Merge branch 'pan/unicode_filename_warnings'

* pan/unicode_filename_warnings:
  Add file:list_dir_all/1 and file:read_link_all/1
  prim_file: Add list_dir_all() and read_link_all()
  Teach prim_file:set_cwd() to avoid entering non-translatable directories
  Make prim_file skip invalid filenames in unicode mode
  prim_file: Refactor functions that return filenames
  prim_file: Refactor handling of responses
  prim_file: Always open non-file ports in binary mode
  Test that list_dir("non-existing-dir") fails with the correct error

6 files changed, 132 insertions, 9 deletions

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index f138324e1f..ce60bb9bbc 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -269,6 +269,7 @@ atom hipe_architecture
 atom http httph https http_response http_request http_header http_eoh http_error http_bin httph_bin
 atom id
 atom if_clause
+atom ignore
 atom imports
 atom in
 atom in_exiting
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 4aaf466008..e313188901 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -522,6 +522,7 @@ bif erlang:nif_error/2
 bif prim_file:internal_name2native/1
 bif prim_file:internal_native2name/1
 bif prim_file:internal_normalize_utf8/1
+bif prim_file:is_translatable/1
 bif file:native_name_encoding/0
 
 #
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 223c9c4d7e..ec3e0d54cb 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -981,19 +981,64 @@ erl_start(int argc, char **argv)
 	    break;
 	case 'f':
 	    if (!strncmp(argv[i],"-fn",3)) {
+		int warning_type =  ERL_FILENAME_WARNING_WARNING;
 		arg = get_arg(argv[i]+3, argv[i+1], &i);
 		switch (*arg) {
 		case 'u':
-		    erts_set_user_requested_filename_encoding(ERL_FILENAME_UTF8);
+		    switch (*(argv[i]+4)) {
+		    case 'w':
+		    case 0:
+			break;
+		    case 'i':
+			warning_type =  ERL_FILENAME_WARNING_IGNORE;
+			break;
+		    case 'e':
+			warning_type =  ERL_FILENAME_WARNING_ERROR;
+			break;
+		    default:
+			erts_fprintf(stderr, "bad type of warnings for "
+				     "wrongly coded filename: %s\n", argv[i]+4);
+			erts_usage();
+		    }
+		    erts_set_user_requested_filename_encoding
+			(
+			 ERL_FILENAME_UTF8,
+			 warning_type
+			 );
 		    break;
 		case 'l':
-		    erts_set_user_requested_filename_encoding(ERL_FILENAME_LATIN1);
+		    erts_set_user_requested_filename_encoding
+			(
+			 ERL_FILENAME_LATIN1,
+			 warning_type
+			 );
 		    break;
 		case 'a':
-		    erts_set_user_requested_filename_encoding(ERL_FILENAME_UNKNOWN);
+		    switch (*(argv[i]+4)) {
+		    case 'w':
+		    case 0:
+			break;
+		    case 'i':
+			warning_type =  ERL_FILENAME_WARNING_IGNORE;
+			break;
+		    case 'e':
+			warning_type =  ERL_FILENAME_WARNING_ERROR;
+			break;
+		    default:
+			erts_fprintf(stderr, "bad type of warnings for "
+				     "wrongly coded filename: %s\n", argv[i]+4);
+			erts_usage();
+		    }
+		    erts_set_user_requested_filename_encoding
+			(
+			 ERL_FILENAME_UNKNOWN,
+			 warning_type
+			 );
 		    break;
 		default:
-		    erts_fprintf(stderr, "bad filename encoding %s, can be (l,u or a)\n", arg);
+		    erts_fprintf(stderr, "bad filename encoding %s, can be "
+				 "(l,u or a, optionally followed by w, "
+				 "i or e)\n", arg);
 		    erts_usage();
 		}
 		break;
diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c
index 99108af937..80982f3760 100644
--- a/erts/emulator/beam/erl_unicode.c
+++ b/erts/emulator/beam/erl_unicode.c
@@ -2573,8 +2573,20 @@ BIF_RETTYPE prim_file_internal_native2name_1(BIF_ALIST_1)
     case ERL_FILENAME_UTF8:
 	bytes = erts_get_aligned_binary_bytes(BIF_ARG_1, &temp_alloc);
 	if (erts_analyze_utf8(bytes,size,&err_pos,&num_chars,NULL) != ERTS_UTF8_OK) {
+	    Eterm *hp = HAlloc(BIF_P,3);
+	    Eterm warn_type = NIL;
 	    erts_free_aligned_binary_bytes(temp_alloc);
-	    goto noconvert;
+	    switch (erts_get_filename_warning_type()) {
+	    case ERL_FILENAME_WARNING_IGNORE:
+		warn_type = am_ignore;
+		break;
+	    case ERL_FILENAME_WARNING_ERROR:
+		warn_type = am_error;
+		break;
+	    default:
+		warn_type = am_warning;
+	    }
+	    BIF_RET(TUPLE2(hp,am_error,warn_type));
 	}
 	num_built = 0;
 	num_eaten = 0;
@@ -2607,9 +2619,8 @@ BIF_RETTYPE prim_file_internal_native2name_1(BIF_ALIST_1)
 	erts_free_aligned_binary_bytes(temp_alloc);
 	BIF_RET(ret);
     default:
-	goto noconvert;
+	break;
     }
- noconvert:
     BIF_RET(BIF_ARG_1);
 }
 
@@ -2646,6 +2657,52 @@ BIF_RETTYPE prim_file_internal_normalize_utf8_1(BIF_ALIST_1)
     BIF_RET(ret);
 }  
 
+BIF_RETTYPE prim_file_is_translatable_1(BIF_ALIST_1)
+{
+    ERTS_DECLARE_DUMMY(Eterm real_bin);
+    ERTS_DECLARE_DUMMY(Uint offset);
+    Uint size;
+    Uint num_chars;
+    Uint bitsize;
+    ERTS_DECLARE_DUMMY(Uint bitoffs);
+    byte *temp_alloc = NULL;
+    byte *bytes;
+    byte *err_pos;
+    int status;
+
+    if (is_not_binary(BIF_ARG_1)) {
+	BIF_ERROR(BIF_P,BADARG);
+    }
+    size = binary_size(BIF_ARG_1);
+    ERTS_GET_REAL_BIN(BIF_ARG_1, real_bin, offset, bitoffs, bitsize);
+    if (bitsize != 0) {
+	BIF_ERROR(BIF_P,BADARG);
+    }
+    if (size == 0) {
+	BIF_RET(am_true);
+    }
+
+    /*
+     * If the encoding is latin1, the pathname is always translatable.
+     */
+    switch (erts_get_native_filename_encoding()) {
+    case ERL_FILENAME_LATIN1:
+	BIF_RET(am_true);
+    case ERL_FILENAME_WIN_WCHAR:
+	if (erts_get_user_requested_filename_encoding() == ERL_FILENAME_LATIN1) {
+	    BIF_RET(am_true);
+	}
+    }
+
+    /*
+     * Check whether the binary contains legal UTF-8 sequences.
+     */
+    bytes = erts_get_aligned_binary_bytes(BIF_ARG_1, &temp_alloc);
+    status = erts_analyze_utf8(bytes, size, &err_pos, &num_chars, NULL);
+    erts_free_aligned_binary_bytes(temp_alloc);
+    BIF_RET(status == ERTS_UTF8_OK ? am_true : am_false);
+}  
+
 BIF_RETTYPE file_native_name_encoding_0(BIF_ALIST_0)
 {
     switch (erts_get_native_filename_encoding()) {
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 249a9c05c2..9416a91480 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -1029,10 +1029,22 @@ char* win32_errorstr(int);
 #define ERL_FILENAME_UTF8_MAC  (3)
 #define ERL_FILENAME_WIN_WCHAR (4)
 
+/************************************************************************
+ * If a filename in for example list_dir is not in the right encoding, it
+ * will be skipped in the resulting list, but depending on a startup setting
+ * we will inform the user in different ways. These macros define the
+ * different reactions to wrongly coded filenames. In the error case an
+ * exception will be thrown by prim_file.
+ ************************************************************************/
+#define ERL_FILENAME_WARNING_WARNING (0)
+#define ERL_FILENAME_WARNING_IGNORE (1)
+#define ERL_FILENAME_WARNING_ERROR (2)
+
 int erts_get_native_filename_encoding(void);
 /* The set function is only to be used by erl_init! */
-void erts_set_user_requested_filename_encoding(int encoding); 
+void erts_set_user_requested_filename_encoding(int encoding, int warning);
 int erts_get_user_requested_filename_encoding(void);
+int erts_get_filename_warning_type(void);
 
 void erts_init_sys_common_misc(void);
 
diff --git a/erts/emulator/sys/common/erl_sys_common_misc.c b/erts/emulator/sys/common/erl_sys_common_misc.c
index 1bf5fa89f4..a0cd4cd10a 100644
--- a/erts/emulator/sys/common/erl_sys_common_misc.c
+++ b/erts/emulator/sys/common/erl_sys_common_misc.c
@@ -47,14 +47,16 @@
 /* Written once and only once */
 
 static int filename_encoding = ERL_FILENAME_UNKNOWN;
+static int filename_warning = ERL_FILENAME_WARNING_WARNING;
 #if defined(__WIN32__) || defined(__DARWIN__)
 static int user_filename_encoding = ERL_FILENAME_UTF8; /* Default unicode on windows */
 #else
 static int user_filename_encoding = ERL_FILENAME_LATIN1;
 #endif
-void erts_set_user_requested_filename_encoding(int encoding)
+void erts_set_user_requested_filename_encoding(int encoding, int warning)
 {
     user_filename_encoding = encoding;
+    filename_warning = warning;
 }
 
 int erts_get_user_requested_filename_encoding(void)
@@ -62,6 +64,11 @@ int erts_get_user_requested_filename_encoding(void)
     return user_filename_encoding;
 }
 
+int erts_get_filename_warning_type(void)
+{
+    return filename_warning;
+}
+
 void erts_init_sys_common_misc(void)
 {
 #if defined(__WIN32__)