Merge branch 'sverk/load-nif-unicode'

OTP-11408

* sverk/load-nif-unicode:
  erts: Fix bug in atom to filename conversions
  Fix open_ddll for win
  erts, crypto: Support NIF library with unicode filename on windows
  erts: Factor out erts_convert_filename_to_wchar()
  erts: Fix compiler warning
  erts: Fix loading of NIF library with unicode in path
  erts: Remove unused constant DRIVER_TAB_SIZE

8 files changed, 169 insertions, 119 deletions

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 673012a9af..e87959f0ab 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1568,9 +1568,10 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
     void* init_func = NULL;
     ErlNifEntry* entry = NULL;
     ErlNifEnv env;
-    int len, i, err;
+    int i, err, encoding;
     Module* mod;
     Eterm mod_atom;
+    const Atom* mod_atomp;
     Eterm f_atom;
     BeamInstr* caller;
     ErtsSysDdllError errdesc = ERTS_SYS_DDLL_ERROR_INIT;
@@ -1578,20 +1579,18 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
     int veto;
     struct erl_module_nif* lib = NULL;
     int reload_warning = 0;
-    char tmp_buf[255];
 
-    len = list_length(BIF_ARG_1);
-    if (len < 0) {
-	BIF_ERROR(BIF_P, BADARG);
+    encoding = erts_get_native_filename_encoding();
+    if (encoding == ERL_FILENAME_WIN_WCHAR) {
+        /* Do not convert the lib name to utf-16le yet, do that in win32 specific code */
+        /* since lib_name is used in error messages */
+        encoding = ERL_FILENAME_UTF8;
     }
-
-    lib_name = (char *) erts_alloc(ERTS_ALC_T_TMP, len + 1);
-
-    if (intlist_to_buf(BIF_ARG_1, lib_name, len) != len) {
-	erts_free(ERTS_ALC_T_TMP, lib_name);
+    lib_name = erts_convert_filename_to_encoding(BIF_ARG_1, NULL, 0,
+                                                 ERTS_ALC_T_TMP, 1, 0, encoding, NULL);
+    if (!lib_name) {
 	BIF_ERROR(BIF_P, BADARG);
     }
-    lib_name[len] = '\0';
 
     if (!erts_try_seize_code_write_permission(BIF_P)) {
 	erts_free(ERTS_ALC_T_TMP, lib_name);
@@ -1615,7 +1614,8 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
     mod=erts_get_module(mod_atom, erts_active_code_ix());
     ASSERT(mod != NULL);
 
-    init_func = erts_static_nif_get_nif_init(erts_basename(lib_name,tmp_buf));
+    mod_atomp = atom_tab(atom_val(mod_atom));
+    init_func = erts_static_nif_get_nif_init((char*)mod_atomp->name, mod_atomp->len);
     if (init_func != NULL)
       handle = init_func;
 
diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c
index a363051062..7e3c6681d9 100644
--- a/erts/emulator/beam/erl_unicode.c
+++ b/erts/emulator/beam/erl_unicode.c
@@ -2026,11 +2026,11 @@ char *erts_convert_filename_to_encoding(Eterm name, char *statbuf, size_t statbu
     } else if (is_binary(name)) {
 	byte *temp_alloc = NULL;
 	byte *bytes;
-	byte *err_pos;
-	Uint size,num_chars;
+	Uint size;
 	
 	size = binary_size(name);
 	bytes = erts_get_aligned_binary_bytes(name, &temp_alloc);
+
 	if (encoding != ERL_FILENAME_WIN_WCHAR) {
 	    /*Add 0 termination only*/
 	    if (used) 
@@ -2042,36 +2042,11 @@ char *erts_convert_filename_to_encoding(Eterm name, char *statbuf, size_t statbu
 	    }
 	    memcpy(name_buf,bytes,size);
 	    name_buf[size]=0;
-	} else if (erts_analyze_utf8(bytes,size,&err_pos,&num_chars,NULL) != ERTS_UTF8_OK || 
-		   erts_get_user_requested_filename_encoding() ==  ERL_FILENAME_LATIN1) {
-	    byte *p;
-	    /* What to do now? Maybe latin1, so just take byte for byte instead */
-	    if (used) 
-		*used = (Sint) (size+1)*2;
-	    if ((size+1)*2 > statbuf_size) {
-		name_buf = (char *) erts_alloc(alloc_type, (size+1)*2);
-	    } else {
-		name_buf = statbuf;
-	    }
-	    p = (byte *) name_buf;
-	    while (size--) {
-		*p++ = *bytes++;
-		*p++ = 0;
-	    }
-	    *p++ = 0;
-	    *p++ = 0;
-	} else { /* WIN_WCHAR and valid UTF8 */
-	    if (used) 
-		*used = (Sint) (num_chars+1)*2;
-	    if ((num_chars+1)*2 > statbuf_size) {
-		name_buf = (char *) erts_alloc(alloc_type, (num_chars+1)*2);
-	    } else {
-		name_buf = statbuf;
-	    }
-	    erts_copy_utf8_to_utf16_little((byte *) name_buf, bytes, num_chars);
-	    name_buf[num_chars*2] = 0;
-	    name_buf[num_chars*2+1] = 0;
-	}
+	} else {
+            name_buf = erts_convert_filename_to_wchar(bytes, size,
+                                                      statbuf, statbuf_size,
+                                                      alloc_type, used, 0);
+        }
 	erts_free_aligned_binary_bytes(temp_alloc);
     } else {
 	return NULL;
@@ -2079,6 +2054,50 @@ char *erts_convert_filename_to_encoding(Eterm name, char *statbuf, size_t statbu
     return name_buf;
 }
 
+char* erts_convert_filename_to_wchar(byte* bytes, Uint size,
+                                     char *statbuf, size_t statbuf_size,
+                                     ErtsAlcType_t alloc_type, Sint* used,
+                                     Uint extra_wchars)
+{
+    byte *err_pos;
+    Uint num_chars;
+    char* name_buf = NULL;
+    Sint need;
+    char *p;
+
+    if (erts_analyze_utf8(bytes,size,&err_pos,&num_chars,NULL) != ERTS_UTF8_OK ||
+        erts_get_user_requested_filename_encoding() ==  ERL_FILENAME_LATIN1) {
+
+        /* What to do now? Maybe latin1, so just take byte for byte instead */
+        need = (Sint) (size + extra_wchars + 1) * 2;
+        if (need > statbuf_size) {
+            name_buf = (char *) erts_alloc(alloc_type, need);
+        } else {
+            name_buf = statbuf;
+        }
+        p = name_buf;
+        while (size--) {
+            *p++ = *bytes++;
+            *p++ = 0;
+        }
+    } else { /* WIN_WCHAR and valid UTF8 */
+        need = (Sint) (num_chars + extra_wchars + 1) * 2;
+        if (need > statbuf_size) {
+            name_buf = (char *) erts_alloc(alloc_type, need);
+        } else {
+            name_buf = statbuf;
+        }
+        erts_copy_utf8_to_utf16_little((byte *) name_buf, bytes, num_chars);
+        p = name_buf + num_chars*2;
+    }
+    *p++ = 0;
+    *p++ = 0;
+    if (used)
+        *used = p - name_buf;
+    return name_buf;
+}
+
+
 static int filename_len_16bit(byte *str) 
 {
     byte *p = str;
@@ -2158,16 +2177,31 @@ Sint erts_native_filename_need(Eterm ioterm, int encoding)
 	ap = atom_tab(atom_val(ioterm));
 	switch (encoding) {
 	case ERL_FILENAME_LATIN1:
-	    need = ap->len;
+	    need = ap->latin1_chars;  /* May be -1 */
 	    break;
 	case ERL_FILENAME_UTF8_MAC:
 	case ERL_FILENAME_UTF8:
-	    for (i = 0; i < ap->len; i++) {
-		need += (ap->name[i] >= 0x80) ? 2 : 1;
-	    }
+	    need = ap->len;
 	    break;
 	case ERL_FILENAME_WIN_WCHAR:
-	    need = 2*(ap->len);
+            if (ap->latin1_chars >= 0) {
+		need = 2* ap->latin1_chars;
+            }
+	    else {
+		for (i = 0; i < ap->len; ) {
+                    if (ap->name[i] < 0x80) {
+			i++;
+                    } else if (ap->name[i] < 0xE0) {
+			i += 2;
+                    } else if (ap->name[i] < 0xF0) {
+			i += 3;
+                    } else {
+			need = -1;
+			break;
+		    }
+		    need += 2;
+		}
+	    }
 	    break;
 	default:
 	    need = -1;
@@ -2297,26 +2331,36 @@ void erts_native_filename_put(Eterm ioterm, int encoding, byte *p)
 	switch (encoding) {
 	case ERL_FILENAME_LATIN1:
 	    for (i = 0; i < ap->len; i++) {
-		*p++ = ap->name[i];
-	    }
-	    break;
-	case ERL_FILENAME_UTF8_MAC:
-	case ERL_FILENAME_UTF8:
-	    for (i = 0; i < ap->len; i++) {
-		if(ap->name[i] < 0x80) {
+		if (ap->name[i] < 0x80) {
 		    *p++ = ap->name[i];
 		} else {
-		    *p++ = (((ap->name[i]) >> 6) | ((byte) 0xC0));
-		    *p++ = (((ap->name[i]) & 0x3F) | ((byte) 0x80));
+		    ASSERT(ap->name[i] < 0xC4);
+		    *p++ = ((ap->name[i] & 3) << 6) | (ap->name[i+1] & 0x3F);
+		    i++;
 		}
 	    }
 	    break;
+	case ERL_FILENAME_UTF8_MAC:
+	case ERL_FILENAME_UTF8:
+	    sys_memcpy(p, ap->name, ap->len);
+	    break;
 	case ERL_FILENAME_WIN_WCHAR:
 	    for (i = 0; i < ap->len; i++) {
 		/* Little endian */
-		*p++ = ap->name[i];
-		*p++ = 0;
-	    }
+                if (ap->name[i] < 0x80) {
+		    *p++ = ap->name[i];
+		    *p++ = 0;
+                } else if (ap->name[i] < 0xE0) {
+		    *p++ = ((ap->name[i] & 3) << 6) | (ap->name[i+1] & 0x3F);
+		    *p++ = ((ap->name[i] & 0x1C) >> 2);
+		    i++;
+                } else {
+		    ASSERT(ap->name[i] < 0xF0);
+		    *p++ = ((ap->name[i+1] & 3) << 6) | (ap->name[i+2] & 0x3C);
+		    *p++ = ((ap->name[i] & 0xF) << 4) | ((ap->name[i+1] & 0x3C) >> 2);
+		    i += 2;
+		}
+            }
 	    break;
 	default:
 	    ASSERT(0);
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 1a842918ff..c1fda3f96c 100755
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -186,11 +186,6 @@ extern void erts_ddll_remove_monitor(Process *p,
 extern Eterm erts_ddll_monitor_driver(Process *p,
 				      Eterm description,
 				      ErtsProcLocks plocks);
-/*
- * Max no. of drivers (linked in and dynamically loaded). Each table
- * entry uses 4 bytes.
- */
-#define DRIVER_TAB_SIZE 32
 
 /*
 ** Just like the driver binary but with initial flags
@@ -854,7 +849,7 @@ void erts_lcnt_enable_io_lock_count(int enable);
 
 /* driver_tab.c */
 typedef void *(*ErtsStaticNifInitFPtr)(void);
-ErtsStaticNifInitFPtr erts_static_nif_get_nif_init(const char *name);
+ErtsStaticNifInitFPtr erts_static_nif_get_nif_init(const char *name, int len);
 int erts_is_static_nif(void *handle);
 void erts_init_static_drivers(void);
 
@@ -863,7 +858,6 @@ void erl_drv_thr_init(void);
 
 /* utils.c */
 void erts_cleanup_offheap(ErlOffHeap *offheap);
-const char *erts_basename(const char* path, char* buff);
 
 Uint64 erts_timestamp_millis(void);
 
@@ -924,6 +918,10 @@ char *erts_convert_filename_to_encoding(Eterm name, char *statbuf,
 					int allow_empty, int allow_atom,
 					int encoding,
 					Sint *used /* out */);
+char* erts_convert_filename_to_wchar(byte* bytes, Uint size,
+                                     char *statbuf, size_t statbuf_size,
+                                     ErtsAlcType_t alloc_type, Sint* used,
+                                     Uint extra_wchars);
 Eterm erts_convert_native_to_filename(Process *p, byte *bytes);
 Eterm erts_utf8_to_list(Process *p, Uint num, byte *bytes, Uint sz, Uint left,
 			Uint *num_built, Uint *num_eaten, Eterm tail);
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index efacee3ea5..605a625282 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -4019,28 +4019,6 @@ erts_smp_ensure_later_interval_acqb(erts_interval_t *icp, Uint64 ic)
 #endif
 }
 
-const char *erts_basename(const char* path, char* buff) {
-  /* This function is not compliant with bash basename. Edge cases like "//"
-     and "/path//" do not work properly.
-   */
-  int i;
-  int len = strlen(path);
-  const char *basename = path;
-  for (i = 0; path[i] != '\0'; i++) {
-    if (path[i] == '/') {
-      if (path[i+1] == '\0') {
-	    memcpy(buff,basename,len - (basename-path));
-	    buff[len - (basename-path)-1] = '\0';
-	    basename = buff;
-	    break;
-      } else { basename = path+i;}
-    }
-  }
-  if (basename == path)
-    return path;
-  return basename+1;
-}
-
 /*
  * A millisecond timestamp without time correction where there's no hrtime
  * - for tracing on "long" things...
diff --git a/erts/emulator/sys/win32/erl_win32_sys_ddll.c b/erts/emulator/sys/win32/erl_win32_sys_ddll.c
index 4c8d83ab16..2d3f073cc2 100644
--- a/erts/emulator/sys/win32/erl_win32_sys_ddll.c
+++ b/erts/emulator/sys/win32/erl_win32_sys_ddll.c
@@ -38,7 +38,7 @@
 #include "erl_nif.h"
 
 #define EXT_LEN          4
-#define FILE_EXT         ".dll"
+#define FILE_EXT_WCHAR   L".dll"
 
 static DWORD tls_index = 0;
 static TWinDynDriverCallbacks wddc;
@@ -57,11 +57,15 @@ void erl_sys_ddll_init(void) {
 
 /* 
  * Open a shared object
+ * Expecting 'full_name' as an UTF-8 string.
  */
 int erts_sys_ddll_open2(const char *full_name, void **handle, ErtsSysDdllError* err)
 {
+    HINSTANCE hinstance;
     int len;
-    char dlname[MAXPATHLEN + 1];
+    wchar_t* wcp;
+    Sint used;
+    int code;
     
     if ((len = sys_strlen(full_name)) >= MAXPATHLEN - EXT_LEN) {
 	if (err != NULL) {
@@ -69,10 +73,26 @@ int erts_sys_ddll_open2(const char *full_name, void **handle, ErtsSysDdllError*
 	}
 	return ERL_DE_LOAD_ERROR_NAME_TO_LONG;
     }
-    sys_strcpy(dlname, full_name);
-    sys_strcpy(dlname+len, FILE_EXT);
-    return erts_sys_ddll_open_noext(dlname, handle, err);
+
+    wcp = (wchar_t*)erts_convert_filename_to_wchar((byte*)full_name, len,
+						   NULL, 0,
+						   ERTS_ALC_T_TMP, &used, EXT_LEN);
+    wcscpy(&wcp[used/2 - 1], FILE_EXT_WCHAR);
+
+    if ((hinstance = LoadLibraryW(wcp)) == NULL) {
+	code = ERL_DE_DYNAMIC_ERROR_OFFSET - GetLastError();
+	if (err != NULL) {
+	    err->str = erts_sys_ddll_error(code);
+	}
+    }
+    else {
+        code = ERL_DE_NO_ERROR;
+	*handle = (void *) hinstance;
+    }
+    erts_free(ERTS_ALC_T_TMP, wcp);
+    return code;
 }
+
 int erts_sys_ddll_open_noext(char *dlname, void **handle, ErtsSysDdllError* err)
 {
     HINSTANCE hinstance;
diff --git a/erts/emulator/utils/make_driver_tab b/erts/emulator/utils/make_driver_tab
index 3eedef21a7..5c68143d58 100755
--- a/erts/emulator/utils/make_driver_tab
+++ b/erts/emulator/utils/make_driver_tab
@@ -2,7 +2,7 @@
 #
 # %CopyrightBegin%
 # 
-# Copyright Ericsson AB 1999-2009. All Rights Reserved.
+# Copyright Ericsson AB 1999-2013. All Rights Reserved.
 # 
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
@@ -88,7 +88,7 @@ foreach (@static_drivers) {
 }
 
 # The array itself
-print "\nErlDrvEntry *driver_tab[DRIVER_TAB_SIZE] =\n{\n";
+print "\nErlDrvEntry *driver_tab[] =\n{\n";
 
 foreach (@emu_drivers) {
     print "    &${_}driver_entry,\n";
@@ -126,7 +126,7 @@ foreach (@nifs) {
 }
 
 # The array itself
-print "static ErtsStaticNifEntry static_nif_tab[DRIVER_TAB_SIZE] =\n{\n";
+print "static ErtsStaticNifEntry static_nif_tab[] =\n{\n";
 
 foreach (@nifs) {
     my $d = ${_};
@@ -137,19 +137,19 @@ foreach (@nifs) {
 print "    {NULL,NULL}\n};\n";
 
 print <<EOF;
-ErtsStaticNifInitFPtr erts_static_nif_get_nif_init(const char *name) {
-    int i;
-    for (i = 0; static_nif_tab[i].nif_name != NULL; i++)
-      if (strcmp(name,static_nif_tab[i].nif_name) == 0)
-        return static_nif_tab[i].nif_init;
+ErtsStaticNifInitFPtr erts_static_nif_get_nif_init(const char *name, int len) {
+    ErtsStaticNifEntry* p;
+    for (p = static_nif_tab; p->nif_name != NULL; p++)
+        if (strncmp(p->nif_name, name, len) == 0 && p->nif_name[len] == 0)
+            return p->nif_init;
     return NULL;
 }
 
 int erts_is_static_nif(void *handle) {
-    int i;
-    for (i = 0; static_nif_tab[i].nif_name != NULL; i++)
-	if (((void*)static_nif_tab[i].nif_init) == handle)
-	return 1;
+    ErtsStaticNifEntry* p;
+    for (p = static_nif_tab; p->nif_name != NULL; p++)
+        if (((void*)p->nif_init) == handle)
+            return 1;
     return 0;
 }
 
diff --git a/lib/crypto/c_src/crypto.c b/lib/crypto/c_src/crypto.c
index c28ff8136c..30c8c2554d 100644
--- a/lib/crypto/c_src/crypto.c
+++ b/lib/crypto/c_src/crypto.c
@@ -538,16 +538,17 @@ static ERL_NIF_TERM atom_onbasis;
 #define PRINTF_ERR1(FMT,A1)
 
 #ifdef HAVE_DYNAMIC_CRYPTO_LIB
-static int change_basename(char* buf, int bufsz, const char* newfile)
+static int change_basename(ErlNifBinary* bin, char* buf, int bufsz, const char* newfile)
 {
-    char* p = strrchr(buf, '/');
-    p = (p == NULL) ? buf : p + 1;
+    const unsigned char* p = (unsigned char*)strrchr((char*)bin->data, '/');
+    int i = (p == NULL) ? 0 : (p+1) - bin->data;
     
-    if ((p - buf) + strlen(newfile) >= bufsz) {
+    if (i + strlen(newfile) >= bufsz) {
 	PRINTF_ERR0("CRYPTO: lib name too long");
 	return 0;
     }
-    strcpy(p, newfile);
+    memcpy(buf, bin->data, i);
+    strcpy(buf+i, newfile);
     return 1;
 }
 
@@ -566,14 +567,15 @@ static int init(ErlNifEnv* env, ERL_NIF_TERM load_info)
     int tpl_arity;
     const ERL_NIF_TERM* tpl_array;
     int vernum;
+    ErlNifBinary lib_bin;
     char lib_buf[1000];
 
-    /* load_info: {201, "/full/path/of/this/library"} */
+    /* load_info: {301, <<"/full/path/of/this/library">>} */
     if (!enif_get_tuple(env, load_info, &tpl_arity, &tpl_array)
 	|| tpl_arity != 2
 	|| !enif_get_int(env, tpl_array[0], &vernum)
-	|| vernum != 201
-	|| enif_get_string(env, tpl_array[1], lib_buf, sizeof(lib_buf), ERL_NIF_LATIN1) <= 0) {
+	|| vernum != 301
+	|| !enif_inspect_binary(env, tpl_array[1], &lib_bin)) {
 
 	PRINTF_ERR1("CRYPTO: Invalid load_info '%T'", load_info);
 	return 0;
@@ -633,7 +635,7 @@ static int init(ErlNifEnv* env, ERL_NIF_TERM load_info)
 #ifdef HAVE_DYNAMIC_CRYPTO_LIB
     {
 	void* handle;
-	if (!change_basename(lib_buf, sizeof(lib_buf), "crypto_callback")) {
+	if (!change_basename(&lib_bin, lib_buf, sizeof(lib_buf), "crypto_callback")) {
 	    return 0;
 	}
 	if (!(handle = enif_dlopen(lib_buf, &error_handler, NULL))) {
diff --git a/lib/crypto/src/crypto.erl b/lib/crypto/src/crypto.erl
index 8e8370f3b0..784fcc0119 100644
--- a/lib/crypto/src/crypto.erl
+++ b/lib/crypto/src/crypto.erl
@@ -183,7 +183,7 @@
 %%-type ec_key() :: {Curve :: ec_curve(), PrivKey :: binary() | undefined, PubKey :: ec_point() | undefined}.
 
 -on_load(on_load/0).
--define(CRYPTO_NIF_VSN,201).
+-define(CRYPTO_NIF_VSN,301).
 
 -define(nif_stub,nif_stub_error(?LINE)).
 nif_stub_error(Line) ->
@@ -640,7 +640,7 @@ on_load() ->
 		      end
 	      end,
     Lib = filename:join([PrivDir, "lib", LibName]),
-    Status = case erlang:load_nif(Lib, {?CRYPTO_NIF_VSN,Lib}) of
+    Status = case erlang:load_nif(Lib, {?CRYPTO_NIF_VSN,path2bin(Lib)}) of
 		 ok -> ok;
 		 {error, {load_failed, _}}=Error1 ->
 		     ArchLibDir =
@@ -652,7 +652,7 @@ on_load() ->
 			 [] -> Error1;
 			 _ ->
 			     ArchLib = filename:join([ArchLibDir, LibName]),
-			     erlang:load_nif(ArchLib, {?CRYPTO_NIF_VSN,ArchLib})
+			     erlang:load_nif(ArchLib, {?CRYPTO_NIF_VSN,path2bin(ArchLib)})
 		     end;
 		 Error1 -> Error1
 	     end,
@@ -663,6 +663,14 @@ on_load() ->
 				   "OpenSSL might not be installed on this system.~n",[E,Str]),
 	    Status
     end.
+
+path2bin(Path) when is_list(Path) ->
+    Encoding = file:native_name_encoding(),
+    case unicode:characters_to_binary(Path,Encoding,Encoding) of
+	Bin when is_binary(Bin) ->
+	    Bin
+    end.
+
 %%--------------------------------------------------------------------
 %%% Internal functions (some internal API functions are part of the deprecated API)
 %%--------------------------------------------------------------------