Merge branch 'siri/unicode-merge' into maint

* siri/unicode-merge: (28 commits)
  stdlib: use 'unicode' option for regexp in ets
  ct: use 'unicode' option for regexps
  syntax_tools: add 'unicode' option to re:run for matching paths
  kernel: update simple error logger to print Unicode strings
  stdlib: add Unicode translation modifier in error_logger_tty_h
  wx: add Unicode translation modifier to wx_object
  stdlib: add Unicode translation modifier in debug format funs
  sasl: set encoding of SASL report log
  stdlib: use Unicode translation modifier in error_logger_file_h
  observer: Improve handling of Unicode
  mnesia: Improve handling of Unicode
  stdlib : Improve handling of Unicode
  kernel: Improve handling of Unicode
  hipe: Improve handling of Unicode
  edoc: Improve handling of Unicode
  syntax_tools: Improve handling of Unicode
  dialyzer: Improve handling of Unicode
  et: Improve handling of Unicode
  xref: Improve handling of Unicode
  fprof: Modify handling of Unicode
  eprof: Improve handling of Unicode
  eunit: Improve handling of Unicode
  stdlib: Improve handling of Unicode in escript
  stdlib: Improve handling of Unicode in edlin_expand
  stdlib: Modify handling of Unicode in proc_lib
  sasl: Improve handling of Unicode
  edoc: Improve handling of Unicode
  debugger: Improve handling of Unicode

OTP-14462
OTP-14464

1 files changed, 50 insertions, 19 deletions

diff --git a/lib/observer/src/ttb.erl b/lib/observer/src/ttb.erl
index 09b0bc6710..940fdc9818 100644
--- a/lib/observer/src/ttb.erl
+++ b/lib/observer/src/ttb.erl
@@ -635,7 +635,7 @@ stop(Opts) when is_list(Opts) ->
             ok;
         {_, {stopped, _}} ->
             %% Printout moved out of the ttb loop to avoid occasional deadlock
-            io:format("Stored logs in ~s~n", [element(2, Result)]);
+            io:format("Stored logs in ~ts~n", [element(2, Result)]);
         {_, _} ->
             ok
     end,
@@ -792,7 +792,7 @@ do_stop({FetchOrFormat, UserDir}, Sender, NodeInfo, SessionInfo) ->
     write_config(?last_config, all),
     Localhost = host(node()),
     Dir = get_fetch_dir(UserDir, proplists:get_value(logfile, SessionInfo)),
-    file:make_dir(Dir),
+    ok = filelib:ensure_dir(filename:join(Dir,"*")),
     %% The nodes are traversed twice here because
     %% the meta tracing in observer_backend must be
     %% stopped before dbg is stopped, and dbg must
@@ -900,21 +900,29 @@ fetch_report(Localhost, Dir, Node, MetaFile) ->
 
 fetch(Localhost,Dir,Node,MetaFile) ->
     case (host(Node) == Localhost) orelse is_local(MetaFile) of
-    true -> % same host, just move the files
+        true -> % same host, just move the files
 	    Files = get_filenames(Node,MetaFile),
 	    lists:foreach(
-            fun(File0) ->
-                Dest = filename:join(Dir,filename:basename(File0)),
-                file:rename(File0, Dest)
-            end,
-        Files);
+              fun(File0) ->
+                      Dest = filename:join(Dir,filename:basename(File0)),
+                      file:rename(File0, Dest)
+              end,
+              Files);
 	false ->
 	    {ok, LSock} = gen_tcp:listen(0, [binary,{packet,2},{active,false}]),
 	    {ok,Port} = inet:port(LSock),
-	    rpc:cast(Node,observer_backend,ttb_fetch,
-		     [MetaFile,{Port,Localhost}]),
+            Enc = file:native_name_encoding(),
+            Args =
+                case rpc:call(Node,erlang,function_exported,
+                              [observer_backend,ttb_fetch,3]) of
+                    true ->
+                        [MetaFile,{Port,Localhost},Enc];
+                    false ->
+                        [MetaFile,{Port,Localhost}]
+                end,
+            rpc:cast(Node,observer_backend,ttb_fetch,Args),
 	    {ok, Sock} = gen_tcp:accept(LSock),
-	    receive_files(Dir,Sock,undefined),
+	    receive_files(Dir,Sock,undefined,Enc),
 	    ok = gen_tcp:close(LSock),
 	    ok = gen_tcp:close(Sock)
     end.
@@ -929,25 +937,48 @@ get_filenames(_N, {local,F,_}) ->
 get_filenames(N, F) ->
     rpc:call(N, observer_backend,ttb_get_filenames,[F]).
 
-receive_files(Dir,Sock,Fd) ->
+receive_files(Dir,Sock,Fd,Enc) ->
     case gen_tcp:recv(Sock, 0) of
 	{ok, <<0,Bin/binary>>} ->
 	    file:write(Fd,Bin),
-	    receive_files(Dir,Sock,Fd);
-	{ok, <<1,Bin/binary>>} ->
-	    File0 = binary_to_list(Bin),
+	    receive_files(Dir,Sock,Fd,Enc);
+	{ok, <<Code,Bin/binary>>} when Code==1; Code==2; Code==3 ->
+            File0 = decode_filename(Code,Bin,Enc),
 	    File = filename:join(Dir,File0),
 	    {ok,Fd1} = file:open(File,[raw,write]),
-	    receive_files(Dir,Sock,Fd1);
+	    receive_files(Dir,Sock,Fd1,Enc);
 	{error, closed} ->
 	    ok = file:close(Fd)
     end.    
 
+decode_filename(1,Bin,_Enc) ->
+    %% Old version of observer_backend - filename encoded with
+    %% list_to_binary
+    binary_to_list(Bin);
+decode_filename(2,Bin,Enc) ->
+    %% Successfully encoded filename with correct encoding
+    unicode:characters_to_list(Bin,Enc);
+decode_filename(3,Bin,latin1) ->
+    %% Filename encoded with faulty encoding. This has to be utf8
+    %% remote and latin1 here, and the filename actually containing
+    %% characters outside the latin1 range. So making an escaped
+    %% variant of the filename and warning about it.
+    File0 = unicode:characters_to_list(Bin,utf8),
+    File = [ case X of
+                     High when High > 255 ->
+                         ["\\\\x{",erlang:integer_to_list(X, 16),$}];
+                     Low ->
+                         Low
+                 end || X <- File0 ],
+    io:format("Warning: fetching file with faulty filename encoding ~ts~n"
+              "Will be written as ~ts~n",
+              [File0,File]),
+    File.
+
 host(Node) ->
     [_name,Host] = string:tokens(atom_to_list(Node),"@"),
     Host.
 
-
 wait_for_fetch([]) ->
     ok;
 wait_for_fetch(Nodes) ->
@@ -1087,7 +1118,7 @@ read_traci(File) ->
 	{ok,B} -> 
 	    interpret_binary(B,dict:new(),[]);
 	_ -> 
-	    io:format("Warning: no meta data file: ~s~n",[MetaFile]),
+	    io:format("Warning: no meta data file: ~ts~n",[MetaFile]),
 	    {dict:new(),[]}
     end.
 
@@ -1303,7 +1334,7 @@ get_term(B) ->
     end.
 
 display_warning(Item,Warning) ->
-    io:format("Warning: {~w,~w}~n",[Warning,Item]).
+    io:format("Warning: {~tw,~tw}~n",[Warning,Item]).
 
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%