Merge branch 'siri/unicode-merge' into maint

* siri/unicode-merge: (28 commits)
  stdlib: use 'unicode' option for regexp in ets
  ct: use 'unicode' option for regexps
  syntax_tools: add 'unicode' option to re:run for matching paths
  kernel: update simple error logger to print Unicode strings
  stdlib: add Unicode translation modifier in error_logger_tty_h
  wx: add Unicode translation modifier to wx_object
  stdlib: add Unicode translation modifier in debug format funs
  sasl: set encoding of SASL report log
  stdlib: use Unicode translation modifier in error_logger_file_h
  observer: Improve handling of Unicode
  mnesia: Improve handling of Unicode
  stdlib : Improve handling of Unicode
  kernel: Improve handling of Unicode
  hipe: Improve handling of Unicode
  edoc: Improve handling of Unicode
  syntax_tools: Improve handling of Unicode
  dialyzer: Improve handling of Unicode
  et: Improve handling of Unicode
  xref: Improve handling of Unicode
  fprof: Modify handling of Unicode
  eprof: Improve handling of Unicode
  eunit: Improve handling of Unicode
  stdlib: Improve handling of Unicode in escript
  stdlib: Improve handling of Unicode in edlin_expand
  stdlib: Modify handling of Unicode in proc_lib
  sasl: Improve handling of Unicode
  edoc: Improve handling of Unicode
  debugger: Improve handling of Unicode

OTP-14462
OTP-14464

6 files changed, 81 insertions, 44 deletions

diff --git a/lib/tools/src/eprof.erl b/lib/tools/src/eprof.erl
index 3ae899a078..535ddbcd04 100644
--- a/lib/tools/src/eprof.erl
+++ b/lib/tools/src/eprof.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1996-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1996-2017. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -246,7 +246,7 @@ handle_call(profile_stop, _From, #state{ profiling = true } = S) ->
 
 %% logfile
 handle_call({logfile, File}, _From, #state{ fd = OldFd } = S) ->
-    case file:open(File, [write]) of
+    case file:open(File, [write, {encoding, utf8}]) of
 	{ok, Fd} ->
 	    case OldFd of
 		undefined -> ok;
@@ -478,11 +478,11 @@ string_bp_mfa([{Mfa, {Count, Time}}|Mfas], Tus, {MfaW, CountW, PercW, TimeW, TpC
 	Stpc   = s("~.2f", [divide(Time,Count)]),
 
 	string_bp_mfa(Mfas, Tus, {
-		erlang:max(MfaW,  length(Smfa)),
-		erlang:max(CountW,length(Scount)),
-		erlang:max(PercW, length(Sperc)),
-		erlang:max(TimeW, length(Stime)),
-		erlang:max(TpCW,  length(Stpc))
+		erlang:max(MfaW,  string:length(Smfa)),
+		erlang:max(CountW,string:length(Scount)),
+		erlang:max(PercW, string:length(Sperc)),
+		erlang:max(TimeW, string:length(Stime)),
+		erlang:max(TpCW,  string:length(Stpc))
 	    }, [[Smfa, Scount, Sperc, Stime, Stpc] | Strings]).
 
 print_bp_mfa(Mfas, {Tn, Tus}, Fd, Opts) ->
@@ -491,11 +491,11 @@ print_bp_mfa(Mfas, {Tn, Tus}, Fd, Opts) ->
     TnStr    = s(Tn),
     TusStr   = s(Tus),
     TuspcStr = s("~.2f", [divide(Tus,Tn)]),
-    Ws = {erlang:max(length("FUNCTION"), MfaW),
-          lists:max([length("CALLS"), CountW, length(TnStr)]),
-          erlang:max(length("      %"), PercW),
-          lists:max([length("TIME"), TimeW, length(TusStr)]),
-          lists:max([length("uS / CALLS"), TpCW, length(TuspcStr)])},
+    Ws = {erlang:max(string:length("FUNCTION"), MfaW),
+          lists:max([string:length("CALLS"), CountW, string:length(TnStr)]),
+          erlang:max(string:length("      %"), PercW),
+          lists:max([string:length("TIME"), TimeW, string:length(TusStr)]),
+          lists:max([string:length("uS / CALLS"), TpCW, string:length(TuspcStr)])},
     format(Fd, Ws, ["FUNCTION", "CALLS", "      %", "TIME", "uS / CALLS"]),
     format(Fd, Ws, ["--------", "-----", "-------", "----", "----------"]),
     lists:foreach(fun (String) -> format(Fd, Ws, String) end, Strs),
@@ -503,13 +503,13 @@ print_bp_mfa(Mfas, {Tn, Tus}, Fd, Opts) ->
     format(Fd, Ws, ["Total:", TnStr, "100.00%", TusStr, TuspcStr]),
     ok.
 
-s({M,F,A}) -> s("~w:~w/~w",[M,F,A]);
-s(Term) -> s("~p", [Term]).
+s({M,F,A}) -> s("~w:~tw/~w",[M,F,A]);
+s(Term) -> s("~tp", [Term]).
 s(Format, Terms) -> lists:flatten(io_lib:format(Format, Terms)).
 
 
 format(Fd, {MfaW, CountW, PercW, TimeW, TpCW}, Strings) ->
-    format(Fd, s("~~.~ps  ~~~ps  ~~~ps  ~~~ps  [~~~ps]~~n", [MfaW, CountW, PercW, TimeW, TpCW]), Strings);
+    format(Fd, s("~~.~wts  ~~~ws  ~~~ws  ~~~ws  [~~~ws]~~n", [MfaW, CountW, PercW, TimeW, TpCW]), Strings);
 format(undefined, Format, Strings) ->
     io:format(Format, Strings),
     ok;
diff --git a/lib/tools/src/fprof.erl b/lib/tools/src/fprof.erl
index 436f68d12b..2fe42beb03 100644
--- a/lib/tools/src/fprof.erl
+++ b/lib/tools/src/fprof.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2001-2016. All Rights Reserved.
+%% Copyright Ericsson AB 2001-2017. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -1475,7 +1475,7 @@ info_suspect_call(GroupLeader, GroupLeader, _, _) ->
     ok;
 info_suspect_call(GroupLeader, _, Func, Pid) ->
     io:format(GroupLeader,
-	      "~nWarning: ~p called in ~p - trace may become corrupt!~n",
+	      "~nWarning: ~tp called in ~p - trace may become corrupt!~n",
 	      parsify([Func, Pid])).
 
 info(GroupLeader, GroupLeader, _, _) ->
@@ -1498,13 +1498,13 @@ dump_stack(Dump, Stack, Term) ->
 			{N, length(hd(Stack))}
 		end
 	end,
-     io:format(Dump, "~s~p.~n", [lists:duplicate(Depth, "  "), parsify(Term)]),
+     io:format(Dump, "~s~tp.~n", [lists:duplicate(Depth, "  "), parsify(Term)]),
     true.
 
 dump(undefined, _) ->
     false;
 dump(Dump, Term) ->
-    io:format(Dump, "~p.~n", [parsify(Term)]),
+    io:format(Dump, "~tp.~n", [parsify(Term)]),
     true.
 
 
@@ -2603,17 +2603,17 @@ println({Io, [W1, W2, W3, W4]}, Head,
 println({Io, _}, Head,
 	[],
 	Tail, Comment) ->
-    io:format(Io, "~s~s~s~n",
+    io:format(Io, "~s~ts~ts~n",
 	      [pad(Head, $ , 3), Tail, Comment]);
 println({Io, _}, Head,
 	{Tag, Term},
 	Tail, Comment) ->
-    io:format(Io, "~s~p, ~p~s~s~n",
+    io:format(Io, "~s~tp, ~tp~ts~ts~n",
 	      [pad(Head, $ , 3), parsify(Tag), parsify(Term), Tail, Comment]);
 println({Io, _}, Head,
 	Term,
 	Tail, Comment) ->
-    io:format(Io, "~s~p~s~s~n",
+    io:format(Io, "~s~tp~ts~ts~n",
 	      [pad(Head, $ , 3), parsify(Term), Tail, Comment]).
 
 
@@ -2720,7 +2720,7 @@ postsort_r([[_|C] | L], R) ->
 flat_format(F, Trailer) when is_float(F) ->
     lists:flatten([io_lib:format("~.3f", [F]), Trailer]);
 flat_format(W, Trailer) ->
-    lists:flatten([io_lib:format("~p", [W]), Trailer]).
+    lists:flatten([io_lib:format("~tp", [W]), Trailer]).
 
 %% Format, flatten, and pad.
 flat_format(Term, Trailer, Width) ->
diff --git a/lib/tools/src/xref_base.erl b/lib/tools/src/xref_base.erl
index 8d2cc07e40..3199b28acb 100644
--- a/lib/tools/src/xref_base.erl
+++ b/lib/tools/src/xref_base.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2000-2016. All Rights Reserved.
+%% Copyright Ericsson AB 2000-2017. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -405,21 +405,21 @@ analysis(exports_not_used, _) ->
     %% Local calls are not considered here. "X * UU" would do otherwise.
     "X - XU";
 analysis({call, F}, functions) ->
-    make_query("range (E | ~w : Fun)", [F]);
+    make_query("range (E | ~tw : Fun)", [F]);
 analysis({use, F}, functions) ->
-    make_query("domain (E || ~w : Fun)", [F]);
+    make_query("domain (E || ~tw : Fun)", [F]);
 analysis({module_call, M}, _) ->
-    make_query("range (ME | ~w : Mod)", [M]);
+    make_query("range (ME | ~tw : Mod)", [M]);
 analysis({module_use, M}, _) ->
-    make_query("domain (ME || ~w : Mod)", [M]);
+    make_query("domain (ME || ~tw : Mod)", [M]);
 analysis({application_call, A}, _) ->
-    make_query("range (AE | ~w : App)", [A]);
+    make_query("range (AE | ~tw : App)", [A]);
 analysis({application_use, A}, _) ->
-    make_query("domain (AE || ~w : App)", [A]);
+    make_query("domain (AE || ~tw : App)", [A]);
 analysis({release_call, R}, _) ->
-    make_query("range (RE | ~w : Rel)", [R]);
+    make_query("range (RE | ~tw : Rel)", [R]);
 analysis({release_use, R}, _) ->
-    make_query("domain (RE || ~w : Rel)", [R]);
+    make_query("domain (RE || ~tw : Rel)", [R]);
 analysis(deprecated_function_calls, functions) ->
     "XC || DF";
 analysis({deprecated_function_calls,Flag}, functions) ->
@@ -1833,9 +1833,9 @@ message(true, What, Arg) ->
 	unreadable ->
 	    io:format("Skipping ~ts (unreadable)~n", [Arg]);
 	xref_attr ->
-	    io:format("~ts: Skipping 'xref' attribute ~w~n", Arg);
+	    io:format("~ts: Skipping 'xref' attribute ~tw~n", Arg);
         depr_attr ->
-            io:format("~ts: Skipping 'deprecated' attribute ~w~n", Arg);
+            io:format("~ts: Skipping 'deprecated' attribute ~tw~n", Arg);
 	lib_search ->
 	    io:format("Scanning library path for BEAM files... ", []);
 	lib_check ->
diff --git a/lib/tools/src/xref_parser.yrl b/lib/tools/src/xref_parser.yrl
index 0711da79e2..5ee6419ff5 100644
--- a/lib/tools/src/xref_parser.yrl
+++ b/lib/tools/src/xref_parser.yrl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2000-2016. All Rights Reserved.
+%% Copyright Ericsson AB 2000-2017. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -170,7 +170,7 @@ is_prefix_op('#') -> numeric;
 is_prefix_op(_) -> false.
 
 check_regexp(String) ->
-    case re:compile(String) of
+    case re:compile(String, [unicode]) of
 	{ok, _Expr} ->
 	    {regexpr, String};
 	{error, {ErrString, Position}} ->
@@ -274,7 +274,7 @@ mfa2s({M,F,A}) ->
     [c2s(M),':',c2s(F),'/',A].
 
 c2s(C) ->
-    [S] = io_lib:format("~p", [C]),
+    [S] = io_lib:format("~tp", [C]),
     list_to_atom(S).
 
 re(variable) -> ['_'];
diff --git a/lib/tools/src/xref_utils.erl b/lib/tools/src/xref_utils.erl
index b0c168e018..02e207d40c 100644
--- a/lib/tools/src/xref_utils.erl
+++ b/lib/tools/src/xref_utils.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2000-2016. All Rights Reserved.
+%% Copyright Ericsson AB 2000-2017. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -638,14 +638,14 @@ neighbours([], G, Fun, VT, L, _V, Vs) ->
     neighbours(Vs, G, Fun, VT, L).
 
 match_list(L, RExpr) ->
-    {ok, Expr} = re:compile(RExpr),
+    {ok, Expr} = re:compile(RExpr, [unicode]),
     filter(fun(E) -> match(E, Expr) end, L).
 
 match_one(VarL, Con, Col) ->
     select_each(VarL, fun(E) -> Con =:= element(Col, E) end).
 
 match_many(VarL, RExpr, Col) ->
-    {ok, Expr} = re:compile(RExpr),    
+    {ok, Expr} = re:compile(RExpr, [unicode]),
     select_each(VarL, fun(E) -> match(element(Col, E), Expr) end).
 
 match(I, Expr) when is_integer(I) ->
@@ -653,7 +653,12 @@ match(I, Expr) when is_integer(I) ->
     {match, [{0,length(S)}]} =:= re:run(S, Expr, [{capture, first}]);
 match(A, Expr) when is_atom(A) ->
     S = atom_to_list(A),
-    {match, [{0,length(S)}]} =:= re:run(S, Expr, [{capture, first}]).
+    case re:run(S, Expr, [{capture, first}]) of
+        {match, [{0,Size}]} ->
+            Size =:= byte_size(unicode:characters_to_binary(S));
+        _ ->
+            false
+    end.
 
 select_each([{Mod,Funs} | L], Pred) ->
     case filter(Pred, Funs) of
diff --git a/lib/tools/test/xref_SUITE.erl b/lib/tools/test/xref_SUITE.erl
index 057449d4a2..379a5c09ab 100644
--- a/lib/tools/test/xref_SUITE.erl
+++ b/lib/tools/test/xref_SUITE.erl
@@ -50,7 +50,8 @@
 
 -export([analyze/1, basic/1, md/1, q/1, variables/1, unused_locals/1]).
 
--export([format_error/1, otp_7423/1, otp_7831/1, otp_10192/1, otp_13708/1]).
+-export([format_error/1, otp_7423/1, otp_7831/1, otp_10192/1, otp_13708/1,
+         otp_14464/1]).
 
 -import(lists, [append/2, flatten/1, keysearch/3, member/2, sort/1, usort/1]).
 
@@ -81,8 +82,10 @@ groups() ->
        update, deprecated, trycatch, fun_mfa,
        fun_mfa_r14, fun_mfa_vars, qlc]},
      {analyses, [],
+
       [analyze, basic, md, q, variables, unused_locals]},
-     {misc, [], [format_error, otp_7423, otp_7831, otp_10192, otp_13708]}].
+     {misc, [], [format_error, otp_7423, otp_7831, otp_10192, otp_13708,
+                 otp_14464]}].
 
 
 init_per_suite(Conf) when is_list(Conf) ->
@@ -2396,7 +2399,6 @@ otp_10192(Conf) when is_list(Conf) ->
     xref:stop(s),
     ok.
 
-%% OTP-10192. Allow filenames with character codes greater than 126.
 otp_13708(Conf) when is_list(Conf) ->
     {ok, _} = start(s),
     ok = xref:set_default(s, [{verbose, true}]),
@@ -2409,6 +2411,36 @@ otp_13708(Conf) when is_list(Conf) ->
     ok = xref:set_library_path(s, [Dir], [{verbose, true}]),
     xref:stop(s).
 
+%% OTP-14464. Unicode atoms.
+otp_14464(Conf) when is_list(Conf) ->
+    Dir = ?copydir,
+
+    File1 = fname(Dir, "a.erl"),
+    MFile1 = fname(Dir, "a"),
+    Beam1 = fname(Dir, "a.beam"),
+    Test1 = "-module(a).
+             -export([ärlig/0, 'кlирилли́ческий атомB'/0]).
+
+              ärlig() ->
+                  'кlирилли́ческий атомB'.
+
+              'кlирилли́ческий атомB'() ->
+                  foo.
+             ",
+    ok = file:write_file(File1, unicode:characters_to_binary(Test1)),
+    {ok, a} = compile:file(File1, [debug_info,{outdir,Dir}]),
+
+    {ok, _} = xref:start(s),
+    {ok, a} = xref:add_module(s, MFile1),
+
+    {ok, [{a,ärlig,0}]} = xref:q(s, 'a:"ärlig"/0'),
+    {ok, [{a,'кlирилли́ческий атомB',0}]} =
+        xref:q(s, 'a:"кlирилли́ческий атомB"/0'),
+
+    xref:stop(s),
+    ok = file:delete(File1),
+    ok = file:delete(Beam1).
+
 %%%
 %%% Utilities
 %%%