Merge branch 'hasse/unicode_atoms/OTP-14285'

* hasse/unicode_atoms/OTP-14285:
  compiler: Handle (bad) Unicode parse transform module names
  kernel: Improve handling of Unicode filenames
  stdlib: Handle Unicode atoms in ms_transform
  stdlib: Improve Unicode handling of the Erlang parser
  stdlib: Handle unknown compiler options with Unicode
  stdlib: Handle Unicode macro names
  stdlib: Correct Unicode handling in escript
  dialyzer: Improve handling of Unicode
  parsetools: Improve handling of Unicode atoms
  stdlib: Handle Unicode atoms when formatting stacktraces
  stdlib: Add more checks of module names to the linter
  stdlib: Handle Unicode atoms better in io_lib_format
  stdlib: Handle Unicode atoms in c.erl

4 files changed, 170 insertions, 10 deletions

diff --git a/lib/stdlib/test/epp_SUITE.erl b/lib/stdlib/test/epp_SUITE.erl
index 71d6820c47..915f478dfa 100644
--- a/lib/stdlib/test/epp_SUITE.erl
+++ b/lib/stdlib/test/epp_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1998-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1998-2017. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -28,7 +28,7 @@
          otp_8130/1, overload_mac/1, otp_8388/1, otp_8470/1,
          otp_8562/1, otp_8665/1, otp_8911/1, otp_10302/1, otp_10820/1,
          otp_11728/1, encoding/1, extends/1,  function_macro/1,
-	 test_error/1, test_warning/1]).
+	 test_error/1, test_warning/1, otp_14285/1]).
 
 -export([epp_parse_erl_form/2]).
 
@@ -68,7 +68,8 @@ all() ->
      not_circular, skip_header, otp_6277, otp_7702, otp_8130,
      overload_mac, otp_8388, otp_8470, otp_8562,
      otp_8665, otp_8911, otp_10302, otp_10820, otp_11728,
-     encoding, extends, function_macro, test_error, test_warning].
+     encoding, extends, function_macro, test_error, test_warning,
+     otp_14285].
 
 groups() -> 
     [{upcase_mac, [], [upcase_mac_1, upcase_mac_2]},
@@ -677,7 +678,7 @@ otp_8130(Config) when is_list(Config) ->
           {otp_8130_c6,
            <<"-define(M3(), A).\n"
              "t() -> A = 1, ?3.14159}.\n">>,
-           {errors,[{{2,16},epp,{call,"?3.14159"}}],[]}},
+           {errors,[{{2,16},epp,{call,[$?,"3.14159"]}}],[]}},
 
           {otp_8130_c7,
            <<"\nt() -> ?A.\n">>,
@@ -1384,6 +1385,26 @@ do_otp_10820(File, C, PC) ->
     true = test_server:stop_node(Node),
     ok.
 
+%% OTP_14285: Unicode atoms.
+otp_14285(Config) when is_list(Config) ->
+    %% This is just a sample of errors.
+    Cs = [{otp_8562,
+           <<"-export([f/0]).
+              -define('a\x{400}b', 'a\x{400}d').
+              f() ->
+                  ?'a\x{400}b'.
+              g() ->
+                  ?\"a\x{400}b\".
+              h() ->
+                  ?'a\x{400}no'().
+              "/utf8>>,
+           {errors,[{6,epp,{call,[63,[91,["97",44,"1024",44,"98"],93]]}},
+                    {8,epp,{undefined,'a\x{400}no',0}}],
+            []}}
+         ],
+    [] = compile(Config, Cs),
+    ok.
+
 %% OTP-11728. Bugfix circular macro.
 otp_11728(Config) when is_list(Config) ->
     Dir = proplists:get_value(priv_dir, Config),
diff --git a/lib/stdlib/test/erl_lint_SUITE.erl b/lib/stdlib/test/erl_lint_SUITE.erl
index cc3d605840..6a75eaa737 100644
--- a/lib/stdlib/test/erl_lint_SUITE.erl
+++ b/lib/stdlib/test/erl_lint_SUITE.erl
@@ -66,7 +66,7 @@
          otp_11851/1,otp_11879/1,otp_13230/1,
          record_errors/1, otp_11879_cont/1,
          non_latin1_module/1, otp_14323/1,
-         get_stacktrace/1]).
+         get_stacktrace/1, otp_14285/1]).
 
 suite() ->
     [{ct_hooks,[ts_install_cth]},
@@ -87,7 +87,7 @@ all() ->
      maps, maps_type, maps_parallel_match,
      otp_11851, otp_11879, otp_13230,
      record_errors, otp_11879_cont, non_latin1_module, otp_14323,
-     get_stacktrace].
+     get_stacktrace, otp_14285].
 
 groups() -> 
     [{unused_vars_warn, [],
@@ -3922,10 +3922,72 @@ otp_11879_cont(Config) ->
 
 %% OTP-14285: We currently don't support non-latin1 module names.
 
-non_latin1_module(_Config) ->
+non_latin1_module(Config) ->
     do_non_latin1_module('юникод'),
     do_non_latin1_module(list_to_atom([256,$a,$b,$c])),
     do_non_latin1_module(list_to_atom([$a,$b,256,$c])),
+
+    "module names with non-latin1 characters are not supported" =
+        format_error(non_latin1_module_unsupported),
+    BadCallback =
+        {bad_callback,{'кирилли́ческий атом','кирилли́ческий атом',0}},
+    "explicit module not allowed for callback "
+    "'кирилли́ческий атом':'кирилли́ческий атом'/0" =
+        format_error(BadCallback),
+    UndefBehav = {undefined_behaviour,'кирилли́ческий атом'},
+    "behaviour 'кирилли́ческий атом' undefined" =
+        format_error(UndefBehav),
+    BadDepr = {bad_nowarn_deprecated_function,
+               {'кирилли́ческий атом','кирилли́ческий атом',18}},
+    "'кирилли́ческий атом':'кирилли́ческий атом'/18 is not a deprecated "
+    "function" = format_error(BadDepr),
+    Ts = [{non_latin1_module,
+           <<"
+            %% Report uses of module names with non-Latin-1 characters.
+
+            -import('кирилли́ческий атом', []).
+            -behaviour('кирилли́ческий атом').
+            -behavior('кирилли́ческий атом').
+
+            -callback 'кирилли́ческий атом':'кирилли́ческий атом'() -> a.
+
+            -compile([{nowarn_deprecated_function,
+                       [{'кирилли́ческий атом','кирилли́ческий атом',18}]}]).
+
+            %% erl_lint:gexpr/3 is not extended to check module name here:
+            t1() when 'кирилли́ческий атом':'кирилли́ческий атом'(1) ->
+                b.
+
+            t2() ->
+                'кирилли́ческий атом':'кирилли́ческий атом'().
+
+            -spec 'кирилли́ческий атом':'кирилли́ческий атом'() -> atom().
+
+            -spec 'кирилли́ческий атом'(integer()) ->
+              'кирилли́ческий атом':'кирилли́ческий атом'().
+
+            'кирилли́ческий атом'(1) ->
+                'кирилли́ческий атом':f(),
+                F = f,
+                'кирилли́ческий атом':F()."/utf8>>,
+           [],
+           {error,
+            [{4,erl_lint,non_latin1_module_unsupported},
+             {5,erl_lint,non_latin1_module_unsupported},
+             {6,erl_lint,non_latin1_module_unsupported},
+             {8,erl_lint,non_latin1_module_unsupported},
+             {8,erl_lint,BadCallback},
+             {10,erl_lint,non_latin1_module_unsupported},
+             {14,erl_lint,illegal_guard_expr},
+             {18,erl_lint,non_latin1_module_unsupported},
+             {20,erl_lint,non_latin1_module_unsupported},
+             {23,erl_lint,non_latin1_module_unsupported},
+             {26,erl_lint,non_latin1_module_unsupported},
+             {28,erl_lint,non_latin1_module_unsupported}],
+            [{5,erl_lint,UndefBehav},
+             {6,erl_lint,UndefBehav},
+             {10,erl_lint,BadDepr}]}}],
+    run(Config, Ts),
     ok.
 
 do_non_latin1_module(Mod) ->
@@ -4042,6 +4104,53 @@ get_stacktrace(Config) ->
     run(Config, Ts),
     ok.
 
+%% Unicode atoms.
+otp_14285(Config) ->
+    %% A small sample of all the errors and warnings in module erl_lint.
+    E1 = {redefine_function,{'кирилли́ческий атом',0}},
+    E2 = {attribute,'кирилли́ческий атом'},
+    E3 = {undefined_record,'кирилли́ческий атом'},
+    E4 = {undefined_bittype,'кирилли́ческий атом'},
+    "function 'кирилли́ческий атом'/0 already defined" = format_error(E1),
+    "attribute 'кирилли́ческий атом' after function definitions" =
+        format_error(E2),
+    "record 'кирилли́ческий атом' undefined" = format_error(E3),
+    "bit type 'кирилли́ческий атом' undefined" = format_error(E4),
+    Ts = [{otp_14285_1,
+           <<"'кирилли́ческий атом'() -> a.
+              'кирилли́ческий атом'() -> a.
+             "/utf8>>,
+           [],
+           {errors,
+            [{2,erl_lint,E1}],
+            []}},
+         {otp_14285_2,
+           <<"'кирилли́ческий атом'() -> a.
+              -'кирилли́ческий атом'(a).
+             "/utf8>>,
+           [],
+           {errors,
+            [{2,erl_lint,E2}],
+            []}},
+         {otp_14285_3,
+           <<"'кирилли́ческий атом'() -> #'кирилли́ческий атом'{}.
+             "/utf8>>,
+           [],
+           {errors,
+            [{1,erl_lint,E3}],
+            []}},
+         {otp_14285_4,
+           <<"t() -> <<34/'кирилли́ческий атом'>>.
+             "/utf8>>,
+           [],
+           {errors,
+            [{1,erl_lint,E4}],
+            []}}],
+    run(Config, Ts),
+    ok.
+
+format_error(E) ->
+    lists:flatten(erl_lint:format_error(E)).
 
 run(Config, Tests) ->
     F = fun({N,P,Ws,E}, BadL) ->
diff --git a/lib/stdlib/test/proc_lib_SUITE.erl b/lib/stdlib/test/proc_lib_SUITE.erl
index a53e99afc9..029e6286e4 100644
--- a/lib/stdlib/test/proc_lib_SUITE.erl
+++ b/lib/stdlib/test/proc_lib_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1996-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1996-2017. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -27,7 +27,7 @@
 -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
 	 init_per_group/2,end_per_group/2, 
 	 crash/1, stacktrace/1, sync_start_nolink/1, sync_start_link/1,
-         spawn_opt/1, sp1/0, sp2/0, sp3/1, sp4/2, sp5/1,
+         spawn_opt/1, sp1/0, sp2/0, sp3/1, sp4/2, sp5/1, '\x{447}'/0,
 	 hibernate/1, stop/1, t_format/1]).
 -export([ otp_6345/1, init_dont_hang/1]).
 
@@ -139,6 +139,14 @@ crash(Config) when is_list(Config) ->
 	    {error_info,{exit,abnormal,{stacktrace}}}],
     analyse_crash(Pid5, Exp5, []),
 
+    %% Unicode atom
+    Pid6 = proc_lib:spawn(?MODULE, '\x{447}', []),
+    Pid6 ! die,
+    Exp6 = [{initial_call,{?MODULE,'\x{447}',[]}},
+	    {ancestors,[self()]},
+	    {error_info,{exit,die,{stacktrace}}}],
+    analyse_crash(Pid6, Exp6, []),
+
     error_logger:delete_report_handler(?MODULE),
     ok.
 
@@ -304,6 +312,12 @@ sp4(Parent, Tester) ->
     end,
     proc_lib:init_ack(Parent, self()).
 
+'\x{447}'() ->
+    receive
+	die -> exit(die);
+	_ -> sp1()
+    end.
+
 hibernate(Config) when is_list(Config) ->
     Ref = make_ref(),
     Self = self(),
diff --git a/lib/stdlib/test/shell_SUITE.erl b/lib/stdlib/test/shell_SUITE.erl
index 5a929157d3..4f0fdc4c6a 100644
--- a/lib/stdlib/test/shell_SUITE.erl
+++ b/lib/stdlib/test/shell_SUITE.erl
@@ -2680,7 +2680,7 @@ prompt_err(B) ->
     S = string:strip(S2, both, $"),
     string:strip(S, right, $.).
 
-%% OTP-10302. Unicode.
+%% OTP-10302. Unicode. Also OTP-14285, Unicode atoms.
 otp_10302(Config) when is_list(Config) ->
     {ok,Node} = start_node(shell_suite_helper_2,
 			   "-pa "++proplists:get_value(priv_dir,Config)++
@@ -2818,6 +2818,22 @@ otp_10302(Config) when is_list(Config) ->
     "                    erl_eval:'-inside-an-interpreted-fun-'(65,\"\x{441}\")"
     " .\n" = t({Node,Test13}),
 
+    %% Unicode atoms.
+    Test14 = <<"'\\x{447}\\x{435}'().">>,
+    "** exception error: undefined shell command '\\x{447}\\x{435}'/0.\n" =
+        t(Test14),
+    Test15 = <<"io:setopts([{encoding,utf8}]).
+                '\\x{447}\\x{435}'().">>,
+    "ok.\n** exception error: undefined shell command '\x{447}\x{435}'/0.\n" =
+        t({Node,Test15}),
+    Test16 = <<"shell_SUITE:'\\x{447}\\x{435}'().">>,
+    "** exception error: undefined function "
+   "shell_SUITE:'\\x{447}\\x{435}'/0.\n" = t(Test16),
+    Test17 = <<"io:setopts([{encoding,utf8}]).
+                shell_SUITE:'\\x{447}\\x{435}'().">>,
+    "ok.\n** exception error: undefined function "
+    "shell_SUITE:'\x{447}\x{435}'/0.\n" =
+        t({Node,Test17}),
     test_server:stop_node(Node),
     ok.