Fix some Unicode issues

Also let the Erlang shell use the new function io:printable_range().

5 files changed, 103 insertions, 50 deletions

diff --git a/lib/stdlib/src/erl_pp.erl b/lib/stdlib/src/erl_pp.erl
index 06dae51cc9..7c7566e4ec 100644
--- a/lib/stdlib/src/erl_pp.erl
+++ b/lib/stdlib/src/erl_pp.erl
@@ -42,7 +42,7 @@
                 | {encoding, latin1 | unicode | utf8}).
 -type(options() :: hook_function() | [option()]).
 
--record(pp, {string_fun, char_fun, term_fun}).
+-record(pp, {string_fun, char_fun}).
 
 -record(options, {hook, encoding, opts}).
 
@@ -182,13 +182,11 @@ state(_Hook) ->
 
 state() ->
     #pp{string_fun = fun io_lib:write_string_as_latin1/1,
-        char_fun   = fun io_lib:write_char_as_latin1/1,
-        term_fun   = fun(T) -> io_lib:format("~p", [T]) end}.
+        char_fun   = fun io_lib:write_char_as_latin1/1}.
 
 unicode_state() ->
     #pp{string_fun = fun io_lib:write_string/1,
-        char_fun   = fun io_lib:write_char/1,
-        term_fun   = fun(T) -> io_lib:format("~tp", [T]) end}.
+        char_fun   = fun io_lib:write_char/1}.
 
 encoding(Options) ->
     case proplists:get_value(encoding, Options, epp:default_encoding()) of
@@ -204,10 +202,10 @@ lform({function,Line,Name,Arity,Clauses}, Opts, _State) ->
 lform({rule,Line,Name,Arity,Clauses}, Opts, _State) ->
     lrule({rule,Line,Name,Arity,Clauses}, Opts);
 %% These are specials to make it easier for the compiler.
-lform({error,E}, _Opts, State) ->
-    leaf((State#pp.term_fun)({error,E})++"\n");
-lform({warning,W}, _Opts, State) ->
-    leaf((State#pp.term_fun)({warning,W})++"\n");
+lform({error,E}, _Opts, _State) ->
+    leaf(format("~p\n", [{error,E}]));
+lform({warning,W}, _Opts, _State) ->
+    leaf(format("~p\n", [{warning,W}]));
 lform({eof,_Line}, _Opts, _State) ->
     $\n.
 
@@ -233,7 +231,7 @@ lattribute(import, Name, _Opts, _State) when is_list(Name) ->
 lattribute(import, {From,Falist}, _Opts, _State) ->
     attr("import",[{var,0,pname(From)},falist(Falist)]);
 lattribute(file, {Name,Line}, _Opts, State) ->
-    attr("file", [{var,0,(State#pp.term_fun)(Name)},{integer,0,Line}]);
+    attr("file", [{var,0,(State#pp.string_fun)(Name)},{integer,0,Line}]);
 lattribute(record, {Name,Is}, Opts, _State) ->
     Nl = leaf(format("-record(~w,", [Name])),
     [{first,Nl,record_fields(Is, Opts)},$)];
diff --git a/lib/stdlib/src/shell.erl b/lib/stdlib/src/shell.erl
index df66acb97b..c90b5ad5c8 100644
--- a/lib/stdlib/src/shell.erl
+++ b/lib/stdlib/src/shell.erl
@@ -129,7 +129,7 @@ start_restricted(RShMod) when is_atom(RShMod) ->
 	    error_logger:error_report(
 	      lists:flatten(
 		io_lib:fwrite(
-		  "Restricted shell module ~w not found: ~"++cs_p() ++"\n",
+		  "Restricted shell module ~w not found: ~tp\n",
 		  [RShMod,What]))),
 	    Error
     end.
@@ -214,8 +214,7 @@ server(StartSync) ->
             ok;
 	{RShMod2,What2} -> 
             io:fwrite(
-              ("Warning! Restricted shell module ~w not found: ~"
-               ++cs_p()++".\n"
+              ("Warning! Restricted shell module ~w not found: ~tp.\n"
                "Only the commands q() and init:stop() will be allowed!\n"),
               [RShMod2,What2]),
             application:set_env(stdlib, restricted_shell, ?MODULE)
@@ -337,7 +336,7 @@ get_prompt_func() ->
     end.
 
 bad_prompt_func(M) ->
-    fwrite_severity(benign, "Bad prompt function: ~"++cs_p(), [M]).
+    fwrite_severity(benign, "Bad prompt function: ~tp", [M]).
 
 default_prompt(N) ->
     %% Don't bother flattening the list irrespective of what the
@@ -571,7 +570,7 @@ report_exception(Class, Severity, {Reason,Stacktrace}, RT) ->
     I = iolist_size(Tag) + 1,
     PF = fun(Term, I1) -> pp(Term, I1, RT) end,
     SF = fun(M, _F, _A) -> (M =:= erl_eval) or (M =:= ?MODULE) end,
-    Enc = encoding(),
+    Enc = io:printable_range(),
     Str = lib:format_exception(I, Class, Reason, Stacktrace, SF, PF, Enc),
     io:requests([{put_chars, latin1, Tag},
                  {put_chars, unicode, Str},
@@ -1380,28 +1379,14 @@ pp(V, I, RT, Enc) ->
                              {record_print_fun, record_print_fun(RT)}]
                             ++ Enc)).
 
-%% Control sequence 'p' possibly with Unicode translation modifier
-cs_p() ->
-    case encoding() of
-        latin1 -> "p";
-        unicode -> "tp"
-    end.
-
 columns() ->
     case io:columns() of
         {ok,N} -> N;
         _ -> 80
     end.
 
-encoding() ->
-    [{encoding, Encoding}] = enc(),
-    Encoding.
-
 enc() ->
-    case lists:keyfind(encoding, 1, io:getopts()) of
-        false -> [{encoding,latin1}]; % should never happen
-        Enc -> [Enc]
-    end.
+    [{encoding, io:printable_range()}].
 
 garb(Shell) ->
     erlang:garbage_collect(Shell),
@@ -1424,10 +1409,9 @@ check_env(V) ->
 	{ok, Val} when is_integer(Val), Val >= 0 ->
 	    ok;
         {ok, Val} ->
-            Txt = io_lib:fwrite(
-                    ("Invalid value of STDLIB configuration parameter ~w: ~"
-                     ++cs_p()++"\n"),
-                    [V, Val]),
+            Txt = io_lib:fwrite
+                    ("Invalid value of STDLIB configuration parameter"
+                     "~w: ~tp\n", [V, Val]),
 	    error_logger:info_report(lists:flatten(Txt))
     end.
 	    
diff --git a/lib/stdlib/test/epp_SUITE.erl b/lib/stdlib/test/epp_SUITE.erl
index 041d521514..b2f1aa955a 100644
--- a/lib/stdlib/test/epp_SUITE.erl
+++ b/lib/stdlib/test/epp_SUITE.erl
@@ -25,7 +25,7 @@
 	 variable_1/1, otp_4870/1, otp_4871/1, otp_5362/1,
          pmod/1, not_circular/1, skip_header/1, otp_6277/1, otp_7702/1,
          otp_8130/1, overload_mac/1, otp_8388/1, otp_8470/1, otp_8503/1,
-         otp_8562/1, otp_8665/1, otp_8911/1, otp_10302/1]).
+         otp_8562/1, otp_8665/1, otp_8911/1, otp_10302/1, otp_10820/1]).
 
 -export([epp_parse_erl_form/2]).
 
@@ -67,7 +67,7 @@ all() ->
      {group, variable}, otp_4870, otp_4871, otp_5362, pmod,
      not_circular, skip_header, otp_6277, otp_7702, otp_8130,
      overload_mac, otp_8388, otp_8470, otp_8503, otp_8562,
-     otp_8665, otp_8911, otp_10302].
+     otp_8665, otp_8911, otp_10302, otp_10820].
 
 groups() -> 
     [{upcase_mac, [], [upcase_mac_1, upcase_mac_2]},
@@ -1359,6 +1359,30 @@ encoding_nocom(Enc, File) ->
     ok = file:close(Fd),
     E = epp:read_encoding(File, Options).
 
+otp_10820(doc) ->
+    "OTP-10820. Unicode filenames.";
+otp_10820(suite) ->
+    [];
+otp_10820(Config) when is_list(Config) ->
+    L = [915,953,959,973,957,953,954,959,957,964],
+    Dir = ?config(priv_dir, Config),
+    File = filename:join(Dir, L++".erl"),
+    C1 = <<"%% coding: utf-8\n -module(any).">>,
+    ok = do_otp_10820(File, C1, "+pc latin1"),
+    ok = do_otp_10820(File, C1, "+pc unicode"),
+    C2 = <<"\n-module(any).">>,
+    ok = do_otp_10820(File, C2, "+pc latin1"),
+    ok = do_otp_10820(File, C2, "+pc unicode").
+
+do_otp_10820(File, C, PC) ->
+    {ok,Node} = start_node(erl_pp_helper, "+fnu " ++ PC),
+    ok = rpc:call(Node, file, write_file, [File, C]),
+    {ok,[{attribute,1,file,{File,1}},
+         {attribute,2,module,any},
+         {eof,2}]} = rpc:call(Node, epp, parse_file, [File, [],[]]),
+    true = test_server:stop_node(Node),
+    ok.
+
 check(Config, Tests) ->
     eval_tests(Config, fun check_test/2, Tests).
 
@@ -1475,3 +1499,8 @@ ln2({error,M}) ->
     {error,ln2(M)};
 ln2(M) ->
     M.
+
+%% +fnu means a peer node has to be started; slave will not do
+start_node(Name, Xargs) ->
+    ?line PA = filename:dirname(code:which(?MODULE)),
+    test_server:start_node(Name, peer, [{args, "-pa " ++ PA ++ " " ++ Xargs}]).
diff --git a/lib/stdlib/test/erl_pp_SUITE.erl b/lib/stdlib/test/erl_pp_SUITE.erl
index 37be61d665..9c0a43abcc 100644
--- a/lib/stdlib/test/erl_pp_SUITE.erl
+++ b/lib/stdlib/test/erl_pp_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2006-2012. All Rights Reserved.
+%% Copyright Ericsson AB 2006-2013. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -49,7 +49,7 @@
 
 	  otp_6321/1, otp_6911/1, otp_6914/1, otp_8150/1, otp_8238/1,
 	  otp_8473/1, otp_8522/1, otp_8567/1, otp_8664/1, otp_9147/1,
-          otp_10302/1]).
+          otp_10302/1, otp_10820/1]).
 
 %% Internal export.
 -export([ehook/6]).
@@ -80,7 +80,8 @@ groups() ->
      {attributes, [], [misc_attrs, import_export]},
      {tickets, [],
       [otp_6321, otp_6911, otp_6914, otp_8150, otp_8238,
-       otp_8473, otp_8522, otp_8567, otp_8664, otp_9147, otp_10302]}].
+       otp_8473, otp_8522, otp_8567, otp_8664, otp_9147,
+       otp_10302, otp_10820]}].
 
 init_per_suite(Config) ->
     Config.
@@ -1074,6 +1075,34 @@ otp_10302(Config) when is_list(Config) ->
 unicode_hook({foo,E}, I, P, H) ->
     erl_pp:expr({call,0,{atom,0,foo},[E]}, I, P, H).
 
+otp_10820(doc) ->
+    "OTP-10820. Unicode filenames.";
+otp_10820(suite) -> [];
+otp_10820(Config) when is_list(Config) ->
+    C1 = <<"%% coding: utf-8\n -module(any).">>,
+    ok = do_otp_10820(Config, C1, "+pc latin1"),
+    ok = do_otp_10820(Config, C1, "+pc unicode"),
+    C2 = <<"-module(any).">>,
+    ok = do_otp_10820(Config, C2, "+pc latin1"),
+    ok = do_otp_10820(Config, C2, "+pc unicode").
+
+do_otp_10820(Config, C, PC) ->
+    {ok,Node} = start_node(erl_pp_helper, "+fnu " ++ PC),
+    L = [915,953,959,973,957,953,954,959,957,964],
+    FileName = filename(L++".erl", Config),
+    ok = rpc:call(Node, file, write_file, [FileName, C]),
+    {ok, _, []} =  rpc:call(Node, compile, file,
+                            [FileName, [return,'P',{outdir,?privdir}]]),
+    PFileName = filename(L++".P", Config),
+    {ok, Bin} = rpc:call(Node, file, read_file, [PFileName]),
+    true = test_server:stop_node(Node),
+    true = file_attr_is_string(binary_to_list(Bin)),
+    ok.
+
+file_attr_is_string("-file(\"" ++ _) -> true;
+file_attr_is_string([_ | L]) ->
+    file_attr_is_string(L).
+
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 compile(Config, Tests) ->
@@ -1247,3 +1276,8 @@ filename(Name, Config) ->
 fail() ->
     io:format("failed~n"),
     ?t:fail().
+
+%% +fnu means a peer node has to be started; slave will not do
+start_node(Name, Xargs) ->
+    ?line PA = filename:dirname(code:which(?MODULE)),
+    test_server:start_node(Name, peer, [{args, "-pa " ++ PA ++ " " ++ Xargs}]).
diff --git a/lib/stdlib/test/shell_SUITE.erl b/lib/stdlib/test/shell_SUITE.erl
index f22df96697..cff394bf09 100644
--- a/lib/stdlib/test/shell_SUITE.erl
+++ b/lib/stdlib/test/shell_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2004-2012. All Rights Reserved.
+%% Copyright Ericsson AB 2004-2013. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -2742,6 +2742,9 @@ otp_10302(doc) ->
     "OTP-10302. Unicode.";
 otp_10302(suite) -> [];
 otp_10302(Config) when is_list(Config) ->
+    {ok,Node} = start_node(shell_suite_helper_2,
+			   "-pa "++?config(priv_dir,Config)++
+			   " +pc unicode"),
     Test1 =
         <<"begin
                io:setopts([{encoding,utf8}]),
@@ -2749,13 +2752,13 @@ otp_10302(Config) when is_list(Config) ->
                rd(rec, {a = \"\\x{400}\"}),
                ok = rl(rec)
             end.">>,
-    "-record(rec,{a = \"\x{400}\"}).\nok.\n" = t(Test1),
+    "-record(rec,{a = \"\x{400}\"}).\nok.\n" = t({Node,Test1}),
 
     Test3 =
         <<"io:setopts([{encoding,utf8}]).
            rd(rec, {a = \"\\x{400}\"}).
            ok = rp(#rec{}).">>,
-    "ok.\nrec\n#rec{a = \"\x{400}\"}.\nok.\n" = t(Test3),
+    "ok.\nrec\n#rec{a = \"\x{400}\"}.\nok.\n" = t({Node,Test3}),
 
     Test4 =
         <<"io:setopts([{encoding,utf8}]).
@@ -2766,7 +2769,7 @@ otp_10302(Config) when is_list(Config) ->
     "ok.\n\"\x{400}\"\nA = \"\x{400}\".\nok.\n"
     "1: io:setopts([{encoding,utf8}])\n-> ok.\n"
     "2: A = [1024] = \"\x{400}\"\n-> \"\x{400}\"\n"
-    "3: b()\n-> ok.\nok.\n" = t(Test4),
+    "3: b()\n-> ok.\nok.\n" = t({Node,Test4}),
 
     Test5 =
         <<"begin
@@ -2776,18 +2779,20 @@ otp_10302(Config) when is_list(Config) ->
                b(),
                h()
            end.">>,
-    "A = \"\x{400}\".\nok.\n" = t(Test5),
+    "A = \"\x{400}\".\nok.\n" = t({Node,Test5}),
 
     %% One $" is "lost":
     true =
        "\x{400}\": command not found" =:=
-       prompt_err({<<"io:setopts([{encoding,utf8}]). v(\"\x{400}\")."/utf8>>,
+       prompt_err({Node,
+                   <<"io:setopts([{encoding,utf8}]). v(\"\x{400}\")."/utf8>>,
                    unicode}),
 
     "ok.\ndefault\n* Bad prompt function: \"\x{400}\".\n" =
-        t({<<"io:setopts([{encoding,utf8}]). "
+        t({Node,<<"io:setopts([{encoding,utf8}]). "
              "shell:prompt_func(\"\x{400}\")."/utf8>>,
            unicode}),
+    rpc:call(Node,shell, prompt_func, [default]),
     _ = shell:prompt_func(default),
 
     %% Test lib:format_exception() (cf. OTP-6554)
@@ -2815,7 +2820,7 @@ otp_10302(Config) when is_list(Config) ->
 
     "ok.\n** exception error: an error occurred when evaluating"
         " an arithmetic expression\n     in operator  '/'/2\n"
-        "        called as <<170>> / <<170>>.\n" = t(Test7),
+        "        called as <<170>> / <<170>>.\n" = t({Node,Test7}),
     Test8 =
         <<"begin
                A = [1089],
@@ -2839,7 +2844,7 @@ otp_10302(Config) when is_list(Config) ->
 
     "ok.\n** exception error: an error occurred when evaluating"
         " an arithmetic expression\n     in operator  '/'/2\n"
-        "        called as \"\x{441}\" / \"\x{441}\".\n" = t(Test9),
+        "        called as \"\x{441}\" / \"\x{441}\".\n" = t({Node,Test9}),
     Test10 =
         <<"A = {\"1\\xaa\",
            $\\xaa,
@@ -2861,7 +2866,7 @@ otp_10302(Config) when is_list(Config) ->
     "ok.\n** exception error: no function clause matching \n"
     "                    erl_eval:'-inside-an-interpreted-fun-'"
     "({\"1\xaa\",170,<<\"hi\">>,\n                                           "
-    "                 <<\"1\xaa\"/utf8>>}) .\n"  = t(Test11),
+    "                 <<\"1\xaa\"/utf8>>}) .\n"  = t({Node,Test11}),
     Test12 = <<"fun(a, b) -> false end(65, [1089]).">>,
     "** exception error: no function clause matching \n"
     "                    erl_eval:'-inside-an-interpreted-fun-'(65,[1089])"
@@ -2871,8 +2876,9 @@ otp_10302(Config) when is_list(Config) ->
            fun(a, b) -> false end(65, [1089]).">>,
     "ok.\n** exception error: no function clause matching \n"
     "                    erl_eval:'-inside-an-interpreted-fun-'(65,\"\x{441}\")"
-    " .\n" = t(Test13),
+    " .\n" = t({Node,Test13}),
 
+    test_server:stop_node(Node),
     ok.
 
 scan(B) ->
@@ -2895,6 +2901,8 @@ scan(S0, F) ->
             []
     end.
 
+t({Node,Bin,Enc}) when is_atom(Node),is_binary(Bin), is_atom(Enc) ->
+    t0({Bin,Enc}, fun() -> start_new_shell(Node) end);
 t({Node,Bin}) when is_atom(Node),is_binary(Bin) ->
     t0({Bin,latin1}, fun() -> start_new_shell(Node) end);
 t(Bin) when is_binary(Bin) ->