Merge branch 'hb/stdlib/unicode_corrections/OTP-10820'

* hb/stdlib/unicode_corrections/OTP-10820:
  [stdlib] Add documentation of ~tp
  [stdlib] Correct handling of Unicode filenames

6 files changed, 64 insertions, 38 deletions

diff --git a/lib/stdlib/doc/src/io.xml b/lib/stdlib/doc/src/io.xml
index 9f59fda6b5..63f814ad2e 100644
--- a/lib/stdlib/doc/src/io.xml
+++ b/lib/stdlib/doc/src/io.xml
@@ -494,8 +494,9 @@ ok
               <c>~w</c>, but breaks terms whose printed representation
               is longer than one line into many lines and indents each
               line sensibly. It also tries to detect lists of
-              printable characters and to output these as strings.
-              For example:</p>
+              printable characters and to output these as strings. The
+              Unicode translation modifier is used for determining
+              what characters are printable. For example:</p>
             <pre>
 1> <input>T = [{attributes,[[{id,age,1.50000},{mode,explicit},</input>
 <input>{typename,"INTEGER"}], [{id,cho},{mode,explicit},{typename,'Cho'}]]},</input>
@@ -546,6 +547,19 @@ ok
 [{a,[97]},
  {b,[98]}]
 ok</pre>
+            <p>Binaries that look like UTF-8 encoded strings will be
+              output with the string syntax if the Unicode translation
+              modifier is given:</p>
+            <pre>
+9> <input>io:fwrite("~p~n",[[1024]]).</input>
+[1024]
+10> <input>io:fwrite("~tp~n",[[1024]]).</input>
+"\x{400}"
+11> <input>io:fwrite("~tp~n", [&lt;&lt;128,128&gt;&gt;]).</input>
+&lt;&lt;128,128&gt;&gt;
+12> <input>io:fwrite("~tp~n", [&lt;&lt;208,128&gt;&gt;]).</input>
+&lt;&lt;"\x{400}"/utf8&gt;&gt;
+ok</pre>
           </item>
           <tag><c>W</c></tag>
           <item>
diff --git a/lib/stdlib/src/epp.erl b/lib/stdlib/src/epp.erl
index 1bb3b95ae2..0a1caa7178 100644
--- a/lib/stdlib/src/epp.erl
+++ b/lib/stdlib/src/epp.erl
@@ -614,7 +614,7 @@ enter_file_reply(From, Name, Location, AtLocation) ->
 
 %% Flatten filename to a string. Must be a valid filename.
 
-file_name([C | T]) when is_integer(C), C > 0, C =< 255 ->
+file_name([C | T]) when is_integer(C), C > 0 ->
     [C | file_name(T)];
 file_name([H|T]) ->
     file_name(H) ++ file_name(T);
diff --git a/lib/stdlib/src/erl_pp.erl b/lib/stdlib/src/erl_pp.erl
index a868867a81..06dae51cc9 100644
--- a/lib/stdlib/src/erl_pp.erl
+++ b/lib/stdlib/src/erl_pp.erl
@@ -42,7 +42,7 @@
                 | {encoding, latin1 | unicode | utf8}).
 -type(options() :: hook_function() | [option()]).
 
--record(pp, {string_fun, char_fun}).
+-record(pp, {string_fun, char_fun, term_fun}).
 
 -record(options, {hook, encoding, opts}).
 
@@ -61,7 +61,8 @@ form(Thing) ->
       Options :: options()).
 
 form(Thing, Options) ->
-    frmt(lform(Thing, options(Options)), state(Options)).
+    State = state(Options),
+    frmt(lform(Thing, options(Options), State), State).
 
 -spec(attribute(Attribute) -> io_lib:chars() when
       Attribute :: erl_parse:abstract_form()).
@@ -74,7 +75,8 @@ attribute(Thing) ->
       Options :: options()).
 
 attribute(Thing, Options) ->
-    frmt(lattribute(Thing, options(Options)), state(Options)).
+    State = state(Options),
+    frmt(lattribute(Thing, options(Options), State), State).
 
 -spec(function(Function) -> io_lib:chars() when
       Function :: erl_parse:abstract_form()).
@@ -180,11 +182,13 @@ state(_Hook) ->
 
 state() ->
     #pp{string_fun = fun io_lib:write_string_as_latin1/1,
-        char_fun   = fun io_lib:write_char_as_latin1/1}.
+        char_fun   = fun io_lib:write_char_as_latin1/1,
+        term_fun   = fun(T) -> io_lib:format("~p", [T]) end}.
 
 unicode_state() ->
     #pp{string_fun = fun io_lib:write_string/1,
-        char_fun   = fun io_lib:write_char/1}.
+        char_fun   = fun io_lib:write_char/1,
+        term_fun   = fun(T) -> io_lib:format("~tp", [T]) end}.
 
 encoding(Options) ->
     case proplists:get_value(encoding, Options, epp:default_encoding()) of
@@ -193,47 +197,47 @@ encoding(Options) ->
         unicode -> unicode
     end.
 
-lform({attribute,Line,Name,Arg}, Opts) ->
-    lattribute({attribute,Line,Name,Arg}, Opts);
-lform({function,Line,Name,Arity,Clauses}, Opts) ->
+lform({attribute,Line,Name,Arg}, Opts, State) ->
+    lattribute({attribute,Line,Name,Arg}, Opts, State);
+lform({function,Line,Name,Arity,Clauses}, Opts, _State) ->
     lfunction({function,Line,Name,Arity,Clauses}, Opts);
-lform({rule,Line,Name,Arity,Clauses}, Opts) ->
+lform({rule,Line,Name,Arity,Clauses}, Opts, _State) ->
     lrule({rule,Line,Name,Arity,Clauses}, Opts);
 %% These are specials to make it easier for the compiler.
-lform({error,E}, _Opts) ->
-    leaf(format("~p\n", [{error,E}]));
-lform({warning,W}, _Opts) ->
-    leaf(format("~p\n", [{warning,W}]));
-lform({eof,_Line}, _Opts) ->
+lform({error,E}, _Opts, State) ->
+    leaf((State#pp.term_fun)({error,E})++"\n");
+lform({warning,W}, _Opts, State) ->
+    leaf((State#pp.term_fun)({warning,W})++"\n");
+lform({eof,_Line}, _Opts, _State) ->
     $\n.
 
-lattribute({attribute,_Line,type,Type}, Opts) ->
+lattribute({attribute,_Line,type,Type}, Opts, _State) ->
     [typeattr(type, Type, Opts),leaf(".\n")];
-lattribute({attribute,_Line,opaque,Type}, Opts) ->
+lattribute({attribute,_Line,opaque,Type}, Opts, _State) ->
     [typeattr(opaque, Type, Opts),leaf(".\n")];
-lattribute({attribute,_Line,spec,Arg}, _Opts) ->
+lattribute({attribute,_Line,spec,Arg}, _Opts, _State) ->
     [specattr(Arg),leaf(".\n")];
-lattribute({attribute,_Line,Name,Arg}, Opts) ->
-    [lattribute(Name, Arg, Opts),leaf(".\n")].
+lattribute({attribute,_Line,Name,Arg}, Opts, State) ->
+    [lattribute(Name, Arg, Opts, State),leaf(".\n")].
 
-lattribute(module, {M,Vs}, _Opts) ->
+lattribute(module, {M,Vs}, _Opts, _State) ->
     attr("module",[{var,0,pname(M)},
                    foldr(fun(V, C) -> {cons,0,{var,0,V},C}
                          end, {nil,0}, Vs)]);
-lattribute(module, M, _Opts) ->
+lattribute(module, M, _Opts, _State) ->
     attr("module", [{var,0,pname(M)}]);
-lattribute(export, Falist, _Opts) ->
+lattribute(export, Falist, _Opts, _State) ->
     call({var,0,"-export"}, [falist(Falist)], 0, none);
-lattribute(import, Name, _Opts) when is_list(Name) ->
+lattribute(import, Name, _Opts, _State) when is_list(Name) ->
     attr("import", [{var,0,pname(Name)}]);
-lattribute(import, {From,Falist}, _Opts) ->
+lattribute(import, {From,Falist}, _Opts, _State) ->
     attr("import",[{var,0,pname(From)},falist(Falist)]);
-lattribute(file, {Name,Line}, _Opts) ->
-    attr("file", [{var,0,format("~p", [Name])},{integer,0,Line}]);
-lattribute(record, {Name,Is}, Opts) ->
+lattribute(file, {Name,Line}, _Opts, State) ->
+    attr("file", [{var,0,(State#pp.term_fun)(Name)},{integer,0,Line}]);
+lattribute(record, {Name,Is}, Opts, _State) ->
     Nl = leaf(format("-record(~w,", [Name])),
     [{first,Nl,record_fields(Is, Opts)},$)];
-lattribute(Name, Arg, #options{encoding = Encoding}) ->
+lattribute(Name, Arg, #options{encoding = Encoding}, _State) ->
     attr(write(Name), [erl_parse:abstract(Arg, [{encoding,Encoding}])]).
 
 typeattr(Tag, {TypeName,Type,Args}, _Opts) ->
@@ -423,7 +427,7 @@ lexpr(E, Opts) ->
 
 lexpr({var,_,V}, _, _) when is_integer(V) ->    %Special hack for Robert
     leaf(format("_~w", [V]));
-lexpr({var,_,V}, _, _) -> leaf(format("~s", [V]));
+lexpr({var,_,V}, _, _) -> leaf(format("~ts", [V]));
 lexpr({char,_,C}, _, _) -> {char,C};
 lexpr({integer,_,N}, _, _) -> leaf(write(N));
 lexpr({float,_,F}, _, _) -> leaf(write(F));
@@ -799,7 +803,7 @@ maybe_paren(_P, _Prec, Expr) ->
     Expr.
 
 leaf(S) ->
-    {leaf,iolist_size(S),S}.
+    {leaf,chars_size(S),S}.
 
 %%% Do the formatting. Currently nothing fancy. Could probably have
 %%% done it in one single pass.
@@ -1009,7 +1013,7 @@ incr(I, Incr) ->
     I+Incr.
 
 indentation(E, I) when I < 0 ->
-    iolist_size(E);
+    chars_size(E);
 indentation(E, I0) ->
     I = io_lib_format:indentation(E, I0),
     case has_nl(E) of
@@ -1064,6 +1068,15 @@ write_char(C, PP) ->
 %% Utilities
 %%
 
+chars_size([C | Es]) when is_integer(C) ->
+    1 + chars_size(Es);
+chars_size([E | Es]) ->
+    chars_size(E) + chars_size(Es);
+chars_size([]) ->
+    0;
+chars_size(B) when is_binary(B) ->
+    byte_size(B).
+
 -define(N_SPACES, 30).
 
 spacetab() ->
diff --git a/lib/syntax_tools/src/erl_comment_scan.erl b/lib/syntax_tools/src/erl_comment_scan.erl
index a70e7ba413..b949675f1c 100644
--- a/lib/syntax_tools/src/erl_comment_scan.erl
+++ b/lib/syntax_tools/src/erl_comment_scan.erl
@@ -282,7 +282,7 @@ join_lines([], Txt, L, Col, Ind) ->
 %% =====================================================================
 %% Utility functions for internal use
 
-filename([C|T]) when is_integer(C), C > 0, C =< 255 ->
+filename([C|T]) when is_integer(C), C > 0 ->
     [C | filename(T)];
 filename([]) ->
     [];
diff --git a/lib/syntax_tools/src/erl_tidy.erl b/lib/syntax_tools/src/erl_tidy.erl
index e9a88caff3..35fd0c2e02 100644
--- a/lib/syntax_tools/src/erl_tidy.erl
+++ b/lib/syntax_tools/src/erl_tidy.erl
@@ -435,7 +435,6 @@ file_type(Name, Links) ->
     end.
 
 open_output_file(FName, Options) ->
-io:format("Options ~p~n", [Options]),
     case catch file:open(FName, [write]++Options) of
         {ok, FD} ->
             FD;
@@ -1805,7 +1804,7 @@ get_free_vars_1([{free, B} | _Bs]) -> B;
 get_free_vars_1([_ | Bs]) -> get_free_vars_1(Bs);
 get_free_vars_1([]) -> [].
 
-filename([C | T]) when is_integer(C), C > 0, C =< 255 ->
+filename([C | T]) when is_integer(C), C > 0 ->
     [C | filename(T)];
 filename([H|T]) ->
     filename(H) ++ filename(T);
diff --git a/lib/syntax_tools/src/igor.erl b/lib/syntax_tools/src/igor.erl
index 8abc3f41cb..567bec00cb 100644
--- a/lib/syntax_tools/src/igor.erl
+++ b/lib/syntax_tools/src/igor.erl
@@ -2956,7 +2956,7 @@ timestamp() ->
 				"~2.2.0w:~2.2.0w:~2.2.0w.",
 				[Yr, Mth, Dy, Hr, Mt, Sc])).
 
-filename([C | T]) when is_integer(C), C > 0, C =< 255 ->
+filename([C | T]) when is_integer(C), C > 0 ->
     [C | filename(T)];
 filename([H|T]) ->
     filename(H) ++ filename(T);