[stdlib] Correct handling of Unicode filenames

5 files changed, 48 insertions, 36 deletions

diff --git a/lib/stdlib/src/epp.erl b/lib/stdlib/src/epp.erl
index 1bb3b95ae2..0a1caa7178 100644
--- a/lib/stdlib/src/epp.erl
+++ b/lib/stdlib/src/epp.erl
@@ -614,7 +614,7 @@ enter_file_reply(From, Name, Location, AtLocation) ->
 
 %% Flatten filename to a string. Must be a valid filename.
 
-file_name([C | T]) when is_integer(C), C > 0, C =< 255 ->
+file_name([C | T]) when is_integer(C), C > 0 ->
     [C | file_name(T)];
 file_name([H|T]) ->
     file_name(H) ++ file_name(T);
diff --git a/lib/stdlib/src/erl_pp.erl b/lib/stdlib/src/erl_pp.erl
index a868867a81..06dae51cc9 100644
--- a/lib/stdlib/src/erl_pp.erl
+++ b/lib/stdlib/src/erl_pp.erl
@@ -42,7 +42,7 @@
                 | {encoding, latin1 | unicode | utf8}).
 -type(options() :: hook_function() | [option()]).
 
--record(pp, {string_fun, char_fun}).
+-record(pp, {string_fun, char_fun, term_fun}).
 
 -record(options, {hook, encoding, opts}).
 
@@ -61,7 +61,8 @@ form(Thing) ->
       Options :: options()).
 
 form(Thing, Options) ->
-    frmt(lform(Thing, options(Options)), state(Options)).
+    State = state(Options),
+    frmt(lform(Thing, options(Options), State), State).
 
 -spec(attribute(Attribute) -> io_lib:chars() when
       Attribute :: erl_parse:abstract_form()).
@@ -74,7 +75,8 @@ attribute(Thing) ->
       Options :: options()).
 
 attribute(Thing, Options) ->
-    frmt(lattribute(Thing, options(Options)), state(Options)).
+    State = state(Options),
+    frmt(lattribute(Thing, options(Options), State), State).
 
 -spec(function(Function) -> io_lib:chars() when
       Function :: erl_parse:abstract_form()).
@@ -180,11 +182,13 @@ state(_Hook) ->
 
 state() ->
     #pp{string_fun = fun io_lib:write_string_as_latin1/1,
-        char_fun   = fun io_lib:write_char_as_latin1/1}.
+        char_fun   = fun io_lib:write_char_as_latin1/1,
+        term_fun   = fun(T) -> io_lib:format("~p", [T]) end}.
 
 unicode_state() ->
     #pp{string_fun = fun io_lib:write_string/1,
-        char_fun   = fun io_lib:write_char/1}.
+        char_fun   = fun io_lib:write_char/1,
+        term_fun   = fun(T) -> io_lib:format("~tp", [T]) end}.
 
 encoding(Options) ->
     case proplists:get_value(encoding, Options, epp:default_encoding()) of
@@ -193,47 +197,47 @@ encoding(Options) ->
         unicode -> unicode
     end.
 
-lform({attribute,Line,Name,Arg}, Opts) ->
-    lattribute({attribute,Line,Name,Arg}, Opts);
-lform({function,Line,Name,Arity,Clauses}, Opts) ->
+lform({attribute,Line,Name,Arg}, Opts, State) ->
+    lattribute({attribute,Line,Name,Arg}, Opts, State);
+lform({function,Line,Name,Arity,Clauses}, Opts, _State) ->
     lfunction({function,Line,Name,Arity,Clauses}, Opts);
-lform({rule,Line,Name,Arity,Clauses}, Opts) ->
+lform({rule,Line,Name,Arity,Clauses}, Opts, _State) ->
     lrule({rule,Line,Name,Arity,Clauses}, Opts);
 %% These are specials to make it easier for the compiler.
-lform({error,E}, _Opts) ->
-    leaf(format("~p\n", [{error,E}]));
-lform({warning,W}, _Opts) ->
-    leaf(format("~p\n", [{warning,W}]));
-lform({eof,_Line}, _Opts) ->
+lform({error,E}, _Opts, State) ->
+    leaf((State#pp.term_fun)({error,E})++"\n");
+lform({warning,W}, _Opts, State) ->
+    leaf((State#pp.term_fun)({warning,W})++"\n");
+lform({eof,_Line}, _Opts, _State) ->
     $\n.
 
-lattribute({attribute,_Line,type,Type}, Opts) ->
+lattribute({attribute,_Line,type,Type}, Opts, _State) ->
     [typeattr(type, Type, Opts),leaf(".\n")];
-lattribute({attribute,_Line,opaque,Type}, Opts) ->
+lattribute({attribute,_Line,opaque,Type}, Opts, _State) ->
     [typeattr(opaque, Type, Opts),leaf(".\n")];
-lattribute({attribute,_Line,spec,Arg}, _Opts) ->
+lattribute({attribute,_Line,spec,Arg}, _Opts, _State) ->
     [specattr(Arg),leaf(".\n")];
-lattribute({attribute,_Line,Name,Arg}, Opts) ->
-    [lattribute(Name, Arg, Opts),leaf(".\n")].
+lattribute({attribute,_Line,Name,Arg}, Opts, State) ->
+    [lattribute(Name, Arg, Opts, State),leaf(".\n")].
 
-lattribute(module, {M,Vs}, _Opts) ->
+lattribute(module, {M,Vs}, _Opts, _State) ->
     attr("module",[{var,0,pname(M)},
                    foldr(fun(V, C) -> {cons,0,{var,0,V},C}
                          end, {nil,0}, Vs)]);
-lattribute(module, M, _Opts) ->
+lattribute(module, M, _Opts, _State) ->
     attr("module", [{var,0,pname(M)}]);
-lattribute(export, Falist, _Opts) ->
+lattribute(export, Falist, _Opts, _State) ->
     call({var,0,"-export"}, [falist(Falist)], 0, none);
-lattribute(import, Name, _Opts) when is_list(Name) ->
+lattribute(import, Name, _Opts, _State) when is_list(Name) ->
     attr("import", [{var,0,pname(Name)}]);
-lattribute(import, {From,Falist}, _Opts) ->
+lattribute(import, {From,Falist}, _Opts, _State) ->
     attr("import",[{var,0,pname(From)},falist(Falist)]);
-lattribute(file, {Name,Line}, _Opts) ->
-    attr("file", [{var,0,format("~p", [Name])},{integer,0,Line}]);
-lattribute(record, {Name,Is}, Opts) ->
+lattribute(file, {Name,Line}, _Opts, State) ->
+    attr("file", [{var,0,(State#pp.term_fun)(Name)},{integer,0,Line}]);
+lattribute(record, {Name,Is}, Opts, _State) ->
     Nl = leaf(format("-record(~w,", [Name])),
     [{first,Nl,record_fields(Is, Opts)},$)];
-lattribute(Name, Arg, #options{encoding = Encoding}) ->
+lattribute(Name, Arg, #options{encoding = Encoding}, _State) ->
     attr(write(Name), [erl_parse:abstract(Arg, [{encoding,Encoding}])]).
 
 typeattr(Tag, {TypeName,Type,Args}, _Opts) ->
@@ -423,7 +427,7 @@ lexpr(E, Opts) ->
 
 lexpr({var,_,V}, _, _) when is_integer(V) ->    %Special hack for Robert
     leaf(format("_~w", [V]));
-lexpr({var,_,V}, _, _) -> leaf(format("~s", [V]));
+lexpr({var,_,V}, _, _) -> leaf(format("~ts", [V]));
 lexpr({char,_,C}, _, _) -> {char,C};
 lexpr({integer,_,N}, _, _) -> leaf(write(N));
 lexpr({float,_,F}, _, _) -> leaf(write(F));
@@ -799,7 +803,7 @@ maybe_paren(_P, _Prec, Expr) ->
     Expr.
 
 leaf(S) ->
-    {leaf,iolist_size(S),S}.
+    {leaf,chars_size(S),S}.
 
 %%% Do the formatting. Currently nothing fancy. Could probably have
 %%% done it in one single pass.
@@ -1009,7 +1013,7 @@ incr(I, Incr) ->
     I+Incr.
 
 indentation(E, I) when I < 0 ->
-    iolist_size(E);
+    chars_size(E);
 indentation(E, I0) ->
     I = io_lib_format:indentation(E, I0),
     case has_nl(E) of
@@ -1064,6 +1068,15 @@ write_char(C, PP) ->
 %% Utilities
 %%
 
+chars_size([C | Es]) when is_integer(C) ->
+    1 + chars_size(Es);
+chars_size([E | Es]) ->
+    chars_size(E) + chars_size(Es);
+chars_size([]) ->
+    0;
+chars_size(B) when is_binary(B) ->
+    byte_size(B).
+
 -define(N_SPACES, 30).
 
 spacetab() ->
diff --git a/lib/syntax_tools/src/erl_comment_scan.erl b/lib/syntax_tools/src/erl_comment_scan.erl
index a70e7ba413..b949675f1c 100644
--- a/lib/syntax_tools/src/erl_comment_scan.erl
+++ b/lib/syntax_tools/src/erl_comment_scan.erl
@@ -282,7 +282,7 @@ join_lines([], Txt, L, Col, Ind) ->
 %% =====================================================================
 %% Utility functions for internal use
 
-filename([C|T]) when is_integer(C), C > 0, C =< 255 ->
+filename([C|T]) when is_integer(C), C > 0 ->
     [C | filename(T)];
 filename([]) ->
     [];
diff --git a/lib/syntax_tools/src/erl_tidy.erl b/lib/syntax_tools/src/erl_tidy.erl
index e9a88caff3..35fd0c2e02 100644
--- a/lib/syntax_tools/src/erl_tidy.erl
+++ b/lib/syntax_tools/src/erl_tidy.erl
@@ -435,7 +435,6 @@ file_type(Name, Links) ->
     end.
 
 open_output_file(FName, Options) ->
-io:format("Options ~p~n", [Options]),
     case catch file:open(FName, [write]++Options) of
         {ok, FD} ->
             FD;
@@ -1805,7 +1804,7 @@ get_free_vars_1([{free, B} | _Bs]) -> B;
 get_free_vars_1([_ | Bs]) -> get_free_vars_1(Bs);
 get_free_vars_1([]) -> [].
 
-filename([C | T]) when is_integer(C), C > 0, C =< 255 ->
+filename([C | T]) when is_integer(C), C > 0 ->
     [C | filename(T)];
 filename([H|T]) ->
     filename(H) ++ filename(T);
diff --git a/lib/syntax_tools/src/igor.erl b/lib/syntax_tools/src/igor.erl
index 8abc3f41cb..567bec00cb 100644
--- a/lib/syntax_tools/src/igor.erl
+++ b/lib/syntax_tools/src/igor.erl
@@ -2956,7 +2956,7 @@ timestamp() ->
 				"~2.2.0w:~2.2.0w:~2.2.0w.",
 				[Yr, Mth, Dy, Hr, Mt, Sc])).
 
-filename([C | T]) when is_integer(C), C > 0, C =< 255 ->
+filename([C | T]) when is_integer(C), C > 0 ->
     [C | filename(T)];
 filename([H|T]) ->
     filename(H) ++ filename(T);