Merge branch 'pan/unicode_filename_warnings'

* pan/unicode_filename_warnings:
  Update file_name_SUITE
  Fix broken file:read_link_all/1
  Fix broken prim_file:list_dir_all/1

5 files changed, 89 insertions, 39 deletions

diff --git a/erts/preloaded/ebin/prim_file.beam b/erts/preloaded/ebin/prim_file.beam
index f7b3aac376..b64fe522e8 100644
--- a/erts/preloaded/ebin/prim_file.beam
+++ b/erts/preloaded/ebin/prim_file.beam
diff --git a/erts/preloaded/src/prim_file.erl b/erts/preloaded/src/prim_file.erl
index bf8879c2a0..b40a6d9633 100644
--- a/erts/preloaded/src/prim_file.erl
+++ b/erts/preloaded/src/prim_file.erl
@@ -990,9 +990,9 @@ list_dir_convert_all([Name|Names]) ->
     %% a binary.
     case prim_file:internal_native2name(Name) of
 	{error, _} ->
-	    [Name|list_dir_convert(Names)];
+	    [Name|list_dir_convert_all(Names)];
 	Converted when is_list(Converted) ->
-	    [Converted|list_dir_convert(Names)]
+	    [Converted|list_dir_convert_all(Names)]
     end;
 list_dir_convert_all([]) -> [].
 
diff --git a/lib/kernel/src/file_server.erl b/lib/kernel/src/file_server.erl
index 73202319b9..6b413ff630 100644
--- a/lib/kernel/src/file_server.erl
+++ b/lib/kernel/src/file_server.erl
@@ -170,7 +170,7 @@ handle_call({read_link_info, Name, Opts}, _From, Handle) ->
 handle_call({read_link, Name}, _From, Handle) ->
     {reply, ?PRIM_FILE:read_link(Handle, Name), Handle};
 handle_call({read_link_all, Name}, _From, Handle) ->
-    {reply, ?PRIM_FILE:read_link(Handle, Name), Handle};
+    {reply, ?PRIM_FILE:read_link_all(Handle, Name), Handle};
 
 handle_call({make_link, Old, New}, _From, Handle) ->
     {reply, ?PRIM_FILE:make_link(Handle, Old, New), Handle};
diff --git a/lib/kernel/test/file_SUITE.erl b/lib/kernel/test/file_SUITE.erl
index fd4d5bd24e..eda901f3d6 100644
--- a/lib/kernel/test/file_SUITE.erl
+++ b/lib/kernel/test/file_SUITE.erl
@@ -45,7 +45,7 @@
 	 init_per_testcase/2, end_per_testcase/2,
 	 read_write_file/1, names/1]).
 -export([cur_dir_0/1, cur_dir_1/1, make_del_dir/1,
-	 list_dir/1,list_dir_error/1,
+	 list_dir/1,list_dir_error/1, untranslatable_names/1,
 	 pos1/1, pos2/1]).
 -export([close/1, consult1/1, path_consult/1, delete/1]).
 -export([ eval1/1, path_eval/1, script1/1, path_script/1,
@@ -117,7 +117,7 @@ all() ->
 
 groups() -> 
     [{dirs, [], [make_del_dir, cur_dir_0, cur_dir_1,
-		 list_dir, list_dir_error]},
+		 list_dir, list_dir_error, untranslatable_names]},
      {files, [],
       [{group, open}, {group, pos}, {group, file_info},
        {group, consult}, {group, eval}, {group, script},
@@ -557,6 +557,78 @@ list_dir_1(TestDir, Cnt, Sorted0) ->
     Sorted = lists:sort(DirList1),
     list_dir_1(TestDir, Cnt-1, Sorted).
 
+untranslatable_names(Config) ->
+    case no_untranslatable_names() of
+	true ->
+	    {skip,"Not a problem on this OS"};
+	false ->
+	    untranslatable_names_1(Config)
+    end.
+
+untranslatable_names_1(Config) ->
+    {ok,OldCwd} = file:get_cwd(),
+    PrivDir = ?config(priv_dir, Config),
+    Dir = filename:join(PrivDir, "untranslatable_names"),
+    ok = file:make_dir(Dir),
+    Node = start_node(untranslatable_names, "+fnu"),
+    try
+	ok = file:set_cwd(Dir),
+	[ok = file:write_file(F, F) || {_,F} <- untranslatable_names()],
+
+	ExpectedListDir0 = [unicode:characters_to_list(N, utf8) ||
+			       {utf8,N} <- untranslatable_names()],
+	ExpectedListDir = lists:sort(ExpectedListDir0),
+	io:format("ExpectedListDir: ~p\n", [ExpectedListDir]),
+	ExpectedListDir = call_and_sort(Node, file, list_dir, [Dir]),
+
+	ExpectedListDirAll0 = [case Enc of
+				   utf8 ->
+				       unicode:characters_to_list(N, utf8);
+				   latin1 ->
+				       N
+			       end || {Enc,N} <- untranslatable_names()],
+	ExpectedListDirAll = lists:sort(ExpectedListDirAll0),
+	io:format("ExpectedListDirAll: ~p\n", [ExpectedListDirAll]),
+	ExpectedListDirAll = call_and_sort(Node, file, list_dir_all, [Dir])
+    after
+	catch test_server:stop_node(Node),
+	file:set_cwd(OldCwd),
+	[file:delete(F) || {_,F} <- untranslatable_names()],
+	file:del_dir(Dir)
+    end,
+    ok.
+
+untranslatable_names() ->
+    [{utf8,<<"abc">>},
+     {utf8,<<"def">>},
+     {utf8,<<"Lagerl",195,182,"f">>},
+     {utf8,<<195,150,"stra Emterwik">>},
+     {latin1,<<"M",229,"rbacka">>},
+     {latin1,<<"V",228,"rmland">>}].
+
+call_and_sort(Node, M, F, A) ->
+    {ok,Res} = rpc:call(Node, M, F, A),
+    lists:sort(Res).
+
+no_untranslatable_names() ->
+    case os:type() of
+	{unix,darwin} -> true;
+	{win32,_} -> true;
+	_ -> false
+    end.
+
+start_node(Name, Args) ->
+    [_,Host] = string:tokens(atom_to_list(node()), "@"),
+    ct:log("Trying to start ~w@~s~n", [Name,Host]),
+    case test_server:start_node(Name, peer, [{args,Args}]) of
+	{error,Reason} ->
+	    test_server:fail(Reason);
+	{ok,Node} ->
+	    ct:log("Node ~p started~n", [Node]),
+	    Node
+    end.
+    
+    
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 
diff --git a/lib/kernel/test/file_name_SUITE.erl b/lib/kernel/test/file_name_SUITE.erl
index e8529db1dc..808a10ee27 100644
--- a/lib/kernel/test/file_name_SUITE.erl
+++ b/lib/kernel/test/file_name_SUITE.erl
@@ -340,9 +340,9 @@ check_icky(Mod) ->
 	?line true=(length("åäö") =:= 3),
 	?line UniMode = file:native_name_encoding() =/= latin1,
 	?line make_icky_dir(Mod),
- 	?line {ok, L0} = Mod:list_dir("."),
+ 	{ok, L0} = Mod:list_dir_all("."),
  	?line L1 = lists:sort(L0),
-	io:format("~p ~p~n",[L1,list(icky_dir())]),
+	io:format("~p~n~p~n~n",[L1,lists:sort(list(icky_dir()))]),
  	?line L1 = lists:sort(convlist(list(icky_dir()))),
  	?line {ok,D2} = Mod:get_cwd(),
  	?line true = is_list(D2),
@@ -357,7 +357,8 @@ check_icky(Mod) ->
  	?line Syms = [ {S,conv(Targ),list_to_binary(get_data(Targ,icky_dir()))} 
  		 || {T,S,Targ} <- icky_dir(), T =:= symlink ],
  	?line [ {ok, Cont} = Mod:read_file(SymL) || {SymL,_,Cont} <- Syms ],
- 	?line [ {ok, Targ} = fixlink(Mod:read_link(SymL)) || {SymL,Targ,_} <- Syms ],
+ 	[ {ok, Targ} = fixlink(Mod:read_link_all(SymL)) ||
+	    {SymL,Targ,_} <- Syms ],
         ?line chk_cre_dir(Mod,[{directory,"åäö_dir",icky_dir()}]),
  	?line {ok,BeginAt} = Mod:get_cwd(),
  	?line true = is_list(BeginAt),
@@ -369,7 +370,7 @@ check_icky(Mod) ->
  	?line ok = Mod:set_cwd(".."),
  	?line {ok,BeginAt} = Mod:get_cwd(),
         ?line rm_r2(Mod,"åäö_dir"),
-	{OS,TYPE} = os:type(),
+	{OS,_} = os:type(),
 	% Check that treat_icky really converts to the same as the OS
 	case UniMode of
 	    true ->
@@ -377,7 +378,7 @@ check_icky(Mod) ->
 		?line ok = Mod:set_cwd("åäö_dir"),
 		?line ok = Mod:write_file(<<"ååå">>,<<"hello">>),
 		?line Treated = treat_icky(<<"ååå">>),
-		?line {ok,[Treated]} = Mod:list_dir("."),
+		{ok,[Treated]} = Mod:list_dir_all("."),
 		?line ok = Mod:delete(<<"ååå">>),
 		?line {ok,[]} = Mod:list_dir("."),
 		?line ok = Mod:set_cwd(".."),
@@ -393,15 +394,7 @@ check_icky(Mod) ->
 	    true ->
 		ok
 	end,
-	?line ok = Mod:set_cwd(treat_icky(<<"åäö_dir">>)),
- 	?line {ok, NowAt2} = Mod:get_cwd(),
-	io:format("~p~n",[NowAt2]),
-	% Cannot create raw unicode-breaking filenames on windows or macos
- 	?line true = ((((not UniMode) or (OS =:= win32) or (TYPE=:=darwin)) and is_list(NowAt2)) orelse ((UniMode) and is_binary(NowAt2))),
- 	?line true = BeginAt =/= NowAt2,
- 	?line ok = Mod:set_cwd(".."),
  	?line {ok,BeginAt} = Mod:get_cwd(),
-        ?line rm_r2(Mod,conv(treat_icky(<<"åäö_dir">>))),
 	case has_links() of
 	    true ->
 		?line ok = Mod:make_link("fil1","nisseö"),
@@ -485,7 +478,7 @@ check_very_icky(Mod) ->
 		ok
 	end,
 	?line make_very_icky_dir(Mod),
- 	?line {ok, L0} = Mod:list_dir("."),
+ 	{ok, L0} = Mod:list_dir_all("."),
  	?line L1 = lists:sort(L0),
  	?line L1 = lists:sort(convlist(list(very_icky_dir()))),
  	?line {ok,D2} = Mod:get_cwd(),
@@ -494,7 +487,8 @@ check_very_icky(Mod) ->
  	?line Syms = [ {S,conv(Targ),list_to_binary(get_data(Targ,very_icky_dir()))} 
  		 || {T,S,Targ} <- very_icky_dir(), T =:= symlink ],
  	?line [ {ok, Cont} = Mod:read_file(SymL) || {SymL,_,Cont} <- Syms ],
- 	?line [ {ok, Targ} = fixlink(Mod:read_link(SymL)) || {SymL,Targ,_} <- Syms ],
+ 	?line [ {ok, Targ} = fixlink(Mod:read_link_all(SymL)) ||
+		  {SymL,Targ,_} <- Syms ],
  	?line chk_cre_dir(Mod,[{directory,[1088,1079,1091]++"_dir",very_icky_dir()}]),
  	?line {ok,BeginAt} = Mod:get_cwd(),
  	?line true = is_list(BeginAt),
@@ -559,22 +553,6 @@ check_very_icky(Mod) ->
 				       FI#file_info{mode = NewMode2}),
  	?line {ok,#file_info{mode = NewMode2}} = 
 	          Mod:read_file_info([956,965,963,954,959,49]),
-	?line NumOK0 = case has_links() of
-			  true -> 5;
-			  false -> 3
-		      end,
-	?line NumNOK0 = case has_links() of
-			   true -> 4;
-			   false -> 3
-		       end,
-	?line {NumOK,NumNOK} = case is_binary(treat_icky(<<"foo">>)) of
-				   false ->
-				       {NumOK0+NumNOK0,0};
-				   true ->
-				       {NumOK0,NumNOK0}
-			       end,
-	?line {NumOK,NumNOK} = filelib:fold_files(".",".*",true,fun(_F,{N,M}) when is_list(_F) ->  io:format("~ts~n",[_F]),{N+1,M}; (_F,{N,M}) ->  io:format("~p~n",[_F]),{N,M+1} end,{0,0}),
-	?line ok = filelib:fold_files(".",[1076,1089,1072,124,46,42],true,fun(_F,_) -> ok end,false),
 	ok
     catch
 	throw:need_unicode_mode ->
@@ -593,7 +571,7 @@ check_very_icky(Mod) ->
 rm_rf(Mod,Dir) ->
     case  Mod:read_link_info(Dir) of
 	{ok, #file_info{type = directory}} ->
-	    {ok, Content} = Mod:list_dir(Dir),
+	    {ok, Content} = Mod:list_dir_all(Dir),
 	    [ rm_rf(Mod,filename:join(Dir,C)) || C <- Content ],
 	    Mod:del_dir(Dir),
 	    ok;
@@ -608,7 +586,7 @@ rm_r(Mod,Dir) ->
     case  Mod:read_link_info(Dir) of
 	{ok, #file_info{type = directory}} ->
 	    {ok,#file_info{type = directory}} =  Mod:read_file_info(Dir),
-	    {ok, Content} = Mod:list_dir(Dir),
+	    {ok, Content} = Mod:list_dir_all(Dir),
 	    [ true = is_list(Part) || Part <- Content ],
 	    [ true = is_list(filename:join(Dir,Part)) || Part <- Content ],
 	    [ rm_r(Mod,filename:join(Dir,C)) || C <- Content ],
@@ -626,7 +604,7 @@ rm_r2(Mod,Dir) ->
     case  Mod:read_link_info(Dir) of
 	{ok, #file_info{type = directory}} ->
 	    {ok,#file_info{type = directory}} =  Mod:read_file_info(Dir),
-	    {ok, Content} = Mod:list_dir(Dir),
+	    {ok, Content} = Mod:list_dir_all(Dir),
 	    UniMode = file:native_name_encoding() =/= latin1,
 	    [ true = (is_list(Part) orelse UniMode) || Part <- Content ],
 	    [ true = (is_list(filename:join(Dir,Part)) orelse UniMode) || Part <- Content ],