[test_server] Add tests for unicode support

4 files changed, 333 insertions, 44 deletions

diff --git a/lib/test_server/test/test_server_SUITE.erl b/lib/test_server/test/test_server_SUITE.erl
index fb82a87fd0..bad35b2d51 100644
--- a/lib/test_server/test/test_server_SUITE.erl
+++ b/lib/test_server/test/test_server_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2010-2012. All Rights Reserved.
+%% Copyright Ericsson AB 2010-2013. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -31,6 +31,7 @@
 
 -include_lib("common_test/include/ct.hrl").
 -include("test_server_test_lib.hrl").
+-include_lib("kernel/include/file.hrl").
 
 %%--------------------------------------------------------------------
 %% COMMON TEST CALLBACK FUNCTIONS
@@ -68,6 +69,13 @@ init_per_testcase(_TestCase, Config) ->
 
 %% @spec end_per_testcase(TestCase, Config0) ->
 %%               void() | {save_config,Config1} | {fail,Reason}
+end_per_testcase(test_server_unicode, _Config) ->
+    [_,Host] = string:tokens(atom_to_list(node()), "@"),
+    N1 = list_to_atom("test_server_tester_latin1" ++ "@" ++ Host),
+    N2 = list_to_atom("test_server_tester_utf8" ++ "@" ++ Host),
+    test_server:stop_node(N1),
+    test_server:stop_node(N2),
+    ok;
 end_per_testcase(_TestCase, _Config) ->
     ok.
 
@@ -80,7 +88,8 @@ all() ->
     [test_server_SUITE, test_server_parallel01_SUITE,
      test_server_conf02_SUITE, test_server_conf01_SUITE,
      test_server_skip_SUITE, test_server_shuffle01_SUITE,
-     test_server_break_SUITE, test_server_cover_SUITE].
+     test_server_break_SUITE, test_server_cover_SUITE,
+     test_server_unicode].
 
 
 %%--------------------------------------------------------------------
@@ -171,7 +180,23 @@ test_server_cover_SUITE(Config) ->
 	    ok
     end.
 
+test_server_unicode(Config) ->
+    run_test_server_tests("test_server_unicode_SUITE", [],
+			  5, 0, 3, 3, 0, 0, 0, 0, 5, Config),
 
+    %% Create and run two test suites - one with filename and content
+    %% in latin1 (not on windows) and one with filename and content in
+    %% utf8.  Both have name and content including letters ���.  Check
+    %% that all logs are generated with utf8 encoded filenames.
+    case os:type() of
+	{win32,_} ->
+	    ok;
+	_ ->
+	    generate_and_run_unicode_test(Config,latin1)
+    end,
+    generate_and_run_unicode_test(Config,utf8).
+
+%%%-----------------------------------------------------------------
 run_test_server_tests(SuiteName, Skip, NCases, NFail, NExpected, NSucc,
 		      NUsrSkip, NAutoSkip, 
 		      NActualSkip, NActualFail, NActualSucc, Config) ->
@@ -182,12 +207,13 @@ run_test_server_tests(SuiteName, Skip, NCases, NFail, NExpected, NSucc,
 run_test_server_tests(SuiteName, Skip, NCases, NFail, NExpected, NSucc,
 		      NUsrSkip, NAutoSkip,
 		      NActualSkip, NActualFail, NActualSucc, Cover, Config) ->
-
+    Node = proplists:get_value(node, Config),
+    Encoding = rpc:call(Node,file,native_name_encoding,[]),
     WorkDir = proplists:get_value(work_dir, Config),
-    ct:log("<a href=\"file://~s\">Test case log files</a>\n",
-	   [filename:join(WorkDir, SuiteName++".logs")]),
+    LogDir = filename:join(WorkDir, SuiteName++".logs"),
+    LogDirUri = test_server_ctrl:uri_encode(LogDir, Encoding),
+    ct:log("<a href=\"file://~s\">Test case log files</a>\n", [LogDirUri]),
 
-    Node = proplists:get_value(node, Config),
     {ok,_Pid} = rpc:call(Node,test_server_ctrl, start, []),
     case Cover of
 	false ->
@@ -207,12 +233,10 @@ run_test_server_tests(SuiteName, Skip, NCases, NFail, NExpected, NSucc,
     
     rpc:call(Node,test_server_ctrl, stop, []),
 
-    {ok,Data} =	test_server_test_lib:parse_suite(
-		  lists:last(
-		    lists:sort(
-		      filelib:wildcard(
-			filename:join([WorkDir,SuiteName++".logs",
-				       "run*","suite.log"]))))),
+    LogDir1 = translate_filename(LogDir,Encoding),
+    LastRunDir = get_latest_run_dir(LogDir1),
+    LastSuiteLog = filename:join(LastRunDir,"suite.log"),
+    {ok,Data} =	test_server_test_lib:parse_suite(LastSuiteLog),
     check([{"Number of cases",NCases,Data#suite.n_cases},
 	   {"Number failed",NFail,Data#suite.n_cases_failed},
 	   {"Number expected",NExpected,Data#suite.n_cases_expected},
@@ -229,6 +253,47 @@ run_test_server_tests(SuiteName, Skip, NCases, NFail, NExpected, NSucc,
 			  end,{0,0,0},Data#suite.cases),
     Data.
 
+translate_filename(Filename,EncodingOnTestNode) ->
+    case {file:native_name_encoding(),EncodingOnTestNode} of
+	{X,X} -> Filename;
+	{utf8,latin1} -> list_to_binary(Filename);
+	{latin1,utf8} -> unicode:characters_to_binary(Filename)
+    end.
+
+get_latest_run_dir(Dir) ->
+    %% For the time being, filelib:wildcard can not take a binary
+    %% argument, so we avoid using this here.
+    case file:list_dir(Dir) of
+	{ok,Files} ->
+	    {ok,RE} = re:compile(<<"^run.[1-2][-_\.0-9]*$">>),
+	    RunDirs = lists:filter(
+			fun(F) ->
+				L = l(F),
+				case re:run(F,RE) of
+				    {match,[{0,L}]} -> true;
+				    _ -> false
+				end
+			end, Files),
+	    case RunDirs of
+		[] ->
+		    Dir;
+		[H|T] ->
+		    filename:join(Dir,get_latest_dir(T,H))
+	    end;
+	_ ->
+	    Dir
+    end.
+
+l(X) when is_binary(X) -> size(X);
+l(X) when is_list(X) -> length(X).
+
+get_latest_dir([H|T],Latest) when H>Latest ->
+    get_latest_dir(T,H);
+get_latest_dir([_|T],Latest) ->
+    get_latest_dir(T,Latest);
+get_latest_dir([],Latest) ->
+    Latest.
+
 check([{Str,Same,Same}|T], Status) ->
     io:format("~s: ~p\n", [Str,Same]),
     check(T, Status);
@@ -246,4 +311,139 @@ until(Fun) ->
 	    timer:sleep(100),
 	    until(Fun)
     end.
-	  
+
+generate_and_run_unicode_test(Config0,Encoding) ->
+    DataDir = ?config(data_dir,Config0),
+    Suite = create_unicode_test_suite(DataDir,Encoding),
+
+    %% We can not run this test on default node since it must be
+    %% started with correct file name mode (+fnu/+fnl).
+    %% OBS: the node are stopped by end_per_testcase/2
+    Config1 = lists:keydelete(node,1,Config0),
+    Config2 = lists:keydelete(work_dir,1,Config1),
+    NodeName = list_to_atom("test_server_tester_" ++ atom_to_list(Encoding)),
+    ErtsSwitch = case Encoding of
+		     latin1 -> "+fnl";
+		     utf8 -> "+fnu"
+		 end,
+    Config = start_node(Config2,NodeName,ErtsSwitch),
+
+    %% Compile the suite
+    Node = proplists:get_value(node,Config),
+    {ok,Mod} = rpc:call(Node,compile,file,[Suite,[{outdir,DataDir}]]),
+    ModStr = atom_to_list(Mod),
+
+    %% Clean logdir
+    LogDir0 = filename:join(DataDir,ModStr++".logs"),
+    LogDir = translate_filename(LogDir0,Encoding),
+    rm_dir(LogDir),
+
+    %% Run the test
+    run_test_server_tests(ModStr, [], 3, 0, 1, 1, 0, 0, 0, 0, 3, Config),
+
+    %% Check that all logs are created with utf8 encoded filenames
+    true = filelib:is_dir(LogDir),
+
+    RunDir = get_latest_run_dir(LogDir),
+    true = filelib:is_dir(RunDir),
+
+    LowerModStr = string:to_lower(ModStr),
+    SuiteHtml = translate_filename(LowerModStr++".src.html",Encoding),
+    true = filelib:is_regular(filename:join(RunDir,SuiteHtml)),
+
+    TCLog = translate_filename(LowerModStr++".tc_���.html",Encoding),
+    true = filelib:is_regular(filename:join(RunDir,TCLog)),
+    ok.
+
+%% Same as test_server_test_lib:start_slave, but starts a peer with
+%% additional arguments.
+%% The reason for this is that we need to start nodes with +fnu/+fnl,
+%% and that will not work well with a slave node since slave nodes run
+%% remote file system on master - i.e. they will use same file name
+%% mode as the master.
+start_node(Config,Name,Args) ->
+    [_,Host] = string:tokens(atom_to_list(node()), "@"),
+    ct:log("Trying to start ~w@~s~n",[Name,Host]),
+    case test_server:start_node(Name, peer, [{args,Args}]) of
+	{error,Reason} ->
+	    test_server:fail(Reason);
+	{ok,Node} ->
+	    ct:log("Node ~p started~n", [Node]),
+	    test_server_test_lib:prepare_tester_node(Node,Config)
+    end.
+
+create_unicode_test_suite(Dir,Encoding) ->
+    ModStr = "test_server_"++atom_to_list(Encoding)++"_���_SUITE",
+    File = filename:join(Dir,ModStr++".erl"),
+    Suite =
+	["%% -*- ",epp:encoding_to_string(Encoding)," -*-\n",
+	 "-module(",ModStr,").\n"
+	 "\n"
+	 "-export([all/1, init_per_suite/1, end_per_suite/1]).\n"
+	 "-export([init_per_testcase/2, end_per_testcase/2]).\n"
+	 "-export([tc_���/1]).\n"
+	 "\n"
+	 "-include_lib(\"test_server/include/test_server.hrl\").\n"
+	 "\n"
+	 "all(suite) ->\n"
+	 "    [tc_���].\n"
+	 "\n"
+	 "init_per_suite(Config) ->\n"
+	 "    Config.\n"
+	 "\n"
+	 "end_per_suite(_Config) ->\n"
+	 "    ok.\n"
+	 "\n"
+	 "init_per_testcase(_Case,Config) ->\n"
+	 "    init_timetrap(500,Config).\n"
+	 "\n"
+	 "init_timetrap(T,Config) ->\n"
+	 "    Dog = ?t:timetrap(T),\n"
+	 "    [{watchdog, Dog}|Config].\n"
+	 "\n"
+	 "end_per_testcase(_Case,Config) ->\n"
+	 "    cancel_timetrap(Config).\n"
+	 "\n"
+	 "cancel_timetrap(Config) ->\n"
+	 "    Dog=?config(watchdog, Config),\n"
+	 "    ?t:timetrap_cancel(Dog),\n"
+	 "    ok.\n"
+	 "\n"
+	 "tc_���(Config) when is_list(Config) ->\n"
+	 "    true = filelib:is_dir(?config(priv_dir,Config)),\n"
+	 "    ok.\n"],
+    {ok,Fd} = file:open(raw_filename(File,Encoding),[write,{encoding,Encoding}]),
+    io:put_chars(Fd,Suite),
+    ok = file:close(Fd),
+    File.
+
+raw_filename(Name,latin1) -> list_to_binary(Name);
+raw_filename(Name,utf8)   -> unicode:characters_to_binary(Name).
+
+rm_dir(Dir) ->
+    case file:list_dir(Dir) of
+	{error,enoent} ->
+	    ok;
+	{ok,Files} ->
+	    rm_files([filename:join(Dir, F) || F <- Files]),
+	    file:del_dir(Dir)
+    end.
+
+rm_files([F | Fs]) ->
+    case file:read_file_info(F) of
+	{ok,#file_info{type=directory}} ->
+	    rm_dir(F),
+	    rm_files(Fs);
+	{ok,_Regular} ->
+	    case file:delete(F) of
+		ok ->
+		    rm_files(Fs);
+		{error,Errno} ->
+		    exit({del_failed,F,Errno})
+	    end
+    end;
+rm_files([]) ->
+    ok.
+
+erts_switch(latin1) -> "+fnl";
+erts_switch(utf8)   -> "+fnu".
diff --git a/lib/test_server/test/test_server_SUITE_data/Makefile.src b/lib/test_server/test/test_server_SUITE_data/Makefile.src
index c770627f04..5aeb035572 100644
--- a/lib/test_server/test/test_server_SUITE_data/Makefile.src
+++ b/lib/test_server/test/test_server_SUITE_data/Makefile.src
@@ -7,4 +7,5 @@ all:
 	erlc test_server_skip_SUITE.erl
 	erlc test_server_break_SUITE.erl
 	erlc test_server_cover_SUITE.erl
-	erlc +debug_info test_server_cover_SUITE_data/cover_helper.erl
\ No newline at end of file
+	erlc +debug_info test_server_cover_SUITE_data/cover_helper.erl
+	erlc test_server_unicode_SUITE.erl
diff --git a/lib/test_server/test/test_server_SUITE_data/test_server_unicode_SUITE.erl b/lib/test_server/test/test_server_SUITE_data/test_server_unicode_SUITE.erl
new file mode 100644
index 0000000000..efdc626c24
--- /dev/null
+++ b/lib/test_server/test/test_server_SUITE_data/test_server_unicode_SUITE.erl
@@ -0,0 +1,82 @@
+%% -*- coding: utf-8 -*-
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2013. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+-module(test_server_unicode_SUITE).
+
+-export([all/1, init_per_suite/1, end_per_suite/1]).
+-export([init_per_testcase/2, end_per_testcase/2]).
+-export(['#difficult_case_name_äöå'/1,
+	 print_and_log_unicode/1,
+	 print_and_log_latin1/1]).
+
+-include_lib("test_server/include/test_server.hrl").
+
+all(suite) ->
+    ['#difficult_case_name_äöå',
+     print_and_log_unicode,
+     print_and_log_latin1].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_testcase(_Case,Config) ->
+    init_timetrap(500,Config).
+
+init_timetrap(T,Config) ->
+    Dog = ?t:timetrap(T),
+    [{watchdog, Dog}|Config].
+
+end_per_testcase(_Case,Config) ->
+    cancel_timetrap(Config).
+
+cancel_timetrap(Config) ->
+    Dog=?config(watchdog, Config),
+    ?t:timetrap_cancel(Dog),
+    ok.
+
+
+%%%-----------------------------------------------------------------
+%%% Test cases
+
+'#difficult_case_name_äöå'(Config) when is_list(Config) ->
+    ok.
+
+print_and_log_unicode(Config) when is_list(Config) ->
+    String = "שלום-שלום+של 日本語",
+    test_server:comment(String),
+    test_server:capture_start(),
+    io:format("String with ts: ~ts",[String]),
+    test_server:capture_stop(),
+    "String with ts: "++String = lists:flatten(test_server:capture_get()),
+    ok.
+
+print_and_log_latin1(Config) when is_list(Config) ->
+    String = "æøå",
+    test_server:comment(String),
+    test_server:capture_start(),
+    io:format("String with s: ~s",[String]),
+    io:format("String with ts: ~ts",[String]),
+    test_server:capture_stop(),
+    ["String with s: "++String,
+     "String with ts: "++String] =
+	[lists:flatten(L) || L<- test_server:capture_get()],
+    ok.
diff --git a/lib/test_server/test/test_server_test_lib.erl b/lib/test_server/test/test_server_test_lib.erl
index d466aa0110..cd6804f7ad 100644
--- a/lib/test_server/test/test_server_test_lib.erl
+++ b/lib/test_server/test/test_server_test_lib.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2009-2012. All Rights Reserved.
+%% Copyright Ericsson AB 2009-2013. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -20,6 +20,9 @@
 -export([parse_suite/1]).
 -export([init/2, pre_init_per_testcase/3, post_end_per_testcase/4]).
 
+%% for test_server_SUITE when node can not be started as slave
+-export([prepare_tester_node/2]).
+
 -include("test_server_test_lib.hrl").
 
 %% The CTH hooks all tests
@@ -49,38 +52,41 @@ start_slave(Config,_Level) ->
 	    ct:log("Node ~p started~n", [Node]),
 	    IsCover = test_server:is_cover(),
 	    if IsCover ->
-		cover:start(Node);
-	    true->
-		ok
+		    cover:start(Node);
+	       true->
+		    ok
 	    end,
-	    DataDir = proplists:get_value(data_dir, Config),
-	    %% We would normally use priv_dir for temporary data,
-	    %% but the pathnames gets too long on Windows.
-	    %% Until the run-time system can support long pathnames,
-	    %% use the data dir.
-	    WorkDir = DataDir,
-
-	    %% WorkDir as well as directory of Test Server suites
-	    %% have to be in code path on Test Server node.
-	    [_ | Parts] = lists:reverse(filename:split(DataDir)),
-	    TSDir = filename:join(lists:reverse(Parts)),
-	    AddPathDirs = case proplists:get_value(path_dirs, Config) of
-			      undefined -> [];
-			      Ds -> Ds
-			  end,
-	    PathDirs = [WorkDir,TSDir | AddPathDirs],
-	    [true = rpc:call(Node, code, add_patha, [D]) || D <- PathDirs],
-	    io:format("Dirs added to code path (on ~w):~n",
-		      [Node]),
-	    [io:format("~s~n", [D]) || D <- PathDirs],
-
-	    true = rpc:call(Node, os, putenv, 
-			    ["TEST_SERVER_FRAMEWORK", "undefined"]),
-
-	    ok = rpc:call(Node, file, set_cwd, [WorkDir]),
-	    [{node,Node}, {work_dir,WorkDir} | Config]
+	    prepare_tester_node(Node,Config)
     end.
 
+prepare_tester_node(Node,Config) ->
+    DataDir = proplists:get_value(data_dir, Config),
+    %% We would normally use priv_dir for temporary data,
+    %% but the pathnames gets too long on Windows.
+    %% Until the run-time system can support long pathnames,
+    %% use the data dir.
+    WorkDir = DataDir,
+
+    %% WorkDir as well as directory of Test Server suites
+    %% have to be in code path on Test Server node.
+    [_ | Parts] = lists:reverse(filename:split(DataDir)),
+    TSDir = filename:join(lists:reverse(Parts)),
+    AddPathDirs = case proplists:get_value(path_dirs, Config) of
+		      undefined -> [];
+		      Ds -> Ds
+		  end,
+    PathDirs = [WorkDir,TSDir | AddPathDirs],
+    [true = rpc:call(Node, code, add_patha, [D]) || D <- PathDirs],
+    io:format("Dirs added to code path (on ~w):~n",
+	      [Node]),
+    [io:format("~s~n", [D]) || D <- PathDirs],
+
+    true = rpc:call(Node, os, putenv,
+		    ["TEST_SERVER_FRAMEWORK", "undefined"]),
+
+    ok = rpc:call(Node, file, set_cwd, [WorkDir]),
+    [{node,Node}, {work_dir,WorkDir} | Config].
+
 post_end_per_testcase(_TC, Config, Return, State) ->
     Node = proplists:get_value(node, Config),
     Cover = test_server:is_cover(),