OTP-11660: make eunit unicode safe

All output from eunit is unicode, including the surefire XML files.

7 files changed, 60 insertions, 48 deletions

diff --git a/lib/eunit/src/eunit.erl b/lib/eunit/src/eunit.erl
index 9c589dfa86..fbfd123c43 100644
--- a/lib/eunit/src/eunit.erl
+++ b/lib/eunit/src/eunit.erl
@@ -231,7 +231,7 @@ event_logger(LogFile) ->
 event_logger_loop(Reference, FD) ->
     receive
 	{status, _Id, _Info}=Msg ->
-	    io:fwrite(FD, "~p.\n", [Msg]),
+	    io:fwrite(FD, "~tp.\n", [Msg]),
 	    event_logger_loop(Reference, FD);
 	{stop, Reference, _ReplyTo} ->
 	    %% no need to reply, just exit
diff --git a/lib/eunit/src/eunit_data.erl b/lib/eunit/src/eunit_data.erl
index cbbc6fbc15..8b53a3681d 100644
--- a/lib/eunit/src/eunit_data.erl
+++ b/lib/eunit/src/eunit_data.erl
@@ -391,7 +391,7 @@ parse({with, X, As}=T) when is_list(As) ->
 parse({S, T1} = T) when is_list(S) ->
     case eunit_lib:is_string(S) of
 	true ->
-	    group(#group{tests = T1, desc = list_to_binary(S)});
+	    group(#group{tests = T1, desc = unicode:characters_to_binary(S)});
 	false ->
 	    bad_test(T)
     end;
diff --git a/lib/eunit/src/eunit_internal.hrl b/lib/eunit/src/eunit_internal.hrl
index 92694ec39b..8e1e27811f 100644
--- a/lib/eunit/src/eunit_internal.hrl
+++ b/lib/eunit/src/eunit_internal.hrl
@@ -14,8 +14,8 @@
 -define(DEFAULT_MODULE_WRAPPER_NAME, eunit_wrapper_).
 
 -ifdef(DEBUG).
--define(debugmsg(S),io:fwrite("\n* ~s: ~s\n", [?MODULE,S])).
--define(debugmsg1(S,As),io:fwrite("\n* ~s: " ++ S ++ "\n", [?MODULE] ++ As)).
+-define(debugmsg(S),io:fwrite("\n* ~ts: ~ts\n", [?MODULE,S])).
+-define(debugmsg1(S,As),io:fwrite("\n* ~ts: " ++ S ++ "\n", [?MODULE] ++ As)).
 -else.
 -define(debugmsg(S),ok).
 -define(debugmsg1(S,As),ok).
diff --git a/lib/eunit/src/eunit_lib.erl b/lib/eunit/src/eunit_lib.erl
index 40bae93298..d8f98cffa5 100644
--- a/lib/eunit/src/eunit_lib.erl
+++ b/lib/eunit/src/eunit_lib.erl
@@ -57,7 +57,7 @@ format_exception({Class,Term,Trace}, Depth)
   when is_atom(Class), is_list(Trace) ->
     case is_stacktrace(Trace) of
 	true ->
-	    io_lib:format("~s**~w:~s",
+	    io_lib:format("~ts**~w:~ts",
 			  [format_stacktrace(Trace), Class,
                            format_term(Term, Depth)]);
 	false ->
@@ -67,11 +67,11 @@ format_exception(Term, Depth) ->
     format_term(Term, Depth).
 
 format_term(Term, Depth) ->
-    io_lib:format("~P\n", [Term, Depth]).
+    io_lib:format("~tP\n", [Term, Depth]).
 
 format_exit_term(Term) ->
     {Reason, Trace} = analyze_exit_term(Term),
-    io_lib:format("~P~s", [Reason, 15, Trace]).
+    io_lib:format("~tP~ts", [Reason, 15, Trace]).
 
 analyze_exit_term({Reason, [_|_]=Trace}=Term) ->
     case is_stacktrace(Trace) of
@@ -102,7 +102,7 @@ format_stacktrace(Trace) ->
     format_stacktrace(Trace, "in function", "in call from").
 
 format_stacktrace([{M,F,A,L}|Fs], Pre, Pre1) when is_integer(A) ->
-    [io_lib:fwrite("~s ~w:~w/~w~s\n",
+    [io_lib:fwrite("~ts ~w:~w/~w~ts\n",
                    [Pre, M, F, A, format_stacktrace_location(L)])
      | format_stacktrace(Fs, Pre1, Pre1)];
 format_stacktrace([{M,F,As,L}|Fs], Pre, Pre1) when is_list(As) ->
@@ -110,15 +110,15 @@ format_stacktrace([{M,F,As,L}|Fs], Pre, Pre1) when is_list(As) ->
     C = case is_op(M,F,A) of
 	    true when A =:= 1 ->
 		[A1] = As,
-		io_lib:fwrite("~s ~s", [F,format_arg(A1)]);
+		io_lib:fwrite("~ts ~ts", [F,format_arg(A1)]);
 	    true when A =:= 2 ->
 		[A1, A2] = As,
-		io_lib:fwrite("~s ~s ~s",
+		io_lib:fwrite("~ts ~ts ~ts",
 			      [format_arg(A1),F,format_arg(A2)]);
 	    false ->
-		io_lib:fwrite("~w(~s)", [F,format_arglist(As)])
+		io_lib:fwrite("~w(~ts)", [F,format_arglist(As)])
 	end,
-    [io_lib:fwrite("~s ~w:~w/~w~s\n  called as ~s\n",
+    [io_lib:fwrite("~ts ~w:~w/~w~ts\n  called as ~ts\n",
 		   [Pre,M,F,A,format_stacktrace_location(L),C])
      | format_stacktrace(Fs,Pre1,Pre1)];
 format_stacktrace([{M,F,As}|Fs], Pre, Pre1) ->
@@ -130,18 +130,18 @@ format_stacktrace_location(Location) ->
     File = proplists:get_value(file, Location),
     Line = proplists:get_value(line, Location),
     if File =/= undefined, Line =/= undefined ->
-            io_lib:format(" (~s, line ~w)", [File, Line]);
+            io_lib:format(" (~ts, line ~w)", [File, Line]);
        true ->
             ""
     end.
 
 format_arg(A) ->
-    io_lib:format("~P",[A,15]).
+    io_lib:format("~tP",[A,15]).
 
 format_arglist([A]) ->
     format_arg(A);
 format_arglist([A|As]) ->
-    [io_lib:format("~P,",[A,15]) | format_arglist(As)];
+    [io_lib:format("~tP,",[A,15]) | format_arglist(As)];
 format_arglist([]) ->
     "".
 
@@ -155,41 +155,41 @@ is_op(_M, _F, _A) ->
     false.
 
 format_error({bad_test, Term}) ->
-    error_msg("bad test descriptor", "~P", [Term, 15]);
+    error_msg("bad test descriptor", "~tP", [Term, 15]);
 format_error({bad_generator, {{M,F,A}, Term}}) ->
     error_msg(io_lib:format("result from generator ~w:~w/~w is not a test",
                             [M,F,A]),
-              "~P", [Term, 15]);
+              "~tP", [Term, 15]);
 format_error({generator_failed, {{M,F,A}, Exception}}) ->
     error_msg(io_lib:format("test generator ~w:~w/~w failed",[M,F,A]),
-              "~s", [format_exception(Exception)]);
+              "~ts", [format_exception(Exception)]);
 format_error({no_such_function, {M,F,A}})
   when is_atom(M), is_atom(F), is_integer(A) ->
     error_msg(io_lib:format("no such function: ~w:~w/~w", [M,F,A]),
 	      "", []);
 format_error({module_not_found, M}) ->
-    error_msg("test module not found", "~p", [M]);
+    error_msg("test module not found", "~tp", [M]);
 format_error({application_not_found, A}) when is_atom(A) ->
     error_msg("application not found", "~w", [A]);
 format_error({file_read_error, {_R, Msg, F}}) ->
-    error_msg("error reading file", "~s: ~s", [Msg, F]);
+    error_msg("error reading file", "~ts: ~ts", [Msg, F]);
 format_error({setup_failed, Exception}) ->
-    error_msg("context setup failed", "~s",
+    error_msg("context setup failed", "~ts",
 	      [format_exception(Exception)]);
 format_error({cleanup_failed, Exception}) ->
-    error_msg("context cleanup failed", "~s",
+    error_msg("context cleanup failed", "~ts",
 	      [format_exception(Exception)]);
 format_error({{bad_instantiator, {{M,F,A}, Term}}, _DummyException}) ->
     error_msg(io_lib:format("result from instantiator ~w:~w/~w is not a test",
                             [M,F,A]),
-              "~P", [Term, 15]);
+              "~tP", [Term, 15]);
 format_error({instantiation_failed, Exception}) ->
-    error_msg("instantiation of subtests failed", "~s",
+    error_msg("instantiation of subtests failed", "~ts",
 	      [format_exception(Exception)]).
 
 error_msg(Title, Fmt, Args) ->
     Msg = io_lib:format("**"++Fmt, Args),    % gets indentation right
-    io_lib:fwrite("*** ~s ***\n~s\n\n", [Title, Msg]).
+    io_lib:fwrite("*** ~ts ***\n~ts\n\n", [Title, Msg]).
 
 -ifdef(TEST).
 format_exception_test_() ->
diff --git a/lib/eunit/src/eunit_proc.erl b/lib/eunit/src/eunit_proc.erl
index 03d1a18321..98ae31d54b 100644
--- a/lib/eunit/src/eunit_proc.erl
+++ b/lib/eunit/src/eunit_proc.erl
@@ -230,7 +230,7 @@ insulator_wait(Child, Parent, Buf, St) ->
 	    message_super(Id, {progress, 'begin', {Type, Data}}, St),
 	    insulator_wait(Child, Parent, [[] | Buf], St);
 	{child, Child, Id, {'end', Status, Time}} ->
-	    Data = [{time, Time}, {output, buffer_to_binary(hd(Buf))}],
+	    Data = [{time, Time}, {output, lists:reverse(hd(Buf))}],
 	    message_super(Id, {progress, 'end', {Status, Data}}, St),
 	    insulator_wait(Child, Parent, tl(Buf), St);
 	{child, Child, Id, {skipped, Reason}} ->
@@ -272,9 +272,6 @@ kill_task(Child, St) ->
     exit(Child, kill),
     terminate_insulator(St).
 
-buffer_to_binary([B]) when is_binary(B) -> B;  % avoid unnecessary copying
-buffer_to_binary(Buf) -> list_to_binary(lists:reverse(Buf)).
-
 %% Unlinking before exit avoids polluting the parent process with exit
 %% signals from the insulator. The child process is already dead here.
 
@@ -597,7 +594,7 @@ group_leader_loop(Runner, Wait, Buf) ->
 	    %% no more messages and nothing to wait for; we ought to
 	    %% have collected all immediately pending output now
 	    process_flag(priority, normal),
-	    Runner ! {self(), buffer_to_binary(Buf)}
+	    Runner ! {self(), lists:reverse(Buf)}
     end.
 
 group_leader_sync(G) ->
diff --git a/lib/eunit/src/eunit_surefire.erl b/lib/eunit/src/eunit_surefire.erl
index 2d1f0b1497..d6684f33cb 100644
--- a/lib/eunit/src/eunit_surefire.erl
+++ b/lib/eunit/src/eunit_surefire.erl
@@ -206,6 +206,7 @@ handle_cancel(test, Data, St) ->
 format_name({Module, Function, Arity}, Line) ->
     lists:flatten([atom_to_list(Module), ":", atom_to_list(Function), "/",
 		   integer_to_list(Arity), "_", integer_to_list(Line)]).
+
 format_desc(undefined) ->
     "";
 format_desc(Desc) when is_binary(Desc) ->
@@ -279,7 +280,7 @@ write_report_to(TestSuite, FileDescriptor) ->
 %% Write the XML header.
 %% ----------------------------------------------------------------------------
 write_header(FileDescriptor) ->
-    file:write(FileDescriptor, [<<"<?xml version=\"1.0\" encoding=\"UTF-8\" ?>">>, ?NEWLINE]).
+    io:format(FileDescriptor, "~ts~ts", [<<"<?xml version=\"1.0\" encoding=\"UTF-8\" ?>">>, ?NEWLINE]).
 
 %% ----------------------------------------------------------------------------
 %% Write the testsuite start tag, with attributes describing the statistics
@@ -303,7 +304,7 @@ write_start_tag(
         <<"\" time=\"">>, format_time(Time),
         <<"\" name=\"">>, escape_attr(Name),
         <<"\">">>, ?NEWLINE],
-    file:write(FileDescriptor, StartTag).
+    io:format(FileDescriptor, "~ts", [StartTag]).
 
 %% ----------------------------------------------------------------------------
 %% Recursive function to write the test cases.
@@ -317,7 +318,7 @@ write_testcases([TestCase| Tail], FileDescriptor) ->
 %% Write the testsuite end tag.
 %% ----------------------------------------------------------------------------
 write_end_tag(FileDescriptor) ->
-    file:write(FileDescriptor, [<<"</testsuite>">>, ?NEWLINE]).
+    io:format(FileDescriptor, "~ts~ts", [<<"</testsuite>">>, ?NEWLINE]).
 
 %% ----------------------------------------------------------------------------
 %% Write a test case, as a testcase tag.
@@ -344,7 +345,7 @@ write_testcase(
         {ok, <<>>} -> [<<"/>">>, ?NEWLINE];
         _ -> [<<">">>, ?NEWLINE, format_testcase_result(Result), format_testcase_output(Output), ?INDENT, <<"</testcase>">>, ?NEWLINE]
     end,
-    file:write(FileDescriptor, [StartTag, ContentAndEndTag]).
+    io:format(FileDescriptor, "~ts~ts", [StartTag, ContentAndEndTag]).
 
 %% ----------------------------------------------------------------------------
 %% Format the result of the test.
@@ -427,7 +428,7 @@ escape_suitename([Char | Tail], Acc) -> escape_suitename(Tail, [Char | Acc]).
 %% Replace < with &lt;, > with &gt; and & with &amp;
 %% ----------------------------------------------------------------------------
 escape_text(Text) when is_binary(Text) -> escape_text(binary_to_list(Text));
-escape_text(Text) -> escape_xml(lists:flatten(Text), [], false).
+escape_text(Text) -> escape_xml(to_utf8(lists:flatten(Text)), [], false).
 
 
 %% ----------------------------------------------------------------------------
@@ -435,7 +436,7 @@ escape_text(Text) -> escape_xml(lists:flatten(Text), [], false).
 %% Replace < with &lt;, > with &gt; and & with &amp;
 %% ----------------------------------------------------------------------------
 escape_attr(Text) when is_binary(Text) -> escape_attr(binary_to_list(Text));
-escape_attr(Text) -> escape_xml(lists:flatten(Text), [], true).
+escape_attr(Text) -> escape_xml(to_utf8(lists:flatten(Text)), [], true).
 
 escape_xml([], Acc, _ForAttr) -> lists:reverse(Acc);
 escape_xml([$< | Tail], Acc, ForAttr) -> escape_xml(Tail, [$;, $t, $l, $& | Acc], ForAttr);
@@ -443,3 +444,17 @@ escape_xml([$> | Tail], Acc, ForAttr) -> escape_xml(Tail, [$;, $t, $g, $& | Acc]
 escape_xml([$& | Tail], Acc, ForAttr) -> escape_xml(Tail, [$;, $p, $m, $a, $& | Acc], ForAttr);
 escape_xml([$" | Tail], Acc, true) -> escape_xml(Tail, [$;, $t, $o, $u, $q, $& | Acc], true); % "
 escape_xml([Char | Tail], Acc, ForAttr) when is_integer(Char) -> escape_xml(Tail, [Char | Acc], ForAttr).
+
+%% the input may be utf8 or latin1; the resulting list is unicode
+to_utf8(Desc) when is_binary(Desc) ->
+	case unicode:characters_to_list(Desc) of
+		{_,_,_} -> unicode:characters_to_list(Desc, latin1);
+		X -> X
+	end;
+to_utf8(Desc) when is_list(Desc) ->
+	try
+		to_utf8(list_to_binary(Desc))
+	catch
+		_:_ ->
+			Desc
+	end.
diff --git a/lib/eunit/src/eunit_tty.erl b/lib/eunit/src/eunit_tty.erl
index f21b2da3d3..699d2adaca 100644
--- a/lib/eunit/src/eunit_tty.erl
+++ b/lib/eunit/src/eunit_tty.erl
@@ -83,7 +83,7 @@ terminate({ok, Data}, St) ->
 	    sync_end(error)
     end;
 terminate({error, Reason}, _St) ->
-    fwrite("Internal error: ~P.\n", [Reason, 25]),
+    fwrite("Internal error: ~tP.\n", [Reason, 25]),
     sync_end(error).
 
 sync_end(Result) ->
@@ -177,7 +177,7 @@ indent(_N) ->
 
 print_group_start(I, Desc) ->
     indent(I),
-    fwrite("~s\n", [Desc]).
+    fwrite("~ts\n", [Desc]).
 
 print_group_end(I, Time) ->
     if Time > 0 ->
@@ -195,13 +195,13 @@ print_test_begin(I, Data) ->
 	   true -> io_lib:fwrite("~w:", [Line])
 	end,
     D = if Desc =:= "" ; Desc =:= undefined -> "";
-	   true -> io_lib:fwrite(" (~s)", [Desc])
+	   true -> io_lib:fwrite(" (~ts)", [Desc])
 	end,
     case proplists:get_value(source, Data) of
 	{Module, Name, _Arity} ->
-	    fwrite("~s:~s ~s~s...", [Module, L, Name, D]);
+	    fwrite("~ts:~ts ~ts~ts...", [Module, L, Name, D]);
 	_ ->
-	    fwrite("~s~s...", [L, D])
+	    fwrite("~ts~ts...", [L, D])
     end.
 
 print_test_end(Data) ->
@@ -209,21 +209,21 @@ print_test_end(Data) ->
     T = if Time > 0 -> io_lib:fwrite("[~.3f s] ", [Time/1000]);
 	   true -> ""
 	end,
-    fwrite("~sok\n", [T]).
+    fwrite("~tsok\n", [T]).
 
 print_test_error({error, Exception}, Data) ->
     Output = proplists:get_value(output, Data),
-    fwrite("*failed*\n~s", [eunit_lib:format_exception(Exception)]),
+    fwrite("*failed*\n~ts", [eunit_lib:format_exception(Exception)]),
     case Output of
 	<<>> ->
 	    fwrite("\n\n");
 	<<Text:800/binary, _:1/binary, _/binary>> ->
-	    fwrite("  output:<<\"~s\">>...\n\n", [Text]);
+	    fwrite("  output:<<\"~ts\">>...\n\n", [Text]);
 	_ ->
-	    fwrite("  output:<<\"~s\">>\n\n", [Output])
+	    fwrite("  output:<<\"~ts\">>\n\n", [Output])
     end;
 print_test_error({skipped, Reason}, _) ->
-    fwrite("*did not run*\n::~s\n", [format_skipped(Reason)]).
+    fwrite("*did not run*\n::~ts\n", [format_skipped(Reason)]).
 
 format_skipped({module_not_found, M}) ->
     io_lib:fwrite("missing module: ~w", [M]);
@@ -244,12 +244,12 @@ format_cancel(undefined) ->
 format_cancel(timeout) ->
     "*timed out*\n";
 format_cancel({startup, Reason}) ->
-    io_lib:fwrite("*could not start test process*\n::~P\n\n",
+    io_lib:fwrite("*could not start test process*\n::~tP\n\n",
 		  [Reason, 15]);
 format_cancel({blame, _SubId}) ->
     "*cancelled because of subtask*\n";
 format_cancel({exit, Reason}) ->
-    io_lib:fwrite("*unexpected termination of test process*\n::~P\n\n",
+    io_lib:fwrite("*unexpected termination of test process*\n::~tP\n\n",
 		  [Reason, 15]);
 format_cancel({abort, Reason}) ->
     eunit_lib:format_error(Reason).