Merge branch 'dgud/stdlib/unicode-binary-bug/ERL-777/OTP-15428' into maint

* dgud/stdlib/unicode-binary-bug/ERL-777/OTP-15428:
  unicode_util did not handle binary input data correctly

1 files changed, 20 insertions, 7 deletions

diff --git a/lib/stdlib/test/unicode_util_SUITE.erl b/lib/stdlib/test/unicode_util_SUITE.erl
index 962b307b07..044b4e5834 100644
--- a/lib/stdlib/test/unicode_util_SUITE.erl
+++ b/lib/stdlib/test/unicode_util_SUITE.erl
@@ -126,17 +126,30 @@ verify_gc(Line0, N, Acc) ->
 
     %io:format("Line: ~s~n",[Line]),
     [Data|_Comments] = string:tokens(Line, "#"),
-    %io:format("Data: ~w~n",[string:tokens(Data, " \t")]),
+    %% io:format("Data: ~w~n",[string:tokens(Data, " \t")]),
     {Str,Res} = gc_test_data(string:tokens(Data, " \t"), [], [[]]),
-    try
-        Res = fetch(Str, fun unicode_util:gc/1),
-        Acc
-    catch _Cl:{badmatch, Other} ->
+    %% io:format("InputStr: ~w ~w~n",[Str,unicode:characters_to_binary(Str)]),
+    case verify_gc(Str, Res, N, Line) andalso
+        verify_gc(unicode:characters_to_binary(Str), Res, N, Line0) of
+        true -> Acc;
+        false -> Acc+1
+    end.
+
+verify_gc({error,_,[CP|_]}=Err, _Res, N, Line) ->
+    IsSurrogate = 16#D800 =< CP andalso CP =< 16#DFFF,
+    %% Surrogat is not valid in utf8 encoding only utf16
+    IsSurrogate orelse
+        io:format("~w: ~ts~n Error in unicode:characters_to_binary ~w~n", [N, Line, Err]),
+    IsSurrogate;
+verify_gc(Str, Res, N, Line) ->
+    try fetch(Str, fun unicode_util:gc/1) of
+        Res -> true;
+        Other ->
             io:format("Failed: ~p~nInput: ~ts~n\t=> ~w |~ts|~n",[N, Line, Str, Str]),
             io:format("Expected: ~p~n", [Res]),
             io:format("Got: ~w~n", [Other]),
-            Acc+1;
-          Cl:R:Stacktrace ->
+            false
+    catch Cl:R:Stacktrace ->
             io:format("~p: ~ts => |~tp|~n",[N, Line, Str]),
             io:format("Expected: ~p~n", [Res]),
             erlang:raise(Cl,R,Stacktrace)