Merge branch 'bjorn/compiler/unicode-atoms/OTP-14285'

* bjorn/compiler/unicode-atoms/OTP-14285:
  Correct compile_SUITE:core_roundtrip/1
  Atoms in Core Erlang must be encoded in UTF-8

2 files changed, 69 insertions, 8 deletions

diff --git a/lib/compiler/src/core_scan.erl b/lib/compiler/src/core_scan.erl
index 15bfc78c8b..d7d5f900de 100644
--- a/lib/compiler/src/core_scan.erl
+++ b/lib/compiler/src/core_scan.erl
@@ -283,10 +283,12 @@ scan1([$$|Cs0], Toks, Pos) ->				%Character constant
     scan1(Cs, [{char,Pos,C}|Toks], Pos1);
 scan1([$'|Cs0], Toks, Pos) ->				%Atom (always quoted)
     {S,Cs1,Pos1} = scan_string(Cs0, $', Pos),
-    case catch list_to_atom(S) of
+    try binary_to_atom(list_to_binary(S), utf8) of
 	A when is_atom(A) ->
-	    scan1(Cs1, [{atom,Pos,A}|Toks], Pos1);
-	_Error -> scan_error({illegal,atom}, Pos)
+	    scan1(Cs1, [{atom,Pos,A}|Toks], Pos1)
+    catch
+        error:_ ->
+            scan_error({illegal,atom}, Pos)
     end;
 scan1([$"|Cs0], Toks, Pos) ->				%String
     {S,Cs1,Pos1} = scan_string(Cs0, $", Pos),
diff --git a/lib/compiler/test/compile_SUITE.erl b/lib/compiler/test/compile_SUITE.erl
index 10740ac2b0..474c3e2ca0 100644
--- a/lib/compiler/test/compile_SUITE.erl
+++ b/lib/compiler/test/compile_SUITE.erl
@@ -865,9 +865,7 @@ do_core_roundtrip_2(M, Core0, Outdir) ->
     case cmp_core(Core0, Core, M) of
 	true -> ok;
 	false -> error
-    end,
-
-    ok.
+    end.
 
 undo_var_translation(Tree) ->
     F = fun(Node) ->
@@ -920,11 +918,72 @@ diff(E, E) ->
 diff([H1|T1], [H2|T2]) ->
     [diff(H1, H2)|diff(T1, T2)];
 diff(T1, T2) when tuple_size(T1) =:= tuple_size(T2) ->
-    L = diff(tuple_to_list(T1), tuple_to_list(T2)),
-    list_to_tuple(L);
+    case cerl:is_c_var(T1) andalso cerl:is_c_var(T2) of
+        true ->
+            diff_var(T1, T2);
+        false ->
+            case cerl:is_c_map(T1) andalso cerl:is_c_map(T2) of
+                true ->
+                    diff_map(T1, T2);
+                false ->
+                    diff_tuple(T1, T2)
+            end
+    end;
 diff(E1, E2) ->
     {'DIFF',E1,E2}.
 
+diff_var(V1, V2) ->
+    case {cerl:var_name(V1),cerl:var_name(V2)} of
+        {Same,Same} ->
+            V1;
+        {Name1,Name2} ->
+            %% The inliner uses integers as variable names. Such integers
+            %% are read back as atoms.
+            case is_integer(Name1) andalso
+                list_to_atom(integer_to_list(Name1)) =:= Name2 of
+                true ->
+                    V1;
+                _ ->
+                    cerl:update_c_var(V1, {'DIFF',Name1,Name2})
+            end
+    end.
+
+%% Annotations for maps are not preserved exactly, but that is not
+%% a real problem. Workaround by not comparing all annotations when
+%% comparing maps.
+
+diff_map(M, M) ->
+    M;
+diff_map(M1, M2) ->
+    case cerl:get_ann(M1) =:= cerl:get_ann(M2) of
+        false ->
+            diff_tuple(M1, M2);
+        true ->
+            case remove_compiler_gen(M1) =:= remove_compiler_gen(M2) of
+                true ->
+                    M1;
+                false ->
+                    diff_tuple(M1, M2)
+            end
+    end.
+
+diff_tuple(T1, T2) ->
+    L = diff(tuple_to_list(T1), tuple_to_list(T2)),
+    list_to_tuple(L).
+
+remove_compiler_gen(M) ->
+    Arg0 = cerl:map_arg(M),
+    Arg = cerl:set_ann(Arg0, []),
+    Es0 = cerl:map_es(M),
+    Es = [remove_compiler_gen_1(Pair) || Pair <- Es0],
+    cerl:update_c_map(M, Arg, Es).
+
+remove_compiler_gen_1(Pair) ->
+    Op0 = cerl:map_pair_op(Pair),
+    Op = cerl:set_ann(Op0, []),
+    K = cerl:map_pair_key(Pair),
+    V = cerl:map_pair_val(Pair),
+    cerl:update_c_map_pair(Pair, Op, K, V).
 
 %% Compile to Beam assembly language (.S) and then try to
 %% run .S through the compiler again.