Merge branch sverker/remove-latin1-atom-encoding/OTP-14337

* sverker/remove-latin1-atom-encoding:
  kernel: Fix erl_distribution_wb_SUITE:whitebox
  kernel: Remove pg2_SUITE:compat
  erts: Remove fun_r13_SUITE
  stdlib: Remove test cases for R12 io protocol
  kernel: Make DFLAG_UTF8_ATOMS mandatory
  kernel: Rewrite distribution flag verification
  tools: Update assumptions in lcnt about external atom format
  stdlib: Tweak beam_lib_SUITE whitebox assumptions
  orber: Remove hard dependency to external atom format
  kernel: Try mend disk_log whitebox tests
  erts: Mark latin1 atom encoding as deprecated
  jinterface: Do not generate atoms on old latin1 external format
  erl_interface: Do not generate atoms on old latin1 ext format
  erts: Do not generate atoms on old latin1 external format
  erts: Fix faulty ASSERT for failed dec_term

4 files changed, 55 insertions, 77 deletions

diff --git a/lib/kernel/src/dist_util.erl b/lib/kernel/src/dist_util.erl
index 8d2fc4d4b7..d929179715 100644
--- a/lib/kernel/src/dist_util.erl
+++ b/lib/kernel/src/dist_util.erl
@@ -131,7 +131,7 @@ handshake_other_started(#hs_data{request_type=ReqType}=HSData0) ->
 			     other_version=Version,
 			     other_node=Node,
 			     other_started=true},
-    check_dflag_xnc(HSData),
+    check_dflags(HSData),
     is_allowed(HSData),
     ?debug({"MD5 connection from ~p (V~p)~n",
 	    [Node, HSData#hs_data.other_version]}),
@@ -168,27 +168,24 @@ is_allowed(#hs_data{other_node = Node,
 %% Check that both nodes can handle the same types of extended
 %% node containers. If they can not, abort the connection.
 %%
-check_dflag_xnc(#hs_data{other_node = Node,
-			 other_flags = OtherFlags,
-			 other_started = OtherStarted} = HSData) ->
-    XRFlg = ?DFLAG_EXTENDED_REFERENCES,
-    XPPFlg = case erlang:system_info(compat_rel) of
-		 R when R >= 10 ->
-		     ?DFLAG_EXTENDED_PIDS_PORTS;
-		 _ ->
-		     0
-	     end,
-    ReqXncFlags = XRFlg bor XPPFlg,
-    case OtherFlags band ReqXncFlags =:= ReqXncFlags of
-	true ->
-	    ok;
-	false ->
-	    What = case {OtherFlags band XRFlg =:= XRFlg,
-			 OtherFlags band XPPFlg =:= XPPFlg} of
-		       {false, false} -> "references, pids and ports";
-		       {true, false} -> "pids and ports";
-		       {false, true} -> "references"
-		   end,
+check_dflags(#hs_data{other_node = Node,
+                      other_flags = OtherFlags,
+                      other_started = OtherStarted} = HSData) ->
+
+    Mandatory = [{?DFLAG_EXTENDED_REFERENCES, "EXTENDED_REFERENCES"},
+                 {?DFLAG_EXTENDED_PIDS_PORTS, "EXTENDED_PIDS_PORTS"},
+                 {?DFLAG_UTF8_ATOMS, "UTF8_ATOMS"}],
+    Missing = lists:filtermap(fun({Bit, Str}) ->
+                                      case Bit band OtherFlags of
+                                          Bit -> false;
+                                          0 -> {true, Str}
+                                      end
+                              end,
+                              Mandatory),
+    case Missing of
+        [] ->
+            ok;
+        _ ->
 	    case OtherStarted of
 		true ->
 		    send_status(HSData, not_allowed),
@@ -199,9 +196,9 @@ check_dflag_xnc(#hs_data{other_node = Node,
 		    How = "aborted"
 	    end,
 	    error_msg("** ~w: Connection attempt ~s node ~w ~s "
-		      "since it cannot handle extended ~s. "
-		      "**~n", [node(), Dir, Node, How, What]),
-	    ?shutdown2(Node, {check_dflag_xnc_failed, What})
+		      "since it cannot handle ~p."
+		      "**~n", [node(), Dir, Node, How, Missing]),
+	    ?shutdown2(Node, {check_dflags_failed, Missing})
     end.
 
 
@@ -327,7 +324,7 @@ handshake_we_started(#hs_data{request_type=ReqType,
     NewHSData = HSData#hs_data{this_flags = ThisFlags,
 			       other_flags = OtherFlags, 
 			       other_started = false}, 
-    check_dflag_xnc(NewHSData),
+    check_dflags(NewHSData),
     MyChallenge = gen_challenge(),
     {MyCookie,HisCookie} = get_cookies(Node),
     send_challenge_reply(NewHSData,MyChallenge,
diff --git a/lib/kernel/test/disk_log_SUITE.erl b/lib/kernel/test/disk_log_SUITE.erl
index 23fe975ef7..079cc2f90f 100644
--- a/lib/kernel/test/disk_log_SUITE.erl
+++ b/lib/kernel/test/disk_log_SUITE.erl
@@ -481,7 +481,7 @@ halt_ro_crash(Conf) when is_list(Conf) ->
     %% This is how it was before R6B:
     %% {C1,T1,15} = disk_log:chunk(a,start),
     %% {C2,T2} = disk_log:chunk(a,C1),
-    {C1,_OneItem,7478} = disk_log:chunk(a,start),
+    {C1,_OneItem,7476} = disk_log:chunk(a,start),
     {C2, [], 7} = disk_log:chunk(a,C1),
     eof = disk_log:chunk(a,C2),
     ok = disk_log:close(a),
@@ -2502,8 +2502,8 @@ error_repair(Conf) when is_list(Conf) ->
     ok = disk_log:close(n),
     BadFile = add_ext(File, 2), % current file
     set_opened(BadFile),
-    crash(BadFile, 28), % the binary is now invalid
-    {repaired,n,{recovered,0},{badbytes,26}} =
+    crash(BadFile, 26), % the binary is now invalid
+    {repaired,n,{recovered,0},{badbytes,24}} =
 	disk_log:open([{name, n}, {file, File}, {type, wrap},
 		       {format, internal}, {size, {40,No}}]),
     ok = disk_log:close(n),
@@ -2518,8 +2518,8 @@ error_repair(Conf) when is_list(Conf) ->
     ok = disk_log:close(n),
     BadFile2 = add_ext(File, 1), % current file
     set_opened(BadFile2),
-    crash(BadFile2, 51), % the second binary is now invalid
-    {repaired,n,{recovered,1},{badbytes,26}} =
+    crash(BadFile2, 47), % the second binary is now invalid
+    {repaired,n,{recovered,1},{badbytes,24}} =
 	disk_log:open([{name, n}, {file, File}, {type, wrap},
 		       {format, internal}, {size, {4000,No}}]),
     ok = disk_log:close(n),
@@ -2571,7 +2571,7 @@ error_repair(Conf) when is_list(Conf) ->
     ok = disk_log:close(n),
     set_opened(File),
     crash(File, 30),
-    {repaired,n,{recovered,3},{badbytes,16}} =
+    {repaired,n,{recovered,3},{badbytes,15}} =
         disk_log:open([{name, n}, {file, File}, {type, halt},
 		       {format, internal},{repair,true},
 		       {head_func, {?MODULE, head_fun, [{ok,"head"}]}}]),
@@ -2797,7 +2797,7 @@ chunk(Conf) when is_list(Conf) ->
     ok = disk_log:log_terms(n, [{some,terms}]), % second file full
     2 = curf(n),
     BadFile = add_ext(File, 1),
-    crash(BadFile, 28), % the _binary_ is now invalid
+    crash(BadFile, 26), % the _binary_ is now invalid
     {error, {corrupt_log_file, BFile}} = disk_log:chunk(n, start, 1),
     BadFile = BFile,
     ok = disk_log:close(n),
@@ -2807,7 +2807,7 @@ chunk(Conf) when is_list(Conf) ->
 			     {format, internal}]),
     ok = disk_log:log_terms(n, [{this,is}]),
     ok = disk_log:sync(n),
-    crash(File, 28), % the _binary_ is now invalid
+    crash(File, 26), % the _binary_ is now invalid
     {error, {corrupt_log_file, File2}} = disk_log:chunk(n, start, 1),
     crash(File, 10),
     {error,{corrupt_log_file,_}} = disk_log:bchunk(n, start, 1),
@@ -2901,8 +2901,8 @@ chunk(Conf) when is_list(Conf) ->
     {ok, n} = disk_log:open([{name, n}, {file, File}, {type, wrap},
 			     {format, internal}, {mode, read_only}]),
     CrashFile = add_ext(File, 1),
-    crash(CrashFile, 51), % the binary term {some,terms} is now bad
-    {H1, [{this,is}], 18} = disk_log:chunk(n, start, 10),
+    crash(CrashFile, 46), % the binary term {some,terms} is now bad
+    {H1, [{this,is}], 16} = disk_log:chunk(n, start, 10),
     {H2, [{on,a},{wrap,file}]} = disk_log:chunk(n, H1),
     eof = disk_log:chunk(n, H2),
     ok = disk_log:close(n),
@@ -2916,8 +2916,8 @@ chunk(Conf) when is_list(Conf) ->
     ok = disk_log:close(n),
     {ok, n} = disk_log:open([{name, n}, {file, File}, {type, halt},
 			     {format, internal}, {mode, read_only}]),
-    crash(File, 51), % the binary term {some,terms} is now bad
-    {J1, [{this,is}], 18} = disk_log:chunk(n, start, 10),
+    crash(File, 46), % the binary term {some,terms} is now bad
+    {J1, [{this,is}], 16} = disk_log:chunk(n, start, 10),
     {J2, [{on,a},{halt,file}]} = disk_log:chunk(n, J1),
     eof = disk_log:chunk(n, J2),
     ok = disk_log:close(n),
@@ -2932,8 +2932,8 @@ chunk(Conf) when is_list(Conf) ->
     ok = disk_log:close(n),
     {ok, n} = disk_log:open([{name, n}, {file, File}, {type, halt},
 			     {format, internal}, {mode, read_only}]),
-    crash(File, 44), % the binary term {s} is now bad
-    {J11, [{this,is}], 7} = disk_log:chunk(n, start, 10),
+    crash(File, 40), % the binary term {s} is now bad
+    {J11, [{this,is}], 6} = disk_log:chunk(n, start, 10),
     {J21, [{on,a},{halt,file}]} = disk_log:chunk(n, J11),
     eof = disk_log:chunk(n, J21),
     ok = disk_log:close(n),
@@ -3052,7 +3052,7 @@ truncate(Conf) when is_list(Conf) ->
     ok = disk_log:truncate(n, apa),
     rec(1, {disk_log, node(), n, {truncated, 6}}),
     {0, 0} = no_overflows(n),
-    23 = curb(n),
+    22 = curb(n),
     1 = curf(n),
     1 = cur_cnt(n),
     true = (Size == sz(n)),
@@ -3072,7 +3072,7 @@ truncate(Conf) when is_list(Conf) ->
     ok = disk_log:truncate(n, apa),
     rec(1, {disk_log, node(), n, {truncated, 3}}),
     {0, 0} = no_overflows(n),
-    23 = curb(n),
+    22 = curb(n),
     1 = curf(n),
     1 = cur_cnt(n),
     true = (Size == sz(n)),
@@ -3181,45 +3181,45 @@ info_current(Conf) when is_list(Conf) ->
     %% Internal with header.
     {ok, n} = disk_log:open([{name, n}, {file, File}, {type, wrap},
 			     {head, header}, {size, {100,No}}]),
-    {26, 1} = {curb(n), cur_cnt(n)},
+    {25, 1} = {curb(n), cur_cnt(n)},
     {1, 1}  = {no_written_items(n), no_items(n)},
     ok = disk_log:log(n, B),
-    {94, 2} = {curb(n), cur_cnt(n)},
+    {93, 2} = {curb(n), cur_cnt(n)},
     {2, 2}  = {no_written_items(n), no_items(n)},
     ok = disk_log:close(n),
     {ok, n} = disk_log:open([{name, n}, {file, File}, {type, wrap},
 			     {notify, true},
 			     {head, header}, {size, {100,No}}]),
-    {94, 2} = {curb(n), cur_cnt(n)},
+    {93, 2} = {curb(n), cur_cnt(n)},
     {0, 2}  = {no_written_items(n), no_items(n)},
     ok = disk_log:log(n, B),
     rec(1, {disk_log, node(), n, {wrap, 0}}),
-    {94, 2} = {curb(n), cur_cnt(n)},
+    {93, 2} = {curb(n), cur_cnt(n)},
     {2, 4}  = {no_written_items(n), no_items(n)},
     disk_log:inc_wrap_file(n),
     rec(1, {disk_log, node(), n, {wrap, 0}}),
-    {26, 1} = {curb(n), cur_cnt(n)},
+    {25, 1} = {curb(n), cur_cnt(n)},
     {3, 4}  = {no_written_items(n), no_items(n)},
     ok = disk_log:log_terms(n, [B,B,B]),
     %% Used to be one message, but now one per wrapped file.
     rec(1, {disk_log, node(), n, {wrap, 0}}),
     rec(1, {disk_log, node(), n, {wrap, 2}}),
-    {94, 2} = {curb(n), cur_cnt(n)},
+    {93, 2} = {curb(n), cur_cnt(n)},
     {8, 7}  = {no_written_items(n), no_items(n)},
     ok = disk_log:log_terms(n, [B]),
     rec(1, {disk_log, node(), n, {wrap, 2}}),
     ok = disk_log:log_terms(n, [B]),
     rec(1, {disk_log, node(), n, {wrap, 2}}),
-    {94, 2} = {curb(n), cur_cnt(n)},
+    {93, 2} = {curb(n), cur_cnt(n)},
     {12, 7}  = {no_written_items(n), no_items(n)},
     ok = disk_log:log_terms(n, [BB,BB]),
     %% Used to be one message, but now one per wrapped file.
     rec(2, {disk_log, node(), n, {wrap, 2}}),
-    {194, 2} = {curb(n), cur_cnt(n)},
+    {193, 2} = {curb(n), cur_cnt(n)},
     {16, 7}  = {no_written_items(n), no_items(n)},
     ok = disk_log:log_terms(n, [SB,SB,SB]),
     rec(1, {disk_log, node(), n, {wrap, 2}}),
-    {80, 4} = {curb(n), cur_cnt(n)},
+    {79, 4} = {curb(n), cur_cnt(n)},
     {20, 9}  = {no_written_items(n), no_items(n)},
     ok = disk_log:close(n),
     del(File, No),
diff --git a/lib/kernel/test/erl_distribution_wb_SUITE.erl b/lib/kernel/test/erl_distribution_wb_SUITE.erl
index 61aa3b32ee..c1dc208cc1 100644
--- a/lib/kernel/test/erl_distribution_wb_SUITE.erl
+++ b/lib/kernel/test/erl_distribution_wb_SUITE.erl
@@ -56,10 +56,14 @@
 -define(DFLAG_HIDDEN_ATOM_CACHE,16#40).
 -define(DFLAG_NEW_FUN_TAGS,16#80).
 -define(DFLAG_EXTENDED_PIDS_PORTS,16#100).
+-define(DFLAG_UTF8_ATOMS, 16#10000).
 
 %% From R9 and forward extended references is compulsory
 %% From R10 and forward extended pids and ports are compulsory
--define(COMPULSORY_DFLAGS, (?DFLAG_EXTENDED_REFERENCES bor ?DFLAG_EXTENDED_PIDS_PORTS)).
+%% From R20 and forward UTF8 atoms are compulsory
+-define(COMPULSORY_DFLAGS, (?DFLAG_EXTENDED_REFERENCES bor
+                            ?DFLAG_EXTENDED_PIDS_PORTS bor
+                            ?DFLAG_UTF8_ATOMS)).
 
 
 -define(shutdown(X), exit(X)).
diff --git a/lib/kernel/test/pg2_SUITE.erl b/lib/kernel/test/pg2_SUITE.erl
index fdc268cb5a..9460608a3e 100644
--- a/lib/kernel/test/pg2_SUITE.erl
+++ b/lib/kernel/test/pg2_SUITE.erl
@@ -31,7 +31,7 @@
 
 -export([
 	 otp_7277/1, otp_8259/1, otp_8653/1,
-         compat/1, basic/1]).
+         basic/1]).
 
 -define(TESTCASE, testcase_name).
 -define(testcase, proplists:get_value(?TESTCASE, Config)).
@@ -56,7 +56,7 @@ all() ->
 
 groups() -> 
     [{tickets, [],
-      [otp_7277, otp_8259, otp_8653, compat, basic]}].
+      [otp_7277, otp_8259, otp_8653, basic]}].
 
 init_per_suite(Config) ->
     Config.
@@ -218,29 +218,6 @@ loop() ->
 	    exit(normal)
     end.
 
-%% OTP-8259. Check that 'exchange' and 'del_member' work.
-compat(Config) when is_list(Config) ->
-    case test_server:is_release_available("r13b") of
-        true ->
-            Pid = spawn(forever()),
-            G = a,
-            ok = pg2:create(G),
-            ok = pg2:join(G, Pid),
-            ok = pg2:join(G, Pid),
-            {ok, A} = start_node_rel(r13, r13b, slave),
-            pong = net_adm:ping(A),
-            wait_for_ready_net(Config),
-            {ok, _} = rpc:call(A, pg2, start, []),
-            ?UNTIL([Pid,Pid] =:= rpc:call(A, pg2, get_members, [a])),
-            true = exit(Pid, kill),
-            ?UNTIL([] =:= pg2:get_members(a)),
-            ?UNTIL([] =:= rpc:call(A, pg2, get_members, [a])),
-            test_server:stop_node(A),
-	    ok;
-        false ->
-	    {skipped, "No support for old node"}
-    end.
-
 %% OTP-8259. Some basic tests.
 basic(Config) when is_list(Config) ->
     _ = [pg2:delete(G) || G <- pg2:which_groups()],