Merge branch 'egil/18/fix-maps-binary_to_term/OTP-13125' into maint

* egil/18/fix-maps-binary_to_term/OTP-13125:
  erts: More testcases for map binary_to_term/1
  erts: Fix maps decode in erlang:binary_to_term/1

2 files changed, 143 insertions, 43 deletions

diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index c6d7e3fcc5..a85aa15403 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -1179,7 +1179,7 @@ typedef struct {
     ErtsHeapFactory factory;
     int remaining_n;
     char* remaining_bytes;
-    Eterm* maps_list;
+    ErtsWStack flat_maps;
     ErtsPStack hamt_array;
 } B2TDecodeContext;
 
@@ -1519,7 +1519,7 @@ static BIF_RETTYPE binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binar
             ctx->u.dc.res = (Eterm) (UWord) NULL;
             ctx->u.dc.next = &ctx->u.dc.res;
 	    erts_factory_proc_prealloc_init(&ctx->u.dc.factory, p, ctx->heap_size);
-	    ctx->u.dc.maps_list = NULL;
+	    ctx->u.dc.flat_maps.wstart = NULL;
 	    ctx->u.dc.hamt_array.pstart = NULL;
             ctx->state = B2TDecode;
             /*fall through*/
@@ -2938,7 +2938,7 @@ dec_term(ErtsDistExternal *edep,
     int n;
     ErtsAtomEncoding char_enc;
     register Eterm* hp;        /* Please don't take the address of hp */
-    Eterm *maps_list;   /* for preprocessing of small maps */
+    DECLARE_WSTACK(flat_maps); /* for preprocessing of small maps */
     Eterm* next;
     SWord reds;
 #ifdef DEBUG
@@ -2950,7 +2950,6 @@ dec_term(ErtsDistExternal *edep,
         next     = ctx->u.dc.next;
         ep       = ctx->u.dc.ep;
 	factory  = &ctx->u.dc.factory;
-	maps_list = ctx->u.dc.maps_list;
 
         if (ctx->state != B2TDecode) {
             int n_limit = reds;
@@ -3026,15 +3025,18 @@ dec_term(ErtsDistExternal *edep,
             }
         }
 	PSTACK_CHANGE_ALLOCATOR(hamt_array, ERTS_ALC_T_SAVED_ESTACK);
+        WSTACK_CHANGE_ALLOCATOR(flat_maps, ERTS_ALC_T_SAVED_ESTACK);
 	if (ctx->u.dc.hamt_array.pstart) {
 	    PSTACK_RESTORE(hamt_array, &ctx->u.dc.hamt_array);
 	}
+	if (ctx->u.dc.flat_maps.wstart) {
+	    WSTACK_RESTORE(flat_maps, &ctx->u.dc.flat_maps);
+	}
     }
     else {
         reds = ERTS_SWORD_MAX;
         next = objp;
         *next = (Eterm) (UWord) NULL;
-	maps_list = NULL;
     }
     hp = factory->hp;
 
@@ -3595,14 +3597,8 @@ dec_term_atom_common:
                      * vptr, last word for values
                      */
 
-                    /*
-                     * Use thing_word to link through decoded maps.
-                     * The list of maps is for later validation.
-                     */
-
-                    mp->thing_word = (Eterm) COMPRESS_POINTER(maps_list);
-                    maps_list      = (Eterm *) mp;
-
+                    WSTACK_PUSH(flat_maps, (UWord)mp);
+                    mp->thing_word = MAP_HEADER_FLATMAP;
                     mp->size       = size;
                     mp->keys       = keys;
                     *objp          = make_flatmap(mp);
@@ -3851,7 +3847,9 @@ dec_term_atom_common:
                     ctx->u.dc.ep = ep;
                     ctx->u.dc.next = next;
                     ctx->u.dc.factory.hp = hp;
-		    ctx->u.dc.maps_list = maps_list;
+		    if (!WSTACK_ISEMPTY(flat_maps)) {
+			WSTACK_SAVE(flat_maps, &ctx->u.dc.flat_maps);
+		    }
 		    if (!PSTACK_IS_EMPTY(hamt_array)) {
 			PSTACK_SAVE(hamt_array, &ctx->u.dc.hamt_array);
 		    }
@@ -3865,18 +3863,6 @@ dec_term_atom_common:
         }
     }
 
-    /* Iterate through all the maps and check for validity and sort keys
-     * - done here for when we know it is complete.
-     */
-
-    while (maps_list) {
-	next  = (Eterm *)(EXPAND_POINTER(*maps_list));
-	*maps_list = MAP_HEADER_FLATMAP;
-	if (!erts_validate_and_sort_flatmap((flatmap_t*)maps_list))
-	    goto error;
-	maps_list  = next;
-    }
-
     ASSERT(hp <= factory->hp_end
            || (factory->mode == FACTORY_CLOSED && is_immed(*dbg_resultp)));
     factory->hp = hp;
@@ -3885,20 +3871,31 @@ dec_term_atom_common:
      */
 
     if (!PSTACK_IS_EMPTY(hamt_array)) {
-	do {
-	    struct dec_term_hamt* hamt = PSTACK_TOP(hamt_array);
-
-	    *hamt->objp = erts_hashmap_from_array(factory,
-						  hamt->leaf_array,
-						  hamt->size,
-						  1);
-	    if (is_non_value(*hamt->objp))
-		goto error_hamt;
-
-	    (void) PSTACK_POP(hamt_array);
-	} while (!PSTACK_IS_EMPTY(hamt_array));
-	PSTACK_DESTROY(hamt_array);
+        do {
+            struct dec_term_hamt* hamt = PSTACK_TOP(hamt_array);
+
+            *hamt->objp = erts_hashmap_from_array(factory,
+                                                  hamt->leaf_array,
+                                                  hamt->size,
+                                                  1);
+            if (is_non_value(*hamt->objp))
+                goto error_hamt;
+
+            (void) PSTACK_POP(hamt_array);
+        } while (!PSTACK_IS_EMPTY(hamt_array));
+        PSTACK_DESTROY(hamt_array);
+    }
+
+    /* Iterate through all the (flat)maps and check for validity and sort keys
+     * - done here for when we know it is complete.
+     */
+
+    while(!WSTACK_ISEMPTY(flat_maps)) {
+        next = (Eterm *)WSTACK_POP(flat_maps);
+        if (!erts_validate_and_sort_flatmap((flatmap_t*)next))
+            goto error;
     }
+    WSTACK_DESTROY(flat_maps);
 
     ASSERT((Eterm*)EXPAND_POINTER(*dbg_resultp) != NULL);
 
@@ -3924,6 +3921,7 @@ error_hamt:
 	ctx->state = B2TDecodeFail;
 	ctx->reds = reds;
     }
+    WSTACK_DESTROY(flat_maps);
         
     return NULL;
 }
diff --git a/erts/emulator/test/map_SUITE.erl b/erts/emulator/test/map_SUITE.erl
index 886ae7d516..62a94e5281 100644
--- a/erts/emulator/test/map_SUITE.erl
+++ b/erts/emulator/test/map_SUITE.erl
@@ -2181,7 +2181,9 @@ t_map_encode_decode(Config) when is_list(Config) ->
 	{<<>>, sc9}, {3.14158, sc10},
 	{[3.14158], sc11}, {more_atoms, sc12},
 	{{more_tuples}, sc13}, {self(), sc14},
-	{{},{}},{[],[]}
+	{{},{}},{[],[]},
+        {map_s, #{a=>a, 2=>b, 3=>c}},
+        {map_l, maps:from_list([{I,I}||I <- lists:seq(1,74)])}
     ],
     ok = map_encode_decode_and_match(Pairs,[],#{}),
 
@@ -2245,9 +2247,30 @@ t_map_encode_decode(Config) when is_list(Config) ->
 
     %% bad size (too small) .. should fail just truncate it .. weird.
     %% possibly change external format so truncated will be #{a:=1}
-    #{ a:=b } =
-	erlang:binary_to_term(<<131,116,0,0,0,1,100,0,1,97,100,0,1,98,97,1,97,1>>),
-
+    #{ a:=b } = erlang:binary_to_term(<<131,116,0,0,0,1,100,0,1,97,100,0,1,98,97,1,97,1>>),
+
+    %% specific fannerl (opensource app) binary_to_term error in 18.1
+
+    #{bias := {1,1,0},
+      bit_fail := 0,
+      connections := #{{2,9}   := _,
+                       {8,14}  := _,
+                       {2,12}  := _,
+                       {5,7}   := _,
+                       {11,16} := _,
+                       {11,15} := _},
+      layers := {5,7,3},
+      network_type := fann_nettype_layer,
+      num_input := 5,
+      num_layers := 3,
+      num_output := 3,
+      rprop_delta_max := _,
+      rprop_delta_min := _,
+      total_connections := 66,
+      total_neurons := 17,
+      train_error_function := fann_errorfunc_tanh,
+      train_stop_function := fann_stopfunc_mse,
+      training_algorithm := fann_train_rprop} = erlang:binary_to_term(fannerl()),
     ok.
 
 map_encode_decode_and_match([{K,V}|Pairs], EncodedPairs, M0) ->
@@ -2966,3 +2989,82 @@ do_badmap_17(Config) ->
 
 %% Use this function to avoid compile-time evaluation of an expression.
 id(I) -> I.
+
+
+%% map external_format (fannerl).
+fannerl() ->
+    <<131,116,0,0,0,28,100,0,13,108,101,97,114,110,105,110,103,95,114,
+      97,116,101,70,63,230,102,102,96,0,0,0,100,0,17,108,101,97,114,110,105,110,
+      103,95,109,111,109,101,110,116,117,109,70,0,0,0,0,0,0,0,0,100,0,
+      18,116,114,97,105,110,105,110,103,95,97,108,103,111,114,105,116,104,109,100,0,
+      16,102,97,110,110,95,116,114,97,105,110,95,114,112,114,111,112,
+      100,0,17,109,101,97,110,95,115,113,117,97,114,101,95,101,114,114,111,114,70,
+      0,0,0,0,0,0,0,0,100,0,8,98,105,116,95,102,97,105,108,97,0,100,0,20,
+      116,114,97,105,110,95,101,114,114,111,114,95,102,117,110,99,116,105,111,
+      110,100,0,19,102,97,110,110,95,101,114,114,111,114,102,117,110,99,
+      95,116,97,110,104,100,0,9,110,117,109,95,105,110,112,117,116,97,5,100,0,10,110,
+      117,109,95,111,117,116,112,117,116,97,3,100,0,13,116,111,116,97,108,
+      95,110,101,117,114,111,110,115,97,17,100,0,17,116,111,116,97,108,95,99,111,110,
+      110,101,99,116,105,111,110,115,97,66,100,0,12,110,101,116,119,111,114,107,
+      95,116,121,112,101,100,0,18,102,97,110,110,95,110,101,116,116,121,112,101,
+      95,108,97,121,101,114,100,0,15,99,111,110,110,101,99,116,105,111,110,95,
+      114,97,116,101,70,63,240,0,0,0,0,0,0,100,0,10,110,117,109,95,108,97,121,101,
+      114,115,97,3,100,0,19,116,114,97,105,110,95,115,116,111,112,95,102,117,110,
+      99,116,105,111,110,100,0,17,102,97,110,110,95,115,116,111,112,102,117,110,
+      99,95,109,115,101,100,0,15,113,117,105,99,107,112,114,111,112,95,100,101,99,
+      97,121,70,191,26,54,226,224,0,0,0,100,0,12,113,117,105,99,107,112,114,
+      111,112,95,109,117,70,63,252,0,0,0,0,0,0,100,0,21,114,112,114,111,112,95,105,
+      110,99,114,101,97,115,101,95,102,97,99,116,111,114,70,63,243,51,51,
+      64,0,0,0,100,0,21,114,112,114,111,112,95,100,101,99,114,101,97,115,101,
+      95,102,97,99,116,111,114,70,63,224,0,0,0,0,0,0,100,0,15,114,112,114,111,112,
+      95,100,101,108,116,97,95,109,105,110,70,0,0,0,0,0,0,0,0,100,0,15,114,112,114,
+      111,112,95,100,101,108,116,97,95,109,97,120,70,64,73,0,0,0,0,0,0,100,0,
+      16,114,112,114,111,112,95,100,101,108,116,97,95,122,101,114,111,70,63,185,153,
+      153,160,0,0,0,100,0,26,115,97,114,112,114,111,112,95,119,101,105,103,
+      104,116,95,100,101,99,97,121,95,115,104,105,102,116,70,192,26,147,116,192,0,0,0,
+      100,0,35,115,97,114,112,114,111,112,95,115,116,101,112,95,101,114,
+      114,111,114,95,116,104,114,101,115,104,111,108,100,95,102,97,99,116,111,114,70,
+      63,185,153,153,160,0,0,0,100,0,24,115,97,114,112,114,111,112,95,115,
+      116,101,112,95,101,114,114,111,114,95,115,104,105,102,116,70,63,246,40,245,
+      192,0,0,0,100,0,19,115,97,114,112,114,111,112,95,116,101,109,112,101,114,
+      97,116,117,114,101,70,63,142,184,81,224,0,0,0,100,0,6,108,97,121,101,114,115,
+      104,3,97,5,97,7,97,3,100,0,4,98,105,97,115,104,3,97,1,97,1,97,0,100,0,11,
+      99,111,110,110,101,99,116,105,111,110,115,116,0,0,0,66,104,2,97,0,97,6,70,
+      191,179,51,44,64,0,0,0,104,2,97,1,97,6,70,63,178,130,90,32,0,0,0,104,2,97,2,
+      97,6,70,63,82,90,88,0,0,0,0,104,2,97,3,97,6,70,63,162,91,63,192,0,0,0,104,2,
+      97,4,97,6,70,191,151,70,169,0,0,0,0,104,2,97,5,97,6,70,191,117,52,222,0,0,0,
+      0,104,2,97,0,97,7,70,63,152,240,139,0,0,0,0,104,2,97,1,97,7,70,191,166,31,
+      187,160,0,0,0,104,2,97,2,97,7,70,191,150,70,63,0,0,0,0,104,2,97,3,97,7,70,
+      63,152,181,126,128,0,0,0,104,2,97,4,97,7,70,63,151,187,162,128,0,0,0,104,2,
+      97,5,97,7,70,191,143,161,101,0,0,0,0,104,2,97,0,97,8,70,191,153,102,36,128,0,
+      0,0,104,2,97,1,97,8,70,63,160,139,250,64,0,0,0,104,2,97,2,97,8,70,63,164,62,
+      196,64,0,0,0,104,2,97,3,97,8,70,191,178,78,209,192,0,0,0,104,2,97,4,97,8,70,
+      191,185,19,76,224,0,0,0,104,2,97,5,97,8,70,63,183,142,196,96,0,0,0,104,2,97,0,
+      97,9,70,63,150,104,248,0,0,0,0,104,2,97,1,97,9,70,191,164,4,100,224,0,0,0,
+      104,2,97,2,97,9,70,191,169,42,42,224,0,0,0,104,2,97,3,97,9,70,63,145,54,78,128,0,
+      0,0,104,2,97,4,97,9,70,63,126,243,134,0,0,0,0,104,2,97,5,97,9,70,63,177,
+      203,25,96,0,0,0,104,2,97,0,97,10,70,63,172,104,47,64,0,0,0,104,2,97,1,97,10,
+      70,63,161,242,193,64,0,0,0,104,2,97,2,97,10,70,63,175,208,241,192,0,0,0,104,2,
+      97,3,97,10,70,191,129,202,161,0,0,0,0,104,2,97,4,97,10,70,63,178,151,55,32,0,0,0,
+      104,2,97,5,97,10,70,63,137,155,94,0,0,0,0,104,2,97,0,97,11,70,191,179,
+      106,160,0,0,0,0,104,2,97,1,97,11,70,63,184,253,164,96,0,0,0,104,2,97,2,97,11,
+      70,191,143,30,157,0,0,0,0,104,2,97,3,97,11,70,63,153,225,140,128,0,0,0,104,
+      2,97,4,97,11,70,63,161,35,85,192,0,0,0,104,2,97,5,97,11,70,63,175,200,55,192,
+      0,0,0,104,2,97,0,97,12,70,191,180,116,132,96,0,0,0,104,2,97,1,97,12,70,191,
+      165,151,152,0,0,0,0,104,2,97,2,97,12,70,191,180,197,91,160,0,0,0,104,2,97,3,97,12,
+      70,191,91,30,160,0,0,0,0,104,2,97,4,97,12,70,63,180,251,45,32,0,0,0,
+      104,2,97,5,97,12,70,63,165,134,77,64,0,0,0,104,2,97,6,97,14,70,63,181,56,242,96,
+      0,0,0,104,2,97,7,97,14,70,191,165,239,234,224,0,0,0,104,2,97,8,97,14,
+      70,191,154,65,216,128,0,0,0,104,2,97,9,97,14,70,63,150,250,236,0,0,0,0,104,2,97,
+      10,97,14,70,191,141,105,108,0,0,0,0,104,2,97,11,97,14,70,191,152,40,
+      165,0,0,0,0,104,2,97,12,97,14,70,63,141,159,46,0,0,0,0,104,2,97,13,97,14,70,
+      191,183,172,137,32,0,0,0,104,2,97,6,97,15,70,63,163,26,123,192,0,0,0,104,
+      2,97,7,97,15,70,63,176,184,106,32,0,0,0,104,2,97,8,97,15,70,63,152,234,144,
+      0,0,0,0,104,2,97,9,97,15,70,191,172,58,70,160,0,0,0,104,2,97,10,97,15,70,
+      63,161,211,211,192,0,0,0,104,2,97,11,97,15,70,191,148,171,120,128,0,0,0,104,
+      2,97,12,97,15,70,63,180,117,214,224,0,0,0,104,2,97,13,97,15,70,191,104,
+      230,216,0,0,0,0,104,2,97,6,97,16,70,63,178,53,103,96,0,0,0,104,2,97,7,97,16,
+      70,63,170,230,232,64,0,0,0,104,2,97,8,97,16,70,191,183,45,100,192,0,0,0,
+      104,2,97,9,97,16,70,63,184,100,97,32,0,0,0,104,2,97,10,97,16,70,63,169,174,
+      254,64,0,0,0,104,2,97,11,97,16,70,191,119,121,234,0,0,0,0,104,2,97,12,97,
+      16,70,63,149,12,170,128,0,0,0,104,2,97,13,97,16,70,191,144,193,191,0,0,0,0>>.