Merge branch 'egil/fix-maps-copy-shallow'

* egil/fix-maps-copy-shallow:
  erts: Make hashmap_get halfword safe
  erts: Fix ETS db_has_variable check for large Maps
  stdlib: Strengthen ETS Maps tests
  erts: Fix copy shallow for large Maps
  stdlib: Strengthen ETS Maps tests
  erts: ETS ordered_set cannot use it's optimization with Maps
  stdlib: Strengthen ETS Maps tests
  stdlib: Refactor away ?line macro

6 files changed, 114 insertions, 31 deletions

diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index 4d12dae787..850606dd86 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -608,11 +608,6 @@ Eterm copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 		    erts_refc_inc(&funp->fe->refc, 2);
 		}
 		goto off_heap_common;
-
-	    case MAP_SUBTAG:
-		*hp++ = *tp++;
-		sz--;
-		break;
 	    case EXTERNAL_PID_SUBTAG:
 	    case EXTERNAL_PORT_SUBTAG:
 	    case EXTERNAL_REF_SUBTAG:
@@ -648,7 +643,6 @@ Eterm copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 	}
     }
     *hpp = hp;
-
     return res;
 }
 
diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c
index d90af46659..c7bccc78c3 100644
--- a/erts/emulator/beam/erl_db_tree.c
+++ b/erts/emulator/beam/erl_db_tree.c
@@ -2716,7 +2716,7 @@ static int key_given(DbTableTree *tb, Eterm pattern, TreeDbTerm **ret,
 	*ret = this;
 	return 1;
     } else if (partly_bound != NULL && key != am_Underscore && 
-	       db_is_variable(key) < 0)
+	       db_is_variable(key) < 0 && !db_has_map(key))
 	*partly_bound = key;
 	
     return 0;
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index 0fb1c397c9..c6c3c55a7e 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -2039,7 +2039,7 @@ restart:
             break;
         case matchKey:
             t = (Eterm) *pc++;
-            tp = erts_maps_get_rel(t, make_flatmap_rel(ep, base), base);
+            tp = erts_maps_get_rel(t, make_boxed_rel(ep, base), base);
             if (!tp) {
                 FAIL();
             }
@@ -3347,6 +3347,37 @@ int db_is_variable(Eterm obj)
     return N;
 }
 
+/* check if node is (or contains) a map
+ * return 1 if node contains a map
+ * return 0 otherwise
+ */
+
+int db_has_map(Eterm node) {
+    DECLARE_ESTACK(s);
+
+    ESTACK_PUSH(s,node);
+    while (!ESTACK_ISEMPTY(s)) {
+	node = ESTACK_POP(s);
+        if (is_list(node)) {
+	    while (is_list(node)) {
+		ESTACK_PUSH(s,CAR(list_val(node)));
+		node = CDR(list_val(node));
+	    }
+	    ESTACK_PUSH(s,node);    /* Non wellformed list or [] */
+        } else if (is_tuple(node)) {
+            Eterm *tuple = tuple_val(node);
+            int arity = arityval(*tuple);
+            while(arity--) {
+                ESTACK_PUSH(s,*(++tuple));
+            }
+        } else if is_map(node) {
+            DESTROY_ESTACK(s);
+            return 1;
+        }
+    }
+    DESTROY_ESTACK(s);
+    return 0;
+}
 
 /* check if obj is (or contains) a variable */
 /* return 1 if obj contains a variable or underscore */
@@ -3380,6 +3411,11 @@ int db_has_variable(Eterm node) {
                 while (size--) {
                     ESTACK_PUSH(s, *(values++));
                 }
+            } else if (is_map(node)) { /* other map-nodes or map-heads */
+                Eterm *ptr = hashmap_val(node);
+                int i = hashmap_bitcount(MAP_HEADER_VAL(*ptr));
+                ptr += MAP_HEADER_ARITY(*ptr);
+                while(i--) { ESTACK_PUSH(s, *++ptr); }
             }
 	    break;
 	case TAG_PRIMARY_IMMED1:
diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h
index ca206c7f58..b2d5a306cb 100644
--- a/erts/emulator/beam/erl_db_util.h
+++ b/erts/emulator/beam/erl_db_util.h
@@ -342,6 +342,7 @@ void* db_store_term(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj);
 void* db_store_term_comp(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj);
 Eterm db_copy_element_from_ets(DbTableCommon* tb, Process* p, DbTerm* obj,
 			       Uint pos, Eterm** hpp, Uint extra);
+int db_has_map(Eterm obj);
 int db_has_variable(Eterm obj);
 int db_is_variable(Eterm obj);
 void db_do_update_element(DbUpdateHandle* handle,
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 256d6a8e81..a1bd39dbc8 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -1884,7 +1884,7 @@ erts_hashmap_get(Uint32 hx, Eterm key, Eterm node)
     UseTmpHeapNoproc(2);
 
     ASSERT(is_boxed(node));
-    ptr = boxed_val(node);
+    ptr = boxed_val_rel(node, map_base);
     hdr = *ptr;
     ASSERT(is_header(hdr));
     ASSERT(is_hashmap_header_head(hdr));
@@ -1905,8 +1905,7 @@ erts_hashmap_get(Uint32 hx, Eterm key, Eterm node)
         node  = ptr[ix+1];
 
         if (is_list(node)) { /* LEAF NODE [K|V] */
-            ptr = list_val(node);
-
+            ptr = list_val_rel(node,map_base);
             res = eq_rel(CAR(ptr), map_base, key, NULL) ? &(CDR(ptr)) : NULL;
             break;
         }
@@ -1914,7 +1913,7 @@ erts_hashmap_get(Uint32 hx, Eterm key, Eterm node)
         hx = hashmap_shift_hash(th,hx,lvl,key);
 
         ASSERT(is_boxed(node));
-        ptr = boxed_val(node);
+        ptr = boxed_val_rel(node, map_base);
         hdr = *ptr;
         ASSERT(is_header(hdr));
         ASSERT(!is_hashmap_header_head(hdr));
diff --git a/lib/stdlib/test/ets_SUITE.erl b/lib/stdlib/test/ets_SUITE.erl
index 5774d774b5..41bd4af241 100644
--- a/lib/stdlib/test/ets_SUITE.erl
+++ b/lib/stdlib/test/ets_SUITE.erl
@@ -3821,46 +3821,99 @@ match_object(Config) when is_list(Config) ->
     repeat_for_opts(match_object_do).
 
 match_object_do(Opts) ->
-    ?line EtsMem = etsmem(),
-    ?line Tab = ets_new(foobar, Opts),
-    ?line fill_tab(Tab, foo),
-    ?line ets:insert(Tab, {{one, 4}, 4}),
-    ?line ets:insert(Tab,{{one,5},5}),
-    ?line ets:insert(Tab,{{two,4},4}),
-    ?line ets:insert(Tab,{{two,5},6}),
-    ?line ets:insert(Tab, {#{camembert=>cabécou},7}),
-    ?line case ets:match_object(Tab, {{one, '_'}, '$0'}) of
+    EtsMem = etsmem(),
+    Tab = ets_new(foobar, Opts),
+    fill_tab(Tab, foo),
+    ets:insert(Tab,{{one,4},4}),
+    ets:insert(Tab,{{one,5},5}),
+    ets:insert(Tab,{{two,4},4}),
+    ets:insert(Tab,{{two,5},6}),
+    ets:insert(Tab, {#{camembert=>cabécou},7}),
+    ets:insert(Tab, {#{"hi"=>"hello","wazzup"=>"awesome","1337"=>"42"},8}),
+    ets:insert(Tab, {#{"hi"=>"hello",#{"wazzup"=>3}=>"awesome","1337"=>"42"},9}),
+    ets:insert(Tab, {#{"hi"=>"hello","wazzup"=>#{"awesome"=>3},"1337"=>"42"},10}),
+    Is = lists:seq(1,100),
+    M1 = maps:from_list([{I,I}||I <- Is]),
+    M2 = maps:from_list([{I,"hi"}||I <- Is]),
+    ets:insert(Tab, {M1,11}),
+    ets:insert(Tab, {M2,12}),
+
+    case ets:match_object(Tab, {{one, '_'}, '$0'}) of
 	[{{one,5},5},{{one,4},4}] -> ok;
 	[{{one,4},4},{{one,5},5}] -> ok;
 	_ -> ?t:fail("ets:match_object() returned something funny.")
     end,
-    ?line case ets:match_object(Tab, {{two, '$1'}, '$0'}) of
+    case ets:match_object(Tab, {{two, '$1'}, '$0'}) of
 	[{{two,5},6},{{two,4},4}] -> ok;
 	[{{two,4},4},{{two,5},6}] -> ok;
 	_ -> ?t:fail("ets:match_object() returned something funny.")
     end,
-    ?line case ets:match_object(Tab, {{two, '$9'}, '$4'}) of
+    case ets:match_object(Tab, {{two, '$9'}, '$4'}) of
 	[{{two,5},6},{{two,4},4}] -> ok;
 	[{{two,4},4},{{two,5},6}] -> ok;
 	_ -> ?t:fail("ets:match_object() returned something funny.")
     end,
-    ?line case ets:match_object(Tab, {{two, '$9'}, '$22'}) of
+    case ets:match_object(Tab, {{two, '$9'}, '$22'}) of
 	[{{two,5},6},{{two,4},4}] -> ok;
 	[{{two,4},4},{{two,5},6}] -> ok;
 	_ -> ?t:fail("ets:match_object() returned something funny.")
     end,
+
     % Check that maps are inspected for variables.
-    [{#{camembert:=cabécou},7}] =
-        ets:match_object(Tab, {#{camembert=>'_'},7}),
+    [{#{camembert:=cabécou},7}] = ets:match_object(Tab, {#{camembert=>'_'},7}),
+
+    [{#{"hi":="hello",#{"wazzup"=>3}:="awesome","1337":="42"},9}] =
+        ets:match_object(Tab, {#{#{"wazzup"=>3}=>"awesome","hi"=>"hello","1337"=>"42"},9}),
+    [{#{"hi":="hello",#{"wazzup"=>3}:="awesome","1337":="42"},9}] =
+        ets:match_object(Tab, {#{#{"wazzup"=>3}=>"awesome","hi"=>"hello","1337"=>'_'},'_'}),
+    [{#{"hi":="hello","wazzup":=#{"awesome":=3},"1337":="42"},10}] =
+        ets:match_object(Tab, {#{"wazzup"=>'_',"hi"=>'_',"1337"=>'_'},10}),
+
+    %% multiple patterns
+    Pat = {{#{#{"wazzup"=>3}=>"awesome","hi"=>"hello","1337"=>'_'},'$1'},[{is_integer,'$1'}],['$_']},
+    [{#{"hi":="hello",#{"wazzup"=>3}:="awesome","1337":="42"},9}] =
+        ets:select(Tab, [Pat,Pat,Pat,Pat]),
+    case ets:match_object(Tab, {#{"hi"=>"hello","wazzup"=>'_',"1337"=>"42"},'_'}) of
+        [{#{"1337" := "42","hi" := "hello","wazzup" := "awesome"},8},
+         {#{"1337" := "42","hi" := "hello","wazzup" := #{"awesome" := 3}},10}] -> ok;
+        [{#{"1337" := "42","hi" := "hello","wazzup" := #{"awesome" := 3}},10},
+         {#{"1337" := "42","hi" := "hello","wazzup" := "awesome"},8}] -> ok;
+        _ -> ?t:fail("ets:match_object() returned something funny.")
+    end,
+    case ets:match_object(Tab, {#{"hi"=>'_'},'_'}) of
+        [{#{"1337":="42", "hi":="hello"},_},
+         {#{"1337":="42", "hi":="hello"},_},
+         {#{"1337":="42", "hi":="hello"},_}] -> ok;
+        _ -> ?t:fail("ets:match_object() returned something funny.")
+    end,
+
+    %% match large maps
+    [{#{1:=1,2:=2,99:=99,100:=100},11}] = ets:match_object(Tab, {M1,11}),
+    [{#{1:="hi",2:="hi",99:="hi",100:="hi"},12}] = ets:match_object(Tab, {M2,12}),
+    case ets:match_object(Tab, {#{1=>'_',2=>'_'},'_'}) of
+        %% only match a part of the map
+        [{#{1:=1,5:=5,99:=99,100:=100},11},{#{1:="hi",6:="hi",99:="hi"},12}] -> ok;
+        [{#{1:="hi",2:="hi",59:="hi"},12},{#{1:=1,2:=2,39:=39,100:=100},11}] -> ok;
+        _ -> ?t:fail("ets:match_object() returned something funny.")
+    end,
+    case ets:match_object(Tab, {maps:from_list([{I,'_'}||I<-Is]),'_'}) of
+        %% only match a part of the map
+        [{#{1:=1,5:=5,99:=99,100:=100},11},{#{1:="hi",6:="hi",99:="hi"},12}] -> ok;
+        [{#{1:="hi",2:="hi",59:="hi"},12},{#{1:=1,2:=2,39:=39,100:=100},11}] -> ok;
+        _ -> ?t:fail("ets:match_object() returned something funny.")
+    end,
     {'EXIT',{badarg,_}} = (catch ets:match_object(Tab, {#{'$1'=>'_'},7})),
-    % Check that unsucessful match returns an empty list.
-    ?line [] = ets:match_object(Tab, {{three,'$0'}, '$92'}),
+    Mve = maps:from_list([{list_to_atom([$$|integer_to_list(I)]),'_'}||I<-Is]),
+    {'EXIT',{badarg,_}} = (catch ets:match_object(Tab, {Mve,11})),
+
+    % Check that unsuccessful match returns an empty list.
+    [] = ets:match_object(Tab, {{three,'$0'}, '$92'}),
     % Check that '$0' equals '_'.
     Len = length(ets:match_object(Tab, '$0')),
     Len = length(ets:match_object(Tab, '_')),
-    ?line if Len > 4 -> ok end,
-    ?line true = ets:delete(Tab),
-    ?line verify_etsmem(EtsMem).
+    if Len > 4 -> ok end,
+    true = ets:delete(Tab),
+    verify_etsmem(EtsMem).
 
 match_object2(suite) -> [];
 match_object2(doc) -> ["Tests that db_match_object does not generate "