Merge branch 'sverker/ets_select_hang/OTP-8732' into dev

* sverker/ets_select_hang/OTP-8732:
  Fix ets:select hanging on ordered_set with empty list as key.

2 files changed, 18 insertions, 10 deletions

diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c
index b6b3cabafe..96e577ed90 100644
--- a/erts/emulator/beam/erl_db_tree.c
+++ b/erts/emulator/beam/erl_db_tree.c
@@ -1081,11 +1081,12 @@ static int db_select_continue_tree(Process *p,
 static int db_select_tree(Process *p, DbTable *tbl, 
 			  Eterm pattern, int reverse, Eterm *ret)
 {
+    /* Strategy: Traverse backwards to build resulting list from tail to head */
     DbTableTree *tb = &tbl->tree;
     DbTreeStack* stack;
     struct select_context sc;
     struct mp_info mpi;
-    Eterm lastkey = NIL;
+    Eterm lastkey = THE_NON_VALUE;
     Eterm key;
     Eterm continuation;
     unsigned sz;
@@ -1293,7 +1294,7 @@ static int db_select_count_tree(Process *p, DbTable *tbl,
     DbTreeStack* stack;
     struct select_count_context sc;
     struct mp_info mpi;
-    Eterm lastkey = NIL;
+    Eterm lastkey = THE_NON_VALUE;
     Eterm key;
     Eterm continuation;
     unsigned sz;
@@ -1395,7 +1396,7 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl,
     DbTreeStack* stack;
     struct select_context sc;
     struct mp_info mpi;
-    Eterm lastkey = NIL;
+    Eterm lastkey = THE_NON_VALUE;
     Eterm key;
     Eterm continuation;
     unsigned sz;
@@ -1636,7 +1637,7 @@ static int db_select_delete_tree(Process *p, DbTable *tbl,
     DbTableTree *tb = &tbl->tree;
     struct select_delete_context sc;
     struct mp_info mpi;
-    Eterm lastkey = NIL;
+    Eterm lastkey = THE_NON_VALUE;
     Eterm key;
     Eterm continuation;
     unsigned sz;
@@ -2628,7 +2629,7 @@ static void traverse_backwards(DbTableTree *tb,
 {
     TreeDbTerm *this, *next;
 
-    if (lastkey == NIL) {
+    if (lastkey == THE_NON_VALUE) {
 	stack->pos = stack->slot = 0;
 	if (( this = tb->root ) == NULL) {
 	    return;
@@ -2666,7 +2667,7 @@ static void traverse_forward(DbTableTree *tb,
 {
     TreeDbTerm *this, *next;
 
-    if (lastkey == NIL) {
+    if (lastkey == THE_NON_VALUE) {
 	stack->pos = stack->slot = 0;
 	if (( this = tb->root ) == NULL) {
 	    return;
diff --git a/lib/stdlib/test/ets_SUITE.erl b/lib/stdlib/test/ets_SUITE.erl
index 13c87ca005..fe8e481121 100644
--- a/lib/stdlib/test/ets_SUITE.erl
+++ b/lib/stdlib/test/ets_SUITE.erl
@@ -63,7 +63,8 @@
 	 meta_lookup_unnamed_read/1, meta_lookup_unnamed_write/1, 
 	 meta_lookup_named_read/1, meta_lookup_named_write/1,
 	 meta_newdel_unnamed/1, meta_newdel_named/1]).
--export([smp_insert/1, smp_fixed_delete/1, smp_unfix_fix/1, smp_select_delete/1, otp_8166/1]).
+-export([smp_insert/1, smp_fixed_delete/1, smp_unfix_fix/1, smp_select_delete/1,
+         otp_8166/1, otp_8732/1]).
 -export([exit_large_table_owner/1,
 	 exit_many_large_table_owner/1,
 	 exit_many_tables_owner/1,
@@ -129,7 +130,7 @@ all(suite) ->
      t_select_delete, t_ets_dets, memory,
      t_bucket_disappears,
      select_fail,t_insert_new, t_repair_continuation, otp_5340, otp_6338,
-     otp_6842_select_1000, otp_7665,
+     otp_6842_select_1000, otp_7665, otp_8732,
      meta_wb,
      grow_shrink, grow_pseudo_deleted, shrink_pseudo_deleted,
      meta_smp,
@@ -5010,8 +5011,14 @@ verify_table_load(T) ->
 	       end.
     
 		  
-    
-    
+otp_8732(doc) -> ["ets:select on a tree with NIL key object"];
+otp_8732(Config) when is_list(Config) ->
+    Tab = ets:new(noname,[ordered_set]),
+    filltabstr(Tab,999),
+    ets:insert(Tab,{[],"nasty NIL object"}),
+    ?line [] = ets:match(Tab,{'_',nomatch}), %% Will hang if bug not fixed
+    ok.
+
 
 smp_select_delete(suite) -> [];
 smp_select_delete(doc) ->