erts: Fix bug in enif_whereis_pid/port

that could cause heap corruption if
whereis lookup conflicts with other register updater
AND other thread sends on-heap message while main lock is released.

Also improved enif_whereis from dirty nifs by passing c_p as NULL.

2 files changed, 17 insertions, 12 deletions

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 4815e5e7bb..cdce6abafd 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -889,26 +889,27 @@ static Eterm call_whereis(ErlNifEnv *env, Eterm name)
     Process *c_p;
     Eterm res;
     int scheduler;
-    int unlock;
 
     execution_state(env, &c_p, &scheduler);
     ASSERT((c_p && scheduler) || (!c_p && !scheduler));
 
-    unlock = 0;
     if (scheduler < 0) {
         /* dirty scheduler */
         if (ERTS_PROC_IS_EXITING(c_p))
             return 0;
 
-        if (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC) {
-            erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
-            unlock = 1;
-        }
+        if (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC)
+            c_p = NULL; /* as we don't have main lock */
     }
-    res = erts_whereis_name_to_id(c_p, name);
 
-    if (unlock)
-        erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
+
+    if (c_p) {
+         /* main lock may be released below and c_p->htop updated by others */
+        flush_env(env);
+    }
+    res = erts_whereis_name_to_id(c_p, name);
+    if (c_p)
+        cache_env(env);
 
     return res;
 }
diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl
index 05c250125d..0337274178 100644
--- a/erts/emulator/test/nif_SUITE.erl
+++ b/erts/emulator/test/nif_SUITE.erl
@@ -2886,11 +2886,15 @@ nif_whereis_parallel(Config) when is_list(Config) ->
     true = lists:all(PidReg, Procs),
 
     %% tell them all to 'fire' as fast as we can
-    [P ! {Ref, send_proc} || {_, P, _} <- Procs],
+    repeat(10, fun(_) ->
+                       [P ! {Ref, send_proc} || {_, P, _} <- Procs]
+               end, void),
 
     %% each gets forwarded through two processes
-    true = lists:all(RecvNum, NSeq),
-    true = lists:all(RecvNum, NSeq),
+    repeat(10, fun(_) ->
+                       true = lists:all(RecvNum, NSeq),
+                       true = lists:all(RecvNum, NSeq)
+               end, void),
 
     %% tell them all to 'quit' by name
     [N ! {Ref, quit} || {N, _, _} <- Procs],