erts: Fix bug in enif_whereis_pid/port

that could cause heap corruption if
whereis lookup conflicts with other register updater
AND other thread sends on-heap message while main lock is released.

Also improved enif_whereis from dirty nifs by passing c_p as NULL.

1 files changed, 10 insertions, 9 deletions

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 4815e5e7bb..cdce6abafd 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -889,26 +889,27 @@ static Eterm call_whereis(ErlNifEnv *env, Eterm name)
     Process *c_p;
     Eterm res;
     int scheduler;
-    int unlock;
 
     execution_state(env, &c_p, &scheduler);
     ASSERT((c_p && scheduler) || (!c_p && !scheduler));
 
-    unlock = 0;
     if (scheduler < 0) {
         /* dirty scheduler */
         if (ERTS_PROC_IS_EXITING(c_p))
             return 0;
 
-        if (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC) {
-            erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
-            unlock = 1;
-        }
+        if (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC)
+            c_p = NULL; /* as we don't have main lock */
     }
-    res = erts_whereis_name_to_id(c_p, name);
 
-    if (unlock)
-        erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
+
+    if (c_p) {
+         /* main lock may be released below and c_p->htop updated by others */
+        flush_env(env);
+    }
+    res = erts_whereis_name_to_id(c_p, name);
+    if (c_p)
+        cache_env(env);
 
     return res;
 }