Merge branch 'pan/unregister_error' into dev

* pan/unregister_error:
  Add possibly missing process lock before unregistering oneself
  Keep process lock over trace of unregister

OTP-8663 Unregister may crash VM

Extreme combinations of register/unregister in a highly parallell SMP
application could crash the VM. The error is corrected.

2 files changed, 28 insertions, 13 deletions

diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index 8addfcf5ad..381a182e39 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -1941,11 +1941,13 @@ trace_proc(Process *c_p, Process *t_p, Eterm what, Eterm data)
     Eterm* hp;
     int need;
 
+    ERTS_SMP_LC_ASSERT((erts_proc_lc_my_proc_locks(t_p) != 0) || erts_is_system_blocked(0));
     if (is_internal_port(t_p->tracer_proc)) {
 #define LOCAL_HEAP_SIZE (5+5)
 	DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
 	UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
 
+
 	hp = local_heap;
 	mess = TUPLE4(hp, am_trace, t_p->id, what, data);
 	hp += 5;
@@ -2727,6 +2729,8 @@ trace_port(Port *t_p, Eterm what, Eterm data) {
     Eterm mess;
     Eterm* hp;
 
+    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(t_p) || erts_is_system_blocked(0));
+
     if (is_internal_port(t_p->tracer_proc)) {
 #define LOCAL_HEAP_SIZE (5+5)
 	DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
diff --git a/erts/emulator/beam/register.c b/erts/emulator/beam/register.c
index 964c10a380..900ebcbbf7 100644
--- a/erts/emulator/beam/register.c
+++ b/erts/emulator/beam/register.c
@@ -476,8 +476,9 @@ int erts_unregister_name(Process *c_p,
      *           on c_prt.
      */
 
-    if (!c_p)
+    if (!c_p) {
 	c_p_locks = 0;
+    }
     current_c_p_locks = c_p_locks;
 
  restart:
@@ -489,9 +490,15 @@ int erts_unregister_name(Process *c_p,
     if (is_non_value(name)) {
 	/* Unregister current process name */
 	ASSERT(c_p);
-	if (c_p->reg)
+#ifdef ERTS_SMP
+	if (current_c_p_locks != c_p_locks) {
+	    erts_smp_proc_lock(c_p, c_p_locks);
+	    current_c_p_locks = c_p_locks;
+	}
+#endif
+	if (c_p->reg) {
 	    r.name = c_p->reg->name;
-	else {
+	} else {
 	    /* Name got unregistered while main lock was released */
 	    res = 0;
 	    goto done;
@@ -533,24 +540,25 @@ int erts_unregister_name(Process *c_p,
 	    }
 
 	} else if (rp->p) {
-	    Process* p = rp->p;
+
 #ifdef ERTS_SMP
 	    erts_proc_safelock(c_p,
 			       current_c_p_locks,
 			       c_p_locks,
 			       rp->p,
-			       0,
+			       (c_p == rp->p) ?  current_c_p_locks : 0,
 			       ERTS_PROC_LOCK_MAIN);
 	    current_c_p_locks = c_p_locks;
 #endif
-	    p->reg = NULL;
+	    rp->p->reg = NULL;
+	    if (IS_TRACED_FL(rp->p, F_TRACE_PROCS)) {
+		trace_proc(c_p, rp->p, am_unregister, r.name);
+	    }
 #ifdef ERTS_SMP
-	    if (rp->p != c_p)
+	    if (rp->p != c_p) {
 		erts_smp_proc_unlock(rp->p, ERTS_PROC_LOCK_MAIN);
-#endif
-	    if (IS_TRACED_FL(p, F_TRACE_PROCS)) {
-		trace_proc(c_p, p, am_unregister, r.name);
 	    }
+#endif
 	}
 	hash_erase(&process_reg, (void*) &r);
 	res = 1;
@@ -560,14 +568,17 @@ int erts_unregister_name(Process *c_p,
 
     reg_write_unlock();
     if (c_prt != port) {
-	if (port)
+	if (port) {
 	    erts_smp_port_unlock(port);
-	if (c_prt)
+	}
+	if (c_prt) {
 	    erts_smp_port_lock(c_prt);
+	}
     }
 #ifdef ERTS_SMP
-    if (c_p && !current_c_p_locks)
+    if (c_p && !current_c_p_locks) {
 	erts_smp_proc_lock(c_p, c_p_locks);
+    }
 #endif
     return res;
 }