check_process_code: Sweep HiPE stack for literals

Because check_process_code neglected checking the HiPE stack for
references to the literal area, such references would survive the purge
and subsequent deletion of a module and its literal area. These dangling
references would then cause incorrect behaviour or even hard crashes of
the VM.

By simply adding a scan of the HiPE stack to check_process_code and
erts_garbage_collect_literals, this problem is fixed.

In order to support full stack walks without deleting the graylimit
trap, a new stack walking interface function,
nstack_walk_init_sdesc_ignore_trap() was introduced.

4 files changed, 152 insertions, 0 deletions

diff --git a/erts/emulator/hipe/hipe_gc.c b/erts/emulator/hipe/hipe_gc.c
index 566c65882e..68c65dea27 100644
--- a/erts/emulator/hipe/hipe_gc.c
+++ b/erts/emulator/hipe/hipe_gc.c
@@ -237,3 +237,122 @@ void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop)
     }
     abort();
 }
+
+Eterm *sweep_literals_nstack(Process *p, Eterm *old_htop, char *area,
+			     Uint area_size)
+{
+    /* known nstack walk state */
+    Eterm *nsp;
+    Eterm *nsp_end;
+    const struct sdesc *sdesc;
+    /* arch-specific nstack walk state */
+    struct nstack_walk_state walk_state;
+
+    ASSERT(!p->hipe.gc_is_unsafe);
+
+    if (!p->hipe.nstack) {
+	ASSERT(!p->hipe.nsp && !p->hipe.nstend);
+	return old_htop;
+    }
+    if (!nstack_walk_init_check(p))
+	return old_htop;
+
+    ASSERT(p->hipe.nsp && p->hipe.nstend);
+    nsp = nstack_walk_nsp_begin(p);
+    nsp_end = nstack_walk_nsp_end(p);
+    sdesc = nstack_walk_init_sdesc_ignore_trap(p, &walk_state);
+
+    while (!nstack_walk_nsp_reached_end(nsp, nsp_end)) {
+	unsigned long ra;
+	unsigned sdesc_size = nstack_walk_frame_size(sdesc);
+	unsigned i = 0;
+	unsigned mask = sdesc->livebits[0];
+	for (;;) {
+	    if (mask & 1) {
+		Eterm *nsp_i = nstack_walk_frame_index(nsp, i);
+		Eterm gval = *nsp_i;
+		if (is_boxed(gval)) {
+		    Eterm *ptr = boxed_val(gval);
+		    Eterm val = *ptr;
+		    if (IS_MOVED_BOXED(val)) {
+			ASSERT(is_boxed(val));
+			*nsp_i = val;
+		    } else if (ErtsInArea(ptr, area, area_size)) {
+			MOVE_BOXED(ptr, val, old_htop, nsp_i);
+		    }
+		} else if (is_list(gval)) {
+		    Eterm *ptr = list_val(gval);
+		    Eterm val = *ptr;
+		    if (IS_MOVED_CONS(val)) {
+			*nsp_i = ptr[1];
+		    } else if (ErtsInArea(ptr, area, area_size)) {
+			MOVE_CONS(ptr, val, old_htop, nsp_i);
+		    }
+		}
+	    }
+	    if (++i >= sdesc_size)
+		break;
+	    if (i & 31)
+		mask >>= 1;
+	    else
+		mask = sdesc->livebits[i >> 5];
+	}
+	ra = nstack_walk_frame_ra(nsp, sdesc);
+	if (ra == (unsigned long)nbif_stack_trap_ra)
+	    ra = (unsigned long)p->hipe.ngra;
+	sdesc = hipe_find_sdesc(ra);
+	nsp = nstack_walk_next_frame(nsp, sdesc_size);
+    }
+    return old_htop;
+}
+
+int
+nstack_any_heap_ref_ptrs(Process *rp, char* mod_start, Uint mod_size)
+{
+    Eterm *nsp;
+    Eterm *nsp_end;
+    const struct sdesc *sdesc;
+    /* arch-specific nstack walk state */
+    struct nstack_walk_state walk_state;
+
+    ASSERT(!rp->hipe.gc_is_unsafe);
+
+    if (!rp->hipe.nstack || !nstack_walk_init_check(rp)) return 0;
+    ASSERT(rp->hipe.nsp && rp->hipe.nstend);
+    nsp = nstack_walk_nsp_begin(rp);
+    nsp_end = nstack_walk_nsp_end(rp);
+    sdesc = nstack_walk_init_sdesc_ignore_trap(rp, &walk_state);
+
+    while (!nstack_walk_nsp_reached_end(nsp, nsp_end)) {
+	unsigned long ra;
+	unsigned sdesc_size = nstack_walk_frame_size(sdesc);
+	unsigned i = 0;
+	unsigned mask = sdesc->livebits[0];
+	for (;;) {
+	    if (mask & 1) {
+		Eterm *nsp_i = nstack_walk_frame_index(nsp, i);
+		Eterm val = *nsp_i;
+		switch (primary_tag(val)) {
+		case TAG_PRIMARY_BOXED:
+		case TAG_PRIMARY_LIST:
+		    if (ErtsInArea(val, mod_start, mod_size)) {
+			return 1;
+		    }
+		    break;
+		}
+	    }
+	    if (++i >= sdesc_size)
+		break;
+	    if (i & 31)
+		mask >>= 1;
+	    else
+		mask = sdesc->livebits[i >> 5];
+	}
+	ra = nstack_walk_frame_ra(nsp, sdesc);
+	if (ra == (unsigned long)nbif_stack_trap_ra)
+	    ra = (unsigned long)rp->hipe.ngra;
+	sdesc = hipe_find_sdesc(ra);
+	nsp = nstack_walk_next_frame(nsp, sdesc_size);
+    }
+    return 0;
+}
diff --git a/erts/emulator/hipe/hipe_risc_gc.h b/erts/emulator/hipe/hipe_risc_gc.h
index 315f8e7f9f..09568c140e 100644
--- a/erts/emulator/hipe/hipe_risc_gc.h
+++ b/erts/emulator/hipe/hipe_risc_gc.h
@@ -51,6 +51,19 @@ nstack_walk_init_sdesc(const Process *p, struct nstack_walk_state *state)
     return sdesc;
 }
 
+static inline const struct sdesc*
+nstack_walk_init_sdesc_ignore_trap(const Process *p,
+				   struct nstack_walk_state *state)
+{
+    unsigned long ra = (unsigned long)p->hipe.nra;
+    const struct sdesc *sdesc;
+    if (ra == (unsigned long)&nbif_stack_trap_ra)
+	ra = (unsigned long)p->hipe.ngra;
+    sdesc = hipe_find_sdesc(ra);
+    state->sdesc0 = sdesc;
+    return sdesc;
+}
+
 static inline void nstack_walk_update_trap(Process *p, const struct sdesc *sdesc0)
 {
     Eterm *nsp = p->hipe.nsp;
diff --git a/erts/emulator/hipe/hipe_stack.h b/erts/emulator/hipe/hipe_stack.h
index 4ea7d5c031..afa0ed4256 100644
--- a/erts/emulator/hipe/hipe_stack.h
+++ b/erts/emulator/hipe/hipe_stack.h
@@ -131,5 +131,8 @@ static __inline__ void hipe_check_nstack(Process *p, unsigned nwords)
  */
 extern Eterm *fullsweep_nstack(Process *p, Eterm *n_htop);
 extern void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop);
+extern Eterm *sweep_literals_nstack(Process *p, Eterm *n_htop, char *area,
+				    Uint area_size);
+extern int nstack_any_heap_ref_ptrs(Process *, char* mod_start, Uint mod_size);
 
 #endif /* HIPE_STACK_H */
diff --git a/erts/emulator/hipe/hipe_x86_gc.h b/erts/emulator/hipe/hipe_x86_gc.h
index c22b28c2d5..00fe03d8f9 100644
--- a/erts/emulator/hipe/hipe_x86_gc.h
+++ b/erts/emulator/hipe/hipe_x86_gc.h
@@ -81,6 +81,23 @@ nstack_walk_init_sdesc(const Process *p, struct nstack_walk_state *state)
 #endif
 }
 
+static inline const struct sdesc*
+nstack_walk_init_sdesc_ignore_trap(const Process *p,
+				   struct nstack_walk_state *state)
+{
+#ifdef SKIP_YOUNGEST_FRAME
+    unsigned long ra = p->hipe.nsp[0];
+    const struct sdesc *sdesc;
+    if (ra == (unsigned long)nbif_stack_trap_ra)
+	ra = (unsigned long)p->hipe.ngra;
+    sdesc = hipe_find_sdesc(ra);
+    state->sdesc0 = sdesc;
+    return sdesc;
+#else
+    return nstack_walk_init_sdesc(p, state);
+#endif
+}
+
 static inline void nstack_walk_update_trap(Process *p, const struct sdesc *sdesc0)
 {
 #ifdef SKIP_YOUNGEST_FRAME