Merge branch 'rickard/mbuf_sz/OTP-13851' into maint

* rickard/mbuf_sz/OTP-13851:
  Ensure correct mbuf_sz value in process structure

7 files changed, 74 insertions, 5 deletions

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index b3e6ebf651..ef4cdf9d5a 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -1960,6 +1960,8 @@ void process_main(void)
      ErtsMessage* msgp;
      PROCESS_MAIN_CHK_LOCKS(c_p);
 
+     ERTS_CHK_MBUF_SZ(c_p);
+
      PreFetch(0, next);
      msgp = PEEK_MESSAGE(c_p);
 
@@ -2047,6 +2049,7 @@ void process_main(void)
      }
 
      ERTS_DBG_CHK_REDS(c_p, FCALLS);
+     ERTS_CHK_MBUF_SZ(c_p);
 
      ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
      PROCESS_MAIN_CHK_LOCKS(c_p);
@@ -2583,7 +2586,9 @@ do {						\
 	c_p->fcalls = FCALLS;
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
+	ERTS_CHK_MBUF_SZ(c_p);
 	result = (*bf)(c_p, tmp_reg);
+	ERTS_CHK_MBUF_SZ(c_p);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
 	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 	PROCESS_MAIN_CHK_LOCKS(c_p);
@@ -2614,7 +2619,9 @@ do {						\
 	c_p->fcalls = FCALLS;
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
+	ERTS_CHK_MBUF_SZ(c_p);
 	result = (*bf)(c_p, tmp_reg);
+	ERTS_CHK_MBUF_SZ(c_p);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
 	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 	PROCESS_MAIN_CHK_LOCKS(c_p);
@@ -2644,7 +2651,9 @@ do {						\
 	SWAPOUT;
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
+	ERTS_CHK_MBUF_SZ(c_p);
 	result = (*bf)(c_p, reg, live);
+	ERTS_CHK_MBUF_SZ(c_p);
 	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 	ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
 	PROCESS_MAIN_CHK_LOCKS(c_p);
@@ -2685,7 +2694,9 @@ do {						\
 	SWAPOUT;
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
+	ERTS_CHK_MBUF_SZ(c_p);
 	result = (*bf)(c_p, reg, live);
+	ERTS_CHK_MBUF_SZ(c_p);
 	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 	ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
 	PROCESS_MAIN_CHK_LOCKS(c_p);
@@ -2728,7 +2739,9 @@ do {						\
 	SWAPOUT;
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
+	ERTS_CHK_MBUF_SZ(c_p);
 	result = (*bf)(c_p, reg, live);
+	ERTS_CHK_MBUF_SZ(c_p);
 	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 	ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
 	PROCESS_MAIN_CHK_LOCKS(c_p);
@@ -2766,7 +2779,9 @@ do {						\
 	c_p->fcalls = FCALLS;
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
+	ERTS_CHK_MBUF_SZ(c_p);
 	result = (*bf)(c_p, tmp_reg);
+	ERTS_CHK_MBUF_SZ(c_p);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
 	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 	PROCESS_MAIN_CHK_LOCKS(c_p);
@@ -2793,7 +2808,9 @@ do {						\
 	bf = (BifFunction) Arg(0);
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
+	ERTS_CHK_MBUF_SZ(c_p);
 	result = (*bf)(c_p, tmp_reg);
+	ERTS_CHK_MBUF_SZ(c_p);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
 	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 	PROCESS_MAIN_CHK_LOCKS(c_p);
@@ -2843,7 +2860,9 @@ do {						\
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
 	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 	live_hf_end = c_p->mbuf;
+	ERTS_CHK_MBUF_SZ(c_p);
 	result = (*bf)(c_p, reg, I);
+	ERTS_CHK_MBUF_SZ(c_p);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
 	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 	ERTS_HOLE_CHECK(c_p);
@@ -3556,11 +3575,13 @@ do {						\
 		ASSERT(c_p->scheduler_data);
 #endif
 		live_hf_end = c_p->mbuf;
+		ERTS_CHK_MBUF_SZ(c_p);
 		erts_pre_nif(&env, c_p, (struct erl_module_nif*)I[2], NULL);
 		nif_bif_result = (*fp)(&env, bif_nif_arity, reg);
 		if (env.exception_thrown)
 		    nif_bif_result = THE_NON_VALUE;
 		erts_post_nif(&env);
+		ERTS_CHK_MBUF_SZ(c_p);
 
 		PROCESS_MAIN_CHK_LOCKS(c_p);
 		ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
@@ -3612,7 +3633,9 @@ do {						\
 		Eterm (*bf)(Process*, Eterm*, BeamInstr*) = vbf;
 		ASSERT(!ERTS_PROC_IS_EXITING(c_p));
 		live_hf_end = c_p->mbuf;
+		ERTS_CHK_MBUF_SZ(c_p);
 		nif_bif_result = (*bf)(c_p, reg, I);
+		ERTS_CHK_MBUF_SZ(c_p);
 		ASSERT(!ERTS_PROC_IS_EXITING(c_p) ||
 		       is_non_value(nif_bif_result));
 		ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 8b0dd9a5a6..75f8ebefbd 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -517,6 +517,8 @@ delay_garbage_collection(Process *p, ErlHeapFragment *live_hf_end, int need, int
 	erts_proc_sched_data((p))->virtual_reds += vreds;
     }
 
+    ERTS_CHK_MBUF_SZ(p);
+
     ASSERT(CONTEXT_REDS >= erts_proc_sched_data(p)->virtual_reds);
     return reds_left;
 }
@@ -527,6 +529,8 @@ young_gen_usage(Process *p)
     Uint hsz;
     Eterm *aheap;
 
+    ERTS_CHK_MBUF_SZ(p);
+
     hsz = p->mbuf_sz;
 
     if (p->flags & F_ON_HEAP_MSGQ) {
@@ -590,6 +594,8 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
     DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE);
 #endif
 
+    ERTS_CHK_MBUF_SZ(p);
+
     ASSERT(CONTEXT_REDS - ERTS_REDS_LEFT(p, fcalls)
 	   >= erts_proc_sched_data(p)->virtual_reds);
 
diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index 91e06cde2d..e4c696ae3b 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -1842,3 +1842,20 @@ void erts_factory_undo(ErtsHeapFactory* factory)
     factory->heap_frags = NULL;
 #endif
 }
+
+Uint
+erts_mbuf_size(Process *p)
+{
+    Uint sz = 0;
+    ErlHeapFragment* bp;
+    ErtsMessage* mp;
+
+    for (bp = p->mbuf; bp; bp = bp->next)
+	sz += bp->used_size;
+
+    for (mp = p->msg_frag; mp; mp = mp->next)
+	for (bp = erts_message_to_heap_frag(mp); bp; bp = bp->next)
+	    sz += bp->used_size;
+
+    return sz;
+}
diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h
index 6df969367b..42ed14e69c 100644
--- a/erts/emulator/beam/erl_message.h
+++ b/erts/emulator/beam/erl_message.h
@@ -499,4 +499,15 @@ erts_msgq_replace_msg_ref(ErlMessageQueue *msgq, ErtsMessage *newp, ErtsMessage
 
 #endif
 
+Uint erts_mbuf_size(Process *p);
+#if defined(DEBUG) || 0
+#  define ERTS_CHK_MBUF_SZ(P)				\
+    do {						\
+	Uint actual_mbuf_sz__ = erts_mbuf_size((P));	\
+	ERTS_ASSERT((P)->mbuf_sz >= actual_mbuf_sz__);	\
+    } while (0)
+#else
+#  define ERTS_CHK_MBUF_SZ(P) ((void) 1)
+#endif
+
 #endif
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index ef2fb93106..27392a5996 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -156,7 +156,9 @@ static Eterm* alloc_heap_heavy(ErlNifEnv* env, size_t need, Eterm* hp)
 	HEAP_TOP(env->proc) = env->hp;
     }
     else {
-	env->heap_frag->used_size = hp - env->heap_frag->mem;
+	Uint usz = env->hp - env->heap_frag->mem;
+	env->proc->mbuf_sz += usz - env->heap_frag->used_size;
+	env->heap_frag->used_size = usz;
 	ASSERT(env->heap_frag->used_size <= env->heap_frag->alloc_size);
     }
     hp = erts_heap_alloc(env->proc, need, MIN_HEAP_FRAG_SZ);
@@ -308,11 +310,14 @@ static void full_flush_env(ErlNifEnv* env)
 	    HEAP_TOP(c_p) = env->hp;
 	}
 	else {
+	    Uint usz;
 	    ASSERT(env->hp_end != HEAP_LIMIT(c_p));
 	    ASSERT(env->hp_end - env->hp <= env->heap_frag->alloc_size);
 
 	    HEAP_TOP(c_p) = HEAP_TOP(env->proc);
-	    env->heap_frag->used_size = env->hp - env->heap_frag->mem;
+	    usz = env->hp - env->heap_frag->mem;
+	    env->proc->mbuf_sz += usz - env->heap_frag->used_size;
+	    env->heap_frag->used_size = usz;
 
 	    ASSERT(env->heap_frag->used_size <= env->heap_frag->alloc_size);
 
@@ -394,9 +399,12 @@ static void flush_env(ErlNifEnv* env)
 	HEAP_TOP(env->proc) = env->hp;
     }
     else {
+	Uint usz;
 	ASSERT(env->hp_end != HEAP_LIMIT(env->proc));
 	ASSERT(env->hp_end - env->hp <= env->heap_frag->alloc_size);
-	env->heap_frag->used_size = env->hp - env->heap_frag->mem;
+	usz = env->hp - env->heap_frag->mem;
+	env->proc->mbuf_sz += usz - env->heap_frag->used_size;
+	env->heap_frag->used_size = usz;
 	ASSERT(env->heap_frag->used_size <= env->heap_frag->alloc_size);
     }
 }
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 109d66012a..3347a7a60e 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -1328,10 +1328,13 @@ ERTS_GLB_INLINE void erts_heap_frag_shrink(Process* p, Eterm* hp);
 ERTS_GLB_INLINE void erts_heap_frag_shrink(Process* p, Eterm* hp)
 {
     ErlHeapFragment* hf = MBUF(p);
+    Uint sz;
 
     ASSERT(hf!=NULL && (hp - hf->mem < hf->alloc_size));
 
-    hf->used_size = hp - hf->mem;
+    sz = hp - hf->mem;
+    p->mbuf_sz -= hf->used_size - sz;
+    hf->used_size = sz;
 }	
 #endif /* inline */
 
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 85647b8500..6786657faf 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -102,6 +102,7 @@ erts_heap_alloc(Process* p, Uint need, Uint xtra)
     if (bp != NULL && need <= (bp->alloc_size - bp->used_size)) {
 	Eterm* ret = bp->mem + bp->used_size;
 	bp->used_size += need;
+	p->mbuf_sz += need;
 	return ret;
     }
 #ifdef DEBUG
@@ -124,7 +125,7 @@ erts_heap_alloc(Process* p, Uint need, Uint xtra)
     MBUF(p) = bp;
     bp->alloc_size = n;
     bp->used_size = need;
-    MBUF_SIZE(p) += n;
+    MBUF_SIZE(p) += need;
     bp->off_heap.first = NULL;
     bp->off_heap.overhead = 0;
     return bp->mem;