erts: Yield after trapping term_to_binary if gc has been ordered

or if "too much" offheap binaries has been built

3 files changed, 11 insertions, 10 deletions

diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index f5893f9291..39bbd6b182 100755
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -3603,8 +3603,9 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
 	    default: BIF_ERROR(BIF_P, BADARG); break;
 	    }
  
-	    res = erts_set_gc_state(BIF_P, enable);
-	    BIF_RET(res ? am_true : am_false);
+            res = (BIF_P->flags & F_DISABLE_GC) ? am_false : am_true;
+	    erts_set_gc_state(BIF_P, enable);
+	    BIF_RET(res);
 	}
 	else if (ERTS_IS_ATOM_STR("send_fake_exit_signal", BIF_ARG_1)) {
 	    /* Used by signal_SUITE (emulator) */
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 13b18e9e0e..b88c871ffc 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -8271,25 +8271,22 @@ save_gc_task(Process *c_p, ErtsProcSysTask *st, int prio)
 int
 erts_set_gc_state(Process *c_p, int enable)
 {
-    int res;
     ErtsProcSysTaskQs *dgc_tsk_qs;
     ASSERT(c_p == erts_get_current_process());
     ASSERT((ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS)
 	   & erts_smp_atomic32_read_nob(&c_p->state));
     ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(c_p));
 
-    res = !(c_p->flags & F_DISABLE_GC);
-
     if (!enable) {
 	c_p->flags |= F_DISABLE_GC;
-	return res;
+	return 0;
     }
 
     c_p->flags &= ~F_DISABLE_GC;
 
     dgc_tsk_qs = ERTS_PROC_GET_DELAYED_GC_TASK_QS(c_p);
     if (!dgc_tsk_qs)
-	return res;
+	return 0;
 
     /* Move delayed gc tasks into sys tasks queues. */
 
@@ -8387,7 +8384,7 @@ erts_set_gc_state(Process *c_p, int enable)
     if (dgc_tsk_qs)
 	proc_sys_task_queues_free(dgc_tsk_qs);
 
-    return res;
+    return 1;
 }
 
 void
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index d5f3b19b82..7dc7ba6f98 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -1058,8 +1058,11 @@ static BIF_RETTYPE term_to_binary_trap_1(BIF_ALIST_1)
 	ASSERT(BIF_P->flags & F_DISABLE_GC);
 	BIF_TRAP1(&term_to_binary_trap_export,BIF_P,res);
     } else {
-	erts_set_gc_state(BIF_P, 1);
-	BIF_RET(res);
+        if (erts_set_gc_state(BIF_P, 1)
+            || MSO(BIF_P).overhead > BIN_VHEAP_SZ(BIF_P))
+            ERTS_BIF_YIELD_RETURN(BIF_P, res);
+        else
+            BIF_RET(res);
     }
 }