erts: Remove hashmap probabilistic heap overestimation

by adding a dynamic heap factory.

"binary_to_term" is now a hybrid solution with both
a call to decoded_size() to calculate needed heap space
AND possible dynamic allocation of more heap space
if needed for big maps.

The heap size returned from decoded_size() is guaranteed
to be sufficient for all term heap data except for hashmap
nodes. All hashmap nodes are created at the end of dec_term()
by invoking the heap factory interface that may allocate more
heap space on process heap or in fragments.

With this commit it is no longer guaranteed that a message
is confined to only one heap fragment.

1 files changed, 121 insertions, 126 deletions

diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index ccc7da265e..07a82b42d2 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -1395,31 +1395,26 @@ finalize_force_imm_drv_call(ErtsTryImmDrvCallState *sp)
 static ERTS_INLINE void
 queue_port_sched_op_reply(Process *rp,
 			  ErtsProcLocks *rp_locksp,
-			  Eterm *hp_start,
-			  Eterm *hp,
-			  Uint h_size,
-			  ErlHeapFragment* bp,
+                          ErtsHeapFactory* factory,
 			  Uint32 *ref_num,
 			  Eterm msg)
 {
-    Eterm ref = make_internal_ref(hp);
+    Eterm* hp = erts_produce_heap(factory, ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE, 0);
+    Eterm ref;
+
+    ref= make_internal_ref(hp);
     write_ref_thing(hp, ref_num[0], ref_num[1], ref_num[2]);
     hp += REF_THING_SIZE;
 
     msg = TUPLE2(hp, ref, msg);
     hp += 3;
 
-    if (!bp) {
-	HRelease(rp, hp_start + h_size, hp);
-    }
-    else {
-	Uint used_h_size = hp - hp_start;
-	ASSERT(h_size >= used_h_size);
-	if (h_size > used_h_size)
-	    bp = erts_resize_message_buffer(bp, used_h_size, &msg, 1);
-    }
+    /* SVERK: We used to call erts_resize_message_buffer here
+     * to maybe realloc message.
+     */
+    erts_factory_close(factory);
 
-    erts_queue_message(rp, rp_locksp, bp, msg, NIL);
+    erts_queue_message(rp, rp_locksp, factory->heap_frags, msg, NIL);
 }
 
 static void
@@ -1429,9 +1424,10 @@ port_sched_op_reply(Eterm to, Uint32 *ref_num, Eterm msg)
     if (rp) {
 	ErlOffHeap *ohp;
 	ErlHeapFragment* bp;
+        ErtsHeapFactory factory;
 	Eterm msg_copy;
 	Uint hsz, msg_sz;
-	Eterm *hp, *hp_start;
+	Eterm *hp;
 	ErtsProcLocks rp_locks = 0;
 
 	hsz = ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE;
@@ -1442,22 +1438,22 @@ port_sched_op_reply(Eterm to, Uint32 *ref_num, Eterm msg)
 	    hsz += msg_sz;
 	}
 
-	hp_start = hp = erts_alloc_message_heap(hsz,
+	hp = erts_alloc_message_heap(hsz,
 						&bp,
 						&ohp,
 						rp,
 						&rp_locks);
+        erts_factory_message_init(&factory, rp, hp, bp);
 	if (is_immed(msg))
 	    msg_copy = msg;
-	else
+	else {
 	    msg_copy = copy_struct(msg, msg_sz, &hp, ohp);
+            factory.hp = hp;
+        }
 
 	queue_port_sched_op_reply(rp,
 				  &rp_locks,
-				  hp_start,
-				  hp,
-				  hsz,
-				  bp,
+                                  &factory,
 				  ref_num,
 				  msg_copy);
 
@@ -3896,12 +3892,13 @@ port_sig_control(Port *prt,
 
 	if (res == ERTS_PORT_OP_DONE) {
 	    Eterm msg;
-	    Eterm *hp, *hp_start;
+	    Eterm *hp;
 	    ErlHeapFragment *bp;
 	    ErlOffHeap *ohp;
+            ErtsHeapFactory factory;
 	    Process *rp;
 	    ErtsProcLocks rp_locks = 0;
-	    Uint hsz;
+	    Uint hsz, rsz;
 	    int control_flags;
 
 	    rp = erts_proc_lookup_raw(sigdp->caller);
@@ -3910,17 +3907,19 @@ port_sig_control(Port *prt,
 
 	    control_flags = prt->control_flags;
 
-	    hsz = ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE;
-	    hsz += port_control_result_size(control_flags,
+	    rsz = port_control_result_size(control_flags,
 					    resp_bufp,
 					    &resp_size,
 					    &resp_buf[0]);
+	    hsz = rsz + ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE;
+
 
-	    hp_start = hp = erts_alloc_message_heap(hsz,
+	    hp = erts_alloc_message_heap(hsz,
 						    &bp,
 						    &ohp,
 						    rp,
 						    &rp_locks);
+            erts_factory_message_init(&factory, rp, hp, bp);
 
 	    msg = write_port_control_result(control_flags,
 					    resp_bufp,
@@ -3929,13 +3928,11 @@ port_sig_control(Port *prt,
 					    &hp,
 					    bp,
 					    ohp);
+            factory.hp = hp;
 
 	    queue_port_sched_op_reply(rp,
 				      &rp_locks,
-				      hp_start,
-				      hp,
-				      hsz,
-				      bp,
+                                      &factory,
 				      sigdp->ref,
 				      msg);
 
@@ -4222,8 +4219,6 @@ port_sig_call(Port *prt,
 	if (res == ERTS_PORT_OP_DONE) {
 	    Eterm msg;
 	    Eterm *hp;
-	    ErlHeapFragment *bp;
-	    ErlOffHeap *ohp;
 	    Process *rp;
 	    ErtsProcLocks rp_locks = 0;
 	    Sint hsz;
@@ -4234,29 +4229,31 @@ port_sig_call(Port *prt,
 
 	    hsz = erts_decode_ext_size((byte *) resp_bufp, resp_size);
 	    if (hsz >= 0) {
-		Eterm *hp_start;
+                ErlHeapFragment* bp;
+                ErlOffHeap* ohp;
+                ErtsHeapFactory factory;
 		byte *endp;
 
 		hsz += 3; /* ok tuple */
 		hsz += ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE;
 
-		hp_start = hp = erts_alloc_message_heap(hsz,
-							&bp,
-							&ohp,
-							rp,
-							&rp_locks);
+                hp = erts_alloc_message_heap(hsz,
+                                             &bp,
+                                             &ohp,
+                                             rp,
+                                             &rp_locks);
 		endp = (byte *) resp_bufp;
-		msg = erts_decode_ext(&hp, ohp, &endp);
+                erts_factory_message_init(&factory, rp, hp, bp);
+		msg = erts_decode_ext(&factory, &endp);
 		if (is_value(msg)) {
+                    hp = erts_produce_heap(&factory,
+                                           3,
+                                           ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE);
 		    msg = TUPLE2(hp, am_ok, msg);
-		    hp += 3;
 
 		    queue_port_sched_op_reply(rp,
 					      &rp_locks,
-					      hp_start,
-					      hp,
-					      hsz,
-					      bp,
+                                              &factory,
 					      sigdp->ref,
 					      msg);
 
@@ -4264,8 +4261,6 @@ port_sig_call(Port *prt,
 			erts_smp_proc_unlock(rp, rp_locks);
 		    goto done;
 		}
-		if (bp)
-		    free_message_buffer(bp);
 		if (rp_locks)
 		    erts_smp_proc_unlock(rp, rp_locks);
 	    }
@@ -4342,10 +4337,11 @@ erts_port_call(Process* c_p,
 	try_call_res = try_imm_drv_call(&try_call_state);
 	switch (try_call_res) {
 	case ERTS_TRY_IMM_DRV_CALL_OK: {
-	    Eterm *hp, *hp_end;
+            ErtsHeapFactory factory;
 	    Sint hsz;
 	    unsigned ret_flags = 0U;
 	    Eterm term;
+            Eterm* hp;
 
 	    res = call_driver_call(c_p->common.id,
 				   prt,
@@ -4365,15 +4361,14 @@ erts_port_call(Process* c_p,
 	    if (hsz < 0)
 		return ERTS_PORT_OP_BADARG;
 	    hsz += 3;
-	    hp = HAlloc(c_p, hsz);
-	    hp_end = hp + hsz;
+            erts_factory_proc_prealloc_init(&factory, c_p, hsz);
 	    endp = (byte *) resp_bufp;
-	    term = erts_decode_ext(&hp, &MSO(c_p), &endp);
+	    term = erts_decode_ext(&factory, &endp);
 	    if (term == THE_NON_VALUE)
 		return ERTS_PORT_OP_BADARG;
+            hp = erts_produce_heap(&factory,3,0);
 	    *retvalp = TUPLE2(hp, am_ok, term);
-	    hp += 3;
-	    HRelease(c_p, hp_end, hp);
+            erts_factory_close(&factory);
 	    if (resp_bufp != &resp_buf[0]
 		&& !(ret_flags & DRIVER_CALL_KEEP_BUFFER))
 		driver_free(resp_bufp);
@@ -4508,12 +4503,11 @@ port_sig_info(Port *prt,
 				    prt,
 				    sigdp->u.info.item);
 	if (is_value(value)) {
+            ErtsHeapFactory factory;
+            erts_factory_message_init(&factory, NULL, hp, bp);
 	    queue_port_sched_op_reply(rp,
 				      &rp_locks,
-				      hp_start,
-				      hp,
-				      hsz,
-				      bp,
+                                      &factory,
 				      sigdp->ref,
 				      value);
 	}
@@ -5106,22 +5100,22 @@ cleanup_b2t_states(struct b2t_states__ *b2tsp)
 static int
 driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 {
+#define HEAP_EXTRA 200
 #define ERTS_DDT_FAIL do { res = -1; goto done; } while (0)
     Uint need = 0;
     int depth = 0;
     int res;
-    Eterm *hp = NULL, *hp_start = NULL, *hp_end = NULL;
     ErlDrvTermData* ptr;
     ErlDrvTermData* ptr_end;
     DECLARE_ESTACK(stack); 
     Eterm mess = NIL;		/* keeps compiler happy */
     Process* rp = NULL;
-    ErlHeapFragment *bp = NULL;
-    ErlOffHeap *ohp;
+    ErtsHeapFactory factory;
     ErtsProcLocks rp_locks = 0;
     struct b2t_states__ b2t;
     int scheduler = 1; /* Silence erroneous warning... */
 
+    factory.mode = FACTORY_CLOSED;
     init_b2t_states(&b2t);
 
     /*
@@ -5285,9 +5279,10 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 #ifdef DEBUG
 	    b2t.org_ext[b2t.ix] = ext;
 #endif
-	    hsz = erts_binary2term_prepare(&b2t.state[b2t.ix++], ext, size);
+	    hsz = erts_binary2term_prepare(&b2t.state[b2t.ix], ext, size);
 	    if (hsz < 0)
 		ERTS_DDT_FAIL; /* Invalid data */
+	    b2t.state[b2t.ix++].heap_size = hsz;
 	    need += hsz;
 	    ptr += 2;
 	    depth++;
@@ -5297,7 +5292,7 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 	    ERTS_DDT_CHK_ENOUGH_ARGS(1);
 	    if ((int) ptr[0] < 0) ERTS_DDT_FAIL;
             if (ptr[0] > MAP_SMALL_MAP_LIMIT) {
-                need += hashmap_over_estimated_heap_size(ptr[0]);
+                need += HASHMAP_ESTIMATED_HEAP_SIZE(ptr[0]);
             } else {
                 need += MAP_HEADER_FLATMAP_SZ + 1 + 2*ptr[0];
             }
@@ -5336,8 +5331,14 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 	goto done;
     }
 
-    hp_start = hp = erts_alloc_message_heap(need, &bp, &ohp, rp, &rp_locks);
-    hp_end = hp + need;
+
+    /* We could try to copy directly to destination heap
+     * if we knew there is no big maps.
+
+    erts_alloc_message_heap(need, &bp, &ohp, rp, &rp_locks);
+    */
+    erts_factory_message_init(&factory, NULL, NULL,
+			      new_message_buffer(need));
 
     /*
      * Interpret the instructions and build the term.
@@ -5358,13 +5359,15 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 
 	case ERL_DRV_INT:  /* signed int argument */
 #if HALFWORD_HEAP
-	    mess = erts_bld_sint64(&hp, NULL, (Sint64)ptr[0]);
+	    erts_reserve_heap(&factory, BIG_NEED_SIZE(2));
+	    mess = erts_bld_sint64(&factory.hp, NULL, (Sint64)ptr[0]);
 #else
+	    erts_reserve_heap(&factory, BIG_UINT_HEAP_SIZE);
 	    if (IS_SSMALL((Sint)ptr[0]))
 		mess = make_small((Sint)ptr[0]);
 	    else {
-		mess = small_to_big((Sint)ptr[0], hp);
-		hp += BIG_UINT_HEAP_SIZE;
+		mess = small_to_big((Sint)ptr[0], factory.hp);
+		factory.hp += BIG_UINT_HEAP_SIZE;
 	    }
 #endif
 	    ptr++;
@@ -5372,25 +5375,29 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 
 	case ERL_DRV_UINT:  /* unsigned int argument */
 #if HALFWORD_HEAP
-	    mess = erts_bld_uint64(&hp, NULL, (Uint64)ptr[0]);
+	    erts_reserve_heap(&factory, BIG_NEED_FOR_BITS(64));
+	    mess = erts_bld_uint64(&factory.hp, NULL, (Uint64)ptr[0]);
 #else
+	    erts_reserve_heap(&factory, BIG_UINT_HEAP_SIZE);
 	    if (IS_USMALL(0, (Uint)ptr[0]))
 		mess = make_small((Uint)ptr[0]);
 	    else {
-		mess = uint_to_big((Uint)ptr[0], hp);
-		hp += BIG_UINT_HEAP_SIZE;
+		mess = uint_to_big((Uint)ptr[0], factory.hp);
+		factory.hp += BIG_UINT_HEAP_SIZE;
 	    }
 #endif
 	    ptr++;
 	    break;
 
 	case ERL_DRV_INT64: /* pointer to unsigned 64-bit int argument */
-	    mess = erts_bld_sint64(&hp, NULL, *((Sint64 *) ptr[0]));
+	    erts_reserve_heap(&factory, BIG_NEED_FOR_BITS(64));
+	    mess = erts_bld_sint64(&factory.hp, NULL, *((Sint64 *) ptr[0]));
 	    ptr++;
 	    break;
 
 	case ERL_DRV_UINT64: /* pointer to unsigned 64-bit int argument */
-	    mess = erts_bld_uint64(&hp, NULL, *((Uint64 *) ptr[0]));
+	    erts_reserve_heap(&factory, BIG_NEED_FOR_BITS(64));
+	    mess = erts_bld_uint64(&factory.hp, NULL, *((Uint64 *) ptr[0]));
 	    ptr++;
 	    break;
 
@@ -5407,8 +5414,8 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 	    erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) size);
 
 	    if (size <= ERL_ONHEAP_BIN_LIMIT) {
-		ErlHeapBin* hbp = (ErlHeapBin *) hp;
-		hp += heap_bin_size(size);
+		ErlHeapBin* hbp = (ErlHeapBin *) erts_produce_heap(&factory,
+								   heap_bin_size(size), HEAP_EXTRA);
 		hbp->thing_word = header_heap_bin(size);
 		hbp->size = size;
 		if (size > 0) {
@@ -5417,18 +5424,18 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 		mess = make_binary(hbp);
 	    }
 	    else {
-		ProcBin* pb = (ProcBin *) hp;
+		ProcBin* pb = (ProcBin *) erts_produce_heap(&factory,
+							    PROC_BIN_SIZE, HEAP_EXTRA);
 		driver_binary_inc_refc(b);  /* caller will free binary */
 		pb->thing_word = HEADER_PROC_BIN;
 		pb->size = size;
-		pb->next = ohp->first;
-		ohp->first = (struct erl_off_heap_header*)pb;
+		pb->next = factory.off_heap->first;
+		factory.off_heap->first = (struct erl_off_heap_header*)pb;
 		pb->val = ErlDrvBinary2Binary(b);
 		pb->bytes = ((byte*) b->orig_bytes) + offset;
 		pb->flags = 0;
 		mess =  make_binary(pb);
-		hp += PROC_BIN_SIZE;
-		OH_OVERHEAD(ohp, pb->size / sizeof(Eterm));
+		OH_OVERHEAD(factory.off_heap, pb->size / sizeof(Eterm));
 	    }
 	    ptr += 3;
 	    break;
@@ -5441,8 +5448,9 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 	    erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) size);
 
 	    if (size <= ERL_ONHEAP_BIN_LIMIT) {
-		ErlHeapBin* hbp = (ErlHeapBin *) hp;
-		hp += heap_bin_size(size);
+		ErlHeapBin* hbp = (ErlHeapBin *) erts_produce_heap(&factory,
+								   heap_bin_size(size),
+								   HEAP_EXTRA);
 		hbp->thing_word = header_heap_bin(size);
 		hbp->size = size;
 		if (size > 0) {
@@ -5457,16 +5465,16 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 		ASSERT(bufp);
 		erts_refc_init(&bp->refc, 1);
 		sys_memcpy((void *) bp->orig_bytes, (void *) bufp, size);
-		pbp = (ProcBin *) hp;
-		hp += PROC_BIN_SIZE;
+		pbp = (ProcBin *) erts_produce_heap(&factory,
+						    PROC_BIN_SIZE, HEAP_EXTRA);
 		pbp->thing_word = HEADER_PROC_BIN;
 		pbp->size = size;
-		pbp->next = ohp->first;
-		ohp->first = (struct erl_off_heap_header*)pbp;
+		pbp->next = factory.off_heap->first;
+		factory.off_heap->first = (struct erl_off_heap_header*)pbp;
 		pbp->val = bp;
 		pbp->bytes = (byte*) bp->orig_bytes;
 		pbp->flags = 0;
-		OH_OVERHEAD(ohp, pbp->size / sizeof(Eterm));
+		OH_OVERHEAD(factory.off_heap, pbp->size / sizeof(Eterm));
 		mess = make_binary(pbp);
 	    }
 	    ptr += 2;
@@ -5475,13 +5483,15 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 
 	case ERL_DRV_STRING: /* char*, length */
 	    erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) ptr[1]);
-	    mess = buf_to_intlist(&hp, (char*)ptr[0], ptr[1], NIL);
+	    erts_reserve_heap(&factory, 2*ptr[1]);
+	    mess = buf_to_intlist(&factory.hp, (char*)ptr[0], ptr[1], NIL);
 	    ptr += 2;
 	    break;
 
 	case ERL_DRV_STRING_CONS:  /* char*, length */
 	    mess = ESTACK_POP(stack);
-	    mess = buf_to_intlist(&hp, (char*)ptr[0], ptr[1], mess);
+	    erts_reserve_heap(&factory, 2*ptr[1]);
+	    mess = buf_to_intlist(&factory.hp, (char*)ptr[0], ptr[1], mess);
 	    ptr += 2;
 	    break;
 
@@ -5490,11 +5500,12 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 
 	    mess = ESTACK_POP(stack);
 	    i--;
+	    erts_reserve_heap(&factory, 2*i);
 	    while(i > 0) {
 		Eterm hd = ESTACK_POP(stack);
 
-		mess = CONS(hp, hd, mess);
-		hp += 2;
+		mess = CONS(factory.hp, hd, mess);
+		factory.hp += 2;
 		i--;
 	    }
 	    ptr++;
@@ -5503,13 +5514,12 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 
 	case ERL_DRV_TUPLE: { /* int */
 	    int size = (int)ptr[0];
-	    Eterm* tp = hp;
+	    Eterm* tp = erts_produce_heap(&factory, size+1, HEAP_EXTRA);
 
 	    *tp = make_arityval(size);
 	    mess = make_tuple(tp);
 
 	    tp += size;   /* point at last element */
-	    hp = tp+1;    /* advance "heap" pointer */
 
 	    while(size--) {
 		*tp-- = ESTACK_POP(stack);
@@ -5525,20 +5535,22 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 
 	case ERL_DRV_FLOAT: { /* double * */
 	    FloatDef f;
+	    Eterm* fp = erts_produce_heap(&factory, FLOAT_SIZE_OBJECT, HEAP_EXTRA);
 
-	    mess = make_float(hp);
+	    mess = make_float(fp);
 	    f.fd = *((double *) ptr[0]);
             if (!erts_isfinite(f.fd))
                 ERTS_DDT_FAIL;
-            PUT_DOUBLE(f, hp);
-	    hp += FLOAT_SIZE_OBJECT;
+	    PUT_DOUBLE(f, fp);
 	    ptr++;
 	    break;
 	}
 
 	case ERL_DRV_EXT2TERM: /* char *ext, int size */
 	    ASSERT(b2t.org_ext[b2t.ix] == (byte *) ptr[0]);
-	    mess = erts_binary2term_create(&b2t.state[b2t.ix++], &hp, ohp);
+
+	    erts_reserve_heap(&factory, b2t.state[b2t.ix].heap_size);
+	    mess = erts_binary2term_create(&b2t.state[b2t.ix++], &factory);
 	    if (mess == THE_NON_VALUE)
 		ERTS_DDT_FAIL;
 	    ptr += 2;
@@ -5548,41 +5560,32 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 	    int size = (int)ptr[0];
             if (size > MAP_SMALL_MAP_LIMIT) {
                 int ix = 2*size;
-                ErtsHeapFactory factory;
-                Eterm* leafs = hp;
-
-                hp += 2*size;
-                while(ix--) { *--hp = ESTACK_POP(stack); }
+                Eterm* leafs;
 
-                hp += 2*size;
-                factory.p = NULL;
-                factory.hp = hp;
-                /* We assume heap will suffice (see hashmap_over_estimated_heap_size) */
+		erts_produce_heap(&factory, ix, HEAP_EXTRA);
+		leafs = factory.hp;
+                while(ix--) { *--leafs = ESTACK_POP(stack); }
 
                 mess = erts_hashmap_from_array(&factory, leafs, size, 1);
-
                 if (is_non_value(mess))
                     ERTS_DDT_FAIL;
-
-                hp = factory.hp;
             } else {
-                Eterm* tp = hp;
                 Eterm* vp;
                 flatmap_t *mp;
+		Eterm* tp = erts_produce_heap(&factory,
+					      2*size + 1 + MAP_HEADER_FLATMAP_SZ,
+					      HEAP_EXTRA);
 
                 *tp = make_arityval(size);
 
-                hp += 1 + size;
-                mp = (flatmap_t*)hp;
+                mp = (flatmap_t*) (tp + 1 + size);
                 mp->thing_word = MAP_HEADER_FLATMAP;
                 mp->size = size;
                 mp->keys = make_tuple(tp);
                 mess = make_flatmap(mp);
 
-                hp += MAP_HEADER_FLATMAP_SZ + size;
-
                 tp += size;    /* point at last key */
-                vp = hp - 1;   /* point at last value */
+                vp = factory.hp - 1;   /* point at last value */
 
                 while(size--) {
                     *vp-- = ESTACK_POP(stack);
@@ -5605,25 +5608,16 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 
     if (res > 0) {
 	mess = ESTACK_POP(stack);  /* get resulting value */
-	if (bp)
-	    bp = erts_resize_message_buffer(bp, hp - hp_start, &mess, 1);
-	else {
-	    ASSERT(hp);
-	    HRelease(rp, hp_end, hp);	    
-	}
+	erts_factory_close(&factory);
 	/* send message */
-	erts_queue_message(rp, &rp_locks, bp, mess, am_undefined);
+	erts_queue_message(rp, &rp_locks, factory.heap_frags, mess, am_undefined);
     }
     else {
 	if (b2t.ix > b2t.used)
 	    b2t.used = b2t.ix;
 	for (b2t.ix = 0; b2t.ix < b2t.used; b2t.ix++)
 	    erts_binary2term_abort(&b2t.state[b2t.ix]);
-	if (bp)
-	    free_message_buffer(bp);
-	else if (hp) {
-	    HRelease(rp, hp_end, hp);
-	}
+	erts_factory_undo(&factory);
     }
     if (rp) {
 	if (rp_locks)
@@ -5635,6 +5629,7 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
     DESTROY_ESTACK(stack);
     return res;
 #undef ERTS_DDT_FAIL
+#undef HEAP_EXTRA
 }
 
 static ERTS_INLINE int