aboutsummaryrefslogtreecommitdiffstats
path: root/erts
diff options
context:
space:
mode:
authorSverker Eriksson <[email protected]>2015-06-02 19:23:49 +0200
committerSverker Eriksson <[email protected]>2015-06-15 14:48:22 +0200
commit02d0ce034598297565f2b35ecc3d1af121787f33 (patch)
treea415e4b7d21c240352ae8737806e516caa2f2b43 /erts
parent512c321bdaf25b685124405e287a3839be467149 (diff)
downloadotp-02d0ce034598297565f2b35ecc3d1af121787f33.tar.gz
otp-02d0ce034598297565f2b35ecc3d1af121787f33.tar.bz2
otp-02d0ce034598297565f2b35ecc3d1af121787f33.zip
erts: Remove hashmap probabilistic heap overestimation
by adding a dynamic heap factory. "binary_to_term" is now a hybrid solution with both a call to decoded_size() to calculate needed heap space AND possible dynamic allocation of more heap space if needed for big maps. The heap size returned from decoded_size() is guaranteed to be sufficient for all term heap data except for hashmap nodes. All hashmap nodes are created at the end of dec_term() by invoking the heap factory interface that may allocate more heap space on process heap or in fragments. With this commit it is no longer guaranteed that a message is confined to only one heap fragment.
Diffstat (limited to 'erts')
-rw-r--r--erts/emulator/beam/beam_emu.c3
-rw-r--r--erts/emulator/beam/beam_load.c184
-rw-r--r--erts/emulator/beam/big.h1
-rw-r--r--erts/emulator/beam/copy.c13
-rw-r--r--erts/emulator/beam/dist.c4
-rw-r--r--erts/emulator/beam/erl_bif_info.c58
-rw-r--r--erts/emulator/beam/erl_db_hash.c1
-rw-r--r--erts/emulator/beam/erl_db_util.c25
-rw-r--r--erts/emulator/beam/erl_gc.c93
-rw-r--r--erts/emulator/beam/erl_map.c27
-rw-r--r--erts/emulator/beam/erl_map.h14
-rw-r--r--erts/emulator/beam/erl_message.c307
-rw-r--r--erts/emulator/beam/erl_message.h85
-rw-r--r--erts/emulator/beam/erl_process_lock.c2
-rw-r--r--erts/emulator/beam/external.c291
-rw-r--r--erts/emulator/beam/external.h9
-rw-r--r--erts/emulator/beam/io.c247
17 files changed, 807 insertions, 557 deletions
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index a21622f424..2d4bf4240c 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -6653,8 +6653,9 @@ new_map(Process* p, Eterm* reg, BeamInstr* I)
p->htop = mhp;
- factory.p = p;
+ erts_factory_proc_init(&factory, p);
res = erts_hashmap_from_array(&factory, thp, n/2, 0);
+ erts_factory_close(&factory);
if (p->mbuf) {
Uint live = Arg(2);
reg[live] = res;
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 0d40201934..8a8ad0d7a3 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -205,10 +205,7 @@ typedef struct {
typedef struct {
Eterm term; /* The tagged term (in the heap). */
- Uint heap_size; /* (Exact) size on the heap. */
- SWord offset; /* Offset from temporary location to final. */
- ErlOffHeap off_heap; /* Start of linked list of ProcBins. */
- Eterm* heap; /* Heap for term. */
+ ErlHeapFragment* heap_frags;
} Literal;
/*
@@ -477,6 +474,8 @@ typedef struct LoaderState {
static void free_loader_state(Binary* magic);
+static ErlHeapFragment* new_literal_fragment(Uint size);
+static void free_literal_fragment(ErlHeapFragment*);
static void loader_state_dtor(Binary* magic);
static Eterm insert_new_code(Process *c_p, ErtsProcLocks c_p_locks,
Eterm group_leader, Eterm module,
@@ -883,6 +882,28 @@ free_loader_state(Binary* magic)
}
}
+static ErlHeapFragment* new_literal_fragment(Uint size)
+{
+ ErlHeapFragment* bp;
+ bp = (ErlHeapFragment*) ERTS_HEAP_ALLOC(ERTS_ALC_T_PREPARED_CODE,
+ ERTS_HEAP_FRAG_SIZE(size));
+ ERTS_INIT_HEAP_FRAG(bp, size);
+ return bp;
+}
+
+static void free_literal_fragment(ErlHeapFragment* bp)
+{
+ ASSERT(bp != NULL);
+ do {
+ ErlHeapFragment* next_bp = bp->next;
+
+ erts_cleanup_offheap(&bp->off_heap);
+ ERTS_HEAP_FREE(ERTS_ALC_T_PREPARED_CODE, (void *) bp,
+ ERTS_HEAP_FRAG_SIZE(bp->size));
+ bp = next_bp;
+ }while (bp != NULL);
+}
+
/*
* This destructor function can safely be called multiple times.
*/
@@ -922,10 +943,9 @@ loader_state_dtor(Binary* magic)
if (stp->literals != 0) {
int i;
for (i = 0; i < stp->num_literals; i++) {
- if (stp->literals[i].heap != 0) {
- erts_free(ERTS_ALC_T_PREPARED_CODE,
- (void *) stp->literals[i].heap);
- stp->literals[i].heap = 0;
+ if (stp->literals[i].heap_frags != 0) {
+ free_literal_fragment(stp->literals[i].heap_frags);
+ stp->literals[i].heap_frags = 0;
}
}
erts_free(ERTS_ALC_T_PREPARED_CODE, (void *) stp->literals);
@@ -1450,6 +1470,7 @@ read_lambda_table(LoaderState* stp)
return 0;
}
+
static int
read_literal_table(LoaderState* stp)
{
@@ -1471,7 +1492,7 @@ read_literal_table(LoaderState* stp)
stp->allocated_literals = stp->num_literals;
for (i = 0; i < stp->num_literals; i++) {
- stp->literals[i].heap = 0;
+ stp->literals[i].heap_frags = 0;
}
for (i = 0; i < stp->num_literals; i++) {
@@ -1479,28 +1500,38 @@ read_literal_table(LoaderState* stp)
Sint heap_size;
byte* p;
Eterm val;
- Eterm* hp;
+ ErtsHeapFactory factory;
GetInt(stp, 4, sz); /* Size of external term format. */
GetString(stp, p, sz);
if ((heap_size = erts_decode_ext_size(p, sz)) < 0) {
LoadError1(stp, "literal %d: bad external format", i);
}
- hp = stp->literals[i].heap = erts_alloc(ERTS_ALC_T_PREPARED_CODE,
- heap_size*sizeof(Eterm));
- stp->literals[i].off_heap.first = 0;
- stp->literals[i].off_heap.overhead = 0;
- val = erts_decode_ext(&hp, &stp->literals[i].off_heap, &p);
- stp->literals[i].heap_size = hp - stp->literals[i].heap;
- if (stp->literals[i].heap_size > heap_size) {
- erl_exit(1, "overrun by %d word(s) for literal heap, term %d",
- stp->literals[i].heap_size - heap_size, i);
- }
- if (is_non_value(val)) {
- LoadError1(stp, "literal %d: bad external format", i);
- }
- stp->literals[i].term = val;
- stp->total_literal_size += stp->literals[i].heap_size;
+
+ if (heap_size > 0) {
+ erts_factory_message_init(&factory, NULL, NULL,
+ new_literal_fragment(heap_size));
+ factory.alloc_type = ERTS_ALC_T_PREPARED_CODE;
+ val = erts_decode_ext(&factory, &p);
+
+ if (is_non_value(val)) {
+ LoadError1(stp, "literal %d: bad external format", i);
+ }
+ erts_factory_close(&factory);
+ stp->literals[i].heap_frags = factory.heap_frags;
+ stp->total_literal_size += erts_used_frag_sz(factory.heap_frags);
+ }
+ else {
+ erts_factory_dummy_init(&factory);
+ val = erts_decode_ext(&factory, &p);
+ if (is_non_value(val)) {
+ LoadError1(stp, "literal %d: bad external format", i);
+ }
+ ASSERT(is_immed(val));
+ stp->literals[i].heap_frags = NULL;
+ }
+ stp->literals[i].term = val;
+
}
erts_free(ERTS_ALC_T_TMP, uncompressed);
return 1;
@@ -4370,8 +4401,9 @@ freeze_code(LoaderState* stp)
Uint* low;
Uint* high;
LiteralPatch* lp;
- struct erl_off_heap_header* off_heap = 0;
- struct erl_off_heap_header** off_heap_last = &off_heap;
+ ErlOffHeap code_off_heap;
+
+ ERTS_INIT_OFF_HEAP(&code_off_heap);
low = (Uint *) (code+stp->ci);
high = low + stp->total_literal_size;
@@ -4379,73 +4411,21 @@ freeze_code(LoaderState* stp)
code[MI_LITERALS_END] = (BeamInstr) high;
ptr = low;
for (i = 0; i < stp->num_literals; i++) {
- SWord offset;
- struct erl_off_heap_header* t_off_heap;
-
- sys_memcpy(ptr, stp->literals[i].heap,
- stp->literals[i].heap_size*sizeof(Eterm));
- offset = ptr - stp->literals[i].heap;
- stp->literals[i].offset = offset;
- high = ptr + stp->literals[i].heap_size;
- while (ptr < high) {
- Eterm val = *ptr;
- switch (primary_tag(val)) {
- case TAG_PRIMARY_LIST:
- case TAG_PRIMARY_BOXED:
- *ptr++ = offset_ptr(val, offset);
- break;
- case TAG_PRIMARY_HEADER:
- if (header_is_transparent(val)) {
- ptr++;
- } else {
- if (thing_subtag(val) == REFC_BINARY_SUBTAG) {
- struct erl_off_heap_header* oh;
-
- oh = (struct erl_off_heap_header*) ptr;
- if (oh->next) {
- Eterm** uptr = (Eterm **) (void *) &oh->next;
- *uptr += offset;
- }
- }
- ptr += 1 + thing_arityval(val);
- }
- break;
- default:
- ptr++;
- break;
- }
- }
- ASSERT(ptr == high);
-
- /*
- * Re-link the off_heap list for this term onto the
- * off_heap list for the entire module.
- */
- t_off_heap = stp->literals[i].off_heap.first;
- if (t_off_heap) {
- t_off_heap = (struct erl_off_heap_header *)
- offset_ptr((UWord) t_off_heap, offset);
- while (t_off_heap) {
- *off_heap_last = t_off_heap;
- off_heap_last = &t_off_heap->next;
- t_off_heap = t_off_heap->next;
- }
- }
+ if (stp->literals[i].heap_frags) {
+ move_multi_frags(&ptr, &code_off_heap, stp->literals[i].heap_frags,
+ &stp->literals[i].term, 1);
+ }
+ else ASSERT(is_immed(stp->literals[i].term));
}
- code[MI_LITERALS_OFF_HEAP] = (BeamInstr) off_heap;
+ code[MI_LITERALS_OFF_HEAP] = (BeamInstr) code_off_heap.first;
lp = stp->literal_patches;
while (lp != 0) {
BeamInstr* op_ptr;
- Uint literal;
Literal* lit;
op_ptr = code + lp->pos;
lit = &stp->literals[op_ptr[0]];
- literal = lit->term;
- if (is_boxed(literal) || is_list(literal)) {
- literal = offset_ptr(literal, lit->offset);
- }
- op_ptr[0] = literal;
+ op_ptr[0] = lit->term;
lp = lp->next;
}
literal_end += stp->total_literal_size;
@@ -5376,13 +5356,9 @@ new_literal(LoaderState* stp, Eterm** hpp, Uint heap_size)
stp->total_literal_size += heap_size;
lit = stp->literals + stp->num_literals;
- lit->offset = 0;
- lit->heap_size = heap_size;
- lit->heap = erts_alloc(ERTS_ALC_T_PREPARED_CODE, heap_size*sizeof(Eterm));
- lit->term = make_boxed(lit->heap);
- lit->off_heap.first = 0;
- lit->off_heap.overhead = 0;
- *hpp = lit->heap;
+ lit->heap_frags = new_literal_fragment(heap_size);
+ lit->term = make_boxed(lit->heap_frags->mem);
+ *hpp = lit->heap_frags->mem;
return stp->num_literals++;
}
@@ -5659,10 +5635,8 @@ attributes_for_module(Process* p, /* Process whose heap to use. */
{
Module* modp;
BeamInstr* code;
- Eterm* hp;
byte* ext;
Eterm result = NIL;
- Eterm* end;
if (is_not_atom(mod)) {
return THE_NON_VALUE;
@@ -5675,13 +5649,12 @@ attributes_for_module(Process* p, /* Process whose heap to use. */
code = modp->curr.code;
ext = (byte *) code[MI_ATTR_PTR];
if (ext != NULL) {
- hp = HAlloc(p, code[MI_ATTR_SIZE_ON_HEAP]);
- end = hp + code[MI_ATTR_SIZE_ON_HEAP];
- result = erts_decode_ext(&hp, &MSO(p), &ext);
+ ErtsHeapFactory factory;
+ erts_factory_proc_prealloc_init(&factory, p, code[MI_ATTR_SIZE_ON_HEAP]);
+ result = erts_decode_ext(&factory, &ext);
if (is_value(result)) {
- ASSERT(hp <= end);
+ erts_factory_close(&factory);
}
- HRelease(p,end,hp);
}
return result;
}
@@ -5698,10 +5671,8 @@ compilation_info_for_module(Process* p, /* Process whose heap to use. */
{
Module* modp;
BeamInstr* code;
- Eterm* hp;
byte* ext;
Eterm result = NIL;
- Eterm* end;
if (is_not_atom(mod)) {
return THE_NON_VALUE;
@@ -5714,13 +5685,12 @@ compilation_info_for_module(Process* p, /* Process whose heap to use. */
code = modp->curr.code;
ext = (byte *) code[MI_COMPILE_PTR];
if (ext != NULL) {
- hp = HAlloc(p, code[MI_COMPILE_SIZE_ON_HEAP]);
- end = hp + code[MI_COMPILE_SIZE_ON_HEAP];
- result = erts_decode_ext(&hp, &MSO(p), &ext);
+ ErtsHeapFactory factory;
+ erts_factory_proc_prealloc_init(&factory, p, code[MI_COMPILE_SIZE_ON_HEAP]);
+ result = erts_decode_ext(&factory, &ext);
if (is_value(result)) {
- ASSERT(hp <= end);
+ erts_factory_close(&factory);
}
- HRelease(p,end,hp);
}
return result;
}
diff --git a/erts/emulator/beam/big.h b/erts/emulator/beam/big.h
index 4e4611de16..5b5550da43 100644
--- a/erts/emulator/beam/big.h
+++ b/erts/emulator/beam/big.h
@@ -85,6 +85,7 @@ typedef Uint dsize_t; /* Vector size type */
/* The heap size needed for a bignum */
#define BIG_NEED_SIZE(x) ((x) + 1)
+#define BIG_NEED_FOR_BITS(bits) BIG_NEED_SIZE(((bits)-1)/D_EXP + 1)
#define BIG_UINT_HEAP_SIZE (1 + 1) /* always, since sizeof(Uint) <= sizeof(Eterm) */
diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index 850606dd86..cddadb346c 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -34,7 +34,7 @@
#include "erl_bits.h"
#include "dtrace-wrapper.h"
-static void move_one_frag(Eterm** hpp, Eterm* src, Uint src_sz, ErlOffHeap*);
+static void move_one_frag(Eterm** hpp, ErlHeapFragment*, ErlOffHeap*);
/*
* Copy object "obj" to process p.
@@ -661,8 +661,7 @@ void move_multi_frags(Eterm** hpp, ErlOffHeap* off_heap, ErlHeapFragment* first,
unsigned i;
for (bp=first; bp!=NULL; bp=bp->next) {
- move_one_frag(hpp, bp->mem, bp->used_size, off_heap);
- OH_OVERHEAD(off_heap, bp->off_heap.overhead);
+ move_one_frag(hpp, bp, off_heap);
}
hp_end = *hpp;
for (hp=hp_start; hp<hp_end; ++hp) {
@@ -698,10 +697,10 @@ void move_multi_frags(Eterm** hpp, ErlOffHeap* off_heap, ErlHeapFragment* first,
}
static void
-move_one_frag(Eterm** hpp, Eterm* src, Uint src_sz, ErlOffHeap* off_heap)
+move_one_frag(Eterm** hpp, ErlHeapFragment* frag, ErlOffHeap* off_heap)
{
- Eterm* ptr = src;
- Eterm* end = ptr + src_sz;
+ Eterm* ptr = frag->mem;
+ Eterm* end = ptr + frag->used_size;
Eterm dummy_ref;
Eterm* hp = *hpp;
@@ -732,5 +731,7 @@ move_one_frag(Eterm** hpp, Eterm* src, Uint src_sz, ErlOffHeap* off_heap)
}
}
*hpp = hp;
+ OH_OVERHEAD(off_heap, frag->off_heap.overhead);
+ frag->off_heap.first = NULL;
}
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 142fcb3c00..ae85659522 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -1149,6 +1149,7 @@ int erts_net_message(Port *prt,
DeclareTmpHeapNoproc(ctl_default,DIST_CTL_DEFAULT_SIZE);
Eterm* ctl = ctl_default;
ErlOffHeap off_heap;
+ ErtsHeapFactory factory;
Eterm* hp;
Sint type;
Eterm token;
@@ -1225,7 +1226,8 @@ int erts_net_message(Port *prt,
}
hp = ctl;
- arg = erts_decode_dist_ext(&hp, &off_heap, &ede);
+ erts_factory_static_init(&factory, ctl, ctl_len, &off_heap);
+ arg = erts_decode_dist_ext(&factory, &ede);
if (is_non_value(arg)) {
#ifdef ERTS_DIST_MSG_DBG
erts_fprintf(stderr, "DIST MSG DEBUG: erts_dist_ext_size(CTL) failed:\n");
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index f74aea80a7..15173bd05e 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -1106,19 +1106,18 @@ process_info_aux(Process *BIF_P,
heap_need += mq[i].copy_struct_size;
}
else {
- mq[i].copy_struct_size = 0;
- if (mp->data.attached)
- heap_need += erts_msg_attached_data_size(mp);
+ mq[i].copy_struct_size = mp->data.attached ?
+ erts_msg_attached_data_size(mp) : 0;
}
i++;
}
- hp = HAlloc(BIF_P, heap_need);
- hp_end = hp + heap_need;
- ASSERT(i == n);
- for (i--; i >= 0; i--) {
- Eterm msg = ERL_MESSAGE_TERM(mq[i].msgp);
- if (rp != BIF_P) {
+ if (rp != BIF_P) {
+ hp = HAlloc(BIF_P, heap_need);
+ hp_end = hp + heap_need;
+ ASSERT(i == n);
+ for (i--; i >= 0; i--) {
+ Eterm msg = ERL_MESSAGE_TERM(mq[i].msgp);
if (is_value(msg)) {
if (mq[i].copy_struct_size)
msg = copy_struct(msg,
@@ -1152,9 +1151,9 @@ process_info_aux(Process *BIF_P,
}
else {
/* Make our copy of the message */
- ASSERT(size_object(msg) == hfp->used_size);
+ ASSERT(size_object(msg) == erts_used_frag_sz(hfp));
msg = copy_struct(msg,
- hfp->used_size,
+ erts_used_frag_sz(hfp),
&hp,
&MSO(BIF_P));
}
@@ -1164,27 +1163,38 @@ process_info_aux(Process *BIF_P,
remove_bad_messages = 1;
continue;
}
+ res = CONS(hp, msg, res);
+ hp += 2;
}
- else {
+ HRelease(BIF_P, hp_end, hp+3);
+ }
+ else {
+ for (i--; i >= 0; i--) {
+ ErtsHeapFactory factory;
+ Eterm msg = ERL_MESSAGE_TERM(mq[i].msgp);
+
+ erts_factory_proc_prealloc_init(&factory, BIF_P,
+ mq[i].copy_struct_size+2);
if (mq[i].msgp->data.attached) {
/* Decode it on the heap */
- erts_move_msg_attached_data_to_heap(&hp,
- &MSO(BIF_P),
+ erts_move_msg_attached_data_to_heap(&factory,
mq[i].msgp);
msg = ERL_MESSAGE_TERM(mq[i].msgp);
ASSERT(!mq[i].msgp->data.attached);
- if (is_non_value(msg)) {
- /* Bad distribution message; ignore */
- remove_bad_messages = 1;
- continue;
- }
- }
+ }
+ if (is_value(msg)) {
+ hp = erts_produce_heap(&factory, 2, 0);
+ res = CONS(hp, msg, res);
+ }
+ else {
+ /* Bad distribution message; ignore */
+ remove_bad_messages = 1;
+ continue;
+ }
+ erts_factory_close(&factory);
}
-
- res = CONS(hp, msg, res);
- hp += 2;
+ hp = HAlloc(BIF_P, 3);
}
- HRelease(BIF_P, hp_end, hp+3);
erts_free(ERTS_ALC_T_TMP, mq);
if (remove_bad_messages) {
ErlMessage **mpp;
diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 383ee7c430..6e50e9c5b4 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -1049,7 +1049,6 @@ static int db_get_element_hash(Process *p, DbTable *tbl,
Eterm copy = db_copy_element_from_ets(&tb->common, p,
&b->dbterm, ndex, &hp, 2);
elem_list = CONS(hp, copy, elem_list);
- hp += 2;
}
b = b->next;
}
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index c6c3c55a7e..d47ff03a30 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -2175,11 +2175,12 @@ restart:
{
ErtsHeapFactory factory;
Uint ix;
- factory.p = build_proc;
for (ix = 0; ix < 2*n; ix++){
ehp[ix] = esp[ix];
}
+ erts_factory_proc_init(&factory, build_proc);
t = erts_hashmap_from_array(&factory, ehp, n, 0);
+ erts_factory_close(&factory);
}
*esp++ = t;
break;
@@ -3192,6 +3193,7 @@ Eterm db_copy_from_comp(DbTableCommon* tb, DbTerm* bp, Eterm** hpp,
{
Eterm* hp = *hpp;
int i, arity = arityval(bp->tpl[0]);
+ ErtsHeapFactory factory;
hp[0] = bp->tpl[0];
*hpp += arity + 1;
@@ -3199,17 +3201,23 @@ Eterm db_copy_from_comp(DbTableCommon* tb, DbTerm* bp, Eterm** hpp,
hp[tb->keypos] = copy_struct_rel(bp->tpl[tb->keypos],
size_object_rel(bp->tpl[tb->keypos], bp->tpl),
hpp, off_heap, bp->tpl, NULL);
+
+ erts_factory_static_init(&factory, *hpp, bp->size - (arity+1), off_heap);
+
for (i=arity; i>0; i--) {
if (i != tb->keypos) {
if (is_immed(bp->tpl[i])) {
hp[i] = bp->tpl[i];
}
else {
- hp[i] = erts_decode_ext_ets(hpp, off_heap,
+ hp[i] = erts_decode_ext_ets(&factory,
elem2ext(bp->tpl, i));
}
}
}
+ *hpp = factory.hp;
+ erts_factory_close(&factory);
+
ASSERT((*hpp - hp) <= bp->size);
#ifdef DEBUG_CLONE
ASSERT(eq_rel(make_tuple(hp),NULL,make_tuple(bp->debug_clone),bp->debug_clone));
@@ -3228,12 +3236,13 @@ Eterm db_copy_element_from_ets(DbTableCommon* tb, Process* p,
if (tb->compress && pos != tb->keypos) {
byte* ext = elem2ext(obj->tpl, pos);
Sint sz = erts_decode_ext_size_ets(ext, db_alloced_size_comp(obj)) + extra;
- Eterm* hp = HAlloc(p, sz);
- Eterm* endp = hp + sz;
- Eterm copy = erts_decode_ext_ets(&hp, &MSO(p), ext);
- *hpp = hp;
- hp += extra;
- HRelease(p, endp, hp);
+ Eterm copy;
+ ErtsHeapFactory factory;
+
+ erts_factory_proc_prealloc_init(&factory, p, sz);
+ copy = erts_decode_ext_ets(&factory, ext);
+ *hpp = erts_produce_heap(&factory, extra, 0);
+ erts_factory_close(&factory);
#ifdef DEBUG_CLONE
ASSERT(eq_rel(copy, NULL, obj->debug_clone[pos], obj->debug_clone));
#endif
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 14e7dde778..a456689b5c 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -108,8 +108,7 @@ static Eterm* sweep_one_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop,
char* src, Uint src_size);
static Eterm* collect_heap_frags(Process* p, Eterm* heap,
Eterm* htop, Eterm* objv, int nobj);
-static Uint adjust_after_fullsweep(Process *p, Uint size_before,
- int need, Eterm *objv, int nobj);
+static void adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj);
static void shrink_new_heap(Process *p, Uint new_sz, Eterm *objv, int nobj);
static void grow_new_heap(Process *p, Uint new_sz, Eterm* objv, int nobj);
static void sweep_off_heap(Process *p, int fullsweep);
@@ -868,29 +867,37 @@ minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
ErlMessage *msgp;
Uint size_after;
Uint need_after;
- Uint stack_size = STACK_SZ_ON_HEAP(p);
- Uint fragments = MBUF_SIZE(p) + combined_message_size(p);
- Uint size_before = fragments + (HEAP_TOP(p) - HEAP_START(p));
- Uint new_sz = next_heap_size(p, HEAP_SIZE(p) + fragments, 0);
+ const Uint stack_size = STACK_SZ_ON_HEAP(p);
+ const Uint size_before = MBUF_SIZE(p) + (HEAP_TOP(p) - HEAP_START(p));
+ Uint new_sz = HEAP_SIZE(p) + MBUF_SIZE(p) + combined_message_size(p);
+ new_sz = next_heap_size(p, new_sz, 0);
do_minor(p, new_sz, objv, nobj);
- /*
+ size_after = HEAP_TOP(p) - HEAP_START(p);
+ *recl += (size_before - size_after);
+
+ /*
* Copy newly received message onto the end of the new heap.
*/
- ErtsGcQuickSanityCheck(p);
- for (msgp = p->msg.first; msgp; msgp = msgp->next) {
- if (msgp->data.attached) {
- erts_move_msg_attached_data_to_heap(&p->htop, &p->off_heap, msgp);
- ErtsGcQuickSanityCheck(p);
- }
- }
+ ErtsGcQuickSanityCheck(p);
+ for (msgp = p->msg.first; msgp; msgp = msgp->next) {
+ if (msgp->data.attached) {
+ ErtsHeapFactory factory;
+ erts_factory_proc_prealloc_init(&factory, p,
+ erts_msg_attached_data_size(msgp));
+ erts_move_msg_attached_data_to_heap(&factory, msgp);
+ erts_factory_close(&factory);
+ ErtsGcQuickSanityCheck(p);
+ }
+ }
ErtsGcQuickSanityCheck(p);
GEN_GCS(p)++;
- size_after = HEAP_TOP(p) - HEAP_START(p);
- need_after = size_after + need + stack_size;
- *recl += (size_before - size_after);
+ need_after = ((HEAP_TOP(p) - HEAP_START(p))
+ + erts_used_frag_sz(MBUF(p))
+ + need
+ + stack_size);
/*
* Excessively large heaps should be shrunk, but
@@ -925,6 +932,7 @@ minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
}
ASSERT(HEAP_SIZE(p) == next_heap_size(p, HEAP_SIZE(p), 0));
+ ASSERT(MBUF(p) == NULL);
return 1; /* We are done. */
}
@@ -933,6 +941,7 @@ minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
* The heap size turned out to be just right. We are done.
*/
ASSERT(HEAP_SIZE(p) == next_heap_size(p, HEAP_SIZE(p), 0));
+ ASSERT(MBUF(p) == NULL);
return 1;
}
}
@@ -1212,7 +1221,9 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
{
Rootset rootset;
Roots* roots;
- Uint size_before;
+ const Uint size_before = ((HEAP_TOP(p) - HEAP_START(p))
+ + (OLD_HTOP(p) - OLD_HEAP(p))
+ + MBUF_SIZE(p));
Eterm* n_heap;
Eterm* n_htop;
char* src = (char *) HEAP_START(p);
@@ -1221,25 +1232,15 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
Uint oh_size = (char *) OLD_HTOP(p) - oh;
Uint n;
Uint new_sz;
- Uint fragments = MBUF_SIZE(p) + combined_message_size(p);
-
- size_before = fragments + (HEAP_TOP(p) - HEAP_START(p))
- + (OLD_HTOP(p) - OLD_HEAP(p));
/*
* Do a fullsweep GC. First figure out the size of the heap
* to receive all live data.
*/
- new_sz = HEAP_SIZE(p) + fragments + (OLD_HTOP(p) - OLD_HEAP(p));
- /*
- * We used to do
- *
- * new_sz += STACK_SZ_ON_HEAP(p);
- *
- * here for no obvious reason. (The stack size is already counted once
- * in HEAP_SIZE(p).)
- */
+ new_sz = (HEAP_SIZE(p) + MBUF_SIZE(p)
+ + combined_message_size(p)
+ + (OLD_HTOP(p) - OLD_HEAP(p)));
new_sz = next_heap_size(p, new_sz, 0);
/*
@@ -1432,20 +1433,27 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
ErtsGcQuickSanityCheck(p);
+ *recl += size_before - (HEAP_TOP(p) - HEAP_START(p));
+
{
ErlMessage *msgp;
+
/*
* Copy newly received message onto the end of the new heap.
*/
- for (msgp = p->msg.first; msgp; msgp = msgp->next) {
+ for (msgp = p->msg.first; msgp; msgp = msgp->next) {
if (msgp->data.attached) {
- erts_move_msg_attached_data_to_heap(&p->htop, &p->off_heap, msgp);
+ ErtsHeapFactory factory;
+ erts_factory_proc_prealloc_init(&factory, p,
+ erts_msg_attached_data_size(msgp));
+ erts_move_msg_attached_data_to_heap(&factory, msgp);
+ erts_factory_close(&factory);
ErtsGcQuickSanityCheck(p);
}
}
}
- *recl += adjust_after_fullsweep(p, size_before, need, objv, nobj);
+ adjust_after_fullsweep(p, need, objv, nobj);
#ifdef HARDDEBUG
disallow_heap_frag_ref_in_heap(p);
@@ -1456,21 +1464,17 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
return 1; /* We are done. */
}
-static Uint
-adjust_after_fullsweep(Process *p, Uint size_before, int need, Eterm *objv, int nobj)
+static void
+adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj)
{
- Uint wanted, sz, size_after, need_after;
+ Uint wanted, sz, need_after;
Uint stack_size = STACK_SZ_ON_HEAP(p);
- Uint reclaimed_now;
-
- size_after = (HEAP_TOP(p) - HEAP_START(p));
- reclaimed_now = (size_before - size_after);
/*
* Resize the heap if needed.
*/
- need_after = size_after + need + stack_size;
+ need_after = (HEAP_TOP(p) - HEAP_START(p)) + need + stack_size;
if (HEAP_SIZE(p) < need_after) {
/* Too small - grow to match requested need */
sz = next_heap_size(p, need_after, 0);
@@ -1493,8 +1497,6 @@ adjust_after_fullsweep(Process *p, Uint size_before, int need, Eterm *objv, int
shrink_new_heap(p, sz, objv, nobj);
}
}
-
- return reclaimed_now;
}
/*
@@ -1942,7 +1944,8 @@ collect_heap_frags(Process* p, Eterm* n_hstart, Eterm* n_htop,
* until next GC.
*/
qb = MBUF(p);
- while (qb != NULL) {
+ while (qb != NULL) {
+ ASSERT(!qb->off_heap.first); /* process fragments use the MSO(p) list */
frag_size = qb->used_size * sizeof(Eterm);
if (frag_size != 0) {
frag_begin = (char *) qb->mem;
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index a1bd39dbc8..09fc9a2ad6 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -410,8 +410,9 @@ static Eterm hashmap_from_validated_list(Process *p, Eterm list, Uint size) {
}
UnUseTmpHeap(2,p);
- factory.p = p;
+ erts_factory_proc_init(&factory, p);
res = hashmap_from_unsorted_array(&factory, hxns, size, 0);
+ erts_factory_close(&factory);
erts_free(ERTS_ALC_T_TMP, (void *) hxns);
ERTS_VERIFY_UNUSED_TEMP_ALLOC(p);
@@ -515,8 +516,9 @@ Eterm erts_hashmap_from_ks_and_vs_extra(Process *p, Eterm *ks, Eterm *vs, Uint n
hxns[i].i = i;
}
- factory.p = p;
+ erts_factory_proc_init(&factory, p);
res = hashmap_from_unsorted_array(&factory, hxns, sz, 0);
+ erts_factory_close(&factory);
erts_free(ERTS_ALC_T_TMP, (void *) hxns);
ERTS_VERIFY_UNUSED_TEMP_ALLOC(p);
@@ -1063,8 +1065,9 @@ static Eterm flatmap_merge(Process *p, Eterm nodeA, Eterm nodeB) {
hxns[i].i = i;
}
- factory.p = p;
+ erts_factory_proc_init(&factory, p);
res = hashmap_from_unsorted_array(&factory, hxns, n, 0);
+ erts_factory_close(&factory);
erts_free(ERTS_ALC_T_TMP, (void *) hxns);
ERTS_VERIFY_UNUSED_TEMP_ALLOC(p);
@@ -1107,8 +1110,9 @@ static Eterm map_merge_mixed(Process *p, Eterm flat, Eterm tree, int swap_args)
hxns[i].i = i;
}
- factory.p = p;
+ erts_factory_proc_init(&factory, p);
res = hashmap_from_unsorted_array(&factory, hxns, n, 0);
+ erts_factory_close(&factory);
erts_free(ERTS_ALC_T_TMP, (void *) hxns);
ERTS_VERIFY_UNUSED_TEMP_ALLOC(p);
@@ -2504,6 +2508,9 @@ int erts_validate_and_sort_flatmap(flatmap_t* mp)
return 1;
}
+#if 0 /* Can't get myself to remove this beautiful piece of code
+ for probabilistic overestimation of nr of nodes in a hashmap */
+
/* Really rough estimate of sqrt(x)
* Guaranteed not to be less than sqrt(x)
*/
@@ -2525,7 +2532,10 @@ static int int_sqrt_ceiling(Uint x)
}
}
-Uint hashmap_over_estimated_heap_size(Uint k)
+/* May not be enough if hashing is broken (not uniform)
+ * or if hell freezes over.
+ */
+Uint hashmap_overestimated_node_count(Uint k)
{
/* k is nr of key-value pairs.
N(k) is expected nr of nodes in hamt.
@@ -2539,12 +2549,9 @@ Uint hashmap_over_estimated_heap_size(Uint k)
by 15 std.devs above the average, which gives a probability for overrun
less than 1.0e-49 (same magnitude as a git SHA1 collision).
*/
- Uint max_nodes = 2*k/5 + (15/3)*int_sqrt_ceiling(k);
- return (k*2 + /* leaf cons cells */
- k + /* leaf list terms */
- max_nodes*2); /* headers + parent boxed terms */
+ return 2*k/5 + 1 + (15/3)*int_sqrt_ceiling(k);
}
-
+#endif
BIF_RETTYPE erts_debug_map_info_1(BIF_ALIST_1) {
if (is_hashmap(BIF_ARG_1)) {
diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h
index 2cc6768bfc..b5941c5c9a 100644
--- a/erts/emulator/beam/erl_map.h
+++ b/erts/emulator/beam/erl_map.h
@@ -91,7 +91,6 @@ Eterm erts_hashmap_insert_up(Eterm *hp, Eterm key, Eterm value,
Uint *upsz, struct ErtsEStack_ *sp);
int erts_validate_and_sort_flatmap(flatmap_t* map);
-Uint hashmap_over_estimated_heap_size(Uint n);
void hashmap_iterator_init(struct ErtsWStack_* s, Eterm node, int reverse);
Eterm* hashmap_iterator_next(struct ErtsWStack_* s);
Eterm* hashmap_iterator_prev(struct ErtsWStack_* s);
@@ -191,5 +190,18 @@ typedef struct hashmap_head_s {
#define hashmap_index(hash) (((Uint32)hash) & 0xf)
+/* hashmap heap size:
+ [one cons cell + one list term in parent node] per key
+ [one header + one boxed term in parent node] per inner node
+ [one header + one size word] for root node
+*/
+#define HASHMAP_HEAP_SIZE(KEYS,NODES) ((KEYS)*3 + (NODES)*2)
+#ifdef DEBUG
+# define HASHMAP_ESTIMATED_NODE_COUNT(KEYS) (KEYS)
+#else
+# define HASHMAP_ESTIMATED_NODE_COUNT(KEYS) (2*(KEYS)/5)
+#endif
+#define HASHMAP_ESTIMATED_HEAP_SIZE(KEYS) \
+ HASHMAP_HEAP_SIZE(KEYS,HASHMAP_ESTIMATED_NODE_COUNT(KEYS))
#endif
diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index ccfc2e6458..13e084ce10 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -237,8 +237,7 @@ erts_msg_distext2heap(Process *pp,
Eterm msg;
Uint tok_sz = 0;
Eterm *hp = NULL;
- Eterm *hp_end = NULL;
- ErlOffHeap *ohp;
+ ErtsHeapFactory factory;
Sint sz;
*bpp = NULL;
@@ -250,36 +249,26 @@ erts_msg_distext2heap(Process *pp,
tok_sz = heap_frag->used_size;
sz += tok_sz;
}
- if (pp)
+ if (pp) {
+ ErlOffHeap *ohp;
hp = erts_alloc_message_heap(sz, bpp, &ohp, pp, plcksp);
+ }
else {
*bpp = new_message_buffer(sz);
hp = (*bpp)->mem;
- ohp = &(*bpp)->off_heap;
}
- hp_end = hp + sz;
- msg = erts_decode_dist_ext(&hp, ohp, dist_extp);
+ erts_factory_message_init(&factory, pp, hp, *bpp);
+ msg = erts_decode_dist_ext(&factory, dist_extp);
if (is_non_value(msg))
goto decode_error;
if (is_not_nil(*tokenp)) {
ErlHeapFragment *heap_frag = erts_dist_ext_trailer(dist_extp);
- *tokenp = copy_struct(*tokenp, tok_sz, &hp, ohp);
+ hp = erts_produce_heap(&factory, tok_sz, 0);
+ *tokenp = copy_struct(*tokenp, tok_sz, &hp, factory.off_heap);
erts_cleanup_offheap(&heap_frag->off_heap);
}
erts_free_dist_ext_copy(dist_extp);
- if (hp_end != hp) {
- if (!(*bpp)) {
- HRelease(pp, hp_end, hp);
- }
- else {
- Uint final_size = hp - &(*bpp)->mem[0];
- Eterm brefs[2] = {msg, *tokenp};
- ASSERT(sz - (hp_end - hp) == final_size);
- *bpp = erts_resize_message_buffer(*bpp, final_size, &brefs[0], 2);
- msg = brefs[0];
- *tokenp = brefs[1];
- }
- }
+ erts_factory_close(&factory);
return msg;
decode_error:
@@ -288,13 +277,7 @@ erts_msg_distext2heap(Process *pp,
erts_cleanup_offheap(&heap_frag->off_heap);
}
erts_free_dist_ext_copy(dist_extp);
- if (*bpp) {
- free_message_buffer(*bpp);
- *bpp = NULL;
- }
- else if (hp) {
- HRelease(pp, hp_end, hp);
- }
+ *bpp = NULL;
return THE_NON_VALUE;
}
@@ -851,10 +834,11 @@ erts_msg_attached_data_size_aux(ErlMessage *msg)
}
void
-erts_move_msg_attached_data_to_heap(Eterm **hpp, ErlOffHeap *ohp, ErlMessage *msg)
+erts_move_msg_attached_data_to_heap(ErtsHeapFactory* factory,
+ ErlMessage *msg)
{
if (is_value(ERL_MESSAGE_TERM(msg)))
- erts_move_msg_mbuf_to_heap(hpp, ohp, msg);
+ erts_move_msg_mbuf_to_heap(&factory->hp, factory->off_heap, msg);
else if (msg->data.dist_ext) {
ASSERT(msg->data.dist_ext->heap_size >= 0);
if (is_not_nil(ERL_MESSAGE_TOKEN(msg))) {
@@ -862,12 +846,11 @@ erts_move_msg_attached_data_to_heap(Eterm **hpp, ErlOffHeap *ohp, ErlMessage *ms
heap_frag = erts_dist_ext_trailer(msg->data.dist_ext);
ERL_MESSAGE_TOKEN(msg) = copy_struct(ERL_MESSAGE_TOKEN(msg),
heap_frag->used_size,
- hpp,
- ohp);
+ &factory->hp,
+ factory->off_heap);
erts_cleanup_offheap(&heap_frag->off_heap);
}
- ERL_MESSAGE_TERM(msg) = erts_decode_dist_ext(hpp,
- ohp,
+ ERL_MESSAGE_TERM(msg) = erts_decode_dist_ext(factory,
msg->data.dist_ext);
erts_free_dist_ext_copy(msg->data.dist_ext);
msg->data.dist_ext = NULL;
@@ -1134,15 +1117,263 @@ erts_deliver_exit_message(Eterm from, Process *to, ErtsProcLocks *to_locksp,
}
}
+void erts_factory_proc_init(ErtsHeapFactory* factory,
+ Process* p)
+{
+ erts_factory_proc_prealloc_init(factory, p, HEAP_LIMIT(p) - HEAP_TOP(p));
+}
+
+void erts_factory_proc_prealloc_init(ErtsHeapFactory* factory,
+ Process* p,
+ Sint size)
+{
+ factory->mode = FACTORY_HALLOC;
+ factory->p = p;
+ factory->hp_start = HAlloc(p, size);
+ factory->hp = factory->hp_start;
+ factory->hp_end = factory->hp_start + size;
+ factory->off_heap = &p->off_heap;
+ factory->off_heap_saved.first = p->off_heap.first;
+ factory->off_heap_saved.overhead = p->off_heap.overhead;
+ factory->heap_frags_saved = p->mbuf;
+ factory->heap_frags = NULL; /* not used */
+ factory->alloc_type = 0; /* not used */
+}
+
+void erts_factory_message_init(ErtsHeapFactory* factory,
+ Process* rp,
+ Eterm* hp,
+ ErlHeapFragment* bp)
+{
+ if (bp) {
+ factory->mode = FACTORY_HEAP_FRAGS;
+ factory->p = NULL;
+ factory->hp_start = bp->mem;
+ factory->hp = hp ? hp : bp->mem;
+ factory->hp_end = bp->mem + bp->alloc_size;
+ factory->off_heap = &bp->off_heap;
+ factory->heap_frags = bp;
+ factory->heap_frags_saved = bp;
+ factory->alloc_type = ERTS_ALC_T_HEAP_FRAG;
+ ASSERT(!bp->next);
+ }
+ else {
+ factory->mode = FACTORY_HALLOC;
+ factory->p = rp;
+ factory->hp_start = hp;
+ factory->hp = hp;
+ factory->hp_end = HEAP_TOP(rp);
+ factory->off_heap = &rp->off_heap;
+ factory->heap_frags_saved = rp->mbuf;
+ factory->heap_frags = NULL; /* not used */
+ factory->alloc_type = 0; /* not used */
+ }
+ factory->off_heap_saved.first = factory->off_heap->first;
+ factory->off_heap_saved.overhead = factory->off_heap->overhead;
+
+ ASSERT(factory->hp >= factory->hp_start && factory->hp <= factory->hp_end);
+}
+
+void erts_factory_static_init(ErtsHeapFactory* factory,
+ Eterm* hp,
+ Uint size,
+ ErlOffHeap* off_heap)
+{
+ factory->mode = FACTORY_STATIC;
+ factory->hp_start = hp;
+ factory->hp = hp;
+ factory->hp_end = hp + size;
+ factory->off_heap = off_heap;
+ factory->off_heap_saved.first = factory->off_heap->first;
+ factory->off_heap_saved.overhead = factory->off_heap->overhead;
+}
+
+/* When we know the term is an immediate and need no heap.
+*/
+void erts_factory_dummy_init(ErtsHeapFactory* factory)
+{
+ factory->mode = FACTORY_CLOSED;
+}
+
+static void reserve_heap(ErtsHeapFactory*, Uint need, Uint xtra);
+
Eterm* erts_produce_heap(ErtsHeapFactory* factory, Uint need, Uint xtra)
{
Eterm* res;
- if (factory->p) {
- res = HAllocX(factory->p, need, xtra);
- } else {
- res = factory->hp;
- factory->hp += need;
+
+ ASSERT((unsigned int)factory->mode > (unsigned int)FACTORY_CLOSED);
+ if (factory->hp + need > factory->hp_end) {
+ reserve_heap(factory, need, xtra);
}
+ res = factory->hp;
+ factory->hp += need;
return res;
}
+Eterm* erts_reserve_heap(ErtsHeapFactory* factory, Uint need)
+{
+ ASSERT((unsigned int)factory->mode > (unsigned int)FACTORY_CLOSED);
+ if (factory->hp + need > factory->hp_end) {
+ reserve_heap(factory, need, 200);
+ }
+ return factory->hp;
+}
+
+static void reserve_heap(ErtsHeapFactory* factory, Uint need, Uint xtra)
+{
+ ErlHeapFragment* bp;
+
+ switch (factory->mode) {
+ case FACTORY_HALLOC:
+ HRelease(factory->p, factory->hp_end, factory->hp);
+ factory->hp = HAllocX(factory->p, need, xtra);
+ factory->hp_end = factory->hp + need;
+ return;
+
+ case FACTORY_HEAP_FRAGS:
+ bp = factory->heap_frags;
+
+ if (bp) {
+ ASSERT(factory->hp > bp->mem);
+ ASSERT(factory->hp <= factory->hp_end);
+ ASSERT(factory->hp_end == bp->mem + bp->alloc_size);
+
+ bp->used_size = factory->hp - bp->mem;
+ }
+ bp = (ErlHeapFragment*) ERTS_HEAP_ALLOC(factory->alloc_type,
+ ERTS_HEAP_FRAG_SIZE(need+xtra));
+ bp->next = factory->heap_frags;
+ factory->heap_frags = bp;
+ bp->alloc_size = need + xtra;
+ bp->used_size = need;
+ bp->off_heap.first = NULL;
+ bp->off_heap.overhead = 0;
+
+ factory->hp = bp->mem;
+ factory->hp_end = bp->mem + bp->alloc_size;
+ return;
+
+ case FACTORY_STATIC:
+ case FACTORY_CLOSED:
+ default:
+ ASSERT(!"Invalid factory mode");
+ }
+}
+
+void erts_factory_close(ErtsHeapFactory* factory)
+{
+ ErlHeapFragment* bp;
+
+ switch (factory->mode) {
+ case FACTORY_HALLOC:
+ HRelease(factory->p, factory->hp_end, factory->hp);
+ break;
+
+ case FACTORY_HEAP_FRAGS:
+ bp = factory->heap_frags;
+
+ if (bp) {
+ ASSERT(factory->hp >= bp->mem);
+ ASSERT(factory->hp <= factory->hp_end);
+ ASSERT(factory->hp_end == bp->mem + bp->alloc_size);
+
+ bp->used_size = factory->hp - bp->mem;
+ }
+ break;
+ case FACTORY_STATIC: break;
+ case FACTORY_CLOSED: break;
+ default:
+ ASSERT(!"Invalid factory mode");
+ }
+ factory->mode = FACTORY_CLOSED;
+}
+
+void erts_factory_undo(ErtsHeapFactory* factory)
+{
+ ErlHeapFragment* bp;
+ struct erl_off_heap_header *hdr, **hdr_nextp;
+
+ switch (factory->mode) {
+ case FACTORY_HALLOC:
+ case FACTORY_STATIC:
+ /* Cleanup off-heap
+ */
+ hdr_nextp = NULL;
+ for (hdr = factory->off_heap->first;
+ hdr != factory->off_heap_saved.first;
+ hdr = hdr->next) {
+
+ hdr_nextp = &hdr->next;
+ }
+
+ if (hdr_nextp != NULL) {
+ *hdr_nextp = NULL;
+ erts_cleanup_offheap(factory->off_heap);
+ factory->off_heap->first = factory->off_heap_saved.first;
+ factory->off_heap->overhead = factory->off_heap_saved.overhead;
+ }
+
+ if (factory->mode == FACTORY_HALLOC) {
+ /* Free heap frags
+ */
+ bp = factory->p->mbuf;
+ if (bp != factory->heap_frags_saved) {
+ do {
+ ErlHeapFragment *next_bp = bp->next;
+ ASSERT(bp->off_heap.first == NULL);
+ ERTS_HEAP_FREE(ERTS_ALC_T_HEAP_FRAG, (void *) bp,
+ ERTS_HEAP_FRAG_SIZE(bp->alloc_size));
+ bp = next_bp;
+ } while (bp != factory->heap_frags_saved);
+
+ factory->p->mbuf = bp;
+ }
+
+ /* Rollback heap top
+ */
+ if (factory->heap_frags_saved == NULL) { /* No heap frags when we started */
+ ASSERT(factory->hp_start >= HEAP_START(factory->p));
+ ASSERT(factory->hp_start <= HEAP_LIMIT(factory->p));
+
+ HEAP_TOP(factory->p) = factory->hp_start;
+ }
+ else {
+ ASSERT(factory->heap_frags_saved == factory->p->mbuf);
+ if (factory->hp_start == factory->heap_frags_saved->mem) {
+ factory->p->mbuf = factory->p->mbuf->next;
+ ERTS_HEAP_FREE(ERTS_ALC_T_HEAP_FRAG, factory->heap_frags_saved,
+ ERTS_HEAP_FRAG_SIZE(factory->heap_frags_saved->alloc_size));
+ }
+ else if (factory->hp_start != factory->hp_end) {
+ unsigned remains = factory->hp_start - factory->heap_frags_saved->mem;
+ ASSERT(remains > 0 && remains < factory->heap_frags_saved->used_size);
+ factory->heap_frags_saved->used_size = remains;
+ }
+ }
+ }
+ break;
+
+ case FACTORY_HEAP_FRAGS:
+ bp = factory->heap_frags;
+ do {
+ ErlHeapFragment* next_bp = bp->next;
+
+ erts_cleanup_offheap(&bp->off_heap);
+ ERTS_HEAP_FREE(factory->alloc_type, (void *) bp,
+ ERTS_HEAP_FRAG_SIZE(bp->size));
+ bp = next_bp;
+ }while (bp != NULL);
+ break;
+
+ case FACTORY_CLOSED: break;
+ default:
+ ASSERT(!"Invalid factory mode");
+ }
+ factory->mode = FACTORY_CLOSED;
+#ifdef DEBUG
+ factory->p = NULL;
+ factory->hp = NULL;
+ factory->heap_frags = NULL;
+#endif
+}
+
diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h
index 1e1dafee90..39946f2a14 100644
--- a/erts/emulator/beam/erl_message.h
+++ b/erts/emulator/beam/erl_message.h
@@ -51,6 +51,44 @@ typedef struct erl_off_heap {
(OHP)->first = NULL; \
(OHP)->overhead = 0; \
} while (0)
+
+typedef struct {
+ enum {
+ FACTORY_CLOSED = 0,
+ FACTORY_HALLOC,
+ FACTORY_HEAP_FRAGS,
+ FACTORY_STATIC
+ } mode;
+ Process* p;
+ Eterm* hp_start;
+ Eterm* hp;
+ Eterm* hp_end;
+ struct erl_heap_fragment* heap_frags;
+ struct erl_heap_fragment* heap_frags_saved;
+ ErlOffHeap* off_heap;
+ ErlOffHeap off_heap_saved;
+ Uint32 alloc_type;
+} ErtsHeapFactory;
+
+void erts_factory_proc_init(ErtsHeapFactory*, Process*);
+void erts_factory_proc_prealloc_init(ErtsHeapFactory*, Process*, Sint size);
+void erts_factory_message_init(ErtsHeapFactory*, Process*, Eterm* hp, struct erl_heap_fragment*);
+void erts_factory_static_init(ErtsHeapFactory*, Eterm* hp, Uint size, ErlOffHeap*);
+void erts_factory_dummy_init(ErtsHeapFactory*);
+
+Eterm* erts_produce_heap(ErtsHeapFactory*, Uint need, Uint xtra);
+Eterm* erts_reserve_heap(ErtsHeapFactory*, Uint need);
+void erts_factory_close(ErtsHeapFactory*);
+void erts_factory_undo(ErtsHeapFactory*);
+
+#ifdef CHECK_FOR_HOLES
+# define ERTS_FACTORY_HOLE_CHECK(f) do { \
+ /*if ((f)->p) erts_check_for_holes((f)->p);*/ \
+ } while (0)
+#else
+# define ERTS_FACTORY_HOLE_CHECK(p)
+#endif
+
#include "external.h"
#include "erl_process.h"
@@ -68,21 +106,6 @@ struct erl_heap_fragment {
Eterm mem[1]; /* Data */
};
-typedef struct {
- Process* p;
- Eterm* hp;
-} ErtsHeapFactory;
-
-Eterm* erts_produce_heap(ErtsHeapFactory*, Uint need, Uint xtra);
-#ifdef CHECK_FOR_HOLES
-# define ERTS_FACTORY_HOLE_CHECK(f) do { \
- if ((f)->p) erts_check_for_holes((f)->p); \
- } while (0)
-#else
-# define ERTS_FACTORY_HOLE_CHECK(p)
-#endif
-
-
typedef struct erl_mesg {
struct erl_mesg* next; /* Next message */
union {
@@ -139,7 +162,7 @@ typedef struct {
*(p)->msg.last = (mp); \
(p)->msg.last = &(mp)->next; \
(p)->msg.len++; \
-} while(0)
+} while (0)
#ifdef ERTS_SMP
@@ -212,17 +235,23 @@ do { \
do { \
if ((M)->data.attached) { \
Uint need__ = erts_msg_attached_data_size((M)); \
+ { SWPO ; } \
if ((ST) - (HT) >= need__) { \
- Uint *htop__ = (HT); \
- erts_move_msg_attached_data_to_heap(&htop__, &MSO((P)), (M));\
- ASSERT(htop__ - (HT) <= need__); \
- (HT) = htop__; \
+ ErtsHeapFactory factory__; \
+ erts_factory_proc_prealloc_init(&factory__, (P), need__); \
+ erts_move_msg_attached_data_to_heap(&factory__, (M)); \
+ erts_factory_close(&factory__); \
+ if ((P)->mbuf != NULL) { \
+ /* Heap was exhausted by messages. This is a rare case */ \
+ /* that can currently (OTP 18) only happen if hamts are */ \
+ /* far exceeding the estimated heap size. Do GC. */ \
+ (FC) -= erts_garbage_collect((P), 0, NULL, 0); \
+ } \
} \
else { \
- { SWPO ; } \
(FC) -= erts_garbage_collect((P), 0, NULL, 0); \
- { SWPI ; } \
} \
+ { SWPI ; } \
ASSERT(!(M)->data.attached); \
} \
} while (0)
@@ -266,23 +295,21 @@ void erts_link_mbuf_to_proc(Process *proc, ErlHeapFragment *bp);
void erts_move_msg_mbuf_to_heap(Eterm**, ErlOffHeap*, ErlMessage *);
Uint erts_msg_attached_data_size_aux(ErlMessage *msg);
-void erts_move_msg_attached_data_to_heap(Eterm **, ErlOffHeap *, ErlMessage *);
-
+void erts_move_msg_attached_data_to_heap(ErtsHeapFactory*, ErlMessage *);
Eterm erts_msg_distext2heap(Process *, ErtsProcLocks *, ErlHeapFragment **,
Eterm *, ErtsDistExternal *);
void erts_cleanup_offheap(ErlOffHeap *offheap);
-ERTS_GLB_INLINE Uint erts_msg_used_frag_sz(const ErlMessage *msg);
+ERTS_GLB_INLINE Uint erts_used_frag_sz(const ErlHeapFragment*);
ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErlMessage *msg);
#if ERTS_GLB_INLINE_INCL_FUNC_DEF
-ERTS_GLB_INLINE Uint erts_msg_used_frag_sz(const ErlMessage *msg)
+ERTS_GLB_INLINE Uint erts_used_frag_sz(const ErlHeapFragment* bp)
{
- const ErlHeapFragment *bp;
Uint sz = 0;
- for (bp = msg->data.heap_frag; bp!=NULL; bp=bp->next) {
+ for ( ; bp!=NULL; bp=bp->next) {
sz += bp->used_size;
}
return sz;
@@ -292,7 +319,7 @@ ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErlMessage *msg)
{
ASSERT(msg->data.attached);
if (is_value(ERL_MESSAGE_TERM(msg)))
- return erts_msg_used_frag_sz(msg);
+ return erts_used_frag_sz(msg->data.heap_frag);
else if (msg->data.dist_ext->heap_size < 0)
return erts_msg_attached_data_size_aux(msg);
else {
diff --git a/erts/emulator/beam/erl_process_lock.c b/erts/emulator/beam/erl_process_lock.c
index fff267ff2a..b28bc498f6 100644
--- a/erts/emulator/beam/erl_process_lock.c
+++ b/erts/emulator/beam/erl_process_lock.c
@@ -1003,7 +1003,9 @@ erts_pid2proc_opt(Process *c_p,
void
erts_proc_lock_init(Process *p)
{
+#if ERTS_PROC_LOCK_OWN_IMPL || defined(ERTS_PROC_LOCK_DEBUG)
int i;
+#endif
#if ERTS_PROC_LOCK_OWN_IMPL
/* We always start with all locks locked */
#if ERTS_PROC_LOCK_ATOMIC_IMPL
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index fe48298155..0a69172980 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -94,9 +94,9 @@ static Uint is_external_string(Eterm obj, int* p_is_string);
static byte* enc_atom(ErtsAtomCacheMap *, Eterm, byte*, Uint32);
static byte* enc_pid(ErtsAtomCacheMap *, Eterm, byte*, Uint32);
struct B2TContext_t;
-static byte* dec_term(ErtsDistExternal *, Eterm**, byte*, ErlOffHeap*, Eterm*, struct B2TContext_t*);
+static byte* dec_term(ErtsDistExternal*, ErtsHeapFactory*, byte*, Eterm*, struct B2TContext_t*);
static byte* dec_atom(ErtsDistExternal *, byte*, Eterm*);
-static byte* dec_pid(ErtsDistExternal *, Eterm**, byte*, ErlOffHeap*, Eterm*);
+static byte* dec_pid(ErtsDistExternal *, ErtsHeapFactory*, byte*, Eterm*);
static Sint decoded_size(byte *ep, byte* endp, int internal_tags, struct B2TContext_t*);
static BIF_RETTYPE term_to_binary_trap_1(BIF_ALIST_1);
@@ -930,8 +930,7 @@ Sint erts_decode_ext_size_ets(byte *ext, Uint size)
** on return hpp is updated to point after allocated data
*/
Eterm
-erts_decode_dist_ext(Eterm** hpp,
- ErlOffHeap* off_heap,
+erts_decode_dist_ext(ErtsHeapFactory* factory,
ErtsDistExternal *edep)
{
Eterm obj;
@@ -951,7 +950,7 @@ erts_decode_dist_ext(Eterm** hpp,
goto error;
ep++;
}
- ep = dec_term(edep, hpp, ep, off_heap, &obj, NULL);
+ ep = dec_term(edep, factory, ep, &obj, NULL);
if (!ep)
goto error;
@@ -960,19 +959,22 @@ erts_decode_dist_ext(Eterm** hpp,
return obj;
error:
+ erts_factory_undo(factory);
bad_dist_ext(edep);
return THE_NON_VALUE;
}
-Eterm erts_decode_ext(Eterm **hpp, ErlOffHeap *off_heap, byte **ext)
+Eterm erts_decode_ext(ErtsHeapFactory* factory, byte **ext)
{
Eterm obj;
byte *ep = *ext;
- if (*ep++ != VERSION_MAGIC)
+ if (*ep++ != VERSION_MAGIC) {
+ erts_factory_undo(factory);
return THE_NON_VALUE;
- ep = dec_term(NULL, hpp, ep, off_heap, &obj, NULL);
+ }
+ ep = dec_term(NULL, factory, ep, &obj, NULL);
if (!ep) {
#ifdef DEBUG
bin_write(ERTS_PRINT_STDERR,NULL,*ext,500);
@@ -983,10 +985,10 @@ Eterm erts_decode_ext(Eterm **hpp, ErlOffHeap *off_heap, byte **ext)
return obj;
}
-Eterm erts_decode_ext_ets(Eterm **hpp, ErlOffHeap *off_heap, byte *ext)
+Eterm erts_decode_ext_ets(ErtsHeapFactory* factory, byte *ext)
{
Eterm obj;
- ext = dec_term(NULL, hpp, ext, off_heap, &obj, NULL);
+ ext = dec_term(NULL, factory, ext, &obj, NULL);
ASSERT(ext);
return obj;
}
@@ -995,9 +997,8 @@ Eterm erts_decode_ext_ets(Eterm **hpp, ErlOffHeap *off_heap, byte *ext)
BIF_RETTYPE erts_debug_dist_ext_to_term_2(BIF_ALIST_2)
{
+ ErtsHeapFactory factory;
Eterm res;
- Eterm *hp;
- Eterm *hendp;
Sint hsz;
ErtsDistExternal ede;
Eterm *tp;
@@ -1044,12 +1045,9 @@ BIF_RETTYPE erts_debug_dist_ext_to_term_2(BIF_ALIST_2)
if (hsz < 0)
goto badarg;
- hp = HAlloc(BIF_P, (Uint) hsz);
- hendp = hp + hsz;
-
- res = erts_decode_dist_ext(&hp, &MSO(BIF_P), &ede);
-
- HRelease(BIF_P, hendp, hp);
+ erts_factory_proc_prealloc_init(&factory, BIF_P, hsz);
+ res = erts_decode_dist_ext(&factory, &ede);
+ erts_factory_close(&factory);
if (is_value(res))
BIF_RET(res);
@@ -1177,13 +1175,11 @@ typedef struct {
byte* ep;
Eterm res;
Eterm* next;
- Eterm* hp_start;
- Eterm* hp;
- Eterm* hp_end;
+ ErtsHeapFactory factory;
int remaining_n;
char* remaining_bytes;
Eterm* maps_list;
- struct dec_term_hamt_placeholder* hamt_list;
+ ErtsPStack hamt_array;
} B2TDecodeContext;
typedef struct {
@@ -1307,10 +1303,12 @@ binary2term_abort(ErtsBinary2TermState *state)
}
static ERTS_INLINE Eterm
-binary2term_create(ErtsDistExternal *edep, ErtsBinary2TermState *state, Eterm **hpp, ErlOffHeap *ohp)
+binary2term_create(ErtsDistExternal *edep, ErtsBinary2TermState *state,
+ ErtsHeapFactory* factory)
{
Eterm res;
- if (!dec_term(edep, hpp, state->extp, ohp, &res, NULL))
+
+ if (!dec_term(edep, factory, state->extp, &res, NULL))
res = THE_NON_VALUE;
if (state->exttmp) {
state->exttmp = 0;
@@ -1343,9 +1341,9 @@ erts_binary2term_abort(ErtsBinary2TermState *state)
}
Eterm
-erts_binary2term_create(ErtsBinary2TermState *state, Eterm **hpp, ErlOffHeap *ohp)
+erts_binary2term_create(ErtsBinary2TermState *state, ErtsHeapFactory* factory)
{
- return binary2term_create(NULL,state, hpp, ohp);
+ return binary2term_create(NULL,state, factory);
}
static void b2t_destroy_context(B2TContext* context)
@@ -1354,8 +1352,21 @@ static void b2t_destroy_context(B2TContext* context)
ERTS_ALC_T_EXT_TERM_DATA);
context->aligned_alloc = NULL;
binary2term_abort(&context->b2ts);
- if (context->state == B2TUncompressChunk) {
+ switch (context->state) {
+ case B2TUncompressChunk:
erl_zlib_inflate_finish(&context->u.uc.stream);
+ break;
+ case B2TDecode:
+ case B2TDecodeList:
+ case B2TDecodeTuple:
+ case B2TDecodeString:
+ case B2TDecodeBinary:
+ if (context->u.dc.hamt_array.pstart) {
+ erts_free(context->u.dc.hamt_array.alloc_type,
+ context->u.dc.hamt_array.pstart);
+ }
+ break;
+ default:;
}
}
@@ -1506,11 +1517,9 @@ static BIF_RETTYPE binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binar
ctx->u.dc.ep = ctx->b2ts.extp;
ctx->u.dc.res = (Eterm) (UWord) NULL;
ctx->u.dc.next = &ctx->u.dc.res;
- ctx->u.dc.hp_start = HAlloc(p, ctx->heap_size);
- ctx->u.dc.hp = ctx->u.dc.hp_start;
- ctx->u.dc.hp_end = ctx->u.dc.hp_start + ctx->heap_size;
+ erts_factory_proc_prealloc_init(&ctx->u.dc.factory, p, ctx->heap_size);
ctx->u.dc.maps_list = NULL;
- ctx->u.dc.hamt_list = NULL;
+ ctx->u.dc.hamt_array.pstart = NULL;
ctx->state = B2TDecode;
/*fall through*/
case B2TDecode:
@@ -1520,11 +1529,10 @@ static BIF_RETTYPE binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binar
case B2TDecodeBinary: {
ErtsDistExternal fakedep;
fakedep.flags = ctx->flags;
- dec_term(&fakedep, NULL, NULL, &MSO(p), NULL, ctx);
+ dec_term(&fakedep, NULL, NULL, NULL, ctx);
break;
}
case B2TDecodeFail:
- HRelease(p, ctx->u.dc.hp_end, ctx->u.dc.hp_start);
/*fall through*/
case B2TBadArg:
BUMP_REDS(p, (initial_reds - ctx->reds) / B2T_BYTES_PER_REDUCTION);
@@ -1549,11 +1557,11 @@ static BIF_RETTYPE binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binar
case B2TDone:
b2t_destroy_context(ctx);
- if (ctx->u.dc.hp > ctx->u.dc.hp_end) {
+ if (ctx->u.dc.factory.hp > ctx->u.dc.factory.hp_end) {
erl_exit(1, ":%s, line %d: heap overrun by %d words(s)\n",
- __FILE__, __LINE__, ctx->u.dc.hp - ctx->u.dc.hp_end);
+ __FILE__, __LINE__, ctx->u.dc.factory.hp - ctx->u.dc.factory.hp_end);
}
- HRelease(p, ctx->u.dc.hp_end, ctx->u.dc.hp);
+ erts_factory_close(&ctx->u.dc.factory);
if (!is_first_call) {
erts_set_gc_state(p, 1);
@@ -2247,7 +2255,7 @@ static ERTS_INLINE ErlNode* dec_get_node(Eterm sysname, Uint creation)
}
static byte*
-dec_pid(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Eterm* objp)
+dec_pid(ErtsDistExternal *edep, ErtsHeapFactory* factory, byte* ep, Eterm* objp)
{
Eterm sysname;
Uint data;
@@ -2286,15 +2294,15 @@ dec_pid(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Ete
if(node == erts_this_node) {
*objp = make_internal_pid(data);
} else {
- ExternalThing *etp = (ExternalThing *) *hpp;
- *hpp += EXTERNAL_THING_HEAD_SIZE + 1;
+ ExternalThing *etp = (ExternalThing *) factory->hp;
+ factory->hp += EXTERNAL_THING_HEAD_SIZE + 1;
etp->header = make_external_pid_header(1);
- etp->next = off_heap->first;
+ etp->next = factory->off_heap->first;
etp->node = node;
etp->data.ui[0] = data;
- off_heap->first = (struct erl_off_heap_header*) etp;
+ factory->off_heap->first = (struct erl_off_heap_header*) etp;
*objp = make_external_pid(etp);
}
return ep;
@@ -2905,69 +2913,43 @@ is_external_string(Eterm list, int* p_is_string)
return len;
}
-/* Assumes that the ones to undo are preluding the list. */
-static void
-undo_offheap_in_area(ErlOffHeap* off_heap, Eterm* start, Eterm* end)
-{
- const Uint area_sz = (end - start) * sizeof(Eterm);
- struct erl_off_heap_header* hdr;
- struct erl_off_heap_header** hdr_nextp = NULL;
-
- for (hdr = off_heap->first; ; hdr=hdr->next) {
- if (!in_area(hdr, start, area_sz)) {
- if (hdr_nextp != NULL) {
- *hdr_nextp = NULL;
- erts_cleanup_offheap(off_heap);
- off_heap->first = hdr;
- }
- break;
- }
- hdr_nextp = &hdr->next;
- }
-
- /* Assert that the ones to undo were indeed preluding the list. */
-#ifdef DEBUG
- for (hdr = off_heap->first; hdr != NULL; hdr = hdr->next) {
- ASSERT(!in_area(hdr, start, area_sz));
- }
-#endif /* DEBUG */
-}
-struct dec_term_hamt_placeholder
+struct dec_term_hamt
{
- struct dec_term_hamt_placeholder* next;
Eterm* objp; /* write result here */
Uint size; /* nr of leafs */
- Eterm leafs[1];
+ Eterm* leaf_array;
};
-#define DEC_TERM_HAMT_PLACEHOLDER_SIZE \
- (offsetof(struct dec_term_hamt_placeholder, leafs) / sizeof(Eterm))
/* Decode term from external format into *objp.
-** On failure return NULL and *hpp will be unchanged.
+** On failure calls erts_factory_undo() and returns NULL
*/
static byte*
-dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap,
- Eterm* objp, B2TContext* ctx)
+dec_term(ErtsDistExternal *edep,
+ ErtsHeapFactory* factory,
+ byte* ep,
+ Eterm* objp,
+ B2TContext* ctx)
{
- Eterm* hp_saved;
+#define PSTACK_TYPE struct dec_term_hamt
+ PSTACK_DECLARE(hamt_array, 5);
int n;
ErtsAtomEncoding char_enc;
register Eterm* hp; /* Please don't take the address of hp */
Eterm *maps_list; /* for preprocessing of small maps */
- struct dec_term_hamt_placeholder* hamt_list; /* for preprocessing of big maps */
Eterm* next;
SWord reds;
+#ifdef DEBUG
+ Eterm* dbg_resultp = ctx ? &ctx->u.dc.res : objp;
+#endif
if (ctx) {
- hp_saved = ctx->u.dc.hp_start;
reds = ctx->reds;
next = ctx->u.dc.next;
ep = ctx->u.dc.ep;
- hpp = &ctx->u.dc.hp;
+ factory = &ctx->u.dc.factory;
maps_list = ctx->u.dc.maps_list;
- hamt_list = ctx->u.dc.hamt_list;
if (ctx->state != B2TDecode) {
int n_limit = reds;
@@ -3012,7 +2994,7 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap,
break;
case B2TDecodeString:
- hp = *hpp;
+ hp = factory->hp;
hp[-1] = make_list(hp); /* overwrite the premature NIL */
while (n-- > 0) {
hp[0] = make_small(*ep++);
@@ -3020,7 +3002,7 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap,
hp += 2;
}
hp[-1] = NIL;
- *hpp = hp;
+ factory->hp = hp;
break;
case B2TDecodeBinary:
@@ -3042,16 +3024,18 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap,
return NULL;
}
}
+ PSTACK_CHANGE_ALLOCATOR(hamt_array, ERTS_ALC_T_SAVED_ESTACK);
+ if (ctx->u.dc.hamt_array.pstart) {
+ PSTACK_RESTORE(hamt_array, &ctx->u.dc.hamt_array);
+ }
}
else {
- hp_saved = *hpp;
reds = ERTS_SWORD_MAX;
next = objp;
*next = (Eterm) (UWord) NULL;
maps_list = NULL;
- hamt_list = NULL;
}
- hp = *hpp;
+ hp = factory->hp;
while (next != NULL) {
@@ -3288,9 +3272,9 @@ dec_term_atom_common:
break;
}
case PID_EXT:
- *hpp = hp;
- ep = dec_pid(edep, hpp, ep, off_heap, objp);
- hp = *hpp;
+ factory->hp = hp;
+ ep = dec_pid(edep, factory, ep, objp);
+ hp = factory->hp;
if (ep == NULL) {
goto error;
}
@@ -3324,11 +3308,11 @@ dec_term_atom_common:
hp += EXTERNAL_THING_HEAD_SIZE + 1;
etp->header = make_external_port_header(1);
- etp->next = off_heap->first;
+ etp->next = factory->off_heap->first;
etp->node = node;
etp->data.ui[0] = num;
- off_heap->first = (struct erl_off_heap_header*)etp;
+ factory->off_heap->first = (struct erl_off_heap_header*)etp;
*objp = make_external_port(etp);
}
@@ -3408,10 +3392,10 @@ dec_term_atom_common:
#else
etp->header = make_external_ref_header(ref_words);
#endif
- etp->next = off_heap->first;
+ etp->next = factory->off_heap->first;
etp->node = node;
- off_heap->first = (struct erl_off_heap_header*)etp;
+ factory->off_heap->first = (struct erl_off_heap_header*)etp;
*objp = make_external_ref(etp);
ref_num = &(etp->data.ui32[0]);
}
@@ -3451,9 +3435,9 @@ dec_term_atom_common:
hp += PROC_BIN_SIZE;
pb->thing_word = HEADER_PROC_BIN;
pb->size = n;
- pb->next = off_heap->first;
- off_heap->first = (struct erl_off_heap_header*)pb;
- OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm));
+ pb->next = factory->off_heap->first;
+ factory->off_heap->first = (struct erl_off_heap_header*)pb;
+ OH_OVERHEAD(factory->off_heap, pb->size / sizeof(Eterm));
pb->val = dbin;
pb->bytes = (byte*) dbin->orig_bytes;
pb->flags = 0;
@@ -3503,9 +3487,9 @@ dec_term_atom_common:
pb = (ProcBin *) hp;
pb->thing_word = HEADER_PROC_BIN;
pb->size = n;
- pb->next = off_heap->first;
- off_heap->first = (struct erl_off_heap_header*)pb;
- OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm));
+ pb->next = factory->off_heap->first;
+ factory->off_heap->first = (struct erl_off_heap_header*)pb;
+ OH_OVERHEAD(factory->off_heap, pb->size / sizeof(Eterm));
pb->val = dbin;
pb->bytes = (byte*) dbin->orig_bytes;
pb->flags = 0;
@@ -3557,9 +3541,9 @@ dec_term_atom_common:
if ((ep = dec_atom(edep, ep, &name)) == NULL) {
goto error;
}
- *hpp = hp;
- ep = dec_term(edep, hpp, ep, off_heap, &temp, NULL);
- hp = *hpp;
+ factory->hp = hp;
+ ep = dec_term(edep, factory, ep, &temp, NULL);
+ hp = factory->hp;
if (ep == NULL) {
goto error;
}
@@ -3631,15 +3615,11 @@ dec_term_atom_common:
}
}
else { /* Make hamt */
- struct dec_term_hamt_placeholder* holder =
- (struct dec_term_hamt_placeholder*) hp;
-
- holder->next = hamt_list;
- hamt_list = holder;
- holder->objp = objp;
- holder->size = size;
+ struct dec_term_hamt* hamt = PSTACK_PUSH(hamt_array);
- hp += DEC_TERM_HAMT_PLACEHOLDER_SIZE;
+ hamt->objp = objp;
+ hamt->size = size;
+ hamt->leaf_array = hp;
for (n = size; n; n--) {
CDR(hp) = (Eterm) COMPRESS_POINTER(next);
@@ -3681,9 +3661,9 @@ dec_term_atom_common:
if ((ep = dec_atom(edep, ep, &module)) == NULL) {
goto error;
}
- *hpp = hp;
+ factory->hp = hp;
/* Index */
- if ((ep = dec_term(edep, hpp, ep, off_heap, &temp, NULL)) == NULL) {
+ if ((ep = dec_term(edep, factory, ep, &temp, NULL)) == NULL) {
goto error;
}
if (!is_small(temp)) {
@@ -3692,7 +3672,7 @@ dec_term_atom_common:
old_index = unsigned_val(temp);
/* Uniq */
- if ((ep = dec_term(edep, hpp, ep, off_heap, &temp, NULL)) == NULL) {
+ if ((ep = dec_term(edep, factory, ep, &temp, NULL)) == NULL) {
goto error;
}
if (!is_small(temp)) {
@@ -3704,8 +3684,8 @@ dec_term_atom_common:
* It is safe to link the fun into the fun list only when
* no more validity tests can fail.
*/
- funp->next = off_heap->first;
- off_heap->first = (struct erl_off_heap_header*)funp;
+ funp->next = factory->off_heap->first;
+ factory->off_heap->first = (struct erl_off_heap_header*)funp;
funp->fe = erts_put_fun_entry2(module, old_uniq, old_index,
uniq, index, arity);
@@ -3716,7 +3696,7 @@ dec_term_atom_common:
}
funp->native_address = funp->fe->native_address;
#endif
- hp = *hpp;
+ hp = factory->hp;
/* Environment */
for (i = num_free-1; i >= 0; i--) {
@@ -3742,14 +3722,14 @@ dec_term_atom_common:
ep += 4;
hp += ERL_FUN_SIZE;
hp += num_free;
- *hpp = hp;
+ factory->hp = hp;
funp->thing_word = HEADER_FUN;
funp->num_free = num_free;
*objp = make_fun(funp);
/* Creator pid */
if (*ep != PID_EXT
- || (ep = dec_pid(edep, hpp, ++ep, off_heap,
+ || (ep = dec_pid(edep, factory, ++ep,
&funp->creator))==NULL) {
goto error;
}
@@ -3760,7 +3740,7 @@ dec_term_atom_common:
}
/* Index */
- if ((ep = dec_term(edep, hpp, ep, off_heap, &temp, NULL)) == NULL) {
+ if ((ep = dec_term(edep, factory, ep, &temp, NULL)) == NULL) {
goto error;
}
if (!is_small(temp)) {
@@ -3769,7 +3749,7 @@ dec_term_atom_common:
old_index = unsigned_val(temp);
/* Uniq */
- if ((ep = dec_term(edep, hpp, ep, off_heap, &temp, NULL)) == NULL) {
+ if ((ep = dec_term(edep, factory, ep, &temp, NULL)) == NULL) {
goto error;
}
if (!is_small(temp)) {
@@ -3780,8 +3760,8 @@ dec_term_atom_common:
* It is safe to link the fun into the fun list only when
* no more validity tests can fail.
*/
- funp->next = off_heap->first;
- off_heap->first = (struct erl_off_heap_header*)funp;
+ funp->next = factory->off_heap->first;
+ factory->off_heap->first = (struct erl_off_heap_header*)funp;
old_uniq = unsigned_val(temp);
funp->fe = erts_put_fun_entry(module, old_uniq, old_index);
@@ -3789,7 +3769,7 @@ dec_term_atom_common:
#ifdef HIPE
funp->native_address = funp->fe->native_address;
#endif
- hp = *hpp;
+ hp = factory->hp;
/* Environment */
for (i = num_free-1; i >= 0; i--) {
@@ -3823,9 +3803,9 @@ dec_term_atom_common:
erts_refc_inc(&pb->val->refc, 1);
hp += PROC_BIN_SIZE;
- pb->next = off_heap->first;
- off_heap->first = (struct erl_off_heap_header*)pb;
- OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm));
+ pb->next = factory->off_heap->first;
+ factory->off_heap->first = (struct erl_off_heap_header*)pb;
+ OH_OVERHEAD(factory->off_heap, pb->size / sizeof(Eterm));
pb->flags = 0;
*objp = make_binary(pb);
break;
@@ -3841,9 +3821,9 @@ dec_term_atom_common:
erts_refc_inc(&pb->val->refc, 1);
hp += PROC_BIN_SIZE;
- pb->next = off_heap->first;
- off_heap->first = (struct erl_off_heap_header*)pb;
- OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm));
+ pb->next = factory->off_heap->first;
+ factory->off_heap->first = (struct erl_off_heap_header*)pb;
+ OH_OVERHEAD(factory->off_heap, pb->size / sizeof(Eterm));
pb->flags = 0;
sub = (ErlSubBin*)hp;
@@ -3869,9 +3849,11 @@ dec_term_atom_common:
if (next || ctx->state != B2TDecode) {
ctx->u.dc.ep = ep;
ctx->u.dc.next = next;
- ctx->u.dc.hp = hp;
+ ctx->u.dc.factory.hp = hp;
ctx->u.dc.maps_list = maps_list;
- ctx->u.dc.hamt_list = hamt_list;
+ if (!PSTACK_IS_EMPTY(hamt_array)) {
+ PSTACK_SAVE(hamt_array, &ctx->u.dc.hamt_array);
+ }
ctx->reds = 0;
return NULL;
}
@@ -3894,40 +3876,36 @@ dec_term_atom_common:
maps_list = next;
}
- /* Iterate through all the hamts and build tree nodes.
+ ASSERT(hp <= factory->hp_end
+ || (factory->mode == FACTORY_CLOSED && is_immed(*dbg_resultp)));
+ factory->hp = hp;
+ /*
+ * From here on factory may produce (more) heap fragments
*/
- if (hamt_list) {
- ErtsHeapFactory factory;
- factory.p = NULL;
- factory.hp = hp;
- /* We assume heap will suffice (see hashmap_over_estimated_heap_size) */
+ if (!PSTACK_IS_EMPTY(hamt_array)) {
+ do {
+ struct dec_term_hamt* hamt = PSTACK_TOP(hamt_array);
- do {
- struct dec_term_hamt_placeholder* hamt = hamt_list;
- *hamt->objp = erts_hashmap_from_array(&factory,
- hamt->leafs,
+ *hamt->objp = erts_hashmap_from_array(factory,
+ hamt->leaf_array,
hamt->size,
1);
if (is_non_value(*hamt->objp))
- goto error;
+ goto error_hamt;
- hamt_list = hamt->next;
-
- /* Yes, we waste a couple of heap words per hamt
- for the temporary placeholder */
- *(Eterm*)hamt = make_pos_bignum_header(DEC_TERM_HAMT_PLACEHOLDER_SIZE-1);
- } while (hamt_list);
-
- hp = factory.hp;
+ (void) PSTACK_POP(hamt_array);
+ } while (!PSTACK_IS_EMPTY(hamt_array));
+ PSTACK_DESTROY(hamt_array);
}
+ ASSERT((Eterm*)EXPAND_POINTER(*dbg_resultp) != NULL);
+
if (ctx) {
ctx->state = B2TDone;
ctx->reds = reds;
}
- *hpp = hp;
return ep;
error:
@@ -3935,11 +3913,12 @@ error:
* Must unlink all off-heap objects that may have been
* linked into the process.
*/
- if (hp < *hpp) { /* Sometimes we used hp and sometimes *hpp */
- hp = *hpp; /* the largest must be the freshest */
+ if (factory->hp < hp) { /* Sometimes we used hp and sometimes factory->hp */
+ factory->hp = hp; /* the largest must be the freshest */
}
- undo_offheap_in_area(off_heap, hp_saved, hp);
- *hpp = hp_saved;
+error_hamt:
+ erts_factory_undo(factory);
+ PSTACK_DESTROY(hamt_array);
if (ctx) {
ctx->state = B2TDecodeFail;
ctx->reds = reds;
@@ -4465,7 +4444,7 @@ init_done:
if (n <= MAP_SMALL_MAP_LIMIT) {
heap_size += 3 + n + 1 + n;
} else {
- heap_size += hashmap_over_estimated_heap_size(n);
+ heap_size += HASHMAP_ESTIMATED_HEAP_SIZE(n);
}
break;
case STRING_EXT:
diff --git a/erts/emulator/beam/external.h b/erts/emulator/beam/external.h
index 50fcfa04d6..508ab8dc17 100644
--- a/erts/emulator/beam/external.h
+++ b/erts/emulator/beam/external.h
@@ -148,6 +148,7 @@ typedef struct {
byte *extp;
int exttmp;
Uint extsize;
+ Uint heap_size;
} ErtsBinary2TermState;
@@ -185,18 +186,18 @@ void erts_destroy_dist_ext_copy(ErtsDistExternal *);
int erts_prepare_dist_ext(ErtsDistExternal *, byte *, Uint,
DistEntry *, ErtsAtomCache *);
Sint erts_decode_dist_ext_size(ErtsDistExternal *);
-Eterm erts_decode_dist_ext(Eterm **, ErlOffHeap *, ErtsDistExternal *);
+Eterm erts_decode_dist_ext(ErtsHeapFactory* factory, ErtsDistExternal *);
Sint erts_decode_ext_size(byte*, Uint);
Sint erts_decode_ext_size_ets(byte*, Uint);
-Eterm erts_decode_ext(Eterm **, ErlOffHeap *, byte**);
-Eterm erts_decode_ext_ets(Eterm **, ErlOffHeap *, byte*);
+Eterm erts_decode_ext(ErtsHeapFactory*, byte**);
+Eterm erts_decode_ext_ets(ErtsHeapFactory*, byte*);
Eterm erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags);
Sint erts_binary2term_prepare(ErtsBinary2TermState *, byte *, Sint);
void erts_binary2term_abort(ErtsBinary2TermState *);
-Eterm erts_binary2term_create(ErtsBinary2TermState *, Eterm **hpp, ErlOffHeap *);
+Eterm erts_binary2term_create(ErtsBinary2TermState *, ErtsHeapFactory*);
int erts_debug_max_atom_out_cache_index(void);
int erts_debug_atom_to_out_cache_index(Eterm);
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index ccc7da265e..07a82b42d2 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -1395,31 +1395,26 @@ finalize_force_imm_drv_call(ErtsTryImmDrvCallState *sp)
static ERTS_INLINE void
queue_port_sched_op_reply(Process *rp,
ErtsProcLocks *rp_locksp,
- Eterm *hp_start,
- Eterm *hp,
- Uint h_size,
- ErlHeapFragment* bp,
+ ErtsHeapFactory* factory,
Uint32 *ref_num,
Eterm msg)
{
- Eterm ref = make_internal_ref(hp);
+ Eterm* hp = erts_produce_heap(factory, ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE, 0);
+ Eterm ref;
+
+ ref= make_internal_ref(hp);
write_ref_thing(hp, ref_num[0], ref_num[1], ref_num[2]);
hp += REF_THING_SIZE;
msg = TUPLE2(hp, ref, msg);
hp += 3;
- if (!bp) {
- HRelease(rp, hp_start + h_size, hp);
- }
- else {
- Uint used_h_size = hp - hp_start;
- ASSERT(h_size >= used_h_size);
- if (h_size > used_h_size)
- bp = erts_resize_message_buffer(bp, used_h_size, &msg, 1);
- }
+ /* SVERK: We used to call erts_resize_message_buffer here
+ * to maybe realloc message.
+ */
+ erts_factory_close(factory);
- erts_queue_message(rp, rp_locksp, bp, msg, NIL);
+ erts_queue_message(rp, rp_locksp, factory->heap_frags, msg, NIL);
}
static void
@@ -1429,9 +1424,10 @@ port_sched_op_reply(Eterm to, Uint32 *ref_num, Eterm msg)
if (rp) {
ErlOffHeap *ohp;
ErlHeapFragment* bp;
+ ErtsHeapFactory factory;
Eterm msg_copy;
Uint hsz, msg_sz;
- Eterm *hp, *hp_start;
+ Eterm *hp;
ErtsProcLocks rp_locks = 0;
hsz = ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE;
@@ -1442,22 +1438,22 @@ port_sched_op_reply(Eterm to, Uint32 *ref_num, Eterm msg)
hsz += msg_sz;
}
- hp_start = hp = erts_alloc_message_heap(hsz,
+ hp = erts_alloc_message_heap(hsz,
&bp,
&ohp,
rp,
&rp_locks);
+ erts_factory_message_init(&factory, rp, hp, bp);
if (is_immed(msg))
msg_copy = msg;
- else
+ else {
msg_copy = copy_struct(msg, msg_sz, &hp, ohp);
+ factory.hp = hp;
+ }
queue_port_sched_op_reply(rp,
&rp_locks,
- hp_start,
- hp,
- hsz,
- bp,
+ &factory,
ref_num,
msg_copy);
@@ -3896,12 +3892,13 @@ port_sig_control(Port *prt,
if (res == ERTS_PORT_OP_DONE) {
Eterm msg;
- Eterm *hp, *hp_start;
+ Eterm *hp;
ErlHeapFragment *bp;
ErlOffHeap *ohp;
+ ErtsHeapFactory factory;
Process *rp;
ErtsProcLocks rp_locks = 0;
- Uint hsz;
+ Uint hsz, rsz;
int control_flags;
rp = erts_proc_lookup_raw(sigdp->caller);
@@ -3910,17 +3907,19 @@ port_sig_control(Port *prt,
control_flags = prt->control_flags;
- hsz = ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE;
- hsz += port_control_result_size(control_flags,
+ rsz = port_control_result_size(control_flags,
resp_bufp,
&resp_size,
&resp_buf[0]);
+ hsz = rsz + ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE;
+
- hp_start = hp = erts_alloc_message_heap(hsz,
+ hp = erts_alloc_message_heap(hsz,
&bp,
&ohp,
rp,
&rp_locks);
+ erts_factory_message_init(&factory, rp, hp, bp);
msg = write_port_control_result(control_flags,
resp_bufp,
@@ -3929,13 +3928,11 @@ port_sig_control(Port *prt,
&hp,
bp,
ohp);
+ factory.hp = hp;
queue_port_sched_op_reply(rp,
&rp_locks,
- hp_start,
- hp,
- hsz,
- bp,
+ &factory,
sigdp->ref,
msg);
@@ -4222,8 +4219,6 @@ port_sig_call(Port *prt,
if (res == ERTS_PORT_OP_DONE) {
Eterm msg;
Eterm *hp;
- ErlHeapFragment *bp;
- ErlOffHeap *ohp;
Process *rp;
ErtsProcLocks rp_locks = 0;
Sint hsz;
@@ -4234,29 +4229,31 @@ port_sig_call(Port *prt,
hsz = erts_decode_ext_size((byte *) resp_bufp, resp_size);
if (hsz >= 0) {
- Eterm *hp_start;
+ ErlHeapFragment* bp;
+ ErlOffHeap* ohp;
+ ErtsHeapFactory factory;
byte *endp;
hsz += 3; /* ok tuple */
hsz += ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE;
- hp_start = hp = erts_alloc_message_heap(hsz,
- &bp,
- &ohp,
- rp,
- &rp_locks);
+ hp = erts_alloc_message_heap(hsz,
+ &bp,
+ &ohp,
+ rp,
+ &rp_locks);
endp = (byte *) resp_bufp;
- msg = erts_decode_ext(&hp, ohp, &endp);
+ erts_factory_message_init(&factory, rp, hp, bp);
+ msg = erts_decode_ext(&factory, &endp);
if (is_value(msg)) {
+ hp = erts_produce_heap(&factory,
+ 3,
+ ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE);
msg = TUPLE2(hp, am_ok, msg);
- hp += 3;
queue_port_sched_op_reply(rp,
&rp_locks,
- hp_start,
- hp,
- hsz,
- bp,
+ &factory,
sigdp->ref,
msg);
@@ -4264,8 +4261,6 @@ port_sig_call(Port *prt,
erts_smp_proc_unlock(rp, rp_locks);
goto done;
}
- if (bp)
- free_message_buffer(bp);
if (rp_locks)
erts_smp_proc_unlock(rp, rp_locks);
}
@@ -4342,10 +4337,11 @@ erts_port_call(Process* c_p,
try_call_res = try_imm_drv_call(&try_call_state);
switch (try_call_res) {
case ERTS_TRY_IMM_DRV_CALL_OK: {
- Eterm *hp, *hp_end;
+ ErtsHeapFactory factory;
Sint hsz;
unsigned ret_flags = 0U;
Eterm term;
+ Eterm* hp;
res = call_driver_call(c_p->common.id,
prt,
@@ -4365,15 +4361,14 @@ erts_port_call(Process* c_p,
if (hsz < 0)
return ERTS_PORT_OP_BADARG;
hsz += 3;
- hp = HAlloc(c_p, hsz);
- hp_end = hp + hsz;
+ erts_factory_proc_prealloc_init(&factory, c_p, hsz);
endp = (byte *) resp_bufp;
- term = erts_decode_ext(&hp, &MSO(c_p), &endp);
+ term = erts_decode_ext(&factory, &endp);
if (term == THE_NON_VALUE)
return ERTS_PORT_OP_BADARG;
+ hp = erts_produce_heap(&factory,3,0);
*retvalp = TUPLE2(hp, am_ok, term);
- hp += 3;
- HRelease(c_p, hp_end, hp);
+ erts_factory_close(&factory);
if (resp_bufp != &resp_buf[0]
&& !(ret_flags & DRIVER_CALL_KEEP_BUFFER))
driver_free(resp_bufp);
@@ -4508,12 +4503,11 @@ port_sig_info(Port *prt,
prt,
sigdp->u.info.item);
if (is_value(value)) {
+ ErtsHeapFactory factory;
+ erts_factory_message_init(&factory, NULL, hp, bp);
queue_port_sched_op_reply(rp,
&rp_locks,
- hp_start,
- hp,
- hsz,
- bp,
+ &factory,
sigdp->ref,
value);
}
@@ -5106,22 +5100,22 @@ cleanup_b2t_states(struct b2t_states__ *b2tsp)
static int
driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
{
+#define HEAP_EXTRA 200
#define ERTS_DDT_FAIL do { res = -1; goto done; } while (0)
Uint need = 0;
int depth = 0;
int res;
- Eterm *hp = NULL, *hp_start = NULL, *hp_end = NULL;
ErlDrvTermData* ptr;
ErlDrvTermData* ptr_end;
DECLARE_ESTACK(stack);
Eterm mess = NIL; /* keeps compiler happy */
Process* rp = NULL;
- ErlHeapFragment *bp = NULL;
- ErlOffHeap *ohp;
+ ErtsHeapFactory factory;
ErtsProcLocks rp_locks = 0;
struct b2t_states__ b2t;
int scheduler = 1; /* Silence erroneous warning... */
+ factory.mode = FACTORY_CLOSED;
init_b2t_states(&b2t);
/*
@@ -5285,9 +5279,10 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
#ifdef DEBUG
b2t.org_ext[b2t.ix] = ext;
#endif
- hsz = erts_binary2term_prepare(&b2t.state[b2t.ix++], ext, size);
+ hsz = erts_binary2term_prepare(&b2t.state[b2t.ix], ext, size);
if (hsz < 0)
ERTS_DDT_FAIL; /* Invalid data */
+ b2t.state[b2t.ix++].heap_size = hsz;
need += hsz;
ptr += 2;
depth++;
@@ -5297,7 +5292,7 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
ERTS_DDT_CHK_ENOUGH_ARGS(1);
if ((int) ptr[0] < 0) ERTS_DDT_FAIL;
if (ptr[0] > MAP_SMALL_MAP_LIMIT) {
- need += hashmap_over_estimated_heap_size(ptr[0]);
+ need += HASHMAP_ESTIMATED_HEAP_SIZE(ptr[0]);
} else {
need += MAP_HEADER_FLATMAP_SZ + 1 + 2*ptr[0];
}
@@ -5336,8 +5331,14 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
goto done;
}
- hp_start = hp = erts_alloc_message_heap(need, &bp, &ohp, rp, &rp_locks);
- hp_end = hp + need;
+
+ /* We could try to copy directly to destination heap
+ * if we knew there is no big maps.
+
+ erts_alloc_message_heap(need, &bp, &ohp, rp, &rp_locks);
+ */
+ erts_factory_message_init(&factory, NULL, NULL,
+ new_message_buffer(need));
/*
* Interpret the instructions and build the term.
@@ -5358,13 +5359,15 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
case ERL_DRV_INT: /* signed int argument */
#if HALFWORD_HEAP
- mess = erts_bld_sint64(&hp, NULL, (Sint64)ptr[0]);
+ erts_reserve_heap(&factory, BIG_NEED_SIZE(2));
+ mess = erts_bld_sint64(&factory.hp, NULL, (Sint64)ptr[0]);
#else
+ erts_reserve_heap(&factory, BIG_UINT_HEAP_SIZE);
if (IS_SSMALL((Sint)ptr[0]))
mess = make_small((Sint)ptr[0]);
else {
- mess = small_to_big((Sint)ptr[0], hp);
- hp += BIG_UINT_HEAP_SIZE;
+ mess = small_to_big((Sint)ptr[0], factory.hp);
+ factory.hp += BIG_UINT_HEAP_SIZE;
}
#endif
ptr++;
@@ -5372,25 +5375,29 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
case ERL_DRV_UINT: /* unsigned int argument */
#if HALFWORD_HEAP
- mess = erts_bld_uint64(&hp, NULL, (Uint64)ptr[0]);
+ erts_reserve_heap(&factory, BIG_NEED_FOR_BITS(64));
+ mess = erts_bld_uint64(&factory.hp, NULL, (Uint64)ptr[0]);
#else
+ erts_reserve_heap(&factory, BIG_UINT_HEAP_SIZE);
if (IS_USMALL(0, (Uint)ptr[0]))
mess = make_small((Uint)ptr[0]);
else {
- mess = uint_to_big((Uint)ptr[0], hp);
- hp += BIG_UINT_HEAP_SIZE;
+ mess = uint_to_big((Uint)ptr[0], factory.hp);
+ factory.hp += BIG_UINT_HEAP_SIZE;
}
#endif
ptr++;
break;
case ERL_DRV_INT64: /* pointer to unsigned 64-bit int argument */
- mess = erts_bld_sint64(&hp, NULL, *((Sint64 *) ptr[0]));
+ erts_reserve_heap(&factory, BIG_NEED_FOR_BITS(64));
+ mess = erts_bld_sint64(&factory.hp, NULL, *((Sint64 *) ptr[0]));
ptr++;
break;
case ERL_DRV_UINT64: /* pointer to unsigned 64-bit int argument */
- mess = erts_bld_uint64(&hp, NULL, *((Uint64 *) ptr[0]));
+ erts_reserve_heap(&factory, BIG_NEED_FOR_BITS(64));
+ mess = erts_bld_uint64(&factory.hp, NULL, *((Uint64 *) ptr[0]));
ptr++;
break;
@@ -5407,8 +5414,8 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) size);
if (size <= ERL_ONHEAP_BIN_LIMIT) {
- ErlHeapBin* hbp = (ErlHeapBin *) hp;
- hp += heap_bin_size(size);
+ ErlHeapBin* hbp = (ErlHeapBin *) erts_produce_heap(&factory,
+ heap_bin_size(size), HEAP_EXTRA);
hbp->thing_word = header_heap_bin(size);
hbp->size = size;
if (size > 0) {
@@ -5417,18 +5424,18 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
mess = make_binary(hbp);
}
else {
- ProcBin* pb = (ProcBin *) hp;
+ ProcBin* pb = (ProcBin *) erts_produce_heap(&factory,
+ PROC_BIN_SIZE, HEAP_EXTRA);
driver_binary_inc_refc(b); /* caller will free binary */
pb->thing_word = HEADER_PROC_BIN;
pb->size = size;
- pb->next = ohp->first;
- ohp->first = (struct erl_off_heap_header*)pb;
+ pb->next = factory.off_heap->first;
+ factory.off_heap->first = (struct erl_off_heap_header*)pb;
pb->val = ErlDrvBinary2Binary(b);
pb->bytes = ((byte*) b->orig_bytes) + offset;
pb->flags = 0;
mess = make_binary(pb);
- hp += PROC_BIN_SIZE;
- OH_OVERHEAD(ohp, pb->size / sizeof(Eterm));
+ OH_OVERHEAD(factory.off_heap, pb->size / sizeof(Eterm));
}
ptr += 3;
break;
@@ -5441,8 +5448,9 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) size);
if (size <= ERL_ONHEAP_BIN_LIMIT) {
- ErlHeapBin* hbp = (ErlHeapBin *) hp;
- hp += heap_bin_size(size);
+ ErlHeapBin* hbp = (ErlHeapBin *) erts_produce_heap(&factory,
+ heap_bin_size(size),
+ HEAP_EXTRA);
hbp->thing_word = header_heap_bin(size);
hbp->size = size;
if (size > 0) {
@@ -5457,16 +5465,16 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
ASSERT(bufp);
erts_refc_init(&bp->refc, 1);
sys_memcpy((void *) bp->orig_bytes, (void *) bufp, size);
- pbp = (ProcBin *) hp;
- hp += PROC_BIN_SIZE;
+ pbp = (ProcBin *) erts_produce_heap(&factory,
+ PROC_BIN_SIZE, HEAP_EXTRA);
pbp->thing_word = HEADER_PROC_BIN;
pbp->size = size;
- pbp->next = ohp->first;
- ohp->first = (struct erl_off_heap_header*)pbp;
+ pbp->next = factory.off_heap->first;
+ factory.off_heap->first = (struct erl_off_heap_header*)pbp;
pbp->val = bp;
pbp->bytes = (byte*) bp->orig_bytes;
pbp->flags = 0;
- OH_OVERHEAD(ohp, pbp->size / sizeof(Eterm));
+ OH_OVERHEAD(factory.off_heap, pbp->size / sizeof(Eterm));
mess = make_binary(pbp);
}
ptr += 2;
@@ -5475,13 +5483,15 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
case ERL_DRV_STRING: /* char*, length */
erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) ptr[1]);
- mess = buf_to_intlist(&hp, (char*)ptr[0], ptr[1], NIL);
+ erts_reserve_heap(&factory, 2*ptr[1]);
+ mess = buf_to_intlist(&factory.hp, (char*)ptr[0], ptr[1], NIL);
ptr += 2;
break;
case ERL_DRV_STRING_CONS: /* char*, length */
mess = ESTACK_POP(stack);
- mess = buf_to_intlist(&hp, (char*)ptr[0], ptr[1], mess);
+ erts_reserve_heap(&factory, 2*ptr[1]);
+ mess = buf_to_intlist(&factory.hp, (char*)ptr[0], ptr[1], mess);
ptr += 2;
break;
@@ -5490,11 +5500,12 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
mess = ESTACK_POP(stack);
i--;
+ erts_reserve_heap(&factory, 2*i);
while(i > 0) {
Eterm hd = ESTACK_POP(stack);
- mess = CONS(hp, hd, mess);
- hp += 2;
+ mess = CONS(factory.hp, hd, mess);
+ factory.hp += 2;
i--;
}
ptr++;
@@ -5503,13 +5514,12 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
case ERL_DRV_TUPLE: { /* int */
int size = (int)ptr[0];
- Eterm* tp = hp;
+ Eterm* tp = erts_produce_heap(&factory, size+1, HEAP_EXTRA);
*tp = make_arityval(size);
mess = make_tuple(tp);
tp += size; /* point at last element */
- hp = tp+1; /* advance "heap" pointer */
while(size--) {
*tp-- = ESTACK_POP(stack);
@@ -5525,20 +5535,22 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
case ERL_DRV_FLOAT: { /* double * */
FloatDef f;
+ Eterm* fp = erts_produce_heap(&factory, FLOAT_SIZE_OBJECT, HEAP_EXTRA);
- mess = make_float(hp);
+ mess = make_float(fp);
f.fd = *((double *) ptr[0]);
if (!erts_isfinite(f.fd))
ERTS_DDT_FAIL;
- PUT_DOUBLE(f, hp);
- hp += FLOAT_SIZE_OBJECT;
+ PUT_DOUBLE(f, fp);
ptr++;
break;
}
case ERL_DRV_EXT2TERM: /* char *ext, int size */
ASSERT(b2t.org_ext[b2t.ix] == (byte *) ptr[0]);
- mess = erts_binary2term_create(&b2t.state[b2t.ix++], &hp, ohp);
+
+ erts_reserve_heap(&factory, b2t.state[b2t.ix].heap_size);
+ mess = erts_binary2term_create(&b2t.state[b2t.ix++], &factory);
if (mess == THE_NON_VALUE)
ERTS_DDT_FAIL;
ptr += 2;
@@ -5548,41 +5560,32 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
int size = (int)ptr[0];
if (size > MAP_SMALL_MAP_LIMIT) {
int ix = 2*size;
- ErtsHeapFactory factory;
- Eterm* leafs = hp;
-
- hp += 2*size;
- while(ix--) { *--hp = ESTACK_POP(stack); }
+ Eterm* leafs;
- hp += 2*size;
- factory.p = NULL;
- factory.hp = hp;
- /* We assume heap will suffice (see hashmap_over_estimated_heap_size) */
+ erts_produce_heap(&factory, ix, HEAP_EXTRA);
+ leafs = factory.hp;
+ while(ix--) { *--leafs = ESTACK_POP(stack); }
mess = erts_hashmap_from_array(&factory, leafs, size, 1);
-
if (is_non_value(mess))
ERTS_DDT_FAIL;
-
- hp = factory.hp;
} else {
- Eterm* tp = hp;
Eterm* vp;
flatmap_t *mp;
+ Eterm* tp = erts_produce_heap(&factory,
+ 2*size + 1 + MAP_HEADER_FLATMAP_SZ,
+ HEAP_EXTRA);
*tp = make_arityval(size);
- hp += 1 + size;
- mp = (flatmap_t*)hp;
+ mp = (flatmap_t*) (tp + 1 + size);
mp->thing_word = MAP_HEADER_FLATMAP;
mp->size = size;
mp->keys = make_tuple(tp);
mess = make_flatmap(mp);
- hp += MAP_HEADER_FLATMAP_SZ + size;
-
tp += size; /* point at last key */
- vp = hp - 1; /* point at last value */
+ vp = factory.hp - 1; /* point at last value */
while(size--) {
*vp-- = ESTACK_POP(stack);
@@ -5605,25 +5608,16 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
if (res > 0) {
mess = ESTACK_POP(stack); /* get resulting value */
- if (bp)
- bp = erts_resize_message_buffer(bp, hp - hp_start, &mess, 1);
- else {
- ASSERT(hp);
- HRelease(rp, hp_end, hp);
- }
+ erts_factory_close(&factory);
/* send message */
- erts_queue_message(rp, &rp_locks, bp, mess, am_undefined);
+ erts_queue_message(rp, &rp_locks, factory.heap_frags, mess, am_undefined);
}
else {
if (b2t.ix > b2t.used)
b2t.used = b2t.ix;
for (b2t.ix = 0; b2t.ix < b2t.used; b2t.ix++)
erts_binary2term_abort(&b2t.state[b2t.ix]);
- if (bp)
- free_message_buffer(bp);
- else if (hp) {
- HRelease(rp, hp_end, hp);
- }
+ erts_factory_undo(&factory);
}
if (rp) {
if (rp_locks)
@@ -5635,6 +5629,7 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
DESTROY_ESTACK(stack);
return res;
#undef ERTS_DDT_FAIL
+#undef HEAP_EXTRA
}
static ERTS_INLINE int
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-21 17:27:10 +0000