aboutsummaryrefslogtreecommitdiffstats
path: root/erts/emulator/beam
diff options
context:
space:
mode:
Diffstat (limited to 'erts/emulator/beam')
-rw-r--r--erts/emulator/beam/atom.h1
-rw-r--r--erts/emulator/beam/atom.names5
-rw-r--r--erts/emulator/beam/beam_emu.c24
-rw-r--r--erts/emulator/beam/beam_load.c47
-rw-r--r--erts/emulator/beam/bif.c30
-rw-r--r--erts/emulator/beam/bif.tab2
-rw-r--r--erts/emulator/beam/erl_alloc.c2
-rw-r--r--erts/emulator/beam/erl_bif_info.c37
-rw-r--r--erts/emulator/beam/erl_gc.c218
-rw-r--r--erts/emulator/beam/erl_init.c21
-rw-r--r--erts/emulator/beam/erl_map.c96
-rw-r--r--erts/emulator/beam/erl_message.c160
-rw-r--r--erts/emulator/beam/erl_message.h2
-rw-r--r--erts/emulator/beam/erl_process.c27
-rw-r--r--erts/emulator/beam/erl_process.h34
-rw-r--r--erts/emulator/beam/erl_process_dict.c66
-rw-r--r--erts/emulator/beam/erl_process_dict.h1
-rw-r--r--erts/emulator/beam/ops.tab3
18 files changed, 621 insertions, 155 deletions
diff --git a/erts/emulator/beam/atom.h b/erts/emulator/beam/atom.h
index ead56c83d8..2c002ca92f 100644
--- a/erts/emulator/beam/atom.h
+++ b/erts/emulator/beam/atom.h
@@ -129,6 +129,7 @@ typedef enum {
(erts_is_atom_utf8_bytes((byte *) LSTR, sizeof(LSTR) - 1, (TERM)))
#define ERTS_DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1)
#define ERTS_INIT_AM(S) AM_ ## S = am_atom_put(#S, sizeof(#S) - 1)
+#define ERTS_MAKE_AM(Str) am_atom_put(Str, sizeof(Str) - 1)
int atom_table_size(void); /* number of elements */
int atom_table_sz(void); /* table size in bytes, excluding stored objects */
diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index ea04495574..7424e47ec3 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -350,6 +350,7 @@ atom memory_internal
atom memory_types
atom message
atom message_binary
+atom message_queue_data
atom message_queue_len
atom messages
atom merge_trap
@@ -361,6 +362,7 @@ atom min_heap_size
atom min_bin_vheap_size
atom minor_version
atom Minus='-'
+atom mixed
atom module
atom module_info
atom monitored_by
@@ -423,11 +425,12 @@ atom notify
atom notsup
atom nouse_stdio
atom objects
-atom off_heap_message_queue
+atom off_heap
atom offset
atom ok
atom old_heap_block_size
atom old_heap_size
+atom on_heap
atom on_load
atom open
atom open_error
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 1a4133bceb..4d7b00b032 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -1843,8 +1843,8 @@ void process_main(void)
* in the queue. This since messages with data outside
* the heap will be corrupted by a GC.
*/
- ASSERT(!(c_p->flags & F_DISABLE_GC));
- c_p->flags |= F_DISABLE_GC;
+ ASSERT(!(c_p->flags & F_DELAY_GC));
+ c_p->flags |= F_DELAY_GC;
loop_rec__:
PROCESS_MAIN_CHK_LOCKS(c_p);
@@ -1858,7 +1858,7 @@ void process_main(void)
if (ERTS_PROC_PENDING_EXIT(c_p)) {
erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
SWAPOUT;
- c_p->flags &= ~F_DISABLE_GC;
+ c_p->flags &= ~F_DELAY_GC;
goto do_schedule; /* Will be rescheduled for exit */
}
ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p);
@@ -1868,7 +1868,7 @@ void process_main(void)
else
#endif
{
- c_p->flags &= ~F_DISABLE_GC;
+ c_p->flags &= ~F_DELAY_GC;
SET_I((BeamInstr *) Arg(0));
Goto(*I); /* Jump to a wait or wait_timeout instruction */
}
@@ -1978,7 +1978,7 @@ void process_main(void)
CANCEL_TIMER(c_p);
erts_save_message_in_proc(c_p, msgp);
- c_p->flags &= ~F_DISABLE_GC;
+ c_p->flags &= ~F_DELAY_GC;
if (ERTS_IS_GC_DESIRED_INTERNAL(c_p, HTOP, E)) {
/*
@@ -2000,7 +2000,7 @@ void process_main(void)
*/
OpCase(loop_rec_end_f): {
- ASSERT(c_p->flags & F_DISABLE_GC);
+ ASSERT(c_p->flags & F_DELAY_GC);
SET_I((BeamInstr *) Arg(0));
SAVE_MESSAGE(c_p);
@@ -2009,7 +2009,7 @@ void process_main(void)
goto loop_rec__;
}
- c_p->flags &= ~F_DISABLE_GC;
+ c_p->flags &= ~F_DELAY_GC;
c_p->i = I;
SWAPOUT;
c_p->arity = 0;
@@ -3558,6 +3558,16 @@ do { \
StoreBifResult(1, result);
}
+ OpCase(i_get_hash_cId):
+ {
+ Eterm arg;
+ Eterm result;
+
+ GetArg1(0, arg);
+ result = erts_pd_hash_get_with_hx(c_p, Arg(1), arg);
+ StoreBifResult(2, result);
+ }
+
{
Eterm case_end_val;
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 5db971b6af..d367cce212 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -4284,6 +4284,53 @@ gen_get_map_element(LoaderState* stp, GenOpArg Fail, GenOpArg Src,
return op;
}
+static int
+hash_internal_genop_arg(LoaderState* stp, GenOpArg Key, Uint32* hx)
+{
+ switch (Key.type) {
+ case TAG_a:
+ *hx = atom_tab(atom_val(Key.val))->slot.bucket.hvalue;
+ return 1;
+ case TAG_i:
+ *hx = Key.val;
+ return 1;
+ case TAG_n:
+ *hx = make_internal_hash(NIL);
+ return 1;
+ case TAG_q:
+ *hx = make_internal_hash(stp->literals[Key.val].term);
+ return 1;
+ default:
+ return 0;
+ }
+}
+
+
+static GenOp*
+gen_get(LoaderState* stp, GenOpArg Src, GenOpArg Dst)
+{
+ GenOp* op;
+ Uint32 hx = 0;
+
+ NEW_GENOP(stp, op);
+ op->next = NULL;
+ if (hash_internal_genop_arg(stp, Src, &hx)) {
+ op->arity = 3;
+ op->op = genop_i_get_hash_3;
+ op->a[0] = Src;
+ op->a[1].type = TAG_u;
+ op->a[1].val = (BeamInstr) hx;
+ op->a[2] = Dst;
+ } else {
+ op->arity = 2;
+ op->op = genop_i_get_2;
+ op->a[0] = Src;
+ op->a[1] = Dst;
+ }
+ return op;
+}
+
+
static GenOp*
gen_get_map_elements(LoaderState* stp, GenOpArg Fail, GenOpArg Src,
GenOpArg Size, GenOpArg* Rest)
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 886b19fe6e..bb9165cd79 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -910,13 +910,22 @@ BIF_RETTYPE spawn_opt_1(BIF_ALIST_1)
so.priority = PRIORITY_LOW;
else
goto error;
- } else if (arg == am_off_heap_message_queue) {
- if (val == am_true)
- so.flags |= SPO_OFF_HEAP_MSGQ;
- else if (val == am_false)
+ } else if (arg == am_message_queue_data) {
+ switch (val) {
+ case am_mixed:
+ so.flags &= ~(SPO_OFF_HEAP_MSGQ|SPO_ON_HEAP_MSGQ);
+ break;
+ case am_on_heap:
so.flags &= ~SPO_OFF_HEAP_MSGQ;
- else
+ so.flags |= SPO_ON_HEAP_MSGQ;
+ break;
+ case am_off_heap:
+ so.flags &= ~SPO_ON_HEAP_MSGQ;
+ so.flags |= SPO_OFF_HEAP_MSGQ;
+ break;
+ default:
goto error;
+ }
} else if (arg == am_min_heap_size && is_small(val)) {
Sint min_heap_size = signed_val(val);
if (min_heap_size < 0) {
@@ -1695,15 +1704,10 @@ BIF_RETTYPE process_flag_2(BIF_ALIST_2)
}
BIF_RET(old_value);
}
- else if (BIF_ARG_1 == am_off_heap_message_queue) {
- int enable;
- if (BIF_ARG_2 == am_true)
- enable = 1;
- else if (BIF_ARG_2 == am_false)
- enable = 0;
- else
+ else if (BIF_ARG_1 == am_message_queue_data) {
+ old_value = erts_change_message_queue_management(BIF_P, BIF_ARG_2);
+ if (is_non_value(old_value))
goto error;
- old_value = erts_change_off_heap_message_queue_state(BIF_P, enable);
BIF_RET(old_value);
}
else if (BIF_ARG_1 == am_sensitive) {
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index c49a3ff313..07d4702b92 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -167,7 +167,7 @@ bif erts_internal:request_system_task/3
bif erts_internal:check_process_code/2
bif erts_internal:map_to_tuple_keys/1
-bif erts_internal:map_type/1
+bif erts_internal:term_type/1
bif erts_internal:map_hashmap_children/1
bif erts_internal:time_unit/0
diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 99458b4268..5544712e8d 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -292,7 +292,7 @@ static void
set_default_literal_alloc_opts(struct au_init *ip)
{
SET_DEFAULT_ALLOC_OPTS(ip);
- ip->enable = AU_ALLOC_DEFAULT_ENABLE(1);
+ ip->enable = 1;
ip->thr_spec = 0;
ip->atype = BESTFIT;
ip->init.bf.ao = 1;
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 82c2aa4b9e..f952f937ce 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -592,7 +592,7 @@ static Eterm pi_args[] = {
am_min_bin_vheap_size,
am_current_location,
am_current_stacktrace,
- am_off_heap_message_queue
+ am_message_queue_data
};
#define ERTS_PI_ARGS ((int) (sizeof(pi_args)/sizeof(Eterm)))
@@ -640,7 +640,7 @@ pi_arg2ix(Eterm arg)
case am_min_bin_vheap_size: return 28;
case am_current_location: return 29;
case am_current_stacktrace: return 30;
- case am_off_heap_message_queue: return 31;
+ case am_message_queue_data: return 31;
default: return -1;
}
}
@@ -1499,8 +1499,22 @@ process_info_aux(Process *BIF_P,
break;
}
- case am_off_heap_message_queue:
- res = BIF_P->flags & F_OFF_HEAP_MSGQ ? am_true : am_false;
+ case am_message_queue_data:
+ switch (rp->flags & (F_OFF_HEAP_MSGQ|F_ON_HEAP_MSGQ)) {
+ case F_OFF_HEAP_MSGQ:
+ res = am_off_heap;
+ break;
+ case F_ON_HEAP_MSGQ:
+ res = am_on_heap;
+ break;
+ case 0:
+ res = am_mixed;
+ break;
+ default:
+ res = am_error;
+ ERTS_INTERNAL_ERROR("Inconsistent message queue management state");
+ break;
+ }
hp = HAlloc(BIF_P, 3);
break;
@@ -2665,9 +2679,18 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
BIF_RET(am_true);
}
#endif
- else if (BIF_ARG_1 == am_off_heap_message_queue) {
- BIF_RET(erts_default_spo_flags & SPO_OFF_HEAP_MSGQ
- ? am_true : am_false);
+ else if (BIF_ARG_1 == am_message_queue_data) {
+ switch (erts_default_spo_flags & (SPO_ON_HEAP_MSGQ|SPO_OFF_HEAP_MSGQ)) {
+ case SPO_OFF_HEAP_MSGQ:
+ BIF_RET(am_off_heap);
+ case SPO_ON_HEAP_MSGQ:
+ BIF_RET(am_on_heap);
+ case 0:
+ BIF_RET(am_mixed);
+ default:
+ ERTS_INTERNAL_ERROR("Inconsistent message queue management state");
+ BIF_RET(am_error);
+ }
}
else if (ERTS_IS_ATOM_STR("compile_info",BIF_ARG_1)) {
Uint sz;
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index c50756d56b..15226074c3 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -145,6 +145,7 @@ static void offset_rootset(Process *p, Sint offs, char* area, Uint area_size,
Eterm* objv, int nobj);
static void offset_off_heap(Process* p, Sint offs, char* area, Uint area_size);
static void offset_mqueue(Process *p, Sint offs, char* area, Uint area_size);
+static void move_msgq_to_heap(Process *p);
static void init_gc_info(ErtsGCInfo *gcip);
@@ -440,8 +441,15 @@ delay_garbage_collection(Process *p, ErlHeapFragment *live_hf_end, int need)
ERTS_HOLE_CHECK(p);
- if (p->live_hf_end == ERTS_INVALID_HFRAG_PTR)
+ if ((p->flags & F_DISABLE_GC)
+ && p->live_hf_end == ERTS_INVALID_HFRAG_PTR) {
+ /*
+ * A BIF yielded with disabled GC. Remember
+ * heap fragments created by the BIF until we
+ * do next GC.
+ */
p->live_hf_end = live_hf_end;
+ }
if (need == 0)
return 1;
@@ -513,6 +521,14 @@ young_gen_usage(Process *p)
Eterm *aheap;
hsz = p->mbuf_sz;
+
+ if (p->flags & F_ON_HEAP_MSGQ) {
+ ErtsMessage *mp;
+ for (mp = p->msg.first; mp; mp = mp->next)
+ if (mp->data.attached)
+ hsz += erts_msg_attached_data_size(mp);
+ }
+
aheap = p->abandoned_heap;
if (!aheap)
hsz += p->htop - p->heap;
@@ -564,10 +580,12 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE);
#endif
- if (p->flags & F_DISABLE_GC)
+ if (p->flags & (F_DISABLE_GC|F_DELAY_GC))
return delay_garbage_collection(p, live_hf_end, need);
- if (p->live_hf_end != ERTS_INVALID_HFRAG_PTR)
+ if (p->abandoned_heap)
+ live_hf_end = ERTS_INVALID_HFRAG_PTR;
+ else if (p->live_hf_end != ERTS_INVALID_HFRAG_PTR)
live_hf_end = p->live_hf_end;
esdp = erts_get_scheduler_data();
@@ -734,6 +752,12 @@ erts_garbage_collect_hibernate(Process* p)
p->arg_reg,
p->arity);
+ ERTS_HEAP_FREE(ERTS_ALC_T_HEAP,
+ (p->abandoned_heap
+ ? p->abandoned_heap
+ : p->heap),
+ p->heap_sz * sizeof(Eterm));
+
p->heap = heap;
p->high_water = htop;
p->htop = htop;
@@ -1025,10 +1049,13 @@ minor_collection(Process* p, ErlHeapFragment *live_hf_end,
do_minor(p, live_hf_end, (char *) mature, mature_size*sizeof(Eterm),
new_sz, objv, nobj);
+ if (p->flags & F_ON_HEAP_MSGQ)
+ move_msgq_to_heap(p);
+
new_mature = p->old_htop - prev_old_htop;
size_after = new_mature;
- size_after += HEAP_TOP(p) - HEAP_START(p);
+ size_after += HEAP_TOP(p) - HEAP_START(p) + p->mbuf_sz;
*recl += (size_before - size_after);
ErtsGcQuickSanityCheck(p);
@@ -1441,7 +1468,7 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end,
(p->abandoned_heap
? p->abandoned_heap
: HEAP_START(p)),
- (HEAP_END(p) - HEAP_START(p)) * sizeof(Eterm));
+ p->heap_sz * sizeof(Eterm));
p->abandoned_heap = NULL;
p->flags &= ~F_ABANDONED_HEAP_USE;
HEAP_START(p) = n_heap;
@@ -1452,9 +1479,14 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end,
HIGH_WATER(p) = HEAP_TOP(p);
+ remove_message_buffers(p);
+
+ if (p->flags & F_ON_HEAP_MSGQ)
+ move_msgq_to_heap(p);
+
ErtsGcQuickSanityCheck(p);
- size_after = HEAP_TOP(p) - HEAP_START(p);
+ size_after = HEAP_TOP(p) - HEAP_START(p) + p->mbuf_sz;
*recl += size_before - size_after;
adjusted = adjust_after_fullsweep(p, need, objv, nobj);
@@ -1462,8 +1494,6 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end,
#ifdef HARDDEBUG
disallow_heap_frag_ref_in_heap(p);
#endif
- remove_message_buffers(p);
-
ErtsGcQuickSanityCheck(p);
return gc_cost(size_after, adjusted ? size_after : 0);
@@ -1991,6 +2021,173 @@ collect_live_heap_frags(Process* p, ErlHeapFragment *live_hf_end,
return n_htop;
}
+static ERTS_INLINE void
+copy_one_frag(Eterm** hpp, ErlOffHeap* off_heap,
+ ErlHeapFragment *bp, Eterm *refs, int nrefs)
+{
+ Uint sz;
+ int i;
+ Sint offs;
+ struct erl_off_heap_header* oh;
+ Eterm *fhp, *hp;
+
+ OH_OVERHEAD(off_heap, bp->off_heap.overhead);
+ sz = bp->used_size;
+
+ fhp = bp->mem;
+ hp = *hpp;
+ offs = hp - fhp;
+
+ oh = NULL;
+ while (sz--) {
+ Uint cpy_sz;
+ Eterm val = *fhp++;
+
+ switch (primary_tag(val)) {
+ case TAG_PRIMARY_IMMED1:
+ *hp++ = val;
+ break;
+ case TAG_PRIMARY_LIST:
+ case TAG_PRIMARY_BOXED:
+ *hp++ = offset_ptr(val, offs);
+ break;
+ case TAG_PRIMARY_HEADER:
+ *hp++ = val;
+ switch (val & _HEADER_SUBTAG_MASK) {
+ case ARITYVAL_SUBTAG:
+ break;
+ case REFC_BINARY_SUBTAG:
+ case FUN_SUBTAG:
+ case EXTERNAL_PID_SUBTAG:
+ case EXTERNAL_PORT_SUBTAG:
+ case EXTERNAL_REF_SUBTAG:
+ oh = (struct erl_off_heap_header*) (hp-1);
+ cpy_sz = thing_arityval(val);
+ goto cpy_words;
+ default:
+ cpy_sz = header_arity(val);
+
+ cpy_words:
+ ASSERT(sz >= cpy_sz);
+ sz -= cpy_sz;
+ while (cpy_sz >= 8) {
+ cpy_sz -= 8;
+ *hp++ = *fhp++;
+ *hp++ = *fhp++;
+ *hp++ = *fhp++;
+ *hp++ = *fhp++;
+ *hp++ = *fhp++;
+ *hp++ = *fhp++;
+ *hp++ = *fhp++;
+ *hp++ = *fhp++;
+ }
+ switch (cpy_sz) {
+ case 7: *hp++ = *fhp++;
+ case 6: *hp++ = *fhp++;
+ case 5: *hp++ = *fhp++;
+ case 4: *hp++ = *fhp++;
+ case 3: *hp++ = *fhp++;
+ case 2: *hp++ = *fhp++;
+ case 1: *hp++ = *fhp++;
+ default: break;
+ }
+ if (oh) {
+ /* Add to offheap list */
+ oh->next = off_heap->first;
+ off_heap->first = oh;
+ ASSERT(*hpp <= (Eterm*)oh);
+ ASSERT(hp > (Eterm*)oh);
+ oh = NULL;
+ }
+ break;
+ }
+ break;
+ }
+ }
+
+ ASSERT(bp->used_size == hp - *hpp);
+ *hpp = hp;
+
+ for (i = 0; i < nrefs; i++) {
+ if (is_not_immed(refs[i]))
+ refs[i] = offset_ptr(refs[i], offs);
+ }
+ bp->off_heap.first = NULL;
+}
+
+static void
+move_msgq_to_heap(Process *p)
+{
+ ErtsMessage **mpp = &p->msg.first;
+
+ while (*mpp) {
+ ErtsMessage *mp = *mpp;
+
+ if (mp->data.attached) {
+ ErlHeapFragment *bp;
+ ErtsHeapFactory factory;
+
+ erts_factory_proc_prealloc_init(&factory, p,
+ erts_msg_attached_data_size(mp));
+
+ if (is_non_value(ERL_MESSAGE_TERM(mp))) {
+ if (mp->data.dist_ext) {
+ ASSERT(mp->data.dist_ext->heap_size >= 0);
+ if (is_not_nil(ERL_MESSAGE_TOKEN(mp))) {
+ bp = erts_dist_ext_trailer(mp->data.dist_ext);
+ ERL_MESSAGE_TOKEN(mp) = copy_struct(ERL_MESSAGE_TOKEN(mp),
+ bp->used_size,
+ &factory.hp,
+ factory.off_heap);
+ erts_cleanup_offheap(&bp->off_heap);
+ }
+ ERL_MESSAGE_TERM(mp) = erts_decode_dist_ext(&factory,
+ mp->data.dist_ext);
+ erts_free_dist_ext_copy(mp->data.dist_ext);
+ mp->data.dist_ext = NULL;
+ }
+ }
+ else {
+
+ if (mp->data.attached == ERTS_MSG_COMBINED_HFRAG)
+ bp = &mp->hfrag;
+ else
+ bp = mp->data.heap_frag;
+
+ if (bp->next)
+ erts_move_multi_frags(&factory.hp, factory.off_heap, bp,
+ mp->m, ERL_MESSAGE_REF_ARRAY_SZ, 0);
+ else
+ copy_one_frag(&factory.hp, factory.off_heap, bp,
+ mp->m, ERL_MESSAGE_REF_ARRAY_SZ);
+
+ if (mp->data.attached != ERTS_MSG_COMBINED_HFRAG) {
+ mp->data.heap_frag = NULL;
+ free_message_buffer(bp);
+ }
+ else {
+ ErtsMessage *tmp = erts_alloc_message(0, NULL);
+ sys_memcpy((void *) tmp->m, (void *) mp->m,
+ sizeof(Eterm)*ERL_MESSAGE_REF_ARRAY_SZ);
+ tmp->next = mp->next;
+ if (p->msg.save == &mp->next)
+ p->msg.save = &tmp->next;
+ if (p->msg.last == &mp->next)
+ p->msg.last = &tmp->next;
+ *mpp = tmp;
+ mp->next = NULL;
+ erts_cleanup_messages(mp);
+ mp = tmp;
+ }
+ }
+
+ erts_factory_close(&factory);
+ }
+
+ mpp = &(*mpp)->next;
+ }
+}
+
static Uint
setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset)
{
@@ -2080,9 +2277,8 @@ setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset)
case F_OFF_HEAP_MSGQ_CHNG:
case 0: {
/*
- * Off heap message queue disabled, i.e. we may
- * have references from the message queue to the
- * heap...
+ * We do not have off heap message queue enabled, i.e. we
+ * need to add message queue to rootset...
*/
ErtsMessage *mp;
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 296cfdabc3..99d8a2a987 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -630,7 +630,8 @@ void erts_usage(void)
erts_fprintf(stderr, "-W<i|w|e> set error logger warnings mapping,\n");
erts_fprintf(stderr, " see error_logger documentation for details\n");
- erts_fprintf(stderr, "-xohmq bool set default off_heap_message_queue flag for processes\n");
+ erts_fprintf(stderr, "-xmqd val set default message queue data flag for processes,\n");
+ erts_fprintf(stderr, " valid values are: off_heap | on_heap | mixed\n");
erts_fprintf(stderr, "-zdbbl size set the distribution buffer busy limit in kilobytes\n");
erts_fprintf(stderr, " valid range is [1-%d]\n", INT_MAX/1024);
erts_fprintf(stderr, "-zdntgc time set delayed node table gc in seconds\n");
@@ -2020,15 +2021,21 @@ erl_start(int argc, char **argv)
case 'x': {
char *sub_param = argv[i]+2;
- if (has_prefix("ohmq", sub_param)) {
- arg = get_arg(sub_param+4, argv[i+1], &i);
- if (sys_strcmp(arg, "true") == 0)
- erts_default_spo_flags |= SPO_OFF_HEAP_MSGQ;
- else if (sys_strcmp(arg, "false") == 0)
+ if (has_prefix("mqd", sub_param)) {
+ arg = get_arg(sub_param+3, argv[i+1], &i);
+ if (sys_strcmp(arg, "mixed") == 0)
+ erts_default_spo_flags &= ~(SPO_ON_HEAP_MSGQ|SPO_OFF_HEAP_MSGQ);
+ else if (sys_strcmp(arg, "on_heap") == 0) {
erts_default_spo_flags &= ~SPO_OFF_HEAP_MSGQ;
+ erts_default_spo_flags |= SPO_ON_HEAP_MSGQ;
+ }
+ else if (sys_strcmp(arg, "off_heap") == 0) {
+ erts_default_spo_flags &= ~SPO_ON_HEAP_MSGQ;
+ erts_default_spo_flags |= SPO_OFF_HEAP_MSGQ;
+ }
else {
erts_fprintf(stderr,
- "Invalid off_heap_message_queue flag: %s\n", arg);
+ "Invalid message_queue_data flag: %s\n", arg);
erts_usage();
}
} else {
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 29b3024644..d0ffb11e79 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -2698,32 +2698,88 @@ BIF_RETTYPE erts_internal_map_to_tuple_keys_1(BIF_ALIST_1) {
}
/*
- * erts_internal:map_type/1
+ * erts_internal:term_type/1
*
* Used in erts_debug:size/1
*/
-BIF_RETTYPE erts_internal_map_type_1(BIF_ALIST_1) {
- DECL_AM(hashmap);
- DECL_AM(hashmap_node);
- DECL_AM(flatmap);
- if (is_map(BIF_ARG_1)) {
- Eterm hdr = *(boxed_val(BIF_ARG_1));
- ASSERT(is_header(hdr));
- switch (hdr & _HEADER_MAP_SUBTAG_MASK) {
- case HAMT_SUBTAG_HEAD_FLATMAP:
- BIF_RET(AM_flatmap);
- case HAMT_SUBTAG_HEAD_ARRAY:
- case HAMT_SUBTAG_HEAD_BITMAP:
- BIF_RET(AM_hashmap);
- case HAMT_SUBTAG_NODE_BITMAP:
- BIF_RET(AM_hashmap_node);
- default:
- erl_exit(1, "bad header");
+BIF_RETTYPE erts_internal_term_type_1(BIF_ALIST_1) {
+ Eterm obj = BIF_ARG_1;
+ switch (primary_tag(obj)) {
+ case TAG_PRIMARY_LIST:
+ BIF_RET(ERTS_MAKE_AM("list"));
+ case TAG_PRIMARY_BOXED: {
+ Eterm hdr = *boxed_val(obj);
+ ASSERT(is_header(hdr));
+ switch (hdr & _TAG_HEADER_MASK) {
+ case ARITYVAL_SUBTAG:
+ BIF_RET(ERTS_MAKE_AM("tuple"));
+ case EXPORT_SUBTAG:
+ BIF_RET(ERTS_MAKE_AM("export"));
+ case FUN_SUBTAG:
+ BIF_RET(ERTS_MAKE_AM("fun"));
+ case MAP_SUBTAG:
+ switch (MAP_HEADER_TYPE(hdr)) {
+ case MAP_HEADER_TAG_FLATMAP_HEAD :
+ BIF_RET(ERTS_MAKE_AM("flatmap"));
+ case MAP_HEADER_TAG_HAMT_HEAD_BITMAP :
+ case MAP_HEADER_TAG_HAMT_HEAD_ARRAY :
+ BIF_RET(ERTS_MAKE_AM("hashmap"));
+ case MAP_HEADER_TAG_HAMT_NODE_BITMAP :
+ BIF_RET(ERTS_MAKE_AM("hashmap_node"));
+ default:
+ erl_exit(ERTS_ABORT_EXIT, "term_type: bad map header type %d\n", MAP_HEADER_TYPE(hdr));
+ }
+ case REFC_BINARY_SUBTAG:
+ BIF_RET(ERTS_MAKE_AM("refc_binary"));
+ case HEAP_BINARY_SUBTAG:
+ BIF_RET(ERTS_MAKE_AM("heap_binary"));
+ case SUB_BINARY_SUBTAG:
+ BIF_RET(ERTS_MAKE_AM("sub_binary"));
+ case BIN_MATCHSTATE_SUBTAG:
+ BIF_RET(ERTS_MAKE_AM("matchstate"));
+ case POS_BIG_SUBTAG:
+ case NEG_BIG_SUBTAG:
+ BIF_RET(ERTS_MAKE_AM("bignum"));
+ case REF_SUBTAG:
+ BIF_RET(ERTS_MAKE_AM("reference"));
+ case EXTERNAL_REF_SUBTAG:
+ BIF_RET(ERTS_MAKE_AM("external_reference"));
+ case EXTERNAL_PID_SUBTAG:
+ BIF_RET(ERTS_MAKE_AM("external_pid"));
+ case EXTERNAL_PORT_SUBTAG:
+ BIF_RET(ERTS_MAKE_AM("external_port"));
+ case FLOAT_SUBTAG:
+ BIF_RET(ERTS_MAKE_AM("hfloat"));
+ default:
+ erl_exit(ERTS_ABORT_EXIT, "term_type: Invalid tag (0x%X)\n", hdr);
+ }
}
+ case TAG_PRIMARY_IMMED1:
+ switch (obj & _TAG_IMMED1_MASK) {
+ case _TAG_IMMED1_SMALL:
+ BIF_RET(ERTS_MAKE_AM("fixnum"));
+ case _TAG_IMMED1_PID:
+ BIF_RET(ERTS_MAKE_AM("pid"));
+ case _TAG_IMMED1_PORT:
+ BIF_RET(ERTS_MAKE_AM("port"));
+ case _TAG_IMMED1_IMMED2:
+ switch (obj & _TAG_IMMED2_MASK) {
+ case _TAG_IMMED2_ATOM:
+ BIF_RET(ERTS_MAKE_AM("atom"));
+ case _TAG_IMMED2_CATCH:
+ BIF_RET(ERTS_MAKE_AM("catch"));
+ case _TAG_IMMED2_NIL:
+ BIF_RET(ERTS_MAKE_AM("nil"));
+ default:
+ erl_exit(ERTS_ABORT_EXIT, "term_type: Invalid tag (0x%X)\n", obj);
+ }
+ default:
+ erl_exit(ERTS_ABORT_EXIT, "term_type: Invalid tag (0x%X)\n", obj);
+ }
+ default:
+ erl_exit(ERTS_ABORT_EXIT, "term_type: Invalid tag (0x%X)\n", obj);
}
- BIF_P->fvalue = BIF_ARG_1;
- BIF_ERROR(BIF_P, BADMAP);
}
/*
diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index 3f515f5181..ac5ec79fe4 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -630,9 +630,24 @@ erts_try_alloc_message_on_heap(Process *pp,
#endif
else {
in_message_fragment:
-
- mp = erts_alloc_message(sz, hpp);
- *ohpp = sz == 0 ? NULL : &mp->hfrag.off_heap;
+ if (!((*psp) & ERTS_PSFLG_ON_HEAP_MSGQ)) {
+ mp = erts_alloc_message(sz, hpp);
+ *ohpp = sz == 0 ? NULL : &mp->hfrag.off_heap;
+ }
+ else {
+ mp = erts_alloc_message(0, NULL);
+ if (!sz) {
+ *hpp = NULL;
+ *ohpp = NULL;
+ }
+ else {
+ ErlHeapFragment *bp;
+ bp = new_message_buffer(sz);
+ *hpp = &bp->mem[0];
+ mp->data.heap_frag = bp;
+ *ohpp = &bp->off_heap;
+ }
+ }
*on_heap_p = 0;
}
@@ -1022,12 +1037,12 @@ erts_complete_off_heap_message_queue_change(Process *c_p)
ASSERT(erts_smp_atomic32_read_nob(&c_p->state) & ERTS_PSFLG_OFF_HEAP_MSGQ);
/*
- * This job was first initiated when the process changed
- * "off heap message queue" state from false to true. Since
- * then ERTS_PSFLG_OFF_HEAP_MSGQ has been set. However, the
- * state change might have been changed again (multiple times)
- * since then. Check users last requested state (the flag
- * F_OFF_HEAP_MSGQ), and make the state consistent with that.
+ * This job was first initiated when the process changed to off heap
+ * message queue management. Since then ERTS_PSFLG_OFF_HEAP_MSGQ
+ * has been set. However, the management state might have been changed
+ * again (multiple times) since then. Check users last requested state
+ * (the flags F_OFF_HEAP_MSGQ, and F_ON_HEAP_MSGQ), and make the state
+ * consistent with that.
*/
if (!(c_p->flags & F_OFF_HEAP_MSGQ))
@@ -1068,8 +1083,9 @@ change_off_heap_msgq(void *vcohmq)
}
Eterm
-erts_change_off_heap_message_queue_state(Process *c_p, int enable)
+erts_change_message_queue_management(Process *c_p, Eterm new_state)
{
+ Eterm res;
#ifdef DEBUG
if (c_p->flags & F_OFF_HEAP_MSGQ) {
@@ -1088,57 +1104,117 @@ erts_change_off_heap_message_queue_state(Process *c_p, int enable)
}
#endif
- if (c_p->flags & F_OFF_HEAP_MSGQ) {
- /* Off heap message queue is enabled */
+ switch (c_p->flags & (F_OFF_HEAP_MSGQ|F_ON_HEAP_MSGQ)) {
- if (!enable) {
+ case F_OFF_HEAP_MSGQ:
+ res = am_off_heap;
+
+ switch (new_state) {
+ case am_off_heap:
+ break;
+ case am_on_heap:
+ c_p->flags |= F_ON_HEAP_MSGQ;
+ erts_smp_atomic32_read_bor_nob(&c_p->state,
+ ERTS_PSFLG_ON_HEAP_MSGQ);
+ /* fall through */
+ case am_mixed:
c_p->flags &= ~F_OFF_HEAP_MSGQ;
/*
* We are not allowed to clear ERTS_PSFLG_OFF_HEAP_MSGQ
- * if a change is ongoing. It will be adjusted when the
- * change completes...
+ * if a off heap change is ongoing. It will be adjusted
+ * when the change completes...
*/
if (!(c_p->flags & F_OFF_HEAP_MSGQ_CHNG)) {
/* Safe to clear ERTS_PSFLG_OFF_HEAP_MSGQ... */
erts_smp_atomic32_read_band_nob(&c_p->state,
~ERTS_PSFLG_OFF_HEAP_MSGQ);
}
+ break;
+ default:
+ res = THE_NON_VALUE; /* badarg */
+ break;
}
+ break;
+
+ case F_ON_HEAP_MSGQ:
+ res = am_on_heap;
- return am_true; /* Old state */
+ switch (new_state) {
+ case am_on_heap:
+ break;
+ case am_mixed:
+ c_p->flags &= ~F_ON_HEAP_MSGQ;
+ erts_smp_atomic32_read_band_nob(&c_p->state,
+ ~ERTS_PSFLG_ON_HEAP_MSGQ);
+ break;
+ case am_off_heap:
+ c_p->flags &= ~F_ON_HEAP_MSGQ;
+ erts_smp_atomic32_read_band_nob(&c_p->state,
+ ~ERTS_PSFLG_ON_HEAP_MSGQ);
+ goto change_to_off_heap;
+ default:
+ res = THE_NON_VALUE; /* badarg */
+ break;
+ }
+ break;
+
+ case 0:
+ res = am_mixed;
+
+ switch (new_state) {
+ case am_mixed:
+ break;
+ case am_on_heap:
+ c_p->flags |= F_ON_HEAP_MSGQ;
+ erts_smp_atomic32_read_bor_nob(&c_p->state,
+ ERTS_PSFLG_ON_HEAP_MSGQ);
+ break;
+ case am_off_heap:
+ goto change_to_off_heap;
+ default:
+ res = THE_NON_VALUE; /* badarg */
+ break;
+ }
+ break;
+
+ default:
+ res = am_error;
+ ERTS_INTERNAL_ERROR("Inconsistent message queue management state");
+ break;
}
- /* Off heap message queue is disabled */
+ return res;
+
+change_to_off_heap:
- if (enable) {
- c_p->flags |= F_OFF_HEAP_MSGQ;
+ c_p->flags |= F_OFF_HEAP_MSGQ;
+
+ /*
+ * We do not have to schedule a change if
+ * we have an ongoing off heap change...
+ */
+ if (!(c_p->flags & F_OFF_HEAP_MSGQ_CHNG)) {
+ ErtsChangeOffHeapMessageQueue *cohmq;
/*
- * We do not have to schedule a change if
- * we have an ongoing change...
+ * Need to set ERTS_PSFLG_OFF_HEAP_MSGQ and wait
+ * thread progress before completing the change in
+ * order to ensure that all senders observe that
+ * messages should be passed off heap. When the
+ * change has completed, GC does not need to inspect
+ * the message queue at all.
*/
- if (!(c_p->flags & F_OFF_HEAP_MSGQ_CHNG)) {
- ErtsChangeOffHeapMessageQueue *cohmq;
- /*
- * Need to set ERTS_PSFLG_OFF_HEAP_MSGQ and wait
- * thread progress before completing the change in
- * order to ensure that all senders observe that
- * messages should be passed off heap. When the
- * change has completed, GC does not need to inspect
- * the message queue at all.
- */
- erts_smp_atomic32_read_bor_nob(&c_p->state,
- ERTS_PSFLG_OFF_HEAP_MSGQ);
- c_p->flags |= F_OFF_HEAP_MSGQ_CHNG;
- cohmq = erts_alloc(ERTS_ALC_T_MSGQ_CHNG,
- sizeof(ErtsChangeOffHeapMessageQueue));
- cohmq->pid = c_p->common.id;
- erts_schedule_thr_prgr_later_op(change_off_heap_msgq,
- (void *) cohmq,
- &cohmq->lop);
- }
+ erts_smp_atomic32_read_bor_nob(&c_p->state,
+ ERTS_PSFLG_OFF_HEAP_MSGQ);
+ c_p->flags |= F_OFF_HEAP_MSGQ_CHNG;
+ cohmq = erts_alloc(ERTS_ALC_T_MSGQ_CHNG,
+ sizeof(ErtsChangeOffHeapMessageQueue));
+ cohmq->pid = c_p->common.id;
+ erts_schedule_thr_prgr_later_op(change_off_heap_msgq,
+ (void *) cohmq,
+ &cohmq->lop);
}
- return am_false; /* Old state */
+ return res;
}
int
diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h
index 19528e6b4b..60035d15ae 100644
--- a/erts/emulator/beam/erl_message.h
+++ b/erts/emulator/beam/erl_message.h
@@ -286,7 +286,7 @@ void erts_cleanup_offheap(ErlOffHeap *offheap);
void erts_save_message_in_proc(Process *p, ErtsMessage *msg);
Sint erts_move_messages_off_heap(Process *c_p);
Sint erts_complete_off_heap_message_queue_change(Process *c_p);
-Eterm erts_change_off_heap_message_queue_state(Process *c_p, int enable);
+Eterm erts_change_message_queue_management(Process *c_p, Eterm new_state);
int erts_decode_dist_message(Process *, ErtsProcLocks, ErtsMessage *, int);
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index a714068314..6c132a7e3d 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -335,7 +335,6 @@ static ErtsAlignedSchedulerSleepInfo *aligned_dirty_io_sched_sleep_info;
static Uint last_reductions;
static Uint last_exact_reductions;
-Uint erts_default_process_flags;
Eterm erts_system_monitor;
Eterm erts_system_monitor_long_gc;
Uint erts_system_monitor_long_schedule;
@@ -677,7 +676,6 @@ erts_init_process(int ncpu, int proc_tab_size, int legacy_proc_tab)
last_reductions = 0;
last_exact_reductions = 0;
- erts_default_process_flags = 0;
}
void
@@ -9259,6 +9257,8 @@ Process *schedule(Process *p, int calls)
} else {
sched_out_proc:
+ ASSERT(!(p->flags & F_DELAY_GC));
+
#ifdef ERTS_SMP
ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p);
esdp = p->scheduler_data;
@@ -10732,7 +10732,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
Eterm args, /* Arguments for function (must be well-formed list). */
ErlSpawnOpts* so) /* Options for spawn. */
{
- Uint flags = erts_default_process_flags;
+ Uint flags = 0;
ErtsRunQueue *rq = NULL;
Process *p;
Sint arity; /* Number of arguments. */
@@ -10778,6 +10778,10 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
state |= ERTS_PSFLG_OFF_HEAP_MSGQ;
flags |= F_OFF_HEAP_MSGQ;
}
+ else if (so->flags & SPO_ON_HEAP_MSGQ) {
+ state |= ERTS_PSFLG_ON_HEAP_MSGQ;
+ flags |= F_ON_HEAP_MSGQ;
+ }
if (!rq)
rq = erts_get_runq_proc(parent);
@@ -11267,6 +11271,7 @@ erts_cleanup_empty_process(Process* p)
static void
delete_process(Process* p)
{
+ Eterm *heap;
VERBOSE(DEBUG_PROCESSES, ("Removing process: %T\n",p->common.id));
VERBOSE(DEBUG_SHCOPY, ("[pid=%T] delete process: %p %p %p %p\n", p->common.id,
HEAP_START(p), HEAP_END(p), OLD_HEAP(p), OLD_HEND(p)));
@@ -11293,16 +11298,17 @@ delete_process(Process* p)
* Release heaps. Clobber contents in DEBUG build.
*/
-
-#ifdef DEBUG
- sys_memset(p->heap, DEBUG_BAD_BYTE, p->heap_sz*sizeof(Eterm));
-#endif
-
#ifdef HIPE
hipe_delete_process(&p->hipe);
#endif
- ERTS_HEAP_FREE(ERTS_ALC_T_HEAP, (void*) p->heap, p->heap_sz*sizeof(Eterm));
+ heap = p->abandoned_heap ? p->abandoned_heap : p->heap;
+
+#ifdef DEBUG
+ sys_memset(heap, DEBUG_BAD_BYTE, p->heap_sz*sizeof(Eterm));
+#endif
+
+ ERTS_HEAP_FREE(ERTS_ALC_T_HEAP, (void*) heap, p->heap_sz*sizeof(Eterm));
if (p->old_heap != NULL) {
#ifdef DEBUG
@@ -11321,6 +11327,9 @@ delete_process(Process* p)
free_message_buffer(p->mbuf);
}
+ if (p->msg_frag)
+ erts_cleanup_messages(p->msg_frag);
+
erts_erase_dicts(p);
/* free all pending messages */
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index a72c5b8ad4..71065d875a 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -1141,14 +1141,15 @@ void erts_check_for_holes(Process* p);
#define ERTS_PSFLG_PROXY ERTS_PSFLG_BIT(16)
#define ERTS_PSFLG_DELAYED_SYS ERTS_PSFLG_BIT(17)
#define ERTS_PSFLG_OFF_HEAP_MSGQ ERTS_PSFLG_BIT(18)
+#define ERTS_PSFLG_ON_HEAP_MSGQ ERTS_PSFLG_BIT(19)
#ifdef ERTS_DIRTY_SCHEDULERS
-#define ERTS_PSFLG_DIRTY_CPU_PROC ERTS_PSFLG_BIT(19)
-#define ERTS_PSFLG_DIRTY_IO_PROC ERTS_PSFLG_BIT(20)
-#define ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q ERTS_PSFLG_BIT(21)
-#define ERTS_PSFLG_DIRTY_IO_PROC_IN_Q ERTS_PSFLG_BIT(22)
-#define ERTS_PSFLG_MAX (ERTS_PSFLGS_ZERO_BIT_OFFSET + 23)
+#define ERTS_PSFLG_DIRTY_CPU_PROC ERTS_PSFLG_BIT(20)
+#define ERTS_PSFLG_DIRTY_IO_PROC ERTS_PSFLG_BIT(21)
+#define ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q ERTS_PSFLG_BIT(22)
+#define ERTS_PSFLG_DIRTY_IO_PROC_IN_Q ERTS_PSFLG_BIT(23)
+#define ERTS_PSFLG_MAX (ERTS_PSFLGS_ZERO_BIT_OFFSET + 24)
#else
-#define ERTS_PSFLG_MAX (ERTS_PSFLGS_ZERO_BIT_OFFSET + 19)
+#define ERTS_PSFLG_MAX (ERTS_PSFLGS_ZERO_BIT_OFFSET + 20)
#endif
#define ERTS_PSFLGS_IN_PRQ_MASK (ERTS_PSFLG_IN_PRQ_MAX \
@@ -1197,6 +1198,7 @@ void erts_check_for_holes(Process* p);
#define SPO_MONITOR 4
#define SPO_SYSTEM_PROC 8
#define SPO_OFF_HEAP_MSGQ 16
+#define SPO_ON_HEAP_MSGQ 32
extern int erts_default_spo_flags;
@@ -1244,7 +1246,6 @@ Eterm* erts_heap_alloc(Process* p, Uint need, Uint xtra);
Eterm* erts_set_hole_marker(Eterm* ptr, Uint sz);
#endif
-extern Uint erts_default_process_flags;
extern erts_smp_rwmtx_t erts_cpu_bind_rwmtx;
/* If any of the erts_system_monitor_* variables are set (enabled),
** erts_system_monitor must be != NIL, to allow testing on just
@@ -1285,10 +1286,23 @@ extern struct erts_system_profile_flags_t erts_system_profile_flags;
#define F_HAVE_BLCKD_MSCHED (1 << 8) /* Process has blocked multi-scheduling */
#define F_P2PNR_RESCHED (1 << 9) /* Process has been rescheduled via erts_pid2proc_not_running() */
#define F_FORCE_GC (1 << 10) /* Force gc at process in-scheduling */
-#define F_DISABLE_GC (1 << 11) /* Disable GC */
+#define F_DISABLE_GC (1 << 11) /* Disable GC (see below) */
#define F_OFF_HEAP_MSGQ (1 << 12) /* Off heap msg queue */
-#define F_OFF_HEAP_MSGQ_CHNG (1 << 13) /* Off heap msg queue changing */
-#define F_ABANDONED_HEAP_USE (1 << 14) /* Have usage of abandoned heap */
+#define F_ON_HEAP_MSGQ (1 << 13) /* Off heap msg queue */
+#define F_OFF_HEAP_MSGQ_CHNG (1 << 14) /* Off heap msg queue changing */
+#define F_ABANDONED_HEAP_USE (1 << 15) /* Have usage of abandoned heap */
+#define F_DELAY_GC (1 << 16) /* Similar to disable GC (see below) */
+
+/*
+ * F_DISABLE_GC and F_DELAY_GC are similar. Both will prevent
+ * GC of the process, but it is important to use the right
+ * one:
+ * - F_DISABLE_GC should *only* be used by BIFs. This when
+ * the BIF needs to yield while preventig a GC.
+ * - F_DELAY_GC should only be used when GC is temporarily
+ * disabled while the process is scheduled. A process must
+ * not be scheduled out while F_DELAY_GC is set.
+ */
/* process trace_flags */
#define F_SENSITIVE (1 << 0)
diff --git a/erts/emulator/beam/erl_process_dict.c b/erts/emulator/beam/erl_process_dict.c
index f82cad745a..da9ebd92ab 100644
--- a/erts/emulator/beam/erl_process_dict.c
+++ b/erts/emulator/beam/erl_process_dict.c
@@ -53,14 +53,17 @@
/* Hash utility macros */
#define HASH_RANGE(PDict) ((PDict)->homeSize + (PDict)->splitPosition)
-#define MAKE_HASH(Term) \
-((is_small(Term)) ? unsigned_val(Term) : \
- ((is_atom(Term)) ? \
- (atom_tab(atom_val(term))->slot.bucket.hvalue) : \
- make_hash2(Term)))
+#define MAKE_HASH(Term) \
+ ((is_small(Term)) ? unsigned_val(Term) : \
+ ((is_atom(Term)) ? \
+ (atom_tab(atom_val(Term))->slot.bucket.hvalue) : \
+ make_internal_hash(Term)))
#define PD_SZ2BYTES(Sz) (sizeof(ProcDict) + ((Sz) - 1)*sizeof(Eterm))
+#define pd_hash_value(Pdict, Key) \
+ pd_hash_value_to_ix(Pdict, MAKE_HASH((Key)))
+
/* Memory allocation macros */
#define PD_ALLOC(Sz) \
erts_alloc(ERTS_ALC_T_PROC_DICT, (Sz))
@@ -82,6 +85,7 @@
*/
static void pd_hash_erase(Process *p, Eterm id, Eterm *ret);
static void pd_hash_erase_all(Process *p);
+static Eterm pd_hash_get_with_hval(Process *p, Eterm bucket, Eterm id);
static Eterm pd_hash_get_keys(Process *p, Eterm value);
static Eterm pd_hash_get_all_keys(Process *p, ProcDict *pd);
static Eterm pd_hash_get_all(Process *p, ProcDict *pd);
@@ -93,7 +97,7 @@ static void grow(Process *p);
static void array_shrink(ProcDict **ppd, unsigned int need);
static Eterm array_put(ProcDict **ppdict, unsigned int ndx, Eterm term);
-static unsigned int pd_hash_value(ProcDict *pdict, Eterm term);
+static unsigned int pd_hash_value_to_ix(ProcDict *pdict, Uint32 hx);
static unsigned int next_array_size(unsigned int need);
/*
@@ -390,40 +394,55 @@ static void pd_hash_erase_all(Process *p)
}
}
+Eterm erts_pd_hash_get_with_hx(Process *p, Uint32 hx, Eterm id)
+{
+ unsigned int hval;
+ ProcDict *pd = p->dictionary;
+
+ if (pd == NULL)
+ return am_undefined;
+ hval = pd_hash_value_to_ix(pd, hx);
+ return pd_hash_get_with_hval(p, ARRAY_GET(pd, hval), id);
+}
+
Eterm erts_pd_hash_get(Process *p, Eterm id)
{
unsigned int hval;
- Eterm tmp;
ProcDict *pd = p->dictionary;
if (pd == NULL)
return am_undefined;
hval = pd_hash_value(pd, id);
- tmp = ARRAY_GET(pd, hval);
- if (is_boxed(tmp)) { /* Tuple */
- ASSERT(is_tuple(tmp));
- if (EQ(tuple_val(tmp)[1], id)) {
- return tuple_val(tmp)[2];
+ return pd_hash_get_with_hval(p, ARRAY_GET(pd, hval), id);
+}
+
+Eterm pd_hash_get_with_hval(Process *p, Eterm bucket, Eterm id)
+{
+ if (is_boxed(bucket)) { /* Tuple */
+ ASSERT(is_tuple(bucket));
+ if (EQ(tuple_val(bucket)[1], id)) {
+ return tuple_val(bucket)[2];
}
- } else if (is_list(tmp)) {
- for (; tmp != NIL && !EQ(tuple_val(TCAR(tmp))[1], id); tmp = TCDR(tmp)) {
+ } else if (is_list(bucket)) {
+ for (; bucket != NIL && !EQ(tuple_val(TCAR(bucket))[1], id); bucket = TCDR(bucket)) {
;
}
- if (tmp != NIL) {
- return tuple_val(TCAR(tmp))[2];
+ if (bucket != NIL) {
+ return tuple_val(TCAR(bucket))[2];
}
- } else if (is_not_nil(tmp)) {
+ } else if (is_not_nil(bucket)) {
#ifdef DEBUG
erts_fprintf(stderr,
"Process dictionary for process %T is broken, trying to "
"display term found in line %d:\n"
- "%T\n", p->common.id, __LINE__, tmp);
+ "%T\n", p->common.id, __LINE__, bucket);
#endif
erl_exit(1, "Damaged process dictionary found during get/1.");
}
return am_undefined;
}
+
#define PD_GET_TKEY(Dst,Src) \
do { \
ASSERT(is_tuple((Src))); \
@@ -932,17 +951,16 @@ static Eterm array_put(ProcDict **ppdict, unsigned int ndx, Eterm term)
** Basic utilities
*/
-static unsigned int pd_hash_value(ProcDict *pdict, Eterm term)
+static unsigned int pd_hash_value_to_ix(ProcDict *pdict, Uint32 hx)
{
- Uint hash, high;
-
- hash = MAKE_HASH(term);
- high = hash % (pdict->homeSize*2);
+ Uint high;
+ high = hx % (pdict->homeSize*2);
if (high >= HASH_RANGE(pdict))
- return hash % pdict->homeSize;
+ return hx % pdict->homeSize;
return high;
}
+
static unsigned int next_array_size(unsigned int need)
{
static unsigned int tab[] =
diff --git a/erts/emulator/beam/erl_process_dict.h b/erts/emulator/beam/erl_process_dict.h
index cc53800eb5..9aa21b7c38 100644
--- a/erts/emulator/beam/erl_process_dict.h
+++ b/erts/emulator/beam/erl_process_dict.h
@@ -39,5 +39,6 @@ void erts_deep_dictionary_dump(int to, void *to_arg,
Eterm erts_dictionary_copy(struct process *p, ProcDict *pd);
Eterm erts_pd_hash_get(struct process *p, Eterm id);
+Eterm erts_pd_hash_get_with_hx(Process *p, Uint32 hx, Eterm id);
#endif
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 46fefb88af..081c4108a0 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -1036,7 +1036,7 @@ call_bif e
bif0 u$bif:erlang:self/0 Dst=d => self Dst
bif0 u$bif:erlang:node/0 Dst=d => node Dst
-bif1 Fail Bif=u$bif:erlang:get/1 Src=s Dst=d => i_get Src Dst
+bif1 Fail Bif=u$bif:erlang:get/1 Src=s Dst=d => gen_get(Src, Dst)
bif2 Jump=j u$bif:erlang:element/2 S1=s S2=xy Dst=d => gen_element(Jump, S1, S2, Dst)
@@ -1045,6 +1045,7 @@ bif1 p Bif S1 Dst => bif1_body Bif S1 Dst
bif2 p Bif S1 S2 Dst => i_bif2_body Bif S1 S2 Dst
bif2 Fail Bif S1 S2 Dst => i_bif2 Fail Bif S1 S2 Dst
+i_get_hash c I d
i_get s d
%macro: self Self
generated by cgit v1.2.3 (git 2.25.1) at 2024-10-30 00:36:59 +0000