diff options
Diffstat (limited to 'erts/emulator/beam')
93 files changed, 11087 insertions, 7690 deletions
diff --git a/erts/emulator/beam/atom.c b/erts/emulator/beam/atom.c index fe91134ef4..099c00bcf6 100644 --- a/erts/emulator/beam/atom.c +++ b/erts/emulator/beam/atom.c @@ -176,7 +176,7 @@ atom_alloc(Atom* tmpl) /* * Precompute ordinal value of first 3 bytes + 7 bits. - * This is used by utils.c:cmp_atoms(). + * This is used by utils.c:erts_cmp_atoms(). * We cannot use the full 32 bits of the first 4 bytes, * since we use the sign of the difference between two * ordinal values to represent their relative order. @@ -435,6 +435,9 @@ init_atom_table(void) f.cmp = (HCMP_FUN) atom_cmp; f.alloc = (HALLOC_FUN) atom_alloc; f.free = (HFREE_FUN) atom_free; + f.meta_alloc = (HMALLOC_FUN) erts_alloc; + f.meta_free = (HMFREE_FUN) erts_free; + f.meta_print = (HMPRINT_FUN) erts_print; atom_text_pos = NULL; atom_text_end = NULL; diff --git a/erts/emulator/beam/atom.h b/erts/emulator/beam/atom.h index ead56c83d8..2c002ca92f 100644 --- a/erts/emulator/beam/atom.h +++ b/erts/emulator/beam/atom.h @@ -129,6 +129,7 @@ typedef enum { (erts_is_atom_utf8_bytes((byte *) LSTR, sizeof(LSTR) - 1, (TERM))) #define ERTS_DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1) #define ERTS_INIT_AM(S) AM_ ## S = am_atom_put(#S, sizeof(#S) - 1) +#define ERTS_MAKE_AM(Str) am_atom_put(Str, sizeof(Str) - 1) int atom_table_size(void); /* number of elements */ int atom_table_sz(void); /* table size in bytes, excluding stored objects */ diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index cb6d294a41..2777c3d112 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -102,6 +102,7 @@ atom asynchronous atom atom atom atom_used atom attributes +atom await_microstate_accounting_modifications atom await_port_send_result atom await_proc_exit atom await_result @@ -118,10 +119,9 @@ atom bif_timer_server atom binary atom binary_bin_to_list_trap atom binary_copy_trap +atom binary_find_trap atom binary_longest_prefix_trap atom binary_longest_suffix_trap -atom binary_match_trap -atom binary_matches_trap atom binary_to_list_continue atom binary_to_term_trap atom block @@ -173,6 +173,7 @@ atom const atom context_switches atom control atom copy +atom counters atom cpu atom cpu_timestamp atom cr @@ -211,6 +212,7 @@ atom dsend_continue_trap atom dunlink atom duplicate_bag atom dupnames +atom einval atom elib_malloc atom emulator atom enable_trace @@ -258,6 +260,7 @@ atom functions atom function_clause atom garbage_collecting atom garbage_collection +atom garbage_collection_info atom gc_end atom gc_start atom Ge='>=' @@ -268,6 +271,7 @@ atom get_tcw atom getenv atom gather_gc_info_result atom gather_io_bytes +atom gather_microstate_accounting_result atom gather_sched_wall_time_result atom gather_system_check_result atom getting_linked @@ -353,17 +357,20 @@ atom memory_internal atom memory_types atom message atom message_binary +atom message_queue_data atom message_queue_len atom messages atom merge_trap atom meta atom meta_match_spec atom micro_seconds +atom microstate_accounting atom milli_seconds atom min_heap_size atom min_bin_vheap_size atom minor_version atom Minus='-' +atom mixed atom module atom module_info atom monitored_by @@ -427,10 +434,12 @@ atom notify atom notsup atom nouse_stdio atom objects +atom off_heap atom offset atom ok atom old_heap_block_size atom old_heap_size +atom on_heap atom on_load atom open atom open_error @@ -441,13 +450,6 @@ atom orelse atom os_pid atom os_type atom os_version -atom ose_bg_proc -atom ose_int_proc -atom ose_phantom -atom ose_pri_proc -atom ose_process_prio -atom ose_process_type -atom ose_ti_proc atom out atom out_exited atom out_exiting @@ -592,6 +594,8 @@ atom trace trace_ts traced atom trace_control_word atom tracer atom trap_exit +atom trim +atom trim_all atom try_clause atom true atom tuple diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 0e192b1ebd..1b4c022370 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -38,9 +38,9 @@ #include "erl_thr_progress.h" static void set_default_trace_pattern(Eterm module); -static Eterm check_process_code(Process* rp, Module* modp, int allow_gc, int *redsp); +static Eterm check_process_code(Process* rp, Module* modp, Uint flags, int *redsp); static void delete_code(Module* modp); -static void decrement_refc(BeamInstr* code); +static void decrement_refc(BeamCodeHeader*); static int any_heap_ref_ptrs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size); static int any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size); @@ -58,8 +58,8 @@ BIF_RETTYPE code_is_module_native_1(BIF_ALIST_1) return am_undefined; } erts_rlock_old_code(code_ix); - res = (erts_is_module_native(modp->curr.code) || - erts_is_module_native(modp->old.code)) ? + res = (erts_is_module_native(modp->curr.code_hdr) || + erts_is_module_native(modp->old.code_hdr)) ? am_true : am_false; erts_runlock_old_code(code_ix); return res; @@ -81,7 +81,7 @@ BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3) modp = erts_get_module(BIF_ARG_1, erts_active_code_ix()); if (modp && modp->curr.num_breakpoints > 0) { - ASSERT(modp->curr.code != NULL); + ASSERT(modp->curr.code_hdr != NULL); erts_clear_module_break(modp); ASSERT(modp->curr.num_breakpoints == 0); } @@ -154,7 +154,7 @@ static struct /* Protected by code_write_permission */ { Process* stager; ErtsThrPrgrLaterOp lop; -}commiter_state; +} committer_state; #endif static Eterm @@ -179,8 +179,8 @@ exception_list(Process* p, Eterm tag, struct m* mp, Sint exceptions) BIF_RETTYPE finish_loading_1(BIF_ALIST_1) { - int i; - int n; + Sint i; + Sint n; struct m* p = NULL; Uint exceptions; Eterm res; @@ -201,7 +201,7 @@ finish_loading_1(BIF_ALIST_1) */ n = erts_list_length(BIF_ARG_1); - if (n == -1) { + if (n < 0) { ERTS_BIF_PREP_ERROR(res, BIF_P, BADARG); goto done; } @@ -281,7 +281,7 @@ finish_loading_1(BIF_ALIST_1) exceptions = 0; for (i = 0; i < n; i++) { p[i].exception = 0; - if (p[i].modp->curr.code && p[i].modp->old.code) { + if (p[i].modp->curr.code_hdr && p[i].modp->old.code_hdr) { p[i].exception = 1; exceptions++; } @@ -367,9 +367,9 @@ staging_epilogue(Process* c_p, int commit, Eterm res, int is_blocking, * schedulers to read active code_ix in a safe way while executing * without any memory barriers at all. */ - ASSERT(commiter_state.stager == NULL); - commiter_state.stager = c_p; - erts_schedule_thr_prgr_later_op(smp_code_ix_commiter, NULL, &commiter_state.lop); + ASSERT(committer_state.stager == NULL); + committer_state.stager = c_p; + erts_schedule_thr_prgr_later_op(smp_code_ix_commiter, NULL, &committer_state.lop); erts_proc_inc_refc(c_p); erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL); /* @@ -385,11 +385,11 @@ staging_epilogue(Process* c_p, int commit, Eterm res, int is_blocking, #ifdef ERTS_SMP static void smp_code_ix_commiter(void* null) { - Process* p = commiter_state.stager; + Process* p = committer_state.stager; erts_commit_staging_code_ix(); #ifdef DEBUG - commiter_state.stager = NULL; + committer_state.stager = NULL; #endif erts_release_code_write_permission(); erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); @@ -417,7 +417,7 @@ check_old_code_1(BIF_ALIST_1) modp = erts_get_module(BIF_ARG_1, code_ix); if (modp != NULL) { erts_rlock_old_code(code_ix); - if (modp->old.code != NULL) { + if (modp->old.code_hdr) { res = am_true; } erts_runlock_old_code(code_ix); @@ -426,7 +426,7 @@ check_old_code_1(BIF_ALIST_1) } Eterm -erts_check_process_code(Process *c_p, Eterm module, int allow_gc, int *redsp) +erts_check_process_code(Process *c_p, Eterm module, Uint flags, int *redsp) { Module* modp; Eterm res; @@ -441,7 +441,8 @@ erts_check_process_code(Process *c_p, Eterm module, int allow_gc, int *redsp) if (!modp) return am_false; erts_rlock_old_code(code_ix); - res = modp->old.code ? check_process_code(c_p, modp, allow_gc, redsp) : am_false; + res = (!modp->old.code_hdr ? am_false : + check_process_code(c_p, modp, flags, redsp)); erts_runlock_old_code(code_ix); return res; @@ -450,49 +451,21 @@ erts_check_process_code(Process *c_p, Eterm module, int allow_gc, int *redsp) BIF_RETTYPE erts_internal_check_process_code_2(BIF_ALIST_2) { int reds = 0; + Uint flags; Eterm res; - Eterm olist = BIF_ARG_2; - int allow_gc = 1; if (is_not_atom(BIF_ARG_1)) goto badarg; - while (is_list(olist)) { - Eterm *lp = list_val(olist); - Eterm opt = CAR(lp); - if (is_tuple(opt)) { - Eterm* tp = tuple_val(opt); - switch (arityval(tp[0])) { - case 2: - switch (tp[1]) { - case am_allow_gc: - switch (tp[2]) { - case am_false: - allow_gc = 0; - break; - case am_true: - allow_gc = 1; - break; - default: - goto badarg; - } - break; - default: - goto badarg; - } - break; - default: - goto badarg; - } - } - else - goto badarg; - olist = CDR(lp); + if (is_not_small(BIF_ARG_2)) + goto badarg; + + flags = unsigned_val(BIF_ARG_2); + if (flags & ~ERTS_CPC_ALL) { + goto badarg; } - if (is_not_nil(olist)) - goto badarg; - res = erts_check_process_code(BIF_P, BIF_ARG_1, allow_gc, &reds); + res = erts_check_process_code(BIF_P, BIF_ARG_1, flags, &reds); ASSERT(is_value(res)); @@ -525,7 +498,7 @@ BIF_RETTYPE delete_module_1(BIF_ALIST_1) if (!modp) { res = am_undefined; } - else if (modp->old.code != 0) { + else if (modp->old.code_hdr) { erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); erts_dsprintf(dsbufp, "Module %T must be purged before loading\n", BIF_ARG_1); @@ -563,8 +536,8 @@ BIF_RETTYPE module_loaded_1(BIF_ALIST_1) } code_ix = erts_active_code_ix(); if ((modp = erts_get_module(BIF_ARG_1, code_ix)) != NULL) { - if (modp->curr.code != NULL - && modp->curr.code[MI_ON_LOAD_FUNCTION_PTR] == 0) { + if (modp->curr.code_hdr + && modp->curr.code_hdr->on_load_function_ptr == NULL) { res = am_true; } } @@ -611,8 +584,8 @@ BIF_RETTYPE call_on_load_function_1(BIF_ALIST_1) { Module* modp = erts_get_module(BIF_ARG_1, erts_active_code_ix()); - if (modp && modp->curr.code) { - BIF_TRAP_CODE_PTR_0(BIF_P, modp->curr.code[MI_ON_LOAD_FUNCTION_PTR]); + if (modp && modp->curr.code_hdr) { + BIF_TRAP_CODE_PTR_0(BIF_P, modp->curr.code_hdr->on_load_function_ptr); } else { BIF_ERROR(BIF_P, BADARG); @@ -623,7 +596,6 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) { ErtsCodeIndex code_ix; Module* modp; - Eterm on_load; if (!erts_try_seize_code_write_permission(BIF_P)) { ERTS_BIF_YIELD2(bif_export[BIF_finish_after_on_load_2], @@ -638,14 +610,14 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) code_ix = erts_active_code_ix(); modp = erts_get_module(BIF_ARG_1, code_ix); - if (!modp || modp->curr.code == 0) { + if (!modp || !modp->curr.code_hdr) { error: erts_smp_thr_progress_unblock(); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_release_code_write_permission(); BIF_ERROR(BIF_P, BADARG); } - if ((on_load = modp->curr.code[MI_ON_LOAD_FUNCTION_PTR]) == 0) { + if (modp->curr.code_hdr->on_load_function_ptr == NULL) { goto error; } if (BIF_ARG_2 != am_false && BIF_ARG_2 != am_true) { @@ -667,7 +639,7 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) ep->code[4] = 0; } } - modp->curr.code[MI_ON_LOAD_FUNCTION_PTR] = 0; + modp->curr.code_hdr->on_load_function_ptr = NULL; set_default_trace_pattern(BIF_ARG_1); } else if (BIF_ARG_2 == am_false) { BeamInstr* code; @@ -679,13 +651,16 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) * the current code; the old code is not touched. */ erts_total_code_size -= modp->curr.code_length; - code = modp->curr.code; - end = (BeamInstr *)((char *)code + modp->curr.code_length); + code = (BeamInstr*) modp->curr.code_hdr; + end = (BeamInstr *) ((char *)code + modp->curr.code_length); erts_cleanup_funs_on_purge(code, end); beam_catches_delmod(modp->curr.catches, code, modp->curr.code_length, erts_active_code_ix()); - erts_free(ERTS_ALC_T_CODE, (void *) code); - modp->curr.code = NULL; + if (modp->curr.code_hdr->literals_start) { + erts_free(ERTS_ALC_T_LITERAL, modp->curr.code_hdr->literals_start); + } + erts_free(ERTS_ALC_T_CODE, modp->curr.code_hdr); + modp->curr.code_hdr = NULL; modp->curr.code_length = 0; modp->curr.catches = BEAM_CATCHES_NIL; erts_remove_from_ranges(code); @@ -721,31 +696,50 @@ set_default_trace_pattern(Eterm module) } } +static ERTS_INLINE int +check_mod_funs(Process *p, ErlOffHeap *off_heap, char *area, size_t area_size) +{ + struct erl_off_heap_header* oh; + for (oh = off_heap->first; oh; oh = oh->next) { + if (thing_subtag(oh->thing_word) == FUN_SUBTAG) { + ErlFunThing* funp = (ErlFunThing*) oh; + if (ErtsInArea(funp->fe->address, area, area_size)) + return !0; + } + } + return 0; +} + + static Eterm -check_process_code(Process* rp, Module* modp, int allow_gc, int *redsp) +check_process_code(Process* rp, Module* modp, Uint flags, int *redsp) { BeamInstr* start; + char* literals; + Uint lit_bsize; char* mod_start; Uint mod_size; - BeamInstr* end; Eterm* sp; - struct erl_off_heap_header* oh; int done_gc = 0; + int need_gc = 0; + ErtsMessage *msgp; + ErlHeapFragment *hfrag; -#define INSIDE(a) (start <= (a) && (a) < end) +#define ERTS_ORDINARY_GC__ (1 << 0) +#define ERTS_LITERAL_GC__ (1 << 1) /* * Pick up limits for the module. */ - start = modp->old.code; - end = (BeamInstr *)((char *)start + modp->old.code_length); + start = (BeamInstr*) modp->old.code_hdr; mod_start = (char *) start; mod_size = modp->old.code_length; /* * Check if current instruction or continuation pointer points into module. */ - if (INSIDE(rp->i) || INSIDE(rp->cp)) { + if (ErtsInArea(rp->i, mod_start, mod_size) + || ErtsInArea(rp->cp, mod_start, mod_size)) { return am_true; } @@ -753,7 +747,7 @@ check_process_code(Process* rp, Module* modp, int allow_gc, int *redsp) * Check all continuation pointers stored on the stack. */ for (sp = rp->stop; sp < STACK_START(rp); sp++) { - if (is_CP(*sp) && INSIDE(cp_val(*sp))) { + if (is_CP(*sp) && ErtsInArea(cp_val(*sp), mod_start, mod_size)) { return am_true; } } @@ -767,15 +761,15 @@ check_process_code(Process* rp, Module* modp, int allow_gc, int *redsp) struct StackTrace *s; ASSERT(is_list(rp->ftrace)); s = (struct StackTrace *) big_val(CDR(list_val(rp->ftrace))); - if ((s->pc && INSIDE(s->pc)) || - (s->current && INSIDE(s->current))) { + if ((s->pc && ErtsInArea(s->pc, mod_start, mod_size)) || + (s->current && ErtsInArea(s->current, mod_start, mod_size))) { rp->freason = EXC_NULL; rp->fvalue = NIL; rp->ftrace = NIL; } else { int i; for (i = 0; i < s->depth; i++) { - if (INSIDE(s->trace[i])) { + if (ErtsInArea(s->trace[i], mod_start, mod_size)) { rp->freason = EXC_NULL; rp->fvalue = NIL; rp->ftrace = NIL; @@ -796,111 +790,147 @@ check_process_code(Process* rp, Module* modp, int allow_gc, int *redsp) } /* - * See if there are funs that refer to the old version of the module. + * Message queue can contains funs, but (at least currently) no + * constants. If we got references to this module from the message + * queue, a GC cannot remove these... */ - rescan: - for (oh = MSO(rp).first; oh; oh = oh->next) { - if (thing_subtag(oh->thing_word) == FUN_SUBTAG) { - ErlFunThing* funp = (ErlFunThing*) oh; + erts_smp_proc_lock(rp, ERTS_PROC_LOCK_MSGQ); + ERTS_SMP_MSGQ_MV_INQ2PRIVQ(rp); + erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MSGQ); - if (INSIDE((BeamInstr *) funp->fe->address)) { - if (done_gc) { - return am_true; - } else { - if (!allow_gc) - return am_aborted; - /* - * Try to get rid of this fun by garbage collecting. - * Clear both fvalue and ftrace to make sure they - * don't hold any funs. - */ - rp->freason = EXC_NULL; - rp->fvalue = NIL; - rp->ftrace = NIL; - done_gc = 1; - FLAGS(rp) |= F_NEED_FULLSWEEP; - *redsp += erts_garbage_collect(rp, 0, rp->arg_reg, rp->arity); - goto rescan; - } - } + literals = (char*) modp->old.code_hdr->literals_start; + lit_bsize = (char*) modp->old.code_hdr->literals_end - literals; + + for (msgp = rp->msg.first; msgp; msgp = msgp->next) { + if (msgp->data.attached == ERTS_MSG_COMBINED_HFRAG) + hfrag = &msgp->hfrag; + else if (is_value(ERL_MESSAGE_TERM(msgp)) && msgp->data.heap_frag) + hfrag = msgp->data.heap_frag; + else + continue; + for (; hfrag; hfrag = hfrag->next) { + if (check_mod_funs(rp, &hfrag->off_heap, mod_start, mod_size)) + return am_true; + /* Should not contain any constants... */ + ASSERT(!any_heap_refs(&hfrag->mem[0], + &hfrag->mem[hfrag->used_size], + literals, + lit_bsize)); } } - /* - * See if there are constants inside the module referenced by the process. - */ - done_gc = 0; - for (;;) { - ErlMessage* mp; - - if (any_heap_ref_ptrs(&rp->fvalue, &rp->fvalue+1, mod_start, mod_size)) { + while (1) { + + /* Check heap, stack etc... */ + if (check_mod_funs(rp, &rp->off_heap, mod_start, mod_size)) + goto try_gc; + if (!(flags & ERTS_CPC_COPY_LITERALS)) { + /* Process ok. May contain old literals but we will be called + * again before module is purged. + */ + return am_false; + } + if (any_heap_ref_ptrs(&rp->fvalue, &rp->fvalue+1, literals, lit_bsize)) { rp->freason = EXC_NULL; rp->fvalue = NIL; rp->ftrace = NIL; } - if (any_heap_ref_ptrs(rp->stop, rp->hend, mod_start, mod_size)) { - goto need_gc; - } - if (any_heap_refs(rp->heap, rp->htop, mod_start, mod_size)) { - goto need_gc; - } - - if (any_heap_refs(rp->old_heap, rp->old_htop, mod_start, mod_size)) { - goto need_gc; + if (any_heap_ref_ptrs(rp->stop, rp->hend, literals, lit_bsize)) + goto try_literal_gc; + if (any_heap_refs(rp->heap, rp->htop, literals, lit_bsize)) + goto try_literal_gc; + if (any_heap_refs(rp->old_heap, rp->old_htop, literals, lit_bsize)) + goto try_literal_gc; + + /* Check dictionary */ + if (rp->dictionary) { + Eterm* start = ERTS_PD_START(rp->dictionary); + Eterm* end = start + ERTS_PD_SIZE(rp->dictionary); + + if (any_heap_ref_ptrs(start, end, literals, lit_bsize)) + goto try_literal_gc; } - if (rp->dictionary != NULL) { - Eterm* start = rp->dictionary->data; - Eterm* end = start + rp->dictionary->used; + /* Check heap fragments */ + for (hfrag = rp->mbuf; hfrag; hfrag = hfrag->next) { + Eterm *hp, *hp_end; + /* Off heap lists should already have been moved into process */ + ASSERT(!check_mod_funs(rp, &hfrag->off_heap, mod_start, mod_size)); - if (any_heap_ref_ptrs(start, end, mod_start, mod_size)) { - goto need_gc; - } + hp = &hfrag->mem[0]; + hp_end = &hfrag->mem[hfrag->used_size]; + if (any_heap_refs(hp, hp_end, literals, lit_bsize)) + goto try_literal_gc; } - for (mp = rp->msg.first; mp != NULL; mp = mp->next) { - if (any_heap_ref_ptrs(mp->m, mp->m+2, mod_start, mod_size)) { - goto need_gc; +#ifdef DEBUG + /* + * Message buffer fragments should not have any references + * to constants, and off heap lists should already have + * been moved into process off heap structure. + */ + for (msgp = rp->msg_frag; msgp; msgp = msgp->next) { + if (msgp->data.attached == ERTS_MSG_COMBINED_HFRAG) + hfrag = &msgp->hfrag; + else + hfrag = msgp->data.heap_frag; + for (; hfrag; hfrag = hfrag->next) { + Eterm *hp, *hp_end; + ASSERT(!check_mod_funs(rp, &hfrag->off_heap, mod_start, mod_size)); + + hp = &hfrag->mem[0]; + hp_end = &hfrag->mem[hfrag->used_size]; + ASSERT(!any_heap_refs(hp, hp_end, literals, lit_bsize)); } } - break; - need_gc: - if (done_gc) { +#endif + + return am_false; + + try_literal_gc: + need_gc |= ERTS_LITERAL_GC__; + + try_gc: + need_gc |= ERTS_ORDINARY_GC__; + + if ((done_gc & need_gc) == need_gc) return am_true; - } else { - Eterm* literals; - Uint lit_size; - struct erl_off_heap_header* oh; - if (!allow_gc) - return am_aborted; + if (!(flags & ERTS_CPC_ALLOW_GC)) + return am_aborted; - /* - * Try to get rid of constants by by garbage collecting. - * Clear both fvalue and ftrace. - */ - rp->freason = EXC_NULL; - rp->fvalue = NIL; - rp->ftrace = NIL; - done_gc = 1; + need_gc &= ~done_gc; + + /* + * Try to get rid of constants by by garbage collecting. + * Clear both fvalue and ftrace. + */ + + rp->freason = EXC_NULL; + rp->fvalue = NIL; + rp->ftrace = NIL; + + if (need_gc & ERTS_ORDINARY_GC__) { FLAGS(rp) |= F_NEED_FULLSWEEP; - *redsp += erts_garbage_collect(rp, 0, rp->arg_reg, rp->arity); - literals = (Eterm *) modp->old.code[MI_LITERALS_START]; - lit_size = (Eterm *) modp->old.code[MI_LITERALS_END] - literals; - oh = (struct erl_off_heap_header *) - modp->old.code[MI_LITERALS_OFF_HEAP]; - *redsp += lit_size / 10; /* Need, better value... */ - erts_garbage_collect_literals(rp, literals, lit_size, oh); + *redsp += erts_garbage_collect_nobump(rp, 0, rp->arg_reg, rp->arity); + done_gc |= ERTS_ORDINARY_GC__; } + if (need_gc & ERTS_LITERAL_GC__) { + struct erl_off_heap_header* oh; + oh = modp->old.code_hdr->literals_off_heap; + *redsp += lit_bsize / 64; /* Need, better value... */ + erts_garbage_collect_literals(rp, (Eterm*)literals, lit_bsize, oh); + done_gc |= ERTS_LITERAL_GC__; + } + need_gc = 0; } - return am_false; -#undef INSIDE -} -#define in_area(ptr,start,nbytes) \ - ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes)) +#undef ERTS_ORDINARY_GC__ +#undef ERTS_LITERAL_GC__ + +} static int any_heap_ref_ptrs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size) @@ -913,7 +943,7 @@ any_heap_ref_ptrs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size) switch (primary_tag(val)) { case TAG_PRIMARY_BOXED: case TAG_PRIMARY_LIST: - if (in_area(EXPAND_POINTER(val), mod_start, mod_size)) { + if (ErtsInArea(val, mod_start, mod_size)) { return 1; } break; @@ -933,7 +963,7 @@ any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size) switch (primary_tag(val)) { case TAG_PRIMARY_BOXED: case TAG_PRIMARY_LIST: - if (in_area(EXPAND_POINTER(val), mod_start, mod_size)) { + if (ErtsInArea(val, mod_start, mod_size)) { return 1; } break; @@ -943,7 +973,7 @@ any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size) if (header_is_bin_matchstate(val)) { ErlBinMatchState *ms = (ErlBinMatchState*) p; ErlBinMatchBuffer *mb = &(ms->mb); - if (in_area(EXPAND_POINTER(mb->orig), mod_start, mod_size)) { + if (ErtsInArea(mb->orig, mod_start, mod_size)) { return 1; } } @@ -958,7 +988,104 @@ any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size) #undef in_area -BIF_RETTYPE purge_module_1(BIF_ALIST_1) +#ifdef ERTS_SMP +static void copy_literals_commit(void*); +#endif + +copy_literals_t erts_clrange = {NULL, 0, THE_NON_VALUE}; + +/* copy literals + * + * copy_literals.ptr = LitPtr + * copy_literals.sz = LitSz + * ------ THR PROG COMMIT ----- + * + * - check process code + * - check process code + * ... + * copy_literals.ptr = NULL + * copy_literals.sz = 0 + * ------ THR PROG COMMIT ----- + * ... + */ + + +BIF_RETTYPE erts_internal_copy_literals_2(BIF_ALIST_2) +{ + ErtsCodeIndex code_ix; + Eterm res = am_true; + + if (is_not_atom(BIF_ARG_1) || (am_true != BIF_ARG_2 && am_false != BIF_ARG_2)) { + BIF_ERROR(BIF_P, BADARG); + } + + if (!erts_try_seize_code_write_permission(BIF_P)) { + ERTS_BIF_YIELD2(bif_export[BIF_erts_internal_copy_literals_2], + BIF_P, BIF_ARG_1, BIF_ARG_2); + } + + code_ix = erts_active_code_ix(); + + if (BIF_ARG_2 == am_true) { + Module* modp = erts_get_module(BIF_ARG_1, code_ix); + if (!modp || !modp->old.code_hdr) { + res = am_false; + goto done; + } + if (erts_clrange.ptr != NULL + && !(BIF_P->static_flags & ERTS_STC_FLG_SYSTEM_PROC)) { + res = am_aborted; + goto done; + } + erts_clrange.ptr = modp->old.code_hdr->literals_start; + erts_clrange.sz = modp->old.code_hdr->literals_end - erts_clrange.ptr; + erts_clrange.pid = BIF_P->common.id; + } else if (BIF_ARG_2 == am_false) { + if (erts_clrange.pid != BIF_P->common.id) { + res = am_false; + goto done; + } + erts_clrange.ptr = NULL; + erts_clrange.sz = 0; + erts_clrange.pid = THE_NON_VALUE; + } + +#ifdef ERTS_SMP + ASSERT(committer_state.stager == NULL); + committer_state.stager = BIF_P; + erts_schedule_thr_prgr_later_op(copy_literals_commit, NULL, &committer_state.lop); + erts_proc_inc_refc(BIF_P); + erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, NULL); + ERTS_BIF_YIELD_RETURN(BIF_P, am_true); +#endif +done: + erts_release_code_write_permission(); + BIF_RET(res); +} + +#ifdef ERTS_SMP +static void copy_literals_commit(void* null) { + Process* p = committer_state.stager; +#ifdef DEBUG + committer_state.stager = NULL; +#endif + erts_release_code_write_permission(); + erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + if (!ERTS_PROC_IS_EXITING(p)) { + erts_resume(p, ERTS_PROC_LOCK_STATUS); + } + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + erts_proc_dec_refc(p); +} +#endif /* ERTS_SMP */ + + +/* Do the actualy module purging and return: + * true for success + * false if no such old module + * BADARG if not an atom + */ +BIF_RETTYPE erts_internal_purge_module_1(BIF_ALIST_1) { ErtsCodeIndex code_ix; BeamInstr* code; @@ -972,7 +1099,8 @@ BIF_RETTYPE purge_module_1(BIF_ALIST_1) } if (!erts_try_seize_code_write_permission(BIF_P)) { - ERTS_BIF_YIELD1(bif_export[BIF_purge_module_1], BIF_P, BIF_ARG_1); + ERTS_BIF_YIELD1(bif_export[BIF_erts_internal_purge_module_1], + BIF_P, BIF_ARG_1); } code_ix = erts_active_code_ix(); @@ -982,7 +1110,7 @@ BIF_RETTYPE purge_module_1(BIF_ALIST_1) */ if ((modp = erts_get_module(BIF_ARG_1, code_ix)) == NULL) { - ERTS_BIF_PREP_ERROR(ret, BIF_P, BADARG); + ERTS_BIF_PREP_RET(ret, am_false); } else { erts_rwlock_old_code(code_ix); @@ -990,8 +1118,8 @@ BIF_RETTYPE purge_module_1(BIF_ALIST_1) /* * Any code to purge? */ - if (modp->old.code == 0) { - ERTS_BIF_PREP_ERROR(ret, BIF_P, BADARG); + if (!modp->old.code_hdr) { + ERTS_BIF_PREP_RET(ret, am_false); } else { /* @@ -1013,14 +1141,17 @@ BIF_RETTYPE purge_module_1(BIF_ALIST_1) */ ASSERT(erts_total_code_size >= modp->old.code_length); erts_total_code_size -= modp->old.code_length; - code = modp->old.code; + code = (BeamInstr*) modp->old.code_hdr; end = (BeamInstr *)((char *)code + modp->old.code_length); erts_cleanup_funs_on_purge(code, end); beam_catches_delmod(modp->old.catches, code, modp->old.code_length, code_ix); - decrement_refc(code); + decrement_refc(modp->old.code_hdr); + if (modp->old.code_hdr->literals_start) { + erts_free(ERTS_ALC_T_LITERAL, modp->old.code_hdr->literals_start); + } erts_free(ERTS_ALC_T_CODE, (void *) code); - modp->old.code = NULL; + modp->old.code_hdr = NULL; modp->old.code_length = 0; modp->old.catches = BEAM_CATCHES_NIL; erts_remove_from_ranges(code); @@ -1037,10 +1168,9 @@ BIF_RETTYPE purge_module_1(BIF_ALIST_1) } static void -decrement_refc(BeamInstr* code) +decrement_refc(BeamCodeHeader* code_hdr) { - struct erl_off_heap_header* oh = - (struct erl_off_heap_header *) code[MI_LITERALS_OFF_HEAP]; + struct erl_off_heap_header* oh = code_hdr->literals_off_heap; while (oh) { Binary* bptr; @@ -1089,7 +1219,7 @@ delete_code(Module* modp) ASSERT(modp->curr.num_breakpoints == 0); ASSERT(modp->curr.num_traced_exports == 0); modp->old = modp->curr; - modp->curr.code = NULL; + modp->curr.code_hdr = NULL; modp->curr.code_length = 0; modp->curr.catches = BEAM_CATCHES_NIL; modp->curr.nif = NULL; @@ -1106,9 +1236,9 @@ beam_make_current_old(Process *c_p, ErtsProcLocks c_p_locks, Eterm module) * if not, delete old code; error if old code already exists. */ - if (modp->curr.code != NULL && modp->old.code != NULL) { + if (modp->curr.code_hdr && modp->old.code_hdr) { return am_not_purged; - } else if (modp->old.code == NULL) { /* Make the current version old. */ + } else if (!modp->old.code_hdr) { /* Make the current version old. */ delete_code(modp); } return NIL; diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c index 9860968687..5d471d168b 100644 --- a/erts/emulator/beam/beam_bp.c +++ b/erts/emulator/beam/beam_bp.c @@ -154,8 +154,8 @@ erts_bp_match_functions(BpFunctions* f, Eterm mfa[3], int specified) num_modules = 0; for (current = 0; current < max_modules; current++) { modp = module_code(current, code_ix); - if (modp->curr.code) { - max_funcs += modp->curr.code[MI_NUM_FUNCTIONS]; + if (modp->curr.code_hdr) { + max_funcs += modp->curr.code_hdr->num_functions; module[num_modules++] = modp; } } @@ -163,9 +163,9 @@ erts_bp_match_functions(BpFunctions* f, Eterm mfa[3], int specified) f->matching = (BpFunction *) Alloc(max_funcs*sizeof(BpFunction)); i = 0; for (current = 0; current < num_modules; current++) { - BeamInstr** code_base = (BeamInstr **) module[current]->curr.code; + BeamCodeHeader* code_hdr = module[current]->curr.code_hdr; BeamInstr* code; - Uint num_functions = (Uint)(UWord) code_base[MI_NUM_FUNCTIONS]; + Uint num_functions = (Uint)(UWord) code_hdr->num_functions; Uint fi; if (specified > 0) { @@ -179,7 +179,7 @@ erts_bp_match_functions(BpFunctions* f, Eterm mfa[3], int specified) BeamInstr* pc; int wi; - code = code_base[MI_FUNCTIONS+fi]; + code = code_hdr->functions[fi]; ASSERT(code[0] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); pc = code+5; if (erts_is_native_break(pc)) { @@ -549,21 +549,21 @@ erts_clear_all_breaks(BpFunctions* f) int erts_clear_module_break(Module *modp) { - BeamInstr** code_base; + BeamCodeHeader* code_hdr; Uint n; Uint i; ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); ASSERT(modp); - code_base = (BeamInstr **) modp->curr.code; - if (code_base == NULL) { + code_hdr = modp->curr.code_hdr; + if (!code_hdr) { return 0; } - n = (Uint)(UWord) code_base[MI_NUM_FUNCTIONS]; + n = (Uint)(UWord) code_hdr->num_functions; for (i = 0; i < n; ++i) { BeamInstr* pc; - pc = code_base[MI_FUNCTIONS+i] + 5; + pc = code_hdr->functions[i] + 5; if (erts_is_native_break(pc)) { continue; } @@ -575,7 +575,7 @@ erts_clear_module_break(Module *modp) { for (i = 0; i < n; ++i) { BeamInstr* pc; - pc = code_base[MI_FUNCTIONS+i] + 5; + pc = code_hdr->functions[i] + 5; if (erts_is_native_break(pc)) { continue; } @@ -864,7 +864,7 @@ do_call_trace(Process* c_p, BeamInstr* I, Eterm* reg, Eterm* cpp; int return_to_trace = 0; BeamInstr w; - BeamInstr *cp_save; + BeamInstr *cp_save = c_p->cp; Uint32 flags; Uint need = 0; Eterm* E = c_p->stop; @@ -1205,17 +1205,17 @@ int erts_is_time_break(Process *p, BeamInstr *pc, Eterm *retval) { BeamInstr * erts_find_local_func(Eterm mfa[3]) { Module *modp; - BeamInstr** code_base; + BeamCodeHeader* code_hdr; BeamInstr* code_ptr; Uint i,n; if ((modp = erts_get_module(mfa[0], erts_active_code_ix())) == NULL) return NULL; - if ((code_base = (BeamInstr **) modp->curr.code) == NULL) + if ((code_hdr = modp->curr.code_hdr) == NULL) return NULL; - n = (BeamInstr) code_base[MI_NUM_FUNCTIONS]; + n = (BeamInstr) code_hdr->num_functions; for (i = 0; i < n; ++i) { - code_ptr = code_base[MI_FUNCTIONS+i]; + code_ptr = code_hdr->functions[i]; ASSERT(((BeamInstr) BeamOp(op_i_func_info_IaaI)) == code_ptr[0]); ASSERT(mfa[0] == ((Eterm) code_ptr[2]) || is_nil((Eterm) code_ptr[2])); diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c index 90985e4f53..e37bd4d78c 100644 --- a/erts/emulator/beam/beam_debug.c +++ b/erts/emulator/beam/beam_debug.c @@ -73,6 +73,40 @@ erts_debug_flat_size_1(BIF_ALIST_1) } } +BIF_RETTYPE +erts_debug_size_shared_1(BIF_ALIST_1) +{ + Process* p = BIF_P; + Eterm term = BIF_ARG_1; + Uint size = size_shared(term); + + if (IS_USMALL(0, size)) { + BIF_RET(make_small(size)); + } else { + Eterm* hp = HAlloc(p, BIG_UINT_HEAP_SIZE); + BIF_RET(uint_to_big(size, hp)); + } +} + +BIF_RETTYPE +erts_debug_copy_shared_1(BIF_ALIST_1) +{ + Process* p = BIF_P; + Eterm term = BIF_ARG_1; + Uint size; + Eterm* hp; + Eterm copy; + erts_shcopy_t info; + INITIALIZE_SHCOPY(info); + + size = copy_shared_calculate(term, &info); + if (size > 0) { + hp = HAlloc(p, size); + } + copy = copy_shared_perform(term, size, &info, &hp, &p->off_heap); + DESTROY_SHCOPY(info); + BIF_RET(copy); +} BIF_RETTYPE erts_debug_breakpoint_2(BIF_ALIST_2) @@ -208,7 +242,7 @@ erts_debug_disassemble_1(BIF_ALIST_1) Eterm bin; Eterm mfa; BeamInstr* funcinfo = NULL; /* Initialized to eliminate warning. */ - BeamInstr* code_base; + BeamCodeHeader* code_hdr; BeamInstr* code_ptr = NULL; /* Initialized to eliminate warning. */ BeamInstr instr; BeamInstr uaddr; @@ -258,12 +292,12 @@ erts_debug_disassemble_1(BIF_ALIST_1) */ code_ptr = ((BeamInstr *) ep->addressv[code_ix]) - 5; funcinfo = code_ptr+2; - } else if (modp == NULL || (code_base = modp->curr.code) == NULL) { + } else if (modp == NULL || (code_hdr = modp->curr.code_hdr) == NULL) { BIF_RET(am_undef); } else { - n = code_base[MI_NUM_FUNCTIONS]; + n = code_hdr->num_functions; for (i = 0; i < n; i++) { - code_ptr = (BeamInstr *) code_base[MI_FUNCTIONS+i]; + code_ptr = code_hdr->functions[i]; if (code_ptr[3] == name && code_ptr[4] == arity) { funcinfo = code_ptr+2; break; @@ -432,39 +466,33 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) while (*sign) { switch (*sign) { case 'r': /* x(0) */ - erts_print(to, to_arg, "x(0)"); + erts_print(to, to_arg, "r(0)"); break; case 'x': /* x(N) */ - if (reg_index(ap[0]) == 0) { - erts_print(to, to_arg, "x[0]"); - } else { - erts_print(to, to_arg, "x(%d)", reg_index(ap[0])); + { + Uint n = ap[0] / sizeof(Eterm); + erts_print(to, to_arg, "x(%d)", n); + ap++; } - ap++; break; case 'y': /* y(N) */ - erts_print(to, to_arg, "y(%d)", reg_index(ap[0]) - CP_SIZE); - ap++; + { + Uint n = ap[0] / sizeof(Eterm) - CP_SIZE; + erts_print(to, to_arg, "y(%d)", n); + ap++; + } break; case 'n': /* Nil */ erts_print(to, to_arg, "[]"); break; case 's': /* Any source (tagged constant or register) */ - tag = beam_reg_tag(*ap); - if (tag == X_REG_DEF) { - if (reg_index(*ap) == 0) { - erts_print(to, to_arg, "x[0]"); - } else { - erts_print(to, to_arg, "x(%d)", reg_index(*ap)); - } - ap++; - break; - } else if (tag == Y_REG_DEF) { - erts_print(to, to_arg, "y(%d)", reg_index(*ap) - CP_SIZE); + tag = loader_tag(*ap); + if (tag == LOADER_X_REG) { + erts_print(to, to_arg, "x(%d)", loader_x_reg_index(*ap)); ap++; break; - } else if (tag == R_REG_DEF) { - erts_print(to, to_arg, "x(0)"); + } else if (tag == LOADER_Y_REG) { + erts_print(to, to_arg, "y(%d)", loader_y_reg_index(*ap) - CP_SIZE); ap++; break; } @@ -481,20 +509,12 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) ap++; break; case 'd': /* Destination (x(0), x(N), y(N)) */ - switch (beam_reg_tag(*ap)) { - case X_REG_DEF: - if (reg_index(*ap) == 0) { - erts_print(to, to_arg, "x[0]"); - } else { - erts_print(to, to_arg, "x(%d)", reg_index(*ap)); - } - break; - case Y_REG_DEF: - erts_print(to, to_arg, "y(%d)", reg_index(*ap) - CP_SIZE); - break; - case R_REG_DEF: - erts_print(to, to_arg, "x(0)"); - break; + if (*ap & 1) { + erts_print(to, to_arg, "y(%d)", + *ap / sizeof(Eterm) - CP_SIZE); + } else { + erts_print(to, to_arg, "x(%d)", + *ap / sizeof(Eterm)); } ap++; break; @@ -561,7 +581,7 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) ap++; break; case 'l': /* fr(N) */ - erts_print(to, to_arg, "fr(%d)", reg_index(ap[0])); + erts_print(to, to_arg, "fr(%d)", loader_reg_index(ap[0])); ap++; break; default: @@ -580,7 +600,6 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) unpacked = ap; ap = addr + size; switch (op) { - case op_i_select_val_lins_rfI: case op_i_select_val_lins_xfI: case op_i_select_val_lins_yfI: { @@ -600,7 +619,6 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) } } break; - case op_i_select_val_bins_rfI: case op_i_select_val_bins_xfI: case op_i_select_val_bins_yfI: { @@ -614,7 +632,6 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) } } break; - case op_i_select_tuple_arity_rfI: case op_i_select_tuple_arity_xfI: case op_i_select_tuple_arity_yfI: { @@ -639,7 +656,6 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) } } break; - case op_i_jump_on_val_rfII: case op_i_jump_on_val_xfII: case op_i_jump_on_val_yfII: { @@ -651,7 +667,6 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) } } break; - case op_i_jump_on_val_zero_rfI: case op_i_jump_on_val_zero_xfI: case op_i_jump_on_val_zero_yfI: { @@ -663,7 +678,6 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) } } break; - case op_i_put_tuple_rI: case op_i_put_tuple_xI: case op_i_put_tuple_yI: case op_new_map_dII: @@ -673,20 +687,16 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) int n = unpacked[-1]; while (n > 0) { - if (!is_header(ap[0])) { + switch (loader_tag(ap[0])) { + case LOADER_X_REG: + erts_print(to, to_arg, " x(%d)", loader_x_reg_index(ap[0])); + break; + case LOADER_Y_REG: + erts_print(to, to_arg, " x(%d)", loader_y_reg_index(ap[0])); + break; + default: erts_print(to, to_arg, " %T", (Eterm) ap[0]); - } else { - switch ((ap[0] >> 2) & 0x03) { - case R_REG_DEF: - erts_print(to, to_arg, " x(0)"); - break; - case X_REG_DEF: - erts_print(to, to_arg, " x(%d)", ap[0] >> 4); - break; - case Y_REG_DEF: - erts_print(to, to_arg, " y(%d)", ap[0] >> 4); - break; - } + break; } ap++, size++, n--; } @@ -699,18 +709,16 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) while (n > 0) { if (n % 3 == 1) { erts_print(to, to_arg, " %X", ap[0]); - } else if (!is_header(ap[0])) { - erts_print(to, to_arg, " %T", (Eterm) ap[0]); } else { - switch ((ap[0] >> 2) & 0x03) { - case R_REG_DEF: - erts_print(to, to_arg, " x(0)"); + switch (loader_tag(ap[0])) { + case LOADER_X_REG: + erts_print(to, to_arg, " x(%d)", loader_x_reg_index(ap[0])); break; - case X_REG_DEF: - erts_print(to, to_arg, " x(%d)", ap[0] >> 4); + case LOADER_Y_REG: + erts_print(to, to_arg, " y(%d)", loader_y_reg_index(ap[0])); break; - case Y_REG_DEF: - erts_print(to, to_arg, " y(%d)", ap[0] >> 4); + default: + erts_print(to, to_arg, " %T", (Eterm) ap[0]); break; } } diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 73292885ce..10f132abfc 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -99,10 +99,7 @@ do { \ do { \ int i_; \ int Arity_ = PC[-1]; \ - if (Arity_ > 0) { \ - CHECK_TERM(r(0)); \ - } \ - for (i_ = 1; i_ < Arity_; i_++) { \ + for (i_ = 0; i_ < Arity_; i_++) { \ CHECK_TERM(x(i_)); \ } \ } while (0) @@ -116,6 +113,9 @@ do { \ #define MAX(x, y) (((x) > (y)) ? (x) : (y)) #endif +#define GET_BIF_MODULE(p) ((Eterm) (((Export *) p)->code[0])) +#define GET_BIF_FUNCTION(p) ((Eterm) (((Export *) p)->code[1])) +#define GET_BIF_ARITY(p) ((Eterm) (((Export *) p)->code[2])) #define GET_BIF_ADDRESS(p) ((BifFunction) (((Export *) p)->code[4])) #define TermWords(t) (((t) / (sizeof(BeamInstr)/sizeof(Eterm))) + !!((t) % (sizeof(BeamInstr)/sizeof(Eterm)))) @@ -151,25 +151,21 @@ do { \ ASSERT(VALID_INSTR(* (Eterm *)(ip))); \ I = (ip) -#define FetchArgs(S1, S2) tmp_arg1 = (S1); tmp_arg2 = (S2) +/* + * Register target (X or Y register). + */ +#define REG_TARGET(Target) (*(((Target) & 1) ? &yb(Target-1) : &xb(Target))) /* * Store a result into a register given a destination descriptor. */ -#define StoreResult(Result, DestDesc) \ - do { \ - Eterm stb_reg; \ - stb_reg = (DestDesc); \ - CHECK_TERM(Result); \ - switch (beam_reg_tag(stb_reg)) { \ - case R_REG_DEF: \ - r(0) = (Result); break; \ - case X_REG_DEF: \ - xb(x_reg_offset(stb_reg)) = (Result); break; \ - default: \ - yb(y_reg_offset(stb_reg)) = (Result); break; \ - } \ +#define StoreResult(Result, DestDesc) \ + do { \ + Eterm stb_reg; \ + stb_reg = (DestDesc); \ + CHECK_TERM(Result); \ + REG_TARGET(stb_reg) = (Result); \ } while (0) #define StoreSimpleDest(Src, Dest) Dest = (Src) @@ -180,22 +176,16 @@ do { \ * be just before the next instruction. */ -#define StoreBifResult(Dst, Result) \ - do { \ - BeamInstr* stb_next; \ - Eterm stb_reg; \ - stb_reg = Arg(Dst); \ - I += (Dst) + 2; \ - stb_next = (BeamInstr *) *I; \ - CHECK_TERM(Result); \ - switch (beam_reg_tag(stb_reg)) { \ - case R_REG_DEF: \ - r(0) = (Result); Goto(stb_next); \ - case X_REG_DEF: \ - xb(x_reg_offset(stb_reg)) = (Result); Goto(stb_next); \ - default: \ - yb(y_reg_offset(stb_reg)) = (Result); Goto(stb_next); \ - } \ +#define StoreBifResult(Dst, Result) \ + do { \ + BeamInstr* stb_next; \ + Eterm stb_reg; \ + stb_reg = Arg(Dst); \ + I += (Dst) + 2; \ + stb_next = (BeamInstr *) *I; \ + CHECK_TERM(Result); \ + REG_TARGET(stb_reg) = (Result); \ + Goto(stb_next); \ } while (0) #define ClauseFail() goto jump_f @@ -250,6 +240,14 @@ void** beam_ops; HEAP_TOP(c_p) = HTOP; \ c_p->stop = E +#define HEAVY_SWAPIN \ + SWAPIN; \ + FCALLS = c_p->fcalls + +#define HEAVY_SWAPOUT \ + SWAPOUT; \ + c_p->fcalls = FCALLS + /* * Use LIGHT_SWAPOUT when the called function * will call HeapOnlyAlloc() (and never HAlloc()). @@ -293,7 +291,7 @@ void** beam_ops; #define Ib(N) (N) #define x(N) reg[N] #define y(N) E[N] -#define r(N) x##N +#define r(N) x(N) /* * Makes sure that there are StackNeed + HeapNeed + 1 words available @@ -309,12 +307,10 @@ void** beam_ops; needed = (StackNeed) + 1; \ if (E - HTOP < (needed + (HeapNeed))) { \ SWAPOUT; \ - reg[0] = r(0); \ PROCESS_MAIN_CHK_LOCKS(c_p); \ - FCALLS -= erts_garbage_collect(c_p, needed + (HeapNeed), reg, (M)); \ + FCALLS -= erts_garbage_collect_nobump(c_p, needed + (HeapNeed), reg, (M)); \ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \ PROCESS_MAIN_CHK_LOCKS(c_p); \ - r(0) = reg[0]; \ SWAPIN; \ } \ E -= needed; \ @@ -363,12 +359,10 @@ void** beam_ops; unsigned need = (Nh); \ if ((E - HTOP < need) || (MSO(c_p).overhead + (VNh) >= BIN_VHEAP_SZ(c_p))) {\ SWAPOUT; \ - reg[0] = r(0); \ PROCESS_MAIN_CHK_LOCKS(c_p); \ - FCALLS -= erts_garbage_collect(c_p, need, reg, (Live)); \ + FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live)); \ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \ PROCESS_MAIN_CHK_LOCKS(c_p); \ - r(0) = reg[0]; \ SWAPIN; \ } \ HEAP_SPACE_VERIFIED(need); \ @@ -386,12 +380,10 @@ void** beam_ops; unsigned need = (Nh); \ if (E - HTOP < need) { \ SWAPOUT; \ - reg[0] = r(0); \ PROCESS_MAIN_CHK_LOCKS(c_p); \ - FCALLS -= erts_garbage_collect(c_p, need, reg, (Live)); \ + FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live));\ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \ PROCESS_MAIN_CHK_LOCKS(c_p); \ - r(0) = reg[0]; \ SWAPIN; \ } \ HEAP_SPACE_VERIFIED(need); \ @@ -408,15 +400,11 @@ void** beam_ops; unsigned need = (Nh); \ if (E - HTOP < need) { \ SWAPOUT; \ - reg[0] = r(0); \ reg[Live] = Extra; \ PROCESS_MAIN_CHK_LOCKS(c_p); \ - FCALLS -= erts_garbage_collect(c_p, need, reg, (Live)+1); \ + FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live)+1); \ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \ PROCESS_MAIN_CHK_LOCKS(c_p); \ - if (Live > 0) { \ - r(0) = reg[0]; \ - } \ Extra = reg[Live]; \ SWAPIN; \ } \ @@ -438,10 +426,9 @@ void** beam_ops; #define MakeFun(FunP, NumFree) \ do { \ - SWAPOUT; \ - reg[0] = r(0); \ + HEAVY_SWAPOUT; \ r(0) = new_fun(c_p, reg, (ErlFunEntry *) FunP, NumFree); \ - SWAPIN; \ + HEAVY_SWAPIN; \ } while (0) #define PutTuple(Dst, Arity) \ @@ -531,15 +518,19 @@ void** beam_ops; ASSERT(VALID_INSTR(Dst)); \ Goto(Dst) -#define GetR(pos, tr) \ - do { \ - tr = Arg(pos); \ - switch (beam_reg_tag(tr)) { \ - case R_REG_DEF: tr = r(0); break; \ - case X_REG_DEF: tr = xb(x_reg_offset(tr)); break; \ - case Y_REG_DEF: ASSERT(y_reg_offset(tr) >= 1); tr = yb(y_reg_offset(tr)); break; \ - } \ - CHECK_TERM(tr); \ +#define GetR(pos, tr) \ + do { \ + tr = Arg(pos); \ + switch (loader_tag(tr)) { \ + case LOADER_X_REG: \ + tr = x(loader_x_reg_index(tr)); \ + break; \ + case LOADER_Y_REG: \ + ASSERT(loader_y_reg_index(tr) >= 1); \ + tr = y(loader_y_reg_index(tr)); \ + break; \ + } \ + CHECK_TERM(tr); \ } while (0) #define GetArg1(N, Dst) GetR((N), Dst) @@ -557,24 +548,93 @@ void** beam_ops; HTOP += 2; \ } while (0) +#define Swap(R1, R2) \ + do { \ + Eterm V = R1; \ + R1 = R2; \ + R2 = V; \ + } while (0) + +#define SwapTemp(R1, R2, Tmp) \ + do { \ + Eterm V = R1; \ + R1 = R2; \ + R2 = Tmp = V; \ + } while (0) + #define Move(Src, Dst, Store) \ do { \ Eterm term = (Src); \ Store(term, Dst); \ } while (0) -#define Move2(S1, D1, S2, D2) D1 = (S1); D2 = (S2) +#define Move2Par(S1, D1, S2, D2) \ + do { \ + Eterm V1, V2; \ + V1 = (S1); V2 = (S2); D1 = V1; D2 = V2; \ + } while (0) + +#define MoveShift(Src, SD, D) \ + do { \ + Eterm V; \ + V = Src; D = SD; SD = V; \ + } while (0) + +#define MoveDup(Src, D1, D2) \ + do { \ + D1 = D2 = (Src); \ + } while (0) + #define Move3(S1, D1, S2, D2, S3, D3) D1 = (S1); D2 = (S2); D3 = (S3) -#define MoveGenDest(src, dstp) \ - if ((dstp) == NULL) { r(0) = (src); } else { *(dstp) = src; } +#define MoveWindow3(S1, S2, S3, D) \ + do { \ + Eterm xt0, xt1, xt2; \ + Eterm *y = &D; \ + xt0 = S1; \ + xt1 = S2; \ + xt2 = S3; \ + y[0] = xt0; \ + y[1] = xt1; \ + y[2] = xt2; \ + } while (0) + +#define MoveWindow4(S1, S2, S3, S4, D) \ + do { \ + Eterm xt0, xt1, xt2, xt3; \ + Eterm *y = &D; \ + xt0 = S1; \ + xt1 = S2; \ + xt2 = S3; \ + xt3 = S4; \ + y[0] = xt0; \ + y[1] = xt1; \ + y[2] = xt2; \ + y[3] = xt3; \ + } while (0) + +#define MoveWindow5(S1, S2, S3, S4, S5, D) \ + do { \ + Eterm xt0, xt1, xt2, xt3, xt4; \ + Eterm *y = &D; \ + xt0 = S1; \ + xt1 = S2; \ + xt2 = S3; \ + xt3 = S4; \ + xt4 = S5; \ + y[0] = xt0; \ + y[1] = xt1; \ + y[2] = xt2; \ + y[3] = xt3; \ + y[4] = xt4; \ + } while (0) -#define MoveReturn(Src, Dest) \ - (Dest) = (Src); \ - I = c_p->cp; \ - ASSERT(VALID_INSTR(*c_p->cp)); \ - c_p->cp = 0; \ - CHECK_TERM(r(0)); \ +#define MoveReturn(Src) \ + x(0) = (Src); \ + I = c_p->cp; \ + ASSERT(VALID_INSTR(*c_p->cp)); \ + c_p->cp = 0; \ + CHECK_TERM(r(0)); \ Goto(*I) #define DeallocateReturn(Deallocate) \ @@ -586,26 +646,26 @@ void** beam_ops; Goto(*I); \ } while (0) -#define MoveDeallocateReturn(Src, Dest, Deallocate) \ - (Dest) = (Src); \ +#define MoveDeallocateReturn(Src, Deallocate) \ + x(0) = (Src); \ DeallocateReturn(Deallocate) -#define MoveCall(Src, Dest, CallDest, Size) \ - (Dest) = (Src); \ +#define MoveCall(Src, CallDest, Size) \ + x(0) = (Src); \ SET_CP(c_p, I+Size+1); \ - SET_I((BeamInstr *) CallDest); \ + SET_I((BeamInstr *) CallDest); \ Dispatch(); -#define MoveCallLast(Src, Dest, CallDest, Deallocate) \ - (Dest) = (Src); \ - RESTORE_CP(E); \ - E = ADD_BYTE_OFFSET(E, (Deallocate)); \ - SET_I((BeamInstr *) CallDest); \ +#define MoveCallLast(Src, CallDest, Deallocate) \ + x(0) = (Src); \ + RESTORE_CP(E); \ + E = ADD_BYTE_OFFSET(E, (Deallocate)); \ + SET_I((BeamInstr *) CallDest); \ Dispatch(); -#define MoveCallOnly(Src, Dest, CallDest) \ - (Dest) = (Src); \ - SET_I((BeamInstr *) CallDest); \ +#define MoveCallOnly(Src, CallDest) \ + x(0) = (Src); \ + SET_I((BeamInstr *) CallDest); \ Dispatch(); #define MoveJump(Src) \ @@ -613,60 +673,58 @@ void** beam_ops; SET_I((BeamInstr *) Arg(0)); \ Goto(*I); -#define GetList(Src, H, T) do { \ - Eterm* tmp_ptr = list_val(Src); \ - H = CAR(tmp_ptr); \ - T = CDR(tmp_ptr); } while (0) - -#define GetTupleElement(Src, Element, Dest) \ - do { \ - tmp_arg1 = (Eterm) COMPRESS_POINTER(((unsigned char *) tuple_val(Src)) + \ - (Element)); \ - (Dest) = (*(Eterm *) EXPAND_POINTER(tmp_arg1)); \ - } while (0) - -#define ExtractNextElement(Dest) \ - tmp_arg1 += sizeof(Eterm); \ - (Dest) = (* (Eterm *) (((unsigned char *) EXPAND_POINTER(tmp_arg1)))) - -#define ExtractNextElement2(Dest) \ - do { \ - Eterm* ene_dstp = &(Dest); \ - ene_dstp[0] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[1]; \ - ene_dstp[1] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[2]; \ - tmp_arg1 += sizeof(Eterm) + sizeof(Eterm); \ +#define GetList(Src, H, T) \ + do { \ + Eterm* tmp_ptr = list_val(Src); \ + Eterm hd, tl; \ + hd = CAR(tmp_ptr); \ + tl = CDR(tmp_ptr); \ + H = hd; T = tl; \ } while (0) -#define ExtractNextElement3(Dest) \ - do { \ - Eterm* ene_dstp = &(Dest); \ - ene_dstp[0] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[1]; \ - ene_dstp[1] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[2]; \ - ene_dstp[2] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[3]; \ - tmp_arg1 += 3*sizeof(Eterm); \ +#define GetTupleElement(Src, Element, Dest) \ + do { \ + Eterm* src; \ + src = ADD_BYTE_OFFSET(tuple_val(Src), (Element)); \ + (Dest) = *src; \ } while (0) -#define ExtractNextElement4(Dest) \ - do { \ - Eterm* ene_dstp = &(Dest); \ - ene_dstp[0] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[1]; \ - ene_dstp[1] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[2]; \ - ene_dstp[2] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[3]; \ - ene_dstp[3] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[4]; \ - tmp_arg1 += 4*sizeof(Eterm); \ +#define GetTupleElement2(Src, Element, Dest) \ + do { \ + Eterm* src; \ + Eterm* dst; \ + Eterm E1, E2; \ + src = ADD_BYTE_OFFSET(tuple_val(Src), (Element)); \ + dst = &(Dest); \ + E1 = src[0]; \ + E2 = src[1]; \ + dst[0] = E1; \ + dst[1] = E2; \ } while (0) -#define ExtractElement(Element, Dest) \ - do { \ - tmp_arg1 += (Element); \ - (Dest) = (* (Eterm *) EXPAND_POINTER(tmp_arg1)); \ +#define GetTupleElement3(Src, Element, Dest) \ + do { \ + Eterm* src; \ + Eterm* dst; \ + Eterm E1, E2, E3; \ + src = ADD_BYTE_OFFSET(tuple_val(Src), (Element)); \ + dst = &(Dest); \ + E1 = src[0]; \ + E2 = src[1]; \ + E3 = src[2]; \ + dst[0] = E1; \ + dst[1] = E2; \ + dst[2] = E3; \ } while (0) #define EqualImmed(X, Y, Action) if (X != Y) { Action; } #define NotEqualImmed(X, Y, Action) if (X == Y) { Action; } #define EqualExact(X, Y, Action) if (!EQ(X,Y)) { Action; } -#define IsLessThan(X, Y, Action) if (CMP_GE(X, Y)) { Action; } -#define IsGreaterEqual(X, Y, Action) if (CMP_LT(X, Y)) { Action; } +#define NotEqualExact(X, Y, Action) if (EQ(X,Y)) { Action; } +#define Equal(X, Y, Action) CMP_EQ_ACTION(X,Y,Action) +#define NotEqual(X, Y, Action) CMP_NE_ACTION(X,Y,Action) +#define IsLessThan(X, Y, Action) CMP_LT_ACTION(X,Y,Action) +#define IsGreaterEqual(X, Y, Action) CMP_GE_ACTION(X,Y,Action) #define IsFloat(Src, Fail) if (is_not_float(Src)) { Fail; } @@ -690,18 +748,26 @@ void** beam_ops; if (is_not_list(Src)) { Fail; } \ A(Need, Alive) -#define IsNonemptyListTestHeap(Src, Need, Alive, Fail) \ - if (is_not_list(Src)) { Fail; } \ +#define IsNonemptyListTestHeap(Need, Alive, Fail) \ + if (is_not_list(x(0))) { Fail; } \ TestHeap(Need, Alive) +#define IsNonemptyListGetList(Src, H, T, Fail) \ + if (is_not_list(Src)) { \ + Fail; \ + } else { \ + Eterm* tmp_ptr = list_val(Src); \ + Eterm hd, tl; \ + hd = CAR(tmp_ptr); \ + tl = CDR(tmp_ptr); \ + H = hd; T = tl; \ + } + #define IsTuple(X, Action) if (is_not_tuple(X)) Action -#define IsArity(Pointer, Arity, Fail) \ - if (*(Eterm *) \ - EXPAND_POINTER(tmp_arg1 = (Eterm) \ - COMPRESS_POINTER(tuple_val(Pointer))) != (Arity)) \ - { \ - Fail; \ +#define IsArity(Pointer, Arity, Fail) \ + if (*tuple_val(Pointer) != (Arity)) { \ + Fail; \ } #define IsMap(Src, Fail) if (!is_map(Src)) { Fail; } @@ -738,15 +804,21 @@ void** beam_ops; } \ } while (0) -#define IsTupleOfArity(Src, Arity, Fail) \ - do { \ - if (is_not_tuple(Src) || \ - *(Eterm *) \ - EXPAND_POINTER(tmp_arg1 = \ - (Eterm) COMPRESS_POINTER(tuple_val(Src))) != Arity) { \ - Fail; \ - } \ +#ifdef DEBUG +#define IsTupleOfArity(Src, Arityval, Fail) \ + do { \ + if (!(is_tuple(Src) && *tuple_val(Src) == Arityval)) { \ + Fail; \ + } \ } while (0) +#else +#define IsTupleOfArity(Src, Arityval, Fail) \ + do { \ + if (!(is_boxed(Src) && *tuple_val(Src) == Arityval)) { \ + Fail; \ + } \ + } while (0) +#endif #define IsBoolean(X, Fail) if ((X) != am_true && (X) != am_false) { Fail; } @@ -756,7 +828,7 @@ void** beam_ops; #define IsBitstring(Src, Fail) \ if (is_not_binary(Src)) { Fail; } -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) #define BsSafeMul(A, B, Fail, Target) \ do { Uint64 _res = (A) * (B); \ if (_res / B != A) { Fail; } \ @@ -773,6 +845,7 @@ void** beam_ops; #define BsGetFieldSize(Bits, Unit, Fail, Target) \ do { \ Sint _signed_size; Uint _uint_size; \ + Uint temp_bits; \ if (is_small(Bits)) { \ _signed_size = signed_val(Bits); \ if (_signed_size < 0) { Fail; } \ @@ -787,6 +860,7 @@ void** beam_ops; #define BsGetUncheckedFieldSize(Bits, Unit, Fail, Target) \ do { \ Sint _signed_size; Uint _uint_size; \ + Uint temp_bits; \ if (is_small(Bits)) { \ _signed_size = signed_val(Bits); \ if (_signed_size < 0) { Fail; } \ @@ -998,23 +1072,17 @@ init_emulator(void) */ #if defined(__GNUC__) && defined(sparc) && !defined(DEBUG) -# define REG_x0 asm("%l0") # define REG_xregs asm("%l1") # define REG_htop asm("%l2") # define REG_stop asm("%l3") # define REG_I asm("%l4") # define REG_fcalls asm("%l5") -# define REG_tmp_arg1 asm("%l6") -# define REG_tmp_arg2 asm("%l7") #else -# define REG_x0 # define REG_xregs # define REG_htop # define REG_stop # define REG_I # define REG_fcalls -# define REG_tmp_arg1 -# define REG_tmp_arg2 #endif #ifdef USE_VM_PROBES @@ -1132,11 +1200,6 @@ void process_main(void) ERTS_DECLARE_DUMMY(Eterm pid); #endif - /* - * X register zero; also called r(0) - */ - register Eterm x0 REG_x0 = NIL; - /* Pointer to X registers: x(1)..x(N); reg[0] is used when doing GC, * in all other cases x0 is used. */ @@ -1164,17 +1227,6 @@ void process_main(void) register Sint FCALLS REG_fcalls = 0; /* - * Temporaries used for picking up arguments for instructions. - */ - register Eterm tmp_arg1 REG_tmp_arg1 = NIL; - register Eterm tmp_arg2 REG_tmp_arg2 = NIL; -#if HEAP_ON_C_STACK - Eterm tmp_big[2]; /* Temporary buffer for small bignums if HEAP_ON_C_STACK. */ -#else - Eterm *tmp_big; /* Temporary buffer for small bignums if !HEAP_ON_C_STACK. */ -#endif - - /* * X registers and floating point registers are located in * scheduler specific data. */ @@ -1185,8 +1237,6 @@ void process_main(void) */ int neg_o_reds = 0; - Eterm (*arith_func)(Process* p, Eterm* reg, Uint live); - #ifdef ERTS_OPCODE_COUNTER_SUPPORT static void* counting_opcodes[] = { DEFINE_COUNTING_OPCODES }; #else @@ -1197,13 +1247,13 @@ void process_main(void) #endif #endif - Uint temp_bits; /* Temporary used by BsSkipBits2 & BsGetInteger2 */ - Eterm pt_arity; /* Used by do_put_tuple */ Uint64 start_time = 0; /* Monitor long schedule */ BeamInstr* start_time_i = NULL; + ERTS_MSACC_DECLARE_CACHE_X() /* a cached value of the tsd pointer for msacc */ + ERL_BITS_DECLARE_STATEP; /* Has to be last declaration */ @@ -1247,9 +1297,6 @@ void process_main(void) PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); -#if HALFWORD_HEAP - ASSERT(erts_get_scheduler_data()->num_tmp_heap_used == 0); -#endif ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); c_p = schedule(c_p, reds_used); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); @@ -1260,6 +1307,8 @@ void process_main(void) ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); + ERTS_MSACC_UPDATE_CACHE_X(); + if (erts_system_monitor_long_schedule != 0) { start_time = erts_timestamp_millis(); start_time_i = c_p->i; @@ -1267,9 +1316,6 @@ void process_main(void) reg = ERTS_PROC_GET_SCHDATA(c_p)->x_reg_array; freg = ERTS_PROC_GET_SCHDATA(c_p)->f_reg_array; -#if !HEAP_ON_C_STACK - tmp_big = ERTS_PROC_GET_SCHDATA(c_p)->beam_emu_tmp_heap; -#endif ERL_BITS_RELOAD_STATEP(c_p); { int reds; @@ -1278,7 +1324,7 @@ void process_main(void) int i; argp = c_p->arg_reg; - for (i = c_p->arity - 1; i > 0; i--) { + for (i = c_p->arity - 1; i >= 0; i--) { reg[i] = argp[i]; CHECK_TERM(reg[i]); } @@ -1302,12 +1348,6 @@ void process_main(void) } next = (BeamInstr *) *I; - r(0) = c_p->arg_reg[0]; -#ifdef HARDDEBUG - if (c_p->arity > 0) { - CHECK_TERM(r(0)); - } -#endif SWAPIN; ASSERT(VALID_INSTR(next)); @@ -1346,26 +1386,24 @@ void process_main(void) #endif #include "beam_hot.h" -#define STORE_ARITH_RESULT(res) StoreBifResult(2, (res)); -#define ARITH_FUNC(name) erts_gc_##name - { Eterm increment_reg_val; Eterm increment_val; Uint live; Eterm result; - OpCase(i_increment_yIId): - increment_reg_val = yb(Arg(0)); + OpCase(i_increment_rIId): + increment_reg_val = x(0); + I--; goto do_increment; OpCase(i_increment_xIId): increment_reg_val = xb(Arg(0)); goto do_increment; - OpCase(i_increment_rIId): - increment_reg_val = r(0); - I--; + OpCase(i_increment_yIId): + increment_reg_val = yb(Arg(0)); + goto do_increment; do_increment: increment_val = Arg(1); @@ -1374,229 +1412,133 @@ void process_main(void) ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i)); if (MY_IS_SSMALL(i)) { result = make_small(i); - store_result: StoreBifResult(3, result); } } live = Arg(2); - SWAPOUT; - reg[0] = r(0); + HEAVY_SWAPOUT; reg[live] = increment_reg_val; reg[live+1] = make_small(increment_val); result = erts_gc_mixed_plus(c_p, reg, live); - r(0) = reg[0]; - SWAPIN; + HEAVY_SWAPIN; ERTS_HOLE_CHECK(c_p); if (is_value(result)) { - goto store_result; + StoreBifResult(3, result); } ASSERT(c_p->freason != BADMATCH || is_value(c_p->fvalue)); goto find_func_info; } -#define DO_BIG_ARITH(Func,Arg1,Arg2) \ - do { \ - Uint live = Arg(1); \ - SWAPOUT; \ - reg[0] = r(0); \ - reg[live] = (Arg1); \ - reg[live+1] = (Arg2); \ - result = (Func)(c_p, reg, live); \ - r(0) = reg[0]; \ - SWAPIN; \ - ERTS_HOLE_CHECK(c_p); \ - if (is_value(result)) { \ - StoreBifResult(4,result); \ - } \ - goto lb_Cl_error; \ - } while(0) +#define DO_OUTLINED_ARITH_2(name, Op1, Op2) \ + do { \ + Eterm result; \ + Uint live = Arg(1); \ + \ + HEAVY_SWAPOUT; \ + reg[live] = Op1; \ + reg[live+1] = Op2; \ + result = erts_gc_##name(c_p, reg, live); \ + HEAVY_SWAPIN; \ + ERTS_HOLE_CHECK(c_p); \ + if (is_value(result)) { \ + StoreBifResult(4, result); \ + } \ + goto lb_Cl_error; \ + } while (0) - OpCase(i_plus_jIxxd): { + Eterm PlusOp1, PlusOp2; Eterm result; - if (is_both_small(xb(Arg(2)), xb(Arg(3)))) { - Sint i = signed_val(xb(Arg(2))) + signed_val(xb(Arg(3))); + OpCase(i_plus_jIxxd): + PlusOp1 = xb(Arg(2)); + PlusOp2 = xb(Arg(3)); + goto do_plus; + + OpCase(i_plus_jIxyd): + PlusOp1 = xb(Arg(2)); + PlusOp2 = yb(Arg(3)); + goto do_plus; + + OpCase(i_plus_jIssd): + GetArg2(2, PlusOp1, PlusOp2); + goto do_plus; + + do_plus: + if (is_both_small(PlusOp1, PlusOp2)) { + Sint i = signed_val(PlusOp1) + signed_val(PlusOp2); ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i)); if (MY_IS_SSMALL(i)) { result = make_small(i); StoreBifResult(4, result); } } - DO_BIG_ARITH(ARITH_FUNC(mixed_plus), xb(Arg(2)), xb(Arg(3))); + DO_OUTLINED_ARITH_2(mixed_plus, PlusOp1, PlusOp2); } - OpCase(i_plus_jId): { + Eterm MinusOp1, MinusOp2; Eterm result; - if (is_both_small(tmp_arg1, tmp_arg2)) { - Sint i = signed_val(tmp_arg1) + signed_val(tmp_arg2); - ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i)); - if (MY_IS_SSMALL(i)) { - result = make_small(i); - STORE_ARITH_RESULT(result); - } - } - arith_func = ARITH_FUNC(mixed_plus); - goto do_big_arith2; - } - OpCase(i_minus_jIxxd): - { - Eterm result; + MinusOp1 = xb(Arg(2)); + MinusOp2 = xb(Arg(3)); + goto do_minus; - if (is_both_small(xb(Arg(2)), xb(Arg(3)))) { - Sint i = signed_val(xb(Arg(2))) - signed_val(xb(Arg(3))); - ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i)); - if (MY_IS_SSMALL(i)) { - result = make_small(i); - StoreBifResult(4, result); - } - } - DO_BIG_ARITH(ARITH_FUNC(mixed_minus), xb(Arg(2)), xb(Arg(3))); - } + OpCase(i_minus_jIssd): + GetArg2(2, MinusOp1, MinusOp2); + goto do_minus; - OpCase(i_minus_jId): - { - Eterm result; - - if (is_both_small(tmp_arg1, tmp_arg2)) { - Sint i = signed_val(tmp_arg1) - signed_val(tmp_arg2); + do_minus: + if (is_both_small(MinusOp1, MinusOp2)) { + Sint i = signed_val(MinusOp1) - signed_val(MinusOp2); ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i)); if (MY_IS_SSMALL(i)) { result = make_small(i); - STORE_ARITH_RESULT(result); + StoreBifResult(4, result); } } - arith_func = ARITH_FUNC(mixed_minus); - goto do_big_arith2; + DO_OUTLINED_ARITH_2(mixed_minus, MinusOp1, MinusOp2); } - OpCase(i_is_lt_f): - if (CMP_GE(tmp_arg1, tmp_arg2)) { - ClauseFail(); - } - Next(1); - - OpCase(i_is_ge_f): - if (CMP_LT(tmp_arg1, tmp_arg2)) { - ClauseFail(); - } - Next(1); - - OpCase(i_is_eq_f): - if (CMP_NE(tmp_arg1, tmp_arg2)) { - ClauseFail(); - } - Next(1); - - OpCase(i_is_ne_f): - if (CMP_EQ(tmp_arg1, tmp_arg2)) { - ClauseFail(); - } - Next(1); - - OpCase(i_is_eq_exact_f): - if (!EQ(tmp_arg1, tmp_arg2)) { - ClauseFail(); - } - Next(1); - { Eterm is_eq_exact_lit_val; - OpCase(i_is_eq_exact_literal_xfc): - is_eq_exact_lit_val = xb(Arg(0)); - I++; + OpCase(i_is_eq_exact_literal_fxc): + is_eq_exact_lit_val = xb(Arg(1)); goto do_is_eq_exact_literal; - OpCase(i_is_eq_exact_literal_yfc): - is_eq_exact_lit_val = yb(Arg(0)); - I++; + OpCase(i_is_eq_exact_literal_fyc): + is_eq_exact_lit_val = yb(Arg(1)); goto do_is_eq_exact_literal; - OpCase(i_is_eq_exact_literal_rfc): - is_eq_exact_lit_val = r(0); - do_is_eq_exact_literal: - if (!eq(Arg(1), is_eq_exact_lit_val)) { + if (!eq(Arg(2), is_eq_exact_lit_val)) { ClauseFail(); } - Next(2); + Next(3); } { Eterm is_ne_exact_lit_val; - OpCase(i_is_ne_exact_literal_xfc): - is_ne_exact_lit_val = xb(Arg(0)); - I++; + OpCase(i_is_ne_exact_literal_fxc): + is_ne_exact_lit_val = xb(Arg(1)); goto do_is_ne_exact_literal; - OpCase(i_is_ne_exact_literal_yfc): - is_ne_exact_lit_val = yb(Arg(0)); - I++; + OpCase(i_is_ne_exact_literal_fyc): + is_ne_exact_lit_val = yb(Arg(1)); goto do_is_ne_exact_literal; - OpCase(i_is_ne_exact_literal_rfc): - is_ne_exact_lit_val = r(0); - do_is_ne_exact_literal: - if (eq(Arg(1), is_ne_exact_lit_val)) { + if (eq(Arg(2), is_ne_exact_lit_val)) { ClauseFail(); } - Next(2); + Next(3); } - OpCase(move_window3_xxxy): { - BeamInstr *next; - Eterm xt0, xt1, xt2; - Eterm *y = (Eterm *)(((unsigned char *)E) + (Arg(3))); - PreFetch(4, next); - xt0 = xb(Arg(0)); - xt1 = xb(Arg(1)); - xt2 = xb(Arg(2)); - y[0] = xt0; - y[1] = xt1; - y[2] = xt2; - NextPF(4, next); - } - OpCase(move_window4_xxxxy): { - BeamInstr *next; - Eterm xt0, xt1, xt2, xt3; - Eterm *y = (Eterm *)(((unsigned char *)E) + (Arg(4))); - PreFetch(5, next); - xt0 = xb(Arg(0)); - xt1 = xb(Arg(1)); - xt2 = xb(Arg(2)); - xt3 = xb(Arg(3)); - y[0] = xt0; - y[1] = xt1; - y[2] = xt2; - y[3] = xt3; - NextPF(5, next); - } - OpCase(move_window5_xxxxxy): { - BeamInstr *next; - Eterm xt0, xt1, xt2, xt3, xt4; - Eterm *y = (Eterm *)(((unsigned char *)E) + (Arg(5))); - PreFetch(6, next); - xt0 = xb(Arg(0)); - xt1 = xb(Arg(1)); - xt2 = xb(Arg(2)); - xt3 = xb(Arg(3)); - xt4 = xb(Arg(4)); - y[0] = xt0; - y[1] = xt1; - y[2] = xt2; - y[3] = xt3; - y[4] = xt4; - NextPF(6, next); - } - - OpCase(i_move_call_only_fcr): { + OpCase(i_move_call_only_fc): { r(0) = Arg(1); } /* FALL THROUGH */ @@ -1606,7 +1548,7 @@ void process_main(void) Dispatch(); } - OpCase(i_move_call_last_fPcr): { + OpCase(i_move_call_last_fPc): { r(0) = Arg(2); } /* FALL THROUGH */ @@ -1618,7 +1560,7 @@ void process_main(void) Dispatch(); } - OpCase(i_move_call_crf): { + OpCase(i_move_call_cf): { r(0) = Arg(0); I++; } @@ -1630,7 +1572,7 @@ void process_main(void) Dispatch(); } - OpCase(i_move_call_ext_last_ePcr): { + OpCase(i_move_call_ext_last_ePc): { r(0) = Arg(2); } /* FALL THROUGH */ @@ -1646,7 +1588,7 @@ void process_main(void) DTRACE_GLOBAL_CALL_FROM_EXPORT(c_p, Arg(0)); Dispatchx(); - OpCase(i_move_call_ext_cre): { + OpCase(i_move_call_ext_ce): { r(0) = Arg(0); I++; } @@ -1656,7 +1598,7 @@ void process_main(void) DTRACE_GLOBAL_CALL_FROM_EXPORT(c_p, Arg(0)); Dispatchx(); - OpCase(i_move_call_ext_only_ecr): { + OpCase(i_move_call_ext_only_ec): { r(0) = Arg(1); } /* FALL THROUGH */ @@ -1722,17 +1664,11 @@ void process_main(void) PRE_BIF_SWAPOUT(c_p); c_p->fcalls = FCALLS - 1; - reg[0] = r(0); result = erl_send(c_p, r(0), x(1)); PreFetch(0, next); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); - if (c_p->mbuf || MSO(c_p).overhead >= BIN_VHEAP_SZ(c_p)) { - result = erts_gc_after_bif_call(c_p, result, reg, 2); - r(0) = reg[0]; - E = c_p->stop; - } HTOP = HEAP_TOP(c_p); FCALLS = c_p->fcalls; if (is_value(result)) { @@ -1743,7 +1679,6 @@ void process_main(void) SET_CP(c_p, I+1); SET_I(c_p->i); SWAPIN; - r(0) = reg[0]; Dispatch(); } goto find_func_info; @@ -1753,29 +1688,23 @@ void process_main(void) Eterm element_index; Eterm element_tuple; - OpCase(i_element_xjsd): - element_tuple = xb(Arg(0)); - I++; + OpCase(i_element_jxsd): + element_tuple = xb(Arg(1)); goto do_element; - OpCase(i_element_yjsd): - element_tuple = yb(Arg(0)); - I++; + OpCase(i_element_jysd): + element_tuple = yb(Arg(1)); goto do_element; - OpCase(i_element_rjsd): - element_tuple = r(0); - /* Fall through */ - do_element: - GetArg1(1, element_index); + GetArg1(2, element_index); if (is_small(element_index) && is_tuple(element_tuple)) { Eterm* tp = tuple_val(element_tuple); if ((signed_val(element_index) >= 1) && (signed_val(element_index) <= arityval(*tp))) { Eterm result = tp[signed_val(element_index)]; - StoreBifResult(2, result); + StoreBifResult(3, result); } } } @@ -1789,29 +1718,24 @@ void process_main(void) { Eterm fast_element_tuple; - OpCase(i_fast_element_rjId): - fast_element_tuple = r(0); + OpCase(i_fast_element_jxId): + fast_element_tuple = xb(Arg(1)); + goto do_fast_element; + + OpCase(i_fast_element_jyId): + fast_element_tuple = yb(Arg(1)); + goto do_fast_element; do_fast_element: if (is_tuple(fast_element_tuple)) { Eterm* tp = tuple_val(fast_element_tuple); - Eterm pos = Arg(1); /* Untagged integer >= 1 */ + Eterm pos = Arg(2); /* Untagged integer >= 1 */ if (pos <= arityval(*tp)) { Eterm result = tp[pos]; - StoreBifResult(2, result); + StoreBifResult(3, result); } } goto badarg; - - OpCase(i_fast_element_xjId): - fast_element_tuple = xb(Arg(0)); - I++; - goto do_fast_element; - - OpCase(i_fast_element_yjId): - fast_element_tuple = yb(Arg(0)); - I++; - goto do_fast_element; } OpCase(catch_yf): @@ -1832,11 +1756,10 @@ void process_main(void) SWAPIN; } /* only x(2) is included in the rootset here */ - if (E - HTOP < 3 || c_p->mbuf) { /* Force GC in case add_stacktrace() - * created heap fragments */ + if (E - HTOP < 3) { SWAPOUT; PROCESS_MAIN_CHK_LOCKS(c_p); - FCALLS -= erts_garbage_collect(c_p, 3, reg+2, 1); + FCALLS -= erts_garbage_collect_nobump(c_p, 3, reg+2, 1); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); SWAPIN; @@ -1917,13 +1840,20 @@ void process_main(void) * Pick up the next message and place it in x(0). * If no message, jump to a wait or wait_timeout instruction. */ - OpCase(i_loop_rec_fr): + OpCase(i_loop_rec_f): { BeamInstr *next; - ErlMessage* msgp; + ErtsMessage* msgp; - loop_rec__: + /* + * We need to disable GC while matching messages + * in the queue. This since messages with data outside + * the heap will be corrupted by a GC. + */ + ASSERT(!(c_p->flags & F_DELAY_GC)); + c_p->flags |= F_DELAY_GC; + loop_rec__: PROCESS_MAIN_CHK_LOCKS(c_p); msgp = PEEK_MESSAGE(c_p); @@ -1935,6 +1865,7 @@ void process_main(void) if (ERTS_PROC_PENDING_EXIT(c_p)) { erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); SWAPOUT; + c_p->flags &= ~F_DELAY_GC; goto do_schedule; /* Will be rescheduled for exit */ } ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p); @@ -1944,32 +1875,27 @@ void process_main(void) else #endif { + c_p->flags &= ~F_DELAY_GC; SET_I((BeamInstr *) Arg(0)); Goto(*I); /* Jump to a wait or wait_timeout instruction */ } } - ErtsMoveMsgAttachmentIntoProc(msgp, c_p, E, HTOP, FCALLS, - { - SWAPOUT; - reg[0] = r(0); - PROCESS_MAIN_CHK_LOCKS(c_p); - }, - { - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - PROCESS_MAIN_CHK_LOCKS(c_p); - r(0) = reg[0]; - SWAPIN; - }); if (is_non_value(ERL_MESSAGE_TERM(msgp))) { - /* - * A corrupt distribution message that we weren't able to decode; - * remove it... - */ - ASSERT(!msgp->data.attached); - /* TODO: Add DTrace probe for this bad message situation? */ - UNLINK_MESSAGE(c_p, msgp); - free_message(msgp); - goto loop_rec__; + SWAPOUT; /* erts_decode_dist_message() may write to heap... */ + if (!erts_decode_dist_message(c_p, ERTS_PROC_LOCK_MAIN, msgp, 0)) { + /* + * A corrupt distribution message that we weren't able to decode; + * remove it... + */ + /* No swapin should be needed */ + ASSERT(HTOP == c_p->htop && E == c_p->stop); + /* TODO: Add DTrace probe for this bad message situation? */ + UNLINK_MESSAGE(c_p, msgp); + msgp->next = NULL; + erts_cleanup_messages(msgp); + goto loop_rec__; + } + SWAPIN; } PreFetch(1, next); r(0) = ERL_MESSAGE_TERM(msgp); @@ -1981,8 +1907,7 @@ void process_main(void) */ OpCase(remove_message): { BeamInstr *next; - ErlMessage* msgp; - + ErtsMessage* msgp; PROCESS_MAIN_CHK_LOCKS(c_p); PreFetch(0, next); @@ -1996,20 +1921,7 @@ void process_main(void) if (DT_UTAG(c_p) != NIL) { if (DT_UTAG_FLAGS(c_p) & DT_UTAG_PERMANENT) { SEQ_TRACE_TOKEN(c_p) = am_have_dt_utag; -#ifdef DTRACE_TAG_HARDDEBUG - if (DT_UTAG_FLAGS(c_p) & DT_UTAG_SPREADING) - erts_fprintf(stderr, - "Dtrace -> (%T) stop spreading " - "tag %T with message %T\r\n", - c_p->common.id,DT_UTAG(c_p),ERL_MESSAGE_TERM(msgp)); -#endif } else { -#ifdef DTRACE_TAG_HARDDEBUG - erts_fprintf(stderr, - "Dtrace -> (%T) kill tag %T with " - "message %T\r\n", - c_p->common.id,DT_UTAG(c_p),ERL_MESSAGE_TERM(msgp)); -#endif DT_UTAG(c_p) = NIL; SEQ_TRACE_TOKEN(c_p) = NIL; } @@ -2029,12 +1941,6 @@ void process_main(void) DT_UTAG(c_p) = ERL_MESSAGE_DT_UTAG(msgp); } DT_UTAG_FLAGS(c_p) |= DT_UTAG_SPREADING; -#ifdef DTRACE_TAG_HARDDEBUG - erts_fprintf(stderr, - "Dtrace -> (%T) receive tag (%T) " - "with message %T\r\n", - c_p->common.id, DT_UTAG(c_p), ERL_MESSAGE_TERM(msgp)); -#endif } else { #endif ASSERT(is_tuple(SEQ_TRACE_TOKEN(c_p))); @@ -2064,7 +1970,7 @@ void process_main(void) dtrace_proc_str(c_p, receiver_name); token2 = SEQ_TRACE_TOKEN(c_p); - if (token2 != NIL && token2 != am_have_dt_utag) { + if (have_seqtrace(token2)) { tok_label = signed_val(SEQ_TRACE_T_LABEL(token2)); tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token2)); tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token2)); @@ -2077,11 +1983,21 @@ void process_main(void) UNLINK_MESSAGE(c_p, msgp); JOIN_MESSAGE(c_p); CANCEL_TIMER(c_p); - free_message(msgp); + + erts_save_message_in_proc(c_p, msgp); + c_p->flags &= ~F_DELAY_GC; + + if (ERTS_IS_GC_DESIRED_INTERNAL(c_p, HTOP, E)) { + /* + * We want to GC soon but we leave a few + * reductions giving the message some time + * to turn into garbage. + */ + ERTS_VBUMP_LEAVE_REDS_INTERNAL(c_p, 5, FCALLS); + } ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); - NextPF(0, next); } @@ -2090,9 +2006,22 @@ void process_main(void) * message didn't match), then jump to the loop_rec instruction. */ OpCase(loop_rec_end_f): { + + ASSERT(c_p->flags & F_DELAY_GC); + SET_I((BeamInstr *) Arg(0)); SAVE_MESSAGE(c_p); - goto loop_rec__; + if (FCALLS > 0 || FCALLS > neg_o_reds) { + FCALLS--; + goto loop_rec__; + } + + c_p->flags &= ~F_DELAY_GC; + c_p->i = I; + SWAPOUT; + c_p->arity = 0; + c_p->current = NULL; + goto do_schedule; } /* * Prepare to wait for a message or a timeout, whichever occurs first. @@ -2131,8 +2060,6 @@ void process_main(void) * c_p->def_arg_reg[0]. Note that it is safe to use this * location because there are no living x registers in * a receive statement. - * Note that for the halfword emulator, the two first elements - * of the array are used. */ BeamInstr** pi = (BeamInstr**) c_p->def_arg_reg; *pi = I+3; @@ -2249,10 +2176,6 @@ void process_main(void) select_val2 = xb(Arg(0)); goto do_select_tuple_arity2; - OpCase(i_select_tuple_arity2_rfAAff): - select_val2 = r(0); - I--; - do_select_tuple_arity2: if (is_not_tuple(select_val2)) { goto select_val2_fail; @@ -2268,10 +2191,6 @@ void process_main(void) select_val2 = xb(Arg(0)); goto do_select_val2; - OpCase(i_select_val2_rfccff): - select_val2 = r(0); - I--; - do_select_val2: if (select_val2 == Arg(2)) { I += 3; @@ -2295,10 +2214,6 @@ void process_main(void) select_val = yb(Arg(0)); goto do_select_tuple_arity; - OpCase(i_select_tuple_arity_rfI): - select_val = r(0); - I--; - do_select_tuple_arity: if (is_tuple(select_val)) { select_val = *tuple_val(select_val); @@ -2315,10 +2230,6 @@ void process_main(void) select_val = yb(Arg(0)); goto do_linear_search; - OpCase(i_select_val_lins_rfI): - select_val = r(0); - I--; - do_linear_search: { BeamInstr *vs = &Arg(3); int ix = 0; @@ -2345,10 +2256,6 @@ void process_main(void) select_val = yb(Arg(0)); goto do_binary_search; - OpCase(i_select_val_bins_rfI): - select_val = r(0); - I--; - do_binary_search: { struct Pairs { @@ -2409,10 +2316,6 @@ void process_main(void) jump_on_val_zero_index = xb(Arg(0)); goto do_jump_on_val_zero_index; - OpCase(i_jump_on_val_zero_rfI): - jump_on_val_zero_index = r(0); - I--; - do_jump_on_val_zero_index: if (is_small(jump_on_val_zero_index)) { jump_on_val_zero_index = signed_val(jump_on_val_zero_index); @@ -2437,10 +2340,6 @@ void process_main(void) jump_on_val_index = xb(Arg(0)); goto do_jump_on_val_index; - OpCase(i_jump_on_val_rfII): - jump_on_val_index = r(0); - I--; - do_jump_on_val_index: if (is_small(jump_on_val_index)) { jump_on_val_index = (Uint) (signed_val(jump_on_val_index) - Arg(3)); @@ -2460,15 +2359,12 @@ void process_main(void) do { Eterm term = *I++; - switch (term & _TAG_IMMED1_MASK) { - case (R_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER: - *hp++ = r(0); - break; - case (X_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER: - *hp++ = x(term >> _TAG_IMMED1_SIZE); + switch (loader_tag(term)) { + case LOADER_X_REG: + *hp++ = x(loader_x_reg_index(term)); break; - case (Y_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER: - *hp++ = y(term >> _TAG_IMMED1_SIZE); + case LOADER_Y_REG: + *hp++ = y(loader_y_reg_index(term)); break; default: *hp++ = term; @@ -2482,31 +2378,26 @@ void process_main(void) OpCase(new_map_dII): { Eterm res; - x(0) = r(0); - SWAPOUT; + HEAVY_SWAPOUT; res = new_map(c_p, reg, I-1); - SWAPIN; - r(0) = x(0); + HEAVY_SWAPIN; StoreResult(res, Arg(0)); Next(3+Arg(2)); } -#define PUT_TERM_REG(term, desc) \ -do { \ - switch ((desc) & _TAG_IMMED1_MASK) { \ - case (R_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER: \ - r(0) = (term); \ - break; \ - case (X_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER: \ - x((desc) >> _TAG_IMMED1_SIZE) = (term); \ - break; \ - case (Y_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER: \ - y((desc) >> _TAG_IMMED1_SIZE) = (term); \ - break; \ - default: \ - ASSERT(0); \ - break; \ - } \ +#define PUT_TERM_REG(term, desc) \ +do { \ + switch (loader_tag(desc)) { \ + case LOADER_X_REG: \ + x(loader_x_reg_index(desc)) = (term); \ + break; \ + case LOADER_Y_REG: \ + y(loader_y_reg_index(desc)) = (term); \ + break; \ + default: \ + ASSERT(0); \ + break; \ + } \ } while(0) OpCase(i_get_map_elements_fsI): { @@ -2577,12 +2468,10 @@ do { \ Eterm map; GetArg1(1, map); - x(0) = r(0); - SWAPOUT; + HEAVY_SWAPOUT; res = update_map_assoc(c_p, reg, map, I); - SWAPIN; + HEAVY_SWAPIN; if (is_value(res)) { - r(0) = x(0); StoreResult(res, Arg(2)); Next(5+Arg(4)); } else { @@ -2601,12 +2490,10 @@ do { \ Eterm map; GetArg1(1, map); - x(0) = r(0); - SWAPOUT; + HEAVY_SWAPOUT; res = update_map_exact(c_p, reg, map, I); - SWAPIN; + HEAVY_SWAPIN; if (is_value(res)) { - r(0) = x(0); StoreResult(res, Arg(2)); Next(5+Arg(4)); } else { @@ -2688,13 +2575,10 @@ do { \ { typedef Eterm (*GcBifFunction)(Process*, Eterm*, Uint); GcBifFunction bf; - Eterm arg; Eterm result; Uint live = (Uint) Arg(3); - GetArg1(2, arg); - reg[0] = r(0); - reg[live] = arg; + GetArg1(2, x(live)); bf = (GcBifFunction) Arg(1); c_p->fcalls = FCALLS; SWAPOUT; @@ -2705,7 +2589,6 @@ do { \ ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); SWAPIN; - r(0) = reg[0]; ERTS_HOLE_CHECK(c_p); FCALLS = c_p->fcalls; if (is_value(result)) { @@ -2715,12 +2598,12 @@ do { \ SET_I((BeamInstr *) Arg(0)); Goto(*I); } - reg[0] = arg; + x(0) = x(live); I = handle_error(c_p, I, reg, translate_gc_bif((void *) bf)); goto post_error_handling; } - OpCase(i_gc_bif2_jIId): /* Note, one less parameter than the i_gc_bif1 + OpCase(i_gc_bif2_jIIssd): /* Note, one less parameter than the i_gc_bif1 and i_gc_bif3 */ { typedef Eterm (*GcBifFunction)(Process*, Eterm*, Uint); @@ -2728,9 +2611,13 @@ do { \ Eterm result; Uint live = (Uint) Arg(2); - reg[0] = r(0); - reg[live++] = tmp_arg1; - reg[live] = tmp_arg2; + GetArg2(3, x(live), x(live+1)); + /* + * XXX This calling convention does not make sense. 'live' + * should point out the first argument, not the second + * (i.e. 'live' should not be incremented below). + */ + live++; bf = (GcBifFunction) Arg(1); c_p->fcalls = FCALLS; SWAPOUT; @@ -2741,35 +2628,37 @@ do { \ ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); SWAPIN; - r(0) = reg[0]; ERTS_HOLE_CHECK(c_p); FCALLS = c_p->fcalls; if (is_value(result)) { - StoreBifResult(3, result); + StoreBifResult(5, result); } if (Arg(0) != 0) { SET_I((BeamInstr *) Arg(0)); Goto(*I); } - reg[0] = tmp_arg1; - reg[1] = tmp_arg2; + live--; + x(0) = x(live); + x(1) = x(live+1); I = handle_error(c_p, I, reg, translate_gc_bif((void *) bf)); goto post_error_handling; } - OpCase(i_gc_bif3_jIsId): + OpCase(i_gc_bif3_jIIssd): { typedef Eterm (*GcBifFunction)(Process*, Eterm*, Uint); GcBifFunction bf; - Eterm arg; Eterm result; - Uint live = (Uint) Arg(3); + Uint live = (Uint) Arg(2); - GetArg1(2, arg); - reg[0] = r(0); - reg[live++] = arg; - reg[live++] = tmp_arg1; - reg[live] = tmp_arg2; + x(live) = x(SCRATCH_X_REG); + GetArg2(3, x(live+1), x(live+2)); + /* + * XXX This calling convention does not make sense. 'live' + * should point out the first argument, not the third + * (i.e. 'live' should not be incremented below). + */ + live += 2; bf = (GcBifFunction) Arg(1); c_p->fcalls = FCALLS; SWAPOUT; @@ -2780,19 +2669,19 @@ do { \ ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); SWAPIN; - r(0) = reg[0]; ERTS_HOLE_CHECK(c_p); FCALLS = c_p->fcalls; if (is_value(result)) { - StoreBifResult(4, result); + StoreBifResult(5, result); } if (Arg(0) != 0) { SET_I((BeamInstr *) Arg(0)); Goto(*I); } - reg[0] = arg; - reg[1] = tmp_arg1; - reg[2] = tmp_arg2; + live -= 2; + x(0) = x(live); + x(1) = x(live+1); + x(2) = x(live+2); I = handle_error(c_p, I, reg, translate_gc_bif((void *) bf)); goto post_error_handling; } @@ -2800,12 +2689,13 @@ do { \ /* * Guards bifs and, or, xor in guards. */ - OpCase(i_bif2_fbd): + OpCase(i_bif2_fbssd): { - Eterm tmp_reg[2] = {tmp_arg1, tmp_arg2}; + Eterm tmp_reg[2]; Eterm (*bf)(Process*, Eterm*); Eterm result; + GetArg2(2, tmp_reg[0], tmp_reg[1]); bf = (BifFunction) Arg(1); c_p->fcalls = FCALLS; PROCESS_MAIN_CHK_LOCKS(c_p); @@ -2817,7 +2707,7 @@ do { \ ERTS_HOLE_CHECK(c_p); FCALLS = c_p->fcalls; if (is_value(result)) { - StoreBifResult(2, result); + StoreBifResult(4, result); } SET_I((BeamInstr *) Arg(0)); Goto(*I); @@ -2826,12 +2716,13 @@ do { \ /* * Guards bifs and, or, xor, relational operators in body. */ - OpCase(i_bif2_body_bd): + OpCase(i_bif2_body_bssd): { - Eterm tmp_reg[2] = {tmp_arg1, tmp_arg2}; + Eterm tmp_reg[2]; Eterm (*bf)(Process*, Eterm*); Eterm result; + GetArg2(1, tmp_reg[0], tmp_reg[1]); bf = (BifFunction) Arg(0); PROCESS_MAIN_CHK_LOCKS(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); @@ -2842,10 +2733,10 @@ do { \ ERTS_HOLE_CHECK(c_p); if (is_value(result)) { ASSERT(!is_CP(result)); - StoreBifResult(1, result); + StoreBifResult(3, result); } - reg[0] = tmp_arg1; - reg[1] = tmp_arg2; + reg[0] = tmp_reg[0]; + reg[1] = tmp_reg[1]; SWAPOUT; I = handle_error(c_p, I, reg, bf); goto post_error_handling; @@ -2857,9 +2748,20 @@ do { \ */ OpCase(call_bif_e): { - Eterm (*bf)(Process*, Eterm*, BeamInstr*) = GET_BIF_ADDRESS(Arg(0)); + Eterm (*bf)(Process*, Eterm*, BeamInstr*); Eterm result; BeamInstr *next; + ErlHeapFragment *live_hf_end; + + if (ERTS_MSACC_IS_ENABLED_CACHED_X()) { + if (GET_BIF_MODULE(Arg(0)) == am_ets) { + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_ETS); + } else { + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_BIF); + } + } + + bf = GET_BIF_ADDRESS(Arg(0)); PRE_BIF_SWAPOUT(c_p); c_p->fcalls = FCALLS - 1; @@ -2869,20 +2771,26 @@ do { \ PreFetch(1, next); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - reg[0] = r(0); + live_hf_end = c_p->mbuf; result = (*bf)(c_p, reg, I); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); ERTS_HOLE_CHECK(c_p); ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); - PROCESS_MAIN_CHK_LOCKS(c_p); - if (c_p->mbuf || MSO(c_p).overhead >= BIN_VHEAP_SZ(c_p)) { + if (ERTS_IS_GC_DESIRED(c_p)) { Uint arity = ((Export *)Arg(0))->code[2]; - result = erts_gc_after_bif_call(c_p, result, reg, arity); + result = erts_gc_after_bif_call_lhf(c_p, live_hf_end, result, reg, arity); E = c_p->stop; } + PROCESS_MAIN_CHK_LOCKS(c_p); HTOP = HEAP_TOP(c_p); FCALLS = c_p->fcalls; + /* We have to update the cache if we are enabled in order + to make sure no book keeping is done after we disabled + msacc. We don't always do this as it is quite expensive. */ + if (ERTS_MSACC_IS_ENABLED_CACHED_X()) + ERTS_MSACC_UPDATE_CACHE_X(); + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR); if (is_value(result)) { r(0) = result; CHECK_TERM(r(0)); @@ -2891,7 +2799,6 @@ do { \ SET_CP(c_p, I+2); SET_I(c_p->i); SWAPIN; - r(0) = reg[0]; Dispatch(); } @@ -2907,111 +2814,81 @@ do { \ * Arithmetic operations. */ - OpCase(i_times_jId): + OpCase(i_times_jIssd): { - arith_func = ARITH_FUNC(mixed_times); - goto do_big_arith2; + Eterm Op1, Op2; + GetArg2(2, Op1, Op2); + DO_OUTLINED_ARITH_2(mixed_times, Op1, Op2); } - OpCase(i_m_div_jId): + OpCase(i_m_div_jIssd): { - arith_func = ARITH_FUNC(mixed_div); - goto do_big_arith2; + Eterm Op1, Op2; + GetArg2(2, Op1, Op2); + DO_OUTLINED_ARITH_2(mixed_div, Op1, Op2); } - OpCase(i_int_div_jId): + OpCase(i_int_div_jIssd): { - Eterm result; + Eterm Op1, Op2; - if (tmp_arg2 == SMALL_ZERO) { + GetArg2(2, Op1, Op2); + if (Op2 == SMALL_ZERO) { goto badarith; - } else if (is_both_small(tmp_arg1, tmp_arg2)) { - Sint ires = signed_val(tmp_arg1) / signed_val(tmp_arg2); + } else if (is_both_small(Op1, Op2)) { + Sint ires = signed_val(Op1) / signed_val(Op2); if (MY_IS_SSMALL(ires)) { - result = make_small(ires); - STORE_ARITH_RESULT(result); + Eterm result = make_small(ires); + StoreBifResult(4, result); } } - arith_func = ARITH_FUNC(int_div); - goto do_big_arith2; + DO_OUTLINED_ARITH_2(int_div, Op1, Op2); } - OpCase(i_rem_jIxxd): { - Eterm result; + Eterm RemOp1, RemOp2; - if (xb(Arg(3)) == SMALL_ZERO) { - goto badarith; - } else if (is_both_small(xb(Arg(2)), xb(Arg(3)))) { - result = make_small(signed_val(xb(Arg(2))) % signed_val(xb(Arg(3)))); + OpCase(i_rem_jIxxd): + RemOp1 = xb(Arg(2)); + RemOp2 = xb(Arg(3)); + goto do_rem; + + OpCase(i_rem_jIssd): + GetArg2(2, RemOp1, RemOp2); + goto do_rem; + + do_rem: + if (RemOp2 == SMALL_ZERO) { + goto badarith; + } else if (is_both_small(RemOp1, RemOp2)) { + Eterm result = make_small(signed_val(RemOp1) % signed_val(RemOp2)); StoreBifResult(4, result); + } else { + DO_OUTLINED_ARITH_2(int_rem, RemOp1, RemOp2); } - DO_BIG_ARITH(ARITH_FUNC(int_rem),xb(Arg(2)),xb(Arg(3))); } - OpCase(i_rem_jId): { - Eterm result; - - if (tmp_arg2 == SMALL_ZERO) { - goto badarith; - } else if (is_both_small(tmp_arg1, tmp_arg2)) { - result = make_small(signed_val(tmp_arg1) % signed_val(tmp_arg2)); - STORE_ARITH_RESULT(result); - } else { - arith_func = ARITH_FUNC(int_rem); - goto do_big_arith2; - } - } + Eterm BandOp1, BandOp2; OpCase(i_band_jIxcd): - { - Eterm result; + BandOp1 = xb(Arg(2)); + BandOp2 = Arg(3); + goto do_band; - if (is_both_small(xb(Arg(2)), Arg(3))) { + OpCase(i_band_jIssd): + GetArg2(2, BandOp1, BandOp2); + goto do_band; + + do_band: + if (is_both_small(BandOp1, BandOp2)) { /* * No need to untag -- TAG & TAG == TAG. */ - result = xb(Arg(2)) & Arg(3); + Eterm result = BandOp1 & BandOp2; StoreBifResult(4, result); } - DO_BIG_ARITH(ARITH_FUNC(band),xb(Arg(2)),Arg(3)); - } - - OpCase(i_band_jId): - { - Eterm result; - - if (is_both_small(tmp_arg1, tmp_arg2)) { - /* - * No need to untag -- TAG & TAG == TAG. - */ - result = tmp_arg1 & tmp_arg2; - STORE_ARITH_RESULT(result); - } - arith_func = ARITH_FUNC(band); - goto do_big_arith2; - } - -#undef DO_BIG_ARITH - - do_big_arith2: - { - Eterm result; - Uint live = Arg(1); - - SWAPOUT; - reg[0] = r(0); - reg[live] = tmp_arg1; - reg[live+1] = tmp_arg2; - result = arith_func(c_p, reg, live); - r(0) = reg[0]; - SWAPIN; - ERTS_HOLE_CHECK(c_p); - if (is_value(result)) { - STORE_ARITH_RESULT(result); - } - goto lb_Cl_error; + DO_OUTLINED_ARITH_2(band, BandOp1, BandOp2); } /* @@ -3032,97 +2909,105 @@ do { \ goto find_func_info; } - OpCase(i_bor_jId): + OpCase(i_bor_jIssd): { - Eterm result; + Eterm Op1, Op2; - if (is_both_small(tmp_arg1, tmp_arg2)) { + GetArg2(2, Op1, Op2); + if (is_both_small(Op1, Op2)) { /* * No need to untag -- TAG | TAG == TAG. */ - result = tmp_arg1 | tmp_arg2; - STORE_ARITH_RESULT(result); + Eterm result = Op1 | Op2; + StoreBifResult(4, result); } - arith_func = ARITH_FUNC(bor); - goto do_big_arith2; + DO_OUTLINED_ARITH_2(bor, Op1, Op2); } - OpCase(i_bxor_jId): + OpCase(i_bxor_jIssd): { - Eterm result; + Eterm Op1, Op2; - if (is_both_small(tmp_arg1, tmp_arg2)) { + GetArg2(2, Op1, Op2); + if (is_both_small(Op1, Op2)) { /* * We could extract the tag from one argument, but a tag extraction * could mean a shift. Therefore, play it safe here. */ - result = make_small(signed_val(tmp_arg1) ^ signed_val(tmp_arg2)); - STORE_ARITH_RESULT(result); + Eterm result = make_small(signed_val(Op1) ^ signed_val(Op2)); + StoreBifResult(4, result); } - arith_func = ARITH_FUNC(bxor); - goto do_big_arith2; + DO_OUTLINED_ARITH_2(bxor, Op1, Op2); } { + Eterm Op1, Op2; Sint i; Sint ires; Eterm* bigp; + Eterm tmp_big[2]; - OpCase(i_bsr_jId): - if (is_small(tmp_arg2)) { - i = -signed_val(tmp_arg2); - if (is_small(tmp_arg1)) { + OpCase(i_bsr_jIssd): + GetArg2(2, Op1, Op2); + if (is_small(Op2)) { + i = -signed_val(Op2); + if (is_small(Op1)) { goto small_shift; - } else if (is_big(tmp_arg1)) { + } else if (is_big(Op1)) { if (i == 0) { - StoreBifResult(2, tmp_arg1); + StoreBifResult(4, Op1); } goto big_shift; } - } else if (is_big(tmp_arg2)) { + } else if (is_big(Op2)) { /* * N bsr NegativeBigNum == N bsl MAX_SMALL * N bsr PositiveBigNum == N bsl MIN_SMALL */ - tmp_arg2 = make_small(bignum_header_is_neg(*big_val(tmp_arg2)) ? + Op2 = make_small(bignum_header_is_neg(*big_val(Op2)) ? MAX_SMALL : MIN_SMALL); goto do_bsl; } goto badarith; - OpCase(i_bsl_jId): + OpCase(i_bsl_jIssd): + GetArg2(2, Op1, Op2); + do_bsl: - if (is_small(tmp_arg2)) { - i = signed_val(tmp_arg2); + if (is_small(Op2)) { + i = signed_val(Op2); - if (is_small(tmp_arg1)) { + if (is_small(Op1)) { small_shift: - ires = signed_val(tmp_arg1); + ires = signed_val(Op1); if (i == 0 || ires == 0) { - StoreBifResult(2, tmp_arg1); + StoreBifResult(4, Op1); } else if (i < 0) { /* Right shift */ i = -i; if (i >= SMALL_BITS-1) { - tmp_arg1 = (ires < 0) ? SMALL_MINUS_ONE : SMALL_ZERO; + Op1 = (ires < 0) ? SMALL_MINUS_ONE : SMALL_ZERO; } else { - tmp_arg1 = make_small(ires >> i); + Op1 = make_small(ires >> i); } - StoreBifResult(2, tmp_arg1); + StoreBifResult(4, Op1); } else if (i < SMALL_BITS-1) { /* Left shift */ if ((ires > 0 && ((~(Uint)0 << ((SMALL_BITS-1)-i)) & ires) == 0) || ((~(Uint)0 << ((SMALL_BITS-1)-i)) & ~ires) == 0) { - tmp_arg1 = make_small(ires << i); - StoreBifResult(2, tmp_arg1); + Op1 = make_small(ires << i); + StoreBifResult(4, Op1); } } - tmp_arg1 = small_to_big(ires, tmp_big); + Op1 = small_to_big(ires, tmp_big); +#ifdef TAG_LITERAL_PTR + Op1 |= TAG_LITERAL_PTR; +#endif big_shift: if (i > 0) { /* Left shift. */ - ires = big_size(tmp_arg1) + (i / D_EXP); + ires = big_size(Op1) + (i / D_EXP); } else { /* Right shift. */ - ires = big_size(tmp_arg1); + ires = big_size(Op1); if (ires <= (-i / D_EXP)) ires = 3; /* ??? */ else @@ -3140,14 +3025,14 @@ do { \ c_p->freason = SYSTEM_LIMIT; goto lb_Cl_error; } - TestHeapPreserve(ires+1, Arg(1), tmp_arg1); + TestHeapPreserve(ires+1, Arg(1), Op1); bigp = HTOP; - tmp_arg1 = big_lshift(tmp_arg1, i, bigp); - if (is_big(tmp_arg1)) { + Op1 = big_lshift(Op1, i, bigp); + if (is_big(Op1)) { HTOP += bignum_header_arity(*HTOP) + 1; } HEAP_SPACE_VERIFIED(0); - if (is_nil(tmp_arg1)) { + if (is_nil(Op1)) { /* * This result must have been only slight larger * than allowed since it wasn't caught by the @@ -3157,25 +3042,25 @@ do { \ goto lb_Cl_error; } ERTS_HOLE_CHECK(c_p); - StoreBifResult(2, tmp_arg1); + StoreBifResult(4, Op1); } - } else if (is_big(tmp_arg1)) { + } else if (is_big(Op1)) { if (i == 0) { - StoreBifResult(2, tmp_arg1); + StoreBifResult(4, Op1); } goto big_shift; } - } else if (is_big(tmp_arg2)) { - if (bignum_header_is_neg(*big_val(tmp_arg2))) { + } else if (is_big(Op2)) { + if (bignum_header_is_neg(*big_val(Op2))) { /* * N bsl NegativeBigNum is either 0 or -1, depending on * the sign of N. Since we don't believe this case * is common, do the calculation with the minimum * amount of code. */ - tmp_arg2 = make_small(MIN_SMALL); + Op2 = make_small(MIN_SMALL); goto do_bsl; - } else if (is_small(tmp_arg1) || is_big(tmp_arg1)) { + } else if (is_small(Op1) || is_big(Op1)) { /* * N bsl PositiveBigNum is too large to represent. */ @@ -3199,12 +3084,10 @@ do { \ bnot_val = make_small(~signed_val(bnot_val)); } else { Uint live = Arg(2); - SWAPOUT; - reg[0] = r(0); + HEAVY_SWAPOUT; reg[live] = bnot_val; bnot_val = erts_gc_bnot(c_p, reg, live); - r(0) = reg[0]; - SWAPIN; + HEAVY_SWAPIN; ERTS_HOLE_CHECK(c_p); if (is_nil(bnot_val)) { goto lb_Cl_error; @@ -3219,11 +3102,10 @@ do { \ OpCase(i_apply): { BeamInstr *next; - SWAPOUT; + HEAVY_SWAPOUT; next = apply(c_p, r(0), x(1), x(2), reg); - SWAPIN; + HEAVY_SWAPIN; if (next != NULL) { - r(0) = reg[0]; SET_CP(c_p, I+1); SET_I(next); Dispatch(); @@ -3234,12 +3116,11 @@ do { \ OpCase(i_apply_last_P): { BeamInstr *next; - SWAPOUT; + HEAVY_SWAPOUT; next = apply(c_p, r(0), x(1), x(2), reg); - SWAPIN; + HEAVY_SWAPIN; if (next != NULL) { - r(0) = reg[0]; - SET_CP(c_p, (BeamInstr *) EXPAND_POINTER(E[0])); + SET_CP(c_p, (BeamInstr *) E[0]); E = ADD_BYTE_OFFSET(E, Arg(0)); SET_I(next); Dispatch(); @@ -3250,11 +3131,10 @@ do { \ OpCase(i_apply_only): { BeamInstr *next; - SWAPOUT; + HEAVY_SWAPOUT; next = apply(c_p, r(0), x(1), x(2), reg); - SWAPIN; + HEAVY_SWAPIN; if (next != NULL) { - r(0) = reg[0]; SET_I(next); Dispatch(); } @@ -3265,12 +3145,10 @@ do { \ OpCase(apply_I): { BeamInstr *next; - reg[0] = r(0); - SWAPOUT; + HEAVY_SWAPOUT; next = fixed_apply(c_p, reg, Arg(0)); - SWAPIN; + HEAVY_SWAPIN; if (next != NULL) { - r(0) = reg[0]; SET_CP(c_p, I+2); SET_I(next); Dispatch(); @@ -3282,13 +3160,11 @@ do { \ OpCase(apply_last_IP): { BeamInstr *next; - reg[0] = r(0); - SWAPOUT; + HEAVY_SWAPOUT; next = fixed_apply(c_p, reg, Arg(0)); - SWAPIN; + HEAVY_SWAPIN; if (next != NULL) { - r(0) = reg[0]; - SET_CP(c_p, (BeamInstr *) EXPAND_POINTER(E[0])); + SET_CP(c_p, (BeamInstr *) E[0]); E = ADD_BYTE_OFFSET(E, Arg(1)); SET_I(next); Dispatch(); @@ -3300,11 +3176,10 @@ do { \ OpCase(i_apply_fun): { BeamInstr *next; - SWAPOUT; + HEAVY_SWAPOUT; next = apply_fun(c_p, r(0), x(1), reg); - SWAPIN; + HEAVY_SWAPIN; if (next != NULL) { - r(0) = reg[0]; SET_CP(c_p, I+1); SET_I(next); Dispatchfun(); @@ -3315,12 +3190,11 @@ do { \ OpCase(i_apply_fun_last_P): { BeamInstr *next; - SWAPOUT; + HEAVY_SWAPOUT; next = apply_fun(c_p, r(0), x(1), reg); - SWAPIN; + HEAVY_SWAPIN; if (next != NULL) { - r(0) = reg[0]; - SET_CP(c_p, (BeamInstr *) EXPAND_POINTER(E[0])); + SET_CP(c_p, (BeamInstr *) E[0]); E = ADD_BYTE_OFFSET(E, Arg(0)); SET_I(next); Dispatchfun(); @@ -3331,11 +3205,10 @@ do { \ OpCase(i_apply_fun_only): { BeamInstr *next; - SWAPOUT; + HEAVY_SWAPOUT; next = apply_fun(c_p, r(0), x(1), reg); - SWAPIN; + HEAVY_SWAPIN; if (next != NULL) { - r(0) = reg[0]; SET_I(next); Dispatchfun(); } @@ -3345,13 +3218,10 @@ do { \ OpCase(i_call_fun_I): { BeamInstr *next; - SWAPOUT; - reg[0] = r(0); - + HEAVY_SWAPOUT; next = call_fun(c_p, Arg(0), reg, THE_NON_VALUE); - SWAPIN; + HEAVY_SWAPIN; if (next != NULL) { - r(0) = reg[0]; SET_CP(c_p, I+2); SET_I(next); Dispatchfun(); @@ -3362,13 +3232,11 @@ do { \ OpCase(i_call_fun_last_IP): { BeamInstr *next; - SWAPOUT; - reg[0] = r(0); + HEAVY_SWAPOUT; next = call_fun(c_p, Arg(0), reg, THE_NON_VALUE); - SWAPIN; + HEAVY_SWAPIN; if (next != NULL) { - r(0) = reg[0]; - SET_CP(c_p, (BeamInstr *) EXPAND_POINTER(E[0])); + SET_CP(c_p, (BeamInstr *) E[0]); E = ADD_BYTE_OFFSET(E, Arg(1)); SET_I(next); Dispatchfun(); @@ -3452,10 +3320,9 @@ do { \ */ argp = c_p->arg_reg; - for (i = c_p->arity - 1; i > 0; i--) { + for (i = c_p->arity - 1; i >= 0; i--) { argp[i] = reg[i]; } - c_p->arg_reg[0] = r(0); SWAPOUT; c_p->i = I; goto do_schedule1; @@ -3468,19 +3335,14 @@ do { \ Eterm* p; PreFetch(3, next); - GetArg2(0, element, tuple); + GetArg1(0, element); + tuple = REG_TARGET(Arg(1)); ASSERT(is_tuple(tuple)); p = (Eterm *) ((unsigned char *) tuple_val(tuple) + Arg(2)); *p = element; NextPF(3, next); } - OpCase(i_is_ne_exact_f): - if (EQ(tmp_arg1, tmp_arg2)) { - ClauseFail(); - } - Next(1); - OpCase(normal_exit): { SWAPOUT; c_p->freason = EXC_NORMAL; @@ -3547,25 +3409,14 @@ do { \ { Eterm badmatch_val; - OpCase(badmatch_y): - badmatch_val = yb(Arg(0)); - goto do_badmatch; - OpCase(badmatch_x): badmatch_val = xb(Arg(0)); - goto do_badmatch; - - OpCase(badmatch_r): - badmatch_val = r(0); - - do_badmatch: c_p->fvalue = badmatch_val; c_p->freason = BADMATCH; } /* Fall through here */ find_func_info: { - reg[0] = r(0); SWAPOUT; I = handle_error(c_p, I, reg, NULL); goto post_error_handling; @@ -3582,11 +3433,9 @@ do { \ * code[3]: &&call_error_handler * code[4]: Not used */ - SWAPOUT; - reg[0] = r(0); + HEAVY_SWAPOUT; I = call_error_handler(c_p, I-3, reg, am_undefined_function); - r(0) = reg[0]; - SWAPIN; + HEAVY_SWAPIN; if (I) { Goto(*I); } @@ -3594,18 +3443,13 @@ do { \ /* Fall through */ OpCase(error_action_code): { handle_error: - reg[0] = r(0); SWAPOUT; I = handle_error(c_p, NULL, reg, NULL); post_error_handling: if (I == 0) { goto do_schedule; } else { - r(0) = reg[0]; ASSERT(!is_value(r(0))); - if (c_p->mbuf) { - erts_garbage_collect(c_p, 0, reg+1, 3); - } SWAPIN; Goto(*I); } @@ -3629,6 +3473,9 @@ do { \ * I[3]: Function pointer to dirty NIF */ BifFunction vbf; + ErlHeapFragment *live_hf_end; + + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_NIF); DTRACE_NIF_ENTRY(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]); c_p->current = I-3; /* current and vbf set to please handle_error */ @@ -3644,7 +3491,7 @@ do { \ NifF* fp = vbf = (NifF*) I[1]; struct enif_environment_t env; erts_pre_nif(&env, c_p, (struct erl_module_nif*)I[2]); - reg[0] = r(0); + live_hf_end = c_p->mbuf; nif_bif_result = (*fp)(&env, bif_nif_arity, reg); if (env.exception_thrown) nif_bif_result = THE_NON_VALUE; @@ -3654,6 +3501,8 @@ do { \ PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR); + DTRACE_NIF_RETURN(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]); goto apply_bif_or_nif_epilogue; @@ -3668,6 +3517,13 @@ do { \ * code[3]: &&apply_bif * code[4]: Function pointer to BIF function */ + if (ERTS_MSACC_IS_ENABLED_CACHED_X()) { + if ((Eterm)I[-3] == am_ets) { + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_ETS); + } else { + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_BIF); + } + } c_p->current = I-3; /* In case we apply process_info/1,2 or load_nif/1 */ c_p->i = I; /* In case we apply check_process_code/2. */ @@ -3683,25 +3539,32 @@ do { \ bif_nif_arity = I[-1]; ASSERT(bif_nif_arity <= 4); ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); - reg[0] = r(0); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); { Eterm (*bf)(Process*, Eterm*, BeamInstr*) = vbf; ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + live_hf_end = c_p->mbuf; nif_bif_result = (*bf)(c_p, reg, I); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(nif_bif_result)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); } - + /* We have to update the cache if we are enabled in order + to make sure no book keeping is done after we disabled + msacc. We don't always do this as it is quite expensive. */ + if (ERTS_MSACC_IS_ENABLED_CACHED_X()) + ERTS_MSACC_UPDATE_CACHE_X(); + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR); DTRACE_BIF_RETURN(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]); apply_bif_or_nif_epilogue: ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); ERTS_HOLE_CHECK(c_p); - if (c_p->mbuf) { - nif_bif_result = erts_gc_after_bif_call(c_p, nif_bif_result, - reg, bif_nif_arity); + if (ERTS_IS_GC_DESIRED(c_p)) { + nif_bif_result = erts_gc_after_bif_call_lhf(c_p, live_hf_end, + nif_bif_result, + reg, bif_nif_arity); } SWAPIN; /* There might have been a garbage collection. */ FCALLS = c_p->fcalls; @@ -3713,7 +3576,6 @@ do { \ Goto(*I); } else if (c_p->freason == TRAP) { SET_I(c_p->i); - r(0) = reg[0]; if (c_p->flags & F_HIBERNATE_SCHED) { c_p->flags &= ~F_HIBERNATE_SCHED; goto do_schedule; @@ -3735,21 +3597,21 @@ do { \ StoreBifResult(1, result); } + OpCase(i_get_hash_cId): + { + Eterm arg; + Eterm result; + + GetArg1(0, arg); + result = erts_pd_hash_get_with_hx(c_p, Arg(1), arg); + StoreBifResult(2, result); + } + { Eterm case_end_val; OpCase(case_end_x): case_end_val = xb(Arg(0)); - goto do_case_end; - - OpCase(case_end_y): - case_end_val = yb(Arg(0)); - goto do_case_end; - - OpCase(case_end_r): - case_end_val = r(0); - - do_case_end: c_p->fvalue = case_end_val; c_p->freason = EXC_CASE_CLAUSE; goto find_func_info; @@ -3797,19 +3659,13 @@ do { \ goto do_bs_init_bits_known; } - OpCase(i_bs_init_bits_fail_heap_IjId): { - /* tmp_arg1 was fetched by an i_fetch instruction */ - num_bits_term = tmp_arg1; - alloc = Arg(0); - I++; + OpCase(i_bs_init_bits_fail_heap_sIjId): { + GetArg1(0, num_bits_term); + alloc = Arg(1); + I += 2; goto do_bs_init_bits; } - OpCase(i_bs_init_bits_fail_rjId): { - num_bits_term = r(0); - alloc = 0; - goto do_bs_init_bits; - } OpCase(i_bs_init_bits_fail_yjId): { num_bits_term = yb(Arg(0)); I++; @@ -3929,52 +3785,48 @@ do { \ } { - OpCase(i_bs_init_fail_heap_IjId): { - /* tmp_arg1 was fetched by an i_fetch instruction */ - tmp_arg2 = Arg(0); - I++; - goto do_bs_init; - } + Eterm BsOp1, BsOp2; - OpCase(i_bs_init_fail_rjId): { - tmp_arg1 = r(0); - tmp_arg2 = 0; + OpCase(i_bs_init_fail_heap_sIjId): { + GetArg1(0, BsOp1); + BsOp2 = Arg(1); + I += 2; goto do_bs_init; } OpCase(i_bs_init_fail_yjId): { - tmp_arg1 = yb(Arg(0)); - tmp_arg2 = 0; + BsOp1 = yb(Arg(0)); + BsOp2 = 0; I++; goto do_bs_init; } OpCase(i_bs_init_fail_xjId): { - tmp_arg1 = xb(Arg(0)); - tmp_arg2 = 0; + BsOp1 = xb(Arg(0)); + BsOp2 = 0; I++; } /* FALL THROUGH */ do_bs_init: - if (is_small(tmp_arg1)) { - Sint size = signed_val(tmp_arg1); + if (is_small(BsOp1)) { + Sint size = signed_val(BsOp1); if (size < 0) { goto badarg; } - tmp_arg1 = (Eterm) size; + BsOp1 = (Eterm) size; } else { Uint bytes; - if (!term_to_Uint(tmp_arg1, &bytes)) { + if (!term_to_Uint(BsOp1, &bytes)) { c_p->freason = bytes; goto lb_Cl_error; } if ((bytes >> (8*sizeof(Uint)-3)) != 0) { goto system_limit; } - tmp_arg1 = (Eterm) bytes; + BsOp1 = (Eterm) bytes; } - if (tmp_arg1 <= ERL_ONHEAP_BIN_LIMIT) { + if (BsOp1 <= ERL_ONHEAP_BIN_LIMIT) { goto do_heap_bin_alloc; } else { goto do_proc_bin_alloc; @@ -3982,15 +3834,15 @@ do { \ OpCase(i_bs_init_heap_IIId): { - tmp_arg1 = Arg(0); - tmp_arg2 = Arg(1); + BsOp1 = Arg(0); + BsOp2 = Arg(1); I++; goto do_proc_bin_alloc; } OpCase(i_bs_init_IId): { - tmp_arg1 = Arg(0); - tmp_arg2 = 0; + BsOp1 = Arg(0); + BsOp2 = 0; } /* FALL THROUGH */ do_proc_bin_alloc: { @@ -3999,13 +3851,13 @@ do { \ erts_bin_offset = 0; erts_writable_bin = 0; - TestBinVHeap(tmp_arg1 / sizeof(Eterm), - tmp_arg2 + PROC_BIN_SIZE + ERL_SUB_BIN_SIZE, Arg(1)); + TestBinVHeap(BsOp1 / sizeof(Eterm), + BsOp2 + PROC_BIN_SIZE + ERL_SUB_BIN_SIZE, Arg(1)); /* * Allocate the binary struct itself. */ - bptr = erts_bin_nrml_alloc(tmp_arg1); + bptr = erts_bin_nrml_alloc(BsOp1); erts_refc_init(&bptr->refc, 1); erts_current_bin = (byte *) bptr->orig_bytes; @@ -4015,28 +3867,28 @@ do { \ pb = (ProcBin *) HTOP; HTOP += PROC_BIN_SIZE; pb->thing_word = HEADER_PROC_BIN; - pb->size = tmp_arg1; + pb->size = BsOp1; pb->next = MSO(c_p).first; MSO(c_p).first = (struct erl_off_heap_header*) pb; pb->val = bptr; pb->bytes = (byte*) bptr->orig_bytes; pb->flags = 0; - OH_OVERHEAD(&(MSO(c_p)), tmp_arg1 / sizeof(Eterm)); + OH_OVERHEAD(&(MSO(c_p)), BsOp1 / sizeof(Eterm)); StoreBifResult(2, make_binary(pb)); } OpCase(i_bs_init_heap_bin_heap_IIId): { - tmp_arg1 = Arg(0); - tmp_arg2 = Arg(1); + BsOp1 = Arg(0); + BsOp2 = Arg(1); I++; goto do_heap_bin_alloc; } OpCase(i_bs_init_heap_bin_IId): { - tmp_arg1 = Arg(0); - tmp_arg2 = 0; + BsOp1 = Arg(0); + BsOp2 = 0; } /* Fall through */ do_heap_bin_alloc: @@ -4044,33 +3896,36 @@ do { \ ErlHeapBin* hb; Uint bin_need; - bin_need = heap_bin_size(tmp_arg1); + bin_need = heap_bin_size(BsOp1); erts_bin_offset = 0; erts_writable_bin = 0; - TestHeap(bin_need+tmp_arg2+ERL_SUB_BIN_SIZE, Arg(1)); + TestHeap(bin_need+BsOp2+ERL_SUB_BIN_SIZE, Arg(1)); hb = (ErlHeapBin *) HTOP; HTOP += bin_need; - hb->thing_word = header_heap_bin(tmp_arg1); - hb->size = tmp_arg1; + hb->thing_word = header_heap_bin(BsOp1); + hb->size = BsOp1; erts_current_bin = (byte *) hb->data; - tmp_arg1 = make_binary(hb); - StoreBifResult(2, tmp_arg1); + BsOp1 = make_binary(hb); + StoreBifResult(2, BsOp1); } } - OpCase(i_bs_add_jId): { - Uint Unit = Arg(1); - if (is_both_small(tmp_arg1, tmp_arg2)) { - Sint Arg1 = signed_val(tmp_arg1); - Sint Arg2 = signed_val(tmp_arg2); + OpCase(bs_add_jssId): { + Eterm Op1, Op2; + Uint Unit = Arg(3); + + GetArg2(1, Op1, Op2); + if (is_both_small(Op1, Op2)) { + Sint Arg1 = signed_val(Op1); + Sint Arg2 = signed_val(Op2); if (Arg1 >= 0 && Arg2 >= 0) { - BsSafeMul(Arg2, Unit, goto system_limit, tmp_arg1); - tmp_arg1 += Arg1; + BsSafeMul(Arg2, Unit, goto system_limit, Op1); + Op1 += Arg1; store_bs_add_result: - if (tmp_arg1 <= MAX_SMALL) { - tmp_arg1 = make_small(tmp_arg1); + if (Op1 <= MAX_SMALL) { + Op1 = make_small(Op1); } else { /* * May generate a heap fragment, but in this @@ -4082,10 +3937,10 @@ do { \ * references (such as the heap). */ SWAPOUT; - tmp_arg1 = erts_make_integer(tmp_arg1, c_p); + Op1 = erts_make_integer(Op1, c_p); HTOP = HEAP_TOP(c_p); } - StoreBifResult(2, tmp_arg1); + StoreBifResult(4, Op1); } goto badarg; } else { @@ -4108,16 +3963,16 @@ do { \ * an Uint, the reason is SYSTEM_LIMIT. */ - if (!term_to_Uint(tmp_arg1, &a)) { + if (!term_to_Uint(Op1, &a)) { if (a == BADARG) { goto badarg; } - if (!term_to_Uint(tmp_arg2, &b)) { + if (!term_to_Uint(Op2, &b)) { c_p->freason = b; goto lb_Cl_error; } goto system_limit; - } else if (!term_to_Uint(tmp_arg2, &b)) { + } else if (!term_to_Uint(Op2, &b)) { c_p->freason = b; goto lb_Cl_error; } @@ -4127,8 +3982,8 @@ do { \ */ BsSafeMul(b, Unit, goto system_limit, c); - tmp_arg1 = a + c; - if (tmp_arg1 < a) { + Op1 = a + c; + if (Op1 < a) { /* * If the result is less than one of the * arguments, there must have been an overflow. @@ -4150,52 +4005,47 @@ do { \ } /* - * tmp_arg1 = Number of bytes to build - * tmp_arg2 = Source binary - * Operands: Fail ExtraHeap Live Unit Dst + * x(SCRATCH_X_REG); + * Operands: Fail ExtraHeap Live Unit Size Dst */ - OpCase(i_bs_append_jIIId): { + OpCase(i_bs_append_jIIIsd): { Uint live = Arg(2); Uint res; + Eterm Size; - SWAPOUT; - reg[0] = r(0); - reg[live] = tmp_arg2; - res = erts_bs_append(c_p, reg, live, tmp_arg1, Arg(1), Arg(3)); - r(0) = reg[0]; - SWAPIN; + GetArg1(4, Size); + HEAVY_SWAPOUT; + reg[live] = x(SCRATCH_X_REG); + res = erts_bs_append(c_p, reg, live, Size, Arg(1), Arg(3)); + HEAVY_SWAPIN; if (is_non_value(res)) { /* c_p->freason is already set (may be either BADARG or SYSTEM_LIMIT). */ goto lb_Cl_error; } - StoreBifResult(4, res); + StoreBifResult(5, res); } /* - * tmp_arg1 = Number of bytes to build - * tmp_arg2 = Source binary - * Operands: Fail Unit Dst + * Operands: Fail Size Src Unit Dst */ - OpCase(i_bs_private_append_jId): { + OpCase(i_bs_private_append_jIssd): { Eterm res; + Eterm Size, Src; - res = erts_bs_private_append(c_p, tmp_arg2, tmp_arg1, Arg(1)); + GetArg2(2, Size, Src); + res = erts_bs_private_append(c_p, Src, Size, Arg(1)); if (is_non_value(res)) { /* c_p->freason is already set (may be either BADARG or SYSTEM_LIMIT). */ goto lb_Cl_error; } - StoreBifResult(2, res); + StoreBifResult(4, res); } - /* - * tmp_arg1 = Initial size of writable binary - * Operands: Live Dst - */ OpCase(bs_init_writable): { - SWAPOUT; + HEAVY_SWAPOUT; r(0) = erts_bs_init_writable(c_p, r(0)); - SWAPIN; + HEAVY_SWAPIN; Next(0); } @@ -4286,26 +4136,29 @@ do { \ /* * Only used for validating a value matched out. - * - * tmp_arg1 = Integer to validate - * tmp_arg2 = Match context */ - OpCase(i_bs_validate_unicode_retract_j): { - /* - * There is no need to untag the integer, but it IS necessary - * to make sure it is small (a bignum pointer could fall in - * the valid range). - */ - if (is_not_small(tmp_arg1) || tmp_arg1 > make_small(0x10FFFFUL) || - (make_small(0xD800UL) <= tmp_arg1 && - tmp_arg1 <= make_small(0xDFFFUL))) { - ErlBinMatchBuffer *mb = ms_matchbuffer(tmp_arg2); + OpCase(i_bs_validate_unicode_retract_jss): { + Eterm i; /* Integer to validate */ - mb->offset -= 32; - goto badarg; - } - Next(1); - } + /* + * There is no need to untag the integer, but it IS necessary + * to make sure it is small (a bignum pointer could fall in + * the valid range). + */ + + GetArg1(1, i); + if (is_not_small(i) || i > make_small(0x10FFFFUL) || + (make_small(0xD800UL) <= i && i <= make_small(0xDFFFUL))) { + Eterm ms; /* Match context */ + ErlBinMatchBuffer* mb; + + GetArg1(2, ms); + mb = ms_matchbuffer(ms); + mb->offset -= 32; + goto badarg; + } + Next(3); + } /* * Matching of binaries. @@ -4317,9 +4170,6 @@ do { \ Uint slots; Eterm context; - OpCase(i_bs_start_match2_rfIId): { - context = r(0); - do_start_match: slots = Arg(2); if (!is_boxed(context)) { @@ -4366,7 +4216,7 @@ do { \ ClauseFail(); } NextPF(4, next); - } + OpCase(i_bs_start_match2_xfIId): { context = xb(Arg(0)); I++; @@ -4379,18 +4229,6 @@ do { \ } } - OpCase(bs_test_zero_tail2_fr): { - BeamInstr *next; - ErlBinMatchBuffer *_mb; - - PreFetch(1, next); - _mb = (ErlBinMatchBuffer*) ms_matchbuffer(r(0)); - if (_mb->size != _mb->offset) { - ClauseFail(); - } - NextPF(1, next); - } - OpCase(bs_test_zero_tail2_fx): { BeamInstr *next; ErlBinMatchBuffer *_mb; @@ -4403,16 +4241,6 @@ do { \ NextPF(2, next); } - OpCase(bs_test_tail_imm2_frI): { - BeamInstr *next; - ErlBinMatchBuffer *_mb; - PreFetch(2, next); - _mb = ms_matchbuffer(r(0)); - if (_mb->size - _mb->offset != Arg(1)) { - ClauseFail(); - } - NextPF(2, next); - } OpCase(bs_test_tail_imm2_fxI): { BeamInstr *next; ErlBinMatchBuffer *_mb; @@ -4424,16 +4252,6 @@ do { \ NextPF(3, next); } - OpCase(bs_test_unit_frI): { - BeamInstr *next; - ErlBinMatchBuffer *_mb; - PreFetch(2, next); - _mb = ms_matchbuffer(r(0)); - if ((_mb->size - _mb->offset) % Arg(1)) { - ClauseFail(); - } - NextPF(2, next); - } OpCase(bs_test_unit_fxI): { BeamInstr *next; ErlBinMatchBuffer *_mb; @@ -4445,16 +4263,6 @@ do { \ NextPF(3, next); } - OpCase(bs_test_unit8_fr): { - BeamInstr *next; - ErlBinMatchBuffer *_mb; - PreFetch(1, next); - _mb = ms_matchbuffer(r(0)); - if ((_mb->size - _mb->offset) & 7) { - ClauseFail(); - } - NextPF(1, next); - } OpCase(bs_test_unit8_fx): { BeamInstr *next; ErlBinMatchBuffer *_mb; @@ -4469,19 +4277,11 @@ do { \ { Eterm bs_get_integer8_context; - OpCase(i_bs_get_integer_8_rfd): { - bs_get_integer8_context = r(0); - goto do_bs_get_integer_8; - } - OpCase(i_bs_get_integer_8_xfd): { - bs_get_integer8_context = xb(Arg(0)); - I++; - } - - do_bs_get_integer_8: { ErlBinMatchBuffer *_mb; Eterm _result; + bs_get_integer8_context = xb(Arg(0)); + I++; _mb = ms_matchbuffer(bs_get_integer8_context); if (_mb->size - _mb->offset < 8) { ClauseFail(); @@ -4499,15 +4299,10 @@ do { \ { Eterm bs_get_integer_16_context; - OpCase(i_bs_get_integer_16_rfd): - bs_get_integer_16_context = r(0); - goto do_bs_get_integer_16; - OpCase(i_bs_get_integer_16_xfd): bs_get_integer_16_context = xb(Arg(0)); I++; - do_bs_get_integer_16: { ErlBinMatchBuffer *_mb; Eterm _result; @@ -4528,17 +4323,10 @@ do { \ { Eterm bs_get_integer_32_context; - OpCase(i_bs_get_integer_32_rfId): - bs_get_integer_32_context = r(0); - goto do_bs_get_integer_32; - - OpCase(i_bs_get_integer_32_xfId): bs_get_integer_32_context = xb(Arg(0)); I++; - - do_bs_get_integer_32: { ErlBinMatchBuffer *_mb; Uint32 _integer; @@ -4551,11 +4339,11 @@ do { \ _integer = get_int32(_mb->base + _mb->offset/8); } _mb->offset += 32; -#if !defined(ARCH_64) || HALFWORD_HEAP +#if !defined(ARCH_64) if (IS_USMALL(0, _integer)) { #endif _result = make_small(_integer); -#if !defined(ARCH_64) || HALFWORD_HEAP +#if !defined(ARCH_64) } else { TestHeap(BIG_UINT_HEAP_SIZE, Arg(1)); _result = uint_to_big((Uint) _integer, HTOP); @@ -4567,103 +4355,82 @@ do { \ } } - /* Operands: Size Live Fail Flags Dst */ - OpCase(i_bs_get_integer_imm_rIIfId): { - tmp_arg1 = r(0); - /* Operands: Size Live Fail Flags Dst */ - goto do_bs_get_integer_imm_test_heap; - } + { + Eterm Ms, Sz; - /* Operands: x(Reg) Size Live Fail Flags Dst */ + /* Operands: x(Reg) Size Live Fail Flags Dst */ OpCase(i_bs_get_integer_imm_xIIfId): { - tmp_arg1 = xb(Arg(0)); - I++; - /* Operands: Size Live Fail Flags Dst */ - goto do_bs_get_integer_imm_test_heap; - } - - /* - * tmp_arg1 = match context - * Operands: Size Live Fail Flags Dst - */ - do_bs_get_integer_imm_test_heap: { - Uint wordsneeded; - tmp_arg2 = Arg(0); - wordsneeded = 1+WSIZE(NBYTES(tmp_arg2)); - TestHeapPreserve(wordsneeded, Arg(1), tmp_arg1); - I += 2; - /* Operands: Fail Flags Dst */ - goto do_bs_get_integer_imm; - } - - /* Operands: Size Fail Flags Dst */ - OpCase(i_bs_get_integer_small_imm_rIfId): { - tmp_arg1 = r(0); - tmp_arg2 = Arg(0); - I++; - /* Operands: Fail Flags Dst */ - goto do_bs_get_integer_imm; - } + Uint wordsneeded; + Ms = xb(Arg(0)); + Sz = Arg(1); + wordsneeded = 1+WSIZE(NBYTES(Sz)); + TestHeapPreserve(wordsneeded, Arg(2), Ms); + I += 3; + /* Operands: Fail Flags Dst */ + goto do_bs_get_integer_imm; + } - /* Operands: x(Reg) Size Fail Flags Dst */ + /* Operands: x(Reg) Size Fail Flags Dst */ OpCase(i_bs_get_integer_small_imm_xIfId): { - tmp_arg1 = xb(Arg(0)); - tmp_arg2 = Arg(1); - I += 2; - /* Operands: Fail Flags Dst */ - goto do_bs_get_integer_imm; - } + Ms = xb(Arg(0)); + Sz = Arg(1); + I += 2; + /* Operands: Fail Flags Dst */ + goto do_bs_get_integer_imm; + } - /* - * tmp_arg1 = match context - * tmp_arg2 = size of field - * Operands: Fail Flags Dst - */ + /* + * Ms = match context + * Sz = size of field + * Operands: Fail Flags Dst + */ do_bs_get_integer_imm: { - ErlBinMatchBuffer* mb; - Eterm result; + ErlBinMatchBuffer* mb; + Eterm result; - mb = ms_matchbuffer(tmp_arg1); - LIGHT_SWAPOUT; - result = erts_bs_get_integer_2(c_p, tmp_arg2, Arg(1), mb); - LIGHT_SWAPIN; - HEAP_SPACE_VERIFIED(0); - if (is_non_value(result)) { - ClauseFail(); + mb = ms_matchbuffer(Ms); + LIGHT_SWAPOUT; + result = erts_bs_get_integer_2(c_p, Sz, Arg(1), mb); + LIGHT_SWAPIN; + HEAP_SPACE_VERIFIED(0); + if (is_non_value(result)) { + ClauseFail(); + } + StoreBifResult(2, result); } - StoreBifResult(2, result); } /* - * tmp_arg1 = Match context - * tmp_arg2 = Size field - * Operands: Fail Live FlagsAndUnit Dst + * Operands: Fail Live FlagsAndUnit Ms Sz Dst */ - OpCase(i_bs_get_integer_fIId): { + OpCase(i_bs_get_integer_fIIssd): { Uint flags; Uint size; + Eterm Ms; + Eterm Sz; ErlBinMatchBuffer* mb; Eterm result; flags = Arg(2); - BsGetFieldSize(tmp_arg2, (flags >> 3), ClauseFail(), size); + GetArg2(3, Ms, Sz); + BsGetFieldSize(Sz, (flags >> 3), ClauseFail(), size); if (size >= SMALL_BITS) { Uint wordsneeded; - /* check bits size before potential gc. + /* Check bits size before potential gc. * We do not want a gc and then realize we don't need - * the allocated space (i.e. if the op fails) + * the allocated space (i.e. if the op fails). * - * remember to reacquire the matchbuffer after gc. + * Remember to re-acquire the matchbuffer after gc. */ - mb = ms_matchbuffer(tmp_arg1); + mb = ms_matchbuffer(Ms); if (mb->size - mb->offset < size) { ClauseFail(); } wordsneeded = 1+WSIZE(NBYTES((Uint) size)); - TestHeapPreserve(wordsneeded, Arg(1), tmp_arg1); + TestHeapPreserve(wordsneeded, Arg(1), Ms); } - mb = ms_matchbuffer(tmp_arg1); + mb = ms_matchbuffer(Ms); LIGHT_SWAPOUT; result = erts_bs_get_integer_2(c_p, size, flags, mb); LIGHT_SWAPIN; @@ -4671,18 +4438,13 @@ do { \ if (is_non_value(result)) { ClauseFail(); } - StoreBifResult(3, result); + StoreBifResult(5, result); } { Eterm get_utf8_context; /* Operands: MatchContext Fail Dst */ - OpCase(i_bs_get_utf8_rfd): { - get_utf8_context = r(0); - goto do_bs_get_utf8; - } - OpCase(i_bs_get_utf8_xfd): { get_utf8_context = xb(Arg(0)); I++; @@ -4693,7 +4455,7 @@ do { \ * Operands: Fail Dst */ - do_bs_get_utf8: { + { Eterm result = erts_bs_get_utf8(ms_matchbuffer(get_utf8_context)); if (is_non_value(result)) { ClauseFail(); @@ -4706,12 +4468,7 @@ do { \ Eterm get_utf16_context; /* Operands: MatchContext Fail Flags Dst */ - OpCase(i_bs_get_utf16_rfId): { - get_utf16_context = r(0); - goto do_bs_get_utf16; - } - - OpCase(i_bs_get_utf16_xfId): { + OpCase(i_bs_get_utf16_xfId): { get_utf16_context = xb(Arg(0)); I++; } @@ -4720,7 +4477,7 @@ do { \ * get_utf16_context = match_context * Operands: Fail Flags Dst */ - do_bs_get_utf16: { + { Eterm result = erts_bs_get_utf16(ms_matchbuffer(get_utf16_context), Arg(1)); if (is_non_value(result)) { @@ -4739,26 +4496,10 @@ do { \ Uint orig; Uint hole_size; - OpCase(bs_context_to_binary_r): { - context_to_binary_context = x0; - I -= 2; - goto do_context_to_binary; - } - - /* Unfortunately, inlining can generate this instruction. */ - OpCase(bs_context_to_binary_y): { - context_to_binary_context = yb(Arg(0)); - goto do_context_to_binary0; - } - - OpCase(bs_context_to_binary_x): { + OpCase(bs_context_to_binary_x): context_to_binary_context = xb(Arg(0)); - - do_context_to_binary0: I--; - } - do_context_to_binary: if (is_boxed(context_to_binary_context) && header_is_bin_matchstate(*boxed_val(context_to_binary_context))) { ErlBinMatchState* ms; @@ -4770,17 +4511,11 @@ do { \ } Next(2); - OpCase(i_bs_get_binary_all_reuse_rfI): { - context_to_binary_context = x0; - goto do_bs_get_binary_all_reuse; - } - OpCase(i_bs_get_binary_all_reuse_xfI): { context_to_binary_context = xb(Arg(0)); I++; } - do_bs_get_binary_all_reuse: mb = ms_matchbuffer(context_to_binary_context); size = mb->size - mb->offset; if (size % Arg(1) != 0) { @@ -4808,16 +4543,11 @@ do { \ { Eterm match_string_context; - OpCase(i_bs_match_string_rfII): { - match_string_context = r(0); - goto do_bs_match_string; - } OpCase(i_bs_match_string_xfII): { match_string_context = xb(Arg(0)); I++; } - do_bs_match_string: { BeamInstr *next; byte* bytes; @@ -4845,14 +4575,6 @@ do { \ } } - OpCase(i_bs_save2_rI): { - BeamInstr *next; - ErlBinMatchState *_ms; - PreFetch(1, next); - _ms = (ErlBinMatchState*) boxed_val((Eterm) r(0)); - _ms->save_offset[Arg(0)] = _ms->mb.offset; - NextPF(1, next); - } OpCase(i_bs_save2_xI): { BeamInstr *next; ErlBinMatchState *_ms; @@ -4862,14 +4584,6 @@ do { \ NextPF(2, next); } - OpCase(i_bs_restore2_rI): { - BeamInstr *next; - ErlBinMatchState *_ms; - PreFetch(1, next); - _ms = (ErlBinMatchState*) boxed_val((Eterm) r(0)); - _ms->mb.offset = _ms->save_offset[Arg(0)]; - NextPF(1, next); - } OpCase(i_bs_restore2_xI): { BeamInstr *next; ErlBinMatchState *_ms; @@ -4917,9 +4631,7 @@ do { \ BeamInstr real_I; ASSERT(I[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); SWAPOUT; - reg[0] = r(0); real_I = erts_generic_breakpoint(c_p, I, reg); - r(0) = reg[0]; SWAPIN; ASSERT(VALID_INSTR(real_I)); Goto(real_I); @@ -4979,7 +4691,7 @@ do { \ BeamInstr *next; PreFetch(2, next); - GetR(0, targ1); + targ1 = REG_TARGET(Arg(0)); /* Arg(0) == HEADER_FLONUM */ GET_DOUBLE(targ1, *(FloatDef*)ADD_BYTE_OFFSET(freg, fr)); NextPF(2, next); @@ -4999,7 +4711,7 @@ do { \ Eterm fr = Arg(1); BeamInstr *next; - GetR(0, targ1); + targ1 = REG_TARGET(Arg(0)); PreFetch(2, next); if (is_small(targ1)) { fb(fr) = (double) signed_val(targ1); @@ -5095,7 +4807,12 @@ do { \ #ifdef HIPE { - unsigned cmd; +#define HIPE_MODE_SWITCH(Cmd) \ + SWAPOUT; \ + c_p->fcalls = FCALLS; \ + c_p->def_arg_reg[4] = -neg_o_reds; \ + c_p = hipe_mode_switch(c_p, Cmd, reg); \ + goto L_post_hipe_mode_switch OpCase(hipe_trap_call): { /* @@ -5109,52 +4826,45 @@ do { \ */ ASSERT(I[-5] == (Uint) OpCode(i_func_info_IaaI)); c_p->hipe.u.ncallee = (void(*)(void)) I[-4]; - cmd = HIPE_MODE_SWITCH_CMD_CALL | (I[-1] << 8); ++hipe_trap_count; - goto L_hipe_mode_switch; + HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_CALL | (I[-1] << 8)); } OpCase(hipe_trap_call_closure): { ASSERT(I[-5] == (Uint) OpCode(i_func_info_IaaI)); c_p->hipe.u.ncallee = (void(*)(void)) I[-4]; - cmd = HIPE_MODE_SWITCH_CMD_CALL_CLOSURE | (I[-1] << 8); ++hipe_trap_count; - goto L_hipe_mode_switch; + HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_CALL_CLOSURE | (I[-1] << 8)); } OpCase(hipe_trap_return): { - cmd = HIPE_MODE_SWITCH_CMD_RETURN; - goto L_hipe_mode_switch; + HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_RETURN); } OpCase(hipe_trap_throw): { - cmd = HIPE_MODE_SWITCH_CMD_THROW; - goto L_hipe_mode_switch; + HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_THROW); } OpCase(hipe_trap_resume): { - cmd = HIPE_MODE_SWITCH_CMD_RESUME; - goto L_hipe_mode_switch; + HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_RESUME); } - L_hipe_mode_switch: - /* XXX: this abuse of def_arg_reg[] is horrid! */ - SWAPOUT; - c_p->fcalls = FCALLS; - c_p->def_arg_reg[4] = -neg_o_reds; - reg[0] = r(0); - c_p = hipe_mode_switch(c_p, cmd, reg); +#undef HIPE_MODE_SWITCH + + L_post_hipe_mode_switch: reg = ERTS_PROC_GET_SCHDATA(c_p)->x_reg_array; freg = ERTS_PROC_GET_SCHDATA(c_p)->f_reg_array; ERL_BITS_RELOAD_STATEP(c_p); + /* XXX: this abuse of def_arg_reg[] is horrid! */ neg_o_reds = -c_p->def_arg_reg[4]; FCALLS = c_p->fcalls; SWAPIN; - switch( c_p->def_arg_reg[3] ) { /* Halfword wont work with hipe yet! */ + switch( c_p->def_arg_reg[3] ) { case HIPE_MODE_SWITCH_RES_RETURN: ASSERT(is_value(reg[0])); - MoveReturn(reg[0], r(0)); + SET_I(c_p->cp); + c_p->cp = 0; + Goto(*I); case HIPE_MODE_SWITCH_RES_CALL_EXPORTED: c_p->i = c_p->hipe.u.callee_exp->addressv[erts_active_code_ix()]; /*fall through*/ case HIPE_MODE_SWITCH_RES_CALL_BEAM: SET_I(c_p->i); - r(0) = reg[0]; Dispatch(); case HIPE_MODE_SWITCH_RES_CALL_CLOSURE: /* This can be used to call any function value, but currently it's @@ -5163,9 +4873,8 @@ do { \ BeamInstr *next; next = call_fun(c_p, c_p->arity - 1, reg, THE_NON_VALUE); - SWAPIN; + HEAVY_SWAPIN; if (next != NULL) { - r(0) = reg[0]; SET_I(next); Dispatchfun(); } @@ -5213,22 +4922,54 @@ do { \ } OpCase(i_hibernate): { - SWAPOUT; + HEAVY_SWAPOUT; if (erts_hibernate(c_p, r(0), x(1), x(2), reg)) { + FCALLS = c_p->fcalls; c_p->flags &= ~F_HIBERNATE_SCHED; goto do_schedule; } else { + HEAVY_SWAPIN; I = handle_error(c_p, I, reg, hibernate_3); goto post_error_handling; } } + /* This is optimised as an instruction because + it has to be very very fast */ + OpCase(i_perf_counter): { + BeamInstr* next; + ErtsSysPerfCounter ts; + PreFetch(0, next); + + ts = erts_sys_perf_counter(); + + if (IS_SSMALL(ts)) { + r(0) = make_small((Sint)ts); + } else { + TestHeap(ERTS_SINT64_HEAP_SIZE(ts),0); + r(0) = make_big(HTOP); +#if defined(ARCH_32) || HALFWORD_HEAP + if (ts >= (((Uint64) 1) << 32)) { + *HTOP = make_pos_bignum_header(2); + BIG_DIGIT(HTOP, 0) = (Uint) (ts & ((Uint) 0xffffffff)); + BIG_DIGIT(HTOP, 1) = (Uint) ((ts >> 32) & ((Uint) 0xffffffff)); + HTOP += 3; + } + else +#endif + { + *HTOP = make_pos_bignum_header(1); + BIG_DIGIT(HTOP, 0) = (Uint) ts; + HTOP += 2; + } + } + NextPF(0, next); + } + OpCase(i_debug_breakpoint): { - SWAPOUT; - reg[0] = r(0); + HEAVY_SWAPOUT; I = call_error_handler(c_p, I-3, reg, am_breakpoint); - r(0) = reg[0]; - SWAPIN; + HEAVY_SWAPIN; if (I) { Goto(*I); } @@ -6439,7 +6180,7 @@ call_fun(Process* p, /* Current process. */ */ module = fe->module; if ((modp = erts_get_module(module, code_ix)) != NULL - && modp->curr.code != NULL) { + && modp->curr.code_hdr != NULL) { /* * There is a module loaded, but obviously the fun is not * defined in it. We must not call the error_handler @@ -6644,23 +6385,20 @@ static Eterm get_map_element_hash(Eterm map, Eterm key, Uint32 hx) return vs ? *vs : THE_NON_VALUE; } -#define GET_TERM(term, dest) \ -do { \ - Eterm src = (Eterm)(term); \ - switch (src & _TAG_IMMED1_MASK) { \ - case (R_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER: \ - dest = x(0); \ - break; \ - case (X_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER: \ - dest = x(src >> _TAG_IMMED1_SIZE); \ - break; \ - case (Y_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER: \ - dest = y(src >> _TAG_IMMED1_SIZE); \ - break; \ - default: \ - dest = src; \ - break; \ - } \ +#define GET_TERM(term, dest) \ +do { \ + Eterm src = (Eterm)(term); \ + switch (loader_tag(src)) { \ + case LOADER_X_REG: \ + dest = x(loader_x_reg_index(src)); \ + break; \ + case LOADER_Y_REG: \ + dest = y(loader_y_reg_index(src)); \ + break; \ + default: \ + dest = src; \ + break; \ + } \ } while(0) @@ -6699,13 +6437,6 @@ new_map(Process* p, Eterm* reg, BeamInstr* I) erts_factory_proc_init(&factory, p); res = erts_hashmap_from_array(&factory, thp, n/2, 0); erts_factory_close(&factory); - if (p->mbuf) { - Uint live = Arg(2); - reg[live] = res; - erts_garbage_collect(p, 0, reg, live+1); - res = reg[live]; - E = p->stop; - } return res; } @@ -6771,13 +6502,6 @@ update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I) hx = hashmap_make_hash(new_key); res = erts_hashmap_insert(p, hx, new_key, val, res, 0); - if (p->mbuf) { - Uint live = Arg(3); - reg[live] = res; - erts_garbage_collect(p, 0, reg, live+1); - res = reg[live]; - E = p->stop; - } new_p += 2; } @@ -6937,12 +6661,6 @@ update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I) /* The expensive case, need to build a hashmap */ if (n > MAP_SMALL_MAP_LIMIT) { res = erts_hashmap_from_ks_and_vs(p,flatmap_get_keys(mp),flatmap_get_values(mp),n); - if (p->mbuf) { - Uint live = Arg(3); - reg[live] = res; - erts_garbage_collect(p, 0, reg, live+1); - res = reg[live]; - } } return res; } @@ -6998,14 +6716,6 @@ update_map_exact(Process* p, Eterm* reg, Eterm map, BeamInstr* I) return res; } - if (p->mbuf) { - Uint live = Arg(3); - reg[live] = res; - erts_garbage_collect(p, 0, reg, live+1); - res = reg[live]; - E = p->stop; - } - new_p += 2; } return res; @@ -7114,6 +6824,15 @@ erts_is_builtin(Eterm Mod, Eterm Name, int arity) Export e; Export* ep; + if (Mod == am_erlang && Name == am_apply && arity == 3) { + /* + * Special case. apply/3 is built-in (implemented in C), + * but implemented in a different way than all other + * BIFs. + */ + return 1; + } + e.code[0] = Mod; e.code[1] = Name; e.code[2] = arity; diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index a6dce2d1d2..1511a4f935 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -80,7 +80,7 @@ ErlDrvBinary* erts_gzinflate_buffer(char*, int); typedef struct { Uint value; /* Value of label (NULL if not known yet). */ - Uint patches; /* Index (into code buffer) to first location + Sint patches; /* Index (into code buffer) to first location * which must be patched with the value of this label. */ #ifdef ERTS_SMP @@ -284,9 +284,10 @@ typedef struct LoaderState { int specific_op; /* Specific opcode (-1 if not found). */ int num_functions; /* Number of functions in module. */ int num_labels; /* Number of labels. */ - int code_buffer_size; /* Size of code buffer in words. */ - BeamInstr* code; /* Loaded code. */ - int ci; /* Current index into loaded code. */ + BeamCodeHeader* hdr; /* Loaded code header */ + BeamInstr* codev; /* Loaded code buffer */ + int codev_size; /* Size of code buffer in words. */ + int ci; /* Current index into loaded code buffer. */ Label* labels; StringPatch* string_patches; /* Linked list of position into string table to patch. */ BeamInstr catches; /* Linked list of catch_yf instructions. */ @@ -480,7 +481,7 @@ static void free_literal_fragment(ErlHeapFragment*); static void loader_state_dtor(Binary* magic); static Eterm insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, Eterm group_leader, Eterm module, - BeamInstr* code, Uint size); + BeamCodeHeader* code, Uint size); static int init_iff_file(LoaderState* stp, byte* code, Uint size); static int scan_iff_file(LoaderState* stp, Uint* chunk_types, Uint num_types, Uint num_mandatory); @@ -526,15 +527,15 @@ static void new_string_patch(LoaderState* stp, int pos); static Uint new_literal(LoaderState* stp, Eterm** hpp, Uint heap_size); static int genopargcompare(GenOpArg* a, GenOpArg* b); static Eterm get_module_info(Process* p, ErtsCodeIndex code_ix, - BeamInstr* code, Eterm module, Eterm what); + BeamCodeHeader*, Eterm module, Eterm what); static Eterm exported_from_module(Process* p, ErtsCodeIndex code_ix, Eterm mod); -static Eterm functions_in_module(Process* p, BeamInstr* code); -static Eterm attributes_for_module(Process* p, BeamInstr* code); -static Eterm compilation_info_for_module(Process* p, BeamInstr* code); -static Eterm md5_of_module(Process* p, BeamInstr* code); -static Eterm has_native(BeamInstr* code); -static Eterm native_addresses(Process* p, BeamInstr* code); +static Eterm functions_in_module(Process* p, BeamCodeHeader*); +static Eterm attributes_for_module(Process* p, BeamCodeHeader*); +static Eterm compilation_info_for_module(Process* p, BeamCodeHeader*); +static Eterm md5_of_module(Process* p, BeamCodeHeader*); +static Eterm has_native(BeamCodeHeader*); +static Eterm native_addresses(Process* p, BeamCodeHeader*); int patch_funentries(Eterm Patchlist); int patch(Eterm Addresses, Uint fe); static int safe_mul(UWord a, UWord b, UWord* resp); @@ -601,6 +602,7 @@ extern void check_allocated_block(Uint type, void *blk); #define CHKBLK(TYPE,BLK) /* nothing */ #endif + Eterm erts_prepare_loading(Binary* magic, Process *c_p, Eterm group_leader, Eterm* modp, byte* code, Uint unloaded_size) @@ -641,20 +643,27 @@ erts_prepare_loading(Binary* magic, Process *c_p, Eterm group_leader, /* * Initialize code area. */ - stp->code_buffer_size = 2048 + stp->num_functions; - stp->code = (BeamInstr *) erts_alloc(ERTS_ALC_T_CODE, - sizeof(BeamInstr) * stp->code_buffer_size); + stp->codev_size = 2048 + stp->num_functions; + stp->hdr = (BeamCodeHeader*) erts_alloc(ERTS_ALC_T_CODE, + (offsetof(BeamCodeHeader,functions) + + sizeof(BeamInstr) * stp->codev_size)); - stp->code[MI_NUM_FUNCTIONS] = stp->num_functions; - stp->ci = MI_FUNCTIONS + stp->num_functions + 1; + stp->hdr->num_functions = stp->num_functions; - stp->code[MI_ATTR_PTR] = 0; - stp->code[MI_ATTR_SIZE] = 0; - stp->code[MI_ATTR_SIZE_ON_HEAP] = 0; - stp->code[MI_COMPILE_PTR] = 0; - stp->code[MI_COMPILE_SIZE] = 0; - stp->code[MI_COMPILE_SIZE_ON_HEAP] = 0; - stp->code[MI_MD5_PTR] = 0; + /* Let the codev array start at functions[0] in order to index + * both function pointers and the loaded code itself that follows. + */ + stp->codev = (BeamInstr*) &stp->hdr->functions; + stp->ci = stp->num_functions + 1; + + stp->hdr->attr_ptr = NULL; + stp->hdr->attr_size = 0; + stp->hdr->attr_size_on_heap = 0; + stp->hdr->compile_ptr = NULL; + stp->hdr->compile_size = 0; + stp->hdr->compile_size_on_heap = 0; + stp->hdr->literals_start = NULL; + stp->hdr->md5_ptr = NULL; /* * Read the atom table. @@ -776,7 +785,7 @@ erts_finish_loading(Binary* magic, Process* c_p, CHKBLK(ERTS_ALC_T_CODE,stp->code); retval = insert_new_code(c_p, c_p_locks, stp->group_leader, stp->module, - stp->code, stp->loaded_size); + stp->hdr, stp->loaded_size); if (retval != NIL) { goto load_error; } @@ -799,7 +808,8 @@ erts_finish_loading(Binary* magic, Process* c_p, debug_dump_code(stp->code,stp->ci); #endif #endif - stp->code = NULL; /* Prevent code from being freed. */ + stp->hdr = NULL; /* Prevent code from being freed. */ + stp->codev = NULL; *modp = stp->module; /* @@ -831,7 +841,8 @@ erts_alloc_loader_state(void) stp->specific_op = -1; stp->genop = NULL; stp->atom = NULL; - stp->code = NULL; + stp->hdr = NULL; + stp->codev = NULL; stp->labels = NULL; stp->import = NULL; stp->export = NULL; @@ -870,7 +881,7 @@ erts_module_for_prepared_code(Binary* magic) return NIL; } stp = ERTS_MAGIC_BIN_DATA(magic); - if (stp->code != 0) { + if (stp->hdr != 0) { return stp->module; } else { return NIL; @@ -891,7 +902,7 @@ static ErlHeapFragment* new_literal_fragment(Uint size) ErlHeapFragment* bp; bp = (ErlHeapFragment*) ERTS_HEAP_ALLOC(ERTS_ALC_T_PREPARED_CODE, ERTS_HEAP_FRAG_SIZE(size)); - ERTS_INIT_HEAP_FRAG(bp, size); + ERTS_INIT_HEAP_FRAG(bp, size, size); return bp; } @@ -920,9 +931,13 @@ loader_state_dtor(Binary* magic) driver_free_binary(stp->bin); stp->bin = 0; } - if (stp->code != 0) { - erts_free(ERTS_ALC_T_CODE, stp->code); - stp->code = 0; + if (stp->hdr != 0) { + if (stp->hdr->literals_start) { + erts_free(ERTS_ALC_T_LITERAL, stp->hdr->literals_start); + } + erts_free(ERTS_ALC_T_CODE, stp->hdr); + stp->hdr = 0; + stp->codev = 0; } if (stp->labels != 0) { erts_free(ERTS_ALC_T_PREPARED_CODE, (void *) stp->labels); @@ -995,7 +1010,7 @@ loader_state_dtor(Binary* magic) static Eterm insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, - Eterm group_leader, Eterm module, BeamInstr* code, + Eterm group_leader, Eterm module, BeamCodeHeader* code_hdr, Uint size) { Module* modp; @@ -1016,7 +1031,7 @@ insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, erts_total_code_size += size; modp = erts_put_module(module); - modp->curr.code = code; + modp->curr.code_hdr = code_hdr; modp->curr.code_length = size; modp->curr.catches = BEAM_CATCHES_NIL; /* Will be filled in later. */ @@ -1024,7 +1039,7 @@ insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, * Update ranges (used for finding a function from a PC value). */ - erts_update_ranges(code, size); + erts_update_ranges((BeamInstr*)modp->curr.code_hdr, size); return NIL; } @@ -1373,7 +1388,7 @@ read_export_table(LoaderState* stp) if (value == 0) { LoadError2(stp, "export table entry %d: label %d not resolved", i, n); } - stp->export[i].address = address = stp->code + value; + stp->export[i].address = address = stp->codev + value; /* * Find out if there is a BIF with the same name. @@ -1392,7 +1407,7 @@ read_export_table(LoaderState* stp) * any other functions that walk through all local functions. */ - if (stp->labels[n].patches) { + if (stp->labels[n].patches >= 0) { LoadError3(stp, "there are local calls to the stub for " "the BIF %T:%T/%d", stp->module, func, arity); @@ -1513,8 +1528,8 @@ read_literal_table(LoaderState* stp) } if (heap_size > 0) { - erts_factory_message_init(&factory, NULL, NULL, - new_literal_fragment(heap_size)); + erts_factory_heap_frag_init(&factory, + new_literal_fragment(heap_size)); factory.alloc_type = ERTS_ALC_T_PREPARED_CODE; val = erts_decode_ext(&factory, &p); @@ -1735,7 +1750,7 @@ read_code_header(LoaderState* stp) stp->num_labels * sizeof(Label)); for (i = 0; i < stp->num_labels; i++) { stp->labels[i].value = 0; - stp->labels[i].patches = 0; + stp->labels[i].patches = -1; #ifdef ERTS_SMP stp->labels[i].looprec_targeted = 0; #endif @@ -1754,13 +1769,14 @@ read_code_header(LoaderState* stp) } else {} #define CodeNeed(w) do { \ - ASSERT(ci <= code_buffer_size); \ - if (code_buffer_size < ci+(w)) { \ - code_buffer_size = 2*ci+(w); \ - stp->code = code = \ - (BeamInstr *) erts_realloc(ERTS_ALC_T_CODE, \ - (void *) code, \ - code_buffer_size * sizeof(BeamInstr)); \ + ASSERT(ci <= codev_size); \ + if (codev_size < ci+(w)) { \ + codev_size = 2*ci+(w); \ + stp->hdr = (BeamCodeHeader*) erts_realloc(ERTS_ALC_T_CODE, \ + (void *) stp->hdr, \ + (offsetof(BeamCodeHeader,functions) \ + + codev_size * sizeof(BeamInstr))); \ + code = stp->codev = (BeamInstr*) &stp->hdr->functions; \ } \ } while (0) @@ -1776,7 +1792,7 @@ load_code(LoaderState* stp) int arg; /* Number of current argument. */ int num_specific; /* Number of specific ops for current. */ BeamInstr* code; - int code_buffer_size; + int codev_size; int specific; Uint last_label = 0; /* Number of last label. */ Uint function_number = 0; @@ -1793,15 +1809,15 @@ load_code(LoaderState* stp) FUNC_INFO_SZ = 5 }; - code = stp->code; - code_buffer_size = stp->code_buffer_size; + code = stp->codev; + codev_size = stp->codev_size; ci = stp->ci; for (;;) { int new_op; GenOp* tmp_op; - ASSERT(ci <= code_buffer_size); + ASSERT(ci <= codev_size); get_next_instr: GetByte(stp, new_op); @@ -1847,9 +1863,7 @@ load_code(LoaderState* stp) case TAG_o: break; case TAG_x: - if (last_op->a[arg].val == 0) { - last_op->a[arg].type = TAG_r; - } else if (last_op->a[arg].val >= MAX_REG) { + if (last_op->a[arg].val >= MAX_REG) { LoadError1(stp, "invalid x register number: %u", last_op->a[arg].val); } @@ -1894,15 +1908,14 @@ load_code(LoaderState* stp) */ { Eterm* hp; -/* XXX:PaN - Halfword should use ARCH_64 variant instead */ -#if !defined(ARCH_64) || HALFWORD_HEAP +#if !defined(ARCH_64) Uint high, low; # endif last_op->a[arg].val = new_literal(stp, &hp, FLOAT_SIZE_OBJECT); hp[0] = HEADER_FLONUM; last_op->a[arg].type = TAG_q; -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) GetInt(stp, 8, hp[1]); # else GetInt(stp, 4, high); @@ -2056,7 +2069,42 @@ load_code(LoaderState* stp) if (((opc[specific].mask[0] & mask[0]) == mask[0]) && ((opc[specific].mask[1] & mask[1]) == mask[1]) && ((opc[specific].mask[2] & mask[2]) == mask[2])) { - break; + + if (!opc[specific].involves_r) { + break; /* No complications - match */ + } + + /* + * The specific operation uses the 'r' operand, + * which is shorthand for x(0). Now things + * get complicated. First we must check whether + * all operands that should be of type 'r' use + * x(0) (as opposed to some other X register). + */ + for (arg = 0; arg < arity; arg++) { + if (opc[specific].involves_r & (1 << arg) && + tmp_op->a[arg].type == TAG_x) { + if (tmp_op->a[arg].val != 0) { + break; /* Other X register than 0 */ + } + } + } + + if (arg == arity) { + /* + * All 'r' operands use x(0) in the generic + * operation. That means a match. Now we + * will need to rewrite the generic instruction + * to actually use 'r' instead of 'x(0)'. + */ + for (arg = 0; arg < arity; arg++) { + if (opc[specific].involves_r & (1 << arg) && + tmp_op->a[arg].type == TAG_x) { + tmp_op->a[arg].type = TAG_r; + } + } + break; /* Match */ + } } specific++; } @@ -2167,14 +2215,11 @@ load_code(LoaderState* stp) break; case 's': /* Any source (tagged constant or register) */ switch (tag) { - case TAG_r: - code[ci++] = make_rreg(); - break; case TAG_x: - code[ci++] = make_xreg(tmp_op->a[arg].val); + code[ci++] = make_loader_x_reg(tmp_op->a[arg].val); break; case TAG_y: - code[ci++] = make_yreg(tmp_op->a[arg].val); + code[ci++] = make_loader_y_reg(tmp_op->a[arg].val); break; case TAG_i: code[ci++] = (BeamInstr) make_small((Uint)tmp_op->a[arg].val); @@ -2185,6 +2230,10 @@ load_code(LoaderState* stp) case TAG_n: code[ci++] = NIL; break; + case TAG_q: + new_literal_patch(stp, ci); + code[ci++] = tmp_op->a[arg].val; + break; default: LoadError1(stp, "bad tag %d for general source", tmp_op->a[arg].type); @@ -2193,14 +2242,11 @@ load_code(LoaderState* stp) break; case 'd': /* Destination (x(0), x(N), y(N) */ switch (tag) { - case TAG_r: - code[ci++] = make_rreg(); - break; case TAG_x: - code[ci++] = make_xreg(tmp_op->a[arg].val); + code[ci++] = tmp_op->a[arg].val * sizeof(Eterm); break; case TAG_y: - code[ci++] = make_yreg(tmp_op->a[arg].val); + code[ci++] = tmp_op->a[arg].val * sizeof(Eterm) + 1; break; default: LoadError1(stp, "bad tag %d for destination", @@ -2357,20 +2403,13 @@ load_code(LoaderState* stp) stp->labels[tmp_op->a[arg].val].patches = ci; ci++; break; - case TAG_r: - CodeNeed(1); - code[ci++] = (R_REG_DEF << _TAG_PRIMARY_SIZE) | - TAG_PRIMARY_HEADER; - break; case TAG_x: CodeNeed(1); - code[ci++] = (tmp_op->a[arg].val << _TAG_IMMED1_SIZE) | - (X_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER; + code[ci++] = make_loader_x_reg(tmp_op->a[arg].val); break; case TAG_y: CodeNeed(1); - code[ci++] = (tmp_op->a[arg].val << _TAG_IMMED1_SIZE) | - (Y_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER; + code[ci++] = make_loader_y_reg(tmp_op->a[arg].val); break; case TAG_n: CodeNeed(1); @@ -2393,8 +2432,7 @@ load_code(LoaderState* stp) switch (stp->specific_op) { case op_i_func_info_IaaI: { - Uint offset; - + Sint offset; if (function_number >= stp->num_functions) { LoadError1(stp, "too many functions in module (header said %d)", stp->num_functions); @@ -2436,15 +2474,15 @@ load_code(LoaderState* stp) stp->arity = code[ci-1]; ASSERT(stp->labels[last_label].value == ci - FUNC_INFO_SZ); - offset = MI_FUNCTIONS + function_number; - code[offset] = stp->labels[last_label].patches; + stp->hdr->functions[function_number] = (BeamInstr*) stp->labels[last_label].patches; + offset = function_number; stp->labels[last_label].patches = offset; function_number++; if (stp->arity > MAX_ARG) { LoadError1(stp, "too many arguments: %d", stp->arity); } #ifdef DEBUG - ASSERT(stp->labels[0].patches == 0); /* Should not be referenced. */ + ASSERT(stp->labels[0].patches < 0); /* Should not be referenced. */ for (i = 1; i < stp->num_labels; i++) { ASSERT(stp->labels[i].patches < ci); } @@ -2458,7 +2496,6 @@ load_code(LoaderState* stp) stp->on_load = ci; break; case op_bs_put_string_II: - case op_i_bs_match_string_rfII: case op_i_bs_match_string_xfII: new_string_patch(stp, ci-1); break; @@ -2515,7 +2552,7 @@ load_code(LoaderState* stp) * End of code found. */ case op_int_code_end: - stp->code_buffer_size = code_buffer_size; + stp->codev_size = codev_size; stp->ci = ci; stp->function = THE_NON_VALUE; stp->genop = NULL; @@ -2680,6 +2717,12 @@ same_label(LoaderState* stp, GenOpArg Target, GenOpArg Label) Target.val == Label.val; } +static int +is_killed(LoaderState* stp, GenOpArg Reg, GenOpArg Live) +{ + return Reg.type == TAG_x && Live.type == TAG_u && + Live.val <= Reg.val; +} /* * Generate an instruction for element/2. @@ -2696,17 +2739,17 @@ gen_element(LoaderState* stp, GenOpArg Fail, GenOpArg Index, op->next = NULL; if (Index.type == TAG_i && Index.val > 0 && - (Tuple.type == TAG_r || Tuple.type == TAG_x || Tuple.type == TAG_y)) { + (Tuple.type == TAG_x || Tuple.type == TAG_y)) { op->op = genop_i_fast_element_4; - op->a[0] = Tuple; - op->a[1] = Fail; + op->a[0] = Fail; + op->a[1] = Tuple; op->a[2].type = TAG_u; op->a[2].val = Index.val; op->a[3] = Dst; } else { op->op = genop_i_element_4; - op->a[0] = Tuple; - op->a[1] = Fail; + op->a[0] = Fail; + op->a[1] = Tuple; op->a[2] = Index; op->a[3] = Dst; } @@ -2832,23 +2875,16 @@ gen_get_integer2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Live, goto generic; } } else { - GenOp* op2; - NEW_GENOP(stp, op2); - - op->op = genop_i_fetch_2; - op->arity = 2; - op->a[0] = Ms; - op->a[1] = Size; - op->next = op2; - - op2->op = genop_i_bs_get_integer_4; - op2->arity = 4; - op2->a[0] = Fail; - op2->a[1] = Live; - op2->a[2].type = TAG_u; - op2->a[2].val = (Unit.val << 3) | Flags.val; - op2->a[3] = Dst; - op2->next = NULL; + op->op = genop_i_bs_get_integer_6; + op->arity = 6; + op->a[0] = Fail; + op->a[1] = Live; + op->a[2].type = TAG_u; + op->a[2].val = (Unit.val << 3) | Flags.val; + op->a[3] = Ms; + op->a[4] = Size; + op->a[5] = Dst; + op->next = NULL; return op; } op->next = NULL; @@ -3272,14 +3308,14 @@ gen_literal_timeout(LoaderState* stp, GenOpArg Fail, GenOpArg Time) op->a[1].type = TAG_u; if (Time.type == TAG_i && (timeout = Time.val) >= 0 && -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) (timeout >> 32) == 0 #else 1 #endif ) { op->a[1].val = timeout; -#if !defined(ARCH_64) || HALFWORD_HEAP +#if !defined(ARCH_64) } else if (Time.type == TAG_q) { Eterm big; @@ -3296,7 +3332,7 @@ gen_literal_timeout(LoaderState* stp, GenOpArg Fail, GenOpArg Time) } #endif } else { -#if !defined(ARCH_64) || HALFWORD_HEAP +#if !defined(ARCH_64) error: #endif op->op = genop_i_wait_error_0; @@ -3319,14 +3355,14 @@ gen_literal_timeout_locked(LoaderState* stp, GenOpArg Fail, GenOpArg Time) op->a[1].type = TAG_u; if (Time.type == TAG_i && (timeout = Time.val) >= 0 && -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) (timeout >> 32) == 0 #else 1 #endif ) { op->a[1].val = timeout; -#if !defined(ARCH_64) || HALFWORD_HEAP +#if !defined(ARCH_64) } else if (Time.type == TAG_q) { Eterm big; @@ -3343,7 +3379,7 @@ gen_literal_timeout_locked(LoaderState* stp, GenOpArg Fail, GenOpArg Time) } #endif } else { -#if !defined(ARCH_64) || HALFWORD_HEAP +#if !defined(ARCH_64) error: #endif op->op = genop_i_wait_error_locked_0; @@ -3899,9 +3935,7 @@ gen_make_fun2(LoaderState* stp, GenOpArg idx) /* * Rewrite gc_bifs with one parameter (the common case). Utilized * in ops.tab to rewrite instructions calling bif's in guards - * to use a garbage collecting implementation. The instructions - * are sometimes once again rewritten to handle literals (putting the - * parameter in the mostly unused r[0] before the instruction is executed). + * to use a garbage collecting implementation. */ static GenOp* gen_guard_bif1(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, @@ -3956,10 +3990,6 @@ gen_guard_bif1(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, /* * This is used by the ops.tab rule that rewrites gc_bifs with two parameters. - * The instruction returned is then again rewritten to an i_load instruction - * followed by i_gc_bif2_jIId, to handle literals properly. - * As opposed to the i_gc_bif1_jIsId, the instruction i_gc_bif2_jIId is - * always rewritten, regardless of if there actually are any literals. */ static GenOp* gen_guard_bif2(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, @@ -3986,23 +4016,19 @@ gen_guard_bif2(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, op->a[2].val = stp->import[Bif.val].arity; return op; } - op->op = genop_ii_gc_bif2_6; + op->op = genop_i_gc_bif2_6; op->arity = 6; op->a[0] = Fail; op->a[1].type = TAG_u; - op->a[2] = S1; - op->a[3] = S2; - op->a[4] = Live; + op->a[2] = Live; + op->a[3] = S1; + op->a[4] = S2; op->a[5] = Dst; return op; } /* * This is used by the ops.tab rule that rewrites gc_bifs with three parameters. - * The instruction returned is then again rewritten to a move instruction that - * uses r[0] for temp storage, followed by an i_load instruction, - * followed by i_gc_bif3_jIsId, to handle literals properly. Rewriting - * always occur, as with the gc_bif2 counterpart. */ static GenOp* gen_guard_bif3(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, @@ -4033,10 +4059,10 @@ gen_guard_bif3(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, op->arity = 7; op->a[0] = Fail; op->a[1].type = TAG_u; - op->a[2] = S1; - op->a[3] = S2; - op->a[4] = S3; - op->a[5] = Live; + op->a[2] = Live; + op->a[3] = S1; + op->a[4] = S2; + op->a[5] = S3; op->a[6] = Dst; op->next = NULL; return op; @@ -4258,6 +4284,53 @@ gen_get_map_element(LoaderState* stp, GenOpArg Fail, GenOpArg Src, return op; } +static int +hash_internal_genop_arg(LoaderState* stp, GenOpArg Key, Uint32* hx) +{ + switch (Key.type) { + case TAG_a: + *hx = atom_tab(atom_val(Key.val))->slot.bucket.hvalue; + return 1; + case TAG_i: + *hx = Key.val; + return 1; + case TAG_n: + *hx = make_internal_hash(NIL); + return 1; + case TAG_q: + *hx = make_internal_hash(stp->literals[Key.val].term); + return 1; + default: + return 0; + } +} + + +static GenOp* +gen_get(LoaderState* stp, GenOpArg Src, GenOpArg Dst) +{ + GenOp* op; + Uint32 hx = 0; + + NEW_GENOP(stp, op); + op->next = NULL; + if (hash_internal_genop_arg(stp, Src, &hx)) { + op->arity = 3; + op->op = genop_i_get_hash_3; + op->a[0] = Src; + op->a[1].type = TAG_u; + op->a[1].val = (BeamInstr) hx; + op->a[2] = Dst; + } else { + op->arity = 2; + op->op = genop_i_get_2; + op->a[0] = Src; + op->a[1] = Dst; + } + return op; +} + + static GenOp* gen_get_map_elements(LoaderState* stp, GenOpArg Fail, GenOpArg Src, GenOpArg Size, GenOpArg* Rest) @@ -4323,7 +4396,7 @@ gen_has_map_fields(LoaderState* stp, GenOpArg Fail, GenOpArg Src, for (i = 0; i < n; i++) { op->a[3+2*i] = Rest[i]; op->a[3+2*i+1].type = TAG_x; - op->a[3+2*i+1].val = 0; /* x(0); normally not used */ + op->a[3+2*i+1].val = SCRATCH_X_REG; /* Ignore result */ } return op; } @@ -4336,8 +4409,8 @@ gen_has_map_fields(LoaderState* stp, GenOpArg Fail, GenOpArg Src, static int freeze_code(LoaderState* stp) { - BeamInstr* code = stp->code; - Uint *literal_end = NULL; + BeamCodeHeader* code_hdr = stp->hdr; + BeamInstr* codev = (BeamInstr*) &stp->hdr->functions; int i; byte* str_table; unsigned strtab_size = stp->chunks[STR_CHUNK].size; @@ -4362,77 +4435,76 @@ freeze_code(LoaderState* stp) if (stp->line_instr == 0) { line_size = 0; } else { - line_size = (MI_LINE_FUNC_TAB + (stp->num_functions + 1) + - (stp->current_li+1) + stp->num_fnames) * - sizeof(Eterm) + (stp->current_li+1) * stp->loc_size; + line_size = (offsetof(BeamCodeLineTab,func_tab) + + (stp->num_functions + 1) * sizeof(BeamInstr**) /* func_tab */ + + (stp->current_li + 1) * sizeof(BeamInstr*) /* line items */ + + stp->num_fnames * sizeof(Eterm) /* fname table */ + + (stp->current_li + 1) * stp->loc_size); /* loc_tab */ } - size = (stp->ci * sizeof(BeamInstr)) + - (stp->total_literal_size * sizeof(Eterm)) + + size = offsetof(BeamCodeHeader,functions) + (stp->ci * sizeof(BeamInstr)) + strtab_size + attr_size + compile_size + MD5_SIZE + line_size; /* * Move the code to its final location. */ - code = (BeamInstr *) erts_realloc(ERTS_ALC_T_CODE, (void *) code, size); - CHKBLK(ERTS_ALC_T_CODE,code); + code_hdr = (BeamCodeHeader*) erts_realloc(ERTS_ALC_T_CODE, (void *) code_hdr, size); + codev = (BeamInstr*) &code_hdr->functions; + CHKBLK(ERTS_ALC_T_CODE,code_hdr); /* * Place a pointer to the op_int_code_end instruction in the * function table in the beginning of the file. */ - code[MI_FUNCTIONS+stp->num_functions] = (BeamInstr) (code + stp->ci - 1); - CHKBLK(ERTS_ALC_T_CODE,code); + code_hdr->functions[stp->num_functions] = (codev + stp->ci - 1); + CHKBLK(ERTS_ALC_T_CODE,code_hdr); /* * Store the pointer to the on_load function. */ if (stp->on_load) { - code[MI_ON_LOAD_FUNCTION_PTR] = (BeamInstr) (code + stp->on_load); + code_hdr->on_load_function_ptr = codev + stp->on_load; } else { - code[MI_ON_LOAD_FUNCTION_PTR] = 0; + code_hdr->on_load_function_ptr = NULL; } - CHKBLK(ERTS_ALC_T_CODE,code); + CHKBLK(ERTS_ALC_T_CODE,code_hdr); - literal_end = (Uint *) (code+stp->ci); /* - * Place the literal heap directly after the code and fix up all - * instructions that refer to it. + * Place the literals in their own allocated heap (for fast range check) + * and fix up all instructions that refer to it. */ { - Uint* ptr; - Uint* low; - Uint* high; + Eterm* ptr; LiteralPatch* lp; ErlOffHeap code_off_heap; ERTS_INIT_OFF_HEAP(&code_off_heap); - low = (Uint *) (code+stp->ci); - high = low + stp->total_literal_size; - code[MI_LITERALS_START] = (BeamInstr) low; - code[MI_LITERALS_END] = (BeamInstr) high; - ptr = low; + ptr = (Eterm*)erts_alloc(ERTS_ALC_T_LITERAL, + stp->total_literal_size*sizeof(Eterm)); + code_hdr->literals_start = ptr; + code_hdr->literals_end = ptr + stp->total_literal_size; for (i = 0; i < stp->num_literals; i++) { - if (stp->literals[i].heap_frags) { - move_multi_frags(&ptr, &code_off_heap, stp->literals[i].heap_frags, - &stp->literals[i].term, 1); + if (is_not_immed(stp->literals[i].term)) { + erts_move_multi_frags(&ptr, &code_off_heap, + stp->literals[i].heap_frags, + &stp->literals[i].term, 1, 1); + ASSERT(erts_is_literal(stp->literals[i].term, + ptr_val(stp->literals[i].term))); } - else ASSERT(is_immed(stp->literals[i].term)); } - code[MI_LITERALS_OFF_HEAP] = (BeamInstr) code_off_heap.first; + code_hdr->literals_off_heap = code_off_heap.first; lp = stp->literal_patches; while (lp != 0) { BeamInstr* op_ptr; Literal* lit; - op_ptr = code + lp->pos; + op_ptr = codev + lp->pos; lit = &stp->literals[op_ptr[0]]; op_ptr[0] = lit->term; lp = lp->next; } - literal_end += stp->total_literal_size; } CHKBLK(ERTS_ALC_T_CODE,code); @@ -4440,52 +4512,49 @@ freeze_code(LoaderState* stp) * If there is line information, place it here. */ if (stp->line_instr == 0) { - code[MI_LINE_TABLE] = (BeamInstr) 0; - str_table = (byte *) literal_end; + code_hdr->line_table = NULL; + str_table = (byte *) (codev + stp->ci); } else { - Eterm* line_tab = (Eterm *) literal_end; - Eterm* p; - int ftab_size = stp->num_functions; - int num_instrs = stp->current_li; - Eterm* first_line_item; + BeamCodeLineTab* const line_tab = (BeamCodeLineTab *) (codev+stp->ci); + const int ftab_size = stp->num_functions; + const int num_instrs = stp->current_li; + const BeamInstr** const line_items = + (const BeamInstr**) &line_tab->func_tab[ftab_size + 1]; - code[MI_LINE_TABLE] = (BeamInstr) line_tab; - p = line_tab + MI_LINE_FUNC_TAB; + code_hdr->line_table = line_tab; - first_line_item = (p + ftab_size + 1); for (i = 0; i < ftab_size; i++) { - *p++ = (Eterm) (BeamInstr) (first_line_item + stp->func_line[i]); + line_tab->func_tab[i] = line_items + stp->func_line[i]; } - *p++ = (Eterm) (BeamInstr) (first_line_item + num_instrs); - ASSERT(p == first_line_item); + line_tab->func_tab[i] = line_items + num_instrs; + for (i = 0; i < num_instrs; i++) { - *p++ = (Eterm) (BeamInstr) (code + stp->line_instr[i].pos); + line_items[i] = codev + stp->line_instr[i].pos; } - *p++ = (Eterm) (BeamInstr) (code + stp->ci - 1); + line_items[i] = codev + stp->ci - 1; - line_tab[MI_LINE_FNAME_PTR] = (Eterm) (BeamInstr) p; - memcpy(p, stp->fname, stp->num_fnames*sizeof(Eterm)); - p += stp->num_fnames; + line_tab->fname_ptr = (Eterm*) &line_items[i + 1]; + memcpy(line_tab->fname_ptr, stp->fname, stp->num_fnames*sizeof(Eterm)); - line_tab[MI_LINE_LOC_TAB] = (Eterm) (BeamInstr) p; - line_tab[MI_LINE_LOC_SIZE] = stp->loc_size; + line_tab->loc_size = stp->loc_size; if (stp->loc_size == 2) { - Uint16* locp = (Uint16 *) p; - for (i = 0; i < num_instrs; i++) { + Uint16* locp = (Uint16 *) &line_tab->fname_ptr[stp->num_fnames]; + line_tab->loc_tab.p2 = locp; + for (i = 0; i < num_instrs; i++) { *locp++ = (Uint16) stp->line_instr[i].loc; - } - *locp++ = LINE_INVALID_LOCATION; - str_table = (byte *) locp; + } + *locp++ = LINE_INVALID_LOCATION; + str_table = (byte *) locp; } else { - Uint32* locp = (Uint32 *) p; - ASSERT(stp->loc_size == 4); + Uint32* locp = (Uint32 *) &line_tab->fname_ptr[stp->num_fnames]; + ASSERT(stp->loc_size == 4); + line_tab->loc_tab.p4 = locp; for (i = 0; i < num_instrs; i++) { *locp++ = stp->line_instr[i].loc; } *locp++ = LINE_INVALID_LOCATION; - str_table = (byte *) locp; + str_table = (byte *) locp; } - CHKBLK(ERTS_ALC_T_CODE,code); } @@ -4497,13 +4566,13 @@ freeze_code(LoaderState* stp) if (attr_size) { byte* attr = str_table + strtab_size; sys_memcpy(attr, stp->chunks[ATTR_CHUNK].start, stp->chunks[ATTR_CHUNK].size); - code[MI_ATTR_PTR] = (BeamInstr) attr; - code[MI_ATTR_SIZE] = (BeamInstr) stp->chunks[ATTR_CHUNK].size; + code_hdr->attr_ptr = attr; + code_hdr->attr_size = (BeamInstr) stp->chunks[ATTR_CHUNK].size; decoded_size = erts_decode_ext_size(attr, attr_size); if (decoded_size < 0) { LoadError0(stp, "bad external term representation of module attributes"); } - code[MI_ATTR_SIZE_ON_HEAP] = decoded_size; + code_hdr->attr_size_on_heap = decoded_size; } CHKBLK(ERTS_ALC_T_CODE,code); if (compile_size) { @@ -4513,9 +4582,9 @@ freeze_code(LoaderState* stp) stp->chunks[COMPILE_CHUNK].size); CHKBLK(ERTS_ALC_T_CODE,code); - code[MI_COMPILE_PTR] = (BeamInstr) compile_info; + code_hdr->compile_ptr = compile_info; CHKBLK(ERTS_ALC_T_CODE,code); - code[MI_COMPILE_SIZE] = (BeamInstr) stp->chunks[COMPILE_CHUNK].size; + code_hdr->compile_size = (BeamInstr) stp->chunks[COMPILE_CHUNK].size; CHKBLK(ERTS_ALC_T_CODE,code); decoded_size = erts_decode_ext_size(compile_info, compile_size); CHKBLK(ERTS_ALC_T_CODE,code); @@ -4523,7 +4592,7 @@ freeze_code(LoaderState* stp) LoadError0(stp, "bad external term representation of compilation information"); } CHKBLK(ERTS_ALC_T_CODE,code); - code[MI_COMPILE_SIZE_ON_HEAP] = decoded_size; + code_hdr->compile_size_on_heap = decoded_size; } CHKBLK(ERTS_ALC_T_CODE,code); { @@ -4531,7 +4600,7 @@ freeze_code(LoaderState* stp) CHKBLK(ERTS_ALC_T_CODE,code); sys_memcpy(md5_sum, stp->mod_md5, MD5_SIZE); CHKBLK(ERTS_ALC_T_CODE,code); - code[MI_MD5_PTR] = (BeamInstr) md5_sum; + code_hdr->md5_ptr = md5_sum; CHKBLK(ERTS_ALC_T_CODE,code); } CHKBLK(ERTS_ALC_T_CODE,code); @@ -4540,7 +4609,7 @@ freeze_code(LoaderState* stp) * Make sure that we have not overflowed the allocated code space. */ ASSERT(str_table + strtab_size + attr_size + compile_size + MD5_SIZE == - ((byte *) code) + size); + ((byte *) code_hdr) + size); /* * Patch all instructions that refer to the string table. @@ -4552,46 +4621,47 @@ freeze_code(LoaderState* stp) BeamInstr* op_ptr; byte* strp; - op_ptr = code + sp->pos; + op_ptr = codev + sp->pos; strp = str_table + op_ptr[0]; op_ptr[0] = (BeamInstr) strp; sp = sp->next; } } - CHKBLK(ERTS_ALC_T_CODE,code); + CHKBLK(ERTS_ALC_T_CODE,code_hdr); /* * Resolve all labels. */ for (i = 0; i < stp->num_labels; i++) { - Uint this_patch; - Uint next_patch; + Sint this_patch; + Sint next_patch; Uint value = stp->labels[i].value; - if (value == 0 && stp->labels[i].patches != 0) { + if (value == 0 && stp->labels[i].patches >= 0) { LoadError1(stp, "label %d not resolved", i); } ASSERT(value < stp->ci); this_patch = stp->labels[i].patches; - while (this_patch != 0) { + while (this_patch >= 0) { ASSERT(this_patch < stp->ci); - next_patch = code[this_patch]; + next_patch = codev[this_patch]; ASSERT(next_patch < stp->ci); - code[this_patch] = (BeamInstr) (code + value); + codev[this_patch] = (BeamInstr) (codev + value); this_patch = next_patch; } } - CHKBLK(ERTS_ALC_T_CODE,code); + CHKBLK(ERTS_ALC_T_CODE,code_hdr); /* * Save the updated code pointer and code size. */ - stp->code = code; + stp->hdr = code_hdr; + stp->codev = codev; stp->loaded_size = size; - CHKBLK(ERTS_ALC_T_CODE,code); + CHKBLK(ERTS_ALC_T_CODE,code_hdr); return 1; load_error: @@ -4599,7 +4669,8 @@ freeze_code(LoaderState* stp) * Make sure that the caller frees the newly reallocated block, and * not the old one (in case it has moved). */ - stp->code = code; + stp->hdr = code_hdr; + stp->codev = codev; return 0; } @@ -4610,7 +4681,7 @@ final_touch(LoaderState* stp) int on_load = stp->on_load; unsigned catches; Uint index; - BeamInstr* code = stp->code; + BeamInstr* codev = stp->codev; Module* modp; /* @@ -4620,10 +4691,10 @@ final_touch(LoaderState* stp) index = stp->catches; catches = BEAM_CATCHES_NIL; while (index != 0) { - BeamInstr next = code[index]; - code[index] = BeamOpCode(op_catch_yf); - catches = beam_catches_cons((BeamInstr *)code[index+2], catches); - code[index+2] = make_catch(catches); + BeamInstr next = codev[index]; + codev[index] = BeamOpCode(op_catch_yf); + catches = beam_catches_cons((BeamInstr *)codev[index+2], catches); + codev[index+2] = make_catch(catches); index = next; } modp = erts_put_module(stp->module); @@ -4674,8 +4745,8 @@ final_touch(LoaderState* stp) current = stp->import[i].patches; while (current != 0) { ASSERT(current < stp->ci); - next = stp->code[current]; - stp->code[current] = import; + next = stp->codev[current]; + stp->codev[current] = import; current = next; } } @@ -4688,7 +4759,7 @@ final_touch(LoaderState* stp) for (i = 0; i < stp->num_lambdas; i++) { unsigned entry_label = stp->lambdas[i].label; ErlFunEntry* fe = stp->lambdas[i].fe; - BeamInstr* code_ptr = (BeamInstr *) (stp->code + stp->labels[entry_label].value); + BeamInstr* code_ptr = stp->codev + stp->labels[entry_label].value; if (fe->address[0] != 0) { /* @@ -4801,7 +4872,8 @@ transform_engine(LoaderState* st) if (var[i].type != instr->a[ap].type) goto restart; switch (var[i].type) { - case TAG_r: case TAG_n: break; + case TAG_n: + break; default: if (var[i].val != instr->a[ap].val) goto restart; @@ -5311,7 +5383,7 @@ new_label(LoaderState* stp) (void *) stp->labels, stp->num_labels * sizeof(Label)); stp->labels[num].value = 0; - stp->labels[num].patches = 0; + stp->labels[num].patches = -1; return num; } @@ -5371,7 +5443,7 @@ erts_module_info_0(Process* p, Eterm module) { Module* modp; ErtsCodeIndex code_ix = erts_active_code_ix(); - BeamInstr* code; + BeamCodeHeader* code_hdr; Eterm *hp; Eterm list = NIL; Eterm tup; @@ -5385,13 +5457,13 @@ erts_module_info_0(Process* p, Eterm module) return THE_NON_VALUE; } - code = modp->curr.code; - if (code == NULL) { + code_hdr = modp->curr.code_hdr; + if (code_hdr == NULL) { return THE_NON_VALUE; } #define BUILD_INFO(What) \ - tup = get_module_info(p, code_ix, code, module, What); \ + tup = get_module_info(p, code_ix, code_hdr, module, What); \ hp = HAlloc(p, 5); \ tup = TUPLE2(hp, What, tup); \ hp += 3; \ @@ -5414,7 +5486,7 @@ erts_module_info_1(Process* p, Eterm module, Eterm what) { Module* modp; ErtsCodeIndex code_ix = erts_active_code_ix(); - BeamInstr* code; + BeamCodeHeader* code_hdr; if (is_not_atom(module)) { return THE_NON_VALUE; @@ -5425,34 +5497,34 @@ erts_module_info_1(Process* p, Eterm module, Eterm what) return THE_NON_VALUE; } - code = modp->curr.code; - if (code == NULL) { + code_hdr = modp->curr.code_hdr; + if (code_hdr == NULL) { return THE_NON_VALUE; } - return get_module_info(p, code_ix, code, module, what); + return get_module_info(p, code_ix, code_hdr, module, what); } static Eterm -get_module_info(Process* p, ErtsCodeIndex code_ix, BeamInstr* code, +get_module_info(Process* p, ErtsCodeIndex code_ix, BeamCodeHeader* code_hdr, Eterm module, Eterm what) { if (what == am_module) { return module; } else if (what == am_md5) { - return md5_of_module(p, code); + return md5_of_module(p, code_hdr); } else if (what == am_exports) { return exported_from_module(p, code_ix, module); } else if (what == am_functions) { - return functions_in_module(p, code); + return functions_in_module(p, code_hdr); } else if (what == am_attributes) { - return attributes_for_module(p, code); + return attributes_for_module(p, code_hdr); } else if (what == am_compile) { - return compilation_info_for_module(p, code); + return compilation_info_for_module(p, code_hdr); } else if (what == am_native_addresses) { - return native_addresses(p, code); + return native_addresses(p, code_hdr); } else if (what == am_native) { - return has_native(code); + return has_native(code_hdr); } return THE_NON_VALUE; } @@ -5464,7 +5536,7 @@ get_module_info(Process* p, ErtsCodeIndex code_ix, BeamInstr* code, Eterm functions_in_module(Process* p, /* Process whose heap to use. */ - BeamInstr* code) + BeamCodeHeader* code_hdr) { int i; Uint num_functions; @@ -5473,12 +5545,12 @@ functions_in_module(Process* p, /* Process whose heap to use. */ Eterm* hp_end; Eterm result = NIL; - num_functions = code[MI_NUM_FUNCTIONS]; + num_functions = code_hdr->num_functions; need = 5*num_functions; hp = HAlloc(p, need); hp_end = hp + need; for (i = num_functions-1; i >= 0 ; i--) { - BeamInstr* func_info = (BeamInstr *) code[MI_FUNCTIONS+i]; + BeamInstr* func_info = code_hdr->functions[i]; Eterm name = (Eterm) func_info[3]; int arity = (int) func_info[4]; Eterm tuple; @@ -5504,11 +5576,11 @@ functions_in_module(Process* p, /* Process whose heap to use. */ */ static Eterm -has_native(BeamInstr *code) +has_native(BeamCodeHeader *code_hdr) { Eterm result = am_false; #ifdef HIPE - if (erts_is_module_native(code)) { + if (erts_is_module_native(code_hdr)) { result = am_true; } #endif @@ -5516,15 +5588,15 @@ has_native(BeamInstr *code) } int -erts_is_module_native(BeamInstr* code) +erts_is_module_native(BeamCodeHeader* code_hdr) { Uint i, num_functions; /* Check NativeAdress of first real function in module */ - if (code != NULL) { - num_functions = code[MI_NUM_FUNCTIONS]; + if (code_hdr != NULL) { + num_functions = code_hdr->num_functions; for (i=0; i<num_functions; i++) { - BeamInstr* func_info = (BeamInstr *) code[MI_FUNCTIONS+i]; + BeamInstr* func_info = (BeamInstr *) code_hdr->functions[i]; Eterm name = (Eterm) func_info[3]; if (is_atom(name)) { return func_info[1] != 0; @@ -5541,7 +5613,7 @@ erts_is_module_native(BeamInstr* code) */ static Eterm -native_addresses(Process* p, BeamInstr* code) +native_addresses(Process* p, BeamCodeHeader* code_hdr) { int i; Eterm* hp; @@ -5550,12 +5622,12 @@ native_addresses(Process* p, BeamInstr* code) Eterm* hp_end; Eterm result = NIL; - num_functions = code[MI_NUM_FUNCTIONS]; + num_functions = code_hdr->num_functions; need = (6+BIG_UINT_HEAP_SIZE)*num_functions; hp = HAlloc(p, need); hp_end = hp + need; for (i = num_functions-1; i >= 0 ; i--) { - BeamInstr* func_info = (BeamInstr *) code[MI_FUNCTIONS+i]; + BeamInstr* func_info = code_hdr->functions[i]; Eterm name = (Eterm) func_info[3]; int arity = (int) func_info[4]; Eterm tuple; @@ -5621,15 +5693,15 @@ exported_from_module(Process* p, /* Process whose heap to use. */ Eterm attributes_for_module(Process* p, /* Process whose heap to use. */ - BeamInstr* code) + BeamCodeHeader* code_hdr) { byte* ext; Eterm result = NIL; - ext = (byte *) code[MI_ATTR_PTR]; + ext = code_hdr->attr_ptr; if (ext != NULL) { ErtsHeapFactory factory; - erts_factory_proc_prealloc_init(&factory, p, code[MI_ATTR_SIZE_ON_HEAP]); + erts_factory_proc_prealloc_init(&factory, p, code_hdr->attr_size_on_heap); result = erts_decode_ext(&factory, &ext); if (is_value(result)) { erts_factory_close(&factory); @@ -5644,15 +5716,15 @@ attributes_for_module(Process* p, /* Process whose heap to use. */ Eterm compilation_info_for_module(Process* p, /* Process whose heap to use. */ - BeamInstr* code) + BeamCodeHeader* code_hdr) { byte* ext; Eterm result = NIL; - ext = (byte *) code[MI_COMPILE_PTR]; + ext = code_hdr->compile_ptr; if (ext != NULL) { ErtsHeapFactory factory; - erts_factory_proc_prealloc_init(&factory, p, code[MI_COMPILE_SIZE_ON_HEAP]); + erts_factory_proc_prealloc_init(&factory, p, code_hdr->compile_size_on_heap); result = erts_decode_ext(&factory, &ext); if (is_value(result)) { erts_factory_close(&factory); @@ -5667,9 +5739,9 @@ compilation_info_for_module(Process* p, /* Process whose heap to use. */ Eterm md5_of_module(Process* p, /* Process whose heap to use. */ - BeamInstr* code) + BeamCodeHeader* code_hdr) { - return new_binary(p, (byte *) code[MI_MD5_PTR], MD5_SIZE); + return new_binary(p, code_hdr->md5_ptr, MD5_SIZE); } /* @@ -5868,7 +5940,7 @@ make_stub(BeamInstr* fp, Eterm mod, Eterm func, Uint arity, Uint native, BeamIns fp[4] = arity; #ifdef HIPE if (native) { - fp[5] = BeamOpCode(op_move_return_nr); + fp[5] = BeamOpCode(op_move_return_n); hipe_mfa_save_orig_beam_op(mod, func, arity, fp+5); } #endif @@ -5880,7 +5952,7 @@ static byte* stub_copy_info(LoaderState* stp, int chunk, /* Chunk: ATTR_CHUNK or COMPILE_CHUNK */ byte* info, /* Where to store info. */ - BeamInstr* ptr_word, /* Where to store pointer into info. */ + byte** ptr_word, /* Where to store pointer into info. */ BeamInstr* size_word, /* Where to store size into info. */ BeamInstr* size_on_heap_word) /* Where to store size on heap. */ { @@ -5888,7 +5960,7 @@ stub_copy_info(LoaderState* stp, Uint size = stp->chunks[chunk].size; if (size != 0) { memcpy(info, stp->chunks[chunk].start, size); - *ptr_word = (BeamInstr) info; + *ptr_word = info; decoded_size = erts_decode_ext_size(info, size); if (decoded_size < 0) { return 0; @@ -6143,15 +6215,14 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) BeamInstr Patchlist; Eterm MD5Bin; Eterm* tp; - BeamInstr* code = NULL; - BeamInstr* ptrs; + BeamCodeHeader* code_hdr; + BeamInstr* code_base; BeamInstr* fp; byte* info; - Uint ci; - int n; + Sint n; int code_size; int rval; - int i; + Sint i; byte* temp_alloc = NULL; byte* bytes; Uint size; @@ -6225,40 +6296,40 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) * Allocate memory for the stub module. */ - code_size = ((WORDS_PER_FUNCTION+1)*n + MI_FUNCTIONS + 2) * sizeof(BeamInstr); - code_size += stp->chunks[ATTR_CHUNK].size; - code_size += stp->chunks[COMPILE_CHUNK].size; - code_size += MD5_SIZE; - code = erts_alloc_fnf(ERTS_ALC_T_CODE, code_size); - if (!code) { + code_size = (offsetof(BeamCodeHeader,functions) + + ((n+1) * sizeof(BeamInstr*)) + + (WORDS_PER_FUNCTION*n + 1) * sizeof(BeamInstr) + + stp->chunks[ATTR_CHUNK].size + + stp->chunks[COMPILE_CHUNK].size + + MD5_SIZE); + code_hdr = erts_alloc_fnf(ERTS_ALC_T_CODE, code_size); + if (!code_hdr) { goto error; } /* - * Initialize code area. + * Initialize code header. */ - code[MI_NUM_FUNCTIONS] = n; - code[MI_ATTR_PTR] = 0; - code[MI_ATTR_SIZE] = 0; - code[MI_ATTR_SIZE_ON_HEAP] = 0; - code[MI_COMPILE_PTR] = 0; - code[MI_COMPILE_SIZE] = 0; - code[MI_COMPILE_SIZE_ON_HEAP] = 0; - code[MI_LITERALS_START] = 0; - code[MI_LITERALS_END] = 0; - code[MI_LITERALS_OFF_HEAP] = 0; - code[MI_ON_LOAD_FUNCTION_PTR] = 0; - code[MI_LINE_TABLE] = 0; - code[MI_MD5_PTR] = 0; - ci = MI_FUNCTIONS + n + 1; + code_hdr->num_functions = n; + code_hdr->attr_ptr = NULL; + code_hdr->attr_size = 0; + code_hdr->attr_size_on_heap = 0; + code_hdr->compile_ptr = NULL; + code_hdr->compile_size = 0; + code_hdr->compile_size_on_heap = 0; + code_hdr->literals_start = NULL; + code_hdr->literals_end = NULL; + code_hdr->literals_off_heap = 0; + code_hdr->on_load_function_ptr = NULL; + code_hdr->line_table = NULL; + code_hdr->md5_ptr = NULL; /* * Make stubs for all functions. */ - ptrs = code + MI_FUNCTIONS; - fp = code + ci; + fp = code_base = (BeamInstr*) &code_hdr->functions[n+1]; for (i = 0; i < n; i++) { Eterm* listp; Eterm tuple; @@ -6301,11 +6372,11 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) * Set the pointer and make the stub. Put a return instruction * as the body until we know what kind of trap we should put there. */ - ptrs[i] = (BeamInstr) fp; + code_hdr->functions[i] = fp; #ifdef HIPE op = (Eterm) BeamOpCode(op_hipe_trap_call); /* Might be changed later. */ #else - op = (Eterm) BeamOpCode(op_move_return_nr); + op = (Eterm) BeamOpCode(op_move_return_n); #endif fp = make_stub(fp, Mod, func, arity, (Uint)native_address, op); } @@ -6314,7 +6385,7 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) * Insert the last pointer and the int_code_end instruction. */ - ptrs[i] = (BeamInstr) fp; + code_hdr->functions[i] = fp; *fp++ = (BeamInstr) BeamOp(op_int_code_end); /* @@ -6323,16 +6394,16 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) info = (byte *) fp; info = stub_copy_info(stp, ATTR_CHUNK, info, - code+MI_ATTR_PTR, - code+MI_ATTR_SIZE, - code+MI_ATTR_SIZE_ON_HEAP); + &code_hdr->attr_ptr, + &code_hdr->attr_size, + &code_hdr->attr_size_on_heap); if (info == NULL) { goto error; } info = stub_copy_info(stp, COMPILE_CHUNK, info, - code+MI_COMPILE_PTR, - code+MI_COMPILE_SIZE, - code+MI_COMPILE_SIZE_ON_HEAP); + &code_hdr->compile_ptr, + &code_hdr->compile_size, + &code_hdr->compile_size_on_heap); if (info == NULL) { goto error; } @@ -6341,7 +6412,7 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) byte *md5 = NULL; if ((md5 = erts_get_aligned_binary_bytes(MD5Bin, &tmp)) != NULL) { sys_memcpy(info, md5, MD5_SIZE); - code[MI_MD5_PTR] = (BeamInstr) info; + code_hdr->md5_ptr = info; } erts_free_aligned_binary_bytes(tmp); } @@ -6350,7 +6421,7 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) * Insert the module in the module table. */ - rval = insert_new_code(p, 0, p->group_leader, Mod, code, code_size); + rval = insert_new_code(p, 0, p->group_leader, Mod, code_hdr, code_size); if (rval != NIL) { goto error; } @@ -6359,7 +6430,7 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) * Export all stub functions and insert the correct type of HiPE trap. */ - fp = code + ci; + fp = code_base; for (i = 0; i < n; i++) { stub_final_touch(stp, fp); fp += WORDS_PER_FUNCTION; diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h index d5af634fad..22ab71c868 100644 --- a/erts/emulator/beam/beam_load.h +++ b/erts/emulator/beam/beam_load.h @@ -24,7 +24,6 @@ #include "beam_opcodes.h" #include "erl_process.h" -int erts_is_module_native(BeamInstr* code); Eterm beam_make_current_old(Process *c_p, ErtsProcLocks c_p_locks, Eterm module); @@ -52,6 +51,7 @@ extern BeamInstr* em_call_error_handler; extern BeamInstr* em_apply_bif; extern BeamInstr* em_call_nif; + /* * The following variables keep a sorted list of address ranges for * each module. It allows us to quickly find a function given an @@ -60,72 +60,81 @@ extern BeamInstr* em_call_nif; /* Total code size in bytes */ extern Uint erts_total_code_size; -/* - * Index into start of code chunks which contains additional information - * about the loaded module. - * - * First number of functions. - */ -#define MI_NUM_FUNCTIONS 0 +typedef struct BeamCodeLineTab_ BeamCodeLineTab; /* - * The attributes retrieved by Mod:module_info(attributes). - */ - -#define MI_ATTR_PTR 1 -#define MI_ATTR_SIZE 2 -#define MI_ATTR_SIZE_ON_HEAP 3 - -/* - * The compilation information retrieved by Mod:module_info(compile). - */ - -#define MI_COMPILE_PTR 4 -#define MI_COMPILE_SIZE 5 -#define MI_COMPILE_SIZE_ON_HEAP 6 - -/* - * Literal area (constant pool). - */ -#define MI_LITERALS_START 7 -#define MI_LITERALS_END 8 -#define MI_LITERALS_OFF_HEAP 9 - -/* - * Pointer to the on_load function (or NULL if none). - */ -#define MI_ON_LOAD_FUNCTION_PTR 10 - -/* - * Pointer to the line table (or NULL if none). - */ -#define MI_LINE_TABLE 11 - -/* - * Pointer to the module MD5 sum (16 bytes) - */ -#define MI_MD5_PTR 12 - -/* - * Start of function pointer table. This table contains pointers to - * all functions in the module plus an additional pointer just beyond - * the end of the last function. - * - * The actual loaded code (for the first function) start just beyond - * this table. + * Header of code chunks which contains additional information + * about the loaded module. */ - -#define MI_FUNCTIONS 13 +typedef struct beam_code_header { + /* + * Number of functions. + */ + UWord num_functions; + + /* + * The attributes retrieved by Mod:module_info(attributes). + */ + byte* attr_ptr; + UWord attr_size; + UWord attr_size_on_heap; + + /* + * The compilation information retrieved by Mod:module_info(compile). + */ + byte* compile_ptr; + UWord compile_size; + UWord compile_size_on_heap; + + /* + * Literal area (constant pool). + */ + Eterm* literals_start; + Eterm* literals_end; + struct erl_off_heap_header* literals_off_heap; + + /* + * Pointer to the on_load function (or NULL if none). + */ + BeamInstr* on_load_function_ptr; + + /* + * Pointer to the line table (or NULL if none). + */ + BeamCodeLineTab* line_table; + + /* + * Pointer to the module MD5 sum (16 bytes) + */ + byte* md5_ptr; + + /* + * Start of function pointer table. This table contains pointers to + * all functions in the module plus an additional pointer just beyond + * the end of the last function. + * + * The actual loaded code (for the first function) start just beyond + * this table. + */ + BeamInstr* functions[1]; + +}BeamCodeHeader; + +int erts_is_module_native(BeamCodeHeader* code); /* * Layout of the line table. */ - -#define MI_LINE_FNAME_PTR 0 -#define MI_LINE_LOC_TAB 1 -#define MI_LINE_LOC_SIZE 2 -#define MI_LINE_FUNC_TAB 3 +struct BeamCodeLineTab_ { + Eterm* fname_ptr; + int loc_size; + union { + Uint16* p2; + Uint32* p4; + }loc_tab; + const BeamInstr** func_tab[1]; +}; #define LINE_INVALID_LOCATION (0) diff --git a/erts/emulator/beam/beam_ranges.c b/erts/emulator/beam/beam_ranges.c index 19079ba150..5a2b66727a 100644 --- a/erts/emulator/beam/beam_ranges.c +++ b/erts/emulator/beam/beam_ranges.c @@ -37,8 +37,8 @@ typedef struct { #define RANGE_END(R) ((BeamInstr*)erts_smp_atomic_read_nob(&(R)->end)) static Range* find_range(BeamInstr* pc); -static void lookup_loc(FunctionInfo* fi, BeamInstr* pc, - BeamInstr* modp, int idx); +static void lookup_loc(FunctionInfo* fi, const BeamInstr* pc, + BeamCodeHeader*, int idx); /* * The following variables keep a sorted list of address ranges for @@ -241,6 +241,7 @@ erts_lookup_function_info(FunctionInfo* fi, BeamInstr* pc, int full_info) BeamInstr** high; BeamInstr** mid; Range* rp; + BeamCodeHeader* hdr; fi->current = NULL; fi->needed = 5; @@ -249,9 +250,10 @@ erts_lookup_function_info(FunctionInfo* fi, BeamInstr* pc, int full_info) if (rp == 0) { return; } + hdr = (BeamCodeHeader*) rp->start; - low = (BeamInstr **) (rp->start + MI_FUNCTIONS); - high = low + rp->start[MI_NUM_FUNCTIONS]; + low = hdr->functions; + high = low + hdr->num_functions; while (low < high) { mid = low + (high-low) / 2; if (pc < mid[0]) { @@ -259,10 +261,9 @@ erts_lookup_function_info(FunctionInfo* fi, BeamInstr* pc, int full_info) } else if (pc < mid[1]) { fi->current = mid[0]+2; if (full_info) { - BeamInstr** fp = (BeamInstr **) (rp->start + - MI_FUNCTIONS); + BeamInstr** fp = hdr->functions; int idx = mid - fp; - lookup_loc(fi, pc, rp->start, idx); + lookup_loc(fi, pc, hdr, idx); } return; } else { @@ -295,39 +296,34 @@ find_range(BeamInstr* pc) } static void -lookup_loc(FunctionInfo* fi, BeamInstr* orig_pc, BeamInstr* modp, int idx) +lookup_loc(FunctionInfo* fi, const BeamInstr* pc, + BeamCodeHeader* code_hdr, int idx) { - Eterm* line = (Eterm *) modp[MI_LINE_TABLE]; - Eterm* low; - Eterm* high; - Eterm* mid; - Eterm pc; + BeamCodeLineTab* lt = code_hdr->line_table; + const BeamInstr** low; + const BeamInstr** high; + const BeamInstr** mid; - if (line == 0) { + if (lt == NULL) { return; } - pc = (Eterm) (BeamInstr) orig_pc; - fi->fname_ptr = (Eterm *) (BeamInstr) line[MI_LINE_FNAME_PTR]; - low = (Eterm *) (BeamInstr) line[MI_LINE_FUNC_TAB+idx]; - high = (Eterm *) (BeamInstr) line[MI_LINE_FUNC_TAB+idx+1]; + fi->fname_ptr = lt->fname_ptr; + low = lt->func_tab[idx]; + high = lt->func_tab[idx+1]; while (high > low) { mid = low + (high-low) / 2; if (pc < mid[0]) { high = mid; } else if (pc < mid[1]) { int file; - int index = mid - (Eterm *) (BeamInstr) line[MI_LINE_FUNC_TAB]; + int index = mid - lt->func_tab[0]; - if (line[MI_LINE_LOC_SIZE] == 2) { - Uint16* loc_table = - (Uint16 *) (BeamInstr) line[MI_LINE_LOC_TAB]; - fi->loc = loc_table[index]; + if (lt->loc_size == 2) { + fi->loc = lt->loc_tab.p2[index]; } else { - Uint32* loc_table = - (Uint32 *) (BeamInstr) line[MI_LINE_LOC_TAB]; - ASSERT(line[MI_LINE_LOC_SIZE] == 4); - fi->loc = loc_table[index]; + ASSERT(lt->loc_size == 4); + fi->loc = lt->loc_tab.p4[index]; } if (fi->loc == LINE_INVALID_LOCATION) { return; diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index a8cc19ee1f..2654785ffc 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -44,6 +44,7 @@ #include "erl_ptab.h" #include "erl_bits.h" #include "erl_bif_unique.h" +#include "erl_msacc.h" Export *erts_await_result; static Export* flush_monitor_messages_trap = NULL; @@ -54,6 +55,9 @@ Export* erts_format_cpu_topology_trap = NULL; static Export dsend_continue_trap_export; Export *erts_convert_time_unit_trap = NULL; +static Export *await_msacc_mod_trap = NULL; +static erts_smp_atomic32_t msacc; + static Export *await_sched_wall_time_mod_trap; static erts_smp_atomic32_t sched_wall_time; @@ -72,7 +76,7 @@ BIF_RETTYPE spawn_3(BIF_ALIST_3) ErlSpawnOpts so; Eterm pid; - so.flags = 0; + so.flags = erts_default_spo_flags; pid = erl_create_process(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, &so); if (is_non_value(pid)) { BIF_ERROR(BIF_P, so.error_code); @@ -589,7 +593,7 @@ erts_queue_monitor_message(Process *p, Eterm reason_copy, ref_copy, item_copy; Uint reason_size, ref_size, item_size, heap_size; ErlOffHeap *ohp; - ErlHeapFragment *bp; + ErtsMessage *msgp; reason_size = IS_CONST(reason) ? 0 : size_object(reason); item_size = IS_CONST(item) ? 0 : size_object(item); @@ -597,11 +601,8 @@ erts_queue_monitor_message(Process *p, heap_size = 6+reason_size+ref_size+item_size; - hp = erts_alloc_message_heap(heap_size, - &bp, - &ohp, - p, - p_locksp); + msgp = erts_alloc_message_heap(p, p_locksp, heap_size, + &hp, &ohp); reason_copy = (IS_CONST(reason) ? reason @@ -612,7 +613,7 @@ erts_queue_monitor_message(Process *p, ref_copy = copy_struct(ref, ref_size, &hp, ohp); tup = TUPLE5(hp, am_DOWN, ref_copy, type, item_copy, reason_copy); - erts_queue_message(p, p_locksp, bp, tup, NIL); + erts_queue_message(p, p_locksp, msgp, tup, NIL); } static BIF_RETTYPE @@ -841,7 +842,7 @@ BIF_RETTYPE spawn_link_3(BIF_ALIST_3) ErlSpawnOpts so; Eterm pid; - so.flags = SPO_LINK; + so.flags = erts_default_spo_flags|SPO_LINK; pid = erl_create_process(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, &so); if (is_non_value(pid)) { BIF_ERROR(BIF_P, so.error_code); @@ -878,7 +879,7 @@ BIF_RETTYPE spawn_opt_1(BIF_ALIST_1) /* * Store default values for options. */ - so.flags = SPO_USE_ARGS; + so.flags = erts_default_spo_flags|SPO_USE_ARGS; so.min_heap_size = H_MIN_SIZE; so.min_vheap_size = BIN_VH_MIN_SIZE; so.priority = PRIORITY_NORMAL; @@ -913,6 +914,22 @@ BIF_RETTYPE spawn_opt_1(BIF_ALIST_1) so.priority = PRIORITY_LOW; else goto error; + } else if (arg == am_message_queue_data) { + switch (val) { + case am_mixed: + so.flags &= ~(SPO_OFF_HEAP_MSGQ|SPO_ON_HEAP_MSGQ); + break; + case am_on_heap: + so.flags &= ~SPO_OFF_HEAP_MSGQ; + so.flags |= SPO_ON_HEAP_MSGQ; + break; + case am_off_heap: + so.flags &= ~SPO_ON_HEAP_MSGQ; + so.flags |= SPO_OFF_HEAP_MSGQ; + break; + default: + goto error; + } } else if (arg == am_min_heap_size && is_small(val)) { Sint min_heap_size = signed_val(val); if (min_heap_size < 0) { @@ -1691,6 +1708,12 @@ BIF_RETTYPE process_flag_2(BIF_ALIST_2) } BIF_RET(old_value); } + else if (BIF_ARG_1 == am_message_queue_data) { + old_value = erts_change_message_queue_management(BIF_P, BIF_ARG_2); + if (is_non_value(old_value)) + goto error; + BIF_RET(old_value); + } else if (BIF_ARG_1 == am_sensitive) { Uint is_sensitive; if (BIF_ARG_2 == am_true) { @@ -1931,7 +1954,7 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext* ctx) } else if (is_atom(to)) { Eterm id = erts_whereis_name_to_id(p, to); - rp = erts_proc_lookup(id); + rp = erts_proc_lookup_raw(id); if (rp) { if (IS_TRACED(p)) trace_send(p, to, msg); @@ -2023,11 +2046,7 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext* ctx) if (ERTS_PROC_GET_SAVED_CALLS_BUF(p)) save_calls(p, &exp_send); - if (SEQ_TRACE_TOKEN(p) != NIL -#ifdef USE_VM_PROBES - && SEQ_TRACE_TOKEN(p) != am_have_dt_utag -#endif - ) { + if (have_seqtrace(SEQ_TRACE_TOKEN(p))) { seq_trace_update_send(p); seq_trace_output(SEQ_TRACE_TOKEN(p), msg, SEQ_TRACE_SEND, portid, p); @@ -2128,7 +2147,11 @@ BIF_RETTYPE send_3(BIF_ALIST_3) int connect = !0; Eterm l = opts; Sint result; + DeclareTypedTmpHeap(ErtsSendContext, ctx, BIF_P); + + ERTS_MSACC_PUSH_STATE_M_X(); + UseTmpHeap(sizeof(ErtsSendContext)/sizeof(Eterm), BIF_P); ctx->suspend = !0; @@ -2157,7 +2180,10 @@ BIF_RETTYPE send_3(BIF_ALIST_3) ref = NIL; #endif + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_SEND); result = do_send(p, to, msg, &ref, ctx); + ERTS_MSACC_POP_STATE_M_X(); + if (result > 0) { ERTS_VBUMP_REDS(p, result); if (ERTS_IS_PROC_OUT_OF_REDS(p)) @@ -2273,8 +2299,8 @@ Eterm erl_send(Process *p, Eterm to, Eterm msg) Eterm ref; Sint result; DeclareTypedTmpHeap(ErtsSendContext, ctx, p); + ERTS_MSACC_PUSH_AND_SET_STATE_M_X(ERTS_MSACC_STATE_SEND); UseTmpHeap(sizeof(ErtsSendContext)/sizeof(Eterm), p); - #ifdef DEBUG ref = NIL; #endif @@ -2285,7 +2311,9 @@ Eterm erl_send(Process *p, Eterm to, Eterm msg) ctx->dss.phase = ERTS_DSIG_SEND_PHASE_INIT; result = do_send(p, to, msg, &ref, ctx); - + + ERTS_MSACC_POP_STATE_M_X(); + if (result > 0) { ERTS_VBUMP_REDS(p, result); if (ERTS_IS_PROC_OUT_OF_REDS(p)) @@ -2809,7 +2837,7 @@ BIF_RETTYPE list_to_atom_1(BIF_ALIST_1) { Eterm res; char *buf = (char *) erts_alloc(ERTS_ALC_T_TMP, MAX_ATOM_CHARACTERS); - int i = intlist_to_buf(BIF_ARG_1, buf, MAX_ATOM_CHARACTERS); + Sint i = intlist_to_buf(BIF_ARG_1, buf, MAX_ATOM_CHARACTERS); if (i < 0) { erts_free(ERTS_ALC_T_TMP, (void *) buf); @@ -2829,7 +2857,7 @@ BIF_RETTYPE list_to_atom_1(BIF_ALIST_1) BIF_RETTYPE list_to_existing_atom_1(BIF_ALIST_1) { - int i; + Sint i; char *buf = (char *) erts_alloc(ERTS_ALC_T_TMP, MAX_ATOM_CHARACTERS); if ((i = intlist_to_buf(BIF_ARG_1, buf, MAX_ATOM_CHARACTERS)) < 0) { @@ -2889,175 +2917,20 @@ BIF_RETTYPE integer_to_list_1(BIF_ALIST_1) /**********************************************************************/ -/* convert a list of ascii ascii integer value to an integer */ - - -#define LTI_BAD_STRUCTURE 0 -#define LTI_NO_INTEGER 1 -#define LTI_SOME_INTEGER 2 -#define LTI_ALL_INTEGER 3 - -static int do_list_to_integer(Process *p, Eterm orig_list, - Eterm *integer, Eterm *rest) -{ - Sint i = 0; - Uint ui = 0; - int skip = 0; - int neg = 0; - Sint n = 0; - int m; - int lg2; - Eterm res; - Eterm* hp; - Eterm *hp_end; - Eterm lst = orig_list; - Eterm tail = lst; - int error_res = LTI_BAD_STRUCTURE; - - if (is_nil(lst)) { - error_res = LTI_NO_INTEGER; - error: - *rest = tail; - *integer = make_small(0); - return error_res; - } - if (is_not_list(lst)) - goto error; - - /* if first char is a '-' then it is a negative integer */ - if (CAR(list_val(lst)) == make_small('-')) { - neg = 1; - skip = 1; - lst = CDR(list_val(lst)); - if (is_not_list(lst)) { - tail = lst; - error_res = LTI_NO_INTEGER; - goto error; - } - } else if (CAR(list_val(lst)) == make_small('+')) { - /* ignore plus */ - skip = 1; - lst = CDR(list_val(lst)); - if (is_not_list(lst)) { - tail = lst; - error_res = LTI_NO_INTEGER; - goto error; - } - } - - /* Calculate size and do type check */ - - while(1) { - if (is_not_small(CAR(list_val(lst)))) { - break; - } - if (unsigned_val(CAR(list_val(lst))) < '0' || - unsigned_val(CAR(list_val(lst))) > '9') { - break; - } - ui = ui * 10; - ui = ui + unsigned_val(CAR(list_val(lst))) - '0'; - n++; - lst = CDR(list_val(lst)); - if (is_nil(lst)) { - break; - } - if (is_not_list(lst)) { - break; - } - } - - tail = lst; - if (!n) { - error_res = LTI_NO_INTEGER; - goto error; - } - - - /* If n <= 8 then we know it's a small int - ** since 2^27 = 134217728. If n > 8 then we must - ** construct a bignum and let that routine do the checking - */ - - if (n <= SMALL_DIGITS) { /* It must be small */ - if (neg) i = -(Sint)ui; - else i = (Sint)ui; - res = make_small(i); - } else { - /* Convert from log10 to log2 by multiplying with 1/log10(2)=3.3219 - which we round up to (3 + 1/3) */ - lg2 = (n+1)*3 + (n+1)/3 + 1; - m = (lg2+D_EXP-1)/D_EXP; /* number of digits */ - m = BIG_NEED_SIZE(m); /* number of words + thing */ - - hp = HAlloc(p, m); - hp_end = hp + m; - - lst = orig_list; - if (skip) - lst = CDR(list_val(lst)); - - /* load first digits (at least one digit) */ - if ((i = (n % D_DECIMAL_EXP)) == 0) - i = D_DECIMAL_EXP; - n -= i; - m = 0; - while(i--) { - m = 10*m + (unsigned_val(CAR(list_val(lst))) - '0'); - lst = CDR(list_val(lst)); - } - res = small_to_big(m, hp); /* load first digits */ - - while(n) { - i = D_DECIMAL_EXP; - n -= D_DECIMAL_EXP; - m = 0; - while(i--) { - m = 10*m + (unsigned_val(CAR(list_val(lst))) - '0'); - lst = CDR(list_val(lst)); - } - if (is_small(res)) - res = small_to_big(signed_val(res), hp); - res = big_times_small(res, D_DECIMAL_BASE, hp); - if (is_small(res)) - res = small_to_big(signed_val(res), hp); - res = big_plus_small(res, m, hp); - } - - if (neg) { - if (is_small(res)) - res = make_small(-signed_val(res)); - else { - Uint *big = big_val(res); /* point to thing */ - *big = bignum_header_neg(*big); - } - } - - if (is_not_small(res)) { - res = big_plus_small(res, 0, hp); /* includes conversion to small */ +/* + * Converts a list of ascii base10 digits to an integer fully or partially. + * Returns result and the remaining tail. + * On error returns: {error,not_a_list}, or {error, no_integer} + */ - if (is_not_small(res)) { - hp += (big_arity(res)+1); - } - } - HRelease(p,hp_end,hp); - } - *integer = res; - *rest = tail; - if (tail != NIL) { - return LTI_SOME_INTEGER; - } - return LTI_ALL_INTEGER; -} BIF_RETTYPE string_to_integer_1(BIF_ALIST_1) { Eterm res; Eterm tail; Eterm *hp; /* must be a list */ - switch (do_list_to_integer(BIF_P,BIF_ARG_1,&res,&tail)) { - /* HAlloc after do_list_to_integer as it - might HAlloc itself (bignum) */ + switch (erts_list_to_integer(BIF_P, BIF_ARG_1, 10, &res, &tail)) { + /* HAlloc after erts_list_to_integer as it might HAlloc itself (bignum) */ case LTI_BAD_STRUCTURE: hp = HAlloc(BIF_P,3); BIF_RET(TUPLE2(hp, am_error, am_not_a_list)); @@ -3072,13 +2945,14 @@ BIF_RETTYPE string_to_integer_1(BIF_ALIST_1) BIF_RETTYPE list_to_integer_1(BIF_ALIST_1) { - /* Using do_list_to_integer is about twice as fast as using + /* Using erts_list_to_integer is about twice as fast as using erts_chars_to_integer because we do not have to copy the entire list */ Eterm res; Eterm dummy; /* must be a list */ - if (do_list_to_integer(BIF_P,BIF_ARG_1,&res,&dummy) != LTI_ALL_INTEGER) { + if (erts_list_to_integer(BIF_P, BIF_ARG_1, 10, + &res, &dummy) != LTI_ALL_INTEGER) { BIF_ERROR(BIF_P,BADARG); } BIF_RET(res); @@ -3086,14 +2960,12 @@ BIF_RETTYPE list_to_integer_1(BIF_ALIST_1) BIF_RETTYPE list_to_integer_2(BIF_ALIST_2) { - /* Bif implementation is about 50% faster than pure erlang, and since we have erts_chars_to_integer now it is simpler as well. This could be optmized further if we did not have to copy the list to buf. */ - int i; - Eterm res; - char *buf = NULL; + Sint i; + Eterm res, dummy; int base; i = erts_list_length(BIF_ARG_1); @@ -3101,31 +2973,16 @@ BIF_RETTYPE list_to_integer_2(BIF_ALIST_2) BIF_ERROR(BIF_P, BADARG); base = signed_val(BIF_ARG_2); - + if (base < 2 || base > 36) BIF_ERROR(BIF_P, BADARG); - /* Take fast path if base it 10 */ - if (base == 10) - return list_to_integer_1(BIF_P,&BIF_ARG_1); - - buf = (char *) erts_alloc(ERTS_ALC_T_TMP, i + 1); - - if (intlist_to_buf(BIF_ARG_1, buf, i) < 0) - goto list_to_integer_1_error; - buf[i] = '\0'; /* null terminal */ - - if ((res = erts_chars_to_integer(BIF_P,buf,i,base)) == THE_NON_VALUE) - goto list_to_integer_1_error; - - erts_free(ERTS_ALC_T_TMP, (void *) buf); + if (erts_list_to_integer(BIF_P, BIF_ARG_1, base, + &res, &dummy) != LTI_ALL_INTEGER) { + BIF_ERROR(BIF_P,BADARG); + } BIF_RET(res); - - list_to_integer_1_error: - erts_free(ERTS_ALC_T_TMP, (void *) buf); - BIF_ERROR(BIF_P, BADARG); - - } +} /**********************************************************************/ @@ -3431,7 +3288,7 @@ static BIF_RETTYPE do_charbuf_to_float(Process *BIF_P,char *buf) { BIF_RETTYPE list_to_float_1(BIF_ALIST_1) { - int i; + Sint i; Eterm res; char *buf = NULL; @@ -3548,7 +3405,7 @@ BIF_RETTYPE list_to_tuple_1(BIF_ALIST_1) Eterm* cons; Eterm res; Eterm* hp; - int len; + Sint len; if ((len = erts_list_length(list)) < 0 || len > ERTS_MAX_TUPLE_SIZE) { BIF_ERROR(BIF_P, BADARG); @@ -3824,11 +3681,9 @@ BIF_RETTYPE now_0(BIF_ALIST_0) BIF_RETTYPE garbage_collect_0(BIF_ALIST_0) { - int reds; - FLAGS(BIF_P) |= F_NEED_FULLSWEEP; - reds = erts_garbage_collect(BIF_P, 0, NULL, 0); - BIF_RET2(am_true, reds); + erts_garbage_collect(BIF_P, 0, NULL, 0); + BIF_RET(am_true); } /**********************************************************************/ @@ -3897,7 +3752,7 @@ BIF_RETTYPE display_string_1(BIF_ALIST_1) { Process* p = BIF_P; Eterm string = BIF_ARG_1; - int len = is_string(string); + Sint len = is_string(string); char *str; if (len <= 0) { @@ -3951,7 +3806,7 @@ BIF_RETTYPE halt_1(BIF_ALIST_1) erl_exit(ERTS_ABORT_EXIT, ""); } else if (is_string(BIF_ARG_1) || BIF_ARG_1 == NIL) { - int i; + Sint i; if ((i = intlist_to_buf(BIF_ARG_1, halt_msg, HALT_MSG_SIZE-1)) < 0) { goto error; @@ -4020,7 +3875,7 @@ BIF_RETTYPE halt_2(BIF_ALIST_2) erl_exit(ERTS_ABORT_EXIT, ""); } else if (is_string(BIF_ARG_1) || BIF_ARG_1 == NIL) { - int i; + Sint i; if ((i = intlist_to_buf(BIF_ARG_1, halt_msg, HALT_MSG_SIZE-1)) < 0) { goto error; @@ -4117,16 +3972,9 @@ BIF_RETTYPE make_fun_3(BIF_ALIST_3) if (arity < 0) { goto error; } -#if HALFWORD_HEAP - hp = HAlloc(BIF_P, 3); - hp[0] = HEADER_EXPORT; - /* Yes, May be misaligned, but X86_64 will fix it... */ - *((Export **) (hp+1)) = erts_export_get_or_make_stub(BIF_ARG_1, BIF_ARG_2, (Uint) arity); -#else hp = HAlloc(BIF_P, 2); hp[0] = HEADER_EXPORT; hp[1] = (Eterm) erts_export_get_or_make_stub(BIF_ARG_1, BIF_ARG_2, (Uint) arity); -#endif BIF_RET(make_export(hp)); } @@ -4170,7 +4018,7 @@ BIF_RETTYPE list_to_pid_1(BIF_ALIST_1) { Uint a = 0, b = 0, c = 0; char* cp; - int i; + Sint i; DistEntry *dep = NULL; char *buf = (char *) erts_alloc(ERTS_ALC_T_TMP, 65); /* @@ -4487,7 +4335,7 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) } } else if (BIF_ARG_1 == make_small(1)) { int i, max; - ErlMessage* mp; + ErtsMessage* mp; erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_smp_thr_progress_block(); @@ -4580,6 +4428,31 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) default: ERTS_INTERNAL_ERROR("Unknown state"); } +#ifdef ERTS_ENABLE_MSACC + } else if (BIF_ARG_1 == am_microstate_accounting) { + Eterm threads; + if (BIF_ARG_2 == am_true || BIF_ARG_2 == am_false) { + erts_aint32_t new = BIF_ARG_2 == am_true ? ERTS_MSACC_ENABLE : ERTS_MSACC_DISABLE; + erts_aint32_t old = erts_smp_atomic32_xchg_nob(&msacc, new); + Eterm ref = erts_msacc_request(BIF_P, new, &threads); + if (is_non_value(ref)) + BIF_RET(old ? am_true : am_false); + BIF_TRAP3(await_msacc_mod_trap, + BIF_P, + ref, + old ? am_true : am_false, + threads); + } else if (BIF_ARG_2 == am_reset) { + Eterm ref = erts_msacc_request(BIF_P, ERTS_MSACC_RESET, &threads); + erts_aint32_t old = erts_smp_atomic32_read_nob(&msacc); + ASSERT(is_value(ref)); + BIF_TRAP3(await_msacc_mod_trap, + BIF_P, + ref, + old ? am_true : am_false, + threads); + } +#endif } else if (ERTS_IS_ATOM_STR("scheduling_statistics", BIF_ARG_1)) { int what; if (ERTS_IS_ATOM_STR("disable", BIF_ARG_2)) @@ -4632,7 +4505,7 @@ BIF_RETTYPE hash_2(BIF_ALIST_2) if ((range = signed_val(BIF_ARG_2)) <= 0) { /* [1..MAX_SMALL] */ BIF_ERROR(BIF_P, BADARG); } -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) if (range > ((1L << 27) - 1)) BIF_ERROR(BIF_P, BADARG); #endif @@ -4704,7 +4577,7 @@ BIF_RETTYPE phash2_2(BIF_ALIST_2) /* * Return either a small or a big. Use the heap for bigs if there is room. */ -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) BIF_RET(make_small(final_hash)); #else if (IS_USMALL(0, final_hash)) { @@ -4909,8 +4782,12 @@ void erts_init_bif(void) await_port_send_result_trap = erts_export_put(am_erts_internal, am_await_port_send_result, 3); await_sched_wall_time_mod_trap - = erts_export_put(am_erlang, am_await_sched_wall_time_modifications, 2); + = erts_export_put(am_erlang, am_await_sched_wall_time_modifications, 2); + await_msacc_mod_trap + = erts_export_put(am_erts_internal, am_await_microstate_accounting_modifications, 3); + erts_smp_atomic32_init_nob(&sched_wall_time, 0); + erts_smp_atomic32_init_nob(&msacc, ERTS_MSACC_IS_ENABLED()); } #ifdef HARDDEBUG diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h index c6ed60376a..a62eddf36b 100644 --- a/erts/emulator/beam/bif.h +++ b/erts/emulator/beam/bif.h @@ -54,22 +54,24 @@ extern Export *erts_convert_time_unit_trap; (p)->fcalls = -CONTEXT_REDS; \ } while(0) - -#define ERTS_VBUMP_ALL_REDS(p) \ +#define ERTS_VBUMP_ALL_REDS_INTERNAL(p, fcalls) \ do { \ if (!ERTS_PROC_GET_SAVED_CALLS_BUF((p))) { \ - if ((p)->fcalls > 0) \ - ERTS_PROC_GET_SCHDATA((p))->virtual_reds += (p)->fcalls; \ - (p)->fcalls = 0; \ + if ((fcalls) > 0) \ + ERTS_PROC_GET_SCHDATA((p))->virtual_reds += (fcalls); \ + (fcalls) = 0; \ } \ else { \ - if ((p)->fcalls > -CONTEXT_REDS) \ + if ((fcalls) > -CONTEXT_REDS) \ ERTS_PROC_GET_SCHDATA((p))->virtual_reds \ - += ((p)->fcalls - (-CONTEXT_REDS)); \ - (p)->fcalls = -CONTEXT_REDS; \ + += ((fcalls) - (-CONTEXT_REDS)); \ + (fcalls) = -CONTEXT_REDS; \ } \ } while(0) +#define ERTS_VBUMP_ALL_REDS(p) \ + ERTS_VBUMP_ALL_REDS_INTERNAL((p), (p)->fcalls) + #define BUMP_REDS(p, gc) do { \ ASSERT(p); \ ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p));\ @@ -110,10 +112,34 @@ do { \ } \ } while(0) -#define ERTS_BIF_REDS_LEFT(p) \ +#define ERTS_VBUMP_LEAVE_REDS_INTERNAL(P, Reds, FCalls) \ + do { \ + if (ERTS_PROC_GET_SAVED_CALLS_BUF((P))) { \ + int nreds__ = ((int)(Reds)) - CONTEXT_REDS; \ + if ((FCalls) > nreds__) { \ + ERTS_PROC_GET_SCHDATA((P))->virtual_reds \ + += (FCalls) - nreds__; \ + (FCalls) = nreds__; \ + } \ + } \ + else { \ + if ((FCalls) > (Reds)) { \ + ERTS_PROC_GET_SCHDATA((P))->virtual_reds \ + += (FCalls) - (Reds); \ + (FCalls) = (Reds); \ + } \ + } \ + } while (0) + +#define ERTS_VBUMP_LEAVE_REDS(P, Reds) \ + ERTS_VBUMP_LEAVE_REDS_INTERNAL(P, Reds, (P)->fcalls) + +#define ERTS_REDS_LEFT(p, FCalls) \ (ERTS_PROC_GET_SAVED_CALLS_BUF((p)) \ - ? ((p)->fcalls > -CONTEXT_REDS ? ((p)->fcalls - (-CONTEXT_REDS)) : 0)\ - : ((p)->fcalls > 0 ? (p)->fcalls : 0)) + ? ((FCalls) > -CONTEXT_REDS ? ((FCalls) - (-CONTEXT_REDS)) : 0) \ + : ((FCalls) > 0 ? (FCalls) : 0)) + +#define ERTS_BIF_REDS_LEFT(p) ERTS_REDS_LEFT(p, p->fcalls) #define BIF_RET2(x, gc) do { \ BUMP_REDS(BIF_P, (gc)); \ diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index 3177d5dae7..77d3d95673 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -115,7 +115,7 @@ bif erlang:time_offset/0 bif erlang:time_offset/1 bif erlang:timestamp/0 -bif erlang:open_port/2 +bif erts_internal:open_port/2 bif erlang:pid_to_list/1 bif erlang:ports/0 @@ -125,7 +125,6 @@ bif erlang:process_flag/3 bif erlang:process_info/1 bif erlang:process_info/2 bif erlang:processes/0 -bif erlang:purge_module/1 bif erlang:put/2 bif erlang:register/2 bif erlang:registered/0 @@ -167,10 +166,11 @@ bif erts_internal:request_system_task/3 bif erts_internal:check_process_code/2 bif erts_internal:map_to_tuple_keys/1 -bif erts_internal:map_type/1 +bif erts_internal:term_type/1 bif erts_internal:map_hashmap_children/1 bif erts_internal:time_unit/0 +bif erts_internal:perf_counter_unit/0 bif erts_internal:is_system_process/1 @@ -372,6 +372,7 @@ bif os:getpid/0 bif os:timestamp/0 bif os:system_time/0 bif os:system_time/1 +bif os:perf_counter/0 # # Bifs in the erl_ddll module (the module actually does not exist) @@ -641,6 +642,17 @@ bif ets:update_counter/4 bif erts_debug:map_info/1 # +# New in 19.0 +# + +bif erts_internal:copy_literals/2 +bif erts_internal:purge_module/1 +bif binary:split/2 +bif binary:split/3 +bif erts_debug:size_shared/1 +bif erts_debug:copy_shared/1 + +# # Obsolete # diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c index 15bcd44fb9..11838e24ef 100644 --- a/erts/emulator/beam/big.c +++ b/erts/emulator/beam/big.c @@ -48,7 +48,7 @@ _t_dst = (dst)+((sz)-1); \ _t_src = (src)+((sz)-1); \ while(_t_sz--) *_t_dst-- = *_t_src--; \ - } \ + } \ } while(0) /* add a and b with carry in + out */ @@ -423,6 +423,25 @@ #endif +/* Forward declaration of lookup tables (See below in this file) used in list to + * integer conversions for different bases. Also used in bignum printing. + */ +static const byte digits_per_sint_lookup[36-1]; +static const byte digits_per_small_lookup[36-1]; +static const Sint largest_power_of_base_lookup[36-1]; + +static ERTS_INLINE byte get_digits_per_signed_int(Uint base) { + return digits_per_sint_lookup[base-2]; +} + +static ERTS_INLINE byte get_digits_per_small(Uint base) { + return digits_per_small_lookup[base-2]; +} + +static ERTS_INLINE Sint get_largest_power_of_base(Uint base) { + return largest_power_of_base_lookup[base-2]; +} + /* ** compare two number vectors */ @@ -1487,20 +1506,8 @@ Eterm uint_to_big(Uint x, Eterm *y) Eterm uword_to_big(UWord x, Eterm *y) { -#if HALFWORD_HEAP - Uint upper = x >> 32; - Uint lower = x & 0xFFFFFFFFUL; - if (upper == 0) { - *y = make_pos_bignum_header(1); - } else { - *y = make_pos_bignum_header(2); - BIG_DIGIT(y, 1) = upper; - } - BIG_DIGIT(y, 0) = lower; -#else *y = make_pos_bignum_header(1); BIG_DIGIT(y, 0) = x; -#endif return make_big(y); } @@ -1525,7 +1532,7 @@ Eterm small_to_big(Sint x, Eterm *y) Eterm erts_uint64_to_big(Uint64 x, Eterm **hpp) { Eterm *hp = *hpp; -#if defined(ARCH_32) || HALFWORD_HEAP +#if defined(ARCH_32) if (x >= (((Uint64) 1) << 32)) { *hp = make_pos_bignum_header(2); BIG_DIGIT(hp, 0) = (Uint) (x & ((Uint) 0xffffffff)); @@ -1555,7 +1562,7 @@ Eterm erts_sint64_to_big(Sint64 x, Eterm **hpp) neg = 1; ux = -(Uint64)x; } -#if defined(ARCH_32) || HALFWORD_HEAP +#if defined(ARCH_32) if (ux >= (((Uint64) 1) << 32)) { if (neg) *hp = make_neg_bignum_header(2); @@ -1588,7 +1595,7 @@ erts_uint64_array_to_big(Uint **hpp, int neg, int len, Uint64 *array) pot_digits = digits = 0; for (i = 0; i < len; i++) { -#if defined(ARCH_32) || HALFWORD_HEAP +#if defined(ARCH_32) Uint low_val = array[i] & ((Uint) 0xffffffff); Uint high_val = (array[i] >> 32) & ((Uint) 0xffffffff); BIG_DIGIT(headerp, pot_digits) = low_val; @@ -1651,8 +1658,6 @@ big_to_double(Wterm x, double* resp) /* * Logic has been copied from erl_bif_guard.c and slightly * modified to use a static instead of dynamic heap - * - * HALFWORD: Return relative term with 'heap' as base. */ Eterm double_to_big(double x, Eterm *heap, Uint hsz) @@ -1683,7 +1688,7 @@ double_to_big(double x, Eterm *heap, Uint hsz) sz = BIG_NEED_SIZE(ds); /* number of words including arity */ hp = heap; - res = make_big_rel(hp, heap); + res = make_big(hp); xp = (ErtsDigit*) (hp + 1); ASSERT(ds < hsz); @@ -1733,8 +1738,10 @@ static Uint write_big(Wterm x, void (*write_func)(void *, char), void *arg) short sign = BIG_SIGN(xp); ErtsDigit rem; Uint n = 0; + const Uint digits_per_Sint = get_digits_per_signed_int(10); + const Sint largest_pow_of_base = get_largest_power_of_base(10); - if (xl == 1 && *dx < D_DECIMAL_BASE) { + if (xl == 1 && *dx < largest_pow_of_base) { rem = *dx; if (rem == 0) { (*write_func)(arg, '0'); n++; @@ -1752,7 +1759,7 @@ static Uint write_big(Wterm x, void (*write_func)(void *, char), void *arg) MOVE_DIGITS(tmp, dx, xl); while(1) { - tmpl = D_div(tmp, tmpl, D_DECIMAL_BASE, tmp, &rem); + tmpl = D_div(tmp, tmpl, largest_pow_of_base, tmp, &rem); if (tmpl == 1 && *tmp == 0) { while(rem) { (*write_func)(arg, (rem % 10)+'0'); n++; @@ -1760,7 +1767,7 @@ static Uint write_big(Wterm x, void (*write_func)(void *, char), void *arg) } break; } else { - int i = D_DECIMAL_EXP; + Uint i = digits_per_Sint; while(i--) { (*write_func)(arg, (rem % 10)+'0'); n++; rem /= 10; @@ -2536,63 +2543,100 @@ int term_equals_2pow32(Eterm x) } } +static ERTS_INLINE int c2int_is_invalid_char(byte ch, int base) { + return (ch < '0' + || (ch > ('0' + base - 1) + && !(base > 10 + && ((ch >= 'a' && ch < ('a' + base - 10)) + || (ch >= 'A' && ch < ('A' + base - 10)))))); +} -#define IS_VALID_CHARACTER(CHAR,BASE) \ - (CHAR < '0' \ - || (CHAR > ('0' + BASE - 1) \ - && !(BASE > 10 \ - && ((CHAR >= 'a' && CHAR < ('a' + BASE - 10)) \ - || (CHAR >= 'A' && CHAR < ('A' + BASE - 10)))))) -#define CHARACTER_FROM_BASE(CHAR) \ - ((CHAR <= '9') ? CHAR - '0' : 10 + ((CHAR <= 'Z') ? CHAR - 'A' : CHAR - 'a')) -#define D_BASE_EXP(BASE) (d_base_exp_lookup[BASE-2]) -#define D_BASE_BASE(BASE) (d_base_base_lookup[BASE-2]) -#define LG2_LOOKUP(BASE) (lg2_lookup[base-2]) +static ERTS_INLINE byte c2int_digit_from_base(byte ch) { + return ch <= '9' ? ch - '0' + : (10 + (ch <= 'Z' ? ch - 'A' : ch - 'a')); +} /* - * for i in 2..64 do - * lg2_lookup[i-2] = log2(i) - * end - * How many bits are needed to store string of size n + * How many bits are needed to store 1 digit of given base in binary + * Wo.Alpha formula: Table [log2[n], {n,2,36}] */ -const double lg2_lookup[] = { 1.0, 1.58496, 2, 2.32193, 2.58496, 2.80735, 3.0, - 3.16993, 3.32193, 3.45943, 3.58496, 3.70044, 3.80735, 3.90689, 4.0, - 4.08746, 4.16993, 4.24793, 4.32193, 4.39232, 4.45943, 4.52356, 4.58496, - 4.64386, 4.70044, 4.75489, 4.80735, 4.85798, 4.90689, 4.9542, 5.0, - 5.04439, 5.08746, 5.12928, 5.16993, 5.20945, 5.24793, 5.2854, 5.32193, - 5.35755, 5.39232, 5.42626, 5.45943, 5.49185, 5.52356, 5.55459, 5.58496, - 5.61471, 5.64386, 5.67243, 5.70044, 5.72792, 5.75489, 5.78136, 5.80735, - 5.83289, 5.85798, 5.88264, 5.90689, 5.93074, 5.9542, 5.97728, 6.0 }; +static const double lg2_lookup[36-1] = { + 1.0, 1.58496, 2.0, 2.32193, 2.58496, 2.80735, 3.0, 3.16993, 3.32193, + 3.45943, 3.58496, 3.70044, 3.80735, 3.90689, 4.0, 4.08746, 4.16993, 4.24793, + 4.32193, 4.39232, 4.45943, 4.52356, 4.58496, 4.64386, 4.70044, 4.75489, + 4.80735, 4.85798, 4.90689, 4.9542, 5.0, 5.04439, 5.08746, 5.12928, 5.16993 +}; +static ERTS_INLINE double lookup_log2(Uint base) { + return lg2_lookup[base - 2]; +} /* - * for i in 2..64 do - * d_base_exp_lookup[i-2] = 31 / lg2_lookup[i-2]; - * end - * How many characters can fit in 31 bits + * How many digits can fit into a signed int (Sint) for given base, we take + * one digit away just to be on the safer side (some corner cases). */ -const byte d_base_exp_lookup[] = { 31, 19, 15, 13, 11, 11, 10, 9, 9, 8, 8, 8, 8, - 7, 7, 7, 7, 7, 7, 7, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5 }; +static const byte digits_per_sint_lookup[36-1] = { +#if (SIZEOF_VOID_P == 4) + /* Wo.Alpha formula: Table [Trunc[31 / log[2,n]]-1, {n, 2, 36}] */ + 30, 18, 14, 12, 10, 10, 9, 8, 8, 7, 7, 7, 7, 6, 6, 6, 6, 6, 6, 6, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 4 +#elif (SIZEOF_VOID_P == 8) + /* Wo.Alpha formula: Table [Trunc[63 / log[2,n]]-1, {n, 2, 36}] */ + 62, 38, 30, 26, 23, 21, 20, 18, 17, 17, 16, 16, 15, 15, 14, 14, 14, 13, 13, + 13, 13, 12, 12, 12, 12, 12, 12, 11, 11, 11, 11, 11, 11, 11, 11 +#else + #error "Please produce a lookup table for the new architecture" +#endif +}; + +/* + * How many digits can fit into Erlang Small (SMALL_BITS-1) counting sign bit + */ +static const byte digits_per_small_lookup[36-1] = { +#if (SIZEOF_VOID_P == 4) + /* Wo.Alpha formula: Table [Trunc[27 / log[2,n]]-1, {n, 2, 36}] */ + 27, 17, 13, 11, 10, 9, 9, 8, 8, 7, 7, 7, 7, 6, 6, 6, 6, 6, 6, 6, 6, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5 +#elif (SIZEOF_VOID_P == 8) + /* Wo.Alpha formula: Table [Trunc[59 / log[2,n]]-1, {n, 2, 36}] */ + 59, 37, 29, 25, 22, 21, 19, 18, 17, 17, 16, 15, 15, 15, 14, 14, 14, 13, 13, + 13, 13, 13, 12, 12, 12, 12, 12, 12, 12, 11, 11, 11, 11, 11, 11 +#else + #error "Please produce a lookup table for the new architecture" +#endif +}; /* - * for i in 2..64 do - * d_base_base_lookup[i-2] = pow(i,d_base_exp_lookup[i-2]); - * end - * How much can the characters which fit in 31 bit represent + * Largest power of base which can be represented in a signed int (Sint). + * Calculated by base 2..36 to the power of corresponding element from + * digits_per_sint_lookup. */ -const Uint d_base_base_lookup[] = { 2147483648u, 1162261467u, 1073741824u, - 1220703125u, 362797056u, 1977326743u, 1073741824u, 387420489u, - 1000000000u, 214358881u, 429981696u, 815730721u, 1475789056u, - 170859375u, 268435456u, 410338673u, 612220032u, 893871739u, 1280000000u, - 1801088541u, 113379904u, 148035889u, 191102976u, 244140625u, 308915776u, - 387420489u, 481890304u, 594823321u, 729000000u, 887503681u, 1073741824u, - 1291467969u, 1544804416u, 1838265625u, 60466176u, 69343957u, 79235168u, - 90224199u, 102400000u, 115856201u, 130691232u, 147008443u, 164916224u, - 184528125u, 205962976u, 229345007u, 254803968u, 282475249u, 312500000u, - 345025251u, 380204032u, 418195493u, 459165024u, 503284375u, 550731776u, - 601692057u, 656356768u, 714924299u, 777600000u, 844596301u, 916132832u, - 992436543u, 1073741824u }; +static const Sint largest_power_of_base_lookup[36-1] = { +#if (SIZEOF_VOID_P == 4) + /* Wo.Alpha formula: Table [Pow[n, Trunc[31 / log[2,n]]-1], {n, 2, 36}] */ + 1073741824, 387420489, 268435456, 244140625, 60466176, 282475249, 134217728, + 43046721, 100000000, 19487171, 35831808, 62748517, 105413504, 11390625, + 16777216, 24137569, 34012224, 47045881, 64000000, 85766121, 5153632, + 6436343,7962624, 9765625, 11881376, 14348907, 17210368, 20511149, 24300000, + 28629151, 33554432, 39135393, 45435424, 52521875, 1679616 +#elif (SIZEOF_VOID_P == 8) + /* Wo.Alpha formula: Table [Pow[n, Trunc[63 / log[2,n]]-1], {n, 2, 36}] + * with LL added after each element manually */ + 4611686018427387904LL, 1350851717672992089LL, 1152921504606846976LL, + 1490116119384765625LL, 789730223053602816LL, 558545864083284007LL, + 1152921504606846976LL, 150094635296999121LL, 100000000000000000LL, + 505447028499293771LL, 184884258895036416LL, 665416609183179841LL, + 155568095557812224LL, 437893890380859375LL, 72057594037927936LL, + 168377826559400929LL, 374813367582081024LL, 42052983462257059LL, + 81920000000000000LL, 154472377739119461LL, 282810057883082752LL, + 21914624432020321LL, 36520347436056576LL, 59604644775390625LL, + 95428956661682176LL, 150094635296999121LL, 232218265089212416LL, + 12200509765705829LL, 17714700000000000LL, 25408476896404831LL, + 36028797018963968LL, 50542106513726817LL, 70188843638032384LL, + 96549157373046875LL, 131621703842267136LL +#else + #error "Please produce a lookup table for the new architecture" +#endif +}; Eterm erts_chars_to_integer(Process *BIF_P, char *bytes, Uint size, const int base) { @@ -2602,8 +2646,11 @@ Eterm erts_chars_to_integer(Process *BIF_P, char *bytes, int neg = 0; byte b; Eterm *hp, *hp_end; - int m; + Sint m; int lg2; + const Uint digits_per_small = get_digits_per_small(base); + const Uint digits_per_Sint = get_digits_per_signed_int(base); + const Sint largest_pow_of_base = get_largest_power_of_base(base); if (size == 0) goto bytebuf_to_integer_1_error; @@ -2618,57 +2665,68 @@ Eterm erts_chars_to_integer(Process *BIF_P, char *bytes, size--; } + /* Trim leading zeroes */ + if (size) { + while (*bytes == '0') { + bytes++; + size--; + if (!size) { + /* All zero! */ + res = make_small(0); + goto bytebuf_to_integer_1_done; + } + } + } + if (size == 0) goto bytebuf_to_integer_1_error; - if (size < SMALL_DIGITS && base <= 10) { - /* * - * Take shortcut if we know that all chars are '0' < b < '9' and - * fit in a small. This improves speed by about 10% over the generic - * small case. - * */ - while (size--) { - b = *bytes++; + if (size < digits_per_small) { + if (base <= 10) { + /* * + * Take shortcut if we know that all chars are '0' < b < '9' and + * fit in a small. This improves speed by about 10% over the generic + * small case. + * */ + while (size--) { + b = *bytes++; - if (b < '0' || b > ('0'+base-1)) - goto bytebuf_to_integer_1_error; + if (b < '0' || b > ('0'+base-1)) + goto bytebuf_to_integer_1_error; - i = i * base + b - '0'; - } + i = i * base + b - '0'; + } - if (neg) - i = -i; - res = make_small(i); - goto bytebuf_to_integer_1_done; + if (neg) + i = -i; + res = make_small(i); + goto bytebuf_to_integer_1_done; + } + + /* Take shortcut if we know it will fit in a small. + * This improves speed by about 30%. + */ + while (size) { + b = *bytes++; + size--; + + if (c2int_is_invalid_char(b, base)) + goto bytebuf_to_integer_1_error; + + i = i * base + c2int_digit_from_base(b); + } + + if (neg) + i = -i; + res = make_small(i); + goto bytebuf_to_integer_1_done; } /* * Calculate the maximum number of bits which will * be needed to represent the binary */ - lg2 = ((size+2)*LG2_LOOKUP(base)+1); - - if (lg2 < SMALL_BITS) { - /* Take shortcut if we know it will fit in a small. - * This improves speed by about 30%. - */ - while (size) { - b = *bytes++; - size--; - - if (IS_VALID_CHARACTER(b,base)) - goto bytebuf_to_integer_1_error; - - i = i * base + CHARACTER_FROM_BASE(b); - - } - - if (neg) - i = -i; - res = make_small(i); - goto bytebuf_to_integer_1_done; - - } + lg2 = ((size+2)*lookup_log2(base)+1); /* Start calculating bignum */ m = (lg2 + D_EXP-1)/D_EXP; @@ -2677,8 +2735,8 @@ Eterm erts_chars_to_integer(Process *BIF_P, char *bytes, hp = HAlloc(BIF_P, m); hp_end = hp + m; - if ((i = (size % D_BASE_EXP(base))) == 0) - i = D_BASE_EXP(base); + if ((i = (size % digits_per_Sint)) == 0) + i = digits_per_Sint; n = size - i; m = 0; @@ -2686,34 +2744,34 @@ Eterm erts_chars_to_integer(Process *BIF_P, char *bytes, while (i--) { b = *bytes++; - if (IS_VALID_CHARACTER(b,base)) { + if (c2int_is_invalid_char(b,base)) { HRelease(BIF_P, hp_end, hp); goto bytebuf_to_integer_1_error; } - m = base * m + CHARACTER_FROM_BASE(b); + m = base * m + c2int_digit_from_base(b); } res = small_to_big(m, hp); while (n) { - i = D_BASE_EXP(base); - n -= D_BASE_EXP(base); + i = digits_per_Sint; + n -= digits_per_Sint; m = 0; while (i--) { b = *bytes++; - if (IS_VALID_CHARACTER(b,base)) { + if (c2int_is_invalid_char(b,base)) { HRelease(BIF_P, hp_end, hp); goto bytebuf_to_integer_1_error; } - m = base * m + CHARACTER_FROM_BASE(b); + m = base * m + c2int_digit_from_base(b); } if (is_small(res)) { res = small_to_big(signed_val(res), hp); } - res = big_times_small(res, D_BASE_BASE(base), hp); + res = big_times_small(res, largest_pow_of_base, hp); if (is_small(res)) { res = small_to_big(signed_val(res), hp); } @@ -2744,5 +2802,166 @@ bytebuf_to_integer_1_error: bytebuf_to_integer_1_done: return res; +} +/* Converts list of digits with given 'base' to integer sequentially. Returns + * result in 'integer_out', remaining tail goes to 'tail_out' and returns result + * code if the list was consumed fully or partially or there was an error + */ +LTI_result_t erts_list_to_integer(Process *BIF_P, Eterm orig_list, + const Uint base, + Eterm *integer_out, Eterm *tail_out) +{ + Sint i = 0; + Uint ui = 0; + int skip = 0; + int neg = 0; + Sint n = 0; + Sint m; + int lg2; + Eterm res; + Eterm lst = orig_list; + Eterm tail = lst; + int error_res = LTI_BAD_STRUCTURE; + const Uint digits_per_small = get_digits_per_small(base); + const Uint digits_per_Sint = get_digits_per_signed_int(base); + + if (is_nil(lst)) { + error_res = LTI_NO_INTEGER; + error: + *tail_out = tail; + *integer_out = make_small(0); + return error_res; + } + if (is_not_list(lst)) + goto error; + + /* if first char is a '-' then it is a negative integer */ + if (CAR(list_val(lst)) == make_small('-')) { + neg = 1; + skip = 1; + lst = CDR(list_val(lst)); + if (is_not_list(lst)) { + tail = lst; + error_res = LTI_NO_INTEGER; + goto error; + } + } else if (CAR(list_val(lst)) == make_small('+')) { + /* ignore plus */ + skip = 1; + lst = CDR(list_val(lst)); + if (is_not_list(lst)) { + tail = lst; + error_res = LTI_NO_INTEGER; + goto error; + } + } + + /* Calculate size and do type check */ + + while(1) { + byte ch; + if (is_not_small(CAR(list_val(lst)))) { + break; + } + ch = unsigned_val(CAR(list_val(lst))); + if (c2int_is_invalid_char(ch, base)) { + break; + } + ui = ui * base; + ui = ui + c2int_digit_from_base(ch); + n++; + lst = CDR(list_val(lst)); + if (is_nil(lst)) { + break; + } + if (is_not_list(lst)) { + break; + } + } + + tail = lst; + if (!n) { + error_res = LTI_NO_INTEGER; + goto error; + } + + + /* If length fits inside Sint then we know it's a small int. Else we + * must construct a bignum and let that routine do the checking + */ + + if (n <= digits_per_small) { /* It must be small */ + i = neg ? -(Sint)ui : (Sint)ui; + res = make_small(i); + } else { + const Sint largest_pow_of_base = get_largest_power_of_base(base); + Eterm *hp; + Eterm *hp_end; + + /* Convert from log_base to log2 using lookup table */ + lg2 = ((n+2)*lookup_log2(base)+1); + m = (lg2+D_EXP-1)/D_EXP; /* number of digits */ + m = BIG_NEED_SIZE(m); /* number of words + thing */ + + hp = HAlloc(BIF_P, m); + hp_end = hp + m; + + lst = orig_list; + if (skip) + lst = CDR(list_val(lst)); + + /* load first digits (at least one digit) */ + if ((i = (n % digits_per_Sint)) == 0) + i = digits_per_Sint; + n -= i; + m = 0; + while(i--) { + m *= base; + m += c2int_digit_from_base(unsigned_val(CAR(list_val(lst)))); + lst = CDR(list_val(lst)); + } + res = small_to_big(m, hp); /* load first digits */ + + while(n) { + i = digits_per_Sint; + n -= digits_per_Sint; + m = 0; + while(i--) { + m *= base; + m += c2int_digit_from_base(unsigned_val(CAR(list_val(lst)))); + lst = CDR(list_val(lst)); + } + if (is_small(res)) + res = small_to_big(signed_val(res), hp); + res = big_times_small(res, largest_pow_of_base, hp); + if (is_small(res)) + res = small_to_big(signed_val(res), hp); + res = big_plus_small(res, m, hp); + } + + if (neg) { + if (is_small(res)) + res = make_small(-signed_val(res)); + else { + Uint *big = big_val(res); /* point to thing */ + *big = bignum_header_neg(*big); + } + } + + if (is_not_small(res)) { + res = big_plus_small(res, 0, hp); /* includes conversion to small */ + + if (is_not_small(res)) { + hp += (big_arity(res)+1); + } + } + HRelease(BIF_P, hp_end, hp); + } + *integer_out = res; + *tail_out = tail; + if (tail != NIL) { + return LTI_SOME_INTEGER; + } + return LTI_ALL_INTEGER; } diff --git a/erts/emulator/beam/big.h b/erts/emulator/beam/big.h index 4aa9724ae3..9c92de6b55 100644 --- a/erts/emulator/beam/big.h +++ b/erts/emulator/beam/big.h @@ -35,7 +35,7 @@ typedef Uint ErtsDigit; -#if ((SIZEOF_VOID_P == 4) || HALFWORD_HEAP) && defined(SIZEOF_LONG_LONG) && (SIZEOF_LONG_LONG == 8) +#if (SIZEOF_VOID_P == 4) && defined(SIZEOF_LONG_LONG) && (SIZEOF_LONG_LONG == 8) /* Assume 32-bit machine with long long support */ typedef Uint64 ErtsDoubleDigit; typedef Uint16 ErtsHalfDigit; @@ -54,9 +54,6 @@ typedef Uint32 ErtsHalfDigit; #error "can not determine machine size" #endif -#define D_DECIMAL_EXP 9 -#define D_DECIMAL_BASE 1000000000 - typedef Uint dsize_t; /* Vector size type */ #define D_EXP (ERTS_SIZEOF_ETERM*8) @@ -90,13 +87,9 @@ typedef Uint dsize_t; /* Vector size type */ #define BIG_UINT_HEAP_SIZE (1 + 1) /* always, since sizeof(Uint) <= sizeof(Eterm) */ -#if HALFWORD_HEAP -#define BIG_UWORD_HEAP_SIZE(UW) (((UW) >> (sizeof(Uint) * 8)) ? 3 : 2) -#else #define BIG_UWORD_HEAP_SIZE(UW) BIG_UINT_HEAP_SIZE -#endif -#if defined(ARCH_32) || HALFWORD_HEAP +#if defined(ARCH_32) #define ERTS_UINT64_BIG_HEAP_SIZE__(X) \ ((X) >= (((Uint64) 1) << 32) ? (1 + 2) : (1 + 1)) @@ -177,5 +170,15 @@ Eterm erts_sint64_to_big(Sint64, Eterm **); Eterm erts_chars_to_integer(Process *, char*, Uint, const int); +/* How list_to_integer classifies the input, was it even a string? */ +typedef enum { + LTI_BAD_STRUCTURE = 0, + LTI_NO_INTEGER = 1, + LTI_SOME_INTEGER = 2, + LTI_ALL_INTEGER = 3 +} LTI_result_t; + +LTI_result_t erts_list_to_integer(Process *BIF_P, Eterm orig_list, + const Uint base, + Eterm *integer_out, Eterm *tail_out); #endif - diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c index e670fbf31c..d3e481c7f9 100644 --- a/erts/emulator/beam/binary.c +++ b/erts/emulator/beam/binary.c @@ -34,6 +34,9 @@ #include "erl_binary.h" #include "erl_bits.h" +#define L2B_B2L_MIN_EXEC_REDS (CONTEXT_REDS/4) +#define L2B_B2L_RESCHED_REDS (CONTEXT_REDS/40) + static Export binary_to_list_continue_export; static Export list_to_binary_continue_export; @@ -415,10 +418,10 @@ binary_to_list_chunk(Process *c_p, } static ERTS_INLINE BIF_RETTYPE -binary_to_list(Process *c_p, Eterm *hp, Eterm tail, byte *bytes, Uint size, Uint bitoffs) +binary_to_list(Process *c_p, Eterm *hp, Eterm tail, byte *bytes, + Uint size, Uint bitoffs, int reds_left, int one_chunk) { - int reds_left = ERTS_BIF_REDS_LEFT(c_p); - if (size < reds_left*ERTS_B2L_BYTES_PER_REDUCTION) { + if (one_chunk) { Eterm res; BIF_RETTYPE ret; int bump_reds = (size - 1)/ERTS_B2L_BYTES_PER_REDUCTION + 1; @@ -472,11 +475,29 @@ BIF_RETTYPE binary_to_list_1(BIF_ALIST_1) Uint size; Uint bitsize; Uint bitoffs; + int reds_left; + int one_chunk; if (is_not_binary(BIF_ARG_1)) { goto error; } + size = binary_size(BIF_ARG_1); + reds_left = ERTS_BIF_REDS_LEFT(BIF_P); + one_chunk = size < reds_left*ERTS_B2L_BYTES_PER_REDUCTION; + if (!one_chunk) { + if (size < L2B_B2L_MIN_EXEC_REDS*ERTS_B2L_BYTES_PER_REDUCTION) { + if (reds_left <= L2B_B2L_RESCHED_REDS) { + /* Yield and do it with full context reds... */ + ERTS_BIF_YIELD1(bif_export[BIF_binary_to_list_1], + BIF_P, BIF_ARG_1); + } + /* Allow a bit more reductions... */ + one_chunk = 1; + reds_left = L2B_B2L_MIN_EXEC_REDS; + } + } + ERTS_GET_REAL_BIN(BIF_ARG_1, real_bin, offset, bitoffs, bitsize); if (bitsize != 0) { goto error; @@ -486,7 +507,8 @@ BIF_RETTYPE binary_to_list_1(BIF_ALIST_1) } else { Eterm* hp = HAlloc(BIF_P, 2 * size); byte* bytes = binary_bytes(real_bin)+offset; - return binary_to_list(BIF_P, hp, NIL, bytes, size, bitoffs); + return binary_to_list(BIF_P, hp, NIL, bytes, size, + bitoffs, reds_left, one_chunk); } error: @@ -505,6 +527,8 @@ BIF_RETTYPE binary_to_list_3(BIF_ALIST_3) Uint start; Uint stop; Eterm* hp; + int reds_left; + int one_chunk; if (is_not_binary(BIF_ARG_1)) { goto error; @@ -513,6 +537,21 @@ BIF_RETTYPE binary_to_list_3(BIF_ALIST_3) goto error; } size = binary_size(BIF_ARG_1); + reds_left = ERTS_BIF_REDS_LEFT(BIF_P); + one_chunk = size < reds_left*ERTS_B2L_BYTES_PER_REDUCTION; + if (!one_chunk) { + if (size < L2B_B2L_MIN_EXEC_REDS*ERTS_B2L_BYTES_PER_REDUCTION) { + if (reds_left <= L2B_B2L_RESCHED_REDS) { + /* Yield and do it with full context reds... */ + ERTS_BIF_YIELD3(bif_export[BIF_binary_to_list_3], + BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); + } + /* Allow a bit more reductions... */ + one_chunk = 1; + reds_left = L2B_B2L_MIN_EXEC_REDS; + } + } + ERTS_GET_BINARY_BYTES(BIF_ARG_1, bytes, bitoffs, bitsize); if (start < 1 || start > size || stop < 1 || stop > size || stop < start ) { @@ -520,7 +559,8 @@ BIF_RETTYPE binary_to_list_3(BIF_ALIST_3) } i = stop-start+1; hp = HAlloc(BIF_P, 2*i); - return binary_to_list(BIF_P, hp, NIL, bytes+start-1, i, bitoffs); + return binary_to_list(BIF_P, hp, NIL, bytes+start-1, i, + bitoffs, reds_left, one_chunk); error: BIF_ERROR(BIF_P, BADARG); } @@ -537,11 +577,27 @@ BIF_RETTYPE bitstring_to_list_1(BIF_ALIST_1) byte* bytes; Eterm previous = NIL; Eterm* hp; + int reds_left; + int one_chunk; if (is_not_binary(BIF_ARG_1)) { BIF_ERROR(BIF_P, BADARG); } size = binary_size(BIF_ARG_1); + reds_left = ERTS_BIF_REDS_LEFT(BIF_P); + one_chunk = size < reds_left*ERTS_B2L_BYTES_PER_REDUCTION; + if (!one_chunk) { + if (size < L2B_B2L_MIN_EXEC_REDS*ERTS_B2L_BYTES_PER_REDUCTION) { + if (reds_left <= L2B_B2L_RESCHED_REDS) { + /* Yield and do it with full context reds... */ + ERTS_BIF_YIELD1(bif_export[BIF_bitstring_to_list_1], + BIF_P, BIF_ARG_1); + } + /* Allow a bit more reductions... */ + one_chunk = 1; + reds_left = L2B_B2L_MIN_EXEC_REDS; + } + } ERTS_GET_REAL_BIN(BIF_ARG_1, real_bin, offset, bitoffs, bitsize); bytes = binary_bytes(real_bin)+offset; if (bitsize == 0) { @@ -566,7 +622,8 @@ BIF_RETTYPE bitstring_to_list_1(BIF_ALIST_1) hp += 2; } - return binary_to_list(BIF_P, hp, previous, bytes, size, bitoffs); + return binary_to_list(BIF_P, hp, previous, bytes, size, + bitoffs, reds_left, one_chunk); } @@ -795,8 +852,19 @@ static BIF_RETTYPE list_to_binary_continue(BIF_ALIST_1) BIF_RETTYPE erts_list_to_binary_bif(Process *c_p, Eterm arg, Export *bif) { + int orig_reds_left = ERTS_BIF_REDS_LEFT(c_p); BIF_RETTYPE ret; + if (orig_reds_left < L2B_B2L_MIN_EXEC_REDS) { + if (orig_reds_left <= L2B_B2L_RESCHED_REDS) { + /* Yield and do it with full context reds... */ + ERTS_BIF_PREP_YIELD1(ret, bif, c_p, arg); + return ret; + } + /* Allow a bit more reductions... */ + orig_reds_left = L2B_B2L_MIN_EXEC_REDS; + } + if (is_nil(arg)) ERTS_BIF_PREP_RET(ret, new_binary(c_p, (byte *) "", 0)); else if (is_not_list(arg)) @@ -818,7 +886,6 @@ BIF_RETTYPE erts_list_to_binary_bif(Process *c_p, Eterm arg, Export *bif) bif, erts_iolist_size_yielding, erts_iolist_to_buf_yielding); - int orig_reds_left = ERTS_BIF_REDS_LEFT(c_p); /* * First try to do it all at once without having to use diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index 4ce9d24479..2c8ecf04be 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -252,7 +252,7 @@ print_process_info(int to, void *to_arg, Process *p) /* display the message queue only if there is anything in it */ if (!ERTS_IS_CRASH_DUMPING && p->msg.first != NULL && !garbing) { - ErlMessage* mp; + ErtsMessage* mp; erts_print(to, to_arg, "Message queue: ["); for (mp = p->msg.first; mp; mp = mp->next) erts_print(to, to_arg, mp->next ? "%T," : "%T", ERL_MESSAGE_TERM(mp)); @@ -323,7 +323,7 @@ print_process_info(int to, void *to_arg, Process *p) erts_print(to, to_arg, "Heap unused: %bpu\n", (p->hend - p->htop)); erts_print(to, to_arg, "OldHeap unused: %bpu\n", (OLD_HEAP(p) == NULL) ? 0 : (OLD_HEND(p) - OLD_HTOP(p)) ); - erts_print(to, to_arg, "Memory: %beu\n", erts_process_memory(p)); + erts_print(to, to_arg, "Memory: %beu\n", erts_process_memory(p, !0)); if (garbing) { print_garb_info(to, to_arg, p); @@ -381,7 +381,7 @@ loaded(int to, void *to_arg) int i; int old = 0; int cur = 0; - BeamInstr* code; + BeamCodeHeader* code; Module* modp; ErtsCodeIndex code_ix; @@ -439,30 +439,30 @@ loaded(int to, void *to_arg) erts_print(to, to_arg, "\n"); erts_print(to, to_arg, "Current size: %d\n", modp->curr.code_length); - code = modp->curr.code; - if (code != NULL && code[MI_ATTR_PTR]) { + code = modp->curr.code_hdr; + if (code != NULL && code->attr_ptr) { erts_print(to, to_arg, "Current attributes: "); - dump_attributes(to, to_arg, (byte *) code[MI_ATTR_PTR], - code[MI_ATTR_SIZE]); + dump_attributes(to, to_arg, code->attr_ptr, + code->attr_size); } - if (code != NULL && code[MI_COMPILE_PTR]) { + if (code != NULL && code->compile_ptr) { erts_print(to, to_arg, "Current compilation info: "); - dump_attributes(to, to_arg, (byte *) code[MI_COMPILE_PTR], - code[MI_COMPILE_SIZE]); + dump_attributes(to, to_arg, code->compile_ptr, + code->compile_size); } if (modp->old.code_length != 0) { erts_print(to, to_arg, "Old size: %d\n", modp->old.code_length); - code = modp->old.code; - if (code[MI_ATTR_PTR]) { + code = modp->old.code_hdr; + if (code->attr_ptr) { erts_print(to, to_arg, "Old attributes: "); - dump_attributes(to, to_arg, (byte *) code[MI_ATTR_PTR], - code[MI_ATTR_SIZE]); + dump_attributes(to, to_arg, code->attr_ptr, + code->attr_size); } - if (code[MI_COMPILE_PTR]) { + if (code->compile_ptr) { erts_print(to, to_arg, "Old compilation info: "); - dump_attributes(to, to_arg, (byte *) code[MI_COMPILE_PTR], - code[MI_COMPILE_SIZE]); + dump_attributes(to, to_arg, code->compile_ptr, + code->compile_size); } } } diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index 8849dadd00..67a96f6442 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -35,56 +35,59 @@ #include "erl_bits.h" #include "dtrace-wrapper.h" -static void move_one_frag(Eterm** hpp, ErlHeapFragment*, ErlOffHeap*); +static void move_one_frag(Eterm** hpp, ErlHeapFragment*, ErlOffHeap*, int); /* * Copy object "obj" to process p. */ -Eterm -copy_object(Eterm obj, Process* to) -{ - Uint size = size_object(obj); - Eterm* hp = HAlloc(to, size); - Eterm res; +Eterm copy_object_x(Eterm obj, Process* to, Uint extra) { + if (!is_immed(obj)) { + Uint size = size_object(obj); + Eterm* hp = HAllocX(to, size, extra); + Eterm res; #ifdef USE_VM_PROBES - if (DTRACE_ENABLED(copy_object)) { - DTRACE_CHARBUF(proc_name, 64); + if (DTRACE_ENABLED(copy_object)) { + DTRACE_CHARBUF(proc_name, 64); - erts_snprintf(proc_name, sizeof(DTRACE_CHARBUF_NAME(proc_name)), - "%T", to->common.id); - DTRACE2(copy_object, proc_name, size); - } + erts_snprintf(proc_name, sizeof(DTRACE_CHARBUF_NAME(proc_name)), + "%T", to->common.id); + DTRACE2(copy_object, proc_name, size); + } #endif - res = copy_struct(obj, size, &hp, &to->off_heap); + res = copy_struct(obj, size, &hp, &to->off_heap); #ifdef DEBUG - if (eq(obj, res) == 0) { - erl_exit(ERTS_ABORT_EXIT, "copy not equal to source\n"); - } + if (eq(obj, res) == 0) { + erl_exit(ERTS_ABORT_EXIT, "copy not equal to source\n"); + } #endif - return res; + return res; + } + return obj; } /* * Return the "flat" size of the object. */ -#if HALFWORD_HEAP -Uint size_object_rel(Eterm obj, Eterm* base) -#else Uint size_object(Eterm obj) -#endif { Uint sum = 0; Eterm* ptr; int arity; +#ifdef DEBUG + Eterm mypid = erts_get_current_pid(); +#endif DECLARE_ESTACK(s); + + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] size_object %p\n", mypid, obj)); + for (;;) { switch (primary_tag(obj)) { case TAG_PRIMARY_LIST: sum += 2; - ptr = list_val_rel(obj,base); + ptr = list_val(obj); obj = *ptr++; if (!IS_CONST(obj)) { ESTACK_PUSH(s, obj); @@ -93,11 +96,11 @@ Uint size_object(Eterm obj) break; case TAG_PRIMARY_BOXED: { - Eterm hdr = *boxed_val_rel(obj,base); + Eterm hdr = *boxed_val(obj); ASSERT(is_header(hdr)); switch (hdr & _TAG_HEADER_MASK) { case ARITYVAL_SUBTAG: - ptr = tuple_val_rel(obj,base); + ptr = tuple_val(obj); arity = header_arity(hdr); sum += arity + 1; if (arity == 0) { /* Empty tuple -- unusual. */ @@ -113,7 +116,7 @@ Uint size_object(Eterm obj) break; case FUN_SUBTAG: { - Eterm* bptr = fun_val_rel(obj,base); + Eterm* bptr = fun_val(obj); ErlFunThing* funp = (ErlFunThing *) bptr; unsigned eterms = 1 /* creator */ + funp->num_free; unsigned sz = thing_arityval(hdr); @@ -134,7 +137,7 @@ Uint size_object(Eterm obj) { Uint n; flatmap_t *mp; - mp = (flatmap_t*)flatmap_val_rel(obj,base); + mp = (flatmap_t*)flatmap_val(obj); ptr = (Eterm *)mp; n = flatmap_get_size(mp) + 1; sum += n + 2; @@ -153,7 +156,7 @@ Uint size_object(Eterm obj) { Eterm *head; Uint sz; - head = hashmap_val_rel(obj, base); + head = hashmap_val(obj); sz = hashmap_bitcount(MAP_HEADER_VAL(hdr)); sum += 1 + sz + header_arity(hdr); head += 1 + header_arity(hdr); @@ -182,7 +185,7 @@ Uint size_object(Eterm obj) Uint bitoffs; Uint extra_bytes; Eterm hdr; - ERTS_GET_REAL_BIN_REL(obj, real_bin, offset, bitoffs, bitsize, base); + ERTS_GET_REAL_BIN(obj, real_bin, offset, bitoffs, bitsize); if ((bitsize + bitoffs) > 8) { sum += ERL_SUB_BIN_SIZE; extra_bytes = 2; @@ -192,11 +195,11 @@ Uint size_object(Eterm obj) } else { extra_bytes = 0; } - hdr = *binary_val_rel(real_bin,base); + hdr = *binary_val(real_bin); if (thing_subtag(hdr) == REFC_BINARY_SUBTAG) { sum += PROC_BIN_SIZE; } else { - sum += heap_bin_size(binary_size_rel(obj,base)+extra_bytes); + sum += heap_bin_size(binary_size(obj)+extra_bytes); } goto pop_next; } @@ -214,6 +217,7 @@ Uint size_object(Eterm obj) pop_next: if (ESTACK_ISEMPTY(s)) { DESTROY_ESTACK(s); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] size was: %u\n", mypid, sum)); return sum; } obj = ESTACK_POP(s); @@ -225,14 +229,377 @@ Uint size_object(Eterm obj) } /* + * Machinery for sharing preserving information + * Using a WSTACK but not very transparently; consider refactoring + */ + +#define DECLARE_BITSTORE(s) \ + DECLARE_WSTACK(s); \ + int WSTK_CONCAT(s,_bitoffs) = 0; \ + int WSTK_CONCAT(s,_offset) = 0; \ + UWord WSTK_CONCAT(s,_buffer) = 0 + +#define DESTROY_BITSTORE(s) DESTROY_WSTACK(s) +#define BITSTORE_PUT(s,i) \ +do { \ + WSTK_CONCAT(s,_buffer) |= i << WSTK_CONCAT(s,_bitoffs); \ + WSTK_CONCAT(s,_bitoffs) += 2; \ + if (WSTK_CONCAT(s,_bitoffs) >= 8*sizeof(UWord)) { \ + WSTACK_PUSH(s, WSTK_CONCAT(s,_buffer)); \ + WSTK_CONCAT(s,_bitoffs) = 0; \ + WSTK_CONCAT(s,_buffer) = 0; \ + } \ +} while(0) +#define BITSTORE_CLOSE(s) \ +do { \ + if (WSTK_CONCAT(s,_bitoffs) > 0) { \ + WSTACK_PUSH(s, WSTK_CONCAT(s,_buffer)); \ + WSTK_CONCAT(s,_bitoffs) = 0; \ + } \ +} while(0) + +#define BITSTORE_FETCH(s,dst) \ +do { \ + UWord result; \ + if (WSTK_CONCAT(s,_bitoffs) <= 0) { \ + WSTK_CONCAT(s,_buffer) = s.wstart[WSTK_CONCAT(s,_offset)]; \ + WSTK_CONCAT(s,_offset)++; \ + WSTK_CONCAT(s,_bitoffs) = 8*sizeof(UWord); \ + } \ + WSTK_CONCAT(s,_bitoffs) -= 2; \ + result = WSTK_CONCAT(s,_buffer) & 3; \ + WSTK_CONCAT(s,_buffer) >>= 2; \ + (dst) = result; \ +} while(0) + +#define BOXED_VISITED_MASK ((Eterm) 3) +#define BOXED_VISITED ((Eterm) 1) +#define BOXED_SHARED_UNPROCESSED ((Eterm) 2) +#define BOXED_SHARED_PROCESSED ((Eterm) 3) + +#define COUNT_OFF_HEAP (0) + +#define IN_LITERAL_PURGE_AREA(info, ptr) \ + ((info)->range_ptr && ( \ + (info)->range_ptr <= (ptr) && \ + (ptr) < ((info)->range_ptr + (info)->range_sz))) +/* + * Return the real size of an object and find sharing information + * This currently returns the same as erts_debug:size/1. + * It is argued whether the size of subterms in constant pools + * should be counted or not. + */ + +Uint size_shared(Eterm obj) +{ + Eterm saved_obj = obj; + Uint sum = 0; + Eterm* ptr; + + DECLARE_EQUEUE(s); + DECLARE_BITSTORE(b); + + for (;;) { + switch (primary_tag(obj)) { + case TAG_PRIMARY_LIST: { + Eterm head, tail; + ptr = list_val(obj); + /* we're not counting anything that's outside our heap */ + if (!COUNT_OFF_HEAP && erts_is_literal(obj,ptr)) { + goto pop_next; + } + head = CAR(ptr); + tail = CDR(ptr); + /* if it's visited, don't count it */ + if (primary_tag(tail) == TAG_PRIMARY_HEADER || + primary_tag(head) == TAG_PRIMARY_HEADER) { + goto pop_next; + } + /* else make it visited now */ + switch (primary_tag(tail)) { + case TAG_PRIMARY_LIST: + ptr[1] = (tail - TAG_PRIMARY_LIST) | TAG_PRIMARY_HEADER; + break; + case TAG_PRIMARY_IMMED1: + CAR(ptr) = (head - primary_tag(head)) | TAG_PRIMARY_HEADER; + CDR(ptr) = (tail - TAG_PRIMARY_IMMED1) | primary_tag(head); + break; + case TAG_PRIMARY_BOXED: + BITSTORE_PUT(b, primary_tag(head)); + CAR(ptr) = (head - primary_tag(head)) | TAG_PRIMARY_HEADER; + CDR(ptr) = (tail - TAG_PRIMARY_BOXED) | TAG_PRIMARY_HEADER; + break; + } + /* and count it */ + sum += 2; + if (!IS_CONST(head)) { + EQUEUE_PUT(s, head); + } + obj = tail; + break; + } + case TAG_PRIMARY_BOXED: { + Eterm hdr; + ptr = boxed_val(obj); + /* we're not counting anything that's outside our heap */ + if (!COUNT_OFF_HEAP && erts_is_literal(obj,ptr)) { + goto pop_next; + } + hdr = *ptr; + /* if it's visited, don't count it */ + if (primary_tag(hdr) != TAG_PRIMARY_HEADER) { + goto pop_next; + } + /* else make it visited now */ + *ptr = (hdr - primary_tag(hdr)) + BOXED_VISITED; + /* and count it */ + ASSERT(is_header(hdr)); + switch (hdr & _TAG_HEADER_MASK) { + case ARITYVAL_SUBTAG: { + int arity = header_arity(hdr); + sum += arity + 1; + if (arity == 0) { /* Empty tuple -- unusual. */ + goto pop_next; + } + while (arity-- > 0) { + obj = *++ptr; + if (!IS_CONST(obj)) { + EQUEUE_PUT(s, obj); + } + } + goto pop_next; + } + case FUN_SUBTAG: { + ErlFunThing* funp = (ErlFunThing *) ptr; + unsigned eterms = 1 /* creator */ + funp->num_free; + unsigned sz = thing_arityval(hdr); + sum += 1 /* header */ + sz + eterms; + ptr += 1 /* header */ + sz; + while (eterms-- > 0) { + obj = *ptr++; + if (!IS_CONST(obj)) { + EQUEUE_PUT(s, obj); + } + } + goto pop_next; + } + case SUB_BINARY_SUBTAG: { + ErlSubBin* sb = (ErlSubBin *) ptr; + Uint extra_bytes; + Eterm hdr; + ASSERT((sb->thing_word & ~BOXED_VISITED_MASK) == HEADER_SUB_BIN); + if (sb->bitsize + sb->bitoffs > 8) { + sum += ERL_SUB_BIN_SIZE; + extra_bytes = 2; + } else if (sb->bitsize + sb->bitoffs > 0) { + sum += ERL_SUB_BIN_SIZE; + extra_bytes = 1; + } else { + extra_bytes = 0; + } + ptr = binary_val(sb->orig); + hdr = (*ptr) & ~BOXED_VISITED_MASK; + if (thing_subtag(hdr) == REFC_BINARY_SUBTAG) { + sum += PROC_BIN_SIZE; + } else { + ASSERT(thing_subtag(hdr) == HEAP_BINARY_SUBTAG); + sum += heap_bin_size(binary_size(obj) + extra_bytes); + } + goto pop_next; + } + case MAP_SUBTAG: + switch (MAP_HEADER_TYPE(hdr)) { + case MAP_HEADER_TAG_FLATMAP_HEAD : { + flatmap_t *mp = (flatmap_t*)flatmap_val(obj); + Uint n = flatmap_get_size(mp) + 1; + ptr = (Eterm *)mp; + sum += n + 2; + ptr += 2; /* hdr + size words */ + while (n--) { + obj = *ptr++; + if (!IS_CONST(obj)) { + EQUEUE_PUT(s, obj); + } + } + goto pop_next; + } + case MAP_HEADER_TAG_HAMT_HEAD_BITMAP : + case MAP_HEADER_TAG_HAMT_HEAD_ARRAY : + case MAP_HEADER_TAG_HAMT_NODE_BITMAP : { + Uint n = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + sum += 1 + n + header_arity(hdr); + ptr += 1 + header_arity(hdr); + while (n--) { + obj = *ptr++; + if (!IS_CONST(obj)) { + EQUEUE_PUT(s, obj); + } + } + goto pop_next; + } + default: + erl_exit(ERTS_ABORT_EXIT, "size_shared: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr)); + } + case BIN_MATCHSTATE_SUBTAG: + erl_exit(ERTS_ABORT_EXIT, + "size_shared: matchstate term not allowed"); + default: + sum += thing_arityval(hdr) + 1; + goto pop_next; + } + break; + } + case TAG_PRIMARY_IMMED1: + pop_next: + if (EQUEUE_ISEMPTY(s)) { + goto cleanup; + } + obj = EQUEUE_GET(s); + break; + default: + erl_exit(ERTS_ABORT_EXIT, "size_shared: bad tag for %#x\n", obj); + } + } + +cleanup: + obj = saved_obj; + BITSTORE_CLOSE(b); + for (;;) { + switch (primary_tag(obj)) { + case TAG_PRIMARY_LIST: { + Eterm head, tail; + ptr = list_val(obj); + if (!COUNT_OFF_HEAP && erts_is_literal(obj,ptr)) { + goto cleanup_next; + } + head = CAR(ptr); + tail = CDR(ptr); + /* if not already clean, clean it up */ + if (primary_tag(tail) == TAG_PRIMARY_HEADER) { + if (primary_tag(head) == TAG_PRIMARY_HEADER) { + Eterm saved; + BITSTORE_FETCH(b, saved); + CAR(ptr) = head = (head - TAG_PRIMARY_HEADER) | saved; + CDR(ptr) = tail = (tail - TAG_PRIMARY_HEADER) | TAG_PRIMARY_BOXED; + } else { + CDR(ptr) = tail = (tail - TAG_PRIMARY_HEADER) | TAG_PRIMARY_LIST; + } + } else if (primary_tag(head) == TAG_PRIMARY_HEADER) { + CAR(ptr) = head = (head - TAG_PRIMARY_HEADER) | primary_tag(tail); + CDR(ptr) = tail = (tail - primary_tag(tail)) | TAG_PRIMARY_IMMED1; + } else { + goto cleanup_next; + } + /* and its children too */ + if (!IS_CONST(head)) { + EQUEUE_PUT_UNCHECKED(s, head); + } + obj = tail; + break; + } + case TAG_PRIMARY_BOXED: { + Eterm hdr; + ptr = boxed_val(obj); + if (!COUNT_OFF_HEAP && erts_is_literal(obj,ptr)) { + goto cleanup_next; + } + hdr = *ptr; + /* if not already clean, clean it up */ + if (primary_tag(hdr) == TAG_PRIMARY_HEADER) { + goto cleanup_next; + } + else { + ASSERT(primary_tag(hdr) == BOXED_VISITED); + *ptr = hdr = (hdr - BOXED_VISITED) + TAG_PRIMARY_HEADER; + } + /* and its children too */ + switch (hdr & _TAG_HEADER_MASK) { + case ARITYVAL_SUBTAG: { + int arity = header_arity(hdr); + if (arity == 0) { /* Empty tuple -- unusual. */ + goto cleanup_next; + } + while (arity-- > 0) { + obj = *++ptr; + if (!IS_CONST(obj)) { + EQUEUE_PUT_UNCHECKED(s, obj); + } + } + goto cleanup_next; + } + case FUN_SUBTAG: { + ErlFunThing* funp = (ErlFunThing *) ptr; + unsigned eterms = 1 /* creator */ + funp->num_free; + unsigned sz = thing_arityval(hdr); + ptr += 1 /* header */ + sz; + while (eterms-- > 0) { + obj = *ptr++; + if (!IS_CONST(obj)) { + EQUEUE_PUT_UNCHECKED(s, obj); + } + } + goto cleanup_next; + } + case MAP_SUBTAG: + switch (MAP_HEADER_TYPE(hdr)) { + case MAP_HEADER_TAG_FLATMAP_HEAD : { + flatmap_t *mp = (flatmap_t *) ptr; + Uint n = flatmap_get_size(mp) + 1; + ptr += 2; /* hdr + size words */ + while (n--) { + obj = *ptr++; + if (!IS_CONST(obj)) { + EQUEUE_PUT_UNCHECKED(s, obj); + } + } + goto cleanup_next; + } + case MAP_HEADER_TAG_HAMT_HEAD_BITMAP : + case MAP_HEADER_TAG_HAMT_HEAD_ARRAY : + case MAP_HEADER_TAG_HAMT_NODE_BITMAP : { + Uint n = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + sum += 1 + n + header_arity(hdr); + ptr += 1 + header_arity(hdr); + while (n--) { + obj = *ptr++; + if (!IS_CONST(obj)) { + EQUEUE_PUT_UNCHECKED(s, obj); + } + } + goto cleanup_next; + } + default: + erl_exit(ERTS_ABORT_EXIT, "size_shared: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr)); + } + default: + goto cleanup_next; + } + break; + } + case TAG_PRIMARY_IMMED1: + cleanup_next: + if (EQUEUE_ISEMPTY(s)) { + goto all_clean; + } + obj = EQUEUE_GET(s); + break; + default: + erl_exit(ERTS_ABORT_EXIT, "size_shared: bad tag for %#x\n", obj); + } + } + + all_clean: + /* Return the result */ + DESTROY_EQUEUE(s); + DESTROY_BITSTORE(b); + return sum; +} + + +/* * Copy a structure to a heap. */ -#if HALFWORD_HEAP -Eterm copy_struct_rel(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, - Eterm* src_base, Eterm* dst_base) -#else -Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) -#endif +Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint *bsz) { char* hstart; Uint hsize; @@ -247,19 +614,23 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) Eterm* argp; Eterm* const_tuple; Eterm hdr; + Eterm *hend; int i; #ifdef DEBUG Eterm org_obj = obj; Uint org_sz = sz; + Eterm mypid = erts_get_current_pid(); #endif if (IS_CONST(obj)) return obj; + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] copy_struct %p\n", mypid, obj)); + DTRACE1(copy_struct, (int32_t)sz); hp = htop = *hpp; - hbot = htop + sz; + hbot = hend = htop + sz; hstart = (char *)htop; hsize = (char*) hbot - hstart; const_tuple = 0; @@ -268,7 +639,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) switch (primary_tag(obj)) { case TAG_PRIMARY_LIST: argp = &res; - objp = list_val_rel(obj,src_base); + objp = list_val(obj); goto L_copy_list; case TAG_PRIMARY_BOXED: argp = &res; goto L_copy_boxed; default: @@ -286,14 +657,11 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) hp++; break; case TAG_PRIMARY_LIST: - objp = list_val_rel(obj,src_base); - #if !HALFWORD_HEAP || defined(DEBUG) - if (in_area(objp,hstart,hsize)) { - ASSERT(!HALFWORD_HEAP); + objp = list_val(obj); + if (ErtsInArea(objp,hstart,hsize)) { hp++; break; } - #endif argp = hp++; /* Fall through */ @@ -309,23 +677,15 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) } else { CAR(htop) = elem; - #if HALFWORD_HEAP - CDR(htop) = CDR(objp); - *tailp = make_list_rel(htop,dst_base); - htop += 2; - goto L_copy; - #else tailp = &CDR(htop); htop += 2; - #endif } - ASSERT(!HALFWORD_HEAP || tp < hp || tp >= hbot); - *tp = make_list_rel(tailp - 1, dst_base); + *tp = make_list(tailp - 1); obj = CDR(objp); if (!is_list(obj)) { break; } - objp = list_val_rel(obj,src_base); + objp = list_val(obj); } switch (primary_tag(obj)) { case TAG_PRIMARY_IMMED1: *tailp = obj; goto L_copy; @@ -337,24 +697,21 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) } case TAG_PRIMARY_BOXED: - #if !HALFWORD_HEAP || defined(DEBUG) - if (in_area(boxed_val_rel(obj,src_base),hstart,hsize)) { - ASSERT(!HALFWORD_HEAP); + if (ErtsInArea(boxed_val(obj),hstart,hsize)) { hp++; break; } - #endif argp = hp++; L_copy_boxed: - objp = boxed_val_rel(obj, src_base); + objp = boxed_val(obj); hdr = *objp; switch (hdr & _TAG_HEADER_MASK) { case ARITYVAL_SUBTAG: { int const_flag = 1; /* assume constant tuple */ i = arityval(hdr); - *argp = make_tuple_rel(htop, dst_base); + *argp = make_tuple(htop); tp = htop; /* tp is pointer to new arity value */ *htop++ = *objp++; /* copy arity value */ while (i--) { @@ -383,7 +740,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) while (i--) { *tp++ = *objp++; } - *argp = make_binary_rel(hbot, dst_base); + *argp = make_binary(hbot); pb = (ProcBin*) hbot; erts_refc_inc(&pb->val->refc, 2); pb->next = off_heap->first; @@ -410,7 +767,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) extra_bytes = 0; } real_size = size+extra_bytes; - objp = binary_val_rel(real_bin,src_base); + objp = binary_val(real_bin); if (thing_subtag(*objp) == HEAP_BINARY_SUBTAG) { ErlHeapBin* from = (ErlHeapBin *) objp; ErlHeapBin* to; @@ -440,7 +797,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) off_heap->first = (struct erl_off_heap_header*) to; OH_OVERHEAD(off_heap, to->size / sizeof(Eterm)); } - *argp = make_binary_rel(hbot, dst_base); + *argp = make_binary(hbot); if (extra_bytes != 0) { ErlSubBin* res; hbot -= ERL_SUB_BIN_SIZE; @@ -452,7 +809,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) res->offs = 0; res->is_writable = 0; res->orig = *argp; - *argp = make_binary_rel(hbot, dst_base); + *argp = make_binary(hbot); } break; } @@ -470,7 +827,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) funp->next = off_heap->first; off_heap->first = (struct erl_off_heap_header*) funp; erts_refc_inc(&funp->fe->refc, 2); - *argp = make_fun_rel(tp, dst_base); + *argp = make_fun(tp); } break; case EXTERNAL_PID_SUBTAG: @@ -490,7 +847,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) off_heap->first = (struct erl_off_heap_header*)etp; erts_refc_inc(&etp->node->refc, 2); - *argp = make_external_rel(tp, dst_base); + *argp = make_external(tp); } break; case MAP_SUBTAG: @@ -498,7 +855,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) switch (MAP_HEADER_TYPE(hdr)) { case MAP_HEADER_TAG_FLATMAP_HEAD : i = flatmap_get_size(objp) + 3; - *argp = make_flatmap_rel(htop, dst_base); + *argp = make_flatmap(htop); while (i--) { *htop++ = *objp++; } @@ -509,7 +866,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) case MAP_HEADER_TAG_HAMT_NODE_BITMAP : i = 1 + hashmap_bitcount(MAP_HEADER_VAL(hdr)); while (i--) { *htop++ = *objp++; } - *argp = make_hashmap_rel(tp, dst_base); + *argp = make_hashmap(tp); break; default: erl_exit(ERTS_ABORT_EXIT, "copy_struct: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr)); @@ -522,7 +879,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) i = thing_arityval(hdr)+1; hbot -= i; tp = hbot; - *argp = make_boxed_rel(hbot, dst_base); + *argp = make_boxed(hbot); while (i--) { *tp++ = *objp++; } @@ -538,22 +895,870 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) } } + if (bsz) { + *hpp = htop; + *bsz = hend - hbot; + } else { #ifdef DEBUG - if (htop != hbot) - erl_exit(ERTS_ABORT_EXIT, - "Internal error in copy_struct() when copying %T:" - " htop=%p != hbot=%p (sz=%beu)\n", - org_obj, htop, hbot, org_sz); + if (htop != hbot) + erl_exit(ERTS_ABORT_EXIT, + "Internal error in copy_struct() when copying %T:" + " htop=%p != hbot=%p (sz=%beu)\n", + org_obj, htop, hbot, org_sz); #else - if (htop > hbot) { - erl_exit(ERTS_ABORT_EXIT, - "Internal error in copy_struct(): htop, hbot overrun\n"); - } + if (htop > hbot) { + erl_exit(ERTS_ABORT_EXIT, + "Internal error in copy_struct(): htop, hbot overrun\n"); + } #endif - *hpp = (Eterm *) (hstart+hsize); + *hpp = (Eterm *) (hstart+hsize); + } + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] result is at %p\n", mypid, res)); return res; } + +/* + * Machinery for the table used by the sharing preserving copier + * Using an ESTACK but not very transparently; consider refactoring + */ + +#define DECLARE_SHTABLE(s) \ + DECLARE_ESTACK(s); \ + Uint ESTK_CONCAT(s,_offset) = 0 +#define DESTROY_SHTABLE(s) DESTROY_ESTACK(s) +#define SHTABLE_INCR 4 +#define SHTABLE_NEXT(s) ESTK_CONCAT(s,_offset) +#define SHTABLE_PUSH(s,x,y,b) \ +do { \ + if (s.sp > s.end - SHTABLE_INCR) { \ + erl_grow_estack(&(s), SHTABLE_INCR); \ + } \ + *s.sp++ = (x); \ + *s.sp++ = (y); \ + *s.sp++ = (Eterm) NULL; \ + *s.sp++ = (Eterm) (b); \ + ESTK_CONCAT(s,_offset) += SHTABLE_INCR; \ +} while(0) +#define SHTABLE_X(s,e) (s.start[e]) +#define SHTABLE_Y(s,e) (s.start[(e)+1]) +#define SHTABLE_FWD(s,e) ((Eterm *) (s.start[(e)+2])) +#define SHTABLE_FWD_UPD(s,e,p) (s.start[(e)+2] = (Eterm) (p)) +#define SHTABLE_REV(s,e) ((Eterm *) (s.start[(e)+3])) + +#define LIST_SHARED_UNPROCESSED ((Eterm) 0) +#define LIST_SHARED_PROCESSED ((Eterm) 1) + +#define HEAP_ELEM_TO_BE_FILLED _unchecked_make_list(NULL) + + +/* + * Specialized macros for using/reusing the persistent state + */ + +#define DECLARE_EQUEUE_INIT_INFO(q, info) \ + UWord* EQUE_DEF_QUEUE(q) = info->queue_default; \ + ErtsEQueue q = { \ + EQUE_DEF_QUEUE(q), /* start */ \ + EQUE_DEF_QUEUE(q), /* front */ \ + EQUE_DEF_QUEUE(q), /* back */ \ + 1, /* possibly_empty */ \ + EQUE_DEF_QUEUE(q) + DEF_EQUEUE_SIZE, /* end */ \ + ERTS_ALC_T_ESTACK /* alloc_type */ \ + } + +#define DECLARE_EQUEUE_FROM_INFO(q, info) \ + /* no EQUE_DEF_QUEUE(q), read-only */ \ + ErtsEQueue q = { \ + info->queue_start, /* start */ \ + info->queue_start, /* front */ \ + info->queue_start, /* back */ \ + 1, /* possibly_empty */ \ + info->queue_end, /* end */ \ + info->queue_alloc_type /* alloc_type */ \ + } + +#define DECLARE_BITSTORE_INIT_INFO(s, info) \ + UWord* WSTK_DEF_STACK(s) = info->bitstore_default; \ + ErtsWStack s = { \ + WSTK_DEF_STACK(s), /* wstart */ \ + WSTK_DEF_STACK(s), /* wsp */ \ + WSTK_DEF_STACK(s) + DEF_WSTACK_SIZE, /* wend */ \ + WSTK_DEF_STACK(s), /* wdflt */ \ + ERTS_ALC_T_ESTACK /* alloc_type */ \ + }; \ + int WSTK_CONCAT(s,_bitoffs) = 0; \ + /* no WSTK_CONCAT(s,_offset), write-only */ \ + UWord WSTK_CONCAT(s,_buffer) = 0 + +#define DECLARE_BITSTORE_FROM_INFO(s, info) \ + /* no WSTK_DEF_STACK(s), read-only */ \ + ErtsWStack s = { \ + info->bitstore_start, /* wstart */ \ + NULL, /* wsp, read-only */ \ + NULL, /* wend, read-only */ \ + NULL, /* wdef, read-only */ \ + info->bitstore_alloc_type /* alloc_type */ \ + }; \ + int WSTK_CONCAT(s,_bitoffs) = 0; \ + int WSTK_CONCAT(s,_offset) = 0; \ + UWord WSTK_CONCAT(s,_buffer) = 0 + +#define DECLARE_SHTABLE_INIT_INFO(s, info) \ + Eterm* ESTK_DEF_STACK(s) = info->shtable_default; \ + ErtsEStack s = { \ + ESTK_DEF_STACK(s), /* start */ \ + ESTK_DEF_STACK(s), /* sp */ \ + ESTK_DEF_STACK(s) + DEF_ESTACK_SIZE, /* end */ \ + ESTK_DEF_STACK(s), /* default */ \ + ERTS_ALC_T_ESTACK /* alloc_type */ \ + }; \ + Uint ESTK_CONCAT(s,_offset) = 0 + +#define DECLARE_SHTABLE_FROM_INFO(s, info) \ + /* no ESTK_DEF_STACK(s), read-only */ \ + ErtsEStack s = { \ + info->shtable_start, /* start */ \ + NULL, /* sp, read-only */ \ + NULL, /* end, read-only */ \ + NULL, /* def, read-only */ \ + info->shtable_alloc_type /* alloc_type */ \ + }; \ + /* no ESTK_CONCAT(s,_offset), read-only */ + +/* + * Copy object "obj" preserving sharing. + * First half: count size and calculate sharing. + */ +Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info) +{ + Uint sum; + Uint e; + unsigned sz; + Eterm* ptr; +#ifdef DEBUG + Eterm mypid = erts_get_current_pid(); +#endif + + DECLARE_EQUEUE_INIT_INFO(s, info); + DECLARE_BITSTORE_INIT_INFO(b, info); + DECLARE_SHTABLE_INIT_INFO(t, info); + + /* step #0: + ------------------------------------------------------- + get rid of the easy cases first: + - copying constants + - if not a proper process, do flat copy + */ + + if (IS_CONST(obj)) + return 0; + + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] copy_shared_calculate %p\n", mypid, obj)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] message is %T\n", mypid, obj)); + + /* step #1: + ------------------------------------------------------- + traverse the term and calculate the size; + when traversing, transform as you do in size_shared + but when you find shared objects: + + a. add entry in the table, indexed by i + b. mark them: + b1. boxed terms, set header to (i | 11) + store (old header, NONV, NULL, backptr) in the entry + b2. cons cells, set CDR to NONV, set CAR to i + store (old CAR, old CDR, NULL, backptr) in the entry + */ + + sum = 0; + + for (;;) { + switch (primary_tag(obj)) { + case TAG_PRIMARY_LIST: { + Eterm head, tail; + ptr = list_val(obj); + /* off heap list pointers are copied verbatim */ + if (erts_is_literal(obj,ptr)) { + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] bypassed copying %p is %T\n", mypid, ptr, obj)); + if (IN_LITERAL_PURGE_AREA(info,ptr)) + info->literal_size += size_object(obj); + goto pop_next; + } + head = CAR(ptr); + tail = CDR(ptr); + /* if it's visited, don't count it; + if not already shared, make it shared and store it in the table */ + if (primary_tag(tail) == TAG_PRIMARY_HEADER || + primary_tag(head) == TAG_PRIMARY_HEADER) { + if (tail != THE_NON_VALUE) { + e = SHTABLE_NEXT(t); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] tabling L %p\n", mypid, ptr)); + SHTABLE_PUSH(t, head, tail, ptr); + CAR(ptr) = (e << _TAG_PRIMARY_SIZE) | LIST_SHARED_UNPROCESSED; + CDR(ptr) = THE_NON_VALUE; + } + goto pop_next; + } + /* else make it visited now */ + switch (primary_tag(tail)) { + case TAG_PRIMARY_LIST: + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] mangling L/L %p\n", mypid, ptr)); + CDR(ptr) = (tail - TAG_PRIMARY_LIST) | TAG_PRIMARY_HEADER; + break; + case TAG_PRIMARY_IMMED1: + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] mangling L/I %p\n", mypid, ptr)); + CAR(ptr) = (head - primary_tag(head)) | TAG_PRIMARY_HEADER; + CDR(ptr) = (tail - TAG_PRIMARY_IMMED1) | primary_tag(head); + break; + case TAG_PRIMARY_BOXED: + BITSTORE_PUT(b, primary_tag(head)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] mangling L/B %p\n", mypid, ptr)); + CAR(ptr) = (head - primary_tag(head)) | TAG_PRIMARY_HEADER; + CDR(ptr) = (tail - TAG_PRIMARY_BOXED) | TAG_PRIMARY_HEADER; + break; + } + /* and count it */ + sum += 2; + if (!IS_CONST(head)) { + EQUEUE_PUT(s, head); + } + obj = tail; + break; + } + case TAG_PRIMARY_BOXED: { + Eterm hdr; + ptr = boxed_val(obj); + /* off heap pointers to boxes are copied verbatim */ + if (erts_is_literal(obj,ptr)) { + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] bypassed copying %p is %T\n", mypid, ptr, obj)); + if (IN_LITERAL_PURGE_AREA(info,ptr)) + info->literal_size += size_object(obj); + goto pop_next; + } + hdr = *ptr; + /* if it's visited, don't count it; + if not already shared, make it shared and store it in the table */ + if (primary_tag(hdr) != TAG_PRIMARY_HEADER) { + if (primary_tag(hdr) == BOXED_VISITED) { + e = SHTABLE_NEXT(t); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] tabling B %p\n", mypid, ptr)); + SHTABLE_PUSH(t, hdr, THE_NON_VALUE, ptr); + *ptr = (e << _TAG_PRIMARY_SIZE) | BOXED_SHARED_UNPROCESSED; + } + goto pop_next; + } + /* else make it visited now */ + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] mangling B %p\n", mypid, ptr)); + *ptr = (hdr - primary_tag(hdr)) + BOXED_VISITED; + /* and count it */ + ASSERT(is_header(hdr)); + switch (hdr & _TAG_HEADER_MASK) { + case ARITYVAL_SUBTAG: { + int arity = header_arity(hdr); + sum += arity + 1; + if (arity == 0) { /* Empty tuple -- unusual. */ + goto pop_next; + } + while (arity-- > 0) { + obj = *++ptr; + if (!IS_CONST(obj)) { + EQUEUE_PUT(s, obj); + } + } + goto pop_next; + } + case FUN_SUBTAG: { + ErlFunThing* funp = (ErlFunThing *) ptr; + unsigned eterms = 1 /* creator */ + funp->num_free; + sz = thing_arityval(hdr); + sum += 1 /* header */ + sz + eterms; + ptr += 1 /* header */ + sz; + while (eterms-- > 0) { + obj = *ptr++; + if (!IS_CONST(obj)) { + EQUEUE_PUT(s, obj); + } + } + goto pop_next; + } + case SUB_BINARY_SUBTAG: { + ErlSubBin* sb = (ErlSubBin *) ptr; + Eterm real_bin = sb->orig; + Uint bit_offset = sb->bitoffs; + Uint bit_size = sb->bitsize; + size_t size = sb->size; + Uint extra_bytes; + Eterm hdr; + if (bit_size + bit_offset > 8) { + sum += ERL_SUB_BIN_SIZE; + extra_bytes = 2; + } else if (bit_size + bit_offset > 0) { + sum += ERL_SUB_BIN_SIZE; + extra_bytes = 1; + } else { + extra_bytes = 0; + } + ASSERT(is_boxed(real_bin) && + (((*boxed_val(real_bin)) & + (_TAG_HEADER_MASK - _BINARY_XXX_MASK - BOXED_VISITED_MASK)) + == _TAG_HEADER_REFC_BIN)); + hdr = *_unchecked_binary_val(real_bin) & ~BOXED_VISITED_MASK; + if (thing_subtag(hdr) == HEAP_BINARY_SUBTAG) { + sum += heap_bin_size(size+extra_bytes); + } else { + ASSERT(thing_subtag(hdr) == REFC_BINARY_SUBTAG); + sum += PROC_BIN_SIZE; + } + goto pop_next; + } + case MAP_SUBTAG: + switch (MAP_HEADER_TYPE(hdr)) { + case MAP_HEADER_TAG_FLATMAP_HEAD : { + flatmap_t *mp = (flatmap_t *) ptr; + Uint n = flatmap_get_size(mp) + 1; + sum += n + 2; + ptr += 2; /* hdr + size words */ + while (n--) { + obj = *ptr++; + if (!IS_CONST(obj)) { + EQUEUE_PUT(s, obj); + } + } + goto pop_next; + } + case MAP_HEADER_TAG_HAMT_HEAD_BITMAP : + case MAP_HEADER_TAG_HAMT_HEAD_ARRAY : + case MAP_HEADER_TAG_HAMT_NODE_BITMAP : { + Uint n = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + sum += 1 + n + header_arity(hdr); + ptr += 1 + header_arity(hdr); + + if (n == 0) { + goto pop_next; + } + while(n--) { + obj = *ptr++; + if (!IS_CONST(obj)) { + EQUEUE_PUT(s, obj); + } + } + goto pop_next; + } + default: + erl_exit(ERTS_ABORT_EXIT, "copy_shared_calculate: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr)); + } + case BIN_MATCHSTATE_SUBTAG: + erl_exit(ERTS_ABORT_EXIT, + "size_shared: matchstate term not allowed"); + default: + sum += thing_arityval(hdr) + 1; + goto pop_next; + } + break; + } + case TAG_PRIMARY_IMMED1: + pop_next: + if (EQUEUE_ISEMPTY(s)) { + /* add sentinel to the table */ + SHTABLE_PUSH(t, THE_NON_VALUE, THE_NON_VALUE, NULL); + /* store persistent info */ + BITSTORE_CLOSE(b); + info->queue_start = s.start; + info->queue_end = s.end; + info->queue_alloc_type = s.alloc_type; + info->bitstore_start = b.wstart; + info->bitstore_alloc_type = b.alloc_type; + info->shtable_start = t.start; + info->shtable_alloc_type = t.alloc_type; + /* single point of return: the size of the object */ + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] size was: %u\n", mypid, sum)); + return sum + info->literal_size; + } + obj = EQUEUE_GET(s); + break; + default: + erl_exit(ERTS_ABORT_EXIT, "[pid=%T] size_shared: bad tag for %#x\n", obj); + } + } +} + +/* + * Copy object "obj" preserving sharing. + * Second half: copy and restore the object. + */ +Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, + Eterm** hpp, ErlOffHeap* off_heap) { + Uint e; + unsigned sz; + Eterm* ptr; + Eterm* hp; + Eterm* hscan; + Eterm result; + Eterm* resp; + Eterm *hbot, *hend; + unsigned remaining; +#ifdef DEBUG + Eterm mypid = erts_get_current_pid(); + Eterm saved_obj = obj; +#endif + + DECLARE_EQUEUE_FROM_INFO(s, info); + DECLARE_BITSTORE_FROM_INFO(b, info); + DECLARE_SHTABLE_FROM_INFO(t, info); + + /* step #0: + ------------------------------------------------------- + get rid of the easy cases first: + - copying constants + - if not a proper process, do flat copy + */ + + if (IS_CONST(obj)) + return obj; + + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] copy_shared_perform %p\n", mypid, obj)); + + /* step #2: was performed before this function was called + ------------------------------------------------------- + allocate new space + */ + + hscan = hp = *hpp; + hbot = hend = hp + size; + + /* step #3: + ------------------------------------------------------- + traverse the term a second time and when traversing: + a. if the object is marked as shared + a1. if the entry contains a forwarding ptr, use that + a2. otherwise, copy it to the new space and store the + forwarding ptr to the entry + b. otherwise, reverse-transform as you do in size_shared + and copy to the new space + */ + + resp = &result; + remaining = 0; + for (;;) { + switch (primary_tag(obj)) { + case TAG_PRIMARY_LIST: { + Eterm head, tail; + ptr = list_val(obj); + /* off heap list pointers are copied verbatim */ + if (erts_is_literal(obj,ptr)) { + if (!IN_LITERAL_PURGE_AREA(info,ptr)) { + *resp = obj; + } else { + Uint bsz = 0; + *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz); + hbot -= bsz; + } + goto cleanup_next; + } + head = CAR(ptr); + tail = CDR(ptr); + /* if it is shared */ + if (tail == THE_NON_VALUE) { + e = head >> _TAG_PRIMARY_SIZE; + /* if it has been processed, just use the forwarding pointer */ + if (primary_tag(head) == LIST_SHARED_PROCESSED) { + *resp = make_list(SHTABLE_FWD(t, e)); + goto cleanup_next; + } + /* else, let's process it now, + copy it and keep the forwarding pointer */ + else { + CAR(ptr) = (head - primary_tag(head)) + LIST_SHARED_PROCESSED; + head = SHTABLE_X(t, e); + tail = SHTABLE_Y(t, e); + ptr = &(SHTABLE_X(t, e)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] tabled L %p is %p\n", mypid, ptr, SHTABLE_REV(t, e))); + SHTABLE_FWD_UPD(t, e, hp); + } + } + /* if not already clean, clean it up and copy it */ + if (primary_tag(tail) == TAG_PRIMARY_HEADER) { + if (primary_tag(head) == TAG_PRIMARY_HEADER) { + Eterm saved; + BITSTORE_FETCH(b, saved); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling L/B %p\n", mypid, ptr)); + CAR(ptr) = head = (head - TAG_PRIMARY_HEADER) + saved; + CDR(ptr) = tail = (tail - TAG_PRIMARY_HEADER) + TAG_PRIMARY_BOXED; + } else { + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling L/L %p\n", mypid, ptr)); + CDR(ptr) = tail = (tail - TAG_PRIMARY_HEADER) + TAG_PRIMARY_LIST; + } + } else if (primary_tag(head) == TAG_PRIMARY_HEADER) { + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling L/I %p\n", mypid, ptr)); + CAR(ptr) = head = (head - TAG_PRIMARY_HEADER) | primary_tag(tail); + CDR(ptr) = tail = (tail - primary_tag(tail)) | TAG_PRIMARY_IMMED1; + } else { + ASSERT(0 && "cannot come here"); + goto cleanup_next; + } + /* and its children too */ + if (IS_CONST(head)) { + CAR(hp) = head; + } else { + EQUEUE_PUT_UNCHECKED(s, head); + CAR(hp) = HEAP_ELEM_TO_BE_FILLED; + } + *resp = make_list(hp); + resp = &(CDR(hp)); + hp += 2; + obj = tail; + break; + } + case TAG_PRIMARY_BOXED: { + Eterm hdr; + ptr = boxed_val(obj); + /* off heap pointers to boxes are copied verbatim */ + if (erts_is_literal(obj,ptr)) { + if (!IN_LITERAL_PURGE_AREA(info,ptr)) { + *resp = obj; + } else { + Uint bsz = 0; + *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz); + hbot -= bsz; + } + goto cleanup_next; + } + hdr = *ptr; + /* clean it up, unless it's already clean or shared and processed */ + switch (primary_tag(hdr)) { + case TAG_PRIMARY_HEADER: + ASSERT(0 && "cannot come here"); + /* if it is shared and has been processed, + just use the forwarding pointer */ + case BOXED_SHARED_PROCESSED: + e = hdr >> _TAG_PRIMARY_SIZE; + *resp = make_boxed(SHTABLE_FWD(t, e)); + goto cleanup_next; + /* if it is shared but has not been processed yet, let's process + it now: copy it and keep the forwarding pointer */ + case BOXED_SHARED_UNPROCESSED: + e = hdr >> _TAG_PRIMARY_SIZE; + *ptr = (hdr - primary_tag(hdr)) + BOXED_SHARED_PROCESSED; + hdr = SHTABLE_X(t, e); + ASSERT(primary_tag(hdr) == BOXED_VISITED); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] tabled B %p is %p\n", mypid, ptr, SHTABLE_REV(t, e))); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling B %p\n", mypid, ptr)); + SHTABLE_X(t, e) = hdr = (hdr - BOXED_VISITED) + TAG_PRIMARY_HEADER; + SHTABLE_FWD_UPD(t, e, hp); + break; + case BOXED_VISITED: + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling B %p\n", mypid, ptr)); + *ptr = hdr = (hdr - BOXED_VISITED) + TAG_PRIMARY_HEADER; + break; + } + /* and its children too */ + switch (hdr & _TAG_HEADER_MASK) { + case ARITYVAL_SUBTAG: { + int arity = header_arity(hdr); + *resp = make_boxed(hp); + *hp++ = hdr; + while (arity-- > 0) { + obj = *++ptr; + if (IS_CONST(obj)) { + *hp++ = obj; + } else { + EQUEUE_PUT_UNCHECKED(s, obj); + *hp++ = HEAP_ELEM_TO_BE_FILLED; + } + } + goto cleanup_next; + } + case FUN_SUBTAG: { + ErlFunThing* funp = (ErlFunThing *) ptr; + unsigned eterms = 1 /* creator */ + funp->num_free; + sz = thing_arityval(hdr); + funp = (ErlFunThing *) hp; + *resp = make_fun(hp); + *hp++ = hdr; + ptr++; + while (sz-- > 0) { + *hp++ = *ptr++; + } + while (eterms-- > 0) { + obj = *ptr++; + if (IS_CONST(obj)) { + *hp++ = obj; + } else { + EQUEUE_PUT_UNCHECKED(s, obj); + *hp++ = HEAP_ELEM_TO_BE_FILLED; + } + } + funp->next = off_heap->first; + off_heap->first = (struct erl_off_heap_header*) funp; + erts_refc_inc(&funp->fe->refc, 2); + goto cleanup_next; + } + case MAP_SUBTAG: + *resp = make_flatmap(hp); + *hp++ = hdr; + switch (MAP_HEADER_TYPE(hdr)) { + case MAP_HEADER_TAG_FLATMAP_HEAD : { + flatmap_t *mp = (flatmap_t *) ptr; + Uint n = flatmap_get_size(mp) + 1; + *hp++ = *++ptr; /* keys */ + while (n--) { + obj = *++ptr; + if (IS_CONST(obj)) { + *hp++ = obj; + } else { + EQUEUE_PUT_UNCHECKED(s, obj); + *hp++ = HEAP_ELEM_TO_BE_FILLED; + } + } + goto cleanup_next; + } + case MAP_HEADER_TAG_HAMT_HEAD_BITMAP : + case MAP_HEADER_TAG_HAMT_HEAD_ARRAY : + *hp++ = *++ptr; /* total map size */ + case MAP_HEADER_TAG_HAMT_NODE_BITMAP : { + Uint n = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + while (n--) { + obj = *++ptr; + if (IS_CONST(obj)) { + *hp++ = obj; + } else { + EQUEUE_PUT_UNCHECKED(s, obj); + *hp++ = HEAP_ELEM_TO_BE_FILLED; + } + } + goto cleanup_next; + } + default: + erl_exit(ERTS_ABORT_EXIT, "copy_shared_perform: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr)); + } + case REFC_BINARY_SUBTAG: { + ProcBin* pb = (ProcBin *) ptr; + sz = thing_arityval(hdr); + if (pb->flags) { + erts_emasculate_writable_binary(pb); + } + pb = (ProcBin *) hp; + *resp = make_binary(hp); + *hp++ = hdr; + ptr++; + while (sz-- > 0) { + *hp++ = *ptr++; + } + erts_refc_inc(&pb->val->refc, 2); + pb->next = off_heap->first; + pb->flags = 0; + off_heap->first = (struct erl_off_heap_header*) pb; + OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm)); + goto cleanup_next; + } + case SUB_BINARY_SUBTAG: { + ErlSubBin* sb = (ErlSubBin *) ptr; + Eterm real_bin = sb->orig; + Uint bit_offset = sb->bitoffs; + Uint bit_size = sb->bitsize; + Uint offset = sb->offs; + size_t size = sb->size; + Uint extra_bytes; + Uint real_size; + if ((bit_size + bit_offset) > 8) { + extra_bytes = 2; + } else if ((bit_size + bit_offset) > 0) { + extra_bytes = 1; + } else { + extra_bytes = 0; + } + real_size = size+extra_bytes; + ASSERT(is_boxed(real_bin) && + (((*boxed_val(real_bin)) & + (_TAG_HEADER_MASK - _BINARY_XXX_MASK - BOXED_VISITED_MASK)) + == _TAG_HEADER_REFC_BIN)); + ptr = _unchecked_binary_val(real_bin); + *resp = make_binary(hp); + if (extra_bytes != 0) { + ErlSubBin* res = (ErlSubBin *) hp; + hp += ERL_SUB_BIN_SIZE; + res->thing_word = HEADER_SUB_BIN; + res->size = size; + res->bitsize = bit_size; + res->bitoffs = bit_offset; + res->offs = 0; + res->is_writable = 0; + res->orig = make_binary(hp); + } + if (thing_subtag(*ptr & ~BOXED_VISITED_MASK) == HEAP_BINARY_SUBTAG) { + ErlHeapBin* from = (ErlHeapBin *) ptr; + ErlHeapBin* to = (ErlHeapBin *) hp; + hp += heap_bin_size(real_size); + to->thing_word = header_heap_bin(real_size); + to->size = real_size; + sys_memcpy(to->data, ((byte *)from->data)+offset, real_size); + } else { + ProcBin* from = (ProcBin *) ptr; + ProcBin* to = (ProcBin *) hp; + ASSERT(thing_subtag(*ptr & ~BOXED_VISITED_MASK) == REFC_BINARY_SUBTAG); + if (from->flags) { + erts_emasculate_writable_binary(from); + } + hp += PROC_BIN_SIZE; + to->thing_word = HEADER_PROC_BIN; + to->size = real_size; + to->val = from->val; + erts_refc_inc(&to->val->refc, 2); + to->bytes = from->bytes + offset; + to->next = off_heap->first; + to->flags = 0; + off_heap->first = (struct erl_off_heap_header*) to; + OH_OVERHEAD(off_heap, to->size / sizeof(Eterm)); + } + goto cleanup_next; + } + case EXTERNAL_PID_SUBTAG: + case EXTERNAL_PORT_SUBTAG: + case EXTERNAL_REF_SUBTAG: { + ExternalThing *etp = (ExternalThing *) hp; + sz = thing_arityval(hdr); + *resp = make_external(hp); + *hp++ = hdr; + ptr++; + while (sz-- > 0) { + *hp++ = *ptr++; + } + etp->next = off_heap->first; + off_heap->first = (struct erl_off_heap_header*) etp; + erts_refc_inc(&etp->node->refc, 2); + goto cleanup_next; + } + default: + sz = thing_arityval(hdr); + *resp = make_boxed(hp); + *hp++ = hdr; + ptr++; + while (sz-- > 0) { + *hp++ = *ptr++; + } + goto cleanup_next; + } + break; + } + case TAG_PRIMARY_IMMED1: + *resp = obj; + cleanup_next: + if (EQUEUE_ISEMPTY(s)) { + goto all_clean; + } + obj = EQUEUE_GET(s); + for (;;) { + ASSERT(hscan < hp); + if (remaining == 0) { + if (*hscan == HEAP_ELEM_TO_BE_FILLED) { + resp = hscan; + hscan += 2; + break; /* scanning loop */ + } else if (primary_tag(*hscan) == TAG_PRIMARY_HEADER) { + switch (*hscan & _TAG_HEADER_MASK) { + case ARITYVAL_SUBTAG: + remaining = header_arity(*hscan); + hscan++; + break; + case FUN_SUBTAG: { + ErlFunThing* funp = (ErlFunThing *) hscan; + hscan += 1 + thing_arityval(*hscan); + remaining = 1 + funp->num_free; + break; + } + case MAP_SUBTAG: + switch (MAP_HEADER_TYPE(*hscan)) { + case MAP_HEADER_TAG_FLATMAP_HEAD : { + flatmap_t *mp = (flatmap_t *) hscan; + remaining = flatmap_get_size(mp) + 1; + hscan += 2; + break; + } + case MAP_HEADER_TAG_HAMT_HEAD_BITMAP : + case MAP_HEADER_TAG_HAMT_HEAD_ARRAY : + case MAP_HEADER_TAG_HAMT_NODE_BITMAP : + remaining = hashmap_bitcount(MAP_HEADER_VAL(*hscan)); + hscan += MAP_HEADER_ARITY(*hscan) + 1; + break; + default: + erl_exit(ERTS_ABORT_EXIT, + "copy_shared_perform: bad hashmap type %d\n", + MAP_HEADER_TYPE(*hscan)); + } + break; + case SUB_BINARY_SUBTAG: + ASSERT(((ErlSubBin *) hscan)->bitoffs + + ((ErlSubBin *) hscan)->bitsize > 0); + hscan += ERL_SUB_BIN_SIZE; + break; + default: + hscan += 1 + thing_arityval(*hscan); + break; + } + } else { + hscan++; + } + } else if (*hscan == HEAP_ELEM_TO_BE_FILLED) { + resp = hscan++; + remaining--; + break; /* scanning loop */ + } else { + hscan++; + remaining--; + } + } + ASSERT(resp < hp); + break; + default: + erl_exit(ERTS_ABORT_EXIT, "size_shared: bad tag for %#x\n", obj); + } + } + + /* step #4: + ------------------------------------------------------- + traverse the table and reverse-transform all stored entries + */ + +all_clean: + for (e = 0; ; e += SHTABLE_INCR) { + ptr = SHTABLE_REV(t, e); + if (ptr == NULL) + break; + VERBOSE(DEBUG_SHCOPY, ("[copy] restoring shared: %x\n", ptr)); + /* entry was a list */ + if (SHTABLE_Y(t, e) != THE_NON_VALUE) { + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] untabling L %p\n", mypid, ptr)); + CAR(ptr) = SHTABLE_X(t, e); + CDR(ptr) = SHTABLE_Y(t, e); + } + /* entry was boxed */ + else { + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] untabling B %p\n", mypid, ptr)); + *ptr = SHTABLE_X(t, e); + ASSERT(primary_tag(*ptr) == TAG_PRIMARY_HEADER); + } + } + +#ifdef DEBUG + if (eq(saved_obj, result) == 0) { + erts_fprintf(stderr, "original = %T\n", saved_obj); + erts_fprintf(stderr, "copy = %T\n", result); + erl_exit(ERTS_ABORT_EXIT, "copy (shared) not equal to source\n"); + } +#endif + + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] original was %T\n", mypid, saved_obj)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] copy is %T\n", mypid, result)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] result is at %p\n", mypid, result)); + + ASSERT(hbot == hp); + ASSERT(size == ((hp - *hpp) + (hend - hbot))); + *hpp = hend; + return result; +} + + /* * Copy a term that is guaranteed to be contained in a single * heap block. The heap block is copied word by word, and any @@ -563,21 +1768,12 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) * * NOTE: Assumes that term is a tuple (ptr is an untagged tuple ptr). */ -#if HALFWORD_HEAP -Eterm copy_shallow_rel(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, - Eterm* src_base) -#else Eterm copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) -#endif { Eterm* tp = ptr; Eterm* hp = *hpp; const Eterm res = make_tuple(hp); -#if HALFWORD_HEAP - const Sint offs = COMPRESS_POINTER(hp - (tp - src_base)); -#else const Sint offs = (hp - tp) * sizeof(Eterm); -#endif while (sz--) { Eterm val = *tp++; @@ -652,17 +1848,24 @@ Eterm copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) * move markers. * Typically used to copy a multi-fragmented message (from NIF). */ -void move_multi_frags(Eterm** hpp, ErlOffHeap* off_heap, ErlHeapFragment* first, - Eterm* refs, unsigned nrefs) +void erts_move_multi_frags(Eterm** hpp, ErlOffHeap* off_heap, ErlHeapFragment* first, + Eterm* refs, unsigned nrefs, int literals) { ErlHeapFragment* bp; Eterm* hp_start = *hpp; Eterm* hp_end; Eterm* hp; unsigned i; + Eterm literal_tag; + +#ifdef TAG_LITERAL_PTR + literal_tag = (Eterm) literals ? TAG_LITERAL_PTR : 0; +#else + literal_tag = (Eterm) 0; +#endif for (bp=first; bp!=NULL; bp=bp->next) { - move_one_frag(hpp, bp, off_heap); + move_one_frag(hpp, bp, off_heap, literals); } hp_end = *hpp; for (hp=hp_start; hp<hp_end; ++hp) { @@ -675,6 +1878,9 @@ void move_multi_frags(Eterm** hpp, ErlOffHeap* off_heap, ErlHeapFragment* first, val = *ptr; if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); +#ifdef TAG_LITERAL_PTR + val |= literal_tag; +#endif *hp = val; } break; @@ -682,7 +1888,11 @@ void move_multi_frags(Eterm** hpp, ErlOffHeap* off_heap, ErlHeapFragment* first, ptr = list_val(gval); val = *ptr; if (IS_MOVED_CONS(val)) { - *hp = ptr[1]; + val = ptr[1]; +#ifdef TAG_LITERAL_PTR + val |= literal_tag; +#endif + *hp = val; } break; case TAG_PRIMARY_HEADER: @@ -693,12 +1903,12 @@ void move_multi_frags(Eterm** hpp, ErlOffHeap* off_heap, ErlHeapFragment* first, } } for (i=0; i<nrefs; ++i) { - refs[i] = follow_moved(refs[i]); + refs[i] = follow_moved(refs[i], literal_tag); } } static void -move_one_frag(Eterm** hpp, ErlHeapFragment* frag, ErlOffHeap* off_heap) +move_one_frag(Eterm** hpp, ErlHeapFragment* frag, ErlOffHeap* off_heap, int literals) { Eterm* ptr = frag->mem; Eterm* end = ptr + frag->used_size; @@ -735,4 +1945,3 @@ move_one_frag(Eterm** hpp, ErlHeapFragment* frag, ErlOffHeap* off_heap) OH_OVERHEAD(off_heap, frag->off_heap.overhead); frag->off_heap.first = NULL; } - diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index 7be2b77a3b..e31ef29562 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -378,10 +378,11 @@ static void doit_node_link_net_exits(ErtsLink *lnk, void *vnecp) ASSERT(lnk->type == LINK_NODE); if (is_internal_pid(lnk->pid)) { ErtsProcLocks rp_locks = ERTS_PROC_LOCK_LINK; - rp = erts_pid2proc(NULL, 0, lnk->pid, rp_locks); - if (!rp) { + ErlOffHeap *ohp; + rp = erts_proc_lookup(lnk->pid); + if (!rp) goto done; - } + erts_smp_proc_lock(rp, rp_locks); rlnk = erts_remove_link(&ERTS_P_LINKS(rp), name); if (rlnk != NULL) { ASSERT(is_atom(rlnk->pid) && (rlnk->type == LINK_NODE)); @@ -389,12 +390,14 @@ static void doit_node_link_net_exits(ErtsLink *lnk, void *vnecp) } n = ERTS_LINK_REFC(lnk); for (i = 0; i < n; ++i) { - ErlHeapFragment* bp; - ErlOffHeap *ohp; Eterm tup; - Eterm *hp = erts_alloc_message_heap(3,&bp,&ohp,rp,&rp_locks); + Eterm *hp; + ErtsMessage *msgp; + + msgp = erts_alloc_message_heap(rp, &rp_locks, + 3, &hp, &ohp); tup = TUPLE2(hp, am_nodedown, name); - erts_queue_message(rp, &rp_locks, bp, tup, NIL); + erts_queue_message(rp, &rp_locks, msgp, tup, NIL); } erts_smp_proc_unlock(rp, rp_locks); } @@ -737,19 +740,11 @@ Eterm erts_dsend_export_trap_context(Process* p, ErtsSendContext* ctx) Binary* ctx_bin = erts_create_magic_binary(sizeof(struct exported_ctx), erts_dsend_context_dtor); struct exported_ctx* dst = ERTS_MAGIC_BIN_DATA(ctx_bin); - Uint ctl_size = !HALFWORD_HEAP ? 0 : (arityval(ctx->ctl_heap[0]) + 1); - Eterm* hp = HAlloc(p, ctl_size + PROC_BIN_SIZE); + Eterm* hp = HAlloc(p, PROC_BIN_SIZE); sys_memcpy(&dst->ctx, ctx, sizeof(ErtsSendContext)); ASSERT(ctx->dss.ctl == make_tuple(ctx->ctl_heap)); -#if !HALFWORD_HEAP dst->ctx.dss.ctl = make_tuple(dst->ctx.ctl_heap); -#else - /* Must put control tuple in low mem */ - sys_memcpy(hp, ctx->ctl_heap, ctl_size*sizeof(Eterm)); - dst->ctx.dss.ctl = make_tuple(hp); - hp += ctl_size; -#endif if (ctx->dss.acmp) { sys_memcpy(&dst->acm, ctx->dss.acmp, sizeof(ErtsAtomCacheMap)); dst->ctx.dss.acmp = &dst->acm; @@ -886,11 +881,7 @@ erts_dsig_send_msg(Eterm remote, Eterm message, ErtsSendContext* ctx) DTRACE_CHARBUF(receiver_name, 64); #endif - if (SEQ_TRACE_TOKEN(sender) != NIL -#ifdef USE_VM_PROBES - && SEQ_TRACE_TOKEN(sender) != am_have_dt_utag -#endif - ) { + if (have_seqtrace(SEQ_TRACE_TOKEN(sender))) { seq_trace_update_send(sender); token = SEQ_TRACE_TOKEN(sender); seq_trace_output(token, message, SEQ_TRACE_SEND, remote, sender); @@ -905,7 +896,7 @@ erts_dsig_send_msg(Eterm remote, Eterm message, ErtsSendContext* ctx) erts_snprintf(receiver_name, sizeof(DTRACE_CHARBUF_NAME(receiver_name)), "%T", remote); msize = size_object(message); - if (token != NIL && token != am_have_dt_utag) { + if (have_seqtrace(token)) { tok_label = signed_val(SEQ_TRACE_T_LABEL(token)); tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token)); tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token)); @@ -947,11 +938,7 @@ erts_dsig_send_reg_msg(Eterm remote_name, Eterm message, DTRACE_CHARBUF(receiver_name, 128); #endif - if (SEQ_TRACE_TOKEN(sender) != NIL -#ifdef USE_VM_PROBES - && SEQ_TRACE_TOKEN(sender) != am_have_dt_utag -#endif - ) { + if (have_seqtrace(SEQ_TRACE_TOKEN(sender))) { seq_trace_update_send(sender); token = SEQ_TRACE_TOKEN(sender); seq_trace_output(token, message, SEQ_TRACE_SEND, remote_name, sender); @@ -966,7 +953,7 @@ erts_dsig_send_reg_msg(Eterm remote_name, Eterm message, erts_snprintf(receiver_name, sizeof(DTRACE_CHARBUF_NAME(receiver_name)), "{%T,%s}", remote_name, node_name); msize = size_object(message); - if (token != NIL && token != am_have_dt_utag) { + if (have_seqtrace(token)) { tok_label = signed_val(SEQ_TRACE_T_LABEL(token)); tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token)); tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token)); @@ -1011,11 +998,7 @@ erts_dsig_send_exit_tt(ErtsDSigData *dsdp, Eterm local, Eterm remote, #endif UseTmpHeapNoproc(6); - if (token != NIL -#ifdef USE_VM_PROBES - && token != am_have_dt_utag -#endif - ) { + if (have_seqtrace(token)) { seq_trace_update_send(dsdp->proc); seq_trace_output_exit(token, reason, SEQ_TRACE_SEND, remote, local); ctl = TUPLE5(&ctl_heap[0], @@ -1034,7 +1017,7 @@ erts_dsig_send_exit_tt(ErtsDSigData *dsdp, Eterm local, Eterm remote, "{%T,%s}", remote, node_name); erts_snprintf(reason_str, sizeof(DTRACE_CHARBUF_NAME(reason_str)), "%T", reason); - if (token != NIL && token != am_have_dt_utag) { + if (have_seqtrace(token)) { tok_label = signed_val(SEQ_TRACE_T_LABEL(token)); tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token)); tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token)); @@ -1466,7 +1449,7 @@ int erts_net_message(Port *prt, ErlOffHeap *ohp; ASSERT(xsize); heap_frag = erts_dist_ext_trailer(ede_copy); - ERTS_INIT_HEAP_FRAG(heap_frag, token_size); + ERTS_INIT_HEAP_FRAG(heap_frag, token_size, token_size); hp = heap_frag->mem; ohp = &heap_frag->off_heap; token = tuple[5]; @@ -1515,7 +1498,7 @@ int erts_net_message(Port *prt, ErlOffHeap *ohp; ASSERT(xsize); heap_frag = erts_dist_ext_trailer(ede_copy); - ERTS_INIT_HEAP_FRAG(heap_frag, token_size); + ERTS_INIT_HEAP_FRAG(heap_frag, token_size, token_size); hp = heap_frag->mem; ohp = &heap_frag->off_heap; token = tuple[4]; @@ -2062,9 +2045,9 @@ dist_port_commandv(Port *prt, ErtsDistOutputBuf *obuf) } -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) #define ERTS_PORT_REDS_MASK__ 0x003fffffffffffffL -#elif defined(ARCH_32) || HALFWORD_HEAP +#elif defined(ARCH_32) #define ERTS_PORT_REDS_MASK__ 0x003fffff #else # error "Ohh come on ... !?!" @@ -3275,11 +3258,16 @@ send_nodes_mon_msg(Process *rp, Uint sz) { Eterm msg; - ErlHeapFragment* bp; + Eterm *hp; + ErtsMessage *mp; ErlOffHeap *ohp; - Eterm *hp = erts_alloc_message_heap(sz, &bp, &ohp, rp, rp_locksp); #ifdef DEBUG - Eterm *hend = hp + sz; + Eterm *hend; +#endif + + mp = erts_alloc_message_heap(rp, rp_locksp, sz, &hp, &ohp); +#ifdef DEBUG + hend = hp + sz; #endif if (!nmp->opts) { @@ -3325,7 +3313,7 @@ send_nodes_mon_msg(Process *rp, } ASSERT(hend == hp); - erts_queue_message(rp, rp_locksp, bp, msg, NIL); + erts_queue_message(rp, rp_locksp, mp, msg, NIL); } static void diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index 2a97069ac2..806f569c38 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -103,9 +103,9 @@ static Uint install_debug_functions(void); static int lock_all_physical_memory = 0; -ErtsAllocatorFunctions_t erts_allctrs[ERTS_ALC_A_MAX+1]; +ErtsAllocatorFunctions_t ERTS_WRITE_UNLIKELY(erts_allctrs[ERTS_ALC_A_MAX+1]); ErtsAllocatorInfo_t erts_allctrs_info[ERTS_ALC_A_MAX+1]; -ErtsAllocatorThrSpec_t erts_allctr_thr_spec[ERTS_ALC_A_MAX+1]; +ErtsAllocatorThrSpec_t ERTS_WRITE_UNLIKELY(erts_allctr_thr_spec[ERTS_ALC_A_MAX+1]); #define ERTS_MIN(A, B) ((A) < (B) ? (A) : (B)) #define ERTS_MAX(A, B) ((A) > (B) ? (A) : (B)) @@ -123,10 +123,6 @@ typedef union { static ErtsAllocatorState_t std_alloc_state; static ErtsAllocatorState_t ll_alloc_state; -#if HALFWORD_HEAP -static ErtsAllocatorState_t std_low_alloc_state; -static ErtsAllocatorState_t ll_low_alloc_state; -#endif static ErtsAllocatorState_t sl_alloc_state; static ErtsAllocatorState_t temp_alloc_state; static ErtsAllocatorState_t eheap_alloc_state; @@ -134,6 +130,7 @@ static ErtsAllocatorState_t binary_alloc_state; static ErtsAllocatorState_t ets_alloc_state; static ErtsAllocatorState_t driver_alloc_state; static ErtsAllocatorState_t fix_alloc_state; +static ErtsAllocatorState_t literal_alloc_state; static ErtsAllocatorState_t test_alloc_state; typedef struct { @@ -151,24 +148,10 @@ typedef struct { #define ERTS_ALC_INFO_A_MSEG_ALLOC (ERTS_ALC_A_MAX + 2) #define ERTS_ALC_INFO_A_MAX ERTS_ALC_INFO_A_MSEG_ALLOC -#if !HALFWORD_HEAP ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(aireq, - ErtsAllocInfoReq, - 5, - ERTS_ALC_T_AINFO_REQ) -#else -static ERTS_INLINE ErtsAllocInfoReq * -aireq_alloc(void) -{ - return erts_alloc(ERTS_ALC_T_AINFO_REQ, sizeof(ErtsAllocInfoReq)); -} - -static ERTS_INLINE void -aireq_free(ErtsAllocInfoReq *ptr) -{ - erts_free(ERTS_ALC_T_AINFO_REQ, ptr); -} -#endif + ErtsAllocInfoReq, + 5, + ERTS_ALC_T_AINFO_REQ) ErtsAlcType_t erts_fix_core_allocator_ix; @@ -230,10 +213,7 @@ typedef struct { struct au_init ets_alloc; struct au_init driver_alloc; struct au_init fix_alloc; -#if HALFWORD_HEAP - struct au_init std_low_alloc; - struct au_init ll_low_alloc; -#endif + struct au_init literal_alloc; struct au_init test_alloc; } erts_alc_hndl_args_init_t; @@ -261,10 +241,6 @@ set_default_sl_alloc_opts(struct au_init *ip) #endif ip->init.util.ts = ERTS_ALC_MTA_SHORT_LIVED; ip->init.util.rsbcst = 80; -#if HALFWORD_HEAP - ip->init.util.force = 1; - ip->init.util.low_mem = 1; -#endif ip->init.util.acul = ERTS_ALC_DEFAULT_ACUL; } @@ -300,9 +276,9 @@ set_default_ll_alloc_opts(struct au_init *ip) ip->init.util.name_prefix = "ll_"; ip->init.util.alloc_no = ERTS_ALC_A_LONG_LIVED; #ifndef SMALL_MEMORY - ip->init.util.mmbcs = 2*1024*1024 - 40; /* Main carrier size */ + ip->init.util.mmbcs = 2*1024*1024; /* Main carrier size */ #else - ip->init.util.mmbcs = 1*1024*1024 - 40; /* Main carrier size */ + ip->init.util.mmbcs = 1*1024*1024; /* Main carrier size */ #endif ip->init.util.ts = ERTS_ALC_MTA_LONG_LIVED; ip->init.util.asbcst = 0; @@ -313,6 +289,51 @@ set_default_ll_alloc_opts(struct au_init *ip) } static void +set_default_literal_alloc_opts(struct au_init *ip) +{ + SET_DEFAULT_ALLOC_OPTS(ip); + ip->enable = 1; + ip->thr_spec = 0; + ip->atype = BESTFIT; + ip->init.bf.ao = 1; + ip->init.util.ramv = 0; + ip->init.util.mmsbc = 0; + ip->init.util.sbct = ~((UWord) 0); + ip->init.util.name_prefix = "literal_"; + ip->init.util.alloc_no = ERTS_ALC_A_LITERAL; +#ifndef SMALL_MEMORY + ip->init.util.mmbcs = 2*1024*1024; /* Main carrier size */ +#else + ip->init.util.mmbcs = 1*1024*1024; /* Main carrier size */ +#endif + ip->init.util.ts = ERTS_ALC_MTA_LITERAL; + ip->init.util.asbcst = 0; + ip->init.util.rsbcst = 0; + ip->init.util.rsbcmt = 0; + ip->init.util.rmbcmt = 0; + ip->init.util.acul = 0; + +#if defined(ARCH_32) +# if HAVE_ERTS_MSEG + ip->init.util.mseg_alloc = &erts_alcu_literal_32_mseg_alloc; + ip->init.util.mseg_realloc = &erts_alcu_literal_32_mseg_realloc; + ip->init.util.mseg_dealloc = &erts_alcu_literal_32_mseg_dealloc; +# endif + ip->init.util.sys_alloc = &erts_alcu_literal_32_sys_alloc; + ip->init.util.sys_realloc = &erts_alcu_literal_32_sys_realloc; + ip->init.util.sys_dealloc = &erts_alcu_literal_32_sys_dealloc; +#elif defined(ARCH_64) +# ifdef ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION + ip->init.util.mseg_alloc = &erts_alcu_literal_64_mseg_alloc; + ip->init.util.mseg_realloc = &erts_alcu_literal_64_mseg_realloc; + ip->init.util.mseg_dealloc = &erts_alcu_literal_64_mseg_dealloc; +# endif +#else +# error Unknown architecture +#endif +} + +static void set_default_temp_alloc_opts(struct au_init *ip) { SET_DEFAULT_ALLOC_OPTS(ip); @@ -330,10 +351,6 @@ set_default_temp_alloc_opts(struct au_init *ip) ip->init.util.ts = ERTS_ALC_MTA_TEMPORARY; ip->init.util.rsbcst = 90; ip->init.util.rmbcmt = 100; -#if HALFWORD_HEAP - ip->init.util.force = 1; - ip->init.util.low_mem = 1; -#endif } static void @@ -352,10 +369,6 @@ set_default_eheap_alloc_opts(struct au_init *ip) #endif ip->init.util.ts = ERTS_ALC_MTA_EHEAP; ip->init.util.rsbcst = 50; -#if HALFWORD_HEAP - ip->init.util.force = 1; - ip->init.util.low_mem = 1; -#endif ip->init.util.acul = ERTS_ALC_DEFAULT_ACUL_EHEAP_ALLOC; } @@ -589,18 +602,16 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_PROC)] = sizeof(Process); -#if !HALFWORD_HEAP fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MONITOR_SH)] = ERTS_MONITOR_SH_SIZE * sizeof(Uint); fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_NLINK_SH)] = ERTS_LINK_SH_SIZE * sizeof(Uint); -#endif fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_DRV_EV_D_STATE)] = sizeof(ErtsDrvEventDataState); fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_DRV_SEL_D_STATE)] = sizeof(ErtsDrvSelectDataState); fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MSG_REF)] - = sizeof(ErlMessage); + = sizeof(ErtsMessageRef); #ifdef ERTS_SMP fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_THR_Q_EL_SL)] = sizeof(ErtsThrQElement_t); @@ -642,6 +653,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) set_default_driver_alloc_opts(&init.driver_alloc); set_default_fix_alloc_opts(&init.fix_alloc, fix_type_sizes); + set_default_literal_alloc_opts(&init.literal_alloc); set_default_test_alloc_opts(&init.test_alloc); if (argc && argv) @@ -670,6 +682,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) init.ets_alloc.thr_spec = 0; init.driver_alloc.thr_spec = 0; init.fix_alloc.thr_spec = 0; + init.literal_alloc.thr_spec = 0; #endif /* Make adjustments for carrier migration support */ @@ -682,6 +695,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) adjust_carrier_migration_support(&init.ets_alloc); adjust_carrier_migration_support(&init.driver_alloc); adjust_carrier_migration_support(&init.fix_alloc); + adjust_carrier_migration_support(&init.literal_alloc); if (init.erts_alloc_config) { /* Adjust flags that erts_alloc_config won't like */ @@ -696,6 +710,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) init.ets_alloc.thr_spec = 0; init.driver_alloc.thr_spec = 0; init.fix_alloc.thr_spec = 0; + init.literal_alloc.thr_spec = 0; /* No carrier migration */ init.temp_alloc.init.util.acul = 0; @@ -707,6 +722,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) init.ets_alloc.init.util.acul = 0; init.driver_alloc.init.util.acul = 0; init.fix_alloc.init.util.acul = 0; + init.literal_alloc.init.util.acul = 0; } #ifdef ERTS_SMP @@ -723,6 +739,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) adjust_tpref(&init.ets_alloc, erts_no_schedulers); adjust_tpref(&init.driver_alloc, erts_no_schedulers); adjust_tpref(&init.fix_alloc, erts_no_schedulers); + adjust_tpref(&init.literal_alloc, erts_no_schedulers); #else /* No thread specific if not smp */ @@ -741,6 +758,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) refuse_af_strategy(&init.ets_alloc); refuse_af_strategy(&init.driver_alloc); refuse_af_strategy(&init.fix_alloc); + refuse_af_strategy(&init.literal_alloc); #ifdef ERTS_SMP if (!init.temp_alloc.thr_spec) @@ -751,6 +769,9 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) #if HAVE_ERTS_MSEG init.mseg.nos = erts_no_schedulers; erts_mseg_init(&init.mseg); +# if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) + erts_mmap_init(&erts_literal_mmapper, &init.mseg.literal_mmap); +# endif #endif erts_alcu_init(&init.alloc_util); @@ -775,24 +796,6 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) erts_allctrs[ERTS_ALC_A_SYSTEM].free = erts_sys_free; erts_allctrs_info[ERTS_ALC_A_SYSTEM].enabled = 1; -#if HALFWORD_HEAP - /* Init low memory variants by cloning */ - init.std_low_alloc = init.std_alloc; - init.std_low_alloc.init.util.name_prefix = "std_low_"; - init.std_low_alloc.init.util.alloc_no = ERTS_ALC_A_STANDARD_LOW; - init.std_low_alloc.init.util.force = 1; - init.std_low_alloc.init.util.low_mem = 1; - - init.ll_low_alloc = init.ll_alloc; - init.ll_low_alloc.init.util.name_prefix = "ll_low_"; - init.ll_low_alloc.init.util.alloc_no = ERTS_ALC_A_LONG_LIVED_LOW; - init.ll_low_alloc.init.util.force = 1; - init.ll_low_alloc.init.util.low_mem = 1; - - set_au_allocator(ERTS_ALC_A_STANDARD_LOW, &init.std_low_alloc, ncpu); - set_au_allocator(ERTS_ALC_A_LONG_LIVED_LOW, &init.ll_low_alloc, ncpu); -#endif /* HALFWORD */ - set_au_allocator(ERTS_ALC_A_TEMPORARY, &init.temp_alloc, ncpu); set_au_allocator(ERTS_ALC_A_SHORT_LIVED, &init.sl_alloc, ncpu); set_au_allocator(ERTS_ALC_A_STANDARD, &init.std_alloc, ncpu); @@ -802,6 +805,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) set_au_allocator(ERTS_ALC_A_ETS, &init.ets_alloc, ncpu); set_au_allocator(ERTS_ALC_A_DRIVER, &init.driver_alloc, ncpu); set_au_allocator(ERTS_ALC_A_FIXED_SIZE, &init.fix_alloc, ncpu); + set_au_allocator(ERTS_ALC_A_LITERAL, &init.literal_alloc, ncpu); set_au_allocator(ERTS_ALC_A_TEST, &init.test_alloc, ncpu); for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) { @@ -836,14 +840,6 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) start_au_allocator(ERTS_ALC_A_LONG_LIVED, &init.ll_alloc, &ll_alloc_state); -#if HALFWORD_HEAP - start_au_allocator(ERTS_ALC_A_LONG_LIVED_LOW, - &init.ll_low_alloc, - &ll_low_alloc_state); - start_au_allocator(ERTS_ALC_A_STANDARD_LOW, - &init.std_low_alloc, - &std_low_alloc_state); -#endif start_au_allocator(ERTS_ALC_A_EHEAP, &init.eheap_alloc, &eheap_alloc_state); @@ -863,6 +859,9 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) start_au_allocator(ERTS_ALC_A_FIXED_SIZE, &init.fix_alloc, &fix_alloc_state); + start_au_allocator(ERTS_ALC_A_LITERAL, + &init.literal_alloc, + &literal_alloc_state); start_au_allocator(ERTS_ALC_A_TEST, &init.test_alloc, @@ -871,9 +870,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) erts_mtrace_install_wrapper_functions(); extra_block_size += erts_instr_init(init.instr.stat, init.instr.map); -#if !HALFWORD_HEAP init_aireq_alloc(); -#endif #ifdef DEBUG extra_block_size += install_debug_functions(); @@ -971,6 +968,10 @@ set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init, int ncpu) else #endif { +#ifdef ERTS_SMP + erl_exit(ERTS_ABORT_EXIT, "%salloc is not thread safe\n", + init->init.util.name_prefix); +#else af->alloc = erts_alcu_alloc; if (init->init.util.fix_type_size) af->realloc = erts_realloc_fixed_size; @@ -979,6 +980,7 @@ set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init, int ncpu) else af->realloc = erts_alcu_realloc; af->free = erts_alcu_free; +#endif } af->extra = NULL; ai->alloc_util = 1; @@ -1051,7 +1053,7 @@ start_au_allocator(ErtsAlcType_t alctr_n, } for (i = 0; i < size; i++) { - void *as; + Allctr_t *as; atype = init->atype; if (!init->thr_spec) @@ -1088,22 +1090,22 @@ start_au_allocator(ErtsAlcType_t alctr_n, switch (atype) { case GOODFIT: - as = (void *) erts_gfalc_start((GFAllctr_t *) as0, + as = erts_gfalc_start((GFAllctr_t *) as0, &init->init.gf, &init->init.util); break; case BESTFIT: - as = (void *) erts_bfalc_start((BFAllctr_t *) as0, + as = erts_bfalc_start((BFAllctr_t *) as0, &init->init.bf, &init->init.util); break; case AFIT: - as = (void *) erts_afalc_start((AFAllctr_t *) as0, + as = erts_afalc_start((AFAllctr_t *) as0, &init->init.af, &init->init.util); break; case AOFIRSTFIT: - as = (void *) erts_aoffalc_start((AOFFAllctr_t *) as0, + as = erts_aoffalc_start((AOFFAllctr_t *) as0, &init->init.aoff, &init->init.util); break; @@ -1273,9 +1275,6 @@ get_acul_value(struct au_init *auip, char *param_end, char** argv, int* ip) if (sys_strcmp(value, "de") == 0) { switch (auip->init.util.alloc_no) { case ERTS_ALC_A_LONG_LIVED: -#if HALFWORD_HEAP - case ERTS_ALC_A_LONG_LIVED_LOW: -#endif return ERTS_ALC_DEFAULT_ENABLED_ACUL_LL_ALLOC; case ERTS_ALC_A_EHEAP: return ERTS_ALC_DEFAULT_ENABLED_ACUL_EHEAP_ALLOC; @@ -1509,25 +1508,25 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) } else if (has_prefix("scs", argv[i]+3)) { #if HAVE_ERTS_MSEG - init->mseg.mmap.scs = + init->mseg.dflt_mmap.scs = #endif get_mb_value(argv[i]+6, argv, &i); } else if (has_prefix("sco", argv[i]+3)) { #if HAVE_ERTS_MSEG - init->mseg.mmap.sco = + init->mseg.dflt_mmap.sco = #endif get_bool_value(argv[i]+6, argv, &i); } else if (has_prefix("scrpm", argv[i]+3)) { #if HAVE_ERTS_MSEG - init->mseg.mmap.scrpm = + init->mseg.dflt_mmap.scrpm = #endif get_bool_value(argv[i]+8, argv, &i); } else if (has_prefix("scrfsd", argv[i]+3)) { #if HAVE_ERTS_MSEG - init->mseg.mmap.scrfsd = + init->mseg.dflt_mmap.scrfsd = #endif get_amount_value(argv[i]+9, argv, &i); } @@ -1996,48 +1995,6 @@ alcu_size(ErtsAlcType_t ai, ErtsAlcUFixInfo_t *fi, int fisz) return res; } -#if HALFWORD_HEAP -static ERTS_INLINE int -alcu_is_low(ErtsAlcType_t ai) -{ - int is_low = 0; - ASSERT(erts_allctrs_info[ai].enabled); - ASSERT(erts_allctrs_info[ai].alloc_util); - - if (!erts_allctrs_info[ai].thr_spec) { - Allctr_t *allctr = erts_allctrs_info[ai].extra; - is_low = allctr->mseg_opt.low_mem; - } - else { - ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[ai]; - int i; -# ifdef DEBUG - int found_one = 0; -# endif - - ASSERT(tspec->enabled); - - for (i = tspec->size - 1; i >= 0; i--) { - Allctr_t *allctr = tspec->allctr[i]; - if (allctr) { -# ifdef DEBUG - if (!found_one) { - is_low = allctr->mseg_opt.low_mem; - found_one = 1; - } - else ASSERT(is_low == allctr->mseg_opt.low_mem); -# else - is_low = allctr->mseg_opt.low_mem; - break; -# endif - } - } - ASSERT(found_one); - } - return is_low; -} -#endif /* HALFWORD */ - static ERTS_INLINE void add_fix_values(UWord *ap, UWord *up, ErtsAlcUFixInfo_t *fi, ErtsAlcType_t type) { @@ -2067,9 +2024,6 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) int code; int ets; int maximum; -#if HALFWORD_HEAP - int low; -#endif } want = {0}; struct { UWord total; @@ -2082,9 +2036,6 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) UWord code; UWord ets; UWord maximum; -#if HALFWORD_HEAP - UWord low; -#endif } size = {0}; Eterm atoms[sizeof(size)/sizeof(UWord)]; UWord *uintps[sizeof(size)/sizeof(UWord)]; @@ -2143,11 +2094,6 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) atoms[length] = am_maximum; uintps[length++] = &size.maximum; } -#if HALFWORD_HEAP - want.low = 1; - atoms[length] = am_low; - uintps[length++] = &size.low; -#endif } else { DeclareTmpHeapNoproc(tmp_heap,2); @@ -2241,15 +2187,6 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) return am_badarg; } break; -#if HALFWORD_HEAP - case am_low: - if (!want.low) { - want.low = 1; - atoms[length] = am_low; - uintps[length++] = &size.low; - } - break; -#endif default: UnUseTmpHeapNoproc(2); return am_badarg; @@ -2330,11 +2267,6 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) if (save) *save = asz; size.total += asz; -#if HALFWORD_HEAP - if (alcu_is_low(ai)) { - size.low += asz; - } -#endif } } } @@ -2361,7 +2293,6 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) &size.processes_used, fi, ERTS_ALC_T_PROC); -#if !HALFWORD_HEAP add_fix_values(&size.processes, &size.processes_used, fi, @@ -2371,7 +2302,6 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) &size.processes_used, fi, ERTS_ALC_T_NLINK_SH); -#endif add_fix_values(&size.processes, &size.processes_used, fi, @@ -2810,7 +2740,7 @@ erts_allocator_info(int to, void *arg) erts_mseg_info(i, &to, arg, 0, NULL, NULL); } erts_print(to, arg, "=allocator:mseg_alloc.erts_mmap\n"); - erts_mmap_info(&to, arg, NULL, NULL, &emis); + erts_mmap_info(&erts_dflt_mmapper, &to, arg, NULL, NULL, &emis); } #endif @@ -3036,12 +2966,12 @@ reply_alloc_info(void *vair) int global_instances = air->req_sched == sched_id; ErtsProcLocks rp_locks; Process *rp = air->proc; - Eterm ref_copy = NIL, ai_list, msg; - Eterm *hp = NULL, *hp_end = NULL, *hp_start = NULL; + Eterm ref_copy = NIL, ai_list, msg = NIL; + Eterm *hp = NULL, *hp_start = NULL, *hp_end = NULL; Eterm **hpp; Uint sz, *szp; ErlOffHeap *ohp = NULL; - ErlHeapFragment *bp = NULL; + ErtsMessage *mp = NULL; struct erts_mmap_info_struct emis; int i; Eterm (*info_func)(Allctr_t *, @@ -3162,7 +3092,8 @@ reply_alloc_info(void *vair) ai_list = erts_bld_cons(hpp, szp, ainfo, ai_list); - ainfo = (air->only_sz ? NIL : erts_mmap_info(NULL, NULL, hpp, szp, &emis)); + ainfo = (air->only_sz ? NIL : + erts_mmap_info(&erts_dflt_mmapper, NULL, NULL, hpp, szp, &emis)); ainfo = erts_bld_tuple3(hpp, szp, alloc_atom, erts_bld_atom(hpp,szp,"erts_mmap"), @@ -3242,20 +3173,17 @@ reply_alloc_info(void *vair) if (hpp) break; - hp = erts_alloc_message_heap(sz, &bp, &ohp, rp, &rp_locks); + mp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp); hp_start = hp; hp_end = hp + sz; szp = NULL; hpp = &hp; } - if (bp) - bp = erts_resize_message_buffer(bp, hp - hp_start, &msg, 1); - else { - ASSERT(hp); - HRelease(rp, hp_end, hp); - } - erts_queue_message(rp, &rp_locks, bp, msg, NIL); + if (hp != hp_end) + erts_shrink_message_heap(&mp, rp, hp_start, hp, hp_end, &msg, 1); + + erts_queue_message(rp, &rp_locks, mp, msg, NIL); if (air->req_sched == sched_id) rp_locks &= ~ERTS_PROC_LOCK_MAIN; @@ -3479,8 +3407,11 @@ UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3) init.enable = 1; init.atype = GOODFIT; init.init.util.name_prefix = (char *) a1; - init.init.util.ts = a2 ? 1 : 0; - +#ifdef ERTS_SMP + init.init.util.ts = 1; +#else + init.init.util.ts = a2 ? 1 : 0; +#endif if ((char **) a3) { char **argv = (char **) a3; int i = 0; diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h index f540bae20d..71e4713624 100644 --- a/erts/emulator/beam/erl_alloc.h +++ b/erts/emulator/beam/erl_alloc.h @@ -30,6 +30,7 @@ #ifdef USE_THREADS #include "erl_threads.h" #endif +#include "erl_mmap.h" #ifdef DEBUG # undef ERTS_ALC_WANT_INLINE @@ -43,9 +44,11 @@ #if ERTS_CAN_INLINE && ERTS_ALC_WANT_INLINE # define ERTS_ALC_DO_INLINE 1 # define ERTS_ALC_INLINE static ERTS_INLINE +# define ERTS_ALC_FORCE_INLINE static ERTS_FORCE_INLINE #else # define ERTS_ALC_DO_INLINE 0 # define ERTS_ALC_INLINE +# define ERTS_ALC_FORCE_INLINE #endif #define ERTS_ALC_NO_FIXED_SIZES \ @@ -177,6 +180,12 @@ void sys_free(void *) __deprecated; /* erts_free() */ void *sys_alloc(Uint ) __deprecated; /* erts_alloc_fnf() */ void *sys_realloc(void *, Uint) __deprecated; /* erts_realloc_fnf() */ +#undef ERTS_HAVE_IS_IN_LITERAL_RANGE +#if defined(ARCH_32) || defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) +# define ERTS_HAVE_IS_IN_LITERAL_RANGE +#endif + + /* * erts_alloc[_fnf](), erts_realloc[_fnf](), erts_free() works as * malloc(), realloc(), and free() with the following exceptions: @@ -204,6 +213,9 @@ void erts_free(ErtsAlcType_t type, void *ptr); void *erts_alloc_fnf(ErtsAlcType_t type, Uint size); void *erts_realloc_fnf(ErtsAlcType_t type, void *ptr, Uint size); int erts_is_allctr_wrapper_prelocked(void); +#ifdef ERTS_HAVE_IS_IN_LITERAL_RANGE +int erts_is_in_literal_range(void* ptr); +#endif #endif /* #if !ERTS_ALC_DO_INLINE */ @@ -221,12 +233,14 @@ ERTS_ALC_INLINE void *erts_alloc(ErtsAlcType_t type, Uint size) { void *res; + ERTS_MSACC_PUSH_AND_SET_STATE_X(ERTS_MSACC_STATE_ALLOC); res = (*erts_allctrs[ERTS_ALC_T2A(type)].alloc)( ERTS_ALC_T2N(type), erts_allctrs[ERTS_ALC_T2A(type)].extra, size); if (!res) erts_alloc_n_enomem(ERTS_ALC_T2N(type), size); + ERTS_MSACC_POP_STATE_X(); return res; } @@ -234,6 +248,7 @@ ERTS_ALC_INLINE void *erts_realloc(ErtsAlcType_t type, void *ptr, Uint size) { void *res; + ERTS_MSACC_PUSH_AND_SET_STATE_X(ERTS_MSACC_STATE_ALLOC); res = (*erts_allctrs[ERTS_ALC_T2A(type)].realloc)( ERTS_ALC_T2N(type), erts_allctrs[ERTS_ALC_T2A(type)].extra, @@ -241,37 +256,48 @@ void *erts_realloc(ErtsAlcType_t type, void *ptr, Uint size) size); if (!res) erts_realloc_n_enomem(ERTS_ALC_T2N(type), ptr, size); + ERTS_MSACC_POP_STATE_X(); return res; } ERTS_ALC_INLINE void erts_free(ErtsAlcType_t type, void *ptr) { + ERTS_MSACC_PUSH_AND_SET_STATE_X(ERTS_MSACC_STATE_ALLOC); (*erts_allctrs[ERTS_ALC_T2A(type)].free)( ERTS_ALC_T2N(type), erts_allctrs[ERTS_ALC_T2A(type)].extra, ptr); + ERTS_MSACC_POP_STATE_X(); } ERTS_ALC_INLINE void *erts_alloc_fnf(ErtsAlcType_t type, Uint size) { - return (*erts_allctrs[ERTS_ALC_T2A(type)].alloc)( + void *res; + ERTS_MSACC_PUSH_AND_SET_STATE_X(ERTS_MSACC_STATE_ALLOC); + res = (*erts_allctrs[ERTS_ALC_T2A(type)].alloc)( ERTS_ALC_T2N(type), erts_allctrs[ERTS_ALC_T2A(type)].extra, size); + ERTS_MSACC_POP_STATE_X(); + return res; } ERTS_ALC_INLINE void *erts_realloc_fnf(ErtsAlcType_t type, void *ptr, Uint size) { - return (*erts_allctrs[ERTS_ALC_T2A(type)].realloc)( + void *res; + ERTS_MSACC_PUSH_AND_SET_STATE_X(ERTS_MSACC_STATE_ALLOC); + res = (*erts_allctrs[ERTS_ALC_T2A(type)].realloc)( ERTS_ALC_T2N(type), erts_allctrs[ERTS_ALC_T2A(type)].extra, ptr, size); + ERTS_MSACC_POP_STATE_X(); + return res; } ERTS_ALC_INLINE @@ -281,6 +307,28 @@ int erts_is_allctr_wrapper_prelocked(void) && !!erts_tsd_get(erts_allctr_prelock_tsd_key); /* by me */ } +#ifdef ERTS_HAVE_IS_IN_LITERAL_RANGE + +ERTS_ALC_FORCE_INLINE +int erts_is_in_literal_range(void* ptr) +{ +#if defined(ARCH_32) + Uint ix = (UWord)ptr >> ERTS_MMAP_SUPERALIGNED_BITS; + + return erts_literal_vspace_map[ix / ERTS_VSPACE_WORD_BITS] + & ((UWord)1 << (ix % ERTS_VSPACE_WORD_BITS)); + +#elif defined(ARCH_64) + extern char* erts_literals_start; + extern UWord erts_literals_size; + return ErtsInArea(ptr, erts_literals_start, erts_literals_size); +#else +# error No ARCH_xx +#endif +} + +#endif /* ERTS_HAVE_IS_IN_LITERAL_RANGE */ + #endif /* #if ERTS_ALC_DO_INLINE || defined(ERTS_ALC_INTERNAL__) */ #define ERTS_ALC_GET_THR_IX() ((int) erts_get_scheduler_id()) diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index 7738531142..2932adca84 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -86,11 +86,7 @@ allocator LONG_LIVED true ll_alloc allocator EHEAP true eheap_alloc allocator ETS true ets_alloc allocator FIXED_SIZE true fix_alloc - -+if halfword -allocator LONG_LIVED_LOW true ll_low_alloc -allocator STANDARD_LOW true std_low_alloc -+endif +allocator LITERAL true literal_alloc +else # Non smp build @@ -101,11 +97,7 @@ allocator LONG_LIVED false ll_alloc allocator EHEAP false eheap_alloc allocator ETS false ets_alloc allocator FIXED_SIZE false fix_alloc - -+if halfword -allocator LONG_LIVED_LOW false ll_low_alloc -allocator STANDARD_LOW false std_low_alloc -+endif +allocator LITERAL false literal_alloc +endif @@ -160,6 +152,8 @@ type OLD_HEAP EHEAP PROCESSES old_heap type HEAP_FRAG EHEAP PROCESSES heap_frag type TMP_HEAP TEMPORARY PROCESSES tmp_heap type MSG_REF FIXED_SIZE PROCESSES msg_ref +type MSG EHEAP PROCESSES message +type MSGQ_CHNG SHORT_LIVED PROCESSES messages_queue_change type MSG_ROOTS TEMPORARY PROCESSES msg_roots type ROOTSET TEMPORARY PROCESSES root_set type LOADER_TMP TEMPORARY CODE loader_tmp @@ -349,34 +343,11 @@ type SSB SHORT_LIVED PROCESSES ssb +endif - -+if halfword - -type DDLL_PROCESS STANDARD_LOW SYSTEM ddll_processes -type MONITOR_LH STANDARD_LOW PROCESSES monitor_lh -type NLINK_LH STANDARD_LOW PROCESSES nlink_lh -type CODE LONG_LIVED_LOW CODE code -type DB_HEIR_DATA STANDARD_LOW ETS db_heir_data -type DB_MS_PSDO_PROC LONG_LIVED_LOW ETS db_match_pseudo_proc -type SCHDLR_DATA LONG_LIVED_LOW SYSTEM scheduler_data -type LL_TEMP_TERM LONG_LIVED_LOW SYSTEM ll_temp_term - -type NIF_TRAP_EXPORT STANDARD_LOW CODE nif_trap_export_entry -type EXPORT LONG_LIVED_LOW CODE export_entry -type MONITOR_SH STANDARD_LOW PROCESSES monitor_sh -type NLINK_SH STANDARD_LOW PROCESSES nlink_sh -type AINFO_REQ STANDARD_LOW SYSTEM alloc_info_request -type SCHED_WTIME_REQ STANDARD_LOW SYSTEM sched_wall_time_request -type GC_INFO_REQ STANDARD_LOW SYSTEM gc_info_request -type PORT_DATA_HEAP STANDARD_LOW SYSTEM port_data_heap -type SYS_CHECK_REQ STANDARD_LOW SYSTEM system_check_request - -+else # "fullword" - type DDLL_PROCESS STANDARD SYSTEM ddll_processes type MONITOR_LH STANDARD PROCESSES monitor_lh type NLINK_LH STANDARD PROCESSES nlink_lh type CODE LONG_LIVED CODE code +type LITERAL LITERAL CODE literal type DB_HEIR_DATA STANDARD ETS db_heir_data type DB_MS_PSDO_PROC LONG_LIVED ETS db_match_pseudo_proc type SCHDLR_DATA LONG_LIVED SYSTEM scheduler_data @@ -390,11 +361,9 @@ type AINFO_REQ SHORT_LIVED SYSTEM alloc_info_request type SCHED_WTIME_REQ SHORT_LIVED SYSTEM sched_wall_time_request type GC_INFO_REQ SHORT_LIVED SYSTEM gc_info_request type PORT_DATA_HEAP STANDARD SYSTEM port_data_heap +type MSACC DRIVER SYSTEM microstate_accounting type SYS_CHECK_REQ SHORT_LIVED SYSTEM system_check_request -+endif - - # # Types used by system specific code # @@ -432,21 +401,6 @@ type SYS_WRITE_BUF BINARY SYSTEM sys_write_buf +endif -+if ose - -type SYS_READ_BUF TEMPORARY SYSTEM sys_read_buf -type FD_TAB LONG_LIVED SYSTEM fd_tab -type FD_ENTRY_BUF STANDARD SYSTEM fd_entry_buf -type FD_SIG_LIST SHORT_LIVED SYSTEM fd_sig_list -type DRV_EV STANDARD SYSTEM driver_event -type CS_PROG_PATH LONG_LIVED SYSTEM cs_prog_path -type ENVIRONMENT TEMPORARY SYSTEM environment -type PUTENV_STR SYSTEM SYSTEM putenv_string -type PRT_REP_EXIT STANDARD SYSTEM port_report_exit - -+endif - - +if win32 type DRV_DATA_BUF SYSTEM SYSTEM drv_data_buf diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index 8229a15824..3230b6ef34 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -751,12 +751,77 @@ internal_free(void *ptr) #endif +#ifdef ARCH_32 + +/* + * Bit vector for the entire 32-bit virtual address space + * with one bit for each super aligned memory segment. + */ + +#define VSPACE_MAP_BITS (1 << (32 - ERTS_MMAP_SUPERALIGNED_BITS)) +#define VSPACE_MAP_SZ (VSPACE_MAP_BITS / ERTS_VSPACE_WORD_BITS) + +static ERTS_INLINE void set_bit(UWord* map, Uint ix) +{ + ASSERT(ix / ERTS_VSPACE_WORD_BITS < VSPACE_MAP_SZ); + map[ix / ERTS_VSPACE_WORD_BITS] + |= ((UWord)1 << (ix % ERTS_VSPACE_WORD_BITS)); +} + +static ERTS_INLINE void clr_bit(UWord* map, Uint ix) +{ + ASSERT(ix / ERTS_VSPACE_WORD_BITS < VSPACE_MAP_SZ); + map[ix / ERTS_VSPACE_WORD_BITS] + &= ~((UWord)1 << (ix % ERTS_VSPACE_WORD_BITS)); +} + +static ERTS_INLINE int is_bit_set(UWord* map, Uint ix) +{ + ASSERT(ix / ERTS_VSPACE_WORD_BITS < VSPACE_MAP_SZ); + return map[ix / ERTS_VSPACE_WORD_BITS] + & ((UWord)1 << (ix % ERTS_VSPACE_WORD_BITS)); +} + +UWord erts_literal_vspace_map[VSPACE_MAP_SZ]; + +static void set_literal_range(void* start, Uint size) +{ + Uint ix = (UWord)start >> ERTS_MMAP_SUPERALIGNED_BITS; + Uint n = size >> ERTS_MMAP_SUPERALIGNED_BITS; + + ASSERT(!((UWord)start & ERTS_INV_SUPERALIGNED_MASK)); + ASSERT(!((UWord)size & ERTS_INV_SUPERALIGNED_MASK)); + ASSERT(n); + while (n--) { + ASSERT(!is_bit_set(erts_literal_vspace_map, ix)); + set_bit(erts_literal_vspace_map, ix); + ix++; + } +} + +static void clear_literal_range(void* start, Uint size) +{ + Uint ix = (UWord)start >> ERTS_MMAP_SUPERALIGNED_BITS; + Uint n = size >> ERTS_MMAP_SUPERALIGNED_BITS; + + ASSERT(!((UWord)start & ERTS_INV_SUPERALIGNED_MASK)); + ASSERT(!((UWord)size & ERTS_INV_SUPERALIGNED_MASK)); + ASSERT(n); + while (n--) { + ASSERT(is_bit_set(erts_literal_vspace_map, ix)); + clr_bit(erts_literal_vspace_map, ix); + ix++; + } +} + +#endif /* ARCH_32 */ + /* mseg ... */ #if HAVE_ERTS_MSEG -static ERTS_INLINE void * -alcu_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags) +void* +erts_alcu_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags) { void *res; UWord size = (UWord) *size_p; @@ -766,8 +831,9 @@ alcu_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags) return res; } -static ERTS_INLINE void * -alcu_mseg_realloc(Allctr_t *allctr, void *seg, Uint old_size, Uint *new_size_p) +void* +erts_alcu_mseg_realloc(Allctr_t *allctr, void *seg, + Uint old_size, Uint *new_size_p) { void *res; UWord new_size = (UWord) *new_size_p; @@ -778,17 +844,103 @@ alcu_mseg_realloc(Allctr_t *allctr, void *seg, Uint old_size, Uint *new_size_p) return res; } -static ERTS_INLINE void -alcu_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, Uint flags) +void +erts_alcu_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, Uint flags) { erts_mseg_dealloc_opt(allctr->alloc_no, seg, (UWord) size, flags, &allctr->mseg_opt); INC_CC(allctr->calls.mseg_dealloc); } -#endif -static ERTS_INLINE void * -alcu_sys_alloc(Allctr_t *allctr, Uint size, int superalign) +#if defined(ARCH_32) + +void* +erts_alcu_literal_32_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags) +{ + void* res; + ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL + && !allctr->t && allctr->thread_safe); + + res = erts_alcu_mseg_alloc(allctr, size_p, flags); + if (res) + set_literal_range(res, *size_p); + return res; +} + +void* +erts_alcu_literal_32_mseg_realloc(Allctr_t *allctr, void *seg, + Uint old_size, Uint *new_size_p) +{ + void* res; + ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL + && !allctr->t && allctr->thread_safe); + + if (seg && old_size) + clear_literal_range(seg, old_size); + res = erts_alcu_mseg_realloc(allctr, seg, old_size, new_size_p); + if (res) + set_literal_range(res, *new_size_p); + return res; +} + +void +erts_alcu_literal_32_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, + Uint flags) +{ + ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL + && !allctr->t && allctr->thread_safe); + + erts_alcu_mseg_dealloc(allctr, seg, size, flags); + + clear_literal_range(seg, size); +} + +#elif defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) + +void* +erts_alcu_literal_64_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags) +{ + void* res; + UWord size = (UWord) *size_p; + Uint32 mmap_flags = ERTS_MMAPFLG_SUPERCARRIER_ONLY; + if (flags & ERTS_MSEG_FLG_2POW) + mmap_flags |= ERTS_MMAPFLG_SUPERALIGNED; + + res = erts_mmap(&erts_literal_mmapper, mmap_flags, &size); + *size_p = (Uint)size; + INC_CC(allctr->calls.mseg_alloc); + return res; +} + +void* +erts_alcu_literal_64_mseg_realloc(Allctr_t *allctr, void *seg, + Uint old_size, Uint *new_size_p) +{ + void *res; + UWord new_size = (UWord) *new_size_p; + res = erts_mremap(&erts_literal_mmapper, ERTS_MSEG_FLG_NONE, seg, old_size, &new_size); + *new_size_p = (Uint) new_size; + INC_CC(allctr->calls.mseg_realloc); + return res; +} + +void +erts_alcu_literal_64_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, + Uint flags) +{ + Uint32 mmap_flags = ERTS_MMAPFLG_SUPERCARRIER_ONLY; + if (flags & ERTS_MSEG_FLG_2POW) + mmap_flags |= ERTS_MMAPFLG_SUPERALIGNED; + + erts_munmap(&erts_literal_mmapper, mmap_flags, seg, (UWord)size); + INC_CC(allctr->calls.mseg_dealloc); +} +#endif /* ARCH_64 && ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION */ + +#endif /* HAVE_ERTS_MSEG */ + +void* +erts_alcu_sys_alloc(Allctr_t *allctr, Uint size, int superalign) { void *res; #if ERTS_SA_MB_CARRIERS && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC @@ -803,8 +955,8 @@ alcu_sys_alloc(Allctr_t *allctr, Uint size, int superalign) return res; } -static ERTS_INLINE void * -alcu_sys_realloc(Allctr_t *allctr, void *ptr, Uint size, Uint old_size, int superalign) +void* +erts_alcu_sys_realloc(Allctr_t *allctr, void *ptr, Uint size, Uint old_size, int superalign) { void *res; @@ -824,8 +976,8 @@ alcu_sys_realloc(Allctr_t *allctr, void *ptr, Uint size, Uint old_size, int supe return res; } -static ERTS_INLINE void -alcu_sys_free(Allctr_t *allctr, void *ptr, int superalign) +void +erts_alcu_sys_dealloc(Allctr_t *allctr, void *ptr, Uint size, int superalign) { #if ERTS_SA_MB_CARRIERS && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC if (superalign) @@ -838,6 +990,49 @@ alcu_sys_free(Allctr_t *allctr, void *ptr, int superalign) erts_mtrace_crr_free(allctr->alloc_no, ERTS_ALC_A_SYSTEM, ptr); } +#ifdef ARCH_32 + +void* +erts_alcu_literal_32_sys_alloc(Allctr_t *allctr, Uint size, int superalign) +{ + void* res; + ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL + && !allctr->t && allctr->thread_safe); + + res = erts_alcu_sys_alloc(allctr, size, 1); + if (res) + set_literal_range(res, size); + return res; +} + +void* +erts_alcu_literal_32_sys_realloc(Allctr_t *allctr, void *ptr, Uint size, Uint old_size, int superalign) +{ + void* res; + ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL + && !allctr->t && allctr->thread_safe); + + if (ptr && old_size) + clear_literal_range(ptr, old_size); + res = erts_alcu_sys_realloc(allctr, ptr, size, old_size, 1); + if (res) + set_literal_range(res, size); + return res; +} + +void +erts_alcu_literal_32_sys_dealloc(Allctr_t *allctr, void *ptr, Uint size, int superalign) +{ + ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL + && !allctr->t && allctr->thread_safe); + + erts_alcu_sys_dealloc(allctr, ptr, size, 1); + + clear_literal_range(ptr, size); +} + +#endif /* ARCH_32 */ + static Uint get_next_mbc_size(Allctr_t *allctr) { @@ -2084,7 +2279,7 @@ mbc_alloc_block(Allctr_t *allctr, Uint size, Uint *blk_szp) if (!blk) { blk = create_carrier(allctr, get_blk_sz, CFLG_MBC); -#if !HALFWORD_HEAP && !ERTS_SUPER_ALIGNED_MSEG_ONLY +#if !ERTS_SUPER_ALIGNED_MSEG_ONLY if (!blk) { /* Emergency! We couldn't create the carrier as we wanted. Try to place it in a sys_alloced sbc. */ @@ -3531,8 +3726,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) int is_mseg = 0; #endif - if (HALFWORD_HEAP - || (ERTS_SUPER_ALIGNED_MSEG_ONLY && (flags & CFLG_MBC)) + if ((ERTS_SUPER_ALIGNED_MSEG_ONLY && (flags & CFLG_MBC)) || !allow_sys_alloc_carriers) { flags |= CFLG_FORCE_MSEG; flags &= ~CFLG_FORCE_SYS_ALLOC; @@ -3540,6 +3734,8 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) return NULL; #endif } + flags |= allctr->crr_set_flgs; + flags &= ~allctr->crr_clr_flgs; ASSERT((flags & CFLG_SBC && !(flags & CFLG_MBC)) || (flags & CFLG_MBC && !(flags & CFLG_SBC))); @@ -3555,7 +3751,21 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) return NULL; } - blk_sz = UMEMSZ2BLKSZ(allctr, umem_sz); + if (flags & CFLG_MAIN_CARRIER) { + ASSERT(flags & CFLG_MBC); + ASSERT(flags & CFLG_NO_CPOOL); + ASSERT(umem_sz == allctr->main_carrier_size); + ERTS_UNDEF(blk_sz, 0); + + if (allctr->main_carrier_size < allctr->min_mbc_size) + allctr->main_carrier_size = allctr->min_mbc_size; + crr_sz = bcrr_sz = allctr->main_carrier_size; + } + else { + ERTS_UNDEF(bcrr_sz, 0); + ERTS_UNDEF(crr_sz, 0); + blk_sz = UMEMSZ2BLKSZ(allctr, umem_sz); + } #ifdef ERTS_SMP allctr->cpool.disable_abandon = ERTS_ALC_CPOOL_MAX_DISABLE_ABANDON; @@ -3601,13 +3811,15 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) mseg_flags = ERTS_MSEG_FLG_NONE; } else { - crr_sz = (*allctr->get_next_mbc_size)(allctr); - if (crr_sz < MBC_HEADER_SIZE(allctr) + blk_sz) - crr_sz = MBC_HEADER_SIZE(allctr) + blk_sz; - mseg_flags = ERTS_MSEG_FLG_2POW; + if (!(flags & CFLG_MAIN_CARRIER)) { + crr_sz = (*allctr->get_next_mbc_size)(allctr); + if (crr_sz < MBC_HEADER_SIZE(allctr) + blk_sz) + crr_sz = MBC_HEADER_SIZE(allctr) + blk_sz; + } + mseg_flags = ERTS_MSEG_FLG_2POW; } - crr = (Carrier_t *) alcu_mseg_alloc(allctr, &crr_sz, mseg_flags); + crr = (Carrier_t *) allctr->mseg_alloc(allctr, &crr_sz, mseg_flags); if (!crr) { have_tried_mseg = 1; if (!(have_tried_sys_alloc || flags & CFLG_FORCE_MSEG)) @@ -3639,23 +3851,22 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) if (flags & CFLG_SBC) { bcrr_sz = blk_sz + SBC_HEADER_SIZE; } - else { + else if (!(flags & CFLG_MAIN_CARRIER)) { bcrr_sz = MBC_HEADER_SIZE(allctr) + blk_sz; - if (!(flags & CFLG_MAIN_CARRIER) - && bcrr_sz < allctr->smallest_mbc_size) - bcrr_sz = allctr->smallest_mbc_size; + if (bcrr_sz < allctr->smallest_mbc_size) + bcrr_sz = allctr->smallest_mbc_size; } crr_sz = (flags & CFLG_FORCE_SIZE ? UNIT_CEILING(bcrr_sz) : SYS_ALLOC_CARRIER_CEILING(bcrr_sz)); - crr = (Carrier_t *) alcu_sys_alloc(allctr, crr_sz, flags & CFLG_MBC); + crr = (Carrier_t *) allctr->sys_alloc(allctr, crr_sz, flags & CFLG_MBC); if (!crr) { if (crr_sz > UNIT_CEILING(bcrr_sz)) { crr_sz = UNIT_CEILING(bcrr_sz); - crr = (Carrier_t *) alcu_sys_alloc(allctr, crr_sz, flags & CFLG_MBC); + crr = (Carrier_t *) allctr->sys_alloc(allctr, crr_sz, flags & CFLG_MBC); } if (!crr) { #if HAVE_ERTS_MSEG @@ -3754,7 +3965,7 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) new_crr_sz = new_blk_sz + SBC_HEADER_SIZE; new_crr_sz = ERTS_SACRR_UNIT_CEILING(new_crr_sz); - new_crr = (Carrier_t *) alcu_mseg_realloc(allctr, + new_crr = (Carrier_t *) allctr->mseg_realloc(allctr, old_crr, old_crr_sz, &new_crr_sz); @@ -3769,11 +3980,6 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) DEBUG_SAVE_ALIGNMENT(new_crr); return new_blk; } -#if HALFWORD_HEAP - /* Old carrier unchanged; restore stat */ - STAT_MSEG_SBC_ALLOC(allctr, old_crr_sz, old_blk_sz); - return NULL; -#endif create_flags |= CFLG_FORCE_SYS_ALLOC; /* since mseg_realloc() failed */ } @@ -3784,7 +3990,7 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) (void *) BLK2UMEM(old_blk), MIN(new_blk_sz, old_blk_sz) - ABLK_HDR_SZ); unlink_carrier(&allctr->sbc_list, old_crr); - alcu_mseg_dealloc(allctr, old_crr, old_crr_sz, ERTS_MSEG_FLG_NONE); + allctr->mseg_dealloc(allctr, old_crr, old_crr_sz, ERTS_MSEG_FLG_NONE); } else { /* Old carrier unchanged; restore stat */ @@ -3801,7 +4007,7 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) ? UNIT_CEILING(new_bcrr_sz) : SYS_ALLOC_CARRIER_CEILING(new_bcrr_sz)); - new_crr = (Carrier_t *) alcu_sys_realloc(allctr, + new_crr = (Carrier_t *) allctr->sys_realloc(allctr, (void *) old_crr, new_crr_sz, old_crr_sz, @@ -3822,7 +4028,7 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) else if (new_crr_sz > UNIT_CEILING(new_bcrr_sz)) { new_crr_sz = new_blk_sz + SBC_HEADER_SIZE; new_crr_sz = UNIT_CEILING(new_crr_sz); - new_crr = (Carrier_t *) alcu_sys_realloc(allctr, + new_crr = (Carrier_t *) allctr->sys_realloc(allctr, (void *) old_crr, new_crr_sz, old_crr_sz, @@ -3845,7 +4051,7 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) (void *) BLK2UMEM(old_blk), MIN(new_blk_sz, old_blk_sz) - ABLK_HDR_SZ); unlink_carrier(&allctr->sbc_list, old_crr); - alcu_sys_free(allctr, old_crr, 0); + allctr->sys_dealloc(allctr, old_crr, CARRIER_SZ(old_crr), 0); } else { /* Old carrier unchanged; restore... */ @@ -3861,13 +4067,13 @@ dealloc_carrier(Allctr_t *allctr, Carrier_t *crr, int superaligned) { #if HAVE_ERTS_MSEG if (IS_MSEG_CARRIER(crr)) - alcu_mseg_dealloc(allctr, crr, CARRIER_SZ(crr), + allctr->mseg_dealloc(allctr, crr, CARRIER_SZ(crr), (superaligned ? ERTS_MSEG_FLG_2POW : ERTS_MSEG_FLG_NONE)); else #endif - alcu_sys_free(allctr, crr, superaligned); + allctr->sys_dealloc(allctr, crr, CARRIER_SZ(crr), superaligned); } static void @@ -3962,9 +4168,6 @@ static struct { Eterm e; Eterm t; Eterm ramv; -#if HALFWORD_HEAP - Eterm low; -#endif Eterm sbct; #if HAVE_ERTS_MSEG Eterm asbcst; @@ -4055,9 +4258,6 @@ init_atoms(Allctr_t *allctr) AM_INIT(e); AM_INIT(t); AM_INIT(ramv); -#if HALFWORD_HEAP - AM_INIT(low); -#endif AM_INIT(sbct); #if HAVE_ERTS_MSEG AM_INIT(asbcst); @@ -4663,9 +4863,6 @@ info_options(Allctr_t *allctr, "option e: true\n" "option t: %s\n" "option ramv: %s\n" -#if HALFWORD_HEAP - "option low: %s\n" -#endif "option sbct: %beu\n" #if HAVE_ERTS_MSEG "option asbcst: %bpu\n" @@ -4684,9 +4881,6 @@ info_options(Allctr_t *allctr, "option acul: %d\n", topt, allctr->ramv ? "true" : "false", -#if HALFWORD_HEAP - allctr->mseg_opt.low_mem ? "true" : "false", -#endif allctr->sbc_threshold, #if HAVE_ERTS_MSEG allctr->mseg_opt.abs_shrink_th, @@ -4749,9 +4943,6 @@ info_options(Allctr_t *allctr, add_2tup(hpp, szp, &res, am.sbct, bld_uint(hpp, szp, allctr->sbc_threshold)); -#if HALFWORD_HEAP - add_2tup(hpp, szp, &res, am.low, allctr->mseg_opt.low_mem ? am_true : am_false); -#endif add_2tup(hpp, szp, &res, am.ramv, allctr->ramv ? am_true : am_false); add_2tup(hpp, szp, &res, am.t, (allctr->t ? am_true : am_false)); add_2tup(hpp, szp, &res, am.e, am_true); @@ -5146,6 +5337,11 @@ do_erts_alcu_alloc(ErtsAlcType_t type, void *extra, Uint size) void *erts_alcu_alloc(ErtsAlcType_t type, void *extra, Uint size) { void *res; +#ifdef ERTS_SMP + ASSERT(!"This is not thread safe"); +#elif defined(USE_THREADS) + ASSERT(erts_equal_tids(erts_main_thread, erts_thr_self())); +#endif res = do_erts_alcu_alloc(type, extra, size); DEBUG_CHECK_ALIGNMENT(res); return res; @@ -5436,11 +5632,7 @@ do_erts_alcu_realloc(ErtsAlcType_t type, Block_t *new_blk; if(IS_SBC_BLK(blk)) { do_carrier_resize: -#if HALFWORD_HEAP - new_blk = resize_carrier(allctr, blk, size, CFLG_SBC | CFLG_FORCE_MSEG); -#else new_blk = resize_carrier(allctr, blk, size, CFLG_SBC); -#endif res = new_blk ? BLK2UMEM(new_blk) : NULL; } else if (alcu_flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE) @@ -5734,11 +5926,8 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) #ifdef ERTS_SMP if (init->tspec || init->tpref) allctr->mseg_opt.sched_spec = 1; -#endif -# if HALFWORD_HEAP - allctr->mseg_opt.low_mem = init->low_mem; -# endif -#endif +#endif /* ERTS_SMP */ +#endif /* HAVE_ERTS_MSEG */ allctr->name_prefix = init->name_prefix; if (!allctr->name_prefix) @@ -5887,17 +6076,52 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) + ABLK_HDR_SZ) - ABLK_HDR_SZ); + if (init->sys_alloc) { + ASSERT(init->sys_realloc && init->sys_dealloc); + allctr->sys_alloc = init->sys_alloc; + allctr->sys_realloc = init->sys_realloc; + allctr->sys_dealloc = init->sys_dealloc; + } + else { + ASSERT(!init->sys_realloc && !init->sys_dealloc); + allctr->sys_alloc = &erts_alcu_sys_alloc; + allctr->sys_realloc = &erts_alcu_sys_realloc; + allctr->sys_dealloc = &erts_alcu_sys_dealloc; + } +#if HAVE_ERTS_MSEG + if (init->mseg_alloc) { + ASSERT(init->mseg_realloc && init->mseg_dealloc); + allctr->mseg_alloc = init->mseg_alloc; + allctr->mseg_realloc = init->mseg_realloc; + allctr->mseg_dealloc = init->mseg_dealloc; + } + else { + ASSERT(!init->mseg_realloc && !init->mseg_dealloc); + allctr->mseg_alloc = &erts_alcu_mseg_alloc; + allctr->mseg_realloc = &erts_alcu_mseg_realloc; + allctr->mseg_dealloc = &erts_alcu_mseg_dealloc; + } + /* If a custom carrier alloc function is specified, make sure it's used */ + if (init->mseg_alloc && !init->sys_alloc) { + allctr->crr_set_flgs = CFLG_FORCE_MSEG; + allctr->crr_clr_flgs = CFLG_FORCE_SYS_ALLOC; + } + else if (!init->mseg_alloc && init->sys_alloc) { + allctr->crr_set_flgs = CFLG_FORCE_SYS_ALLOC; + allctr->crr_clr_flgs = CFLG_FORCE_MSEG; + } +#endif + if (allctr->main_carrier_size) { Block_t *blk; blk = create_carrier(allctr, allctr->main_carrier_size, - CFLG_MBC + (ERTS_SUPER_ALIGNED_MSEG_ONLY + ? CFLG_FORCE_MSEG : CFLG_FORCE_SYS_ALLOC) + | CFLG_MBC | CFLG_FORCE_SIZE | CFLG_NO_CPOOL -#if !HALFWORD_HEAP && !ERTS_SUPER_ALIGNED_MSEG_ONLY - | CFLG_FORCE_SYS_ALLOC -#endif | CFLG_MAIN_CARRIER); if (!blk) { #ifdef USE_THREADS diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index f4a2ae7ff3..b7d717ed23 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -24,6 +24,12 @@ #define ERTS_ALCU_VSN_STR "3.0" #include "erl_alloc_types.h" +#ifdef USE_THREADS +#define ERL_THREADS_EMU_INTERNAL__ +#include "erl_threads.h" +#endif + +#include "erl_mseg.h" #define ERTS_AU_PREF_ALLOC_BITS 11 #define ERTS_AU_MAX_PREF_ALLOC_INSTANCES (1 << ERTS_AU_PREF_ALLOC_BITS) @@ -45,7 +51,6 @@ typedef struct { int tspec; int tpref; int ramv; - int low_mem; /* HALFWORD only */ UWord sbct; UWord asbcst; UWord rsbcst; @@ -61,6 +66,15 @@ typedef struct { void *fix; size_t *fix_type_size; + +#if HAVE_ERTS_MSEG + void* (*mseg_alloc)(Allctr_t*, Uint *size_p, Uint flags); + void* (*mseg_realloc)(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p); + void (*mseg_dealloc)(Allctr_t*, void *seg, Uint size, Uint flags); +#endif + void* (*sys_alloc)(Allctr_t *allctr, Uint size, int superalign); + void* (*sys_realloc)(Allctr_t *allctr, void *ptr, Uint size, Uint old_size, int superalign); + void (*sys_dealloc)(Allctr_t *allctr, void *ptr, Uint size, int superalign); } AllctrInit_t; typedef struct { @@ -90,7 +104,6 @@ typedef struct { 0, /* (bool) tspec: thread specific */\ 0, /* (bool) tpref: thread preferred */\ 0, /* (bool) ramv: realloc always moves */\ - 0, /* (bool) low_mem: HALFWORD only */\ 512*1024, /* (bytes) sbct: sbc threshold */\ 2*1024*2024, /* (amount) asbcst: abs sbc shrink threshold */\ 20, /* (%) rsbcst: rel sbc shrink threshold */\ @@ -125,7 +138,6 @@ typedef struct { 0, /* (bool) tspec: thread specific */\ 0, /* (bool) tpref: thread preferred */\ 0, /* (bool) ramv: realloc always moves */\ - 0, /* (bool) low_mem: HALFWORD only */\ 64*1024, /* (bytes) sbct: sbc threshold */\ 2*1024*2024, /* (amount) asbcst: abs sbc shrink threshold */\ 20, /* (%) rsbcst: rel sbc shrink threshold */\ @@ -176,20 +188,44 @@ void erts_alcu_check_delayed_dealloc(Allctr_t *, int, int *, ErtsThrPrgrVal * #endif erts_aint32_t erts_alcu_fix_alloc_shrink(Allctr_t *, erts_aint32_t); +#ifdef ARCH_32 +extern UWord erts_literal_vspace_map[]; +# define ERTS_VSPACE_WORD_BITS (sizeof(UWord)*8) +#endif + +void* erts_alcu_mseg_alloc(Allctr_t*, Uint *size_p, Uint flags); +void* erts_alcu_mseg_realloc(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p); +void erts_alcu_mseg_dealloc(Allctr_t*, void *seg, Uint size, Uint flags); + +#if HAVE_ERTS_MSEG +# if defined(ARCH_32) +void* erts_alcu_literal_32_mseg_alloc(Allctr_t*, Uint *size_p, Uint flags); +void* erts_alcu_literal_32_mseg_realloc(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p); +void erts_alcu_literal_32_mseg_dealloc(Allctr_t*, void *seg, Uint size, Uint flags); + +# elif defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) +void* erts_alcu_literal_64_mseg_alloc(Allctr_t*, Uint *size_p, Uint flags); +void* erts_alcu_literal_64_mseg_realloc(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p); +void erts_alcu_literal_64_mseg_dealloc(Allctr_t*, void *seg, Uint size, Uint flags); +# endif +#endif /* HAVE_ERTS_MSEG */ + +void* erts_alcu_sys_alloc(Allctr_t*, Uint size, int superalign); +void* erts_alcu_sys_realloc(Allctr_t*, void *ptr, Uint size, Uint old_size, int superalign); +void erts_alcu_sys_dealloc(Allctr_t*, void *ptr, Uint size, int superalign); +#ifdef ARCH_32 +void* erts_alcu_literal_32_sys_alloc(Allctr_t*, Uint size, int superalign); +void* erts_alcu_literal_32_sys_realloc(Allctr_t*, void *ptr, Uint size, Uint old_size, int superalign); +void erts_alcu_literal_32_sys_dealloc(Allctr_t*, void *ptr, Uint size, int superalign); #endif +#endif /* !ERL_ALLOC_UTIL__ */ + #if defined(GET_ERL_ALLOC_UTIL_IMPL) && !defined(ERL_ALLOC_UTIL_IMPL__) #define ERL_ALLOC_UTIL_IMPL__ #define ERTS_ALCU_FLG_FAIL_REALLOC_MOVE (((Uint32) 1) << 0) -#ifdef USE_THREADS -#define ERL_THREADS_EMU_INTERNAL__ -#include "erl_threads.h" -#endif - -#include "erl_mseg.h" - #undef ERTS_ALLOC_UTIL_HARD_DEBUG #ifdef DEBUG # if 0 @@ -223,7 +259,7 @@ erts_aint32_t erts_alcu_fix_alloc_shrink(Allctr_t *, erts_aint32_t); #if ERTS_HAVE_MSEG_SUPER_ALIGNED \ || (!HAVE_ERTS_MSEG && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC) -# ifndef MSEG_ALIGN_BITS +# ifdef MSEG_ALIGN_BITS # define ERTS_SUPER_ALIGN_BITS MSEG_ALIGN_BITS # else # define ERTS_SUPER_ALIGN_BITS 18 @@ -501,6 +537,8 @@ struct Allctr_t_ { Uint min_mbc_size; Uint min_mbc_first_free_size; Uint min_block_size; + UWord crr_set_flgs; + UWord crr_clr_flgs; /* Carriers */ CarrierList_t mbc_list; @@ -546,6 +584,15 @@ struct Allctr_t_ { void (*remove_mbc) (Allctr_t *, Carrier_t *); UWord (*largest_fblk_in_mbc) (Allctr_t *, Carrier_t *); +#if HAVE_ERTS_MSEG + void* (*mseg_alloc)(Allctr_t*, Uint *size_p, Uint flags); + void* (*mseg_realloc)(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p); + void (*mseg_dealloc)(Allctr_t*, void *seg, Uint size, Uint flags); +#endif + void* (*sys_alloc)(Allctr_t *allctr, Uint size, int superalign); + void* (*sys_realloc)(Allctr_t *allctr, void *ptr, Uint size, Uint old_size, int superalign); + void (*sys_dealloc)(Allctr_t *allctr, void *ptr, Uint size, int superalign); + void (*init_atoms) (void); #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG diff --git a/erts/emulator/beam/erl_arith.c b/erts/emulator/beam/erl_arith.c index b8c5ef9b09..3671025d22 100644 --- a/erts/emulator/beam/erl_arith.c +++ b/erts/emulator/beam/erl_arith.c @@ -42,15 +42,8 @@ # define MAX(x, y) (((x) > (y)) ? (x) : (y)) #endif -#if !HEAP_ON_C_STACK -# define DECLARE_TMP(VariableName,N,P) \ - Eterm *VariableName = ((ERTS_PROC_GET_SCHDATA(P)->erl_arith_tmp_heap) + (2 * N)) -#else -# define DECLARE_TMP(VariableName,N,P) \ - Eterm VariableName[2] -#endif -# define ARG_IS_NOT_TMP(Arg,Tmp) ((Arg) != make_big((Tmp))) - +#define DECLARE_TMP(VariableName,N,P) Eterm VariableName[2] +#define ARG_IS_NOT_TMP(Arg,Tmp) ((Arg) != make_big((Tmp))) static Eterm shift(Process* p, Eterm arg1, Eterm arg2, int right); diff --git a/erts/emulator/beam/erl_async.c b/erts/emulator/beam/erl_async.c index f071898046..cdeeb5281b 100644 --- a/erts/emulator/beam/erl_async.c +++ b/erts/emulator/beam/erl_async.c @@ -167,7 +167,6 @@ async_ready_q(Uint sched_id) #endif - void erts_init_async(void) { @@ -401,13 +400,19 @@ static ERTS_INLINE void call_async_ready(ErtsAsync *a) ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP); #endif if (!p) { - if (a->async_free) + if (a->async_free) { + ERTS_MSACC_PUSH_AND_SET_STATE(ERTS_MSACC_STATE_PORT); a->async_free(a->async_data); + ERTS_MSACC_POP_STATE(); + } } else { if (async_ready(p, a->async_data)) { - if (a->async_free) + if (a->async_free) { + ERTS_MSACC_PUSH_AND_SET_STATE(ERTS_MSACC_STATE_PORT); a->async_free(a->async_data); + ERTS_MSACC_POP_STATE(); + } } #if ERTS_USE_ASYNC_READY_Q erts_port_release(p); @@ -461,6 +466,8 @@ static erts_tse_t *async_thread_init(ErtsAsyncQ *aq) { ErtsThrQInit_t qinit = ERTS_THR_Q_INIT_DEFAULT; erts_tse_t *tse = erts_tse_fetch(); + ERTS_DECLARE_DUMMY(Uint no); + #ifdef ERTS_SMP ErtsThrPrgrCallbacks callbacks; @@ -484,10 +491,12 @@ static erts_tse_t *async_thread_init(ErtsAsyncQ *aq) /* Inform main thread that we are done initializing... */ erts_mtx_lock(&async->init.data.mtx); - async->init.data.no_initialized++; + no = async->init.data.no_initialized++; erts_cnd_signal(&async->init.data.cnd); erts_mtx_unlock(&async->init.data.mtx); + erts_msacc_init_thread("async", no, 0); + return tse; } @@ -495,6 +504,7 @@ static void *async_main(void* arg) { ErtsAsyncQ *aq = (ErtsAsyncQ *) arg; erts_tse_t *tse = async_thread_init(aq); + ERTS_MSACC_DECLARE_CACHE(); while (1) { ErtsThrQPrepEnQ_t *prep_enq; @@ -502,11 +512,14 @@ static void *async_main(void* arg) if (is_nil(a->port)) break; /* Time to die */ + ERTS_MSACC_UPDATE_CACHE(); + #if ERTS_ASYNC_PRINT_JOB erts_fprintf(stderr, "<- %ld\n", a->async_id); #endif - + ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_PORT); a->async_invoke(a->async_data); + ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_OTHER); async_reply(a, prep_enq); } @@ -629,10 +642,13 @@ long driver_async(ErlDrvPort ix, unsigned int* key, unsigned int qix; #if ERTS_USE_ASYNC_READY_Q Uint sched_id; + ERTS_MSACC_PUSH_STATE(); sched_id = erts_get_scheduler_id(); if (!sched_id) sched_id = 1; +#else + ERTS_MSACC_PUSH_STATE(); #endif prt = erts_drvport2port(ix); @@ -685,12 +701,17 @@ long driver_async(ErlDrvPort ix, unsigned int* key, return id; } #endif - + + ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_PORT); (*a->async_invoke)(a->async_data); + ERTS_MSACC_POP_STATE(); if (async_ready(prt, a->async_data)) { - if (a->async_free != NULL) + if (a->async_free != NULL) { + ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_PORT); (*a->async_free)(a->async_data); + ERTS_MSACC_POP_STATE(); + } } erts_free(ERTS_ALC_T_ASYNC, (void *) a); diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c index 134aa2d396..aec72bd61a 100644 --- a/erts/emulator/beam/erl_bif_binary.c +++ b/erts/emulator/beam/erl_bif_binary.c @@ -55,10 +55,8 @@ /* Init and local variables */ -static Export binary_match_trap_export; -static BIF_RETTYPE binary_match_trap(BIF_ALIST_3); -static Export binary_matches_trap_export; -static BIF_RETTYPE binary_matches_trap(BIF_ALIST_3); +static Export binary_find_trap_export; +static BIF_RETTYPE binary_find_trap(BIF_ALIST_3); static Export binary_longest_prefix_trap_export; static BIF_RETTYPE binary_longest_prefix_trap(BIF_ALIST_3); static Export binary_longest_suffix_trap_export; @@ -70,19 +68,15 @@ static BIF_RETTYPE binary_copy_trap(BIF_ALIST_2); static Uint max_loop_limit; static BIF_RETTYPE -binary_match(Process *p, Eterm arg1, Eterm arg2, Eterm arg3); +binary_match(Process *p, Eterm arg1, Eterm arg2, Eterm arg3, Uint flags); static BIF_RETTYPE -binary_matches(Process *p, Eterm arg1, Eterm arg2, Eterm arg3); +binary_split(Process *p, Eterm arg1, Eterm arg2, Eterm arg3); void erts_init_bif_binary(void) { - erts_init_trap_export(&binary_match_trap_export, - am_erlang, am_binary_match_trap, 3, - &binary_match_trap); - - erts_init_trap_export(&binary_matches_trap_export, - am_erlang, am_binary_matches_trap, 3, - &binary_matches_trap); + erts_init_trap_export(&binary_find_trap_export, + am_erlang, am_binary_find_trap, 3, + &binary_find_trap); erts_init_trap_export(&binary_longest_prefix_trap_export, am_erlang, am_binary_longest_prefix_trap, 3, @@ -314,8 +308,8 @@ static BMData *create_bmdata(MyAllocator *my, byte *x, Uint len, /* * Aho Corasick - Build a Trie and fill in the failure functions * when all strings are added. - * The algorithm is nicely described by Dieter B�hler of University of - * T�bingen: + * The algorithm is nicely described by Dieter Bühler of University of + * Tübingen: * http://www-sr.informatik.uni-tuebingen.de/~buehler/AC/AC.html */ @@ -537,21 +531,23 @@ static void ac_init_find_all(ACFindAllState *state, ACTrie *act, Sint startpos, state->out = NULL; } -static void ac_restore_find_all(ACFindAllState *state, char *buff) +static void ac_restore_find_all(ACFindAllState *state, + const ACFindAllState *src) { - memcpy(state,buff,sizeof(ACFindAllState)); + memcpy(state, src, sizeof(ACFindAllState)); if (state->allocated > 0) { state->out = erts_alloc(ERTS_ALC_T_TMP, sizeof(FindallData) * (state->allocated)); - memcpy(state->out,buff+sizeof(ACFindAllState),sizeof(FindallData)*state->m); + memcpy(state->out, src+1, sizeof(FindallData)*state->m); } else { state->out = NULL; } } -static void ac_serialize_find_all(ACFindAllState *state, char *buff) +static void ac_serialize_find_all(const ACFindAllState *state, + ACFindAllState *dst) { - memcpy(buff,state,sizeof(ACFindAllState)); - memcpy(buff+sizeof(ACFindAllState),state->out,sizeof(FindallData)*state->m); + memcpy(dst, state, sizeof(ACFindAllState)); + memcpy(dst+1, state->out, sizeof(FindallData)*state->m); } static void ac_clean_find_all(ACFindAllState *state) @@ -565,9 +561,6 @@ static void ac_clean_find_all(ACFindAllState *state) #endif } -#define SIZEOF_AC_SERIALIZED_FIND_ALL_STATE(S) \ - (sizeof(ACFindAllState)+(sizeof(FindallData)*(S).m)) - /* * Differs to the find_first function in that it stores all matches and the values * arte returned only in the state. @@ -814,24 +807,24 @@ static void bm_init_find_all(BMFindAllState *state, Sint startpos, Uint len) state->out = NULL; } -static void bm_restore_find_all(BMFindAllState *state, char *buff) +static void bm_restore_find_all(BMFindAllState *state, + const BMFindAllState *src) { - memcpy(state,buff,sizeof(BMFindAllState)); + memcpy(state, src, sizeof(BMFindAllState)); if (state->allocated > 0) { state->out = erts_alloc(ERTS_ALC_T_TMP, sizeof(FindallData) * (state->allocated)); - memcpy(state->out,buff+sizeof(BMFindAllState), - sizeof(FindallData)*state->m); + memcpy(state->out, src+1, sizeof(FindallData)*state->m); } else { state->out = NULL; } } -static void bm_serialize_find_all(BMFindAllState *state, char *buff) +static void bm_serialize_find_all(const BMFindAllState *state, + BMFindAllState *dst) { - memcpy(buff,state,sizeof(BMFindAllState)); - memcpy(buff+sizeof(BMFindAllState),state->out, - sizeof(FindallData)*state->m); + memcpy(dst, state, sizeof(BMFindAllState)); + memcpy(dst+1, state->out, sizeof(FindallData)*state->m); } static void bm_clean_find_all(BMFindAllState *state) @@ -845,9 +838,6 @@ static void bm_clean_find_all(BMFindAllState *state) #endif } -#define SIZEOF_BM_SERIALIZED_FIND_ALL_STATE(S) \ - (sizeof(BMFindAllState)+(sizeof(FindallData)*(S).m)) - /* * Differs to the find_first function in that it stores all matches and the * values are returned only in the state. @@ -1030,283 +1020,142 @@ BIF_RETTYPE binary_compile_pattern_1(BIF_ALIST_1) #define DO_BIN_MATCH_BADARG -1 #define DO_BIN_MATCH_RESTART -2 -static int do_binary_match(Process *p, Eterm subject, Uint hsstart, Uint hsend, - Eterm type, Binary *bin, Eterm state_term, - Eterm *res_term) -{ - byte *bytes; - Uint bitoffs, bitsize; - byte *temp_alloc = NULL; - - ERTS_GET_BINARY_BYTES(subject, bytes, bitoffs, bitsize); - if (bitsize != 0) { - goto badarg; - } - if (bitoffs != 0) { - bytes = erts_get_aligned_binary_bytes(subject, &temp_alloc); - } - if (state_term != NIL) { - Eterm *ptr = big_val(state_term); - type = ptr[1]; - } - - if (type == am_bm) { - BMData *bm; - Sint pos; - Eterm ret; - Eterm *hp; - BMFindFirstState state; - Uint reds = get_reds(p, BM_LOOP_FACTOR); - Uint save_reds = reds; - - bm = (BMData *) ERTS_MAGIC_BIN_DATA(bin); -#ifdef HARDDEBUG - dump_bm_data(bm); -#endif - if (state_term == NIL) { - bm_init_find_first_match(&state, hsstart, hsend); - } else { - Eterm *ptr = big_val(state_term); - memcpy(&state,ptr+2,sizeof(state)); - } -#ifdef HARDDEBUG - erts_printf("(bm) state->pos = %ld, state->len = %lu\n",state.pos, - state.len); -#endif - pos = bm_find_first_match(&state, bm, bytes, &reds); - if (pos == BM_NOT_FOUND) { - ret = am_nomatch; - } else if (pos == BM_RESTART) { - int x = (sizeof(BMFindFirstState) / sizeof(Eterm)) + - !!(sizeof(BMFindFirstState) % sizeof(Eterm)); -#ifdef HARDDEBUG - erts_printf("Trap bm!\n"); -#endif - hp = HAlloc(p,x+2); - hp[0] = make_pos_bignum_header(x+1); - hp[1] = type; - memcpy(hp+2,&state,sizeof(state)); - *res_term = make_big(hp); - erts_free_aligned_binary_bytes(temp_alloc); - return DO_BIN_MATCH_RESTART; - } else { - Eterm erlen = erts_make_integer((Uint) bm->len, p); - ret = erts_make_integer(pos,p); - hp = HAlloc(p,3); - ret = TUPLE2(hp, ret, erlen); - } - erts_free_aligned_binary_bytes(temp_alloc); - BUMP_REDS(p, (save_reds - reds) / BM_LOOP_FACTOR); - *res_term = ret; - return DO_BIN_MATCH_OK; - } else if (type == am_ac) { - ACTrie *act; - Uint pos, rlen; - int acr; - ACFindFirstState state; - Eterm ret; - Eterm *hp; - Uint reds = get_reds(p, AC_LOOP_FACTOR); - Uint save_reds = reds; +#define BINARY_FIND_ALL 0x01 +#define BINARY_SPLIT_TRIM 0x02 +#define BINARY_SPLIT_TRIM_ALL 0x04 - act = (ACTrie *) ERTS_MAGIC_BIN_DATA(bin); -#ifdef HARDDEBUG - dump_ac_trie(act); -#endif - if (state_term == NIL) { - ac_init_find_first_match(&state, act, hsstart, hsend); - } else { - Eterm *ptr = big_val(state_term); - memcpy(&state,ptr+2,sizeof(state)); - } - acr = ac_find_first_match(&state, bytes, &pos, &rlen, &reds); - if (acr == AC_NOT_FOUND) { - ret = am_nomatch; - } else if (acr == AC_RESTART) { - int x = (sizeof(state) / sizeof(Eterm)) + - !!(sizeof(ACFindFirstState) % sizeof(Eterm)); -#ifdef HARDDEBUG - erts_printf("Trap ac!\n"); -#endif - hp = HAlloc(p,x+2); - hp[0] = make_pos_bignum_header(x+1); - hp[1] = type; - memcpy(hp+2,&state,sizeof(state)); - *res_term = make_big(hp); - erts_free_aligned_binary_bytes(temp_alloc); - return DO_BIN_MATCH_RESTART; - } else { - Eterm epos = erts_make_integer(pos,p); - Eterm erlen = erts_make_integer(rlen,p); - hp = HAlloc(p,3); - ret = TUPLE2(hp, epos, erlen); - } - erts_free_aligned_binary_bytes(temp_alloc); - BUMP_REDS(p, (save_reds - reds) / AC_LOOP_FACTOR); - *res_term = ret; - return DO_BIN_MATCH_OK; - } - badarg: - return DO_BIN_MATCH_BADARG; -} +typedef struct BinaryFindState { + Eterm type; + Uint flags; + Uint hsstart; + Uint hsend; + Eterm (*not_found_result) (Process *, Eterm, struct BinaryFindState *); + Eterm (*single_result) (Process *, Eterm, struct BinaryFindState *, Sint, Sint); + Eterm (*global_result) (Process *, Eterm, struct BinaryFindState *, FindallData *, Uint); +} BinaryFindState; + +typedef struct BinaryFindState_bignum { + Eterm bignum_hdr; + BinaryFindState bfs; + union { + BMFindFirstState bmffs; + BMFindAllState bmfas; + ACFindFirstState acffs; + ACFindAllState acfas; + } data; +} BinaryFindState_bignum; + +#define SIZEOF_BINARY_FIND_STATE(S) \ + (sizeof(BinaryFindState)+sizeof(S)) + +#define SIZEOF_BINARY_FIND_ALL_STATE(S) \ + (sizeof(BinaryFindState)+sizeof(S)+(sizeof(FindallData)*(S).m)) + +static Eterm do_match_not_found_result(Process *p, Eterm subject, BinaryFindState *bfs); +static Eterm do_match_single_result(Process *p, Eterm subject, BinaryFindState *bfs, + Sint pos, Sint len); +static Eterm do_match_global_result(Process *p, Eterm subject, BinaryFindState *bfs, + FindallData *fad, Uint fad_sz); +static Eterm do_split_not_found_result(Process *p, Eterm subject, BinaryFindState *bfs); +static Eterm do_split_single_result(Process *p, Eterm subject, BinaryFindState *bfs, + Sint pos, Sint len); +static Eterm do_split_global_result(Process *p, Eterm subject, BinaryFindState *bfs, + FindallData *fad, Uint fad_sz); -static int do_binary_matches(Process *p, Eterm subject, Uint hsstart, - Uint hsend, Eterm type, Binary *bin, - Eterm state_term, Eterm *res_term) +static int parse_match_opts_list(Eterm l, Eterm bin, Uint *posp, Uint *endp) { - byte *bytes; - Uint bitoffs, bitsize; - byte *temp_alloc = NULL; - - ERTS_GET_BINARY_BYTES(subject, bytes, bitoffs, bitsize); - if (bitsize != 0) { - goto badarg; - } - if (bitoffs != 0) { - bytes = erts_get_aligned_binary_bytes(subject, &temp_alloc); - } - if (state_term != NIL) { - Eterm *ptr = big_val(state_term); - type = ptr[1]; - } - - if (type == am_bm) { - BMData *bm; - Sint pos; - Eterm ret,tpl; - Eterm *hp; - BMFindAllState state; - Uint reds = get_reds(p, BM_LOOP_FACTOR); - Uint save_reds = reds; - - bm = (BMData *) ERTS_MAGIC_BIN_DATA(bin); -#ifdef HARDDEBUG - dump_bm_data(bm); -#endif - if (state_term == NIL) { - bm_init_find_all(&state, hsstart, hsend); - } else { - Eterm *ptr = big_val(state_term); - bm_restore_find_all(&state,(char *) (ptr+2)); - } - - pos = bm_find_all_non_overlapping(&state, bm, bytes, &reds); - if (pos == BM_NOT_FOUND) { - ret = NIL; - } else if (pos == BM_RESTART) { - int x = - (SIZEOF_BM_SERIALIZED_FIND_ALL_STATE(state) / sizeof(Eterm)) + - !!(SIZEOF_BM_SERIALIZED_FIND_ALL_STATE(state) % sizeof(Eterm)); -#ifdef HARDDEBUG - erts_printf("Trap bm!\n"); -#endif - hp = HAlloc(p,x+2); - hp[0] = make_pos_bignum_header(x+1); - hp[1] = type; - bm_serialize_find_all(&state, (char *) (hp+2)); - *res_term = make_big(hp); - erts_free_aligned_binary_bytes(temp_alloc); - bm_clean_find_all(&state); - return DO_BIN_MATCH_RESTART; - } else { - FindallData *fad = state.out; - int i; - for (i = 0; i < state.m; ++i) { - fad[i].epos = erts_make_integer(fad[i].pos,p); - fad[i].elen = erts_make_integer(fad[i].len,p); + Eterm *tp; + Uint pos; + Sint len; + if (l == THE_NON_VALUE || l == NIL) { + /* Invalid term or NIL, we're called from binary_match(es)_2 or + have no options*/ + *posp = 0; + *endp = binary_size(bin); + return 0; + } else if (is_list(l)) { + while(is_list(l)) { + Eterm t = CAR(list_val(l)); + Uint orig_size; + if (!is_tuple(t)) { + goto badarg; } - hp = HAlloc(p,state.m * (3 + 2)); - ret = NIL; - for (i = state.m - 1; i >= 0; --i) { - tpl = TUPLE2(hp, fad[i].epos, fad[i].elen); - hp +=3; - ret = CONS(hp,tpl,ret); - hp += 2; + tp = tuple_val(t); + if (arityval(*tp) != 2) { + goto badarg; } - } - erts_free_aligned_binary_bytes(temp_alloc); - bm_clean_find_all(&state); - BUMP_REDS(p, (save_reds - reds) / BM_LOOP_FACTOR); - *res_term = ret; - return DO_BIN_MATCH_OK; - } else if (type == am_ac) { - ACTrie *act; - int acr; - ACFindAllState state; - Eterm ret,tpl; - Eterm *hp; - Uint reds = get_reds(p, AC_LOOP_FACTOR); - Uint save_reds = reds; - - act = (ACTrie *) ERTS_MAGIC_BIN_DATA(bin); -#ifdef HARDDEBUG - dump_ac_trie(act); -#endif - if (state_term == NIL) { - ac_init_find_all(&state, act, hsstart, hsend); - } else { - Eterm *ptr = big_val(state_term); - ac_restore_find_all(&state,(char *) (ptr+2)); - } - acr = ac_find_all_non_overlapping(&state, bytes, &reds); - if (acr == AC_NOT_FOUND) { - ret = NIL; - } else if (acr == AC_RESTART) { - int x = - (SIZEOF_AC_SERIALIZED_FIND_ALL_STATE(state) / sizeof(Eterm)) + - !!(SIZEOF_AC_SERIALIZED_FIND_ALL_STATE(state) % sizeof(Eterm)); -#ifdef HARDDEBUG - erts_printf("Trap ac!\n"); -#endif - hp = HAlloc(p,x+2); - hp[0] = make_pos_bignum_header(x+1); - hp[1] = type; - ac_serialize_find_all(&state, (char *) (hp+2)); - *res_term = make_big(hp); - erts_free_aligned_binary_bytes(temp_alloc); - ac_clean_find_all(&state); - return DO_BIN_MATCH_RESTART; - } else { - FindallData *fad = state.out; - int i; - for (i = 0; i < state.m; ++i) { - fad[i].epos = erts_make_integer(fad[i].pos,p); - fad[i].elen = erts_make_integer(fad[i].len,p); + if (tp[1] != am_scope || is_not_tuple(tp[2])) { + goto badarg; } - hp = HAlloc(p,state.m * (3 + 2)); - ret = NIL; - for (i = state.m - 1; i >= 0; --i) { - tpl = TUPLE2(hp, fad[i].epos, fad[i].elen); - hp +=3; - ret = CONS(hp,tpl,ret); - hp += 2; + tp = tuple_val(tp[2]); + if (arityval(*tp) != 2) { + goto badarg; + } + if (!term_to_Uint(tp[1], &pos)) { + goto badarg; } + if (!term_to_Sint(tp[2], &len)) { + goto badarg; + } + if (len < 0) { + Uint lentmp = -(Uint)len; + /* overflow */ + if ((Sint)lentmp < 0) { + goto badarg; + } + len = lentmp; + pos -= len; + } + /* overflow */ + if ((pos + len) < pos || (len > 0 && (pos + len) == pos)) { + goto badarg; + } + *endp = len + pos; + *posp = pos; + if ((orig_size = binary_size(bin)) < pos || + orig_size < (*endp)) { + goto badarg; + } + l = CDR(list_val(l)); } - erts_free_aligned_binary_bytes(temp_alloc); - ac_clean_find_all(&state); - BUMP_REDS(p, (save_reds - reds) / AC_LOOP_FACTOR); - *res_term = ret; - return DO_BIN_MATCH_OK; + return 0; + } else { + badarg: + return 1; } - badarg: - return DO_BIN_MATCH_BADARG; } -static int parse_match_opts_list(Eterm l, Eterm bin, Uint *posp, Uint *endp) +static int parse_split_opts_list(Eterm l, Eterm bin, Uint *posp, Uint *endp, Uint *optp) { Eterm *tp; Uint pos; Sint len; - if (l == ((Eterm) 0) || l == NIL) { - /* Invalid term or NIL, we're called from binary_match(es)_2 or - have no options*/ - *posp = 0; - *endp = binary_size(bin); + *optp = 0; + *posp = 0; + *endp = binary_size(bin); + if (l == THE_NON_VALUE || l == NIL) { return 0; } else if (is_list(l)) { while(is_list(l)) { Eterm t = CAR(list_val(l)); Uint orig_size; + if (is_atom(t)) { + if (t == am_global) { + *optp |= BINARY_FIND_ALL; + l = CDR(list_val(l)); + continue; + } + if (t == am_trim) { + *optp |= BINARY_SPLIT_TRIM; + l = CDR(list_val(l)); + continue; + } + if (t == am_trim_all) { + *optp |= BINARY_SPLIT_TRIM_ALL; + l = CDR(list_val(l)); + continue; + } + } if (!is_tuple(t)) { goto badarg; } @@ -1355,48 +1204,207 @@ static int parse_match_opts_list(Eterm l, Eterm bin, Uint *posp, Uint *endp) } } -static BIF_RETTYPE binary_match_trap(BIF_ALIST_3) +static int do_binary_find(Process *p, Eterm subject, BinaryFindState *bfs, Binary *bin, + Eterm state_term, Eterm *res_term) { - int runres; - Eterm result; - Binary *bin = ((ProcBin *) binary_val(BIF_ARG_3))->val; - runres = do_binary_match(BIF_P,BIF_ARG_1,0,0,NIL,bin,BIF_ARG_2,&result); - if (runres == DO_BIN_MATCH_OK) { - BIF_RET(result); - } else { - BUMP_ALL_REDS(BIF_P); - BIF_TRAP3(&binary_match_trap_export, BIF_P, BIF_ARG_1, result, - BIF_ARG_3); + byte *bytes; + Uint bitoffs, bitsize; + byte *temp_alloc = NULL; + BinaryFindState_bignum *state_ptr = NULL; + + ERTS_GET_BINARY_BYTES(subject, bytes, bitoffs, bitsize); + if (bitsize != 0) { + goto badarg; + } + if (bitoffs != 0) { + bytes = erts_get_aligned_binary_bytes(subject, &temp_alloc); + } + if (state_term != NIL) { + state_ptr = (BinaryFindState_bignum *)(big_val(state_term)); + bfs = &(state_ptr->bfs); } -} -static BIF_RETTYPE binary_matches_trap(BIF_ALIST_3) -{ - int runres; - Eterm result; - Binary *bin = ((ProcBin *) binary_val(BIF_ARG_3))->val; - runres = do_binary_matches(BIF_P,BIF_ARG_1,0,0,NIL,bin,BIF_ARG_2,&result); - if (runres == DO_BIN_MATCH_OK) { - BIF_RET(result); + if (bfs->flags & BINARY_FIND_ALL) { + if (bfs->type == am_bm) { + BMData *bm; + Sint pos; + BMFindAllState state; + Uint reds = get_reds(p, BM_LOOP_FACTOR); + Uint save_reds = reds; + + bm = (BMData *) ERTS_MAGIC_BIN_DATA(bin); +#ifdef HARDDEBUG + dump_bm_data(bm); +#endif + if (state_term == NIL) { + bm_init_find_all(&state, bfs->hsstart, bfs->hsend); + } else { + bm_restore_find_all(&state, &(state_ptr->data.bmfas)); + } + + pos = bm_find_all_non_overlapping(&state, bm, bytes, &reds); + if (pos == BM_NOT_FOUND) { + *res_term = bfs->not_found_result(p, subject, bfs); + } else if (pos == BM_RESTART) { + int x = + (SIZEOF_BINARY_FIND_ALL_STATE(state) / sizeof(Eterm)) + + !!(SIZEOF_BINARY_FIND_ALL_STATE(state) % sizeof(Eterm)); +#ifdef HARDDEBUG + erts_printf("Trap bm!\n"); +#endif + state_ptr = (BinaryFindState_bignum*) HAlloc(p, x+1); + state_ptr->bignum_hdr = make_pos_bignum_header(x); + memcpy(&state_ptr->bfs, bfs, sizeof(BinaryFindState)); + bm_serialize_find_all(&state, &state_ptr->data.bmfas); + *res_term = make_big(&state_ptr->bignum_hdr); + erts_free_aligned_binary_bytes(temp_alloc); + bm_clean_find_all(&state); + return DO_BIN_MATCH_RESTART; + } else { + *res_term = bfs->global_result(p, subject, bfs, state.out, state.m); + } + erts_free_aligned_binary_bytes(temp_alloc); + bm_clean_find_all(&state); + BUMP_REDS(p, (save_reds - reds) / BM_LOOP_FACTOR); + return DO_BIN_MATCH_OK; + } else if (bfs->type == am_ac) { + ACTrie *act; + int acr; + ACFindAllState state; + Uint reds = get_reds(p, AC_LOOP_FACTOR); + Uint save_reds = reds; + + act = (ACTrie *) ERTS_MAGIC_BIN_DATA(bin); +#ifdef HARDDEBUG + dump_ac_trie(act); +#endif + if (state_term == NIL) { + ac_init_find_all(&state, act, bfs->hsstart, bfs->hsend); + } else { + ac_restore_find_all(&state, &(state_ptr->data.acfas)); + } + acr = ac_find_all_non_overlapping(&state, bytes, &reds); + if (acr == AC_NOT_FOUND) { + *res_term = bfs->not_found_result(p, subject, bfs); + } else if (acr == AC_RESTART) { + int x = + (SIZEOF_BINARY_FIND_ALL_STATE(state) / sizeof(Eterm)) + + !!(SIZEOF_BINARY_FIND_ALL_STATE(state) % sizeof(Eterm)); +#ifdef HARDDEBUG + erts_printf("Trap ac!\n"); +#endif + state_ptr = (BinaryFindState_bignum*) HAlloc(p, x+1); + state_ptr->bignum_hdr = make_pos_bignum_header(x); + memcpy(&state_ptr->bfs, bfs, sizeof(BinaryFindState)); + ac_serialize_find_all(&state, &state_ptr->data.acfas); + *res_term = make_big(&state_ptr->bignum_hdr); + erts_free_aligned_binary_bytes(temp_alloc); + ac_clean_find_all(&state); + return DO_BIN_MATCH_RESTART; + } else { + *res_term = bfs->global_result(p, subject, bfs, state.out, state.m); + } + erts_free_aligned_binary_bytes(temp_alloc); + ac_clean_find_all(&state); + BUMP_REDS(p, (save_reds - reds) / AC_LOOP_FACTOR); + return DO_BIN_MATCH_OK; + } } else { - BUMP_ALL_REDS(BIF_P); - BIF_TRAP3(&binary_matches_trap_export, BIF_P, BIF_ARG_1, result, - BIF_ARG_3); - } -} + if (bfs->type == am_bm) { + BMData *bm; + Sint pos; + BMFindFirstState state; + Uint reds = get_reds(p, BM_LOOP_FACTOR); + Uint save_reds = reds; + + bm = (BMData *) ERTS_MAGIC_BIN_DATA(bin); +#ifdef HARDDEBUG + dump_bm_data(bm); +#endif + if (state_term == NIL) { + bm_init_find_first_match(&state, bfs->hsstart, bfs->hsend); + } else { + memcpy(&state, &state_ptr->data.bmffs, sizeof(BMFindFirstState)); + } -BIF_RETTYPE binary_match_3(BIF_ALIST_3) -{ - return binary_match(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); +#ifdef HARDDEBUG + erts_printf("(bm) state->pos = %ld, state->len = %lu\n",state.pos, + state.len); +#endif + pos = bm_find_first_match(&state, bm, bytes, &reds); + if (pos == BM_NOT_FOUND) { + *res_term = bfs->not_found_result(p, subject, bfs); + } else if (pos == BM_RESTART) { + int x = + (SIZEOF_BINARY_FIND_STATE(state) / sizeof(Eterm)) + + !!(SIZEOF_BINARY_FIND_STATE(state) % sizeof(Eterm)); +#ifdef HARDDEBUG + erts_printf("Trap bm!\n"); +#endif + state_ptr = (BinaryFindState_bignum*) HAlloc(p, x+1); + state_ptr->bignum_hdr = make_pos_bignum_header(x); + memcpy(&state_ptr->bfs, bfs, sizeof(BinaryFindState)); + memcpy(&state_ptr->data.acffs, &state, sizeof(BMFindFirstState)); + *res_term = make_big(&state_ptr->bignum_hdr); + erts_free_aligned_binary_bytes(temp_alloc); + return DO_BIN_MATCH_RESTART; + } else { + *res_term = bfs->single_result(p, subject, bfs, pos, bm->len); + } + erts_free_aligned_binary_bytes(temp_alloc); + BUMP_REDS(p, (save_reds - reds) / BM_LOOP_FACTOR); + return DO_BIN_MATCH_OK; + } else if (bfs->type == am_ac) { + ACTrie *act; + Uint pos, rlen; + int acr; + ACFindFirstState state; + Uint reds = get_reds(p, AC_LOOP_FACTOR); + Uint save_reds = reds; + + act = (ACTrie *) ERTS_MAGIC_BIN_DATA(bin); +#ifdef HARDDEBUG + dump_ac_trie(act); +#endif + if (state_term == NIL) { + ac_init_find_first_match(&state, act, bfs->hsstart, bfs->hsend); + } else { + memcpy(&state, &state_ptr->data.acffs, sizeof(ACFindFirstState)); + } + acr = ac_find_first_match(&state, bytes, &pos, &rlen, &reds); + if (acr == AC_NOT_FOUND) { + *res_term = bfs->not_found_result(p, subject, bfs); + } else if (acr == AC_RESTART) { + int x = + (SIZEOF_BINARY_FIND_STATE(state) / sizeof(Eterm)) + + !!(SIZEOF_BINARY_FIND_STATE(state) % sizeof(Eterm)); +#ifdef HARDDEBUG + erts_printf("Trap ac!\n"); +#endif + state_ptr = (BinaryFindState_bignum*) HAlloc(p, x+1); + state_ptr->bignum_hdr = make_pos_bignum_header(x); + memcpy(&state_ptr->bfs, bfs, sizeof(BinaryFindState)); + memcpy(&state_ptr->data.acffs, &state, sizeof(ACFindFirstState)); + *res_term = make_big(&state_ptr->bignum_hdr); + erts_free_aligned_binary_bytes(temp_alloc); + return DO_BIN_MATCH_RESTART; + } else { + *res_term = bfs->single_result(p, subject, bfs, pos, rlen); + } + erts_free_aligned_binary_bytes(temp_alloc); + BUMP_REDS(p, (save_reds - reds) / AC_LOOP_FACTOR); + return DO_BIN_MATCH_OK; + } + } + badarg: + return DO_BIN_MATCH_BADARG; } static BIF_RETTYPE -binary_match(Process *p, Eterm arg1, Eterm arg2, Eterm arg3) +binary_match(Process *p, Eterm arg1, Eterm arg2, Eterm arg3, Uint flags) { - Uint hsstart; - Uint hsend; + BinaryFindState bfs; Eterm *tp; - Eterm type; Binary *bin; Eterm bin_term = NIL; int runres; @@ -1405,11 +1413,12 @@ binary_match(Process *p, Eterm arg1, Eterm arg2, Eterm arg3) if (is_not_binary(arg1)) { goto badarg; } - if (parse_match_opts_list(arg3,arg1,&hsstart,&hsend)) { + bfs.flags = flags; + if (parse_match_opts_list(arg3, arg1, &(bfs.hsstart), &(bfs.hsend))) { goto badarg; } - if (hsend == 0) { - BIF_RET(am_nomatch); + if (bfs.hsend == 0) { + BIF_RET(do_match_not_found_result(p, arg1, &bfs)); } if (is_tuple(arg2)) { tp = tuple_val(arg2); @@ -1420,21 +1429,24 @@ binary_match(Process *p, Eterm arg1, Eterm arg2, Eterm arg3) !ERTS_TERM_IS_MAGIC_BINARY(tp[2])) { goto badarg; } - type = tp[1]; + bfs.type = tp[1]; bin = ((ProcBin *) binary_val(tp[2]))->val; - if (type == am_bm && + if (bfs.type == am_bm && ERTS_MAGIC_BIN_DESTRUCTOR(bin) != cleanup_my_data_bm) { goto badarg; } - if (type == am_ac && + if (bfs.type == am_ac && ERTS_MAGIC_BIN_DESTRUCTOR(bin) != cleanup_my_data_ac) { goto badarg; } bin_term = tp[2]; - } else if (do_binary_match_compile(arg2,&type,&bin)) { + } else if (do_binary_match_compile(arg2, &(bfs.type), &bin)) { goto badarg; } - runres = do_binary_match(p,arg1,hsstart,hsend,type,bin,NIL,&result); + bfs.not_found_result = &do_match_not_found_result; + bfs.single_result = &do_match_single_result; + bfs.global_result = &do_match_global_result; + runres = do_binary_find(p, arg1, &bfs, bin, NIL, &result); if (runres == DO_BIN_MATCH_RESTART && bin_term == NIL) { Eterm *hp = HAlloc(p, PROC_BIN_SIZE); bin_term = erts_mk_magic_binary_term(&hp, &MSO(p), bin); @@ -1446,7 +1458,7 @@ binary_match(Process *p, Eterm arg1, Eterm arg2, Eterm arg3) BIF_RET(result); case DO_BIN_MATCH_RESTART: BUMP_ALL_REDS(p); - BIF_TRAP3(&binary_match_trap_export, p, arg1, result, bin_term); + BIF_TRAP3(&binary_find_trap_export, p, arg1, result, bin_term); default: goto badarg; } @@ -1454,17 +1466,31 @@ binary_match(Process *p, Eterm arg1, Eterm arg2, Eterm arg3) BIF_ERROR(p,BADARG); } +BIF_RETTYPE binary_match_2(BIF_ALIST_2) +{ + return binary_match(BIF_P, BIF_ARG_1, BIF_ARG_2, THE_NON_VALUE, 0); +} + +BIF_RETTYPE binary_match_3(BIF_ALIST_3) +{ + return binary_match(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, 0); +} + +BIF_RETTYPE binary_matches_2(BIF_ALIST_2) +{ + return binary_match(BIF_P, BIF_ARG_1, BIF_ARG_2, THE_NON_VALUE, BINARY_FIND_ALL); +} + BIF_RETTYPE binary_matches_3(BIF_ALIST_3) { - return binary_matches(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); + return binary_match(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, BINARY_FIND_ALL); } static BIF_RETTYPE -binary_matches(Process *p, Eterm arg1, Eterm arg2, Eterm arg3) +binary_split(Process *p, Eterm arg1, Eterm arg2, Eterm arg3) { - Uint hsstart, hsend; + BinaryFindState bfs; Eterm *tp; - Eterm type; Binary *bin; Eterm bin_term = NIL; int runres; @@ -1473,11 +1499,12 @@ binary_matches(Process *p, Eterm arg1, Eterm arg2, Eterm arg3) if (is_not_binary(arg1)) { goto badarg; } - if (parse_match_opts_list(arg3,arg1,&hsstart,&hsend)) { + if (parse_split_opts_list(arg3, arg1, &(bfs.hsstart), &(bfs.hsend), &(bfs.flags))) { goto badarg; } - if (hsend == 0) { - BIF_RET(NIL); + if (bfs.hsend == 0) { + result = do_split_not_found_result(p, arg1, &bfs); + BIF_RET(result); } if (is_tuple(arg2)) { tp = tuple_val(arg2); @@ -1488,54 +1515,267 @@ binary_matches(Process *p, Eterm arg1, Eterm arg2, Eterm arg3) !ERTS_TERM_IS_MAGIC_BINARY(tp[2])) { goto badarg; } - type = tp[1]; + bfs.type = tp[1]; bin = ((ProcBin *) binary_val(tp[2]))->val; - if (type == am_bm && + if (bfs.type == am_bm && ERTS_MAGIC_BIN_DESTRUCTOR(bin) != cleanup_my_data_bm) { goto badarg; } - if (type == am_ac && + if (bfs.type == am_ac && ERTS_MAGIC_BIN_DESTRUCTOR(bin) != cleanup_my_data_ac) { goto badarg; } bin_term = tp[2]; - } else if (do_binary_match_compile(arg2,&type,&bin)) { + } else if (do_binary_match_compile(arg2, &(bfs.type), &bin)) { goto badarg; } - runres = do_binary_matches(p,arg1,hsstart,hsend,type,bin, - NIL,&result); + bfs.not_found_result = &do_split_not_found_result; + bfs.single_result = &do_split_single_result; + bfs.global_result = &do_split_global_result; + runres = do_binary_find(p, arg1, &bfs, bin, NIL, &result); if (runres == DO_BIN_MATCH_RESTART && bin_term == NIL) { Eterm *hp = HAlloc(p, PROC_BIN_SIZE); bin_term = erts_mk_magic_binary_term(&hp, &MSO(p), bin); } else if (bin_term == NIL) { erts_bin_free(bin); } - switch (runres) { + switch(runres) { case DO_BIN_MATCH_OK: BIF_RET(result); case DO_BIN_MATCH_RESTART: - BUMP_ALL_REDS(p); - BIF_TRAP3(&binary_matches_trap_export, p, arg1, result, - bin_term); + BIF_TRAP3(&binary_find_trap_export, p, arg1, result, bin_term); default: goto badarg; } badarg: - BIF_ERROR(p,BADARG); + BIF_ERROR(p, BADARG); } +BIF_RETTYPE binary_split_2(BIF_ALIST_2) +{ + return binary_split(BIF_P, BIF_ARG_1, BIF_ARG_2, THE_NON_VALUE); +} -BIF_RETTYPE binary_match_2(BIF_ALIST_2) +BIF_RETTYPE binary_split_3(BIF_ALIST_3) { - return binary_match(BIF_P,BIF_ARG_1,BIF_ARG_2,((Eterm) 0)); + return binary_split(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); +} + +static Eterm do_match_not_found_result(Process *p, Eterm subject, BinaryFindState *bfs) +{ + if (bfs->flags & BINARY_FIND_ALL) { + return NIL; + } else { + return am_nomatch; + } } +static Eterm do_match_single_result(Process *p, Eterm subject, BinaryFindState *bfs, + Sint pos, Sint len) +{ + Eterm erlen; + Eterm *hp; + Eterm ret; -BIF_RETTYPE binary_matches_2(BIF_ALIST_2) + erlen = erts_make_integer((Uint)(len), p); + ret = erts_make_integer(pos, p); + hp = HAlloc(p, 3); + ret = TUPLE2(hp, ret, erlen); + + return ret; +} + +static Eterm do_match_global_result(Process *p, Eterm subject, BinaryFindState *bfs, + FindallData *fad, Uint fad_sz) +{ + Sint i; + Eterm tpl; + Eterm *hp; + Eterm ret; + + for (i = 0; i < fad_sz; ++i) { + fad[i].epos = erts_make_integer(fad[i].pos, p); + fad[i].elen = erts_make_integer(fad[i].len, p); + } + hp = HAlloc(p, fad_sz * (3 + 2)); + ret = NIL; + for (i = fad_sz - 1; i >= 0; --i) { + tpl = TUPLE2(hp, fad[i].epos, fad[i].elen); + hp += 3; + ret = CONS(hp, tpl, ret); + hp += 2; + } + + return ret; +} + +static Eterm do_split_not_found_result(Process *p, Eterm subject, BinaryFindState *bfs) { - return binary_matches(BIF_P,BIF_ARG_1,BIF_ARG_2,((Eterm) 0)); + Eterm *hp; + Eterm ret; + + if (bfs->flags & (BINARY_SPLIT_TRIM | BINARY_SPLIT_TRIM_ALL) + && binary_size(subject) == 0) { + return NIL; + } + hp = HAlloc(p, 2); + ret = CONS(hp, subject, NIL); + + return ret; } +static Eterm do_split_single_result(Process *p, Eterm subject, BinaryFindState *bfs, + Sint pos, Sint len) +{ + size_t orig_size; + Eterm orig; + Uint offset; + Uint bit_offset; + Uint bit_size; + ErlSubBin *sb1; + ErlSubBin *sb2; + Eterm *hp; + Eterm ret; + + orig_size = binary_size(subject); + + if ((bfs->flags & (BINARY_SPLIT_TRIM | BINARY_SPLIT_TRIM_ALL)) && + (orig_size - pos - len) == 0) { + if (pos == 0) { + ret = NIL; + } else { + hp = HAlloc(p, (ERL_SUB_BIN_SIZE + 2)); + ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size); + sb1 = (ErlSubBin *) hp; + sb1->thing_word = HEADER_SUB_BIN; + sb1->size = pos; + sb1->offs = offset; + sb1->orig = orig; + sb1->bitoffs = bit_offset; + sb1->bitsize = bit_size; + sb1->is_writable = 0; + hp += ERL_SUB_BIN_SIZE; + + ret = CONS(hp, make_binary(sb1), NIL); + hp += 2; + } + } else { + if ((bfs->flags & BINARY_SPLIT_TRIM_ALL) && (pos == 0)) { + hp = HAlloc(p, 1 * (ERL_SUB_BIN_SIZE + 2)); + ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size); + sb1 = NULL; + } else { + hp = HAlloc(p, 2 * (ERL_SUB_BIN_SIZE + 2)); + ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size); + sb1 = (ErlSubBin *) hp; + sb1->thing_word = HEADER_SUB_BIN; + sb1->size = pos; + sb1->offs = offset; + sb1->orig = orig; + sb1->bitoffs = bit_offset; + sb1->bitsize = 0; + sb1->is_writable = 0; + hp += ERL_SUB_BIN_SIZE; + } + + sb2 = (ErlSubBin *) hp; + sb2->thing_word = HEADER_SUB_BIN; + sb2->size = orig_size - pos - len; + sb2->offs = offset + pos + len; + sb2->orig = orig; + sb2->bitoffs = bit_offset; + sb2->bitsize = bit_size; + sb2->is_writable = 0; + hp += ERL_SUB_BIN_SIZE; + + ret = CONS(hp, make_binary(sb2), NIL); + hp += 2; + if (sb1 != NULL) { + ret = CONS(hp, make_binary(sb1), ret); + hp += 2; + } + } + return ret; +} + +static Eterm do_split_global_result(Process *p, Eterm subject, BinaryFindState *bfs, + FindallData *fad, Uint fad_sz) +{ + size_t orig_size; + Eterm orig; + Uint offset; + Uint bit_offset; + Uint bit_size; + ErlSubBin *sb; + Sint i; + Sint tail; + Uint list_size; + Uint end_pos; + Uint do_trim = bfs->flags & (BINARY_SPLIT_TRIM | BINARY_SPLIT_TRIM_ALL); + Eterm *hp; + Eterm *hendp; + Eterm ret; + + tail = fad_sz - 1; + list_size = fad_sz + 1; + orig_size = binary_size(subject); + end_pos = (Uint)(orig_size); + + hp = HAlloc(p, list_size * (ERL_SUB_BIN_SIZE + 2)); + hendp = hp + list_size * (ERL_SUB_BIN_SIZE + 2); + ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size); + ASSERT(bit_size == 0); + + ret = NIL; + + for (i = tail; i >= 0; --i) { + sb = (ErlSubBin *)(hp); + sb->size = end_pos - (fad[i].pos + fad[i].len); + if (!(sb->size == 0 && do_trim)) { + sb->thing_word = HEADER_SUB_BIN; + sb->offs = offset + fad[i].pos + fad[i].len; + sb->orig = orig; + sb->bitoffs = bit_offset; + sb->bitsize = 0; + sb->is_writable = 0; + hp += ERL_SUB_BIN_SIZE; + ret = CONS(hp, make_binary(sb), ret); + hp += 2; + do_trim &= ~BINARY_SPLIT_TRIM; + } + end_pos = fad[i].pos; + } + + sb = (ErlSubBin *)(hp); + sb->size = fad[0].pos; + if (!(sb->size == 0 && do_trim)) { + sb->thing_word = HEADER_SUB_BIN; + sb->offs = offset; + sb->orig = orig; + sb->bitoffs = bit_offset; + sb->bitsize = 0; + sb->is_writable = 0; + hp += ERL_SUB_BIN_SIZE; + ret = CONS(hp, make_binary(sb), ret); + hp += 2; + } + HRelease(p, hendp, hp); + return ret; +} + +static BIF_RETTYPE binary_find_trap(BIF_ALIST_3) +{ + int runres; + Eterm result; + Binary *bin = ((ProcBin *) binary_val(BIF_ARG_3))->val; + runres = do_binary_find(BIF_P, BIF_ARG_1, NULL, bin, BIF_ARG_2, &result); + if (runres == DO_BIN_MATCH_OK) { + BIF_RET(result); + } else { + BUMP_ALL_REDS(BIF_P); + BIF_TRAP3(&binary_find_trap_export, BIF_P, BIF_ARG_1, result, BIF_ARG_3); + } +} BIF_RETTYPE erts_binary_part(Process *p, Eterm binary, Eterm epos, Eterm elen) { @@ -2550,7 +2790,6 @@ BIF_RETTYPE binary_referenced_byte_size_1(BIF_ALIST_1) } pb = (ProcBin *) binary_val(bin); if (pb->thing_word == HEADER_PROC_BIN) { - /* XXX:PaN - Halfword - orig_size is a long, we should handle that */ res = erts_make_integer((Uint) pb->val->orig_size, BIF_P); } else { /* heap binary */ res = erts_make_integer((Uint) ((ErlHeapBin *) pb)->size, BIF_P); @@ -2568,7 +2807,7 @@ BIF_RETTYPE binary_referenced_byte_size_1(BIF_ALIST_1) #endif static int get_need(Uint u) { -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) if (u > 0xFFFFFFFFUL) { if (u > 0xFFFFFFFFFFFFUL) { if (u > 0xFFFFFFFFFFFFFFUL) { diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c index 28bec6325c..2b1d875bfe 100644 --- a/erts/emulator/beam/erl_bif_ddll.c +++ b/erts/emulator/beam/erl_bif_ddll.c @@ -1707,18 +1707,19 @@ static void notify_proc(Process *proc, Eterm ref, Eterm driver_name, Eterm type, Eterm mess; Eterm r; Eterm *hp; - ErlHeapFragment *bp; - ErlOffHeap *ohp; + ErtsMessage *mp; ErtsProcLocks rp_locks = 0; + ErlOffHeap *ohp; ERTS_SMP_CHK_NO_PROC_LOCKS; assert_drv_list_rwlocked(); if (errcode != 0) { int need = load_error_need(errcode); Eterm e; - hp = erts_alloc_message_heap(6 /* tuple */ + 3 /* Error tuple */ + - REF_THING_SIZE + need, &bp, &ohp, - proc, &rp_locks); + mp = erts_alloc_message_heap(proc, &rp_locks, + (6 /* tuple */ + 3 /* Error tuple */ + + REF_THING_SIZE + need), + &hp, &ohp); r = copy_ref(ref,hp); hp += REF_THING_SIZE; e = build_load_error_hp(hp, errcode); @@ -1727,12 +1728,14 @@ static void notify_proc(Process *proc, Eterm ref, Eterm driver_name, Eterm type, hp += 3; mess = TUPLE5(hp,type,r,am_driver,driver_name,mess); } else { - hp = erts_alloc_message_heap(6 /* tuple */ + REF_THING_SIZE, &bp, &ohp, proc, &rp_locks); + mp = erts_alloc_message_heap(proc, &rp_locks, + 6 /* tuple */ + REF_THING_SIZE, + &hp, &ohp); r = copy_ref(ref,hp); hp += REF_THING_SIZE; mess = TUPLE5(hp,type,r,am_driver,driver_name,tag); } - erts_queue_message(proc, &rp_locks, bp, mess, am_undefined); + erts_queue_message(proc, &rp_locks, mp, mess, am_undefined); erts_smp_proc_unlock(proc, rp_locks); ERTS_SMP_CHK_NO_PROC_LOCKS; } diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 8bf6877bea..e4baff87f4 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -64,6 +64,7 @@ static Export* alloc_sizes_trap = NULL; static Export* gather_io_bytes_trap = NULL; static Export *gather_sched_wall_time_res_trap; +static Export *gather_msacc_res_trap; static Export *gather_gc_info_res_trap; static Export *gather_system_check_res_trap; @@ -74,9 +75,6 @@ static char otp_version[] = ERLANG_OTP_VERSION; static char erts_system_version[] = ("Erlang/OTP " ERLANG_OTP_RELEASE "%s" " [erts-" ERLANG_VERSION "]" -#if !HEAP_ON_C_STACK && !HALFWORD_HEAP - " [no-c-stack-objects]" -#endif #ifndef OTP_RELEASE #ifdef ERLANG_GIT_VERSION " [source-" ERLANG_GIT_VERSION "]" @@ -85,12 +83,8 @@ static char erts_system_version[] = ("Erlang/OTP " ERLANG_OTP_RELEASE #endif #endif #ifdef ARCH_64 -#if HALFWORD_HEAP - " [64-bit halfword]" -#else " [64-bit]" #endif -#endif #ifdef ERTS_SMP " [smp:%beu:%beu]" #endif @@ -138,6 +132,9 @@ static char erts_system_version[] = ("Erlang/OTP " ERLANG_OTP_RELEASE #ifdef USE_SYSTEMTAP " [systemtap]" #endif +#ifdef SHCOPY + " [sharing-preserving]" +#endif "\n"); #define ASIZE(a) (sizeof(a)/sizeof(a[0])) @@ -597,6 +594,8 @@ static Eterm pi_args[] = { am_min_bin_vheap_size, am_current_location, am_current_stacktrace, + am_message_queue_data, + am_garbage_collection_info }; #define ERTS_PI_ARGS ((int) (sizeof(pi_args)/sizeof(Eterm))) @@ -644,6 +643,8 @@ pi_arg2ix(Eterm arg) case am_min_bin_vheap_size: return 28; case am_current_location: return 29; case am_current_stacktrace: return 30; + case am_message_queue_data: return 31; + case am_garbage_collection_info: return 32; default: return -1; } } @@ -672,18 +673,12 @@ static Eterm pi_1_keys[] = { #define ERTS_PI_1_NO_OF_KEYS (sizeof(pi_1_keys)/sizeof(Eterm)) static Eterm pi_1_keys_list; -#if HEAP_ON_C_STACK static Eterm pi_1_keys_list_heap[2*ERTS_PI_1_NO_OF_KEYS]; -#endif static void process_info_init(void) { -#if HEAP_ON_C_STACK Eterm *hp = &pi_1_keys_list_heap[0]; -#else - Eterm *hp = erts_alloc(ERTS_ALC_T_LL_TEMP_TERM,sizeof(Eterm)*2*ERTS_PI_1_NO_OF_KEYS); -#endif int i; pi_1_keys_list = NIL; @@ -732,9 +727,10 @@ pi_pid2proc(Process *c_p, Eterm pid, ErtsProcLocks info_locks) -BIF_RETTYPE +static BIF_RETTYPE process_info_aux(Process *BIF_P, Process *rp, + ErtsProcLocks rp_locks, Eterm rpid, Eterm item, int always_wrap); @@ -825,10 +821,31 @@ process_info_list(Process *c_p, Eterm pid, Eterm list, int always_wrap, *fail_type = ERTS_PI_FAIL_TYPE_AWAIT_EXIT; goto done; } - else if (!(locks & ERTS_PROC_LOCK_STATUS)) { - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); + else { + ErtsProcLocks unlock_locks = 0; + + if (c_p == rp) + locks |= ERTS_PROC_LOCK_MAIN; + + if (!(locks & ERTS_PROC_LOCK_STATUS)) + unlock_locks |= ERTS_PROC_LOCK_STATUS; + + if (locks & ERTS_PROC_LOCK_MSGQ) { + /* + * Move in queue into private queue and + * release msgq lock, enabling others to + * send messages to the process while it + * is being inspected... + */ + ASSERT(locks & ERTS_PROC_LOCK_MAIN); + ERTS_SMP_MSGQ_MV_INQ2PRIVQ(rp); + locks &= ~ERTS_PROC_LOCK_MSGQ; + unlock_locks |= ERTS_PROC_LOCK_MSGQ; + } + + if (unlock_locks) + erts_smp_proc_unlock(rp, unlock_locks); } - /* * We always handle 'messages' first if it should be part @@ -840,7 +857,7 @@ process_info_list(Process *c_p, Eterm pid, Eterm list, int always_wrap, if (want_messages) { ix = pi_arg2ix(am_messages); ASSERT(part_res[ix] == THE_NON_VALUE); - part_res[ix] = process_info_aux(c_p, rp, pid, am_messages, always_wrap); + part_res[ix] = process_info_aux(c_p, rp, locks, pid, am_messages, always_wrap); ASSERT(part_res[ix] != THE_NON_VALUE); } @@ -848,7 +865,7 @@ process_info_list(Process *c_p, Eterm pid, Eterm list, int always_wrap, ix = res_elem_ix[res_elem_ix_ix]; if (part_res[ix] == THE_NON_VALUE) { arg = pi_ix2arg(ix); - part_res[ix] = process_info_aux(c_p, rp, pid, arg, always_wrap); + part_res[ix] = process_info_aux(c_p, rp, locks, pid, arg, always_wrap); ASSERT(part_res[ix] != THE_NON_VALUE); } } @@ -979,9 +996,31 @@ BIF_RETTYPE process_info_2(BIF_ALIST_2) ERTS_BIF_AWAIT_X_DATA_TRAP(BIF_P, BIF_ARG_1, am_undefined); } else { + ErtsProcLocks unlock_locks = 0; + + if (BIF_P == rp) + info_locks |= ERTS_PROC_LOCK_MAIN; + if (!(info_locks & ERTS_PROC_LOCK_STATUS)) - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); - res = process_info_aux(BIF_P, rp, pid, BIF_ARG_2, 0); + unlock_locks |= ERTS_PROC_LOCK_STATUS; + + if (info_locks & ERTS_PROC_LOCK_MSGQ) { + /* + * Move in queue into private queue and + * release msgq lock, enabling others to + * send messages to the process while it + * is being inspected... + */ + ASSERT(info_locks & ERTS_PROC_LOCK_MAIN); + ERTS_SMP_MSGQ_MV_INQ2PRIVQ(rp); + info_locks &= ~ERTS_PROC_LOCK_MSGQ; + unlock_locks |= ERTS_PROC_LOCK_MSGQ; + } + + if (unlock_locks) + erts_smp_proc_unlock(rp, unlock_locks); + + res = process_info_aux(BIF_P, rp, info_locks, pid, BIF_ARG_2, 0); } ASSERT(is_value(res)); @@ -999,6 +1038,7 @@ BIF_RETTYPE process_info_2(BIF_ALIST_2) Eterm process_info_aux(Process *BIF_P, Process *rp, + ErtsProcLocks rp_locks, Eterm rpid, Eterm item, int always_wrap) @@ -1070,171 +1110,55 @@ process_info_aux(Process *BIF_P, break; case am_messages: { - ErlMessage* mp; - int n; - ERTS_SMP_MSGQ_MV_INQ2PRIVQ(rp); - n = rp->msg.len; - - if (n == 0 || ERTS_TRACE_FLAGS(rp) & F_SENSITIVE) { + if (rp->msg.len == 0 || ERTS_TRACE_FLAGS(rp) & F_SENSITIVE) { hp = HAlloc(BIF_P, 3); } else { - int remove_bad_messages = 0; - struct { - Uint copy_struct_size; - ErlMessage* msgp; - } *mq = erts_alloc(ERTS_ALC_T_TMP, n*sizeof(*mq)); - Sint i = 0; - Uint heap_need = 3; + ErtsMessageInfo *mip; + Sint i; + Uint heap_need; +#ifdef DEBUG Eterm *hp_end; +#endif - for (mp = rp->msg.first; mp; mp = mp->next) { - heap_need += 2; - mq[i].msgp = mp; - if (rp != BIF_P) { - Eterm msg = ERL_MESSAGE_TERM(mq[i].msgp); - if (is_value(msg)) { - mq[i].copy_struct_size = (is_immed(msg)? 0 : - size_object(msg)); - } - else if (mq[i].msgp->data.attached) { - mq[i].copy_struct_size - = erts_msg_attached_data_size(mq[i].msgp); - } - else { - /* Bad distribution message; ignore */ - remove_bad_messages = 1; - mq[i].copy_struct_size = 0; - } - heap_need += mq[i].copy_struct_size; - } - else { - mq[i].copy_struct_size = mp->data.attached ? - erts_msg_attached_data_size(mp) : 0; - } - i++; - } + mip = erts_alloc(ERTS_ALC_T_TMP, + rp->msg.len*sizeof(ErtsMessageInfo)); - if (rp != BIF_P) { - hp = HAlloc(BIF_P, heap_need); - hp_end = hp + heap_need; - ASSERT(i == n); - for (i--; i >= 0; i--) { - Eterm msg = ERL_MESSAGE_TERM(mq[i].msgp); - if (is_value(msg)) { - if (mq[i].copy_struct_size) - msg = copy_struct(msg, - mq[i].copy_struct_size, - &hp, - &MSO(BIF_P)); - } - else if (mq[i].msgp->data.attached) { - ErlHeapFragment *hfp; - /* - * Decode it into a message buffer and attach it - * to the message instead of the attached external - * term. - * - * Note that we may not pass a process pointer - * to erts_msg_distext2heap(), since it would then - * try to alter locks on that process. - */ - msg = erts_msg_distext2heap( - NULL, NULL, &hfp, &ERL_MESSAGE_TOKEN(mq[i].msgp), - mq[i].msgp->data.dist_ext); - - ERL_MESSAGE_TERM(mq[i].msgp) = msg; - mq[i].msgp->data.heap_frag = hfp; - - if (is_non_value(msg)) { - ASSERT(!mq[i].msgp->data.heap_frag); - /* Bad distribution message; ignore */ - remove_bad_messages = 1; - continue; - } - else { - /* Make our copy of the message */ - ASSERT(size_object(msg) == erts_used_frag_sz(hfp)); - msg = copy_struct(msg, - erts_used_frag_sz(hfp), - &hp, - &MSO(BIF_P)); - } - } - else { - /* Bad distribution message; ignore */ - remove_bad_messages = 1; - continue; - } - res = CONS(hp, msg, res); - hp += 2; - } - HRelease(BIF_P, hp_end, hp+3); - } - else { - for (i--; i >= 0; i--) { - ErtsHeapFactory factory; - Eterm msg = ERL_MESSAGE_TERM(mq[i].msgp); - - erts_factory_proc_prealloc_init(&factory, BIF_P, - mq[i].copy_struct_size+2); - if (mq[i].msgp->data.attached) { - /* Decode it on the heap */ - erts_move_msg_attached_data_to_heap(&factory, - mq[i].msgp); - msg = ERL_MESSAGE_TERM(mq[i].msgp); - ASSERT(!mq[i].msgp->data.attached); - } - if (is_value(msg)) { - hp = erts_produce_heap(&factory, 2, 0); - res = CONS(hp, msg, res); - } - else { - /* Bad distribution message; ignore */ - remove_bad_messages = 1; - continue; - } - erts_factory_close(&factory); - } - hp = HAlloc(BIF_P, 3); - } - erts_free(ERTS_ALC_T_TMP, mq); - if (remove_bad_messages) { - ErlMessage **mpp; - /* - * We need to remove bad distribution messages from - * the queue, so that the value returned for - * 'message_queue_len' is consistent with the value - * returned for 'messages'. - */ - mpp = &rp->msg.first; - mp = rp->msg.first; - while (mp) { - if (is_value(ERL_MESSAGE_TERM(mp))) { - mpp = &mp->next; - mp = mp->next; - } - else { - ErlMessage* bad_mp = mp; - ASSERT(!mp->data.attached); - if (rp->msg.save == &mp->next) - rp->msg.save = mpp; - if (rp->msg.last == &mp->next) - rp->msg.last = mpp; - *mpp = mp->next; - mp = mp->next; - rp->msg.len--; - free_message(bad_mp); - } - } + /* + * Note that message queue may shrink when calling + * erts_prep_msgq_for_inspection() since it removes + * corrupt distribution messages. + */ + heap_need = erts_prep_msgq_for_inspection(BIF_P, rp, rp_locks, mip); + heap_need += 3; /* top 2-tuple */ + heap_need += rp->msg.len*2; /* Cons cells */ + + hp = HAlloc(BIF_P, heap_need); /* heap_need is exact */ +#ifdef DEBUG + hp_end = hp + heap_need; +#endif + + /* Build list of messages... */ + for (i = rp->msg.len - 1, res = NIL; i >= 0; i--) { + Eterm msg = ERL_MESSAGE_TERM(mip[i].msgp); + Uint sz = mip[i].size; + + if (sz != 0) + msg = copy_struct(msg, sz, &hp, &BIF_P->off_heap); + + res = CONS(hp, msg, res); + hp += 2; } + + ASSERT(hp_end == hp + 3); + + erts_free(ERTS_ALC_T_TMP, mip); } break; } case am_message_queue_len: hp = HAlloc(BIF_P, 3); - ERTS_SMP_MSGQ_MV_INQ2PRIVQ(rp); res = make_small(rp->msg.len); break; @@ -1422,7 +1346,7 @@ process_info_aux(Process *BIF_P, } case am_total_heap_size: { - ErlMessage *mp; + ErtsMessage *mp; Uint total_heap_size; Uint hsz = 3; @@ -1432,8 +1356,6 @@ process_info_aux(Process *BIF_P, total_heap_size += rp->mbuf_sz; - ERTS_SMP_MSGQ_MV_INQ2PRIVQ(rp); - for (mp = rp->msg.first; mp; mp = mp->next) if (mp->data.attached) total_heap_size += erts_msg_attached_data_size(mp); @@ -1455,7 +1377,7 @@ process_info_aux(Process *BIF_P, case am_memory: { /* Memory consumed in bytes */ Uint hsz = 3; - Uint size = erts_process_memory(rp); + Uint size = erts_process_memory(rp, 0); (void) erts_bld_uint(NULL, &hsz, size); hp = HAlloc(BIF_P, hsz); res = erts_bld_uint(&hp, NULL, size); @@ -1480,6 +1402,32 @@ process_info_aux(Process *BIF_P, break; } + case am_garbage_collection_info: { + Uint sz = 0, actual_sz = 0; + + if (rp == BIF_P) { + sz += ERTS_PROCESS_GC_INFO_MAX_SIZE; + } else { + erts_process_gc_info(rp, &sz, NULL); + sz += 3; + } + + hp = HAlloc(BIF_P, sz); + res = erts_process_gc_info(rp, &actual_sz, &hp); + + /* We may have some extra space, fill with 0 tuples */ + if (actual_sz <= sz - 3) { + for (; actual_sz < sz - 3; hp++, actual_sz++) + hp[0] = make_arityval(0); + } else { + for (; actual_sz < sz; hp++, actual_sz++) + hp[0] = make_arityval(0); + hp = HAlloc(BIF_P, 3); + } + + break; + } + case am_group_leader: { int sz = NC_HEAP_SIZE(rp->group_leader); hp = HAlloc(BIF_P, 3 + sz); @@ -1581,6 +1529,25 @@ process_info_aux(Process *BIF_P, break; } + case am_message_queue_data: + switch (rp->flags & (F_OFF_HEAP_MSGQ|F_ON_HEAP_MSGQ)) { + case F_OFF_HEAP_MSGQ: + res = am_off_heap; + break; + case F_ON_HEAP_MSGQ: + res = am_on_heap; + break; + case 0: + res = am_mixed; + break; + default: + res = am_error; + ERTS_INTERNAL_ERROR("Inconsistent message queue management state"); + break; + } + hp = HAlloc(BIF_P, 3); + break; + default: return THE_NON_VALUE; /* will produce badarg */ @@ -1755,7 +1722,7 @@ info_1_tuple(Process* BIF_P, /* Pointer to current process. */ if (arity == 2) { Eterm res = THE_NON_VALUE; char *buf; - int len = is_string(*tp); + Sint len = is_string(*tp); if (len <= 0) return res; buf = (char *) erts_alloc(ERTS_ALC_T_TMP, len+1); @@ -1774,7 +1741,7 @@ info_1_tuple(Process* BIF_P, /* Pointer to current process. */ else { Eterm res = THE_NON_VALUE; char *buf; - int len = is_string(tp[1]); + Sint len = is_string(tp[1]); if (len <= 0) return res; buf = (char *) erts_alloc(ERTS_ALC_T_TMP, len+1); @@ -2742,6 +2709,19 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) BIF_RET(am_true); } #endif + else if (BIF_ARG_1 == am_message_queue_data) { + switch (erts_default_spo_flags & (SPO_ON_HEAP_MSGQ|SPO_OFF_HEAP_MSGQ)) { + case SPO_OFF_HEAP_MSGQ: + BIF_RET(am_off_heap); + case SPO_ON_HEAP_MSGQ: + BIF_RET(am_on_heap); + case 0: + BIF_RET(am_mixed); + default: + ERTS_INTERNAL_ERROR("Inconsistent message queue management state"); + BIF_RET(am_error); + } + } else if (ERTS_IS_ATOM_STR("compile_info",BIF_ARG_1)) { Uint sz; Eterm res = NIL, tup, text; @@ -2780,6 +2760,20 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) else if (ERTS_IS_ATOM_STR("eager_check_io",BIF_ARG_1)) { BIF_RET(erts_eager_check_io ? am_true : am_false); } + else if (ERTS_IS_ATOM_STR("literal_test",BIF_ARG_1)) { +#ifdef ERTS_HAVE_IS_IN_LITERAL_RANGE +#ifdef ARCH_64 + DECL_AM(range); + BIF_RET(AM_range); +#else /* ARCH_32 */ + DECL_AM(range_bitmask); + BIF_RET(AM_range_bitmask); +#endif /* ARCH_32 */ +#else /* ! ERTS_HAVE_IS_IN_LITERAL_RANGE */ + DECL_AM(tag); + BIF_RET(AM_tag); +#endif + } BIF_ERROR(BIF_P, BADARG); } @@ -3268,6 +3262,14 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1) szp = NULL; hpp = &hp; } +#ifdef ERTS_ENABLE_MSACC + } else if (BIF_ARG_1 == am_microstate_accounting) { + Eterm threads; + res = erts_msacc_request(BIF_P, ERTS_MSACC_GATHER, &threads); + if (is_non_value(res)) + BIF_RET(am_undefined); + BIF_TRAP2(gather_msacc_res_trap, BIF_P, res, threads); +#endif } else if (BIF_ARG_1 == am_context_switches) { Eterm cs = erts_make_integer(erts_get_total_context_switches(), BIF_P); hp = HAlloc(BIF_P, 3); @@ -3488,7 +3490,7 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) BIF_RET(res); } else if (ERTS_IS_ATOM_STR("mmap", BIF_ARG_1)) { - BIF_RET(erts_mmap_debug_info(BIF_P)); + BIF_RET(erts_mmap_debug_info(&erts_dflt_mmapper, BIF_P)); } else if (ERTS_IS_ATOM_STR("unique_monotonic_integer_state", BIF_ARG_1)) { BIF_RET(erts_debug_get_unique_monotonic_integer_state(BIF_P)); @@ -3910,9 +3912,7 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) BIF_RET(am_false); } else { - FLAGS(rp) |= F_FORCE_GC; - if (BIF_P != rp) - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MAIN); + ERTS_FORCE_GC(BIF_P); BIF_RET(am_true); } } @@ -4131,6 +4131,17 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) BIF_RET(am_ok); } } + else if (ERTS_IS_ATOM_STR("fill_heap", BIF_ARG_1)) { + UWord left = HeapWordsLeft(BIF_P); + if (left > 1) { + Eterm* hp = HAlloc(BIF_P, left); + *hp = make_pos_bignum_header(left - 1); + } + if (BIF_ARG_2 == am_true) { + FLAGS(BIF_P) |= F_NEED_FULLSWEEP; + } + BIF_RET(am_ok); + } } BIF_ERROR(BIF_P, BADARG); @@ -4380,14 +4391,17 @@ static void os_info_init(void) os_flavor(buf, 1024); flav = erts_atom_put((byte *) buf, strlen(buf), ERTS_ATOM_ENC_LATIN1, 1); erts_free(ERTS_ALC_T_TMP, (void *) buf); - hp = erts_alloc(ERTS_ALC_T_LL_TEMP_TERM, (3+4)*sizeof(Eterm)); + hp = erts_alloc(ERTS_ALC_T_LITERAL, (3+4)*sizeof(Eterm)); os_type_tuple = TUPLE2(hp, type, flav); + erts_set_literal_tag(&os_type_tuple, hp, 3); + hp += 3; os_version(&major, &minor, &build); os_version_tuple = TUPLE3(hp, make_small(major), make_small(minor), make_small(build)); + erts_set_literal_tag(&os_version_tuple, hp, 4); } void @@ -4404,6 +4418,8 @@ erts_bif_info_init(void) = erts_export_put(am_erlang, am_gather_gc_info_result, 1); gather_io_bytes_trap = erts_export_put(am_erts_internal, am_gather_io_bytes, 2); + gather_msacc_res_trap + = erts_export_put(am_erts_internal, am_gather_microstate_accounting_result, 2); gather_system_check_res_trap = erts_export_put(am_erts_internal, am_gather_system_check_result, 1); process_info_init(); diff --git a/erts/emulator/beam/erl_bif_lists.c b/erts/emulator/beam/erl_bif_lists.c index 5583dcb371..fe64e76575 100644 --- a/erts/emulator/beam/erl_bif_lists.c +++ b/erts/emulator/beam/erl_bif_lists.c @@ -42,7 +42,7 @@ static BIF_RETTYPE append(Process* p, Eterm A, Eterm B) Eterm last; size_t need; Eterm* hp; - int i; + Sint i; if ((i = erts_list_length(A)) < 0) { BIF_ERROR(p, BADARG); @@ -99,9 +99,9 @@ static Eterm subtract(Process* p, Eterm A, Eterm B) Eterm small_vec[SMALL_VEC_SIZE]; /* Preallocated memory for small lists */ Eterm* vec_p; Eterm* vp; - int i; - int n; - int m; + Sint i; + Sint n; + Sint m; if ((n = erts_list_length(A)) < 0) { BIF_ERROR(p, BADARG); diff --git a/erts/emulator/beam/erl_bif_op.c b/erts/emulator/beam/erl_bif_op.c index c9192fc420..0f20ded1d6 100644 --- a/erts/emulator/beam/erl_bif_op.c +++ b/erts/emulator/beam/erl_bif_op.c @@ -89,22 +89,22 @@ BIF_RETTYPE not_1(BIF_ALIST_1) BIF_RETTYPE sgt_2(BIF_ALIST_2) { - BIF_RET(cmp_gt(BIF_ARG_1, BIF_ARG_2) ? am_true : am_false); + BIF_RET(CMP_GT(BIF_ARG_1, BIF_ARG_2) ? am_true : am_false); } BIF_RETTYPE sge_2(BIF_ALIST_2) { - BIF_RET(cmp_ge(BIF_ARG_1, BIF_ARG_2) ? am_true : am_false); + BIF_RET(CMP_GE(BIF_ARG_1, BIF_ARG_2) ? am_true : am_false); } BIF_RETTYPE slt_2(BIF_ALIST_2) { - BIF_RET(cmp_lt(BIF_ARG_1, BIF_ARG_2) ? am_true : am_false); + BIF_RET(CMP_LT(BIF_ARG_1, BIF_ARG_2) ? am_true : am_false); } BIF_RETTYPE sle_2(BIF_ALIST_2) { - BIF_RET(cmp_le(BIF_ARG_1, BIF_ARG_2) ? am_true : am_false); + BIF_RET(CMP_LE(BIF_ARG_1, BIF_ARG_2) ? am_true : am_false); } BIF_RETTYPE seq_2(BIF_ALIST_2) @@ -114,7 +114,7 @@ BIF_RETTYPE seq_2(BIF_ALIST_2) BIF_RETTYPE seqeq_2(BIF_ALIST_2) { - BIF_RET(cmp_eq(BIF_ARG_1, BIF_ARG_2) ? am_true : am_false); + BIF_RET(CMP_EQ(BIF_ARG_1, BIF_ARG_2) ? am_true : am_false); } BIF_RETTYPE sneq_2(BIF_ALIST_2) @@ -124,7 +124,7 @@ BIF_RETTYPE sneq_2(BIF_ALIST_2) BIF_RETTYPE sneqeq_2(BIF_ALIST_2) { - BIF_RET(cmp_ne(BIF_ARG_1, BIF_ARG_2) ? am_true : am_false); + BIF_RET(CMP_NE(BIF_ARG_1, BIF_ARG_2) ? am_true : am_false); } BIF_RETTYPE is_atom_1(BIF_ALIST_1) @@ -258,7 +258,7 @@ Eterm erl_is_function(Process* p, Eterm arg1, Eterm arg2) BIF_RET(am_true); } } else if (is_export(arg1)) { - Export* exp = (Export *) EXPAND_POINTER((export_val(arg1))[1]); + Export* exp = (Export *) (export_val(arg1)[1]); if (exp->code[2] == (Uint) arity) { BIF_RET(am_true); diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c index e47d7bcbbb..3acc1d7bbd 100644 --- a/erts/emulator/beam/erl_bif_port.c +++ b/erts/emulator/beam/erl_bif_port.c @@ -41,6 +41,7 @@ #include "external.h" #include "packet_parser.h" #include "erl_bits.h" +#include "erl_bif_unique.h" #include "dtrace-wrapper.h" static Port *open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump); @@ -50,10 +51,10 @@ static void free_args(char **); char *erts_default_arg0 = "default"; -BIF_RETTYPE open_port_2(BIF_ALIST_2) +BIF_RETTYPE erts_internal_open_port_2(BIF_ALIST_2) { Port *port; - Eterm port_id; + Eterm res; char *str; int err_type, err_num; @@ -61,27 +62,58 @@ BIF_RETTYPE open_port_2(BIF_ALIST_2) if (!port) { if (err_type == -3) { ASSERT(err_num == BADARG || err_num == SYSTEM_LIMIT); - BIF_ERROR(BIF_P, err_num); + if (err_num == BADARG) + res = am_badarg; + else if (err_num == SYSTEM_LIMIT) + res = am_system_limit; + else + /* this is only here to silence gcc, it should not happen */ + BIF_ERROR(BIF_P, EXC_INTERNAL_ERROR); } else if (err_type == -2) { str = erl_errno_id(err_num); + res = erts_atom_put((byte *) str, strlen(str), ERTS_ATOM_ENC_LATIN1, 1); } else { - str = "einval"; + res = am_einval; } - BIF_P->fvalue = erts_atom_put((byte *) str, strlen(str), ERTS_ATOM_ENC_LATIN1, 1); - BIF_ERROR(BIF_P, EXC_ERROR); - } + BIF_RET(res); + } + + if (port->drv_ptr->flags & ERL_DRV_FLAG_USE_INIT_ACK) { + /* Copied from erl_port_task.c */ + port->async_open_port = erts_alloc(ERTS_ALC_T_PRTSD, + sizeof(*port->async_open_port)); + erts_make_ref_in_array(port->async_open_port->ref); + port->async_open_port->to = BIF_P->common.id; + + erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCKS_MSG_RECEIVE | ERTS_PROC_LOCK_LINK); + if (ERTS_PROC_PENDING_EXIT(BIF_P)) { + /* need to exit caller instead */ + erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCKS_MSG_RECEIVE | ERTS_PROC_LOCK_LINK); + KILL_CATCHES(BIF_P); + BIF_P->freason = EXC_EXIT; + erts_port_release(port); + BIF_RET(am_badarg); + } + + ERTS_SMP_MSGQ_MV_INQ2PRIVQ(BIF_P); + BIF_P->msg.save = BIF_P->msg.last; - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK); + erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCKS_MSG_RECEIVE); + + res = erts_proc_store_ref(BIF_P, port->async_open_port->ref); + } else { + res = port->common.id; + erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK); + } - port_id = port->common.id; erts_add_link(&ERTS_P_LINKS(port), LINK_PID, BIF_P->common.id); - erts_add_link(&ERTS_P_LINKS(BIF_P), LINK_PID, port_id); + erts_add_link(&ERTS_P_LINKS(BIF_P), LINK_PID, port->common.id); erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK); erts_port_release(port); - BIF_RET(port_id); + BIF_RET(res); } static ERTS_INLINE Port * @@ -617,7 +649,7 @@ BIF_RETTYPE port_get_data_1(BIF_ALIST_1) static Port * open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump) { - int i; + Sint i; Eterm option; Uint arity; Eterm* tp; @@ -945,8 +977,8 @@ static char **convert_args(Eterm l) { char **pp; char *b; - int n; - int i = 0; + Sint n; + Sint i = 0; Eterm str; if (is_not_list(l) && is_not_nil(l)) { return NULL; @@ -992,7 +1024,7 @@ static byte* convert_environment(Process* p, Eterm env) Eterm* temp_heap; Eterm* hp; Uint heap_size; - int n; + Sint n; Sint size; byte* bytes; int encoding = erts_get_native_filename_encoding(); diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c index 86951f32b0..7f7cd376ac 100644 --- a/erts/emulator/beam/erl_bif_re.c +++ b/erts/emulator/beam/erl_bif_re.c @@ -100,7 +100,7 @@ Sint erts_re_set_loop_limit(Sint limit) static int term_to_int(Eterm term, int *sp) { -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) if (is_small(term)) { Uint x = signed_val(term); @@ -151,7 +151,7 @@ static int term_to_int(Eterm term, int *sp) static Eterm make_signed_integer(int x, Process *p) { -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) return make_small(x); #else Eterm* hp; diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c index 08807d72c9..71a7079b09 100644 --- a/erts/emulator/beam/erl_bif_trace.c +++ b/erts/emulator/beam/erl_bif_trace.c @@ -1887,11 +1887,7 @@ new_seq_trace_token(Process* p) { Eterm* hp; - if (SEQ_TRACE_TOKEN(p) == NIL -#ifdef USE_VM_PROBES - || SEQ_TRACE_TOKEN(p) == am_have_dt_utag -#endif - ) { + if (have_no_seqtrace(SEQ_TRACE_TOKEN(p))) { hp = HAlloc(p, 6); SEQ_TRACE_TOKEN(p) = TUPLE5(hp, make_small(0), /* Flags */ make_small(0), /* Label */ @@ -1911,12 +1907,8 @@ BIF_RETTYPE erl_seq_trace_info(Process *p, Eterm item) BIF_ERROR(p, BADARG); } - if (SEQ_TRACE_TOKEN(p) == NIL -#ifdef USE_VM_PROBES - || SEQ_TRACE_TOKEN(p) == am_have_dt_utag -#endif - ) { - if ((item == am_send) || (item == am_receive) || + if (have_no_seqtrace(SEQ_TRACE_TOKEN(p))) { + if ((item == am_send) || (item == am_receive) || (item == am_print) || (item == am_timestamp) || (item == am_monotonic_timestamp) || (item == am_strict_monotonic_timestamp)) { @@ -1978,11 +1970,7 @@ BIF_RETTYPE seq_trace_info_1(BIF_ALIST_1) */ BIF_RETTYPE seq_trace_print_1(BIF_ALIST_1) { - if (SEQ_TRACE_TOKEN(BIF_P) == NIL -#ifdef USE_VM_PROBES - || SEQ_TRACE_TOKEN(BIF_P) == am_have_dt_utag -#endif - ) { + if (have_no_seqtrace(SEQ_TRACE_TOKEN(BIF_P))) { BIF_RET(am_false); } seq_trace_update_send(BIF_P); @@ -2001,11 +1989,7 @@ BIF_RETTYPE seq_trace_print_1(BIF_ALIST_1) */ BIF_RETTYPE seq_trace_print_2(BIF_ALIST_2) { - if (SEQ_TRACE_TOKEN(BIF_P) == NIL -#ifdef USE_VM_PROBES - || SEQ_TRACE_TOKEN(BIF_P) == am_have_dt_utag -#endif - ) { + if (have_no_seqtrace(SEQ_TRACE_TOKEN(BIF_P))) { BIF_RET(am_false); } if (!(is_atom(BIF_ARG_1) || is_small(BIF_ARG_1))) { diff --git a/erts/emulator/beam/erl_bif_unique.c b/erts/emulator/beam/erl_bif_unique.c index 5eca09c5a6..c4a39b8897 100644 --- a/erts/emulator/beam/erl_bif_unique.c +++ b/erts/emulator/beam/erl_bif_unique.c @@ -338,7 +338,7 @@ static struct { } w; } raw_unique_monotonic_integer erts_align_attribute(ERTS_CACHE_LINE_SIZE); -#if defined(ARCH_32) || HALFWORD_HEAP +#if defined(ARCH_32) # define ERTS_UNIQUE_MONOTONIC_OFFSET ERTS_SINT64_MIN #else # define ERTS_UNIQUE_MONOTONIC_OFFSET MIN_SMALL @@ -368,7 +368,7 @@ get_unique_monotonic_integer_heap_size(Uint64 raw, int positive) Sint64 value = ((Sint64) raw) + ERTS_UNIQUE_MONOTONIC_OFFSET; if (IS_SSMALL(value)) return 0; -#if defined(ARCH_32) || HALFWORD_HEAP +#if defined(ARCH_32) return ERTS_SINT64_HEAP_SIZE(value); #else return ERTS_UINT64_HEAP_SIZE((Uint64) value); @@ -393,7 +393,7 @@ make_unique_monotonic_integer_value(Eterm *hp, Uint hsz, Uint64 raw, int positiv if (hsz == 0) res = make_small(value); else { -#if defined(ARCH_32) || HALFWORD_HEAP +#if defined(ARCH_32) res = erts_sint64_to_big(value, &hp); #else res = erts_uint64_to_big((Uint64) value, &hp); diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h index ea01bf08f0..e181b5555d 100644 --- a/erts/emulator/beam/erl_binary.h +++ b/erts/emulator/beam/erl_binary.h @@ -72,7 +72,6 @@ typedef struct erl_heap_bin { */ #define binary_size(Bin) (binary_val(Bin)[1]) -#define binary_size_rel(Bin,BasePtr) (binary_val_rel(Bin,BasePtr)[1]) #define binary_bitsize(Bin) \ ((*binary_val(Bin) == HEADER_SUB_BIN) ? \ @@ -95,12 +94,9 @@ typedef struct erl_heap_bin { * Bitsize: output variable (Uint) */ -#define ERTS_GET_BINARY_BYTES(Bin,Bytep,Bitoffs,Bitsize) \ - ERTS_GET_BINARY_BYTES_REL(Bin,Bytep,Bitoffs,Bitsize,NULL) - -#define ERTS_GET_BINARY_BYTES_REL(Bin,Bytep,Bitoffs,Bitsize,BasePtr) \ +#define ERTS_GET_BINARY_BYTES(Bin,Bytep,Bitoffs,Bitsize) \ do { \ - Eterm* _real_bin = binary_val_rel(Bin,BasePtr); \ + Eterm* _real_bin = binary_val(Bin); \ Uint _offs = 0; \ Bitoffs = Bitsize = 0; \ if (*_real_bin == HEADER_SUB_BIN) { \ @@ -108,7 +104,7 @@ do { \ _offs = _sb->offs; \ Bitoffs = _sb->bitoffs; \ Bitsize = _sb->bitsize; \ - _real_bin = binary_val_rel(_sb->orig,BasePtr); \ + _real_bin = binary_val(_sb->orig); \ } \ if (*_real_bin == HEADER_PROC_BIN) { \ Bytep = ((ProcBin *) _real_bin)->bytes + _offs; \ @@ -131,11 +127,8 @@ do { \ */ #define ERTS_GET_REAL_BIN(Bin, RealBin, ByteOffset, BitOffset, BitSize) \ - ERTS_GET_REAL_BIN_REL(Bin, RealBin, ByteOffset, BitOffset, BitSize, NULL) - -#define ERTS_GET_REAL_BIN_REL(Bin, RealBin, ByteOffset, BitOffset, BitSize, BasePtr) \ do { \ - ErlSubBin* _sb = (ErlSubBin *) binary_val_rel(Bin,BasePtr); \ + ErlSubBin* _sb = (ErlSubBin *) binary_val(Bin); \ if (_sb->thing_word == HEADER_SUB_BIN) { \ RealBin = _sb->orig; \ ByteOffset = _sb->offs; \ diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c index 01734c55d7..11d83686a3 100644 --- a/erts/emulator/beam/erl_bits.c +++ b/erts/emulator/beam/erl_bits.c @@ -282,7 +282,7 @@ erts_bs_get_integer_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuff * Simply shift whole bytes into the result. */ switch (BYTE_OFFSET(n)) { -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) case 7: w = (w << 8) | *bp++; case 6: w = (w << 8) | *bp++; case 5: w = (w << 8) | *bp++; @@ -387,7 +387,7 @@ erts_bs_get_integer_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuff case 3: v32 = LSB[0] + (LSB[1]<<8) + (LSB[2]<<16); goto big_small; -#if !defined(ARCH_64) || HALFWORD_HEAP +#if !defined(ARCH_64) case 4: v32 = (LSB[0] + (LSB[1]<<8) + (LSB[2]<<16) + (LSB[3]<<24)); if (!IS_USMALL(sgn, v32)) { diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index 645f9e3c28..55b8789e29 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -1773,15 +1773,9 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1) * (it looks like an continuation pointer), but that is will crash the * emulator if this BIF is call traced. */ -#if HALFWORD_HEAP - Eterm *hp = HAlloc(BIF_P, 3); - hp[0] = make_pos_bignum_header(2); - *((UWord *) (UWord) (hp+1)) = (UWord) tb; -#else Eterm *hp = HAlloc(BIF_P, 2); hp[0] = make_pos_bignum_header(1); hp[1] = (Eterm) tb; -#endif BIF_TRAP1(&ets_delete_continue_exp, BIF_P, make_big(hp)); } else { @@ -1837,7 +1831,7 @@ BIF_RETTYPE ets_give_away_3(BIF_ALIST_3) tb->common.id, from_pid, BIF_ARG_3), - 0); + 0); erts_smp_proc_unlock(to_proc, to_locks); UnUseTmpHeap(5,BIF_P); BIF_RET(am_true); @@ -2839,7 +2833,7 @@ BIF_RETTYPE ets_match_spec_run_r_3(BIF_ALIST_3) BIF_TRAP3(bif_export[BIF_ets_match_spec_run_r_3], BIF_P,lst,BIF_ARG_2,ret); } - res = db_prog_match(BIF_P, mp, CAR(list_val(lst)), NULL, NULL, 0, + res = db_prog_match(BIF_P, mp, CAR(list_val(lst)), NULL, 0, ERTS_PAM_COPY_RESULT, &dummy); if (is_value(res)) { hp = HAlloc(BIF_P, 2); @@ -3216,7 +3210,7 @@ retry: tb->common.id, p->common.id, heir_data), - 0); + 0); erts_smp_proc_unlock(to_proc, to_locks); return !0; } @@ -3651,11 +3645,8 @@ static BIF_RETTYPE ets_delete_trap(BIF_ALIST_1) Eterm* ptr = big_val(cont); DbTable *tb = *((DbTable **) (UWord) (ptr + 1)); -#if HALFWORD_HEAP - ASSERT(*ptr == make_pos_bignum_header(2)); -#else ASSERT(*ptr == make_pos_bignum_header(1)); -#endif + db_lock(tb, LCK_WRITE); trap = free_table_cont(p, tb, 0, 1); db_unlock(tb, LCK_WRITE); diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 98a2e2842a..cff65f244d 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -469,9 +469,6 @@ static ERTS_INLINE void try_shrink(DbTableHash* tb) } } -#define EQ_REL(x,y,y_base) \ - (is_same(x,NULL,y,y_base) || (is_not_both_immed((x),(y)) && eq_rel((x),NULL,(y),y_base))) - /* Is this a live object (not pseodo-deleted) with the specified key? */ static ERTS_INLINE int has_live_key(DbTableHash* tb, HashDbTerm* b, @@ -481,7 +478,7 @@ static ERTS_INLINE int has_live_key(DbTableHash* tb, HashDbTerm* b, else { Eterm itemKey = GETKEY(tb, b->dbterm.tpl); ASSERT(!is_header(itemKey)); - return EQ_REL(key, itemKey, b->dbterm.tpl); + return EQ(key, itemKey); } } @@ -494,7 +491,7 @@ static ERTS_INLINE int has_key(DbTableHash* tb, HashDbTerm* b, else { Eterm itemKey = GETKEY(tb, b->dbterm.tpl); ASSERT(!is_header(itemKey)); - return EQ_REL(key, itemKey, b->dbterm.tpl); + return EQ(key, itemKey); } } @@ -2204,11 +2201,11 @@ static void db_print_hash(int to, void *to_arg, int show, DbTable *tbl) erts_print(to, to_arg, "*"); if (tb->common.compress) { Eterm key = GETKEY(tb, list->dbterm.tpl); - erts_print(to, to_arg, "key=%R", key, list->dbterm.tpl); + erts_print(to, to_arg, "key=%T", key); } else { - Eterm obj = make_tuple_rel(list->dbterm.tpl,list->dbterm.tpl); - erts_print(to, to_arg, "%R", obj, list->dbterm.tpl); + Eterm obj = make_tuple(list->dbterm.tpl); + erts_print(to, to_arg, "%T", obj); } if (list->next != 0) erts_print(to, to_arg, ","); @@ -2899,9 +2896,6 @@ Ldone: handle->dbterm = &b->dbterm; handle->flags = flags; handle->new_size = b->dbterm.size; -#if HALFWORD_HEAP - handle->abs_vec = NULL; -#endif handle->lck = lck; return 1; } diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c index 465aa566ad..311b69d114 100644 --- a/erts/emulator/beam/erl_db_tree.c +++ b/erts/emulator/beam/erl_db_tree.c @@ -280,7 +280,7 @@ struct select_delete_context { /* ** Forward declarations */ -static TreeDbTerm *linkout_tree(DbTableTree *tb, Eterm key, Eterm* key_base); +static TreeDbTerm *linkout_tree(DbTableTree *tb, Eterm key); static TreeDbTerm *linkout_object_tree(DbTableTree *tb, Eterm object); static int do_free_tree_cont(DbTableTree *tb, int num_left); @@ -291,15 +291,15 @@ static int delsub(TreeDbTerm **this); static TreeDbTerm *slot_search(Process *p, DbTableTree *tb, Sint slot); static TreeDbTerm *find_node(DbTableTree *tb, Eterm key); static TreeDbTerm **find_node2(DbTableTree *tb, Eterm key); -static TreeDbTerm *find_next(DbTableTree *tb, DbTreeStack*, Eterm key, Eterm* kbase); -static TreeDbTerm *find_prev(DbTableTree *tb, DbTreeStack*, Eterm key, Eterm* kbase); +static TreeDbTerm *find_next(DbTableTree *tb, DbTreeStack*, Eterm key); +static TreeDbTerm *find_prev(DbTableTree *tb, DbTreeStack*, Eterm key); static TreeDbTerm *find_next_from_pb_key(DbTableTree *tb, DbTreeStack*, Eterm key); static TreeDbTerm *find_prev_from_pb_key(DbTableTree *tb, DbTreeStack*, Eterm key); static void traverse_backwards(DbTableTree *tb, DbTreeStack*, - Eterm lastkey, Eterm* lk_base, + Eterm lastkey, int (*doit)(DbTableTree *tb, TreeDbTerm *, void *, @@ -307,7 +307,7 @@ static void traverse_backwards(DbTableTree *tb, void *context); static void traverse_forward(DbTableTree *tb, DbTreeStack*, - Eterm lastkey, Eterm* lk_base, + Eterm lastkey, int (*doit)(DbTableTree *tb, TreeDbTerm *, void *, @@ -315,8 +315,8 @@ static void traverse_forward(DbTableTree *tb, void *context); static int key_given(DbTableTree *tb, Eterm pattern, TreeDbTerm **ret, Eterm *partly_bound_key); -static Sint cmp_partly_bound(Eterm partly_bound_key, Eterm bound_key, Eterm* bk_base); -static Sint do_cmp_partly_bound(Eterm a, Eterm b, Eterm* b_base, int *done); +static Sint cmp_partly_bound(Eterm partly_bound_key, Eterm bound_key); +static Sint do_cmp_partly_bound(Eterm a, Eterm b, int *done); static int analyze_pattern(DbTableTree *tb, Eterm pattern, struct mp_info *mpi); @@ -517,7 +517,7 @@ static int db_next_tree(Process *p, DbTable *tbl, Eterm key, Eterm *ret) if (is_atom(key) && key == am_EOT) return DB_ERROR_BADKEY; stack = get_any_stack(tb); - this = find_next(tb, stack, key, NULL); + this = find_next(tb, stack, key); release_stack(tb,stack); if (this == NULL) { *ret = am_EOT; @@ -563,7 +563,7 @@ static int db_prev_tree(Process *p, DbTable *tbl, Eterm key, Eterm *ret) if (is_atom(key) && key == am_EOT) return DB_ERROR_BADKEY; stack = get_any_stack(tb); - this = find_prev(tb, stack, key, NULL); + this = find_prev(tb, stack, key); release_stack(tb,stack); if (this == NULL) { *ret = am_EOT; @@ -573,19 +573,13 @@ static int db_prev_tree(Process *p, DbTable *tbl, Eterm key, Eterm *ret) return DB_ERROR_NONE; } -static ERTS_INLINE Sint cmp_key(DbTableTree* tb, Eterm key, Eterm* key_base, - TreeDbTerm* obj) -{ - return cmp_rel(key, key_base, - GETKEY(tb,obj->dbterm.tpl), obj->dbterm.tpl); +static ERTS_INLINE Sint cmp_key(DbTableTree* tb, Eterm key, TreeDbTerm* obj) { + return CMP(key, GETKEY(tb,obj->dbterm.tpl)); } -static ERTS_INLINE int cmp_key_eq(DbTableTree* tb, Eterm key, Eterm* key_base, - TreeDbTerm* obj) -{ +static ERTS_INLINE int cmp_key_eq(DbTableTree* tb, Eterm key, TreeDbTerm* obj) { Eterm obj_key = GETKEY(tb,obj->dbterm.tpl); - return is_same(key, key_base, obj_key, obj->dbterm.tpl) - || cmp_rel(key, key_base, obj_key, obj->dbterm.tpl) == 0; + return is_same(key, obj_key) || CMP(key, obj_key) == 0; } static int db_put_tree(DbTable *tbl, Eterm obj, int key_clash_fail) @@ -619,7 +613,7 @@ static int db_put_tree(DbTable *tbl, Eterm obj, int key_clash_fail) (*this)->balance = 0; (*this)->left = (*this)->right = NULL; break; - } else if ((c = cmp_key(tb, key, NULL, *this)) < 0) { + } else if ((c = cmp_key(tb, key, *this)) < 0) { /* go lefts */ dstack[dpos++] = DIR_LEFT; tstack[tpos++] = this; @@ -774,7 +768,7 @@ static int db_erase_tree(DbTable *tbl, Eterm key, Eterm *ret) *ret = am_true; - if ((res = linkout_tree(tb, key, NULL)) != NULL) { + if ((res = linkout_tree(tb, key)) != NULL) { free_term(tb, res); } return DB_ERROR_NONE; @@ -970,15 +964,15 @@ static int db_select_continue_tree(Process *p, stack = get_any_stack(tb); if (chunk_size) { if (reverse) { - traverse_backwards(tb, stack, lastkey, NULL, &doit_select_chunk, &sc); + traverse_backwards(tb, stack, lastkey, &doit_select_chunk, &sc); } else { - traverse_forward(tb, stack, lastkey, NULL, &doit_select_chunk, &sc); + traverse_forward(tb, stack, lastkey, &doit_select_chunk, &sc); } } else { if (reverse) { - traverse_forward(tb, stack, lastkey, NULL, &doit_select, &sc); + traverse_forward(tb, stack, lastkey, &doit_select, &sc); } else { - traverse_backwards(tb, stack, lastkey, NULL, &doit_select, &sc); + traverse_backwards(tb, stack, lastkey, &doit_select, &sc); } } release_stack(tb,stack); @@ -1003,9 +997,9 @@ static int db_select_continue_tree(Process *p, } key = GETKEY(tb, sc.lastobj); - sz = size_object_rel(key,sc.lastobj); + sz = size_object(key); hp = HAlloc(p, 9 + sz); - key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL); + key = copy_struct(key, sz, &hp, &MSO(p)); continuation = TUPLE8 (hp, tptr[1], @@ -1026,8 +1020,8 @@ static int db_select_continue_tree(Process *p, key = GETKEY(tb, sc.lastobj); if (chunk_size) { if (end_condition != NIL && - ((!reverse && cmp_partly_bound(end_condition,key,sc.lastobj) < 0) || - (reverse && cmp_partly_bound(end_condition,key,sc.lastobj) > 0))) { + ((!reverse && cmp_partly_bound(end_condition,key) < 0) || + (reverse && cmp_partly_bound(end_condition,key) > 0))) { /* done anyway */ if (!sc.got) { RET_TO_BIF(am_EOT, DB_ERROR_NONE); @@ -1039,16 +1033,16 @@ static int db_select_continue_tree(Process *p, } } else { if (end_condition != NIL && - ((!reverse && cmp_partly_bound(end_condition,key,sc.lastobj) > 0) || - (reverse && cmp_partly_bound(end_condition,key,sc.lastobj) < 0))) { + ((!reverse && cmp_partly_bound(end_condition,key) > 0) || + (reverse && cmp_partly_bound(end_condition,key) < 0))) { /* done anyway */ RET_TO_BIF(sc.accum,DB_ERROR_NONE); } } /* Not done yet, let's trap. */ - sz = size_object_rel(key,sc.lastobj); + sz = size_object(key); hp = HAlloc(p, 9 + sz); - key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL); + key = copy_struct(key, sz, &hp, &MSO(p)); continuation = TUPLE8 (hp, tptr[1], @@ -1075,7 +1069,6 @@ static int db_select_tree(Process *p, DbTable *tbl, struct select_context sc; struct mp_info mpi; Eterm lastkey = THE_NON_VALUE; - Eterm* lk_base = NULL; Eterm key; Eterm continuation; unsigned sz; @@ -1127,20 +1120,18 @@ static int db_select_tree(Process *p, DbTable *tbl, if (mpi.some_limitation) { if ((this = find_prev_from_pb_key(tb, stack, mpi.least)) != NULL) { lastkey = GETKEY(tb, this->dbterm.tpl); - lk_base = this->dbterm.tpl; } sc.end_condition = mpi.most; } - traverse_forward(tb, stack, lastkey, lk_base, &doit_select, &sc); + traverse_forward(tb, stack, lastkey, &doit_select, &sc); } else { if (mpi.some_limitation) { if ((this = find_next_from_pb_key(tb, stack, mpi.most)) != NULL) { lastkey = GETKEY(tb, this->dbterm.tpl); - lk_base = this->dbterm.tpl; } sc.end_condition = mpi.least; } - traverse_backwards(tb, stack, lastkey, lk_base, &doit_select, &sc); + traverse_backwards(tb, stack, lastkey, &doit_select, &sc); } release_stack(tb,stack); #ifdef HARDDEBUG @@ -1153,9 +1144,9 @@ static int db_select_tree(Process *p, DbTable *tbl, } key = GETKEY(tb, sc.lastobj); - sz = size_object_rel(key, sc.lastobj); + sz = size_object(key); hp = HAlloc(p, 9 + sz + PROC_BIN_SIZE); - key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL); + key = copy_struct(key, sz, &hp, &MSO(p)); if (mpi.all_objects) (mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS; mpb=db_make_mp_binary(p,mpi.mp,&hp); @@ -1236,7 +1227,7 @@ static int db_select_count_continue_tree(Process *p, } stack = get_any_stack(tb); - traverse_backwards(tb, stack, lastkey, NULL, &doit_select_count, &sc); + traverse_backwards(tb, stack, lastkey, &doit_select_count, &sc); release_stack(tb,stack); BUMP_REDS(p, 1000 - sc.max); @@ -1246,12 +1237,12 @@ static int db_select_count_continue_tree(Process *p, } key = GETKEY(tb, sc.lastobj); if (end_condition != NIL && - (cmp_partly_bound(end_condition,key,sc.lastobj) > 0)) { + (cmp_partly_bound(end_condition,key) > 0)) { /* done anyway */ RET_TO_BIF(make_small(sc.got),DB_ERROR_NONE); } /* Not done yet, let's trap. */ - sz = size_object_rel(key, sc.lastobj); + sz = size_object(key); if (IS_USMALL(0, sc.got)) { hp = HAlloc(p, sz + 6); egot = make_small(sc.got); @@ -1261,7 +1252,7 @@ static int db_select_count_continue_tree(Process *p, egot = uint_to_big(sc.got, hp); hp += BIG_UINT_HEAP_SIZE; } - key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL); + key = copy_struct(key, sz, &hp, &MSO(p)); continuation = TUPLE5 (hp, tptr[1], @@ -1284,7 +1275,6 @@ static int db_select_count_tree(Process *p, DbTable *tbl, struct select_count_context sc; struct mp_info mpi; Eterm lastkey = THE_NON_VALUE; - Eterm* lk_base = NULL; Eterm key; Eterm continuation; unsigned sz; @@ -1334,12 +1324,11 @@ static int db_select_count_tree(Process *p, DbTable *tbl, if (mpi.some_limitation) { if ((this = find_next_from_pb_key(tb, stack, mpi.most)) != NULL) { lastkey = GETKEY(tb, this->dbterm.tpl); - lk_base = this->dbterm.tpl; } sc.end_condition = mpi.least; } - traverse_backwards(tb, stack, lastkey, lk_base, &doit_select_count, &sc); + traverse_backwards(tb, stack, lastkey, &doit_select_count, &sc); release_stack(tb,stack); BUMP_REDS(p, 1000 - sc.max); if (sc.max > 0) { @@ -1347,7 +1336,7 @@ static int db_select_count_tree(Process *p, DbTable *tbl, } key = GETKEY(tb, sc.lastobj); - sz = size_object_rel(key, sc.lastobj); + sz = size_object(key); if (IS_USMALL(0, sc.got)) { hp = HAlloc(p, sz + PROC_BIN_SIZE + 6); egot = make_small(sc.got); @@ -1357,7 +1346,7 @@ static int db_select_count_tree(Process *p, DbTable *tbl, egot = uint_to_big(sc.got, hp); hp += BIG_UINT_HEAP_SIZE; } - key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL); + key = copy_struct(key, sz, &hp, &MSO(p)); if (mpi.all_objects) (mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS; mpb = db_make_mp_binary(p,mpi.mp,&hp); @@ -1388,7 +1377,6 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl, struct select_context sc; struct mp_info mpi; Eterm lastkey = THE_NON_VALUE; - Eterm* lk_base = NULL; Eterm key; Eterm continuation; unsigned sz; @@ -1445,20 +1433,18 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl, if (mpi.some_limitation) { if ((this = find_next_from_pb_key(tb, stack, mpi.most)) != NULL) { lastkey = GETKEY(tb, this->dbterm.tpl); - lk_base = this->dbterm.tpl; } sc.end_condition = mpi.least; } - traverse_backwards(tb, stack, lastkey, lk_base, &doit_select_chunk, &sc); + traverse_backwards(tb, stack, lastkey, &doit_select_chunk, &sc); } else { if (mpi.some_limitation) { if ((this = find_prev_from_pb_key(tb, stack, mpi.least)) != NULL) { lastkey = GETKEY(tb, this->dbterm.tpl); - lk_base = this->dbterm.tpl; } sc.end_condition = mpi.most; } - traverse_forward(tb, stack, lastkey, lk_base, &doit_select_chunk, &sc); + traverse_forward(tb, stack, lastkey, &doit_select_chunk, &sc); } release_stack(tb,stack); @@ -1483,9 +1469,9 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl, } key = GETKEY(tb, sc.lastobj); - sz = size_object_rel(key, sc.lastobj); + sz = size_object(key); hp = HAlloc(p, 9 + sz + PROC_BIN_SIZE); - key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL); + key = copy_struct(key, sz, &hp, &MSO(p)); if (mpi.all_objects) (mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS; mpb = db_make_mp_binary(p,mpi.mp,&hp); @@ -1508,9 +1494,9 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl, } key = GETKEY(tb, sc.lastobj); - sz = size_object_rel(key, sc.lastobj); + sz = size_object(key); hp = HAlloc(p, 9 + sz + PROC_BIN_SIZE); - key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL); + key = copy_struct(key, sz, &hp, &MSO(p)); if (mpi.all_objects) (mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS; @@ -1586,7 +1572,7 @@ static int db_select_delete_continue_tree(Process *p, sc.keypos = tb->common.keypos; ASSERT(!erts_smp_atomic_read_nob(&tb->is_stack_busy)); - traverse_backwards(tb, &tb->static_stack, lastkey, NULL, &doit_select_delete, &sc); + traverse_backwards(tb, &tb->static_stack, lastkey, &doit_select_delete, &sc); BUMP_REDS(p, 1000 - sc.max); @@ -1595,11 +1581,11 @@ static int db_select_delete_continue_tree(Process *p, } key = GETKEY(tb, (sc.lastterm)->dbterm.tpl); if (end_condition != NIL && - cmp_partly_bound(end_condition,key,sc.lastterm->dbterm.tpl) > 0) { /* done anyway */ + cmp_partly_bound(end_condition,key) > 0) { /* done anyway */ RET_TO_BIF(erts_make_integer(sc.accum,p),DB_ERROR_NONE); } /* Not done yet, let's trap. */ - sz = size_object_rel(key, sc.lastterm->dbterm.tpl); + sz = size_object(key); if (IS_USMALL(0, sc.accum)) { hp = HAlloc(p, sz + 6); eaccsum = make_small(sc.accum); @@ -1609,7 +1595,7 @@ static int db_select_delete_continue_tree(Process *p, eaccsum = uint_to_big(sc.accum, hp); hp += BIG_UINT_HEAP_SIZE; } - key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastterm->dbterm.tpl, NULL); + key = copy_struct(key, sz, &hp, &MSO(p)); continuation = TUPLE5 (hp, tptr[1], @@ -1630,7 +1616,6 @@ static int db_select_delete_tree(Process *p, DbTable *tbl, struct select_delete_context sc; struct mp_info mpi; Eterm lastkey = THE_NON_VALUE; - Eterm* lk_base = NULL; Eterm key; Eterm continuation; unsigned sz; @@ -1683,12 +1668,11 @@ static int db_select_delete_tree(Process *p, DbTable *tbl, if (mpi.some_limitation) { if ((this = find_next_from_pb_key(tb, &tb->static_stack, mpi.most)) != NULL) { lastkey = GETKEY(tb, this->dbterm.tpl); - lk_base = this->dbterm.tpl; } sc.end_condition = mpi.least; } - traverse_backwards(tb, &tb->static_stack, lastkey, lk_base, &doit_select_delete, &sc); + traverse_backwards(tb, &tb->static_stack, lastkey, &doit_select_delete, &sc); BUMP_REDS(p, 1000 - sc.max); if (sc.max > 0) { @@ -1696,7 +1680,7 @@ static int db_select_delete_tree(Process *p, DbTable *tbl, } key = GETKEY(tb, (sc.lastterm)->dbterm.tpl); - sz = size_object_rel(key, sc.lastterm->dbterm.tpl); + sz = size_object(key); if (IS_USMALL(0, sc.accum)) { hp = HAlloc(p, sz + PROC_BIN_SIZE + 6); eaccsum = make_small(sc.accum); @@ -1706,7 +1690,7 @@ static int db_select_delete_tree(Process *p, DbTable *tbl, eaccsum = uint_to_big(sc.accum, hp); hp += BIG_UINT_HEAP_SIZE; } - key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastterm->dbterm.tpl, NULL); + key = copy_struct(key, sz, &hp, &MSO(p)); mpb = db_make_mp_binary(p,mpi.mp,&hp); continuation = TUPLE5 @@ -1734,7 +1718,7 @@ static int db_take_tree(Process *p, DbTable *tbl, Eterm key, Eterm *ret) TreeDbTerm *this; *ret = NIL; - this = linkout_tree(tb, key, NULL); + this = linkout_tree(tb, key); if (this) { Eterm copy, *hp, *hend; @@ -1845,9 +1829,7 @@ do_db_tree_foreach_offheap(TreeDbTerm *tdbt, do_db_tree_foreach_offheap(tdbt->right, func, arg); } -static TreeDbTerm *linkout_tree(DbTableTree *tb, - Eterm key, Eterm* key_base) -{ +static TreeDbTerm *linkout_tree(DbTableTree *tb, Eterm key) { TreeDbTerm **tstack[STACK_NEED]; int tpos = 0; int dstack[STACK_NEED+1]; @@ -1869,7 +1851,7 @@ static TreeDbTerm *linkout_tree(DbTableTree *tb, for (;;) { if (!*this) { /* Failure */ return NULL; - } else if ((c = cmp_key(tb, key, key_base, *this)) < 0) { + } else if ((c = cmp_key(tb, key, *this)) < 0) { dstack[dpos++] = DIR_LEFT; tstack[tpos++] = this; this = &((*this)->left); @@ -1933,7 +1915,7 @@ static TreeDbTerm *linkout_object_tree(DbTableTree *tb, for (;;) { if (!*this) { /* Failure */ return NULL; - } else if ((c = cmp_key(tb,key,NULL,*this)) < 0) { + } else if ((c = cmp_key(tb,key,*this)) < 0) { dstack[dpos++] = DIR_LEFT; tstack[tpos++] = this; this = &((*this)->left); @@ -2319,15 +2301,13 @@ done: * Find next and previous in sort order */ -static TreeDbTerm *find_next(DbTableTree *tb, DbTreeStack* stack, - Eterm key, Eterm* key_base) -{ +static TreeDbTerm *find_next(DbTableTree *tb, DbTreeStack* stack, Eterm key) { TreeDbTerm *this; TreeDbTerm *tmp; Sint c; if(( this = TOP_NODE(stack)) != NULL) { - if (!cmp_key_eq(tb,key,key_base,this)) { + if (!cmp_key_eq(tb,key,this)) { /* Start from the beginning */ stack->pos = stack->slot = 0; } @@ -2337,7 +2317,7 @@ static TreeDbTerm *find_next(DbTableTree *tb, DbTreeStack* stack, return NULL; for (;;) { PUSH_NODE(stack, this); - if (( c = cmp_key(tb,key,key_base,this) ) > 0) { + if (( c = cmp_key(tb,key,this) ) > 0) { if (this->right == NULL) /* We are at the previos and the element does not exist */ @@ -2377,15 +2357,13 @@ static TreeDbTerm *find_next(DbTableTree *tb, DbTreeStack* stack, return this; } -static TreeDbTerm *find_prev(DbTableTree *tb, DbTreeStack* stack, - Eterm key, Eterm* key_base) -{ +static TreeDbTerm *find_prev(DbTableTree *tb, DbTreeStack* stack, Eterm key) { TreeDbTerm *this; TreeDbTerm *tmp; Sint c; if(( this = TOP_NODE(stack)) != NULL) { - if (!cmp_key_eq(tb,key,key_base,this)) { + if (!cmp_key_eq(tb,key,this)) { /* Start from the beginning */ stack->pos = stack->slot = 0; } @@ -2395,7 +2373,7 @@ static TreeDbTerm *find_prev(DbTableTree *tb, DbTreeStack* stack, return NULL; for (;;) { PUSH_NODE(stack, this); - if (( c = cmp_key(tb,key,key_base,this) ) < 0) { + if (( c = cmp_key(tb,key,this) ) < 0) { if (this->left == NULL) /* We are at the next and the element does not exist */ @@ -2448,8 +2426,7 @@ static TreeDbTerm *find_next_from_pb_key(DbTableTree *tb, DbTreeStack* stack, return NULL; for (;;) { PUSH_NODE(stack, this); - if (( c = cmp_partly_bound(key,GETKEY(tb, this->dbterm.tpl), - this->dbterm.tpl) ) >= 0) { + if (( c = cmp_partly_bound(key,GETKEY(tb, this->dbterm.tpl))) >= 0) { if (this->right == NULL) { do { tmp = POP_NODE(stack); @@ -2482,8 +2459,7 @@ static TreeDbTerm *find_prev_from_pb_key(DbTableTree *tb, DbTreeStack* stack, return NULL; for (;;) { PUSH_NODE(stack, this); - if (( c = cmp_partly_bound(key,GETKEY(tb, this->dbterm.tpl), - this->dbterm.tpl) ) <= 0) { + if (( c = cmp_partly_bound(key,GETKEY(tb, this->dbterm.tpl))) <= 0) { if (this->left == NULL) { do { tmp = POP_NODE(stack); @@ -2514,10 +2490,10 @@ static TreeDbTerm *find_node(DbTableTree *tb, Eterm key) DbTreeStack* stack = get_static_stack(tb); if(!stack || EMPTY_NODE(stack) - || !cmp_key_eq(tb, key, NULL, (this=TOP_NODE(stack)))) { + || !cmp_key_eq(tb, key, (this=TOP_NODE(stack)))) { this = tb->root; - while (this != NULL && (res = cmp_key(tb,key,NULL,this)) != 0) { + while (this != NULL && (res = cmp_key(tb,key,this)) != 0) { if (res < 0) this = this->left; else @@ -2539,7 +2515,7 @@ static TreeDbTerm **find_node2(DbTableTree *tb, Eterm key) Sint res; this = &tb->root; - while ((*this) != NULL && (res = cmp_key(tb, key, NULL, *this)) != 0) { + while ((*this) != NULL && (res = cmp_key(tb, key, *this)) != 0) { if (res < 0) this = &((*this)->left); else @@ -2589,9 +2565,6 @@ db_lookup_dbterm_tree(Process *p, DbTable *tbl, Eterm key, Eterm obj, handle->flags = flags; handle->bp = (void**) pp; handle->new_size = (*pp)->dbterm.size; -#if HALFWORD_HEAP - handle->abs_vec = NULL; -#endif return 1; } @@ -2622,7 +2595,7 @@ db_finalize_dbterm_tree(int cret, DbUpdateHandle *handle) */ static void traverse_backwards(DbTableTree *tb, DbTreeStack* stack, - Eterm lastkey, Eterm* lk_base, + Eterm lastkey, int (*doit)(DbTableTree *, TreeDbTerm *, void *, @@ -2641,16 +2614,15 @@ static void traverse_backwards(DbTableTree *tb, this = this->right; } this = TOP_NODE(stack); - next = find_prev(tb, stack, GETKEY(tb, this->dbterm.tpl), - this->dbterm.tpl); + next = find_prev(tb, stack, GETKEY(tb, this->dbterm.tpl)); if (!((*doit)(tb, this, context, 0))) return; } else { - next = find_prev(tb, stack, lastkey, lk_base); + next = find_prev(tb, stack, lastkey); } while ((this = next) != NULL) { - next = find_prev(tb, stack, GETKEY(tb, this->dbterm.tpl), this->dbterm.tpl); + next = find_prev(tb, stack, GETKEY(tb, this->dbterm.tpl)); if (!((*doit)(tb, this, context, 0))) return; } @@ -2661,7 +2633,7 @@ static void traverse_backwards(DbTableTree *tb, */ static void traverse_forward(DbTableTree *tb, DbTreeStack* stack, - Eterm lastkey, Eterm* lk_base, + Eterm lastkey, int (*doit)(DbTableTree *, TreeDbTerm *, void *, @@ -2680,15 +2652,15 @@ static void traverse_forward(DbTableTree *tb, this = this->left; } this = TOP_NODE(stack); - next = find_next(tb, stack, GETKEY(tb, this->dbterm.tpl), this->dbterm.tpl); + next = find_next(tb, stack, GETKEY(tb, this->dbterm.tpl)); if (!((*doit)(tb, this, context, 1))) return; } else { - next = find_next(tb, stack, lastkey, lk_base); + next = find_next(tb, stack, lastkey); } while ((this = next) != NULL) { - next = find_next(tb, stack, GETKEY(tb, this->dbterm.tpl), this->dbterm.tpl); + next = find_next(tb, stack, GETKEY(tb, this->dbterm.tpl)); if (!((*doit)(tb, this, context, 1))) return; } @@ -2725,7 +2697,7 @@ static int key_given(DbTableTree *tb, Eterm pattern, TreeDbTerm **ret, -static Sint do_cmp_partly_bound(Eterm a, Eterm b, Eterm* b_base, int *done) +static Sint do_cmp_partly_bound(Eterm a, Eterm b, int *done) { Eterm* aa; Eterm* bb; @@ -2739,44 +2711,44 @@ static Sint do_cmp_partly_bound(Eterm a, Eterm b, Eterm* b_base, int *done) *done = 1; return 0; } - if (is_same(a,NULL,b,b_base)) + if (is_same(a,b)) return 0; switch (a & _TAG_PRIMARY_MASK) { case TAG_PRIMARY_LIST: if (!is_list(b)) { - return cmp_rel(a,NULL,b,b_base); + return CMP(a,b); } aa = list_val(a); - bb = list_val_rel(b,b_base); + bb = list_val(b); while (1) { - if ((j = do_cmp_partly_bound(*aa++, *bb++, b_base, done)) != 0 || *done) + if ((j = do_cmp_partly_bound(*aa++, *bb++, done)) != 0 || *done) return j; - if (is_same(*aa, NULL, *bb, b_base)) + if (is_same(*aa, *bb)) return 0; if (is_not_list(*aa) || is_not_list(*bb)) - return do_cmp_partly_bound(*aa, *bb, b_base, done); + return do_cmp_partly_bound(*aa, *bb, done); aa = list_val(*aa); - bb = list_val_rel(*bb,b_base); + bb = list_val(*bb); } case TAG_PRIMARY_BOXED: if ((b & _TAG_PRIMARY_MASK) != TAG_PRIMARY_BOXED) { - return cmp_rel(a,NULL,b,b_base); + return CMP(a,b); } a_hdr = ((*boxed_val(a)) & _TAG_HEADER_MASK) >> _TAG_PRIMARY_SIZE; - b_hdr = ((*boxed_val_rel(b,b_base)) & _TAG_HEADER_MASK) >> _TAG_PRIMARY_SIZE; + b_hdr = ((*boxed_val(b)) & _TAG_HEADER_MASK) >> _TAG_PRIMARY_SIZE; if (a_hdr != b_hdr) { - return cmp_rel(a, NULL, b, b_base); + return CMP(a,b); } if (a_hdr == (_TAG_HEADER_ARITYVAL >> _TAG_PRIMARY_SIZE)) { aa = tuple_val(a); - bb = tuple_val_rel(b, b_base); + bb = tuple_val(b); /* compare the arities */ i = arityval(*aa); /* get the arity*/ if (i < arityval(*bb)) return(-1); if (i > arityval(*bb)) return(1); while (i--) { - if ((j = do_cmp_partly_bound(*++aa, *++bb, b_base, done)) != 0 + if ((j = do_cmp_partly_bound(*++aa, *++bb, done)) != 0 || *done) return j; } @@ -2784,14 +2756,13 @@ static Sint do_cmp_partly_bound(Eterm a, Eterm b, Eterm* b_base, int *done) } /* Drop through */ default: - return cmp_rel(a, NULL, b, b_base); + return CMP(a,b); } } -static Sint cmp_partly_bound(Eterm partly_bound_key, Eterm bound_key, Eterm* bk_base) -{ +static Sint cmp_partly_bound(Eterm partly_bound_key, Eterm bound_key) { int done = 0; - Sint ret = do_cmp_partly_bound(partly_bound_key, bound_key, bk_base, &done); + Sint ret = do_cmp_partly_bound(partly_bound_key, bound_key, &done); #ifdef HARDDEBUG erts_fprintf(stderr,"\ncmp_partly_bound: %T", partly_bound_key); if (ret < 0) @@ -2800,7 +2771,7 @@ static Sint cmp_partly_bound(Eterm partly_bound_key, Eterm bound_key, Eterm* bk_ erts_fprintf(stderr," > "); else erts_fprintf(stderr," == "); - erts_fprintf(stderr,"%R\n", bound_key, bk_base); + erts_fprintf(stderr,"%T\n", bound_key); #endif return ret; } @@ -3017,12 +2988,10 @@ static int doit_select(DbTableTree *tb, TreeDbTerm *this, void *ptr, if (sc->end_condition != NIL && ((forward && cmp_partly_bound(sc->end_condition, - GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl), - this->dbterm.tpl) < 0) || + GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl)) < 0) || (!forward && cmp_partly_bound(sc->end_condition, - GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl), - this->dbterm.tpl) > 0))) { + GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl)) > 0))) { return 0; } ret = db_match_dbterm(&tb->common,sc->p,sc->mp,sc->all_objects, @@ -3054,8 +3023,7 @@ static int doit_select_count(DbTableTree *tb, TreeDbTerm *this, void *ptr, /* Always backwards traversing */ if (sc->end_condition != NIL && (cmp_partly_bound(sc->end_condition, - GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl), - this->dbterm.tpl) > 0)) { + GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl)) > 0)) { return 0; } ret = db_match_dbterm(&tb->common, sc->p, sc->mp, 0, @@ -3081,12 +3049,10 @@ static int doit_select_chunk(DbTableTree *tb, TreeDbTerm *this, void *ptr, if (sc->end_condition != NIL && ((forward && cmp_partly_bound(sc->end_condition, - GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl), - this->dbterm.tpl) < 0) || + GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl)) < 0) || (!forward && cmp_partly_bound(sc->end_condition, - GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl), - this->dbterm.tpl) > 0))) { + GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl)) > 0))) { return 0; } @@ -3124,14 +3090,13 @@ static int doit_select_delete(DbTableTree *tb, TreeDbTerm *this, void *ptr, if (sc->end_condition != NIL && cmp_partly_bound(sc->end_condition, - GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl), - this->dbterm.tpl) > 0) + GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl)) > 0) return 0; ret = db_match_dbterm(&tb->common, sc->p, sc->mp, 0, &this->dbterm, NULL, 0); if (ret == am_true) { key = GETKEY(sc->tb, this->dbterm.tpl); - linkout_tree(sc->tb, key, this->dbterm.tpl); + linkout_tree(sc->tb, key); sc->erase_lastterm = 1; ++sc->accum; } @@ -3157,11 +3122,10 @@ static void do_dump_tree2(DbTableTree* tb, int to, void *to_arg, int show, } else { prefix = ""; - term = make_tuple_rel(t->dbterm.tpl,t->dbterm.tpl); + term = make_tuple(t->dbterm.tpl); } - erts_print(to, to_arg, "%*s%s%R (addr = %p, bal = %d)\n", - offset, "", prefix, term, t->dbterm.tpl, - t, t->balance); + erts_print(to, to_arg, "%*s%s%T (addr = %p, bal = %d)\n", + offset, "", prefix, term, t, t->balance); } do_dump_tree2(tb, to, to_arg, show, t->left, offset + 4); } diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index dab357a079..399be6058c 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -239,11 +239,7 @@ typedef enum { matchCall2, matchCall3, matchPushV, -#if HALFWORD_HEAP - matchPushVGuard, /* First guard-only variable reference */ -#endif - matchPushVResult, /* First variable reference in result, or (if HALFWORD) - in guard if also referenced in result */ + matchPushVResult, /* First variable reference in result */ matchPushExpr, /* Push the whole expression we're matching ('$_') */ matchPushArrayAsList, /* Only when parameter is an Array and not an erlang term (DCOMP_TRACE) */ @@ -310,9 +306,6 @@ DMC_DECLARE_STACK_TYPE(unsigned); typedef struct DMCVariable { int is_bound; int is_in_body; -#if HALFWORD_HEAP - int first_guard_label; /* to maybe change from PushVGuard to PushVResult */ -#endif } DMCVariable; typedef struct DMCHeap { @@ -374,7 +367,6 @@ typedef struct MatchVariable { Eterm term; #ifdef DEBUG Process* proc; - Eterm* base; #endif } MatchVariable; @@ -415,7 +407,7 @@ cleanup_match_pseudo_process(ErtsMatchPseudoProcess *mpsp, int keep_heap) else { int i; for (i = 0; i < ERTS_DEFAULT_MS_HEAP_SIZE; i++) { -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) mpsp->default_heap[i] = (Eterm) 0xdeadbeefdeadbeef; #else mpsp->default_heap[i] = (Eterm) 0xdeadbeef; @@ -1240,7 +1232,7 @@ Eterm erts_match_set_run(Process *p, Binary *mpsp, { Eterm ret; - ret = db_prog_match(p, mpsp, NIL, NULL, args, num_args, + ret = db_prog_match(p, mpsp, NIL, args, num_args, in_flags, return_flags); #if defined(HARDDEBUG) if (is_non_value(ret)) { @@ -1265,7 +1257,7 @@ static Eterm erts_match_set_run_ets(Process *p, Binary *mpsp, { Eterm ret; - ret = db_prog_match(p, mpsp, args, NULL, NULL, num_args, + ret = db_prog_match(p, mpsp, args, NULL, num_args, ERTS_PAM_COPY_RESULT, return_flags); #if defined(HARDDEBUG) @@ -1755,71 +1747,6 @@ static Eterm dpm_array_to_list(Process *psp, Eterm *arr, int arity) return ret; } - -#if HALFWORD_HEAP -struct heap_checkpoint_t -{ - Process *p; - Eterm* htop; - ErlHeapFragment* mbuf; - unsigned used_size; - ErlOffHeap off_heap; -}; - -static void heap_checkpoint_init(Process* p, struct heap_checkpoint_t* hcp) -{ - hcp->p = p; - hcp->htop = HEAP_TOP(p); - hcp->mbuf = MBUF(p); - hcp->used_size = hcp->mbuf ? hcp->mbuf->used_size : 0; - hcp->off_heap = MSO(p); -} - -static void heap_checkpoint_revert(struct heap_checkpoint_t* hcp) -{ - struct erl_off_heap_header* oh = MSO(hcp->p).first; - - if (oh != hcp->off_heap.first) { - ASSERT(oh != NULL); - if (hcp->off_heap.first) { - while (oh->next != hcp->off_heap.first) { - oh = oh->next; - } - oh->next = NULL; - } - erts_cleanup_offheap(&MSO(hcp->p)); - MSO(hcp->p) = hcp->off_heap; - } - if (MBUF(hcp->p) != hcp->mbuf) { - ErlHeapFragment* hf = MBUF(hcp->p); - ASSERT(hf != NULL); - if (hcp->mbuf) { - while (hf->next != hcp->mbuf) { - hf = hf->next; - } - hf->next = NULL; - } - free_message_buffer(MBUF(hcp->p)); - MBUF(hcp->p) = hcp->mbuf; - } - if (hcp->mbuf != NULL && hcp->mbuf->used_size != hcp->used_size) { - hcp->mbuf->used_size = hcp->used_size; - } - HEAP_TOP(hcp->p) = hcp->htop; -} -#endif /* HALFWORD_HEAP */ - -static ERTS_INLINE Eterm copy_object_rel(Process* p, Eterm term, Eterm* base) -{ - if (!is_immed(term)) { - Uint sz = size_object_rel(term, base); - Eterm* top = HAllocX(p, sz, HEAP_XTRA); - return copy_struct_rel(term, sz, &top, &MSO(p), base, NULL); - } - return term; -} - - /* ** Execution of the match program, this is Pam. ** May return THE_NON_VALUE, which is a bailout. @@ -1827,7 +1754,7 @@ static ERTS_INLINE Eterm copy_object_rel(Process* p, Eterm term, Eterm* base) ** i.e. 'DCOMP_TRACE' was specified */ Eterm db_prog_match(Process *c_p, Binary *bprog, - Eterm term, Eterm* base, + Eterm term, Eterm *termp, int arity, enum erts_pam_run_flags in_flags, @@ -1855,17 +1782,12 @@ Eterm db_prog_match(Process *c_p, Binary *bprog, Eterm bif_args[3]; int fail_label; int atomic_trace; -#if HALFWORD_HEAP - struct heap_checkpoint_t c_p_checkpoint = {}; -#endif #ifdef DMC_DEBUG Uint *heap_fence; Uint *stack_fence; Uint save_op; #endif /* DMC_DEBUG */ - ASSERT(base==NULL || HALFWORD_HEAP); - mpsp = get_match_pseudo_process(c_p, prog->heap_size); psp = &mpsp->process; @@ -1916,11 +1838,7 @@ Eterm db_prog_match(Process *c_p, Binary *bprog, do_catch != 0 */ *return_flags = 0U; - variables = mpsp->u.variables; -#if HALFWORD_HEAP - c_p_checkpoint.p = NULL; -#endif restart: ep = &term; @@ -1931,14 +1849,12 @@ restart: fail_label = -1; build_proc = psp; esdp->current_process = psp; - ASSERT_HALFWORD(!c_p_checkpoint.p); #ifdef DEBUG ASSERT(variables == mpsp->u.variables); for (i=0; i<prog->num_bindings; i++) { variables[i].term = THE_NON_VALUE; variables[i].proc = NULL; - variables[i].base = base; } #endif @@ -1974,9 +1890,9 @@ restart: variables[n].term = dpm_array_to_list(psp, termp, arity); break; case matchTuple: /* *ep is a tuple of arity n */ - if (!is_tuple_rel(*ep,base)) + if (!is_tuple(*ep)) FAIL(); - ep = tuple_val_rel(*ep,base); + ep = tuple_val(*ep); n = *pc++; if (arityval(*ep) != n) FAIL(); @@ -1984,9 +1900,9 @@ restart: break; case matchPushT: /* *ep is a tuple of arity n, push ptr to first element */ - if (!is_tuple_rel(*ep,base)) + if (!is_tuple(*ep)) FAIL(); - tp = tuple_val_rel(*ep,base); + tp = tuple_val(*ep); n = *pc++; if (arityval(*tp) != n) FAIL(); @@ -1996,51 +1912,51 @@ restart: case matchList: if (!is_list(*ep)) FAIL(); - ep = list_val_rel(*ep,base); + ep = list_val(*ep); break; case matchPushL: if (!is_list(*ep)) FAIL(); - *sp++ = list_val_rel(*ep,base); + *sp++ = list_val(*ep); ++ep; break; case matchMap: - if (!is_map_rel(*ep, base)) { + if (!is_map(*ep)) { FAIL(); } n = *pc++; - if (is_flatmap_rel(*ep,base)) { - if (flatmap_get_size(flatmap_val_rel(*ep, base)) < n) { + if (is_flatmap(*ep)) { + if (flatmap_get_size(flatmap_val(*ep)) < n) { FAIL(); } } else { - ASSERT(is_hashmap_rel(*ep,base)); - if (hashmap_size_rel(*ep, base) < n) { + ASSERT(is_hashmap(*ep)); + if (hashmap_size(*ep) < n) { FAIL(); } } - ep = flatmap_val_rel(*ep, base); + ep = flatmap_val(*ep); break; case matchPushM: - if (!is_map_rel(*ep, base)) { + if (!is_map(*ep)) { FAIL(); } n = *pc++; - if (is_flatmap_rel(*ep,base)) { - if (flatmap_get_size(flatmap_val_rel(*ep, base)) < n) { + if (is_flatmap(*ep)) { + if (flatmap_get_size(flatmap_val(*ep)) < n) { FAIL(); } } else { - ASSERT(is_hashmap_rel(*ep,base)); - if (hashmap_size_rel(*ep, base) < n) { + ASSERT(is_hashmap(*ep)); + if (hashmap_size(*ep) < n) { FAIL(); } } - *sp++ = flatmap_val_rel(*ep++, base); + *sp++ = flatmap_val(*ep++); break; case matchKey: t = (Eterm) *pc++; - tp = erts_maps_get_rel(t, make_boxed_rel(ep, base), base); + tp = erts_maps_get(t, make_boxed(ep)); if (!tp) { FAIL(); } @@ -2061,29 +1977,29 @@ restart: break; case matchCmp: n = *pc++; - if (!eq_rel(variables[n].term, base, *ep, base)) + if (!EQ(variables[n].term, *ep)) FAIL(); ++ep; break; case matchEqBin: t = (Eterm) *pc++; - if (!eq_rel(t,NULL,*ep,base)) + if (!EQ(t,*ep)) FAIL(); ++ep; break; case matchEqFloat: - if (!is_float_rel(*ep,base)) + if (!is_float(*ep)) FAIL(); - if (memcmp(float_val_rel(*ep,base) + 1, pc, sizeof(double))) + if (memcmp(float_val(*ep) + 1, pc, sizeof(double))) FAIL(); pc += TermWords(2); ++ep; break; case matchEqRef: { Eterm* epc = (Eterm*)pc; - if (!is_ref_rel(*ep,base)) + if (!is_ref(*ep)) FAIL(); - if (!eq_rel(make_internal_ref_rel(epc, epc), epc, *ep, base)) { + if (!EQ(make_internal_ref(epc), *ep)) { FAIL(); } i = thing_arityval(*epc); @@ -2092,9 +2008,9 @@ restart: break; } case matchEqBig: - if (!is_big_rel(*ep,base)) + if (!is_big(*ep)) FAIL(); - tp = big_val_rel(*ep,base); + tp = big_val(*ep); { Eterm *epc = (Eterm *) pc; if (*tp != *epc) @@ -2236,62 +2152,36 @@ restart: esp -= 2; esp[-1] = t; break; - - #if HALFWORD_HEAP - case matchPushVGuard: - if (!base) goto case_matchPushV; - /* Build NULL-based copy on pseudo heap for easy disposal */ - n = *pc++; - ASSERT(is_value(variables[n].term)); - ASSERT(!variables[n].proc); - variables[n].term = copy_object_rel(psp, variables[n].term, base); - *esp++ = variables[n].term; - #ifdef DEBUG - variables[n].proc = psp; - variables[n].base = NULL; - #endif - break; - #endif case matchPushVResult: if (!(in_flags & ERTS_PAM_COPY_RESULT)) goto case_matchPushV; - - /* Build (NULL-based) copy on callers heap */ - #if HALFWORD_HEAP - if (!do_catch && !c_p_checkpoint.p) { - heap_checkpoint_init(c_p, &c_p_checkpoint); - } - #endif + /* Build copy on callers heap */ n = *pc++; ASSERT(is_value(variables[n].term)); ASSERT(!variables[n].proc); - variables[n].term = copy_object_rel(c_p, variables[n].term, base); + variables[n].term = copy_object_x(variables[n].term, c_p, HEAP_XTRA); *esp++ = variables[n].term; #ifdef DEBUG variables[n].proc = c_p; - variables[n].base = NULL; #endif break; case matchPushV: case_matchPushV: n = *pc++; ASSERT(is_value(variables[n].term)); - ASSERT(!variables[n].base); *esp++ = variables[n].term; break; case matchPushExpr: if (in_flags & ERTS_PAM_COPY_RESULT) { Uint sz; Eterm* top; - sz = size_object_rel(term, base); + sz = size_object(term); top = HAllocX(build_proc, sz, HEAP_XTRA); if (in_flags & ERTS_PAM_CONTIGUOUS_TUPLE) { - ASSERT(is_tuple_rel(term,base)); - *esp++ = copy_shallow_rel(tuple_val_rel(term,base), sz, - &top, &MSO(build_proc), base); + ASSERT(is_tuple(term)); + *esp++ = copy_shallow(tuple_val(term), sz, &top, &MSO(build_proc)); } else { - *esp++ = copy_struct_rel(term, sz, &top, &MSO(build_proc), - base, NULL); + *esp++ = copy_struct(term, sz, &top, &MSO(build_proc)); } } else { @@ -2299,7 +2189,6 @@ restart: } break; case matchPushArrayAsList: - ASSERT_HALFWORD(base == NULL); n = arity; /* Only happens when 'term' is an array */ tp = termp; ehp = HAllocX(build_proc, n*2, HEAP_XTRA); @@ -2315,7 +2204,6 @@ restart: break; case matchPushArrayAsListU: /* This instruction is NOT efficient. */ - ASSERT_HALFWORD(base == NULL); *esp++ = dpm_array_to_list(build_proc, termp, arity); break; case matchTrue: @@ -2420,11 +2308,7 @@ restart: *esp++ = am_true; break; case matchIsSeqTrace: - if (SEQ_TRACE_TOKEN(c_p) != NIL -#ifdef USE_VM_PROBES - && SEQ_TRACE_TOKEN(c_p) != am_have_dt_utag -#endif - ) + if (have_seqtrace(SEQ_TRACE_TOKEN(c_p))) *esp++ = am_true; else *esp++ = am_false; @@ -2448,11 +2332,7 @@ restart: --esp; break; case matchGetSeqToken: - if (SEQ_TRACE_TOKEN(c_p) == NIL -#ifdef USE_VM_PROBES - || SEQ_TRACE_TOKEN(c_p) == am_have_dt_utag -#endif - ) + if (have_no_seqtrace(SEQ_TRACE_TOKEN(c_p))) *esp++ = NIL; else { Eterm sender = SEQ_TRACE_TOKEN_SENDER(c_p); @@ -2619,13 +2499,6 @@ restart: } } fail: -#if HALFWORD_HEAP - if (c_p_checkpoint.p) { - /* Dispose garbage built by guards on caller heap */ - heap_checkpoint_revert(&c_p_checkpoint); - c_p_checkpoint.p = NULL; - } -#endif *return_flags = 0U; if (fail_label >= 0) { /* We failed during a "TryMeElse", lets restart, with the next match @@ -2788,13 +2661,6 @@ Wterm db_do_read_element(DbUpdateHandle* handle, Sint position) { Eterm elem = handle->dbterm->tpl[position]; if (!is_header(elem)) { -#if HALFWORD_HEAP - if (!is_immed(elem) - && !handle->tb->common.compress - && !(handle->abs_vec && handle->abs_vec[position])) { - return rterm2wterm(elem, handle->dbterm->tpl); - } -#endif return elem; } @@ -2822,9 +2688,6 @@ void db_do_update_element(DbUpdateHandle* handle, Eterm* oldp; Uint newval_sz; Uint oldval_sz; -#if HALFWORD_HEAP - Eterm* old_base; -#endif if (is_both_immed(newval,oldval)) { handle->dbterm->tpl[position] = newval; @@ -2841,15 +2704,8 @@ void db_do_update_element(DbUpdateHandle* handle, handle->dbterm); handle->flags |= DB_MUST_RESIZE; oldval = handle->dbterm->tpl[position]; - #if HALFWORD_HEAP - old_base = NULL; - #endif } else { - #if HALFWORD_HEAP - ASSERT(!handle->abs_vec); - old_base = handle->dbterm->tpl; - #endif if (is_boxed(newval)) { newp = boxed_val(newval); switch (*newp & _TAG_HEADER_MASK) { @@ -2859,7 +2715,7 @@ void db_do_update_element(DbUpdateHandle* handle, case _TAG_HEADER_HEAP_BIN: newval_sz = header_arity(*newp) + 1; if (is_boxed(oldval)) { - oldp = boxed_val_rel(oldval,old_base); + oldp = boxed_val(oldval); switch (*oldp & _TAG_HEADER_MASK) { case _TAG_HEADER_POS_BIG: case _TAG_HEADER_NEG_BIG: @@ -2879,20 +2735,13 @@ void db_do_update_element(DbUpdateHandle* handle, } } } -#if HALFWORD_HEAP - else { - old_base = (handle->tb->common.compress - || (handle->abs_vec && handle->abs_vec[position])) ? - NULL : handle->dbterm->tpl; - } -#endif /* Not possible for simple memcpy or dbterm is already non-contiguous, */ /* need to realloc... */ newval_sz = is_immed(newval) ? 0 : size_object(newval); new_size_set: - oldval_sz = is_immed(oldval) ? 0 : size_object_rel(oldval,old_base); + oldval_sz = is_immed(oldval) ? 0 : size_object(oldval); both_size_set: handle->new_size = handle->new_size - oldval_sz + newval_sz; @@ -2900,19 +2749,6 @@ both_size_set: /* write new value in old dbterm, finalize will make a flat copy */ handle->dbterm->tpl[position] = newval; handle->flags |= DB_MUST_RESIZE; - -#if HALFWORD_HEAP - if (old_base && newval_sz > 0) { - ASSERT(!handle->tb->common.compress); - if (!handle->abs_vec) { - int i = header_arity(handle->dbterm->tpl[0]); - handle->abs_vec = erts_alloc(ERTS_ALC_T_TMP, (i+1)*sizeof(char)); - sys_memset(handle->abs_vec, 0, i+1); - /* abs_vec[0] not used */ - } - handle->abs_vec[position] = 1; - } -#endif } static ERTS_INLINE byte* db_realloc_term(DbTableCommon* tb, void* old, @@ -3011,7 +2847,7 @@ static void* copy_to_comp(DbTableCommon* tb, Eterm obj, DbTerm* dest, tpl[arity + 1] = alloc_size; tmp_offheap.first = NULL; - tpl[tb->keypos] = copy_struct_rel(key, size_object(key), &top.ep, &tmp_offheap, NULL, tpl); + tpl[tb->keypos] = copy_struct(key, size_object(key), &top.ep, &tmp_offheap); dest->first_oh = tmp_offheap.first; for (i=1; i<=arity; i++) { if (i != tb->keypos) { @@ -3030,7 +2866,7 @@ static void* copy_to_comp(DbTableCommon* tb, Eterm obj, DbTerm* dest, Eterm* dbg_top = erts_alloc(ERTS_ALC_T_DB_TERM, dest->size * sizeof(Eterm)); dest->debug_clone = dbg_top; tmp_offheap.first = dest->first_oh; - copy_struct_rel(obj, dest->size, &dbg_top, &tmp_offheap, NULL, dbg_top); + copy_struct(obj, dest->size, &dbg_top, &tmp_offheap); dest->first_oh = tmp_offheap.first; ASSERT(dbg_top == dest->debug_clone + dest->size); } @@ -3077,7 +2913,7 @@ void* db_store_term(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj) newp->size = size; top = newp->tpl; tmp_offheap.first = NULL; - copy_struct_rel(obj, size, &top, &tmp_offheap, NULL, top); + copy_struct(obj, size, &top, &tmp_offheap); newp->first_oh = tmp_offheap.first; #ifdef DEBUG_CLONE newp->debug_clone = NULL; @@ -3160,29 +2996,8 @@ void db_finalize_resize(DbUpdateHandle* handle, Uint offset) tmp_offheap.first = NULL; - #if HALFWORD_HEAP - if (handle->abs_vec) { - int i, arity = header_arity(handle->dbterm->tpl[0]); - - top[0] = tpl[0]; - top += arity + 1; - for (i=1; i<=arity; i++) { - Eterm* src_base = handle->abs_vec[i] ? NULL : tpl; - - newDbTerm->tpl[i] = copy_struct_rel(tpl[i], - size_object_rel(tpl[i],src_base), - &top, &tmp_offheap, src_base, - newDbTerm->tpl); - } - newDbTerm->first_oh = tmp_offheap.first; - ASSERT((byte*)top <= (newp + alloc_sz)); - erts_free(ERTS_ALC_T_TMP, handle->abs_vec); - } - else - #endif /* HALFWORD_HEAP */ { - copy_struct_rel(make_tuple_rel(tpl,tpl), handle->new_size, &top, - &tmp_offheap, tpl, top); + copy_struct(make_tuple(tpl), handle->new_size, &top, &tmp_offheap); newDbTerm->first_oh = tmp_offheap.first; ASSERT((byte*)top == (newp + alloc_sz)); } @@ -3199,9 +3014,9 @@ Eterm db_copy_from_comp(DbTableCommon* tb, DbTerm* bp, Eterm** hpp, hp[0] = bp->tpl[0]; *hpp += arity + 1; - hp[tb->keypos] = copy_struct_rel(bp->tpl[tb->keypos], - size_object_rel(bp->tpl[tb->keypos], bp->tpl), - hpp, off_heap, bp->tpl, NULL); + hp[tb->keypos] = copy_struct(bp->tpl[tb->keypos], + size_object(bp->tpl[tb->keypos]), + hpp, off_heap); erts_factory_static_init(&factory, *hpp, bp->size - (arity+1), off_heap); @@ -3221,7 +3036,7 @@ Eterm db_copy_from_comp(DbTableCommon* tb, DbTerm* bp, Eterm** hpp, ASSERT((*hpp - hp) <= bp->size); #ifdef DEBUG_CLONE - ASSERT(eq_rel(make_tuple(hp),NULL,make_tuple(bp->debug_clone),bp->debug_clone)); + ASSERT(EQ(make_tuple(hp),make_tuple(bp->debug_clone))); #endif return make_tuple(hp); } @@ -3245,14 +3060,14 @@ Eterm db_copy_element_from_ets(DbTableCommon* tb, Process* p, *hpp = erts_produce_heap(&factory, extra, 0); erts_factory_close(&factory); #ifdef DEBUG_CLONE - ASSERT(eq_rel(copy, NULL, obj->debug_clone[pos], obj->debug_clone)); + ASSERT(EQ(copy, obj->debug_clone[pos])); #endif return copy; } else { - Uint sz = size_object_rel(obj->tpl[pos], obj->tpl); + Uint sz = size_object(obj->tpl[pos]); *hpp = HAlloc(p, sz + extra); - return copy_struct_rel(obj->tpl[pos], sz, hpp, &MSO(p), obj->tpl, NULL); + return copy_struct(obj->tpl[pos], sz, hpp, &MSO(p)); } } @@ -3599,26 +3414,10 @@ static DMCRet dmc_one_term(DMCContext *context, { Eterm* ref_val = internal_ref_val(c); DMC_PUSH(*text, matchEqRef); -#if HALFWORD_HEAP - { - union { - UWord u; - Uint t[2]; - } fiddle; - ASSERT(thing_arityval(ref_val[0]) == 3); - fiddle.t[0] = ref_val[0]; - fiddle.t[1] = ref_val[1]; - DMC_PUSH(*text, fiddle.u); - fiddle.t[0] = ref_val[2]; - fiddle.t[1] = ref_val[3]; - DMC_PUSH(*text, fiddle.u); - } -#else n = thing_arityval(ref_val[0]); for (i = 0; i <= n; ++i) { DMC_PUSH(*text, ref_val[i]); } -#endif break; } case (_TAG_HEADER_POS_BIG >> _TAG_PRIMARY_SIZE): @@ -3627,53 +3426,19 @@ static DMCRet dmc_one_term(DMCContext *context, Eterm* bval = big_val(c); n = thing_arityval(bval[0]); DMC_PUSH(*text, matchEqBig); -#if HALFWORD_HEAP - { - union { - UWord u; - Uint t[2]; - } fiddle; - ASSERT(n >= 1); - fiddle.t[0] = bval[0]; - fiddle.t[1] = bval[1]; - DMC_PUSH(*text, fiddle.u); - for (i = 2; i <= n; ++i) { - fiddle.t[0] = bval[i]; - if (++i <= n) { - fiddle.t[1] = bval[i]; - } else { - fiddle.t[1] = (Uint) 0; - } - DMC_PUSH(*text, fiddle.u); - } - } -#else for (i = 0; i <= n; ++i) { DMC_PUSH(*text, (Uint) bval[i]); } -#endif break; } case (_TAG_HEADER_FLOAT >> _TAG_PRIMARY_SIZE): DMC_PUSH(*text,matchEqFloat); -#if HALFWORD_HEAP - { - union { - UWord u; - Uint t[2]; - } fiddle; - fiddle.t[0] = float_val(c)[1]; - fiddle.t[1] = float_val(c)[2]; - DMC_PUSH(*text, fiddle.u); - } -#else DMC_PUSH(*text, (Uint) float_val(c)[1]); #ifdef ARCH_64 DMC_PUSH(*text, (Uint) 0); #else DMC_PUSH(*text, (Uint) float_val(c)[2]); #endif -#endif break; default: /* BINARY, FUN, VECTOR, or EXTERNAL */ DMC_PUSH(*text, matchEqBin); @@ -3997,24 +3762,8 @@ static void dmc_add_pushv_variant(DMCContext *context, DMCHeap *heap, MatchOps instr = matchPushV; ASSERT(n < heap->vars_used && v->is_bound); - if (context->is_guard) { - #if HALFWORD_HEAP - if (!v->first_guard_label) { - v->first_guard_label = DMC_STACK_NUM(*text); - ASSERT(v->first_guard_label); - instr = matchPushVGuard; /* may be changed to PushVResult below */ - } - #endif - } - else { /* body */ - #if HALFWORD_HEAP - if (v->first_guard_label) { - /* Avoid double-copy, copy to result heap at first encounter in guard */ - DMC_POKE(*text, v->first_guard_label, matchPushVResult); - v->is_in_body = 1; - } - #endif - if (!v->is_in_body) { + if (!context->is_guard) { + if(!v->is_in_body) { instr = matchPushVResult; v->is_in_body = 1; } @@ -5443,16 +5192,13 @@ Eterm db_match_dbterm(DbTableCommon* tb, Process* c_p, Binary* bprog, int all, DbTerm* obj, Eterm** hpp, Uint extra) { Uint32 dummy; - Eterm* base; Eterm res; if (tb->compress) { obj = db_alloc_tmp_uncompressed(tb, obj); - base = NULL; } - else base = HALFWORD_HEAP ? obj->tpl : NULL; - res = db_prog_match(c_p, bprog, make_tuple_rel(obj->tpl,base), base, NULL, 0, + res = db_prog_match(c_p, bprog, make_tuple(obj->tpl), NULL, 0, ERTS_PAM_COPY_RESULT|ERTS_PAM_CONTIGUOUS_TUPLE, &dummy); if (is_value(res) && hpp!=NULL) { @@ -5573,7 +5319,7 @@ void db_match_dis(Binary *bp) first = 0; else erts_printf(", "); -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) erts_printf("0x%016bex", rt->data.ui[ri]); #else erts_printf("0x%08bex", rt->data.ui[ri]); @@ -5597,7 +5343,7 @@ void db_match_dis(Binary *bp) first = 0; else erts_printf(", "); -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) erts_printf("0x%016bex", *et); #else erts_printf("0x%08bex", *et); @@ -5720,13 +5466,6 @@ void db_match_dis(Binary *bp) ++t; erts_printf("PushV\t%beu\n", n); break; - #if HALFWORD_HEAP - case matchPushVGuard: - n = (Uint) *++t; - ++t; - erts_printf("PushVGuard\t%beu\n", n); - break; - #endif case matchPushVResult: n = (Uint) *++t; ++t; diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h index 0903a40460..911ed37aef 100644 --- a/erts/emulator/beam/erl_db_util.h +++ b/erts/emulator/beam/erl_db_util.h @@ -90,9 +90,6 @@ typedef struct { Uint new_size; int flags; void* lck; -#if HALFWORD_HEAP - unsigned char* abs_vec; /* [i] true if dbterm->tpl[i] is absolute Eterm */ -#endif } DbUpdateHandle; @@ -290,10 +287,10 @@ ERTS_GLB_INLINE Eterm db_copy_key(Process* p, DbTable* tb, DbTerm* obj) Eterm key = GETKEY(tb, obj->tpl); if IS_CONST(key) return key; else { - Uint size = size_object_rel(key, obj->tpl); + Uint size = size_object(key); Eterm* hp = HAlloc(p, size); - Eterm res = copy_struct_rel(key, size, &hp, &MSO(p), obj->tpl, NULL); - ASSERT(eq_rel(res,NULL,key,obj->tpl)); + Eterm res = copy_struct(key, size, &hp, &MSO(p)); + ASSERT(EQ(res,key)); return res; } } @@ -305,14 +302,14 @@ ERTS_GLB_INLINE Eterm db_copy_object_from_ets(DbTableCommon* tb, DbTerm* bp, return db_copy_from_comp(tb, bp, hpp, off_heap); } else { - return copy_shallow_rel(bp->tpl, bp->size, hpp, off_heap, bp->tpl); + return copy_shallow(bp->tpl, bp->size, hpp, off_heap); } } ERTS_GLB_INLINE int db_eq(DbTableCommon* tb, Eterm a, DbTerm* b) { if (!tb->compress) { - return eq_rel(a, NULL, make_tuple_rel(b->tpl,b->tpl), b->tpl); + return EQ(a, make_tuple(b->tpl)); } else { return db_eq_comp(tb, a, b); @@ -438,7 +435,7 @@ Binary *db_match_compile(Eterm *matchexpr, Eterm *guards, Eterm db_match_dbterm(DbTableCommon* tb, Process* c_p, Binary* bprog, int all, DbTerm* obj, Eterm** hpp, Uint extra); -Eterm db_prog_match(Process *p, Binary *prog, Eterm term, Eterm* base, +Eterm db_prog_match(Process *p, Binary *prog, Eterm term, Eterm *termp, int arity, enum erts_pam_run_flags in_flags, Uint32 *return_flags /* Zeroed on enter */); diff --git a/erts/emulator/beam/erl_debug.c b/erts/emulator/beam/erl_debug.c index 2dcfb79f00..7b13cf2c83 100644 --- a/erts/emulator/beam/erl_debug.c +++ b/erts/emulator/beam/erl_debug.c @@ -312,6 +312,8 @@ void erts_check_for_holes(Process* p) p->last_htop = HEAP_TOP(p); for (hf = MBUF(p); hf != 0; hf = hf->next) { + if (hf == p->heap_hfrag) + continue; if (hf == p->last_mbuf) { break; } @@ -402,7 +404,7 @@ void verify_process(Process *p) erl_exit(1,"Wild pointer found in " name " of %T!\n",p->common.id); } - ErlMessage* mp = p->msg.first; + ErtsMessage* mp = p->msg.first; VERBOSE(DEBUG_MEMORY,("Verify process: %T...\n",p->common.id)); @@ -416,7 +418,7 @@ void verify_process(Process *p) erts_check_heap(p); if (p->dictionary) - VERIFY_AREA("dictionary",p->dictionary->data, p->dictionary->used); + VERIFY_AREA("dictionary", ERTS_PD_START(p->dictionary), ERTS_PD_SIZE(p->dictionary)); VERIFY_ETERM("seq trace token",p->seq_trace_token); VERIFY_ETERM("group leader",p->group_leader); VERIFY_ETERM("fvalue",p->fvalue); @@ -531,7 +533,7 @@ static void print_process_memory(Process *p) PTR_SIZE, "PCB", dashes, dashes, dashes, dashes); if (p->msg.first != NULL) { - ErlMessage* mp; + ErtsMessage* mp; erts_printf(" Message Queue:\n"); mp = p->msg.first; while (mp != NULL) { @@ -542,8 +544,8 @@ static void print_process_memory(Process *p) } if (p->dictionary != NULL) { - int n = p->dictionary->used; - Eterm *ptr = p->dictionary->data; + int n = ERTS_PD_SIZE(p->dictionary); + Eterm *ptr = ERTS_PD_START(p->dictionary); erts_printf(" Dictionary: "); while (n--) erts_printf("0x%0*lx ",PTR_SIZE,(unsigned long)ptr++); erts_printf("\n"); @@ -631,29 +633,4 @@ void print_memory_info(Process *p) } erts_printf("+-----------------%s-%s-%s-%s-+\n",dashes,dashes,dashes,dashes); } -#if !HEAP_ON_C_STACK && defined(DEBUG) -Eterm *erts_debug_allocate_tmp_heap(int size, Process *p) -{ - ErtsSchedulerData *sd = ((p == NULL) ? erts_get_scheduler_data() : ERTS_PROC_GET_SCHDATA(p)); - int offset = sd->num_tmp_heap_used; - - ASSERT(offset+size <= TMP_HEAP_SIZE); - return (sd->tmp_heap)+offset; -} -void erts_debug_use_tmp_heap(int size, Process *p) -{ - ErtsSchedulerData *sd = ((p == NULL) ? erts_get_scheduler_data() : ERTS_PROC_GET_SCHDATA(p)); - - sd->num_tmp_heap_used += size; - ASSERT(sd->num_tmp_heap_used <= TMP_HEAP_SIZE); -} -void erts_debug_unuse_tmp_heap(int size, Process *p) -{ - ErtsSchedulerData *sd = ((p == NULL) ? erts_get_scheduler_data() : ERTS_PROC_GET_SCHDATA(p)); - - sd->num_tmp_heap_used -= size; - ASSERT(sd->num_tmp_heap_used >= 0); -} -#endif #endif - diff --git a/erts/emulator/beam/erl_debug.h b/erts/emulator/beam/erl_debug.h index 4905e64f07..029320691d 100644 --- a/erts/emulator/beam/erl_debug.h +++ b/erts/emulator/beam/erl_debug.h @@ -48,6 +48,7 @@ #define DEBUG_THREADS 0x0010 /* Thread-related stuff */ #define DEBUG_PROCESSES 0x0020 /* Process creation and removal */ #define DEBUG_MEMORY 0x0040 /* Display results of memory checks */ +#define DEBUG_SHCOPY 0x0080 /* Sharing-preserving copying of terms */ extern Uint32 verbose; @@ -92,10 +93,4 @@ extern void print_tagged_memory(Eterm *start, Eterm *end); extern void print_untagged_memory(Eterm *start, Eterm *end); extern void print_memory(Process *p); extern void print_memory_info(Process *p); -#if defined(DEBUG) && !HEAP_ON_C_STACK -extern Eterm *erts_debug_allocate_tmp_heap(int, Process *); -extern void erts_debug_use_tmp_heap(int, Process *); -extern void erts_debug_unuse_tmp_heap(int, Process *); -#endif - #endif /* _ERL_DEBUG_H_ */ diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h index 7ab58e336b..0234b9b0e4 100644 --- a/erts/emulator/beam/erl_driver.h +++ b/erts/emulator/beam/erl_driver.h @@ -37,11 +37,6 @@ # endif #endif -/* This is OK to override by the NIF/driver implementor */ -#if defined(HALFWORD_HEAP_EMULATOR_SAVED__) && !defined(HALFWORD_HEAP_EMULATOR) -#define HALFWORD_HEAP_EMULATOR HALFWORD_HEAP_EMULATOR_SAVED__ -#endif - #include "erl_drv_nif.h" #include <stdlib.h> @@ -131,6 +126,7 @@ typedef struct { #define ERL_DRV_FLAG_USE_PORT_LOCKING (1 << 0) #define ERL_DRV_FLAG_SOFT_BUSY (1 << 1) #define ERL_DRV_FLAG_NO_BUSY_MSGQ (1 << 2) +#define ERL_DRV_FLAG_USE_INIT_ACK (1 << 3) /* * Integer types @@ -340,22 +336,23 @@ typedef struct erl_drv_entry { #ifdef STATIC_ERLANG_DRIVER # define ERLANG_DRIVER_NAME(NAME) NAME ## _driver_init +# define ERL_DRIVER_EXPORT #else # define ERLANG_DRIVER_NAME(NAME) driver_init +# if defined(__GNUC__) && __GNUC__ >= 4 +# define ERL_DRIVER_EXPORT __attribute__ ((visibility("default"))) +# elif defined (__SUNPRO_C) && (__SUNPRO_C >= 0x550) +# define ERL_DRIVER_EXPORT __global +# else +# define ERL_DRIVER_EXPORT +# endif #endif -/* For windows dynamic drivers */ #ifndef ERL_DRIVER_TYPES_ONLY -#if defined(__WIN32__) -# define DRIVER_INIT(DRIVER_NAME) \ - __declspec(dllexport) ErlDrvEntry* ERLANG_DRIVER_NAME(DRIVER_NAME)(void); \ - __declspec(dllexport) ErlDrvEntry* ERLANG_DRIVER_NAME(DRIVER_NAME)(void) -#else -# define DRIVER_INIT(DRIVER_NAME) \ - ErlDrvEntry* ERLANG_DRIVER_NAME(DRIVER_NAME)(void); \ - ErlDrvEntry* ERLANG_DRIVER_NAME(DRIVER_NAME)(void) -#endif +#define DRIVER_INIT(DRIVER_NAME) \ + ERL_DRIVER_EXPORT ErlDrvEntry* ERLANG_DRIVER_NAME(DRIVER_NAME)(void); \ + ERL_DRIVER_EXPORT ErlDrvEntry* ERLANG_DRIVER_NAME(DRIVER_NAME)(void) #define ERL_DRV_BUSY_MSGQ_DISABLED (~((ErlDrvSizeT) 0)) #define ERL_DRV_BUSY_MSGQ_READ_ONLY ((ErlDrvSizeT) 0) @@ -661,15 +658,11 @@ EXTERN char *driver_dl_error(void); EXTERN int erl_drv_putenv(const char *key, char *value); EXTERN int erl_drv_getenv(const char *key, char *value, size_t *value_size); -#ifdef __OSE__ -typedef ErlDrvUInt ErlDrvOseEventId; -EXTERN union SIGNAL *erl_drv_ose_get_signal(ErlDrvEvent ev); -EXTERN ErlDrvEvent erl_drv_ose_event_alloc(SIGSELECT sig, ErlDrvOseEventId handle, - ErlDrvOseEventId (*resolve_signal)(union SIGNAL *sig), void *extra); -EXTERN void erl_drv_ose_event_free(ErlDrvEvent ev); -EXTERN void erl_drv_ose_event_fetch(ErlDrvEvent ev, SIGSELECT *sig, - ErlDrvOseEventId *handle, void **extra); -#endif +/* spawn start init ack */ +EXTERN void erl_drv_init_ack(ErlDrvPort ix, ErlDrvData res); + +/* set the pid seen in port_info */ +EXTERN void erl_drv_set_os_pid(ErlDrvPort ix, ErlDrvSInt pid); #endif /* !ERL_DRIVER_TYPES_ONLY */ diff --git a/erts/emulator/beam/erl_fun.c b/erts/emulator/beam/erl_fun.c index 4268e2d40a..cff476694c 100644 --- a/erts/emulator/beam/erl_fun.c +++ b/erts/emulator/beam/erl_fun.c @@ -66,6 +66,9 @@ erts_init_fun_table(void) f.cmp = (HCMP_FUN) fun_cmp; f.alloc = (HALLOC_FUN) fun_alloc; f.free = (HFREE_FUN) fun_free; + f.meta_alloc = (HMALLOC_FUN) erts_alloc; + f.meta_free = (HMFREE_FUN) erts_free; + f.meta_print = (HMPRINT_FUN) erts_print; hash_init(ERTS_ALC_T_FUN_TABLE, &erts_fun_table, "fun_table", 16, f); } diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 2f21111a2e..21b03ae8bd 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -47,6 +47,17 @@ #define ERTS_INACT_WR_PB_LEAVE_LIMIT 10 #define ERTS_INACT_WR_PB_LEAVE_PERCENTAGE 10 +#if defined(DEBUG) || 0 +#define ERTS_GC_DEBUG +#else +#undef ERTS_GC_DEBUG +#endif +#ifdef ERTS_GC_DEBUG +# define ERTS_GC_ASSERT ASSERT +#else +# define ERTS_GC_ASSERT(B) ((void) 1) +#endif + /* * Returns number of elements in an array. */ @@ -67,10 +78,10 @@ #define ErtsGcQuickSanityCheck(P) \ do { \ ASSERT((P)->heap < (P)->hend); \ - ASSERT((P)->heap_sz == (P)->hend - (P)->heap); \ + ASSERT((p)->abandoned_heap || (P)->heap_sz == (P)->hend - (P)->heap); \ ASSERT((P)->heap <= (P)->htop && (P)->htop <= (P)->hend); \ ASSERT((P)->heap <= (P)->stop && (P)->stop <= (P)->hend); \ - ASSERT((P)->heap <= (P)->high_water && (P)->high_water <= (P)->hend);\ + ASSERT((p)->abandoned_heap || ((P)->heap <= (P)->high_water && (P)->high_water <= (P)->hend)); \ OverRunCheck((P)); \ } while (0) #else @@ -98,18 +109,33 @@ typedef struct { static Uint setup_rootset(Process*, Eterm*, int, Rootset*); static void cleanup_rootset(Rootset *rootset); -static Uint combined_message_size(Process* p); static void remove_message_buffers(Process* p); -static int major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl); -static int minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl); -static void do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj); -static Eterm* sweep_rootset(Rootset *rootset, Eterm* htop, char* src, Uint src_size); -static Eterm* sweep_one_area(Eterm* n_hp, Eterm* n_htop, char* src, Uint src_size); -static Eterm* sweep_one_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, - char* src, Uint src_size); -static Eterm* collect_heap_frags(Process* p, Eterm* heap, - Eterm* htop, Eterm* objv, int nobj); -static void adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj); +static Eterm *full_sweep_heaps(Process *p, + int hibernate, + Eterm *n_heap, Eterm* n_htop, + char *oh, Uint oh_size, + Eterm *objv, int nobj); +static int garbage_collect(Process* p, ErlHeapFragment *live_hf_end, + int need, Eterm* objv, int nobj); +static int major_collection(Process* p, ErlHeapFragment *live_hf_end, + int need, Eterm* objv, int nobj, Uint *recl); +static int minor_collection(Process* p, ErlHeapFragment *live_hf_end, + int need, Eterm* objv, int nobj, Uint *recl); +static void do_minor(Process *p, ErlHeapFragment *live_hf_end, + char *mature, Uint mature_size, + Uint new_sz, Eterm* objv, int nobj); +static Eterm *sweep_new_heap(Eterm *n_hp, Eterm *n_htop, + char* old_heap, Uint old_heap_size); +static Eterm *sweep_heaps(Eterm *n_hp, Eterm *n_htop, + char* old_heap, Uint old_heap_size); +static Eterm* sweep_literal_area(Eterm* n_hp, Eterm* n_htop, + char* old_heap, Uint old_heap_size, + char* src, Uint src_size); +static Eterm* sweep_literals_to_old_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, + char* src, Uint src_size); +static Eterm* collect_live_heap_frags(Process* p, ErlHeapFragment *live_hf_end, + Eterm* heap, Eterm* htop, Eterm* objv, int nobj); +static int adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj); static void shrink_new_heap(Process *p, Uint new_sz, Eterm *objv, int nobj); static void grow_new_heap(Process *p, Uint new_sz, Eterm* objv, int nobj); static void sweep_off_heap(Process *p, int fullsweep); @@ -119,16 +145,16 @@ static void offset_rootset(Process *p, Sint offs, char* area, Uint area_size, Eterm* objv, int nobj); static void offset_off_heap(Process* p, Sint offs, char* area, Uint area_size); static void offset_mqueue(Process *p, Sint offs, char* area, Uint area_size); +static void move_msgq_to_heap(Process *p); static void init_gc_info(ErtsGCInfo *gcip); #ifdef HARDDEBUG static void disallow_heap_frag_ref_in_heap(Process* p); static void disallow_heap_frag_ref_in_old_heap(Process* p); -static void disallow_heap_frag_ref(Process* p, Eterm* n_htop, Eterm* objv, int nobj); #endif -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) # define MAX_HEAP_SIZES 154 #else # define MAX_HEAP_SIZES 59 @@ -147,26 +173,24 @@ typedef struct { erts_smp_atomic32_t refc; } ErtsGCInfoReq; -#if !HALFWORD_HEAP -ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(gcireq, - ErtsGCInfoReq, - 5, - ERTS_ALC_T_GC_INFO_REQ) -#else -static ERTS_INLINE ErtsGCInfoReq * -gcireq_alloc(void) +static ERTS_INLINE int +gc_cost(Uint gc_moved_live_words, Uint resize_moved_words) { - return erts_alloc(ERTS_ALC_T_GC_INFO_REQ, - sizeof(ErtsGCInfoReq)); -} + Sint reds; -static ERTS_INLINE void -gcireq_free(ErtsGCInfoReq *ptr) -{ - erts_free(ERTS_ALC_T_GC_INFO_REQ, ptr); + reds = gc_moved_live_words/10; + reds += resize_moved_words/100; + if (reds < 1) + return 1; + if (reds > INT_MAX) + return INT_MAX; + return (int) reds; } -#endif +ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(gcireq, + ErtsGCInfoReq, + 5, + ERTS_ALC_T_GC_INFO_REQ) /* * Initialize GC global data. */ @@ -208,7 +232,7 @@ erts_init_gc(void) } - /* for 32 bit we want max_heap_size to be MAX(32bit) / 4 [words] (and halfword) + /* for 32 bit we want max_heap_size to be MAX(32bit) / 4 [words] * for 64 bit we want max_heap_size to be MAX(52bit) / 8 [words] */ @@ -232,10 +256,7 @@ erts_init_gc(void) init_gc_info(&esdp->gc_info); } -#if !HALFWORD_HEAP init_gcireq_alloc(); -#endif - } /* @@ -351,10 +372,19 @@ erts_offset_off_heap(ErlOffHeap *ohp, Sint offs, Eterm* low, Eterm* high) #undef ptr_within Eterm -erts_gc_after_bif_call(Process* p, Eterm result, Eterm* regs, Uint arity) +erts_gc_after_bif_call_lhf(Process* p, ErlHeapFragment *live_hf_end, + Eterm result, Eterm* regs, Uint arity) { int cost; + if (p->flags & F_HIBERNATE_SCHED) { + /* + * We just hibernated. We do *not* want to mess + * up the hibernation by an ordinary GC... + */ + return result; + } + if (is_non_value(result)) { if (p->freason == TRAP) { #if HIPE @@ -362,21 +392,28 @@ erts_gc_after_bif_call(Process* p, Eterm result, Eterm* regs, Uint arity) regs = ERTS_PROC_GET_SCHDATA(p)->x_reg_array; } #endif - cost = erts_garbage_collect(p, 0, regs, p->arity); + cost = garbage_collect(p, live_hf_end, 0, regs, p->arity); } else { - cost = erts_garbage_collect(p, 0, regs, arity); + cost = garbage_collect(p, live_hf_end, 0, regs, arity); } } else { Eterm val[1]; val[0] = result; - cost = erts_garbage_collect(p, 0, val, 1); + cost = garbage_collect(p, live_hf_end, 0, val, 1); result = val[0]; } BUMP_REDS(p, cost); return result; } +Eterm +erts_gc_after_bif_call(Process* p, Eterm result, Eterm* regs, Uint arity) +{ + return erts_gc_after_bif_call_lhf(p, ERTS_INVALID_HFRAG_PTR, + result, regs, arity); +} + static ERTS_INLINE void reset_active_writer(Process *p) { struct erl_off_heap_header* ptr; @@ -390,6 +427,139 @@ static ERTS_INLINE void reset_active_writer(Process *p) } } +#define ERTS_DELAY_GC_EXTRA_FREE 40 +#define ERTS_ABANDON_HEAP_COST 10 + +static int +delay_garbage_collection(Process *p, ErlHeapFragment *live_hf_end, int need) +{ + ErlHeapFragment *hfrag; + Eterm *orig_heap, *orig_hend, *orig_htop, *orig_stop; + Eterm *stop, *hend; + Uint hsz, ssz; + int reds_left; + + ERTS_HOLE_CHECK(p); + + if ((p->flags & F_DISABLE_GC) + && p->live_hf_end == ERTS_INVALID_HFRAG_PTR) { + /* + * A BIF yielded with disabled GC. Remember + * heap fragments created by the BIF until we + * do next GC. + */ + p->live_hf_end = live_hf_end; + } + + if (need == 0) + return 1; + + /* + * Satisfy need in a heap fragment... + */ + ASSERT(need > 0); + + orig_heap = p->heap; + orig_hend = p->hend; + orig_htop = p->htop; + orig_stop = p->stop; + + ssz = orig_hend - orig_stop; + hsz = ssz + need + ERTS_DELAY_GC_EXTRA_FREE; + + hfrag = new_message_buffer(hsz); + hfrag->next = p->mbuf; + p->mbuf = hfrag; + p->mbuf_sz += hsz; + p->heap = p->htop = &hfrag->mem[0]; + p->hend = hend = &hfrag->mem[hsz]; + p->stop = stop = hend - ssz; + sys_memcpy((void *) stop, (void *) orig_stop, ssz * sizeof(Eterm)); + + if (p->abandoned_heap) { + /* Active heap already in a fragment; adjust it... */ + ErlHeapFragment *hfrag = ((ErlHeapFragment *) + (((char *) orig_heap) + - offsetof(ErlHeapFragment, mem))); + Uint unused = orig_hend - orig_htop; + ASSERT(hfrag->used_size == hfrag->alloc_size); + ASSERT(hfrag->used_size >= unused); + hfrag->used_size -= unused; + p->mbuf_sz -= unused; + } + else { + /* Do not leave a hole in the abandoned heap... */ + if (orig_htop < orig_hend) { + *orig_htop = make_pos_bignum_header(orig_hend-orig_htop-1); + if (orig_htop + 1 < orig_hend) { + orig_hend[-1] = (Uint) (orig_htop - orig_heap); + p->flags |= F_ABANDONED_HEAP_USE; + } + } + p->abandoned_heap = orig_heap; + } + +#ifdef CHECK_FOR_HOLES + p->last_htop = p->htop; + p->heap_hfrag = hfrag; +#endif + + /* Make sure that we do a proper GC as soon as possible... */ + p->flags |= F_FORCE_GC; + reds_left = ERTS_BIF_REDS_LEFT(p); + if (reds_left > ERTS_ABANDON_HEAP_COST) { + int vreds = reds_left - ERTS_ABANDON_HEAP_COST; + ERTS_VBUMP_REDS(p, vreds); + } + return ERTS_ABANDON_HEAP_COST; +} + +static ERTS_FORCE_INLINE Uint +young_gen_usage(Process *p) +{ + Uint hsz; + Eterm *aheap; + + hsz = p->mbuf_sz; + + if (p->flags & F_ON_HEAP_MSGQ) { + ErtsMessage *mp; + for (mp = p->msg.first; mp; mp = mp->next) + if (mp->data.attached) + hsz += erts_msg_attached_data_size(mp); + } + + aheap = p->abandoned_heap; + if (!aheap) + hsz += p->htop - p->heap; + else { + /* used in orig heap */ + if (p->flags & F_ABANDONED_HEAP_USE) + hsz += aheap[p->heap_sz-1]; + else + hsz += p->heap_sz; + /* Remove unused part in latest fragment */ + hsz -= p->hend - p->htop; + } + return hsz; +} + +#define ERTS_GET_ORIG_HEAP(Proc, Heap, HTop) \ + do { \ + Eterm *aheap__ = (Proc)->abandoned_heap; \ + if (!aheap__) { \ + (Heap) = (Proc)->heap; \ + (HTop) = (Proc)->htop; \ + } \ + else { \ + (Heap) = aheap__; \ + if ((Proc)->flags & F_ABANDONED_HEAP_USE) \ + (HTop) = aheap__ + aheap__[(Proc)->heap_sz-1]; \ + else \ + (HTop) = aheap__ + (Proc)->heap_sz; \ + } \ + } while (0) + /* * Garbage collect a process. * @@ -398,21 +568,28 @@ static ERTS_INLINE void reset_active_writer(Process *p) * objv: Array of terms to add to rootset; that is to preserve. * nobj: Number of objects in objv. */ -int -erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj) +static int +garbage_collect(Process* p, ErlHeapFragment *live_hf_end, + int need, Eterm* objv, int nobj) { Uint reclaimed_now = 0; - int done = 0; + int reds; ErtsMonotonicTime start_time = 0; /* Shut up faulty warning... */ ErtsSchedulerData *esdp; + ERTS_MSACC_PUSH_STATE_M(); #ifdef USE_VM_PROBES DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE); #endif - if (p->flags & F_DISABLE_GC) { - ASSERT(need == 0); - return 1; - } + if (p->flags & (F_DISABLE_GC|F_DELAY_GC)) + return delay_garbage_collection(p, live_hf_end, need); + + if (p->abandoned_heap) + live_hf_end = ERTS_INVALID_HFRAG_PTR; + else if (p->live_hf_end != ERTS_INVALID_HFRAG_PTR) + live_hf_end = p->live_hf_end; + + ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_GC); esdp = erts_get_scheduler_data(); @@ -420,16 +597,14 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj) trace_gc(p, am_gc_start); } - (void) erts_smp_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC); + erts_smp_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC); if (erts_system_monitor_long_gc != 0) start_time = erts_get_monotonic_time(esdp); ERTS_CHK_OFFHEAP(p); ErtsGcQuickSanityCheck(p); - if (GEN_GCS(p) >= MAX_GEN_GCS(p)) { - FLAGS(p) |= F_NEED_FULLSWEEP; - } + #ifdef USE_VM_PROBES *pidbuf = '\0'; if (DTRACE_ENABLED(gc_major_start) @@ -442,17 +617,23 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj) /* * Test which type of GC to do. */ - while (!done) { - if ((FLAGS(p) & F_NEED_FULLSWEEP) != 0) { - DTRACE2(gc_major_start, pidbuf, need); - done = major_collection(p, need, objv, nobj, &reclaimed_now); - DTRACE2(gc_major_end, pidbuf, reclaimed_now); - } else { - DTRACE2(gc_minor_start, pidbuf, need); - done = minor_collection(p, need, objv, nobj, &reclaimed_now); - DTRACE2(gc_minor_end, pidbuf, reclaimed_now); - } + + if (GEN_GCS(p) < MAX_GEN_GCS(p) && !(FLAGS(p) & F_NEED_FULLSWEEP)) { + DTRACE2(gc_minor_start, pidbuf, need); + reds = minor_collection(p, live_hf_end, need, objv, nobj, &reclaimed_now); + DTRACE2(gc_minor_end, pidbuf, reclaimed_now); + if (reds < 0) + goto do_major_collection; + } + else { + do_major_collection: + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_GC_FULL); + DTRACE2(gc_major_start, pidbuf, need); + reds = major_collection(p, live_hf_end, need, objv, nobj, &reclaimed_now); + DTRACE2(gc_major_end, pidbuf, reclaimed_now); + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_GC); } + reset_active_writer(p); /* @@ -491,6 +672,9 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj) esdp->gc_info.reclaimed += reclaimed_now; FLAGS(p) &= ~F_FORCE_GC; + p->live_hf_end = ERTS_INVALID_HFRAG_PTR; + + ERTS_MSACC_POP_STATE_M(); #ifdef CHECK_FOR_HOLES /* @@ -512,15 +696,20 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj) p->last_old_htop = p->old_htop; #endif - /* FIXME: This function should really return an Sint, i.e., a possibly - 64 bit wide signed integer, but that requires updating all the code - that calls it. For now, we just return INT_MAX if the result is too - large for an int. */ - { - Sint result = (HEAP_TOP(p) - HEAP_START(p)) / 10; - if (result >= INT_MAX) return INT_MAX; - else return (int) result; - } + return reds; +} + +int +erts_garbage_collect_nobump(Process* p, int need, Eterm* objv, int nobj) +{ + return garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj); +} + +void +erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj) +{ + int reds = garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj); + BUMP_REDS(p, reds); } /* @@ -533,13 +722,11 @@ erts_garbage_collect_hibernate(Process* p) Uint heap_size; Eterm* heap; Eterm* htop; - Rootset rootset; - char* src; - Uint src_size; Uint actual_size; char* area; Uint area_size; Sint offs; + int reds; if (p->flags & F_DISABLE_GC) ERTS_INTERNAL_ERROR("GC disabled"); @@ -549,8 +736,6 @@ erts_garbage_collect_hibernate(Process* p) */ erts_smp_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC); ErtsGcQuickSanityCheck(p); - ASSERT(p->mbuf_sz == 0); - ASSERT(p->mbuf == 0); ASSERT(p->stop == p->hend); /* Stack must be empty. */ /* @@ -558,46 +743,25 @@ erts_garbage_collect_hibernate(Process* p) */ - heap_size = p->heap_sz + (p->old_htop - p->old_heap); + heap_size = p->heap_sz + (p->old_htop - p->old_heap) + p->mbuf_sz; heap = (Eterm*) ERTS_HEAP_ALLOC(ERTS_ALC_T_TMP_HEAP, sizeof(Eterm)*heap_size); htop = heap; - (void) setup_rootset(p, p->arg_reg, p->arity, &rootset); -#if HIPE - hipe_empty_nstack(p); -#endif - - src = (char *) p->heap; - src_size = (char *) p->htop - src; - htop = sweep_rootset(&rootset, htop, src, src_size); - htop = sweep_one_area(heap, htop, src, src_size); + htop = full_sweep_heaps(p, + 1, + heap, + htop, + (char *) p->old_heap, + (char *) p->old_htop - (char *) p->old_heap, + p->arg_reg, + p->arity); - if (p->old_heap) { - src = (char *) p->old_heap; - src_size = (char *) p->old_htop - src; - htop = sweep_rootset(&rootset, htop, src, src_size); - htop = sweep_one_area(heap, htop, src, src_size); - } - - cleanup_rootset(&rootset); - - if (MSO(p).first) { - sweep_off_heap(p, 1); - } - - /* - * Update all pointers. - */ ERTS_HEAP_FREE(ERTS_ALC_T_HEAP, - (void*)HEAP_START(p), - HEAP_SIZE(p) * sizeof(Eterm)); - if (p->old_heap) { - ERTS_HEAP_FREE(ERTS_ALC_T_OLD_HEAP, - (void*)p->old_heap, - (p->old_hend - p->old_heap) * sizeof(Eterm)); - p->old_heap = p->old_htop = p->old_hend = 0; - } + (p->abandoned_heap + ? p->abandoned_heap + : p->heap), + p->heap_sz * sizeof(Eterm)); p->heap = heap; p->high_water = htop; @@ -612,6 +776,7 @@ erts_garbage_collect_hibernate(Process* p) } FLAGS(p) &= ~F_FORCE_GC; + p->live_hf_end = ERTS_INVALID_HFRAG_PTR; /* * Move the heap to its final destination. @@ -631,6 +796,8 @@ erts_garbage_collect_hibernate(Process* p) sys_memcpy((void *) heap, (void *) p->heap, actual_size*sizeof(Eterm)); ERTS_HEAP_FREE(ERTS_ALC_T_TMP_HEAP, p->heap, p->heap_sz*sizeof(Eterm)); + remove_message_buffers(p); + p->stop = p->hend = heap + heap_size; offs = heap - p->heap; @@ -659,15 +826,18 @@ erts_garbage_collect_hibernate(Process* p) ErtsGcQuickSanityCheck(p); erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC); + + reds = gc_cost(actual_size, actual_size); + BUMP_REDS(p, reds); } void erts_garbage_collect_literals(Process* p, Eterm* literals, - Uint lit_size, + Uint byte_lit_size, struct erl_off_heap_header* oh) { - Uint byte_lit_size = sizeof(Eterm)*lit_size; + Uint lit_size = byte_lit_size / sizeof(Eterm); Uint old_heap_size; Eterm* temp_lit; Sint offs; @@ -743,7 +913,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); *g_ptr++ = val; - } else if (in_area(ptr, area, area_size)) { + } else if (ErtsInArea(ptr, area, area_size)) { MOVE_BOXED(ptr,val,old_htop,g_ptr++); } else { g_ptr++; @@ -754,7 +924,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, val = *ptr; if (IS_MOVED_CONS(val)) { /* Moved */ *g_ptr++ = ptr[1]; - } else if (in_area(ptr, area, area_size)) { + } else if (ErtsInArea(ptr, area, area_size)) { MOVE_CONS(ptr,val,old_htop,g_ptr++); } else { g_ptr++; @@ -774,8 +944,10 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, * Now we'll have to go through all heaps updating all other references. */ - old_htop = sweep_one_heap(p->heap, p->htop, old_htop, area, area_size); - old_htop = sweep_one_area(p->old_heap, old_htop, area, area_size); + old_htop = sweep_literals_to_old_heap(p->heap, p->htop, old_htop, area, area_size); + old_htop = sweep_literal_area(p->old_heap, old_htop, + (char *) p->old_heap, sizeof(Eterm)*old_heap_size, + area, area_size); ASSERT(p->old_htop <= old_htop && old_htop <= p->old_hend); p->old_htop = old_htop; @@ -834,15 +1006,18 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, } static int -minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) +minor_collection(Process* p, ErlHeapFragment *live_hf_end, + int need, Eterm* objv, int nobj, Uint *recl) { - Uint mature = HIGH_WATER(p) - HEAP_START(p); + Eterm *mature = p->abandoned_heap ? p->abandoned_heap : p->heap; + Uint mature_size = p->high_water - mature; + Uint size_before = young_gen_usage(p); /* * Allocate an old heap if we don't have one and if we'll need one. */ - if (OLD_HEAP(p) == NULL && mature != 0) { + if (OLD_HEAP(p) == NULL && mature_size != 0) { Eterm* n_old; /* Note: We choose a larger heap size than strictly needed, @@ -850,7 +1025,7 @@ minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) * This improved Estone by more than 1200 estones on my computer * (Ultra Sparc 10). */ - Uint new_sz = erts_next_heap_size(HEAP_TOP(p) - HEAP_START(p), 1); + Uint new_sz = erts_next_heap_size(size_before, 1); /* Create new, empty old_heap */ n_old = (Eterm *) ERTS_HEAP_ALLOC(ERTS_ALC_T_OLD_HEAP, @@ -866,41 +1041,32 @@ minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) */ if (OLD_HEAP(p) && - ((mature <= OLD_HEND(p) - OLD_HTOP(p)) && - ((BIN_VHEAP_MATURE(p) < ( BIN_OLD_VHEAP_SZ(p) - BIN_OLD_VHEAP(p)))) && - ((BIN_OLD_VHEAP_SZ(p) > BIN_OLD_VHEAP(p))) ) ) { - ErlMessage *msgp; - Uint size_after; - Uint need_after; - const Uint stack_size = STACK_SZ_ON_HEAP(p); - const Uint size_before = MBUF_SIZE(p) + (HEAP_TOP(p) - HEAP_START(p)); - Uint new_sz = HEAP_SIZE(p) + MBUF_SIZE(p) + combined_message_size(p); + ((mature_size <= OLD_HEND(p) - OLD_HTOP(p)) && + ((BIN_OLD_VHEAP_SZ(p) > BIN_OLD_VHEAP(p))) ) ) { + Eterm *prev_old_htop; + Uint stack_size, size_after, adjust_size, need_after, new_sz, new_mature; + + stack_size = p->hend - p->stop; + new_sz = stack_size + size_before; new_sz = next_heap_size(p, new_sz, 0); - do_minor(p, new_sz, objv, nobj); + prev_old_htop = p->old_htop; + do_minor(p, live_hf_end, (char *) mature, mature_size*sizeof(Eterm), + new_sz, objv, nobj); + + if (p->flags & F_ON_HEAP_MSGQ) + move_msgq_to_heap(p); - size_after = HEAP_TOP(p) - HEAP_START(p); + new_mature = p->old_htop - prev_old_htop; + + size_after = new_mature; + size_after += HEAP_TOP(p) - HEAP_START(p) + p->mbuf_sz; *recl += (size_before - size_after); - /* - * Copy newly received message onto the end of the new heap. - */ - ErtsGcQuickSanityCheck(p); - for (msgp = p->msg.first; msgp; msgp = msgp->next) { - if (msgp->data.attached) { - ErtsHeapFactory factory; - erts_factory_proc_prealloc_init(&factory, p, - erts_msg_attached_data_size(msgp)); - erts_move_msg_attached_data_to_heap(&factory, msgp); - erts_factory_close(&factory); - ErtsGcQuickSanityCheck(p); - } - } ErtsGcQuickSanityCheck(p); GEN_GCS(p)++; need_after = ((HEAP_TOP(p) - HEAP_START(p)) - + erts_used_frag_sz(MBUF(p)) + need + stack_size); @@ -913,6 +1079,8 @@ minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) * the heap size is substantial, we don't want to shrink. */ + adjust_size = 0; + if ((HEAP_SIZE(p) > 3000) && (4 * need_after < HEAP_SIZE(p)) && ((HEAP_SIZE(p) > 8000) || (HEAP_SIZE(p) > (OLD_HEND(p) - OLD_HEAP(p))))) { @@ -934,28 +1102,33 @@ minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) : next_heap_size(p, wanted, 0); if (wanted < HEAP_SIZE(p)) { shrink_new_heap(p, wanted, objv, nobj); + adjust_size = p->htop - p->heap; } - ASSERT(HEAP_SIZE(p) == next_heap_size(p, HEAP_SIZE(p), 0)); - ASSERT(MBUF(p) == NULL); - return 1; /* We are done. */ + goto done; } if (HEAP_SIZE(p) >= need_after) { /* * The heap size turned out to be just right. We are done. */ - ASSERT(HEAP_SIZE(p) == next_heap_size(p, HEAP_SIZE(p), 0)); - ASSERT(MBUF(p) == NULL); - return 1; + goto done; } + + grow_new_heap(p, next_heap_size(p, need_after, 0), objv, nobj); + adjust_size = p->htop - p->heap; + + done: + ASSERT(HEAP_SIZE(p) == next_heap_size(p, HEAP_SIZE(p), 0)); + ASSERT(MBUF(p) == NULL); + + return gc_cost(size_after, adjust_size); } /* - * Still not enough room after minor collection. Must force a major collection. + * Not enough room for a minor collection. Must force a major collection. */ - FLAGS(p) |= F_NEED_FULLSWEEP; - return 0; + return -1; } /* @@ -1009,7 +1182,9 @@ static ERTS_INLINE void offset_nstack(Process* p, Sint offs, #endif /* HIPE */ static void -do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj) +do_minor(Process *p, ErlHeapFragment *live_hf_end, + char *mature, Uint mature_size, + Uint new_sz, Eterm* objv, int nobj) { Rootset rootset; /* Rootset for GC (stack, dictionary, etc). */ Roots* roots; @@ -1018,17 +1193,24 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj) Eterm* ptr; Eterm val; Eterm gval; - char* heap = (char *) HEAP_START(p); - Uint heap_size = (char *) HEAP_TOP(p) - heap; - Uint mature_size = (char *) HIGH_WATER(p) - heap; Eterm* old_htop = OLD_HTOP(p); Eterm* n_heap; + char* oh = (char *) OLD_HEAP(p); + Uint oh_size = (char *) OLD_HTOP(p) - oh; + + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] MINOR GC: %p %p %p %p\n", p->common.id, + HEAP_START(p), HEAP_END(p), OLD_HEAP(p), OLD_HEND(p))); n_htop = n_heap = (Eterm*) ERTS_HEAP_ALLOC(ERTS_ALC_T_HEAP, sizeof(Eterm)*new_sz); - if (MBUF(p) != NULL) { - n_htop = collect_heap_frags(p, n_heap, n_htop, objv, nobj); + if (live_hf_end != ERTS_INVALID_HFRAG_PTR) { + /* + * Move heap frags that we know are completely live + * directly into the new young heap generation. + */ + n_htop = collect_live_heap_frags(p, live_hf_end, n_heap, n_htop, + objv, nobj); } n = setup_rootset(p, objv, nobj, &rootset); @@ -1051,9 +1233,9 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj) if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); *g_ptr++ = val; - } else if (in_area(ptr, heap, mature_size)) { + } else if (ErtsInArea(ptr, mature, mature_size)) { MOVE_BOXED(ptr,val,old_htop,g_ptr++); - } else if (in_area(ptr, heap, heap_size)) { + } else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) { MOVE_BOXED(ptr,val,n_htop,g_ptr++); } else { g_ptr++; @@ -1066,9 +1248,9 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj) val = *ptr; if (IS_MOVED_CONS(val)) { /* Moved */ *g_ptr++ = ptr[1]; - } else if (in_area(ptr, heap, mature_size)) { + } else if (ErtsInArea(ptr, mature, mature_size)) { MOVE_CONS(ptr,val,old_htop,g_ptr++); - } else if (in_area(ptr, heap, heap_size)) { + } else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) { MOVE_CONS(ptr,val,n_htop,g_ptr++); } else { g_ptr++; @@ -1093,7 +1275,7 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj) */ if (mature_size == 0) { - n_htop = sweep_one_area(n_heap, n_htop, heap, heap_size); + n_htop = sweep_new_heap(n_heap, n_htop, oh, oh_size); } else { Eterm* n_hp = n_heap; Eterm* ptr; @@ -1110,9 +1292,9 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj) if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); *n_hp++ = val; - } else if (in_area(ptr, heap, mature_size)) { + } else if (ErtsInArea(ptr, mature, mature_size)) { MOVE_BOXED(ptr,val,old_htop,n_hp++); - } else if (in_area(ptr, heap, heap_size)) { + } else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) { MOVE_BOXED(ptr,val,n_htop,n_hp++); } else { n_hp++; @@ -1124,9 +1306,9 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj) val = *ptr; if (IS_MOVED_CONS(val)) { *n_hp++ = ptr[1]; - } else if (in_area(ptr, heap, mature_size)) { + } else if (ErtsInArea(ptr, mature, mature_size)) { MOVE_CONS(ptr,val,old_htop,n_hp++); - } else if (in_area(ptr, heap, heap_size)) { + } else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) { MOVE_CONS(ptr,val,n_htop,n_hp++); } else { n_hp++; @@ -1146,10 +1328,10 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj) if (IS_MOVED_BOXED(val)) { *origptr = val; mb->base = binary_bytes(val); - } else if (in_area(ptr, heap, mature_size)) { + } else if (ErtsInArea(ptr, mature, mature_size)) { MOVE_BOXED(ptr,val,old_htop,origptr); mb->base = binary_bytes(mb->orig); - } else if (in_area(ptr, heap, heap_size)) { + } else if (ErtsInYoungGen(*origptr, ptr, oh, oh_size)) { MOVE_BOXED(ptr,val,n_htop,origptr); mb->base = binary_bytes(mb->orig); } @@ -1170,9 +1352,8 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj) * may point to the old (soon to be deleted) new_heap. */ - if (OLD_HTOP(p) < old_htop) { - old_htop = sweep_one_area(OLD_HTOP(p), old_htop, heap, heap_size); - } + if (OLD_HTOP(p) < old_htop) + old_htop = sweep_new_heap(OLD_HTOP(p), old_htop, oh, oh_size); OLD_HTOP(p) = old_htop; HIGH_WATER(p) = n_htop; @@ -1204,8 +1385,12 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj) #endif ERTS_HEAP_FREE(ERTS_ALC_T_HEAP, - (void*)HEAP_START(p), + (p->abandoned_heap + ? p->abandoned_heap + : HEAP_START(p)), HEAP_SIZE(p) * sizeof(Eterm)); + p->abandoned_heap = NULL; + p->flags &= ~F_ABANDONED_HEAP_USE; HEAP_START(p) = n_heap; HEAP_TOP(p) = n_htop; HEAP_SIZE(p) = new_sz; @@ -1222,31 +1407,30 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj) */ static int -major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) +major_collection(Process* p, ErlHeapFragment *live_hf_end, + int need, Eterm* objv, int nobj, Uint *recl) { - Rootset rootset; - Roots* roots; - const Uint size_before = ((HEAP_TOP(p) - HEAP_START(p)) - + (OLD_HTOP(p) - OLD_HEAP(p)) - + MBUF_SIZE(p)); + Uint size_before, size_after, stack_size; Eterm* n_heap; Eterm* n_htop; - char* src = (char *) HEAP_START(p); - Uint src_size = (char *) HEAP_TOP(p) - src; char* oh = (char *) OLD_HEAP(p); Uint oh_size = (char *) OLD_HTOP(p) - oh; - Uint n; - Uint new_sz; - int done; + Uint new_sz, stk_sz; + int adjusted; + + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] MAJOR GC: %p %p %p %p\n", p->common.id, + HEAP_START(p), HEAP_END(p), OLD_HEAP(p), OLD_HEND(p))); /* * Do a fullsweep GC. First figure out the size of the heap * to receive all live data. */ - new_sz = (HEAP_SIZE(p) + MBUF_SIZE(p) - + combined_message_size(p) - + (OLD_HTOP(p) - OLD_HEAP(p))); + size_before = young_gen_usage(p); + size_before += p->old_htop - p->old_heap; + stack_size = p->hend - p->stop; + + new_sz = stack_size + size_before; new_sz = next_heap_size(p, new_sz, 0); /* @@ -1260,13 +1444,76 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) n_htop = n_heap = (Eterm *) ERTS_HEAP_ALLOC(ERTS_ALC_T_HEAP, sizeof(Eterm)*new_sz); - /* - * Get rid of heap fragments. - */ + if (live_hf_end != ERTS_INVALID_HFRAG_PTR) { + /* + * Move heap frags that we know are completely live + * directly into the heap. + */ + n_htop = collect_live_heap_frags(p, live_hf_end, n_heap, n_htop, + objv, nobj); + } - if (MBUF(p) != NULL) { - n_htop = collect_heap_frags(p, n_heap, n_htop, objv, nobj); + n_htop = full_sweep_heaps(p, 0, n_heap, n_htop, oh, oh_size, objv, nobj); + + /* Move the stack to the end of the heap */ + stk_sz = HEAP_END(p) - p->stop; + sys_memcpy(n_heap + new_sz - stk_sz, p->stop, stk_sz * sizeof(Eterm)); + p->stop = n_heap + new_sz - stk_sz; + +#ifdef USE_VM_PROBES + if (HEAP_SIZE(p) != new_sz && DTRACE_ENABLED(process_heap_grow)) { + DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE); + + dtrace_proc_str(p, pidbuf); + DTRACE3(process_heap_grow, pidbuf, HEAP_SIZE(p), new_sz); } +#endif + + ERTS_HEAP_FREE(ERTS_ALC_T_HEAP, + (p->abandoned_heap + ? p->abandoned_heap + : HEAP_START(p)), + p->heap_sz * sizeof(Eterm)); + p->abandoned_heap = NULL; + p->flags &= ~F_ABANDONED_HEAP_USE; + HEAP_START(p) = n_heap; + HEAP_TOP(p) = n_htop; + HEAP_SIZE(p) = new_sz; + HEAP_END(p) = n_heap + new_sz; + GEN_GCS(p) = 0; + + HIGH_WATER(p) = HEAP_TOP(p); + + remove_message_buffers(p); + + if (p->flags & F_ON_HEAP_MSGQ) + move_msgq_to_heap(p); + + ErtsGcQuickSanityCheck(p); + + size_after = HEAP_TOP(p) - HEAP_START(p) + p->mbuf_sz; + *recl += size_before - size_after; + + adjusted = adjust_after_fullsweep(p, need, objv, nobj); + +#ifdef HARDDEBUG + disallow_heap_frag_ref_in_heap(p); +#endif + ErtsGcQuickSanityCheck(p); + + return gc_cost(size_after, adjusted ? size_after : 0); +} + +static Eterm * +full_sweep_heaps(Process *p, + int hibernate, + Eterm *n_heap, Eterm* n_htop, + char *oh, Uint oh_size, + Eterm *objv, int nobj) +{ + Rootset rootset; + Roots *roots; + Uint n; /* * Copy all top-level terms directly referenced by the rootset to @@ -1274,7 +1521,14 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) */ n = setup_rootset(p, objv, nobj, &rootset); - n_htop = fullsweep_nstack(p, n_htop); + +#ifdef HIPE + if (hibernate) + hipe_empty_nstack(p); + else + n_htop = fullsweep_nstack(p, n_htop); +#endif + roots = rootset.roots; while (n--) { Eterm* g_ptr = roots->v; @@ -1294,7 +1548,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); *g_ptr++ = val; - } else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) { + } else if (!erts_is_literal(gval, ptr)) { MOVE_BOXED(ptr,val,n_htop,g_ptr++); } else { g_ptr++; @@ -1307,7 +1561,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) val = *ptr; if (IS_MOVED_CONS(val)) { *g_ptr++ = ptr[1]; - } else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) { + } else if (!erts_is_literal(gval, ptr)) { MOVE_CONS(ptr,val,n_htop,g_ptr++); } else { g_ptr++; @@ -1332,74 +1586,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) * until all is copied. */ - if (oh_size == 0) { - n_htop = sweep_one_area(n_heap, n_htop, src, src_size); - } else { - Eterm* n_hp = n_heap; - - while (n_hp != n_htop) { - Eterm* ptr; - Eterm val; - Eterm gval = *n_hp; - - switch (primary_tag(gval)) { - case TAG_PRIMARY_BOXED: { - ptr = boxed_val(gval); - val = *ptr; - if (IS_MOVED_BOXED(val)) { - ASSERT(is_boxed(val)); - *n_hp++ = val; - } else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) { - MOVE_BOXED(ptr,val,n_htop,n_hp++); - } else { - n_hp++; - } - break; - } - case TAG_PRIMARY_LIST: { - ptr = list_val(gval); - val = *ptr; - if (IS_MOVED_CONS(val)) { - *n_hp++ = ptr[1]; - } else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) { - MOVE_CONS(ptr,val,n_htop,n_hp++); - } else { - n_hp++; - } - break; - } - case TAG_PRIMARY_HEADER: { - if (!header_is_thing(gval)) - n_hp++; - else { - if (header_is_bin_matchstate(gval)) { - ErlBinMatchState *ms = (ErlBinMatchState*) n_hp; - ErlBinMatchBuffer *mb = &(ms->mb); - Eterm* origptr; - origptr = &(mb->orig); - ptr = boxed_val(*origptr); - val = *ptr; - if (IS_MOVED_BOXED(val)) { - *origptr = val; - mb->base = binary_bytes(*origptr); - } else if (in_area(ptr, src, src_size) || - in_area(ptr, oh, oh_size)) { - MOVE_BOXED(ptr,val,n_htop,origptr); - mb->base = binary_bytes(*origptr); - ptr = boxed_val(*origptr); - val = *ptr; - } - } - n_hp += (thing_arityval(gval)+1); - } - break; - } - default: - n_hp++; - break; - } - } - } + n_htop = sweep_heaps(n_heap, n_htop, oh, oh_size); if (MSO(p).first) { sweep_off_heap(p, 1); @@ -1412,75 +1599,13 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) OLD_HEAP(p) = OLD_HTOP(p) = OLD_HEND(p) = NULL; } - /* Move the stack to the end of the heap */ - n = HEAP_END(p) - p->stop; - sys_memcpy(n_heap + new_sz - n, p->stop, n * sizeof(Eterm)); - p->stop = n_heap + new_sz - n; - -#ifdef USE_VM_PROBES - if (HEAP_SIZE(p) != new_sz && DTRACE_ENABLED(process_heap_grow)) { - DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE); - - dtrace_proc_str(p, pidbuf); - DTRACE3(process_heap_grow, pidbuf, HEAP_SIZE(p), new_sz); - } -#endif - - ERTS_HEAP_FREE(ERTS_ALC_T_HEAP, - (void *) HEAP_START(p), - (HEAP_END(p) - HEAP_START(p)) * sizeof(Eterm)); - HEAP_START(p) = n_heap; - HEAP_TOP(p) = n_htop; - HEAP_SIZE(p) = new_sz; - HEAP_END(p) = n_heap + new_sz; - GEN_GCS(p) = 0; - - HIGH_WATER(p) = HEAP_TOP(p); - - ErtsGcQuickSanityCheck(p); - - *recl += size_before - (HEAP_TOP(p) - HEAP_START(p)); - - remove_message_buffers(p); - - { - ErlMessage *msgp; - - /* - * Copy newly received message onto the end of the new heap. - */ - for (msgp = p->msg.first; msgp; msgp = msgp->next) { - if (msgp->data.attached) { - ErtsHeapFactory factory; - erts_factory_proc_prealloc_init(&factory, p, - erts_msg_attached_data_size(msgp)); - erts_move_msg_attached_data_to_heap(&factory, msgp); - erts_factory_close(&factory); - ErtsGcQuickSanityCheck(p); - } - } - } - - if (MBUF(p)) { - /* This is a very rare case when distributed messages copied above - * contained maps so big they did not fit on the heap causing the - * factory to create heap frags. - * Solution: Trigger a minor gc (without tenuring) - */ - HIGH_WATER(p) = HEAP_START(p); - done = 0; - } else { - adjust_after_fullsweep(p, need, objv, nobj); - done = 1; - } - - ErtsGcQuickSanityCheck(p); - return done; + return n_htop; } -static void +static int adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj) { + int adjusted = 0; Uint wanted, sz, need_after; Uint stack_size = STACK_SZ_ON_HEAP(p); @@ -1493,6 +1618,7 @@ adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj) /* Too small - grow to match requested need */ sz = next_heap_size(p, need_after, 0); grow_new_heap(p, sz, objv, nobj); + adjusted = 1; } else if (3 * HEAP_SIZE(p) < 4 * need_after){ /* Need more than 75% of current, postpone to next GC.*/ FLAGS(p) |= F_HEAP_GROW; @@ -1509,25 +1635,10 @@ adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj) if (sz < HEAP_SIZE(p)) { shrink_new_heap(p, sz, objv, nobj); + adjusted = 1; } } -} - -/* - * Return the size of all message buffers that are NOT linked in the - * mbuf list. - */ -static Uint -combined_message_size(Process* p) -{ - Uint sz; - ErlMessage *msgp; - - for (sz = 0, msgp = p->msg.first; msgp; msgp = msgp->next) { - if (msgp->data.attached) - sz += erts_msg_attached_data_size(msgp); - } - return sz; + return adjusted; } /* @@ -1540,6 +1651,10 @@ remove_message_buffers(Process* p) free_message_buffer(MBUF(p)); MBUF(p) = NULL; } + if (p->msg_frag) { + erts_cleanup_messages(p->msg_frag); + p->msg_frag = NULL; + } MBUF_SIZE(p) = 0; } #ifdef HARDDEBUG @@ -1551,64 +1666,6 @@ remove_message_buffers(Process* p) * For performance reasons, we use _unchecked_list_val(), _unchecked_boxed_val(), * and so on to avoid a function call. */ - -static void -disallow_heap_frag_ref(Process* p, Eterm* n_htop, Eterm* objv, int nobj) -{ - ErlHeapFragment* mbuf; - ErlHeapFragment* qb; - Eterm gval; - Eterm* ptr; - Eterm val; - - ASSERT(p->htop != NULL); - mbuf = MBUF(p); - - while (nobj--) { - gval = *objv; - - switch (primary_tag(gval)) { - - case TAG_PRIMARY_BOXED: { - ptr = _unchecked_boxed_val(gval); - val = *ptr; - if (IS_MOVED_BOXED(val)) { - ASSERT(is_boxed(val)); - objv++; - } else { - for (qb = mbuf; qb != NULL; qb = qb->next) { - if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) { - abort(); - } - } - objv++; - } - break; - } - - case TAG_PRIMARY_LIST: { - ptr = _unchecked_list_val(gval); - val = *ptr; - if (IS_MOVED_CONS(val)) { - objv++; - } else { - for (qb = mbuf; qb != NULL; qb = qb->next) { - if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) { - abort(); - } - } - objv++; - } - break; - } - - default: { - objv++; - break; - } - } - } -} static void disallow_heap_frag_ref_in_heap(Process* p) @@ -1636,9 +1693,9 @@ disallow_heap_frag_ref_in_heap(Process* p) switch (primary_tag(val)) { case TAG_PRIMARY_BOXED: ptr = _unchecked_boxed_val(val); - if (!in_area(ptr, heap, heap_size)) { + if (!ErtsInArea(ptr, heap, heap_size)) { for (qb = MBUF(p); qb != NULL; qb = qb->next) { - if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) { + if (ErtsInArea(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) { abort(); } } @@ -1646,9 +1703,9 @@ disallow_heap_frag_ref_in_heap(Process* p) break; case TAG_PRIMARY_LIST: ptr = _unchecked_list_val(val); - if (!in_area(ptr, heap, heap_size)) { + if (!ErtsInArea(ptr, heap, heap_size)) { for (qb = MBUF(p); qb != NULL; qb = qb->next) { - if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) { + if (ErtsInArea(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) { abort(); } } @@ -1690,26 +1747,26 @@ disallow_heap_frag_ref_in_old_heap(Process* p) val = *hp++; switch (primary_tag(val)) { case TAG_PRIMARY_BOXED: - ptr = (Eterm *) EXPAND_POINTER(val); - if (!in_area(ptr, old_heap, old_heap_size)) { - if (in_area(ptr, new_heap, new_heap_size)) { + ptr = (Eterm *) val; + if (!ErtsInArea(ptr, old_heap, old_heap_size)) { + if (ErtsInArea(ptr, new_heap, new_heap_size)) { abort(); } for (qb = MBUF(p); qb != NULL; qb = qb->next) { - if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) { + if (ErtsInArea(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) { abort(); } } } break; case TAG_PRIMARY_LIST: - ptr = (Eterm *) EXPAND_POINTER(val); - if (!in_area(ptr, old_heap, old_heap_size)) { - if (in_area(ptr, new_heap, new_heap_size)) { + ptr = (Eterm *) val; + if (!ErtsInArea(ptr, old_heap, old_heap_size)) { + if (ErtsInArea(ptr, new_heap, new_heap_size)) { abort(); } for (qb = MBUF(p); qb != NULL; qb = qb->next) { - if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) { + if (ErtsInArea(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) { abort(); } } @@ -1718,7 +1775,7 @@ disallow_heap_frag_ref_in_old_heap(Process* p) case TAG_PRIMARY_HEADER: if (header_is_thing(val)) { hp += _unchecked_thing_arityval(val); - if (!in_area(hp, old_heap, old_heap_size+1)) { + if (!ErtsInArea(hp, old_heap, old_heap_size+1)) { abort(); } } @@ -1728,66 +1785,30 @@ disallow_heap_frag_ref_in_old_heap(Process* p) } #endif -static Eterm* -sweep_rootset(Rootset* rootset, Eterm* htop, char* src, Uint src_size) +typedef enum { + ErtsSweepNewHeap, + ErtsSweepHeaps, + ErtsSweepLiteralArea +} ErtsSweepType; + +static ERTS_FORCE_INLINE Eterm * +sweep(Eterm *n_hp, Eterm *n_htop, + ErtsSweepType type, + char *oh, Uint ohsz, + char *src, Uint src_size) { - Roots* roots = rootset->roots; - Uint n = rootset->num_roots; Eterm* ptr; - Eterm gval; Eterm val; + Eterm gval; - while (n--) { - Eterm* g_ptr = roots->v; - Uint g_sz = roots->sz; - - roots++; - while (g_sz--) { - gval = *g_ptr; - - switch (primary_tag(gval)) { - case TAG_PRIMARY_BOXED: { - ptr = boxed_val(gval); - val = *ptr; - if (IS_MOVED_BOXED(val)) { - ASSERT(is_boxed(val)); - *g_ptr++ = val; - } else if (in_area(ptr, src, src_size)) { - MOVE_BOXED(ptr,val,htop,g_ptr++); - } else { - g_ptr++; - } - break; - } - case TAG_PRIMARY_LIST: { - ptr = list_val(gval); - val = *ptr; - if (IS_MOVED_CONS(val)) { - *g_ptr++ = ptr[1]; - } else if (in_area(ptr, src, src_size)) { - MOVE_CONS(ptr,val,htop,g_ptr++); - } else { - g_ptr++; - } - break; - } - - default: - g_ptr++; - break; - } - } - } - return htop; -} +#undef ERTS_IS_IN_SWEEP_AREA - -static Eterm* -sweep_one_area(Eterm* n_hp, Eterm* n_htop, char* src, Uint src_size) -{ - Eterm* ptr; - Eterm val; - Eterm gval; +#define ERTS_IS_IN_SWEEP_AREA(TPtr, Ptr) \ + (type == ErtsSweepHeaps \ + ? !erts_is_literal((TPtr), (Ptr)) \ + : (type == ErtsSweepNewHeap \ + ? ErtsInYoungGen((TPtr), (Ptr), oh, ohsz) \ + : ErtsInArea((Ptr), src, src_size))) while (n_hp != n_htop) { ASSERT(n_hp < n_htop); @@ -1799,7 +1820,7 @@ sweep_one_area(Eterm* n_hp, Eterm* n_htop, char* src, Uint src_size) if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); *n_hp++ = val; - } else if (in_area(ptr, src, src_size)) { + } else if (ERTS_IS_IN_SWEEP_AREA(gval, ptr)) { MOVE_BOXED(ptr,val,n_htop,n_hp++); } else { n_hp++; @@ -1811,7 +1832,7 @@ sweep_one_area(Eterm* n_hp, Eterm* n_htop, char* src, Uint src_size) val = *ptr; if (IS_MOVED_CONS(val)) { *n_hp++ = ptr[1]; - } else if (in_area(ptr, src, src_size)) { + } else if (ERTS_IS_IN_SWEEP_AREA(gval, ptr)) { MOVE_CONS(ptr,val,n_htop,n_hp++); } else { n_hp++; @@ -1832,7 +1853,7 @@ sweep_one_area(Eterm* n_hp, Eterm* n_htop, char* src, Uint src_size) if (IS_MOVED_BOXED(val)) { *origptr = val; mb->base = binary_bytes(*origptr); - } else if (in_area(ptr, src, src_size)) { + } else if (ERTS_IS_IN_SWEEP_AREA(*origptr, ptr)) { MOVE_BOXED(ptr,val,n_htop,origptr); mb->base = binary_bytes(*origptr); } @@ -1847,10 +1868,41 @@ sweep_one_area(Eterm* n_hp, Eterm* n_htop, char* src, Uint src_size) } } return n_htop; +#undef ERTS_IS_IN_SWEEP_AREA +} + +static Eterm * +sweep_new_heap(Eterm *n_hp, Eterm *n_htop, char* old_heap, Uint old_heap_size) +{ + return sweep(n_hp, n_htop, + ErtsSweepNewHeap, + old_heap, old_heap_size, + NULL, 0); +} + +static Eterm * +sweep_heaps(Eterm *n_hp, Eterm *n_htop, char* old_heap, Uint old_heap_size) +{ + return sweep(n_hp, n_htop, + ErtsSweepHeaps, + old_heap, old_heap_size, + NULL, 0); +} + +static Eterm * +sweep_literal_area(Eterm *n_hp, Eterm *n_htop, + char* old_heap, Uint old_heap_size, + char* src, Uint src_size) +{ + return sweep(n_hp, n_htop, + ErtsSweepLiteralArea, + old_heap, old_heap_size, + src, src_size); } static Eterm* -sweep_one_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, char* src, Uint src_size) +sweep_literals_to_old_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, + char* src, Uint src_size) { while (heap_ptr < heap_end) { Eterm* ptr; @@ -1864,7 +1916,7 @@ sweep_one_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, char* src, Uint sr if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); *heap_ptr++ = val; - } else if (in_area(ptr, src, src_size)) { + } else if (ErtsInArea(ptr, src, src_size)) { MOVE_BOXED(ptr,val,htop,heap_ptr++); } else { heap_ptr++; @@ -1876,7 +1928,7 @@ sweep_one_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, char* src, Uint sr val = *ptr; if (IS_MOVED_CONS(val)) { *heap_ptr++ = ptr[1]; - } else if (in_area(ptr, src, src_size)) { + } else if (ErtsInArea(ptr, src, src_size)) { MOVE_CONS(ptr,val,htop,heap_ptr++); } else { heap_ptr++; @@ -1897,7 +1949,7 @@ sweep_one_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, char* src, Uint sr if (IS_MOVED_BOXED(val)) { *origptr = val; mb->base = binary_bytes(*origptr); - } else if (in_area(ptr, src, src_size)) { + } else if (ErtsInArea(ptr, src, src_size)) { MOVE_BOXED(ptr,val,htop,origptr); mb->base = binary_bytes(*origptr); } @@ -1948,43 +2000,21 @@ move_one_area(Eterm* n_htop, char* src, Uint src_size) */ static Eterm* -collect_heap_frags(Process* p, Eterm* n_hstart, Eterm* n_htop, - Eterm* objv, int nobj) +collect_live_heap_frags(Process* p, ErlHeapFragment *live_hf_end, + Eterm* n_hstart, Eterm* n_htop, + Eterm* objv, int nobj) { ErlHeapFragment* qb; char* frag_begin; Uint frag_size; /* - * We don't allow references to a heap fragments from the stack, heap, - * or process dictionary. - */ -#ifdef HARDDEBUG - disallow_heap_frag_ref(p, n_htop, p->stop, STACK_START(p) - p->stop); - if (p->dictionary != NULL) { - disallow_heap_frag_ref(p, n_htop, p->dictionary->data, p->dictionary->used); - } - /* OTP-18: Actually we do allow references from heap to heap fragments now. - This can happen when doing "binary_to_term" with a "fat" map contained - in another term. A "fat" map is a hashmap with higher heap demand than - first estimated by "binary_to_term" causing the factory to allocate - additional heap (fragments) for the hashmap tree nodes. - Run map_SUITE:t_gc_rare_map_overflow to provoke this. - - Inverted references like this does not matter however. The copy done - below by move_one_area() with move markers in the fragments and the - sweeping done later by the GC should make everything ok in the end. - */ - /***disallow_heap_frag_ref_in_heap(p);***/ -#endif - - /* * Move the heap fragments to the new heap. Note that no GC is done on * the heap fragments. Any garbage will thus be moved as well and survive * until next GC. */ qb = MBUF(p); - while (qb != NULL) { + while (qb != live_hf_end) { ASSERT(!qb->off_heap.first); /* process fragments use the MSO(p) list */ frag_size = qb->used_size * sizeof(Eterm); if (frag_size != 0) { @@ -1996,12 +2026,177 @@ collect_heap_frags(Process* p, Eterm* n_hstart, Eterm* n_htop, return n_htop; } +static ERTS_INLINE void +copy_one_frag(Eterm** hpp, ErlOffHeap* off_heap, + ErlHeapFragment *bp, Eterm *refs, int nrefs) +{ + Uint sz; + int i; + Sint offs; + struct erl_off_heap_header* oh; + Eterm *fhp, *hp; + + OH_OVERHEAD(off_heap, bp->off_heap.overhead); + sz = bp->used_size; + + fhp = bp->mem; + hp = *hpp; + offs = hp - fhp; + + oh = NULL; + while (sz--) { + Uint cpy_sz; + Eterm val = *fhp++; + + switch (primary_tag(val)) { + case TAG_PRIMARY_IMMED1: + *hp++ = val; + break; + case TAG_PRIMARY_LIST: + case TAG_PRIMARY_BOXED: + *hp++ = offset_ptr(val, offs); + break; + case TAG_PRIMARY_HEADER: + *hp++ = val; + switch (val & _HEADER_SUBTAG_MASK) { + case ARITYVAL_SUBTAG: + break; + case REFC_BINARY_SUBTAG: + case FUN_SUBTAG: + case EXTERNAL_PID_SUBTAG: + case EXTERNAL_PORT_SUBTAG: + case EXTERNAL_REF_SUBTAG: + oh = (struct erl_off_heap_header*) (hp-1); + cpy_sz = thing_arityval(val); + goto cpy_words; + default: + cpy_sz = header_arity(val); + + cpy_words: + ASSERT(sz >= cpy_sz); + sz -= cpy_sz; + while (cpy_sz >= 8) { + cpy_sz -= 8; + *hp++ = *fhp++; + *hp++ = *fhp++; + *hp++ = *fhp++; + *hp++ = *fhp++; + *hp++ = *fhp++; + *hp++ = *fhp++; + *hp++ = *fhp++; + *hp++ = *fhp++; + } + switch (cpy_sz) { + case 7: *hp++ = *fhp++; + case 6: *hp++ = *fhp++; + case 5: *hp++ = *fhp++; + case 4: *hp++ = *fhp++; + case 3: *hp++ = *fhp++; + case 2: *hp++ = *fhp++; + case 1: *hp++ = *fhp++; + default: break; + } + if (oh) { + /* Add to offheap list */ + oh->next = off_heap->first; + off_heap->first = oh; + ASSERT(*hpp <= (Eterm*)oh); + ASSERT(hp > (Eterm*)oh); + oh = NULL; + } + break; + } + break; + } + } + + ASSERT(bp->used_size == hp - *hpp); + *hpp = hp; + + for (i = 0; i < nrefs; i++) { + if (is_not_immed(refs[i])) + refs[i] = offset_ptr(refs[i], offs); + } + bp->off_heap.first = NULL; +} + +static void +move_msgq_to_heap(Process *p) +{ + ErtsMessage **mpp = &p->msg.first; + + while (*mpp) { + ErtsMessage *mp = *mpp; + + if (mp->data.attached) { + ErlHeapFragment *bp; + ErtsHeapFactory factory; + + erts_factory_proc_prealloc_init(&factory, p, + erts_msg_attached_data_size(mp)); + + if (is_non_value(ERL_MESSAGE_TERM(mp))) { + if (mp->data.dist_ext) { + ASSERT(mp->data.dist_ext->heap_size >= 0); + if (is_not_nil(ERL_MESSAGE_TOKEN(mp))) { + bp = erts_dist_ext_trailer(mp->data.dist_ext); + ERL_MESSAGE_TOKEN(mp) = copy_struct(ERL_MESSAGE_TOKEN(mp), + bp->used_size, + &factory.hp, + factory.off_heap); + erts_cleanup_offheap(&bp->off_heap); + } + ERL_MESSAGE_TERM(mp) = erts_decode_dist_ext(&factory, + mp->data.dist_ext); + erts_free_dist_ext_copy(mp->data.dist_ext); + mp->data.dist_ext = NULL; + } + } + else { + + if (mp->data.attached == ERTS_MSG_COMBINED_HFRAG) + bp = &mp->hfrag; + else + bp = mp->data.heap_frag; + + if (bp->next) + erts_move_multi_frags(&factory.hp, factory.off_heap, bp, + mp->m, ERL_MESSAGE_REF_ARRAY_SZ, 0); + else + copy_one_frag(&factory.hp, factory.off_heap, bp, + mp->m, ERL_MESSAGE_REF_ARRAY_SZ); + + if (mp->data.attached != ERTS_MSG_COMBINED_HFRAG) { + mp->data.heap_frag = NULL; + free_message_buffer(bp); + } + else { + ErtsMessage *tmp = erts_alloc_message(0, NULL); + sys_memcpy((void *) tmp->m, (void *) mp->m, + sizeof(Eterm)*ERL_MESSAGE_REF_ARRAY_SZ); + tmp->next = mp->next; + if (p->msg.save == &mp->next) + p->msg.save = &tmp->next; + if (p->msg.last == &mp->next) + p->msg.last = &tmp->next; + *mpp = tmp; + mp->next = NULL; + erts_cleanup_messages(mp); + mp = tmp; + } + } + + erts_factory_close(&factory); + } + + mpp = &(*mpp)->next; + } +} + static Uint setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset) { - Uint avail; Roots* roots; - ErlMessage* mp; Uint n; n = 0; @@ -2013,8 +2208,8 @@ setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset) ++n; if (p->dictionary != NULL) { - roots[n].v = p->dictionary->data; - roots[n].sz = p->dictionary->used; + roots[n].v = ERTS_PD_START(p->dictionary); + roots[n].sz = ERTS_PD_SIZE(p->dictionary); ++n; } if (nobj > 0) { @@ -2024,7 +2219,7 @@ setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset) } ASSERT((is_nil(p->seq_trace_token) || - is_tuple(follow_moved(p->seq_trace_token)) || + is_tuple(follow_moved(p->seq_trace_token, (Eterm) 0)) || is_atom(p->seq_trace_token))); if (is_not_immed(p->seq_trace_token)) { roots[n].v = &p->seq_trace_token; @@ -2042,7 +2237,7 @@ setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset) is_internal_pid(ERTS_TRACER_PROC(p)) || is_internal_port(ERTS_TRACER_PROC(p))); - ASSERT(is_pid(follow_moved(p->group_leader))); + ASSERT(is_pid(follow_moved(p->group_leader, (Eterm) 0))); if (is_not_immed(p->group_leader)) { roots[n].v = &p->group_leader; roots[n].sz = 1; @@ -2079,31 +2274,47 @@ setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset) ASSERT(n <= rootset->size); - mp = p->msg.first; - avail = rootset->size - n; - while (mp != NULL) { - if (avail == 0) { - Uint new_size = 2*rootset->size; - if (roots == rootset->def) { - roots = erts_alloc(ERTS_ALC_T_ROOTSET, - new_size*sizeof(Roots)); - sys_memcpy(roots, rootset->def, sizeof(rootset->def)); - } else { - roots = erts_realloc(ERTS_ALC_T_ROOTSET, - (void *) roots, - new_size*sizeof(Roots)); - } + switch (p->flags & (F_OFF_HEAP_MSGQ|F_OFF_HEAP_MSGQ_CHNG)) { + case F_OFF_HEAP_MSGQ|F_OFF_HEAP_MSGQ_CHNG: + (void) erts_move_messages_off_heap(p); + case F_OFF_HEAP_MSGQ: + break; + case F_OFF_HEAP_MSGQ_CHNG: + case 0: { + /* + * We do not have off heap message queue enabled, i.e. we + * need to add message queue to rootset... + */ + ErtsMessage *mp; + + /* Ensure large enough rootset... */ + if (n + p->msg.len > rootset->size) { + Uint new_size = n + p->msg.len; + ERTS_GC_ASSERT(roots == rootset->def); + roots = erts_alloc(ERTS_ALC_T_ROOTSET, + new_size*sizeof(Roots)); + sys_memcpy(roots, rootset->def, n*sizeof(Roots)); rootset->size = new_size; - avail = new_size - n; } - if (mp->data.attached == NULL) { - roots[n].v = mp->m; - roots[n].sz = 2; - n++; - avail--; + + for (mp = p->msg.first; mp; mp = mp->next) { + + if (!mp->data.attached) { + /* + * Message may refer data on heap; + * add it to rootset... + */ + roots[n].v = mp->m; + roots[n].sz = ERL_MESSAGE_REF_ARRAY_SZ; + n++; + } } - mp = mp->next; + break; } + } + + ASSERT(rootset->size >= n); + rootset->roots = roots; rootset->num_roots = n; return n; @@ -2343,11 +2554,11 @@ sweep_off_heap(Process *p, int fullsweep) */ while (ptr) { if (IS_MOVED_BOXED(ptr->thing_word)) { - ASSERT(!in_area(ptr, oheap, oheap_sz)); + ASSERT(!ErtsInArea(ptr, oheap, oheap_sz)); *prev = ptr = (struct erl_off_heap_header*) boxed_val(ptr->thing_word); ASSERT(!IS_MOVED_BOXED(ptr->thing_word)); if (ptr->thing_word == HEADER_PROC_BIN) { - int to_new_heap = !in_area(ptr, oheap, oheap_sz); + int to_new_heap = !ErtsInArea(ptr, oheap, oheap_sz); ASSERT(to_new_heap == !seen_mature || (!to_new_heap && (seen_mature=1))); if (to_new_heap) { bin_vheap += ptr->size / sizeof(Eterm); @@ -2361,7 +2572,7 @@ sweep_off_heap(Process *p, int fullsweep) ptr = ptr->next; } } - else if (!in_area(ptr, oheap, oheap_sz)) { + else if (!ErtsInArea(ptr, oheap, oheap_sz)) { /* garbage */ switch (thing_subtag(ptr->thing_word)) { case REFC_BINARY_SUBTAG: @@ -2393,7 +2604,7 @@ sweep_off_heap(Process *p, int fullsweep) * generational collection - keep objects in list. */ while (ptr) { - ASSERT(in_area(ptr, oheap, oheap_sz)); + ASSERT(ErtsInArea(ptr, oheap, oheap_sz)); ASSERT(!IS_MOVED_BOXED(ptr->thing_word)); if (ptr->thing_word == HEADER_PROC_BIN) { BIN_OLD_VHEAP(p) += ptr->size / sizeof(Eterm); /* for binary gc (words)*/ @@ -2412,7 +2623,6 @@ sweep_off_heap(Process *p, int fullsweep) } BIN_VHEAP_SZ(p) = next_vheap_size(p, bin_vheap, BIN_VHEAP_SZ(p)); MSO(p).overhead = bin_vheap; - BIN_VHEAP_MATURE(p) = bin_vheap; /* * If we got any shrink candidates, check them out. @@ -2483,7 +2693,7 @@ offset_heap(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size) switch (primary_tag(val)) { case TAG_PRIMARY_LIST: case TAG_PRIMARY_BOXED: - if (in_area(ptr_val(val), area, area_size)) { + if (ErtsInArea(ptr_val(val), area, area_size)) { *hp = offset_ptr(val, offs); } hp++; @@ -2505,7 +2715,7 @@ offset_heap(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size) { struct erl_off_heap_header* oh = (struct erl_off_heap_header*) hp; - if (in_area(oh->next, area, area_size)) { + if (ErtsInArea(oh->next, area, area_size)) { Eterm** uptr = (Eterm **) (void *) &oh->next; *uptr += offs; /* Patch the mso chain */ } @@ -2515,7 +2725,7 @@ offset_heap(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size) { ErlBinMatchState *ms = (ErlBinMatchState*) hp; ErlBinMatchBuffer *mb = &(ms->mb); - if (in_area(ptr_val(mb->orig), area, area_size)) { + if (ErtsInArea(ptr_val(mb->orig), area, area_size)) { mb->orig = offset_ptr(mb->orig, offs); mb->base = binary_bytes(mb->orig); } @@ -2545,7 +2755,7 @@ offset_heap_ptr(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size) switch (primary_tag(val)) { case TAG_PRIMARY_LIST: case TAG_PRIMARY_BOXED: - if (in_area(ptr_val(val), area, area_size)) { + if (ErtsInArea(ptr_val(val), area, area_size)) { *hp = offset_ptr(val, offs); } hp++; @@ -2560,7 +2770,7 @@ offset_heap_ptr(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size) static void offset_off_heap(Process* p, Sint offs, char* area, Uint area_size) { - if (MSO(p).first && in_area((Eterm *)MSO(p).first, area, area_size)) { + if (MSO(p).first && ErtsInArea((Eterm *)MSO(p).first, area, area_size)) { Eterm** uptr = (Eterm**) (void *) &MSO(p).first; *uptr += offs; } @@ -2572,35 +2782,39 @@ offset_off_heap(Process* p, Sint offs, char* area, Uint area_size) static void offset_mqueue(Process *p, Sint offs, char* area, Uint area_size) { - ErlMessage* mp = p->msg.first; - - while (mp != NULL) { - Eterm mesg = ERL_MESSAGE_TERM(mp); - if (is_value(mesg)) { - switch (primary_tag(mesg)) { - case TAG_PRIMARY_LIST: - case TAG_PRIMARY_BOXED: - if (in_area(ptr_val(mesg), area, area_size)) { - ERL_MESSAGE_TERM(mp) = offset_ptr(mesg, offs); + ErtsMessage* mp = p->msg.first; + + if ((p->flags & (F_OFF_HEAP_MSGQ|F_OFF_HEAP_MSGQ_CHNG)) != F_OFF_HEAP_MSGQ) { + + while (mp != NULL) { + Eterm mesg = ERL_MESSAGE_TERM(mp); + if (is_value(mesg)) { + switch (primary_tag(mesg)) { + case TAG_PRIMARY_LIST: + case TAG_PRIMARY_BOXED: + if (ErtsInArea(ptr_val(mesg), area, area_size)) { + ERL_MESSAGE_TERM(mp) = offset_ptr(mesg, offs); + } + break; } - break; } - } - mesg = ERL_MESSAGE_TOKEN(mp); - if (is_boxed(mesg) && in_area(ptr_val(mesg), area, area_size)) { - ERL_MESSAGE_TOKEN(mp) = offset_ptr(mesg, offs); - } + mesg = ERL_MESSAGE_TOKEN(mp); + if (is_boxed(mesg) && ErtsInArea(ptr_val(mesg), area, area_size)) { + ERL_MESSAGE_TOKEN(mp) = offset_ptr(mesg, offs); + } #ifdef USE_VM_PROBES - mesg = ERL_MESSAGE_DT_UTAG(mp); - if (is_boxed(mesg) && in_area(ptr_val(mesg), area, area_size)) { - ERL_MESSAGE_DT_UTAG(mp) = offset_ptr(mesg, offs); - } + mesg = ERL_MESSAGE_DT_UTAG(mp); + if (is_boxed(mesg) && ErtsInArea(ptr_val(mesg), area, area_size)) { + ERL_MESSAGE_DT_UTAG(mp) = offset_ptr(mesg, offs); + } #endif - ASSERT((is_nil(ERL_MESSAGE_TOKEN(mp)) || - is_tuple(ERL_MESSAGE_TOKEN(mp)) || - is_atom(ERL_MESSAGE_TOKEN(mp)))); - mp = mp->next; + ASSERT((is_nil(ERL_MESSAGE_TOKEN(mp)) || + is_tuple(ERL_MESSAGE_TOKEN(mp)) || + is_atom(ERL_MESSAGE_TOKEN(mp)))); + mp = mp->next; + } + } } @@ -2609,8 +2823,8 @@ offset_one_rootset(Process *p, Sint offs, char* area, Uint area_size, Eterm* objv, int nobj) { if (p->dictionary) { - offset_heap(p->dictionary->data, - p->dictionary->used, + offset_heap(ERTS_PD_START(p->dictionary), + ERTS_PD_SIZE(p->dictionary), offs, area, area_size); } @@ -2659,7 +2873,7 @@ reply_gc_info(void *vgcirp) Eterm **hpp; Uint sz, *szp; ErlOffHeap *ohp = NULL; - ErlHeapFragment *bp = NULL; + ErtsMessage *mp = NULL; ASSERT(esdp); @@ -2685,12 +2899,13 @@ reply_gc_info(void *vgcirp) if (hpp) break; - hp = erts_alloc_message_heap(sz, &bp, &ohp, rp, &rp_locks); + mp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp); + szp = NULL; hpp = &hp; } - erts_queue_message(rp, &rp_locks, bp, msg, NIL); + erts_queue_message(rp, &rp_locks, mp, msg, NIL); if (gcirp->req_sched == esdp->no) rp_locks &= ~ERTS_PROC_LOCK_MAIN; @@ -2737,41 +2952,105 @@ erts_gc_info_request(Process *c_p) return ref; } +Eterm +erts_process_gc_info(Process *p, Uint *sizep, Eterm **hpp) +{ + ERTS_DECL_AM(bin_vheap_size); + ERTS_DECL_AM(bin_vheap_block_size); + ERTS_DECL_AM(bin_old_vheap_size); + ERTS_DECL_AM(bin_old_vheap_block_size); + Eterm tags[] = { + /* If you increase the number of elements here, make sure to update + any call sites as they may have stack allocations that depend + on the number of elements here. */ + am_old_heap_block_size, + am_heap_block_size, + am_mbuf_size, + am_recent_size, + am_stack_size, + am_old_heap_size, + am_heap_size, + AM_bin_vheap_size, + AM_bin_vheap_block_size, + AM_bin_old_vheap_size, + AM_bin_old_vheap_block_size + }; + UWord values[] = { + OLD_HEAP(p) ? OLD_HEND(p) - OLD_HEAP(p) : 0, + HEAP_SIZE(p), + MBUF_SIZE(p), + HIGH_WATER(p) - HEAP_START(p), + STACK_START(p) - p->stop, + OLD_HEAP(p) ? OLD_HTOP(p) - OLD_HEAP(p) : 0, + HEAP_TOP(p) - HEAP_START(p), + MSO(p).overhead, + BIN_VHEAP_SZ(p), + BIN_OLD_VHEAP(p), + BIN_OLD_VHEAP_SZ(p) + }; + + Eterm res = THE_NON_VALUE; + + ERTS_CT_ASSERT(sizeof(values)/sizeof(*values) == sizeof(tags)/sizeof(*tags)); + ERTS_CT_ASSERT(sizeof(values)/sizeof(*values) == ERTS_PROCESS_GC_INFO_MAX_TERMS); + + res = erts_bld_atom_uword_2tup_list(hpp, + sizep, + sizeof(values)/sizeof(*values), + tags, + values); + + return res; +} + #if defined(DEBUG) || defined(ERTS_OFFHEAP_DEBUG) static int within2(Eterm *ptr, Process *p, Eterm *real_htop) { - ErlHeapFragment* bp = MBUF(p); - ErlMessage* mp = p->msg.first; - Eterm *htop = real_htop ? real_htop : HEAP_TOP(p); + ErlHeapFragment* bp; + ErtsMessage* mp; + Eterm *htop, *heap; + + if (p->abandoned_heap) + ERTS_GET_ORIG_HEAP(p, heap, htop); + else { + heap = p->heap; + htop = real_htop ? real_htop : HEAP_TOP(p); + } if (OLD_HEAP(p) && (OLD_HEAP(p) <= ptr && ptr < OLD_HEND(p))) { return 1; } - if (HEAP_START(p) <= ptr && ptr < htop) { + if (heap <= ptr && ptr < htop) { return 1; } - while (bp != NULL) { - if (bp->mem <= ptr && ptr < bp->mem + bp->used_size) { - return 1; - } - bp = bp->next; - } + + mp = p->msg_frag; + bp = p->mbuf; + + if (bp) + goto search_heap_frags; + while (mp) { - if (mp->data.attached) { - ErlHeapFragment *hfp; - if (is_value(ERL_MESSAGE_TERM(mp))) - hfp = mp->data.heap_frag; - else if (is_not_nil(ERL_MESSAGE_TOKEN(mp))) - hfp = erts_dist_ext_trailer(mp->data.dist_ext); - else - hfp = NULL; - if (hfp && hfp->mem <= ptr && ptr < hfp->mem + hfp->used_size) + + if (mp->data.attached == ERTS_MSG_COMBINED_HFRAG) + bp = &mp->hfrag; + else + bp = mp->data.heap_frag; + + mp = mp->next; + + search_heap_frags: + + while (bp) { + if (bp->mem <= ptr && ptr < bp->mem + bp->used_size) { return 1; + } + bp = bp->next; } - mp = mp->next; } + return 0; } @@ -2793,11 +3072,11 @@ do { \ __FILE__, __LINE__, #EXP); \ } while (0) + #ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_LIST # define ERTS_OFFHEAP_VISITED_BIT ((Eterm) 1 << 31) #endif - void erts_check_off_heap2(Process *p, Eterm *htop) { @@ -2826,7 +3105,7 @@ erts_check_off_heap2(Process *p, Eterm *htop) } ERTS_CHK_OFFHEAP_ASSERT(refc >= 1); #ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_LIST - ERTS_CHK_OFFHEAP_ASSERT(!(u.hdr->thing_word & ERTS_EXTERNAL_VISITED_BIT)); + ERTS_CHK_OFFHEAP_ASSERT(!(u.hdr->thing_word & ERTS_OFFHEAP_VISITED_BIT)); u.hdr->thing_word |= ERTS_OFFHEAP_VISITED_BIT; #endif if (old) { @@ -2839,7 +3118,7 @@ erts_check_off_heap2(Process *p, Eterm *htop) } } -#ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_EXTERNAL_LIST +#ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_LIST for (u.hdr = MSO(p).first; u.hdr; u.hdr = u.hdr->next) u.hdr->thing_word &= ~ERTS_OFFHEAP_VISITED_BIT; #endif diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h index ecd1bf4d22..2cedd9361f 100644 --- a/erts/emulator/beam/erl_gc.h +++ b/erts/emulator/beam/erl_gc.h @@ -69,17 +69,18 @@ do { \ while (nelts--) *HTOP++ = *PTR++; \ } while(0) -#define in_area(ptr,start,nbytes) \ - ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes)) - #if defined(DEBUG) || defined(ERTS_OFFHEAP_DEBUG) int within(Eterm *ptr, Process *p); #endif -ERTS_GLB_INLINE Eterm follow_moved(Eterm term); +#define ErtsInYoungGen(TPtr, Ptr, OldHeap, OldHeapSz) \ + (!erts_is_literal((TPtr), (Ptr)) \ + & !ErtsInArea((Ptr), (OldHeap), (OldHeapSz))) + +ERTS_GLB_INLINE Eterm follow_moved(Eterm term, Eterm xptr_tag); #if ERTS_GLB_INLINE_INCL_FUNC_DEF -ERTS_GLB_INLINE Eterm follow_moved(Eterm term) +ERTS_GLB_INLINE Eterm follow_moved(Eterm term, Eterm xptr_tag) { Eterm* ptr; switch (primary_tag(term)) { @@ -87,17 +88,18 @@ ERTS_GLB_INLINE Eterm follow_moved(Eterm term) break; case TAG_PRIMARY_BOXED: ptr = boxed_val(term); - if (IS_MOVED_BOXED(*ptr)) term = *ptr; + if (IS_MOVED_BOXED(*ptr)) term = (*ptr) | xptr_tag; break; case TAG_PRIMARY_LIST: ptr = list_val(term); - if (IS_MOVED_CONS(ptr[0])) term = ptr[1]; + if (IS_MOVED_CONS(ptr[0])) term = (ptr[1]) | xptr_tag; break; default: ASSERT(!"strange tag in follow_moved"); } return term; } + #endif #endif /* ERL_GC_C__ || HIPE_GC_C__ */ @@ -106,6 +108,23 @@ ERTS_GLB_INLINE Eterm follow_moved(Eterm term) * Global exported */ +#define ERTS_IS_GC_DESIRED_INTERNAL(Proc, HTop, STop) \ + ((((STop) - (HTop) < (Proc)->mbuf_sz)) \ + | ((Proc)->off_heap.overhead > (Proc)->bin_vheap_sz) \ + | !!((Proc)->flags & F_FORCE_GC)) + +#define ERTS_IS_GC_DESIRED(Proc) \ + ERTS_IS_GC_DESIRED_INTERNAL((Proc), (Proc)->htop, (Proc)->stop) + +#define ERTS_FORCE_GC_INTERNAL(Proc, FCalls) \ + do { \ + (Proc)->flags |= F_FORCE_GC; \ + ERTS_VBUMP_ALL_REDS_INTERNAL((Proc), (FCalls)); \ + } while (0) + +#define ERTS_FORCE_GC(Proc) \ + ERTS_FORCE_GC_INTERNAL((Proc), (Proc)->fcalls) + extern Uint erts_test_long_gc_sleep; typedef struct { @@ -113,10 +132,18 @@ typedef struct { Uint64 garbage_cols; } ErtsGCInfo; +#define ERTS_PROCESS_GC_INFO_MAX_TERMS (11) /* number of elements in process_gc_info*/ +#define ERTS_PROCESS_GC_INFO_MAX_SIZE \ + (ERTS_PROCESS_GC_INFO_MAX_TERMS * (2/*cons*/ + 3/*2-tuple*/ + BIG_UINT_HEAP_SIZE)) +Eterm erts_process_gc_info(struct process*, Uint *, Eterm **); + void erts_gc_info(ErtsGCInfo *gcip); void erts_init_gc(void); -int erts_garbage_collect(struct process*, int, Eterm*, int); +int erts_garbage_collect_nobump(struct process*, int, Eterm*, int); +void erts_garbage_collect(struct process*, int, Eterm*, int); void erts_garbage_collect_hibernate(struct process* p); +Eterm erts_gc_after_bif_call_lhf(struct process* p, ErlHeapFragment *live_hf_end, + Eterm result, Eterm* regs, Uint arity); Eterm erts_gc_after_bif_call(struct process* p, Eterm result, Eterm* regs, Uint arity); void erts_garbage_collect_literals(struct process* p, Eterm* literals, Uint lit_size, diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c index fb6d249145..491d2f8c84 100644 --- a/erts/emulator/beam/erl_hl_timer.c +++ b/erts/emulator/beam/erl_hl_timer.c @@ -1245,7 +1245,9 @@ hlt_bif_timer_timeout(ErtsHLTimer *tmr, Uint32 roflgs) * the middle of tree destruction). */ if (!ERTS_PROC_IS_EXITING(proc)) { - erts_queue_message(proc, &proc_locks, tmr->btm.bp, + ErtsMessage *mp = erts_alloc_message(0, NULL); + mp->data.heap_frag = tmr->btm.bp; + erts_queue_message(proc, &proc_locks, mp, tmr->btm.message, NIL); erts_smp_proc_unlock(proc, ERTS_PROC_LOCKS_MSG_SEND); queued_message = 1; @@ -1926,36 +1928,31 @@ access_sched_local_btm(Process *c_p, Eterm pid, if (proc) { Uint hsz; - ErlOffHeap *ohp; - ErlHeapFragment* bp; + ErtsMessage *mp; Eterm *hp, msg, ref, result; + ErlOffHeap *ohp; + Uint32 *refn; #ifdef ERTS_HLT_DEBUG Eterm *hp_end; #endif - hsz = 3; /* 2-tuple */ - if (!async) - hsz += REF_THING_SIZE; + hsz = REF_THING_SIZE; + if (async) { + refn = trefn; /* timer ref */ + hsz += 4; /* 3-tuple */ + } else { - if (is_non_value(tref) || proc != c_p) - hsz += REF_THING_SIZE; - hsz += 1; /* upgrade to 3-tuple */ + refn = rrefn; /* request ref */ + hsz += 3; /* 2-tuple */ } + + ERTS_HLT_ASSERT(refn); + if (time_left > (Sint64) MAX_SMALL) hsz += ERTS_SINT64_HEAP_SIZE(time_left); - if (proc == c_p) { - bp = NULL; - ohp = NULL; - hp = HAlloc(c_p, hsz); - } - else { - hp = erts_alloc_message_heap(hsz, - &bp, - &ohp, - proc, - &proc_locks); - } + mp = erts_alloc_message_heap(proc, &proc_locks, + hsz, &hp, &ohp); #ifdef ERTS_HLT_DEBUG hp_end = hp + hsz; @@ -1968,35 +1965,22 @@ access_sched_local_btm(Process *c_p, Eterm pid, else result = erts_sint64_to_big(time_left, &hp); - if (!async) { - write_ref_thing(hp, - rrefn[0], - rrefn[1], - rrefn[2]); - ref = make_internal_ref(hp); - hp += REF_THING_SIZE; - msg = TUPLE2(hp, ref, result); + write_ref_thing(hp, + refn[0], + refn[1], + refn[2]); + ref = make_internal_ref(hp); + hp += REF_THING_SIZE; - ERTS_HLT_ASSERT(hp + 3 == hp_end); - } - else { - Eterm tag = cancel ? am_cancel_timer : am_read_timer; - if (is_value(tref) && proc == c_p) - ref = tref; - else { - write_ref_thing(hp, - trefn[0], - trefn[1], - trefn[2]); - ref = make_internal_ref(hp); - hp += REF_THING_SIZE; - } - msg = TUPLE3(hp, tag, ref, result); + msg = (async + ? TUPLE3(hp, (cancel + ? am_cancel_timer + : am_read_timer), ref, result) + : TUPLE2(hp, ref, result)); - ERTS_HLT_ASSERT(hp + 4 == hp_end); + ERTS_HLT_ASSERT(hp + (async ? 4 : 3) == hp_end); - } - erts_queue_message(proc, &proc_locks, bp, msg, NIL); + erts_queue_message(proc, &proc_locks, mp, msg, NIL); if (c_p) proc_locks &= ~ERTS_PROC_LOCK_MAIN; @@ -2034,7 +2018,7 @@ bif_timer_access_request(void *vreq) static int try_access_sched_remote_btm(ErtsSchedulerData *esdp, Process *c_p, Uint32 sid, - Eterm tref, Uint32 *trefn, + Uint32 *trefn, int async, int cancel, int info, Eterm *resp) { @@ -2093,21 +2077,31 @@ try_access_sched_remote_btm(ErtsSchedulerData *esdp, } } else { - Eterm tag, res, msg; + ErtsMessage *mp; + Eterm tag, res, msg, tref; Uint hsz; Eterm *hp; ErtsProcLocks proc_locks = ERTS_PROC_LOCK_MAIN; + ErlOffHeap *ohp; - hsz = 4; + hsz = 4 + REF_THING_SIZE; if (time_left > (Sint64) MAX_SMALL) hsz += ERTS_SINT64_HEAP_SIZE(time_left); - hp = HAlloc(c_p, hsz); + mp = erts_alloc_message_heap(c_p, &proc_locks, + hsz, &hp, &ohp); if (cancel) tag = am_cancel_timer; else tag = am_read_timer; + write_ref_thing(hp, + trefn[0], + trefn[1], + trefn[2]); + tref = make_internal_ref(hp); + hp += REF_THING_SIZE; + if (time_left < 0) res = am_false; else if (time_left <= (Sint64) MAX_SMALL) @@ -2117,7 +2111,7 @@ try_access_sched_remote_btm(ErtsSchedulerData *esdp, msg = TUPLE3(hp, tag, tref, res); - erts_queue_message(c_p, &proc_locks, NULL, msg, NIL); + erts_queue_message(c_p, &proc_locks, mp, msg, NIL); proc_locks &= ~ERTS_PROC_LOCK_MAIN; if (proc_locks) @@ -2158,8 +2152,8 @@ access_bif_timer(Process *c_p, Eterm tref, int cancel, int async, int info) info); ERTS_BIF_PREP_RET(ret, res); } - else if (try_access_sched_remote_btm(esdp, c_p, sid, - tref, trefn, + else if (try_access_sched_remote_btm(esdp, c_p, + sid, trefn, async, cancel, info, &res)) { ERTS_BIF_PREP_RET(ret, res); diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index d9c3b0dcf4..67089df863 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -91,11 +91,6 @@ const int etp_arch_bits = 32; #else # error "Not 64-bit, nor 32-bit arch" #endif -#if HALFWORD_HEAP -const int etp_halfword = 1; -#else -const int etp_halfword = 0; -#endif #ifdef HIPE const int etp_hipe = 1; #else @@ -157,7 +152,7 @@ volatile int erts_writing_erl_crash_dump = 0; int erts_initialized = 0; #if defined(USE_THREADS) && !defined(ERTS_SMP) -static erts_tid_t main_thread; +erts_tid_t erts_main_thread; #endif int erts_use_sender_punish; @@ -197,7 +192,7 @@ Uint32 verbose; /* See erl_debug.h for information about verbose */ int erts_atom_table_size = ATOM_LIMIT; /* Maximum number of atoms */ -int erts_pd_initial_size = 10; +int erts_pd_initial_size = 8; /* must be power of 2 */ int erts_modified_timing_level; @@ -393,11 +388,13 @@ erl_init(int ncpu, erts_mseg_late_init(); /* Must be after timer (erts_init_time()) and thread initializations */ #endif + erl_sys_late_init(); #ifdef HIPE hipe_mode_switch_init(); /* Must be after init_load/beam_catches/init */ #endif packet_parser_init(); erl_nif_init(); + erts_msacc_init(); } static Eterm @@ -436,13 +433,36 @@ erl_first_process_otp(char* modname, void* code, unsigned size, int argc, char** hp += 2; args = CONS(hp, env, args); - so.flags = SPO_SYSTEM_PROC; + so.flags = erts_default_spo_flags|SPO_SYSTEM_PROC; res = erl_create_process(&parent, start_mod, am_start, args, &so); erts_smp_proc_unlock(&parent, ERTS_PROC_LOCK_MAIN); erts_cleanup_empty_process(&parent); return res; } +static Eterm +erl_system_process_otp(Eterm parent_pid, char* modname) +{ + Eterm start_mod; + Process* parent; + ErlSpawnOpts so; + Eterm res; + + start_mod = erts_atom_put((byte *) modname, sys_strlen(modname), ERTS_ATOM_ENC_LATIN1, 1); + if (erts_find_function(start_mod, am_start, 0, + erts_active_code_ix()) == NULL) { + erl_exit(5, "No function %s:start/0\n", modname); + } + + parent = erts_pid2proc(NULL, 0, parent_pid, ERTS_PROC_LOCK_MAIN); + + so.flags = erts_default_spo_flags|SPO_SYSTEM_PROC; + res = erl_create_process(parent, start_mod, am_start, NIL, &so); + erts_smp_proc_unlock(parent, ERTS_PROC_LOCK_MAIN); + return res; +} + + Eterm erts_preloaded(Process* p) { @@ -635,6 +655,8 @@ void erts_usage(void) erts_fprintf(stderr, "-W<i|w|e> set error logger warnings mapping,\n"); erts_fprintf(stderr, " see error_logger documentation for details\n"); + erts_fprintf(stderr, "-xmqd val set default message queue data flag for processes,\n"); + erts_fprintf(stderr, " valid values are: off_heap | on_heap | mixed\n"); erts_fprintf(stderr, "-zdbbl size set the distribution buffer busy limit in kilobytes\n"); erts_fprintf(stderr, " valid range is [1-%d]\n", INT_MAX/1024); erts_fprintf(stderr, "-zdntgc time set delayed node table gc in seconds\n"); @@ -723,6 +745,10 @@ early_init(int *argc, char **argv) /* char envbuf[21]; /* enough for any 64-bit integer */ size_t envbufsz; +#if defined(USE_THREADS) && !defined(ERTS_SMP) + erts_main_thread = erts_thr_self(); +#endif + erts_save_emu_args(*argc, argv); erts_sched_compact_load = 1; @@ -776,9 +802,6 @@ early_init(int *argc, char **argv) /* (erts_aint32_t) ((Uint16) -1)); erts_pre_init_process(); -#if defined(USE_THREADS) && !defined(ERTS_SMP) - main_thread = erts_thr_self(); -#endif /* * We need to know the number of schedulers to use before we @@ -1172,6 +1195,7 @@ early_init(int *argc, char **argv) /* erts_thr_late_init(&elid); } #endif + erts_msacc_early_init(); #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_late_init(); @@ -1236,6 +1260,7 @@ erl_start(int argc, char **argv) ErtsTimeWarpMode time_warp_mode; int node_tab_delete_delay = ERTS_NODE_TAB_DELAY_GC_DEFAULT; ErtsDbSpinCount db_spin_count = ERTS_DB_SPNCNT_NORMAL; + Eterm otp_ring0_pid; set_default_time_adj(&time_correction, &time_warp_mode); @@ -1405,6 +1430,7 @@ erl_start(int argc, char **argv) case 't': verbose |= DEBUG_THREADS; break; case 'p': verbose |= DEBUG_PROCESSES; break; case 'm': verbose |= DEBUG_MESSAGES; break; + case 'c': verbose |= DEBUG_SHCOPY; break; default : erts_fprintf(stderr,"Unknown verbose option: %c\n",*ch); } } @@ -1417,6 +1443,7 @@ erl_start(int argc, char **argv) if (verbose & DEBUG_THREADS) erts_printf("THREADS "); if (verbose & DEBUG_PROCESSES) erts_printf("PROCESSES "); if (verbose & DEBUG_MESSAGES) erts_printf("MESSAGES "); + if (verbose & DEBUG_SHCOPY) erts_printf("SHCOPY "); erts_printf("\n"); #else erts_fprintf(stderr, "warning: -v (only in debug compiled code)\n"); @@ -1479,7 +1506,7 @@ erl_start(int argc, char **argv) VERBOSE(DEBUG_SYSTEM, ("using minimum heap size %d\n", H_MIN_SIZE)); } else if (has_prefix("pds", sub_param)) { arg = get_arg(sub_param+3, argv[i+1], &i); - if ((erts_pd_initial_size = atoi(arg)) <= 0) { + if (!erts_pd_set_initial_size(atoi(arg))) { erts_fprintf(stderr, "bad initial process dictionary size %s\n", arg); erts_usage(); } @@ -2020,6 +2047,32 @@ erl_start(int argc, char **argv) } break; + case 'x': { + char *sub_param = argv[i]+2; + if (has_prefix("mqd", sub_param)) { + arg = get_arg(sub_param+3, argv[i+1], &i); + if (sys_strcmp(arg, "mixed") == 0) + erts_default_spo_flags &= ~(SPO_ON_HEAP_MSGQ|SPO_OFF_HEAP_MSGQ); + else if (sys_strcmp(arg, "on_heap") == 0) { + erts_default_spo_flags &= ~SPO_OFF_HEAP_MSGQ; + erts_default_spo_flags |= SPO_ON_HEAP_MSGQ; + } + else if (sys_strcmp(arg, "off_heap") == 0) { + erts_default_spo_flags &= ~SPO_ON_HEAP_MSGQ; + erts_default_spo_flags |= SPO_OFF_HEAP_MSGQ; + } + else { + erts_fprintf(stderr, + "Invalid message_queue_data flag: %s\n", arg); + erts_usage(); + } + } else { + erts_fprintf(stderr, "bad -x option %s\n", argv[i]); + erts_usage(); + } + break; + } + case 'z': { char *sub_param = argv[i]+2; @@ -2073,7 +2126,8 @@ erl_start(int argc, char **argv) "Invalid ets busy wait threshold: %s\n", arg); erts_usage(); } - } else { + } + else { erts_fprintf(stderr, "bad -z option %s\n", argv[i]); erts_usage(); } @@ -2156,8 +2210,10 @@ erl_start(int argc, char **argv) erts_initialized = 1; - (void) erl_first_process_otp("otp_ring0", NULL, 0, - boot_argc, boot_argv); + otp_ring0_pid = erl_first_process_otp("otp_ring0", NULL, 0, + boot_argc, boot_argv); + + (void) erl_system_process_otp(otp_ring0_pid, "erts_code_purger"); #ifdef ERTS_SMP erts_start_schedulers(); @@ -2167,6 +2223,7 @@ erl_start(int argc, char **argv) #else { ErtsSchedulerData *esdp = erts_get_scheduler_data(); + erts_msacc_init_thread("scheduler", 1, 1); erts_thr_set_main_status(1, 1); #if ERTS_USE_ASYNC_READY_Q esdp->aux_work_data.async_ready.queue @@ -2232,7 +2289,7 @@ system_cleanup(int flush_async) if (!flush_async || !erts_initialized #if defined(USE_THREADS) && !defined(ERTS_SMP) - || !erts_equal_tids(main_thread, erts_thr_self()) + || !erts_equal_tids(erts_main_thread, erts_thr_self()) #endif ) return; diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c index 3b3b247020..598bf84c0b 100644 --- a/erts/emulator/beam/erl_lock_check.c +++ b/erts/emulator/beam/erl_lock_check.c @@ -113,9 +113,6 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "environ", NULL }, #endif { "efile_drv", "address" }, -#if defined(ENABLE_CHILD_WAITER_THREAD) || defined(ERTS_SMP) - { "child_status", NULL }, -#endif { "drv_ev_state_grow", NULL, }, { "drv_ev_state", "address" }, { "safe_hash", "address" }, @@ -140,6 +137,8 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "mmap_init_atoms", NULL }, { "drv_tsd", NULL }, { "async_enq_mtx", NULL }, + { "msacc_list_mutex", NULL }, + { "msacc_unmanaged_mutex", NULL }, #ifdef ERTS_SMP { "atom_tab", NULL }, { "misc_op_list_pre_alloc_lock", "address" }, @@ -156,9 +155,6 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "instr", NULL }, { "alcu_allocator", "index" }, { "mseg", NULL }, -#if HALFWORD_HEAP - { "pmmap", NULL }, -#endif #ifdef ERTS_SMP { "port_task_pre_alloc_lock", "address" }, { "proclist_pre_alloc_lock", "address" }, diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c index ff2a355309..d0ffb11e79 100644 --- a/erts/emulator/beam/erl_map.c +++ b/erts/emulator/beam/erl_map.c @@ -172,26 +172,22 @@ BIF_RETTYPE maps_to_list_1(BIF_ALIST_1) { */ const Eterm * -#if HALFWORD_HEAP -erts_maps_get_rel(Eterm key, Eterm map, Eterm *map_base) -#else erts_maps_get(Eterm key, Eterm map) -#endif { Uint32 hx; - if (is_flatmap_rel(map, map_base)) { + if (is_flatmap(map)) { Eterm *ks, *vs; flatmap_t *mp; Uint n, i; - mp = (flatmap_t *)flatmap_val_rel(map, map_base); + mp = (flatmap_t *)flatmap_val(map); n = flatmap_get_size(mp); if (n == 0) { return NULL; } - ks = (Eterm *)tuple_val_rel(mp->keys, map_base) + 1; + ks = (Eterm *)tuple_val(mp->keys) + 1; vs = flatmap_get_values(mp); if (is_immed(key)) { @@ -203,16 +199,16 @@ erts_maps_get(Eterm key, Eterm map) } for (i = 0; i < n; i++) { - if (eq_rel(ks[i], map_base, key, NULL)) { + if (EQ(ks[i], key)) { return &vs[i]; } } return NULL; } - ASSERT(is_hashmap_rel(map, map_base)); + ASSERT(is_hashmap(map)); hx = hashmap_make_hash(key); - return erts_hashmap_get_rel(hx, key, map, map_base); + return erts_hashmap_get(hx, key, map); } BIF_RETTYPE maps_find_2(BIF_ALIST_2) { @@ -1998,11 +1994,7 @@ Eterm* hashmap_iterator_prev(ErtsWStack* s) { } const Eterm * -#if HALFWORD_HEAP -erts_hashmap_get_rel(Uint32 hx, Eterm key, Eterm node, Eterm *map_base) -#else erts_hashmap_get(Uint32 hx, Eterm key, Eterm node) -#endif { Eterm *ptr, hdr, *res; Uint ix, lvl = 0; @@ -2011,7 +2003,7 @@ erts_hashmap_get(Uint32 hx, Eterm key, Eterm node) UseTmpHeapNoproc(2); ASSERT(is_boxed(node)); - ptr = boxed_val_rel(node, map_base); + ptr = boxed_val(node); hdr = *ptr; ASSERT(is_header(hdr)); ASSERT(is_hashmap_header_head(hdr)); @@ -2032,15 +2024,15 @@ erts_hashmap_get(Uint32 hx, Eterm key, Eterm node) node = ptr[ix+1]; if (is_list(node)) { /* LEAF NODE [K|V] */ - ptr = list_val_rel(node,map_base); - res = eq_rel(CAR(ptr), map_base, key, NULL) ? &(CDR(ptr)) : NULL; + ptr = list_val(node); + res = EQ(CAR(ptr), key) ? &(CDR(ptr)) : NULL; break; } hx = hashmap_shift_hash(th,hx,lvl,key); ASSERT(is_boxed(node)); - ptr = boxed_val_rel(node, map_base); + ptr = boxed_val(node); hdr = *ptr; ASSERT(is_header(hdr)); ASSERT(!is_hashmap_header_head(hdr)); @@ -2706,32 +2698,88 @@ BIF_RETTYPE erts_internal_map_to_tuple_keys_1(BIF_ALIST_1) { } /* - * erts_internal:map_type/1 + * erts_internal:term_type/1 * * Used in erts_debug:size/1 */ -BIF_RETTYPE erts_internal_map_type_1(BIF_ALIST_1) { - DECL_AM(hashmap); - DECL_AM(hashmap_node); - DECL_AM(flatmap); - if (is_map(BIF_ARG_1)) { - Eterm hdr = *(boxed_val(BIF_ARG_1)); - ASSERT(is_header(hdr)); - switch (hdr & _HEADER_MAP_SUBTAG_MASK) { - case HAMT_SUBTAG_HEAD_FLATMAP: - BIF_RET(AM_flatmap); - case HAMT_SUBTAG_HEAD_ARRAY: - case HAMT_SUBTAG_HEAD_BITMAP: - BIF_RET(AM_hashmap); - case HAMT_SUBTAG_NODE_BITMAP: - BIF_RET(AM_hashmap_node); - default: - erl_exit(1, "bad header"); +BIF_RETTYPE erts_internal_term_type_1(BIF_ALIST_1) { + Eterm obj = BIF_ARG_1; + switch (primary_tag(obj)) { + case TAG_PRIMARY_LIST: + BIF_RET(ERTS_MAKE_AM("list")); + case TAG_PRIMARY_BOXED: { + Eterm hdr = *boxed_val(obj); + ASSERT(is_header(hdr)); + switch (hdr & _TAG_HEADER_MASK) { + case ARITYVAL_SUBTAG: + BIF_RET(ERTS_MAKE_AM("tuple")); + case EXPORT_SUBTAG: + BIF_RET(ERTS_MAKE_AM("export")); + case FUN_SUBTAG: + BIF_RET(ERTS_MAKE_AM("fun")); + case MAP_SUBTAG: + switch (MAP_HEADER_TYPE(hdr)) { + case MAP_HEADER_TAG_FLATMAP_HEAD : + BIF_RET(ERTS_MAKE_AM("flatmap")); + case MAP_HEADER_TAG_HAMT_HEAD_BITMAP : + case MAP_HEADER_TAG_HAMT_HEAD_ARRAY : + BIF_RET(ERTS_MAKE_AM("hashmap")); + case MAP_HEADER_TAG_HAMT_NODE_BITMAP : + BIF_RET(ERTS_MAKE_AM("hashmap_node")); + default: + erl_exit(ERTS_ABORT_EXIT, "term_type: bad map header type %d\n", MAP_HEADER_TYPE(hdr)); + } + case REFC_BINARY_SUBTAG: + BIF_RET(ERTS_MAKE_AM("refc_binary")); + case HEAP_BINARY_SUBTAG: + BIF_RET(ERTS_MAKE_AM("heap_binary")); + case SUB_BINARY_SUBTAG: + BIF_RET(ERTS_MAKE_AM("sub_binary")); + case BIN_MATCHSTATE_SUBTAG: + BIF_RET(ERTS_MAKE_AM("matchstate")); + case POS_BIG_SUBTAG: + case NEG_BIG_SUBTAG: + BIF_RET(ERTS_MAKE_AM("bignum")); + case REF_SUBTAG: + BIF_RET(ERTS_MAKE_AM("reference")); + case EXTERNAL_REF_SUBTAG: + BIF_RET(ERTS_MAKE_AM("external_reference")); + case EXTERNAL_PID_SUBTAG: + BIF_RET(ERTS_MAKE_AM("external_pid")); + case EXTERNAL_PORT_SUBTAG: + BIF_RET(ERTS_MAKE_AM("external_port")); + case FLOAT_SUBTAG: + BIF_RET(ERTS_MAKE_AM("hfloat")); + default: + erl_exit(ERTS_ABORT_EXIT, "term_type: Invalid tag (0x%X)\n", hdr); + } } + case TAG_PRIMARY_IMMED1: + switch (obj & _TAG_IMMED1_MASK) { + case _TAG_IMMED1_SMALL: + BIF_RET(ERTS_MAKE_AM("fixnum")); + case _TAG_IMMED1_PID: + BIF_RET(ERTS_MAKE_AM("pid")); + case _TAG_IMMED1_PORT: + BIF_RET(ERTS_MAKE_AM("port")); + case _TAG_IMMED1_IMMED2: + switch (obj & _TAG_IMMED2_MASK) { + case _TAG_IMMED2_ATOM: + BIF_RET(ERTS_MAKE_AM("atom")); + case _TAG_IMMED2_CATCH: + BIF_RET(ERTS_MAKE_AM("catch")); + case _TAG_IMMED2_NIL: + BIF_RET(ERTS_MAKE_AM("nil")); + default: + erl_exit(ERTS_ABORT_EXIT, "term_type: Invalid tag (0x%X)\n", obj); + } + default: + erl_exit(ERTS_ABORT_EXIT, "term_type: Invalid tag (0x%X)\n", obj); + } + default: + erl_exit(ERTS_ABORT_EXIT, "term_type: Invalid tag (0x%X)\n", obj); } - BIF_P->fvalue = BIF_ARG_1; - BIF_ERROR(BIF_P, BADMAP); } /* diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h index 4d9d74bc37..052fa99f03 100644 --- a/erts/emulator/beam/erl_map.h +++ b/erts/emulator/beam/erl_map.h @@ -57,7 +57,6 @@ typedef struct flatmap_s { #define hashmap_size(x) (((hashmap_head_t*) hashmap_val(x))->size) -#define hashmap_size_rel(RTERM, BASE) hashmap_size(rterm2wterm(RTERM, BASE)) #define hashmap_make_hash(Key) make_internal_hash(Key) #define hashmap_restore_hash(Heap,Lvl,Key) \ @@ -104,23 +103,9 @@ Eterm erts_hashmap_from_array(ErtsHeapFactory*, Eterm *leafs, Uint n, int rejec Eterm erts_hashmap_from_ks_and_vs_extra(Process *p, Eterm *ks, Eterm *vs, Uint n, Eterm k, Eterm v); -const Eterm * -#if HALFWORD_HEAP -erts_maps_get_rel(Eterm key, Eterm map, Eterm *map_base); -# define erts_maps_get(A, B) erts_maps_get_rel(A, B, NULL) -#else -erts_maps_get(Eterm key, Eterm map); -# define erts_maps_get_rel(A, B, B_BASE) erts_maps_get(A, B) -#endif +const Eterm *erts_maps_get(Eterm key, Eterm map); -const Eterm * -#if HALFWORD_HEAP -erts_hashmap_get_rel(Uint32 hx, Eterm key, Eterm node, Eterm *map_base); -# define erts_hashmap_get(Hx, K, M) erts_hashmap_get_rel(Hx, K, M, NULL) -#else -erts_hashmap_get(Uint32 hx, Eterm key, Eterm map); -# define erts_hashmap_get_rel(Hx, K, M, M_BASE) erts_hashmap_get(Hx, K, M) -#endif +const Eterm *erts_hashmap_get(Uint32 hx, Eterm key, Eterm map); /* hamt nodes v2.0 * diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index fa6b2fc613..88efb2c59f 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -33,8 +33,8 @@ #include "erl_binary.h" #include "dtrace-wrapper.h" -ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(message, - ErlMessage, +ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(message_ref, + ErtsMessageRef, ERL_MESSAGE_BUF_SZ, ERTS_ALC_T_MSG_REF) @@ -44,27 +44,20 @@ ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(message, #undef HARD_DEBUG #endif - - - -#ifdef DEBUG -static ERTS_INLINE int in_heapfrag(const Eterm* ptr, const ErlHeapFragment *bp) +void +init_message(void) { - return ((unsigned)(ptr - bp->mem) < bp->used_size); + init_message_ref_alloc(); } -#endif - -void -init_message(void) +void *erts_alloc_message_ref(void) { - init_message_alloc(); + return (void *) message_ref_alloc(); } -void -free_message(ErlMessage* mp) +void erts_free_message_ref(void *mp) { - message_free(mp); + message_ref_free((ErtsMessageRef *) mp); } /* Allocate message buffer (size in words) */ @@ -74,7 +67,8 @@ new_message_buffer(Uint size) ErlHeapFragment* bp; bp = (ErlHeapFragment*) ERTS_HEAP_ALLOC(ERTS_ALC_T_HEAP_FRAG, ERTS_HEAP_FRAG_SIZE(size)); - ERTS_INIT_HEAP_FRAG(bp, size); + ERTS_INIT_HEAP_FRAG(bp, size, size); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] new message buffer %p\n", erts_get_current_pid(), bp->mem)); return bp; } @@ -203,83 +197,57 @@ free_message_buffer(ErlHeapFragment* bp) }while (bp != NULL); } -static ERTS_INLINE void -link_mbuf_to_proc(Process *proc, ErlHeapFragment *bp) +void +erts_cleanup_messages(ErtsMessage *msgp) { - if (bp) { - /* Link the message buffer */ - bp->next = MBUF(proc); - MBUF(proc) = bp; - MBUF_SIZE(proc) += bp->used_size; - FLAGS(proc) |= F_FORCE_GC; - - /* Move any off_heap's into the process */ - if (bp->off_heap.first != NULL) { - struct erl_off_heap_header** next_p = &bp->off_heap.first; - while (*next_p != NULL) { - next_p = &((*next_p)->next); + ErtsMessage *mp = msgp; + while (mp) { + ErtsMessage *fmp; + ErlHeapFragment *bp; + if (is_non_value(ERL_MESSAGE_TERM(mp))) { + if (is_not_immed(ERL_MESSAGE_TOKEN(mp))) { + bp = (ErlHeapFragment *) mp->data.dist_ext->ext_endp; + erts_cleanup_offheap(&bp->off_heap); } - *next_p = MSO(proc).first; - MSO(proc).first = bp->off_heap.first; - bp->off_heap.first = NULL; - OH_OVERHEAD(&(MSO(proc)), bp->off_heap.overhead); + if (mp->data.dist_ext) + erts_free_dist_ext_copy(mp->data.dist_ext); } + else { + if (mp->data.attached != ERTS_MSG_COMBINED_HFRAG) + bp = mp->data.heap_frag; + else { + bp = mp->hfrag.next; + erts_cleanup_offheap(&mp->hfrag.off_heap); + } + if (bp) + free_message_buffer(bp); + } + fmp = mp; + mp = mp->next; + erts_free_message(fmp); } } -Eterm -erts_msg_distext2heap(Process *pp, - ErtsProcLocks *plcksp, - ErlHeapFragment **bpp, - Eterm *tokenp, - ErtsDistExternal *dist_extp) +ErtsMessage * +erts_realloc_shrink_message(ErtsMessage *mp, Uint sz, Eterm *brefs, Uint brefs_size) { - Eterm msg; - Uint tok_sz = 0; - Eterm *hp = NULL; - ErtsHeapFactory factory; - Sint sz; - - *bpp = NULL; - sz = erts_decode_dist_ext_size(dist_extp); - if (sz < 0) - goto decode_error; - if (is_not_nil(*tokenp)) { - ErlHeapFragment *heap_frag = erts_dist_ext_trailer(dist_extp); - tok_sz = heap_frag->used_size; - sz += tok_sz; - } - if (pp) { - ErlOffHeap *ohp; - hp = erts_alloc_message_heap(sz, bpp, &ohp, pp, plcksp); - } - else { - *bpp = new_message_buffer(sz); - hp = (*bpp)->mem; - } - erts_factory_message_init(&factory, pp, hp, *bpp); - msg = erts_decode_dist_ext(&factory, dist_extp); - if (is_non_value(msg)) - goto decode_error; - if (is_not_nil(*tokenp)) { - ErlHeapFragment *heap_frag = erts_dist_ext_trailer(dist_extp); - hp = erts_produce_heap(&factory, tok_sz, 0); - *tokenp = copy_struct(*tokenp, tok_sz, &hp, factory.off_heap); - erts_cleanup_offheap(&heap_frag->off_heap); + ErtsMessage *nmp = erts_realloc(ERTS_ALC_T_MSG, mp, + sizeof(ErtsMessage) + (sz - 1)*sizeof(Eterm)); + if (nmp != mp) { + Eterm *sp = &mp->hfrag.mem[0]; + Eterm *ep = sp + sz; + Sint offs = &nmp->hfrag.mem[0] - sp; + erts_offset_off_heap(&nmp->hfrag.off_heap, offs, sp, ep); + erts_offset_heap(&nmp->hfrag.mem[0], sz, offs, sp, ep); + if (brefs && brefs_size) + erts_offset_heap_ptr(brefs, brefs_size, offs, sp, ep); } - erts_free_dist_ext_copy(dist_extp); - erts_factory_close(&factory); - return msg; - decode_error: - if (is_not_nil(*tokenp)) { - ErlHeapFragment *heap_frag = erts_dist_ext_trailer(dist_extp); - erts_cleanup_offheap(&heap_frag->off_heap); - } - erts_free_dist_ext_copy(dist_extp); - *bpp = NULL; - return THE_NON_VALUE; - } + nmp->hfrag.used_size = sz; + nmp->hfrag.alloc_size = sz; + + return nmp; +} void erts_queue_dist_message(Process *rcvr, @@ -287,7 +255,7 @@ erts_queue_dist_message(Process *rcvr, ErtsDistExternal *dist_ext, Eterm token) { - ErlMessage* mp; + ErtsMessage* mp; #ifdef USE_VM_PROBES Sint tok_label = 0; Sint tok_lastcnt = 0; @@ -299,7 +267,17 @@ erts_queue_dist_message(Process *rcvr, ERTS_SMP_LC_ASSERT(*rcvr_locks == erts_proc_lc_my_proc_locks(rcvr)); - mp = message_alloc(); + mp = erts_alloc_message(0, NULL); + mp->data.dist_ext = dist_ext; + + ERL_MESSAGE_TERM(mp) = THE_NON_VALUE; +#ifdef USE_VM_PROBES + ERL_MESSAGE_DT_UTAG(mp) = NIL; + if (token == am_have_dt_utag) + ERL_MESSAGE_TOKEN(mp) = NIL; + else +#endif + ERL_MESSAGE_TOKEN(mp) = token; #ifdef ERTS_SMP if (!(*rcvr_locks & ERTS_PROC_LOCK_MSGQ)) { @@ -318,64 +296,46 @@ erts_queue_dist_message(Process *rcvr, if (!(*rcvr_locks & ERTS_PROC_LOCK_MSGQ)) erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ); /* Drop message if receiver is exiting or has a pending exit ... */ - if (is_not_nil(token)) { - ErlHeapFragment *heap_frag; - heap_frag = erts_dist_ext_trailer(mp->data.dist_ext); - erts_cleanup_offheap(&heap_frag->off_heap); - } - erts_free_dist_ext_copy(dist_ext); - message_free(mp); + erts_cleanup_messages(mp); } else #endif if (IS_TRACED_FL(rcvr, F_TRACE_RECEIVE)) { /* Ahh... need to decode it in order to trace it... */ - ErlHeapFragment *mbuf; - Eterm msg; if (!(*rcvr_locks & ERTS_PROC_LOCK_MSGQ)) erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ); - message_free(mp); - msg = erts_msg_distext2heap(rcvr, rcvr_locks, &mbuf, &token, dist_ext); - if (is_value(msg)) + if (!erts_decode_dist_message(rcvr, *rcvr_locks, mp, 0)) + erts_free_message(mp); + else { + Eterm msg = ERL_MESSAGE_TERM(mp); + token = ERL_MESSAGE_TOKEN(mp); #ifdef USE_VM_PROBES - if (DTRACE_ENABLED(message_queued)) { - DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE); - - dtrace_proc_str(rcvr, receiver_name); - if (token != NIL && token != am_have_dt_utag) { - tok_label = signed_val(SEQ_TRACE_T_LABEL(token)); - tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token)); - tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token)); - } - DTRACE6(message_queued, - receiver_name, size_object(msg), rcvr->msg.len, - tok_label, tok_lastcnt, tok_serial); - } + if (DTRACE_ENABLED(message_queued)) { + DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE); + + dtrace_proc_str(rcvr, receiver_name); + if (have_seqtrace(token)) { + tok_label = signed_val(SEQ_TRACE_T_LABEL(token)); + tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token)); + tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token)); + } + DTRACE6(message_queued, + receiver_name, size_object(msg), rcvr->msg.len, + tok_label, tok_lastcnt, tok_serial); + } #endif - erts_queue_message(rcvr, rcvr_locks, mbuf, msg, token); + erts_queue_message(rcvr, rcvr_locks, mp, msg, token); + } } else { /* Enqueue message on external format */ - ERL_MESSAGE_TERM(mp) = THE_NON_VALUE; -#ifdef USE_VM_PROBES - ERL_MESSAGE_DT_UTAG(mp) = NIL; - if (token == am_have_dt_utag) { - ERL_MESSAGE_TOKEN(mp) = NIL; - } else { -#endif - ERL_MESSAGE_TOKEN(mp) = token; -#ifdef USE_VM_PROBES - } -#endif - mp->next = NULL; - #ifdef USE_VM_PROBES if (DTRACE_ENABLED(message_queued)) { DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE); dtrace_proc_str(rcvr, receiver_name); - if (token != NIL && token != am_have_dt_utag) { + if (have_seqtrace(token)) { tok_label = signed_val(SEQ_TRACE_T_LABEL(token)); tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token)); tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token)); @@ -388,7 +348,7 @@ erts_queue_dist_message(Process *rcvr, tok_label, tok_lastcnt, tok_serial); } #endif - mp->data.dist_ext = dist_ext; + LINK_MESSAGE(rcvr, mp); if (!(*rcvr_locks & ERTS_PROC_LOCK_MSGQ)) @@ -408,9 +368,9 @@ erts_queue_dist_message(Process *rcvr, static Sint queue_message(Process *c_p, Process* receiver, - ErtsProcLocks *receiver_locks, erts_aint32_t *receiver_state, - ErlHeapFragment* bp, + ErtsProcLocks *receiver_locks, + ErtsMessage* mp, Eterm message, Eterm seq_trace_token #ifdef USE_VM_PROBES @@ -419,31 +379,24 @@ queue_message(Process *c_p, ) { Sint res; - ErlMessage* mp; int locked_msgq = 0; - erts_aint_t state; - -#ifndef ERTS_SMP - ASSERT(bp != NULL || receiver->mbuf == NULL); -#endif + erts_aint32_t state; ERTS_SMP_LC_ASSERT(*receiver_locks == erts_proc_lc_my_proc_locks(receiver)); - mp = message_alloc(); - - if (receiver_state) - state = *receiver_state; - else - state = erts_smp_atomic32_read_acqb(&receiver->state); - #ifdef ERTS_SMP - if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) - goto exiting; - if (!(*receiver_locks & ERTS_PROC_LOCK_MSGQ)) { if (erts_smp_proc_trylock(receiver, ERTS_PROC_LOCK_MSGQ) == EBUSY) { ErtsProcLocks need_locks = ERTS_PROC_LOCK_MSGQ; + + if (receiver_state) + state = *receiver_state; + else + state = erts_smp_atomic32_read_nob(&receiver->state); + if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) + goto exiting; + if (*receiver_locks & ERTS_PROC_LOCK_STATUS) { erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_STATUS); need_locks |= ERTS_PROC_LOCK_STATUS; @@ -451,13 +404,12 @@ queue_message(Process *c_p, erts_smp_proc_lock(receiver, need_locks); } locked_msgq = 1; - state = erts_smp_atomic32_read_nob(&receiver->state); - if (receiver_state) - *receiver_state = state; } #endif + state = erts_smp_atomic32_read_nob(&receiver->state); + if (state & (ERTS_PSFLG_PENDING_EXIT|ERTS_PSFLG_EXITING)) { #ifdef ERTS_SMP exiting: @@ -465,9 +417,7 @@ queue_message(Process *c_p, /* Drop message if receiver is exiting or has a pending exit... */ if (locked_msgq) erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_MSGQ); - if (bp) - free_message_buffer(bp); - message_free(mp); + erts_cleanup_messages(mp); return 0; } @@ -476,13 +426,9 @@ queue_message(Process *c_p, #ifdef USE_VM_PROBES ERL_MESSAGE_DT_UTAG(mp) = dt_utag; #endif - mp->next = NULL; - mp->data.heap_frag = bp; -#ifndef ERTS_SMP res = receiver->msg.len; -#else - res = receiver->msg_inq.len; +#ifdef ERTS_SMP if (*receiver_locks & ERTS_PROC_LOCK_MAIN) { /* * We move 'in queue' to 'private queue' and place @@ -492,7 +438,7 @@ queue_message(Process *c_p, * we don't need to include the 'in queue' in * the root set when garbage collecting. */ - res += receiver->msg.len; + res += receiver->msg_inq.len; ERTS_SMP_MSGQ_MV_INQ2PRIVQ(receiver); LINK_MESSAGE_PRIVQ(receiver, mp); } @@ -544,19 +490,19 @@ queue_message(Process *c_p, void #ifdef USE_VM_PROBES erts_queue_message_probe(Process* receiver, ErtsProcLocks *receiver_locks, - ErlHeapFragment* bp, + ErtsMessage* mp, Eterm message, Eterm seq_trace_token, Eterm dt_utag) #else erts_queue_message(Process* receiver, ErtsProcLocks *receiver_locks, - ErlHeapFragment* bp, + ErtsMessage* mp, Eterm message, Eterm seq_trace_token) #endif { queue_message(NULL, receiver, - receiver_locks, NULL, - bp, + receiver_locks, + mp, message, seq_trace_token #ifdef USE_VM_PROBES @@ -566,245 +512,37 @@ erts_queue_message(Process* receiver, ErtsProcLocks *receiver_locks, } void -erts_link_mbuf_to_proc(struct process *proc, ErlHeapFragment *bp) +erts_link_mbuf_to_proc(Process *proc, ErlHeapFragment *first_bp) { - Eterm* htop = HEAP_TOP(proc); - - link_mbuf_to_proc(proc, bp); - if (htop < HEAP_LIMIT(proc)) { - *htop = make_pos_bignum_header(HEAP_LIMIT(proc)-htop-1); - HEAP_TOP(proc) = HEAP_LIMIT(proc); - } -} - -/* - * Moves content of message buffer attached to a message into a heap. - * The message buffer is deallocated. - */ -void -erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg) -{ - struct erl_off_heap_header* oh; - Eterm term, token, *fhp, *hp; - Sint offs; - Uint sz; - ErlHeapFragment *bp; -#ifdef USE_VM_PROBES - Eterm utag; -#endif - -#ifdef HARD_DEBUG - struct erl_off_heap_header* dbg_oh_start = off_heap->first; - Eterm dbg_term, dbg_token; - ErlHeapFragment *dbg_bp; - Uint *dbg_hp, *dbg_thp_start; - Uint dbg_term_sz, dbg_token_sz; -#ifdef USE_VM_PROBES - Eterm dbg_utag; - Uint dbg_utag_sz; -#endif -#endif - - bp = msg->data.heap_frag; - term = ERL_MESSAGE_TERM(msg); - token = ERL_MESSAGE_TOKEN(msg); -#ifdef USE_VM_PROBES - utag = ERL_MESSAGE_DT_UTAG(msg); -#endif - if (!bp) { -#ifdef USE_VM_PROBES - ASSERT(is_immed(term) && is_immed(token) && is_immed(utag)); -#else - ASSERT(is_immed(term) && is_immed(token)); -#endif - return; - } - -#ifdef HARD_DEBUG - dbg_term_sz = size_object(term); - dbg_token_sz = size_object(token); - dbg_bp = new_message_buffer(dbg_term_sz + dbg_token_sz); -#ifdef USE_VM_PROBES - dbg_utag_sz = size_object(utag); - dbg_bp = new_message_buffer(dbg_term_sz + dbg_token_sz + dbg_utag_sz ); -#endif - /*ASSERT(dbg_term_sz + dbg_token_sz == erts_msg_used_frag_sz(msg)); - Copied size may be smaller due to removed SubBins's or garbage. - Copied size may be larger due to duplicated shared terms. - */ - dbg_hp = dbg_bp->mem; - dbg_term = copy_struct(term, dbg_term_sz, &dbg_hp, &dbg_bp->off_heap); - dbg_token = copy_struct(token, dbg_token_sz, &dbg_hp, &dbg_bp->off_heap); -#ifdef USE_VM_PROBES - dbg_utag = copy_struct(utag, dbg_utag_sz, &dbg_hp, &dbg_bp->off_heap); -#endif - dbg_thp_start = *hpp; -#endif - - if (bp->next != NULL) { - move_multi_frags(hpp, off_heap, bp, msg->m, -#ifdef USE_VM_PROBES - 3 -#else - 2 -#endif - ); - goto copy_done; - } - - OH_OVERHEAD(off_heap, bp->off_heap.overhead); - sz = bp->used_size; - - ASSERT(is_immed(term) || in_heapfrag(ptr_val(term),bp)); - ASSERT(is_immed(token) || in_heapfrag(ptr_val(token),bp)); - - fhp = bp->mem; - hp = *hpp; - offs = hp - fhp; - - oh = NULL; - while (sz--) { - Uint cpy_sz; - Eterm val = *fhp++; - - switch (primary_tag(val)) { - case TAG_PRIMARY_IMMED1: - *hp++ = val; - break; - case TAG_PRIMARY_LIST: - case TAG_PRIMARY_BOXED: - ASSERT(in_heapfrag(ptr_val(val), bp)); - *hp++ = offset_ptr(val, offs); - break; - case TAG_PRIMARY_HEADER: - *hp++ = val; - switch (val & _HEADER_SUBTAG_MASK) { - case ARITYVAL_SUBTAG: - break; - case REFC_BINARY_SUBTAG: - case FUN_SUBTAG: - case EXTERNAL_PID_SUBTAG: - case EXTERNAL_PORT_SUBTAG: - case EXTERNAL_REF_SUBTAG: - oh = (struct erl_off_heap_header*) (hp-1); - cpy_sz = thing_arityval(val); - goto cpy_words; - default: - cpy_sz = header_arity(val); - - cpy_words: - ASSERT(sz >= cpy_sz); - sz -= cpy_sz; - while (cpy_sz >= 8) { - cpy_sz -= 8; - *hp++ = *fhp++; - *hp++ = *fhp++; - *hp++ = *fhp++; - *hp++ = *fhp++; - *hp++ = *fhp++; - *hp++ = *fhp++; - *hp++ = *fhp++; - *hp++ = *fhp++; - } - switch (cpy_sz) { - case 7: *hp++ = *fhp++; - case 6: *hp++ = *fhp++; - case 5: *hp++ = *fhp++; - case 4: *hp++ = *fhp++; - case 3: *hp++ = *fhp++; - case 2: *hp++ = *fhp++; - case 1: *hp++ = *fhp++; - default: break; + if (first_bp) { + ErlHeapFragment *bp = first_bp; + + while (1) { + /* Move any off_heap's into the process */ + if (bp->off_heap.first != NULL) { + struct erl_off_heap_header** next_p = &bp->off_heap.first; + while (*next_p != NULL) { + next_p = &((*next_p)->next); } - if (oh) { - /* Add to offheap list */ - oh->next = off_heap->first; - off_heap->first = oh; - ASSERT(*hpp <= (Eterm*)oh); - ASSERT(hp > (Eterm*)oh); - oh = NULL; - } - break; + *next_p = MSO(proc).first; + MSO(proc).first = bp->off_heap.first; + bp->off_heap.first = NULL; + OH_OVERHEAD(&(MSO(proc)), bp->off_heap.overhead); } - break; + MBUF_SIZE(proc) += bp->used_size; + if (!bp->next) + break; + bp = bp->next; } - } - - ASSERT(bp->used_size == hp - *hpp); - *hpp = hp; - if (is_not_immed(token)) { - ASSERT(in_heapfrag(ptr_val(token), bp)); - ERL_MESSAGE_TOKEN(msg) = offset_ptr(token, offs); -#ifdef HARD_DEBUG - ASSERT(dbg_thp_start <= ptr_val(ERL_MESSAGE_TOKEN(msg))); - ASSERT(hp > ptr_val(ERL_MESSAGE_TOKEN(msg))); -#endif - } - - if (is_not_immed(term)) { - ASSERT(in_heapfrag(ptr_val(term),bp)); - ERL_MESSAGE_TERM(msg) = offset_ptr(term, offs); -#ifdef HARD_DEBUG - ASSERT(dbg_thp_start <= ptr_val(ERL_MESSAGE_TERM(msg))); - ASSERT(hp > ptr_val(ERL_MESSAGE_TERM(msg))); -#endif - } -#ifdef USE_VM_PROBES - if (is_not_immed(utag)) { - ASSERT(in_heapfrag(ptr_val(utag), bp)); - ERL_MESSAGE_DT_UTAG(msg) = offset_ptr(utag, offs); -#ifdef HARD_DEBUG - ASSERT(dbg_thp_start <= ptr_val(ERL_MESSAGE_DT_UTAG(msg))); - ASSERT(hp > ptr_val(ERL_MESSAGE_DT_UTAG(msg))); -#endif - } -#endif - -copy_done: - -#ifdef HARD_DEBUG - { - int i, j; - ErlHeapFragment* frag; - { - struct erl_off_heap_header* dbg_oh = off_heap->first; - i = j = 0; - while (dbg_oh != dbg_oh_start) { - dbg_oh = dbg_oh->next; - i++; - } - for (frag=bp; frag; frag=frag->next) { - dbg_oh = frag->off_heap.first; - while (dbg_oh) { - dbg_oh = dbg_oh->next; - j++; - } - } - ASSERT(i == j); - } + /* Link the message buffer */ + bp->next = MBUF(proc); + MBUF(proc) = first_bp; } -#endif - - - bp->off_heap.first = NULL; - free_message_buffer(bp); - msg->data.heap_frag = NULL; - -#ifdef HARD_DEBUG - ASSERT(eq(ERL_MESSAGE_TERM(msg), dbg_term)); - ASSERT(eq(ERL_MESSAGE_TOKEN(msg), dbg_token)); -#ifdef USE_VM_PROBES - ASSERT(eq(ERL_MESSAGE_DT_UTAG(msg), dbg_utag)); -#endif - free_message_buffer(dbg_bp); -#endif - } - Uint -erts_msg_attached_data_size_aux(ErlMessage *msg) +erts_msg_attached_data_size_aux(ErtsMessage *msg) { Sint sz; ASSERT(is_non_value(ERL_MESSAGE_TERM(msg))); @@ -833,29 +571,87 @@ erts_msg_attached_data_size_aux(ErlMessage *msg) return sz; } -void -erts_move_msg_attached_data_to_heap(ErtsHeapFactory* factory, - ErlMessage *msg) +ErtsMessage * +erts_try_alloc_message_on_heap(Process *pp, + erts_aint32_t *psp, + ErtsProcLocks *plp, + Uint sz, + Eterm **hpp, + ErlOffHeap **ohpp, + int *on_heap_p) { - if (is_value(ERL_MESSAGE_TERM(msg))) - erts_move_msg_mbuf_to_heap(&factory->hp, factory->off_heap, msg); - else if (msg->data.dist_ext) { - ASSERT(msg->data.dist_ext->heap_size >= 0); - if (is_not_nil(ERL_MESSAGE_TOKEN(msg))) { - ErlHeapFragment *heap_frag; - heap_frag = erts_dist_ext_trailer(msg->data.dist_ext); - ERL_MESSAGE_TOKEN(msg) = copy_struct(ERL_MESSAGE_TOKEN(msg), - heap_frag->used_size, - &factory->hp, - factory->off_heap); - erts_cleanup_offheap(&heap_frag->off_heap); +#ifdef ERTS_SMP + int locked_main = 0; +#endif + ErtsMessage *mp; + + ASSERT(!(*psp & ERTS_PSFLG_OFF_HEAP_MSGQ)); + + if ( +#if defined(ERTS_SMP) + *plp & ERTS_PROC_LOCK_MAIN +#else + 1 +#endif + ) { +#ifdef ERTS_SMP + try_on_heap: +#endif + if ((*psp & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) + || (pp->flags & F_DISABLE_GC) + || HEAP_LIMIT(pp) - HEAP_TOP(pp) <= sz) { + /* + * The heap is either potentially in an inconsistent + * state, or not large enough. + */ +#ifdef ERTS_SMP + if (locked_main) { + *plp &= ~ERTS_PROC_LOCK_MAIN; + erts_smp_proc_unlock(pp, ERTS_PROC_LOCK_MAIN); + } +#endif + goto in_message_fragment; } - ERL_MESSAGE_TERM(msg) = erts_decode_dist_ext(factory, - msg->data.dist_ext); - erts_free_dist_ext_copy(msg->data.dist_ext); - msg->data.dist_ext = NULL; + + *hpp = HEAP_TOP(pp); + HEAP_TOP(pp) = *hpp + sz; + *ohpp = &MSO(pp); + mp = erts_alloc_message(0, NULL); + mp->data.attached = NULL; + *on_heap_p = !0; + } +#ifdef ERTS_SMP + else if (erts_smp_proc_trylock(pp, ERTS_PROC_LOCK_MAIN) == 0) { + locked_main = 1; + *psp = erts_smp_atomic32_read_nob(&pp->state); + *plp |= ERTS_PROC_LOCK_MAIN; + goto try_on_heap; } - /* else: bad external detected when calculating size */ +#endif + else { + in_message_fragment: + if (!((*psp) & ERTS_PSFLG_ON_HEAP_MSGQ)) { + mp = erts_alloc_message(sz, hpp); + *ohpp = sz == 0 ? NULL : &mp->hfrag.off_heap; + } + else { + mp = erts_alloc_message(0, NULL); + if (!sz) { + *hpp = NULL; + *ohpp = NULL; + } + else { + ErlHeapFragment *bp; + bp = new_message_buffer(sz); + *hpp = &bp->mem[0]; + mp->data.heap_frag = bp; + *ohpp = &bp->off_heap; + } + } + *on_heap_p = 0; + } + + return mp; } /* @@ -870,7 +666,8 @@ erts_send_message(Process* sender, unsigned flags) { Uint msize; - ErlHeapFragment* bp = NULL; + ErtsMessage* mp; + ErlOffHeap *ohp; Eterm token = NIL; Sint res = 0; #ifdef USE_VM_PROBES @@ -879,67 +676,96 @@ erts_send_message(Process* sender, Sint tok_label = 0; Sint tok_lastcnt = 0; Sint tok_serial = 0; + Eterm utag = NIL; +#endif + erts_aint32_t receiver_state; +#ifdef SHCOPY_SEND + erts_shcopy_t info; #endif BM_STOP_TIMER(system); BM_MESSAGE(message,sender,receiver); BM_START_TIMER(send); - #ifdef USE_VM_PROBES +#ifdef USE_VM_PROBES *sender_name = *receiver_name = '\0'; - if (DTRACE_ENABLED(message_send)) { + if (DTRACE_ENABLED(message_send)) { erts_snprintf(sender_name, sizeof(DTRACE_CHARBUF_NAME(sender_name)), "%T", sender->common.id); erts_snprintf(receiver_name, sizeof(DTRACE_CHARBUF_NAME(receiver_name)), "%T", receiver->common.id); } #endif + + receiver_state = erts_smp_atomic32_read_nob(&receiver->state); + if (SEQ_TRACE_TOKEN(sender) != NIL && !(flags & ERTS_SND_FLG_NO_SEQ_TRACE)) { Eterm* hp; Eterm stoken = SEQ_TRACE_TOKEN(sender); Uint seq_trace_size = 0; #ifdef USE_VM_PROBES Uint dt_utag_size = 0; - Eterm utag = NIL; #endif - - BM_SWAP_TIMER(send,size); - msize = size_object(message); - BM_SWAP_TIMER(size,send); - -#ifdef USE_VM_PROBES - if (stoken != am_have_dt_utag) { -#endif - + BM_SWAP_TIMER(send,size); + + /* SHCOPY corrupts the heap between + * copy_shared_calculate, and + * copy_shared_perform. (it inserts move_markers like the gc). + * Make sure we don't use the heap between those instances. + */ + if (have_seqtrace(stoken)) { seq_trace_update_send(sender); seq_trace_output(stoken, message, SEQ_TRACE_SEND, receiver->common.id, sender); seq_trace_size = 6; /* TUPLE5 */ -#ifdef USE_VM_PROBES - } - if (DT_UTAG_FLAGS(sender) & DT_UTAG_SPREADING) { - dt_utag_size = size_object(DT_UTAG(sender)); - } else if (stoken == am_have_dt_utag ) { - stoken = NIL; } +#ifdef USE_VM_PROBES + if (DT_UTAG_FLAGS(sender) & DT_UTAG_SPREADING) { + dt_utag_size = size_object(DT_UTAG(sender)); + } else if (stoken == am_have_dt_utag ) { + stoken = NIL; + } +#endif + +#ifdef SHCOPY_SEND + INITIALIZE_SHCOPY(info); + msize = copy_shared_calculate(message, &info); +#else + msize = size_object(message); #endif + BM_SWAP_TIMER(size,send); - bp = new_message_buffer(msize + seq_trace_size + mp = erts_alloc_message_heap_state(receiver, + &receiver_state, + receiver_locks, + (msize #ifdef USE_VM_PROBES - + dt_utag_size + + dt_utag_size #endif - ); - hp = bp->mem; + + seq_trace_size), + &hp, + &ohp); BM_SWAP_TIMER(send,copy); - token = copy_struct(stoken, - seq_trace_size, - &hp, - &bp->off_heap); - message = copy_struct(message, msize, &hp, &bp->off_heap); +#ifdef SHCOPY_SEND + if (is_not_immed(message)) + message = copy_shared_perform(message, msize, &info, &hp, ohp); + DESTROY_SHCOPY(info); +#else + if (is_not_immed(message)) + message = copy_struct(message, msize, &hp, ohp); +#endif + if (is_immed(stoken)) + token = stoken; + else + token = copy_struct(stoken, seq_trace_size, &hp, ohp); + #ifdef USE_VM_PROBES if (DT_UTAG_FLAGS(sender) & DT_UTAG_SPREADING) { - utag = copy_struct(DT_UTAG(sender), dt_utag_size, &hp, &bp->off_heap); + if (is_immed(DT_UTAG(sender))) + utag = DT_UTAG(sender); + else + utag = copy_struct(DT_UTAG(sender), dt_utag_size, &hp, ohp); #ifdef DTRACE_TAG_HARDDEBUG erts_fprintf(stderr, "Dtrace -> (%T) Spreading tag (%T) with " @@ -952,7 +778,7 @@ erts_send_message(Process* sender, #ifdef USE_VM_PROBES if (DTRACE_ENABLED(message_send)) { - if (stoken != NIL && stoken != am_have_dt_utag) { + if (have_seqtrace(stoken)) { tok_label = signed_val(SEQ_TRACE_T_LABEL(stoken)); tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(stoken)); tok_serial = signed_val(SEQ_TRACE_T_SERIAL(stoken)); @@ -961,103 +787,63 @@ erts_send_message(Process* sender, msize, tok_label, tok_lastcnt, tok_serial); } #endif - res = queue_message(NULL, - receiver, - receiver_locks, - NULL, - bp, - message, - token -#ifdef USE_VM_PROBES - , utag -#endif - ); - BM_SWAP_TIMER(send,system); - } else if (sender == receiver) { - /* Drop message if receiver has a pending exit ... */ -#ifdef ERTS_SMP - ErtsProcLocks need_locks = (~(*receiver_locks) - & (ERTS_PROC_LOCK_MSGQ - | ERTS_PROC_LOCK_STATUS)); - if (need_locks) { - *receiver_locks |= need_locks; - if (erts_smp_proc_trylock(receiver, need_locks) == EBUSY) { - if (need_locks == ERTS_PROC_LOCK_MSGQ) { - erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_STATUS); - need_locks = ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS; - } - erts_smp_proc_lock(receiver, need_locks); - } + } else { + Eterm *hp; + + if (receiver == sender && !(receiver_state & ERTS_PSFLG_OFF_HEAP_MSGQ)) { + mp = erts_alloc_message(0, NULL); + msize = 0; } - if (!ERTS_PROC_PENDING_EXIT(receiver)) + else { + BM_SWAP_TIMER(send,size); +#ifdef SHCOPY_SEND + INITIALIZE_SHCOPY(info); + msize = copy_shared_calculate(message, &info); +#else + msize = size_object(message); #endif - { - ErlMessage* mp = message_alloc(); - - DTRACE6(message_send, sender_name, receiver_name, - size_object(message), tok_label, tok_lastcnt, tok_serial); - mp->data.attached = NULL; - ERL_MESSAGE_TERM(mp) = message; - ERL_MESSAGE_TOKEN(mp) = NIL; -#ifdef USE_VM_PROBES - ERL_MESSAGE_DT_UTAG(mp) = NIL; + BM_SWAP_TIMER(size,send); + + mp = erts_alloc_message_heap_state(receiver, + &receiver_state, + receiver_locks, + msize, + &hp, + &ohp); + BM_SWAP_TIMER(send,copy); +#ifdef SHCOPY_SEND + if (is_not_immed(message)) + message = copy_shared_perform(message, msize, &info, &hp, ohp); + DESTROY_SHCOPY(info); +#else + if (is_not_immed(message)) + message = copy_struct(message, msize, &hp, ohp); #endif - mp->next = NULL; - /* - * We move 'in queue' to 'private queue' and place - * message at the end of 'private queue' in order - * to ensure that the 'in queue' doesn't contain - * references into the heap. By ensuring this, - * we don't need to include the 'in queue' in - * the root set when garbage collecting. - */ - - ERTS_SMP_MSGQ_MV_INQ2PRIVQ(receiver); - LINK_MESSAGE_PRIVQ(receiver, mp); - - res = receiver->msg.len; - - if (IS_TRACED_FL(receiver, F_TRACE_RECEIVE)) { - trace_receive(receiver, message); - } + BM_MESSAGE_COPIED(msz); + BM_SWAP_TIMER(copy,send); } - BM_SWAP_TIMER(send,system); - } else { - ErlOffHeap *ohp; - Eterm *hp; - erts_aint32_t state; - - BM_SWAP_TIMER(send,size); - msize = size_object(message); - BM_SWAP_TIMER(size,send); - hp = erts_alloc_message_heap_state(msize, - &bp, - &ohp, - receiver, - receiver_locks, - &state); - BM_SWAP_TIMER(send,copy); - message = copy_struct(message, msize, &hp, ohp); - BM_MESSAGE_COPIED(msz); - BM_SWAP_TIMER(copy,send); DTRACE6(message_send, sender_name, receiver_name, msize, tok_label, tok_lastcnt, tok_serial); - res = queue_message(sender, - receiver, - receiver_locks, - &state, - bp, - message, - token + } + + res = queue_message(sender, + receiver, + &receiver_state, + receiver_locks, + mp, + message, + token #ifdef USE_VM_PROBES - , NIL + , utag #endif - ); - BM_SWAP_TIMER(send,system); - } - return res; + ); + + BM_SWAP_TIMER(send,system); + + return res; } + /* * This function delivers an EXIT message to a process * which is trapping EXITs. @@ -1075,46 +861,572 @@ erts_deliver_exit_message(Eterm from, Process *to, ErtsProcLocks *to_locksp, Uint sz_from; Eterm* hp; Eterm temptoken; - ErlHeapFragment* bp = NULL; - - if (token != NIL -#ifdef USE_VM_PROBES - && token != am_have_dt_utag + ErtsMessage* mp; + ErlOffHeap *ohp; +#ifdef SHCOPY_SEND + erts_shcopy_t info; #endif - ) { + if (have_seqtrace(token)) { ASSERT(is_tuple(token)); - sz_reason = size_object(reason); sz_token = size_object(token); sz_from = size_object(from); - bp = new_message_buffer(sz_reason + sz_from + sz_token + 4); - hp = bp->mem; - mess = copy_struct(reason, sz_reason, &hp, &bp->off_heap); - from_copy = copy_struct(from, sz_from, &hp, &bp->off_heap); +#ifdef SHCOPY_SEND + INITIALIZE_SHCOPY(info); + sz_reason = copy_shared_calculate(reason, &info); +#else + sz_reason = size_object(reason); +#endif + mp = erts_alloc_message_heap(to, to_locksp, + sz_reason + sz_from + sz_token + 4, + &hp, &ohp); +#ifdef SHCOPY_SEND + mess = copy_shared_perform(reason, sz_reason, &info, &hp, ohp); + DESTROY_SHCOPY(info); +#else + mess = copy_struct(reason, sz_reason, &hp, ohp); +#endif + from_copy = copy_struct(from, sz_from, &hp, ohp); save = TUPLE3(hp, am_EXIT, from_copy, mess); hp += 4; /* the trace token must in this case be updated by the caller */ seq_trace_output(token, save, SEQ_TRACE_SEND, to->common.id, NULL); - temptoken = copy_struct(token, sz_token, &hp, &bp->off_heap); - erts_queue_message(to, to_locksp, bp, save, temptoken); + temptoken = copy_struct(token, sz_token, &hp, ohp); + erts_queue_message(to, to_locksp, mp, save, temptoken); } else { - ErlOffHeap *ohp; - sz_reason = size_object(reason); sz_from = IS_CONST(from) ? 0 : size_object(from); +#ifdef SHCOPY_SEND + INITIALIZE_SHCOPY(info); + sz_reason = copy_shared_calculate(reason, &info); +#else + sz_reason = size_object(reason); +#endif + mp = erts_alloc_message_heap(to, to_locksp, + sz_reason+sz_from+4, &hp, &ohp); - hp = erts_alloc_message_heap(sz_reason+sz_from+4, - &bp, - &ohp, - to, - to_locksp); - +#ifdef SHCOPY_SEND + mess = copy_shared_perform(reason, sz_reason, &info, &hp, ohp); + DESTROY_SHCOPY(info); +#else mess = copy_struct(reason, sz_reason, &hp, ohp); +#endif from_copy = (IS_CONST(from) ? from : copy_struct(from, sz_from, &hp, ohp)); save = TUPLE3(hp, am_EXIT, from_copy, mess); - erts_queue_message(to, to_locksp, bp, save, NIL); + erts_queue_message(to, to_locksp, mp, save, NIL); + } +} + +void erts_save_message_in_proc(Process *p, ErtsMessage *msgp) +{ + ErlHeapFragment *hfp; + + if (msgp->data.attached == ERTS_MSG_COMBINED_HFRAG) + hfp = &msgp->hfrag; + else if (msgp->data.attached) { + hfp = msgp->data.heap_frag; + } + else { + erts_free_message(msgp); + return; /* Nothing to save */ + } + + while (1) { + struct erl_off_heap_header *ohhp = hfp->off_heap.first; + if (ohhp) { + for ( ; ohhp->next; ohhp = ohhp->next) + ; + ohhp->next = p->off_heap.first; + p->off_heap.first = hfp->off_heap.first; + hfp->off_heap.first = NULL; + } + p->off_heap.overhead += hfp->off_heap.overhead; + hfp->off_heap.overhead = 0; + p->mbuf_sz += hfp->used_size; + + if (!hfp->next) + break; + hfp = hfp->next; + } + + msgp->next = p->msg_frag; + p->msg_frag = msgp; +} + +Sint +erts_move_messages_off_heap(Process *c_p) +{ + int reds = 1; + /* + * Move all messages off heap. This *only* occurs when the + * process had off heap message disabled and just enabled + * it... + */ + ErtsMessage *mp; + + reds += c_p->msg.len / 10; + + ASSERT(erts_smp_atomic32_read_nob(&c_p->state) + & ERTS_PSFLG_OFF_HEAP_MSGQ); + ASSERT(c_p->flags & F_OFF_HEAP_MSGQ_CHNG); + + for (mp = c_p->msg.first; mp; mp = mp->next) { + Uint msg_sz, token_sz; +#ifdef USE_VM_PROBES + Uint utag_sz; +#endif + Eterm *hp; + ErlHeapFragment *hfrag; + + if (mp->data.attached) + continue; + + if (is_immed(ERL_MESSAGE_TERM(mp)) +#ifdef USE_VM_PROBES + && is_immed(ERL_MESSAGE_DT_UTAG(mp)) +#endif + && is_not_immed(ERL_MESSAGE_TOKEN(mp))) + continue; + + /* + * The message refers into the heap. Copy the message + * from the heap into a heap fragment and attach + * it to the message... + */ + msg_sz = size_object(ERL_MESSAGE_TERM(mp)); +#ifdef USE_VM_PROBES + utag_sz = size_object(ERL_MESSAGE_DT_UTAG(mp)); +#endif + token_sz = size_object(ERL_MESSAGE_TOKEN(mp)); + + hfrag = new_message_buffer(msg_sz +#ifdef USE_VM_PROBES + + utag_sz +#endif + + token_sz); + hp = hfrag->mem; + if (is_not_immed(ERL_MESSAGE_TERM(mp))) + ERL_MESSAGE_TERM(mp) = copy_struct(ERL_MESSAGE_TERM(mp), + msg_sz, &hp, + &hfrag->off_heap); + if (is_not_immed(ERL_MESSAGE_TOKEN(mp))) + ERL_MESSAGE_TOKEN(mp) = copy_struct(ERL_MESSAGE_TOKEN(mp), + token_sz, &hp, + &hfrag->off_heap); +#ifdef USE_VM_PROBES + if (is_not_immed(ERL_MESSAGE_DT_UTAG(mp))) + ERL_MESSAGE_DT_UTAG(mp) = copy_struct(ERL_MESSAGE_DT_UTAG(mp), + utag_sz, &hp, + &hfrag->off_heap); +#endif + mp->data.heap_frag = hfrag; + reds += 1; + } + + return reds; +} + +Sint +erts_complete_off_heap_message_queue_change(Process *c_p) +{ + int reds = 1; + + ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(c_p)); + ASSERT(c_p->flags & F_OFF_HEAP_MSGQ_CHNG); + ASSERT(erts_smp_atomic32_read_nob(&c_p->state) & ERTS_PSFLG_OFF_HEAP_MSGQ); + + /* + * This job was first initiated when the process changed to off heap + * message queue management. Since then ERTS_PSFLG_OFF_HEAP_MSGQ + * has been set. However, the management state might have been changed + * again (multiple times) since then. Check users last requested state + * (the flags F_OFF_HEAP_MSGQ, and F_ON_HEAP_MSGQ), and make the state + * consistent with that. + */ + + if (!(c_p->flags & F_OFF_HEAP_MSGQ)) + erts_smp_atomic32_read_band_nob(&c_p->state, + ~ERTS_PSFLG_OFF_HEAP_MSGQ); + else { + reds += 2; + erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ); + ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p); + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ); + reds += erts_move_messages_off_heap(c_p); } + c_p->flags &= ~F_OFF_HEAP_MSGQ_CHNG; + return reds; +} + +typedef struct { + Eterm pid; + ErtsThrPrgrLaterOp lop; +} ErtsChangeOffHeapMessageQueue; + +static void +change_off_heap_msgq(void *vcohmq) +{ + ErtsChangeOffHeapMessageQueue *cohmq; + /* + * Now we've waited thread progress which ensures that all + * messages to the process are enqueued off heap. Schedule + * completion of this change as a system task on the process + * itself. This in order to avoid lock contention on its + * main lock. We will be called in + * erts_complete_off_heap_message_queue_change() (above) when + * the system task has been selected for execution. + */ + cohmq = (ErtsChangeOffHeapMessageQueue *) vcohmq; + erts_schedule_complete_off_heap_message_queue_change(cohmq->pid); + erts_free(ERTS_ALC_T_MSGQ_CHNG, vcohmq); +} + +Eterm +erts_change_message_queue_management(Process *c_p, Eterm new_state) +{ + Eterm res; + +#ifdef DEBUG + if (c_p->flags & F_OFF_HEAP_MSGQ) { + ASSERT(erts_smp_atomic32_read_nob(&c_p->state) + & ERTS_PSFLG_OFF_HEAP_MSGQ); + } + else { + if (c_p->flags & F_OFF_HEAP_MSGQ_CHNG) { + ASSERT(erts_smp_atomic32_read_nob(&c_p->state) + & ERTS_PSFLG_OFF_HEAP_MSGQ); + } + else { + ASSERT(!(erts_smp_atomic32_read_nob(&c_p->state) + & ERTS_PSFLG_OFF_HEAP_MSGQ)); + } + } +#endif + + switch (c_p->flags & (F_OFF_HEAP_MSGQ|F_ON_HEAP_MSGQ)) { + + case F_OFF_HEAP_MSGQ: + res = am_off_heap; + + switch (new_state) { + case am_off_heap: + break; + case am_on_heap: + c_p->flags |= F_ON_HEAP_MSGQ; + erts_smp_atomic32_read_bor_nob(&c_p->state, + ERTS_PSFLG_ON_HEAP_MSGQ); + /* fall through */ + case am_mixed: + c_p->flags &= ~F_OFF_HEAP_MSGQ; + /* + * We are not allowed to clear ERTS_PSFLG_OFF_HEAP_MSGQ + * if a off heap change is ongoing. It will be adjusted + * when the change completes... + */ + if (!(c_p->flags & F_OFF_HEAP_MSGQ_CHNG)) { + /* Safe to clear ERTS_PSFLG_OFF_HEAP_MSGQ... */ + erts_smp_atomic32_read_band_nob(&c_p->state, + ~ERTS_PSFLG_OFF_HEAP_MSGQ); + } + break; + default: + res = THE_NON_VALUE; /* badarg */ + break; + } + break; + + case F_ON_HEAP_MSGQ: + res = am_on_heap; + + switch (new_state) { + case am_on_heap: + break; + case am_mixed: + c_p->flags &= ~F_ON_HEAP_MSGQ; + erts_smp_atomic32_read_band_nob(&c_p->state, + ~ERTS_PSFLG_ON_HEAP_MSGQ); + break; + case am_off_heap: + c_p->flags &= ~F_ON_HEAP_MSGQ; + erts_smp_atomic32_read_band_nob(&c_p->state, + ~ERTS_PSFLG_ON_HEAP_MSGQ); + goto change_to_off_heap; + default: + res = THE_NON_VALUE; /* badarg */ + break; + } + break; + + case 0: + res = am_mixed; + + switch (new_state) { + case am_mixed: + break; + case am_on_heap: + c_p->flags |= F_ON_HEAP_MSGQ; + erts_smp_atomic32_read_bor_nob(&c_p->state, + ERTS_PSFLG_ON_HEAP_MSGQ); + break; + case am_off_heap: + goto change_to_off_heap; + default: + res = THE_NON_VALUE; /* badarg */ + break; + } + break; + + default: + res = am_error; + ERTS_INTERNAL_ERROR("Inconsistent message queue management state"); + break; + } + + return res; + +change_to_off_heap: + + c_p->flags |= F_OFF_HEAP_MSGQ; + + /* + * We do not have to schedule a change if + * we have an ongoing off heap change... + */ + if (!(c_p->flags & F_OFF_HEAP_MSGQ_CHNG)) { + ErtsChangeOffHeapMessageQueue *cohmq; + /* + * Need to set ERTS_PSFLG_OFF_HEAP_MSGQ and wait + * thread progress before completing the change in + * order to ensure that all senders observe that + * messages should be passed off heap. When the + * change has completed, GC does not need to inspect + * the message queue at all. + */ + erts_smp_atomic32_read_bor_nob(&c_p->state, + ERTS_PSFLG_OFF_HEAP_MSGQ); + c_p->flags |= F_OFF_HEAP_MSGQ_CHNG; + cohmq = erts_alloc(ERTS_ALC_T_MSGQ_CHNG, + sizeof(ErtsChangeOffHeapMessageQueue)); + cohmq->pid = c_p->common.id; + erts_schedule_thr_prgr_later_op(change_off_heap_msgq, + (void *) cohmq, + &cohmq->lop); + } + + return res; +} + +int +erts_decode_dist_message(Process *proc, ErtsProcLocks proc_locks, + ErtsMessage *msgp, int force_off_heap) +{ + ErtsHeapFactory factory; + Eterm msg; + ErlHeapFragment *bp; + Sint need; + int decode_in_heap_frag; + + decode_in_heap_frag = (force_off_heap + || !(proc_locks & ERTS_PROC_LOCK_MAIN) + || (proc->flags & F_OFF_HEAP_MSGQ)); + + if (msgp->data.dist_ext->heap_size >= 0) + need = msgp->data.dist_ext->heap_size; + else { + need = erts_decode_dist_ext_size(msgp->data.dist_ext); + if (need < 0) { + /* bad msg; remove it... */ + if (is_not_immed(ERL_MESSAGE_TOKEN(msgp))) { + bp = erts_dist_ext_trailer(msgp->data.dist_ext); + erts_cleanup_offheap(&bp->off_heap); + } + erts_free_dist_ext_copy(msgp->data.dist_ext); + msgp->data.dist_ext = NULL; + return 0; + } + + msgp->data.dist_ext->heap_size = need; + } + + if (is_not_immed(ERL_MESSAGE_TOKEN(msgp))) { + bp = erts_dist_ext_trailer(msgp->data.dist_ext); + need += bp->used_size; + } + + if (decode_in_heap_frag) + erts_factory_heap_frag_init(&factory, new_message_buffer(need)); + else + erts_factory_proc_prealloc_init(&factory, proc, need); + + ASSERT(msgp->data.dist_ext->heap_size >= 0); + if (is_not_immed(ERL_MESSAGE_TOKEN(msgp))) { + ErlHeapFragment *heap_frag; + heap_frag = erts_dist_ext_trailer(msgp->data.dist_ext); + ERL_MESSAGE_TOKEN(msgp) = copy_struct(ERL_MESSAGE_TOKEN(msgp), + heap_frag->used_size, + &factory.hp, + factory.off_heap); + erts_cleanup_offheap(&heap_frag->off_heap); + } + + msg = erts_decode_dist_ext(&factory, msgp->data.dist_ext); + ERL_MESSAGE_TERM(msgp) = msg; + erts_free_dist_ext_copy(msgp->data.dist_ext); + msgp->data.attached = NULL; + + if (is_non_value(msg)) { + erts_factory_undo(&factory); + return 0; + } + + erts_factory_trim_and_close(&factory, msgp->m, + ERL_MESSAGE_REF_ARRAY_SZ); + + ASSERT(!msgp->data.heap_frag); + + if (decode_in_heap_frag) + msgp->data.heap_frag = factory.heap_frags; + + return 1; +} + +/* + * ERTS_INSPECT_MSGQ_KEEP_OH_MSGS == 0 will move off heap messages + * into the heap of the inspected process if off_heap_message_queue + * is false when process_info(_, messages) is called. That is, the + * following GC will have more data in the rootset compared to the + * scenario when process_info(_, messages) had not been called. + * + * ERTS_INSPECT_MSGQ_KEEP_OH_MSGS != 0 will keep off heap messages + * off heap when process_info(_, messages) is called regardless of + * the off_heap_message_queue setting of the process. That is, it + * will change the following execution of the process as little as + * possible. + */ +#define ERTS_INSPECT_MSGQ_KEEP_OH_MSGS 1 + +Uint +erts_prep_msgq_for_inspection(Process *c_p, Process *rp, + ErtsProcLocks rp_locks, ErtsMessageInfo *mip) +{ + Uint tot_heap_size; + ErtsMessage* mp; + Sint i; + int self_on_heap; + + /* + * Prepare the message queue for inspection + * by process_info(). + * + * + * - Decode all messages on external format + * - Remove all corrupt dist messages from queue + * - Save pointer to, and heap size need of each + * message in the mip array. + * - Return total heap size need for all messages + * that needs to be copied. + * + * If ERTS_INSPECT_MSGQ_KEEP_OH_MSGS == 0: + * - In case off heap messages is disabled and + * we are inspecting our own queue, move all + * off heap data into the heap. + */ + + self_on_heap = c_p == rp && !(c_p->flags & F_OFF_HEAP_MSGQ); + + tot_heap_size = 0; + i = 0; + mp = rp->msg.first; + while (mp) { + Eterm msg = ERL_MESSAGE_TERM(mp); + + mip[i].size = 0; + + if (is_non_value(msg)) { + /* Dist message on external format; decode it... */ + if (mp->data.attached) + erts_decode_dist_message(rp, rp_locks, mp, + ERTS_INSPECT_MSGQ_KEEP_OH_MSGS); + + msg = ERL_MESSAGE_TERM(mp); + + if (is_non_value(msg)) { + ErtsMessage **mpp; + ErtsMessage *bad_mp = mp; + /* + * Bad distribution message; remove + * it from the queue... + */ + ASSERT(!mp->data.attached); + + mpp = i == 0 ? &rp->msg.first : &mip[i-1].msgp->next; + + if (rp->msg.save == &bad_mp->next) + rp->msg.save = mpp; + if (rp->msg.last == &bad_mp->next) + rp->msg.last = mpp; + mp = mp->next; + *mpp = mp; + rp->msg.len--; + bad_mp->next = NULL; + erts_cleanup_messages(bad_mp); + continue; + } + } + + ASSERT(is_value(msg)); + +#if ERTS_INSPECT_MSGQ_KEEP_OH_MSGS + if (is_not_immed(msg) && (!self_on_heap || mp->data.attached)) { + Uint sz = size_object(msg); + mip[i].size = sz; + tot_heap_size += sz; + } +#else + if (self_on_heap) { + if (mp->data.attached) { + ErtsMessage *tmp = NULL; + if (mp->data.attached != ERTS_MSG_COMBINED_HFRAG) { + erts_link_mbuf_to_proc(rp, mp->data.heap_frag); + mp->data.attached = NULL; + } + else { + /* + * Need to replace the message reference since + * we will get references to the message data + * from the heap... + */ + ErtsMessage **mpp; + tmp = erts_alloc_message(0, NULL); + sys_memcpy((void *) tmp->m, (void *) mp->m, + sizeof(Eterm)*ERL_MESSAGE_REF_ARRAY_SZ); + mpp = i == 0 ? &rp->msg.first : &mip[i-1].msgp->next; + tmp->next = mp->next; + if (rp->msg.save == &mp->next) + rp->msg.save = &tmp->next; + if (rp->msg.last == &mp->next) + rp->msg.last = &tmp->next; + *mpp = tmp; + erts_save_message_in_proc(rp, mp); + mp = tmp; + } + } + } + else if (is_not_immed(msg)) { + Uint sz = size_object(msg); + mip[i].size = sz; + tot_heap_size += sz; + } + +#endif + + mip[i].msgp = mp; + i++; + mp = mp->next; + } + + return tot_heap_size; } void erts_factory_proc_init(ErtsHeapFactory* factory, @@ -1127,47 +1439,138 @@ void erts_factory_proc_prealloc_init(ErtsHeapFactory* factory, Process* p, Sint size) { + ErlHeapFragment *bp = p->mbuf; factory->mode = FACTORY_HALLOC; factory->p = p; factory->hp_start = HAlloc(p, size); factory->hp = factory->hp_start; factory->hp_end = factory->hp_start + size; factory->off_heap = &p->off_heap; + factory->message = NULL; factory->off_heap_saved.first = p->off_heap.first; factory->off_heap_saved.overhead = p->off_heap.overhead; - factory->heap_frags_saved = p->mbuf; + factory->heap_frags_saved = bp; + factory->heap_frags_saved_used = bp ? bp->used_size : 0; factory->heap_frags = NULL; /* not used */ factory->alloc_type = 0; /* not used */ } -void erts_factory_message_init(ErtsHeapFactory* factory, - Process* rp, - Eterm* hp, - ErlHeapFragment* bp) +void erts_factory_heap_frag_init(ErtsHeapFactory* factory, + ErlHeapFragment* bp) { - if (bp) { - factory->mode = FACTORY_HEAP_FRAGS; - factory->p = NULL; - factory->hp_start = bp->mem; - factory->hp = hp ? hp : bp->mem; - factory->hp_end = bp->mem + bp->alloc_size; - factory->off_heap = &bp->off_heap; - factory->heap_frags = bp; - factory->heap_frags_saved = bp; - factory->alloc_type = ERTS_ALC_T_HEAP_FRAG; - ASSERT(!bp->next); + factory->mode = FACTORY_HEAP_FRAGS; + factory->p = NULL; + factory->hp_start = bp->mem; + factory->hp = bp->mem; + factory->hp_end = bp->mem + bp->alloc_size; + factory->off_heap = &bp->off_heap; + factory->message = NULL; + factory->heap_frags = bp; + factory->heap_frags_saved = NULL; + factory->heap_frags_saved_used = 0; + factory->alloc_type = ERTS_ALC_T_HEAP_FRAG; + ASSERT(!bp->next); + factory->off_heap_saved.first = factory->off_heap->first; + factory->off_heap_saved.overhead = factory->off_heap->overhead; + + ASSERT(factory->hp >= factory->hp_start && factory->hp <= factory->hp_end); +} + + +ErtsMessage * +erts_factory_message_create(ErtsHeapFactory* factory, + Process *proc, + ErtsProcLocks *proc_locksp, + Uint sz) +{ + Eterm *hp; + ErlOffHeap *ohp; + ErtsMessage *msgp; + int on_heap; + erts_aint32_t state; + + state = erts_smp_atomic32_read_nob(&proc->state); + + if (state & ERTS_PSFLG_OFF_HEAP_MSGQ) { + msgp = erts_alloc_message(sz, &hp); + ohp = sz == 0 ? NULL : &msgp->hfrag.off_heap; + on_heap = 0; } else { - factory->mode = FACTORY_HALLOC; - factory->p = rp; - factory->hp_start = hp; - factory->hp = hp; - factory->hp_end = HEAP_TOP(rp); - factory->off_heap = &rp->off_heap; - factory->heap_frags_saved = rp->mbuf; - factory->heap_frags = NULL; /* not used */ - factory->alloc_type = 0; /* not used */ + msgp = erts_try_alloc_message_on_heap(proc, &state, + proc_locksp, + sz, &hp, &ohp, + &on_heap); + } + + if (on_heap) { + ERTS_SMP_ASSERT(*proc_locksp & ERTS_PROC_LOCK_MAIN); + ASSERT(ohp == &proc->off_heap); + factory->mode = FACTORY_HALLOC; + factory->p = proc; + factory->heap_frags_saved = proc->mbuf; + factory->heap_frags_saved_used = proc->mbuf ? proc->mbuf->used_size : 0; + } + else { + factory->mode = FACTORY_MESSAGE; + factory->p = NULL; + factory->heap_frags_saved = NULL; + factory->heap_frags_saved_used = 0; + + if (msgp->data.attached == ERTS_MSG_COMBINED_HFRAG) { + ASSERT(!msgp->hfrag.next); + factory->heap_frags = NULL; + } + else { + ASSERT(!msgp->data.heap_frag + || !msgp->data.heap_frag->next); + factory->heap_frags = msgp->data.heap_frag; + } + } + factory->hp_start = hp; + factory->hp = hp; + factory->hp_end = hp + sz; + factory->message = msgp; + factory->off_heap = ohp; + factory->alloc_type = ERTS_ALC_T_HEAP_FRAG; + if (ohp) { + factory->off_heap_saved.first = ohp->first; + factory->off_heap_saved.overhead = ohp->overhead; } + else { + factory->off_heap_saved.first = NULL; + factory->off_heap_saved.overhead = 0; + } + + ASSERT(factory->hp >= factory->hp_start && factory->hp <= factory->hp_end); + + return msgp; +} + +void erts_factory_selfcontained_message_init(ErtsHeapFactory* factory, + ErtsMessage *msgp, + Eterm *hp) +{ + ErlHeapFragment* bp; + if (msgp->data.attached == ERTS_MSG_COMBINED_HFRAG) { + bp = &msgp->hfrag; + factory->heap_frags = NULL; + } + else { + bp = msgp->data.heap_frag; + factory->heap_frags = bp; + } + factory->mode = FACTORY_MESSAGE; + factory->p = NULL; + factory->hp_start = bp->mem; + factory->hp = hp; + factory->hp_end = bp->mem + bp->alloc_size; + factory->message = msgp; + factory->off_heap = &bp->off_heap; + factory->heap_frags_saved = NULL; + factory->heap_frags_saved_used = 0; + factory->alloc_type = ERTS_ALC_T_HEAP_FRAG; + ASSERT(!bp->next); factory->off_heap_saved.first = factory->off_heap->first; factory->off_heap_saved.overhead = factory->off_heap->overhead; @@ -1250,9 +1653,17 @@ static void reserve_heap(ErtsHeapFactory* factory, Uint need, Uint xtra) factory->hp_end = factory->hp + need; return; - case FACTORY_HEAP_FRAGS: - case FACTORY_TMP: - bp = factory->heap_frags; + case FACTORY_MESSAGE: + if (!factory->heap_frags) { + ASSERT(factory->message->data.attached == ERTS_MSG_COMBINED_HFRAG); + bp = &factory->message->hfrag; + } + else { + /* Fall through */ + case FACTORY_HEAP_FRAGS: + case FACTORY_TMP: + bp = factory->heap_frags; + } if (bp) { ASSERT(factory->hp > bp->mem); @@ -1290,8 +1701,23 @@ void erts_factory_close(ErtsHeapFactory* factory) HRelease(factory->p, factory->hp_end, factory->hp); break; - case FACTORY_HEAP_FRAGS: - bp = factory->heap_frags; + case FACTORY_MESSAGE: + if (!factory->heap_frags) { + if (factory->message->data.attached == ERTS_MSG_COMBINED_HFRAG) + bp = &factory->message->hfrag; + else + bp = NULL; + } + else { + if (factory->message->data.attached == ERTS_MSG_COMBINED_HFRAG) + factory->message->hfrag.next = factory->heap_frags; + else + factory->message->data.heap_frag = factory->heap_frags; + + /* Fall through */ + case FACTORY_HEAP_FRAGS: + bp = factory->heap_frags; + } if (bp) { ASSERT(factory->hp >= bp->mem); @@ -1315,17 +1741,47 @@ void erts_factory_close(ErtsHeapFactory* factory) void erts_factory_trim_and_close(ErtsHeapFactory* factory, Eterm *brefs, Uint brefs_size) { - if (factory->mode == FACTORY_HEAP_FRAGS) { - ErlHeapFragment* bp = factory->heap_frags; + ErlHeapFragment *bp; + + switch (factory->mode) { + case FACTORY_MESSAGE: { + ErtsMessage *mp = factory->message; + if (mp->data.attached == ERTS_MSG_COMBINED_HFRAG) { + if (!mp->hfrag.next) { + Uint sz = factory->hp - factory->hp_start; + mp = erts_shrink_message(mp, sz, brefs, brefs_size); + factory->message = mp; + factory->mode = FACTORY_CLOSED; + return; + } + /*else we don't trim multi fragmented messages for now (off_heap...) */ + break; + } + /* Fall through... */ + } + case FACTORY_HEAP_FRAGS: + bp = factory->heap_frags; + if (!bp) + break; if (bp->next == NULL) { Uint used_sz = factory->hp - bp->mem; ASSERT(used_sz <= bp->alloc_size); - factory->heap_frags = erts_resize_message_buffer(bp, used_sz, - brefs, brefs_size); + if (used_sz > 0) + bp = erts_resize_message_buffer(bp, used_sz, + brefs, brefs_size); + else { + free_message_buffer(bp); + bp = NULL; + } + factory->heap_frags = bp; + if (factory->mode == FACTORY_MESSAGE) + factory->message->data.heap_frag = bp; factory->mode = FACTORY_CLOSED; return; } - /*else we don't trim multi fragmented messages for now */ + /*else we don't trim multi fragmented messages for now (off_heap...) */ + default: + break; } erts_factory_close(factory); } @@ -1373,28 +1829,33 @@ void erts_factory_undo(ErtsHeapFactory* factory) /* Rollback heap top */ - if (factory->heap_frags_saved == NULL) { /* No heap frags when we started */ - ASSERT(factory->hp_start >= HEAP_START(factory->p)); - ASSERT(factory->hp_start <= HEAP_LIMIT(factory->p)); - HEAP_TOP(factory->p) = factory->hp_start; - } - else { + if (HEAP_START(factory->p) <= factory->hp_start + && factory->hp_start <= HEAP_LIMIT(factory->p)) { + HEAP_TOP(factory->p) = factory->hp_start; + } + + /* Fix last heap frag */ + if (factory->heap_frags_saved) { ASSERT(factory->heap_frags_saved == factory->p->mbuf); - if (factory->hp_start == factory->heap_frags_saved->mem) { + if (factory->hp_start != factory->heap_frags_saved->mem) + factory->heap_frags_saved->used_size = factory->heap_frags_saved_used; + else { factory->p->mbuf = factory->p->mbuf->next; ERTS_HEAP_FREE(ERTS_ALC_T_HEAP_FRAG, factory->heap_frags_saved, ERTS_HEAP_FRAG_SIZE(factory->heap_frags_saved->alloc_size)); } - else if (factory->hp_start != factory->hp_end) { - unsigned remains = factory->hp_start - factory->heap_frags_saved->mem; - ASSERT(remains > 0 && remains < factory->heap_frags_saved->used_size); - factory->heap_frags_saved->used_size = remains; - } } } break; + case FACTORY_MESSAGE: + if (factory->message->data.attached == ERTS_MSG_COMBINED_HFRAG) + factory->message->hfrag.next = factory->heap_frags; + else + factory->message->data.heap_frag = factory->heap_frags; + erts_cleanup_messages(factory->message); + break; case FACTORY_TMP: case FACTORY_HEAP_FRAGS: erts_cleanup_offheap(factory->off_heap); @@ -1422,4 +1883,3 @@ void erts_factory_undo(ErtsHeapFactory* factory) factory->heap_frags = NULL; #endif } - diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index 92ba3e571c..60035d15ae 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -24,6 +24,8 @@ struct proc_bin; struct external_thing_; +typedef struct erl_mesg ErtsMessage; + /* * This struct represents data that must be updated by structure copy, * but is stored outside of any heap. @@ -32,9 +34,6 @@ struct external_thing_; struct erl_off_heap_header { Eterm thing_word; Uint size; -#if HALFWORD_HEAP - void* dummy_ptr_padding__; -#endif struct erl_off_heap_header* next; }; @@ -57,6 +56,7 @@ typedef struct { enum { FACTORY_CLOSED = 0, FACTORY_HALLOC, + FACTORY_MESSAGE, FACTORY_HEAP_FRAGS, FACTORY_STATIC, FACTORY_TMP @@ -65,8 +65,10 @@ typedef struct { Eterm* hp_start; Eterm* hp; Eterm* hp_end; + ErtsMessage *message; struct erl_heap_fragment* heap_frags; struct erl_heap_fragment* heap_frags_saved; + Uint heap_frags_saved_used; ErlOffHeap* off_heap; ErlOffHeap off_heap_saved; Uint32 alloc_type; @@ -74,7 +76,10 @@ typedef struct { void erts_factory_proc_init(ErtsHeapFactory*, Process*); void erts_factory_proc_prealloc_init(ErtsHeapFactory*, Process*, Sint size); -void erts_factory_message_init(ErtsHeapFactory*, Process*, Eterm* hp, struct erl_heap_fragment*); +void erts_factory_heap_frag_init(ErtsHeapFactory*, struct erl_heap_fragment*); +ErtsMessage *erts_factory_message_create(ErtsHeapFactory *, Process *, + ErtsProcLocks *, Uint sz); +void erts_factory_selfcontained_message_init(ErtsHeapFactory*, ErtsMessage *, Eterm *); void erts_factory_static_init(ErtsHeapFactory*, Eterm* hp, Uint size, ErlOffHeap*); void erts_factory_tmp_init(ErtsHeapFactory*, Eterm* hp, Uint size, Uint32 atype); void erts_factory_dummy_init(ErtsHeapFactory*); @@ -96,6 +101,8 @@ void erts_factory_undo(ErtsHeapFactory*); #include "external.h" #include "erl_process.h" +#define ERTS_INVALID_HFRAG_PTR ((ErlHeapFragment *) ~((UWord) 7)) + /* * This struct represents a heap fragment, which is used when there * isn't sufficient room in the process heap and we can't do a GC. @@ -110,33 +117,53 @@ struct erl_heap_fragment { Eterm mem[1]; /* Data */ }; -typedef struct erl_mesg { - struct erl_mesg* next; /* Next message */ - union { - ErtsDistExternal *dist_ext; - ErlHeapFragment *heap_frag; - void *attached; - } data; +/* m[0] = message, m[1] = seq trace token */ +#define ERL_MESSAGE_REF_ARRAY_SZ 2 +#define ERL_MESSAGE_TERM(mp) ((mp)->m[0]) +#define ERL_MESSAGE_TOKEN(mp) ((mp)->m[1]) + #ifdef USE_VM_PROBES - Eterm m[3]; /* m[0] = message, m[1] = seq trace token, m[3] = dynamic trace user tag */ +/* m[2] = dynamic trace user tag */ +#undef ERL_MESSAGE_REF_ARRAY_SZ +#define ERL_MESSAGE_REF_ARRAY_SZ 3 +#define ERL_MESSAGE_DT_UTAG(mp) ((mp)->m[2]) #else - Eterm m[2]; /* m[0] = message, m[1] = seq trace token */ #endif -} ErlMessage; -#define ERL_MESSAGE_TERM(mp) ((mp)->m[0]) -#define ERL_MESSAGE_TOKEN(mp) ((mp)->m[1]) #ifdef USE_VM_PROBES -#define ERL_MESSAGE_DT_UTAG(mp) ((mp)->m[2]) +#define have_no_seqtrace(T) ((T) == NIL || (T) == am_have_dt_utag) +#else +#define have_no_seqtrace(T) ((T) == NIL) #endif +#define have_seqtrace(T) (!have_no_seqtrace(T)) + +#define ERL_MESSAGE_REF_FIELDS__ \ + ErtsMessage *next; /* Next message */ \ + union { \ + ErtsDistExternal *dist_ext; \ + ErlHeapFragment *heap_frag; \ + void *attached; \ + } data; \ + Eterm m[ERL_MESSAGE_REF_ARRAY_SZ] + + +typedef struct erl_msg_ref__ { + ERL_MESSAGE_REF_FIELDS__; +} ErtsMessageRef; + +struct erl_mesg { + ERL_MESSAGE_REF_FIELDS__; + + ErlHeapFragment hfrag; +}; /* Size of default message buffer (erl_message.c) */ #define ERL_MESSAGE_BUF_SZ 500 typedef struct { - ErlMessage* first; - ErlMessage** last; /* point to the last next pointer */ - ErlMessage** save; + ErtsMessage* first; + ErtsMessage** last; /* point to the last next pointer */ + ErtsMessage** save; Sint len; /* queue length */ /* @@ -144,14 +171,14 @@ typedef struct { * recv_set/1 instructions. */ BeamInstr* mark; /* address to rec_loop/2 instruction */ - ErlMessage** saved_last; /* saved last pointer */ + ErtsMessage** saved_last; /* saved last pointer */ } ErlMessageQueue; #ifdef ERTS_SMP typedef struct { - ErlMessage* first; - ErlMessage** last; /* point to the last next pointer */ + ErtsMessage* first; + ErtsMessage** last; /* point to the last next pointer */ Sint len; /* queue length */ } ErlMessageInQueue; @@ -202,7 +229,7 @@ do { \ /* Unlink current message */ #define UNLINK_MESSAGE(p,msgp) do { \ - ErlMessage* __mp = (msgp)->next; \ + ErtsMessage* __mp = (msgp)->next; \ *(p)->msg.save = __mp; \ (p)->msg.len--; \ if (__mp == NULL) \ @@ -218,76 +245,33 @@ do { \ #define SAVE_MESSAGE(p) \ (p)->msg.save = &(*(p)->msg.save)->next -/* - * ErtsMoveMsgAttachmentIntoProc() moves data attached to a message - * onto the heap of a process. The attached data is the content of - * the the message either on the internal format or on the external - * format, and also possibly a seq trace token on the internal format. - * If the message content is on the external format, the decode might - * fail. If the decoding fails, ERL_MESSAGE_TERM(M) will contain - * THE_NON_VALUE. That is, ERL_MESSAGE_TERM(M) *has* to be checked - * afterwards and taken care of appropriately. - * - * ErtsMoveMsgAttachmentIntoProc() will shallow copy to heap if - * possible; otherwise, move to heap via garbage collection. - * - * ErtsMoveMsgAttachmentIntoProc() is used when receiveing messages - * in process_main() and in hipe_check_get_msg(). - */ - -#define ErtsMoveMsgAttachmentIntoProc(M, P, ST, HT, FC, SWPO, SWPI) \ -do { \ - if ((M)->data.attached) { \ - Uint need__ = erts_msg_attached_data_size((M)); \ - { SWPO ; } \ - if ((ST) - (HT) >= need__) { \ - ErtsHeapFactory factory__; \ - erts_factory_proc_prealloc_init(&factory__, (P), need__); \ - erts_move_msg_attached_data_to_heap(&factory__, (M)); \ - erts_factory_close(&factory__); \ - if ((P)->mbuf != NULL) { \ - /* Heap was exhausted by messages. This is a rare case */ \ - /* that can currently (OTP 18) only happen if hamts are */ \ - /* far exceeding the estimated heap size. Do GC. */ \ - (FC) -= erts_garbage_collect((P), 0, NULL, 0); \ - } \ - } \ - else { \ - (FC) -= erts_garbage_collect((P), 0, NULL, 0); \ - } \ - { SWPI ; } \ - ASSERT(!(M)->data.attached); \ - } \ -} while (0) - #define ERTS_SND_FLG_NO_SEQ_TRACE (((unsigned) 1) << 0) #define ERTS_HEAP_FRAG_SIZE(DATA_WORDS) \ (sizeof(ErlHeapFragment) - sizeof(Eterm) + (DATA_WORDS)*sizeof(Eterm)) -#define ERTS_INIT_HEAP_FRAG(HEAP_FRAG_P, DATA_WORDS) \ -do { \ - (HEAP_FRAG_P)->next = NULL; \ - (HEAP_FRAG_P)->alloc_size = (DATA_WORDS); \ - (HEAP_FRAG_P)->used_size = (DATA_WORDS); \ - (HEAP_FRAG_P)->off_heap.first = NULL; \ - (HEAP_FRAG_P)->off_heap.overhead = 0; \ -} while (0) +#define ERTS_INIT_HEAP_FRAG(HEAP_FRAG_P, USED_WORDS, DATA_WORDS) \ + do { \ + (HEAP_FRAG_P)->next = NULL; \ + (HEAP_FRAG_P)->alloc_size = (DATA_WORDS); \ + (HEAP_FRAG_P)->used_size = (USED_WORDS); \ + (HEAP_FRAG_P)->off_heap.first = NULL; \ + (HEAP_FRAG_P)->off_heap.overhead = 0; \ + } while (0) void init_message(void); -void free_message(ErlMessage *); ErlHeapFragment* new_message_buffer(Uint); ErlHeapFragment* erts_resize_message_buffer(ErlHeapFragment *, Uint, Eterm *, Uint); void free_message_buffer(ErlHeapFragment *); void erts_queue_dist_message(Process*, ErtsProcLocks*, ErtsDistExternal *, Eterm); #ifdef USE_VM_PROBES -void erts_queue_message_probe(Process*, ErtsProcLocks*, ErlHeapFragment*, +void erts_queue_message_probe(Process*, ErtsProcLocks*, ErtsMessage*, Eterm message, Eterm seq_trace_token, Eterm dt_utag); #define erts_queue_message(RP,RL,BP,Msg,SEQ) \ erts_queue_message_probe((RP),(RL),(BP),(Msg),(SEQ),NIL) #else -void erts_queue_message(Process*, ErtsProcLocks*, ErlHeapFragment*, +void erts_queue_message(Process*, ErtsProcLocks*, ErtsMessage*, Eterm message, Eterm seq_trace_token); #define erts_queue_message_probe(RP,RL,BP,Msg,SEQ,TAG) \ erts_queue_message((RP),(RL),(BP),(Msg),(SEQ)) @@ -296,20 +280,141 @@ void erts_deliver_exit_message(Eterm, Process*, ErtsProcLocks *, Eterm, Eterm); Sint erts_send_message(Process*, Process*, ErtsProcLocks*, Eterm, unsigned); void erts_link_mbuf_to_proc(Process *proc, ErlHeapFragment *bp); -void erts_move_msg_mbuf_to_heap(Eterm**, ErlOffHeap*, ErlMessage *); - -Uint erts_msg_attached_data_size_aux(ErlMessage *msg); -void erts_move_msg_attached_data_to_heap(ErtsHeapFactory*, ErlMessage *); -Eterm erts_msg_distext2heap(Process *, ErtsProcLocks *, ErlHeapFragment **, - Eterm *, ErtsDistExternal *); +Uint erts_msg_attached_data_size_aux(ErtsMessage *msg); void erts_cleanup_offheap(ErlOffHeap *offheap); +void erts_save_message_in_proc(Process *p, ErtsMessage *msg); +Sint erts_move_messages_off_heap(Process *c_p); +Sint erts_complete_off_heap_message_queue_change(Process *c_p); +Eterm erts_change_message_queue_management(Process *c_p, Eterm new_state); + +int erts_decode_dist_message(Process *, ErtsProcLocks, ErtsMessage *, int); + +void erts_cleanup_messages(ErtsMessage *mp); + +typedef struct { + Uint size; + ErtsMessage *msgp; +} ErtsMessageInfo; + +Uint erts_prep_msgq_for_inspection(Process *c_p, + Process *rp, + ErtsProcLocks rp_locks, + ErtsMessageInfo *mip); + +void *erts_alloc_message_ref(void); +void erts_free_message_ref(void *); +#define ERTS_SMALL_FIX_MSG_SZ 10 +#define ERTS_MEDIUM_FIX_MSG_SZ 20 +#define ERTS_LARGE_FIX_MSG_SZ 30 +void *erts_alloc_small_message(void); +void erts_free_small_message(void *mp); + +typedef struct { + ErtsMessage m; + Eterm data[ERTS_SMALL_FIX_MSG_SZ-1]; +} ErtsSmallFixSzMessage; + +typedef struct { + ErtsMessage m; + Eterm data[ERTS_MEDIUM_FIX_MSG_SZ-1]; +} ErtsMediumFixSzMessage; + +typedef struct { + ErtsMessage m; + Eterm data[ERTS_LARGE_FIX_MSG_SZ-1]; +} ErtsLargeFixSzMessage; + +ErtsMessage *erts_try_alloc_message_on_heap(Process *pp, + erts_aint32_t *psp, + ErtsProcLocks *plp, + Uint sz, + Eterm **hpp, + ErlOffHeap **ohpp, + int *on_heap_p); +ErtsMessage *erts_realloc_shrink_message(ErtsMessage *mp, Uint sz, + Eterm *brefs, Uint brefs_size); + +ERTS_GLB_FORCE_INLINE ErtsMessage *erts_alloc_message(Uint sz, Eterm **hpp); +ERTS_GLB_FORCE_INLINE ErtsMessage *erts_shrink_message(ErtsMessage *mp, Uint sz, + Eterm *brefs, Uint brefs_size); +ERTS_GLB_FORCE_INLINE void erts_free_message(ErtsMessage *mp); ERTS_GLB_INLINE Uint erts_used_frag_sz(const ErlHeapFragment*); -ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErlMessage *msg); +ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErtsMessage *msg); + +#define ERTS_MSG_COMBINED_HFRAG ((void *) 0x1) #if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_FORCE_INLINE ErtsMessage *erts_alloc_message(Uint sz, Eterm **hpp) +{ + ErtsMessage *mp; + + if (sz == 0) { + mp = erts_alloc_message_ref(); + mp->next = NULL; + ERL_MESSAGE_TERM(mp) = NIL; + mp->data.attached = NULL; + if (hpp) + *hpp = NULL; + return mp; + } + + mp = erts_alloc(ERTS_ALC_T_MSG, + sizeof(ErtsMessage) + (sz - 1)*sizeof(Eterm)); + + mp->next = NULL; + ERL_MESSAGE_TERM(mp) = NIL; + mp->data.attached = ERTS_MSG_COMBINED_HFRAG; + ERTS_INIT_HEAP_FRAG(&mp->hfrag, sz, sz); + + if (hpp) + *hpp = &mp->hfrag.mem[0]; + + return mp; +} + +ERTS_GLB_FORCE_INLINE ErtsMessage * +erts_shrink_message(ErtsMessage *mp, Uint sz, Eterm *brefs, Uint brefs_size) +{ + if (sz == 0) { + ErtsMessage *nmp; + if (!mp->data.attached) + return mp; + ASSERT(mp->data.attached == ERTS_MSG_COMBINED_HFRAG); + nmp = erts_alloc_message_ref(); +#ifdef DEBUG + if (brefs && brefs_size) { + int i; + for (i = 0; i < brefs_size; i++) + ASSERT(is_non_value(brefs[i]) || is_immed(brefs[i])); + } +#endif + erts_free(ERTS_ALC_T_MSG, mp); + return nmp; + } + + ASSERT(mp->data.attached == ERTS_MSG_COMBINED_HFRAG); + ASSERT(mp->hfrag.used_size >= sz); + + if (sz >= (mp->hfrag.alloc_size - mp->hfrag.alloc_size / 16)) { + mp->hfrag.used_size = sz; + return mp; + } + + return erts_realloc_shrink_message(mp, sz, brefs, brefs_size); +} + +ERTS_GLB_FORCE_INLINE void erts_free_message(ErtsMessage *mp) +{ + if (mp->data.attached != ERTS_MSG_COMBINED_HFRAG) + erts_free_message_ref(mp); + else + erts_free(ERTS_ALC_T_MSG, mp); +} + ERTS_GLB_INLINE Uint erts_used_frag_sz(const ErlHeapFragment* bp) { Uint sz = 0; @@ -319,11 +424,17 @@ ERTS_GLB_INLINE Uint erts_used_frag_sz(const ErlHeapFragment* bp) return sz; } -ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErlMessage *msg) +ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErtsMessage *msg) { ASSERT(msg->data.attached); - if (is_value(ERL_MESSAGE_TERM(msg))) - return erts_used_frag_sz(msg->data.heap_frag); + if (is_value(ERL_MESSAGE_TERM(msg))) { + ErlHeapFragment *bp; + if (msg->data.attached == ERTS_MSG_COMBINED_HFRAG) + bp = &msg->hfrag; + else + bp = msg->data.heap_frag; + return erts_used_frag_sz(bp); + } else if (msg->data.dist_ext->heap_size < 0) return erts_msg_attached_data_size_aux(msg); else { diff --git a/erts/emulator/beam/erl_msacc.c b/erts/emulator/beam/erl_msacc.c new file mode 100644 index 0000000000..71e3fd8b6e --- /dev/null +++ b/erts/emulator/beam/erl_msacc.c @@ -0,0 +1,486 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2014-2015. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Microstate accounting. + * + * We keep track of the different states that the + * Erlang VM threads are in, in order to provide + * performance/debugging statistics. There is a + * small overhead in enabling this, but in the big + * scheme of things it should be negligible. + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#define ERTS_MSACC_STATE_STRINGS 1 + +#include "sys.h" +#include "global.h" +#include "erl_threads.h" +#include "erl_bif_unique.h" +#include "erl_map.h" +#include "erl_msacc.h" + +#if ERTS_ENABLE_MSACC + +static Eterm erts_msacc_gather_stats(ErtsMsAcc *msacc, Eterm **hpp, Uint *szp); +static void erts_msacc_reset(ErtsMsAcc *msacc); +static ErtsMsAcc* get_msacc(void); + +#ifdef USE_THREADS +erts_tsd_key_t ERTS_WRITE_UNLIKELY(erts_msacc_key); +#else +ErtsMsAcc *ERTS_WRITE_UNLIKELY(erts_msacc) = NULL; +#endif +int ERTS_WRITE_UNLIKELY(erts_msacc_enabled); + +static Eterm *erts_msacc_state_atoms = NULL; +static erts_rwmtx_t msacc_mutex; +static ErtsMsAcc *msacc_managed = NULL; +#ifdef USE_THREADS +static ErtsMsAcc *msacc_unmanaged = NULL; +static Uint msacc_unmanaged_count = 0; +#endif + +/* we have to split initiation as atoms are not inited in early init */ +void erts_msacc_early_init(void) { +#ifndef ERTS_MSACC_ALWAYS_ON + erts_msacc_enabled = 0; +#endif + erts_rwmtx_init(&msacc_mutex,"msacc_list_mutex"); +#ifdef USE_THREADS + erts_tsd_key_create(&erts_msacc_key,"erts_msacc_key"); +#else + erts_msacc = NULL; +#endif +} + +void erts_msacc_init(void) { + int i; + erts_msacc_state_atoms = erts_alloc(ERTS_ALC_T_MSACC, + sizeof(Eterm)*ERTS_MSACC_STATE_COUNT); + for (i = 0; i < ERTS_MSACC_STATE_COUNT; i++) { + erts_msacc_state_atoms[i] = am_atom_put(erts_msacc_states[i], + strlen(erts_msacc_states[i])); + } +} + +void erts_msacc_init_thread(char *type, int id, int managed) { + ErtsMsAcc *msacc; + + msacc = erts_alloc(ERTS_ALC_T_MSACC, sizeof(ErtsMsAcc)); + + msacc->type = strdup(type); + msacc->id = make_small(id); + msacc->unmanaged = !managed; + msacc->tid = erts_thr_self(); + msacc->perf_counter = 0; + +#ifdef USE_THREADS + erts_rwmtx_rwlock(&msacc_mutex); + if (!managed) { + erts_mtx_init(&msacc->mtx,"msacc_unmanaged_mutex"); + msacc->next = msacc_unmanaged; + msacc_unmanaged = msacc; + msacc_unmanaged_count++; + ERTS_MSACC_TSD_SET(msacc); + } else { + msacc->next = msacc_managed; + msacc_managed = msacc; + } + erts_rwmtx_rwunlock(&msacc_mutex); +#else + msacc_managed = msacc; +#endif + + erts_msacc_reset(msacc); + +#ifdef ERTS_MSACC_ALWAYS_ON + ERTS_MSACC_TSD_SET(msacc); + msacc->perf_counter = erts_sys_perf_counter(); + msacc->state = ERTS_MSACC_STATE_OTHER; +#endif +} + +/* + * Creates a structure looking like this + * #{ type => scheduler, id => 1, counters => #{ State1 => Counter1 ... StateN => CounterN}} + */ +static +Eterm erts_msacc_gather_stats(ErtsMsAcc *msacc, Eterm **hpp, Uint *szp) { + int i; + Eterm *hp; + Eterm key, state_key, state_map; + Eterm res = THE_NON_VALUE; + flatmap_t *map; + + if (szp) { + *szp += MAP_HEADER_FLATMAP_SZ + 1 + 2*(3); + *szp += MAP_HEADER_FLATMAP_SZ + 1 + 2*(ERTS_MSACC_STATE_COUNT); + for (i = 0; i < ERTS_MSACC_STATE_COUNT; i++) { + (void)erts_bld_sint64(NULL,szp,(Sint64)msacc->perf_counters[i]); +#ifdef ERTS_MSACC_STATE_COUNTERS + (void)erts_bld_uint64(NULL,szp,msacc->state_counters[i]); + *szp += 3; /* tuple to put state+perf counter in */ +#endif + } + } + + if (hpp) { + Eterm counters[ERTS_MSACC_STATE_COUNT]; + hp = *hpp; + for (i = 0; i < ERTS_MSACC_STATE_COUNT; i++) { + Eterm counter = erts_bld_sint64(&hp,NULL,(Sint64)msacc->perf_counters[i]); +#ifdef ERTS_MSACC_STATE_COUNTERS + Eterm counter__ = erts_bld_uint64(&hp,NULL,msacc->state_counters[i]); + counters[i] = TUPLE2(hp,counter,counter__); + hp += 3; +#else + counters[i] = counter; +#endif + } + + key = TUPLE3(hp,am_counters,am_id,am_type); + hp += 4; + + state_key = make_tuple(hp); + hp[0] = make_arityval(ERTS_MSACC_STATE_COUNT); + + for (i = 0; i < ERTS_MSACC_STATE_COUNT; i++) + hp[1+i] = erts_msacc_state_atoms[i]; + hp += 1 + ERTS_MSACC_STATE_COUNT; + + map = (flatmap_t*)hp; + hp += MAP_HEADER_FLATMAP_SZ; + map->thing_word = MAP_HEADER_FLATMAP; + map->size = ERTS_MSACC_STATE_COUNT; + map->keys = state_key; + for (i = 0; i < ERTS_MSACC_STATE_COUNT; i++) + hp[i] = counters[i]; + hp += ERTS_MSACC_STATE_COUNT; + state_map = make_flatmap(map); + + map = (flatmap_t*)hp; + hp += MAP_HEADER_FLATMAP_SZ; + map->thing_word = MAP_HEADER_FLATMAP; + map->size = 3; + map->keys = key; + hp[0] = state_map; + hp[1] = msacc->id; + hp[2] = am_atom_put(msacc->type,strlen(msacc->type)); + hp += 3; + + *hpp = hp; + res = make_flatmap(map); + } + + return res; +} + +typedef struct { + int action; + Process *proc; + Eterm ref; + Eterm ref_heap[REF_THING_SIZE]; + Uint req_sched; + erts_smp_atomic32_t refc; +} ErtsMSAccReq; + +static ErtsMsAcc* get_msacc(void) { + ErtsMsAcc *msacc; + erts_rwmtx_rlock(&msacc_mutex); + msacc = msacc_managed; + while (!erts_equal_tids(msacc->tid,erts_thr_self())) { + msacc = msacc->next; + ASSERT(msacc != NULL); + } + erts_rwmtx_runlock(&msacc_mutex); + return msacc; +} + +static void send_reply(ErtsMsAcc *msacc, ErtsMSAccReq *msaccrp) { + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + Process *rp = msaccrp->proc; + ErtsMessage *msgp = NULL; + Eterm **hpp, *hp; + Eterm ref_copy = NIL, msg; + Uint sz, *szp; + ErlOffHeap *ohp = NULL; + ErtsProcLocks rp_locks = (esdp && msaccrp->req_sched == esdp->no + ? ERTS_PROC_LOCK_MAIN : 0); + + sz = 0; + hpp = NULL; + szp = &sz; + + if (msacc->unmanaged) erts_mtx_lock(&msacc->mtx); + + while (1) { + if (hpp) + ref_copy = STORE_NC(hpp, ohp, msaccrp->ref); + else + *szp += REF_THING_SIZE; + + if (msaccrp->action != ERTS_MSACC_GATHER) + msg = ref_copy; + else { + msg = erts_msacc_gather_stats(msacc, hpp, szp); + msg = erts_bld_tuple(hpp, szp, 2, ref_copy, msg); + } + if (hpp) + break; + + msgp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp); + hpp = &hp; + szp = NULL; + } + + if (msacc->unmanaged) erts_mtx_unlock(&msacc->mtx); + + erts_queue_message(rp, &rp_locks, msgp, msg, NIL); + + if (esdp && msaccrp->req_sched == esdp->no) + rp_locks &= ~ERTS_PROC_LOCK_MAIN; + + if (rp_locks) + erts_smp_proc_unlock(rp, rp_locks); + +} + +static void +reply_msacc(void *vmsaccrp) +{ + ErtsMsAcc *msacc = ERTS_MSACC_TSD_GET(); + ErtsMSAccReq *msaccrp = (ErtsMSAccReq *) vmsaccrp; + + ASSERT(!msacc || !msacc->unmanaged); + + if (msaccrp->action == ERTS_MSACC_ENABLE && !msacc) { + msacc = get_msacc(); + + msacc->perf_counter = erts_sys_perf_counter(); + + msacc->state = ERTS_MSACC_STATE_OTHER; + + ERTS_MSACC_TSD_SET(msacc); + + } else if (msaccrp->action == ERTS_MSACC_DISABLE && msacc) { + ERTS_MSACC_TSD_SET(NULL); + } else if (msaccrp->action == ERTS_MSACC_RESET) { + msacc = msacc ? msacc : get_msacc(); + erts_msacc_reset(msacc); + } else if (msaccrp->action == ERTS_MSACC_GATHER && !msacc) { + msacc = get_msacc(); + } + + ASSERT(!msacc || !msacc->unmanaged); + + send_reply(msacc, msaccrp); + + erts_proc_dec_refc(msaccrp->proc); + + if (erts_smp_atomic32_dec_read_nob(&msaccrp->refc) == 0) + erts_free(ERTS_ALC_T_MSACC, vmsaccrp); +} + +static void erts_msacc_reset(ErtsMsAcc *msacc) { + int i; + if (msacc->unmanaged) erts_mtx_lock(&msacc->mtx); + + for (i = 0; i < ERTS_MSACC_STATE_COUNT; i++) { + msacc->perf_counters[i] = 0; +#ifdef ERTS_MSACC_STATE_COUNTERS + msacc->state_counters[i] = 0; +#endif + } + + if (msacc->unmanaged) erts_mtx_unlock(&msacc->mtx); +} + +#endif /* ERTS_ENABLE_MSACC */ + + +/* + * This function is responsible for enabling, disabling, resetting and + * gathering data related to microstate accounting. + * + * Managed threads and unmanaged threads are handled differently. + * - managed threads get a misc_aux job telling them to switch on msacc + * - unmanaged have some fields protected by a mutex that has to be taken + * before any values can be updated + * + * For performance reasons there is also a global value erts_msacc_enabled + * that controls the state of all threads. Statistics gathering is only on + * if erts_msacc_enabled && msacc is true. + */ +Eterm +erts_msacc_request(Process *c_p, int action, Eterm *threads) +{ +#ifdef ERTS_ENABLE_MSACC + ErtsMsAcc *msacc = ERTS_MSACC_TSD_GET(); + ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(c_p); + Eterm ref; + ErtsMSAccReq *msaccrp; + Eterm *hp; + + +#ifdef ERTS_MSACC_ALWAYS_ON + if (action == ERTS_MSACC_ENABLE || action == ERTS_MSACC_DISABLE) + return THE_NON_VALUE; +#else + /* take care of double enable, and double disable here */ + if (msacc && action == ERTS_MSACC_ENABLE) { + return THE_NON_VALUE; + } else if (!msacc && action == ERTS_MSACC_DISABLE) { + return THE_NON_VALUE; + } +#endif + + ref = erts_make_ref(c_p); + + msaccrp = erts_alloc(ERTS_ALC_T_MSACC, sizeof(ErtsMSAccReq)); + hp = &msaccrp->ref_heap[0]; + + msaccrp->action = action; + msaccrp->proc = c_p; + msaccrp->ref = STORE_NC(&hp, NULL, ref); + msaccrp->req_sched = esdp->no; + +#ifdef ERTS_SMP + *threads = erts_no_schedulers; + *threads += 1; /* aux thread */ +#else + *threads = 1; +#endif + + erts_smp_atomic32_init_nob(&msaccrp->refc,(erts_aint32_t)*threads); + + erts_proc_add_refc(c_p, *threads); + + if (erts_no_schedulers > 1) + erts_schedule_multi_misc_aux_work(1, + erts_no_schedulers, + reply_msacc, + (void *) msaccrp); +#ifdef ERTS_SMP + /* aux thread */ + erts_schedule_misc_aux_work(0, reply_msacc, (void *) msaccrp); +#endif + +#ifdef USE_THREADS + /* Manage unmanaged threads */ + switch (action) { + case ERTS_MSACC_GATHER: { + Uint unmanaged_count; + ErtsMsAcc *msacc, **unmanaged; + int i = 0; + + /* we copy a list of pointers here so that we do not have to have + the msacc_mutex when sending messages */ + erts_rwmtx_rlock(&msacc_mutex); + unmanaged_count = msacc_unmanaged_count; + unmanaged = erts_alloc(ERTS_ALC_T_MSACC, + sizeof(ErtsMsAcc*)*unmanaged_count); + + for (i = 0, msacc = msacc_unmanaged; + i < unmanaged_count; + i++, msacc = msacc->next) { + unmanaged[i] = msacc; + } + erts_rwmtx_runlock(&msacc_mutex); + + for (i = 0; i < unmanaged_count; i++) { + erts_mtx_lock(&unmanaged[i]->mtx); + if (unmanaged[i]->perf_counter) { + ErtsSysPerfCounter perf_counter; + /* if enabled update stats */ + perf_counter = erts_sys_perf_counter(); + unmanaged[i]->perf_counters[unmanaged[i]->state] += + perf_counter - unmanaged[i]->perf_counter; + unmanaged[i]->perf_counter = perf_counter; + } + erts_mtx_unlock(&unmanaged[i]->mtx); + send_reply(unmanaged[i],msaccrp); + } + erts_free(ERTS_ALC_T_MSACC,unmanaged); + /* We have just sent unmanaged_count messages, so bump no of threads */ + *threads += unmanaged_count; + break; + } + case ERTS_MSACC_RESET: { + ErtsMsAcc *msacc; + erts_rwmtx_rlock(&msacc_mutex); + for (msacc = msacc_unmanaged; msacc != NULL; msacc = msacc->next) + erts_msacc_reset(msacc); + erts_rwmtx_runlock(&msacc_mutex); + break; + } + case ERTS_MSACC_ENABLE: { + erts_rwmtx_rlock(&msacc_mutex); + for (msacc = msacc_unmanaged; msacc != NULL; msacc = msacc->next) { + erts_mtx_lock(&msacc->mtx); + msacc->perf_counter = erts_sys_perf_counter(); + /* we assume the unmanaged thread is sleeping */ + msacc->state = ERTS_MSACC_STATE_SLEEP; + erts_mtx_unlock(&msacc->mtx); + } + erts_rwmtx_runlock(&msacc_mutex); + break; + } + case ERTS_MSACC_DISABLE: { + ErtsSysPerfCounter perf_counter; + erts_rwmtx_rlock(&msacc_mutex); + /* make sure to update stats with latest results */ + for (msacc = msacc_unmanaged; msacc != NULL; msacc = msacc->next) { + erts_mtx_lock(&msacc->mtx); + perf_counter = erts_sys_perf_counter(); + msacc->perf_counters[msacc->state] += perf_counter - msacc->perf_counter; + msacc->perf_counter = 0; + erts_mtx_unlock(&msacc->mtx); + } + erts_rwmtx_runlock(&msacc_mutex); + break; + } + default: { ASSERT(0); } + } + +#endif + + *threads = make_small(*threads); + + reply_msacc((void *) msaccrp); + +#ifndef ERTS_MSACC_ALWAYS_ON + /* enable/disable the global value */ + if (action == ERTS_MSACC_ENABLE) { + erts_msacc_enabled = 1; + } else if (action == ERTS_MSACC_DISABLE) { + erts_msacc_enabled = 0; + } +#endif + + return ref; +#else + return THE_NON_VALUE; +#endif +} diff --git a/erts/emulator/beam/erl_msacc.h b/erts/emulator/beam/erl_msacc.h new file mode 100644 index 0000000000..284388f7aa --- /dev/null +++ b/erts/emulator/beam/erl_msacc.h @@ -0,0 +1,409 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2014-2015. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifndef ERL_MSACC_H__ +#define ERL_MSACC_H__ + +/* Can be enabled/disabled via configure */ +#if ERTS_ENABLE_MSACC == 2 +#define ERTS_MSACC_EXTENDED_STATES 1 +#endif + +/* Uncomment this to also count the number of + transitions to a state. This will add a count + to the counter map. */ +/* #define ERTS_MSACC_STATE_COUNTERS 1 */ + +/* Uncomment this to make msacc to always be on, + this reduces overhead a little bit when profiling */ +/* #define ERTS_MSACC_ALWAYS_ON 1 */ + +#define ERTS_MSACC_DISABLE 0 +#define ERTS_MSACC_ENABLE 1 +#define ERTS_MSACC_RESET 2 +#define ERTS_MSACC_GATHER 3 + +/* + * When adding a new state, you have to: + * * Add it here + * * Increment ERTS_MSACC_STATE_COUNT + * * Add string value to erts_msacc_states + * * Have to be in alphabetical order! + * * Only add states to the non-extended section after + * careful benchmarking to make sure the overhead + * when disabled is minimal. + */ + +#ifndef ERTS_MSACC_EXTENDED_STATES +#define ERTS_MSACC_STATE_AUX 0 +#define ERTS_MSACC_STATE_CHECK_IO 1 +#define ERTS_MSACC_STATE_EMULATOR 2 +#define ERTS_MSACC_STATE_GC 3 +#define ERTS_MSACC_STATE_OTHER 4 +#define ERTS_MSACC_STATE_PORT 5 +#define ERTS_MSACC_STATE_SLEEP 6 + +#define ERTS_MSACC_STATE_COUNT 7 + +#if ERTS_MSACC_STATE_STRINGS && ERTS_ENABLE_MSACC +static char *erts_msacc_states[] = { + "aux", + "check_io", + "emulator", + "gc", + "other", + "port", + "sleep" +}; +#endif + +#else + +#define ERTS_MSACC_STATE_ALLOC 0 +#define ERTS_MSACC_STATE_AUX 1 +#define ERTS_MSACC_STATE_BIF 2 +#define ERTS_MSACC_STATE_BUSY_WAIT 3 +#define ERTS_MSACC_STATE_CHECK_IO 4 +#define ERTS_MSACC_STATE_EMULATOR 5 +#define ERTS_MSACC_STATE_ETS 6 +#define ERTS_MSACC_STATE_GC 7 +#define ERTS_MSACC_STATE_GC_FULL 8 +#define ERTS_MSACC_STATE_NIF 9 +#define ERTS_MSACC_STATE_OTHER 10 +#define ERTS_MSACC_STATE_PORT 11 +#define ERTS_MSACC_STATE_SEND 12 +#define ERTS_MSACC_STATE_SLEEP 13 +#define ERTS_MSACC_STATE_TIMERS 14 + +#define ERTS_MSACC_STATE_COUNT 15 + +#if ERTS_MSACC_STATE_STRINGS +static char *erts_msacc_states[] = { + "alloc", + "aux", + "bif", + "busy_wait", + "check_io", + "emulator", + "ets", + "gc", + "gc_full", + "nif", + "other", + "port", + "send", + "sleep", + "timers" +}; +#endif + +#endif + +typedef struct erl_msacc_t_ ErtsMsAcc; + +struct erl_msacc_t_ { + + /* the the values below are protected by mtx iff unmanaged = 1 */ + ErtsSysPerfCounter perf_counter; + ErtsSysPerfCounter perf_counters[ERTS_MSACC_STATE_COUNT]; +#ifdef ERTS_MSACC_STATE_COUNTERS + Uint64 state_counters[ERTS_MSACC_STATE_COUNT]; +#endif + Uint state; + + /* protected by msacc_mutex in erl_msacc.c, and should be constant */ + int unmanaged; + erts_mtx_t mtx; + ErtsMsAcc *next; + erts_tid_t tid; + Eterm id; + char *type; +}; + +#if ERTS_ENABLE_MSACC + +#define ERTS_MSACC_INLINE ERTS_GLB_INLINE + +#ifdef USE_THREADS +extern erts_tsd_key_t erts_msacc_key; +#else +extern ErtsMsAcc *erts_msacc; +#endif + +#ifdef ERTS_MSACC_ALWAYS_ON +#define erts_msacc_enabled 1 +#else +extern int erts_msacc_enabled; +#endif + +#ifdef USE_THREADS +#define ERTS_MSACC_TSD_GET() erts_tsd_get(erts_msacc_key) +#define ERTS_MSACC_TSD_SET(tsd) erts_tsd_set(erts_msacc_key,tsd) +#else +#define ERTS_MSACC_TSD_GET() erts_msacc +#define ERTS_MSACC_TSD_SET(tsd) erts_msacc = tsd +#endif + +void erts_msacc_early_init(void); +void erts_msacc_init(void); +void erts_msacc_init_thread(char *type, int id, int liberty); + +/* The defines below are used to instrument the vm code + * with different state changes. There are two variants + * of each define. One that has a cached ErtsMsAcc * + * that it can use, and one that does not. + * The cached values are necessary to have in order to get + * low enough overhead when running without msacc enabled. + * + * The two most common patterns to use the function with are: + * + * ERTS_MSACC_PUSH_AND_SET_STATE(ERTS_MSACC_STATE_NEWSTATE); + * ... call other function in new state ... + * ERTS_MSACC_POP_STATE(); + * + * Note that the erts_msacc_push* function declare new variables, so + * to conform with C89 we have to call it in the beginning of a function. + * We might not want to change state it the beginning though, so we use this: + * + * ERTS_MSACC_PUSH_STATE(); + * ... some other code ... + * ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_NEWSTATE); + * ... call other function in new state ... + * ERTS_MSACC_POP_STATE(); + * + * Notice that we used the cached version of set_state as push_state already + * read the erts_msacc_enabled to the cache. + * + * Most macros also have other variants with the suffix _m which means that + * they are known to only be called in managed threads, or with the _x suffix + * which means that it should only be used in an emulator compiled with + * extended states. + * + * Here is a listing of the entire api: + * + * void ERTS_MSACC_DECLARE_CACHE() + * void ERTS_MSACC_UPDATE_CACHE() + * void ERTS_MSACC_IS_ENABLED() + * void ERTS_MSACC_IS_ENABLED_CACHED() + * + * void ERTS_MSACC_PUSH_STATE() + * void ERTS_MSACC_SET_STATE(int state) + * void ERTS_MSACC_PUSH_AND_SET_STATE(int state) + * + * void ERTS_MSACC_PUSH_STATE_CACHED() + * void ERTS_MSACC_SET_STATE_CACHED(int state) + * void ERTS_MSACC_PUSH_AND_SET_STATE_CACHED(int state) + * void ERTS_MSACC_POP_STATE() + * + * void ERTS_MSACC_PUSH_STATE_M() + * void ERTS_MSACC_PUSH_STATE_CACHED_M() + * void ERTS_MSACC_SET_STATE_CACHED_M(int state) + * void ERTS_MSACC_SET_STATE_M(int state) + * void ERTS_MSACC_POP_STATE_M() + * void ERTS_MSACC_PUSH_AND_SET_STATE_M(int state) + * + * Most functions are also available with an _x suffix that are only enabled + * when using the extra states. If they are not, just add them to the end + * of this file. + */ + +/* cache handling functions */ +#define ERTS_MSACC_IS_ENABLED() ERTS_UNLIKELY(erts_msacc_enabled) +#define ERTS_MSACC_DECLARE_CACHE() \ + ErtsMsAcc *ERTS_MSACC_UPDATE_CACHE(); \ + ERTS_DECLARE_DUMMY(Uint __erts_msacc_state) = ERTS_MSACC_STATE_OTHER; +#define ERTS_MSACC_IS_ENABLED_CACHED() ERTS_UNLIKELY(__erts_msacc_cache != NULL) +#define ERTS_MSACC_UPDATE_CACHE() \ + __erts_msacc_cache = erts_msacc_enabled ? ERTS_MSACC_TSD_GET() : NULL + + +/* The defines below implicitly declare and load a new cache */ +#define ERTS_MSACC_PUSH_STATE() \ + ERTS_MSACC_DECLARE_CACHE(); \ + ERTS_MSACC_PUSH_STATE_CACHED() +#define ERTS_MSACC_SET_STATE(state) \ + ERTS_MSACC_DECLARE_CACHE(); \ + ERTS_MSACC_SET_STATE_CACHED(state) +#define ERTS_MSACC_PUSH_AND_SET_STATE(state) \ + ERTS_MSACC_PUSH_STATE(); ERTS_MSACC_SET_STATE_CACHED(state) + +/* The defines below need an already declared cache to work */ +#define ERTS_MSACC_PUSH_STATE_CACHED() \ + __erts_msacc_state = ERTS_MSACC_IS_ENABLED_CACHED() ? \ + erts_msacc_get_state_um__(__erts_msacc_cache) : ERTS_MSACC_STATE_OTHER +#define ERTS_MSACC_SET_STATE_CACHED(state) \ + if (ERTS_MSACC_IS_ENABLED_CACHED()) \ + erts_msacc_set_state_um__(__erts_msacc_cache, state, 1) +#define ERTS_MSACC_PUSH_AND_SET_STATE_CACHED(state) \ + ERTS_MSACC_PUSH_STATE_CACHED(); ERTS_MSACC_SET_STATE_CACHED(state) +#define ERTS_MSACC_POP_STATE() \ + if (ERTS_MSACC_IS_ENABLED_CACHED()) \ + erts_msacc_set_state_um__(__erts_msacc_cache, __erts_msacc_state, 0) + +/* Only use these defines when we know that we have in a managed thread */ +#define ERTS_MSACC_PUSH_STATE_M() \ + ERTS_MSACC_DECLARE_CACHE(); \ + ERTS_MSACC_PUSH_STATE_CACHED_M() +#define ERTS_MSACC_PUSH_STATE_CACHED_M() \ + __erts_msacc_state = ERTS_MSACC_IS_ENABLED_CACHED() ? \ + erts_msacc_get_state_m__(__erts_msacc_cache) : ERTS_MSACC_STATE_OTHER +#define ERTS_MSACC_SET_STATE_M(state) \ + ERTS_MSACC_DECLARE_CACHE(); \ + ERTS_MSACC_SET_STATE_CACHED_M(state) +#define ERTS_MSACC_SET_STATE_CACHED_M(state) \ + if (ERTS_MSACC_IS_ENABLED_CACHED()) \ + erts_msacc_set_state_m__(__erts_msacc_cache, state, 1) +#define ERTS_MSACC_POP_STATE_M() \ + if (ERTS_MSACC_IS_ENABLED_CACHED()) \ + erts_msacc_set_state_m__(__erts_msacc_cache, __erts_msacc_state, 0) +#define ERTS_MSACC_PUSH_AND_SET_STATE_M(state) \ + ERTS_MSACC_PUSH_STATE_M(); ERTS_MSACC_SET_STATE_CACHED_M(state) + +ERTS_MSACC_INLINE +void erts_msacc_set_state_um__(ErtsMsAcc *msacc,Uint state,int increment); +ERTS_MSACC_INLINE +void erts_msacc_set_state_m__(ErtsMsAcc *msacc,Uint state,int increment); + +ERTS_MSACC_INLINE +Uint erts_msacc_get_state_um__(ErtsMsAcc *msacc); +ERTS_MSACC_INLINE +Uint erts_msacc_get_state_m__(ErtsMsAcc *msacc); + + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_MSACC_INLINE +Uint erts_msacc_get_state_um__(ErtsMsAcc *msacc) { + Uint state; + if (msacc->unmanaged) + erts_mtx_lock(&msacc->mtx); + state = msacc->state; + if (msacc->unmanaged) + erts_mtx_unlock(&msacc->mtx); + return state; +} + +ERTS_MSACC_INLINE +Uint erts_msacc_get_state_m__(ErtsMsAcc *msacc) { + return msacc->state; +} + +ERTS_MSACC_INLINE +void erts_msacc_set_state_um__(ErtsMsAcc *msacc, Uint new_state, int increment) { + if (ERTS_UNLIKELY(msacc->unmanaged)) { + erts_mtx_lock(&msacc->mtx); + msacc->state = new_state; + if (ERTS_LIKELY(!msacc->perf_counter)) { + erts_mtx_unlock(&msacc->mtx); + return; + } + } + + erts_msacc_set_state_m__(msacc,new_state,increment); + + if (ERTS_UNLIKELY(msacc->unmanaged)) + erts_mtx_unlock(&msacc->mtx); +} + +ERTS_MSACC_INLINE +void erts_msacc_set_state_m__(ErtsMsAcc *msacc, Uint new_state, int increment) { + ErtsSysPerfCounter prev_perf_counter; + Sint64 diff; + + if (new_state == msacc->state) + return; + + prev_perf_counter = msacc->perf_counter; + msacc->perf_counter = erts_sys_perf_counter(); + diff = msacc->perf_counter - prev_perf_counter; + ASSERT(diff >= 0); + msacc->perf_counters[msacc->state] += diff; +#ifdef ERTS_MSACC_STATE_COUNTERS + msacc->state_counters[new_state] += increment; +#endif + msacc->state = new_state; +} + +#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ + +#else + +#define ERTS_MSACC_IS_ENABLED() 0 +#define erts_msacc_early_init() +#define erts_msacc_init() +#define erts_msacc_init_thread(type, id, liberty) +#define ERTS_MSACC_PUSH_STATE() +#define ERTS_MSACC_PUSH_STATE_CACHED() +#define ERTS_MSACC_POP_STATE() +#define ERTS_MSACC_SET_STATE(state) +#define ERTS_MSACC_SET_STATE_CACHED(state) +#define ERTS_MSACC_PUSH_AND_SET_STATE(state) +#define ERTS_MSACC_PUSH_AND_SET_STATE_CACHED(state) +#define ERTS_MSACC_UPDATE_CACHE() +#define ERTS_MSACC_IS_ENABLED_CACHED() +#define ERTS_MSACC_DECLARE_CACHE() +#define ERTS_MSACC_PUSH_STATE_M() +#define ERTS_MSACC_PUSH_STATE_CACHED_M() +#define ERTS_MSACC_SET_STATE_CACHED_M(state) +#define ERTS_MSACC_POP_STATE_M() +#define ERTS_MSACC_PUSH_AND_SET_STATE_M(state) + + +#endif /* ERTS_ENABLE_MSACC */ + +#ifndef ERTS_MSACC_EXTENDED_STATES + +#define ERTS_MSACC_PUSH_STATE_X() +#define ERTS_MSACC_POP_STATE_X() +#define ERTS_MSACC_SET_STATE_X(state) +#define ERTS_MSACC_SET_STATE_M_X(state) +#define ERTS_MSACC_SET_STATE_CACHED_X(state) +#define ERTS_MSACC_PUSH_AND_SET_STATE_X(state) +#define ERTS_MSACC_PUSH_AND_SET_STATE_CACHED_X(state) +#define ERTS_MSACC_UPDATE_CACHE_X() +#define ERTS_MSACC_IS_ENABLED_CACHED_X() 0 +#define ERTS_MSACC_DECLARE_CACHE_X() +#define ERTS_MSACC_PUSH_STATE_M_X() +#define ERTS_MSACC_PUSH_STATE_CACHED_M_X() +#define ERTS_MSACC_SET_STATE_CACHED_M_X(state) +#define ERTS_MSACC_POP_STATE_M_X() +#define ERTS_MSACC_PUSH_AND_SET_STATE_M_X(state) + +#else + +#define ERTS_MSACC_PUSH_STATE_X() ERTS_MSACC_PUSH_STATE() +#define ERTS_MSACC_POP_STATE_X() ERTS_MSACC_POP_STATE() +#define ERTS_MSACC_SET_STATE_X(state) ERTS_MSACC_SET_STATE(state) +#define ERTS_MSACC_SET_STATE_M_X(state) ERTS_MSACC_SET_STATE_M(state) +#define ERTS_MSACC_SET_STATE_CACHED_X(state) ERTS_MSACC_SET_STATE_CACHED(state) +#define ERTS_MSACC_PUSH_AND_SET_STATE_X(state) ERTS_MSACC_PUSH_AND_SET_STATE(state) +#define ERTS_MSACC_PUSH_AND_SET_STATE_CACHED_X(state) ERTS_MSACC_PUSH_AND_SET_STATE_CACHED(state) +#define ERTS_MSACC_UPDATE_CACHE_X() ERTS_MSACC_UPDATE_CACHE() +#define ERTS_MSACC_IS_ENABLED_CACHED_X() ERTS_MSACC_IS_ENABLED_CACHED() +#define ERTS_MSACC_DECLARE_CACHE_X() ERTS_MSACC_DECLARE_CACHE() +#define ERTS_MSACC_PUSH_STATE_M_X() ERTS_MSACC_PUSH_STATE_M() +#define ERTS_MSACC_PUSH_STATE_CACHED_M_X() ERTS_MSACC_PUSH_STATE_CACHED_M() +#define ERTS_MSACC_SET_STATE_CACHED_M_X(state) ERTS_MSACC_SET_STATE_CACHED_M(state) +#define ERTS_MSACC_POP_STATE_M_X() ERTS_MSACC_POP_STATE_M() +#define ERTS_MSACC_PUSH_AND_SET_STATE_M_X(state) ERTS_MSACC_PUSH_AND_SET_STATE_M(state) + +#endif /* !ERTS_MSACC_EXTENDED_STATES */ + +#endif /* ERL_MSACC_H__ */ diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 3141b05e2b..2204231748 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -314,6 +314,7 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, ErtsProcLocks rp_locks = 0; Process* rp; Process* c_p; + ErtsMessage *mp; ErlHeapFragment* frags; Eterm receiver = to_pid->pid; int flush_me = 0; @@ -347,7 +348,7 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, ASSERT(frags == MBUF(&menv->phony_proc)); if (frags != NULL) { /* Move all offheap's from phony proc to the first fragment. - Quick and dirty, but erts_move_msg_mbuf_to_heap doesn't care. */ + Quick and dirty... */ ASSERT(!is_offheap(&frags->off_heap)); frags->off_heap = MSO(&menv->phony_proc); clear_offheap(&MSO(&menv->phony_proc)); @@ -359,7 +360,9 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, if (flush_me) { flush_env(env); /* Needed for ERTS_HOLE_CHECK */ } - erts_queue_message(rp, &rp_locks, frags, msg, am_undefined); + mp = erts_alloc_message(0, NULL); + mp->data.heap_frag = frags; + erts_queue_message(rp, &rp_locks, mp, msg, am_undefined); if (c_p == rp) rp_locks &= ~ERTS_PROC_LOCK_MAIN; if (rp_locks) @@ -376,9 +379,19 @@ ERL_NIF_TERM enif_make_copy(ErlNifEnv* dst_env, ERL_NIF_TERM src_term) { Uint sz; Eterm* hp; +#ifdef SHCOPY + erts_shcopy_t info; + INITIALIZE_SHCOPY(info); + sz = copy_shared_calculate(src_term, &info); + hp = alloc_heap(dst_env, sz); + src_term = copy_shared_perform(src_term, sz, &info, &hp, &MSO(dst_env->proc)); + DESTROY_SHCOPY(info); + return src_term; +#else sz = size_object(src_term); hp = alloc_heap(dst_env, sz); return copy_struct(src_term, sz, &hp, &MSO(dst_env->proc)); +#endif } @@ -898,8 +911,13 @@ int enif_get_list_cell(ErlNifEnv* env, Eterm term, Eterm* head, Eterm* tail) int enif_get_list_length(ErlNifEnv* env, Eterm term, unsigned* len) { - if (is_not_list(term) && is_not_nil(term)) return 0; - *len = erts_list_length(term); + Sint i; + Uint u; + + if ((i = erts_list_length(term)) < 0) return 0; + u = (Uint)i; + if ((unsigned)u != u) return 0; + *len = u; return 1; } @@ -1792,12 +1810,10 @@ execute_dirty_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); ASSERT(ep); ep->fp = NULL; + erts_smp_atomic32_read_band_mb(&proc->state, ~(ERTS_PSFLG_DIRTY_CPU_PROC + | ERTS_PSFLG_DIRTY_IO_PROC)); result = (*fp)(env, argc, argv); - erts_smp_atomic32_read_band_mb(&proc->state, - ~(ERTS_PSFLG_DIRTY_CPU_PROC - |ERTS_PSFLG_DIRTY_IO_PROC - |ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q - |ERTS_PSFLG_DIRTY_IO_PROC_IN_Q)); + if (erts_refc_dectest(&env->mod_nif->rt_dtor_cnt, 0) == 0 && env->mod_nif->mod == NULL) close_lib(env->mod_nif); /* @@ -1845,13 +1861,6 @@ schedule_dirty_nif(ErlNifEnv* env, int flags, int argc, const ERL_NIF_TERM argv[ a = erts_smp_atomic32_read_acqb(&proc->state); while (1) { n = state = a; - /* - * clear any current dirty flags and dirty queue indicators, - * in case the application is shifting a job from one type - * of dirty scheduler to the other - */ - n &= ~(ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC - |ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q|ERTS_PSFLG_DIRTY_IO_PROC_IN_Q); if (flags == ERL_NIF_DIRTY_JOB_CPU_BOUND) n |= ERTS_PSFLG_DIRTY_CPU_PROC; else @@ -2251,17 +2260,17 @@ int enif_map_iterator_get_pair(ErlNifEnv *env, ***************************************************************************/ -static BeamInstr** get_func_pp(BeamInstr* mod_code, Eterm f_atom, unsigned arity) +static BeamInstr** get_func_pp(BeamCodeHeader* mod_code, Eterm f_atom, unsigned arity) { - int n = (int) mod_code[MI_NUM_FUNCTIONS]; + int n = (int) mod_code->num_functions; int j; for (j = 0; j < n; ++j) { - BeamInstr* code_ptr = (BeamInstr*) mod_code[MI_FUNCTIONS+j]; + BeamInstr* code_ptr = (BeamInstr*) mod_code->functions[j]; ASSERT(code_ptr[0] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); if (f_atom == ((Eterm) code_ptr[3]) && arity == ((unsigned) code_ptr[4])) { - return (BeamInstr**) &mod_code[MI_FUNCTIONS+j]; + return (BeamInstr**) &mod_code->functions[j]; } } return NULL; @@ -2444,8 +2453,8 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) if (init_func != NULL) handle = init_func; - if (!in_area(caller, mod->curr.code, mod->curr.code_length)) { - ASSERT(in_area(caller, mod->old.code, mod->old.code_length)); + if (!in_area(caller, mod->curr.code_hdr, mod->curr.code_length)) { + ASSERT(in_area(caller, mod->old.code_hdr, mod->old.code_length)); ret = load_nif_error(BIF_P, "old_code", "Calling load_nif from old " "module '%T' not allowed", mod_atom); @@ -2499,7 +2508,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) for (i=0; i < entry->num_of_funcs && ret==am_ok; i++) { BeamInstr** code_pp; if (!erts_atom_get(f->name, sys_strlen(f->name), &f_atom, ERTS_ATOM_ENC_LATIN1) - || (code_pp = get_func_pp(mod->curr.code, f_atom, f->arity))==NULL) { + || (code_pp = get_func_pp(mod->curr.code_hdr, f_atom, f->arity))==NULL) { ret = load_nif_error(BIF_P,bad_lib,"Function not found %T:%s/%u", mod_atom, f->name, f->arity); } @@ -2642,7 +2651,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) { BeamInstr* code_ptr; erts_atom_get(f->name, sys_strlen(f->name), &f_atom, ERTS_ATOM_ENC_LATIN1); - code_ptr = *get_func_pp(mod->curr.code, f_atom, f->arity); + code_ptr = *get_func_pp(mod->curr.code_hdr, f_atom, f->arity); if (code_ptr[1] == 0) { code_ptr[5+0] = (BeamInstr) BeamOp(op_call_nif); diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h index 75070ad901..c9c8b1d0af 100644 --- a/erts/emulator/beam/erl_nif.h +++ b/erts/emulator/beam/erl_nif.h @@ -77,13 +77,8 @@ typedef ErlNapiSInt64 ErlNifSInt64; typedef ErlNapiUInt ErlNifUInt; typedef ErlNapiSInt ErlNifSInt; -#ifdef HALFWORD_HEAP_EMULATOR -# define ERL_NIF_VM_VARIANT "beam.halfword" -typedef unsigned int ERL_NIF_TERM; -#else # define ERL_NIF_VM_VARIANT "beam.vanilla" typedef ErlNifUInt ERL_NIF_TERM; -#endif typedef ERL_NIF_TERM ERL_NIF_UINT; @@ -226,22 +221,28 @@ extern TWinDynNifCallbacks WinDynNifCallbacks; #if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_)) # define ERL_NIF_INIT_GLOB TWinDynNifCallbacks WinDynNifCallbacks; -# ifdef STATIC_ERLANG_NIF -# define ERL_NIF_INIT_DECL(MODNAME) __declspec(dllexport) ErlNifEntry* MODNAME ## _nif_init(TWinDynNifCallbacks* callbacks) -# else -# define ERL_NIF_INIT_DECL(MODNAME) __declspec(dllexport) ErlNifEntry* nif_init(TWinDynNifCallbacks* callbacks) -# endif +# define ERL_NIF_INIT_ARGS TWinDynNifCallbacks* callbacks # define ERL_NIF_INIT_BODY memcpy(&WinDynNifCallbacks,callbacks,sizeof(TWinDynNifCallbacks)) +# define ERL_NIF_INIT_EXPORT __declspec(dllexport) #else # define ERL_NIF_INIT_GLOB +# define ERL_NIF_INIT_ARGS void # define ERL_NIF_INIT_BODY -# ifdef STATIC_ERLANG_NIF -# define ERL_NIF_INIT_DECL(MODNAME) ErlNifEntry* MODNAME ## _nif_init(void) +# if defined(__GNUC__) && __GNUC__ >= 4 +# define ERL_NIF_INIT_EXPORT __attribute__ ((visibility("default"))) +# elif defined (__SUNPRO_C) && (__SUNPRO_C >= 0x550) +# define ERL_NIF_INIT_EXPORT __global # else -# define ERL_NIF_INIT_DECL(MODNAME) ErlNifEntry* nif_init(void) +# define ERL_NIF_INIT_EXPORT # endif #endif +#ifdef STATIC_ERLANG_NIF +# define ERL_NIF_INIT_DECL(MODNAME) ErlNifEntry* MODNAME ## _nif_init(ERL_NIF_INIT_ARGS) +#else +# define ERL_NIF_INIT_DECL(MODNAME) ERL_NIF_INIT_EXPORT ErlNifEntry* nif_init(ERL_NIF_INIT_ARGS) +#endif + #ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT # define ERL_NIF_ENTRY_OPTIONS ERL_NIF_DIRTY_NIF_OPTION #else diff --git a/erts/emulator/beam/erl_node_container_utils.h b/erts/emulator/beam/erl_node_container_utils.h index 211b1a0090..b6a3531e28 100644 --- a/erts/emulator/beam/erl_node_container_utils.h +++ b/erts/emulator/beam/erl_node_container_utils.h @@ -255,7 +255,7 @@ extern ErtsPTab erts_port; * Refs * \* */ -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) #define internal_ref_no_of_numbers(x) \ (internal_ref_data((x))[0]) @@ -328,8 +328,6 @@ extern ErtsPTab erts_port; : external_ref_channel_no((x))) #define is_ref(x) (is_internal_ref((x)) \ || is_external_ref((x))) -#define is_ref_rel(x,Base) (is_internal_ref_rel((x),Base) \ - || is_external_ref_rel((x),Base)) #define is_not_ref(x) (!is_ref(x)) #endif diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c index 707de39556..13f14adbab 100644 --- a/erts/emulator/beam/erl_node_tables.c +++ b/erts/emulator/beam/erl_node_tables.c @@ -787,10 +787,13 @@ void erts_init_node_tables(int dd_sec) erts_smp_rwmtx_init_opt(&erts_node_table_rwmtx, &rwmtx_opt, "node_table"); erts_smp_rwmtx_init_opt(&erts_dist_table_rwmtx, &rwmtx_opt, "dist_table"); - f.hash = (H_FUN) dist_table_hash; - f.cmp = (HCMP_FUN) dist_table_cmp; - f.alloc = (HALLOC_FUN) dist_table_alloc; - f.free = (HFREE_FUN) dist_table_free; + f.hash = (H_FUN) dist_table_hash; + f.cmp = (HCMP_FUN) dist_table_cmp; + f.alloc = (HALLOC_FUN) dist_table_alloc; + f.free = (HFREE_FUN) dist_table_free; + f.meta_alloc = (HMALLOC_FUN) erts_alloc; + f.meta_free = (HMFREE_FUN) erts_free; + f.meta_print = (HMPRINT_FUN) erts_print; hash_init(ERTS_ALC_T_DIST_TABLE, &erts_dist_table, "dist_table", 11, f); f.hash = (H_FUN) node_table_hash; @@ -1157,23 +1160,11 @@ insert_offheap(ErlOffHeap *oh, int type, Eterm id) break; } if (insert_bin) { -#if HALFWORD_HEAP - UWord val = (UWord) u.pb->val; - DeclareTmpHeapNoproc(id_heap,BIG_UINT_HEAP_SIZE*2); /* extra place allocated */ -#else DeclareTmpHeapNoproc(id_heap,BIG_UINT_HEAP_SIZE); -#endif Uint *hp = &id_heap[0]; InsertedBin *nib; -#if HALFWORD_HEAP - int actual_need = BIG_UWORD_HEAP_SIZE(val); - ASSERT(actual_need <= (BIG_UINT_HEAP_SIZE*2)); - UseTmpHeapNoproc(actual_need); - a.id = erts_bld_uword(&hp, NULL, (UWord) val); -#else UseTmpHeapNoproc(BIG_UINT_HEAP_SIZE); a.id = erts_bld_uint(&hp, NULL, (Uint) u.pb->val); -#endif erts_match_prog_foreach_offheap(u.pb->val, insert_offheap2, (void *) &a); @@ -1181,11 +1172,7 @@ insert_offheap(ErlOffHeap *oh, int type, Eterm id) nib->bin_val = u.pb->val; nib->next = inserted_bins; inserted_bins = nib; -#if HALFWORD_HEAP - UnUseTmpHeapNoproc(actual_need); -#else UnUseTmpHeapNoproc(BIG_UINT_HEAP_SIZE); -#endif } } break; @@ -1371,56 +1358,50 @@ setup_reference_table(void) for (i = 0; i < max; i++) { Process *proc = erts_pix2proc(i); if (proc) { - ErlMessage *msg; + int mli; + ErtsMessage *msg_list[] = { + proc->msg.first, +#ifdef ERTS_SMP + proc->msg_inq.first, +#endif + proc->msg_frag}; /* Insert Heap */ insert_offheap(&(proc->off_heap), HEAP_REF, proc->common.id); - /* Insert message buffers */ + /* Insert heap fragments buffers */ for(hfp = proc->mbuf; hfp; hfp = hfp->next) insert_offheap(&(hfp->off_heap), HEAP_REF, proc->common.id); - /* Insert msg msg buffers */ - for (msg = proc->msg.first; msg; msg = msg->next) { - ErlHeapFragment *heap_frag = NULL; - if (msg->data.attached) { - if (is_value(ERL_MESSAGE_TERM(msg))) - heap_frag = msg->data.heap_frag; - else { - if (msg->data.dist_ext->dep) - insert_dist_entry(msg->data.dist_ext->dep, - HEAP_REF, proc->common.id, 0); - if (is_not_nil(ERL_MESSAGE_TOKEN(msg))) - heap_frag = erts_dist_ext_trailer(msg->data.dist_ext); + + /* Insert msg buffers */ + for (mli = 0; mli < sizeof(msg_list)/sizeof(msg_list[0]); mli++) { + ErtsMessage *msg; + for (msg = msg_list[mli]; msg; msg = msg->next) { + ErlHeapFragment *heap_frag = NULL; + if (msg->data.attached) { + if (msg->data.attached == ERTS_MSG_COMBINED_HFRAG) + heap_frag = &msg->hfrag; + else if (is_value(ERL_MESSAGE_TERM(msg))) + heap_frag = msg->data.heap_frag; + else { + if (msg->data.dist_ext->dep) + insert_dist_entry(msg->data.dist_ext->dep, + HEAP_REF, proc->common.id, 0); + if (is_not_nil(ERL_MESSAGE_TOKEN(msg))) + heap_frag = erts_dist_ext_trailer(msg->data.dist_ext); + } } - } - if (heap_frag) - insert_offheap(&(heap_frag->off_heap), - HEAP_REF, - proc->common.id); - } -#ifdef ERTS_SMP - for (msg = proc->msg_inq.first; msg; msg = msg->next) { - ErlHeapFragment *heap_frag = NULL; - if (msg->data.attached) { - if (is_value(ERL_MESSAGE_TERM(msg))) - heap_frag = msg->data.heap_frag; - else { - if (msg->data.dist_ext->dep) - insert_dist_entry(msg->data.dist_ext->dep, - HEAP_REF, proc->common.id, 0); - if (is_not_nil(ERL_MESSAGE_TOKEN(msg))) - heap_frag = erts_dist_ext_trailer(msg->data.dist_ext); + while (heap_frag) { + insert_offheap(&(heap_frag->off_heap), + HEAP_REF, + proc->common.id); + heap_frag = heap_frag->next; } } - if (heap_frag) - insert_offheap(&(heap_frag->off_heap), - HEAP_REF, - proc->common.id); } -#endif /* Insert links */ if (ERTS_P_LINKS(proc)) insert_links(ERTS_P_LINKS(proc), proc->common.id); diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h index 694ac84232..fb2f2a5407 100644 --- a/erts/emulator/beam/erl_node_tables.h +++ b/erts/emulator/beam/erl_node_tables.h @@ -41,6 +41,7 @@ #include "sys.h" #include "hash.h" +#include "erl_alloc.h" #include "erl_process.h" #include "erl_monitors.h" #include "erl_smp.h" @@ -66,7 +67,7 @@ #define ERTS_DE_QFLGS_ALL (ERTS_DE_QFLG_BUSY \ | ERTS_DE_QFLG_EXIT) -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) #define ERTS_DIST_OUTPUT_BUF_DBG_PATTERN ((Uint) 0xf713f713f713f713UL) #else #define ERTS_DIST_OUTPUT_BUF_DBG_PATTERN ((Uint) 0xf713f713) diff --git a/erts/emulator/beam/erl_port.h b/erts/emulator/beam/erl_port.h index acd68ef0ad..fa97707a87 100644 --- a/erts/emulator/beam/erl_port.h +++ b/erts/emulator/beam/erl_port.h @@ -187,6 +187,11 @@ struct _erl_drv_port { ErtsPrtSD *psd; /* Port specific data */ int reds; /* Only used while executing driver callbacks */ + + struct { + Eterm to; + Uint32 ref[ERTS_MAX_REF_NUMBERS]; + } *async_open_port; /* Reference used with async open port */ }; @@ -687,7 +692,7 @@ erts_drvport2port_state(ErlDrvPort drvport, erts_aint32_t *statep) Port *prt = ERTS_ErlDrvPort2Port(drvport); erts_aint32_t state; ASSERT(prt); - ERTS_LC_ASSERT(erts_lc_is_emu_thr()); +// ERTS_LC_ASSERT(erts_lc_is_emu_thr()); if (prt == ERTS_INVALID_ERL_DRV_PORT) return ERTS_INVALID_ERL_DRV_PORT; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt) @@ -944,4 +949,9 @@ ErtsPortOpResult erts_port_control(Process *, Port *, unsigned int, Eterm, Eterm ErtsPortOpResult erts_port_call(Process *, Port *, unsigned int, Eterm, Eterm *); ErtsPortOpResult erts_port_info(Process *, Port *, Eterm, Eterm *); +/* + * Signals from ports to ports. Used by sys drivers. + */ +int erl_drv_port_control(Eterm, char, char*, ErlDrvSizeT); + #endif diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c index 5c38db1cbc..197b328fe2 100644 --- a/erts/emulator/beam/erl_port_task.c +++ b/erts/emulator/beam/erl_port_task.c @@ -1648,6 +1648,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) int active; Uint64 start_time = 0; ErtsSchedulerData *esdp = runq->scheduler; + ERTS_MSACC_PUSH_STATE_M(); ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq)); @@ -1690,6 +1691,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) state = erts_atomic32_read_nob(&pp->state); pp->reds = ERTS_PORT_REDS_EXECUTE; + ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_PORT); goto begin_handle_tasks; while (1) { @@ -1734,7 +1736,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) reds = ERTS_PORT_REDS_INPUT; ASSERT((state & ERTS_PORT_SFLGS_DEAD) == 0); DTRACE_DRIVER(driver_ready_input, pp); - /* NOTE some windows/ose drivers use ->ready_input + /* NOTE some windows drivers use ->ready_input for input and output */ (*pp->drv_ptr->ready_input)((ErlDrvData) pp->drv_data, ptp->u.alive.td.io.event); @@ -1822,6 +1824,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) } erts_unblock_fpe(fpe_was_unmasked); + ERTS_MSACC_POP_STATE_M(); if (io_tasks_executed) { diff --git a/erts/emulator/beam/erl_printf_term.c b/erts/emulator/beam/erl_printf_term.c index 2917d58932..45ba4371dc 100644 --- a/erts/emulator/beam/erl_printf_term.c +++ b/erts/emulator/beam/erl_printf_term.c @@ -117,13 +117,12 @@ do { \ /* return 0 if list is not a non-empty flat list of printable characters */ static int -is_printable_string(Eterm list, Eterm* base) -{ +is_printable_string(Eterm list) { int len = 0; int c; while(is_list(list)) { - Eterm* consp = list_val_rel(list, base); + Eterm* consp = list_val(list); Eterm hd = CAR(consp); if (!is_byte(hd)) @@ -260,9 +259,7 @@ static char *format_binary(Uint16 x, char *b) { #endif static int -print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount, - Eterm* obj_base) /* ignored if !HALFWORD_HEAP */ -{ +print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount) { DECLARE_WSTACK(s); int res; int i; @@ -308,7 +305,7 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount, obj = (Eterm) popped.word; L_print_one_cons: { - Eterm* cons = list_val_rel(obj, obj_base); + Eterm* cons = list_val(obj); Eterm tl; obj = CAR(cons); @@ -344,11 +341,7 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount, PRINT_CHAR(res, fn, arg, '>'); goto L_done; } -#if HALFWORD_HEAP - wobj = is_immed(obj) ? (Wterm)obj : rterm2wterm(obj, obj_base); -#else wobj = (Wterm)obj; -#endif switch (tag_val_def(wobj)) { case NIL_DEF: PRINT_STRING(res, fn, arg, "[]"); @@ -424,10 +417,10 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount, PRINT_CHAR(res, fn, arg, '>'); break; case LIST_DEF: - if (is_printable_string(obj, obj_base)) { + if (is_printable_string(obj)) { int c; PRINT_CHAR(res, fn, arg, '"'); - nobj = list_val_rel(obj, obj_base); + nobj = list_val(obj); while (1) { if ((*dcount)-- <= 0) goto L_done; @@ -441,7 +434,7 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount, } if (is_not_list(*nobj)) break; - nobj = list_val_rel(*nobj, obj_base); + nobj = list_val(*nobj); } PRINT_CHAR(res, fn, arg, '"'); } else { @@ -469,11 +462,11 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount, case BINARY_DEF: { byte* bytep; - Uint bytesize = binary_size_rel(obj,obj_base); + Uint bytesize = binary_size(obj); Uint bitoffs; Uint bitsize; byte octet; - ERTS_GET_BINARY_BYTES_REL(obj, bytep, bitoffs, bitsize, obj_base); + ERTS_GET_BINARY_BYTES(obj, bytep, bitoffs, bitsize); if (bitsize || !bytesize || !is_printable_ascii(bytep, bytesize, bitoffs)) { @@ -648,13 +641,11 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount, int -erts_printf_term(fmtfn_t fn, void* arg, ErlPfEterm term, long precision, - ErlPfEterm* term_base) -{ +erts_printf_term(fmtfn_t fn, void* arg, ErlPfEterm term, long precision) { int res; ERTS_CT_ASSERT(sizeof(ErlPfEterm) == sizeof(Eterm)); - res = print_term(fn, arg, (Eterm)term, &precision, (Eterm*)term_base); + res = print_term(fn, arg, (Eterm)term, &precision); if (res < 0) return res; if (precision <= 0) diff --git a/erts/emulator/beam/erl_printf_term.h b/erts/emulator/beam/erl_printf_term.h index 87618a36d7..4618457f27 100644 --- a/erts/emulator/beam/erl_printf_term.h +++ b/erts/emulator/beam/erl_printf_term.h @@ -22,6 +22,5 @@ #define ERL_PRINTF_TERM_H__ #include "erl_printf_format.h" -int erts_printf_term(fmtfn_t fn, void* arg, ErlPfEterm term, long precision, - ErlPfEterm* term_base); +int erts_printf_term(fmtfn_t fn, void* arg, ErlPfEterm term, long precision); #endif diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 99a4011b8f..e4584e7cc2 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -58,11 +58,7 @@ #define ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST (CONTEXT_REDS/10) -#ifndef ERTS_SCHED_MIN_SPIN #define ERTS_SCHED_SPIN_UNTIL_YIELD 100 -#else -#define ERTS_SCHED_SPIN_UNTIL_YIELD 1 -#endif #define ERTS_SCHED_SYS_SLEEP_SPINCOUNT_VERY_LONG 40 #define ERTS_SCHED_AUX_WORK_SLEEP_SPINCOUNT_FACT_VERY_LONG 1000 @@ -119,7 +115,7 @@ #define RUNQ_SET_RQ(X, RQ) erts_smp_atomic_set_nob((X), (erts_aint_t) (RQ)) #ifdef DEBUG -# if defined(ARCH_64) && !HALFWORD_HEAP +# if defined(ARCH_64) # define ERTS_DBG_SET_INVALID_RUNQP(RQP, N) \ (RUNQ_SET_RQ((RQP), (0xdeadbeefdead0003LL | ((N) << 4))) # define ERTS_DBG_VERIFY_VALID_RUNQP(RQP) \ @@ -152,18 +148,14 @@ extern BeamInstr beam_apply[]; extern BeamInstr beam_exit[]; extern BeamInstr beam_continue_exit[]; -#ifdef __OSE__ -/* Eager check I/O not supported on OSE yet. */ -int erts_eager_check_io = 0; -#else -int erts_eager_check_io = 1; -#endif -int erts_sched_compact_load; -int erts_sched_balance_util = 0; -Uint erts_no_schedulers; +int ERTS_WRITE_UNLIKELY(erts_default_spo_flags) = 0; +int ERTS_WRITE_UNLIKELY(erts_eager_check_io) = 1; +int ERTS_WRITE_UNLIKELY(erts_sched_compact_load); +int ERTS_WRITE_UNLIKELY(erts_sched_balance_util) = 0; +Uint ERTS_WRITE_UNLIKELY(erts_no_schedulers); #ifdef ERTS_DIRTY_SCHEDULERS -Uint erts_no_dirty_cpu_schedulers; -Uint erts_no_dirty_io_schedulers; +Uint ERTS_WRITE_UNLIKELY(erts_no_dirty_cpu_schedulers); +Uint ERTS_WRITE_UNLIKELY(erts_no_dirty_io_schedulers); #endif static char *erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_NO_FLAGS] = {0}; @@ -303,7 +295,7 @@ do { \ erts_sched_stat_t erts_sched_stat; #ifdef USE_THREADS -static erts_tsd_key_t sched_data_key; +static erts_tsd_key_t ERTS_WRITE_UNLIKELY(sched_data_key); #endif static erts_smp_atomic32_t function_calls; @@ -343,11 +335,10 @@ static ErtsAlignedSchedulerSleepInfo *aligned_dirty_io_sched_sleep_info; static Uint last_reductions; static Uint last_exact_reductions; -Uint erts_default_process_flags; -Eterm erts_system_monitor; -Eterm erts_system_monitor_long_gc; -Uint erts_system_monitor_long_schedule; -Eterm erts_system_monitor_large_heap; +Eterm ERTS_WRITE_UNLIKELY(erts_system_monitor); +Eterm ERTS_WRITE_UNLIKELY(erts_system_monitor_long_gc); +Uint ERTS_WRITE_UNLIKELY(erts_system_monitor_long_schedule); +Eterm ERTS_WRITE_UNLIKELY(erts_system_monitor_large_heap); struct erts_system_monitor_flags_t erts_system_monitor_flags; /* system performance monitor */ @@ -361,7 +352,8 @@ int erts_system_profile_ts_type = ERTS_TRACE_FLG_NOW_TIMESTAMP; typedef enum { ERTS_PSTT_GC, /* Garbage Collect */ - ERTS_PSTT_CPC /* Check Process Code */ + ERTS_PSTT_CPC, /* Check Process Code */ + ERTS_PSTT_COHMQ /* Change off heap message queue */ } ErtsProcSysTaskType; #define ERTS_MAX_PROC_SYS_TASK_ARGS 2 @@ -507,9 +499,6 @@ dbg_chk_aux_work_val(erts_aint32_t value) #if HAVE_ERTS_MSEG valid |= ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK; #endif -#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN - valid |= ERTS_SSI_AUX_WORK_CHECK_CHILDREN; -#endif #ifdef ERTS_SSI_AUX_WORK_REAP_PORTS valid |= ERTS_SSI_AUX_WORK_REAP_PORTS; #endif @@ -596,8 +585,6 @@ erts_pre_init_process(void) = "MISC_THR_PRGR"; erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_MISC_IX] = "MISC"; - erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_CHECK_CHILDREN_IX] - = "CHECK_CHILDREN"; erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_SET_TMO_IX] = "SET_TMO"; erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX] @@ -685,7 +672,6 @@ erts_init_process(int ncpu, int proc_tab_size, int legacy_proc_tab) last_reductions = 0; last_exact_reductions = 0; - erts_default_process_flags = 0; } void @@ -955,6 +941,12 @@ sched_wall_time_change(ErtsSchedulerData *esdp, int working) } } } + if (!working) { + ERTS_MSACC_SET_STATE_M_X(ERTS_MSACC_STATE_BUSY_WAIT); + } else { + ERTS_MSACC_SET_STATE_M_X(ERTS_MSACC_STATE_OTHER); + } + } typedef struct { @@ -976,7 +968,6 @@ typedef struct { } ErtsSystemCheckReq; -#if !HALFWORD_HEAP ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(swtreq, ErtsSchedWallTimeReq, 5, @@ -986,33 +977,6 @@ ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(screq, ErtsSystemCheckReq, 5, ERTS_ALC_T_SYS_CHECK_REQ) -#else -static ERTS_INLINE ErtsSchedWallTimeReq * -swtreq_alloc(void) -{ - return erts_alloc(ERTS_ALC_T_SCHED_WTIME_REQ, - sizeof(ErtsSchedWallTimeReq)); -} - -static ERTS_INLINE void -swtreq_free(ErtsSchedWallTimeReq *ptr) -{ - erts_free(ERTS_ALC_T_SCHED_WTIME_REQ, ptr); -} - -static ERTS_INLINE ErtsSystemCheckReq * -screq_alloc(void) -{ - return erts_alloc(ERTS_ALC_T_SYS_CHECK_REQ, - sizeof(ErtsSystemCheckReq)); -} - -static ERTS_INLINE void -screq_free(ErtsSystemCheckReq *ptr) -{ - erts_free(ERTS_ALC_T_SYS_CHECK_REQ, ptr); -} -#endif static void reply_sched_wall_time(void *vswtrp) @@ -1029,7 +993,7 @@ reply_sched_wall_time(void *vswtrp) Eterm **hpp; Uint sz, *szp; ErlOffHeap *ohp = NULL; - ErlHeapFragment *bp = NULL; + ErtsMessage *mp = NULL; ASSERT(esdp); #ifdef ERTS_DIRTY_SCHEDULERS @@ -1085,12 +1049,12 @@ reply_sched_wall_time(void *vswtrp) if (hpp) break; - hp = erts_alloc_message_heap(sz, &bp, &ohp, rp, &rp_locks); + mp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp); szp = NULL; hpp = &hp; } - erts_queue_message(rp, &rp_locks, bp, msg, NIL); + erts_queue_message(rp, &rp_locks, mp, msg, NIL); if (swtrp->req_sched == esdp->no) rp_locks &= ~ERTS_PROC_LOCK_MAIN; @@ -1157,7 +1121,7 @@ reply_system_check(void *vscrp) Eterm **hpp; Uint sz; ErlOffHeap *ohp = NULL; - ErlHeapFragment *bp = NULL; + ErtsMessage *mp = NULL; ASSERT(esdp); #ifdef ERTS_DIRTY_SCHEDULERS @@ -1165,11 +1129,11 @@ reply_system_check(void *vscrp) #endif sz = REF_THING_SIZE; - hp = erts_alloc_message_heap(sz, &bp, &ohp, rp, &rp_locks); + mp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp); hpp = &hp; msg = STORE_NC(hpp, ohp, scrp->ref); - erts_queue_message(rp, &rp_locks, bp, msg, NIL); + erts_queue_message(rp, &rp_locks, mp, msg, NIL); if (scrp->req_sched == esdp->no) rp_locks &= ~ERTS_PROC_LOCK_MAIN; @@ -1821,15 +1785,17 @@ handle_delayed_dealloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waitin int need_thr_progress = 0; ErtsThrPrgrVal wakeup = ERTS_THR_PRGR_INVALID; int more_work = 0; - + ERTS_MSACC_PUSH_STATE_M_X(); #ifdef ERTS_DIRTY_SCHEDULERS ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp)); #endif unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD); + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_ALLOC); erts_alloc_scheduler_handle_delayed_dealloc((void *) awdp->esdp, &need_thr_progress, &wakeup, &more_work); + ERTS_MSACC_POP_STATE_M_X(); if (more_work) { if (set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD) & ERTS_SSI_AUX_WORK_DD_THR_PRGR) { @@ -2216,34 +2182,6 @@ erts_debug_wait_completed(Process *c_p, int flags) } -#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN -void -erts_smp_notify_check_children_needed(void) -{ - int i; - for (i = 0; i < erts_no_schedulers; i++) - set_aux_work_flags_wakeup_nob(ERTS_SCHED_SLEEP_INFO_IX(i), - ERTS_SSI_AUX_WORK_CHECK_CHILDREN); -#ifdef ERTS_DIRTY_SCHEDULERS - for (i = 0; i < erts_no_dirty_cpu_schedulers; i++) - set_aux_work_flags_wakeup_nob(ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(i), - ERTS_SSI_AUX_WORK_CHECK_CHILDREN); - for (i = 0; i < erts_no_dirty_io_schedulers; i++) - set_aux_work_flags_wakeup_nob(ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(i), - ERTS_SSI_AUX_WORK_CHECK_CHILDREN); -#endif -} - -static ERTS_INLINE erts_aint32_t -handle_check_children(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting) -{ - unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_CHECK_CHILDREN); - erts_check_children(); - return aux_work & ~ERTS_SSI_AUX_WORK_CHECK_CHILDREN; -} - -#endif - static void notify_reap_ports_relb(void) { @@ -2329,12 +2267,15 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work, int waiting) ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); \ if (!(aux_work & ~ignore)) { \ ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); \ + ERTS_MSACC_UPDATE_CACHE(); \ + ERTS_MSACC_POP_STATE_M(); \ return aux_work; \ } \ } erts_aint32_t aux_work = orig_aux_work; erts_aint32_t ignore = 0; + ERTS_MSACC_PUSH_AND_SET_STATE_M(ERTS_MSACC_STATE_AUX); ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp)); #ifdef ERTS_SMP @@ -2398,10 +2339,6 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work, int waiting) HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_MISC, handle_misc_aux_work); -#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN - HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_CHECK_CHILDREN, - handle_check_children); -#endif HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_SET_TMO, handle_setup_aux_work_timer); @@ -2429,6 +2366,8 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work, int waiting) haw_thr_prgr_current_check_progress(awdp); #endif + ERTS_MSACC_UPDATE_CACHE(); + ERTS_MSACC_POP_STATE_M(); return aux_work; #undef HANDLE_AUX_WORK @@ -2629,19 +2568,10 @@ try_set_sys_scheduling(void) #endif static ERTS_INLINE int -prepare_for_sys_schedule(ErtsSchedulerData *esdp, int non_blocking) +prepare_for_sys_schedule(int non_blocking) { if (non_blocking && erts_eager_check_io) { #ifdef ERTS_SMP -#ifdef ERTS_SCHED_ONLY_POLL_SCHED_1 - if (esdp->no != 1) { - /* If we are not scheduler 1 and ERTS_SCHED_ONLY_POLL_SCHED_1 is used - then we make sure to wake scheduler 1 */ - ErtsRunQueue *rq = ERTS_RUNQ_IX(0); - wake_scheduler(rq); - return 0; - } -#endif return try_set_sys_scheduling(); #else return 1; @@ -2651,16 +2581,6 @@ prepare_for_sys_schedule(ErtsSchedulerData *esdp, int non_blocking) #ifdef ERTS_SMP while (!erts_port_task_have_outstanding_io_tasks() && try_set_sys_scheduling()) { -#ifdef ERTS_SCHED_ONLY_POLL_SCHED_1 - if (esdp->no != 1) { - /* If we are not scheduler 1 and ERTS_SCHED_ONLY_POLL_SCHED_1 is used - then we make sure to wake scheduler 1 */ - ErtsRunQueue *rq = ERTS_RUNQ_IX(0); - clear_sys_scheduling(); - wake_scheduler(rq); - return 0; - } -#endif if (!erts_port_task_have_outstanding_io_tasks()) return 1; clear_sys_scheduling(); @@ -2955,6 +2875,8 @@ aux_thread(void *unused) ssi->event = erts_tse_fetch(); + erts_msacc_init_thread("aux", 1, 1); + callbacks.arg = (void *) ssi; callbacks.wakeup = thr_prgr_wakeup; callbacks.prepare_wait = thr_prgr_prep_wait; @@ -2965,6 +2887,7 @@ aux_thread(void *unused) init_aux_work_data(awdp, NULL, NULL); awdp->ssi = ssi; + sched_prep_spin_wait(ssi); while (1) { @@ -2984,8 +2907,6 @@ aux_thread(void *unused) erts_thr_progress_active(NULL, thr_prgr_active = 0); erts_thr_progress_prepare_wait(NULL); - ERTS_SCHED_FAIR_YIELD(); - flgs = sched_spin_wait(ssi, 0); if (flgs & ERTS_SSI_FLG_SLEEPING) { @@ -3020,6 +2941,9 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) #ifdef ERTS_SMP int thr_prgr_active = 1; erts_aint32_t flgs; +#endif + ERTS_MSACC_PUSH_STATE_M(); +#ifdef ERTS_SMP ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); @@ -3053,7 +2977,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) * be waiting in erl_sys_schedule() */ - if (ERTS_SCHEDULER_IS_DIRTY(esdp) || !prepare_for_sys_schedule(esdp, 0)) { + if (ERTS_SCHEDULER_IS_DIRTY(esdp) || !prepare_for_sys_schedule(0)) { sched_waiting(esdp->no, rq); @@ -3067,7 +2991,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) sched_wall_time_change(esdp, thr_prgr_active); while (1) { - ErtsMonotonicTime current_time; + ErtsMonotonicTime current_time = 0; aux_work = erts_atomic32_read_acqb(&ssi->aux_work); if (aux_work && !ERTS_SCHEDULER_IS_DIRTY(esdp)) { @@ -3076,6 +3000,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) sched_wall_time_change(esdp, 1); } aux_work = handle_aux_work(&esdp->aux_work_data, aux_work, 1); + ERTS_MSACC_UPDATE_CACHE(); if (aux_work && erts_thr_progress_update(esdp)) erts_thr_progress_leader_update(esdp); } @@ -3117,8 +3042,6 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) erts_thr_progress_prepare_wait(esdp); } - ERTS_SCHED_FAIR_YIELD(); - flgs = sched_spin_wait(ssi, spincount); if (flgs & ERTS_SSI_FLG_SLEEPING) { ASSERT(flgs & ERTS_SSI_FLG_WAITING); @@ -3139,7 +3062,9 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) - 1) + 1; } else timeout = -1; + ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_SLEEP); res = erts_tse_twait(ssi->event, timeout); + ERTS_MSACC_POP_STATE_M(); current_time = ERTS_SCHEDULER_IS_DIRTY(esdp) ? 0 : erts_get_monotonic_time(esdp); } while (res == EINTR); @@ -3189,13 +3114,8 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) #ifdef ERTS_DIRTY_SCHEDULERS ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp)); #endif - -#ifdef ERTS_SCHED_ONLY_POLL_SCHED_1 - ASSERT(esdp->no == 1); -#endif sched_waiting_sys(esdp->no, rq); - erts_smp_runq_unlock(rq); ASSERT(working); @@ -3213,9 +3133,13 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) if (working) sched_wall_time_change(esdp, working = 0); + ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_CHECK_IO); + ASSERT(!erts_port_task_have_outstanding_io_tasks()); erl_sys_schedule(1); /* Might give us something to do */ + ERTS_MSACC_POP_STATE_M(); + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { current_time = erts_get_monotonic_time(esdp); if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) @@ -3236,6 +3160,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) erts_thr_progress_active(esdp, thr_prgr_active = 1); #endif aux_work = handle_aux_work(&esdp->aux_work_data, aux_work, 1); + ERTS_MSACC_UPDATE_CACHE(); #ifdef ERTS_SMP if (aux_work && erts_thr_progress_update(esdp)) erts_thr_progress_leader_update(esdp); @@ -3262,7 +3187,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) * Got to check that we still got I/O tasks; otherwise * we have to continue checking for I/O... */ - if (!prepare_for_sys_schedule(esdp, 0)) { + if (!prepare_for_sys_schedule(0)) { spincount *= ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT; goto tse_wait; } @@ -3284,7 +3209,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) * Got to check that we still got I/O tasks; otherwise * we have to wait in erl_sys_schedule() after all... */ - if (!prepare_for_sys_schedule(esdp, 0)) { + if (!prepare_for_sys_schedule(0)) { /* * Not allowed to wait in erl_sys_schedule; * do tse wait instead... @@ -3333,8 +3258,12 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) ASSERT(!erts_port_task_have_outstanding_io_tasks()); + ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_CHECK_IO); + erl_sys_schedule(0); + ERTS_MSACC_POP_STATE_M(); + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { ErtsMonotonicTime current_time = erts_get_monotonic_time(esdp); if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) @@ -3729,6 +3658,13 @@ check_requeue_process(ErtsRunQueue *rq, int prio_q) return 0; } +static ERTS_INLINE void +free_proxy_proc(Process *proxy) +{ + ASSERT(erts_smp_atomic32_read_nob(&proxy->state) & ERTS_PSFLG_PROXY); + erts_free(ERTS_ALC_T_PROC, proxy); +} + #ifdef ERTS_SMP static ErtsRunQueue * @@ -4043,9 +3979,6 @@ evacuate_run_queue(ErtsRunQueue *rq, erts_aint32_t state; Process *proc; int notify = 0; -#ifdef ERTS_DIRTY_SCHEDULERS - int requeue; -#endif to_rq = NULL; #ifdef ERTS_DIRTY_SCHEDULERS @@ -4060,49 +3993,94 @@ evacuate_run_queue(ErtsRunQueue *rq, proc = dequeue_process(rq, prio_q, &state); while (proc) { -#ifdef ERTS_DIRTY_SCHEDULERS - requeue = 1; + Process *real_proc; + int prio; + erts_aint32_t max_qbit, qbit, real_state; + + prio = ERTS_PSFLGS_GET_PRQ_PRIO(state); + qbit = ((erts_aint32_t) 1) << prio; + + if (!(state & ERTS_PSFLG_PROXY)) { + real_proc = proc; + real_state = state; + } + else { + real_proc = erts_proc_lookup_raw(proc->common.id); + if (!real_proc) { + free_proxy_proc(proc); + goto handle_next_proc; + } + real_state = erts_smp_atomic32_read_acqb(&real_proc->state); + } + + max_qbit = (state >> ERTS_PSFLGS_IN_PRQ_MASK_OFFSET); + max_qbit &= ERTS_PSFLGS_QMASK; + max_qbit |= 1 << ERTS_PSFLGS_QMASK_BITS; + max_qbit &= -max_qbit; + + if (qbit > max_qbit) { + /* Process already queued with higher prio; drop it... */ + if (real_proc != proc) + free_proxy_proc(proc); + else { + erts_aint32_t clr_bits; +#ifdef DEBUG + erts_aint32_t old; #endif - if (ERTS_PSFLG_BOUND & state) { - /* Bound processes get stuck here... */ - proc->next = NULL; - if (sbpp->last) - sbpp->last->next = proc; - else - sbpp->first = proc; - sbpp->last = proc; -#ifdef ERTS_DIRTY_SCHEDULERS - requeue = 0; + + clr_bits = ERTS_PSFLG_IN_RUNQ; + clr_bits |= qbit << ERTS_PSFLGS_IN_PRQ_MASK_OFFSET; + +#ifdef DEBUG + old = +#else + (void) #endif + erts_smp_atomic32_read_band_mb(&proc->state, + ~clr_bits); + ASSERT((old & clr_bits) == clr_bits); + + } + + goto handle_next_proc; } + #ifdef ERTS_DIRTY_SCHEDULERS - else if (state & ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q) { + + if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) { + erts_aint32_t dqbit = qbit; #ifdef DEBUG - erts_aint32_t old = -#else - (void) + erts_aint32_t old_dqbit; #endif - erts_smp_atomic32_read_band_nob(&proc->state, - ~(ERTS_PSFLG_DIRTY_CPU_PROC - | ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q)); - /* assert that no other dirty flags are set */ - ASSERT(!(old & (ERTS_PSFLG_DIRTY_IO_PROC|ERTS_PSFLG_DIRTY_IO_PROC_IN_Q))); - } else if (state & ERTS_PSFLG_DIRTY_IO_PROC_IN_Q) { + + if (rq == ERTS_DIRTY_CPU_RUNQ) + dqbit <<= ERTS_PDSFLGS_IN_CPU_PRQ_MASK_OFFSET; + else { + ASSERT(rq == ERTS_DIRTY_IO_RUNQ); + dqbit <<= ERTS_PDSFLGS_IN_IO_PRQ_MASK_OFFSET; + } + #ifdef DEBUG - erts_aint32_t old = + old_dqbit = (int) #else - (void) + (void) #endif - erts_smp_atomic32_read_band_nob(&proc->state, - ~(ERTS_PSFLG_DIRTY_IO_PROC - | ERTS_PSFLG_DIRTY_IO_PROC_IN_Q)); - /* assert that no other dirty flags are set */ - ASSERT(!(old & (ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q))); + erts_smp_atomic32_read_band_mb(&real_proc->dirty_state, + ~dqbit); + ASSERT(old_dqbit & dqbit); } - if (requeue) { -#else - else { #endif + + if (ERTS_PSFLG_BOUND & real_state) { + /* Bound processes get stuck here... */ + proc->next = NULL; + if (sbpp->last) + sbpp->last->next = proc; + else + sbpp->first = proc; + sbpp->last = proc; + } + else { int prio = (int) ERTS_PSFLGS_GET_PRQ_PRIO(state); erts_smp_runq_unlock(rq); @@ -4125,6 +4103,8 @@ evacuate_run_queue(ErtsRunQueue *rq, erts_smp_runq_lock(rq); } + + handle_next_proc: proc = dequeue_process(rq, prio_q, &state); } if (notify) @@ -5405,17 +5385,11 @@ erts_early_init_scheduling(int no_schedulers) wakeup_other.threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_MEDIUM; wakeup_other.type = ERTS_SCHED_WAKEUP_OTHER_TYPE_DEFAULT; #endif -#ifndef ERTS_SCHED_MIN_SPIN sched_busy_wait.sys_schedule = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_MEDIUM; sched_busy_wait.tse = (ERTS_SCHED_SYS_SLEEP_SPINCOUNT_MEDIUM * ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT); sched_busy_wait.aux_work = (ERTS_SCHED_SYS_SLEEP_SPINCOUNT_MEDIUM * ERTS_SCHED_AUX_WORK_SLEEP_SPINCOUNT_FACT_MEDIUM); -#else - sched_busy_wait.sys_schedule = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_NONE; - sched_busy_wait.tse = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_NONE; - sched_busy_wait.aux_work = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_NONE; -#endif } int @@ -5599,9 +5573,6 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num, esdp->f_reg_array = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_BEAM_REGISTER, MAX_REG * sizeof(FloatDef)); -#if !HEAP_ON_C_STACK - esdp->num_tmp_heap_used = 0; -#endif #ifdef ERTS_DIRTY_SCHEDULERS if (ERTS_RUNQ_IX_IS_DIRTY(runq->ix)) { esdp->no = 0; @@ -5886,10 +5857,8 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online #endif init_misc_aux_work(); -#if !HALFWORD_HEAP init_swtreq_alloc(); init_screq_alloc(); -#endif erts_atomic32_init_nob(&debug_wait_completed_count, 0); /* debug only */ debug_wait_completed_flags = 0; @@ -6069,19 +6038,96 @@ make_proxy_proc(Process *prev_proxy, Process *proc, erts_aint32_t prio) return proxy; } -static ERTS_INLINE void -free_proxy_proc(Process *proxy) -{ - ASSERT(erts_smp_atomic32_read_nob(&proxy->state) & ERTS_PSFLG_PROXY); - erts_free(ERTS_ALC_T_PROC, proxy); -} - #define ERTS_ENQUEUE_NOT 0 #define ERTS_ENQUEUE_NORMAL_QUEUE 1 -#ifdef ERTS_DIRTY_SCHEDULERS #define ERTS_ENQUEUE_DIRTY_CPU_QUEUE 2 #define ERTS_ENQUEUE_DIRTY_IO_QUEUE 3 + +#ifdef ERTS_DIRTY_SCHEDULERS + +static int +check_dirty_enqueue_in_prio_queue(Process *c_p, + erts_aint32_t *newp, + erts_aint32_t actual, + erts_aint32_t aprio, + erts_aint32_t qbit) +{ + int queue; + erts_aint32_t dact, max_qbit; + + /* Termination should be done on an ordinary scheduler */ + if (actual & ERTS_PSFLG_EXITING) { + *newp &= ~ERTS_PSFLGS_DIRTY_WORK; + return ERTS_ENQUEUE_NORMAL_QUEUE; + } + + /* + * If we have system tasks, we enqueue on ordinary run-queue + * and take care of those system tasks first. + */ + if (actual & ERTS_PSFLG_ACTIVE_SYS) + return ERTS_ENQUEUE_NORMAL_QUEUE; + + dact = erts_smp_atomic32_read_mb(&c_p->dirty_state); + if (actual & (ERTS_PSFLG_DIRTY_ACTIVE_SYS + | ERTS_PSFLG_DIRTY_CPU_PROC)) { + max_qbit = ((dact >> ERTS_PDSFLGS_IN_CPU_PRQ_MASK_OFFSET) + & ERTS_PDSFLGS_QMASK); + queue = ERTS_ENQUEUE_DIRTY_CPU_QUEUE; + } + else { + ASSERT(actual & ERTS_PSFLG_DIRTY_IO_PROC); + max_qbit = ((dact >> ERTS_PDSFLGS_IN_IO_PRQ_MASK_OFFSET) + & ERTS_PDSFLGS_QMASK); + queue = ERTS_ENQUEUE_DIRTY_IO_QUEUE; + } + + max_qbit |= 1 << ERTS_PSFLGS_QMASK_BITS; + max_qbit &= -max_qbit; + + if (qbit >= max_qbit) + return ERTS_ENQUEUE_NOT; /* Already queued in higher or equal prio */ + if ((actual & (ERTS_PSFLG_IN_RUNQ|ERTS_PSFLGS_USR_PRIO_MASK)) + != (aprio << ERTS_PSFLGS_USR_PRIO_OFFSET)) { + /* + * Process struct already enqueued, or actual prio not + * equal to user prio, i.e., enqueue using proxy. + */ + return -1*queue; + } + + *newp |= ERTS_PSFLG_IN_RUNQ; + return queue; +} + +static ERTS_INLINE int +fin_dirty_enq_s_change(Process *p, + int pstruct_reserved, + erts_aint32_t enq_prio, + int qmask_offset) +{ + erts_aint32_t qbit = 1 << enq_prio; + qbit <<= qmask_offset; + + if (qbit & erts_smp_atomic32_read_bor_mb(&p->dirty_state, qbit)) { + /* Already enqueue by someone else... */ + if (pstruct_reserved) { + /* We reserved process struct for enqueue; clear it... */ +#ifdef DEBUG + erts_aint32_t old = +#else + (void) #endif + erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_IN_RUNQ); + ASSERT(old & ERTS_PSFLG_IN_RUNQ); + } + return 0; + } + + return !0; +} + +#endif /* ERTS_DIRTY_SCHEDULERS */ static ERTS_INLINE int check_enqueue_in_prio_queue(Process *c_p, @@ -6097,61 +6143,14 @@ check_enqueue_in_prio_queue(Process *c_p, *prq_prio_p = aprio; #ifdef ERTS_DIRTY_SCHEDULERS - if (actual & (ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC)) { - /* - * If we have system tasks of a priority higher - * or equal to the user priority, we enqueue - * on ordinary run-queue and take care of - * those system tasks first. - */ - if (actual & ERTS_PSFLG_ACTIVE_SYS) { - erts_aint32_t uprio, stprio, qmask; - uprio = (actual >> ERTS_PSFLGS_USR_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK; - if (aprio < uprio) - goto enqueue_normal_runq; /* system tasks with higher prio */ - erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_STATUS); - qmask = c_p->sys_task_qs->qmask; - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS); - switch (qmask & -qmask) { - case MAX_BIT: - stprio = PRIORITY_MAX; - break; - case HIGH_BIT: - stprio = PRIORITY_HIGH; - break; - case NORMAL_BIT: - stprio = PRIORITY_NORMAL; - break; - case LOW_BIT: - stprio = PRIORITY_LOW; - break; - default: - stprio = PRIORITY_LOW+1; - break; - } - if (stprio <= uprio) - goto enqueue_normal_runq; /* system tasks with higher prio */ - } - - /* Enqueue in dirty run queue if not already enqueued */ - if (actual & (ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q|ERTS_PSFLG_DIRTY_IO_PROC_IN_Q)) - return ERTS_ENQUEUE_NOT; /* already in queue */ - if (actual & ERTS_PSFLG_DIRTY_CPU_PROC) { - *newp |= ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q; - if (actual & ERTS_PSFLG_IN_RUNQ) - return -ERTS_ENQUEUE_DIRTY_CPU_QUEUE; /* use proxy */ - *newp |= ERTS_PSFLG_IN_RUNQ; - return ERTS_ENQUEUE_DIRTY_CPU_QUEUE; - } - *newp |= ERTS_PSFLG_DIRTY_IO_PROC_IN_Q; - if (actual & ERTS_PSFLG_IN_RUNQ) - return -ERTS_ENQUEUE_DIRTY_IO_QUEUE; /* use proxy */ - *newp |= ERTS_PSFLG_IN_RUNQ; - return ERTS_ENQUEUE_DIRTY_IO_QUEUE; + if (actual & ERTS_PSFLGS_DIRTY_WORK) { + int res = check_dirty_enqueue_in_prio_queue(c_p, newp, actual, + aprio, qbit); + if (res != ERTS_ENQUEUE_NORMAL_QUEUE) + return res; } - - enqueue_normal_runq: #endif + max_qbit = (actual >> ERTS_PSFLGS_IN_PRQ_MASK_OFFSET) & ERTS_PSFLGS_QMASK; max_qbit |= 1 << ERTS_PSFLGS_QMASK_BITS; max_qbit &= -max_qbit; @@ -6183,6 +6182,65 @@ check_enqueue_in_prio_queue(Process *c_p, return ERTS_ENQUEUE_NORMAL_QUEUE; } +static ERTS_INLINE ErtsRunQueue * +select_enqueue_run_queue(int enqueue, int enq_prio, Process *p, erts_aint32_t state) +{ + + switch (enqueue) { + + case ERTS_ENQUEUE_NOT: + + return NULL; + +#ifdef ERTS_DIRTY_SCHEDULERS + + case ERTS_ENQUEUE_DIRTY_CPU_QUEUE: + case -ERTS_ENQUEUE_DIRTY_CPU_QUEUE: + + if (fin_dirty_enq_s_change(p, enqueue > 0, enq_prio, + ERTS_PDSFLGS_IN_CPU_PRQ_MASK_OFFSET)) + return ERTS_DIRTY_CPU_RUNQ; + + return NULL; + + + case ERTS_ENQUEUE_DIRTY_IO_QUEUE: + case -ERTS_ENQUEUE_DIRTY_IO_QUEUE: + + if (fin_dirty_enq_s_change(p, enqueue > 0, enq_prio, + ERTS_PDSFLGS_IN_IO_PRQ_MASK_OFFSET)) + return ERTS_DIRTY_IO_RUNQ; + + return NULL; + +#endif + + default: { + ErtsRunQueue* runq; + + ASSERT(enqueue == ERTS_ENQUEUE_NORMAL_QUEUE + || enqueue == -ERTS_ENQUEUE_NORMAL_QUEUE); + + runq = erts_get_runq_proc(p); + +#ifdef ERTS_SMP + if (!(ERTS_PSFLG_BOUND & state)) { + ErtsRunQueue *new_runq = erts_check_emigration_need(runq, enq_prio); + if (new_runq) { + RUNQ_SET_RQ(&p->run_queue, new_runq); + runq = new_runq; + } + } +#endif + + ASSERT(runq); + + return runq; + } + } +} + + /* * schedule_out_process() return with c_rq locked. */ @@ -6191,11 +6249,7 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, Proces { erts_aint32_t a, e, n, enq_prio = -1; int enqueue; /* < 0 -> use proxy */ - Process* sched_p; ErtsRunQueue* runq; -#ifdef ERTS_SMP - int check_emigration_need; -#endif a = state; @@ -6207,7 +6261,7 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, Proces enqueue = ERTS_ENQUEUE_NOT; n &= ~(ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS); - if (a & ERTS_PSFLG_ACTIVE_SYS + if (a & (ERTS_PSFLG_ACTIVE_SYS|ERTS_PSFLG_DIRTY_ACTIVE_SYS) || (a & (ERTS_PSFLG_ACTIVE|ERTS_PSFLG_SUSPENDED)) == ERTS_PSFLG_ACTIVE) { enqueue = check_enqueue_in_prio_queue(p, &enq_prio, &n, a); } @@ -6216,16 +6270,17 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, Proces break; } - switch (enqueue) { - case ERTS_ENQUEUE_NOT: + runq = select_enqueue_run_queue(enqueue, enq_prio, p, n); + + if (!runq) { + if (erts_system_profile_flags.runnable_procs) { /* Status lock prevents out of order "runnable proc" trace msgs */ ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); - if (!(a & ERTS_PSFLG_ACTIVE_SYS) - && (!(a & ERTS_PSFLG_ACTIVE) - || (a & ERTS_PSFLG_SUSPENDED))) { + if (!(a & (ERTS_PSFLG_ACTIVE_SYS|ERTS_PSFLG_DIRTY_ACTIVE_SYS)) + && (!(a & ERTS_PSFLG_ACTIVE) || (a & ERTS_PSFLG_SUSPENDED))) { /* Process inactive */ profile_runnable_proc(p, am_inactive); } @@ -6238,98 +6293,76 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, Proces return 0; -#ifdef ERTS_DIRTY_SCHEDULERS -#ifdef ERTS_SMP - case ERTS_ENQUEUE_DIRTY_CPU_QUEUE: - case -ERTS_ENQUEUE_DIRTY_CPU_QUEUE: - runq = ERTS_DIRTY_CPU_RUNQ; - ASSERT(ERTS_SCHEDULER_IS_DIRTY_CPU(runq->scheduler)); -#ifdef ERTS_SMP - check_emigration_need = 0; -#endif - break; + } + else { + Process* sched_p; - case ERTS_ENQUEUE_DIRTY_IO_QUEUE: - case -ERTS_ENQUEUE_DIRTY_IO_QUEUE: - runq = ERTS_DIRTY_IO_RUNQ; - ASSERT(ERTS_SCHEDULER_IS_DIRTY_IO(runq->scheduler)); -#ifdef ERTS_SMP - check_emigration_need = 0; -#endif - break; -#endif -#endif + ASSERT(!(n & ERTS_PSFLG_SUSPENDED) || (n & (ERTS_PSFLG_ACTIVE_SYS + | ERTS_PSFLG_DIRTY_ACTIVE_SYS))); - default: - ASSERT(enqueue == ERTS_ENQUEUE_NORMAL_QUEUE - || enqueue == -ERTS_ENQUEUE_NORMAL_QUEUE); + if (enqueue < 0) + sched_p = make_proxy_proc(proxy, p, enq_prio); + else { + sched_p = p; + if (proxy) + free_proxy_proc(proxy); + } - runq = erts_get_runq_proc(p); -#ifdef ERTS_SMP - check_emigration_need = !(ERTS_PSFLG_BOUND & n); -#endif - break; - } + ASSERT(runq); - ASSERT(!(n & ERTS_PSFLG_SUSPENDED) || (n & ERTS_PSFLG_ACTIVE_SYS)); + erts_smp_runq_lock(runq); - if (enqueue < 0) - sched_p = make_proxy_proc(proxy, p, enq_prio); - else { - sched_p = p; - if (proxy) - free_proxy_proc(proxy); - } + /* Enqueue the process */ + enqueue_process(runq, (int) enq_prio, sched_p); -#ifdef ERTS_SMP - if (check_emigration_need) { - ErtsRunQueue *new_runq = erts_check_emigration_need(runq, enq_prio); - if (new_runq) { - RUNQ_SET_RQ(&sched_p->run_queue, new_runq); - runq = new_runq; - } - } -#endif + if (runq == c_rq) + return 1; - ASSERT(runq); + erts_smp_runq_unlock(runq); - erts_smp_runq_lock(runq); + smp_notify_inc_runq(runq); - /* Enqueue the process */ - enqueue_process(runq, (int) enq_prio, sched_p); + erts_smp_runq_lock(c_rq); - if (runq == c_rq) return 1; - erts_smp_runq_unlock(runq); - smp_notify_inc_runq(runq); - erts_smp_runq_lock(c_rq); - return 1; + } + } static ERTS_INLINE void -add2runq(Process *p, erts_aint32_t state, erts_aint32_t prio) +add2runq(int enqueue, erts_aint32_t prio, + Process *proc, erts_aint32_t state, + Process **proxy) { - ErtsRunQueue *runq = erts_get_runq_proc(p); + ErtsRunQueue *runq; -#ifdef ERTS_SMP - if (!(ERTS_PSFLG_BOUND & state)) { - ErtsRunQueue *new_runq = erts_check_emigration_need(runq, (int) prio); - if (new_runq) { - RUNQ_SET_RQ(&p->run_queue, new_runq); - runq = new_runq; - } - } -#endif - ASSERT(runq); + runq = select_enqueue_run_queue(enqueue, prio, proc, state); - erts_smp_runq_lock(runq); + if (runq) { + Process *sched_p; - /* Enqueue the process */ - enqueue_process(runq, (int) prio, p); + if (enqueue > 0) + sched_p = proc; + else { + Process *pxy; - erts_smp_runq_unlock(runq); - smp_notify_inc_runq(runq); + if (!proxy) + pxy = NULL; + else { + pxy = *proxy; + *proxy = NULL; + } + sched_p = make_proxy_proc(pxy, proc, prio); + } + + erts_smp_runq_lock(runq); + /* Enqueue the process */ + enqueue_process(runq, (int) prio, sched_p); + + erts_smp_runq_unlock(runq); + smp_notify_inc_runq(runq); + } } static ERTS_INLINE int @@ -6435,10 +6468,7 @@ schedule_process(Process *p, erts_aint32_t in_state, ErtsProcLocks locks) &state, &enq_prio, locks); - if (enqueue != ERTS_ENQUEUE_NOT) - add2runq(enqueue > 0 ? p : make_proxy_proc(NULL, p, enq_prio), - state, - enq_prio); + add2runq(enqueue, enq_prio, p, state, NULL); } void @@ -6447,22 +6477,99 @@ erts_schedule_process(Process *p, erts_aint32_t state, ErtsProcLocks locks) schedule_process(p, state, locks); } -static void -schedule_process_sys_task(Process *p, erts_aint32_t state, Process *proxy) +static int +schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st) { - /* - * Expects status lock to be locked when called, and - * returns with status lock unlocked... - */ - erts_aint32_t a = state, n, enq_prio = -1; + int res; + int locked; + ErtsProcSysTaskQs *stqs, *free_stqs; + erts_aint32_t state, a, n, enq_prio; int enqueue; /* < 0 -> use proxy */ - unsigned int prof_runnable_procs = erts_system_profile_flags.runnable_procs; + unsigned int prof_runnable_procs; + + res = 1; /* prepare for success */ + st->next = st->prev = st; /* Prep for empty prio queue */ + state = erts_smp_atomic32_read_nob(&p->state); + prof_runnable_procs = erts_system_profile_flags.runnable_procs; + locked = 0; + free_stqs = NULL; + if (state & ERTS_PSFLG_ACTIVE_SYS) + stqs = NULL; + else { + alloc_qs: + stqs = proc_sys_task_queues_alloc(); + stqs->qmask = 1 << prio; + stqs->ncount = 0; + stqs->q[PRIORITY_MAX] = NULL; + stqs->q[PRIORITY_HIGH] = NULL; + stqs->q[PRIORITY_NORMAL] = NULL; + stqs->q[PRIORITY_LOW] = NULL; + stqs->q[prio] = st; + } + + if (!locked) { + locked = 1; + erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + + state = erts_smp_atomic32_read_nob(&p->state); + if (state & ERTS_PSFLG_EXITING) { + free_stqs = stqs; + res = 0; + goto cleanup; + } + } + + if (!p->sys_task_qs) { + if (stqs) + p->sys_task_qs = stqs; + else + goto alloc_qs; + } + else { + free_stqs = stqs; + stqs = p->sys_task_qs; + if (!stqs->q[prio]) { + stqs->q[prio] = st; + stqs->qmask |= 1 << prio; + } + else { + st->next = stqs->q[prio]; + st->prev = stqs->q[prio]->prev; + st->next->prev = st; + st->prev->next = st; + ASSERT(stqs->qmask & (1 << prio)); + } + } + + if (ERTS_PSFLGS_GET_ACT_PRIO(state) > prio) { + erts_aint32_t n, a, e; + /* Need to elevate actual prio */ + + a = state; + do { + if (ERTS_PSFLGS_GET_ACT_PRIO(a) <= prio) { + n = a; + break; + } + n = e = a; + n &= ~ERTS_PSFLGS_ACT_PRIO_MASK; + n |= (prio << ERTS_PSFLGS_ACT_PRIO_OFFSET); + a = erts_smp_atomic32_cmpxchg_nob(&p->state, n, e); + } while (a != e); + state = n; + } + + + a = state; + enq_prio = -1; /* Status lock prevents out of order "runnable proc" trace msgs */ ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); - if (!prof_runnable_procs) + if (!prof_runnable_procs) { erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + locked = 0; + } ASSERT(!(state & ERTS_PSFLG_PROXY)); @@ -6470,8 +6577,10 @@ schedule_process_sys_task(Process *p, erts_aint32_t state, Process *proxy) erts_aint32_t e; n = e = a; - if (a & ERTS_PSFLG_FREE) + if (a & ERTS_PSFLG_FREE) { + res = 0; goto cleanup; /* We don't want to schedule free processes... */ + } enqueue = ERTS_ENQUEUE_NOT; n |= ERTS_PSFLG_ACTIVE_SYS; @@ -6495,29 +6604,22 @@ schedule_process_sys_task(Process *p, erts_aint32_t state, Process *proxy) } erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); - prof_runnable_procs = 0; + locked = 0; } - if (enqueue != ERTS_ENQUEUE_NOT) { - Process *sched_p; - if (enqueue > 0) - sched_p = p; - else { - sched_p = make_proxy_proc(proxy, p, enq_prio); - proxy = NULL; - } - add2runq(sched_p, n, enq_prio); - } + add2runq(enqueue, enq_prio, p, n, NULL); cleanup: - if (prof_runnable_procs) + if (locked) erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); - if (proxy) - free_proxy_proc(proxy); + if (free_stqs) + proc_sys_task_queues_free(free_stqs); ERTS_SMP_LC_ASSERT(!(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p))); + + return res; } static ERTS_INLINE int @@ -6600,10 +6702,7 @@ resume_process(Process *p, ErtsProcLocks locks) &state, &enq_prio, locks); - if (enqueue) - add2runq(enqueue > 0 ? p : make_proxy_proc(NULL, p, enq_prio), - state, - enq_prio); + add2runq(enqueue, enq_prio, p, state, NULL); } int @@ -8046,6 +8145,8 @@ sched_thread_func(void *vesdp) callbacks.wait = thr_prgr_wait; callbacks.finalize_wait = thr_prgr_fin_wait; + erts_msacc_init_thread("scheduler", no, 1); + erts_thr_progress_register_managed_thread(esdp, &callbacks, 0); erts_alloc_register_scheduler(vesdp); #endif @@ -8301,18 +8402,12 @@ erts_start_schedulers(void) erts_snprintf(opts.name, 16, "%lu_scheduler", actual + 1); -#ifdef __OSE__ - /* This should be done in the bind strategy */ - opts.coreNo = (actual+1) % ose_num_cpus(); -#endif - res = ethr_thr_create(&esdp->tid, sched_thread_func, (void*)esdp, &opts); if (res != 0) { break; } } - erts_no_schedulers = actual; #ifdef ERTS_DIRTY_SCHEDULERS @@ -8341,10 +8436,6 @@ erts_start_schedulers(void) erts_snprintf(opts.name, 16, "aux"); -#ifdef __OSE__ - opts.coreNo = 0; -#endif /* __OSE__ */ - res = ethr_thr_create(&aux_tid, aux_thread, NULL, &opts); if (res != 0) erl_exit(1, "Failed to create aux thread\n"); @@ -8364,7 +8455,6 @@ erts_start_schedulers(void) actual, actual == 1 ? " was" : "s were"); erts_send_error_to_logger_nogl(dsbufp); } - } #endif /* ERTS_SMP */ @@ -8383,7 +8473,7 @@ add_pend_suspend(Process *suspendee, sizeof(ErtsPendingSuspend)); psp->next = NULL; #ifdef DEBUG -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) psp->end = (ErtsPendingSuspend *) 0xdeaddeaddeaddead; #else psp->end = (ErtsPendingSuspend *) 0xdeaddead; @@ -9378,6 +9468,8 @@ Process *schedule(Process *p, int calls) Uint32 flags; erts_aint32_t state = 0; /* Supress warning... */ + ERTS_MSACC_DECLARE_CACHE(); + #ifdef USE_VM_PROBES if (p != NULL && DTRACE_ENABLED(process_unscheduled)) { DTRACE_CHARBUF(process_buf, DTRACE_TERM_BUF_SIZE); @@ -9412,6 +9504,8 @@ Process *schedule(Process *p, int calls) } else { sched_out_proc: + ASSERT(!(p->flags & F_DELAY_GC)); + #ifdef ERTS_SMP ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); esdp = p->scheduler_data; @@ -9480,6 +9574,8 @@ Process *schedule(Process *p, int calls) erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS); + ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_OTHER); + if (state & ERTS_PSFLG_FREE) { #ifdef ERTS_SMP ASSERT(esdp->free_process == p); @@ -9516,15 +9612,16 @@ Process *schedule(Process *p, int calls) #ifdef ERTS_SMP ErtsMigrationPaths *mps; ErtsMigrationPath *mp; - ErtsProcList *pnd_xtrs = rq->procs.pending_exiters; - if (erts_proclist_fetch(&pnd_xtrs, NULL)) { + + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + ErtsProcList *pnd_xtrs = rq->procs.pending_exiters; + if (erts_proclist_fetch(&pnd_xtrs, NULL)) { rq->procs.pending_exiters = NULL; erts_smp_runq_unlock(rq); handle_pending_exiters(pnd_xtrs); erts_smp_runq_lock(rq); } - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { if (rq->check_balance_reds <= 0) check_balance(rq); @@ -9568,12 +9665,10 @@ Process *schedule(Process *p, int calls) erts_aint32_t aux_work; int leader_update = erts_thr_progress_update(esdp); aux_work = erts_atomic32_read_acqb(&esdp->ssi->aux_work); - if (aux_work | leader_update | ERTS_SCHED_FAIR) { + if (aux_work | leader_update) { erts_smp_runq_unlock(rq); if (leader_update) erts_thr_progress_leader_update(esdp); - else if (ERTS_SCHED_FAIR) - ERTS_SCHED_FAIR_YIELD(); if (aux_work) handle_aux_work(&esdp->aux_work_data, aux_work, 0); erts_smp_runq_lock(rq); @@ -9642,7 +9737,7 @@ Process *schedule(Process *p, int calls) scheduler_wait(&fcalls, esdp, rq); flags = ERTS_RUNQ_FLGS_SET_NOB(rq, ERTS_RUNQ_FLG_EXEC); flags |= ERTS_RUNQ_FLG_EXEC; - + ERTS_MSACC_UPDATE_CACHE(); #ifdef ERTS_SMP non_empty_runq(rq); #endif @@ -9651,12 +9746,14 @@ Process *schedule(Process *p, int calls) } else if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && (fcalls > input_reductions && - prepare_for_sys_schedule(esdp, !0))) { + prepare_for_sys_schedule(!0))) { ErtsMonotonicTime current_time; /* * Schedule system-level activities. */ + ERTS_MSACC_PUSH_STATE_CACHED_M(); + erts_smp_atomic32_set_relb(&function_calls, 0); fcalls = 0; @@ -9664,7 +9761,9 @@ Process *schedule(Process *p, int calls) erts_sys_schedule_interrupt(0); #endif erts_smp_runq_unlock(rq); + ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_CHECK_IO); erl_sys_schedule(1); + ERTS_MSACC_POP_STATE_M(); current_time = erts_get_monotonic_time(esdp); if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) @@ -9722,11 +9821,12 @@ Process *schedule(Process *p, int calls) pick_next_process: { erts_aint32_t psflg_band_mask; int prio_q; - int qmask; + int qmask, qbit; flags = ERTS_RUNQ_FLGS_GET_NOB(rq); qmask = (int) (flags & ERTS_RUNQ_FLGS_PROCS_QMASK); - switch (qmask & -qmask) { + qbit = qmask & -qmask; + switch (qbit) { case MAX_BIT: prio_q = PRIORITY_MAX; break; @@ -9742,9 +9842,12 @@ Process *schedule(Process *p, int calls) case 0: /* No process at all */ default: ASSERT(qmask == 0); + ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_OTHER); goto check_activities_to_run; } + ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_EMULATOR); + BM_START_TIMER(system); /* @@ -9754,20 +9857,11 @@ Process *schedule(Process *p, int calls) ASSERT(p); /* Wrong qmask in rq->flags? */ - psflg_band_mask = ~(((erts_aint32_t) 1) << (ERTS_PSFLGS_GET_PRQ_PRIO(state) - + ERTS_PSFLGS_IN_PRQ_MASK_OFFSET)); - -#ifdef ERTS_DIRTY_SCHEDULERS - ASSERT((state & (ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC)) != - (ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC)); - if (state & (ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC)) { - ASSERT((ERTS_SCHEDULER_IS_DIRTY_CPU(esdp) && (state & ERTS_PSFLG_DIRTY_CPU_PROC)) || - (ERTS_SCHEDULER_IS_DIRTY_IO(esdp) && (state & ERTS_PSFLG_DIRTY_IO_PROC))); - if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && !(state & ERTS_PSFLG_ACTIVE_SYS)) - goto pick_next_process; - state &= ~(ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q|ERTS_PSFLG_DIRTY_IO_PROC_IN_Q); - } -#endif + if (ERTS_SCHEDULER_IS_DIRTY(esdp)) + psflg_band_mask = ~((erts_aint32_t) 0); + else + psflg_band_mask = ~(((erts_aint32_t) 1) << (ERTS_PSFLGS_GET_PRQ_PRIO(state) + + ERTS_PSFLGS_IN_PRQ_MASK_OFFSET)); if (!(state & ERTS_PSFLG_PROXY)) psflg_band_mask &= ~ERTS_PSFLG_IN_RUNQ; @@ -9790,9 +9884,11 @@ Process *schedule(Process *p, int calls) | ERTS_PSFLG_RUNNING_SYS))) { tmp = state & (ERTS_PSFLG_SUSPENDED | ERTS_PSFLG_PENDING_EXIT - | ERTS_PSFLG_ACTIVE_SYS); + | ERTS_PSFLG_ACTIVE_SYS + | ERTS_PSFLG_DIRTY_ACTIVE_SYS); if (tmp != ERTS_PSFLG_SUSPENDED) { - if (state & ERTS_PSFLG_ACTIVE_SYS) + if (state & (ERTS_PSFLG_ACTIVE_SYS + | ERTS_PSFLG_DIRTY_ACTIVE_SYS)) new |= ERTS_PSFLG_RUNNING_SYS; else new |= ERTS_PSFLG_RUNNING; @@ -9805,7 +9901,8 @@ Process *schedule(Process *p, int calls) | ERTS_PSFLG_FREE)) || ((state & (ERTS_PSFLG_SUSPENDED | ERTS_PSFLG_PENDING_EXIT - | ERTS_PSFLG_ACTIVE_SYS)) + | ERTS_PSFLG_ACTIVE_SYS + | ERTS_PSFLG_DIRTY_ACTIVE_SYS)) == ERTS_PSFLG_SUSPENDED)) { if (state & ERTS_PSFLG_FREE) erts_proc_dec_refc(p); @@ -9824,10 +9921,42 @@ Process *schedule(Process *p, int calls) esdp->current_process = p; + + reds = context_reds; + +#ifdef ERTS_SMP + + erts_smp_runq_unlock(rq); + +#ifdef ERTS_DIRTY_SCHEDULERS + if (ERTS_SCHEDULER_IS_DIRTY(esdp)) { +#ifdef DEBUG + int old_dqbit; +#endif + int dqbit = qbit; + + if (rq == ERTS_DIRTY_CPU_RUNQ) + dqbit <<= ERTS_PDSFLGS_IN_CPU_PRQ_MASK_OFFSET; + else { + ASSERT(rq == ERTS_DIRTY_IO_RUNQ); + dqbit <<= ERTS_PDSFLGS_IN_IO_PRQ_MASK_OFFSET; + } + +#ifdef DEBUG + old_dqbit = (int) +#else + (void) +#endif + erts_smp_atomic32_read_band_mb(&p->dirty_state, ~dqbit); + ASSERT(old_dqbit & dqbit); + } +#endif /* ERTS_DIRTY_SCHEDULERS */ + +#endif /* ERTS_SMP */ + } #ifdef ERTS_SMP - erts_smp_runq_unlock(rq); if (flags & ERTS_RUNQ_FLG_PROTECTED) (void) ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_PROTECTED); @@ -9862,15 +9991,56 @@ Process *schedule(Process *p, int calls) erts_smp_spin_unlock(&erts_sched_stat.lock); } - if (ERTS_PROC_PENDING_EXIT(p)) { + ASSERT(!p->scheduler_data); + p->scheduler_data = esdp; + + state = erts_smp_atomic32_read_nob(&p->state); + +#ifdef ERTS_DIRTY_SCHEDULERS + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + if (!!(state & ERTS_PSFLGS_DIRTY_WORK) + & !(state & ERTS_PSFLG_ACTIVE_SYS)) { + /* Migrate to dirty scheduler... */ + sunlock_sched_out_proc: + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + p->fcalls = reds; + goto sched_out_proc; + + } + } + else { + if (state & (ERTS_PSFLG_ACTIVE_SYS + | ERTS_PSFLG_PENDING_EXIT + | ERTS_PSFLG_EXITING)) { + /* Migrate to normal scheduler... */ + goto sunlock_sched_out_proc; + } + if ((state & ERTS_PSFLG_DIRTY_ACTIVE_SYS) + && rq == ERTS_DIRTY_IO_RUNQ) { + /* Migrate to dirty cpu scheduler... */ + goto sunlock_sched_out_proc; + } + + ASSERT((state & ERTS_PSFLG_DIRTY_ACTIVE_SYS) + || *p->i == (BeamInstr) em_call_nif); + + ASSERT(rq == ERTS_DIRTY_CPU_RUNQ + ? (state & (ERTS_PSFLG_DIRTY_CPU_PROC + | ERTS_PSFLG_DIRTY_ACTIVE_SYS)) + : (rq == ERTS_DIRTY_IO_RUNQ + && (state & ERTS_PSFLG_DIRTY_IO_PROC))); + } +#endif + + if (state & ERTS_PSFLG_PENDING_EXIT) { erts_handle_pending_exit(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS); state = erts_smp_atomic32_read_nob(&p->state); } - ASSERT(!p->scheduler_data); - p->scheduler_data = esdp; -#endif - reds = context_reds; + +#endif /* ERTS_SMP */ + + p->fcalls = reds; if (IS_TRACED(p)) { if (state & ERTS_PSFLG_EXITING) { @@ -9899,7 +10069,8 @@ Process *schedule(Process *p, int calls) reds -= execute_sys_tasks(p, &state, reds); if (reds <= 0 #ifdef ERTS_DIRTY_SCHEDULERS - || (state & (ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC)) + || ERTS_SCHEDULER_IS_DIRTY(esdp) + || (state & ERTS_PSFLGS_DIRTY_WORK) #endif ) { p->fcalls = reds; @@ -9932,13 +10103,13 @@ Process *schedule(Process *p, int calls) } } - if (!(state & ERTS_PSFLG_EXITING) - && ((FLAGS(p) & F_FORCE_GC) - || (MSO(p).overhead > BIN_VHEAP_SZ(p)))) { - reds -= erts_garbage_collect(p, 0, p->arg_reg, p->arity); - if (reds <= 0) { - p->fcalls = reds; - goto sched_out_proc; + if (ERTS_IS_GC_DESIRED(p)) { + if (!(state & ERTS_PSFLG_EXITING) && !(p->flags & F_DISABLE_GC)) { + reds -= erts_garbage_collect_nobump(p, 0, p->arg_reg, p->arity); + if (reds <= 0) { + p->fcalls = reds; + goto sched_out_proc; + } } } @@ -9947,7 +10118,6 @@ Process *schedule(Process *p, int calls) proxy_p = NULL; } - p->fcalls = reds; ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); /* Never run a suspended process */ @@ -9978,7 +10148,7 @@ notify_sys_task_executed(Process *c_p, ErtsProcSysTask *st, Eterm st_result) if (rp) { ErtsProcLocks rp_locks; ErlOffHeap *ohp; - ErlHeapFragment* bp; + ErtsMessage *mp; Eterm *hp, msg, req_id, result; Uint st_result_sz, hsz; #ifdef DEBUG @@ -9990,11 +10160,7 @@ notify_sys_task_executed(Process *c_p, ErtsProcSysTask *st, Eterm st_result) st_result_sz = is_immed(st_result) ? 0 : size_object(st_result); hsz = st->req_id_sz + st_result_sz + 4 /* 3-tuple */; - hp = erts_alloc_message_heap(hsz, - &bp, - &ohp, - rp, - &rp_locks); + mp = erts_alloc_message_heap(rp, &rp_locks, hsz, &hp, &ohp); #ifdef DEBUG hp_start = hp; @@ -10019,7 +10185,7 @@ notify_sys_task_executed(Process *c_p, ErtsProcSysTask *st, Eterm st_result) ASSERT(hp_start + hsz == hp); #endif - erts_queue_message(rp, &rp_locks, bp, msg, NIL); + erts_queue_message(rp, &rp_locks, mp, msg, NIL); if (c_p == rp) rp_locks &= ~ERTS_PROC_LOCK_MAIN; @@ -10221,10 +10387,10 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) else { if (!garbage_collected) { FLAGS(c_p) |= F_NEED_FULLSWEEP; - reds += erts_garbage_collect(c_p, - 0, - c_p->arg_reg, - c_p->arity); + reds += erts_garbage_collect_nobump(c_p, + 0, + c_p->arg_reg, + c_p->arity); garbage_collected = 1; } st_res = am_true; @@ -10233,7 +10399,7 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) case ERTS_PSTT_CPC: st_res = erts_check_process_code(c_p, st->arg[0], - st->arg[1] == am_true, + unsigned_val(st->arg[1]), &reds); if (is_non_value(st_res)) { /* Needed gc, but gc was disabled */ @@ -10241,6 +10407,10 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) st = NULL; } break; + case ERTS_PSTT_COHMQ: + reds += erts_complete_off_heap_message_queue_change(c_p); + st_res = am_true; + break; default: ERTS_INTERNAL_ERROR("Invalid process sys task type"); st_res = am_false; @@ -10283,6 +10453,9 @@ cleanup_sys_tasks(Process *c_p, erts_aint32_t in_state, int in_reds) case ERTS_PSTT_CPC: st_res = am_false; break; + case ERTS_PSTT_COHMQ: + st_res = am_false; + break; default: ERTS_INTERNAL_ERROR("Invalid process sys task type"); st_res = am_false; @@ -10301,10 +10474,8 @@ BIF_RETTYPE erts_internal_request_system_task_3(BIF_ALIST_3) { Process *rp = erts_proc_lookup(BIF_ARG_1); - ErtsProcSysTaskQs *stqs, *free_stqs = NULL; ErtsProcSysTask *st = NULL; - erts_aint32_t prio, rp_state; - int rp_locked; + erts_aint32_t prio; Eterm noproc_res, req_type; if (!rp && !is_internal_pid(BIF_ARG_1)) { @@ -10361,7 +10532,6 @@ erts_internal_request_system_task_3(BIF_ALIST_3) } st = erts_alloc(ERTS_ALC_T_PROC_SYS_TSK, ERTS_PROC_SYS_TASK_SIZE(tot_sz)); - st->next = st->prev = st; /* Prep for empty prio queue */ ERTS_INIT_OFF_HEAP(&st->off_heap); hp = &st->heap[0]; @@ -10393,7 +10563,7 @@ erts_internal_request_system_task_3(BIF_ALIST_3) case am_check_process_code: if (is_not_atom(st->arg[0])) goto badarg; - if (st->arg[1] != am_true && st->arg[1] != am_false) + if (is_not_small(st->arg[1]) || (unsigned_val(st->arg[1]) & ~ERTS_CPC_ALL)) goto badarg; noproc_res = am_false; st->type = ERTS_PSTT_CPC; @@ -10405,95 +10575,11 @@ erts_internal_request_system_task_3(BIF_ALIST_3) goto badarg; } - rp_state = erts_smp_atomic32_read_nob(&rp->state); - - rp_locked = 0; - - free_stqs = NULL; - if (rp_state & ERTS_PSFLG_ACTIVE_SYS) - stqs = NULL; - else { - alloc_qs: - stqs = proc_sys_task_queues_alloc(); - stqs->qmask = 1 << prio; - stqs->ncount = 0; - stqs->q[PRIORITY_MAX] = NULL; - stqs->q[PRIORITY_HIGH] = NULL; - stqs->q[PRIORITY_NORMAL] = NULL; - stqs->q[PRIORITY_LOW] = NULL; - stqs->q[prio] = st; - } - - if (!rp_locked) { - rp_locked = 1; - erts_smp_proc_lock(rp, ERTS_PROC_LOCK_STATUS); - - rp_state = erts_smp_atomic32_read_nob(&rp->state); - if (rp_state & ERTS_PSFLG_EXITING) { - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); - rp = NULL; - free_stqs = stqs; - goto noproc; - } - } - - if (!rp->sys_task_qs) { - if (stqs) - rp->sys_task_qs = stqs; - else - goto alloc_qs; - } - else { - if (stqs) - free_stqs = stqs; - stqs = rp->sys_task_qs; - if (!stqs->q[prio]) { - stqs->q[prio] = st; - stqs->qmask |= 1 << prio; - } - else { - st->next = stqs->q[prio]; - st->prev = stqs->q[prio]->prev; - st->next->prev = st; - st->prev->next = st; - ASSERT(stqs->qmask & (1 << prio)); - } - } - - if (ERTS_PSFLGS_GET_ACT_PRIO(rp_state) > prio) { - erts_aint32_t n, a, e; - /* Need to elevate actual prio */ - - a = rp_state; - do { - if (ERTS_PSFLGS_GET_ACT_PRIO(a) <= prio) { - n = a; - break; - } - n = e = a; - n &= ~ERTS_PSFLGS_ACT_PRIO_MASK; - n |= (prio << ERTS_PSFLGS_ACT_PRIO_OFFSET); - a = erts_smp_atomic32_cmpxchg_nob(&rp->state, n, e); - } while (a != e); - rp_state = n; + if (!schedule_process_sys_task(rp, prio, st)) { + noproc: + notify_sys_task_executed(BIF_P, st, noproc_res); } - /* - * schedule_process_sys_task() unlocks status - * lock on process. - */ - schedule_process_sys_task(rp, rp_state, NULL); - - if (free_stqs) - proc_sys_task_queues_free(free_stqs); - - BIF_RET(am_ok); - -noproc: - - notify_sys_task_executed(BIF_P, st, noproc_res); - if (free_stqs) - proc_sys_task_queues_free(free_stqs); BIF_RET(am_ok); badarg: @@ -10502,11 +10588,35 @@ badarg: erts_cleanup_offheap(&st->off_heap); erts_free(ERTS_ALC_T_PROC_SYS_TSK, st); } - if (free_stqs) - proc_sys_task_queues_free(free_stqs); BIF_ERROR(BIF_P, BADARG); } +void +erts_schedule_complete_off_heap_message_queue_change(Eterm pid) +{ + Process *rp = erts_proc_lookup(pid); + if (rp) { + ErtsProcSysTask *st; + erts_aint32_t state; + int i; + + st = erts_alloc(ERTS_ALC_T_PROC_SYS_TSK, + ERTS_PROC_SYS_TASK_SIZE(0)); + st->type = ERTS_PSTT_COHMQ; + st->requester = NIL; + st->reply_tag = NIL; + st->req_id = NIL; + st->req_id_sz = 0; + for (i = 0; i < ERTS_MAX_PROC_SYS_TASK_ARGS; i++) + st->arg[i] = NIL; + ERTS_INIT_OFF_HEAP(&st->off_heap); + state = erts_smp_atomic32_read_nob(&rp->state); + + if (!schedule_process_sys_task(rp, ERTS_PSFLGS_GET_USR_PRIO(state), st)) + erts_free(ERTS_ALC_T_PROC_SYS_TSK, st); + } +} + static void save_gc_task(Process *c_p, ErtsProcSysTask *st, int prio) { @@ -10892,6 +11002,9 @@ static void early_init_process_struct(void *varg, Eterm data) Process *proc = arg->proc; proc->common.id = make_internal_pid(data); +#ifdef ERTS_DIRTY_SCHEDULERS + erts_smp_atomic32_init_nob(&proc->dirty_state, 0); +#endif erts_smp_atomic32_init_relb(&proc->state, arg->state); #ifdef ERTS_SMP @@ -10952,6 +11065,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). Eterm args, /* Arguments for function (must be well-formed list). */ ErlSpawnOpts* so) /* Options for spawn. */ { + Uint flags = 0; ErtsRunQueue *rq = NULL; Process *p; Sint arity; /* Number of arguments. */ @@ -10961,6 +11075,10 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). Eterm res = THE_NON_VALUE; erts_aint32_t state = 0; erts_aint32_t prio = (erts_aint32_t) PRIORITY_NORMAL; +#ifdef SHCOPY_SPAWN + erts_shcopy_t info; + INITIALIZE_SHCOPY(info); +#endif #ifdef ERTS_SMP erts_smp_proc_lock(parent, ERTS_PROC_LOCKS_ALL_MINOR); @@ -10989,6 +11107,15 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). state |= (((prio & ERTS_PSFLGS_PRIO_MASK) << ERTS_PSFLGS_ACT_PRIO_OFFSET) | ((prio & ERTS_PSFLGS_PRIO_MASK) << ERTS_PSFLGS_USR_PRIO_OFFSET)); + if (so->flags & SPO_OFF_HEAP_MSGQ) { + state |= ERTS_PSFLG_OFF_HEAP_MSGQ; + flags |= F_OFF_HEAP_MSGQ; + } + else if (so->flags & SPO_ON_HEAP_MSGQ) { + state |= ERTS_PSFLG_ON_HEAP_MSGQ; + flags |= F_ON_HEAP_MSGQ; + } + if (!rq) rq = erts_get_runq_proc(parent); @@ -11007,11 +11134,15 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). BM_COUNT(processes_spawned); BM_SWAP_TIMER(system,size); +#ifdef SHCOPY_SPAWN + arg_size = copy_shared_calculate(args, &info); +#else arg_size = size_object(args); +#endif BM_SWAP_TIMER(size,system); heap_need = arg_size; - p->flags = erts_default_process_flags; + p->flags = flags; p->static_flags = 0; if (so->flags & SPO_SYSTEM_PROC) @@ -11060,12 +11191,13 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->stop = p->hend = p->heap + sz; p->htop = p->heap; p->heap_sz = sz; + p->abandoned_heap = NULL; + p->live_hf_end = ERTS_INVALID_HFRAG_PTR; p->catches = 0; p->bin_vheap_sz = p->min_vheap_size; p->bin_old_vheap_sz = p->min_vheap_size; p->bin_old_vheap = 0; - p->bin_vheap_mature = 0; p->sys_task_qs = NULL; @@ -11084,7 +11216,12 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). BM_MESSAGE(args,p,parent); BM_START_TIMER(system); BM_SWAP_TIMER(system,copy); +#ifdef SHCOPY_SPAWN + p->arg_reg[2] = copy_shared_perform(args, arg_size, &info, &p->htop, &p->off_heap); + DESTROY_SHCOPY(info); +#else p->arg_reg[2] = copy_struct(args, arg_size, &p->htop, &p->off_heap); +#endif BM_MESSAGE_COPIED(arg_size); BM_SWAP_TIMER(copy,system); p->arity = 3; @@ -11130,6 +11267,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->accessor_bif_timers = NULL; #endif p->mbuf = NULL; + p->msg_frag = NULL; p->mbuf_sz = 0; p->psd = NULL; p->dictionary = NULL; @@ -11265,6 +11403,8 @@ void erts_init_empty_process(Process *p) p->stop = NULL; p->hend = NULL; p->heap = NULL; + p->abandoned_heap = NULL; + p->live_hf_end = ERTS_INVALID_HFRAG_PTR; p->gen_gcs = 0; p->max_gen_gcs = 0; p->min_heap_size = 0; @@ -11285,7 +11425,6 @@ void erts_init_empty_process(Process *p) p->bin_old_vheap_sz = BIN_VH_MIN_SIZE; p->bin_old_vheap = 0; p->sys_task_qs = NULL; - p->bin_vheap_mature = 0; ERTS_PTMR_INIT(p); p->next = NULL; p->off_heap.first = NULL; @@ -11297,6 +11436,7 @@ void erts_init_empty_process(Process *p) p->old_htop = NULL; p->old_heap = NULL; p->mbuf = NULL; + p->msg_frag = NULL; p->mbuf_sz = 0; p->psd = NULL; ERTS_P_MONITORS(p) = NULL; @@ -11354,6 +11494,9 @@ void erts_init_empty_process(Process *p) p->last_old_htop = NULL; #endif +#ifdef ERTS_DIRTY_SCHEDULERS + erts_smp_atomic32_init_nob(&p->dirty_state, 0); +#endif erts_smp_atomic32_init_nob(&p->state, (erts_aint32_t) PRIORITY_NORMAL); #ifdef ERTS_SMP @@ -11385,6 +11528,8 @@ erts_debug_verify_clean_empty_process(Process* p) ASSERT(p->htop == NULL); ASSERT(p->stop == NULL); ASSERT(p->hend == NULL); + ASSERT(p->abandoned_heap == NULL); + ASSERT(p->live_hf_end == ERTS_INVALID_HFRAG_PTR); ASSERT(p->heap == NULL); ASSERT(p->common.id == ERTS_INVALID_PID); ASSERT(ERTS_TRACER_PROC(p) == NIL); @@ -11462,9 +11607,10 @@ erts_cleanup_empty_process(Process* p) static void delete_process(Process* p) { - ErlMessage* mp; - + Eterm *heap; VERBOSE(DEBUG_PROCESSES, ("Removing process: %T\n",p->common.id)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] delete process: %p %p %p %p\n", p->common.id, + HEAP_START(p), HEAP_END(p), OLD_HEAP(p), OLD_HEND(p))); /* Cleanup psd */ @@ -11488,16 +11634,17 @@ delete_process(Process* p) * Release heaps. Clobber contents in DEBUG build. */ - -#ifdef DEBUG - sys_memset(p->heap, DEBUG_BAD_BYTE, p->heap_sz*sizeof(Eterm)); -#endif - #ifdef HIPE hipe_delete_process(&p->hipe); #endif - ERTS_HEAP_FREE(ERTS_ALC_T_HEAP, (void*) p->heap, p->heap_sz*sizeof(Eterm)); + heap = p->abandoned_heap ? p->abandoned_heap : p->heap; + +#ifdef DEBUG + sys_memset(heap, DEBUG_BAD_BYTE, p->heap_sz*sizeof(Eterm)); +#endif + + ERTS_HEAP_FREE(ERTS_ALC_T_HEAP, (void*) heap, p->heap_sz*sizeof(Eterm)); if (p->old_heap != NULL) { #ifdef DEBUG @@ -11516,27 +11663,14 @@ delete_process(Process* p) free_message_buffer(p->mbuf); } + if (p->msg_frag) + erts_cleanup_messages(p->msg_frag); + erts_erase_dicts(p); /* free all pending messages */ - mp = p->msg.first; - while(mp != NULL) { - ErlMessage* next_mp = mp->next; - if (mp->data.attached) { - if (is_value(mp->m[0])) - free_message_buffer(mp->data.heap_frag); - else { - if (is_not_nil(mp->m[1])) { - ErlHeapFragment *heap_frag; - heap_frag = (ErlHeapFragment *) mp->data.dist_ext->ext_endp; - erts_cleanup_offheap(&heap_frag->off_heap); - } - erts_free_dist_ext_copy(mp->data.dist_ext); - } - } - free_message(mp); - mp = next_mp; - } + erts_cleanup_messages(p->msg.first); + p->msg.first = NULL; ASSERT(!p->nodes_monitors); ASSERT(!p->suspend_monitors); @@ -11555,7 +11689,9 @@ set_proc_exiting(Process *p, ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(p) == ERTS_PROC_LOCKS_ALL); enqueue = change_proc_schedule_state(p, - ERTS_PSFLG_SUSPENDED|ERTS_PSFLG_PENDING_EXIT, + (ERTS_PSFLG_SUSPENDED + | ERTS_PSFLG_PENDING_EXIT + | ERTS_PSFLGS_DIRTY_WORK), ERTS_PSFLG_EXITING|ERTS_PSFLG_ACTIVE, &state, &enq_prio, @@ -11589,10 +11725,7 @@ set_proc_exiting(Process *p, } #endif - if (enqueue) - add2runq(enqueue > 0 ? p : make_proxy_proc(NULL, p, enq_prio), - state, - enq_prio); + add2runq(enqueue, enq_prio, p, state, NULL); } static ERTS_INLINE erts_aint32_t @@ -11695,6 +11828,11 @@ save_pending_exiter(Process *p) else rq = esdp->run_queue; +#ifdef ERTS_DIRTY_SCHEDULERS + if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) + rq = ERTS_RUNQ_IX(0); /* Handle on ordinary scheduler */ +#endif + plp = proclist_create(p); erts_smp_runq_lock(rq); @@ -11704,13 +11842,8 @@ save_pending_exiter(Process *p) non_empty_runq(rq); erts_smp_runq_unlock(rq); -#ifdef ERTS_DIRTY_SCHEDULERS - if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) - wake_dirty_schedulers(rq, 0); - else -#endif - wake_scheduler(rq); + wake_scheduler(rq); } #endif @@ -11724,35 +11857,46 @@ static ERTS_INLINE void send_exit_message(Process *to, ErtsProcLocks *to_locksp, Eterm exit_term, Uint term_size, Eterm token) { - if (token == NIL -#ifdef USE_VM_PROBES - || token == am_have_dt_utag -#endif - ) { - Eterm* hp; - Eterm mess; - ErlHeapFragment* bp; - ErlOffHeap *ohp; - - hp = erts_alloc_message_heap(term_size, &bp, &ohp, to, to_locksp); + ErtsMessage *mp; + ErlOffHeap *ohp; + Eterm* hp; + Eterm mess; +#ifdef SHCOPY_SEND + erts_shcopy_t info; +#endif + + if (!have_seqtrace(token)) { +#ifdef SHCOPY_SEND + INITIALIZE_SHCOPY(info); + term_size = copy_shared_calculate(exit_term, &info); + mp = erts_alloc_message_heap(to, to_locksp, term_size, &hp, &ohp); + mess = copy_shared_perform(exit_term, term_size, &info, &hp, ohp); + DESTROY_SHCOPY(info); +#else + mp = erts_alloc_message_heap(to, to_locksp, term_size, &hp, &ohp); mess = copy_struct(exit_term, term_size, &hp, ohp); - erts_queue_message(to, to_locksp, bp, mess, NIL); +#endif + erts_queue_message(to, to_locksp, mp, mess, NIL); } else { - ErlHeapFragment* bp; - Eterm* hp; - Eterm mess; Eterm temp_token; Uint sz_token; ASSERT(is_tuple(token)); sz_token = size_object(token); - bp = new_message_buffer(term_size+sz_token); - hp = bp->mem; - mess = copy_struct(exit_term, term_size, &hp, &bp->off_heap); +#ifdef SHCOPY_SEND + INITIALIZE_SHCOPY(info); + term_size = copy_shared_calculate(exit_term, &info); + mp = erts_alloc_message_heap(to, to_locksp, term_size+sz_token, &hp, &ohp); + mess = copy_shared_perform(exit_term, term_size, &info, &hp, ohp); + DESTROY_SHCOPY(info); +#else + mp = erts_alloc_message_heap(to, to_locksp, term_size+sz_token, &hp, &ohp); + mess = copy_struct(exit_term, term_size, &hp, ohp); +#endif /* the trace token must in this case be updated by the caller */ seq_trace_output(token, mess, SEQ_TRACE_SEND, to->common.id, NULL); - temp_token = copy_struct(token, sz_token, &hp, &bp->off_heap); - erts_queue_message(to, to_locksp, bp, mess, temp_token); + temp_token = copy_struct(token, sz_token, &hp, ohp); + erts_queue_message(to, to_locksp, mp, mess, temp_token); } } @@ -11861,11 +12005,7 @@ send_exit_signal(Process *c_p, /* current process if and only if ((state & ERTS_PSFLG_TRAP_EXIT) && (reason != am_kill || (flags & ERTS_XSIG_FLG_IGN_KILL))) { - if (is_not_nil(token) -#ifdef USE_VM_PROBES - && token != am_have_dt_utag -#endif - && token_update) + if (have_seqtrace(token) && token_update) seq_trace_update_send(token_update); if (is_value(exit_tuple)) send_exit_message(rp, rp_locks, exit_tuple, exit_tuple_sz, token); @@ -12698,7 +12838,7 @@ stack_element_dump(int to, void *to_arg, Eterm* sp, int yreg) } if (is_CP(x)) { - erts_print(to, to_arg, "Return addr %p (", (Eterm *) EXPAND_POINTER(x)); + erts_print(to, to_arg, "Return addr %p (", (Eterm *) x); print_function_from_pc(to, to_arg, cp_val(x)); erts_print(to, to_arg, ")\n"); yreg = 0; diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index ee8a2e1b7b..0c7ad74614 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -304,7 +304,6 @@ typedef enum { ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN_IX, ERTS_SSI_AUX_WORK_MISC_THR_PRGR_IX, ERTS_SSI_AUX_WORK_MISC_IX, - ERTS_SSI_AUX_WORK_CHECK_CHILDREN_IX, ERTS_SSI_AUX_WORK_SET_TMO_IX, ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX, ERTS_SSI_AUX_WORK_REAP_PORTS_IX, @@ -337,8 +336,6 @@ typedef enum { (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MISC_THR_PRGR_IX) #define ERTS_SSI_AUX_WORK_MISC \ (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MISC_IX) -#define ERTS_SSI_AUX_WORK_CHECK_CHILDREN \ - (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_CHECK_CHILDREN_IX) #define ERTS_SSI_AUX_WORK_SET_TMO \ (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_SET_TMO_IX) #define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK \ @@ -643,12 +640,6 @@ struct ErtsSchedulerData_ { Process *free_process; ErtsThrPrgrData thr_progress_data; #endif -#if !HEAP_ON_C_STACK - Eterm tmp_heap[TMP_HEAP_SIZE]; - int num_tmp_heap_used; - Eterm beam_emu_tmp_heap[BEAM_EMU_TMP_HEAP_SIZE]; - Eterm erl_arith_tmp_heap[ERL_ARITH_TMP_HEAP_SIZE]; -#endif ErtsSchedulerSleepInfo *ssi; Process *current_process; Uint no; /* Scheduler number for normal schedulers */ @@ -702,13 +693,6 @@ extern ErtsAlignedSchedulerData *erts_aligned_dirty_io_scheduler_data; extern ErtsSchedulerData *erts_scheduler_data; #endif -#ifdef ERTS_SCHED_FAIR -#define ERTS_SCHED_FAIR_YIELD() ETHR_YIELD() -#else -#define ERTS_SCHED_FAIR 0 -#define ERTS_SCHED_FAIR_YIELD() -#endif - #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) int erts_smp_lc_runq_is_locked(ErtsRunQueue *); #endif @@ -924,7 +908,6 @@ struct ErtsPendingSuspend_ { # define MIN_VHEAP_SIZE(p) (p)->min_vheap_size # define BIN_VHEAP_SZ(p) (p)->bin_vheap_sz -# define BIN_VHEAP_MATURE(p) (p)->bin_vheap_mature # define BIN_OLD_VHEAP_SZ(p) (p)->bin_old_vheap_sz # define BIN_OLD_VHEAP(p) (p)->bin_old_vheap @@ -942,6 +925,7 @@ struct process { Eterm* stop; /* Stack top */ Eterm* heap; /* Heap start */ Eterm* hend; /* Heap end */ + Eterm* abandoned_heap; Uint heap_sz; /* Size of heap in words */ Uint min_heap_size; /* Minimum size of heap (in words). */ Uint min_vheap_size; /* Minimum size of virtual heap (in words). */ @@ -1038,18 +1022,22 @@ struct process { Uint16 gen_gcs; /* Number of (minor) generational GCs. */ Uint16 max_gen_gcs; /* Max minor gen GCs before fullsweep. */ ErlOffHeap off_heap; /* Off-heap data updated by copy_struct(). */ - ErlHeapFragment* mbuf; /* Pointer to message buffer list */ - Uint mbuf_sz; /* Size of all message buffers */ + ErlHeapFragment* mbuf; /* Pointer to heap fragment list */ + ErlHeapFragment* live_hf_end; + ErtsMessage *msg_frag; /* Pointer to message fragment list */ + Uint mbuf_sz; /* Total size of heap fragments and message fragments */ ErtsPSD *psd; /* Rarely used process specific data */ Uint64 bin_vheap_sz; /* Virtual heap block size for binaries */ - Uint64 bin_vheap_mature; /* Virtual heap block size for binaries */ Uint64 bin_old_vheap_sz; /* Virtual old heap block size for binaries */ Uint64 bin_old_vheap; /* Virtual old heap size for binaries */ ErtsProcSysTaskQs *sys_task_qs; erts_smp_atomic32_t state; /* Process state flags (see ERTS_PSFLG_*) */ +#ifdef ERTS_DIRTY_SCHEDULERS + erts_smp_atomic32_t dirty_state; /* Process dirty state flags (see ERTS_PDSFLG_*) */ +#endif #ifdef ERTS_SMP ErlMessageInQueue msg_inq; @@ -1067,6 +1055,7 @@ struct process { #ifdef CHECK_FOR_HOLES Eterm* last_htop; /* No need to scan the heap below this point. */ ErlHeapFragment* last_mbuf; /* No need to scan beyond this mbuf. */ + ErlHeapFragment* heap_hfrag; /* Heap abandoned, htop now lives in this frag */ #endif #ifdef DEBUG @@ -1090,6 +1079,7 @@ extern const Process erts_invalid_process; do { \ (p)->last_htop = 0; \ (p)->last_mbuf = 0; \ + (p)->heap_hfrag = NULL; \ } while (0) # define ERTS_HOLE_CHECK(p) erts_check_for_holes((p)) @@ -1167,15 +1157,16 @@ void erts_check_for_holes(Process* p); #define ERTS_PSFLG_RUNNING_SYS ERTS_PSFLG_BIT(15) #define ERTS_PSFLG_PROXY ERTS_PSFLG_BIT(16) #define ERTS_PSFLG_DELAYED_SYS ERTS_PSFLG_BIT(17) -#ifdef ERTS_DIRTY_SCHEDULERS -#define ERTS_PSFLG_DIRTY_CPU_PROC ERTS_PSFLG_BIT(18) -#define ERTS_PSFLG_DIRTY_IO_PROC ERTS_PSFLG_BIT(19) -#define ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q ERTS_PSFLG_BIT(20) -#define ERTS_PSFLG_DIRTY_IO_PROC_IN_Q ERTS_PSFLG_BIT(21) +#define ERTS_PSFLG_OFF_HEAP_MSGQ ERTS_PSFLG_BIT(18) +#define ERTS_PSFLG_ON_HEAP_MSGQ ERTS_PSFLG_BIT(19) +#define ERTS_PSFLG_DIRTY_CPU_PROC ERTS_PSFLG_BIT(20) +#define ERTS_PSFLG_DIRTY_IO_PROC ERTS_PSFLG_BIT(21) +#define ERTS_PSFLG_DIRTY_ACTIVE_SYS ERTS_PSFLG_BIT(22) #define ERTS_PSFLG_MAX (ERTS_PSFLGS_ZERO_BIT_OFFSET + 22) -#else -#define ERTS_PSFLG_MAX (ERTS_PSFLGS_ZERO_BIT_OFFSET + 18) -#endif + +#define ERTS_PSFLGS_DIRTY_WORK (ERTS_PSFLG_DIRTY_CPU_PROC \ + | ERTS_PSFLG_DIRTY_IO_PROC \ + | ERTS_PSFLG_DIRTY_ACTIVE_SYS) #define ERTS_PSFLGS_IN_PRQ_MASK (ERTS_PSFLG_IN_PRQ_MAX \ | ERTS_PSFLG_IN_PRQ_HIGH \ @@ -1189,6 +1180,36 @@ void erts_check_for_holes(Process* p); #define ERTS_PSFLGS_GET_PRQ_PRIO(PSFLGS) \ (((PSFLGS) >> ERTS_PSFLGS_PRQ_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK) +#ifdef ERTS_DIRTY_SCHEDULERS + +/* + * Flags in the dirty_state field. + */ + +#define ERTS_PDSFLG_IN_CPU_PRQ_MAX (((erts_aint32_t) 1) << 0) +#define ERTS_PDSFLG_IN_CPU_PRQ_HIGH (((erts_aint32_t) 1) << 1) +#define ERTS_PDSFLG_IN_CPU_PRQ_NORMAL (((erts_aint32_t) 1) << 2) +#define ERTS_PDSFLG_IN_CPU_PRQ_LOW (((erts_aint32_t) 1) << 3) +#define ERTS_PDSFLG_IN_IO_PRQ_MAX (((erts_aint32_t) 1) << 4) +#define ERTS_PDSFLG_IN_IO_PRQ_HIGH (((erts_aint32_t) 1) << 5) +#define ERTS_PDSFLG_IN_IO_PRQ_NORMAL (((erts_aint32_t) 1) << 6) +#define ERTS_PDSFLG_IN_IO_PRQ_LOW (((erts_aint32_t) 1) << 7) + +#define ERTS_PDSFLGS_QMASK ERTS_PSFLGS_QMASK +#define ERTS_PDSFLGS_IN_CPU_PRQ_MASK_OFFSET 0 +#define ERTS_PDSFLGS_IN_IO_PRQ_MASK_OFFSET ERTS_PSFLGS_QMASK_BITS + +#define ERTS_PDSFLG_IN_CPU_PRQ_MASK (ERTS_PDSFLG_IN_CPU_PRQ_MAX \ + | ERTS_PDSFLG_IN_CPU_PRQ_HIGH \ + | ERTS_PDSFLG_IN_CPU_PRQ_NORMAL\ + | ERTS_PDSFLG_IN_CPU_PRQ_LOW) +#define ERTS_PDSFLG_IN_IO_PRQ_MASK (ERTS_PDSFLG_IN_CPU_PRQ_MAX \ + | ERTS_PDSFLG_IN_CPU_PRQ_HIGH \ + | ERTS_PDSFLG_IN_CPU_PRQ_NORMAL\ + | ERTS_PDSFLG_IN_CPU_PRQ_LOW) +#endif + + /* * Static flags that do not change after process creation. */ @@ -1222,12 +1243,16 @@ void erts_check_for_holes(Process* p); #define SPO_USE_ARGS 2 #define SPO_MONITOR 4 #define SPO_SYSTEM_PROC 8 +#define SPO_OFF_HEAP_MSGQ 16 +#define SPO_ON_HEAP_MSGQ 32 + +extern int erts_default_spo_flags; /* * The following struct contains options for a process to be spawned. */ typedef struct { - Uint flags; + int flags; int error_code; /* Error code returned from create_process(). */ Eterm mref; /* Monitor ref returned (if SPO_MONITOR was given). */ @@ -1267,7 +1292,6 @@ Eterm* erts_heap_alloc(Process* p, Uint need, Uint xtra); Eterm* erts_set_hole_marker(Eterm* ptr, Uint sz); #endif -extern Uint erts_default_process_flags; extern erts_smp_rwmtx_t erts_cpu_bind_rwmtx; /* If any of the erts_system_monitor_* variables are set (enabled), ** erts_system_monitor must be != NIL, to allow testing on just @@ -1309,7 +1333,23 @@ extern int erts_system_profile_ts_type; #define F_HAVE_BLCKD_MSCHED (1 << 8) /* Process has blocked multi-scheduling */ #define F_P2PNR_RESCHED (1 << 9) /* Process has been rescheduled via erts_pid2proc_not_running() */ #define F_FORCE_GC (1 << 10) /* Force gc at process in-scheduling */ -#define F_DISABLE_GC (1 << 11) /* Disable GC */ +#define F_DISABLE_GC (1 << 11) /* Disable GC (see below) */ +#define F_OFF_HEAP_MSGQ (1 << 12) /* Off heap msg queue */ +#define F_ON_HEAP_MSGQ (1 << 13) /* Off heap msg queue */ +#define F_OFF_HEAP_MSGQ_CHNG (1 << 14) /* Off heap msg queue changing */ +#define F_ABANDONED_HEAP_USE (1 << 15) /* Have usage of abandoned heap */ +#define F_DELAY_GC (1 << 16) /* Similar to disable GC (see below) */ + +/* + * F_DISABLE_GC and F_DELAY_GC are similar. Both will prevent + * GC of the process, but it is important to use the right + * one: + * - F_DISABLE_GC should *only* be used by BIFs. This when + * the BIF needs to yield while preventig a GC. + * - F_DELAY_GC should only be used when GC is temporarily + * disabled while the process is scheduled. A process must + * not be scheduled out while F_DELAY_GC is set. + */ #define ERTS_TRACE_FLAGS_TS_TYPE_SHIFT 0 @@ -1673,6 +1713,7 @@ void erts_schedule_thr_prgr_later_cleanup_op(void (*)(void *), void *, ErtsThrPrgrLaterOp *, UWord); +void erts_schedule_complete_off_heap_message_queue_change(Eterm pid); #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) int erts_dbg_check_halloc_lock(Process *p); @@ -1702,11 +1743,8 @@ Eterm erts_multi_scheduling_blockers(Process *); void erts_start_schedulers(void); void erts_alloc_notify_delayed_dealloc(int); void erts_alloc_ensure_handle_delayed_dealloc_call(int); -#ifdef ERTS_SMP void erts_notify_canceled_timer(ErtsSchedulerData *, int); #endif -void erts_smp_notify_check_children_needed(void); -#endif #if ERTS_USE_ASYNC_READY_Q void erts_notify_check_async_ready_queue(void *); #endif @@ -1800,7 +1838,7 @@ Uint erts_debug_nbalance(void); int erts_debug_wait_completed(Process *c_p, int flags); -Uint erts_process_memory(Process *c_p); +Uint erts_process_memory(Process *c_p, int incl_msg_inq); #ifdef ERTS_SMP # define ERTS_GET_SCHEDULER_DATA_FROM_PROC(PROC) ((PROC)->scheduler_data) @@ -2110,6 +2148,22 @@ ERTS_GLB_INLINE void erts_smp_xrunq_unlock(ErtsRunQueue *rq, ErtsRunQueue *xrq); ERTS_GLB_INLINE void erts_smp_runqs_lock(ErtsRunQueue *rq1, ErtsRunQueue *rq2); ERTS_GLB_INLINE void erts_smp_runqs_unlock(ErtsRunQueue *rq1, ErtsRunQueue *rq2); +ERTS_GLB_INLINE ErtsMessage *erts_alloc_message_heap_state(Process *pp, + erts_aint32_t *psp, + ErtsProcLocks *plp, + Uint sz, + Eterm **hpp, + ErlOffHeap **ohpp); +ERTS_GLB_INLINE ErtsMessage *erts_alloc_message_heap(Process *pp, + ErtsProcLocks *plp, + Uint sz, + Eterm **hpp, + ErlOffHeap **ohpp); + +ERTS_GLB_INLINE void erts_shrink_message_heap(ErtsMessage **msgpp, Process *pp, + Eterm *start_hp, Eterm *used_hp, Eterm *end_hp, + Eterm *brefs, Uint brefs_size); + #if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_INLINE @@ -2258,6 +2312,63 @@ erts_smp_runqs_unlock(ErtsRunQueue *rq1, ErtsRunQueue *rq2) #endif } +ERTS_GLB_INLINE ErtsMessage * +erts_alloc_message_heap_state(Process *pp, + erts_aint32_t *psp, + ErtsProcLocks *plp, + Uint sz, + Eterm **hpp, + ErlOffHeap **ohpp) +{ + int on_heap; + + if ((*psp) & ERTS_PSFLG_OFF_HEAP_MSGQ) { + ErtsMessage *mp = erts_alloc_message(sz, hpp); + *ohpp = sz == 0 ? NULL : &mp->hfrag.off_heap; + return mp; + } + + return erts_try_alloc_message_on_heap(pp, psp, plp, sz, hpp, ohpp, &on_heap); +} + +ERTS_GLB_INLINE ErtsMessage * +erts_alloc_message_heap(Process *pp, + ErtsProcLocks *plp, + Uint sz, + Eterm **hpp, + ErlOffHeap **ohpp) +{ + erts_aint32_t state = erts_smp_atomic32_read_nob(&pp->state); + return erts_alloc_message_heap_state(pp, &state, plp, sz, hpp, ohpp); +} + +ERTS_GLB_INLINE void +erts_shrink_message_heap(ErtsMessage **msgpp, Process *pp, + Eterm *start_hp, Eterm *used_hp, Eterm *end_hp, + Eterm *brefs, Uint brefs_size) +{ + ASSERT(start_hp <= used_hp && used_hp <= end_hp); + if ((*msgpp)->data.attached == ERTS_MSG_COMBINED_HFRAG) + *msgpp = erts_shrink_message(*msgpp, used_hp - start_hp, + brefs, brefs_size); + else if (!(*msgpp)->data.attached) { + ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN + & erts_proc_lc_my_proc_locks(pp)); + HRelease(pp, end_hp, used_hp); + } + else { + ErlHeapFragment *hfrag = (*msgpp)->data.heap_frag; + if (start_hp != used_hp) + hfrag = erts_resize_message_buffer(hfrag, used_hp - start_hp, + brefs, brefs_size); + else { + free_message_buffer(hfrag); + hfrag = NULL; + } + (*msgpp)->data.heap_frag = hfrag; + } +} + #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ ERTS_GLB_INLINE ErtsAtomCacheMap *erts_get_atom_cache_map(Process *c_p); diff --git a/erts/emulator/beam/erl_process_dict.c b/erts/emulator/beam/erl_process_dict.c index 8606371bdf..36d16f7f42 100644 --- a/erts/emulator/beam/erl_process_dict.c +++ b/erts/emulator/beam/erl_process_dict.c @@ -51,16 +51,19 @@ #define INITIAL_SIZE (erts_pd_initial_size) /* Hash utility macros */ -#define HASH_RANGE(PDict) ((PDict)->homeSize + (PDict)->splitPosition) +#define HASH_RANGE(PDict) ((PDict)->usedSlots) -#define MAKE_HASH(Term) \ -((is_small(Term)) ? unsigned_val(Term) : \ - ((is_atom(Term)) ? \ - (atom_tab(atom_val(term))->slot.bucket.hvalue) : \ - make_hash2(Term))) +#define MAKE_HASH(Term) \ + ((is_small(Term)) ? unsigned_val(Term) : \ + ((is_atom(Term)) ? \ + (atom_tab(atom_val(Term))->slot.bucket.hvalue) : \ + make_internal_hash(Term))) #define PD_SZ2BYTES(Sz) (sizeof(ProcDict) + ((Sz) - 1)*sizeof(Eterm)) +#define pd_hash_value(Pdict, Key) \ + pd_hash_value_to_ix(Pdict, MAKE_HASH((Key))) + /* Memory allocation macros */ #define PD_ALLOC(Sz) \ erts_alloc(ERTS_ALC_T_PROC_DICT, (Sz)) @@ -74,14 +77,19 @@ #define TCDR(Term) CDR(list_val(Term)) /* Array access macro */ -#define ARRAY_GET(PDict, Index) (((PDict)->size > (Index)) ? \ - (PDict)->data[Index] : NIL) +#define ARRAY_GET(PDict, Index) (ASSERT((Index) < (PDict)->arraySize), \ + (PDict)->data[Index]) +#define ARRAY_PUT(PDict, Index, Val) (ASSERT((Index) < (PDict)->arraySize), \ + (PDict)->data[Index] = (Val)) + +#define IS_POW2(X) ((X) && !((X) & ((X)-1))) /* * Forward decalarations */ static void pd_hash_erase(Process *p, Eterm id, Eterm *ret); static void pd_hash_erase_all(Process *p); +static Eterm pd_hash_get_with_hval(Process *p, Eterm bucket, Eterm id); static Eterm pd_hash_get_keys(Process *p, Eterm value); static Eterm pd_hash_get_all_keys(Process *p, ProcDict *pd); static Eterm pd_hash_get_all(Process *p, ProcDict *pd); @@ -91,9 +99,9 @@ static void shrink(Process *p, Eterm* ret); static void grow(Process *p); static void array_shrink(ProcDict **ppd, unsigned int need); -static Eterm array_put(ProcDict **ppdict, unsigned int ndx, Eterm term); +static void ensure_array_size(ProcDict**, unsigned int size); -static unsigned int pd_hash_value(ProcDict *pdict, Eterm term); +static unsigned int pd_hash_value_to_ix(ProcDict *pdict, Uint32 hx); static unsigned int next_array_size(unsigned int need); /* @@ -134,6 +142,16 @@ static void pd_check(ProcDict *pd); ** External interface */ +int +erts_pd_set_initial_size(int size) +{ + if (size <= 0) + return 0; + + erts_pd_initial_size = 1 << erts_fit_in_bits_uint(size-1); + return 1; +} + /* * Called from break handler */ @@ -146,9 +164,9 @@ erts_dictionary_dump(int to, void *to_arg, ProcDict *pd) /*PD_CHECK(pd);*/ if (pd == NULL) return; - erts_print(to, to_arg, "(size = %d, used = %d, homeSize = %d, " + erts_print(to, to_arg, "(size = %d, usedSlots = %d, " "splitPosition = %d, numElements = %d)\n", - pd->size, pd->used, pd->homeSize, + pd->arraySize, pd->usedSlots, pd->splitPosition, (unsigned int) pd->numElements); for (i = 0; i < HASH_RANGE(pd); ++i) { erts_print(to, to_arg, "%d: %T\n", i, ARRAY_GET(pd, i)); @@ -203,7 +221,7 @@ erts_dicts_mem_size(Process *p) { Uint size = 0; if (p->dictionary) - size += PD_SZ2BYTES(p->dictionary->size); + size += PD_SZ2BYTES(p->dictionary->arraySize); return size; } @@ -345,7 +363,7 @@ static void pd_hash_erase(Process *p, Eterm id, Eterm *ret) if (is_boxed(old)) { /* Tuple */ ASSERT(is_tuple(old)); if (EQ(tuple_val(old)[1], id)) { - array_put(&(p->dictionary), hval, NIL); + ARRAY_PUT(p->dictionary, hval, NIL); --(p->dictionary->numElements); *ret = tuple_val(old)[2]; } @@ -365,7 +383,7 @@ static void pd_hash_erase(Process *p, Eterm id, Eterm *ret) old = ARRAY_GET(p->dictionary, hval); ASSERT(is_list(old)); if (is_nil(TCDR(old))) { - array_put(&p->dictionary, hval, TCAR(old)); + ARRAY_PUT(p->dictionary, hval, TCAR(old)); } } else if (is_not_nil(old)) { #ifdef DEBUG @@ -390,40 +408,56 @@ static void pd_hash_erase_all(Process *p) } } +Eterm erts_pd_hash_get_with_hx(Process *p, Uint32 hx, Eterm id) +{ + unsigned int hval; + ProcDict *pd = p->dictionary; + + ASSERT(hx == MAKE_HASH(id)); + if (pd == NULL) + return am_undefined; + hval = pd_hash_value_to_ix(pd, hx); + return pd_hash_get_with_hval(p, ARRAY_GET(pd, hval), id); +} + Eterm erts_pd_hash_get(Process *p, Eterm id) { unsigned int hval; - Eterm tmp; ProcDict *pd = p->dictionary; if (pd == NULL) return am_undefined; hval = pd_hash_value(pd, id); - tmp = ARRAY_GET(pd, hval); - if (is_boxed(tmp)) { /* Tuple */ - ASSERT(is_tuple(tmp)); - if (EQ(tuple_val(tmp)[1], id)) { - return tuple_val(tmp)[2]; + return pd_hash_get_with_hval(p, ARRAY_GET(pd, hval), id); +} + +Eterm pd_hash_get_with_hval(Process *p, Eterm bucket, Eterm id) +{ + if (is_boxed(bucket)) { /* Tuple */ + ASSERT(is_tuple(bucket)); + if (EQ(tuple_val(bucket)[1], id)) { + return tuple_val(bucket)[2]; } - } else if (is_list(tmp)) { - for (; tmp != NIL && !EQ(tuple_val(TCAR(tmp))[1], id); tmp = TCDR(tmp)) { + } else if (is_list(bucket)) { + for (; bucket != NIL && !EQ(tuple_val(TCAR(bucket))[1], id); bucket = TCDR(bucket)) { ; } - if (tmp != NIL) { - return tuple_val(TCAR(tmp))[2]; + if (bucket != NIL) { + return tuple_val(TCAR(bucket))[2]; } - } else if (is_not_nil(tmp)) { + } else if (is_not_nil(bucket)) { #ifdef DEBUG erts_fprintf(stderr, "Process dictionary for process %T is broken, trying to " "display term found in line %d:\n" - "%T\n", p->common.id, __LINE__, tmp); + "%T\n", p->common.id, __LINE__, bucket); #endif erl_exit(1, "Damaged process dictionary found during get/1."); } return am_undefined; } + #define PD_GET_TKEY(Dst,Src) \ do { \ ASSERT(is_tuple((Src))); \ @@ -549,8 +583,11 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value) if (p->dictionary == NULL) { /* Create it */ - array_put(&(p->dictionary), INITIAL_SIZE - 1, NIL); - p->dictionary->homeSize = INITIAL_SIZE; + ensure_array_size(&p->dictionary, INITIAL_SIZE); + p->dictionary->usedSlots = INITIAL_SIZE; + p->dictionary->sizeMask = INITIAL_SIZE*2 - 1; + p->dictionary->splitPosition = 0; + p->dictionary->numElements = 0; } hval = pd_hash_value(p->dictionary, id); old = ARRAY_GET(p->dictionary, hval); @@ -583,7 +620,7 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value) root[0] = id; root[1] = value; root[2] = old; - BUMP_REDS(p, erts_garbage_collect(p, needed, root, 3)); + erts_garbage_collect(p, needed, root, 3); id = root[0]; value = root[1]; old = root[2]; @@ -602,19 +639,19 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value) * Update the dictionary. */ if (is_nil(old)) { - array_put(&(p->dictionary), hval, tpl); + ARRAY_PUT(p->dictionary, hval, tpl); ++(p->dictionary->numElements); } else if (is_boxed(old)) { ASSERT(is_tuple(old)); if (EQ(tuple_val(old)[1],id)) { - array_put(&(p->dictionary), hval, tpl); + ARRAY_PUT(p->dictionary, hval, tpl); return tuple_val(old)[2]; } else { hp = HeapOnlyAlloc(p, 4); tmp = CONS(hp, old, NIL); hp += 2; ++(p->dictionary->numElements); - array_put(&(p->dictionary), hval, CONS(hp, tpl, tmp)); + ARRAY_PUT(p->dictionary, hval, CONS(hp, tpl, tmp)); hp += 2; ASSERT(hp <= hp_limit); } @@ -624,7 +661,7 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value) * New key. Simply prepend the tuple to the beginning of the list. */ hp = HeapOnlyAlloc(p, 2); - array_put(&(p->dictionary), hval, CONS(hp, tpl, old)); + ARRAY_PUT(p->dictionary, hval, CONS(hp, tpl, old)); hp += 2; ASSERT(hp <= hp_limit); ++(p->dictionary->numElements); @@ -659,7 +696,7 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value) nlist = CONS(hp, tpl, nlist); hp += 2; ASSERT(hp <= hp_limit); - array_put(&(p->dictionary), hval, nlist); + ARRAY_PUT(p->dictionary, hval, nlist); return tuple_val(TCAR(tmp))[2]; } } else { @@ -684,6 +721,7 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value) static void shrink(Process *p, Eterm* ret) { + ProcDict *pd = p->dictionary; unsigned int range = HASH_RANGE(p->dictionary); unsigned int steps = (range*3) / 10; Eterm hi, lo, tmp; @@ -698,25 +736,26 @@ static void shrink(Process *p, Eterm* ret) } for (i = 0; i < steps; ++i) { - ProcDict *pd = p->dictionary; if (pd->splitPosition == 0) { - pd->homeSize /= 2; - pd->splitPosition = pd->homeSize; + ASSERT(IS_POW2(pd->usedSlots)); + pd->sizeMask = pd->usedSlots - 1; + pd->splitPosition = pd->usedSlots / 2; } --(pd->splitPosition); - hi = ARRAY_GET(pd, (pd->splitPosition + pd->homeSize)); + /* Must wait to decrement 'usedSlots' for GC rootset below */ + hi = ARRAY_GET(pd, pd->usedSlots - 1); lo = ARRAY_GET(pd, pd->splitPosition); if (hi != NIL) { if (lo == NIL) { - array_put(&(p->dictionary), pd->splitPosition, hi); + ARRAY_PUT(pd, pd->splitPosition, hi); } else { - int needed = 4; + Sint needed = 4; if (is_list(hi) && is_list(lo)) { needed = 2*erts_list_length(hi); } if (HeapWordsLeft(p) < needed) { - BUMP_REDS(p, erts_garbage_collect(p, needed, ret, 1)); - hi = pd->data[(pd->splitPosition + pd->homeSize)]; + erts_garbage_collect(p, needed, ret, 1); + hi = pd->data[pd->usedSlots - 1]; lo = pd->data[pd->splitPosition]; } #ifdef DEBUG @@ -727,13 +766,13 @@ static void shrink(Process *p, Eterm* ret) hp = HeapOnlyAlloc(p, 4); tmp = CONS(hp, hi, NIL); hp += 2; - array_put(&(p->dictionary), pd->splitPosition, + ARRAY_PUT(pd, pd->splitPosition, CONS(hp,lo,tmp)); hp += 2; ASSERT(hp <= hp_limit); } else { /* hi is a list */ hp = HeapOnlyAlloc(p, 2); - array_put(&(p->dictionary), pd->splitPosition, + ARRAY_PUT(pd, pd->splitPosition, CONS(hp, lo, hi)); hp += 2; ASSERT(hp <= hp_limit); @@ -741,7 +780,7 @@ static void shrink(Process *p, Eterm* ret) } else { /* lo is a list */ if (is_tuple(hi)) { hp = HeapOnlyAlloc(p, 2); - array_put(&(p->dictionary), pd->splitPosition, + ARRAY_PUT(pd, pd->splitPosition, CONS(hp, hi, lo)); hp += 2; ASSERT(hp <= hp_limit); @@ -753,14 +792,15 @@ static void shrink(Process *p, Eterm* ret) hp += 2; } ASSERT(hp <= hp_limit); - array_put(&(p->dictionary), pd->splitPosition, lo); + ARRAY_PUT(pd, pd->splitPosition, lo); } } } } - array_put(&(p->dictionary), (pd->splitPosition + pd->homeSize), NIL); + --pd->usedSlots; + ARRAY_PUT(pd, pd->usedSlots, NIL); } - if (HASH_RANGE(p->dictionary) <= (p->dictionary->size / 4)) { + if (HASH_RANGE(p->dictionary) <= (p->dictionary->arraySize / 4)) { array_shrink(&(p->dictionary), (HASH_RANGE(p->dictionary) * 3) / 2); } } @@ -768,14 +808,14 @@ static void shrink(Process *p, Eterm* ret) static void grow(Process *p) { unsigned int i,j; - unsigned int steps = p->dictionary->homeSize / 5; + unsigned int steps = (p->dictionary->usedSlots / 4) & 0xf; Eterm l1,l2; Eterm l; Eterm *hp; unsigned int pos; unsigned int homeSize; - int needed = 0; - ProcDict *pd; + Sint needed = 0; + ProcDict *pd = p->dictionary; #ifdef DEBUG Eterm *hp_limit; #endif @@ -784,18 +824,20 @@ static void grow(Process *p) if (steps == 0) steps = 1; /* Dont grow over MAX_HASH */ - if ((MAX_HASH - steps) <= HASH_RANGE(p->dictionary)) { + if ((MAX_HASH - steps) <= HASH_RANGE(pd)) { return; } + ensure_array_size(&p->dictionary, HASH_RANGE(pd) + steps); + pd = p->dictionary; + /* * Calculate total number of heap words needed, and garbage collect * if necessary. */ - pd = p->dictionary; pos = pd->splitPosition; - homeSize = pd->homeSize; + homeSize = pd->usedSlots - pd->splitPosition; for (i = 0; i < steps; ++i) { if (pos == homeSize) { homeSize *= 2; @@ -811,7 +853,7 @@ static void grow(Process *p) } } if (HeapWordsLeft(p) < needed) { - BUMP_REDS(p, erts_garbage_collect(p, needed, 0, 0)); + erts_garbage_collect(p, needed, 0, 0); } #ifdef DEBUG hp_limit = p->htop + needed; @@ -820,21 +862,22 @@ static void grow(Process *p) /* * Now grow. */ - + homeSize = pd->usedSlots - pd->splitPosition; for (i = 0; i < steps; ++i) { - ProcDict *pd = p->dictionary; - if (pd->splitPosition == pd->homeSize) { - pd->homeSize *= 2; - pd->splitPosition = 0; + if (pd->splitPosition == homeSize) { + homeSize *= 2; + pd->sizeMask = homeSize*2 - 1; + pd->splitPosition = 0; } pos = pd->splitPosition; ++pd->splitPosition; /* For the hashes */ + ++pd->usedSlots; + ASSERT(pos + homeSize == pd->usedSlots - 1); l = ARRAY_GET(pd, pos); if (is_tuple(l)) { if (pd_hash_value(pd, tuple_val(l)[1]) != pos) { - array_put(&(p->dictionary), pos + - p->dictionary->homeSize, l); - array_put(&(p->dictionary), pos, NIL); + ARRAY_PUT(pd, pos + homeSize, l); + ARRAY_PUT(pd, pos, NIL); } } else { l2 = NIL; @@ -856,10 +899,8 @@ static void grow(Process *p) if (l2 != NIL && TCDR(l2) == NIL) l2 = TCAR(l2); ASSERT(hp <= hp_limit); - /* After array_put pd is no longer valid */ - array_put(&(p->dictionary), pos, l1); - array_put(&(p->dictionary), pos + - p->dictionary->homeSize, l2); + ARRAY_PUT(pd, pos, l1); + ARRAY_PUT(pd, pos + homeSize, l2); } } @@ -876,73 +917,65 @@ static void array_shrink(ProcDict **ppd, unsigned int need) { unsigned int siz = next_array_size(need); - HDEBUGF(("array_shrink: size = %d, used = %d, need = %d", - (*ppd)->size, (*ppd)->used, need)); + HDEBUGF(("array_shrink: size = %d, need = %d", + (*ppd)->arraySize, need)); - if (siz > (*ppd)->size) + if (siz >= (*ppd)->arraySize) return; /* Only shrink */ *ppd = PD_REALLOC(((void *) *ppd), - PD_SZ2BYTES((*ppd)->size), + PD_SZ2BYTES((*ppd)->arraySize), PD_SZ2BYTES(siz)); - (*ppd)->size = siz; - if ((*ppd)->size < (*ppd)->used) - (*ppd)->used = (*ppd)->size; + (*ppd)->arraySize = siz; } -static Eterm array_put(ProcDict **ppdict, unsigned int ndx, Eterm term) +static void ensure_array_size(ProcDict **ppdict, unsigned int size) { + ProcDict *pd = *ppdict; unsigned int i; - Eterm ret; - if (*ppdict == NULL) { - Uint siz = next_array_size(ndx+1); - ProcDict *p; - p = PD_ALLOC(PD_SZ2BYTES(siz)); + if (pd == NULL) { + Uint siz = next_array_size(size); + + pd = PD_ALLOC(PD_SZ2BYTES(siz)); for (i = 0; i < siz; ++i) - p->data[i] = NIL; - p->size = siz; - p->homeSize = p->splitPosition = p->numElements = p->used = 0; - *ppdict = p; - } else if (ndx >= (*ppdict)->size) { - Uint osize = (*ppdict)->size; - Uint nsize = next_array_size(ndx+1); - *ppdict = PD_REALLOC(((void *) *ppdict), + pd->data[i] = NIL; + pd->arraySize = siz; + *ppdict = pd; + } else if (size > pd->arraySize) { + Uint osize = pd->arraySize; + Uint nsize = next_array_size(size); + pd = PD_REALLOC(((void *) pd), PD_SZ2BYTES(osize), PD_SZ2BYTES(nsize)); for (i = osize; i < nsize; ++i) - (*ppdict)->data[i] = NIL; - (*ppdict)->size = nsize; + pd->data[i] = NIL; + pd->arraySize = nsize; + *ppdict = pd; } - ret = (*ppdict)->data[ndx]; - (*ppdict)->data[ndx] = term; - if ((ndx + 1) > (*ppdict)->used) - (*ppdict)->used = ndx + 1; -#ifdef HARDDEBUG - HDEBUGF(("array_put: (*ppdict)->size = %d, (*ppdict)->used = %d, ndx = %d", - (*ppdict)->size, (*ppdict)->used, ndx)); - erts_fprintf(stderr, "%T", term); -#endif /* HARDDEBUG */ - return ret; } /* ** Basic utilities */ -static unsigned int pd_hash_value(ProcDict *pdict, Eterm term) +static unsigned int pd_hash_value_to_ix(ProcDict *pdict, Uint32 hx) { - Uint hash, high; + Uint high; - hash = MAKE_HASH(term); - high = hash % (pdict->homeSize*2); + ASSERT(IS_POW2(pdict->sizeMask+1)); + ASSERT(HASH_RANGE(pdict) >= (pdict->sizeMask >> 1)); + ASSERT(HASH_RANGE(pdict) <= (pdict->sizeMask + 1)); + + high = hx & pdict->sizeMask; if (high >= HASH_RANGE(pdict)) - return hash % pdict->homeSize; + return hx & (pdict->sizeMask >> 1); return high; } + static unsigned int next_array_size(unsigned int need) { static unsigned int tab[] = @@ -1002,24 +1035,28 @@ static unsigned int next_array_size(unsigned int need) static void pd_check(ProcDict *pd) { unsigned int i; + unsigned int used; Uint num; if (pd == NULL) return; - ASSERT(pd->size >= pd->used); + used = HASH_RANGE(pd); + ASSERT(pd->arraySize >= used); ASSERT(HASH_RANGE(pd) <= MAX_HASH); - for (i = 0, num = 0; i < pd->used; ++i) { + for (i = 0, num = 0; i < used; ++i) { Eterm t = pd->data[i]; if (is_nil(t)) { continue; } else if (is_tuple(t)) { ++num; ASSERT(arityval(*tuple_val(t)) == 2); + ASSERT(pd_hash_value(pd, tuple_val(t)[1]) == i); continue; } else if (is_list(t)) { while (t != NIL) { ++num; ASSERT(is_tuple(TCAR(t))); ASSERT(arityval(*(tuple_val(TCAR(t)))) == 2); + ASSERT(pd_hash_value(pd, tuple_val(TCAR(t))[1]) == i); t = TCDR(t); } continue; @@ -1030,7 +1067,7 @@ static void pd_check(ProcDict *pd) } } ASSERT(num == pd->numElements); - ASSERT(pd->splitPosition <= pd->homeSize); + ASSERT(pd->usedSlots >= pd->splitPosition*2); } #endif /* DEBUG */ diff --git a/erts/emulator/beam/erl_process_dict.h b/erts/emulator/beam/erl_process_dict.h index cc53800eb5..dac214c8a1 100644 --- a/erts/emulator/beam/erl_process_dict.h +++ b/erts/emulator/beam/erl_process_dict.h @@ -23,14 +23,18 @@ #include "sys.h" typedef struct proc_dict { - unsigned int size; - unsigned int used; - unsigned int homeSize; + unsigned int sizeMask; + unsigned int usedSlots; + unsigned int arraySize; unsigned int splitPosition; Uint numElements; Eterm data[1]; /* The beginning of an array of erlang terms */ } ProcDict; +#define ERTS_PD_START(PD) ((PD)->data) +#define ERTS_PD_SIZE(PD) ((PD)->usedSlots) + +int erts_pd_set_initial_size(int size); Uint erts_dicts_mem_size(struct process *p); void erts_erase_dicts(struct process *p); void erts_dictionary_dump(int to, void *to_arg, ProcDict *pd); @@ -39,5 +43,6 @@ void erts_deep_dictionary_dump(int to, void *to_arg, Eterm erts_dictionary_copy(struct process *p, ProcDict *pd); Eterm erts_pd_hash_get(struct process *p, Eterm id); +Eterm erts_pd_hash_get_with_hx(Process *p, Uint32 hx, Eterm id); #endif diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c index 25f0b1ed38..39c36ee7a9 100644 --- a/erts/emulator/beam/erl_process_dump.c +++ b/erts/emulator/beam/erl_process_dump.c @@ -78,13 +78,14 @@ erts_deep_process_dump(int to, void *to_arg) dump_binaries(to, to_arg, all_binaries); } -Uint erts_process_memory(Process *p) { - ErlMessage *mp; +Uint erts_process_memory(Process *p, int incl_msg_inq) { + ErtsMessage *mp; Uint size = 0; struct saved_calls *scb; size += sizeof(Process); - ERTS_SMP_MSGQ_MV_INQ2PRIVQ(p); + if (incl_msg_inq) + ERTS_SMP_MSGQ_MV_INQ2PRIVQ(p); erts_doforall_links(ERTS_P_LINKS(p), &erts_one_link_size, &size); erts_doforall_monitors(ERTS_P_MONITORS(p), &erts_one_mon_size, &size); @@ -92,7 +93,7 @@ Uint erts_process_memory(Process *p) { if (p->old_hend && p->old_heap) size += (p->old_hend - p->old_heap) * sizeof(Eterm); - size += p->msg.len * sizeof(ErlMessage); + size += p->msg.len * sizeof(ErtsMessage); for (mp = p->msg.first; mp; mp = mp->next) if (mp->data.attached) @@ -119,7 +120,7 @@ static void dump_process_info(int to, void *to_arg, Process *p) { Eterm* sp; - ErlMessage* mp; + ErtsMessage* mp; int yreg = -1; ERTS_SMP_MSGQ_MV_INQ2PRIVQ(p); @@ -365,7 +366,7 @@ heap_dump(int to, void *to_arg, Eterm x) while (x != OUR_NIL) { if (is_CP(x)) { - next = (Eterm *) EXPAND_POINTER(x); + next = (Eterm *) x; } else if (is_list(x)) { ptr = list_val(x); if (ptr[0] != OUR_NIL) { @@ -378,7 +379,7 @@ heap_dump(int to, void *to_arg, Eterm x) ptr[1] = make_small(0); } x = ptr[0]; - ptr[0] = (Eterm) COMPRESS_POINTER(next); + ptr[0] = (Eterm) next; next = ptr + 1; continue; } @@ -408,7 +409,7 @@ heap_dump(int to, void *to_arg, Eterm x) ptr[0] = OUR_NIL; } else { x = ptr[arity]; - ptr[0] = (Eterm) COMPRESS_POINTER(next); + ptr[0] = (Eterm) next; next = ptr + arity - 1; continue; } @@ -617,7 +618,7 @@ erts_dump_extended_process_state(int to, void *to_arg, erts_aint32_t psflg) { if (psflg) erts_print(to, to_arg, " | "); - for (i = 0; i < ERTS_PSFLG_MAX && psflg; i++) { + for (i = 0; i <= ERTS_PSFLG_MAX && psflg; i++) { erts_aint32_t chk = (1 << i); if (psflg & chk) { switch (chk) { @@ -657,16 +658,16 @@ erts_dump_extended_process_state(int to, void *to_arg, erts_aint32_t psflg) { erts_print(to, to_arg, "PROXY"); break; case ERTS_PSFLG_DELAYED_SYS: erts_print(to, to_arg, "DELAYED_SYS"); break; -#ifdef ERTS_DIRTY_SCHEDULERS + case ERTS_PSFLG_OFF_HEAP_MSGQ: + erts_print(to, to_arg, "OFF_HEAP_MSGQ"); break; + case ERTS_PSFLG_ON_HEAP_MSGQ: + erts_print(to, to_arg, "ON_HEAP_MSGQ"); break; case ERTS_PSFLG_DIRTY_CPU_PROC: erts_print(to, to_arg, "DIRTY_CPU_PROC"); break; case ERTS_PSFLG_DIRTY_IO_PROC: erts_print(to, to_arg, "DIRTY_IO_PROC"); break; - case ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q: - erts_print(to, to_arg, "DIRTY_CPU_PROC_IN_Q"); break; - case ERTS_PSFLG_DIRTY_IO_PROC_IN_Q: - erts_print(to, to_arg, "DIRTY_IO_PROC_IN_Q"); break; -#endif + case ERTS_PSFLG_DIRTY_ACTIVE_SYS: + erts_print(to, to_arg, "DIRTY_ACTIVE_SYS"); break; default: erts_print(to, to_arg, "UNKNOWN(%d)", chk); break; } diff --git a/erts/emulator/beam/erl_term.c b/erts/emulator/beam/erl_term.c index e5050bfaa5..e31d3a951d 100644 --- a/erts/emulator/beam/erl_term.c +++ b/erts/emulator/beam/erl_term.c @@ -28,6 +28,37 @@ #include <stdlib.h> #include <stdio.h> +void +erts_set_literal_tag(Eterm *term, Eterm *hp_start, Eterm hsz) +{ +#ifdef TAG_LITERAL_PTR + Eterm *hp_end, *hp; + + hp_end = hp_start + hsz; + hp = hp_start; + + while (hp < hp_end) { + switch (primary_tag(*hp)) { + case TAG_PRIMARY_BOXED: + case TAG_PRIMARY_LIST: + *hp |= TAG_LITERAL_PTR; + break; + case TAG_PRIMARY_HEADER: + if (header_is_thing(*hp)) { + hp += thing_arityval(*hp); + } + break; + default: + break; + } + + hp++; + } + if (is_boxed(*term) || is_list(*term)) + *term |= TAG_LITERAL_PTR; +#endif +} + __decl_noreturn static void __noreturn et_abort(const char *expr, const char *file, unsigned line) { @@ -174,9 +205,7 @@ ET_DEFINE_CHECKED(Uint,external_thing_data_words,ExternalThing*,is_thing_ptr); ET_DEFINE_CHECKED(Eterm,make_cp,UWord *,_is_taggable_pointer); ET_DEFINE_CHECKED(UWord *,cp_val,Eterm,is_CP); ET_DEFINE_CHECKED(Uint,catch_val,Eterm,is_catch); -ET_DEFINE_CHECKED(Uint,x_reg_offset,Uint,_is_xreg); -ET_DEFINE_CHECKED(Uint,y_reg_offset,Uint,_is_yreg); -ET_DEFINE_CHECKED(Uint,x_reg_index,Uint,_is_xreg); -ET_DEFINE_CHECKED(Uint,y_reg_index,Uint,_is_yreg); +ET_DEFINE_CHECKED(Uint,loader_x_reg_index,Uint,_is_loader_x_reg); +ET_DEFINE_CHECKED(Uint,loader_y_reg_index,Uint,_is_loader_y_reg); #endif /* ET_DEBUG */ diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h index d5e91d80d9..2b28762db5 100644 --- a/erts/emulator/beam/erl_term.h +++ b/erts/emulator/beam/erl_term.h @@ -21,34 +21,10 @@ #ifndef __ERL_TERM_H #define __ERL_TERM_H -#include "sys.h" /* defines HALFWORD_HEAP */ +#include "erl_mmap.h" typedef UWord Wterm; /* Full word terms */ -#if HALFWORD_HEAP -# define HEAP_ON_C_STACK 0 -# if HALFWORD_ASSERT -# ifdef ET_DEBUG -# undef ET_DEBUG -# endif -# define ET_DEBUG 1 -# endif -# if 1 -# define CHECK_POINTER_MASK 0xFFFFFFFF00000000UL -# define COMPRESS_POINTER(APointer) ((Eterm) (UWord) (APointer)) -# define EXPAND_POINTER(AnEterm) ((UWord) (AnEterm)) -# else -# define CHECK_POINTER_MASK 0x0UL -# define COMPRESS_POINTER(AnUint) (AnUint) -# define EXPAND_POINTER(APointer) (APointer) -# endif -#else -# define HEAP_ON_C_STACK 1 -# define CHECK_POINTER_MASK 0x0UL -# define COMPRESS_POINTER(AnUint) (AnUint) -# define EXPAND_POINTER(APointer) (APointer) -#endif - struct erl_node_; /* Declared in erl_node_tables.h */ /* @@ -74,6 +50,24 @@ struct erl_node_; /* Declared in erl_node_tables.h */ #define _ET_APPLY(F,X) _unchecked_##F(X) #endif +#if defined(ARCH_64) +# define TAG_PTR_MASK__ 0x7 +# if !defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) +# ifdef HIPE +# error Hipe on 64-bit needs a real mmap as it does not support the literal tag +# endif +# define TAG_LITERAL_PTR 0x4 +# else +# undef TAG_LITERAL_PTR +# endif +#elif defined(ARCH_32) +# define TAG_PTR_MASK__ 0x3 +# undef TAG_LITERAL_PTR +#else +# error Not supported arch +#endif + + #define _TAG_PRIMARY_SIZE 2 #define _TAG_PRIMARY_MASK 0x3 #define TAG_PRIMARY_HEADER 0x0 @@ -190,15 +184,13 @@ struct erl_node_; /* Declared in erl_node_tables.h */ /* boxed object access methods */ -#if HALFWORD_HEAP -#define _is_taggable_pointer(x) (((UWord)(x) & (CHECK_POINTER_MASK | 0x3)) == 0) -#define _boxed_precond(x) (is_boxed(x)) -#else -#define _is_taggable_pointer(x) (((Uint)(x) & 0x3) == 0) + +#define _is_taggable_pointer(x) (((Uint)(x) & TAG_PTR_MASK__) == 0) + #define _boxed_precond(x) (is_boxed(x)) -#endif -#define _is_aligned(x) (((Uint)(x) & 0x3) == 0) -#define _unchecked_make_boxed(x) ((Uint) COMPRESS_POINTER(x) + TAG_PRIMARY_BOXED) + +#define _is_aligned(x) (((Uint)(x) & TAG_PTR_MASK__) == 0) +#define _unchecked_make_boxed(x) ((Uint)(x) + TAG_PRIMARY_BOXED) _ET_DECLARE_CHECKED(Eterm,make_boxed,const Eterm*) #define make_boxed(x) _ET_APPLY(make_boxed,(x)) #if 1 @@ -209,12 +201,16 @@ _ET_DECLARE_CHECKED(int,is_boxed,Eterm) #else #define is_boxed(x) (((x) & _TAG_PRIMARY_MASK) == TAG_PRIMARY_BOXED) #endif -#define _unchecked_boxed_val(x) ((Eterm*) EXPAND_POINTER(((x) - TAG_PRIMARY_BOXED))) +#ifdef TAG_LITERAL_PTR +#define _unchecked_boxed_val(x) _unchecked_ptr_val(x) +#else +#define _unchecked_boxed_val(x) ((Eterm*) ((x) - TAG_PRIMARY_BOXED)) +#endif _ET_DECLARE_CHECKED(Eterm*,boxed_val,Wterm) #define boxed_val(x) _ET_APPLY(boxed_val,(x)) /* cons cell ("list") access methods */ -#define _unchecked_make_list(x) ((Uint) COMPRESS_POINTER(x) + TAG_PRIMARY_LIST) +#define _unchecked_make_list(x) ((Uint)(x) + TAG_PRIMARY_LIST) _ET_DECLARE_CHECKED(Eterm,make_list,const Eterm*) #define make_list(x) _ET_APPLY(make_list,(x)) #if 1 @@ -226,12 +222,12 @@ _ET_DECLARE_CHECKED(int,is_not_list,Eterm) #define is_list(x) (((x) & _TAG_PRIMARY_MASK) == TAG_PRIMARY_LIST) #define is_not_list(x) (!is_list((x))) #endif -#if HALFWORD_HEAP #define _list_precond(x) (is_list(x)) +#ifdef TAG_LITERAL_PTR +#define _unchecked_list_val(x) _unchecked_ptr_val(x) #else -#define _list_precond(x) (is_list(x)) +#define _unchecked_list_val(x) ((Eterm*) ((x) - TAG_PRIMARY_LIST)) #endif -#define _unchecked_list_val(x) ((Eterm*) EXPAND_POINTER((x) - TAG_PRIMARY_LIST)) _ET_DECLARE_CHECKED(Eterm*,list_val,Wterm) #define list_val(x) _ET_APPLY(list_val,(x)) @@ -242,15 +238,22 @@ _ET_DECLARE_CHECKED(Eterm*,list_val,Wterm) #define CDR(x) ((x)[1]) /* generic tagged pointer (boxed or list) access methods */ -#define _unchecked_ptr_val(x) ((Eterm*) EXPAND_POINTER((x) & ~((Uint) 0x3))) +#define _unchecked_ptr_val(x) ((Eterm*) ((x) & ~((Uint) TAG_PTR_MASK__))) #define ptr_val(x) _unchecked_ptr_val((x)) /*XXX*/ #define _unchecked_offset_ptr(x,offs) ((x)+((offs)*sizeof(Eterm))) #define offset_ptr(x,offs) _unchecked_offset_ptr(x,offs) /*XXX*/ #define _unchecked_byte_offset_ptr(x,byte_offs) ((x)+(offs)) #define byte_offset_ptr(x,offs) _unchecked_byte_offset_ptr(x,offs) /*XXX*/ +#ifdef TAG_LITERAL_PTR +#define _unchecked_is_not_literal_ptr(x) (!((x) & TAG_LITERAL_PTR)) +#define is_not_literal_ptr(x) _unchecked_is_not_literal_ptr((x)) /*XXX*/ +#define is_literal_ptr(x) (!is_not_literal_ptr((x))) /*XXX*/ +#endif + + /* fixnum ("small") access methods */ -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) #define SMALL_BITS (64-4) #define SMALL_DIGITS (17) #else @@ -396,11 +399,7 @@ _ET_DECLARE_CHECKED(Eterm*,fun_val,Wterm) _ET_DECLARE_CHECKED(Eterm*,export_val,Wterm) #define export_val(x) _ET_APPLY(export_val,(x)) #define is_export_header(x) ((x) == HEADER_EXPORT) -#if HALFWORD_HEAP -#define HEADER_EXPORT _make_header(2,_TAG_HEADER_EXPORT) -#else #define HEADER_EXPORT _make_header(1,_TAG_HEADER_EXPORT) -#endif /* bignum access methods */ #define make_pos_bignum_header(sz) _make_header((sz),_TAG_HEADER_POS_BIG) @@ -424,7 +423,7 @@ _ET_DECLARE_CHECKED(Eterm*,big_val,Wterm) #define big_val(x) _ET_APPLY(big_val,(x)) /* flonum ("float") access methods */ -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) #define HEADER_FLONUM _make_header(1,_TAG_HEADER_FLOAT) #else #define HEADER_FLONUM _make_header(2,_TAG_HEADER_FLOAT) @@ -445,12 +444,12 @@ typedef union float_def byte fb[sizeof(ieee754_8)]; Uint16 fs[sizeof(ieee754_8) / sizeof(Uint16)]; Uint32 fw[sizeof(ieee754_8) / sizeof(Uint32)]; -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) Uint fdw; #endif } FloatDef; -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) #define FLOAT_VAL_GET_DOUBLE(fval, f) (f).fdw = *((fval)+1) @@ -727,7 +726,7 @@ _ET_DECLARE_CHECKED(struct erl_node_*,internal_port_node,Eterm) #define ERTS_MAX_REF_NUMBERS 3 #define ERTS_REF_NUMBERS ERTS_MAX_REF_NUMBERS -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) # define ERTS_REF_WORDS (ERTS_REF_NUMBERS/2 + 1) # define ERTS_REF_32BIT_WORDS (ERTS_REF_NUMBERS+1) #else @@ -749,7 +748,7 @@ typedef struct { #define make_ref_thing_header(DW) \ _make_header((DW)+REF_THING_HEAD_SIZE-1,_TAG_HEADER_REF) -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) /* * Ref layout on a 64-bit little endian machine: @@ -1009,27 +1008,20 @@ _ET_DECLARE_CHECKED(struct erl_node_*,external_ref_node,Eterm) #define MAP_HEADER_VAL(Hdr) (((Hdr) >> (_HEADER_ARITY_OFFS + MAP_HEADER_TAG_SZ + MAP_HEADER_ARITY_SZ)) & (0xffff)) #define make_hashmap(x) make_boxed((Eterm*)(x)) -#define make_hashmap_rel make_boxed_rel #define is_hashmap(x) (is_boxed((x)) && is_hashmap_header(*boxed_val((x)))) #define is_not_hashmap(x) (!is_hashmap(x)) -#define is_hashmap_rel(RTERM,BASE) is_hashmap(rterm2wterm(RTERM,BASE)) #define is_hashmap_header(x) (((x) & (_HEADER_MAP_HASHMAP_HEAD_MASK)) == HAMT_SUBTAG_HEAD_ARRAY) #define hashmap_val(x) _unchecked_boxed_val((x)) -#define hashmap_val_rel(RTERM, BASE) hashmap_val(rterm2wterm(RTERM, BASE)) #define make_flatmap(x) make_boxed((Eterm*)(x)) -#define make_flatmap_rel(x, BASE) make_boxed_rel((Eterm*)(x),(BASE)) #define is_flatmap(x) (is_boxed((x)) && is_flatmap_header(*boxed_val((x)))) -#define is_flatmap_rel(RTERM,BASE) is_flatmap(rterm2wterm(RTERM,BASE)) #define is_not_flatmap(x) (!is_flatmap((x))) #define is_flatmap_header(x) (((x) & (_HEADER_MAP_SUBTAG_MASK)) == HAMT_SUBTAG_HEAD_FLATMAP) #define flatmap_val(x) (_unchecked_boxed_val((x))) -#define flatmap_val_rel(RTERM, BASE) flatmap_val(rterm2wterm(RTERM, BASE)) #define is_map_header(x) (((x) & (_TAG_HEADER_MASK)) == _TAG_HEADER_MAP) #define is_map(x) (is_boxed((x)) && is_map_header(*boxed_val(x))) #define is_not_map(x) (!is_map(x)) -#define is_map_rel(RTERM,BASE) is_map(rterm2wterm(RTERM,BASE)) /* number tests */ @@ -1048,14 +1040,14 @@ _ET_DECLARE_CHECKED(struct erl_node_*,external_ref_node,Eterm) #error "fix yer arch, like" #endif -#define _unchecked_make_cp(x) ((Eterm) COMPRESS_POINTER(x)) +#define _unchecked_make_cp(x) ((Eterm)(x)) _ET_DECLARE_CHECKED(Eterm,make_cp,BeamInstr*) #define make_cp(x) _ET_APPLY(make_cp,(x)) #define is_not_CP(x) ((x) & _CPMASK) #define is_CP(x) (!is_not_CP(x)) -#define _unchecked_cp_val(x) ((BeamInstr*) EXPAND_POINTER(x)) +#define _unchecked_cp_val(x) ((BeamInstr*) (x)) _ET_DECLARE_CHECKED(BeamInstr*,cp_val,Eterm) #define cp_val(x) _ET_APPLY(cp_val,(x)) @@ -1071,44 +1063,40 @@ _ET_DECLARE_CHECKED(Uint,catch_val,Eterm) /* * Overloaded tags. * - * SMALL = 15 - * ATOM/NIL=7 + * In the loader, we want to tag a term in a way so that it can + * be any literal (atom/integer/float/tuple/list/binary) or a + * register. * - * Note that the two least significant bits in SMALL/ATOM/NIL always are 3; - * thus, we can distinguish register from literals by looking at only these - * two bits. + * We can achive that by overloading the PID and PORT tags to + * mean X and Y registers. That works because there are no + * pid or port literals. */ -#define X_REG_DEF 0 -#define Y_REG_DEF 1 -#define R_REG_DEF 2 +#define _LOADER_TAG_XREG _TAG_IMMED1_PID +#define _LOADER_TAG_YREG _TAG_IMMED1_PORT +#define _LOADER_TAG_SIZE _TAG_IMMED1_SIZE +#define _LOADER_MASK _TAG_IMMED1_MASK -#define beam_reg_tag(x) ((x) & 3) +#define LOADER_X_REG _LOADER_TAG_XREG +#define LOADER_Y_REG _LOADER_TAG_YREG -#define make_rreg() R_REG_DEF -#define make_xreg(ix) (((ix) * sizeof(Eterm)) | X_REG_DEF) -#define make_yreg(ix) (((ix) * sizeof(Eterm)) | Y_REG_DEF) +#define make_loader_x_reg(R) (((R) << _LOADER_TAG_SIZE) | _LOADER_TAG_XREG) +#define make_loader_y_reg(R) (((R) << _LOADER_TAG_SIZE) | _LOADER_TAG_YREG) -#define _is_xreg(x) (beam_reg_tag(x) == X_REG_DEF) -#define _is_yreg(x) (beam_reg_tag(x) == Y_REG_DEF) +#define loader_reg_index(R) ((R) >> _LOADER_TAG_SIZE) -#define _unchecked_x_reg_offset(R) ((R) - X_REG_DEF) -_ET_DECLARE_CHECKED(Uint,x_reg_offset,Uint) -#define x_reg_offset(R) _ET_APPLY(x_reg_offset,(R)) +#define loader_tag(T) ((T) & _LOADER_MASK) -#define _unchecked_y_reg_offset(R) ((R) - Y_REG_DEF) -_ET_DECLARE_CHECKED(Uint,y_reg_offset,Uint) -#define y_reg_offset(R) _ET_APPLY(y_reg_offset,(R)) +#define _is_loader_x_reg(x) (loader_tag(x) == _LOADER_TAG_XREG) +#define _is_loader_y_reg(x) (loader_tag(x) == _LOADER_TAG_YREG) -#define reg_index(R) ((R) / sizeof(Eterm)) +#define _unchecked_loader_x_reg_index(R) ((R) >> _LOADER_TAG_SIZE) +_ET_DECLARE_CHECKED(Uint,loader_x_reg_index,Uint) +#define loader_x_reg_index(R) _ET_APPLY(loader_x_reg_index,(R)) -#define _unchecked_x_reg_index(R) ((R) >> 2) -_ET_DECLARE_CHECKED(Uint,x_reg_index,Uint) -#define x_reg_index(R) _ET_APPLY(x_reg_index,(R)) - -#define _unchecked_y_reg_index(R) ((R) >> 2) -_ET_DECLARE_CHECKED(Uint,y_reg_index,Uint) -#define y_reg_index(R) _ET_APPLY(y_reg_index,(R)) +#define _unchecked_loader_y_reg_index(R) ((R) >> _LOADER_TAG_SIZE) +_ET_DECLARE_CHECKED(Uint,loader_y_reg_index,Uint) +#define loader_y_reg_index(R) _ET_APPLY(loader_y_reg_index,(R)) /* * Backwards compatibility definitions: @@ -1160,82 +1148,9 @@ extern unsigned tag_val_def(Wterm); #define FLOAT_BIG _NUMBER_CODE(FLOAT_DEF,BIG_DEF) #define FLOAT_FLOAT _NUMBER_CODE(FLOAT_DEF,FLOAT_DEF) -#if HALFWORD_HEAP -#define ptr2rel(PTR,BASE) ((Eterm*)((char*)(PTR) - (char*)(BASE))) -#define rterm2wterm(REL,BASE) ((Wterm)(REL) + (Wterm)(BASE)) - -#else /* HALFWORD_HEAP */ - -#define ptr2rel(PTR,BASE) (PTR) -#define rterm2wterm(REL,BASE) (REL) - -#endif /* !HALFWORD_HEAP */ - -#define make_list_rel(PTR, BASE) make_list(ptr2rel(PTR,BASE)) -#define make_boxed_rel(PTR, BASE) make_boxed(ptr2rel(PTR,BASE)) -#define make_fun_rel make_boxed_rel -#define make_binary_rel make_boxed_rel -#define make_tuple_rel make_boxed_rel -#define make_external_rel make_boxed_rel -#define make_internal_ref_rel make_boxed_rel -#define make_big_rel make_boxed_rel - -#define binary_val_rel(RTERM, BASE) binary_val(rterm2wterm(RTERM, BASE)) -#define list_val_rel(RTERM, BASE) list_val(rterm2wterm(RTERM, BASE)) -#define boxed_val_rel(RTERM, BASE) boxed_val(rterm2wterm(RTERM, BASE)) -#define tuple_val_rel(RTERM, BASE) tuple_val(rterm2wterm(RTERM, BASE)) -#define export_val_rel(RTERM, BASE) export_val(rterm2wterm(RTERM, BASE)) -#define fun_val_rel(RTERM, BASE) fun_val(rterm2wterm(RTERM, BASE)) -#define big_val_rel(RTERM,BASE) big_val(rterm2wterm(RTERM,BASE)) -#define float_val_rel(RTERM,BASE) float_val(rterm2wterm(RTERM,BASE)) -#define internal_ref_val_rel(RTERM,BASE) internal_ref_val(rterm2wterm(RTERM,BASE)) - -#define external_thing_ptr_rel(RTERM, BASE) external_thing_ptr(rterm2wterm(RTERM, BASE)) -#define external_data_words_rel(RTERM,BASE) external_data_words(rterm2wterm(RTERM,BASE)) - -#define external_port_node_rel(RTERM,BASE) external_port_node(rterm2wterm(RTERM,BASE)) -#define external_port_data_rel(RTERM,BASE) external_port_data(rterm2wterm(RTERM,BASE)) - -#define is_external_pid_rel(RTERM,BASE) is_external_pid(rterm2wterm(RTERM,BASE)) -#define external_pid_node_rel(RTERM,BASE) external_pid_node(rterm2wterm(RTERM,BASE)) -#define external_pid_data_rel(RTERM,BASE) external_pid_data(rterm2wterm(RTERM,BASE)) +#define is_same(A,B) ((A)==(B)) -#define is_binary_rel(RTERM,BASE) is_binary(rterm2wterm(RTERM,BASE)) -#define is_float_rel(RTERM,BASE) is_float(rterm2wterm(RTERM,BASE)) -#define is_fun_rel(RTERM,BASE) is_fun(rterm2wterm(RTERM,BASE)) -#define is_big_rel(RTERM,BASE) is_big(rterm2wterm(RTERM,BASE)) -#define is_export_rel(RTERM,BASE) is_export(rterm2wterm(RTERM,BASE)) -#define is_tuple_rel(RTERM,BASE) is_tuple(rterm2wterm(RTERM,BASE)) - -#define GET_DOUBLE_REL(RTERM, f, BASE) GET_DOUBLE(rterm2wterm(RTERM,BASE), f) - -#define ref_thing_ptr_rel(RTERM,BASE) ref_thing_ptr(rterm2wterm(RTERM,BASE)) -#define is_internal_ref_rel(RTERM,BASE) is_internal_ref(rterm2wterm(RTERM,BASE)) -#define is_external_rel(RTERM,BASE) is_external(rterm2wterm(RTERM,BASE)) -#define is_external_port_rel(RTERM,BASE) is_external_port(rterm2wterm(RTERM,BASE)) -#define is_external_ref_rel(RTERM,BASE) is_external_ref(rterm2wterm(RTERM,BASE)) - -#define external_node_rel(RTERM,BASE) external_node(rterm2wterm(RTERM,BASE)) - - -#if HALFWORD_HEAP -ERTS_GLB_INLINE int is_same(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base); - -#if ERTS_GLB_INLINE_INCL_FUNC_DEF -ERTS_GLB_INLINE int is_same(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base) -{ - /* If bases differ, assume a and b are on different "heaps", - ie can only be same if immed */ - ASSERT(a_base == b_base || is_immed(a) || is_immed(b) - || rterm2wterm(a,a_base) != rterm2wterm(b,b_base)); - - return a == b && (a_base == b_base || is_immed(a)); -} -#endif - -#else /* !HALFWORD_HEAP */ -#define is_same(A,A_BASE,B,B_BASE) ((A)==(B)) -#endif +void erts_set_literal_tag(Eterm *term, Eterm *hp_start, Eterm hsz); #endif /* __ERL_TERM_H */ diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h index 34f91e2ec8..e689547d1d 100644 --- a/erts/emulator/beam/erl_threads.h +++ b/erts/emulator/beam/erl_threads.h @@ -2056,6 +2056,8 @@ erts_atomic64_read_dirty(erts_atomic64_t *var) #endif /* !USE_THREADS */ +#include "erl_msacc.h" + #if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_INLINE void @@ -2414,6 +2416,7 @@ erts_cnd_wait(erts_cnd_t *cnd, erts_mtx_t *mtx) { #ifdef USE_THREADS int res; + ERTS_MSACC_PUSH_AND_SET_STATE(ERTS_MSACC_STATE_SLEEP); #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_unlock(&mtx->lc); #endif @@ -2432,6 +2435,7 @@ erts_cnd_wait(erts_cnd_t *cnd, erts_mtx_t *mtx) #endif if (res != 0 && res != EINTR) erts_thr_fatal_error(res, "wait on condition variable"); + ERTS_MSACC_POP_STATE(); #endif } @@ -3488,7 +3492,11 @@ ERTS_GLB_INLINE void erts_tse_reset(erts_tse_t *ep) ERTS_GLB_INLINE int erts_tse_wait(erts_tse_t *ep) { #ifdef USE_THREADS - return ethr_event_wait(&((ethr_ts_event *) ep)->event); + int res; + ERTS_MSACC_PUSH_AND_SET_STATE(ERTS_MSACC_STATE_SLEEP); + res = ethr_event_wait(&((ethr_ts_event *) ep)->event); + ERTS_MSACC_POP_STATE(); + return res; #else return ENOTSUP; #endif @@ -3497,7 +3505,11 @@ ERTS_GLB_INLINE int erts_tse_wait(erts_tse_t *ep) ERTS_GLB_INLINE int erts_tse_swait(erts_tse_t *ep, int spincount) { #ifdef USE_THREADS - return ethr_event_swait(&((ethr_ts_event *) ep)->event, spincount); + int res; + ERTS_MSACC_PUSH_AND_SET_STATE(ERTS_MSACC_STATE_SLEEP); + res = ethr_event_swait(&((ethr_ts_event *) ep)->event, spincount); + ERTS_MSACC_POP_STATE(); + return res; #else return ENOTSUP; #endif @@ -3506,8 +3518,12 @@ ERTS_GLB_INLINE int erts_tse_swait(erts_tse_t *ep, int spincount) ERTS_GLB_INLINE int erts_tse_twait(erts_tse_t *ep, Sint64 tmo) { #ifdef USE_THREADS - return ethr_event_twait(&((ethr_ts_event *) ep)->event, - (ethr_sint64_t) tmo); + int res; + ERTS_MSACC_PUSH_AND_SET_STATE(ERTS_MSACC_STATE_SLEEP); + res = ethr_event_twait(&((ethr_ts_event *) ep)->event, + (ethr_sint64_t) tmo); + ERTS_MSACC_POP_STATE(); + return res; #else return ENOTSUP; #endif @@ -3516,9 +3532,13 @@ ERTS_GLB_INLINE int erts_tse_twait(erts_tse_t *ep, Sint64 tmo) ERTS_GLB_INLINE int erts_tse_stwait(erts_tse_t *ep, int spincount, Sint64 tmo) { #ifdef USE_THREADS - return ethr_event_stwait(&((ethr_ts_event *) ep)->event, - spincount, - (ethr_sint64_t) tmo); + int res; + ERTS_MSACC_PUSH_AND_SET_STATE(ERTS_MSACC_STATE_SLEEP); + res = ethr_event_stwait(&((ethr_ts_event *) ep)->event, + spincount, + (ethr_sint64_t) tmo); + ERTS_MSACC_POP_STATE(); + return res; #else return ENOTSUP; #endif diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h index 93a0d556bf..5242063550 100644 --- a/erts/emulator/beam/erl_time.h +++ b/erts/emulator/beam/erl_time.h @@ -142,6 +142,8 @@ erts_time_unit_conversion(Uint64 value, Uint32 from_time_unit, Uint32 to_time_unit); +ErtsSysPerfCounter erts_perf_counter_unit(void); + #if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_INLINE Uint64 diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c index 5f12c7809a..98159fdf72 100644 --- a/erts/emulator/beam/erl_time_sup.c +++ b/erts/emulator/beam/erl_time_sup.c @@ -1749,7 +1749,7 @@ erts_get_monotonic_time(ErtsSchedulerData *esdp) { ErtsMonotonicTime mtime = time_sup.r.o.get_time(); update_last_mtime(esdp, mtime); - return mtime; + return mtime; } ErtsMonotonicTime @@ -1957,15 +1957,16 @@ send_time_offset_changed_notifications(void *new_offsetp) ErtsProcLocks rp_locks = ERTS_PROC_LOCK_LINK; erts_smp_proc_lock(rp, ERTS_PROC_LOCK_LINK); if (erts_lookup_monitor(ERTS_P_MONITORS(rp), ref)) { - ErlHeapFragment *bp; + ErtsMessage *mp; ErlOffHeap *ohp; Eterm message; - hp = erts_alloc_message_heap(hsz, &bp, &ohp, rp, &rp_locks); + mp = erts_alloc_message_heap(rp, &rp_locks, + hsz, &hp, &ohp); *patch_refp = ref; ASSERT(hsz == size_object(message_template)); message = copy_struct(message_template, hsz, &hp, ohp); - erts_queue_message(rp, &rp_locks, bp, message, NIL); + erts_queue_message(rp, &rp_locks, mp, message, NIL); } erts_smp_proc_unlock(rp, rp_locks); } @@ -2434,9 +2435,19 @@ BIF_RETTYPE os_system_time_0(BIF_ALIST_0) BIF_RET(make_time_val(BIF_P, stime)); } -BIF_RETTYPE os_system_time_1(BIF_ALIST_0) +BIF_RETTYPE os_system_time_1(BIF_ALIST_1) { ErtsSystemTime stime = erts_os_system_time(); BIF_RET(time_unit_conversion(BIF_P, BIF_ARG_1, stime, 0)); } +BIF_RETTYPE +os_perf_counter_0(BIF_ALIST_0) +{ + BIF_RET(make_time_val(BIF_P, erts_sys_perf_counter())); +} + +BIF_RETTYPE erts_internal_perf_counter_unit_0(BIF_ALIST_0) +{ + BIF_RET(make_time_val(BIF_P, erts_sys_perf_counter_unit())); +} diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index 8a4c0ab1f2..25393369a7 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -373,15 +373,10 @@ void erts_init_trace(void) { static Eterm system_seq_tracer; -#ifdef ERTS_SMP #define ERTS_ALLOC_SYSMSG_HEAP(SZ, BPP, OHPP, UNUSED) \ (*(BPP) = new_message_buffer((SZ)), \ *(OHPP) = &(*(BPP))->off_heap, \ (*(BPP))->mem) -#else -#define ERTS_ALLOC_SYSMSG_HEAP(SZ, BPP, OHPP, RPP) \ - erts_alloc_message_heap((SZ), (BPP), (OHPP), (RPP), 0) -#endif #ifdef ERTS_SMP #define ERTS_ENQ_TRACE_MSG(FPID, TPID, MSG, BP) \ @@ -390,8 +385,12 @@ do { \ enqueue_sys_msg_unlocked(SYS_MSG_TYPE_TRACE, (FPID), (TPID), (MSG), (BP)); \ } while(0) #else -#define ERTS_ENQ_TRACE_MSG(FPID, TPROC, MSG, BP) \ - erts_queue_message((TPROC), NULL, (BP), (MSG), NIL) +#define ERTS_ENQ_TRACE_MSG(FPID, TPROC, MSG, BP) \ + do { \ + ErtsMessage *mp__ = erts_alloc_message(0, NULL); \ + mp__->data.heap_frag = (BP); \ + erts_queue_message((TPROC), NULL, mp__, (MSG), NIL); \ + } while (0) #endif /* @@ -826,11 +825,9 @@ send_to_port(Process *c_p, Eterm message, static void profile_send(Eterm from, Eterm message) { Uint sz = 0; - ErlHeapFragment *bp = NULL; Uint *hp = NULL; Eterm msg = NIL; Process *profile_p = NULL; - ErlOffHeap *off_heap = NULL; Eterm profiler = erts_get_system_profile(); @@ -856,6 +853,7 @@ profile_send(Eterm from, Eterm message) { } } else { + ErtsMessage *mp; ASSERT(is_internal_pid(profiler)); profile_p = erts_proc_lookup(profiler); @@ -864,10 +862,13 @@ profile_send(Eterm from, Eterm message) { return; sz = size_object(message); - hp = erts_alloc_message_heap(sz, &bp, &off_heap, profile_p, 0); - msg = copy_struct(message, sz, &hp, &bp->off_heap); - - erts_queue_message(profile_p, NULL, bp, msg, NIL); + mp = erts_alloc_message(sz, &hp); + if (sz == 0) + msg = message; + else + msg = copy_struct(message, sz, &hp, &mp->hfrag.off_heap); + + erts_queue_message(profile_p, NULL, mp, msg, NIL); } } @@ -1235,11 +1236,7 @@ seq_trace_update_send(Process *p) { Eterm seq_tracer = erts_get_system_seq_tracer(); ASSERT((is_tuple(SEQ_TRACE_TOKEN(p)) || is_nil(SEQ_TRACE_TOKEN(p)))); - if ( (p->common.id == seq_tracer) || (SEQ_TRACE_TOKEN(p) == NIL) -#ifdef USE_VM_PROBES - || (SEQ_TRACE_TOKEN(p) == am_have_dt_utag) -#endif - ) { + if ((p->common.id == seq_tracer) || have_no_seqtrace(SEQ_TRACE_TOKEN(p))) { return 0; } SEQ_TRACE_TOKEN_SENDER(p) = p->common.id; @@ -1431,7 +1428,11 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type, erts_smp_mtx_unlock(&smq_mtx); #else /* trace_token must be NIL here */ - erts_queue_message(tracer, NULL, bp, mess, NIL); + { + ErtsMessage *mp = erts_alloc_message(0, NULL); + mp->data.heap_frag = bp; + erts_queue_message(tracer, NULL, mp, mess, NIL); + } #endif } } @@ -1851,12 +1852,7 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, ts_type = ERTS_TFLGS2TSTYPE(*tracee_flags); if (is_internal_port(*tracer_pid)) { -#if HEAP_ON_C_STACK - Eterm local_heap[60+ERTS_TRACE_PATCH_TS_MAX_SIZE+MAX_ARG]; -#else - Eterm *local_heap = erts_alloc(ERTS_ALC_T_TEMP_TERM, - sizeof(Eterm)*(60+ERTS_TRACE_PATCH_TS_MAX_SIZE+MAX_ARG)); -#endif + Eterm local_heap[64+ERTS_TRACE_PATCH_TS_MAX_SIZE+MAX_ARG]; hp = local_heap; if (!erts_is_valid_tracer_port(*tracer_pid)) { @@ -1871,9 +1867,6 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, if (is_not_nil(tracee)) erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL_MINOR); #endif -#if !HEAP_ON_C_STACK - erts_free(ERTS_ALC_T_TEMP_TERM,local_heap); -#endif UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); return 0; } @@ -1901,9 +1894,6 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, ERTS_PAM_TMP_RESULT, &return_flags); if (is_non_value(pam_result)) { erts_match_set_release_result(p); -#if !HEAP_ON_C_STACK - erts_free(ERTS_ALC_T_TEMP_TERM,local_heap); -#endif UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); return 0; } @@ -1912,9 +1902,6 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, /* Meta trace */ if (pam_result == am_false) { erts_match_set_release_result(p); -#if !HEAP_ON_C_STACK - erts_free(ERTS_ALC_T_TEMP_TERM,local_heap); -#endif UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); return return_flags; } @@ -1922,17 +1909,11 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, /* Non-meta trace */ if (*tracee_flags & F_TRACE_SILENT) { erts_match_set_release_result(p); -#if !HEAP_ON_C_STACK - erts_free(ERTS_ALC_T_TEMP_TERM,local_heap); -#endif UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); return 0; } if (pam_result == am_false) { erts_match_set_release_result(p); -#if !HEAP_ON_C_STACK - erts_free(ERTS_ALC_T_TEMP_TERM,local_heap); -#endif UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); return return_flags; } @@ -1974,9 +1955,6 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, send_to_port(p, mess, tracer_pid, tracee_flags); erts_smp_mtx_unlock(&smq_mtx); erts_match_set_release_result(p); -#if !HEAP_ON_C_STACK - erts_free(ERTS_ALC_T_TEMP_TERM,local_heap); -#endif UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); return *tracer_pid == NIL ? 0 : return_flags; @@ -2314,11 +2292,6 @@ void save_calls(Process *p, Export *e) void trace_gc(Process *p, Eterm what) { - ERTS_DECL_AM(bin_vheap_size); - ERTS_DECL_AM(bin_vheap_block_size); - ERTS_DECL_AM(bin_old_vheap_size); - ERTS_DECL_AM(bin_old_vheap_block_size); - ErlHeapFragment *bp = NULL; ErlOffHeap *off_heap; ERTS_TRACER_REF_TYPE tracer_ref = ERTS_NULL_TRACER_REF; /* Initialized @@ -2329,52 +2302,19 @@ trace_gc(Process *p, Eterm what) Eterm msg = NIL; Uint size; - Eterm tags[] = { - am_old_heap_block_size, - am_heap_block_size, - am_mbuf_size, - am_recent_size, - am_stack_size, - am_old_heap_size, - am_heap_size, - AM_bin_vheap_size, - AM_bin_vheap_block_size, - AM_bin_old_vheap_size, - AM_bin_old_vheap_block_size - }; - - UWord values[] = { - OLD_HEAP(p) ? OLD_HEND(p) - OLD_HEAP(p) : 0, - HEAP_SIZE(p), - MBUF_SIZE(p), - HIGH_WATER(p) - HEAP_START(p), - STACK_START(p) - p->stop, - OLD_HEAP(p) ? OLD_HTOP(p) - OLD_HEAP(p) : 0, - HEAP_TOP(p) - HEAP_START(p), - MSO(p).overhead, - BIN_VHEAP_SZ(p), - BIN_OLD_VHEAP(p), - BIN_OLD_VHEAP_SZ(p) - }; #define LOCAL_HEAP_SIZE \ - (sizeof(values)/sizeof(*values)) * \ - (2/*cons*/ + 3/*2-tuple*/ + BIG_UINT_HEAP_SIZE) + \ + (ERTS_PROCESS_GC_INFO_MAX_SIZE) + \ 5/*4-tuple */ + ERTS_TRACE_PATCH_TS_MAX_SIZE DeclareTmpHeap(local_heap,LOCAL_HEAP_SIZE,p); - ERTS_CT_ASSERT(sizeof(values)/sizeof(*values) == sizeof(tags)/sizeof(Eterm)); - UseTmpHeap(LOCAL_HEAP_SIZE,p); if (is_internal_port(ERTS_TRACER_PROC(p))) { hp = local_heap; #ifdef DEBUG size = 0; - (void) erts_bld_atom_uword_2tup_list(NULL, - &size, - sizeof(values)/sizeof(*values), - tags, - values); + (void) erts_process_gc_info(p, &size, NULL); + size += 5/*4-tuple*/ + PATCH_TS_SIZE(p); #endif } else { @@ -2385,11 +2325,8 @@ trace_gc(Process *p, Eterm what) ERTS_TRACE_FLAGS(p)); size = 0; - (void) erts_bld_atom_uword_2tup_list(NULL, - &size, - sizeof(values)/sizeof(*values), - tags, - values); + (void) erts_process_gc_info(p, &size, NULL); + size += 5/*4-tuple*/ + PATCH_TS_SIZE(p); hp = ERTS_ALLOC_SYSMSG_HEAP(size, &bp, &off_heap, tracer_ref); @@ -2397,11 +2334,7 @@ trace_gc(Process *p, Eterm what) ASSERT(size <= LOCAL_HEAP_SIZE); - msg = erts_bld_atom_uword_2tup_list(&hp, - NULL, - sizeof(values)/sizeof(*values), - tags, - values); + msg = erts_process_gc_info(p, NULL, &hp); msg = TUPLE4(hp, am_trace, p->common.id, what, msg); hp += 5; @@ -2477,7 +2410,11 @@ monitor_long_schedule_proc(Process *p, BeamInstr *in_fp, BeamInstr *out_fp, Uint #ifdef ERTS_SMP enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, p->common.id, NIL, msg, bp); #else - erts_queue_message(monitor_p, NULL, bp, msg, NIL); + { + ErtsMessage *mp = erts_alloc_message(0, NULL); + mp->data.heap_frag = bp; + erts_queue_message(monitor_p, NULL, mp, msg, NIL); + } #endif } void @@ -2538,7 +2475,11 @@ monitor_long_schedule_port(Port *pp, ErtsPortTaskType type, Uint time) #ifdef ERTS_SMP enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, pp->common.id, NIL, msg, bp); #else - erts_queue_message(monitor_p, NULL, bp, msg, NIL); + { + ErtsMessage *mp = erts_alloc_message(0, NULL); + mp->data.heap_frag = bp; + erts_queue_message(monitor_p, NULL, mp, msg, NIL); + } #endif } @@ -2609,7 +2550,11 @@ monitor_long_gc(Process *p, Uint time) { #ifdef ERTS_SMP enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, p->common.id, NIL, msg, bp); #else - erts_queue_message(monitor_p, NULL, bp, msg, NIL); + { + ErtsMessage *mp = erts_alloc_message(0, NULL); + mp->data.heap_frag = bp; + erts_queue_message(monitor_p, NULL, mp, msg, NIL); + } #endif } @@ -2680,7 +2625,11 @@ monitor_large_heap(Process *p) { #ifdef ERTS_SMP enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, p->common.id, NIL, msg, bp); #else - erts_queue_message(monitor_p, NULL, bp, msg, NIL); + { + ErtsMessage *mp = erts_alloc_message(0, NULL); + mp->data.heap_frag = bp; + erts_queue_message(monitor_p, NULL, mp, msg, NIL); + } #endif } @@ -2708,7 +2657,11 @@ monitor_generic(Process *p, Eterm type, Eterm spec) { #ifdef ERTS_SMP enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, p->common.id, NIL, msg, bp); #else - erts_queue_message(monitor_p, NULL, bp, msg, NIL); + { + ErtsMessage *mp = erts_alloc_message(0, NULL); + mp->data.heap_frag = bp; + erts_queue_message(monitor_p, NULL, mp, msg, NIL); + } #endif } @@ -3436,8 +3389,6 @@ sys_msg_dispatcher_func(void *unused) if (erts_thr_progress_update(NULL)) erts_thr_progress_leader_update(NULL); - ERTS_SCHED_FAIR_YIELD(); - #ifdef DEBUG_PRINTOUTS print_msg_type(smqp); #endif @@ -3496,8 +3447,11 @@ sys_msg_dispatcher_func(void *unused) goto failure; } else { + ErtsMessage *mp; queue_proc_msg: - erts_queue_message(proc,&proc_locks,smqp->bp,smqp->msg,NIL); + mp = erts_alloc_message(0, NULL); + mp->data.heap_frag = smqp->bp; + erts_queue_message(proc,&proc_locks,mp,smqp->msg,NIL); #ifdef DEBUG_PRINTOUTS erts_fprintf(stderr, "delivered\n"); #endif @@ -3592,9 +3546,6 @@ static void init_sys_msg_dispatcher(void) { erts_smp_thr_opts_t thr_opts = ERTS_SMP_THR_OPTS_DEFAULT_INITER; -#ifdef __OSE__ - thr_opts.coreNo = 0; -#endif thr_opts.detached = 1; thr_opts.name = "sys_msg_dispatcher"; init_smq_element_alloc(); @@ -3602,7 +3553,6 @@ init_sys_msg_dispatcher(void) sys_message_queue_end = NULL; erts_smp_cnd_init(&smq_cnd); erts_smp_mtx_init(&smq_mtx, "sys_msg_q"); - erts_smp_thr_create(&sys_msg_dispatcher_tid, sys_msg_dispatcher_func, NULL, diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h index 6dab3bf297..640969daf0 100644 --- a/erts/emulator/beam/erl_utils.h +++ b/erts/emulator/beam/erl_utils.h @@ -115,7 +115,7 @@ void erts_silence_warn_unused_result(long unused); int erts_fit_in_bits_int64(Sint64); int erts_fit_in_bits_int32(Sint32); int erts_fit_in_bits_uint(Uint); -int erts_list_length(Eterm); +Sint erts_list_length(Eterm); int erts_is_builtin(Eterm, Eterm, int); Uint32 make_broken_hash(Eterm); Uint32 block_hash(byte *, unsigned, Uint32); @@ -157,51 +157,46 @@ void erts_init_utils_mem(void); erts_dsprintf_buf_t *erts_create_tmp_dsbuf(Uint); void erts_destroy_tmp_dsbuf(erts_dsprintf_buf_t *); -#if HALFWORD_HEAP -int eq_rel(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base); -# define eq(A,B) eq_rel(A,NULL,B,NULL) -#else int eq(Eterm, Eterm); -# define eq_rel(A,A_BASE,B,B_BASE) eq(A,B) -#endif #define EQ(x,y) (((x) == (y)) || (is_not_both_immed((x),(y)) && eq((x),(y)))) -#if HALFWORD_HEAP -Sint erts_cmp_rel_opt(Eterm, Eterm*, Eterm, Eterm*, int, int); -#define cmp_rel(A,A_BASE,B,B_BASE) erts_cmp_rel_opt(A,A_BASE,B,B_BASE,0,0) -#define cmp_rel_term(A,A_BASE,B,B_BASE) erts_cmp_rel_opt(A,A_BASE,B,B_BASE,1,0) -#define CMP(A,B) erts_cmp_rel_opt(A,NULL,B,NULL,0,0) -#define CMP_TERM(A,B) erts_cmp_rel_opt(A,NULL,B,NULL,1,0) -#define CMP_EQ_ONLY(A,B) erts_cmp_rel_opt(A,NULL,B,NULL,0,1) -#else +int erts_cmp_atoms(Eterm a, Eterm b); Sint erts_cmp(Eterm, Eterm, int, int); +Sint erts_cmp_compound(Eterm, Eterm, int, int); Sint cmp(Eterm a, Eterm b); -#define cmp_rel(A,A_BASE,B,B_BASE) erts_cmp(A,B,0,0) -#define cmp_rel_term(A,A_BASE,B,B_BASE) erts_cmp(A,B,1,0) #define CMP(A,B) erts_cmp(A,B,0,0) #define CMP_TERM(A,B) erts_cmp(A,B,1,0) #define CMP_EQ_ONLY(A,B) erts_cmp(A,B,0,1) -#endif -#define cmp_lt(a,b) (CMP((a),(b)) < 0) -#define cmp_le(a,b) (CMP((a),(b)) <= 0) -#define cmp_eq(a,b) (CMP_EQ_ONLY((a),(b)) == 0) -#define cmp_ne(a,b) (CMP_EQ_ONLY((a),(b)) != 0) -#define cmp_ge(a,b) (CMP((a),(b)) >= 0) -#define cmp_gt(a,b) (CMP((a),(b)) > 0) - -#define cmp_lt_term(a,b) (CMP_TERM((a),(b)) < 0) -#define cmp_le_term(a,b) (CMP_TERM((a),(b)) <= 0) -#define cmp_ge_term(a,b) (CMP_TERM((a),(b)) >= 0) -#define cmp_gt_term(a,b) (CMP_TERM((a),(b)) > 0) - -#define CMP_LT(a,b) ((a) != (b) && cmp_lt((a),(b))) -#define CMP_GE(a,b) ((a) == (b) || cmp_ge((a),(b))) -#define CMP_EQ(a,b) ((a) == (b) || cmp_eq((a),(b))) -#define CMP_NE(a,b) ((a) != (b) && cmp_ne((a),(b))) - -#define CMP_LT_TERM(a,b) ((a) != (b) && cmp_lt_term((a),(b))) -#define CMP_GE_TERM(a,b) ((a) == (b) || cmp_ge_term((a),(b))) +#define CMP_LT(a,b) ((a) != (b) && CMP((a),(b)) < 0) +#define CMP_LE(a,b) ((a) == (b) || CMP((a),(b)) <= 0) +#define CMP_EQ(a,b) ((a) == (b) || CMP_EQ_ONLY((a),(b)) == 0) +#define CMP_NE(a,b) ((a) != (b) && CMP_EQ_ONLY((a),(b)) != 0) +#define CMP_GE(a,b) ((a) == (b) || CMP((a),(b)) >= 0) +#define CMP_GT(a,b) ((a) != (b) && CMP((a),(b)) > 0) + +#define CMP_EQ_ACTION(X,Y,Action) \ + if ((X) != (Y)) { CMP_SPEC((X),(Y),!=,Action,1); } +#define CMP_NE_ACTION(X,Y,Action) \ + if ((X) == (Y)) { Action; } else { CMP_SPEC((X),(Y),==,Action,1); } +#define CMP_GE_ACTION(X,Y,Action) \ + if ((X) != (Y)) { CMP_SPEC((X),(Y),<,Action,0); } +#define CMP_LT_ACTION(X,Y,Action) \ + if ((X) == (Y)) { Action; } else { CMP_SPEC((X),(Y),>=,Action,0); } + +#define CMP_SPEC(X,Y,Op,Action,EqOnly) \ + if (is_atom(X) && is_atom(Y)) { \ + if (erts_cmp_atoms(X, Y) Op 0) { Action; }; \ + } else if (is_both_small(X, Y)) { \ + if (signed_val(X) Op signed_val(Y)) { Action; }; \ + } else if (is_float(X) && is_float(Y)) { \ + FloatDef af, bf; \ + GET_DOUBLE(X, af); \ + GET_DOUBLE(Y, bf); \ + if (af.fd Op bf.fd) { Action; }; \ + } else { \ + if (erts_cmp_compound(X,Y,0,EqOnly) Op 0) { Action; }; \ + } #endif diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h index 8948ca2ea9..98f27a1725 100644 --- a/erts/emulator/beam/erl_vm.h +++ b/erts/emulator/beam/erl_vm.h @@ -40,14 +40,6 @@ #define MAX_ARG 255 /* Max number of arguments allowed */ #define MAX_REG 1024 /* Max number of x(N) registers used */ -/* Scheduler stores data for temporary heaps if - !HEAP_ON_C_STACK. Macros (*TmpHeap*) in global.h selects if we put temporary - heap data on the C stack or if we use the buffers in the scheduler data. */ -#define TMP_HEAP_SIZE 128 /* Number of Eterm in the schedulers - small heap for transient heap data */ -#define ERL_ARITH_TMP_HEAP_SIZE 4 /* as does erl_arith... */ -#define BEAM_EMU_TMP_HEAP_SIZE 2 /* and beam_emu... */ - /* * The new arithmetic operations need some extra X registers in the register array. * so does the gc_bif's (i_gc_bif3 need 3 extra). @@ -117,12 +109,8 @@ #define HeapWordsLeft(p) (HEAP_LIMIT(p) - HEAP_TOP(p)) #if defined(DEBUG) || defined(CHECK_FOR_HOLES) -#if HALFWORD_HEAP -# define ERTS_HOLE_MARKER (0xdeadbeef) -#else # define ERTS_HOLE_MARKER (((0xdeadbeef << 24) << 8) | 0xdeadbeef) -#endif -#endif +#endif /* egil: 32-bit ? */ /* * Allocate heap memory on the ordinary heap, NEVER in a heap @@ -153,6 +141,7 @@ typedef struct op_entry { char* name; /* Name of instruction. */ Uint32 mask[3]; /* Signature mask. */ + unsigned involves_r; /* Needs special attention when matching. */ int sz; /* Number of loaded words. */ char* pack; /* Instructions for packing engine. */ char* sign; /* Signature string. */ diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c index 2420df36b5..581efe6eec 100644 --- a/erts/emulator/beam/export.c +++ b/erts/emulator/beam/export.c @@ -184,6 +184,9 @@ init_export_table(void) f.cmp = (HCMP_FUN) export_cmp; f.alloc = (HALLOC_FUN) export_alloc; f.free = (HFREE_FUN) export_free; + f.meta_alloc = (HMALLOC_FUN) erts_alloc; + f.meta_free = (HMFREE_FUN) erts_free; + f.meta_print = (HMPRINT_FUN) erts_print; for (i=0; i<ERTS_NUM_CODE_IX; i++) { erts_index_init(ERTS_ALC_T_EXPORT_TABLE, &export_tables[i], "export_list", diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index a85aa15403..58010dcb72 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -2339,10 +2339,6 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Eterm val; FloatDef f; Sint r = 0; -#if HALFWORD_HEAP - UWord wobj; -#endif - if (ctx) { WSTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK); @@ -2362,11 +2358,8 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, outer_loop: while (!WSTACK_ISEMPTY(s)) { -#if HALFWORD_HEAP - obj = (Eterm) (wobj = WSTACK_POP(s)); -#else obj = WSTACK_POP(s); -#endif + switch (val = WSTACK_POP(s)) { case ENC_TERM: break; @@ -2384,11 +2377,7 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, break; case ENC_PATCH_FUN_SIZE: { -#if HALFWORD_HEAP - byte* size_p = (byte *) wobj; -#else byte* size_p = (byte *) obj; -#endif put_int32(ep - size_p, size_p); } goto outer_loop; @@ -2435,21 +2424,13 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, case ENC_LAST_ARRAY_ELEMENT: /* obj is the tuple */ { -#if HALFWORD_HEAP - Eterm* ptr = (Eterm *) wobj; -#else Eterm* ptr = (Eterm *) obj; -#endif obj = *ptr; } break; default: /* ENC_LAST_ARRAY_ELEMENT+1 and upwards */ { -#if HALFWORD_HEAP - Eterm* ptr = (Eterm *) wobj; -#else Eterm* ptr = (Eterm *) obj; -#endif obj = *ptr++; WSTACK_PUSH2(s, val-1, (UWord)ptr); } @@ -2976,7 +2957,7 @@ dec_term(ErtsDistExternal *edep, case B2TDecodeList: objp = next - 2; while (n > 0) { - objp[0] = (Eterm) COMPRESS_POINTER(next); + objp[0] = (Eterm) next; objp[1] = make_list(next); next = objp; objp -= 2; @@ -2987,7 +2968,7 @@ dec_term(ErtsDistExternal *edep, case B2TDecodeTuple: objp = next - 1; while (n-- > 0) { - objp[0] = (Eterm) COMPRESS_POINTER(next); + objp[0] = (Eterm) next; next = objp; objp--; } @@ -3043,7 +3024,7 @@ dec_term(ErtsDistExternal *edep, while (next != NULL) { objp = next; - next = (Eterm *) EXPAND_POINTER(*objp); + next = (Eterm *) *objp; switch (*ep++) { case INTEGER_EXT: @@ -3051,7 +3032,7 @@ dec_term(ErtsDistExternal *edep, Sint sn = get_int32(ep); ep += 4; -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) *objp = make_small(sn); #else if (MY_IS_SSMALL(sn)) { @@ -3174,7 +3155,7 @@ dec_term_atom_common: reds -= n; } while (n-- > 0) { - objp[0] = (Eterm) COMPRESS_POINTER(next); + objp[0] = (Eterm) next; next = objp; objp--; } @@ -3192,8 +3173,8 @@ dec_term_atom_common: *objp = make_list(hp); hp += 2 * n; objp = hp - 2; - objp[0] = (Eterm) COMPRESS_POINTER((objp+1)); - objp[1] = (Eterm) COMPRESS_POINTER(next); + objp[0] = (Eterm) (objp+1); + objp[1] = (Eterm) next; next = objp; objp -= 2; n--; @@ -3206,7 +3187,7 @@ dec_term_atom_common: reds -= n; } while (n > 0) { - objp[0] = (Eterm) COMPRESS_POINTER(next); + objp[0] = (Eterm) next; objp[1] = make_list(next); next = objp; objp -= 2; @@ -3373,7 +3354,7 @@ dec_term_atom_common: RefThing *rtp = (RefThing *) hp; ref_num = (Uint32 *) (hp + REF_THING_HEAD_SIZE); -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) hp += REF_THING_HEAD_SIZE + ref_words/2 + 1; rtp->header = make_ref_thing_header(ref_words/2 + 1); #else @@ -3384,13 +3365,13 @@ dec_term_atom_common: } else { ExternalThing *etp = (ExternalThing *) hp; -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) hp += EXTERNAL_THING_HEAD_SIZE + ref_words/2 + 1; #else hp += EXTERNAL_THING_HEAD_SIZE + ref_words; #endif -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) etp->header = make_external_ref_header(ref_words/2 + 1); #else etp->header = make_external_ref_header(ref_words); @@ -3403,7 +3384,7 @@ dec_term_atom_common: ref_num = &(etp->data.ui32[0]); } -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) *(ref_num++) = ref_words /* 32-bit arity */; #endif ref_num[0] = r0; @@ -3411,7 +3392,7 @@ dec_term_atom_common: ref_num[i] = get_int32(ep); ep += 4; } -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) if ((1 + ref_words) % 2) ref_num[ref_words] = 0; #endif @@ -3563,12 +3544,7 @@ dec_term_atom_common: } *objp = make_export(hp); *hp++ = HEADER_EXPORT; -#if HALFWORD_HEAP - *((UWord *) (UWord) hp) = (UWord) erts_export_get_or_make_stub(mod, name, arity); - hp += 2; -#else *hp++ = (Eterm) erts_export_get_or_make_stub(mod, name, arity); -#endif break; } break; @@ -3604,8 +3580,8 @@ dec_term_atom_common: *objp = make_flatmap(mp); for (n = size; n; n--) { - *vptr = (Eterm) COMPRESS_POINTER(next); - *kptr = (Eterm) COMPRESS_POINTER(vptr); + *vptr = (Eterm) next; + *kptr = (Eterm) vptr; next = kptr; vptr--; kptr--; @@ -3619,8 +3595,8 @@ dec_term_atom_common: hamt->leaf_array = hp; for (n = size; n; n--) { - CDR(hp) = (Eterm) COMPRESS_POINTER(next); - CAR(hp) = (Eterm) COMPRESS_POINTER(&CDR(hp)); + CDR(hp) = (Eterm) next; + CAR(hp) = (Eterm) &CDR(hp); next = &CAR(hp); hp += 2; } @@ -3697,11 +3673,11 @@ dec_term_atom_common: /* Environment */ for (i = num_free-1; i >= 0; i--) { - funp->env[i] = (Eterm) COMPRESS_POINTER(next); + funp->env[i] = (Eterm) next; next = funp->env + i; } /* Creator */ - funp->creator = (Eterm) COMPRESS_POINTER(next); + funp->creator = (Eterm) next; next = &(funp->creator); break; } @@ -3770,7 +3746,7 @@ dec_term_atom_common: /* Environment */ for (i = num_free-1; i >= 0; i--) { - funp->env[i] = (Eterm) COMPRESS_POINTER(next); + funp->env[i] = (Eterm) next; next = funp->env + i; } break; @@ -3897,7 +3873,7 @@ dec_term_atom_common: } WSTACK_DESTROY(flat_maps); - ASSERT((Eterm*)EXPAND_POINTER(*dbg_resultp) != NULL); + ASSERT((Eterm*)*dbg_resultp != NULL); if (ctx) { ctx->state = B2TDone; @@ -4190,11 +4166,7 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, case EXPORT_DEF: { Export* ep = *((Export **) (export_val(obj) + 1)); -#if HALFWORD_HEAP - result += 2; -#else result += 1; -#endif result += encode_size_struct2(acmp, ep->code[0], dflags); result += encode_size_struct2(acmp, ep->code[1], dflags); result += encode_size_struct2(acmp, make_small(ep->code[2]), dflags); @@ -4309,7 +4281,7 @@ init_done: switch (tag) { case INTEGER_EXT: SKIP(4); -#if !defined(ARCH_64) || HALFWORD_HEAP +#if !defined(ARCH_64) heap_size += BIG_UINT_HEAP_SIZE; #endif break; @@ -4398,7 +4370,7 @@ init_done: ep += 2; atom_extra_skip = 1 + 4*id_words; /* In case it is an external ref */ -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) heap_size += EXTERNAL_THING_HEAD_SIZE + id_words/2 + 1; #else heap_size += EXTERNAL_THING_HEAD_SIZE + id_words; @@ -4484,11 +4456,7 @@ init_done: break; case EXPORT_EXT: terms += 3; -#if HALFWORD_HEAP - heap_size += 3; -#else heap_size += 2; -#endif break; case NEW_FUN_EXT: { diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index ec9296d034..2a8bdb6ee3 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -310,9 +310,6 @@ typedef union { typedef struct proc_bin { Eterm thing_word; /* Subtag REFC_BINARY_SUBTAG. */ Uint size; /* Binary size in bytes. */ -#if HALFWORD_HEAP - void* dummy_ptr_padding__; -#endif struct erl_off_heap_header *next; Binary *val; /* Pointer to Binary structure. */ byte *bytes; /* Pointer to the actual data bytes. */ @@ -383,7 +380,7 @@ extern int bif_reductions; /* reductions + fcalls (when doing call_bif) */ extern int stackdump_on_exit; /* - * Here is an implementation of a lightweiht stack. + * Here is an implementation of a lightweight stack. * * Use it like this: * @@ -848,6 +845,94 @@ do {\ } while(0) +/* + * An implementation of lightweight unbounded queues, + * using a circular dynamic array. + * It does not include support for change_allocator. + * + * Use it like this: + * + * DECLARE_EQUEUE(Queue) (At the start of a block) + * ... + * EQUEUE_PUT(Queue, Term) + * ... + * if (EQUEUE_ISEMPTY(Queue)) { + * Queue is empty + * } else { + * Term = EQUEUE_GET(Stack); + * Process popped Term here + * } + * ... + * DESTROY_EQUEUE(Queue) + */ + +typedef struct { + Eterm* start; + Eterm* front; + Eterm* back; + int possibly_empty; + Eterm* end; + ErtsAlcType_t alloc_type; +} ErtsEQueue; + +#define DEF_EQUEUE_SIZE (16) + +void erl_grow_equeue(ErtsEQueue*, Eterm* def_queue); +#define EQUE_CONCAT(a,b) a##b +#define EQUE_DEF_QUEUE(q) EQUE_CONCAT(q,_default_equeue) + +#define DECLARE_EQUEUE(q) \ + UWord EQUE_DEF_QUEUE(q)[DEF_EQUEUE_SIZE]; \ + ErtsEQueue q = { \ + EQUE_DEF_QUEUE(q), /* start */ \ + EQUE_DEF_QUEUE(q), /* front */ \ + EQUE_DEF_QUEUE(q), /* back */ \ + 1, /* possibly_empty */ \ + EQUE_DEF_QUEUE(q) + DEF_EQUEUE_SIZE, /* end */ \ + ERTS_ALC_T_ESTACK /* alloc_type */ \ + } + +#define DESTROY_EQUEUE(q) \ +do { \ + if (q.start != EQUE_DEF_QUEUE(q)) { \ + erts_free(q.alloc_type, q.start); \ + } \ +} while(0) + +#define EQUEUE_PUT_UNCHECKED(q, x) \ +do { \ + q.possibly_empty = 0; \ + *(q.back) = (x); \ + if (++(q.back) == q.end) { \ + q.back = q.start; \ + } \ +} while(0) + +#define EQUEUE_PUT(q, x) \ +do { \ + if (q.back == q.front && !q.possibly_empty) { \ + erl_grow_equeue(&q, EQUE_DEF_QUEUE(q)); \ + } \ + EQUEUE_PUT_UNCHECKED(q, x); \ +} while(0) + +#define EQUEUE_ISEMPTY(q) (q.back == q.front && q.possibly_empty) + +ERTS_GLB_INLINE Eterm erts_equeue_get(ErtsEQueue *q); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE Eterm erts_equeue_get(ErtsEQueue *q) { + Eterm x; + q->possibly_empty = 1; + x = *(q->front); + if (++(q->front) == q->end) { + q->front = q->start; + } + return x; +} +#endif +#define EQUEUE_GET(q) erts_equeue_get(&(q)); + /* binary.c */ void erts_emasculate_writable_binary(ProcBin* pb); @@ -900,8 +985,18 @@ Eterm erl_send(Process *p, Eterm to, Eterm msg); Eterm erl_is_function(Process* p, Eterm arg1, Eterm arg2); /* beam_bif_load.c */ -Eterm erts_check_process_code(Process *c_p, Eterm module, int allow_gc, int *redsp); +#define ERTS_CPC_ALLOW_GC (1 << 0) +#define ERTS_CPC_COPY_LITERALS (1 << 1) +#define ERTS_CPC_ALL (ERTS_CPC_ALLOW_GC | ERTS_CPC_COPY_LITERALS) +Eterm erts_check_process_code(Process *c_p, Eterm module, Uint flags, int *redsp); +typedef struct { + Eterm *ptr; + Uint sz; + Eterm pid; +} copy_literals_t; + +extern copy_literals_t erts_clrange; /* beam_load.c */ typedef struct { @@ -956,35 +1051,71 @@ __decl_noreturn void __noreturn erl_exit(int n, char*, ...); __decl_noreturn void __noreturn erl_exit_flush_async(int n, char*, ...); void erl_error(char*, va_list); -/* copy.c */ -Eterm copy_object(Eterm, Process*); +/* This controls whether sharing-preserving copy is used by Erlang */ -#if HALFWORD_HEAP -Uint size_object_rel(Eterm, Eterm*); -# define size_object(A) size_object_rel(A,NULL) +#ifdef SHCOPY +#define SHCOPY_SEND +#define SHCOPY_SPAWN +#endif + +/* The persistent state while the sharing-preserving copier works */ -Eterm copy_struct_rel(Eterm, Uint, Eterm**, ErlOffHeap*, Eterm* src_base, Eterm* dst_base); -# define copy_struct(OBJ,SZ,HPP,OH) copy_struct_rel(OBJ,SZ,HPP,OH, NULL,NULL) +typedef struct { + Eterm queue_default[DEF_EQUEUE_SIZE]; + Eterm* queue_start; + Eterm* queue_end; + ErtsAlcType_t queue_alloc_type; + UWord bitstore_default[DEF_WSTACK_SIZE]; + UWord* bitstore_start; + ErtsAlcType_t bitstore_alloc_type; + Eterm shtable_default[DEF_ESTACK_SIZE]; + Eterm* shtable_start; + ErtsAlcType_t shtable_alloc_type; + Uint literal_size; + Eterm *range_ptr; + Uint range_sz; +} erts_shcopy_t; + +#define INITIALIZE_SHCOPY(info) \ +do { \ + info.queue_start = info.queue_default; \ + info.bitstore_start = info.bitstore_default; \ + info.shtable_start = info.shtable_default; \ + info.literal_size = 0; \ + info.range_ptr = erts_clrange.ptr; \ + info.range_sz = erts_clrange.sz; \ +} while(0) -Eterm copy_shallow_rel(Eterm*, Uint, Eterm**, ErlOffHeap*, Eterm* src_base); -# define copy_shallow(A,B,C,D) copy_shallow_rel(A,B,C,D,NULL) +#define DESTROY_SHCOPY(info) \ +do { \ + if (info.queue_start != info.queue_default) { \ + erts_free(info.queue_alloc_type, info.queue_start); \ + } \ + if (info.bitstore_start != info.bitstore_default) { \ + erts_free(info.bitstore_alloc_type, info.bitstore_start); \ + } \ + if (info.shtable_start != info.shtable_default) { \ + erts_free(info.shtable_alloc_type, info.shtable_start); \ + } \ +} while(0) -#else /* !HALFWORD_HEAP */ +/* copy.c */ +Eterm copy_object_x(Eterm, Process*, Uint); +#define copy_object(Term, Proc) copy_object_x(Term,Proc,0) Uint size_object(Eterm); -# define size_object_rel(A,B) size_object(A) +Uint copy_shared_calculate(Eterm, erts_shcopy_t*); +Eterm copy_shared_perform(Eterm, Uint, erts_shcopy_t*, Eterm**, ErlOffHeap*); -Eterm copy_struct(Eterm, Uint, Eterm**, ErlOffHeap*); -# define copy_struct_rel(OBJ,SZ,HPP,OH, SB,DB) copy_struct(OBJ,SZ,HPP,OH) +Uint size_shared(Eterm); +Eterm copy_struct_x(Eterm, Uint, Eterm**, ErlOffHeap*, Uint* bsz); +#define copy_struct(Obj,Sz,HPP,OH) \ + copy_struct_x(Obj,Sz,HPP,OH,NULL) Eterm copy_shallow(Eterm*, Uint, Eterm**, ErlOffHeap*); -# define copy_shallow_rel(A,B,C,D, BASE) copy_shallow(A,B,C,D) -#endif - - -void move_multi_frags(Eterm** hpp, ErlOffHeap*, ErlHeapFragment* first, - Eterm* refs, unsigned nrefs); +void erts_move_multi_frags(Eterm** hpp, ErlOffHeap*, ErlHeapFragment* first, + Eterm* refs, unsigned nrefs, int literals); /* Utilities */ extern void erts_delete_nodes_monitors(Process *, ErtsProcLocks); @@ -1037,6 +1168,9 @@ extern ErtsModifiedTimings erts_modified_timings[]; extern int erts_no_line_info; extern Eterm erts_error_logger_warnings; extern int erts_initialized; +#if defined(USE_THREADS) && !defined(ERTS_SMP) +extern erts_tid_t erts_main_thread; +#endif extern int erts_compat_rel; extern int erts_use_sender_punish; void erts_short_init(void); @@ -1177,10 +1311,10 @@ int erts_utf8_to_latin1(byte* dest, const byte* source, int slen); #define ERTS_UTF8_OK_MAX_CHARS 4 void bin_write(int, void*, byte*, size_t); -int intlist_to_buf(Eterm, char*, int); /* most callers pass plain char*'s */ +Sint intlist_to_buf(Eterm, char*, Sint); /* most callers pass plain char*'s */ struct Sint_buf { -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) char s[22]; #else char s[12]; @@ -1250,7 +1384,7 @@ ErlDrvSizeT erts_iolist_to_buf(Eterm, char*, ErlDrvSizeT); ErlDrvSizeT erts_iolist_to_buf_yielding(ErtsIOList2BufState *); int erts_iolist_size_yielding(ErtsIOListState *state); int erts_iolist_size(Eterm, ErlDrvSizeT *); -int is_string(Eterm); +Sint is_string(Eterm); void erl_at_exit(void (*) (void*), void*); Eterm collect_memory(Process *); void dump_memory_to_fd(int); @@ -1296,6 +1430,29 @@ int erts_print_system_version(int to, void *arg, Process *c_p); int erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* reg); +ERTS_GLB_FORCE_INLINE int erts_is_literal(Eterm tptr, Eterm *ptr); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_FORCE_INLINE int erts_is_literal(Eterm tptr, Eterm *ptr) +{ + ASSERT(is_boxed(tptr) || is_list(tptr)); + ASSERT(ptr == ptr_val(tptr)); + +#if defined(ERTS_HAVE_IS_IN_LITERAL_RANGE) + return erts_is_in_literal_range(ptr); +#elif defined(TAG_LITERAL_PTR) + return is_literal_ptr(tptr); +#else +# error Not able to detect literals... +#endif + +} + +#endif + +Eterm erts_msacc_request(Process *c_p, int action, Eterm *threads); + /* ** Call_trace uses this API for the parameter matching functions */ @@ -1344,191 +1501,21 @@ extern void erts_match_prog_foreach_offheap(Binary *b, extern erts_driver_t vanilla_driver; extern erts_driver_t spawn_driver; +extern erts_driver_t forker_driver; extern erts_driver_t fd_driver; int erts_beam_jump_table(void); -/* Should maybe be placed in erl_message.h, but then we get an include mess. */ -ERTS_GLB_INLINE Eterm * -erts_alloc_message_heap_state(Uint size, - ErlHeapFragment **bpp, - ErlOffHeap **ohpp, - Process *receiver, - ErtsProcLocks *receiver_locks, - erts_aint32_t *statep); - -ERTS_GLB_INLINE Eterm * -erts_alloc_message_heap(Uint size, - ErlHeapFragment **bpp, - ErlOffHeap **ohpp, - Process *receiver, - ErtsProcLocks *receiver_locks); - -#if ERTS_GLB_INLINE_INCL_FUNC_DEF - -/* - * NOTE: erts_alloc_message_heap() releases msg q and status - * lock on receiver without ensuring that other locks are - * held. User is responsible to ensure that the receiver - * pointer cannot become invalid until after message has - * been passed. This is normal done either by increasing - * reference count on process (preferred) or by holding - * main or link lock over the whole message passing - * operation. - */ - -ERTS_GLB_INLINE Eterm * -erts_alloc_message_heap_state(Uint size, - ErlHeapFragment **bpp, - ErlOffHeap **ohpp, - Process *receiver, - ErtsProcLocks *receiver_locks, - erts_aint32_t *statep) -{ - Eterm *hp; - erts_aint32_t state; -#ifdef ERTS_SMP - int locked_main = 0; - state = erts_smp_atomic32_read_acqb(&receiver->state); - if (statep) - *statep = state; - if (state & (ERTS_PSFLG_EXITING - | ERTS_PSFLG_PENDING_EXIT)) - goto allocate_in_mbuf; -#endif - - if (size > (Uint) INT_MAX) - erl_exit(ERTS_ABORT_EXIT, "HUGE size (%beu)\n", size); - - if ( -#if defined(ERTS_SMP) - *receiver_locks & ERTS_PROC_LOCK_MAIN -#else - 1 -#endif - ) { -#ifdef ERTS_SMP - try_allocate_on_heap: -#endif - state = erts_smp_atomic32_read_nob(&receiver->state); - if (statep) - *statep = state; - if ((state & (ERTS_PSFLG_EXITING - | ERTS_PSFLG_PENDING_EXIT)) - || (receiver->flags & F_DISABLE_GC) - || HEAP_LIMIT(receiver) - HEAP_TOP(receiver) <= size) { - /* - * The heap is either potentially in an inconsistent - * state, or not large enough. - */ -#ifdef ERTS_SMP - if (locked_main) { - *receiver_locks &= ~ERTS_PROC_LOCK_MAIN; - erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_MAIN); - } -#endif - goto allocate_in_mbuf; - } - hp = HEAP_TOP(receiver); - HEAP_TOP(receiver) = hp + size; - *bpp = NULL; - *ohpp = &MSO(receiver); - } -#ifdef ERTS_SMP - else if (erts_smp_proc_trylock(receiver, ERTS_PROC_LOCK_MAIN) == 0) { - locked_main = 1; - *receiver_locks |= ERTS_PROC_LOCK_MAIN; - goto try_allocate_on_heap; - } -#endif - else { - ErlHeapFragment *bp; - allocate_in_mbuf: - bp = new_message_buffer(size); - hp = bp->mem; - *bpp = bp; - *ohpp = &bp->off_heap; - } - - return hp; -} - -ERTS_GLB_INLINE Eterm * -erts_alloc_message_heap(Uint size, - ErlHeapFragment **bpp, - ErlOffHeap **ohpp, - Process *receiver, - ErtsProcLocks *receiver_locks) -{ - return erts_alloc_message_heap_state(size, bpp, ohpp, receiver, - receiver_locks, NULL); -} - -#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ - -#if !HEAP_ON_C_STACK -# if defined(DEBUG) -# define DeclareTmpHeap(VariableName,Size,Process) \ - Eterm *VariableName = erts_debug_allocate_tmp_heap(Size,Process) -# define DeclareTypedTmpHeap(Type,VariableName,Process) \ - Type *VariableName = (Type *) erts_debug_allocate_tmp_heap(sizeof(Type)/sizeof(Eterm),Process) -# define DeclareTmpHeapNoproc(VariableName,Size) \ - Eterm *VariableName = erts_debug_allocate_tmp_heap(Size,NULL) -# define UseTmpHeap(Size,Proc) \ - do { \ - erts_debug_use_tmp_heap((Size),(Proc)); \ - } while (0) -# define UnUseTmpHeap(Size,Proc) \ - do { \ - erts_debug_unuse_tmp_heap((Size),(Proc)); \ - } while (0) -# define UseTmpHeapNoproc(Size) \ - do { \ - erts_debug_use_tmp_heap(Size,NULL); \ - } while (0) -# define UnUseTmpHeapNoproc(Size) \ - do { \ - erts_debug_unuse_tmp_heap(Size,NULL); \ - } while (0) -# else -# define DeclareTmpHeap(VariableName,Size,Process) \ - Eterm *VariableName = (ERTS_PROC_GET_SCHDATA(Process)->tmp_heap)+(ERTS_PROC_GET_SCHDATA(Process)->num_tmp_heap_used) -# define DeclareTypedTmpHeap(Type,VariableName,Process) \ - Type *VariableName = (Type *) (ERTS_PROC_GET_SCHDATA(Process)->tmp_heap)+(ERTS_PROC_GET_SCHDATA(Process)->num_tmp_heap_used) -# define DeclareTmpHeapNoproc(VariableName,Size) \ - Eterm *VariableName = (erts_get_scheduler_data()->tmp_heap)+(erts_get_scheduler_data()->num_tmp_heap_used) -# define UseTmpHeap(Size,Proc) \ - do { \ - ERTS_PROC_GET_SCHDATA(Proc)->num_tmp_heap_used += (Size); \ - } while (0) -# define UnUseTmpHeap(Size,Proc) \ - do { \ - ERTS_PROC_GET_SCHDATA(Proc)->num_tmp_heap_used -= (Size); \ - } while (0) -# define UseTmpHeapNoproc(Size) \ - do { \ - erts_get_scheduler_data()->num_tmp_heap_used += (Size); \ - } while (0) -# define UnUseTmpHeapNoproc(Size) \ - do { \ - erts_get_scheduler_data()->num_tmp_heap_used -= (Size); \ - } while (0) - - -# endif - -#else -# define DeclareTmpHeap(VariableName,Size,Process) \ +#define DeclareTmpHeap(VariableName,Size,Process) \ Eterm VariableName[Size] -# define DeclareTypedTmpHeap(Type,VariableName,Process) \ +#define DeclareTypedTmpHeap(Type,VariableName,Process) \ Type VariableName[1] -# define DeclareTmpHeapNoproc(VariableName,Size) \ +#define DeclareTmpHeapNoproc(VariableName,Size) \ Eterm VariableName[Size] -# define UseTmpHeap(Size,Proc) /* Nothing */ -# define UnUseTmpHeap(Size,Proc) /* Nothing */ -# define UseTmpHeapNoproc(Size) /* Nothing */ -# define UnUseTmpHeapNoproc(Size) /* Nothing */ -#endif /* HEAP_ON_C_STACK */ +#define UseTmpHeap(Size,Proc) /* Nothing */ +#define UnUseTmpHeap(Size,Proc) /* Nothing */ +#define UseTmpHeapNoproc(Size) /* Nothing */ +#define UnUseTmpHeapNoproc(Size) /* Nothing */ ERTS_GLB_INLINE void dtrace_pid_str(Eterm pid, char *process_buf); ERTS_GLB_INLINE void dtrace_proc_str(Process *process, char *process_buf); @@ -1576,6 +1563,7 @@ dtrace_fun_decode(Process *process, erts_snprintf(mfa_buf, DTRACE_TERM_BUF_SIZE, "%T:%T/%d", module, function, arity); } + #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ #endif /* !__GLOBAL_H__ */ diff --git a/erts/emulator/beam/hash.c b/erts/emulator/beam/hash.c index e0fde337f2..5a0b93f693 100644 --- a/erts/emulator/beam/hash.c +++ b/erts/emulator/beam/hash.c @@ -27,8 +27,6 @@ #endif #include "sys.h" -#include "erl_vm.h" -#include "global.h" #include "hash.h" /* @@ -66,6 +64,7 @@ void hash_get_info(HashInfo *hi, Hash *h) int i; int max_depth = 0; int objects = 0; + int used = 0; for (i = 0; i < size; i++) { int depth = 0; @@ -76,14 +75,18 @@ void hash_get_info(HashInfo *hi, Hash *h) depth++; b = b->next; } - if (depth > max_depth) - max_depth = depth; + if (depth) { + used++; + if (depth > max_depth) + max_depth = depth; + } } + ASSERT(objects == h->nobjs); hi->name = h->name; hi->size = h->size; - hi->used = h->used; - hi->objs = objects; + hi->used = used; + hi->objs = h->nobjs; hi->depth = max_depth; } @@ -98,11 +101,11 @@ void hash_info(int to, void *arg, Hash* h) hash_get_info(&hi, h); - erts_print(to, arg, "=hash_table:%s\n", hi.name); - erts_print(to, arg, "size: %d\n", hi.size); - erts_print(to, arg, "used: %d\n", hi.used); - erts_print(to, arg, "objs: %d\n", hi.objs); - erts_print(to, arg, "depth: %d\n", hi.depth); + h->fun.meta_print(to, arg, "=hash_table:%s\n", hi.name); + h->fun.meta_print(to, arg, "size: %d\n", hi.size); + h->fun.meta_print(to, arg, "used: %d\n", hi.used); + h->fun.meta_print(to, arg, "objs: %d\n", hi.objs); + h->fun.meta_print(to, arg, "depth: %d\n", hi.depth); } @@ -119,47 +122,56 @@ hash_table_sz(Hash *h) } +static ERTS_INLINE void set_thresholds(Hash* h) +{ + h->grow_threshold = (8*h->size)/5; /* grow at 160% load */ + if (h->size_ix > h->min_size_ix) + h->shrink_threshold = h->size / 5; /* shrink at 20% load */ + else + h->shrink_threshold = -1; /* never shrink below inital size */ +} + /* ** init a pre allocated or static hash structure ** and allocate buckets. */ -Hash* hash_init(ErtsAlcType_t type, Hash* h, char* name, int size, HashFunctions fun) +Hash* hash_init(int type, Hash* h, char* name, int size, HashFunctions fun) { int sz; int ix = 0; - h->type = type; + h->meta_alloc_type = type; while (h_size_table[ix] != -1 && h_size_table[ix] < size) ix++; if (h_size_table[ix] == -1) - erl_exit(1, "panic: too large hash table size (%d)\n", size); + return NULL; size = h_size_table[ix]; sz = size*sizeof(HashBucket*); - h->bucket = (HashBucket**) erts_alloc(h->type, sz); + h->bucket = (HashBucket**) fun.meta_alloc(h->meta_alloc_type, sz); sys_memzero(h->bucket, sz); h->is_allocated = 0; h->name = name; h->fun = fun; h->size = size; - h->size20percent = h->size/5; - h->size80percent = (4*h->size)/5; - h->ix = ix; - h->used = 0; + h->size_ix = ix; + h->min_size_ix = ix; + h->nobjs = 0; + set_thresholds(h); return h; } /* ** Create a new hash table */ -Hash* hash_new(ErtsAlcType_t type, char* name, int size, HashFunctions fun) +Hash* hash_new(int type, char* name, int size, HashFunctions fun) { Hash* h; - h = erts_alloc(type, sizeof(Hash)); + h = fun.meta_alloc(type, sizeof(Hash)); h = hash_init(type, h, name, size, fun); h->is_allocated = 1; @@ -183,9 +195,9 @@ void hash_delete(Hash* h) b = b_next; } } - erts_free(h->type, h->bucket); + h->fun.meta_free(h->meta_alloc_type, h->bucket); if (h->is_allocated) - erts_free(h->type, (void*) h); + h->fun.meta_free(h->meta_alloc_type, (void*) h); } /* @@ -199,39 +211,34 @@ static void rehash(Hash* h, int grow) int i; if (grow) { - if ((h_size_table[h->ix+1]) == -1) + if ((h_size_table[h->size_ix+1]) == -1) return; - h->ix++; + h->size_ix++; } else { - if (h->ix == 0) + if (h->size_ix == 0) return; - h->ix--; + h->size_ix--; } - h->size = h_size_table[h->ix]; - h->size20percent = h->size/5; - h->size80percent = (4*h->size)/5; + h->size = h_size_table[h->size_ix]; sz = h->size*sizeof(HashBucket*); - new_bucket = (HashBucket **) erts_alloc(h->type, sz); + new_bucket = (HashBucket **) h->fun.meta_alloc(h->meta_alloc_type, sz); sys_memzero(new_bucket, sz); - h->used = 0; - for (i = 0; i < old_size; i++) { HashBucket* b = h->bucket[i]; while (b != (HashBucket*) 0) { HashBucket* b_next = b->next; int ix = b->hvalue % h->size; - if (new_bucket[ix] == NULL) - h->used++; b->next = new_bucket[ix]; new_bucket[ix] = b; b = b_next; } } - erts_free(h->type, (void *) h->bucket); + h->fun.meta_free(h->meta_alloc_type, (void *) h->bucket); h->bucket = new_bucket; + set_thresholds(h); } /* @@ -268,68 +275,15 @@ void* hash_put(Hash* h, void* tmpl) } b = (HashBucket*) h->fun.alloc(tmpl); - if (h->bucket[ix] == NULL) - h->used++; - b->hvalue = hval; b->next = h->bucket[ix]; h->bucket[ix] = b; - if (h->used > h->size80percent) /* rehash at 80% */ + if (++h->nobjs > h->grow_threshold) rehash(h, 1); return (void*) b; } -static void -hash_insert_entry(Hash* h, HashBucket* entry) -{ - HashValue hval = entry->hvalue; - int ix = hval % h->size; - HashBucket* b = h->bucket[ix]; - - while (b != (HashBucket*) 0) { - if ((b->hvalue == hval) && (h->fun.cmp((void*)entry, (void*)b) == 0)) { - abort(); /* Should not happen */ - } - b = b->next; - } - - if (h->bucket[ix] == NULL) - h->used++; - - entry->next = h->bucket[ix]; - h->bucket[ix] = entry; - - if (h->used > h->size80percent) /* rehash at 80% */ - rehash(h, 1); -} - - -/* - * Move all entries in src into dst; empty src. - * Entries in src must not exist in dst. - */ -void -erts_hash_merge(Hash* src, Hash* dst) -{ - int limit = src->size; - HashBucket** bucket = src->bucket; - int i; - - src->used = 0; - for (i = 0; i < limit; i++) { - HashBucket* b = bucket[i]; - HashBucket* next; - - bucket[i] = NULL; - while (b) { - next = b->next; - hash_insert_entry(dst, b); - b = next; - } - } -} - /* ** Erase hash entry return template if erased ** return 0 if not erased @@ -348,9 +302,7 @@ void* hash_erase(Hash* h, void* tmpl) else h->bucket[ix] = b->next; h->fun.free((void*)b); - if (h->bucket[ix] == NULL) - h->used--; - if (h->used < h->size20percent) /* rehash at 20% */ + if (--h->nobjs < h->shrink_threshold) rehash(h, 0); return tmpl; } @@ -381,9 +333,7 @@ hash_remove(Hash *h, void *tmpl) prev->next = b->next; else h->bucket[ix] = b->next; - if (h->bucket[ix] == NULL) - h->used--; - if (h->used < h->size20percent) /* rehash at 20% */ + if (--h->nobjs < h->shrink_threshold) rehash(h, 0); return (void *) b; } diff --git a/erts/emulator/beam/hash.h b/erts/emulator/beam/hash.h index 87fdb360e3..e94aaa0a84 100644 --- a/erts/emulator/beam/hash.h +++ b/erts/emulator/beam/hash.h @@ -29,14 +29,17 @@ #include "sys.h" #endif -#include "erl_alloc.h" - typedef unsigned long HashValue; +typedef struct hash Hash; typedef int (*HCMP_FUN)(void*, void*); typedef HashValue (*H_FUN)(void*); typedef void* (*HALLOC_FUN)(void*); typedef void (*HFREE_FUN)(void*); +/* Meta functions */ +typedef void* (*HMALLOC_FUN)(int,size_t); +typedef void (*HMFREE_FUN)(int,void*); +typedef int (*HMPRINT_FUN)(int,void*,char*, ...); /* ** This bucket must be placed in top of @@ -55,6 +58,9 @@ typedef struct hash_functions HCMP_FUN cmp; HALLOC_FUN alloc; HFREE_FUN free; + HMALLOC_FUN meta_alloc; + HMFREE_FUN meta_free; + HMPRINT_FUN meta_print; } HashFunctions; typedef struct { @@ -65,22 +71,23 @@ typedef struct { int depth; } HashInfo; -typedef struct hash +struct hash { HashFunctions fun; /* Function block */ int is_allocated; /* 0 iff hash structure is on stack or is static */ - ErtsAlcType_t type; + int meta_alloc_type; /* argument to pass to meta_alloc and meta_free */ char* name; /* Table name (static string, for debugging) */ int size; /* Number of slots */ - int size20percent; /* 20 percent of number of slots */ - int size80percent; /* 80 percent of number of slots */ - int ix; /* Size index in size table */ - int used; /* Number of slots used */ + int shrink_threshold; + int grow_threshold; + int size_ix; /* Size index in size table */ + int min_size_ix; /* Never shrink table smaller than this */ + int nobjs; /* Number of objects in table */ HashBucket** bucket; /* Vector of bucket pointers (objects) */ -} Hash; +}; -Hash* hash_new(ErtsAlcType_t, char*, int, HashFunctions); -Hash* hash_init(ErtsAlcType_t, Hash*, char*, int, HashFunctions); +Hash* hash_new(int, char*, int, HashFunctions); +Hash* hash_init(int, Hash*, char*, int, HashFunctions); void hash_delete(Hash*); void hash_get_info(HashInfo*, Hash*); @@ -93,6 +100,4 @@ void* hash_erase(Hash*, void*); void* hash_remove(Hash*, void*); void hash_foreach(Hash*, void (*func)(void *, void *), void *); -void erts_hash_merge(Hash* src, Hash* dst); - #endif diff --git a/erts/emulator/beam/index.h b/erts/emulator/beam/index.h index 14fab41026..99b2bdfab0 100644 --- a/erts/emulator/beam/index.h +++ b/erts/emulator/beam/index.h @@ -30,6 +30,10 @@ #include "hash.h" #endif +#ifndef ERL_ALLOC_H__ +#include "erl_alloc.h" +#endif + typedef struct index_slot { HashBucket bucket; diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index 2bd31ee97e..41b0fcdd1b 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -53,10 +53,11 @@ #include "erl_time.h" extern ErlDrvEntry fd_driver_entry; -#ifndef __OSE__ extern ErlDrvEntry vanilla_driver_entry; -#endif extern ErlDrvEntry spawn_driver_entry; +#ifndef __WIN32__ +extern ErlDrvEntry forker_driver_entry; +#endif extern ErlDrvEntry *driver_tab[]; /* table of static drivers, only used during initialization */ erts_driver_t *driver_list; /* List of all drivers, static and dynamic. */ @@ -74,6 +75,9 @@ const Port erts_invalid_port = {{ERTS_INVALID_PORT}}; erts_driver_t vanilla_driver; erts_driver_t spawn_driver; +#ifndef __WIN32__ +erts_driver_t forker_driver; +#endif erts_driver_t fd_driver; int erts_port_synchronous_ops = 0; @@ -87,6 +91,7 @@ static void deliver_result(Eterm sender, Eterm pid, Eterm res); static int init_driver(erts_driver_t *, ErlDrvEntry *, DE_Handle *); static void terminate_port(Port *p); static void pdl_init(void); +static int driver_failure_term(ErlDrvPort ix, Eterm term, int eof); #ifdef ERTS_SMP static void driver_monitor_lock_pdl(Port *p); static void driver_monitor_unlock_pdl(Port *p); @@ -308,12 +313,9 @@ static Port *create_port(char *name, size_t port_size, busy_port_queue_size, size; erts_aint32_t state = ERTS_PORT_SFLG_CONNECTED; erts_aint32_t x_pts_flgs = 0; -#ifdef DEBUG - /* Make sure the debug flags survives until port is freed */ - state |= ERTS_PORT_SFLG_PORT_DEBUG; -#endif #ifdef ERTS_SMP + ErtsRunQueue *runq; if (!driver_lock) { /* Align size for mutex following port struct */ port_size = size = ERTS_ALC_DATA_ALIGN_SIZE(sizeof(Port)); @@ -323,6 +325,12 @@ static Port *create_port(char *name, #endif port_size = size = ERTS_ALC_DATA_ALIGN_SIZE(sizeof(Port)); +#ifdef DEBUG + /* Make sure the debug flags survives until port is freed */ + state |= ERTS_PORT_SFLG_PORT_DEBUG; +#endif + + busy_port_queue_size = ((driver->flags & ERL_DRV_FLAG_NO_BUSY_MSGQ) ? 0 @@ -358,8 +366,12 @@ static Port *create_port(char *name, p += sizeof(erts_mtx_t); state |= ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK; } - erts_smp_atomic_set_nob(&prt->run_queue, - (erts_aint_t) erts_get_runq_current(NULL)); + if (erts_get_scheduler_data()) + runq = erts_get_runq_current(NULL); + else + runq = ERTS_RUNQ_IX(0); + erts_smp_atomic_set_nob(&prt->run_queue, (erts_aint_t) runq); + prt->xports = NULL; #else erts_atomic32_init_nob(&prt->refc, 1); @@ -386,6 +398,7 @@ static Port *create_port(char *name, ERTS_PTMR_INIT(prt); erts_port_task_handle_init(&prt->timeout_task); prt->psd = NULL; + prt->async_open_port = NULL; prt->drv_data = (SWord) 0; prt->os_pid = -1; @@ -467,6 +480,11 @@ erts_port_free(Port *prt) erts_port_task_fini_sched(&prt->sched); + if (prt->async_open_port) { + erts_free(ERTS_ALC_T_PRTSD, prt->async_open_port); + prt->async_open_port = NULL; + } + #ifdef ERTS_SMP ASSERT(prt->lock); if (state & ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK) @@ -688,6 +706,7 @@ erts_open_driver(erts_driver_t* driver, /* Pointer to driver. */ error_number = error_type = 0; if (driver->start) { + ERTS_MSACC_PUSH_STATE_M(); if (IS_TRACED_FL(port, F_TRACE_SCHED_PORTS)) { trace_sched_ports_where(port, am_in, am_start); } @@ -698,6 +717,7 @@ erts_open_driver(erts_driver_t* driver, /* Pointer to driver. */ DTRACE3(driver_start, process_str, driver->name, port_str); } #endif + ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_PORT); fpe_was_unmasked = erts_block_fpe(); drv_data = (*driver->start)(ERTS_Port2ErlDrvPort(port), name, opts); if (((SWord) drv_data) == -1) @@ -717,6 +737,7 @@ erts_open_driver(erts_driver_t* driver, /* Pointer to driver. */ } erts_unblock_fpe(fpe_was_unmasked); + ERTS_MSACC_POP_STATE_M(); port->caller = NIL; if (IS_TRACED_FL(port, F_TRACE_SCHED_PORTS)) { trace_sched_ports_where(port, am_out, am_start); @@ -1106,7 +1127,7 @@ io_list_vec_len(Eterm obj, int* vsize, Uint* csize, Uint p_v_size = 0; Uint p_c_size = 0; Uint p_in_clist = 0; - Uint total; /* Uint due to halfword emulator */ + Uint total; goto L_jump_start; /* avoid a push */ @@ -1413,7 +1434,7 @@ queue_port_sched_op_reply(Process *rp, erts_factory_trim_and_close(factory, &msg, 1); - erts_queue_message(rp, rp_locksp, factory->heap_frags, msg, NIL); + erts_queue_message(rp, rp_locksp, factory->message, msg, NIL); } static void @@ -1421,12 +1442,9 @@ port_sched_op_reply(Eterm to, Uint32 *ref_num, Eterm msg) { Process *rp = erts_proc_lookup_raw(to); if (rp) { - ErlOffHeap *ohp; - ErlHeapFragment* bp; ErtsHeapFactory factory; Eterm msg_copy; Uint hsz, msg_sz; - Eterm *hp; ErtsProcLocks rp_locks = 0; hsz = ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE; @@ -1437,18 +1455,13 @@ port_sched_op_reply(Eterm to, Uint32 *ref_num, Eterm msg) hsz += msg_sz; } - hp = erts_alloc_message_heap(hsz, - &bp, - &ohp, - rp, - &rp_locks); - erts_factory_message_init(&factory, rp, hp, bp); - if (is_immed(msg)) - msg_copy = msg; - else { - msg_copy = copy_struct(msg, msg_sz, &hp, ohp); - factory.hp = hp; - } + (void) erts_factory_message_create(&factory, rp, + &rp_locks, hsz); + msg_copy = (is_immed(msg) + ? msg + : copy_struct(msg, msg_sz, + &factory.hp, + factory.off_heap)); queue_port_sched_op_reply(rp, &rp_locks, @@ -1536,6 +1549,26 @@ erts_schedule_proc2port_signal(Process *c_p, return ERTS_PORT_OP_SCHEDULED; } +static int +erts_schedule_port2port_signal(Eterm port_num, ErtsProc2PortSigData *sigdp, + int task_flags, + ErtsProc2PortSigCallback callback) +{ + Port *prt = erts_port_lookup_raw(port_num); + + if (!prt) + return -1; + + sigdp->caller = ERTS_INVALID_PID; + + return erts_port_task_schedule(prt->common.id, + NULL, + ERTS_PORT_TASK_PROC_SIG, + sigdp, + callback, + task_flags); +} + static ERTS_INLINE void send_badsig(Port *prt) { ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND; @@ -1680,6 +1713,7 @@ call_driver_outputv(int bang_op, else { ErtsSchedulerData *esdp = erts_get_scheduler_data(); ErlDrvSizeT size = evp->size; + ERTS_MSACC_PUSH_AND_SET_STATE_M(ERTS_MSACC_STATE_PORT); ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt) || ERTS_IS_CRASH_DUMPING); @@ -1700,6 +1734,8 @@ call_driver_outputv(int bang_op, esdp->io.out += (Uint64) size; else erts_atomic64_add_nob(&bytes_out, (erts_aint64_t) size); + + ERTS_MSACC_POP_STATE_M(); } } @@ -1780,6 +1816,7 @@ call_driver_output(int bang_op, send_badsig(prt); else { ErtsSchedulerData *esdp = erts_get_scheduler_data(); + ERTS_MSACC_PUSH_AND_SET_STATE_M(ERTS_MSACC_STATE_PORT); ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt) || ERTS_IS_CRASH_DUMPING); @@ -1799,6 +1836,8 @@ call_driver_output(int bang_op, esdp->io.out += (Uint64) size; else erts_atomic64_add_nob(&bytes_out, (erts_aint64_t) size); + + ERTS_MSACC_POP_STATE_M(); } } @@ -2371,6 +2410,11 @@ erts_port_exit(Process *c_p, | ERTS_PORT_SIG_FLG_BROKEN_LINK | ERTS_PORT_SIG_FLG_FORCE_SCHED)) == 0); +#ifndef __WIN32__ + if (prt->drv_ptr == &forker_driver) + return ERTS_PORT_OP_DROPPED; +#endif + if (!(flags & ERTS_PORT_SIG_FLG_FORCE_SCHED)) { ErtsTryImmDrvCallState try_call_state = ERTS_INIT_TRY_IMM_DRV_CALL_STATE(c_p, @@ -2735,6 +2779,72 @@ erts_port_link(Process *c_p, Port *prt, Eterm to, Eterm *refp) port_sig_link); } +static void +init_ack_send_reply(Port *port, Eterm resp) +{ + + if (!is_internal_port(resp)) { + Process *rp = erts_proc_lookup_raw(port->async_open_port->to); + erts_smp_proc_lock(rp, ERTS_PROC_LOCK_LINK); + erts_remove_link(&ERTS_P_LINKS(port), port->async_open_port->to); + erts_remove_link(&ERTS_P_LINKS(rp), port->common.id); + erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); + } + port_sched_op_reply(port->async_open_port->to, + port->async_open_port->ref, + resp); + + erts_free(ERTS_ALC_T_PRTSD, port->async_open_port); + port->async_open_port = NULL; +} + +void +erl_drv_init_ack(ErlDrvPort ix, ErlDrvData res) { + Port *port = erts_drvport2port(ix); + SWord err_type = (SWord)res; + Eterm resp; + + if (port == ERTS_INVALID_ERL_DRV_PORT && port->async_open_port) + return; + + if (port->async_open_port) { + switch(err_type) { + case -3: + resp = am_badarg; + break; + case -2: { + char *str = erl_errno_id(errno); + resp = erts_atom_put((byte *) str, strlen(str), + ERTS_ATOM_ENC_LATIN1, 1); + break; + } + case -1: + resp = am_einval; + break; + default: + resp = port->common.id; + break; + } + + init_ack_send_reply(port, resp); + + if (err_type == -1 || err_type == -2 || err_type == -3) + driver_failure_term(ix, am_normal, 0); + port->drv_data = err_type; + } +} + +void +erl_drv_set_os_pid(ErlDrvPort ix, ErlDrvSInt pid) { + Port *port = erts_drvport2port(ix); + + if (port == ERTS_INVALID_ERL_DRV_PORT) + return; + + port->os_pid = (SWord)pid; + +} + void erts_init_io(int port_tab_size, int port_tab_size_ignore_files, int legacy_port_tab) @@ -2795,10 +2905,11 @@ void erts_init_io(int port_tab_size, erts_smp_rwmtx_rwlock(&erts_driver_list_lock); init_driver(&fd_driver, &fd_driver_entry, NULL); -#ifndef __OSE__ init_driver(&vanilla_driver, &vanilla_driver_entry, NULL); -#endif init_driver(&spawn_driver, &spawn_driver_entry, NULL); +#ifndef __WIN32__ + init_driver(&forker_driver, &forker_driver_entry, NULL); +#endif erts_init_static_drivers(); for (dp = driver_tab; *dp != NULL; dp++) erts_add_driver_entry(*dp, NULL, 1); @@ -2860,6 +2971,9 @@ void erts_lcnt_enable_io_lock_count(int enable) { lcnt_enable_drv_lock_count(&vanilla_driver, enable); lcnt_enable_drv_lock_count(&spawn_driver, enable); +#ifndef __WIN32__ + lcnt_enable_drv_lock_count(&forker_driver, enable); +#endif lcnt_enable_drv_lock_count(&fd_driver, enable); /* enable lock counting in all drivers */ for (dp = driver_list; dp; dp = dp->next) { @@ -3055,16 +3169,17 @@ deliver_result(Eterm sender, Eterm pid, Eterm res) if (rp) { Eterm tuple; - ErlHeapFragment *bp; + ErtsMessage *mp; ErlOffHeap *ohp; Eterm* hp; Uint sz_res; sz_res = size_object(res); - hp = erts_alloc_message_heap(sz_res + 3, &bp, &ohp, rp, &rp_locks); + mp = erts_alloc_message_heap(rp, &rp_locks, + sz_res + 3, &hp, &ohp); res = copy_struct(res, sz_res, &hp, ohp); tuple = TUPLE2(hp, sender, res); - erts_queue_message(rp, &rp_locks, bp, tuple, NIL); + erts_queue_message(rp, &rp_locks, mp, tuple, NIL); if (rp_locks) erts_smp_proc_unlock(rp, rp_locks); @@ -3092,7 +3207,7 @@ static void deliver_read_message(Port* prt, erts_aint32_t state, Eterm to, Eterm tuple; Process* rp; Eterm* hp; - ErlHeapFragment *bp; + ErtsMessage *mp; ErlOffHeap *ohp; ErtsProcLocks rp_locks = 0; int scheduler = erts_get_scheduler_id() != 0; @@ -3118,7 +3233,7 @@ static void deliver_read_message(Port* prt, erts_aint32_t state, Eterm to, if (!rp) return; - hp = erts_alloc_message_heap(need, &bp, &ohp, rp, &rp_locks); + mp = erts_alloc_message_heap(rp, &rp_locks, need, &hp, &ohp); listp = NIL; if ((state & ERTS_PORT_SFLG_BINARY_IO) == 0) { @@ -3160,7 +3275,7 @@ static void deliver_read_message(Port* prt, erts_aint32_t state, Eterm to, tuple = TUPLE2(hp, prt->common.id, tuple); hp += 3; - erts_queue_message(rp, &rp_locks, bp, tuple, am_undefined); + erts_queue_message(rp, &rp_locks, mp, tuple, am_undefined); if (rp_locks) erts_smp_proc_unlock(rp, rp_locks); if (!scheduler) @@ -3234,7 +3349,7 @@ deliver_vec_message(Port* prt, /* Port */ Eterm tuple; Process* rp; Eterm* hp; - ErlHeapFragment *bp; + ErtsMessage *mp; ErlOffHeap *ohp; ErtsProcLocks rp_locks = 0; int scheduler = erts_get_scheduler_id() != 0; @@ -3266,7 +3381,7 @@ deliver_vec_message(Port* prt, /* Port */ need += (hlen+csize)*2; } - hp = erts_alloc_message_heap(need, &bp, &ohp, rp, &rp_locks); + mp = erts_alloc_message_heap(rp, &rp_locks, need, &hp, &ohp); listp = NIL; iov += vsize; @@ -3327,7 +3442,7 @@ deliver_vec_message(Port* prt, /* Port */ tuple = TUPLE2(hp, prt->common.id, tuple); hp += 3; - erts_queue_message(rp, &rp_locks, bp, tuple, am_undefined); + erts_queue_message(rp, &rp_locks, mp, tuple, am_undefined); erts_smp_proc_unlock(rp, rp_locks); if (!scheduler) erts_proc_dec_refc(rp); @@ -3367,6 +3482,7 @@ static void flush_port(Port *p) ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(p)); if (p->drv_ptr->flush != NULL) { + ERTS_MSACC_PUSH_STATE_M(); #ifdef USE_VM_PROBES if (DTRACE_ENABLED(driver_flush)) { DTRACE_FORMAT_COMMON_PID_AND_PORT(ERTS_PORT_GET_CONNECTED(p), p) @@ -3376,9 +3492,11 @@ static void flush_port(Port *p) if (IS_TRACED_FL(p, F_TRACE_SCHED_PORTS)) { trace_sched_ports_where(p, am_in, am_flush); } + ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_PORT); fpe_was_unmasked = erts_block_fpe(); (*p->drv_ptr->flush)((ErlDrvData)p->drv_data); erts_unblock_fpe(fpe_was_unmasked); + ERTS_MSACC_POP_STATE_M(); if (IS_TRACED_FL(p, F_TRACE_SCHED_PORTS)) { trace_sched_ports_where(p, am_out, am_flush); } @@ -3426,6 +3544,7 @@ terminate_port(Port *prt) drv = prt->drv_ptr; if ((drv != NULL) && (drv->stop != NULL)) { int fpe_was_unmasked = erts_block_fpe(); + ERTS_MSACC_PUSH_AND_SET_STATE_M(ERTS_MSACC_STATE_PORT); #ifdef USE_VM_PROBES if (DTRACE_ENABLED(driver_stop)) { DTRACE_FORMAT_COMMON_PID_AND_PORT(connected_id, prt) @@ -3434,6 +3553,7 @@ terminate_port(Port *prt) #endif (*drv->stop)((ErlDrvData)prt->drv_data); erts_unblock_fpe(fpe_was_unmasked); + ERTS_MSACC_POP_STATE_M(); #ifdef ERTS_SMP if (prt->xports) erts_port_handle_xports(prt); @@ -3744,6 +3864,7 @@ call_driver_control(Eterm caller, ErlDrvSizeT *from_size) { ErlDrvSSizeT cres; + ERTS_MSACC_PUSH_STATE_M(); if (!prt->drv_ptr->control) return ERTS_PORT_OP_BADARG; @@ -3757,6 +3878,8 @@ call_driver_control(Eterm caller, command, size); } #endif + + ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_PORT); prt->caller = caller; cres = prt->drv_ptr->control((ErlDrvData) prt->drv_data, @@ -3767,6 +3890,8 @@ call_driver_control(Eterm caller, *from_size); prt->caller = NIL; + ERTS_MSACC_POP_STATE_M(); + if (cres < 0) return ERTS_PORT_OP_BADARG; @@ -3818,7 +3943,6 @@ write_port_control_result(int control_flags, ErlDrvSizeT resp_size, char *pre_alloc_buf, Eterm **hpp, - ErlHeapFragment *bp, ErlOffHeap *ohp) { Eterm res; @@ -3892,16 +4016,13 @@ port_sig_control(Port *prt, if (res == ERTS_PORT_OP_DONE) { Eterm msg; - Eterm *hp; - ErlHeapFragment *bp; - ErlOffHeap *ohp; ErtsHeapFactory factory; Process *rp; ErtsProcLocks rp_locks = 0; Uint hsz, rsz; int control_flags; - rp = erts_proc_lookup_raw(sigdp->caller); + rp = sigdp->caller == ERTS_INVALID_PID ? NULL : erts_proc_lookup_raw(sigdp->caller); if (!rp) goto done; @@ -3914,22 +4035,15 @@ port_sig_control(Port *prt, hsz = rsz + ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE; - hp = erts_alloc_message_heap(hsz, - &bp, - &ohp, - rp, - &rp_locks); - erts_factory_message_init(&factory, rp, hp, bp); + (void) erts_factory_message_create(&factory, rp, + &rp_locks, hsz); msg = write_port_control_result(control_flags, resp_bufp, resp_size, &resp_buf[0], - &hp, - bp, - ohp); - factory.hp = hp; - + &factory.hp, + factory.off_heap); queue_port_sched_op_reply(rp, &rp_locks, &factory, @@ -3944,7 +4058,8 @@ port_sig_control(Port *prt, /* failure */ - port_sched_op_reply(sigdp->caller, sigdp->ref, am_badarg); + if (sigdp->caller != ERTS_INVALID_PID) + port_sched_op_reply(sigdp->caller, sigdp->ref, am_badarg); done: @@ -3954,6 +4069,23 @@ done: return ERTS_PORT_REDS_CONTROL; } +/* + * This is an asynchronous control call. I.e. it will not return anything + * to the caller. + */ +int +erl_drv_port_control(Eterm port_num, char cmd, char* buff, ErlDrvSizeT size) +{ + ErtsProc2PortSigData *sigdp = erts_port_task_alloc_p2p_sig_data(); + + sigdp->flags = ERTS_P2P_SIG_TYPE_CONTROL | ERTS_P2P_SIG_DATA_FLG_REPLY; + sigdp->u.control.binp = NULL; + sigdp->u.control.command = cmd; + sigdp->u.control.bufp = buff; + sigdp->u.control.size = size; + + return erts_schedule_port2port_signal(port_num, sigdp, 0, port_sig_control); +} ErtsPortOpResult erts_port_control(Process* c_p, @@ -4070,7 +4202,6 @@ erts_port_control(Process* c_p, resp_size, &resp_buf[0], &hp, - NULL, &c_p->off_heap); BUMP_REDS(c_p, ERTS_PORT_REDS_CONTROL); return ERTS_PORT_OP_DONE; @@ -4092,10 +4223,10 @@ erts_port_control(Process* c_p, binp = NULL; if (is_binary(data) && binary_bitoffset(data) == 0) { - Eterm *ebinp = binary_val_rel(data, NULL); + Eterm *ebinp = binary_val(data); ASSERT(!tmp_alloced); if (*ebinp == HEADER_SUB_BIN) - ebinp = binary_val_rel(((ErlSubBin *) ebinp)->orig, NULL); + ebinp = binary_val(((ErlSubBin *) ebinp)->orig); if (*ebinp != HEADER_PROC_BIN) copy = 1; else { @@ -4148,6 +4279,7 @@ call_driver_call(Eterm caller, unsigned *ret_flagsp) { ErlDrvSSizeT cres; + ERTS_MSACC_PUSH_STATE_M(); if (!prt->drv_ptr->call) return ERTS_PORT_OP_BADARG; @@ -4163,6 +4295,8 @@ call_driver_call(Eterm caller, } #endif + ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_PORT); + prt->caller = caller; cres = prt->drv_ptr->call((ErlDrvData) prt->drv_data, command, @@ -4173,6 +4307,8 @@ call_driver_call(Eterm caller, ret_flagsp); prt->caller = NIL; + ERTS_MSACC_POP_STATE_M(); + if (cres <= 0 || ((byte) (*resp_bufp)[0]) != VERSION_MAGIC) return ERTS_PORT_OP_BADARG; @@ -4229,21 +4365,14 @@ port_sig_call(Port *prt, hsz = erts_decode_ext_size((byte *) resp_bufp, resp_size); if (hsz >= 0) { - ErlHeapFragment* bp; - ErlOffHeap* ohp; ErtsHeapFactory factory; byte *endp; hsz += 3; /* ok tuple */ hsz += ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE; - hp = erts_alloc_message_heap(hsz, - &bp, - &ohp, - rp, - &rp_locks); + (void) erts_factory_message_create(&factory, rp, &rp_locks, hsz); endp = (byte *) resp_bufp; - erts_factory_message_init(&factory, rp, hp, bp); msg = erts_decode_ext(&factory, &endp); if (is_value(msg)) { hp = erts_produce_heap(&factory, @@ -4504,7 +4633,9 @@ port_sig_info(Port *prt, sigdp->u.info.item); if (is_value(value)) { ErtsHeapFactory factory; - erts_factory_message_init(&factory, NULL, hp, bp); + ErtsMessage *mp = erts_alloc_message(0, NULL); + mp->data.heap_frag = bp; + erts_factory_selfcontained_message_init(&factory, mp, hp); queue_port_sched_op_reply(rp, &rp_locks, &factory, @@ -4592,8 +4723,8 @@ reply_io_bytes(void *vreq) rp = erts_proc_lookup(req->pid); if (rp) { - ErlOffHeap *ohp = NULL; - ErlHeapFragment *bp = NULL; + ErlOffHeap *ohp; + ErtsMessage *mp; ErtsProcLocks rp_locks; Eterm ref, msg, ein, eout, *hp; Uint64 in, out; @@ -4615,7 +4746,7 @@ reply_io_bytes(void *vreq) erts_bld_uint64(NULL, &hsz, in); erts_bld_uint64(NULL, &hsz, out); - hp = erts_alloc_message_heap(hsz, &bp, &ohp, rp, &rp_locks); + mp = erts_alloc_message_heap(rp, &rp_locks, hsz, &hp, &ohp); ref = make_internal_ref(hp); write_ref_thing(hp, req->refn[0], req->refn[1], req->refn[2]); @@ -4625,7 +4756,7 @@ reply_io_bytes(void *vreq) eout = erts_bld_uint64(&hp, NULL, out); msg = TUPLE4(hp, ref, make_small(sched_id), ein, eout); - erts_queue_message(rp, &rp_locks, bp, msg, NIL); + erts_queue_message(rp, &rp_locks, mp, msg, NIL); if (req->sched_id == sched_id) rp_locks &= ~ERTS_PROC_LOCK_MAIN; @@ -4731,6 +4862,10 @@ print_port_info(Port *p, int to, void *arg) erts_print(to, arg, "Port is a file: %s\n",p->name); } else if (p->drv_ptr == &spawn_driver) { erts_print(to, arg, "Port controls external process: %s\n",p->name); +#ifndef __WIN32__ + } else if (p->drv_ptr == &forker_driver) { + erts_print(to, arg, "Port controls forker process: %s\n",p->name); +#endif } else { erts_print(to, arg, "Port controls linked-in driver: %s\n",p->name); } @@ -4877,6 +5012,7 @@ int get_port_flags(ErlDrvPort ix) void erts_raw_port_command(Port* p, byte* buf, Uint len) { int fpe_was_unmasked; + ERTS_MSACC_PUSH_STATE_M(); ERTS_SMP_CHK_NO_PROC_LOCKS; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(p)); @@ -4897,9 +5033,11 @@ void erts_raw_port_command(Port* p, byte* buf, Uint len) DTRACE4(driver_output, "-raw-", port_str, p->name, len); } #endif + ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_PORT); fpe_was_unmasked = erts_block_fpe(); (*p->drv_ptr->output)((ErlDrvData)p->drv_data, (char*) buf, (int) len); erts_unblock_fpe(fpe_was_unmasked); + ERTS_MSACC_POP_STATE_M(); } int async_ready(Port *p, void* data) @@ -4911,6 +5049,7 @@ int async_ready(Port *p, void* data) if (p) { ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(p)); if (p->drv_ptr->ready_async != NULL) { + ERTS_MSACC_PUSH_AND_SET_STATE_M(ERTS_MSACC_STATE_PORT); #ifdef USE_VM_PROBES if (DTRACE_ENABLED(driver_ready_async)) { DTRACE_FORMAT_COMMON_PID_AND_PORT(ERTS_PORT_GET_CONNECTED(p), p) @@ -4919,6 +5058,7 @@ int async_ready(Port *p, void* data) #endif (*p->drv_ptr->ready_async)((ErlDrvData)p->drv_data, data); need_free = 0; + ERTS_MSACC_POP_STATE_M(); } erts_port_driver_callback_epilogue(p, NULL); @@ -5070,11 +5210,11 @@ ErlDrvTermData driver_mk_term_nil(void) void driver_report_exit(ErlDrvPort ix, int status) { Eterm* hp; + ErlOffHeap *ohp; Eterm tuple; Process *rp; Eterm pid; - ErlHeapFragment *bp = NULL; - ErlOffHeap *ohp; + ErtsMessage *mp; ErtsProcLocks rp_locks = 0; int scheduler = erts_get_scheduler_id() != 0; Port* prt = erts_drvport2port(ix); @@ -5094,13 +5234,13 @@ void driver_report_exit(ErlDrvPort ix, int status) if (!rp) return; - hp = erts_alloc_message_heap(3+3, &bp, &ohp, rp, &rp_locks); + mp = erts_alloc_message_heap(rp, &rp_locks, 3+3, &hp, &ohp); tuple = TUPLE2(hp, am_exit_status, make_small(status)); hp += 3; tuple = TUPLE2(hp, prt->common.id, tuple); - erts_queue_message(rp, &rp_locks, bp, tuple, am_undefined); + erts_queue_message(rp, &rp_locks, mp, tuple, am_undefined); erts_smp_proc_unlock(rp, rp_locks); if (!scheduler) @@ -5210,7 +5350,6 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) ErtsProcLocks rp_locks = 0; struct b2t_states__ b2t; int scheduler; - int is_heap_need_limited = 1; ErtsSchedulerData *esdp = erts_get_scheduler_data(); ERTS_UNDEF(mess,NIL); @@ -5249,25 +5388,17 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) break; case ERL_DRV_INT: /* signed int argument */ ERTS_DDT_CHK_ENOUGH_ARGS(1); -#if HALFWORD_HEAP - erts_bld_sint64(NULL, &need, (Sint64)ptr[0]); -#else /* check for bignum */ if (!IS_SSMALL((Sint)ptr[0])) need += BIG_UINT_HEAP_SIZE; /* use small_to_big */ -#endif ptr++; depth++; break; case ERL_DRV_UINT: /* unsigned int argument */ ERTS_DDT_CHK_ENOUGH_ARGS(1); -#if HALFWORD_HEAP - erts_bld_uint64(NULL, &need, (Uint64)ptr[0]); -#else /* check for bignum */ if (!IS_USMALL(0, (Uint)ptr[0])) need += BIG_UINT_HEAP_SIZE; /* use small_to_big */ -#endif ptr++; depth++; break; @@ -5387,9 +5518,6 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) need += hsz; ptr += 2; depth++; - if (size > MAP_SMALL_MAP_LIMIT*3) { /* may contain big map */ - is_heap_need_limited = 0; - } break; } case ERL_DRV_MAP: { /* int */ @@ -5397,7 +5525,6 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) if ((int) ptr[0] < 0) ERTS_DDT_FAIL; if (ptr[0] > MAP_SMALL_MAP_LIMIT) { need += HASHMAP_ESTIMATED_HEAP_SIZE(ptr[0]); - is_heap_need_limited = 0; } else { need += MAP_HEADER_FLATMAP_SZ + 1 + 2*ptr[0]; } @@ -5436,17 +5563,7 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) goto done; } - /* Try copy directly to destination heap if we know there are no big maps */ - if (is_heap_need_limited) { - ErlOffHeap *ohp; - ErlHeapFragment* bp; - Eterm* hp = erts_alloc_message_heap(need, &bp, &ohp, rp, &rp_locks); - erts_factory_message_init(&factory, rp, hp, bp); - } - else { - erts_factory_message_init(&factory, NULL, NULL, - new_message_buffer(need)); - } + (void) erts_factory_message_create(&factory, rp, &rp_locks, need); /* * Interpret the instructions and build the term. @@ -5466,10 +5583,6 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) break; case ERL_DRV_INT: /* signed int argument */ -#if HALFWORD_HEAP - erts_reserve_heap(&factory, BIG_NEED_SIZE(2)); - mess = erts_bld_sint64(&factory.hp, NULL, (Sint64)ptr[0]); -#else erts_reserve_heap(&factory, BIG_UINT_HEAP_SIZE); if (IS_SSMALL((Sint)ptr[0])) mess = make_small((Sint)ptr[0]); @@ -5477,15 +5590,10 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) mess = small_to_big((Sint)ptr[0], factory.hp); factory.hp += BIG_UINT_HEAP_SIZE; } -#endif ptr++; break; case ERL_DRV_UINT: /* unsigned int argument */ -#if HALFWORD_HEAP - erts_reserve_heap(&factory, BIG_NEED_FOR_BITS(64)); - mess = erts_bld_uint64(&factory.hp, NULL, (Uint64)ptr[0]); -#else erts_reserve_heap(&factory, BIG_UINT_HEAP_SIZE); if (IS_USMALL(0, (Uint)ptr[0])) mess = make_small((Uint)ptr[0]); @@ -5493,7 +5601,6 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) mess = uint_to_big((Uint)ptr[0], factory.hp); factory.hp += BIG_UINT_HEAP_SIZE; } -#endif ptr++; break; @@ -5725,9 +5832,9 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) if (res > 0) { mess = ESTACK_POP(stack); /* get resulting value */ - erts_factory_close(&factory); + erts_factory_trim_and_close(&factory, &mess, 1); /* send message */ - erts_queue_message(rp, &rp_locks, factory.heap_frags, mess, am_undefined); + erts_queue_message(rp, &rp_locks, factory.message, mess, am_undefined); } else { if (b2t.ix > b2t.used) @@ -6832,7 +6939,7 @@ int driver_monitor_process(ErlDrvPort drvport, { Port *prt; int ret; -#if !HEAP_ON_C_STACK || (defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)) +#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) ErtsSchedulerData *sched = erts_get_scheduler_data(); #endif @@ -6843,16 +6950,6 @@ int driver_monitor_process(ErlDrvPort drvport, /* Now (in SMP) we should have either the port lock (if we have a scheduler) or the port data lock (if we're a driver thread) */ ERTS_SMP_LC_ASSERT((sched != NULL || prt->port_data_lock)); - -#if !HEAP_ON_C_STACK - if (!sched) { - /* Need a separate allocation for the ref :( */ - Eterm *buf = erts_alloc(ERTS_ALC_T_TEMP_TERM, - sizeof(Eterm)*REF_THING_SIZE); - ret = do_driver_monitor_process(prt,buf,process,monitor); - erts_free(ERTS_ALC_T_TEMP_TERM,buf); - } else -#endif { DeclareTmpHeapNoproc(buf,REF_THING_SIZE); UseTmpHeapNoproc(REF_THING_SIZE); @@ -6905,7 +7002,7 @@ int driver_demonitor_process(ErlDrvPort drvport, { Port *prt; int ret; -#if !HEAP_ON_C_STACK || (defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)) +#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) ErtsSchedulerData *sched = erts_get_scheduler_data(); #endif @@ -6916,15 +7013,6 @@ int driver_demonitor_process(ErlDrvPort drvport, /* Now we should have either the port lock (if we have a scheduler) or the port data lock (if we're a driver thread) */ ERTS_SMP_LC_ASSERT((sched != NULL || prt->port_data_lock)); -#if !HEAP_ON_C_STACK - if (!sched) { - /* Need a separate allocation for the ref :( */ - Eterm *buf = erts_alloc(ERTS_ALC_T_TEMP_TERM, - sizeof(Eterm)*REF_THING_SIZE); - ret = do_driver_demonitor_process(prt,buf,monitor); - erts_free(ERTS_ALC_T_TEMP_TERM,buf); - } else -#endif { DeclareTmpHeapNoproc(buf,REF_THING_SIZE); UseTmpHeapNoproc(REF_THING_SIZE); @@ -6960,7 +7048,7 @@ ErlDrvTermData driver_get_monitored_process(ErlDrvPort drvport, { Port *prt; ErlDrvTermData ret; -#if !HEAP_ON_C_STACK || (defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)) +#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) ErtsSchedulerData *sched = erts_get_scheduler_data(); #endif @@ -6971,16 +7059,6 @@ ErlDrvTermData driver_get_monitored_process(ErlDrvPort drvport, /* Now we should have either the port lock (if we have a scheduler) or the port data lock (if we're a driver thread) */ ERTS_SMP_LC_ASSERT((sched != NULL || prt->port_data_lock)); - -#if !HEAP_ON_C_STACK - if (!sched) { - /* Need a separate allocation for the ref :( */ - Eterm *buf = erts_alloc(ERTS_ALC_T_TEMP_TERM, - sizeof(Eterm)*REF_THING_SIZE); - ret = do_driver_get_monitored_process(prt,buf,monitor); - erts_free(ERTS_ALC_T_TEMP_TERM,buf); - } else -#endif { DeclareTmpHeapNoproc(buf,REF_THING_SIZE); UseTmpHeapNoproc(REF_THING_SIZE); @@ -7004,6 +7082,7 @@ void erts_fire_port_monitor(Port *prt, Eterm ref) void (*callback)(ErlDrvData drv_data, ErlDrvMonitor *monitor); ErlDrvMonitor drv_monitor; int fpe_was_unmasked; + ERTS_MSACC_PUSH_STATE_M(); ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); ASSERT(prt->drv_ptr != NULL); @@ -7015,6 +7094,7 @@ void erts_fire_port_monitor(Port *prt, Eterm ref) callback = prt->drv_ptr->process_exit; ASSERT(callback != NULL); ref_to_driver_monitor(ref,&drv_monitor); + ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_PORT); DRV_MONITOR_UNLOCK_PDL(prt); #ifdef USE_VM_PROBES if (DTRACE_ENABLED(driver_process_exit)) { @@ -7026,6 +7106,7 @@ void erts_fire_port_monitor(Port *prt, Eterm ref) (*callback)((ErlDrvData) (prt->drv_data), &drv_monitor); erts_unblock_fpe(fpe_was_unmasked); DRV_MONITOR_LOCK_PDL(prt); + ERTS_MSACC_POP_STATE_M(); /* remove monitor *after* callback */ rmon = erts_remove_monitor(&ERTS_P_MONITORS(prt), ref); DRV_MONITOR_UNLOCK_PDL(prt); @@ -7046,6 +7127,9 @@ driver_failure_term(ErlDrvPort ix, Eterm term, int eof) if (prt == ERTS_INVALID_ERL_DRV_PORT) return -1; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); + + if (prt->async_open_port) + init_ack_send_reply(prt, prt->common.id); if (eof) flush_linebuf_messages(prt, state); if (state & ERTS_PORT_SFLG_CLOSING) { diff --git a/erts/emulator/beam/module.c b/erts/emulator/beam/module.c index 86dd3b5aac..f5c7b177d3 100644 --- a/erts/emulator/beam/module.c +++ b/erts/emulator/beam/module.c @@ -74,8 +74,8 @@ static Module* module_alloc(Module* tmpl) erts_smp_atomic_add_nob(&tot_module_bytes, sizeof(Module)); obj->module = tmpl->module; - obj->curr.code = 0; - obj->old.code = 0; + obj->curr.code_hdr = 0; + obj->old.code_hdr = 0; obj->curr.code_length = 0; obj->old.code_length = 0; obj->slot.index = -1; @@ -103,6 +103,9 @@ void init_module_table(void) f.cmp = (HCMP_FUN) module_cmp; f.alloc = (HALLOC_FUN) module_alloc; f.free = (HFREE_FUN) module_free; + f.meta_alloc = (HMALLOC_FUN) erts_alloc; + f.meta_free = (HMFREE_FUN) erts_free; + f.meta_print = (HMPRINT_FUN) erts_print; for (i = 0; i < ERTS_NUM_CODE_IX; i++) { erts_index_init(ERTS_ALC_T_MODULE_TABLE, &module_tables[i], "module_code", diff --git a/erts/emulator/beam/module.h b/erts/emulator/beam/module.h index c8a6351b04..e66d628ca9 100644 --- a/erts/emulator/beam/module.h +++ b/erts/emulator/beam/module.h @@ -26,7 +26,7 @@ #endif struct erl_module_instance { - BeamInstr* code; + BeamCodeHeader* code_hdr; int code_length; /* Length of loaded code in bytes. */ unsigned catches; struct erl_module_nif* nif; diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab index 1d32e72247..9e53b4bfcc 100644 --- a/erts/emulator/beam/ops.tab +++ b/erts/emulator/beam/ops.tab @@ -39,8 +39,8 @@ too_old_compiler | never() => # necessary.) Since the instructions don't work correctly in R12B, simply # refuse to load the module. -func_info M=a a==am_module_info A=u==0 | label L | move n r => too_old_compiler -func_info M=a a==am_module_info A=u==1 | label L | move n r => too_old_compiler +func_info M=a a==am_module_info A=u==0 | label L | move n x==0 => too_old_compiler +func_info M=a a==am_module_info A=u==1 | label L | move n x==0 => too_old_compiler # The undocumented and unsupported guard BIF is_constant/1 was removed # in R13. The is_constant/2 operation is marked as obsolete in genop.tab, @@ -76,17 +76,6 @@ return # with the following call instruction, we need to make sure that # there is no line/1 instruction between the move and the call. # - -move S r | line Loc | call_ext Ar Func => \ - line Loc | move S r | call_ext Ar Func -move S r | line Loc | call_ext_last Ar Func=u$is_bif D => \ - line Loc | move S r | call_ext_last Ar Func D -move S r | line Loc | call_ext_only Ar Func=u$is_bif => \ - line Loc | move S r | call_ext_only Ar Func -move S r | line Loc | call Ar Func => \ - line Loc | move S r | call Ar Func - -# # A tail-recursive call to an external function (non-BIF) will # never be saved on the stack, so there is no reason to keep # the line instruction. (The compiler did not remove the line @@ -94,10 +83,14 @@ move S r | line Loc | call Ar Func => \ # BIFs and ordinary Erlang functions.) # -line Loc | call_ext_last Ar Func=u$is_not_bif D => \ - call_ext_last Ar Func D -line Loc | call_ext_only Ar Func=u$is_not_bif => \ - call_ext_only Ar Func +move S X0=x==0 | line Loc | call_ext Ar Func => \ + line Loc | move S X0 | call_ext Ar Func +move S X0=x==0 | line Loc | call_ext_last Ar Func=u$is_not_bif D => \ + move S X0 | call_ext_last Ar Func D +move S X0=x==0 | line Loc | call_ext_only Ar Func=u$is_not_bif => \ + move S X0 | call_ext_only Ar Func +move S X0=x==0 | line Loc | call Ar Func => \ + line Loc | move S X0 | call Ar Func line Loc | func_info M F A => func_info M F A | line Loc @@ -167,31 +160,24 @@ is_tuple Fail=f S | select_tuple_arity S=d Fail=f Size=u Rest=* => \ select_tuple_arity S=d Fail=f Size=u Rest=* => \ gen_select_tuple_arity(S, Fail, Size, Rest) -i_select_val_bins r f I i_select_val_bins x f I i_select_val_bins y f I -i_select_val_lins r f I i_select_val_lins x f I i_select_val_lins y f I -i_select_val2 r f c c f f i_select_val2 x f c c f f i_select_val2 y f c c f f -i_select_tuple_arity r f I i_select_tuple_arity x f I i_select_tuple_arity y f I -i_select_tuple_arity2 r f A A f f i_select_tuple_arity2 x f A A f f i_select_tuple_arity2 y f A A f f -i_jump_on_val_zero r f I i_jump_on_val_zero x f I i_jump_on_val_zero y f I -i_jump_on_val r f I I i_jump_on_val x f I I i_jump_on_val y f I I @@ -203,30 +189,21 @@ is_ne_exact L1 S1 S2 | jump Fail | label L2 | same_label(L1, L2) => \ %macro: get_list GetList -pack get_list x x x get_list x x y -get_list x x r get_list x y x get_list x y y -get_list x y r -get_list x r x -get_list x r y get_list y x x get_list y x y -get_list y x r get_list y y x get_list y y y -get_list y y r -get_list y r x -get_list y r y +# The following get_list instructions using x(0) are frequently used. get_list r x x +get_list r r y +get_list x r x get_list r x y -get_list r x r -get_list r y x -get_list r y y get_list r y r -get_list r r x -get_list r r y +get_list r x r # Old-style catch. catch y f @@ -237,33 +214,31 @@ try Y F => catch Y F try_case Y => try_end Y try_end y -try_case_end Literal=q => move Literal x | try_case_end x try_case_end s # Destructive set tuple element -set_tuple_element Lit=q Tuple Pos => move Lit x | set_tuple_element x Tuple Pos set_tuple_element s d P # Get tuple element %macro: i_get_tuple_element GetTupleElement -pack i_get_tuple_element x P x -i_get_tuple_element r P x i_get_tuple_element y P x -i_get_tuple_element x P r -i_get_tuple_element y P r %cold -i_get_tuple_element r P r i_get_tuple_element x P y -i_get_tuple_element r P y i_get_tuple_element y P y %hot +%macro: i_get_tuple_element2 GetTupleElement2 -pack +i_get_tuple_element2 x P x + +%macro: i_get_tuple_element3 GetTupleElement3 -pack +i_get_tuple_element3 x P x + %macro: is_number IsNumber -fail_action %cold -is_number f r is_number f x is_number f y %hot @@ -273,16 +248,12 @@ is_number Fail Literal=q => move Literal x | is_number Fail x jump f -case_end Literal=cq => move Literal x | case_end x -badmatch Literal=cq => move Literal x | badmatch x +case_end NotInX=cy => move NotInX x | case_end x +badmatch NotInX=cy => move NotInX x | badmatch x -case_end r case_end x -case_end y -badmatch r badmatch x -badmatch y if_end raise s s @@ -291,7 +262,7 @@ raise s s badarg j system_limit j -move C=cxy r | jump Lbl => move_jump Lbl C +move C=cxy x==0 | jump Lbl => move_jump Lbl C %macro: move_jump MoveJump -nonext move_jump f n @@ -309,8 +280,6 @@ move_window/6 # x -> y -move S1=r S2=y | move X1=x Y1=y => move2 S1 S2 X1 Y1 - move X1=x Y1=y | move X2=x Y2=y | move X3=x Y3=y | succ(Y1,Y2) | succ(Y2,Y3) => \ move_window X1 X2 X3 Y1 Y3 @@ -323,24 +292,76 @@ move_window X1=x X2=x X3=x X4=x Y1=y Y4=y | move X5=x Y5=y | succ(Y4,Y5) => \ move_window X1=x X2=x X3=x Y1=y Y3=y => move_window3 X1 X2 X3 Y1 move_window X1=x X2=x X3=x X4=x Y1=y Y4=y => move_window4 X1 X2 X3 X4 Y1 +%macro: move_window3 MoveWindow3 -pack +%macro: move_window4 MoveWindow4 -pack +%macro: move_window5 MoveWindow5 -pack + move_window3 x x x y move_window4 x x x x y move_window5 x x x x x y -move X1=x Y1=y | move X2=x Y2=y => move2 X1 Y1 X2 Y2 -move Y1=y X1=x | move Y2=y X2=x => move2 Y1 X1 Y2 X2 -move X1=x X2=x | move X3=x X4=x => move2 X1 X2 X3 X4 +# Swap registers. +move R1=x Tmp=x | move R2=xy R1 | move Tmp R2 => swap_temp R1 R2 Tmp + +swap_temp R1 R2 Tmp | line Loc | apply Live | is_killed(Tmp, Live) => \ + swap R1 R2 | line Loc | apply Live + +swap_temp R1 R2 Tmp | line Loc | call Live Addr | is_killed(Tmp, Live) => \ + swap R1 R2 | line Loc | call Live Addr +swap_temp R1 R2 Tmp | call_only Live Addr | \ + is_killed(Tmp, Live) => swap R1 R2 | call_only Live Addr +swap_temp R1 R2 Tmp | call_last Live Addr D | \ + is_killed(Tmp, Live) => swap R1 R2 | call_last Live Addr D -move S1=x S2=r | move S3=x S4=x => move2 S1 S2 S3 S4 -move S1=x S2=r | move X1=x Y1=y => move2 S1 S2 X1 Y1 -move S1=y S2=r | move X1=x Y1=y => move2 S1 S2 X1 Y1 +swap_temp R1 R2 Tmp | line Loc | call_ext Live Addr | is_killed(Tmp, Live) => \ + swap R1 R2 | line Loc | call_ext Live Addr +swap_temp R1 R2 Tmp | line Loc | call_ext_only Live Addr | \ + is_killed(Tmp, Live) => swap R1 R2 | line Loc | call_ext_only Live Addr +swap_temp R1 R2 Tmp | line Loc | call_ext_last Live Addr D | \ + is_killed(Tmp, Live) => swap R1 R2 | line Loc | call_ext_last Live Addr D -move Y1=y X1=x | move S1=r D1=x => move2 Y1 X1 S1 D1 -move S1=r D1=x | move Y1=y X1=x => move2 S1 D1 Y1 X1 +%macro: swap_temp SwapTemp -pack +swap_temp x x x +swap_temp x y x -move2 X1=x Y1=y X2=x Y2=y | move X3=x Y3=y => move3 X1 Y1 X2 Y2 X3 Y3 -move2 Y1=y X1=x Y2=y X2=x | move Y3=y X3=x => move3 Y1 X1 Y2 X2 Y3 X3 -move2 X1=x X2=x X3=x X4=x | move X5=x X6=x => move3 X1 X2 X3 X4 X5 X6 +%macro: swap Swap -pack +swap x x +swap x y + +move Src=x D1=x | move Src=x D2=x => move_dup Src D1 D2 +move Src=x SD=x | move SD=x D=x => move_dup Src SD D +move Src=x D1=x | move Src=x D2=y => move_dup Src D1 D2 +move Src=y SD=x | move SD=x D=y => move_dup Src SD D +move Src=x SD=x | move SD=x D=y => move_dup Src SD D +move Src=y SD=x | move SD=x D=x => move_dup Src SD D + +move SD=x D=x | move Src=xy SD=x => move_shift Src SD D +move SD=y D=x | move Src=x SD=y => move_shift Src SD D +move SD=x D=y | move Src=x SD=x => move_shift Src SD D + +# The transformations above guarantee that the source for +# the second move is not the same as the destination for +# the first move. That means that we can do the moves in +# parallel (fetch both values, then store them) which could +# be faster. + +move X1=x Y1=y | move X2=x Y2=y => move2_par X1 Y1 X2 Y2 +move Y1=y X1=x | move Y2=y X2=x => move2_par Y1 X1 Y2 X2 + +move X1=x X2=x | move X3=x X4=x => move2_par X1 X2 X3 X4 + +move X1=x X2=x | move X3=x Y1=y => move2_par X1 X2 X3 Y1 + +move S1=x S2=x | move X1=x Y1=y => move2_par S1 S2 X1 Y1 + +move S1=y S2=x | move X1=x Y1=y => move2_par S1 S2 X1 Y1 + +move Y1=y X1=x | move S1=x D1=x => move2_par Y1 X1 S1 D1 +move S1=x D1=x | move Y1=y X1=x => move2_par S1 D1 Y1 X1 + +move2_par X1=x Y1=y X2=x Y2=y | move X3=x Y3=y => move3 X1 Y1 X2 Y2 X3 Y3 +move2_par Y1=y X1=x Y2=y X2=x | move Y3=y X3=x => move3 Y1 X1 Y2 X2 Y3 X3 +move2_par X1=x X2=x X3=x X4=x | move X5=x X6=x => move3 X1 X2 X3 X4 X5 X6 move C=aiq X=x==1 => move_x1 C move C=aiq X=x==2 => move_x2 C @@ -348,21 +369,32 @@ move C=aiq X=x==2 => move_x2 C move_x1 c move_x2 c -%macro: move2 Move2 -pack -move2 x y x y -move2 y x y x -move2 x x x x +%macro: move_shift MoveShift -pack +move_shift x x x +move_shift y x x +move_shift x y x +move_shift x x y + +%macro: move_dup MoveDup -pack +move_dup x x x +move_dup x x y +move_dup y x x +move_dup y x y -move2 x r x x +%macro: move2_par Move2Par -pack -move2 x r x y -move2 r y x y -move2 y r x y +move2_par x y x y +move2_par y x y x +move2_par x x x x -move2 r x y x -move2 y x r x +move2_par x x x y -%macro: move3 Move3 +move2_par y x x y + +move2_par x x y x +move2_par y x x x + +%macro: move3 Move3 -pack move3 x y x y x y move3 y x y x y x move3 x x x x x x @@ -375,20 +407,22 @@ move S=c D=y => move S x | move x D %macro:move Move -pack -gen_dest move x x move x y -move x r move y x -move y r -move r x -move r y -move c r move c x move n x -move n r move y y +# The following move instructions using x(0) are frequently used. + +move x r +move r x +move y r +move c r +move r y + # Receive operations. -loop_rec Fail Src | smp_mark_target_label(Fail) => i_loop_rec Fail Src +loop_rec Fail x==0 | smp_mark_target_label(Fail) => i_loop_rec Fail label L | wait_timeout Fail Src | smp_already_locked(L) => label L | i_wait_timeout_locked Fail Src wait_timeout Fail Src => i_wait_timeout Fail Src @@ -403,7 +437,7 @@ label L | timeout | smp_already_locked(L) => label L | timeout_locked remove_message timeout timeout_locked -i_loop_rec f r +i_loop_rec f loop_rec_end f wait f wait_locked f @@ -421,93 +455,57 @@ send # Optimized comparisons with one immediate/literal operand. # -is_eq_exact Lbl R=rxy C=ian => i_is_eq_exact_immed Lbl R C -is_eq_exact Lbl R=rxy C=q => i_is_eq_exact_literal R Lbl C +is_eq_exact Lbl R=xy C=ian => i_is_eq_exact_immed Lbl R C +is_eq_exact Lbl R=xy C=q => i_is_eq_exact_literal Lbl R C -is_ne_exact Lbl R=rxy C=ian => i_is_ne_exact_immed Lbl R C -is_ne_exact Lbl R=rxy C=q => i_is_ne_exact_literal R Lbl C +is_ne_exact Lbl R=xy C=ian => i_is_ne_exact_immed Lbl R C +is_ne_exact Lbl R=xy C=q => i_is_ne_exact_literal Lbl R C %macro: i_is_eq_exact_immed EqualImmed -fail_action i_is_eq_exact_immed f r c i_is_eq_exact_immed f x c i_is_eq_exact_immed f y c -i_is_eq_exact_literal r f c -i_is_eq_exact_literal x f c -i_is_eq_exact_literal y f c +i_is_eq_exact_literal f x c +i_is_eq_exact_literal f y c %macro: i_is_ne_exact_immed NotEqualImmed -fail_action -i_is_ne_exact_immed f r c i_is_ne_exact_immed f x c i_is_ne_exact_immed f y c -i_is_ne_exact_literal r f c -i_is_ne_exact_literal x f c -i_is_ne_exact_literal y f c +i_is_ne_exact_literal f x c +i_is_ne_exact_literal f y c -# -# Common Compare Specializations -# We don't do all of them since we want -# to keep the instruction set small-ish -# - -is_eq_exact Lbl S1=xy S2=r => is_eq_exact Lbl S2 S1 -is_eq_exact Lbl S1=rx S2=xy => i_is_eq_exact_spec Lbl S1 S2 -%macro: i_is_eq_exact_spec EqualExact -fail_action +is_eq_exact Lbl Y=y X=x => is_eq_exact Lbl X Y +%macro: is_eq_exact EqualExact -fail_action -pack +is_eq_exact f x x +is_eq_exact f x y +is_eq_exact f s s -i_is_eq_exact_spec f x x -i_is_eq_exact_spec f x y -i_is_eq_exact_spec f r x -i_is_eq_exact_spec f r y +%macro: is_lt IsLessThan -fail_action +is_lt f x x +is_lt f x c +is_lt f c x %cold -i_is_eq_exact_spec f r r +is_lt f s s %hot -is_lt Lbl S1=rxc S2=rxc => i_is_lt_spec Lbl S1 S2 - -%macro: i_is_lt_spec IsLessThan -fail_action - -i_is_lt_spec f x x -i_is_lt_spec f x r -i_is_lt_spec f x c -i_is_lt_spec f r x -i_is_lt_spec f r c -i_is_lt_spec f c x -i_is_lt_spec f c r +%macro: is_ge IsGreaterEqual -fail_action +is_ge f x x +is_ge f x c +is_ge f c x %cold -i_is_lt_spec f r r -i_is_lt_spec f c c +is_ge f s s %hot -is_ge Lbl S1=xc S2=xc => i_is_ge_spec Lbl S1 S2 - -%macro: i_is_ge_spec IsGreaterEqual -fail_action - -i_is_ge_spec f x x -i_is_ge_spec f x c -i_is_ge_spec f c x -%cold -i_is_ge_spec f c c -%hot - -# -# All other comparisons. -# - -is_eq_exact Lbl S1 S2 => i_fetch S1 S2 | i_is_eq_exact Lbl -is_ne_exact Lbl S1 S2 => i_fetch S1 S2 | i_is_ne_exact Lbl +%macro: is_ne_exact NotEqualExact -fail_action +is_ne_exact f s s -is_lt Lbl S1 S2 => i_fetch S1 S2 | i_is_lt Lbl -is_ge Lbl S1 S2 => i_fetch S1 S2 | i_is_ge Lbl -is_eq Lbl S1 S2 => i_fetch S1 S2 | i_is_eq Lbl -is_ne Lbl S1 S2 => i_fetch S1 S2 | i_is_ne Lbl +%macro: is_eq Equal -fail_action +is_eq f s s -i_is_eq_exact f -i_is_ne_exact f -i_is_lt f -i_is_ge f -i_is_eq f -i_is_ne f +%macro: is_ne NotEqual -fail_action +is_ne f s s # # Putting things. @@ -525,7 +523,6 @@ i_put_tuple Dst Arity Puts=* | put S => \ i_put_tuple/2 %macro:i_put_tuple PutTuple -pack -goto:do_put_tuple -i_put_tuple r I i_put_tuple x I i_put_tuple y I @@ -542,74 +539,36 @@ put_list x n x put_list y n x put_list x x x put_list y x x -put_list x x r -put_list y r r put_list y y x put_list x y x -put_list r x x -put_list r y x -put_list r x r -put_list y y r -put_list y r x -put_list r n x -put_list x r x -put_list x y r -put_list y x r put_list y x x -put_list x r r - # put_list SrcReg Constant Dst -put_list r c r -put_list r c x -put_list r c y -put_list x c r put_list x c x put_list x c y -put_list y c r put_list y c x -put_list y c y # put_list Constant SrcReg Dst -put_list c r r -put_list c r x -put_list c r y -put_list c x r put_list c x x -put_list c x y - -put_list c y r put_list c y x -put_list c y y -%cold -put_list s s d -%hot +# The following put_list instructions using x(0) are frequently used. -%macro: i_fetch FetchArgs -pack -i_fetch c r -i_fetch c x -i_fetch c y -i_fetch r c -i_fetch r x -i_fetch r y -i_fetch x c -i_fetch x r -i_fetch x x -i_fetch x y -i_fetch y c -i_fetch y r -i_fetch y x -i_fetch y y +put_list y r r +put_list x r r +put_list r n r +put_list r n x +put_list r x x +put_list r x r +put_list x x r %cold -i_fetch c c -i_fetch s s +put_list s s d %hot # @@ -631,27 +590,27 @@ return_trace # Note: There is no 'move_return y r', since there never are any y registers # when we do move_return (if we have y registers, we must do move_deallocate_return). -move S r | return => move_return S r +move S x==0 | return => move_return S %macro: move_return MoveReturn -nonext -move_return x r -move_return c r -move_return n r +move_return x +move_return c +move_return n -move S r | deallocate D | return => move_deallocate_return S r D +move S x==0 | deallocate D | return => move_deallocate_return S D %macro: move_deallocate_return MoveDeallocateReturn -pack -nonext -move_deallocate_return x r Q -move_deallocate_return y r Q -move_deallocate_return c r Q -move_deallocate_return n r Q +move_deallocate_return x Q +move_deallocate_return y Q +move_deallocate_return c Q +move_deallocate_return n Q deallocate D | return => deallocate_return D %macro: deallocate_return DeallocateReturn -nonext deallocate_return Q -test_heap Need u==1 | put_list Y=y r r => test_heap_1_put_list Need Y +test_heap Need u==1 | put_list Y=y x==0 x==0 => test_heap_1_put_list Need Y %macro: test_heap_1_put_list TestHeapPutList -pack test_heap_1_put_list I y @@ -660,18 +619,18 @@ test_heap_1_put_list I y is_tuple Fail Literal=q => move Literal x | is_tuple Fail x is_tuple Fail=f c => jump Fail -is_tuple Fail=f S=rxy | test_arity Fail=f S=rxy Arity => is_tuple_of_arity Fail S Arity +is_tuple Fail=f S=xy | test_arity Fail=f S=xy Arity => is_tuple_of_arity Fail S Arity %macro:is_tuple_of_arity IsTupleOfArity -fail_action +is_tuple_of_arity f r A is_tuple_of_arity f x A is_tuple_of_arity f y A -is_tuple_of_arity f r A %macro: is_tuple IsTuple -fail_action +is_tuple f r is_tuple f x is_tuple f y -is_tuple f r test_arity Fail Literal=q Arity => move Literal x | test_arity Fail x Arity test_arity Fail=f c Arity => jump Fail @@ -679,95 +638,63 @@ test_arity Fail=f c Arity => jump Fail %macro: test_arity IsArity -fail_action test_arity f x A test_arity f y A -test_arity f r A - -is_tuple_of_arity Fail=f Reg Arity | get_tuple_element Reg P=u==0 Dst=xy => \ - is_tuple_of_arity Fail Reg Arity | extract_next_element Dst | original_reg Reg P - -test_arity Fail Reg Arity | get_tuple_element Reg P=u==0 Dst=xy => \ - test_arity Fail Reg Arity | extract_next_element Dst | original_reg Reg P - -original_reg Reg P1 | get_tuple_element Reg P2 Dst=xy | succ(P1, P2) => \ - extract_next_element Dst | original_reg Reg P2 - -get_tuple_element Reg P Dst => i_get_tuple_element Reg P Dst | original_reg Reg P - -original_reg Reg Pos => -original_reg/2 +get_tuple_element Reg=x P1 D1=x | get_tuple_element Reg=x P2 D2=x | \ + get_tuple_element Reg=x P3 D3=x | \ + succ(P1, P2) | succ(P2, P3) | \ + succ(D1, D2) | succ(D2, D3) => i_get_tuple_element3 Reg P1 D1 -extract_next_element D1=xy | original_reg Reg P1 | get_tuple_element Reg P2 D2=xy | \ -succ(P1, P2) | succ(D1, D2) => \ - extract_next_element2 D1 | original_reg Reg P2 +get_tuple_element Reg=x P1 D1=x | get_tuple_element Reg=x P2 D2=x | \ + succ(P1, P2) | succ(D1, D2) => i_get_tuple_element2 Reg P1 D1 -extract_next_element2 D1=xy | original_reg Reg P1 | get_tuple_element Reg P2 D2=xy | \ -succ(P1, P2) | succ2(D1, D2) => \ - extract_next_element3 D1 | original_reg Reg P2 - -#extract_next_element3 D1=xy | original_reg Reg P1 | get_tuple_element Reg P2 D2=xy | \ -#succ(P1, P2) | succ3(D1, D2) => \ -# extract_next_element4 D1 | original_reg Reg P2 - -%macro: extract_next_element ExtractNextElement -pack -extract_next_element x -extract_next_element y - -%macro: extract_next_element2 ExtractNextElement2 -pack -extract_next_element2 x -extract_next_element2 y - -%macro: extract_next_element3 ExtractNextElement3 -pack -extract_next_element3 x -extract_next_element3 y - -#%macro: extract_next_element4 ExtractNextElement4 -pack -#extract_next_element4 x -#extract_next_element4 y +get_tuple_element Reg P Dst => i_get_tuple_element Reg P Dst is_integer Fail=f i => is_integer Fail=f an => jump Fail is_integer Fail Literal=q => move Literal x | is_integer Fail x -is_integer Fail=f S=rx | allocate Need Regs => is_integer_allocate Fail S Need Regs +is_integer Fail=f S=x | allocate Need Regs => is_integer_allocate Fail S Need Regs %macro: is_integer_allocate IsIntegerAllocate -fail_action is_integer_allocate f x I I -is_integer_allocate f r I I %macro: is_integer IsInteger -fail_action is_integer f x is_integer f y -is_integer f r is_list Fail=f n => is_list Fail Literal=q => move Literal x | is_list Fail x is_list Fail=f c => jump Fail %macro: is_list IsList -fail_action -is_list f r is_list f x %cold is_list f y %hot -is_nonempty_list Fail=f S=rx | allocate Need Rs => is_nonempty_list_allocate Fail S Need Rs +is_nonempty_list Fail=f S=x | allocate Need Rs => is_nonempty_list_allocate Fail S Need Rs %macro:is_nonempty_list_allocate IsNonemptyListAllocate -fail_action -pack -is_nonempty_list_allocate f x I t is_nonempty_list_allocate f r I t +is_nonempty_list_allocate f x I t -is_nonempty_list F=f r | test_heap I1 I2 => is_non_empty_list_test_heap F r I1 I2 +is_nonempty_list F=f x==0 | test_heap I1 I2 => is_non_empty_list_test_heap F I1 I2 %macro: is_non_empty_list_test_heap IsNonemptyListTestHeap -fail_action -pack -is_non_empty_list_test_heap f r I t +is_non_empty_list_test_heap f I t + +is_nonempty_list Fail=f S=x | get_list S D1=x D2=x => \ + is_nonempty_list_get_list Fail S D1 D2 + +%macro: is_nonempty_list_get_list IsNonemptyListGetList -fail_action -pack +is_nonempty_list_get_list f r x x +is_nonempty_list_get_list f x x x %macro: is_nonempty_list IsNonemptyList -fail_action is_nonempty_list f x is_nonempty_list f y -is_nonempty_list f r %macro: is_atom IsAtom -fail_action is_atom f x -is_atom f r %cold is_atom f y %hot @@ -775,7 +702,6 @@ is_atom Fail=f a => is_atom Fail=f niq => jump Fail %macro: is_float IsFloat -fail_action -is_float f r is_float f x %cold is_float f y @@ -789,12 +715,10 @@ is_nil Fail=f qia => jump Fail %macro: is_nil IsNil -fail_action is_nil f x is_nil f y -is_nil f r is_binary Fail Literal=q => move Literal x | is_binary Fail x is_binary Fail=f c => jump Fail %macro: is_binary IsBinary -fail_action -is_binary f r is_binary f x %cold is_binary f y @@ -806,7 +730,6 @@ is_bitstr Fail Term => is_bitstring Fail Term is_bitstring Fail Literal=q => move Literal x | is_bitstring Fail x is_bitstring Fail=f c => jump Fail %macro: is_bitstring IsBitstring -fail_action -is_bitstring f r is_bitstring f x %cold is_bitstring f y @@ -814,7 +737,6 @@ is_bitstring f y is_reference Fail=f cq => jump Fail %macro: is_reference IsRef -fail_action -is_reference f r is_reference f x %cold is_reference f y @@ -822,7 +744,6 @@ is_reference f y is_pid Fail=f cq => jump Fail %macro: is_pid IsPid -fail_action -is_pid f r is_pid f x %cold is_pid f y @@ -830,7 +751,6 @@ is_pid f y is_port Fail=f cq => jump Fail %macro: is_port IsPort -fail_action -is_port f r is_port f x %cold is_port f y @@ -842,14 +762,12 @@ is_boolean Fail=f ac => jump Fail %cold %macro: is_boolean IsBoolean -fail_action -is_boolean f r is_boolean f x is_boolean f y %hot is_function2 Fail=f acq Arity => jump Fail is_function2 Fail=f Fun a => jump Fail -is_function2 Fail Fun Literal=q => move Literal x | is_function2 Fail Fun x is_function2 f s s %macro: is_function2 IsFunction2 -fail_action @@ -989,76 +907,76 @@ call_ext_only u==3 u$func:erlang:hibernate/3 => i_hibernate %unless USE_VM_PROBES call_ext Arity u$func:erlang:dt_get_tag/0 => \ - move a=am_undefined r + move a=am_undefined x=0 call_ext_last Arity u$func:erlang:dt_get_tag/0 D => \ - move a=am_undefined r | deallocate D | return + move a=am_undefined x=0 | deallocate D | return call_ext_only Arity u$func:erlang:dt_get_tag/0 => \ - move a=am_undefined r | return - -move Any r | call_ext Arity u$func:erlang:dt_put_tag/1 => \ - move a=am_undefined r -move Any r | call_ext_last Arity u$func:erlang:dt_put_tag/1 D => \ - move a=am_undefined r | deallocate D | return -move Any r | call_ext_only Arity u$func:erlang:dt_put_tag/1 => \ - move a=am_undefined r | return + move a=am_undefined x=0 | return + +move Any x==0 | call_ext Arity u$func:erlang:dt_put_tag/1 => \ + move a=am_undefined x=0 +move Any x==0 | call_ext_last Arity u$func:erlang:dt_put_tag/1 D => \ + move a=am_undefined x=0 | deallocate D | return +move Any x==0 | call_ext_only Arity u$func:erlang:dt_put_tag/1 => \ + move a=am_undefined x=0 | return call_ext Arity u$func:erlang:dt_put_tag/1 => \ - move a=am_undefined r + move a=am_undefined x=0 call_ext_last Arity u$func:erlang:dt_put_tag/1 D => \ - move a=am_undefined r | deallocate D | return + move a=am_undefined x=0 | deallocate D | return call_ext_only Arity u$func:erlang:dt_put_tag/1 => \ - move a=am_undefined r | return + move a=am_undefined x=0 | return call_ext Arity u$func:erlang:dt_get_tag_data/0 => \ - move a=am_undefined r + move a=am_undefined x=0 call_ext_last Arity u$func:erlang:dt_get_tag_data/0 D => \ - move a=am_undefined r | deallocate D | return + move a=am_undefined x=0 | deallocate D | return call_ext_only Arity u$func:erlang:dt_get_tag_data/0 => \ - move a=am_undefined r | return - -move Any r | call_ext Arity u$func:erlang:dt_spread_tag/1 => \ - move a=am_true r -move Any r | call_ext_last Arity u$func:erlang:dt_spread_tag/1 D => \ - move a=am_true r | deallocate D | return -move Any r | call_ext_only Arity u$func:erlang:dt_spread_tag/1 => \ - move a=am_true r | return + move a=am_undefined x=0 | return + +move Any x==0 | call_ext Arity u$func:erlang:dt_spread_tag/1 => \ + move a=am_true x=0 +move Any x==0 | call_ext_last Arity u$func:erlang:dt_spread_tag/1 D => \ + move a=am_true x=0 | deallocate D | return +move Any x==0 | call_ext_only Arity u$func:erlang:dt_spread_tag/1 => \ + move a=am_true x=0 | return call_ext Arity u$func:erlang:dt_spread_tag/1 => \ - move a=am_true r + move a=am_true x=0 call_ext_last Arity u$func:erlang:dt_spread_tag/1 D => \ - move a=am_true r | deallocate D | return + move a=am_true x=0 | deallocate D | return call_ext_only Arity u$func:erlang:dt_spread_tag/1 => \ - move a=am_true r | return - -move Any r | call_ext Arity u$func:erlang:dt_restore_tag/1 => \ - move a=am_true r -move Any r | call_ext_last Arity u$func:erlang:dt_restore_tag/1 D => \ - move a=am_true r | deallocate D | return -move Any r | call_ext_only Arity u$func:erlang:dt_restore_tag/1 => \ - move a=am_true r | return + move a=am_true x=0 | return + +move Any x==0 | call_ext Arity u$func:erlang:dt_restore_tag/1 => \ + move a=am_true x=0 +move Any x==0 | call_ext_last Arity u$func:erlang:dt_restore_tag/1 D => \ + move a=am_true x=0 | deallocate D | return +move Any x==0 | call_ext_only Arity u$func:erlang:dt_restore_tag/1 => \ + move a=am_true x=0 | return call_ext Arity u$func:erlang:dt_restore_tag/1 => \ - move a=am_true r + move a=am_true x=0 call_ext_last Arity u$func:erlang:dt_restore_tag/1 D => \ - move a=am_true r | deallocate D | return + move a=am_true x=0 | deallocate D | return call_ext_only Arity u$func:erlang:dt_restore_tag/1 => \ - move a=am_true r | return - -move Any r | call_ext Arity u$func:erlang:dt_prepend_vm_tag_data/1 => \ - move Any r -move Any r | call_ext_last Arity u$func:erlang:dt_prepend_vm_tag_data/1 D => \ - move Any r | deallocate D | return -move Any r | call_ext_only Arity u$func:erlang:dt_prepend_vm_tag_data/1 => \ - move Any r | return + move a=am_true x=0 | return + +move Any x==0 | call_ext Arity u$func:erlang:dt_prepend_vm_tag_data/1 => \ + move Any x=0 +move Any x==0 | call_ext_last Arity u$func:erlang:dt_prepend_vm_tag_data/1 D => \ + move Any x=0 | deallocate D | return +move Any x==0 | call_ext_only Arity u$func:erlang:dt_prepend_vm_tag_data/1 => \ + move Any x=0 | return call_ext Arity u$func:erlang:dt_prepend_vm_tag_data/1 => call_ext_last Arity u$func:erlang:dt_prepend_vm_tag_data/1 D => \ deallocate D | return call_ext_only Arity u$func:erlang:dt_prepend_vm_tag_data/1 => \ return -move Any r | call_ext Arity u$func:erlang:dt_append_vm_tag_data/1 => \ - move Any r -move Any r | call_ext_last Arity u$func:erlang:dt_append_vm_tag_data/1 D => \ - move Any r | deallocate D | return -move Any r | call_ext_only Arity u$func:erlang:dt_append_vm_tag_data/1 => \ - move Any r | return +move Any x==0 | call_ext Arity u$func:erlang:dt_append_vm_tag_data/1 => \ + move Any x=0 +move Any x==0 | call_ext_last Arity u$func:erlang:dt_append_vm_tag_data/1 D => \ + move Any x=0 | deallocate D | return +move Any x==0 | call_ext_only Arity u$func:erlang:dt_append_vm_tag_data/1 => \ + move Any x=0 | return call_ext Arity u$func:erlang:dt_append_vm_tag_data/1 => call_ext_last Arity u$func:erlang:dt_append_vm_tag_data/1 D => \ deallocate D | return @@ -1066,10 +984,17 @@ call_ext_only Arity u$func:erlang:dt_append_vm_tag_data/1 => \ return # Can happen after one of the transformations above. -move Discarded r | move Something r => move Something r +move Discarded x==0 | move Something x==0 => move Something x=0 %endif +call_ext u==0 u$func:os:perf_counter/0 => \ + i_perf_counter +call_ext_last u==0 u$func:os:perf_counter/0 D => \ + i_perf_counter | deallocate_return D +call_ext_only u==0 u$func:os:perf_counter/0 => \ + i_perf_counter | return + # # The general case for BIFs that have no special instructions. # A BIF used in the tail must be followed by a return instruction. @@ -1091,9 +1016,9 @@ call_ext_only Ar=u Bif=u$is_bif => \ # with call instructions. # -move S=c r | call_ext Ar=u Func=u$is_not_bif => i_move_call_ext S r Func -move S=c r | call_ext_last Ar=u Func=u$is_not_bif D => i_move_call_ext_last Func D S r -move S=c r | call_ext_only Ar=u Func=u$is_not_bif => i_move_call_ext_only Func S r +move S=c x==0 | call_ext Ar=u Func=u$is_not_bif => i_move_call_ext S Func +move S=c x==0 | call_ext_last Ar=u Func=u$is_not_bif D => i_move_call_ext_last Func D S +move S=c x==0 | call_ext_only Ar=u Func=u$is_not_bif => i_move_call_ext_only Func S call_ext Ar Func => i_call_ext Func call_ext_last Ar Func D => i_call_ext_last Func D @@ -1109,6 +1034,8 @@ i_apply_fun_only i_hibernate +i_perf_counter + call_bif e # @@ -1118,80 +1045,74 @@ call_bif e bif0 u$bif:erlang:self/0 Dst=d => self Dst bif0 u$bif:erlang:node/0 Dst=d => node Dst -bif1 Fail Bif=u$bif:erlang:get/1 Src=s Dst=d => i_get Src Dst +bif1 Fail Bif=u$bif:erlang:get/1 Src=s Dst=d => gen_get(Src, Dst) -bif2 Jump=j u$bif:erlang:element/2 S1=s S2=rxy Dst=d => gen_element(Jump, S1, S2, Dst) +bif2 Jump=j u$bif:erlang:element/2 S1=s S2=xy Dst=d => gen_element(Jump, S1, S2, Dst) -bif1 Fail Bif Literal=q Dst => move Literal x | bif1 Fail Bif x Dst bif1 p Bif S1 Dst => bif1_body Bif S1 Dst -bif1_body Bif Literal=q Dst => move Literal x | bif1_body Bif x Dst - -bif2 p Bif S1 S2 Dst => i_fetch S1 S2 | i_bif2_body Bif Dst -bif2 Fail Bif S1 S2 Dst => i_fetch S1 S2 | i_bif2 Fail Bif Dst +bif2 p Bif S1 S2 Dst => i_bif2_body Bif S1 S2 Dst +bif2 Fail Bif S1 S2 Dst => i_bif2 Fail Bif S1 S2 Dst +i_get_hash c I d i_get s d %macro: self Self -self r self x self y %macro: node Node -node r node x %cold node y %hot -i_fast_element r j I d -i_fast_element x j I d -i_fast_element y j I d +i_fast_element j x I d +i_fast_element j y I d -i_element r j s d -i_element x j s d -i_element y j s d +i_element j x s d +i_element j y s d bif1 f b s d bif1_body b s d -i_bif2 f b d -i_bif2_body b d +i_bif2 f b s s d +i_bif2_body b s s d # # Internal calls. # -move S=c r | call Ar P=f => i_move_call S r P -move S=s r | call Ar P=f => move_call S r P +move S=c x==0 | call Ar P=f => i_move_call S P +move S=s x==0 | call Ar P=f => move_call S P -i_move_call c r f +i_move_call c f %macro:move_call MoveCall -arg_f -size -nonext -move_call/3 +move_call/2 -move_call x r f -move_call y r f +move_call x f +move_call y f -move S=c r | call_last Ar P=f D => i_move_call_last P D S r -move S r | call_last Ar P=f D => move_call_last S r P D +move S=c x==0 | call_last Ar P=f D => i_move_call_last P D S +move S x==0 | call_last Ar P=f D => move_call_last S P D -i_move_call_last f P c r +i_move_call_last f P c %macro:move_call_last MoveCallLast -arg_f -nonext -pack -move_call_last/4 -move_call_last x r f Q -move_call_last y r f Q +move_call_last/3 +move_call_last x f Q +move_call_last y f Q -move S=c r | call_only Ar P=f => i_move_call_only P S r -move S=x r | call_only Ar P=f => move_call_only S r P +move S=c x==0 | call_only Ar P=f => i_move_call_only P S +move S=x x==0 | call_only Ar P=f => move_call_only S P -i_move_call_only f c r +i_move_call_only f c %macro:move_call_only MoveCallOnly -arg_f -nonext -move_call_only/3 +move_call_only/2 -move_call_only x r f +move_call_only x f call Ar Func => i_call Func call_last Ar Func D => i_call_last Func D @@ -1205,9 +1126,9 @@ i_call_ext e i_call_ext_last e P i_call_ext_only e -i_move_call_ext c r e -i_move_call_ext_last e P c r -i_move_call_ext_only e c r +i_move_call_ext c e +i_move_call_ext_last e P c +i_move_call_ext_only e c # Fun calls. @@ -1227,7 +1148,6 @@ i_make_fun I t %macro: is_function IsFunction -fail_action is_function f x is_function f y -is_function f r is_function Fail=f c => jump Fail func_info M F A => i_func_info u M F A @@ -1239,131 +1159,105 @@ func_info M F A => i_func_info u M F A %cold bs_start_match2 Fail=f ica X Y D => jump Fail bs_start_match2 Fail Bin X Y D => i_bs_start_match2 Bin Fail X Y D -i_bs_start_match2 r f I I d i_bs_start_match2 x f I I d i_bs_start_match2 y f I I d bs_save2 Reg Index => gen_bs_save(Reg, Index) -i_bs_save2 r I i_bs_save2 x I bs_restore2 Reg Index => gen_bs_restore(Reg, Index) -i_bs_restore2 r I i_bs_restore2 x I # Matching integers bs_match_string Fail Ms Bits Val => i_bs_match_string Ms Fail Bits Val -i_bs_match_string r f I I i_bs_match_string x f I I # Fetching integers from binaries. -bs_get_integer2 Fail=f Ms=rx Live=u Sz=sq Unit=u Flags=u Dst=d => \ +bs_get_integer2 Fail=f Ms=x Live=u Sz=sq Unit=u Flags=u Dst=d => \ gen_get_integer2(Fail, Ms, Live, Sz, Unit, Flags, Dst) -i_bs_get_integer_small_imm r I f I d i_bs_get_integer_small_imm x I f I d -i_bs_get_integer_imm r I I f I d i_bs_get_integer_imm x I I f I d -i_bs_get_integer f I I d -i_bs_get_integer_8 r f d +i_bs_get_integer f I I s s d i_bs_get_integer_8 x f d -i_bs_get_integer_16 r f d i_bs_get_integer_16 x f d -i_bs_get_integer_32 r f I d i_bs_get_integer_32 x f I d # Fetching binaries from binaries. -bs_get_binary2 Fail=f Ms=rx Live=u Sz=sq Unit=u Flags=u Dst=d => \ +bs_get_binary2 Fail=f Ms=x Live=u Sz=sq Unit=u Flags=u Dst=d => \ gen_get_binary2(Fail, Ms, Live, Sz, Unit, Flags, Dst) %macro: i_bs_get_binary_imm2 BsGetBinaryImm_2 -fail_action -gen_dest %macro: i_bs_get_binary2 BsGetBinary_2 -fail_action -gen_dest %macro: i_bs_get_binary_all2 BsGetBinaryAll_2 -fail_action -gen_dest -i_bs_get_binary_imm2 f r I I I d i_bs_get_binary_imm2 f x I I I d -i_bs_get_binary2 f r I s I d i_bs_get_binary2 f x I s I d -i_bs_get_binary_all2 f r I I d i_bs_get_binary_all2 f x I I d -i_bs_get_binary_all_reuse r f I i_bs_get_binary_all_reuse x f I # Fetching float from binaries. -bs_get_float2 Fail=f Ms=rx Live=u Sz=s Unit=u Flags=u Dst=d => \ +bs_get_float2 Fail=f Ms=x Live=u Sz=s Unit=u Flags=u Dst=d => \ gen_get_float2(Fail, Ms, Live, Sz, Unit, Flags, Dst) -bs_get_float2 Fail=f Ms=rx Live=u Sz=q Unit=u Flags=u Dst=d => jump Fail +bs_get_float2 Fail=f Ms=x Live=u Sz=q Unit=u Flags=u Dst=d => jump Fail %macro: i_bs_get_float2 BsGetFloat2 -fail_action -gen_dest -i_bs_get_float2 f r I s I d i_bs_get_float2 f x I s I d # Miscellanous -bs_skip_bits2 Fail=f Ms=rx Sz=s Unit=u Flags=u => \ - gen_skip_bits2(Fail, Ms, Sz, Unit, Flags) -bs_skip_bits2 Fail=f Ms=rx Sz=q Unit=u Flags=u => \ +bs_skip_bits2 Fail=f Ms=x Sz=sq Unit=u Flags=u => \ gen_skip_bits2(Fail, Ms, Sz, Unit, Flags) %macro: i_bs_skip_bits_imm2 BsSkipBitsImm2 -fail_action -i_bs_skip_bits_imm2 f r I i_bs_skip_bits_imm2 f x I %macro: i_bs_skip_bits2 BsSkipBits2 -fail_action -i_bs_skip_bits2 f r x I -i_bs_skip_bits2 f r y I i_bs_skip_bits2 f x x I -i_bs_skip_bits2 f x r I i_bs_skip_bits2 f x y I %macro: i_bs_skip_bits_all2 BsSkipBitsAll2 -fail_action -i_bs_skip_bits_all2 f r I i_bs_skip_bits_all2 f x I -bs_test_tail2 Fail=f Ms=rx Bits=u==0 => bs_test_zero_tail2 Fail Ms -bs_test_tail2 Fail=f Ms=rx Bits=u => bs_test_tail_imm2 Fail Ms Bits -bs_test_zero_tail2 f r +bs_test_tail2 Fail=f Ms=x Bits=u==0 => bs_test_zero_tail2 Fail Ms +bs_test_tail2 Fail=f Ms=x Bits=u => bs_test_tail_imm2 Fail Ms Bits bs_test_zero_tail2 f x -bs_test_tail_imm2 f r I bs_test_tail_imm2 f x I bs_test_unit F Ms Unit=u==8 => bs_test_unit8 F Ms -bs_test_unit f r I bs_test_unit f x I -bs_test_unit8 f r bs_test_unit8 f x -bs_context_to_binary r +# An y register operand for bs_context_to_binary is rare, +# but can happen because of inlining. + +bs_context_to_binary Y=y => move Y x | bs_context_to_binary x + bs_context_to_binary x -bs_context_to_binary y # # Utf8/utf16/utf32 support. (R12B-5) # -bs_get_utf8 Fail=f Ms=rx u u Dst=d => i_bs_get_utf8 Ms Fail Dst -i_bs_get_utf8 r f d +bs_get_utf8 Fail=f Ms=x u u Dst=d => i_bs_get_utf8 Ms Fail Dst i_bs_get_utf8 x f d -bs_skip_utf8 Fail=f Ms=rx u u => i_bs_get_utf8 Ms Fail x +bs_skip_utf8 Fail=f Ms=x u u => i_bs_get_utf8 Ms Fail x -bs_get_utf16 Fail=f Ms=rx u Flags=u Dst=d => i_bs_get_utf16 Ms Fail Flags Dst -bs_skip_utf16 Fail=f Ms=rx u Flags=u => i_bs_get_utf16 Ms Fail Flags x +bs_get_utf16 Fail=f Ms=x u Flags=u Dst=d => i_bs_get_utf16 Ms Fail Flags Dst +bs_skip_utf16 Fail=f Ms=x u Flags=u => i_bs_get_utf16 Ms Fail Flags x -i_bs_get_utf16 r f I d i_bs_get_utf16 x f I d -bs_get_utf32 Fail=f Ms=rx Live=u Flags=u Dst=d => \ +bs_get_utf32 Fail=f Ms=x Live=u Flags=u Dst=d => \ bs_get_integer2 Fail Ms Live i=32 u=1 Flags Dst | \ - i_fetch Dst Ms | \ - i_bs_validate_unicode_retract Fail -bs_skip_utf32 Fail=f Ms=rx Live=u Flags=u => \ + i_bs_validate_unicode_retract Fail Dst Ms +bs_skip_utf32 Fail=f Ms=x Live=u Flags=u => \ bs_get_integer2 Fail Ms Live i=32 u=1 Flags x | \ - i_fetch x Ms | \ - i_bs_validate_unicode_retract Fail + i_bs_validate_unicode_retract Fail x Ms -i_bs_validate_unicode_retract j +i_bs_validate_unicode_retract j s s %hot # @@ -1385,13 +1279,12 @@ bs_init2 Fail Sz=u Words Regs Flags Dst => \ bs_init2 Fail Sz Words=u==0 Regs Flags Dst => \ i_bs_init_fail Sz Fail Regs Dst bs_init2 Fail Sz Words Regs Flags Dst => \ - i_fetch Sz r | i_bs_init_fail_heap Words Fail Regs Dst + i_bs_init_fail_heap Sz Words Fail Regs Dst -i_bs_init_fail r j I d i_bs_init_fail x j I d i_bs_init_fail y j I d -i_bs_init_fail_heap I j I d +i_bs_init_fail_heap s I j I d i_bs_init I I d i_bs_init_heap_bin I I d @@ -1408,39 +1301,35 @@ bs_init_bits Fail Sz=u Words Regs Flags Dst => i_bs_init_bits_heap Sz Words Reg bs_init_bits Fail Sz Words=u==0 Regs Flags Dst => \ i_bs_init_bits_fail Sz Fail Regs Dst bs_init_bits Fail Sz Words Regs Flags Dst => \ - i_fetch Sz r | i_bs_init_bits_fail_heap Words Fail Regs Dst + i_bs_init_bits_fail_heap Sz Words Fail Regs Dst -i_bs_init_bits_fail r j I d i_bs_init_bits_fail x j I d i_bs_init_bits_fail y j I d -i_bs_init_bits_fail_heap I j I d +i_bs_init_bits_fail_heap s I j I d i_bs_init_bits I I d i_bs_init_bits_heap I I I d bs_add Fail S1=i==0 S2 Unit=u==1 D => move S2 D -bs_add Fail S1 S2 Unit D => i_fetch S1 S2 | i_bs_add Fail Unit D -i_bs_add j I d +bs_add j s s I d bs_append Fail Size Extra Live Unit Bin Flags Dst => \ - i_fetch Size Bin | i_bs_append Fail Extra Live Unit Dst + move Bin x | i_bs_append Fail Extra Live Unit Size Dst bs_private_append Fail Size Unit Bin Flags Dst => \ - i_fetch Size Bin | i_bs_private_append Fail Unit Dst + i_bs_private_append Fail Unit Size Bin Dst bs_init_writable -i_bs_append j I I I d -i_bs_private_append j I d +i_bs_append j I I I s d +i_bs_private_append j I s s d # # Storing integers into binaries. # -bs_put_integer Fail=j Sz=s Unit=u Flags=u Literal=q => \ - move Literal x | bs_put_integer Fail Sz Unit Flags x bs_put_integer Fail=j Sz=sq Unit=u Flags=u Src=s => \ gen_put_integer(Fail, Sz, Unit, Flags, Src) @@ -1454,32 +1343,22 @@ i_new_bs_put_integer_imm j I I s # Utf8/utf16/utf32 support. (R12B-5) # -bs_utf8_size Fail Literal=q Dst=d => \ - move Literal x | bs_utf8_size Fail x Dst bs_utf8_size j Src=s Dst=d => i_bs_utf8_size Src Dst i_bs_utf8_size s d -bs_utf16_size Fail Literal=q Dst=d => \ - move Literal x | bs_utf16_size Fail x Dst bs_utf16_size j Src=s Dst=d => i_bs_utf16_size Src Dst i_bs_utf16_size s d -bs_put_utf8 Fail=j Flags=u Literal=q => \ - move Literal x | bs_put_utf8 Fail Flags x bs_put_utf8 Fail u Src=s => i_bs_put_utf8 Fail Src i_bs_put_utf8 j s -bs_put_utf16 Fail=j Flags=u Literal=q => \ - move Literal x | bs_put_utf16 Fail Flags x bs_put_utf16 Fail Flags=u Src=s => i_bs_put_utf16 Fail Flags Src i_bs_put_utf16 j I s -bs_put_utf32 Fail=j Flags=u Literal=q => \ - move Literal x | bs_put_utf32 Fail Flags x bs_put_utf32 Fail=j Flags=u Src=s => \ i_bs_validate_unicode Fail Src | bs_put_integer Fail i=32 u=1 Flags Src @@ -1490,9 +1369,6 @@ i_bs_validate_unicode j s # bs_put_float Fail Sz=q Unit Flags Val => badarg Fail -bs_put_float Fail=j Sz Unit=u Flags=u Literal=q => \ - move Literal x | bs_put_float Fail Sz Unit Flags x - bs_put_float Fail=j Sz=s Unit=u Flags=u Src=s => \ gen_put_float(Fail, Sz, Unit, Flags, Src) @@ -1506,8 +1382,6 @@ i_new_bs_put_float_imm j I I s # Storing binaries into binaries. # -bs_put_binary Fail Sz Unit Flags Literal=q => \ - move Literal x | bs_put_binary Fail Sz Unit Flags x bs_put_binary Fail=j Sz=s Unit=u Flags=u Src=s => \ gen_put_binary(Fail, Sz, Unit, Flags, Src) @@ -1600,7 +1474,6 @@ is_map Fail Lit=q | literal_is_map(Lit) => is_map Fail cq => jump Fail %macro: is_map IsMap -fail_action -is_map f r is_map f x is_map f y @@ -1611,101 +1484,120 @@ has_map_fields Fail Src Size Rest=* => \ ## Transform get_map_elements(s) #{ K1 := V1, K2 := V2 } -get_map_elements Fail Src=rxy Size=u==2 Rest=* => \ +get_map_elements Fail Src=xy Size=u==2 Rest=* => \ gen_get_map_element(Fail, Src, Size, Rest) get_map_elements Fail Src Size Rest=* | map_key_sort(Size, Rest) => \ gen_get_map_elements(Fail, Src, Size, Rest) i_get_map_elements f s I -i_get_map_element Fail Src=rxy Key=ry Dst => \ +i_get_map_element Fail Src=xy Key=y Dst => \ move Key x | i_get_map_element Fail Src x Dst %macro: i_get_map_element_hash GetMapElementHash -fail_action -i_get_map_element_hash f r c I r -i_get_map_element_hash f x c I r -i_get_map_element_hash f y c I r -i_get_map_element_hash f r c I x i_get_map_element_hash f x c I x i_get_map_element_hash f y c I x -i_get_map_element_hash f r c I y i_get_map_element_hash f x c I y i_get_map_element_hash f y c I y %macro: i_get_map_element GetMapElement -fail_action -i_get_map_element f r x r -i_get_map_element f x x r -i_get_map_element f y x r -i_get_map_element f r x x i_get_map_element f x x x i_get_map_element f y x x -i_get_map_element f r x y i_get_map_element f x x y i_get_map_element f y x y # +# Convert the plus operations to a generic plus instruction. +# +gen_plus/5 +gen_minus/5 + +gc_bif1 Fail Live u$bif:erlang:splus/1 Src Dst => \ + gen_plus Fail Live Src i Dst +gc_bif2 Fail Live u$bif:erlang:splus/2 S1 S2 Dst => \ + gen_plus Fail Live S1 S2 Dst + +gc_bif1 Fail Live u$bif:erlang:sminus/1 Src Dst => \ + gen_minus Fail Live i Src Dst +gc_bif2 Fail Live u$bif:erlang:sminus/2 S1 S2 Dst => \ + gen_minus Fail Live S1 S2 Dst + +# # Optimize addition and subtraction of small literals using # the i_increment/4 instruction (in bodies, not in guards). # -gc_bif2 p Live u$bif:erlang:splus/2 Int=i Reg=d Dst => \ +gen_plus p Live Int=i Reg=d Dst => \ gen_increment(Reg, Int, Live, Dst) -gc_bif2 p Live u$bif:erlang:splus/2 Reg=d Int=i Dst => \ +gen_plus p Live Reg=d Int=i Dst => \ gen_increment(Reg, Int, Live, Dst) -gc_bif2 p Live u$bif:erlang:sminus/2 Reg=d Int=i Dst | \ - negation_is_small(Int) => \ +gen_minus p Live Reg=d Int=i Dst | negation_is_small(Int) => \ gen_increment_from_minus(Reg, Int, Live, Dst) # # GCing arithmetic instructions. # -gc_bif2 Fail I u$bif:erlang:splus/2 S1=x S2=x Dst=d => i_plus Fail I S1 S2 Dst -gc_bif2 Fail I u$bif:erlang:splus/2 S1 S2 Dst=d => i_fetch S1 S2 | i_plus Fail I Dst -gc_bif2 Fail I u$bif:erlang:sminus/2 S1=x S2=x Dst=d => i_minus Fail I S1 S2 Dst -gc_bif2 Fail I u$bif:erlang:sminus/2 S1 S2 Dst=d => i_fetch S1 S2 | i_minus Fail I Dst -gc_bif2 Fail I u$bif:erlang:stimes/2 S1 S2 Dst=d => i_fetch S1 S2 | i_times Fail I Dst -gc_bif2 Fail I u$bif:erlang:div/2 S1 S2 Dst=d => i_fetch S1 S2 | i_m_div Fail I Dst +gen_plus Fail Live Y=y X=x Dst => i_plus Fail Live X Y Dst +gen_plus Fail Live S1 S2 Dst => i_plus Fail Live S1 S2 Dst -gc_bif2 Fail I u$bif:erlang:intdiv/2 S1 S2 Dst=d => i_fetch S1 S2 | i_int_div Fail I Dst -gc_bif2 Fail I u$bif:erlang:rem/2 S1=x S2=x Dst=d => i_rem Fail I S1 S2 Dst -gc_bif2 Fail I u$bif:erlang:rem/2 S1 S2 Dst=d => i_fetch S1 S2 | i_rem Fail I Dst +gen_minus Fail Live S1 S2 Dst => i_minus Fail Live S1 S2 Dst -gc_bif2 Fail I u$bif:erlang:bsl/2 S1 S2 Dst=d => i_fetch S1 S2 | i_bsl Fail I Dst -gc_bif2 Fail I u$bif:erlang:bsr/2 S1 S2 Dst=d => i_fetch S1 S2 | i_bsr Fail I Dst +gc_bif2 Fail Live u$bif:erlang:stimes/2 S1 S2 Dst => \ + i_times Fail Live S1 S2 Dst -gc_bif2 Fail I u$bif:erlang:band/2 S1=x S2=c Dst=d => i_band Fail I S1 S2 Dst -gc_bif2 Fail I u$bif:erlang:band/2 S1 S2 Dst=d => i_fetch S1 S2 | i_band Fail I Dst -gc_bif2 Fail I u$bif:erlang:bor/2 S1 S2 Dst=d => i_fetch S1 S2 | i_bor Fail I Dst -gc_bif2 Fail I u$bif:erlang:bxor/2 S1 S2 Dst=d => i_fetch S1 S2 | i_bxor Fail I Dst +gc_bif2 Fail Live u$bif:erlang:div/2 S1 S2 Dst => \ + i_m_div Fail Live S1 S2 Dst +gc_bif2 Fail Live u$bif:erlang:intdiv/2 S1 S2 Dst => \ + i_int_div Fail Live S1 S2 Dst -gc_bif1 Fail I u$bif:erlang:bnot/1 Src Dst=d => i_int_bnot Fail Src I Dst +gc_bif2 Fail Live u$bif:erlang:rem/2 S1 S2 Dst => \ + i_rem Fail Live S1 S2 Dst + +gc_bif2 Fail Live u$bif:erlang:bsl/2 S1 S2 Dst => \ + i_bsl Fail Live S1 S2 Dst +gc_bif2 Fail Live u$bif:erlang:bsr/2 S1 S2 Dst => \ + i_bsr Fail Live S1 S2 Dst + +gc_bif2 Fail Live u$bif:erlang:band/2 S1 S2 Dst => \ + i_band Fail Live S1 S2 Dst + +gc_bif2 Fail Live u$bif:erlang:bor/2 S1 S2 Dst => \ + i_bor Fail Live S1 S2 Dst -gc_bif1 Fail I u$bif:erlang:sminus/1 Src Dst=d => i_fetch i Src | i_minus Fail I Dst -gc_bif1 Fail I u$bif:erlang:splus/1 Src Dst=d => i_fetch i Src | i_plus Fail I Dst +gc_bif2 Fail Live u$bif:erlang:bxor/2 S1 S2 Dst => \ + i_bxor Fail Live S1 S2 Dst + +gc_bif1 Fail I u$bif:erlang:bnot/1 Src Dst=d => i_int_bnot Fail Src I Dst i_increment r I I d i_increment x I I d i_increment y I I d i_plus j I x x d -i_plus j I d +i_plus j I x y d +i_plus j I s s d + i_minus j I x x d -i_minus j I d -i_times j I d -i_m_div j I d -i_int_div j I d +i_minus j I s s d + +i_times j I s s d + +i_m_div j I s s d +i_int_div j I s s d + i_rem j I x x d -i_rem j I d +i_rem j I s s d -i_bsl j I d -i_bsr j I d +i_bsl j I s s d +i_bsr j I s s d i_band j I x c d -i_band j I d -i_bor j I d -i_bxor j I d +i_band j I s s d + +i_bor j I s s d +i_bxor j I s s d i_int_bnot j s I d @@ -1731,21 +1623,18 @@ gc_bif2 Fail I Bif S1 S2 Dst => \ gc_bif3 Fail I Bif S1 S2 S3 Dst => \ gen_guard_bif3(Fail, I, Bif, S1, S2, S3, Dst) -i_gc_bif1 Fail Bif V=q Live D => move V x | i_gc_bif1 Fail Bif x Live D - i_gc_bif1 j I s I d -ii_gc_bif2/6 - -ii_gc_bif2 Fail Bif S1 S2 Live D => i_fetch S1 S2 | i_gc_bif2 Fail Bif Live D - -i_gc_bif2 j I I d +i_gc_bif2 j I I s s d ii_gc_bif3/7 -ii_gc_bif3 Fail Bif S1 S2 S3 Live D => move S1 x | i_fetch S2 S3 | i_gc_bif3 Fail Bif x Live D +# A specific instruction can only have 6 operands, so we must +# pass one of the arguments in an x register. +ii_gc_bif3 Fail Bif Live S1 S2 S3 Dst => \ + move S1 x | i_gc_bif3 Fail Bif Live S2 S3 Dst -i_gc_bif3 j I s I d +i_gc_bif3 j I I s s d # # The following instruction is specially handled in beam_load.c diff --git a/erts/emulator/beam/register.c b/erts/emulator/beam/register.c index 7ade8bca0f..fdb6cbc813 100644 --- a/erts/emulator/beam/register.c +++ b/erts/emulator/beam/register.c @@ -151,6 +151,9 @@ void init_register_table(void) f.cmp = (HCMP_FUN) reg_cmp; f.alloc = (HALLOC_FUN) reg_alloc; f.free = (HFREE_FUN) reg_free; + f.meta_alloc = (HMALLOC_FUN) erts_alloc; + f.meta_free = (HMFREE_FUN) erts_free; + f.meta_print = (HMPRINT_FUN) erts_print; hash_init(ERTS_ALC_T_REG_TABLE, &process_reg, "process_reg", PREG_HASH_SIZE, f); diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index 03e30573de..068d636a40 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -21,6 +21,19 @@ #ifndef __SYS_H__ #define __SYS_H__ +#if !defined(__GNUC__) +# define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) 0 +#elif !defined(__GNUC_MINOR__) +# define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \ + ((__GNUC__ << 24) >= (((MAJ) << 24) | ((MIN) << 12) | (PL))) +#elif !defined(__GNUC_PATCHLEVEL__) +# define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \ + (((__GNUC__ << 24) | (__GNUC_MINOR__ << 12)) >= (((MAJ) << 24) | ((MIN) << 12) | (PL))) +#else +# define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \ + (((__GNUC__ << 24) | (__GNUC_MINOR__ << 12) | __GNUC_PATCHLEVEL__) >= (((MAJ) << 24) | ((MIN) << 12) | (PL))) +#endif + #ifdef ERTS_INLINE # ifndef ERTS_CAN_INLINE # define ERTS_CAN_INLINE 1 @@ -38,6 +51,17 @@ # endif #endif +#ifndef ERTS_FORCE_INLINE +# if ERTS_AT_LEAST_GCC_VSN__(3,1,1) +# define ERTS_FORCE_INLINE __inline__ __attribute__((__always_inline__)) +# elif defined(__WIN32__) +# define ERTS_FORCE_INLINE __forceinline +# endif +# ifndef ERTS_FORCE_INLINE +# define ERTS_FORCE_INLINE ERTS_INLINE +# endif +#endif + #if defined(DEBUG) || defined(ERTS_ENABLE_LOCK_CHECK) # undef ERTS_CAN_INLINE # define ERTS_CAN_INLINE 0 @@ -46,8 +70,10 @@ #endif #if ERTS_CAN_INLINE +#define ERTS_GLB_FORCE_INLINE static ERTS_FORCE_INLINE #define ERTS_GLB_INLINE static ERTS_INLINE #else +#define ERTS_GLB_FORCE_INLINE #define ERTS_GLB_INLINE #endif @@ -61,22 +87,14 @@ # define NO_FPE_SIGNALS #endif -#ifdef DISABLE_CHILD_WAITER_THREAD -#undef ENABLE_CHILD_WAITER_THREAD -#endif - -#if defined(ERTS_SMP) && !defined(DISABLE_CHILD_WAITER_THREAD) -#undef ENABLE_CHILD_WAITER_THREAD -#define ENABLE_CHILD_WAITER_THREAD 1 -#endif - #define ERTS_I64_LITERAL(X) X##LL +#define ErtsInArea(ptr,start,nbytes) \ + ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes)) + #if defined (__WIN32__) # include "erl_win_sys.h" -#elif defined (__OSE__) -# include "erl_ose_sys.h" -#else +#else # include "erl_unix_sys.h" #ifndef UNIX # define UNIX 1 @@ -111,19 +129,6 @@ typedef int ErtsSysFdType; typedef ERTS_SYS_FD_TYPE ErtsSysFdType; #endif -#if !defined(__GNUC__) -# define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) 0 -#elif !defined(__GNUC_MINOR__) -# define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \ - ((__GNUC__ << 24) >= (((MAJ) << 24) | ((MIN) << 12) | (PL))) -#elif !defined(__GNUC_PATCHLEVEL__) -# define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \ - (((__GNUC__ << 24) | (__GNUC_MINOR__ << 12)) >= (((MAJ) << 24) | ((MIN) << 12) | (PL))) -#else -# define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \ - (((__GNUC__ << 24) | (__GNUC_MINOR__ << 12) | __GNUC_PATCHLEVEL__) >= (((MAJ) << 24) | ((MIN) << 12) | (PL))) -#endif - #if ERTS_AT_LEAST_GCC_VSN__(2, 96, 0) # define ERTS_LIKELY(BOOL) __builtin_expect((BOOL), !0) # define ERTS_UNLIKELY(BOOL) __builtin_expect((BOOL), 0) @@ -131,6 +136,17 @@ typedef ERTS_SYS_FD_TYPE ErtsSysFdType; # define ERTS_LIKELY(BOOL) (BOOL) # define ERTS_UNLIKELY(BOOL) (BOOL) #endif + +#if ERTS_AT_LEAST_GCC_VSN__(2, 96, 0) +#ifndef __llvm__ +# define ERTS_WRITE_UNLIKELY(X) X __attribute__ ((section ("ERTS_LOW_WRITE") )) +#else +# define ERTS_WRITE_UNLIKELY(X) X __attribute__ ((section ("__DATA,ERTS_LOW_WRITE") )) +#endif +#else +# define ERTS_WRITE_UNLIKELY(X) X +#endif + #ifdef __GNUC__ # if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ > 5) # define ERTS_DECLARE_DUMMY(X) X __attribute__ ((unused)) @@ -191,6 +207,12 @@ __decl_noreturn void __noreturn erl_assert_error(const char* expr, const char *f # define ASSERT(e) ((void) 1) #endif +#ifdef ERTS_SMP +# define ERTS_SMP_ASSERT(e) ASSERT(e) +#else +# define ERTS_SMP_ASSERT(e) ((void)1) +#endif + /* ERTS_UNDEF can be used to silence false warnings about * "variable may be used uninitialized" while keeping the variable * marked as undefined by valgrind. @@ -285,62 +307,11 @@ __decl_noreturn void __noreturn erl_assert_error(const char* expr, const char *f #else #error Neither 32 nor 64 bit architecture #endif -#if defined(ARCH_64) && defined(HALFWORD_HEAP_EMULATOR) -# define HALFWORD_HEAP 1 -# define HALFWORD_ASSERT 0 -# define ASSERT_HALFWORD(COND) ASSERT(COND) -# undef ERTS_SIZEOF_TERM -# define ERTS_SIZEOF_TERM 4 -#else -# define HALFWORD_HEAP 0 -# define HALFWORD_ASSERT 0 -# define ASSERT_HALFWORD(COND) -#endif #if SIZEOF_VOID_P != SIZEOF_SIZE_T #error sizeof(void*) != sizeof(size_t) #endif -#if HALFWORD_HEAP - -#if SIZEOF_INT == 4 -typedef unsigned int Eterm; -typedef unsigned int Uint; -typedef int Sint; -#define ERTS_UINT_MAX UINT_MAX -#define ERTS_SIZEOF_ETERM SIZEOF_INT -#define ErtsStrToSint strtol -#else -#error Found no appropriate type to use for 'Eterm', 'Uint' and 'Sint' -#endif - -#if SIZEOF_VOID_P == SIZEOF_LONG -typedef unsigned long UWord; -typedef long SWord; -#define SWORD_CONSTANT(Const) Const##L -#define UWORD_CONSTANT(Const) Const##UL -#define ERTS_UWORD_MAX ULONG_MAX -#define ERTS_SWORD_MAX LONG_MAX -#elif SIZEOF_VOID_P == SIZEOF_INT -typedef unsigned int UWord; -typedef int SWord; -#define SWORD_CONSTANT(Const) Const -#define UWORD_CONSTANT(Const) Const##U -#define ERTS_UWORD_MAX UINT_MAX -#define ERTS_SWORD_MAX INT_MAX -#elif SIZEOF_VOID_P == SIZEOF_LONG_LONG -typedef unsigned long long UWord; -typedef long long SWord; -#define SWORD_CONSTANT(Const) Const##LL -#define UWORD_CONSTANT(Const) Const##ULL -#define ERTS_UWORD_MAX ULLONG_MAX -#define ERTS_SWORD_MAX LLONG_MAX -#else -#error Found no appropriate type to use for 'Eterm', 'Uint' and 'Sint' -#endif - -#else /* !HALFWORD_HEAP */ - #if SIZEOF_VOID_P == SIZEOF_LONG typedef unsigned long Eterm; typedef unsigned long Uint; @@ -383,8 +354,6 @@ typedef Uint UWord; typedef Sint SWord; #define ERTS_UINT_MAX ERTS_UWORD_MAX -#endif /* HALFWORD_HEAP */ - typedef UWord BeamInstr; #ifndef HAVE_INT64 @@ -785,6 +754,7 @@ void erts_sys_main_thread(void); extern int erts_sys_prepare_crash_dump(int secs); extern void erts_sys_pre_init(void); extern void erl_sys_init(void); +extern void erl_sys_late_init(void); extern void erl_sys_args(int *argc, char **argv); extern void erl_sys_schedule(int); void sys_tty_reset(int); @@ -1102,7 +1072,6 @@ extern int erts_use_kernel_poll; #define put_int8(i, s) do {((unsigned char*)(s))[0] = (i) & 0xff;} while (0) - /* * Use DEBUGF as you would use printf, but use double parentheses: * diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c index 0ab6661c9f..48cb39333a 100644 --- a/erts/emulator/beam/time.c +++ b/erts/emulator/beam/time.c @@ -278,9 +278,14 @@ ErtsMonotonicTime erts_check_next_timeout_time(ErtsSchedulerData *esdp) { ErtsTimerWheel *tiw = esdp->timer_wheel; + ErtsMonotonicTime time; + ERTS_MSACC_DECLARE_CACHE_X(); if (tiw->true_next_timeout_time) return tiw->next_timeout_time; - return find_next_timeout(esdp, tiw, 1, 0, 0); + ERTS_MSACC_PUSH_AND_SET_STATE_CACHED_X(ERTS_MSACC_STATE_TIMERS); + time = find_next_timeout(esdp, tiw, 1, 0, 0); + ERTS_MSACC_POP_STATE_M_X(); + return time; } #ifndef ERTS_TW_DEBUG @@ -336,6 +341,7 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) { int tiw_pos_ix, slots, yielded_slot_restarted, yield_count; ErtsMonotonicTime bump_to, tmp_slots, old_pos; + ERTS_MSACC_PUSH_AND_SET_STATE_M_X(ERTS_MSACC_STATE_TIMERS); yield_count = ERTS_TWHEEL_BUMP_YIELD_LIMIT; @@ -371,6 +377,7 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) tiw->next_timeout_time = curr_time + ERTS_MONOTONIC_DAY; tiw->pos = bump_to; tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE; + ERTS_MSACC_POP_STATE_M_X(); return; } @@ -382,6 +389,7 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) tiw->yield_slot = ERTS_TWHEEL_SLOT_AT_ONCE; tiw->true_next_timeout_time = 1; tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(old_pos); + ERTS_MSACC_POP_STATE_M_X(); return; } @@ -400,8 +408,10 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) p = tiw->at_once.head; } - if (tiw->pos >= bump_to) + if (tiw->pos >= bump_to) { + ERTS_MSACC_POP_STATE_M_X(); break; + } if (tiw->nto == 0) goto empty_wheel; @@ -478,6 +488,7 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) tiw->yield_slot = tiw_pos_ix; tiw->yield_slots_left = slots; tiw->yield_start_pos = old_pos; + ERTS_MSACC_POP_STATE_M_X(); return; /* Yield! */ } @@ -500,6 +511,7 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) /* Search at most two seconds ahead... */ (void) find_next_timeout(NULL, tiw, 0, curr_time, ERTS_SEC_TO_MONOTONIC(2)); + ERTS_MSACC_POP_STATE_M_X(); } Uint @@ -569,6 +581,7 @@ erts_twheel_set_timer(ErtsTimerWheel *tiw, ErtsMonotonicTime timeout_pos) { ErtsMonotonicTime timeout_time; + ERTS_MSACC_PUSH_AND_SET_STATE_M_X(ERTS_MSACC_STATE_TIMERS); p->u.func.timeout = timeout; p->u.func.cancel = cancel; @@ -612,6 +625,7 @@ erts_twheel_set_timer(ErtsTimerWheel *tiw, tiw->true_next_timeout_time = 1; tiw->next_timeout_time = timeout_time; } + ERTS_MSACC_POP_STATE_M_X(); } void @@ -620,11 +634,13 @@ erts_twheel_cancel_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p) if (p->slot != ERTS_TWHEEL_SLOT_INACTIVE) { ErlCancelProc cancel; void *arg; + ERTS_MSACC_PUSH_AND_SET_STATE_M_X(ERTS_MSACC_STATE_TIMERS); remove_timer(tiw, p); cancel = p->u.func.cancel; arg = p->u.func.arg; if (cancel) (*cancel)(arg); + ERTS_MSACC_POP_STATE_M_X(); } } diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index 5286391746..3ba02023a1 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -110,7 +110,6 @@ Eterm* erts_heap_alloc(Process* p, Uint need, Uint xtra) { ErlHeapFragment* bp; - Eterm* htop; Uint n; #if defined(DEBUG) || defined(CHECK_FOR_HOLES) Uint i; @@ -156,16 +155,6 @@ erts_heap_alloc(Process* p, Uint need, Uint xtra) n--; #endif - /* - * When we have created a heap fragment, we are no longer allowed - * to store anything more on the heap. - */ - htop = HEAP_TOP(p); - if (htop < HEAP_LIMIT(p)) { - *htop = make_pos_bignum_header(HEAP_LIMIT(p)-htop-1); - HEAP_TOP(p) = HEAP_LIMIT(p); - } - bp->next = MBUF(p); MBUF(p) = bp; bp->alloc_size = n; @@ -269,6 +258,31 @@ erl_grow_pstack(ErtsPStack* s, void* default_pstack, unsigned need_bytes) s->psp = s->pstart + sp_offs; } +/* + * Helper function for the EQUEUE macros defined in global.h. + */ + +void +erl_grow_equeue(ErtsEQueue* q, Eterm* default_equeue) +{ + Uint old_size = (q->end - q->start); + Uint new_size = old_size * 2; + Uint first_part = (q->end - q->front); + Uint second_part = (q->back - q->start); + Eterm* new_ptr = erts_alloc(q->alloc_type, new_size*sizeof(Eterm)); + ASSERT(q->back == q->front); // of course the queue is full now! + if (first_part > 0) + sys_memcpy(new_ptr, q->front, first_part*sizeof(Eterm)); + if (second_part > 0) + sys_memcpy(new_ptr+first_part, q->start, second_part*sizeof(Eterm)); + if (q->start != default_equeue) + erts_free(q->alloc_type, q->start); + q->start = new_ptr; + q->end = q->start + new_size; + q->front = q->start; + q->back = q->start + old_size; +} + /* CTYPE macros */ #define LATIN1 @@ -303,10 +317,10 @@ erl_grow_pstack(ErtsPStack* s, void* default_pstack, unsigned need_bytes) * Calculate length of a list. * Returns -1 if not a proper list (i.e. not terminated with NIL) */ -int +Sint erts_list_length(Eterm list) { - int i = 0; + Sint i = 0; while(is_list(list)) { i++; @@ -894,7 +908,7 @@ tail_recur: Uint y2 = y1 < 0 ? -(Uint)y1 : y1; UINT32_HASH_STEP(y2, FUNNY_NUMBER2); -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) if (y2 >> 32) UINT32_HASH_STEP(y2 >> 32, FUNNY_NUMBER2); #endif @@ -1015,7 +1029,7 @@ tail_recur: } d = BIG_DIGIT(ptr, k); k = sizeof(ErtsDigit); -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) if (!(d >> 32)) k /= 2; #endif @@ -1985,7 +1999,7 @@ tail_recur: (atom_tab(atom_val(term))->slot.bucket.hvalue); break; case SMALL_DEF: -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) { Sint y1 = signed_val(term); Uint y2 = y1 < 0 ? -(Uint)y1 : y1; @@ -2281,7 +2295,11 @@ static void do_send_logger_message(Eterm *hp, ErlOffHeap *ohp, ErlHeapFragment * erts_queue_error_logger_message(from, message, bp); } #else - erts_queue_message(p, NULL /* only used for smp build */, bp, message, NIL); + { + ErtsMessage *mp = erts_alloc_message(0, NULL); + mp->data.heap_frag = bp; + erts_queue_message(p, NULL /* only used for smp build */, mp, message, NIL); + } #endif } @@ -2594,11 +2612,7 @@ erts_destroy_tmp_dsbuf(erts_dsprintf_buf_t *dsbufp) * Test for equality of two terms. * Returns 0 if not equal, or a non-zero value otherwise. */ -#if HALFWORD_HEAP -int eq_rel(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base) -#else int eq(Eterm a, Eterm b) -#endif { DECLARE_WSTACK(stack); Sint sz; @@ -2606,18 +2620,18 @@ int eq(Eterm a, Eterm b) Eterm* bb; tailrecur: - if (is_same(a, a_base, b, b_base)) goto pop_next; + if (is_same(a, b)) goto pop_next; tailrecur_ne: switch (primary_tag(a)) { case TAG_PRIMARY_LIST: if (is_list(b)) { - Eterm* aval = list_val_rel(a, a_base); - Eterm* bval = list_val_rel(b, b_base); + Eterm* aval = list_val(a); + Eterm* bval = list_val(b); while (1) { Eterm atmp = CAR(aval); Eterm btmp = CAR(bval); - if (!is_same(atmp,a_base,btmp,b_base)) { + if (!is_same(atmp,btmp)) { WSTACK_PUSH2(stack,(UWord) CDR(bval),(UWord) CDR(aval)); a = atmp; b = btmp; @@ -2625,7 +2639,7 @@ tailrecur_ne: } atmp = CDR(aval); btmp = CDR(bval); - if (is_same(atmp,a_base,btmp,b_base)) { + if (is_same(atmp,btmp)) { goto pop_next; } if (is_not_list(atmp) || is_not_list(btmp)) { @@ -2633,22 +2647,22 @@ tailrecur_ne: b = btmp; goto tailrecur_ne; } - aval = list_val_rel(atmp, a_base); - bval = list_val_rel(btmp, b_base); + aval = list_val(atmp); + bval = list_val(btmp); } } break; /* not equal */ case TAG_PRIMARY_BOXED: { - Eterm hdr = *boxed_val_rel(a,a_base); + Eterm hdr = *boxed_val(a); switch (hdr & _TAG_HEADER_MASK) { case ARITYVAL_SUBTAG: { - aa = tuple_val_rel(a, a_base); - if (!is_boxed(b) || *boxed_val_rel(b,b_base) != *aa) + aa = tuple_val(a); + if (!is_boxed(b) || *boxed_val(b) != *aa) goto not_equal; - bb = tuple_val_rel(b,b_base); + bb = tuple_val(b); if ((sz = arityval(*aa)) == 0) goto pop_next; ++aa; ++bb; @@ -2667,16 +2681,16 @@ tailrecur_ne: Uint a_bitoffs; Uint b_bitoffs; - if (!is_binary_rel(b,b_base)) { + if (!is_binary(b)) { goto not_equal; } - a_size = binary_size_rel(a,a_base); - b_size = binary_size_rel(b,b_base); + a_size = binary_size(a); + b_size = binary_size(b); if (a_size != b_size) { goto not_equal; } - ERTS_GET_BINARY_BYTES_REL(a, a_ptr, a_bitoffs, a_bitsize, a_base); - ERTS_GET_BINARY_BYTES_REL(b, b_ptr, b_bitoffs, b_bitsize, b_base); + ERTS_GET_BINARY_BYTES(a, a_ptr, a_bitoffs, a_bitsize); + ERTS_GET_BINARY_BYTES(b, b_ptr, b_bitoffs, b_bitsize); if ((a_bitsize | b_bitsize | a_bitoffs | b_bitoffs) == 0) { if (sys_memcmp(a_ptr, b_ptr, a_size) == 0) goto pop_next; } else if (a_bitsize == b_bitsize) { @@ -2687,9 +2701,9 @@ tailrecur_ne: } case EXPORT_SUBTAG: { - if (is_export_rel(b,b_base)) { - Export* a_exp = *((Export **) (export_val_rel(a,a_base) + 1)); - Export* b_exp = *((Export **) (export_val_rel(b,b_base) + 1)); + if (is_export(b)) { + Export* a_exp = *((Export **) (export_val(a) + 1)); + Export* b_exp = *((Export **) (export_val(b) + 1)); if (a_exp == b_exp) goto pop_next; } break; /* not equal */ @@ -2699,10 +2713,10 @@ tailrecur_ne: ErlFunThing* f1; ErlFunThing* f2; - if (!is_fun_rel(b,b_base)) + if (!is_fun(b)) goto not_equal; - f1 = (ErlFunThing *) fun_val_rel(a,a_base); - f2 = (ErlFunThing *) fun_val_rel(b,b_base); + f1 = (ErlFunThing *) fun_val(a); + f2 = (ErlFunThing *) fun_val(b); if (f1->fe->module != f2->fe->module || f1->fe->old_index != f2->fe->old_index || f1->fe->old_uniq != f2->fe->old_uniq || @@ -2720,15 +2734,15 @@ tailrecur_ne: ExternalThing *ap; ExternalThing *bp; - if(!is_external_rel(b,b_base)) + if(!is_external(b)) goto not_equal; - ap = external_thing_ptr_rel(a,a_base); - bp = external_thing_ptr_rel(b,b_base); + ap = external_thing_ptr(a); + bp = external_thing_ptr(b); if(ap->header == bp->header && ap->node == bp->node) { - ASSERT(1 == external_data_words_rel(a,a_base)); - ASSERT(1 == external_data_words_rel(b,b_base)); + ASSERT(1 == external_data_words(a)); + ASSERT(1 == external_data_words(b)); if (ap->data.ui[0] == bp->data.ui[0]) goto pop_next; } @@ -2749,11 +2763,11 @@ tailrecur_ne: ExternalThing* athing; ExternalThing* bthing; - if(!is_external_ref_rel(b,b_base)) + if(!is_external_ref(b)) goto not_equal; - athing = external_thing_ptr_rel(a,a_base); - bthing = external_thing_ptr_rel(b,b_base); + athing = external_thing_ptr(a); + bthing = external_thing_ptr(b); if(athing->node != bthing->node) goto not_equal; @@ -2765,12 +2779,12 @@ tailrecur_ne: goto ref_common; case REF_SUBTAG: - if (!is_internal_ref_rel(b,b_base)) + if (!is_internal_ref(b)) goto not_equal; { - RefThing* athing = ref_thing_ptr_rel(a,a_base); - RefThing* bthing = ref_thing_ptr_rel(b,b_base); + RefThing* athing = ref_thing_ptr(a); + RefThing* bthing = ref_thing_ptr(b); alen = internal_thing_ref_no_of_numbers(athing); blen = internal_thing_ref_no_of_numbers(bthing); anum = internal_thing_ref_numbers(athing); @@ -2820,10 +2834,10 @@ tailrecur_ne: { int i; - if (!is_big_rel(b,b_base)) + if (!is_big(b)) goto not_equal; - aa = big_val_rel(a,a_base); - bb = big_val_rel(b,b_base); + aa = big_val(a); + bb = big_val(b); if (*aa != *bb) goto not_equal; i = BIG_ARITY(aa); @@ -2838,19 +2852,19 @@ tailrecur_ne: FloatDef af; FloatDef bf; - if (is_float_rel(b,b_base)) { - GET_DOUBLE_REL(a, af, a_base); - GET_DOUBLE_REL(b, bf, b_base); + if (is_float(b)) { + GET_DOUBLE(a, af); + GET_DOUBLE(b, bf); if (af.fd == bf.fd) goto pop_next; } break; /* not equal */ } case MAP_SUBTAG: - if (is_flatmap_rel(a, a_base)) { - aa = flatmap_val_rel(a, a_base); - if (!is_boxed(b) || *boxed_val_rel(b,b_base) != *aa) + if (is_flatmap(a)) { + aa = flatmap_val(a); + if (!is_boxed(b) || *boxed_val(b) != *aa) goto not_equal; - bb = flatmap_val_rel(b,b_base); + bb = flatmap_val(b); sz = flatmap_get_size((flatmap_t*)aa); if (sz != flatmap_get_size((flatmap_t*)bb)) goto not_equal; @@ -2862,11 +2876,11 @@ tailrecur_ne: goto term_array; } else { - if (!is_boxed(b) || *boxed_val_rel(b,b_base) != hdr) + if (!is_boxed(b) || *boxed_val(b) != hdr) goto not_equal; - aa = hashmap_val_rel(a, a_base) + 1; - bb = hashmap_val_rel(b, b_base) + 1; + aa = hashmap_val(a) + 1; + bb = hashmap_val(b) + 1; switch (hdr & _HEADER_MAP_SUBTAG_MASK) { case HAMT_SUBTAG_HEAD_ARRAY: aa++; bb++; @@ -2899,7 +2913,7 @@ term_array: /* arrays in 'aa' and 'bb', length in 'sz' */ Eterm* bp = bb; Sint i = sz; for (;;) { - if (!is_same(*ap,a_base,*bp,b_base)) break; + if (!is_same(*ap,*bp)) break; if (--i == 0) goto pop_next; ++ap; ++bp; @@ -2977,7 +2991,7 @@ static int cmpbytes(byte *s1, int l1, byte *s2, int l2) #define float_comp(x,y) (((x)<(y)) ? -1 : (((x)==(y)) ? 0 : 1)) -static int cmp_atoms(Eterm a, Eterm b) +int erts_cmp_atoms(Eterm a, Eterm b) { Atom *aa = atom_tab(atom_val(a)); Atom *bb = atom_tab(atom_val(b)); @@ -2988,7 +3002,6 @@ static int cmp_atoms(Eterm a, Eterm b) bb->name+3, bb->len-3); } -#if !HALFWORD_HEAP /* cmp(Eterm a, Eterm b) * For compatibility with HiPE - arith-based compare. */ @@ -2996,39 +3009,22 @@ Sint cmp(Eterm a, Eterm b) { return erts_cmp(a, b, 0, 0); } -#endif -#if HALFWORD_HEAP -static Sint erts_cmp_compound_rel_opt(Eterm a, Eterm* a_base, - Eterm b, Eterm* b_base, - int exact, int eq_only); -#else -static Sint erts_cmp_compound(Eterm a, Eterm b, int exact, int eq_only); -#endif +Sint erts_cmp_compound(Eterm a, Eterm b, int exact, int eq_only); -#if HALFWORD_HEAP -Sint erts_cmp_rel_opt(Eterm a, Eterm* a_base, - Eterm b, Eterm* b_base, - int exact, int eq_only) -#else Sint erts_cmp(Eterm a, Eterm b, int exact, int eq_only) -#endif { if (is_atom(a) && is_atom(b)) { - return cmp_atoms(a, b); + return erts_cmp_atoms(a, b); } else if (is_both_small(a, b)) { return (signed_val(a) - signed_val(b)); - } else if (is_float_rel(a, a_base) && is_float_rel(b, b_base)) { + } else if (is_float(a) && is_float(b)) { FloatDef af, bf; - GET_DOUBLE_REL(a, af, a_base); - GET_DOUBLE_REL(b, bf, b_base); + GET_DOUBLE(a, af); + GET_DOUBLE(b, bf); return float_comp(af.fd, bf.fd); } -#if HALFWORD_HEAP - return erts_cmp_compound_rel_opt(a,a_base,b,b_base,exact,eq_only); -#else return erts_cmp_compound(a,b,exact,eq_only); -#endif } @@ -3036,13 +3032,7 @@ Sint erts_cmp(Eterm a, Eterm b, int exact, int eq_only) * exact = 1 -> term-based compare * exact = 0 -> arith-based compare */ -#if HALFWORD_HEAP -static Sint erts_cmp_compound_rel_opt(Eterm a, Eterm* a_base, - Eterm b, Eterm* b_base, - int exact, int eq_only) -#else -static Sint erts_cmp_compound(Eterm a, Eterm b, int exact, int eq_only) -#endif +Sint erts_cmp_compound(Eterm a, Eterm b, int exact, int eq_only) { #define PSTACK_TYPE struct erts_cmp_hashmap_state struct erts_cmp_hashmap_state { @@ -3099,7 +3089,7 @@ static Sint erts_cmp_compound(Eterm a, Eterm b, int exact, int eq_only) do { \ if((AN) != (BN)) { \ if((AN)->sysname != (BN)->sysname) \ - RETURN_NEQ(cmp_atoms((AN)->sysname, (BN)->sysname)); \ + RETURN_NEQ(erts_cmp_atoms((AN)->sysname, (BN)->sysname)); \ ASSERT((AN)->creation != (BN)->creation); \ RETURN_NEQ(((AN)->creation < (BN)->creation) ? -1 : 1); \ } \ @@ -3109,7 +3099,7 @@ static Sint erts_cmp_compound(Eterm a, Eterm b, int exact, int eq_only) bodyrecur: j = 0; tailrecur: - if (is_same(a,a_base,b,b_base)) { /* Equal values or pointers. */ + if (is_same(a,b)) { /* Equal values or pointers. */ goto pop_next; } tailrecur_ne: @@ -3117,7 +3107,7 @@ tailrecur_ne: /* deal with majority (?) cases by brute-force */ if (is_atom(a)) { if (is_atom(b)) { - ON_CMP_GOTO(cmp_atoms(a, b)); + ON_CMP_GOTO(erts_cmp_atoms(a, b)); } } else if (is_both_small(a, b)) { ON_CMP_GOTO(signed_val(a) - signed_val(b)); @@ -3135,9 +3125,9 @@ tailrecur_ne: if (is_internal_port(b)) { bnode = erts_this_node; bdata = internal_port_data(b); - } else if (is_external_port_rel(b,b_base)) { - bnode = external_port_node_rel(b,b_base); - bdata = external_port_data_rel(b,b_base); + } else if (is_external_port(b)) { + bnode = external_port_node(b); + bdata = external_port_data(b); } else { a_tag = PORT_DEF; goto mixed_types; @@ -3153,9 +3143,9 @@ tailrecur_ne: if (is_internal_pid(b)) { bnode = erts_this_node; bdata = internal_pid_data(b); - } else if (is_external_pid_rel(b,b_base)) { - bnode = external_pid_node_rel(b,b_base); - bdata = external_pid_data_rel(b,b_base); + } else if (is_external_pid(b)) { + bnode = external_pid_node(b); + bdata = external_pid_data(b); } else { a_tag = PID_DEF; goto mixed_types; @@ -3188,12 +3178,12 @@ tailrecur_ne: a_tag = LIST_DEF; goto mixed_types; } - aa = list_val_rel(a,a_base); - bb = list_val_rel(b,b_base); + aa = list_val(a); + bb = list_val(b); while (1) { Eterm atmp = CAR(aa); Eterm btmp = CAR(bb); - if (!is_same(atmp,a_base,btmp,b_base)) { + if (!is_same(atmp,btmp)) { WSTACK_PUSH2(stack,(UWord) CDR(bb),(UWord) CDR(aa)); a = atmp; b = btmp; @@ -3201,7 +3191,7 @@ tailrecur_ne: } atmp = CDR(aa); btmp = CDR(bb); - if (is_same(atmp,a_base,btmp,b_base)) { + if (is_same(atmp,btmp)) { goto pop_next; } if (is_not_list(atmp) || is_not_list(btmp)) { @@ -3209,20 +3199,20 @@ tailrecur_ne: b = btmp; goto tailrecur_ne; } - aa = list_val_rel(atmp,a_base); - bb = list_val_rel(btmp,b_base); + aa = list_val(atmp); + bb = list_val(btmp); } case TAG_PRIMARY_BOXED: { - Eterm ahdr = *boxed_val_rel(a,a_base); + Eterm ahdr = *boxed_val(a); switch ((ahdr & _TAG_HEADER_MASK) >> _TAG_PRIMARY_SIZE) { case (_TAG_HEADER_ARITYVAL >> _TAG_PRIMARY_SIZE): - if (!is_tuple_rel(b,b_base)) { + if (!is_tuple(b)) { a_tag = TUPLE_DEF; goto mixed_types; } - aa = tuple_val_rel(a,a_base); - bb = tuple_val_rel(b,b_base); + aa = tuple_val(a); + bb = tuple_val(b); /* compare the arities */ i = arityval(ahdr); /* get the arity*/ if (i != arityval(*bb)) { @@ -3238,18 +3228,18 @@ tailrecur_ne: { struct erts_cmp_hashmap_state* sp; if (is_flatmap_header(ahdr)) { - if (!is_flatmap_rel(b,b_base)) { - if (is_hashmap_rel(b,b_base)) { - aa = (Eterm *)flatmap_val_rel(a,a_base); - i = flatmap_get_size((flatmap_t*)aa) - hashmap_size_rel(b,b_base); + if (!is_flatmap(b)) { + if (is_hashmap(b)) { + aa = (Eterm *)flatmap_val(a); + i = flatmap_get_size((flatmap_t*)aa) - hashmap_size(b); ASSERT(i != 0); RETURN_NEQ(i); } a_tag = MAP_DEF; goto mixed_types; } - aa = (Eterm *)flatmap_val_rel(a,a_base); - bb = (Eterm *)flatmap_val_rel(b,b_base); + aa = (Eterm *)flatmap_val(a); + bb = (Eterm *)flatmap_val(b); i = flatmap_get_size((flatmap_t*)aa); if (i != flatmap_get_size((flatmap_t*)bb)) { @@ -3276,21 +3266,21 @@ tailrecur_ne: goto bodyrecur; } } - if (!is_hashmap_rel(b,b_base)) { - if (is_flatmap_rel(b,b_base)) { - bb = (Eterm *)flatmap_val_rel(b,b_base); - i = hashmap_size_rel(a,a_base) - flatmap_get_size((flatmap_t*)bb); + if (!is_hashmap(b)) { + if (is_flatmap(b)) { + bb = (Eterm *)flatmap_val(b); + i = hashmap_size(a) - flatmap_get_size((flatmap_t*)bb); ASSERT(i != 0); RETURN_NEQ(i); } a_tag = MAP_DEF; goto mixed_types; } - i = hashmap_size_rel(a,a_base) - hashmap_size_rel(b,b_base); + i = hashmap_size(a) - hashmap_size(b); if (i) { RETURN_NEQ(i); } - if (hashmap_size_rel(a,a_base) == 0) { + if (hashmap_size(a) == 0) { goto pop_next; } @@ -3325,48 +3315,48 @@ tailrecur_ne: goto bodyrecur; } case (_TAG_HEADER_FLOAT >> _TAG_PRIMARY_SIZE): - if (!is_float_rel(b,b_base)) { + if (!is_float(b)) { a_tag = FLOAT_DEF; goto mixed_types; } else { FloatDef af; FloatDef bf; - GET_DOUBLE_REL(a, af, a_base); - GET_DOUBLE_REL(b, bf, b_base); + GET_DOUBLE(a, af); + GET_DOUBLE(b, bf); ON_CMP_GOTO(float_comp(af.fd, bf.fd)); } case (_TAG_HEADER_POS_BIG >> _TAG_PRIMARY_SIZE): case (_TAG_HEADER_NEG_BIG >> _TAG_PRIMARY_SIZE): - if (!is_big_rel(b,b_base)) { + if (!is_big(b)) { a_tag = BIG_DEF; goto mixed_types; } - ON_CMP_GOTO(big_comp(rterm2wterm(a,a_base), rterm2wterm(b,b_base))); + ON_CMP_GOTO(big_comp(a, b)); case (_TAG_HEADER_EXPORT >> _TAG_PRIMARY_SIZE): - if (!is_export_rel(b,b_base)) { + if (!is_export(b)) { a_tag = EXPORT_DEF; goto mixed_types; } else { - Export* a_exp = *((Export **) (export_val_rel(a,a_base) + 1)); - Export* b_exp = *((Export **) (export_val_rel(b,b_base) + 1)); + Export* a_exp = *((Export **) (export_val(a) + 1)); + Export* b_exp = *((Export **) (export_val(b) + 1)); - if ((j = cmp_atoms(a_exp->code[0], b_exp->code[0])) != 0) { + if ((j = erts_cmp_atoms(a_exp->code[0], b_exp->code[0])) != 0) { RETURN_NEQ(j); } - if ((j = cmp_atoms(a_exp->code[1], b_exp->code[1])) != 0) { + if ((j = erts_cmp_atoms(a_exp->code[1], b_exp->code[1])) != 0) { RETURN_NEQ(j); } ON_CMP_GOTO((Sint) a_exp->code[2] - (Sint) b_exp->code[2]); } break; case (_TAG_HEADER_FUN >> _TAG_PRIMARY_SIZE): - if (!is_fun_rel(b,b_base)) { + if (!is_fun(b)) { a_tag = FUN_DEF; goto mixed_types; } else { - ErlFunThing* f1 = (ErlFunThing *) fun_val_rel(a,a_base); - ErlFunThing* f2 = (ErlFunThing *) fun_val_rel(b,b_base); + ErlFunThing* f1 = (ErlFunThing *) fun_val(a); + ErlFunThing* f2 = (ErlFunThing *) fun_val(b); Sint diff; diff = cmpbytes(atom_tab(atom_val(f1->fe->module))->name, @@ -3398,29 +3388,29 @@ tailrecur_ne: if (is_internal_pid(b)) { bnode = erts_this_node; bdata = internal_pid_data(b); - } else if (is_external_pid_rel(b,b_base)) { - bnode = external_pid_node_rel(b,b_base); - bdata = external_pid_data_rel(b,b_base); + } else if (is_external_pid(b)) { + bnode = external_pid_node(b); + bdata = external_pid_data(b); } else { a_tag = EXTERNAL_PID_DEF; goto mixed_types; } - anode = external_pid_node_rel(a,a_base); - adata = external_pid_data_rel(a,a_base); + anode = external_pid_node(a); + adata = external_pid_data(a); goto pid_common; case (_TAG_HEADER_EXTERNAL_PORT >> _TAG_PRIMARY_SIZE): if (is_internal_port(b)) { bnode = erts_this_node; bdata = internal_port_data(b); - } else if (is_external_port_rel(b,b_base)) { - bnode = external_port_node_rel(b,b_base); - bdata = external_port_data_rel(b,b_base); + } else if (is_external_port(b)) { + bnode = external_port_node(b); + bdata = external_port_data(b); } else { a_tag = EXTERNAL_PORT_DEF; goto mixed_types; } - anode = external_port_node_rel(a,a_base); - adata = external_port_data_rel(a,a_base); + anode = external_port_node(a); + adata = external_port_data(a); goto port_common; case (_TAG_HEADER_REF >> _TAG_PRIMARY_SIZE): /* @@ -3428,14 +3418,13 @@ tailrecur_ne: * (32-bit words), *not* ref data words. */ - - if (is_internal_ref_rel(b,b_base)) { - RefThing* bthing = ref_thing_ptr_rel(b,b_base); + if (is_internal_ref(b)) { + RefThing* bthing = ref_thing_ptr(b); bnode = erts_this_node; bnum = internal_thing_ref_numbers(bthing); blen = internal_thing_ref_no_of_numbers(bthing); - } else if(is_external_ref_rel(b,b_base)) { - ExternalThing* bthing = external_thing_ptr_rel(b,b_base); + } else if(is_external_ref(b)) { + ExternalThing* bthing = external_thing_ptr(b); bnode = bthing->node; bnum = external_thing_ref_numbers(bthing); blen = external_thing_ref_no_of_numbers(bthing); @@ -3444,7 +3433,7 @@ tailrecur_ne: goto mixed_types; } { - RefThing* athing = ref_thing_ptr_rel(a,a_base); + RefThing* athing = ref_thing_ptr(a); anode = erts_this_node; anum = internal_thing_ref_numbers(athing); alen = internal_thing_ref_no_of_numbers(athing); @@ -3477,13 +3466,13 @@ tailrecur_ne: RETURN_NEQ((Sint32) (anum[i] - bnum[i])); goto pop_next; case (_TAG_HEADER_EXTERNAL_REF >> _TAG_PRIMARY_SIZE): - if (is_internal_ref_rel(b,b_base)) { - RefThing* bthing = ref_thing_ptr_rel(b,b_base); + if (is_internal_ref(b)) { + RefThing* bthing = ref_thing_ptr(b); bnode = erts_this_node; bnum = internal_thing_ref_numbers(bthing); blen = internal_thing_ref_no_of_numbers(bthing); - } else if (is_external_ref_rel(b,b_base)) { - ExternalThing* bthing = external_thing_ptr_rel(b,b_base); + } else if (is_external_ref(b)) { + ExternalThing* bthing = external_thing_ptr(b); bnode = bthing->node; bnum = external_thing_ref_numbers(bthing); blen = external_thing_ref_no_of_numbers(bthing); @@ -3492,7 +3481,7 @@ tailrecur_ne: goto mixed_types; } { - ExternalThing* athing = external_thing_ptr_rel(a,a_base); + ExternalThing* athing = external_thing_ptr(a); anode = athing->node; anum = external_thing_ref_numbers(athing); alen = external_thing_ref_no_of_numbers(athing); @@ -3500,13 +3489,13 @@ tailrecur_ne: goto ref_common; default: /* Must be a binary */ - ASSERT(is_binary_rel(a,a_base)); - if (!is_binary_rel(b,b_base)) { + ASSERT(is_binary(a)); + if (!is_binary(b)) { a_tag = BINARY_DEF; goto mixed_types; } else { - Uint a_size = binary_size_rel(a,a_base); - Uint b_size = binary_size_rel(b,b_base); + Uint a_size = binary_size(a); + Uint b_size = binary_size(b); Uint a_bitsize; Uint b_bitsize; Uint a_bitoffs; @@ -3515,8 +3504,8 @@ tailrecur_ne: int cmp; byte* a_ptr; byte* b_ptr; - ERTS_GET_BINARY_BYTES_REL(a, a_ptr, a_bitoffs, a_bitsize, a_base); - ERTS_GET_BINARY_BYTES_REL(b, b_ptr, b_bitoffs, b_bitsize, b_base); + ERTS_GET_BINARY_BYTES(a, a_ptr, a_bitoffs, a_bitsize); + ERTS_GET_BINARY_BYTES(b, b_ptr, b_bitoffs, b_bitsize); if ((a_bitsize | b_bitsize | a_bitoffs | b_bitoffs) == 0) { min_size = (a_size < b_size) ? a_size : b_size; if ((cmp = sys_memcmp(a_ptr, b_ptr, min_size)) != 0) { @@ -3547,13 +3536,8 @@ tailrecur_ne: { FloatDef f1, f2; Eterm big; -#if HALFWORD_HEAP - Wterm aw = is_immed(a) ? a : rterm2wterm(a,a_base); - Wterm bw = is_immed(b) ? b : rterm2wterm(b,b_base); -#else Eterm aw = a; Eterm bw = b; -#endif #define MAX_LOSSLESS_FLOAT ((double)((1LL << 53) - 2)) #define MIN_LOSSLESS_FLOAT ((double)(((1LL << 53) - 2)*-1)) #define BIG_ARITY_FLOAT_MAX (1024 / D_EXP) /* arity of max float as a bignum */ @@ -3625,7 +3609,7 @@ tailrecur_ne: } } else { big = double_to_big(f2.fd, big_buf, sizeof(big_buf)/sizeof(Eterm)); - j = big_comp(aw, rterm2wterm(big,big_buf)); + j = big_comp(aw, big); } if (_NUMBER_CODE(a_tag, b_tag) == FLOAT_BIG) { j = -j; @@ -3673,9 +3657,9 @@ term_array: /* arrays in 'aa' and 'bb', length in 'i' */ while (--i) { a = *aa++; b = *bb++; - if (!is_same(a,a_base, b,b_base)) { + if (!is_same(a, b)) { if (is_atom(a) && is_atom(b)) { - if ((j = cmp_atoms(a, b)) != 0) { + if ((j = erts_cmp_atoms(a, b)) != 0) { goto not_equal; } } else if (is_both_small(a, b)) { @@ -3946,11 +3930,11 @@ void bin_write(int to, void *to_arg, byte* buf, size_t sz) /* Fill buf with the contents of bytelist list return number of chars in list or -1 for error */ -int -intlist_to_buf(Eterm list, char *buf, int len) +Sint +intlist_to_buf(Eterm list, char *buf, Sint len) { Eterm* listptr; - int sz = 0; + Sint sz = 0; if (is_nil(list)) return 0; @@ -4365,7 +4349,7 @@ iolist_size(const int yield_support, ErtsIOListState *state, Eterm obj, ErlDrvSi { int res, init_yield_count, yield_count; Eterm* objp; - Uint size = (Uint) *sizep; /* Intentionally Uint due to halfword heap */ + Uint size = (Uint) *sizep; DECLARE_ESTACK(s); if (!yield_support) @@ -4497,11 +4481,12 @@ int erts_iolist_size(Eterm obj, ErlDrvSizeT* sizep) return iolist_size(0, NULL, obj, sizep); } -/* return 0 if item is not a non-empty flat list of bytes */ -int +/* return 0 if item is not a non-empty flat list of bytes + otherwise return the nonzero length of the list */ +Sint is_string(Eterm list) { - int len = 0; + Sint len = 0; while(is_list(list)) { Eterm* consp = list_val(list); |