diff options
Diffstat (limited to 'erts/emulator')
40 files changed, 1467 insertions, 701 deletions
diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index 8f65e71531..badd69856e 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -228,9 +228,6 @@ atom endian atom env atom eof atom eol -atom exception_from -atom exception_trace -atom extended atom Eq='=:=' atom Eqeq='==' atom erl_tracer @@ -243,6 +240,8 @@ atom ets atom ETS_TRANSFER='ETS-TRANSFER' atom event atom exact_reductions +atom exception_from +atom exception_trace atom exclusive atom exit_status atom existing @@ -251,7 +250,9 @@ atom existing_ports atom existing atom exiting atom exports +atom extended atom external +atom extra atom false atom fcgi atom fd @@ -389,7 +390,6 @@ atom min_heap_size atom min_bin_vheap_size atom minor_version atom Minus='-' -atom mixed atom module atom module_info atom monitored_by diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 40d44dda4c..15e878ba65 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -762,6 +762,11 @@ check_mod_funs(Process *p, ErlOffHeap *off_heap, char *area, size_t area_size) return 0; } +static Uint hfrag_literal_size(Eterm* start, Eterm* end, + char* lit_start, Uint lit_size); +static void hfrag_literal_copy(Eterm **hpp, ErlOffHeap *ohp, + Eterm *start, Eterm *end, + char *lit_start, Uint lit_size); static Eterm check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls) @@ -842,9 +847,14 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls } /* - * Message queue can contains funs, but (at least currently) no + * Message queue can contains funs, and may contain * literals. If we got references to this module from the message - * queue, a GC cannot remove these... + * queue. + * + * If a literal is in the message queue we maka an explicit copy of + * and attach it to the heap fragment. Each message needs to be + * self contained, we cannot save the literal in the old_heap or + * any other heap than the message it self. */ erts_smp_proc_lock(rp, ERTS_PROC_LOCK_MSGQ); @@ -861,15 +871,31 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls hfrag = msgp->data.heap_frag; else continue; - for (; hfrag; hfrag = hfrag->next) { - if (check_mod_funs(rp, &hfrag->off_heap, mod_start, mod_size)) - return am_true; - /* Should not contain any literals... */ - ASSERT(!any_heap_refs(&hfrag->mem[0], - &hfrag->mem[hfrag->used_size], - literals, - lit_bsize)); - } + { + ErlHeapFragment *hf; + Uint lit_sz; + for (hf=hfrag; hf; hf = hf->next) { + if (check_mod_funs(rp, &hfrag->off_heap, mod_start, mod_size)) + return am_true; + lit_sz = hfrag_literal_size(&hf->mem[0], &hf->mem[hf->used_size], + literals, lit_bsize); + } + if (lit_sz > 0) { + ErlHeapFragment *bp = new_message_buffer(lit_sz); + Eterm *hp = bp->mem; + + for (hf=hfrag; hf; hf = hf->next) { + hfrag_literal_copy(&hp, &bp->off_heap, + &hf->mem[0], &hf->mem[hf->used_size], + literals, lit_bsize); + hfrag=hf; + } + /* link new hfrag last */ + ASSERT(hfrag->next == NULL); + hfrag->next = bp; + bp->next = NULL; + } + } } while (1) { @@ -916,29 +942,26 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls goto try_literal_gc; } -#ifdef DEBUG /* - * Message buffer fragments should not have any references - * to literals, and off heap lists should already have - * been moved into process off heap structure. + * Message buffer fragments (matched messages) + * - off heap lists should already have been moved into + * process off heap structure. + * - Check for literals */ for (msgp = rp->msg_frag; msgp; msgp = msgp->next) { - if (msgp->data.attached == ERTS_MSG_COMBINED_HFRAG) - hfrag = &msgp->hfrag; - else - hfrag = msgp->data.heap_frag; + hfrag = erts_message_to_heap_frag(msgp); for (; hfrag; hfrag = hfrag->next) { Eterm *hp, *hp_end; ASSERT(!check_mod_funs(rp, &hfrag->off_heap, mod_start, mod_size)); hp = &hfrag->mem[0]; hp_end = &hfrag->mem[hfrag->used_size]; - ASSERT(!any_heap_refs(hp, hp_end, literals, lit_bsize)); + + if (any_heap_refs(hp, hp_end, literals, lit_bsize)) + goto try_literal_gc; } } -#endif - return am_false; try_literal_gc: @@ -1038,6 +1061,80 @@ any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size) return 0; } +static Uint +hfrag_literal_size(Eterm* start, Eterm* end, char* lit_start, Uint lit_size) +{ + Eterm* p; + Eterm val; + Uint sz = 0; + + for (p = start; p < end; p++) { + val = *p; + switch (primary_tag(val)) { + case TAG_PRIMARY_BOXED: + case TAG_PRIMARY_LIST: + if (ErtsInArea(val, lit_start, lit_size)) { + sz += size_object(val); + } + break; + case TAG_PRIMARY_HEADER: + if (!header_is_transparent(val)) { + Eterm* new_p; + if (header_is_bin_matchstate(val)) { + ErlBinMatchState *ms = (ErlBinMatchState*) p; + ErlBinMatchBuffer *mb = &(ms->mb); + if (ErtsInArea(mb->orig, lit_start, lit_size)) { + sz += size_object(mb->orig); + } + } + new_p = p + thing_arityval(val); + ASSERT(start <= new_p && new_p < end); + p = new_p; + } + } + } + return sz; +} + +static void +hfrag_literal_copy(Eterm **hpp, ErlOffHeap *ohp, + Eterm *start, Eterm *end, + char *lit_start, Uint lit_size) { + Eterm* p; + Eterm val; + Uint sz; + + for (p = start; p < end; p++) { + val = *p; + switch (primary_tag(val)) { + case TAG_PRIMARY_BOXED: + case TAG_PRIMARY_LIST: + if (ErtsInArea(val, lit_start, lit_size)) { + sz = size_object(val); + val = copy_struct(val, sz, hpp, ohp); + *p = val; + } + break; + case TAG_PRIMARY_HEADER: + if (!header_is_transparent(val)) { + Eterm* new_p; + /* matchstate in message, not possible. */ + if (header_is_bin_matchstate(val)) { + ErlBinMatchState *ms = (ErlBinMatchState*) p; + ErlBinMatchBuffer *mb = &(ms->mb); + if (ErtsInArea(mb->orig, lit_start, lit_size)) { + sz = size_object(mb->orig); + mb->orig = copy_struct(mb->orig, sz, hpp, ohp); + } + } + new_p = p + thing_arityval(val); + ASSERT(start <= new_p && new_p < end); + p = new_p; + } + } + } +} + #undef in_area #ifdef ERTS_SMP diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index f8f2e29c95..4716460a6b 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -1323,11 +1323,7 @@ void process_main(void) if (start_time != 0) { Sint64 diff = erts_timestamp_millis() - start_time; - if (diff > 0 && (Uint) diff > erts_system_monitor_long_schedule -#if defined(ERTS_SMP) && defined(ERTS_DIRTY_SCHEDULERS) - && !ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(c_p)) -#endif - ) { + if (diff > 0 && (Uint) diff > erts_system_monitor_long_schedule) { BeamInstr *inptr = find_function_from_pc(start_time_i); BeamInstr *outptr = find_function_from_pc(c_p->i); monitor_long_schedule_proc(c_p,inptr,outptr,(Uint) diff); @@ -1337,7 +1333,7 @@ void process_main(void) PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - c_p = schedule(c_p, reds_used); + c_p = erts_schedule(NULL, c_p, reds_used); ASSERT(!(c_p->flags & F_HIPE_MODE)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); start_time = 0; @@ -3559,12 +3555,10 @@ do { \ typedef Eterm NifF(struct enif_environment_t*, int argc, Eterm argv[]); NifF* fp = vbf = (NifF*) I[1]; struct enif_environment_t env; -#ifdef ERTS_DIRTY_SCHEDULERS - if (!c_p->scheduler_data) - live_hf_end = ERTS_INVALID_HFRAG_PTR; /* On dirty scheduler */ - else +#ifdef ERTS_SMP + ASSERT(c_p->scheduler_data); #endif - live_hf_end = c_p->mbuf; + live_hf_end = c_p->mbuf; erts_pre_nif(&env, c_p, (struct erl_module_nif*)I[2], NULL); nif_bif_result = (*fp)(&env, bif_nif_arity, reg); if (env.exception_thrown) @@ -3574,10 +3568,7 @@ do { \ PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR); - if (env.exiting) { - ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); - goto do_schedule; - } + ASSERT(!env.exiting); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); } @@ -5162,6 +5153,278 @@ do { \ } } +/* + * erts_dirty_process_main() is what dirty schedulers execute. Since they handle + * only NIF calls they do not need to be able to execute all BEAM + * instructions. + */ +void erts_dirty_process_main(ErtsSchedulerData *esdp) +{ +#ifdef ERTS_DIRTY_SCHEDULERS + Process* c_p = NULL; + ErtsMonotonicTime start_time; +#ifdef DEBUG + ERTS_DECLARE_DUMMY(Eterm pid); +#endif + + /* Pointer to X registers: x(1)..x(N); reg[0] is used when doing GC, + * in all other cases x0 is used. + */ + register Eterm* reg REG_xregs = NULL; + + /* + * Top of heap (next free location); grows upwards. + */ + register Eterm* HTOP REG_htop = NULL; + + /* Stack pointer. Grows downwards; points + * to last item pushed (normally a saved + * continuation pointer). + */ + register Eterm* E REG_stop = NULL; + + /* + * Pointer to next threaded instruction. + */ + register BeamInstr *I REG_I = NULL; + + ERTS_MSACC_DECLARE_CACHE_X() /* a cached value of the tsd pointer for msacc */ + + /* + * start_time always positive for dirty CPU schedulers, + * and negative for dirty I/O schedulers. + */ + + if (ERTS_SCHEDULER_IS_DIRTY_CPU(esdp)) { + start_time = erts_get_monotonic_time(NULL); + ASSERT(start_time >= 0); + } + else { + start_time = ERTS_SINT64_MIN; + ASSERT(start_time < 0); + } + + goto do_dirty_schedule; + + context_switch: + c_p->arity = I[-1]; + c_p->current = I-3; /* Pointer to Mod, Func, Arity */ + + { + int reds_used; + Eterm* argp; + int i; + + /* + * Make sure that there is enough room for the argument registers to be saved. + */ + if (c_p->arity > c_p->max_arg_reg) { + /* + * Yes, this is an expensive operation, but you only pay it the first + * time you call a function with more than 6 arguments which is + * scheduled out. This is better than paying for 26 words of wasted + * space for most processes which never call functions with more than + * 6 arguments. + */ + Uint size = c_p->arity * sizeof(c_p->arg_reg[0]); + if (c_p->arg_reg != c_p->def_arg_reg) { + c_p->arg_reg = (Eterm *) erts_realloc(ERTS_ALC_T_ARG_REG, + (void *) c_p->arg_reg, + size); + } else { + c_p->arg_reg = (Eterm *) erts_alloc(ERTS_ALC_T_ARG_REG, size); + } + c_p->max_arg_reg = c_p->arity; + } + + /* + * Save the argument registers and everything else. + */ + + argp = c_p->arg_reg; + for (i = c_p->arity - 1; i >= 0; i--) { + argp[i] = reg[i]; + } + SWAPOUT; + c_p->i = I; + + do_dirty_schedule: + + if (start_time < 0) { + /* + * Dirty I/O scheduler: + * One reduction consumed regardless of + * time spent in the dirty NIF. + */ + reds_used = esdp->virtual_reds + 1; + } + else { + /* + * Dirty CPU scheduler: + * Currently two reductions consumed per + * micro second spent in the dirty NIF. + */ + ErtsMonotonicTime time; + time = erts_get_monotonic_time(esdp); + time -= start_time; + time = ERTS_MONOTONIC_TO_USEC(time); + time *= (CONTEXT_REDS-1)/1000 + 1; + ASSERT(time >= 0); + if (time == 0) + time = 1; /* At least one reduction */ + time += esdp->virtual_reds; + reds_used = time > INT_MAX ? INT_MAX : (int) time; + } + + PROCESS_MAIN_CHK_LOCKS(c_p); + ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + c_p = erts_schedule(esdp, c_p, reds_used); + + if (start_time >= 0) { + start_time = erts_get_monotonic_time(esdp); + ASSERT(start_time >= 0); + } + } + + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); +#ifdef DEBUG + pid = c_p->common.id; /* Save for debugging purposes */ +#endif + ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); + PROCESS_MAIN_CHK_LOCKS(c_p); + + ASSERT(!(c_p->flags & F_HIPE_MODE)); + ERTS_MSACC_UPDATE_CACHE_X(); + + reg = esdp->x_reg_array; + { + Eterm* argp; + int i; + + argp = c_p->arg_reg; + for (i = c_p->arity - 1; i >= 0; i--) { + reg[i] = argp[i]; + CHECK_TERM(reg[i]); + } + + /* + * We put the original reduction count in the process structure, to reduce + * the code size (referencing a field in a struct through a pointer stored + * in a register gives smaller code than referencing a global variable). + */ + + I = c_p->i; + + ASSERT(BeamOp(op_call_nif) == (BeamInstr *) *I); + + /* + * Set fcalls even though we ignore it, so we don't + * confuse code accessing it... + */ + if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) + c_p->fcalls = 0; + else + c_p->fcalls = CONTEXT_REDS; + + SWAPIN; + +#ifdef USE_VM_PROBES + if (DTRACE_ENABLED(process_scheduled)) { + DTRACE_CHARBUF(process_buf, DTRACE_TERM_BUF_SIZE); + DTRACE_CHARBUF(fun_buf, DTRACE_TERM_BUF_SIZE); + dtrace_proc_str(c_p, process_buf); + + if (ERTS_PROC_IS_EXITING(c_p)) { + strcpy(fun_buf, "<exiting>"); + } else { + BeamInstr *fptr = find_function_from_pc(c_p->i); + if (fptr) { + dtrace_fun_decode(c_p, (Eterm)fptr[0], + (Eterm)fptr[1], (Uint)fptr[2], + NULL, fun_buf); + } else { + erts_snprintf(fun_buf, sizeof(DTRACE_CHARBUF_NAME(fun_buf)), + "<unknown/%p>", *I); + } + } + + DTRACE2(process_scheduled, process_buf, fun_buf); + } +#endif + } + + { +#ifdef DEBUG + Eterm result; +#endif + Eterm arity; + + { + /* + * call_nif is always first instruction in function: + * + * I[-3]: Module + * I[-2]: Function + * I[-1]: Arity + * I[0]: &&call_nif + * I[1]: Function pointer to NIF function + * I[2]: Pointer to erl_module_nif + * I[3]: Function pointer to dirty NIF + */ + BifFunction vbf; + + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_NIF); + + DTRACE_NIF_ENTRY(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]); + c_p->current = I-3; /* current and vbf set to please handle_error */ + SWAPOUT; + PROCESS_MAIN_CHK_LOCKS(c_p); + arity = I[-1]; + ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); + + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + { + typedef Eterm NifF(struct enif_environment_t*, int argc, Eterm argv[]); + NifF* fp = vbf = (NifF*) I[1]; + struct enif_environment_t env; + ASSERT(!c_p->scheduler_data); + + erts_pre_dirty_nif(esdp, &env, c_p, + (struct erl_module_nif*)I[2], NULL); + +#ifdef DEBUG + result = +#else + (void) +#endif + (*fp)(&env, arity, reg); + + erts_post_nif(&env); + + ASSERT(!is_value(result)); + ASSERT(c_p->freason == TRAP); + ASSERT(!(c_p->flags & F_HIBERNATE_SCHED)); + + PROCESS_MAIN_CHK_LOCKS(c_p); + ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR); + if (env.exiting) + goto do_dirty_schedule; + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + } + + DTRACE_NIF_RETURN(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]); + ERTS_HOLE_CHECK(c_p); + SWAPIN; + I = c_p->i; + goto context_switch; + } + } +#endif /* ERTS_DIRTY_SCHEDULERS */ +} + static BifFunction translate_gc_bif(void* gcf) { diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 2a3bd4afe5..b18910e2c7 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -917,9 +917,6 @@ BIF_RETTYPE spawn_opt_1(BIF_ALIST_1) goto error; } else if (arg == am_message_queue_data) { switch (val) { - case am_mixed: - so.flags &= ~(SPO_OFF_HEAP_MSGQ|SPO_ON_HEAP_MSGQ); - break; case am_on_heap: so.flags &= ~SPO_OFF_HEAP_MSGQ; so.flags |= SPO_ON_HEAP_MSGQ; @@ -3836,59 +3833,11 @@ BIF_RETTYPE display_nl_0(BIF_ALIST_0) /**********************************************************************/ -/* stop the system */ -/* ARGSUSED */ -BIF_RETTYPE halt_0(BIF_ALIST_0) -{ - VERBOSE(DEBUG_SYSTEM,("System halted by BIF halt()\n")); - erts_halt(0); - ERTS_BIF_YIELD1(bif_export[BIF_halt_1], BIF_P, am_undefined); -} - -/**********************************************************************/ #define HALT_MSG_SIZE 200 -static char halt_msg[HALT_MSG_SIZE]; - -/* stop the system with exit code */ -/* ARGSUSED */ -BIF_RETTYPE halt_1(BIF_ALIST_1) -{ - Uint code; - - if (term_to_Uint_mask(BIF_ARG_1, &code)) { - int pos_int_code = (int) (code & INT_MAX); - VERBOSE(DEBUG_SYSTEM,("System halted by BIF halt(%T)\n", BIF_ARG_1)); - erts_halt(pos_int_code); - ERTS_BIF_YIELD1(bif_export[BIF_halt_1], BIF_P, am_undefined); - } - else if (ERTS_IS_ATOM_STR("abort", BIF_ARG_1)) { - VERBOSE(DEBUG_SYSTEM,("System halted by BIF halt(%T)\n", BIF_ARG_1)); - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_exit(ERTS_ABORT_EXIT, ""); - } - else if (is_string(BIF_ARG_1) || BIF_ARG_1 == NIL) { - Sint i; - - if ((i = intlist_to_buf(BIF_ARG_1, halt_msg, HALT_MSG_SIZE-1)) < 0) { - goto error; - } - halt_msg[i] = '\0'; - VERBOSE(DEBUG_SYSTEM,("System halted by BIF halt(%T)\n", BIF_ARG_1)); - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_exit(ERTS_DUMP_EXIT, "%s\n", halt_msg); - } - else - goto error; - return NIL; /* Pedantic (lint does not know about erts_exit) */ - error: - BIF_ERROR(BIF_P, BADARG); -} - -/**********************************************************************/ +static char halt_msg[HALT_MSG_SIZE+1]; /* stop the system with exit code and flags */ -/* ARGSUSED */ BIF_RETTYPE halt_2(BIF_ALIST_2) { Uint code; @@ -3924,7 +3873,7 @@ BIF_RETTYPE halt_2(BIF_ALIST_2) ("System halted by BIF halt(%T, %T)\n", BIF_ARG_1, BIF_ARG_2)); if (flush) { erts_halt(pos_int_code); - ERTS_BIF_YIELD1(bif_export[BIF_halt_1], BIF_P, am_undefined); + ERTS_BIF_YIELD2(bif_export[BIF_halt_2], BIF_P, am_undefined, am_undefined); } else { erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); @@ -3940,9 +3889,12 @@ BIF_RETTYPE halt_2(BIF_ALIST_2) else if (is_string(BIF_ARG_1) || BIF_ARG_1 == NIL) { Sint i; - if ((i = intlist_to_buf(BIF_ARG_1, halt_msg, HALT_MSG_SIZE-1)) < 0) { - goto error; - } + if ((i = intlist_to_buf(BIF_ARG_1, halt_msg, HALT_MSG_SIZE)) == -1) { + goto error; + } + if (i == -2) /* truncated string */ + i = HALT_MSG_SIZE; + ASSERT(i >= 0 && i <= HALT_MSG_SIZE); halt_msg[i] = '\0'; VERBOSE(DEBUG_SYSTEM, ("System halted by BIF halt(%T, %T)\n", BIF_ARG_1, BIF_ARG_2)); diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index 872f0f9b2a..065018514a 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -72,8 +72,6 @@ bif erlang:get/1 bif erlang:get_keys/1 bif erlang:group_leader/0 bif erlang:group_leader/2 -bif erlang:halt/0 -bif erlang:halt/1 bif erlang:halt/2 bif erlang:phash/2 bif erlang:phash2/1 diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.c b/erts/emulator/beam/erl_ao_firstfit_alloc.c index fbe4724047..7e239d1f5d 100644 --- a/erts/emulator/beam/erl_ao_firstfit_alloc.c +++ b/erts/emulator/beam/erl_ao_firstfit_alloc.c @@ -123,7 +123,7 @@ struct AOFF_Carrier_t_ { AOFF_RBTree_t rbt_node; /* My node in the carrier tree */ AOFF_RBTree_t* root; /* Root of my block tree */ }; -#define RBT_NODE_TO_MBC(PTR) ((AOFF_Carrier_t*)((char*)(PTR) - offsetof(AOFF_Carrier_t, rbt_node))) +#define RBT_NODE_TO_MBC(PTR) ErtsContainerStruct((PTR), AOFF_Carrier_t, rbt_node) /* To support carrier migration we keep two kinds of rb-trees: diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 2e195db0ee..b410578d37 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -1565,9 +1565,6 @@ process_info_aux(Process *BIF_P, case F_ON_HEAP_MSGQ: res = am_on_heap; break; - case 0: - res = am_mixed; - break; default: res = am_error; ERTS_INTERNAL_ERROR("Inconsistent message queue management state"); @@ -2809,8 +2806,6 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) BIF_RET(am_off_heap); case SPO_ON_HEAP_MSGQ: BIF_RET(am_on_heap); - case 0: - BIF_RET(am_mixed); default: ERTS_INTERNAL_ERROR("Inconsistent message queue management state"); BIF_RET(am_error); diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c index 37f4e1de49..fefa9d8391 100644 --- a/erts/emulator/beam/erl_bif_port.c +++ b/erts/emulator/beam/erl_bif_port.c @@ -1411,7 +1411,7 @@ BIF_RETTYPE decode_packet_3(BIF_ALIST_3) trunc_len = val; goto next_option; case am_line_delimiter: - if (type == TCP_PB_LINE_LF && val >= 0 && val <= 255) { + if (type == TCP_PB_LINE_LF && val <= 255) { delimiter = (char)val; goto next_option; } diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 74979f984a..074ac6d64e 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -95,7 +95,8 @@ /* * The following symbols can be manipulated to "tune" the linear hash array */ -#define CHAIN_LEN 6 /* Medium bucket chain len */ +#define GROW_LIMIT(NACTIVE) ((NACTIVE)*2) +#define SHRINK_LIMIT(NACTIVE) ((NACTIVE) / 2) /* Number of slots per segment */ #define SEGSZ_EXP 8 @@ -463,7 +464,7 @@ db_finalize_dbterm_hash(int cret, DbUpdateHandle* handle); static ERTS_INLINE void try_shrink(DbTableHash* tb) { int nactive = NACTIVE(tb); - if (nactive > SEGSZ && NITEMS(tb) < (nactive * CHAIN_LEN) + if (nactive > SEGSZ && NITEMS(tb) < SHRINK_LIMIT(nactive) && !IS_FIXED(tb)) { shrink(tb, nactive); } @@ -670,8 +671,8 @@ int db_create_hash(Process *p, DbTable *tbl) tb->nsegs = NSEG_1; tb->nslots = SEGSZ; - erts_smp_atomic_init_nob(&tb->is_resizing, 0); #ifdef ERTS_SMP + erts_smp_atomic_init_nob(&tb->is_resizing, 0); if (tb->common.type & DB_FINE_LOCKED) { erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; int i; @@ -862,7 +863,7 @@ Lnew: WUNLOCK_HASH(lck); { int nactive = NACTIVE(tb); - if (nitems > nactive * (CHAIN_LEN+1) && !IS_FIXED(tb)) { + if (nitems > GROW_LIMIT(nactive) && !IS_FIXED(tb)) { grow(tb, nactive); } } @@ -2250,12 +2251,12 @@ static int db_free_table_continue_hash(DbTable *tbl) done /= 2; while(tb->nslots != 0) { - free_seg(tb, 1); + done += 1 + SEGSZ/64 + free_seg(tb, 1); /* * If we have done enough work, get out here. */ - if (++done >= (DELETE_RECORD_LIMIT / CHAIN_LEN / SEGSZ)) { + if (done >= DELETE_RECORD_LIMIT) { return 0; /* Not done */ } } @@ -2604,23 +2605,22 @@ static Eterm build_term_list(Process* p, HashDbTerm* ptr1, HashDbTerm* ptr2, static ERTS_INLINE int begin_resizing(DbTableHash* tb) { +#ifdef ERTS_SMP if (DB_USING_FINE_LOCKING(tb)) - return !erts_smp_atomic_xchg_acqb(&tb->is_resizing, 1); - else { - if (erts_smp_atomic_read_nob(&tb->is_resizing)) - return 0; - erts_smp_atomic_set_nob(&tb->is_resizing, 1); - return 1; - } + return !erts_atomic_xchg_acqb(&tb->is_resizing, 1); + else + ERTS_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&tb->common.rwlock)); +#endif + return 1; } static ERTS_INLINE void done_resizing(DbTableHash* tb) { +#ifdef ERTS_SMP if (DB_USING_FINE_LOCKING(tb)) - erts_smp_atomic_set_relb(&tb->is_resizing, 0); - else - erts_smp_atomic_set_nob(&tb->is_resizing, 0); + erts_atomic_set_relb(&tb->is_resizing, 0); +#endif } /* Grow table with one new bucket. @@ -2871,7 +2871,7 @@ db_lookup_dbterm_hash(Process *p, DbTable *tbl, Eterm key, Eterm obj, int nitems = erts_smp_atomic_inc_read_nob(&tb->common.nitems); int nactive = NACTIVE(tb); - if (nitems > nactive * (CHAIN_LEN + 1) && !IS_FIXED(tb)) { + if (nitems > GROW_LIMIT(nactive) && !IS_FIXED(tb)) { grow(tb, nactive); } } diff --git a/erts/emulator/beam/erl_db_hash.h b/erts/emulator/beam/erl_db_hash.h index e654363cd5..081ff8fafc 100644 --- a/erts/emulator/beam/erl_db_hash.h +++ b/erts/emulator/beam/erl_db_hash.h @@ -60,8 +60,8 @@ typedef struct db_table_hash { /* List of slots where elements have been deleted while table was fixed */ erts_smp_atomic_t fixdel; /* (FixedDeletion*) */ erts_smp_atomic_t nactive; /* Number of "active" slots */ - erts_smp_atomic_t is_resizing; /* grow/shrink in progress */ #ifdef ERTS_SMP + erts_smp_atomic_t is_resizing; /* grow/shrink in progress */ DbTableHashFineLocks* locks; #endif #ifdef VALGRIND diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index d740b2baec..d0d74bbf44 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -1183,22 +1183,14 @@ minor_collection(Process* p, ErlHeapFragment *live_hf_end, adjust_size = p->htop - p->heap; } - goto done; } + else if (need_after > HEAP_SIZE(p)) { + grow_new_heap(p, next_heap_size(p, need_after, 0), objv, nobj); + adjust_size = p->htop - p->heap; + } + /*else: The heap size turned out to be just right. We are done. */ - if (HEAP_SIZE(p) >= need_after) { - /* - * The heap size turned out to be just right. We are done. - */ - goto done; - } - - grow_new_heap(p, next_heap_size(p, need_after, 0), objv, nobj); - adjust_size = p->htop - p->heap; - - done: ASSERT(HEAP_SIZE(p) == next_heap_size(p, HEAP_SIZE(p), 0)); - ASSERT(MBUF(p) == NULL); /* The heap usage during GC should be larger than what we end up after a GC, even if we grow it. If this assertion is not true @@ -1591,6 +1583,9 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end, HIGH_WATER(p) = HEAP_TOP(p); +#ifdef HARDDEBUG + disallow_heap_frag_ref_in_heap(p); +#endif remove_message_buffers(p); if (p->flags & F_ON_HEAP_MSGQ) @@ -1603,9 +1598,6 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end, adjusted = adjust_after_fullsweep(p, need, objv, nobj); -#ifdef HARDDEBUG - disallow_heap_frag_ref_in_heap(p); -#endif ErtsGcQuickSanityCheck(p); return gc_cost(size_after, adjusted ? size_after : 0); @@ -2279,10 +2271,7 @@ move_msgq_to_heap(Process *p) } else { - if (mp->data.attached == ERTS_MSG_COMBINED_HFRAG) - bp = &mp->hfrag; - else - bp = mp->data.heap_frag; + bp = erts_message_to_heap_frag(mp); if (bp->next) erts_move_multi_frags(&factory.hp, factory.off_heap, bp, @@ -2296,18 +2285,13 @@ move_msgq_to_heap(Process *p) free_message_buffer(bp); } else { - ErtsMessage *tmp = erts_alloc_message(0, NULL); - sys_memcpy((void *) tmp->m, (void *) mp->m, - sizeof(Eterm)*ERL_MESSAGE_REF_ARRAY_SZ); - tmp->next = mp->next; - if (p->msg.save == &mp->next) - p->msg.save = &tmp->next; - if (p->msg.last == &mp->next) - p->msg.last = &tmp->next; - *mpp = tmp; + ErtsMessage *new_mp = erts_alloc_message(0, NULL); + sys_memcpy((void *) new_mp->m, (void *) mp->m, + sizeof(Eterm)*ERL_MESSAGE_REF_ARRAY_SZ); + erts_msgq_replace_msg_ref(&p->msg, new_mp, mpp); mp->next = NULL; erts_cleanup_messages(mp); - mp = tmp; + mp = new_mp; } } @@ -3304,11 +3288,7 @@ within2(Eterm *ptr, Process *p, Eterm *real_htop) while (mp) { - if (mp->data.attached == ERTS_MSG_COMBINED_HFRAG) - bp = &mp->hfrag; - else - bp = mp->data.heap_frag; - + bp = erts_message_to_heap_frag(mp); mp = mp->next; search_heap_frags: diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index 0649fb68de..fbdafec4ef 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -585,7 +585,7 @@ void erts_usage(void) erts_fprintf(stderr, "-hpds size initial process dictionary size (default %d)\n", erts_pd_initial_size); erts_fprintf(stderr, "-hmqd val set default message queue data flag for processes,\n"); - erts_fprintf(stderr, " valid values are: off_heap | on_heap | mixed\n"); + erts_fprintf(stderr, " valid values are: off_heap | on_heap\n"); /* erts_fprintf(stderr, "-i module set the boot module (default init)\n"); */ @@ -1526,9 +1526,7 @@ erl_start(int argc, char **argv) erts_pd_initial_size)); } else if (has_prefix("mqd", sub_param)) { arg = get_arg(sub_param+3, argv[i+1], &i); - if (sys_strcmp(arg, "mixed") == 0) - erts_default_spo_flags &= ~(SPO_ON_HEAP_MSGQ|SPO_OFF_HEAP_MSGQ); - else if (sys_strcmp(arg, "on_heap") == 0) { + if (sys_strcmp(arg, "on_heap") == 0) { erts_default_spo_flags &= ~SPO_OFF_HEAP_MSGQ; erts_default_spo_flags |= SPO_ON_HEAP_MSGQ; } diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index 579f6e427d..71ab92937d 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -1123,11 +1123,9 @@ erts_change_message_queue_management(Process *c_p, Eterm new_state) break; case am_on_heap: c_p->flags |= F_ON_HEAP_MSGQ; + c_p->flags &= ~F_OFF_HEAP_MSGQ; erts_smp_atomic32_read_bor_nob(&c_p->state, ERTS_PSFLG_ON_HEAP_MSGQ); - /* fall through */ - case am_mixed: - c_p->flags &= ~F_OFF_HEAP_MSGQ; /* * We are not allowed to clear ERTS_PSFLG_OFF_HEAP_MSGQ * if a off heap change is ongoing. It will be adjusted @@ -1151,11 +1149,6 @@ erts_change_message_queue_management(Process *c_p, Eterm new_state) switch (new_state) { case am_on_heap: break; - case am_mixed: - c_p->flags &= ~F_ON_HEAP_MSGQ; - erts_smp_atomic32_read_band_nob(&c_p->state, - ~ERTS_PSFLG_ON_HEAP_MSGQ); - break; case am_off_heap: c_p->flags &= ~F_ON_HEAP_MSGQ; erts_smp_atomic32_read_band_nob(&c_p->state, @@ -1167,25 +1160,6 @@ erts_change_message_queue_management(Process *c_p, Eterm new_state) } break; - case 0: - res = am_mixed; - - switch (new_state) { - case am_mixed: - break; - case am_on_heap: - c_p->flags |= F_ON_HEAP_MSGQ; - erts_smp_atomic32_read_bor_nob(&c_p->state, - ERTS_PSFLG_ON_HEAP_MSGQ); - break; - case am_off_heap: - goto change_to_off_heap; - default: - res = THE_NON_VALUE; /* badarg */ - break; - } - break; - default: res = am_error; ERTS_INTERNAL_ERROR("Inconsistent message queue management state"); @@ -1371,10 +1345,10 @@ erts_prep_msgq_for_inspection(Process *c_p, Process *rp, mpp = i == 0 ? &rp->msg.first : &mip[i-1].msgp->next; - if (rp->msg.save == &bad_mp->next) - rp->msg.save = mpp; - if (rp->msg.last == &bad_mp->next) - rp->msg.last = mpp; + ASSERT(*mpp == bad_mp); + + erts_msgq_update_internal_pointers(&rp->msg, mpp, &bad_mp->next); + mp = mp->next; *mpp = mp; rp->msg.len--; @@ -1411,12 +1385,7 @@ erts_prep_msgq_for_inspection(Process *c_p, Process *rp, sys_memcpy((void *) tmp->m, (void *) mp->m, sizeof(Eterm)*ERL_MESSAGE_REF_ARRAY_SZ); mpp = i == 0 ? &rp->msg.first : &mip[i-1].msgp->next; - tmp->next = mp->next; - if (rp->msg.save == &mp->next) - rp->msg.save = &tmp->next; - if (rp->msg.last == &mp->next) - rp->msg.last = &tmp->next; - *mpp = tmp; + erts_msgq_replace_msg_ref(&rp->msg, tmp, mpp); erts_save_message_in_proc(rp, mp); mp = tmp; } @@ -1756,7 +1725,7 @@ void erts_factory_trim_and_close(ErtsHeapFactory* factory, case FACTORY_MESSAGE: { ErtsMessage *mp = factory->message; if (mp->data.attached == ERTS_MSG_COMBINED_HFRAG) { - if (!mp->hfrag.next) { + if (!factory->heap_frags) { Uint sz = factory->hp - factory->hp_start; mp = erts_shrink_message(mp, sz, brefs, brefs_size); factory->message = mp; diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index 4493df1c1a..6df969367b 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -366,9 +366,19 @@ ERTS_GLB_FORCE_INLINE ErtsMessage *erts_shrink_message(ErtsMessage *mp, Uint sz, ERTS_GLB_FORCE_INLINE void erts_free_message(ErtsMessage *mp); ERTS_GLB_INLINE Uint erts_used_frag_sz(const ErlHeapFragment*); ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErtsMessage *msg); +ERTS_GLB_INLINE void erts_msgq_update_internal_pointers(ErlMessageQueue *msgq, + ErtsMessage **newpp, + ErtsMessage **oldpp); +ERTS_GLB_INLINE void erts_msgq_replace_msg_ref(ErlMessageQueue *msgq, + ErtsMessage *newp, + ErtsMessage **oldpp); #define ERTS_MSG_COMBINED_HFRAG ((void *) 0x1) +#define erts_message_to_heap_frag(MP) \ + (((MP)->data.attached == ERTS_MSG_COMBINED_HFRAG) ? \ + &(MP)->hfrag : (MP)->data.heap_frag) + #if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_FORCE_INLINE ErtsMessage *erts_alloc_message(Uint sz, Eterm **hpp) @@ -449,10 +459,7 @@ ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErtsMessage *msg) ASSERT(msg->data.attached); if (is_value(ERL_MESSAGE_TERM(msg))) { ErlHeapFragment *bp; - if (msg->data.attached == ERTS_MSG_COMBINED_HFRAG) - bp = &msg->hfrag; - else - bp = msg->data.heap_frag; + bp = erts_message_to_heap_frag(msg); return erts_used_frag_sz(bp); } else if (msg->data.dist_ext->heap_size < 0) @@ -467,6 +474,29 @@ ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErtsMessage *msg) return sz; } } + +ERTS_GLB_INLINE void +erts_msgq_update_internal_pointers(ErlMessageQueue *msgq, + ErtsMessage **newpp, + ErtsMessage **oldpp) +{ + if (msgq->save == oldpp) + msgq->save = newpp; + if (msgq->last == oldpp) + msgq->last = newpp; + if (msgq->saved_last == oldpp) + msgq->saved_last = newpp; +} + +ERTS_GLB_INLINE void +erts_msgq_replace_msg_ref(ErlMessageQueue *msgq, ErtsMessage *newp, ErtsMessage **oldpp) +{ + ErtsMessage *oldp = *oldpp; + newp->next = oldp->next; + erts_msgq_update_internal_pointers(msgq, &newp->next, &oldp->next); + *oldpp = newp; +} + #endif #endif diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 606b73c7b5..4fd82bad10 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -178,9 +178,6 @@ static ERTS_INLINE void ensure_heap(ErlNifEnv* env, size_t may_need) void erts_pre_nif(ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif, Process* tracee) { -#ifdef ERTS_DIRTY_SCHEDULERS - ErtsSchedulerData *esdp; -#endif env->mod_nif = mod_nif; env->proc = p; env->hp = HEAP_TOP(p); @@ -193,57 +190,65 @@ void erts_pre_nif(ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif, ASSERT(p->common.id != ERTS_INVALID_PID); -#ifdef ERTS_DIRTY_SCHEDULERS - esdp = erts_get_scheduler_data(); - ASSERT(esdp); +#if defined(DEBUG) && defined(ERTS_DIRTY_SCHEDULERS) + { + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + ASSERT(esdp); - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { -#ifdef DEBUG - erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state); + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state); - ASSERT(p->scheduler_data == esdp); - ASSERT((state & (ERTS_PSFLG_RUNNING - | ERTS_PSFLG_RUNNING_SYS)) - && !(state & (ERTS_PSFLG_DIRTY_RUNNING - | ERTS_PSFLG_DIRTY_RUNNING_SYS))); + ASSERT(p->scheduler_data == esdp); + ASSERT((state & (ERTS_PSFLG_RUNNING + | ERTS_PSFLG_RUNNING_SYS)) + && !(state & (ERTS_PSFLG_DIRTY_RUNNING + | ERTS_PSFLG_DIRTY_RUNNING_SYS))); + } + } #endif +} - } - else { - Process *sproc; +void erts_pre_dirty_nif(ErtsSchedulerData *esdp, + ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif, + Process* tracee) +{ +#ifdef ERTS_DIRTY_SCHEDULERS + Process *sproc; #ifdef DEBUG - erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state); + erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state); - ASSERT(!p->scheduler_data); - ASSERT((state & ERTS_PSFLG_DIRTY_RUNNING) - && !(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS))); + ASSERT(!p->scheduler_data); + ASSERT((state & ERTS_PSFLG_DIRTY_RUNNING) + && !(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS))); + ASSERT(esdp); #endif - sproc = esdp->dirty_shadow_process; - ASSERT(sproc); - ASSERT(sproc->static_flags & ERTS_STC_FLG_SHADOW_PROC); - ASSERT(erts_smp_atomic32_read_nob(&sproc->state) - == (ERTS_PSFLG_ACTIVE - | ERTS_PSFLG_DIRTY_RUNNING - | ERTS_PSFLG_PROXY)); - - sproc->next = p; - sproc->common.id = p->common.id; - sproc->htop = p->htop; - sproc->stop = p->stop; - sproc->hend = p->hend; - sproc->heap = p->heap; - sproc->abandoned_heap = p->abandoned_heap; - sproc->heap_sz = p->heap_sz; - sproc->high_water = p->high_water; - sproc->old_hend = p->old_hend; - sproc->old_htop = p->old_htop; - sproc->old_heap = p->old_heap; - sproc->mbuf = NULL; - sproc->mbuf_sz = 0; - ERTS_INIT_OFF_HEAP(&sproc->off_heap); - env->proc = sproc; - } + erts_pre_nif(env, p, mod_nif, tracee); + + sproc = esdp->dirty_shadow_process; + ASSERT(sproc); + ASSERT(sproc->static_flags & ERTS_STC_FLG_SHADOW_PROC); + ASSERT(erts_smp_atomic32_read_nob(&sproc->state) + == (ERTS_PSFLG_ACTIVE + | ERTS_PSFLG_DIRTY_RUNNING + | ERTS_PSFLG_PROXY)); + + sproc->next = p; + sproc->common.id = p->common.id; + sproc->htop = p->htop; + sproc->stop = p->stop; + sproc->hend = p->hend; + sproc->heap = p->heap; + sproc->abandoned_heap = p->abandoned_heap; + sproc->heap_sz = p->heap_sz; + sproc->high_water = p->high_water; + sproc->old_hend = p->old_hend; + sproc->old_htop = p->old_htop; + sproc->old_heap = p->old_heap; + sproc->mbuf = NULL; + sproc->mbuf_sz = 0; + ERTS_INIT_OFF_HEAP(&sproc->off_heap); + env->proc = sproc; #endif } @@ -623,10 +628,28 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, } } else { Uint sz = size_object(msg); + ErlOffHeap *ohp; Eterm *hp; - mp = erts_alloc_message(sz, &hp); - msg = copy_struct(msg, sz, &hp, &mp->hfrag.off_heap); - ASSERT(hp == mp->hfrag.mem+mp->hfrag.used_size); + if (env && !env->tracee) { + flush_env(env); + mp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp); + cache_env(env); + } + else { + erts_aint_t state = erts_smp_atomic32_read_nob(&rp->state); + if (state & ERTS_PSFLG_OFF_HEAP_MSGQ) { + mp = erts_alloc_message(sz, &hp); + ohp = sz == 0 ? NULL : &mp->hfrag.off_heap; + } + else { + ErlHeapFragment *bp = new_message_buffer(sz); + mp = erts_alloc_message(0, NULL); + mp->data.heap_frag = bp; + hp = bp->mem; + ohp = &bp->off_heap; + } + } + msg = copy_struct(msg, sz, &hp, ohp); } ERL_MESSAGE_TERM(mp) = msg; @@ -780,19 +803,13 @@ ERL_NIF_TERM enif_make_copy(ErlNifEnv* dst_env, ERL_NIF_TERM src_term) { Uint sz; Eterm* hp; -#ifdef SHCOPY - erts_shcopy_t info; - INITIALIZE_SHCOPY(info); - sz = copy_shared_calculate(src_term, &info); - hp = alloc_heap(dst_env, sz); - src_term = copy_shared_perform(src_term, sz, &info, &hp, &MSO(dst_env->proc)); - DESTROY_SHCOPY(info); - return src_term; -#else + /* + * No preserved sharing allowed as long as literals are also preserved. + * Process independent environment can not be reached by purge. + */ sz = size_object(src_term); hp = alloc_heap(dst_env, sz); return copy_struct(src_term, sz, &hp, &MSO(dst_env->proc)); -#endif } diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index a853ec585b..c0b1d7246c 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -149,7 +149,7 @@ extern BeamInstr beam_apply[]; extern BeamInstr beam_exit[]; extern BeamInstr beam_continue_exit[]; -int ERTS_WRITE_UNLIKELY(erts_default_spo_flags) = 0; +int ERTS_WRITE_UNLIKELY(erts_default_spo_flags) = SPO_ON_HEAP_MSGQ; int ERTS_WRITE_UNLIKELY(erts_eager_check_io) = 1; int ERTS_WRITE_UNLIKELY(erts_sched_compact_load); int ERTS_WRITE_UNLIKELY(erts_sched_balance_util) = 0; @@ -8197,7 +8197,7 @@ sched_dirty_cpu_thread_func(void *vesdp) #endif erts_thread_init_float(); - process_main(); + erts_dirty_process_main(esdp); /* No schedulers should *ever* terminate */ erts_exit(ERTS_ABORT_EXIT, "Dirty CPU scheduler thread number %beu terminated\n", @@ -8242,7 +8242,7 @@ sched_dirty_io_thread_func(void *vesdp) #endif erts_thread_init_float(); - process_main(); + erts_dirty_process_main(esdp); /* No schedulers should *ever* terminate */ erts_exit(ERTS_ABORT_EXIT, "Dirty I/O scheduler thread number %beu terminated\n", @@ -9377,77 +9377,6 @@ scheduler_gc_proc(Process *c_p, int reds_left) return reds; } -static ERTS_INLINE void -clean_dirty_start(Process *p) -{ -#if defined(ERTS_DIRTY_SCHEDULERS) && !defined(ARCH_64) - void *ptr = ERTS_PROC_SET_DIRTY_CPU_START(p, NULL); - if (ptr) - erts_free(ERTS_ALC_T_DIRTY_START, ptr); -#endif -} - -static ERTS_INLINE void -save_dirty_start(ErtsSchedulerData *esdp, Process *c_p) -{ -#ifdef ERTS_DIRTY_SCHEDULERS - if (ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(esdp->run_queue)) { - ErtsMonotonicTime time = erts_get_monotonic_time(esdp); -#ifdef ARCH_64 - ERTS_PROC_SET_DIRTY_CPU_START(c_p, (void *) time); -#else - ErtsMonotonicTime *stimep; - - stimep = (ErtsMonotonicTime *) ERTS_PROC_GET_DIRTY_CPU_START(c_p); - if (!stimep) { - stimep = erts_alloc(ERTS_ALC_T_DIRTY_START, - sizeof(ErtsMonotonicTime)); - ERTS_PROC_SET_DIRTY_CPU_START(c_p, (void *) stimep); - } - *stimep = time; -#endif - } -#endif -} - -static ERTS_INLINE int -get_dirty_reds(ErtsSchedulerData *esdp, Process *c_p) -{ - -#ifndef ERTS_DIRTY_SCHEDULERS - return -1; -#else - ErtsMonotonicTime stime, time; - - if (!ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(esdp->run_queue)) - return 1; - -#ifdef ARCH_64 - stime = (ErtsMonotonicTime) ERTS_PROC_GET_DIRTY_CPU_START(c_p); -#else - { - ErtsMonotonicTime *stimep; - stimep = (ErtsMonotonicTime *) ERTS_PROC_GET_DIRTY_CPU_START(c_p); - ASSERT(stimep); - stime = *stimep; - } -#endif - - time = erts_get_monotonic_time(esdp); - - ASSERT(stime && stime < time); - - time -= stime; - time = ERTS_MONOTONIC_TO_USEC(time); - time *= 2; - - if (time > INT_MAX) - return INT_MAX; - return (int) time; -#endif - -} - /* * schedule() is called from BEAM (process_main()) or HiPE * (hipe_mode_switch()) when the current process is to be @@ -9466,11 +9395,10 @@ get_dirty_reds(ErtsSchedulerData *esdp, Process *c_p) * so that normal processes get to run more frequently. */ -Process *schedule(Process *p, int calls) +Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) { Process *proxy_p = NULL; ErtsRunQueue *rq; - ErtsSchedulerData *esdp; int context_reds; int fcalls; int input_reductions; @@ -9507,8 +9435,19 @@ Process *schedule(Process *p, int calls) * Clean up after the process being scheduled out. */ if (!p) { /* NULL in the very first schedule() call */ +#ifdef ERTS_DIRTY_SCHEDULERS + is_normal_sched = !esdp; + if (is_normal_sched) { + esdp = erts_get_scheduler_data(); + ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp)); + } + else { + ASSERT(ERTS_SCHEDULER_IS_DIRTY(esdp)); + } +#else esdp = erts_get_scheduler_data(); - is_normal_sched = !ERTS_SCHEDULER_IS_DIRTY(esdp); + is_normal_sched = 1; +#endif rq = erts_get_runq_current(esdp); ASSERT(esdp); fcalls = (int) erts_smp_atomic32_read_acqb(&function_calls); @@ -9517,12 +9456,12 @@ Process *schedule(Process *p, int calls) } else { #ifdef ERTS_SMP #ifdef ERTS_DIRTY_SCHEDULERS - esdp = p->scheduler_data; - is_normal_sched = esdp != NULL; - if (is_normal_sched) + is_normal_sched = !esdp; + if (is_normal_sched) { + esdp = p->scheduler_data; ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp)); + } else { - esdp = erts_get_scheduler_data(); ASSERT(ERTS_SCHEDULER_IS_DIRTY(esdp)); } #else @@ -9541,10 +9480,7 @@ Process *schedule(Process *p, int calls) ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); - if (is_normal_sched) - reds = actual_reds = calls - esdp->virtual_reds; - else - reds = actual_reds = get_dirty_reds(esdp, p); + reds = actual_reds = calls - esdp->virtual_reds; ASSERT(actual_reds >= 0); if (reds < ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST) @@ -9994,17 +9930,10 @@ Process *schedule(Process *p, int calls) calls = 0; reds = context_reds; -#ifdef ERTS_SMP - erts_smp_runq_unlock(rq); -#endif /* ERTS_SMP */ - } - if (!is_normal_sched) - save_dirty_start(esdp, p); - #ifdef ERTS_SMP if (flags & ERTS_RUNQ_FLG_PROTECTED) @@ -11206,6 +11135,8 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). flags |= F_ON_HEAP_MSGQ; } + ASSERT((flags & F_ON_HEAP_MSGQ) || (flags & F_OFF_HEAP_MSGQ)); + if (!rq) rq = erts_get_runq_proc(parent); @@ -11218,6 +11149,11 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). goto error; } + ASSERT((erts_smp_atomic32_read_nob(&p->state) + & ERTS_PSFLG_ON_HEAP_MSGQ) + || (erts_smp_atomic32_read_nob(&p->state) + & ERTS_PSFLG_OFF_HEAP_MSGQ)); + #ifdef BM_COUNTERS processes_busy++; #endif @@ -11738,8 +11674,6 @@ delete_process(Process* p) if (nif_export) erts_destroy_nif_export(nif_export); - clean_dirty_start(p); - /* Cleanup psd */ psd = (ErtsPSD *) erts_smp_atomic_read_nob(&p->psd); diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index b44ac442aa..7c98b60647 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -810,25 +810,13 @@ erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi) #define ERTS_PSD_DELAYED_GC_TASK_QS 4 #define ERTS_PSD_NIF_TRAP_EXPORT 5 #define ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF 6 -#define ERTS_PSD_DIRTY_CPU_START 7 -#define ERTS_PSD_SIZE 8 +#define ERTS_PSD_SIZE 7 -#if !defined(HIPE) && !defined(ERTS_DIRTY_SCHEDULERS) +#if !defined(HIPE) # undef ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF -# undef ERTS_PSD_DIRTY_CPU_START # undef ERTS_PSD_SIZE # define ERTS_PSD_SIZE 6 -#elif !defined(HIPE) -# undef ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF -# undef ERTS_PSD_DIRTY_CPU_START -# undef ERTS_PSD_SIZE -# define ERTS_PSD_DIRTY_CPU_START 6 -# define ERTS_PSD_SIZE 7 -#elif !defined(ERTS_DIRTY_SCHEDULERS) -# undef ERTS_PSD_DIRTY_CPU_START -# undef ERTS_PSD_SIZE -# define ERTS_PSD_SIZE 7 #endif typedef struct { @@ -1179,6 +1167,9 @@ void erts_check_for_holes(Process* p); * USR_PRIO -> User prio. i.e., prio the user has set. * PRQ_PRIO -> Prio queue prio, i.e., prio queue currently * enqueued in. + * + * Update etp-proc-state-int in $ERL_TOP/erts/etc/unix/etp-commands.in + * when changing ERTS_PSFLG_*. */ #define ERTS_PSFLGS_ACT_PRIO_MASK \ (ERTS_PSFLGS_PRIO_MASK << ERTS_PSFLGS_ACT_PRIO_OFFSET) @@ -1831,7 +1822,7 @@ Eterm erts_get_schedulers_binds(Process *c_p); Eterm erts_set_cpu_topology(Process *c_p, Eterm term); Eterm erts_bind_schedulers(Process *c_p, Eterm how); ErtsRunQueue *erts_schedid2runq(Uint); -Process *schedule(Process*, int); +Process *erts_schedule(ErtsSchedulerData *, Process*, int); void erts_schedule_misc_op(void (*)(void *), void *); Eterm erl_create_process(Process*, Eterm, Eterm, Eterm, ErlSpawnOpts*); void erts_do_exit_process(Process*, Eterm); @@ -2061,13 +2052,6 @@ erts_psd_set(Process *p, int ix, void *data) ((struct saved_calls *) erts_psd_set((P), ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF, (void *) (SCB))) #endif -#ifdef ERTS_DIRTY_SCHEDULERS -#define ERTS_PROC_GET_DIRTY_CPU_START(P) \ - ((void *) erts_psd_get((P), ERTS_PSD_DIRTY_CPU_START)) -#define ERTS_PROC_SET_DIRTY_CPU_START(P, DCS) \ - ((void *) erts_psd_set((P), ERTS_PSD_DIRTY_CPU_START, (void *) (DCS))) -#endif - ERTS_GLB_INLINE Eterm erts_proc_get_error_handler(Process *p); ERTS_GLB_INLINE Eterm erts_proc_set_error_handler(Process *p, Eterm handler); diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index ca001fc156..3dca58d60b 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -2625,7 +2625,7 @@ static void init_tracer_template(ErtsTracerNif *tnif) { /* default tracer functions */ tnif->tracers[TRACE_FUN_DEFAULT].name = "trace"; - tnif->tracers[TRACE_FUN_DEFAULT].arity = 6; + tnif->tracers[TRACE_FUN_DEFAULT].arity = 5; tnif->tracers[TRACE_FUN_DEFAULT].cb = NULL; tnif->tracers[TRACE_FUN_ENABLED].name = "enabled"; @@ -2634,35 +2634,35 @@ static void init_tracer_template(ErtsTracerNif *tnif) { /* specific tracer functions */ tnif->tracers[TRACE_FUN_T_SEND].name = "trace_send"; - tnif->tracers[TRACE_FUN_T_SEND].arity = 6; + tnif->tracers[TRACE_FUN_T_SEND].arity = 5; tnif->tracers[TRACE_FUN_T_SEND].cb = NULL; tnif->tracers[TRACE_FUN_T_RECEIVE].name = "trace_receive"; - tnif->tracers[TRACE_FUN_T_RECEIVE].arity = 6; + tnif->tracers[TRACE_FUN_T_RECEIVE].arity = 5; tnif->tracers[TRACE_FUN_T_RECEIVE].cb = NULL; tnif->tracers[TRACE_FUN_T_CALL].name = "trace_call"; - tnif->tracers[TRACE_FUN_T_CALL].arity = 6; + tnif->tracers[TRACE_FUN_T_CALL].arity = 5; tnif->tracers[TRACE_FUN_T_CALL].cb = NULL; tnif->tracers[TRACE_FUN_T_SCHED_PROC].name = "trace_running_procs"; - tnif->tracers[TRACE_FUN_T_SCHED_PROC].arity = 6; + tnif->tracers[TRACE_FUN_T_SCHED_PROC].arity = 5; tnif->tracers[TRACE_FUN_T_SCHED_PROC].cb = NULL; tnif->tracers[TRACE_FUN_T_SCHED_PORT].name = "trace_running_ports"; - tnif->tracers[TRACE_FUN_T_SCHED_PORT].arity = 6; + tnif->tracers[TRACE_FUN_T_SCHED_PORT].arity = 5; tnif->tracers[TRACE_FUN_T_SCHED_PORT].cb = NULL; tnif->tracers[TRACE_FUN_T_GC].name = "trace_garbage_collection"; - tnif->tracers[TRACE_FUN_T_GC].arity = 6; + tnif->tracers[TRACE_FUN_T_GC].arity = 5; tnif->tracers[TRACE_FUN_T_GC].cb = NULL; tnif->tracers[TRACE_FUN_T_PROCS].name = "trace_procs"; - tnif->tracers[TRACE_FUN_T_PROCS].arity = 6; + tnif->tracers[TRACE_FUN_T_PROCS].arity = 5; tnif->tracers[TRACE_FUN_T_PROCS].cb = NULL; tnif->tracers[TRACE_FUN_T_PORTS].name = "trace_ports"; - tnif->tracers[TRACE_FUN_T_PORTS].arity = 6; + tnif->tracers[TRACE_FUN_T_PORTS].arity = 5; tnif->tracers[TRACE_FUN_T_PORTS].cb = NULL; /* specific enabled functions */ @@ -2834,10 +2834,12 @@ send_to_tracer_nif_raw(Process *c_p, Process *tracee, Eterm tag, Eterm msg, Eterm extra, Eterm pam_result) { if (tnif || (tnif = lookup_tracer_nif(tracer)) != NULL) { -#define MAP_SIZE 3 - Eterm argv[6], local_heap[3+MAP_SIZE /* values */ + (MAP_SIZE+1 /* keys */)]; +#define MAP_SIZE 4 + Eterm argv[5], local_heap[3+MAP_SIZE /* values */ + (MAP_SIZE+1 /* keys */)]; flatmap_t *map = (flatmap_t*)(local_heap+(MAP_SIZE+1)); Eterm *map_values = flatmap_get_values(map); + Eterm *map_keys = local_heap + 1; + Uint map_elem_count = 0; topt = (tnif->tracers[topt].cb) ? topt : TRACE_FUN_DEFAULT; ASSERT(topt < NIF_TRACER_TYPES); @@ -2846,31 +2848,40 @@ send_to_tracer_nif_raw(Process *c_p, Process *tracee, argv[1] = ERTS_TRACER_STATE(tracer); argv[2] = t_p_id; argv[3] = msg; - argv[4] = extra == THE_NON_VALUE ? am_undefined : extra; - argv[5] = make_flatmap(map); + argv[4] = make_flatmap(map); map->thing_word = MAP_HEADER_FLATMAP; - map->size = MAP_SIZE; - map->keys = TUPLE3(local_heap, am_match_spec_result, am_scheduler_id, am_timestamp); - - *map_values++ = pam_result; - if (tracee_flags & F_TRACE_SCHED_NO) - *map_values++ = make_small(erts_get_scheduler_id()); - else - *map_values++ = am_undefined; + + if (extra != THE_NON_VALUE) { + map_keys[map_elem_count] = am_extra; + map_values[map_elem_count++] = extra; + } + + if (pam_result != am_true) { + map_keys[map_elem_count] = am_match_spec_result; + map_values[map_elem_count++] = pam_result; + } + + if (tracee_flags & F_TRACE_SCHED_NO) { + map_keys[map_elem_count] = am_scheduler_id; + map_values[map_elem_count++] = make_small(erts_get_scheduler_id()); + } + map_keys[map_elem_count] = am_timestamp; if (tracee_flags & F_NOW_TS) #ifdef HAVE_ERTS_NOW_CPU if (erts_cpu_timestamp) - *map_values++ = am_cpu_timestamp; + map_values[map_elem_count++] = am_cpu_timestamp; else #endif - *map_values++ = am_timestamp; + map_values[map_elem_count++] = am_timestamp; else if (tracee_flags & F_STRICT_MON_TS) - *map_values++ = am_strict_monotonic; + map_values[map_elem_count++] = am_strict_monotonic; else if (tracee_flags & F_MON_TS) - *map_values++ = am_monotonic; - else - *map_values++ = am_undefined; + map_values[map_elem_count++] = am_monotonic; + + map->size = map_elem_count; + map->keys = make_tuple(local_heap); + local_heap[0] = make_arityval(map_elem_count); #undef MAP_SIZE erts_nif_call_function(c_p, tracee ? tracee : c_p, diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c index 02c24557c1..2a19211987 100644 --- a/erts/emulator/beam/export.c +++ b/erts/emulator/beam/export.c @@ -31,7 +31,7 @@ #define EXPORT_INITIAL_SIZE 4000 #define EXPORT_LIMIT (512*1024) -#define EXPORT_HASH(m,f,a) ((m)*(f)+(a)) +#define EXPORT_HASH(m,f,a) ((atom_val(m) * atom_val(f)) ^ (a)) #ifdef DEBUG # define IF_DEBUG(x) x @@ -79,8 +79,7 @@ struct export_templ static struct export_blob* entry_to_blob(struct export_entry* ee) { - return (struct export_blob*) - ((char*)ee->ep - offsetof(struct export_blob,exp)); + return ErtsContainerStruct(ee->ep, struct export_blob, exp); } void diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index b76b9cd874..f3d4ac56cd 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -62,6 +62,9 @@ struct enif_environment_t /* ErlNifEnv */ extern void erts_pre_nif(struct enif_environment_t*, Process*, struct erl_module_nif*, Process* tracee); extern void erts_post_nif(struct enif_environment_t* env); +extern void erts_pre_dirty_nif(ErtsSchedulerData *, + struct enif_environment_t*, Process*, + struct erl_module_nif*, Process* tracee); extern Eterm erts_nif_taints(Process* p); extern void erts_print_nif_taints(int to, void* to_arg); void erts_unload_nif(struct erl_module_nif* nif); @@ -1152,6 +1155,7 @@ void print_pass_through(int, byte*, int); int catchlevel(Process*); void init_emulator(void); void process_main(void); +void erts_dirty_process_main(ErtsSchedulerData *); Eterm build_stacktrace(Process* c_p, Eterm exc); Eterm expand_error_value(Process* c_p, Uint freason, Eterm Value); void erts_save_stacktrace(Process* p, struct StackTrace* s, int depth); diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index 0377f6cb5e..01df5476db 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -938,18 +938,32 @@ int erts_port_handle_xports(Port *prt) ** -2 on type error */ -#define SET_VEC(iov, bv, bin, ptr, len, vlen) do { \ - (iov)->iov_base = (ptr); \ - (iov)->iov_len = (len); \ - if (sizeof((iov)->iov_len) < sizeof(len) \ - /* Check if (len) overflowed (iov)->iov_len */ \ - && (iov)->iov_len != (len)) { \ - goto L_overflow; \ - } \ - *(bv)++ = (bin); \ - (iov)++; \ - (vlen)++; \ -} while(0) +#ifdef DEBUG +#define MAX_SYSIOVEC_IOVLEN (1ull << (32 - 1)) +#else +#define MAX_SYSIOVEC_IOVLEN (1ull << (sizeof(((SysIOVec*)0)->iov_len) * 8 - 1)) +#endif + +static ERTS_INLINE void +io_list_to_vec_set_vec(SysIOVec **iov, ErlDrvBinary ***binv, + ErlDrvBinary *bin, byte *ptr, Uint len, + int *vlen) +{ + while (len > MAX_SYSIOVEC_IOVLEN) { + (*iov)->iov_base = ptr; + (*iov)->iov_len = MAX_SYSIOVEC_IOVLEN; + ptr += MAX_SYSIOVEC_IOVLEN; + len -= MAX_SYSIOVEC_IOVLEN; + (*iov)++; + (*vlen)++; + *(*binv)++ = bin; + } + (*iov)->iov_base = ptr; + (*iov)->iov_len = len; + *(*binv)++ = bin; + (*iov)++; + (*vlen)++; +} static int io_list_to_vec(Eterm obj, /* io-list */ @@ -960,11 +974,11 @@ io_list_to_vec(Eterm obj, /* io-list */ { DECLARE_ESTACK(s); Eterm* objp; - char *buf = cbin->orig_bytes; + byte *buf = (byte*)cbin->orig_bytes; Uint len = cbin->orig_size; Uint csize = 0; int vlen = 0; - char* cptr = buf; + byte* cptr = buf; goto L_jump_start; /* avoid push */ @@ -1032,15 +1046,17 @@ io_list_to_vec(Eterm obj, /* io-list */ len -= size; } else { if (csize != 0) { - SET_VEC(iov, binv, cbin, cptr, csize, vlen); + io_list_to_vec_set_vec(&iov, &binv, cbin, + cptr, csize, &vlen); cptr = buf; csize = 0; } if (pb->flags) { erts_emasculate_writable_binary(pb); } - SET_VEC(iov, binv, Binary2ErlDrvBinary(pb->val), - pb->bytes+offset, size, vlen); + io_list_to_vec_set_vec( + &iov, &binv, Binary2ErlDrvBinary(pb->val), + pb->bytes+offset, size, &vlen); } } else { ErlHeapBin* hb = (ErlHeapBin *) bptr; @@ -1060,7 +1076,7 @@ io_list_to_vec(Eterm obj, /* io-list */ } if (csize != 0) { - SET_VEC(iov, binv, cbin, cptr, csize, vlen); + io_list_to_vec_set_vec(&iov, &binv, cbin, cptr, csize, &vlen); } DESTROY_ESTACK(s); @@ -1086,10 +1102,13 @@ do { \ if (_bitsize != 0) goto L_type_error; \ if (thing_subtag(*binary_val(_real)) == REFC_BINARY_SUBTAG && \ _bitoffs == 0) { \ - b_size += _size; \ + b_size += _size; \ if (b_size < _size) goto L_overflow_error; \ in_clist = 0; \ - v_size++; \ + v_size++; \ + /* If iov_len is smaller then Uint we split the binary into*/ \ + /* multiple smaller (2GB) elements in the iolist.*/ \ + v_size += _size / MAX_SYSIOVEC_IOVLEN; \ if (_size >= ERL_SMALL_IO_BIN_LIMIT) { \ p_in_clist = 0; \ p_v_size++; \ diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index f303d4f167..dfe82cab44 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -97,7 +97,7 @@ ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes)) #define ErtsContainerStruct(ptr, type, member) \ - (type *)((char *)(1 ? (ptr) : &((type *)0)->member) - offsetof(type, member)) + ((type *)((char *)(1 ? (ptr) : &((type *)0)->member) - offsetof(type, member))) #if defined (__WIN32__) # include "erl_win_sys.h" @@ -154,8 +154,9 @@ typedef ERTS_SYS_FD_TYPE ErtsSysFdType; # define ERTS_WRITE_UNLIKELY(X) X #endif +/* clang may have too low __GNUC__ versions but can handle it */ #ifdef __GNUC__ -# if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ > 5) +# if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ > 5) || defined(__clang__) # define ERTS_DECLARE_DUMMY(X) X __attribute__ ((unused)) # else # define ERTS_DECLARE_DUMMY(X) X diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index cedc88e5fe..675fafa726 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -2210,7 +2210,9 @@ do_allocate_logger_message(Eterm gleader, Eterm **hp, ErlOffHeap **ohp, #ifndef ERTS_SMP #ifdef USE_THREADS - if (erts_get_scheduler_data()) /* Must be scheduler thread */ + if (!erts_get_scheduler_data()) /* Must be scheduler thread */ + *p = NULL; + else #endif { *p = erts_whereis_process(NULL, 0, am_error_logger, 0, 0); @@ -2226,18 +2228,10 @@ do_allocate_logger_message(Eterm gleader, Eterm **hp, ErlOffHeap **ohp, } /* So we have an error logger, lets build the message */ - if (sz <= HeapWordsLeft(*p)) { - *ohp = &MSO(*p); - *hp = HEAP_TOP(*p); - HEAP_TOP(*p) += sz; - } else { -#endif - *bp = new_message_buffer(sz); - *ohp = &(*bp)->off_heap; - *hp = (*bp)->mem; -#ifndef ERTS_SMP - } #endif + *bp = new_message_buffer(sz); + *ohp = &(*bp)->off_heap; + *hp = (*bp)->mem; return (is_nil(gleader) ? am_noproc @@ -3893,8 +3887,10 @@ void bin_write(int to, void *to_arg, byte* buf, size_t sz) } /* Fill buf with the contents of bytelist list - return number of chars in list or -1 for error */ - + * return number of chars in list + * or -1 for type error + * or -2 for not enough buffer space (buffer contains truncated result) + */ Sint intlist_to_buf(Eterm list, char *buf, Sint len) { @@ -3917,7 +3913,7 @@ intlist_to_buf(Eterm list, char *buf, Sint len) return -1; listptr = list_val(*(listptr + 1)); } - return -1; /* not enough space */ + return -2; /* not enough space */ } /* diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c index e87d141ddb..06dfb2dd10 100644 --- a/erts/emulator/drivers/common/inet_drv.c +++ b/erts/emulator/drivers/common/inet_drv.c @@ -34,6 +34,7 @@ #include <stdio.h> #include <stdlib.h> +#include <stddef.h> #include <ctype.h> #include <sys/types.h> #include <errno.h> @@ -58,6 +59,9 @@ #ifdef HAVE_NETPACKET_PACKET_H #include <netpacket/packet.h> #endif +#ifdef HAVE_SYS_UN_H +#include <sys/un.h> +#endif /* All platforms fail on malloc errors. */ #define FATAL_MALLOC @@ -573,7 +577,7 @@ static int my_strncasecmp(const char *s1, const char *s2, size_t n) #ifdef HAVE_SOCKLEN_T # define SOCKLEN_T socklen_t #else -# define SOCKLEN_T int +# define SOCKLEN_T size_t #endif #include "packet_parser.h" @@ -587,6 +591,22 @@ static int my_strncasecmp(const char *s1, const char *s2, size_t n) (((unsigned char*) (s))[1] << 8) | \ (((unsigned char*) (s))[0])) +/* strnlen doesn't exist everywhere */ +static size_t my_strnlen(const char *s, size_t maxlen) +{ + size_t i = 0; + while (i < maxlen && s[i] != '\0') + i++; + return i; +} + +/* Check that some character in the buffer != '\0' */ +static int is_nonzero(const char *s, size_t n) +{ + size_t i; + for (i = 0; i < n; i++) if (s[i] != '\0') return !0; + return 0; +} #ifdef VALGRIND # include <valgrind/memcheck.h> @@ -611,6 +631,8 @@ static int my_strncasecmp(const char *s1, const char *s2, size_t n) #define INET_AF_INET6 2 #define INET_AF_ANY 3 /* INADDR_ANY or IN6ADDR_ANY_INIT */ #define INET_AF_LOOPBACK 4 /* INADDR_LOOPBACK or IN6ADDR_LOOPBACK_INIT */ +#define INET_AF_LOCAL 5 +#define INET_AF_UNDEFINED 6 /* Unknown */ /* open and INET_REQ_GETTYPE enumeration */ #define INET_TYPE_STREAM 1 @@ -896,19 +918,35 @@ typedef union { #ifdef HAVE_IN6 struct sockaddr_in6 sai6; #endif +#ifdef HAVE_SYS_UN_H + struct sockaddr_un sal; +#endif } inet_address; -/* for AF_INET & AF_INET6 */ -#define inet_address_port(x) ((x)->sai.sin_port) +#define inet_address_port(x) \ + ((((x)->sai.sin_family == AF_INET) || \ + ((x)->sai.sin_family == AF_INET6)) ? \ + ((x)->sai.sin_port) : -1) + +#ifdef HAVE_SYS_UN_H +#define localaddrlen(data) \ + ((((unsigned char*)(data))[0] == INET_AF_LOCAL) ? \ + (1 + 1 + ((unsigned char*)(data))[1]) : 1) +#else +#define localaddrlen(data) (1) +#endif #if defined(HAVE_IN6) && defined(AF_INET6) -#define addrlen(family) \ - ((family == AF_INET) ? sizeof(struct in_addr) : \ - ((family == AF_INET6) ? sizeof(struct in6_addr) : 0)) +#define addrlen(data) \ + ((((unsigned char*)(data))[0] == INET_AF_INET) ? \ + (1 + 2 + 4) : \ + ((((unsigned char*)(data))[0] == INET_AF_INET6) ? \ + (1 + 2 + 16) : localaddrlen(data))) #else -#define addrlen(family) \ - ((family == AF_INET) ? sizeof(struct in_addr) : 0) +#define addrlen(data) \ + ((((unsigned char*)(data))[0] == INET_AF_INET) ? \ + (1 + 2 + 4) : localaddrlen(data)) #endif typedef struct _multi_timer_data { @@ -999,8 +1037,10 @@ typedef struct { inet_address peer_addr; /* fake peer address */ inet_address name_addr; /* fake local address */ - inet_address* peer_ptr; /* fake peername or NULL */ - inet_address* name_ptr; /* fake sockname or NULL */ + inet_address* peer_ptr; /* fake peername or NULL */ + inet_address* name_ptr; /* fake sockname or NULL */ + SOCKLEN_T peer_addr_len; /* fake peername size */ + SOCKLEN_T name_addr_len; /* fake sockname size */ int bufsz; /* minimum buffer constraint */ unsigned int hsz; /* the list header size, -1 is large !!! */ @@ -1246,6 +1286,7 @@ static int async_ref = 0; /* async reference id generator */ } while (0) static ErlDrvTermData am_ok; +static ErlDrvTermData am_undefined; static ErlDrvTermData am_tcp; static ErlDrvTermData am_error; static ErlDrvTermData am_einval; @@ -1262,6 +1303,7 @@ static ErlDrvTermData am_ssl_tls; static ErlDrvTermData am_udp; static ErlDrvTermData am_udp_passive; static ErlDrvTermData am_udp_error; +static ErlDrvTermData am_local; #endif #ifdef HAVE_SCTP static ErlDrvTermData am_sctp; @@ -1471,40 +1513,59 @@ static void *realloc_wrapper(void *current, ErlDrvSizeT size){ #endif #ifdef HAVE_UDP -static int load_ip_port(ErlDrvTermData* spec, int i, char* buf) -{ - spec[i++] = ERL_DRV_INT; - spec[i++] = (ErlDrvTermData) get_int16(buf); - return i; -} - -static int load_ip_address(ErlDrvTermData* spec, int i, int family, char* buf) +static int load_address(ErlDrvTermData* spec, int i, char* buf) { int n; - if (family == AF_INET) { - for (n = 0; n < 4; n++) { + switch (*buf++) { /* Family */ + case INET_AF_INET: { + for (n = 2; n < 2+4; n++) { spec[i++] = ERL_DRV_INT; spec[i++] = (ErlDrvTermData) ((unsigned char)buf[n]); } spec[i++] = ERL_DRV_TUPLE; spec[i++] = 4; + spec[i++] = ERL_DRV_INT; + spec[i++] = (ErlDrvTermData) get_int16(buf); + break; } #if defined(HAVE_IN6) && defined(AF_INET6) - else if (family == AF_INET6) { - for (n = 0; n < 16; n += 2) { + case INET_AF_INET6: { + for (n = 2; n < 2+16; n += 2) { spec[i++] = ERL_DRV_INT; spec[i++] = (ErlDrvTermData) get_int16(buf+n); } spec[i++] = ERL_DRV_TUPLE; spec[i++] = 8; + spec[i++] = ERL_DRV_INT; + spec[i++] = (ErlDrvTermData) get_int16(buf); + break; } #endif - else { +#ifdef HAVE_SYS_UN_H + case INET_AF_LOCAL: { + int len = *(unsigned char*)buf++; + i = LOAD_ATOM(spec, i, am_local); + i = LOAD_BUF2BINARY(spec, i, buf, len); + spec[i++] = ERL_DRV_TUPLE; + spec[i++] = 2; + spec[i++] = ERL_DRV_INT; + spec[i++] = 0; + break; + } +#endif + default: { /* INET_AF_UNDEFINED */ + i = LOAD_ATOM(spec, i, am_undefined); + spec[i++] = ERL_DRV_INT; + spec[i++] = 0; spec[i++] = ERL_DRV_TUPLE; + spec[i++] = 2; + spec[i++] = ERL_DRV_INT; spec[i++] = 0; + break; + } } return i; -} + } #endif @@ -1512,10 +1573,13 @@ static int load_ip_address(ErlDrvTermData* spec, int i, int family, char* buf) /* For SCTP, we often need to return {IP, Port} tuples: */ static int inet_get_address(char* dst, inet_address* src, unsigned int* len); -#define LOAD_IP_AND_PORT_CNT \ +/* Max of {{int()*8},int()} | {{int()*4},int()} | + * {{'local',binary()},int()} + */ +#define LOAD_INET_GET_ADDRESS_CNT \ (8*LOAD_INT_CNT + LOAD_TUPLE_CNT + LOAD_INT_CNT + LOAD_TUPLE_CNT) -static int load_ip_and_port +static int load_inet_get_address (ErlDrvTermData* spec, int i, inet_descriptor* desc, struct sockaddr_storage* addr) { @@ -1533,8 +1597,7 @@ static int load_ip_and_port /* NB: the following functions are safe to use, as they create tuples of copied Ints on the "spec", and do not install any String pts -- a ptr to "abuf" would be dangling upon exiting this function: */ - i = load_ip_address(spec, i, desc->sfamily, abuf+3); - i = load_ip_port (spec, i, abuf+1); + i = load_address(spec, i, abuf); /* IP,Port | Family,Addr */ i = LOAD_TUPLE (spec, i, 2); return i; } @@ -2226,7 +2289,6 @@ static ErlDrvTermData am_http_error; static ErlDrvTermData am_abs_path; static ErlDrvTermData am_absoluteURI; static ErlDrvTermData am_star; -static ErlDrvTermData am_undefined; static ErlDrvTermData am_http; static ErlDrvTermData am_https; static ErlDrvTermData am_scheme; @@ -2917,7 +2979,7 @@ static int sctp_parse_async_event ASSERT(sptr->spc_length <= sz); /* No buffer overrun */ i = LOAD_ATOM (spec, i, am_sctp_paddr_change); - i = load_ip_and_port(spec, i, desc, &sptr->spc_aaddr); + i = load_inet_get_address(spec, i, desc, &sptr->spc_aaddr); switch (sptr->spc_state) { @@ -3344,10 +3406,11 @@ static int tcp_error_message(tcp_descriptor* desc, int err) #ifdef HAVE_UDP /* ** active mode message: -** {udp, S, IP, Port, [H1,...Hsz | Data]} or -** {sctp, S, IP, Port, {[AncilData], Event_or_Data}} +** {udp, S, IP, Port, [H1,...Hsz | Data]} or +** {sctp, S, IP, Port, {[AncilData], Event_or_Data}} ** where ** [H1,...,HSz] are msg headers (without IP/Port, UDP only), +** [AddrLen, H2,...,HSz] are msg headers for UDP AF_UNIX only ** Data : List() | Binary() */ static int packet_binary_message @@ -3357,6 +3420,7 @@ static int packet_binary_message ErlDrvTermData spec [PACKET_ERL_DRV_TERM_DATA_LEN]; int i = 0; int alen; + char* data = bin->orig_bytes+offs; DEBUGF(("packet_binary_message(%ld): len = %d\r\n", (long)desc->port, len)); @@ -3366,13 +3430,12 @@ static int packet_binary_message i = LOAD_ATOM(spec, i, am_udp ); /* UDP only */ # endif i = LOAD_PORT(spec, i, desc->dport); /* S */ - - alen = addrlen(desc->sfamily); - i = load_ip_address(spec, i, desc->sfamily, bin->orig_bytes+offs+3); - i = load_ip_port(spec, i, bin->orig_bytes+offs+1); /* IP, Port */ - - offs += (alen + 3); - len -= (alen + 3); + + alen = addrlen(data); + i = load_address(spec, i, data); /* IP,Port | Family,Addr */ + + offs += alen; + len -= alen; # ifdef HAVE_SCTP if (!IS_SCTP(desc)) @@ -3759,6 +3822,7 @@ static int inet_init() # endif INIT_ATOM(ok); + INIT_ATOM(undefined); INIT_ATOM(tcp); #ifdef HAVE_UDP INIT_ATOM(udp); @@ -3775,6 +3839,7 @@ static int inet_init() #ifdef HAVE_UDP INIT_ATOM(udp_passive); INIT_ATOM(udp_error); + INIT_ATOM(local); #endif INIT_ATOM(empty_out_q); INIT_ATOM(ssl_tls); @@ -3787,7 +3852,6 @@ static int inet_init() INIT_ATOM(abs_path); INIT_ATOM(absoluteURI); am_star = driver_mk_atom("*"); - INIT_ATOM(undefined); INIT_ATOM(http); INIT_ATOM(https); INIT_ATOM(scheme); @@ -3883,7 +3947,7 @@ static int inet_init() /* -** Set a inaddr structure: +** Set an inaddr structure: ** src = [P1,P0,X1,X2,.....] ** dst points to a structure large enugh to keep any kind ** of inaddr. @@ -3923,6 +3987,18 @@ static char* inet_set_address(int family, inet_address* dst, return src + 2+16; } #endif +#ifdef HAVE_SYS_UN_H + else if ((family == AF_UNIX) && (*len >= 1)) { + int n = *((unsigned char*)src); + if ((*len < 1+n) || (sizeof(dst->sal.sun_path) < n+1)) + return NULL; + sys_memzero((char*)dst, sizeof(struct sockaddr_un)); + dst->sal.sun_family = family; + sys_memcpy(dst->sal.sun_path, src+1, n); + *len = offsetof(struct sockaddr_un, sun_path) + n; + return src + 1 + n; + } +#endif return NULL; } @@ -3931,7 +4007,7 @@ static char* inet_set_address(int family, inet_address* dst, ** or from argument if source data specifies constant address. ** ** src = [TAG,P1,P0] when TAG = INET_AF_ANY | INET_AF_LOOPBACK -** src = [TAG,P1,P0,X1,X2,...] when TAG = INET_AF_INET | INET_AF_INET6 +** src = [TAG,P1,P0,X1,X2,...] when TAG = INET_AF_INET | INET_AF_INET6 | INET_AF_LOCAL */ static char *inet_set_faddress(int family, inet_address* dst, char *src, ErlDrvSizeT* len) { @@ -3949,6 +4025,12 @@ static char *inet_set_faddress(int family, inet_address* dst, family = AF_INET6; break; # endif +# ifdef HAVE_SYS_UN_H + case INET_AF_LOCAL: { + family = AF_UNIX; + break; + } +# endif case INET_AF_ANY: case INET_AF_LOOPBACK: { int port; @@ -4012,7 +4094,6 @@ static char *inet_set_faddress(int family, inet_address* dst, return inet_set_address(family, dst, src, len); } - /* Get a inaddr structure ** src = inaddr structure ** *len is the lenght of structure @@ -4023,6 +4104,7 @@ static char *inet_set_faddress(int family, inet_address* dst, */ static int inet_get_address(char* dst, inet_address* src, unsigned int* len) { + /* Compare the code with inet_address_to_erlang() */ int family; short port; @@ -4045,6 +4127,32 @@ static int inet_get_address(char* dst, inet_address* src, unsigned int* len) return 0; } #endif +#ifdef HAVE_SYS_UN_H + else if (family == AF_UNIX) { + size_t n, m; + if (*len < offsetof(struct sockaddr_un, sun_path)) return -1; + n = *len - offsetof(struct sockaddr_un, sun_path); + if (255 < n) return -1; + /* Portability fix: Assume that the address is a zero terminated + * string, except when the first byte is \0 i.e the + * string length is 0. Then use the reported length instead. + * This fix handles Linux's abstract socket address + * nonportable extension. + */ + m = my_strnlen(src->sal.sun_path, n); + if ((m == 0) && is_nonzero(src->sal.sun_path, n)) + m = n; + dst[0] = INET_AF_LOCAL; + dst[1] = (char) ((unsigned char) m); + sys_memcpy(dst+2, src->sal.sun_path, m); + *len = 1 + 1 + m; + return 0; + } +#endif + else { + dst[0] = INET_AF_UNDEFINED; + *len = 1; + } return -1; } @@ -4053,7 +4161,9 @@ static int inet_get_address(char* dst, inet_address* src, unsigned int* len) ** according to the size of the current, ** and return the resulting encoded size */ -static int inet_address_to_erlang(char *dst, inet_address **src) { +static int +inet_address_to_erlang(char *dst, inet_address **src, SOCKLEN_T sz) { + /* Compare the code with inet_get_address() */ short port; switch ((*src)->sa.sa_family) { @@ -4078,6 +4188,30 @@ static int inet_address_to_erlang(char *dst, inet_address **src) { (*src) = (inet_address *) (&(*src)->sai6 + 1); return 1 + 2 + 16; #endif +#ifdef HAVE_SYS_UN_H + case AF_UNIX: { + size_t n, m; + if (sz < offsetof(struct sockaddr_un, sun_path)) return -1; + n = sz - offsetof(struct sockaddr_un, sun_path); + if (255 < n) return -1; + /* Portability fix: Assume that the address is a zero terminated + * string, except when the first byte is \0 i.e the + * string length is 0. Then use the reported length instead. + * This fix handles Linux's abstract socket address + * nonportable extension. + */ + m = my_strnlen((*src)->sal.sun_path, n); + if ((m == 0) && is_nonzero((*src)->sal.sun_path, n)) + m = n; + if (dst) { + dst[0] = INET_AF_LOCAL; + dst[1] = (char) ((unsigned char) m); + sys_memcpy(dst+2, (*src)->sal.sun_path, m); + } + (*src) = (inet_address *) (&(*src)->sal + 1); + return 1 + 1 + m; + } +#endif default: return -1; } @@ -4086,7 +4220,7 @@ static int inet_address_to_erlang(char *dst, inet_address **src) { /* Encode n encoded addresses from addrs in the result buffer */ static ErlDrvSizeT reply_inet_addrs -(int n, inet_address *addrs, char **rbuf, ErlDrvSizeT rsize) { +(int n, inet_address *addrs, char **rbuf, ErlDrvSizeT rsize, SOCKLEN_T sz) { inet_address *ia; int i, s; ErlDrvSizeT rlen; @@ -4094,11 +4228,19 @@ static ErlDrvSizeT reply_inet_addrs if (IS_SOCKET_ERROR(n)) return ctl_error(sock_errno(), rbuf, rsize); if (n == 0) return ctl_reply(INET_REP_OK, NULL, 0, rbuf, rsize); + /* The sz argument is only used when we have got an actual size + * of addrs[0] from e.g getsockname() and then n == 1 + * so we will loop over 1 element below. Otherwise sz + * would be expected to differ between addresses but that + * can only happen for AF_UNIX and we will only be called with + * n > 1 for SCTP and that will never (?) happen with AF_UNIX + */ + /* Calculate result length */ rlen = 1; ia = addrs; for (i = 0; i < n; i++) { - s = inet_address_to_erlang(NULL, &ia); + s = inet_address_to_erlang(NULL, &ia, sz); if (s < 0) break; rlen += s; } @@ -4109,7 +4251,7 @@ static ErlDrvSizeT reply_inet_addrs rlen = 1; ia = addrs; for (i = 0; i < n; i++) { - s = inet_address_to_erlang((*rbuf)+rlen, &ia); + s = inet_address_to_erlang((*rbuf)+rlen, &ia, sz); if (s < 0) break; rlen += s; } @@ -4180,6 +4322,7 @@ static ErlDrvSSizeT inet_ctl_open(inet_descriptor* desc, int domain, int type, char** rbuf, ErlDrvSizeT rsize) { int save_errno; + int protocol; #ifdef HAVE_SETNS int current_ns, new_ns; current_ns = new_ns = 0; @@ -4218,7 +4361,11 @@ static ErlDrvSSizeT inet_ctl_open(inet_descriptor* desc, int domain, int type, } } #endif - if ((desc->s = sock_open(domain, type, desc->sprotocol)) == INVALID_SOCKET) + protocol = desc->sprotocol; +#ifdef HAVE_SYS_UN_H + if (domain == AF_UNIX) protocol = 0; +#endif + if ((desc->s = sock_open(domain, type, protocol)) == INVALID_SOCKET) save_errno = sock_errno(); #ifdef HAVE_SETNS if (desc->netns != NULL) { @@ -7119,14 +7266,14 @@ static ErlDrvSSizeT inet_fill_opts(inet_descriptor* desc, #ifdef HAVE_SCTP #define LOAD_PADDRINFO_CNT \ - (2*LOAD_ATOM_CNT + LOAD_ASSOC_ID_CNT + LOAD_IP_AND_PORT_CNT + \ + (2*LOAD_ATOM_CNT + LOAD_ASSOC_ID_CNT + LOAD_INET_GET_ADDRESS_CNT + \ 4*LOAD_INT_CNT + LOAD_TUPLE_CNT) static int load_paddrinfo (ErlDrvTermData * spec, int i, inet_descriptor* desc, struct sctp_paddrinfo* pai) { i = LOAD_ATOM (spec, i, am_sctp_paddrinfo); i = LOAD_ASSOC_ID (spec, i, pai->spinfo_assoc_id); - i = load_ip_and_port(spec, i, desc, &pai->spinfo_address); + i = load_inet_get_address(spec, i, desc, &pai->spinfo_address); switch(pai->spinfo_state) { case SCTP_ACTIVE: @@ -7542,7 +7689,7 @@ static ErlDrvSSizeT sctp_fill_opts(inet_descriptor* desc, /* Fill in the response: */ PLACE_FOR(spec, i, 2*LOAD_ATOM_CNT + LOAD_ASSOC_ID_CNT + - LOAD_IP_AND_PORT_CNT + 2*LOAD_TUPLE_CNT); + LOAD_INET_GET_ADDRESS_CNT + 2*LOAD_TUPLE_CNT); switch (eopt) { case SCTP_OPT_PRIMARY_ADDR: i = LOAD_ATOM(spec, i, am_sctp_primary_addr); @@ -7556,7 +7703,7 @@ static ErlDrvSSizeT sctp_fill_opts(inet_descriptor* desc, ASSERT(0); } i = LOAD_ASSOC_ID (spec, i, sp.sspp_assoc_id); - i = load_ip_and_port(spec, i, desc, &sp.sspp_addr); + i = load_inet_get_address(spec, i, desc, &sp.sspp_addr); i = LOAD_TUPLE (spec, i, 3); i = LOAD_TUPLE (spec, i, 2); break; @@ -7603,11 +7750,11 @@ static ErlDrvSSizeT sctp_fill_opts(inet_descriptor* desc, /* Fill in the response: */ PLACE_FOR(spec, i, 2*LOAD_ATOM_CNT + LOAD_ASSOC_ID_CNT + - LOAD_IP_AND_PORT_CNT + 4*LOAD_INT_CNT); + LOAD_INET_GET_ADDRESS_CNT + 4*LOAD_INT_CNT); i = LOAD_ATOM (spec, i, am_sctp_peer_addr_params); i = LOAD_ATOM (spec, i, am_sctp_paddrparams); i = LOAD_ASSOC_ID (spec, i, ap.spp_assoc_id); - i = load_ip_and_port(spec, i, desc, &ap.spp_address); + i = load_inet_get_address(spec, i, desc, &ap.spp_address); i = LOAD_INT (spec, i, ap.spp_hbinterval); i = LOAD_INT (spec, i, ap.spp_pathmaxrxt); @@ -8248,6 +8395,11 @@ static ErlDrvSSizeT inet_ctl(inet_descriptor* desc, int cmd, char* buf, put_int32(INET_AF_INET6, &tbuf[0]); } #endif +#ifdef HAVE_SYS_UN_H + else if (desc->sfamily == AF_UNIX) { + put_int32(INET_AF_LOCAL, &tbuf[0]); + } +#endif else return ctl_error(EINVAL, rbuf, rsize); @@ -8307,19 +8459,19 @@ static ErlDrvSSizeT inet_ctl(inet_descriptor* desc, int cmd, char* buf, assoc_id = get_int32(buf); n = p_sctp_getpaddrs(desc->s, assoc_id, &sa); - rlen = reply_inet_addrs(n, (inet_address *) sa, rbuf, rsize); + rlen = reply_inet_addrs(n, (inet_address *) sa, rbuf, rsize, 0); if (n > 0) p_sctp_freepaddrs(sa); return rlen; } #endif { /* Fallback to sock_peer */ inet_address addr; - unsigned int sz; + SOCKLEN_T sz; int i; sz = sizeof(addr); i = sock_peer(desc->s, (struct sockaddr *) &addr, &sz); - return reply_inet_addrs(i >= 0 ? 1 : i, &addr, rbuf, rsize); + return reply_inet_addrs(i >= 0 ? 1 : i, &addr, rbuf, rsize, sz); } } @@ -8327,15 +8479,21 @@ static ErlDrvSSizeT inet_ctl(inet_descriptor* desc, int cmd, char* buf, char tbuf[sizeof(inet_address)]; inet_address peer; inet_address* ptr; - unsigned int sz = sizeof(peer); + unsigned int sz; DEBUGF(("inet_ctl(%ld): PEER\r\n", (long)desc->port)); if (!(desc->state & INET_F_ACTIVE)) return ctl_error(ENOTCONN, rbuf, rsize); - if ((ptr = desc->peer_ptr) == NULL) { + if ((ptr = desc->peer_ptr) != NULL) { + sz = desc->peer_addr_len; + } + else { ptr = &peer; - if (IS_SOCKET_ERROR(sock_peer(desc->s, (struct sockaddr*)ptr,&sz))) + sz = sizeof(peer); + if (IS_SOCKET_ERROR + (sock_peer + (desc->s, (struct sockaddr*)ptr, &sz))) return ctl_error(sock_errno(), rbuf, rsize); } if (inet_get_address(tbuf, ptr, &sz) < 0) @@ -8350,11 +8508,12 @@ static ErlDrvSSizeT inet_ctl(inet_descriptor* desc, int cmd, char* buf, } else if (len < 2) return ctl_error(EINVAL, rbuf, rsize); - else if (inet_set_address(desc->sfamily, &desc->peer_addr, - buf, &len) == NULL) + else if (inet_set_faddress + (desc->sfamily, &desc->peer_addr, buf, &len) == NULL) return ctl_error(EINVAL, rbuf, rsize); else { desc->peer_ptr = &desc->peer_addr; + desc->peer_addr_len = (SOCKLEN_T) len; return ctl_reply(INET_REP_OK, NULL, 0, rbuf, rsize); } } @@ -8376,19 +8535,19 @@ static ErlDrvSSizeT inet_ctl(inet_descriptor* desc, int cmd, char* buf, assoc_id = get_int32(buf); n = p_sctp_getladdrs(desc->s, assoc_id, &sa); - rlen = reply_inet_addrs(n, (inet_address *) sa, rbuf, rsize); + rlen = reply_inet_addrs(n, (inet_address *) sa, rbuf, rsize, 0); if (n > 0) p_sctp_freeladdrs(sa); return rlen; } #endif { /* Fallback to sock_name */ inet_address addr; - unsigned int sz; + SOCKLEN_T sz; int i; sz = sizeof(addr); i = sock_name(desc->s, (struct sockaddr *) &addr, &sz); - return reply_inet_addrs(i >= 0 ? 1 : i, &addr, rbuf, rsize); + return reply_inet_addrs(i >= 0 ? 1 : i, &addr, rbuf, rsize, sz); } } @@ -8396,16 +8555,21 @@ static ErlDrvSSizeT inet_ctl(inet_descriptor* desc, int cmd, char* buf, char tbuf[sizeof(inet_address)]; inet_address name; inet_address* ptr; - unsigned int sz = sizeof(name); + unsigned int sz; DEBUGF(("inet_ctl(%ld): NAME\r\n", (long)desc->port)); if (!IS_BOUND(desc)) return ctl_error(EINVAL, rbuf, rsize); /* address is not valid */ - if ((ptr = desc->name_ptr) == NULL) { + if ((ptr = desc->name_ptr) != NULL) { + sz = desc->name_addr_len; + } + else { ptr = &name; - if (IS_SOCKET_ERROR(sock_name(desc->s, (struct sockaddr*)ptr, &sz))) + sz = sizeof(name); + if (IS_SOCKET_ERROR + (sock_name(desc->s, (struct sockaddr*)ptr, &sz))) return ctl_error(sock_errno(), rbuf, rsize); } if (inet_get_address(tbuf, ptr, &sz) < 0) @@ -8413,18 +8577,19 @@ static ErlDrvSSizeT inet_ctl(inet_descriptor* desc, int cmd, char* buf, return ctl_reply(INET_REP_OK, tbuf, sz, rbuf, rsize); } - case INET_REQ_SETNAME: { /* set fake peername Port Address */ + case INET_REQ_SETNAME: { /* set fake sockname Port Address */ if (len == 0) { desc->name_ptr = NULL; return ctl_reply(INET_REP_OK, NULL, 0, rbuf, rsize); } else if (len < 2) return ctl_error(EINVAL, rbuf, rsize); - else if (inet_set_address(desc->sfamily, &desc->name_addr, - buf, &len) == NULL) + else if (inet_set_faddress + (desc->sfamily, &desc->name_addr, buf, &len) == NULL) return ctl_error(EINVAL, rbuf, rsize); else { desc->name_ptr = &desc->name_addr; + desc->name_addr_len = (SOCKLEN_T) len; return ctl_reply(INET_REP_OK, NULL, 0, rbuf, rsize); } } @@ -8432,7 +8597,7 @@ static ErlDrvSSizeT inet_ctl(inet_descriptor* desc, int cmd, char* buf, case INET_REQ_BIND: { /* bind socket */ char tbuf[2]; inet_address local; - short port; + int port; DEBUGF(("inet_ctl(%ld): BIND\r\n", (long)desc->port)); @@ -8449,13 +8614,14 @@ static ErlDrvSSizeT inet_ctl(inet_descriptor* desc, int cmd, char* buf, desc->state = INET_STATE_BOUND; - if ((port = inet_address_port(&local)) == 0) { + port = inet_address_port(&local); + if (port == 0) { SOCKLEN_T adrlen = sizeof(local); sock_name(desc->s, &local.sa, &adrlen); port = inet_address_port(&local); } - port = sock_ntohs(port); - put_int16(port, tbuf); + else if (port == -1) port = 0; + put_int16(sock_ntohs((Uint16) port), tbuf); return ctl_reply(INET_REP_OK, tbuf, 2, rbuf, rsize); } @@ -8912,6 +9078,11 @@ static ErlDrvSSizeT tcp_inet_ctl(ErlDrvData e, unsigned int cmd, return ctl_xerror("eafnosupport", rbuf, rsize); break; #endif +#ifdef HAVE_SYS_UN_H + case INET_AF_LOCAL: + domain = AF_UNIX; + break; +#endif default: return ctl_error(EINVAL, rbuf, rsize); } @@ -8938,6 +9109,11 @@ static ErlDrvSSizeT tcp_inet_ctl(ErlDrvData e, unsigned int cmd, return ctl_xerror("eafnosupport", rbuf, rsize); break; #endif +#ifdef HAVE_SYS_UN_H + case INET_AF_LOCAL: + domain = AF_UNIX; + break; +#endif default: return ctl_error(EINVAL, rbuf, rsize); } @@ -8993,8 +9169,8 @@ static ErlDrvSSizeT tcp_inet_ctl(ErlDrvData e, unsigned int cmd, timeout = get_int32(buf); buf += 4; len -= 4; - if (inet_set_address(desc->inet.sfamily, &desc->inet.remote, - buf, &len) == NULL) + if (inet_set_faddress + (desc->inet.sfamily, &desc->inet.remote, buf, &len) == NULL) return ctl_error(EINVAL, rbuf, rsize); code = sock_connect(desc->inet.s, @@ -10777,6 +10953,9 @@ static ErlDrvSSizeT packet_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf, return ctl_xerror("eafnosupport", rbuf, rsize); break; #endif +#ifdef HAVE_SYS_UN_H + case INET_AF_LOCAL: af = AF_UNIX; break; +#endif default: return ctl_error(EINVAL, rbuf, rsize); } @@ -10827,6 +11006,9 @@ static ErlDrvSSizeT packet_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf, return ctl_xerror("eafnosupport", rbuf, rsize); break; #endif +#ifdef HAVE_SYS_UN_H + case INET_AF_LOCAL: af = AF_UNIX; break; +#endif default: return ctl_error(EINVAL, rbuf, rsize); } @@ -10909,7 +11091,7 @@ static ErlDrvSSizeT packet_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf, /* For SCTP, we do not set the peer's addr in desc->remote, as multiple peers are possible: */ - if (inet_set_address(desc->sfamily, &remote, buf, &len) == NULL) + if (inet_set_faddress(desc->sfamily, &remote, buf, &len) == NULL) return ctl_error(EINVAL, rbuf, rsize); sock_select(desc, FD_CONNECT, 1); @@ -10949,8 +11131,8 @@ static ErlDrvSSizeT packet_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf, /* Ignore timeout */ buf += 4; len -= 4; - if (inet_set_address(desc->sfamily, - &desc->remote, buf, &len) == NULL) + if (inet_set_faddress + (desc->sfamily, &desc->remote, buf, &len) == NULL) return ctl_error(EINVAL, rbuf, rsize); code = sock_connect(desc->s, @@ -11130,12 +11312,12 @@ static void packet_inet_timeout(ErlDrvData e) /* THIS IS A "send*" REQUEST; on the Erlang side: "port_command". -** input should be: P1 P0 Address buffer . +** input should be: Family Address buffer . ** For UDP, buffer (after Address) is just data to be sent. ** For SCTP, buffer contains a list representing 2 items: ** (1) 6 parms for sctp_sndrcvinfo, as in sctp_get_sendparams(); ** (2) 0+ real data bytes. -** There is no destination address -- SCTYP send is performed over +** There is no destination address -- SCTP send is performed over ** an existing association, using "sctp_sndrcvinfo" specified. */ static void packet_inet_command(ErlDrvData e, char* buf, ErlDrvSizeT len) @@ -11210,7 +11392,7 @@ static void packet_inet_command(ErlDrvData e, char* buf, ErlDrvSizeT len) /* UDP socket. Even if it is connected, there is an address prefix here -- ignored for connected sockets: */ sz = len; - qtr = inet_set_address(desc->sfamily, &other, ptr, &sz); + qtr = inet_set_faddress(desc->sfamily, &other, ptr, &sz); if (qtr == NULL) { inet_reply_error(desc, EINVAL); return; @@ -11809,9 +11991,8 @@ int erts_sock_connect(erts_sock_t socket, byte *ip_addr, int len, Uint16 port) if (!inet_set_address(AF_INET, &addr, buf, &blen)) return 0; - if (IS_SOCKET_ERROR(sock_connect(s, - (struct sockaddr *) &addr, - sizeof(struct sockaddr_in)))) + if (IS_SOCKET_ERROR + (sock_connect(s, (struct sockaddr *) &addr, blen))) return 0; return 1; } diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c index 4063cbf306..3336fded7a 100644 --- a/erts/emulator/hipe/hipe_bif0.c +++ b/erts/emulator/hipe/hipe_bif0.c @@ -418,6 +418,8 @@ BIF_RETTYPE hipe_bifs_enter_code_2(BIF_ALIST_2) BIF_RET(make_tuple(hp)); } +#define IS_POWER_OF_TWO(Val) (((Val) > 0) && (((Val) & ((Val)-1)) == 0)) + /* * Allocate memory for arbitrary non-Erlang data. */ @@ -427,16 +429,18 @@ BIF_RETTYPE hipe_bifs_alloc_data_2(BIF_ALIST_2) void *block; if (is_not_small(BIF_ARG_1) || is_not_small(BIF_ARG_2) || - (align = unsigned_val(BIF_ARG_1), - align != sizeof(long) && align != sizeof(double))) + (align = unsigned_val(BIF_ARG_1), !IS_POWER_OF_TWO(align))) BIF_ERROR(BIF_P, BADARG); nrbytes = unsigned_val(BIF_ARG_2); if (nrbytes == 0) BIF_RET(make_small(0)); block = erts_alloc(ERTS_ALC_T_HIPE, nrbytes); - if ((unsigned long)block & (align-1)) + if ((unsigned long)block & (align-1)) { fprintf(stderr, "%s: erts_alloc(%lu) returned %p which is not %lu-byte aligned\r\n", __FUNCTION__, (unsigned long)nrbytes, block, (unsigned long)align); + erts_free(ERTS_ALC_T_HIPE, block); + BIF_ERROR(BIF_P, EXC_NOTSUP); + } BIF_RET(address_to_term(block, BIF_P)); } @@ -693,7 +697,7 @@ static struct nbif nbifs[BIF_SIZE] = { #undef BIF_LIST }; -#define NBIF_HASH(m,f,a) ((m)*(f)+(a)) +#define NBIF_HASH(m,f,a) (atom_val(m) ^ atom_val(f) ^ (a)) static Hash nbif_table; static HashValue nbif_hash(struct nbif *x) @@ -1059,7 +1063,7 @@ static inline void hipe_mfa_info_table_rwunlock(void) erts_smp_rwmtx_rwunlock(&hipe_mfa_info_table.lock); } -#define HIPE_MFA_HASH(M,F,A) ((M) * (F) + (A)) +#define HIPE_MFA_HASH(M,F,A) (atom_val(M) ^ atom_val(F) ^ (A)) static struct hipe_mfa_info **hipe_mfa_info_table_alloc_bucket(unsigned int size) { @@ -1140,10 +1144,13 @@ static inline struct hipe_mfa_info *hipe_mfa_info_table_get_locked(Eterm m, Eter h = HIPE_MFA_HASH(m, f, arity); i = h & hipe_mfa_info_table.mask; p = hipe_mfa_info_table.bucket[i]; - for (; p; p = p->bucket.next) - /* XXX: do we want to compare p->bucket.hvalue as well? */ - if (p->m == m && p->f == f && p->a == arity) - return p; + for (; p; p = p->bucket.next) { + if (p->bucket.hvalue == h) { + if (p->m == m && p->f == f && p->a == arity) + return p; + } + else ASSERT(!(p->m == m && p->f == f && p->a == arity)); + } return NULL; } @@ -1167,10 +1174,13 @@ static struct hipe_mfa_info *hipe_mfa_info_table_put_rwlocked(Eterm m, Eterm f, h = HIPE_MFA_HASH(m, f, arity); i = h & hipe_mfa_info_table.mask; p = hipe_mfa_info_table.bucket[i]; - for (; p; p = p->bucket.next) - /* XXX: do we want to compare p->bucket.hvalue as well? */ - if (p->m == m && p->f == f && p->a == arity) - return p; + for (; p; p = p->bucket.next) { + if (p->bucket.hvalue == h) { + if (p->m == m && p->f == f && p->a == arity) + return p; + } + else ASSERT(!(p->m == m && p->f == f && p->a == arity)); + } p = hipe_mfa_info_table_alloc(m, f, arity); p->bucket.hvalue = h; p->bucket.next = hipe_mfa_info_table.bucket[i]; diff --git a/erts/emulator/hipe/hipe_mode_switch.c b/erts/emulator/hipe/hipe_mode_switch.c index 884331e969..ed95045292 100644 --- a/erts/emulator/hipe/hipe_mode_switch.c +++ b/erts/emulator/hipe/hipe_mode_switch.c @@ -547,7 +547,7 @@ Process *hipe_mode_switch(Process *p, unsigned cmd, Eterm reg[]) p->flags &= ~F_HIPE_MODE; ERTS_SMP_UNREQ_PROC_MAIN_LOCK(p); - p = schedule(p, reds_in - p->fcalls); + p = erts_schedule(NULL, p, reds_in - p->fcalls); ERTS_SMP_REQ_PROC_MAIN_LOCK(p); ASSERT(!(p->flags & F_HIPE_MODE)); #ifdef ERTS_SMP diff --git a/erts/emulator/hipe/hipe_x86.c b/erts/emulator/hipe/hipe_x86.c index 3d25646231..5f6c8c200e 100644 --- a/erts/emulator/hipe/hipe_x86.c +++ b/erts/emulator/hipe/hipe_x86.c @@ -37,7 +37,7 @@ void hipe_patch_load_fe(Uint32 *address, Uint32 value) { /* address points to a disp32 or imm32 operand */ - *address = value; + *address += value; } int hipe_patch_insn(void *address, Uint32 value, Eterm type) @@ -54,7 +54,7 @@ int hipe_patch_insn(void *address, Uint32 value, Eterm type) default: return -1; } - *(Uint32*)address = value; + *(Uint32*)address += value; return 0; } diff --git a/erts/emulator/nifs/common/erl_tracer_nif.c b/erts/emulator/nifs/common/erl_tracer_nif.c index 6dddc80607..c0cc48ff42 100644 --- a/erts/emulator/nifs/common/erl_tracer_nif.c +++ b/erts/emulator/nifs/common/erl_tracer_nif.c @@ -45,7 +45,7 @@ static ERL_NIF_TERM trace(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); static ErlNifFunc nif_funcs[] = { {"enabled", 3, enabled}, - {"trace", 6, trace} + {"trace", 5, trace} }; @@ -57,6 +57,7 @@ ERL_NIF_INIT(erl_tracer, nif_funcs, load, NULL, upgrade, unload) ATOM_DECL(cpu_timestamp); \ ATOM_DECL(discard); \ ATOM_DECL(exception_from); \ + ATOM_DECL(extra); \ ATOM_DECL(match_spec_result); \ ATOM_DECL(monotonic); \ ATOM_DECL(ok); \ @@ -76,8 +77,7 @@ ERL_NIF_INIT(erl_tracer, nif_funcs, load, NULL, upgrade, unload) ATOM_DECL(gc_minor_start); \ ATOM_DECL(gc_minor_end); \ ATOM_DECL(gc_major_start); \ - ATOM_DECL(gc_major_end); \ - ATOM_DECL(undefined); + ATOM_DECL(gc_major_end); #define ATOM_DECL(A) static ERL_NIF_TERM atom_##A ATOMS @@ -154,11 +154,6 @@ static ERL_NIF_TERM enabled(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) Tracee :: pid() || port() || undefined, Msg :: term(), Opts :: map()) -> ignored(). - -spec trace(Tag :: atom(), TracerState :: pid() | port(), - Tracee :: pid() || port() || undefined, - Msg :: term(), - Extra :: term(), - Opts :: map()) -> ignored(). */ static ERL_NIF_TERM trace(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { @@ -167,7 +162,8 @@ static ERL_NIF_TERM trace(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) ErlNifPort to_port; size_t tt_sz = 0; int is_port = 0; - ASSERT(argc == 6); + size_t opts_sz = 0; + ASSERT(argc == 5); if (!enif_get_local_pid(env, argv[1], &to_pid)) { if (!enif_get_local_port(env, argv[1], &to_port)) { @@ -179,12 +175,17 @@ static ERL_NIF_TERM trace(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) is_port = 1; } - if (!enif_is_identical(argv[4], atom_undefined)) { + opts = argv[4]; + + if (!enif_get_map_size(env, opts, &opts_sz)) + opts_sz = 0; + + if (opts_sz && enif_get_map_value(env, opts, atom_extra, &value)) { tt[tt_sz++] = atom_trace; tt[tt_sz++] = argv[2]; tt[tt_sz++] = argv[0]; tt[tt_sz++] = argv[3]; - tt[tt_sz++] = argv[4]; + tt[tt_sz++] = value; } else { if (enif_is_identical(argv[0], atom_seq_trace)) { tt[tt_sz++] = atom_seq_trace; @@ -198,21 +199,16 @@ static ERL_NIF_TERM trace(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) } } - opts = argv[5]; - if (enif_get_map_value(env, opts, atom_match_spec_result, - &value) - && !enif_is_identical(value, atom_true)) { + if (opts_sz && enif_get_map_value(env, opts, atom_match_spec_result, &value)) { tt[tt_sz++] = value; } - if (enif_get_map_value(env, opts, atom_scheduler_id, &value) - && !enif_is_identical(value, atom_undefined)) { + if (opts_sz && enif_get_map_value(env, opts, atom_scheduler_id, &value)) { tt[tt_sz++] = value; } - if (enif_get_map_value(env, opts, atom_timestamp, &value) - && !enif_is_identical(value, atom_undefined)) { + if (opts_sz && enif_get_map_value(env, opts, atom_timestamp, &value)) { ERL_NIF_TERM ts; if (enif_is_identical(value, atom_monotonic)) { ErlNifTime mon = enif_monotonic_time(ERL_NIF_NSEC); diff --git a/erts/emulator/sys/win32/erl_poll.c b/erts/emulator/sys/win32/erl_poll.c index 94f3840b5f..f23c7ab03d 100644 --- a/erts/emulator/sys/win32/erl_poll.c +++ b/erts/emulator/sys/win32/erl_poll.c @@ -424,7 +424,7 @@ static ERTS_INLINE int wakeup_cause(ErtsPollSet ps) { int res; - erts_aint32_t wakeup_state = erts_atomic32_read_nob(&ps->wakeup_state); + erts_aint32_t wakeup_state = erts_atomic32_read_acqb(&ps->wakeup_state); switch (wakeup_state) { case ERTS_POLL_WOKEN_IO_READY: res = 0; @@ -487,9 +487,8 @@ wake_poller(ErtsPollSet ps, int io_ready) { erts_aint32_t wakeup_state; if (io_ready) { - /* We may set the event multiple times. This is, however, harmless. */ - wakeup_state = erts_atomic32_read_nob(&ps->wakeup_state); - erts_atomic32_set_relb(&ps->wakeup_state, ERTS_POLL_WOKEN_IO_READY); + wakeup_state = erts_atomic32_xchg_relb(&ps->wakeup_state, + ERTS_POLL_WOKEN_IO_READY); } else { ERTS_THR_MEMORY_BARRIER; diff --git a/erts/emulator/test/bif_SUITE.erl b/erts/emulator/test/bif_SUITE.erl index 26bb416bf0..d31399e4af 100644 --- a/erts/emulator/test/bif_SUITE.erl +++ b/erts/emulator/test/bif_SUITE.erl @@ -141,9 +141,11 @@ guard_bifs_in_erl_bif_types(_Config) -> shadow_comments(_Config) -> ensure_erl_bif_types_compiled(), + ErlangList = [{erlang,F,A} || {F,A} <- erlang:module_info(exports), + not is_operator(F,A)], List0 = erlang:system_info(snifs), - List1 = [MFA || {M,_,_}=MFA <- List0, M =/= hipe_bifs], - List = [MFA || MFA <- List1, not is_operator(MFA)], + List1 = [MFA || {M,_,_}=MFA <- List0, M =/= hipe_bifs, M =/= erlang], + List = List1 ++ ErlangList, HasTypes = [MFA || {M,F,A}=MFA <- List, erl_bif_types:is_known(M, F, A)], Path = get_code_path(), @@ -253,12 +255,15 @@ specs(_) -> end. is_operator({erlang,F,A}) -> + is_operator(F,A); +is_operator(_) -> false. + +is_operator(F,A) -> erl_internal:arith_op(F, A) orelse erl_internal:bool_op(F, A) orelse erl_internal:comp_op(F, A) orelse erl_internal:list_op(F, A) orelse - erl_internal:send_op(F, A); -is_operator(_) -> false. + erl_internal:send_op(F, A). extract_specs(M, Abstr) -> [{make_mfa(M, Name),Spec} || {attribute,_,spec,{Name,Spec}} <- Abstr]. @@ -646,6 +651,8 @@ erlang_halt(Config) when is_list(Config) -> {badrpc,nodedown} = rpc:call(N2, erlang, halt, [0]), {ok,N3} = slave:start(H, halt_node3), {badrpc,nodedown} = rpc:call(N3, erlang, halt, [0,[]]), + {ok,N4} = slave:start(H, halt_node4), + {badrpc,nodedown} = rpc:call(N4, erlang, halt, [lists:duplicate(300,$x)]), % This test triggers a segfault when dumping a crash dump % to make sure that we can handle it properly. diff --git a/erts/emulator/test/bs_construct_SUITE.erl b/erts/emulator/test/bs_construct_SUITE.erl index 941cb435f7..22a1c0b765 100644 --- a/erts/emulator/test/bs_construct_SUITE.erl +++ b/erts/emulator/test/bs_construct_SUITE.erl @@ -527,7 +527,7 @@ huge_float_check({'EXIT',{system_limit,_}}) -> ok; huge_float_check({'EXIT',{badarg,_}}) -> ok. huge_binary(Config) when is_list(Config) -> - ct:timetrap({seconds, 30}), + ct:timetrap({seconds, 60}), 16777216 = size(<<0:(id(1 bsl 26)),(-1):(id(1 bsl 26))>>), garbage_collect(), {Shift,Return} = case free_mem() of @@ -561,30 +561,13 @@ huge_binary(Config) when is_list(Config) -> end. free_mem() -> - Cmd = "uname; free", - Output = string:tokens(os:cmd(Cmd), "\n"), - io:format("Output from command ~p\n~p\n",[Cmd,Output]), - case Output of - [OS, ColumnNames, Values | _] -> - case string:str(OS,"Linux") of - 0 -> - io:format("Unknown OS\n",[]), - undefined; - _ -> - case {string:tokens(ColumnNames, " \t"), - string:tokens(Values, " \t")} of - {[_,_,"free"|_],["Mem:",_,_,FreeKb|_]} -> - list_to_integer(FreeKb) div 1024; - _ -> - io:format("Failed to parse output from 'free':\n",[]), - undefined - end - end; - _ -> - io:format("Too few lines in output\n",[]), - undefined + {ok,Apps} = application:ensure_all_started(os_mon), + Mem = memsup:get_system_memory_data(), + [ok = application:stop(App)||App <- Apps], + case proplists:get_value(free_memory,Mem) of + undefined -> undefined; + Val -> Val div 1024 end. - system_limit(Config) when is_list(Config) -> WordSize = erlang:system_info(wordsize), @@ -614,8 +597,7 @@ system_limit_32() -> {'EXIT',{system_limit,_}} = (catch <<42:536870912/unit:8>>), {'EXIT',{system_limit,_}} = (catch <<42:(id(536870912))/unit:8>>), {'EXIT',{system_limit,_}} = (catch <<0:(id(8)),42:536870912/unit:8>>), - {'EXIT',{system_limit,_}} = - (catch <<0:(id(8)),42:(id(536870912))/unit:8>>), + {'EXIT',{system_limit,_}} = (catch <<0:(id(8)),42:(id(536870912))/unit:8>>), %% The size would be silently truncated, resulting in a crash. {'EXIT',{system_limit,_}} = (catch <<0:(1 bsl 35)>>), @@ -627,16 +609,10 @@ system_limit_32() -> ok. badarg(Config) when is_list(Config) -> - {'EXIT',{badarg,_}} = - (catch <<0:(id(1 bsl 100)),0:(id(-1))>>), - {'EXIT',{badarg,_}} = - (catch <<0:(id(1 bsl 100)),0:(id(-(1 bsl 70)))>>), - {'EXIT',{badarg,_}} = - (catch <<0:(id(-(1 bsl 70))),0:(id(1 bsl 100))>>), - - {'EXIT',{badarg,_}} = - (catch <<(id(<<>>))/binary,0:(id(-(1 bsl 100)))>>), - + {'EXIT',{badarg,_}} = (catch <<0:(id(1 bsl 100)),0:(id(-1))>>), + {'EXIT',{badarg,_}} = (catch <<0:(id(1 bsl 100)),0:(id(-(1 bsl 70)))>>), + {'EXIT',{badarg,_}} = (catch <<0:(id(-(1 bsl 70))),0:(id(1 bsl 100))>>), + {'EXIT',{badarg,_}} = (catch <<(id(<<>>))/binary,0:(id(-(1 bsl 100)))>>), ok. copy_writable_binary(Config) when is_list(Config) -> diff --git a/erts/emulator/test/code_SUITE.erl b/erts/emulator/test/code_SUITE.erl index 29b95ef674..2347a3d4ef 100644 --- a/erts/emulator/test/code_SUITE.erl +++ b/erts/emulator/test/code_SUITE.erl @@ -26,7 +26,7 @@ external_fun/1,get_chunk/1,module_md5/1,make_stub/1, make_stub_many_funs/1,constant_pools/1,constant_refc_binaries/1, false_dependency/1,coverage/1,fun_confusion/1, - t_copy_literals/1]). + t_copy_literals/1, t_copy_literals_frags/1]). -define(line_trace, 1). -include_lib("common_test/include/ct.hrl"). @@ -38,7 +38,7 @@ all() -> t_check_process_code_ets, t_check_old_code, external_fun, get_chunk, module_md5, make_stub, make_stub_many_funs, constant_pools, constant_refc_binaries, false_dependency, - coverage, fun_confusion, t_copy_literals]. + coverage, fun_confusion, t_copy_literals, t_copy_literals_frags]. init_per_suite(Config) -> erts_debug:set_internal_state(available_internal_state, true), @@ -766,6 +766,134 @@ t_copy_literals(Config) when is_list(Config) -> ok = flush(), ok. +-define(mod, t_copy_literals_frags). +t_copy_literals_frags(Config) when is_list(Config) -> + Bin = gen_lit(?mod,[{a,{1,2,3,4,5,6,7}}, + {b,"hello world"}, + {c, <<"hello world">>}, + {d, {"hello world", {1.0, 2.0, <<"some">>, "string"}}}, + {e, <<"off heap", 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9,10,11,12,13,14,15, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9,10,11,12,13,14,15, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9,10,11,12,13,14,15, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9,10,11,12,13,14,15>>}]), + + {module, ?mod} = erlang:load_module(?mod, Bin), + N = 6000, + Recv = spawn_opt(fun() -> receive + read -> + io:format("reading"), + literal_receiver() + end + end, [link,{min_heap_size, 10000}]), + Switcher = spawn_link(fun() -> literal_switcher() end), + Pids = [spawn_opt(fun() -> receive + {Pid, go, Recv, N} -> + io:format("sender batch (~w) start ~w~n",[N,self()]), + literal_sender(N,Recv), + Pid ! {self(), ok} + end + end, [link,{min_heap_size,800}]) || _ <- lists:seq(1,100)], + _ = [Pid ! {self(), go, Recv, N} || Pid <- Pids], + %% don't read immediately + timer:sleep(5), + Recv ! read, + Switcher ! {switch,?mod,Bin,[Recv|Pids],200}, + _ = [receive {Pid, ok} -> ok end || Pid <- Pids], + Switcher ! {self(), done}, + receive {Switcher, ok} -> ok end, + Recv ! {self(), done}, + receive {Recv, ok} -> ok end, + ok. + +literal_receiver() -> + receive + {Pid, done} -> + io:format("reader_done~n"), + Pid ! {self(), ok}; + {_Pid, msg, [A,B,C,D,E]} -> + A = ?mod:a(), + B = ?mod:b(), + C = ?mod:c(), + D = ?mod:d(), + E = ?mod:e(), + literal_receiver(); + {Pid, sender_confirm} -> + io:format("sender confirm ~w~n", [Pid]), + Pid ! {self(), ok}, + literal_receiver() + end. + +literal_sender(0, Recv) -> + Recv ! {self(), sender_confirm}, + receive {Recv, ok} -> ok end; +literal_sender(N, Recv) -> + Recv ! {self(), msg, [?mod:a(), + ?mod:b(), + ?mod:c(), + ?mod:d(), + ?mod:e()]}, + literal_sender(N - 1, Recv). + +literal_switcher() -> + receive + {switch,Mod,Bin,Pids,Tmo} -> + literal_switcher(Mod,Bin,Pids,Tmo) + end. +literal_switcher(Mod,Bin,Pids,Tmo) -> + receive + {Pid,done} -> + Pid ! {self(),ok} + after Tmo -> + io:format("load module ~w~n", [Mod]), + {module, Mod} = erlang:load_module(Mod,Bin), + ok = check_and_purge(Pids,Mod), + io:format("purge complete ~w~n", [Mod]), + literal_switcher(Mod,Bin,Pids,Tmo+Tmo) + end. + +check_and_purge([],Mod) -> + erlang:purge_module(Mod), + ok; +check_and_purge(Pids,Mod) -> + io:format("purge ~w~n", [Mod]), + Tag = make_ref(), + _ = [begin + erlang:check_process_code(Pid,Mod,[{async,{Tag,Pid}}]) + end || Pid <- Pids], + Retry = check_and_purge_receive(Pids,Tag,[]), + check_and_purge(Retry,Mod). + +check_and_purge_receive([Pid|Pids],Tag,Retry) -> + receive + {check_process_code, {Tag, Pid}, false} -> + check_and_purge_receive(Pids,Tag,Retry); + {check_process_code, {Tag, Pid}, true} -> + check_and_purge_receive(Pids,Tag,[Pid|Retry]) + end; +check_and_purge_receive([],_,Retry) -> + Retry. + + +gen_lit(Module,Terms) -> + FunStrings = [lists:flatten(io_lib:format("~w() -> ~w.~n", [F,Term]))||{F,Term}<-Terms], + FunForms = function_forms(FunStrings), + Forms = [{attribute,erl_anno:new(1),module,Module}, + {attribute,erl_anno:new(2),export,[FA || {FA,_} <- FunForms]}] ++ + [Function || {_, Function} <- FunForms], + {ok, Module, Bin} = compile:forms(Forms), + Bin. + +function_forms([]) -> []; +function_forms([S|Ss]) -> + {ok, Ts,_} = erl_scan:string(S), + {ok, Form} = erl_parse:parse_form(Ts), + Fun = element(3, Form), + Arity = element(4, Form), + [{{Fun,Arity}, Form}|function_forms(Ss)]. chase_msg(0, Pid) -> chase_loop(Pid); diff --git a/erts/emulator/test/dirty_nif_SUITE.erl b/erts/emulator/test/dirty_nif_SUITE.erl index c3afbc0803..83b098a704 100644 --- a/erts/emulator/test/dirty_nif_SUITE.erl +++ b/erts/emulator/test/dirty_nif_SUITE.erl @@ -32,19 +32,23 @@ dirty_nif/1, dirty_nif_send/1, dirty_nif_exception/1, call_dirty_nif_exception/1, dirty_scheduler_exit/1, dirty_call_while_terminated/1, - dirty_heap_access/1]). + dirty_heap_access/1, dirty_process_info/1, + dirty_process_register/1, dirty_process_trace/1]). -define(nif_stub,nif_stub_error(?LINE)). suite() -> [{ct_hooks,[ts_install_cth]}]. -all() -> +all() -> [dirty_nif, dirty_nif_send, dirty_nif_exception, dirty_scheduler_exit, dirty_call_while_terminated, - dirty_heap_access]. + dirty_heap_access, + dirty_process_info, + dirty_process_register, + dirty_process_trace]. init_per_suite(Config) -> try erlang:system_info(dirty_cpu_schedulers) of @@ -187,7 +191,7 @@ dirty_call_while_terminated(Config) when is_list(Config) -> blipp:blupp(Bin) end, [monitor,link]), - receive {dirty_alive, Pid} -> ok end, + receive {dirty_alive, _Pid} -> ok end, {value, {BinAddr, 4711, 2}} = lists:keysearch(4711, 2, element(2, process_info(self(), @@ -241,7 +245,7 @@ dirty_heap_access(Config) when is_list(Config) -> end), {N, R} = access_dirty_heap(Dirty, RGL, 0, 0), receive - {Pid, Res} -> + {_Pid, Res} -> 1000 = length(Res), lists:foreach(fun (X) -> Ref = X end, Res) end, @@ -269,12 +273,123 @@ access_dirty_heap(Dirty, RGL, N, R) -> end) end. +%% These tests verify that processes that access a process executing a +%% dirty NIF where the main lock is needed for that access do not get +%% blocked. Each test passes its pid to dirty_sleeper, which sends a +%% 'ready' message when it's running on a dirty scheduler and just before +%% it starts a 6 second sleep. When it receives the message, it verifies +%% that access to the dirty process is as it expects. After the dirty +%% process finishes its 6 second sleep but before it returns from the dirty +%% scheduler, it sends a 'done' message. If the tester already received +%% that message, the test fails because it means attempting to access the +%% dirty process waited for that process to return to a regular scheduler, +%% so verify that we haven't received that message, and also verify that +%% the dirty process is still alive immediately after accessing it. +dirty_process_info(Config) when is_list(Config) -> + access_dirty_process( + Config, + fun() -> ok end, + fun(NifPid) -> + PI = process_info(NifPid), + {current_function,{?MODULE,dirty_sleeper,1}} = + lists:keyfind(current_function, 1, PI), + ok + end, + fun(_) -> ok end). + +dirty_process_register(Config) when is_list(Config) -> + access_dirty_process( + Config, + fun() -> ok end, + fun(NifPid) -> + register(test_dirty_process_register, NifPid), + NifPid = whereis(test_dirty_process_register), + unregister(test_dirty_process_register), + false = lists:member(test_dirty_process_register, + registered()), + ok + end, + fun(_) -> ok end). + +dirty_process_trace(Config) when is_list(Config) -> + access_dirty_process( + Config, + fun() -> + erlang:trace_pattern({?MODULE,dirty_sleeper,1}, + [{'_',[],[{return_trace}]}], + [local,meta]), + ok + end, + fun(NifPid) -> + erlang:trace(NifPid, true, [call,timestamp]), + ok + end, + fun(NifPid) -> + receive + done -> + receive + {trace_ts,NifPid,call,{?MODULE,dirty_sleeper,_},_} -> + ok + after + 0 -> + error(missing_trace_call_message) + end, + receive + {trace_ts,NifPid,return_from,{?MODULE,dirty_sleeper,1}, + ok,_} -> + ok + after + 100 -> + error(missing_trace_return_message) + end + after + 6500 -> + error(missing_done_message) + end, + ok + end). + %% %% Internal... %% +access_dirty_process(Config, Start, Test, Finish) -> + {ok, Node} = start_node(Config, ""), + [ok] = mcall(Node, + [fun() -> + Path = ?config(data_dir, Config), + Lib = atom_to_list(?MODULE), + ok = erlang:load_nif(filename:join(Path,Lib), []), + ok = test_dirty_process_access(Start, Test, Finish) + end]), + stop_node(Node), + ok. + +test_dirty_process_access(Start, Test, Finish) -> + ok = Start(), + Self = self(), + NifPid = spawn_link(fun() -> + ok = dirty_sleeper(Self) + end), + ok = receive + ready -> + ok = Test(NifPid), + receive + done -> + error(dirty_process_info_blocked) + after + 0 -> + true = erlang:is_process_alive(NifPid), + ok + end + after + 3000 -> + error(timeout) + end, + ok = Finish(NifPid). + receive_any() -> - receive M -> M end. + receive M -> M end. start_node(Config) -> start_node(Config, ""). @@ -314,13 +429,13 @@ mcall(Node, Funs) -> %% The NIFs: lib_loaded() -> false. -call_nif_schedule(_,_) -> ?nif_stub. call_dirty_nif(_,_,_) -> ?nif_stub. send_from_dirty_nif(_) -> ?nif_stub. call_dirty_nif_exception(_) -> ?nif_stub. call_dirty_nif_zero_args() -> ?nif_stub. dirty_call_while_terminated_nif(_) -> ?nif_stub. dirty_sleeper() -> ?nif_stub. +dirty_sleeper(_) -> ?nif_stub. dirty_heap_access_nif(_) -> ?nif_stub. nif_stub_error(Line) -> diff --git a/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c b/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c index 2013c88167..e38bececde 100644 --- a/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c +++ b/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c @@ -19,7 +19,9 @@ */ #include "erl_nif.h" #include <assert.h> -#ifndef __WIN32__ +#ifdef __WIN32__ +#include <windows.h> +#else #include <unistd.h> #endif @@ -146,12 +148,31 @@ static ERL_NIF_TERM call_dirty_nif_zero_args(ErlNifEnv* env, int argc, const ERL static ERL_NIF_TERM dirty_sleeper(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { + ErlNifPid pid; + ErlNifEnv* msg_env = NULL; + assert(enif_is_on_dirty_scheduler(env)); + + /* If we get a pid argument, it indicates a process involved in the + test wants a message from us. Prior to the sleep we send a 'ready' + message, and then after the sleep, send a 'done' message. */ + if (argc == 1 && enif_get_local_pid(env, argv[0], &pid)) { + msg_env = enif_alloc_env(); + enif_send(env, &pid, msg_env, enif_make_atom(msg_env, "ready")); + } + #ifdef __WIN32__ Sleep(6000); #else sleep(6); #endif + + if (argc == 1) { + assert(msg_env != NULL); + enif_send(env, &pid, msg_env, enif_make_atom(msg_env, "done")); + enif_free_env(msg_env); + } + return enif_make_atom(env, "ok"); } @@ -216,6 +237,7 @@ static ErlNifFunc nif_funcs[] = {"call_dirty_nif_exception", 1, call_dirty_nif_exception, ERL_NIF_DIRTY_JOB_IO_BOUND}, {"call_dirty_nif_zero_args", 0, call_dirty_nif_zero_args, ERL_NIF_DIRTY_JOB_CPU_BOUND}, {"dirty_sleeper", 0, dirty_sleeper, ERL_NIF_DIRTY_JOB_IO_BOUND}, + {"dirty_sleeper", 1, dirty_sleeper, ERL_NIF_DIRTY_JOB_CPU_BOUND}, {"dirty_call_while_terminated_nif", 1, dirty_call_while_terminated_nif, ERL_NIF_DIRTY_JOB_CPU_BOUND}, {"dirty_heap_access_nif", 1, dirty_heap_access_nif, ERL_NIF_DIRTY_JOB_CPU_BOUND} }; diff --git a/erts/emulator/test/distribution_SUITE.erl b/erts/emulator/test/distribution_SUITE.erl index d0096fb1bc..26780f6017 100644 --- a/erts/emulator/test/distribution_SUITE.erl +++ b/erts/emulator/test/distribution_SUITE.erl @@ -55,7 +55,8 @@ bad_dist_ext_receive/1, bad_dist_ext_process_info/1, bad_dist_ext_control/1, - bad_dist_ext_connection_id/1]). + bad_dist_ext_connection_id/1, + start_epmd_false/1, epmd_module/1]). %% Internal exports. -export([sender/3, receiver2/2, dummy_waiter/0, dead_process/0, @@ -64,6 +65,9 @@ dist_evil_parallel_receiver/0, sendersender/4, sendersender2/4]). +%% epmd_module exports +-export([start_link/0, register_node/2, port_please/2]). + suite() -> [{ct_hooks,[ts_install_cth]}, {timetrap, {minutes, 4}}]. @@ -76,7 +80,8 @@ all() -> {group, trap_bif}, {group, dist_auto_connect}, dist_parallel_send, atom_roundtrip, unicode_atom_roundtrip, atom_roundtrip_r15b, contended_atom_cache_entry, contended_unicode_atom_cache_entry, - bad_dist_structure, {group, bad_dist_ext}]. + bad_dist_structure, {group, bad_dist_ext}, + start_epmd_false, epmd_module]. groups() -> [{bulk_send, [], [bulk_send_small, bulk_send_big, bulk_send_bigbig]}, @@ -1881,6 +1886,66 @@ dmsg_ext(Term) -> dmsg_bad_atom_cache_ref() -> [$R, 137]. +start_epmd_false(Config) when is_list(Config) -> + %% Start a node with the option -start_epmd false. + {ok, OtherNode} = start_node(start_epmd_false, "-start_epmd false"), + %% We should be able to ping it, as epmd was started by us: + pong = net_adm:ping(OtherNode), + stop_node(OtherNode), + + ok. + +epmd_module(Config) when is_list(Config) -> + %% We need a relay node to test this, since the test node uses the + %% standard epmd module. + Sock1 = start_relay_node(epmd_module_node1, "-epmd_module " ++ ?MODULE_STRING), + Node1 = inet_rpc_nodename(Sock1), + %% Ask what port it's listening on - it won't have registered with + %% epmd. + {ok, {ok, Port1}} = do_inet_rpc(Sock1, application, get_env, [kernel, dist_listen_port]), + + %% Start a second node, passing the port number as a secret + %% argument. + Sock2 = start_relay_node(epmd_module_node2, "-epmd_module " ++ ?MODULE_STRING + ++ " -other_node_port " ++ integer_to_list(Port1)), + Node2 = inet_rpc_nodename(Sock2), + %% Node 1 can't ping node 2 + {ok, pang} = do_inet_rpc(Sock1, net_adm, ping, [Node2]), + {ok, []} = do_inet_rpc(Sock1, erlang, nodes, []), + {ok, []} = do_inet_rpc(Sock2, erlang, nodes, []), + %% But node 2 can ping node 1 + {ok, pong} = do_inet_rpc(Sock2, net_adm, ping, [Node1]), + {ok, [Node2]} = do_inet_rpc(Sock1, erlang, nodes, []), + {ok, [Node1]} = do_inet_rpc(Sock2, erlang, nodes, []), + + stop_relay_node(Sock2), + stop_relay_node(Sock1). + +%% epmd_module functions: + +start_link() -> + ignore. + +register_node(_Name, Port) -> + %% Save the port number we're listening on. + application:set_env(kernel, dist_listen_port, Port), + Creation = rand:uniform(3), + {ok, Creation}. + +port_please(_Name, _Ip) -> + case init:get_argument(other_node_port) of + error -> + %% None specified. Default to 42. + Port = 42, + Version = 5, + {port, Port, Version}; + {ok, [[PortS]]} -> + %% Port number given on command line. + Port = list_to_integer(PortS), + Version = 5, + {port, Port, Version} + end. + %%% Utilities timestamp() -> diff --git a/erts/emulator/test/lttng_SUITE.erl b/erts/emulator/test/lttng_SUITE.erl index efc79f42ed..1360751aee 100644 --- a/erts/emulator/test/lttng_SUITE.erl +++ b/erts/emulator/test/lttng_SUITE.erl @@ -44,6 +44,7 @@ suite() -> all() -> [t_lttng_list, + t_memory_carrier, t_carrier_pool, t_async_io_pool, t_driver_start_stop, @@ -52,8 +53,7 @@ all() -> t_driver_timeout, t_driver_caller, t_driver_flush, - t_scheduler_poll, - t_memory_carrier]. + t_scheduler_poll]. init_per_suite(Config) -> @@ -120,7 +120,7 @@ t_lttng_list(_Config) -> %% com_ericsson_otp:carrier_pool_get %% com_ericsson_otp:carrier_pool_put t_carrier_pool(Config) -> - case have_carriers() of + case have_carriers(ets_alloc) of false -> {skip, "No Memory Carriers configured on system."}; true -> @@ -137,7 +137,7 @@ t_carrier_pool(Config) -> %% com_ericsson_otp:carrier_destroy %% com_ericsson_otp:carrier_create t_memory_carrier(Config) -> - case have_carriers() of + case have_carriers(ets_alloc) of false -> {skip, "No Memory Carriers configured on system."}; true -> @@ -178,11 +178,13 @@ t_async_io_pool(Config) -> %% com_ericsson_otp:driver_stop t_driver_start_stop(Config) -> ok = lttng_start_event("com_ericsson_otp:driver_*", Config), + timer:sleep(500), Path = proplists:get_value(priv_dir, Config), Name = filename:join(Path, "sometext.txt"), Bin = txt(), ok = file:write_file(Name, Bin), {ok, Bin} = file:read_file(Name), + timer:sleep(500), Res = lttng_stop_and_view(Config), ok = check_tracepoint("com_ericsson_otp:driver_start", Res), ok = check_tracepoint("com_ericsson_otp:driver_stop", Res), @@ -213,6 +215,7 @@ t_driver_control_ready_async(Config) -> %% com_ericsson_otp:driver_ready_output t_driver_ready_input_output(Config) -> ok = lttng_start_event("com_ericsson_otp:driver_ready_*", Config), + timer:sleep(500), Me = self(), Pid = spawn_link(fun() -> tcp_server(Me, active) end), receive {Pid, accept} -> ok end, @@ -225,6 +228,7 @@ t_driver_ready_input_output(Config) -> ok = gen_tcp:close(Sock), receive {Pid, done} -> ok end, + timer:sleep(500), Res = lttng_stop_and_view(Config), ok = check_tracepoint("com_ericsson_otp:driver_ready_input", Res), ok = check_tracepoint("com_ericsson_otp:driver_ready_output", Res), @@ -446,11 +450,10 @@ load_driver(Dir, Driver) -> %% check -have_carriers() -> - Cap = element(3,erlang:system_info(allocator)), - case Cap -- [sys_alloc,sys_aligned_alloc] of - [] -> false; - _ -> true +have_carriers(Alloc) -> + case erlang:system_info({allocator,Alloc}) of + false -> false; + _ -> true end. have_async_threads() -> diff --git a/erts/emulator/test/message_queue_data_SUITE.erl b/erts/emulator/test/message_queue_data_SUITE.erl index 226462676c..44e77dfad0 100644 --- a/erts/emulator/test/message_queue_data_SUITE.erl +++ b/erts/emulator/test/message_queue_data_SUITE.erl @@ -52,18 +52,12 @@ basic(Config) when is_list(Config) -> ok = rpc:call(Node2, ?MODULE, basic_test, [on_heap]), stop_node(Node2), - {ok, Node3} = start_node(Config, "+hmqd mixed"), - ok = rpc:call(Node3, ?MODULE, basic_test, [mixed]), - stop_node(Node3), - ok. is_valid_mqd_value(off_heap) -> true; is_valid_mqd_value(on_heap) -> true; -is_valid_mqd_value(mixed) -> - true; is_valid_mqd_value(_) -> false. @@ -78,9 +72,6 @@ basic_test(Default) -> {message_queue_data, off_heap} = process_info(self(), message_queue_data), off_heap = process_flag(message_queue_data, on_heap), {message_queue_data, on_heap} = process_info(self(), message_queue_data), - on_heap = process_flag(message_queue_data, mixed), - {message_queue_data, mixed} = process_info(self(), message_queue_data), - mixed = process_flag(message_queue_data, Default), {'EXIT', _} = (catch process_flag(message_queue_data, blupp)), P1 = spawn_opt(fun () -> receive after infinity -> ok end end, @@ -101,12 +92,6 @@ basic_test(Default) -> unlink(P3), exit(P3, bye), - P4 = spawn_opt(fun () -> receive after infinity -> ok end end, - [link, {message_queue_data, mixed}]), - {message_queue_data, mixed} = process_info(P4, message_queue_data), - unlink(P4), - exit(P4, bye), - {'EXIT', _} = (catch spawn_opt(fun () -> receive after infinity -> ok end end, [link, {message_queue_data, blapp}])), @@ -116,21 +101,18 @@ process_info_messages(Config) when is_list(Config) -> Tester = self(), P1 = spawn_opt(fun () -> receive after 500 -> ok end, - mixed = process_flag(message_queue_data, off_heap), + on_heap = process_flag(message_queue_data, off_heap), Tester ! first, receive after 500 -> ok end, off_heap = process_flag(message_queue_data, on_heap), Tester ! second, receive after 500 -> ok end, - on_heap = process_flag(message_queue_data, mixed), + on_heap = process_flag(message_queue_data, off_heap), Tester ! third, - receive after 500 -> ok end, - mixed = process_flag(message_queue_data, off_heap), - Tester ! fourth, receive after infinity -> ok end end, - [link, {message_queue_data, mixed}]), + [link, {message_queue_data, on_heap}]), P1 ! "A", receive first -> ok end, @@ -139,25 +121,20 @@ process_info_messages(Config) when is_list(Config) -> P1 ! "C", receive third -> ok end, P1 ! "D", - receive fourth -> ok end, - P1 ! "E", - {messages, ["A", "B", "C", "D", "E"]} = process_info(P1, messages), + {messages, ["A", "B", "C", "D"]} = process_info(P1, messages), P2 = spawn_opt(fun () -> receive after 500 -> ok end, - mixed = process_flag(message_queue_data, off_heap), + on_heap = process_flag(message_queue_data, off_heap), Tester ! first, receive after 500 -> ok end, off_heap = process_flag(message_queue_data, on_heap), Tester ! second, receive after 500 -> ok end, - on_heap = process_flag(message_queue_data, mixed), + on_heap = process_flag(message_queue_data, off_heap), Tester ! third, receive after 500 -> ok end, - mixed = process_flag(message_queue_data, off_heap), - Tester ! fourth, - receive after 500 -> ok end, Tester ! process_info(self(), messages), @@ -165,11 +142,10 @@ process_info_messages(Config) when is_list(Config) -> receive M2 -> M2 = "B" end, receive M3 -> M3 = "C" end, receive M4 -> M4 = "D" end, - receive M5 -> M5 = "E" end, Tester ! self() end, - [link, {message_queue_data, mixed}]), + [link, {message_queue_data, on_heap}]), P2 ! "A", receive first -> ok end, @@ -178,12 +154,10 @@ process_info_messages(Config) when is_list(Config) -> P2 ! "C", receive third -> ok end, P2 ! "D", - receive fourth -> ok end, - P2 ! "E", receive Msg -> - {messages, ["A", "B", "C", "D", "E"]} = Msg + {messages, ["A", "B", "C", "D"]} = Msg end, receive P2 -> ok end, diff --git a/erts/emulator/test/tracer_SUITE.erl b/erts/emulator/test/tracer_SUITE.erl index 20fb7e475e..9eb55c9af3 100644 --- a/erts/emulator/test/tracer_SUITE.erl +++ b/erts/emulator/test/tracer_SUITE.erl @@ -28,9 +28,9 @@ init_per_group/2,end_per_group/2, init_per_testcase/2, end_per_testcase/2]). -export([load/1, unload/1, reload/1, invalid_tracers/1]). --export([send/1, recv/1, spawn/1, exit/1, link/1, unlink/1, - getting_linked/1, getting_unlinked/1, register/1, unregister/1, - in/1, out/1, gc_start/1, gc_end/1]). +-export([send/1, recv/1, call/1, call_return/1, spawn/1, exit/1, + link/1, unlink/1, getting_linked/1, getting_unlinked/1, + register/1, unregister/1, in/1, out/1, gc_start/1, gc_end/1]). suite() -> [{ct_hooks,[ts_install_cth]}, {timetrap, {minutes, 1}}]. @@ -39,9 +39,9 @@ all() -> [load, unload, reload, invalid_tracers, {group, basic}]. groups() -> - [{ basic, [], [send, recv, spawn, exit, link, unlink, getting_linked, - getting_unlinked, register, unregister, in, out, - gc_start, gc_end]}]. + [{ basic, [], [send, recv, call, call_return, spawn, exit, + link, unlink, getting_linked, getting_unlinked, + register, unregister, in, out, gc_start, gc_end]}]. init_per_suite(Config) -> erlang:trace_pattern({'_','_','_'}, false, [local]), @@ -223,8 +223,8 @@ send(_Config) -> Expect = fun(Pid, State, EOpts) -> receive Msg -> - {send, State, Pid, ok, Self, Opts} = Msg, - check_opts(EOpts, Opts) + {send, State, Pid, ok, Opts} = Msg, + check_opts(EOpts, Opts, Self) end end, test(send, Tc, Expect). @@ -239,13 +239,59 @@ recv(_Config) -> Expect = fun(Pid, State, EOpts) -> receive Msg -> - {'receive', State, Pid, ok, undefined, Opts} = Msg, + {'receive', State, Pid, ok, Opts} = Msg, check_opts(EOpts, Opts) end end, test('receive', Tc, Expect, false). +call(_Config) -> + + Self = self(), + Tc = fun(Pid) -> + Pid ! fun() -> call_test(Self), Self ! ok end, + receive ok -> ok after 100 -> ct:fail(timeout) end + end, + + erlang:trace_pattern({?MODULE, call_test, 1}, [], [local]), + + Expect = fun(Pid, State, EOpts) -> + receive + Msg -> + {call, State, Pid, {?MODULE, call_test, [Self]}, Opts} = Msg, + check_opts(EOpts, Opts) + end + end, + test(call, Tc, Expect). + +call_return(_Config) -> + + Self = self(), + Tc = fun(Pid) -> + Pid ! fun() -> call_test(undefined), Self ! ok end, + receive ok -> ok after 100 -> ct:fail(timeout) end + end, + + 1 = erlang:trace_pattern({?MODULE, call_test, 1}, [{'_',[],[{return_trace}]}], [local]), + + Expect = fun(Pid, State, EOpts) -> + receive + CallMsg -> + {call, State, Pid, {?MODULE, call_test, [undefined]}, COpts} = CallMsg, + check_opts(EOpts, COpts) + end, + receive + RetMsg -> + {return_from, State, Pid, {?MODULE, call_test, 1}, ROpts} = RetMsg, + check_opts(EOpts, ROpts, undefined) + end + end, + test(call, Tc, Expect). + +call_test(Arg) -> + Arg. + spawn(_Config) -> Tc = fun(Pid) -> @@ -256,9 +302,8 @@ spawn(_Config) -> fun(Pid, State, EOpts) -> receive Msg -> - {spawn, State, Pid, NewPid, - {lists,seq,[1,10]}, Opts} = Msg, - check_opts(EOpts, Opts), + {spawn, State, Pid, NewPid, Opts} = Msg, + check_opts(EOpts, Opts, {lists,seq,[1,10]}), true = is_pid(NewPid) andalso NewPid /= Pid end end, @@ -274,7 +319,7 @@ exit(_Config) -> fun(Pid, State, EOpts) -> receive Msg -> - {exit, State, Pid, normal, undefined, Opts} = Msg, + {exit, State, Pid, normal, Opts} = Msg, check_opts(EOpts, Opts) end end, @@ -295,7 +340,7 @@ link(_Config) -> fun(Pid, State, EOpts) -> receive Msg -> - {link, State, Pid, NewPid, undefined, Opts} = Msg, + {link, State, Pid, NewPid, Opts} = Msg, check_opts(EOpts, Opts), true = is_pid(NewPid) andalso NewPid /= Pid end @@ -318,7 +363,7 @@ unlink(_Config) -> fun(Pid, State, EOpts) -> receive Msg -> - {unlink, State, Pid, NewPid, undefined, Opts} = Msg, + {unlink, State, Pid, NewPid, Opts} = Msg, check_opts(EOpts, Opts), true = is_pid(NewPid) andalso NewPid /= Pid end @@ -340,7 +385,7 @@ getting_linked(_Config) -> fun(Pid, State, EOpts) -> receive Msg -> - {getting_linked, State, Pid, NewPid, undefined, Opts} = Msg, + {getting_linked, State, Pid, NewPid, Opts} = Msg, check_opts(EOpts, Opts), true = is_pid(NewPid) andalso NewPid /= Pid end @@ -364,7 +409,7 @@ getting_unlinked(_Config) -> fun(Pid, State, EOpts) -> receive Msg -> - {getting_unlinked, State, Pid, NewPid, undefined, Opts} = Msg, + {getting_unlinked, State, Pid, NewPid, Opts} = Msg, check_opts(EOpts, Opts), true = is_pid(NewPid) andalso NewPid /= Pid end @@ -386,7 +431,7 @@ register(_Config) -> fun(Pid, State, EOpts) -> receive Msg -> - {register, State, Pid, ?MODULE, undefined, Opts} = Msg, + {register, State, Pid, ?MODULE, Opts} = Msg, check_opts(EOpts, Opts) end end, @@ -407,7 +452,7 @@ unregister(_Config) -> fun(Pid, State, EOpts) -> receive Msg -> - {unregister, State, Pid, ?MODULE, undefined, Opts} = Msg, + {unregister, State, Pid, ?MODULE, Opts} = Msg, check_opts(EOpts, Opts) end end, @@ -427,8 +472,7 @@ in(_Config) -> N = (fun F(N) -> receive Msg -> - {in, State, Pid, _, - undefined, Opts} = Msg, + {in, State, Pid, _, Opts} = Msg, check_opts(EOpts, Opts), F(N+1) after 0 -> N @@ -452,8 +496,7 @@ out(_Config) -> N = (fun F(N) -> receive Msg -> - {out, State, Pid, _, - undefined, Opts} = Msg, + {out, State, Pid, _, Opts} = Msg, check_opts(EOpts, Opts), F(N+1) after 0 -> N @@ -477,7 +520,7 @@ gc_start(_Config) -> fun(Pid, State, EOpts) -> receive Msg -> - {gc_major_start, State, Pid, _, undefined, Opts} = Msg, + {gc_major_start, State, Pid, _, Opts} = Msg, check_opts(EOpts, Opts) end end, @@ -497,7 +540,7 @@ gc_end(_Config) -> fun(Pid, State, EOpts) -> receive Msg -> - {gc_major_end, State, Pid, _, undefined, Opts} = Msg, + {gc_major_end, State, Pid, _, Opts} = Msg, check_opts(EOpts, Opts) end end, @@ -513,9 +556,7 @@ test(Event, TraceFlag, Tc, Expect, Removes) -> test(Event, TraceFlag, Tc, Expect, _Removes, Dies) -> ComplexState = {fun() -> ok end, <<0:(128*8)>>}, - Opts = #{ timestamp => undefined, - scheduler_id => undefined, - match_spec_result => true }, + Opts = #{ }, %% Test that trace works State1 = {#{ Event => trace }, self(), ComplexState}, @@ -540,8 +581,8 @@ test(Event, TraceFlag, Tc, Expect, _Removes, Dies) -> Tc(Pid1T), ok = trace_delivered(Pid1T), - Expect(Pid1T, State1, Opts#{ scheduler_id := number, - timestamp := timestamp}), + Expect(Pid1T, State1, Opts#{ scheduler_id => number, + timestamp => timestamp}), receive M11T -> ct:fail({unexpected, M11T}) after 0 -> ok end, if not Dies -> {flags, [scheduler_id, TraceFlag, timestamp]} @@ -568,6 +609,8 @@ test(Event, TraceFlag, Tc, Expect, _Removes, Dies) -> ok. +check_opts(E, O, Extra) -> + check_opts(E#{ extra => Extra }, O). check_opts(#{ scheduler_id := number } = E, #{ scheduler_id := N } = O) when is_integer(N) -> E1 = maps:remove(scheduler_id, E), diff --git a/erts/emulator/test/tracer_SUITE_data/tracer_test.c b/erts/emulator/test/tracer_SUITE_data/tracer_test.c index 908f35da9c..a26bb33600 100644 --- a/erts/emulator/test/tracer_SUITE_data/tracer_test.c +++ b/erts/emulator/test/tracer_SUITE_data/tracer_test.c @@ -36,7 +36,7 @@ static ERL_NIF_TERM trace(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); static ErlNifFunc nif_funcs[] = { {"enabled", 3, enabled}, - {"trace", 6, trace} + {"trace", 5, trace} }; ERL_NIF_INIT(tracer_test, nif_funcs, load, NULL, upgrade, unload) @@ -100,7 +100,7 @@ static ERL_NIF_TERM trace(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) ErlNifPid self, to; ERL_NIF_TERM *tuple, msg; const ERL_NIF_TERM *state_tuple; - ASSERT(argc == 6); + ASSERT(argc == 5); enif_get_tuple(env, argv[1], &state_arity, &state_tuple); diff --git a/erts/emulator/test/tracer_test.erl b/erts/emulator/test/tracer_test.erl index d4778f4531..1da80bfe31 100644 --- a/erts/emulator/test/tracer_test.erl +++ b/erts/emulator/test/tracer_test.erl @@ -24,14 +24,14 @@ %%% Test tracer %%% --export([enabled/3, trace/6]). +-export([enabled/3, trace/5]). -export([load/1, load/2]). -on_load(load/0). enabled(_, _, _) -> erlang:nif_error(nif_not_loaded). -trace(_, _, _, _, _, _) -> +trace(_, _, _, _, _) -> erlang:nif_error(nif_not_loaded). load() -> |