diff options
Diffstat (limited to 'erts')
29 files changed, 649 insertions, 367 deletions
diff --git a/erts/doc/src/erl_nif.xml b/erts/doc/src/erl_nif.xml index 57e047af08..4efd155b09 100644 --- a/erts/doc/src/erl_nif.xml +++ b/erts/doc/src/erl_nif.xml @@ -322,13 +322,13 @@ ok <p>The <seealso marker="#enif_consume_timeslice">enif_consume_timeslice()</seealso> - function can be used to inform the runtime system about the lenght of the + function can be used to inform the runtime system about the length of the NIF call. It should typically always be used unless the NIF executes very quickly.</p> - <p>If the NIF call is too lenghty one needs to handle this in one of the + <p>If the NIF call is too lengthy one needs to handle this in one of the following ways in order to avoid degraded responsiveness, scheduler load - balancing problems, and other strange behaviours:</p> + balancing problems, and other strange behaviors:</p> <taglist> <tag>Yielding NIF</tag> @@ -410,14 +410,14 @@ ok <seealso marker="erlang#system_flag_multi_scheduling"><c>erlang:system_flag(multi_scheduling, block)</c></seealso>, might also take a very long time to complete. This since all ongoing dirty operations on all - dirty schedulers need to complete before the the block + dirty schedulers need to complete before the block operation can complete. </p> <p> A lot of operations communicating with a process executing a dirty NIF can, however, complete while it is executing the - dirty NIF. For example, retreiving information about it via + dirty NIF. For example, retrieving information about it via <c>process_info()</c>, setting its group leader, register/unregister its name, etc. </p> @@ -425,10 +425,10 @@ ok <p> Termination of a process executing a dirty NIF can only be completed up to a certain point while it is executing the - dirty NIF. All Erlang resources such as registered names, - ETS tables, etc will be released. All links and monitors + dirty NIF. All Erlang resources such as its registered name, + its ETS tables, etc will be released. All links and monitors will be triggered. The actual execution of the NIF will - however <em>not</em> be stopped. The NIF can safely contiue + however <em>not</em> be stopped. The NIF can safely continue execution, allocate heap memory, etc, but it is of course better to stop executing as soon as possible. The NIF can check whether current process is alive or not using @@ -450,8 +450,8 @@ ok collect a process in order to determine if it has references to the module, a process executing a dirty NIF might delay purging for a very long time. Delaying - a purge operatin implies delaying <em>all</em> code - loding operations which might cause severe problems for + a purge operation implies delaying <em>all</em> code + loading operations which might cause severe problems for the system as a whole. </p> </item> diff --git a/erts/doc/src/erlang.xml b/erts/doc/src/erlang.xml index ddde1c96c4..e0c3fed0c2 100644 --- a/erts/doc/src/erlang.xml +++ b/erts/doc/src/erlang.xml @@ -1802,6 +1802,8 @@ os_prompt% </pre> <tag>string()</tag> <item>An Erlang crash dump is produced with <c><anno>Status</anno></c> as slogan. Then the runtime system exits with status code <c>1</c>. + Note that only code points in the range 0-255 may be used + and the string will be truncated if longer than 200 characters. </item> <tag><c>abort</c></tag> <item> diff --git a/erts/doc/src/init.xml b/erts/doc/src/init.xml index 878a33c9ca..a88a815ef6 100644 --- a/erts/doc/src/init.xml +++ b/erts/doc/src/init.xml @@ -178,14 +178,8 @@ <name name="stop" arity="0"/> <fsummary>Take down an Erlang node smoothly</fsummary> <desc> - <p>All applications are taken down smoothly, all code is - unloaded, and all ports are closed before the system - terminates. If the <c>-heart</c> command line flag was given, - the <c>heart</c> program is terminated before the Erlang node - terminates. Refer to <c>heart(3)</c> for more information.</p> - <p>To limit the shutdown time, the time <c>init</c> is allowed - to spend taking down applications, the <c>-shutdown_time</c> - command line flag should be used.</p> + <p>The same as + <seealso marker="#stop/1"><c>stop(0)</c></seealso>.</p> </desc> </func> <func> diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index f8f2e29c95..0a59f8785c 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -1323,11 +1323,7 @@ void process_main(void) if (start_time != 0) { Sint64 diff = erts_timestamp_millis() - start_time; - if (diff > 0 && (Uint) diff > erts_system_monitor_long_schedule -#if defined(ERTS_SMP) && defined(ERTS_DIRTY_SCHEDULERS) - && !ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(c_p)) -#endif - ) { + if (diff > 0 && (Uint) diff > erts_system_monitor_long_schedule) { BeamInstr *inptr = find_function_from_pc(start_time_i); BeamInstr *outptr = find_function_from_pc(c_p->i); monitor_long_schedule_proc(c_p,inptr,outptr,(Uint) diff); @@ -1337,7 +1333,7 @@ void process_main(void) PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - c_p = schedule(c_p, reds_used); + c_p = erts_schedule(NULL, c_p, reds_used); ASSERT(!(c_p->flags & F_HIPE_MODE)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); start_time = 0; @@ -3559,12 +3555,10 @@ do { \ typedef Eterm NifF(struct enif_environment_t*, int argc, Eterm argv[]); NifF* fp = vbf = (NifF*) I[1]; struct enif_environment_t env; -#ifdef ERTS_DIRTY_SCHEDULERS - if (!c_p->scheduler_data) - live_hf_end = ERTS_INVALID_HFRAG_PTR; /* On dirty scheduler */ - else +#ifdef ERTS_SMP + ASSERT(c_p->scheduler_data); #endif - live_hf_end = c_p->mbuf; + live_hf_end = c_p->mbuf; erts_pre_nif(&env, c_p, (struct erl_module_nif*)I[2], NULL); nif_bif_result = (*fp)(&env, bif_nif_arity, reg); if (env.exception_thrown) @@ -3574,10 +3568,7 @@ do { \ PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR); - if (env.exiting) { - ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); - goto do_schedule; - } + ASSERT(!env.exiting); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); } @@ -5162,6 +5153,283 @@ do { \ } } +/* + * erts_dirty_process_main() is what dirty schedulers execute. Since they handle + * only NIF calls they do not need to be able to execute all BEAM + * instructions. + */ +void erts_dirty_process_main(ErtsSchedulerData *esdp) +{ +#ifdef ERTS_DIRTY_SCHEDULERS + Process* c_p = NULL; + ErtsMonotonicTime start_time; +#ifdef DEBUG + ERTS_DECLARE_DUMMY(Eterm pid); +#endif + + /* Pointer to X registers: x(1)..x(N); reg[0] is used when doing GC, + * in all other cases x0 is used. + */ + register Eterm* reg REG_xregs = NULL; + + /* + * Top of heap (next free location); grows upwards. + */ + register Eterm* HTOP REG_htop = NULL; + + /* Stack pointer. Grows downwards; points + * to last item pushed (normally a saved + * continuation pointer). + */ + register Eterm* E REG_stop = NULL; + + /* + * Pointer to next threaded instruction. + */ + register BeamInstr *I REG_I = NULL; + + ERTS_MSACC_DECLARE_CACHE_X() /* a cached value of the tsd pointer for msacc */ + + /* + * start_time always positive for dirty CPU schedulers, + * and negative for dirty I/O schedulers. + */ + + if (ERTS_SCHEDULER_IS_DIRTY_CPU(esdp)) { + start_time = erts_get_monotonic_time(NULL); + ASSERT(start_time >= 0); + } + else { + start_time = ERTS_SINT64_MIN; + ASSERT(start_time < 0); + } + + goto do_dirty_schedule; + + context_switch: + c_p->arity = I[-1]; + c_p->current = I-3; /* Pointer to Mod, Func, Arity */ + + { + int reds_used; + Eterm* argp; + int i; + + /* + * Make sure that there is enough room for the argument registers to be saved. + */ + if (c_p->arity > c_p->max_arg_reg) { + /* + * Yes, this is an expensive operation, but you only pay it the first + * time you call a function with more than 6 arguments which is + * scheduled out. This is better than paying for 26 words of wasted + * space for most processes which never call functions with more than + * 6 arguments. + */ + Uint size = c_p->arity * sizeof(c_p->arg_reg[0]); + if (c_p->arg_reg != c_p->def_arg_reg) { + c_p->arg_reg = (Eterm *) erts_realloc(ERTS_ALC_T_ARG_REG, + (void *) c_p->arg_reg, + size); + } else { + c_p->arg_reg = (Eterm *) erts_alloc(ERTS_ALC_T_ARG_REG, size); + } + c_p->max_arg_reg = c_p->arity; + } + + /* + * Save the argument registers and everything else. + */ + + argp = c_p->arg_reg; + for (i = c_p->arity - 1; i >= 0; i--) { + argp[i] = reg[i]; + } + SWAPOUT; + c_p->i = I; + + do_dirty_schedule: + + if (start_time < 0) { + /* + * Dirty I/O scheduler: + * One reduction consumed regardless of + * time spent in the dirty NIF. + */ + reds_used = esdp->virtual_reds + 1; + } + else { + /* + * Dirty CPU scheduler: + * Currently two reductions consumed per + * micro second spent in the dirty NIF. + */ + ErtsMonotonicTime time; + time = erts_get_monotonic_time(esdp); + time -= start_time; + time = ERTS_MONOTONIC_TO_USEC(time); + time *= (CONTEXT_REDS-1)/1000 + 1; + ASSERT(time >= 0); + if (time == 0) + time = 1; /* At least one reduction */ + time += esdp->virtual_reds; + reds_used = time > INT_MAX ? INT_MAX : (int) time; + } + + PROCESS_MAIN_CHK_LOCKS(c_p); + ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + c_p = erts_schedule(esdp, c_p, reds_used); + + if (start_time >= 0) { + start_time = erts_get_monotonic_time(esdp); + ASSERT(start_time >= 0); + } + } + + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); +#ifdef DEBUG + pid = c_p->common.id; /* Save for debugging purposes */ +#endif + ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); + PROCESS_MAIN_CHK_LOCKS(c_p); + + ASSERT(!(c_p->flags & F_HIPE_MODE)); + ERTS_MSACC_UPDATE_CACHE_X(); + + reg = esdp->x_reg_array; + { + Eterm* argp; + int i; + + argp = c_p->arg_reg; + for (i = c_p->arity - 1; i >= 0; i--) { + reg[i] = argp[i]; + CHECK_TERM(reg[i]); + } + + /* + * We put the original reduction count in the process structure, to reduce + * the code size (referencing a field in a struct through a pointer stored + * in a register gives smaller code than referencing a global variable). + */ + + I = c_p->i; + + ASSERT(BeamOp(op_call_nif) == (BeamInstr *) *I); + + if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p) + && (ERTS_TRACE_FLAGS(c_p) & F_SENSITIVE) == 0) { + c_p->fcalls = REDS_IN(c_p) = 0; + } + + SWAPIN; + +#ifdef USE_VM_PROBES + if (DTRACE_ENABLED(process_scheduled)) { + DTRACE_CHARBUF(process_buf, DTRACE_TERM_BUF_SIZE); + DTRACE_CHARBUF(fun_buf, DTRACE_TERM_BUF_SIZE); + dtrace_proc_str(c_p, process_buf); + + if (ERTS_PROC_IS_EXITING(c_p)) { + strcpy(fun_buf, "<exiting>"); + } else { + BeamInstr *fptr = find_function_from_pc(c_p->i); + if (fptr) { + dtrace_fun_decode(c_p, (Eterm)fptr[0], + (Eterm)fptr[1], (Uint)fptr[2], + NULL, fun_buf); + } else { + erts_snprintf(fun_buf, sizeof(DTRACE_CHARBUF_NAME(fun_buf)), + "<unknown/%p>", next); + } + } + + DTRACE2(process_scheduled, process_buf, fun_buf); + } +#endif + } + + { + Eterm nif_bif_result; + Eterm bif_nif_arity; + + { + /* + * call_nif is always first instruction in function: + * + * I[-3]: Module + * I[-2]: Function + * I[-1]: Arity + * I[0]: &&call_nif + * I[1]: Function pointer to NIF function + * I[2]: Pointer to erl_module_nif + * I[3]: Function pointer to dirty NIF + */ + BifFunction vbf; + + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_NIF); + + DTRACE_NIF_ENTRY(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]); + c_p->current = I-3; /* current and vbf set to please handle_error */ + SWAPOUT; + PROCESS_MAIN_CHK_LOCKS(c_p); + bif_nif_arity = I[-1]; + ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); + + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + { + typedef Eterm NifF(struct enif_environment_t*, int argc, Eterm argv[]); + NifF* fp = vbf = (NifF*) I[1]; + struct enif_environment_t env; + ASSERT(!c_p->scheduler_data); + erts_pre_dirty_nif(esdp, &env, c_p, (struct erl_module_nif*)I[2], NULL); + nif_bif_result = (*fp)(&env, bif_nif_arity, reg); + if (env.exception_thrown) + nif_bif_result = THE_NON_VALUE; + erts_post_nif(&env); + PROCESS_MAIN_CHK_LOCKS(c_p); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR); + if (env.exiting) { + ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); + goto do_dirty_schedule; + } + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + } + DTRACE_NIF_RETURN(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]); + ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); + ERTS_HOLE_CHECK(c_p); + if (ERTS_IS_GC_DESIRED(c_p)) { + nif_bif_result = erts_gc_after_bif_call(c_p, + nif_bif_result, + reg, bif_nif_arity); + } + SWAPIN; /* There might have been a garbage collection. */ + if (is_value(nif_bif_result)) { + r(0) = nif_bif_result; + CHECK_TERM(r(0)); + I = c_p->cp; + c_p->cp = 0; + Goto(*I); + } else if (c_p->freason == TRAP) { + I = c_p->i; + ASSERT(!(c_p->flags & F_HIBERNATE_SCHED)); + goto context_switch; + } + I = handle_error(c_p, c_p->cp, reg, vbf); + } + } + if (I == 0) { + goto do_dirty_schedule; + } else { + ASSERT(!is_value(r(0))); + SWAPIN; + goto context_switch; + } +#endif /* ERTS_DIRTY_SCHEDULERS */ +} + static BifFunction translate_gc_bif(void* gcf) { diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index e77da526fd..b18910e2c7 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -3833,59 +3833,11 @@ BIF_RETTYPE display_nl_0(BIF_ALIST_0) /**********************************************************************/ -/* stop the system */ -/* ARGSUSED */ -BIF_RETTYPE halt_0(BIF_ALIST_0) -{ - VERBOSE(DEBUG_SYSTEM,("System halted by BIF halt()\n")); - erts_halt(0); - ERTS_BIF_YIELD1(bif_export[BIF_halt_1], BIF_P, am_undefined); -} - -/**********************************************************************/ #define HALT_MSG_SIZE 200 -static char halt_msg[HALT_MSG_SIZE]; - -/* stop the system with exit code */ -/* ARGSUSED */ -BIF_RETTYPE halt_1(BIF_ALIST_1) -{ - Uint code; - - if (term_to_Uint_mask(BIF_ARG_1, &code)) { - int pos_int_code = (int) (code & INT_MAX); - VERBOSE(DEBUG_SYSTEM,("System halted by BIF halt(%T)\n", BIF_ARG_1)); - erts_halt(pos_int_code); - ERTS_BIF_YIELD1(bif_export[BIF_halt_1], BIF_P, am_undefined); - } - else if (ERTS_IS_ATOM_STR("abort", BIF_ARG_1)) { - VERBOSE(DEBUG_SYSTEM,("System halted by BIF halt(%T)\n", BIF_ARG_1)); - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_exit(ERTS_ABORT_EXIT, ""); - } - else if (is_string(BIF_ARG_1) || BIF_ARG_1 == NIL) { - Sint i; - - if ((i = intlist_to_buf(BIF_ARG_1, halt_msg, HALT_MSG_SIZE-1)) < 0) { - goto error; - } - halt_msg[i] = '\0'; - VERBOSE(DEBUG_SYSTEM,("System halted by BIF halt(%T)\n", BIF_ARG_1)); - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_exit(ERTS_DUMP_EXIT, "%s\n", halt_msg); - } - else - goto error; - return NIL; /* Pedantic (lint does not know about erts_exit) */ - error: - BIF_ERROR(BIF_P, BADARG); -} - -/**********************************************************************/ +static char halt_msg[HALT_MSG_SIZE+1]; /* stop the system with exit code and flags */ -/* ARGSUSED */ BIF_RETTYPE halt_2(BIF_ALIST_2) { Uint code; @@ -3921,7 +3873,7 @@ BIF_RETTYPE halt_2(BIF_ALIST_2) ("System halted by BIF halt(%T, %T)\n", BIF_ARG_1, BIF_ARG_2)); if (flush) { erts_halt(pos_int_code); - ERTS_BIF_YIELD1(bif_export[BIF_halt_1], BIF_P, am_undefined); + ERTS_BIF_YIELD2(bif_export[BIF_halt_2], BIF_P, am_undefined, am_undefined); } else { erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); @@ -3937,9 +3889,12 @@ BIF_RETTYPE halt_2(BIF_ALIST_2) else if (is_string(BIF_ARG_1) || BIF_ARG_1 == NIL) { Sint i; - if ((i = intlist_to_buf(BIF_ARG_1, halt_msg, HALT_MSG_SIZE-1)) < 0) { - goto error; - } + if ((i = intlist_to_buf(BIF_ARG_1, halt_msg, HALT_MSG_SIZE)) == -1) { + goto error; + } + if (i == -2) /* truncated string */ + i = HALT_MSG_SIZE; + ASSERT(i >= 0 && i <= HALT_MSG_SIZE); halt_msg[i] = '\0'; VERBOSE(DEBUG_SYSTEM, ("System halted by BIF halt(%T, %T)\n", BIF_ARG_1, BIF_ARG_2)); diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index 872f0f9b2a..065018514a 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -72,8 +72,6 @@ bif erlang:get/1 bif erlang:get_keys/1 bif erlang:group_leader/0 bif erlang:group_leader/2 -bif erlang:halt/0 -bif erlang:halt/1 bif erlang:halt/2 bif erlang:phash/2 bif erlang:phash2/1 diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.c b/erts/emulator/beam/erl_ao_firstfit_alloc.c index fbe4724047..7e239d1f5d 100644 --- a/erts/emulator/beam/erl_ao_firstfit_alloc.c +++ b/erts/emulator/beam/erl_ao_firstfit_alloc.c @@ -123,7 +123,7 @@ struct AOFF_Carrier_t_ { AOFF_RBTree_t rbt_node; /* My node in the carrier tree */ AOFF_RBTree_t* root; /* Root of my block tree */ }; -#define RBT_NODE_TO_MBC(PTR) ((AOFF_Carrier_t*)((char*)(PTR) - offsetof(AOFF_Carrier_t, rbt_node))) +#define RBT_NODE_TO_MBC(PTR) ErtsContainerStruct((PTR), AOFF_Carrier_t, rbt_node) /* To support carrier migration we keep two kinds of rb-trees: diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index c7bbbd5ca0..d0d74bbf44 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -1183,22 +1183,14 @@ minor_collection(Process* p, ErlHeapFragment *live_hf_end, adjust_size = p->htop - p->heap; } - goto done; } + else if (need_after > HEAP_SIZE(p)) { + grow_new_heap(p, next_heap_size(p, need_after, 0), objv, nobj); + adjust_size = p->htop - p->heap; + } + /*else: The heap size turned out to be just right. We are done. */ - if (HEAP_SIZE(p) >= need_after) { - /* - * The heap size turned out to be just right. We are done. - */ - goto done; - } - - grow_new_heap(p, next_heap_size(p, need_after, 0), objv, nobj); - adjust_size = p->htop - p->heap; - - done: ASSERT(HEAP_SIZE(p) == next_heap_size(p, HEAP_SIZE(p), 0)); - ASSERT(MBUF(p) == NULL); /* The heap usage during GC should be larger than what we end up after a GC, even if we grow it. If this assertion is not true @@ -1591,6 +1583,9 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end, HIGH_WATER(p) = HEAP_TOP(p); +#ifdef HARDDEBUG + disallow_heap_frag_ref_in_heap(p); +#endif remove_message_buffers(p); if (p->flags & F_ON_HEAP_MSGQ) @@ -1603,9 +1598,6 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end, adjusted = adjust_after_fullsweep(p, need, objv, nobj); -#ifdef HARDDEBUG - disallow_heap_frag_ref_in_heap(p); -#endif ErtsGcQuickSanityCheck(p); return gc_cost(size_after, adjusted ? size_after : 0); diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index ac7b9d6606..71ab92937d 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -1345,7 +1345,7 @@ erts_prep_msgq_for_inspection(Process *c_p, Process *rp, mpp = i == 0 ? &rp->msg.first : &mip[i-1].msgp->next; - ASSERT((*mpp)->next == bad_mp); + ASSERT(*mpp == bad_mp); erts_msgq_update_internal_pointers(&rp->msg, mpp, &bad_mp->next); @@ -1725,7 +1725,7 @@ void erts_factory_trim_and_close(ErtsHeapFactory* factory, case FACTORY_MESSAGE: { ErtsMessage *mp = factory->message; if (mp->data.attached == ERTS_MSG_COMBINED_HFRAG) { - if (!mp->hfrag.next) { + if (!factory->heap_frags) { Uint sz = factory->hp - factory->hp_start; mp = erts_shrink_message(mp, sz, brefs, brefs_size); factory->message = mp; diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 2bbb8e3c91..f1c35cd5a5 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -178,9 +178,6 @@ static ERTS_INLINE void ensure_heap(ErlNifEnv* env, size_t may_need) void erts_pre_nif(ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif, Process* tracee) { -#ifdef ERTS_DIRTY_SCHEDULERS - ErtsSchedulerData *esdp; -#endif env->mod_nif = mod_nif; env->proc = p; env->hp = HEAP_TOP(p); @@ -193,57 +190,65 @@ void erts_pre_nif(ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif, ASSERT(p->common.id != ERTS_INVALID_PID); -#ifdef ERTS_DIRTY_SCHEDULERS - esdp = erts_get_scheduler_data(); - ASSERT(esdp); +#if defined(DEBUG) && defined(ERTS_DIRTY_SCHEDULERS) + { + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + ASSERT(esdp); - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { -#ifdef DEBUG - erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state); + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state); - ASSERT(p->scheduler_data == esdp); - ASSERT((state & (ERTS_PSFLG_RUNNING - | ERTS_PSFLG_RUNNING_SYS)) - && !(state & (ERTS_PSFLG_DIRTY_RUNNING - | ERTS_PSFLG_DIRTY_RUNNING_SYS))); + ASSERT(p->scheduler_data == esdp); + ASSERT((state & (ERTS_PSFLG_RUNNING + | ERTS_PSFLG_RUNNING_SYS)) + && !(state & (ERTS_PSFLG_DIRTY_RUNNING + | ERTS_PSFLG_DIRTY_RUNNING_SYS))); + } + } #endif +} - } - else { - Process *sproc; +void erts_pre_dirty_nif(ErtsSchedulerData *esdp, + ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif, + Process* tracee) +{ +#ifdef ERTS_DIRTY_SCHEDULERS + Process *sproc; #ifdef DEBUG - erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state); + erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state); - ASSERT(!p->scheduler_data); - ASSERT((state & ERTS_PSFLG_DIRTY_RUNNING) - && !(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS))); + ASSERT(!p->scheduler_data); + ASSERT((state & ERTS_PSFLG_DIRTY_RUNNING) + && !(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS))); + ASSERT(esdp); #endif - sproc = esdp->dirty_shadow_process; - ASSERT(sproc); - ASSERT(sproc->static_flags & ERTS_STC_FLG_SHADOW_PROC); - ASSERT(erts_smp_atomic32_read_nob(&sproc->state) - == (ERTS_PSFLG_ACTIVE - | ERTS_PSFLG_DIRTY_RUNNING - | ERTS_PSFLG_PROXY)); - - sproc->next = p; - sproc->common.id = p->common.id; - sproc->htop = p->htop; - sproc->stop = p->stop; - sproc->hend = p->hend; - sproc->heap = p->heap; - sproc->abandoned_heap = p->abandoned_heap; - sproc->heap_sz = p->heap_sz; - sproc->high_water = p->high_water; - sproc->old_hend = p->old_hend; - sproc->old_htop = p->old_htop; - sproc->old_heap = p->old_heap; - sproc->mbuf = NULL; - sproc->mbuf_sz = 0; - ERTS_INIT_OFF_HEAP(&sproc->off_heap); - env->proc = sproc; - } + erts_pre_nif(env, p, mod_nif, tracee); + + sproc = esdp->dirty_shadow_process; + ASSERT(sproc); + ASSERT(sproc->static_flags & ERTS_STC_FLG_SHADOW_PROC); + ASSERT(erts_smp_atomic32_read_nob(&sproc->state) + == (ERTS_PSFLG_ACTIVE + | ERTS_PSFLG_DIRTY_RUNNING + | ERTS_PSFLG_PROXY)); + + sproc->next = p; + sproc->common.id = p->common.id; + sproc->htop = p->htop; + sproc->stop = p->stop; + sproc->hend = p->hend; + sproc->heap = p->heap; + sproc->abandoned_heap = p->abandoned_heap; + sproc->heap_sz = p->heap_sz; + sproc->high_water = p->high_water; + sproc->old_hend = p->old_hend; + sproc->old_htop = p->old_htop; + sproc->old_heap = p->old_heap; + sproc->mbuf = NULL; + sproc->mbuf_sz = 0; + ERTS_INIT_OFF_HEAP(&sproc->off_heap); + env->proc = sproc; #endif } diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index f8cbe60e76..c0b1d7246c 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -8197,7 +8197,7 @@ sched_dirty_cpu_thread_func(void *vesdp) #endif erts_thread_init_float(); - process_main(); + erts_dirty_process_main(esdp); /* No schedulers should *ever* terminate */ erts_exit(ERTS_ABORT_EXIT, "Dirty CPU scheduler thread number %beu terminated\n", @@ -8242,7 +8242,7 @@ sched_dirty_io_thread_func(void *vesdp) #endif erts_thread_init_float(); - process_main(); + erts_dirty_process_main(esdp); /* No schedulers should *ever* terminate */ erts_exit(ERTS_ABORT_EXIT, "Dirty I/O scheduler thread number %beu terminated\n", @@ -9377,77 +9377,6 @@ scheduler_gc_proc(Process *c_p, int reds_left) return reds; } -static ERTS_INLINE void -clean_dirty_start(Process *p) -{ -#if defined(ERTS_DIRTY_SCHEDULERS) && !defined(ARCH_64) - void *ptr = ERTS_PROC_SET_DIRTY_CPU_START(p, NULL); - if (ptr) - erts_free(ERTS_ALC_T_DIRTY_START, ptr); -#endif -} - -static ERTS_INLINE void -save_dirty_start(ErtsSchedulerData *esdp, Process *c_p) -{ -#ifdef ERTS_DIRTY_SCHEDULERS - if (ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(esdp->run_queue)) { - ErtsMonotonicTime time = erts_get_monotonic_time(esdp); -#ifdef ARCH_64 - ERTS_PROC_SET_DIRTY_CPU_START(c_p, (void *) time); -#else - ErtsMonotonicTime *stimep; - - stimep = (ErtsMonotonicTime *) ERTS_PROC_GET_DIRTY_CPU_START(c_p); - if (!stimep) { - stimep = erts_alloc(ERTS_ALC_T_DIRTY_START, - sizeof(ErtsMonotonicTime)); - ERTS_PROC_SET_DIRTY_CPU_START(c_p, (void *) stimep); - } - *stimep = time; -#endif - } -#endif -} - -static ERTS_INLINE int -get_dirty_reds(ErtsSchedulerData *esdp, Process *c_p) -{ - -#ifndef ERTS_DIRTY_SCHEDULERS - return -1; -#else - ErtsMonotonicTime stime, time; - - if (!ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(esdp->run_queue)) - return 1; - -#ifdef ARCH_64 - stime = (ErtsMonotonicTime) ERTS_PROC_GET_DIRTY_CPU_START(c_p); -#else - { - ErtsMonotonicTime *stimep; - stimep = (ErtsMonotonicTime *) ERTS_PROC_GET_DIRTY_CPU_START(c_p); - ASSERT(stimep); - stime = *stimep; - } -#endif - - time = erts_get_monotonic_time(esdp); - - ASSERT(stime && stime < time); - - time -= stime; - time = ERTS_MONOTONIC_TO_USEC(time); - time *= 2; - - if (time > INT_MAX) - return INT_MAX; - return (int) time; -#endif - -} - /* * schedule() is called from BEAM (process_main()) or HiPE * (hipe_mode_switch()) when the current process is to be @@ -9466,11 +9395,10 @@ get_dirty_reds(ErtsSchedulerData *esdp, Process *c_p) * so that normal processes get to run more frequently. */ -Process *schedule(Process *p, int calls) +Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) { Process *proxy_p = NULL; ErtsRunQueue *rq; - ErtsSchedulerData *esdp; int context_reds; int fcalls; int input_reductions; @@ -9507,8 +9435,19 @@ Process *schedule(Process *p, int calls) * Clean up after the process being scheduled out. */ if (!p) { /* NULL in the very first schedule() call */ +#ifdef ERTS_DIRTY_SCHEDULERS + is_normal_sched = !esdp; + if (is_normal_sched) { + esdp = erts_get_scheduler_data(); + ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp)); + } + else { + ASSERT(ERTS_SCHEDULER_IS_DIRTY(esdp)); + } +#else esdp = erts_get_scheduler_data(); - is_normal_sched = !ERTS_SCHEDULER_IS_DIRTY(esdp); + is_normal_sched = 1; +#endif rq = erts_get_runq_current(esdp); ASSERT(esdp); fcalls = (int) erts_smp_atomic32_read_acqb(&function_calls); @@ -9517,12 +9456,12 @@ Process *schedule(Process *p, int calls) } else { #ifdef ERTS_SMP #ifdef ERTS_DIRTY_SCHEDULERS - esdp = p->scheduler_data; - is_normal_sched = esdp != NULL; - if (is_normal_sched) + is_normal_sched = !esdp; + if (is_normal_sched) { + esdp = p->scheduler_data; ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp)); + } else { - esdp = erts_get_scheduler_data(); ASSERT(ERTS_SCHEDULER_IS_DIRTY(esdp)); } #else @@ -9541,10 +9480,7 @@ Process *schedule(Process *p, int calls) ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); - if (is_normal_sched) - reds = actual_reds = calls - esdp->virtual_reds; - else - reds = actual_reds = get_dirty_reds(esdp, p); + reds = actual_reds = calls - esdp->virtual_reds; ASSERT(actual_reds >= 0); if (reds < ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST) @@ -9994,17 +9930,10 @@ Process *schedule(Process *p, int calls) calls = 0; reds = context_reds; -#ifdef ERTS_SMP - erts_smp_runq_unlock(rq); -#endif /* ERTS_SMP */ - } - if (!is_normal_sched) - save_dirty_start(esdp, p); - #ifdef ERTS_SMP if (flags & ERTS_RUNQ_FLG_PROTECTED) @@ -11745,8 +11674,6 @@ delete_process(Process* p) if (nif_export) erts_destroy_nif_export(nif_export); - clean_dirty_start(p); - /* Cleanup psd */ psd = (ErtsPSD *) erts_smp_atomic_read_nob(&p->psd); diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index b44ac442aa..7c98b60647 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -810,25 +810,13 @@ erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi) #define ERTS_PSD_DELAYED_GC_TASK_QS 4 #define ERTS_PSD_NIF_TRAP_EXPORT 5 #define ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF 6 -#define ERTS_PSD_DIRTY_CPU_START 7 -#define ERTS_PSD_SIZE 8 +#define ERTS_PSD_SIZE 7 -#if !defined(HIPE) && !defined(ERTS_DIRTY_SCHEDULERS) +#if !defined(HIPE) # undef ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF -# undef ERTS_PSD_DIRTY_CPU_START # undef ERTS_PSD_SIZE # define ERTS_PSD_SIZE 6 -#elif !defined(HIPE) -# undef ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF -# undef ERTS_PSD_DIRTY_CPU_START -# undef ERTS_PSD_SIZE -# define ERTS_PSD_DIRTY_CPU_START 6 -# define ERTS_PSD_SIZE 7 -#elif !defined(ERTS_DIRTY_SCHEDULERS) -# undef ERTS_PSD_DIRTY_CPU_START -# undef ERTS_PSD_SIZE -# define ERTS_PSD_SIZE 7 #endif typedef struct { @@ -1179,6 +1167,9 @@ void erts_check_for_holes(Process* p); * USR_PRIO -> User prio. i.e., prio the user has set. * PRQ_PRIO -> Prio queue prio, i.e., prio queue currently * enqueued in. + * + * Update etp-proc-state-int in $ERL_TOP/erts/etc/unix/etp-commands.in + * when changing ERTS_PSFLG_*. */ #define ERTS_PSFLGS_ACT_PRIO_MASK \ (ERTS_PSFLGS_PRIO_MASK << ERTS_PSFLGS_ACT_PRIO_OFFSET) @@ -1831,7 +1822,7 @@ Eterm erts_get_schedulers_binds(Process *c_p); Eterm erts_set_cpu_topology(Process *c_p, Eterm term); Eterm erts_bind_schedulers(Process *c_p, Eterm how); ErtsRunQueue *erts_schedid2runq(Uint); -Process *schedule(Process*, int); +Process *erts_schedule(ErtsSchedulerData *, Process*, int); void erts_schedule_misc_op(void (*)(void *), void *); Eterm erl_create_process(Process*, Eterm, Eterm, Eterm, ErlSpawnOpts*); void erts_do_exit_process(Process*, Eterm); @@ -2061,13 +2052,6 @@ erts_psd_set(Process *p, int ix, void *data) ((struct saved_calls *) erts_psd_set((P), ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF, (void *) (SCB))) #endif -#ifdef ERTS_DIRTY_SCHEDULERS -#define ERTS_PROC_GET_DIRTY_CPU_START(P) \ - ((void *) erts_psd_get((P), ERTS_PSD_DIRTY_CPU_START)) -#define ERTS_PROC_SET_DIRTY_CPU_START(P, DCS) \ - ((void *) erts_psd_set((P), ERTS_PSD_DIRTY_CPU_START, (void *) (DCS))) -#endif - ERTS_GLB_INLINE Eterm erts_proc_get_error_handler(Process *p); ERTS_GLB_INLINE Eterm erts_proc_set_error_handler(Process *p, Eterm handler); diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c index 02c24557c1..2a19211987 100644 --- a/erts/emulator/beam/export.c +++ b/erts/emulator/beam/export.c @@ -31,7 +31,7 @@ #define EXPORT_INITIAL_SIZE 4000 #define EXPORT_LIMIT (512*1024) -#define EXPORT_HASH(m,f,a) ((m)*(f)+(a)) +#define EXPORT_HASH(m,f,a) ((atom_val(m) * atom_val(f)) ^ (a)) #ifdef DEBUG # define IF_DEBUG(x) x @@ -79,8 +79,7 @@ struct export_templ static struct export_blob* entry_to_blob(struct export_entry* ee) { - return (struct export_blob*) - ((char*)ee->ep - offsetof(struct export_blob,exp)); + return ErtsContainerStruct(ee->ep, struct export_blob, exp); } void diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index b76b9cd874..f3d4ac56cd 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -62,6 +62,9 @@ struct enif_environment_t /* ErlNifEnv */ extern void erts_pre_nif(struct enif_environment_t*, Process*, struct erl_module_nif*, Process* tracee); extern void erts_post_nif(struct enif_environment_t* env); +extern void erts_pre_dirty_nif(ErtsSchedulerData *, + struct enif_environment_t*, Process*, + struct erl_module_nif*, Process* tracee); extern Eterm erts_nif_taints(Process* p); extern void erts_print_nif_taints(int to, void* to_arg); void erts_unload_nif(struct erl_module_nif* nif); @@ -1152,6 +1155,7 @@ void print_pass_through(int, byte*, int); int catchlevel(Process*); void init_emulator(void); void process_main(void); +void erts_dirty_process_main(ErtsSchedulerData *); Eterm build_stacktrace(Process* c_p, Eterm exc); Eterm expand_error_value(Process* c_p, Uint freason, Eterm Value); void erts_save_stacktrace(Process* p, struct StackTrace* s, int depth); diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index f303d4f167..9a205d50d3 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -97,7 +97,7 @@ ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes)) #define ErtsContainerStruct(ptr, type, member) \ - (type *)((char *)(1 ? (ptr) : &((type *)0)->member) - offsetof(type, member)) + ((type *)((char *)(1 ? (ptr) : &((type *)0)->member) - offsetof(type, member))) #if defined (__WIN32__) # include "erl_win_sys.h" diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index cedc88e5fe..675fafa726 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -2210,7 +2210,9 @@ do_allocate_logger_message(Eterm gleader, Eterm **hp, ErlOffHeap **ohp, #ifndef ERTS_SMP #ifdef USE_THREADS - if (erts_get_scheduler_data()) /* Must be scheduler thread */ + if (!erts_get_scheduler_data()) /* Must be scheduler thread */ + *p = NULL; + else #endif { *p = erts_whereis_process(NULL, 0, am_error_logger, 0, 0); @@ -2226,18 +2228,10 @@ do_allocate_logger_message(Eterm gleader, Eterm **hp, ErlOffHeap **ohp, } /* So we have an error logger, lets build the message */ - if (sz <= HeapWordsLeft(*p)) { - *ohp = &MSO(*p); - *hp = HEAP_TOP(*p); - HEAP_TOP(*p) += sz; - } else { -#endif - *bp = new_message_buffer(sz); - *ohp = &(*bp)->off_heap; - *hp = (*bp)->mem; -#ifndef ERTS_SMP - } #endif + *bp = new_message_buffer(sz); + *ohp = &(*bp)->off_heap; + *hp = (*bp)->mem; return (is_nil(gleader) ? am_noproc @@ -3893,8 +3887,10 @@ void bin_write(int to, void *to_arg, byte* buf, size_t sz) } /* Fill buf with the contents of bytelist list - return number of chars in list or -1 for error */ - + * return number of chars in list + * or -1 for type error + * or -2 for not enough buffer space (buffer contains truncated result) + */ Sint intlist_to_buf(Eterm list, char *buf, Sint len) { @@ -3917,7 +3913,7 @@ intlist_to_buf(Eterm list, char *buf, Sint len) return -1; listptr = list_val(*(listptr + 1)); } - return -1; /* not enough space */ + return -2; /* not enough space */ } /* diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c index 58b5be3906..3336fded7a 100644 --- a/erts/emulator/hipe/hipe_bif0.c +++ b/erts/emulator/hipe/hipe_bif0.c @@ -697,7 +697,7 @@ static struct nbif nbifs[BIF_SIZE] = { #undef BIF_LIST }; -#define NBIF_HASH(m,f,a) ((m)*(f)+(a)) +#define NBIF_HASH(m,f,a) (atom_val(m) ^ atom_val(f) ^ (a)) static Hash nbif_table; static HashValue nbif_hash(struct nbif *x) @@ -1063,7 +1063,7 @@ static inline void hipe_mfa_info_table_rwunlock(void) erts_smp_rwmtx_rwunlock(&hipe_mfa_info_table.lock); } -#define HIPE_MFA_HASH(M,F,A) ((M) * (F) + (A)) +#define HIPE_MFA_HASH(M,F,A) (atom_val(M) ^ atom_val(F) ^ (A)) static struct hipe_mfa_info **hipe_mfa_info_table_alloc_bucket(unsigned int size) { @@ -1144,10 +1144,13 @@ static inline struct hipe_mfa_info *hipe_mfa_info_table_get_locked(Eterm m, Eter h = HIPE_MFA_HASH(m, f, arity); i = h & hipe_mfa_info_table.mask; p = hipe_mfa_info_table.bucket[i]; - for (; p; p = p->bucket.next) - /* XXX: do we want to compare p->bucket.hvalue as well? */ - if (p->m == m && p->f == f && p->a == arity) - return p; + for (; p; p = p->bucket.next) { + if (p->bucket.hvalue == h) { + if (p->m == m && p->f == f && p->a == arity) + return p; + } + else ASSERT(!(p->m == m && p->f == f && p->a == arity)); + } return NULL; } @@ -1171,10 +1174,13 @@ static struct hipe_mfa_info *hipe_mfa_info_table_put_rwlocked(Eterm m, Eterm f, h = HIPE_MFA_HASH(m, f, arity); i = h & hipe_mfa_info_table.mask; p = hipe_mfa_info_table.bucket[i]; - for (; p; p = p->bucket.next) - /* XXX: do we want to compare p->bucket.hvalue as well? */ - if (p->m == m && p->f == f && p->a == arity) - return p; + for (; p; p = p->bucket.next) { + if (p->bucket.hvalue == h) { + if (p->m == m && p->f == f && p->a == arity) + return p; + } + else ASSERT(!(p->m == m && p->f == f && p->a == arity)); + } p = hipe_mfa_info_table_alloc(m, f, arity); p->bucket.hvalue = h; p->bucket.next = hipe_mfa_info_table.bucket[i]; diff --git a/erts/emulator/hipe/hipe_mode_switch.c b/erts/emulator/hipe/hipe_mode_switch.c index 884331e969..ed95045292 100644 --- a/erts/emulator/hipe/hipe_mode_switch.c +++ b/erts/emulator/hipe/hipe_mode_switch.c @@ -547,7 +547,7 @@ Process *hipe_mode_switch(Process *p, unsigned cmd, Eterm reg[]) p->flags &= ~F_HIPE_MODE; ERTS_SMP_UNREQ_PROC_MAIN_LOCK(p); - p = schedule(p, reds_in - p->fcalls); + p = erts_schedule(NULL, p, reds_in - p->fcalls); ERTS_SMP_REQ_PROC_MAIN_LOCK(p); ASSERT(!(p->flags & F_HIPE_MODE)); #ifdef ERTS_SMP diff --git a/erts/emulator/sys/win32/erl_poll.c b/erts/emulator/sys/win32/erl_poll.c index 94f3840b5f..f23c7ab03d 100644 --- a/erts/emulator/sys/win32/erl_poll.c +++ b/erts/emulator/sys/win32/erl_poll.c @@ -424,7 +424,7 @@ static ERTS_INLINE int wakeup_cause(ErtsPollSet ps) { int res; - erts_aint32_t wakeup_state = erts_atomic32_read_nob(&ps->wakeup_state); + erts_aint32_t wakeup_state = erts_atomic32_read_acqb(&ps->wakeup_state); switch (wakeup_state) { case ERTS_POLL_WOKEN_IO_READY: res = 0; @@ -487,9 +487,8 @@ wake_poller(ErtsPollSet ps, int io_ready) { erts_aint32_t wakeup_state; if (io_ready) { - /* We may set the event multiple times. This is, however, harmless. */ - wakeup_state = erts_atomic32_read_nob(&ps->wakeup_state); - erts_atomic32_set_relb(&ps->wakeup_state, ERTS_POLL_WOKEN_IO_READY); + wakeup_state = erts_atomic32_xchg_relb(&ps->wakeup_state, + ERTS_POLL_WOKEN_IO_READY); } else { ERTS_THR_MEMORY_BARRIER; diff --git a/erts/emulator/test/bif_SUITE.erl b/erts/emulator/test/bif_SUITE.erl index 26bb416bf0..d31399e4af 100644 --- a/erts/emulator/test/bif_SUITE.erl +++ b/erts/emulator/test/bif_SUITE.erl @@ -141,9 +141,11 @@ guard_bifs_in_erl_bif_types(_Config) -> shadow_comments(_Config) -> ensure_erl_bif_types_compiled(), + ErlangList = [{erlang,F,A} || {F,A} <- erlang:module_info(exports), + not is_operator(F,A)], List0 = erlang:system_info(snifs), - List1 = [MFA || {M,_,_}=MFA <- List0, M =/= hipe_bifs], - List = [MFA || MFA <- List1, not is_operator(MFA)], + List1 = [MFA || {M,_,_}=MFA <- List0, M =/= hipe_bifs, M =/= erlang], + List = List1 ++ ErlangList, HasTypes = [MFA || {M,F,A}=MFA <- List, erl_bif_types:is_known(M, F, A)], Path = get_code_path(), @@ -253,12 +255,15 @@ specs(_) -> end. is_operator({erlang,F,A}) -> + is_operator(F,A); +is_operator(_) -> false. + +is_operator(F,A) -> erl_internal:arith_op(F, A) orelse erl_internal:bool_op(F, A) orelse erl_internal:comp_op(F, A) orelse erl_internal:list_op(F, A) orelse - erl_internal:send_op(F, A); -is_operator(_) -> false. + erl_internal:send_op(F, A). extract_specs(M, Abstr) -> [{make_mfa(M, Name),Spec} || {attribute,_,spec,{Name,Spec}} <- Abstr]. @@ -646,6 +651,8 @@ erlang_halt(Config) when is_list(Config) -> {badrpc,nodedown} = rpc:call(N2, erlang, halt, [0]), {ok,N3} = slave:start(H, halt_node3), {badrpc,nodedown} = rpc:call(N3, erlang, halt, [0,[]]), + {ok,N4} = slave:start(H, halt_node4), + {badrpc,nodedown} = rpc:call(N4, erlang, halt, [lists:duplicate(300,$x)]), % This test triggers a segfault when dumping a crash dump % to make sure that we can handle it properly. diff --git a/erts/emulator/test/bs_construct_SUITE.erl b/erts/emulator/test/bs_construct_SUITE.erl index 941cb435f7..22a1c0b765 100644 --- a/erts/emulator/test/bs_construct_SUITE.erl +++ b/erts/emulator/test/bs_construct_SUITE.erl @@ -527,7 +527,7 @@ huge_float_check({'EXIT',{system_limit,_}}) -> ok; huge_float_check({'EXIT',{badarg,_}}) -> ok. huge_binary(Config) when is_list(Config) -> - ct:timetrap({seconds, 30}), + ct:timetrap({seconds, 60}), 16777216 = size(<<0:(id(1 bsl 26)),(-1):(id(1 bsl 26))>>), garbage_collect(), {Shift,Return} = case free_mem() of @@ -561,30 +561,13 @@ huge_binary(Config) when is_list(Config) -> end. free_mem() -> - Cmd = "uname; free", - Output = string:tokens(os:cmd(Cmd), "\n"), - io:format("Output from command ~p\n~p\n",[Cmd,Output]), - case Output of - [OS, ColumnNames, Values | _] -> - case string:str(OS,"Linux") of - 0 -> - io:format("Unknown OS\n",[]), - undefined; - _ -> - case {string:tokens(ColumnNames, " \t"), - string:tokens(Values, " \t")} of - {[_,_,"free"|_],["Mem:",_,_,FreeKb|_]} -> - list_to_integer(FreeKb) div 1024; - _ -> - io:format("Failed to parse output from 'free':\n",[]), - undefined - end - end; - _ -> - io:format("Too few lines in output\n",[]), - undefined + {ok,Apps} = application:ensure_all_started(os_mon), + Mem = memsup:get_system_memory_data(), + [ok = application:stop(App)||App <- Apps], + case proplists:get_value(free_memory,Mem) of + undefined -> undefined; + Val -> Val div 1024 end. - system_limit(Config) when is_list(Config) -> WordSize = erlang:system_info(wordsize), @@ -614,8 +597,7 @@ system_limit_32() -> {'EXIT',{system_limit,_}} = (catch <<42:536870912/unit:8>>), {'EXIT',{system_limit,_}} = (catch <<42:(id(536870912))/unit:8>>), {'EXIT',{system_limit,_}} = (catch <<0:(id(8)),42:536870912/unit:8>>), - {'EXIT',{system_limit,_}} = - (catch <<0:(id(8)),42:(id(536870912))/unit:8>>), + {'EXIT',{system_limit,_}} = (catch <<0:(id(8)),42:(id(536870912))/unit:8>>), %% The size would be silently truncated, resulting in a crash. {'EXIT',{system_limit,_}} = (catch <<0:(1 bsl 35)>>), @@ -627,16 +609,10 @@ system_limit_32() -> ok. badarg(Config) when is_list(Config) -> - {'EXIT',{badarg,_}} = - (catch <<0:(id(1 bsl 100)),0:(id(-1))>>), - {'EXIT',{badarg,_}} = - (catch <<0:(id(1 bsl 100)),0:(id(-(1 bsl 70)))>>), - {'EXIT',{badarg,_}} = - (catch <<0:(id(-(1 bsl 70))),0:(id(1 bsl 100))>>), - - {'EXIT',{badarg,_}} = - (catch <<(id(<<>>))/binary,0:(id(-(1 bsl 100)))>>), - + {'EXIT',{badarg,_}} = (catch <<0:(id(1 bsl 100)),0:(id(-1))>>), + {'EXIT',{badarg,_}} = (catch <<0:(id(1 bsl 100)),0:(id(-(1 bsl 70)))>>), + {'EXIT',{badarg,_}} = (catch <<0:(id(-(1 bsl 70))),0:(id(1 bsl 100))>>), + {'EXIT',{badarg,_}} = (catch <<(id(<<>>))/binary,0:(id(-(1 bsl 100)))>>), ok. copy_writable_binary(Config) when is_list(Config) -> diff --git a/erts/emulator/test/dirty_nif_SUITE.erl b/erts/emulator/test/dirty_nif_SUITE.erl index c3afbc0803..83b098a704 100644 --- a/erts/emulator/test/dirty_nif_SUITE.erl +++ b/erts/emulator/test/dirty_nif_SUITE.erl @@ -32,19 +32,23 @@ dirty_nif/1, dirty_nif_send/1, dirty_nif_exception/1, call_dirty_nif_exception/1, dirty_scheduler_exit/1, dirty_call_while_terminated/1, - dirty_heap_access/1]). + dirty_heap_access/1, dirty_process_info/1, + dirty_process_register/1, dirty_process_trace/1]). -define(nif_stub,nif_stub_error(?LINE)). suite() -> [{ct_hooks,[ts_install_cth]}]. -all() -> +all() -> [dirty_nif, dirty_nif_send, dirty_nif_exception, dirty_scheduler_exit, dirty_call_while_terminated, - dirty_heap_access]. + dirty_heap_access, + dirty_process_info, + dirty_process_register, + dirty_process_trace]. init_per_suite(Config) -> try erlang:system_info(dirty_cpu_schedulers) of @@ -187,7 +191,7 @@ dirty_call_while_terminated(Config) when is_list(Config) -> blipp:blupp(Bin) end, [monitor,link]), - receive {dirty_alive, Pid} -> ok end, + receive {dirty_alive, _Pid} -> ok end, {value, {BinAddr, 4711, 2}} = lists:keysearch(4711, 2, element(2, process_info(self(), @@ -241,7 +245,7 @@ dirty_heap_access(Config) when is_list(Config) -> end), {N, R} = access_dirty_heap(Dirty, RGL, 0, 0), receive - {Pid, Res} -> + {_Pid, Res} -> 1000 = length(Res), lists:foreach(fun (X) -> Ref = X end, Res) end, @@ -269,12 +273,123 @@ access_dirty_heap(Dirty, RGL, N, R) -> end) end. +%% These tests verify that processes that access a process executing a +%% dirty NIF where the main lock is needed for that access do not get +%% blocked. Each test passes its pid to dirty_sleeper, which sends a +%% 'ready' message when it's running on a dirty scheduler and just before +%% it starts a 6 second sleep. When it receives the message, it verifies +%% that access to the dirty process is as it expects. After the dirty +%% process finishes its 6 second sleep but before it returns from the dirty +%% scheduler, it sends a 'done' message. If the tester already received +%% that message, the test fails because it means attempting to access the +%% dirty process waited for that process to return to a regular scheduler, +%% so verify that we haven't received that message, and also verify that +%% the dirty process is still alive immediately after accessing it. +dirty_process_info(Config) when is_list(Config) -> + access_dirty_process( + Config, + fun() -> ok end, + fun(NifPid) -> + PI = process_info(NifPid), + {current_function,{?MODULE,dirty_sleeper,1}} = + lists:keyfind(current_function, 1, PI), + ok + end, + fun(_) -> ok end). + +dirty_process_register(Config) when is_list(Config) -> + access_dirty_process( + Config, + fun() -> ok end, + fun(NifPid) -> + register(test_dirty_process_register, NifPid), + NifPid = whereis(test_dirty_process_register), + unregister(test_dirty_process_register), + false = lists:member(test_dirty_process_register, + registered()), + ok + end, + fun(_) -> ok end). + +dirty_process_trace(Config) when is_list(Config) -> + access_dirty_process( + Config, + fun() -> + erlang:trace_pattern({?MODULE,dirty_sleeper,1}, + [{'_',[],[{return_trace}]}], + [local,meta]), + ok + end, + fun(NifPid) -> + erlang:trace(NifPid, true, [call,timestamp]), + ok + end, + fun(NifPid) -> + receive + done -> + receive + {trace_ts,NifPid,call,{?MODULE,dirty_sleeper,_},_} -> + ok + after + 0 -> + error(missing_trace_call_message) + end, + receive + {trace_ts,NifPid,return_from,{?MODULE,dirty_sleeper,1}, + ok,_} -> + ok + after + 100 -> + error(missing_trace_return_message) + end + after + 6500 -> + error(missing_done_message) + end, + ok + end). + %% %% Internal... %% +access_dirty_process(Config, Start, Test, Finish) -> + {ok, Node} = start_node(Config, ""), + [ok] = mcall(Node, + [fun() -> + Path = ?config(data_dir, Config), + Lib = atom_to_list(?MODULE), + ok = erlang:load_nif(filename:join(Path,Lib), []), + ok = test_dirty_process_access(Start, Test, Finish) + end]), + stop_node(Node), + ok. + +test_dirty_process_access(Start, Test, Finish) -> + ok = Start(), + Self = self(), + NifPid = spawn_link(fun() -> + ok = dirty_sleeper(Self) + end), + ok = receive + ready -> + ok = Test(NifPid), + receive + done -> + error(dirty_process_info_blocked) + after + 0 -> + true = erlang:is_process_alive(NifPid), + ok + end + after + 3000 -> + error(timeout) + end, + ok = Finish(NifPid). + receive_any() -> - receive M -> M end. + receive M -> M end. start_node(Config) -> start_node(Config, ""). @@ -314,13 +429,13 @@ mcall(Node, Funs) -> %% The NIFs: lib_loaded() -> false. -call_nif_schedule(_,_) -> ?nif_stub. call_dirty_nif(_,_,_) -> ?nif_stub. send_from_dirty_nif(_) -> ?nif_stub. call_dirty_nif_exception(_) -> ?nif_stub. call_dirty_nif_zero_args() -> ?nif_stub. dirty_call_while_terminated_nif(_) -> ?nif_stub. dirty_sleeper() -> ?nif_stub. +dirty_sleeper(_) -> ?nif_stub. dirty_heap_access_nif(_) -> ?nif_stub. nif_stub_error(Line) -> diff --git a/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c b/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c index 2013c88167..e38bececde 100644 --- a/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c +++ b/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c @@ -19,7 +19,9 @@ */ #include "erl_nif.h" #include <assert.h> -#ifndef __WIN32__ +#ifdef __WIN32__ +#include <windows.h> +#else #include <unistd.h> #endif @@ -146,12 +148,31 @@ static ERL_NIF_TERM call_dirty_nif_zero_args(ErlNifEnv* env, int argc, const ERL static ERL_NIF_TERM dirty_sleeper(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { + ErlNifPid pid; + ErlNifEnv* msg_env = NULL; + assert(enif_is_on_dirty_scheduler(env)); + + /* If we get a pid argument, it indicates a process involved in the + test wants a message from us. Prior to the sleep we send a 'ready' + message, and then after the sleep, send a 'done' message. */ + if (argc == 1 && enif_get_local_pid(env, argv[0], &pid)) { + msg_env = enif_alloc_env(); + enif_send(env, &pid, msg_env, enif_make_atom(msg_env, "ready")); + } + #ifdef __WIN32__ Sleep(6000); #else sleep(6); #endif + + if (argc == 1) { + assert(msg_env != NULL); + enif_send(env, &pid, msg_env, enif_make_atom(msg_env, "done")); + enif_free_env(msg_env); + } + return enif_make_atom(env, "ok"); } @@ -216,6 +237,7 @@ static ErlNifFunc nif_funcs[] = {"call_dirty_nif_exception", 1, call_dirty_nif_exception, ERL_NIF_DIRTY_JOB_IO_BOUND}, {"call_dirty_nif_zero_args", 0, call_dirty_nif_zero_args, ERL_NIF_DIRTY_JOB_CPU_BOUND}, {"dirty_sleeper", 0, dirty_sleeper, ERL_NIF_DIRTY_JOB_IO_BOUND}, + {"dirty_sleeper", 1, dirty_sleeper, ERL_NIF_DIRTY_JOB_CPU_BOUND}, {"dirty_call_while_terminated_nif", 1, dirty_call_while_terminated_nif, ERL_NIF_DIRTY_JOB_CPU_BOUND}, {"dirty_heap_access_nif", 1, dirty_heap_access_nif, ERL_NIF_DIRTY_JOB_CPU_BOUND} }; diff --git a/erts/etc/unix/etp-commands.in b/erts/etc/unix/etp-commands.in index 4dc24d68b4..8f9945c4b4 100644 --- a/erts/etc/unix/etp-commands.in +++ b/erts/etc/unix/etp-commands.in @@ -1657,8 +1657,29 @@ end define etp-proc-state-int # Args: int # - if ($arg0 & 0xff000000) - printf "GARBAGE | " + if ($arg0 & 0x80000000) + printf "GARBAGE<0x80000000> | " + end + if ($arg0 & 0x40000000) + printf "dirty-running-sys | " + end + if ($arg0 & 0x20000000) + printf "dirty-running | " + end + if ($arg0 & 0x10000000) + printf "dirty-active-sys | " + end + if ($arg0 & 0x8000000) + printf "dirty-io-proc | " + end + if ($arg0 & 0x4000000) + printf "dirty-cpu-proc | " + end + if ($arg0 & 0x2000000) + printf "on-heap-msgq | " + end + if ($arg0 & 0x1000000) + printf "off-heap-msgq | " end if ($arg0 & 0x800000) printf "delayed-sys | " diff --git a/erts/preloaded/ebin/erlang.beam b/erts/preloaded/ebin/erlang.beam Binary files differindex 12262bb5b0..cde8c9ab72 100644 --- a/erts/preloaded/ebin/erlang.beam +++ b/erts/preloaded/ebin/erlang.beam diff --git a/erts/preloaded/ebin/init.beam b/erts/preloaded/ebin/init.beam Binary files differindex 7b5797e90a..6fc95b914e 100644 --- a/erts/preloaded/ebin/init.beam +++ b/erts/preloaded/ebin/init.beam diff --git a/erts/preloaded/src/erlang.erl b/erts/preloaded/src/erlang.erl index f3f8696f32..94f3078173 100644 --- a/erts/preloaded/src/erlang.erl +++ b/erts/preloaded/src/erlang.erl @@ -980,14 +980,14 @@ group_leader(_GroupLeader, _Pid) -> %% Shadowed by erl_bif_types: erlang:halt/0 -spec halt() -> no_return(). halt() -> - erlang:nif_error(undefined). + erlang:halt(0, []). %% halt/1 %% Shadowed by erl_bif_types: erlang:halt/1 -spec halt(Status) -> no_return() when Status :: non_neg_integer() | 'abort' | string(). -halt(_Status) -> - erlang:nif_error(undefined). +halt(Status) -> + erlang:halt(Status, []). %% halt/2 %% Shadowed by erl_bif_types: erlang:halt/2 @@ -2578,6 +2578,7 @@ universaltime_to_localtime(_Universaltime) -> %%-------------------------------------------------------------------------- +%% Shadowed by erl_bif_types: erlang:apply/2 -spec apply(Fun, Args) -> term() when Fun :: function(), Args :: [term()]. diff --git a/erts/preloaded/src/erts.app.src b/erts/preloaded/src/erts.app.src index 98e0224a5f..e18da28905 100644 --- a/erts/preloaded/src/erts.app.src +++ b/erts/preloaded/src/erts.app.src @@ -37,7 +37,7 @@ {registered, []}, {applications, []}, {env, []}, - {runtime_dependencies, ["stdlib-2.5", "kernel-4.0", "sasl-2.4"]} + {runtime_dependencies, ["stdlib-3.0", "kernel-5.0", "sasl-3.0"]} ]}. %% vim: ft=erlang diff --git a/erts/preloaded/src/init.erl b/erts/preloaded/src/init.erl index 618b53f6bb..45468b3b9c 100644 --- a/erts/preloaded/src/init.erl +++ b/erts/preloaded/src/init.erl @@ -90,6 +90,7 @@ -define(ON_LOAD_HANDLER, init__boot__on_load_handler). + debug(false, _) -> ok; debug(_, T) -> erlang:display(T). @@ -173,7 +174,25 @@ stop() -> init ! {stop,stop}, ok. -spec stop(Status) -> 'ok' when Status :: non_neg_integer() | string(). -stop(Status) -> init ! {stop,{stop,Status}}, ok. +stop(Status) when is_integer(Status), Status >= 0 -> + stop_1(Status); +stop(Status) when is_list(Status) -> + case is_bytelist(Status) of + true -> + stop_1(Status); + false -> + erlang:error(badarg) + end; +stop(_) -> + erlang:error(badarg). + +is_bytelist([B|Bs]) when is_integer(B), B >= 0, B < 256 -> is_bytelist(Bs); +is_bytelist([]) -> true; +is_bytelist(_) -> false. + +%% Note that we check the type of Status beforehand to ensure that +%% the call to halt(Status) by the init process cannot fail +stop_1(Status) -> init ! {stop,{stop,Status}}, ok. -spec boot(BootArgs) -> no_return() when BootArgs :: [binary()]. @@ -285,21 +304,13 @@ things_to_string([]) -> "". halt_string(String, List) -> - HaltString = String ++ things_to_string(List), - if - length(HaltString)<199 -> HaltString; - true -> first198(HaltString, 198) - end. - -first198([H|T], N) when N>0 -> - [H|first198(T, N-1)]; -first198(_, 0) -> - []. + String ++ things_to_string(List). %% String = string() %% List = [string() | atom() | pid() | number()] %% Any other items in List, such as tuples, are ignored when creating %% the string used as argument to erlang:halt/1. +-spec crash(_, _) -> no_return(). crash(String, List) -> halt(halt_string(String, List)). |