diff options
Diffstat (limited to 'erts/emulator/beam')
64 files changed, 6434 insertions, 4226 deletions
diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index ada2d152b7..4bf1c13421 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -37,25 +37,83 @@ static void set_default_trace_pattern(Eterm module); static Eterm check_process_code(Process* rp, Module* modp); -static void delete_code(Process *c_p, ErtsProcLocks c_p_locks, Module* modp); -static void delete_export_references(Eterm module); -static int purge_module(int module); +static void delete_code(Module* modp); static void decrement_refc(BeamInstr* code); static int is_native(BeamInstr* code); static int any_heap_ref_ptrs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size); static int any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size); -static void remove_from_address_table(BeamInstr* code); -Eterm -load_module_2(BIF_ALIST_2) + + +BIF_RETTYPE code_is_module_native_1(BIF_ALIST_1) { - Eterm reason; - Eterm* hp; - int sz; - byte* code; + Module* modp; Eterm res; + ErtsCodeIndex code_ix; + + if (is_not_atom(BIF_ARG_1)) { + BIF_ERROR(BIF_P, BADARG); + } + code_ix = erts_active_code_ix(); + if ((modp = erts_get_module(BIF_ARG_1, code_ix)) == NULL) { + return am_undefined; + } + erts_rlock_old_code(code_ix); + res = ((modp->curr.code && is_native(modp->curr.code)) || + (modp->old.code != 0 && is_native(modp->old.code))) ? + am_true : am_false; + erts_runlock_old_code(code_ix); + return res; +} + +BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3) +{ + Module* modp; + Eterm res; + + if (!erts_try_seize_code_write_permission(BIF_P)) { + ERTS_BIF_YIELD3(bif_export[BIF_code_make_stub_module_3], + BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); + } + + erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_smp_thr_progress_block(); + + modp = erts_get_module(BIF_ARG_1, erts_active_code_ix()); + + if (modp && modp->curr.num_breakpoints > 0) { + ASSERT(modp->curr.code != NULL); + erts_clear_module_break(modp); + ASSERT(modp->curr.num_breakpoints == 0); + } + + erts_start_staging_code_ix(); + + res = erts_make_stub_module(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); + + if (res == BIF_ARG_1) { + erts_end_staging_code_ix(); + erts_commit_staging_code_ix(); + } + else { + erts_abort_staging_code_ix(); + } + erts_smp_thr_progress_unblock(); + erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_release_code_write_permission(); + return res; +} + +BIF_RETTYPE +prepare_loading_2(BIF_ALIST_2) +{ byte* temp_alloc = NULL; - struct LoaderState* stp; + byte* code; + Uint sz; + Binary* magic; + Eterm reason; + Eterm* hp; + Eterm res; if (is_not_atom(BIF_ARG_1)) { error: @@ -65,108 +123,277 @@ load_module_2(BIF_ALIST_2) if ((code = erts_get_aligned_binary_bytes(BIF_ARG_2, &temp_alloc)) == NULL) { goto error; } - hp = HAlloc(BIF_P, 3); - /* - * Read the BEAM file and prepare the module for loading. - */ - stp = erts_alloc_loader_state(); + magic = erts_alloc_loader_state(); sz = binary_size(BIF_ARG_2); - reason = erts_prepare_loading(stp, BIF_P, BIF_P->group_leader, + reason = erts_prepare_loading(magic, BIF_P, BIF_P->group_leader, &BIF_ARG_1, code, sz); erts_free_aligned_binary_bytes(temp_alloc); if (reason != NIL) { + hp = HAlloc(BIF_P, 3); res = TUPLE2(hp, am_error, reason); BIF_RET(res); } + hp = HAlloc(BIF_P, PROC_BIN_SIZE); + res = erts_mk_magic_binary_term(&hp, &MSO(BIF_P), magic); + erts_refc_dec(&magic->refc, 1); + BIF_RET(res); +} - /* - * Stop all other processes and finish the loading of the module. - */ - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_thr_progress_block(); +struct m { + Binary* code; + Eterm module; + Module* modp; + Uint exception; +}; - reason = erts_finish_loading(stp, BIF_P, 0, &BIF_ARG_1); - if (reason != NIL) { - res = TUPLE2(hp, am_error, reason); - } else { - set_default_trace_pattern(BIF_ARG_1); - res = TUPLE2(hp, am_module, BIF_ARG_1); - } +static Eterm staging_epilogue(Process* c_p, int, Eterm res, int, struct m*, int); - erts_smp_thr_progress_unblock(); - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); - BIF_RET(res); +static Eterm +exception_list(Process* p, Eterm tag, struct m* mp, Sint exceptions) +{ + Eterm* hp = HAlloc(p, 3 + 2*exceptions); + Eterm res = NIL; + + mp += exceptions - 1; + while (exceptions > 0) { + if (mp->exception) { + res = CONS(hp, mp->module, res); + hp += 2; + exceptions--; + } + mp--; + } + return TUPLE2(hp, tag, res); } -BIF_RETTYPE purge_module_1(BIF_ALIST_1) + +BIF_RETTYPE +finish_loading_1(BIF_ALIST_1) { - int purge_res; + int i; + int n; + struct m* p = NULL; + Uint exceptions; + Eterm res; + int is_blocking = 0; + int do_commit = 0; - if (is_not_atom(BIF_ARG_1)) { - BIF_ERROR(BIF_P, BADARG); + if (!erts_try_seize_code_write_permission(BIF_P)) { + ERTS_BIF_YIELD1(bif_export[BIF_finish_loading_1], BIF_P, BIF_ARG_1); } - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_thr_progress_block(); + /* + * Validate the argument before we start loading; it must be a + * proper list where each element is a magic binary containing + * prepared (not previously loaded) code. + * + * First count the number of elements and allocate an array + * to keep the elements in. + */ - erts_export_consolidate(); - purge_res = purge_module(atom_val(BIF_ARG_1)); + n = list_length(BIF_ARG_1); + if (n == -1) { + ERTS_BIF_PREP_ERROR(res, BIF_P, BADARG); + goto done; + } + p = erts_alloc(ERTS_ALC_T_LOADER_TMP, n*sizeof(struct m)); - erts_smp_thr_progress_unblock(); - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + /* + * We now know that the argument is a proper list. Validate + * and collect the binaries into the array. + */ - if (purge_res < 0) { - BIF_ERROR(BIF_P, BADARG); + for (i = 0; i < n; i++) { + Eterm* cons = list_val(BIF_ARG_1); + Eterm term = CAR(cons); + ProcBin* pb; + + if (!ERTS_TERM_IS_MAGIC_BINARY(term)) { + ERTS_BIF_PREP_ERROR(res, BIF_P, BADARG); + goto done; + } + pb = (ProcBin*) binary_val(term); + p[i].code = pb->val; + p[i].module = erts_module_for_prepared_code(p[i].code); + if (p[i].module == NIL) { + ERTS_BIF_PREP_ERROR(res, BIF_P, BADARG); + goto done; + } + BIF_ARG_1 = CDR(cons); } - BIF_RET(am_true); -} -BIF_RETTYPE code_is_module_native_1(BIF_ALIST_1) -{ - Module* modp; + /* + * Since we cannot handle atomic loading of a group of modules + * if one or more of them uses on_load, we will only allow one + * element in the list. This limitation is intended to be + * lifted in the future. + */ - if (is_not_atom(BIF_ARG_1)) { - BIF_ERROR(BIF_P, BADARG); + if (n > 1) { + ERTS_BIF_PREP_ERROR(res, BIF_P, SYSTEM_LIMIT); + goto done; } - if ((modp = erts_get_module(BIF_ARG_1)) == NULL) { - return am_undefined; + + /* + * All types are correct. There cannot be a BADARG from now on. + * Before we can start loading, we must check whether any of + * the modules already has old code. To avoid a race, we must + * not allow other process to initiate a code loading operation + * from now on. + */ + + res = am_ok; + erts_start_staging_code_ix(); + + for (i = 0; i < n; i++) { + p[i].modp = erts_put_module(p[i].module); + } + for (i = 0; i < n; i++) { + if (p[i].modp->curr.num_breakpoints > 0 || + p[i].modp->curr.num_traced_exports > 0 || + erts_is_default_trace_enabled()) { + /* tracing involved, fallback with thread blocking */ + erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_smp_thr_progress_block(); + is_blocking = 1; + break; + } } - return ((modp->code && is_native(modp->code)) || - (modp->old_code != 0 && is_native(modp->old_code))) ? - am_true : am_false; -} -BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3) -{ - Eterm res; + if (is_blocking) { + for (i = 0; i < n; i++) { + if (p[i].modp->curr.num_breakpoints) { + erts_clear_module_break(p[i].modp); + ASSERT(p[i].modp->curr.num_breakpoints == 0); + } + } + } - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_thr_progress_block(); + exceptions = 0; + for (i = 0; i < n; i++) { + p[i].exception = 0; + if (p[i].modp->curr.code && p[i].modp->old.code) { + p[i].exception = 1; + exceptions++; + } + } - erts_export_consolidate(); - res = erts_make_stub_module(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); + if (exceptions) { + res = exception_list(BIF_P, am_not_purged, p, exceptions); + } else { + /* + * Now we can load all code. This can't fail. + */ - erts_smp_thr_progress_unblock(); - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); - return res; + exceptions = 0; + for (i = 0; i < n; i++) { + Eterm mod; + Eterm retval; + + erts_refc_inc(&p[i].code->refc, 1); + retval = erts_finish_loading(p[i].code, BIF_P, 0, &mod); + ASSERT(retval == NIL || retval == am_on_load); + if (retval == am_on_load) { + p[i].exception = 1; + exceptions++; + } + } + + /* + * Check whether any module has an on_load_handler. + */ + + if (exceptions) { + res = exception_list(BIF_P, am_on_load, p, exceptions); + } + do_commit = 1; + } + +done: + return staging_epilogue(BIF_P, do_commit, res, is_blocking, p, n); +} + +static Eterm +staging_epilogue(Process* c_p, int commit, Eterm res, int is_blocking, + struct m* loaded, int nloaded) +{ +#ifdef ERTS_SMP + if (is_blocking || !commit) +#endif + { + if (commit) { + erts_end_staging_code_ix(); + erts_commit_staging_code_ix(); + if (loaded) { + int i; + for (i=0; i < nloaded; i++) { + set_default_trace_pattern(loaded[i].module); + } + } + } + else { + erts_abort_staging_code_ix(); + } + if (loaded) { + erts_free(ERTS_ALC_T_LOADER_TMP, loaded); + } + if (is_blocking) { + erts_smp_thr_progress_unblock(); + erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); + } + erts_release_code_write_permission(); + return res; + } +#ifdef ERTS_SMP + else { + ErtsThrPrgrVal later; + ASSERT(is_value(res)); + + if (loaded) { + erts_free(ERTS_ALC_T_LOADER_TMP, loaded); + } + erts_end_staging_code_ix(); + /* + * Now we must wait for all schedulers to do a memory barrier before + * we can activate and let them access the new staged code. This allows + * schedulers to read active code_ix in a safe way while executing + * without any memory barriers at all. + */ + + later = erts_thr_progress_later(); + erts_thr_progress_wakeup(c_p->scheduler_data, later); + erts_notify_code_ix_activation(c_p, later); + erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL); + /* + * handle_code_ix_activation() will do the rest "later" + * and resume this process to return 'res'. + */ + ERTS_BIF_YIELD_RETURN(c_p, res); + } +#endif } BIF_RETTYPE check_old_code_1(BIF_ALIST_1) { + ErtsCodeIndex code_ix; Module* modp; + Eterm res = am_false; if (is_not_atom(BIF_ARG_1)) { BIF_ERROR(BIF_P, BADARG); } - modp = erts_get_module(BIF_ARG_1); - if (modp == NULL) { /* Doesn't exist. */ - BIF_RET(am_false); - } else if (modp->old_code == NULL) { /* No old code. */ - BIF_RET(am_false); + code_ix = erts_active_code_ix(); + modp = erts_get_module(BIF_ARG_1, code_ix); + if (modp != NULL) { + erts_rlock_old_code(code_ix); + if (modp->old.code != NULL) { + res = am_true; + } + erts_runlock_old_code(code_ix); } - BIF_RET(am_true); + BIF_RET(res); } Eterm @@ -180,14 +407,20 @@ check_process_code_2(BIF_ALIST_2) } if (is_internal_pid(BIF_ARG_1)) { Eterm res; + ErtsCodeIndex code_ix; if (internal_pid_index(BIF_ARG_1) >= erts_max_processes) goto error; - modp = erts_get_module(BIF_ARG_2); + code_ix = erts_active_code_ix(); + modp = erts_get_module(BIF_ARG_2, code_ix); if (modp == NULL) { /* Doesn't exist. */ return am_false; - } else if (modp->old_code == NULL) { /* No old code. */ + } + erts_rlock_old_code(code_ix); + if (modp->old.code == NULL) { /* No old code. */ + erts_runlock_old_code(code_ix); return am_false; } + erts_runlock_old_code(code_ix); #ifdef ERTS_SMP rp = erts_pid2proc_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, @@ -202,7 +435,14 @@ check_process_code_2(BIF_ALIST_2) ERTS_BIF_YIELD2(bif_export[BIF_check_process_code_2], BIF_P, BIF_ARG_1, BIF_ARG_2); } - res = check_process_code(rp, modp); + erts_rlock_old_code(code_ix); + if (modp->old.code != NULL) { /* must check again */ + res = check_process_code(rp, modp); + } + else { + res = am_false; + } + erts_runlock_old_code(code_ix); #ifdef ERTS_SMP if (BIF_P != rp) { erts_resume(rp, ERTS_PROC_LOCK_MAIN); @@ -223,56 +463,71 @@ check_process_code_2(BIF_ALIST_2) BIF_RETTYPE delete_module_1(BIF_ALIST_1) { - int res; + ErtsCodeIndex code_ix; + Module* modp; + int is_blocking = 0; + int success = 0; + Eterm res = NIL; - if (is_not_atom(BIF_ARG_1)) - goto badarg; + if (is_not_atom(BIF_ARG_1)) { + BIF_ERROR(BIF_P, BADARG); + } - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_thr_progress_block(); + if (!erts_try_seize_code_write_permission(BIF_P)) { + ERTS_BIF_YIELD1(bif_export[BIF_delete_module_1], BIF_P, BIF_ARG_1); + } { - Module *modp = erts_get_module(BIF_ARG_1); + erts_start_staging_code_ix(); + code_ix = erts_staging_code_ix(); + modp = erts_get_module(BIF_ARG_1, code_ix); if (!modp) { res = am_undefined; } - else if (modp->old_code != 0) { + else if (modp->old.code != 0) { erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); erts_dsprintf(dsbufp, "Module %T must be purged before loading\n", BIF_ARG_1); erts_send_error_to_logger(BIF_P->group_leader, dsbufp); - res = am_badarg; + ERTS_BIF_PREP_ERROR(res, BIF_P, BADARG); } else { - delete_export_references(BIF_ARG_1); - delete_code(BIF_P, 0, modp); + if (modp->curr.num_breakpoints > 0 || + modp->curr.num_traced_exports > 0) { + /* we have tracing, retry single threaded */ + erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_smp_thr_progress_block(); + is_blocking = 1; + if (modp->curr.num_breakpoints) { + erts_clear_module_break(modp); + ASSERT(modp->curr.num_breakpoints == 0); + } + } + delete_code(modp); res = am_true; + success = 1; } } - - erts_smp_thr_progress_unblock(); - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); - - if (res == am_badarg) { - badarg: - BIF_ERROR(BIF_P, BADARG); - } - BIF_RET(res); + return staging_epilogue(BIF_P, success, res, is_blocking, NULL, 0); } BIF_RETTYPE module_loaded_1(BIF_ALIST_1) { Module* modp; + ErtsCodeIndex code_ix; + Eterm res = am_false; if (is_not_atom(BIF_ARG_1)) { BIF_ERROR(BIF_P, BADARG); } - if ((modp = erts_get_module(BIF_ARG_1)) == NULL || - modp->code == NULL || - modp->code[MI_ON_LOAD_FUNCTION_PTR] != 0) { - BIF_RET(am_false); + code_ix = erts_active_code_ix(); + if ((modp = erts_get_module(BIF_ARG_1, code_ix)) != NULL) { + if (modp->curr.code != NULL + && modp->curr.code[MI_ON_LOAD_FUNCTION_PTR] == 0) { + res = am_true; + } } - BIF_RET(am_true); + BIF_RET(res); } BIF_RETTYPE pre_loaded_0(BIF_ALIST_0) @@ -282,27 +537,28 @@ BIF_RETTYPE pre_loaded_0(BIF_ALIST_0) BIF_RETTYPE loaded_0(BIF_ALIST_0) { + ErtsCodeIndex code_ix = erts_active_code_ix(); + Module* modp; Eterm previous = NIL; Eterm* hp; int i; int j = 0; - - for (i = 0; i < module_code_size(); i++) { - if (module_code(i) != NULL && - ((module_code(i)->code_length != 0) || - (module_code(i)->old_code_length != 0))) { + + for (i = 0; i < module_code_size(code_ix); i++) { + if ((modp = module_code(i, code_ix)) != NULL && + ((modp->curr.code_length != 0) || + (modp->old.code_length != 0))) { j++; } } if (j > 0) { hp = HAlloc(BIF_P, j*2); - for (i = 0; i < module_code_size(); i++) { - if (module_code(i) != NULL && - ((module_code(i)->code_length != 0) || - (module_code(i)->old_code_length != 0))) { - previous = CONS(hp, make_atom(module_code(i)->module), - previous); + for (i = 0; i < module_code_size(code_ix); i++) { + if ((modp=module_code(i,code_ix)) != NULL && + ((modp->curr.code_length != 0) || + (modp->old.code_length != 0))) { + previous = CONS(hp, make_atom(modp->module), previous); hp += 2; } } @@ -312,54 +568,65 @@ BIF_RETTYPE loaded_0(BIF_ALIST_0) BIF_RETTYPE call_on_load_function_1(BIF_ALIST_1) { - Module* modp = erts_get_module(BIF_ARG_1); - Eterm on_load; + Module* modp = erts_get_module(BIF_ARG_1, erts_active_code_ix()); - if (!modp || modp->code == 0) { - error: - BIF_ERROR(BIF_P, BADARG); + if (modp && modp->curr.code) { + BIF_TRAP_CODE_PTR_0(BIF_P, modp->curr.code[MI_ON_LOAD_FUNCTION_PTR]); } - if ((on_load = modp->code[MI_ON_LOAD_FUNCTION_PTR]) == 0) { - goto error; + else { + BIF_ERROR(BIF_P, BADARG); } - BIF_TRAP_CODE_PTR_0(BIF_P, on_load); } BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) { - Module* modp = erts_get_module(BIF_ARG_1); + ErtsCodeIndex code_ix; + Module* modp; Eterm on_load; - if (!modp || modp->code == 0) { + if (!erts_try_seize_code_write_permission(BIF_P)) { + ERTS_BIF_YIELD2(bif_export[BIF_finish_after_on_load_2], + BIF_P, BIF_ARG_1, BIF_ARG_2); + } + + /* ToDo: Use code_ix staging instead of thread blocking */ + + erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_smp_thr_progress_block(); + + code_ix = erts_active_code_ix(); + modp = erts_get_module(BIF_ARG_1, code_ix); + + if (!modp || modp->curr.code == 0) { error: + erts_smp_thr_progress_unblock(); + erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_release_code_write_permission(); BIF_ERROR(BIF_P, BADARG); } - if ((on_load = modp->code[MI_ON_LOAD_FUNCTION_PTR]) == 0) { + if ((on_load = modp->curr.code[MI_ON_LOAD_FUNCTION_PTR]) == 0) { goto error; } if (BIF_ARG_2 != am_false && BIF_ARG_2 != am_true) { goto error; } - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_thr_progress_block(); - if (BIF_ARG_2 == am_true) { int i; /* * The on_load function succeded. Fix up export entries. */ - for (i = 0; i < export_list_size(); i++) { - Export *ep = export_list(i); + for (i = 0; i < export_list_size(code_ix); i++) { + Export *ep = export_list(i,code_ix); if (ep != NULL && ep->code[0] == BIF_ARG_1 && ep->code[4] != 0) { - ep->address = (void *) ep->code[4]; + ep->addressv[code_ix] = (void *) ep->code[4]; ep->code[4] = 0; } } - modp->code[MI_ON_LOAD_FUNCTION_PTR] = 0; + modp->curr.code[MI_ON_LOAD_FUNCTION_PTR] = 0; set_default_trace_pattern(BIF_ARG_1); } else if (BIF_ARG_2 == am_false) { BeamInstr* code; @@ -370,19 +637,21 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) * This is an combination of delete and purge. We purge * the current code; the old code is not touched. */ - erts_total_code_size -= modp->code_length; - code = modp->code; - end = (BeamInstr *)((char *)code + modp->code_length); + erts_total_code_size -= modp->curr.code_length; + code = modp->curr.code; + end = (BeamInstr *)((char *)code + modp->curr.code_length); erts_cleanup_funs_on_purge(code, end); - beam_catches_delmod(modp->catches, code, modp->code_length); + beam_catches_delmod(modp->curr.catches, code, modp->curr.code_length, + erts_active_code_ix()); erts_free(ERTS_ALC_T_CODE, (void *) code); - modp->code = NULL; - modp->code_length = 0; - modp->catches = BEAM_CATCHES_NIL; - remove_from_address_table(code); + modp->curr.code = NULL; + modp->curr.code_length = 0; + modp->curr.catches = BEAM_CATCHES_NIL; + erts_remove_from_ranges(code); } erts_smp_thr_progress_unblock(); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_release_code_write_permission(); BIF_RET(am_true); } @@ -427,10 +696,10 @@ check_process_code(Process* rp, Module* modp) /* * Pick up limits for the module. */ - start = modp->old_code; - end = (BeamInstr *)((char *)start + modp->old_code_length); + start = modp->old.code; + end = (BeamInstr *)((char *)start + modp->old.code_length); mod_start = (char *) start; - mod_size = modp->old_code_length; + mod_size = modp->old.code_length; /* * Check if current instruction or continuation pointer points into module. @@ -562,10 +831,10 @@ check_process_code(Process* rp, Module* modp) done_gc = 1; FLAGS(rp) |= F_NEED_FULLSWEEP; (void) erts_garbage_collect(rp, 0, rp->arg_reg, rp->arity); - literals = (Eterm *) modp->old_code[MI_LITERALS_START]; - lit_size = (Eterm *) modp->old_code[MI_LITERALS_END] - literals; + literals = (Eterm *) modp->old.code[MI_LITERALS_START]; + lit_size = (Eterm *) modp->old.code[MI_LITERALS_END] - literals; oh = (struct erl_off_heap_header *) - modp->old_code[MI_LITERALS_OFF_HEAP]; + modp->old.code[MI_LITERALS_OFF_HEAP]; erts_garbage_collect_literals(rp, literals, lit_size, oh); } } @@ -624,53 +893,82 @@ any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size) #undef in_area - -static int -purge_module(int module) +BIF_RETTYPE purge_module_1(BIF_ALIST_1) { + ErtsCodeIndex code_ix; BeamInstr* code; BeamInstr* end; Module* modp; + int is_blocking = 0; + Eterm ret; - /* - * Correct module? - */ - - if ((modp = erts_get_module(make_atom(module))) == NULL) { - return -2; + if (is_not_atom(BIF_ARG_1)) { + BIF_ERROR(BIF_P, BADARG); } - /* - * Any code to purge? - */ - if (modp->old_code == 0) { - return -1; + if (!erts_try_seize_code_write_permission(BIF_P)) { + ERTS_BIF_YIELD1(bif_export[BIF_purge_module_1], BIF_P, BIF_ARG_1); } + code_ix = erts_active_code_ix(); + /* - * Unload any NIF library + * Correct module? */ - if (modp->old_nif != NULL) { - erts_unload_nif(modp->old_nif); - modp->old_nif = NULL; + + if ((modp = erts_get_module(BIF_ARG_1, code_ix)) == NULL) { + ERTS_BIF_PREP_ERROR(ret, BIF_P, BADARG); } + else { + erts_rwlock_old_code(code_ix); - /* - * Remove the old code. - */ - ASSERT(erts_total_code_size >= modp->old_code_length); - erts_total_code_size -= modp->old_code_length; - code = modp->old_code; - end = (BeamInstr *)((char *)code + modp->old_code_length); - erts_cleanup_funs_on_purge(code, end); - beam_catches_delmod(modp->old_catches, code, modp->old_code_length); - decrement_refc(code); - erts_free(ERTS_ALC_T_CODE, (void *) code); - modp->old_code = NULL; - modp->old_code_length = 0; - modp->old_catches = BEAM_CATCHES_NIL; - remove_from_address_table(code); - return 0; + /* + * Any code to purge? + */ + if (modp->old.code == 0) { + ERTS_BIF_PREP_ERROR(ret, BIF_P, BADARG); + } + else { + /* + * Unload any NIF library + */ + if (modp->old.nif != NULL) { + /* ToDo: Do unload nif without blocking */ + erts_rwunlock_old_code(code_ix); + erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_smp_thr_progress_block(); + is_blocking = 1; + erts_rwlock_old_code(code_ix); + erts_unload_nif(modp->old.nif); + modp->old.nif = NULL; + } + + /* + * Remove the old code. + */ + ASSERT(erts_total_code_size >= modp->old.code_length); + erts_total_code_size -= modp->old.code_length; + code = modp->old.code; + end = (BeamInstr *)((char *)code + modp->old.code_length); + erts_cleanup_funs_on_purge(code, end); + beam_catches_delmod(modp->old.catches, code, modp->old.code_length, + code_ix); + decrement_refc(code); + erts_free(ERTS_ALC_T_CODE, (void *) code); + modp->old.code = NULL; + modp->old.code_length = 0; + modp->old.catches = BEAM_CATCHES_NIL; + erts_remove_from_ranges(code); + ERTS_BIF_PREP_RET(ret, am_true); + } + erts_rwunlock_old_code(code_ix); + } + if (is_blocking) { + erts_smp_thr_progress_unblock(); + erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + } + erts_release_code_write_permission(); + return ret; } static void @@ -690,92 +988,50 @@ decrement_refc(BeamInstr* code) } } -static void -remove_from_address_table(BeamInstr* code) -{ - int i; - - for (i = 0; i < num_loaded_modules; i++) { - if (modules[i].start == code) { - num_loaded_modules--; - while (i < num_loaded_modules) { - modules[i] = modules[i+1]; - i++; - } - mid_module = &modules[num_loaded_modules/2]; - return; - } - } - ASSERT(0); /* Not found? */ -} - - /* - * Move code from current to old. + * Move code from current to old and null all export entries for the module */ -static void -delete_code(Process *c_p, ErtsProcLocks c_p_locks, Module* modp) -{ -#ifdef ERTS_ENABLE_LOCK_CHECK -#ifdef ERTS_SMP - if (c_p && c_p_locks) - erts_proc_lc_chk_only_proc_main(c_p); - else -#endif - erts_lc_check_exact(NULL, 0); -#endif - - /* - * Clear breakpoints if any - */ - if (modp->code != NULL && modp->code[MI_NUM_BREAKPOINTS] > 0) { - if (c_p && c_p_locks) - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); - erts_smp_thr_progress_block(); - erts_clear_module_break(modp); - modp->code[MI_NUM_BREAKPOINTS] = 0; - erts_smp_thr_progress_unblock(); - if (c_p && c_p_locks) - erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); - } - modp->old_code = modp->code; - modp->old_code_length = modp->code_length; - modp->old_catches = modp->catches; - modp->old_nif = modp->nif; - modp->code = NULL; - modp->code_length = 0; - modp->catches = BEAM_CATCHES_NIL; - modp->nif = NULL; -} - - -/* null all references on the export table for the module called with the - atom index below */ - static void -delete_export_references(Eterm module) +delete_code(Module* modp) { + ErtsCodeIndex code_ix = erts_staging_code_ix(); + Eterm module = make_atom(modp->module); int i; - ASSERT(is_atom(module)); - - for (i = 0; i < export_list_size(); i++) { - Export *ep = export_list(i); + for (i = 0; i < export_list_size(code_ix); i++) { + Export *ep = export_list(i, code_ix); if (ep != NULL && (ep->code[0] == module)) { - if (ep->address == ep->code+3 && - (ep->code[3] == (BeamInstr) em_apply_bif)) { - continue; + if (ep->addressv[code_ix] == ep->code+3) { + if (ep->code[3] == (BeamInstr) em_apply_bif) { + continue; + } + else if (ep->code[3] == (BeamInstr) em_call_traced_function) { + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); + ASSERT(modp->curr.num_traced_exports > 0); + --modp->curr.num_traced_exports; + MatchSetUnref(ep->match_prog_set); + ep->match_prog_set = NULL; + } + else ASSERT(ep->code[3] == (BeamInstr) em_call_error_handler + || !erts_initialized); } - ep->address = ep->code+3; + ep->addressv[code_ix] = ep->code+3; ep->code[3] = (BeamInstr) em_call_error_handler; ep->code[4] = 0; - MatchSetUnref(ep->match_prog_set); - ep->match_prog_set = NULL; + ASSERT(ep->match_prog_set == NULL); } } + + ASSERT(modp->curr.num_breakpoints == 0); + ASSERT(modp->curr.num_traced_exports == 0); + modp->old = modp->curr; + modp->curr.code = NULL; + modp->curr.code_length = 0; + modp->curr.catches = BEAM_CATCHES_NIL; + modp->curr.nif = NULL; } - + Eterm beam_make_current_old(Process *c_p, ErtsProcLocks c_p_locks, Eterm module) @@ -787,11 +1043,10 @@ beam_make_current_old(Process *c_p, ErtsProcLocks c_p_locks, Eterm module) * if not, delete old code; error if old code already exists. */ - if (modp->code != NULL && modp->old_code != NULL) { + if (modp->curr.code != NULL && modp->old.code != NULL) { return am_not_purged; - } else if (modp->old_code == NULL) { /* Make the current version old. */ - delete_code(c_p, c_p_locks, modp); - delete_export_references(module); + } else if (modp->old.code == NULL) { /* Make the current version old. */ + delete_code(modp); } return NIL; } diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c index d772bea02f..26dadfbbc0 100644 --- a/erts/emulator/beam/beam_bp.c +++ b/erts/emulator/beam/beam_bp.c @@ -478,15 +478,16 @@ erts_find_local_func(Eterm mfa[3]) { BeamInstr* code_ptr; Uint i,n; - if ((modp = erts_get_module(mfa[0])) == NULL) + if ((modp = erts_get_module(mfa[0], erts_active_code_ix())) == NULL) return NULL; - if ((code_base = (BeamInstr **) modp->code) == NULL) + if ((code_base = (BeamInstr **) modp->curr.code) == NULL) return NULL; n = (BeamInstr) code_base[MI_NUM_FUNCTIONS]; for (i = 0; i < n; ++i) { code_ptr = code_base[MI_FUNCTIONS+i]; ASSERT(((BeamInstr) BeamOp(op_i_func_info_IaaI)) == code_ptr[0]); - ASSERT(mfa[0] == ((Eterm) code_ptr[2])); + ASSERT(mfa[0] == ((Eterm) code_ptr[2]) || + is_nil((Eterm) code_ptr[2])); if (mfa[1] == ((Eterm) code_ptr[3]) && ((BeamInstr) mfa[2]) == code_ptr[4]) { return code_ptr + 5; @@ -713,7 +714,7 @@ void erts_trace_time_break(Process *p, BeamInstr *pc, BpDataTime *bdt, Uint type BpDataTime *pbdt = NULL; ASSERT(p); - ASSERT(p->status == P_RUNNING); + ASSERT(ERTS_PSFLG_RUNNING & erts_smp_atomic32_read_acqb(&p->state)); /* get previous timestamp and breakpoint * from the process psd */ @@ -849,12 +850,13 @@ static int set_break(Eterm mfa[3], int specified, { Module *modp; int num_processed = 0; + ErtsCodeIndex code_ix = erts_active_code_ix(); if (!specified) { /* Find and process all modules in the system... */ int current; - int last = module_code_size(); + int last = module_code_size(code_ix); for (current = 0; current < last; current++) { - modp = module_code(current); + modp = module_code(current, code_ix); ASSERT(modp != NULL); num_processed += set_module_break(modp, mfa, specified, @@ -863,7 +865,7 @@ static int set_break(Eterm mfa[3], int specified, } } else { /* Process a single module */ - if ((modp = erts_get_module(mfa[0])) != NULL) { + if ((modp = erts_get_module(mfa[0], code_ix)) != NULL) { num_processed += set_module_break(modp, mfa, specified, match_spec, break_op, count_op, @@ -883,7 +885,7 @@ static int set_module_break(Module *modp, Eterm mfa[3], int specified, ASSERT(break_op); ASSERT(modp); - code_base = (BeamInstr **) modp->code; + code_base = (BeamInstr **) modp->curr.code; if (code_base == NULL) { return 0; } @@ -913,10 +915,10 @@ static int set_function_break(Module *modp, BeamInstr *pc, int bif, Uint ix = 0; if (bif == BREAK_IS_ERL) { - code_base = (BeamInstr **)modp->code; + code_base = (BeamInstr **)modp->curr.code; ASSERT(code_base); ASSERT(code_base <= (BeamInstr **)pc); - ASSERT((BeamInstr **)pc < code_base + (modp->code_length/sizeof(BeamInstr *))); + ASSERT((BeamInstr **)pc < code_base + (modp->curr.code_length/sizeof(BeamInstr *))); } else { ASSERT(*pc == (BeamInstr) em_apply_bif); ASSERT(modp == NULL); @@ -1097,29 +1099,30 @@ static int set_function_break(Module *modp, BeamInstr *pc, int bif, } if (bif == BREAK_IS_ERL) { - ++(*(BeamInstr*)&code_base[MI_NUM_BREAKPOINTS]); + ++modp->curr.num_breakpoints; } return 1; } static int clear_break(Eterm mfa[3], int specified, BeamInstr break_op) { + ErtsCodeIndex code_ix = erts_active_code_ix(); int num_processed = 0; Module *modp; if (!specified) { /* Iterate over all modules */ int current; - int last = module_code_size(); + int last = module_code_size(code_ix); for (current = 0; current < last; current++) { - modp = module_code(current); + modp = module_code(current, code_ix); ASSERT(modp != NULL); num_processed += clear_module_break(modp, mfa, specified, break_op); } } else { /* Process a single module */ - if ((modp = erts_get_module(mfa[0])) != NULL) { + if ((modp = erts_get_module(mfa[0], code_ix)) != NULL) { num_processed += clear_module_break(modp, mfa, specified, break_op); } @@ -1136,7 +1139,7 @@ static int clear_module_break(Module *m, Eterm mfa[3], int specified, BeamInstr n; ASSERT(m); - code_base = (BeamInstr **) m->code; + code_base = (BeamInstr **) m->curr.code; if (code_base == NULL) { return 0; } @@ -1160,10 +1163,10 @@ static int clear_function_break(Module *m, BeamInstr *pc, int bif, BeamInstr bre BeamInstr **code_base = NULL; if (bif == BREAK_IS_ERL) { - code_base = (BeamInstr **)m->code; + code_base = (BeamInstr **)m->curr.code; ASSERT(code_base); ASSERT(code_base <= (BeamInstr **)pc); - ASSERT((BeamInstr **)pc < code_base + (m->code_length/sizeof(BeamInstr *))); + ASSERT((BeamInstr **)pc < code_base + (m->curr.code_length/sizeof(BeamInstr *))); } else { ASSERT(*pc == (BeamInstr) em_apply_bif); ASSERT(m == NULL); @@ -1255,7 +1258,7 @@ static int clear_function_break(Module *m, BeamInstr *pc, int bif, BeamInstr bre for (j = 0; j < bdt->hash[i].n; ++j) { item = &(bdt->hash[i].item[j]); if (item->pid != NIL) { - h_p = process_tab[internal_pid_index(item->pid)]; + h_p = erts_proc_lookup(item->pid); if (h_p) { pbt = ERTS_PROC_SET_CALL_TIME(h_p, ERTS_PROC_LOCK_MAIN, NULL); if (pbt) { @@ -1273,8 +1276,8 @@ static int clear_function_break(Module *m, BeamInstr *pc, int bif, BeamInstr bre } Free(bd); if (bif == BREAK_IS_ERL) { - ASSERT(((BeamInstr) code_base[MI_NUM_BREAKPOINTS]) > 0); - --(*(BeamInstr*)&code_base[MI_NUM_BREAKPOINTS]); + ASSERT(m->curr.num_breakpoints > 0); + --m->curr.num_breakpoints; } if (*rs) { for (ix = 1; ix < erts_no_schedulers; ++ix) { diff --git a/erts/emulator/beam/beam_catches.c b/erts/emulator/beam/beam_catches.c index 406ef1db5f..92f7ffe5a2 100644 --- a/erts/emulator/beam/beam_catches.c +++ b/erts/emulator/beam/beam_catches.c @@ -31,78 +31,144 @@ typedef struct { unsigned cdr; } beam_catch_t; -static int free_list; -static unsigned high_mark; -static unsigned tabsize; -static beam_catch_t *beam_catches; +#ifdef DEBUG +# define IF_DEBUG(x) x +#else +# define IF_DEBUG(x) +#endif + +struct bc_pool { + int free_list; + unsigned high_mark; + unsigned tabsize; + beam_catch_t *beam_catches; + /* + * Note that the 'beam_catches' area is shared by pools. Used slots + * are readonly as long as the module is not purgable. The free-list is + * protected by the code_ix lock. + */ + + IF_DEBUG(int is_staging;) +}; + +static struct bc_pool bccix[ERTS_NUM_CODE_IX]; void beam_catches_init(void) { - tabsize = DEFAULT_TABSIZE; - free_list = -1; - high_mark = 0; + int i; + + bccix[0].tabsize = DEFAULT_TABSIZE; + bccix[0].free_list = -1; + bccix[0].high_mark = 0; + bccix[0].beam_catches = erts_alloc(ERTS_ALC_T_CODE, + sizeof(beam_catch_t)*DEFAULT_TABSIZE); + IF_DEBUG(bccix[0].is_staging = 0); + for (i=1; i<ERTS_NUM_CODE_IX; i++) { + bccix[i] = bccix[i-1]; + } + /* For initial load: */ + IF_DEBUG(bccix[erts_staging_code_ix()].is_staging = 1); +} - beam_catches = erts_alloc(ERTS_ALC_T_CODE, sizeof(beam_catch_t)*DEFAULT_TABSIZE); + +static void gc_old_vec(beam_catch_t* vec) +{ + int i; + for (i=0; i<ERTS_NUM_CODE_IX; i++) { + if (bccix[i].beam_catches == vec) { + return; + } + } + erts_free(ERTS_ALC_T_CODE, vec); +} + + +void beam_catches_start_staging(void) +{ + ErtsCodeIndex dst = erts_staging_code_ix(); + ErtsCodeIndex src = erts_active_code_ix(); + beam_catch_t* prev_vec = bccix[dst].beam_catches; + + ASSERT(!bccix[src].is_staging && !bccix[dst].is_staging); + + bccix[dst] = bccix[src]; + gc_old_vec(prev_vec); + IF_DEBUG(bccix[dst].is_staging = 1); +} + +void beam_catches_end_staging(int commit) +{ + IF_DEBUG(bccix[erts_staging_code_ix()].is_staging = 0); } unsigned beam_catches_cons(BeamInstr *cp, unsigned cdr) { int i; + struct bc_pool* p = &bccix[erts_staging_code_ix()]; + ASSERT(p->is_staging); /* * Allocate from free_list while it is non-empty. * If free_list is empty, allocate at high_mark. - * - * This avoids the need to initialise the free list in - * beam_catches_init(), which would cost O(TABSIZ) time. */ - if( free_list >= 0 ) { - i = free_list; - free_list = beam_catches[i].cdr; - } else if( high_mark < tabsize ) { - i = high_mark; - high_mark++; - } else { - /* No free slots and table is full: realloc table */ - tabsize = 2*tabsize; - beam_catches = erts_realloc(ERTS_ALC_T_CODE, beam_catches, sizeof(beam_catch_t)*tabsize); - i = high_mark; - high_mark++; + if (p->free_list >= 0) { + i = p->free_list; + p->free_list = p->beam_catches[i].cdr; + } + else { + if (p->high_mark >= p->tabsize) { + /* No free slots and table is full: realloc table */ + beam_catch_t* prev_vec = p->beam_catches; + unsigned newsize = p->tabsize*2; + + p->beam_catches = erts_alloc(ERTS_ALC_T_CODE, + newsize*sizeof(beam_catch_t)); + sys_memcpy(p->beam_catches, prev_vec, + p->tabsize*sizeof(beam_catch_t)); + gc_old_vec(prev_vec); + p->tabsize = newsize; + } + i = p->high_mark++; } - beam_catches[i].cp = cp; - beam_catches[i].cdr = cdr; + p->beam_catches[i].cp = cp; + p->beam_catches[i].cdr = cdr; return i; } BeamInstr *beam_catches_car(unsigned i) { - if( i >= tabsize ) { + struct bc_pool* p = &bccix[erts_active_code_ix()]; + + if (i >= p->tabsize ) { erl_exit(1, "beam_catches_delmod: index %#x is out of range\r\n", i); } - return beam_catches[i].cp; + return p->beam_catches[i].cp; } -void beam_catches_delmod(unsigned head, BeamInstr *code, unsigned code_bytes) +void beam_catches_delmod(unsigned head, BeamInstr *code, unsigned code_bytes, + ErtsCodeIndex code_ix) { + struct bc_pool* p = &bccix[code_ix]; unsigned i, cdr; + ASSERT((code_ix == erts_active_code_ix()) != bccix[erts_staging_code_ix()].is_staging); for(i = head; i != (unsigned)-1;) { - if( i >= tabsize ) { + if (i >= p->tabsize) { erl_exit(1, "beam_catches_delmod: index %#x is out of range\r\n", i); } - if( (char*)beam_catches[i].cp - (char*)code >= code_bytes ) { + if( (char*)p->beam_catches[i].cp - (char*)code >= code_bytes ) { erl_exit(1, "beam_catches_delmod: item %#x has cp %#lx which is not " "in module's range [%#lx,%#lx[\r\n", - i, (long)beam_catches[i].cp, + i, (long)p->beam_catches[i].cp, (long)code, (long)((char*)code + code_bytes)); } - beam_catches[i].cp = 0; - cdr = beam_catches[i].cdr; - beam_catches[i].cdr = free_list; - free_list = i; + p->beam_catches[i].cp = 0; + cdr = p->beam_catches[i].cdr; + p->beam_catches[i].cdr = p->free_list; + p->free_list = i; i = cdr; } } diff --git a/erts/emulator/beam/beam_catches.h b/erts/emulator/beam/beam_catches.h index 6223427f0d..b2bd2351a5 100644 --- a/erts/emulator/beam/beam_catches.h +++ b/erts/emulator/beam/beam_catches.h @@ -20,12 +20,21 @@ #ifndef __BEAM_CATCHES_H #define __BEAM_CATCHES_H +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif +#include "sys.h" +#include "code_ix.h" + #define BEAM_CATCHES_NIL (-1) void beam_catches_init(void); +void beam_catches_start_staging(void); +void beam_catches_end_staging(int commit); unsigned beam_catches_cons(BeamInstr* cp, unsigned cdr); BeamInstr *beam_catches_car(unsigned i); -void beam_catches_delmod(unsigned head, BeamInstr* code, unsigned code_bytes); +void beam_catches_delmod(unsigned head, BeamInstr* code, unsigned code_bytes, + ErtsCodeIndex); #define catch_pc(x) beam_catches_car(catch_val((x))) diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c index 8041c92162..e69cbc3048 100644 --- a/erts/emulator/beam/beam_debug.c +++ b/erts/emulator/beam/beam_debug.c @@ -114,6 +114,10 @@ erts_debug_breakpoint_2(BIF_ALIST_2) mfa[2] = signed_val(mfa[2]); } + if (!erts_try_seize_code_write_permission(BIF_P)) { + ERTS_BIF_YIELD2(bif_export[BIF_erts_debug_breakpoint_2], + BIF_P, BIF_ARG_1, BIF_ARG_2); + } erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN); erts_smp_thr_progress_block(); @@ -125,7 +129,7 @@ erts_debug_breakpoint_2(BIF_ALIST_2) erts_smp_thr_progress_unblock(); erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); - + erts_release_code_write_permission(); return res; error: @@ -207,6 +211,7 @@ erts_debug_disassemble_1(BIF_ALIST_1) BIF_RET(am_false); } } else if (is_tuple(addr)) { + ErtsCodeIndex code_ix; Module* modp; Eterm mod; Eterm name; @@ -225,14 +230,14 @@ erts_debug_disassemble_1(BIF_ALIST_1) goto error; } arity = signed_val(tp[3]); - modp = erts_get_module(mod); + code_ix = erts_active_code_ix(); + modp = erts_get_module(mod, code_ix); /* * Try the export entry first to allow disassembly of special functions * such as erts_debug:apply/4. Then search for it in the module. */ - - if ((ep = erts_find_function(mod, name, arity)) != NULL) { + if ((ep = erts_find_function(mod, name, arity, code_ix)) != NULL) { /* XXX: add "&& ep->address != ep->code+3" condition? * Consider a traced function. * Its ep will have ep->address == ep->code+3. @@ -241,9 +246,9 @@ erts_debug_disassemble_1(BIF_ALIST_1) * But this code_ptr will point to the start of the Export, * not the function's func_info instruction. BOOM !? */ - code_ptr = ((BeamInstr *) ep->address) - 5; + code_ptr = ((BeamInstr *) ep->addressv[code_ix]) - 5; funcinfo = code_ptr+2; - } else if (modp == NULL || (code_base = modp->code) == NULL) { + } else if (modp == NULL || (code_base = modp->curr.code) == NULL) { BIF_RET(am_undef); } else { n = code_base[MI_NUM_FUNCTIONS]; diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 6d3b15cd46..15a937289d 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -63,11 +63,7 @@ # define PROCESS_MAIN_CHK_LOCKS(P) \ do { \ if ((P)) { \ - erts_pix_lock_t *pix_lock__ = ERTS_PIX2PIXLOCK(internal_pid_index((P)->id));\ erts_proc_lc_chk_only_proc_main((P)); \ - erts_pix_lock(pix_lock__); \ - ASSERT(0 < (P)->lock.refc && (P)->lock.refc < erts_no_schedulers*5);\ - erts_pix_unlock(pix_lock__); \ } \ else \ erts_lc_check_exact(NULL, 0); \ @@ -495,7 +491,7 @@ extern int count_instructions; do { \ if (FCALLS > 0) { \ Eterm* dis_next; \ - SET_I(((Export *) Arg(0))->address); \ + SET_I(((Export *) Arg(0))->addressv[erts_active_code_ix()]); \ dis_next = (Eterm *) *I; \ FCALLS--; \ CHECK_ARGS(I); \ @@ -504,7 +500,7 @@ extern int count_instructions; && FCALLS > neg_o_reds) { \ goto save_calls1; \ } else { \ - SET_I(((Export *) Arg(0))->address); \ + SET_I(((Export *) Arg(0))->addressv[erts_active_code_ix()]); \ CHECK_ARGS(I); \ goto context_switch; \ } \ @@ -1507,7 +1503,7 @@ void process_main(void) */ #ifdef USE_VM_CALL_PROBES if (DTRACE_ENABLED(global_function_entry)) { - BeamInstr* fp = (BeamInstr *) (((Export *) Arg(0))->address); + BeamInstr* fp = (BeamInstr *) (((Export *) Arg(0))->addressv[erts_active_code_ix()]); DTRACE_GLOBAL_CALL(c_p, (Eterm)fp[-3], (Eterm)fp[-2], fp[-1]); } #endif @@ -1522,7 +1518,7 @@ void process_main(void) SET_CP(c_p, I+2); #ifdef USE_VM_CALL_PROBES if (DTRACE_ENABLED(global_function_entry)) { - BeamInstr* fp = (BeamInstr *) (((Export *) Arg(0))->address); + BeamInstr* fp = (BeamInstr *) (((Export *) Arg(0))->addressv[erts_active_code_ix()]); DTRACE_GLOBAL_CALL(c_p, (Eterm)fp[-3], (Eterm)fp[-2], fp[-1]); } #endif @@ -1535,7 +1531,7 @@ void process_main(void) OpCase(i_call_ext_only_e): #ifdef USE_VM_CALL_PROBES if (DTRACE_ENABLED(global_function_entry)) { - BeamInstr* fp = (BeamInstr *) (((Export *) Arg(0))->address); + BeamInstr* fp = (BeamInstr *) (((Export *) Arg(0))->addressv[erts_active_code_ix()]); DTRACE_GLOBAL_CALL(c_p, (Eterm)fp[-3], (Eterm)fp[-2], fp[-1]); } #endif @@ -1826,13 +1822,12 @@ void process_main(void) msgp = PEEK_MESSAGE(c_p); if (msgp) erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); - else { + else #endif + { SET_I((BeamInstr *) Arg(0)); Goto(*I); /* Jump to a wait or wait_timeout instruction */ -#ifdef ERTS_SMP } -#endif } ErtsMoveMsgAttachmentIntoProc(msgp, c_p, E, HTOP, FCALLS, { @@ -2061,11 +2056,11 @@ void process_main(void) OpCase(wait_f): wait2: { - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); c_p->i = (BeamInstr *) Arg(0); /* L1 */ SWAPOUT; c_p->arity = 0; - c_p->status = P_WAITING; + erts_smp_atomic32_read_band_relb(&c_p->state, ~ERTS_PSFLG_ACTIVE); + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); c_p->current = NULL; goto do_schedule; @@ -3144,10 +3139,6 @@ void process_main(void) c_p->arg_reg[0] = r(0); SWAPOUT; c_p->i = I; - erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_STATUS); - if (c_p->status != P_SUSPENDED) - erts_add_to_runq(c_p); - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS); goto do_schedule1; } @@ -5110,9 +5101,6 @@ void process_main(void) c_p->arity = 1; /* One living register (the 'true' return value) */ SWAPOUT; c_p->i = I + 1; /* Next instruction */ - erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_STATUS); - erts_add_to_runq(c_p); - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS); c_p->current = NULL; goto do_schedule; } @@ -5234,7 +5222,7 @@ void process_main(void) save_calls(c_p, (Export *) Arg(0)); - SET_I(((Export *) Arg(0))->address); + SET_I(((Export *) Arg(0))->addressv[erts_active_code_ix()]); dis_next = (Eterm *) *I; FCALLS--; @@ -5911,7 +5899,8 @@ call_error_handler(Process* p, BeamInstr* fi, Eterm* reg, Eterm func) /* * Search for the error_handler module. */ - ep = erts_find_function(erts_proc_get_error_handler(p), func, 3); + ep = erts_find_function(erts_proc_get_error_handler(p), func, 3, + erts_active_code_ix()); if (ep == NULL) { /* No error handler */ p->current = fi; p->freason = EXC_UNDEF; @@ -5941,7 +5930,7 @@ call_error_handler(Process* p, BeamInstr* fi, Eterm* reg, Eterm func) reg[0] = fi[0]; reg[1] = fi[1]; reg[2] = args; - return ep->address; + return ep->addressv[erts_active_code_ix()]; } @@ -5955,7 +5944,7 @@ apply_setup_error_handler(Process* p, Eterm module, Eterm function, Uint arity, * there is no error handler module. */ - if ((ep = erts_find_export_entry(erts_proc_get_error_handler(p), + if ((ep = erts_active_export_entry(erts_proc_get_error_handler(p), am_undefined_function, 3)) == NULL) { return NULL; } else { @@ -6062,7 +6051,7 @@ apply(Process* p, Eterm module, Eterm function, Eterm args, Eterm* reg) * Note: All BIFs have export entries; thus, no special case is needed. */ - if ((ep = erts_find_export_entry(module, function, arity)) == NULL) { + if ((ep = erts_active_export_entry(module, function, arity)) == NULL) { if ((ep = apply_setup_error_handler(p, module, function, arity, reg)) == NULL) goto error; } else if (ERTS_PROC_GET_SAVED_CALLS_BUF(p)) { save_calls(p, ep); @@ -6070,11 +6059,11 @@ apply(Process* p, Eterm module, Eterm function, Eterm args, Eterm* reg) #ifdef USE_VM_CALL_PROBES if (DTRACE_ENABLED(global_function_entry)) { - BeamInstr *fptr = (BeamInstr *) ep->address; + BeamInstr *fptr = (BeamInstr *) ep->addressv[erts_active_code_ix()]; DTRACE_GLOBAL_CALL(p, (Eterm)fptr[-3], (Eterm)fptr[-2], (Uint)fptr[-1]); } #endif - return ep->address; + return ep->addressv[erts_active_code_ix()]; } static BeamInstr* @@ -6116,7 +6105,7 @@ fixed_apply(Process* p, Eterm* reg, Uint arity) * Note: All BIFs have export entries; thus, no special case is needed. */ - if ((ep = erts_find_export_entry(module, function, arity)) == NULL) { + if ((ep = erts_active_export_entry(module, function, arity)) == NULL) { if ((ep = apply_setup_error_handler(p, module, function, arity, reg)) == NULL) goto error; } else if (ERTS_PROC_GET_SAVED_CALLS_BUF(p)) { @@ -6125,11 +6114,11 @@ fixed_apply(Process* p, Eterm* reg, Uint arity) #ifdef USE_VM_CALL_PROBES if (DTRACE_ENABLED(global_function_entry)) { - BeamInstr *fptr = (BeamInstr *) ep->address; + BeamInstr *fptr = (BeamInstr *) ep->addressv[erts_active_code_ix()]; DTRACE_GLOBAL_CALL(p, (Eterm)fptr[-3], (Eterm)fptr[-2], (Uint)fptr[-1]); } #endif - return ep->address; + return ep->addressv[erts_active_code_ix()]; } int @@ -6206,9 +6195,7 @@ erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* re */ erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS); ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p); - if (c_p->msg.len > 0) { - erts_add_to_runq(c_p); - } else { + if (!c_p->msg.len) { erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS); c_p->fvalue = NIL; PROCESS_MAIN_CHK_LOCKS(c_p); @@ -6216,14 +6203,12 @@ erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* re ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS); - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); #ifdef ERTS_SMP ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p); - if (c_p->msg.len > 0) - erts_add_to_runq(c_p); - else + if (!c_p->msg.len) #endif - c_p->status = P_WAITING; + erts_smp_atomic32_read_band_relb(&c_p->state, ~ERTS_PSFLG_ACTIVE); + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); } erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS); c_p->current = bif_export[BIF_hibernate_3]->code; @@ -6311,7 +6296,7 @@ call_fun(Process* p, /* Current process. */ Export* ep; Module* modp; Eterm module; - + ErtsCodeIndex code_ix = erts_active_code_ix(); /* * No arity. There is no module loaded that defines the fun, @@ -6319,9 +6304,9 @@ call_fun(Process* p, /* Current process. */ * representation (the module has never been loaded), * or the module defining the fun has been unloaded. */ - module = fe->module; - if ((modp = erts_get_module(module)) != NULL && modp->code != NULL) { + if ((modp = erts_get_module(module, code_ix)) != NULL + && modp->curr.code != NULL) { /* * There is a module loaded, but obviously the fun is not * defined in it. We must not call the error_handler @@ -6336,7 +6321,7 @@ call_fun(Process* p, /* Current process. */ */ ep = erts_find_function(erts_proc_get_error_handler(p), - am_undefined_lambda, 3); + am_undefined_lambda, 3, code_ix); if (ep == NULL) { /* No error handler */ p->current = NULL; p->freason = EXC_UNDEF; @@ -6346,7 +6331,7 @@ call_fun(Process* p, /* Current process. */ reg[1] = fun; reg[2] = args; reg[3] = NIL; - return ep->address; + return ep->addressv[erts_active_code_ix()]; } } } else if (is_export_header(hdr)) { @@ -6358,7 +6343,7 @@ call_fun(Process* p, /* Current process. */ if (arity == actual_arity) { DTRACE_GLOBAL_CALL(p, ep->code[0], ep->code[1], (Uint)ep->code[2]); - return ep->address; + return ep->addressv[erts_active_code_ix()]; } else { /* * Wrong arity. First build a list of the arguments. @@ -6409,8 +6394,8 @@ call_fun(Process* p, /* Current process. */ erts_send_warning_to_logger(p->group_leader, dsbufp); } - if ((ep = erts_find_export_entry(module, function, arity)) == NULL) { - ep = erts_find_export_entry(erts_proc_get_error_handler(p), + if ((ep = erts_active_export_entry(module, function, arity)) == NULL) { + ep = erts_active_export_entry(erts_proc_get_error_handler(p), am_undefined_function, 3); if (ep == NULL) { p->freason = EXC_UNDEF; @@ -6434,7 +6419,7 @@ call_fun(Process* p, /* Current process. */ reg[2] = args; } DTRACE_GLOBAL_CALL(p, module, function, arity); - return ep->address; + return ep->addressv[erts_active_code_ix()]; } else { badfun: p->current = NULL; @@ -6537,7 +6522,8 @@ erts_is_builtin(Eterm Mod, Eterm Name, int arity) if ((ep = export_get(&e)) == NULL) { return 0; } - return ep->address == ep->code+3 && (ep->code[3] == (BeamInstr) em_apply_bif); + return ep->addressv[erts_active_code_ix()] == ep->code+3 + && (ep->code[3] == (BeamInstr) em_apply_bif); } diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index dd788df6e4..d3f55a2ba4 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -352,27 +352,6 @@ typedef struct LoaderState { int loc_size; /* Size of location info in bytes (2/4) */ } LoaderState; -/* - * Layout of the line table. - */ - -#define MI_LINE_FNAME_PTR 0 -#define MI_LINE_LOC_TAB 1 -#define MI_LINE_LOC_SIZE 2 -#define MI_LINE_FUNC_TAB 3 - -#define LINE_INVALID_LOCATION (0) - -/* - * Macros for manipulating locations. - */ - -#define IS_VALID_LOCATION(File, Line) \ - ((unsigned) (File) < 255 && (unsigned) (Line) < ((1 << 24) - 1)) -#define MAKE_LOCATION(File, Line) (((File) << 24) | (Line)) -#define LOC_FILE(Loc) ((Loc) >> 24) -#define LOC_LINE(Loc) ((Loc) & ((1 << 24)-1)) - #define GetTagAndValue(Stp, Tag, Val) \ do { \ BeamInstr __w; \ @@ -496,7 +475,8 @@ typedef struct LoaderState { } while (0) -static void free_state(LoaderState* stp); +static void free_loader_state(Binary* magic); +static void loader_state_dtor(Binary* magic); static Eterm insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, Eterm group_leader, Eterm module, BeamInstr* code, Uint size); @@ -507,6 +487,7 @@ static int verify_chunks(LoaderState* stp); static int load_atom_table(LoaderState* stp); static int load_import_table(LoaderState* stp); static int read_export_table(LoaderState* stp); +static int is_bif(Eterm mod, Eterm func, unsigned arity); static int read_lambda_table(LoaderState* stp); static int read_literal_table(LoaderState* stp); static int read_line_table(LoaderState* stp); @@ -548,22 +529,9 @@ static Eterm native_addresses(Process* p, Eterm mod); int patch_funentries(Eterm Patchlist); int patch(Eterm Addresses, Uint fe); static int safe_mul(UWord a, UWord b, UWord* resp); -static void lookup_loc(FunctionInfo* fi, BeamInstr* pc, - BeamInstr* modp, int idx); - static int must_swap_floats; -/* - * The following variables keep a sorted list of address ranges for - * each module. It allows us to quickly find a function given an - * instruction pointer. - */ -Range* modules = NULL; /* Sorted lists of module addresses. */ -int num_loaded_modules; /* Number of loaded modules. */ -int allocated_modules; /* Number of slots allocated. */ -Range* mid_module = NULL; /* Cached search start point */ - Uint erts_total_code_size; /**********************************************************************/ @@ -579,11 +547,7 @@ void init_load(void) f.fd = 1.0; must_swap_floats = (f.fw[0] == 0); - allocated_modules = 128; - modules = (Range *) erts_alloc(ERTS_ALC_T_MODULE_REFS, - allocated_modules*sizeof(Range)); - mid_module = modules; - num_loaded_modules = 0; + erts_init_ranges(); } static void @@ -595,7 +559,7 @@ define_file(LoaderState* stp, char* name, int idx) } Eterm -erts_load_module(Process *c_p, +erts_preload_module(Process *c_p, ErtsProcLocks c_p_locks, Eterm group_leader, /* Group leader or NIL if none. */ Eterm* modp, /* @@ -605,15 +569,16 @@ erts_load_module(Process *c_p, byte* code, /* Points to the code to load */ Uint size) /* Size of code to load. */ { - LoaderState* stp = erts_alloc_loader_state(); + Binary* magic = erts_alloc_loader_state(); Eterm retval; - retval = erts_prepare_loading(stp, c_p, group_leader, modp, + ASSERT(!erts_initialized); + retval = erts_prepare_loading(magic, c_p, group_leader, modp, code, size); if (retval != NIL) { return retval; } - return erts_finish_loading(stp, c_p, c_p_locks, modp); + return erts_finish_loading(magic, c_p, c_p_locks, modp); } /* #define LOAD_MEMORY_HARD_DEBUG 1*/ @@ -629,11 +594,13 @@ extern void check_allocated_block(Uint type, void *blk); #endif Eterm -erts_prepare_loading(LoaderState* stp, Process *c_p, Eterm group_leader, +erts_prepare_loading(Binary* magic, Process *c_p, Eterm group_leader, Eterm* modp, byte* code, Uint unloaded_size) { Eterm retval = am_badfile; + LoaderState* stp; + stp = ERTS_MAGIC_BIN_DATA(magic); stp->module = *modp; stp->group_leader = group_leader; @@ -666,7 +633,7 @@ erts_prepare_loading(LoaderState* stp, Process *c_p, Eterm group_leader, /* * Initialize code area. */ - stp->code_buffer_size = erts_next_heap_size(2048 + stp->num_functions, 0); + stp->code_buffer_size = 2048 + stp->num_functions; stp->code = (BeamInstr *) erts_alloc(ERTS_ALC_T_CODE, sizeof(BeamInstr) * stp->code_buffer_size); @@ -679,8 +646,6 @@ erts_prepare_loading(LoaderState* stp, Process *c_p, Eterm group_leader, stp->code[MI_COMPILE_PTR] = 0; stp->code[MI_COMPILE_SIZE] = 0; stp->code[MI_COMPILE_SIZE_ON_HEAP] = 0; - stp->code[MI_NUM_BREAKPOINTS] = 0; - /* * Read the atom table. @@ -774,23 +739,24 @@ erts_prepare_loading(LoaderState* stp, Process *c_p, Eterm group_leader, load_error: if (retval != NIL) { - free_state(stp); + free_loader_state(magic); } return retval; } Eterm -erts_finish_loading(LoaderState* stp, Process* c_p, +erts_finish_loading(Binary* magic, Process* c_p, ErtsProcLocks c_p_locks, Eterm* modp) { Eterm retval; + LoaderState* stp = ERTS_MAGIC_BIN_DATA(magic); /* * No other process may run since we will update the export * table which is not protected by any locks. */ - ERTS_SMP_LC_ASSERT(erts_initialized == 0 || + ERTS_SMP_LC_ASSERT(erts_initialized == 0 || erts_is_code_ix_locked() || erts_smp_thr_progress_is_blocking()); /* @@ -811,7 +777,6 @@ erts_finish_loading(LoaderState* stp, Process* c_p, * exported and imported functions. This can't fail. */ - erts_export_consolidate(); CHKBLK(ERTS_ALC_T_CODE,stp->code); final_touch(stp); @@ -837,16 +802,20 @@ erts_finish_loading(LoaderState* stp, Process* c_p, } load_error: - free_state(stp); + free_loader_state(magic); return retval; } -LoaderState* +Binary* erts_alloc_loader_state(void) { LoaderState* stp; + Binary* magic; - stp = erts_alloc(ERTS_ALC_T_LOADER_TMP, sizeof(LoaderState)); + magic = erts_create_magic_binary(sizeof(LoaderState), + loader_state_dtor); + erts_refc_inc(&magic->refc, 1); + stp = ERTS_MAGIC_BIN_DATA(magic); stp->bin = NULL; stp->function = THE_NON_VALUE; /* Function not known yet */ stp->arity = 0; @@ -875,76 +844,123 @@ erts_alloc_loader_state(void) stp->line_instr = 0; stp->func_line = 0; stp->fname = 0; - return stp; + return magic; +} + +/* + * Return the module name (a tagged atom) for the prepared code + * in the magic binary, or NIL if the binary does not contain + * prepared code. + */ +Eterm +erts_module_for_prepared_code(Binary* magic) +{ + LoaderState* stp; + + if (ERTS_MAGIC_BIN_DESTRUCTOR(magic) != loader_state_dtor) { + return NIL; + } + stp = ERTS_MAGIC_BIN_DATA(magic); + if (stp->code != 0) { + return stp->module; + } else { + return NIL; + } } static void -free_state(LoaderState* stp) +free_loader_state(Binary* magic) { + loader_state_dtor(magic); + if (erts_refc_dectest(&magic->refc, 0) == 0) { + erts_bin_free(magic); + } +} + +/* + * This destructor function can safely be called multiple times. + */ +static void +loader_state_dtor(Binary* magic) +{ + LoaderState* stp = ERTS_MAGIC_BIN_DATA(magic); + if (stp->bin != 0) { driver_free_binary(stp->bin); + stp->bin = 0; } if (stp->code != 0) { erts_free(ERTS_ALC_T_CODE, stp->code); + stp->code = 0; } - if (stp->labels != NULL) { - erts_free(ERTS_ALC_T_LOADER_TMP, (void *) stp->labels); + if (stp->labels != 0) { + erts_free(ERTS_ALC_T_PREPARED_CODE, (void *) stp->labels); + stp->labels = 0; } - if (stp->atom != NULL) { - erts_free(ERTS_ALC_T_LOADER_TMP, (void *) stp->atom); + if (stp->atom != 0) { + erts_free(ERTS_ALC_T_PREPARED_CODE, (void *) stp->atom); + stp->atom = 0; } - if (stp->import != NULL) { - erts_free(ERTS_ALC_T_LOADER_TMP, (void *) stp->import); + if (stp->import != 0) { + erts_free(ERTS_ALC_T_PREPARED_CODE, (void *) stp->import); + stp->import = 0; } - if (stp->export != NULL) { - erts_free(ERTS_ALC_T_LOADER_TMP, (void *) stp->export); + if (stp->export != 0) { + erts_free(ERTS_ALC_T_PREPARED_CODE, (void *) stp->export); + stp->export = 0; } if (stp->lambdas != stp->def_lambdas) { - erts_free(ERTS_ALC_T_LOADER_TMP, (void *) stp->lambdas); + erts_free(ERTS_ALC_T_PREPARED_CODE, (void *) stp->lambdas); + stp->lambdas = stp->def_lambdas; } - if (stp->literals != NULL) { + if (stp->literals != 0) { int i; for (i = 0; i < stp->num_literals; i++) { - if (stp->literals[i].heap != NULL) { - erts_free(ERTS_ALC_T_LOADER_TMP, + if (stp->literals[i].heap != 0) { + erts_free(ERTS_ALC_T_PREPARED_CODE, (void *) stp->literals[i].heap); + stp->literals[i].heap = 0; } } - erts_free(ERTS_ALC_T_LOADER_TMP, (void *) stp->literals); + erts_free(ERTS_ALC_T_PREPARED_CODE, (void *) stp->literals); + stp->literals = 0; } - while (stp->literal_patches != NULL) { + while (stp->literal_patches != 0) { LiteralPatch* next = stp->literal_patches->next; - erts_free(ERTS_ALC_T_LOADER_TMP, (void *) stp->literal_patches); + erts_free(ERTS_ALC_T_PREPARED_CODE, (void *) stp->literal_patches); stp->literal_patches = next; } - while (stp->string_patches != NULL) { + while (stp->string_patches != 0) { StringPatch* next = stp->string_patches->next; - erts_free(ERTS_ALC_T_LOADER_TMP, (void *) stp->string_patches); + erts_free(ERTS_ALC_T_PREPARED_CODE, (void *) stp->string_patches); stp->string_patches = next; } - while (stp->genop_blocks) { - GenOpBlock* next = stp->genop_blocks->next; - erts_free(ERTS_ALC_T_LOADER_TMP, (void *) stp->genop_blocks); - stp->genop_blocks = next; - } if (stp->line_item != 0) { - erts_free(ERTS_ALC_T_LOADER_TMP, stp->line_item); + erts_free(ERTS_ALC_T_PREPARED_CODE, stp->line_item); + stp->line_item = 0; } if (stp->line_instr != 0) { - erts_free(ERTS_ALC_T_LOADER_TMP, stp->line_instr); + erts_free(ERTS_ALC_T_PREPARED_CODE, stp->line_instr); + stp->line_instr = 0; } if (stp->func_line != 0) { - erts_free(ERTS_ALC_T_LOADER_TMP, stp->func_line); + erts_free(ERTS_ALC_T_PREPARED_CODE, stp->func_line); + stp->func_line = 0; } if (stp->fname != 0) { - erts_free(ERTS_ALC_T_LOADER_TMP, stp->fname); + erts_free(ERTS_ALC_T_PREPARED_CODE, stp->fname); + stp->fname = 0; } - erts_free(ERTS_ALC_T_LOADER_TMP, stp); + /* + * The following data items should have been freed earlier. + */ + + ASSERT(stp->genop_blocks == 0); } static Eterm @@ -954,7 +970,6 @@ insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, { Module* modp; Eterm retval; - int i; if ((retval = beam_make_current_old(c_p, c_p_locks, module)) != NIL) { erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); @@ -971,30 +986,15 @@ insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, erts_total_code_size += size; modp = erts_put_module(module); - modp->code = code; - modp->code_length = size; - modp->catches = BEAM_CATCHES_NIL; /* Will be filled in later. */ + modp->curr.code = code; + modp->curr.code_length = size; + modp->curr.catches = BEAM_CATCHES_NIL; /* Will be filled in later. */ /* - * Update address table (used for finding a function from a PC value). + * Update ranges (used for finding a function from a PC value). */ - if (num_loaded_modules == allocated_modules) { - allocated_modules *= 2; - modules = (Range *) erts_realloc(ERTS_ALC_T_MODULE_REFS, - (void *) modules, - allocated_modules * sizeof(Range)); - } - for (i = num_loaded_modules; i > 0; i--) { - if (code > modules[i-1].start) { - break; - } - modules[i] = modules[i-1]; - } - modules[i].start = code; - modules[i].end = (BeamInstr *) (((byte *)code) + size); - num_loaded_modules++; - mid_module = &modules[num_loaded_modules/2]; + erts_update_ranges(code, size); return NIL; } @@ -1217,9 +1217,8 @@ load_atom_table(LoaderState* stp) GetInt(stp, 4, stp->num_atoms); stp->num_atoms++; - stp->atom = erts_alloc(ERTS_ALC_T_LOADER_TMP, - erts_next_heap_size((stp->num_atoms*sizeof(Eterm)), - 0)); + stp->atom = erts_alloc(ERTS_ALC_T_PREPARED_CODE, + stp->num_atoms*sizeof(Eterm)); /* * Read all atoms. @@ -1263,10 +1262,8 @@ load_import_table(LoaderState* stp) int i; GetInt(stp, 4, stp->num_imports); - stp->import = erts_alloc(ERTS_ALC_T_LOADER_TMP, - erts_next_heap_size((stp->num_imports * - sizeof(ImportEntry)), - 0)); + stp->import = erts_alloc(ERTS_ALC_T_PREPARED_CODE, + stp->num_imports * sizeof(ImportEntry)); for (i = 0; i < stp->num_imports; i++) { int n; Eterm mod; @@ -1296,7 +1293,7 @@ load_import_table(LoaderState* stp) * If the export entry refers to a BIF, get the pointer to * the BIF function. */ - if ((e = erts_find_export_entry(mod, func, arity)) != NULL) { + if ((e = erts_active_export_entry(mod, func, arity)) != NULL) { if (e->code[3] == (BeamInstr) em_apply_bif) { stp->import[i].bf = (BifFunction) e->code[4]; if (func == am_load_nif && mod == am_erlang && arity == 2) { @@ -1315,16 +1312,8 @@ load_import_table(LoaderState* stp) static int read_export_table(LoaderState* stp) { - static struct { - Eterm mod; - Eterm func; - int arity; - } allow_redef[] = { - /* The BIFs that are allowed to be redefined by Erlang code */ - {am_erlang,am_apply,2}, - {am_erlang,am_apply,3}, - }; int i; + BeamInstr* address; GetInt(stp, 4, stp->num_exps); if (stp->num_exps > stp->num_functions) { @@ -1332,7 +1321,7 @@ read_export_table(LoaderState* stp) stp->num_exps, stp->num_functions); } stp->export - = (ExportEntry *) erts_alloc(ERTS_ALC_T_LOADER_TMP, + = (ExportEntry *) erts_alloc(ERTS_ALC_T_PREPARED_CODE, (stp->num_exps * sizeof(ExportEntry))); for (i = 0; i < stp->num_exps; i++) { @@ -1340,7 +1329,6 @@ read_export_table(LoaderState* stp) Uint value; Eterm func; Uint arity; - Export* e; GetInt(stp, 4, n); GetAtom(stp, n, func); @@ -1358,29 +1346,34 @@ read_export_table(LoaderState* stp) if (value == 0) { LoadError2(stp, "export table entry %d: label %d not resolved", i, n); } - stp->export[i].address = stp->code + value; + stp->export[i].address = address = stp->code + value; /* - * Check that we are not redefining a BIF (except the ones allowed to - * redefine). + * Find out if there is a BIF with the same name. */ - if ((e = erts_find_export_entry(stp->module, func, arity)) != NULL) { - if (e->code[3] == (BeamInstr) em_apply_bif) { - int j; - for (j = 0; j < sizeof(allow_redef)/sizeof(allow_redef[0]); j++) { - if (stp->module == allow_redef[j].mod && - func == allow_redef[j].func && - arity == allow_redef[j].arity) { - break; - } - } - if (j == sizeof(allow_redef)/sizeof(allow_redef[0])) { - LoadError2(stp, "exported function %T/%d redefines BIF", - func, arity); - } - } + if (!is_bif(stp->module, func, arity)) { + continue; + } + + /* + * This is a stub for a BIF. + * + * It should not be exported, and the information in its + * func_info instruction should be invalidated so that it + * can be filtered out by module_info(functions) and by + * any other functions that walk through all local functions. + */ + + if (stp->labels[n].patches) { + LoadError3(stp, "there are local calls to the stub for " + "the BIF %T:%T/%d", + stp->module, func, arity); } + stp->export[i].address = NULL; + address[-1] = 0; + address[-2] = NIL; + address[-3] = NIL; } return 1; @@ -1388,15 +1381,39 @@ read_export_table(LoaderState* stp) return 0; } + +static int +is_bif(Eterm mod, Eterm func, unsigned arity) +{ + Export* e = erts_active_export_entry(mod, func, arity); + if (e == NULL) { + return 0; + } + if (e->code[3] != (BeamInstr) em_apply_bif) { + return 0; + } + if (mod == am_erlang && func == am_apply && arity == 3) { + /* + * erlang:apply/3 is a special case -- it is implemented + * as an instruction and it is OK to redefine it. + */ + return 0; + } + return 1; +} + static int read_lambda_table(LoaderState* stp) { int i; GetInt(stp, 4, stp->num_lambdas); - stp->lambdas_allocated = stp->num_lambdas; - stp->lambdas = (Lambda *) erts_alloc(ERTS_ALC_T_LOADER_TMP, - stp->num_lambdas * sizeof(Lambda)); + if (stp->num_lambdas > stp->lambdas_allocated) { + ASSERT(stp->lambdas == stp->def_lambdas); + stp->lambdas_allocated = stp->num_lambdas; + stp->lambdas = (Lambda *) erts_alloc(ERTS_ALC_T_PREPARED_CODE, + stp->num_lambdas * sizeof(Lambda)); + } for (i = 0; i < stp->num_lambdas; i++) { Uint n; Uint32 Index; @@ -1446,7 +1463,7 @@ read_literal_table(LoaderState* stp) stp->file_p = uncompressed; stp->file_left = (unsigned) uncompressed_sz; GetInt(stp, 4, stp->num_literals); - stp->literals = (Literal *) erts_alloc(ERTS_ALC_T_LOADER_TMP, + stp->literals = (Literal *) erts_alloc(ERTS_ALC_T_PREPARED_CODE, stp->num_literals * sizeof(Literal)); stp->allocated_literals = stp->num_literals; @@ -1466,7 +1483,7 @@ read_literal_table(LoaderState* stp) if ((heap_size = erts_decode_ext_size(p, sz)) < 0) { LoadError1(stp, "literal %d: bad external format", i); } - hp = stp->literals[i].heap = erts_alloc(ERTS_ALC_T_LOADER_TMP, + hp = stp->literals[i].heap = erts_alloc(ERTS_ALC_T_PREPARED_CODE, heap_size*sizeof(Eterm)); stp->literals[i].off_heap.first = 0; stp->literals[i].off_heap.overhead = 0; @@ -1538,7 +1555,7 @@ read_line_table(LoaderState* stp) */ num_line_items++; - lp = (BeamInstr *) erts_alloc(ERTS_ALC_T_LOADER_TMP, + lp = (BeamInstr *) erts_alloc(ERTS_ALC_T_PREPARED_CODE, num_line_items * sizeof(BeamInstr)); stp->line_item = lp; stp->num_line_items = num_line_items; @@ -1594,7 +1611,7 @@ read_line_table(LoaderState* stp) */ if (stp->num_fnames != 0) { - stp->fname = (Eterm *) erts_alloc(ERTS_ALC_T_LOADER_TMP, + stp->fname = (Eterm *) erts_alloc(ERTS_ALC_T_PREPARED_CODE, stp->num_fnames * sizeof(Eterm)); for (i = 0; i < stp->num_fnames; i++) { @@ -1610,11 +1627,11 @@ read_line_table(LoaderState* stp) /* * Allocate the arrays to be filled while code is being loaded. */ - stp->line_instr = (LineInstr *) erts_alloc(ERTS_ALC_T_LOADER_TMP, + stp->line_instr = (LineInstr *) erts_alloc(ERTS_ALC_T_PREPARED_CODE, stp->num_line_instrs * sizeof(LineInstr)); stp->current_li = 0; - stp->func_line = (int *) erts_alloc(ERTS_ALC_T_LOADER_TMP, + stp->func_line = (int *) erts_alloc(ERTS_ALC_T_PREPARED_CODE, stp->num_functions * sizeof(int)); @@ -1677,7 +1694,7 @@ read_code_header(LoaderState* stp) * Initialize label table. */ - stp->labels = (Label *) erts_alloc(ERTS_ALC_T_LOADER_TMP, + stp->labels = (Label *) erts_alloc(ERTS_ALC_T_PREPARED_CODE, stp->num_labels * sizeof(Label)); for (i = 0; i < stp->num_labels; i++) { stp->labels[i].value = 0; @@ -1703,9 +1720,9 @@ read_code_header(LoaderState* stp) #define CodeNeed(w) do { \ ASSERT(ci <= code_buffer_size); \ if (code_buffer_size < ci+(w)) { \ - code_buffer_size = erts_next_heap_size(ci+(w), 0); \ - stp->code = code \ - = (BeamInstr *) erts_realloc(ERTS_ALC_T_CODE, \ + code_buffer_size = 2*ci+(w); \ + stp->code = code = \ + (BeamInstr *) erts_realloc(ERTS_ALC_T_CODE, \ (void *) code, \ code_buffer_size * sizeof(BeamInstr)); \ } \ @@ -1731,6 +1748,7 @@ load_code(LoaderState* stp) GenOp* last_op = NULL; GenOp** last_op_next = NULL; int arity; + int retval = 1; /* * The size of the loaded func_info instruction is needed @@ -2457,7 +2475,11 @@ load_code(LoaderState* stp) case op_int_code_end: stp->code_buffer_size = code_buffer_size; stp->ci = ci; - return 1; + stp->function = THE_NON_VALUE; + stp->genop = NULL; + stp->specific_op = -1; + retval = 1; + goto cleanup; } /* @@ -2471,9 +2493,20 @@ load_code(LoaderState* stp) } } - load_error: - return 0; + retval = 0; + + cleanup: + /* + * Clean up everything that is not needed any longer. + */ + + while (stp->genop_blocks) { + GenOpBlock* next = stp->genop_blocks->next; + erts_free(ERTS_ALC_T_LOADER_TMP, (void *) stp->genop_blocks); + stp->genop_blocks = next; + } + return retval; } @@ -4265,24 +4298,31 @@ final_touch(LoaderState* stp) index = next; } modp = erts_put_module(stp->module); - modp->catches = catches; + modp->curr.catches = catches; /* * Export functions. */ for (i = 0; i < stp->num_exps; i++) { - Export* ep = erts_export_put(stp->module, stp->export[i].function, - stp->export[i].arity); + Export* ep; + BeamInstr* address = stp->export[i].address; + + if (address == NULL) { + /* Skip stub for a BIF */ + continue; + } + ep = erts_export_put(stp->module, stp->export[i].function, + stp->export[i].arity); if (!on_load) { - ep->address = stp->export[i].address; + ep->addressv[erts_staging_code_ix()] = address; } else { /* * Don't make any of the exported functions * callable yet. */ - ep->address = ep->code+3; - ep->code[4] = (BeamInstr) stp->export[i].address; + ep->addressv[erts_staging_code_ix()] = ep->code+3; + ep->code[4] = (BeamInstr) address; } } @@ -4926,7 +4966,7 @@ new_label(LoaderState* stp) int num = stp->num_labels; stp->num_labels++; - stp->labels = (Label *) erts_realloc(ERTS_ALC_T_LOADER_TMP, + stp->labels = (Label *) erts_realloc(ERTS_ALC_T_PREPARED_CODE, (void *) stp->labels, stp->num_labels * sizeof(Label)); stp->labels[num].value = 0; @@ -4937,7 +4977,8 @@ new_label(LoaderState* stp) static void new_literal_patch(LoaderState* stp, int pos) { - LiteralPatch* p = erts_alloc(ERTS_ALC_T_LOADER_TMP, sizeof(LiteralPatch)); + LiteralPatch* p = erts_alloc(ERTS_ALC_T_PREPARED_CODE, + sizeof(LiteralPatch)); p->pos = pos; p->next = stp->literal_patches; stp->literal_patches = p; @@ -4946,7 +4987,7 @@ new_literal_patch(LoaderState* stp, int pos) static void new_string_patch(LoaderState* stp, int pos) { - StringPatch* p = erts_alloc(ERTS_ALC_T_LOADER_TMP, sizeof(StringPatch)); + StringPatch* p = erts_alloc(ERTS_ALC_T_PREPARED_CODE, sizeof(StringPatch)); p->pos = pos; p->next = stp->string_patches; stp->string_patches = p; @@ -4964,14 +5005,14 @@ new_literal(LoaderState* stp, Eterm** hpp, Uint heap_size) ASSERT(stp->num_literals == 0); stp->allocated_literals = 8; need = stp->allocated_literals * sizeof(Literal); - stp->literals = (Literal *) erts_alloc(ERTS_ALC_T_LOADER_TMP, + stp->literals = (Literal *) erts_alloc(ERTS_ALC_T_PREPARED_CODE, need); } else if (stp->allocated_literals <= stp->num_literals) { Uint need; stp->allocated_literals *= 2; need = stp->allocated_literals * sizeof(Literal); - stp->literals = (Literal *) erts_realloc(ERTS_ALC_T_LOADER_TMP, + stp->literals = (Literal *) erts_realloc(ERTS_ALC_T_PREPARED_CODE, (void *) stp->literals, need); } @@ -4980,7 +5021,7 @@ new_literal(LoaderState* stp, Eterm** hpp, Uint heap_size) lit = stp->literals + stp->num_literals; lit->offset = 0; lit->heap_size = heap_size; - lit->heap = erts_alloc(ERTS_ALC_T_LOADER_TMP, heap_size*sizeof(Eterm)); + lit->heap = erts_alloc(ERTS_ALC_T_PREPARED_CODE, heap_size*sizeof(Eterm)); lit->term = make_boxed(lit->heap); lit->off_heap.first = 0; lit->off_heap.overhead = 0; @@ -4999,7 +5040,7 @@ erts_module_info_0(Process* p, Eterm module) return THE_NON_VALUE; } - if (erts_get_module(module) == NULL) { + if (erts_get_module(module, erts_active_code_ix()) == NULL) { return THE_NON_VALUE; } @@ -5054,32 +5095,43 @@ functions_in_module(Process* p, /* Process whose heap to use. */ BeamInstr* code; int i; Uint num_functions; + Uint need; Eterm* hp; + Eterm* hp_end; Eterm result = NIL; if (is_not_atom(mod)) { return THE_NON_VALUE; } - modp = erts_get_module(mod); + modp = erts_get_module(mod, erts_active_code_ix()); if (modp == NULL) { return THE_NON_VALUE; } - code = modp->code; + code = modp->curr.code; num_functions = code[MI_NUM_FUNCTIONS]; - hp = HAlloc(p, 5*num_functions); + need = 5*num_functions; + hp = HAlloc(p, need); + hp_end = hp + need; for (i = num_functions-1; i >= 0 ; i--) { BeamInstr* func_info = (BeamInstr *) code[MI_FUNCTIONS+i]; Eterm name = (Eterm) func_info[3]; int arity = (int) func_info[4]; Eterm tuple; - ASSERT(is_atom(name)); - tuple = TUPLE2(hp, name, make_small(arity)); - hp += 3; - result = CONS(hp, tuple, result); - hp += 2; + /* + * If the function name is [], this entry is a stub for + * a BIF that should be ignored. + */ + ASSERT(is_atom(name) || is_nil(name)); + if (is_atom(name)) { + tuple = TUPLE2(hp, name, make_small(arity)); + hp += 3; + result = CONS(hp, tuple, result); + hp += 2; + } } + HRelease(p, hp_end, hp); return result; } @@ -5106,12 +5158,12 @@ native_addresses(Process* p, Eterm mod) return THE_NON_VALUE; } - modp = erts_get_module(mod); + modp = erts_get_module(mod, erts_active_code_ix()); if (modp == NULL) { return THE_NON_VALUE; } - code = modp->code; + code = modp->curr.code; num_functions = code[MI_NUM_FUNCTIONS]; need = (6+BIG_UINT_HEAP_SIZE)*num_functions; hp = HAlloc(p, need); @@ -5122,9 +5174,11 @@ native_addresses(Process* p, Eterm mod) int arity = (int) func_info[4]; Eterm tuple; - ASSERT(is_atom(name)); + ASSERT(is_atom(name) || is_nil(name)); /* [] if BIF stub */ if (func_info[1] != 0) { - Eterm addr = erts_bld_uint(&hp, NULL, func_info[1]); + Eterm addr; + ASSERT(is_atom(name)); + addr = erts_bld_uint(&hp, NULL, func_info[1]); tuple = erts_bld_tuple(&hp, NULL, 3, name, make_small(arity), addr); result = erts_bld_cons(&hp, NULL, tuple, result); } @@ -5149,18 +5203,20 @@ exported_from_module(Process* p, /* Process whose heap to use. */ Eterm* hp = NULL; Eterm* hend = NULL; Eterm result = NIL; + ErtsCodeIndex code_ix; if (is_not_atom(mod)) { return THE_NON_VALUE; } - for (i = 0; i < export_list_size(); i++) { - Export* ep = export_list(i); + code_ix = erts_active_code_ix(); + for (i = 0; i < export_list_size(code_ix); i++) { + Export* ep = export_list(i,code_ix); if (ep->code[0] == mod) { Eterm tuple; - if (ep->address == ep->code+3 && + if (ep->addressv[code_ix] == ep->code+3 && ep->code[3] == (BeamInstr) em_call_error_handler) { /* There is a call to the function, but it does not exist. */ continue; @@ -5204,11 +5260,11 @@ attributes_for_module(Process* p, /* Process whose heap to use. */ return THE_NON_VALUE; } - modp = erts_get_module(mod); + modp = erts_get_module(mod, erts_active_code_ix()); if (modp == NULL) { return THE_NON_VALUE; } - code = modp->code; + code = modp->curr.code; ext = (byte *) code[MI_ATTR_PTR]; if (ext != NULL) { hp = HAlloc(p, code[MI_ATTR_SIZE_ON_HEAP]); @@ -5244,11 +5300,11 @@ compilation_info_for_module(Process* p, /* Process whose heap to use. */ return THE_NON_VALUE; } - modp = erts_get_module(mod); + modp = erts_get_module(mod, erts_active_code_ix()); if (modp == NULL) { return THE_NON_VALUE; } - code = modp->code; + code = modp->curr.code; ext = (byte *) code[MI_COMPILE_PTR]; if (ext != NULL) { hp = HAlloc(p, code[MI_COMPILE_SIZE_ON_HEAP]); @@ -5263,113 +5319,6 @@ compilation_info_for_module(Process* p, /* Process whose heap to use. */ } /* - * Find a function from the given pc and fill information in - * the FunctionInfo struct. If the full_info is non-zero, fill - * in all available information (including location in the - * source code). If no function is found, the 'current' field - * will be set to NULL. - */ - -void -erts_lookup_function_info(FunctionInfo* fi, BeamInstr* pc, int full_info) -{ - Range* low = modules; - Range* high = low + num_loaded_modules; - Range* mid = mid_module; - - fi->current = NULL; - fi->needed = 5; - fi->loc = LINE_INVALID_LOCATION; - while (low < high) { - if (pc < mid->start) { - high = mid; - } else if (pc > mid->end) { - low = mid + 1; - } else { - BeamInstr** low1 = (BeamInstr **) (mid->start + MI_FUNCTIONS); - BeamInstr** high1 = low1 + mid->start[MI_NUM_FUNCTIONS]; - BeamInstr** mid1; - - while (low1 < high1) { - mid1 = low1 + (high1-low1) / 2; - if (pc < mid1[0]) { - high1 = mid1; - } else if (pc < mid1[1]) { - mid_module = mid; - fi->current = mid1[0]+2; - if (full_info) { - BeamInstr** fp = (BeamInstr **) (mid->start + - MI_FUNCTIONS); - int idx = mid1 - fp; - lookup_loc(fi, pc, mid->start, idx); - } - return; - } else { - low1 = mid1 + 1; - } - } - return; - } - mid = low + (high-low) / 2; - } -} - -static void -lookup_loc(FunctionInfo* fi, BeamInstr* orig_pc, BeamInstr* modp, int idx) -{ - Eterm* line = (Eterm *) modp[MI_LINE_TABLE]; - Eterm* low; - Eterm* high; - Eterm* mid; - Eterm pc; - - if (line == 0) { - return; - } - - pc = (Eterm) (BeamInstr) orig_pc; - fi->fname_ptr = (Eterm *) (BeamInstr) line[MI_LINE_FNAME_PTR]; - low = (Eterm *) (BeamInstr) line[MI_LINE_FUNC_TAB+idx]; - high = (Eterm *) (BeamInstr) line[MI_LINE_FUNC_TAB+idx+1]; - while (high > low) { - mid = low + (high-low) / 2; - if (pc < mid[0]) { - high = mid; - } else if (pc < mid[1]) { - int file; - int index = mid - (Eterm *) (BeamInstr) line[MI_LINE_FUNC_TAB]; - - if (line[MI_LINE_LOC_SIZE] == 2) { - Uint16* loc_table = - (Uint16 *) (BeamInstr) line[MI_LINE_LOC_TAB]; - fi->loc = loc_table[index]; - } else { - Uint32* loc_table = - (Uint32 *) (BeamInstr) line[MI_LINE_LOC_TAB]; - ASSERT(line[MI_LINE_LOC_SIZE] == 4); - fi->loc = loc_table[index]; - } - if (fi->loc == LINE_INVALID_LOCATION) { - return; - } - fi->needed += 3+2+3+2; - file = LOC_FILE(fi->loc); - if (file == 0) { - /* Special case: Module name with ".erl" appended */ - Atom* mod_atom = atom_tab(atom_val(fi->current[0])); - fi->needed += 2*(mod_atom->len+4); - } else { - Atom* ap = atom_tab(atom_val((fi->fname_ptr)[file-1])); - fi->needed += 2*ap->len; - } - return; - } else { - low = mid + 1; - } - } -} - -/* * Build a single {M,F,A,Loction} item to be part of * a stack trace. */ @@ -5449,6 +5398,7 @@ code_get_chunk_2(BIF_ALIST_2) Process* p = BIF_P; Eterm Bin = BIF_ARG_1; Eterm Chunk = BIF_ARG_2; + Binary* magic = 0; LoaderState* stp; Uint chunk = 0; ErlSubBin* sb; @@ -5461,12 +5411,13 @@ code_get_chunk_2(BIF_ALIST_2) Eterm real_bin; byte* temp_alloc = NULL; - stp = erts_alloc_loader_state(); + magic = erts_alloc_loader_state(); + stp = ERTS_MAGIC_BIN_DATA(magic); if ((start = erts_get_aligned_binary_bytes(Bin, &temp_alloc)) == NULL) { error: erts_free_aligned_binary_bytes(temp_alloc); - if (stp) { - free_state(stp); + if (magic) { + free_loader_state(magic); } BIF_ERROR(p, BADARG); } @@ -5511,7 +5462,7 @@ code_get_chunk_2(BIF_ALIST_2) done: erts_free_aligned_binary_bytes(temp_alloc); - free_state(stp); + free_loader_state(magic); return res; } @@ -5524,14 +5475,16 @@ code_module_md5_1(BIF_ALIST_1) { Process* p = BIF_P; Eterm Bin = BIF_ARG_1; + Binary* magic; LoaderState* stp; byte* bytes; byte* temp_alloc = NULL; Eterm res; - stp = erts_alloc_loader_state(); + magic = erts_alloc_loader_state(); + stp = ERTS_MAGIC_BIN_DATA(magic); if ((bytes = erts_get_aligned_binary_bytes(Bin, &temp_alloc)) == NULL) { - free_state(stp); + free_loader_state(magic); BIF_ERROR(p, BADARG); } stp->module = THE_NON_VALUE; /* Suppress diagnostiscs */ @@ -5545,7 +5498,7 @@ code_module_md5_1(BIF_ALIST_1) done: erts_free_aligned_binary_bytes(temp_alloc); - free_state(stp); + free_loader_state(magic); return res; } @@ -5601,7 +5554,7 @@ stub_read_export_table(LoaderState* stp) stp->num_exps, stp->num_functions); } stp->export - = (ExportEntry *) erts_alloc(ERTS_ALC_T_LOADER_TMP, + = (ExportEntry *) erts_alloc(ERTS_ALC_T_PREPARED_CODE, stp->num_exps * sizeof(ExportEntry)); for (i = 0; i < stp->num_exps; i++) { @@ -5627,20 +5580,29 @@ stub_final_touch(LoaderState* stp, BeamInstr* fp) { int i; int n = stp->num_exps; + Eterm mod = fp[2]; Eterm function = fp[3]; int arity = fp[4]; #ifdef HIPE Lambda* lp; #endif + if (is_bif(mod, function, arity)) { + fp[1] = 0; + fp[2] = 0; + fp[3] = 0; + fp[4] = 0; + return; + } + /* * Test if the function should be exported. */ for (i = 0; i < n; i++) { if (stp->export[i].function == function && stp->export[i].arity == arity) { - Export* ep = erts_export_put(fp[2], function, arity); - ep->address = fp+5; + Export* ep = erts_export_put(mod, function, arity); + ep->addressv[erts_staging_code_ix()] = fp+5; return; } } @@ -5819,6 +5781,7 @@ patch_funentries(Eterm Patchlist) Eterm erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) { + Binary* magic; LoaderState* stp; BeamInstr Funcs; BeamInstr Patchlist; @@ -5840,7 +5803,8 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) * Must initialize stp->lambdas here because the error handling code * at label 'error' uses it. */ - stp = erts_alloc_loader_state(); + magic = erts_alloc_loader_state(); + stp = ERTS_MAGIC_BIN_DATA(magic); if (is_not_atom(Mod)) { goto error; @@ -5920,7 +5884,6 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) code[MI_COMPILE_PTR] = 0; code[MI_COMPILE_SIZE] = 0; code[MI_COMPILE_SIZE_ON_HEAP] = 0; - code[MI_NUM_BREAKPOINTS] = 0; code[MI_LITERALS_START] = 0; code[MI_LITERALS_END] = 0; code[MI_LITERALS_OFF_HEAP] = 0; @@ -6028,13 +5991,13 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) if (patch_funentries(Patchlist)) { erts_free_aligned_binary_bytes(temp_alloc); - free_state(stp); + free_loader_state(magic); return Mod; } error: erts_free_aligned_binary_bytes(temp_alloc); - free_state(stp); + free_loader_state(magic); BIF_ERROR(p, BADARG); } @@ -6051,3 +6014,4 @@ static int safe_mul(UWord a, UWord b, UWord* resp) return (res / b) == a; } } + diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h index 997ba197db..f1506a8684 100644 --- a/erts/emulator/beam/beam_load.h +++ b/erts/emulator/beam/beam_load.h @@ -50,10 +50,6 @@ extern BeamInstr beam_debug_apply[]; extern BeamInstr* em_call_error_handler; extern BeamInstr* em_apply_bif; extern BeamInstr* em_call_traced_function; -typedef struct { - BeamInstr* start; /* Pointer to start of module. */ - BeamInstr* end; /* Points one word beyond last function in module. */ -} Range; /* * The following variables keep a sorted list of address ranges for @@ -61,11 +57,6 @@ typedef struct { * instruction pointer. */ -extern Range* modules; -extern int num_loaded_modules; -extern int allocated_modules; -extern Range* mid_module; - /* Total code size in bytes */ extern Uint erts_total_code_size; /* @@ -94,27 +85,22 @@ extern Uint erts_total_code_size; #define MI_COMPILE_SIZE_ON_HEAP 6 /* - * Number of breakpoints in module is stored in this word - */ -#define MI_NUM_BREAKPOINTS 7 - -/* * Literal area (constant pool). */ -#define MI_LITERALS_START 8 -#define MI_LITERALS_END 9 -#define MI_LITERALS_OFF_HEAP 10 +#define MI_LITERALS_START 7 +#define MI_LITERALS_END 8 +#define MI_LITERALS_OFF_HEAP 9 /* * Pointer to the on_load function (or NULL if none). */ -#define MI_ON_LOAD_FUNCTION_PTR 11 +#define MI_ON_LOAD_FUNCTION_PTR 10 /* * Pointer to the line table (or NULL if none). */ -#define MI_LINE_TABLE 12 +#define MI_LINE_TABLE 11 /* * Start of function pointer table. This table contains pointers to @@ -125,5 +111,27 @@ extern Uint erts_total_code_size; * this table. */ -#define MI_FUNCTIONS 13 +#define MI_FUNCTIONS 12 + +/* + * Layout of the line table. + */ + +#define MI_LINE_FNAME_PTR 0 +#define MI_LINE_LOC_TAB 1 +#define MI_LINE_LOC_SIZE 2 +#define MI_LINE_FUNC_TAB 3 + +#define LINE_INVALID_LOCATION (0) + +/* + * Macros for manipulating locations. + */ + +#define IS_VALID_LOCATION(File, Line) \ + ((unsigned) (File) < 255 && (unsigned) (Line) < ((1 << 24) - 1)) +#define MAKE_LOCATION(File, Line) (((File) << 24) | (Line)) +#define LOC_FILE(Loc) ((Loc) >> 24) +#define LOC_LINE(Loc) ((Loc) & ((1 << 24)-1)) + #endif /* _BEAM_LOAD_H */ diff --git a/erts/emulator/beam/beam_ranges.c b/erts/emulator/beam/beam_ranges.c new file mode 100644 index 0000000000..0f2d5d0c2a --- /dev/null +++ b/erts/emulator/beam/beam_ranges.c @@ -0,0 +1,349 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2012. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "sys.h" +#include "erl_vm.h" +#include "global.h" +#include "beam_load.h" + +typedef struct { + BeamInstr* start; /* Pointer to start of module. */ + erts_smp_atomic_t end; /* (BeamInstr*) Points one word beyond last function in module. */ +} Range; + +/* Range 'end' needs to be atomic as we purge module + by setting end=start in active code_ix */ +#define RANGE_END(R) ((BeamInstr*)erts_smp_atomic_read_nob(&(R)->end)) + +static Range* find_range(BeamInstr* pc); +static void lookup_loc(FunctionInfo* fi, BeamInstr* pc, + BeamInstr* modp, int idx); + +/* + * The following variables keep a sorted list of address ranges for + * each module. It allows us to quickly find a function given an + * instruction pointer. + */ +struct ranges { + Range* modules; /* Sorted lists of module addresses. */ + Sint n; /* Number of range entries. */ + Sint allocated; /* Number of allocated entries. */ + erts_smp_atomic_t mid; /* Cached search start point */ +}; +static struct ranges r[ERTS_NUM_CODE_IX]; +static erts_smp_atomic_t mem_used; + +#ifdef HARD_DEBUG +static void check_consistency(struct ranges* p) +{ + int i; + + ASSERT(p->n <= p->allocated); + ASSERT((Uint)(p->mid - p->modules) < p->n || + (p->mid == p->modules && p->n == 0)); + for (i = 0; i < p->n; i++) { + ASSERT(p->modules[i].start <= RANGE_END(&p->modules[i])); + ASSERT(!i || RANGE_END(&p->modules[i-1]) < p->modules[i].start); + } +} +# define CHECK(r) check_consistency(r) +#else +# define CHECK(r) +#endif /* HARD_DEBUG */ + + +void +erts_init_ranges(void) +{ + Sint i; + + erts_smp_atomic_init_nob(&mem_used, 0); + for (i = 0; i < ERTS_NUM_CODE_IX; i++) { + r[i].modules = 0; + r[i].n = 0; + r[i].allocated = 0; + erts_smp_atomic_init_nob(&r[i].mid, 0); + } +} + +void +erts_start_staging_ranges(void) +{ + ErtsCodeIndex dst = erts_staging_code_ix(); + + if (r[dst].modules) { + erts_smp_atomic_add_nob(&mem_used, -r[dst].allocated); + erts_free(ERTS_ALC_T_MODULE_REFS, r[dst].modules); + r[dst].modules = NULL; + } +} + +void +erts_end_staging_ranges(int commit) +{ + ErtsCodeIndex dst = erts_staging_code_ix(); + + if (commit && r[dst].modules == NULL) { + Sint i; + Sint n; + + /* No modules added, just clone src and remove purged code. */ + ErtsCodeIndex src = erts_active_code_ix(); + + erts_smp_atomic_add_nob(&mem_used, r[src].n); + r[dst].modules = erts_alloc(ERTS_ALC_T_MODULE_REFS, + r[src].n * sizeof(Range)); + r[dst].allocated = r[src].n; + n = 0; + for (i = 0; i < r[src].n; i++) { + Range* rp = r[src].modules+i; + if (rp->start < RANGE_END(rp)) { + /* Only insert a module that has not been purged. */ + r[dst].modules[n] = *rp; + n++; + } + } + r[dst].n = n; + erts_smp_atomic_set_nob(&r[dst].mid, + (erts_aint_t) (r[dst].modules + n / 2)); + } +} + +void +erts_update_ranges(BeamInstr* code, Uint size) +{ + ErtsCodeIndex dst = erts_staging_code_ix(); + ErtsCodeIndex src = erts_active_code_ix(); + Sint i; + Sint n; + Sint need; + + if (src == dst) { + ASSERT(!erts_initialized); + + /* + * During start-up of system, the indices are the same. + * Handle this by faking a source area. + */ + src = (src+1) % ERTS_NUM_CODE_IX; + if (r[src].modules) { + erts_smp_atomic_add_nob(&mem_used, -r[src].allocated); + erts_free(ERTS_ALC_T_MODULE_REFS, r[src].modules); + } + r[src] = r[dst]; + r[dst].modules = 0; + } + + CHECK(&r[src]); + + ASSERT(r[dst].modules == NULL); + need = r[dst].allocated = r[src].n + 1; + erts_smp_atomic_add_nob(&mem_used, need); + r[dst].modules = (Range *) erts_alloc(ERTS_ALC_T_MODULE_REFS, + need * sizeof(Range)); + n = 0; + for (i = 0; i < r[src].n; i++) { + Range* rp = r[src].modules+i; + if (code < rp->start) { + r[dst].modules[n].start = code; + erts_smp_atomic_init_nob(&r[dst].modules[n].end, + (erts_aint_t)(((byte *)code) + size)); + ASSERT(!n || RANGE_END(&r[dst].modules[n-1]) < code); + n++; + break; + } + if (rp->start < RANGE_END(rp)) { + /* Only insert a module that has not been purged. */ + r[dst].modules[n].start = rp->start; + erts_smp_atomic_init_nob(&r[dst].modules[n].end, + (erts_aint_t)(RANGE_END(rp))); + ASSERT(!n || RANGE_END(&r[dst].modules[n-1]) < rp->start); + n++; + } + } + + while (i < r[src].n) { + Range* rp = r[src].modules+i; + if (rp->start < RANGE_END(rp)) { + /* Only insert a module that has not been purged. */ + r[dst].modules[n].start = rp->start; + erts_smp_atomic_init_nob(&r[dst].modules[n].end, + (erts_aint_t)(RANGE_END(rp))); + ASSERT(!n || RANGE_END(&r[dst].modules[n-1]) < rp->start); + n++; + } + i++; + } + + if (n == 0 || code > r[dst].modules[n-1].start) { + r[dst].modules[n].start = code; + erts_smp_atomic_init_nob(&r[dst].modules[n].end, + (erts_aint_t)(((byte *)code) + size)); + ASSERT(!n || RANGE_END(&r[dst].modules[n-1]) < code); + n++; + } + + ASSERT(n <= r[src].n+1); + r[dst].n = n; + erts_smp_atomic_set_nob(&r[dst].mid, + (erts_aint_t) (r[dst].modules + n / 2)); + + CHECK(&r[dst]); + CHECK(&r[src]); +} + +void +erts_remove_from_ranges(BeamInstr* code) +{ + Range* rp = find_range(code); + erts_smp_atomic_set_nob(&rp->end, (erts_aint_t)rp->start); +} + +UWord +erts_ranges_sz(void) +{ + return erts_smp_atomic_read_nob(&mem_used) * sizeof(Range); +} + +/* + * Find a function from the given pc and fill information in + * the FunctionInfo struct. If the full_info is non-zero, fill + * in all available information (including location in the + * source code). If no function is found, the 'current' field + * will be set to NULL. + */ + +void +erts_lookup_function_info(FunctionInfo* fi, BeamInstr* pc, int full_info) +{ + BeamInstr** low; + BeamInstr** high; + BeamInstr** mid; + Range* rp; + + fi->current = NULL; + fi->needed = 5; + fi->loc = LINE_INVALID_LOCATION; + rp = find_range(pc); + if (rp == 0) { + return; + } + + low = (BeamInstr **) (rp->start + MI_FUNCTIONS); + high = low + rp->start[MI_NUM_FUNCTIONS]; + while (low < high) { + mid = low + (high-low) / 2; + if (pc < mid[0]) { + high = mid; + } else if (pc < mid[1]) { + fi->current = mid[0]+2; + if (full_info) { + BeamInstr** fp = (BeamInstr **) (rp->start + + MI_FUNCTIONS); + int idx = mid - fp; + lookup_loc(fi, pc, rp->start, idx); + } + return; + } else { + low = mid + 1; + } + } +} + +static Range* +find_range(BeamInstr* pc) +{ + ErtsCodeIndex active = erts_active_code_ix(); + Range* low = r[active].modules; + Range* high = low + r[active].n; + Range* mid = (Range *) erts_smp_atomic_read_nob(&r[active].mid); + + CHECK(&r[active]); + while (low < high) { + if (pc < mid->start) { + high = mid; + } else if (pc > RANGE_END(mid)) { + low = mid + 1; + } else { + erts_smp_atomic_set_nob(&r[active].mid, (erts_aint_t) mid); + return mid; + } + mid = low + (high-low) / 2; + } + return 0; +} + +static void +lookup_loc(FunctionInfo* fi, BeamInstr* orig_pc, BeamInstr* modp, int idx) +{ + Eterm* line = (Eterm *) modp[MI_LINE_TABLE]; + Eterm* low; + Eterm* high; + Eterm* mid; + Eterm pc; + + if (line == 0) { + return; + } + + pc = (Eterm) (BeamInstr) orig_pc; + fi->fname_ptr = (Eterm *) (BeamInstr) line[MI_LINE_FNAME_PTR]; + low = (Eterm *) (BeamInstr) line[MI_LINE_FUNC_TAB+idx]; + high = (Eterm *) (BeamInstr) line[MI_LINE_FUNC_TAB+idx+1]; + while (high > low) { + mid = low + (high-low) / 2; + if (pc < mid[0]) { + high = mid; + } else if (pc < mid[1]) { + int file; + int index = mid - (Eterm *) (BeamInstr) line[MI_LINE_FUNC_TAB]; + + if (line[MI_LINE_LOC_SIZE] == 2) { + Uint16* loc_table = + (Uint16 *) (BeamInstr) line[MI_LINE_LOC_TAB]; + fi->loc = loc_table[index]; + } else { + Uint32* loc_table = + (Uint32 *) (BeamInstr) line[MI_LINE_LOC_TAB]; + ASSERT(line[MI_LINE_LOC_SIZE] == 4); + fi->loc = loc_table[index]; + } + if (fi->loc == LINE_INVALID_LOCATION) { + return; + } + fi->needed += 3+2+3+2; + file = LOC_FILE(fi->loc); + if (file == 0) { + /* Special case: Module name with ".erl" appended */ + Atom* mod_atom = atom_tab(atom_val(fi->current[0])); + fi->needed += 2*(mod_atom->len+4); + } else { + Atom* ap = atom_tab(atom_val((fi->fname_ptr)[file-1])); + fi->needed += 2*ap->len; + } + return; + } else { + low = mid + 1; + } + } +} diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index fc00b42454..6943c8852c 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -233,15 +233,17 @@ BIF_RETTYPE link_1(BIF_ALIST_1) BIF_ERROR(BIF_P, BADARG); - res_no_proc: - if (BIF_P->flags & F_TRAPEXIT) { - ErtsProcLocks locks = ERTS_PROC_LOCK_MAIN; - erts_deliver_exit_message(BIF_ARG_1, BIF_P, &locks, am_noproc, NIL); - erts_smp_proc_unlock(BIF_P, ~ERTS_PROC_LOCK_MAIN & locks); - BIF_RET(am_true); +res_no_proc: { + erts_aint32_t state = erts_smp_atomic32_read_nob(&BIF_P->state); + if (state & ERTS_PSFLG_TRAP_EXIT) { + ErtsProcLocks locks = ERTS_PROC_LOCK_MAIN; + erts_deliver_exit_message(BIF_ARG_1, BIF_P, &locks, am_noproc, NIL); + erts_smp_proc_unlock(BIF_P, ~ERTS_PROC_LOCK_MAIN & locks); + BIF_RET(am_true); + } + else + BIF_ERROR(BIF_P, EXC_NOPROC); } - else - BIF_ERROR(BIF_P, EXC_NOPROC); } #define ERTS_DEMONITOR_FALSE 2 @@ -1103,8 +1105,9 @@ BIF_RETTYPE hibernate_3(BIF_ALIST_3) if (erts_hibernate(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, reg)) { /* - * If hibernate succeeded, TRAP. The process will be suspended - * if status is P_WAITING or continue (if any message was in the queue). + * If hibernate succeeded, TRAP. The process will be wait in a + * hibernated state if its state is inactive (!ERTS_PSFLG_ACTIVE); + * otherwise, continue executing (if any message was in the queue). */ BIF_TRAP_CODE_PTR_(BIF_P, BIF_P->i); } @@ -1403,9 +1406,8 @@ BIF_RETTYPE exit_2(BIF_ALIST_2) } else { rp_locks = ERTS_PROC_LOCKS_XSIG_SEND; - rp = erts_pid2proc_opt(BIF_P, ERTS_PROC_LOCK_MAIN, - BIF_ARG_1, rp_locks, - ERTS_P2P_FLG_SMP_INC_REFC); + rp = erts_pid2proc(BIF_P, ERTS_PROC_LOCK_MAIN, + BIF_ARG_1, rp_locks); if (!rp) { BIF_RET(am_true); } @@ -1427,8 +1429,6 @@ BIF_RETTYPE exit_2(BIF_ALIST_2) rp_locks &= ~ERTS_PROC_LOCK_MAIN; if (rp_locks) erts_smp_proc_unlock(rp, rp_locks); - if (rp != BIF_P) - erts_smp_proc_dec_refc(rp); #endif /* * We may have exited ourselves and may have to take action. @@ -1502,14 +1502,13 @@ BIF_RETTYPE process_flag_2(BIF_ALIST_2) BIF_RET(old_value); } else if (BIF_ARG_1 == am_priority) { - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_STATUS); old_value = erts_set_process_priority(BIF_P, BIF_ARG_2); - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_STATUS); if (old_value == THE_NON_VALUE) goto error; BIF_RET(old_value); } else if (BIF_ARG_1 == am_trap_exit) { + erts_aint32_t state; Uint trap_exit; if (BIF_ARG_2 == am_true) { trap_exit = 1; @@ -1524,59 +1523,52 @@ BIF_RETTYPE process_flag_2(BIF_ALIST_2) * For more info, see implementation of erts_send_exit_signal(). */ erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_STATUS); + ERTS_SMP_LC_ASSERT((ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS) + & erts_proc_lc_my_proc_locks(BIF_P)); ERTS_SMP_BIF_CHK_PENDING_EXIT(BIF_P, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS); - old_value = ERTS_PROC_IS_TRAPPING_EXITS(BIF_P) ? am_true : am_false; - if (trap_exit) { - ERTS_PROC_SET_TRAP_EXIT(BIF_P); - } else { - ERTS_PROC_UNSET_TRAP_EXIT(BIF_P); - } + if (trap_exit) + state = erts_smp_atomic32_read_bor_nob(&BIF_P->state, + ERTS_PSFLG_TRAP_EXIT); + else + state = erts_smp_atomic32_read_band_nob(&BIF_P->state, + ~ERTS_PSFLG_TRAP_EXIT); + old_value = (state & ERTS_PSFLG_TRAP_EXIT) ? am_true : am_false; erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_STATUS); BIF_RET(old_value); } else if (BIF_ARG_1 == am_scheduler) { - int yield; - ErtsRunQueue *old; - ErtsRunQueue *new; + ErtsRunQueue *old, *new, *curr; Sint sched; + erts_aint32_t state; + if (!is_small(BIF_ARG_2)) goto error; sched = signed_val(BIF_ARG_2); if (sched < 0 || erts_no_schedulers < sched) goto error; - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_STATUS); - old = BIF_P->bound_runq; -#ifdef ERTS_SMP - ASSERT(!old || old == BIF_P->run_queue); -#endif - new = !sched ? NULL : erts_schedid2runq(sched); -#ifndef ERTS_SMP - yield = 0; -#else - if (new == old) - yield = 0; + + if (sched == 0) { + new = NULL; + state = erts_smp_atomic32_read_band_mb(&BIF_P->state, + ~ERTS_PSFLG_BOUND); + } else { - ErtsRunQueue *curr = BIF_P->run_queue; - if (!new) - erts_smp_runq_lock(curr); - else - erts_smp_runqs_lock(curr, new); - yield = new && BIF_P->run_queue != new; -#endif - BIF_P->bound_runq = new; + new = erts_schedid2runq(sched); #ifdef ERTS_SMP - if (new) - BIF_P->run_queue = new; - if (!new) - erts_smp_runq_unlock(curr); - else - erts_smp_runqs_unlock(curr, new); - } + erts_atomic_set_nob(&BIF_P->run_queue, (erts_aint_t) new); #endif - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_STATUS); + state = erts_smp_atomic32_read_bor_mb(&BIF_P->state, + ERTS_PSFLG_BOUND); + } + + curr = ERTS_GET_SCHEDULER_DATA_FROM_PROC(BIF_P)->run_queue; + old = (ERTS_PSFLG_BOUND & state) ? curr : NULL; + + ASSERT(!old || old == curr); + old_value = old ? make_small(old->ix+1) : make_small(0); - if (yield) + if (new && new != curr) ERTS_BIF_YIELD_RETURN_X(BIF_P, old_value, am_scheduler); else BIF_RET(old_value); @@ -1826,8 +1818,7 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend) { if (internal_pid_index(to) >= erts_max_processes) return SEND_BADARG; - rp = erts_pid2proc_opt(p, ERTS_PROC_LOCK_MAIN, - to, 0, ERTS_P2P_FLG_SMP_INC_REFC); + rp = erts_proc_lookup_raw(to); if (!rp) { ERTS_SMP_ASSERT_IS_NOT_EXITING(p); @@ -1865,7 +1856,7 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend) { } erts_whereis_name(p, ERTS_PROC_LOCK_MAIN, to, - &rp, 0, ERTS_P2P_FLG_SMP_INC_REFC, + &rp, 0, 0, &pt); if (pt) { @@ -2017,7 +2008,7 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend) { erts_whereis_name(p, ERTS_PROC_LOCK_MAIN, tp[1], - &rp, 0, ERTS_P2P_FLG_SMP_INC_REFC, + &rp, 0, 0, &pt); if (pt) { portid = pt->id; @@ -2076,7 +2067,6 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend) { p == rp ? (rp_locks & ~ERTS_PROC_LOCK_MAIN) : rp_locks); - erts_smp_proc_dec_refc(rp); return res; } } @@ -3486,7 +3476,7 @@ BIF_RETTYPE garbage_collect_1(BIF_ALIST_1) if (rp == ERTS_PROC_LOCK_BUSY) ERTS_BIF_YIELD1(bif_export[BIF_garbage_collect_1], BIF_P, BIF_ARG_1); #else - rp = erts_pid2proc(BIF_P, 0, BIF_ARG_1, 0); + rp = erts_proc_lookup(BIF_ARG_1); #endif if (!rp) BIF_RET(am_false); @@ -3780,7 +3770,8 @@ BIF_RETTYPE function_exported_3(BIF_ALIST_3) is_not_small(BIF_ARG_3)) { BIF_ERROR(BIF_P, BADARG); } - if (erts_find_function(BIF_ARG_1, BIF_ARG_2, signed_val(BIF_ARG_3)) == NULL) { + if (erts_find_function(BIF_ARG_1, BIF_ARG_2, signed_val(BIF_ARG_3), + erts_active_code_ix()) == NULL) { BIF_RET(am_false); } BIF_RET(am_true); @@ -4216,8 +4207,8 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) erts_smp_thr_progress_block(); for (i = 0; i < erts_max_processes; i++) { - if (process_tab[i] != (Process*) 0) { - Process* p = process_tab[i]; + Process *p = erts_pix2proc(i); + if (p) { #ifdef USE_VM_PROBES p->seq_trace_token = (p->dt_utag != NIL) ? am_have_dt_utag : NIL; #else @@ -4513,6 +4504,21 @@ erts_bif_prep_await_proc_exit_apply_trap(Process *c_p, Export bif_return_trap_export; +void erts_init_trap_export(Export* ep, Eterm m, Eterm f, Uint a, + Eterm (*bif)(BIF_ALIST_0)) +{ + int i; + sys_memset((void *) ep, 0, sizeof(Export)); + for (i=0; i<ERTS_NUM_CODE_IX; i++) { + ep->addressv[i] = &ep->code[3]; + } + ep->code[0] = m; + ep->code[1] = f; + ep->code[2] = a; + ep->code[3] = (BeamInstr) em_apply_bif; + ep->code[4] = (BeamInstr) bif; +} + void erts_init_bif(void) { reference0 = 0; @@ -4528,17 +4534,13 @@ void erts_init_bif(void) * yield the calling process traps to. The only thing it does: * return the value passed as argument. */ - sys_memset((void *) &bif_return_trap_export, 0, sizeof(Export)); - bif_return_trap_export.address = &bif_return_trap_export.code[3]; - bif_return_trap_export.code[0] = am_erlang; - bif_return_trap_export.code[1] = am_bif_return_trap; + erts_init_trap_export(&bif_return_trap_export, am_erlang, am_bif_return_trap, #ifdef DEBUG - bif_return_trap_export.code[2] = 2; + 2 #else - bif_return_trap_export.code[2] = 1; + 1 #endif - bif_return_trap_export.code[3] = (BeamInstr) em_apply_bif; - bif_return_trap_export.code[4] = (BeamInstr) &bif_return_trap; + , &bif_return_trap); flush_monitor_message_trap = erts_export_put(am_erlang, am_flush_monitor_message, diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h index d20089a9fb..7cb2c78815 100644 --- a/erts/emulator/beam/bif.h +++ b/erts/emulator/beam/bif.h @@ -125,7 +125,7 @@ do { \ #define ERTS_BIF_PREP_TRAP0(Ret, Trap, Proc) \ do { \ (Proc)->arity = 0; \ - (Proc)->i = (BeamInstr*) ((Trap)->address); \ + (Proc)->i = (BeamInstr*) ((Trap)->addressv[erts_active_code_ix()]); \ (Proc)->freason = TRAP; \ (Ret) = THE_NON_VALUE; \ } while (0) @@ -135,7 +135,7 @@ do { \ Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array; \ (Proc)->arity = 1; \ reg[0] = (Eterm) (A0); \ - (Proc)->i = (BeamInstr*) ((Trap)->address); \ + (Proc)->i = (BeamInstr*) ((Trap)->addressv[erts_active_code_ix()]); \ (Proc)->freason = TRAP; \ (Ret) = THE_NON_VALUE; \ } while (0) @@ -146,7 +146,7 @@ do { \ (Proc)->arity = 2; \ reg[0] = (Eterm) (A0); \ reg[1] = (Eterm) (A1); \ - (Proc)->i = (BeamInstr*) ((Trap)->address); \ + (Proc)->i = (BeamInstr*) ((Trap)->addressv[erts_active_code_ix()]); \ (Proc)->freason = TRAP; \ (Ret) = THE_NON_VALUE; \ } while (0) @@ -158,7 +158,7 @@ do { \ reg[0] = (Eterm) (A0); \ reg[1] = (Eterm) (A1); \ reg[2] = (Eterm) (A2); \ - (Proc)->i = (BeamInstr*) ((Trap)->address); \ + (Proc)->i = (BeamInstr*) ((Trap)->addressv[erts_active_code_ix()]); \ (Proc)->freason = TRAP; \ (Ret) = THE_NON_VALUE; \ } while (0) @@ -170,13 +170,13 @@ do { \ reg[0] = (Eterm) (A0); \ reg[1] = (Eterm) (A1); \ reg[2] = (Eterm) (A2); \ - (Proc)->i = (BeamInstr*) ((Trap)->address); \ + (Proc)->i = (BeamInstr*) ((Trap)->addressv[erts_active_code_ix()]); \ (Proc)->freason = TRAP; \ } while (0) #define BIF_TRAP0(p, Trap_) do { \ (p)->arity = 0; \ - (p)->i = (BeamInstr*) ((Trap_)->address); \ + (p)->i = (BeamInstr*) ((Trap_)->addressv[erts_active_code_ix()]); \ (p)->freason = TRAP; \ return THE_NON_VALUE; \ } while(0) @@ -185,7 +185,7 @@ do { \ Eterm* reg = ERTS_PROC_GET_SCHDATA((p))->x_reg_array; \ (p)->arity = 1; \ reg[0] = (A0); \ - (p)->i = (BeamInstr*) ((Trap_)->address); \ + (p)->i = (BeamInstr*) ((Trap_)->addressv[erts_active_code_ix()]); \ (p)->freason = TRAP; \ return THE_NON_VALUE; \ } while(0) @@ -195,7 +195,7 @@ do { \ (p)->arity = 2; \ reg[0] = (A0); \ reg[1] = (A1); \ - (p)->i = (BeamInstr*) ((Trap_)->address); \ + (p)->i = (BeamInstr*) ((Trap_)->addressv[erts_active_code_ix()]); \ (p)->freason = TRAP; \ return THE_NON_VALUE; \ } while(0) @@ -206,7 +206,7 @@ do { \ reg[0] = (A0); \ reg[1] = (A1); \ reg[2] = (A2); \ - (p)->i = (BeamInstr*) ((Trap_)->address); \ + (p)->i = (BeamInstr*) ((Trap_)->addressv[erts_active_code_ix()]); \ (p)->freason = TRAP; \ return THE_NON_VALUE; \ } while(0) diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index 797bce43ab..f7dad2767f 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -142,8 +142,6 @@ bif erlang:list_to_pid/1 bif 'erl.lang.proc':string_to_pid/1 ebif_string_to_pid_1 list_to_pid_1 bif erlang:list_to_tuple/1 bif 'erl.lang.tuple':from_list/1 ebif_list_to_tuple_1 -bif erlang:load_module/2 -bif 'erl.system.code':load/2 ebif_load_module_2 bif erlang:loaded/0 bif 'erl.system.code':loaded/0 ebif_loaded_0 bif erlang:localtime/0 @@ -829,6 +827,13 @@ bif erlang:dt_restore_tag/1 bif erlang:dt_prepend_vm_tag_data/1 bif erlang:dt_append_vm_tag_data/1 + +# +# New in R16B. +# +bif erlang:prepare_loading/2 +bif erlang:finish_loading/1 + # # Obsolete # diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index 6f5020dc14..0a51e00016 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -71,9 +71,10 @@ process_info(int to, void *to_arg) { int i; for (i = 0; i < erts_max_processes; i++) { - if ((process_tab[i] != NULL) && (process_tab[i]->i != ENULL)) { - if (process_tab[i]->status != P_EXITING) - print_process_info(to, to_arg, process_tab[i]); + Process *p = erts_pix2proc(i); + if (p && p->i != ENULL) { + if (!ERTS_PROC_IS_EXITING(p)) + print_process_info(to, to_arg, p); } } @@ -89,7 +90,8 @@ process_killer(void) erts_printf("\n\nProcess Information\n\n"); erts_printf("--------------------------------------------------\n"); for (i = erts_max_processes-1; i >= 0; i--) { - if (((rp = process_tab[i]) != NULL) && rp->i != ENULL) { + rp = erts_pix2proc(i); + if (rp && rp->i != ENULL) { int br; print_process_info(ERTS_PRINT_STDOUT, NULL, rp); erts_printf("(k)ill (n)ext (r)eturn:\n"); @@ -97,11 +99,20 @@ process_killer(void) if ((j = sys_get_key(0)) <= 0) erl_exit(0, ""); switch(j) { - case 'k': - if (rp->status == P_WAITING) { - ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND; - erts_smp_proc_inc_refc(rp); - erts_smp_proc_lock(rp, rp_locks); + case 'k': { + ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND; + erts_aint32_t state; + erts_smp_proc_inc_refc(rp); + erts_smp_proc_lock(rp, rp_locks); + state = erts_smp_atomic32_read_acqb(&rp->state); + if (state & (ERTS_PSFLG_FREE + | ERTS_PSFLG_EXITING + | ERTS_PSFLG_ACTIVE + | ERTS_PSFLG_IN_RUNQ + | ERTS_PSFLG_RUNNING)) { + erts_printf("Can only kill WAITING processes this way\n"); + } + else { (void) erts_send_exit_signal(NULL, NIL, rp, @@ -110,12 +121,10 @@ process_killer(void) NIL, NULL, 0); - erts_smp_proc_unlock(rp, rp_locks); - erts_smp_proc_dec_refc(rp); } - else - erts_printf("Can only kill WAITING processes this way\n"); - + erts_smp_proc_unlock(rp, rp_locks); + erts_smp_proc_dec_refc(rp); + } case 'n': br = 1; break; case 'r': return; default: return; @@ -180,42 +189,38 @@ static void doit_print_monitor(ErtsMonitor *mon, void *vpcontext) void print_process_info(int to, void *to_arg, Process *p) { + time_t approx_started; int garbing = 0; int running = 0; - time_t tmp_t; struct saved_calls *scb; + erts_aint32_t state; /* display the PID */ erts_print(to, to_arg, "=proc:%T\n", p->id); /* Display the state */ erts_print(to, to_arg, "State: "); - switch (p->status) { - case P_FREE: + + state = erts_smp_atomic32_read_acqb(&p->state); + if (state & ERTS_PSFLG_FREE) erts_print(to, to_arg, "Non Existing\n"); /* Should never happen */ - break; - case P_RUNABLE: - erts_print(to, to_arg, "Scheduled\n"); - break; - case P_WAITING: - erts_print(to, to_arg, "Waiting\n"); - break; - case P_SUSPENDED: - erts_print(to, to_arg, "Suspended\n"); - break; - case P_RUNNING: - erts_print(to, to_arg, "Running\n"); - running = 1; - break; - case P_EXITING: + else if (state & ERTS_PSFLG_EXITING) erts_print(to, to_arg, "Exiting\n"); - break; - case P_GARBING: - erts_print(to, to_arg, "Garbing\n"); + else if (state & ERTS_PSFLG_GC) { garbing = 1; running = 1; - break; + erts_print(to, to_arg, "Garbing\n"); + } + else if (state & ERTS_PSFLG_SUSPENDED) + erts_print(to, to_arg, "Suspended\n"); + else if (state & ERTS_PSFLG_RUNNING) { + running = 1; + erts_print(to, to_arg, "Running\n"); } + else if (state & ERTS_PSFLG_ACTIVE) + erts_print(to, to_arg, "Scheduled\n"); + else + erts_print(to, to_arg, "Waiting\n"); /* * If the process is registered as a global process, display the @@ -245,8 +250,8 @@ print_process_info(int to, void *to_arg, Process *p) } erts_print(to, to_arg, "Spawned by: %T\n", p->parent); - tmp_t = p->started.tv_sec; - erts_print(to, to_arg, "Started: %s", ctime(&tmp_t)); + approx_started = (time_t) p->approx_started; + erts_print(to, to_arg, "Started: %s", ctime(&approx_started)); ERTS_SMP_MSGQ_MV_INQ2PRIVQ(p); erts_print(to, to_arg, "Message queue length: %d\n", p->msg.len); @@ -377,17 +382,22 @@ loaded(int to, void *to_arg) int old = 0; int cur = 0; BeamInstr* code; + Module* modp; + ErtsCodeIndex code_ix; + + code_ix = erts_active_code_ix(); + erts_rlock_old_code(code_ix); /* * Calculate and print totals. */ - for (i = 0; i < module_code_size(); i++) { - if (module_code(i) != NULL && - ((module_code(i)->code_length != 0) || - (module_code(i)->old_code_length != 0))) { - cur += module_code(i)->code_length; - if (module_code(i)->old_code_length != 0) { - old += module_code(i)->old_code_length; + for (i = 0; i < module_code_size(code_ix); i++) { + if ((modp = module_code(i, code_ix)) != NULL && + ((modp->curr.code_length != 0) || + (modp->old.code_length != 0))) { + cur += modp->curr.code_length; + if (modp->old.code_length != 0) { + old += modp->old.code_length; } } } @@ -398,21 +408,22 @@ loaded(int to, void *to_arg) * Print one line per module. */ - for (i = 0; i < module_code_size(); i++) { + for (i = 0; i < module_code_size(code_ix); i++) { + modp = module_code(i, code_ix); if (!ERTS_IS_CRASH_DUMPING) { /* * Interactive dump; keep it brief. */ - if (module_code(i) != NULL && - ((module_code(i)->code_length != 0) || - (module_code(i)->old_code_length != 0))) { - erts_print(to, to_arg, "%T", make_atom(module_code(i)->module)); - cur += module_code(i)->code_length; - erts_print(to, to_arg, " %d", module_code(i)->code_length ); - if (module_code(i)->old_code_length != 0) { + if (modp != NULL && + ((modp->curr.code_length != 0) || + (modp->old.code_length != 0))) { + erts_print(to, to_arg, "%T", make_atom(modp->module)); + cur += modp->curr.code_length; + erts_print(to, to_arg, " %d", modp->curr.code_length ); + if (modp->old.code_length != 0) { erts_print(to, to_arg, " (%d old)", - module_code(i)->old_code_length ); - old += module_code(i)->old_code_length; + modp->old.code_length ); + old += modp->old.code_length; } erts_print(to, to_arg, "\n"); } @@ -420,15 +431,15 @@ loaded(int to, void *to_arg) /* * To crash dump; make it parseable. */ - if (module_code(i) != NULL && - ((module_code(i)->code_length != 0) || - (module_code(i)->old_code_length != 0))) { + if (modp != NULL && + ((modp->curr.code_length != 0) || + (modp->old.code_length != 0))) { erts_print(to, to_arg, "=mod:"); - erts_print(to, to_arg, "%T", make_atom(module_code(i)->module)); + erts_print(to, to_arg, "%T", make_atom(modp->module)); erts_print(to, to_arg, "\n"); erts_print(to, to_arg, "Current size: %d\n", - module_code(i)->code_length); - code = module_code(i)->code; + modp->curr.code_length); + code = modp->curr.code; if (code != NULL && code[MI_ATTR_PTR]) { erts_print(to, to_arg, "Current attributes: "); dump_attributes(to, to_arg, (byte *) code[MI_ATTR_PTR], @@ -440,9 +451,9 @@ loaded(int to, void *to_arg) code[MI_COMPILE_SIZE]); } - if (module_code(i)->old_code_length != 0) { - erts_print(to, to_arg, "Old size: %d\n", module_code(i)->old_code_length); - code = module_code(i)->old_code; + if (modp->old.code_length != 0) { + erts_print(to, to_arg, "Old size: %d\n", modp->old.code_length); + code = modp->old.code; if (code[MI_ATTR_PTR]) { erts_print(to, to_arg, "Old attributes: "); dump_attributes(to, to_arg, (byte *) code[MI_ATTR_PTR], @@ -457,6 +468,7 @@ loaded(int to, void *to_arg) } } } + erts_runlock_old_code(code_ix); } @@ -616,7 +628,8 @@ bin_check(void) int i, printed = 0; for (i=0; i < erts_max_processes; i++) { - if ((rp = process_tab[i]) == NULL) + rp = erts_pix2proc(i); + if (!rp) continue; for (hdr = rp->off_heap.first; hdr; hdr = hdr->next) { if (hdr->thing_word == HEADER_PROC_BIN) { @@ -704,7 +717,7 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) erts_print_nif_taints(fd, NULL); erts_fdprintf(fd, "Atoms: %d\n", atom_table_size()); info(fd, NULL); /* General system info */ - if (process_tab != NULL) /* XXX true at init */ + if (erts_proc.tab) process_info(fd, NULL); /* Info about each process and port */ db_info(fd, NULL, 0); erts_print_bif_timer_info(fd, NULL); diff --git a/erts/emulator/beam/code_ix.c b/erts/emulator/beam/code_ix.c new file mode 100644 index 0000000000..ae4cca1e58 --- /dev/null +++ b/erts/emulator/beam/code_ix.c @@ -0,0 +1,152 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2012. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "code_ix.h" +#include "global.h" +#include "beam_catches.h" + + + +#if 0 +# define CIX_TRACE(text) erts_fprintf(stderr, "CIX_TRACE: " text " act=%u load=%u\r\n", erts_active_code_ix(), erts_staging_code_ix()) +#else +# define CIX_TRACE(text) +#endif + +erts_smp_atomic32_t the_active_code_index; +erts_smp_atomic32_t the_staging_code_index; + +static int the_code_ix_lock = 0; +struct code_ix_queue_item { + Process *p; + struct code_ix_queue_item* next; +}; +static struct code_ix_queue_item* the_code_ix_queue = NULL; +static erts_smp_mtx_t the_code_ix_queue_lock; + +void erts_code_ix_init(void) +{ + /* We start emulator by initializing preloaded modules + * single threaded with active and staging set both to zero. + * Preloading is finished by a commit that will set things straight. + */ + erts_smp_atomic32_init_nob(&the_active_code_index, 0); + erts_smp_atomic32_init_nob(&the_staging_code_index, 0); + erts_smp_mtx_init(&the_code_ix_queue_lock, "code_ix_queue"); + CIX_TRACE("init"); +} + +void erts_start_staging_code_ix(void) +{ + beam_catches_start_staging(); + export_start_staging(); + module_start_staging(); + erts_start_staging_ranges(); + CIX_TRACE("start"); +} + + +void erts_end_staging_code_ix(void) +{ + beam_catches_end_staging(1); + export_end_staging(1); + module_end_staging(1); + erts_end_staging_ranges(1); + CIX_TRACE("end"); +} + +void erts_commit_staging_code_ix(void) +{ + ErtsCodeIndex ix; + /* We need to this lock as we are now making the staging export table active */ + export_staging_lock(); + ix = erts_staging_code_ix(); + erts_smp_atomic32_set_nob(&the_active_code_index, ix); + ix = (ix + 1) % ERTS_NUM_CODE_IX; + erts_smp_atomic32_set_nob(&the_staging_code_index, ix); + export_staging_unlock(); + CIX_TRACE("activate"); +} + +void erts_abort_staging_code_ix(void) +{ + beam_catches_end_staging(0); + export_end_staging(0); + module_end_staging(0); + erts_end_staging_ranges(0); + CIX_TRACE("abort"); +} + + +/* + * Calller _must_ yield if we return 0 + */ +int erts_try_seize_code_write_permission(Process* c_p) +{ + int success; +#ifdef ERTS_SMP + ASSERT(!erts_smp_thr_progress_is_blocking()); /* to avoid deadlock */ +#endif + + erts_smp_mtx_lock(&the_code_ix_queue_lock); + success = !the_code_ix_lock; + if (success) { + the_code_ix_lock = 1; + } + else { /* Already locked */ + struct code_ix_queue_item* qitem; + qitem = erts_alloc(ERTS_ALC_T_CODE_IX_LOCK_Q, sizeof(*qitem)); + qitem->p = c_p; + erts_smp_proc_inc_refc(c_p); + qitem->next = the_code_ix_queue; + the_code_ix_queue = qitem; + erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL); + } + erts_smp_mtx_unlock(&the_code_ix_queue_lock); + return success; +} + +void erts_release_code_write_permission(void) +{ + erts_smp_mtx_lock(&the_code_ix_queue_lock); + while (the_code_ix_queue != NULL) { /* unleash the entire herd */ + struct code_ix_queue_item* qitem = the_code_ix_queue; + erts_smp_proc_lock(qitem->p, ERTS_PROC_LOCK_STATUS); + if (!ERTS_PROC_IS_EXITING(qitem->p)) { + erts_resume(qitem->p, ERTS_PROC_LOCK_STATUS); + } + erts_smp_proc_unlock(qitem->p, ERTS_PROC_LOCK_STATUS); + the_code_ix_queue = qitem->next; + erts_smp_proc_dec_refc(qitem->p); + erts_free(ERTS_ALC_T_CODE_IX_LOCK_Q, qitem); + } + the_code_ix_lock = 0; + erts_smp_mtx_unlock(&the_code_ix_queue_lock); +} + +#ifdef ERTS_ENABLE_LOCK_CHECK +int erts_is_code_ix_locked(void) +{ + return the_code_ix_lock; +} +#endif diff --git a/erts/emulator/beam/code_ix.h b/erts/emulator/beam/code_ix.h new file mode 100644 index 0000000000..6b2680044e --- /dev/null +++ b/erts/emulator/beam/code_ix.h @@ -0,0 +1,141 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2012. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* Description: + * This is the interface that facilitate changing the beam code + * (load,upgrade,delete) while allowing executing Erlang processes to + * access the code without any locks or other expensive memory barriers. + * + * The basic idea is to maintain several "logical copies" of the code. These + * copies are identified by a global 'code index', an integer of 0, 1 or 2. + * The code index is used as argument to code access structures like + * export, module, beam_catches, beam_ranges. + * + * The current 'active' code index is used to access the current running + * code. The 'staging' code index is used by the process that performs + * a code change operation. When a code change operation completes + * succesfully, the staging code index becomes the new active code index. + * + * The third code index is not explicitly used. It can be thought of as + * the "previous active" or the "next staging" index. It is needed to make + * sure that we do not reuse a code index for staging until we are sure + * that no executing BIFs are still referring it. + * We could get by with only two (0 and 1), but that would require that we + * must wait for all schedulers to re-schedule before each code change + * operation can start staging. + * + * Note that the 'code index' is very loosely coupled to the concept of + * 'current' and 'old' module versions. You can almost say that they are + * orthogonal to each other. Code index is an emulator global concept while + * 'current' and 'old' is specific for each module. + */ + +#ifndef __CODE_IX_H__ +#define __CODE_IX_H__ + +#ifndef __SYS_H__ +# ifdef HAVE_CONFIG_H +# include "config.h" +# endif +# include "sys.h" +#endif +struct process; + + +#define ERTS_NUM_CODE_IX 3 +typedef unsigned ErtsCodeIndex; + + +/* Called once at emulator initialization. + */ +void erts_code_ix_init(void); + +/* Return active code index. + * Is guaranteed to be valid until the calling BIF returns. + * To get a consistent view of the code, only one call to erts_active_code_ix() + * should be made and the returned ix reused within the same BIF call. + */ +ERTS_GLB_INLINE +ErtsCodeIndex erts_active_code_ix(void); + +/* Return staging code ix. + * Only used by a process performing code loading/upgrading/deleting/purging. + * code_ix must be locked. + */ +ERTS_GLB_INLINE +ErtsCodeIndex erts_staging_code_ix(void); + +/* Try seize exclusive code write permission. Needed for code staging. + * Main process lock (only) must be held. + * System thread progress must not be blocked. + * Caller is suspended and *must* yield if 0 is returned. + */ +int erts_try_seize_code_write_permission(struct process*); + +/* Release code write permission. + * Will resume any suspended waiters. + */ +void erts_release_code_write_permission(void); + +/* Prepare the "staging area" to be a complete copy of the active code. + * Code write permission must have been seized. + * Must be followed by calls to either "end" and "commit" or "abort" before + * code write permission can be released. + */ +void erts_start_staging_code_ix(void); + +/* End the staging. + * Preceded by "start" and must be followed by "commit". + */ +void erts_end_staging_code_ix(void); + +/* Set staging code index as new active code index. + * Preceded by "end". + */ +void erts_commit_staging_code_ix(void); + +/* Abort the staging. + * Preceded by "start". + */ +void erts_abort_staging_code_ix(void); + +#ifdef ERTS_ENABLE_LOCK_CHECK +int erts_is_code_ix_locked(void); +#endif + + + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +extern erts_smp_atomic32_t the_active_code_index; +extern erts_smp_atomic32_t the_staging_code_index; + +ERTS_GLB_INLINE ErtsCodeIndex erts_active_code_ix(void) +{ + return erts_smp_atomic32_read_nob(&the_active_code_index); +} +ERTS_GLB_INLINE ErtsCodeIndex erts_staging_code_ix(void) +{ + return erts_smp_atomic32_read_nob(&the_staging_code_index); +} + +#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ + +#endif /* !__CODE_IX_H__ */ + diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index 01fd63d5d9..300c2b708f 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -1312,7 +1312,7 @@ int erts_net_message(Port *prt, if (is_not_pid(from) || is_not_atom(to)){ goto invalid_message; } - rp = erts_whereis_process(NULL, 0, to, 0, ERTS_P2P_FLG_SMP_INC_REFC); + rp = erts_whereis_process(NULL, 0, to, 0, 0); if (rp) { Uint xsize = (type == DOP_REG_SEND ? 0 @@ -1338,7 +1338,6 @@ int erts_net_message(Port *prt, erts_queue_dist_message(rp, &locks, ede_copy, token); if (locks) erts_smp_proc_unlock(rp, locks); - erts_smp_proc_dec_refc(rp); } break; @@ -1364,7 +1363,7 @@ int erts_net_message(Port *prt, if (is_not_pid(to)) { goto invalid_message; } - rp = erts_pid2proc_opt(NULL, 0, to, 0, ERTS_P2P_FLG_SMP_INC_REFC); + rp = erts_proc_lookup(to); if (rp) { Uint xsize = type == DOP_SEND ? 0 : ERTS_HEAP_FRAG_SIZE(token_size); ErtsProcLocks locks = 0; @@ -1388,7 +1387,6 @@ int erts_net_message(Port *prt, erts_queue_dist_message(rp, &locks, ede_copy, token); if (locks) erts_smp_proc_unlock(rp, locks); - erts_smp_proc_dec_refc(rp); } break; @@ -1544,8 +1542,7 @@ int erts_net_message(Port *prt, if (is_not_pid(from) || is_not_internal_pid(to)) { goto invalid_message; } - rp = erts_pid2proc_opt(NULL, 0, to, rp_locks, - ERTS_P2P_FLG_SMP_INC_REFC); + rp = erts_pid2proc(NULL, 0, to, rp_locks); if (rp) { (void) erts_send_exit_signal(NULL, from, @@ -1556,7 +1553,6 @@ int erts_net_message(Port *prt, NULL, 0); erts_smp_proc_unlock(rp, rp_locks); - erts_smp_proc_dec_refc(rp); } break; } @@ -2242,7 +2238,7 @@ static void doit_print_monitor_info(ErtsMonitor *mon, void *vptdp) void *arg = ((struct print_to_data *) vptdp)->arg; Process *rp; ErtsMonitor *rmon; - rp = erts_pid2proc_unlocked(mon->pid); + rp = erts_proc_lookup(mon->pid); if (!rp || (rmon = erts_lookup_monitor(rp->monitors, mon->ref)) == NULL) { erts_print(to, arg, "Warning, stray monitor for: %T\n", mon->pid); } else if (mon->type == MON_ORIGIN) { @@ -2281,7 +2277,7 @@ static void doit_print_link_info2(ErtsLink *lnk, void *vpplc) static void doit_print_link_info(ErtsLink *lnk, void *vptdp) { - if (is_internal_pid(lnk->pid) && erts_pid2proc_unlocked(lnk->pid)) { + if (is_internal_pid(lnk->pid) && erts_proc_lookup(lnk->pid)) { PrintLinkContext plc = {(struct print_to_data *) vptdp, lnk->pid}; erts_doforall_links(ERTS_LINK_ROOT(lnk), &doit_print_link_info2, &plc); } @@ -2303,7 +2299,7 @@ static void doit_print_nodelink_info(ErtsLink *lnk, void *vpcontext) { PrintNodeLinkContext *pcontext = vpcontext; - if (is_internal_pid(lnk->pid) && erts_pid2proc_unlocked(lnk->pid)) + if (is_internal_pid(lnk->pid) && erts_proc_lookup(lnk->pid)) erts_print(pcontext->ptd.to, pcontext->ptd.arg, "Remote monitoring: %T %T\n", lnk->pid, pcontext->sysname); } @@ -2451,15 +2447,15 @@ BIF_RETTYPE setnode_2(BIF_ALIST_2) goto error; /* Check that all trap functions are defined !! */ - if (dsend2_trap->address == NULL || - dsend3_trap->address == NULL || + if (dsend2_trap->addressv[0] == NULL || + dsend3_trap->addressv[0] == NULL || /* dsend_nosuspend_trap->address == NULL ||*/ - dlink_trap->address == NULL || - dunlink_trap->address == NULL || - dmonitor_node_trap->address == NULL || - dgroup_leader_trap->address == NULL || - dmonitor_p_trap->address == NULL || - dexit_trap->address == NULL) { + dlink_trap->addressv[0] == NULL || + dunlink_trap->addressv[0] == NULL || + dmonitor_node_trap->addressv[0] == NULL || + dgroup_leader_trap->addressv[0] == NULL || + dmonitor_p_trap->addressv[0] == NULL || + dexit_trap->addressv[0] == NULL) { goto error; } @@ -2706,9 +2702,8 @@ BIF_RETTYPE dist_exit_3(BIF_ALIST_3) } else { lp_locks = ERTS_PROC_LOCKS_XSIG_SEND; - lp = erts_pid2proc_opt(BIF_P, ERTS_PROC_LOCK_MAIN, - local, lp_locks, - ERTS_P2P_FLG_SMP_INC_REFC); + lp = erts_pid2proc(BIF_P, ERTS_PROC_LOCK_MAIN, + local, lp_locks); if (!lp) { BIF_RET(am_true); /* ignore */ } @@ -2727,9 +2722,7 @@ BIF_RETTYPE dist_exit_3(BIF_ALIST_3) lp_locks &= ~ERTS_PROC_LOCK_MAIN; #endif erts_smp_proc_unlock(lp, lp_locks); - if (lp != BIF_P) - erts_smp_proc_dec_refc(lp); - else { + if (lp == BIF_P) { /* * We may have exited current process and may have to take action. */ diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index 6fce032f9d..918a800555 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -2182,9 +2182,9 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) if (want.code) { size.code = module_table_sz(); size.code += export_table_sz(); - size.code += export_list_size() * sizeof(Export); + size.code += export_entries_sz(); size.code += erts_fun_table_sz(); - size.code += allocated_modules*sizeof(Range); + size.code += erts_ranges_sz(); size.code += erts_total_code_size; } @@ -2332,7 +2332,7 @@ erts_allocated_areas(int *print_to_p, void *print_to_arg, void *proc) values[i].arity = 2; values[i].name = "export_list"; - values[i].ui[0] = export_list_size() * sizeof(Export); + values[i].ui[0] = export_entries_sz(); i++; values[i].arity = 2; @@ -2347,7 +2347,7 @@ erts_allocated_areas(int *print_to_p, void *print_to_arg, void *proc) values[i].arity = 2; values[i].name = "module_refs"; - values[i].ui[0] = allocated_modules*sizeof(Range); + values[i].ui[0] = erts_ranges_sz(); i++; values[i].arity = 2; diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index d4ef9cc553..134de6dfa6 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -164,6 +164,7 @@ type MSG_REF FIXED_SIZE PROCESSES msg_ref type MSG_ROOTS TEMPORARY PROCESSES msg_roots type ROOTSET TEMPORARY PROCESSES root_set type LOADER_TMP TEMPORARY CODE loader_tmp +type PREPARED_CODE SHORT_LIVED CODE prepared_code type BIF_TIMER_TABLE LONG_LIVED SYSTEM bif_timer_table type SL_BIF_TIMER SHORT_LIVED PROCESSES bif_timer_sl type LL_BIF_TIMER STANDARD PROCESSES bif_timer_ll @@ -263,6 +264,8 @@ type ZLIB STANDARD SYSTEM zlib type CPU_GRPS_MAP LONG_LIVED SYSTEM cpu_groups_map type AUX_WORK_TMO LONG_LIVED SYSTEM aux_work_timeouts type MISC_AUX_WORK_Q LONG_LIVED SYSTEM misc_aux_work_q +type CODE_IX_LOCK_Q SHORT_LIVED SYSTEM code_ix_lock_q +type PROC_INTERVAL LONG_LIVED SYSTEM process_interval +if threads_no_smp # Need thread safe allocs, but std_alloc and fix_alloc are not; @@ -322,6 +325,8 @@ type SL_PTIMER SHORT_LIVED SYSTEM ptimer_sl type LL_PTIMER STANDARD SYSTEM ptimer_ll type SYS_MSG_Q SHORT_LIVED PROCESSES system_messages_queue type FP_EXCEPTION LONG_LIVED SYSTEM fp_exception +type LL_MPATHS LONG_LIVED SYSTEM ll_migration_paths +type SL_MPATHS SHORT_LIVED SYSTEM sl_migration_paths +endif +if hipe diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index e0d525bdde..62225d3572 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -832,14 +832,14 @@ init_dd_queue(ErtsAllctrDDQueue_t *ddq) static ERTS_INLINE erts_aint_t ddq_managed_thread_enqueue(ErtsAllctrDDQueue_t *ddq, void *ptr) { - erts_aint_t ilast, itmp; + erts_aint_t first_ilast, ilast, itmp; ErtsAllctrDDBlock_t *this = ptr; erts_atomic_init_nob(&this->atmc_next, ERTS_AINT_NULL); /* Enqueue at end of list... */ - ilast = erts_atomic_read_nob(&ddq->tail.data.last); + first_ilast = ilast = erts_atomic_read_nob(&ddq->tail.data.last); while (1) { ErtsAllctrDDBlock_t *last = (ErtsAllctrDDBlock_t *) ilast; itmp = erts_atomic_cmpxchg_mb(&last->atmc_next, @@ -853,8 +853,11 @@ ddq_managed_thread_enqueue(ErtsAllctrDDQueue_t *ddq, void *ptr) /* Move last pointer forward... */ while (1) { if (erts_atomic_read_rb(&this->atmc_next) != ERTS_AINT_NULL) { - /* Someone else will move it forward */ - return erts_atomic_read_rb(&ddq->tail.data.last); + ilast = erts_atomic_read_rb(&ddq->tail.data.last); + if (first_ilast != ilast) { + /* Someone else will move it forward */ + return ilast; + } } itmp = erts_atomic_cmpxchg_mb(&ddq->tail.data.last, (erts_aint_t) this, diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c index cc4f2be8eb..474151d454 100644 --- a/erts/emulator/beam/erl_bif_binary.c +++ b/erts/emulator/beam/erl_bif_binary.c @@ -72,52 +72,29 @@ binary_matches(Process *p, Eterm arg1, Eterm arg2, Eterm arg3); void erts_init_bif_binary(void) { - sys_memset((void *) &binary_match_trap_export, 0, sizeof(Export)); - binary_match_trap_export.address = &binary_match_trap_export.code[3]; - binary_match_trap_export.code[0] = am_erlang; - binary_match_trap_export.code[1] = am_binary_match_trap; - binary_match_trap_export.code[2] = 3; - binary_match_trap_export.code[3] = (BeamInstr) em_apply_bif; - binary_match_trap_export.code[4] = (BeamInstr) &binary_match_trap; - - sys_memset((void *) &binary_matches_trap_export, 0, sizeof(Export)); - binary_matches_trap_export.address = &binary_matches_trap_export.code[3]; - binary_matches_trap_export.code[0] = am_erlang; - binary_matches_trap_export.code[1] = am_binary_matches_trap; - binary_matches_trap_export.code[2] = 3; - binary_matches_trap_export.code[3] = (BeamInstr) em_apply_bif; - binary_matches_trap_export.code[4] = (BeamInstr) &binary_matches_trap; - - sys_memset((void *) &binary_longest_prefix_trap_export, 0, sizeof(Export)); - binary_longest_prefix_trap_export.address = &binary_longest_prefix_trap_export.code[3]; - binary_longest_prefix_trap_export.code[0] = am_erlang; - binary_longest_prefix_trap_export.code[1] = am_binary_longest_prefix_trap; - binary_longest_prefix_trap_export.code[2] = 3; - binary_longest_prefix_trap_export.code[3] = (BeamInstr) em_apply_bif; - binary_longest_prefix_trap_export.code[4] = (BeamInstr) &binary_longest_prefix_trap; - - sys_memset((void *) &binary_longest_suffix_trap_export, 0, sizeof(Export)); - binary_longest_suffix_trap_export.address = &binary_longest_suffix_trap_export.code[3]; - binary_longest_suffix_trap_export.code[0] = am_erlang; - binary_longest_suffix_trap_export.code[1] = am_binary_longest_suffix_trap; - binary_longest_suffix_trap_export.code[2] = 3; - binary_longest_suffix_trap_export.code[3] = (BeamInstr) em_apply_bif; - binary_longest_suffix_trap_export.code[4] = (BeamInstr) &binary_longest_suffix_trap; - - sys_memset((void *) &binary_bin_to_list_trap_export, 0, sizeof(Export)); - binary_bin_to_list_trap_export.address = &binary_bin_to_list_trap_export.code[3]; - binary_bin_to_list_trap_export.code[0] = am_erlang; - binary_bin_to_list_trap_export.code[1] = am_binary_bin_to_list_trap; - binary_bin_to_list_trap_export.code[2] = 3; - binary_bin_to_list_trap_export.code[3] = (BeamInstr) em_apply_bif; - binary_bin_to_list_trap_export.code[4] = (BeamInstr) &binary_bin_to_list_trap; - sys_memset((void *) &binary_copy_trap_export, 0, sizeof(Export)); - binary_copy_trap_export.address = &binary_copy_trap_export.code[3]; - binary_copy_trap_export.code[0] = am_erlang; - binary_copy_trap_export.code[1] = am_binary_copy_trap; - binary_copy_trap_export.code[2] = 2; - binary_copy_trap_export.code[3] = (BeamInstr) em_apply_bif; - binary_copy_trap_export.code[4] = (BeamInstr) &binary_copy_trap; + erts_init_trap_export(&binary_match_trap_export, + am_erlang, am_binary_match_trap, 3, + &binary_match_trap); + + erts_init_trap_export(&binary_matches_trap_export, + am_erlang, am_binary_matches_trap, 3, + &binary_matches_trap); + + erts_init_trap_export(&binary_longest_prefix_trap_export, + am_erlang, am_binary_longest_prefix_trap, 3, + &binary_longest_prefix_trap); + + erts_init_trap_export(&binary_longest_suffix_trap_export, + am_erlang, am_binary_longest_suffix_trap, 3, + &binary_longest_suffix_trap); + + erts_init_trap_export(&binary_bin_to_list_trap_export, + am_erlang, am_binary_bin_to_list_trap, 3, + &binary_bin_to_list_trap); + + erts_init_trap_export(&binary_copy_trap_export, + am_erlang, am_binary_copy_trap, 2, + &binary_copy_trap); max_loop_limit = 0; return; diff --git a/erts/emulator/beam/erl_bif_chksum.c b/erts/emulator/beam/erl_bif_chksum.c index 06b7ffdf32..ff5ce3cc7a 100644 --- a/erts/emulator/beam/erl_bif_chksum.c +++ b/erts/emulator/beam/erl_bif_chksum.c @@ -42,16 +42,9 @@ static Export chksum_md5_2_exp; void erts_init_bif_chksum(void) { /* Non visual BIF to trap to. */ - memset(&chksum_md5_2_exp, 0, sizeof(Export)); - chksum_md5_2_exp.address = - &chksum_md5_2_exp.code[3]; - chksum_md5_2_exp.code[0] = am_erlang; - chksum_md5_2_exp.code[1] = am_atom_put("md5_trap",8); - chksum_md5_2_exp.code[2] = 2; - chksum_md5_2_exp.code[3] = - (BeamInstr) em_apply_bif; - chksum_md5_2_exp.code[4] = - (BeamInstr) &md5_2; + erts_init_trap_export(&chksum_md5_2_exp, + am_erlang, am_atom_put("md5_trap",8), 2, + &md5_2); } diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c index c338ee1c4b..7f7c975e78 100644 --- a/erts/emulator/beam/erl_bif_ddll.c +++ b/erts/emulator/beam/erl_bif_ddll.c @@ -368,13 +368,11 @@ BIF_RETTYPE erl_ddll_try_load_3(BIF_ALIST_3) #endif for (j = 0; j < erts_max_ports; j++) { Port* prt = &erts_port[j]; -#ifdef DDLL_SMP erts_smp_port_state_lock(prt); -#endif if (!(prt->status & FREE_PORT_FLAGS) && prt->drv_ptr->handle == dh) { -#if DDLL_SMP erts_smp_atomic_inc_nob(&prt->refc); +#if DDLL_SMP /* Extremely rare spinlock */ while(prt->status & ERTS_PORT_SFLG_INITIALIZING) { erts_smp_port_state_unlock(prt); @@ -598,13 +596,11 @@ done: #endif for (j = 0; j < erts_max_ports; j++) { Port* prt = &erts_port[j]; -#if DDLL_SMP erts_smp_port_state_lock(prt); -#endif if (!(prt->status & FREE_PORT_FLAGS) && prt->drv_ptr->handle == dh) { -#if DDLL_SMP erts_smp_atomic_inc_nob(&prt->refc); +#if DDLL_SMP /* Extremely rare spinlock */ while(prt->status & ERTS_PORT_SFLG_INITIALIZING) { erts_smp_port_state_unlock(prt); @@ -1060,13 +1056,11 @@ void erts_ddll_proc_dead(Process *p, ErtsProcLocks plocks) #endif for (j = 0; j < erts_max_ports; j++) { Port* prt = &erts_port[j]; -#if DDLL_SMP erts_smp_port_state_lock(prt); -#endif if (!(prt->status & FREE_PORT_FLAGS) && prt->drv_ptr->handle == dh) { -#if DDLL_SMP erts_smp_atomic_inc_nob(&prt->refc); +#if DDLL_SMP while(prt->status & ERTS_PORT_SFLG_INITIALIZING) { erts_smp_port_state_unlock(prt); erts_smp_port_state_lock(prt); diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index f56b00287f..e2f7c8673f 100755 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -1338,13 +1338,15 @@ process_info_aux(Process *BIF_P, hp = HAlloc(BIF_P, 3); break; - case am_trap_exit: + case am_trap_exit: { + erts_aint32_t state = erts_smp_atomic32_read_nob(&rp->state); hp = HAlloc(BIF_P, 3); - if (rp->flags & F_TRAPEXIT) + if (state & ERTS_PSFLG_TRAP_EXIT) res = am_true; else res = am_false; break; + } case am_error_handler: hp = HAlloc(BIF_P, 3); @@ -3098,15 +3100,13 @@ BIF_RETTYPE is_process_alive_1(BIF_ALIST_1) if(internal_pid_index(BIF_ARG_1) >= erts_max_processes) BIF_ERROR(BIF_P, BADARG); - rp = erts_pid2proc(BIF_P, ERTS_PROC_LOCK_MAIN, - BIF_ARG_1, ERTS_PROC_LOCK_STATUS); + rp = erts_proc_lookup(BIF_ARG_1); if (!rp) { BIF_RET(am_false); } else { - int have_pending_exit = ERTS_PROC_PENDING_EXIT(rp); - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); - if (have_pending_exit) + if (erts_smp_atomic32_read_acqb(&rp->state) + & (ERTS_PSFLG_PENDING_EXIT|ERTS_PSFLG_EXITING)) ERTS_BIF_AWAIT_X_DATA_TRAP(BIF_P, BIF_ARG_1, am_false); else BIF_RET(am_true); @@ -3710,9 +3710,8 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) && is_internal_pid(tp[2])) { int xres; ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND; - Process *rp = erts_pid2proc_opt(BIF_P, ERTS_PROC_LOCK_MAIN, - tp[2], rp_locks, - ERTS_P2P_FLG_SMP_INC_REFC); + Process *rp = erts_pid2proc(BIF_P, ERTS_PROC_LOCK_MAIN, + tp[2], rp_locks); if (!rp) { DECL_AM(dead); BIF_RET(AM_dead); @@ -3738,7 +3737,6 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) rp_locks &= ~ERTS_PROC_LOCK_MAIN; #endif erts_smp_proc_unlock(rp, rp_locks); - erts_smp_proc_dec_refc(rp); if (xres > 1) { DECL_AM(message); BIF_RET(AM_message); @@ -3949,7 +3947,7 @@ static Eterm lcnt_build_lock_term(Eterm **hpp, Uint *szp, erts_lcnt_lock_t *lock id = am_atom_put(ltype, strlen(ltype)); } else if (lock->flag & ERTS_LCNT_LT_PROCLOCK) { /* use registered names as id's for process locks if available */ - proc = erts_pid2proc_unlocked(lock->id); + proc = erts_proc_lookup(lock->id); if (proc && proc->reg) { id = proc->reg->name; } else { diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c index 6b843d2e08..b036c5ef5c 100644 --- a/erts/emulator/beam/erl_bif_re.c +++ b/erts/emulator/beam/erl_bif_re.c @@ -71,14 +71,9 @@ void erts_init_bif_re(void) erts_pcre_stack_free = &erts_erts_pcre_stack_free; default_table = NULL; /* ISO8859-1 default, forced into pcre */ max_loop_limit = CONTEXT_REDS * LOOP_FACTOR; - - sys_memset((void *) &re_exec_trap_export, 0, sizeof(Export)); - re_exec_trap_export.address = &re_exec_trap_export.code[3]; - re_exec_trap_export.code[0] = am_erlang; - re_exec_trap_export.code[1] = am_re_run_trap; - re_exec_trap_export.code[2] = 3; - re_exec_trap_export.code[3] = (BeamInstr) em_apply_bif; - re_exec_trap_export.code[4] = (BeamInstr) &re_exec_trap; + + erts_init_trap_export(&re_exec_trap_export, am_erlang, am_re_run_trap, 3, + &re_exec_trap); grun_trap_exportp = erts_export_put(am_re,am_grun,3); urun_trap_exportp = erts_export_put(am_re,am_urun,3); diff --git a/erts/emulator/beam/erl_bif_timer.c b/erts/emulator/beam/erl_bif_timer.c index d806be0704..525b11f61c 100644 --- a/erts/emulator/beam/erl_bif_timer.c +++ b/erts/emulator/beam/erl_bif_timer.c @@ -324,10 +324,9 @@ bif_timer_timeout(ErtsBifTimer* btm) ASSERT(!erts_get_current_process()); if (btm->flags & BTM_FLG_BYNAME) - rp = erts_whereis_process(NULL,0,btm->receiver.name,0,ERTS_P2P_FLG_SMP_INC_REFC); + rp = erts_whereis_process(NULL, 0, btm->receiver.name, 0, 0); else { rp = btm->receiver.proc.ess; - erts_smp_proc_inc_refc(rp); unlink_proc(btm); } @@ -379,7 +378,6 @@ bif_timer_timeout(ErtsBifTimer* btm) #endif ); erts_smp_proc_unlock(rp, rp_locks); - erts_smp_proc_dec_refc(rp); } } diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c index 80f774523c..7f1b02b9b4 100644 --- a/erts/emulator/beam/erl_bif_trace.c +++ b/erts/emulator/beam/erl_bif_trace.c @@ -60,8 +60,8 @@ static Eterm trace_info_pid(Process* p, Eterm pid_spec, Eterm key); static Eterm trace_info_func(Process* p, Eterm pid_spec, Eterm key); static Eterm trace_info_on_load(Process* p, Eterm key); -static int setup_func_trace(Export* ep, void* match_prog); -static int reset_func_trace(Export* ep); +static int setup_func_trace(Export* ep, void* match_prog, ErtsCodeIndex); +static int reset_func_trace(Export* ep, ErtsCodeIndex); static void reset_bif_trace(int bif_index); static void setup_bif_trace(int bif_index); static void set_trace_bif(int bif_index, void* match_prog); @@ -98,7 +98,7 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) { DeclareTmpHeap(mfa,3,p); /* Not really heap here, but might be when setting pattern */ int i; - int matches = 0; + int matches = -1; int specified = 0; enum erts_break_op on; Binary* match_prog_set; @@ -108,6 +108,9 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) Process *meta_tracer_proc = p; Eterm meta_tracer_pid = p->id; + if (!erts_try_seize_code_write_permission(p)) { + ERTS_BIF_YIELD3(bif_export[BIF_trace_pattern_3], p, MFA, Pattern, flaglist); + } erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN); erts_smp_thr_progress_block(); @@ -241,7 +244,6 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) erts_default_meta_match_spec = NULL; erts_default_meta_tracer_pid = NIL; } - MatchSetUnref(match_prog_set); if (erts_default_trace_pattern_flags.breakpoint && flags.breakpoint) { /* Breakpoint trace -> breakpoint trace */ @@ -297,8 +299,7 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) erts_default_trace_pattern_is_on = !!flags.breakpoint; } } - - goto done; + matches = 0; } else if (is_tuple(MFA)) { Eterm *tp = tuple_val(MFA); if (tp[0] != make_arityval(3)) { @@ -322,35 +323,29 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) if (is_small(mfa[2])) { mfa[2] = signed_val(mfa[2]); } - } else { - goto error; - } - if (meta_tracer_proc) { - meta_tracer_proc->trace_flags |= F_TRACER; - } - - - matches = erts_set_trace_pattern(mfa, specified, - match_prog_set, match_prog_set, - on, flags, meta_tracer_pid); - MatchSetUnref(match_prog_set); - - done: - UnUseTmpHeap(3,p); - erts_smp_thr_progress_unblock(); - erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); + if (meta_tracer_proc) { + meta_tracer_proc->trace_flags |= F_TRACER; + } - return make_small(matches); + matches = erts_set_trace_pattern(mfa, specified, + match_prog_set, match_prog_set, + on, flags, meta_tracer_pid); + } error: - MatchSetUnref(match_prog_set); - UnUseTmpHeap(3,p); erts_smp_thr_progress_unblock(); erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); - BIF_ERROR(p, BADARG); + erts_release_code_write_permission(); + + if (matches >= 0) { + return make_small(matches); + } + else { + BIF_ERROR(p, BADARG); + } } void @@ -372,7 +367,10 @@ erts_get_default_trace_pattern(int *trace_pattern_is_on, *meta_tracer_pid = erts_default_meta_tracer_pid; } - +int erts_is_default_trace_enabled(void) +{ + return erts_default_trace_pattern_is_on; +} Uint erts_trace_flag2bit(Eterm flag) @@ -466,6 +464,10 @@ Eterm trace_3(BIF_ALIST_3) BIF_ERROR(p, BADARG); } + if (!erts_try_seize_code_write_permission(BIF_P)) { + ERTS_BIF_YIELD3(bif_export[BIF_trace_3], BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); + } + if (is_nil(tracer) || is_internal_pid(tracer)) { Process *tracer_proc = erts_pid2proc(p, ERTS_PROC_LOCK_MAIN, @@ -653,8 +655,7 @@ Eterm trace_3(BIF_ALIST_3) if (procs || mods) { /* tracing of processes */ for (i = 0; i < erts_max_processes; i++) { - Process* tracee_p = process_tab[i]; - + Process* tracee_p = erts_pix2proc(i); if (! tracee_p) continue; if (tracer != NIL) { @@ -730,6 +731,7 @@ Eterm trace_3(BIF_ALIST_3) erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); } #endif + erts_release_code_write_permission(); BIF_RET(make_small(matches)); @@ -745,6 +747,7 @@ Eterm trace_3(BIF_ALIST_3) erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); } #endif + erts_release_code_write_permission(); BIF_ERROR(p, BADARG); } @@ -772,8 +775,7 @@ static int port_already_traced(Process *c_p, Port *tracee_port, Eterm tracer) return 1; } else if(is_internal_pid(tracee_port->tracer_proc)) { - Process *tracer_p = erts_pid2proc(c_p, ERTS_PROC_LOCK_MAIN, - tracee_port->tracer_proc, 0); + Process *tracer_p = erts_proc_lookup(tracee_port->tracer_proc); if (!tracer_p) { /* Current trace process now invalid * - discard it and approve the new. */ @@ -813,8 +815,7 @@ static int already_traced(Process *c_p, Process *tracee_p, Eterm tracer) return 1; } else if(is_internal_pid(tracee_p->tracer_proc)) { - Process *tracer_p = erts_pid2proc(c_p, ERTS_PROC_LOCK_MAIN, - tracee_p->tracer_proc, 0); + Process *tracer_p = erts_proc_lookup(tracee_p->tracer_proc); if (!tracer_p) { /* Current trace process now invalid * - discard it and approve the new. */ @@ -876,7 +877,7 @@ trace_info_pid(Process* p, Eterm pid_spec, Eterm key) } if (is_internal_pid(tracer)) { - if (!erts_pid2proc(p, ERTS_PROC_LOCK_MAIN, tracer, 0)) { + if (!erts_proc_lookup(tracer)) { reset_tracer: tracee->trace_flags &= ~TRACEE_FLAGS; trace_flags = tracee->trace_flags; @@ -990,7 +991,7 @@ static int function_is_traced(Process *p, e.code[1] = mfa[1]; e.code[2] = mfa[2]; if ((ep = export_get(&e)) != NULL) { - if (ep->address == ep->code+3 && + if (ep->addressv[erts_active_code_ix()] == ep->code+3 && ep->code[3] != (BeamInstr) em_call_error_handler) { if (ep->code[3] == (BeamInstr) em_call_traced_function) { *ms = ep->match_prog_set; @@ -1333,6 +1334,7 @@ erts_set_trace_pattern(Eterm* mfa, int specified, int on, struct trace_pattern_flags flags, Eterm meta_tracer_pid) { + const ErtsCodeIndex code_ix = erts_active_code_ix(); int matches = 0; int i; @@ -1340,8 +1342,8 @@ erts_set_trace_pattern(Eterm* mfa, int specified, * First work on normal functions (not real BIFs). */ - for (i = 0; i < export_list_size(); i++) { - Export* ep = export_list(i); + for (i = 0; i < export_list_size(code_ix); i++) { + Export* ep = export_list(i, code_ix); int j; if (ExportIsBuiltIn(ep)) { @@ -1355,11 +1357,11 @@ erts_set_trace_pattern(Eterm* mfa, int specified, if (j == specified) { if (on) { if (! flags.breakpoint) - matches += setup_func_trace(ep, match_prog_set); + matches += setup_func_trace(ep, match_prog_set, code_ix); else - reset_func_trace(ep); + reset_func_trace(ep, code_ix); } else if (! flags.breakpoint) { - matches += reset_func_trace(ep); + matches += reset_func_trace(ep, code_ix); } } } @@ -1522,9 +1524,11 @@ erts_set_trace_pattern(Eterm* mfa, int specified, */ static int -setup_func_trace(Export* ep, void* match_prog) +setup_func_trace(Export* ep, void* match_prog, ErtsCodeIndex code_ix) { - if (ep->address == ep->code+3) { + Module* modp; + + if (ep->addressv[code_ix] == ep->code+3) { if (ep->code[3] == (BeamInstr) em_call_error_handler) { return 0; } else if (ep->code[3] == (BeamInstr) em_call_traced_function) { @@ -1543,15 +1547,19 @@ setup_func_trace(Export* ep, void* match_prog) /* * Currently no trace support for native code. */ - if (erts_is_native_break(ep->address)) { + if (erts_is_native_break(ep->addressv[code_ix])) { return 0; } - + ep->code[3] = (BeamInstr) em_call_traced_function; - ep->code[4] = (BeamInstr) ep->address; - ep->address = ep->code+3; + ep->code[4] = (BeamInstr) ep->addressv[code_ix]; + ep->addressv[code_ix] = ep->code+3; ep->match_prog_set = match_prog; MatchSetRef(ep->match_prog_set); + + modp = erts_get_module(ep->code[0], code_ix); + ASSERT(modp); + modp->curr.num_traced_exports++; return 1; } @@ -1584,13 +1592,17 @@ static void set_trace_bif(int bif_index, void* match_prog) { */ static int -reset_func_trace(Export* ep) +reset_func_trace(Export* ep, ErtsCodeIndex code_ix) { - if (ep->address == ep->code+3) { + if (ep->addressv[code_ix] == ep->code+3) { if (ep->code[3] == (BeamInstr) em_call_error_handler) { return 0; } else if (ep->code[3] == (BeamInstr) em_call_traced_function) { - ep->address = (Uint *) ep->code[4]; + Module* modp = erts_get_module(ep->code[0], code_ix); + ASSERT(modp); + modp->curr.num_traced_exports--; + + ep->addressv[code_ix] = (Uint *) ep->code[4]; MatchSetUnref(ep->match_prog_set); ep->match_prog_set = NULL; return 1; @@ -1605,7 +1617,7 @@ reset_func_trace(Export* ep) /* * Currently no trace support for native code. */ - if (erts_is_native_break(ep->address)) { + if (erts_is_native_break(ep->addressv[code_ix])) { return 0; } @@ -2142,9 +2154,9 @@ BIF_RETTYPE system_profile_2(BIF_ALIST_2) /* Check if valid process, no locks are taken */ if (is_internal_pid(profiler)) { - if (internal_pid_index(profiler) >= erts_max_processes) goto error; - profiler_p = process_tab[internal_pid_index(profiler)]; - if (INVALID_PID(profiler_p, profiler)) goto error; + profiler_p = erts_proc_lookup(profiler); + if (!profiler_p) + goto error; } else if (is_internal_port(profiler)) { if (internal_port_index(profiler) >= erts_max_ports) goto error; profiler_port = &erts_port[internal_port_index(profiler)]; diff --git a/erts/emulator/beam/erl_cpu_topology.c b/erts/emulator/beam/erl_cpu_topology.c index fe3693d0ca..41e71881a0 100644 --- a/erts/emulator/beam/erl_cpu_topology.c +++ b/erts/emulator/beam/erl_cpu_topology.c @@ -486,7 +486,7 @@ erts_sched_check_cpu_bind_post_suspend(ErtsSchedulerData *esdp) erts_thr_set_main_status(1, (int) esdp->no); /* Make sure we check if we should bind to a cpu or not... */ - esdp->run_queue->flags |= ERTS_RUNQ_FLG_CHK_CPU_BIND; + ERTS_RUNQ_FLGS_SET(esdp->run_queue, ERTS_RUNQ_FLG_CHK_CPU_BIND); } #endif @@ -498,9 +498,6 @@ erts_sched_check_cpu_bind(ErtsSchedulerData *esdp) erts_cpu_groups_map_t *cgm; erts_cpu_groups_callback_list_t *cgcl; erts_cpu_groups_callback_call_t *cgcc; -#ifdef ERTS_SMP - esdp->run_queue->flags &= ~ERTS_RUNQ_FLG_CHK_CPU_BIND; -#endif erts_smp_runq_unlock(esdp->run_queue); erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx); cpu_id = scheduler2cpu_map[esdp->no].bind_id; diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index 51bdf53823..5bd8163968 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -2816,7 +2816,6 @@ void init_db(void) { DbTable init_tb; int i; - extern BeamInstr* em_apply_bif; Eterm *hp; unsigned bits; size_t size; @@ -2850,7 +2849,7 @@ void init_db(void) else db_max_tabs = user_requested_db_max_tabs; - bits = erts_fit_in_bits(db_max_tabs-1); + bits = erts_fit_in_bits_int32(db_max_tabs-1); if (bits > SMALL_BITS) { erl_exit(1,"Max limit for ets tabled too high %u (max %u).", db_max_tabs, ((Uint)1)<<SMALL_BITS); @@ -2949,49 +2948,24 @@ void init_db(void) } /* Non visual BIF to trap to. */ - memset(&ets_select_delete_continue_exp, 0, sizeof(Export)); - ets_select_delete_continue_exp.address = - &ets_select_delete_continue_exp.code[3]; - ets_select_delete_continue_exp.code[0] = am_ets; - ets_select_delete_continue_exp.code[1] = am_atom_put("delete_trap",11); - ets_select_delete_continue_exp.code[2] = 1; - ets_select_delete_continue_exp.code[3] = - (BeamInstr) em_apply_bif; - ets_select_delete_continue_exp.code[4] = - (BeamInstr) &ets_select_delete_1; + erts_init_trap_export(&ets_select_delete_continue_exp, + am_ets, am_atom_put("delete_trap",11), 1, + &ets_select_delete_1); /* Non visual BIF to trap to. */ - memset(&ets_select_count_continue_exp, 0, sizeof(Export)); - ets_select_count_continue_exp.address = - &ets_select_count_continue_exp.code[3]; - ets_select_count_continue_exp.code[0] = am_ets; - ets_select_count_continue_exp.code[1] = am_atom_put("count_trap",11); - ets_select_count_continue_exp.code[2] = 1; - ets_select_count_continue_exp.code[3] = - (BeamInstr) em_apply_bif; - ets_select_count_continue_exp.code[4] = - (BeamInstr) &ets_select_count_1; + erts_init_trap_export(&ets_select_count_continue_exp, + am_ets, am_atom_put("count_trap",11), 1, + &ets_select_count_1); /* Non visual BIF to trap to. */ - memset(&ets_select_continue_exp, 0, sizeof(Export)); - ets_select_continue_exp.address = - &ets_select_continue_exp.code[3]; - ets_select_continue_exp.code[0] = am_ets; - ets_select_continue_exp.code[1] = am_atom_put("select_trap",11); - ets_select_continue_exp.code[2] = 1; - ets_select_continue_exp.code[3] = - (BeamInstr) em_apply_bif; - ets_select_continue_exp.code[4] = - (BeamInstr) &ets_select_trap_1; + erts_init_trap_export(&ets_select_continue_exp, + am_ets, am_atom_put("select_trap",11), 1, + &ets_select_trap_1); /* Non visual BIF to trap to. */ - memset(&ets_delete_continue_exp, 0, sizeof(Export)); - ets_delete_continue_exp.address = &ets_delete_continue_exp.code[3]; - ets_delete_continue_exp.code[0] = am_ets; - ets_delete_continue_exp.code[1] = am_atom_put("delete_trap",11); - ets_delete_continue_exp.code[2] = 1; - ets_delete_continue_exp.code[3] = (BeamInstr) em_apply_bif; - ets_delete_continue_exp.code[4] = (BeamInstr) &ets_delete_trap; + erts_init_trap_export(&ets_delete_continue_exp, + am_ets, am_atom_put("delete_trap",11), 1, + &ets_delete_trap); hp = ms_delete_all_buff; ms_delete_all = CONS(hp, am_true, NIL); @@ -3124,7 +3098,7 @@ retry: if (to_proc == NULL) { return 0; /* heir not alive, table still mine */ } - if (erts_cmp_timeval(&to_proc->started, &tb->common.heir_started) != 0) { + if (to_proc->started_interval != tb->common.heir_started_interval) { erts_smp_proc_unlock(to_proc, to_locks); return 0; /* heir dead and pid reused, table still mine */ } @@ -3520,14 +3494,14 @@ static void set_heir(Process* me, DbTable* tb, Eterm heir, UWord heir_data) return; } if (heir == me->id) { - tb->common.heir_started = me->started; + erts_ensure_later_proc_interval(me->started_interval); + tb->common.heir_started_interval = me->started_interval; } else { - Process* heir_proc= erts_pid2proc_opt(me, ERTS_PROC_LOCK_MAIN, heir, - 0, ERTS_P2P_FLG_SMP_INC_REFC); + Process* heir_proc= erts_proc_lookup(heir); if (heir_proc != NULL) { - tb->common.heir_started = heir_proc->started; - erts_smp_proc_dec_refc(heir_proc); + erts_ensure_later_proc_interval(heir_proc->started_interval); + tb->common.heir_started_interval = heir_proc->started_interval; } else { tb->common.heir = am_none; } diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c index 312050b931..faa7f31d99 100644 --- a/erts/emulator/beam/erl_db_tree.c +++ b/erts/emulator/beam/erl_db_tree.c @@ -452,16 +452,8 @@ DbTableMethod db_tree = void db_initialize_tree(void) { - memset(&ets_select_reverse_exp, 0, sizeof(Export)); - ets_select_reverse_exp.address = - &ets_select_reverse_exp.code[3]; - ets_select_reverse_exp.code[0] = am_ets; - ets_select_reverse_exp.code[1] = am_reverse; - ets_select_reverse_exp.code[2] = 3; - ets_select_reverse_exp.code[3] = - (BeamInstr) em_apply_bif; - ets_select_reverse_exp.code[4] = - (BeamInstr) &ets_select_reverse; + erts_init_trap_export(&ets_select_reverse_exp, am_ets, am_reverse, 3, + &ets_select_reverse); return; }; diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h index 6a96e174e1..ff5982640d 100644 --- a/erts/emulator/beam/erl_db_util.h +++ b/erts/emulator/beam/erl_db_util.h @@ -219,7 +219,7 @@ typedef struct db_table_common { Eterm owner; /* Pid of the creator */ Eterm heir; /* Pid of the heir */ UWord heir_data; /* To send in ETS-TRANSFER (is_immed or (DbTerm*) */ - SysTimeval heir_started; /* To further identify the heir */ + Uint64 heir_started_interval; /* To further identify the heir */ Eterm the_name; /* an atom */ Eterm id; /* atom | integer */ DbTableMethod* meth; /* table methods */ diff --git a/erts/emulator/beam/erl_fun.h b/erts/emulator/beam/erl_fun.h index 54cfd6aa83..217066ab11 100644 --- a/erts/emulator/beam/erl_fun.h +++ b/erts/emulator/beam/erl_fun.h @@ -77,8 +77,6 @@ ErlFunEntry* erts_get_fun_entry(Eterm mod, int uniq, int index); ErlFunEntry* erts_put_fun_entry2(Eterm mod, int old_uniq, int old_index, byte* uniq, int index, int arity); -ErlFunEntry* erts_get_fun_entry2(Eterm mod, int old_uniq, int old_index, - byte* uniq, int index, int arity); void erts_erase_fun_entry(ErlFunEntry* fe); void erts_cleanup_funs(ErlFunThing* funp); diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 52a6e52e6c..6075a527c3 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -357,11 +357,7 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj) trace_gc(p, am_gc_start); } - erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); - p->gcstatus = p->status; - p->status = P_GARBING; - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); - + erts_smp_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC); if (erts_system_monitor_long_gc != 0) { get_now(&ms1, &s1, &us1); } @@ -404,9 +400,8 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj) ErtsGcQuickSanityCheck(p); - erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); - p->status = p->gcstatus; - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC); + if (IS_TRACED_FL(p, F_TRACE_GC)) { trace_gc(p, am_gc_end); } @@ -490,10 +485,7 @@ erts_garbage_collect_hibernate(Process* p) /* * Preliminaries. */ - erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); - p->gcstatus = p->status; - p->status = P_GARBING; - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + erts_smp_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC); ErtsGcQuickSanityCheck(p); ASSERT(p->mbuf_sz == 0); ASSERT(p->mbuf == 0); @@ -604,9 +596,7 @@ erts_garbage_collect_hibernate(Process* p) ErtsGcQuickSanityCheck(p); - erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); - p->status = p->gcstatus; - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC); } @@ -630,10 +620,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, /* * Set GC state. */ - erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); - p->gcstatus = p->status; - p->status = P_GARBING; - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + erts_smp_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC); /* * We assume that the caller has already done a major collection @@ -775,9 +762,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, /* * Restore status. */ - erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); - p->status = p->gcstatus; - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC); } static int diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index 02164728fe..d382e421e6 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -55,6 +55,61 @@ #endif /* + * The variables below (prefixed with etp_) are for erts/etc/unix/etp-commands + * only. Do not remove even though they aren't used elsewhere in the emulator! + */ +#ifdef ERTS_SMP +const int etp_smp_compiled = 1; +#else +const int etp_smp_compiled = 0; +#endif +#ifdef USE_THREADS +const int etp_thread_compiled = 1; +#else +const int etp_thread_compiled = 0; +#endif +const char etp_erts_version[] = ERLANG_VERSION; +const char etp_otp_release[] = ERLANG_OTP_RELEASE; +const char etp_compile_date[] = ERLANG_COMPILE_DATE; +const char etp_arch[] = ERLANG_ARCHITECTURE; +#ifdef ERTS_ENABLE_KERNEL_POLL +const int etp_kernel_poll_support = 1; +#else +const int etp_kernel_poll_support = 0; +#endif +#if defined(ARCH_64) +const int etp_arch_bits = 64; +#elif defined(ARCH_32) +const int etp_arch_bits = 32; +#else +# error "Not 64-bit, nor 32-bit arch" +#endif +#if HALFWORD_HEAP +const int etp_halfword = 1; +#else +const int etp_halfword = 0; +#endif +#ifdef HIPE +const int etp_hipe = 1; +#else +const int etp_hipe = 0; +#endif +#ifdef DEBUG +const int etp_debug_compiled = 1; +#else +const int etp_debug_compiled = 0; +#endif +#ifdef ERTS_ENABLE_LOCK_COUNT +const int etp_lock_count = 1; +#else +const int etp_lock_count = 0; +#endif +#ifdef ERTS_ENABLE_LOCK_CHECK +const int etp_lock_check = 1; +#else +const int etp_lock_check = 0; +#endif +/* * Note about VxWorks: All variables must be initialized by executable code, * not by an initializer. Otherwise a new instance of the emulator will * inherit previous values. @@ -241,6 +296,7 @@ erl_init(int ncpu) erts_init_trace(); erts_init_binary(); erts_init_bits(); + erts_code_ix_init(); erts_init_fun_table(); init_atom_table(); init_export_table(); @@ -289,7 +345,8 @@ erl_first_process_otp(char* modname, void* code, unsigned size, int argc, char** Eterm env; start_mod = am_atom_put(modname, sys_strlen(modname)); - if (erts_find_function(start_mod, am_start, 2) == NULL) { + if (erts_find_function(start_mod, am_start, 2, + erts_active_code_ix()) == NULL) { erl_exit(5, "No function %s:start/2\n", modname); } @@ -388,7 +445,7 @@ load_preloaded(void) if ((code = sys_preload_begin(&preload_p[i])) == 0) erl_exit(1, "Failed to find preloaded code for module %s\n", name); - res = erts_load_module(NULL, 0, NIL, &module_name, code, length); + res = erts_preload_module(NULL, 0, NIL, &module_name, code, length); sys_preload_end(&preload_p[i]); if (res != NIL) erl_exit(1,"Failed loading preloaded module %s (%T)\n", @@ -555,6 +612,7 @@ early_init(int *argc, char **argv) /* erts_printf_eterm_func = erts_printf_term; erts_disable_tolerant_timeofday = 0; display_items = 200; + erts_proc.max = ERTS_DEFAULT_MAX_PROCESSES; erts_backtrace_depth = DEFAULT_BACKTRACE_SIZE; erts_async_max_threads = 0; erts_async_thread_suggested_stack_size = ERTS_ASYNC_THREAD_MIN_STACK_SIZE; @@ -1097,7 +1155,7 @@ erl_start(int argc, char **argv) case 'P': /* set maximum number of processes */ Parg = get_arg(argv[i]+2, argv[i+1], &i); - erts_max_processes = atoi(Parg); + erts_proc.max = atoi(Parg); /* Check of result is delayed until later. This is because +R may be given after +P. */ break; @@ -1396,10 +1454,10 @@ erl_start(int argc, char **argv) } /* Delayed check of +P flag */ - if (erts_max_processes < ERTS_MIN_PROCESSES - || erts_max_processes > ERTS_MAX_PROCESSES + if (erts_proc.max < ERTS_MIN_PROCESSES + || erts_proc.max > ERTS_MAX_PROCESSES || (erts_use_r9_pids_ports - && erts_max_processes > ERTS_MAX_R9_PROCESSES)) { + && erts_proc.max > ERTS_MAX_R9_PROCESSES)) { erts_fprintf(stderr, "bad number of processes %s\n", Parg); erts_usage(); } @@ -1427,6 +1485,8 @@ erl_start(int argc, char **argv) erl_init(ncpu); load_preloaded(); + erts_end_staging_code_ix(); + erts_commit_staging_code_ix(); erts_initialized = 1; diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c index a0f744be9d..95efd9efc3 100644 --- a/erts/emulator/beam/erl_lock_check.c +++ b/erts/emulator/beam/erl_lock_check.c @@ -82,8 +82,8 @@ static erts_lc_lock_order_t erts_lock_order[] = { #ifdef ERTS_SMP { "bif_timers", NULL }, { "reg_tab", NULL }, - { "migration_info_update", NULL }, { "proc_main", "pid" }, + { "old_code", "address" }, #ifdef HIPE { "hipe_mfait_lock", NULL }, #endif @@ -93,6 +93,7 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "proc_msgq", "pid" }, { "dist_entry", "address" }, { "dist_entry_links", "address" }, + { "code_ix_queue", NULL }, { "proc_status", "pid" }, { "proc_tab", NULL }, { "ports_snapshot", NULL }, @@ -124,6 +125,7 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "removed_fd_pre_alloc_lock", "address" }, { "state_prealloc", NULL }, { "schdlr_sspnd", NULL }, + { "migration_info_update", NULL }, { "run_queue", "address" }, { "cpu_info", NULL }, { "pollset", "address" }, @@ -446,7 +448,7 @@ print_lock2(char *prefix, Sint16 id, Wterm extra, Uint16 flags, char *suffix) "%s'%s:%p%s'%s%s", prefix, lname, - boxed_val(extra), + _unchecked_boxed_val(extra), lock_type(flags), rw_op_str(flags), suffix); diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index 499a2aac53..241d61b868 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -296,36 +296,6 @@ erts_msg_distext2heap(Process *pp, return THE_NON_VALUE; } -static ERTS_INLINE void -notify_new_message(Process *receiver) -{ - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS - & erts_proc_lc_my_proc_locks(receiver)); - - switch (receiver->status) { - case P_GARBING: - switch (receiver->gcstatus) { - case P_SUSPENDED: - goto suspended; - case P_WAITING: - goto waiting; - default: - break; - } - break; - case P_SUSPENDED: - suspended: - receiver->rstatus = P_RUNABLE; - break; - case P_WAITING: - waiting: - erts_add_to_runq(receiver); - break; - default: - break; - } -} - void erts_queue_dist_message(Process *rcvr, ErtsProcLocks *rcvr_locks, @@ -339,7 +309,7 @@ erts_queue_dist_message(Process *rcvr, Sint tok_serial = 0; #endif #ifdef ERTS_SMP - ErtsProcLocks need_locks; + erts_aint_t state; #endif ERTS_SMP_LC_ASSERT(*rcvr_locks == erts_proc_lc_my_proc_locks(rcvr)); @@ -347,20 +317,21 @@ erts_queue_dist_message(Process *rcvr, mp = message_alloc(); #ifdef ERTS_SMP - need_locks = ~(*rcvr_locks) & (ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS); - if (need_locks) { - *rcvr_locks |= need_locks; - if (erts_smp_proc_trylock(rcvr, need_locks) == EBUSY) { - if (need_locks == ERTS_PROC_LOCK_MSGQ) { + if (!(*rcvr_locks & ERTS_PROC_LOCK_MSGQ)) { + if (erts_smp_proc_trylock(rcvr, ERTS_PROC_LOCK_MSGQ) == EBUSY) { + ErtsProcLocks need_locks = ERTS_PROC_LOCK_MSGQ; + if (*rcvr_locks & ERTS_PROC_LOCK_STATUS) { erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_STATUS); - need_locks = (ERTS_PROC_LOCK_MSGQ - | ERTS_PROC_LOCK_STATUS); + need_locks |= ERTS_PROC_LOCK_STATUS; } erts_smp_proc_lock(rcvr, need_locks); } } - if (rcvr->is_exiting || ERTS_PROC_PENDING_EXIT(rcvr)) { + state = erts_smp_atomic32_read_acqb(&rcvr->state); + if (state & (ERTS_PSFLG_PENDING_EXIT|ERTS_PSFLG_EXITING)) { + if (!(*rcvr_locks & ERTS_PROC_LOCK_MSGQ)) + erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ); /* Drop message if receiver is exiting or has a pending exit ... */ if (is_not_nil(token)) { ErlHeapFragment *heap_frag; @@ -376,6 +347,8 @@ erts_queue_dist_message(Process *rcvr, /* Ahh... need to decode it in order to trace it... */ ErlHeapFragment *mbuf; Eterm msg; + if (!(*rcvr_locks & ERTS_PROC_LOCK_MSGQ)) + erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ); message_free(mp); msg = erts_msg_distext2heap(rcvr, rcvr_locks, &mbuf, &token, dist_ext); if (is_value(msg)) @@ -437,26 +410,32 @@ erts_queue_dist_message(Process *rcvr, mp->data.dist_ext = dist_ext; LINK_MESSAGE(rcvr, mp); - notify_new_message(rcvr); + if (!(*rcvr_locks & ERTS_PROC_LOCK_MSGQ)) + erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ); + + erts_proc_notify_new_message(rcvr); } } /* Add a message last in message queue */ -void -erts_queue_message(Process* receiver, - ErtsProcLocks *receiver_locks, - ErlHeapFragment* bp, - Eterm message, - Eterm seq_trace_token +static int +queue_message(Process *c_p, + Process* receiver, + ErtsProcLocks *receiver_locks, + erts_aint32_t *receiver_state, + ErlHeapFragment* bp, + Eterm message, + Eterm seq_trace_token #ifdef USE_VM_PROBES , Eterm dt_utag #endif -) + ) { ErlMessage* mp; -#ifdef ERTS_SMP - ErtsProcLocks need_locks; -#else + int locked_msgq = 0; + erts_aint_t state; + +#ifndef ERTS_SMP ASSERT(bp != NULL || receiver->mbuf == NULL); #endif @@ -464,31 +443,45 @@ erts_queue_message(Process* receiver, mp = message_alloc(); + if (receiver_state) + state = *receiver_state; + else + state = erts_smp_atomic32_read_acqb(&receiver->state); + #ifdef ERTS_SMP - need_locks = ~(*receiver_locks) & (ERTS_PROC_LOCK_MSGQ - | ERTS_PROC_LOCK_STATUS); - if (need_locks) { - *receiver_locks |= need_locks; - if (erts_smp_proc_trylock(receiver, need_locks) == EBUSY) { - if (need_locks == ERTS_PROC_LOCK_MSGQ) { + + if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) + goto exiting; + + if (!(*receiver_locks & ERTS_PROC_LOCK_MSGQ)) { + if (erts_smp_proc_trylock(receiver, ERTS_PROC_LOCK_MSGQ) == EBUSY) { + ErtsProcLocks need_locks = ERTS_PROC_LOCK_MSGQ; + if (*receiver_locks & ERTS_PROC_LOCK_STATUS) { erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_STATUS); - need_locks = (ERTS_PROC_LOCK_MSGQ - | ERTS_PROC_LOCK_STATUS); + need_locks |= ERTS_PROC_LOCK_STATUS; } erts_smp_proc_lock(receiver, need_locks); } + locked_msgq = 1; + state = erts_smp_atomic32_read_nob(&receiver->state); + if (receiver_state) + *receiver_state = state; } - if (receiver->is_exiting || ERTS_PROC_PENDING_EXIT(receiver)) { - /* Drop message if receiver is exiting or has a pending - * exit ... - */ +#endif + + if (state & (ERTS_PSFLG_PENDING_EXIT|ERTS_PSFLG_EXITING)) { +#ifdef ERTS_SMP + exiting: +#endif + /* Drop message if receiver is exiting or has a pending exit... */ + if (locked_msgq) + erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_MSGQ); if (bp) free_message_buffer(bp); message_free(mp); - return; + return 0; } -#endif ERL_MESSAGE_TERM(mp) = message; ERL_MESSAGE_TOKEN(mp) = seq_trace_token; @@ -511,12 +504,11 @@ erts_queue_message(Process* receiver, ERTS_SMP_MSGQ_MV_INQ2PRIVQ(receiver); LINK_MESSAGE_PRIVQ(receiver, mp); } - else { + else +#endif + { LINK_MESSAGE(receiver, mp); } -#else - LINK_MESSAGE(receiver, mp); -#endif #ifdef USE_VM_PROBES if (DTRACE_ENABLED(message_queued)) { @@ -536,15 +528,43 @@ erts_queue_message(Process* receiver, tok_label, tok_lastcnt, tok_serial); } #endif - notify_new_message(receiver); - if (IS_TRACED_FL(receiver, F_TRACE_RECEIVE)) { + if (IS_TRACED_FL(receiver, F_TRACE_RECEIVE)) trace_receive(receiver, message); - } + + if (locked_msgq) + erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_MSGQ); + + erts_proc_notify_new_message(receiver); #ifndef ERTS_SMP ERTS_HOLE_CHECK(receiver); #endif + return 0; +} + +void +erts_queue_message(Process* receiver, + ErtsProcLocks *receiver_locks, + ErlHeapFragment* bp, + Eterm message, + Eterm seq_trace_token +#ifdef USE_VM_PROBES + , Eterm dt_utag +#endif + ) +{ + queue_message(NULL, + receiver, + receiver_locks, + NULL, + bp, + message, + seq_trace_token +#ifdef USE_VM_PROBES + , dt_utag +#endif + ); } void @@ -1036,21 +1056,34 @@ erts_send_message(Process* sender, #ifdef ERTS_SMP ErlOffHeap *ohp; Eterm *hp; + erts_aint32_t state; + BM_SWAP_TIMER(send,size); msize = size_object(message); BM_SWAP_TIMER(size,send); - hp = erts_alloc_message_heap(msize,&bp,&ohp,receiver,receiver_locks); + hp = erts_alloc_message_heap_state(msize, + &bp, + &ohp, + receiver, + receiver_locks, + &state); BM_SWAP_TIMER(send,copy); message = copy_struct(message, msize, &hp, ohp); BM_MESSAGE_COPIED(msz); BM_SWAP_TIMER(copy,send); DTRACE6(message_send, sender_name, receiver_name, msize, tok_label, tok_lastcnt, tok_serial); - erts_queue_message(receiver, receiver_locks, bp, message, token + queue_message(sender, + receiver, + receiver_locks, + &state, + bp, + message, + token #ifdef USE_VM_PROBES - , NIL + , NIL #endif - ); + ); BM_SWAP_TIMER(send,system); #else ErlMessage* mp = message_alloc(); @@ -1081,11 +1114,8 @@ erts_send_message(Process* sender, mp->data.attached = NULL; LINK_MESSAGE(receiver, mp); - if (receiver->status == P_WAITING) { - erts_add_to_runq(receiver); - } else if (receiver->status == P_SUSPENDED) { - receiver->rstatus = P_RUNABLE; - } + erts_proc_notify_new_message(receiver); + if (IS_TRACED_FL(receiver, F_TRACE_RECEIVE)) { trace_receive(receiver, message); } diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index b323bc7f69..da4376fd0a 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -311,6 +311,7 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, #endif Eterm receiver = to_pid->pid; int flush_me = 0; + int scheduler = erts_get_scheduler_id() != 0; if (env != NULL) { c_p = env->proc; @@ -330,8 +331,11 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, #if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP) rp_had_locks = rp_locks; #endif - rp = erts_pid2proc_opt(c_p, ERTS_PROC_LOCK_MAIN, - receiver, rp_locks, ERTS_P2P_FLG_SMP_INC_REFC); + + rp = (scheduler + ? erts_proc_lookup(receiver) + : erts_pid2proc_opt(c_p, ERTS_PROC_LOCK_MAIN, + receiver, rp_locks, ERTS_P2P_FLG_SMP_INC_REFC)); if (rp == NULL) { ASSERT(env == NULL || receiver != c_p->id); return 0; @@ -358,12 +362,12 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, , NIL #endif ); - if (rp_locks) { - ERTS_SMP_LC_ASSERT(rp_locks == (rp_had_locks | (ERTS_PROC_LOCK_MSGQ | - ERTS_PROC_LOCK_STATUS))); - erts_smp_proc_unlock(rp, (ERTS_PROC_LOCK_MSGQ | ERTS_PROC_LOCK_STATUS)); - } - erts_smp_proc_dec_refc(rp); + if (c_p == rp) + rp_locks &= ~ERTS_PROC_LOCK_MAIN; + if (rp_locks) + erts_smp_proc_unlock(rp, rp_locks); + if (!scheduler) + erts_smp_proc_dec_refc(rp); if (flush_me) { cache_env(env); } @@ -1520,6 +1524,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) if (len < 0) { BIF_ERROR(BIF_P, BADARG); } + lib_name = (char *) erts_alloc(ERTS_ALC_T_TMP, len + 1); if (intlist_to_buf(BIF_ARG_1, lib_name, len) != len) { @@ -1528,6 +1533,12 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) } lib_name[len] = '\0'; + if (!erts_try_seize_code_write_permission(BIF_P)) { + erts_free(ERTS_ALC_T_TMP, lib_name); + ERTS_BIF_YIELD2(bif_export[BIF_load_nif_2], + BIF_P, BIF_ARG_1, BIF_ARG_2); + } + /* Block system (is this the right place to do it?) */ erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_smp_thr_progress_block(); @@ -1541,11 +1552,11 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) ASSERT(caller != NULL); mod_atom = caller[0]; ASSERT(is_atom(mod_atom)); - mod=erts_get_module(mod_atom); + mod=erts_get_module(mod_atom, erts_active_code_ix()); ASSERT(mod != NULL); - if (!in_area(caller, mod->code, mod->code_length)) { - ASSERT(in_area(caller, mod->old_code, mod->old_code_length)); + if (!in_area(caller, mod->curr.code, mod->curr.code_length)) { + ASSERT(in_area(caller, mod->old.code, mod->old.code_length)); ret = load_nif_error(BIF_P, "old_code", "Calling load_nif from old " "module '%T' not allowed", mod_atom); @@ -1591,7 +1602,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) BeamInstr** code_pp; ErlNifFunc* f = &entry->funcs[i]; if (!erts_atom_get(f->name, sys_strlen(f->name), &f_atom) - || (code_pp = get_func_pp(mod->code, f_atom, f->arity))==NULL) { + || (code_pp = get_func_pp(mod->curr.code, f_atom, f->arity))==NULL) { ret = load_nif_error(BIF_P,bad_lib,"Function not found %T:%s/%u", mod_atom, f->name, f->arity); } @@ -1620,18 +1631,18 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) erts_refc_init(&lib->rt_dtor_cnt, 0); lib->mod = mod; env.mod_nif = lib; - if (mod->nif != NULL) { /* Reload */ + if (mod->curr.nif != NULL) { /* Reload */ int k; - lib->priv_data = mod->nif->priv_data; + lib->priv_data = mod->curr.nif->priv_data; - ASSERT(mod->nif->entry != NULL); + ASSERT(mod->curr.nif->entry != NULL); if (entry->reload == NULL) { ret = load_nif_error(BIF_P,reload,"Reload not supported by this NIF library."); goto error; } /* Check that no NIF is removed */ - for (k=0; k < mod->nif->entry->num_of_funcs; k++) { - ErlNifFunc* old_func = &mod->nif->entry->funcs[k]; + for (k=0; k < mod->curr.nif->entry->num_of_funcs; k++) { + ErlNifFunc* old_func = &mod->curr.nif->entry->funcs[k]; for (i=0; i < entry->num_of_funcs; i++) { if (old_func->arity == entry->funcs[i].arity && sys_strcmp(old_func->name, entry->funcs[i].name) == 0) { @@ -1652,24 +1663,24 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) ret = load_nif_error(BIF_P, reload, "Library reload-call unsuccessful."); } else { - mod->nif->entry = NULL; /* to prevent 'unload' callback */ - erts_unload_nif(mod->nif); + mod->curr.nif->entry = NULL; /* to prevent 'unload' callback */ + erts_unload_nif(mod->curr.nif); reload_warning = 1; } } else { lib->priv_data = NULL; - if (mod->old_nif != NULL) { /* Upgrade */ - void* prev_old_data = mod->old_nif->priv_data; + if (mod->old.nif != NULL) { /* Upgrade */ + void* prev_old_data = mod->old.nif->priv_data; if (entry->upgrade == NULL) { ret = load_nif_error(BIF_P, upgrade, "Upgrade not supported by this NIF library."); goto error; } erts_pre_nif(&env, BIF_P, lib); - veto = entry->upgrade(&env, &lib->priv_data, &mod->old_nif->priv_data, BIF_ARG_2); + veto = entry->upgrade(&env, &lib->priv_data, &mod->old.nif->priv_data, BIF_ARG_2); erts_post_nif(&env); if (veto) { - mod->old_nif->priv_data = prev_old_data; + mod->old.nif->priv_data = prev_old_data; ret = load_nif_error(BIF_P, upgrade, "Library upgrade-call unsuccessful."); } /*else if (mod->old_nif->priv_data != prev_old_data) { @@ -1689,12 +1700,12 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) /* ** Everything ok, patch the beam code with op_call_nif */ - mod->nif = lib; + mod->curr.nif = lib; for (i=0; i < entry->num_of_funcs; i++) { BeamInstr* code_ptr; erts_atom_get(entry->funcs[i].name, sys_strlen(entry->funcs[i].name), &f_atom); - code_ptr = *get_func_pp(mod->code, f_atom, entry->funcs[i].arity); + code_ptr = *get_func_pp(mod->curr.code, f_atom, entry->funcs[i].arity); if (code_ptr[1] == 0) { code_ptr[5+0] = (BeamInstr) BeamOp(op_call_nif); @@ -1722,6 +1733,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) erts_smp_thr_progress_unblock(); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_release_code_write_permission(); erts_free(ERTS_ALC_T_TMP, lib_name); if (reload_warning) { diff --git a/erts/emulator/beam/erl_node_container_utils.h b/erts/emulator/beam/erl_node_container_utils.h index 329a2204cc..7b4cb7b042 100644 --- a/erts/emulator/beam/erl_node_container_utils.h +++ b/erts/emulator/beam/erl_node_container_utils.h @@ -128,8 +128,47 @@ extern int erts_use_r9_pids_ports; * Pids * \* */ -#define internal_pid_index(x) (internal_pid_data((x)) \ - & erts_process_tab_index_mask) +#define erts_max_processes erts_proc.max + +typedef struct { + erts_smp_atomic_t *tab; + int max; + int tab_cache_lines; + int pix_per_cache_line; + int pix_cl_mask; + int pix_cl_shift; + int pix_cli_mask; + int pix_cli_shift; +} ErtsProcTab; + +extern ErtsProcTab erts_proc; + +ERTS_GLB_INLINE int erts_pid_data2ix(Eterm pid_data); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE int erts_pid_data2ix(Eterm pid_data) +{ + int n, pix; + + n = (int) pid_data; + if (erts_proc.pix_cl_mask) { + pix = ((n & erts_proc.pix_cl_mask) << erts_proc.pix_cl_shift); + pix += ((n >> erts_proc.pix_cli_shift) & erts_proc.pix_cli_mask); + } + else { + n %= erts_proc.max; + pix = n % erts_proc.tab_cache_lines; + pix *= erts_proc.pix_per_cache_line; + pix += n / erts_proc.tab_cache_lines; + } + ASSERT(0 <= pix && pix < erts_proc.max); + return pix; +} + +#endif + +#define internal_pid_index(x) erts_pid_data2ix(internal_pid_data((x))) #define internal_pid_node_name(x) (internal_pid_node((x))->sysname) #define external_pid_node_name(x) (external_pid_node((x))->sysname) diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c index c7fd379367..649be3f454 100644 --- a/erts/emulator/beam/erl_node_tables.c +++ b/erts/emulator/beam/erl_node_tables.c @@ -1298,21 +1298,22 @@ setup_reference_table(void) UnUseTmpHeapNoproc(3); /* Insert all processes */ - for (i = 0; i < erts_max_processes; i++) - if (process_tab[i]) { + for (i = 0; i < erts_max_processes; i++) { + Process *proc = erts_pix2proc(i); + if (proc) { ErlMessage *msg; /* Insert Heap */ - insert_offheap(&(process_tab[i]->off_heap), + insert_offheap(&(proc->off_heap), HEAP_REF, - process_tab[i]->id); + proc->id); /* Insert message buffers */ - for(hfp = process_tab[i]->mbuf; hfp; hfp = hfp->next) + for(hfp = proc->mbuf; hfp; hfp = hfp->next) insert_offheap(&(hfp->off_heap), HEAP_REF, - process_tab[i]->id); + proc->id); /* Insert msg msg buffers */ - for (msg = process_tab[i]->msg.first; msg; msg = msg->next) { + for (msg = proc->msg.first; msg; msg = msg->next) { ErlHeapFragment *heap_frag = NULL; if (msg->data.attached) { if (is_value(ERL_MESSAGE_TERM(msg))) @@ -1320,7 +1321,7 @@ setup_reference_table(void) else { if (msg->data.dist_ext->dep) insert_dist_entry(msg->data.dist_ext->dep, - HEAP_REF, process_tab[i]->id, 0); + HEAP_REF, proc->id, 0); if (is_not_nil(ERL_MESSAGE_TOKEN(msg))) heap_frag = erts_dist_ext_trailer(msg->data.dist_ext); } @@ -1328,10 +1329,10 @@ setup_reference_table(void) if (heap_frag) insert_offheap(&(heap_frag->off_heap), HEAP_REF, - process_tab[i]->id); + proc->id); } #ifdef ERTS_SMP - for (msg = process_tab[i]->msg_inq.first; msg; msg = msg->next) { + for (msg = proc->msg_inq.first; msg; msg = msg->next) { ErlHeapFragment *heap_frag = NULL; if (msg->data.attached) { if (is_value(ERL_MESSAGE_TERM(msg))) @@ -1339,7 +1340,7 @@ setup_reference_table(void) else { if (msg->data.dist_ext->dep) insert_dist_entry(msg->data.dist_ext->dep, - HEAP_REF, process_tab[i]->id, 0); + HEAP_REF, proc->id, 0); if (is_not_nil(ERL_MESSAGE_TOKEN(msg))) heap_frag = erts_dist_ext_trailer(msg->data.dist_ext); } @@ -1347,21 +1348,22 @@ setup_reference_table(void) if (heap_frag) insert_offheap(&(heap_frag->off_heap), HEAP_REF, - process_tab[i]->id); + proc->id); } #endif /* Insert links */ - if(process_tab[i]->nlinks) - insert_links(process_tab[i]->nlinks, process_tab[i]->id); - if(process_tab[i]->monitors) - insert_monitors(process_tab[i]->monitors, process_tab[i]->id); + if(proc->nlinks) + insert_links(proc->nlinks, proc->id); + if(proc->monitors) + insert_monitors(proc->monitors, proc->id); /* Insert controller */ { - DistEntry *dep = ERTS_PROC_GET_DIST_ENTRY(process_tab[i]); + DistEntry *dep = ERTS_PROC_GET_DIST_ENTRY(proc); if (dep) - insert_dist_entry(dep, CTRL_REF, process_tab[i]->id, 0); + insert_dist_entry(dep, CTRL_REF, proc->id, 0); } } + } #ifdef ERTS_SMP erts_foreach_sys_msg_in_q(insert_sys_msg); diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c index 0f1a0d441a..86454fe1fa 100644 --- a/erts/emulator/beam/erl_port_task.c +++ b/erts/emulator/beam/erl_port_task.c @@ -56,12 +56,6 @@ #define ERTS_PORT_IS_IN_RUNQ(RQ, P) \ ((P)->sched.next || (P)->sched.prev || (RQ)->ports.start == (P)) -#define ERTS_PORT_NOT_IN_RUNQ(P) \ -do { \ - (P)->sched.prev = NULL; \ - (P)->sched.next = NULL; \ -} while (0) - #ifdef USE_VM_PROBES #define DTRACE_DRIVER(PROBE_NAME, PP) \ if (DTRACE_ENABLED(driver_ready_input)) { \ @@ -167,7 +161,7 @@ enqueue_port(ErtsRunQueue *runq, Port *pp) { ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq)); pp->sched.next = NULL; - pp->sched.prev = runq->ports.end; + pp->sched.in_runq = 1; if (runq->ports.end) { ASSERT(runq->ports.start); runq->ports.end->sched.next = pp; @@ -177,39 +171,10 @@ enqueue_port(ErtsRunQueue *runq, Port *pp) runq->ports.start = pp; } - runq->ports.info.len++; - if (runq->ports.info.max_len < runq->ports.info.len) - runq->ports.info.max_len = runq->ports.info.len; - runq->len++; - if (runq->max_len < runq->len) - runq->max_len = runq->len; runq->ports.end = pp; ASSERT(runq->ports.start && runq->ports.end); -} -static ERTS_INLINE void -dequeue_port(ErtsRunQueue *runq, Port *pp) -{ - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq)); - if (pp->sched.next) - pp->sched.next->sched.prev = pp->sched.prev; - else { - ASSERT(runq->ports.end == pp); - runq->ports.end = pp->sched.prev; - } - if (pp->sched.prev) - pp->sched.prev->sched.next = pp->sched.next; - else { - ASSERT(runq->ports.start == pp); - runq->ports.start = pp->sched.next; - } - - ASSERT(runq->ports.info.len > 0); - runq->ports.info.len--; - ASSERT(runq->len > 0); - runq->len--; - ASSERT(runq->ports.start || !runq->ports.end); - ASSERT(runq->ports.end || !runq->ports.start); + erts_smp_inc_runq_len(runq, &runq->ports.info, ERTS_PORT_PRIO_LEVEL); } static ERTS_INLINE Port * @@ -222,16 +187,11 @@ pop_port(ErtsRunQueue *runq) } else { runq->ports.start = runq->ports.start->sched.next; - if (runq->ports.start) - runq->ports.start->sched.prev = NULL; - else { + if (!runq->ports.start) { ASSERT(runq->ports.end == pp); runq->ports.end = NULL; } - ASSERT(runq->ports.info.len > 0); - runq->ports.info.len--; - ASSERT(runq->len > 0); - runq->len--; + erts_smp_dec_runq_len(runq, &runq->ports.info, ERTS_PORT_PRIO_LEVEL); } ASSERT(runq->ports.start || !runq->ports.end); @@ -285,7 +245,7 @@ check_port_queue(ErtsRunQueue *runq, Port *chk_pp, int inq) } ASSERT(no_forward == no_backward); } - ASSERT(no_forward == runq->ports.info.len); + ASSERT(no_forward == RUNQ_READ_LEN(&runq->ports.info.len)); if (chk_pp) { if (chk_pp->sched.taskq || chk_pp->sched.exe_taskq) { ASSERT(chk_pp->sched.taskq != chk_pp->sched.exe_taskq); @@ -467,10 +427,11 @@ erts_port_task_abort(Eterm id, ErtsPortTaskHandle *pthp) ErtsPortTaskQueue *ptqp; ErtsPortTask *ptp; Port *pp; - int port_is_dequeued = 0; pp = &erts_port[internal_port_index(id)]; runq = erts_port_runq(pp); + if (!runq) + return 1; ptp = handle2task(pthp); @@ -505,22 +466,12 @@ erts_port_task_abort(Eterm id, ErtsPortTaskHandle *pthp) if (ptqp->first || pp->sched.taskq != ptqp) ptqp = NULL; - else { + else pp->sched.taskq = NULL; - if (!pp->sched.exe_taskq) { - dequeue_port(runq, pp); - ERTS_PORT_NOT_IN_RUNQ(pp); - port_is_dequeued = 1; - } - } ERTS_PT_CHK_PRES_PORTQ(runq, pp); erts_smp_runq_unlock(runq); - - if (erts_system_profile_flags.runnable_ports && port_is_dequeued) { - profile_runnable_port(pp, am_inactive); - } port_task_free(ptp); if (ptqp) @@ -575,7 +526,7 @@ erts_port_task_schedule(Eterm id, if (!pp->sched.taskq) { pp->sched.taskq = port_taskq_init(port_taskq_alloc(), pp); - enq_port = !pp->sched.exe_taskq; + enq_port = !pp->sched.in_runq && !pp->sched.exe_taskq; } #ifdef ERTS_SMP @@ -585,13 +536,13 @@ erts_port_task_schedule(Eterm id, /* Port emigrated ... */ erts_smp_atomic_set_nob(&pp->run_queue, (erts_aint_t) xrunq); erts_smp_runq_unlock(runq); - runq = xrunq; + runq = erts_port_runq(pp); + if (!runq) + return -1; } } #endif - ASSERT(!enq_port || !(runq->flags & ERTS_RUNQ_FLG_SUSPENDED)); - ASSERT(pp->sched.taskq); ASSERT(ptp); @@ -633,7 +584,7 @@ erts_port_task_schedule(Eterm id, #elif defined(DEBUG) if (!enq_port && !pp->sched.exe_taskq) { /* We should be in port run q */ - ASSERT(pp->sched.prev || runq->ports.start == pp); + ASSERT(pp->sched.in_runq); } #endif #endif @@ -661,63 +612,54 @@ void erts_port_task_free_port(Port *pp) { ErtsRunQueue *runq; - int port_is_dequeued = 0; + ErtsPortTaskQueue *ptqp; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(pp)); ASSERT(!(pp->status & ERTS_PORT_SFLGS_DEAD)); runq = erts_port_runq(pp); ASSERT(runq); ERTS_PT_CHK_PRES_PORTQ(runq, pp); - if (pp->sched.exe_taskq) { + ptqp = pp->sched.exe_taskq; + if (ptqp) { /* I (this thread) am currently executing this port, free it when scheduled out... */ - ErtsPortTask *ptp = port_task_alloc(); + ErtsPortTask *ptp; + enqueue_free: + ptp = port_task_alloc(); erts_smp_port_state_lock(pp); pp->status &= ~ERTS_PORT_SFLG_CLOSING; pp->status |= ERTS_PORT_SFLG_FREE_SCHEDULED; erts_may_save_closed_port(pp); erts_smp_port_state_unlock(pp); - ERTS_SMP_LC_ASSERT(erts_smp_atomic_read_nob(&pp->refc) > 1); + ERTS_LC_ASSERT(erts_smp_atomic_read_nob(&pp->refc) > 1); ptp->type = ERTS_PORT_TASK_FREE; ptp->event = (ErlDrvEvent) -1; ptp->event_data = NULL; set_handle(ptp, NULL); - push_task(pp->sched.exe_taskq, ptp); + push_task(ptqp, ptp); ERTS_PT_CHK_PRES_PORTQ(runq, pp); erts_smp_runq_unlock(runq); } else { - ErtsPortTaskQueue *ptqp = pp->sched.taskq; - if (ptqp) { - dequeue_port(runq, pp); - ERTS_PORT_NOT_IN_RUNQ(pp); - port_is_dequeued = 1; + if (pp->sched.in_runq) { + ptqp = pp->sched.taskq; + if (!ptqp) + pp->sched.taskq = ptqp = port_taskq_init(port_taskq_alloc(), pp); + goto enqueue_free; } + ASSERT(!pp->sched.taskq); erts_smp_port_state_lock(pp); pp->status &= ~ERTS_PORT_SFLG_CLOSING; pp->status |= ERTS_PORT_SFLG_FREE_SCHEDULED; erts_may_save_closed_port(pp); erts_smp_port_state_unlock(pp); -#ifdef ERTS_SMP erts_smp_atomic_dec_nob(&pp->refc); /* Not alive */ -#endif - ERTS_SMP_LC_ASSERT(erts_smp_atomic_read_nob(&pp->refc) > 0); /* Lock */ + ERTS_LC_ASSERT(erts_smp_atomic_read_nob(&pp->refc) > 0); /* Lock */ handle_remaining_tasks(runq, pp); /* May release runq lock */ ASSERT(!pp->sched.exe_taskq && (!ptqp || !ptqp->first)); pp->sched.taskq = NULL; ERTS_PT_CHK_PRES_PORTQ(runq, pp); -#ifndef ERTS_SMP - ASSERT(pp->status & ERTS_PORT_SFLG_PORT_DEBUG); - erts_port_status_set(pp, ERTS_PORT_SFLG_FREE); -#endif erts_smp_runq_unlock(runq); - - if (erts_system_profile_flags.runnable_ports && port_is_dequeued) { - profile_runnable_port(pp, am_inactive); - } - - if (ptqp) - port_taskq_free(ptqp); } } @@ -738,7 +680,6 @@ typedef struct { int erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) { - int port_was_enqueued = 0; Port *pp; ErtsPortTaskQueue *ptqp; ErtsPortTask *ptp; @@ -757,11 +698,18 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) goto done; } - ERTS_PORT_NOT_IN_RUNQ(pp); + ASSERT(pp->sched.in_runq); + pp->sched.in_runq = 0; + if (!pp->sched.taskq) { + if (erts_system_profile_flags.runnable_ports) + profile_runnable_port(pp, am_inactive); + res = (erts_smp_atomic_read_nob(&erts_port_task_outstanding_io_tasks) + != (erts_aint_t) 0); + goto done; + } *curr_port_pp = pp; - ASSERT(pp->sched.taskq); ASSERT(pp->sched.taskq->first); ptqp = pp->sched.taskq; pp->sched.taskq = NULL; @@ -823,12 +771,8 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) handle_remaining_tasks(runq, pp); ASSERT(!ptqp->first && (!pp->sched.taskq || !pp->sched.taskq->first)); -#ifdef ERTS_SMP erts_smp_atomic_dec_nob(&pp->refc); /* Not alive */ - ERTS_SMP_LC_ASSERT(erts_smp_atomic_read_nob(&pp->refc) > 0); /* Lock */ -#else - erts_port_status_bor_set(pp, ERTS_PORT_SFLG_FREE); -#endif + ERTS_LC_ASSERT(erts_smp_atomic_read_nob(&pp->refc) > 0); /* Lock */ port_task_free(ptp); if (pp->sched.taskq) @@ -918,6 +862,8 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) if (!pp->sched.taskq) { ASSERT(pp->sched.exe_taskq); pp->sched.exe_taskq = NULL; + if (erts_system_profile_flags.runnable_ports) + profile_runnable_port(pp, am_inactive); } else { #ifdef ERTS_SMP @@ -940,14 +886,20 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) else { /* Port emigrated ... */ erts_smp_atomic_set_nob(&pp->run_queue, (erts_aint_t) xrunq); - enqueue_port(xrunq, pp); - ASSERT(pp->sched.exe_taskq); - pp->sched.exe_taskq = NULL; - erts_smp_runq_unlock(xrunq); - erts_smp_notify_inc_runq(xrunq); + erts_smp_runq_unlock(runq); + + xrunq = erts_port_runq(pp); + if (xrunq) { + enqueue_port(xrunq, pp); + ASSERT(pp->sched.exe_taskq); + pp->sched.exe_taskq = NULL; + erts_smp_runq_unlock(xrunq); + erts_smp_notify_inc_runq(xrunq); + } + + erts_smp_runq_lock(runq); } #endif - port_was_enqueued = 1; } res = (erts_smp_atomic_read_nob(&erts_port_task_outstanding_io_tasks) @@ -957,10 +909,6 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) port_taskq_free(ptqp); - if (erts_system_profile_flags.runnable_ports && (port_was_enqueued != 1)) { - profile_runnable_port(pp, am_inactive); - } - /* trace port scheduling, out */ if (IS_TRACED_FL(pp, F_TRACE_SCHED_PORTS)) { trace_sched_ports(pp, am_out); @@ -1049,63 +997,33 @@ erts_port_is_scheduled(Port *pp) { int res; ErtsRunQueue *runq = erts_port_runq(pp); + if (!runq) + return 0; res = pp->sched.taskq || pp->sched.exe_taskq; erts_smp_runq_unlock(runq); return res; } #ifdef ERTS_SMP +void +erts_enqueue_port(ErtsRunQueue *rq, Port *pp) +{ + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); + ASSERT(rq == (ErtsRunQueue *) erts_smp_atomic_read_nob(&pp->run_queue)); + ASSERT(pp->sched.in_runq); + enqueue_port(rq, pp); +} -ErtsMigrateResult -erts_port_migrate(Port *prt, int *prt_locked, - ErtsRunQueue *from_rq, int *from_locked, - ErtsRunQueue *to_rq, int *to_locked) +Port * +erts_dequeue_port(ErtsRunQueue *rq) { - ERTS_SMP_LC_ASSERT(*from_locked); - ERTS_SMP_LC_CHK_RUNQ_LOCK(from_rq, *from_locked); - ERTS_SMP_LC_CHK_RUNQ_LOCK(to_rq, *to_locked); - - if (!*from_locked || !*to_locked) { - if (from_rq < to_rq) { - if (!*to_locked) { - if (!*from_locked) - erts_smp_runq_lock(from_rq); - erts_smp_runq_lock(to_rq); - } - else if (erts_smp_runq_trylock(from_rq) == EBUSY) { - erts_smp_runq_unlock(to_rq); - erts_smp_runq_lock(from_rq); - erts_smp_runq_lock(to_rq); - } - } - else { - if (!*from_locked) { - if (!*to_locked) - erts_smp_runq_lock(to_rq); - erts_smp_runq_lock(from_rq); - } - else if (erts_smp_runq_trylock(to_rq) == EBUSY) { - erts_smp_runq_unlock(from_rq); - erts_smp_runq_lock(to_rq); - erts_smp_runq_lock(from_rq); - } - } - *to_locked = *from_locked = 1; - } - ERTS_SMP_LC_CHK_RUNQ_LOCK(from_rq, *from_locked); - ERTS_SMP_LC_CHK_RUNQ_LOCK(to_rq, *to_locked); - - /* Refuse to migrate to a suspended run queue */ - if (to_rq->flags & ERTS_RUNQ_FLG_SUSPENDED) - return ERTS_MIGRATE_FAILED_RUNQ_SUSPENDED; - if (from_rq != (ErtsRunQueue *) erts_smp_atomic_read_nob(&prt->run_queue)) - return ERTS_MIGRATE_FAILED_RUNQ_CHANGED; - if (!ERTS_PORT_IS_IN_RUNQ(from_rq, prt)) - return ERTS_MIGRATE_FAILED_NOT_IN_RUNQ; - dequeue_port(from_rq, prt); - erts_smp_atomic_set_nob(&prt->run_queue, (erts_aint_t) to_rq); - enqueue_port(to_rq, prt); - return ERTS_MIGRATE_SUCCESS; + Port *pp; + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); + pp = pop_port(rq); + ASSERT(!pp + || rq == (ErtsRunQueue *) erts_smp_atomic_read_nob(&pp->run_queue)); + ASSERT(!pp || pp->sched.in_runq); + return pp; } #endif diff --git a/erts/emulator/beam/erl_port_task.h b/erts/emulator/beam/erl_port_task.h index d7104e1143..fd88b1c1ff 100644 --- a/erts/emulator/beam/erl_port_task.h +++ b/erts/emulator/beam/erl_port_task.h @@ -62,7 +62,7 @@ typedef struct ErtsPortTaskQueue_ ErtsPortTaskQueue; typedef struct { Port *next; - Port *prev; + int in_runq; ErtsPortTaskQueue *taskq; ErtsPortTaskQueue *exe_taskq; } ErtsPortTaskSched; @@ -92,7 +92,7 @@ ERTS_GLB_INLINE void erts_port_task_init_sched(ErtsPortTaskSched *ptsp) { ptsp->next = NULL; - ptsp->prev = NULL; + ptsp->in_runq = 0; ptsp->taskq = NULL; ptsp->exe_taskq = NULL; } @@ -123,13 +123,10 @@ int erts_port_task_schedule(Eterm, ErlDrvEventData); void erts_port_task_free_port(Port *); int erts_port_is_scheduled(Port *); + #ifdef ERTS_SMP -ErtsMigrateResult erts_port_migrate(Port *, - int *, - ErtsRunQueue *, - int *, - ErtsRunQueue *, - int *); +void erts_enqueue_port(ErtsRunQueue *rq, Port *pp); +Port *erts_dequeue_port(ErtsRunQueue *rq); #endif #undef ERTS_INCLUDE_SCHEDULER_INTERNALS #endif /* ERL_PORT_TASK_H__ */ diff --git a/erts/emulator/beam/erl_printf_term.c b/erts/emulator/beam/erl_printf_term.c index 34da9cab84..2320b64295 100644 --- a/erts/emulator/beam/erl_printf_term.c +++ b/erts/emulator/beam/erl_printf_term.c @@ -437,7 +437,10 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount, } break; case BINARY_DEF: - { + if (header_is_bin_matchstate(*boxed_val(wobj))) { + PRINT_STRING(res, fn, arg, "#MatchState"); + } + else { ProcBin* pb = (ProcBin *) binary_val(wobj); if (pb->size == 1) PRINT_STRING(res, fn, arg, "<<1 byte>>"); diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index bd4c56eaa4..bca8fac259 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -94,34 +94,194 @@ #define HIGH_BIT (1 << PRIORITY_HIGH) #define NORMAL_BIT (1 << PRIORITY_NORMAL) #define LOW_BIT (1 << PRIORITY_LOW) +#define PORT_BIT (1 << ERTS_PORT_PRIO_LEVEL) -#define ERTS_MAYBE_SAVE_TERMINATING_PROCESS(P) \ -do { \ - ERTS_SMP_LC_ASSERT(erts_lc_mtx_is_locked(&proc_tab_mtx)); \ - if (saved_term_procs.end) \ - save_terminating_process((P)); \ -} while (0) +#define ERTS_EMPTY_RUNQ(RQ) \ + ((ERTS_RUNQ_FLGS_GET_NOB((RQ)) & ERTS_RUNQ_FLGS_QMASK) == 0 \ + && (RQ)->misc.start == NULL) + +#undef RUNQ_READ_RQ +#undef RUNQ_SET_RQ +#define RUNQ_READ_RQ(X) ((ErtsRunQueue *) erts_smp_atomic_read_nob((X))) +#define RUNQ_SET_RQ(X, RQ) erts_smp_atomic_set_nob((X), (erts_aint_t) (RQ)) -#define ERTS_EMPTY_RUNQ(RQ) \ - ((RQ)->len == 0 && (RQ)->misc.start == NULL) +#ifdef DEBUG +# if defined(ARCH_64) && !HALFWORD_HEAP +# define ERTS_DBG_SET_INVALID_RUNQP(RQP, N) \ + (RUNQ_SET_RQ((RQP), (0xdeadbeefdead0003LL | ((N) << 4))) +# define ERTS_DBG_VERIFY_VALID_RUNQP(RQP) \ +do { \ + ASSERT((RQP) != NULL); \ + ASSERT(((((Uint) (RQP)) & ((Uint) 0x3))) == ((Uint) 0)); \ + ASSERT((((Uint) (RQP)) & ~((Uint) 0xffff)) != ((Uint) 0xdeadbeefdead0000LL));\ +} while (0) +# else +# define ERTS_DBG_SET_INVALID_RUNQP(RQP, N) \ + (RUNQ_SET_RQ((RQP), (0xdead0003 | ((N) << 4)))) +# define ERTS_DBG_VERIFY_VALID_RUNQP(RQP) \ +do { \ + ASSERT((RQP) != NULL); \ + ASSERT(((((UWord) (RQP)) & ((UWord) 1))) == ((UWord) 0)); \ + ASSERT((((UWord) (RQP)) & ~((UWord) 0xffff)) != ((UWord) 0xdead0000)); \ +} while (0) +# endif +#else +# define ERTS_DBG_SET_INVALID_RUNQP(RQP, N) +# define ERTS_DBG_VERIFY_VALID_RUNQP(RQP) +#endif #define ERTS_EMPTY_RUNQ_PORTS(RQ) \ - ((RQ)->ports.info.len == 0 && (RQ)->misc.start == NULL) + (RUNQ_READ_LEN(&(RQ)->ports.info.len) == 0 && (RQ)->misc.start == NULL) extern BeamInstr beam_apply[]; extern BeamInstr beam_exit[]; extern BeamInstr beam_continue_exit[]; -static Sint p_last; -static Sint p_next; -static Sint p_serial; -static Uint p_serial_mask; -static Uint p_serial_shift; +#ifdef ARCH_32 + +union { + erts_smp_dw_atomic_t pid_data; + char align[ERTS_CACHE_LINE_SIZE]; +} last erts_align_attribute(ERTS_CACHE_LINE_SIZE); + + +static ERTS_INLINE Uint64 +dw_aint_to_uint64(erts_dw_aint_t *dw) +{ +#ifdef ETHR_SU_DW_NAINT_T__ + return (Uint64) dw->dw_sint; +#else + Uint64 res; + res = (Uint64) ((Uint32) dw->sint[ERTS_DW_AINT_HIGH_WORD]); + res <<= 32; + res |= (Uint64) ((Uint32) dw->sint[ERTS_DW_AINT_LOW_WORD]); + return res; +#endif +} + +static void +unint64_to_dw_aint(erts_dw_aint_t *dw, Uint64 val) +{ +#ifdef ETHR_SU_DW_NAINT_T__ + dw->dw_sint = (ETHR_SU_DW_NAINT_T__) val; +#else + dw->sint[ERTS_DW_AINT_LOW_WORD] = (erts_aint_t) (val & 0xffffffff); + dw->sint[ERTS_DW_AINT_HIGH_WORD] = (erts_aint_t) ((val >> 32) & 0xffffffff); +#endif +} + +static ERTS_INLINE void +last_pid_data_init_nob(Uint64 val) +{ + erts_dw_aint_t dw; + unint64_to_dw_aint(&dw, val); + erts_smp_dw_atomic_init_nob(&last.pid_data, &dw); +} + +static ERTS_INLINE void +last_pid_data_set_relb(Uint64 val) +{ + erts_dw_aint_t dw; + unint64_to_dw_aint(&dw, val); + erts_smp_dw_atomic_set_relb(&last.pid_data, &dw); +} + +static ERTS_INLINE Uint64 +last_pid_data_read_nob(void) +{ + erts_dw_aint_t dw; + erts_smp_dw_atomic_read_nob(&last.pid_data, &dw); + return dw_aint_to_uint64(&dw); +} + +static ERTS_INLINE Uint64 +last_pid_data_read_acqb(void) +{ + erts_dw_aint_t dw; + erts_smp_dw_atomic_read_acqb(&last.pid_data, &dw); + return dw_aint_to_uint64(&dw); +} + +static ERTS_INLINE Uint64 +last_pid_data_cmpxchg_relb(Uint64 new, Uint64 exp) +{ + erts_dw_aint_t dw_new, dw_xchg; + + unint64_to_dw_aint(&dw_new, new); + unint64_to_dw_aint(&dw_xchg, exp); + + if (erts_smp_dw_atomic_cmpxchg_relb(&last.pid_data, &dw_new, &dw_xchg)) + return exp; + else + return dw_aint_to_uint64(&dw_xchg); +} + +#elif defined(ARCH_64) + +union { + erts_smp_atomic_t pid_data; + char align[ERTS_CACHE_LINE_SIZE]; +} last erts_align_attribute(ERTS_CACHE_LINE_SIZE); + +static ERTS_INLINE void +last_pid_data_init_nob(Uint64 val) +{ + erts_smp_atomic_init_nob(&last.pid_data, (erts_aint_t) val); +} + +static ERTS_INLINE void +last_pid_data_set_relb(Uint64 val) +{ + erts_smp_atomic_set_relb(&last.pid_data, (erts_aint_t) val); +} + +static ERTS_INLINE Uint64 +last_pid_data_read_nob(void) +{ + return (Uint64) erts_smp_atomic_read_nob(&last.pid_data); +} + +static ERTS_INLINE Uint64 +last_pid_data_read_acqb(void) +{ + return (Uint64) erts_smp_atomic_read_acqb(&last.pid_data); +} + +static ERTS_INLINE Uint64 +last_pid_data_cmpxchg_relb(Uint64 new, Uint64 exp) +{ + return (Uint64) erts_smp_atomic_cmpxchg_relb(&last.pid_data, + (erts_aint_t) new, + (erts_aint_t) exp); +} + +#else +# error "Not 64-bit, nor 32-bit architecture..." +#endif + +static ERTS_INLINE int +last_pid_data_cmp(Uint64 lpd1, Uint64 lpd2) +{ + Uint64 lpd1_wrap; + + if (lpd1 == lpd2) + return 0; + + lpd1_wrap = lpd1 + (((Uint64) 1) << 63); + + if (lpd1 < lpd1_wrap) + return (lpd1 < lpd2 && lpd2 < lpd1_wrap) ? -1 : 1; + else + return (lpd1_wrap <= lpd2 && lpd2 < lpd1) ? 1 : -1; +} + + +#define ERTS_PID_DATA_MASK__ ((1 << _PID_DATA_SIZE) - 1) int erts_sched_compact_load; Uint erts_no_schedulers; -Uint erts_max_processes = ERTS_DEFAULT_MAX_PROCESSES; -Uint erts_process_tab_index_mask; + +ErtsProcTab erts_proc erts_align_attribute(ERTS_CACHE_LINE_SIZE); int erts_sched_thread_suggested_stack_size = -1; @@ -188,7 +348,7 @@ static struct { struct { int active_runqs; int reds; - int max_len; + erts_aint32_t max_len; } prev_rise; Uint n; } balance_info; @@ -208,7 +368,7 @@ erts_sched_stat_t erts_sched_stat; static erts_tsd_key_t sched_data_key; #endif -static erts_smp_mtx_t proc_tab_mtx; +erts_smp_rwmtx_t erts_proc_tab_rwmtx; static erts_smp_atomic32_t function_calls; @@ -231,7 +391,6 @@ typedef union { static ErtsAlignedSchedulerSleepInfo *aligned_sched_sleep_info; -Process** process_tab; static Uint last_reductions; static Uint last_exact_reductions; Uint erts_default_process_flags; @@ -257,11 +416,11 @@ struct ErtsTermProcElement_ { union { struct { Eterm pid; - SysTimeval spawned; - SysTimeval exited; + Uint64 spawned; + Uint64 exited; } process; struct { - SysTimeval time; + Uint64 interval; } bif_invocation; } u; }; @@ -369,6 +528,9 @@ dbg_chk_aux_work_val(erts_aint32_t value) #ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN valid |= ERTS_SSI_AUX_WORK_CHECK_CHILDREN; #endif +#ifdef ERTS_SMP + valid |= ERTS_SSI_AUX_WORK_CODE_IX_ACTIVATION; +#endif #ifdef ERTS_SSI_AUX_WORK_REAP_PORTS valid |= ERTS_SSI_AUX_WORK_REAP_PORTS; #endif @@ -398,6 +560,53 @@ erts_smp_lc_runq_is_locked(ErtsRunQueue *runq) } #endif +static erts_interval_t *proc_interval; + +static void +proc_interval_init(void) +{ + proc_interval = erts_alloc_permanent_cache_aligned( + ERTS_ALC_T_PROC_INTERVAL, + sizeof(erts_interval_t)); + erts_smp_interval_init(proc_interval); +} + +static ERTS_INLINE Uint64 +get_proc_interval(void) +{ + return erts_smp_current_interval_nob(proc_interval); +} + +static ERTS_INLINE Uint64 +ensure_later_proc_interval(Uint64 interval) +{ + return erts_smp_ensure_later_interval_nob(proc_interval, interval); +} + +static ERTS_INLINE Uint64 +step_proc_interval(void) +{ + return erts_smp_step_interval_nob(proc_interval); +} + +Uint64 +erts_get_proc_interval(void) +{ + return get_proc_interval(); +} + +Uint64 +erts_ensure_later_proc_interval(Uint64 interval) +{ + return ensure_later_proc_interval(interval); +} + +Uint64 +erts_step_proc_interval(void) +{ + return step_proc_interval(); +} + void erts_pre_init_process(void) { @@ -447,7 +656,16 @@ erts_pre_init_process(void) void erts_init_process(int ncpu) { - Uint proc_bits = ERTS_PROC_BITS; + int proc_tab_sz; + int max_proc_bits; + int proc_bits = ERTS_PROC_BITS; + erts_smp_atomic_t *proc_entry; + char *proc_tab_end; + erts_smp_rwmtx_opt_t proc_tab_rwmtx_opts = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; + proc_tab_rwmtx_opts.type = ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ; + proc_tab_rwmtx_opts.lived = ERTS_SMP_RWMTX_LONG_LIVED; + + proc_interval_init(); #ifdef ERTS_SMP erts_disable_proc_not_running_opt = 0; @@ -458,23 +676,51 @@ erts_init_process(int ncpu) erts_smp_atomic32_init_nob(&process_count, 0); - if (erts_use_r9_pids_ports) { + if (erts_use_r9_pids_ports) proc_bits = ERTS_R9_PROC_BITS; - ASSERT(erts_max_processes <= (1 << ERTS_R9_PROC_BITS)); + + if (erts_proc.max > (1 << proc_bits)) + erts_proc.max = 1 << proc_bits; + + proc_tab_sz = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(erts_proc.max + * sizeof(erts_smp_atomic_t)); + erts_proc.tab = erts_alloc(ERTS_ALC_T_PROC_TABLE, proc_tab_sz); + proc_tab_end = ((char *) erts_proc.tab) + proc_tab_sz; + proc_entry = erts_proc.tab; + while (proc_tab_end > ((char *) proc_entry)) { + erts_smp_atomic_init_nob(proc_entry, ERTS_AINT_NULL); + proc_entry++; } - process_tab = (Process**) erts_alloc(ERTS_ALC_T_PROC_TABLE, - erts_max_processes*sizeof(Process*)); - sys_memzero(process_tab, erts_max_processes * sizeof(Process*)); + erts_smp_rwmtx_init_opt(&erts_proc_tab_rwmtx, + &proc_tab_rwmtx_opts, + "proc_tab"); + last_pid_data_init_nob(~((Uint64) 0)); - erts_smp_mtx_init(&proc_tab_mtx, "proc_tab"); - p_last = -1; - p_next = 0; - p_serial = 0; + max_proc_bits = erts_fit_in_bits_int32((Sint32) erts_proc.max - 1); + + erts_proc.tab_cache_lines = proc_tab_sz/ERTS_CACHE_LINE_SIZE; + erts_proc.pix_per_cache_line = ERTS_CACHE_LINE_SIZE/sizeof(erts_smp_atomic_t); + if ((erts_proc.max & (erts_proc.max - 1)) + | (erts_proc.pix_per_cache_line & (erts_proc.pix_per_cache_line - 1))) { + /* + * erts_proc.max or erts_proc.pix_per_cache_line + * not a power of 2 :( + */ + erts_proc.pix_cl_mask = 0; + erts_proc.pix_cl_shift = 0; + erts_proc.pix_cli_mask = 0; + erts_proc.pix_cli_shift = 0; + } + else { + ASSERT((erts_proc.tab_cache_lines + & (erts_proc.tab_cache_lines - 1)) == 0); + erts_proc.pix_cl_mask = erts_proc.tab_cache_lines-1; + erts_proc.pix_cl_shift = erts_fit_in_bits_int32(erts_proc.pix_per_cache_line-1); + erts_proc.pix_cli_shift = erts_fit_in_bits_int32(erts_proc.pix_cl_mask); + erts_proc.pix_cli_mask = (1 << (max_proc_bits - erts_proc.pix_cli_shift)) - 1; + } - p_serial_shift = erts_fit_in_bits(erts_max_processes - 1); - p_serial_mask = ((~(~((Uint) 0) << proc_bits)) >> p_serial_shift); - erts_process_tab_index_mask = ~(~((Uint) 0) << p_serial_shift); last_reductions = 0; last_exact_reductions = 0; erts_default_process_flags = 0; @@ -732,8 +978,9 @@ static ERTS_INLINE ErtsProcList * proclist_create(Process *p) { ErtsProcList *plp = proclist_alloc(); + ensure_later_proc_interval(p->started_interval); plp->pid = p->id; - plp->started = p->started; + plp->started_interval = p->started_interval; return plp; } @@ -746,8 +993,7 @@ proclist_destroy(ErtsProcList *plp) static ERTS_INLINE int proclist_same(ErtsProcList *plp, Process *p) { - return (plp->pid == p->id - && erts_cmp_timeval(&plp->started, &p->started) == 0); + return plp->pid == p->id && plp->started_interval == p->started_interval; } ErtsProcList * @@ -1145,7 +1391,45 @@ handle_async_ready_clean(ErtsAuxWorkData *awdp, } } +#endif /* ERTS_USE_ASYNC_READY_Q */ + +#ifdef ERTS_SMP +void +erts_notify_code_ix_activation(Process* p, ErtsThrPrgrVal later) +{ + ErtsAuxWorkData* awdp = &p->scheduler_data->aux_work_data; + ASSERT(awdp->code_ix_activation.code_stager == NULL); + awdp->code_ix_activation.code_stager = p; + awdp->code_ix_activation.thr_prgr = later; + erts_smp_proc_inc_refc(p); + set_aux_work_flags_wakeup_relb(p->scheduler_data->ssi, + ERTS_SSI_AUX_WORK_CODE_IX_ACTIVATION); +} + +static erts_aint32_t +handle_code_ix_activation(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) +{ + Process* p; + if (!erts_thr_progress_has_reached(awdp->code_ix_activation.thr_prgr)) { + return aux_work & ~ERTS_SSI_AUX_WORK_CODE_IX_ACTIVATION; + } + unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_CODE_IX_ACTIVATION); + p = awdp->code_ix_activation.code_stager; + ASSERT(p); +#ifdef DEBUG + awdp->code_ix_activation.code_stager = NULL; #endif + erts_commit_staging_code_ix(); + erts_release_code_write_permission(); + erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + if (!ERTS_PROC_IS_EXITING(p)) { + erts_resume(p, ERTS_PROC_LOCK_STATUS); + } + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + erts_smp_proc_dec_refc(p); + return aux_work & ~ERTS_SSI_AUX_WORK_CODE_IX_ACTIVATION; +} +#endif /* ERTS_SMP */ static ERTS_INLINE erts_aint32_t handle_fix_alloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) @@ -1510,6 +1794,11 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work) handle_mseg_cache_check); #endif +#ifdef ERTS_SMP + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_CODE_IX_ACTIVATION, + handle_code_ix_activation); +#endif + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_REAP_PORTS, handle_reap_ports); @@ -1666,8 +1955,8 @@ sched_waiting_sys(Uint no, ErtsRunQueue *rq) { ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); ASSERT(rq->waiting >= 0); - rq->flags |= (ERTS_RUNQ_FLG_OUT_OF_WORK - | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK); + ERTS_RUNQ_FLGS_SET(rq, (ERTS_RUNQ_FLG_OUT_OF_WORK + | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK)); rq->waiting++; rq->waiting *= -1; rq->woken = 0; @@ -1743,8 +2032,8 @@ static ERTS_INLINE void sched_waiting(Uint no, ErtsRunQueue *rq) { ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); - rq->flags |= (ERTS_RUNQ_FLG_OUT_OF_WORK - | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK); + ERTS_RUNQ_FLGS_SET(rq, (ERTS_RUNQ_FLG_OUT_OF_WORK + | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK)); if (rq->waiting < 0) rq->waiting--; else @@ -1776,9 +2065,8 @@ ongoing_multi_scheduling_block(void) static ERTS_INLINE void empty_runq(ErtsRunQueue *rq) { - erts_aint32_t oifls = erts_smp_atomic32_read_band_nob(&rq->info_flags, - ~ERTS_RUNQ_IFLG_NONEMPTY); - if (oifls & ERTS_RUNQ_IFLG_NONEMPTY) { + Uint32 old_flags = ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_NONEMPTY|ERTS_RUNQ_FLG_PROTECTED); + if (old_flags & ERTS_RUNQ_FLG_NONEMPTY) { #ifdef DEBUG erts_aint32_t empty = erts_smp_atomic32_read_nob(&no_empty_run_queues); /* @@ -1794,9 +2082,8 @@ empty_runq(ErtsRunQueue *rq) static ERTS_INLINE void non_empty_runq(ErtsRunQueue *rq) { - erts_aint32_t oifls = erts_smp_atomic32_read_bor_nob(&rq->info_flags, - ERTS_RUNQ_IFLG_NONEMPTY); - if (!(oifls & ERTS_RUNQ_IFLG_NONEMPTY)) { + Uint32 old_flags = ERTS_RUNQ_FLGS_SET(rq, ERTS_RUNQ_FLG_NONEMPTY); + if (!(old_flags & ERTS_RUNQ_FLG_NONEMPTY)) { #ifdef DEBUG erts_aint32_t empty = erts_smp_atomic32_read_nob(&no_empty_run_queues); /* @@ -2411,19 +2698,16 @@ try_inc_no_active_runqs(int active) static ERTS_INLINE int chk_wake_sched(ErtsRunQueue *crq, int ix, int activate) { - erts_aint32_t iflgs; + Uint32 flags; ErtsRunQueue *wrq; if (crq->ix == ix) return 0; wrq = ERTS_RUNQ_IX(ix); - iflgs = erts_smp_atomic32_read_nob(&wrq->info_flags); - if (!(iflgs & (ERTS_RUNQ_IFLG_SUSPENDED|ERTS_RUNQ_IFLG_NONEMPTY))) { + flags = ERTS_RUNQ_FLGS_GET(wrq); + if (!(flags & (ERTS_RUNQ_FLG_SUSPENDED|ERTS_RUNQ_FLG_NONEMPTY))) { if (activate) { - if (try_inc_no_active_runqs(ix+1)) { - erts_smp_xrunq_lock(crq, wrq); - wrq->flags &= ~ERTS_RUNQ_FLG_INACTIVE; - erts_smp_xrunq_unlock(crq, wrq); - } + if (try_inc_no_active_runqs(ix+1)) + ERTS_RUNQ_FLGS_UNSET(wrq, ERTS_RUNQ_FLG_INACTIVE); } wake_scheduler(wrq, 0); return 1; @@ -2490,9 +2774,7 @@ erts_sched_notify_check_cpu_bind(void) int ix; for (ix = 0; ix < erts_no_run_queues; ix++) { ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); - erts_smp_runq_lock(rq); - rq->flags |= ERTS_RUNQ_FLG_CHK_CPU_BIND; - erts_smp_runq_unlock(rq); + ERTS_RUNQ_FLGS_SET(rq, ERTS_RUNQ_FLG_CHK_CPU_BIND); wake_scheduler(rq, 0); } #else @@ -2501,409 +2783,547 @@ erts_sched_notify_check_cpu_bind(void) } -#ifdef ERTS_SMP +static ERTS_INLINE void +enqueue_process(ErtsRunQueue *runq, int prio, Process *p) +{ + ErtsRunPrioQueue *rpq; -ErtsRunQueue * -erts_prepare_emigrate(ErtsRunQueue *c_rq, ErtsRunQueueInfo *c_rqi, int prio) + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq)); + + erts_smp_inc_runq_len(runq, &runq->procs.prio_info[prio], prio); + + if (prio == PRIORITY_LOW) { + p->schedule_count = RESCHEDULE_LOW; + rpq = &runq->procs.prio[PRIORITY_NORMAL]; + } + else { + p->schedule_count = 1; + rpq = &runq->procs.prio[prio]; + } + + p->next = NULL; + if (rpq->last) + rpq->last->next = p; + else + rpq->first = p; + rpq->last = p; +} + + +static ERTS_INLINE void +unqueue_process(ErtsRunQueue *runq, + ErtsRunPrioQueue *rpq, + ErtsRunQueueInfo *rqi, + int prio, + Process *prev_proc, + Process *proc) { - ASSERT(ERTS_CHK_RUNQ_FLG_EMIGRATE(c_rq->flags, prio)); - ASSERT(ERTS_CHK_RUNQ_FLG_EVACUATE(c_rq->flags, prio) - || c_rqi->len >= c_rqi->migrate.limit.this); + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq)); - while (1) { - ErtsRunQueue *n_rq = c_rqi->migrate.runq; - ERTS_DBG_VERIFY_VALID_RUNQP(n_rq); - erts_smp_xrunq_lock(c_rq, n_rq); - - /* - * erts_smp_xrunq_lock() may release lock on c_rq! We have - * to check that we still want to emigrate and emigrate - * to the same run queue as before. - */ + if (prev_proc) + prev_proc->next = proc->next; + else + rpq->first = proc->next; + if (!proc->next) + rpq->last = prev_proc; - if (ERTS_CHK_RUNQ_FLG_EMIGRATE(c_rq->flags, prio)) { - Uint32 force = (ERTS_CHK_RUNQ_FLG_EVACUATE(c_rq->flags, prio) - | (c_rq->flags & ERTS_RUNQ_FLG_INACTIVE)); - if (force || c_rqi->len > c_rqi->migrate.limit.this) { - ErtsRunQueueInfo *n_rqi; - /* We still want to emigrate */ - - if (n_rq != c_rqi->migrate.runq) { - /* Ahh... run queue changed; need to do it all over again... */ - erts_smp_runq_unlock(n_rq); - continue; - } - else { + if (!rpq->first) + rpq->last = NULL; - if (prio == ERTS_PORT_PRIO_LEVEL) - n_rqi = &n_rq->ports.info; - else - n_rqi = &n_rq->procs.prio_info[prio]; + erts_smp_dec_runq_len(runq, rqi, prio); +} - if (force || (n_rqi->len < c_rqi->migrate.limit.other)) { - /* emigrate ... */ - return n_rq; - } - } - } - } - ASSERT(n_rq != c_rq); - erts_smp_runq_unlock(n_rq); - if (!(c_rq->flags & ERTS_RUNQ_FLG_INACTIVE)) { - /* No more emigrations to this runq */ - ERTS_UNSET_RUNQ_FLG_EMIGRATE(c_rq->flags, prio); - ERTS_DBG_SET_INVALID_RUNQP(c_rqi->migrate.runq, 0x3); - } +static ERTS_INLINE Process * +dequeue_process(ErtsRunQueue *runq, int prio_q, erts_aint32_t *statep) +{ + erts_aint32_t state; + int prio; + ErtsRunPrioQueue *rpq; + ErtsRunQueueInfo *rqi; + Process *p; + + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq)); + + ASSERT(PRIORITY_NORMAL == prio_q + || PRIORITY_HIGH == prio_q + || PRIORITY_MAX == prio_q); + + rpq = &runq->procs.prio[prio_q]; + p = rpq->first; + if (!p) + return NULL; + + ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER; + + state = erts_smp_atomic32_read_nob(&p->state); + if (statep) + *statep = state; + + prio = (int) (ERTS_PSFLG_PRIO_MASK & state); + + rqi = &runq->procs.prio_info[prio]; + + if (p) + unqueue_process(runq, rpq, rqi, prio, NULL, p); + return p; +} + +static ERTS_INLINE int +check_requeue_process(ErtsRunQueue *rq, int prio_q) +{ + ErtsRunPrioQueue *rpq = &rq->procs.prio[prio_q]; + Process *p = rpq->first; + if (--p->schedule_count > 0 && p != rpq->last) { + /* reschedule */ + rpq->first = p->next; + rpq->last->next = p; + rpq->last = p; + p->next = NULL; + return 1; + } + return 0; +} + +#ifdef ERTS_SMP + +static ErtsRunQueue * +check_immigration_need(ErtsRunQueue *c_rq, ErtsMigrationPath *mp, int prio) +{ + int len; + Uint32 f_flags, f_rq_flags; + ErtsRunQueue *f_rq; + + f_flags = mp->prio[prio].flags; + + ASSERT(ERTS_CHK_RUNQ_FLG_IMMIGRATE(mp->flags, prio)); + + f_rq = mp->prio[prio].runq; + if (!f_rq) + return NULL; + + f_rq_flags = ERTS_RUNQ_FLGS_GET(f_rq); + if (f_rq_flags & ERTS_RUNQ_FLG_PROTECTED) return NULL; + + if (ERTS_CHK_RUNQ_FLG_EVACUATE(f_flags, prio)) + return f_rq; + + if (f_rq_flags & ERTS_RUNQ_FLG_INACTIVE) + return f_rq; + + if (prio == ERTS_PORT_PRIO_LEVEL) + len = RUNQ_READ_LEN(&c_rq->ports.info.len); + else + len = RUNQ_READ_LEN(&c_rq->procs.prio_info[prio].len); + + if (len < mp->prio[prio].limit.this) { + if (prio == ERTS_PORT_PRIO_LEVEL) + len = RUNQ_READ_LEN(&f_rq->ports.info.len); + else + len = RUNQ_READ_LEN(&f_rq->procs.prio_info[prio].len); + + if (len > mp->prio[prio].limit.other) + return f_rq; } + return NULL; } static void -immigrate(ErtsRunQueue *rq) +immigrate(ErtsRunQueue *c_rq, ErtsMigrationPath *mp) { - int prio; + Uint32 iflags, iflag; + erts_smp_runq_unlock(c_rq); - ASSERT(rq->flags & ERTS_RUNQ_FLGS_IMMIGRATE_QMASK); + ASSERT(erts_thr_progress_is_managed_thread()); - for (prio = 0; prio < ERTS_NO_PRIO_LEVELS; prio++) { - if (ERTS_CHK_RUNQ_FLG_IMMIGRATE(rq->flags, prio)) { - ErtsRunQueueInfo *rqi = (prio == ERTS_PORT_PRIO_LEVEL - ? &rq->ports.info - : &rq->procs.prio_info[prio]); - ErtsRunQueue *from_rq = rqi->migrate.runq; - int rq_locked, from_rq_locked; + iflags = mp->flags & ERTS_RUNQ_FLGS_IMMIGRATE_QMASK; - ERTS_DBG_VERIFY_VALID_RUNQP(from_rq); + iflag = iflags & -iflags; - rq_locked = 1; - from_rq_locked = 1; - erts_smp_xrunq_lock(rq, from_rq); - /* - * erts_smp_xrunq_lock() may release lock on rq! We have - * to check that we still want to immigrate from the same - * run queue as before. - */ - if (ERTS_CHK_RUNQ_FLG_IMMIGRATE(rq->flags, prio) - && from_rq == rqi->migrate.runq) { - ErtsRunQueueInfo *from_rqi = (prio == ERTS_PORT_PRIO_LEVEL - ? &from_rq->ports.info - : &from_rq->procs.prio_info[prio]); - if ((ERTS_CHK_RUNQ_FLG_EVACUATE(rq->flags, prio) - && ERTS_CHK_RUNQ_FLG_EVACUATE(from_rq->flags, prio) - && from_rqi->len) - || (from_rqi->len > rqi->migrate.limit.other - && rqi->len < rqi->migrate.limit.this)) { - if (prio == ERTS_PORT_PRIO_LEVEL) { - Port *prt = from_rq->ports.start; - if (prt) { - int prt_locked = 0; - (void) erts_port_migrate(prt, &prt_locked, - from_rq, &from_rq_locked, - rq, &rq_locked); - if (prt_locked) - erts_smp_port_unlock(prt); - } - } - else { - Process *proc; - ErtsRunPrioQueue *from_rpq; - from_rpq = (prio == PRIORITY_LOW - ? &from_rq->procs.prio[PRIORITY_NORMAL] - : &from_rq->procs.prio[prio]); - for (proc = from_rpq->first; proc; proc = proc->next) - if (proc->prio == prio && !proc->bound_runq) - break; - if (proc) { - ErtsProcLocks proc_locks = 0; - (void) erts_proc_migrate(proc, &proc_locks, - from_rq, &from_rq_locked, - rq, &rq_locked); - if (proc_locks) - erts_smp_proc_unlock(proc, proc_locks); - } + while (iflag) { + ErtsRunQueue *rq; + int prio; + + switch (iflag) { + case (MAX_BIT << ERTS_RUNQ_FLGS_IMMIGRATE_SHFT): + prio = PRIORITY_MAX; + break; + case (HIGH_BIT << ERTS_RUNQ_FLGS_IMMIGRATE_SHFT): + prio = PRIORITY_HIGH; + break; + case (NORMAL_BIT << ERTS_RUNQ_FLGS_IMMIGRATE_SHFT): + prio = PRIORITY_NORMAL; + break; + case (LOW_BIT << ERTS_RUNQ_FLGS_IMMIGRATE_SHFT): + prio = PRIORITY_LOW; + break; + case (PORT_BIT << ERTS_RUNQ_FLGS_IMMIGRATE_SHFT): + prio = ERTS_PORT_PRIO_LEVEL; + break; + default: + erl_exit(ERTS_ABORT_EXIT, + "%s:%d:%s(): Invalid immigrate queue mask", + __FILE__, __LINE__, __func__); + prio = 0; + break; + } + + iflags &= ~iflag; + iflag = iflags & -iflags; + + rq = check_immigration_need(c_rq, mp, prio); + if (rq) { + erts_smp_runq_lock(rq); + if (prio == ERTS_PORT_PRIO_LEVEL) { + Port *prt; + prt = erts_dequeue_port(rq); + if (prt) + RUNQ_SET_RQ(&prt->run_queue, c_rq); + erts_smp_runq_unlock(rq); + if (prt) { + /* port might terminate while we have no lock... */ + rq = erts_port_runq(prt); + if (rq) { + if (rq != c_rq) + erl_exit(ERTS_ABORT_EXIT, + "%s:%d:%s(): Internal error", + __FILE__, __LINE__, __func__); + erts_enqueue_port(c_rq, prt); + if (!iflag) + return; /* done */ + erts_smp_runq_unlock(c_rq); } } - else { - ERTS_UNSET_RUNQ_FLG_IMMIGRATE(rq->flags, prio); - ERTS_DBG_SET_INVALID_RUNQP(rqi->migrate.runq, 0x1); + } + else { + ErtsRunPrioQueue *rpq = &rq->procs.prio[prio == PRIORITY_LOW + ? PRIORITY_NORMAL + : prio]; + Process *prev_proc = NULL; + Process *proc = rpq->first; + int rq_locked = 1; + + while (proc) { + erts_aint32_t state; + state = erts_smp_atomic32_read_acqb(&proc->state); + if (!(ERTS_PSFLG_BOUND & state) + && (prio == (int) (ERTS_PSFLG_PRIO_MASK & state))) { + ErtsRunQueueInfo *rqi = &rq->procs.prio_info[prio]; + unqueue_process(rq, rpq, rqi, prio, prev_proc, proc); + erts_smp_runq_unlock(rq); + RUNQ_SET_RQ(&proc->run_queue, c_rq); + rq_locked = 0; + + erts_smp_runq_lock(c_rq); + enqueue_process(c_rq, prio, proc); + if (!iflag) + return; /* done */ + erts_smp_runq_unlock(c_rq); + break; + } + prev_proc = proc; + proc = proc->next; } + if (rq_locked) + erts_smp_runq_unlock(rq); } - if (from_rq_locked) - erts_smp_runq_unlock(from_rq); - if (!rq_locked) - erts_smp_runq_lock(rq); } } + + erts_smp_runq_lock(c_rq); +} + +static ERTS_INLINE void +suspend_run_queue(ErtsRunQueue *rq) +{ + erts_smp_atomic32_read_bor_nob(&rq->scheduler->ssi->flags, + ERTS_SSI_FLG_SUSPENDED); + ERTS_RUNQ_FLGS_SET(rq, ERTS_RUNQ_FLG_SUSPENDED); + + wake_scheduler(rq, 0); } +static void scheduler_ix_resume_wake(Uint ix); + +static ERTS_INLINE void +resume_run_queue(ErtsRunQueue *rq) +{ + int pix; + + erts_smp_runq_lock(rq); + + (void) ERTS_RUNQ_FLGS_MASK_SET(rq, + (ERTS_RUNQ_FLG_OUT_OF_WORK + | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK + | ERTS_RUNQ_FLG_SUSPENDED), + (ERTS_RUNQ_FLG_OUT_OF_WORK + | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK)); + + rq->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS; + for (pix = 0; pix < ERTS_NO_PROC_PRIO_LEVELS; pix++) { + rq->procs.prio_info[pix].max_len = 0; + rq->procs.prio_info[pix].reds = 0; + } + rq->ports.info.max_len = 0; + rq->ports.info.reds = 0; + rq->max_len = 0; + + erts_smp_runq_unlock(rq); + + scheduler_ix_resume_wake(rq->ix); +} + +typedef struct { + Process *first; + Process *last; +} ErtsStuckBoundProcesses; + static void -evacuate_run_queue(ErtsRunQueue *evac_rq, ErtsRunQueue *rq) +schedule_bound_processes(ErtsRunQueue *rq, + ErtsStuckBoundProcesses *sbpp) { - Port *prt; - int notify_to_rq = 0; - int prio; - int prt_locked = 0; - int rq_locked = 0; - int evac_rq_locked = 1; - ErtsMigrateResult mres; + Process *proc, *next; + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); - erts_smp_runq_lock(evac_rq); + proc = sbpp->first; + while (proc) { + erts_aint32_t state = erts_smp_atomic32_read_acqb(&proc->state); + next = proc->next; + enqueue_process(rq, (int) (ERTS_PSFLG_PRIO_MASK & state), proc); + proc = next; + } +} - erts_smp_atomic32_read_bor_nob(&evac_rq->scheduler->ssi->flags, - ERTS_SSI_FLG_SUSPENDED); +static void +evacuate_run_queue(ErtsRunQueue *rq, + ErtsStuckBoundProcesses *sbpp) +{ + int prio_q; + ErtsRunQueue *to_rq; + ErtsMigrationPaths *mps; + ErtsMigrationPath *mp; - evac_rq->flags &= ~ERTS_RUNQ_FLGS_IMMIGRATE_QMASK; - evac_rq->flags |= (ERTS_RUNQ_FLGS_EMIGRATE_QMASK - | ERTS_RUNQ_FLGS_EVACUATE_QMASK - | ERTS_RUNQ_FLG_SUSPENDED); + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); - erts_smp_atomic32_read_bor_nob(&evac_rq->info_flags, ERTS_RUNQ_IFLG_SUSPENDED); - /* - * Need to set up evacuation paths first since we - * may release the run queue lock on evac_rq - * when evacuating. - */ - evac_rq->misc.evac_runq = rq; - evac_rq->ports.info.migrate.runq = rq; - for (prio = 0; prio < ERTS_NO_PROC_PRIO_LEVELS; prio++) - evac_rq->procs.prio_info[prio].migrate.runq = rq; + ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_PROTECTED); + + mps = erts_get_migration_paths_managed(); + mp = &mps->mpath[rq->ix]; /* Evacuate scheduled misc ops */ - if (evac_rq->misc.start) { - rq_locked = 1; - erts_smp_xrunq_lock(evac_rq, rq); - if (rq->misc.end) - rq->misc.end->next = evac_rq->misc.start; + if (rq->misc.start) { + ErtsMiscOpList *start, *end; + + to_rq = mp->misc_evac_runq; + if (!to_rq) + return; + + start = rq->misc.start; + end = rq->misc.end; + rq->misc.start = NULL; + rq->misc.end = NULL; + erts_smp_runq_unlock(rq); + + erts_smp_runq_lock(to_rq); + if (to_rq->misc.end) + to_rq->misc.end->next = start; else - rq->misc.start = evac_rq->misc.start; - rq->misc.end = evac_rq->misc.end; - evac_rq->misc.start = NULL; - evac_rq->misc.end = NULL; + to_rq->misc.start = start; + + to_rq->misc.end = end; + erts_smp_runq_unlock(to_rq); + smp_notify_inc_runq(to_rq); + erts_smp_runq_lock(to_rq); } - /* Evacuate scheduled ports */ - prt = evac_rq->ports.start; - while (prt) { - mres = erts_port_migrate(prt, &prt_locked, - evac_rq, &evac_rq_locked, - rq, &rq_locked); - if (mres == ERTS_MIGRATE_SUCCESS) - notify_to_rq = 1; - if (prt_locked) - erts_smp_port_unlock(prt); - if (!evac_rq_locked) { - evac_rq_locked = 1; - erts_smp_runq_lock(evac_rq); + if (rq->ports.start) { + Port *prt; + + to_rq = mp->prio[ERTS_PORT_PRIO_LEVEL].runq; + if (!to_rq) + return; + + /* Evacuate scheduled ports */ + prt = rq->ports.start; + while (prt) { + ErtsRunQueue *prt_rq; + prt = erts_dequeue_port(rq); + RUNQ_SET_RQ(&prt->run_queue, to_rq); + erts_smp_runq_unlock(rq); + /* + * The port might terminate while + * we have no lock on it... + */ + prt_rq = erts_port_runq(prt); + if (prt_rq) { + if (prt_rq != to_rq) + erl_exit(ERTS_ABORT_EXIT, + "%s:%d:%s() internal error\n", + __FILE__, __LINE__, __func__); + erts_enqueue_port(to_rq, prt); + erts_smp_runq_unlock(to_rq); + } + erts_smp_runq_lock(rq); + prt = rq->ports.start; } - prt = evac_rq->ports.start; + smp_notify_inc_runq(to_rq); } /* Evacuate scheduled processes */ - for (prio = 0; prio < ERTS_NO_PROC_PRIO_LEVELS; prio++) { + for (prio_q = 0; prio_q < ERTS_NO_PROC_PRIO_QUEUES; prio_q++) { + erts_aint32_t state; Process *proc; + int notify = 0; + to_rq = NULL; - switch (prio) { - case PRIORITY_MAX: - case PRIORITY_HIGH: - case PRIORITY_NORMAL: - proc = evac_rq->procs.prio[prio].first; - while (proc) { - ErtsProcLocks proc_locks = 0; - - /* Bound processes are stuck... */ - while (proc->bound_runq) { - proc = proc->next; - if (!proc) - goto end_of_proc; - } - - mres = erts_proc_migrate(proc, &proc_locks, - evac_rq, &evac_rq_locked, - rq, &rq_locked); - if (mres == ERTS_MIGRATE_SUCCESS) - notify_to_rq = 1; - if (proc_locks) - erts_smp_proc_unlock(proc, proc_locks); - if (!evac_rq_locked) { - erts_smp_runq_lock(evac_rq); - evac_rq_locked = 1; - } + if (!mp->prio[prio_q].runq) + return; + if (prio_q == PRIORITY_NORMAL && !mp->prio[PRIORITY_LOW].runq) + return; - proc = evac_rq->procs.prio[prio].first; + proc = dequeue_process(rq, prio_q, &state); + while (proc) { + if (ERTS_PSFLG_BOUND & state) { + /* Bound processes get stuck here... */ + proc->next = NULL; + if (sbpp->last) + sbpp->last->next = proc; + else + sbpp->first = proc; + sbpp->last = proc; } + else { + int prio = (int) (ERTS_PSFLG_PRIO_MASK & state); + erts_smp_runq_unlock(rq); - end_of_proc: + to_rq = mp->prio[prio].runq; + RUNQ_SET_RQ(&proc->run_queue, to_rq); -#ifdef DEBUG - for (proc = evac_rq->procs.prio[prio].first; - proc; - proc = proc->next) { - ASSERT(proc->bound_runq); + erts_smp_runq_lock(to_rq); + enqueue_process(to_rq, prio, proc); + erts_smp_runq_unlock(to_rq); + notify = 1; + + erts_smp_runq_lock(rq); } -#endif - break; - case PRIORITY_LOW: - break; - default: - ASSERT(!"Invalid process priority"); - break; + proc = dequeue_process(rq, prio_q, &state); } + if (notify) + smp_notify_inc_runq(to_rq); } - if (rq_locked) - erts_smp_runq_unlock(rq); - - if (evac_rq_locked) - erts_smp_runq_unlock(evac_rq); - - if (notify_to_rq) - smp_notify_inc_runq(rq); - - wake_scheduler(evac_rq, 0); + if (ERTS_EMPTY_RUNQ(rq)) + empty_runq(rq); } static int -try_steal_task_from_victim(ErtsRunQueue *rq, int *rq_lockedp, ErtsRunQueue *vrq) +try_steal_task_from_victim(ErtsRunQueue *rq, int *rq_lockedp, ErtsRunQueue *vrq, Uint32 flags) { - Process *proc; - int vrq_locked; + Uint32 procs_qmask = flags & ERTS_RUNQ_FLGS_PROCS_QMASK; + int max_prio_bit; + ErtsRunPrioQueue *rpq; - if (*rq_lockedp) - erts_smp_xrunq_lock(rq, vrq); - else - erts_smp_runq_lock(vrq); - vrq_locked = 1; + if (*rq_lockedp) { + erts_smp_runq_unlock(rq); + *rq_lockedp = 0; + } - ERTS_SMP_LC_CHK_RUNQ_LOCK(rq, *rq_lockedp); - ERTS_SMP_LC_CHK_RUNQ_LOCK(vrq, vrq_locked); + ERTS_SMP_LC_ASSERT(!erts_smp_lc_runq_is_locked(rq)); + + erts_smp_runq_lock(vrq); if (rq->halt_in_progress) - goto try_steal_port; + goto no_procs; /* * Check for a runnable process to steal... */ - switch (vrq->flags & ERTS_RUNQ_FLGS_PROCS_QMASK) { - case MAX_BIT: - case MAX_BIT|HIGH_BIT: - case MAX_BIT|NORMAL_BIT: - case MAX_BIT|LOW_BIT: - case MAX_BIT|HIGH_BIT|NORMAL_BIT: - case MAX_BIT|HIGH_BIT|LOW_BIT: - case MAX_BIT|NORMAL_BIT|LOW_BIT: - case MAX_BIT|HIGH_BIT|NORMAL_BIT|LOW_BIT: - for (proc = vrq->procs.prio[PRIORITY_MAX].last; - proc; - proc = proc->prev) { - if (!proc->bound_runq) - break; - } - if (proc) + while (procs_qmask) { + Process *prev_proc; + Process *proc; + + max_prio_bit = procs_qmask & -procs_qmask; + switch (max_prio_bit) { + case MAX_BIT: + rpq = &vrq->procs.prio[PRIORITY_MAX]; break; - case HIGH_BIT: - case HIGH_BIT|NORMAL_BIT: - case HIGH_BIT|LOW_BIT: - case HIGH_BIT|NORMAL_BIT|LOW_BIT: - for (proc = vrq->procs.prio[PRIORITY_HIGH].last; - proc; - proc = proc->prev) { - if (!proc->bound_runq) - break; - } - if (proc) + case HIGH_BIT: + rpq = &vrq->procs.prio[PRIORITY_HIGH]; break; - case NORMAL_BIT: - case LOW_BIT: - case NORMAL_BIT|LOW_BIT: - for (proc = vrq->procs.prio[PRIORITY_NORMAL].last; - proc; - proc = proc->prev) { - if (!proc->bound_runq) - break; - } - if (proc) + case NORMAL_BIT: + case LOW_BIT: + rpq = &vrq->procs.prio[PRIORITY_NORMAL]; break; - case 0: - proc = NULL; - break; - default: - ASSERT(!"Invalid queue mask"); - proc = NULL; - break; - } + case 0: + goto no_procs; + default: + ASSERT(!"Invalid queue mask"); + goto no_procs; + } + + prev_proc = NULL; + proc = rpq->first; - if (proc) { - ErtsProcLocks proc_locks = 0; - int res; - ErtsMigrateResult mres; - mres = erts_proc_migrate(proc, &proc_locks, - vrq, &vrq_locked, - rq, rq_lockedp); - if (proc_locks) - erts_smp_proc_unlock(proc, proc_locks); - res = !0; - switch (mres) { - case ERTS_MIGRATE_FAILED_RUNQ_SUSPENDED: - res = 0; - case ERTS_MIGRATE_SUCCESS: - if (vrq_locked) + while (proc) { + erts_aint32_t state = erts_smp_atomic32_read_acqb(&proc->state); + if (!(ERTS_PSFLG_BOUND & state)) { + /* Steal process */ + int prio = (int) (ERTS_PSFLG_PRIO_MASK & state); + ErtsRunQueueInfo *rqi = &vrq->procs.prio_info[prio]; + unqueue_process(vrq, rpq, rqi, prio, prev_proc, proc); erts_smp_runq_unlock(vrq); - return res; - default: /* Other failures */ - break; - } - } + RUNQ_SET_RQ(&proc->run_queue, rq); - ERTS_SMP_LC_CHK_RUNQ_LOCK(rq, *rq_lockedp); - ERTS_SMP_LC_CHK_RUNQ_LOCK(vrq, vrq_locked); + erts_smp_runq_lock(rq); + *rq_lockedp = 1; + enqueue_process(rq, prio, proc); + return !0; + } + prev_proc = proc; + proc = proc->next; + } - if (!vrq_locked) { - if (*rq_lockedp) - erts_smp_xrunq_lock(rq, vrq); - else - erts_smp_runq_lock(vrq); - vrq_locked = 1; + procs_qmask &= ~max_prio_bit; } - try_steal_port: +no_procs: - ERTS_SMP_LC_CHK_RUNQ_LOCK(rq, *rq_lockedp); - ERTS_SMP_LC_CHK_RUNQ_LOCK(vrq, vrq_locked); + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(vrq)); /* * Check for a runnable port to steal... */ - if (vrq->ports.info.len) { - Port *prt = vrq->ports.end; - int prt_locked = 0; - int res; - ErtsMigrateResult mres; - - mres = erts_port_migrate(prt, &prt_locked, - vrq, &vrq_locked, - rq, rq_lockedp); - if (prt_locked) - erts_smp_port_unlock(prt); - res = !0; - switch (mres) { - case ERTS_MIGRATE_FAILED_RUNQ_SUSPENDED: - res = 0; - case ERTS_MIGRATE_SUCCESS: - if (vrq_locked) - erts_smp_runq_unlock(vrq); - return res; - default: /* Other failures */ - break; + if (vrq->ports.start) { + ErtsRunQueue *prt_rq; + Port *prt = erts_dequeue_port(vrq); + RUNQ_SET_RQ(&prt->run_queue, rq); + erts_smp_runq_unlock(vrq); + + /* + * The port might terminate while + * we have no lock on it... + */ + + prt_rq = erts_port_runq(prt); + if (!prt_rq) + return 0; + else { + if (prt_rq != rq) + erl_exit(ERTS_ABORT_EXIT, + "%s:%d:%s() internal error\n", + __FILE__, __LINE__, __func__); + *rq_lockedp = 1; + erts_enqueue_port(rq, prt); + return !0; } } - if (vrq_locked) - erts_smp_runq_unlock(vrq); + erts_smp_runq_unlock(vrq); return 0; } @@ -2913,9 +3333,10 @@ static ERTS_INLINE int check_possible_steal_victim(ErtsRunQueue *rq, int *rq_lockedp, int vix) { ErtsRunQueue *vrq = ERTS_RUNQ_IX(vix); - erts_aint32_t iflgs = erts_smp_atomic32_read_nob(&vrq->info_flags); - if (iflgs & ERTS_RUNQ_IFLG_NONEMPTY) - return try_steal_task_from_victim(rq, rq_lockedp, vrq); + Uint32 flags = ERTS_RUNQ_FLGS_GET(vrq); + if ((flags & (ERTS_RUNQ_FLG_NONEMPTY + | ERTS_RUNQ_FLG_PROTECTED)) == ERTS_RUNQ_FLG_NONEMPTY) + return try_steal_task_from_victim(rq, rq_lockedp, vrq, flags); else return 0; } @@ -2925,15 +3346,12 @@ static int try_steal_task(ErtsRunQueue *rq) { int res, rq_locked, vix, active_rqs, blnc_rqs; + Uint32 flags; - /* - * We are not allowed to steal jobs to this run queue - * if it is suspended. Note that it might get suspended - * at any time when we don't have the lock on the run - * queue. - */ - if (rq->flags & ERTS_RUNQ_FLG_SUSPENDED) - return 0; + /* Protect jobs we steal from getting stolen from us... */ + flags = ERTS_RUNQ_FLGS_SET(rq, ERTS_RUNQ_FLG_PROTECTED); + if (flags & ERTS_RUNQ_FLG_SUSPENDED) + return 0; /* go suspend instead... */ res = 0; rq_locked = 1; @@ -3052,12 +3470,123 @@ do { \ ASSERT(sum__ == (RQ)->full_reds_history_sum); \ } while (0); +#define ERTS_PRE_ALLOCED_MPATHS 8 + +erts_atomic_t erts_migration_paths; + +static struct { + size_t size; + ErtsMigrationPaths *freelist; + struct { + ErtsMigrationPaths *first; + ErtsMigrationPaths *last; + } retired; +} mpaths; + +static void +init_migration_paths(void) +{ + int qix, i; + char *p; + ErtsMigrationPaths *mps; + + mpaths.size = sizeof(ErtsMigrationPaths); + mpaths.size += sizeof(ErtsMigrationPath)*(erts_no_schedulers-1); + mpaths.size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(mpaths.size); + + p = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_LL_MPATHS, + (mpaths.size + * ERTS_PRE_ALLOCED_MPATHS)); + mpaths.freelist = NULL; + for (i = 0; i < ERTS_PRE_ALLOCED_MPATHS-1; i++) { + mps = (ErtsMigrationPaths *) p; + mps->next = mpaths.freelist; + mpaths.freelist = mps; + p += mpaths.size; + } + + mps = (ErtsMigrationPaths *) p; + mps->block = NULL; + for (qix = 0; qix < erts_no_run_queues; qix++) { + int pix; + mps->mpath[qix].flags = 0; + mps->mpath[qix].misc_evac_runq = NULL; + for (pix = 0; pix < ERTS_NO_PRIO_LEVELS; pix++) { + mps->mpath[qix].prio[pix].limit.this = -1; + mps->mpath[qix].prio[pix].limit.other = -1; + mps->mpath[qix].prio[pix].runq = NULL; + mps->mpath[qix].prio[pix].flags = 0; + } + } + erts_atomic_init_wb(&erts_migration_paths, (erts_aint_t) mps); +} + +static ERTS_INLINE ErtsMigrationPaths * +alloc_mpaths(void) +{ + void *block; + ErtsMigrationPaths *res; + ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&balance_info.update_mtx)); + + res = mpaths.freelist; + if (res) { + mpaths.freelist = res->next; + res->block = NULL; + return res; + } + res = erts_alloc(ERTS_ALC_T_SL_MPATHS, + mpaths.size+ERTS_CACHE_LINE_SIZE); + block = (void *) res; + if (((UWord) res) & ERTS_CACHE_LINE_MASK) + res = (ErtsMigrationPaths *) ((((UWord) res) & ~ERTS_CACHE_LINE_MASK) + + ERTS_CACHE_LINE_SIZE); + res->block = block; + return res; +} + +static ERTS_INLINE void +retire_mpaths(ErtsMigrationPaths *mps) +{ + ErtsThrPrgrVal current; + + ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&balance_info.update_mtx)); + + current = erts_thr_progress_current(); + + while (mpaths.retired.first) { + ErtsMigrationPaths *tmp = mpaths.retired.first; + if (!erts_thr_progress_has_reached_this(current, tmp->thr_prgr)) + break; + mpaths.retired.first = tmp->next; + if (tmp->block) { + erts_free(ERTS_ALC_T_SL_MPATHS, tmp->block); + } + else { + tmp->next = mpaths.freelist; + mpaths.freelist = tmp; + } + } + + if (!mpaths.retired.first) + mpaths.retired.last = NULL; + + mps->thr_prgr = erts_thr_progress_later_than(current); + mps->next = NULL; + + if (mpaths.retired.last) + mpaths.retired.last->next = mps; + else + mpaths.retired.first = mps; + mpaths.retired.last = mps; +} + static void check_balance(ErtsRunQueue *c_rq) { #if ERTS_MAX_PROCESSES >= (1 << 27) # error check_balance() assumes ERTS_MAX_PROCESS < (1 << 27) #endif + ErtsMigrationPaths *new_mpaths, *old_mpaths; ErtsRunQueueBalance avg = {0}; Sint64 scheds_reds, full_scheds_reds; int forced, active, current_active, oowc, half_full_scheds, full_scheds, @@ -3083,9 +3612,9 @@ check_balance(ErtsRunQueue *c_rq) ERTS_FOREACH_RUNQ(rq, { if (rq->waiting) - rq->flags |= ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK; + ERTS_RUNQ_FLGS_SET(rq, ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK); else - rq->flags &= ~ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK; + ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK); rq->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS; }); @@ -3127,7 +3656,7 @@ check_balance(ErtsRunQueue *c_rq) ErtsRunQueue *rq = ERTS_RUNQ_IX(qix); erts_smp_runq_lock(rq); - run_queue_info[qix].flags = rq->flags; + run_queue_info[qix].flags = ERTS_RUNQ_FLGS_GET_NOB(rq); for (pix = 0; pix < ERTS_NO_PROC_PRIO_LEVELS; pix++) { run_queue_info[qix].prio[pix].max_len = rq->procs.prio_info[pix].max_len; @@ -3461,47 +3990,27 @@ erts_fprintf(stderr, "--------------------------------\n"); set_no_active_runqs(active); balance_info.halftime = 1; - erts_smp_atomic32_set_nob(&balance_info.checking_balance, 0); + new_mpaths = alloc_mpaths(); + + /* Write migration paths */ - /* Write migration paths and reset balance statistics in all queues */ for (qix = 0; qix < blnc_no_rqs; qix++) { int mqix; - Uint32 flags; - ErtsRunQueue *rq = ERTS_RUNQ_IX(qix); - ErtsRunQueueInfo *rqi; - flags = run_queue_info[qix].flags; - erts_smp_runq_lock(rq); - flags |= (rq->flags & ~ERTS_RUNQ_FLGS_MIGRATION_INFO); - ASSERT(!(flags & ERTS_RUNQ_FLG_OUT_OF_WORK)); - if (rq->waiting) - flags |= ERTS_RUNQ_FLG_OUT_OF_WORK; - - rq->full_reds_history_sum - = run_queue_info[qix].full_reds_history_sum; - rq->full_reds_history[freds_hist_ix] - = run_queue_info[qix].full_reds_history_change; + Uint32 flags = run_queue_info[qix].flags; + ErtsMigrationPath *mp = &new_mpaths->mpath[qix]; - ERTS_DBG_CHK_FULL_REDS_HISTORY(rq); + mp->flags = flags; + mp->misc_evac_runq = NULL; - rq->out_of_work_count = 0; - rq->flags = flags; - rq->max_len = rq->len; for (pix = 0; pix < ERTS_NO_PRIO_LEVELS; pix++) { - rqi = (pix == ERTS_PORT_PRIO_LEVEL - ? &rq->ports.info - : &rq->procs.prio_info[pix]); - rqi->max_len = rqi->len; - rqi->reds = 0; if (!(ERTS_CHK_RUNQ_FLG_EMIGRATE(flags, pix) | ERTS_CHK_RUNQ_FLG_IMMIGRATE(flags, pix))) { ASSERT(run_queue_info[qix].prio[pix].immigrate_from < 0); ASSERT(run_queue_info[qix].prio[pix].emigrate_to < 0); -#ifdef DEBUG - rqi->migrate.limit.this = -1; - rqi->migrate.limit.other = -1; - ERTS_DBG_SET_INVALID_RUNQP(rqi->migrate.runq, 0x2); -#endif - + mp->prio[pix].limit.this = -1; + mp->prio[pix].limit.other = -1; + mp->prio[pix].runq = NULL; + mp->prio[pix].flags = 0; } else if (ERTS_CHK_RUNQ_FLG_EMIGRATE(flags, pix)) { ASSERT(!ERTS_CHK_RUNQ_FLG_IMMIGRATE(flags, pix)); @@ -3509,11 +4018,12 @@ erts_fprintf(stderr, "--------------------------------\n"); ASSERT(run_queue_info[qix].prio[pix].emigrate_to >= 0); mqix = run_queue_info[qix].prio[pix].emigrate_to; - rqi->migrate.limit.this + mp->prio[pix].limit.this = run_queue_info[qix].prio[pix].migration_limit; - rqi->migrate.limit.other + mp->prio[pix].limit.other = run_queue_info[mqix].prio[pix].migration_limit; - rqi->migrate.runq = ERTS_RUNQ_IX(mqix); + mp->prio[pix].runq = ERTS_RUNQ_IX(mqix); + mp->prio[pix].flags = run_queue_info[mqix].flags; } else { ASSERT(ERTS_CHK_RUNQ_FLG_IMMIGRATE(flags, pix)); @@ -3521,24 +4031,142 @@ erts_fprintf(stderr, "--------------------------------\n"); ASSERT(run_queue_info[qix].prio[pix].immigrate_from >= 0); mqix = run_queue_info[qix].prio[pix].immigrate_from; - rqi->migrate.limit.this + mp->prio[pix].limit.this = run_queue_info[qix].prio[pix].migration_limit; - rqi->migrate.limit.other + mp->prio[pix].limit.other = run_queue_info[mqix].prio[pix].migration_limit; - rqi->migrate.runq = ERTS_RUNQ_IX(mqix); + mp->prio[pix].runq = ERTS_RUNQ_IX(mqix); + mp->prio[pix].flags = run_queue_info[mqix].flags; } } + } + + old_mpaths = erts_get_migration_paths_managed(); + + /* Keep offline run-queues as is */ + for (qix = blnc_no_rqs; qix < erts_no_schedulers; qix++) { + ErtsMigrationPath *nmp = &new_mpaths->mpath[qix]; + ErtsMigrationPath *omp = &old_mpaths->mpath[qix]; + + nmp->flags = omp->flags; + nmp->misc_evac_runq = omp->misc_evac_runq; + + for (pix = 0; pix < ERTS_NO_PRIO_LEVELS; pix++) { + nmp->prio[pix].limit.this = omp->prio[pix].limit.this; + nmp->prio[pix].limit.other = omp->prio[pix].limit.other; + nmp->prio[pix].runq = omp->prio[pix].runq; + nmp->prio[pix].flags = omp->prio[pix].flags; + } + } + + + /* Publish new migration paths... */ + erts_atomic_set_wb(&erts_migration_paths, (erts_aint_t) new_mpaths); + + /* Reset balance statistics in all online queues */ + for (qix = 0; qix < blnc_no_rqs; qix++) { + Uint32 flags = run_queue_info[qix].flags; + ErtsRunQueue *rq = ERTS_RUNQ_IX(qix); + + erts_smp_runq_lock(rq); + ASSERT(!(flags & ERTS_RUNQ_FLG_OUT_OF_WORK)); + if (rq->waiting) + flags |= ERTS_RUNQ_FLG_OUT_OF_WORK; + + rq->full_reds_history_sum + = run_queue_info[qix].full_reds_history_sum; + rq->full_reds_history[freds_hist_ix] + = run_queue_info[qix].full_reds_history_change; + + ERTS_DBG_CHK_FULL_REDS_HISTORY(rq); + + rq->out_of_work_count = 0; + (void) ERTS_RUNQ_FLGS_MASK_SET(rq, ERTS_RUNQ_FLGS_MIGRATION_INFO, flags); + + rq->max_len = rq->len; + for (pix = 0; pix < ERTS_NO_PRIO_LEVELS; pix++) { + ErtsRunQueueInfo *rqi; + rqi = (pix == ERTS_PORT_PRIO_LEVEL + ? &rq->ports.info + : &rq->procs.prio_info[pix]); + erts_smp_reset_max_len(rq, rqi); + rqi->reds = 0; + } rq->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS; erts_smp_runq_unlock(rq); } + erts_smp_atomic32_set_nob(&balance_info.checking_balance, 0); + balance_info.n++; + retire_mpaths(old_mpaths); erts_smp_mtx_unlock(&balance_info.update_mtx); erts_smp_runq_lock(c_rq); } +static void +change_no_used_runqs(int used) +{ + ErtsMigrationPaths *new_mpaths, *old_mpaths; + int active, qix; + erts_smp_mtx_lock(&balance_info.update_mtx); + get_no_runqs(&active, NULL); + set_no_used_runqs(used); + + old_mpaths = erts_get_migration_paths_managed(); + new_mpaths = alloc_mpaths(); + + /* Write migration paths... */ + + for (qix = 0; qix < used; qix++) { + int pix; + ErtsMigrationPath *omp = &old_mpaths->mpath[qix]; + ErtsMigrationPath *nmp = &new_mpaths->mpath[qix]; + + nmp->flags = omp->flags & ~ERTS_RUNQ_FLGS_MIGRATION_QMASKS; + nmp->misc_evac_runq = NULL; + + for (pix = 0; pix < ERTS_NO_PRIO_LEVELS; pix++) { + nmp->prio[pix].limit.this = -1; + nmp->prio[pix].limit.other = -1; + nmp->prio[pix].runq = NULL; + nmp->prio[pix].flags = 0; + } + } + for (qix = used; qix < erts_no_run_queues; qix++) { + int pix; + ErtsRunQueue *to_rq = ERTS_RUNQ_IX(qix % used); + ErtsMigrationPath *nmp = &new_mpaths->mpath[qix]; + + nmp->flags = (ERTS_RUNQ_FLGS_EMIGRATE_QMASK + | ERTS_RUNQ_FLGS_EVACUATE_QMASK); + nmp->misc_evac_runq = to_rq; + for (pix = 0; pix < ERTS_NO_PRIO_LEVELS; pix++) { + nmp->prio[pix].limit.this = -1; + nmp->prio[pix].limit.other = -1; + nmp->prio[pix].runq = to_rq; + nmp->prio[pix].flags = 0; + } + } + + /* ... and publish them. */ + erts_atomic_set_wb(&erts_migration_paths, (erts_aint_t) new_mpaths); + + retire_mpaths(old_mpaths); + + /* Make sure that we balance soon... */ + balance_info.forced_check_balance = 1; + + erts_smp_mtx_unlock(&balance_info.update_mtx); + + erts_smp_runq_lock(ERTS_RUNQ_IX(0)); + ERTS_RUNQ_IX(0)->check_balance_reds = 0; + erts_smp_runq_unlock(ERTS_RUNQ_IX(0)); +} + + #endif /* #ifdef ERTS_SMP */ Uint @@ -3606,11 +4234,11 @@ static struct { int limit; int dec_shift; int dec_mask; - void (*check)(ErtsRunQueue *rq); + void (*check)(ErtsRunQueue *rq, Uint32 flags); } wakeup_other; static void -wakeup_other_check(ErtsRunQueue *rq) +wakeup_other_check(ErtsRunQueue *rq, Uint32 flags) { int wo_reds = rq->wakeup_other_reds; if (wo_reds) { @@ -3628,6 +4256,8 @@ wakeup_other_check(ErtsRunQueue *rq) if (rq->wakeup_other > wakeup_other.limit) { int empty_rqs = erts_smp_atomic32_read_acqb(&no_empty_run_queues); + if (flags & ERTS_RUNQ_FLG_PROTECTED) + ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_PROTECTED); if (empty_rqs != 0) wake_scheduler_on_empty_runq(rq); rq->wakeup_other = 0; @@ -3672,18 +4302,21 @@ wakeup_other_set_limit(void) } static void -wakeup_other_check_legacy(ErtsRunQueue *rq) +wakeup_other_check_legacy(ErtsRunQueue *rq, Uint32 flags) { int wo_reds = rq->wakeup_other_reds; if (wo_reds) { - if (rq->len < 2) { + erts_aint32_t len = rq->len; + if (len < 2) { rq->wakeup_other -= ERTS_WAKEUP_OTHER_DEC_LEGACY*wo_reds; if (rq->wakeup_other < 0) rq->wakeup_other = 0; } else if (rq->wakeup_other < wakeup_other.limit) - rq->wakeup_other += rq->len*wo_reds + ERTS_WAKEUP_OTHER_FIXED_INC_LEGACY; + rq->wakeup_other += len*wo_reds + ERTS_WAKEUP_OTHER_FIXED_INC_LEGACY; else { + if (flags & ERTS_RUNQ_FLG_PROTECTED) + ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_PROTECTED); if (erts_smp_atomic32_read_acqb(&no_empty_run_queues) != 0) { wake_scheduler_on_empty_runq(rq); rq->wakeup_other = 0; @@ -3883,7 +4516,6 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online) ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); rq->ix = ix; - erts_smp_atomic32_init_nob(&rq->info_flags, ERTS_RUNQ_IFLG_NONEMPTY); /* make sure that the "extra" id correponds to the schedulers * id if the esdp->no <-> ix+1 mapping change. @@ -3894,7 +4526,7 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online) rq->waiting = 0; rq->woken = 0; - rq->flags = 0; + ERTS_RUNQ_FLGS_INIT(rq, ERTS_RUNQ_FLG_NONEMPTY); rq->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS; rq->full_reds_history_sum = 0; for (rix = 0; rix < ERTS_FULL_REDS_HISTORY_SIZE; rix++) { @@ -3908,19 +4540,14 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online) rq->wakeup_other_reds = 0; rq->halt_in_progress = 0; - rq->procs.len = 0; rq->procs.pending_exiters = NULL; rq->procs.context_switches = 0; rq->procs.reductions = 0; for (pix = 0; pix < ERTS_NO_PROC_PRIO_LEVELS; pix++) { - rq->procs.prio_info[pix].len = 0; + erts_smp_atomic32_init_nob(&rq->procs.prio_info[pix].len, 0); rq->procs.prio_info[pix].max_len = 0; rq->procs.prio_info[pix].reds = 0; - rq->procs.prio_info[pix].migrate.limit.this = 0; - rq->procs.prio_info[pix].migrate.limit.other = 0; - ERTS_DBG_SET_INVALID_RUNQP(rq->procs.prio_info[pix].migrate.runq, - 0x0); if (pix < ERTS_NO_PROC_PRIO_LEVELS - 1) { rq->procs.prio[pix].first = NULL; rq->procs.prio[pix].last = NULL; @@ -3929,14 +4556,10 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online) rq->misc.start = NULL; rq->misc.end = NULL; - rq->misc.evac_runq = NULL; - rq->ports.info.len = 0; + erts_smp_atomic32_init_nob(&rq->ports.info.len, 0); rq->ports.info.max_len = 0; rq->ports.info.reds = 0; - rq->ports.info.migrate.limit.this = 0; - rq->ports.info.migrate.limit.other = 0; - rq->ports.info.migrate.runq = NULL; rq->ports.start = NULL; rq->ports.end = NULL; } @@ -4059,10 +4682,12 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online) balance_info.prev_rise.reds = 0; balance_info.n = 0; + init_migration_paths(); + if (no_schedulers_online < no_schedulers) { + change_no_used_runqs(no_schedulers_online); for (ix = no_schedulers_online; ix < erts_no_run_queues; ix++) - evacuate_run_queue(ERTS_RUNQ_IX(ix), - ERTS_RUNQ_IX(ix % no_schedulers_online)); + suspend_run_queue(ERTS_RUNQ_IX(ix)); } schdlr_sspnd.wait_curr_online = no_schedulers_online; @@ -4125,91 +4750,208 @@ erts_get_scheduler_data(void) #endif -static int remove_proc_from_runq(ErtsRunQueue *rq, Process *p, int to_inactive); +/* + * scheduler_out_process() return with c_rq locked. + */ +static ERTS_INLINE int +schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p) +{ + erts_aint32_t a, e, n; + int res = 0; + + a = state; + + while (1) { + n = e = a; + + ASSERT(a & ERTS_PSFLG_RUNNING); + ASSERT(!(a & ERTS_PSFLG_IN_RUNQ)); + + n &= ~ERTS_PSFLG_RUNNING; + if ((a & (ERTS_PSFLG_ACTIVE|ERTS_PSFLG_SUSPENDED)) == ERTS_PSFLG_ACTIVE) + n |= ERTS_PSFLG_IN_RUNQ; + a = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e); + if (a == e) + break; + } + + if (!(n & ERTS_PSFLG_IN_RUNQ)) { + if (erts_system_profile_flags.runnable_procs) + profile_runnable_proc(p, am_inactive); + } + else { + int prio = (int) (ERTS_PSFLG_PRIO_MASK & n); + ErtsRunQueue *runq = erts_get_runq_proc(p); + + ASSERT(!(n & ERTS_PSFLG_SUSPENDED)); + +#ifdef ERTS_SMP + if (!(ERTS_PSFLG_BOUND & n)) { + ErtsRunQueue *new_runq = erts_check_emigration_need(runq, prio); + if (new_runq) { + RUNQ_SET_RQ(&p->run_queue, new_runq); + runq = new_runq; + } + } +#endif + ASSERT(runq); + res = 1; + + erts_smp_runq_lock(runq); + + /* Enqueue the process */ + enqueue_process(runq, prio, p); + + if (runq == c_rq) + return res; + erts_smp_runq_unlock(runq); + smp_notify_inc_runq(runq); + } + erts_smp_runq_lock(c_rq); + return res; +} static ERTS_INLINE void -suspend_process(ErtsRunQueue *rq, Process *p) +add2runq(Process *p, erts_aint32_t state) { - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); - p->rcount++; /* count number of suspend */ -#ifdef ERTS_SMP - ASSERT(!(p->runq_flags & ERTS_PROC_RUNQ_FLG_RUNNING) - || p == erts_get_current_process()); - ASSERT(p->status != P_RUNNING - || p->runq_flags & ERTS_PROC_RUNQ_FLG_RUNNING); - if (p->status_flags & ERTS_PROC_SFLG_PENDADD2SCHEDQ) - goto runable; -#endif - switch(p->status) { - case P_SUSPENDED: - break; - case P_RUNABLE: + int prio = (int) (ERTS_PSFLG_PRIO_MASK & state); + ErtsRunQueue *runq = erts_get_runq_proc(p); + #ifdef ERTS_SMP - runable: - if (!ERTS_PROC_PENDING_EXIT(p)) + if (!(ERTS_PSFLG_BOUND & state)) { + ErtsRunQueue *new_runq = erts_check_emigration_need(runq, prio); + if (new_runq) { + RUNQ_SET_RQ(&p->run_queue, new_runq); + runq = new_runq; + } + } #endif - remove_proc_from_runq(rq, p, 1); - /* else: - * leave process in schedq so it will discover the pending exit - */ - p->rstatus = P_RUNABLE; /* wakeup as runnable */ - break; - case P_RUNNING: - p->rstatus = P_RUNABLE; /* wakeup as runnable */ - break; - case P_WAITING: - p->rstatus = P_WAITING; /* wakeup as waiting */ - break; - case P_EXITING: - return; /* ignore this */ - case P_GARBING: - case P_FREE: - erl_exit(1, "bad state in suspend_process()\n"); + ASSERT(runq); + + erts_smp_runq_lock(runq); + + /* Enqueue the process */ + enqueue_process(runq, prio, p); + + erts_smp_runq_unlock(runq); + smp_notify_inc_runq(runq); + +} + +static ERTS_INLINE void +schedule_process(Process *p, erts_aint32_t state, int active_enq) +{ + erts_aint32_t a = state, n; + + while (1) { + erts_aint32_t e; + n = e = a; + ASSERT(!(a & ERTS_PSFLG_FREE)); + n |= ERTS_PSFLG_ACTIVE; + if (!(a & (ERTS_PSFLG_SUSPENDED|ERTS_PSFLG_RUNNING))) + n |= ERTS_PSFLG_IN_RUNQ; + a = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e); + if (a == e) + break; + if (!active_enq && (a & ERTS_PSFLG_ACTIVE)) + return; /* Someone else activated process ... */ + } + +#ifdef RRR_DEBUG + if (active_enq) + erts_fprintf(stderr, "! state=0x%x\n", n); +#endif + + if (erts_system_profile_flags.runnable_procs + && !(a & (ERTS_PSFLG_ACTIVE|ERTS_PSFLG_SUSPENDED))) { + profile_runnable_proc(p, am_active); } - if ((erts_system_profile_flags.runnable_procs) && (p->rcount == 1) && (p->status != P_WAITING)) { - profile_runnable_proc(p, am_inactive); + if ((n & ERTS_PSFLG_IN_RUNQ) && !(a & ERTS_PSFLG_IN_RUNQ)) { +#ifdef RRR_DEBUG + if (active_enq) + erts_fprintf(stderr, "-->\n"); +#endif + add2runq(p, n); } +} - p->status = P_SUSPENDED; - +void +erts_schedule_process(Process *p, erts_aint32_t state) +{ + schedule_process(p, state, 0); } -static ERTS_INLINE void -resume_process(Process *p) +static ERTS_INLINE int +suspend_process(Process *c_p, Process *p) { - Uint32 *statusp; + erts_aint32_t state = erts_smp_atomic32_read_acqb(&p->state); + int suspended = 0; ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); - switch (p->status) { - case P_SUSPENDED: - statusp = &p->status; - break; - case P_GARBING: - if (p->gcstatus == P_SUSPENDED) { - statusp = &p->gcstatus; - break; + + if ((state & ERTS_PSFLG_SUSPENDED)) + suspended = -1; + else { + if (c_p == p) { + state = erts_smp_atomic32_read_bor_relb(&p->state, + ERTS_PSFLG_SUSPENDED); + state |= ERTS_PSFLG_SUSPENDED; + ASSERT(state & ERTS_PSFLG_RUNNING); + suspended = 1; } - /* Fall through */ - default: - return; + else { + while (!(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_EXITING))) { + erts_aint32_t e, n; + n = e = state; + n |= ERTS_PSFLG_SUSPENDED; + state = erts_smp_atomic32_cmpxchg_relb(&p->state, n, e); + if (state == e) { + state = n; + suspended = 1; + break; + } + } + } + } + + if (state & ERTS_PSFLG_SUSPENDED) { + + ASSERT(!(ERTS_PSFLG_RUNNING & state) + || p == erts_get_current_process()); + + if (erts_system_profile_flags.runnable_procs + && (p->rcount == 0) + && (state & ERTS_PSFLG_ACTIVE)) { + profile_runnable_proc(p, am_inactive); + } + + p->rcount++; /* count number of suspend */ } + return suspended; +} + +static ERTS_INLINE void +resume_process(Process *p) +{ + erts_aint32_t state; + ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); ASSERT(p->rcount > 0); if (--p->rcount > 0) /* multiple suspend */ return; - switch(p->rstatus) { - case P_RUNABLE: - erts_add_to_runq(p); - break; - case P_WAITING: - *statusp = P_WAITING; - break; - default: - erl_exit(1, "bad state in resume_process()\n"); + + state = erts_smp_atomic32_read_band_mb(&p->state, ~ERTS_PSFLG_SUSPENDED); + state &= ~ERTS_PSFLG_SUSPENDED; +#ifdef RRR_DEBUG + erts_fprintf(stderr, "%T - state=0x%x\n", p->id, state); +#endif + if ((state & (ERTS_PSFLG_EXITING + | ERTS_PSFLG_ACTIVE + | ERTS_PSFLG_IN_RUNQ + | ERTS_PSFLG_RUNNING)) == ERTS_PSFLG_ACTIVE) { + schedule_process(p, state, 1); } - p->rstatus = P_FREE; } int @@ -4333,6 +5075,7 @@ suspend_scheduler(ErtsSchedulerData *esdp) int wake = 0; erts_aint32_t aux_work; int thr_prgr_active = 1; + ErtsStuckBoundProcesses sbp = {NULL, NULL}; /* * Schedulers may be suspended in two different ways: @@ -4347,6 +5090,8 @@ suspend_scheduler(ErtsSchedulerData *esdp) ASSERT(no != 1); + evacuate_run_queue(esdp->run_queue, &sbp); + erts_smp_runq_unlock(esdp->run_queue); erts_sched_check_cpu_bind_prep_suspend(esdp); @@ -4410,17 +5155,26 @@ suspend_scheduler(ErtsSchedulerData *esdp) erts_smp_mtx_unlock(&schdlr_sspnd.mtx); while (1) { + erts_aint32_t qmask; erts_aint32_t flgs; + qmask = (ERTS_RUNQ_FLGS_GET(esdp->run_queue) + & ERTS_RUNQ_FLGS_QMASK); aux_work = erts_atomic32_read_acqb(&ssi->aux_work); - if (aux_work) { + if (aux_work|qmask) { if (!thr_prgr_active) { erts_thr_progress_active(esdp, thr_prgr_active = 1); sched_wall_time_change(esdp, 1); } - aux_work = handle_aux_work(&esdp->aux_work_data, aux_work); + if (aux_work) + aux_work = handle_aux_work(&esdp->aux_work_data, aux_work); if (aux_work && erts_thr_progress_update(esdp)) erts_thr_progress_leader_update(esdp); + if (qmask) { + erts_smp_runq_lock(esdp->run_queue); + evacuate_run_queue(esdp->run_queue, &sbp); + erts_smp_runq_unlock(esdp->run_queue); + } } if (!aux_work) { @@ -4490,56 +5244,11 @@ suspend_scheduler(ErtsSchedulerData *esdp) erts_smp_runq_lock(esdp->run_queue); non_empty_runq(esdp->run_queue); + schedule_bound_processes(esdp->run_queue, &sbp); + erts_sched_check_cpu_bind_post_suspend(esdp); } -#define ERTS_RUNQ_RESET_SUSPEND_INFO(RQ, DBG_ID) \ -do { \ - int pix__; \ - (RQ)->misc.evac_runq = NULL; \ - (RQ)->ports.info.migrate.runq = NULL; \ - (RQ)->flags &= ~(ERTS_RUNQ_FLGS_IMMIGRATE_QMASK \ - | ERTS_RUNQ_FLGS_EMIGRATE_QMASK \ - | ERTS_RUNQ_FLGS_EVACUATE_QMASK \ - | ERTS_RUNQ_FLG_SUSPENDED); \ - (RQ)->flags |= (ERTS_RUNQ_FLG_OUT_OF_WORK \ - | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK); \ - (RQ)->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS; \ - erts_smp_atomic32_read_band_nob(&(RQ)->info_flags, ~ERTS_RUNQ_IFLG_SUSPENDED);\ - for (pix__ = 0; pix__ < ERTS_NO_PROC_PRIO_LEVELS; pix__++) { \ - (RQ)->procs.prio_info[pix__].max_len = 0; \ - (RQ)->procs.prio_info[pix__].reds = 0; \ - ERTS_DBG_SET_INVALID_RUNQP((RQ)->procs.prio_info[pix__].migrate.runq,\ - (DBG_ID)); \ - } \ - (RQ)->ports.info.max_len = 0; \ - (RQ)->ports.info.reds = 0; \ -} while (0) - -#define ERTS_RUNQ_RESET_MIGRATION_PATHS__(RQ) \ -do { \ - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked((RQ))); \ - (RQ)->misc.evac_runq = NULL; \ - (RQ)->ports.info.migrate.runq = NULL; \ - (RQ)->flags &= ~(ERTS_RUNQ_FLGS_IMMIGRATE_QMASK \ - | ERTS_RUNQ_FLGS_EMIGRATE_QMASK \ - | ERTS_RUNQ_FLGS_EVACUATE_QMASK); \ -} while (0) - -#ifdef DEBUG -#define ERTS_RUNQ_RESET_MIGRATION_PATHS(RQ, DBG_ID) \ -do { \ - int pix__; \ - ERTS_RUNQ_RESET_MIGRATION_PATHS__((RQ)); \ - for (pix__ = 0; pix__ < ERTS_NO_PROC_PRIO_LEVELS; pix__++) \ - ERTS_DBG_SET_INVALID_RUNQP((RQ)->procs.prio_info[pix__].migrate.runq,\ - (DBG_ID)); \ -} while (0) -#else -#define ERTS_RUNQ_RESET_MIGRATION_PATHS(RQ, DBG_ID) \ - ERTS_RUNQ_RESET_MIGRATION_PATHS__((RQ)) -#endif - ErtsSchedSuspendResult erts_schedulers_state(Uint *total, Uint *online, @@ -4609,27 +5318,13 @@ erts_set_schedulers_online(Process *p, have_unlocked_plocks = 1; erts_smp_proc_unlock(p, plocks); } - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - erts_smp_mtx_lock(&balance_info.update_mtx); - for (ix = online; ix < no; ix++) { - ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); - erts_smp_runq_lock(rq); - ERTS_RUNQ_RESET_SUSPEND_INFO(rq, 0x5); - erts_smp_runq_unlock(rq); - scheduler_ix_resume_wake(ix); - } - /* - * Spread evacuation paths among all online - * run queues. - */ - for (ix = no; ix < erts_no_run_queues; ix++) { - ErtsRunQueue *from_rq = ERTS_RUNQ_IX(ix); - ErtsRunQueue *to_rq = ERTS_RUNQ_IX(ix % no); - evacuate_run_queue(from_rq, to_rq); - } - set_no_used_runqs(no); - erts_smp_mtx_unlock(&balance_info.update_mtx); - erts_smp_mtx_lock(&schdlr_sspnd.mtx); + change_no_used_runqs(no); + + for (ix = online; ix < no; ix++) + resume_run_queue(ERTS_RUNQ_IX(ix)); + + for (ix = no; ix < erts_no_run_queues; ix++) + suspend_run_queue(ERTS_RUNQ_IX(ix)); } res = ERTS_SCHDLR_SSPND_DONE; } @@ -4656,25 +5351,11 @@ erts_set_schedulers_online(Process *p, have_unlocked_plocks = 1; erts_smp_proc_unlock(p, plocks); } - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - erts_smp_mtx_lock(&balance_info.update_mtx); - - for (ix = 0; ix < online; ix++) { - ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); - erts_smp_runq_lock(rq); - ERTS_RUNQ_RESET_MIGRATION_PATHS(rq, 0x6); - erts_smp_runq_unlock(rq); - } - /* - * Evacutation order important! Newly suspended run queues - * has to be evacuated last. - */ - for (ix = erts_no_run_queues-1; ix >= no; ix--) - evacuate_run_queue(ERTS_RUNQ_IX(ix), - ERTS_RUNQ_IX(ix % no)); - set_no_used_runqs(no); - erts_smp_mtx_unlock(&balance_info.update_mtx); - erts_smp_mtx_lock(&schdlr_sspnd.mtx); + + change_no_used_runqs(no); + for (ix = no; ix < erts_no_run_queues; ix++) + suspend_run_queue(ERTS_RUNQ_IX(ix)); + for (ix = no; ix < online; ix++) { ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); wake_scheduler(rq, 0); @@ -4771,25 +5452,14 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) schdlr_sspnd.msb.wait_active = 2; } - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - erts_smp_mtx_lock(&balance_info.update_mtx); - set_no_used_runqs(1); - for (ix = 0; ix < online; ix++) { + change_no_used_runqs(1); + for (ix = 1; ix < erts_no_run_queues; ix++) + suspend_run_queue(ERTS_RUNQ_IX(ix)); + + for (ix = 1; ix < online; ix++) { ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); - erts_smp_runq_lock(rq); - ASSERT(!(rq->flags & ERTS_RUNQ_FLG_SUSPENDED)); - ERTS_RUNQ_RESET_MIGRATION_PATHS(rq, 0x7); - erts_smp_runq_unlock(rq); + wake_scheduler(rq, 0); } - /* - * Evacuate all activities in all other run queues - * into the first run queue. Note order is important, - * online run queues has to be evacuated last. - */ - for (ix = erts_no_run_queues-1; ix >= 1; ix--) - evacuate_run_queue(ERTS_RUNQ_IX(ix), ERTS_RUNQ_IX(0)); - erts_smp_mtx_unlock(&balance_info.update_mtx); - erts_smp_mtx_lock(&schdlr_sspnd.mtx); if (erts_smp_atomic32_read_nob(&schdlr_sspnd.active) != schdlr_sspnd.msb.wait_active) { @@ -4833,15 +5503,6 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) plp = proclist_create(p); plp->next = schdlr_sspnd.msb.procs; schdlr_sspnd.msb.procs = plp; -#ifdef DEBUG - ERTS_FOREACH_RUNQ(srq, - { - if (srq != ERTS_RUNQ_IX(0)) { - ASSERT(ERTS_EMPTY_RUNQ(srq)); - ASSERT(srq->flags & ERTS_RUNQ_FLG_SUSPENDED); - } - }); -#endif ASSERT(p->scheduler_data); } } @@ -4873,27 +5534,6 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; else { ERTS_SCHDLR_SSPND_CHNG_SET(ERTS_SCHDLR_SSPND_CHNG_MSB, 0); -#ifdef DEBUG - ERTS_FOREACH_RUNQ(rq, - { - if (rq != p->scheduler_data->run_queue) { - if (!ERTS_EMPTY_RUNQ(rq)) { - Process *rp; - int pix; - ASSERT(rq->ports.info.len == 0); - for (pix = 0; pix < ERTS_NO_PROC_PRIO_LEVELS; pix++) { - for (rp = rq->procs.prio[pix].first; - rp; - rp = rp->next) { - ASSERT(rp->bound_runq); - } - } - } - - ASSERT(rq->flags & ERTS_RUNQ_FLG_SUSPENDED); - } - }); -#endif p->flags &= ~F_HAVE_BLCKD_MSCHED; schdlr_sspnd.msb.ongoing = 0; if (schdlr_sspnd.online == 1) { @@ -4903,35 +5543,19 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) } else { int online = schdlr_sspnd.online; - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); if (plocks) { have_unlocked_plocks = 1; erts_smp_proc_unlock(p, plocks); } - erts_smp_mtx_lock(&balance_info.update_mtx); + + change_no_used_runqs(online); /* Resume all online run queues */ - for (ix = 1; ix < online; ix++) { - ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); - erts_smp_runq_lock(rq); - ERTS_RUNQ_RESET_SUSPEND_INFO(rq, 0x4); - erts_smp_runq_unlock(rq); - scheduler_ix_resume_wake(ix); - } + for (ix = 1; ix < online; ix++) + resume_run_queue(ERTS_RUNQ_IX(ix)); - /* Spread evacuation paths among all online run queues */ for (ix = online; ix < erts_no_run_queues; ix++) - evacuate_run_queue(ERTS_RUNQ_IX(ix), - ERTS_RUNQ_IX(ix % online)); - - set_no_used_runqs(online); - /* Make sure that we balance soon... */ - balance_info.forced_check_balance = 1; - erts_smp_runq_lock(ERTS_RUNQ_IX(0)); - ERTS_RUNQ_IX(0)->check_balance_reds = 0; - erts_smp_runq_unlock(ERTS_RUNQ_IX(0)); - erts_smp_mtx_unlock(&balance_info.update_mtx); - erts_smp_mtx_lock(&schdlr_sspnd.mtx); + suspend_run_queue(ERTS_RUNQ_IX(ix)); } res = ERTS_SCHDLR_SSPND_DONE; } @@ -5238,10 +5862,7 @@ handle_pend_sync_suspend(Process *suspendee, if (suspender) { ASSERT(is_nil(suspender->suspendee)); if (suspendee_alive) { - ErtsRunQueue *rq = erts_get_runq_proc(suspendee); - erts_smp_runq_lock(rq); - suspend_process(rq, suspendee); - erts_smp_runq_unlock(rq); + erts_suspend(suspendee, suspendee_locks, NULL); suspender->suspendee = suspendee->id; } /* suspender is suspended waiting for suspendee to suspend; @@ -5284,10 +5905,9 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks, resume_process(rp); } else { - ErtsRunQueue *cp_rq, *rp_rq; rp = erts_pid2proc(c_p, c_p_locks|ERTS_PROC_LOCK_STATUS, - pid, ERTS_PROC_LOCK_STATUS); + pid, pid_locks|ERTS_PROC_LOCK_STATUS); if (!rp) { c_p->flags &= ~F_P2PNR_RESCHED; @@ -5296,58 +5916,36 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks, ASSERT(!(c_p->flags & F_P2PNR_RESCHED)); - cp_rq = erts_get_runq_proc(c_p); - rp_rq = erts_get_runq_proc(rp); - erts_smp_runqs_lock(cp_rq, rp_rq); - if (rp->runq_flags & ERTS_PROC_RUNQ_FLG_RUNNING) { - running: - /* Phiu... */ - - /* - * If we got pending suspenders and suspend ourselves waiting - * to suspend another process we might deadlock. - * In this case we have to yield, be suspended by - * someone else and then do it all over again. - */ - if (!c_p->pending_suspenders) { - /* Mark rp pending for suspend by c_p */ - add_pend_suspend(rp, c_p->id, handle_pend_sync_suspend); - ASSERT(is_nil(c_p->suspendee)); - - /* Suspend c_p; when rp is suspended c_p will be resumed. */ - suspend_process(cp_rq, c_p); - c_p->flags |= F_P2PNR_RESCHED; - } - /* Yield (caller is assumed to yield immediately in bif). */ - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); - rp = ERTS_PROC_LOCK_BUSY; + if (suspend) { + if (suspend_process(c_p, rp)) + goto done; } else { - ErtsProcLocks need_locks = pid_locks & ~ERTS_PROC_LOCK_STATUS; - if (need_locks && erts_smp_proc_trylock(rp, need_locks) == EBUSY) { - erts_smp_runqs_unlock(cp_rq, rp_rq); - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); - rp = erts_pid2proc(c_p, c_p_locks|ERTS_PROC_LOCK_STATUS, - pid, pid_locks|ERTS_PROC_LOCK_STATUS); - if (!rp) - goto done; - /* run-queues may have changed */ - cp_rq = erts_get_runq_proc(c_p); - rp_rq = erts_get_runq_proc(rp); - erts_smp_runqs_lock(cp_rq, rp_rq); - if (rp->runq_flags & ERTS_PROC_RUNQ_FLG_RUNNING) { - /* Ahh... */ - erts_smp_proc_unlock(rp, - pid_locks & ~ERTS_PROC_LOCK_STATUS); - goto running; - } - } + if (!(ERTS_PSFLG_RUNNING & erts_smp_atomic32_read_acqb(&rp->state))) + goto done; + + } + + /* Other process running */ - /* rp is not running and we got the locks we want... */ - if (suspend) - suspend_process(rp_rq, rp); + /* + * If we got pending suspenders and suspend ourselves waiting + * to suspend another process we might deadlock. + * In this case we have to yield, be suspended by + * someone else and then do it all over again. + */ + if (!c_p->pending_suspenders) { + /* Mark rp pending for suspend by c_p */ + add_pend_suspend(rp, c_p->id, handle_pend_sync_suspend); + ASSERT(is_nil(c_p->suspendee)); + + /* Suspend c_p; when rp is suspended c_p will be resumed. */ + suspend_process(c_p, c_p); + c_p->flags |= F_P2PNR_RESCHED; } - erts_smp_runqs_unlock(cp_rq, rp_rq); + /* Yield (caller is assumed to yield immediately in bif). */ + erts_smp_proc_unlock(rp, pid_locks|ERTS_PROC_LOCK_STATUS); + rp = ERTS_PROC_LOCK_BUSY; } done: @@ -5404,36 +6002,26 @@ erts_pid2proc_nropt(Process *c_p, ErtsProcLocks c_p_locks, return erts_pid2proc_not_running(c_p, c_p_locks, pid, pid_locks); } -static ERTS_INLINE void -do_bif_suspend_process(ErtsSuspendMonitor *smon, - Process *suspendee, - ErtsRunQueue *locked_runq) +static ERTS_INLINE int +do_bif_suspend_process(Process *c_p, + ErtsSuspendMonitor *smon, + Process *suspendee) { ASSERT(suspendee); - ASSERT(!suspendee->is_exiting); + ASSERT(!ERTS_PROC_IS_EXITING(suspendee)); ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(suspendee)); if (smon) { if (!smon->active) { - ErtsRunQueue *rq; - - if (locked_runq) - rq = locked_runq; - else { - rq = erts_get_runq_proc(suspendee); - erts_smp_runq_lock(rq); - } - - suspend_process(rq, suspendee); - - if (!locked_runq) - erts_smp_runq_unlock(rq); + if (!suspend_process(c_p, suspendee)) + return 0; } smon->active += smon->pending; ASSERT(smon->active); smon->pending = 0; + return 1; } - + return 0; } static void @@ -5456,10 +6044,17 @@ handle_pend_bif_sync_suspend(Process *suspendee, erts_delete_suspend_monitor(&suspender->suspend_monitors, suspendee->id); else { +#ifdef DEBUG + int res; +#endif ErtsSuspendMonitor *smon; smon = erts_lookup_suspend_monitor(suspender->suspend_monitors, suspendee->id); - do_bif_suspend_process(smon, suspendee, NULL); +#ifdef DEBUG + res = +#endif + do_bif_suspend_process(suspendee, smon, suspendee); + ASSERT(!smon || res != 0); suspender->suspendee = suspendee->id; } /* suspender is suspended waiting for suspendee to suspend; @@ -5491,10 +6086,17 @@ handle_pend_bif_async_suspend(Process *suspendee, erts_delete_suspend_monitor(&suspender->suspend_monitors, suspendee->id); else { +#ifdef DEBUG + int res; +#endif ErtsSuspendMonitor *smon; smon = erts_lookup_suspend_monitor(suspender->suspend_monitors, suspendee->id); - do_bif_suspend_process(smon, suspendee, NULL); +#ifdef DEBUG + res = +#endif + do_bif_suspend_process(suspendee, smon, suspendee); + ASSERT(!smon || res != 0); } erts_smp_proc_unlock(suspender, ERTS_PROC_LOCK_LINK); } @@ -5579,7 +6181,8 @@ suspend_process_2(BIF_ALIST_2) /* This is really a piece of cake without SMP support... */ if (!smon->active) { - suspend_process(ERTS_RUNQ_IX(0), suspendee); + erts_smp_atomic32_read_bor_nob(&suspendee->state, ERTS_PSFLG_SUSPENDED); + suspend_process(BIF_P, suspendee); smon->active++; res = am_true; } @@ -5622,21 +6225,15 @@ suspend_process_2(BIF_ALIST_2) if (smon->pending && unless_suspending) res = am_false; else { - ErtsRunQueue *rq; if (smon->pending == INT_MAX) goto system_limit; smon->pending++; - rq = erts_get_runq_proc(suspendee); - erts_smp_runq_lock(rq); - if (suspendee->runq_flags & ERTS_PROC_RUNQ_FLG_RUNNING) + if (!do_bif_suspend_process(BIF_P, smon, suspendee)) add_pend_suspend(suspendee, BIF_P->id, handle_pend_bif_async_suspend); - else - do_bif_suspend_process(smon, suspendee, rq); - erts_smp_runq_unlock(rq); res = am_true; } @@ -5673,7 +6270,6 @@ suspend_process_2(BIF_ALIST_2) /* done */ } else { - ErtsRunQueue *cp_rq, *s_rq; /* We haven't got any active suspends on the suspendee */ /* @@ -5690,12 +6286,7 @@ suspend_process_2(BIF_ALIST_2) if (!unless_suspending || smon->pending == 0) smon->pending++; - cp_rq = erts_get_runq_proc(BIF_P); - s_rq = erts_get_runq_proc(suspendee); - erts_smp_runqs_lock(cp_rq, s_rq); - if (!(suspendee->runq_flags & ERTS_PROC_RUNQ_FLG_RUNNING)) { - do_bif_suspend_process(smon, suspendee, s_rq); - erts_smp_runqs_unlock(cp_rq, s_rq); + if (do_bif_suspend_process(BIF_P, smon, suspendee)) { res = (!unless_suspending || smon->active == 1 ? am_true : am_false); @@ -5715,8 +6306,7 @@ suspend_process_2(BIF_ALIST_2) * This time with BIF_P->suspendee == BIF_ARG_1 (see * above). */ - suspend_process(cp_rq, BIF_P); - erts_smp_runqs_unlock(cp_rq, s_rq); + suspend_process(BIF_P, BIF_P); goto yield; } } @@ -5724,9 +6314,15 @@ suspend_process_2(BIF_ALIST_2) } #endif /* ERTS_SMP */ - - ASSERT(suspendee->status == P_SUSPENDED || (asynchronous && smon->pending)); - ASSERT(suspendee->status == P_SUSPENDED || !smon->active); +#ifdef DEBUG + { + erts_aint32_t state = erts_smp_atomic32_read_acqb(&suspendee->state); + ASSERT((state & ERTS_PSFLG_SUSPENDED) + || (asynchronous && smon->pending)); + ASSERT((state & ERTS_PSFLG_SUSPENDED) + || !smon->active); + } +#endif erts_smp_proc_unlock(suspendee, ERTS_PROC_LOCK_STATUS); erts_smp_proc_unlock(BIF_P, xlocks); @@ -5821,9 +6417,8 @@ resume_process_1(BIF_ALIST_1) if (!suspendee) goto no_suspendee; - ASSERT(suspendee->status == P_SUSPENDED - || (suspendee->status == P_GARBING - && suspendee->gcstatus == P_SUSPENDED)); + ASSERT(ERTS_PSFLG_SUSPENDED + & erts_smp_atomic32_read_nob(&suspendee->state)); resume_process(suspendee); erts_smp_proc_unlock(suspendee, ERTS_PROC_LOCK_STATUS); @@ -5852,449 +6447,46 @@ erts_run_queues_len(Uint *qlen) Uint len = 0; ERTS_ATOMIC_FOREACH_RUNQ(rq, { - if (qlen) - qlen[i++] = rq->procs.len; - len += rq->procs.len; + Sint pqlen = 0; + int pix; + for (pix = 0; pix < ERTS_NO_PROC_PRIO_LEVELS; pix++) + pqlen += RUNQ_READ_LEN(&rq->procs.prio_info[pix].len); + + if (pqlen < 0) + pqlen = 0; + if (qlen) + qlen[i++] = pqlen; + len += pqlen; } ); return len; } -#ifdef HARDDEBUG_RUNQS -static void -check_procs_runq(ErtsRunQueue *runq, Process *p_in_q, Process *p_not_in_q) -{ - int len[ERTS_NO_PROC_PRIO_LEVELS] = {0}; - int tot_len; - int prioq, prio; - int found_p_in_q; - Process *p, *prevp; - - found_p_in_q = 0; - for (prioq = 0; prioq < ERTS_NO_PROC_PRIO_LEVELS - 1; prioq++) { - prevp = NULL; - for (p = runq->procs.prio[prioq].first; p; p = p->next) { - ASSERT(p != p_not_in_q); - if (p == p_in_q) - found_p_in_q = 1; - switch (p->prio) { - case PRIORITY_MAX: - case PRIORITY_HIGH: - case PRIORITY_NORMAL: - ASSERT(prioq == p->prio); - break; - case PRIORITY_LOW: - ASSERT(prioq == PRIORITY_NORMAL); - break; - default: - ASSERT(!"Bad prio on process"); - } - len[p->prio]++; - ASSERT(prevp == p->prev); - if (p->prev) { - ASSERT(p->prev->next == p); - } - else { - ASSERT(runq->procs.prio[prioq].first == p); - } - if (p->next) { - ASSERT(p->next->prev == p); - } - else { - ASSERT(runq->procs.prio[prioq].last == p); - } - ASSERT(p->run_queue == runq); - prevp = p; - } - } - - ASSERT(!p_in_q || found_p_in_q); - - tot_len = 0; - for (prio = 0; prio < ERTS_NO_PROC_PRIO_LEVELS; prio++) { - ASSERT(len[prio] == runq->procs.prio_info[prio].len); - if (len[prio]) { - ASSERT(runq->flags & (1 << prio)); - } - else { - ASSERT(!(runq->flags & (1 << prio))); - } - tot_len += len[prio]; - } - ASSERT(runq->procs.len == tot_len); -} -# define ERTS_DBG_CHK_PROCS_RUNQ(RQ) check_procs_runq((RQ), NULL, NULL) -# define ERTS_DBG_CHK_PROCS_RUNQ_PROC(RQ, P) check_procs_runq((RQ), (P), NULL) -# define ERTS_DBG_CHK_PROCS_RUNQ_NOPROC(RQ, P) check_procs_runq((RQ), NULL, (P)) -#else -# define ERTS_DBG_CHK_PROCS_RUNQ(RQ) -# define ERTS_DBG_CHK_PROCS_RUNQ_PROC(RQ, P) -# define ERTS_DBG_CHK_PROCS_RUNQ_NOPROC(RQ, P) -#endif - - -static ERTS_INLINE void -enqueue_process(ErtsRunQueue *runq, Process *p) -{ - ErtsRunPrioQueue *rpq; - ErtsRunQueueInfo *rqi; - - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq)); - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); - - ASSERT(p->bound_runq || !(runq->flags & ERTS_RUNQ_FLG_SUSPENDED)); - - rqi = &runq->procs.prio_info[p->prio]; - rqi->len++; - if (rqi->max_len < rqi->len) - rqi->max_len = rqi->len; - - runq->procs.len++; - runq->len++; - if (runq->max_len < runq->len) - runq->max_len = runq->len; - - runq->flags |= (1 << p->prio); - - rpq = (p->prio == PRIORITY_LOW - ? &runq->procs.prio[PRIORITY_NORMAL] - : &runq->procs.prio[p->prio]); - - p->next = NULL; - p->prev = rpq->last; - if (rpq->last) - rpq->last->next = p; - else - rpq->first = p; - rpq->last = p; - - switch (p->status) { - case P_EXITING: - break; - case P_GARBING: - p->gcstatus = P_RUNABLE; - break; - default: - p->status = P_RUNABLE; - break; - } - -#ifdef ERTS_SMP - p->status_flags |= ERTS_PROC_SFLG_INRUNQ; -#endif - - ERTS_DBG_CHK_PROCS_RUNQ_PROC(runq, p); -} - - -static ERTS_INLINE int -dequeue_process(ErtsRunQueue *runq, Process *p) -{ - ErtsRunPrioQueue *rpq; - int res = 1; - - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq)); - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); - - ERTS_DBG_CHK_PROCS_RUNQ(runq); - - rpq = &runq->procs.prio[p->prio == PRIORITY_LOW ? PRIORITY_NORMAL : p->prio]; - if (p->prev) { - p->prev->next = p->next; - } - else if (rpq->first == p) { - rpq->first = p->next; - } - else { - res = 0; - } - if (p->next) { - p->next->prev = p->prev; - } - else if (rpq->last == p) { - rpq->last = p->prev; - } - else { - ASSERT(res == 0); - } - - if (res) { - - if (--runq->procs.prio_info[p->prio].len == 0) - runq->flags &= ~(1 << p->prio); - runq->procs.len--; - runq->len--; - -#ifdef ERTS_SMP - p->status_flags &= ~ERTS_PROC_SFLG_INRUNQ; -#endif - } - - ERTS_DBG_CHK_PROCS_RUNQ_NOPROC(runq, p); - return res; -} - -/* schedule a process */ -static ERTS_INLINE ErtsRunQueue * -internal_add_to_runq(ErtsRunQueue *runq, Process *p) -{ - Uint32 prev_status = p->status; - ErtsRunQueue *add_runq; -#ifdef ERTS_SMP - - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq)); - - if (p->status_flags & ERTS_PROC_SFLG_INRUNQ) - return NULL; - else if (p->runq_flags & ERTS_PROC_RUNQ_FLG_RUNNING) { - ASSERT(ERTS_PROC_IS_EXITING(p) || p->rcount == 0); - ERTS_DBG_CHK_PROCS_RUNQ_NOPROC(runq, p); - p->status_flags |= ERTS_PROC_SFLG_PENDADD2SCHEDQ; - return NULL; - } - ASSERT(!p->scheduler_data); -#endif - - ERTS_DBG_CHK_PROCS_RUNQ_NOPROC(runq, p); -#ifndef ERTS_SMP - /* Never schedule a suspended process (ok in smp case) */ - ASSERT(ERTS_PROC_IS_EXITING(p) || p->rcount == 0); - add_runq = runq; -#else - ASSERT(!p->bound_runq || p->bound_runq == p->run_queue); - if (p->bound_runq) { - if (p->bound_runq == runq) - add_runq = runq; - else { - add_runq = p->bound_runq; - erts_smp_xrunq_lock(runq, add_runq); - } - } - else { - add_runq = erts_check_emigration_need(runq, p->prio); - if (!add_runq) - add_runq = runq; - else /* Process emigrated */ - p->run_queue = add_runq; - } -#endif - - /* Enqueue the process */ - enqueue_process(add_runq, p); - - if ((erts_system_profile_flags.runnable_procs) - && (prev_status == P_WAITING - || prev_status == P_SUSPENDED)) { - profile_runnable_proc(p, am_active); - } - - if (add_runq != runq) - erts_smp_runq_unlock(add_runq); - - return add_runq; -} - - -void -erts_add_to_runq(Process *p) -{ - ErtsRunQueue *notify_runq; - ErtsRunQueue *runq = erts_get_runq_proc(p); - erts_smp_runq_lock(runq); - notify_runq = internal_add_to_runq(runq, p); - erts_smp_runq_unlock(runq); - smp_notify_inc_runq(notify_runq); - -} - -/* Possibly remove a scheduled process we need to suspend */ - -static int -remove_proc_from_runq(ErtsRunQueue *rq, Process *p, int to_inactive) -{ - int res; - - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); - -#ifdef ERTS_SMP - if (p->status_flags & ERTS_PROC_SFLG_PENDADD2SCHEDQ) { - p->status_flags &= ~ERTS_PROC_SFLG_PENDADD2SCHEDQ; - ASSERT(!remove_proc_from_runq(rq, p, 0)); - return 1; - } -#endif - - res = dequeue_process(rq, p); - - if (res && erts_system_profile_flags.runnable_procs && to_inactive) - profile_runnable_proc(p, am_inactive); - -#ifdef ERTS_SMP - ASSERT(!(p->status_flags & ERTS_PROC_SFLG_INRUNQ)); -#endif - - return res; -} - -#ifdef ERTS_SMP - -ErtsMigrateResult -erts_proc_migrate(Process *p, ErtsProcLocks *plcks, - ErtsRunQueue *from_rq, int *from_locked, - ErtsRunQueue *to_rq, int *to_locked) -{ - ERTS_SMP_LC_ASSERT(*plcks == erts_proc_lc_my_proc_locks(p)); - ERTS_SMP_LC_ASSERT((ERTS_PROC_LOCK_STATUS & *plcks) - || from_locked); - ERTS_SMP_LC_CHK_RUNQ_LOCK(from_rq, *from_locked); - ERTS_SMP_LC_CHK_RUNQ_LOCK(to_rq, *to_locked); - - /* - * If we have the lock on the run queue to migrate to, - * check that it isn't suspended. If it is suspended, - * we will refuse to migrate to it anyway. - */ - if (*to_locked && (to_rq->flags & ERTS_RUNQ_FLG_SUSPENDED)) - return ERTS_MIGRATE_FAILED_RUNQ_SUSPENDED; - - /* We need status lock on process and locks on both run queues */ - - if (!(ERTS_PROC_LOCK_STATUS & *plcks)) { - if (erts_smp_proc_trylock(p, ERTS_PROC_LOCK_STATUS) == EBUSY) { - ErtsProcLocks lcks = *plcks; - Eterm pid = p->id; - Process *proc = *plcks ? p : NULL; - - if (*from_locked) { - *from_locked = 0; - erts_smp_runq_unlock(from_rq); - } - if (*to_locked) { - *to_locked = 0; - erts_smp_runq_unlock(to_rq); - } - - proc = erts_pid2proc_opt(proc, - lcks, - pid, - lcks|ERTS_PROC_LOCK_STATUS, - ERTS_P2P_FLG_ALLOW_OTHER_X); - if (!proc) { - *plcks = 0; - return ERTS_MIGRATE_FAILED_NOT_IN_RUNQ; - } - ASSERT(proc == p); - } - *plcks |= ERTS_PROC_LOCK_STATUS; - } - - ASSERT(!p->bound_runq); - - ERTS_SMP_LC_CHK_RUNQ_LOCK(from_rq, *from_locked); - ERTS_SMP_LC_CHK_RUNQ_LOCK(to_rq, *to_locked); - - if (p->run_queue != from_rq) - return ERTS_MIGRATE_FAILED_RUNQ_CHANGED; - - if (!*from_locked || !*to_locked) { - if (from_rq < to_rq) { - if (!*to_locked) { - if (!*from_locked) - erts_smp_runq_lock(from_rq); - erts_smp_runq_lock(to_rq); - } - else if (erts_smp_runq_trylock(from_rq) == EBUSY) { - erts_smp_runq_unlock(to_rq); - erts_smp_runq_lock(from_rq); - erts_smp_runq_lock(to_rq); - } - } - else { - if (!*from_locked) { - if (!*to_locked) - erts_smp_runq_lock(to_rq); - erts_smp_runq_lock(from_rq); - } - else if (erts_smp_runq_trylock(to_rq) == EBUSY) { - erts_smp_runq_unlock(from_rq); - erts_smp_runq_lock(to_rq); - erts_smp_runq_lock(from_rq); - } - } - *to_locked = *from_locked = 1; - } - - ERTS_SMP_LC_CHK_RUNQ_LOCK(from_rq, *from_locked); - ERTS_SMP_LC_CHK_RUNQ_LOCK(to_rq, *to_locked); - - /* Ok we now got all locks we need; do it... */ - - /* Refuse to migrate to a suspended run queue */ - if (to_rq->flags & ERTS_RUNQ_FLG_SUSPENDED) - return ERTS_MIGRATE_FAILED_RUNQ_SUSPENDED; - - if ((p->runq_flags & ERTS_PROC_RUNQ_FLG_RUNNING) - || !(p->status_flags & ERTS_PROC_SFLG_INRUNQ)) - return ERTS_MIGRATE_FAILED_NOT_IN_RUNQ; - - dequeue_process(from_rq, p); - p->run_queue = to_rq; - enqueue_process(to_rq, p); - - return ERTS_MIGRATE_SUCCESS; -} -#endif /* ERTS_SMP */ - Eterm erts_process_status(Process *c_p, ErtsProcLocks c_p_locks, Process *rp, Eterm rpid) { Eterm res = am_undefined; - Process *p; - - if (rp) { - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS - & erts_proc_lc_my_proc_locks(rp)); - p = rp; - } - else { - p = erts_pid2proc_opt(c_p, c_p_locks, - rpid, ERTS_PROC_LOCK_STATUS, - ERTS_P2P_FLG_ALLOW_OTHER_X); - } + Process *p = rp ? rp : erts_proc_lookup_raw(rpid); if (p) { - switch (p->status) { - case P_RUNABLE: - res = am_runnable; - break; - case P_WAITING: - res = am_waiting; - break; - case P_RUNNING: - res = am_running; - break; - case P_EXITING: + erts_aint32_t state = erts_smp_atomic32_read_acqb(&p->state); + if (state & ERTS_PSFLG_FREE) + res = am_free; + else if (state & ERTS_PSFLG_EXITING) res = am_exiting; - break; - case P_GARBING: + else if (state & ERTS_PSFLG_GC) res = am_garbage_collecting; - break; - case P_SUSPENDED: + else if (state & ERTS_PSFLG_SUSPENDED) res = am_suspended; - break; - case P_FREE: /* We cannot look up a process in P_FREE... */ - default: /* Not a valid status... */ - erl_exit(1, "Bad status (%b32u) found for process %T\n", - p->status, p->id); - break; - } - -#ifdef ERTS_SMP - if (!rp && (p != c_p || !(ERTS_PROC_LOCK_STATUS & c_p_locks))) - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + else if (state & ERTS_PSFLG_RUNNING) + res = am_running; + else if (state & ERTS_PSFLG_ACTIVE) + res = am_runnable; + else + res = am_waiting; } +#ifdef ERTS_SMP else { int i; ErtsSchedulerData *esdp; @@ -6309,42 +6501,40 @@ erts_process_status(Process *c_p, ErtsProcLocks c_p_locks, } erts_smp_runq_unlock(esdp->run_queue); } - -#endif - } - +#endif return res; } /* -** Suspend a process +** Suspend a currently executing process ** If we are to suspend on a port the busy_port is the thing ** otherwise busy_port is NIL */ void -erts_suspend(Process* process, ErtsProcLocks process_locks, Port *busy_port) +erts_suspend(Process* c_p, ErtsProcLocks c_p_locks, Port *busy_port) { - ErtsRunQueue *rq; - - ERTS_SMP_LC_ASSERT(process_locks == erts_proc_lc_my_proc_locks(process)); - if (!(process_locks & ERTS_PROC_LOCK_STATUS)) - erts_smp_proc_lock(process, ERTS_PROC_LOCK_STATUS); - - rq = erts_get_runq_proc(process); - - erts_smp_runq_lock(rq); +#ifdef DEBUG + int res; +#endif + ASSERT(c_p == erts_get_current_process()); + ERTS_SMP_LC_ASSERT(c_p_locks == erts_proc_lc_my_proc_locks(c_p)); + if (!(c_p_locks & ERTS_PROC_LOCK_STATUS)) + erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_STATUS); - suspend_process(rq, process); +#ifdef DEBUG + res = +#endif + suspend_process(c_p, c_p); - erts_smp_runq_unlock(rq); + ASSERT(res); if (busy_port) - erts_wake_process_later(busy_port, process); + erts_wake_process_later(busy_port, c_p); - if (!(process_locks & ERTS_PROC_LOCK_STATUS)) - erts_smp_proc_unlock(process, ERTS_PROC_LOCK_STATUS); + if (!(c_p_locks & ERTS_PROC_LOCK_STATUS)) + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS); } @@ -6385,57 +6575,54 @@ erts_resume_processes(ErtsProcList *plp) Eterm erts_get_process_priority(Process *p) { - ErtsRunQueue *rq; - Eterm value; - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); - rq = erts_get_runq_proc(p); - erts_smp_runq_lock(rq); - switch(p->prio) { - case PRIORITY_MAX: value = am_max; break; - case PRIORITY_HIGH: value = am_high; break; - case PRIORITY_NORMAL: value = am_normal; break; - case PRIORITY_LOW: value = am_low; break; - default: ASSERT(0); value = am_undefined; break; + erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state); + switch (state & ERTS_PSFLG_PRIO_MASK) { + case PRIORITY_MAX: return am_max; + case PRIORITY_HIGH: return am_high; + case PRIORITY_NORMAL: return am_normal; + case PRIORITY_LOW: return am_low; + default: ASSERT(0); return am_undefined; } - erts_smp_runq_unlock(rq); - return value; } Eterm -erts_set_process_priority(Process *p, Eterm new_value) +erts_set_process_priority(Process *p, Eterm value) { - ErtsRunQueue *rq; - Eterm old_value; - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); - rq = erts_get_runq_proc(p); -#ifdef ERTS_SMP - ASSERT(!(p->status_flags & ERTS_PROC_SFLG_INRUNQ)); -#endif - erts_smp_runq_lock(rq); - switch(p->prio) { - case PRIORITY_MAX: old_value = am_max; break; - case PRIORITY_HIGH: old_value = am_high; break; - case PRIORITY_NORMAL: old_value = am_normal; break; - case PRIORITY_LOW: old_value = am_low; break; - default: ASSERT(0); old_value = am_undefined; break; - } - switch (new_value) { - case am_max: p->prio = PRIORITY_MAX; break; - case am_high: p->prio = PRIORITY_HIGH; break; - case am_normal: p->prio = PRIORITY_NORMAL; break; - case am_low: p->prio = PRIORITY_LOW; break; - default: old_value = THE_NON_VALUE; break; + erts_aint32_t a, oprio, nprio; + + switch (value) { + case am_max: nprio = (erts_aint32_t) PRIORITY_MAX; break; + case am_high: nprio = (erts_aint32_t) PRIORITY_HIGH; break; + case am_normal: nprio = (erts_aint32_t) PRIORITY_NORMAL; break; + case am_low: nprio = (erts_aint32_t) PRIORITY_LOW; break; + default: return THE_NON_VALUE; break; } - erts_smp_runq_unlock(rq); - return old_value; -} -/* note that P_RUNNING is only set so that we don't try to remove -** running processes from the schedule queue if they exit - a running -** process not being in the schedule queue!! -** Schedule for up to INPUT_REDUCTIONS context switches, -** return 1 if more to do. -*/ + a = erts_smp_atomic32_read_nob(&p->state); + if (nprio == (a & ERTS_PSFLG_PRIO_MASK)) + oprio = nprio; + else { + erts_aint32_t e, n; + do { + oprio = a & ERTS_PSFLG_PRIO_MASK; + n = e = a; + + ASSERT(!(a & ERTS_PSFLG_IN_RUNQ)); + + n &= ~ERTS_PSFLG_PRIO_MASK; + n |= nprio; + a = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e); + } while (a != e); + } + + switch (oprio) { + case PRIORITY_MAX: return am_max; + case PRIORITY_HIGH: return am_high; + case PRIORITY_NORMAL: return am_normal; + case PRIORITY_LOW: return am_low; + default: ASSERT(0); return am_undefined; + } +} /* * schedule() is called from BEAM (process_main()) or HiPE @@ -6458,7 +6645,6 @@ erts_set_process_priority(Process *p, Eterm new_value) Process *schedule(Process *p, int calls) { ErtsRunQueue *rq; - ErtsRunPrioQueue *rpq; erts_aint_t dt; ErtsSchedulerData *esdp; int context_reds; @@ -6466,6 +6652,8 @@ Process *schedule(Process *p, int calls) int input_reductions; int actual_reds; int reds; + Uint32 flags; + erts_aint32_t state = 0; /* Supress warning... */ #ifdef USE_VM_PROBES if (p != NULL && DTRACE_ENABLED(process_unscheduled)) { @@ -6521,92 +6709,60 @@ Process *schedule(Process *p, int calls) erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); - if ((erts_system_profile_flags.runnable_procs) - && (p->status == P_WAITING)) { - profile_runnable_proc(p, am_inactive); - } + state = erts_smp_atomic32_read_acqb(&p->state); if (IS_TRACED(p)) { - if (IS_TRACED_FL(p, F_TRACE_CALLS) && p->status != P_FREE) { + if (IS_TRACED_FL(p, F_TRACE_CALLS) && !(state & ERTS_PSFLG_FREE)) { erts_schedule_time_break(p, ERTS_BP_CALL_TIME_SCHEDULE_OUT); } - switch (p->status) { - case P_EXITING: - if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED_EXIT)) - trace_sched(p, am_out_exiting); - break; - case P_FREE: + if (state & (ERTS_PSFLG_FREE|ERTS_PSFLG_EXITING)) { if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED_EXIT)) - trace_sched(p, am_out_exited); - break; - default: + trace_sched(p, ((state & ERTS_PSFLG_FREE) + ? am_out_exited + : am_out_exiting)); + } + else { if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED)) trace_sched(p, am_out); else if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED_PROCS)) trace_virtual_sched(p, am_out); - break; } } #ifdef ERTS_SMP - if (ERTS_PROC_PENDING_EXIT(p)) { - erts_handle_pending_exit(p, - ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS); - p->status_flags |= ERTS_PROC_SFLG_PENDADD2SCHEDQ; - } - - if (p->pending_suspenders) { - handle_pending_suspend(p, - ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS); - ASSERT(!(p->status_flags & ERTS_PROC_SFLG_PENDADD2SCHEDQ) - || p->rcount == 0); - } + if (state & ERTS_PSFLG_PENDING_EXIT) + erts_handle_pending_exit(p, (ERTS_PROC_LOCK_MAIN + | ERTS_PROC_LOCK_STATUS)); + if (p->pending_suspenders) + handle_pending_suspend(p, (ERTS_PROC_LOCK_MAIN + | ERTS_PROC_LOCK_STATUS)); #endif - erts_smp_runq_lock(rq); - ERTS_PROC_REDUCTIONS_EXECUTED(rq, p->prio, reds, actual_reds); + schedule_out_process(rq, state, p); /* Returns with rq locked! */ + + ERTS_PROC_REDUCTIONS_EXECUTED(rq, + (int) (state & ERTS_PSFLG_PRIO_MASK), + reds, + actual_reds); esdp->current_process = NULL; #ifdef ERTS_SMP p->scheduler_data = NULL; - p->runq_flags &= ~ERTS_PROC_RUNQ_FLG_RUNNING; - p->status_flags &= ~ERTS_PROC_SFLG_RUNNING; - - if (p->status_flags & ERTS_PROC_SFLG_PENDADD2SCHEDQ) { - ErtsRunQueue *notify_runq; - p->status_flags &= ~ERTS_PROC_SFLG_PENDADD2SCHEDQ; - notify_runq = internal_add_to_runq(rq, p); - if (notify_runq != rq) - smp_notify_inc_runq(notify_runq); - } #endif - if (p->status == P_FREE) { + if (state & ERTS_PSFLG_FREE) { #ifdef ERTS_SMP ASSERT(esdp->free_process == p); esdp->free_process = NULL; - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS); - erts_smp_proc_dec_refc(p); #else erts_free_proc(p); #endif - } else { - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS); } -#ifdef ERTS_SMP - { - ErtsProcList *pnd_xtrs = rq->procs.pending_exiters; - rq->procs.pending_exiters = NULL; + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS); - if (pnd_xtrs) { - erts_smp_runq_unlock(rq); - handle_pending_exiters(pnd_xtrs); - erts_smp_runq_lock(rq); - } - - } +#ifdef ERTS_SMP ASSERT(!esdp->free_process); #endif ASSERT(!esdp->current_process); @@ -6626,8 +6782,23 @@ Process *schedule(Process *p, int calls) ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking()); check_activities_to_run: { +#ifdef ERTS_SMP + ErtsMigrationPaths *mps; + ErtsMigrationPath *mp; #ifdef ERTS_SMP + { + ErtsProcList *pnd_xtrs = rq->procs.pending_exiters; + rq->procs.pending_exiters = NULL; + + if (pnd_xtrs) { + erts_smp_runq_unlock(rq); + handle_pending_exiters(pnd_xtrs); + erts_smp_runq_lock(rq); + } + + } +#endif if (rq->check_balance_reds <= 0) check_balance(rq); @@ -6635,20 +6806,28 @@ Process *schedule(Process *p, int calls) ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking()); ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); - if (rq->flags & ERTS_RUNQ_FLGS_IMMIGRATE_QMASK) - immigrate(rq); + mps = erts_get_migration_paths_managed(); + mp = &mps->mpath[rq->ix]; + + if (mp->flags & ERTS_RUNQ_FLGS_IMMIGRATE_QMASK) + immigrate(rq, mp); - continue_check_activities_to_run: + continue_check_activities_to_run: + flags = ERTS_RUNQ_FLGS_GET_NOB(rq); + continue_check_activities_to_run_known_flags: - if (rq->flags & (ERTS_RUNQ_FLG_CHK_CPU_BIND - | ERTS_RUNQ_FLG_SUSPENDED)) { - if (rq->flags & ERTS_RUNQ_FLG_SUSPENDED) { - ASSERT(erts_smp_atomic32_read_nob(&esdp->ssi->flags) - & ERTS_SSI_FLG_SUSPENDED); + + if (flags & (ERTS_RUNQ_FLG_CHK_CPU_BIND|ERTS_RUNQ_FLG_SUSPENDED)) { + + if (flags & ERTS_RUNQ_FLG_SUSPENDED) { suspend_scheduler(esdp); + flags = ERTS_RUNQ_FLGS_GET_NOB(rq); } - if (rq->flags & ERTS_RUNQ_FLG_CHK_CPU_BIND) + if (flags & ERTS_RUNQ_FLG_CHK_CPU_BIND) { + flags = ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_CHK_CPU_BIND); + flags &= ~ ERTS_RUNQ_FLG_CHK_CPU_BIND; erts_sched_check_cpu_bind(esdp); + } } { @@ -6677,14 +6856,12 @@ Process *schedule(Process *p, int calls) } #endif /* ERTS_SMP */ - ASSERT(rq->len == rq->procs.len + rq->ports.info.len); - - if ((rq->len == 0 && !rq->misc.start) - || (rq->halt_in_progress - && rq->ports.info.len == 0 && !rq->misc.start)) { + flags = ERTS_RUNQ_FLGS_GET_NOB(rq); + if ((!(flags & ERTS_RUNQ_FLGS_QMASK) && !rq->misc.start) + || (rq->halt_in_progress && ERTS_EMPTY_RUNQ_PORTS(rq))) { + /* Prepare for scheduler wait */ #ifdef ERTS_SMP - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); rq->wakeup_other = 0; @@ -6692,21 +6869,27 @@ Process *schedule(Process *p, int calls) empty_runq(rq); - if (rq->flags & ERTS_RUNQ_FLG_SUSPENDED) { - ASSERT(erts_smp_atomic32_read_nob(&esdp->ssi->flags) - & ERTS_SSI_FLG_SUSPENDED); + flags = ERTS_RUNQ_FLGS_GET_NOB(rq); + if (flags & ERTS_RUNQ_FLG_SUSPENDED) { non_empty_runq(rq); - goto continue_check_activities_to_run; + goto continue_check_activities_to_run_known_flags; } - else if (!(rq->flags & ERTS_RUNQ_FLG_INACTIVE)) { + else if (!(flags & ERTS_RUNQ_FLG_INACTIVE)) { + if (try_steal_task(rq)) { + non_empty_runq(rq); + goto continue_check_activities_to_run; + } + + ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_PROTECTED); + /* * Check for ERTS_RUNQ_FLG_SUSPENDED has to be done * after trying to steal a task. */ - if (try_steal_task(rq) - || (rq->flags & ERTS_RUNQ_FLG_SUSPENDED)) { + flags = ERTS_RUNQ_FLGS_GET_NOB(rq); + if (flags & ERTS_RUNQ_FLG_SUSPENDED) { non_empty_runq(rq); - goto continue_check_activities_to_run; + goto continue_check_activities_to_run_known_flags; } } @@ -6737,6 +6920,7 @@ Process *schedule(Process *p, int calls) erl_sys_schedule(1); dt = erts_do_time_read_and_reset(); if (dt) erts_bump_timer(dt); + #ifdef ERTS_SMP erts_smp_runq_lock(rq); clear_sys_scheduling(); @@ -6750,14 +6934,14 @@ Process *schedule(Process *p, int calls) exec_misc_ops(rq); #ifdef ERTS_SMP - wakeup_other.check(rq); + wakeup_other.check(rq, flags); #endif /* * Find a new port to run. */ - if (rq->ports.info.len) { + if (RUNQ_READ_LEN(&rq->ports.info.len)) { int have_outstanding_io; have_outstanding_io = erts_port_task_execute(rq, &esdp->current_port); if ((have_outstanding_io && fcalls > 2*input_reductions) @@ -6784,159 +6968,122 @@ Process *schedule(Process *p, int calls) /* * Find a new process to run. */ - pick_next_process: - - ERTS_DBG_CHK_PROCS_RUNQ(rq); - - switch (rq->flags & ERTS_RUNQ_FLGS_PROCS_QMASK) { - case MAX_BIT: - case MAX_BIT|HIGH_BIT: - case MAX_BIT|NORMAL_BIT: - case MAX_BIT|LOW_BIT: - case MAX_BIT|HIGH_BIT|NORMAL_BIT: - case MAX_BIT|HIGH_BIT|LOW_BIT: - case MAX_BIT|NORMAL_BIT|LOW_BIT: - case MAX_BIT|HIGH_BIT|NORMAL_BIT|LOW_BIT: - rpq = &rq->procs.prio[PRIORITY_MAX]; - break; - case HIGH_BIT: - case HIGH_BIT|NORMAL_BIT: - case HIGH_BIT|LOW_BIT: - case HIGH_BIT|NORMAL_BIT|LOW_BIT: - rpq = &rq->procs.prio[PRIORITY_HIGH]; - break; - case NORMAL_BIT: - rpq = &rq->procs.prio[PRIORITY_NORMAL]; - break; - case LOW_BIT: - rpq = &rq->procs.prio[PRIORITY_NORMAL]; - break; - case NORMAL_BIT|LOW_BIT: - rpq = &rq->procs.prio[PRIORITY_NORMAL]; - ASSERT(rpq->first != NULL); - p = rpq->first; - if (p->prio == PRIORITY_LOW) { - if (p == rpq->last || p->skipped >= RESCHEDULE_LOW-1) - p->skipped = 0; - else { - /* skip it */ - p->skipped++; - rpq->first = p->next; - rpq->first->prev = NULL; - rpq->last->next = p; - p->prev = rpq->last; - p->next = NULL; - rpq->last = p; + pick_next_process: { + int prio_q; + int qmask; + + flags = ERTS_RUNQ_FLGS_GET_NOB(rq); + qmask = (int) (flags & ERTS_RUNQ_FLGS_PROCS_QMASK); + switch (qmask & -qmask) { + case MAX_BIT: + prio_q = PRIORITY_MAX; + break; + case HIGH_BIT: + prio_q = PRIORITY_HIGH; + break; + case NORMAL_BIT: + case LOW_BIT: + prio_q = PRIORITY_NORMAL; + if (check_requeue_process(rq, PRIORITY_NORMAL)) goto pick_next_process; - } + break; + case 0: /* No process at all */ + default: + ASSERT(qmask == 0); + goto check_activities_to_run; } - break; - case 0: /* No process at all */ - default: - ASSERT((rq->flags & ERTS_RUNQ_FLGS_PROCS_QMASK) == 0); - ASSERT(rq->procs.len == 0); - goto check_activities_to_run; - } - - BM_START_TIMER(system); - - /* - * Take the chosen process out of the queue. - */ - ASSERT(rpq->first); /* Wrong qmask in rq->flags? */ - p = rpq->first; -#ifdef ERTS_SMP - ERTS_SMP_LC_ASSERT(rq == p->run_queue); -#endif - rpq->first = p->next; - if (!rpq->first) - rpq->last = NULL; - else - rpq->first->prev = NULL; - p->next = p->prev = NULL; + BM_START_TIMER(system); - if (--rq->procs.prio_info[p->prio].len == 0) - rq->flags &= ~(1 << p->prio); - ASSERT(rq->procs.len > 0); - rq->procs.len--; - ASSERT(rq->len > 0); - rq->len--; + /* + * Take the chosen process out of the queue. + */ + p = dequeue_process(rq, prio_q, &state); - { - Uint32 ee_flgs = (ERTS_RUNQ_FLG_EVACUATE(p->prio) - | ERTS_RUNQ_FLG_EMIGRATE(p->prio)); + ASSERT(p); /* Wrong qmask in rq->flags? */ - if ((rq->flags & (ERTS_RUNQ_FLG_SUSPENDED|ee_flgs)) == ee_flgs) - ERTS_UNSET_RUNQ_FLG_EVACUATE(rq->flags, p->prio); - } + while (1) { + erts_aint32_t exp, new, tmp; + tmp = new = exp = state; + new &= ~ERTS_PSFLG_IN_RUNQ; + tmp = state & (ERTS_PSFLG_SUSPENDED|ERTS_PSFLG_PENDING_EXIT); + if (tmp != ERTS_PSFLG_SUSPENDED) + new |= ERTS_PSFLG_RUNNING; + state = erts_smp_atomic32_cmpxchg_relb(&p->state, new, exp); + if (state == exp) { + tmp = state & (ERTS_PSFLG_SUSPENDED|ERTS_PSFLG_PENDING_EXIT); + if (tmp == ERTS_PSFLG_SUSPENDED) + goto pick_next_process; + state = new; + break; + } + } - ERTS_DBG_CHK_PROCS_RUNQ_NOPROC(rq, p); + rq->procs.context_switches++; - rq->procs.context_switches++; + esdp->current_process = p; - esdp->current_process = p; + } #ifdef ERTS_SMP - p->runq_flags |= ERTS_PROC_RUNQ_FLG_RUNNING; erts_smp_runq_unlock(rq); + if (flags & ERTS_RUNQ_FLG_PROTECTED) + ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_PROTECTED); + ERTS_SMP_CHK_NO_PROC_LOCKS; erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS); if (erts_sched_stat.enabled) { + int prio; UWord old = ERTS_PROC_SCHED_ID(p, (ERTS_PROC_LOCK_MAIN | ERTS_PROC_LOCK_STATUS), (UWord) esdp->no); int migrated = old && old != esdp->no; + prio = (int) (state & ERTS_PSFLG_PRIO_MASK); + erts_smp_spin_lock(&erts_sched_stat.lock); - erts_sched_stat.prio[p->prio].total_executed++; - erts_sched_stat.prio[p->prio].executed++; + erts_sched_stat.prio[prio].total_executed++; + erts_sched_stat.prio[prio].executed++; if (migrated) { - erts_sched_stat.prio[p->prio].total_migrated++; - erts_sched_stat.prio[p->prio].migrated++; + erts_sched_stat.prio[prio].total_migrated++; + erts_sched_stat.prio[prio].migrated++; } erts_smp_spin_unlock(&erts_sched_stat.lock); } - p->status_flags |= ERTS_PROC_SFLG_RUNNING; - p->status_flags &= ~ERTS_PROC_SFLG_INRUNQ; if (ERTS_PROC_PENDING_EXIT(p)) { erts_handle_pending_exit(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS); + state = erts_smp_atomic32_read_nob(&p->state); } ASSERT(!p->scheduler_data); p->scheduler_data = esdp; - #endif - ASSERT(p->status != P_SUSPENDED); /* Never run a suspended process */ + /* Never run a suspended process */ + ASSERT(!(ERTS_PSFLG_SUSPENDED & erts_smp_atomic32_read_nob(&p->state))); reds = context_reds; if (IS_TRACED(p)) { - switch (p->status) { - case P_EXITING: + if (state & ERTS_PSFLG_EXITING) { if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED_EXIT)) trace_sched(p, am_in_exiting); - break; - default: + } + else { if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED)) trace_sched(p, am_in); else if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED_PROCS)) trace_virtual_sched(p, am_in); - break; } if (IS_TRACED_FL(p, F_TRACE_CALLS)) { erts_schedule_time_break(p, ERTS_BP_CALL_TIME_SCHEDULE_IN); } } - if (p->status != P_EXITING) - p->status = P_RUNNING; - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); #ifdef ERTS_SMP @@ -6944,7 +7091,7 @@ Process *schedule(Process *p, int calls) erts_check_my_tracer_proc(p); #endif - if (!ERTS_PROC_IS_EXITING(p) + if (!(state & ERTS_PSFLG_EXITING) && ((FLAGS(p) & F_FORCE_GC) || (MSO(p).overhead > BIN_VHEAP_SZ(p)))) { reds -= erts_garbage_collect(p, 0, p->arg_reg, p->arity); @@ -7035,17 +7182,19 @@ erts_schedule_misc_op(void (*func)(void *), void *arg) ErtsSchedulerData *esdp = erts_get_scheduler_data(); ErtsRunQueue *rq = esdp ? esdp->run_queue : ERTS_RUNQ_IX(0); ErtsMiscOpList *molp = misc_op_list_alloc(); +#ifdef ERTS_SMP + ErtsMigrationPaths *mpaths = erts_get_migration_paths(); - erts_smp_runq_lock(rq); - - while (rq->misc.evac_runq) { - ErtsRunQueue *tmp_rq = rq->misc.evac_runq; - erts_smp_runq_unlock(rq); - rq = tmp_rq; - erts_smp_runq_lock(rq); + if (!mpaths) + rq = ERTS_RUNQ_IX(0); + else { + ErtsRunQueue *erq = mpaths->mpath[rq->ix].misc_evac_runq; + if (erq) + rq = erq; } +#endif - ASSERT(!(rq->flags & ERTS_RUNQ_FLG_SUSPENDED)); + erts_smp_runq_lock(rq); molp->next = NULL; molp->func = func; @@ -7055,7 +7204,9 @@ erts_schedule_misc_op(void (*func)(void *), void *arg) else rq->misc.start = molp; rq->misc.end = molp; + erts_smp_runq_unlock(rq); + smp_notify_inc_runq(rq); } @@ -7145,44 +7296,50 @@ erts_get_exact_total_reductions(Process *c_p, Uint *redsp, Uint *diffp) Sint erts_test_next_pid(int set, Uint next) { + Uint64 lpd; Sint res; - Sint p_prev; - + Eterm pid_data; + int first_pix = -1; - erts_smp_mtx_lock(&proc_tab_mtx); + erts_smp_rwmtx_rwlock(&erts_proc_tab_rwmtx); - if (!set) { - res = p_next < 0 ? -1 : (p_serial << p_serial_shift | p_next); - } + if (!set) + lpd = last_pid_data_read_nob(); else { - p_serial = (Sint) ((next >> p_serial_shift) & p_serial_mask); - p_next = (Sint) (erts_process_tab_index_mask & next); - - if (p_next >= erts_max_processes) { - p_next = 0; - p_serial++; - p_serial &= p_serial_mask; + lpd = (Uint64) next; + pid_data = (Eterm) (lpd & ERTS_PID_DATA_MASK__); + if (ERTS_INVALID_PID == make_internal_pid(pid_data)) { + lpd += erts_proc.max; + ASSERT(erts_pid_data2ix(pid_data) + == erts_pid_data2ix(lpd & ERTS_PID_DATA_MASK__)); } + last_pid_data_set_relb(lpd); + } - p_prev = p_next; - - do { - if (!process_tab[p_next]) - break; - p_next++; - if(p_next >= erts_max_processes) { - p_next = 0; - p_serial++; - p_serial &= p_serial_mask; + while (1) { + int pix; + lpd++; + pix = (int) (lpd % erts_proc.max); + if (first_pix < 0) + first_pix = pix; + else if (pix == first_pix) { + res = -1; + break; + } + if (ERTS_AINT_NULL == erts_smp_atomic_read_nob(&erts_proc.tab[pix])) { + pid_data = (Eterm) (lpd & ERTS_PID_DATA_MASK__); + if (ERTS_INVALID_PID == make_internal_pid(pid_data)) { + lpd += erts_proc.max; + ASSERT(erts_pid_data2ix(pid_data) + == erts_pid_data2ix(lpd & ERTS_PID_DATA_MASK__)); } - } while (p_prev != p_next); - - res = process_tab[p_next] ? -1 : (p_serial << p_serial_shift | p_next); - + res = lpd & ERTS_PID_DATA_MASK__; + break; + } } - erts_smp_mtx_unlock(&proc_tab_mtx); + erts_smp_rwmtx_rwunlock(&erts_proc_tab_rwmtx); return res; @@ -7191,6 +7348,8 @@ erts_test_next_pid(int set, Uint next) Uint erts_process_count(void) { erts_aint32_t res = erts_smp_atomic32_read_nob(&process_count); + if (res > erts_proc.max) + return erts_proc.max; ASSERT(res >= 0); return (Uint) res; } @@ -7198,97 +7357,138 @@ Uint erts_process_count(void) void erts_free_proc(Process *p) { -#if defined(ERTS_ENABLE_LOCK_COUNT) && defined(ERTS_SMP) - erts_lcnt_proc_lock_destroy(p); +#ifdef ERTS_SMP + erts_proc_lock_fin(p); #endif erts_free(ERTS_ALC_T_PROC, (void *) p); } - /* ** Allocate process and find out where to place next process. */ static Process* -alloc_process(void) +alloc_process(ErtsRunQueue *rq, erts_aint32_t state) { -#ifdef ERTS_SMP - erts_pix_lock_t *pix_lock; -#endif + int pix; Process* p; - int p_prev; + Uint64 lpd, exp_lpd; + Eterm pid_data; + erts_aint32_t proc_count; +#ifdef DEBUG + Eterm pid; +#endif - erts_smp_mtx_lock(&proc_tab_mtx); + erts_smp_rwmtx_rlock(&erts_proc_tab_rwmtx); + + proc_count = erts_smp_atomic32_inc_read_acqb(&process_count); + if (proc_count > erts_proc.max) { + while (1) { + erts_aint32_t act_proc_count; - if (p_next == -1) { - p = NULL; - goto error; /* Process table full! */ + act_proc_count = erts_smp_atomic32_cmpxchg_relb(&process_count, + proc_count-1, + proc_count); + if (act_proc_count == proc_count) + goto system_limit; + proc_count = act_proc_count; + if (proc_count <= erts_proc.max) + break; + } } p = (Process*) erts_alloc_fnf(ERTS_ALC_T_PROC, sizeof(Process)); if (!p) - goto error; /* ENOMEM */ + goto enomem; - p_last = p_next; + p->approx_started = erts_get_approx_time(); + p->started_interval = get_proc_interval(); - erts_get_emu_time(&p->started); + lpd = last_pid_data_read_acqb(); -#ifdef ERTS_SMP - pix_lock = ERTS_PIX2PIXLOCK(p_next); - erts_pix_lock(pix_lock); -#endif - ASSERT(!process_tab[p_next]); + /* Reserve slot */ + while (1) { + lpd++; + pix = erts_pid_data2ix((Eterm) (lpd & ERTS_PID_DATA_MASK__)); + if (erts_smp_atomic_read_nob(&erts_proc.tab[pix]) == ERTS_AINT_NULL) { + erts_aint_t val; + val = erts_smp_atomic_cmpxchg_relb(&erts_proc.tab[pix], + ((erts_aint_t) + ERTS_PROC_LOCK_BUSY), + ERTS_AINT_NULL); + + if (ERTS_AINT_NULL == val) + break; + } + } - process_tab[p_next] = p; - erts_smp_atomic32_inc_nob(&process_count); - p->id = make_internal_pid(p_serial << p_serial_shift | p_next); + pid_data = (Eterm) lpd & ERTS_PID_DATA_MASK__; + + p->id = make_internal_pid(pid_data); if (p->id == ERTS_INVALID_PID) { /* Do not use the invalid pid; change serial */ - p_serial++; - p_serial &= p_serial_mask; - p->id = make_internal_pid(p_serial << p_serial_shift | p_next); + lpd += erts_proc.max; + ASSERT(pix == erts_pid_data2ix((Eterm) (lpd & ERTS_PID_DATA_MASK__))); + pid_data = (Eterm) lpd & ERTS_PID_DATA_MASK__; + p->id = make_internal_pid(pid_data); ASSERT(p->id != ERTS_INVALID_PID); } - ASSERT(internal_pid_serial(p->id) <= (erts_use_r9_pids_ports - ? ERTS_MAX_PID_R9_SERIAL - : ERTS_MAX_PID_SERIAL)); + + exp_lpd = last_pid_data_read_nob(); + + /* Move last pid data forward */ + while (1) { + Uint64 act_lpd; + if (last_pid_data_cmp(lpd, exp_lpd) < 0) + break; + act_lpd = last_pid_data_cmpxchg_relb(lpd, exp_lpd); + if (act_lpd == exp_lpd) + break; + exp_lpd = act_lpd; + } #ifdef ERTS_SMP - erts_proc_lock_init(p); /* All locks locked */ - erts_pix_unlock(pix_lock); + RUNQ_SET_RQ(&p->run_queue, rq); #endif - p->rstatus = P_FREE; - p->rcount = 0; + erts_smp_atomic32_init_relb(&p->state, state); - /* - * set p_next to the next available slot - */ +#ifdef DEBUG + pid = p->id; +#endif - p_prev = p_next; +#ifdef ERTS_SMP + erts_proc_lock_init(p); /* All locks locked */ +#endif - while (1) { - p_next++; - if(p_next >= erts_max_processes) { - p_serial++; - p_serial &= p_serial_mask; - p_next = 0; - } + /* Move into slot reserved */ +#ifdef DEBUG + ASSERT(ERTS_PROC_LOCK_BUSY + == (Process *) erts_smp_atomic_xchg_relb(&erts_proc.tab[pix], + (erts_aint_t) p)); +#else + erts_smp_atomic_set_relb(&erts_proc.tab[pix], (erts_aint_t) p); +#endif - if (p_prev == p_next) { - p_next = -1; - break; /* Table full! */ - } + ASSERT(internal_pid_serial(p->id) <= (erts_use_r9_pids_ports + ? ERTS_MAX_PID_R9_SERIAL + : ERTS_MAX_PID_SERIAL)); - if (!process_tab[p_next]) - break; /* found a free slot */ - } + erts_smp_rwmtx_runlock(&erts_proc_tab_rwmtx); - error: + p->rcount = 0; - erts_smp_mtx_unlock(&proc_tab_mtx); + ASSERT(p == (Process *) + erts_smp_atomic_read_nob( + &erts_proc.tab[internal_pid_index(pid)])); return p; +enomem: +system_limit: + + erts_smp_rwmtx_runlock(&erts_proc_tab_rwmtx); + return NULL; + } Eterm @@ -7298,13 +7498,15 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). Eterm args, /* Arguments for function (must be well-formed list). */ ErlSpawnOpts* so) /* Options for spawn. */ { - ErtsRunQueue *rq, *notify_runq; + ErtsRunQueue *rq = NULL; Process *p; Sint arity; /* Number of arguments. */ Uint arg_size; /* Size of arguments. */ Uint sz; /* Needed words on heap. */ Uint heap_need; /* Size needed on heap. */ Eterm res = THE_NON_VALUE; + erts_aint32_t state = 0; + erts_aint32_t prio = (erts_aint32_t) PRIORITY_NORMAL; #ifdef ERTS_SMP erts_smp_proc_lock(parent, ERTS_PROC_LOCKS_ALL_MINOR); @@ -7318,8 +7520,24 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). so->error_code = BADARG; goto error; } - p = alloc_process(); /* All proc locks are locked by this thread - on success */ + + if (so->flags & SPO_USE_ARGS) { + if (so->scheduler) { + int ix = so->scheduler-1; + ASSERT(0 <= ix && ix < erts_no_run_queues); + rq = ERTS_RUNQ_IX(ix); + state |= ERTS_PSFLG_BOUND; + } + prio = (erts_aint32_t) so->priority; + } + + state |= (prio & ERTS_PSFLG_PRIO_MASK); + + if (!rq) + rq = erts_get_runq_proc(parent); + + p = alloc_process(rq, state); /* All proc locks are locked by this thread + on success */ if (!p) { erts_send_error_to_logger_str(parent->group_leader, "Too many processes\n"); @@ -7339,22 +7557,16 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->flags = erts_default_process_flags; - /* Scheduler queue mutex should be locked when changeing - * prio. In this case we don't have to lock it, since - * noone except us has access to the process. - */ if (so->flags & SPO_USE_ARGS) { p->min_heap_size = so->min_heap_size; p->min_vheap_size = so->min_vheap_size; - p->prio = so->priority; p->max_gen_gcs = so->max_gen_gcs; } else { p->min_heap_size = H_MIN_SIZE; p->min_vheap_size = BIN_VH_MIN_SIZE; - p->prio = PRIORITY_NORMAL; p->max_gen_gcs = (Uint16) erts_smp_atomic32_read_nob(&erts_max_gen_gcs); } - p->skipped = 0; + p->schedule_count = 0; ASSERT(p->min_heap_size == erts_next_heap_size(p->min_heap_size, 0)); p->initial[INITIAL_MOD] = mod; @@ -7456,7 +7668,6 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->msg_inq.first = NULL; p->msg_inq.last = &p->msg_inq.first; p->msg_inq.len = 0; - p->bound_runq = NULL; #endif p->bif_timers = NULL; p->mbuf = NULL; @@ -7542,9 +7753,6 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). #ifdef ERTS_SMP p->scheduler_data = NULL; - p->is_exiting = 0; - p->status_flags = 0; - p->runq_flags = 0; p->suspendee = NIL; p->pending_suspenders = NULL; p->pending_exit.reason = THE_NON_VALUE; @@ -7555,34 +7763,15 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->fp_exception = 0; #endif + erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL); + + res = p->id; + /* * Schedule process for execution. */ - if (!((so->flags & SPO_USE_ARGS) && so->scheduler)) - rq = erts_get_runq_proc(parent); - else { - int ix = so->scheduler-1; - ASSERT(0 <= ix && ix < erts_no_run_queues); - rq = ERTS_RUNQ_IX(ix); - p->bound_runq = rq; - } - - erts_smp_runq_lock(rq); - -#ifdef ERTS_SMP - p->run_queue = rq; -#endif - - p->status = P_WAITING; - notify_runq = internal_add_to_runq(rq, p); - - erts_smp_runq_unlock(rq); - - smp_notify_inc_runq(notify_runq); - - res = p->id; - erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL); + schedule_process(p, state, 0); VERBOSE(DEBUG_PROCESSES, ("Created a new process: %T\n",p->id)); @@ -7618,12 +7807,8 @@ void erts_init_empty_process(Process *p) p->max_gen_gcs = 0; p->min_heap_size = 0; p->min_vheap_size = 0; - p->status = P_RUNABLE; - p->gcstatus = P_RUNABLE; - p->rstatus = P_RUNABLE; p->rcount = 0; p->id = ERTS_INVALID_PID; - p->prio = PRIORITY_NORMAL; p->reds = 0; p->tracer_proc = NIL; p->trace_flags = F_INITIAL_TRACE_FLAGS; @@ -7640,7 +7825,6 @@ void erts_init_empty_process(Process *p) p->bin_vheap_mature = 0; #ifdef ERTS_SMP p->u.ptimer = NULL; - p->bound_runq = NULL; #else memset(&(p->u.tm), 0, sizeof(ErlTimer)); #endif @@ -7691,8 +7875,8 @@ void erts_init_empty_process(Process *p) p->def_arg_reg[5] = 0; p->parent = NIL; - p->started.tv_sec = 0; - p->started.tv_usec = 0; + p->approx_started = 0; + p->started_interval = 0; #ifdef HIPE hipe_init_process(&p->hipe); @@ -7706,12 +7890,10 @@ void erts_init_empty_process(Process *p) p->last_old_htop = NULL; #endif + erts_smp_atomic32_init_nob(&p->state, (erts_aint32_t) PRIORITY_NORMAL); #ifdef ERTS_SMP p->scheduler_data = NULL; - p->is_exiting = 0; - p->status_flags = 0; - p->runq_flags = 0; p->msg_inq.first = NULL; p->msg_inq.last = &p->msg_inq.first; p->msg_inq.len = 0; @@ -7721,7 +7903,7 @@ void erts_init_empty_process(Process *p) p->pending_exit.bp = NULL; erts_proc_lock_init(p); erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL); - p->run_queue = ERTS_RUNQ_IX(0); + RUNQ_SET_RQ(&p->run_queue, ERTS_RUNQ_IX(0)); #endif #if !defined(NO_FPE_SIGNALS) || defined(HIPE) @@ -7799,8 +7981,8 @@ erts_cleanup_empty_process(Process* p) free_message_buffer(p->mbuf); p->mbuf = NULL; } -#if defined(ERTS_ENABLE_LOCK_COUNT) && defined(ERTS_SMP) - erts_lcnt_proc_lock_destroy(p); +#ifdef ERTS_SMP + erts_proc_lock_fin(p); #endif #ifdef DEBUG erts_debug_verify_clean_empty_process(p); @@ -7897,26 +8079,34 @@ delete_process(Process* p) p->fvalue = NIL; } +static ERTS_INLINE erts_aint32_t +set_proc_exiting_state(Process *p, erts_aint32_t state) +{ + erts_aint32_t a, n, e; + a = state; + while (1) { + n = e = a; + n &= ~(ERTS_PSFLG_SUSPENDED|ERTS_PSFLG_PENDING_EXIT); + n |= ERTS_PSFLG_EXITING|ERTS_PSFLG_ACTIVE; + if (!(a & (ERTS_PSFLG_IN_RUNQ|ERTS_PSFLG_RUNNING))) + n |= ERTS_PSFLG_IN_RUNQ; + a = erts_smp_atomic32_cmpxchg_relb(&p->state, n, e); + if (a == e) + break; + } + return a; +} + static ERTS_INLINE void -set_proc_exiting(Process *p, Eterm reason, ErlHeapFragment *bp) +set_proc_exiting(Process *p, + erts_aint32_t state, + Eterm reason, + ErlHeapFragment *bp) { -#ifdef ERTS_SMP - erts_pix_lock_t *pix_lock = ERTS_PID2PIXLOCK(p->id); ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(p) == ERTS_PROC_LOCKS_ALL); - /* - * You are required to have all proc locks and the pix lock when going - * to status P_EXITING. This makes it is enough to take any lock when - * looking up a process (pid2proc()) to prevent the looked up process - * from exiting until the lock has been released. - */ - erts_pix_lock(pix_lock); - p->is_exiting = 1; -#endif - p->status = P_EXITING; -#ifdef ERTS_SMP - erts_pix_unlock(pix_lock); -#endif + state = set_proc_exiting_state(p, state); + p->fvalue = reason; if (bp) erts_link_mbuf_to_proc(p, bp); @@ -7929,6 +8119,14 @@ set_proc_exiting(Process *p, Eterm reason, ErlHeapFragment *bp) KILL_CATCHES(p); cancel_timer(p); p->i = (BeamInstr *) beam_exit; + + if (erts_system_profile_flags.runnable_procs + && !(state & (ERTS_PSFLG_ACTIVE|ERTS_PSFLG_SUSPENDED))) { + profile_runnable_proc(p, am_active); + } + + if (!(state & (ERTS_PSFLG_IN_RUNQ|ERTS_PSFLG_RUNNING))) + add2runq(p, state); } @@ -7941,8 +8139,8 @@ erts_handle_pending_exit(Process *c_p, ErtsProcLocks locks) ASSERT(is_value(c_p->pending_exit.reason)); ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == locks); ERTS_SMP_LC_ASSERT(locks & ERTS_PROC_LOCK_MAIN); - ERTS_SMP_LC_ASSERT(c_p->status != P_EXITING); - ERTS_SMP_LC_ASSERT(c_p->status != P_FREE); + ERTS_SMP_LC_ASSERT(!((ERTS_PSFLG_EXITING|ERTS_PSFLG_FREE) + & erts_smp_atomic32_read_nob(&c_p->state))); /* Ensure that all locks on c_p are locked before proceeding... */ if (locks == ERTS_PROC_LOCKS_ALL) @@ -7955,7 +8153,10 @@ erts_handle_pending_exit(Process *c_p, ErtsProcLocks locks) } } - set_proc_exiting(c_p, c_p->pending_exit.reason, c_p->pending_exit.bp); + set_proc_exiting(c_p, + erts_smp_atomic32_read_acqb(&c_p->state), + c_p->pending_exit.reason, + c_p->pending_exit.bp); c_p->pending_exit.reason = THE_NON_VALUE; c_p->pending_exit.bp = NULL; @@ -7971,11 +8172,12 @@ handle_pending_exiters(ErtsProcList *pnd_xtrs) while (plp) { Process *p = erts_pid2proc(NULL, 0, plp->pid, ERTS_PROC_LOCKS_ALL); if (p) { - if (proclist_same(plp, p) - && !(p->status_flags & ERTS_PROC_SFLG_RUNNING)) { - ASSERT(p->status_flags & ERTS_PROC_SFLG_INRUNQ); - ASSERT(ERTS_PROC_PENDING_EXIT(p)); - erts_handle_pending_exit(p, ERTS_PROC_LOCKS_ALL); + if (proclist_same(plp, p)) { + erts_aint32_t state = erts_smp_atomic32_read_acqb(&p->state); + if (!(state & ERTS_PSFLG_RUNNING)) { + ASSERT(state & ERTS_PSFLG_PENDING_EXIT); + erts_handle_pending_exit(p, ERTS_PROC_LOCKS_ALL); + } } erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL); } @@ -8003,7 +8205,7 @@ save_pending_exiter(Process *p) rq->procs.pending_exiters = plp; erts_smp_runq_unlock(rq); - + wake_scheduler(rq, 1); } #endif @@ -8065,7 +8267,7 @@ send_exit_message(Process *to, ErtsProcLocks *to_locksp, * SMP emulator). When the signal is received the receiver receives an * 'EXIT' message if it is trapping exits; otherwise, it will either * ignore the signal if the exit reason is normal, or go into an - * exiting state (status P_EXITING). When a process has gone into the + * exiting state (ERTS_PSFLG_EXITING). When a process has gone into the * exiting state it will not execute any more Erlang code, but it might * take a while before it actually exits. The exit signal is being * received when the 'EXIT' message is put in the message queue, the @@ -8138,6 +8340,7 @@ send_exit_signal(Process *c_p, /* current process if and only Uint32 flags /* flags */ ) { + erts_aint32_t state = erts_smp_atomic32_read_nob(&rp->state); Eterm rsn = reason == am_kill ? am_killed : reason; ERTS_SMP_LC_ASSERT(*rp_locks == erts_proc_lc_my_proc_locks(rp)); @@ -8159,7 +8362,7 @@ send_exit_signal(Process *c_p, /* current process if and only } #endif - if (ERTS_PROC_IS_TRAPPING_EXITS(rp) + if ((state & ERTS_PSFLG_TRAP_EXIT) && (reason != am_kill || (flags & ERTS_XSIG_FLG_IGN_KILL))) { if (is_not_nil(token) #ifdef USE_VM_PROBES @@ -8175,9 +8378,7 @@ send_exit_signal(Process *c_p, /* current process if and only } else if (reason != am_normal || (flags & ERTS_XSIG_FLG_NO_IGN_NORMAL)) { #ifdef ERTS_SMP - if (!ERTS_PROC_PENDING_EXIT(rp) && !rp->is_exiting) { - ASSERT(rp->status != P_EXITING); - ASSERT(rp->status != P_FREE); + if (!(state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT))) { ASSERT(!rp->pending_exit.bp); if (rp == c_p && (*rp_locks & ERTS_PROC_LOCK_MAIN)) { @@ -8193,9 +8394,9 @@ send_exit_signal(Process *c_p, /* current process if and only } *rp_locks = ERTS_PROC_LOCKS_ALL; } - set_proc_exiting(c_p, rsn, NULL); + set_proc_exiting(c_p, state, rsn, NULL); } - else if (!(rp->status_flags & ERTS_PROC_SFLG_RUNNING)) { + else if (!(state & ERTS_PSFLG_RUNNING)) { /* Process not running ... */ ErtsProcLocks need_locks = ~(*rp_locks) & ERTS_PROC_LOCKS_ALL; if (need_locks @@ -8212,6 +8413,7 @@ send_exit_signal(Process *c_p, /* current process if and only /* ...and we have all locks on it... */ *rp_locks = ERTS_PROC_LOCKS_ALL; set_proc_exiting(rp, + state, (is_immed(rsn) ? rsn : copy_object(rsn, rp)), @@ -8241,11 +8443,9 @@ send_exit_signal(Process *c_p, /* current process if and only &bp->off_heap); rp->pending_exit.bp = bp; } - ASSERT(ERTS_PROC_PENDING_EXIT(rp)); + erts_smp_atomic32_read_bor_relb(&rp->state, + ERTS_PSFLG_PENDING_EXIT); } - if (!(rp->status_flags - & (ERTS_PROC_SFLG_INRUNQ|ERTS_PROC_SFLG_RUNNING))) - erts_add_to_runq(rp); } /* else: * @@ -8257,17 +8457,14 @@ send_exit_signal(Process *c_p, /* current process if and only * exit or by itself before seeing the pending exit. */ #else /* !ERTS_SMP */ - if (c_p == rp) { - rp->status = P_EXITING; - c_p->fvalue = rsn; - } - else if (rp->status != P_EXITING) { /* No recursive process exits /PaN */ - Eterm old_status = rp->status; + erts_aint32_t state = erts_smp_atomic32_read_nob(&rp->state); + if (!(state & ERTS_PSFLG_EXITING)) { set_proc_exiting(rp, - is_immed(rsn) ? rsn : copy_object(rsn, rp), + state, + (is_immed(rsn) || c_p == rp + ? rsn + : copy_object(rsn, rp)), NULL); - if (old_status != P_RUNABLE && old_status != P_RUNNING) - erts_add_to_runq(rp); } #endif return -1; /* Receiver will exit */ @@ -8570,12 +8767,14 @@ resume_suspend_monitor(ErtsSuspendMonitor *smon, void *vc_p) erts_destroy_suspend_monitor(smon); } -static void -continue_exit_process(Process *p #ifdef ERTS_SMP - , erts_pix_lock_t *pix_lock +static void +proc_dec_refc(void *vproc) +{ + erts_smp_proc_dec_refc((Process *) vproc); +} #endif - ); + /* this function fishishes a process and propagates exit messages - called by process_main when a process dies */ @@ -8583,9 +8782,8 @@ void erts_do_exit_process(Process* p, Eterm reason) { #ifdef ERTS_SMP - erts_pix_lock_t *pix_lock = ERTS_PID2PIXLOCK(p->id); + erts_aint32_t state; #endif - p->arity = 0; /* No live registers */ p->fvalue = reason; @@ -8603,27 +8801,17 @@ erts_do_exit_process(Process* p, Eterm reason) #ifdef ERTS_SMP ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); /* By locking all locks (main lock is already locked) when going - to status P_EXITING, it is enough to take any lock when + to exiting state (ERTS_PSFLG_EXITING), it is enough to take any lock when looking up a process (erts_pid2proc()) to prevent the looked up process from exiting until the lock has been released. */ erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR); #endif - - if (erts_system_profile_flags.runnable_procs && (p->status != P_WAITING)) { - profile_runnable_proc(p, am_inactive); - } - -#ifdef ERTS_SMP - erts_pix_lock(pix_lock); - p->is_exiting = 1; -#endif - - p->status = P_EXITING; -#ifdef ERTS_SMP - erts_pix_unlock(pix_lock); - - if (ERTS_PROC_PENDING_EXIT(p)) { +#ifndef ERTS_SMP + set_proc_exiting_state(p, erts_smp_atomic32_read_nob(&p->state)); +#else + state = set_proc_exiting_state(p, erts_smp_atomic32_read_nob(&p->state)); + if (state & ERTS_PSFLG_PENDING_EXIT) { /* Process exited before pending exit was received... */ p->pending_exit.reason = THE_NON_VALUE; if (p->pending_exit.bp) { @@ -8662,29 +8850,11 @@ erts_do_exit_process(Process* p, Eterm reason) erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL_MINOR); -#ifdef ERTS_SMP - continue_exit_process(p, pix_lock); -#else - continue_exit_process(p); -#endif + erts_continue_exit_process(p); } void -erts_continue_exit_process(Process *c_p) -{ -#ifdef ERTS_SMP - continue_exit_process(c_p, ERTS_PID2PIXLOCK(c_p->id)); -#else - continue_exit_process(c_p); -#endif -} - -static void -continue_exit_process(Process *p -#ifdef ERTS_SMP - , erts_pix_lock_t *pix_lock -#endif - ) +erts_continue_exit_process(Process *p) { ErtsLink* lnk; ErtsMonitor *mon; @@ -8700,11 +8870,7 @@ continue_exit_process(Process *p ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(p)); -#ifdef DEBUG - erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); - ASSERT(p->status == P_EXITING); - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); -#endif + ASSERT(ERTS_PROC_IS_EXITING(p)); #ifdef ERTS_SMP if (p->flags & F_HAVE_BLCKD_MSCHED) { @@ -8772,48 +8938,47 @@ continue_exit_process(Process *p #endif { + int maybe_save; int pix; /* Do *not* use erts_get_runq_proc() */ ErtsRunQueue *rq; rq = erts_get_runq_current(ERTS_GET_SCHEDULER_DATA_FROM_PROC(p)); - ASSERT(internal_pid_index(p->id) < erts_max_processes); pix = internal_pid_index(p->id); - erts_smp_mtx_lock(&proc_tab_mtx); + erts_smp_rwmtx_rlock(&erts_proc_tab_rwmtx); + maybe_save = saved_term_procs.end != NULL; + if (maybe_save) { + erts_smp_rwmtx_runlock(&erts_proc_tab_rwmtx); + erts_smp_rwmtx_rwlock(&erts_proc_tab_rwmtx); + } + erts_smp_runq_lock(rq); #ifdef ERTS_SMP - erts_pix_lock(pix_lock); - ASSERT(p->scheduler_data); ASSERT(p->scheduler_data->current_process == p); ASSERT(p->scheduler_data->free_process == NULL); p->scheduler_data->current_process = NULL; p->scheduler_data->free_process = p; - p->status_flags = 0; #endif - process_tab[pix] = NULL; /* Time of death! */ + /* Time of death! */ + erts_smp_atomic_set_relb(&erts_proc.tab[pix], ERTS_AINT_NULL); + ASSERT(erts_smp_atomic32_read_nob(&process_count) > 0); - erts_smp_atomic32_dec_nob(&process_count); + erts_smp_atomic32_dec_relb(&process_count); -#ifdef ERTS_SMP - erts_pix_unlock(pix_lock); -#endif erts_smp_runq_unlock(rq); - if (p_next < 0) { - if (p_last >= p_next) { - p_serial++; - p_serial &= p_serial_mask; - } - p_next = pix; + if (!maybe_save) + erts_smp_rwmtx_runlock(&erts_proc_tab_rwmtx); + else { + if (saved_term_procs.end) + save_terminating_process(p); + erts_smp_rwmtx_rwunlock(&erts_proc_tab_rwmtx); } - ERTS_MAYBE_SAVE_TERMINATING_PROCESS(p); - - erts_smp_mtx_unlock(&proc_tab_mtx); } /* @@ -8828,7 +8993,21 @@ continue_exit_process(Process *p lnk = p->nlinks; p->nlinks = NULL; - p->status = P_FREE; + + { + /* Inactivate and notify free */ + erts_aint32_t n, e, a = erts_smp_atomic32_read_nob(&p->state); + while (1) { + n = e = a; + ASSERT(a & ERTS_PSFLG_EXITING); + n |= ERTS_PSFLG_FREE; + n &= ~ERTS_PSFLG_ACTIVE; + a = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e); + if (a == e) + break; + } + } + dep = ((p->flags & F_DISTRIBUTION) ? ERTS_PROC_SET_DIST_ENTRY(p, ERTS_PROC_LOCKS_ALL, NULL) : NULL); @@ -8883,8 +9062,25 @@ continue_exit_process(Process *p delete_process(p); +#ifdef ERTS_SMP + /* + * Each scheduler will decrease refc by one via misc aux work; + * we have one refc for reference from process table which we + * now want to remove, i.e. we increase refc with schedulers-1. + * + * Process struct wont be deallocated until (earliest) when + * all schedulers have decreased refc via misc aux work... + */ + if (erts_no_schedulers != 1) + erts_smp_proc_add_refc(p, (Sint32) erts_no_schedulers-1); + erts_schedule_multi_misc_aux_work(0, + erts_no_schedulers, + proc_dec_refc, + (void *) p); + erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); +#endif return; @@ -8897,8 +9093,6 @@ continue_exit_process(Process *p ERTS_SMP_LC_ASSERT(curr_locks == erts_proc_lc_my_proc_locks(p)); ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & curr_locks); - ASSERT(p->status == P_EXITING); - p->i = (BeamInstr *) beam_continue_exit; if (!(curr_locks & ERTS_PROC_LOCK_STATUS)) { @@ -8906,8 +9100,6 @@ continue_exit_process(Process *p curr_locks |= ERTS_PROC_LOCK_STATUS; } - erts_add_to_runq(p); - if (curr_locks != ERTS_PROC_LOCK_MAIN) erts_smp_proc_unlock(p, ~ERTS_PROC_LOCK_MAIN & curr_locks); @@ -8919,33 +9111,15 @@ continue_exit_process(Process *p static void timeout_proc(Process* p) { + erts_aint32_t state; BeamInstr** pi = (BeamInstr **) p->def_arg_reg; p->i = *pi; p->flags |= F_TIMO; p->flags &= ~F_INSLPQUEUE; - switch (p->status) { - case P_GARBING: - switch (p->gcstatus) { - case P_SUSPENDED: - goto suspended; - case P_WAITING: - goto waiting; - default: - break; - } - break; - case P_WAITING: - waiting: - erts_add_to_runq(p); - break; - case P_SUSPENDED: - suspended: - p->rstatus = P_RUNABLE; /* MUST set resume status to runnable */ - break; - default: - break; - } + state = erts_smp_atomic32_read_acqb(&p->state); + if (!(state & ERTS_PSFLG_ACTIVE)) + schedule_process(p, state, 0); } @@ -9013,6 +9187,7 @@ erts_stack_dump(int to, void *to_arg, Process *p) void erts_program_counter_info(int to, void *to_arg, Process *p) { + erts_aint32_t state; int i; erts_print(to, to_arg, "Program counter: %p (", p->i); @@ -9021,7 +9196,8 @@ erts_program_counter_info(int to, void *to_arg, Process *p) erts_print(to, to_arg, "CP: %p (", p->cp); print_function_from_pc(to, to_arg, p->cp); erts_print(to, to_arg, ")\n"); - if (!((p->status == P_RUNNING) || (p->status == P_GARBING))) { + state = erts_smp_atomic32_read_acqb(&p->state); + if (!(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_GC))) { erts_print(to, to_arg, "arity = %d\n",p->arity); if (!ERTS_IS_CRASH_DUMPING) { /* @@ -9194,13 +9370,13 @@ do { \ #endif #if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS -# define ERTS_PROCS_DBG_CHK_PID_FOUND(PBDP, PID, TVP) \ - debug_processes_check_found_pid((PBDP), (PID), (TVP), 1) -# define ERTS_PROCS_DBG_CHK_PID_NOT_FOUND(PBDP, PID, TVP) \ - debug_processes_check_found_pid((PBDP), (PID), (TVP), 0) +# define ERTS_PROCS_DBG_CHK_PID_FOUND(PBDP, PID, IC) \ + debug_processes_check_found_pid((PBDP), (PID), (IC), 1) +# define ERTS_PROCS_DBG_CHK_PID_NOT_FOUND(PBDP, PID, IC) \ + debug_processes_check_found_pid((PBDP), (PID), (IC), 0) #else -# define ERTS_PROCS_DBG_CHK_PID_FOUND(PBDP, PID, TVP) -# define ERTS_PROCS_DBG_CHK_PID_NOT_FOUND(PBDP, PID, TVP) +# define ERTS_PROCS_DBG_CHK_PID_FOUND(PBDP, PID, IC) +# define ERTS_PROCS_DBG_CHK_PID_NOT_FOUND(PBDP, PID, IC) #endif #if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_TERM_PROC_LIST @@ -9245,7 +9421,7 @@ static Uint processes_bif_tab_chunks; static Export processes_trap_export; typedef struct { - SysTimeval time; + Uint64 interval; } ErtsProcessesBifChunkInfo; typedef enum { @@ -9270,7 +9446,7 @@ typedef struct { struct { Eterm caller; #if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS - SysTimeval *pid_started; + Uint64 *pid_started; #endif #if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_HALLOC Eterm *heap; @@ -9299,11 +9475,10 @@ static void debug_processes_verify_all_pids(ErtsProcessesBifData *pbdp); #if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS static void debug_processes_check_found_pid(ErtsProcessesBifData *pbdp, Eterm pid, - SysTimeval *started, + Uint64 ic, int pid_should_be_found); #endif #if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_TERM_PROC_LIST -static SysTimeval debug_tv_start; static void debug_processes_check_term_proc_list(void); static void debug_processes_check_term_proc_free_list(ErtsTermProcElement *tpep); #endif @@ -9314,7 +9489,7 @@ save_terminating_process(Process *p) ErtsTermProcElement *tpep = erts_alloc(ERTS_ALC_T_PROCS_TPROC_EL, sizeof(ErtsTermProcElement)); ERTS_PROCS_ASSERT(saved_term_procs.start && saved_term_procs.end); - ERTS_SMP_LC_ASSERT(erts_lc_mtx_is_locked(&proc_tab_mtx)); + ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&erts_proc_tab_rwmtx)); ERTS_PROCS_DBG_CHK_TPLIST(); @@ -9322,19 +9497,19 @@ save_terminating_process(Process *p) tpep->next = NULL; tpep->ix = internal_pid_index(p->id); tpep->u.process.pid = p->id; - tpep->u.process.spawned = p->started; - erts_get_emu_time(&tpep->u.process.exited); + tpep->u.process.spawned = p->started_interval; + tpep->u.process.exited = get_proc_interval(); saved_term_procs.end->next = tpep; saved_term_procs.end = tpep; ERTS_PROCS_DBG_CHK_TPLIST(); - ERTS_PROCS_ASSERT((tpep->prev->ix >= 0 - ? erts_cmp_timeval(&tpep->u.process.exited, - &tpep->prev->u.process.exited) - : erts_cmp_timeval(&tpep->u.process.exited, - &tpep->prev->u.bif_invocation.time)) > 0); + ERTS_PROCS_ASSERT(tpep->prev->ix >= 0 + ? (tpep->u.process.exited + >= tpep->prev->u.process.exited) + : (tpep->u.process.exited + >= tpep->prev->u.bif_invocation.interval)); } static void @@ -9365,7 +9540,7 @@ cleanup_processes_bif_data(Binary *bp) if (pbdp->bif_invocation) { ErtsTermProcElement *tpep; - erts_smp_mtx_lock(&proc_tab_mtx); + erts_smp_rwmtx_rwlock(&erts_proc_tab_rwmtx); ERTS_PROCS_DBG_TRACE(pbdp->debug.caller, cleanup_processes_bif_data, @@ -9419,7 +9594,7 @@ cleanup_processes_bif_data(Binary *bp) ERTS_PROCS_DBG_CHK_TPLIST(); - erts_smp_mtx_unlock(&proc_tab_mtx); + erts_smp_rwmtx_rwunlock(&erts_proc_tab_rwmtx); } } @@ -9447,7 +9622,7 @@ processes_bif_engine(Process *p, Eterm *res_accp, Binary *mbp) pbdp->tix = 0; pbdp->pid_ix = 0; - erts_smp_mtx_lock(&proc_tab_mtx); + erts_smp_rwmtx_rwlock(&erts_proc_tab_rwmtx); locked = 1; ERTS_PROCS_DBG_TRACE(p->id, processes_bif_engine, init); @@ -9458,7 +9633,7 @@ processes_bif_engine(Process *p, Eterm *res_accp, Binary *mbp) #if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS pbdp->debug.pid_started = erts_alloc(ERTS_ALC_T_PROCS_PIDS, - sizeof(SysTimeval)*pbdp->pid_sz); + sizeof(Uint64)*pbdp->pid_sz); #endif ERTS_PROCS_DBG_SAVE_PIDS(pbdp); @@ -9473,7 +9648,8 @@ processes_bif_engine(Process *p, Eterm *res_accp, Binary *mbp) pbdp->bif_invocation = erts_alloc(ERTS_ALC_T_PROCS_TPROC_EL, sizeof(ErtsTermProcElement)); pbdp->bif_invocation->ix = -1; - erts_get_emu_time(&pbdp->bif_invocation->u.bif_invocation.time); + pbdp->bif_invocation->u.bif_invocation.interval + = step_proc_interval(); ERTS_PROCS_DBG_CHK_TPLIST(); pbdp->bif_invocation->next = NULL; @@ -9500,30 +9676,31 @@ processes_bif_engine(Process *p, Eterm *res_accp, Binary *mbp) int indices = ERTS_PROCESSES_BIF_TAB_CHUNK_SIZE; int cix = ix / ERTS_PROCESSES_BIF_TAB_CHUNK_SIZE; int end_ix = ix + indices; - SysTimeval *invocation_timep; + Uint64 *invocation_interval_p; - invocation_timep = (pbdp->bif_invocation - ? &pbdp->bif_invocation->u.bif_invocation.time - : NULL); + invocation_interval_p + = (pbdp->bif_invocation + ? &pbdp->bif_invocation->u.bif_invocation.interval + : NULL); ERTS_PROCS_ASSERT(is_nil(*res_accp)); if (!locked) { - erts_smp_mtx_lock(&proc_tab_mtx); + erts_smp_rwmtx_rwlock(&erts_proc_tab_rwmtx); locked = 1; } - ERTS_SMP_LC_ASSERT(erts_lc_mtx_is_locked(&proc_tab_mtx)); + ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&erts_proc_tab_rwmtx)); ERTS_PROCS_DBG_TRACE(p->id, processes_bif_engine, insp_table); if (cix != 0) - erts_get_emu_time(&pbdp->chunk[cix].time); + pbdp->chunk[cix].interval = step_proc_interval(); else if (pbdp->bif_invocation) - pbdp->chunk[0].time = *invocation_timep; - /* else: Time is irrelevant */ + pbdp->chunk[0].interval = *invocation_interval_p; + /* else: interval is irrelevant */ - if (end_ix >= erts_max_processes) { + if (end_ix >= erts_proc.max) { ERTS_PROCS_ASSERT(cix+1 == processes_bif_tab_chunks); - end_ix = erts_max_processes; + end_ix = erts_proc.max; indices = end_ix - ix; /* What to do when done with this chunk */ pbdp->state = (processes_bif_tab_chunks == 1 @@ -9532,16 +9709,15 @@ processes_bif_engine(Process *p, Eterm *res_accp, Binary *mbp) } for (; ix < end_ix; ix++) { - Process *rp = process_tab[ix]; + Process *rp = erts_pix2proc(ix); if (rp - && (!invocation_timep - || erts_cmp_timeval(&rp->started, - invocation_timep) < 0)) { + && (!invocation_interval_p + || rp->started_interval < *invocation_interval_p)) { ERTS_PROCS_ASSERT(is_internal_pid(rp->id)); pbdp->pid[pbdp->pid_ix] = rp->id; #if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS - pbdp->debug.pid_started[pbdp->pid_ix] = rp->started; + pbdp->debug.pid_started[pbdp->pid_ix] = rp->started_interval; #endif pbdp->pid_ix++; @@ -9551,7 +9727,7 @@ processes_bif_engine(Process *p, Eterm *res_accp, Binary *mbp) pbdp->tix = end_ix; - erts_smp_mtx_unlock(&proc_tab_mtx); + erts_smp_rwmtx_rwunlock(&erts_proc_tab_rwmtx); locked = 0; reds = indices/ERTS_PROCESSES_BIF_TAB_INSPECT_INDICES_PER_RED; @@ -9563,8 +9739,8 @@ processes_bif_engine(Process *p, Eterm *res_accp, Binary *mbp) ix = pbdp->tix; indices = ERTS_PROCESSES_BIF_TAB_CHUNK_SIZE; end_ix = ix + indices; - if (end_ix > erts_max_processes) { - end_ix = erts_max_processes; + if (end_ix > erts_proc.max) { + end_ix = erts_proc.max; indices = end_ix - ix; } @@ -9583,20 +9759,20 @@ processes_bif_engine(Process *p, Eterm *res_accp, Binary *mbp) int i; int max_reds; int free_term_procs = 0; - SysTimeval *invocation_timep; + Uint64 invocation_interval; ErtsTermProcElement *tpep; ErtsTermProcElement *free_list = NULL; tpep = pbdp->bif_invocation; ERTS_PROCS_ASSERT(tpep); - invocation_timep = &tpep->u.bif_invocation.time; + invocation_interval = tpep->u.bif_invocation.interval; max_reds = have_reds = ERTS_BIF_REDS_LEFT(p); if (max_reds > ERTS_PROCESSES_INSPECT_TERM_PROC_MAX_REDS) max_reds = ERTS_PROCESSES_INSPECT_TERM_PROC_MAX_REDS; reds = 0; - erts_smp_mtx_lock(&proc_tab_mtx); + erts_smp_rwmtx_rwlock(&erts_proc_tab_rwmtx); ERTS_PROCS_DBG_TRACE(p->id, processes_bif_engine, insp_term_procs); ERTS_PROCS_DBG_CHK_TPLIST(); @@ -9635,20 +9811,19 @@ processes_bif_engine(Process *p, Eterm *res_accp, Binary *mbp) } else { int cix = tpep->ix/ERTS_PROCESSES_BIF_TAB_CHUNK_SIZE; - SysTimeval *chunk_timep = &pbdp->chunk[cix].time; + Uint64 chunk_interval = pbdp->chunk[cix].interval; Eterm pid = tpep->u.process.pid; ERTS_PROCS_ASSERT(is_internal_pid(pid)); - if (erts_cmp_timeval(&tpep->u.process.spawned, - invocation_timep) < 0) { - if (erts_cmp_timeval(&tpep->u.process.exited, - chunk_timep) < 0) { + if (tpep->u.process.spawned < invocation_interval) { + if (tpep->u.process.exited < chunk_interval) { ERTS_PROCS_DBG_CHK_PID_NOT_FOUND(pbdp, pid, - &tpep->u.process.spawned); + tpep->u.process.spawned); pbdp->pid[pbdp->pid_ix] = pid; #if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS - pbdp->debug.pid_started[pbdp->pid_ix] = tpep->u.process.spawned; + pbdp->debug.pid_started[pbdp->pid_ix] + = tpep->u.process.spawned; #endif pbdp->pid_ix++; ERTS_PROCS_ASSERT(pbdp->pid_ix <= pbdp->pid_sz); @@ -9656,13 +9831,13 @@ processes_bif_engine(Process *p, Eterm *res_accp, Binary *mbp) else { ERTS_PROCS_DBG_CHK_PID_FOUND(pbdp, pid, - &tpep->u.process.spawned); + tpep->u.process.spawned); } } else { ERTS_PROCS_DBG_CHK_PID_NOT_FOUND(pbdp, pid, - &tpep->u.process.spawned); + tpep->u.process.spawned); } i++; @@ -9711,7 +9886,7 @@ processes_bif_engine(Process *p, Eterm *res_accp, Binary *mbp) ERTS_PROCS_DBG_CHK_TPLIST(); ERTS_PROCS_DBG_CHK_FREELIST(free_list); - erts_smp_mtx_unlock(&proc_tab_mtx); + erts_smp_rwmtx_rwunlock(&erts_proc_tab_rwmtx); /* * We do the actual free of term proc structures now when we @@ -9771,8 +9946,9 @@ processes_bif_engine(Process *p, Eterm *res_accp, Binary *mbp) sizeof(Eterm)*pbdp->pid_sz); #if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS pbdp->debug.pid_started = erts_realloc(ERTS_ALC_T_PROCS_PIDS, - pbdp->debug.pid_started, - sizeof(SysTimeval)*pbdp->pid_sz); + pbdp->debug.pid_started, + (sizeof(Uint64) + * pbdp->pid_sz)); #endif } reds = conses/ERTS_PROCESSES_BIF_BUILD_RESULT_CONSES_PER_RED; @@ -9892,22 +10068,13 @@ init_processes_bif(void) { saved_term_procs.start = NULL; saved_term_procs.end = NULL; - processes_bif_tab_chunks = (((erts_max_processes - 1) + processes_bif_tab_chunks = (((erts_proc.max - 1) / ERTS_PROCESSES_BIF_TAB_CHUNK_SIZE) + 1); /* processes_trap/2 is a hidden BIF that the processes/0 BIF traps to. */ - sys_memset((void *) &processes_trap_export, 0, sizeof(Export)); - processes_trap_export.address = &processes_trap_export.code[3]; - processes_trap_export.code[0] = am_erlang; - processes_trap_export.code[1] = am_processes_trap; - processes_trap_export.code[2] = 2; - processes_trap_export.code[3] = (BeamInstr) em_apply_bif; - processes_trap_export.code[4] = (BeamInstr) &processes_trap; - -#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_TERM_PROC_LIST - erts_get_emu_time(&debug_tv_start); -#endif + erts_init_trap_export(&processes_trap_export, am_erlang, am_processes_trap, 2, + &processes_trap); } @@ -9940,31 +10107,29 @@ erts_debug_processes(Process *c_p) Eterm res; Eterm* hp; Process *p; -#ifdef DEBUG Eterm *hp_end; -#endif - erts_smp_mtx_lock(&proc_tab_mtx); + erts_smp_rwmtx_rwlock(&erts_proc_tab_rwmtx); res = NIL; need = erts_process_count() * 2; hp = HAlloc(c_p, need); /* we need two heap words for each pid */ -#ifdef DEBUG hp_end = hp + need; -#endif /* make the list by scanning bakward */ - for (i = erts_max_processes-1; i >= 0; i--) { - if ((p = process_tab[i]) != NULL) { - res = CONS(hp, process_tab[i]->id, res); + for (i = erts_proc.max-1; i >= 0; i--) { + p = erts_pix2proc(i); + if (p) { + res = CONS(hp, p->id, res); hp += 2; } } - ASSERT(hp == hp_end); - erts_smp_mtx_unlock(&proc_tab_mtx); + erts_smp_rwmtx_rwunlock(&erts_proc_tab_rwmtx); + + HRelease(c_p, hp_end, hp); return res; } @@ -9996,14 +10161,12 @@ erts_debug_processes_bif_info(Process *c_p) static void debug_processes_check_found_pid(ErtsProcessesBifData *pbdp, Eterm pid, - SysTimeval *tvp, + Uint64 ic, int pid_should_be_found) { int i; for (i = 0; i < pbdp->pid_ix; i++) { - if (pbdp->pid[i] == pid - && pbdp->debug.pid_started[i].tv_sec == tvp->tv_sec - && pbdp->debug.pid_started[i].tv_usec == tvp->tv_usec) { + if (pbdp->pid[i] == pid && pbdp->debug.pid_started[i] == ic) { ERTS_PROCS_ASSERT(pid_should_be_found); return; } @@ -10037,8 +10200,8 @@ debug_processes_save_all_pids(ErtsProcessesBifData *pbdp) pbdp->debug.correct_pids = erts_alloc(ERTS_ALC_T_PROCS_PIDS, sizeof(Eterm)*pbdp->pid_sz); - for (tix = 0, cpix = 0; tix < erts_max_processes; tix++) { - Process *rp = process_tab[tix]; + for (tix = 0, cpix = 0; tix < erts_proc.max; tix++) { + Process *rp = erts_pix2proc(tix); if (rp) { ERTS_PROCS_ASSERT(is_internal_pid(rp->id)); pbdp->debug.correct_pids[cpix++] = rp->id; @@ -10091,14 +10254,13 @@ debug_processes_verify_all_pids(ErtsProcessesBifData *pbdp) static void debug_processes_check_term_proc_list(void) { - ERTS_SMP_LC_ASSERT(erts_lc_mtx_is_locked(&proc_tab_mtx)); + ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&erts_proc_tab_rwmtx)); if (!saved_term_procs.start) ERTS_PROCS_ASSERT(!saved_term_procs.end); else { - SysTimeval tv_now; - SysTimeval *prev_xtvp = NULL; + Uint64 curr_interval = get_proc_interval(); + Uint64 *prev_x_interval_p = NULL; ErtsTermProcElement *tpep; - erts_get_emu_time(&tv_now); for (tpep = saved_term_procs.start; tpep; tpep = tpep->next) { if (!tpep->prev) @@ -10110,20 +10272,17 @@ debug_processes_check_term_proc_list(void) else ERTS_PROCS_ASSERT(tpep->next->prev == tpep); if (tpep->ix < 0) { - SysTimeval *tvp = &tpep->u.bif_invocation.time; - ERTS_PROCS_ASSERT(erts_cmp_timeval(&debug_tv_start, tvp) < 0 - && erts_cmp_timeval(tvp, &tv_now) < 0); + Uint64 interval = tpep->u.bif_invocation.interval; + ERTS_PROCS_ASSERT(interval <= curr_interval); } else { - SysTimeval *stvp = &tpep->u.process.spawned; - SysTimeval *xtvp = &tpep->u.process.exited; + Uint64 s_interval = tpep->u.process.spawned; + Uint64 x_interval = tpep->u.process.exited; - ERTS_PROCS_ASSERT(erts_cmp_timeval(&debug_tv_start, - stvp) < 0); - ERTS_PROCS_ASSERT(erts_cmp_timeval(stvp, xtvp) < 0); - if (prev_xtvp) - ERTS_PROCS_ASSERT(erts_cmp_timeval(prev_xtvp, xtvp) < 0); - prev_xtvp = xtvp; + ERTS_PROCS_ASSERT(s_interval <= x_interval); + if (prev_x_interval_p) + ERTS_PROCS_ASSERT(*prev_x_interval_p <= x_interval); + prev_x_interval_p = &tpep->u.process.exited; ERTS_PROCS_ASSERT(is_internal_pid(tpep->u.process.pid)); ERTS_PROCS_ASSERT(tpep->ix == internal_pid_index(tpep->u.process.pid)); diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 5b79c40d93..e9e5a3365f 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -112,28 +112,29 @@ extern int erts_sched_thread_suggested_stack_size; #define PRIORITY_NORMAL 2 #define PRIORITY_LOW 3 #define ERTS_NO_PROC_PRIO_LEVELS 4 +#define ERTS_NO_PROC_PRIO_QUEUES 3 #define ERTS_PORT_PRIO_LEVEL ERTS_NO_PROC_PRIO_LEVELS +#define ERTS_NO_PRIO_LEVELS (ERTS_NO_PROC_PRIO_LEVELS + 1) #define ERTS_RUNQ_FLGS_PROCS_QMASK \ ((((Uint32) 1) << ERTS_NO_PROC_PRIO_LEVELS) - 1) -#define ERTS_NO_PRIO_LEVELS (ERTS_NO_PROC_PRIO_LEVELS + 1) -#define ERTS_RUNQ_FLGS_MIGRATE_QMASK \ +#define ERTS_RUNQ_FLGS_QMASK \ ((((Uint32) 1) << ERTS_NO_PRIO_LEVELS) - 1) #define ERTS_RUNQ_FLGS_EMIGRATE_SHFT \ - ERTS_NO_PROC_PRIO_LEVELS + ERTS_NO_PRIO_LEVELS #define ERTS_RUNQ_FLGS_IMMIGRATE_SHFT \ (ERTS_RUNQ_FLGS_EMIGRATE_SHFT + ERTS_NO_PRIO_LEVELS) #define ERTS_RUNQ_FLGS_EVACUATE_SHFT \ (ERTS_RUNQ_FLGS_IMMIGRATE_SHFT + ERTS_NO_PRIO_LEVELS) #define ERTS_RUNQ_FLGS_EMIGRATE_QMASK \ - (ERTS_RUNQ_FLGS_MIGRATE_QMASK << ERTS_RUNQ_FLGS_EMIGRATE_SHFT) + (ERTS_RUNQ_FLGS_QMASK << ERTS_RUNQ_FLGS_EMIGRATE_SHFT) #define ERTS_RUNQ_FLGS_IMMIGRATE_QMASK \ - (ERTS_RUNQ_FLGS_MIGRATE_QMASK << ERTS_RUNQ_FLGS_IMMIGRATE_SHFT) + (ERTS_RUNQ_FLGS_QMASK << ERTS_RUNQ_FLGS_IMMIGRATE_SHFT) #define ERTS_RUNQ_FLGS_EVACUATE_QMASK \ - (ERTS_RUNQ_FLGS_MIGRATE_QMASK << ERTS_RUNQ_FLGS_EVACUATE_SHFT) + (ERTS_RUNQ_FLGS_QMASK << ERTS_RUNQ_FLGS_EVACUATE_SHFT) #define ERTS_RUNQ_FLG_BASE2 \ (ERTS_RUNQ_FLGS_EVACUATE_SHFT + ERTS_NO_PRIO_LEVELS) @@ -148,14 +149,18 @@ extern int erts_sched_thread_suggested_stack_size; (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 3)) #define ERTS_RUNQ_FLG_INACTIVE \ (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 4)) +#define ERTS_RUNQ_FLG_NONEMPTY \ + (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 5)) +#define ERTS_RUNQ_FLG_PROTECTED \ + (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 6)) #define ERTS_RUNQ_FLGS_MIGRATION_QMASKS \ (ERTS_RUNQ_FLGS_EMIGRATE_QMASK \ | ERTS_RUNQ_FLGS_IMMIGRATE_QMASK \ | ERTS_RUNQ_FLGS_EVACUATE_QMASK) + #define ERTS_RUNQ_FLGS_MIGRATION_INFO \ - (ERTS_RUNQ_FLGS_MIGRATION_QMASKS \ - | ERTS_RUNQ_FLG_INACTIVE \ + (ERTS_RUNQ_FLG_INACTIVE \ | ERTS_RUNQ_FLG_OUT_OF_WORK \ | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK) @@ -186,33 +191,45 @@ extern int erts_sched_thread_suggested_stack_size; #define ERTS_UNSET_RUNQ_FLG_EVACUATE(FLGS, PRIO) \ ((FLGS) &= ~ERTS_RUNQ_FLG_EVACUATE((PRIO))) -#define ERTS_RUNQ_IFLG_SUSPENDED (((erts_aint32_t) 1) << 0) -#define ERTS_RUNQ_IFLG_NONEMPTY (((erts_aint32_t) 1) << 1) - - -#ifdef DEBUG -# if defined(ARCH_64) && !HALFWORD_HEAP -# define ERTS_DBG_SET_INVALID_RUNQP(RQP, N) \ - (*((char **) &(RQP)) = (char *) (0xdeadbeefdead0003 | ((N) << 4))) -# define ERTS_DBG_VERIFY_VALID_RUNQP(RQP) \ -do { \ - ASSERT((RQP) != NULL); \ - ASSERT(((((Uint) (RQP)) & ((Uint) 0x3))) == ((Uint) 0)); \ - ASSERT((((Uint) (RQP)) & ~((Uint) 0xffff)) != ((Uint) 0xdeadbeefdead0000));\ -} while (0) -# else -# define ERTS_DBG_SET_INVALID_RUNQP(RQP, N) \ - (*((char **) &(RQP)) = (char *) (0xdead0003 | ((N) << 4))) -# define ERTS_DBG_VERIFY_VALID_RUNQP(RQP) \ -do { \ - ASSERT((RQP) != NULL); \ - ASSERT(((((UWord) (RQP)) & ((UWord) 1))) == ((UWord) 0)); \ - ASSERT((((UWord) (RQP)) & ~((UWord) 0xffff)) != ((UWord) 0xdead0000)); \ -} while (0) -# endif -#else -# define ERTS_DBG_SET_INVALID_RUNQP(RQP, N) -# define ERTS_DBG_VERIFY_VALID_RUNQP(RQP) +#define ERTS_RUNQ_FLGS_INIT(RQ, INIT) \ + erts_smp_atomic32_init_nob(&(RQ)->flags, (erts_aint32_t) (INIT)) +#define ERTS_RUNQ_FLGS_SET(RQ, FLGS) \ + ((Uint32) erts_smp_atomic32_read_bor_relb(&(RQ)->flags, \ + (erts_aint32_t) (FLGS))) +#define ERTS_RUNQ_FLGS_UNSET(RQ, FLGS) \ + ((Uint32) erts_smp_atomic32_read_band_relb(&(RQ)->flags, \ + (erts_aint32_t) ~(FLGS))) +#define ERTS_RUNQ_FLGS_GET(RQ) \ + ((Uint32) erts_smp_atomic32_read_acqb(&(RQ)->flags)) +#define ERTS_RUNQ_FLGS_GET_NOB(RQ) \ + ((Uint32) erts_smp_atomic32_read_nob(&(RQ)->flags)) +#define ERTS_RUNQ_FLGS_GET_MB(RQ) \ + ((Uint32) erts_smp_atomic32_read_mb(&(RQ)->flags)) +#define ERTS_RUNQ_FLGS_MASK_SET(RQ, MSK, FLGS) \ + ((Uint32) erts_smp_atomic32_mask_set_relb(&(RQ)->flags, \ + (erts_aint32_t) (MSK), \ + (erts_aint32_t) (FLGS))) + +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_mask_set_relb(erts_smp_atomic32_t *a32p, + erts_aint32_t mask, + erts_aint32_t set); +#if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_mask_set_relb(erts_smp_atomic32_t *a32p, + erts_aint32_t mask, + erts_aint32_t set) +{ + erts_aint32_t act = erts_smp_atomic32_read_nob(a32p); + while (1) { + erts_aint32_t exp = act; + erts_aint32_t new = exp & ~mask; + new |= (mask & set); + act = erts_smp_atomic32_cmpxchg_relb(a32p, new, exp); + if (act == exp) + return act; + } +} #endif typedef enum { @@ -264,7 +281,8 @@ typedef enum { #define ERTS_SSI_AUX_WORK_CHECK_CHILDREN (((erts_aint32_t) 1) << 8) #define ERTS_SSI_AUX_WORK_SET_TMO (((erts_aint32_t) 1) << 9) #define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK (((erts_aint32_t) 1) << 10) -#define ERTS_SSI_AUX_WORK_REAP_PORTS (((erts_aint32_t) 1) << 11) +#define ERTS_SSI_AUX_WORK_CODE_IX_ACTIVATION (((erts_aint32_t) 1) << 11) +#define ERTS_SSI_AUX_WORK_REAP_PORTS (((erts_aint32_t) 1) << 12) typedef struct ErtsSchedulerSleepInfo_ ErtsSchedulerSleepInfo; @@ -290,7 +308,7 @@ struct ErtsSchedulerSleepInfo_ { typedef struct ErtsProcList_ ErtsProcList; struct ErtsProcList_ { Eterm pid; - SysTimeval started; + Uint64 started_interval; ErtsProcList* next; }; @@ -311,21 +329,39 @@ typedef struct ErtsSchedulerData_ ErtsSchedulerData; typedef struct ErtsRunQueue_ ErtsRunQueue; typedef struct { - int len; - int max_len; + erts_smp_atomic32_t len; + erts_aint32_t max_len; int reds; +} ErtsRunQueueInfo; + +#ifdef ERTS_SMP + +typedef struct { + Uint32 flags; + ErtsRunQueue *misc_evac_runq; struct { struct { int this; int other; } limit; ErtsRunQueue *runq; - } migrate; -} ErtsRunQueueInfo; + Uint32 flags; + } prio[ERTS_NO_PRIO_LEVELS]; +} ErtsMigrationPath; + +typedef struct ErtsMigrationPaths_ ErtsMigrationPaths; + +struct ErtsMigrationPaths_ { + void *block; + ErtsMigrationPaths *next; + ErtsThrPrgrVal thr_prgr; + ErtsMigrationPath mpath[1]; +}; + +#endif /* ERTS_SMP */ struct ErtsRunQueue_ { int ix; - erts_smp_atomic32_t info_flags; erts_smp_mtx_t mtx; erts_smp_cnd_t cnd; @@ -333,19 +369,18 @@ struct ErtsRunQueue_ { ErtsSchedulerData *scheduler; int waiting; /* < 0 in sys schedule; > 0 on cnd variable */ int woken; - Uint32 flags; + erts_smp_atomic32_t flags; int check_balance_reds; int full_reds_history_sum; int full_reds_history[ERTS_FULL_REDS_HISTORY_SIZE]; int out_of_work_count; - int max_len; - int len; + erts_aint32_t max_len; + erts_aint32_t len; int wakeup_other; int wakeup_other_reds; int halt_in_progress; struct { - int len; ErtsProcList *pending_exiters; Uint context_switches; Uint reductions; @@ -360,7 +395,7 @@ struct ErtsRunQueue_ { struct { ErtsMiscOpList *start; ErtsMiscOpList *end; - ErtsRunQueue *evac_runq; + erts_smp_atomic_t evac_runq; } misc; struct { @@ -380,7 +415,7 @@ extern ErtsAlignedRunQueue *erts_aligned_run_queues; #define ERTS_PROC_REDUCTIONS_EXECUTED(RQ, PRIO, REDS, AREDS) \ do { \ (RQ)->procs.reductions += (AREDS); \ - (RQ)->procs.prio_info[p->prio].reds += (REDS); \ + (RQ)->procs.prio_info[(PRIO)].reds += (REDS); \ (RQ)->check_balance_reds -= (REDS); \ (RQ)->wakeup_other_reds += (AREDS); \ } while (0) @@ -431,6 +466,12 @@ typedef struct { void *queue; } async_ready; #endif +#ifdef ERTS_SMP + struct { + Process* code_stager; + ErtsThrPrgrVal thr_prgr; + } code_ix_activation; +#endif } ErtsAuxWorkData; struct ErtsSchedulerData_ { @@ -488,6 +529,90 @@ extern ErtsAlignedSchedulerData *erts_aligned_scheduler_data; extern ErtsSchedulerData *erts_scheduler_data; #endif +#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) +int erts_smp_lc_runq_is_locked(ErtsRunQueue *); +#endif + +#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS + +/* + * Run queue locked during modifications. We use atomic ops since + * other threads peek at values without run queue lock. + */ + +ERTS_GLB_INLINE void erts_smp_inc_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio); +ERTS_GLB_INLINE void erts_smp_dec_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio); +ERTS_GLB_INLINE void erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE void +erts_smp_inc_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio) +{ + erts_aint32_t len; + + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); + + len = erts_smp_atomic32_read_nob(&rqi->len); + ASSERT(len >= 0); + if (len == 0) { + ASSERT((erts_smp_atomic32_read_nob(&rq->flags) + & ((erts_aint32_t) (1 << prio))) == 0); + erts_smp_atomic32_read_bor_nob(&rq->flags, + (erts_aint32_t) (1 << prio)); + } + len++; + if (rqi->max_len < len) + rqi->max_len = len; + + erts_smp_atomic32_set_relb(&rqi->len, len); + + rq->len++; + if (rq->max_len < rq->len) + rq->max_len = len; + ASSERT(rq->len > 0); +} + +ERTS_GLB_INLINE void +erts_smp_dec_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio) +{ + erts_aint32_t len; + + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); + + len = erts_smp_atomic32_read_nob(&rqi->len); + len--; + ASSERT(len >= 0); + if (len == 0) { + ASSERT((erts_smp_atomic32_read_nob(&rq->flags) + & ((erts_aint32_t) (1 << prio)))); + erts_smp_atomic32_read_band_nob(&rq->flags, + ~((erts_aint32_t) (1 << prio))); + } + erts_smp_atomic32_set_relb(&rqi->len, len); + + rq->len--; + ASSERT(rq->len >= 0); +} + +ERTS_GLB_INLINE void +erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi) +{ + erts_aint32_t len; + + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); + + len = erts_smp_atomic32_read_nob(&rqi->len); + ASSERT(rqi->max_len >= len); + rqi->max_len = len; +} + +#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ + +#define RUNQ_READ_LEN(X) erts_smp_atomic32_read_nob((X)) + +#endif /* ERTS_INCLUDE_SCHEDULER_INTERNALS */ + /* * Process Specific Data. * @@ -600,6 +725,8 @@ struct ErtsPendingSuspend_ { # define BIN_OLD_VHEAP(p) (p)->bin_old_vheap struct process { + Eterm id; /* The pid of this process + (need to be first in struct) */ /* All fields in the PCB that differs between different heap * architectures, have been moved to the end of this struct to * make sure that as few offsets as possible differ. Different @@ -642,13 +769,9 @@ struct process { * Number of reductions left to execute. * Only valid for the current process. */ - Uint32 status; /* process STATE */ - Uint32 gcstatus; /* process gc STATE */ - Uint32 rstatus; /* process resume STATE */ Uint32 rcount; /* suspend count */ - Eterm id; /* The pid of this process */ int prio; /* Priority of process */ - int skipped; /* Times a low prio process has been rescheduled */ + int schedule_count; /* Times left to reschedule a low prio process */ Uint reds; /* No of reductions for this process */ Eterm tracer_proc; /* If proc is traced, this is the tracer (can NOT be boxed) */ @@ -661,7 +784,6 @@ struct process { Eterm ftrace; /* Latest exception stack trace dump */ Process *next; /* Pointer to next process in run queue */ - Process *prev; /* Pointer to prev process in run queue */ struct reg_proc *reg; /* NULL iff not registered */ ErtsLink *nlinks; @@ -699,8 +821,8 @@ struct process { * Information mainly for post-mortem use (erl crash dump). */ Eterm parent; /* Pid of process that created this process. */ - SysTimeval started; /* Time when started. */ - + erts_approx_time_t approx_started; /* Time when started. */ + Uint64 started_interval; /* This is the place, where all fields that differs between memory * architectures, have gone to. @@ -731,19 +853,16 @@ struct process { void *exit_data; /* Misc data referred during termination */ } u; - ErtsRunQueue *bound_runq; + erts_smp_atomic32_t state; /* Process state flags (see ERTS_PSFLG_*) */ #ifdef ERTS_SMP erts_proc_lock_t lock; ErtsSchedulerData *scheduler_data; - int is_exiting; - Uint32 runq_flags; - Uint32 status_flags; ErlMessageInQueue msg_inq; Eterm suspendee; ErtsPendingSuspend *pending_suspenders; ErtsPendExit pending_exit; - ErtsRunQueue *run_queue; + erts_smp_atomic_t run_queue; #ifdef HIPE struct hipe_process_state_smp hipe_smp; #endif @@ -797,6 +916,29 @@ void erts_check_for_holes(Process* p); #define SEQ_TRACE_TOKEN(p) ((p)->seq_trace_token) +#if ERTS_NO_PROC_PRIO_LEVELS > 4 +# error "Need to increase ERTS_PSFLG_PRIO_SHIFT" +#endif + +#define ERTS_PSFLG_PRIO_SHIFT 2 + +#define ERTS_PSFLG_BIT(N) \ + (((erts_aint32_t) 1) << (ERTS_PSFLG_PRIO_SHIFT + (N))) + +#define ERTS_PSFLG_PRIO_MASK (ERTS_PSFLG_BIT(0) - 1) + +#define ERTS_PSFLG_FREE ERTS_PSFLG_BIT(0) +#define ERTS_PSFLG_EXITING ERTS_PSFLG_BIT(1) +#define ERTS_PSFLG_PENDING_EXIT ERTS_PSFLG_BIT(2) +#define ERTS_PSFLG_ACTIVE ERTS_PSFLG_BIT(3) +#define ERTS_PSFLG_IN_RUNQ ERTS_PSFLG_BIT(4) +#define ERTS_PSFLG_RUNNING ERTS_PSFLG_BIT(5) +#define ERTS_PSFLG_SUSPENDED ERTS_PSFLG_BIT(6) +#define ERTS_PSFLG_GC ERTS_PSFLG_BIT(7) +#define ERTS_PSFLG_BOUND ERTS_PSFLG_BIT(8) +#define ERTS_PSFLG_TRAP_EXIT ERTS_PSFLG_BIT(9) + + /* The sequential tracing token is a tuple of size 5: * * {Flags, Label, Serial, Sender} @@ -869,9 +1011,8 @@ Eterm* erts_heap_alloc(Process* p, Uint need, Uint xtra); Eterm* erts_set_hole_marker(Eterm* ptr, Uint sz); #endif -extern Process** process_tab; -extern Uint erts_max_processes; -extern Uint erts_process_tab_index_mask; +extern erts_smp_rwmtx_t erts_proc_tab_rwmtx; +extern erts_smp_atomic_t *erts_proc_tab; extern Uint erts_default_process_flags; extern erts_smp_rwmtx_t erts_cpu_bind_rwmtx; /* If any of the erts_system_monitor_* variables are set (enabled), @@ -899,17 +1040,13 @@ struct erts_system_profile_flags_t { unsigned int exclusive : 1; }; extern struct erts_system_profile_flags_t erts_system_profile_flags; - -#define INVALID_PID(p, pid) ((p) == NULL \ - || (p)->id != (pid) \ - || (p)->status == P_EXITING) #define IS_TRACED(p) ( (p)->tracer_proc != NIL ) #define ARE_TRACE_FLAGS_ON(p,tf) ( ((p)->trace_flags & (tf|F_SENSITIVE)) == (tf) ) #define IS_TRACED_FL(p,tf) ( IS_TRACED(p) && ARE_TRACE_FLAGS_ON(p,tf) ) /* process flags */ -#define F_TRAPEXIT (1 << 0) +#define F_HIBERNATE_SCHED (1 << 0) /* Schedule out after hibernate op */ #define F_INSLPQUEUE (1 << 1) /* Set if in timer queue */ #define F_TIMO (1 << 2) /* Set if timeout */ #define F_HEAP_GROW (1 << 3) @@ -920,7 +1057,6 @@ extern struct erts_system_profile_flags_t erts_system_profile_flags; #define F_HAVE_BLCKD_MSCHED (1 << 8) /* Process has blocked multi-scheduling */ #define F_P2PNR_RESCHED (1 << 9) /* Process has been rescheduled via erts_pid2proc_not_running() */ #define F_FORCE_GC (1 << 10) /* Force gc at process in-scheduling */ -#define F_HIBERNATE_SCHED (1 << 11) /* Schedule out after hibernate op */ /* process trace_flags */ #define F_SENSITIVE (1 << 0) @@ -987,67 +1123,10 @@ extern struct erts_system_profile_flags_t erts_system_profile_flags; #define DT_UTAG_FLAGS(P) ((P)->dt_utag_flags) #endif - -#ifdef ERTS_SMP -/* Status flags ... */ -#define ERTS_PROC_SFLG_PENDADD2SCHEDQ (((Uint32) 1) << 0) /* Pending - add to - schedule q */ -#define ERTS_PROC_SFLG_INRUNQ (((Uint32) 1) << 1) /* Process is - in run q */ -#define ERTS_PROC_SFLG_TRAPEXIT (((Uint32) 1) << 2) /* Process is - trapping - exit */ -#define ERTS_PROC_SFLG_RUNNING (((Uint32) 1) << 3) /* Process is - running */ -/* Scheduler flags in process struct... */ -#define ERTS_PROC_RUNQ_FLG_RUNNING (((Uint32) 1) << 0) /* Process is - running */ - -#endif - - -#ifdef ERTS_SMP -#define ERTS_PROC_IS_TRAPPING_EXITS(P) \ - (ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks((P)) \ - & ERTS_PROC_LOCK_STATUS), \ - (P)->status_flags & ERTS_PROC_SFLG_TRAPEXIT) - -#define ERTS_PROC_SET_TRAP_EXIT(P) \ - (ERTS_SMP_LC_ASSERT(((ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS) \ - & erts_proc_lc_my_proc_locks((P))) \ - == (ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS)), \ - (P)->status_flags |= ERTS_PROC_SFLG_TRAPEXIT, \ - (P)->flags |= F_TRAPEXIT, \ - 1) - -#define ERTS_PROC_UNSET_TRAP_EXIT(P) \ - (ERTS_SMP_LC_ASSERT(((ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS) \ - & erts_proc_lc_my_proc_locks((P))) \ - == (ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS)), \ - (P)->status_flags &= ~ERTS_PROC_SFLG_TRAPEXIT, \ - (P)->flags &= ~F_TRAPEXIT, \ - 0) -#else -#define ERTS_PROC_IS_TRAPPING_EXITS(P) ((P)->flags & F_TRAPEXIT) -#define ERTS_PROC_SET_TRAP_EXIT(P) ((P)->flags |= F_TRAPEXIT, 1) -#define ERTS_PROC_UNSET_TRAP_EXIT(P) ((P)->flags &= ~F_TRAPEXIT, 0) -#endif - /* Option flags to erts_send_exit_signal() */ #define ERTS_XSIG_FLG_IGN_KILL (((Uint32) 1) << 0) #define ERTS_XSIG_FLG_NO_IGN_NORMAL (((Uint32) 1) << 1) - -/* Process status values */ -#define P_FREE 0 -#define P_RUNABLE 1 -#define P_WAITING 2 -#define P_RUNNING 3 -#define P_EXITING 4 -#define P_GARBING 5 -#define P_SUSPENDED 6 - #define CANCEL_TIMER(p) \ do { \ if ((p)->flags & (F_INSLPQUEUE)) \ @@ -1069,6 +1148,9 @@ void erts_early_init_scheduling(int); void erts_init_scheduling(int, int); Eterm erts_sched_wall_time_request(Process *c_p, int set, int enable); +Uint64 erts_get_proc_interval(void); +Uint64 erts_ensure_later_proc_interval(Uint64); +Uint64 erts_step_proc_interval(void); ErtsProcList *erts_proclist_create(Process *); void erts_proclist_destroy(ErtsProcList *); @@ -1104,6 +1186,9 @@ void erts_smp_notify_check_children_needed(void); #if ERTS_USE_ASYNC_READY_Q void erts_notify_check_async_ready_queue(void *); #endif +#ifdef ERTS_SMP +void erts_notify_code_ix_activation(Process* p, ErtsThrPrgrVal later); +#endif void erts_schedule_misc_aux_work(int sched_id, void (*func)(void *), void *arg); @@ -1124,14 +1209,6 @@ Eterm erts_get_schedulers_binds(Process *c_p); Eterm erts_set_cpu_topology(Process *c_p, Eterm term); Eterm erts_bind_schedulers(Process *c_p, Eterm how); ErtsRunQueue *erts_schedid2runq(Uint); -#ifdef ERTS_SMP -ErtsMigrateResult erts_proc_migrate(Process *, - ErtsProcLocks *, - ErtsRunQueue *, - int *, - ErtsRunQueue *, - int *); -#endif Process *schedule(Process*, int); void erts_schedule_misc_op(void (*)(void *), void *); Eterm erl_create_process(Process*, Eterm, Eterm, Eterm, ErlSpawnOpts*); @@ -1180,8 +1257,7 @@ int erts_send_exit_signal(Process *, #ifdef ERTS_SMP void erts_handle_pending_exit(Process *, ErtsProcLocks); #define ERTS_PROC_PENDING_EXIT(P) \ - (ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks((P)) & ERTS_PROC_LOCK_STATUS),\ - (P)->pending_exit.reason != THE_NON_VALUE) + (ERTS_PSFLG_PENDING_EXIT & erts_smp_atomic32_read_acqb(&(P)->state)) #else #define ERTS_PROC_PENDING_EXIT(P) 0 #endif @@ -1233,13 +1309,26 @@ ErtsSchedulerData *erts_get_scheduler_data(void) #endif #endif +void erts_schedule_process(Process *, erts_aint32_t); + +ERTS_GLB_INLINE void erts_proc_notify_new_message(Process *p); +#if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE void +erts_proc_notify_new_message(Process *p) +{ + /* No barrier needed, due to msg lock */ + erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state); + if (!(state & ERTS_PSFLG_ACTIVE)) + erts_schedule_process(p, state); +} +#endif + #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) #define ERTS_PROCESS_LOCK_ONLY_LOCK_CHECK_PROTO__ #include "erl_process_lock.h" #undef ERTS_PROCESS_LOCK_ONLY_LOCK_CHECK_PROTO__ -int erts_smp_lc_runq_is_locked(ErtsRunQueue *); #define ERTS_SMP_LC_CHK_RUNQ_LOCK(RQ, L) \ do { \ if ((L)) \ @@ -1365,13 +1454,91 @@ erts_proc_set_error_handler(Process *p, ErtsProcLocks plocks, Eterm handler) #endif +#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS + #ifdef ERTS_SMP -ErtsRunQueue *erts_prepare_emigrate(ErtsRunQueue *c_rq, - ErtsRunQueueInfo *c_rqi, - int prio); +#include "erl_thr_progress.h" + +extern erts_atomic_t erts_migration_paths; + +ERTS_GLB_INLINE ErtsMigrationPaths *erts_get_migration_paths_managed(void); +ERTS_GLB_INLINE ErtsMigrationPaths *erts_get_migration_paths(void); ERTS_GLB_INLINE ErtsRunQueue *erts_check_emigration_need(ErtsRunQueue *c_rq, int prio); +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE ErtsMigrationPaths * +erts_get_migration_paths_managed(void) +{ + return (ErtsMigrationPaths *) erts_atomic_read_ddrb(&erts_migration_paths); +} + +ERTS_GLB_INLINE ErtsMigrationPaths * +erts_get_migration_paths(void) +{ + if (erts_thr_progress_is_managed_thread()) + return erts_get_migration_paths_managed(); + else + return NULL; +} + +ERTS_GLB_INLINE ErtsRunQueue * +erts_check_emigration_need(ErtsRunQueue *c_rq, int prio) +{ + ErtsMigrationPaths *mps = erts_get_migration_paths(); + ErtsMigrationPath *mp; + Uint32 flags; + + if (!mps) + return NULL; + + mp = &mps->mpath[c_rq->ix]; + flags = mp->flags; + + if (ERTS_CHK_RUNQ_FLG_EMIGRATE(flags, prio)) { + int len; + + if (ERTS_CHK_RUNQ_FLG_EVACUATE(flags, prio)) { + /* force emigration */ + return mp->prio[prio].runq; + } + + if (flags & ERTS_RUNQ_FLG_INACTIVE) { + /* + * Run queue was inactive at last balance. Verify that + * it still is before forcing emigration. + */ + if (ERTS_RUNQ_FLGS_GET(c_rq) & ERTS_RUNQ_FLG_INACTIVE) + return mp->prio[prio].runq; + } + + + if (prio == ERTS_PORT_PRIO_LEVEL) + len = RUNQ_READ_LEN(&c_rq->ports.info.len); + else + len = RUNQ_READ_LEN(&c_rq->procs.prio_info[prio].len); + + if (len > mp->prio[prio].limit.this) { + ErtsRunQueue *n_rq = mp->prio[prio].runq; + if (n_rq) { + if (prio == ERTS_PORT_PRIO_LEVEL) + len = RUNQ_READ_LEN(&n_rq->ports.info.len); + else + len = RUNQ_READ_LEN(&n_rq->procs.prio_info[prio].len); + + if (len < mp->prio[prio].limit.other) + return n_rq; + } + } + } + return NULL; +} + +#endif + +#endif + #endif ERTS_GLB_INLINE int erts_is_scheduler_bound(ErtsSchedulerData *esdp); @@ -1394,29 +1561,6 @@ ERTS_GLB_INLINE void erts_smp_runqs_unlock(ErtsRunQueue *rq1, ErtsRunQueue *rq2) #if ERTS_GLB_INLINE_INCL_FUNC_DEF -#ifdef ERTS_SMP -ERTS_GLB_INLINE ErtsRunQueue * -erts_check_emigration_need(ErtsRunQueue *c_rq, int prio) -{ - ErtsRunQueueInfo *c_rqi; - - if (!ERTS_CHK_RUNQ_FLG_EMIGRATE(c_rq->flags, prio)) - return NULL; - - if (prio == ERTS_PORT_PRIO_LEVEL) - c_rqi = &c_rq->ports.info; - else - c_rqi = &c_rq->procs.prio_info[prio]; - - if (!ERTS_CHK_RUNQ_FLG_EVACUATE(c_rq->flags, prio) - && !(c_rq->flags & ERTS_RUNQ_FLG_INACTIVE) - && c_rqi->len <= c_rqi->migrate.limit.this) - return NULL; - - return erts_prepare_emigrate(c_rq, c_rqi, prio); -} -#endif - ERTS_GLB_INLINE int erts_is_scheduler_bound(ErtsSchedulerData *esdp) { @@ -1454,10 +1598,9 @@ Uint erts_get_scheduler_id(void) ERTS_GLB_INLINE ErtsRunQueue * erts_get_runq_proc(Process *p) { - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); #ifdef ERTS_SMP - ASSERT(p->run_queue); - return p->run_queue; + ASSERT(ERTS_AINT_NULL != erts_atomic_read_nob(&p->run_queue)); + return (ErtsRunQueue *) erts_atomic_read_nob(&p->run_queue); #else return ERTS_RUNQ_IX(0); #endif @@ -1628,25 +1771,13 @@ extern int erts_disable_proc_not_running_opt; #ifdef DEBUG #define ERTS_SMP_ASSERT_IS_NOT_EXITING(P) \ - do { ASSERT(!(P)->is_exiting); } while (0) + do { ASSERT(!ERTS_PROC_IS_EXITING((P))); } while (0) #else #define ERTS_SMP_ASSERT_IS_NOT_EXITING(P) #endif -/* NOTE: At least one process lock has to be held on P! */ -#ifdef ERTS_ENABLE_LOCK_CHECK -#define ERTS_PROC_IS_EXITING(P) \ - (ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks((P)) != 0 \ - || erts_lc_pix_lock_is_locked(ERTS_PID2PIXLOCK((P)->id))),\ - (P)->is_exiting) -#else -#define ERTS_PROC_IS_EXITING(P) ((P)->is_exiting) -#endif - #else /* !ERTS_SMP */ -#define ERTS_PROC_IS_EXITING(P) ((P)->status == P_EXITING) - #define ERTS_SMP_ASSERT_IS_NOT_EXITING(P) #define erts_pid2proc_not_running erts_pid2proc @@ -1654,6 +1785,10 @@ extern int erts_disable_proc_not_running_opt; #endif +#define ERTS_PROC_IS_EXITING(P) \ + (ERTS_PSFLG_EXITING & erts_smp_atomic32_read_acqb(&(P)->state)) + + /* Minimum NUMBER of processes for a small system to start */ #ifdef ERTS_SMP #define ERTS_MIN_PROCESSES ERTS_NO_OF_PIX_LOCKS diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c index 3550f1396c..964dc1ae3e 100644 --- a/erts/emulator/beam/erl_process_dump.c +++ b/erts/emulator/beam/erl_process_dump.c @@ -65,14 +65,11 @@ erts_deep_process_dump(int to, void *to_arg) all_binaries = NULL; for (i = 0; i < erts_max_processes; i++) { - if ((process_tab[i] != NULL) && (process_tab[i]->i != ENULL)) { - if (process_tab[i]->status != P_EXITING) { - Process* p = process_tab[i]; - - if (p->status != P_GARBING) { - dump_process_info(to, to_arg, p); - } - } + Process *p = erts_pix2proc(i); + if (p && p->i != ENULL) { + erts_aint32_t state = erts_smp_atomic32_read_acqb(&p->state); + if (!(state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_GC))) + dump_process_info(to, to_arg, p); } } diff --git a/erts/emulator/beam/erl_process_lock.c b/erts/emulator/beam/erl_process_lock.c index b3b4601a31..bae2b383a4 100644 --- a/erts/emulator/beam/erl_process_lock.c +++ b/erts/emulator/beam/erl_process_lock.c @@ -67,10 +67,12 @@ #include "erl_process.h" -const Process erts_proc_lock_busy; +const Process erts_proc_lock_busy = {ERTS_INVALID_PID}; #ifdef ERTS_SMP +#if ERTS_PROC_LOCK_OWN_IMPL + #define ERTS_PROC_LOCK_SPIN_COUNT_MAX 2000 #define ERTS_PROC_LOCK_SPIN_COUNT_SCHED_INC 32 #define ERTS_PROC_LOCK_SPIN_COUNT_BASE 1000 @@ -100,6 +102,13 @@ static erts_proc_lock_queues_t zeroqs = {0}; static erts_smp_spinlock_t qs_lock; static erts_proc_lock_queues_t *queue_free_list; +static int proc_lock_spin_count; +static int aux_thr_proc_lock_spin_count; + +static void cleanup_tse(void); + +#endif /* ERTS_PROC_LOCK_OWN_IMPL */ + #ifdef ERTS_ENABLE_LOCK_CHECK static struct { Sint16 proc_lock_main; @@ -111,16 +120,11 @@ static struct { erts_pix_lock_t erts_pix_locks[ERTS_NO_OF_PIX_LOCKS]; -static int proc_lock_spin_count; -static int aux_thr_proc_lock_spin_count; - -static void cleanup_tse(void); void erts_init_proc_lock(int cpus) { int i; - erts_smp_spinlock_init(&qs_lock, "proc_lck_qs_alloc"); for (i = 0; i < ERTS_NO_OF_PIX_LOCKS; i++) { #ifdef ERTS_ENABLE_LOCK_COUNT erts_mtx_init_x(&erts_pix_locks[i].u.mtx, @@ -129,14 +133,10 @@ erts_init_proc_lock(int cpus) erts_mtx_init(&erts_pix_locks[i].u.mtx, "pix_lock"); #endif } +#if ERTS_PROC_LOCK_OWN_IMPL + erts_smp_spinlock_init(&qs_lock, "proc_lck_qs_alloc"); queue_free_list = NULL; erts_thr_install_exit_handler(cleanup_tse); -#ifdef ERTS_ENABLE_LOCK_CHECK - lc_id.proc_lock_main = erts_lc_get_lock_order_id("proc_main"); - lc_id.proc_lock_link = erts_lc_get_lock_order_id("proc_link"); - lc_id.proc_lock_msgq = erts_lc_get_lock_order_id("proc_msgq"); - lc_id.proc_lock_status = erts_lc_get_lock_order_id("proc_status"); -#endif if (cpus > 1) { proc_lock_spin_count = ERTS_PROC_LOCK_SPIN_COUNT_BASE; proc_lock_spin_count += (ERTS_PROC_LOCK_SPIN_COUNT_SCHED_INC @@ -153,8 +153,17 @@ erts_init_proc_lock(int cpus) } if (proc_lock_spin_count > ERTS_PROC_LOCK_SPIN_COUNT_MAX) proc_lock_spin_count = ERTS_PROC_LOCK_SPIN_COUNT_MAX; +#endif +#ifdef ERTS_ENABLE_LOCK_CHECK + lc_id.proc_lock_main = erts_lc_get_lock_order_id("proc_main"); + lc_id.proc_lock_link = erts_lc_get_lock_order_id("proc_link"); + lc_id.proc_lock_msgq = erts_lc_get_lock_order_id("proc_msgq"); + lc_id.proc_lock_status = erts_lc_get_lock_order_id("proc_status"); +#endif } +#if ERTS_PROC_LOCK_OWN_IMPL + #ifdef ERTS_ENABLE_LOCK_CHECK static void check_unused_tse(erts_tse_t *wtr) @@ -220,13 +229,6 @@ tse_return(erts_tse_t *tse, int force_free_q) erts_tse_return(tse); } -void -erts_proc_lock_prepare_proc_lock_waiter(void) -{ - tse_return(tse_fetch(NULL), 0); -} - - static void cleanup_tse(void) { @@ -651,6 +653,16 @@ erts_proc_unlock_failed(Process *p, transfer_locks(p, wait_locks, pix_lock, 1); /* unlocks pix_lock */ } +#endif /* ERTS_PROC_LOCK_OWN_IMPL */ + +void +erts_proc_lock_prepare_proc_lock_waiter(void) +{ +#if ERTS_PROC_LOCK_OWN_IMPL + tse_return(tse_fetch(NULL), 0); +#endif +} + /* * proc_safelock() locks process locks on two processes. In order * to avoid a deadlock, proc_safelock() unlocks those locks that @@ -659,12 +671,11 @@ erts_proc_unlock_failed(Process *p, */ static void -proc_safelock(Process *a_proc, - erts_pix_lock_t *a_pix_lck, +proc_safelock(int is_sched, + Process *a_proc, ErtsProcLocks a_have_locks, ErtsProcLocks a_need_locks, Process *b_proc, - erts_pix_lock_t *b_pix_lck, ErtsProcLocks b_have_locks, ErtsProcLocks b_need_locks) { @@ -672,7 +683,6 @@ proc_safelock(Process *a_proc, #ifdef ERTS_ENABLE_LOCK_CHECK Eterm pid1, pid2; #endif - erts_pix_lock_t *pix_lck1, *pix_lck2; ErtsProcLocks need_locks1, have_locks1, need_locks2, have_locks2; ErtsProcLocks unlock_mask; int lock_no, refc1 = 0, refc2 = 0; @@ -689,14 +699,12 @@ proc_safelock(Process *a_proc, #ifdef ERTS_ENABLE_LOCK_CHECK pid1 = a_proc->id; #endif - pix_lck1 = a_pix_lck; need_locks1 = a_need_locks; have_locks1 = a_have_locks; p2 = b_proc; #ifdef ERTS_ENABLE_LOCK_CHECK pid2 = b_proc->id; #endif - pix_lck2 = b_pix_lck; need_locks2 = b_need_locks; have_locks2 = b_have_locks; } @@ -705,14 +713,12 @@ proc_safelock(Process *a_proc, #ifdef ERTS_ENABLE_LOCK_CHECK pid1 = b_proc->id; #endif - pix_lck1 = b_pix_lck; need_locks1 = b_need_locks; have_locks1 = b_have_locks; p2 = a_proc; #ifdef ERTS_ENABLE_LOCK_CHECK pid2 = a_proc->id; #endif - pix_lck2 = a_pix_lck; need_locks2 = a_need_locks; have_locks2 = a_have_locks; } @@ -723,14 +729,12 @@ proc_safelock(Process *a_proc, #ifdef ERTS_ENABLE_LOCK_CHECK pid1 = a_proc->id; #endif - pix_lck1 = a_pix_lck; need_locks1 = a_need_locks | b_need_locks; have_locks1 = a_have_locks | b_have_locks; p2 = NULL; #ifdef ERTS_ENABLE_LOCK_CHECK pid2 = 0; #endif - pix_lck2 = NULL; need_locks2 = 0; have_locks2 = 0; } @@ -740,14 +744,12 @@ proc_safelock(Process *a_proc, #ifdef ERTS_ENABLE_LOCK_CHECK pid1 = b_proc->id; #endif - pix_lck1 = b_pix_lck; need_locks1 = b_need_locks; have_locks1 = b_have_locks; p2 = NULL; #ifdef ERTS_ENABLE_LOCK_CHECK pid2 = 0; #endif - pix_lck2 = NULL; need_locks2 = 0; have_locks2 = 0; #ifdef ERTS_ENABLE_LOCK_CHECK @@ -795,21 +797,21 @@ proc_safelock(Process *a_proc, if (unlock_locks) { have_locks1 &= ~unlock_locks; need_locks1 |= unlock_locks; - if (!have_locks1) { + if (!is_sched && !have_locks1) { refc1 = 1; erts_smp_proc_inc_refc(p1); } - erts_smp_proc_unlock__(p1, pix_lck1, unlock_locks); + erts_smp_proc_unlock(p1, unlock_locks); } unlock_locks = unlock_mask & have_locks2; if (unlock_locks) { have_locks2 &= ~unlock_locks; need_locks2 |= unlock_locks; - if (!have_locks2) { + if (!is_sched && !have_locks2) { refc2 = 1; erts_smp_proc_inc_refc(p2); } - erts_smp_proc_unlock__(p2, pix_lck2, unlock_locks); + erts_smp_proc_unlock(p2, unlock_locks); } } @@ -840,7 +842,7 @@ proc_safelock(Process *a_proc, if (need_locks2 & lock) lock_no--; locks = need_locks1 & lock_mask; - erts_smp_proc_lock__(p1, pix_lck1, locks); + erts_smp_proc_lock(p1, locks); have_locks1 |= locks; need_locks1 &= ~locks; } @@ -851,7 +853,7 @@ proc_safelock(Process *a_proc, lock = (1 << ++lock_no); } locks = need_locks2 & lock_mask; - erts_smp_proc_lock__(p2, pix_lck2, locks); + erts_smp_proc_lock(p2, locks); have_locks2 |= locks; need_locks2 &= ~locks; } @@ -886,10 +888,12 @@ proc_safelock(Process *a_proc, } #endif - if (refc1) - erts_smp_proc_dec_refc(p1); - if (refc2) - erts_smp_proc_dec_refc(p2); + if (!is_sched) { + if (refc1) + erts_smp_proc_dec_refc(p1); + if (refc2) + erts_smp_proc_dec_refc(p2); + } } void @@ -900,77 +904,191 @@ erts_proc_safelock(Process *a_proc, ErtsProcLocks b_have_locks, ErtsProcLocks b_need_locks) { - proc_safelock(a_proc, - a_proc ? ERTS_PID2PIXLOCK(a_proc->id) : NULL, + proc_safelock(erts_get_scheduler_id() != 0, + a_proc, a_have_locks, a_need_locks, b_proc, - b_proc ? ERTS_PID2PIXLOCK(b_proc->id) : NULL, b_have_locks, b_need_locks); } -/* - * erts_pid2proc_safelock() is called from erts_pid2proc_opt() when - * it wasn't possible to trylock all locks needed. - * c_p - current process - * c_p_have_locks - locks held on c_p - * pid - process id of process we are looking up - * proc - process struct of process we are looking - * up (both in and out argument) - * need_locks - all locks we need (including have_locks) - * pix_lock - pix lock for process we are looking up - * flags - option flags - */ -void -erts_pid2proc_safelock(Process *c_p, - ErtsProcLocks c_p_have_locks, - Process **proc, - ErtsProcLocks need_locks, - erts_pix_lock_t *pix_lock, - int flags) +Process * +erts_pid2proc_opt(Process *c_p, + ErtsProcLocks c_p_have_locks, + Eterm pid, + ErtsProcLocks pid_need_locks, + int flags) { - Process *p = *proc; - ERTS_LC_ASSERT(p->lock.refc > 0); - ERTS_LC_ASSERT(process_tab[internal_pid_index(p->id)] == p); - p->lock.refc++; - erts_pix_unlock(pix_lock); - - proc_safelock(c_p, - c_p ? ERTS_PID2PIXLOCK(c_p->id) : NULL, - c_p_have_locks, - c_p_have_locks, - p, - pix_lock, - 0, - need_locks); + Process *dec_refc_proc = NULL; + int need_ptl; + ErtsProcLocks need_locks; + Uint pix; + Process *proc; +#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT) + ErtsProcLocks lcnt_locks; +#endif - erts_pix_lock(pix_lock); +#ifdef ERTS_ENABLE_LOCK_CHECK + if (c_p) { + ErtsProcLocks might_unlock = c_p_have_locks & pid_need_locks; + if (might_unlock) + erts_proc_lc_might_unlock(c_p, might_unlock); + } +#endif - if (!p->is_exiting - || ((flags & ERTS_P2P_FLG_ALLOW_OTHER_X) - && process_tab[internal_pid_index(p->id)] == p)) { - ERTS_LC_ASSERT(p->lock.refc > 1); - p->lock.refc--; + if (is_not_internal_pid(pid)) + return NULL; + pix = internal_pid_index(pid); + + ERTS_LC_ASSERT((pid_need_locks & ERTS_PROC_LOCKS_ALL) == pid_need_locks); + need_locks = pid_need_locks; + + if (c_p && c_p->id == pid) { + ASSERT(c_p->id != ERTS_INVALID_PID); + ASSERT(c_p == erts_pix2proc(pix)); + + if (!(flags & ERTS_P2P_FLG_ALLOW_OTHER_X) + && ERTS_PROC_IS_EXITING(c_p)) + return NULL; + need_locks &= ~c_p_have_locks; + if (!need_locks) { + if (flags & ERTS_P2P_FLG_SMP_INC_REFC) + erts_smp_proc_inc_refc(c_p); + return c_p; + } } - else { - /* No proc. Note, we need to keep refc until after process unlock */ - erts_pix_unlock(pix_lock); - erts_smp_proc_unlock__(p, pix_lock, need_locks); - *proc = NULL; - erts_pix_lock(pix_lock); - ERTS_LC_ASSERT(p->lock.refc > 0); - if (--p->lock.refc == 0) { - erts_pix_unlock(pix_lock); - erts_free_proc(p); - erts_pix_lock(pix_lock); + + need_ptl = !erts_get_scheduler_id(); + + if (need_ptl) + erts_smp_rwmtx_rwlock(&erts_proc_tab_rwmtx); + + proc = (Process *) erts_smp_atomic_read_ddrb(&erts_proc.tab[pix]); + + if (proc) { + if (proc->id != pid) + proc = NULL; + else if (!need_locks) { + if (flags & ERTS_P2P_FLG_SMP_INC_REFC) + erts_smp_proc_inc_refc(proc); } + else { + int busy; + +#if ERTS_PROC_LOCK_OWN_IMPL +#ifdef ERTS_ENABLE_LOCK_COUNT + lcnt_locks = need_locks; + if (!(flags & ERTS_P2P_FLG_TRY_LOCK)) { + erts_lcnt_proc_lock(&proc->lock, need_locks); + } +#endif + +#ifdef ERTS_ENABLE_LOCK_CHECK + /* Make sure erts_pid2proc_safelock() is enough to handle + a potential lock order violation situation... */ + busy = erts_proc_lc_trylock_force_busy(proc, need_locks); + if (!busy) +#endif +#endif /* ERTS_PROC_LOCK_OWN_IMPL */ + { + /* Try a quick trylock to grab all the locks we need. */ + busy = (int) erts_smp_proc_raw_trylock__(proc, need_locks); + +#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_CHECK) + erts_proc_lc_trylock(proc, need_locks, !busy); +#endif +#ifdef ERTS_PROC_LOCK_DEBUG + if (!busy) + erts_proc_lock_op_debug(proc, need_locks, 1); +#endif + } + +#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT) + if (flags & ERTS_P2P_FLG_TRY_LOCK) + erts_lcnt_proc_trylock(&proc->lock, need_locks, + busy ? EBUSY : 0); +#endif + + if (!busy) { + if (flags & ERTS_P2P_FLG_SMP_INC_REFC) + erts_smp_proc_inc_refc(proc); + +#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT) + /* all is great */ + if (!(flags & ERTS_P2P_FLG_TRY_LOCK)) + erts_lcnt_proc_lock_post_x(&proc->lock, lcnt_locks, + __FILE__, __LINE__); +#endif + + } + else { + if (flags & ERTS_P2P_FLG_TRY_LOCK) + proc = ERTS_PROC_LOCK_BUSY; + else { + if (flags & ERTS_P2P_FLG_SMP_INC_REFC) + erts_smp_proc_inc_refc(proc); + +#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT) + erts_lcnt_proc_lock_unaquire(&proc->lock, lcnt_locks); +#endif + + if (need_ptl) { + erts_smp_proc_inc_refc(proc); + dec_refc_proc = proc; + erts_smp_rwmtx_rwunlock(&erts_proc_tab_rwmtx); + need_ptl = 0; + } + + proc_safelock(!need_ptl, + c_p, + c_p_have_locks, + c_p_have_locks, + proc, + 0, + need_locks); + } + } + } } + + if (need_ptl) + erts_smp_rwmtx_rwunlock(&erts_proc_tab_rwmtx); + + if (need_locks + && proc + && proc != ERTS_PROC_LOCK_BUSY + && (!(flags & ERTS_P2P_FLG_ALLOW_OTHER_X) + ? ERTS_PROC_IS_EXITING(proc) + : (proc + != (Process *) erts_smp_atomic_read_nob(&erts_proc.tab[pix])))) { + + erts_smp_proc_unlock(proc, need_locks); + + if (flags & ERTS_P2P_FLG_SMP_INC_REFC) + dec_refc_proc = proc; + proc = NULL; + + } + + if (dec_refc_proc) + erts_smp_proc_dec_refc(dec_refc_proc); + +#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_PROC_LOCK_DEBUG) + ERTS_LC_ASSERT(!proc + || proc == ERTS_PROC_LOCK_BUSY + || (pid_need_locks == + (ERTS_PROC_LOCK_FLGS_READ_(&proc->lock) + & pid_need_locks))); +#endif + + return proc; } void erts_proc_lock_init(Process *p) { +#if ERTS_PROC_LOCK_OWN_IMPL /* We always start with all locks locked */ #if ERTS_PROC_LOCK_ATOMIC_IMPL erts_smp_atomic32_init_nob(&p->lock.flags, @@ -979,16 +1097,32 @@ erts_proc_lock_init(Process *p) p->lock.flags = ERTS_PROC_LOCKS_ALL; #endif p->lock.queues = NULL; - p->lock.refc = 1; -#ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_proc_lock_init(p); - erts_lcnt_proc_lock(&(p->lock), ERTS_PROC_LOCKS_ALL); - erts_lcnt_proc_lock_post_x(&(p->lock), ERTS_PROC_LOCKS_ALL, __FILE__, __LINE__); -#endif - #ifdef ERTS_ENABLE_LOCK_CHECK erts_proc_lc_trylock(p, ERTS_PROC_LOCKS_ALL, 1); #endif +#elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL + erts_mtx_init_x(&p->lock.main, "proc_main", p->id); + ethr_mutex_lock(&p->lock.main.mtx); +#ifdef ERTS_ENABLE_LOCK_CHECK + erts_lc_trylock(1, &p->lock.main.lc); +#endif + erts_mtx_init_x(&p->lock.link, "proc_link", p->id); + ethr_mutex_lock(&p->lock.link.mtx); +#ifdef ERTS_ENABLE_LOCK_CHECK + erts_lc_trylock(1, &p->lock.link.lc); +#endif + erts_mtx_init_x(&p->lock.msgq, "proc_msgq", p->id); + ethr_mutex_lock(&p->lock.msgq.mtx); +#ifdef ERTS_ENABLE_LOCK_CHECK + erts_lc_trylock(1, &p->lock.msgq.lc); +#endif + erts_mtx_init_x(&p->lock.status, "proc_status", p->id); + ethr_mutex_lock(&p->lock.status.mtx); +#ifdef ERTS_ENABLE_LOCK_CHECK + erts_lc_trylock(1, &p->lock.status.lc); +#endif +#endif + erts_atomic32_init_nob(&p->lock.refc, 1); #ifdef ERTS_PROC_LOCK_DEBUG { int i; @@ -996,11 +1130,30 @@ erts_proc_lock_init(Process *p) erts_smp_atomic32_init_nob(&p->lock.locked[i], (erts_aint32_t) 1); } #endif +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_proc_lock_init(p); + erts_lcnt_proc_lock(&(p->lock), ERTS_PROC_LOCKS_ALL); + erts_lcnt_proc_lock_post_x(&(p->lock), ERTS_PROC_LOCKS_ALL, __FILE__, __LINE__); +#endif +} + +void +erts_proc_lock_fin(Process *p) +{ +#if ERTS_PROC_LOCK_RAW_MUTEX_IMPL + erts_mtx_destroy(&p->lock.main); + erts_mtx_destroy(&p->lock.link); + erts_mtx_destroy(&p->lock.msgq); + erts_mtx_destroy(&p->lock.status); +#endif +#if defined(ERTS_ENABLE_LOCK_COUNT) && defined(ERTS_SMP) + erts_lcnt_proc_lock_destroy(p); +#endif } /* --- Process lock counting ----------------------------------------------- */ -#ifdef ERTS_ENABLE_LOCK_COUNT +#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT) void erts_lcnt_proc_lock_init(Process *p) { if (erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK) { if (p->id != ERTS_INVALID_PID) { @@ -1141,6 +1294,8 @@ void erts_lcnt_enable_proc_lock_count(int enable) { #ifdef ERTS_ENABLE_LOCK_CHECK +#if ERTS_PROC_LOCK_OWN_IMPL + void erts_proc_lc_lock(Process *p, ErtsProcLocks locks) { @@ -1213,9 +1368,12 @@ erts_proc_lc_unlock(Process *p, ErtsProcLocks locks) } } +#endif /* ERTS_PROC_LOCK_OWN_IMPL */ + void erts_proc_lc_might_unlock(Process *p, ErtsProcLocks locks) { +#if ERTS_PROC_LOCK_OWN_IMPL erts_lc_lock_t lck = ERTS_LC_LOCK_INIT(-1, p->id, ERTS_LC_FLG_LT_PROCLOCK); @@ -1235,11 +1393,22 @@ erts_proc_lc_might_unlock(Process *p, ErtsProcLocks locks) lck.id = lc_id.proc_lock_main; erts_lc_might_unlock(&lck); } +#elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL + if (locks & ERTS_PROC_LOCK_MAIN) + erts_lc_might_unlock(&p->lock.main.lc); + if (locks & ERTS_PROC_LOCK_LINK) + erts_lc_might_unlock(&p->lock.link.lc); + if (locks & ERTS_PROC_LOCK_MSGQ) + erts_lc_might_unlock(&p->lock.msgq.lc); + if (locks & ERTS_PROC_LOCK_STATUS) + erts_lc_might_unlock(&p->lock.status.lc); +#endif } void erts_proc_lc_require_lock(Process *p, ErtsProcLocks locks) { +#if ERTS_PROC_LOCK_OWN_IMPL erts_lc_lock_t lck = ERTS_LC_LOCK_INIT(-1, p->id, ERTS_LC_FLG_LT_PROCLOCK); @@ -1259,11 +1428,22 @@ erts_proc_lc_require_lock(Process *p, ErtsProcLocks locks) lck.id = lc_id.proc_lock_status; erts_lc_require_lock(&lck); } +#elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL + if (locks & ERTS_PROC_LOCK_MAIN) + erts_lc_require_lock(&p->lock.main.lc); + if (locks & ERTS_PROC_LOCK_LINK) + erts_lc_require_lock(&p->lock.link.lc); + if (locks & ERTS_PROC_LOCK_MSGQ) + erts_lc_require_lock(&p->lock.msgq.lc); + if (locks & ERTS_PROC_LOCK_STATUS) + erts_lc_require_lock(&p->lock.status.lc); +#endif } void erts_proc_lc_unrequire_lock(Process *p, ErtsProcLocks locks) { +#if ERTS_PROC_LOCK_OWN_IMPL erts_lc_lock_t lck = ERTS_LC_LOCK_INIT(-1, p->id, ERTS_LC_FLG_LT_PROCLOCK); @@ -1283,8 +1463,19 @@ erts_proc_lc_unrequire_lock(Process *p, ErtsProcLocks locks) lck.id = lc_id.proc_lock_main; erts_lc_unrequire_lock(&lck); } +#elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL + if (locks & ERTS_PROC_LOCK_MAIN) + erts_lc_unrequire_lock(&p->lock.main.lc); + if (locks & ERTS_PROC_LOCK_LINK) + erts_lc_unrequire_lock(&p->lock.link.lc); + if (locks & ERTS_PROC_LOCK_MSGQ) + erts_lc_unrequire_lock(&p->lock.msgq.lc); + if (locks & ERTS_PROC_LOCK_STATUS) + erts_lc_unrequire_lock(&p->lock.status.lc); +#endif } +#if ERTS_PROC_LOCK_OWN_IMPL int erts_proc_lc_trylock_force_busy(Process *p, ErtsProcLocks locks) @@ -1310,21 +1501,30 @@ erts_proc_lc_trylock_force_busy(Process *p, ErtsProcLocks locks) return 0; } +#endif /* ERTS_PROC_LOCK_OWN_IMPL */ + void erts_proc_lc_chk_only_proc_main(Process *p) { +#if ERTS_PROC_LOCK_OWN_IMPL erts_lc_lock_t proc_main = ERTS_LC_LOCK_INIT(lc_id.proc_lock_main, p->id, ERTS_LC_FLG_LT_PROCLOCK); erts_lc_check_exact(&proc_main, 1); +#elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL + erts_lc_check_exact(&p->lock.main.lc, 1); +#endif } +#if ERTS_PROC_LOCK_OWN_IMPL #define ERTS_PROC_LC_EMPTY_LOCK_INIT \ ERTS_LC_LOCK_INIT(-1, THE_NON_VALUE, ERTS_LC_FLG_LT_PROCLOCK) +#endif /* ERTS_PROC_LOCK_OWN_IMPL */ void erts_proc_lc_chk_have_proc_locks(Process *p, ErtsProcLocks locks) { int have_locks_len = 0; +#if ERTS_PROC_LOCK_OWN_IMPL erts_lc_lock_t have_locks[4] = {ERTS_PROC_LC_EMPTY_LOCK_INIT, ERTS_PROC_LC_EMPTY_LOCK_INIT, ERTS_PROC_LC_EMPTY_LOCK_INIT, @@ -1345,7 +1545,17 @@ erts_proc_lc_chk_have_proc_locks(Process *p, ErtsProcLocks locks) have_locks[have_locks_len].id = lc_id.proc_lock_status; have_locks[have_locks_len++].extra = p->id; } - +#elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL + erts_lc_lock_t have_locks[4]; + if (locks & ERTS_PROC_LOCK_MAIN) + have_locks[have_locks_len++] = p->lock.main.lc; + if (locks & ERTS_PROC_LOCK_LINK) + have_locks[have_locks_len++] = p->lock.link.lc; + if (locks & ERTS_PROC_LOCK_MSGQ) + have_locks[have_locks_len++] = p->lock.msgq.lc; + if (locks & ERTS_PROC_LOCK_STATUS) + have_locks[have_locks_len++] = p->lock.status.lc; +#endif erts_lc_check(have_locks, have_locks_len, NULL, 0); } @@ -1354,6 +1564,7 @@ erts_proc_lc_chk_proc_locks(Process *p, ErtsProcLocks locks) { int have_locks_len = 0; int have_not_locks_len = 0; +#if ERTS_PROC_LOCK_OWN_IMPL erts_lc_lock_t have_locks[4] = {ERTS_PROC_LC_EMPTY_LOCK_INIT, ERTS_PROC_LC_EMPTY_LOCK_INIT, ERTS_PROC_LC_EMPTY_LOCK_INIT, @@ -1395,6 +1606,27 @@ erts_proc_lc_chk_proc_locks(Process *p, ErtsProcLocks locks) have_not_locks[have_not_locks_len].id = lc_id.proc_lock_status; have_not_locks[have_not_locks_len++].extra = p->id; } +#elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL + erts_lc_lock_t have_locks[4]; + erts_lc_lock_t have_not_locks[4]; + + if (locks & ERTS_PROC_LOCK_MAIN) + have_locks[have_locks_len++] = p->lock.main.lc; + else + have_not_locks[have_not_locks_len++] = p->lock.main.lc; + if (locks & ERTS_PROC_LOCK_LINK) + have_locks[have_locks_len++] = p->lock.link.lc; + else + have_not_locks[have_not_locks_len++] = p->lock.link.lc; + if (locks & ERTS_PROC_LOCK_MSGQ) + have_locks[have_locks_len++] = p->lock.msgq.lc; + else + have_not_locks[have_not_locks_len++] = p->lock.msgq.lc; + if (locks & ERTS_PROC_LOCK_STATUS) + have_locks[have_locks_len++] = p->lock.status.lc; + else + have_not_locks[have_not_locks_len++] = p->lock.status.lc; +#endif erts_lc_check(have_locks, have_locks_len, have_not_locks, have_not_locks_len); @@ -1404,6 +1636,8 @@ ErtsProcLocks erts_proc_lc_my_proc_locks(Process *p) { int resv[4]; + ErtsProcLocks res = 0; +#if ERTS_PROC_LOCK_OWN_IMPL erts_lc_lock_t locks[4] = {ERTS_LC_LOCK_INIT(lc_id.proc_lock_main, p->id, ERTS_LC_FLG_LT_PROCLOCK), @@ -1416,8 +1650,12 @@ erts_proc_lc_my_proc_locks(Process *p) ERTS_LC_LOCK_INIT(lc_id.proc_lock_status, p->id, ERTS_LC_FLG_LT_PROCLOCK)}; - - ErtsProcLocks res = 0; +#elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL + erts_lc_lock_t locks[4] = {p->lock.main.lc, + p->lock.link.lc, + p->lock.msgq.lc, + p->lock.status.lc}; +#endif erts_lc_have_locks(resv, locks, 4); if (resv[0]) @@ -1450,7 +1688,7 @@ erts_proc_lc_chk_no_proc_locks(char *file, int line) #endif /* #ifdef ERTS_ENABLE_LOCK_CHECK */ -#ifdef ERTS_PROC_LOCK_HARD_DEBUG +#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_PROC_LOCK_HARD_DEBUG) void check_queue(erts_proc_lock_t *lck) { @@ -1483,4 +1721,4 @@ check_queue(erts_proc_lock_t *lck) } #endif -#endif /* ERTS_SMP (the whole file) */ +#endif /* ERTS_SMP */ diff --git a/erts/emulator/beam/erl_process_lock.h b/erts/emulator/beam/erl_process_lock.h index 413c45480c..7367e03b44 100644 --- a/erts/emulator/beam/erl_process_lock.h +++ b/erts/emulator/beam/erl_process_lock.h @@ -37,10 +37,21 @@ #include "erl_smp.h" +#if defined(VALGRIND) || defined(ETHR_DISABLE_NATIVE_IMPLS) +# define ERTS_PROC_LOCK_OWN_IMPL 0 +#else +# define ERTS_PROC_LOCK_OWN_IMPL 1 +#endif + #define ERTS_PROC_LOCK_ATOMIC_IMPL 0 #define ERTS_PROC_LOCK_SPINLOCK_IMPL 0 #define ERTS_PROC_LOCK_MUTEX_IMPL 0 +#if !ERTS_PROC_LOCK_OWN_IMPL +#define ERTS_PROC_LOCK_RAW_MUTEX_IMPL 1 +#else +#define ERTS_PROC_LOCK_RAW_MUTEX_IMPL 0 + #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) # undef ERTS_PROC_LOCK_ATOMIC_IMPL # define ERTS_PROC_LOCK_ATOMIC_IMPL 1 @@ -52,6 +63,8 @@ # define ERTS_PROC_LOCK_MUTEX_IMPL 1 #endif +#endif + #define ERTS_PROC_LOCK_MAX_BIT 3 typedef erts_aint32_t ErtsProcLocks; @@ -59,22 +72,31 @@ typedef erts_aint32_t ErtsProcLocks; typedef struct erts_proc_lock_queues_t_ erts_proc_lock_queues_t; typedef struct erts_proc_lock_t_ { +#if ERTS_PROC_LOCK_OWN_IMPL #if ERTS_PROC_LOCK_ATOMIC_IMPL erts_smp_atomic32_t flags; #else ErtsProcLocks flags; #endif erts_proc_lock_queues_t *queues; - Sint32 refc; -#ifdef ERTS_PROC_LOCK_DEBUG - erts_smp_atomic32_t locked[ERTS_PROC_LOCK_MAX_BIT+1]; -#endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_t lcnt_main; erts_lcnt_lock_t lcnt_link; erts_lcnt_lock_t lcnt_msgq; erts_lcnt_lock_t lcnt_status; #endif +#elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL + erts_mtx_t main; + erts_mtx_t link; + erts_mtx_t msgq; + erts_mtx_t status; +#else +# error "no implementation" +#endif + erts_atomic32_t refc; +#ifdef ERTS_PROC_LOCK_DEBUG + erts_smp_atomic32_t locked[ERTS_PROC_LOCK_MAX_BIT+1]; +#endif } erts_proc_lock_t; /* Process lock flags */ @@ -107,11 +129,9 @@ typedef struct erts_proc_lock_t_ { /* * Status lock: * Protects the following fields in the process structure: - * * status - * * rstatus - * * status_flags * * pending_suspenders * * suspendee + * * ... */ #define ERTS_PROC_LOCK_STATUS (((ErtsProcLocks) 1) << ERTS_PROC_LOCK_MAX_BIT) @@ -143,14 +163,11 @@ typedef struct erts_proc_lock_t_ { * Other rules regarding process locking: * * Exiting processes: - * When changing status to P_EXITING on a process, you are required - * to take all process locks (ERTS_PROC_LOCKS_ALL). Thus, by holding - * at least one process lock (whichever one doesn't matter) you - * are guaranteed that the process won't exit until the lock you are - * holding has been released. Appart from all process locks also - * the pix lock corresponding to the process has to be held. - * At the same time as status is changed to P_EXITING, also the - * field 'is_exiting' in the process structure is set to a value != 0. + * When changing state to exiting (ERTS_PSFLG_EXITING) on a process, + * you are required to take all process locks (ERTS_PROC_LOCKS_ALL). + * Thus, by holding at least one process lock (whichever one doesn't + * matter) you are guaranteed that the process won't exit until the + * lock you are holding has been released. * * Lock order: * Process locks with low numeric values has to be locked before @@ -262,12 +279,10 @@ typedef struct { } u; } erts_pix_lock_t; -#define ERTS_PIX2PIXLOCKIX(PIX) \ - ((PIX) & ((1 << ERTS_PIX_LOCKS_BITS) - 1)) -#define ERTS_PIX2PIXLOCK(PIX) \ - (&erts_pix_locks[ERTS_PIX2PIXLOCKIX((PIX))]) #define ERTS_PID2PIXLOCK(PID) \ - ERTS_PIX2PIXLOCK(internal_pid_data((PID))) + (&erts_pix_locks[(internal_pid_data((PID)) & ((1 << ERTS_PIX_LOCKS_BITS) - 1))]) + +#if ERTS_PROC_LOCK_OWN_IMPL #if ERTS_PROC_LOCK_ATOMIC_IMPL @@ -337,11 +352,13 @@ erts_proc_lock_flags_cmpxchg(erts_proc_lock_t *lck, ErtsProcLocks new, #define ERTS_PROC_LOCK_FLGS_READ_(L) ((L)->flags) #endif /* end no opt atomic ops */ +#endif /* ERTS_PROC_LOCK_OWN_IMPL */ extern erts_pix_lock_t erts_pix_locks[ERTS_NO_OF_PIX_LOCKS]; void erts_init_proc_lock(int cpus); void erts_proc_lock_prepare_proc_lock_waiter(void); +#if ERTS_PROC_LOCK_OWN_IMPL void erts_proc_lock_failed(Process *, erts_pix_lock_t *, ErtsProcLocks, @@ -349,6 +366,7 @@ void erts_proc_lock_failed(Process *, void erts_proc_unlock_failed(Process *, erts_pix_lock_t *, ErtsProcLocks); +#endif ERTS_GLB_INLINE void erts_pix_lock(erts_pix_lock_t *); ERTS_GLB_INLINE void erts_pix_unlock(erts_pix_lock_t *); @@ -412,6 +430,7 @@ ERTS_GLB_INLINE int erts_lc_pix_lock_is_locked(erts_pix_lock_t *pixlck) ERTS_GLB_INLINE ErtsProcLocks erts_smp_proc_raw_trylock__(Process *p, ErtsProcLocks locks) { +#if ERTS_PROC_LOCK_OWN_IMPL ErtsProcLocks expct_lflgs = 0; while (1) { @@ -431,8 +450,38 @@ erts_smp_proc_raw_trylock__(Process *p, ErtsProcLocks locks) /* cmpxchg failed, try again (should be rare). */ expct_lflgs = lflgs; } -} +#elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL + + if (locks & ERTS_PROC_LOCK_MAIN) + if (erts_mtx_trylock(&p->lock.main) == EBUSY) + goto busy_main; + if (locks & ERTS_PROC_LOCK_LINK) + if (erts_mtx_trylock(&p->lock.link) == EBUSY) + goto busy_link; + if (locks & ERTS_PROC_LOCK_MSGQ) + if (erts_mtx_trylock(&p->lock.msgq) == EBUSY) + goto busy_msgq; + if (locks & ERTS_PROC_LOCK_STATUS) + if (erts_mtx_trylock(&p->lock.status) == EBUSY) + goto busy_status; + + return 0; + +busy_status: + if (locks & ERTS_PROC_LOCK_MSGQ) + erts_mtx_unlock(&p->lock.msgq); +busy_msgq: + if (locks & ERTS_PROC_LOCK_LINK) + erts_mtx_unlock(&p->lock.link); +busy_link: + if (locks & ERTS_PROC_LOCK_MAIN) + erts_mtx_unlock(&p->lock.main); +busy_main: + + return EBUSY; +#endif +} ERTS_GLB_INLINE void #ifdef ERTS_ENABLE_LOCK_COUNT @@ -446,10 +495,13 @@ erts_smp_proc_lock__(Process *p, ErtsProcLocks locks) #endif { +#if ERTS_PROC_LOCK_OWN_IMPL + ErtsProcLocks old_lflgs; #if !ERTS_PROC_LOCK_ATOMIC_IMPL erts_pix_lock(pix_lck); #endif + #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_proc_lock(&(p->lock), locks); #endif @@ -473,12 +525,14 @@ erts_smp_proc_lock__(Process *p, erts_pix_unlock(pix_lck); } #endif + #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_proc_lock_post_x(&(p->lock), locks, file, line); #endif #ifdef ERTS_ENABLE_LOCK_CHECK erts_proc_lc_lock(p, locks); #endif + #ifdef ERTS_PROC_LOCK_DEBUG erts_proc_lock_op_debug(p, locks, 1); #endif @@ -486,6 +540,22 @@ erts_smp_proc_lock__(Process *p, #if ERTS_PROC_LOCK_ATOMIC_IMPL ETHR_COMPILER_BARRIER; #endif + +#elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL + if (locks & ERTS_PROC_LOCK_MAIN) + erts_mtx_lock(&p->lock.main); + if (locks & ERTS_PROC_LOCK_LINK) + erts_mtx_lock(&p->lock.link); + if (locks & ERTS_PROC_LOCK_MSGQ) + erts_mtx_lock(&p->lock.msgq); + if (locks & ERTS_PROC_LOCK_STATUS) + erts_mtx_lock(&p->lock.status); + +#ifdef ERTS_PROC_LOCK_DEBUG + erts_proc_lock_op_debug(p, locks, 1); +#endif + +#endif } ERTS_GLB_INLINE void @@ -493,6 +563,7 @@ erts_smp_proc_unlock__(Process *p, erts_pix_lock_t *pix_lck, ErtsProcLocks locks) { +#if ERTS_PROC_LOCK_OWN_IMPL ErtsProcLocks old_lflgs; #if ERTS_PROC_LOCK_ATOMIC_IMPL @@ -557,6 +628,23 @@ erts_smp_proc_unlock__(Process *p, break; } + +#elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL + +#ifdef ERTS_PROC_LOCK_DEBUG + erts_proc_lock_op_debug(p, locks, 0); +#endif + + if (locks & ERTS_PROC_LOCK_STATUS) + erts_mtx_unlock(&p->lock.status); + if (locks & ERTS_PROC_LOCK_MSGQ) + erts_mtx_unlock(&p->lock.msgq); + if (locks & ERTS_PROC_LOCK_LINK) + erts_mtx_unlock(&p->lock.link); + if (locks & ERTS_PROC_LOCK_MAIN) + erts_mtx_unlock(&p->lock.main); +#endif + } ERTS_GLB_INLINE int @@ -564,6 +652,7 @@ erts_smp_proc_trylock__(Process *p, erts_pix_lock_t *pix_lck, ErtsProcLocks locks) { +#if ERTS_PROC_LOCK_OWN_IMPL int res; #ifdef ERTS_ENABLE_LOCK_CHECK @@ -575,6 +664,7 @@ erts_smp_proc_trylock__(Process *p, else #endif { + #if !ERTS_PROC_LOCK_ATOMIC_IMPL erts_pix_lock(pix_lck); #endif @@ -613,8 +703,18 @@ erts_smp_proc_trylock__(Process *p, #if ERTS_PROC_LOCK_ATOMIC_IMPL ETHR_COMPILER_BARRIER; #endif - return res; + +#elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL + if (erts_smp_proc_raw_trylock__(p, locks) != 0) + return EBUSY; + else { +#ifdef ERTS_PROC_LOCK_DEBUG + erts_proc_lock_op_debug(p, locks, 1); +#endif + return 0; + } +#endif } #ifdef ERTS_PROC_LOCK_DEBUG @@ -713,44 +813,36 @@ erts_smp_proc_trylock(Process *p, ErtsProcLocks locks) #endif } - ERTS_GLB_INLINE void erts_smp_proc_inc_refc(Process *p) { #ifdef ERTS_SMP - erts_pix_lock_t *pixlck = ERTS_PID2PIXLOCK(p->id); - erts_pix_lock(pixlck); - ERTS_LC_ASSERT(p->lock.refc > 0); - p->lock.refc++; - erts_pix_unlock(pixlck); +#ifdef ERTS_ENABLE_LOCK_CHECK + erts_aint32_t refc = erts_atomic32_inc_read_nob(&p->lock.refc); + ERTS_SMP_LC_ASSERT(refc > 1); +#else + erts_atomic32_inc_nob(&p->lock.refc); +#endif #endif } ERTS_GLB_INLINE void erts_smp_proc_dec_refc(Process *p) { #ifdef ERTS_SMP - Process *fp; - erts_pix_lock_t *pixlck = ERTS_PID2PIXLOCK(p->id); - erts_pix_lock(pixlck); - ERTS_LC_ASSERT(p->lock.refc > 0); - fp = --p->lock.refc == 0 ? p : NULL; - erts_pix_unlock(pixlck); - if (fp) - erts_free_proc(fp); + erts_aint32_t refc = erts_atomic32_dec_read_nob(&p->lock.refc); + ERTS_SMP_LC_ASSERT(refc >= 0); + if (refc == 0) + erts_free_proc(p); #endif } -ERTS_GLB_INLINE void erts_smp_proc_add_refc(Process *p, Sint32 refc) +ERTS_GLB_INLINE void erts_smp_proc_add_refc(Process *p, Sint32 add_refc) { #ifdef ERTS_SMP - Process *fp; - erts_pix_lock_t *pixlck = ERTS_PID2PIXLOCK(p->id); - erts_pix_lock(pixlck); - ERTS_LC_ASSERT(p->lock.refc > 0); - p->lock.refc += refc; - fp = p->lock.refc == 0 ? p : NULL; - erts_pix_unlock(pixlck); - if (fp) - erts_free_proc(fp); + erts_aint32_t refc = erts_atomic32_add_read_nob(&p->lock.refc, + (erts_aint32_t) add_refc); + ERTS_SMP_LC_ASSERT(refc >= 0); + if (refc == 0) + erts_free_proc(p); #endif } @@ -758,6 +850,7 @@ ERTS_GLB_INLINE void erts_smp_proc_add_refc(Process *p, Sint32 refc) #ifdef ERTS_SMP void erts_proc_lock_init(Process *); +void erts_proc_lock_fin(Process *); void erts_proc_safelock(Process *a_proc, ErtsProcLocks a_have_locks, ErtsProcLocks a_need_locks, @@ -790,211 +883,75 @@ extern const Process erts_proc_lock_busy; #define erts_pid2proc(PROC, HL, PID, NL) \ erts_pid2proc_opt((PROC), (HL), (PID), (NL), 0) -ERTS_GLB_INLINE Process * -erts_pid2proc_opt(Process *, ErtsProcLocks, Eterm, ErtsProcLocks, int); -#ifdef ERTS_SMP -void -erts_pid2proc_safelock(Process *c_p, - ErtsProcLocks c_p_have_locks, - Process **proc, - ErtsProcLocks need_locks, - erts_pix_lock_t *pix_lock, - int flags); -ERTS_GLB_INLINE Process *erts_pid2proc_unlocked_opt(Eterm pid, int flags); -#define erts_pid2proc_unlocked(PID) erts_pid2proc_unlocked_opt((PID), 0) -#else -#define erts_pid2proc_unlocked_opt(PID, FLGS) \ - erts_pid2proc_opt(NULL, 0, (PID), 0, FLGS) -#define erts_pid2proc_unlocked(PID) erts_pid2proc_opt(NULL, 0, (PID), 0, 0) +ERTS_GLB_INLINE Process *erts_pix2proc(int ix); +ERTS_GLB_INLINE Process *erts_proc_lookup_raw(Eterm pid); +ERTS_GLB_INLINE Process *erts_proc_lookup(Eterm pid); + +#ifndef ERTS_SMP +ERTS_GLB_INLINE #endif +Process *erts_pid2proc_opt(Process *, ErtsProcLocks, Eterm, ErtsProcLocks, int); #if ERTS_GLB_INLINE_INCL_FUNC_DEF -ERTS_GLB_INLINE Process * -#ifdef ERTS_SMP -erts_pid2proc_unlocked_opt(Eterm pid, int flags) -#else -erts_pid2proc_opt(Process *c_p_unused, - ErtsProcLocks c_p_have_locks_unused, - Eterm pid, - ErtsProcLocks pid_need_locks_unused, - int flags) -#endif +ERTS_GLB_INLINE Process *erts_pix2proc(int ix) { - Uint pix; Process *proc; + ASSERT(0 <= ix && ix < erts_proc.max); + proc = (Process *) erts_smp_atomic_read_nob(&erts_proc.tab[ix]); + return proc == ERTS_PROC_LOCK_BUSY ? NULL : proc; +} + +ERTS_GLB_INLINE Process *erts_proc_lookup_raw(Eterm pid) +{ + Process *proc; + int pix; + + /* + * In SMP case: Only scheduler threads are allowed + * to use this function. Other threads need to + * atomicaly increment refc at lookup, i.e., use + * erts_pid2proc_opt() with ERTS_P2P_FLG_SMP_INC_REFC. + */ + ERTS_SMP_LC_ASSERT(erts_get_scheduler_id()); if (is_not_internal_pid(pid)) return NULL; pix = internal_pid_index(pid); - if(pix >= erts_max_processes) + + proc = (Process *) erts_smp_atomic_read_ddrb(&erts_proc.tab[pix]); + + if (proc && proc->id != pid) return NULL; - proc = process_tab[pix]; - if (proc) { - if (proc->id != pid - || (!(flags & ERTS_P2P_FLG_ALLOW_OTHER_X) - && proc->status == P_EXITING)) - proc = NULL; - } + return proc; } -#ifdef ERTS_SMP +ERTS_GLB_INLINE Process *erts_proc_lookup(Eterm pid) +{ + Process *proc = erts_proc_lookup_raw(pid); + if (proc && ERTS_PROC_IS_EXITING(proc)) + return NULL; + return proc; +} +#ifndef ERTS_SMP ERTS_GLB_INLINE Process * -erts_pid2proc_opt(Process *c_p, - ErtsProcLocks c_p_have_locks, +erts_pid2proc_opt(Process *c_p_unused, + ErtsProcLocks c_p_have_locks_unused, Eterm pid, - ErtsProcLocks pid_need_locks, + ErtsProcLocks pid_need_locks_unused, int flags) { - erts_pix_lock_t *pix_lock; - ErtsProcLocks need_locks; - Uint pix; - Process *proc; -#ifdef ERTS_ENABLE_LOCK_COUNT - ErtsProcLocks lcnt_locks; -#endif - -#ifdef ERTS_ENABLE_LOCK_CHECK - if (c_p) { - ErtsProcLocks might_unlock = c_p_have_locks & pid_need_locks; - if (might_unlock) - erts_proc_lc_might_unlock(c_p, might_unlock); - } -#endif - if (is_not_internal_pid(pid)) { - proc = NULL; - goto done; - } - pix = internal_pid_index(pid); - if(pix >= erts_max_processes) { - proc = NULL; - goto done; - } - - ERTS_LC_ASSERT((pid_need_locks & ERTS_PROC_LOCKS_ALL) == pid_need_locks); - need_locks = pid_need_locks; - - pix_lock = ERTS_PIX2PIXLOCK(pix); - - if (c_p && c_p->id == pid) { - ASSERT(c_p->id != ERTS_INVALID_PID); - ASSERT(c_p == process_tab[pix]); - if (!(flags & ERTS_P2P_FLG_ALLOW_OTHER_X) && c_p->is_exiting) { - proc = NULL; - goto done; - } - need_locks &= ~c_p_have_locks; - if (!need_locks) { - proc = c_p; - erts_pix_lock(pix_lock); - if (flags & ERTS_P2P_FLG_SMP_INC_REFC) - proc->lock.refc++; - erts_pix_unlock(pix_lock); - goto done; - } - } - - erts_pix_lock(pix_lock); - - proc = process_tab[pix]; - if (proc) { - if (proc->id != pid || (!(flags & ERTS_P2P_FLG_ALLOW_OTHER_X) - && ERTS_PROC_IS_EXITING(proc))) { - proc = NULL; - } - else if (!need_locks) { - if (flags & ERTS_P2P_FLG_SMP_INC_REFC) - proc->lock.refc++; - } - else { - int busy; - -#ifdef ERTS_ENABLE_LOCK_COUNT - lcnt_locks = need_locks; - if (!(flags & ERTS_P2P_FLG_TRY_LOCK)) { - erts_lcnt_proc_lock(&proc->lock, need_locks); - } -#endif - -#ifdef ERTS_ENABLE_LOCK_CHECK - /* Make sure erts_pid2proc_safelock() is enough to handle - a potential lock order violation situation... */ - busy = erts_proc_lc_trylock_force_busy(proc, need_locks); - if (!busy) -#endif - { - /* Try a quick trylock to grab all the locks we need. */ - busy = (int) erts_smp_proc_raw_trylock__(proc, need_locks); -#ifdef ERTS_ENABLE_LOCK_CHECK - erts_proc_lc_trylock(proc, need_locks, !busy); -#endif -#ifdef ERTS_PROC_LOCK_DEBUG - if (!busy) - erts_proc_lock_op_debug(proc, need_locks, 1); -#endif - } - -#ifdef ERTS_ENABLE_LOCK_COUNT - if (flags & ERTS_P2P_FLG_TRY_LOCK) { - if (busy) { - erts_lcnt_proc_trylock(&proc->lock, need_locks, EBUSY); - } else { - erts_lcnt_proc_trylock(&proc->lock, need_locks, 0); - } - } -#endif - if (!busy) { - if (flags & ERTS_P2P_FLG_SMP_INC_REFC) - proc->lock.refc++; -#ifdef ERTS_ENABLE_LOCK_COUNT - /* all is great */ - if (!(flags & ERTS_P2P_FLG_TRY_LOCK)) { - erts_lcnt_proc_lock_post_x(&proc->lock, lcnt_locks, __FILE__, __LINE__); - } -#endif - } - else { - if (flags & ERTS_P2P_FLG_TRY_LOCK) - proc = ERTS_PROC_LOCK_BUSY; - else { -#ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_proc_lock_unaquire(&proc->lock, lcnt_locks); -#endif - erts_pid2proc_safelock(c_p, - c_p_have_locks, - &proc, - pid_need_locks, - pix_lock, - flags); - if (proc && (flags & ERTS_P2P_FLG_SMP_INC_REFC)) - proc->lock.refc++; - } - } - } - } - - erts_pix_unlock(pix_lock); -#ifdef ERTS_PROC_LOCK_DEBUG - ERTS_LC_ASSERT(!proc - || proc == ERTS_PROC_LOCK_BUSY - || (pid_need_locks == - (ERTS_PROC_LOCK_FLGS_READ_(&proc->lock) - & pid_need_locks))); -#endif - - - done: - -#if ERTS_PROC_LOCK_ATOMIC_IMPL - ETHR_COMPILER_BARRIER; -#endif - - return proc; + Process *proc = erts_proc_lookup_raw(pid); + return ((!(flags & ERTS_P2P_FLG_ALLOW_OTHER_X) + && proc + && ERTS_PROC_IS_EXITING(proc)) + ? NULL + : proc); } -#endif /* ERTS_SMP */ +#endif /* !ERTS_SMP */ #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h index ee47c98009..17628286bc 100644 --- a/erts/emulator/beam/erl_threads.h +++ b/erts/emulator/beam/erl_threads.h @@ -258,10 +258,6 @@ #include "sys.h" -typedef struct { SWord sint[2]; } erts_no_dw_atomic_t; -typedef SWord erts_no_atomic_t; -typedef Sint32 erts_no_atomic32_t; - #ifdef USE_THREADS #define ETHR_TRY_INLINE_FUNCS @@ -411,12 +407,15 @@ typedef struct { typedef int erts_rwmtx_t; typedef int erts_tsd_key_t; typedef int erts_tse_t; -#define erts_dw_aint_t erts_no_dw_atomic_t -#define erts_dw_atomic_t erts_no_dw_atomic_t -#define erts_aint_t SWord -#define erts_atomic_t erts_no_atomic_t -#define erts_aint32_t Sint32 -#define erts_atomic32_t erts_no_atomic32_t + +typedef struct { SWord sint[2]; } erts_dw_aint_t; +typedef SWord erts_aint_t; +typedef Sint32 erts_aint32_t; + +#define erts_dw_atomic_t erts_dw_aint_t +#define erts_atomic_t erts_aint_t +#define erts_atomic32_t erts_aint32_t + #if __GNUC__ > 2 typedef struct { } erts_spinlock_t; typedef struct { } erts_rwlock_t; @@ -425,6 +424,14 @@ typedef struct { int gcc_is_buggy; } erts_spinlock_t; typedef struct { int gcc_is_buggy; } erts_rwlock_t; #endif +#ifdef WORDS_BIGENDIAN +#define ERTS_DW_AINT_LOW_WORD 1 +#define ERTS_DW_AINT_HIGH_WORD 0 +#else +#define ERTS_DW_AINT_LOW_WORD 0 +#define ERTS_DW_AINT_HIGH_WORD 1 +#endif + #define ERTS_MTX_INITER 0 #define ERTS_CND_INITER 0 #define ERTS_THR_INIT_DATA_DEF_INITER 0 @@ -433,6 +440,10 @@ typedef struct { int gcc_is_buggy; } erts_rwlock_t; #endif /* #ifdef USE_THREADS */ +#define erts_no_dw_atomic_t erts_dw_aint_t +#define erts_no_atomic_t erts_aint_t +#define erts_no_atomic32_t erts_aint32_t + #define ERTS_AINT_NULL ((erts_aint_t) NULL) #define ERTS_AINT_T_MAX (~(((erts_aint_t) 1) << (sizeof(erts_aint_t)*8-1))) diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h index 6c6e193818..4bbdcaa3e3 100644 --- a/erts/emulator/beam/erl_time.h +++ b/erts/emulator/beam/erl_time.h @@ -118,9 +118,11 @@ ERTS_GLB_INLINE void erts_do_time_add(erts_short_time_t elapsed) void erts_get_now_cpu(Uint* megasec, Uint* sec, Uint* microsec); #endif +typedef UWord erts_approx_time_t; +erts_approx_time_t erts_get_approx_time(void); + void erts_get_timeval(SysTimeval *tv); erts_time_t erts_get_time(void); -void erts_get_emu_time(SysTimeval *); ERTS_GLB_INLINE int erts_cmp_timeval(SysTimeval *t1p, SysTimeval *t2p); diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c index 1d0735aa99..c1b4408c06 100644 --- a/erts/emulator/beam/erl_time_sup.c +++ b/erts/emulator/beam/erl_time_sup.c @@ -91,6 +91,41 @@ static SysTimeval then; /* Used in get_now */ static SysTimeval last_emu_time; /* Used in erts_get_emu_time() */ SysTimeval erts_first_emu_time; /* Used in erts_get_emu_time() */ +union { + erts_smp_atomic_t time; + char align[ERTS_CACHE_LINE_SIZE]; +} approx erts_align_attribute(ERTS_CACHE_LINE_SIZE); + +static void +init_approx_time(void) +{ + erts_smp_atomic_init_nob(&approx.time, 0); +} + +static ERTS_INLINE erts_approx_time_t +get_approx_time(void) +{ + return (erts_approx_time_t) erts_smp_atomic_read_nob(&approx.time); +} + +static ERTS_INLINE void +update_approx_time(SysTimeval *tv) +{ + erts_approx_time_t new_secs = (erts_approx_time_t) tv->tv_sec; + erts_approx_time_t old_secs = get_approx_time(); + if (old_secs != new_secs) + erts_smp_atomic_set_nob(&approx.time, new_secs); +} + +/* + * erts_get_approx_time() returns an *approximate* time + * in seconds. NOTE that this time may jump backwards!!! + */ +erts_approx_time_t +erts_get_approx_time(void) +{ + return get_approx_time(); +} #ifdef HAVE_GETHRTIME @@ -398,6 +433,8 @@ erts_init_time_sup(void) { erts_smp_mtx_init(&erts_timeofday_mtx, "timeofday"); + init_approx_time(); + last_emu_time.tv_sec = 0; last_emu_time.tv_usec = 0; @@ -417,7 +454,7 @@ erts_init_time_sup(void) gtv = inittv; then.tv_sec = then.tv_usec = 0; - erts_get_emu_time(&erts_first_emu_time); + erts_deliver_time(); return CLOCK_RESOLUTION; } @@ -873,6 +910,8 @@ get_now(Uint* megasec, Uint* sec, Uint* microsec) *megasec = (Uint) (now.tv_sec / 1000000); *sec = (Uint) (now.tv_sec % 1000000); *microsec = (Uint) (now.tv_usec); + + update_approx_time(&now); } void @@ -885,6 +924,8 @@ get_sys_now(Uint* megasec, Uint* sec, Uint* microsec) *megasec = (Uint) (now.tv_sec / 1000000); *sec = (Uint) (now.tv_sec % 1000000); *microsec = (Uint) (now.tv_usec); + + update_approx_time(&now); } @@ -901,6 +942,8 @@ void erts_deliver_time(void) { do_erts_deliver_time(&now); erts_smp_mtx_unlock(&erts_timeofday_mtx); + + update_approx_time(&now); } /* get *real* time (not ticks) remaining until next timeout - if there @@ -949,6 +992,7 @@ void erts_get_timeval(SysTimeval *tv) erts_smp_mtx_lock(&erts_timeofday_mtx); get_tolerant_timeofday(tv); erts_smp_mtx_unlock(&erts_timeofday_mtx); + update_approx_time(tv); } erts_time_t @@ -961,7 +1005,9 @@ erts_get_time(void) get_tolerant_timeofday(&sys_tv); erts_smp_mtx_unlock(&erts_timeofday_mtx); - + + update_approx_time(&sys_tv); + return sys_tv.tv_sec; } @@ -977,38 +1023,3 @@ void erts_get_now_cpu(Uint* megasec, Uint* sec, Uint* microsec) { *sec = (Uint)(tp.tv_sec % 1000000); } #endif - - -/* - * erts_get_emu_time() is similar to get_now(). You will - * always get different times from erts_get_emu_time(), but they - * may equal a time from get_now(). - * - * erts_get_emu_time() is only used internally in the emulator in - * order to order emulator internal events. - */ - -void -erts_get_emu_time(SysTimeval *this_emu_time_p) -{ - erts_smp_mtx_lock(&erts_timeofday_mtx); - - get_tolerant_timeofday(this_emu_time_p); - - /* Make sure time is later than last */ - if (last_emu_time.tv_sec > this_emu_time_p->tv_sec || - (last_emu_time.tv_sec == this_emu_time_p->tv_sec - && last_emu_time.tv_usec >= this_emu_time_p->tv_usec)) { - *this_emu_time_p = last_emu_time; - this_emu_time_p->tv_usec++; - } - /* Check for carry from above + general reasonability */ - if (this_emu_time_p->tv_usec >= 1000000) { - this_emu_time_p->tv_usec = 0; - this_emu_time_p->tv_sec++; - } - - last_emu_time = *this_emu_time_p; - - erts_smp_mtx_unlock(&erts_timeofday_mtx); -} diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index 009ca1eb52..bc988cd61b 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -64,7 +64,7 @@ int erts_cpu_timestamp; #endif static erts_smp_mtx_t smq_mtx; -static erts_smp_mtx_t sys_trace_mtx; +static erts_smp_rwmtx_t sys_trace_rwmtx; enum ErtsSysMsgType { SYS_MSG_TYPE_UNDEFINED, @@ -91,7 +91,12 @@ static void init_sys_msg_dispatcher(void); #endif void erts_init_trace(void) { - erts_smp_mtx_init(&sys_trace_mtx, "sys_tracers"); + erts_smp_rwmtx_opt_t rwmtx_opts = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; + rwmtx_opts.type = ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ; + rwmtx_opts.lived = ERTS_SMP_RWMTX_LONG_LIVED; + + erts_smp_rwmtx_init_opt(&sys_trace_rwmtx, &rwmtx_opts, "sys_tracers"); + #ifdef HAVE_ERTS_NOW_CPU erts_cpu_timestamp = 0; #endif @@ -151,8 +156,8 @@ do { (RES) = (TPID); } while(0) #define ERTS_TRACER_REF_TYPE Process * #define ERTS_GET_TRACER_REF(RES, TPID, TRACEE_FLGS) \ do { \ - (RES) = process_tab[internal_pid_index((TPID))]; \ - if (INVALID_PID((RES), (TPID)) || !((RES)->trace_flags & F_TRACER)) { \ + (RES) = erts_proc_lookup((TPID)); \ + if (!(RES) || !((RES)->trace_flags & F_TRACER)) { \ (TPID) = NIL; \ (TRACEE_FLGS) &= ~TRACEE_FLAGS; \ return; \ @@ -169,10 +174,10 @@ erts_system_profile_setup_active_schedulers(void) active_sched = erts_active_schedulers(); } -void -erts_trace_check_exiting(Eterm exiting) +static void +exiting_reset(Eterm exiting) { - erts_smp_mtx_lock(&sys_trace_mtx); + erts_smp_rwmtx_rwlock(&sys_trace_rwmtx); if (exiting == default_tracer) { default_tracer = NIL; default_trace_flags &= TRACEE_FLAGS; @@ -202,29 +207,49 @@ erts_trace_check_exiting(Eterm exiting) erts_system_profile_clear(NULL); #endif } - erts_smp_mtx_unlock(&sys_trace_mtx); + erts_smp_rwmtx_rwunlock(&sys_trace_rwmtx); +} + +void +erts_trace_check_exiting(Eterm exiting) +{ + int reset = 0; + erts_smp_rwmtx_rlock(&sys_trace_rwmtx); + if (exiting == default_tracer) + reset = 1; + else if (exiting == system_seq_tracer) + reset = 1; + else if (exiting == system_monitor) + reset = 1; + else if (exiting == system_profile) + reset = 1; + erts_smp_rwmtx_runlock(&sys_trace_rwmtx); + if (reset) + exiting_reset(exiting); +} + +static ERTS_INLINE int +is_valid_tracer(Eterm tracer) +{ + return erts_proc_lookup(tracer) || erts_is_valid_tracer_port(tracer); } Eterm erts_set_system_seq_tracer(Process *c_p, ErtsProcLocks c_p_locks, Eterm new) { - Eterm old = THE_NON_VALUE; + Eterm old; - if (new != am_false) { - if (!erts_pid2proc(c_p, c_p_locks, new, 0) - && !erts_is_valid_tracer_port(new)) { - return old; - } - } + if (new != am_false && !is_valid_tracer(new)) + return THE_NON_VALUE; - erts_smp_mtx_lock(&sys_trace_mtx); + erts_smp_rwmtx_rwlock(&sys_trace_rwmtx); old = system_seq_tracer; system_seq_tracer = new; #ifdef DEBUG_PRINTOUTS erts_fprintf(stderr, "set seq tracer new=%T old=%T\n", new, old); #endif - erts_smp_mtx_unlock(&sys_trace_mtx); + erts_smp_rwmtx_rwunlock(&sys_trace_rwmtx); return old; } @@ -232,12 +257,12 @@ Eterm erts_get_system_seq_tracer(void) { Eterm st; - erts_smp_mtx_lock(&sys_trace_mtx); + erts_smp_rwmtx_rlock(&sys_trace_rwmtx); st = system_seq_tracer; #ifdef DEBUG_PRINTOUTS erts_fprintf(stderr, "get seq tracer %T\n", st); #endif - erts_smp_mtx_unlock(&sys_trace_mtx); + erts_smp_rwmtx_runlock(&sys_trace_rwmtx); return st; } @@ -250,7 +275,7 @@ get_default_tracing(Uint *flagsp, Eterm *tracerp) if (is_nil(default_tracer)) { default_trace_flags &= ~TRACEE_FLAGS; } else if (is_internal_pid(default_tracer)) { - if (!erts_pid2proc(NULL, 0, default_tracer, 0)) { + if (!erts_proc_lookup(default_tracer)) { reset_tracer: default_trace_flags &= ~TRACEE_FLAGS; default_tracer = NIL; @@ -270,7 +295,7 @@ get_default_tracing(Uint *flagsp, Eterm *tracerp) void erts_change_default_tracing(int setflags, Uint *flagsp, Eterm *tracerp) { - erts_smp_mtx_lock(&sys_trace_mtx); + erts_smp_rwmtx_rwlock(&sys_trace_rwmtx); if (flagsp) { if (setflags) default_trace_flags |= *flagsp; @@ -280,48 +305,48 @@ erts_change_default_tracing(int setflags, Uint *flagsp, Eterm *tracerp) if (tracerp) default_tracer = *tracerp; get_default_tracing(flagsp, tracerp); - erts_smp_mtx_unlock(&sys_trace_mtx); + erts_smp_rwmtx_rwunlock(&sys_trace_rwmtx); } void erts_get_default_tracing(Uint *flagsp, Eterm *tracerp) { - erts_smp_mtx_lock(&sys_trace_mtx); + erts_smp_rwmtx_rlock(&sys_trace_rwmtx); get_default_tracing(flagsp, tracerp); - erts_smp_mtx_unlock(&sys_trace_mtx); + erts_smp_rwmtx_runlock(&sys_trace_rwmtx); } void erts_set_system_monitor(Eterm monitor) { - erts_smp_mtx_lock(&sys_trace_mtx); + erts_smp_rwmtx_rwlock(&sys_trace_rwmtx); system_monitor = monitor; - erts_smp_mtx_unlock(&sys_trace_mtx); + erts_smp_rwmtx_rwunlock(&sys_trace_rwmtx); } Eterm erts_get_system_monitor(void) { Eterm monitor; - erts_smp_mtx_lock(&sys_trace_mtx); + erts_smp_rwmtx_rlock(&sys_trace_rwmtx); monitor = system_monitor; - erts_smp_mtx_unlock(&sys_trace_mtx); + erts_smp_rwmtx_runlock(&sys_trace_rwmtx); return monitor; } /* Performance monitoring */ void erts_set_system_profile(Eterm profile) { - erts_smp_mtx_lock(&sys_trace_mtx); + erts_smp_rwmtx_rwlock(&sys_trace_rwmtx); system_profile = profile; - erts_smp_mtx_unlock(&sys_trace_mtx); + erts_smp_rwmtx_rwunlock(&sys_trace_rwmtx); } Eterm erts_get_system_profile(void) { Eterm profile; - erts_smp_mtx_lock(&sys_trace_mtx); + erts_smp_rwmtx_rlock(&sys_trace_rwmtx); profile = system_profile; - erts_smp_mtx_unlock(&sys_trace_mtx); + erts_smp_rwmtx_runlock(&sys_trace_rwmtx); return profile; } @@ -384,13 +409,9 @@ WRITE_SYS_MSG_TO_PORT(Eterm unused_to, } #ifndef ERTS_SMP - if (!INVALID_TRACER_PORT(trace_port, trace_port->id)) { + if (!INVALID_TRACER_PORT(trace_port, trace_port->id)) #endif erts_raw_port_command(trace_port, buffer, ptr-buffer); -#ifndef ERTS_SMP - erts_port_release(trace_port); - } -#endif erts_free(ERTS_ALC_T_TMP, (void *) buffer); } @@ -465,13 +486,13 @@ send_to_port(Process *c_p, Eterm message, trace_port = NULL; #else - if (is_not_internal_port(*tracer_pid)) - goto invalid_tracer_port; - trace_port = &erts_port[internal_port_index(*tracer_pid)]; + trace_port = erts_id2port_sflgs(*tracer_pid, + NULL, + 0, + ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP); - if (INVALID_TRACER_PORT(trace_port, *tracer_pid)) { - invalid_tracer_port: + if (!trace_port) { *tracee_flags &= ~TRACEE_FLAGS; *tracer_pid = NIL; return; @@ -491,6 +512,7 @@ send_to_port(Process *c_p, Eterm message, SYS_MSG_TYPE_TRACE, message); #ifndef ERTS_SMP + erts_port_release(trace_port); return; } @@ -537,6 +559,9 @@ send_to_port(Process *c_p, Eterm message, */ do_send_schedfix_to_port(trace_port, c_p->id, ts); } + + erts_port_release(trace_port); + UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); #undef LOCAL_HEAP_SIZE #endif @@ -566,23 +591,28 @@ profile_send(Eterm from, Eterm message) { Port *profiler_port = NULL; /* not smp */ - - - profiler_port = &erts_port[internal_port_index(profiler)]; - - do_send_to_port(profiler, - profiler_port, - NIL, /* or current process->id */ - SYS_MSG_TYPE_SYSPROF, - message); + + profiler_port = erts_id2port_sflgs(profiler, + NULL, + 0, + ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP); + if (profiler_port) { + do_send_to_port(profiler, + profiler_port, + NIL, /* or current process->id */ + SYS_MSG_TYPE_SYSPROF, + message); + erts_port_release(profiler_port); + } } else { ASSERT(is_internal_pid(profiler) && internal_pid_index(profiler) < erts_max_processes); - profile_p = process_tab[internal_pid_index(profiler)]; + profile_p = erts_proc_lookup(profiler); - if (INVALID_PID(profile_p, profiler)) return; + if (!profile_p) + return; sz = size_object(message); hp = erts_alloc_message_heap(sz, &bp, &off_heap, profile_p, 0); @@ -626,13 +656,11 @@ seq_trace_send_to_port(Process *c_p, trace_port = NULL; #else - if (is_not_internal_port(seq_tracer)) - goto invalid_tracer_port; - - trace_port = &erts_port[internal_port_index(seq_tracer)]; - - if (INVALID_TRACER_PORT(trace_port, seq_tracer)) { - invalid_tracer_port: + trace_port = erts_id2port_sflgs(seq_tracer, + NULL, + 0, + ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP); + if (!trace_port) { system_seq_tracer = am_false; #ifndef ERTS_SMP UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); @@ -650,6 +678,7 @@ seq_trace_send_to_port(Process *c_p, message); #ifndef ERTS_SMP + erts_port_release(trace_port); UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); return; } @@ -691,6 +720,9 @@ seq_trace_send_to_port(Process *c_p, */ do_send_schedfix_to_port(trace_port, c_p->id, ts); } + + erts_port_release(trace_port); + UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); #undef LOCAL_HEAP_SIZE #endif @@ -789,13 +821,8 @@ trace_sched_aux(Process *p, Eterm what, int never_fake_sched) ERTS_GET_TRACER_REF(tracer_ref, p->tracer_proc, p->trace_flags); } - if (ERTS_PROC_IS_EXITING(p) -#ifndef ERTS_SMP - || p->status == P_FREE -#endif - ) { + if (ERTS_PROC_IS_EXITING(p)) curr_func = 0; - } else { if (!p->current) p->current = find_function_from_pc(p->i); @@ -874,7 +901,7 @@ trace_send(Process *p, Eterm to, Eterm msg) operation = am_send; if (is_internal_pid(to)) { - if (!erts_pid2proc(p, ERTS_PROC_LOCK_MAIN, to, 0)) + if (!erts_proc_lookup(to)) goto send_to_non_existing_process; } else if(is_external_pid(to) @@ -1116,8 +1143,8 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type, #ifndef ERTS_SMP - tracer = process_tab[internal_pid_index(seq_tracer)]; - if (INVALID_PID(tracer, tracer->id)) { + tracer = erts_proc_lookup(seq_tracer); + if (!tracer) { system_seq_tracer = am_false; return; /* no need to send anything */ } @@ -2451,10 +2478,9 @@ monitor_long_gc(Process *p, Uint time) { #ifndef ERTS_SMP ASSERT(is_internal_pid(system_monitor) && internal_pid_index(system_monitor) < erts_max_processes); - monitor_p = process_tab[internal_pid_index(system_monitor)]; - if (INVALID_PID(monitor_p, system_monitor) || p == monitor_p) { + monitor_p = erts_proc_lookup(system_monitor); + if (!monitor_p || p == monitor_p) return; - } #endif hsz = 0; @@ -2527,8 +2553,8 @@ monitor_large_heap(Process *p) { #ifndef ERTS_SMP ASSERT(is_internal_pid(system_monitor) && internal_pid_index(system_monitor) < erts_max_processes); - monitor_p = process_tab[internal_pid_index(system_monitor)]; - if (INVALID_PID(monitor_p, system_monitor) || p == monitor_p) { + monitor_p = erts_proc_lookup(system_monitor); + if (monitor_p || p == monitor_p) { return; } #endif @@ -2582,10 +2608,9 @@ monitor_generic(Process *p, Eterm type, Eterm spec) { #ifndef ERTS_SMP ASSERT(is_internal_pid(system_monitor) && internal_pid_index(system_monitor) < erts_max_processes); - monitor_p = process_tab[internal_pid_index(system_monitor)]; - if (INVALID_PID(monitor_p, system_monitor) || p == monitor_p) { + monitor_p = erts_proc_lookup(system_monitor); + if (!monitor_p || p == monitor_p) return; - } #endif hp = ERTS_ALLOC_SYSMSG_HEAP(5, &bp, &off_heap, monitor_p); @@ -3149,10 +3174,10 @@ sys_msg_disp_failure(ErtsSysMsgQ *smqp, Eterm receiver) break; case SYS_MSG_TYPE_SEQTRACE: /* Reset seq_tracer if it hasn't changed */ - erts_smp_mtx_lock(&sys_trace_mtx); + erts_smp_rwmtx_rwlock(&sys_trace_rwmtx); if (system_seq_tracer == receiver) system_seq_tracer = am_false; - erts_smp_mtx_unlock(&sys_trace_mtx); + erts_smp_rwmtx_rwunlock(&sys_trace_rwmtx); break; case SYS_MSG_TYPE_SYSMON: if (receiver == NIL @@ -3407,12 +3432,12 @@ sys_msg_dispatcher_func(void *unused) goto queue_proc_msg; } else if (is_internal_port(receiver)) { - port = erts_id2port(receiver, NULL, 0); - if (INVALID_TRACER_PORT(port, receiver)) { - if (port) - erts_port_release(port); + port = erts_id2port_sflgs(receiver, + NULL, + 0, + ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP); + if (!port) goto failure; - } else { write_sys_msg_to_port(receiver, port, diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c index 6d5eae73b0..049226ab7d 100644 --- a/erts/emulator/beam/erl_unicode.c +++ b/erts/emulator/beam/erl_unicode.c @@ -77,66 +77,25 @@ void erts_init_unicode(void) { max_loop_limit = CONTEXT_REDS * LOOP_FACTOR; /* Non visual BIFs to trap to. */ - memset(&characters_to_utf8_trap_exp, 0, sizeof(Export)); - characters_to_utf8_trap_exp.address = - &characters_to_utf8_trap_exp.code[3]; - characters_to_utf8_trap_exp.code[0] = am_erlang; - characters_to_utf8_trap_exp.code[1] = - am_atom_put("characters_to_utf8_trap",23); - characters_to_utf8_trap_exp.code[2] = 3; - characters_to_utf8_trap_exp.code[3] = - (BeamInstr) em_apply_bif; - characters_to_utf8_trap_exp.code[4] = - (BeamInstr) &characters_to_utf8_trap; - - memset(&characters_to_list_trap_1_exp, 0, sizeof(Export)); - characters_to_list_trap_1_exp.address = - &characters_to_list_trap_1_exp.code[3]; - characters_to_list_trap_1_exp.code[0] = am_erlang; - characters_to_list_trap_1_exp.code[1] = - am_atom_put("characters_to_list_trap_1",25); - characters_to_list_trap_1_exp.code[2] = 3; - characters_to_list_trap_1_exp.code[3] = - (BeamInstr) em_apply_bif; - characters_to_list_trap_1_exp.code[4] = - (BeamInstr) &characters_to_list_trap_1; - - memset(&characters_to_list_trap_2_exp, 0, sizeof(Export)); - characters_to_list_trap_2_exp.address = - &characters_to_list_trap_2_exp.code[3]; - characters_to_list_trap_2_exp.code[0] = am_erlang; - characters_to_list_trap_2_exp.code[1] = - am_atom_put("characters_to_list_trap_2",25); - characters_to_list_trap_2_exp.code[2] = 3; - characters_to_list_trap_2_exp.code[3] = - (BeamInstr) em_apply_bif; - characters_to_list_trap_2_exp.code[4] = - (BeamInstr) &characters_to_list_trap_2; - - - memset(&characters_to_list_trap_3_exp, 0, sizeof(Export)); - characters_to_list_trap_3_exp.address = - &characters_to_list_trap_3_exp.code[3]; - characters_to_list_trap_3_exp.code[0] = am_erlang; - characters_to_list_trap_3_exp.code[1] = - am_atom_put("characters_to_list_trap_3",25); - characters_to_list_trap_3_exp.code[2] = 3; - characters_to_list_trap_3_exp.code[3] = - (BeamInstr) em_apply_bif; - characters_to_list_trap_3_exp.code[4] = - (BeamInstr) &characters_to_list_trap_3; - - memset(&characters_to_list_trap_4_exp, 0, sizeof(Export)); - characters_to_list_trap_4_exp.address = - &characters_to_list_trap_4_exp.code[3]; - characters_to_list_trap_4_exp.code[0] = am_erlang; - characters_to_list_trap_4_exp.code[1] = - am_atom_put("characters_to_list_trap_4",25); - characters_to_list_trap_4_exp.code[2] = 1; - characters_to_list_trap_4_exp.code[3] = - (BeamInstr) em_apply_bif; - characters_to_list_trap_4_exp.code[4] = - (BeamInstr) &characters_to_list_trap_4; + erts_init_trap_export(&characters_to_utf8_trap_exp, + am_erlang, am_atom_put("characters_to_utf8_trap",23), 3, + &characters_to_utf8_trap); + + erts_init_trap_export(&characters_to_list_trap_1_exp, + am_erlang, am_atom_put("characters_to_list_trap_1",25), 3, + &characters_to_list_trap_1); + + erts_init_trap_export(&characters_to_list_trap_2_exp, + am_erlang, am_atom_put("characters_to_list_trap_2",25), 3, + &characters_to_list_trap_2); + + erts_init_trap_export(&characters_to_list_trap_3_exp, + am_erlang, am_atom_put("characters_to_list_trap_3",25), 3, + &characters_to_list_trap_3); + + erts_init_trap_export(&characters_to_list_trap_4_exp, + am_erlang, am_atom_put("characters_to_list_trap_4",25), 1, + &characters_to_list_trap_4); c_to_b_int_trap_exportp = erts_export_put(am_unicode,am_characters_to_binary_int,2); c_to_l_int_trap_exportp = erts_export_put(am_unicode,am_characters_to_list_int,2); diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c index fb0ee99119..229641cb32 100644 --- a/erts/emulator/beam/export.c +++ b/erts/emulator/beam/export.c @@ -32,98 +32,163 @@ #define EXPORT_HASH(m,f,a) ((m)*(f)+(a)) -static IndexTable export_table; /* Not locked. */ -static Hash secondary_export_table; /* Locked. */ +#ifdef DEBUG +# define IF_DEBUG(x) x +#else +# define IF_DEBUG(x) +#endif -#include "erl_smp.h" +static IndexTable export_tables[ERTS_NUM_CODE_IX]; /* Active not locked */ -static erts_smp_rwmtx_t export_table_lock; /* Locks the secondary export table. */ +static erts_smp_atomic_t total_entries_bytes; -#define export_read_lock() erts_smp_rwmtx_rlock(&export_table_lock) -#define export_read_unlock() erts_smp_rwmtx_runlock(&export_table_lock) -#define export_write_lock() erts_smp_rwmtx_rwlock(&export_table_lock) -#define export_write_unlock() erts_smp_rwmtx_rwunlock(&export_table_lock) +#include "erl_smp.h" + +/* This lock protects the staging export table from concurrent access + * AND it protects the staging table from becoming active. + */ +erts_smp_mtx_t export_staging_lock; extern BeamInstr* em_call_error_handler; extern BeamInstr* em_call_traced_function; +struct export_entry +{ + IndexSlot slot; /* MUST BE LOCATED AT TOP OF STRUCT!!! */ + Export* ep; +}; + +/* Helper struct that brings things together in one allocation +*/ +struct export_blob +{ + Export exp; + struct export_entry entryv[ERTS_NUM_CODE_IX]; + /* Note that entryv is not indexed by "code_ix". + */ +}; + +/* Helper struct only used as template +*/ +struct export_templ +{ + struct export_entry entry; + Export exp; +}; + +static struct export_blob* entry_to_blob(struct export_entry* ee) +{ + return (struct export_blob*) + ((char*)ee->ep - offsetof(struct export_blob,exp)); +} + void export_info(int to, void *to_arg) { #ifdef ERTS_SMP int lock = !ERTS_IS_CRASH_DUMPING; if (lock) - export_read_lock(); + export_staging_lock(); #endif - index_info(to, to_arg, &export_table); - hash_info(to, to_arg, &secondary_export_table); + index_info(to, to_arg, &export_tables[erts_active_code_ix()]); + hash_info(to, to_arg, &export_tables[erts_staging_code_ix()].htable); #ifdef ERTS_SMP if (lock) - export_read_unlock(); + export_staging_unlock(); #endif } static HashValue -export_hash(Export* x) +export_hash(struct export_entry* ee) { + Export* x = ee->ep; return EXPORT_HASH(x->code[0], x->code[1], x->code[2]); } static int -export_cmp(Export* tmpl, Export* obj) +export_cmp(struct export_entry* tmpl_e, struct export_entry* obj_e) { + Export* tmpl = tmpl_e->ep; + Export* obj = obj_e->ep; return !(tmpl->code[0] == obj->code[0] && tmpl->code[1] == obj->code[1] && tmpl->code[2] == obj->code[2]); } -static Export* -export_alloc(Export* tmpl) +static struct export_entry* +export_alloc(struct export_entry* tmpl_e) { - Export* obj = (Export*) erts_alloc(ERTS_ALC_T_EXPORT, sizeof(Export)); - - obj->fake_op_func_info_for_hipe[0] = 0; - obj->fake_op_func_info_for_hipe[1] = 0; - obj->code[0] = tmpl->code[0]; - obj->code[1] = tmpl->code[1]; - obj->code[2] = tmpl->code[2]; - obj->slot.index = -1; - obj->address = obj->code+3; - obj->code[3] = (BeamInstr) em_call_error_handler; - obj->code[4] = 0; - obj->match_prog_set = NULL; - return obj; + struct export_blob* blob; + unsigned ix; + + if (tmpl_e->slot.index == -1) { /* Template, allocate blob */ + Export* tmpl = tmpl_e->ep; + Export* obj; + + blob = (struct export_blob*) erts_alloc(ERTS_ALC_T_EXPORT, sizeof(*blob)); + erts_smp_atomic_add_nob(&total_entries_bytes, sizeof(*blob)); + obj = &blob->exp; + obj->fake_op_func_info_for_hipe[0] = 0; + obj->fake_op_func_info_for_hipe[1] = 0; + obj->code[0] = tmpl->code[0]; + obj->code[1] = tmpl->code[1]; + obj->code[2] = tmpl->code[2]; + obj->code[3] = (BeamInstr) em_call_error_handler; + obj->code[4] = 0; + obj->match_prog_set = NULL; + + for (ix=0; ix<ERTS_NUM_CODE_IX; ix++) { + obj->addressv[ix] = obj->code+3; + + blob->entryv[ix].slot.index = -1; + blob->entryv[ix].ep = &blob->exp; + } + ix = 0; + } + else { /* Existing entry in another table, use free entry in blob */ + blob = entry_to_blob(tmpl_e); + for (ix = 0; blob->entryv[ix].slot.index >= 0; ix++) { + ASSERT(ix < ERTS_NUM_CODE_IX); + } + } + return &blob->entryv[ix]; } - -static void -export_free(Export* obj) +static void +export_free(struct export_entry* obj) { - erts_free(ERTS_ALC_T_EXPORT, (void*) obj); + struct export_blob* blob = entry_to_blob(obj); + int i; + obj->slot.index = -1; + for (i=0; i < ERTS_NUM_CODE_IX; i++) { + if (blob->entryv[i].slot.index >= 0) { + return; + } + } + erts_free(ERTS_ALC_T_EXPORT, blob); + erts_smp_atomic_add_nob(&total_entries_bytes, -sizeof(*blob)); } - void init_export_table(void) { HashFunctions f; - erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; - rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; - rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED; + int i; - erts_smp_rwmtx_init_opt(&export_table_lock, &rwmtx_opt, "export_tab"); + erts_smp_mtx_init(&export_staging_lock, "export_tab"); + erts_smp_atomic_init_nob(&total_entries_bytes, 0); f.hash = (H_FUN) export_hash; f.cmp = (HCMP_FUN) export_cmp; f.alloc = (HALLOC_FUN) export_alloc; f.free = (HFREE_FUN) export_free; - erts_index_init(ERTS_ALC_T_EXPORT_TABLE, &export_table, "export_list", - EXPORT_INITIAL_SIZE, EXPORT_LIMIT, f); - hash_init(ERTS_ALC_T_EXPORT_TABLE, &secondary_export_table, - "secondary_export_table", 50, f); + for (i=0; i<ERTS_NUM_CODE_IX; i++) { + erts_index_init(ERTS_ALC_T_EXPORT_TABLE, &export_tables[i], "export_list", + EXPORT_INITIAL_SIZE, EXPORT_LIMIT, f); + } } /* @@ -138,29 +203,42 @@ init_export_table(void) * called through such an export entry. * 3) This function is suitable for the implementation of erlang:apply/3. */ +extern Export* /* inline-helper */ +erts_find_export_entry(Eterm m, Eterm f, unsigned int a,ErtsCodeIndex code_ix); Export* -erts_find_export_entry(Eterm m, Eterm f, unsigned int a) +erts_find_export_entry(Eterm m, Eterm f, unsigned int a, ErtsCodeIndex code_ix) { HashValue hval = EXPORT_HASH((BeamInstr) m, (BeamInstr) f, (BeamInstr) a); int ix; HashBucket* b; - ix = hval % export_table.htable.size; - b = export_table.htable.bucket[ix]; + ix = hval % export_tables[code_ix].htable.size; + b = export_tables[code_ix].htable.bucket[ix]; /* * Note: We have inlined the code from hash.c for speed. */ while (b != (HashBucket*) 0) { - Export* ep = (Export *) b; + Export* ep = ((struct export_entry*) b)->ep; if (ep->code[0] == m && ep->code[1] == f && ep->code[2] == a) { - break; + return ep; } b = b->next; } - return (Export*)b; + return NULL; +} + +static struct export_entry* init_template(struct export_templ* templ, + Eterm m, Eterm f, unsigned a) +{ + templ->entry.ep = &templ->exp; + templ->entry.slot.index = -1; + templ->exp.code[0] = m; + templ->exp.code[1] = f; + templ->exp.code[2] = a; + return &templ->entry; } @@ -176,47 +254,41 @@ erts_find_export_entry(Eterm m, Eterm f, unsigned int a) */ Export* -erts_find_function(Eterm m, Eterm f, unsigned int a) +erts_find_function(Eterm m, Eterm f, unsigned int a, ErtsCodeIndex code_ix) { - Export e; - Export* ep; - - e.code[0] = m; - e.code[1] = f; - e.code[2] = a; + struct export_templ templ; + struct export_entry* ee; - ep = hash_get(&export_table.htable, (void*) &e); - if (ep != NULL && ep->address == ep->code+3 && - ep->code[3] != (BeamInstr) em_call_traced_function) { - ep = NULL; + ee = hash_get(&export_tables[code_ix].htable, init_template(&templ, m, f, a)); + if (ee == NULL || (ee->ep->addressv[code_ix] == ee->ep->code+3 && + ee->ep->code[3] != (BeamInstr) em_call_traced_function)) { + return NULL; } - return ep; + return ee->ep; } /* * Returns a pointer to an existing export entry for a MFA, * or creates a new one and returns the pointer. * - * This function provides unlocked write access to the main export - * table. It should only be used during start up or when - * all other threads are blocked. + * This function acts on the staging export table. It should only be used + * to load new code. */ Export* erts_export_put(Eterm mod, Eterm func, unsigned int arity) { - Export e; - int ix; + ErtsCodeIndex code_ix = erts_staging_code_ix(); + struct export_templ templ; + struct export_entry* ee; - ERTS_SMP_LC_ASSERT(erts_initialized == 0 - || erts_smp_thr_progress_is_blocking()); ASSERT(is_atom(mod)); ASSERT(is_atom(func)); - e.code[0] = mod; - e.code[1] = func; - e.code[2] = arity; - ix = index_put(&export_table, (void*) &e); - return (Export*) erts_index_lookup(&export_table, ix); + export_staging_lock(); + ee = (struct export_entry*) index_put_entry(&export_tables[code_ix], + init_template(&templ, mod, func, arity)); + export_staging_unlock(); + return ee->ep; } /* @@ -224,77 +296,117 @@ erts_export_put(Eterm mod, Eterm func, unsigned int arity) * export entry (making a call through it will cause the error_handler to * be called). * - * Stub export entries will be placed in the secondary export table. - * erts_export_consolidate() will move all stub export entries into the - * main export table (will be done the next time code is loaded). + * Stub export entries will be placed in the loader export table. */ Export* erts_export_get_or_make_stub(Eterm mod, Eterm func, unsigned int arity) { - Export e; + ErtsCodeIndex code_ix; Export* ep; + IF_DEBUG(int retrying = 0;) ASSERT(is_atom(mod)); ASSERT(is_atom(func)); - - e.code[0] = mod; - e.code[1] = func; - e.code[2] = arity; - ep = erts_find_export_entry(mod, func, arity); - if (ep == 0) { - /* - * The code is not loaded (yet). Put the export in the secondary - * export table, to avoid having to lock the main export table. - */ - export_write_lock(); - ep = (Export *) hash_put(&secondary_export_table, (void*) &e); - export_write_unlock(); - } + + do { + code_ix = erts_active_code_ix(); + ep = erts_find_export_entry(mod, func, arity, code_ix); + if (ep == 0) { + /* + * The code is not loaded (yet). Put the export in the staging + * export table, to avoid having to lock the active export table. + */ + export_staging_lock(); + if (erts_active_code_ix() == code_ix) { + struct export_templ templ; + struct export_entry* entry; + + IndexTable* tab = &export_tables[erts_staging_code_ix()]; + init_template(&templ, mod, func, arity); + entry = (struct export_entry *) index_put_entry(tab, &templ.entry); + ep = entry->ep; + ASSERT(ep); + } + else { /* race */ + ASSERT(!retrying); + IF_DEBUG(retrying = 1); + } + export_staging_unlock(); + } + } while (!ep); return ep; } -/* - * To be called before loading code (with other threads blocked). - * This function will move all export entries from the secondary - * export table into the primary. - */ -void -erts_export_consolidate(void) +Export *export_list(int i, ErtsCodeIndex code_ix) { -#ifdef DEBUG - HashInfo hi; -#endif - - ERTS_SMP_LC_ASSERT(erts_initialized == 0 - || erts_smp_thr_progress_is_blocking()); - - export_write_lock(); - erts_index_merge(&secondary_export_table, &export_table); - erts_hash_merge(&secondary_export_table, &export_table.htable); - export_write_unlock(); -#ifdef DEBUG - hash_get_info(&hi, &export_table.htable); - ASSERT(export_table.entries == hi.objs); -#endif + return ((struct export_entry*) erts_index_lookup(&export_tables[code_ix], i))->ep; } -Export *export_list(int i) +int export_list_size(ErtsCodeIndex code_ix) { - return (Export*) erts_index_lookup(&export_table, i); + return export_tables[code_ix].entries; } -int export_list_size(void) +int export_table_sz(void) { - return export_table.entries; + int i, bytes = 0; + + export_staging_lock(); + for (i=0; i<ERTS_NUM_CODE_IX; i++) { + bytes += index_table_sz(&export_tables[i]); + } + export_staging_unlock(); + return bytes; +} +int export_entries_sz(void) +{ + return erts_smp_atomic_read_nob(&total_entries_bytes); +} +Export *export_get(Export *e) +{ + struct export_entry ee; + struct export_entry* entry; + + ee.ep = e; + entry = (struct export_entry*)hash_get(&export_tables[erts_active_code_ix()].htable, &ee); + return entry ? entry->ep : NULL; } -int export_table_sz(void) +IF_DEBUG(static ErtsCodeIndex debug_start_load_ix = 0;) + + +void export_start_staging(void) { - return index_table_sz(&export_table); + ErtsCodeIndex dst_ix = erts_staging_code_ix(); + ErtsCodeIndex src_ix = erts_active_code_ix(); + IndexTable* dst = &export_tables[dst_ix]; + IndexTable* src = &export_tables[src_ix]; + struct export_entry* src_entry; + struct export_entry* dst_entry; + int i; + + ASSERT(dst_ix != src_ix); + ASSERT(debug_start_load_ix == -1); + + export_staging_lock(); + /* + * Insert all entries in src into dst table + */ + for (i = 0; i < src->entries; i++) { + src_entry = (struct export_entry*) erts_index_lookup(src, i); + src_entry->ep->addressv[dst_ix] = src_entry->ep->addressv[src_ix]; + dst_entry = (struct export_entry*) index_put_entry(dst, src_entry); + ASSERT(entry_to_blob(src_entry) == entry_to_blob(dst_entry)); + } + export_staging_unlock(); + + IF_DEBUG(debug_start_load_ix = dst_ix); } -Export *export_get(Export *e) +void export_end_staging(int commit) { - return hash_get(&export_table.htable, e); + ASSERT(debug_start_load_ix == erts_staging_code_ix()); + IF_DEBUG(debug_start_load_ix = -1); } + diff --git a/erts/emulator/beam/export.h b/erts/emulator/beam/export.h index c604fdf7c3..ec9fcb26f2 100644 --- a/erts/emulator/beam/export.h +++ b/erts/emulator/beam/export.h @@ -28,13 +28,15 @@ #include "index.h" #endif +#include "code_ix.h" + /* ** Export entry */ + typedef struct export { - IndexSlot slot; /* MUST BE LOCATED AT TOP OF STRUCT!!! */ - void* address; /* Pointer to code for function. */ + void* addressv[ERTS_NUM_CODE_IX]; /* Pointer to code for function. */ struct binary* match_prog_set; /* Match program for tracing. */ BeamInstr fake_op_func_info_for_hipe[2]; /* MUST be just before code[] */ @@ -59,21 +61,38 @@ typedef struct export void init_export_table(void); void export_info(int, void *); -Export* erts_find_export_entry(Eterm m, Eterm f, unsigned int a); +ERTS_GLB_INLINE Export* erts_active_export_entry(Eterm m, Eterm f, unsigned a); Export* erts_export_put(Eterm mod, Eterm func, unsigned int arity); - Export* erts_export_get_or_make_stub(Eterm, Eterm, unsigned); -void erts_export_consolidate(void); -Export *export_list(int); -int export_list_size(void); +Export *export_list(int,ErtsCodeIndex); +int export_list_size(ErtsCodeIndex); int export_table_sz(void); +int export_entries_sz(void); Export *export_get(Export*); +void export_start_staging(void); +void export_end_staging(int commit); + +extern erts_smp_mtx_t export_staging_lock; +#define export_staging_lock() erts_smp_mtx_lock(&export_staging_lock) +#define export_staging_unlock() erts_smp_mtx_unlock(&export_staging_lock) #include "beam_load.h" /* For em_* extern declarations */ #define ExportIsBuiltIn(EntryPtr) \ -(((EntryPtr)->address == (EntryPtr)->code + 3) && \ +(((EntryPtr)->addressv[erts_active_code_ix()] == (EntryPtr)->code + 3) && \ ((EntryPtr)->code[3] == (BeamInstr) em_apply_bif)) -#endif +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE Export* +erts_active_export_entry(Eterm m, Eterm f, unsigned int a) +{ + extern Export* erts_find_export_entry(Eterm m, Eterm f, unsigned a, ErtsCodeIndex); + return erts_find_export_entry(m, f, a, erts_active_code_ix()); +} + +#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ + +#endif /* __EXPORT_H__ */ + diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 4348578694..feb1740649 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -2562,7 +2562,7 @@ dec_term_atom_common: goto error; } if (edep && (edep->flags & ERTS_DIST_EXT_BTT_SAFE)) { - if (!erts_find_export_entry(mod, name, arity)) + if (!erts_active_export_entry(mod, name, arity)) goto error; } *objp = make_export(hp); diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 1b15c4ac3b..9e387c550f 100755 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -28,6 +28,7 @@ #include "hash.h" #include "index.h" #include "atom.h" +#include "code_ix.h" #include "export.h" #include "module.h" #include "register.h" @@ -143,8 +144,8 @@ typedef struct ErtsXPortsList_ ErtsXPortsList; struct port { ErtsPortTaskSched sched; ErtsPortTaskHandle timeout_task; -#ifdef ERTS_SMP erts_smp_atomic_t refc; +#ifdef ERTS_SMP erts_smp_mtx_t *lock; ErtsXPortsList *xports; erts_smp_atomic_t run_queue; @@ -197,6 +198,8 @@ erts_port_runq(Port *prt) #ifdef ERTS_SMP ErtsRunQueue *rq1, *rq2; rq1 = (ErtsRunQueue *) erts_smp_atomic_read_nob(&prt->run_queue); + if (!rq1) + return NULL; while (1) { erts_smp_runq_lock(rq1); rq2 = (ErtsRunQueue *) erts_smp_atomic_read_nob(&prt->run_queue); @@ -204,6 +207,8 @@ erts_port_runq(Port *prt) return rq1; erts_smp_runq_unlock(rq1); rq1 = rq2; + if (!rq1) + return NULL; } #else return ERTS_RUNQ_IX(0); @@ -764,6 +769,8 @@ void erts_queue_monitor_message(Process *, Eterm, Eterm, Eterm); +void erts_init_trap_export(Export* ep, Eterm m, Eterm f, Uint a, + Eterm (*bif)(Process*,Eterm*)); void erts_init_bif(void); Eterm erl_send(Process *p, Eterm to, Eterm msg); @@ -786,24 +793,33 @@ typedef struct { Eterm* fname_ptr; /* Pointer to fname table */ } FunctionInfo; -struct LoaderState* erts_alloc_loader_state(void); -Eterm erts_prepare_loading(struct LoaderState*, Process *c_p, +Binary* erts_alloc_loader_state(void); +Eterm erts_module_for_prepared_code(Binary* magic); +Eterm erts_prepare_loading(Binary* loader_state, Process *c_p, Eterm group_leader, Eterm* modp, byte* code, Uint size); -Eterm erts_finish_loading(struct LoaderState* stp, Process* c_p, +Eterm erts_finish_loading(Binary* loader_state, Process* c_p, ErtsProcLocks c_p_locks, Eterm* modp); -Eterm erts_load_module(Process *c_p, ErtsProcLocks c_p_locks, - Eterm group_leader, Eterm* mod, byte* code, Uint size); +Eterm erts_preload_module(Process *c_p, ErtsProcLocks c_p_locks, + Eterm group_leader, Eterm* mod, byte* code, Uint size); void init_load(void); BeamInstr* find_function_from_pc(BeamInstr* pc); Eterm* erts_build_mfa_item(FunctionInfo* fi, Eterm* hp, Eterm args, Eterm* mfa_p); -void erts_lookup_function_info(FunctionInfo* fi, BeamInstr* pc, int full_info); void erts_set_current_function(FunctionInfo* fi, BeamInstr* current); Eterm erts_module_info_0(Process* p, Eterm module); Eterm erts_module_info_1(Process* p, Eterm module, Eterm what); Eterm erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info); +/* beam_ranges.c */ +void erts_init_ranges(void); +void erts_start_staging_ranges(void); +void erts_end_staging_ranges(int commit); +void erts_update_ranges(BeamInstr* code, Uint size); +void erts_remove_from_ranges(BeamInstr* code); +UWord erts_ranges_sz(void); +void erts_lookup_function_info(FunctionInfo* fi, BeamInstr* pc, int full_info); + /* break.c */ void init_break_handler(void); void erts_set_ignore_break(void); @@ -1024,28 +1040,29 @@ erts_smp_port_state_unlock(Port *prt) ERTS_GLB_INLINE int erts_smp_port_trylock(Port *prt) { -#ifdef ERTS_SMP int res; ASSERT(erts_smp_atomic_read_nob(&prt->refc) > 0); erts_smp_atomic_inc_nob(&prt->refc); + +#ifdef ERTS_SMP res = erts_smp_mtx_trylock(prt->lock); if (res == EBUSY) { erts_smp_atomic_dec_nob(&prt->refc); } +#else + res = 0; +#endif return res; -#else /* !ERTS_SMP */ - return 0; -#endif } ERTS_GLB_INLINE void erts_smp_port_lock(Port *prt) { -#ifdef ERTS_SMP ASSERT(erts_smp_atomic_read_nob(&prt->refc) > 0); erts_smp_atomic_inc_nob(&prt->refc); +#ifdef ERTS_SMP erts_smp_mtx_lock(prt->lock); #endif } @@ -1053,14 +1070,14 @@ erts_smp_port_lock(Port *prt) ERTS_GLB_INLINE void erts_smp_port_unlock(Port *prt) { -#ifdef ERTS_SMP erts_aint_t refc; +#ifdef ERTS_SMP erts_smp_mtx_unlock(prt->lock); +#endif refc = erts_smp_atomic_dec_read_nob(&prt->refc); ASSERT(refc >= 0); if (refc == 0) erts_port_cleanup(prt); -#endif } #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ @@ -1123,12 +1140,12 @@ erts_id2port_sflgs(Eterm id, Process *c_p, ErtsProcLocks c_p_locks, Uint32 sflgs erts_smp_port_state_unlock(prt); prt = NULL; } -#ifdef ERTS_SMP else { erts_smp_atomic_inc_nob(&prt->refc); erts_smp_port_state_unlock(prt); - if (no_proc_locks) +#ifdef ERTS_SMP + if (no_proc_locks) erts_smp_mtx_lock(prt->lock); else if (erts_smp_mtx_trylock(prt->lock) == EBUSY) { /* Unlock process locks, and acquire locks in lock order... */ @@ -1144,21 +1161,17 @@ erts_id2port_sflgs(Eterm id, Process *c_p, ErtsProcLocks c_p_locks, Uint32 sflgs erts_smp_port_unlock(prt); /* Also decrements refc... */ prt = NULL; } - } #endif + } + return prt; } ERTS_GLB_INLINE void erts_port_release(Port *prt) { -#ifdef ERTS_SMP erts_smp_port_unlock(prt); -#else - if (prt->status & ERTS_PORT_SFLGS_DEAD) - erts_port_cleanup(prt); -#endif } ERTS_GLB_INLINE Port* @@ -1275,6 +1288,122 @@ void erl_drv_thr_init(void); /* utils.c */ +typedef struct { +#ifdef DEBUG + int smp_api; +#endif + union { + Uint64 not_atomic; +#ifdef ARCH_64 + erts_atomic_t atomic; +#else + erts_dw_atomic_t atomic; +#endif + } counter; +} erts_interval_t; + +void erts_interval_init(erts_interval_t *); +void erts_smp_interval_init(erts_interval_t *); +Uint64 erts_step_interval_nob(erts_interval_t *); +Uint64 erts_step_interval_relb(erts_interval_t *); +Uint64 erts_smp_step_interval_nob(erts_interval_t *); +Uint64 erts_smp_step_interval_relb(erts_interval_t *); +Uint64 erts_ensure_later_interval_nob(erts_interval_t *, Uint64); +Uint64 erts_ensure_later_interval_acqb(erts_interval_t *, Uint64); +Uint64 erts_smp_ensure_later_interval_nob(erts_interval_t *, Uint64); +Uint64 erts_smp_ensure_later_interval_acqb(erts_interval_t *, Uint64); +#ifdef ARCH_32 +ERTS_GLB_INLINE Uint64 erts_interval_dw_aint_to_val__(erts_dw_aint_t *); +#endif +ERTS_GLB_INLINE Uint64 erts_current_interval_nob__(erts_interval_t *); +ERTS_GLB_INLINE Uint64 erts_current_interval_acqb__(erts_interval_t *); +ERTS_GLB_INLINE Uint64 erts_current_interval_nob(erts_interval_t *); +ERTS_GLB_INLINE Uint64 erts_current_interval_acqb(erts_interval_t *); +ERTS_GLB_INLINE Uint64 erts_smp_current_interval_nob(erts_interval_t *); +ERTS_GLB_INLINE Uint64 erts_smp_current_interval_acqb(erts_interval_t *); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +#ifdef ARCH_32 + +ERTS_GLB_INLINE Uint64 +erts_interval_dw_aint_to_val__(erts_dw_aint_t *dw) +{ +#ifdef ETHR_SU_DW_NAINT_T__ + return (Uint64) dw->dw_sint; +#else + Uint64 res; + res = (Uint64) ((Uint32) dw->sint[ERTS_DW_AINT_HIGH_WORD]); + res <<= 32; + res |= (Uint64) ((Uint32) dw->sint[ERTS_DW_AINT_LOW_WORD]); + return res; +#endif +} + +#endif + +ERTS_GLB_INLINE Uint64 +erts_current_interval_nob__(erts_interval_t *icp) +{ +#ifdef ARCH_64 + return (Uint64) erts_atomic_read_nob(&icp->counter.atomic); +#else + erts_dw_aint_t dw; + erts_dw_atomic_read_nob(&icp->counter.atomic, &dw); + return erts_interval_dw_aint_to_val__(&dw); +#endif +} + +ERTS_GLB_INLINE Uint64 +erts_current_interval_acqb__(erts_interval_t *icp) +{ +#ifdef ARCH_64 + return (Uint64) erts_atomic_read_acqb(&icp->counter.atomic); +#else + erts_dw_aint_t dw; + erts_dw_atomic_read_acqb(&icp->counter.atomic, &dw); + return erts_interval_dw_aint_to_val__(&dw); +#endif +} + +ERTS_GLB_INLINE Uint64 +erts_current_interval_nob(erts_interval_t *icp) +{ + ASSERT(!icp->smp_api); + return erts_current_interval_nob__(icp); +} + +ERTS_GLB_INLINE Uint64 +erts_current_interval_acqb(erts_interval_t *icp) +{ + ASSERT(!icp->smp_api); + return erts_current_interval_acqb__(icp); +} + +ERTS_GLB_INLINE Uint64 +erts_smp_current_interval_nob(erts_interval_t *icp) +{ + ASSERT(icp->smp_api); +#ifdef ERTS_SMP + return erts_current_interval_nob__(icp); +#else + return icp->counter.not_atomic; +#endif +} + +ERTS_GLB_INLINE Uint64 +erts_smp_current_interval_acqb(erts_interval_t *icp) +{ + ASSERT(icp->smp_api); +#ifdef ERTS_SMP + return erts_current_interval_acqb__(icp); +#else + return icp->counter.not_atomic; +#endif +} + +#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ + /* * To be used to silence unused result warnings, but do not abuse it. */ @@ -1282,9 +1411,10 @@ void erts_silence_warn_unused_result(long unused); void erts_cleanup_offheap(ErlOffHeap *offheap); -Uint erts_fit_in_bits(Uint); +int erts_fit_in_bits_int64(Sint64); +int erts_fit_in_bits_int32(Sint32); int list_length(Eterm); -Export* erts_find_function(Eterm, Eterm, unsigned int); +Export* erts_find_function(Eterm, Eterm, unsigned int, ErtsCodeIndex); int erts_is_builtin(Eterm, Eterm, int); Uint32 make_broken_hash(Eterm); Uint32 block_hash(byte *, unsigned, Uint32); @@ -1562,6 +1692,7 @@ erts_get_default_trace_pattern(int *trace_pattern_is_on, Binary **meta_match_spec, struct trace_pattern_flags *trace_pattern_flags, Eterm *meta_tracer_pid); +int erts_is_default_trace_enabled(void); void erts_bif_trace_init(void); /* @@ -1622,6 +1753,13 @@ extern erts_driver_t spawn_driver; extern erts_driver_t fd_driver; /* Should maybe be placed in erl_message.h, but then we get an include mess. */ +ERTS_GLB_INLINE Eterm * +erts_alloc_message_heap_state(Uint size, + ErlHeapFragment **bpp, + ErlOffHeap **ohpp, + Process *receiver, + ErtsProcLocks *receiver_locks, + erts_aint32_t *statep); ERTS_GLB_INLINE Eterm * erts_alloc_message_heap(Uint size, @@ -1644,16 +1782,22 @@ erts_alloc_message_heap(Uint size, */ ERTS_GLB_INLINE Eterm * -erts_alloc_message_heap(Uint size, - ErlHeapFragment **bpp, - ErlOffHeap **ohpp, - Process *receiver, - ErtsProcLocks *receiver_locks) +erts_alloc_message_heap_state(Uint size, + ErlHeapFragment **bpp, + ErlOffHeap **ohpp, + Process *receiver, + ErtsProcLocks *receiver_locks, + erts_aint32_t *statep) { Eterm *hp; + erts_aint32_t state; #ifdef ERTS_SMP int locked_main = 0; - ErtsProcLocks ulocks = *receiver_locks & ERTS_PROC_LOCKS_MSG_SEND; + state = erts_smp_atomic32_read_acqb(&receiver->state); + if (statep) + *statep = state; + if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) + goto allocate_in_mbuf; #endif if (size > (Uint) INT_MAX) @@ -1669,20 +1813,19 @@ erts_alloc_message_heap(Uint size, #ifdef ERTS_SMP try_allocate_on_heap: #endif - if (ERTS_PROC_IS_EXITING(receiver) + state = erts_smp_atomic32_read_nob(&receiver->state); + if (statep) + *statep = state; + if ((state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) || HEAP_LIMIT(receiver) - HEAP_TOP(receiver) <= size) { #ifdef ERTS_SMP - if (locked_main) - ulocks |= ERTS_PROC_LOCK_MAIN; + if (locked_main) { + *receiver_locks &= ~ERTS_PROC_LOCK_MAIN; + erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_MAIN); + } #endif goto allocate_in_mbuf; } -#ifdef ERTS_SMP - if (ulocks) { - erts_smp_proc_unlock(receiver, ulocks); - *receiver_locks &= ~ulocks; - } -#endif hp = HEAP_TOP(receiver); HEAP_TOP(receiver) = hp + size; *bpp = NULL; @@ -1698,12 +1841,6 @@ erts_alloc_message_heap(Uint size, else { ErlHeapFragment *bp; allocate_in_mbuf: -#ifdef ERTS_SMP - if (ulocks) { - *receiver_locks &= ~ulocks; - erts_smp_proc_unlock(receiver, ulocks); - } -#endif bp = new_message_buffer(size); hp = bp->mem; *bpp = bp; @@ -1713,6 +1850,17 @@ erts_alloc_message_heap(Uint size, return hp; } +ERTS_GLB_INLINE Eterm * +erts_alloc_message_heap(Uint size, + ErlHeapFragment **bpp, + ErlOffHeap **ohpp, + Process *receiver, + ErtsProcLocks *receiver_locks) +{ + return erts_alloc_message_heap_state(size, bpp, ohpp, receiver, + receiver_locks, NULL); +} + #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ #if !HEAP_ON_C_STACK diff --git a/erts/emulator/beam/index.c b/erts/emulator/beam/index.c index a4a3007f93..25d5cce0f3 100644 --- a/erts/emulator/beam/index.c +++ b/erts/emulator/beam/index.c @@ -68,14 +68,14 @@ erts_index_init(ErtsAlcType_t type, IndexTable* t, char* name, return t; } -int -index_put(IndexTable* t, void* tmpl) +IndexSlot* +index_put_entry(IndexTable* t, void* tmpl) { int ix; IndexSlot* p = (IndexSlot*) hash_put(&t->htable, tmpl); if (p->index >= 0) { - return p->index; + return p; } ix = t->entries; @@ -92,7 +92,7 @@ index_put(IndexTable* t, void* tmpl) t->entries++; p->index = ix; t->seg_table[ix>>INDEX_PAGE_SHIFT][ix&INDEX_PAGE_MASK] = p; - return ix; + return p; } int index_get(IndexTable* t, void* tmpl) @@ -135,3 +135,18 @@ void erts_index_merge(Hash* src, IndexTable* dst) } } } + +void index_erase_latest_from(IndexTable* t, Uint from_ix) +{ + if(from_ix < (Uint)t->entries) { + int ix; + for (ix = from_ix; ix < t->entries; ix++) { + IndexSlot* obj = t->seg_table[ix>>INDEX_PAGE_SHIFT][ix&INDEX_PAGE_MASK]; + hash_erase(&t->htable, obj); + } + t->entries = from_ix; + } + else { + ASSERT(from_ix == t->entries); + } +} diff --git a/erts/emulator/beam/index.h b/erts/emulator/beam/index.h index 4eb9b1f992..3afe48d007 100644 --- a/erts/emulator/beam/index.h +++ b/erts/emulator/beam/index.h @@ -55,12 +55,24 @@ void index_info(int, void *, IndexTable*); int index_table_sz(IndexTable *); int index_get(IndexTable*, void*); -int index_put(IndexTable*, void*); + +IndexSlot* index_put_entry(IndexTable*, void*); void erts_index_merge(Hash*, IndexTable*); +/* Erase all entries with index 'ix' and higher +*/ +void index_erase_latest_from(IndexTable*, Uint ix); + +ERTS_GLB_INLINE int index_put(IndexTable*, void*); ERTS_GLB_INLINE IndexSlot* erts_index_lookup(IndexTable*, Uint); #if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE int index_put(IndexTable* t, void* tmpl) +{ + return index_put_entry(t, tmpl)->index; +} + ERTS_GLB_INLINE IndexSlot* erts_index_lookup(IndexTable* t, Uint ix) { diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index 204bff299e..845593d0b2 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -259,10 +259,8 @@ get_free_port(void) } } port->status = ERTS_PORT_SFLG_INITIALIZING; -#ifdef ERTS_SMP - ERTS_SMP_LC_ASSERT(erts_smp_atomic_read_nob(&port->refc) == 0); + ERTS_LC_ASSERT(erts_smp_atomic_read_nob(&port->refc) == 0); erts_smp_atomic_set_nob(&port->refc, 2); /* Port alive + lock */ -#endif erts_smp_port_state_unlock(port); return num & port_num_mask; } @@ -340,10 +338,14 @@ port_cleanup(Port *prt) prt->drv_ptr = NULL; ASSERT(driver); -#ifdef ERTS_SMP - ASSERT(prt->status & ERTS_PORT_SFLG_FREE_SCHEDULED); - ERTS_SMP_LC_ASSERT(erts_smp_atomic_read_nob(&prt->refc) == 0); + ERTS_LC_ASSERT(erts_smp_atomic_read_nob(&prt->refc) == 0); + + ASSERT(prt->status & ERTS_PORT_SFLG_PORT_DEBUG); + ASSERT(!(prt->status & ERTS_PORT_SFLG_FREE)); + prt->status = ERTS_PORT_SFLG_FREE; + +#ifdef ERTS_SMP port_specific = (prt->status & ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK); @@ -352,10 +354,6 @@ port_cleanup(Port *prt) prt->lock = NULL; - ASSERT(prt->status & ERTS_PORT_SFLG_PORT_DEBUG); - ASSERT(!(prt->status & ERTS_PORT_SFLG_FREE)); - prt->status = ERTS_PORT_SFLG_FREE; - erts_smp_port_state_unlock(prt); erts_smp_mtx_unlock(mtx); @@ -606,10 +604,8 @@ erts_open_driver(erts_driver_t* driver, /* Pointer to driver. */ /* Need to mark the port as free again */ erts_smp_port_state_lock(port); port->status = ERTS_PORT_SFLG_FREE; -#ifdef ERTS_SMP - ERTS_SMP_LC_ASSERT(erts_smp_atomic_read_nob(&port->refc) == 2); + ERTS_LC_ASSERT(erts_smp_atomic_read_nob(&port->refc) == 2); erts_smp_atomic_set_nob(&port->refc, 0); -#endif erts_smp_port_state_unlock(port); return -3; } @@ -1330,7 +1326,7 @@ void init_io(void) erts_max_ports = ERTS_MAX_R9_PORTS; } - port_extra_shift = erts_fit_in_bits(erts_max_ports - 1); + port_extra_shift = erts_fit_in_bits_int32(erts_max_ports - 1); port_num_mask = (1 << ports_bits) - 1; erts_port_tab_index_mask = ~(~((Uint) 0) << port_extra_shift); @@ -1354,8 +1350,8 @@ void init_io(void) for (i = 0; i < erts_max_ports; i++) { erts_port_task_init_sched(&erts_port[i].sched); -#ifdef ERTS_SMP erts_smp_atomic_init_nob(&erts_port[i].refc, 0); +#ifdef ERTS_SMP erts_port[i].lock = NULL; erts_port[i].xports = NULL; erts_smp_spinlock_init_x(&erts_port[i].state_lck, @@ -1594,6 +1590,7 @@ deliver_result(Eterm sender, Eterm pid, Eterm res) { Process *rp; ErtsProcLocks rp_locks = 0; + int scheduler = erts_get_scheduler_id() != 0; ERTS_SMP_CHK_NO_PROC_LOCKS; @@ -1601,7 +1598,9 @@ deliver_result(Eterm sender, Eterm pid, Eterm res) && is_internal_pid(pid) && internal_pid_index(pid) < erts_max_processes); - rp = erts_pid2proc_opt(NULL, 0, pid, 0, ERTS_P2P_FLG_SMP_INC_REFC); + rp = (scheduler + ? erts_proc_lookup(pid) + : erts_pid2proc_opt(NULL, 0, pid, 0, ERTS_P2P_FLG_SMP_INC_REFC)); if (rp) { Eterm tuple; @@ -1619,7 +1618,9 @@ deliver_result(Eterm sender, Eterm pid, Eterm res) #endif ); erts_smp_proc_unlock(rp, rp_locks); - erts_smp_proc_dec_refc(rp); + if (!scheduler) + erts_smp_proc_dec_refc(rp); + } } @@ -1644,6 +1645,7 @@ static void deliver_read_message(Port* prt, Eterm to, ErlHeapFragment *bp; ErlOffHeap *ohp; ErtsProcLocks rp_locks = 0; + int scheduler = erts_get_scheduler_id() != 0; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); ERTS_SMP_CHK_NO_PROC_LOCKS; @@ -1658,7 +1660,10 @@ static void deliver_read_message(Port* prt, Eterm to, need += 2*len; } - rp = erts_pid2proc_opt(NULL, 0, to, 0, ERTS_P2P_FLG_SMP_INC_REFC); + rp = (scheduler + ? erts_proc_lookup(to) + : erts_pid2proc_opt(NULL, 0, to, 0, ERTS_P2P_FLG_SMP_INC_REFC)); + if (!rp) return; @@ -1711,8 +1716,10 @@ static void deliver_read_message(Port* prt, Eterm to, , NIL #endif ); - erts_smp_proc_unlock(rp, rp_locks); - erts_smp_proc_dec_refc(rp); + if (rp_locks) + erts_smp_proc_unlock(rp, rp_locks); + if (!scheduler) + erts_smp_proc_dec_refc(rp); } /* @@ -1779,6 +1786,7 @@ deliver_vec_message(Port* prt, /* Port */ ErlHeapFragment *bp; ErlOffHeap *ohp; ErtsProcLocks rp_locks = 0; + int scheduler = erts_get_scheduler_id() != 0; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); ERTS_SMP_CHK_NO_PROC_LOCKS; @@ -1787,7 +1795,10 @@ deliver_vec_message(Port* prt, /* Port */ * Check arguments for validity. */ - rp = erts_pid2proc_opt(NULL, 0, to, 0, ERTS_P2P_FLG_SMP_INC_REFC); + + rp = (scheduler + ? erts_proc_lookup(to) + : erts_pid2proc_opt(NULL, 0, to, 0, ERTS_P2P_FLG_SMP_INC_REFC)); if (!rp) return; @@ -1869,7 +1880,8 @@ deliver_vec_message(Port* prt, /* Port */ #endif ); erts_smp_proc_unlock(rp, rp_locks); - erts_smp_proc_dec_refc(rp); + if (!scheduler) + erts_smp_proc_dec_refc(rp); } @@ -2277,9 +2289,8 @@ void erts_port_command(Process *proc, { ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND; - Process* rp = erts_pid2proc_opt(NULL, 0, - port->connected, rp_locks, - ERTS_P2P_FLG_SMP_INC_REFC); + Process* rp = erts_pid2proc(NULL, 0, + port->connected, rp_locks); if (rp) { (void) erts_send_exit_signal(NULL, port->id, @@ -2290,7 +2301,6 @@ void erts_port_command(Process *proc, NULL, 0); erts_smp_proc_unlock(rp, rp_locks); - erts_smp_proc_dec_refc(rp); } } @@ -2841,13 +2851,17 @@ void driver_report_exit(int ix, int status) ErlHeapFragment *bp = NULL; ErlOffHeap *ohp; ErtsProcLocks rp_locks = 0; + int scheduler = erts_get_scheduler_id() != 0; ERTS_SMP_CHK_NO_PROC_LOCKS; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); pid = prt->connected; ASSERT(is_internal_pid(pid)); - rp = erts_pid2proc_opt(NULL, 0, pid, 0, ERTS_P2P_FLG_SMP_INC_REFC); + + rp = (scheduler + ? erts_proc_lookup(pid) + : erts_pid2proc_opt(NULL, 0, pid, 0, ERTS_P2P_FLG_SMP_INC_REFC)); if (!rp) return; @@ -2864,7 +2878,8 @@ void driver_report_exit(int ix, int status) ); erts_smp_proc_unlock(rp, rp_locks); - erts_smp_proc_dec_refc(rp); + if (!scheduler) + erts_smp_proc_dec_refc(rp); } @@ -2995,6 +3010,7 @@ driver_deliver_term(ErlDrvPort port, ErlOffHeap *ohp; ErtsProcLocks rp_locks = 0; struct b2t_states__ b2t; + int scheduler = 1; /* Silence erroneous warning... */ init_b2t_states(&b2t); @@ -3186,7 +3202,13 @@ driver_deliver_term(ErlDrvPort port, if (res <= 0) goto done; - rp = erts_pid2proc_opt(NULL, 0, to, rp_locks, ERTS_P2P_FLG_SMP_INC_REFC); + /* + * Increase refc on proc if done from a non-scheduler thread. + */ + scheduler = erts_get_scheduler_id() != 0; + rp = (scheduler + ? erts_proc_lookup(to) + : erts_pid2proc_opt(NULL, 0, to, 0, ERTS_P2P_FLG_SMP_INC_REFC)); if (!rp) { res = 0; goto done; @@ -3438,7 +3460,8 @@ driver_deliver_term(ErlDrvPort port, if (rp) { if (rp_locks) erts_smp_proc_unlock(rp, rp_locks); - erts_smp_proc_dec_refc(rp); + if (!scheduler) + erts_smp_proc_dec_refc(rp); } #endif cleanup_b2t_states(&b2t); diff --git a/erts/emulator/beam/module.c b/erts/emulator/beam/module.c index b93b1ad09a..1ef71cda79 100644 --- a/erts/emulator/beam/module.c +++ b/erts/emulator/beam/module.c @@ -26,21 +26,32 @@ #include "global.h" #include "module.h" +#ifdef DEBUG +# define IF_DEBUG(x) x +#else +# define IF_DEBUG(x) +#endif + #define MODULE_SIZE 50 #define MODULE_LIMIT (64*1024) -static IndexTable module_table; +static IndexTable module_tables[ERTS_NUM_CODE_IX]; -/* - * SMP note: We don't need to look accesses to the module table because - * there is one only scheduler thread when we update it. +erts_smp_rwmtx_t the_old_code_rwlocks[ERTS_NUM_CODE_IX]; + +static erts_smp_atomic_t tot_module_bytes; + +/* SMP note: Active module table lookup and current module instance can be + * read without any locks. Old module instances are protected by + * "the_old_code_rwlocks" as purging is done on active module table. + * Staging table is protected by the "code_ix lock". */ #include "erl_smp.h" void module_info(int to, void *to_arg) { - index_info(to, to_arg, &module_table); + index_info(to, to_arg, &module_tables[erts_active_code_ix()]); } @@ -59,45 +70,67 @@ static int module_cmp(Module* tmpl, Module* obj) static Module* module_alloc(Module* tmpl) { Module* obj = (Module*) erts_alloc(ERTS_ALC_T_MODULE, sizeof(Module)); + erts_smp_atomic_add_nob(&tot_module_bytes, sizeof(Module)); obj->module = tmpl->module; - obj->code = 0; - obj->old_code = 0; - obj->code_length = 0; - obj->old_code_length = 0; + obj->curr.code = 0; + obj->old.code = 0; + obj->curr.code_length = 0; + obj->old.code_length = 0; obj->slot.index = -1; - obj->nif = NULL; - obj->old_nif = NULL; + obj->curr.nif = NULL; + obj->old.nif = NULL; + obj->curr.num_breakpoints = 0; + obj->old.num_breakpoints = 0; + obj->curr.num_traced_exports = 0; + obj->old.num_traced_exports = 0; return obj; } +static void module_free(Module* mod) +{ + erts_free(ERTS_ALC_T_MODULE, mod); + erts_smp_atomic_add_nob(&tot_module_bytes, -sizeof(Module)); +} void init_module_table(void) { HashFunctions f; + int i; f.hash = (H_FUN) module_hash; f.cmp = (HCMP_FUN) module_cmp; f.alloc = (HALLOC_FUN) module_alloc; - f.free = 0; + f.free = (HFREE_FUN) module_free; + + for (i = 0; i < ERTS_NUM_CODE_IX; i++) { + erts_index_init(ERTS_ALC_T_MODULE_TABLE, &module_tables[i], "module_code", + MODULE_SIZE, MODULE_LIMIT, f); + } - erts_index_init(ERTS_ALC_T_MODULE_TABLE, &module_table, "module_code", - MODULE_SIZE, MODULE_LIMIT, f); + for (i=0; i<ERTS_NUM_CODE_IX; i++) { + erts_smp_rwmtx_init_x(&the_old_code_rwlocks[i], "old_code", make_small(i)); + } + erts_smp_atomic_init_nob(&tot_module_bytes, 0); } Module* -erts_get_module(Eterm mod) +erts_get_module(Eterm mod, ErtsCodeIndex code_ix) { Module e; int index; + IndexTable* mod_tab; ASSERT(is_atom(mod)); + + mod_tab = &module_tables[code_ix]; + e.module = atom_val(mod); - index = index_get(&module_table, (void*) &e); + index = index_get(mod_tab, (void*) &e); if (index == -1) { return NULL; } else { - return (Module*) erts_index_lookup(&module_table, index); + return (Module*) erts_index_lookup(mod_tab, index); } } @@ -105,27 +138,101 @@ Module* erts_put_module(Eterm mod) { Module e; - int index; + IndexTable* mod_tab; + int oldsz, newsz; + Module* res; ASSERT(is_atom(mod)); ERTS_SMP_LC_ASSERT(erts_initialized == 0 - || erts_smp_thr_progress_is_blocking()); + || erts_is_code_ix_locked()); + + mod_tab = &module_tables[erts_staging_code_ix()]; e.module = atom_val(mod); - index = index_put(&module_table, (void*) &e); - return (Module*) erts_index_lookup(&module_table, index); + oldsz = index_table_sz(mod_tab); + res = (Module*) index_put_entry(mod_tab, (void*) &e); + newsz = index_table_sz(mod_tab); + erts_smp_atomic_add_nob(&tot_module_bytes, (newsz - oldsz)); + return res; } -Module *module_code(int i) +Module *module_code(int i, ErtsCodeIndex code_ix) { - return (Module*) erts_index_lookup(&module_table, i); + return (Module*) erts_index_lookup(&module_tables[code_ix], i); } -int module_code_size(void) +int module_code_size(ErtsCodeIndex code_ix) { - return module_table.entries; + return module_tables[code_ix].entries; } int module_table_sz(void) { - return index_table_sz(&module_table); + return erts_smp_atomic_read_nob(&tot_module_bytes); +} + +#ifdef DEBUG +static ErtsCodeIndex dbg_load_code_ix = 0; +#endif + +static int entries_at_start_staging = 0; + +void module_start_staging(void) +{ + IndexTable* src = &module_tables[erts_active_code_ix()]; + IndexTable* dst = &module_tables[erts_staging_code_ix()]; + Module* src_mod; + Module* dst_mod; + int i, oldsz, newsz; + + ASSERT(dbg_load_code_ix == -1); + ASSERT(dst->entries <= src->entries); + + /* + * Make sure our existing modules are up-to-date + */ + for (i = 0; i < dst->entries; i++) { + src_mod = (Module*) erts_index_lookup(src, i); + dst_mod = (Module*) erts_index_lookup(dst, i); + ASSERT(src_mod->module == dst_mod->module); + + dst_mod->curr = src_mod->curr; + dst_mod->old = src_mod->old; + } + + /* + * Copy all new modules from active table + */ + oldsz = index_table_sz(dst); + for (i = dst->entries; i < src->entries; i++) { + src_mod = (Module*) erts_index_lookup(src, i); + dst_mod = (Module*) index_put_entry(dst, src_mod); + ASSERT(dst_mod != src_mod); + + dst_mod->curr = src_mod->curr; + dst_mod->old = src_mod->old; + } + newsz = index_table_sz(dst); + erts_smp_atomic_add_nob(&tot_module_bytes, (newsz - oldsz)); + + entries_at_start_staging = dst->entries; + IF_DEBUG(dbg_load_code_ix = erts_staging_code_ix()); +} + +void module_end_staging(int commit) +{ + ASSERT(dbg_load_code_ix == erts_staging_code_ix()); + + if (!commit) { /* abort */ + IndexTable* tab = &module_tables[erts_staging_code_ix()]; + int oldsz, newsz; + + ASSERT(entries_at_start_staging <= tab->entries); + oldsz = index_table_sz(tab); + index_erase_latest_from(tab, entries_at_start_staging); + newsz = index_table_sz(tab); + erts_smp_atomic_add_nob(&tot_module_bytes, (newsz - oldsz)); + } + + IF_DEBUG(dbg_load_code_ix = -1); } + diff --git a/erts/emulator/beam/module.h b/erts/emulator/beam/module.h index 694e4ab72f..6e123a0ffe 100644 --- a/erts/emulator/beam/module.h +++ b/erts/emulator/beam/module.h @@ -24,28 +24,72 @@ #include "index.h" #endif +struct erl_module_instance { + BeamInstr* code; + int code_length; /* Length of loaded code in bytes. */ + unsigned catches; + struct erl_module_nif* nif; + int num_breakpoints; + int num_traced_exports; +}; typedef struct erl_module { IndexSlot slot; /* Must be located at top of struct! */ int module; /* Atom index for module (not tagged). */ - BeamInstr* code; - BeamInstr* old_code; - int code_length; /* Length of loaded code in bytes. */ - int old_code_length; /* Length of old loaded code in bytes */ - unsigned catches, old_catches; - struct erl_module_nif* nif; - struct erl_module_nif* old_nif; + struct erl_module_instance curr; + struct erl_module_instance old; /* protected by "old_code" rwlock */ } Module; -Module* erts_get_module(Eterm mod); +Module* erts_get_module(Eterm mod, ErtsCodeIndex code_ix); Module* erts_put_module(Eterm mod); void init_module_table(void); +void module_start_staging(void); +void module_end_staging(int commit); void module_info(int, void *); -Module *module_code(int); -int module_code_size(void); +Module *module_code(int, ErtsCodeIndex); +int module_code_size(ErtsCodeIndex); int module_table_sz(void); +ERTS_GLB_INLINE void erts_rwlock_old_code(ErtsCodeIndex); +ERTS_GLB_INLINE void erts_rwunlock_old_code(ErtsCodeIndex); +ERTS_GLB_INLINE void erts_rlock_old_code(ErtsCodeIndex); +ERTS_GLB_INLINE void erts_runlock_old_code(ErtsCodeIndex); +#ifdef ERTS_ENABLE_LOCK_CHECK +int erts_is_old_code_rlocked(ErtsCodeIndex); +#endif + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +extern erts_smp_rwmtx_t the_old_code_rwlocks[ERTS_NUM_CODE_IX]; + +ERTS_GLB_INLINE void erts_rwlock_old_code(ErtsCodeIndex code_ix) +{ + erts_smp_rwmtx_rwlock(&the_old_code_rwlocks[code_ix]); +} +ERTS_GLB_INLINE void erts_rwunlock_old_code(ErtsCodeIndex code_ix) +{ + erts_smp_rwmtx_rwunlock(&the_old_code_rwlocks[code_ix]); +} +ERTS_GLB_INLINE void erts_rlock_old_code(ErtsCodeIndex code_ix) +{ + erts_smp_rwmtx_rlock(&the_old_code_rwlocks[code_ix]); +} +ERTS_GLB_INLINE void erts_runlock_old_code(ErtsCodeIndex code_ix) +{ + erts_smp_rwmtx_runlock(&the_old_code_rwlocks[code_ix]); +} + +#ifdef ERTS_ENABLE_LOCK_CHECK +ERTS_GLB_INLINE int erts_is_old_code_rlocked(ErtsCodeIndex code_ix) +{ + return erts_smp_lc_rwmtx_is_rlocked(&the_old_code_rwlocks[code_ix]); +} #endif + +#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ + + +#endif /* !__MODULE_H__ */ diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab index 9b168889dd..c58b36231c 100644 --- a/erts/emulator/beam/ops.tab +++ b/erts/emulator/beam/ops.tab @@ -829,16 +829,20 @@ call_ext_only Ar=u==2 Bif=u$bif:erlang:load_nif/2 => allocate u Ar | i_call_ext # -# The apply/2 and apply/3 BIFs are instructions. +# apply/2 is an instruction, not a BIF. # call_ext u==2 u$func:erlang:apply/2 => i_apply_fun call_ext_last u==2 u$func:erlang:apply/2 D => i_apply_fun_last D call_ext_only u==2 u$func:erlang:apply/2 => i_apply_fun_only -call_ext u==3 u$func:erlang:apply/3 => i_apply -call_ext_last u==3 u$func:erlang:apply/3 D => i_apply_last D -call_ext_only u==3 u$func:erlang:apply/3 => i_apply_only +# +# The apply/3 BIF is an instruction. +# + +call_ext u==3 u$bif:erlang:apply/3 => i_apply +call_ext_last u==3 u$bif:erlang:apply/3 D => i_apply_last D +call_ext_only u==3 u$bif:erlang:apply/3 => i_apply_only # # The exit/1 and throw/1 BIFs never execute the instruction following them; diff --git a/erts/emulator/beam/register.c b/erts/emulator/beam/register.c index 26d64887d0..c02872ef80 100644 --- a/erts/emulator/beam/register.c +++ b/erts/emulator/beam/register.c @@ -93,19 +93,14 @@ reg_safe_write_lock(Process *c_p, ErtsProcLocks *c_p_locks) reg_write_lock(); } +#endif + static ERTS_INLINE int is_proc_alive(Process *p) { - int res; - erts_pix_lock_t *pixlck = ERTS_PID2PIXLOCK(p->id); - erts_pix_lock(pixlck); - res = !p->is_exiting; - erts_pix_unlock(pixlck); - return res; + return !ERTS_PROC_IS_EXITING(p); } -#endif - void register_info(int to, void *to_arg) { int lock = !ERTS_IS_CRASH_DUMPING; @@ -389,8 +384,7 @@ erts_whereis_name(Process *c_p, } #else if (rp->p - && ((flags & ERTS_P2P_FLG_ALLOW_OTHER_X) - || rp->p->status != P_EXITING)) + && ((flags & ERTS_P2P_FLG_ALLOW_OTHER_X) || is_proc_alive(rp->p))) *proc = rp->p; else *proc = NULL; @@ -402,7 +396,9 @@ erts_whereis_name(Process *c_p, if (!rp || !rp->pt) *port = NULL; else { -#ifdef ERTS_SMP +#ifndef ERTS_SMP + erts_smp_atomic_inc_nob(&rp->pt->refc); +#else if (pending_port == rp->pt) pending_port = NULL; else { @@ -509,9 +505,11 @@ int erts_unregister_name(Process *c_p, if ((rp = (RegProc*) hash_get(&process_reg, (void*) &r)) != NULL) { if (rp->pt) { if (port != rp->pt) { -#ifdef ERTS_SMP +#ifndef ERTS_SMP + erts_smp_atomic_inc_nob(&rp->pt->refc); +#else if (port) { - ERTS_SMP_LC_ASSERT(port != c_prt); + ASSERT(port != c_prt); erts_smp_port_unlock(port); port = NULL; } diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index 7b2bb81f62..0d3b910278 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -39,6 +39,8 @@ #define ENABLE_CHILD_WAITER_THREAD 1 #endif +#define ERTS_I64_LITERAL(X) X##LL + #if defined (__WIN32__) # include "erl_win_sys.h" #elif defined (VXWORKS) @@ -91,14 +93,22 @@ typedef ERTS_SYS_FD_TYPE ErtsSysFdType; # endif #endif -#ifdef __GNUC__ -# if __GNUC__ < 3 && (__GNUC__ != 2 || __GNUC_MINOR__ < 96) -# define ERTS_LIKELY(BOOL) (BOOL) -# define ERTS_UNLIKELY(BOOL) (BOOL) -# else -# define ERTS_LIKELY(BOOL) __builtin_expect((BOOL), !0) -# define ERTS_UNLIKELY(BOOL) __builtin_expect((BOOL), 0) -# endif +#if !defined(__GNUC__) +# define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) 0 +#elif !defined(__GNUC_MINOR__) +# define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \ + ((__GNUC__ << 24) >= (((MAJ) << 24) | ((MIN) << 12) | (PL))) +#elif !defined(__GNUC_PATCHLEVEL__) +# define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \ + (((__GNUC__ << 24) | (__GNUC_MINOR__ << 12)) >= (((MAJ) << 24) | ((MIN) << 12) | (PL))) +#else +# define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \ + (((__GNUC__ << 24) | (__GNUC_MINOR__ << 12) | __GNUC_PATCHLEVEL__) >= (((MAJ) << 24) | ((MIN) << 12) | (PL))) +#endif + +#if ERTS_AT_LEAST_GCC_VSN__(2, 96, 0) +# define ERTS_LIKELY(BOOL) __builtin_expect((BOOL), !0) +# define ERTS_UNLIKELY(BOOL) __builtin_expect((BOOL), 0) #else # define ERTS_LIKELY(BOOL) (BOOL) # define ERTS_UNLIKELY(BOOL) (BOOL) @@ -178,6 +188,18 @@ int real_printf(const char *fmt, ...); # define printf real_printf #endif +#undef __deprecated +#if ERTS_AT_LEAST_GCC_VSN__(3, 0, 0) +# define __deprecated __attribute__((deprecated)) +#else +# define __deprecated +#endif +#if ERTS_AT_LEAST_GCC_VSN__(3, 0, 4) +# define erts_align_attribute(SZ) __attribute__ ((aligned (SZ))) +#else +# define erts_align_attribute(SZ) +#endif + /* In VC++, noreturn is a declspec that has to be before the types, * but in GNUC it is an att ribute to be placed between return type * and function name, hence __decl_noreturn <types> __noreturn <function name> @@ -185,12 +207,6 @@ int real_printf(const char *fmt, ...); #if __GNUC__ # define __decl_noreturn # define __noreturn __attribute__((noreturn)) -# undef __deprecated -# if __GNUC__ >= 3 -# define __deprecated __attribute__((deprecated)) -# else -# define __deprecated -# endif #else # if defined(__WIN32__) && defined(_MSC_VER) # define __noreturn @@ -199,7 +215,6 @@ int real_printf(const char *fmt, ...); # define __noreturn # define __decl_noreturn # endif -# define __deprecated #endif /* diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index a36d15204e..db6597dc7c 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -268,16 +268,42 @@ list_length(Eterm list) return i; } -Uint erts_fit_in_bits(Uint n) +static const struct { + Sint64 mask; + int bits; +} fib_data[] = {{ERTS_I64_LITERAL(0x2), 1}, + {ERTS_I64_LITERAL(0xc), 2}, + {ERTS_I64_LITERAL(0xf0), 4}, + {ERTS_I64_LITERAL(0xff00), 8}, + {ERTS_I64_LITERAL(0xffff0000), 16}, + {ERTS_I64_LITERAL(0xffffffff00000000), 32}}; + +static ERTS_INLINE int +fit_in_bits(Sint64 value, int start) { - Uint i; + int bits = 0; + int i; - i = 0; - while (n > 0) { - i++; - n >>= 1; - } - return i; + for (i = start; i >= 0; i--) { + if (value & fib_data[i].mask) { + value >>= fib_data[i].bits; + bits |= fib_data[i].bits; + } + } + + bits++; + + return bits; +} + +int erts_fit_in_bits_int64(Sint64 value) +{ + return fit_in_bits(value, 5); +} + +int erts_fit_in_bits_int32(Sint32 value) +{ + return fit_in_bits((Sint64) (Uint32) value, 4); } int @@ -1640,12 +1666,20 @@ static int do_send_to_logger(Eterm tag, Eterm gleader, char *buf, int len) } #ifndef ERTS_SMP - if ( #ifdef USE_THREADS - !erts_get_scheduler_data() || /* Must be scheduler thread */ + p = NULL; + if (erts_get_scheduler_data()) /* Must be scheduler thread */ #endif - (p = erts_whereis_process(NULL, 0, am_error_logger, 0, 0)) == NULL - || p->status == P_RUNNING) { + { + p = erts_whereis_process(NULL, 0, am_error_logger, 0, 0); + if (p) { + erts_aint32_t state = erts_smp_atomic32_read_acqb(&p->state); + if (state & ERTS_PSFLG_RUNNING) + p = NULL; + } + } + + if (!p) { /* buf *always* points to a null terminated string */ erts_fprintf(stderr, "(no error logger present) %T: \"%s\"\n", tag, buf); @@ -3240,7 +3274,7 @@ ptimer_timeout(ErtsSmpPTimer *ptimer) ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS, ERTS_P2P_FLG_ALLOW_OTHER_X); if (p) { - if (!p->is_exiting + if (!ERTS_PROC_IS_EXITING(p) && !(ptimer->timer.flags & ERTS_PTMR_FLG_CANCELLED)) { ASSERT(*ptimer->timer.timer_ref == ptimer); *ptimer->timer.timer_ref = NULL; @@ -3458,6 +3492,254 @@ void erts_silence_warn_unused_result(long unused) } +/* + * Interval counts + */ +void +erts_interval_init(erts_interval_t *icp) +{ +#ifdef ARCH_64 + erts_atomic_init_nob(&icp->counter.atomic, 0); +#else + erts_dw_aint_t dw; +#ifdef ETHR_SU_DW_NAINT_T__ + dw.dw_sint = 0; +#else + dw.sint[ERTS_DW_AINT_HIGH_WORD] = 0; + dw.sint[ERTS_DW_AINT_LOW_WORD] = 0; +#endif + erts_dw_atomic_init_nob(&icp->counter.atomic, &dw); + +#endif +#ifdef DEBUG + icp->smp_api = 0; +#endif +} + +void +erts_smp_interval_init(erts_interval_t *icp) +{ +#ifdef ERTS_SMP + erts_interval_init(icp); +#else + icp->counter.not_atomic = 0; +#endif +#ifdef DEBUG + icp->smp_api = 1; +#endif +} + +static ERTS_INLINE Uint64 +step_interval_nob(erts_interval_t *icp) +{ +#ifdef ARCH_64 + return (Uint64) erts_atomic_inc_read_nob(&icp->counter.atomic); +#else + erts_dw_aint_t exp; + + erts_dw_atomic_read_nob(&icp->counter.atomic, &exp); + while (1) { + erts_dw_aint_t new = exp; + +#ifdef ETHR_SU_DW_NAINT_T__ + new.dw_sint++; +#else + new.sint[ERTS_DW_AINT_LOW_WORD]++; + if (new.sint[ERTS_DW_AINT_LOW_WORD] == 0) + new.sint[ERTS_DW_AINT_HIGH_WORD]++; +#endif + + if (erts_dw_atomic_cmpxchg_nob(&icp->counter.atomic, &new, &exp)) + return erts_interval_dw_aint_to_val__(&new); + + } +#endif +} + +static ERTS_INLINE Uint64 +step_interval_relb(erts_interval_t *icp) +{ +#ifdef ARCH_64 + return (Uint64) erts_atomic_inc_read_relb(&icp->counter.atomic); +#else + erts_dw_aint_t exp; + + erts_dw_atomic_read_nob(&icp->counter.atomic, &exp); + while (1) { + erts_dw_aint_t new = exp; + +#ifdef ETHR_SU_DW_NAINT_T__ + new.dw_sint++; +#else + new.sint[ERTS_DW_AINT_LOW_WORD]++; + if (new.sint[ERTS_DW_AINT_LOW_WORD] == 0) + new.sint[ERTS_DW_AINT_HIGH_WORD]++; +#endif + + if (erts_dw_atomic_cmpxchg_relb(&icp->counter.atomic, &new, &exp)) + return erts_interval_dw_aint_to_val__(&new); + + } +#endif +} + + +static ERTS_INLINE Uint64 +ensure_later_interval_nob(erts_interval_t *icp, Uint64 ic) +{ + Uint64 curr_ic; +#ifdef ARCH_64 + curr_ic = (Uint64) erts_atomic_read_nob(&icp->counter.atomic); + if (curr_ic > ic) + return curr_ic; + return (Uint64) erts_atomic_inc_read_nob(&icp->counter.atomic); +#else + erts_dw_aint_t exp; + + erts_dw_atomic_read_nob(&icp->counter.atomic, &exp); + curr_ic = erts_interval_dw_aint_to_val__(&exp); + if (curr_ic > ic) + return curr_ic; + + while (1) { + erts_dw_aint_t new = exp; + +#ifdef ETHR_SU_DW_NAINT_T__ + new.dw_sint++; +#else + new.sint[ERTS_DW_AINT_LOW_WORD]++; + if (new.sint[ERTS_DW_AINT_LOW_WORD] == 0) + new.sint[ERTS_DW_AINT_HIGH_WORD]++; +#endif + + if (erts_dw_atomic_cmpxchg_nob(&icp->counter.atomic, &new, &exp)) + return erts_interval_dw_aint_to_val__(&new); + + curr_ic = erts_interval_dw_aint_to_val__(&exp); + if (curr_ic > ic) + return curr_ic; + } +#endif +} + + +static ERTS_INLINE Uint64 +ensure_later_interval_acqb(erts_interval_t *icp, Uint64 ic) +{ + Uint64 curr_ic; +#ifdef ARCH_64 + curr_ic = (Uint64) erts_atomic_read_acqb(&icp->counter.atomic); + if (curr_ic > ic) + return curr_ic; + return (Uint64) erts_atomic_inc_read_acqb(&icp->counter.atomic); +#else + erts_dw_aint_t exp; + + erts_dw_atomic_read_acqb(&icp->counter.atomic, &exp); + curr_ic = erts_interval_dw_aint_to_val__(&exp); + if (curr_ic > ic) + return curr_ic; + + while (1) { + erts_dw_aint_t new = exp; + +#ifdef ETHR_SU_DW_NAINT_T__ + new.dw_sint++; +#else + new.sint[ERTS_DW_AINT_LOW_WORD]++; + if (new.sint[ERTS_DW_AINT_LOW_WORD] == 0) + new.sint[ERTS_DW_AINT_HIGH_WORD]++; +#endif + + if (erts_dw_atomic_cmpxchg_acqb(&icp->counter.atomic, &new, &exp)) + return erts_interval_dw_aint_to_val__(&new); + + curr_ic = erts_interval_dw_aint_to_val__(&exp); + if (curr_ic > ic) + return curr_ic; + } +#endif +} + +Uint64 +erts_step_interval_nob(erts_interval_t *icp) +{ + ASSERT(!icp->smp_api); + return step_interval_nob(icp); +} + +Uint64 +erts_step_interval_relb(erts_interval_t *icp) +{ + ASSERT(!icp->smp_api); + return step_interval_relb(icp); +} + +Uint64 +erts_smp_step_interval_nob(erts_interval_t *icp) +{ + ASSERT(icp->smp_api); +#ifdef ERTS_SMP + return step_interval_nob(icp); +#else + return ++icp->counter.not_atomic; +#endif +} + +Uint64 +erts_smp_step_interval_relb(erts_interval_t *icp) +{ + ASSERT(icp->smp_api); +#ifdef ERTS_SMP + return step_interval_relb(icp); +#else + return ++icp->counter.not_atomic; +#endif +} + +Uint64 +erts_ensure_later_interval_nob(erts_interval_t *icp, Uint64 ic) +{ + ASSERT(!icp->smp_api); + return ensure_later_interval_nob(icp, ic); +} + +Uint64 +erts_ensure_later_interval_acqb(erts_interval_t *icp, Uint64 ic) +{ + ASSERT(!icp->smp_api); + return ensure_later_interval_acqb(icp, ic); +} + +Uint64 +erts_smp_ensure_later_interval_nob(erts_interval_t *icp, Uint64 ic) +{ + ASSERT(icp->smp_api); +#ifdef ERTS_SMP + return ensure_later_interval_nob(icp, ic); +#else + if (icp->counter.not_atomic > ic) + return icp->counter.not_atomic; + else + return ++icp->counter.not_atomic; +#endif +} + +Uint64 +erts_smp_ensure_later_interval_acqb(erts_interval_t *icp, Uint64 ic) +{ + ASSERT(icp->smp_api); +#ifdef ERTS_SMP + return ensure_later_interval_acqb(icp, ic); +#else + if (icp->counter.not_atomic > ic) + return icp->counter.not_atomic; + else + return ++icp->counter.not_atomic; +#endif +} + + #ifdef DEBUG /* * Handy functions when using a debugger - don't use in the code! @@ -3490,7 +3772,7 @@ Process *p; void ppi(Eterm pid) { - pp(erts_pid2proc_unlocked(pid)); + pp(erts_proc_lookup(pid)); } void td(Eterm x) |