From dc2b0ecba50cbe6b2f2321b1f24579a0353ced18 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 15 Feb 2012 15:12:18 +0100 Subject: erts: Activate staged code in a thread safe way Activation of staged code is scheduled for a later moment when all schedulers have done a full memory barrier. This allow them to read active code index while executing without any memory barriers at all. --- erts/emulator/beam/beam_bif_load.c | 95 +++++++++++++++++++++++++------------- erts/emulator/beam/code_ix.c | 27 ++++++----- erts/emulator/beam/code_ix.h | 23 +++++---- erts/emulator/beam/erl_init.c | 3 +- erts/emulator/beam/erl_process.c | 53 ++++++++++++++++++++- erts/emulator/beam/erl_process.h | 10 ++++ 6 files changed, 154 insertions(+), 57 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 1227af8cbb..6df5774fdb 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -35,6 +35,7 @@ #include "erl_nif.h" #include "erl_thr_progress.h" +static Eterm staging_epilogue(Process* c_p, int, Eterm res, int); static void set_default_trace_pattern(Eterm module); static Eterm check_process_code(Process* rp, Module* modp); static void delete_code(Module* modp); @@ -86,7 +87,8 @@ BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3) res = erts_make_stub_module(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); if (res == BIF_ARG_1) { - erts_commit_staging_code_ix(); + erts_end_staging_code_ix(); + erts_activate_staging_code_ix(); } else { erts_abort_staging_code_ix(); @@ -166,6 +168,7 @@ finish_loading_1(BIF_ALIST_1) Uint exceptions; Eterm res; int is_blocking = 0; + int do_commit = 0; if (!erts_try_lock_code_ix(BIF_P)) { ERTS_BIF_YIELD1(bif_export[BIF_finish_loading_1], BIF_P, BIF_ARG_1); @@ -272,7 +275,6 @@ finish_loading_1(BIF_ALIST_1) if (exceptions) { res = exception_list(BIF_P, am_not_purged, p, exceptions); - erts_abort_staging_code_ix(); } else { /* * Now we can load all code. This can't fail. @@ -299,20 +301,63 @@ finish_loading_1(BIF_ALIST_1) if (exceptions) { res = exception_list(BIF_P, am_on_load, p, exceptions); } - erts_commit_staging_code_ix(); + do_commit = 1; } done: if (p) { erts_free(ERTS_ALC_T_LOADER_TMP, p); } - if (!is_blocking) { - erts_unlock_code_ix(); - } else { - erts_smp_thr_progress_unblock(); - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + return staging_epilogue(BIF_P, do_commit, res, is_blocking); +} + +static Eterm +staging_epilogue(Process* c_p, int commit, Eterm res, int is_blocking) +{ +#ifdef ERTS_SMP + if (is_blocking || !commit) +#endif + { + if (commit) { + erts_end_staging_code_ix(); + erts_activate_staging_code_ix(); + } + else { + erts_abort_staging_code_ix(); + } + if (is_blocking) { + erts_smp_thr_progress_unblock(); + erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); + } + else { + erts_unlock_code_ix(); + } + return res; } - BIF_RET(res); +#ifdef ERTS_SMP + else { + ErtsThrPrgrVal later; + ASSERT(is_value(res)); + + erts_end_staging_code_ix(); + /* + * Now we must wait for all schedulers to do a memory barrier before + * we can activate and let them access the new staged code. This allows + * schedulers to read active code_ix in a safe way while executing + * without any memory barriers at all. + */ + + later = erts_thr_progress_later(); + erts_thr_progress_wakeup(c_p->scheduler_data, later); + erts_notify_code_ix_activation(c_p, later); + erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL); + /* + * handle_code_ix_activation() will do the rest "later" + * and resume this process in bif_return_trap() to return 'res'. + */ + ERTS_BIF_YIELD1(&bif_return_trap_export, c_p, res); + } +#endif } BIF_RETTYPE @@ -407,17 +452,18 @@ BIF_RETTYPE delete_module_1(BIF_ALIST_1) ErtsCodeIndex code_ix; Module* modp; int is_blocking = 0; + int success = 0; Eterm res = NIL; if (is_not_atom(BIF_ARG_1)) { - goto badarg; + BIF_ERROR(BIF_P, BADARG); } if (!erts_try_lock_code_ix(BIF_P)) { ERTS_BIF_YIELD1(bif_export[BIF_delete_module_1], BIF_P, BIF_ARG_1); } - - do { +retry: + { erts_start_staging_code_ix(); code_ix = erts_staging_code_ix(); modp = erts_get_module(BIF_ARG_1, code_ix); @@ -429,7 +475,7 @@ BIF_RETTYPE delete_module_1(BIF_ALIST_1) erts_dsprintf(dsbufp, "Module %T must be purged before loading\n", BIF_ARG_1); erts_send_error_to_logger(BIF_P->group_leader, dsbufp); - res = am_badarg; + ERTS_BIF_PREP_ERROR(res, BIF_P, BADARG); } else { if (!is_blocking) { @@ -441,7 +487,7 @@ BIF_RETTYPE delete_module_1(BIF_ALIST_1) erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_smp_thr_progress_block(); is_blocking = 1; - continue; + goto retry; } } else if (modp->curr.num_breakpoints) { @@ -450,28 +496,11 @@ BIF_RETTYPE delete_module_1(BIF_ALIST_1) } delete_code(modp); res = am_true; + success = 1; } - } while (res == NIL); - - if (res == am_true) { - erts_commit_staging_code_ix(); - } - else { - erts_abort_staging_code_ix(); - } - if (is_blocking) { - erts_smp_thr_progress_unblock(); - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); - } - else { - erts_unlock_code_ix(); } - if (res == am_badarg) { - badarg: - BIF_ERROR(BIF_P, BADARG); - } - BIF_RET(res); + return staging_epilogue(BIF_P, success, res, is_blocking); } BIF_RETTYPE module_loaded_1(BIF_ALIST_1) diff --git a/erts/emulator/beam/code_ix.c b/erts/emulator/beam/code_ix.c index a1db003127..5565b51e7e 100644 --- a/erts/emulator/beam/code_ix.c +++ b/erts/emulator/beam/code_ix.c @@ -66,22 +66,25 @@ void erts_start_staging_code_ix(void) } -void erts_commit_staging_code_ix(void) +void erts_end_staging_code_ix(void) { beam_catches_end_staging(1); export_end_staging(1); module_end_staging(1); erts_end_staging_ranges(1); - { - ErtsCodeIndex ix; - export_write_lock(); - ix = erts_staging_code_ix(); - erts_smp_atomic32_set_nob(&the_active_code_index, ix); - ix = (ix + 1) % ERTS_NUM_CODE_IX; - erts_smp_atomic32_set_nob(&the_staging_code_index, ix); - export_write_unlock(); - } - CIX_TRACE("commit"); + CIX_TRACE("end"); +} + +void erts_activate_staging_code_ix(void) +{ + ErtsCodeIndex ix; + export_write_lock(); + ix = erts_staging_code_ix(); + erts_smp_atomic32_set_nob(&the_active_code_index, ix); + ix = (ix + 1) % ERTS_NUM_CODE_IX; + erts_smp_atomic32_set_nob(&the_staging_code_index, ix); + export_write_unlock(); + CIX_TRACE("activate"); } void erts_abort_staging_code_ix(void) @@ -121,7 +124,7 @@ int erts_try_lock_code_ix(Process* c_p) /* Unlock code_ix (resume all waiters) */ -void erts_unlock_code_ix() +void erts_unlock_code_ix(void) { erts_smp_mtx_lock(&the_code_ix_queue_lock); while (the_code_ix_queue != NULL) { /* unleash the entire herd */ diff --git a/erts/emulator/beam/code_ix.h b/erts/emulator/beam/code_ix.h index 66543fa2a8..bac8c3365c 100644 --- a/erts/emulator/beam/code_ix.h +++ b/erts/emulator/beam/code_ix.h @@ -91,22 +91,27 @@ int erts_try_lock_code_ix(struct process*); */ void erts_unlock_code_ix(void); -/* Make the "staging area" a complete copy of the active code. +/* Prepare the "staging area" to be a complete copy of the active code. * code_ix must be locked. - * Must be followed by a call to either "commit" or "abort" before code_ix lock - * is released. + * Must be followed by calls to either "end" and "activate" or "abort" before + * code_ix lock is released. */ void erts_start_staging_code_ix(void); -/* Commit the staging area and update the active code index. - * code_ix must be locked and erts_start_staging_code_ix() called. - * ToDo: Updating active code index should be done according to Rickard's recipe. - * This function might need to be split into two. +/* End the staging. + * code_ix must be locked. + * Must be followed by a call to either "activate" or "abort" + * before code_ix lock is released. + */ +void erts_end_staging_code_ix(void); + +/* Set staging code index as new active code index. + * code_ix must be locked and "start" and "end" called. */ -void erts_commit_staging_code_ix(void); +void erts_activate_staging_code_ix(void); /* Abort the staging. - * code_ix must be locked and erts_start_staging_code_ix() called. + * code_ix must be locked and "start" called. */ void erts_abort_staging_code_ix(void); diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index 2d192b3fc4..a4690891d5 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -1473,7 +1473,8 @@ erl_start(int argc, char **argv) init_shared_memory(boot_argc, boot_argv); load_preloaded(); - erts_commit_staging_code_ix(); + erts_end_staging_code_ix(); + erts_activate_staging_code_ix(); erts_initialized = 1; diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 44a99e1f84..aaee951107 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -366,6 +366,9 @@ dbg_chk_aux_work_val(erts_aint32_t value) #ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN valid |= ERTS_SSI_AUX_WORK_CHECK_CHILDREN; #endif +#ifdef ERTS_SMP + valid |= ERTS_SSI_AUX_WORK_CODE_IX_ACTIVATION; +#endif if (~valid & value) erl_exit(ERTS_ABORT_EXIT, @@ -861,7 +864,7 @@ set_aux_work_flags_wakeup_nob(ErtsSchedulerSleepInfo *ssi, } } -#if 0 /* Currently not used */ +#ifdef ERTS_SMP static ERTS_INLINE void set_aux_work_flags_wakeup_relb(ErtsSchedulerSleepInfo *ssi, @@ -882,7 +885,7 @@ set_aux_work_flags_wakeup_relb(ErtsSchedulerSleepInfo *ssi, } } -#endif +#endif /* ERTS_SMP */ static ERTS_INLINE erts_aint32_t set_aux_work_flags(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flgs) @@ -1145,7 +1148,49 @@ handle_async_ready_clean(ErtsAuxWorkData *awdp, } } +#endif /* ERTS_USE_ASYNC_READY_Q */ + +#ifdef ERTS_SMP +void +erts_notify_code_ix_activation(Process* p, ErtsThrPrgrVal later) +{ + ErtsAuxWorkData* awdp = &p->scheduler_data->aux_work_data; + ASSERT(awdp->code_ix_activation.code_stager == NULL); + awdp->code_ix_activation.code_stager = p; + awdp->code_ix_activation.thr_prgr = later; + erts_smp_proc_inc_refc(p); + set_aux_work_flags_wakeup_relb(p->scheduler_data->ssi, + ERTS_SSI_AUX_WORK_CODE_IX_ACTIVATION); +} + +static erts_aint32_t +handle_code_ix_activation(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) +{ + Process* p; + if (!erts_thr_progress_has_reached(awdp->code_ix_activation.thr_prgr)) { + return aux_work & ~ERTS_SSI_AUX_WORK_CODE_IX_ACTIVATION; + } + unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_CODE_IX_ACTIVATION); + p = awdp->code_ix_activation.code_stager; + ASSERT(p); +#ifdef DEBUG + awdp->code_ix_activation.code_stager = NULL; #endif + erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + if (!ERTS_PROC_IS_EXITING(p)) { + erts_activate_staging_code_ix(); + erts_resume(p, ERTS_PROC_LOCK_STATUS); + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + } + else { + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + erts_abort_staging_code_ix(); + } + erts_unlock_code_ix(); + erts_smp_proc_dec_refc(p); + return aux_work & ~ERTS_SSI_AUX_WORK_CODE_IX_ACTIVATION; +} +#endif /* ERTS_SMP */ static ERTS_INLINE erts_aint32_t handle_fix_alloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) @@ -1451,6 +1496,10 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work) handle_mseg_cache_check); #endif +#ifdef ERTS_SMP + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_CODE_IX_ACTIVATION, + handle_code_ix_activation); +#endif ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); return aux_work; diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index c23810f15a..3f19d92fcd 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -264,6 +264,7 @@ typedef enum { #define ERTS_SSI_AUX_WORK_CHECK_CHILDREN (((erts_aint32_t) 1) << 8) #define ERTS_SSI_AUX_WORK_SET_TMO (((erts_aint32_t) 1) << 9) #define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK (((erts_aint32_t) 1) << 10) +#define ERTS_SSI_AUX_WORK_CODE_IX_ACTIVATION (((erts_aint32_t) 1) << 11) typedef struct ErtsSchedulerSleepInfo_ ErtsSchedulerSleepInfo; @@ -429,6 +430,12 @@ typedef struct { void *queue; } async_ready; #endif +#ifdef ERTS_SMP + struct { + Process* code_stager; + ErtsThrPrgrVal thr_prgr; + } code_ix_activation; +#endif } ErtsAuxWorkData; struct ErtsSchedulerData_ { @@ -1110,6 +1117,9 @@ void erts_smp_notify_check_children_needed(void); #if ERTS_USE_ASYNC_READY_Q void erts_notify_check_async_ready_queue(void *); #endif +#ifdef ERTS_SMP +void erts_notify_code_ix_activation(Process* p, ErtsThrPrgrVal later); +#endif void erts_schedule_misc_aux_work(int sched_id, void (*func)(void *), void *arg); -- cgit v1.2.3