aboutsummaryrefslogtreecommitdiffstats
path: root/erts/emulator/beam
diff options
context:
space:
mode:
Diffstat (limited to 'erts/emulator/beam')
-rw-r--r--erts/emulator/beam/beam_bif_load.c95
-rw-r--r--erts/emulator/beam/code_ix.c27
-rw-r--r--erts/emulator/beam/code_ix.h23
-rw-r--r--erts/emulator/beam/erl_init.c3
-rw-r--r--erts/emulator/beam/erl_process.c53
-rw-r--r--erts/emulator/beam/erl_process.h10
6 files changed, 154 insertions, 57 deletions
diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index 1227af8cbb..6df5774fdb 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -35,6 +35,7 @@
#include "erl_nif.h"
#include "erl_thr_progress.h"
+static Eterm staging_epilogue(Process* c_p, int, Eterm res, int);
static void set_default_trace_pattern(Eterm module);
static Eterm check_process_code(Process* rp, Module* modp);
static void delete_code(Module* modp);
@@ -86,7 +87,8 @@ BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3)
res = erts_make_stub_module(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3);
if (res == BIF_ARG_1) {
- erts_commit_staging_code_ix();
+ erts_end_staging_code_ix();
+ erts_activate_staging_code_ix();
}
else {
erts_abort_staging_code_ix();
@@ -166,6 +168,7 @@ finish_loading_1(BIF_ALIST_1)
Uint exceptions;
Eterm res;
int is_blocking = 0;
+ int do_commit = 0;
if (!erts_try_lock_code_ix(BIF_P)) {
ERTS_BIF_YIELD1(bif_export[BIF_finish_loading_1], BIF_P, BIF_ARG_1);
@@ -272,7 +275,6 @@ finish_loading_1(BIF_ALIST_1)
if (exceptions) {
res = exception_list(BIF_P, am_not_purged, p, exceptions);
- erts_abort_staging_code_ix();
} else {
/*
* Now we can load all code. This can't fail.
@@ -299,20 +301,63 @@ finish_loading_1(BIF_ALIST_1)
if (exceptions) {
res = exception_list(BIF_P, am_on_load, p, exceptions);
}
- erts_commit_staging_code_ix();
+ do_commit = 1;
}
done:
if (p) {
erts_free(ERTS_ALC_T_LOADER_TMP, p);
}
- if (!is_blocking) {
- erts_unlock_code_ix();
- } else {
- erts_smp_thr_progress_unblock();
- erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
+ return staging_epilogue(BIF_P, do_commit, res, is_blocking);
+}
+
+static Eterm
+staging_epilogue(Process* c_p, int commit, Eterm res, int is_blocking)
+{
+#ifdef ERTS_SMP
+ if (is_blocking || !commit)
+#endif
+ {
+ if (commit) {
+ erts_end_staging_code_ix();
+ erts_activate_staging_code_ix();
+ }
+ else {
+ erts_abort_staging_code_ix();
+ }
+ if (is_blocking) {
+ erts_smp_thr_progress_unblock();
+ erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
+ }
+ else {
+ erts_unlock_code_ix();
+ }
+ return res;
}
- BIF_RET(res);
+#ifdef ERTS_SMP
+ else {
+ ErtsThrPrgrVal later;
+ ASSERT(is_value(res));
+
+ erts_end_staging_code_ix();
+ /*
+ * Now we must wait for all schedulers to do a memory barrier before
+ * we can activate and let them access the new staged code. This allows
+ * schedulers to read active code_ix in a safe way while executing
+ * without any memory barriers at all.
+ */
+
+ later = erts_thr_progress_later();
+ erts_thr_progress_wakeup(c_p->scheduler_data, later);
+ erts_notify_code_ix_activation(c_p, later);
+ erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL);
+ /*
+ * handle_code_ix_activation() will do the rest "later"
+ * and resume this process in bif_return_trap() to return 'res'.
+ */
+ ERTS_BIF_YIELD1(&bif_return_trap_export, c_p, res);
+ }
+#endif
}
BIF_RETTYPE
@@ -407,17 +452,18 @@ BIF_RETTYPE delete_module_1(BIF_ALIST_1)
ErtsCodeIndex code_ix;
Module* modp;
int is_blocking = 0;
+ int success = 0;
Eterm res = NIL;
if (is_not_atom(BIF_ARG_1)) {
- goto badarg;
+ BIF_ERROR(BIF_P, BADARG);
}
if (!erts_try_lock_code_ix(BIF_P)) {
ERTS_BIF_YIELD1(bif_export[BIF_delete_module_1], BIF_P, BIF_ARG_1);
}
-
- do {
+retry:
+ {
erts_start_staging_code_ix();
code_ix = erts_staging_code_ix();
modp = erts_get_module(BIF_ARG_1, code_ix);
@@ -429,7 +475,7 @@ BIF_RETTYPE delete_module_1(BIF_ALIST_1)
erts_dsprintf(dsbufp, "Module %T must be purged before loading\n",
BIF_ARG_1);
erts_send_error_to_logger(BIF_P->group_leader, dsbufp);
- res = am_badarg;
+ ERTS_BIF_PREP_ERROR(res, BIF_P, BADARG);
}
else {
if (!is_blocking) {
@@ -441,7 +487,7 @@ BIF_RETTYPE delete_module_1(BIF_ALIST_1)
erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
erts_smp_thr_progress_block();
is_blocking = 1;
- continue;
+ goto retry;
}
}
else if (modp->curr.num_breakpoints) {
@@ -450,28 +496,11 @@ BIF_RETTYPE delete_module_1(BIF_ALIST_1)
}
delete_code(modp);
res = am_true;
+ success = 1;
}
- } while (res == NIL);
-
- if (res == am_true) {
- erts_commit_staging_code_ix();
- }
- else {
- erts_abort_staging_code_ix();
- }
- if (is_blocking) {
- erts_smp_thr_progress_unblock();
- erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
- }
- else {
- erts_unlock_code_ix();
}
- if (res == am_badarg) {
- badarg:
- BIF_ERROR(BIF_P, BADARG);
- }
- BIF_RET(res);
+ return staging_epilogue(BIF_P, success, res, is_blocking);
}
BIF_RETTYPE module_loaded_1(BIF_ALIST_1)
diff --git a/erts/emulator/beam/code_ix.c b/erts/emulator/beam/code_ix.c
index a1db003127..5565b51e7e 100644
--- a/erts/emulator/beam/code_ix.c
+++ b/erts/emulator/beam/code_ix.c
@@ -66,22 +66,25 @@ void erts_start_staging_code_ix(void)
}
-void erts_commit_staging_code_ix(void)
+void erts_end_staging_code_ix(void)
{
beam_catches_end_staging(1);
export_end_staging(1);
module_end_staging(1);
erts_end_staging_ranges(1);
- {
- ErtsCodeIndex ix;
- export_write_lock();
- ix = erts_staging_code_ix();
- erts_smp_atomic32_set_nob(&the_active_code_index, ix);
- ix = (ix + 1) % ERTS_NUM_CODE_IX;
- erts_smp_atomic32_set_nob(&the_staging_code_index, ix);
- export_write_unlock();
- }
- CIX_TRACE("commit");
+ CIX_TRACE("end");
+}
+
+void erts_activate_staging_code_ix(void)
+{
+ ErtsCodeIndex ix;
+ export_write_lock();
+ ix = erts_staging_code_ix();
+ erts_smp_atomic32_set_nob(&the_active_code_index, ix);
+ ix = (ix + 1) % ERTS_NUM_CODE_IX;
+ erts_smp_atomic32_set_nob(&the_staging_code_index, ix);
+ export_write_unlock();
+ CIX_TRACE("activate");
}
void erts_abort_staging_code_ix(void)
@@ -121,7 +124,7 @@ int erts_try_lock_code_ix(Process* c_p)
/* Unlock code_ix (resume all waiters)
*/
-void erts_unlock_code_ix()
+void erts_unlock_code_ix(void)
{
erts_smp_mtx_lock(&the_code_ix_queue_lock);
while (the_code_ix_queue != NULL) { /* unleash the entire herd */
diff --git a/erts/emulator/beam/code_ix.h b/erts/emulator/beam/code_ix.h
index 66543fa2a8..bac8c3365c 100644
--- a/erts/emulator/beam/code_ix.h
+++ b/erts/emulator/beam/code_ix.h
@@ -91,22 +91,27 @@ int erts_try_lock_code_ix(struct process*);
*/
void erts_unlock_code_ix(void);
-/* Make the "staging area" a complete copy of the active code.
+/* Prepare the "staging area" to be a complete copy of the active code.
* code_ix must be locked.
- * Must be followed by a call to either "commit" or "abort" before code_ix lock
- * is released.
+ * Must be followed by calls to either "end" and "activate" or "abort" before
+ * code_ix lock is released.
*/
void erts_start_staging_code_ix(void);
-/* Commit the staging area and update the active code index.
- * code_ix must be locked and erts_start_staging_code_ix() called.
- * ToDo: Updating active code index should be done according to Rickard's recipe.
- * This function might need to be split into two.
+/* End the staging.
+ * code_ix must be locked.
+ * Must be followed by a call to either "activate" or "abort"
+ * before code_ix lock is released.
+ */
+void erts_end_staging_code_ix(void);
+
+/* Set staging code index as new active code index.
+ * code_ix must be locked and "start" and "end" called.
*/
-void erts_commit_staging_code_ix(void);
+void erts_activate_staging_code_ix(void);
/* Abort the staging.
- * code_ix must be locked and erts_start_staging_code_ix() called.
+ * code_ix must be locked and "start" called.
*/
void erts_abort_staging_code_ix(void);
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 2d192b3fc4..a4690891d5 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -1473,7 +1473,8 @@ erl_start(int argc, char **argv)
init_shared_memory(boot_argc, boot_argv);
load_preloaded();
- erts_commit_staging_code_ix();
+ erts_end_staging_code_ix();
+ erts_activate_staging_code_ix();
erts_initialized = 1;
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 44a99e1f84..aaee951107 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -366,6 +366,9 @@ dbg_chk_aux_work_val(erts_aint32_t value)
#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN
valid |= ERTS_SSI_AUX_WORK_CHECK_CHILDREN;
#endif
+#ifdef ERTS_SMP
+ valid |= ERTS_SSI_AUX_WORK_CODE_IX_ACTIVATION;
+#endif
if (~valid & value)
erl_exit(ERTS_ABORT_EXIT,
@@ -861,7 +864,7 @@ set_aux_work_flags_wakeup_nob(ErtsSchedulerSleepInfo *ssi,
}
}
-#if 0 /* Currently not used */
+#ifdef ERTS_SMP
static ERTS_INLINE void
set_aux_work_flags_wakeup_relb(ErtsSchedulerSleepInfo *ssi,
@@ -882,7 +885,7 @@ set_aux_work_flags_wakeup_relb(ErtsSchedulerSleepInfo *ssi,
}
}
-#endif
+#endif /* ERTS_SMP */
static ERTS_INLINE erts_aint32_t
set_aux_work_flags(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flgs)
@@ -1145,7 +1148,49 @@ handle_async_ready_clean(ErtsAuxWorkData *awdp,
}
}
+#endif /* ERTS_USE_ASYNC_READY_Q */
+
+#ifdef ERTS_SMP
+void
+erts_notify_code_ix_activation(Process* p, ErtsThrPrgrVal later)
+{
+ ErtsAuxWorkData* awdp = &p->scheduler_data->aux_work_data;
+ ASSERT(awdp->code_ix_activation.code_stager == NULL);
+ awdp->code_ix_activation.code_stager = p;
+ awdp->code_ix_activation.thr_prgr = later;
+ erts_smp_proc_inc_refc(p);
+ set_aux_work_flags_wakeup_relb(p->scheduler_data->ssi,
+ ERTS_SSI_AUX_WORK_CODE_IX_ACTIVATION);
+}
+
+static erts_aint32_t
+handle_code_ix_activation(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
+{
+ Process* p;
+ if (!erts_thr_progress_has_reached(awdp->code_ix_activation.thr_prgr)) {
+ return aux_work & ~ERTS_SSI_AUX_WORK_CODE_IX_ACTIVATION;
+ }
+ unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_CODE_IX_ACTIVATION);
+ p = awdp->code_ix_activation.code_stager;
+ ASSERT(p);
+#ifdef DEBUG
+ awdp->code_ix_activation.code_stager = NULL;
#endif
+ erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
+ if (!ERTS_PROC_IS_EXITING(p)) {
+ erts_activate_staging_code_ix();
+ erts_resume(p, ERTS_PROC_LOCK_STATUS);
+ erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+ }
+ else {
+ erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+ erts_abort_staging_code_ix();
+ }
+ erts_unlock_code_ix();
+ erts_smp_proc_dec_refc(p);
+ return aux_work & ~ERTS_SSI_AUX_WORK_CODE_IX_ACTIVATION;
+}
+#endif /* ERTS_SMP */
static ERTS_INLINE erts_aint32_t
handle_fix_alloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
@@ -1451,6 +1496,10 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work)
handle_mseg_cache_check);
#endif
+#ifdef ERTS_SMP
+ HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_CODE_IX_ACTIVATION,
+ handle_code_ix_activation);
+#endif
ERTS_DBG_CHK_AUX_WORK_VAL(aux_work);
return aux_work;
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index c23810f15a..3f19d92fcd 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -264,6 +264,7 @@ typedef enum {
#define ERTS_SSI_AUX_WORK_CHECK_CHILDREN (((erts_aint32_t) 1) << 8)
#define ERTS_SSI_AUX_WORK_SET_TMO (((erts_aint32_t) 1) << 9)
#define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK (((erts_aint32_t) 1) << 10)
+#define ERTS_SSI_AUX_WORK_CODE_IX_ACTIVATION (((erts_aint32_t) 1) << 11)
typedef struct ErtsSchedulerSleepInfo_ ErtsSchedulerSleepInfo;
@@ -429,6 +430,12 @@ typedef struct {
void *queue;
} async_ready;
#endif
+#ifdef ERTS_SMP
+ struct {
+ Process* code_stager;
+ ErtsThrPrgrVal thr_prgr;
+ } code_ix_activation;
+#endif
} ErtsAuxWorkData;
struct ErtsSchedulerData_ {
@@ -1110,6 +1117,9 @@ void erts_smp_notify_check_children_needed(void);
#if ERTS_USE_ASYNC_READY_Q
void erts_notify_check_async_ready_queue(void *);
#endif
+#ifdef ERTS_SMP
+void erts_notify_code_ix_activation(Process* p, ErtsThrPrgrVal later);
+#endif
void erts_schedule_misc_aux_work(int sched_id,
void (*func)(void *),
void *arg);
generated by cgit v1.2.3 (git 2.25.1) at 2024-12-02 20:41:53 +0000