aboutsummaryrefslogtreecommitdiffstats
path: root/erts/emulator/beam/erl_process.c
diff options
context:
space:
mode:
Diffstat (limited to 'erts/emulator/beam/erl_process.c')
-rw-r--r--erts/emulator/beam/erl_process.c170
1 files changed, 117 insertions, 53 deletions
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 8153cdce1c..c58bf40435 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -522,6 +522,7 @@ dbg_chk_aux_work_val(erts_aint32_t value)
valid |= ERTS_SSI_AUX_WORK_MISC_THR_PRGR;
valid |= ERTS_SSI_AUX_WORK_DD;
valid |= ERTS_SSI_AUX_WORK_DD_THR_PRGR;
+ valid |= ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP;
#endif
#if HAVE_ERTS_MSEG
valid |= ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK;
@@ -1653,6 +1654,74 @@ handle_delayed_dealloc_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
return aux_work & ~ERTS_SSI_AUX_WORK_DD_THR_PRGR;
}
+/*
+ * Handle scheduled thread progress later operations.
+ */
+#define ERTS_MAX_THR_PRGR_LATER_OPS 50
+
+static ERTS_INLINE erts_aint32_t
+handle_thr_prgr_later_op(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
+{
+ int lops;
+ ErtsThrPrgrVal current = haw_thr_prgr_current(awdp);
+
+ for (lops = 0; lops < ERTS_MAX_THR_PRGR_LATER_OPS; lops++) {
+ ErtsThrPrgrLaterOp *lop = awdp->later_op.first;
+ if (!erts_thr_progress_has_reached_this(current, lop->later))
+ return aux_work & ~ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP;
+ awdp->later_op.first = lop->next;
+ lop->func(lop->data);
+ if (!awdp->later_op.first) {
+ awdp->later_op.last = NULL;
+ unset_aux_work_flags(awdp->ssi,
+ ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP);
+ return aux_work & ~ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP;
+ }
+ }
+
+ return aux_work;
+}
+
+#endif /* ERTS_SMP */
+
+void
+erts_schedule_thr_prgr_later_op(void (*later_func)(void *),
+ void *later_data,
+ ErtsThrPrgrLaterOp *lop)
+{
+#ifndef ERTS_SMP
+ later_func(later_data);
+#else
+ ErtsSchedulerData *esdp;
+ ErtsAuxWorkData *awdp;
+ int request_wakeup = 1;
+
+ esdp = erts_get_scheduler_data();
+ ASSERT(esdp);
+ awdp = &esdp->aux_work_data;
+
+ lop->func = later_func;
+ lop->data = later_data;
+ lop->later = erts_thr_progress_later(esdp);
+ lop->next = NULL;
+ if (!awdp->later_op.last)
+ awdp->later_op.first = lop;
+ else {
+ ErtsThrPrgrLaterOp *last = awdp->later_op.last;
+ last->next = lop;
+ if (erts_thr_progress_equal(last->later, lop->later))
+ request_wakeup = 0;
+ }
+ awdp->later_op.last = lop;
+ set_aux_work_flags_wakeup_nob(awdp->ssi,
+ ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP);
+ if (request_wakeup)
+ erts_thr_progress_wakeup(esdp, lop->later);
+#endif
+}
+
+#ifdef ERTS_SMP
+
static erts_atomic32_t completed_dealloc_count;
static void
@@ -1878,6 +1947,11 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work, int waiting)
| ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC),
handle_fix_alloc);
+#ifdef ERTS_SMP
+ HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP,
+ handle_thr_prgr_later_op);
+#endif
+
#if ERTS_USE_ASYNC_READY_Q
HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_ASYNC_READY,
handle_async_ready);
@@ -4596,6 +4670,8 @@ init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp)
awdp->dd.thr_prgr = ERTS_THR_PRGR_VAL_WAITING;
awdp->dd.completed_callback = NULL;
awdp->dd.completed_arg = NULL;
+ awdp->later_op.first = NULL;
+ awdp->later_op.last = NULL;
#endif
#ifdef ERTS_USE_ASYNC_READY_Q
#ifdef ERTS_SMP
@@ -7781,9 +7857,9 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
p->reds = 0;
#ifdef ERTS_SMP
- p->u.ptimer = NULL;
+ p->u.alive.ptimer = NULL;
#else
- sys_memset(&p->u.tm, 0, sizeof(ErlTimer));
+ sys_memset(&p->u.alive.tm, 0, sizeof(ErlTimer));
#endif
p->reg = NULL;
@@ -7811,9 +7887,9 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
p->msg.save = &p->msg.first;
p->msg.len = 0;
#ifdef ERTS_SMP
- p->msg_inq.first = NULL;
- p->msg_inq.last = &p->msg_inq.first;
- p->msg_inq.len = 0;
+ p->u.alive.msg_inq.first = NULL;
+ p->u.alive.msg_inq.last = &p->u.alive.msg_inq.first;
+ p->u.alive.msg_inq.len = 0;
#endif
p->bif_timers = NULL;
p->mbuf = NULL;
@@ -7901,8 +7977,8 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
p->scheduler_data = NULL;
p->suspendee = NIL;
p->pending_suspenders = NULL;
- p->pending_exit.reason = THE_NON_VALUE;
- p->pending_exit.bp = NULL;
+ p->u.alive.pending_exit.reason = THE_NON_VALUE;
+ p->u.alive.pending_exit.bp = NULL;
#endif
#if !defined(NO_FPE_SIGNALS) || defined(HIPE)
@@ -7970,9 +8046,9 @@ void erts_init_empty_process(Process *p)
p->bin_old_vheap = 0;
p->bin_vheap_mature = 0;
#ifdef ERTS_SMP
- p->u.ptimer = NULL;
+ p->u.alive.ptimer = NULL;
#else
- memset(&(p->u.tm), 0, sizeof(ErlTimer));
+ memset(&(p->u.alive.tm), 0, sizeof(ErlTimer));
#endif
p->next = NULL;
p->off_heap.first = NULL;
@@ -8040,13 +8116,13 @@ void erts_init_empty_process(Process *p)
#ifdef ERTS_SMP
p->scheduler_data = NULL;
- p->msg_inq.first = NULL;
- p->msg_inq.last = &p->msg_inq.first;
- p->msg_inq.len = 0;
+ p->u.alive.msg_inq.first = NULL;
+ p->u.alive.msg_inq.last = &p->u.alive.msg_inq.first;
+ p->u.alive.msg_inq.len = 0;
p->suspendee = NIL;
p->pending_suspenders = NULL;
- p->pending_exit.reason = THE_NON_VALUE;
- p->pending_exit.bp = NULL;
+ p->u.alive.pending_exit.reason = THE_NON_VALUE;
+ p->u.alive.pending_exit.bp = NULL;
erts_proc_lock_init(p);
erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL);
RUNQ_SET_RQ(&p->run_queue, ERTS_RUNQ_IX(0));
@@ -8096,12 +8172,12 @@ erts_debug_verify_clean_empty_process(Process* p)
ASSERT(p->parent == NIL);
#ifdef ERTS_SMP
- ASSERT(p->msg_inq.first == NULL);
- ASSERT(p->msg_inq.len == 0);
+ ASSERT(p->u.alive.msg_inq.first == NULL);
+ ASSERT(p->u.alive.msg_inq.len == 0);
ASSERT(p->suspendee == NIL);
ASSERT(p->pending_suspenders == NULL);
- ASSERT(p->pending_exit.reason == THE_NON_VALUE);
- ASSERT(p->pending_exit.bp == NULL);
+ ASSERT(p->u.alive.pending_exit.reason == THE_NON_VALUE);
+ ASSERT(p->u.alive.pending_exit.bp == NULL);
#endif
/* Thing that erts_cleanup_empty_process() cleans up */
@@ -8282,7 +8358,7 @@ void
erts_handle_pending_exit(Process *c_p, ErtsProcLocks locks)
{
ErtsProcLocks xlocks;
- ASSERT(is_value(c_p->pending_exit.reason));
+ ASSERT(is_value(c_p->u.alive.pending_exit.reason));
ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == locks);
ERTS_SMP_LC_ASSERT(locks & ERTS_PROC_LOCK_MAIN);
ERTS_SMP_LC_ASSERT(!((ERTS_PSFLG_EXITING|ERTS_PSFLG_FREE)
@@ -8301,10 +8377,10 @@ erts_handle_pending_exit(Process *c_p, ErtsProcLocks locks)
set_proc_exiting(c_p,
erts_smp_atomic32_read_acqb(&c_p->state),
- c_p->pending_exit.reason,
- c_p->pending_exit.bp);
- c_p->pending_exit.reason = THE_NON_VALUE;
- c_p->pending_exit.bp = NULL;
+ c_p->u.alive.pending_exit.reason,
+ c_p->u.alive.pending_exit.bp);
+ c_p->u.alive.pending_exit.reason = THE_NON_VALUE;
+ c_p->u.alive.pending_exit.bp = NULL;
if (xlocks)
erts_smp_proc_unlock(c_p, xlocks);
@@ -8525,7 +8601,7 @@ send_exit_signal(Process *c_p, /* current process if and only
else if (reason != am_normal || (flags & ERTS_XSIG_FLG_NO_IGN_NORMAL)) {
#ifdef ERTS_SMP
if (!(state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT))) {
- ASSERT(!rp->pending_exit.bp);
+ ASSERT(!rp->u.alive.pending_exit.bp);
if (rp == c_p && (*rp_locks & ERTS_PROC_LOCK_MAIN)) {
/* Ensure that all locks on c_p are locked before
@@ -8575,7 +8651,7 @@ send_exit_signal(Process *c_p, /* current process if and only
set_pending_exit:
if (is_immed(rsn)) {
- rp->pending_exit.reason = rsn;
+ rp->u.alive.pending_exit.reason = rsn;
}
else {
Eterm *hp;
@@ -8583,11 +8659,11 @@ send_exit_signal(Process *c_p, /* current process if and only
ErlHeapFragment *bp = new_message_buffer(sz);
hp = &bp->mem[0];
- rp->pending_exit.reason = copy_struct(rsn,
- sz,
- &hp,
- &bp->off_heap);
- rp->pending_exit.bp = bp;
+ rp->u.alive.pending_exit.reason = copy_struct(rsn,
+ sz,
+ &hp,
+ &bp->off_heap);
+ rp->u.alive.pending_exit.bp = bp;
}
erts_smp_atomic32_read_bor_relb(&rp->state,
ERTS_PSFLG_PENDING_EXIT);
@@ -8959,10 +9035,10 @@ erts_do_exit_process(Process* p, Eterm reason)
state = set_proc_exiting_state(p, erts_smp_atomic32_read_nob(&p->state));
if (state & ERTS_PSFLG_PENDING_EXIT) {
/* Process exited before pending exit was received... */
- p->pending_exit.reason = THE_NON_VALUE;
- if (p->pending_exit.bp) {
- free_message_buffer(p->pending_exit.bp);
- p->pending_exit.bp = NULL;
+ p->u.alive.pending_exit.reason = THE_NON_VALUE;
+ if (p->u.alive.pending_exit.bp) {
+ free_message_buffer(p->u.alive.pending_exit.bp);
+ p->u.alive.pending_exit.bp = NULL;
}
}
@@ -9209,21 +9285,9 @@ erts_continue_exit_process(Process *p)
delete_process(p);
#ifdef ERTS_SMP
- /*
- * Each scheduler will decrease refc by one via misc aux work;
- * we have one refc for reference from process table which we
- * now want to remove, i.e. we increase refc with schedulers-1.
- *
- * Process struct wont be deallocated until (earliest) when
- * all schedulers have decreased refc via misc aux work...
- */
- if (erts_no_schedulers != 1)
- erts_smp_proc_add_refc(p, (Sint32) erts_no_schedulers-1);
- erts_schedule_multi_misc_aux_work(0,
- erts_no_schedulers,
- proc_dec_refc,
- (void *) p);
-
+ erts_schedule_thr_prgr_later_op(proc_dec_refc,
+ (void *) p,
+ &p->u.release_data);
erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN);
ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p);
#endif
@@ -9275,9 +9339,9 @@ cancel_timer(Process* p)
ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p));
p->flags &= ~(F_INSLPQUEUE|F_TIMO);
#ifdef ERTS_SMP
- erts_cancel_smp_ptimer(p->u.ptimer);
+ erts_cancel_smp_ptimer(p->u.alive.ptimer);
#else
- erts_cancel_timer(&p->u.tm);
+ erts_cancel_timer(&p->u.alive.tm);
#endif
}
@@ -9298,12 +9362,12 @@ set_timer(Process* p, Uint timeout)
p->flags &= ~F_TIMO;
#ifdef ERTS_SMP
- erts_create_smp_ptimer(&p->u.ptimer,
+ erts_create_smp_ptimer(&p->u.alive.ptimer,
p->id,
(ErlTimeoutProc) timeout_proc,
timeout);
#else
- erts_set_timer(&p->u.tm,
+ erts_set_timer(&p->u.alive.tm,
(ErlTimeoutProc) timeout_proc,
NULL,
(void*) p,
generated by cgit v1.2.3 (git 2.25.1) at 2024-12-28 01:15:20 +0000