aboutsummaryrefslogtreecommitdiffstats
path: root/erts/emulator
diff options
context:
space:
mode:
Diffstat (limited to 'erts/emulator')
-rw-r--r--erts/emulator/beam/beam_bp.c4
-rw-r--r--erts/emulator/beam/beam_emu.c32
-rw-r--r--erts/emulator/beam/bif.c114
-rw-r--r--erts/emulator/beam/break.c86
-rw-r--r--erts/emulator/beam/dist.c25
-rw-r--r--erts/emulator/beam/erl_alloc.types3
-rw-r--r--erts/emulator/beam/erl_alloc_util.c11
-rw-r--r--erts/emulator/beam/erl_bif_ddll.c12
-rwxr-xr-xerts/emulator/beam/erl_bif_info.c22
-rw-r--r--erts/emulator/beam/erl_bif_timer.c4
-rw-r--r--erts/emulator/beam/erl_bif_trace.c17
-rw-r--r--erts/emulator/beam/erl_cpu_topology.c5
-rw-r--r--erts/emulator/beam/erl_db.c14
-rw-r--r--erts/emulator/beam/erl_db_util.h2
-rw-r--r--erts/emulator/beam/erl_gc.c29
-rw-r--r--erts/emulator/beam/erl_init.c64
-rw-r--r--erts/emulator/beam/erl_lock_check.c2
-rw-r--r--erts/emulator/beam/erl_message.c200
-rw-r--r--erts/emulator/beam/erl_nif.c20
-rw-r--r--erts/emulator/beam/erl_node_container_utils.h43
-rw-r--r--erts/emulator/beam/erl_node_tables.c38
-rw-r--r--erts/emulator/beam/erl_port_task.c224
-rw-r--r--erts/emulator/beam/erl_port_task.h13
-rw-r--r--erts/emulator/beam/erl_process.c3742
-rw-r--r--erts/emulator/beam/erl_process.h483
-rw-r--r--erts/emulator/beam/erl_process_dump.c13
-rw-r--r--erts/emulator/beam/erl_process_lock.c452
-rw-r--r--erts/emulator/beam/erl_process_lock.h417
-rw-r--r--erts/emulator/beam/erl_threads.h31
-rw-r--r--erts/emulator/beam/erl_time.h4
-rw-r--r--erts/emulator/beam/erl_time_sup.c85
-rw-r--r--erts/emulator/beam/erl_trace.c193
-rwxr-xr-xerts/emulator/beam/global.h211
-rw-r--r--erts/emulator/beam/io.c83
-rw-r--r--erts/emulator/beam/register.c24
-rw-r--r--erts/emulator/beam/sys.h45
-rw-r--r--erts/emulator/beam/utils.c310
-rw-r--r--erts/emulator/hipe/hipe_debug.c2
-rw-r--r--erts/emulator/hipe/hipe_mode_switch.c10
-rw-r--r--erts/emulator/sys/win32/erl_win_sys.h5
-rw-r--r--erts/emulator/sys/win32/sys_time.c6
41 files changed, 4010 insertions, 3090 deletions
diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c
index 113f3ac6a8..26dadfbbc0 100644
--- a/erts/emulator/beam/beam_bp.c
+++ b/erts/emulator/beam/beam_bp.c
@@ -714,7 +714,7 @@ void erts_trace_time_break(Process *p, BeamInstr *pc, BpDataTime *bdt, Uint type
BpDataTime *pbdt = NULL;
ASSERT(p);
- ASSERT(p->status == P_RUNNING);
+ ASSERT(ERTS_PSFLG_RUNNING & erts_smp_atomic32_read_acqb(&p->state));
/* get previous timestamp and breakpoint
* from the process psd */
@@ -1258,7 +1258,7 @@ static int clear_function_break(Module *m, BeamInstr *pc, int bif, BeamInstr bre
for (j = 0; j < bdt->hash[i].n; ++j) {
item = &(bdt->hash[i].item[j]);
if (item->pid != NIL) {
- h_p = process_tab[internal_pid_index(item->pid)];
+ h_p = erts_proc_lookup(item->pid);
if (h_p) {
pbt = ERTS_PROC_SET_CALL_TIME(h_p, ERTS_PROC_LOCK_MAIN, NULL);
if (pbt) {
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 49c4ac1b00..3973d1d378 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -64,11 +64,7 @@
# define PROCESS_MAIN_CHK_LOCKS(P) \
do { \
if ((P)) { \
- erts_pix_lock_t *pix_lock__ = ERTS_PIX2PIXLOCK(internal_pid_index((P)->id));\
erts_proc_lc_chk_only_proc_main((P)); \
- erts_pix_lock(pix_lock__); \
- ASSERT(0 < (P)->lock.refc && (P)->lock.refc < erts_no_schedulers*5);\
- erts_pix_unlock(pix_lock__); \
} \
else \
erts_lc_check_exact(NULL, 0); \
@@ -1879,13 +1875,12 @@ void process_main(void)
msgp = PEEK_MESSAGE(c_p);
if (msgp)
erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
- else {
+ else
#endif
+ {
SET_I((BeamInstr *) Arg(0));
Goto(*I); /* Jump to a wait or wait_timeout instruction */
-#ifdef ERTS_SMP
}
-#endif
}
ErtsMoveMsgAttachmentIntoProc(msgp, c_p, E, HTOP, FCALLS,
{
@@ -2114,11 +2109,11 @@ void process_main(void)
OpCase(wait_f):
wait2: {
- ASSERT(!ERTS_PROC_IS_EXITING(c_p));
c_p->i = (BeamInstr *) Arg(0); /* L1 */
SWAPOUT;
c_p->arity = 0;
- c_p->status = P_WAITING;
+ erts_smp_atomic32_read_band_relb(&c_p->state, ~ERTS_PSFLG_ACTIVE);
+ ASSERT(!ERTS_PROC_IS_EXITING(c_p));
erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
c_p->current = NULL;
goto do_schedule;
@@ -3197,10 +3192,6 @@ void process_main(void)
c_p->arg_reg[0] = r(0);
SWAPOUT;
c_p->i = I;
- erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_STATUS);
- if (c_p->status != P_SUSPENDED)
- erts_add_to_runq(c_p);
- erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS);
goto do_schedule1;
}
@@ -5163,9 +5154,6 @@ void process_main(void)
c_p->arity = 1; /* One living register (the 'true' return value) */
SWAPOUT;
c_p->i = I + 1; /* Next instruction */
- erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_STATUS);
- erts_add_to_runq(c_p);
- erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS);
c_p->current = NULL;
goto do_schedule;
}
@@ -6260,9 +6248,7 @@ erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* re
*/
erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p);
- if (c_p->msg.len > 0) {
- erts_add_to_runq(c_p);
- } else {
+ if (!c_p->msg.len) {
erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
c_p->fvalue = NIL;
PROCESS_MAIN_CHK_LOCKS(c_p);
@@ -6270,14 +6256,12 @@ erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* re
ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
PROCESS_MAIN_CHK_LOCKS(c_p);
erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
- ASSERT(!ERTS_PROC_IS_EXITING(c_p));
#ifdef ERTS_SMP
ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p);
- if (c_p->msg.len > 0)
- erts_add_to_runq(c_p);
- else
+ if (!c_p->msg.len)
#endif
- c_p->status = P_WAITING;
+ erts_smp_atomic32_read_band_relb(&c_p->state, ~ERTS_PSFLG_ACTIVE);
+ ASSERT(!ERTS_PROC_IS_EXITING(c_p));
}
erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
c_p->current = bif_export[BIF_hibernate_3]->code;
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index ee93cde243..fcb130655a 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -233,15 +233,17 @@ BIF_RETTYPE link_1(BIF_ALIST_1)
BIF_ERROR(BIF_P, BADARG);
- res_no_proc:
- if (BIF_P->flags & F_TRAPEXIT) {
- ErtsProcLocks locks = ERTS_PROC_LOCK_MAIN;
- erts_deliver_exit_message(BIF_ARG_1, BIF_P, &locks, am_noproc, NIL);
- erts_smp_proc_unlock(BIF_P, ~ERTS_PROC_LOCK_MAIN & locks);
- BIF_RET(am_true);
+res_no_proc: {
+ erts_aint32_t state = erts_smp_atomic32_read_nob(&BIF_P->state);
+ if (state & ERTS_PSFLG_TRAP_EXIT) {
+ ErtsProcLocks locks = ERTS_PROC_LOCK_MAIN;
+ erts_deliver_exit_message(BIF_ARG_1, BIF_P, &locks, am_noproc, NIL);
+ erts_smp_proc_unlock(BIF_P, ~ERTS_PROC_LOCK_MAIN & locks);
+ BIF_RET(am_true);
+ }
+ else
+ BIF_ERROR(BIF_P, EXC_NOPROC);
}
- else
- BIF_ERROR(BIF_P, EXC_NOPROC);
}
#define ERTS_DEMONITOR_FALSE 2
@@ -1103,8 +1105,9 @@ BIF_RETTYPE hibernate_3(BIF_ALIST_3)
if (erts_hibernate(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, reg)) {
/*
- * If hibernate succeeded, TRAP. The process will be suspended
- * if status is P_WAITING or continue (if any message was in the queue).
+ * If hibernate succeeded, TRAP. The process will be wait in a
+ * hibernated state if its state is inactive (!ERTS_PSFLG_ACTIVE);
+ * otherwise, continue executing (if any message was in the queue).
*/
BIF_TRAP_CODE_PTR_(BIF_P, BIF_P->i);
}
@@ -1403,9 +1406,8 @@ BIF_RETTYPE exit_2(BIF_ALIST_2)
}
else {
rp_locks = ERTS_PROC_LOCKS_XSIG_SEND;
- rp = erts_pid2proc_opt(BIF_P, ERTS_PROC_LOCK_MAIN,
- BIF_ARG_1, rp_locks,
- ERTS_P2P_FLG_SMP_INC_REFC);
+ rp = erts_pid2proc(BIF_P, ERTS_PROC_LOCK_MAIN,
+ BIF_ARG_1, rp_locks);
if (!rp) {
BIF_RET(am_true);
}
@@ -1427,8 +1429,6 @@ BIF_RETTYPE exit_2(BIF_ALIST_2)
rp_locks &= ~ERTS_PROC_LOCK_MAIN;
if (rp_locks)
erts_smp_proc_unlock(rp, rp_locks);
- if (rp != BIF_P)
- erts_smp_proc_dec_refc(rp);
#endif
/*
* We may have exited ourselves and may have to take action.
@@ -1502,14 +1502,13 @@ BIF_RETTYPE process_flag_2(BIF_ALIST_2)
BIF_RET(old_value);
}
else if (BIF_ARG_1 == am_priority) {
- erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_STATUS);
old_value = erts_set_process_priority(BIF_P, BIF_ARG_2);
- erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_STATUS);
if (old_value == THE_NON_VALUE)
goto error;
BIF_RET(old_value);
}
else if (BIF_ARG_1 == am_trap_exit) {
+ erts_aint32_t state;
Uint trap_exit;
if (BIF_ARG_2 == am_true) {
trap_exit = 1;
@@ -1524,59 +1523,52 @@ BIF_RETTYPE process_flag_2(BIF_ALIST_2)
* For more info, see implementation of erts_send_exit_signal().
*/
erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_STATUS);
+ ERTS_SMP_LC_ASSERT((ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS)
+ & erts_proc_lc_my_proc_locks(BIF_P));
ERTS_SMP_BIF_CHK_PENDING_EXIT(BIF_P,
ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS);
- old_value = ERTS_PROC_IS_TRAPPING_EXITS(BIF_P) ? am_true : am_false;
- if (trap_exit) {
- ERTS_PROC_SET_TRAP_EXIT(BIF_P);
- } else {
- ERTS_PROC_UNSET_TRAP_EXIT(BIF_P);
- }
+ if (trap_exit)
+ state = erts_smp_atomic32_read_bor_nob(&BIF_P->state,
+ ERTS_PSFLG_TRAP_EXIT);
+ else
+ state = erts_smp_atomic32_read_band_nob(&BIF_P->state,
+ ~ERTS_PSFLG_TRAP_EXIT);
+ old_value = (state & ERTS_PSFLG_TRAP_EXIT) ? am_true : am_false;
erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_STATUS);
BIF_RET(old_value);
}
else if (BIF_ARG_1 == am_scheduler) {
- int yield;
- ErtsRunQueue *old;
- ErtsRunQueue *new;
+ ErtsRunQueue *old, *new, *curr;
Sint sched;
+ erts_aint32_t state;
+
if (!is_small(BIF_ARG_2))
goto error;
sched = signed_val(BIF_ARG_2);
if (sched < 0 || erts_no_schedulers < sched)
goto error;
- erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_STATUS);
- old = BIF_P->bound_runq;
-#ifdef ERTS_SMP
- ASSERT(!old || old == BIF_P->run_queue);
-#endif
- new = !sched ? NULL : erts_schedid2runq(sched);
-#ifndef ERTS_SMP
- yield = 0;
-#else
- if (new == old)
- yield = 0;
+
+ if (sched == 0) {
+ new = NULL;
+ state = erts_smp_atomic32_read_band_mb(&BIF_P->state,
+ ~ERTS_PSFLG_BOUND);
+ }
else {
- ErtsRunQueue *curr = BIF_P->run_queue;
- if (!new)
- erts_smp_runq_lock(curr);
- else
- erts_smp_runqs_lock(curr, new);
- yield = new && BIF_P->run_queue != new;
-#endif
- BIF_P->bound_runq = new;
+ new = erts_schedid2runq(sched);
#ifdef ERTS_SMP
- if (new)
- BIF_P->run_queue = new;
- if (!new)
- erts_smp_runq_unlock(curr);
- else
- erts_smp_runqs_unlock(curr, new);
- }
+ erts_atomic_set_nob(&BIF_P->run_queue, (erts_aint_t) new);
#endif
- erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_STATUS);
+ state = erts_smp_atomic32_read_bor_mb(&BIF_P->state,
+ ERTS_PSFLG_BOUND);
+ }
+
+ curr = ERTS_GET_SCHEDULER_DATA_FROM_PROC(BIF_P)->run_queue;
+ old = (ERTS_PSFLG_BOUND & state) ? curr : NULL;
+
+ ASSERT(!old || old == curr);
+
old_value = old ? make_small(old->ix+1) : make_small(0);
- if (yield)
+ if (new && new != curr)
ERTS_BIF_YIELD_RETURN_X(BIF_P, old_value, am_scheduler);
else
BIF_RET(old_value);
@@ -1826,8 +1818,7 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend) {
if (internal_pid_index(to) >= erts_max_processes)
return SEND_BADARG;
- rp = erts_pid2proc_opt(p, ERTS_PROC_LOCK_MAIN,
- to, 0, ERTS_P2P_FLG_SMP_INC_REFC);
+ rp = erts_proc_lookup_raw(to);
if (!rp) {
ERTS_SMP_ASSERT_IS_NOT_EXITING(p);
@@ -1865,7 +1856,7 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend) {
}
erts_whereis_name(p, ERTS_PROC_LOCK_MAIN,
to,
- &rp, 0, ERTS_P2P_FLG_SMP_INC_REFC,
+ &rp, 0, 0,
&pt);
if (pt) {
@@ -2017,7 +2008,7 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend) {
erts_whereis_name(p, ERTS_PROC_LOCK_MAIN,
tp[1],
- &rp, 0, ERTS_P2P_FLG_SMP_INC_REFC,
+ &rp, 0, 0,
&pt);
if (pt) {
portid = pt->id;
@@ -2076,7 +2067,6 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend) {
p == rp
? (rp_locks & ~ERTS_PROC_LOCK_MAIN)
: rp_locks);
- erts_smp_proc_dec_refc(rp);
return res;
}
}
@@ -3486,7 +3476,7 @@ BIF_RETTYPE garbage_collect_1(BIF_ALIST_1)
if (rp == ERTS_PROC_LOCK_BUSY)
ERTS_BIF_YIELD1(bif_export[BIF_garbage_collect_1], BIF_P, BIF_ARG_1);
#else
- rp = erts_pid2proc(BIF_P, 0, BIF_ARG_1, 0);
+ rp = erts_proc_lookup(BIF_ARG_1);
#endif
if (!rp)
BIF_RET(am_false);
@@ -4233,8 +4223,8 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2)
erts_smp_thr_progress_block();
for (i = 0; i < erts_max_processes; i++) {
- if (process_tab[i] != (Process*) 0) {
- Process* p = process_tab[i];
+ Process *p = erts_pix2proc(i);
+ if (p) {
#ifdef USE_VM_PROBES
p->seq_trace_token = (p->dt_utag != NIL) ? am_have_dt_utag : NIL;
#else
diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index 39f91be7fc..0a51e00016 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -71,9 +71,10 @@ process_info(int to, void *to_arg)
{
int i;
for (i = 0; i < erts_max_processes; i++) {
- if ((process_tab[i] != NULL) && (process_tab[i]->i != ENULL)) {
- if (process_tab[i]->status != P_EXITING)
- print_process_info(to, to_arg, process_tab[i]);
+ Process *p = erts_pix2proc(i);
+ if (p && p->i != ENULL) {
+ if (!ERTS_PROC_IS_EXITING(p))
+ print_process_info(to, to_arg, p);
}
}
@@ -89,7 +90,8 @@ process_killer(void)
erts_printf("\n\nProcess Information\n\n");
erts_printf("--------------------------------------------------\n");
for (i = erts_max_processes-1; i >= 0; i--) {
- if (((rp = process_tab[i]) != NULL) && rp->i != ENULL) {
+ rp = erts_pix2proc(i);
+ if (rp && rp->i != ENULL) {
int br;
print_process_info(ERTS_PRINT_STDOUT, NULL, rp);
erts_printf("(k)ill (n)ext (r)eturn:\n");
@@ -97,11 +99,20 @@ process_killer(void)
if ((j = sys_get_key(0)) <= 0)
erl_exit(0, "");
switch(j) {
- case 'k':
- if (rp->status == P_WAITING) {
- ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND;
- erts_smp_proc_inc_refc(rp);
- erts_smp_proc_lock(rp, rp_locks);
+ case 'k': {
+ ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND;
+ erts_aint32_t state;
+ erts_smp_proc_inc_refc(rp);
+ erts_smp_proc_lock(rp, rp_locks);
+ state = erts_smp_atomic32_read_acqb(&rp->state);
+ if (state & (ERTS_PSFLG_FREE
+ | ERTS_PSFLG_EXITING
+ | ERTS_PSFLG_ACTIVE
+ | ERTS_PSFLG_IN_RUNQ
+ | ERTS_PSFLG_RUNNING)) {
+ erts_printf("Can only kill WAITING processes this way\n");
+ }
+ else {
(void) erts_send_exit_signal(NULL,
NIL,
rp,
@@ -110,12 +121,10 @@ process_killer(void)
NIL,
NULL,
0);
- erts_smp_proc_unlock(rp, rp_locks);
- erts_smp_proc_dec_refc(rp);
}
- else
- erts_printf("Can only kill WAITING processes this way\n");
-
+ erts_smp_proc_unlock(rp, rp_locks);
+ erts_smp_proc_dec_refc(rp);
+ }
case 'n': br = 1; break;
case 'r': return;
default: return;
@@ -180,42 +189,38 @@ static void doit_print_monitor(ErtsMonitor *mon, void *vpcontext)
void
print_process_info(int to, void *to_arg, Process *p)
{
+ time_t approx_started;
int garbing = 0;
int running = 0;
- time_t tmp_t;
struct saved_calls *scb;
+ erts_aint32_t state;
/* display the PID */
erts_print(to, to_arg, "=proc:%T\n", p->id);
/* Display the state */
erts_print(to, to_arg, "State: ");
- switch (p->status) {
- case P_FREE:
+
+ state = erts_smp_atomic32_read_acqb(&p->state);
+ if (state & ERTS_PSFLG_FREE)
erts_print(to, to_arg, "Non Existing\n"); /* Should never happen */
- break;
- case P_RUNABLE:
- erts_print(to, to_arg, "Scheduled\n");
- break;
- case P_WAITING:
- erts_print(to, to_arg, "Waiting\n");
- break;
- case P_SUSPENDED:
- erts_print(to, to_arg, "Suspended\n");
- break;
- case P_RUNNING:
- erts_print(to, to_arg, "Running\n");
- running = 1;
- break;
- case P_EXITING:
+ else if (state & ERTS_PSFLG_EXITING)
erts_print(to, to_arg, "Exiting\n");
- break;
- case P_GARBING:
- erts_print(to, to_arg, "Garbing\n");
+ else if (state & ERTS_PSFLG_GC) {
garbing = 1;
running = 1;
- break;
+ erts_print(to, to_arg, "Garbing\n");
+ }
+ else if (state & ERTS_PSFLG_SUSPENDED)
+ erts_print(to, to_arg, "Suspended\n");
+ else if (state & ERTS_PSFLG_RUNNING) {
+ running = 1;
+ erts_print(to, to_arg, "Running\n");
}
+ else if (state & ERTS_PSFLG_ACTIVE)
+ erts_print(to, to_arg, "Scheduled\n");
+ else
+ erts_print(to, to_arg, "Waiting\n");
/*
* If the process is registered as a global process, display the
@@ -245,8 +250,8 @@ print_process_info(int to, void *to_arg, Process *p)
}
erts_print(to, to_arg, "Spawned by: %T\n", p->parent);
- tmp_t = p->started.tv_sec;
- erts_print(to, to_arg, "Started: %s", ctime(&tmp_t));
+ approx_started = (time_t) p->approx_started;
+ erts_print(to, to_arg, "Started: %s", ctime(&approx_started));
ERTS_SMP_MSGQ_MV_INQ2PRIVQ(p);
erts_print(to, to_arg, "Message queue length: %d\n", p->msg.len);
@@ -623,7 +628,8 @@ bin_check(void)
int i, printed = 0;
for (i=0; i < erts_max_processes; i++) {
- if ((rp = process_tab[i]) == NULL)
+ rp = erts_pix2proc(i);
+ if (!rp)
continue;
for (hdr = rp->off_heap.first; hdr; hdr = hdr->next) {
if (hdr->thing_word == HEADER_PROC_BIN) {
@@ -711,7 +717,7 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
erts_print_nif_taints(fd, NULL);
erts_fdprintf(fd, "Atoms: %d\n", atom_table_size());
info(fd, NULL); /* General system info */
- if (process_tab != NULL) /* XXX true at init */
+ if (erts_proc.tab)
process_info(fd, NULL); /* Info about each process and port */
db_info(fd, NULL, 0);
erts_print_bif_timer_info(fd, NULL);
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 9e5a720f42..4b85909828 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -1312,7 +1312,7 @@ int erts_net_message(Port *prt,
if (is_not_pid(from) || is_not_atom(to)){
goto invalid_message;
}
- rp = erts_whereis_process(NULL, 0, to, 0, ERTS_P2P_FLG_SMP_INC_REFC);
+ rp = erts_whereis_process(NULL, 0, to, 0, 0);
if (rp) {
Uint xsize = (type == DOP_REG_SEND
? 0
@@ -1338,7 +1338,6 @@ int erts_net_message(Port *prt,
erts_queue_dist_message(rp, &locks, ede_copy, token);
if (locks)
erts_smp_proc_unlock(rp, locks);
- erts_smp_proc_dec_refc(rp);
}
break;
@@ -1364,7 +1363,7 @@ int erts_net_message(Port *prt,
if (is_not_pid(to)) {
goto invalid_message;
}
- rp = erts_pid2proc_opt(NULL, 0, to, 0, ERTS_P2P_FLG_SMP_INC_REFC);
+ rp = erts_proc_lookup(to);
if (rp) {
Uint xsize = type == DOP_SEND ? 0 : ERTS_HEAP_FRAG_SIZE(token_size);
ErtsProcLocks locks = 0;
@@ -1388,7 +1387,6 @@ int erts_net_message(Port *prt,
erts_queue_dist_message(rp, &locks, ede_copy, token);
if (locks)
erts_smp_proc_unlock(rp, locks);
- erts_smp_proc_dec_refc(rp);
}
break;
@@ -1544,8 +1542,7 @@ int erts_net_message(Port *prt,
if (is_not_pid(from) || is_not_internal_pid(to)) {
goto invalid_message;
}
- rp = erts_pid2proc_opt(NULL, 0, to, rp_locks,
- ERTS_P2P_FLG_SMP_INC_REFC);
+ rp = erts_pid2proc(NULL, 0, to, rp_locks);
if (rp) {
(void) erts_send_exit_signal(NULL,
from,
@@ -1556,7 +1553,6 @@ int erts_net_message(Port *prt,
NULL,
0);
erts_smp_proc_unlock(rp, rp_locks);
- erts_smp_proc_dec_refc(rp);
}
break;
}
@@ -2246,7 +2242,7 @@ static void doit_print_monitor_info(ErtsMonitor *mon, void *vptdp)
void *arg = ((struct print_to_data *) vptdp)->arg;
Process *rp;
ErtsMonitor *rmon;
- rp = erts_pid2proc_unlocked(mon->pid);
+ rp = erts_proc_lookup(mon->pid);
if (!rp || (rmon = erts_lookup_monitor(rp->monitors, mon->ref)) == NULL) {
erts_print(to, arg, "Warning, stray monitor for: %T\n", mon->pid);
} else if (mon->type == MON_ORIGIN) {
@@ -2285,7 +2281,7 @@ static void doit_print_link_info2(ErtsLink *lnk, void *vpplc)
static void doit_print_link_info(ErtsLink *lnk, void *vptdp)
{
- if (is_internal_pid(lnk->pid) && erts_pid2proc_unlocked(lnk->pid)) {
+ if (is_internal_pid(lnk->pid) && erts_proc_lookup(lnk->pid)) {
PrintLinkContext plc = {(struct print_to_data *) vptdp, lnk->pid};
erts_doforall_links(ERTS_LINK_ROOT(lnk), &doit_print_link_info2, &plc);
}
@@ -2307,7 +2303,7 @@ static void doit_print_nodelink_info(ErtsLink *lnk, void *vpcontext)
{
PrintNodeLinkContext *pcontext = vpcontext;
- if (is_internal_pid(lnk->pid) && erts_pid2proc_unlocked(lnk->pid))
+ if (is_internal_pid(lnk->pid) && erts_proc_lookup(lnk->pid))
erts_print(pcontext->ptd.to, pcontext->ptd.arg,
"Remote monitoring: %T %T\n", lnk->pid, pcontext->sysname);
}
@@ -2710,9 +2706,8 @@ BIF_RETTYPE dist_exit_3(BIF_ALIST_3)
}
else {
lp_locks = ERTS_PROC_LOCKS_XSIG_SEND;
- lp = erts_pid2proc_opt(BIF_P, ERTS_PROC_LOCK_MAIN,
- local, lp_locks,
- ERTS_P2P_FLG_SMP_INC_REFC);
+ lp = erts_pid2proc(BIF_P, ERTS_PROC_LOCK_MAIN,
+ local, lp_locks);
if (!lp) {
BIF_RET(am_true); /* ignore */
}
@@ -2731,9 +2726,7 @@ BIF_RETTYPE dist_exit_3(BIF_ALIST_3)
lp_locks &= ~ERTS_PROC_LOCK_MAIN;
#endif
erts_smp_proc_unlock(lp, lp_locks);
- if (lp != BIF_P)
- erts_smp_proc_dec_refc(lp);
- else {
+ if (lp == BIF_P) {
/*
* We may have exited current process and may have to take action.
*/
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index f04e9ab051..4c1424350f 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -265,6 +265,7 @@ type CPU_GRPS_MAP LONG_LIVED SYSTEM cpu_groups_map
type AUX_WORK_TMO LONG_LIVED SYSTEM aux_work_timeouts
type MISC_AUX_WORK_Q LONG_LIVED SYSTEM misc_aux_work_q
type CODE_IX_LOCK_Q SHORT_LIVED SYSTEM code_ix_lock_q
+type PROC_INTERVAL LONG_LIVED SYSTEM process_interval
+if threads_no_smp
# Need thread safe allocs, but std_alloc and fix_alloc are not;
@@ -337,6 +338,8 @@ type SL_PTIMER SHORT_LIVED SYSTEM ptimer_sl
type LL_PTIMER STANDARD SYSTEM ptimer_ll
type SYS_MSG_Q SHORT_LIVED PROCESSES system_messages_queue
type FP_EXCEPTION LONG_LIVED SYSTEM fp_exception
+type LL_MPATHS LONG_LIVED SYSTEM ll_migration_paths
+type SL_MPATHS SHORT_LIVED SYSTEM sl_migration_paths
+endif
+if hipe
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index e0d525bdde..62225d3572 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -832,14 +832,14 @@ init_dd_queue(ErtsAllctrDDQueue_t *ddq)
static ERTS_INLINE erts_aint_t
ddq_managed_thread_enqueue(ErtsAllctrDDQueue_t *ddq, void *ptr)
{
- erts_aint_t ilast, itmp;
+ erts_aint_t first_ilast, ilast, itmp;
ErtsAllctrDDBlock_t *this = ptr;
erts_atomic_init_nob(&this->atmc_next, ERTS_AINT_NULL);
/* Enqueue at end of list... */
- ilast = erts_atomic_read_nob(&ddq->tail.data.last);
+ first_ilast = ilast = erts_atomic_read_nob(&ddq->tail.data.last);
while (1) {
ErtsAllctrDDBlock_t *last = (ErtsAllctrDDBlock_t *) ilast;
itmp = erts_atomic_cmpxchg_mb(&last->atmc_next,
@@ -853,8 +853,11 @@ ddq_managed_thread_enqueue(ErtsAllctrDDQueue_t *ddq, void *ptr)
/* Move last pointer forward... */
while (1) {
if (erts_atomic_read_rb(&this->atmc_next) != ERTS_AINT_NULL) {
- /* Someone else will move it forward */
- return erts_atomic_read_rb(&ddq->tail.data.last);
+ ilast = erts_atomic_read_rb(&ddq->tail.data.last);
+ if (first_ilast != ilast) {
+ /* Someone else will move it forward */
+ return ilast;
+ }
}
itmp = erts_atomic_cmpxchg_mb(&ddq->tail.data.last,
(erts_aint_t) this,
diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c
index c338ee1c4b..7f7c975e78 100644
--- a/erts/emulator/beam/erl_bif_ddll.c
+++ b/erts/emulator/beam/erl_bif_ddll.c
@@ -368,13 +368,11 @@ BIF_RETTYPE erl_ddll_try_load_3(BIF_ALIST_3)
#endif
for (j = 0; j < erts_max_ports; j++) {
Port* prt = &erts_port[j];
-#ifdef DDLL_SMP
erts_smp_port_state_lock(prt);
-#endif
if (!(prt->status & FREE_PORT_FLAGS) &&
prt->drv_ptr->handle == dh) {
-#if DDLL_SMP
erts_smp_atomic_inc_nob(&prt->refc);
+#if DDLL_SMP
/* Extremely rare spinlock */
while(prt->status & ERTS_PORT_SFLG_INITIALIZING) {
erts_smp_port_state_unlock(prt);
@@ -598,13 +596,11 @@ done:
#endif
for (j = 0; j < erts_max_ports; j++) {
Port* prt = &erts_port[j];
-#if DDLL_SMP
erts_smp_port_state_lock(prt);
-#endif
if (!(prt->status & FREE_PORT_FLAGS)
&& prt->drv_ptr->handle == dh) {
-#if DDLL_SMP
erts_smp_atomic_inc_nob(&prt->refc);
+#if DDLL_SMP
/* Extremely rare spinlock */
while(prt->status & ERTS_PORT_SFLG_INITIALIZING) {
erts_smp_port_state_unlock(prt);
@@ -1060,13 +1056,11 @@ void erts_ddll_proc_dead(Process *p, ErtsProcLocks plocks)
#endif
for (j = 0; j < erts_max_ports; j++) {
Port* prt = &erts_port[j];
-#if DDLL_SMP
erts_smp_port_state_lock(prt);
-#endif
if (!(prt->status & FREE_PORT_FLAGS) &&
prt->drv_ptr->handle == dh) {
-#if DDLL_SMP
erts_smp_atomic_inc_nob(&prt->refc);
+#if DDLL_SMP
while(prt->status & ERTS_PORT_SFLG_INITIALIZING) {
erts_smp_port_state_unlock(prt);
erts_smp_port_state_lock(prt);
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 2373dc7af4..eb98d2f6dd 100755
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -1355,13 +1355,15 @@ process_info_aux(Process *BIF_P,
hp = HAlloc(BIF_P, 3);
break;
- case am_trap_exit:
+ case am_trap_exit: {
+ erts_aint32_t state = erts_smp_atomic32_read_nob(&rp->state);
hp = HAlloc(BIF_P, 3);
- if (rp->flags & F_TRAPEXIT)
+ if (state & ERTS_PSFLG_TRAP_EXIT)
res = am_true;
else
res = am_false;
break;
+ }
case am_error_handler:
hp = HAlloc(BIF_P, 3);
@@ -3153,15 +3155,13 @@ BIF_RETTYPE is_process_alive_1(BIF_ALIST_1)
if(internal_pid_index(BIF_ARG_1) >= erts_max_processes)
BIF_ERROR(BIF_P, BADARG);
- rp = erts_pid2proc(BIF_P, ERTS_PROC_LOCK_MAIN,
- BIF_ARG_1, ERTS_PROC_LOCK_STATUS);
+ rp = erts_proc_lookup(BIF_ARG_1);
if (!rp) {
BIF_RET(am_false);
}
else {
- int have_pending_exit = ERTS_PROC_PENDING_EXIT(rp);
- erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS);
- if (have_pending_exit)
+ if (erts_smp_atomic32_read_acqb(&rp->state)
+ & (ERTS_PSFLG_PENDING_EXIT|ERTS_PSFLG_EXITING))
ERTS_BIF_AWAIT_X_DATA_TRAP(BIF_P, BIF_ARG_1, am_false);
else
BIF_RET(am_true);
@@ -3765,9 +3765,8 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
&& is_internal_pid(tp[2])) {
int xres;
ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND;
- Process *rp = erts_pid2proc_opt(BIF_P, ERTS_PROC_LOCK_MAIN,
- tp[2], rp_locks,
- ERTS_P2P_FLG_SMP_INC_REFC);
+ Process *rp = erts_pid2proc(BIF_P, ERTS_PROC_LOCK_MAIN,
+ tp[2], rp_locks);
if (!rp) {
DECL_AM(dead);
BIF_RET(AM_dead);
@@ -3793,7 +3792,6 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
rp_locks &= ~ERTS_PROC_LOCK_MAIN;
#endif
erts_smp_proc_unlock(rp, rp_locks);
- erts_smp_proc_dec_refc(rp);
if (xres > 1) {
DECL_AM(message);
BIF_RET(AM_message);
@@ -4004,7 +4002,7 @@ static Eterm lcnt_build_lock_term(Eterm **hpp, Uint *szp, erts_lcnt_lock_t *lock
id = am_atom_put(ltype, strlen(ltype));
} else if (lock->flag & ERTS_LCNT_LT_PROCLOCK) {
/* use registered names as id's for process locks if available */
- proc = erts_pid2proc_unlocked(lock->id);
+ proc = erts_proc_lookup(lock->id);
if (proc && proc->reg) {
id = proc->reg->name;
} else {
diff --git a/erts/emulator/beam/erl_bif_timer.c b/erts/emulator/beam/erl_bif_timer.c
index d806be0704..525b11f61c 100644
--- a/erts/emulator/beam/erl_bif_timer.c
+++ b/erts/emulator/beam/erl_bif_timer.c
@@ -324,10 +324,9 @@ bif_timer_timeout(ErtsBifTimer* btm)
ASSERT(!erts_get_current_process());
if (btm->flags & BTM_FLG_BYNAME)
- rp = erts_whereis_process(NULL,0,btm->receiver.name,0,ERTS_P2P_FLG_SMP_INC_REFC);
+ rp = erts_whereis_process(NULL, 0, btm->receiver.name, 0, 0);
else {
rp = btm->receiver.proc.ess;
- erts_smp_proc_inc_refc(rp);
unlink_proc(btm);
}
@@ -379,7 +378,6 @@ bif_timer_timeout(ErtsBifTimer* btm)
#endif
);
erts_smp_proc_unlock(rp, rp_locks);
- erts_smp_proc_dec_refc(rp);
}
}
diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c
index 4cbfd7bab5..7f1b02b9b4 100644
--- a/erts/emulator/beam/erl_bif_trace.c
+++ b/erts/emulator/beam/erl_bif_trace.c
@@ -655,8 +655,7 @@ Eterm trace_3(BIF_ALIST_3)
if (procs || mods) {
/* tracing of processes */
for (i = 0; i < erts_max_processes; i++) {
- Process* tracee_p = process_tab[i];
-
+ Process* tracee_p = erts_pix2proc(i);
if (! tracee_p)
continue;
if (tracer != NIL) {
@@ -776,8 +775,7 @@ static int port_already_traced(Process *c_p, Port *tracee_port, Eterm tracer)
return 1;
}
else if(is_internal_pid(tracee_port->tracer_proc)) {
- Process *tracer_p = erts_pid2proc(c_p, ERTS_PROC_LOCK_MAIN,
- tracee_port->tracer_proc, 0);
+ Process *tracer_p = erts_proc_lookup(tracee_port->tracer_proc);
if (!tracer_p) {
/* Current trace process now invalid
* - discard it and approve the new. */
@@ -817,8 +815,7 @@ static int already_traced(Process *c_p, Process *tracee_p, Eterm tracer)
return 1;
}
else if(is_internal_pid(tracee_p->tracer_proc)) {
- Process *tracer_p = erts_pid2proc(c_p, ERTS_PROC_LOCK_MAIN,
- tracee_p->tracer_proc, 0);
+ Process *tracer_p = erts_proc_lookup(tracee_p->tracer_proc);
if (!tracer_p) {
/* Current trace process now invalid
* - discard it and approve the new. */
@@ -880,7 +877,7 @@ trace_info_pid(Process* p, Eterm pid_spec, Eterm key)
}
if (is_internal_pid(tracer)) {
- if (!erts_pid2proc(p, ERTS_PROC_LOCK_MAIN, tracer, 0)) {
+ if (!erts_proc_lookup(tracer)) {
reset_tracer:
tracee->trace_flags &= ~TRACEE_FLAGS;
trace_flags = tracee->trace_flags;
@@ -2157,9 +2154,9 @@ BIF_RETTYPE system_profile_2(BIF_ALIST_2)
/* Check if valid process, no locks are taken */
if (is_internal_pid(profiler)) {
- if (internal_pid_index(profiler) >= erts_max_processes) goto error;
- profiler_p = process_tab[internal_pid_index(profiler)];
- if (INVALID_PID(profiler_p, profiler)) goto error;
+ profiler_p = erts_proc_lookup(profiler);
+ if (!profiler_p)
+ goto error;
} else if (is_internal_port(profiler)) {
if (internal_port_index(profiler) >= erts_max_ports) goto error;
profiler_port = &erts_port[internal_port_index(profiler)];
diff --git a/erts/emulator/beam/erl_cpu_topology.c b/erts/emulator/beam/erl_cpu_topology.c
index fe3693d0ca..41e71881a0 100644
--- a/erts/emulator/beam/erl_cpu_topology.c
+++ b/erts/emulator/beam/erl_cpu_topology.c
@@ -486,7 +486,7 @@ erts_sched_check_cpu_bind_post_suspend(ErtsSchedulerData *esdp)
erts_thr_set_main_status(1, (int) esdp->no);
/* Make sure we check if we should bind to a cpu or not... */
- esdp->run_queue->flags |= ERTS_RUNQ_FLG_CHK_CPU_BIND;
+ ERTS_RUNQ_FLGS_SET(esdp->run_queue, ERTS_RUNQ_FLG_CHK_CPU_BIND);
}
#endif
@@ -498,9 +498,6 @@ erts_sched_check_cpu_bind(ErtsSchedulerData *esdp)
erts_cpu_groups_map_t *cgm;
erts_cpu_groups_callback_list_t *cgcl;
erts_cpu_groups_callback_call_t *cgcc;
-#ifdef ERTS_SMP
- esdp->run_queue->flags &= ~ERTS_RUNQ_FLG_CHK_CPU_BIND;
-#endif
erts_smp_runq_unlock(esdp->run_queue);
erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx);
cpu_id = scheduler2cpu_map[esdp->no].bind_id;
diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index 5cbfa65378..5bd8163968 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -2849,7 +2849,7 @@ void init_db(void)
else
db_max_tabs = user_requested_db_max_tabs;
- bits = erts_fit_in_bits(db_max_tabs-1);
+ bits = erts_fit_in_bits_int32(db_max_tabs-1);
if (bits > SMALL_BITS) {
erl_exit(1,"Max limit for ets tabled too high %u (max %u).",
db_max_tabs, ((Uint)1)<<SMALL_BITS);
@@ -3098,7 +3098,7 @@ retry:
if (to_proc == NULL) {
return 0; /* heir not alive, table still mine */
}
- if (erts_cmp_timeval(&to_proc->started, &tb->common.heir_started) != 0) {
+ if (to_proc->started_interval != tb->common.heir_started_interval) {
erts_smp_proc_unlock(to_proc, to_locks);
return 0; /* heir dead and pid reused, table still mine */
}
@@ -3494,14 +3494,14 @@ static void set_heir(Process* me, DbTable* tb, Eterm heir, UWord heir_data)
return;
}
if (heir == me->id) {
- tb->common.heir_started = me->started;
+ erts_ensure_later_proc_interval(me->started_interval);
+ tb->common.heir_started_interval = me->started_interval;
}
else {
- Process* heir_proc= erts_pid2proc_opt(me, ERTS_PROC_LOCK_MAIN, heir,
- 0, ERTS_P2P_FLG_SMP_INC_REFC);
+ Process* heir_proc= erts_proc_lookup(heir);
if (heir_proc != NULL) {
- tb->common.heir_started = heir_proc->started;
- erts_smp_proc_dec_refc(heir_proc);
+ erts_ensure_later_proc_interval(heir_proc->started_interval);
+ tb->common.heir_started_interval = heir_proc->started_interval;
} else {
tb->common.heir = am_none;
}
diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h
index 6a96e174e1..ff5982640d 100644
--- a/erts/emulator/beam/erl_db_util.h
+++ b/erts/emulator/beam/erl_db_util.h
@@ -219,7 +219,7 @@ typedef struct db_table_common {
Eterm owner; /* Pid of the creator */
Eterm heir; /* Pid of the heir */
UWord heir_data; /* To send in ETS-TRANSFER (is_immed or (DbTerm*) */
- SysTimeval heir_started; /* To further identify the heir */
+ Uint64 heir_started_interval; /* To further identify the heir */
Eterm the_name; /* an atom */
Eterm id; /* atom | integer */
DbTableMethod* meth; /* table methods */
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 52a6e52e6c..6075a527c3 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -357,11 +357,7 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj)
trace_gc(p, am_gc_start);
}
- erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
- p->gcstatus = p->status;
- p->status = P_GARBING;
- erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
-
+ erts_smp_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC);
if (erts_system_monitor_long_gc != 0) {
get_now(&ms1, &s1, &us1);
}
@@ -404,9 +400,8 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj)
ErtsGcQuickSanityCheck(p);
- erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
- p->status = p->gcstatus;
- erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+ erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC);
+
if (IS_TRACED_FL(p, F_TRACE_GC)) {
trace_gc(p, am_gc_end);
}
@@ -490,10 +485,7 @@ erts_garbage_collect_hibernate(Process* p)
/*
* Preliminaries.
*/
- erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
- p->gcstatus = p->status;
- p->status = P_GARBING;
- erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+ erts_smp_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC);
ErtsGcQuickSanityCheck(p);
ASSERT(p->mbuf_sz == 0);
ASSERT(p->mbuf == 0);
@@ -604,9 +596,7 @@ erts_garbage_collect_hibernate(Process* p)
ErtsGcQuickSanityCheck(p);
- erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
- p->status = p->gcstatus;
- erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+ erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC);
}
@@ -630,10 +620,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals,
/*
* Set GC state.
*/
- erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
- p->gcstatus = p->status;
- p->status = P_GARBING;
- erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+ erts_smp_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC);
/*
* We assume that the caller has already done a major collection
@@ -775,9 +762,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals,
/*
* Restore status.
*/
- erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
- p->status = p->gcstatus;
- erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+ erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC);
}
static int
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index f56ccac33d..51d3153a3b 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -56,6 +56,61 @@
#endif
/*
+ * The variables below (prefixed with etp_) are for erts/etc/unix/etp-commands
+ * only. Do not remove even though they aren't used elsewhere in the emulator!
+ */
+#ifdef ERTS_SMP
+const int etp_smp_compiled = 1;
+#else
+const int etp_smp_compiled = 0;
+#endif
+#ifdef USE_THREADS
+const int etp_thread_compiled = 1;
+#else
+const int etp_thread_compiled = 0;
+#endif
+const char etp_erts_version[] = ERLANG_VERSION;
+const char etp_otp_release[] = ERLANG_OTP_RELEASE;
+const char etp_compile_date[] = ERLANG_COMPILE_DATE;
+const char etp_arch[] = ERLANG_ARCHITECTURE;
+#ifdef ERTS_ENABLE_KERNEL_POLL
+const int etp_kernel_poll_support = 1;
+#else
+const int etp_kernel_poll_support = 0;
+#endif
+#if defined(ARCH_64)
+const int etp_arch_bits = 64;
+#elif defined(ARCH_32)
+const int etp_arch_bits = 32;
+#else
+# error "Not 64-bit, nor 32-bit arch"
+#endif
+#if HALFWORD_HEAP
+const int etp_halfword = 1;
+#else
+const int etp_halfword = 0;
+#endif
+#ifdef HIPE
+const int etp_hipe = 1;
+#else
+const int etp_hipe = 0;
+#endif
+#ifdef DEBUG
+const int etp_debug_compiled = 1;
+#else
+const int etp_debug_compiled = 0;
+#endif
+#ifdef ERTS_ENABLE_LOCK_COUNT
+const int etp_lock_count = 1;
+#else
+const int etp_lock_count = 0;
+#endif
+#ifdef ERTS_ENABLE_LOCK_CHECK
+const int etp_lock_check = 1;
+#else
+const int etp_lock_check = 0;
+#endif
+/*
* Note about VxWorks: All variables must be initialized by executable code,
* not by an initializer. Otherwise a new instance of the emulator will
* inherit previous values.
@@ -616,6 +671,7 @@ early_init(int *argc, char **argv) /*
erts_printf_eterm_func = erts_printf_term;
erts_disable_tolerant_timeofday = 0;
display_items = 200;
+ erts_proc.max = ERTS_DEFAULT_MAX_PROCESSES;
erts_backtrace_depth = DEFAULT_BACKTRACE_SIZE;
erts_async_max_threads = 0;
erts_async_thread_suggested_stack_size = ERTS_ASYNC_THREAD_MIN_STACK_SIZE;
@@ -1162,7 +1218,7 @@ erl_start(int argc, char **argv)
case 'P':
/* set maximum number of processes */
Parg = get_arg(argv[i]+2, argv[i+1], &i);
- erts_max_processes = atoi(Parg);
+ erts_proc.max = atoi(Parg);
/* Check of result is delayed until later. This is because +R
may be given after +P. */
break;
@@ -1441,10 +1497,10 @@ erl_start(int argc, char **argv)
}
/* Delayed check of +P flag */
- if (erts_max_processes < ERTS_MIN_PROCESSES
- || erts_max_processes > ERTS_MAX_PROCESSES
+ if (erts_proc.max < ERTS_MIN_PROCESSES
+ || erts_proc.max > ERTS_MAX_PROCESSES
|| (erts_use_r9_pids_ports
- && erts_max_processes > ERTS_MAX_R9_PROCESSES)) {
+ && erts_proc.max > ERTS_MAX_R9_PROCESSES)) {
erts_fprintf(stderr, "bad number of processes %s\n", Parg);
erts_usage();
}
diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c
index c4fe64c566..95efd9efc3 100644
--- a/erts/emulator/beam/erl_lock_check.c
+++ b/erts/emulator/beam/erl_lock_check.c
@@ -82,7 +82,6 @@ static erts_lc_lock_order_t erts_lock_order[] = {
#ifdef ERTS_SMP
{ "bif_timers", NULL },
{ "reg_tab", NULL },
- { "migration_info_update", NULL },
{ "proc_main", "pid" },
{ "old_code", "address" },
#ifdef HIPE
@@ -126,6 +125,7 @@ static erts_lc_lock_order_t erts_lock_order[] = {
{ "removed_fd_pre_alloc_lock", "address" },
{ "state_prealloc", NULL },
{ "schdlr_sspnd", NULL },
+ { "migration_info_update", NULL },
{ "run_queue", "address" },
{ "cpu_info", NULL },
{ "pollset", "address" },
diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index bd86e3ea9e..b10964da52 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -297,38 +297,6 @@ erts_msg_distext2heap(Process *pp,
return THE_NON_VALUE;
}
-static ERTS_INLINE void
-notify_new_message(Process *receiver)
-{
- ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS
- & erts_proc_lc_my_proc_locks(receiver));
-
- ACTIVATE(receiver);
-
- switch (receiver->status) {
- case P_GARBING:
- switch (receiver->gcstatus) {
- case P_SUSPENDED:
- goto suspended;
- case P_WAITING:
- goto waiting;
- default:
- break;
- }
- break;
- case P_SUSPENDED:
- suspended:
- receiver->rstatus = P_RUNABLE;
- break;
- case P_WAITING:
- waiting:
- erts_add_to_runq(receiver);
- break;
- default:
- break;
- }
-}
-
void
erts_queue_dist_message(Process *rcvr,
ErtsProcLocks *rcvr_locks,
@@ -342,7 +310,7 @@ erts_queue_dist_message(Process *rcvr,
Sint tok_serial = 0;
#endif
#ifdef ERTS_SMP
- ErtsProcLocks need_locks;
+ erts_aint_t state;
#endif
ERTS_SMP_LC_ASSERT(*rcvr_locks == erts_proc_lc_my_proc_locks(rcvr));
@@ -350,20 +318,21 @@ erts_queue_dist_message(Process *rcvr,
mp = message_alloc();
#ifdef ERTS_SMP
- need_locks = ~(*rcvr_locks) & (ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
- if (need_locks) {
- *rcvr_locks |= need_locks;
- if (erts_smp_proc_trylock(rcvr, need_locks) == EBUSY) {
- if (need_locks == ERTS_PROC_LOCK_MSGQ) {
+ if (!(*rcvr_locks & ERTS_PROC_LOCK_MSGQ)) {
+ if (erts_smp_proc_trylock(rcvr, ERTS_PROC_LOCK_MSGQ) == EBUSY) {
+ ErtsProcLocks need_locks = ERTS_PROC_LOCK_MSGQ;
+ if (*rcvr_locks & ERTS_PROC_LOCK_STATUS) {
erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_STATUS);
- need_locks = (ERTS_PROC_LOCK_MSGQ
- | ERTS_PROC_LOCK_STATUS);
+ need_locks |= ERTS_PROC_LOCK_STATUS;
}
erts_smp_proc_lock(rcvr, need_locks);
}
}
- if (rcvr->is_exiting || ERTS_PROC_PENDING_EXIT(rcvr)) {
+ state = erts_smp_atomic32_read_acqb(&rcvr->state);
+ if (state & (ERTS_PSFLG_PENDING_EXIT|ERTS_PSFLG_EXITING)) {
+ if (!(*rcvr_locks & ERTS_PROC_LOCK_MSGQ))
+ erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ);
/* Drop message if receiver is exiting or has a pending exit ... */
if (is_not_nil(token)) {
ErlHeapFragment *heap_frag;
@@ -379,6 +348,8 @@ erts_queue_dist_message(Process *rcvr,
/* Ahh... need to decode it in order to trace it... */
ErlHeapFragment *mbuf;
Eterm msg;
+ if (!(*rcvr_locks & ERTS_PROC_LOCK_MSGQ))
+ erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ);
message_free(mp);
msg = erts_msg_distext2heap(rcvr, rcvr_locks, &mbuf, &token, dist_ext);
if (is_value(msg))
@@ -440,26 +411,32 @@ erts_queue_dist_message(Process *rcvr,
mp->data.dist_ext = dist_ext;
LINK_MESSAGE(rcvr, mp);
- notify_new_message(rcvr);
+ if (!(*rcvr_locks & ERTS_PROC_LOCK_MSGQ))
+ erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ);
+
+ erts_proc_notify_new_message(rcvr);
}
}
/* Add a message last in message queue */
-void
-erts_queue_message(Process* receiver,
- ErtsProcLocks *receiver_locks,
- ErlHeapFragment* bp,
- Eterm message,
- Eterm seq_trace_token
+static int
+queue_message(Process *c_p,
+ Process* receiver,
+ ErtsProcLocks *receiver_locks,
+ erts_aint32_t *receiver_state,
+ ErlHeapFragment* bp,
+ Eterm message,
+ Eterm seq_trace_token
#ifdef USE_VM_PROBES
, Eterm dt_utag
#endif
-)
+ )
{
ErlMessage* mp;
-#ifdef ERTS_SMP
- ErtsProcLocks need_locks;
-#else
+ int locked_msgq = 0;
+ erts_aint_t state;
+
+#ifndef ERTS_SMP
ASSERT(bp != NULL || receiver->mbuf == NULL);
#endif
@@ -467,31 +444,45 @@ erts_queue_message(Process* receiver,
mp = message_alloc();
+ if (receiver_state)
+ state = *receiver_state;
+ else
+ state = erts_smp_atomic32_read_acqb(&receiver->state);
+
#ifdef ERTS_SMP
- need_locks = ~(*receiver_locks) & (ERTS_PROC_LOCK_MSGQ
- | ERTS_PROC_LOCK_STATUS);
- if (need_locks) {
- *receiver_locks |= need_locks;
- if (erts_smp_proc_trylock(receiver, need_locks) == EBUSY) {
- if (need_locks == ERTS_PROC_LOCK_MSGQ) {
+
+ if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT))
+ goto exiting;
+
+ if (!(*receiver_locks & ERTS_PROC_LOCK_MSGQ)) {
+ if (erts_smp_proc_trylock(receiver, ERTS_PROC_LOCK_MSGQ) == EBUSY) {
+ ErtsProcLocks need_locks = ERTS_PROC_LOCK_MSGQ;
+ if (*receiver_locks & ERTS_PROC_LOCK_STATUS) {
erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_STATUS);
- need_locks = (ERTS_PROC_LOCK_MSGQ
- | ERTS_PROC_LOCK_STATUS);
+ need_locks |= ERTS_PROC_LOCK_STATUS;
}
erts_smp_proc_lock(receiver, need_locks);
}
+ locked_msgq = 1;
+ state = erts_smp_atomic32_read_nob(&receiver->state);
+ if (receiver_state)
+ *receiver_state = state;
}
- if (receiver->is_exiting || ERTS_PROC_PENDING_EXIT(receiver)) {
- /* Drop message if receiver is exiting or has a pending
- * exit ...
- */
+#endif
+
+ if (state & (ERTS_PSFLG_PENDING_EXIT|ERTS_PSFLG_EXITING)) {
+#ifdef ERTS_SMP
+ exiting:
+#endif
+ /* Drop message if receiver is exiting or has a pending exit... */
+ if (locked_msgq)
+ erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_MSGQ);
if (bp)
free_message_buffer(bp);
message_free(mp);
- return;
+ return 0;
}
-#endif
ERL_MESSAGE_TERM(mp) = message;
ERL_MESSAGE_TOKEN(mp) = seq_trace_token;
@@ -514,12 +505,11 @@ erts_queue_message(Process* receiver,
ERTS_SMP_MSGQ_MV_INQ2PRIVQ(receiver);
LINK_MESSAGE_PRIVQ(receiver, mp);
}
- else {
+ else
+#endif
+ {
LINK_MESSAGE(receiver, mp);
}
-#else
- LINK_MESSAGE(receiver, mp);
-#endif
#ifdef USE_VM_PROBES
if (DTRACE_ENABLED(message_queued)) {
@@ -539,15 +529,43 @@ erts_queue_message(Process* receiver,
tok_label, tok_lastcnt, tok_serial);
}
#endif
- notify_new_message(receiver);
- if (IS_TRACED_FL(receiver, F_TRACE_RECEIVE)) {
+ if (IS_TRACED_FL(receiver, F_TRACE_RECEIVE))
trace_receive(receiver, message);
- }
+
+ if (locked_msgq)
+ erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_MSGQ);
+
+ erts_proc_notify_new_message(receiver);
#ifndef ERTS_SMP
ERTS_HOLE_CHECK(receiver);
#endif
+ return 0;
+}
+
+void
+erts_queue_message(Process* receiver,
+ ErtsProcLocks *receiver_locks,
+ ErlHeapFragment* bp,
+ Eterm message,
+ Eterm seq_trace_token
+#ifdef USE_VM_PROBES
+ , Eterm dt_utag
+#endif
+ )
+{
+ queue_message(NULL,
+ receiver,
+ receiver_locks,
+ NULL,
+ bp,
+ message,
+ seq_trace_token
+#ifdef USE_VM_PROBES
+ , dt_utag
+#endif
+ );
}
void
@@ -1025,11 +1043,8 @@ erts_send_message(Process* sender,
LINK_MESSAGE(receiver, mp);
ACTIVATE(receiver);
- if (receiver->status == P_WAITING) {
- erts_add_to_runq(receiver);
- } else if (receiver->status == P_SUSPENDED) {
- receiver->rstatus = P_RUNABLE;
- }
+ erts_proc_notify_new_message(receiver);
+
if (IS_TRACED_FL(receiver, F_TRACE_RECEIVE)) {
trace_receive(receiver, message);
}
@@ -1089,21 +1104,34 @@ erts_send_message(Process* sender,
#ifdef ERTS_SMP
ErlOffHeap *ohp;
Eterm *hp;
+ erts_aint32_t state;
+
BM_SWAP_TIMER(send,size);
msize = size_object(message);
BM_SWAP_TIMER(size,send);
- hp = erts_alloc_message_heap(msize,&bp,&ohp,receiver,receiver_locks);
+ hp = erts_alloc_message_heap_state(msize,
+ &bp,
+ &ohp,
+ receiver,
+ receiver_locks,
+ &state);
BM_SWAP_TIMER(send,copy);
message = copy_struct(message, msize, &hp, ohp);
BM_MESSAGE_COPIED(msz);
BM_SWAP_TIMER(copy,send);
DTRACE6(message_send, sender_name, receiver_name,
msize, tok_label, tok_lastcnt, tok_serial);
- erts_queue_message(receiver, receiver_locks, bp, message, token
+ queue_message(sender,
+ receiver,
+ receiver_locks,
+ &state,
+ bp,
+ message,
+ token
#ifdef USE_VM_PROBES
- , NIL
+ , NIL
#endif
- );
+ );
BM_SWAP_TIMER(send,system);
#else
ErlMessage* mp = message_alloc();
@@ -1134,17 +1162,13 @@ erts_send_message(Process* sender,
mp->data.attached = NULL;
LINK_MESSAGE(receiver, mp);
- if (receiver->status == P_WAITING) {
- erts_add_to_runq(receiver);
- } else if (receiver->status == P_SUSPENDED) {
- receiver->rstatus = P_RUNABLE;
- }
+ erts_proc_notify_new_message(receiver);
+
if (IS_TRACED_FL(receiver, F_TRACE_RECEIVE)) {
trace_receive(receiver, message);
}
BM_SWAP_TIMER(send,system);
#endif /* #ifndef ERTS_SMP */
- return;
#endif /* HYBRID */
}
}
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 9564d4168e..da4376fd0a 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -311,6 +311,7 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
#endif
Eterm receiver = to_pid->pid;
int flush_me = 0;
+ int scheduler = erts_get_scheduler_id() != 0;
if (env != NULL) {
c_p = env->proc;
@@ -330,8 +331,11 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP)
rp_had_locks = rp_locks;
#endif
- rp = erts_pid2proc_opt(c_p, ERTS_PROC_LOCK_MAIN,
- receiver, rp_locks, ERTS_P2P_FLG_SMP_INC_REFC);
+
+ rp = (scheduler
+ ? erts_proc_lookup(receiver)
+ : erts_pid2proc_opt(c_p, ERTS_PROC_LOCK_MAIN,
+ receiver, rp_locks, ERTS_P2P_FLG_SMP_INC_REFC));
if (rp == NULL) {
ASSERT(env == NULL || receiver != c_p->id);
return 0;
@@ -358,12 +362,12 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
, NIL
#endif
);
- if (rp_locks) {
- ERTS_SMP_LC_ASSERT(rp_locks == (rp_had_locks | (ERTS_PROC_LOCK_MSGQ |
- ERTS_PROC_LOCK_STATUS)));
- erts_smp_proc_unlock(rp, (ERTS_PROC_LOCK_MSGQ | ERTS_PROC_LOCK_STATUS));
- }
- erts_smp_proc_dec_refc(rp);
+ if (c_p == rp)
+ rp_locks &= ~ERTS_PROC_LOCK_MAIN;
+ if (rp_locks)
+ erts_smp_proc_unlock(rp, rp_locks);
+ if (!scheduler)
+ erts_smp_proc_dec_refc(rp);
if (flush_me) {
cache_env(env);
}
diff --git a/erts/emulator/beam/erl_node_container_utils.h b/erts/emulator/beam/erl_node_container_utils.h
index 329a2204cc..7b4cb7b042 100644
--- a/erts/emulator/beam/erl_node_container_utils.h
+++ b/erts/emulator/beam/erl_node_container_utils.h
@@ -128,8 +128,47 @@ extern int erts_use_r9_pids_ports;
* Pids *
\* */
-#define internal_pid_index(x) (internal_pid_data((x)) \
- & erts_process_tab_index_mask)
+#define erts_max_processes erts_proc.max
+
+typedef struct {
+ erts_smp_atomic_t *tab;
+ int max;
+ int tab_cache_lines;
+ int pix_per_cache_line;
+ int pix_cl_mask;
+ int pix_cl_shift;
+ int pix_cli_mask;
+ int pix_cli_shift;
+} ErtsProcTab;
+
+extern ErtsProcTab erts_proc;
+
+ERTS_GLB_INLINE int erts_pid_data2ix(Eterm pid_data);
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+ERTS_GLB_INLINE int erts_pid_data2ix(Eterm pid_data)
+{
+ int n, pix;
+
+ n = (int) pid_data;
+ if (erts_proc.pix_cl_mask) {
+ pix = ((n & erts_proc.pix_cl_mask) << erts_proc.pix_cl_shift);
+ pix += ((n >> erts_proc.pix_cli_shift) & erts_proc.pix_cli_mask);
+ }
+ else {
+ n %= erts_proc.max;
+ pix = n % erts_proc.tab_cache_lines;
+ pix *= erts_proc.pix_per_cache_line;
+ pix += n / erts_proc.tab_cache_lines;
+ }
+ ASSERT(0 <= pix && pix < erts_proc.max);
+ return pix;
+}
+
+#endif
+
+#define internal_pid_index(x) erts_pid_data2ix(internal_pid_data((x)))
#define internal_pid_node_name(x) (internal_pid_node((x))->sysname)
#define external_pid_node_name(x) (external_pid_node((x))->sysname)
diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index 1481f66b55..16367c305d 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -1310,21 +1310,22 @@ setup_reference_table(void)
UnUseTmpHeapNoproc(3);
/* Insert all processes */
- for (i = 0; i < erts_max_processes; i++)
- if (process_tab[i]) {
+ for (i = 0; i < erts_max_processes; i++) {
+ Process *proc = erts_pix2proc(i);
+ if (proc) {
ErlMessage *msg;
/* Insert Heap */
- insert_offheap(&(process_tab[i]->off_heap),
+ insert_offheap(&(proc->off_heap),
HEAP_REF,
- process_tab[i]->id);
+ proc->id);
/* Insert message buffers */
- for(hfp = process_tab[i]->mbuf; hfp; hfp = hfp->next)
+ for(hfp = proc->mbuf; hfp; hfp = hfp->next)
insert_offheap(&(hfp->off_heap),
HEAP_REF,
- process_tab[i]->id);
+ proc->id);
/* Insert msg msg buffers */
- for (msg = process_tab[i]->msg.first; msg; msg = msg->next) {
+ for (msg = proc->msg.first; msg; msg = msg->next) {
ErlHeapFragment *heap_frag = NULL;
if (msg->data.attached) {
if (is_value(ERL_MESSAGE_TERM(msg)))
@@ -1332,7 +1333,7 @@ setup_reference_table(void)
else {
if (msg->data.dist_ext->dep)
insert_dist_entry(msg->data.dist_ext->dep,
- HEAP_REF, process_tab[i]->id, 0);
+ HEAP_REF, proc->id, 0);
if (is_not_nil(ERL_MESSAGE_TOKEN(msg)))
heap_frag = erts_dist_ext_trailer(msg->data.dist_ext);
}
@@ -1340,10 +1341,10 @@ setup_reference_table(void)
if (heap_frag)
insert_offheap(&(heap_frag->off_heap),
HEAP_REF,
- process_tab[i]->id);
+ proc->id);
}
#ifdef ERTS_SMP
- for (msg = process_tab[i]->msg_inq.first; msg; msg = msg->next) {
+ for (msg = proc->msg_inq.first; msg; msg = msg->next) {
ErlHeapFragment *heap_frag = NULL;
if (msg->data.attached) {
if (is_value(ERL_MESSAGE_TERM(msg)))
@@ -1351,7 +1352,7 @@ setup_reference_table(void)
else {
if (msg->data.dist_ext->dep)
insert_dist_entry(msg->data.dist_ext->dep,
- HEAP_REF, process_tab[i]->id, 0);
+ HEAP_REF, proc->id, 0);
if (is_not_nil(ERL_MESSAGE_TOKEN(msg)))
heap_frag = erts_dist_ext_trailer(msg->data.dist_ext);
}
@@ -1359,21 +1360,22 @@ setup_reference_table(void)
if (heap_frag)
insert_offheap(&(heap_frag->off_heap),
HEAP_REF,
- process_tab[i]->id);
+ proc->id);
}
#endif
/* Insert links */
- if(process_tab[i]->nlinks)
- insert_links(process_tab[i]->nlinks, process_tab[i]->id);
- if(process_tab[i]->monitors)
- insert_monitors(process_tab[i]->monitors, process_tab[i]->id);
+ if(proc->nlinks)
+ insert_links(proc->nlinks, proc->id);
+ if(proc->monitors)
+ insert_monitors(proc->monitors, proc->id);
/* Insert controller */
{
- DistEntry *dep = ERTS_PROC_GET_DIST_ENTRY(process_tab[i]);
+ DistEntry *dep = ERTS_PROC_GET_DIST_ENTRY(proc);
if (dep)
- insert_dist_entry(dep, CTRL_REF, process_tab[i]->id, 0);
+ insert_dist_entry(dep, CTRL_REF, proc->id, 0);
}
}
+ }
#ifdef ERTS_SMP
erts_foreach_sys_msg_in_q(insert_sys_msg);
diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c
index 0f1a0d441a..86454fe1fa 100644
--- a/erts/emulator/beam/erl_port_task.c
+++ b/erts/emulator/beam/erl_port_task.c
@@ -56,12 +56,6 @@
#define ERTS_PORT_IS_IN_RUNQ(RQ, P) \
((P)->sched.next || (P)->sched.prev || (RQ)->ports.start == (P))
-#define ERTS_PORT_NOT_IN_RUNQ(P) \
-do { \
- (P)->sched.prev = NULL; \
- (P)->sched.next = NULL; \
-} while (0)
-
#ifdef USE_VM_PROBES
#define DTRACE_DRIVER(PROBE_NAME, PP) \
if (DTRACE_ENABLED(driver_ready_input)) { \
@@ -167,7 +161,7 @@ enqueue_port(ErtsRunQueue *runq, Port *pp)
{
ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq));
pp->sched.next = NULL;
- pp->sched.prev = runq->ports.end;
+ pp->sched.in_runq = 1;
if (runq->ports.end) {
ASSERT(runq->ports.start);
runq->ports.end->sched.next = pp;
@@ -177,39 +171,10 @@ enqueue_port(ErtsRunQueue *runq, Port *pp)
runq->ports.start = pp;
}
- runq->ports.info.len++;
- if (runq->ports.info.max_len < runq->ports.info.len)
- runq->ports.info.max_len = runq->ports.info.len;
- runq->len++;
- if (runq->max_len < runq->len)
- runq->max_len = runq->len;
runq->ports.end = pp;
ASSERT(runq->ports.start && runq->ports.end);
-}
-static ERTS_INLINE void
-dequeue_port(ErtsRunQueue *runq, Port *pp)
-{
- ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq));
- if (pp->sched.next)
- pp->sched.next->sched.prev = pp->sched.prev;
- else {
- ASSERT(runq->ports.end == pp);
- runq->ports.end = pp->sched.prev;
- }
- if (pp->sched.prev)
- pp->sched.prev->sched.next = pp->sched.next;
- else {
- ASSERT(runq->ports.start == pp);
- runq->ports.start = pp->sched.next;
- }
-
- ASSERT(runq->ports.info.len > 0);
- runq->ports.info.len--;
- ASSERT(runq->len > 0);
- runq->len--;
- ASSERT(runq->ports.start || !runq->ports.end);
- ASSERT(runq->ports.end || !runq->ports.start);
+ erts_smp_inc_runq_len(runq, &runq->ports.info, ERTS_PORT_PRIO_LEVEL);
}
static ERTS_INLINE Port *
@@ -222,16 +187,11 @@ pop_port(ErtsRunQueue *runq)
}
else {
runq->ports.start = runq->ports.start->sched.next;
- if (runq->ports.start)
- runq->ports.start->sched.prev = NULL;
- else {
+ if (!runq->ports.start) {
ASSERT(runq->ports.end == pp);
runq->ports.end = NULL;
}
- ASSERT(runq->ports.info.len > 0);
- runq->ports.info.len--;
- ASSERT(runq->len > 0);
- runq->len--;
+ erts_smp_dec_runq_len(runq, &runq->ports.info, ERTS_PORT_PRIO_LEVEL);
}
ASSERT(runq->ports.start || !runq->ports.end);
@@ -285,7 +245,7 @@ check_port_queue(ErtsRunQueue *runq, Port *chk_pp, int inq)
}
ASSERT(no_forward == no_backward);
}
- ASSERT(no_forward == runq->ports.info.len);
+ ASSERT(no_forward == RUNQ_READ_LEN(&runq->ports.info.len));
if (chk_pp) {
if (chk_pp->sched.taskq || chk_pp->sched.exe_taskq) {
ASSERT(chk_pp->sched.taskq != chk_pp->sched.exe_taskq);
@@ -467,10 +427,11 @@ erts_port_task_abort(Eterm id, ErtsPortTaskHandle *pthp)
ErtsPortTaskQueue *ptqp;
ErtsPortTask *ptp;
Port *pp;
- int port_is_dequeued = 0;
pp = &erts_port[internal_port_index(id)];
runq = erts_port_runq(pp);
+ if (!runq)
+ return 1;
ptp = handle2task(pthp);
@@ -505,22 +466,12 @@ erts_port_task_abort(Eterm id, ErtsPortTaskHandle *pthp)
if (ptqp->first || pp->sched.taskq != ptqp)
ptqp = NULL;
- else {
+ else
pp->sched.taskq = NULL;
- if (!pp->sched.exe_taskq) {
- dequeue_port(runq, pp);
- ERTS_PORT_NOT_IN_RUNQ(pp);
- port_is_dequeued = 1;
- }
- }
ERTS_PT_CHK_PRES_PORTQ(runq, pp);
erts_smp_runq_unlock(runq);
-
- if (erts_system_profile_flags.runnable_ports && port_is_dequeued) {
- profile_runnable_port(pp, am_inactive);
- }
port_task_free(ptp);
if (ptqp)
@@ -575,7 +526,7 @@ erts_port_task_schedule(Eterm id,
if (!pp->sched.taskq) {
pp->sched.taskq = port_taskq_init(port_taskq_alloc(), pp);
- enq_port = !pp->sched.exe_taskq;
+ enq_port = !pp->sched.in_runq && !pp->sched.exe_taskq;
}
#ifdef ERTS_SMP
@@ -585,13 +536,13 @@ erts_port_task_schedule(Eterm id,
/* Port emigrated ... */
erts_smp_atomic_set_nob(&pp->run_queue, (erts_aint_t) xrunq);
erts_smp_runq_unlock(runq);
- runq = xrunq;
+ runq = erts_port_runq(pp);
+ if (!runq)
+ return -1;
}
}
#endif
- ASSERT(!enq_port || !(runq->flags & ERTS_RUNQ_FLG_SUSPENDED));
-
ASSERT(pp->sched.taskq);
ASSERT(ptp);
@@ -633,7 +584,7 @@ erts_port_task_schedule(Eterm id,
#elif defined(DEBUG)
if (!enq_port && !pp->sched.exe_taskq) {
/* We should be in port run q */
- ASSERT(pp->sched.prev || runq->ports.start == pp);
+ ASSERT(pp->sched.in_runq);
}
#endif
#endif
@@ -661,63 +612,54 @@ void
erts_port_task_free_port(Port *pp)
{
ErtsRunQueue *runq;
- int port_is_dequeued = 0;
+ ErtsPortTaskQueue *ptqp;
ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(pp));
ASSERT(!(pp->status & ERTS_PORT_SFLGS_DEAD));
runq = erts_port_runq(pp);
ASSERT(runq);
ERTS_PT_CHK_PRES_PORTQ(runq, pp);
- if (pp->sched.exe_taskq) {
+ ptqp = pp->sched.exe_taskq;
+ if (ptqp) {
/* I (this thread) am currently executing this port, free it
when scheduled out... */
- ErtsPortTask *ptp = port_task_alloc();
+ ErtsPortTask *ptp;
+ enqueue_free:
+ ptp = port_task_alloc();
erts_smp_port_state_lock(pp);
pp->status &= ~ERTS_PORT_SFLG_CLOSING;
pp->status |= ERTS_PORT_SFLG_FREE_SCHEDULED;
erts_may_save_closed_port(pp);
erts_smp_port_state_unlock(pp);
- ERTS_SMP_LC_ASSERT(erts_smp_atomic_read_nob(&pp->refc) > 1);
+ ERTS_LC_ASSERT(erts_smp_atomic_read_nob(&pp->refc) > 1);
ptp->type = ERTS_PORT_TASK_FREE;
ptp->event = (ErlDrvEvent) -1;
ptp->event_data = NULL;
set_handle(ptp, NULL);
- push_task(pp->sched.exe_taskq, ptp);
+ push_task(ptqp, ptp);
ERTS_PT_CHK_PRES_PORTQ(runq, pp);
erts_smp_runq_unlock(runq);
}
else {
- ErtsPortTaskQueue *ptqp = pp->sched.taskq;
- if (ptqp) {
- dequeue_port(runq, pp);
- ERTS_PORT_NOT_IN_RUNQ(pp);
- port_is_dequeued = 1;
+ if (pp->sched.in_runq) {
+ ptqp = pp->sched.taskq;
+ if (!ptqp)
+ pp->sched.taskq = ptqp = port_taskq_init(port_taskq_alloc(), pp);
+ goto enqueue_free;
}
+ ASSERT(!pp->sched.taskq);
erts_smp_port_state_lock(pp);
pp->status &= ~ERTS_PORT_SFLG_CLOSING;
pp->status |= ERTS_PORT_SFLG_FREE_SCHEDULED;
erts_may_save_closed_port(pp);
erts_smp_port_state_unlock(pp);
-#ifdef ERTS_SMP
erts_smp_atomic_dec_nob(&pp->refc); /* Not alive */
-#endif
- ERTS_SMP_LC_ASSERT(erts_smp_atomic_read_nob(&pp->refc) > 0); /* Lock */
+ ERTS_LC_ASSERT(erts_smp_atomic_read_nob(&pp->refc) > 0); /* Lock */
handle_remaining_tasks(runq, pp); /* May release runq lock */
ASSERT(!pp->sched.exe_taskq && (!ptqp || !ptqp->first));
pp->sched.taskq = NULL;
ERTS_PT_CHK_PRES_PORTQ(runq, pp);
-#ifndef ERTS_SMP
- ASSERT(pp->status & ERTS_PORT_SFLG_PORT_DEBUG);
- erts_port_status_set(pp, ERTS_PORT_SFLG_FREE);
-#endif
erts_smp_runq_unlock(runq);
-
- if (erts_system_profile_flags.runnable_ports && port_is_dequeued) {
- profile_runnable_port(pp, am_inactive);
- }
-
- if (ptqp)
- port_taskq_free(ptqp);
}
}
@@ -738,7 +680,6 @@ typedef struct {
int
erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
{
- int port_was_enqueued = 0;
Port *pp;
ErtsPortTaskQueue *ptqp;
ErtsPortTask *ptp;
@@ -757,11 +698,18 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
goto done;
}
- ERTS_PORT_NOT_IN_RUNQ(pp);
+ ASSERT(pp->sched.in_runq);
+ pp->sched.in_runq = 0;
+ if (!pp->sched.taskq) {
+ if (erts_system_profile_flags.runnable_ports)
+ profile_runnable_port(pp, am_inactive);
+ res = (erts_smp_atomic_read_nob(&erts_port_task_outstanding_io_tasks)
+ != (erts_aint_t) 0);
+ goto done;
+ }
*curr_port_pp = pp;
- ASSERT(pp->sched.taskq);
ASSERT(pp->sched.taskq->first);
ptqp = pp->sched.taskq;
pp->sched.taskq = NULL;
@@ -823,12 +771,8 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
handle_remaining_tasks(runq, pp);
ASSERT(!ptqp->first
&& (!pp->sched.taskq || !pp->sched.taskq->first));
-#ifdef ERTS_SMP
erts_smp_atomic_dec_nob(&pp->refc); /* Not alive */
- ERTS_SMP_LC_ASSERT(erts_smp_atomic_read_nob(&pp->refc) > 0); /* Lock */
-#else
- erts_port_status_bor_set(pp, ERTS_PORT_SFLG_FREE);
-#endif
+ ERTS_LC_ASSERT(erts_smp_atomic_read_nob(&pp->refc) > 0); /* Lock */
port_task_free(ptp);
if (pp->sched.taskq)
@@ -918,6 +862,8 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
if (!pp->sched.taskq) {
ASSERT(pp->sched.exe_taskq);
pp->sched.exe_taskq = NULL;
+ if (erts_system_profile_flags.runnable_ports)
+ profile_runnable_port(pp, am_inactive);
}
else {
#ifdef ERTS_SMP
@@ -940,14 +886,20 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
else {
/* Port emigrated ... */
erts_smp_atomic_set_nob(&pp->run_queue, (erts_aint_t) xrunq);
- enqueue_port(xrunq, pp);
- ASSERT(pp->sched.exe_taskq);
- pp->sched.exe_taskq = NULL;
- erts_smp_runq_unlock(xrunq);
- erts_smp_notify_inc_runq(xrunq);
+ erts_smp_runq_unlock(runq);
+
+ xrunq = erts_port_runq(pp);
+ if (xrunq) {
+ enqueue_port(xrunq, pp);
+ ASSERT(pp->sched.exe_taskq);
+ pp->sched.exe_taskq = NULL;
+ erts_smp_runq_unlock(xrunq);
+ erts_smp_notify_inc_runq(xrunq);
+ }
+
+ erts_smp_runq_lock(runq);
}
#endif
- port_was_enqueued = 1;
}
res = (erts_smp_atomic_read_nob(&erts_port_task_outstanding_io_tasks)
@@ -957,10 +909,6 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
port_taskq_free(ptqp);
- if (erts_system_profile_flags.runnable_ports && (port_was_enqueued != 1)) {
- profile_runnable_port(pp, am_inactive);
- }
-
/* trace port scheduling, out */
if (IS_TRACED_FL(pp, F_TRACE_SCHED_PORTS)) {
trace_sched_ports(pp, am_out);
@@ -1049,63 +997,33 @@ erts_port_is_scheduled(Port *pp)
{
int res;
ErtsRunQueue *runq = erts_port_runq(pp);
+ if (!runq)
+ return 0;
res = pp->sched.taskq || pp->sched.exe_taskq;
erts_smp_runq_unlock(runq);
return res;
}
#ifdef ERTS_SMP
+void
+erts_enqueue_port(ErtsRunQueue *rq, Port *pp)
+{
+ ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
+ ASSERT(rq == (ErtsRunQueue *) erts_smp_atomic_read_nob(&pp->run_queue));
+ ASSERT(pp->sched.in_runq);
+ enqueue_port(rq, pp);
+}
-ErtsMigrateResult
-erts_port_migrate(Port *prt, int *prt_locked,
- ErtsRunQueue *from_rq, int *from_locked,
- ErtsRunQueue *to_rq, int *to_locked)
+Port *
+erts_dequeue_port(ErtsRunQueue *rq)
{
- ERTS_SMP_LC_ASSERT(*from_locked);
- ERTS_SMP_LC_CHK_RUNQ_LOCK(from_rq, *from_locked);
- ERTS_SMP_LC_CHK_RUNQ_LOCK(to_rq, *to_locked);
-
- if (!*from_locked || !*to_locked) {
- if (from_rq < to_rq) {
- if (!*to_locked) {
- if (!*from_locked)
- erts_smp_runq_lock(from_rq);
- erts_smp_runq_lock(to_rq);
- }
- else if (erts_smp_runq_trylock(from_rq) == EBUSY) {
- erts_smp_runq_unlock(to_rq);
- erts_smp_runq_lock(from_rq);
- erts_smp_runq_lock(to_rq);
- }
- }
- else {
- if (!*from_locked) {
- if (!*to_locked)
- erts_smp_runq_lock(to_rq);
- erts_smp_runq_lock(from_rq);
- }
- else if (erts_smp_runq_trylock(to_rq) == EBUSY) {
- erts_smp_runq_unlock(from_rq);
- erts_smp_runq_lock(to_rq);
- erts_smp_runq_lock(from_rq);
- }
- }
- *to_locked = *from_locked = 1;
- }
- ERTS_SMP_LC_CHK_RUNQ_LOCK(from_rq, *from_locked);
- ERTS_SMP_LC_CHK_RUNQ_LOCK(to_rq, *to_locked);
-
- /* Refuse to migrate to a suspended run queue */
- if (to_rq->flags & ERTS_RUNQ_FLG_SUSPENDED)
- return ERTS_MIGRATE_FAILED_RUNQ_SUSPENDED;
- if (from_rq != (ErtsRunQueue *) erts_smp_atomic_read_nob(&prt->run_queue))
- return ERTS_MIGRATE_FAILED_RUNQ_CHANGED;
- if (!ERTS_PORT_IS_IN_RUNQ(from_rq, prt))
- return ERTS_MIGRATE_FAILED_NOT_IN_RUNQ;
- dequeue_port(from_rq, prt);
- erts_smp_atomic_set_nob(&prt->run_queue, (erts_aint_t) to_rq);
- enqueue_port(to_rq, prt);
- return ERTS_MIGRATE_SUCCESS;
+ Port *pp;
+ ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
+ pp = pop_port(rq);
+ ASSERT(!pp
+ || rq == (ErtsRunQueue *) erts_smp_atomic_read_nob(&pp->run_queue));
+ ASSERT(!pp || pp->sched.in_runq);
+ return pp;
}
#endif
diff --git a/erts/emulator/beam/erl_port_task.h b/erts/emulator/beam/erl_port_task.h
index d7104e1143..fd88b1c1ff 100644
--- a/erts/emulator/beam/erl_port_task.h
+++ b/erts/emulator/beam/erl_port_task.h
@@ -62,7 +62,7 @@ typedef struct ErtsPortTaskQueue_ ErtsPortTaskQueue;
typedef struct {
Port *next;
- Port *prev;
+ int in_runq;
ErtsPortTaskQueue *taskq;
ErtsPortTaskQueue *exe_taskq;
} ErtsPortTaskSched;
@@ -92,7 +92,7 @@ ERTS_GLB_INLINE void
erts_port_task_init_sched(ErtsPortTaskSched *ptsp)
{
ptsp->next = NULL;
- ptsp->prev = NULL;
+ ptsp->in_runq = 0;
ptsp->taskq = NULL;
ptsp->exe_taskq = NULL;
}
@@ -123,13 +123,10 @@ int erts_port_task_schedule(Eterm,
ErlDrvEventData);
void erts_port_task_free_port(Port *);
int erts_port_is_scheduled(Port *);
+
#ifdef ERTS_SMP
-ErtsMigrateResult erts_port_migrate(Port *,
- int *,
- ErtsRunQueue *,
- int *,
- ErtsRunQueue *,
- int *);
+void erts_enqueue_port(ErtsRunQueue *rq, Port *pp);
+Port *erts_dequeue_port(ErtsRunQueue *rq);
#endif
#undef ERTS_INCLUDE_SCHEDULER_INTERNALS
#endif /* ERL_PORT_TASK_H__ */
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 7942a05153..bb8edd2624 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -94,34 +94,194 @@
#define HIGH_BIT (1 << PRIORITY_HIGH)
#define NORMAL_BIT (1 << PRIORITY_NORMAL)
#define LOW_BIT (1 << PRIORITY_LOW)
+#define PORT_BIT (1 << ERTS_PORT_PRIO_LEVEL)
-#define ERTS_MAYBE_SAVE_TERMINATING_PROCESS(P) \
-do { \
- ERTS_SMP_LC_ASSERT(erts_lc_mtx_is_locked(&proc_tab_mtx)); \
- if (saved_term_procs.end) \
- save_terminating_process((P)); \
-} while (0)
+#define ERTS_EMPTY_RUNQ(RQ) \
+ ((ERTS_RUNQ_FLGS_GET_NOB((RQ)) & ERTS_RUNQ_FLGS_QMASK) == 0 \
+ && (RQ)->misc.start == NULL)
+
+#undef RUNQ_READ_RQ
+#undef RUNQ_SET_RQ
+#define RUNQ_READ_RQ(X) ((ErtsRunQueue *) erts_smp_atomic_read_nob((X)))
+#define RUNQ_SET_RQ(X, RQ) erts_smp_atomic_set_nob((X), (erts_aint_t) (RQ))
-#define ERTS_EMPTY_RUNQ(RQ) \
- ((RQ)->len == 0 && (RQ)->misc.start == NULL)
+#ifdef DEBUG
+# if defined(ARCH_64) && !HALFWORD_HEAP
+# define ERTS_DBG_SET_INVALID_RUNQP(RQP, N) \
+ (RUNQ_SET_RQ((RQP), (0xdeadbeefdead0003LL | ((N) << 4)))
+# define ERTS_DBG_VERIFY_VALID_RUNQP(RQP) \
+do { \
+ ASSERT((RQP) != NULL); \
+ ASSERT(((((Uint) (RQP)) & ((Uint) 0x3))) == ((Uint) 0)); \
+ ASSERT((((Uint) (RQP)) & ~((Uint) 0xffff)) != ((Uint) 0xdeadbeefdead0000LL));\
+} while (0)
+# else
+# define ERTS_DBG_SET_INVALID_RUNQP(RQP, N) \
+ (RUNQ_SET_RQ((RQP), (0xdead0003 | ((N) << 4))))
+# define ERTS_DBG_VERIFY_VALID_RUNQP(RQP) \
+do { \
+ ASSERT((RQP) != NULL); \
+ ASSERT(((((UWord) (RQP)) & ((UWord) 1))) == ((UWord) 0)); \
+ ASSERT((((UWord) (RQP)) & ~((UWord) 0xffff)) != ((UWord) 0xdead0000)); \
+} while (0)
+# endif
+#else
+# define ERTS_DBG_SET_INVALID_RUNQP(RQP, N)
+# define ERTS_DBG_VERIFY_VALID_RUNQP(RQP)
+#endif
#define ERTS_EMPTY_RUNQ_PORTS(RQ) \
- ((RQ)->ports.info.len == 0 && (RQ)->misc.start == NULL)
+ (RUNQ_READ_LEN(&(RQ)->ports.info.len) == 0 && (RQ)->misc.start == NULL)
extern BeamInstr beam_apply[];
extern BeamInstr beam_exit[];
extern BeamInstr beam_continue_exit[];
-static Sint p_last;
-static Sint p_next;
-static Sint p_serial;
-static Uint p_serial_mask;
-static Uint p_serial_shift;
+#ifdef ARCH_32
+
+union {
+ erts_smp_dw_atomic_t pid_data;
+ char align[ERTS_CACHE_LINE_SIZE];
+} last erts_align_attribute(ERTS_CACHE_LINE_SIZE);
+
+
+static ERTS_INLINE Uint64
+dw_aint_to_uint64(erts_dw_aint_t *dw)
+{
+#ifdef ETHR_SU_DW_NAINT_T__
+ return (Uint64) dw->dw_sint;
+#else
+ Uint64 res;
+ res = (Uint64) ((Uint32) dw->sint[ERTS_DW_AINT_HIGH_WORD]);
+ res <<= 32;
+ res |= (Uint64) ((Uint32) dw->sint[ERTS_DW_AINT_LOW_WORD]);
+ return res;
+#endif
+}
+
+static void
+unint64_to_dw_aint(erts_dw_aint_t *dw, Uint64 val)
+{
+#ifdef ETHR_SU_DW_NAINT_T__
+ dw->dw_sint = (ETHR_SU_DW_NAINT_T__) val;
+#else
+ dw->sint[ERTS_DW_AINT_LOW_WORD] = (erts_aint_t) (val & 0xffffffff);
+ dw->sint[ERTS_DW_AINT_HIGH_WORD] = (erts_aint_t) ((val >> 32) & 0xffffffff);
+#endif
+}
+
+static ERTS_INLINE void
+last_pid_data_init_nob(Uint64 val)
+{
+ erts_dw_aint_t dw;
+ unint64_to_dw_aint(&dw, val);
+ erts_smp_dw_atomic_init_nob(&last.pid_data, &dw);
+}
+
+static ERTS_INLINE void
+last_pid_data_set_relb(Uint64 val)
+{
+ erts_dw_aint_t dw;
+ unint64_to_dw_aint(&dw, val);
+ erts_smp_dw_atomic_set_relb(&last.pid_data, &dw);
+}
+
+static ERTS_INLINE Uint64
+last_pid_data_read_nob(void)
+{
+ erts_dw_aint_t dw;
+ erts_smp_dw_atomic_read_nob(&last.pid_data, &dw);
+ return dw_aint_to_uint64(&dw);
+}
+
+static ERTS_INLINE Uint64
+last_pid_data_read_acqb(void)
+{
+ erts_dw_aint_t dw;
+ erts_smp_dw_atomic_read_acqb(&last.pid_data, &dw);
+ return dw_aint_to_uint64(&dw);
+}
+
+static ERTS_INLINE Uint64
+last_pid_data_cmpxchg_relb(Uint64 new, Uint64 exp)
+{
+ erts_dw_aint_t dw_new, dw_xchg;
+
+ unint64_to_dw_aint(&dw_new, new);
+ unint64_to_dw_aint(&dw_xchg, exp);
+
+ if (erts_smp_dw_atomic_cmpxchg_relb(&last.pid_data, &dw_new, &dw_xchg))
+ return exp;
+ else
+ return dw_aint_to_uint64(&dw_xchg);
+}
+
+#elif defined(ARCH_64)
+
+union {
+ erts_smp_atomic_t pid_data;
+ char align[ERTS_CACHE_LINE_SIZE];
+} last erts_align_attribute(ERTS_CACHE_LINE_SIZE);
+
+static ERTS_INLINE void
+last_pid_data_init_nob(Uint64 val)
+{
+ erts_smp_atomic_init_nob(&last.pid_data, (erts_aint_t) val);
+}
+
+static ERTS_INLINE void
+last_pid_data_set_relb(Uint64 val)
+{
+ erts_smp_atomic_set_relb(&last.pid_data, (erts_aint_t) val);
+}
+
+static ERTS_INLINE Uint64
+last_pid_data_read_nob(void)
+{
+ return (Uint64) erts_smp_atomic_read_nob(&last.pid_data);
+}
+
+static ERTS_INLINE Uint64
+last_pid_data_read_acqb(void)
+{
+ return (Uint64) erts_smp_atomic_read_acqb(&last.pid_data);
+}
+
+static ERTS_INLINE Uint64
+last_pid_data_cmpxchg_relb(Uint64 new, Uint64 exp)
+{
+ return (Uint64) erts_smp_atomic_cmpxchg_relb(&last.pid_data,
+ (erts_aint_t) new,
+ (erts_aint_t) exp);
+}
+
+#else
+# error "Not 64-bit, nor 32-bit architecture..."
+#endif
+
+static ERTS_INLINE int
+last_pid_data_cmp(Uint64 lpd1, Uint64 lpd2)
+{
+ Uint64 lpd1_wrap;
+
+ if (lpd1 == lpd2)
+ return 0;
+
+ lpd1_wrap = lpd1 + (((Uint64) 1) << 63);
+
+ if (lpd1 < lpd1_wrap)
+ return (lpd1 < lpd2 && lpd2 < lpd1_wrap) ? -1 : 1;
+ else
+ return (lpd1_wrap <= lpd2 && lpd2 < lpd1) ? 1 : -1;
+}
+
+
+#define ERTS_PID_DATA_MASK__ ((1 << _PID_DATA_SIZE) - 1)
int erts_sched_compact_load;
Uint erts_no_schedulers;
-Uint erts_max_processes = ERTS_DEFAULT_MAX_PROCESSES;
-Uint erts_process_tab_index_mask;
+
+ErtsProcTab erts_proc erts_align_attribute(ERTS_CACHE_LINE_SIZE);
static int wakeup_other_limit;
@@ -184,7 +344,7 @@ static struct {
struct {
int active_runqs;
int reds;
- int max_len;
+ erts_aint32_t max_len;
} prev_rise;
Uint n;
} balance_info;
@@ -204,7 +364,7 @@ erts_sched_stat_t erts_sched_stat;
static erts_tsd_key_t sched_data_key;
#endif
-static erts_smp_mtx_t proc_tab_mtx;
+erts_smp_rwmtx_t erts_proc_tab_rwmtx;
static erts_smp_atomic32_t function_calls;
@@ -227,7 +387,6 @@ typedef union {
static ErtsAlignedSchedulerSleepInfo *aligned_sched_sleep_info;
-Process** process_tab;
static Uint last_reductions;
static Uint last_exact_reductions;
Uint erts_default_process_flags;
@@ -258,11 +417,11 @@ struct ErtsTermProcElement_ {
union {
struct {
Eterm pid;
- SysTimeval spawned;
- SysTimeval exited;
+ Uint64 spawned;
+ Uint64 exited;
} process;
struct {
- SysTimeval time;
+ Uint64 interval;
} bif_invocation;
} u;
};
@@ -402,6 +561,53 @@ erts_smp_lc_runq_is_locked(ErtsRunQueue *runq)
}
#endif
+static erts_interval_t *proc_interval;
+
+static void
+proc_interval_init(void)
+{
+ proc_interval = erts_alloc_permanent_cache_aligned(
+ ERTS_ALC_T_PROC_INTERVAL,
+ sizeof(erts_interval_t));
+ erts_smp_interval_init(proc_interval);
+}
+
+static ERTS_INLINE Uint64
+get_proc_interval(void)
+{
+ return erts_smp_current_interval_nob(proc_interval);
+}
+
+static ERTS_INLINE Uint64
+ensure_later_proc_interval(Uint64 interval)
+{
+ return erts_smp_ensure_later_interval_nob(proc_interval, interval);
+}
+
+static ERTS_INLINE Uint64
+step_proc_interval(void)
+{
+ return erts_smp_step_interval_nob(proc_interval);
+}
+
+Uint64
+erts_get_proc_interval(void)
+{
+ return get_proc_interval();
+}
+
+Uint64
+erts_ensure_later_proc_interval(Uint64 interval)
+{
+ return ensure_later_proc_interval(interval);
+}
+
+Uint64
+erts_step_proc_interval(void)
+{
+ return step_proc_interval();
+}
+
void
erts_pre_init_process(void)
{
@@ -451,7 +657,16 @@ erts_pre_init_process(void)
void
erts_init_process(int ncpu)
{
- Uint proc_bits = ERTS_PROC_BITS;
+ int proc_tab_sz;
+ int max_proc_bits;
+ int proc_bits = ERTS_PROC_BITS;
+ erts_smp_atomic_t *proc_entry;
+ char *proc_tab_end;
+ erts_smp_rwmtx_opt_t proc_tab_rwmtx_opts = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
+ proc_tab_rwmtx_opts.type = ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ;
+ proc_tab_rwmtx_opts.lived = ERTS_SMP_RWMTX_LONG_LIVED;
+
+ proc_interval_init();
#ifdef ERTS_SMP
erts_disable_proc_not_running_opt = 0;
@@ -462,29 +677,57 @@ erts_init_process(int ncpu)
erts_smp_atomic32_init_nob(&process_count, 0);
- if (erts_use_r9_pids_ports) {
+ if (erts_use_r9_pids_ports)
proc_bits = ERTS_R9_PROC_BITS;
- ASSERT(erts_max_processes <= (1 << ERTS_R9_PROC_BITS));
- }
- process_tab = (Process**) erts_alloc(ERTS_ALC_T_PROC_TABLE,
- erts_max_processes*sizeof(Process*));
- sys_memzero(process_tab, erts_max_processes * sizeof(Process*));
+ if (erts_proc.max > (1 << proc_bits))
+ erts_proc.max = 1 << proc_bits;
+
+ proc_tab_sz = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(erts_proc.max
+ * sizeof(erts_smp_atomic_t));
+ erts_proc.tab = erts_alloc(ERTS_ALC_T_PROC_TABLE, proc_tab_sz);
+ proc_tab_end = ((char *) erts_proc.tab) + proc_tab_sz;
+ proc_entry = erts_proc.tab;
+ while (proc_tab_end > ((char *) proc_entry)) {
+ erts_smp_atomic_init_nob(proc_entry, ERTS_AINT_NULL);
+ proc_entry++;
+ }
#ifdef HYBRID
erts_active_procs = (Process**)
erts_alloc(ERTS_ALC_T_ACTIVE_PROCS,
- erts_max_processes * sizeof(Process*));
+ erts_proc.max * sizeof(Process*));
erts_num_active_procs = 0;
#endif
- erts_smp_mtx_init(&proc_tab_mtx, "proc_tab");
- p_last = -1;
- p_next = 0;
- p_serial = 0;
+ erts_smp_rwmtx_init_opt(&erts_proc_tab_rwmtx,
+ &proc_tab_rwmtx_opts,
+ "proc_tab");
+ last_pid_data_init_nob(~((Uint64) 0));
+
+ max_proc_bits = erts_fit_in_bits_int32((Sint32) erts_proc.max - 1);
+
+ erts_proc.tab_cache_lines = proc_tab_sz/ERTS_CACHE_LINE_SIZE;
+ erts_proc.pix_per_cache_line = ERTS_CACHE_LINE_SIZE/sizeof(erts_smp_atomic_t);
+ if ((erts_proc.max & (erts_proc.max - 1))
+ | (erts_proc.pix_per_cache_line & (erts_proc.pix_per_cache_line - 1))) {
+ /*
+ * erts_proc.max or erts_proc.pix_per_cache_line
+ * not a power of 2 :(
+ */
+ erts_proc.pix_cl_mask = 0;
+ erts_proc.pix_cl_shift = 0;
+ erts_proc.pix_cli_mask = 0;
+ erts_proc.pix_cli_shift = 0;
+ }
+ else {
+ ASSERT((erts_proc.tab_cache_lines
+ & (erts_proc.tab_cache_lines - 1)) == 0);
+ erts_proc.pix_cl_mask = erts_proc.tab_cache_lines-1;
+ erts_proc.pix_cl_shift = erts_fit_in_bits_int32(erts_proc.pix_per_cache_line-1);
+ erts_proc.pix_cli_shift = erts_fit_in_bits_int32(erts_proc.pix_cl_mask);
+ erts_proc.pix_cli_mask = (1 << (max_proc_bits - erts_proc.pix_cli_shift)) - 1;
+ }
- p_serial_shift = erts_fit_in_bits(erts_max_processes - 1);
- p_serial_mask = ((~(~((Uint) 0) << proc_bits)) >> p_serial_shift);
- erts_process_tab_index_mask = ~(~((Uint) 0) << p_serial_shift);
last_reductions = 0;
last_exact_reductions = 0;
erts_default_process_flags = 0;
@@ -742,8 +985,9 @@ static ERTS_INLINE ErtsProcList *
proclist_create(Process *p)
{
ErtsProcList *plp = proclist_alloc();
+ ensure_later_proc_interval(p->started_interval);
plp->pid = p->id;
- plp->started = p->started;
+ plp->started_interval = p->started_interval;
return plp;
}
@@ -756,8 +1000,7 @@ proclist_destroy(ErtsProcList *plp)
static ERTS_INLINE int
proclist_same(ErtsProcList *plp, Process *p)
{
- return (plp->pid == p->id
- && erts_cmp_timeval(&plp->started, &p->started) == 0);
+ return plp->pid == p->id && plp->started_interval == p->started_interval;
}
ErtsProcList *
@@ -1719,8 +1962,8 @@ sched_waiting_sys(Uint no, ErtsRunQueue *rq)
{
ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
ASSERT(rq->waiting >= 0);
- rq->flags |= (ERTS_RUNQ_FLG_OUT_OF_WORK
- | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK);
+ ERTS_RUNQ_FLGS_SET(rq, (ERTS_RUNQ_FLG_OUT_OF_WORK
+ | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK));
rq->waiting++;
rq->waiting *= -1;
rq->woken = 0;
@@ -1796,8 +2039,8 @@ static ERTS_INLINE void
sched_waiting(Uint no, ErtsRunQueue *rq)
{
ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
- rq->flags |= (ERTS_RUNQ_FLG_OUT_OF_WORK
- | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK);
+ ERTS_RUNQ_FLGS_SET(rq, (ERTS_RUNQ_FLG_OUT_OF_WORK
+ | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK));
if (rq->waiting < 0)
rq->waiting--;
else
@@ -1829,9 +2072,8 @@ ongoing_multi_scheduling_block(void)
static ERTS_INLINE void
empty_runq(ErtsRunQueue *rq)
{
- erts_aint32_t oifls = erts_smp_atomic32_read_band_nob(&rq->info_flags,
- ~ERTS_RUNQ_IFLG_NONEMPTY);
- if (oifls & ERTS_RUNQ_IFLG_NONEMPTY) {
+ Uint32 old_flags = ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_NONEMPTY|ERTS_RUNQ_FLG_PROTECTED);
+ if (old_flags & ERTS_RUNQ_FLG_NONEMPTY) {
#ifdef DEBUG
erts_aint32_t empty = erts_smp_atomic32_read_nob(&no_empty_run_queues);
/*
@@ -1847,9 +2089,8 @@ empty_runq(ErtsRunQueue *rq)
static ERTS_INLINE void
non_empty_runq(ErtsRunQueue *rq)
{
- erts_aint32_t oifls = erts_smp_atomic32_read_bor_nob(&rq->info_flags,
- ERTS_RUNQ_IFLG_NONEMPTY);
- if (!(oifls & ERTS_RUNQ_IFLG_NONEMPTY)) {
+ Uint32 old_flags = ERTS_RUNQ_FLGS_SET(rq, ERTS_RUNQ_FLG_NONEMPTY);
+ if (!(old_flags & ERTS_RUNQ_FLG_NONEMPTY)) {
#ifdef DEBUG
erts_aint32_t empty = erts_smp_atomic32_read_nob(&no_empty_run_queues);
/*
@@ -2462,19 +2703,16 @@ try_inc_no_active_runqs(int active)
static ERTS_INLINE int
chk_wake_sched(ErtsRunQueue *crq, int ix, int activate)
{
- erts_aint32_t iflgs;
+ Uint32 flags;
ErtsRunQueue *wrq;
if (crq->ix == ix)
return 0;
wrq = ERTS_RUNQ_IX(ix);
- iflgs = erts_smp_atomic32_read_nob(&wrq->info_flags);
- if (!(iflgs & (ERTS_RUNQ_IFLG_SUSPENDED|ERTS_RUNQ_IFLG_NONEMPTY))) {
+ flags = ERTS_RUNQ_FLGS_GET(wrq);
+ if (!(flags & (ERTS_RUNQ_FLG_SUSPENDED|ERTS_RUNQ_FLG_NONEMPTY))) {
if (activate) {
- if (try_inc_no_active_runqs(ix+1)) {
- erts_smp_xrunq_lock(crq, wrq);
- wrq->flags &= ~ERTS_RUNQ_FLG_INACTIVE;
- erts_smp_xrunq_unlock(crq, wrq);
- }
+ if (try_inc_no_active_runqs(ix+1))
+ ERTS_RUNQ_FLGS_UNSET(wrq, ERTS_RUNQ_FLG_INACTIVE);
}
wake_scheduler(wrq, 0);
return 1;
@@ -2541,9 +2779,7 @@ erts_sched_notify_check_cpu_bind(void)
int ix;
for (ix = 0; ix < erts_no_run_queues; ix++) {
ErtsRunQueue *rq = ERTS_RUNQ_IX(ix);
- erts_smp_runq_lock(rq);
- rq->flags |= ERTS_RUNQ_FLG_CHK_CPU_BIND;
- erts_smp_runq_unlock(rq);
+ ERTS_RUNQ_FLGS_SET(rq, ERTS_RUNQ_FLG_CHK_CPU_BIND);
wake_scheduler(rq, 0);
}
#else
@@ -2552,409 +2788,547 @@ erts_sched_notify_check_cpu_bind(void)
}
-#ifdef ERTS_SMP
+static ERTS_INLINE void
+enqueue_process(ErtsRunQueue *runq, int prio, Process *p)
+{
+ ErtsRunPrioQueue *rpq;
-ErtsRunQueue *
-erts_prepare_emigrate(ErtsRunQueue *c_rq, ErtsRunQueueInfo *c_rqi, int prio)
+ ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq));
+
+ erts_smp_inc_runq_len(runq, &runq->procs.prio_info[prio], prio);
+
+ if (prio == PRIORITY_LOW) {
+ p->schedule_count = RESCHEDULE_LOW;
+ rpq = &runq->procs.prio[PRIORITY_NORMAL];
+ }
+ else {
+ p->schedule_count = 1;
+ rpq = &runq->procs.prio[prio];
+ }
+
+ p->next = NULL;
+ if (rpq->last)
+ rpq->last->next = p;
+ else
+ rpq->first = p;
+ rpq->last = p;
+}
+
+
+static ERTS_INLINE void
+unqueue_process(ErtsRunQueue *runq,
+ ErtsRunPrioQueue *rpq,
+ ErtsRunQueueInfo *rqi,
+ int prio,
+ Process *prev_proc,
+ Process *proc)
{
- ASSERT(ERTS_CHK_RUNQ_FLG_EMIGRATE(c_rq->flags, prio));
- ASSERT(ERTS_CHK_RUNQ_FLG_EVACUATE(c_rq->flags, prio)
- || c_rqi->len >= c_rqi->migrate.limit.this);
+ ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq));
- while (1) {
- ErtsRunQueue *n_rq = c_rqi->migrate.runq;
- ERTS_DBG_VERIFY_VALID_RUNQP(n_rq);
- erts_smp_xrunq_lock(c_rq, n_rq);
-
- /*
- * erts_smp_xrunq_lock() may release lock on c_rq! We have
- * to check that we still want to emigrate and emigrate
- * to the same run queue as before.
- */
+ if (prev_proc)
+ prev_proc->next = proc->next;
+ else
+ rpq->first = proc->next;
+ if (!proc->next)
+ rpq->last = prev_proc;
- if (ERTS_CHK_RUNQ_FLG_EMIGRATE(c_rq->flags, prio)) {
- Uint32 force = (ERTS_CHK_RUNQ_FLG_EVACUATE(c_rq->flags, prio)
- | (c_rq->flags & ERTS_RUNQ_FLG_INACTIVE));
- if (force || c_rqi->len > c_rqi->migrate.limit.this) {
- ErtsRunQueueInfo *n_rqi;
- /* We still want to emigrate */
-
- if (n_rq != c_rqi->migrate.runq) {
- /* Ahh... run queue changed; need to do it all over again... */
- erts_smp_runq_unlock(n_rq);
- continue;
- }
- else {
+ if (!rpq->first)
+ rpq->last = NULL;
- if (prio == ERTS_PORT_PRIO_LEVEL)
- n_rqi = &n_rq->ports.info;
- else
- n_rqi = &n_rq->procs.prio_info[prio];
+ erts_smp_dec_runq_len(runq, rqi, prio);
+}
- if (force || (n_rqi->len < c_rqi->migrate.limit.other)) {
- /* emigrate ... */
- return n_rq;
- }
- }
- }
- }
- ASSERT(n_rq != c_rq);
- erts_smp_runq_unlock(n_rq);
- if (!(c_rq->flags & ERTS_RUNQ_FLG_INACTIVE)) {
- /* No more emigrations to this runq */
- ERTS_UNSET_RUNQ_FLG_EMIGRATE(c_rq->flags, prio);
- ERTS_DBG_SET_INVALID_RUNQP(c_rqi->migrate.runq, 0x3);
- }
+static ERTS_INLINE Process *
+dequeue_process(ErtsRunQueue *runq, int prio_q, erts_aint32_t *statep)
+{
+ erts_aint32_t state;
+ int prio;
+ ErtsRunPrioQueue *rpq;
+ ErtsRunQueueInfo *rqi;
+ Process *p;
+
+ ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq));
+
+ ASSERT(PRIORITY_NORMAL == prio_q
+ || PRIORITY_HIGH == prio_q
+ || PRIORITY_MAX == prio_q);
+
+ rpq = &runq->procs.prio[prio_q];
+ p = rpq->first;
+ if (!p)
+ return NULL;
+
+ ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER;
+
+ state = erts_smp_atomic32_read_nob(&p->state);
+ if (statep)
+ *statep = state;
+
+ prio = (int) (ERTS_PSFLG_PRIO_MASK & state);
+
+ rqi = &runq->procs.prio_info[prio];
+
+ if (p)
+ unqueue_process(runq, rpq, rqi, prio, NULL, p);
+
+ return p;
+}
+
+static ERTS_INLINE int
+check_requeue_process(ErtsRunQueue *rq, int prio_q)
+{
+ ErtsRunPrioQueue *rpq = &rq->procs.prio[prio_q];
+ Process *p = rpq->first;
+ if (--p->schedule_count > 0 && p != rpq->last) {
+ /* reschedule */
+ rpq->first = p->next;
+ rpq->last->next = p;
+ rpq->last = p;
+ p->next = NULL;
+ return 1;
+ }
+ return 0;
+}
+
+#ifdef ERTS_SMP
+
+static ErtsRunQueue *
+check_immigration_need(ErtsRunQueue *c_rq, ErtsMigrationPath *mp, int prio)
+{
+ int len;
+ Uint32 f_flags, f_rq_flags;
+ ErtsRunQueue *f_rq;
+
+ f_flags = mp->prio[prio].flags;
+
+ ASSERT(ERTS_CHK_RUNQ_FLG_IMMIGRATE(mp->flags, prio));
+
+ f_rq = mp->prio[prio].runq;
+ if (!f_rq)
+ return NULL;
+ f_rq_flags = ERTS_RUNQ_FLGS_GET(f_rq);
+ if (f_rq_flags & ERTS_RUNQ_FLG_PROTECTED)
return NULL;
+
+ if (ERTS_CHK_RUNQ_FLG_EVACUATE(f_flags, prio))
+ return f_rq;
+
+ if (f_rq_flags & ERTS_RUNQ_FLG_INACTIVE)
+ return f_rq;
+
+ if (prio == ERTS_PORT_PRIO_LEVEL)
+ len = RUNQ_READ_LEN(&c_rq->ports.info.len);
+ else
+ len = RUNQ_READ_LEN(&c_rq->procs.prio_info[prio].len);
+
+ if (len < mp->prio[prio].limit.this) {
+ if (prio == ERTS_PORT_PRIO_LEVEL)
+ len = RUNQ_READ_LEN(&f_rq->ports.info.len);
+ else
+ len = RUNQ_READ_LEN(&f_rq->procs.prio_info[prio].len);
+
+ if (len > mp->prio[prio].limit.other)
+ return f_rq;
}
+ return NULL;
}
static void
-immigrate(ErtsRunQueue *rq)
+immigrate(ErtsRunQueue *c_rq, ErtsMigrationPath *mp)
{
- int prio;
+ Uint32 iflags, iflag;
+ erts_smp_runq_unlock(c_rq);
- ASSERT(rq->flags & ERTS_RUNQ_FLGS_IMMIGRATE_QMASK);
+ ASSERT(erts_thr_progress_is_managed_thread());
- for (prio = 0; prio < ERTS_NO_PRIO_LEVELS; prio++) {
- if (ERTS_CHK_RUNQ_FLG_IMMIGRATE(rq->flags, prio)) {
- ErtsRunQueueInfo *rqi = (prio == ERTS_PORT_PRIO_LEVEL
- ? &rq->ports.info
- : &rq->procs.prio_info[prio]);
- ErtsRunQueue *from_rq = rqi->migrate.runq;
- int rq_locked, from_rq_locked;
+ iflags = mp->flags & ERTS_RUNQ_FLGS_IMMIGRATE_QMASK;
- ERTS_DBG_VERIFY_VALID_RUNQP(from_rq);
+ iflag = iflags & -iflags;
- rq_locked = 1;
- from_rq_locked = 1;
- erts_smp_xrunq_lock(rq, from_rq);
- /*
- * erts_smp_xrunq_lock() may release lock on rq! We have
- * to check that we still want to immigrate from the same
- * run queue as before.
- */
- if (ERTS_CHK_RUNQ_FLG_IMMIGRATE(rq->flags, prio)
- && from_rq == rqi->migrate.runq) {
- ErtsRunQueueInfo *from_rqi = (prio == ERTS_PORT_PRIO_LEVEL
- ? &from_rq->ports.info
- : &from_rq->procs.prio_info[prio]);
- if ((ERTS_CHK_RUNQ_FLG_EVACUATE(rq->flags, prio)
- && ERTS_CHK_RUNQ_FLG_EVACUATE(from_rq->flags, prio)
- && from_rqi->len)
- || (from_rqi->len > rqi->migrate.limit.other
- && rqi->len < rqi->migrate.limit.this)) {
- if (prio == ERTS_PORT_PRIO_LEVEL) {
- Port *prt = from_rq->ports.start;
- if (prt) {
- int prt_locked = 0;
- (void) erts_port_migrate(prt, &prt_locked,
- from_rq, &from_rq_locked,
- rq, &rq_locked);
- if (prt_locked)
- erts_smp_port_unlock(prt);
- }
- }
- else {
- Process *proc;
- ErtsRunPrioQueue *from_rpq;
- from_rpq = (prio == PRIORITY_LOW
- ? &from_rq->procs.prio[PRIORITY_NORMAL]
- : &from_rq->procs.prio[prio]);
- for (proc = from_rpq->first; proc; proc = proc->next)
- if (proc->prio == prio && !proc->bound_runq)
- break;
- if (proc) {
- ErtsProcLocks proc_locks = 0;
- (void) erts_proc_migrate(proc, &proc_locks,
- from_rq, &from_rq_locked,
- rq, &rq_locked);
- if (proc_locks)
- erts_smp_proc_unlock(proc, proc_locks);
- }
+ while (iflag) {
+ ErtsRunQueue *rq;
+ int prio;
+
+ switch (iflag) {
+ case (MAX_BIT << ERTS_RUNQ_FLGS_IMMIGRATE_SHFT):
+ prio = PRIORITY_MAX;
+ break;
+ case (HIGH_BIT << ERTS_RUNQ_FLGS_IMMIGRATE_SHFT):
+ prio = PRIORITY_HIGH;
+ break;
+ case (NORMAL_BIT << ERTS_RUNQ_FLGS_IMMIGRATE_SHFT):
+ prio = PRIORITY_NORMAL;
+ break;
+ case (LOW_BIT << ERTS_RUNQ_FLGS_IMMIGRATE_SHFT):
+ prio = PRIORITY_LOW;
+ break;
+ case (PORT_BIT << ERTS_RUNQ_FLGS_IMMIGRATE_SHFT):
+ prio = ERTS_PORT_PRIO_LEVEL;
+ break;
+ default:
+ erl_exit(ERTS_ABORT_EXIT,
+ "%s:%d:%s(): Invalid immigrate queue mask",
+ __FILE__, __LINE__, __func__);
+ prio = 0;
+ break;
+ }
+
+ iflags &= ~iflag;
+ iflag = iflags & -iflags;
+
+ rq = check_immigration_need(c_rq, mp, prio);
+ if (rq) {
+ erts_smp_runq_lock(rq);
+ if (prio == ERTS_PORT_PRIO_LEVEL) {
+ Port *prt;
+ prt = erts_dequeue_port(rq);
+ if (prt)
+ RUNQ_SET_RQ(&prt->run_queue, c_rq);
+ erts_smp_runq_unlock(rq);
+ if (prt) {
+ /* port might terminate while we have no lock... */
+ rq = erts_port_runq(prt);
+ if (rq) {
+ if (rq != c_rq)
+ erl_exit(ERTS_ABORT_EXIT,
+ "%s:%d:%s(): Internal error",
+ __FILE__, __LINE__, __func__);
+ erts_enqueue_port(c_rq, prt);
+ if (!iflag)
+ return; /* done */
+ erts_smp_runq_unlock(c_rq);
}
}
- else {
- ERTS_UNSET_RUNQ_FLG_IMMIGRATE(rq->flags, prio);
- ERTS_DBG_SET_INVALID_RUNQP(rqi->migrate.runq, 0x1);
+ }
+ else {
+ ErtsRunPrioQueue *rpq = &rq->procs.prio[prio == PRIORITY_LOW
+ ? PRIORITY_NORMAL
+ : prio];
+ Process *prev_proc = NULL;
+ Process *proc = rpq->first;
+ int rq_locked = 1;
+
+ while (proc) {
+ erts_aint32_t state;
+ state = erts_smp_atomic32_read_acqb(&proc->state);
+ if (!(ERTS_PSFLG_BOUND & state)
+ && (prio == (int) (ERTS_PSFLG_PRIO_MASK & state))) {
+ ErtsRunQueueInfo *rqi = &rq->procs.prio_info[prio];
+ unqueue_process(rq, rpq, rqi, prio, prev_proc, proc);
+ erts_smp_runq_unlock(rq);
+ RUNQ_SET_RQ(&proc->run_queue, c_rq);
+ rq_locked = 0;
+
+ erts_smp_runq_lock(c_rq);
+ enqueue_process(c_rq, prio, proc);
+ if (!iflag)
+ return; /* done */
+ erts_smp_runq_unlock(c_rq);
+ break;
+ }
+ prev_proc = proc;
+ proc = proc->next;
}
+ if (rq_locked)
+ erts_smp_runq_unlock(rq);
}
- if (from_rq_locked)
- erts_smp_runq_unlock(from_rq);
- if (!rq_locked)
- erts_smp_runq_lock(rq);
}
}
+
+ erts_smp_runq_lock(c_rq);
+}
+
+static ERTS_INLINE void
+suspend_run_queue(ErtsRunQueue *rq)
+{
+ erts_smp_atomic32_read_bor_nob(&rq->scheduler->ssi->flags,
+ ERTS_SSI_FLG_SUSPENDED);
+ ERTS_RUNQ_FLGS_SET(rq, ERTS_RUNQ_FLG_SUSPENDED);
+
+ wake_scheduler(rq, 0);
}
+static void scheduler_ix_resume_wake(Uint ix);
+
+static ERTS_INLINE void
+resume_run_queue(ErtsRunQueue *rq)
+{
+ int pix;
+
+ erts_smp_runq_lock(rq);
+
+ (void) ERTS_RUNQ_FLGS_MASK_SET(rq,
+ (ERTS_RUNQ_FLG_OUT_OF_WORK
+ | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK
+ | ERTS_RUNQ_FLG_SUSPENDED),
+ (ERTS_RUNQ_FLG_OUT_OF_WORK
+ | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK));
+
+ rq->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS;
+ for (pix = 0; pix < ERTS_NO_PROC_PRIO_LEVELS; pix++) {
+ rq->procs.prio_info[pix].max_len = 0;
+ rq->procs.prio_info[pix].reds = 0;
+ }
+ rq->ports.info.max_len = 0;
+ rq->ports.info.reds = 0;
+ rq->max_len = 0;
+
+ erts_smp_runq_unlock(rq);
+
+ scheduler_ix_resume_wake(rq->ix);
+}
+
+typedef struct {
+ Process *first;
+ Process *last;
+} ErtsStuckBoundProcesses;
+
static void
-evacuate_run_queue(ErtsRunQueue *evac_rq, ErtsRunQueue *rq)
+schedule_bound_processes(ErtsRunQueue *rq,
+ ErtsStuckBoundProcesses *sbpp)
{
- Port *prt;
- int notify_to_rq = 0;
- int prio;
- int prt_locked = 0;
- int rq_locked = 0;
- int evac_rq_locked = 1;
- ErtsMigrateResult mres;
+ Process *proc, *next;
+ ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
- erts_smp_runq_lock(evac_rq);
+ proc = sbpp->first;
+ while (proc) {
+ erts_aint32_t state = erts_smp_atomic32_read_acqb(&proc->state);
+ next = proc->next;
+ enqueue_process(rq, (int) (ERTS_PSFLG_PRIO_MASK & state), proc);
+ proc = next;
+ }
+}
- erts_smp_atomic32_read_bor_nob(&evac_rq->scheduler->ssi->flags,
- ERTS_SSI_FLG_SUSPENDED);
+static void
+evacuate_run_queue(ErtsRunQueue *rq,
+ ErtsStuckBoundProcesses *sbpp)
+{
+ int prio_q;
+ ErtsRunQueue *to_rq;
+ ErtsMigrationPaths *mps;
+ ErtsMigrationPath *mp;
- evac_rq->flags &= ~ERTS_RUNQ_FLGS_IMMIGRATE_QMASK;
- evac_rq->flags |= (ERTS_RUNQ_FLGS_EMIGRATE_QMASK
- | ERTS_RUNQ_FLGS_EVACUATE_QMASK
- | ERTS_RUNQ_FLG_SUSPENDED);
+ ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
- erts_smp_atomic32_read_bor_nob(&evac_rq->info_flags, ERTS_RUNQ_IFLG_SUSPENDED);
- /*
- * Need to set up evacuation paths first since we
- * may release the run queue lock on evac_rq
- * when evacuating.
- */
- evac_rq->misc.evac_runq = rq;
- evac_rq->ports.info.migrate.runq = rq;
- for (prio = 0; prio < ERTS_NO_PROC_PRIO_LEVELS; prio++)
- evac_rq->procs.prio_info[prio].migrate.runq = rq;
+ ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_PROTECTED);
+
+ mps = erts_get_migration_paths_managed();
+ mp = &mps->mpath[rq->ix];
/* Evacuate scheduled misc ops */
- if (evac_rq->misc.start) {
- rq_locked = 1;
- erts_smp_xrunq_lock(evac_rq, rq);
- if (rq->misc.end)
- rq->misc.end->next = evac_rq->misc.start;
+ if (rq->misc.start) {
+ ErtsMiscOpList *start, *end;
+
+ to_rq = mp->misc_evac_runq;
+ if (!to_rq)
+ return;
+
+ start = rq->misc.start;
+ end = rq->misc.end;
+ rq->misc.start = NULL;
+ rq->misc.end = NULL;
+ erts_smp_runq_unlock(rq);
+
+ erts_smp_runq_lock(to_rq);
+ if (to_rq->misc.end)
+ to_rq->misc.end->next = start;
else
- rq->misc.start = evac_rq->misc.start;
- rq->misc.end = evac_rq->misc.end;
- evac_rq->misc.start = NULL;
- evac_rq->misc.end = NULL;
+ to_rq->misc.start = start;
+
+ to_rq->misc.end = end;
+ erts_smp_runq_unlock(to_rq);
+ smp_notify_inc_runq(to_rq);
+ erts_smp_runq_lock(to_rq);
}
- /* Evacuate scheduled ports */
- prt = evac_rq->ports.start;
- while (prt) {
- mres = erts_port_migrate(prt, &prt_locked,
- evac_rq, &evac_rq_locked,
- rq, &rq_locked);
- if (mres == ERTS_MIGRATE_SUCCESS)
- notify_to_rq = 1;
- if (prt_locked)
- erts_smp_port_unlock(prt);
- if (!evac_rq_locked) {
- evac_rq_locked = 1;
- erts_smp_runq_lock(evac_rq);
+ if (rq->ports.start) {
+ Port *prt;
+
+ to_rq = mp->prio[ERTS_PORT_PRIO_LEVEL].runq;
+ if (!to_rq)
+ return;
+
+ /* Evacuate scheduled ports */
+ prt = rq->ports.start;
+ while (prt) {
+ ErtsRunQueue *prt_rq;
+ prt = erts_dequeue_port(rq);
+ RUNQ_SET_RQ(&prt->run_queue, to_rq);
+ erts_smp_runq_unlock(rq);
+ /*
+ * The port might terminate while
+ * we have no lock on it...
+ */
+ prt_rq = erts_port_runq(prt);
+ if (prt_rq) {
+ if (prt_rq != to_rq)
+ erl_exit(ERTS_ABORT_EXIT,
+ "%s:%d:%s() internal error\n",
+ __FILE__, __LINE__, __func__);
+ erts_enqueue_port(to_rq, prt);
+ erts_smp_runq_unlock(to_rq);
+ }
+ erts_smp_runq_lock(rq);
+ prt = rq->ports.start;
}
- prt = evac_rq->ports.start;
+ smp_notify_inc_runq(to_rq);
}
/* Evacuate scheduled processes */
- for (prio = 0; prio < ERTS_NO_PROC_PRIO_LEVELS; prio++) {
+ for (prio_q = 0; prio_q < ERTS_NO_PROC_PRIO_QUEUES; prio_q++) {
+ erts_aint32_t state;
Process *proc;
+ int notify = 0;
+ to_rq = NULL;
- switch (prio) {
- case PRIORITY_MAX:
- case PRIORITY_HIGH:
- case PRIORITY_NORMAL:
- proc = evac_rq->procs.prio[prio].first;
- while (proc) {
- ErtsProcLocks proc_locks = 0;
-
- /* Bound processes are stuck... */
- while (proc->bound_runq) {
- proc = proc->next;
- if (!proc)
- goto end_of_proc;
- }
-
- mres = erts_proc_migrate(proc, &proc_locks,
- evac_rq, &evac_rq_locked,
- rq, &rq_locked);
- if (mres == ERTS_MIGRATE_SUCCESS)
- notify_to_rq = 1;
- if (proc_locks)
- erts_smp_proc_unlock(proc, proc_locks);
- if (!evac_rq_locked) {
- erts_smp_runq_lock(evac_rq);
- evac_rq_locked = 1;
- }
+ if (!mp->prio[prio_q].runq)
+ return;
+ if (prio_q == PRIORITY_NORMAL && !mp->prio[PRIORITY_LOW].runq)
+ return;
- proc = evac_rq->procs.prio[prio].first;
+ proc = dequeue_process(rq, prio_q, &state);
+ while (proc) {
+ if (ERTS_PSFLG_BOUND & state) {
+ /* Bound processes get stuck here... */
+ proc->next = NULL;
+ if (sbpp->last)
+ sbpp->last->next = proc;
+ else
+ sbpp->first = proc;
+ sbpp->last = proc;
}
+ else {
+ int prio = (int) (ERTS_PSFLG_PRIO_MASK & state);
+ erts_smp_runq_unlock(rq);
- end_of_proc:
+ to_rq = mp->prio[prio].runq;
+ RUNQ_SET_RQ(&proc->run_queue, to_rq);
-#ifdef DEBUG
- for (proc = evac_rq->procs.prio[prio].first;
- proc;
- proc = proc->next) {
- ASSERT(proc->bound_runq);
+ erts_smp_runq_lock(to_rq);
+ enqueue_process(to_rq, prio, proc);
+ erts_smp_runq_unlock(to_rq);
+ notify = 1;
+
+ erts_smp_runq_lock(rq);
}
-#endif
- break;
- case PRIORITY_LOW:
- break;
- default:
- ASSERT(!"Invalid process priority");
- break;
+ proc = dequeue_process(rq, prio_q, &state);
}
+ if (notify)
+ smp_notify_inc_runq(to_rq);
}
- if (rq_locked)
- erts_smp_runq_unlock(rq);
-
- if (evac_rq_locked)
- erts_smp_runq_unlock(evac_rq);
-
- if (notify_to_rq)
- smp_notify_inc_runq(rq);
-
- wake_scheduler(evac_rq, 0);
+ if (ERTS_EMPTY_RUNQ(rq))
+ empty_runq(rq);
}
static int
-try_steal_task_from_victim(ErtsRunQueue *rq, int *rq_lockedp, ErtsRunQueue *vrq)
+try_steal_task_from_victim(ErtsRunQueue *rq, int *rq_lockedp, ErtsRunQueue *vrq, Uint32 flags)
{
- Process *proc;
- int vrq_locked;
+ Uint32 procs_qmask = flags & ERTS_RUNQ_FLGS_PROCS_QMASK;
+ int max_prio_bit;
+ ErtsRunPrioQueue *rpq;
- if (*rq_lockedp)
- erts_smp_xrunq_lock(rq, vrq);
- else
- erts_smp_runq_lock(vrq);
- vrq_locked = 1;
+ if (*rq_lockedp) {
+ erts_smp_runq_unlock(rq);
+ *rq_lockedp = 0;
+ }
+
+ ERTS_SMP_LC_ASSERT(!erts_smp_lc_runq_is_locked(rq));
- ERTS_SMP_LC_CHK_RUNQ_LOCK(rq, *rq_lockedp);
- ERTS_SMP_LC_CHK_RUNQ_LOCK(vrq, vrq_locked);
+ erts_smp_runq_lock(vrq);
if (rq->halt_in_progress)
- goto try_steal_port;
+ goto no_procs;
/*
* Check for a runnable process to steal...
*/
- switch (vrq->flags & ERTS_RUNQ_FLGS_PROCS_QMASK) {
- case MAX_BIT:
- case MAX_BIT|HIGH_BIT:
- case MAX_BIT|NORMAL_BIT:
- case MAX_BIT|LOW_BIT:
- case MAX_BIT|HIGH_BIT|NORMAL_BIT:
- case MAX_BIT|HIGH_BIT|LOW_BIT:
- case MAX_BIT|NORMAL_BIT|LOW_BIT:
- case MAX_BIT|HIGH_BIT|NORMAL_BIT|LOW_BIT:
- for (proc = vrq->procs.prio[PRIORITY_MAX].last;
- proc;
- proc = proc->prev) {
- if (!proc->bound_runq)
- break;
- }
- if (proc)
+ while (procs_qmask) {
+ Process *prev_proc;
+ Process *proc;
+
+ max_prio_bit = procs_qmask & -procs_qmask;
+ switch (max_prio_bit) {
+ case MAX_BIT:
+ rpq = &vrq->procs.prio[PRIORITY_MAX];
break;
- case HIGH_BIT:
- case HIGH_BIT|NORMAL_BIT:
- case HIGH_BIT|LOW_BIT:
- case HIGH_BIT|NORMAL_BIT|LOW_BIT:
- for (proc = vrq->procs.prio[PRIORITY_HIGH].last;
- proc;
- proc = proc->prev) {
- if (!proc->bound_runq)
- break;
- }
- if (proc)
+ case HIGH_BIT:
+ rpq = &vrq->procs.prio[PRIORITY_HIGH];
break;
- case NORMAL_BIT:
- case LOW_BIT:
- case NORMAL_BIT|LOW_BIT:
- for (proc = vrq->procs.prio[PRIORITY_NORMAL].last;
- proc;
- proc = proc->prev) {
- if (!proc->bound_runq)
- break;
- }
- if (proc)
+ case NORMAL_BIT:
+ case LOW_BIT:
+ rpq = &vrq->procs.prio[PRIORITY_NORMAL];
break;
- case 0:
- proc = NULL;
- break;
- default:
- ASSERT(!"Invalid queue mask");
- proc = NULL;
- break;
- }
+ case 0:
+ goto no_procs;
+ default:
+ ASSERT(!"Invalid queue mask");
+ goto no_procs;
+ }
+
+ prev_proc = NULL;
+ proc = rpq->first;
- if (proc) {
- ErtsProcLocks proc_locks = 0;
- int res;
- ErtsMigrateResult mres;
- mres = erts_proc_migrate(proc, &proc_locks,
- vrq, &vrq_locked,
- rq, rq_lockedp);
- if (proc_locks)
- erts_smp_proc_unlock(proc, proc_locks);
- res = !0;
- switch (mres) {
- case ERTS_MIGRATE_FAILED_RUNQ_SUSPENDED:
- res = 0;
- case ERTS_MIGRATE_SUCCESS:
- if (vrq_locked)
+ while (proc) {
+ erts_aint32_t state = erts_smp_atomic32_read_acqb(&proc->state);
+ if (!(ERTS_PSFLG_BOUND & state)) {
+ /* Steal process */
+ int prio = (int) (ERTS_PSFLG_PRIO_MASK & state);
+ ErtsRunQueueInfo *rqi = &vrq->procs.prio_info[prio];
+ unqueue_process(vrq, rpq, rqi, prio, prev_proc, proc);
erts_smp_runq_unlock(vrq);
- return res;
- default: /* Other failures */
- break;
- }
- }
+ RUNQ_SET_RQ(&proc->run_queue, rq);
- ERTS_SMP_LC_CHK_RUNQ_LOCK(rq, *rq_lockedp);
- ERTS_SMP_LC_CHK_RUNQ_LOCK(vrq, vrq_locked);
+ erts_smp_runq_lock(rq);
+ *rq_lockedp = 1;
+ enqueue_process(rq, prio, proc);
+ return !0;
+ }
+ prev_proc = proc;
+ proc = proc->next;
+ }
- if (!vrq_locked) {
- if (*rq_lockedp)
- erts_smp_xrunq_lock(rq, vrq);
- else
- erts_smp_runq_lock(vrq);
- vrq_locked = 1;
+ procs_qmask &= ~max_prio_bit;
}
- try_steal_port:
+no_procs:
- ERTS_SMP_LC_CHK_RUNQ_LOCK(rq, *rq_lockedp);
- ERTS_SMP_LC_CHK_RUNQ_LOCK(vrq, vrq_locked);
+ ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(vrq));
/*
* Check for a runnable port to steal...
*/
- if (vrq->ports.info.len) {
- Port *prt = vrq->ports.end;
- int prt_locked = 0;
- int res;
- ErtsMigrateResult mres;
-
- mres = erts_port_migrate(prt, &prt_locked,
- vrq, &vrq_locked,
- rq, rq_lockedp);
- if (prt_locked)
- erts_smp_port_unlock(prt);
- res = !0;
- switch (mres) {
- case ERTS_MIGRATE_FAILED_RUNQ_SUSPENDED:
- res = 0;
- case ERTS_MIGRATE_SUCCESS:
- if (vrq_locked)
- erts_smp_runq_unlock(vrq);
- return res;
- default: /* Other failures */
- break;
+ if (vrq->ports.start) {
+ ErtsRunQueue *prt_rq;
+ Port *prt = erts_dequeue_port(vrq);
+ RUNQ_SET_RQ(&prt->run_queue, rq);
+ erts_smp_runq_unlock(vrq);
+
+ /*
+ * The port might terminate while
+ * we have no lock on it...
+ */
+
+ prt_rq = erts_port_runq(prt);
+ if (!prt_rq)
+ return 0;
+ else {
+ if (prt_rq != rq)
+ erl_exit(ERTS_ABORT_EXIT,
+ "%s:%d:%s() internal error\n",
+ __FILE__, __LINE__, __func__);
+ *rq_lockedp = 1;
+ erts_enqueue_port(rq, prt);
+ return !0;
}
}
- if (vrq_locked)
- erts_smp_runq_unlock(vrq);
+ erts_smp_runq_unlock(vrq);
return 0;
}
@@ -2964,9 +3338,10 @@ static ERTS_INLINE int
check_possible_steal_victim(ErtsRunQueue *rq, int *rq_lockedp, int vix)
{
ErtsRunQueue *vrq = ERTS_RUNQ_IX(vix);
- erts_aint32_t iflgs = erts_smp_atomic32_read_nob(&vrq->info_flags);
- if (iflgs & ERTS_RUNQ_IFLG_NONEMPTY)
- return try_steal_task_from_victim(rq, rq_lockedp, vrq);
+ Uint32 flags = ERTS_RUNQ_FLGS_GET(vrq);
+ if ((flags & (ERTS_RUNQ_FLG_NONEMPTY
+ | ERTS_RUNQ_FLG_PROTECTED)) == ERTS_RUNQ_FLG_NONEMPTY)
+ return try_steal_task_from_victim(rq, rq_lockedp, vrq, flags);
else
return 0;
}
@@ -2976,15 +3351,12 @@ static int
try_steal_task(ErtsRunQueue *rq)
{
int res, rq_locked, vix, active_rqs, blnc_rqs;
+ Uint32 flags;
- /*
- * We are not allowed to steal jobs to this run queue
- * if it is suspended. Note that it might get suspended
- * at any time when we don't have the lock on the run
- * queue.
- */
- if (rq->flags & ERTS_RUNQ_FLG_SUSPENDED)
- return 0;
+ /* Protect jobs we steal from getting stolen from us... */
+ flags = ERTS_RUNQ_FLGS_SET(rq, ERTS_RUNQ_FLG_PROTECTED);
+ if (flags & ERTS_RUNQ_FLG_SUSPENDED)
+ return 0; /* go suspend instead... */
res = 0;
rq_locked = 1;
@@ -3103,12 +3475,123 @@ do { \
ASSERT(sum__ == (RQ)->full_reds_history_sum); \
} while (0);
+#define ERTS_PRE_ALLOCED_MPATHS 8
+
+erts_atomic_t erts_migration_paths;
+
+static struct {
+ size_t size;
+ ErtsMigrationPaths *freelist;
+ struct {
+ ErtsMigrationPaths *first;
+ ErtsMigrationPaths *last;
+ } retired;
+} mpaths;
+
+static void
+init_migration_paths(void)
+{
+ int qix, i;
+ char *p;
+ ErtsMigrationPaths *mps;
+
+ mpaths.size = sizeof(ErtsMigrationPaths);
+ mpaths.size += sizeof(ErtsMigrationPath)*(erts_no_schedulers-1);
+ mpaths.size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(mpaths.size);
+
+ p = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_LL_MPATHS,
+ (mpaths.size
+ * ERTS_PRE_ALLOCED_MPATHS));
+ mpaths.freelist = NULL;
+ for (i = 0; i < ERTS_PRE_ALLOCED_MPATHS-1; i++) {
+ mps = (ErtsMigrationPaths *) p;
+ mps->next = mpaths.freelist;
+ mpaths.freelist = mps;
+ p += mpaths.size;
+ }
+
+ mps = (ErtsMigrationPaths *) p;
+ mps->block = NULL;
+ for (qix = 0; qix < erts_no_run_queues; qix++) {
+ int pix;
+ mps->mpath[qix].flags = 0;
+ mps->mpath[qix].misc_evac_runq = NULL;
+ for (pix = 0; pix < ERTS_NO_PRIO_LEVELS; pix++) {
+ mps->mpath[qix].prio[pix].limit.this = -1;
+ mps->mpath[qix].prio[pix].limit.other = -1;
+ mps->mpath[qix].prio[pix].runq = NULL;
+ mps->mpath[qix].prio[pix].flags = 0;
+ }
+ }
+ erts_atomic_init_wb(&erts_migration_paths, (erts_aint_t) mps);
+}
+
+static ERTS_INLINE ErtsMigrationPaths *
+alloc_mpaths(void)
+{
+ void *block;
+ ErtsMigrationPaths *res;
+ ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&balance_info.update_mtx));
+
+ res = mpaths.freelist;
+ if (res) {
+ mpaths.freelist = res->next;
+ res->block = NULL;
+ return res;
+ }
+ res = erts_alloc(ERTS_ALC_T_SL_MPATHS,
+ mpaths.size+ERTS_CACHE_LINE_SIZE);
+ block = (void *) res;
+ if (((UWord) res) & ERTS_CACHE_LINE_MASK)
+ res = (ErtsMigrationPaths *) ((((UWord) res) & ~ERTS_CACHE_LINE_MASK)
+ + ERTS_CACHE_LINE_SIZE);
+ res->block = block;
+ return res;
+}
+
+static ERTS_INLINE void
+retire_mpaths(ErtsMigrationPaths *mps)
+{
+ ErtsThrPrgrVal current;
+
+ ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&balance_info.update_mtx));
+
+ current = erts_thr_progress_current();
+
+ while (mpaths.retired.first) {
+ ErtsMigrationPaths *tmp = mpaths.retired.first;
+ if (!erts_thr_progress_has_reached_this(current, tmp->thr_prgr))
+ break;
+ mpaths.retired.first = tmp->next;
+ if (tmp->block) {
+ erts_free(ERTS_ALC_T_SL_MPATHS, tmp->block);
+ }
+ else {
+ tmp->next = mpaths.freelist;
+ mpaths.freelist = tmp;
+ }
+ }
+
+ if (!mpaths.retired.first)
+ mpaths.retired.last = NULL;
+
+ mps->thr_prgr = erts_thr_progress_later_than(current);
+ mps->next = NULL;
+
+ if (mpaths.retired.last)
+ mpaths.retired.last->next = mps;
+ else
+ mpaths.retired.first = mps;
+ mpaths.retired.last = mps;
+}
+
static void
check_balance(ErtsRunQueue *c_rq)
{
#if ERTS_MAX_PROCESSES >= (1 << 27)
# error check_balance() assumes ERTS_MAX_PROCESS < (1 << 27)
#endif
+ ErtsMigrationPaths *new_mpaths, *old_mpaths;
ErtsRunQueueBalance avg = {0};
Sint64 scheds_reds, full_scheds_reds;
int forced, active, current_active, oowc, half_full_scheds, full_scheds,
@@ -3134,9 +3617,9 @@ check_balance(ErtsRunQueue *c_rq)
ERTS_FOREACH_RUNQ(rq,
{
if (rq->waiting)
- rq->flags |= ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK;
+ ERTS_RUNQ_FLGS_SET(rq, ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK);
else
- rq->flags &= ~ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK;
+ ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK);
rq->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS;
});
@@ -3178,7 +3661,7 @@ check_balance(ErtsRunQueue *c_rq)
ErtsRunQueue *rq = ERTS_RUNQ_IX(qix);
erts_smp_runq_lock(rq);
- run_queue_info[qix].flags = rq->flags;
+ run_queue_info[qix].flags = ERTS_RUNQ_FLGS_GET_NOB(rq);
for (pix = 0; pix < ERTS_NO_PROC_PRIO_LEVELS; pix++) {
run_queue_info[qix].prio[pix].max_len
= rq->procs.prio_info[pix].max_len;
@@ -3512,47 +3995,27 @@ erts_fprintf(stderr, "--------------------------------\n");
set_no_active_runqs(active);
balance_info.halftime = 1;
- erts_smp_atomic32_set_nob(&balance_info.checking_balance, 0);
+ new_mpaths = alloc_mpaths();
+
+ /* Write migration paths */
- /* Write migration paths and reset balance statistics in all queues */
for (qix = 0; qix < blnc_no_rqs; qix++) {
int mqix;
- Uint32 flags;
- ErtsRunQueue *rq = ERTS_RUNQ_IX(qix);
- ErtsRunQueueInfo *rqi;
- flags = run_queue_info[qix].flags;
- erts_smp_runq_lock(rq);
- flags |= (rq->flags & ~ERTS_RUNQ_FLGS_MIGRATION_INFO);
- ASSERT(!(flags & ERTS_RUNQ_FLG_OUT_OF_WORK));
- if (rq->waiting)
- flags |= ERTS_RUNQ_FLG_OUT_OF_WORK;
+ Uint32 flags = run_queue_info[qix].flags;
+ ErtsMigrationPath *mp = &new_mpaths->mpath[qix];
- rq->full_reds_history_sum
- = run_queue_info[qix].full_reds_history_sum;
- rq->full_reds_history[freds_hist_ix]
- = run_queue_info[qix].full_reds_history_change;
+ mp->flags = flags;
+ mp->misc_evac_runq = NULL;
- ERTS_DBG_CHK_FULL_REDS_HISTORY(rq);
-
- rq->out_of_work_count = 0;
- rq->flags = flags;
- rq->max_len = rq->len;
for (pix = 0; pix < ERTS_NO_PRIO_LEVELS; pix++) {
- rqi = (pix == ERTS_PORT_PRIO_LEVEL
- ? &rq->ports.info
- : &rq->procs.prio_info[pix]);
- rqi->max_len = rqi->len;
- rqi->reds = 0;
if (!(ERTS_CHK_RUNQ_FLG_EMIGRATE(flags, pix)
| ERTS_CHK_RUNQ_FLG_IMMIGRATE(flags, pix))) {
ASSERT(run_queue_info[qix].prio[pix].immigrate_from < 0);
ASSERT(run_queue_info[qix].prio[pix].emigrate_to < 0);
-#ifdef DEBUG
- rqi->migrate.limit.this = -1;
- rqi->migrate.limit.other = -1;
- ERTS_DBG_SET_INVALID_RUNQP(rqi->migrate.runq, 0x2);
-#endif
-
+ mp->prio[pix].limit.this = -1;
+ mp->prio[pix].limit.other = -1;
+ mp->prio[pix].runq = NULL;
+ mp->prio[pix].flags = 0;
}
else if (ERTS_CHK_RUNQ_FLG_EMIGRATE(flags, pix)) {
ASSERT(!ERTS_CHK_RUNQ_FLG_IMMIGRATE(flags, pix));
@@ -3560,11 +4023,12 @@ erts_fprintf(stderr, "--------------------------------\n");
ASSERT(run_queue_info[qix].prio[pix].emigrate_to >= 0);
mqix = run_queue_info[qix].prio[pix].emigrate_to;
- rqi->migrate.limit.this
+ mp->prio[pix].limit.this
= run_queue_info[qix].prio[pix].migration_limit;
- rqi->migrate.limit.other
+ mp->prio[pix].limit.other
= run_queue_info[mqix].prio[pix].migration_limit;
- rqi->migrate.runq = ERTS_RUNQ_IX(mqix);
+ mp->prio[pix].runq = ERTS_RUNQ_IX(mqix);
+ mp->prio[pix].flags = run_queue_info[mqix].flags;
}
else {
ASSERT(ERTS_CHK_RUNQ_FLG_IMMIGRATE(flags, pix));
@@ -3572,24 +4036,142 @@ erts_fprintf(stderr, "--------------------------------\n");
ASSERT(run_queue_info[qix].prio[pix].immigrate_from >= 0);
mqix = run_queue_info[qix].prio[pix].immigrate_from;
- rqi->migrate.limit.this
+ mp->prio[pix].limit.this
= run_queue_info[qix].prio[pix].migration_limit;
- rqi->migrate.limit.other
+ mp->prio[pix].limit.other
= run_queue_info[mqix].prio[pix].migration_limit;
- rqi->migrate.runq = ERTS_RUNQ_IX(mqix);
+ mp->prio[pix].runq = ERTS_RUNQ_IX(mqix);
+ mp->prio[pix].flags = run_queue_info[mqix].flags;
}
}
+ }
+
+ old_mpaths = erts_get_migration_paths_managed();
+
+ /* Keep offline run-queues as is */
+ for (qix = blnc_no_rqs; qix < erts_no_schedulers; qix++) {
+ ErtsMigrationPath *nmp = &new_mpaths->mpath[qix];
+ ErtsMigrationPath *omp = &old_mpaths->mpath[qix];
+
+ nmp->flags = omp->flags;
+ nmp->misc_evac_runq = omp->misc_evac_runq;
+
+ for (pix = 0; pix < ERTS_NO_PRIO_LEVELS; pix++) {
+ nmp->prio[pix].limit.this = omp->prio[pix].limit.this;
+ nmp->prio[pix].limit.other = omp->prio[pix].limit.other;
+ nmp->prio[pix].runq = omp->prio[pix].runq;
+ nmp->prio[pix].flags = omp->prio[pix].flags;
+ }
+ }
+
+
+ /* Publish new migration paths... */
+ erts_atomic_set_wb(&erts_migration_paths, (erts_aint_t) new_mpaths);
+
+ /* Reset balance statistics in all online queues */
+ for (qix = 0; qix < blnc_no_rqs; qix++) {
+ Uint32 flags = run_queue_info[qix].flags;
+ ErtsRunQueue *rq = ERTS_RUNQ_IX(qix);
+
+ erts_smp_runq_lock(rq);
+ ASSERT(!(flags & ERTS_RUNQ_FLG_OUT_OF_WORK));
+ if (rq->waiting)
+ flags |= ERTS_RUNQ_FLG_OUT_OF_WORK;
+
+ rq->full_reds_history_sum
+ = run_queue_info[qix].full_reds_history_sum;
+ rq->full_reds_history[freds_hist_ix]
+ = run_queue_info[qix].full_reds_history_change;
+
+ ERTS_DBG_CHK_FULL_REDS_HISTORY(rq);
+
+ rq->out_of_work_count = 0;
+ (void) ERTS_RUNQ_FLGS_MASK_SET(rq, ERTS_RUNQ_FLGS_MIGRATION_INFO, flags);
+
+ rq->max_len = rq->len;
+ for (pix = 0; pix < ERTS_NO_PRIO_LEVELS; pix++) {
+ ErtsRunQueueInfo *rqi;
+ rqi = (pix == ERTS_PORT_PRIO_LEVEL
+ ? &rq->ports.info
+ : &rq->procs.prio_info[pix]);
+ erts_smp_reset_max_len(rq, rqi);
+ rqi->reds = 0;
+ }
rq->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS;
erts_smp_runq_unlock(rq);
}
+ erts_smp_atomic32_set_nob(&balance_info.checking_balance, 0);
+
balance_info.n++;
+ retire_mpaths(old_mpaths);
erts_smp_mtx_unlock(&balance_info.update_mtx);
erts_smp_runq_lock(c_rq);
}
+static void
+change_no_used_runqs(int used)
+{
+ ErtsMigrationPaths *new_mpaths, *old_mpaths;
+ int active, qix;
+ erts_smp_mtx_lock(&balance_info.update_mtx);
+ get_no_runqs(&active, NULL);
+ set_no_used_runqs(used);
+
+ old_mpaths = erts_get_migration_paths_managed();
+ new_mpaths = alloc_mpaths();
+
+ /* Write migration paths... */
+
+ for (qix = 0; qix < used; qix++) {
+ int pix;
+ ErtsMigrationPath *omp = &old_mpaths->mpath[qix];
+ ErtsMigrationPath *nmp = &new_mpaths->mpath[qix];
+
+ nmp->flags = omp->flags & ~ERTS_RUNQ_FLGS_MIGRATION_QMASKS;
+ nmp->misc_evac_runq = NULL;
+
+ for (pix = 0; pix < ERTS_NO_PRIO_LEVELS; pix++) {
+ nmp->prio[pix].limit.this = -1;
+ nmp->prio[pix].limit.other = -1;
+ nmp->prio[pix].runq = NULL;
+ nmp->prio[pix].flags = 0;
+ }
+ }
+ for (qix = used; qix < erts_no_run_queues; qix++) {
+ int pix;
+ ErtsRunQueue *to_rq = ERTS_RUNQ_IX(qix % used);
+ ErtsMigrationPath *nmp = &new_mpaths->mpath[qix];
+
+ nmp->flags = (ERTS_RUNQ_FLGS_EMIGRATE_QMASK
+ | ERTS_RUNQ_FLGS_EVACUATE_QMASK);
+ nmp->misc_evac_runq = to_rq;
+ for (pix = 0; pix < ERTS_NO_PRIO_LEVELS; pix++) {
+ nmp->prio[pix].limit.this = -1;
+ nmp->prio[pix].limit.other = -1;
+ nmp->prio[pix].runq = to_rq;
+ nmp->prio[pix].flags = 0;
+ }
+ }
+
+ /* ... and publish them. */
+ erts_atomic_set_wb(&erts_migration_paths, (erts_aint_t) new_mpaths);
+
+ retire_mpaths(old_mpaths);
+
+ /* Make sure that we balance soon... */
+ balance_info.forced_check_balance = 1;
+
+ erts_smp_mtx_unlock(&balance_info.update_mtx);
+
+ erts_smp_runq_lock(ERTS_RUNQ_IX(0));
+ ERTS_RUNQ_IX(0)->check_balance_reds = 0;
+ erts_smp_runq_unlock(ERTS_RUNQ_IX(0));
+}
+
+
#endif /* #ifdef ERTS_SMP */
Uint
@@ -3681,7 +4263,6 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online)
ErtsRunQueue *rq = ERTS_RUNQ_IX(ix);
rq->ix = ix;
- erts_smp_atomic32_init_nob(&rq->info_flags, ERTS_RUNQ_IFLG_NONEMPTY);
/* make sure that the "extra" id correponds to the schedulers
* id if the esdp->no <-> ix+1 mapping change.
@@ -3692,7 +4273,7 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online)
rq->waiting = 0;
rq->woken = 0;
- rq->flags = 0;
+ ERTS_RUNQ_FLGS_INIT(rq, ERTS_RUNQ_FLG_NONEMPTY);
rq->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS;
rq->full_reds_history_sum = 0;
for (rix = 0; rix < ERTS_FULL_REDS_HISTORY_SIZE; rix++) {
@@ -3706,19 +4287,14 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online)
rq->wakeup_other_reds = 0;
rq->halt_in_progress = 0;
- rq->procs.len = 0;
rq->procs.pending_exiters = NULL;
rq->procs.context_switches = 0;
rq->procs.reductions = 0;
for (pix = 0; pix < ERTS_NO_PROC_PRIO_LEVELS; pix++) {
- rq->procs.prio_info[pix].len = 0;
+ erts_smp_atomic32_init_nob(&rq->procs.prio_info[pix].len, 0);
rq->procs.prio_info[pix].max_len = 0;
rq->procs.prio_info[pix].reds = 0;
- rq->procs.prio_info[pix].migrate.limit.this = 0;
- rq->procs.prio_info[pix].migrate.limit.other = 0;
- ERTS_DBG_SET_INVALID_RUNQP(rq->procs.prio_info[pix].migrate.runq,
- 0x0);
if (pix < ERTS_NO_PROC_PRIO_LEVELS - 1) {
rq->procs.prio[pix].first = NULL;
rq->procs.prio[pix].last = NULL;
@@ -3727,14 +4303,10 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online)
rq->misc.start = NULL;
rq->misc.end = NULL;
- rq->misc.evac_runq = NULL;
- rq->ports.info.len = 0;
+ erts_smp_atomic32_init_nob(&rq->ports.info.len, 0);
rq->ports.info.max_len = 0;
rq->ports.info.reds = 0;
- rq->ports.info.migrate.limit.this = 0;
- rq->ports.info.migrate.limit.other = 0;
- rq->ports.info.migrate.runq = NULL;
rq->ports.start = NULL;
rq->ports.end = NULL;
}
@@ -3857,10 +4429,12 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online)
balance_info.prev_rise.reds = 0;
balance_info.n = 0;
+ init_migration_paths();
+
if (no_schedulers_online < no_schedulers) {
+ change_no_used_runqs(no_schedulers_online);
for (ix = no_schedulers_online; ix < erts_no_run_queues; ix++)
- evacuate_run_queue(ERTS_RUNQ_IX(ix),
- ERTS_RUNQ_IX(ix % no_schedulers_online));
+ suspend_run_queue(ERTS_RUNQ_IX(ix));
}
schdlr_sspnd.wait_curr_online = no_schedulers_online;
@@ -3923,91 +4497,208 @@ erts_get_scheduler_data(void)
#endif
-static int remove_proc_from_runq(ErtsRunQueue *rq, Process *p, int to_inactive);
+/*
+ * scheduler_out_process() return with c_rq locked.
+ */
+static ERTS_INLINE int
+schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p)
+{
+ erts_aint32_t a, e, n;
+ int res = 0;
+
+ a = state;
+
+ while (1) {
+ n = e = a;
+
+ ASSERT(a & ERTS_PSFLG_RUNNING);
+ ASSERT(!(a & ERTS_PSFLG_IN_RUNQ));
+
+ n &= ~ERTS_PSFLG_RUNNING;
+ if ((a & (ERTS_PSFLG_ACTIVE|ERTS_PSFLG_SUSPENDED)) == ERTS_PSFLG_ACTIVE)
+ n |= ERTS_PSFLG_IN_RUNQ;
+ a = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e);
+ if (a == e)
+ break;
+ }
+
+ if (!(n & ERTS_PSFLG_IN_RUNQ)) {
+ if (erts_system_profile_flags.runnable_procs)
+ profile_runnable_proc(p, am_inactive);
+ }
+ else {
+ int prio = (int) (ERTS_PSFLG_PRIO_MASK & n);
+ ErtsRunQueue *runq = erts_get_runq_proc(p);
+
+ ASSERT(!(n & ERTS_PSFLG_SUSPENDED));
+
+#ifdef ERTS_SMP
+ if (!(ERTS_PSFLG_BOUND & n)) {
+ ErtsRunQueue *new_runq = erts_check_emigration_need(runq, prio);
+ if (new_runq) {
+ RUNQ_SET_RQ(&p->run_queue, new_runq);
+ runq = new_runq;
+ }
+ }
+#endif
+ ASSERT(runq);
+ res = 1;
+
+ erts_smp_runq_lock(runq);
+
+ /* Enqueue the process */
+ enqueue_process(runq, prio, p);
+
+ if (runq == c_rq)
+ return res;
+ erts_smp_runq_unlock(runq);
+ smp_notify_inc_runq(runq);
+ }
+ erts_smp_runq_lock(c_rq);
+ return res;
+}
static ERTS_INLINE void
-suspend_process(ErtsRunQueue *rq, Process *p)
+add2runq(Process *p, erts_aint32_t state)
{
- ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p));
- ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
- p->rcount++; /* count number of suspend */
-#ifdef ERTS_SMP
- ASSERT(!(p->runq_flags & ERTS_PROC_RUNQ_FLG_RUNNING)
- || p == erts_get_current_process());
- ASSERT(p->status != P_RUNNING
- || p->runq_flags & ERTS_PROC_RUNQ_FLG_RUNNING);
- if (p->status_flags & ERTS_PROC_SFLG_PENDADD2SCHEDQ)
- goto runable;
-#endif
- switch(p->status) {
- case P_SUSPENDED:
- break;
- case P_RUNABLE:
+ int prio = (int) (ERTS_PSFLG_PRIO_MASK & state);
+ ErtsRunQueue *runq = erts_get_runq_proc(p);
+
#ifdef ERTS_SMP
- runable:
- if (!ERTS_PROC_PENDING_EXIT(p))
+ if (!(ERTS_PSFLG_BOUND & state)) {
+ ErtsRunQueue *new_runq = erts_check_emigration_need(runq, prio);
+ if (new_runq) {
+ RUNQ_SET_RQ(&p->run_queue, new_runq);
+ runq = new_runq;
+ }
+ }
#endif
- remove_proc_from_runq(rq, p, 1);
- /* else:
- * leave process in schedq so it will discover the pending exit
- */
- p->rstatus = P_RUNABLE; /* wakeup as runnable */
- break;
- case P_RUNNING:
- p->rstatus = P_RUNABLE; /* wakeup as runnable */
- break;
- case P_WAITING:
- p->rstatus = P_WAITING; /* wakeup as waiting */
- break;
- case P_EXITING:
- return; /* ignore this */
- case P_GARBING:
- case P_FREE:
- erl_exit(1, "bad state in suspend_process()\n");
+ ASSERT(runq);
+
+ erts_smp_runq_lock(runq);
+
+ /* Enqueue the process */
+ enqueue_process(runq, prio, p);
+
+ erts_smp_runq_unlock(runq);
+ smp_notify_inc_runq(runq);
+
+}
+
+static ERTS_INLINE void
+schedule_process(Process *p, erts_aint32_t state, int active_enq)
+{
+ erts_aint32_t a = state, n;
+
+ while (1) {
+ erts_aint32_t e;
+ n = e = a;
+ ASSERT(!(a & ERTS_PSFLG_FREE));
+ n |= ERTS_PSFLG_ACTIVE;
+ if (!(a & (ERTS_PSFLG_SUSPENDED|ERTS_PSFLG_RUNNING)))
+ n |= ERTS_PSFLG_IN_RUNQ;
+ a = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e);
+ if (a == e)
+ break;
+ if (!active_enq && (a & ERTS_PSFLG_ACTIVE))
+ return; /* Someone else activated process ... */
}
- if ((erts_system_profile_flags.runnable_procs) && (p->rcount == 1) && (p->status != P_WAITING)) {
- profile_runnable_proc(p, am_inactive);
+#ifdef RRR_DEBUG
+ if (active_enq)
+ erts_fprintf(stderr, "! state=0x%x\n", n);
+#endif
+
+ if (erts_system_profile_flags.runnable_procs
+ && !(a & (ERTS_PSFLG_ACTIVE|ERTS_PSFLG_SUSPENDED))) {
+ profile_runnable_proc(p, am_active);
}
- p->status = P_SUSPENDED;
-
+ if ((n & ERTS_PSFLG_IN_RUNQ) && !(a & ERTS_PSFLG_IN_RUNQ)) {
+#ifdef RRR_DEBUG
+ if (active_enq)
+ erts_fprintf(stderr, "-->\n");
+#endif
+ add2runq(p, n);
+ }
}
-static ERTS_INLINE void
-resume_process(Process *p)
+void
+erts_schedule_process(Process *p, erts_aint32_t state)
{
- Uint32 *statusp;
+ schedule_process(p, state, 0);
+}
+
+static ERTS_INLINE int
+suspend_process(Process *c_p, Process *p)
+{
+ erts_aint32_t state = erts_smp_atomic32_read_acqb(&p->state);
+ int suspended = 0;
ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p));
- switch (p->status) {
- case P_SUSPENDED:
- statusp = &p->status;
- break;
- case P_GARBING:
- if (p->gcstatus == P_SUSPENDED) {
- statusp = &p->gcstatus;
- break;
+
+ if ((state & ERTS_PSFLG_SUSPENDED))
+ suspended = -1;
+ else {
+ if (c_p == p) {
+ state = erts_smp_atomic32_read_bor_relb(&p->state,
+ ERTS_PSFLG_SUSPENDED);
+ state |= ERTS_PSFLG_SUSPENDED;
+ ASSERT(state & ERTS_PSFLG_RUNNING);
+ suspended = 1;
+ }
+ else {
+ while (!(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_EXITING))) {
+ erts_aint32_t e, n;
+ n = e = state;
+ n |= ERTS_PSFLG_SUSPENDED;
+ state = erts_smp_atomic32_cmpxchg_relb(&p->state, n, e);
+ if (state == e) {
+ state = n;
+ suspended = 1;
+ break;
+ }
+ }
}
- /* Fall through */
- default:
- return;
}
+ if (state & ERTS_PSFLG_SUSPENDED) {
+
+ ASSERT(!(ERTS_PSFLG_RUNNING & state)
+ || p == erts_get_current_process());
+
+ if (erts_system_profile_flags.runnable_procs
+ && (p->rcount == 0)
+ && (state & ERTS_PSFLG_ACTIVE)) {
+ profile_runnable_proc(p, am_inactive);
+ }
+
+ p->rcount++; /* count number of suspend */
+ }
+ return suspended;
+}
+
+static ERTS_INLINE void
+resume_process(Process *p)
+{
+ erts_aint32_t state;
+ ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p));
+
ASSERT(p->rcount > 0);
if (--p->rcount > 0) /* multiple suspend */
return;
- switch(p->rstatus) {
- case P_RUNABLE:
- erts_add_to_runq(p);
- break;
- case P_WAITING:
- *statusp = P_WAITING;
- break;
- default:
- erl_exit(1, "bad state in resume_process()\n");
+
+ state = erts_smp_atomic32_read_band_mb(&p->state, ~ERTS_PSFLG_SUSPENDED);
+ state &= ~ERTS_PSFLG_SUSPENDED;
+#ifdef RRR_DEBUG
+ erts_fprintf(stderr, "%T - state=0x%x\n", p->id, state);
+#endif
+ if ((state & (ERTS_PSFLG_EXITING
+ | ERTS_PSFLG_ACTIVE
+ | ERTS_PSFLG_IN_RUNQ
+ | ERTS_PSFLG_RUNNING)) == ERTS_PSFLG_ACTIVE) {
+ schedule_process(p, state, 1);
}
- p->rstatus = P_FREE;
}
int
@@ -4131,6 +4822,7 @@ suspend_scheduler(ErtsSchedulerData *esdp)
int wake = 0;
erts_aint32_t aux_work;
int thr_prgr_active = 1;
+ ErtsStuckBoundProcesses sbp = {NULL, NULL};
/*
* Schedulers may be suspended in two different ways:
@@ -4145,6 +4837,8 @@ suspend_scheduler(ErtsSchedulerData *esdp)
ASSERT(no != 1);
+ evacuate_run_queue(esdp->run_queue, &sbp);
+
erts_smp_runq_unlock(esdp->run_queue);
erts_sched_check_cpu_bind_prep_suspend(esdp);
@@ -4208,17 +4902,26 @@ suspend_scheduler(ErtsSchedulerData *esdp)
erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
while (1) {
+ erts_aint32_t qmask;
erts_aint32_t flgs;
+ qmask = (ERTS_RUNQ_FLGS_GET(esdp->run_queue)
+ & ERTS_RUNQ_FLGS_QMASK);
aux_work = erts_atomic32_read_acqb(&ssi->aux_work);
- if (aux_work) {
+ if (aux_work|qmask) {
if (!thr_prgr_active) {
erts_thr_progress_active(esdp, thr_prgr_active = 1);
sched_wall_time_change(esdp, 1);
}
- aux_work = handle_aux_work(&esdp->aux_work_data, aux_work);
+ if (aux_work)
+ aux_work = handle_aux_work(&esdp->aux_work_data, aux_work);
if (aux_work && erts_thr_progress_update(esdp))
erts_thr_progress_leader_update(esdp);
+ if (qmask) {
+ erts_smp_runq_lock(esdp->run_queue);
+ evacuate_run_queue(esdp->run_queue, &sbp);
+ erts_smp_runq_unlock(esdp->run_queue);
+ }
}
if (!aux_work) {
@@ -4288,56 +4991,11 @@ suspend_scheduler(ErtsSchedulerData *esdp)
erts_smp_runq_lock(esdp->run_queue);
non_empty_runq(esdp->run_queue);
+ schedule_bound_processes(esdp->run_queue, &sbp);
+
erts_sched_check_cpu_bind_post_suspend(esdp);
}
-#define ERTS_RUNQ_RESET_SUSPEND_INFO(RQ, DBG_ID) \
-do { \
- int pix__; \
- (RQ)->misc.evac_runq = NULL; \
- (RQ)->ports.info.migrate.runq = NULL; \
- (RQ)->flags &= ~(ERTS_RUNQ_FLGS_IMMIGRATE_QMASK \
- | ERTS_RUNQ_FLGS_EMIGRATE_QMASK \
- | ERTS_RUNQ_FLGS_EVACUATE_QMASK \
- | ERTS_RUNQ_FLG_SUSPENDED); \
- (RQ)->flags |= (ERTS_RUNQ_FLG_OUT_OF_WORK \
- | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK); \
- (RQ)->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS; \
- erts_smp_atomic32_read_band_nob(&(RQ)->info_flags, ~ERTS_RUNQ_IFLG_SUSPENDED);\
- for (pix__ = 0; pix__ < ERTS_NO_PROC_PRIO_LEVELS; pix__++) { \
- (RQ)->procs.prio_info[pix__].max_len = 0; \
- (RQ)->procs.prio_info[pix__].reds = 0; \
- ERTS_DBG_SET_INVALID_RUNQP((RQ)->procs.prio_info[pix__].migrate.runq,\
- (DBG_ID)); \
- } \
- (RQ)->ports.info.max_len = 0; \
- (RQ)->ports.info.reds = 0; \
-} while (0)
-
-#define ERTS_RUNQ_RESET_MIGRATION_PATHS__(RQ) \
-do { \
- ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked((RQ))); \
- (RQ)->misc.evac_runq = NULL; \
- (RQ)->ports.info.migrate.runq = NULL; \
- (RQ)->flags &= ~(ERTS_RUNQ_FLGS_IMMIGRATE_QMASK \
- | ERTS_RUNQ_FLGS_EMIGRATE_QMASK \
- | ERTS_RUNQ_FLGS_EVACUATE_QMASK); \
-} while (0)
-
-#ifdef DEBUG
-#define ERTS_RUNQ_RESET_MIGRATION_PATHS(RQ, DBG_ID) \
-do { \
- int pix__; \
- ERTS_RUNQ_RESET_MIGRATION_PATHS__((RQ)); \
- for (pix__ = 0; pix__ < ERTS_NO_PROC_PRIO_LEVELS; pix__++) \
- ERTS_DBG_SET_INVALID_RUNQP((RQ)->procs.prio_info[pix__].migrate.runq,\
- (DBG_ID)); \
-} while (0)
-#else
-#define ERTS_RUNQ_RESET_MIGRATION_PATHS(RQ, DBG_ID) \
- ERTS_RUNQ_RESET_MIGRATION_PATHS__((RQ))
-#endif
-
ErtsSchedSuspendResult
erts_schedulers_state(Uint *total,
Uint *online,
@@ -4407,27 +5065,13 @@ erts_set_schedulers_online(Process *p,
have_unlocked_plocks = 1;
erts_smp_proc_unlock(p, plocks);
}
- erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
- erts_smp_mtx_lock(&balance_info.update_mtx);
- for (ix = online; ix < no; ix++) {
- ErtsRunQueue *rq = ERTS_RUNQ_IX(ix);
- erts_smp_runq_lock(rq);
- ERTS_RUNQ_RESET_SUSPEND_INFO(rq, 0x5);
- erts_smp_runq_unlock(rq);
- scheduler_ix_resume_wake(ix);
- }
- /*
- * Spread evacuation paths among all online
- * run queues.
- */
- for (ix = no; ix < erts_no_run_queues; ix++) {
- ErtsRunQueue *from_rq = ERTS_RUNQ_IX(ix);
- ErtsRunQueue *to_rq = ERTS_RUNQ_IX(ix % no);
- evacuate_run_queue(from_rq, to_rq);
- }
- set_no_used_runqs(no);
- erts_smp_mtx_unlock(&balance_info.update_mtx);
- erts_smp_mtx_lock(&schdlr_sspnd.mtx);
+ change_no_used_runqs(no);
+
+ for (ix = online; ix < no; ix++)
+ resume_run_queue(ERTS_RUNQ_IX(ix));
+
+ for (ix = no; ix < erts_no_run_queues; ix++)
+ suspend_run_queue(ERTS_RUNQ_IX(ix));
}
res = ERTS_SCHDLR_SSPND_DONE;
}
@@ -4454,25 +5098,11 @@ erts_set_schedulers_online(Process *p,
have_unlocked_plocks = 1;
erts_smp_proc_unlock(p, plocks);
}
- erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
- erts_smp_mtx_lock(&balance_info.update_mtx);
-
- for (ix = 0; ix < online; ix++) {
- ErtsRunQueue *rq = ERTS_RUNQ_IX(ix);
- erts_smp_runq_lock(rq);
- ERTS_RUNQ_RESET_MIGRATION_PATHS(rq, 0x6);
- erts_smp_runq_unlock(rq);
- }
- /*
- * Evacutation order important! Newly suspended run queues
- * has to be evacuated last.
- */
- for (ix = erts_no_run_queues-1; ix >= no; ix--)
- evacuate_run_queue(ERTS_RUNQ_IX(ix),
- ERTS_RUNQ_IX(ix % no));
- set_no_used_runqs(no);
- erts_smp_mtx_unlock(&balance_info.update_mtx);
- erts_smp_mtx_lock(&schdlr_sspnd.mtx);
+
+ change_no_used_runqs(no);
+ for (ix = no; ix < erts_no_run_queues; ix++)
+ suspend_run_queue(ERTS_RUNQ_IX(ix));
+
for (ix = no; ix < online; ix++) {
ErtsRunQueue *rq = ERTS_RUNQ_IX(ix);
wake_scheduler(rq, 0);
@@ -4569,25 +5199,14 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all)
schdlr_sspnd.msb.wait_active = 2;
}
- erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
- erts_smp_mtx_lock(&balance_info.update_mtx);
- set_no_used_runqs(1);
- for (ix = 0; ix < online; ix++) {
+ change_no_used_runqs(1);
+ for (ix = 1; ix < erts_no_run_queues; ix++)
+ suspend_run_queue(ERTS_RUNQ_IX(ix));
+
+ for (ix = 1; ix < online; ix++) {
ErtsRunQueue *rq = ERTS_RUNQ_IX(ix);
- erts_smp_runq_lock(rq);
- ASSERT(!(rq->flags & ERTS_RUNQ_FLG_SUSPENDED));
- ERTS_RUNQ_RESET_MIGRATION_PATHS(rq, 0x7);
- erts_smp_runq_unlock(rq);
+ wake_scheduler(rq, 0);
}
- /*
- * Evacuate all activities in all other run queues
- * into the first run queue. Note order is important,
- * online run queues has to be evacuated last.
- */
- for (ix = erts_no_run_queues-1; ix >= 1; ix--)
- evacuate_run_queue(ERTS_RUNQ_IX(ix), ERTS_RUNQ_IX(0));
- erts_smp_mtx_unlock(&balance_info.update_mtx);
- erts_smp_mtx_lock(&schdlr_sspnd.mtx);
if (erts_smp_atomic32_read_nob(&schdlr_sspnd.active)
!= schdlr_sspnd.msb.wait_active) {
@@ -4631,15 +5250,6 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all)
plp = proclist_create(p);
plp->next = schdlr_sspnd.msb.procs;
schdlr_sspnd.msb.procs = plp;
-#ifdef DEBUG
- ERTS_FOREACH_RUNQ(srq,
- {
- if (srq != ERTS_RUNQ_IX(0)) {
- ASSERT(ERTS_EMPTY_RUNQ(srq));
- ASSERT(srq->flags & ERTS_RUNQ_FLG_SUSPENDED);
- }
- });
-#endif
ASSERT(p->scheduler_data);
}
}
@@ -4671,27 +5281,6 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all)
res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED;
else {
ERTS_SCHDLR_SSPND_CHNG_SET(ERTS_SCHDLR_SSPND_CHNG_MSB, 0);
-#ifdef DEBUG
- ERTS_FOREACH_RUNQ(rq,
- {
- if (rq != p->scheduler_data->run_queue) {
- if (!ERTS_EMPTY_RUNQ(rq)) {
- Process *rp;
- int pix;
- ASSERT(rq->ports.info.len == 0);
- for (pix = 0; pix < ERTS_NO_PROC_PRIO_LEVELS; pix++) {
- for (rp = rq->procs.prio[pix].first;
- rp;
- rp = rp->next) {
- ASSERT(rp->bound_runq);
- }
- }
- }
-
- ASSERT(rq->flags & ERTS_RUNQ_FLG_SUSPENDED);
- }
- });
-#endif
p->flags &= ~F_HAVE_BLCKD_MSCHED;
schdlr_sspnd.msb.ongoing = 0;
if (schdlr_sspnd.online == 1) {
@@ -4701,35 +5290,19 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all)
}
else {
int online = schdlr_sspnd.online;
- erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
if (plocks) {
have_unlocked_plocks = 1;
erts_smp_proc_unlock(p, plocks);
}
- erts_smp_mtx_lock(&balance_info.update_mtx);
+
+ change_no_used_runqs(online);
/* Resume all online run queues */
- for (ix = 1; ix < online; ix++) {
- ErtsRunQueue *rq = ERTS_RUNQ_IX(ix);
- erts_smp_runq_lock(rq);
- ERTS_RUNQ_RESET_SUSPEND_INFO(rq, 0x4);
- erts_smp_runq_unlock(rq);
- scheduler_ix_resume_wake(ix);
- }
+ for (ix = 1; ix < online; ix++)
+ resume_run_queue(ERTS_RUNQ_IX(ix));
- /* Spread evacuation paths among all online run queues */
for (ix = online; ix < erts_no_run_queues; ix++)
- evacuate_run_queue(ERTS_RUNQ_IX(ix),
- ERTS_RUNQ_IX(ix % online));
-
- set_no_used_runqs(online);
- /* Make sure that we balance soon... */
- balance_info.forced_check_balance = 1;
- erts_smp_runq_lock(ERTS_RUNQ_IX(0));
- ERTS_RUNQ_IX(0)->check_balance_reds = 0;
- erts_smp_runq_unlock(ERTS_RUNQ_IX(0));
- erts_smp_mtx_unlock(&balance_info.update_mtx);
- erts_smp_mtx_lock(&schdlr_sspnd.mtx);
+ suspend_run_queue(ERTS_RUNQ_IX(ix));
}
res = ERTS_SCHDLR_SSPND_DONE;
}
@@ -5036,10 +5609,7 @@ handle_pend_sync_suspend(Process *suspendee,
if (suspender) {
ASSERT(is_nil(suspender->suspendee));
if (suspendee_alive) {
- ErtsRunQueue *rq = erts_get_runq_proc(suspendee);
- erts_smp_runq_lock(rq);
- suspend_process(rq, suspendee);
- erts_smp_runq_unlock(rq);
+ erts_suspend(suspendee, suspendee_locks, NULL);
suspender->suspendee = suspendee->id;
}
/* suspender is suspended waiting for suspendee to suspend;
@@ -5082,10 +5652,9 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks,
resume_process(rp);
}
else {
- ErtsRunQueue *cp_rq, *rp_rq;
rp = erts_pid2proc(c_p, c_p_locks|ERTS_PROC_LOCK_STATUS,
- pid, ERTS_PROC_LOCK_STATUS);
+ pid, pid_locks|ERTS_PROC_LOCK_STATUS);
if (!rp) {
c_p->flags &= ~F_P2PNR_RESCHED;
@@ -5094,58 +5663,36 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks,
ASSERT(!(c_p->flags & F_P2PNR_RESCHED));
- cp_rq = erts_get_runq_proc(c_p);
- rp_rq = erts_get_runq_proc(rp);
- erts_smp_runqs_lock(cp_rq, rp_rq);
- if (rp->runq_flags & ERTS_PROC_RUNQ_FLG_RUNNING) {
- running:
- /* Phiu... */
-
- /*
- * If we got pending suspenders and suspend ourselves waiting
- * to suspend another process we might deadlock.
- * In this case we have to yield, be suspended by
- * someone else and then do it all over again.
- */
- if (!c_p->pending_suspenders) {
- /* Mark rp pending for suspend by c_p */
- add_pend_suspend(rp, c_p->id, handle_pend_sync_suspend);
- ASSERT(is_nil(c_p->suspendee));
-
- /* Suspend c_p; when rp is suspended c_p will be resumed. */
- suspend_process(cp_rq, c_p);
- c_p->flags |= F_P2PNR_RESCHED;
- }
- /* Yield (caller is assumed to yield immediately in bif). */
- erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS);
- rp = ERTS_PROC_LOCK_BUSY;
+ if (suspend) {
+ if (suspend_process(c_p, rp))
+ goto done;
}
else {
- ErtsProcLocks need_locks = pid_locks & ~ERTS_PROC_LOCK_STATUS;
- if (need_locks && erts_smp_proc_trylock(rp, need_locks) == EBUSY) {
- erts_smp_runqs_unlock(cp_rq, rp_rq);
- erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS);
- rp = erts_pid2proc(c_p, c_p_locks|ERTS_PROC_LOCK_STATUS,
- pid, pid_locks|ERTS_PROC_LOCK_STATUS);
- if (!rp)
- goto done;
- /* run-queues may have changed */
- cp_rq = erts_get_runq_proc(c_p);
- rp_rq = erts_get_runq_proc(rp);
- erts_smp_runqs_lock(cp_rq, rp_rq);
- if (rp->runq_flags & ERTS_PROC_RUNQ_FLG_RUNNING) {
- /* Ahh... */
- erts_smp_proc_unlock(rp,
- pid_locks & ~ERTS_PROC_LOCK_STATUS);
- goto running;
- }
- }
+ if (!(ERTS_PSFLG_RUNNING & erts_smp_atomic32_read_acqb(&rp->state)))
+ goto done;
+
+ }
+
+ /* Other process running */
- /* rp is not running and we got the locks we want... */
- if (suspend)
- suspend_process(rp_rq, rp);
+ /*
+ * If we got pending suspenders and suspend ourselves waiting
+ * to suspend another process we might deadlock.
+ * In this case we have to yield, be suspended by
+ * someone else and then do it all over again.
+ */
+ if (!c_p->pending_suspenders) {
+ /* Mark rp pending for suspend by c_p */
+ add_pend_suspend(rp, c_p->id, handle_pend_sync_suspend);
+ ASSERT(is_nil(c_p->suspendee));
+
+ /* Suspend c_p; when rp is suspended c_p will be resumed. */
+ suspend_process(c_p, c_p);
+ c_p->flags |= F_P2PNR_RESCHED;
}
- erts_smp_runqs_unlock(cp_rq, rp_rq);
+ /* Yield (caller is assumed to yield immediately in bif). */
+ erts_smp_proc_unlock(rp, pid_locks|ERTS_PROC_LOCK_STATUS);
+ rp = ERTS_PROC_LOCK_BUSY;
}
done:
@@ -5202,36 +5749,26 @@ erts_pid2proc_nropt(Process *c_p, ErtsProcLocks c_p_locks,
return erts_pid2proc_not_running(c_p, c_p_locks, pid, pid_locks);
}
-static ERTS_INLINE void
-do_bif_suspend_process(ErtsSuspendMonitor *smon,
- Process *suspendee,
- ErtsRunQueue *locked_runq)
+static ERTS_INLINE int
+do_bif_suspend_process(Process *c_p,
+ ErtsSuspendMonitor *smon,
+ Process *suspendee)
{
ASSERT(suspendee);
- ASSERT(!suspendee->is_exiting);
+ ASSERT(!ERTS_PROC_IS_EXITING(suspendee));
ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS
& erts_proc_lc_my_proc_locks(suspendee));
if (smon) {
if (!smon->active) {
- ErtsRunQueue *rq;
-
- if (locked_runq)
- rq = locked_runq;
- else {
- rq = erts_get_runq_proc(suspendee);
- erts_smp_runq_lock(rq);
- }
-
- suspend_process(rq, suspendee);
-
- if (!locked_runq)
- erts_smp_runq_unlock(rq);
+ if (!suspend_process(c_p, suspendee))
+ return 0;
}
smon->active += smon->pending;
ASSERT(smon->active);
smon->pending = 0;
+ return 1;
}
-
+ return 0;
}
static void
@@ -5254,10 +5791,17 @@ handle_pend_bif_sync_suspend(Process *suspendee,
erts_delete_suspend_monitor(&suspender->suspend_monitors,
suspendee->id);
else {
+#ifdef DEBUG
+ int res;
+#endif
ErtsSuspendMonitor *smon;
smon = erts_lookup_suspend_monitor(suspender->suspend_monitors,
suspendee->id);
- do_bif_suspend_process(smon, suspendee, NULL);
+#ifdef DEBUG
+ res =
+#endif
+ do_bif_suspend_process(suspendee, smon, suspendee);
+ ASSERT(!smon || res != 0);
suspender->suspendee = suspendee->id;
}
/* suspender is suspended waiting for suspendee to suspend;
@@ -5289,10 +5833,17 @@ handle_pend_bif_async_suspend(Process *suspendee,
erts_delete_suspend_monitor(&suspender->suspend_monitors,
suspendee->id);
else {
+#ifdef DEBUG
+ int res;
+#endif
ErtsSuspendMonitor *smon;
smon = erts_lookup_suspend_monitor(suspender->suspend_monitors,
suspendee->id);
- do_bif_suspend_process(smon, suspendee, NULL);
+#ifdef DEBUG
+ res =
+#endif
+ do_bif_suspend_process(suspendee, smon, suspendee);
+ ASSERT(!smon || res != 0);
}
erts_smp_proc_unlock(suspender, ERTS_PROC_LOCK_LINK);
}
@@ -5377,7 +5928,8 @@ suspend_process_2(BIF_ALIST_2)
/* This is really a piece of cake without SMP support... */
if (!smon->active) {
- suspend_process(ERTS_RUNQ_IX(0), suspendee);
+ erts_smp_atomic32_read_bor_nob(&suspendee->state, ERTS_PSFLG_SUSPENDED);
+ suspend_process(BIF_P, suspendee);
smon->active++;
res = am_true;
}
@@ -5420,21 +5972,15 @@ suspend_process_2(BIF_ALIST_2)
if (smon->pending && unless_suspending)
res = am_false;
else {
- ErtsRunQueue *rq;
if (smon->pending == INT_MAX)
goto system_limit;
smon->pending++;
- rq = erts_get_runq_proc(suspendee);
- erts_smp_runq_lock(rq);
- if (suspendee->runq_flags & ERTS_PROC_RUNQ_FLG_RUNNING)
+ if (!do_bif_suspend_process(BIF_P, smon, suspendee))
add_pend_suspend(suspendee,
BIF_P->id,
handle_pend_bif_async_suspend);
- else
- do_bif_suspend_process(smon, suspendee, rq);
- erts_smp_runq_unlock(rq);
res = am_true;
}
@@ -5471,7 +6017,6 @@ suspend_process_2(BIF_ALIST_2)
/* done */
}
else {
- ErtsRunQueue *cp_rq, *s_rq;
/* We haven't got any active suspends on the suspendee */
/*
@@ -5488,12 +6033,7 @@ suspend_process_2(BIF_ALIST_2)
if (!unless_suspending || smon->pending == 0)
smon->pending++;
- cp_rq = erts_get_runq_proc(BIF_P);
- s_rq = erts_get_runq_proc(suspendee);
- erts_smp_runqs_lock(cp_rq, s_rq);
- if (!(suspendee->runq_flags & ERTS_PROC_RUNQ_FLG_RUNNING)) {
- do_bif_suspend_process(smon, suspendee, s_rq);
- erts_smp_runqs_unlock(cp_rq, s_rq);
+ if (do_bif_suspend_process(BIF_P, smon, suspendee)) {
res = (!unless_suspending || smon->active == 1
? am_true
: am_false);
@@ -5513,8 +6053,7 @@ suspend_process_2(BIF_ALIST_2)
* This time with BIF_P->suspendee == BIF_ARG_1 (see
* above).
*/
- suspend_process(cp_rq, BIF_P);
- erts_smp_runqs_unlock(cp_rq, s_rq);
+ suspend_process(BIF_P, BIF_P);
goto yield;
}
}
@@ -5522,9 +6061,15 @@ suspend_process_2(BIF_ALIST_2)
}
#endif /* ERTS_SMP */
-
- ASSERT(suspendee->status == P_SUSPENDED || (asynchronous && smon->pending));
- ASSERT(suspendee->status == P_SUSPENDED || !smon->active);
+#ifdef DEBUG
+ {
+ erts_aint32_t state = erts_smp_atomic32_read_acqb(&suspendee->state);
+ ASSERT((state & ERTS_PSFLG_SUSPENDED)
+ || (asynchronous && smon->pending));
+ ASSERT((state & ERTS_PSFLG_SUSPENDED)
+ || !smon->active);
+ }
+#endif
erts_smp_proc_unlock(suspendee, ERTS_PROC_LOCK_STATUS);
erts_smp_proc_unlock(BIF_P, xlocks);
@@ -5619,9 +6164,8 @@ resume_process_1(BIF_ALIST_1)
if (!suspendee)
goto no_suspendee;
- ASSERT(suspendee->status == P_SUSPENDED
- || (suspendee->status == P_GARBING
- && suspendee->gcstatus == P_SUSPENDED));
+ ASSERT(ERTS_PSFLG_SUSPENDED
+ & erts_smp_atomic32_read_nob(&suspendee->state));
resume_process(suspendee);
erts_smp_proc_unlock(suspendee, ERTS_PROC_LOCK_STATUS);
@@ -5650,449 +6194,46 @@ erts_run_queues_len(Uint *qlen)
Uint len = 0;
ERTS_ATOMIC_FOREACH_RUNQ(rq,
{
- if (qlen)
- qlen[i++] = rq->procs.len;
- len += rq->procs.len;
+ Sint pqlen = 0;
+ int pix;
+ for (pix = 0; pix < ERTS_NO_PROC_PRIO_LEVELS; pix++)
+ pqlen += RUNQ_READ_LEN(&rq->procs.prio_info[pix].len);
+
+ if (pqlen < 0)
+ pqlen = 0;
+ if (qlen)
+ qlen[i++] = pqlen;
+ len += pqlen;
}
);
return len;
}
-#ifdef HARDDEBUG_RUNQS
-static void
-check_procs_runq(ErtsRunQueue *runq, Process *p_in_q, Process *p_not_in_q)
-{
- int len[ERTS_NO_PROC_PRIO_LEVELS] = {0};
- int tot_len;
- int prioq, prio;
- int found_p_in_q;
- Process *p, *prevp;
-
- found_p_in_q = 0;
- for (prioq = 0; prioq < ERTS_NO_PROC_PRIO_LEVELS - 1; prioq++) {
- prevp = NULL;
- for (p = runq->procs.prio[prioq].first; p; p = p->next) {
- ASSERT(p != p_not_in_q);
- if (p == p_in_q)
- found_p_in_q = 1;
- switch (p->prio) {
- case PRIORITY_MAX:
- case PRIORITY_HIGH:
- case PRIORITY_NORMAL:
- ASSERT(prioq == p->prio);
- break;
- case PRIORITY_LOW:
- ASSERT(prioq == PRIORITY_NORMAL);
- break;
- default:
- ASSERT(!"Bad prio on process");
- }
- len[p->prio]++;
- ASSERT(prevp == p->prev);
- if (p->prev) {
- ASSERT(p->prev->next == p);
- }
- else {
- ASSERT(runq->procs.prio[prioq].first == p);
- }
- if (p->next) {
- ASSERT(p->next->prev == p);
- }
- else {
- ASSERT(runq->procs.prio[prioq].last == p);
- }
- ASSERT(p->run_queue == runq);
- prevp = p;
- }
- }
-
- ASSERT(!p_in_q || found_p_in_q);
-
- tot_len = 0;
- for (prio = 0; prio < ERTS_NO_PROC_PRIO_LEVELS; prio++) {
- ASSERT(len[prio] == runq->procs.prio_info[prio].len);
- if (len[prio]) {
- ASSERT(runq->flags & (1 << prio));
- }
- else {
- ASSERT(!(runq->flags & (1 << prio)));
- }
- tot_len += len[prio];
- }
- ASSERT(runq->procs.len == tot_len);
-}
-# define ERTS_DBG_CHK_PROCS_RUNQ(RQ) check_procs_runq((RQ), NULL, NULL)
-# define ERTS_DBG_CHK_PROCS_RUNQ_PROC(RQ, P) check_procs_runq((RQ), (P), NULL)
-# define ERTS_DBG_CHK_PROCS_RUNQ_NOPROC(RQ, P) check_procs_runq((RQ), NULL, (P))
-#else
-# define ERTS_DBG_CHK_PROCS_RUNQ(RQ)
-# define ERTS_DBG_CHK_PROCS_RUNQ_PROC(RQ, P)
-# define ERTS_DBG_CHK_PROCS_RUNQ_NOPROC(RQ, P)
-#endif
-
-
-static ERTS_INLINE void
-enqueue_process(ErtsRunQueue *runq, Process *p)
-{
- ErtsRunPrioQueue *rpq;
- ErtsRunQueueInfo *rqi;
-
- ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq));
- ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p));
-
- ASSERT(p->bound_runq || !(runq->flags & ERTS_RUNQ_FLG_SUSPENDED));
-
- rqi = &runq->procs.prio_info[p->prio];
- rqi->len++;
- if (rqi->max_len < rqi->len)
- rqi->max_len = rqi->len;
-
- runq->procs.len++;
- runq->len++;
- if (runq->max_len < runq->len)
- runq->max_len = runq->len;
-
- runq->flags |= (1 << p->prio);
-
- rpq = (p->prio == PRIORITY_LOW
- ? &runq->procs.prio[PRIORITY_NORMAL]
- : &runq->procs.prio[p->prio]);
-
- p->next = NULL;
- p->prev = rpq->last;
- if (rpq->last)
- rpq->last->next = p;
- else
- rpq->first = p;
- rpq->last = p;
-
- switch (p->status) {
- case P_EXITING:
- break;
- case P_GARBING:
- p->gcstatus = P_RUNABLE;
- break;
- default:
- p->status = P_RUNABLE;
- break;
- }
-
-#ifdef ERTS_SMP
- p->status_flags |= ERTS_PROC_SFLG_INRUNQ;
-#endif
-
- ERTS_DBG_CHK_PROCS_RUNQ_PROC(runq, p);
-}
-
-
-static ERTS_INLINE int
-dequeue_process(ErtsRunQueue *runq, Process *p)
-{
- ErtsRunPrioQueue *rpq;
- int res = 1;
-
- ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq));
- ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p));
-
- ERTS_DBG_CHK_PROCS_RUNQ(runq);
-
- rpq = &runq->procs.prio[p->prio == PRIORITY_LOW ? PRIORITY_NORMAL : p->prio];
- if (p->prev) {
- p->prev->next = p->next;
- }
- else if (rpq->first == p) {
- rpq->first = p->next;
- }
- else {
- res = 0;
- }
- if (p->next) {
- p->next->prev = p->prev;
- }
- else if (rpq->last == p) {
- rpq->last = p->prev;
- }
- else {
- ASSERT(res == 0);
- }
-
- if (res) {
-
- if (--runq->procs.prio_info[p->prio].len == 0)
- runq->flags &= ~(1 << p->prio);
- runq->procs.len--;
- runq->len--;
-
-#ifdef ERTS_SMP
- p->status_flags &= ~ERTS_PROC_SFLG_INRUNQ;
-#endif
- }
-
- ERTS_DBG_CHK_PROCS_RUNQ_NOPROC(runq, p);
- return res;
-}
-
-/* schedule a process */
-static ERTS_INLINE ErtsRunQueue *
-internal_add_to_runq(ErtsRunQueue *runq, Process *p)
-{
- Uint32 prev_status = p->status;
- ErtsRunQueue *add_runq;
-#ifdef ERTS_SMP
-
- ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p));
- ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq));
-
- if (p->status_flags & ERTS_PROC_SFLG_INRUNQ)
- return NULL;
- else if (p->runq_flags & ERTS_PROC_RUNQ_FLG_RUNNING) {
- ASSERT(ERTS_PROC_IS_EXITING(p) || p->rcount == 0);
- ERTS_DBG_CHK_PROCS_RUNQ_NOPROC(runq, p);
- p->status_flags |= ERTS_PROC_SFLG_PENDADD2SCHEDQ;
- return NULL;
- }
- ASSERT(!p->scheduler_data);
-#endif
-
- ERTS_DBG_CHK_PROCS_RUNQ_NOPROC(runq, p);
-#ifndef ERTS_SMP
- /* Never schedule a suspended process (ok in smp case) */
- ASSERT(ERTS_PROC_IS_EXITING(p) || p->rcount == 0);
- add_runq = runq;
-#else
- ASSERT(!p->bound_runq || p->bound_runq == p->run_queue);
- if (p->bound_runq) {
- if (p->bound_runq == runq)
- add_runq = runq;
- else {
- add_runq = p->bound_runq;
- erts_smp_xrunq_lock(runq, add_runq);
- }
- }
- else {
- add_runq = erts_check_emigration_need(runq, p->prio);
- if (!add_runq)
- add_runq = runq;
- else /* Process emigrated */
- p->run_queue = add_runq;
- }
-#endif
-
- /* Enqueue the process */
- enqueue_process(add_runq, p);
-
- if ((erts_system_profile_flags.runnable_procs)
- && (prev_status == P_WAITING
- || prev_status == P_SUSPENDED)) {
- profile_runnable_proc(p, am_active);
- }
-
- if (add_runq != runq)
- erts_smp_runq_unlock(add_runq);
-
- return add_runq;
-}
-
-
-void
-erts_add_to_runq(Process *p)
-{
- ErtsRunQueue *notify_runq;
- ErtsRunQueue *runq = erts_get_runq_proc(p);
- erts_smp_runq_lock(runq);
- notify_runq = internal_add_to_runq(runq, p);
- erts_smp_runq_unlock(runq);
- smp_notify_inc_runq(notify_runq);
-
-}
-
-/* Possibly remove a scheduled process we need to suspend */
-
-static int
-remove_proc_from_runq(ErtsRunQueue *rq, Process *p, int to_inactive)
-{
- int res;
-
- ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p));
-
-#ifdef ERTS_SMP
- if (p->status_flags & ERTS_PROC_SFLG_PENDADD2SCHEDQ) {
- p->status_flags &= ~ERTS_PROC_SFLG_PENDADD2SCHEDQ;
- ASSERT(!remove_proc_from_runq(rq, p, 0));
- return 1;
- }
-#endif
-
- res = dequeue_process(rq, p);
-
- if (res && erts_system_profile_flags.runnable_procs && to_inactive)
- profile_runnable_proc(p, am_inactive);
-
-#ifdef ERTS_SMP
- ASSERT(!(p->status_flags & ERTS_PROC_SFLG_INRUNQ));
-#endif
-
- return res;
-}
-
-#ifdef ERTS_SMP
-
-ErtsMigrateResult
-erts_proc_migrate(Process *p, ErtsProcLocks *plcks,
- ErtsRunQueue *from_rq, int *from_locked,
- ErtsRunQueue *to_rq, int *to_locked)
-{
- ERTS_SMP_LC_ASSERT(*plcks == erts_proc_lc_my_proc_locks(p));
- ERTS_SMP_LC_ASSERT((ERTS_PROC_LOCK_STATUS & *plcks)
- || from_locked);
- ERTS_SMP_LC_CHK_RUNQ_LOCK(from_rq, *from_locked);
- ERTS_SMP_LC_CHK_RUNQ_LOCK(to_rq, *to_locked);
-
- /*
- * If we have the lock on the run queue to migrate to,
- * check that it isn't suspended. If it is suspended,
- * we will refuse to migrate to it anyway.
- */
- if (*to_locked && (to_rq->flags & ERTS_RUNQ_FLG_SUSPENDED))
- return ERTS_MIGRATE_FAILED_RUNQ_SUSPENDED;
-
- /* We need status lock on process and locks on both run queues */
-
- if (!(ERTS_PROC_LOCK_STATUS & *plcks)) {
- if (erts_smp_proc_trylock(p, ERTS_PROC_LOCK_STATUS) == EBUSY) {
- ErtsProcLocks lcks = *plcks;
- Eterm pid = p->id;
- Process *proc = *plcks ? p : NULL;
-
- if (*from_locked) {
- *from_locked = 0;
- erts_smp_runq_unlock(from_rq);
- }
- if (*to_locked) {
- *to_locked = 0;
- erts_smp_runq_unlock(to_rq);
- }
-
- proc = erts_pid2proc_opt(proc,
- lcks,
- pid,
- lcks|ERTS_PROC_LOCK_STATUS,
- ERTS_P2P_FLG_ALLOW_OTHER_X);
- if (!proc) {
- *plcks = 0;
- return ERTS_MIGRATE_FAILED_NOT_IN_RUNQ;
- }
- ASSERT(proc == p);
- }
- *plcks |= ERTS_PROC_LOCK_STATUS;
- }
-
- ASSERT(!p->bound_runq);
-
- ERTS_SMP_LC_CHK_RUNQ_LOCK(from_rq, *from_locked);
- ERTS_SMP_LC_CHK_RUNQ_LOCK(to_rq, *to_locked);
-
- if (p->run_queue != from_rq)
- return ERTS_MIGRATE_FAILED_RUNQ_CHANGED;
-
- if (!*from_locked || !*to_locked) {
- if (from_rq < to_rq) {
- if (!*to_locked) {
- if (!*from_locked)
- erts_smp_runq_lock(from_rq);
- erts_smp_runq_lock(to_rq);
- }
- else if (erts_smp_runq_trylock(from_rq) == EBUSY) {
- erts_smp_runq_unlock(to_rq);
- erts_smp_runq_lock(from_rq);
- erts_smp_runq_lock(to_rq);
- }
- }
- else {
- if (!*from_locked) {
- if (!*to_locked)
- erts_smp_runq_lock(to_rq);
- erts_smp_runq_lock(from_rq);
- }
- else if (erts_smp_runq_trylock(to_rq) == EBUSY) {
- erts_smp_runq_unlock(from_rq);
- erts_smp_runq_lock(to_rq);
- erts_smp_runq_lock(from_rq);
- }
- }
- *to_locked = *from_locked = 1;
- }
-
- ERTS_SMP_LC_CHK_RUNQ_LOCK(from_rq, *from_locked);
- ERTS_SMP_LC_CHK_RUNQ_LOCK(to_rq, *to_locked);
-
- /* Ok we now got all locks we need; do it... */
-
- /* Refuse to migrate to a suspended run queue */
- if (to_rq->flags & ERTS_RUNQ_FLG_SUSPENDED)
- return ERTS_MIGRATE_FAILED_RUNQ_SUSPENDED;
-
- if ((p->runq_flags & ERTS_PROC_RUNQ_FLG_RUNNING)
- || !(p->status_flags & ERTS_PROC_SFLG_INRUNQ))
- return ERTS_MIGRATE_FAILED_NOT_IN_RUNQ;
-
- dequeue_process(from_rq, p);
- p->run_queue = to_rq;
- enqueue_process(to_rq, p);
-
- return ERTS_MIGRATE_SUCCESS;
-}
-#endif /* ERTS_SMP */
-
Eterm
erts_process_status(Process *c_p, ErtsProcLocks c_p_locks,
Process *rp, Eterm rpid)
{
Eterm res = am_undefined;
- Process *p;
-
- if (rp) {
- ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS
- & erts_proc_lc_my_proc_locks(rp));
- p = rp;
- }
- else {
- p = erts_pid2proc_opt(c_p, c_p_locks,
- rpid, ERTS_PROC_LOCK_STATUS,
- ERTS_P2P_FLG_ALLOW_OTHER_X);
- }
+ Process *p = rp ? rp : erts_proc_lookup_raw(rpid);
if (p) {
- switch (p->status) {
- case P_RUNABLE:
- res = am_runnable;
- break;
- case P_WAITING:
- res = am_waiting;
- break;
- case P_RUNNING:
- res = am_running;
- break;
- case P_EXITING:
+ erts_aint32_t state = erts_smp_atomic32_read_acqb(&p->state);
+ if (state & ERTS_PSFLG_FREE)
+ res = am_free;
+ else if (state & ERTS_PSFLG_EXITING)
res = am_exiting;
- break;
- case P_GARBING:
+ else if (state & ERTS_PSFLG_GC)
res = am_garbage_collecting;
- break;
- case P_SUSPENDED:
+ else if (state & ERTS_PSFLG_SUSPENDED)
res = am_suspended;
- break;
- case P_FREE: /* We cannot look up a process in P_FREE... */
- default: /* Not a valid status... */
- erl_exit(1, "Bad status (%b32u) found for process %T\n",
- p->status, p->id);
- break;
- }
-
-#ifdef ERTS_SMP
- if (!rp && (p != c_p || !(ERTS_PROC_LOCK_STATUS & c_p_locks)))
- erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+ else if (state & ERTS_PSFLG_RUNNING)
+ res = am_running;
+ else if (state & ERTS_PSFLG_ACTIVE)
+ res = am_runnable;
+ else
+ res = am_waiting;
}
+#ifdef ERTS_SMP
else {
int i;
ErtsSchedulerData *esdp;
@@ -6107,42 +6248,40 @@ erts_process_status(Process *c_p, ErtsProcLocks c_p_locks,
}
erts_smp_runq_unlock(esdp->run_queue);
}
-
-#endif
-
}
-
+#endif
return res;
}
/*
-** Suspend a process
+** Suspend a currently executing process
** If we are to suspend on a port the busy_port is the thing
** otherwise busy_port is NIL
*/
void
-erts_suspend(Process* process, ErtsProcLocks process_locks, Port *busy_port)
+erts_suspend(Process* c_p, ErtsProcLocks c_p_locks, Port *busy_port)
{
- ErtsRunQueue *rq;
-
- ERTS_SMP_LC_ASSERT(process_locks == erts_proc_lc_my_proc_locks(process));
- if (!(process_locks & ERTS_PROC_LOCK_STATUS))
- erts_smp_proc_lock(process, ERTS_PROC_LOCK_STATUS);
-
- rq = erts_get_runq_proc(process);
-
- erts_smp_runq_lock(rq);
+#ifdef DEBUG
+ int res;
+#endif
+ ASSERT(c_p == erts_get_current_process());
+ ERTS_SMP_LC_ASSERT(c_p_locks == erts_proc_lc_my_proc_locks(c_p));
+ if (!(c_p_locks & ERTS_PROC_LOCK_STATUS))
+ erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_STATUS);
- suspend_process(rq, process);
+#ifdef DEBUG
+ res =
+#endif
+ suspend_process(c_p, c_p);
- erts_smp_runq_unlock(rq);
+ ASSERT(res);
if (busy_port)
- erts_wake_process_later(busy_port, process);
+ erts_wake_process_later(busy_port, c_p);
- if (!(process_locks & ERTS_PROC_LOCK_STATUS))
- erts_smp_proc_unlock(process, ERTS_PROC_LOCK_STATUS);
+ if (!(c_p_locks & ERTS_PROC_LOCK_STATUS))
+ erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS);
}
@@ -6183,57 +6322,54 @@ erts_resume_processes(ErtsProcList *plp)
Eterm
erts_get_process_priority(Process *p)
{
- ErtsRunQueue *rq;
- Eterm value;
- ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p));
- rq = erts_get_runq_proc(p);
- erts_smp_runq_lock(rq);
- switch(p->prio) {
- case PRIORITY_MAX: value = am_max; break;
- case PRIORITY_HIGH: value = am_high; break;
- case PRIORITY_NORMAL: value = am_normal; break;
- case PRIORITY_LOW: value = am_low; break;
- default: ASSERT(0); value = am_undefined; break;
+ erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state);
+ switch (state & ERTS_PSFLG_PRIO_MASK) {
+ case PRIORITY_MAX: return am_max;
+ case PRIORITY_HIGH: return am_high;
+ case PRIORITY_NORMAL: return am_normal;
+ case PRIORITY_LOW: return am_low;
+ default: ASSERT(0); return am_undefined;
}
- erts_smp_runq_unlock(rq);
- return value;
}
Eterm
-erts_set_process_priority(Process *p, Eterm new_value)
+erts_set_process_priority(Process *p, Eterm value)
{
- ErtsRunQueue *rq;
- Eterm old_value;
- ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p));
- rq = erts_get_runq_proc(p);
-#ifdef ERTS_SMP
- ASSERT(!(p->status_flags & ERTS_PROC_SFLG_INRUNQ));
-#endif
- erts_smp_runq_lock(rq);
- switch(p->prio) {
- case PRIORITY_MAX: old_value = am_max; break;
- case PRIORITY_HIGH: old_value = am_high; break;
- case PRIORITY_NORMAL: old_value = am_normal; break;
- case PRIORITY_LOW: old_value = am_low; break;
- default: ASSERT(0); old_value = am_undefined; break;
- }
- switch (new_value) {
- case am_max: p->prio = PRIORITY_MAX; break;
- case am_high: p->prio = PRIORITY_HIGH; break;
- case am_normal: p->prio = PRIORITY_NORMAL; break;
- case am_low: p->prio = PRIORITY_LOW; break;
- default: old_value = THE_NON_VALUE; break;
+ erts_aint32_t a, oprio, nprio;
+
+ switch (value) {
+ case am_max: nprio = (erts_aint32_t) PRIORITY_MAX; break;
+ case am_high: nprio = (erts_aint32_t) PRIORITY_HIGH; break;
+ case am_normal: nprio = (erts_aint32_t) PRIORITY_NORMAL; break;
+ case am_low: nprio = (erts_aint32_t) PRIORITY_LOW; break;
+ default: return THE_NON_VALUE; break;
}
- erts_smp_runq_unlock(rq);
- return old_value;
-}
-/* note that P_RUNNING is only set so that we don't try to remove
-** running processes from the schedule queue if they exit - a running
-** process not being in the schedule queue!!
-** Schedule for up to INPUT_REDUCTIONS context switches,
-** return 1 if more to do.
-*/
+ a = erts_smp_atomic32_read_nob(&p->state);
+ if (nprio == (a & ERTS_PSFLG_PRIO_MASK))
+ oprio = nprio;
+ else {
+ erts_aint32_t e, n;
+ do {
+ oprio = a & ERTS_PSFLG_PRIO_MASK;
+ n = e = a;
+
+ ASSERT(!(a & ERTS_PSFLG_IN_RUNQ));
+
+ n &= ~ERTS_PSFLG_PRIO_MASK;
+ n |= nprio;
+ a = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e);
+ } while (a != e);
+ }
+
+ switch (oprio) {
+ case PRIORITY_MAX: return am_max;
+ case PRIORITY_HIGH: return am_high;
+ case PRIORITY_NORMAL: return am_normal;
+ case PRIORITY_LOW: return am_low;
+ default: ASSERT(0); return am_undefined;
+ }
+}
/*
* schedule() is called from BEAM (process_main()) or HiPE
@@ -6256,7 +6392,6 @@ erts_set_process_priority(Process *p, Eterm new_value)
Process *schedule(Process *p, int calls)
{
ErtsRunQueue *rq;
- ErtsRunPrioQueue *rpq;
erts_aint_t dt;
ErtsSchedulerData *esdp;
int context_reds;
@@ -6264,6 +6399,8 @@ Process *schedule(Process *p, int calls)
int input_reductions;
int actual_reds;
int reds;
+ Uint32 flags;
+ erts_aint32_t state = 0; /* Supress warning... */
#ifdef USE_VM_PROBES
if (p != NULL && DTRACE_ENABLED(process_unscheduled)) {
@@ -6319,92 +6456,60 @@ Process *schedule(Process *p, int calls)
erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
- if ((erts_system_profile_flags.runnable_procs)
- && (p->status == P_WAITING)) {
- profile_runnable_proc(p, am_inactive);
- }
+ state = erts_smp_atomic32_read_acqb(&p->state);
if (IS_TRACED(p)) {
- if (IS_TRACED_FL(p, F_TRACE_CALLS) && p->status != P_FREE) {
+ if (IS_TRACED_FL(p, F_TRACE_CALLS) && !(state & ERTS_PSFLG_FREE)) {
erts_schedule_time_break(p, ERTS_BP_CALL_TIME_SCHEDULE_OUT);
}
- switch (p->status) {
- case P_EXITING:
- if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED_EXIT))
- trace_sched(p, am_out_exiting);
- break;
- case P_FREE:
+ if (state & (ERTS_PSFLG_FREE|ERTS_PSFLG_EXITING)) {
if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED_EXIT))
- trace_sched(p, am_out_exited);
- break;
- default:
+ trace_sched(p, ((state & ERTS_PSFLG_FREE)
+ ? am_out_exited
+ : am_out_exiting));
+ }
+ else {
if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED))
trace_sched(p, am_out);
else if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED_PROCS))
trace_virtual_sched(p, am_out);
- break;
}
}
#ifdef ERTS_SMP
- if (ERTS_PROC_PENDING_EXIT(p)) {
- erts_handle_pending_exit(p,
- ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS);
- p->status_flags |= ERTS_PROC_SFLG_PENDADD2SCHEDQ;
- }
-
- if (p->pending_suspenders) {
- handle_pending_suspend(p,
- ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS);
- ASSERT(!(p->status_flags & ERTS_PROC_SFLG_PENDADD2SCHEDQ)
- || p->rcount == 0);
- }
+ if (state & ERTS_PSFLG_PENDING_EXIT)
+ erts_handle_pending_exit(p, (ERTS_PROC_LOCK_MAIN
+ | ERTS_PROC_LOCK_STATUS));
+ if (p->pending_suspenders)
+ handle_pending_suspend(p, (ERTS_PROC_LOCK_MAIN
+ | ERTS_PROC_LOCK_STATUS));
#endif
- erts_smp_runq_lock(rq);
- ERTS_PROC_REDUCTIONS_EXECUTED(rq, p->prio, reds, actual_reds);
+ schedule_out_process(rq, state, p); /* Returns with rq locked! */
+
+ ERTS_PROC_REDUCTIONS_EXECUTED(rq,
+ (int) (state & ERTS_PSFLG_PRIO_MASK),
+ reds,
+ actual_reds);
esdp->current_process = NULL;
#ifdef ERTS_SMP
p->scheduler_data = NULL;
- p->runq_flags &= ~ERTS_PROC_RUNQ_FLG_RUNNING;
- p->status_flags &= ~ERTS_PROC_SFLG_RUNNING;
-
- if (p->status_flags & ERTS_PROC_SFLG_PENDADD2SCHEDQ) {
- ErtsRunQueue *notify_runq;
- p->status_flags &= ~ERTS_PROC_SFLG_PENDADD2SCHEDQ;
- notify_runq = internal_add_to_runq(rq, p);
- if (notify_runq != rq)
- smp_notify_inc_runq(notify_runq);
- }
#endif
- if (p->status == P_FREE) {
+ if (state & ERTS_PSFLG_FREE) {
#ifdef ERTS_SMP
ASSERT(esdp->free_process == p);
esdp->free_process = NULL;
- erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS);
- erts_smp_proc_dec_refc(p);
#else
erts_free_proc(p);
#endif
- } else {
- erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS);
}
-#ifdef ERTS_SMP
- {
- ErtsProcList *pnd_xtrs = rq->procs.pending_exiters;
- rq->procs.pending_exiters = NULL;
+ erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS);
- if (pnd_xtrs) {
- erts_smp_runq_unlock(rq);
- handle_pending_exiters(pnd_xtrs);
- erts_smp_runq_lock(rq);
- }
-
- }
+#ifdef ERTS_SMP
ASSERT(!esdp->free_process);
#endif
ASSERT(!esdp->current_process);
@@ -6424,8 +6529,23 @@ Process *schedule(Process *p, int calls)
ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking());
check_activities_to_run: {
+#ifdef ERTS_SMP
+ ErtsMigrationPaths *mps;
+ ErtsMigrationPath *mp;
#ifdef ERTS_SMP
+ {
+ ErtsProcList *pnd_xtrs = rq->procs.pending_exiters;
+ rq->procs.pending_exiters = NULL;
+
+ if (pnd_xtrs) {
+ erts_smp_runq_unlock(rq);
+ handle_pending_exiters(pnd_xtrs);
+ erts_smp_runq_lock(rq);
+ }
+
+ }
+#endif
if (rq->check_balance_reds <= 0)
check_balance(rq);
@@ -6433,20 +6553,28 @@ Process *schedule(Process *p, int calls)
ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking());
ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
- if (rq->flags & ERTS_RUNQ_FLGS_IMMIGRATE_QMASK)
- immigrate(rq);
+ mps = erts_get_migration_paths_managed();
+ mp = &mps->mpath[rq->ix];
- continue_check_activities_to_run:
+ if (mp->flags & ERTS_RUNQ_FLGS_IMMIGRATE_QMASK)
+ immigrate(rq, mp);
- if (rq->flags & (ERTS_RUNQ_FLG_CHK_CPU_BIND
- | ERTS_RUNQ_FLG_SUSPENDED)) {
- if (rq->flags & ERTS_RUNQ_FLG_SUSPENDED) {
- ASSERT(erts_smp_atomic32_read_nob(&esdp->ssi->flags)
- & ERTS_SSI_FLG_SUSPENDED);
+ continue_check_activities_to_run:
+ flags = ERTS_RUNQ_FLGS_GET_NOB(rq);
+ continue_check_activities_to_run_known_flags:
+
+
+ if (flags & (ERTS_RUNQ_FLG_CHK_CPU_BIND|ERTS_RUNQ_FLG_SUSPENDED)) {
+
+ if (flags & ERTS_RUNQ_FLG_SUSPENDED) {
suspend_scheduler(esdp);
+ flags = ERTS_RUNQ_FLGS_GET_NOB(rq);
}
- if (rq->flags & ERTS_RUNQ_FLG_CHK_CPU_BIND)
+ if (flags & ERTS_RUNQ_FLG_CHK_CPU_BIND) {
+ flags = ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_CHK_CPU_BIND);
+ flags &= ~ ERTS_RUNQ_FLG_CHK_CPU_BIND;
erts_sched_check_cpu_bind(esdp);
+ }
}
{
@@ -6475,14 +6603,12 @@ Process *schedule(Process *p, int calls)
}
#endif /* ERTS_SMP */
- ASSERT(rq->len == rq->procs.len + rq->ports.info.len);
-
- if ((rq->len == 0 && !rq->misc.start)
- || (rq->halt_in_progress
- && rq->ports.info.len == 0 && !rq->misc.start)) {
+ flags = ERTS_RUNQ_FLGS_GET_NOB(rq);
+ if ((!(flags & ERTS_RUNQ_FLGS_QMASK) && !rq->misc.start)
+ || (rq->halt_in_progress && ERTS_EMPTY_RUNQ_PORTS(rq))) {
+ /* Prepare for scheduler wait */
#ifdef ERTS_SMP
-
ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
rq->wakeup_other = 0;
@@ -6490,21 +6616,27 @@ Process *schedule(Process *p, int calls)
empty_runq(rq);
- if (rq->flags & ERTS_RUNQ_FLG_SUSPENDED) {
- ASSERT(erts_smp_atomic32_read_nob(&esdp->ssi->flags)
- & ERTS_SSI_FLG_SUSPENDED);
+ flags = ERTS_RUNQ_FLGS_GET_NOB(rq);
+ if (flags & ERTS_RUNQ_FLG_SUSPENDED) {
non_empty_runq(rq);
- goto continue_check_activities_to_run;
+ goto continue_check_activities_to_run_known_flags;
}
- else if (!(rq->flags & ERTS_RUNQ_FLG_INACTIVE)) {
+ else if (!(flags & ERTS_RUNQ_FLG_INACTIVE)) {
+ if (try_steal_task(rq)) {
+ non_empty_runq(rq);
+ goto continue_check_activities_to_run;
+ }
+
+ ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_PROTECTED);
+
/*
* Check for ERTS_RUNQ_FLG_SUSPENDED has to be done
* after trying to steal a task.
*/
- if (try_steal_task(rq)
- || (rq->flags & ERTS_RUNQ_FLG_SUSPENDED)) {
+ flags = ERTS_RUNQ_FLGS_GET_NOB(rq);
+ if (flags & ERTS_RUNQ_FLG_SUSPENDED) {
non_empty_runq(rq);
- goto continue_check_activities_to_run;
+ goto continue_check_activities_to_run_known_flags;
}
}
@@ -6535,6 +6667,7 @@ Process *schedule(Process *p, int calls)
erl_sys_schedule(1);
dt = erts_do_time_read_and_reset();
if (dt) erts_bump_timer(dt);
+
#ifdef ERTS_SMP
erts_smp_runq_lock(rq);
clear_sys_scheduling();
@@ -6551,14 +6684,17 @@ Process *schedule(Process *p, int calls)
{
int wo_reds = rq->wakeup_other_reds;
if (wo_reds) {
- if (rq->len < 2) {
+ erts_aint32_t len = rq->len;
+ if (len < 2) {
rq->wakeup_other -= ERTS_WAKEUP_OTHER_DEC*wo_reds;
if (rq->wakeup_other < 0)
rq->wakeup_other = 0;
}
else if (rq->wakeup_other < wakeup_other_limit)
- rq->wakeup_other += rq->len*wo_reds + ERTS_WAKEUP_OTHER_FIXED_INC;
+ rq->wakeup_other += len*wo_reds + ERTS_WAKEUP_OTHER_FIXED_INC;
else {
+ if (flags & ERTS_RUNQ_FLG_PROTECTED)
+ ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_PROTECTED);
if (erts_smp_atomic32_read_acqb(&no_empty_run_queues) != 0) {
wake_scheduler_on_empty_runq(rq);
rq->wakeup_other = 0;
@@ -6574,7 +6710,7 @@ Process *schedule(Process *p, int calls)
* Find a new port to run.
*/
- if (rq->ports.info.len) {
+ if (RUNQ_READ_LEN(&rq->ports.info.len)) {
int have_outstanding_io;
have_outstanding_io = erts_port_task_execute(rq, &esdp->current_port);
if ((have_outstanding_io && fcalls > 2*input_reductions)
@@ -6601,160 +6737,123 @@ Process *schedule(Process *p, int calls)
/*
* Find a new process to run.
*/
- pick_next_process:
-
- ERTS_DBG_CHK_PROCS_RUNQ(rq);
-
- switch (rq->flags & ERTS_RUNQ_FLGS_PROCS_QMASK) {
- case MAX_BIT:
- case MAX_BIT|HIGH_BIT:
- case MAX_BIT|NORMAL_BIT:
- case MAX_BIT|LOW_BIT:
- case MAX_BIT|HIGH_BIT|NORMAL_BIT:
- case MAX_BIT|HIGH_BIT|LOW_BIT:
- case MAX_BIT|NORMAL_BIT|LOW_BIT:
- case MAX_BIT|HIGH_BIT|NORMAL_BIT|LOW_BIT:
- rpq = &rq->procs.prio[PRIORITY_MAX];
- break;
- case HIGH_BIT:
- case HIGH_BIT|NORMAL_BIT:
- case HIGH_BIT|LOW_BIT:
- case HIGH_BIT|NORMAL_BIT|LOW_BIT:
- rpq = &rq->procs.prio[PRIORITY_HIGH];
- break;
- case NORMAL_BIT:
- rpq = &rq->procs.prio[PRIORITY_NORMAL];
- break;
- case LOW_BIT:
- rpq = &rq->procs.prio[PRIORITY_NORMAL];
- break;
- case NORMAL_BIT|LOW_BIT:
- rpq = &rq->procs.prio[PRIORITY_NORMAL];
- ASSERT(rpq->first != NULL);
- p = rpq->first;
- if (p->prio == PRIORITY_LOW) {
- if (p == rpq->last || p->skipped >= RESCHEDULE_LOW-1)
- p->skipped = 0;
- else {
- /* skip it */
- p->skipped++;
- rpq->first = p->next;
- rpq->first->prev = NULL;
- rpq->last->next = p;
- p->prev = rpq->last;
- p->next = NULL;
- rpq->last = p;
+ pick_next_process: {
+ int prio_q;
+ int qmask;
+
+ flags = ERTS_RUNQ_FLGS_GET_NOB(rq);
+ qmask = (int) (flags & ERTS_RUNQ_FLGS_PROCS_QMASK);
+ switch (qmask & -qmask) {
+ case MAX_BIT:
+ prio_q = PRIORITY_MAX;
+ break;
+ case HIGH_BIT:
+ prio_q = PRIORITY_HIGH;
+ break;
+ case NORMAL_BIT:
+ case LOW_BIT:
+ prio_q = PRIORITY_NORMAL;
+ if (check_requeue_process(rq, PRIORITY_NORMAL))
goto pick_next_process;
- }
+ break;
+ case 0: /* No process at all */
+ default:
+ ASSERT(qmask == 0);
+ goto check_activities_to_run;
}
- break;
- case 0: /* No process at all */
- default:
- ASSERT((rq->flags & ERTS_RUNQ_FLGS_PROCS_QMASK) == 0);
- ASSERT(rq->procs.len == 0);
- goto check_activities_to_run;
- }
-
- BM_START_TIMER(system);
-
- /*
- * Take the chosen process out of the queue.
- */
- ASSERT(rpq->first); /* Wrong qmask in rq->flags? */
- p = rpq->first;
-#ifdef ERTS_SMP
- ERTS_SMP_LC_ASSERT(rq == p->run_queue);
-#endif
- rpq->first = p->next;
- if (!rpq->first)
- rpq->last = NULL;
- else
- rpq->first->prev = NULL;
- p->next = p->prev = NULL;
+ BM_START_TIMER(system);
- if (--rq->procs.prio_info[p->prio].len == 0)
- rq->flags &= ~(1 << p->prio);
- ASSERT(rq->procs.len > 0);
- rq->procs.len--;
- ASSERT(rq->len > 0);
- rq->len--;
+ /*
+ * Take the chosen process out of the queue.
+ */
+ p = dequeue_process(rq, prio_q, &state);
- {
- Uint32 ee_flgs = (ERTS_RUNQ_FLG_EVACUATE(p->prio)
- | ERTS_RUNQ_FLG_EMIGRATE(p->prio));
+ ASSERT(p); /* Wrong qmask in rq->flags? */
- if ((rq->flags & (ERTS_RUNQ_FLG_SUSPENDED|ee_flgs)) == ee_flgs)
- ERTS_UNSET_RUNQ_FLG_EVACUATE(rq->flags, p->prio);
- }
+ while (1) {
+ erts_aint32_t exp, new, tmp;
+ tmp = new = exp = state;
+ new &= ~ERTS_PSFLG_IN_RUNQ;
+ tmp = state & (ERTS_PSFLG_SUSPENDED|ERTS_PSFLG_PENDING_EXIT);
+ if (tmp != ERTS_PSFLG_SUSPENDED)
+ new |= ERTS_PSFLG_RUNNING;
+ state = erts_smp_atomic32_cmpxchg_relb(&p->state, new, exp);
+ if (state == exp) {
+ tmp = state & (ERTS_PSFLG_SUSPENDED|ERTS_PSFLG_PENDING_EXIT);
+ if (tmp == ERTS_PSFLG_SUSPENDED)
+ goto pick_next_process;
+ state = new;
+ break;
+ }
+ }
- ERTS_DBG_CHK_PROCS_RUNQ_NOPROC(rq, p);
+ rq->procs.context_switches++;
- rq->procs.context_switches++;
+ esdp->current_process = p;
- esdp->current_process = p;
+ }
#ifdef ERTS_SMP
- p->runq_flags |= ERTS_PROC_RUNQ_FLG_RUNNING;
erts_smp_runq_unlock(rq);
+ if (flags & ERTS_RUNQ_FLG_PROTECTED)
+ ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_PROTECTED);
+
ERTS_SMP_CHK_NO_PROC_LOCKS;
erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS);
if (erts_sched_stat.enabled) {
+ int prio;
UWord old = ERTS_PROC_SCHED_ID(p,
(ERTS_PROC_LOCK_MAIN
| ERTS_PROC_LOCK_STATUS),
(UWord) esdp->no);
int migrated = old && old != esdp->no;
+ prio = (int) (state & ERTS_PSFLG_PRIO_MASK);
+
erts_smp_spin_lock(&erts_sched_stat.lock);
- erts_sched_stat.prio[p->prio].total_executed++;
- erts_sched_stat.prio[p->prio].executed++;
+ erts_sched_stat.prio[prio].total_executed++;
+ erts_sched_stat.prio[prio].executed++;
if (migrated) {
- erts_sched_stat.prio[p->prio].total_migrated++;
- erts_sched_stat.prio[p->prio].migrated++;
+ erts_sched_stat.prio[prio].total_migrated++;
+ erts_sched_stat.prio[prio].migrated++;
}
erts_smp_spin_unlock(&erts_sched_stat.lock);
}
- p->status_flags |= ERTS_PROC_SFLG_RUNNING;
- p->status_flags &= ~ERTS_PROC_SFLG_INRUNQ;
if (ERTS_PROC_PENDING_EXIT(p)) {
erts_handle_pending_exit(p,
ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS);
+ state = erts_smp_atomic32_read_nob(&p->state);
}
ASSERT(!p->scheduler_data);
p->scheduler_data = esdp;
-
#endif
- ASSERT(p->status != P_SUSPENDED); /* Never run a suspended process */
+ /* Never run a suspended process */
+ ASSERT(!(ERTS_PSFLG_SUSPENDED & erts_smp_atomic32_read_nob(&p->state)));
ACTIVATE(p);
reds = context_reds;
if (IS_TRACED(p)) {
- switch (p->status) {
- case P_EXITING:
+ if (state & ERTS_PSFLG_EXITING) {
if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED_EXIT))
trace_sched(p, am_in_exiting);
- break;
- default:
+ }
+ else {
if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED))
trace_sched(p, am_in);
else if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED_PROCS))
trace_virtual_sched(p, am_in);
- break;
}
if (IS_TRACED_FL(p, F_TRACE_CALLS)) {
erts_schedule_time_break(p, ERTS_BP_CALL_TIME_SCHEDULE_IN);
}
}
- if (p->status != P_EXITING)
- p->status = P_RUNNING;
-
erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
#ifdef ERTS_SMP
@@ -6762,7 +6861,7 @@ Process *schedule(Process *p, int calls)
erts_check_my_tracer_proc(p);
#endif
- if (!ERTS_PROC_IS_EXITING(p)
+ if (!(state & ERTS_PSFLG_EXITING)
&& ((FLAGS(p) & F_FORCE_GC)
|| (MSO(p).overhead > BIN_VHEAP_SZ(p)))) {
reds -= erts_garbage_collect(p, 0, p->arg_reg, p->arity);
@@ -6854,17 +6953,19 @@ erts_schedule_misc_op(void (*func)(void *), void *arg)
ErtsSchedulerData *esdp = erts_get_scheduler_data();
ErtsRunQueue *rq = esdp ? esdp->run_queue : ERTS_RUNQ_IX(0);
ErtsMiscOpList *molp = misc_op_list_alloc();
+#ifdef ERTS_SMP
+ ErtsMigrationPaths *mpaths = erts_get_migration_paths();
- erts_smp_runq_lock(rq);
-
- while (rq->misc.evac_runq) {
- ErtsRunQueue *tmp_rq = rq->misc.evac_runq;
- erts_smp_runq_unlock(rq);
- rq = tmp_rq;
- erts_smp_runq_lock(rq);
+ if (!mpaths)
+ rq = ERTS_RUNQ_IX(0);
+ else {
+ ErtsRunQueue *erq = mpaths->mpath[rq->ix].misc_evac_runq;
+ if (erq)
+ rq = erq;
}
+#endif
- ASSERT(!(rq->flags & ERTS_RUNQ_FLG_SUSPENDED));
+ erts_smp_runq_lock(rq);
molp->next = NULL;
molp->func = func;
@@ -6874,7 +6975,9 @@ erts_schedule_misc_op(void (*func)(void *), void *arg)
else
rq->misc.start = molp;
rq->misc.end = molp;
+
erts_smp_runq_unlock(rq);
+
smp_notify_inc_runq(rq);
}
@@ -6964,44 +7067,50 @@ erts_get_exact_total_reductions(Process *c_p, Uint *redsp, Uint *diffp)
Sint
erts_test_next_pid(int set, Uint next)
{
+ Uint64 lpd;
Sint res;
- Sint p_prev;
+ Eterm pid_data;
+ int first_pix = -1;
+ erts_smp_rwmtx_rwlock(&erts_proc_tab_rwmtx);
- erts_smp_mtx_lock(&proc_tab_mtx);
-
- if (!set) {
- res = p_next < 0 ? -1 : (p_serial << p_serial_shift | p_next);
- }
+ if (!set)
+ lpd = last_pid_data_read_nob();
else {
- p_serial = (Sint) ((next >> p_serial_shift) & p_serial_mask);
- p_next = (Sint) (erts_process_tab_index_mask & next);
-
- if (p_next >= erts_max_processes) {
- p_next = 0;
- p_serial++;
- p_serial &= p_serial_mask;
+ lpd = (Uint64) next;
+ pid_data = (Eterm) (lpd & ERTS_PID_DATA_MASK__);
+ if (ERTS_INVALID_PID == make_internal_pid(pid_data)) {
+ lpd += erts_proc.max;
+ ASSERT(erts_pid_data2ix(pid_data)
+ == erts_pid_data2ix(lpd & ERTS_PID_DATA_MASK__));
}
+ last_pid_data_set_relb(lpd);
+ }
- p_prev = p_next;
-
- do {
- if (!process_tab[p_next])
- break;
- p_next++;
- if(p_next >= erts_max_processes) {
- p_next = 0;
- p_serial++;
- p_serial &= p_serial_mask;
+ while (1) {
+ int pix;
+ lpd++;
+ pix = (int) (lpd % erts_proc.max);
+ if (first_pix < 0)
+ first_pix = pix;
+ else if (pix == first_pix) {
+ res = -1;
+ break;
+ }
+ if (ERTS_AINT_NULL == erts_smp_atomic_read_nob(&erts_proc.tab[pix])) {
+ pid_data = (Eterm) (lpd & ERTS_PID_DATA_MASK__);
+ if (ERTS_INVALID_PID == make_internal_pid(pid_data)) {
+ lpd += erts_proc.max;
+ ASSERT(erts_pid_data2ix(pid_data)
+ == erts_pid_data2ix(lpd & ERTS_PID_DATA_MASK__));
}
- } while (p_prev != p_next);
-
- res = process_tab[p_next] ? -1 : (p_serial << p_serial_shift | p_next);
-
+ res = lpd & ERTS_PID_DATA_MASK__;
+ break;
+ }
}
- erts_smp_mtx_unlock(&proc_tab_mtx);
+ erts_smp_rwmtx_rwunlock(&erts_proc_tab_rwmtx);
return res;
@@ -7010,6 +7119,8 @@ erts_test_next_pid(int set, Uint next)
Uint erts_process_count(void)
{
erts_aint32_t res = erts_smp_atomic32_read_nob(&process_count);
+ if (res > erts_proc.max)
+ return erts_proc.max;
ASSERT(res >= 0);
return (Uint) res;
}
@@ -7017,97 +7128,138 @@ Uint erts_process_count(void)
void
erts_free_proc(Process *p)
{
-#if defined(ERTS_ENABLE_LOCK_COUNT) && defined(ERTS_SMP)
- erts_lcnt_proc_lock_destroy(p);
+#ifdef ERTS_SMP
+ erts_proc_lock_fin(p);
#endif
erts_free(ERTS_ALC_T_PROC, (void *) p);
}
-
/*
** Allocate process and find out where to place next process.
*/
static Process*
-alloc_process(void)
+alloc_process(ErtsRunQueue *rq, erts_aint32_t state)
{
-#ifdef ERTS_SMP
- erts_pix_lock_t *pix_lock;
-#endif
+ int pix;
Process* p;
- int p_prev;
+ Uint64 lpd, exp_lpd;
+ Eterm pid_data;
+ erts_aint32_t proc_count;
+#ifdef DEBUG
+ Eterm pid;
+#endif
+
+ erts_smp_rwmtx_rlock(&erts_proc_tab_rwmtx);
- erts_smp_mtx_lock(&proc_tab_mtx);
+ proc_count = erts_smp_atomic32_inc_read_acqb(&process_count);
+ if (proc_count > erts_proc.max) {
+ while (1) {
+ erts_aint32_t act_proc_count;
- if (p_next == -1) {
- p = NULL;
- goto error; /* Process table full! */
+ act_proc_count = erts_smp_atomic32_cmpxchg_relb(&process_count,
+ proc_count-1,
+ proc_count);
+ if (act_proc_count == proc_count)
+ goto system_limit;
+ proc_count = act_proc_count;
+ if (proc_count <= erts_proc.max)
+ break;
+ }
}
p = (Process*) erts_alloc_fnf(ERTS_ALC_T_PROC, sizeof(Process));
if (!p)
- goto error; /* ENOMEM */
+ goto enomem;
- p_last = p_next;
+ p->approx_started = erts_get_approx_time();
+ p->started_interval = get_proc_interval();
- erts_get_emu_time(&p->started);
+ lpd = last_pid_data_read_acqb();
-#ifdef ERTS_SMP
- pix_lock = ERTS_PIX2PIXLOCK(p_next);
- erts_pix_lock(pix_lock);
-#endif
- ASSERT(!process_tab[p_next]);
+ /* Reserve slot */
+ while (1) {
+ lpd++;
+ pix = erts_pid_data2ix((Eterm) (lpd & ERTS_PID_DATA_MASK__));
+ if (erts_smp_atomic_read_nob(&erts_proc.tab[pix]) == ERTS_AINT_NULL) {
+ erts_aint_t val;
+ val = erts_smp_atomic_cmpxchg_relb(&erts_proc.tab[pix],
+ ((erts_aint_t)
+ ERTS_PROC_LOCK_BUSY),
+ ERTS_AINT_NULL);
+
+ if (ERTS_AINT_NULL == val)
+ break;
+ }
+ }
- process_tab[p_next] = p;
- erts_smp_atomic32_inc_nob(&process_count);
- p->id = make_internal_pid(p_serial << p_serial_shift | p_next);
+ pid_data = (Eterm) lpd & ERTS_PID_DATA_MASK__;
+
+ p->id = make_internal_pid(pid_data);
if (p->id == ERTS_INVALID_PID) {
/* Do not use the invalid pid; change serial */
- p_serial++;
- p_serial &= p_serial_mask;
- p->id = make_internal_pid(p_serial << p_serial_shift | p_next);
+ lpd += erts_proc.max;
+ ASSERT(pix == erts_pid_data2ix((Eterm) (lpd & ERTS_PID_DATA_MASK__)));
+ pid_data = (Eterm) lpd & ERTS_PID_DATA_MASK__;
+ p->id = make_internal_pid(pid_data);
ASSERT(p->id != ERTS_INVALID_PID);
}
- ASSERT(internal_pid_serial(p->id) <= (erts_use_r9_pids_ports
- ? ERTS_MAX_PID_R9_SERIAL
- : ERTS_MAX_PID_SERIAL));
+
+ exp_lpd = last_pid_data_read_nob();
+
+ /* Move last pid data forward */
+ while (1) {
+ Uint64 act_lpd;
+ if (last_pid_data_cmp(lpd, exp_lpd) < 0)
+ break;
+ act_lpd = last_pid_data_cmpxchg_relb(lpd, exp_lpd);
+ if (act_lpd == exp_lpd)
+ break;
+ exp_lpd = act_lpd;
+ }
#ifdef ERTS_SMP
- erts_proc_lock_init(p); /* All locks locked */
- erts_pix_unlock(pix_lock);
+ RUNQ_SET_RQ(&p->run_queue, rq);
#endif
- p->rstatus = P_FREE;
- p->rcount = 0;
+ erts_smp_atomic32_init_relb(&p->state, state);
- /*
- * set p_next to the next available slot
- */
+#ifdef DEBUG
+ pid = p->id;
+#endif
- p_prev = p_next;
+#ifdef ERTS_SMP
+ erts_proc_lock_init(p); /* All locks locked */
+#endif
- while (1) {
- p_next++;
- if(p_next >= erts_max_processes) {
- p_serial++;
- p_serial &= p_serial_mask;
- p_next = 0;
- }
+ /* Move into slot reserved */
+#ifdef DEBUG
+ ASSERT(ERTS_PROC_LOCK_BUSY
+ == (Process *) erts_smp_atomic_xchg_relb(&erts_proc.tab[pix],
+ (erts_aint_t) p));
+#else
+ erts_smp_atomic_set_relb(&erts_proc.tab[pix], (erts_aint_t) p);
+#endif
- if (p_prev == p_next) {
- p_next = -1;
- break; /* Table full! */
- }
+ ASSERT(internal_pid_serial(p->id) <= (erts_use_r9_pids_ports
+ ? ERTS_MAX_PID_R9_SERIAL
+ : ERTS_MAX_PID_SERIAL));
- if (!process_tab[p_next])
- break; /* found a free slot */
- }
+ erts_smp_rwmtx_runlock(&erts_proc_tab_rwmtx);
- error:
+ p->rcount = 0;
- erts_smp_mtx_unlock(&proc_tab_mtx);
+ ASSERT(p == (Process *)
+ erts_smp_atomic_read_nob(
+ &erts_proc.tab[internal_pid_index(pid)]));
return p;
+enomem:
+system_limit:
+
+ erts_smp_rwmtx_runlock(&erts_proc_tab_rwmtx);
+ return NULL;
+
}
Eterm
@@ -7117,7 +7269,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
Eterm args, /* Arguments for function (must be well-formed list). */
ErlSpawnOpts* so) /* Options for spawn. */
{
- ErtsRunQueue *rq, *notify_runq;
+ ErtsRunQueue *rq = NULL;
Process *p;
Sint arity; /* Number of arguments. */
#ifndef HYBRID
@@ -7126,6 +7278,8 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
Uint sz; /* Needed words on heap. */
Uint heap_need; /* Size needed on heap. */
Eterm res = THE_NON_VALUE;
+ erts_aint32_t state = 0;
+ erts_aint32_t prio = (erts_aint32_t) PRIORITY_NORMAL;
#ifdef ERTS_SMP
erts_smp_proc_lock(parent, ERTS_PROC_LOCKS_ALL_MINOR);
@@ -7135,7 +7289,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
/*
* Copy the arguments to the global heap
* Since global GC might occur we want to do this before adding the
- * new process to the process_tab.
+ * new process to the erts_proc.tab.
*/
BM_SWAP_TIMER(system,copy);
LAZY_COPY(parent,args);
@@ -7150,8 +7304,24 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
so->error_code = BADARG;
goto error;
}
- p = alloc_process(); /* All proc locks are locked by this thread
- on success */
+
+ if (so->flags & SPO_USE_ARGS) {
+ if (so->scheduler) {
+ int ix = so->scheduler-1;
+ ASSERT(0 <= ix && ix < erts_no_run_queues);
+ rq = ERTS_RUNQ_IX(ix);
+ state |= ERTS_PSFLG_BOUND;
+ }
+ prio = (erts_aint32_t) so->priority;
+ }
+
+ state |= (prio & ERTS_PSFLG_PRIO_MASK);
+
+ if (!rq)
+ rq = erts_get_runq_proc(parent);
+
+ p = alloc_process(rq, state); /* All proc locks are locked by this thread
+ on success */
if (!p) {
erts_send_error_to_logger_str(parent->group_leader,
"Too many processes\n");
@@ -7173,22 +7343,16 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
p->flags = erts_default_process_flags;
- /* Scheduler queue mutex should be locked when changeing
- * prio. In this case we don't have to lock it, since
- * noone except us has access to the process.
- */
if (so->flags & SPO_USE_ARGS) {
p->min_heap_size = so->min_heap_size;
p->min_vheap_size = so->min_vheap_size;
- p->prio = so->priority;
p->max_gen_gcs = so->max_gen_gcs;
} else {
p->min_heap_size = H_MIN_SIZE;
p->min_vheap_size = BIN_VH_MIN_SIZE;
- p->prio = PRIORITY_NORMAL;
p->max_gen_gcs = (Uint16) erts_smp_atomic32_read_nob(&erts_max_gen_gcs);
}
- p->skipped = 0;
+ p->schedule_count = 0;
ASSERT(p->min_heap_size == erts_next_heap_size(p->min_heap_size, 0));
p->initial[INITIAL_MOD] = mod;
@@ -7302,7 +7466,6 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
p->msg_inq.first = NULL;
p->msg_inq.last = &p->msg_inq.first;
p->msg_inq.len = 0;
- p->bound_runq = NULL;
#endif
p->bif_timers = NULL;
p->mbuf = NULL;
@@ -7404,9 +7567,6 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
#ifdef ERTS_SMP
p->scheduler_data = NULL;
- p->is_exiting = 0;
- p->status_flags = 0;
- p->runq_flags = 0;
p->suspendee = NIL;
p->pending_suspenders = NULL;
p->pending_exit.reason = THE_NON_VALUE;
@@ -7417,34 +7577,15 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
p->fp_exception = 0;
#endif
+ erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL);
+
+ res = p->id;
+
/*
* Schedule process for execution.
*/
- if (!((so->flags & SPO_USE_ARGS) && so->scheduler))
- rq = erts_get_runq_proc(parent);
- else {
- int ix = so->scheduler-1;
- ASSERT(0 <= ix && ix < erts_no_run_queues);
- rq = ERTS_RUNQ_IX(ix);
- p->bound_runq = rq;
- }
-
- erts_smp_runq_lock(rq);
-
-#ifdef ERTS_SMP
- p->run_queue = rq;
-#endif
-
- p->status = P_WAITING;
- notify_runq = internal_add_to_runq(rq, p);
-
- erts_smp_runq_unlock(rq);
-
- smp_notify_inc_runq(notify_runq);
-
- res = p->id;
- erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL);
+ schedule_process(p, state, 0);
VERBOSE(DEBUG_PROCESSES, ("Created a new process: %T\n",p->id));
@@ -7480,12 +7621,8 @@ void erts_init_empty_process(Process *p)
p->max_gen_gcs = 0;
p->min_heap_size = 0;
p->min_vheap_size = 0;
- p->status = P_RUNABLE;
- p->gcstatus = P_RUNABLE;
- p->rstatus = P_RUNABLE;
p->rcount = 0;
p->id = ERTS_INVALID_PID;
- p->prio = PRIORITY_NORMAL;
p->reds = 0;
p->tracer_proc = NIL;
p->trace_flags = F_INITIAL_TRACE_FLAGS;
@@ -7502,7 +7639,6 @@ void erts_init_empty_process(Process *p)
p->bin_vheap_mature = 0;
#ifdef ERTS_SMP
p->u.ptimer = NULL;
- p->bound_runq = NULL;
#else
memset(&(p->u.tm), 0, sizeof(ErlTimer));
#endif
@@ -7556,8 +7692,8 @@ void erts_init_empty_process(Process *p)
p->def_arg_reg[5] = 0;
p->parent = NIL;
- p->started.tv_sec = 0;
- p->started.tv_usec = 0;
+ p->approx_started = 0;
+ p->started_interval = 0;
#ifdef HIPE
hipe_init_process(&p->hipe);
@@ -7579,12 +7715,10 @@ void erts_init_empty_process(Process *p)
p->last_old_htop = NULL;
#endif
+ erts_smp_atomic32_init_nob(&p->state, (erts_aint32_t) PRIORITY_NORMAL);
#ifdef ERTS_SMP
p->scheduler_data = NULL;
- p->is_exiting = 0;
- p->status_flags = 0;
- p->runq_flags = 0;
p->msg_inq.first = NULL;
p->msg_inq.last = &p->msg_inq.first;
p->msg_inq.len = 0;
@@ -7594,7 +7728,7 @@ void erts_init_empty_process(Process *p)
p->pending_exit.bp = NULL;
erts_proc_lock_init(p);
erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL);
- p->run_queue = ERTS_RUNQ_IX(0);
+ RUNQ_SET_RQ(&p->run_queue, ERTS_RUNQ_IX(0));
#endif
#if !defined(NO_FPE_SIGNALS) || defined(HIPE)
@@ -7675,8 +7809,8 @@ erts_cleanup_empty_process(Process* p)
free_message_buffer(p->mbuf);
p->mbuf = NULL;
}
-#if defined(ERTS_ENABLE_LOCK_COUNT) && defined(ERTS_SMP)
- erts_lcnt_proc_lock_destroy(p);
+#ifdef ERTS_SMP
+ erts_proc_lock_fin(p);
#endif
#ifdef DEBUG
erts_debug_verify_clean_empty_process(p);
@@ -7789,26 +7923,34 @@ delete_process(Process* p)
}
+static ERTS_INLINE erts_aint32_t
+set_proc_exiting_state(Process *p, erts_aint32_t state)
+{
+ erts_aint32_t a, n, e;
+ a = state;
+ while (1) {
+ n = e = a;
+ n &= ~(ERTS_PSFLG_SUSPENDED|ERTS_PSFLG_PENDING_EXIT);
+ n |= ERTS_PSFLG_EXITING|ERTS_PSFLG_ACTIVE;
+ if (!(a & (ERTS_PSFLG_IN_RUNQ|ERTS_PSFLG_RUNNING)))
+ n |= ERTS_PSFLG_IN_RUNQ;
+ a = erts_smp_atomic32_cmpxchg_relb(&p->state, n, e);
+ if (a == e)
+ break;
+ }
+ return a;
+}
+
static ERTS_INLINE void
-set_proc_exiting(Process *p, Eterm reason, ErlHeapFragment *bp)
+set_proc_exiting(Process *p,
+ erts_aint32_t state,
+ Eterm reason,
+ ErlHeapFragment *bp)
{
-#ifdef ERTS_SMP
- erts_pix_lock_t *pix_lock = ERTS_PID2PIXLOCK(p->id);
ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(p) == ERTS_PROC_LOCKS_ALL);
- /*
- * You are required to have all proc locks and the pix lock when going
- * to status P_EXITING. This makes it is enough to take any lock when
- * looking up a process (pid2proc()) to prevent the looked up process
- * from exiting until the lock has been released.
- */
- erts_pix_lock(pix_lock);
- p->is_exiting = 1;
-#endif
- p->status = P_EXITING;
-#ifdef ERTS_SMP
- erts_pix_unlock(pix_lock);
-#endif
+ state = set_proc_exiting_state(p, state);
+
p->fvalue = reason;
if (bp)
erts_link_mbuf_to_proc(p, bp);
@@ -7821,6 +7963,14 @@ set_proc_exiting(Process *p, Eterm reason, ErlHeapFragment *bp)
KILL_CATCHES(p);
cancel_timer(p);
p->i = (BeamInstr *) beam_exit;
+
+ if (erts_system_profile_flags.runnable_procs
+ && !(state & (ERTS_PSFLG_ACTIVE|ERTS_PSFLG_SUSPENDED))) {
+ profile_runnable_proc(p, am_active);
+ }
+
+ if (!(state & (ERTS_PSFLG_IN_RUNQ|ERTS_PSFLG_RUNNING)))
+ add2runq(p, state);
}
@@ -7833,8 +7983,8 @@ erts_handle_pending_exit(Process *c_p, ErtsProcLocks locks)
ASSERT(is_value(c_p->pending_exit.reason));
ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == locks);
ERTS_SMP_LC_ASSERT(locks & ERTS_PROC_LOCK_MAIN);
- ERTS_SMP_LC_ASSERT(c_p->status != P_EXITING);
- ERTS_SMP_LC_ASSERT(c_p->status != P_FREE);
+ ERTS_SMP_LC_ASSERT(!((ERTS_PSFLG_EXITING|ERTS_PSFLG_FREE)
+ & erts_smp_atomic32_read_nob(&c_p->state)));
/* Ensure that all locks on c_p are locked before proceeding... */
if (locks == ERTS_PROC_LOCKS_ALL)
@@ -7847,7 +7997,10 @@ erts_handle_pending_exit(Process *c_p, ErtsProcLocks locks)
}
}
- set_proc_exiting(c_p, c_p->pending_exit.reason, c_p->pending_exit.bp);
+ set_proc_exiting(c_p,
+ erts_smp_atomic32_read_acqb(&c_p->state),
+ c_p->pending_exit.reason,
+ c_p->pending_exit.bp);
c_p->pending_exit.reason = THE_NON_VALUE;
c_p->pending_exit.bp = NULL;
@@ -7863,11 +8016,12 @@ handle_pending_exiters(ErtsProcList *pnd_xtrs)
while (plp) {
Process *p = erts_pid2proc(NULL, 0, plp->pid, ERTS_PROC_LOCKS_ALL);
if (p) {
- if (proclist_same(plp, p)
- && !(p->status_flags & ERTS_PROC_SFLG_RUNNING)) {
- ASSERT(p->status_flags & ERTS_PROC_SFLG_INRUNQ);
- ASSERT(ERTS_PROC_PENDING_EXIT(p));
- erts_handle_pending_exit(p, ERTS_PROC_LOCKS_ALL);
+ if (proclist_same(plp, p)) {
+ erts_aint32_t state = erts_smp_atomic32_read_acqb(&p->state);
+ if (!(state & ERTS_PSFLG_RUNNING)) {
+ ASSERT(state & ERTS_PSFLG_PENDING_EXIT);
+ erts_handle_pending_exit(p, ERTS_PROC_LOCKS_ALL);
+ }
}
erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL);
}
@@ -7895,7 +8049,7 @@ save_pending_exiter(Process *p)
rq->procs.pending_exiters = plp;
erts_smp_runq_unlock(rq);
-
+ wake_scheduler(rq, 1);
}
#endif
@@ -7957,7 +8111,7 @@ send_exit_message(Process *to, ErtsProcLocks *to_locksp,
* SMP emulator). When the signal is received the receiver receives an
* 'EXIT' message if it is trapping exits; otherwise, it will either
* ignore the signal if the exit reason is normal, or go into an
- * exiting state (status P_EXITING). When a process has gone into the
+ * exiting state (ERTS_PSFLG_EXITING). When a process has gone into the
* exiting state it will not execute any more Erlang code, but it might
* take a while before it actually exits. The exit signal is being
* received when the 'EXIT' message is put in the message queue, the
@@ -8030,6 +8184,7 @@ send_exit_signal(Process *c_p, /* current process if and only
Uint32 flags /* flags */
)
{
+ erts_aint32_t state = erts_smp_atomic32_read_nob(&rp->state);
Eterm rsn = reason == am_kill ? am_killed : reason;
ERTS_SMP_LC_ASSERT(*rp_locks == erts_proc_lc_my_proc_locks(rp));
@@ -8051,7 +8206,7 @@ send_exit_signal(Process *c_p, /* current process if and only
}
#endif
- if (ERTS_PROC_IS_TRAPPING_EXITS(rp)
+ if ((state & ERTS_PSFLG_TRAP_EXIT)
&& (reason != am_kill || (flags & ERTS_XSIG_FLG_IGN_KILL))) {
if (is_not_nil(token)
#ifdef USE_VM_PROBES
@@ -8067,9 +8222,7 @@ send_exit_signal(Process *c_p, /* current process if and only
}
else if (reason != am_normal || (flags & ERTS_XSIG_FLG_NO_IGN_NORMAL)) {
#ifdef ERTS_SMP
- if (!ERTS_PROC_PENDING_EXIT(rp) && !rp->is_exiting) {
- ASSERT(rp->status != P_EXITING);
- ASSERT(rp->status != P_FREE);
+ if (!(state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT))) {
ASSERT(!rp->pending_exit.bp);
if (rp == c_p && (*rp_locks & ERTS_PROC_LOCK_MAIN)) {
@@ -8085,9 +8238,9 @@ send_exit_signal(Process *c_p, /* current process if and only
}
*rp_locks = ERTS_PROC_LOCKS_ALL;
}
- set_proc_exiting(c_p, rsn, NULL);
+ set_proc_exiting(c_p, state, rsn, NULL);
}
- else if (!(rp->status_flags & ERTS_PROC_SFLG_RUNNING)) {
+ else if (!(state & ERTS_PSFLG_RUNNING)) {
/* Process not running ... */
ErtsProcLocks need_locks = ~(*rp_locks) & ERTS_PROC_LOCKS_ALL;
if (need_locks
@@ -8104,6 +8257,7 @@ send_exit_signal(Process *c_p, /* current process if and only
/* ...and we have all locks on it... */
*rp_locks = ERTS_PROC_LOCKS_ALL;
set_proc_exiting(rp,
+ state,
(is_immed(rsn)
? rsn
: copy_object(rsn, rp)),
@@ -8133,11 +8287,9 @@ send_exit_signal(Process *c_p, /* current process if and only
&bp->off_heap);
rp->pending_exit.bp = bp;
}
- ASSERT(ERTS_PROC_PENDING_EXIT(rp));
+ erts_smp_atomic32_read_bor_relb(&rp->state,
+ ERTS_PSFLG_PENDING_EXIT);
}
- if (!(rp->status_flags
- & (ERTS_PROC_SFLG_INRUNQ|ERTS_PROC_SFLG_RUNNING)))
- erts_add_to_runq(rp);
}
/* else:
*
@@ -8149,18 +8301,15 @@ send_exit_signal(Process *c_p, /* current process if and only
* exit or by itself before seeing the pending exit.
*/
#else /* !ERTS_SMP */
- if (c_p == rp) {
- rp->status = P_EXITING;
- c_p->fvalue = rsn;
- }
- else if (rp->status != P_EXITING) { /* No recursive process exits /PaN */
- Eterm old_status = rp->status;
+ erts_aint32_t state = erts_smp_atomic32_read_nob(&rp->state);
+ if (!(state & ERTS_PSFLG_EXITING)) {
set_proc_exiting(rp,
- is_immed(rsn) ? rsn : copy_object(rsn, rp),
+ state,
+ (is_immed(rsn) || c_p == rp
+ ? rsn
+ : copy_object(rsn, rp)),
NULL);
ACTIVATE(rp);
- if (old_status != P_RUNABLE && old_status != P_RUNNING)
- erts_add_to_runq(rp);
}
#endif
return -1; /* Receiver will exit */
@@ -8463,12 +8612,14 @@ resume_suspend_monitor(ErtsSuspendMonitor *smon, void *vc_p)
erts_destroy_suspend_monitor(smon);
}
-static void
-continue_exit_process(Process *p
#ifdef ERTS_SMP
- , erts_pix_lock_t *pix_lock
+static void
+proc_dec_refc(void *vproc)
+{
+ erts_smp_proc_dec_refc((Process *) vproc);
+}
#endif
- );
+
/* this function fishishes a process and propagates exit messages - called
by process_main when a process dies */
@@ -8476,9 +8627,8 @@ void
erts_do_exit_process(Process* p, Eterm reason)
{
#ifdef ERTS_SMP
- erts_pix_lock_t *pix_lock = ERTS_PID2PIXLOCK(p->id);
+ erts_aint32_t state;
#endif
-
p->arity = 0; /* No live registers */
p->fvalue = reason;
@@ -8496,27 +8646,17 @@ erts_do_exit_process(Process* p, Eterm reason)
#ifdef ERTS_SMP
ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p);
/* By locking all locks (main lock is already locked) when going
- to status P_EXITING, it is enough to take any lock when
+ to exiting state (ERTS_PSFLG_EXITING), it is enough to take any lock when
looking up a process (erts_pid2proc()) to prevent the looked up
process from exiting until the lock has been released. */
erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR);
#endif
-
- if (erts_system_profile_flags.runnable_procs && (p->status != P_WAITING)) {
- profile_runnable_proc(p, am_inactive);
- }
-
-#ifdef ERTS_SMP
- erts_pix_lock(pix_lock);
- p->is_exiting = 1;
-#endif
-
- p->status = P_EXITING;
-
-#ifdef ERTS_SMP
- erts_pix_unlock(pix_lock);
- if (ERTS_PROC_PENDING_EXIT(p)) {
+#ifndef ERTS_SMP
+ set_proc_exiting_state(p, erts_smp_atomic32_read_nob(&p->state));
+#else
+ state = set_proc_exiting_state(p, erts_smp_atomic32_read_nob(&p->state));
+ if (state & ERTS_PSFLG_PENDING_EXIT) {
/* Process exited before pending exit was received... */
p->pending_exit.reason = THE_NON_VALUE;
if (p->pending_exit.bp) {
@@ -8555,29 +8695,11 @@ erts_do_exit_process(Process* p, Eterm reason)
erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL_MINOR);
-#ifdef ERTS_SMP
- continue_exit_process(p, pix_lock);
-#else
- continue_exit_process(p);
-#endif
+ erts_continue_exit_process(p);
}
void
-erts_continue_exit_process(Process *c_p)
-{
-#ifdef ERTS_SMP
- continue_exit_process(c_p, ERTS_PID2PIXLOCK(c_p->id));
-#else
- continue_exit_process(c_p);
-#endif
-}
-
-static void
-continue_exit_process(Process *p
-#ifdef ERTS_SMP
- , erts_pix_lock_t *pix_lock
-#endif
- )
+erts_continue_exit_process(Process *p)
{
ErtsLink* lnk;
ErtsMonitor *mon;
@@ -8593,11 +8715,7 @@ continue_exit_process(Process *p
ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(p));
-#ifdef DEBUG
- erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
- ASSERT(p->status == P_EXITING);
- erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
-#endif
+ ASSERT(ERTS_PROC_IS_EXITING(p));
#ifdef ERTS_SMP
if (p->flags & F_HAVE_BLCKD_MSCHED) {
@@ -8665,48 +8783,47 @@ continue_exit_process(Process *p
#endif
{
+ int maybe_save;
int pix;
/* Do *not* use erts_get_runq_proc() */
ErtsRunQueue *rq;
rq = erts_get_runq_current(ERTS_GET_SCHEDULER_DATA_FROM_PROC(p));
- ASSERT(internal_pid_index(p->id) < erts_max_processes);
pix = internal_pid_index(p->id);
- erts_smp_mtx_lock(&proc_tab_mtx);
+ erts_smp_rwmtx_rlock(&erts_proc_tab_rwmtx);
+ maybe_save = saved_term_procs.end != NULL;
+ if (maybe_save) {
+ erts_smp_rwmtx_runlock(&erts_proc_tab_rwmtx);
+ erts_smp_rwmtx_rwlock(&erts_proc_tab_rwmtx);
+ }
+
erts_smp_runq_lock(rq);
#ifdef ERTS_SMP
- erts_pix_lock(pix_lock);
-
ASSERT(p->scheduler_data);
ASSERT(p->scheduler_data->current_process == p);
ASSERT(p->scheduler_data->free_process == NULL);
p->scheduler_data->current_process = NULL;
p->scheduler_data->free_process = p;
- p->status_flags = 0;
#endif
- process_tab[pix] = NULL; /* Time of death! */
+ /* Time of death! */
+ erts_smp_atomic_set_relb(&erts_proc.tab[pix], ERTS_AINT_NULL);
+
ASSERT(erts_smp_atomic32_read_nob(&process_count) > 0);
- erts_smp_atomic32_dec_nob(&process_count);
+ erts_smp_atomic32_dec_relb(&process_count);
-#ifdef ERTS_SMP
- erts_pix_unlock(pix_lock);
-#endif
erts_smp_runq_unlock(rq);
- if (p_next < 0) {
- if (p_last >= p_next) {
- p_serial++;
- p_serial &= p_serial_mask;
- }
- p_next = pix;
+ if (!maybe_save)
+ erts_smp_rwmtx_runlock(&erts_proc_tab_rwmtx);
+ else {
+ if (saved_term_procs.end)
+ save_terminating_process(p);
+ erts_smp_rwmtx_rwunlock(&erts_proc_tab_rwmtx);
}
- ERTS_MAYBE_SAVE_TERMINATING_PROCESS(p);
-
- erts_smp_mtx_unlock(&proc_tab_mtx);
}
/*
@@ -8721,7 +8838,21 @@ continue_exit_process(Process *p
lnk = p->nlinks;
p->nlinks = NULL;
- p->status = P_FREE;
+
+ {
+ /* Inactivate and notify free */
+ erts_aint32_t n, e, a = erts_smp_atomic32_read_nob(&p->state);
+ while (1) {
+ n = e = a;
+ ASSERT(a & ERTS_PSFLG_EXITING);
+ n |= ERTS_PSFLG_FREE;
+ n &= ~ERTS_PSFLG_ACTIVE;
+ a = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e);
+ if (a == e)
+ break;
+ }
+ }
+
dep = ((p->flags & F_DISTRIBUTION)
? ERTS_PROC_SET_DIST_ENTRY(p, ERTS_PROC_LOCKS_ALL, NULL)
: NULL);
@@ -8776,8 +8907,25 @@ continue_exit_process(Process *p
delete_process(p);
+#ifdef ERTS_SMP
+ /*
+ * Each scheduler will decrease refc by one via misc aux work;
+ * we have one refc for reference from process table which we
+ * now want to remove, i.e. we increase refc with schedulers-1.
+ *
+ * Process struct wont be deallocated until (earliest) when
+ * all schedulers have decreased refc via misc aux work...
+ */
+ if (erts_no_schedulers != 1)
+ erts_smp_proc_add_refc(p, (Sint32) erts_no_schedulers-1);
+ erts_schedule_multi_misc_aux_work(0,
+ erts_no_schedulers,
+ proc_dec_refc,
+ (void *) p);
+
erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN);
ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p);
+#endif
return;
@@ -8790,8 +8938,6 @@ continue_exit_process(Process *p
ERTS_SMP_LC_ASSERT(curr_locks == erts_proc_lc_my_proc_locks(p));
ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & curr_locks);
- ASSERT(p->status == P_EXITING);
-
p->i = (BeamInstr *) beam_continue_exit;
if (!(curr_locks & ERTS_PROC_LOCK_STATUS)) {
@@ -8799,8 +8945,6 @@ continue_exit_process(Process *p
curr_locks |= ERTS_PROC_LOCK_STATUS;
}
- erts_add_to_runq(p);
-
if (curr_locks != ERTS_PROC_LOCK_MAIN)
erts_smp_proc_unlock(p, ~ERTS_PROC_LOCK_MAIN & curr_locks);
@@ -8812,33 +8956,15 @@ continue_exit_process(Process *p
static void
timeout_proc(Process* p)
{
+ erts_aint32_t state;
BeamInstr** pi = (BeamInstr **) p->def_arg_reg;
p->i = *pi;
p->flags |= F_TIMO;
p->flags &= ~F_INSLPQUEUE;
- switch (p->status) {
- case P_GARBING:
- switch (p->gcstatus) {
- case P_SUSPENDED:
- goto suspended;
- case P_WAITING:
- goto waiting;
- default:
- break;
- }
- break;
- case P_WAITING:
- waiting:
- erts_add_to_runq(p);
- break;
- case P_SUSPENDED:
- suspended:
- p->rstatus = P_RUNABLE; /* MUST set resume status to runnable */
- break;
- default:
- break;
- }
+ state = erts_smp_atomic32_read_acqb(&p->state);
+ if (!(state & ERTS_PSFLG_ACTIVE))
+ schedule_process(p, state, 0);
}
@@ -8906,6 +9032,7 @@ erts_stack_dump(int to, void *to_arg, Process *p)
void
erts_program_counter_info(int to, void *to_arg, Process *p)
{
+ erts_aint32_t state;
int i;
erts_print(to, to_arg, "Program counter: %p (", p->i);
@@ -8914,7 +9041,8 @@ erts_program_counter_info(int to, void *to_arg, Process *p)
erts_print(to, to_arg, "CP: %p (", p->cp);
print_function_from_pc(to, to_arg, p->cp);
erts_print(to, to_arg, ")\n");
- if (!((p->status == P_RUNNING) || (p->status == P_GARBING))) {
+ state = erts_smp_atomic32_read_acqb(&p->state);
+ if (!(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_GC))) {
erts_print(to, to_arg, "arity = %d\n",p->arity);
if (!ERTS_IS_CRASH_DUMPING) {
/*
@@ -9087,13 +9215,13 @@ do { \
#endif
#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS
-# define ERTS_PROCS_DBG_CHK_PID_FOUND(PBDP, PID, TVP) \
- debug_processes_check_found_pid((PBDP), (PID), (TVP), 1)
-# define ERTS_PROCS_DBG_CHK_PID_NOT_FOUND(PBDP, PID, TVP) \
- debug_processes_check_found_pid((PBDP), (PID), (TVP), 0)
+# define ERTS_PROCS_DBG_CHK_PID_FOUND(PBDP, PID, IC) \
+ debug_processes_check_found_pid((PBDP), (PID), (IC), 1)
+# define ERTS_PROCS_DBG_CHK_PID_NOT_FOUND(PBDP, PID, IC) \
+ debug_processes_check_found_pid((PBDP), (PID), (IC), 0)
#else
-# define ERTS_PROCS_DBG_CHK_PID_FOUND(PBDP, PID, TVP)
-# define ERTS_PROCS_DBG_CHK_PID_NOT_FOUND(PBDP, PID, TVP)
+# define ERTS_PROCS_DBG_CHK_PID_FOUND(PBDP, PID, IC)
+# define ERTS_PROCS_DBG_CHK_PID_NOT_FOUND(PBDP, PID, IC)
#endif
#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_TERM_PROC_LIST
@@ -9138,7 +9266,7 @@ static Uint processes_bif_tab_chunks;
static Export processes_trap_export;
typedef struct {
- SysTimeval time;
+ Uint64 interval;
} ErtsProcessesBifChunkInfo;
typedef enum {
@@ -9163,7 +9291,7 @@ typedef struct {
struct {
Eterm caller;
#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS
- SysTimeval *pid_started;
+ Uint64 *pid_started;
#endif
#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_HALLOC
Eterm *heap;
@@ -9192,11 +9320,10 @@ static void debug_processes_verify_all_pids(ErtsProcessesBifData *pbdp);
#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS
static void debug_processes_check_found_pid(ErtsProcessesBifData *pbdp,
Eterm pid,
- SysTimeval *started,
+ Uint64 ic,
int pid_should_be_found);
#endif
#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_TERM_PROC_LIST
-static SysTimeval debug_tv_start;
static void debug_processes_check_term_proc_list(void);
static void debug_processes_check_term_proc_free_list(ErtsTermProcElement *tpep);
#endif
@@ -9207,7 +9334,7 @@ save_terminating_process(Process *p)
ErtsTermProcElement *tpep = erts_alloc(ERTS_ALC_T_PROCS_TPROC_EL,
sizeof(ErtsTermProcElement));
ERTS_PROCS_ASSERT(saved_term_procs.start && saved_term_procs.end);
- ERTS_SMP_LC_ASSERT(erts_lc_mtx_is_locked(&proc_tab_mtx));
+ ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&erts_proc_tab_rwmtx));
ERTS_PROCS_DBG_CHK_TPLIST();
@@ -9215,19 +9342,19 @@ save_terminating_process(Process *p)
tpep->next = NULL;
tpep->ix = internal_pid_index(p->id);
tpep->u.process.pid = p->id;
- tpep->u.process.spawned = p->started;
- erts_get_emu_time(&tpep->u.process.exited);
+ tpep->u.process.spawned = p->started_interval;
+ tpep->u.process.exited = get_proc_interval();
saved_term_procs.end->next = tpep;
saved_term_procs.end = tpep;
ERTS_PROCS_DBG_CHK_TPLIST();
- ERTS_PROCS_ASSERT((tpep->prev->ix >= 0
- ? erts_cmp_timeval(&tpep->u.process.exited,
- &tpep->prev->u.process.exited)
- : erts_cmp_timeval(&tpep->u.process.exited,
- &tpep->prev->u.bif_invocation.time)) > 0);
+ ERTS_PROCS_ASSERT(tpep->prev->ix >= 0
+ ? (tpep->u.process.exited
+ >= tpep->prev->u.process.exited)
+ : (tpep->u.process.exited
+ >= tpep->prev->u.bif_invocation.interval));
}
static void
@@ -9258,7 +9385,7 @@ cleanup_processes_bif_data(Binary *bp)
if (pbdp->bif_invocation) {
ErtsTermProcElement *tpep;
- erts_smp_mtx_lock(&proc_tab_mtx);
+ erts_smp_rwmtx_rwlock(&erts_proc_tab_rwmtx);
ERTS_PROCS_DBG_TRACE(pbdp->debug.caller,
cleanup_processes_bif_data,
@@ -9312,7 +9439,7 @@ cleanup_processes_bif_data(Binary *bp)
ERTS_PROCS_DBG_CHK_TPLIST();
- erts_smp_mtx_unlock(&proc_tab_mtx);
+ erts_smp_rwmtx_rwunlock(&erts_proc_tab_rwmtx);
}
}
@@ -9340,7 +9467,7 @@ processes_bif_engine(Process *p, Eterm *res_accp, Binary *mbp)
pbdp->tix = 0;
pbdp->pid_ix = 0;
- erts_smp_mtx_lock(&proc_tab_mtx);
+ erts_smp_rwmtx_rwlock(&erts_proc_tab_rwmtx);
locked = 1;
ERTS_PROCS_DBG_TRACE(p->id, processes_bif_engine, init);
@@ -9351,7 +9478,7 @@ processes_bif_engine(Process *p, Eterm *res_accp, Binary *mbp)
#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS
pbdp->debug.pid_started = erts_alloc(ERTS_ALC_T_PROCS_PIDS,
- sizeof(SysTimeval)*pbdp->pid_sz);
+ sizeof(Uint64)*pbdp->pid_sz);
#endif
ERTS_PROCS_DBG_SAVE_PIDS(pbdp);
@@ -9366,7 +9493,8 @@ processes_bif_engine(Process *p, Eterm *res_accp, Binary *mbp)
pbdp->bif_invocation = erts_alloc(ERTS_ALC_T_PROCS_TPROC_EL,
sizeof(ErtsTermProcElement));
pbdp->bif_invocation->ix = -1;
- erts_get_emu_time(&pbdp->bif_invocation->u.bif_invocation.time);
+ pbdp->bif_invocation->u.bif_invocation.interval
+ = step_proc_interval();
ERTS_PROCS_DBG_CHK_TPLIST();
pbdp->bif_invocation->next = NULL;
@@ -9393,30 +9521,31 @@ processes_bif_engine(Process *p, Eterm *res_accp, Binary *mbp)
int indices = ERTS_PROCESSES_BIF_TAB_CHUNK_SIZE;
int cix = ix / ERTS_PROCESSES_BIF_TAB_CHUNK_SIZE;
int end_ix = ix + indices;
- SysTimeval *invocation_timep;
+ Uint64 *invocation_interval_p;
- invocation_timep = (pbdp->bif_invocation
- ? &pbdp->bif_invocation->u.bif_invocation.time
- : NULL);
+ invocation_interval_p
+ = (pbdp->bif_invocation
+ ? &pbdp->bif_invocation->u.bif_invocation.interval
+ : NULL);
ERTS_PROCS_ASSERT(is_nil(*res_accp));
if (!locked) {
- erts_smp_mtx_lock(&proc_tab_mtx);
+ erts_smp_rwmtx_rwlock(&erts_proc_tab_rwmtx);
locked = 1;
}
- ERTS_SMP_LC_ASSERT(erts_lc_mtx_is_locked(&proc_tab_mtx));
+ ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&erts_proc_tab_rwmtx));
ERTS_PROCS_DBG_TRACE(p->id, processes_bif_engine, insp_table);
if (cix != 0)
- erts_get_emu_time(&pbdp->chunk[cix].time);
+ pbdp->chunk[cix].interval = step_proc_interval();
else if (pbdp->bif_invocation)
- pbdp->chunk[0].time = *invocation_timep;
- /* else: Time is irrelevant */
+ pbdp->chunk[0].interval = *invocation_interval_p;
+ /* else: interval is irrelevant */
- if (end_ix >= erts_max_processes) {
+ if (end_ix >= erts_proc.max) {
ERTS_PROCS_ASSERT(cix+1 == processes_bif_tab_chunks);
- end_ix = erts_max_processes;
+ end_ix = erts_proc.max;
indices = end_ix - ix;
/* What to do when done with this chunk */
pbdp->state = (processes_bif_tab_chunks == 1
@@ -9425,16 +9554,15 @@ processes_bif_engine(Process *p, Eterm *res_accp, Binary *mbp)
}
for (; ix < end_ix; ix++) {
- Process *rp = process_tab[ix];
+ Process *rp = erts_pix2proc(ix);
if (rp
- && (!invocation_timep
- || erts_cmp_timeval(&rp->started,
- invocation_timep) < 0)) {
+ && (!invocation_interval_p
+ || rp->started_interval < *invocation_interval_p)) {
ERTS_PROCS_ASSERT(is_internal_pid(rp->id));
pbdp->pid[pbdp->pid_ix] = rp->id;
#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS
- pbdp->debug.pid_started[pbdp->pid_ix] = rp->started;
+ pbdp->debug.pid_started[pbdp->pid_ix] = rp->started_interval;
#endif
pbdp->pid_ix++;
@@ -9444,7 +9572,7 @@ processes_bif_engine(Process *p, Eterm *res_accp, Binary *mbp)
pbdp->tix = end_ix;
- erts_smp_mtx_unlock(&proc_tab_mtx);
+ erts_smp_rwmtx_rwunlock(&erts_proc_tab_rwmtx);
locked = 0;
reds = indices/ERTS_PROCESSES_BIF_TAB_INSPECT_INDICES_PER_RED;
@@ -9456,8 +9584,8 @@ processes_bif_engine(Process *p, Eterm *res_accp, Binary *mbp)
ix = pbdp->tix;
indices = ERTS_PROCESSES_BIF_TAB_CHUNK_SIZE;
end_ix = ix + indices;
- if (end_ix > erts_max_processes) {
- end_ix = erts_max_processes;
+ if (end_ix > erts_proc.max) {
+ end_ix = erts_proc.max;
indices = end_ix - ix;
}
@@ -9476,20 +9604,20 @@ processes_bif_engine(Process *p, Eterm *res_accp, Binary *mbp)
int i;
int max_reds;
int free_term_procs = 0;
- SysTimeval *invocation_timep;
+ Uint64 invocation_interval;
ErtsTermProcElement *tpep;
ErtsTermProcElement *free_list = NULL;
tpep = pbdp->bif_invocation;
ERTS_PROCS_ASSERT(tpep);
- invocation_timep = &tpep->u.bif_invocation.time;
+ invocation_interval = tpep->u.bif_invocation.interval;
max_reds = have_reds = ERTS_BIF_REDS_LEFT(p);
if (max_reds > ERTS_PROCESSES_INSPECT_TERM_PROC_MAX_REDS)
max_reds = ERTS_PROCESSES_INSPECT_TERM_PROC_MAX_REDS;
reds = 0;
- erts_smp_mtx_lock(&proc_tab_mtx);
+ erts_smp_rwmtx_rwlock(&erts_proc_tab_rwmtx);
ERTS_PROCS_DBG_TRACE(p->id, processes_bif_engine, insp_term_procs);
ERTS_PROCS_DBG_CHK_TPLIST();
@@ -9528,20 +9656,19 @@ processes_bif_engine(Process *p, Eterm *res_accp, Binary *mbp)
}
else {
int cix = tpep->ix/ERTS_PROCESSES_BIF_TAB_CHUNK_SIZE;
- SysTimeval *chunk_timep = &pbdp->chunk[cix].time;
+ Uint64 chunk_interval = pbdp->chunk[cix].interval;
Eterm pid = tpep->u.process.pid;
ERTS_PROCS_ASSERT(is_internal_pid(pid));
- if (erts_cmp_timeval(&tpep->u.process.spawned,
- invocation_timep) < 0) {
- if (erts_cmp_timeval(&tpep->u.process.exited,
- chunk_timep) < 0) {
+ if (tpep->u.process.spawned < invocation_interval) {
+ if (tpep->u.process.exited < chunk_interval) {
ERTS_PROCS_DBG_CHK_PID_NOT_FOUND(pbdp,
pid,
- &tpep->u.process.spawned);
+ tpep->u.process.spawned);
pbdp->pid[pbdp->pid_ix] = pid;
#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS
- pbdp->debug.pid_started[pbdp->pid_ix] = tpep->u.process.spawned;
+ pbdp->debug.pid_started[pbdp->pid_ix]
+ = tpep->u.process.spawned;
#endif
pbdp->pid_ix++;
ERTS_PROCS_ASSERT(pbdp->pid_ix <= pbdp->pid_sz);
@@ -9549,13 +9676,13 @@ processes_bif_engine(Process *p, Eterm *res_accp, Binary *mbp)
else {
ERTS_PROCS_DBG_CHK_PID_FOUND(pbdp,
pid,
- &tpep->u.process.spawned);
+ tpep->u.process.spawned);
}
}
else {
ERTS_PROCS_DBG_CHK_PID_NOT_FOUND(pbdp,
pid,
- &tpep->u.process.spawned);
+ tpep->u.process.spawned);
}
i++;
@@ -9604,7 +9731,7 @@ processes_bif_engine(Process *p, Eterm *res_accp, Binary *mbp)
ERTS_PROCS_DBG_CHK_TPLIST();
ERTS_PROCS_DBG_CHK_FREELIST(free_list);
- erts_smp_mtx_unlock(&proc_tab_mtx);
+ erts_smp_rwmtx_rwunlock(&erts_proc_tab_rwmtx);
/*
* We do the actual free of term proc structures now when we
@@ -9664,8 +9791,9 @@ processes_bif_engine(Process *p, Eterm *res_accp, Binary *mbp)
sizeof(Eterm)*pbdp->pid_sz);
#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS
pbdp->debug.pid_started = erts_realloc(ERTS_ALC_T_PROCS_PIDS,
- pbdp->debug.pid_started,
- sizeof(SysTimeval)*pbdp->pid_sz);
+ pbdp->debug.pid_started,
+ (sizeof(Uint64)
+ * pbdp->pid_sz));
#endif
}
reds = conses/ERTS_PROCESSES_BIF_BUILD_RESULT_CONSES_PER_RED;
@@ -9785,7 +9913,7 @@ init_processes_bif(void)
{
saved_term_procs.start = NULL;
saved_term_procs.end = NULL;
- processes_bif_tab_chunks = (((erts_max_processes - 1)
+ processes_bif_tab_chunks = (((erts_proc.max - 1)
/ ERTS_PROCESSES_BIF_TAB_CHUNK_SIZE)
+ 1);
@@ -9793,10 +9921,6 @@ init_processes_bif(void)
erts_init_trap_export(&processes_trap_export, am_erlang, am_processes_trap, 2,
&processes_trap);
-#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_TERM_PROC_LIST
- erts_get_emu_time(&debug_tv_start);
-#endif
-
}
/*
@@ -9828,31 +9952,29 @@ erts_debug_processes(Process *c_p)
Eterm res;
Eterm* hp;
Process *p;
-#ifdef DEBUG
Eterm *hp_end;
-#endif
- erts_smp_mtx_lock(&proc_tab_mtx);
+ erts_smp_rwmtx_rwlock(&erts_proc_tab_rwmtx);
res = NIL;
need = erts_process_count() * 2;
hp = HAlloc(c_p, need); /* we need two heap words for each pid */
-#ifdef DEBUG
hp_end = hp + need;
-#endif
/* make the list by scanning bakward */
- for (i = erts_max_processes-1; i >= 0; i--) {
- if ((p = process_tab[i]) != NULL) {
- res = CONS(hp, process_tab[i]->id, res);
+ for (i = erts_proc.max-1; i >= 0; i--) {
+ p = erts_pix2proc(i);
+ if (p) {
+ res = CONS(hp, p->id, res);
hp += 2;
}
}
- ASSERT(hp == hp_end);
- erts_smp_mtx_unlock(&proc_tab_mtx);
+ erts_smp_rwmtx_rwunlock(&erts_proc_tab_rwmtx);
+
+ HRelease(c_p, hp_end, hp);
return res;
}
@@ -9884,14 +10006,12 @@ erts_debug_processes_bif_info(Process *c_p)
static void
debug_processes_check_found_pid(ErtsProcessesBifData *pbdp,
Eterm pid,
- SysTimeval *tvp,
+ Uint64 ic,
int pid_should_be_found)
{
int i;
for (i = 0; i < pbdp->pid_ix; i++) {
- if (pbdp->pid[i] == pid
- && pbdp->debug.pid_started[i].tv_sec == tvp->tv_sec
- && pbdp->debug.pid_started[i].tv_usec == tvp->tv_usec) {
+ if (pbdp->pid[i] == pid && pbdp->debug.pid_started[i] == ic) {
ERTS_PROCS_ASSERT(pid_should_be_found);
return;
}
@@ -9925,8 +10045,8 @@ debug_processes_save_all_pids(ErtsProcessesBifData *pbdp)
pbdp->debug.correct_pids = erts_alloc(ERTS_ALC_T_PROCS_PIDS,
sizeof(Eterm)*pbdp->pid_sz);
- for (tix = 0, cpix = 0; tix < erts_max_processes; tix++) {
- Process *rp = process_tab[tix];
+ for (tix = 0, cpix = 0; tix < erts_proc.max; tix++) {
+ Process *rp = erts_pix2proc(tix);
if (rp) {
ERTS_PROCS_ASSERT(is_internal_pid(rp->id));
pbdp->debug.correct_pids[cpix++] = rp->id;
@@ -9979,14 +10099,13 @@ debug_processes_verify_all_pids(ErtsProcessesBifData *pbdp)
static void
debug_processes_check_term_proc_list(void)
{
- ERTS_SMP_LC_ASSERT(erts_lc_mtx_is_locked(&proc_tab_mtx));
+ ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&erts_proc_tab_rwmtx));
if (!saved_term_procs.start)
ERTS_PROCS_ASSERT(!saved_term_procs.end);
else {
- SysTimeval tv_now;
- SysTimeval *prev_xtvp = NULL;
+ Uint64 curr_interval = get_proc_interval();
+ Uint64 *prev_x_interval_p = NULL;
ErtsTermProcElement *tpep;
- erts_get_emu_time(&tv_now);
for (tpep = saved_term_procs.start; tpep; tpep = tpep->next) {
if (!tpep->prev)
@@ -9998,20 +10117,17 @@ debug_processes_check_term_proc_list(void)
else
ERTS_PROCS_ASSERT(tpep->next->prev == tpep);
if (tpep->ix < 0) {
- SysTimeval *tvp = &tpep->u.bif_invocation.time;
- ERTS_PROCS_ASSERT(erts_cmp_timeval(&debug_tv_start, tvp) < 0
- && erts_cmp_timeval(tvp, &tv_now) < 0);
+ Uint64 interval = tpep->u.bif_invocation.interval;
+ ERTS_PROCS_ASSERT(interval <= curr_interval);
}
else {
- SysTimeval *stvp = &tpep->u.process.spawned;
- SysTimeval *xtvp = &tpep->u.process.exited;
+ Uint64 s_interval = tpep->u.process.spawned;
+ Uint64 x_interval = tpep->u.process.exited;
- ERTS_PROCS_ASSERT(erts_cmp_timeval(&debug_tv_start,
- stvp) < 0);
- ERTS_PROCS_ASSERT(erts_cmp_timeval(stvp, xtvp) < 0);
- if (prev_xtvp)
- ERTS_PROCS_ASSERT(erts_cmp_timeval(prev_xtvp, xtvp) < 0);
- prev_xtvp = xtvp;
+ ERTS_PROCS_ASSERT(s_interval <= x_interval);
+ if (prev_x_interval_p)
+ ERTS_PROCS_ASSERT(*prev_x_interval_p <= x_interval);
+ prev_x_interval_p = &tpep->u.process.exited;
ERTS_PROCS_ASSERT(is_internal_pid(tpep->u.process.pid));
ERTS_PROCS_ASSERT(tpep->ix
== internal_pid_index(tpep->u.process.pid));
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 707e7abe63..2ab7b30906 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -112,28 +112,29 @@ extern int erts_sched_thread_suggested_stack_size;
#define PRIORITY_NORMAL 2
#define PRIORITY_LOW 3
#define ERTS_NO_PROC_PRIO_LEVELS 4
+#define ERTS_NO_PROC_PRIO_QUEUES 3
#define ERTS_PORT_PRIO_LEVEL ERTS_NO_PROC_PRIO_LEVELS
+#define ERTS_NO_PRIO_LEVELS (ERTS_NO_PROC_PRIO_LEVELS + 1)
#define ERTS_RUNQ_FLGS_PROCS_QMASK \
((((Uint32) 1) << ERTS_NO_PROC_PRIO_LEVELS) - 1)
-#define ERTS_NO_PRIO_LEVELS (ERTS_NO_PROC_PRIO_LEVELS + 1)
-#define ERTS_RUNQ_FLGS_MIGRATE_QMASK \
+#define ERTS_RUNQ_FLGS_QMASK \
((((Uint32) 1) << ERTS_NO_PRIO_LEVELS) - 1)
#define ERTS_RUNQ_FLGS_EMIGRATE_SHFT \
- ERTS_NO_PROC_PRIO_LEVELS
+ ERTS_NO_PRIO_LEVELS
#define ERTS_RUNQ_FLGS_IMMIGRATE_SHFT \
(ERTS_RUNQ_FLGS_EMIGRATE_SHFT + ERTS_NO_PRIO_LEVELS)
#define ERTS_RUNQ_FLGS_EVACUATE_SHFT \
(ERTS_RUNQ_FLGS_IMMIGRATE_SHFT + ERTS_NO_PRIO_LEVELS)
#define ERTS_RUNQ_FLGS_EMIGRATE_QMASK \
- (ERTS_RUNQ_FLGS_MIGRATE_QMASK << ERTS_RUNQ_FLGS_EMIGRATE_SHFT)
+ (ERTS_RUNQ_FLGS_QMASK << ERTS_RUNQ_FLGS_EMIGRATE_SHFT)
#define ERTS_RUNQ_FLGS_IMMIGRATE_QMASK \
- (ERTS_RUNQ_FLGS_MIGRATE_QMASK << ERTS_RUNQ_FLGS_IMMIGRATE_SHFT)
+ (ERTS_RUNQ_FLGS_QMASK << ERTS_RUNQ_FLGS_IMMIGRATE_SHFT)
#define ERTS_RUNQ_FLGS_EVACUATE_QMASK \
- (ERTS_RUNQ_FLGS_MIGRATE_QMASK << ERTS_RUNQ_FLGS_EVACUATE_SHFT)
+ (ERTS_RUNQ_FLGS_QMASK << ERTS_RUNQ_FLGS_EVACUATE_SHFT)
#define ERTS_RUNQ_FLG_BASE2 \
(ERTS_RUNQ_FLGS_EVACUATE_SHFT + ERTS_NO_PRIO_LEVELS)
@@ -148,14 +149,18 @@ extern int erts_sched_thread_suggested_stack_size;
(((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 3))
#define ERTS_RUNQ_FLG_INACTIVE \
(((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 4))
+#define ERTS_RUNQ_FLG_NONEMPTY \
+ (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 5))
+#define ERTS_RUNQ_FLG_PROTECTED \
+ (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 6))
#define ERTS_RUNQ_FLGS_MIGRATION_QMASKS \
(ERTS_RUNQ_FLGS_EMIGRATE_QMASK \
| ERTS_RUNQ_FLGS_IMMIGRATE_QMASK \
| ERTS_RUNQ_FLGS_EVACUATE_QMASK)
+
#define ERTS_RUNQ_FLGS_MIGRATION_INFO \
- (ERTS_RUNQ_FLGS_MIGRATION_QMASKS \
- | ERTS_RUNQ_FLG_INACTIVE \
+ (ERTS_RUNQ_FLG_INACTIVE \
| ERTS_RUNQ_FLG_OUT_OF_WORK \
| ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK)
@@ -186,33 +191,45 @@ extern int erts_sched_thread_suggested_stack_size;
#define ERTS_UNSET_RUNQ_FLG_EVACUATE(FLGS, PRIO) \
((FLGS) &= ~ERTS_RUNQ_FLG_EVACUATE((PRIO)))
-#define ERTS_RUNQ_IFLG_SUSPENDED (((erts_aint32_t) 1) << 0)
-#define ERTS_RUNQ_IFLG_NONEMPTY (((erts_aint32_t) 1) << 1)
-
-
-#ifdef DEBUG
-# if defined(ARCH_64) && !HALFWORD_HEAP
-# define ERTS_DBG_SET_INVALID_RUNQP(RQP, N) \
- (*((char **) &(RQP)) = (char *) (0xdeadbeefdead0003 | ((N) << 4)))
-# define ERTS_DBG_VERIFY_VALID_RUNQP(RQP) \
-do { \
- ASSERT((RQP) != NULL); \
- ASSERT(((((Uint) (RQP)) & ((Uint) 0x3))) == ((Uint) 0)); \
- ASSERT((((Uint) (RQP)) & ~((Uint) 0xffff)) != ((Uint) 0xdeadbeefdead0000));\
-} while (0)
-# else
-# define ERTS_DBG_SET_INVALID_RUNQP(RQP, N) \
- (*((char **) &(RQP)) = (char *) (0xdead0003 | ((N) << 4)))
-# define ERTS_DBG_VERIFY_VALID_RUNQP(RQP) \
-do { \
- ASSERT((RQP) != NULL); \
- ASSERT(((((UWord) (RQP)) & ((UWord) 1))) == ((UWord) 0)); \
- ASSERT((((UWord) (RQP)) & ~((UWord) 0xffff)) != ((UWord) 0xdead0000)); \
-} while (0)
-# endif
-#else
-# define ERTS_DBG_SET_INVALID_RUNQP(RQP, N)
-# define ERTS_DBG_VERIFY_VALID_RUNQP(RQP)
+#define ERTS_RUNQ_FLGS_INIT(RQ, INIT) \
+ erts_smp_atomic32_init_nob(&(RQ)->flags, (erts_aint32_t) (INIT))
+#define ERTS_RUNQ_FLGS_SET(RQ, FLGS) \
+ ((Uint32) erts_smp_atomic32_read_bor_relb(&(RQ)->flags, \
+ (erts_aint32_t) (FLGS)))
+#define ERTS_RUNQ_FLGS_UNSET(RQ, FLGS) \
+ ((Uint32) erts_smp_atomic32_read_band_relb(&(RQ)->flags, \
+ (erts_aint32_t) ~(FLGS)))
+#define ERTS_RUNQ_FLGS_GET(RQ) \
+ ((Uint32) erts_smp_atomic32_read_acqb(&(RQ)->flags))
+#define ERTS_RUNQ_FLGS_GET_NOB(RQ) \
+ ((Uint32) erts_smp_atomic32_read_nob(&(RQ)->flags))
+#define ERTS_RUNQ_FLGS_GET_MB(RQ) \
+ ((Uint32) erts_smp_atomic32_read_mb(&(RQ)->flags))
+#define ERTS_RUNQ_FLGS_MASK_SET(RQ, MSK, FLGS) \
+ ((Uint32) erts_smp_atomic32_mask_set_relb(&(RQ)->flags, \
+ (erts_aint32_t) (MSK), \
+ (erts_aint32_t) (FLGS)))
+
+ERTS_GLB_INLINE erts_aint32_t
+erts_smp_atomic32_mask_set_relb(erts_smp_atomic32_t *a32p,
+ erts_aint32_t mask,
+ erts_aint32_t set);
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+ERTS_GLB_INLINE erts_aint32_t
+erts_smp_atomic32_mask_set_relb(erts_smp_atomic32_t *a32p,
+ erts_aint32_t mask,
+ erts_aint32_t set)
+{
+ erts_aint32_t act = erts_smp_atomic32_read_nob(a32p);
+ while (1) {
+ erts_aint32_t exp = act;
+ erts_aint32_t new = exp & ~mask;
+ new |= (mask & set);
+ act = erts_smp_atomic32_cmpxchg_relb(a32p, new, exp);
+ if (act == exp)
+ return act;
+ }
+}
#endif
typedef enum {
@@ -291,7 +308,7 @@ struct ErtsSchedulerSleepInfo_ {
typedef struct ErtsProcList_ ErtsProcList;
struct ErtsProcList_ {
Eterm pid;
- SysTimeval started;
+ Uint64 started_interval;
ErtsProcList* next;
};
@@ -312,21 +329,39 @@ typedef struct ErtsSchedulerData_ ErtsSchedulerData;
typedef struct ErtsRunQueue_ ErtsRunQueue;
typedef struct {
- int len;
- int max_len;
+ erts_smp_atomic32_t len;
+ erts_aint32_t max_len;
int reds;
+} ErtsRunQueueInfo;
+
+#ifdef ERTS_SMP
+
+typedef struct {
+ Uint32 flags;
+ ErtsRunQueue *misc_evac_runq;
struct {
struct {
int this;
int other;
} limit;
ErtsRunQueue *runq;
- } migrate;
-} ErtsRunQueueInfo;
+ Uint32 flags;
+ } prio[ERTS_NO_PRIO_LEVELS];
+} ErtsMigrationPath;
+
+typedef struct ErtsMigrationPaths_ ErtsMigrationPaths;
+
+struct ErtsMigrationPaths_ {
+ void *block;
+ ErtsMigrationPaths *next;
+ ErtsThrPrgrVal thr_prgr;
+ ErtsMigrationPath mpath[1];
+};
+
+#endif /* ERTS_SMP */
struct ErtsRunQueue_ {
int ix;
- erts_smp_atomic32_t info_flags;
erts_smp_mtx_t mtx;
erts_smp_cnd_t cnd;
@@ -334,19 +369,18 @@ struct ErtsRunQueue_ {
ErtsSchedulerData *scheduler;
int waiting; /* < 0 in sys schedule; > 0 on cnd variable */
int woken;
- Uint32 flags;
+ erts_smp_atomic32_t flags;
int check_balance_reds;
int full_reds_history_sum;
int full_reds_history[ERTS_FULL_REDS_HISTORY_SIZE];
int out_of_work_count;
- int max_len;
- int len;
+ erts_aint32_t max_len;
+ erts_aint32_t len;
int wakeup_other;
int wakeup_other_reds;
int halt_in_progress;
struct {
- int len;
ErtsProcList *pending_exiters;
Uint context_switches;
Uint reductions;
@@ -361,7 +395,7 @@ struct ErtsRunQueue_ {
struct {
ErtsMiscOpList *start;
ErtsMiscOpList *end;
- ErtsRunQueue *evac_runq;
+ erts_smp_atomic_t evac_runq;
} misc;
struct {
@@ -381,7 +415,7 @@ extern ErtsAlignedRunQueue *erts_aligned_run_queues;
#define ERTS_PROC_REDUCTIONS_EXECUTED(RQ, PRIO, REDS, AREDS) \
do { \
(RQ)->procs.reductions += (AREDS); \
- (RQ)->procs.prio_info[p->prio].reds += (REDS); \
+ (RQ)->procs.prio_info[(PRIO)].reds += (REDS); \
(RQ)->check_balance_reds -= (REDS); \
(RQ)->wakeup_other_reds += (AREDS); \
} while (0)
@@ -495,6 +529,90 @@ extern ErtsAlignedSchedulerData *erts_aligned_scheduler_data;
extern ErtsSchedulerData *erts_scheduler_data;
#endif
+#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
+int erts_smp_lc_runq_is_locked(ErtsRunQueue *);
+#endif
+
+#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS
+
+/*
+ * Run queue locked during modifications. We use atomic ops since
+ * other threads peek at values without run queue lock.
+ */
+
+ERTS_GLB_INLINE void erts_smp_inc_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio);
+ERTS_GLB_INLINE void erts_smp_dec_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio);
+ERTS_GLB_INLINE void erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi);
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+ERTS_GLB_INLINE void
+erts_smp_inc_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio)
+{
+ erts_aint32_t len;
+
+ ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
+
+ len = erts_smp_atomic32_read_nob(&rqi->len);
+ ASSERT(len >= 0);
+ if (len == 0) {
+ ASSERT((erts_smp_atomic32_read_nob(&rq->flags)
+ & ((erts_aint32_t) (1 << prio))) == 0);
+ erts_smp_atomic32_read_bor_nob(&rq->flags,
+ (erts_aint32_t) (1 << prio));
+ }
+ len++;
+ if (rqi->max_len < len)
+ rqi->max_len = len;
+
+ erts_smp_atomic32_set_relb(&rqi->len, len);
+
+ rq->len++;
+ if (rq->max_len < rq->len)
+ rq->max_len = len;
+ ASSERT(rq->len > 0);
+}
+
+ERTS_GLB_INLINE void
+erts_smp_dec_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio)
+{
+ erts_aint32_t len;
+
+ ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
+
+ len = erts_smp_atomic32_read_nob(&rqi->len);
+ len--;
+ ASSERT(len >= 0);
+ if (len == 0) {
+ ASSERT((erts_smp_atomic32_read_nob(&rq->flags)
+ & ((erts_aint32_t) (1 << prio))));
+ erts_smp_atomic32_read_band_nob(&rq->flags,
+ ~((erts_aint32_t) (1 << prio)));
+ }
+ erts_smp_atomic32_set_relb(&rqi->len, len);
+
+ rq->len--;
+ ASSERT(rq->len >= 0);
+}
+
+ERTS_GLB_INLINE void
+erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi)
+{
+ erts_aint32_t len;
+
+ ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
+
+ len = erts_smp_atomic32_read_nob(&rqi->len);
+ ASSERT(rqi->max_len >= len);
+ rqi->max_len = len;
+}
+
+#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
+
+#define RUNQ_READ_LEN(X) erts_smp_atomic32_read_nob((X))
+
+#endif /* ERTS_INCLUDE_SCHEDULER_INTERNALS */
+
/*
* Process Specific Data.
*
@@ -607,6 +725,8 @@ struct ErtsPendingSuspend_ {
# define BIN_OLD_VHEAP(p) (p)->bin_old_vheap
struct process {
+ Eterm id; /* The pid of this process
+ (need to be first in struct) */
/* All fields in the PCB that differs between different heap
* architectures, have been moved to the end of this struct to
* make sure that as few offsets as possible differ. Different
@@ -649,13 +769,9 @@ struct process {
* Number of reductions left to execute.
* Only valid for the current process.
*/
- Uint32 status; /* process STATE */
- Uint32 gcstatus; /* process gc STATE */
- Uint32 rstatus; /* process resume STATE */
Uint32 rcount; /* suspend count */
- Eterm id; /* The pid of this process */
int prio; /* Priority of process */
- int skipped; /* Times a low prio process has been rescheduled */
+ int schedule_count; /* Times left to reschedule a low prio process */
Uint reds; /* No of reductions for this process */
Eterm tracer_proc; /* If proc is traced, this is the tracer
(can NOT be boxed) */
@@ -668,7 +784,6 @@ struct process {
Eterm ftrace; /* Latest exception stack trace dump */
Process *next; /* Pointer to next process in run queue */
- Process *prev; /* Pointer to prev process in run queue */
struct reg_proc *reg; /* NULL iff not registered */
ErtsLink *nlinks;
@@ -706,8 +821,8 @@ struct process {
* Information mainly for post-mortem use (erl crash dump).
*/
Eterm parent; /* Pid of process that created this process. */
- SysTimeval started; /* Time when started. */
-
+ erts_approx_time_t approx_started; /* Time when started. */
+ Uint64 started_interval;
/* This is the place, where all fields that differs between memory
* architectures, have gone to.
@@ -738,19 +853,16 @@ struct process {
void *exit_data; /* Misc data referred during termination */
} u;
- ErtsRunQueue *bound_runq;
+ erts_smp_atomic32_t state; /* Process state flags (see ERTS_PSFLG_*) */
#ifdef ERTS_SMP
erts_proc_lock_t lock;
ErtsSchedulerData *scheduler_data;
- int is_exiting;
- Uint32 runq_flags;
- Uint32 status_flags;
ErlMessageInQueue msg_inq;
Eterm suspendee;
ErtsPendingSuspend *pending_suspenders;
ErtsPendExit pending_exit;
- ErtsRunQueue *run_queue;
+ erts_smp_atomic_t run_queue;
#ifdef HIPE
struct hipe_process_state_smp hipe_smp;
#endif
@@ -822,6 +934,29 @@ void erts_check_for_holes(Process* p);
#define SEQ_TRACE_TOKEN(p) ((p)->seq_trace_token)
+#if ERTS_NO_PROC_PRIO_LEVELS > 4
+# error "Need to increase ERTS_PSFLG_PRIO_SHIFT"
+#endif
+
+#define ERTS_PSFLG_PRIO_SHIFT 2
+
+#define ERTS_PSFLG_BIT(N) \
+ (((erts_aint32_t) 1) << (ERTS_PSFLG_PRIO_SHIFT + (N)))
+
+#define ERTS_PSFLG_PRIO_MASK (ERTS_PSFLG_BIT(0) - 1)
+
+#define ERTS_PSFLG_FREE ERTS_PSFLG_BIT(0)
+#define ERTS_PSFLG_EXITING ERTS_PSFLG_BIT(1)
+#define ERTS_PSFLG_PENDING_EXIT ERTS_PSFLG_BIT(2)
+#define ERTS_PSFLG_ACTIVE ERTS_PSFLG_BIT(3)
+#define ERTS_PSFLG_IN_RUNQ ERTS_PSFLG_BIT(4)
+#define ERTS_PSFLG_RUNNING ERTS_PSFLG_BIT(5)
+#define ERTS_PSFLG_SUSPENDED ERTS_PSFLG_BIT(6)
+#define ERTS_PSFLG_GC ERTS_PSFLG_BIT(7)
+#define ERTS_PSFLG_BOUND ERTS_PSFLG_BIT(8)
+#define ERTS_PSFLG_TRAP_EXIT ERTS_PSFLG_BIT(9)
+
+
/* The sequential tracing token is a tuple of size 5:
*
* {Flags, Label, Serial, Sender}
@@ -894,13 +1029,12 @@ Eterm* erts_heap_alloc(Process* p, Uint need, Uint xtra);
Eterm* erts_set_hole_marker(Eterm* ptr, Uint sz);
#endif
-extern Process** process_tab;
+extern erts_smp_rwmtx_t erts_proc_tab_rwmtx;
+extern erts_smp_atomic_t *erts_proc_tab;
#ifdef HYBRID
extern Uint erts_num_active_procs;
extern Process** erts_active_procs;
#endif
-extern Uint erts_max_processes;
-extern Uint erts_process_tab_index_mask;
extern Uint erts_default_process_flags;
extern erts_smp_rwmtx_t erts_cpu_bind_rwmtx;
/* If any of the erts_system_monitor_* variables are set (enabled),
@@ -928,17 +1062,13 @@ struct erts_system_profile_flags_t {
unsigned int exclusive : 1;
};
extern struct erts_system_profile_flags_t erts_system_profile_flags;
-
-#define INVALID_PID(p, pid) ((p) == NULL \
- || (p)->id != (pid) \
- || (p)->status == P_EXITING)
#define IS_TRACED(p) ( (p)->tracer_proc != NIL )
#define ARE_TRACE_FLAGS_ON(p,tf) ( ((p)->trace_flags & (tf|F_SENSITIVE)) == (tf) )
#define IS_TRACED_FL(p,tf) ( IS_TRACED(p) && ARE_TRACE_FLAGS_ON(p,tf) )
/* process flags */
-#define F_TRAPEXIT (1 << 0)
+#define F_HIBERNATE_SCHED (1 << 0) /* Schedule out after hibernate op */
#define F_INSLPQUEUE (1 << 1) /* Set if in timer queue */
#define F_TIMO (1 << 2) /* Set if timeout */
#define F_HEAP_GROW (1 << 3)
@@ -949,7 +1079,6 @@ extern struct erts_system_profile_flags_t erts_system_profile_flags;
#define F_HAVE_BLCKD_MSCHED (1 << 8) /* Process has blocked multi-scheduling */
#define F_P2PNR_RESCHED (1 << 9) /* Process has been rescheduled via erts_pid2proc_not_running() */
#define F_FORCE_GC (1 << 10) /* Force gc at process in-scheduling */
-#define F_HIBERNATE_SCHED (1 << 11) /* Schedule out after hibernate op */
/* process trace_flags */
#define F_SENSITIVE (1 << 0)
@@ -1016,67 +1145,10 @@ extern struct erts_system_profile_flags_t erts_system_profile_flags;
#define DT_UTAG_FLAGS(P) ((P)->dt_utag_flags)
#endif
-
-#ifdef ERTS_SMP
-/* Status flags ... */
-#define ERTS_PROC_SFLG_PENDADD2SCHEDQ (((Uint32) 1) << 0) /* Pending
- add to
- schedule q */
-#define ERTS_PROC_SFLG_INRUNQ (((Uint32) 1) << 1) /* Process is
- in run q */
-#define ERTS_PROC_SFLG_TRAPEXIT (((Uint32) 1) << 2) /* Process is
- trapping
- exit */
-#define ERTS_PROC_SFLG_RUNNING (((Uint32) 1) << 3) /* Process is
- running */
-/* Scheduler flags in process struct... */
-#define ERTS_PROC_RUNQ_FLG_RUNNING (((Uint32) 1) << 0) /* Process is
- running */
-
-#endif
-
-
-#ifdef ERTS_SMP
-#define ERTS_PROC_IS_TRAPPING_EXITS(P) \
- (ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks((P)) \
- & ERTS_PROC_LOCK_STATUS), \
- (P)->status_flags & ERTS_PROC_SFLG_TRAPEXIT)
-
-#define ERTS_PROC_SET_TRAP_EXIT(P) \
- (ERTS_SMP_LC_ASSERT(((ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS) \
- & erts_proc_lc_my_proc_locks((P))) \
- == (ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS)), \
- (P)->status_flags |= ERTS_PROC_SFLG_TRAPEXIT, \
- (P)->flags |= F_TRAPEXIT, \
- 1)
-
-#define ERTS_PROC_UNSET_TRAP_EXIT(P) \
- (ERTS_SMP_LC_ASSERT(((ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS) \
- & erts_proc_lc_my_proc_locks((P))) \
- == (ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS)), \
- (P)->status_flags &= ~ERTS_PROC_SFLG_TRAPEXIT, \
- (P)->flags &= ~F_TRAPEXIT, \
- 0)
-#else
-#define ERTS_PROC_IS_TRAPPING_EXITS(P) ((P)->flags & F_TRAPEXIT)
-#define ERTS_PROC_SET_TRAP_EXIT(P) ((P)->flags |= F_TRAPEXIT, 1)
-#define ERTS_PROC_UNSET_TRAP_EXIT(P) ((P)->flags &= ~F_TRAPEXIT, 0)
-#endif
-
/* Option flags to erts_send_exit_signal() */
#define ERTS_XSIG_FLG_IGN_KILL (((Uint32) 1) << 0)
#define ERTS_XSIG_FLG_NO_IGN_NORMAL (((Uint32) 1) << 1)
-
-/* Process status values */
-#define P_FREE 0
-#define P_RUNABLE 1
-#define P_WAITING 2
-#define P_RUNNING 3
-#define P_EXITING 4
-#define P_GARBING 5
-#define P_SUSPENDED 6
-
#define CANCEL_TIMER(p) \
do { \
if ((p)->flags & (F_INSLPQUEUE)) \
@@ -1098,6 +1170,9 @@ void erts_early_init_scheduling(int);
void erts_init_scheduling(int, int);
Eterm erts_sched_wall_time_request(Process *c_p, int set, int enable);
+Uint64 erts_get_proc_interval(void);
+Uint64 erts_ensure_later_proc_interval(Uint64);
+Uint64 erts_step_proc_interval(void);
ErtsProcList *erts_proclist_create(Process *);
void erts_proclist_destroy(ErtsProcList *);
@@ -1154,14 +1229,6 @@ Eterm erts_get_schedulers_binds(Process *c_p);
Eterm erts_set_cpu_topology(Process *c_p, Eterm term);
Eterm erts_bind_schedulers(Process *c_p, Eterm how);
ErtsRunQueue *erts_schedid2runq(Uint);
-#ifdef ERTS_SMP
-ErtsMigrateResult erts_proc_migrate(Process *,
- ErtsProcLocks *,
- ErtsRunQueue *,
- int *,
- ErtsRunQueue *,
- int *);
-#endif
Process *schedule(Process*, int);
void erts_schedule_misc_op(void (*)(void *), void *);
Eterm erl_create_process(Process*, Eterm, Eterm, Eterm, ErlSpawnOpts*);
@@ -1210,8 +1277,7 @@ int erts_send_exit_signal(Process *,
#ifdef ERTS_SMP
void erts_handle_pending_exit(Process *, ErtsProcLocks);
#define ERTS_PROC_PENDING_EXIT(P) \
- (ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks((P)) & ERTS_PROC_LOCK_STATUS),\
- (P)->pending_exit.reason != THE_NON_VALUE)
+ (ERTS_PSFLG_PENDING_EXIT & erts_smp_atomic32_read_acqb(&(P)->state))
#else
#define ERTS_PROC_PENDING_EXIT(P) 0
#endif
@@ -1263,13 +1329,26 @@ ErtsSchedulerData *erts_get_scheduler_data(void)
#endif
#endif
+void erts_schedule_process(Process *, erts_aint32_t);
+
+ERTS_GLB_INLINE void erts_proc_notify_new_message(Process *p);
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+ERTS_GLB_INLINE void
+erts_proc_notify_new_message(Process *p)
+{
+ /* No barrier needed, due to msg lock */
+ erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state);
+ if (!(state & ERTS_PSFLG_ACTIVE))
+ erts_schedule_process(p, state);
+}
+#endif
+
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
#define ERTS_PROCESS_LOCK_ONLY_LOCK_CHECK_PROTO__
#include "erl_process_lock.h"
#undef ERTS_PROCESS_LOCK_ONLY_LOCK_CHECK_PROTO__
-int erts_smp_lc_runq_is_locked(ErtsRunQueue *);
#define ERTS_SMP_LC_CHK_RUNQ_LOCK(RQ, L) \
do { \
if ((L)) \
@@ -1395,13 +1474,91 @@ erts_proc_set_error_handler(Process *p, ErtsProcLocks plocks, Eterm handler)
#endif
+#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS
+
#ifdef ERTS_SMP
-ErtsRunQueue *erts_prepare_emigrate(ErtsRunQueue *c_rq,
- ErtsRunQueueInfo *c_rqi,
- int prio);
+#include "erl_thr_progress.h"
+
+extern erts_atomic_t erts_migration_paths;
+
+ERTS_GLB_INLINE ErtsMigrationPaths *erts_get_migration_paths_managed(void);
+ERTS_GLB_INLINE ErtsMigrationPaths *erts_get_migration_paths(void);
ERTS_GLB_INLINE ErtsRunQueue *erts_check_emigration_need(ErtsRunQueue *c_rq,
int prio);
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+ERTS_GLB_INLINE ErtsMigrationPaths *
+erts_get_migration_paths_managed(void)
+{
+ return (ErtsMigrationPaths *) erts_atomic_read_ddrb(&erts_migration_paths);
+}
+
+ERTS_GLB_INLINE ErtsMigrationPaths *
+erts_get_migration_paths(void)
+{
+ if (erts_thr_progress_is_managed_thread())
+ return erts_get_migration_paths_managed();
+ else
+ return NULL;
+}
+
+ERTS_GLB_INLINE ErtsRunQueue *
+erts_check_emigration_need(ErtsRunQueue *c_rq, int prio)
+{
+ ErtsMigrationPaths *mps = erts_get_migration_paths();
+ ErtsMigrationPath *mp;
+ Uint32 flags;
+
+ if (!mps)
+ return NULL;
+
+ mp = &mps->mpath[c_rq->ix];
+ flags = mp->flags;
+
+ if (ERTS_CHK_RUNQ_FLG_EMIGRATE(flags, prio)) {
+ int len;
+
+ if (ERTS_CHK_RUNQ_FLG_EVACUATE(flags, prio)) {
+ /* force emigration */
+ return mp->prio[prio].runq;
+ }
+
+ if (flags & ERTS_RUNQ_FLG_INACTIVE) {
+ /*
+ * Run queue was inactive at last balance. Verify that
+ * it still is before forcing emigration.
+ */
+ if (ERTS_RUNQ_FLGS_GET(c_rq) & ERTS_RUNQ_FLG_INACTIVE)
+ return mp->prio[prio].runq;
+ }
+
+
+ if (prio == ERTS_PORT_PRIO_LEVEL)
+ len = RUNQ_READ_LEN(&c_rq->ports.info.len);
+ else
+ len = RUNQ_READ_LEN(&c_rq->procs.prio_info[prio].len);
+
+ if (len > mp->prio[prio].limit.this) {
+ ErtsRunQueue *n_rq = mp->prio[prio].runq;
+ if (n_rq) {
+ if (prio == ERTS_PORT_PRIO_LEVEL)
+ len = RUNQ_READ_LEN(&n_rq->ports.info.len);
+ else
+ len = RUNQ_READ_LEN(&n_rq->procs.prio_info[prio].len);
+
+ if (len < mp->prio[prio].limit.other)
+ return n_rq;
+ }
+ }
+ }
+ return NULL;
+}
+
+#endif
+
+#endif
+
#endif
ERTS_GLB_INLINE int erts_is_scheduler_bound(ErtsSchedulerData *esdp);
@@ -1424,29 +1581,6 @@ ERTS_GLB_INLINE void erts_smp_runqs_unlock(ErtsRunQueue *rq1, ErtsRunQueue *rq2)
#if ERTS_GLB_INLINE_INCL_FUNC_DEF
-#ifdef ERTS_SMP
-ERTS_GLB_INLINE ErtsRunQueue *
-erts_check_emigration_need(ErtsRunQueue *c_rq, int prio)
-{
- ErtsRunQueueInfo *c_rqi;
-
- if (!ERTS_CHK_RUNQ_FLG_EMIGRATE(c_rq->flags, prio))
- return NULL;
-
- if (prio == ERTS_PORT_PRIO_LEVEL)
- c_rqi = &c_rq->ports.info;
- else
- c_rqi = &c_rq->procs.prio_info[prio];
-
- if (!ERTS_CHK_RUNQ_FLG_EVACUATE(c_rq->flags, prio)
- && !(c_rq->flags & ERTS_RUNQ_FLG_INACTIVE)
- && c_rqi->len <= c_rqi->migrate.limit.this)
- return NULL;
-
- return erts_prepare_emigrate(c_rq, c_rqi, prio);
-}
-#endif
-
ERTS_GLB_INLINE
int erts_is_scheduler_bound(ErtsSchedulerData *esdp)
{
@@ -1484,10 +1618,9 @@ Uint erts_get_scheduler_id(void)
ERTS_GLB_INLINE ErtsRunQueue *
erts_get_runq_proc(Process *p)
{
- ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p));
#ifdef ERTS_SMP
- ASSERT(p->run_queue);
- return p->run_queue;
+ ASSERT(ERTS_AINT_NULL != erts_atomic_read_nob(&p->run_queue));
+ return (ErtsRunQueue *) erts_atomic_read_nob(&p->run_queue);
#else
return ERTS_RUNQ_IX(0);
#endif
@@ -1658,25 +1791,13 @@ extern int erts_disable_proc_not_running_opt;
#ifdef DEBUG
#define ERTS_SMP_ASSERT_IS_NOT_EXITING(P) \
- do { ASSERT(!(P)->is_exiting); } while (0)
+ do { ASSERT(!ERTS_PROC_IS_EXITING((P))); } while (0)
#else
#define ERTS_SMP_ASSERT_IS_NOT_EXITING(P)
#endif
-/* NOTE: At least one process lock has to be held on P! */
-#ifdef ERTS_ENABLE_LOCK_CHECK
-#define ERTS_PROC_IS_EXITING(P) \
- (ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks((P)) != 0 \
- || erts_lc_pix_lock_is_locked(ERTS_PID2PIXLOCK((P)->id))),\
- (P)->is_exiting)
-#else
-#define ERTS_PROC_IS_EXITING(P) ((P)->is_exiting)
-#endif
-
#else /* !ERTS_SMP */
-#define ERTS_PROC_IS_EXITING(P) ((P)->status == P_EXITING)
-
#define ERTS_SMP_ASSERT_IS_NOT_EXITING(P)
#define erts_pid2proc_not_running erts_pid2proc
@@ -1684,6 +1805,10 @@ extern int erts_disable_proc_not_running_opt;
#endif
+#define ERTS_PROC_IS_EXITING(P) \
+ (ERTS_PSFLG_EXITING & erts_smp_atomic32_read_acqb(&(P)->state))
+
+
/* Minimum NUMBER of processes for a small system to start */
#ifdef ERTS_SMP
#define ERTS_MIN_PROCESSES ERTS_NO_OF_PIX_LOCKS
diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c
index 3550f1396c..964dc1ae3e 100644
--- a/erts/emulator/beam/erl_process_dump.c
+++ b/erts/emulator/beam/erl_process_dump.c
@@ -65,14 +65,11 @@ erts_deep_process_dump(int to, void *to_arg)
all_binaries = NULL;
for (i = 0; i < erts_max_processes; i++) {
- if ((process_tab[i] != NULL) && (process_tab[i]->i != ENULL)) {
- if (process_tab[i]->status != P_EXITING) {
- Process* p = process_tab[i];
-
- if (p->status != P_GARBING) {
- dump_process_info(to, to_arg, p);
- }
- }
+ Process *p = erts_pix2proc(i);
+ if (p && p->i != ENULL) {
+ erts_aint32_t state = erts_smp_atomic32_read_acqb(&p->state);
+ if (!(state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_GC)))
+ dump_process_info(to, to_arg, p);
}
}
diff --git a/erts/emulator/beam/erl_process_lock.c b/erts/emulator/beam/erl_process_lock.c
index b3b4601a31..bae2b383a4 100644
--- a/erts/emulator/beam/erl_process_lock.c
+++ b/erts/emulator/beam/erl_process_lock.c
@@ -67,10 +67,12 @@
#include "erl_process.h"
-const Process erts_proc_lock_busy;
+const Process erts_proc_lock_busy = {ERTS_INVALID_PID};
#ifdef ERTS_SMP
+#if ERTS_PROC_LOCK_OWN_IMPL
+
#define ERTS_PROC_LOCK_SPIN_COUNT_MAX 2000
#define ERTS_PROC_LOCK_SPIN_COUNT_SCHED_INC 32
#define ERTS_PROC_LOCK_SPIN_COUNT_BASE 1000
@@ -100,6 +102,13 @@ static erts_proc_lock_queues_t zeroqs = {0};
static erts_smp_spinlock_t qs_lock;
static erts_proc_lock_queues_t *queue_free_list;
+static int proc_lock_spin_count;
+static int aux_thr_proc_lock_spin_count;
+
+static void cleanup_tse(void);
+
+#endif /* ERTS_PROC_LOCK_OWN_IMPL */
+
#ifdef ERTS_ENABLE_LOCK_CHECK
static struct {
Sint16 proc_lock_main;
@@ -111,16 +120,11 @@ static struct {
erts_pix_lock_t erts_pix_locks[ERTS_NO_OF_PIX_LOCKS];
-static int proc_lock_spin_count;
-static int aux_thr_proc_lock_spin_count;
-
-static void cleanup_tse(void);
void
erts_init_proc_lock(int cpus)
{
int i;
- erts_smp_spinlock_init(&qs_lock, "proc_lck_qs_alloc");
for (i = 0; i < ERTS_NO_OF_PIX_LOCKS; i++) {
#ifdef ERTS_ENABLE_LOCK_COUNT
erts_mtx_init_x(&erts_pix_locks[i].u.mtx,
@@ -129,14 +133,10 @@ erts_init_proc_lock(int cpus)
erts_mtx_init(&erts_pix_locks[i].u.mtx, "pix_lock");
#endif
}
+#if ERTS_PROC_LOCK_OWN_IMPL
+ erts_smp_spinlock_init(&qs_lock, "proc_lck_qs_alloc");
queue_free_list = NULL;
erts_thr_install_exit_handler(cleanup_tse);
-#ifdef ERTS_ENABLE_LOCK_CHECK
- lc_id.proc_lock_main = erts_lc_get_lock_order_id("proc_main");
- lc_id.proc_lock_link = erts_lc_get_lock_order_id("proc_link");
- lc_id.proc_lock_msgq = erts_lc_get_lock_order_id("proc_msgq");
- lc_id.proc_lock_status = erts_lc_get_lock_order_id("proc_status");
-#endif
if (cpus > 1) {
proc_lock_spin_count = ERTS_PROC_LOCK_SPIN_COUNT_BASE;
proc_lock_spin_count += (ERTS_PROC_LOCK_SPIN_COUNT_SCHED_INC
@@ -153,8 +153,17 @@ erts_init_proc_lock(int cpus)
}
if (proc_lock_spin_count > ERTS_PROC_LOCK_SPIN_COUNT_MAX)
proc_lock_spin_count = ERTS_PROC_LOCK_SPIN_COUNT_MAX;
+#endif
+#ifdef ERTS_ENABLE_LOCK_CHECK
+ lc_id.proc_lock_main = erts_lc_get_lock_order_id("proc_main");
+ lc_id.proc_lock_link = erts_lc_get_lock_order_id("proc_link");
+ lc_id.proc_lock_msgq = erts_lc_get_lock_order_id("proc_msgq");
+ lc_id.proc_lock_status = erts_lc_get_lock_order_id("proc_status");
+#endif
}
+#if ERTS_PROC_LOCK_OWN_IMPL
+
#ifdef ERTS_ENABLE_LOCK_CHECK
static void
check_unused_tse(erts_tse_t *wtr)
@@ -220,13 +229,6 @@ tse_return(erts_tse_t *tse, int force_free_q)
erts_tse_return(tse);
}
-void
-erts_proc_lock_prepare_proc_lock_waiter(void)
-{
- tse_return(tse_fetch(NULL), 0);
-}
-
-
static void
cleanup_tse(void)
{
@@ -651,6 +653,16 @@ erts_proc_unlock_failed(Process *p,
transfer_locks(p, wait_locks, pix_lock, 1); /* unlocks pix_lock */
}
+#endif /* ERTS_PROC_LOCK_OWN_IMPL */
+
+void
+erts_proc_lock_prepare_proc_lock_waiter(void)
+{
+#if ERTS_PROC_LOCK_OWN_IMPL
+ tse_return(tse_fetch(NULL), 0);
+#endif
+}
+
/*
* proc_safelock() locks process locks on two processes. In order
* to avoid a deadlock, proc_safelock() unlocks those locks that
@@ -659,12 +671,11 @@ erts_proc_unlock_failed(Process *p,
*/
static void
-proc_safelock(Process *a_proc,
- erts_pix_lock_t *a_pix_lck,
+proc_safelock(int is_sched,
+ Process *a_proc,
ErtsProcLocks a_have_locks,
ErtsProcLocks a_need_locks,
Process *b_proc,
- erts_pix_lock_t *b_pix_lck,
ErtsProcLocks b_have_locks,
ErtsProcLocks b_need_locks)
{
@@ -672,7 +683,6 @@ proc_safelock(Process *a_proc,
#ifdef ERTS_ENABLE_LOCK_CHECK
Eterm pid1, pid2;
#endif
- erts_pix_lock_t *pix_lck1, *pix_lck2;
ErtsProcLocks need_locks1, have_locks1, need_locks2, have_locks2;
ErtsProcLocks unlock_mask;
int lock_no, refc1 = 0, refc2 = 0;
@@ -689,14 +699,12 @@ proc_safelock(Process *a_proc,
#ifdef ERTS_ENABLE_LOCK_CHECK
pid1 = a_proc->id;
#endif
- pix_lck1 = a_pix_lck;
need_locks1 = a_need_locks;
have_locks1 = a_have_locks;
p2 = b_proc;
#ifdef ERTS_ENABLE_LOCK_CHECK
pid2 = b_proc->id;
#endif
- pix_lck2 = b_pix_lck;
need_locks2 = b_need_locks;
have_locks2 = b_have_locks;
}
@@ -705,14 +713,12 @@ proc_safelock(Process *a_proc,
#ifdef ERTS_ENABLE_LOCK_CHECK
pid1 = b_proc->id;
#endif
- pix_lck1 = b_pix_lck;
need_locks1 = b_need_locks;
have_locks1 = b_have_locks;
p2 = a_proc;
#ifdef ERTS_ENABLE_LOCK_CHECK
pid2 = a_proc->id;
#endif
- pix_lck2 = a_pix_lck;
need_locks2 = a_need_locks;
have_locks2 = a_have_locks;
}
@@ -723,14 +729,12 @@ proc_safelock(Process *a_proc,
#ifdef ERTS_ENABLE_LOCK_CHECK
pid1 = a_proc->id;
#endif
- pix_lck1 = a_pix_lck;
need_locks1 = a_need_locks | b_need_locks;
have_locks1 = a_have_locks | b_have_locks;
p2 = NULL;
#ifdef ERTS_ENABLE_LOCK_CHECK
pid2 = 0;
#endif
- pix_lck2 = NULL;
need_locks2 = 0;
have_locks2 = 0;
}
@@ -740,14 +744,12 @@ proc_safelock(Process *a_proc,
#ifdef ERTS_ENABLE_LOCK_CHECK
pid1 = b_proc->id;
#endif
- pix_lck1 = b_pix_lck;
need_locks1 = b_need_locks;
have_locks1 = b_have_locks;
p2 = NULL;
#ifdef ERTS_ENABLE_LOCK_CHECK
pid2 = 0;
#endif
- pix_lck2 = NULL;
need_locks2 = 0;
have_locks2 = 0;
#ifdef ERTS_ENABLE_LOCK_CHECK
@@ -795,21 +797,21 @@ proc_safelock(Process *a_proc,
if (unlock_locks) {
have_locks1 &= ~unlock_locks;
need_locks1 |= unlock_locks;
- if (!have_locks1) {
+ if (!is_sched && !have_locks1) {
refc1 = 1;
erts_smp_proc_inc_refc(p1);
}
- erts_smp_proc_unlock__(p1, pix_lck1, unlock_locks);
+ erts_smp_proc_unlock(p1, unlock_locks);
}
unlock_locks = unlock_mask & have_locks2;
if (unlock_locks) {
have_locks2 &= ~unlock_locks;
need_locks2 |= unlock_locks;
- if (!have_locks2) {
+ if (!is_sched && !have_locks2) {
refc2 = 1;
erts_smp_proc_inc_refc(p2);
}
- erts_smp_proc_unlock__(p2, pix_lck2, unlock_locks);
+ erts_smp_proc_unlock(p2, unlock_locks);
}
}
@@ -840,7 +842,7 @@ proc_safelock(Process *a_proc,
if (need_locks2 & lock)
lock_no--;
locks = need_locks1 & lock_mask;
- erts_smp_proc_lock__(p1, pix_lck1, locks);
+ erts_smp_proc_lock(p1, locks);
have_locks1 |= locks;
need_locks1 &= ~locks;
}
@@ -851,7 +853,7 @@ proc_safelock(Process *a_proc,
lock = (1 << ++lock_no);
}
locks = need_locks2 & lock_mask;
- erts_smp_proc_lock__(p2, pix_lck2, locks);
+ erts_smp_proc_lock(p2, locks);
have_locks2 |= locks;
need_locks2 &= ~locks;
}
@@ -886,10 +888,12 @@ proc_safelock(Process *a_proc,
}
#endif
- if (refc1)
- erts_smp_proc_dec_refc(p1);
- if (refc2)
- erts_smp_proc_dec_refc(p2);
+ if (!is_sched) {
+ if (refc1)
+ erts_smp_proc_dec_refc(p1);
+ if (refc2)
+ erts_smp_proc_dec_refc(p2);
+ }
}
void
@@ -900,77 +904,191 @@ erts_proc_safelock(Process *a_proc,
ErtsProcLocks b_have_locks,
ErtsProcLocks b_need_locks)
{
- proc_safelock(a_proc,
- a_proc ? ERTS_PID2PIXLOCK(a_proc->id) : NULL,
+ proc_safelock(erts_get_scheduler_id() != 0,
+ a_proc,
a_have_locks,
a_need_locks,
b_proc,
- b_proc ? ERTS_PID2PIXLOCK(b_proc->id) : NULL,
b_have_locks,
b_need_locks);
}
-/*
- * erts_pid2proc_safelock() is called from erts_pid2proc_opt() when
- * it wasn't possible to trylock all locks needed.
- * c_p - current process
- * c_p_have_locks - locks held on c_p
- * pid - process id of process we are looking up
- * proc - process struct of process we are looking
- * up (both in and out argument)
- * need_locks - all locks we need (including have_locks)
- * pix_lock - pix lock for process we are looking up
- * flags - option flags
- */
-void
-erts_pid2proc_safelock(Process *c_p,
- ErtsProcLocks c_p_have_locks,
- Process **proc,
- ErtsProcLocks need_locks,
- erts_pix_lock_t *pix_lock,
- int flags)
+Process *
+erts_pid2proc_opt(Process *c_p,
+ ErtsProcLocks c_p_have_locks,
+ Eterm pid,
+ ErtsProcLocks pid_need_locks,
+ int flags)
{
- Process *p = *proc;
- ERTS_LC_ASSERT(p->lock.refc > 0);
- ERTS_LC_ASSERT(process_tab[internal_pid_index(p->id)] == p);
- p->lock.refc++;
- erts_pix_unlock(pix_lock);
-
- proc_safelock(c_p,
- c_p ? ERTS_PID2PIXLOCK(c_p->id) : NULL,
- c_p_have_locks,
- c_p_have_locks,
- p,
- pix_lock,
- 0,
- need_locks);
+ Process *dec_refc_proc = NULL;
+ int need_ptl;
+ ErtsProcLocks need_locks;
+ Uint pix;
+ Process *proc;
+#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT)
+ ErtsProcLocks lcnt_locks;
+#endif
- erts_pix_lock(pix_lock);
+#ifdef ERTS_ENABLE_LOCK_CHECK
+ if (c_p) {
+ ErtsProcLocks might_unlock = c_p_have_locks & pid_need_locks;
+ if (might_unlock)
+ erts_proc_lc_might_unlock(c_p, might_unlock);
+ }
+#endif
- if (!p->is_exiting
- || ((flags & ERTS_P2P_FLG_ALLOW_OTHER_X)
- && process_tab[internal_pid_index(p->id)] == p)) {
- ERTS_LC_ASSERT(p->lock.refc > 1);
- p->lock.refc--;
+ if (is_not_internal_pid(pid))
+ return NULL;
+ pix = internal_pid_index(pid);
+
+ ERTS_LC_ASSERT((pid_need_locks & ERTS_PROC_LOCKS_ALL) == pid_need_locks);
+ need_locks = pid_need_locks;
+
+ if (c_p && c_p->id == pid) {
+ ASSERT(c_p->id != ERTS_INVALID_PID);
+ ASSERT(c_p == erts_pix2proc(pix));
+
+ if (!(flags & ERTS_P2P_FLG_ALLOW_OTHER_X)
+ && ERTS_PROC_IS_EXITING(c_p))
+ return NULL;
+ need_locks &= ~c_p_have_locks;
+ if (!need_locks) {
+ if (flags & ERTS_P2P_FLG_SMP_INC_REFC)
+ erts_smp_proc_inc_refc(c_p);
+ return c_p;
+ }
}
- else {
- /* No proc. Note, we need to keep refc until after process unlock */
- erts_pix_unlock(pix_lock);
- erts_smp_proc_unlock__(p, pix_lock, need_locks);
- *proc = NULL;
- erts_pix_lock(pix_lock);
- ERTS_LC_ASSERT(p->lock.refc > 0);
- if (--p->lock.refc == 0) {
- erts_pix_unlock(pix_lock);
- erts_free_proc(p);
- erts_pix_lock(pix_lock);
+
+ need_ptl = !erts_get_scheduler_id();
+
+ if (need_ptl)
+ erts_smp_rwmtx_rwlock(&erts_proc_tab_rwmtx);
+
+ proc = (Process *) erts_smp_atomic_read_ddrb(&erts_proc.tab[pix]);
+
+ if (proc) {
+ if (proc->id != pid)
+ proc = NULL;
+ else if (!need_locks) {
+ if (flags & ERTS_P2P_FLG_SMP_INC_REFC)
+ erts_smp_proc_inc_refc(proc);
}
+ else {
+ int busy;
+
+#if ERTS_PROC_LOCK_OWN_IMPL
+#ifdef ERTS_ENABLE_LOCK_COUNT
+ lcnt_locks = need_locks;
+ if (!(flags & ERTS_P2P_FLG_TRY_LOCK)) {
+ erts_lcnt_proc_lock(&proc->lock, need_locks);
+ }
+#endif
+
+#ifdef ERTS_ENABLE_LOCK_CHECK
+ /* Make sure erts_pid2proc_safelock() is enough to handle
+ a potential lock order violation situation... */
+ busy = erts_proc_lc_trylock_force_busy(proc, need_locks);
+ if (!busy)
+#endif
+#endif /* ERTS_PROC_LOCK_OWN_IMPL */
+ {
+ /* Try a quick trylock to grab all the locks we need. */
+ busy = (int) erts_smp_proc_raw_trylock__(proc, need_locks);
+
+#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_CHECK)
+ erts_proc_lc_trylock(proc, need_locks, !busy);
+#endif
+#ifdef ERTS_PROC_LOCK_DEBUG
+ if (!busy)
+ erts_proc_lock_op_debug(proc, need_locks, 1);
+#endif
+ }
+
+#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT)
+ if (flags & ERTS_P2P_FLG_TRY_LOCK)
+ erts_lcnt_proc_trylock(&proc->lock, need_locks,
+ busy ? EBUSY : 0);
+#endif
+
+ if (!busy) {
+ if (flags & ERTS_P2P_FLG_SMP_INC_REFC)
+ erts_smp_proc_inc_refc(proc);
+
+#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT)
+ /* all is great */
+ if (!(flags & ERTS_P2P_FLG_TRY_LOCK))
+ erts_lcnt_proc_lock_post_x(&proc->lock, lcnt_locks,
+ __FILE__, __LINE__);
+#endif
+
+ }
+ else {
+ if (flags & ERTS_P2P_FLG_TRY_LOCK)
+ proc = ERTS_PROC_LOCK_BUSY;
+ else {
+ if (flags & ERTS_P2P_FLG_SMP_INC_REFC)
+ erts_smp_proc_inc_refc(proc);
+
+#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT)
+ erts_lcnt_proc_lock_unaquire(&proc->lock, lcnt_locks);
+#endif
+
+ if (need_ptl) {
+ erts_smp_proc_inc_refc(proc);
+ dec_refc_proc = proc;
+ erts_smp_rwmtx_rwunlock(&erts_proc_tab_rwmtx);
+ need_ptl = 0;
+ }
+
+ proc_safelock(!need_ptl,
+ c_p,
+ c_p_have_locks,
+ c_p_have_locks,
+ proc,
+ 0,
+ need_locks);
+ }
+ }
+ }
}
+
+ if (need_ptl)
+ erts_smp_rwmtx_rwunlock(&erts_proc_tab_rwmtx);
+
+ if (need_locks
+ && proc
+ && proc != ERTS_PROC_LOCK_BUSY
+ && (!(flags & ERTS_P2P_FLG_ALLOW_OTHER_X)
+ ? ERTS_PROC_IS_EXITING(proc)
+ : (proc
+ != (Process *) erts_smp_atomic_read_nob(&erts_proc.tab[pix])))) {
+
+ erts_smp_proc_unlock(proc, need_locks);
+
+ if (flags & ERTS_P2P_FLG_SMP_INC_REFC)
+ dec_refc_proc = proc;
+ proc = NULL;
+
+ }
+
+ if (dec_refc_proc)
+ erts_smp_proc_dec_refc(dec_refc_proc);
+
+#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_PROC_LOCK_DEBUG)
+ ERTS_LC_ASSERT(!proc
+ || proc == ERTS_PROC_LOCK_BUSY
+ || (pid_need_locks ==
+ (ERTS_PROC_LOCK_FLGS_READ_(&proc->lock)
+ & pid_need_locks)));
+#endif
+
+ return proc;
}
void
erts_proc_lock_init(Process *p)
{
+#if ERTS_PROC_LOCK_OWN_IMPL
/* We always start with all locks locked */
#if ERTS_PROC_LOCK_ATOMIC_IMPL
erts_smp_atomic32_init_nob(&p->lock.flags,
@@ -979,16 +1097,32 @@ erts_proc_lock_init(Process *p)
p->lock.flags = ERTS_PROC_LOCKS_ALL;
#endif
p->lock.queues = NULL;
- p->lock.refc = 1;
-#ifdef ERTS_ENABLE_LOCK_COUNT
- erts_lcnt_proc_lock_init(p);
- erts_lcnt_proc_lock(&(p->lock), ERTS_PROC_LOCKS_ALL);
- erts_lcnt_proc_lock_post_x(&(p->lock), ERTS_PROC_LOCKS_ALL, __FILE__, __LINE__);
-#endif
-
#ifdef ERTS_ENABLE_LOCK_CHECK
erts_proc_lc_trylock(p, ERTS_PROC_LOCKS_ALL, 1);
#endif
+#elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
+ erts_mtx_init_x(&p->lock.main, "proc_main", p->id);
+ ethr_mutex_lock(&p->lock.main.mtx);
+#ifdef ERTS_ENABLE_LOCK_CHECK
+ erts_lc_trylock(1, &p->lock.main.lc);
+#endif
+ erts_mtx_init_x(&p->lock.link, "proc_link", p->id);
+ ethr_mutex_lock(&p->lock.link.mtx);
+#ifdef ERTS_ENABLE_LOCK_CHECK
+ erts_lc_trylock(1, &p->lock.link.lc);
+#endif
+ erts_mtx_init_x(&p->lock.msgq, "proc_msgq", p->id);
+ ethr_mutex_lock(&p->lock.msgq.mtx);
+#ifdef ERTS_ENABLE_LOCK_CHECK
+ erts_lc_trylock(1, &p->lock.msgq.lc);
+#endif
+ erts_mtx_init_x(&p->lock.status, "proc_status", p->id);
+ ethr_mutex_lock(&p->lock.status.mtx);
+#ifdef ERTS_ENABLE_LOCK_CHECK
+ erts_lc_trylock(1, &p->lock.status.lc);
+#endif
+#endif
+ erts_atomic32_init_nob(&p->lock.refc, 1);
#ifdef ERTS_PROC_LOCK_DEBUG
{
int i;
@@ -996,11 +1130,30 @@ erts_proc_lock_init(Process *p)
erts_smp_atomic32_init_nob(&p->lock.locked[i], (erts_aint32_t) 1);
}
#endif
+#ifdef ERTS_ENABLE_LOCK_COUNT
+ erts_lcnt_proc_lock_init(p);
+ erts_lcnt_proc_lock(&(p->lock), ERTS_PROC_LOCKS_ALL);
+ erts_lcnt_proc_lock_post_x(&(p->lock), ERTS_PROC_LOCKS_ALL, __FILE__, __LINE__);
+#endif
+}
+
+void
+erts_proc_lock_fin(Process *p)
+{
+#if ERTS_PROC_LOCK_RAW_MUTEX_IMPL
+ erts_mtx_destroy(&p->lock.main);
+ erts_mtx_destroy(&p->lock.link);
+ erts_mtx_destroy(&p->lock.msgq);
+ erts_mtx_destroy(&p->lock.status);
+#endif
+#if defined(ERTS_ENABLE_LOCK_COUNT) && defined(ERTS_SMP)
+ erts_lcnt_proc_lock_destroy(p);
+#endif
}
/* --- Process lock counting ----------------------------------------------- */
-#ifdef ERTS_ENABLE_LOCK_COUNT
+#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT)
void erts_lcnt_proc_lock_init(Process *p) {
if (erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK) {
if (p->id != ERTS_INVALID_PID) {
@@ -1141,6 +1294,8 @@ void erts_lcnt_enable_proc_lock_count(int enable) {
#ifdef ERTS_ENABLE_LOCK_CHECK
+#if ERTS_PROC_LOCK_OWN_IMPL
+
void
erts_proc_lc_lock(Process *p, ErtsProcLocks locks)
{
@@ -1213,9 +1368,12 @@ erts_proc_lc_unlock(Process *p, ErtsProcLocks locks)
}
}
+#endif /* ERTS_PROC_LOCK_OWN_IMPL */
+
void
erts_proc_lc_might_unlock(Process *p, ErtsProcLocks locks)
{
+#if ERTS_PROC_LOCK_OWN_IMPL
erts_lc_lock_t lck = ERTS_LC_LOCK_INIT(-1,
p->id,
ERTS_LC_FLG_LT_PROCLOCK);
@@ -1235,11 +1393,22 @@ erts_proc_lc_might_unlock(Process *p, ErtsProcLocks locks)
lck.id = lc_id.proc_lock_main;
erts_lc_might_unlock(&lck);
}
+#elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
+ if (locks & ERTS_PROC_LOCK_MAIN)
+ erts_lc_might_unlock(&p->lock.main.lc);
+ if (locks & ERTS_PROC_LOCK_LINK)
+ erts_lc_might_unlock(&p->lock.link.lc);
+ if (locks & ERTS_PROC_LOCK_MSGQ)
+ erts_lc_might_unlock(&p->lock.msgq.lc);
+ if (locks & ERTS_PROC_LOCK_STATUS)
+ erts_lc_might_unlock(&p->lock.status.lc);
+#endif
}
void
erts_proc_lc_require_lock(Process *p, ErtsProcLocks locks)
{
+#if ERTS_PROC_LOCK_OWN_IMPL
erts_lc_lock_t lck = ERTS_LC_LOCK_INIT(-1,
p->id,
ERTS_LC_FLG_LT_PROCLOCK);
@@ -1259,11 +1428,22 @@ erts_proc_lc_require_lock(Process *p, ErtsProcLocks locks)
lck.id = lc_id.proc_lock_status;
erts_lc_require_lock(&lck);
}
+#elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
+ if (locks & ERTS_PROC_LOCK_MAIN)
+ erts_lc_require_lock(&p->lock.main.lc);
+ if (locks & ERTS_PROC_LOCK_LINK)
+ erts_lc_require_lock(&p->lock.link.lc);
+ if (locks & ERTS_PROC_LOCK_MSGQ)
+ erts_lc_require_lock(&p->lock.msgq.lc);
+ if (locks & ERTS_PROC_LOCK_STATUS)
+ erts_lc_require_lock(&p->lock.status.lc);
+#endif
}
void
erts_proc_lc_unrequire_lock(Process *p, ErtsProcLocks locks)
{
+#if ERTS_PROC_LOCK_OWN_IMPL
erts_lc_lock_t lck = ERTS_LC_LOCK_INIT(-1,
p->id,
ERTS_LC_FLG_LT_PROCLOCK);
@@ -1283,8 +1463,19 @@ erts_proc_lc_unrequire_lock(Process *p, ErtsProcLocks locks)
lck.id = lc_id.proc_lock_main;
erts_lc_unrequire_lock(&lck);
}
+#elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
+ if (locks & ERTS_PROC_LOCK_MAIN)
+ erts_lc_unrequire_lock(&p->lock.main.lc);
+ if (locks & ERTS_PROC_LOCK_LINK)
+ erts_lc_unrequire_lock(&p->lock.link.lc);
+ if (locks & ERTS_PROC_LOCK_MSGQ)
+ erts_lc_unrequire_lock(&p->lock.msgq.lc);
+ if (locks & ERTS_PROC_LOCK_STATUS)
+ erts_lc_unrequire_lock(&p->lock.status.lc);
+#endif
}
+#if ERTS_PROC_LOCK_OWN_IMPL
int
erts_proc_lc_trylock_force_busy(Process *p, ErtsProcLocks locks)
@@ -1310,21 +1501,30 @@ erts_proc_lc_trylock_force_busy(Process *p, ErtsProcLocks locks)
return 0;
}
+#endif /* ERTS_PROC_LOCK_OWN_IMPL */
+
void erts_proc_lc_chk_only_proc_main(Process *p)
{
+#if ERTS_PROC_LOCK_OWN_IMPL
erts_lc_lock_t proc_main = ERTS_LC_LOCK_INIT(lc_id.proc_lock_main,
p->id,
ERTS_LC_FLG_LT_PROCLOCK);
erts_lc_check_exact(&proc_main, 1);
+#elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
+ erts_lc_check_exact(&p->lock.main.lc, 1);
+#endif
}
+#if ERTS_PROC_LOCK_OWN_IMPL
#define ERTS_PROC_LC_EMPTY_LOCK_INIT \
ERTS_LC_LOCK_INIT(-1, THE_NON_VALUE, ERTS_LC_FLG_LT_PROCLOCK)
+#endif /* ERTS_PROC_LOCK_OWN_IMPL */
void
erts_proc_lc_chk_have_proc_locks(Process *p, ErtsProcLocks locks)
{
int have_locks_len = 0;
+#if ERTS_PROC_LOCK_OWN_IMPL
erts_lc_lock_t have_locks[4] = {ERTS_PROC_LC_EMPTY_LOCK_INIT,
ERTS_PROC_LC_EMPTY_LOCK_INIT,
ERTS_PROC_LC_EMPTY_LOCK_INIT,
@@ -1345,7 +1545,17 @@ erts_proc_lc_chk_have_proc_locks(Process *p, ErtsProcLocks locks)
have_locks[have_locks_len].id = lc_id.proc_lock_status;
have_locks[have_locks_len++].extra = p->id;
}
-
+#elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
+ erts_lc_lock_t have_locks[4];
+ if (locks & ERTS_PROC_LOCK_MAIN)
+ have_locks[have_locks_len++] = p->lock.main.lc;
+ if (locks & ERTS_PROC_LOCK_LINK)
+ have_locks[have_locks_len++] = p->lock.link.lc;
+ if (locks & ERTS_PROC_LOCK_MSGQ)
+ have_locks[have_locks_len++] = p->lock.msgq.lc;
+ if (locks & ERTS_PROC_LOCK_STATUS)
+ have_locks[have_locks_len++] = p->lock.status.lc;
+#endif
erts_lc_check(have_locks, have_locks_len, NULL, 0);
}
@@ -1354,6 +1564,7 @@ erts_proc_lc_chk_proc_locks(Process *p, ErtsProcLocks locks)
{
int have_locks_len = 0;
int have_not_locks_len = 0;
+#if ERTS_PROC_LOCK_OWN_IMPL
erts_lc_lock_t have_locks[4] = {ERTS_PROC_LC_EMPTY_LOCK_INIT,
ERTS_PROC_LC_EMPTY_LOCK_INIT,
ERTS_PROC_LC_EMPTY_LOCK_INIT,
@@ -1395,6 +1606,27 @@ erts_proc_lc_chk_proc_locks(Process *p, ErtsProcLocks locks)
have_not_locks[have_not_locks_len].id = lc_id.proc_lock_status;
have_not_locks[have_not_locks_len++].extra = p->id;
}
+#elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
+ erts_lc_lock_t have_locks[4];
+ erts_lc_lock_t have_not_locks[4];
+
+ if (locks & ERTS_PROC_LOCK_MAIN)
+ have_locks[have_locks_len++] = p->lock.main.lc;
+ else
+ have_not_locks[have_not_locks_len++] = p->lock.main.lc;
+ if (locks & ERTS_PROC_LOCK_LINK)
+ have_locks[have_locks_len++] = p->lock.link.lc;
+ else
+ have_not_locks[have_not_locks_len++] = p->lock.link.lc;
+ if (locks & ERTS_PROC_LOCK_MSGQ)
+ have_locks[have_locks_len++] = p->lock.msgq.lc;
+ else
+ have_not_locks[have_not_locks_len++] = p->lock.msgq.lc;
+ if (locks & ERTS_PROC_LOCK_STATUS)
+ have_locks[have_locks_len++] = p->lock.status.lc;
+ else
+ have_not_locks[have_not_locks_len++] = p->lock.status.lc;
+#endif
erts_lc_check(have_locks, have_locks_len,
have_not_locks, have_not_locks_len);
@@ -1404,6 +1636,8 @@ ErtsProcLocks
erts_proc_lc_my_proc_locks(Process *p)
{
int resv[4];
+ ErtsProcLocks res = 0;
+#if ERTS_PROC_LOCK_OWN_IMPL
erts_lc_lock_t locks[4] = {ERTS_LC_LOCK_INIT(lc_id.proc_lock_main,
p->id,
ERTS_LC_FLG_LT_PROCLOCK),
@@ -1416,8 +1650,12 @@ erts_proc_lc_my_proc_locks(Process *p)
ERTS_LC_LOCK_INIT(lc_id.proc_lock_status,
p->id,
ERTS_LC_FLG_LT_PROCLOCK)};
-
- ErtsProcLocks res = 0;
+#elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
+ erts_lc_lock_t locks[4] = {p->lock.main.lc,
+ p->lock.link.lc,
+ p->lock.msgq.lc,
+ p->lock.status.lc};
+#endif
erts_lc_have_locks(resv, locks, 4);
if (resv[0])
@@ -1450,7 +1688,7 @@ erts_proc_lc_chk_no_proc_locks(char *file, int line)
#endif /* #ifdef ERTS_ENABLE_LOCK_CHECK */
-#ifdef ERTS_PROC_LOCK_HARD_DEBUG
+#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_PROC_LOCK_HARD_DEBUG)
void
check_queue(erts_proc_lock_t *lck)
{
@@ -1483,4 +1721,4 @@ check_queue(erts_proc_lock_t *lck)
}
#endif
-#endif /* ERTS_SMP (the whole file) */
+#endif /* ERTS_SMP */
diff --git a/erts/emulator/beam/erl_process_lock.h b/erts/emulator/beam/erl_process_lock.h
index 413c45480c..7367e03b44 100644
--- a/erts/emulator/beam/erl_process_lock.h
+++ b/erts/emulator/beam/erl_process_lock.h
@@ -37,10 +37,21 @@
#include "erl_smp.h"
+#if defined(VALGRIND) || defined(ETHR_DISABLE_NATIVE_IMPLS)
+# define ERTS_PROC_LOCK_OWN_IMPL 0
+#else
+# define ERTS_PROC_LOCK_OWN_IMPL 1
+#endif
+
#define ERTS_PROC_LOCK_ATOMIC_IMPL 0
#define ERTS_PROC_LOCK_SPINLOCK_IMPL 0
#define ERTS_PROC_LOCK_MUTEX_IMPL 0
+#if !ERTS_PROC_LOCK_OWN_IMPL
+#define ERTS_PROC_LOCK_RAW_MUTEX_IMPL 1
+#else
+#define ERTS_PROC_LOCK_RAW_MUTEX_IMPL 0
+
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
# undef ERTS_PROC_LOCK_ATOMIC_IMPL
# define ERTS_PROC_LOCK_ATOMIC_IMPL 1
@@ -52,6 +63,8 @@
# define ERTS_PROC_LOCK_MUTEX_IMPL 1
#endif
+#endif
+
#define ERTS_PROC_LOCK_MAX_BIT 3
typedef erts_aint32_t ErtsProcLocks;
@@ -59,22 +72,31 @@ typedef erts_aint32_t ErtsProcLocks;
typedef struct erts_proc_lock_queues_t_ erts_proc_lock_queues_t;
typedef struct erts_proc_lock_t_ {
+#if ERTS_PROC_LOCK_OWN_IMPL
#if ERTS_PROC_LOCK_ATOMIC_IMPL
erts_smp_atomic32_t flags;
#else
ErtsProcLocks flags;
#endif
erts_proc_lock_queues_t *queues;
- Sint32 refc;
-#ifdef ERTS_PROC_LOCK_DEBUG
- erts_smp_atomic32_t locked[ERTS_PROC_LOCK_MAX_BIT+1];
-#endif
#ifdef ERTS_ENABLE_LOCK_COUNT
erts_lcnt_lock_t lcnt_main;
erts_lcnt_lock_t lcnt_link;
erts_lcnt_lock_t lcnt_msgq;
erts_lcnt_lock_t lcnt_status;
#endif
+#elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
+ erts_mtx_t main;
+ erts_mtx_t link;
+ erts_mtx_t msgq;
+ erts_mtx_t status;
+#else
+# error "no implementation"
+#endif
+ erts_atomic32_t refc;
+#ifdef ERTS_PROC_LOCK_DEBUG
+ erts_smp_atomic32_t locked[ERTS_PROC_LOCK_MAX_BIT+1];
+#endif
} erts_proc_lock_t;
/* Process lock flags */
@@ -107,11 +129,9 @@ typedef struct erts_proc_lock_t_ {
/*
* Status lock:
* Protects the following fields in the process structure:
- * * status
- * * rstatus
- * * status_flags
* * pending_suspenders
* * suspendee
+ * * ...
*/
#define ERTS_PROC_LOCK_STATUS (((ErtsProcLocks) 1) << ERTS_PROC_LOCK_MAX_BIT)
@@ -143,14 +163,11 @@ typedef struct erts_proc_lock_t_ {
* Other rules regarding process locking:
*
* Exiting processes:
- * When changing status to P_EXITING on a process, you are required
- * to take all process locks (ERTS_PROC_LOCKS_ALL). Thus, by holding
- * at least one process lock (whichever one doesn't matter) you
- * are guaranteed that the process won't exit until the lock you are
- * holding has been released. Appart from all process locks also
- * the pix lock corresponding to the process has to be held.
- * At the same time as status is changed to P_EXITING, also the
- * field 'is_exiting' in the process structure is set to a value != 0.
+ * When changing state to exiting (ERTS_PSFLG_EXITING) on a process,
+ * you are required to take all process locks (ERTS_PROC_LOCKS_ALL).
+ * Thus, by holding at least one process lock (whichever one doesn't
+ * matter) you are guaranteed that the process won't exit until the
+ * lock you are holding has been released.
*
* Lock order:
* Process locks with low numeric values has to be locked before
@@ -262,12 +279,10 @@ typedef struct {
} u;
} erts_pix_lock_t;
-#define ERTS_PIX2PIXLOCKIX(PIX) \
- ((PIX) & ((1 << ERTS_PIX_LOCKS_BITS) - 1))
-#define ERTS_PIX2PIXLOCK(PIX) \
- (&erts_pix_locks[ERTS_PIX2PIXLOCKIX((PIX))])
#define ERTS_PID2PIXLOCK(PID) \
- ERTS_PIX2PIXLOCK(internal_pid_data((PID)))
+ (&erts_pix_locks[(internal_pid_data((PID)) & ((1 << ERTS_PIX_LOCKS_BITS) - 1))])
+
+#if ERTS_PROC_LOCK_OWN_IMPL
#if ERTS_PROC_LOCK_ATOMIC_IMPL
@@ -337,11 +352,13 @@ erts_proc_lock_flags_cmpxchg(erts_proc_lock_t *lck, ErtsProcLocks new,
#define ERTS_PROC_LOCK_FLGS_READ_(L) ((L)->flags)
#endif /* end no opt atomic ops */
+#endif /* ERTS_PROC_LOCK_OWN_IMPL */
extern erts_pix_lock_t erts_pix_locks[ERTS_NO_OF_PIX_LOCKS];
void erts_init_proc_lock(int cpus);
void erts_proc_lock_prepare_proc_lock_waiter(void);
+#if ERTS_PROC_LOCK_OWN_IMPL
void erts_proc_lock_failed(Process *,
erts_pix_lock_t *,
ErtsProcLocks,
@@ -349,6 +366,7 @@ void erts_proc_lock_failed(Process *,
void erts_proc_unlock_failed(Process *,
erts_pix_lock_t *,
ErtsProcLocks);
+#endif
ERTS_GLB_INLINE void erts_pix_lock(erts_pix_lock_t *);
ERTS_GLB_INLINE void erts_pix_unlock(erts_pix_lock_t *);
@@ -412,6 +430,7 @@ ERTS_GLB_INLINE int erts_lc_pix_lock_is_locked(erts_pix_lock_t *pixlck)
ERTS_GLB_INLINE ErtsProcLocks
erts_smp_proc_raw_trylock__(Process *p, ErtsProcLocks locks)
{
+#if ERTS_PROC_LOCK_OWN_IMPL
ErtsProcLocks expct_lflgs = 0;
while (1) {
@@ -431,8 +450,38 @@ erts_smp_proc_raw_trylock__(Process *p, ErtsProcLocks locks)
/* cmpxchg failed, try again (should be rare). */
expct_lflgs = lflgs;
}
-}
+#elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
+
+ if (locks & ERTS_PROC_LOCK_MAIN)
+ if (erts_mtx_trylock(&p->lock.main) == EBUSY)
+ goto busy_main;
+ if (locks & ERTS_PROC_LOCK_LINK)
+ if (erts_mtx_trylock(&p->lock.link) == EBUSY)
+ goto busy_link;
+ if (locks & ERTS_PROC_LOCK_MSGQ)
+ if (erts_mtx_trylock(&p->lock.msgq) == EBUSY)
+ goto busy_msgq;
+ if (locks & ERTS_PROC_LOCK_STATUS)
+ if (erts_mtx_trylock(&p->lock.status) == EBUSY)
+ goto busy_status;
+
+ return 0;
+
+busy_status:
+ if (locks & ERTS_PROC_LOCK_MSGQ)
+ erts_mtx_unlock(&p->lock.msgq);
+busy_msgq:
+ if (locks & ERTS_PROC_LOCK_LINK)
+ erts_mtx_unlock(&p->lock.link);
+busy_link:
+ if (locks & ERTS_PROC_LOCK_MAIN)
+ erts_mtx_unlock(&p->lock.main);
+busy_main:
+
+ return EBUSY;
+#endif
+}
ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_COUNT
@@ -446,10 +495,13 @@ erts_smp_proc_lock__(Process *p,
ErtsProcLocks locks)
#endif
{
+#if ERTS_PROC_LOCK_OWN_IMPL
+
ErtsProcLocks old_lflgs;
#if !ERTS_PROC_LOCK_ATOMIC_IMPL
erts_pix_lock(pix_lck);
#endif
+
#ifdef ERTS_ENABLE_LOCK_COUNT
erts_lcnt_proc_lock(&(p->lock), locks);
#endif
@@ -473,12 +525,14 @@ erts_smp_proc_lock__(Process *p,
erts_pix_unlock(pix_lck);
}
#endif
+
#ifdef ERTS_ENABLE_LOCK_COUNT
erts_lcnt_proc_lock_post_x(&(p->lock), locks, file, line);
#endif
#ifdef ERTS_ENABLE_LOCK_CHECK
erts_proc_lc_lock(p, locks);
#endif
+
#ifdef ERTS_PROC_LOCK_DEBUG
erts_proc_lock_op_debug(p, locks, 1);
#endif
@@ -486,6 +540,22 @@ erts_smp_proc_lock__(Process *p,
#if ERTS_PROC_LOCK_ATOMIC_IMPL
ETHR_COMPILER_BARRIER;
#endif
+
+#elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
+ if (locks & ERTS_PROC_LOCK_MAIN)
+ erts_mtx_lock(&p->lock.main);
+ if (locks & ERTS_PROC_LOCK_LINK)
+ erts_mtx_lock(&p->lock.link);
+ if (locks & ERTS_PROC_LOCK_MSGQ)
+ erts_mtx_lock(&p->lock.msgq);
+ if (locks & ERTS_PROC_LOCK_STATUS)
+ erts_mtx_lock(&p->lock.status);
+
+#ifdef ERTS_PROC_LOCK_DEBUG
+ erts_proc_lock_op_debug(p, locks, 1);
+#endif
+
+#endif
}
ERTS_GLB_INLINE void
@@ -493,6 +563,7 @@ erts_smp_proc_unlock__(Process *p,
erts_pix_lock_t *pix_lck,
ErtsProcLocks locks)
{
+#if ERTS_PROC_LOCK_OWN_IMPL
ErtsProcLocks old_lflgs;
#if ERTS_PROC_LOCK_ATOMIC_IMPL
@@ -557,6 +628,23 @@ erts_smp_proc_unlock__(Process *p,
break;
}
+
+#elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
+
+#ifdef ERTS_PROC_LOCK_DEBUG
+ erts_proc_lock_op_debug(p, locks, 0);
+#endif
+
+ if (locks & ERTS_PROC_LOCK_STATUS)
+ erts_mtx_unlock(&p->lock.status);
+ if (locks & ERTS_PROC_LOCK_MSGQ)
+ erts_mtx_unlock(&p->lock.msgq);
+ if (locks & ERTS_PROC_LOCK_LINK)
+ erts_mtx_unlock(&p->lock.link);
+ if (locks & ERTS_PROC_LOCK_MAIN)
+ erts_mtx_unlock(&p->lock.main);
+#endif
+
}
ERTS_GLB_INLINE int
@@ -564,6 +652,7 @@ erts_smp_proc_trylock__(Process *p,
erts_pix_lock_t *pix_lck,
ErtsProcLocks locks)
{
+#if ERTS_PROC_LOCK_OWN_IMPL
int res;
#ifdef ERTS_ENABLE_LOCK_CHECK
@@ -575,6 +664,7 @@ erts_smp_proc_trylock__(Process *p,
else
#endif
{
+
#if !ERTS_PROC_LOCK_ATOMIC_IMPL
erts_pix_lock(pix_lck);
#endif
@@ -613,8 +703,18 @@ erts_smp_proc_trylock__(Process *p,
#if ERTS_PROC_LOCK_ATOMIC_IMPL
ETHR_COMPILER_BARRIER;
#endif
-
return res;
+
+#elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
+ if (erts_smp_proc_raw_trylock__(p, locks) != 0)
+ return EBUSY;
+ else {
+#ifdef ERTS_PROC_LOCK_DEBUG
+ erts_proc_lock_op_debug(p, locks, 1);
+#endif
+ return 0;
+ }
+#endif
}
#ifdef ERTS_PROC_LOCK_DEBUG
@@ -713,44 +813,36 @@ erts_smp_proc_trylock(Process *p, ErtsProcLocks locks)
#endif
}
-
ERTS_GLB_INLINE void erts_smp_proc_inc_refc(Process *p)
{
#ifdef ERTS_SMP
- erts_pix_lock_t *pixlck = ERTS_PID2PIXLOCK(p->id);
- erts_pix_lock(pixlck);
- ERTS_LC_ASSERT(p->lock.refc > 0);
- p->lock.refc++;
- erts_pix_unlock(pixlck);
+#ifdef ERTS_ENABLE_LOCK_CHECK
+ erts_aint32_t refc = erts_atomic32_inc_read_nob(&p->lock.refc);
+ ERTS_SMP_LC_ASSERT(refc > 1);
+#else
+ erts_atomic32_inc_nob(&p->lock.refc);
+#endif
#endif
}
ERTS_GLB_INLINE void erts_smp_proc_dec_refc(Process *p)
{
#ifdef ERTS_SMP
- Process *fp;
- erts_pix_lock_t *pixlck = ERTS_PID2PIXLOCK(p->id);
- erts_pix_lock(pixlck);
- ERTS_LC_ASSERT(p->lock.refc > 0);
- fp = --p->lock.refc == 0 ? p : NULL;
- erts_pix_unlock(pixlck);
- if (fp)
- erts_free_proc(fp);
+ erts_aint32_t refc = erts_atomic32_dec_read_nob(&p->lock.refc);
+ ERTS_SMP_LC_ASSERT(refc >= 0);
+ if (refc == 0)
+ erts_free_proc(p);
#endif
}
-ERTS_GLB_INLINE void erts_smp_proc_add_refc(Process *p, Sint32 refc)
+ERTS_GLB_INLINE void erts_smp_proc_add_refc(Process *p, Sint32 add_refc)
{
#ifdef ERTS_SMP
- Process *fp;
- erts_pix_lock_t *pixlck = ERTS_PID2PIXLOCK(p->id);
- erts_pix_lock(pixlck);
- ERTS_LC_ASSERT(p->lock.refc > 0);
- p->lock.refc += refc;
- fp = p->lock.refc == 0 ? p : NULL;
- erts_pix_unlock(pixlck);
- if (fp)
- erts_free_proc(fp);
+ erts_aint32_t refc = erts_atomic32_add_read_nob(&p->lock.refc,
+ (erts_aint32_t) add_refc);
+ ERTS_SMP_LC_ASSERT(refc >= 0);
+ if (refc == 0)
+ erts_free_proc(p);
#endif
}
@@ -758,6 +850,7 @@ ERTS_GLB_INLINE void erts_smp_proc_add_refc(Process *p, Sint32 refc)
#ifdef ERTS_SMP
void erts_proc_lock_init(Process *);
+void erts_proc_lock_fin(Process *);
void erts_proc_safelock(Process *a_proc,
ErtsProcLocks a_have_locks,
ErtsProcLocks a_need_locks,
@@ -790,211 +883,75 @@ extern const Process erts_proc_lock_busy;
#define erts_pid2proc(PROC, HL, PID, NL) \
erts_pid2proc_opt((PROC), (HL), (PID), (NL), 0)
-ERTS_GLB_INLINE Process *
-erts_pid2proc_opt(Process *, ErtsProcLocks, Eterm, ErtsProcLocks, int);
-#ifdef ERTS_SMP
-void
-erts_pid2proc_safelock(Process *c_p,
- ErtsProcLocks c_p_have_locks,
- Process **proc,
- ErtsProcLocks need_locks,
- erts_pix_lock_t *pix_lock,
- int flags);
-ERTS_GLB_INLINE Process *erts_pid2proc_unlocked_opt(Eterm pid, int flags);
-#define erts_pid2proc_unlocked(PID) erts_pid2proc_unlocked_opt((PID), 0)
-#else
-#define erts_pid2proc_unlocked_opt(PID, FLGS) \
- erts_pid2proc_opt(NULL, 0, (PID), 0, FLGS)
-#define erts_pid2proc_unlocked(PID) erts_pid2proc_opt(NULL, 0, (PID), 0, 0)
+ERTS_GLB_INLINE Process *erts_pix2proc(int ix);
+ERTS_GLB_INLINE Process *erts_proc_lookup_raw(Eterm pid);
+ERTS_GLB_INLINE Process *erts_proc_lookup(Eterm pid);
+
+#ifndef ERTS_SMP
+ERTS_GLB_INLINE
#endif
+Process *erts_pid2proc_opt(Process *, ErtsProcLocks, Eterm, ErtsProcLocks, int);
#if ERTS_GLB_INLINE_INCL_FUNC_DEF
-ERTS_GLB_INLINE Process *
-#ifdef ERTS_SMP
-erts_pid2proc_unlocked_opt(Eterm pid, int flags)
-#else
-erts_pid2proc_opt(Process *c_p_unused,
- ErtsProcLocks c_p_have_locks_unused,
- Eterm pid,
- ErtsProcLocks pid_need_locks_unused,
- int flags)
-#endif
+ERTS_GLB_INLINE Process *erts_pix2proc(int ix)
{
- Uint pix;
Process *proc;
+ ASSERT(0 <= ix && ix < erts_proc.max);
+ proc = (Process *) erts_smp_atomic_read_nob(&erts_proc.tab[ix]);
+ return proc == ERTS_PROC_LOCK_BUSY ? NULL : proc;
+}
+
+ERTS_GLB_INLINE Process *erts_proc_lookup_raw(Eterm pid)
+{
+ Process *proc;
+ int pix;
+
+ /*
+ * In SMP case: Only scheduler threads are allowed
+ * to use this function. Other threads need to
+ * atomicaly increment refc at lookup, i.e., use
+ * erts_pid2proc_opt() with ERTS_P2P_FLG_SMP_INC_REFC.
+ */
+ ERTS_SMP_LC_ASSERT(erts_get_scheduler_id());
if (is_not_internal_pid(pid))
return NULL;
pix = internal_pid_index(pid);
- if(pix >= erts_max_processes)
+
+ proc = (Process *) erts_smp_atomic_read_ddrb(&erts_proc.tab[pix]);
+
+ if (proc && proc->id != pid)
return NULL;
- proc = process_tab[pix];
- if (proc) {
- if (proc->id != pid
- || (!(flags & ERTS_P2P_FLG_ALLOW_OTHER_X)
- && proc->status == P_EXITING))
- proc = NULL;
- }
+
return proc;
}
-#ifdef ERTS_SMP
+ERTS_GLB_INLINE Process *erts_proc_lookup(Eterm pid)
+{
+ Process *proc = erts_proc_lookup_raw(pid);
+ if (proc && ERTS_PROC_IS_EXITING(proc))
+ return NULL;
+ return proc;
+}
+#ifndef ERTS_SMP
ERTS_GLB_INLINE Process *
-erts_pid2proc_opt(Process *c_p,
- ErtsProcLocks c_p_have_locks,
+erts_pid2proc_opt(Process *c_p_unused,
+ ErtsProcLocks c_p_have_locks_unused,
Eterm pid,
- ErtsProcLocks pid_need_locks,
+ ErtsProcLocks pid_need_locks_unused,
int flags)
{
- erts_pix_lock_t *pix_lock;
- ErtsProcLocks need_locks;
- Uint pix;
- Process *proc;
-#ifdef ERTS_ENABLE_LOCK_COUNT
- ErtsProcLocks lcnt_locks;
-#endif
-
-#ifdef ERTS_ENABLE_LOCK_CHECK
- if (c_p) {
- ErtsProcLocks might_unlock = c_p_have_locks & pid_need_locks;
- if (might_unlock)
- erts_proc_lc_might_unlock(c_p, might_unlock);
- }
-#endif
- if (is_not_internal_pid(pid)) {
- proc = NULL;
- goto done;
- }
- pix = internal_pid_index(pid);
- if(pix >= erts_max_processes) {
- proc = NULL;
- goto done;
- }
-
- ERTS_LC_ASSERT((pid_need_locks & ERTS_PROC_LOCKS_ALL) == pid_need_locks);
- need_locks = pid_need_locks;
-
- pix_lock = ERTS_PIX2PIXLOCK(pix);
-
- if (c_p && c_p->id == pid) {
- ASSERT(c_p->id != ERTS_INVALID_PID);
- ASSERT(c_p == process_tab[pix]);
- if (!(flags & ERTS_P2P_FLG_ALLOW_OTHER_X) && c_p->is_exiting) {
- proc = NULL;
- goto done;
- }
- need_locks &= ~c_p_have_locks;
- if (!need_locks) {
- proc = c_p;
- erts_pix_lock(pix_lock);
- if (flags & ERTS_P2P_FLG_SMP_INC_REFC)
- proc->lock.refc++;
- erts_pix_unlock(pix_lock);
- goto done;
- }
- }
-
- erts_pix_lock(pix_lock);
-
- proc = process_tab[pix];
- if (proc) {
- if (proc->id != pid || (!(flags & ERTS_P2P_FLG_ALLOW_OTHER_X)
- && ERTS_PROC_IS_EXITING(proc))) {
- proc = NULL;
- }
- else if (!need_locks) {
- if (flags & ERTS_P2P_FLG_SMP_INC_REFC)
- proc->lock.refc++;
- }
- else {
- int busy;
-
-#ifdef ERTS_ENABLE_LOCK_COUNT
- lcnt_locks = need_locks;
- if (!(flags & ERTS_P2P_FLG_TRY_LOCK)) {
- erts_lcnt_proc_lock(&proc->lock, need_locks);
- }
-#endif
-
-#ifdef ERTS_ENABLE_LOCK_CHECK
- /* Make sure erts_pid2proc_safelock() is enough to handle
- a potential lock order violation situation... */
- busy = erts_proc_lc_trylock_force_busy(proc, need_locks);
- if (!busy)
-#endif
- {
- /* Try a quick trylock to grab all the locks we need. */
- busy = (int) erts_smp_proc_raw_trylock__(proc, need_locks);
-#ifdef ERTS_ENABLE_LOCK_CHECK
- erts_proc_lc_trylock(proc, need_locks, !busy);
-#endif
-#ifdef ERTS_PROC_LOCK_DEBUG
- if (!busy)
- erts_proc_lock_op_debug(proc, need_locks, 1);
-#endif
- }
-
-#ifdef ERTS_ENABLE_LOCK_COUNT
- if (flags & ERTS_P2P_FLG_TRY_LOCK) {
- if (busy) {
- erts_lcnt_proc_trylock(&proc->lock, need_locks, EBUSY);
- } else {
- erts_lcnt_proc_trylock(&proc->lock, need_locks, 0);
- }
- }
-#endif
- if (!busy) {
- if (flags & ERTS_P2P_FLG_SMP_INC_REFC)
- proc->lock.refc++;
-#ifdef ERTS_ENABLE_LOCK_COUNT
- /* all is great */
- if (!(flags & ERTS_P2P_FLG_TRY_LOCK)) {
- erts_lcnt_proc_lock_post_x(&proc->lock, lcnt_locks, __FILE__, __LINE__);
- }
-#endif
- }
- else {
- if (flags & ERTS_P2P_FLG_TRY_LOCK)
- proc = ERTS_PROC_LOCK_BUSY;
- else {
-#ifdef ERTS_ENABLE_LOCK_COUNT
- erts_lcnt_proc_lock_unaquire(&proc->lock, lcnt_locks);
-#endif
- erts_pid2proc_safelock(c_p,
- c_p_have_locks,
- &proc,
- pid_need_locks,
- pix_lock,
- flags);
- if (proc && (flags & ERTS_P2P_FLG_SMP_INC_REFC))
- proc->lock.refc++;
- }
- }
- }
- }
-
- erts_pix_unlock(pix_lock);
-#ifdef ERTS_PROC_LOCK_DEBUG
- ERTS_LC_ASSERT(!proc
- || proc == ERTS_PROC_LOCK_BUSY
- || (pid_need_locks ==
- (ERTS_PROC_LOCK_FLGS_READ_(&proc->lock)
- & pid_need_locks)));
-#endif
-
-
- done:
-
-#if ERTS_PROC_LOCK_ATOMIC_IMPL
- ETHR_COMPILER_BARRIER;
-#endif
-
- return proc;
+ Process *proc = erts_proc_lookup_raw(pid);
+ return ((!(flags & ERTS_P2P_FLG_ALLOW_OTHER_X)
+ && proc
+ && ERTS_PROC_IS_EXITING(proc))
+ ? NULL
+ : proc);
}
-#endif /* ERTS_SMP */
+#endif /* !ERTS_SMP */
#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h
index ee47c98009..17628286bc 100644
--- a/erts/emulator/beam/erl_threads.h
+++ b/erts/emulator/beam/erl_threads.h
@@ -258,10 +258,6 @@
#include "sys.h"
-typedef struct { SWord sint[2]; } erts_no_dw_atomic_t;
-typedef SWord erts_no_atomic_t;
-typedef Sint32 erts_no_atomic32_t;
-
#ifdef USE_THREADS
#define ETHR_TRY_INLINE_FUNCS
@@ -411,12 +407,15 @@ typedef struct {
typedef int erts_rwmtx_t;
typedef int erts_tsd_key_t;
typedef int erts_tse_t;
-#define erts_dw_aint_t erts_no_dw_atomic_t
-#define erts_dw_atomic_t erts_no_dw_atomic_t
-#define erts_aint_t SWord
-#define erts_atomic_t erts_no_atomic_t
-#define erts_aint32_t Sint32
-#define erts_atomic32_t erts_no_atomic32_t
+
+typedef struct { SWord sint[2]; } erts_dw_aint_t;
+typedef SWord erts_aint_t;
+typedef Sint32 erts_aint32_t;
+
+#define erts_dw_atomic_t erts_dw_aint_t
+#define erts_atomic_t erts_aint_t
+#define erts_atomic32_t erts_aint32_t
+
#if __GNUC__ > 2
typedef struct { } erts_spinlock_t;
typedef struct { } erts_rwlock_t;
@@ -425,6 +424,14 @@ typedef struct { int gcc_is_buggy; } erts_spinlock_t;
typedef struct { int gcc_is_buggy; } erts_rwlock_t;
#endif
+#ifdef WORDS_BIGENDIAN
+#define ERTS_DW_AINT_LOW_WORD 1
+#define ERTS_DW_AINT_HIGH_WORD 0
+#else
+#define ERTS_DW_AINT_LOW_WORD 0
+#define ERTS_DW_AINT_HIGH_WORD 1
+#endif
+
#define ERTS_MTX_INITER 0
#define ERTS_CND_INITER 0
#define ERTS_THR_INIT_DATA_DEF_INITER 0
@@ -433,6 +440,10 @@ typedef struct { int gcc_is_buggy; } erts_rwlock_t;
#endif /* #ifdef USE_THREADS */
+#define erts_no_dw_atomic_t erts_dw_aint_t
+#define erts_no_atomic_t erts_aint_t
+#define erts_no_atomic32_t erts_aint32_t
+
#define ERTS_AINT_NULL ((erts_aint_t) NULL)
#define ERTS_AINT_T_MAX (~(((erts_aint_t) 1) << (sizeof(erts_aint_t)*8-1)))
diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h
index 6c6e193818..4bbdcaa3e3 100644
--- a/erts/emulator/beam/erl_time.h
+++ b/erts/emulator/beam/erl_time.h
@@ -118,9 +118,11 @@ ERTS_GLB_INLINE void erts_do_time_add(erts_short_time_t elapsed)
void erts_get_now_cpu(Uint* megasec, Uint* sec, Uint* microsec);
#endif
+typedef UWord erts_approx_time_t;
+erts_approx_time_t erts_get_approx_time(void);
+
void erts_get_timeval(SysTimeval *tv);
erts_time_t erts_get_time(void);
-void erts_get_emu_time(SysTimeval *);
ERTS_GLB_INLINE int erts_cmp_timeval(SysTimeval *t1p, SysTimeval *t2p);
diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c
index 1d0735aa99..c1b4408c06 100644
--- a/erts/emulator/beam/erl_time_sup.c
+++ b/erts/emulator/beam/erl_time_sup.c
@@ -91,6 +91,41 @@ static SysTimeval then; /* Used in get_now */
static SysTimeval last_emu_time; /* Used in erts_get_emu_time() */
SysTimeval erts_first_emu_time; /* Used in erts_get_emu_time() */
+union {
+ erts_smp_atomic_t time;
+ char align[ERTS_CACHE_LINE_SIZE];
+} approx erts_align_attribute(ERTS_CACHE_LINE_SIZE);
+
+static void
+init_approx_time(void)
+{
+ erts_smp_atomic_init_nob(&approx.time, 0);
+}
+
+static ERTS_INLINE erts_approx_time_t
+get_approx_time(void)
+{
+ return (erts_approx_time_t) erts_smp_atomic_read_nob(&approx.time);
+}
+
+static ERTS_INLINE void
+update_approx_time(SysTimeval *tv)
+{
+ erts_approx_time_t new_secs = (erts_approx_time_t) tv->tv_sec;
+ erts_approx_time_t old_secs = get_approx_time();
+ if (old_secs != new_secs)
+ erts_smp_atomic_set_nob(&approx.time, new_secs);
+}
+
+/*
+ * erts_get_approx_time() returns an *approximate* time
+ * in seconds. NOTE that this time may jump backwards!!!
+ */
+erts_approx_time_t
+erts_get_approx_time(void)
+{
+ return get_approx_time();
+}
#ifdef HAVE_GETHRTIME
@@ -398,6 +433,8 @@ erts_init_time_sup(void)
{
erts_smp_mtx_init(&erts_timeofday_mtx, "timeofday");
+ init_approx_time();
+
last_emu_time.tv_sec = 0;
last_emu_time.tv_usec = 0;
@@ -417,7 +454,7 @@ erts_init_time_sup(void)
gtv = inittv;
then.tv_sec = then.tv_usec = 0;
- erts_get_emu_time(&erts_first_emu_time);
+ erts_deliver_time();
return CLOCK_RESOLUTION;
}
@@ -873,6 +910,8 @@ get_now(Uint* megasec, Uint* sec, Uint* microsec)
*megasec = (Uint) (now.tv_sec / 1000000);
*sec = (Uint) (now.tv_sec % 1000000);
*microsec = (Uint) (now.tv_usec);
+
+ update_approx_time(&now);
}
void
@@ -885,6 +924,8 @@ get_sys_now(Uint* megasec, Uint* sec, Uint* microsec)
*megasec = (Uint) (now.tv_sec / 1000000);
*sec = (Uint) (now.tv_sec % 1000000);
*microsec = (Uint) (now.tv_usec);
+
+ update_approx_time(&now);
}
@@ -901,6 +942,8 @@ void erts_deliver_time(void) {
do_erts_deliver_time(&now);
erts_smp_mtx_unlock(&erts_timeofday_mtx);
+
+ update_approx_time(&now);
}
/* get *real* time (not ticks) remaining until next timeout - if there
@@ -949,6 +992,7 @@ void erts_get_timeval(SysTimeval *tv)
erts_smp_mtx_lock(&erts_timeofday_mtx);
get_tolerant_timeofday(tv);
erts_smp_mtx_unlock(&erts_timeofday_mtx);
+ update_approx_time(tv);
}
erts_time_t
@@ -961,7 +1005,9 @@ erts_get_time(void)
get_tolerant_timeofday(&sys_tv);
erts_smp_mtx_unlock(&erts_timeofday_mtx);
-
+
+ update_approx_time(&sys_tv);
+
return sys_tv.tv_sec;
}
@@ -977,38 +1023,3 @@ void erts_get_now_cpu(Uint* megasec, Uint* sec, Uint* microsec) {
*sec = (Uint)(tp.tv_sec % 1000000);
}
#endif
-
-
-/*
- * erts_get_emu_time() is similar to get_now(). You will
- * always get different times from erts_get_emu_time(), but they
- * may equal a time from get_now().
- *
- * erts_get_emu_time() is only used internally in the emulator in
- * order to order emulator internal events.
- */
-
-void
-erts_get_emu_time(SysTimeval *this_emu_time_p)
-{
- erts_smp_mtx_lock(&erts_timeofday_mtx);
-
- get_tolerant_timeofday(this_emu_time_p);
-
- /* Make sure time is later than last */
- if (last_emu_time.tv_sec > this_emu_time_p->tv_sec ||
- (last_emu_time.tv_sec == this_emu_time_p->tv_sec
- && last_emu_time.tv_usec >= this_emu_time_p->tv_usec)) {
- *this_emu_time_p = last_emu_time;
- this_emu_time_p->tv_usec++;
- }
- /* Check for carry from above + general reasonability */
- if (this_emu_time_p->tv_usec >= 1000000) {
- this_emu_time_p->tv_usec = 0;
- this_emu_time_p->tv_sec++;
- }
-
- last_emu_time = *this_emu_time_p;
-
- erts_smp_mtx_unlock(&erts_timeofday_mtx);
-}
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index 009ca1eb52..bc988cd61b 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -64,7 +64,7 @@ int erts_cpu_timestamp;
#endif
static erts_smp_mtx_t smq_mtx;
-static erts_smp_mtx_t sys_trace_mtx;
+static erts_smp_rwmtx_t sys_trace_rwmtx;
enum ErtsSysMsgType {
SYS_MSG_TYPE_UNDEFINED,
@@ -91,7 +91,12 @@ static void init_sys_msg_dispatcher(void);
#endif
void erts_init_trace(void) {
- erts_smp_mtx_init(&sys_trace_mtx, "sys_tracers");
+ erts_smp_rwmtx_opt_t rwmtx_opts = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
+ rwmtx_opts.type = ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ;
+ rwmtx_opts.lived = ERTS_SMP_RWMTX_LONG_LIVED;
+
+ erts_smp_rwmtx_init_opt(&sys_trace_rwmtx, &rwmtx_opts, "sys_tracers");
+
#ifdef HAVE_ERTS_NOW_CPU
erts_cpu_timestamp = 0;
#endif
@@ -151,8 +156,8 @@ do { (RES) = (TPID); } while(0)
#define ERTS_TRACER_REF_TYPE Process *
#define ERTS_GET_TRACER_REF(RES, TPID, TRACEE_FLGS) \
do { \
- (RES) = process_tab[internal_pid_index((TPID))]; \
- if (INVALID_PID((RES), (TPID)) || !((RES)->trace_flags & F_TRACER)) { \
+ (RES) = erts_proc_lookup((TPID)); \
+ if (!(RES) || !((RES)->trace_flags & F_TRACER)) { \
(TPID) = NIL; \
(TRACEE_FLGS) &= ~TRACEE_FLAGS; \
return; \
@@ -169,10 +174,10 @@ erts_system_profile_setup_active_schedulers(void)
active_sched = erts_active_schedulers();
}
-void
-erts_trace_check_exiting(Eterm exiting)
+static void
+exiting_reset(Eterm exiting)
{
- erts_smp_mtx_lock(&sys_trace_mtx);
+ erts_smp_rwmtx_rwlock(&sys_trace_rwmtx);
if (exiting == default_tracer) {
default_tracer = NIL;
default_trace_flags &= TRACEE_FLAGS;
@@ -202,29 +207,49 @@ erts_trace_check_exiting(Eterm exiting)
erts_system_profile_clear(NULL);
#endif
}
- erts_smp_mtx_unlock(&sys_trace_mtx);
+ erts_smp_rwmtx_rwunlock(&sys_trace_rwmtx);
+}
+
+void
+erts_trace_check_exiting(Eterm exiting)
+{
+ int reset = 0;
+ erts_smp_rwmtx_rlock(&sys_trace_rwmtx);
+ if (exiting == default_tracer)
+ reset = 1;
+ else if (exiting == system_seq_tracer)
+ reset = 1;
+ else if (exiting == system_monitor)
+ reset = 1;
+ else if (exiting == system_profile)
+ reset = 1;
+ erts_smp_rwmtx_runlock(&sys_trace_rwmtx);
+ if (reset)
+ exiting_reset(exiting);
+}
+
+static ERTS_INLINE int
+is_valid_tracer(Eterm tracer)
+{
+ return erts_proc_lookup(tracer) || erts_is_valid_tracer_port(tracer);
}
Eterm
erts_set_system_seq_tracer(Process *c_p, ErtsProcLocks c_p_locks, Eterm new)
{
- Eterm old = THE_NON_VALUE;
+ Eterm old;
- if (new != am_false) {
- if (!erts_pid2proc(c_p, c_p_locks, new, 0)
- && !erts_is_valid_tracer_port(new)) {
- return old;
- }
- }
+ if (new != am_false && !is_valid_tracer(new))
+ return THE_NON_VALUE;
- erts_smp_mtx_lock(&sys_trace_mtx);
+ erts_smp_rwmtx_rwlock(&sys_trace_rwmtx);
old = system_seq_tracer;
system_seq_tracer = new;
#ifdef DEBUG_PRINTOUTS
erts_fprintf(stderr, "set seq tracer new=%T old=%T\n", new, old);
#endif
- erts_smp_mtx_unlock(&sys_trace_mtx);
+ erts_smp_rwmtx_rwunlock(&sys_trace_rwmtx);
return old;
}
@@ -232,12 +257,12 @@ Eterm
erts_get_system_seq_tracer(void)
{
Eterm st;
- erts_smp_mtx_lock(&sys_trace_mtx);
+ erts_smp_rwmtx_rlock(&sys_trace_rwmtx);
st = system_seq_tracer;
#ifdef DEBUG_PRINTOUTS
erts_fprintf(stderr, "get seq tracer %T\n", st);
#endif
- erts_smp_mtx_unlock(&sys_trace_mtx);
+ erts_smp_rwmtx_runlock(&sys_trace_rwmtx);
return st;
}
@@ -250,7 +275,7 @@ get_default_tracing(Uint *flagsp, Eterm *tracerp)
if (is_nil(default_tracer)) {
default_trace_flags &= ~TRACEE_FLAGS;
} else if (is_internal_pid(default_tracer)) {
- if (!erts_pid2proc(NULL, 0, default_tracer, 0)) {
+ if (!erts_proc_lookup(default_tracer)) {
reset_tracer:
default_trace_flags &= ~TRACEE_FLAGS;
default_tracer = NIL;
@@ -270,7 +295,7 @@ get_default_tracing(Uint *flagsp, Eterm *tracerp)
void
erts_change_default_tracing(int setflags, Uint *flagsp, Eterm *tracerp)
{
- erts_smp_mtx_lock(&sys_trace_mtx);
+ erts_smp_rwmtx_rwlock(&sys_trace_rwmtx);
if (flagsp) {
if (setflags)
default_trace_flags |= *flagsp;
@@ -280,48 +305,48 @@ erts_change_default_tracing(int setflags, Uint *flagsp, Eterm *tracerp)
if (tracerp)
default_tracer = *tracerp;
get_default_tracing(flagsp, tracerp);
- erts_smp_mtx_unlock(&sys_trace_mtx);
+ erts_smp_rwmtx_rwunlock(&sys_trace_rwmtx);
}
void
erts_get_default_tracing(Uint *flagsp, Eterm *tracerp)
{
- erts_smp_mtx_lock(&sys_trace_mtx);
+ erts_smp_rwmtx_rlock(&sys_trace_rwmtx);
get_default_tracing(flagsp, tracerp);
- erts_smp_mtx_unlock(&sys_trace_mtx);
+ erts_smp_rwmtx_runlock(&sys_trace_rwmtx);
}
void
erts_set_system_monitor(Eterm monitor)
{
- erts_smp_mtx_lock(&sys_trace_mtx);
+ erts_smp_rwmtx_rwlock(&sys_trace_rwmtx);
system_monitor = monitor;
- erts_smp_mtx_unlock(&sys_trace_mtx);
+ erts_smp_rwmtx_rwunlock(&sys_trace_rwmtx);
}
Eterm
erts_get_system_monitor(void)
{
Eterm monitor;
- erts_smp_mtx_lock(&sys_trace_mtx);
+ erts_smp_rwmtx_rlock(&sys_trace_rwmtx);
monitor = system_monitor;
- erts_smp_mtx_unlock(&sys_trace_mtx);
+ erts_smp_rwmtx_runlock(&sys_trace_rwmtx);
return monitor;
}
/* Performance monitoring */
void erts_set_system_profile(Eterm profile) {
- erts_smp_mtx_lock(&sys_trace_mtx);
+ erts_smp_rwmtx_rwlock(&sys_trace_rwmtx);
system_profile = profile;
- erts_smp_mtx_unlock(&sys_trace_mtx);
+ erts_smp_rwmtx_rwunlock(&sys_trace_rwmtx);
}
Eterm
erts_get_system_profile(void) {
Eterm profile;
- erts_smp_mtx_lock(&sys_trace_mtx);
+ erts_smp_rwmtx_rlock(&sys_trace_rwmtx);
profile = system_profile;
- erts_smp_mtx_unlock(&sys_trace_mtx);
+ erts_smp_rwmtx_runlock(&sys_trace_rwmtx);
return profile;
}
@@ -384,13 +409,9 @@ WRITE_SYS_MSG_TO_PORT(Eterm unused_to,
}
#ifndef ERTS_SMP
- if (!INVALID_TRACER_PORT(trace_port, trace_port->id)) {
+ if (!INVALID_TRACER_PORT(trace_port, trace_port->id))
#endif
erts_raw_port_command(trace_port, buffer, ptr-buffer);
-#ifndef ERTS_SMP
- erts_port_release(trace_port);
- }
-#endif
erts_free(ERTS_ALC_T_TMP, (void *) buffer);
}
@@ -465,13 +486,13 @@ send_to_port(Process *c_p, Eterm message,
trace_port = NULL;
#else
- if (is_not_internal_port(*tracer_pid))
- goto invalid_tracer_port;
- trace_port = &erts_port[internal_port_index(*tracer_pid)];
+ trace_port = erts_id2port_sflgs(*tracer_pid,
+ NULL,
+ 0,
+ ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP);
- if (INVALID_TRACER_PORT(trace_port, *tracer_pid)) {
- invalid_tracer_port:
+ if (!trace_port) {
*tracee_flags &= ~TRACEE_FLAGS;
*tracer_pid = NIL;
return;
@@ -491,6 +512,7 @@ send_to_port(Process *c_p, Eterm message,
SYS_MSG_TYPE_TRACE,
message);
#ifndef ERTS_SMP
+ erts_port_release(trace_port);
return;
}
@@ -537,6 +559,9 @@ send_to_port(Process *c_p, Eterm message,
*/
do_send_schedfix_to_port(trace_port, c_p->id, ts);
}
+
+ erts_port_release(trace_port);
+
UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
#undef LOCAL_HEAP_SIZE
#endif
@@ -566,23 +591,28 @@ profile_send(Eterm from, Eterm message) {
Port *profiler_port = NULL;
/* not smp */
-
-
- profiler_port = &erts_port[internal_port_index(profiler)];
-
- do_send_to_port(profiler,
- profiler_port,
- NIL, /* or current process->id */
- SYS_MSG_TYPE_SYSPROF,
- message);
+
+ profiler_port = erts_id2port_sflgs(profiler,
+ NULL,
+ 0,
+ ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP);
+ if (profiler_port) {
+ do_send_to_port(profiler,
+ profiler_port,
+ NIL, /* or current process->id */
+ SYS_MSG_TYPE_SYSPROF,
+ message);
+ erts_port_release(profiler_port);
+ }
} else {
ASSERT(is_internal_pid(profiler)
&& internal_pid_index(profiler) < erts_max_processes);
- profile_p = process_tab[internal_pid_index(profiler)];
+ profile_p = erts_proc_lookup(profiler);
- if (INVALID_PID(profile_p, profiler)) return;
+ if (!profile_p)
+ return;
sz = size_object(message);
hp = erts_alloc_message_heap(sz, &bp, &off_heap, profile_p, 0);
@@ -626,13 +656,11 @@ seq_trace_send_to_port(Process *c_p,
trace_port = NULL;
#else
- if (is_not_internal_port(seq_tracer))
- goto invalid_tracer_port;
-
- trace_port = &erts_port[internal_port_index(seq_tracer)];
-
- if (INVALID_TRACER_PORT(trace_port, seq_tracer)) {
- invalid_tracer_port:
+ trace_port = erts_id2port_sflgs(seq_tracer,
+ NULL,
+ 0,
+ ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP);
+ if (!trace_port) {
system_seq_tracer = am_false;
#ifndef ERTS_SMP
UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
@@ -650,6 +678,7 @@ seq_trace_send_to_port(Process *c_p,
message);
#ifndef ERTS_SMP
+ erts_port_release(trace_port);
UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
return;
}
@@ -691,6 +720,9 @@ seq_trace_send_to_port(Process *c_p,
*/
do_send_schedfix_to_port(trace_port, c_p->id, ts);
}
+
+ erts_port_release(trace_port);
+
UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
#undef LOCAL_HEAP_SIZE
#endif
@@ -789,13 +821,8 @@ trace_sched_aux(Process *p, Eterm what, int never_fake_sched)
ERTS_GET_TRACER_REF(tracer_ref, p->tracer_proc, p->trace_flags);
}
- if (ERTS_PROC_IS_EXITING(p)
-#ifndef ERTS_SMP
- || p->status == P_FREE
-#endif
- ) {
+ if (ERTS_PROC_IS_EXITING(p))
curr_func = 0;
- }
else {
if (!p->current)
p->current = find_function_from_pc(p->i);
@@ -874,7 +901,7 @@ trace_send(Process *p, Eterm to, Eterm msg)
operation = am_send;
if (is_internal_pid(to)) {
- if (!erts_pid2proc(p, ERTS_PROC_LOCK_MAIN, to, 0))
+ if (!erts_proc_lookup(to))
goto send_to_non_existing_process;
}
else if(is_external_pid(to)
@@ -1116,8 +1143,8 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type,
#ifndef ERTS_SMP
- tracer = process_tab[internal_pid_index(seq_tracer)];
- if (INVALID_PID(tracer, tracer->id)) {
+ tracer = erts_proc_lookup(seq_tracer);
+ if (!tracer) {
system_seq_tracer = am_false;
return; /* no need to send anything */
}
@@ -2451,10 +2478,9 @@ monitor_long_gc(Process *p, Uint time) {
#ifndef ERTS_SMP
ASSERT(is_internal_pid(system_monitor)
&& internal_pid_index(system_monitor) < erts_max_processes);
- monitor_p = process_tab[internal_pid_index(system_monitor)];
- if (INVALID_PID(monitor_p, system_monitor) || p == monitor_p) {
+ monitor_p = erts_proc_lookup(system_monitor);
+ if (!monitor_p || p == monitor_p)
return;
- }
#endif
hsz = 0;
@@ -2527,8 +2553,8 @@ monitor_large_heap(Process *p) {
#ifndef ERTS_SMP
ASSERT(is_internal_pid(system_monitor)
&& internal_pid_index(system_monitor) < erts_max_processes);
- monitor_p = process_tab[internal_pid_index(system_monitor)];
- if (INVALID_PID(monitor_p, system_monitor) || p == monitor_p) {
+ monitor_p = erts_proc_lookup(system_monitor);
+ if (monitor_p || p == monitor_p) {
return;
}
#endif
@@ -2582,10 +2608,9 @@ monitor_generic(Process *p, Eterm type, Eterm spec) {
#ifndef ERTS_SMP
ASSERT(is_internal_pid(system_monitor)
&& internal_pid_index(system_monitor) < erts_max_processes);
- monitor_p = process_tab[internal_pid_index(system_monitor)];
- if (INVALID_PID(monitor_p, system_monitor) || p == monitor_p) {
+ monitor_p = erts_proc_lookup(system_monitor);
+ if (!monitor_p || p == monitor_p)
return;
- }
#endif
hp = ERTS_ALLOC_SYSMSG_HEAP(5, &bp, &off_heap, monitor_p);
@@ -3149,10 +3174,10 @@ sys_msg_disp_failure(ErtsSysMsgQ *smqp, Eterm receiver)
break;
case SYS_MSG_TYPE_SEQTRACE:
/* Reset seq_tracer if it hasn't changed */
- erts_smp_mtx_lock(&sys_trace_mtx);
+ erts_smp_rwmtx_rwlock(&sys_trace_rwmtx);
if (system_seq_tracer == receiver)
system_seq_tracer = am_false;
- erts_smp_mtx_unlock(&sys_trace_mtx);
+ erts_smp_rwmtx_rwunlock(&sys_trace_rwmtx);
break;
case SYS_MSG_TYPE_SYSMON:
if (receiver == NIL
@@ -3407,12 +3432,12 @@ sys_msg_dispatcher_func(void *unused)
goto queue_proc_msg;
}
else if (is_internal_port(receiver)) {
- port = erts_id2port(receiver, NULL, 0);
- if (INVALID_TRACER_PORT(port, receiver)) {
- if (port)
- erts_port_release(port);
+ port = erts_id2port_sflgs(receiver,
+ NULL,
+ 0,
+ ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP);
+ if (!port)
goto failure;
- }
else {
write_sys_msg_to_port(receiver,
port,
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index c1be206b90..12fea779da 100755
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -144,8 +144,8 @@ typedef struct ErtsXPortsList_ ErtsXPortsList;
struct port {
ErtsPortTaskSched sched;
ErtsPortTaskHandle timeout_task;
-#ifdef ERTS_SMP
erts_smp_atomic_t refc;
+#ifdef ERTS_SMP
erts_smp_mtx_t *lock;
ErtsXPortsList *xports;
erts_smp_atomic_t run_queue;
@@ -198,6 +198,8 @@ erts_port_runq(Port *prt)
#ifdef ERTS_SMP
ErtsRunQueue *rq1, *rq2;
rq1 = (ErtsRunQueue *) erts_smp_atomic_read_nob(&prt->run_queue);
+ if (!rq1)
+ return NULL;
while (1) {
erts_smp_runq_lock(rq1);
rq2 = (ErtsRunQueue *) erts_smp_atomic_read_nob(&prt->run_queue);
@@ -205,6 +207,8 @@ erts_port_runq(Port *prt)
return rq1;
erts_smp_runq_unlock(rq1);
rq1 = rq2;
+ if (!rq1)
+ return NULL;
}
#else
return ERTS_RUNQ_IX(0);
@@ -1237,28 +1241,29 @@ erts_smp_port_state_unlock(Port *prt)
ERTS_GLB_INLINE int
erts_smp_port_trylock(Port *prt)
{
-#ifdef ERTS_SMP
int res;
ASSERT(erts_smp_atomic_read_nob(&prt->refc) > 0);
erts_smp_atomic_inc_nob(&prt->refc);
+
+#ifdef ERTS_SMP
res = erts_smp_mtx_trylock(prt->lock);
if (res == EBUSY) {
erts_smp_atomic_dec_nob(&prt->refc);
}
+#else
+ res = 0;
+#endif
return res;
-#else /* !ERTS_SMP */
- return 0;
-#endif
}
ERTS_GLB_INLINE void
erts_smp_port_lock(Port *prt)
{
-#ifdef ERTS_SMP
ASSERT(erts_smp_atomic_read_nob(&prt->refc) > 0);
erts_smp_atomic_inc_nob(&prt->refc);
+#ifdef ERTS_SMP
erts_smp_mtx_lock(prt->lock);
#endif
}
@@ -1266,14 +1271,14 @@ erts_smp_port_lock(Port *prt)
ERTS_GLB_INLINE void
erts_smp_port_unlock(Port *prt)
{
-#ifdef ERTS_SMP
erts_aint_t refc;
+#ifdef ERTS_SMP
erts_smp_mtx_unlock(prt->lock);
+#endif
refc = erts_smp_atomic_dec_read_nob(&prt->refc);
ASSERT(refc >= 0);
if (refc == 0)
erts_port_cleanup(prt);
-#endif
}
#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
@@ -1336,12 +1341,12 @@ erts_id2port_sflgs(Eterm id, Process *c_p, ErtsProcLocks c_p_locks, Uint32 sflgs
erts_smp_port_state_unlock(prt);
prt = NULL;
}
-#ifdef ERTS_SMP
else {
erts_smp_atomic_inc_nob(&prt->refc);
erts_smp_port_state_unlock(prt);
- if (no_proc_locks)
+#ifdef ERTS_SMP
+ if (no_proc_locks)
erts_smp_mtx_lock(prt->lock);
else if (erts_smp_mtx_trylock(prt->lock) == EBUSY) {
/* Unlock process locks, and acquire locks in lock order... */
@@ -1357,21 +1362,17 @@ erts_id2port_sflgs(Eterm id, Process *c_p, ErtsProcLocks c_p_locks, Uint32 sflgs
erts_smp_port_unlock(prt); /* Also decrements refc... */
prt = NULL;
}
- }
#endif
+ }
+
return prt;
}
ERTS_GLB_INLINE void
erts_port_release(Port *prt)
{
-#ifdef ERTS_SMP
erts_smp_port_unlock(prt);
-#else
- if (prt->status & ERTS_PORT_SFLGS_DEAD)
- erts_port_cleanup(prt);
-#endif
}
ERTS_GLB_INLINE Port*
@@ -1488,6 +1489,122 @@ void erl_drv_thr_init(void);
/* utils.c */
+typedef struct {
+#ifdef DEBUG
+ int smp_api;
+#endif
+ union {
+ Uint64 not_atomic;
+#ifdef ARCH_64
+ erts_atomic_t atomic;
+#else
+ erts_dw_atomic_t atomic;
+#endif
+ } counter;
+} erts_interval_t;
+
+void erts_interval_init(erts_interval_t *);
+void erts_smp_interval_init(erts_interval_t *);
+Uint64 erts_step_interval_nob(erts_interval_t *);
+Uint64 erts_step_interval_relb(erts_interval_t *);
+Uint64 erts_smp_step_interval_nob(erts_interval_t *);
+Uint64 erts_smp_step_interval_relb(erts_interval_t *);
+Uint64 erts_ensure_later_interval_nob(erts_interval_t *, Uint64);
+Uint64 erts_ensure_later_interval_acqb(erts_interval_t *, Uint64);
+Uint64 erts_smp_ensure_later_interval_nob(erts_interval_t *, Uint64);
+Uint64 erts_smp_ensure_later_interval_acqb(erts_interval_t *, Uint64);
+#ifdef ARCH_32
+ERTS_GLB_INLINE Uint64 erts_interval_dw_aint_to_val__(erts_dw_aint_t *);
+#endif
+ERTS_GLB_INLINE Uint64 erts_current_interval_nob__(erts_interval_t *);
+ERTS_GLB_INLINE Uint64 erts_current_interval_acqb__(erts_interval_t *);
+ERTS_GLB_INLINE Uint64 erts_current_interval_nob(erts_interval_t *);
+ERTS_GLB_INLINE Uint64 erts_current_interval_acqb(erts_interval_t *);
+ERTS_GLB_INLINE Uint64 erts_smp_current_interval_nob(erts_interval_t *);
+ERTS_GLB_INLINE Uint64 erts_smp_current_interval_acqb(erts_interval_t *);
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+#ifdef ARCH_32
+
+ERTS_GLB_INLINE Uint64
+erts_interval_dw_aint_to_val__(erts_dw_aint_t *dw)
+{
+#ifdef ETHR_SU_DW_NAINT_T__
+ return (Uint64) dw->dw_sint;
+#else
+ Uint64 res;
+ res = (Uint64) ((Uint32) dw->sint[ERTS_DW_AINT_HIGH_WORD]);
+ res <<= 32;
+ res |= (Uint64) ((Uint32) dw->sint[ERTS_DW_AINT_LOW_WORD]);
+ return res;
+#endif
+}
+
+#endif
+
+ERTS_GLB_INLINE Uint64
+erts_current_interval_nob__(erts_interval_t *icp)
+{
+#ifdef ARCH_64
+ return (Uint64) erts_atomic_read_nob(&icp->counter.atomic);
+#else
+ erts_dw_aint_t dw;
+ erts_dw_atomic_read_nob(&icp->counter.atomic, &dw);
+ return erts_interval_dw_aint_to_val__(&dw);
+#endif
+}
+
+ERTS_GLB_INLINE Uint64
+erts_current_interval_acqb__(erts_interval_t *icp)
+{
+#ifdef ARCH_64
+ return (Uint64) erts_atomic_read_acqb(&icp->counter.atomic);
+#else
+ erts_dw_aint_t dw;
+ erts_dw_atomic_read_acqb(&icp->counter.atomic, &dw);
+ return erts_interval_dw_aint_to_val__(&dw);
+#endif
+}
+
+ERTS_GLB_INLINE Uint64
+erts_current_interval_nob(erts_interval_t *icp)
+{
+ ASSERT(!icp->smp_api);
+ return erts_current_interval_nob__(icp);
+}
+
+ERTS_GLB_INLINE Uint64
+erts_current_interval_acqb(erts_interval_t *icp)
+{
+ ASSERT(!icp->smp_api);
+ return erts_current_interval_acqb__(icp);
+}
+
+ERTS_GLB_INLINE Uint64
+erts_smp_current_interval_nob(erts_interval_t *icp)
+{
+ ASSERT(icp->smp_api);
+#ifdef ERTS_SMP
+ return erts_current_interval_nob__(icp);
+#else
+ return icp->counter.not_atomic;
+#endif
+}
+
+ERTS_GLB_INLINE Uint64
+erts_smp_current_interval_acqb(erts_interval_t *icp)
+{
+ ASSERT(icp->smp_api);
+#ifdef ERTS_SMP
+ return erts_current_interval_acqb__(icp);
+#else
+ return icp->counter.not_atomic;
+#endif
+}
+
+#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
+
/*
* To be used to silence unused result warnings, but do not abuse it.
*/
@@ -1495,7 +1612,8 @@ void erts_silence_warn_unused_result(long unused);
void erts_cleanup_offheap(ErlOffHeap *offheap);
-Uint erts_fit_in_bits(Uint);
+int erts_fit_in_bits_int64(Sint64);
+int erts_fit_in_bits_int32(Sint32);
int list_length(Eterm);
Export* erts_find_function(Eterm, Eterm, unsigned int, ErtsCodeIndex);
int erts_is_builtin(Eterm, Eterm, int);
@@ -1836,6 +1954,13 @@ extern erts_driver_t spawn_driver;
extern erts_driver_t fd_driver;
/* Should maybe be placed in erl_message.h, but then we get an include mess. */
+ERTS_GLB_INLINE Eterm *
+erts_alloc_message_heap_state(Uint size,
+ ErlHeapFragment **bpp,
+ ErlOffHeap **ohpp,
+ Process *receiver,
+ ErtsProcLocks *receiver_locks,
+ erts_aint32_t *statep);
ERTS_GLB_INLINE Eterm *
erts_alloc_message_heap(Uint size,
@@ -1858,16 +1983,22 @@ erts_alloc_message_heap(Uint size,
*/
ERTS_GLB_INLINE Eterm *
-erts_alloc_message_heap(Uint size,
- ErlHeapFragment **bpp,
- ErlOffHeap **ohpp,
- Process *receiver,
- ErtsProcLocks *receiver_locks)
+erts_alloc_message_heap_state(Uint size,
+ ErlHeapFragment **bpp,
+ ErlOffHeap **ohpp,
+ Process *receiver,
+ ErtsProcLocks *receiver_locks,
+ erts_aint32_t *statep)
{
Eterm *hp;
+ erts_aint32_t state;
#ifdef ERTS_SMP
int locked_main = 0;
- ErtsProcLocks ulocks = *receiver_locks & ERTS_PROC_LOCKS_MSG_SEND;
+ state = erts_smp_atomic32_read_acqb(&receiver->state);
+ if (statep)
+ *statep = state;
+ if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT))
+ goto allocate_in_mbuf;
#endif
if (size > (Uint) INT_MAX)
@@ -1883,20 +2014,19 @@ erts_alloc_message_heap(Uint size,
#ifdef ERTS_SMP
try_allocate_on_heap:
#endif
- if (ERTS_PROC_IS_EXITING(receiver)
+ state = erts_smp_atomic32_read_nob(&receiver->state);
+ if (statep)
+ *statep = state;
+ if ((state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT))
|| HEAP_LIMIT(receiver) - HEAP_TOP(receiver) <= size) {
#ifdef ERTS_SMP
- if (locked_main)
- ulocks |= ERTS_PROC_LOCK_MAIN;
+ if (locked_main) {
+ *receiver_locks &= ~ERTS_PROC_LOCK_MAIN;
+ erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_MAIN);
+ }
#endif
goto allocate_in_mbuf;
}
-#ifdef ERTS_SMP
- if (ulocks) {
- erts_smp_proc_unlock(receiver, ulocks);
- *receiver_locks &= ~ulocks;
- }
-#endif
hp = HEAP_TOP(receiver);
HEAP_TOP(receiver) = hp + size;
*bpp = NULL;
@@ -1912,12 +2042,6 @@ erts_alloc_message_heap(Uint size,
else {
ErlHeapFragment *bp;
allocate_in_mbuf:
-#ifdef ERTS_SMP
- if (ulocks) {
- *receiver_locks &= ~ulocks;
- erts_smp_proc_unlock(receiver, ulocks);
- }
-#endif
bp = new_message_buffer(size);
hp = bp->mem;
*bpp = bp;
@@ -1927,6 +2051,17 @@ erts_alloc_message_heap(Uint size,
return hp;
}
+ERTS_GLB_INLINE Eterm *
+erts_alloc_message_heap(Uint size,
+ ErlHeapFragment **bpp,
+ ErlOffHeap **ohpp,
+ Process *receiver,
+ ErtsProcLocks *receiver_locks)
+{
+ return erts_alloc_message_heap_state(size, bpp, ohpp, receiver,
+ receiver_locks, NULL);
+}
+
#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
#if !HEAP_ON_C_STACK
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index eb8db73bae..7b8bff24e8 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -259,10 +259,8 @@ get_free_port(void)
}
}
port->status = ERTS_PORT_SFLG_INITIALIZING;
-#ifdef ERTS_SMP
- ERTS_SMP_LC_ASSERT(erts_smp_atomic_read_nob(&port->refc) == 0);
+ ERTS_LC_ASSERT(erts_smp_atomic_read_nob(&port->refc) == 0);
erts_smp_atomic_set_nob(&port->refc, 2); /* Port alive + lock */
-#endif
erts_smp_port_state_unlock(port);
return num & port_num_mask;
}
@@ -340,10 +338,14 @@ port_cleanup(Port *prt)
prt->drv_ptr = NULL;
ASSERT(driver);
-#ifdef ERTS_SMP
-
ASSERT(prt->status & ERTS_PORT_SFLG_FREE_SCHEDULED);
- ERTS_SMP_LC_ASSERT(erts_smp_atomic_read_nob(&prt->refc) == 0);
+ ERTS_LC_ASSERT(erts_smp_atomic_read_nob(&prt->refc) == 0);
+
+ ASSERT(prt->status & ERTS_PORT_SFLG_PORT_DEBUG);
+ ASSERT(!(prt->status & ERTS_PORT_SFLG_FREE));
+ prt->status = ERTS_PORT_SFLG_FREE;
+
+#ifdef ERTS_SMP
port_specific = (prt->status & ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK);
@@ -352,10 +354,6 @@ port_cleanup(Port *prt)
prt->lock = NULL;
- ASSERT(prt->status & ERTS_PORT_SFLG_PORT_DEBUG);
- ASSERT(!(prt->status & ERTS_PORT_SFLG_FREE));
- prt->status = ERTS_PORT_SFLG_FREE;
-
erts_smp_port_state_unlock(prt);
erts_smp_mtx_unlock(mtx);
@@ -606,10 +604,8 @@ erts_open_driver(erts_driver_t* driver, /* Pointer to driver. */
/* Need to mark the port as free again */
erts_smp_port_state_lock(port);
port->status = ERTS_PORT_SFLG_FREE;
-#ifdef ERTS_SMP
- ERTS_SMP_LC_ASSERT(erts_smp_atomic_read_nob(&port->refc) == 2);
+ ERTS_LC_ASSERT(erts_smp_atomic_read_nob(&port->refc) == 2);
erts_smp_atomic_set_nob(&port->refc, 0);
-#endif
erts_smp_port_state_unlock(port);
return -3;
}
@@ -1330,7 +1326,7 @@ void init_io(void)
erts_max_ports = ERTS_MAX_R9_PORTS;
}
- port_extra_shift = erts_fit_in_bits(erts_max_ports - 1);
+ port_extra_shift = erts_fit_in_bits_int32(erts_max_ports - 1);
port_num_mask = (1 << ports_bits) - 1;
erts_port_tab_index_mask = ~(~((Uint) 0) << port_extra_shift);
@@ -1354,8 +1350,8 @@ void init_io(void)
for (i = 0; i < erts_max_ports; i++) {
erts_port_task_init_sched(&erts_port[i].sched);
-#ifdef ERTS_SMP
erts_smp_atomic_init_nob(&erts_port[i].refc, 0);
+#ifdef ERTS_SMP
erts_port[i].lock = NULL;
erts_port[i].xports = NULL;
erts_smp_spinlock_init_x(&erts_port[i].state_lck,
@@ -1594,6 +1590,7 @@ deliver_result(Eterm sender, Eterm pid, Eterm res)
{
Process *rp;
ErtsProcLocks rp_locks = 0;
+ int scheduler = erts_get_scheduler_id() != 0;
ERTS_SMP_CHK_NO_PROC_LOCKS;
@@ -1601,7 +1598,9 @@ deliver_result(Eterm sender, Eterm pid, Eterm res)
&& is_internal_pid(pid)
&& internal_pid_index(pid) < erts_max_processes);
- rp = erts_pid2proc_opt(NULL, 0, pid, 0, ERTS_P2P_FLG_SMP_INC_REFC);
+ rp = (scheduler
+ ? erts_proc_lookup(pid)
+ : erts_pid2proc_opt(NULL, 0, pid, 0, ERTS_P2P_FLG_SMP_INC_REFC));
if (rp) {
Eterm tuple;
@@ -1619,7 +1618,9 @@ deliver_result(Eterm sender, Eterm pid, Eterm res)
#endif
);
erts_smp_proc_unlock(rp, rp_locks);
- erts_smp_proc_dec_refc(rp);
+ if (!scheduler)
+ erts_smp_proc_dec_refc(rp);
+
}
}
@@ -1644,6 +1645,7 @@ static void deliver_read_message(Port* prt, Eterm to,
ErlHeapFragment *bp;
ErlOffHeap *ohp;
ErtsProcLocks rp_locks = 0;
+ int scheduler = erts_get_scheduler_id() != 0;
ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
ERTS_SMP_CHK_NO_PROC_LOCKS;
@@ -1658,7 +1660,10 @@ static void deliver_read_message(Port* prt, Eterm to,
need += 2*len;
}
- rp = erts_pid2proc_opt(NULL, 0, to, 0, ERTS_P2P_FLG_SMP_INC_REFC);
+ rp = (scheduler
+ ? erts_proc_lookup(to)
+ : erts_pid2proc_opt(NULL, 0, to, 0, ERTS_P2P_FLG_SMP_INC_REFC));
+
if (!rp)
return;
@@ -1711,8 +1716,10 @@ static void deliver_read_message(Port* prt, Eterm to,
, NIL
#endif
);
- erts_smp_proc_unlock(rp, rp_locks);
- erts_smp_proc_dec_refc(rp);
+ if (rp_locks)
+ erts_smp_proc_unlock(rp, rp_locks);
+ if (!scheduler)
+ erts_smp_proc_dec_refc(rp);
}
/*
@@ -1779,6 +1786,7 @@ deliver_vec_message(Port* prt, /* Port */
ErlHeapFragment *bp;
ErlOffHeap *ohp;
ErtsProcLocks rp_locks = 0;
+ int scheduler = erts_get_scheduler_id() != 0;
ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
ERTS_SMP_CHK_NO_PROC_LOCKS;
@@ -1787,7 +1795,10 @@ deliver_vec_message(Port* prt, /* Port */
* Check arguments for validity.
*/
- rp = erts_pid2proc_opt(NULL, 0, to, 0, ERTS_P2P_FLG_SMP_INC_REFC);
+
+ rp = (scheduler
+ ? erts_proc_lookup(to)
+ : erts_pid2proc_opt(NULL, 0, to, 0, ERTS_P2P_FLG_SMP_INC_REFC));
if (!rp)
return;
@@ -1869,7 +1880,8 @@ deliver_vec_message(Port* prt, /* Port */
#endif
);
erts_smp_proc_unlock(rp, rp_locks);
- erts_smp_proc_dec_refc(rp);
+ if (!scheduler)
+ erts_smp_proc_dec_refc(rp);
}
@@ -2277,9 +2289,8 @@ void erts_port_command(Process *proc,
{
ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND;
- Process* rp = erts_pid2proc_opt(NULL, 0,
- port->connected, rp_locks,
- ERTS_P2P_FLG_SMP_INC_REFC);
+ Process* rp = erts_pid2proc(NULL, 0,
+ port->connected, rp_locks);
if (rp) {
(void) erts_send_exit_signal(NULL,
port->id,
@@ -2290,7 +2301,6 @@ void erts_port_command(Process *proc,
NULL,
0);
erts_smp_proc_unlock(rp, rp_locks);
- erts_smp_proc_dec_refc(rp);
}
}
@@ -2841,13 +2851,17 @@ void driver_report_exit(int ix, int status)
ErlHeapFragment *bp = NULL;
ErlOffHeap *ohp;
ErtsProcLocks rp_locks = 0;
+ int scheduler = erts_get_scheduler_id() != 0;
ERTS_SMP_CHK_NO_PROC_LOCKS;
ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
pid = prt->connected;
ASSERT(is_internal_pid(pid));
- rp = erts_pid2proc_opt(NULL, 0, pid, 0, ERTS_P2P_FLG_SMP_INC_REFC);
+
+ rp = (scheduler
+ ? erts_proc_lookup(pid)
+ : erts_pid2proc_opt(NULL, 0, pid, 0, ERTS_P2P_FLG_SMP_INC_REFC));
if (!rp)
return;
@@ -2864,7 +2878,8 @@ void driver_report_exit(int ix, int status)
);
erts_smp_proc_unlock(rp, rp_locks);
- erts_smp_proc_dec_refc(rp);
+ if (!scheduler)
+ erts_smp_proc_dec_refc(rp);
}
@@ -2995,6 +3010,7 @@ driver_deliver_term(ErlDrvPort port,
ErlOffHeap *ohp;
ErtsProcLocks rp_locks = 0;
struct b2t_states__ b2t;
+ int scheduler = 1; /* Silence erroneous warning... */
init_b2t_states(&b2t);
@@ -3186,7 +3202,13 @@ driver_deliver_term(ErlDrvPort port,
if (res <= 0)
goto done;
- rp = erts_pid2proc_opt(NULL, 0, to, rp_locks, ERTS_P2P_FLG_SMP_INC_REFC);
+ /*
+ * Increase refc on proc if done from a non-scheduler thread.
+ */
+ scheduler = erts_get_scheduler_id() != 0;
+ rp = (scheduler
+ ? erts_proc_lookup(to)
+ : erts_pid2proc_opt(NULL, 0, to, 0, ERTS_P2P_FLG_SMP_INC_REFC));
if (!rp) {
res = 0;
goto done;
@@ -3432,7 +3454,8 @@ driver_deliver_term(ErlDrvPort port,
if (rp) {
if (rp_locks)
erts_smp_proc_unlock(rp, rp_locks);
- erts_smp_proc_dec_refc(rp);
+ if (!scheduler)
+ erts_smp_proc_dec_refc(rp);
}
#endif
cleanup_b2t_states(&b2t);
diff --git a/erts/emulator/beam/register.c b/erts/emulator/beam/register.c
index 26d64887d0..c02872ef80 100644
--- a/erts/emulator/beam/register.c
+++ b/erts/emulator/beam/register.c
@@ -93,19 +93,14 @@ reg_safe_write_lock(Process *c_p, ErtsProcLocks *c_p_locks)
reg_write_lock();
}
+#endif
+
static ERTS_INLINE int
is_proc_alive(Process *p)
{
- int res;
- erts_pix_lock_t *pixlck = ERTS_PID2PIXLOCK(p->id);
- erts_pix_lock(pixlck);
- res = !p->is_exiting;
- erts_pix_unlock(pixlck);
- return res;
+ return !ERTS_PROC_IS_EXITING(p);
}
-#endif
-
void register_info(int to, void *to_arg)
{
int lock = !ERTS_IS_CRASH_DUMPING;
@@ -389,8 +384,7 @@ erts_whereis_name(Process *c_p,
}
#else
if (rp->p
- && ((flags & ERTS_P2P_FLG_ALLOW_OTHER_X)
- || rp->p->status != P_EXITING))
+ && ((flags & ERTS_P2P_FLG_ALLOW_OTHER_X) || is_proc_alive(rp->p)))
*proc = rp->p;
else
*proc = NULL;
@@ -402,7 +396,9 @@ erts_whereis_name(Process *c_p,
if (!rp || !rp->pt)
*port = NULL;
else {
-#ifdef ERTS_SMP
+#ifndef ERTS_SMP
+ erts_smp_atomic_inc_nob(&rp->pt->refc);
+#else
if (pending_port == rp->pt)
pending_port = NULL;
else {
@@ -509,9 +505,11 @@ int erts_unregister_name(Process *c_p,
if ((rp = (RegProc*) hash_get(&process_reg, (void*) &r)) != NULL) {
if (rp->pt) {
if (port != rp->pt) {
-#ifdef ERTS_SMP
+#ifndef ERTS_SMP
+ erts_smp_atomic_inc_nob(&rp->pt->refc);
+#else
if (port) {
- ERTS_SMP_LC_ASSERT(port != c_prt);
+ ASSERT(port != c_prt);
erts_smp_port_unlock(port);
port = NULL;
}
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 7b2bb81f62..0d3b910278 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -39,6 +39,8 @@
#define ENABLE_CHILD_WAITER_THREAD 1
#endif
+#define ERTS_I64_LITERAL(X) X##LL
+
#if defined (__WIN32__)
# include "erl_win_sys.h"
#elif defined (VXWORKS)
@@ -91,14 +93,22 @@ typedef ERTS_SYS_FD_TYPE ErtsSysFdType;
# endif
#endif
-#ifdef __GNUC__
-# if __GNUC__ < 3 && (__GNUC__ != 2 || __GNUC_MINOR__ < 96)
-# define ERTS_LIKELY(BOOL) (BOOL)
-# define ERTS_UNLIKELY(BOOL) (BOOL)
-# else
-# define ERTS_LIKELY(BOOL) __builtin_expect((BOOL), !0)
-# define ERTS_UNLIKELY(BOOL) __builtin_expect((BOOL), 0)
-# endif
+#if !defined(__GNUC__)
+# define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) 0
+#elif !defined(__GNUC_MINOR__)
+# define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \
+ ((__GNUC__ << 24) >= (((MAJ) << 24) | ((MIN) << 12) | (PL)))
+#elif !defined(__GNUC_PATCHLEVEL__)
+# define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \
+ (((__GNUC__ << 24) | (__GNUC_MINOR__ << 12)) >= (((MAJ) << 24) | ((MIN) << 12) | (PL)))
+#else
+# define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \
+ (((__GNUC__ << 24) | (__GNUC_MINOR__ << 12) | __GNUC_PATCHLEVEL__) >= (((MAJ) << 24) | ((MIN) << 12) | (PL)))
+#endif
+
+#if ERTS_AT_LEAST_GCC_VSN__(2, 96, 0)
+# define ERTS_LIKELY(BOOL) __builtin_expect((BOOL), !0)
+# define ERTS_UNLIKELY(BOOL) __builtin_expect((BOOL), 0)
#else
# define ERTS_LIKELY(BOOL) (BOOL)
# define ERTS_UNLIKELY(BOOL) (BOOL)
@@ -178,6 +188,18 @@ int real_printf(const char *fmt, ...);
# define printf real_printf
#endif
+#undef __deprecated
+#if ERTS_AT_LEAST_GCC_VSN__(3, 0, 0)
+# define __deprecated __attribute__((deprecated))
+#else
+# define __deprecated
+#endif
+#if ERTS_AT_LEAST_GCC_VSN__(3, 0, 4)
+# define erts_align_attribute(SZ) __attribute__ ((aligned (SZ)))
+#else
+# define erts_align_attribute(SZ)
+#endif
+
/* In VC++, noreturn is a declspec that has to be before the types,
* but in GNUC it is an att ribute to be placed between return type
* and function name, hence __decl_noreturn <types> __noreturn <function name>
@@ -185,12 +207,6 @@ int real_printf(const char *fmt, ...);
#if __GNUC__
# define __decl_noreturn
# define __noreturn __attribute__((noreturn))
-# undef __deprecated
-# if __GNUC__ >= 3
-# define __deprecated __attribute__((deprecated))
-# else
-# define __deprecated
-# endif
#else
# if defined(__WIN32__) && defined(_MSC_VER)
# define __noreturn
@@ -199,7 +215,6 @@ int real_printf(const char *fmt, ...);
# define __noreturn
# define __decl_noreturn
# endif
-# define __deprecated
#endif
/*
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index a36d15204e..db6597dc7c 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -268,16 +268,42 @@ list_length(Eterm list)
return i;
}
-Uint erts_fit_in_bits(Uint n)
+static const struct {
+ Sint64 mask;
+ int bits;
+} fib_data[] = {{ERTS_I64_LITERAL(0x2), 1},
+ {ERTS_I64_LITERAL(0xc), 2},
+ {ERTS_I64_LITERAL(0xf0), 4},
+ {ERTS_I64_LITERAL(0xff00), 8},
+ {ERTS_I64_LITERAL(0xffff0000), 16},
+ {ERTS_I64_LITERAL(0xffffffff00000000), 32}};
+
+static ERTS_INLINE int
+fit_in_bits(Sint64 value, int start)
{
- Uint i;
+ int bits = 0;
+ int i;
- i = 0;
- while (n > 0) {
- i++;
- n >>= 1;
- }
- return i;
+ for (i = start; i >= 0; i--) {
+ if (value & fib_data[i].mask) {
+ value >>= fib_data[i].bits;
+ bits |= fib_data[i].bits;
+ }
+ }
+
+ bits++;
+
+ return bits;
+}
+
+int erts_fit_in_bits_int64(Sint64 value)
+{
+ return fit_in_bits(value, 5);
+}
+
+int erts_fit_in_bits_int32(Sint32 value)
+{
+ return fit_in_bits((Sint64) (Uint32) value, 4);
}
int
@@ -1640,12 +1666,20 @@ static int do_send_to_logger(Eterm tag, Eterm gleader, char *buf, int len)
}
#ifndef ERTS_SMP
- if (
#ifdef USE_THREADS
- !erts_get_scheduler_data() || /* Must be scheduler thread */
+ p = NULL;
+ if (erts_get_scheduler_data()) /* Must be scheduler thread */
#endif
- (p = erts_whereis_process(NULL, 0, am_error_logger, 0, 0)) == NULL
- || p->status == P_RUNNING) {
+ {
+ p = erts_whereis_process(NULL, 0, am_error_logger, 0, 0);
+ if (p) {
+ erts_aint32_t state = erts_smp_atomic32_read_acqb(&p->state);
+ if (state & ERTS_PSFLG_RUNNING)
+ p = NULL;
+ }
+ }
+
+ if (!p) {
/* buf *always* points to a null terminated string */
erts_fprintf(stderr, "(no error logger present) %T: \"%s\"\n",
tag, buf);
@@ -3240,7 +3274,7 @@ ptimer_timeout(ErtsSmpPTimer *ptimer)
ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS,
ERTS_P2P_FLG_ALLOW_OTHER_X);
if (p) {
- if (!p->is_exiting
+ if (!ERTS_PROC_IS_EXITING(p)
&& !(ptimer->timer.flags & ERTS_PTMR_FLG_CANCELLED)) {
ASSERT(*ptimer->timer.timer_ref == ptimer);
*ptimer->timer.timer_ref = NULL;
@@ -3458,6 +3492,254 @@ void erts_silence_warn_unused_result(long unused)
}
+/*
+ * Interval counts
+ */
+void
+erts_interval_init(erts_interval_t *icp)
+{
+#ifdef ARCH_64
+ erts_atomic_init_nob(&icp->counter.atomic, 0);
+#else
+ erts_dw_aint_t dw;
+#ifdef ETHR_SU_DW_NAINT_T__
+ dw.dw_sint = 0;
+#else
+ dw.sint[ERTS_DW_AINT_HIGH_WORD] = 0;
+ dw.sint[ERTS_DW_AINT_LOW_WORD] = 0;
+#endif
+ erts_dw_atomic_init_nob(&icp->counter.atomic, &dw);
+
+#endif
+#ifdef DEBUG
+ icp->smp_api = 0;
+#endif
+}
+
+void
+erts_smp_interval_init(erts_interval_t *icp)
+{
+#ifdef ERTS_SMP
+ erts_interval_init(icp);
+#else
+ icp->counter.not_atomic = 0;
+#endif
+#ifdef DEBUG
+ icp->smp_api = 1;
+#endif
+}
+
+static ERTS_INLINE Uint64
+step_interval_nob(erts_interval_t *icp)
+{
+#ifdef ARCH_64
+ return (Uint64) erts_atomic_inc_read_nob(&icp->counter.atomic);
+#else
+ erts_dw_aint_t exp;
+
+ erts_dw_atomic_read_nob(&icp->counter.atomic, &exp);
+ while (1) {
+ erts_dw_aint_t new = exp;
+
+#ifdef ETHR_SU_DW_NAINT_T__
+ new.dw_sint++;
+#else
+ new.sint[ERTS_DW_AINT_LOW_WORD]++;
+ if (new.sint[ERTS_DW_AINT_LOW_WORD] == 0)
+ new.sint[ERTS_DW_AINT_HIGH_WORD]++;
+#endif
+
+ if (erts_dw_atomic_cmpxchg_nob(&icp->counter.atomic, &new, &exp))
+ return erts_interval_dw_aint_to_val__(&new);
+
+ }
+#endif
+}
+
+static ERTS_INLINE Uint64
+step_interval_relb(erts_interval_t *icp)
+{
+#ifdef ARCH_64
+ return (Uint64) erts_atomic_inc_read_relb(&icp->counter.atomic);
+#else
+ erts_dw_aint_t exp;
+
+ erts_dw_atomic_read_nob(&icp->counter.atomic, &exp);
+ while (1) {
+ erts_dw_aint_t new = exp;
+
+#ifdef ETHR_SU_DW_NAINT_T__
+ new.dw_sint++;
+#else
+ new.sint[ERTS_DW_AINT_LOW_WORD]++;
+ if (new.sint[ERTS_DW_AINT_LOW_WORD] == 0)
+ new.sint[ERTS_DW_AINT_HIGH_WORD]++;
+#endif
+
+ if (erts_dw_atomic_cmpxchg_relb(&icp->counter.atomic, &new, &exp))
+ return erts_interval_dw_aint_to_val__(&new);
+
+ }
+#endif
+}
+
+
+static ERTS_INLINE Uint64
+ensure_later_interval_nob(erts_interval_t *icp, Uint64 ic)
+{
+ Uint64 curr_ic;
+#ifdef ARCH_64
+ curr_ic = (Uint64) erts_atomic_read_nob(&icp->counter.atomic);
+ if (curr_ic > ic)
+ return curr_ic;
+ return (Uint64) erts_atomic_inc_read_nob(&icp->counter.atomic);
+#else
+ erts_dw_aint_t exp;
+
+ erts_dw_atomic_read_nob(&icp->counter.atomic, &exp);
+ curr_ic = erts_interval_dw_aint_to_val__(&exp);
+ if (curr_ic > ic)
+ return curr_ic;
+
+ while (1) {
+ erts_dw_aint_t new = exp;
+
+#ifdef ETHR_SU_DW_NAINT_T__
+ new.dw_sint++;
+#else
+ new.sint[ERTS_DW_AINT_LOW_WORD]++;
+ if (new.sint[ERTS_DW_AINT_LOW_WORD] == 0)
+ new.sint[ERTS_DW_AINT_HIGH_WORD]++;
+#endif
+
+ if (erts_dw_atomic_cmpxchg_nob(&icp->counter.atomic, &new, &exp))
+ return erts_interval_dw_aint_to_val__(&new);
+
+ curr_ic = erts_interval_dw_aint_to_val__(&exp);
+ if (curr_ic > ic)
+ return curr_ic;
+ }
+#endif
+}
+
+
+static ERTS_INLINE Uint64
+ensure_later_interval_acqb(erts_interval_t *icp, Uint64 ic)
+{
+ Uint64 curr_ic;
+#ifdef ARCH_64
+ curr_ic = (Uint64) erts_atomic_read_acqb(&icp->counter.atomic);
+ if (curr_ic > ic)
+ return curr_ic;
+ return (Uint64) erts_atomic_inc_read_acqb(&icp->counter.atomic);
+#else
+ erts_dw_aint_t exp;
+
+ erts_dw_atomic_read_acqb(&icp->counter.atomic, &exp);
+ curr_ic = erts_interval_dw_aint_to_val__(&exp);
+ if (curr_ic > ic)
+ return curr_ic;
+
+ while (1) {
+ erts_dw_aint_t new = exp;
+
+#ifdef ETHR_SU_DW_NAINT_T__
+ new.dw_sint++;
+#else
+ new.sint[ERTS_DW_AINT_LOW_WORD]++;
+ if (new.sint[ERTS_DW_AINT_LOW_WORD] == 0)
+ new.sint[ERTS_DW_AINT_HIGH_WORD]++;
+#endif
+
+ if (erts_dw_atomic_cmpxchg_acqb(&icp->counter.atomic, &new, &exp))
+ return erts_interval_dw_aint_to_val__(&new);
+
+ curr_ic = erts_interval_dw_aint_to_val__(&exp);
+ if (curr_ic > ic)
+ return curr_ic;
+ }
+#endif
+}
+
+Uint64
+erts_step_interval_nob(erts_interval_t *icp)
+{
+ ASSERT(!icp->smp_api);
+ return step_interval_nob(icp);
+}
+
+Uint64
+erts_step_interval_relb(erts_interval_t *icp)
+{
+ ASSERT(!icp->smp_api);
+ return step_interval_relb(icp);
+}
+
+Uint64
+erts_smp_step_interval_nob(erts_interval_t *icp)
+{
+ ASSERT(icp->smp_api);
+#ifdef ERTS_SMP
+ return step_interval_nob(icp);
+#else
+ return ++icp->counter.not_atomic;
+#endif
+}
+
+Uint64
+erts_smp_step_interval_relb(erts_interval_t *icp)
+{
+ ASSERT(icp->smp_api);
+#ifdef ERTS_SMP
+ return step_interval_relb(icp);
+#else
+ return ++icp->counter.not_atomic;
+#endif
+}
+
+Uint64
+erts_ensure_later_interval_nob(erts_interval_t *icp, Uint64 ic)
+{
+ ASSERT(!icp->smp_api);
+ return ensure_later_interval_nob(icp, ic);
+}
+
+Uint64
+erts_ensure_later_interval_acqb(erts_interval_t *icp, Uint64 ic)
+{
+ ASSERT(!icp->smp_api);
+ return ensure_later_interval_acqb(icp, ic);
+}
+
+Uint64
+erts_smp_ensure_later_interval_nob(erts_interval_t *icp, Uint64 ic)
+{
+ ASSERT(icp->smp_api);
+#ifdef ERTS_SMP
+ return ensure_later_interval_nob(icp, ic);
+#else
+ if (icp->counter.not_atomic > ic)
+ return icp->counter.not_atomic;
+ else
+ return ++icp->counter.not_atomic;
+#endif
+}
+
+Uint64
+erts_smp_ensure_later_interval_acqb(erts_interval_t *icp, Uint64 ic)
+{
+ ASSERT(icp->smp_api);
+#ifdef ERTS_SMP
+ return ensure_later_interval_acqb(icp, ic);
+#else
+ if (icp->counter.not_atomic > ic)
+ return icp->counter.not_atomic;
+ else
+ return ++icp->counter.not_atomic;
+#endif
+}
+
+
#ifdef DEBUG
/*
* Handy functions when using a debugger - don't use in the code!
@@ -3490,7 +3772,7 @@ Process *p;
void ppi(Eterm pid)
{
- pp(erts_pid2proc_unlocked(pid));
+ pp(erts_proc_lookup(pid));
}
void td(Eterm x)
diff --git a/erts/emulator/hipe/hipe_debug.c b/erts/emulator/hipe/hipe_debug.c
index 7ca11f8c6c..37615bf718 100644
--- a/erts/emulator/hipe/hipe_debug.c
+++ b/erts/emulator/hipe/hipe_debug.c
@@ -188,8 +188,6 @@ void hipe_print_pcb(Process *p)
U("old_htop ", old_htop);
U("old_head ", old_heap);
U("min_heap_..", min_heap_size);
- U("status ", status);
- U("rstatus ", rstatus);
U("rcount ", rcount);
U("id ", id);
U("prio ", prio);
diff --git a/erts/emulator/hipe/hipe_mode_switch.c b/erts/emulator/hipe/hipe_mode_switch.c
index 6a3ce5608f..3c782b2f56 100644
--- a/erts/emulator/hipe/hipe_mode_switch.c
+++ b/erts/emulator/hipe/hipe_mode_switch.c
@@ -360,7 +360,8 @@ Process *hipe_mode_switch(Process *p, unsigned cmd, Eterm reg[])
p->flags &= ~F_HIBERNATE_SCHED;
goto do_schedule;
}
- if (p->status == P_WAITING) {
+
+ if (!(erts_smp_atomic32_read_acqb(&p->state) & ERTS_PSFLG_ACTIVE)) {
for (i = 0; i < p->arity; ++i)
p->arg_reg[i] = reg[i];
goto do_schedule;
@@ -451,10 +452,6 @@ Process *hipe_mode_switch(Process *p, unsigned cmd, Eterm reg[])
case HIPE_MODE_SWITCH_RES_SUSPEND: {
p->i = hipe_beam_pc_resume;
p->arity = 0;
- erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
- if (p->status != P_SUSPENDED)
- erts_add_to_runq(p);
- erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
goto do_schedule;
}
case HIPE_MODE_SWITCH_RES_WAIT:
@@ -470,7 +467,8 @@ Process *hipe_mode_switch(Process *p, unsigned cmd, Eterm reg[])
#endif
p->i = hipe_beam_pc_resume;
p->arity = 0;
- p->status = P_WAITING;
+ erts_smp_atomic32_read_band_relb(&p->state,
+ ~ERTS_PSFLG_ACTIVE);
erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_MSG_RECEIVE);
do_schedule:
{
diff --git a/erts/emulator/sys/win32/erl_win_sys.h b/erts/emulator/sys/win32/erl_win_sys.h
index 03298a6c54..21c52ca26d 100644
--- a/erts/emulator/sys/win32/erl_win_sys.h
+++ b/erts/emulator/sys/win32/erl_win_sys.h
@@ -90,6 +90,11 @@
#define strncasecmp _strnicmp
+#ifndef __GNUC__
+# undef ERTS_I64_LITERAL
+# define ERTS_I64_LITERAL(X) X##i64
+#endif
+
/*
* Practial Windows specific macros.
*/
diff --git a/erts/emulator/sys/win32/sys_time.c b/erts/emulator/sys/win32/sys_time.c
index b5123dc45d..2f2dfc8197 100644
--- a/erts/emulator/sys/win32/sys_time.c
+++ b/erts/emulator/sys/win32/sys_time.c
@@ -26,11 +26,7 @@
#include "sys.h"
#include "assert.h"
-#ifdef __GNUC__
-#define LL_LITERAL(X) X##LL
-#else
-#define LL_LITERAL(X) X##i64
-#endif
+#define LL_LITERAL(X) ERTS_I64_LITERAL(X)
/******************* Routines for time measurement *********************/