aboutsummaryrefslogtreecommitdiffstats
path: root/erts/emulator
diff options
context:
space:
mode:
authorRickard Green <[email protected]>2011-12-16 13:02:09 +0100
committerRickard Green <[email protected]>2012-04-16 16:32:28 +0200
commit414f4fb8dc9a188f8148a1f92e5f9125108e170d (patch)
tree319b52ff0171a0f3bec1deb63407caaefe095ab5 /erts/emulator
parent3e454414a40bba082e5dc0be310f71843200dcf4 (diff)
downloadotp-414f4fb8dc9a188f8148a1f92e5f9125108e170d.tar.gz
otp-414f4fb8dc9a188f8148a1f92e5f9125108e170d.tar.bz2
otp-414f4fb8dc9a188f8148a1f92e5f9125108e170d.zip
Optimize process table access
Diffstat (limited to 'erts/emulator')
-rw-r--r--erts/emulator/beam/beam_bp.c2
-rw-r--r--erts/emulator/beam/beam_emu.c4
-rw-r--r--erts/emulator/beam/bif.c21
-rw-r--r--erts/emulator/beam/break.c21
-rw-r--r--erts/emulator/beam/dist.c25
-rw-r--r--erts/emulator/beam/erl_alloc.types1
-rw-r--r--erts/emulator/beam/erl_bif_info.c22
-rw-r--r--erts/emulator/beam/erl_bif_timer.c4
-rw-r--r--erts/emulator/beam/erl_bif_trace.c17
-rw-r--r--erts/emulator/beam/erl_db.c14
-rw-r--r--erts/emulator/beam/erl_db_util.h2
-rw-r--r--erts/emulator/beam/erl_init.c9
-rw-r--r--erts/emulator/beam/erl_message.c4
-rw-r--r--erts/emulator/beam/erl_nif.c20
-rw-r--r--erts/emulator/beam/erl_node_container_utils.h43
-rw-r--r--erts/emulator/beam/erl_node_tables.c38
-rw-r--r--erts/emulator/beam/erl_process.c782
-rw-r--r--erts/emulator/beam/erl_process.h39
-rw-r--r--erts/emulator/beam/erl_process_dump.c7
-rw-r--r--erts/emulator/beam/erl_process_lock.c261
-rw-r--r--erts/emulator/beam/erl_process_lock.h287
-rw-r--r--erts/emulator/beam/erl_threads.h31
-rw-r--r--erts/emulator/beam/erl_time.h4
-rw-r--r--erts/emulator/beam/erl_time_sup.c85
-rw-r--r--erts/emulator/beam/erl_trace.c33
-rw-r--r--erts/emulator/beam/global.h119
-rw-r--r--erts/emulator/beam/io.c59
-rw-r--r--erts/emulator/beam/register.c7
-rw-r--r--erts/emulator/beam/sys.h45
-rw-r--r--erts/emulator/beam/utils.c294
-rw-r--r--erts/emulator/sys/win32/erl_win_sys.h5
-rw-r--r--erts/emulator/sys/win32/sys_time.c6
32 files changed, 1509 insertions, 802 deletions
diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c
index d772bea02f..f4e7119d4d 100644
--- a/erts/emulator/beam/beam_bp.c
+++ b/erts/emulator/beam/beam_bp.c
@@ -1255,7 +1255,7 @@ static int clear_function_break(Module *m, BeamInstr *pc, int bif, BeamInstr bre
for (j = 0; j < bdt->hash[i].n; ++j) {
item = &(bdt->hash[i].item[j]);
if (item->pid != NIL) {
- h_p = process_tab[internal_pid_index(item->pid)];
+ h_p = erts_proc_lookup(item->pid);
if (h_p) {
pbt = ERTS_PROC_SET_CALL_TIME(h_p, ERTS_PROC_LOCK_MAIN, NULL);
if (pbt) {
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 18a57931ae..5ca3e8060b 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -64,11 +64,7 @@
# define PROCESS_MAIN_CHK_LOCKS(P) \
do { \
if ((P)) { \
- erts_pix_lock_t *pix_lock__ = ERTS_PIX2PIXLOCK(internal_pid_index((P)->id));\
erts_proc_lc_chk_only_proc_main((P)); \
- erts_pix_lock(pix_lock__); \
- ASSERT(0 < (P)->lock.refc && (P)->lock.refc < erts_no_schedulers*5);\
- erts_pix_unlock(pix_lock__); \
} \
else \
erts_lc_check_exact(NULL, 0); \
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 39d4582435..dd9488afa6 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -1403,9 +1403,8 @@ BIF_RETTYPE exit_2(BIF_ALIST_2)
}
else {
rp_locks = ERTS_PROC_LOCKS_XSIG_SEND;
- rp = erts_pid2proc_opt(BIF_P, ERTS_PROC_LOCK_MAIN,
- BIF_ARG_1, rp_locks,
- ERTS_P2P_FLG_SMP_INC_REFC);
+ rp = erts_pid2proc(BIF_P, ERTS_PROC_LOCK_MAIN,
+ BIF_ARG_1, rp_locks);
if (!rp) {
BIF_RET(am_true);
}
@@ -1427,8 +1426,6 @@ BIF_RETTYPE exit_2(BIF_ALIST_2)
rp_locks &= ~ERTS_PROC_LOCK_MAIN;
if (rp_locks)
erts_smp_proc_unlock(rp, rp_locks);
- if (rp != BIF_P)
- erts_smp_proc_dec_refc(rp);
#endif
/*
* We may have exited ourselves and may have to take action.
@@ -1826,8 +1823,7 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend) {
if (internal_pid_index(to) >= erts_max_processes)
return SEND_BADARG;
- rp = erts_pid2proc_opt(p, ERTS_PROC_LOCK_MAIN,
- to, 0, ERTS_P2P_FLG_SMP_INC_REFC);
+ rp = erts_proc_lookup_raw(to);
if (!rp) {
ERTS_SMP_ASSERT_IS_NOT_EXITING(p);
@@ -1865,7 +1861,7 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend) {
}
erts_whereis_name(p, ERTS_PROC_LOCK_MAIN,
to,
- &rp, 0, ERTS_P2P_FLG_SMP_INC_REFC,
+ &rp, 0, 0,
&pt);
if (pt) {
@@ -2017,7 +2013,7 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend) {
erts_whereis_name(p, ERTS_PROC_LOCK_MAIN,
tp[1],
- &rp, 0, ERTS_P2P_FLG_SMP_INC_REFC,
+ &rp, 0, 0,
&pt);
if (pt) {
portid = pt->id;
@@ -2076,7 +2072,6 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend) {
p == rp
? (rp_locks & ~ERTS_PROC_LOCK_MAIN)
: rp_locks);
- erts_smp_proc_dec_refc(rp);
return res;
}
}
@@ -3486,7 +3481,7 @@ BIF_RETTYPE garbage_collect_1(BIF_ALIST_1)
if (rp == ERTS_PROC_LOCK_BUSY)
ERTS_BIF_YIELD1(bif_export[BIF_garbage_collect_1], BIF_P, BIF_ARG_1);
#else
- rp = erts_pid2proc(BIF_P, 0, BIF_ARG_1, 0);
+ rp = erts_proc_lookup(BIF_ARG_1);
#endif
if (!rp)
BIF_RET(am_false);
@@ -4232,8 +4227,8 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2)
erts_smp_thr_progress_block();
for (i = 0; i < erts_max_processes; i++) {
- if (process_tab[i] != (Process*) 0) {
- Process* p = process_tab[i];
+ Process *p = erts_pix2proc(i);
+ if (p) {
#ifdef USE_VM_PROBES
p->seq_trace_token = (p->dt_utag != NIL) ? am_have_dt_utag : NIL;
#else
diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index 6f5020dc14..0a5fc26ee8 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -71,9 +71,10 @@ process_info(int to, void *to_arg)
{
int i;
for (i = 0; i < erts_max_processes; i++) {
- if ((process_tab[i] != NULL) && (process_tab[i]->i != ENULL)) {
- if (process_tab[i]->status != P_EXITING)
- print_process_info(to, to_arg, process_tab[i]);
+ Process *p = erts_pix2proc(i);
+ if (p && p->i != ENULL) {
+ if (p->status != P_EXITING)
+ print_process_info(to, to_arg, p);
}
}
@@ -89,7 +90,8 @@ process_killer(void)
erts_printf("\n\nProcess Information\n\n");
erts_printf("--------------------------------------------------\n");
for (i = erts_max_processes-1; i >= 0; i--) {
- if (((rp = process_tab[i]) != NULL) && rp->i != ENULL) {
+ rp = erts_pix2proc(i);
+ if (rp && rp->i != ENULL) {
int br;
print_process_info(ERTS_PRINT_STDOUT, NULL, rp);
erts_printf("(k)ill (n)ext (r)eturn:\n");
@@ -180,9 +182,9 @@ static void doit_print_monitor(ErtsMonitor *mon, void *vpcontext)
void
print_process_info(int to, void *to_arg, Process *p)
{
+ time_t approx_started;
int garbing = 0;
int running = 0;
- time_t tmp_t;
struct saved_calls *scb;
/* display the PID */
@@ -245,8 +247,8 @@ print_process_info(int to, void *to_arg, Process *p)
}
erts_print(to, to_arg, "Spawned by: %T\n", p->parent);
- tmp_t = p->started.tv_sec;
- erts_print(to, to_arg, "Started: %s", ctime(&tmp_t));
+ approx_started = (time_t) p->approx_started;
+ erts_print(to, to_arg, "Started: %s", ctime(&approx_started));
ERTS_SMP_MSGQ_MV_INQ2PRIVQ(p);
erts_print(to, to_arg, "Message queue length: %d\n", p->msg.len);
@@ -616,7 +618,8 @@ bin_check(void)
int i, printed = 0;
for (i=0; i < erts_max_processes; i++) {
- if ((rp = process_tab[i]) == NULL)
+ rp = erts_pix2proc(i);
+ if (!rp)
continue;
for (hdr = rp->off_heap.first; hdr; hdr = hdr->next) {
if (hdr->thing_word == HEADER_PROC_BIN) {
@@ -704,7 +707,7 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
erts_print_nif_taints(fd, NULL);
erts_fdprintf(fd, "Atoms: %d\n", atom_table_size());
info(fd, NULL); /* General system info */
- if (process_tab != NULL) /* XXX true at init */
+ if (erts_proc.tab)
process_info(fd, NULL); /* Info about each process and port */
db_info(fd, NULL, 0);
erts_print_bif_timer_info(fd, NULL);
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 7c75c9fdb7..a7f11740f5 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -1312,7 +1312,7 @@ int erts_net_message(Port *prt,
if (is_not_pid(from) || is_not_atom(to)){
goto invalid_message;
}
- rp = erts_whereis_process(NULL, 0, to, 0, ERTS_P2P_FLG_SMP_INC_REFC);
+ rp = erts_whereis_process(NULL, 0, to, 0, 0);
if (rp) {
Uint xsize = (type == DOP_REG_SEND
? 0
@@ -1338,7 +1338,6 @@ int erts_net_message(Port *prt,
erts_queue_dist_message(rp, &locks, ede_copy, token);
if (locks)
erts_smp_proc_unlock(rp, locks);
- erts_smp_proc_dec_refc(rp);
}
break;
@@ -1364,7 +1363,7 @@ int erts_net_message(Port *prt,
if (is_not_pid(to)) {
goto invalid_message;
}
- rp = erts_pid2proc_opt(NULL, 0, to, 0, ERTS_P2P_FLG_SMP_INC_REFC);
+ rp = erts_proc_lookup(to);
if (rp) {
Uint xsize = type == DOP_SEND ? 0 : ERTS_HEAP_FRAG_SIZE(token_size);
ErtsProcLocks locks = 0;
@@ -1388,7 +1387,6 @@ int erts_net_message(Port *prt,
erts_queue_dist_message(rp, &locks, ede_copy, token);
if (locks)
erts_smp_proc_unlock(rp, locks);
- erts_smp_proc_dec_refc(rp);
}
break;
@@ -1544,8 +1542,7 @@ int erts_net_message(Port *prt,
if (is_not_pid(from) || is_not_internal_pid(to)) {
goto invalid_message;
}
- rp = erts_pid2proc_opt(NULL, 0, to, rp_locks,
- ERTS_P2P_FLG_SMP_INC_REFC);
+ rp = erts_pid2proc(NULL, 0, to, rp_locks);
if (rp) {
(void) erts_send_exit_signal(NULL,
from,
@@ -1556,7 +1553,6 @@ int erts_net_message(Port *prt,
NULL,
0);
erts_smp_proc_unlock(rp, rp_locks);
- erts_smp_proc_dec_refc(rp);
}
break;
}
@@ -2246,7 +2242,7 @@ static void doit_print_monitor_info(ErtsMonitor *mon, void *vptdp)
void *arg = ((struct print_to_data *) vptdp)->arg;
Process *rp;
ErtsMonitor *rmon;
- rp = erts_pid2proc_unlocked(mon->pid);
+ rp = erts_proc_lookup(mon->pid);
if (!rp || (rmon = erts_lookup_monitor(rp->monitors, mon->ref)) == NULL) {
erts_print(to, arg, "Warning, stray monitor for: %T\n", mon->pid);
} else if (mon->type == MON_ORIGIN) {
@@ -2285,7 +2281,7 @@ static void doit_print_link_info2(ErtsLink *lnk, void *vpplc)
static void doit_print_link_info(ErtsLink *lnk, void *vptdp)
{
- if (is_internal_pid(lnk->pid) && erts_pid2proc_unlocked(lnk->pid)) {
+ if (is_internal_pid(lnk->pid) && erts_proc_lookup(lnk->pid)) {
PrintLinkContext plc = {(struct print_to_data *) vptdp, lnk->pid};
erts_doforall_links(ERTS_LINK_ROOT(lnk), &doit_print_link_info2, &plc);
}
@@ -2307,7 +2303,7 @@ static void doit_print_nodelink_info(ErtsLink *lnk, void *vpcontext)
{
PrintNodeLinkContext *pcontext = vpcontext;
- if (is_internal_pid(lnk->pid) && erts_pid2proc_unlocked(lnk->pid))
+ if (is_internal_pid(lnk->pid) && erts_proc_lookup(lnk->pid))
erts_print(pcontext->ptd.to, pcontext->ptd.arg,
"Remote monitoring: %T %T\n", lnk->pid, pcontext->sysname);
}
@@ -2710,9 +2706,8 @@ BIF_RETTYPE dist_exit_3(BIF_ALIST_3)
}
else {
lp_locks = ERTS_PROC_LOCKS_XSIG_SEND;
- lp = erts_pid2proc_opt(BIF_P, ERTS_PROC_LOCK_MAIN,
- local, lp_locks,
- ERTS_P2P_FLG_SMP_INC_REFC);
+ lp = erts_pid2proc(BIF_P, ERTS_PROC_LOCK_MAIN,
+ local, lp_locks);
if (!lp) {
BIF_RET(am_true); /* ignore */
}
@@ -2731,9 +2726,7 @@ BIF_RETTYPE dist_exit_3(BIF_ALIST_3)
lp_locks &= ~ERTS_PROC_LOCK_MAIN;
#endif
erts_smp_proc_unlock(lp, lp_locks);
- if (lp != BIF_P)
- erts_smp_proc_dec_refc(lp);
- else {
+ if (lp == BIF_P) {
/*
* We may have exited current process and may have to take action.
*/
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index bba6b83ac6..b30e6dc018 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -263,6 +263,7 @@ type ZLIB STANDARD SYSTEM zlib
type CPU_GRPS_MAP LONG_LIVED SYSTEM cpu_groups_map
type AUX_WORK_TMO LONG_LIVED SYSTEM aux_work_timeouts
type MISC_AUX_WORK_Q LONG_LIVED SYSTEM misc_aux_work_q
+type PROC_INTERVAL LONG_LIVED SYSTEM process_interval
+if threads_no_smp
# Need thread safe allocs, but std_alloc and fix_alloc are not;
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index f889ccdb93..8117b1bfaa 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -3139,15 +3139,19 @@ BIF_RETTYPE is_process_alive_1(BIF_ALIST_1)
if(internal_pid_index(BIF_ARG_1) >= erts_max_processes)
BIF_ERROR(BIF_P, BADARG);
- rp = erts_pid2proc(BIF_P, ERTS_PROC_LOCK_MAIN,
- BIF_ARG_1, ERTS_PROC_LOCK_STATUS);
+ rp = erts_proc_lookup(BIF_ARG_1);
if (!rp) {
BIF_RET(am_false);
}
else {
- int have_pending_exit = ERTS_PROC_PENDING_EXIT(rp);
- erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS);
- if (have_pending_exit)
+ int px;
+#ifdef ERTS_SMP
+ px = (erts_atomic32_read_acqb(&rp->aflags)
+ & (ERTS_PROC_AFLG_PENDING_EXIT|ERTS_PROC_AFLG_EXITING));
+#else
+ px = 0;
+#endif
+ if (px)
ERTS_BIF_AWAIT_X_DATA_TRAP(BIF_P, BIF_ARG_1, am_false);
else
BIF_RET(am_true);
@@ -3751,9 +3755,8 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
&& is_internal_pid(tp[2])) {
int xres;
ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND;
- Process *rp = erts_pid2proc_opt(BIF_P, ERTS_PROC_LOCK_MAIN,
- tp[2], rp_locks,
- ERTS_P2P_FLG_SMP_INC_REFC);
+ Process *rp = erts_pid2proc(BIF_P, ERTS_PROC_LOCK_MAIN,
+ tp[2], rp_locks);
if (!rp) {
DECL_AM(dead);
BIF_RET(AM_dead);
@@ -3779,7 +3782,6 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
rp_locks &= ~ERTS_PROC_LOCK_MAIN;
#endif
erts_smp_proc_unlock(rp, rp_locks);
- erts_smp_proc_dec_refc(rp);
if (xres > 1) {
DECL_AM(message);
BIF_RET(AM_message);
@@ -3990,7 +3992,7 @@ static Eterm lcnt_build_lock_term(Eterm **hpp, Uint *szp, erts_lcnt_lock_t *lock
id = am_atom_put(ltype, strlen(ltype));
} else if (lock->flag & ERTS_LCNT_LT_PROCLOCK) {
/* use registered names as id's for process locks if available */
- proc = erts_pid2proc_unlocked(lock->id);
+ proc = erts_proc_lookup(lock->id);
if (proc && proc->reg) {
id = proc->reg->name;
} else {
diff --git a/erts/emulator/beam/erl_bif_timer.c b/erts/emulator/beam/erl_bif_timer.c
index d806be0704..525b11f61c 100644
--- a/erts/emulator/beam/erl_bif_timer.c
+++ b/erts/emulator/beam/erl_bif_timer.c
@@ -324,10 +324,9 @@ bif_timer_timeout(ErtsBifTimer* btm)
ASSERT(!erts_get_current_process());
if (btm->flags & BTM_FLG_BYNAME)
- rp = erts_whereis_process(NULL,0,btm->receiver.name,0,ERTS_P2P_FLG_SMP_INC_REFC);
+ rp = erts_whereis_process(NULL, 0, btm->receiver.name, 0, 0);
else {
rp = btm->receiver.proc.ess;
- erts_smp_proc_inc_refc(rp);
unlink_proc(btm);
}
@@ -379,7 +378,6 @@ bif_timer_timeout(ErtsBifTimer* btm)
#endif
);
erts_smp_proc_unlock(rp, rp_locks);
- erts_smp_proc_dec_refc(rp);
}
}
diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c
index 80f774523c..8fc8e363b1 100644
--- a/erts/emulator/beam/erl_bif_trace.c
+++ b/erts/emulator/beam/erl_bif_trace.c
@@ -653,8 +653,7 @@ Eterm trace_3(BIF_ALIST_3)
if (procs || mods) {
/* tracing of processes */
for (i = 0; i < erts_max_processes; i++) {
- Process* tracee_p = process_tab[i];
-
+ Process* tracee_p = erts_pix2proc(i);
if (! tracee_p)
continue;
if (tracer != NIL) {
@@ -772,8 +771,7 @@ static int port_already_traced(Process *c_p, Port *tracee_port, Eterm tracer)
return 1;
}
else if(is_internal_pid(tracee_port->tracer_proc)) {
- Process *tracer_p = erts_pid2proc(c_p, ERTS_PROC_LOCK_MAIN,
- tracee_port->tracer_proc, 0);
+ Process *tracer_p = erts_proc_lookup(tracee_port->tracer_proc);
if (!tracer_p) {
/* Current trace process now invalid
* - discard it and approve the new. */
@@ -813,8 +811,7 @@ static int already_traced(Process *c_p, Process *tracee_p, Eterm tracer)
return 1;
}
else if(is_internal_pid(tracee_p->tracer_proc)) {
- Process *tracer_p = erts_pid2proc(c_p, ERTS_PROC_LOCK_MAIN,
- tracee_p->tracer_proc, 0);
+ Process *tracer_p = erts_proc_lookup(tracee_p->tracer_proc);
if (!tracer_p) {
/* Current trace process now invalid
* - discard it and approve the new. */
@@ -876,7 +873,7 @@ trace_info_pid(Process* p, Eterm pid_spec, Eterm key)
}
if (is_internal_pid(tracer)) {
- if (!erts_pid2proc(p, ERTS_PROC_LOCK_MAIN, tracer, 0)) {
+ if (!erts_proc_lookup(tracer)) {
reset_tracer:
tracee->trace_flags &= ~TRACEE_FLAGS;
trace_flags = tracee->trace_flags;
@@ -2142,9 +2139,9 @@ BIF_RETTYPE system_profile_2(BIF_ALIST_2)
/* Check if valid process, no locks are taken */
if (is_internal_pid(profiler)) {
- if (internal_pid_index(profiler) >= erts_max_processes) goto error;
- profiler_p = process_tab[internal_pid_index(profiler)];
- if (INVALID_PID(profiler_p, profiler)) goto error;
+ profiler_p = erts_proc_lookup(profiler);
+ if (!profiler_p)
+ goto error;
} else if (is_internal_port(profiler)) {
if (internal_port_index(profiler) >= erts_max_ports) goto error;
profiler_port = &erts_port[internal_port_index(profiler)];
diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index 51bdf53823..2b8bcb5bf7 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -2850,7 +2850,7 @@ void init_db(void)
else
db_max_tabs = user_requested_db_max_tabs;
- bits = erts_fit_in_bits(db_max_tabs-1);
+ bits = erts_fit_in_bits_int32(db_max_tabs-1);
if (bits > SMALL_BITS) {
erl_exit(1,"Max limit for ets tabled too high %u (max %u).",
db_max_tabs, ((Uint)1)<<SMALL_BITS);
@@ -3124,7 +3124,7 @@ retry:
if (to_proc == NULL) {
return 0; /* heir not alive, table still mine */
}
- if (erts_cmp_timeval(&to_proc->started, &tb->common.heir_started) != 0) {
+ if (to_proc->started_interval != tb->common.heir_started_interval) {
erts_smp_proc_unlock(to_proc, to_locks);
return 0; /* heir dead and pid reused, table still mine */
}
@@ -3520,14 +3520,14 @@ static void set_heir(Process* me, DbTable* tb, Eterm heir, UWord heir_data)
return;
}
if (heir == me->id) {
- tb->common.heir_started = me->started;
+ erts_ensure_later_proc_interval(me->started_interval);
+ tb->common.heir_started_interval = me->started_interval;
}
else {
- Process* heir_proc= erts_pid2proc_opt(me, ERTS_PROC_LOCK_MAIN, heir,
- 0, ERTS_P2P_FLG_SMP_INC_REFC);
+ Process* heir_proc= erts_proc_lookup(heir);
if (heir_proc != NULL) {
- tb->common.heir_started = heir_proc->started;
- erts_smp_proc_dec_refc(heir_proc);
+ erts_ensure_later_proc_interval(heir_proc->started_interval);
+ tb->common.heir_started_interval = heir_proc->started_interval;
} else {
tb->common.heir = am_none;
}
diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h
index 6a96e174e1..ff5982640d 100644
--- a/erts/emulator/beam/erl_db_util.h
+++ b/erts/emulator/beam/erl_db_util.h
@@ -219,7 +219,7 @@ typedef struct db_table_common {
Eterm owner; /* Pid of the creator */
Eterm heir; /* Pid of the heir */
UWord heir_data; /* To send in ETS-TRANSFER (is_immed or (DbTerm*) */
- SysTimeval heir_started; /* To further identify the heir */
+ Uint64 heir_started_interval; /* To further identify the heir */
Eterm the_name; /* an atom */
Eterm id; /* atom | integer */
DbTableMethod* meth; /* table methods */
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index ca4385dd3a..ead7b40679 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -614,6 +614,7 @@ early_init(int *argc, char **argv) /*
erts_printf_eterm_func = erts_printf_term;
erts_disable_tolerant_timeofday = 0;
display_items = 200;
+ erts_proc.max = ERTS_DEFAULT_MAX_PROCESSES;
erts_backtrace_depth = DEFAULT_BACKTRACE_SIZE;
erts_async_max_threads = 0;
erts_async_thread_suggested_stack_size = ERTS_ASYNC_THREAD_MIN_STACK_SIZE;
@@ -1160,7 +1161,7 @@ erl_start(int argc, char **argv)
case 'P':
/* set maximum number of processes */
Parg = get_arg(argv[i]+2, argv[i+1], &i);
- erts_max_processes = atoi(Parg);
+ erts_proc.max = atoi(Parg);
/* Check of result is delayed until later. This is because +R
may be given after +P. */
break;
@@ -1439,10 +1440,10 @@ erl_start(int argc, char **argv)
}
/* Delayed check of +P flag */
- if (erts_max_processes < ERTS_MIN_PROCESSES
- || erts_max_processes > ERTS_MAX_PROCESSES
+ if (erts_proc.max < ERTS_MIN_PROCESSES
+ || erts_proc.max > ERTS_MAX_PROCESSES
|| (erts_use_r9_pids_ports
- && erts_max_processes > ERTS_MAX_R9_PROCESSES)) {
+ && erts_proc.max > ERTS_MAX_R9_PROCESSES)) {
erts_fprintf(stderr, "bad number of processes %s\n", Parg);
erts_usage();
}
diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index bd86e3ea9e..4a797fbeae 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -363,7 +363,7 @@ erts_queue_dist_message(Process *rcvr,
}
}
- if (rcvr->is_exiting || ERTS_PROC_PENDING_EXIT(rcvr)) {
+ if (ERTS_PROC_IS_EXITING(rcvr) || ERTS_PROC_PENDING_EXIT(rcvr)) {
/* Drop message if receiver is exiting or has a pending exit ... */
if (is_not_nil(token)) {
ErlHeapFragment *heap_frag;
@@ -482,7 +482,7 @@ erts_queue_message(Process* receiver,
}
}
- if (receiver->is_exiting || ERTS_PROC_PENDING_EXIT(receiver)) {
+ if (ERTS_PROC_IS_EXITING(receiver) || ERTS_PROC_PENDING_EXIT(receiver)) {
/* Drop message if receiver is exiting or has a pending
* exit ...
*/
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index b323bc7f69..6d0975f41f 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -311,6 +311,7 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
#endif
Eterm receiver = to_pid->pid;
int flush_me = 0;
+ int scheduler = erts_get_scheduler_id() != 0;
if (env != NULL) {
c_p = env->proc;
@@ -330,8 +331,11 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP)
rp_had_locks = rp_locks;
#endif
- rp = erts_pid2proc_opt(c_p, ERTS_PROC_LOCK_MAIN,
- receiver, rp_locks, ERTS_P2P_FLG_SMP_INC_REFC);
+
+ rp = (scheduler
+ ? erts_proc_lookup(receiver)
+ : erts_pid2proc_opt(c_p, ERTS_PROC_LOCK_MAIN,
+ receiver, rp_locks, ERTS_P2P_FLG_SMP_INC_REFC));
if (rp == NULL) {
ASSERT(env == NULL || receiver != c_p->id);
return 0;
@@ -358,12 +362,12 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
, NIL
#endif
);
- if (rp_locks) {
- ERTS_SMP_LC_ASSERT(rp_locks == (rp_had_locks | (ERTS_PROC_LOCK_MSGQ |
- ERTS_PROC_LOCK_STATUS)));
- erts_smp_proc_unlock(rp, (ERTS_PROC_LOCK_MSGQ | ERTS_PROC_LOCK_STATUS));
- }
- erts_smp_proc_dec_refc(rp);
+ if (c_p == rp)
+ rp_locks &= ~ERTS_PROC_LOCK_MAIN;
+ if (rp_locks)
+ erts_smp_proc_unlock(rp, rp_locks);
+ if (!scheduler)
+ erts_smp_proc_dec_refc(rp);
if (flush_me) {
cache_env(env);
}
diff --git a/erts/emulator/beam/erl_node_container_utils.h b/erts/emulator/beam/erl_node_container_utils.h
index 329a2204cc..7b4cb7b042 100644
--- a/erts/emulator/beam/erl_node_container_utils.h
+++ b/erts/emulator/beam/erl_node_container_utils.h
@@ -128,8 +128,47 @@ extern int erts_use_r9_pids_ports;
* Pids *
\* */
-#define internal_pid_index(x) (internal_pid_data((x)) \
- & erts_process_tab_index_mask)
+#define erts_max_processes erts_proc.max
+
+typedef struct {
+ erts_smp_atomic_t *tab;
+ int max;
+ int tab_cache_lines;
+ int pix_per_cache_line;
+ int pix_cl_mask;
+ int pix_cl_shift;
+ int pix_cli_mask;
+ int pix_cli_shift;
+} ErtsProcTab;
+
+extern ErtsProcTab erts_proc;
+
+ERTS_GLB_INLINE int erts_pid_data2ix(Eterm pid_data);
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+ERTS_GLB_INLINE int erts_pid_data2ix(Eterm pid_data)
+{
+ int n, pix;
+
+ n = (int) pid_data;
+ if (erts_proc.pix_cl_mask) {
+ pix = ((n & erts_proc.pix_cl_mask) << erts_proc.pix_cl_shift);
+ pix += ((n >> erts_proc.pix_cli_shift) & erts_proc.pix_cli_mask);
+ }
+ else {
+ n %= erts_proc.max;
+ pix = n % erts_proc.tab_cache_lines;
+ pix *= erts_proc.pix_per_cache_line;
+ pix += n / erts_proc.tab_cache_lines;
+ }
+ ASSERT(0 <= pix && pix < erts_proc.max);
+ return pix;
+}
+
+#endif
+
+#define internal_pid_index(x) erts_pid_data2ix(internal_pid_data((x)))
#define internal_pid_node_name(x) (internal_pid_node((x))->sysname)
#define external_pid_node_name(x) (external_pid_node((x))->sysname)
diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index 1481f66b55..16367c305d 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -1310,21 +1310,22 @@ setup_reference_table(void)
UnUseTmpHeapNoproc(3);
/* Insert all processes */
- for (i = 0; i < erts_max_processes; i++)
- if (process_tab[i]) {
+ for (i = 0; i < erts_max_processes; i++) {
+ Process *proc = erts_pix2proc(i);
+ if (proc) {
ErlMessage *msg;
/* Insert Heap */
- insert_offheap(&(process_tab[i]->off_heap),
+ insert_offheap(&(proc->off_heap),
HEAP_REF,
- process_tab[i]->id);
+ proc->id);
/* Insert message buffers */
- for(hfp = process_tab[i]->mbuf; hfp; hfp = hfp->next)
+ for(hfp = proc->mbuf; hfp; hfp = hfp->next)
insert_offheap(&(hfp->off_heap),
HEAP_REF,
- process_tab[i]->id);
+ proc->id);
/* Insert msg msg buffers */
- for (msg = process_tab[i]->msg.first; msg; msg = msg->next) {
+ for (msg = proc->msg.first; msg; msg = msg->next) {
ErlHeapFragment *heap_frag = NULL;
if (msg->data.attached) {
if (is_value(ERL_MESSAGE_TERM(msg)))
@@ -1332,7 +1333,7 @@ setup_reference_table(void)
else {
if (msg->data.dist_ext->dep)
insert_dist_entry(msg->data.dist_ext->dep,
- HEAP_REF, process_tab[i]->id, 0);
+ HEAP_REF, proc->id, 0);
if (is_not_nil(ERL_MESSAGE_TOKEN(msg)))
heap_frag = erts_dist_ext_trailer(msg->data.dist_ext);
}
@@ -1340,10 +1341,10 @@ setup_reference_table(void)
if (heap_frag)
insert_offheap(&(heap_frag->off_heap),
HEAP_REF,
- process_tab[i]->id);
+ proc->id);
}
#ifdef ERTS_SMP
- for (msg = process_tab[i]->msg_inq.first; msg; msg = msg->next) {
+ for (msg = proc->msg_inq.first; msg; msg = msg->next) {
ErlHeapFragment *heap_frag = NULL;
if (msg->data.attached) {
if (is_value(ERL_MESSAGE_TERM(msg)))
@@ -1351,7 +1352,7 @@ setup_reference_table(void)
else {
if (msg->data.dist_ext->dep)
insert_dist_entry(msg->data.dist_ext->dep,
- HEAP_REF, process_tab[i]->id, 0);
+ HEAP_REF, proc->id, 0);
if (is_not_nil(ERL_MESSAGE_TOKEN(msg)))
heap_frag = erts_dist_ext_trailer(msg->data.dist_ext);
}
@@ -1359,21 +1360,22 @@ setup_reference_table(void)
if (heap_frag)
insert_offheap(&(heap_frag->off_heap),
HEAP_REF,
- process_tab[i]->id);
+ proc->id);
}
#endif
/* Insert links */
- if(process_tab[i]->nlinks)
- insert_links(process_tab[i]->nlinks, process_tab[i]->id);
- if(process_tab[i]->monitors)
- insert_monitors(process_tab[i]->monitors, process_tab[i]->id);
+ if(proc->nlinks)
+ insert_links(proc->nlinks, proc->id);
+ if(proc->monitors)
+ insert_monitors(proc->monitors, proc->id);
/* Insert controller */
{
- DistEntry *dep = ERTS_PROC_GET_DIST_ENTRY(process_tab[i]);
+ DistEntry *dep = ERTS_PROC_GET_DIST_ENTRY(proc);
if (dep)
- insert_dist_entry(dep, CTRL_REF, process_tab[i]->id, 0);
+ insert_dist_entry(dep, CTRL_REF, proc->id, 0);
}
}
+ }
#ifdef ERTS_SMP
erts_foreach_sys_msg_in_q(insert_sys_msg);
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 4f8e00f7f1..7174991ade 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -95,13 +95,6 @@
#define NORMAL_BIT (1 << PRIORITY_NORMAL)
#define LOW_BIT (1 << PRIORITY_LOW)
-#define ERTS_MAYBE_SAVE_TERMINATING_PROCESS(P) \
-do { \
- ERTS_SMP_LC_ASSERT(erts_lc_mtx_is_locked(&proc_tab_mtx)); \
- if (saved_term_procs.end) \
- save_terminating_process((P)); \
-} while (0)
-
#define ERTS_EMPTY_RUNQ(RQ) \
((RQ)->len == 0 && (RQ)->misc.start == NULL)
@@ -112,16 +105,151 @@ extern BeamInstr beam_apply[];
extern BeamInstr beam_exit[];
extern BeamInstr beam_continue_exit[];
-static Sint p_last;
-static Sint p_next;
-static Sint p_serial;
-static Uint p_serial_mask;
-static Uint p_serial_shift;
+#ifdef ARCH_32
+
+union {
+ erts_smp_dw_atomic_t pid_data;
+ char align[ERTS_CACHE_LINE_SIZE];
+} last erts_align_attribute(ERTS_CACHE_LINE_SIZE);
+
+
+static ERTS_INLINE Uint64
+dw_aint_to_uint64(erts_dw_aint_t *dw)
+{
+#ifdef ETHR_SU_DW_NAINT_T__
+ return (Uint64) dw->dw_sint;
+#else
+ Uint64 res;
+ res = (Uint64) ((Uint32) dw->sint[ERTS_DW_AINT_HIGH_WORD]);
+ res <<= 32;
+ res |= (Uint64) ((Uint32) dw->sint[ERTS_DW_AINT_LOW_WORD]);
+ return res;
+#endif
+}
+
+static void
+unint64_to_dw_aint(erts_dw_aint_t *dw, Uint64 val)
+{
+#ifdef ETHR_SU_DW_NAINT_T__
+ dw->dw_sint = (ETHR_SU_DW_NAINT_T__) val;
+#else
+ dw->sint[ERTS_DW_AINT_LOW_WORD] = (erts_aint_t) (val & 0xffffffff);
+ dw->sint[ERTS_DW_AINT_HIGH_WORD] = (erts_aint_t) ((val >> 32) & 0xffffffff);
+#endif
+}
+
+static ERTS_INLINE void
+last_pid_data_init_nob(Uint64 val)
+{
+ erts_dw_aint_t dw;
+ unint64_to_dw_aint(&dw, val);
+ erts_smp_dw_atomic_init_nob(&last.pid_data, &dw);
+}
+
+static ERTS_INLINE void
+last_pid_data_set_relb(Uint64 val)
+{
+ erts_dw_aint_t dw;
+ unint64_to_dw_aint(&dw, val);
+ erts_smp_dw_atomic_set_relb(&last.pid_data, &dw);
+}
+
+static ERTS_INLINE Uint64
+last_pid_data_read_nob(void)
+{
+ erts_dw_aint_t dw;
+ erts_smp_dw_atomic_read_nob(&last.pid_data, &dw);
+ return dw_aint_to_uint64(&dw);
+}
+
+static ERTS_INLINE Uint64
+last_pid_data_read_acqb(void)
+{
+ erts_dw_aint_t dw;
+ erts_smp_dw_atomic_read_acqb(&last.pid_data, &dw);
+ return dw_aint_to_uint64(&dw);
+}
+
+static ERTS_INLINE Uint64
+last_pid_data_cmpxchg_relb(Uint64 new, Uint64 exp)
+{
+ erts_dw_aint_t dw_new, dw_xchg;
+
+ unint64_to_dw_aint(&dw_new, new);
+ unint64_to_dw_aint(&dw_xchg, exp);
+
+ if (erts_smp_dw_atomic_cmpxchg_relb(&last.pid_data, &dw_new, &dw_xchg))
+ return exp;
+ else
+ return dw_aint_to_uint64(&dw_xchg);
+}
+
+#elif defined(ARCH_64)
+
+union {
+ erts_smp_atomic_t pid_data;
+ char align[ERTS_CACHE_LINE_SIZE];
+} last erts_align_attribute(ERTS_CACHE_LINE_SIZE);
+
+static ERTS_INLINE void
+last_pid_data_init_nob(Uint64 val)
+{
+ erts_smp_atomic_init_nob(&last.pid_data, (erts_aint_t) val);
+}
+
+static ERTS_INLINE void
+last_pid_data_set_relb(Uint64 val)
+{
+ erts_smp_atomic_set_relb(&last.pid_data, (erts_aint_t) val);
+}
+
+static ERTS_INLINE Uint64
+last_pid_data_read_nob(void)
+{
+ return (Uint64) erts_smp_atomic_read_nob(&last.pid_data);
+}
+
+static ERTS_INLINE Uint64
+last_pid_data_read_acqb(void)
+{
+ return (Uint64) erts_smp_atomic_read_acqb(&last.pid_data);
+}
+
+static ERTS_INLINE Uint64
+last_pid_data_cmpxchg_relb(Uint64 new, Uint64 exp)
+{
+ return (Uint64) erts_smp_atomic_cmpxchg_relb(&last.pid_data,
+ (erts_aint_t) new,
+ (erts_aint_t) exp);
+}
+
+#else
+# error "Not 64-bit, nor 32-bit architecture..."
+#endif
+
+static ERTS_INLINE int
+last_pid_data_cmp(Uint64 lpd1, Uint64 lpd2)
+{
+ Uint64 lpd1_wrap;
+
+ if (lpd1 == lpd2)
+ return 0;
+
+ lpd1_wrap = lpd1 + (((Uint64) 1) << 63);
+
+ if (lpd1 < lpd1_wrap)
+ return (lpd1 < lpd2 && lpd2 < lpd1_wrap) ? -1 : 1;
+ else
+ return (lpd1_wrap <= lpd2 && lpd2 < lpd1) ? 1 : -1;
+}
+
+
+#define ERTS_PID_DATA_MASK__ ((1 << _PID_DATA_SIZE) - 1)
int erts_sched_compact_load;
Uint erts_no_schedulers;
-Uint erts_max_processes = ERTS_DEFAULT_MAX_PROCESSES;
-Uint erts_process_tab_index_mask;
+
+ErtsProcTab erts_proc erts_align_attribute(ERTS_CACHE_LINE_SIZE);
static int wakeup_other_limit;
@@ -204,7 +332,7 @@ erts_sched_stat_t erts_sched_stat;
static erts_tsd_key_t sched_data_key;
#endif
-static erts_smp_mtx_t proc_tab_mtx;
+erts_smp_rwmtx_t erts_proc_tab_rwmtx;
static erts_smp_atomic32_t function_calls;
@@ -227,7 +355,6 @@ typedef union {
static ErtsAlignedSchedulerSleepInfo *aligned_sched_sleep_info;
-Process** process_tab;
static Uint last_reductions;
static Uint last_exact_reductions;
Uint erts_default_process_flags;
@@ -258,11 +385,11 @@ struct ErtsTermProcElement_ {
union {
struct {
Eterm pid;
- SysTimeval spawned;
- SysTimeval exited;
+ Uint64 spawned;
+ Uint64 exited;
} process;
struct {
- SysTimeval time;
+ Uint64 interval;
} bif_invocation;
} u;
};
@@ -399,6 +526,53 @@ erts_smp_lc_runq_is_locked(ErtsRunQueue *runq)
}
#endif
+static erts_interval_t *proc_interval;
+
+static void
+proc_interval_init(void)
+{
+ proc_interval = erts_alloc_permanent_cache_aligned(
+ ERTS_ALC_T_PROC_INTERVAL,
+ sizeof(erts_interval_t));
+ erts_smp_interval_init(proc_interval);
+}
+
+static ERTS_INLINE Uint64
+get_proc_interval(void)
+{
+ return erts_smp_current_interval_nob(proc_interval);
+}
+
+static ERTS_INLINE Uint64
+ensure_later_proc_interval(Uint64 interval)
+{
+ return erts_smp_ensure_later_interval_nob(proc_interval, interval);
+}
+
+static ERTS_INLINE Uint64
+step_proc_interval(void)
+{
+ return erts_smp_step_interval_nob(proc_interval);
+}
+
+Uint64
+erts_get_proc_interval(void)
+{
+ return get_proc_interval();
+}
+
+Uint64
+erts_ensure_later_proc_interval(Uint64 interval)
+{
+ return ensure_later_proc_interval(interval);
+}
+
+Uint64
+erts_step_proc_interval(void)
+{
+ return step_proc_interval();
+}
+
void
erts_pre_init_process(void)
{
@@ -448,7 +622,16 @@ erts_pre_init_process(void)
void
erts_init_process(int ncpu)
{
- Uint proc_bits = ERTS_PROC_BITS;
+ int proc_tab_sz;
+ int max_proc_bits;
+ int proc_bits = ERTS_PROC_BITS;
+ erts_smp_atomic_t *proc_entry;
+ char *proc_tab_end;
+ erts_smp_rwmtx_opt_t proc_tab_rwmtx_opts = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
+ proc_tab_rwmtx_opts.type = ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ;
+ proc_tab_rwmtx_opts.lived = ERTS_SMP_RWMTX_LONG_LIVED;
+
+ proc_interval_init();
#ifdef ERTS_SMP
erts_disable_proc_not_running_opt = 0;
@@ -459,29 +642,57 @@ erts_init_process(int ncpu)
erts_smp_atomic32_init_nob(&process_count, 0);
- if (erts_use_r9_pids_ports) {
+ if (erts_use_r9_pids_ports)
proc_bits = ERTS_R9_PROC_BITS;
- ASSERT(erts_max_processes <= (1 << ERTS_R9_PROC_BITS));
- }
- process_tab = (Process**) erts_alloc(ERTS_ALC_T_PROC_TABLE,
- erts_max_processes*sizeof(Process*));
- sys_memzero(process_tab, erts_max_processes * sizeof(Process*));
+ if (erts_proc.max > (1 << proc_bits))
+ erts_proc.max = 1 << proc_bits;
+
+ proc_tab_sz = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(erts_proc.max
+ * sizeof(erts_smp_atomic_t));
+ erts_proc.tab = erts_alloc(ERTS_ALC_T_PROC_TABLE, proc_tab_sz);
+ proc_tab_end = ((char *) erts_proc.tab) + proc_tab_sz;
+ proc_entry = erts_proc.tab;
+ while (proc_tab_end > ((char *) proc_entry)) {
+ erts_smp_atomic_init_nob(proc_entry, ERTS_AINT_NULL);
+ proc_entry++;
+ }
#ifdef HYBRID
erts_active_procs = (Process**)
erts_alloc(ERTS_ALC_T_ACTIVE_PROCS,
- erts_max_processes * sizeof(Process*));
+ erts_proc.max * sizeof(Process*));
erts_num_active_procs = 0;
#endif
- erts_smp_mtx_init(&proc_tab_mtx, "proc_tab");
- p_last = -1;
- p_next = 0;
- p_serial = 0;
+ erts_smp_rwmtx_init_opt(&erts_proc_tab_rwmtx,
+ &proc_tab_rwmtx_opts,
+ "proc_tab");
+ last_pid_data_init_nob(~((Uint64) 0));
+
+ max_proc_bits = erts_fit_in_bits_int32((Sint32) erts_proc.max - 1);
+
+ erts_proc.tab_cache_lines = proc_tab_sz/ERTS_CACHE_LINE_SIZE;
+ erts_proc.pix_per_cache_line = ERTS_CACHE_LINE_SIZE/sizeof(erts_smp_atomic_t);
+ if ((erts_proc.max & (erts_proc.max - 1))
+ | (erts_proc.pix_per_cache_line & (erts_proc.pix_per_cache_line - 1))) {
+ /*
+ * erts_proc.max or erts_proc.pix_per_cache_line
+ * not a power of 2 :(
+ */
+ erts_proc.pix_cl_mask = 0;
+ erts_proc.pix_cl_shift = 0;
+ erts_proc.pix_cli_mask = 0;
+ erts_proc.pix_cli_shift = 0;
+ }
+ else {
+ ASSERT((erts_proc.tab_cache_lines
+ & (erts_proc.tab_cache_lines - 1)) == 0);
+ erts_proc.pix_cl_mask = erts_proc.tab_cache_lines-1;
+ erts_proc.pix_cl_shift = erts_fit_in_bits_int32(erts_proc.pix_per_cache_line-1);
+ erts_proc.pix_cli_shift = erts_fit_in_bits_int32(erts_proc.pix_cl_mask);
+ erts_proc.pix_cli_mask = (1 << (max_proc_bits - erts_proc.pix_cli_shift)) - 1;
+ }
- p_serial_shift = erts_fit_in_bits(erts_max_processes - 1);
- p_serial_mask = ((~(~((Uint) 0) << proc_bits)) >> p_serial_shift);
- erts_process_tab_index_mask = ~(~((Uint) 0) << p_serial_shift);
last_reductions = 0;
last_exact_reductions = 0;
erts_default_process_flags = 0;
@@ -739,8 +950,9 @@ static ERTS_INLINE ErtsProcList *
proclist_create(Process *p)
{
ErtsProcList *plp = proclist_alloc();
+ ensure_later_proc_interval(p->started_interval);
plp->pid = p->id;
- plp->started = p->started;
+ plp->started_interval = p->started_interval;
return plp;
}
@@ -753,8 +965,7 @@ proclist_destroy(ErtsProcList *plp)
static ERTS_INLINE int
proclist_same(ErtsProcList *plp, Process *p)
{
- return (plp->pid == p->id
- && erts_cmp_timeval(&plp->started, &p->started) == 0);
+ return plp->pid == p->id && plp->started_interval == p->started_interval;
}
ErtsProcList *
@@ -5162,7 +5373,7 @@ do_bif_suspend_process(ErtsSuspendMonitor *smon,
ErtsRunQueue *locked_runq)
{
ASSERT(suspendee);
- ASSERT(!suspendee->is_exiting);
+ ASSERT(!ERTS_PROC_IS_EXITING(suspendee));
ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS
& erts_proc_lc_my_proc_locks(suspendee));
if (smon) {
@@ -6338,27 +6549,14 @@ Process *schedule(Process *p, int calls)
#ifdef ERTS_SMP
ASSERT(esdp->free_process == p);
esdp->free_process = NULL;
- erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS);
- erts_smp_proc_dec_refc(p);
#else
erts_free_proc(p);
#endif
- } else {
- erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS);
}
-#ifdef ERTS_SMP
- {
- ErtsProcList *pnd_xtrs = rq->procs.pending_exiters;
- rq->procs.pending_exiters = NULL;
+ erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS);
- if (pnd_xtrs) {
- erts_smp_runq_unlock(rq);
- handle_pending_exiters(pnd_xtrs);
- erts_smp_runq_lock(rq);
- }
-
- }
+#ifdef ERTS_SMP
ASSERT(!esdp->free_process);
#endif
ASSERT(!esdp->current_process);
@@ -6381,6 +6579,20 @@ Process *schedule(Process *p, int calls)
#ifdef ERTS_SMP
+#ifdef ERTS_SMP
+ {
+ ErtsProcList *pnd_xtrs = rq->procs.pending_exiters;
+ rq->procs.pending_exiters = NULL;
+
+ if (pnd_xtrs) {
+ erts_smp_runq_unlock(rq);
+ handle_pending_exiters(pnd_xtrs);
+ erts_smp_runq_lock(rq);
+ }
+
+ }
+#endif
+
if (rq->check_balance_reds <= 0)
check_balance(rq);
@@ -6918,44 +7130,50 @@ erts_get_exact_total_reductions(Process *c_p, Uint *redsp, Uint *diffp)
Sint
erts_test_next_pid(int set, Uint next)
{
+ Uint64 lpd;
Sint res;
- Sint p_prev;
+ Eterm pid_data;
+ int first_pix = -1;
+ erts_smp_rwmtx_rwlock(&erts_proc_tab_rwmtx);
- erts_smp_mtx_lock(&proc_tab_mtx);
-
- if (!set) {
- res = p_next < 0 ? -1 : (p_serial << p_serial_shift | p_next);
- }
+ if (!set)
+ lpd = last_pid_data_read_nob();
else {
- p_serial = (Sint) ((next >> p_serial_shift) & p_serial_mask);
- p_next = (Sint) (erts_process_tab_index_mask & next);
-
- if (p_next >= erts_max_processes) {
- p_next = 0;
- p_serial++;
- p_serial &= p_serial_mask;
+ lpd = (Uint64) next;
+ pid_data = (Eterm) (lpd & ERTS_PID_DATA_MASK__);
+ if (ERTS_INVALID_PID == make_internal_pid(pid_data)) {
+ lpd += erts_proc.max;
+ ASSERT(erts_pid_data2ix(pid_data)
+ == erts_pid_data2ix(lpd & ERTS_PID_DATA_MASK__));
}
+ last_pid_data_set_relb(lpd);
+ }
- p_prev = p_next;
-
- do {
- if (!process_tab[p_next])
- break;
- p_next++;
- if(p_next >= erts_max_processes) {
- p_next = 0;
- p_serial++;
- p_serial &= p_serial_mask;
+ while (1) {
+ int pix;
+ lpd++;
+ pix = (int) (lpd % erts_proc.max);
+ if (first_pix < 0)
+ first_pix = pix;
+ else if (pix == first_pix) {
+ res = -1;
+ break;
+ }
+ if (ERTS_AINT_NULL == erts_smp_atomic_read_nob(&erts_proc.tab[pix])) {
+ pid_data = (Eterm) (lpd & ERTS_PID_DATA_MASK__);
+ if (ERTS_INVALID_PID == make_internal_pid(pid_data)) {
+ lpd += erts_proc.max;
+ ASSERT(erts_pid_data2ix(pid_data)
+ == erts_pid_data2ix(lpd & ERTS_PID_DATA_MASK__));
}
- } while (p_prev != p_next);
-
- res = process_tab[p_next] ? -1 : (p_serial << p_serial_shift | p_next);
-
+ res = lpd & ERTS_PID_DATA_MASK__;
+ break;
+ }
}
- erts_smp_mtx_unlock(&proc_tab_mtx);
+ erts_smp_rwmtx_rwunlock(&erts_proc_tab_rwmtx);
return res;
@@ -6964,6 +7182,8 @@ erts_test_next_pid(int set, Uint next)
Uint erts_process_count(void)
{
erts_aint32_t res = erts_smp_atomic32_read_nob(&process_count);
+ if (res > erts_proc.max)
+ return erts_proc.max;
ASSERT(res >= 0);
return (Uint) res;
}
@@ -6977,91 +7197,126 @@ erts_free_proc(Process *p)
erts_free(ERTS_ALC_T_PROC, (void *) p);
}
-
/*
** Allocate process and find out where to place next process.
*/
static Process*
alloc_process(void)
{
-#ifdef ERTS_SMP
- erts_pix_lock_t *pix_lock;
-#endif
+ int pix;
Process* p;
- int p_prev;
+ Uint64 lpd, exp_lpd;
+ Eterm pid_data;
+ erts_aint32_t proc_count;
+#ifdef DEBUG
+ Eterm pid;
+#endif
- erts_smp_mtx_lock(&proc_tab_mtx);
+ erts_smp_rwmtx_rlock(&erts_proc_tab_rwmtx);
- if (p_next == -1) {
- p = NULL;
- goto error; /* Process table full! */
+ proc_count = erts_smp_atomic32_inc_read_acqb(&process_count);
+ if (proc_count > erts_proc.max) {
+ while (1) {
+ erts_aint32_t act_proc_count;
+
+ act_proc_count = erts_smp_atomic32_cmpxchg_relb(&process_count,
+ proc_count-1,
+ proc_count);
+ if (act_proc_count == proc_count)
+ goto system_limit;
+ proc_count = act_proc_count;
+ if (proc_count <= erts_proc.max)
+ break;
+ }
}
p = (Process*) erts_alloc_fnf(ERTS_ALC_T_PROC, sizeof(Process));
if (!p)
- goto error; /* ENOMEM */
+ goto enomem;
- p_last = p_next;
+ p->approx_started = erts_get_approx_time();
+ p->started_interval = get_proc_interval();
- erts_get_emu_time(&p->started);
+ lpd = last_pid_data_read_acqb();
-#ifdef ERTS_SMP
- pix_lock = ERTS_PIX2PIXLOCK(p_next);
- erts_pix_lock(pix_lock);
-#endif
- ASSERT(!process_tab[p_next]);
+ /* Reserve slot */
+ while (1) {
+ lpd++;
+ pix = erts_pid_data2ix((Eterm) (lpd & ERTS_PID_DATA_MASK__));
+ if (erts_smp_atomic_read_nob(&erts_proc.tab[pix]) == ERTS_AINT_NULL) {
+ erts_aint_t val;
+ val = erts_smp_atomic_cmpxchg_relb(&erts_proc.tab[pix],
+ ((erts_aint_t)
+ ERTS_PROC_LOCK_BUSY),
+ ERTS_AINT_NULL);
+
+ if (ERTS_AINT_NULL == val)
+ break;
+ }
+ }
+
+ pid_data = (Eterm) lpd & ERTS_PID_DATA_MASK__;
- process_tab[p_next] = p;
- erts_smp_atomic32_inc_nob(&process_count);
- p->id = make_internal_pid(p_serial << p_serial_shift | p_next);
+ p->id = make_internal_pid(pid_data);
if (p->id == ERTS_INVALID_PID) {
/* Do not use the invalid pid; change serial */
- p_serial++;
- p_serial &= p_serial_mask;
- p->id = make_internal_pid(p_serial << p_serial_shift | p_next);
+ lpd += erts_proc.max;
+ ASSERT(pix == erts_pid_data2ix((Eterm) (lpd & ERTS_PID_DATA_MASK__)));
+ pid_data = (Eterm) lpd & ERTS_PID_DATA_MASK__;
+ p->id = make_internal_pid(pid_data);
ASSERT(p->id != ERTS_INVALID_PID);
}
- ASSERT(internal_pid_serial(p->id) <= (erts_use_r9_pids_ports
- ? ERTS_MAX_PID_R9_SERIAL
- : ERTS_MAX_PID_SERIAL));
-#ifdef ERTS_SMP
- erts_proc_lock_init(p); /* All locks locked */
- erts_pix_unlock(pix_lock);
-#endif
+ exp_lpd = last_pid_data_read_nob();
- p->rstatus = P_FREE;
- p->rcount = 0;
+ /* Move last pid data forward */
+ while (1) {
+ Uint64 act_lpd;
+ if (last_pid_data_cmp(lpd, exp_lpd) < 0)
+ break;
+ act_lpd = last_pid_data_cmpxchg_relb(lpd, exp_lpd);
+ if (act_lpd == exp_lpd)
+ break;
+ exp_lpd = act_lpd;
+ }
- /*
- * set p_next to the next available slot
- */
+#ifdef DEBUG
+ pid = p->id;
+#endif
- p_prev = p_next;
+#ifdef ERTS_SMP
+ erts_proc_lock_init(p); /* All locks locked */
+#endif
- while (1) {
- p_next++;
- if(p_next >= erts_max_processes) {
- p_serial++;
- p_serial &= p_serial_mask;
- p_next = 0;
- }
+ /* Move into slot reserved */
+#ifdef DEBUG
+ ASSERT(ERTS_PROC_LOCK_BUSY
+ == (Process *) erts_smp_atomic_xchg_relb(&erts_proc.tab[pix],
+ (erts_aint_t) p));
+#else
+ erts_smp_atomic_set_relb(&erts_proc.tab[pix], (erts_aint_t) p);
+#endif
- if (p_prev == p_next) {
- p_next = -1;
- break; /* Table full! */
- }
+ ASSERT(internal_pid_serial(p->id) <= (erts_use_r9_pids_ports
+ ? ERTS_MAX_PID_R9_SERIAL
+ : ERTS_MAX_PID_SERIAL));
- if (!process_tab[p_next])
- break; /* found a free slot */
- }
+ erts_smp_rwmtx_runlock(&erts_proc_tab_rwmtx);
- error:
+ p->rstatus = P_FREE;
+ p->rcount = 0;
- erts_smp_mtx_unlock(&proc_tab_mtx);
+ ASSERT(p == (Process *)
+ erts_smp_atomic_read_nob(
+ &erts_proc.tab[internal_pid_index(pid)]));
return p;
+enomem:
+system_limit:
+
+ erts_smp_rwmtx_runlock(&erts_proc_tab_rwmtx);
+ return NULL;
}
Eterm
@@ -7089,7 +7344,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
/*
* Copy the arguments to the global heap
* Since global GC might occur we want to do this before adding the
- * new process to the process_tab.
+ * new process to the erts_proc.tab.
*/
BM_SWAP_TIMER(system,copy);
LAZY_COPY(parent,args);
@@ -7246,7 +7501,12 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
: STORE_NC(&p->htop, &p->off_heap, parent->group_leader);
}
+#if 1
+ p->trace_flags = ~TRACEE_FLAGS;
+ p->tracer_proc = NIL;
+#else
erts_get_default_tracing(&p->trace_flags, &p->tracer_proc);
+#endif
p->msg.first = NULL;
p->msg.last = &p->msg.first;
@@ -7358,7 +7618,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
#ifdef ERTS_SMP
p->scheduler_data = NULL;
- p->is_exiting = 0;
+ erts_atomic32_init_nob(&p->aflags, 0);
p->status_flags = 0;
p->runq_flags = 0;
p->suspendee = NIL;
@@ -7510,8 +7770,8 @@ void erts_init_empty_process(Process *p)
p->def_arg_reg[5] = 0;
p->parent = NIL;
- p->started.tv_sec = 0;
- p->started.tv_usec = 0;
+ p->approx_started = 0;
+ p->started_interval = 0;
#ifdef HIPE
hipe_init_process(&p->hipe);
@@ -7536,7 +7796,7 @@ void erts_init_empty_process(Process *p)
#ifdef ERTS_SMP
p->scheduler_data = NULL;
- p->is_exiting = 0;
+ erts_atomic32_init_nob(&p->aflags, 0);
p->status_flags = 0;
p->runq_flags = 0;
p->msg_inq.first = NULL;
@@ -7757,7 +8017,7 @@ set_proc_exiting(Process *p, Eterm reason, ErlHeapFragment *bp)
*/
erts_pix_lock(pix_lock);
- p->is_exiting = 1;
+ erts_atomic32_read_bor_relb(&p->aflags, ERTS_PROC_AFLG_EXITING);
#endif
p->status = P_EXITING;
#ifdef ERTS_SMP
@@ -7849,7 +8109,7 @@ save_pending_exiter(Process *p)
rq->procs.pending_exiters = plp;
erts_smp_runq_unlock(rq);
-
+ wake_scheduler(rq, 1);
}
#endif
@@ -8021,7 +8281,7 @@ send_exit_signal(Process *c_p, /* current process if and only
}
else if (reason != am_normal || (flags & ERTS_XSIG_FLG_NO_IGN_NORMAL)) {
#ifdef ERTS_SMP
- if (!ERTS_PROC_PENDING_EXIT(rp) && !rp->is_exiting) {
+ if (!ERTS_PROC_PENDING_EXIT(rp) && !ERTS_PROC_IS_EXITING(rp)) {
ASSERT(rp->status != P_EXITING);
ASSERT(rp->status != P_FREE);
ASSERT(!rp->pending_exit.bp);
@@ -8087,6 +8347,8 @@ send_exit_signal(Process *c_p, /* current process if and only
&bp->off_heap);
rp->pending_exit.bp = bp;
}
+ erts_atomic32_read_bor_relb(&rp->aflags,
+ ERTS_PROC_AFLG_PENDING_EXIT);
ASSERT(ERTS_PROC_PENDING_EXIT(rp));
}
if (!(rp->status_flags
@@ -8417,6 +8679,15 @@ resume_suspend_monitor(ErtsSuspendMonitor *smon, void *vc_p)
erts_destroy_suspend_monitor(smon);
}
+#ifdef ERTS_SMP
+static void
+proc_dec_refc(void *vproc)
+{
+ erts_smp_proc_dec_refc((Process *) vproc);
+}
+#endif
+
+
static void
continue_exit_process(Process *p
#ifdef ERTS_SMP
@@ -8462,7 +8733,7 @@ erts_do_exit_process(Process* p, Eterm reason)
#ifdef ERTS_SMP
erts_pix_lock(pix_lock);
- p->is_exiting = 1;
+ erts_atomic32_read_bor_relb(&p->aflags, ERTS_PROC_AFLG_EXITING);
#endif
p->status = P_EXITING;
@@ -8619,20 +8890,24 @@ continue_exit_process(Process *p
#endif
{
+ int maybe_save;
int pix;
/* Do *not* use erts_get_runq_proc() */
ErtsRunQueue *rq;
rq = erts_get_runq_current(ERTS_GET_SCHEDULER_DATA_FROM_PROC(p));
- ASSERT(internal_pid_index(p->id) < erts_max_processes);
pix = internal_pid_index(p->id);
- erts_smp_mtx_lock(&proc_tab_mtx);
+ erts_smp_rwmtx_rlock(&erts_proc_tab_rwmtx);
+ maybe_save = saved_term_procs.end != NULL;
+ if (maybe_save) {
+ erts_smp_rwmtx_runlock(&erts_proc_tab_rwmtx);
+ erts_smp_rwmtx_rwlock(&erts_proc_tab_rwmtx);
+ }
+
erts_smp_runq_lock(rq);
#ifdef ERTS_SMP
- erts_pix_lock(pix_lock);
-
ASSERT(p->scheduler_data);
ASSERT(p->scheduler_data->current_process == p);
ASSERT(p->scheduler_data->free_process == NULL);
@@ -8641,26 +8916,22 @@ continue_exit_process(Process *p
p->scheduler_data->free_process = p;
p->status_flags = 0;
#endif
- process_tab[pix] = NULL; /* Time of death! */
+ /* Time of death! */
+ erts_smp_atomic_set_relb(&erts_proc.tab[pix], ERTS_AINT_NULL);
+
ASSERT(erts_smp_atomic32_read_nob(&process_count) > 0);
- erts_smp_atomic32_dec_nob(&process_count);
+ erts_smp_atomic32_dec_relb(&process_count);
-#ifdef ERTS_SMP
- erts_pix_unlock(pix_lock);
-#endif
erts_smp_runq_unlock(rq);
- if (p_next < 0) {
- if (p_last >= p_next) {
- p_serial++;
- p_serial &= p_serial_mask;
- }
- p_next = pix;
+ if (!maybe_save)
+ erts_smp_rwmtx_runlock(&erts_proc_tab_rwmtx);
+ else {
+ if (saved_term_procs.end)
+ save_terminating_process(p);
+ erts_smp_rwmtx_rwunlock(&erts_proc_tab_rwmtx);
}
- ERTS_MAYBE_SAVE_TERMINATING_PROCESS(p);
-
- erts_smp_mtx_unlock(&proc_tab_mtx);
}
/*
@@ -8730,8 +9001,25 @@ continue_exit_process(Process *p
delete_process(p);
+#ifdef ERTS_SMP
+ /*
+ * Each scheduler will decrease refc by one via misc aux work;
+ * we have one refc for reference from process table which we
+ * now want to remove, i.e. we increase refc with schedulers-1.
+ *
+ * Process struct wont be deallocated until (earliest) when
+ * all schedulers have decreased refc via misc aux work...
+ */
+ if (erts_no_schedulers != 1)
+ erts_smp_proc_add_refc(p, (Sint32) erts_no_schedulers-1);
+ erts_schedule_multi_misc_aux_work(0,
+ erts_no_schedulers,
+ proc_dec_refc,
+ (void *) p);
+
erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN);
ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p);
+#endif
return;
@@ -9041,13 +9329,13 @@ do { \
#endif
#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS
-# define ERTS_PROCS_DBG_CHK_PID_FOUND(PBDP, PID, TVP) \
- debug_processes_check_found_pid((PBDP), (PID), (TVP), 1)
-# define ERTS_PROCS_DBG_CHK_PID_NOT_FOUND(PBDP, PID, TVP) \
- debug_processes_check_found_pid((PBDP), (PID), (TVP), 0)
+# define ERTS_PROCS_DBG_CHK_PID_FOUND(PBDP, PID, IC) \
+ debug_processes_check_found_pid((PBDP), (PID), (IC), 1)
+# define ERTS_PROCS_DBG_CHK_PID_NOT_FOUND(PBDP, PID, IC) \
+ debug_processes_check_found_pid((PBDP), (PID), (IC), 0)
#else
-# define ERTS_PROCS_DBG_CHK_PID_FOUND(PBDP, PID, TVP)
-# define ERTS_PROCS_DBG_CHK_PID_NOT_FOUND(PBDP, PID, TVP)
+# define ERTS_PROCS_DBG_CHK_PID_FOUND(PBDP, PID, IC)
+# define ERTS_PROCS_DBG_CHK_PID_NOT_FOUND(PBDP, PID, IC)
#endif
#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_TERM_PROC_LIST
@@ -9092,7 +9380,7 @@ static Uint processes_bif_tab_chunks;
static Export processes_trap_export;
typedef struct {
- SysTimeval time;
+ Uint64 interval;
} ErtsProcessesBifChunkInfo;
typedef enum {
@@ -9117,7 +9405,7 @@ typedef struct {
struct {
Eterm caller;
#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS
- SysTimeval *pid_started;
+ Uint64 *pid_started;
#endif
#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_HALLOC
Eterm *heap;
@@ -9146,11 +9434,10 @@ static void debug_processes_verify_all_pids(ErtsProcessesBifData *pbdp);
#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS
static void debug_processes_check_found_pid(ErtsProcessesBifData *pbdp,
Eterm pid,
- SysTimeval *started,
+ Uint64 ic,
int pid_should_be_found);
#endif
#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_TERM_PROC_LIST
-static SysTimeval debug_tv_start;
static void debug_processes_check_term_proc_list(void);
static void debug_processes_check_term_proc_free_list(ErtsTermProcElement *tpep);
#endif
@@ -9161,7 +9448,7 @@ save_terminating_process(Process *p)
ErtsTermProcElement *tpep = erts_alloc(ERTS_ALC_T_PROCS_TPROC_EL,
sizeof(ErtsTermProcElement));
ERTS_PROCS_ASSERT(saved_term_procs.start && saved_term_procs.end);
- ERTS_SMP_LC_ASSERT(erts_lc_mtx_is_locked(&proc_tab_mtx));
+ ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&erts_proc_tab_rwmtx));
ERTS_PROCS_DBG_CHK_TPLIST();
@@ -9169,19 +9456,19 @@ save_terminating_process(Process *p)
tpep->next = NULL;
tpep->ix = internal_pid_index(p->id);
tpep->u.process.pid = p->id;
- tpep->u.process.spawned = p->started;
- erts_get_emu_time(&tpep->u.process.exited);
+ tpep->u.process.spawned = p->started_interval;
+ tpep->u.process.exited = get_proc_interval();
saved_term_procs.end->next = tpep;
saved_term_procs.end = tpep;
ERTS_PROCS_DBG_CHK_TPLIST();
- ERTS_PROCS_ASSERT((tpep->prev->ix >= 0
- ? erts_cmp_timeval(&tpep->u.process.exited,
- &tpep->prev->u.process.exited)
- : erts_cmp_timeval(&tpep->u.process.exited,
- &tpep->prev->u.bif_invocation.time)) > 0);
+ ERTS_PROCS_ASSERT(tpep->prev->ix >= 0
+ ? (tpep->u.process.exited
+ >= tpep->prev->u.process.exited)
+ : (tpep->u.process.exited
+ >= tpep->prev->u.bif_invocation.interval));
}
static void
@@ -9212,7 +9499,7 @@ cleanup_processes_bif_data(Binary *bp)
if (pbdp->bif_invocation) {
ErtsTermProcElement *tpep;
- erts_smp_mtx_lock(&proc_tab_mtx);
+ erts_smp_rwmtx_rwlock(&erts_proc_tab_rwmtx);
ERTS_PROCS_DBG_TRACE(pbdp->debug.caller,
cleanup_processes_bif_data,
@@ -9266,7 +9553,7 @@ cleanup_processes_bif_data(Binary *bp)
ERTS_PROCS_DBG_CHK_TPLIST();
- erts_smp_mtx_unlock(&proc_tab_mtx);
+ erts_smp_rwmtx_rwunlock(&erts_proc_tab_rwmtx);
}
}
@@ -9294,7 +9581,7 @@ processes_bif_engine(Process *p, Eterm *res_accp, Binary *mbp)
pbdp->tix = 0;
pbdp->pid_ix = 0;
- erts_smp_mtx_lock(&proc_tab_mtx);
+ erts_smp_rwmtx_rwlock(&erts_proc_tab_rwmtx);
locked = 1;
ERTS_PROCS_DBG_TRACE(p->id, processes_bif_engine, init);
@@ -9305,7 +9592,7 @@ processes_bif_engine(Process *p, Eterm *res_accp, Binary *mbp)
#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS
pbdp->debug.pid_started = erts_alloc(ERTS_ALC_T_PROCS_PIDS,
- sizeof(SysTimeval)*pbdp->pid_sz);
+ sizeof(Uint64)*pbdp->pid_sz);
#endif
ERTS_PROCS_DBG_SAVE_PIDS(pbdp);
@@ -9320,7 +9607,8 @@ processes_bif_engine(Process *p, Eterm *res_accp, Binary *mbp)
pbdp->bif_invocation = erts_alloc(ERTS_ALC_T_PROCS_TPROC_EL,
sizeof(ErtsTermProcElement));
pbdp->bif_invocation->ix = -1;
- erts_get_emu_time(&pbdp->bif_invocation->u.bif_invocation.time);
+ pbdp->bif_invocation->u.bif_invocation.interval
+ = step_proc_interval();
ERTS_PROCS_DBG_CHK_TPLIST();
pbdp->bif_invocation->next = NULL;
@@ -9347,30 +9635,31 @@ processes_bif_engine(Process *p, Eterm *res_accp, Binary *mbp)
int indices = ERTS_PROCESSES_BIF_TAB_CHUNK_SIZE;
int cix = ix / ERTS_PROCESSES_BIF_TAB_CHUNK_SIZE;
int end_ix = ix + indices;
- SysTimeval *invocation_timep;
+ Uint64 *invocation_interval_p;
- invocation_timep = (pbdp->bif_invocation
- ? &pbdp->bif_invocation->u.bif_invocation.time
- : NULL);
+ invocation_interval_p
+ = (pbdp->bif_invocation
+ ? &pbdp->bif_invocation->u.bif_invocation.interval
+ : NULL);
ERTS_PROCS_ASSERT(is_nil(*res_accp));
if (!locked) {
- erts_smp_mtx_lock(&proc_tab_mtx);
+ erts_smp_rwmtx_rwlock(&erts_proc_tab_rwmtx);
locked = 1;
}
- ERTS_SMP_LC_ASSERT(erts_lc_mtx_is_locked(&proc_tab_mtx));
+ ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&erts_proc_tab_rwmtx));
ERTS_PROCS_DBG_TRACE(p->id, processes_bif_engine, insp_table);
if (cix != 0)
- erts_get_emu_time(&pbdp->chunk[cix].time);
+ pbdp->chunk[cix].interval = step_proc_interval();
else if (pbdp->bif_invocation)
- pbdp->chunk[0].time = *invocation_timep;
- /* else: Time is irrelevant */
+ pbdp->chunk[0].interval = *invocation_interval_p;
+ /* else: interval is irrelevant */
- if (end_ix >= erts_max_processes) {
+ if (end_ix >= erts_proc.max) {
ERTS_PROCS_ASSERT(cix+1 == processes_bif_tab_chunks);
- end_ix = erts_max_processes;
+ end_ix = erts_proc.max;
indices = end_ix - ix;
/* What to do when done with this chunk */
pbdp->state = (processes_bif_tab_chunks == 1
@@ -9379,16 +9668,15 @@ processes_bif_engine(Process *p, Eterm *res_accp, Binary *mbp)
}
for (; ix < end_ix; ix++) {
- Process *rp = process_tab[ix];
+ Process *rp = erts_pix2proc(ix);
if (rp
- && (!invocation_timep
- || erts_cmp_timeval(&rp->started,
- invocation_timep) < 0)) {
+ && (!invocation_interval_p
+ || rp->started_interval < *invocation_interval_p)) {
ERTS_PROCS_ASSERT(is_internal_pid(rp->id));
pbdp->pid[pbdp->pid_ix] = rp->id;
#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS
- pbdp->debug.pid_started[pbdp->pid_ix] = rp->started;
+ pbdp->debug.pid_started[pbdp->pid_ix] = rp->started_interval;
#endif
pbdp->pid_ix++;
@@ -9398,7 +9686,7 @@ processes_bif_engine(Process *p, Eterm *res_accp, Binary *mbp)
pbdp->tix = end_ix;
- erts_smp_mtx_unlock(&proc_tab_mtx);
+ erts_smp_rwmtx_rwunlock(&erts_proc_tab_rwmtx);
locked = 0;
reds = indices/ERTS_PROCESSES_BIF_TAB_INSPECT_INDICES_PER_RED;
@@ -9410,8 +9698,8 @@ processes_bif_engine(Process *p, Eterm *res_accp, Binary *mbp)
ix = pbdp->tix;
indices = ERTS_PROCESSES_BIF_TAB_CHUNK_SIZE;
end_ix = ix + indices;
- if (end_ix > erts_max_processes) {
- end_ix = erts_max_processes;
+ if (end_ix > erts_proc.max) {
+ end_ix = erts_proc.max;
indices = end_ix - ix;
}
@@ -9430,20 +9718,20 @@ processes_bif_engine(Process *p, Eterm *res_accp, Binary *mbp)
int i;
int max_reds;
int free_term_procs = 0;
- SysTimeval *invocation_timep;
+ Uint64 invocation_interval;
ErtsTermProcElement *tpep;
ErtsTermProcElement *free_list = NULL;
tpep = pbdp->bif_invocation;
ERTS_PROCS_ASSERT(tpep);
- invocation_timep = &tpep->u.bif_invocation.time;
+ invocation_interval = tpep->u.bif_invocation.interval;
max_reds = have_reds = ERTS_BIF_REDS_LEFT(p);
if (max_reds > ERTS_PROCESSES_INSPECT_TERM_PROC_MAX_REDS)
max_reds = ERTS_PROCESSES_INSPECT_TERM_PROC_MAX_REDS;
reds = 0;
- erts_smp_mtx_lock(&proc_tab_mtx);
+ erts_smp_rwmtx_rwlock(&erts_proc_tab_rwmtx);
ERTS_PROCS_DBG_TRACE(p->id, processes_bif_engine, insp_term_procs);
ERTS_PROCS_DBG_CHK_TPLIST();
@@ -9482,20 +9770,19 @@ processes_bif_engine(Process *p, Eterm *res_accp, Binary *mbp)
}
else {
int cix = tpep->ix/ERTS_PROCESSES_BIF_TAB_CHUNK_SIZE;
- SysTimeval *chunk_timep = &pbdp->chunk[cix].time;
+ Uint64 chunk_interval = pbdp->chunk[cix].interval;
Eterm pid = tpep->u.process.pid;
ERTS_PROCS_ASSERT(is_internal_pid(pid));
- if (erts_cmp_timeval(&tpep->u.process.spawned,
- invocation_timep) < 0) {
- if (erts_cmp_timeval(&tpep->u.process.exited,
- chunk_timep) < 0) {
+ if (tpep->u.process.spawned < invocation_interval) {
+ if (tpep->u.process.exited < chunk_interval) {
ERTS_PROCS_DBG_CHK_PID_NOT_FOUND(pbdp,
pid,
- &tpep->u.process.spawned);
+ tpep->u.process.spawned);
pbdp->pid[pbdp->pid_ix] = pid;
#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS
- pbdp->debug.pid_started[pbdp->pid_ix] = tpep->u.process.spawned;
+ pbdp->debug.pid_started[pbdp->pid_ix]
+ = tpep->u.process.spawned;
#endif
pbdp->pid_ix++;
ERTS_PROCS_ASSERT(pbdp->pid_ix <= pbdp->pid_sz);
@@ -9503,13 +9790,13 @@ processes_bif_engine(Process *p, Eterm *res_accp, Binary *mbp)
else {
ERTS_PROCS_DBG_CHK_PID_FOUND(pbdp,
pid,
- &tpep->u.process.spawned);
+ tpep->u.process.spawned);
}
}
else {
ERTS_PROCS_DBG_CHK_PID_NOT_FOUND(pbdp,
pid,
- &tpep->u.process.spawned);
+ tpep->u.process.spawned);
}
i++;
@@ -9558,7 +9845,7 @@ processes_bif_engine(Process *p, Eterm *res_accp, Binary *mbp)
ERTS_PROCS_DBG_CHK_TPLIST();
ERTS_PROCS_DBG_CHK_FREELIST(free_list);
- erts_smp_mtx_unlock(&proc_tab_mtx);
+ erts_smp_rwmtx_rwunlock(&erts_proc_tab_rwmtx);
/*
* We do the actual free of term proc structures now when we
@@ -9618,8 +9905,9 @@ processes_bif_engine(Process *p, Eterm *res_accp, Binary *mbp)
sizeof(Eterm)*pbdp->pid_sz);
#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS
pbdp->debug.pid_started = erts_realloc(ERTS_ALC_T_PROCS_PIDS,
- pbdp->debug.pid_started,
- sizeof(SysTimeval)*pbdp->pid_sz);
+ pbdp->debug.pid_started,
+ (sizeof(Uint64)
+ * pbdp->pid_sz));
#endif
}
reds = conses/ERTS_PROCESSES_BIF_BUILD_RESULT_CONSES_PER_RED;
@@ -9739,7 +10027,7 @@ init_processes_bif(void)
{
saved_term_procs.start = NULL;
saved_term_procs.end = NULL;
- processes_bif_tab_chunks = (((erts_max_processes - 1)
+ processes_bif_tab_chunks = (((erts_proc.max - 1)
/ ERTS_PROCESSES_BIF_TAB_CHUNK_SIZE)
+ 1);
@@ -9752,10 +10040,6 @@ init_processes_bif(void)
processes_trap_export.code[3] = (BeamInstr) em_apply_bif;
processes_trap_export.code[4] = (BeamInstr) &processes_trap;
-#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_TERM_PROC_LIST
- erts_get_emu_time(&debug_tv_start);
-#endif
-
}
/*
@@ -9787,31 +10071,29 @@ erts_debug_processes(Process *c_p)
Eterm res;
Eterm* hp;
Process *p;
-#ifdef DEBUG
Eterm *hp_end;
-#endif
- erts_smp_mtx_lock(&proc_tab_mtx);
+ erts_smp_rwmtx_rwlock(&erts_proc_tab_rwmtx);
res = NIL;
need = erts_process_count() * 2;
hp = HAlloc(c_p, need); /* we need two heap words for each pid */
-#ifdef DEBUG
hp_end = hp + need;
-#endif
/* make the list by scanning bakward */
- for (i = erts_max_processes-1; i >= 0; i--) {
- if ((p = process_tab[i]) != NULL) {
- res = CONS(hp, process_tab[i]->id, res);
+ for (i = erts_proc.max-1; i >= 0; i--) {
+ p = erts_pix2proc(i);
+ if (p) {
+ res = CONS(hp, p->id, res);
hp += 2;
}
}
- ASSERT(hp == hp_end);
- erts_smp_mtx_unlock(&proc_tab_mtx);
+ erts_smp_rwmtx_rwunlock(&erts_proc_tab_rwmtx);
+
+ HRelease(c_p, hp_end, hp);
return res;
}
@@ -9843,14 +10125,12 @@ erts_debug_processes_bif_info(Process *c_p)
static void
debug_processes_check_found_pid(ErtsProcessesBifData *pbdp,
Eterm pid,
- SysTimeval *tvp,
+ Uint64 ic,
int pid_should_be_found)
{
int i;
for (i = 0; i < pbdp->pid_ix; i++) {
- if (pbdp->pid[i] == pid
- && pbdp->debug.pid_started[i].tv_sec == tvp->tv_sec
- && pbdp->debug.pid_started[i].tv_usec == tvp->tv_usec) {
+ if (pbdp->pid[i] == pid && pbdp->debug.pid_started[i] == ic) {
ERTS_PROCS_ASSERT(pid_should_be_found);
return;
}
@@ -9884,8 +10164,8 @@ debug_processes_save_all_pids(ErtsProcessesBifData *pbdp)
pbdp->debug.correct_pids = erts_alloc(ERTS_ALC_T_PROCS_PIDS,
sizeof(Eterm)*pbdp->pid_sz);
- for (tix = 0, cpix = 0; tix < erts_max_processes; tix++) {
- Process *rp = process_tab[tix];
+ for (tix = 0, cpix = 0; tix < erts_proc.max; tix++) {
+ Process *rp = erts_pix2proc(tix);
if (rp) {
ERTS_PROCS_ASSERT(is_internal_pid(rp->id));
pbdp->debug.correct_pids[cpix++] = rp->id;
@@ -9938,14 +10218,13 @@ debug_processes_verify_all_pids(ErtsProcessesBifData *pbdp)
static void
debug_processes_check_term_proc_list(void)
{
- ERTS_SMP_LC_ASSERT(erts_lc_mtx_is_locked(&proc_tab_mtx));
+ ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&erts_proc_tab_rwmtx));
if (!saved_term_procs.start)
ERTS_PROCS_ASSERT(!saved_term_procs.end);
else {
- SysTimeval tv_now;
- SysTimeval *prev_xtvp = NULL;
+ Uint64 curr_interval = get_proc_interval();
+ Uint64 *prev_x_interval_p = NULL;
ErtsTermProcElement *tpep;
- erts_get_emu_time(&tv_now);
for (tpep = saved_term_procs.start; tpep; tpep = tpep->next) {
if (!tpep->prev)
@@ -9957,20 +10236,17 @@ debug_processes_check_term_proc_list(void)
else
ERTS_PROCS_ASSERT(tpep->next->prev == tpep);
if (tpep->ix < 0) {
- SysTimeval *tvp = &tpep->u.bif_invocation.time;
- ERTS_PROCS_ASSERT(erts_cmp_timeval(&debug_tv_start, tvp) < 0
- && erts_cmp_timeval(tvp, &tv_now) < 0);
+ Uint64 interval = tpep->u.bif_invocation.interval;
+ ERTS_PROCS_ASSERT(interval <= curr_interval);
}
else {
- SysTimeval *stvp = &tpep->u.process.spawned;
- SysTimeval *xtvp = &tpep->u.process.exited;
+ Uint64 s_interval = tpep->u.process.spawned;
+ Uint64 x_interval = tpep->u.process.exited;
- ERTS_PROCS_ASSERT(erts_cmp_timeval(&debug_tv_start,
- stvp) < 0);
- ERTS_PROCS_ASSERT(erts_cmp_timeval(stvp, xtvp) < 0);
- if (prev_xtvp)
- ERTS_PROCS_ASSERT(erts_cmp_timeval(prev_xtvp, xtvp) < 0);
- prev_xtvp = xtvp;
+ ERTS_PROCS_ASSERT(s_interval <= x_interval);
+ if (prev_x_interval_p)
+ ERTS_PROCS_ASSERT(*prev_x_interval_p <= x_interval);
+ prev_x_interval_p = &tpep->u.process.exited;
ERTS_PROCS_ASSERT(is_internal_pid(tpep->u.process.pid));
ERTS_PROCS_ASSERT(tpep->ix
== internal_pid_index(tpep->u.process.pid));
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index cff0783bc4..d40f50c0af 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -290,7 +290,7 @@ struct ErtsSchedulerSleepInfo_ {
typedef struct ErtsProcList_ ErtsProcList;
struct ErtsProcList_ {
Eterm pid;
- SysTimeval started;
+ Uint64 started_interval;
ErtsProcList* next;
};
@@ -600,6 +600,8 @@ struct ErtsPendingSuspend_ {
# define BIN_OLD_VHEAP(p) (p)->bin_old_vheap
struct process {
+ Eterm id; /* The pid of this process
+ (need to be first in struct) */
/* All fields in the PCB that differs between different heap
* architectures, have been moved to the end of this struct to
* make sure that as few offsets as possible differ. Different
@@ -646,7 +648,6 @@ struct process {
Uint32 gcstatus; /* process gc STATE */
Uint32 rstatus; /* process resume STATE */
Uint32 rcount; /* suspend count */
- Eterm id; /* The pid of this process */
int prio; /* Priority of process */
int skipped; /* Times a low prio process has been rescheduled */
Uint reds; /* No of reductions for this process */
@@ -699,8 +700,8 @@ struct process {
* Information mainly for post-mortem use (erl crash dump).
*/
Eterm parent; /* Pid of process that created this process. */
- SysTimeval started; /* Time when started. */
-
+ erts_approx_time_t approx_started; /* Time when started. */
+ Uint64 started_interval;
/* This is the place, where all fields that differs between memory
* architectures, have gone to.
@@ -735,8 +736,8 @@ struct process {
#ifdef ERTS_SMP
erts_proc_lock_t lock;
+ erts_atomic32_t aflags;
ErtsSchedulerData *scheduler_data;
- int is_exiting;
Uint32 runq_flags;
Uint32 status_flags;
ErlMessageInQueue msg_inq;
@@ -815,6 +816,10 @@ void erts_check_for_holes(Process* p);
#define SEQ_TRACE_TOKEN(p) ((p)->seq_trace_token)
+#define ERTS_PROC_AFLG_EXITING (((erts_aint_t) 1) << 0)
+#define ERTS_PROC_AFLG_PENDING_EXIT (((erts_aint_t) 1) << 1)
+
+
/* The sequential tracing token is a tuple of size 5:
*
* {Flags, Label, Serial, Sender}
@@ -887,13 +892,12 @@ Eterm* erts_heap_alloc(Process* p, Uint need, Uint xtra);
Eterm* erts_set_hole_marker(Eterm* ptr, Uint sz);
#endif
-extern Process** process_tab;
+extern erts_smp_rwmtx_t erts_proc_tab_rwmtx;
+extern erts_smp_atomic_t *erts_proc_tab;
#ifdef HYBRID
extern Uint erts_num_active_procs;
extern Process** erts_active_procs;
#endif
-extern Uint erts_max_processes;
-extern Uint erts_process_tab_index_mask;
extern Uint erts_default_process_flags;
extern erts_smp_rwmtx_t erts_cpu_bind_rwmtx;
/* If any of the erts_system_monitor_* variables are set (enabled),
@@ -1091,6 +1095,9 @@ void erts_early_init_scheduling(int);
void erts_init_scheduling(int, int);
Eterm erts_sched_wall_time_request(Process *c_p, int set, int enable);
+Uint64 erts_get_proc_interval(void);
+Uint64 erts_ensure_later_proc_interval(Uint64);
+Uint64 erts_step_proc_interval(void);
ErtsProcList *erts_proclist_create(Process *);
void erts_proclist_destroy(ErtsProcList *);
@@ -1607,23 +1614,17 @@ Process *erts_pid2proc_nropt(Process *c_p,
ErtsProcLocks pid_locks);
extern int erts_disable_proc_not_running_opt;
+
+#define ERTS_PROC_IS_EXITING(P) \
+ (ERTS_PROC_AFLG_EXITING & erts_atomic32_read_acqb(&(P)->aflags))
+
#ifdef DEBUG
#define ERTS_SMP_ASSERT_IS_NOT_EXITING(P) \
- do { ASSERT(!(P)->is_exiting); } while (0)
+ do { ASSERT(!ERTS_PROC_IS_EXITING((P))); } while (0)
#else
#define ERTS_SMP_ASSERT_IS_NOT_EXITING(P)
#endif
-/* NOTE: At least one process lock has to be held on P! */
-#ifdef ERTS_ENABLE_LOCK_CHECK
-#define ERTS_PROC_IS_EXITING(P) \
- (ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks((P)) != 0 \
- || erts_lc_pix_lock_is_locked(ERTS_PID2PIXLOCK((P)->id))),\
- (P)->is_exiting)
-#else
-#define ERTS_PROC_IS_EXITING(P) ((P)->is_exiting)
-#endif
-
#else /* !ERTS_SMP */
#define ERTS_PROC_IS_EXITING(P) ((P)->status == P_EXITING)
diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c
index 3550f1396c..37cc717156 100644
--- a/erts/emulator/beam/erl_process_dump.c
+++ b/erts/emulator/beam/erl_process_dump.c
@@ -65,10 +65,9 @@ erts_deep_process_dump(int to, void *to_arg)
all_binaries = NULL;
for (i = 0; i < erts_max_processes; i++) {
- if ((process_tab[i] != NULL) && (process_tab[i]->i != ENULL)) {
- if (process_tab[i]->status != P_EXITING) {
- Process* p = process_tab[i];
-
+ Process *p = erts_pix2proc(i);
+ if (p && p->i != ENULL) {
+ if (p->status != P_EXITING) {
if (p->status != P_GARBING) {
dump_process_info(to, to_arg, p);
}
diff --git a/erts/emulator/beam/erl_process_lock.c b/erts/emulator/beam/erl_process_lock.c
index 3e52a8767f..b864340782 100644
--- a/erts/emulator/beam/erl_process_lock.c
+++ b/erts/emulator/beam/erl_process_lock.c
@@ -67,7 +67,7 @@
#include "erl_process.h"
-const Process erts_proc_lock_busy;
+const Process erts_proc_lock_busy = {ERTS_INVALID_PID};
#ifdef ERTS_SMP
@@ -671,12 +671,11 @@ erts_proc_lock_prepare_proc_lock_waiter(void)
*/
static void
-proc_safelock(Process *a_proc,
- erts_pix_lock_t *a_pix_lck,
+proc_safelock(int is_sched,
+ Process *a_proc,
ErtsProcLocks a_have_locks,
ErtsProcLocks a_need_locks,
Process *b_proc,
- erts_pix_lock_t *b_pix_lck,
ErtsProcLocks b_have_locks,
ErtsProcLocks b_need_locks)
{
@@ -684,7 +683,6 @@ proc_safelock(Process *a_proc,
#ifdef ERTS_ENABLE_LOCK_CHECK
Eterm pid1, pid2;
#endif
- erts_pix_lock_t *pix_lck1, *pix_lck2;
ErtsProcLocks need_locks1, have_locks1, need_locks2, have_locks2;
ErtsProcLocks unlock_mask;
int lock_no, refc1 = 0, refc2 = 0;
@@ -701,14 +699,12 @@ proc_safelock(Process *a_proc,
#ifdef ERTS_ENABLE_LOCK_CHECK
pid1 = a_proc->id;
#endif
- pix_lck1 = a_pix_lck;
need_locks1 = a_need_locks;
have_locks1 = a_have_locks;
p2 = b_proc;
#ifdef ERTS_ENABLE_LOCK_CHECK
pid2 = b_proc->id;
#endif
- pix_lck2 = b_pix_lck;
need_locks2 = b_need_locks;
have_locks2 = b_have_locks;
}
@@ -717,14 +713,12 @@ proc_safelock(Process *a_proc,
#ifdef ERTS_ENABLE_LOCK_CHECK
pid1 = b_proc->id;
#endif
- pix_lck1 = b_pix_lck;
need_locks1 = b_need_locks;
have_locks1 = b_have_locks;
p2 = a_proc;
#ifdef ERTS_ENABLE_LOCK_CHECK
pid2 = a_proc->id;
#endif
- pix_lck2 = a_pix_lck;
need_locks2 = a_need_locks;
have_locks2 = a_have_locks;
}
@@ -735,14 +729,12 @@ proc_safelock(Process *a_proc,
#ifdef ERTS_ENABLE_LOCK_CHECK
pid1 = a_proc->id;
#endif
- pix_lck1 = a_pix_lck;
need_locks1 = a_need_locks | b_need_locks;
have_locks1 = a_have_locks | b_have_locks;
p2 = NULL;
#ifdef ERTS_ENABLE_LOCK_CHECK
pid2 = 0;
#endif
- pix_lck2 = NULL;
need_locks2 = 0;
have_locks2 = 0;
}
@@ -752,14 +744,12 @@ proc_safelock(Process *a_proc,
#ifdef ERTS_ENABLE_LOCK_CHECK
pid1 = b_proc->id;
#endif
- pix_lck1 = b_pix_lck;
need_locks1 = b_need_locks;
have_locks1 = b_have_locks;
p2 = NULL;
#ifdef ERTS_ENABLE_LOCK_CHECK
pid2 = 0;
#endif
- pix_lck2 = NULL;
need_locks2 = 0;
have_locks2 = 0;
#ifdef ERTS_ENABLE_LOCK_CHECK
@@ -807,21 +797,21 @@ proc_safelock(Process *a_proc,
if (unlock_locks) {
have_locks1 &= ~unlock_locks;
need_locks1 |= unlock_locks;
- if (!have_locks1) {
+ if (!is_sched && !have_locks1) {
refc1 = 1;
erts_smp_proc_inc_refc(p1);
}
- erts_smp_proc_unlock__(p1, pix_lck1, unlock_locks);
+ erts_smp_proc_unlock(p1, unlock_locks);
}
unlock_locks = unlock_mask & have_locks2;
if (unlock_locks) {
have_locks2 &= ~unlock_locks;
need_locks2 |= unlock_locks;
- if (!have_locks2) {
+ if (!is_sched && !have_locks2) {
refc2 = 1;
erts_smp_proc_inc_refc(p2);
}
- erts_smp_proc_unlock__(p2, pix_lck2, unlock_locks);
+ erts_smp_proc_unlock(p2, unlock_locks);
}
}
@@ -852,7 +842,7 @@ proc_safelock(Process *a_proc,
if (need_locks2 & lock)
lock_no--;
locks = need_locks1 & lock_mask;
- erts_smp_proc_lock__(p1, pix_lck1, locks);
+ erts_smp_proc_lock(p1, locks);
have_locks1 |= locks;
need_locks1 &= ~locks;
}
@@ -863,7 +853,7 @@ proc_safelock(Process *a_proc,
lock = (1 << ++lock_no);
}
locks = need_locks2 & lock_mask;
- erts_smp_proc_lock__(p2, pix_lck2, locks);
+ erts_smp_proc_lock(p2, locks);
have_locks2 |= locks;
need_locks2 &= ~locks;
}
@@ -898,10 +888,12 @@ proc_safelock(Process *a_proc,
}
#endif
- if (refc1)
- erts_smp_proc_dec_refc(p1);
- if (refc2)
- erts_smp_proc_dec_refc(p2);
+ if (!is_sched) {
+ if (refc1)
+ erts_smp_proc_dec_refc(p1);
+ if (refc2)
+ erts_smp_proc_dec_refc(p2);
+ }
}
void
@@ -912,72 +904,185 @@ erts_proc_safelock(Process *a_proc,
ErtsProcLocks b_have_locks,
ErtsProcLocks b_need_locks)
{
- proc_safelock(a_proc,
- a_proc ? ERTS_PID2PIXLOCK(a_proc->id) : NULL,
+ proc_safelock(erts_get_scheduler_id() != 0,
+ a_proc,
a_have_locks,
a_need_locks,
b_proc,
- b_proc ? ERTS_PID2PIXLOCK(b_proc->id) : NULL,
b_have_locks,
b_need_locks);
}
-/*
- * erts_pid2proc_safelock() is called from erts_pid2proc_opt() when
- * it wasn't possible to trylock all locks needed.
- * c_p - current process
- * c_p_have_locks - locks held on c_p
- * pid - process id of process we are looking up
- * proc - process struct of process we are looking
- * up (both in and out argument)
- * need_locks - all locks we need (including have_locks)
- * pix_lock - pix lock for process we are looking up
- * flags - option flags
- */
-void
-erts_pid2proc_safelock(Process *c_p,
- ErtsProcLocks c_p_have_locks,
- Process **proc,
- ErtsProcLocks need_locks,
- erts_pix_lock_t *pix_lock,
- int flags)
+Process *
+erts_pid2proc_opt(Process *c_p,
+ ErtsProcLocks c_p_have_locks,
+ Eterm pid,
+ ErtsProcLocks pid_need_locks,
+ int flags)
{
- Process *p = *proc;
- ERTS_LC_ASSERT(p->lock.refc > 0);
- ERTS_LC_ASSERT(process_tab[internal_pid_index(p->id)] == p);
- p->lock.refc++;
- erts_pix_unlock(pix_lock);
-
- proc_safelock(c_p,
- c_p ? ERTS_PID2PIXLOCK(c_p->id) : NULL,
- c_p_have_locks,
- c_p_have_locks,
- p,
- pix_lock,
- 0,
- need_locks);
+ Process *dec_refc_proc = NULL;
+ int need_ptl;
+ ErtsProcLocks need_locks;
+ Uint pix;
+ Process *proc;
+#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT)
+ ErtsProcLocks lcnt_locks;
+#endif
- erts_pix_lock(pix_lock);
+#ifdef ERTS_ENABLE_LOCK_CHECK
+ if (c_p) {
+ ErtsProcLocks might_unlock = c_p_have_locks & pid_need_locks;
+ if (might_unlock)
+ erts_proc_lc_might_unlock(c_p, might_unlock);
+ }
+#endif
- if (!p->is_exiting
- || ((flags & ERTS_P2P_FLG_ALLOW_OTHER_X)
- && process_tab[internal_pid_index(p->id)] == p)) {
- ERTS_LC_ASSERT(p->lock.refc > 1);
- p->lock.refc--;
+ if (is_not_internal_pid(pid))
+ return NULL;
+ pix = internal_pid_index(pid);
+
+ ERTS_LC_ASSERT((pid_need_locks & ERTS_PROC_LOCKS_ALL) == pid_need_locks);
+ need_locks = pid_need_locks;
+
+ if (c_p && c_p->id == pid) {
+ ASSERT(c_p->id != ERTS_INVALID_PID);
+ ASSERT(c_p == erts_pix2proc(pix));
+
+ if (!(flags & ERTS_P2P_FLG_ALLOW_OTHER_X)
+ && ERTS_PROC_IS_EXITING(c_p))
+ return NULL;
+ need_locks &= ~c_p_have_locks;
+ if (!need_locks) {
+ if (flags & ERTS_P2P_FLG_SMP_INC_REFC)
+ erts_smp_proc_inc_refc(c_p);
+ return c_p;
+ }
}
- else {
- /* No proc. Note, we need to keep refc until after process unlock */
- erts_pix_unlock(pix_lock);
- erts_smp_proc_unlock__(p, pix_lock, need_locks);
- *proc = NULL;
- erts_pix_lock(pix_lock);
- ERTS_LC_ASSERT(p->lock.refc > 0);
- if (--p->lock.refc == 0) {
- erts_pix_unlock(pix_lock);
- erts_free_proc(p);
- erts_pix_lock(pix_lock);
+
+ need_ptl = !erts_get_scheduler_id();
+
+ if (need_ptl)
+ erts_smp_rwmtx_rwlock(&erts_proc_tab_rwmtx);
+
+ proc = (Process *) erts_smp_atomic_read_ddrb(&erts_proc.tab[pix]);
+
+ if (proc) {
+ if (proc->id != pid)
+ proc = NULL;
+ else if (!need_locks) {
+ if (flags & ERTS_P2P_FLG_SMP_INC_REFC)
+ erts_smp_proc_inc_refc(proc);
}
+ else {
+ int busy;
+
+#if ERTS_PROC_LOCK_OWN_IMPL
+#ifdef ERTS_ENABLE_LOCK_COUNT
+ lcnt_locks = need_locks;
+ if (!(flags & ERTS_P2P_FLG_TRY_LOCK)) {
+ erts_lcnt_proc_lock(&proc->lock, need_locks);
+ }
+#endif
+
+#ifdef ERTS_ENABLE_LOCK_CHECK
+ /* Make sure erts_pid2proc_safelock() is enough to handle
+ a potential lock order violation situation... */
+ busy = erts_proc_lc_trylock_force_busy(proc, need_locks);
+ if (!busy)
+#endif
+#endif /* ERTS_PROC_LOCK_OWN_IMPL */
+ {
+ /* Try a quick trylock to grab all the locks we need. */
+ busy = (int) erts_smp_proc_raw_trylock__(proc, need_locks);
+
+#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_CHECK)
+ erts_proc_lc_trylock(proc, need_locks, !busy);
+#endif
+#ifdef ERTS_PROC_LOCK_DEBUG
+ if (!busy)
+ erts_proc_lock_op_debug(proc, need_locks, 1);
+#endif
+ }
+
+#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT)
+ if (flags & ERTS_P2P_FLG_TRY_LOCK)
+ erts_lcnt_proc_trylock(&proc->lock, need_locks,
+ busy ? EBUSY : 0);
+#endif
+
+ if (!busy) {
+ if (flags & ERTS_P2P_FLG_SMP_INC_REFC)
+ erts_smp_proc_inc_refc(proc);
+
+#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT)
+ /* all is great */
+ if (!(flags & ERTS_P2P_FLG_TRY_LOCK))
+ erts_lcnt_proc_lock_post_x(&proc->lock, lcnt_locks,
+ __FILE__, __LINE__);
+#endif
+
+ }
+ else {
+ if (flags & ERTS_P2P_FLG_TRY_LOCK)
+ proc = ERTS_PROC_LOCK_BUSY;
+ else {
+ if (flags & ERTS_P2P_FLG_SMP_INC_REFC)
+ erts_smp_proc_inc_refc(proc);
+
+#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT)
+ erts_lcnt_proc_lock_unaquire(&proc->lock, lcnt_locks);
+#endif
+
+ if (need_ptl) {
+ erts_smp_proc_inc_refc(proc);
+ dec_refc_proc = proc;
+ erts_smp_rwmtx_rwunlock(&erts_proc_tab_rwmtx);
+ need_ptl = 0;
+ }
+
+ proc_safelock(!need_ptl,
+ c_p,
+ c_p_have_locks,
+ c_p_have_locks,
+ proc,
+ 0,
+ need_locks);
+ }
+ }
+ }
}
+
+ if (need_ptl)
+ erts_smp_rwmtx_rwunlock(&erts_proc_tab_rwmtx);
+
+ if (need_locks
+ && proc
+ && proc != ERTS_PROC_LOCK_BUSY
+ && (!(flags & ERTS_P2P_FLG_ALLOW_OTHER_X)
+ ? ERTS_PROC_IS_EXITING(proc)
+ : (proc
+ != (Process *) erts_smp_atomic_read_nob(&erts_proc.tab[pix])))) {
+
+ erts_smp_proc_unlock(proc, need_locks);
+
+ if (flags & ERTS_P2P_FLG_SMP_INC_REFC)
+ dec_refc_proc = proc;
+ proc = NULL;
+
+ }
+
+ if (dec_refc_proc)
+ erts_smp_proc_dec_refc(dec_refc_proc);
+
+#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_PROC_LOCK_DEBUG)
+ ERTS_LC_ASSERT(!proc
+ || proc == ERTS_PROC_LOCK_BUSY
+ || (pid_need_locks ==
+ (ERTS_PROC_LOCK_FLGS_READ_(&proc->lock)
+ & pid_need_locks)));
+#endif
+
+ return proc;
}
void
@@ -1017,7 +1122,7 @@ erts_proc_lock_init(Process *p)
erts_lc_trylock(1, &p->lock.status.lc);
#endif
#endif
- p->lock.refc = 1;
+ erts_atomic32_init_nob(&p->lock.refc, 1);
#ifdef ERTS_PROC_LOCK_DEBUG
{
int i;
@@ -1590,4 +1695,4 @@ check_queue(erts_proc_lock_t *lck)
}
#endif
-#endif /* ERTS_SMP (the whole file) */
+#endif /* ERTS_SMP */
diff --git a/erts/emulator/beam/erl_process_lock.h b/erts/emulator/beam/erl_process_lock.h
index ee341bd951..9981af7a8a 100644
--- a/erts/emulator/beam/erl_process_lock.h
+++ b/erts/emulator/beam/erl_process_lock.h
@@ -93,7 +93,7 @@ typedef struct erts_proc_lock_t_ {
#else
# error "no implementation"
#endif
- Sint32 refc;
+ erts_atomic32_t refc;
#ifdef ERTS_PROC_LOCK_DEBUG
erts_smp_atomic32_t locked[ERTS_PROC_LOCK_MAX_BIT+1];
#endif
@@ -171,8 +171,6 @@ typedef struct erts_proc_lock_t_ {
* are guaranteed that the process won't exit until the lock you are
* holding has been released. Appart from all process locks also
* the pix lock corresponding to the process has to be held.
- * At the same time as status is changed to P_EXITING, also the
- * field 'is_exiting' in the process structure is set to a value != 0.
*
* Lock order:
* Process locks with low numeric values has to be locked before
@@ -282,12 +280,8 @@ typedef struct {
} u;
} erts_pix_lock_t;
-#define ERTS_PIX2PIXLOCKIX(PIX) \
- ((PIX) & ((1 << ERTS_PIX_LOCKS_BITS) - 1))
-#define ERTS_PIX2PIXLOCK(PIX) \
- (&erts_pix_locks[ERTS_PIX2PIXLOCKIX((PIX))])
#define ERTS_PID2PIXLOCK(PID) \
- ERTS_PIX2PIXLOCK(internal_pid_data((PID)))
+ (&erts_pix_locks[(internal_pid_data((PID)) & ((1 << ERTS_PIX_LOCKS_BITS) - 1))])
#if ERTS_PROC_LOCK_OWN_IMPL
@@ -820,44 +814,36 @@ erts_smp_proc_trylock(Process *p, ErtsProcLocks locks)
#endif
}
-
ERTS_GLB_INLINE void erts_smp_proc_inc_refc(Process *p)
{
#ifdef ERTS_SMP
- erts_pix_lock_t *pixlck = ERTS_PID2PIXLOCK(p->id);
- erts_pix_lock(pixlck);
- ERTS_LC_ASSERT(p->lock.refc > 0);
- p->lock.refc++;
- erts_pix_unlock(pixlck);
+#ifdef ERTS_ENABLE_LOCK_CHECK
+ erts_aint32_t refc = erts_atomic32_inc_read_nob(&p->lock.refc);
+ ERTS_SMP_LC_ASSERT(refc > 1);
+#else
+ erts_atomic32_inc_nob(&p->lock.refc);
+#endif
#endif
}
ERTS_GLB_INLINE void erts_smp_proc_dec_refc(Process *p)
{
#ifdef ERTS_SMP
- Process *fp;
- erts_pix_lock_t *pixlck = ERTS_PID2PIXLOCK(p->id);
- erts_pix_lock(pixlck);
- ERTS_LC_ASSERT(p->lock.refc > 0);
- fp = --p->lock.refc == 0 ? p : NULL;
- erts_pix_unlock(pixlck);
- if (fp)
- erts_free_proc(fp);
+ erts_aint32_t refc = erts_atomic32_dec_read_nob(&p->lock.refc);
+ ERTS_SMP_LC_ASSERT(refc >= 0);
+ if (refc == 0)
+ erts_free_proc(p);
#endif
}
-ERTS_GLB_INLINE void erts_smp_proc_add_refc(Process *p, Sint32 refc)
+ERTS_GLB_INLINE void erts_smp_proc_add_refc(Process *p, Sint32 add_refc)
{
#ifdef ERTS_SMP
- Process *fp;
- erts_pix_lock_t *pixlck = ERTS_PID2PIXLOCK(p->id);
- erts_pix_lock(pixlck);
- ERTS_LC_ASSERT(p->lock.refc > 0);
- p->lock.refc += refc;
- fp = p->lock.refc == 0 ? p : NULL;
- erts_pix_unlock(pixlck);
- if (fp)
- erts_free_proc(fp);
+ erts_aint32_t refc = erts_atomic32_add_read_nob(&p->lock.refc,
+ (erts_aint32_t) add_refc);
+ ERTS_SMP_LC_ASSERT(refc >= 0);
+ if (refc == 0)
+ erts_free_proc(p);
#endif
}
@@ -898,218 +884,75 @@ extern const Process erts_proc_lock_busy;
#define erts_pid2proc(PROC, HL, PID, NL) \
erts_pid2proc_opt((PROC), (HL), (PID), (NL), 0)
-ERTS_GLB_INLINE Process *
-erts_pid2proc_opt(Process *, ErtsProcLocks, Eterm, ErtsProcLocks, int);
-#ifdef ERTS_SMP
-void
-erts_pid2proc_safelock(Process *c_p,
- ErtsProcLocks c_p_have_locks,
- Process **proc,
- ErtsProcLocks need_locks,
- erts_pix_lock_t *pix_lock,
- int flags);
-ERTS_GLB_INLINE Process *erts_pid2proc_unlocked_opt(Eterm pid, int flags);
-#define erts_pid2proc_unlocked(PID) erts_pid2proc_unlocked_opt((PID), 0)
-#else
-#define erts_pid2proc_unlocked_opt(PID, FLGS) \
- erts_pid2proc_opt(NULL, 0, (PID), 0, FLGS)
-#define erts_pid2proc_unlocked(PID) erts_pid2proc_opt(NULL, 0, (PID), 0, 0)
+ERTS_GLB_INLINE Process *erts_pix2proc(int ix);
+ERTS_GLB_INLINE Process *erts_proc_lookup_raw(Eterm pid);
+ERTS_GLB_INLINE Process *erts_proc_lookup(Eterm pid);
+
+#ifndef ERTS_SMP
+ERTS_GLB_INLINE
#endif
+Process *erts_pid2proc_opt(Process *, ErtsProcLocks, Eterm, ErtsProcLocks, int);
#if ERTS_GLB_INLINE_INCL_FUNC_DEF
-ERTS_GLB_INLINE Process *
-#ifdef ERTS_SMP
-erts_pid2proc_unlocked_opt(Eterm pid, int flags)
-#else
-erts_pid2proc_opt(Process *c_p_unused,
- ErtsProcLocks c_p_have_locks_unused,
- Eterm pid,
- ErtsProcLocks pid_need_locks_unused,
- int flags)
-#endif
+ERTS_GLB_INLINE Process *erts_pix2proc(int ix)
{
- Uint pix;
Process *proc;
+ ASSERT(0 <= ix && ix < erts_proc.max);
+ proc = (Process *) erts_smp_atomic_read_nob(&erts_proc.tab[ix]);
+ return proc == ERTS_PROC_LOCK_BUSY ? NULL : proc;
+}
+
+ERTS_GLB_INLINE Process *erts_proc_lookup_raw(Eterm pid)
+{
+ Process *proc;
+ int pix;
+
+ /*
+ * In SMP case: Only scheduler threads are allowed
+ * to use this function. Other threads need to
+ * atomicaly increment refc at lookup, i.e., use
+ * erts_pid2proc_opt() with ERTS_P2P_FLG_SMP_INC_REFC.
+ */
+ ERTS_SMP_LC_ASSERT(erts_get_scheduler_id());
if (is_not_internal_pid(pid))
return NULL;
pix = internal_pid_index(pid);
- if(pix >= erts_max_processes)
+
+ proc = (Process *) erts_smp_atomic_read_ddrb(&erts_proc.tab[pix]);
+
+ if (proc && proc->id != pid)
return NULL;
- proc = process_tab[pix];
- if (proc) {
- if (proc->id != pid
- || (!(flags & ERTS_P2P_FLG_ALLOW_OTHER_X)
- && proc->status == P_EXITING))
- proc = NULL;
- }
+
return proc;
}
-#ifdef ERTS_SMP
+ERTS_GLB_INLINE Process *erts_proc_lookup(Eterm pid)
+{
+ Process *proc = erts_proc_lookup_raw(pid);
+ if (proc && ERTS_PROC_IS_EXITING(proc))
+ return NULL;
+ return proc;
+}
+#ifndef ERTS_SMP
ERTS_GLB_INLINE Process *
-erts_pid2proc_opt(Process *c_p,
- ErtsProcLocks c_p_have_locks,
+erts_pid2proc_opt(Process *c_p_unused,
+ ErtsProcLocks c_p_have_locks_unused,
Eterm pid,
- ErtsProcLocks pid_need_locks,
+ ErtsProcLocks pid_need_locks_unused,
int flags)
{
- erts_pix_lock_t *pix_lock;
- ErtsProcLocks need_locks;
- Uint pix;
- Process *proc;
-#if ERTS_PROC_LOCK_OWN_IMPL
-#ifdef ERTS_ENABLE_LOCK_COUNT
- ErtsProcLocks lcnt_locks;
-#endif
-#endif /* ERTS_PROC_LOCK_OWN_IMPL */
-
-#ifdef ERTS_ENABLE_LOCK_CHECK
- if (c_p) {
- ErtsProcLocks might_unlock = c_p_have_locks & pid_need_locks;
- if (might_unlock)
- erts_proc_lc_might_unlock(c_p, might_unlock);
- }
-#endif
-
- if (is_not_internal_pid(pid)) {
- proc = NULL;
- goto done;
- }
- pix = internal_pid_index(pid);
- if(pix >= erts_max_processes) {
- proc = NULL;
- goto done;
- }
-
- ERTS_LC_ASSERT((pid_need_locks & ERTS_PROC_LOCKS_ALL) == pid_need_locks);
- need_locks = pid_need_locks;
-
- pix_lock = ERTS_PIX2PIXLOCK(pix);
-
- if (c_p && c_p->id == pid) {
- ASSERT(c_p->id != ERTS_INVALID_PID);
- ASSERT(c_p == process_tab[pix]);
- if (!(flags & ERTS_P2P_FLG_ALLOW_OTHER_X) && c_p->is_exiting) {
- proc = NULL;
- goto done;
- }
- need_locks &= ~c_p_have_locks;
- if (!need_locks) {
- proc = c_p;
- erts_pix_lock(pix_lock);
- if (flags & ERTS_P2P_FLG_SMP_INC_REFC)
- proc->lock.refc++;
- erts_pix_unlock(pix_lock);
- goto done;
- }
- }
-
- erts_pix_lock(pix_lock);
-
- proc = process_tab[pix];
- if (proc) {
- if (proc->id != pid || (!(flags & ERTS_P2P_FLG_ALLOW_OTHER_X)
- && ERTS_PROC_IS_EXITING(proc))) {
- proc = NULL;
- }
- else if (!need_locks) {
- if (flags & ERTS_P2P_FLG_SMP_INC_REFC)
- proc->lock.refc++;
- }
- else {
- int busy;
-
-#if ERTS_PROC_LOCK_OWN_IMPL
-#ifdef ERTS_ENABLE_LOCK_COUNT
- lcnt_locks = need_locks;
- if (!(flags & ERTS_P2P_FLG_TRY_LOCK)) {
- erts_lcnt_proc_lock(&proc->lock, need_locks);
- }
-#endif
-
-#ifdef ERTS_ENABLE_LOCK_CHECK
- /* Make sure erts_pid2proc_safelock() is enough to handle
- a potential lock order violation situation... */
- busy = erts_proc_lc_trylock_force_busy(proc, need_locks);
- if (!busy)
-#endif
-#endif /* ERTS_PROC_LOCK_OWN_IMPL */
- {
- /* Try a quick trylock to grab all the locks we need. */
- busy = (int) erts_smp_proc_raw_trylock__(proc, need_locks);
-#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_CHECK)
- erts_proc_lc_trylock(proc, need_locks, !busy);
-#endif
-#ifdef ERTS_PROC_LOCK_DEBUG
- if (!busy)
- erts_proc_lock_op_debug(proc, need_locks, 1);
-#endif
- }
-
-#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT)
- if (flags & ERTS_P2P_FLG_TRY_LOCK) {
- if (busy) {
- erts_lcnt_proc_trylock(&proc->lock, need_locks, EBUSY);
- } else {
- erts_lcnt_proc_trylock(&proc->lock, need_locks, 0);
- }
- }
-#endif
- if (!busy) {
- if (flags & ERTS_P2P_FLG_SMP_INC_REFC)
- proc->lock.refc++;
-#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT)
- /* all is great */
- if (!(flags & ERTS_P2P_FLG_TRY_LOCK)) {
- erts_lcnt_proc_lock_post_x(&proc->lock, lcnt_locks, __FILE__, __LINE__);
- }
-#endif
- }
- else {
- if (flags & ERTS_P2P_FLG_TRY_LOCK)
- proc = ERTS_PROC_LOCK_BUSY;
- else {
- if (flags & ERTS_P2P_FLG_SMP_INC_REFC)
- proc->lock.refc++;
-#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT)
- erts_lcnt_proc_lock_unaquire(&proc->lock, lcnt_locks);
-#endif
- erts_pid2proc_safelock(c_p,
- c_p_have_locks,
- &proc,
- pid_need_locks,
- pix_lock,
- flags);
- if (proc && (flags & ERTS_P2P_FLG_SMP_INC_REFC))
- proc->lock.refc++;
- }
- }
- }
- }
-
- erts_pix_unlock(pix_lock);
-#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_PROC_LOCK_DEBUG)
- ERTS_LC_ASSERT(!proc
- || proc == ERTS_PROC_LOCK_BUSY
- || (pid_need_locks ==
- (ERTS_PROC_LOCK_FLGS_READ_(&proc->lock)
- & pid_need_locks)));
-#endif
-
-
- done:
-
-#if ERTS_PROC_LOCK_ATOMIC_IMPL
- ETHR_COMPILER_BARRIER;
-#endif
-
+ Process *proc = erts_proc_lookup_raw(pid);
+ if (!(flags & ERTS_P2P_FLG_ALLOW_OTHER_X)
+ && proc
+ && proc->status == P_EXITING)
+ return NULL;
return proc;
}
-#endif /* ERTS_SMP */
+#endif /* !ERTS_SMP */
#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h
index ee47c98009..17628286bc 100644
--- a/erts/emulator/beam/erl_threads.h
+++ b/erts/emulator/beam/erl_threads.h
@@ -258,10 +258,6 @@
#include "sys.h"
-typedef struct { SWord sint[2]; } erts_no_dw_atomic_t;
-typedef SWord erts_no_atomic_t;
-typedef Sint32 erts_no_atomic32_t;
-
#ifdef USE_THREADS
#define ETHR_TRY_INLINE_FUNCS
@@ -411,12 +407,15 @@ typedef struct {
typedef int erts_rwmtx_t;
typedef int erts_tsd_key_t;
typedef int erts_tse_t;
-#define erts_dw_aint_t erts_no_dw_atomic_t
-#define erts_dw_atomic_t erts_no_dw_atomic_t
-#define erts_aint_t SWord
-#define erts_atomic_t erts_no_atomic_t
-#define erts_aint32_t Sint32
-#define erts_atomic32_t erts_no_atomic32_t
+
+typedef struct { SWord sint[2]; } erts_dw_aint_t;
+typedef SWord erts_aint_t;
+typedef Sint32 erts_aint32_t;
+
+#define erts_dw_atomic_t erts_dw_aint_t
+#define erts_atomic_t erts_aint_t
+#define erts_atomic32_t erts_aint32_t
+
#if __GNUC__ > 2
typedef struct { } erts_spinlock_t;
typedef struct { } erts_rwlock_t;
@@ -425,6 +424,14 @@ typedef struct { int gcc_is_buggy; } erts_spinlock_t;
typedef struct { int gcc_is_buggy; } erts_rwlock_t;
#endif
+#ifdef WORDS_BIGENDIAN
+#define ERTS_DW_AINT_LOW_WORD 1
+#define ERTS_DW_AINT_HIGH_WORD 0
+#else
+#define ERTS_DW_AINT_LOW_WORD 0
+#define ERTS_DW_AINT_HIGH_WORD 1
+#endif
+
#define ERTS_MTX_INITER 0
#define ERTS_CND_INITER 0
#define ERTS_THR_INIT_DATA_DEF_INITER 0
@@ -433,6 +440,10 @@ typedef struct { int gcc_is_buggy; } erts_rwlock_t;
#endif /* #ifdef USE_THREADS */
+#define erts_no_dw_atomic_t erts_dw_aint_t
+#define erts_no_atomic_t erts_aint_t
+#define erts_no_atomic32_t erts_aint32_t
+
#define ERTS_AINT_NULL ((erts_aint_t) NULL)
#define ERTS_AINT_T_MAX (~(((erts_aint_t) 1) << (sizeof(erts_aint_t)*8-1)))
diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h
index 6c6e193818..4bbdcaa3e3 100644
--- a/erts/emulator/beam/erl_time.h
+++ b/erts/emulator/beam/erl_time.h
@@ -118,9 +118,11 @@ ERTS_GLB_INLINE void erts_do_time_add(erts_short_time_t elapsed)
void erts_get_now_cpu(Uint* megasec, Uint* sec, Uint* microsec);
#endif
+typedef UWord erts_approx_time_t;
+erts_approx_time_t erts_get_approx_time(void);
+
void erts_get_timeval(SysTimeval *tv);
erts_time_t erts_get_time(void);
-void erts_get_emu_time(SysTimeval *);
ERTS_GLB_INLINE int erts_cmp_timeval(SysTimeval *t1p, SysTimeval *t2p);
diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c
index 1d0735aa99..c1b4408c06 100644
--- a/erts/emulator/beam/erl_time_sup.c
+++ b/erts/emulator/beam/erl_time_sup.c
@@ -91,6 +91,41 @@ static SysTimeval then; /* Used in get_now */
static SysTimeval last_emu_time; /* Used in erts_get_emu_time() */
SysTimeval erts_first_emu_time; /* Used in erts_get_emu_time() */
+union {
+ erts_smp_atomic_t time;
+ char align[ERTS_CACHE_LINE_SIZE];
+} approx erts_align_attribute(ERTS_CACHE_LINE_SIZE);
+
+static void
+init_approx_time(void)
+{
+ erts_smp_atomic_init_nob(&approx.time, 0);
+}
+
+static ERTS_INLINE erts_approx_time_t
+get_approx_time(void)
+{
+ return (erts_approx_time_t) erts_smp_atomic_read_nob(&approx.time);
+}
+
+static ERTS_INLINE void
+update_approx_time(SysTimeval *tv)
+{
+ erts_approx_time_t new_secs = (erts_approx_time_t) tv->tv_sec;
+ erts_approx_time_t old_secs = get_approx_time();
+ if (old_secs != new_secs)
+ erts_smp_atomic_set_nob(&approx.time, new_secs);
+}
+
+/*
+ * erts_get_approx_time() returns an *approximate* time
+ * in seconds. NOTE that this time may jump backwards!!!
+ */
+erts_approx_time_t
+erts_get_approx_time(void)
+{
+ return get_approx_time();
+}
#ifdef HAVE_GETHRTIME
@@ -398,6 +433,8 @@ erts_init_time_sup(void)
{
erts_smp_mtx_init(&erts_timeofday_mtx, "timeofday");
+ init_approx_time();
+
last_emu_time.tv_sec = 0;
last_emu_time.tv_usec = 0;
@@ -417,7 +454,7 @@ erts_init_time_sup(void)
gtv = inittv;
then.tv_sec = then.tv_usec = 0;
- erts_get_emu_time(&erts_first_emu_time);
+ erts_deliver_time();
return CLOCK_RESOLUTION;
}
@@ -873,6 +910,8 @@ get_now(Uint* megasec, Uint* sec, Uint* microsec)
*megasec = (Uint) (now.tv_sec / 1000000);
*sec = (Uint) (now.tv_sec % 1000000);
*microsec = (Uint) (now.tv_usec);
+
+ update_approx_time(&now);
}
void
@@ -885,6 +924,8 @@ get_sys_now(Uint* megasec, Uint* sec, Uint* microsec)
*megasec = (Uint) (now.tv_sec / 1000000);
*sec = (Uint) (now.tv_sec % 1000000);
*microsec = (Uint) (now.tv_usec);
+
+ update_approx_time(&now);
}
@@ -901,6 +942,8 @@ void erts_deliver_time(void) {
do_erts_deliver_time(&now);
erts_smp_mtx_unlock(&erts_timeofday_mtx);
+
+ update_approx_time(&now);
}
/* get *real* time (not ticks) remaining until next timeout - if there
@@ -949,6 +992,7 @@ void erts_get_timeval(SysTimeval *tv)
erts_smp_mtx_lock(&erts_timeofday_mtx);
get_tolerant_timeofday(tv);
erts_smp_mtx_unlock(&erts_timeofday_mtx);
+ update_approx_time(tv);
}
erts_time_t
@@ -961,7 +1005,9 @@ erts_get_time(void)
get_tolerant_timeofday(&sys_tv);
erts_smp_mtx_unlock(&erts_timeofday_mtx);
-
+
+ update_approx_time(&sys_tv);
+
return sys_tv.tv_sec;
}
@@ -977,38 +1023,3 @@ void erts_get_now_cpu(Uint* megasec, Uint* sec, Uint* microsec) {
*sec = (Uint)(tp.tv_sec % 1000000);
}
#endif
-
-
-/*
- * erts_get_emu_time() is similar to get_now(). You will
- * always get different times from erts_get_emu_time(), but they
- * may equal a time from get_now().
- *
- * erts_get_emu_time() is only used internally in the emulator in
- * order to order emulator internal events.
- */
-
-void
-erts_get_emu_time(SysTimeval *this_emu_time_p)
-{
- erts_smp_mtx_lock(&erts_timeofday_mtx);
-
- get_tolerant_timeofday(this_emu_time_p);
-
- /* Make sure time is later than last */
- if (last_emu_time.tv_sec > this_emu_time_p->tv_sec ||
- (last_emu_time.tv_sec == this_emu_time_p->tv_sec
- && last_emu_time.tv_usec >= this_emu_time_p->tv_usec)) {
- *this_emu_time_p = last_emu_time;
- this_emu_time_p->tv_usec++;
- }
- /* Check for carry from above + general reasonability */
- if (this_emu_time_p->tv_usec >= 1000000) {
- this_emu_time_p->tv_usec = 0;
- this_emu_time_p->tv_sec++;
- }
-
- last_emu_time = *this_emu_time_p;
-
- erts_smp_mtx_unlock(&erts_timeofday_mtx);
-}
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index 009ca1eb52..a321bc504b 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -151,8 +151,8 @@ do { (RES) = (TPID); } while(0)
#define ERTS_TRACER_REF_TYPE Process *
#define ERTS_GET_TRACER_REF(RES, TPID, TRACEE_FLGS) \
do { \
- (RES) = process_tab[internal_pid_index((TPID))]; \
- if (INVALID_PID((RES), (TPID)) || !((RES)->trace_flags & F_TRACER)) { \
+ (RES) = erts_proc_lookup((TPID)); \
+ if (!(RES) || !((RES)->trace_flags & F_TRACER)) { \
(TPID) = NIL; \
(TRACEE_FLGS) &= ~TRACEE_FLAGS; \
return; \
@@ -211,7 +211,7 @@ erts_set_system_seq_tracer(Process *c_p, ErtsProcLocks c_p_locks, Eterm new)
Eterm old = THE_NON_VALUE;
if (new != am_false) {
- if (!erts_pid2proc(c_p, c_p_locks, new, 0)
+ if (!erts_proc_lookup(new)
&& !erts_is_valid_tracer_port(new)) {
return old;
}
@@ -250,7 +250,7 @@ get_default_tracing(Uint *flagsp, Eterm *tracerp)
if (is_nil(default_tracer)) {
default_trace_flags &= ~TRACEE_FLAGS;
} else if (is_internal_pid(default_tracer)) {
- if (!erts_pid2proc(NULL, 0, default_tracer, 0)) {
+ if (!erts_proc_lookup(default_tracer)) {
reset_tracer:
default_trace_flags &= ~TRACEE_FLAGS;
default_tracer = NIL;
@@ -580,9 +580,10 @@ profile_send(Eterm from, Eterm message) {
ASSERT(is_internal_pid(profiler)
&& internal_pid_index(profiler) < erts_max_processes);
- profile_p = process_tab[internal_pid_index(profiler)];
+ profile_p = erts_proc_lookup(profiler);
- if (INVALID_PID(profile_p, profiler)) return;
+ if (!profile_p)
+ return;
sz = size_object(message);
hp = erts_alloc_message_heap(sz, &bp, &off_heap, profile_p, 0);
@@ -874,7 +875,7 @@ trace_send(Process *p, Eterm to, Eterm msg)
operation = am_send;
if (is_internal_pid(to)) {
- if (!erts_pid2proc(p, ERTS_PROC_LOCK_MAIN, to, 0))
+ if (!erts_proc_lookup(to))
goto send_to_non_existing_process;
}
else if(is_external_pid(to)
@@ -1116,8 +1117,8 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type,
#ifndef ERTS_SMP
- tracer = process_tab[internal_pid_index(seq_tracer)];
- if (INVALID_PID(tracer, tracer->id)) {
+ tracer = erts_proc_lookup(seq_tracer);
+ if (!tracer) {
system_seq_tracer = am_false;
return; /* no need to send anything */
}
@@ -2451,10 +2452,9 @@ monitor_long_gc(Process *p, Uint time) {
#ifndef ERTS_SMP
ASSERT(is_internal_pid(system_monitor)
&& internal_pid_index(system_monitor) < erts_max_processes);
- monitor_p = process_tab[internal_pid_index(system_monitor)];
- if (INVALID_PID(monitor_p, system_monitor) || p == monitor_p) {
+ monitor_p = erts_proc_lookup(system_monitor);
+ if (!monitor_p || p == monitor_p)
return;
- }
#endif
hsz = 0;
@@ -2527,8 +2527,8 @@ monitor_large_heap(Process *p) {
#ifndef ERTS_SMP
ASSERT(is_internal_pid(system_monitor)
&& internal_pid_index(system_monitor) < erts_max_processes);
- monitor_p = process_tab[internal_pid_index(system_monitor)];
- if (INVALID_PID(monitor_p, system_monitor) || p == monitor_p) {
+ monitor_p = erts_proc_lookup(system_monitor);
+ if (monitor_p || p == monitor_p) {
return;
}
#endif
@@ -2582,10 +2582,9 @@ monitor_generic(Process *p, Eterm type, Eterm spec) {
#ifndef ERTS_SMP
ASSERT(is_internal_pid(system_monitor)
&& internal_pid_index(system_monitor) < erts_max_processes);
- monitor_p = process_tab[internal_pid_index(system_monitor)];
- if (INVALID_PID(monitor_p, system_monitor) || p == monitor_p) {
+ monitor_p = erts_proc_lookup(system_monitor);
+ if (!monitor_p || p == monitor_p)
return;
- }
#endif
hp = ERTS_ALLOC_SYSMSG_HEAP(5, &bp, &off_heap, monitor_p);
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index b000e2c5d4..ded9646800 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1471,6 +1471,122 @@ void erl_drv_thr_init(void);
/* utils.c */
+typedef struct {
+#ifdef DEBUG
+ int smp_api;
+#endif
+ union {
+ Uint64 not_atomic;
+#ifdef ARCH_64
+ erts_atomic_t atomic;
+#else
+ erts_dw_atomic_t atomic;
+#endif
+ } counter;
+} erts_interval_t;
+
+void erts_interval_init(erts_interval_t *);
+void erts_smp_interval_init(erts_interval_t *);
+Uint64 erts_step_interval_nob(erts_interval_t *);
+Uint64 erts_step_interval_relb(erts_interval_t *);
+Uint64 erts_smp_step_interval_nob(erts_interval_t *);
+Uint64 erts_smp_step_interval_relb(erts_interval_t *);
+Uint64 erts_ensure_later_interval_nob(erts_interval_t *, Uint64);
+Uint64 erts_ensure_later_interval_acqb(erts_interval_t *, Uint64);
+Uint64 erts_smp_ensure_later_interval_nob(erts_interval_t *, Uint64);
+Uint64 erts_smp_ensure_later_interval_acqb(erts_interval_t *, Uint64);
+#ifdef ARCH_32
+ERTS_GLB_INLINE Uint64 erts_interval_dw_aint_to_val__(erts_dw_aint_t *);
+#endif
+ERTS_GLB_INLINE Uint64 erts_current_interval_nob__(erts_interval_t *);
+ERTS_GLB_INLINE Uint64 erts_current_interval_acqb__(erts_interval_t *);
+ERTS_GLB_INLINE Uint64 erts_current_interval_nob(erts_interval_t *);
+ERTS_GLB_INLINE Uint64 erts_current_interval_acqb(erts_interval_t *);
+ERTS_GLB_INLINE Uint64 erts_smp_current_interval_nob(erts_interval_t *);
+ERTS_GLB_INLINE Uint64 erts_smp_current_interval_acqb(erts_interval_t *);
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+#ifdef ARCH_32
+
+ERTS_GLB_INLINE Uint64
+erts_interval_dw_aint_to_val__(erts_dw_aint_t *dw)
+{
+#ifdef ETHR_SU_DW_NAINT_T__
+ return (Uint64) dw->dw_sint;
+#else
+ Uint64 res;
+ res = (Uint64) ((Uint32) dw->sint[ERTS_DW_AINT_HIGH_WORD]);
+ res <<= 32;
+ res |= (Uint64) ((Uint32) dw->sint[ERTS_DW_AINT_LOW_WORD]);
+ return res;
+#endif
+}
+
+#endif
+
+ERTS_GLB_INLINE Uint64
+erts_current_interval_nob__(erts_interval_t *icp)
+{
+#ifdef ARCH_64
+ return (Uint64) erts_atomic_read_nob(&icp->counter.atomic);
+#else
+ erts_dw_aint_t dw;
+ erts_dw_atomic_read_nob(&icp->counter.atomic, &dw);
+ return erts_interval_dw_aint_to_val__(&dw);
+#endif
+}
+
+ERTS_GLB_INLINE Uint64
+erts_current_interval_acqb__(erts_interval_t *icp)
+{
+#ifdef ARCH_64
+ return (Uint64) erts_atomic_read_acqb(&icp->counter.atomic);
+#else
+ erts_dw_aint_t dw;
+ erts_dw_atomic_read_acqb(&icp->counter.atomic, &dw);
+ return erts_interval_dw_aint_to_val__(&dw);
+#endif
+}
+
+ERTS_GLB_INLINE Uint64
+erts_current_interval_nob(erts_interval_t *icp)
+{
+ ASSERT(!icp->smp_api);
+ return erts_current_interval_nob__(icp);
+}
+
+ERTS_GLB_INLINE Uint64
+erts_current_interval_acqb(erts_interval_t *icp)
+{
+ ASSERT(!icp->smp_api);
+ return erts_current_interval_acqb__(icp);
+}
+
+ERTS_GLB_INLINE Uint64
+erts_smp_current_interval_nob(erts_interval_t *icp)
+{
+ ASSERT(icp->smp_api);
+#ifdef ERTS_SMP
+ return erts_current_interval_nob__(icp);
+#else
+ return icp->counter.not_atomic;
+#endif
+}
+
+ERTS_GLB_INLINE Uint64
+erts_smp_current_interval_acqb(erts_interval_t *icp)
+{
+ ASSERT(icp->smp_api);
+#ifdef ERTS_SMP
+ return erts_current_interval_acqb__(icp);
+#else
+ return icp->counter.not_atomic;
+#endif
+}
+
+#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
+
/*
* To be used to silence unused result warnings, but do not abuse it.
*/
@@ -1478,7 +1594,8 @@ void erts_silence_warn_unused_result(long unused);
void erts_cleanup_offheap(ErlOffHeap *offheap);
-Uint erts_fit_in_bits(Uint);
+int erts_fit_in_bits_int64(Sint64);
+int erts_fit_in_bits_int32(Sint32);
int list_length(Eterm);
Export* erts_find_function(Eterm, Eterm, unsigned int);
int erts_is_builtin(Eterm, Eterm, int);
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 8a2a43bebd..8d20178307 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -1319,7 +1319,7 @@ void init_io(void)
erts_max_ports = ERTS_MAX_R9_PORTS;
}
- port_extra_shift = erts_fit_in_bits(erts_max_ports - 1);
+ port_extra_shift = erts_fit_in_bits_int32(erts_max_ports - 1);
port_num_mask = (1 << ports_bits) - 1;
erts_port_tab_index_mask = ~(~((Uint) 0) << port_extra_shift);
@@ -1556,6 +1556,7 @@ deliver_result(Eterm sender, Eterm pid, Eterm res)
{
Process *rp;
ErtsProcLocks rp_locks = 0;
+ int scheduler = erts_get_scheduler_id() != 0;
ERTS_SMP_CHK_NO_PROC_LOCKS;
@@ -1563,7 +1564,9 @@ deliver_result(Eterm sender, Eterm pid, Eterm res)
&& is_internal_pid(pid)
&& internal_pid_index(pid) < erts_max_processes);
- rp = erts_pid2proc_opt(NULL, 0, pid, 0, ERTS_P2P_FLG_SMP_INC_REFC);
+ rp = (scheduler
+ ? erts_proc_lookup(pid)
+ : erts_pid2proc_opt(NULL, 0, pid, 0, ERTS_P2P_FLG_SMP_INC_REFC));
if (rp) {
Eterm tuple;
@@ -1581,7 +1584,9 @@ deliver_result(Eterm sender, Eterm pid, Eterm res)
#endif
);
erts_smp_proc_unlock(rp, rp_locks);
- erts_smp_proc_dec_refc(rp);
+ if (!scheduler)
+ erts_smp_proc_dec_refc(rp);
+
}
}
@@ -1606,6 +1611,7 @@ static void deliver_read_message(Port* prt, Eterm to,
ErlHeapFragment *bp;
ErlOffHeap *ohp;
ErtsProcLocks rp_locks = 0;
+ int scheduler = erts_get_scheduler_id() != 0;
ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
ERTS_SMP_CHK_NO_PROC_LOCKS;
@@ -1620,7 +1626,10 @@ static void deliver_read_message(Port* prt, Eterm to,
need += 2*len;
}
- rp = erts_pid2proc_opt(NULL, 0, to, 0, ERTS_P2P_FLG_SMP_INC_REFC);
+ rp = (scheduler
+ ? erts_proc_lookup(to)
+ : erts_pid2proc_opt(NULL, 0, to, 0, ERTS_P2P_FLG_SMP_INC_REFC));
+
if (!rp)
return;
@@ -1673,8 +1682,10 @@ static void deliver_read_message(Port* prt, Eterm to,
, NIL
#endif
);
- erts_smp_proc_unlock(rp, rp_locks);
- erts_smp_proc_dec_refc(rp);
+ if (rp_locks)
+ erts_smp_proc_unlock(rp, rp_locks);
+ if (!scheduler)
+ erts_smp_proc_dec_refc(rp);
}
/*
@@ -1741,6 +1752,7 @@ deliver_vec_message(Port* prt, /* Port */
ErlHeapFragment *bp;
ErlOffHeap *ohp;
ErtsProcLocks rp_locks = 0;
+ int scheduler = erts_get_scheduler_id() != 0;
ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
ERTS_SMP_CHK_NO_PROC_LOCKS;
@@ -1749,7 +1761,10 @@ deliver_vec_message(Port* prt, /* Port */
* Check arguments for validity.
*/
- rp = erts_pid2proc_opt(NULL, 0, to, 0, ERTS_P2P_FLG_SMP_INC_REFC);
+
+ rp = (scheduler
+ ? erts_proc_lookup(to)
+ : erts_pid2proc_opt(NULL, 0, to, 0, ERTS_P2P_FLG_SMP_INC_REFC));
if (!rp)
return;
@@ -1831,7 +1846,8 @@ deliver_vec_message(Port* prt, /* Port */
#endif
);
erts_smp_proc_unlock(rp, rp_locks);
- erts_smp_proc_dec_refc(rp);
+ if (!scheduler)
+ erts_smp_proc_dec_refc(rp);
}
@@ -2239,9 +2255,8 @@ void erts_port_command(Process *proc,
{
ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND;
- Process* rp = erts_pid2proc_opt(NULL, 0,
- port->connected, rp_locks,
- ERTS_P2P_FLG_SMP_INC_REFC);
+ Process* rp = erts_pid2proc(NULL, 0,
+ port->connected, rp_locks);
if (rp) {
(void) erts_send_exit_signal(NULL,
port->id,
@@ -2252,7 +2267,6 @@ void erts_port_command(Process *proc,
NULL,
0);
erts_smp_proc_unlock(rp, rp_locks);
- erts_smp_proc_dec_refc(rp);
}
}
@@ -2803,13 +2817,17 @@ void driver_report_exit(int ix, int status)
ErlHeapFragment *bp = NULL;
ErlOffHeap *ohp;
ErtsProcLocks rp_locks = 0;
+ int scheduler = erts_get_scheduler_id() != 0;
ERTS_SMP_CHK_NO_PROC_LOCKS;
ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
pid = prt->connected;
ASSERT(is_internal_pid(pid));
- rp = erts_pid2proc_opt(NULL, 0, pid, 0, ERTS_P2P_FLG_SMP_INC_REFC);
+
+ rp = (scheduler
+ ? erts_proc_lookup(pid)
+ : erts_pid2proc_opt(NULL, 0, pid, 0, ERTS_P2P_FLG_SMP_INC_REFC));
if (!rp)
return;
@@ -2826,7 +2844,8 @@ void driver_report_exit(int ix, int status)
);
erts_smp_proc_unlock(rp, rp_locks);
- erts_smp_proc_dec_refc(rp);
+ if (!scheduler)
+ erts_smp_proc_dec_refc(rp);
}
@@ -2957,6 +2976,7 @@ driver_deliver_term(ErlDrvPort port,
ErlOffHeap *ohp;
ErtsProcLocks rp_locks = 0;
struct b2t_states__ b2t;
+ int scheduler = 1; /* Silence erroneous warning... */
init_b2t_states(&b2t);
@@ -3148,7 +3168,13 @@ driver_deliver_term(ErlDrvPort port,
if (res <= 0)
goto done;
- rp = erts_pid2proc_opt(NULL, 0, to, rp_locks, ERTS_P2P_FLG_SMP_INC_REFC);
+ /*
+ * Increase refc on proc if done from a non-scheduler thread.
+ */
+ scheduler = erts_get_scheduler_id() != 0;
+ rp = (scheduler
+ ? erts_proc_lookup(to)
+ : erts_pid2proc_opt(NULL, 0, to, 0, ERTS_P2P_FLG_SMP_INC_REFC));
if (!rp) {
res = 0;
goto done;
@@ -3394,7 +3420,8 @@ driver_deliver_term(ErlDrvPort port,
if (rp) {
if (rp_locks)
erts_smp_proc_unlock(rp, rp_locks);
- erts_smp_proc_dec_refc(rp);
+ if (!scheduler)
+ erts_smp_proc_dec_refc(rp);
}
#endif
cleanup_b2t_states(&b2t);
diff --git a/erts/emulator/beam/register.c b/erts/emulator/beam/register.c
index 26d64887d0..0ddae56322 100644
--- a/erts/emulator/beam/register.c
+++ b/erts/emulator/beam/register.c
@@ -96,12 +96,7 @@ reg_safe_write_lock(Process *c_p, ErtsProcLocks *c_p_locks)
static ERTS_INLINE int
is_proc_alive(Process *p)
{
- int res;
- erts_pix_lock_t *pixlck = ERTS_PID2PIXLOCK(p->id);
- erts_pix_lock(pixlck);
- res = !p->is_exiting;
- erts_pix_unlock(pixlck);
- return res;
+ return !ERTS_PROC_IS_EXITING(p);
}
#endif
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 7b2bb81f62..0d3b910278 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -39,6 +39,8 @@
#define ENABLE_CHILD_WAITER_THREAD 1
#endif
+#define ERTS_I64_LITERAL(X) X##LL
+
#if defined (__WIN32__)
# include "erl_win_sys.h"
#elif defined (VXWORKS)
@@ -91,14 +93,22 @@ typedef ERTS_SYS_FD_TYPE ErtsSysFdType;
# endif
#endif
-#ifdef __GNUC__
-# if __GNUC__ < 3 && (__GNUC__ != 2 || __GNUC_MINOR__ < 96)
-# define ERTS_LIKELY(BOOL) (BOOL)
-# define ERTS_UNLIKELY(BOOL) (BOOL)
-# else
-# define ERTS_LIKELY(BOOL) __builtin_expect((BOOL), !0)
-# define ERTS_UNLIKELY(BOOL) __builtin_expect((BOOL), 0)
-# endif
+#if !defined(__GNUC__)
+# define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) 0
+#elif !defined(__GNUC_MINOR__)
+# define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \
+ ((__GNUC__ << 24) >= (((MAJ) << 24) | ((MIN) << 12) | (PL)))
+#elif !defined(__GNUC_PATCHLEVEL__)
+# define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \
+ (((__GNUC__ << 24) | (__GNUC_MINOR__ << 12)) >= (((MAJ) << 24) | ((MIN) << 12) | (PL)))
+#else
+# define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \
+ (((__GNUC__ << 24) | (__GNUC_MINOR__ << 12) | __GNUC_PATCHLEVEL__) >= (((MAJ) << 24) | ((MIN) << 12) | (PL)))
+#endif
+
+#if ERTS_AT_LEAST_GCC_VSN__(2, 96, 0)
+# define ERTS_LIKELY(BOOL) __builtin_expect((BOOL), !0)
+# define ERTS_UNLIKELY(BOOL) __builtin_expect((BOOL), 0)
#else
# define ERTS_LIKELY(BOOL) (BOOL)
# define ERTS_UNLIKELY(BOOL) (BOOL)
@@ -178,6 +188,18 @@ int real_printf(const char *fmt, ...);
# define printf real_printf
#endif
+#undef __deprecated
+#if ERTS_AT_LEAST_GCC_VSN__(3, 0, 0)
+# define __deprecated __attribute__((deprecated))
+#else
+# define __deprecated
+#endif
+#if ERTS_AT_LEAST_GCC_VSN__(3, 0, 4)
+# define erts_align_attribute(SZ) __attribute__ ((aligned (SZ)))
+#else
+# define erts_align_attribute(SZ)
+#endif
+
/* In VC++, noreturn is a declspec that has to be before the types,
* but in GNUC it is an att ribute to be placed between return type
* and function name, hence __decl_noreturn <types> __noreturn <function name>
@@ -185,12 +207,6 @@ int real_printf(const char *fmt, ...);
#if __GNUC__
# define __decl_noreturn
# define __noreturn __attribute__((noreturn))
-# undef __deprecated
-# if __GNUC__ >= 3
-# define __deprecated __attribute__((deprecated))
-# else
-# define __deprecated
-# endif
#else
# if defined(__WIN32__) && defined(_MSC_VER)
# define __noreturn
@@ -199,7 +215,6 @@ int real_printf(const char *fmt, ...);
# define __noreturn
# define __decl_noreturn
# endif
-# define __deprecated
#endif
/*
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index a36d15204e..245916bc0d 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -268,16 +268,42 @@ list_length(Eterm list)
return i;
}
-Uint erts_fit_in_bits(Uint n)
+static const struct {
+ Sint64 mask;
+ int bits;
+} fib_data[] = {{ERTS_I64_LITERAL(0x2), 1},
+ {ERTS_I64_LITERAL(0xc), 2},
+ {ERTS_I64_LITERAL(0xf0), 4},
+ {ERTS_I64_LITERAL(0xff00), 8},
+ {ERTS_I64_LITERAL(0xffff0000), 16},
+ {ERTS_I64_LITERAL(0xffffffff00000000), 32}};
+
+static ERTS_INLINE int
+fit_in_bits(Sint64 value, int start)
{
- Uint i;
+ int bits = 0;
+ int i;
- i = 0;
- while (n > 0) {
- i++;
- n >>= 1;
- }
- return i;
+ for (i = start; i >= 0; i--) {
+ if (value & fib_data[i].mask) {
+ value >>= fib_data[i].bits;
+ bits |= fib_data[i].bits;
+ }
+ }
+
+ bits++;
+
+ return bits;
+}
+
+int erts_fit_in_bits_int64(Sint64 value)
+{
+ return fit_in_bits(value, 5);
+}
+
+int erts_fit_in_bits_int32(Sint32 value)
+{
+ return fit_in_bits((Sint64) (Uint32) value, 4);
}
int
@@ -3240,7 +3266,7 @@ ptimer_timeout(ErtsSmpPTimer *ptimer)
ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS,
ERTS_P2P_FLG_ALLOW_OTHER_X);
if (p) {
- if (!p->is_exiting
+ if (!ERTS_PROC_IS_EXITING(p)
&& !(ptimer->timer.flags & ERTS_PTMR_FLG_CANCELLED)) {
ASSERT(*ptimer->timer.timer_ref == ptimer);
*ptimer->timer.timer_ref = NULL;
@@ -3458,6 +3484,254 @@ void erts_silence_warn_unused_result(long unused)
}
+/*
+ * Interval counts
+ */
+void
+erts_interval_init(erts_interval_t *icp)
+{
+#ifdef ARCH_64
+ erts_atomic_init_nob(&icp->counter.atomic, 0);
+#else
+ erts_dw_aint_t dw;
+#ifdef ETHR_SU_DW_NAINT_T__
+ dw.dw_sint = 0;
+#else
+ dw.sint[ERTS_DW_AINT_HIGH_WORD] = 0;
+ dw.sint[ERTS_DW_AINT_LOW_WORD] = 0;
+#endif
+ erts_dw_atomic_init_nob(&icp->counter.atomic, &dw);
+
+#endif
+#ifdef DEBUG
+ icp->smp_api = 0;
+#endif
+}
+
+void
+erts_smp_interval_init(erts_interval_t *icp)
+{
+#ifdef ERTS_SMP
+ erts_interval_init(icp);
+#else
+ icp->counter.not_atomic = 0;
+#endif
+#ifdef DEBUG
+ icp->smp_api = 1;
+#endif
+}
+
+static ERTS_INLINE Uint64
+step_interval_nob(erts_interval_t *icp)
+{
+#ifdef ARCH_64
+ return (Uint64) erts_atomic_inc_read_nob(&icp->counter.atomic);
+#else
+ erts_dw_aint_t exp;
+
+ erts_dw_atomic_read_nob(&icp->counter.atomic, &exp);
+ while (1) {
+ erts_dw_aint_t new = exp;
+
+#ifdef ETHR_SU_DW_NAINT_T__
+ new.dw_sint++;
+#else
+ new.sint[ERTS_DW_AINT_LOW_WORD]++;
+ if (new.sint[ERTS_DW_AINT_LOW_WORD] == 0)
+ new.sint[ERTS_DW_AINT_HIGH_WORD]++;
+#endif
+
+ if (erts_dw_atomic_cmpxchg_nob(&icp->counter.atomic, &new, &exp))
+ return erts_interval_dw_aint_to_val__(&new);
+
+ }
+#endif
+}
+
+static ERTS_INLINE Uint64
+step_interval_relb(erts_interval_t *icp)
+{
+#ifdef ARCH_64
+ return (Uint64) erts_atomic_inc_read_relb(&icp->counter.atomic);
+#else
+ erts_dw_aint_t exp;
+
+ erts_dw_atomic_read_nob(&icp->counter.atomic, &exp);
+ while (1) {
+ erts_dw_aint_t new = exp;
+
+#ifdef ETHR_SU_DW_NAINT_T__
+ new.dw_sint++;
+#else
+ new.sint[ERTS_DW_AINT_LOW_WORD]++;
+ if (new.sint[ERTS_DW_AINT_LOW_WORD] == 0)
+ new.sint[ERTS_DW_AINT_HIGH_WORD]++;
+#endif
+
+ if (erts_dw_atomic_cmpxchg_relb(&icp->counter.atomic, &new, &exp))
+ return erts_interval_dw_aint_to_val__(&new);
+
+ }
+#endif
+}
+
+
+static ERTS_INLINE Uint64
+ensure_later_interval_nob(erts_interval_t *icp, Uint64 ic)
+{
+ Uint64 curr_ic;
+#ifdef ARCH_64
+ curr_ic = (Uint64) erts_atomic_read_nob(&icp->counter.atomic);
+ if (curr_ic > ic)
+ return curr_ic;
+ return (Uint64) erts_atomic_inc_read_nob(&icp->counter.atomic);
+#else
+ erts_dw_aint_t exp;
+
+ erts_dw_atomic_read_nob(&icp->counter.atomic, &exp);
+ curr_ic = erts_interval_dw_aint_to_val__(&exp);
+ if (curr_ic > ic)
+ return curr_ic;
+
+ while (1) {
+ erts_dw_aint_t new = exp;
+
+#ifdef ETHR_SU_DW_NAINT_T__
+ new.dw_sint++;
+#else
+ new.sint[ERTS_DW_AINT_LOW_WORD]++;
+ if (new.sint[ERTS_DW_AINT_LOW_WORD] == 0)
+ new.sint[ERTS_DW_AINT_HIGH_WORD]++;
+#endif
+
+ if (erts_dw_atomic_cmpxchg_nob(&icp->counter.atomic, &new, &exp))
+ return erts_interval_dw_aint_to_val__(&new);
+
+ curr_ic = erts_interval_dw_aint_to_val__(&exp);
+ if (curr_ic > ic)
+ return curr_ic;
+ }
+#endif
+}
+
+
+static ERTS_INLINE Uint64
+ensure_later_interval_acqb(erts_interval_t *icp, Uint64 ic)
+{
+ Uint64 curr_ic;
+#ifdef ARCH_64
+ curr_ic = (Uint64) erts_atomic_read_acqb(&icp->counter.atomic);
+ if (curr_ic > ic)
+ return curr_ic;
+ return (Uint64) erts_atomic_inc_read_acqb(&icp->counter.atomic);
+#else
+ erts_dw_aint_t exp;
+
+ erts_dw_atomic_read_acqb(&icp->counter.atomic, &exp);
+ curr_ic = erts_interval_dw_aint_to_val__(&exp);
+ if (curr_ic > ic)
+ return curr_ic;
+
+ while (1) {
+ erts_dw_aint_t new = exp;
+
+#ifdef ETHR_SU_DW_NAINT_T__
+ new.dw_sint++;
+#else
+ new.sint[ERTS_DW_AINT_LOW_WORD]++;
+ if (new.sint[ERTS_DW_AINT_LOW_WORD] == 0)
+ new.sint[ERTS_DW_AINT_HIGH_WORD]++;
+#endif
+
+ if (erts_dw_atomic_cmpxchg_acqb(&icp->counter.atomic, &new, &exp))
+ return erts_interval_dw_aint_to_val__(&new);
+
+ curr_ic = erts_interval_dw_aint_to_val__(&exp);
+ if (curr_ic > ic)
+ return curr_ic;
+ }
+#endif
+}
+
+Uint64
+erts_step_interval_nob(erts_interval_t *icp)
+{
+ ASSERT(!icp->smp_api);
+ return step_interval_nob(icp);
+}
+
+Uint64
+erts_step_interval_relb(erts_interval_t *icp)
+{
+ ASSERT(!icp->smp_api);
+ return step_interval_relb(icp);
+}
+
+Uint64
+erts_smp_step_interval_nob(erts_interval_t *icp)
+{
+ ASSERT(icp->smp_api);
+#ifdef ERTS_SMP
+ return step_interval_nob(icp);
+#else
+ return ++icp->counter.not_atomic;
+#endif
+}
+
+Uint64
+erts_smp_step_interval_relb(erts_interval_t *icp)
+{
+ ASSERT(icp->smp_api);
+#ifdef ERTS_SMP
+ return step_interval_relb(icp);
+#else
+ return ++icp->counter.not_atomic;
+#endif
+}
+
+Uint64
+erts_ensure_later_interval_nob(erts_interval_t *icp, Uint64 ic)
+{
+ ASSERT(!icp->smp_api);
+ return ensure_later_interval_nob(icp, ic);
+}
+
+Uint64
+erts_ensure_later_interval_acqb(erts_interval_t *icp, Uint64 ic)
+{
+ ASSERT(!icp->smp_api);
+ return ensure_later_interval_acqb(icp, ic);
+}
+
+Uint64
+erts_smp_ensure_later_interval_nob(erts_interval_t *icp, Uint64 ic)
+{
+ ASSERT(icp->smp_api);
+#ifdef ERTS_SMP
+ return ensure_later_interval_nob(icp, ic);
+#else
+ if (icp->counter.not_atomic > ic)
+ return icp->counter.not_atomic;
+ else
+ return ++icp->counter.not_atomic;
+#endif
+}
+
+Uint64
+erts_smp_ensure_later_interval_acqb(erts_interval_t *icp, Uint64 ic)
+{
+ ASSERT(icp->smp_api);
+#ifdef ERTS_SMP
+ return ensure_later_interval_acqb(icp, ic);
+#else
+ if (icp->counter.not_atomic > ic)
+ return icp->counter.not_atomic;
+ else
+ return ++icp->counter.not_atomic;
+#endif
+}
+
+
#ifdef DEBUG
/*
* Handy functions when using a debugger - don't use in the code!
@@ -3490,7 +3764,7 @@ Process *p;
void ppi(Eterm pid)
{
- pp(erts_pid2proc_unlocked(pid));
+ pp(erts_proc_lookup(pid));
}
void td(Eterm x)
diff --git a/erts/emulator/sys/win32/erl_win_sys.h b/erts/emulator/sys/win32/erl_win_sys.h
index 03298a6c54..21c52ca26d 100644
--- a/erts/emulator/sys/win32/erl_win_sys.h
+++ b/erts/emulator/sys/win32/erl_win_sys.h
@@ -90,6 +90,11 @@
#define strncasecmp _strnicmp
+#ifndef __GNUC__
+# undef ERTS_I64_LITERAL
+# define ERTS_I64_LITERAL(X) X##i64
+#endif
+
/*
* Practial Windows specific macros.
*/
diff --git a/erts/emulator/sys/win32/sys_time.c b/erts/emulator/sys/win32/sys_time.c
index b5123dc45d..2f2dfc8197 100644
--- a/erts/emulator/sys/win32/sys_time.c
+++ b/erts/emulator/sys/win32/sys_time.c
@@ -26,11 +26,7 @@
#include "sys.h"
#include "assert.h"
-#ifdef __GNUC__
-#define LL_LITERAL(X) X##LL
-#else
-#define LL_LITERAL(X) X##i64
-#endif
+#define LL_LITERAL(X) ERTS_I64_LITERAL(X)
/******************* Routines for time measurement *********************/
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-03 05:52:24 +0000