aboutsummaryrefslogtreecommitdiffstats
path: root/erts/emulator/beam/erl_process.c
diff options
context:
space:
mode:
Diffstat (limited to 'erts/emulator/beam/erl_process.c')
-rw-r--r--erts/emulator/beam/erl_process.c2076
1 files changed, 318 insertions, 1758 deletions
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 61554780c4..aaca4b5f59 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -42,6 +42,7 @@
#include "erl_thr_queue.h"
#include "erl_async.h"
#include "dtrace-wrapper.h"
+#include "erl_ptab.h"
#define ERTS_DELAYED_WAKEUP_INFINITY (~(Uint64) 0)
#define ERTS_DELAYED_WAKEUP_REDUCTIONS ((Uint64) CONTEXT_REDS/2)
@@ -136,155 +137,16 @@ do { \
#define ERTS_EMPTY_RUNQ_PORTS(RQ) \
(RUNQ_READ_LEN(&(RQ)->ports.info.len) == 0 && (RQ)->misc.start == NULL)
+const Process erts_invalid_process = {{ERTS_INVALID_PID}};
+
extern BeamInstr beam_apply[];
extern BeamInstr beam_exit[];
extern BeamInstr beam_continue_exit[];
-#ifdef ARCH_32
-
-union {
- erts_smp_dw_atomic_t pid_data;
- char align[ERTS_CACHE_LINE_SIZE];
-} last erts_align_attribute(ERTS_CACHE_LINE_SIZE);
-
-
-static ERTS_INLINE Uint64
-dw_aint_to_uint64(erts_dw_aint_t *dw)
-{
-#ifdef ETHR_SU_DW_NAINT_T__
- return (Uint64) dw->dw_sint;
-#else
- Uint64 res;
- res = (Uint64) ((Uint32) dw->sint[ERTS_DW_AINT_HIGH_WORD]);
- res <<= 32;
- res |= (Uint64) ((Uint32) dw->sint[ERTS_DW_AINT_LOW_WORD]);
- return res;
-#endif
-}
-
-static void
-unint64_to_dw_aint(erts_dw_aint_t *dw, Uint64 val)
-{
-#ifdef ETHR_SU_DW_NAINT_T__
- dw->dw_sint = (ETHR_SU_DW_NAINT_T__) val;
-#else
- dw->sint[ERTS_DW_AINT_LOW_WORD] = (erts_aint_t) (val & 0xffffffff);
- dw->sint[ERTS_DW_AINT_HIGH_WORD] = (erts_aint_t) ((val >> 32) & 0xffffffff);
-#endif
-}
-
-static ERTS_INLINE void
-last_pid_data_init_nob(Uint64 val)
-{
- erts_dw_aint_t dw;
- unint64_to_dw_aint(&dw, val);
- erts_smp_dw_atomic_init_nob(&last.pid_data, &dw);
-}
-
-static ERTS_INLINE void
-last_pid_data_set_relb(Uint64 val)
-{
- erts_dw_aint_t dw;
- unint64_to_dw_aint(&dw, val);
- erts_smp_dw_atomic_set_relb(&last.pid_data, &dw);
-}
-
-static ERTS_INLINE Uint64
-last_pid_data_read_nob(void)
-{
- erts_dw_aint_t dw;
- erts_smp_dw_atomic_read_nob(&last.pid_data, &dw);
- return dw_aint_to_uint64(&dw);
-}
-
-static ERTS_INLINE Uint64
-last_pid_data_read_acqb(void)
-{
- erts_dw_aint_t dw;
- erts_smp_dw_atomic_read_acqb(&last.pid_data, &dw);
- return dw_aint_to_uint64(&dw);
-}
-
-static ERTS_INLINE Uint64
-last_pid_data_cmpxchg_relb(Uint64 new, Uint64 exp)
-{
- erts_dw_aint_t dw_new, dw_xchg;
-
- unint64_to_dw_aint(&dw_new, new);
- unint64_to_dw_aint(&dw_xchg, exp);
-
- if (erts_smp_dw_atomic_cmpxchg_relb(&last.pid_data, &dw_new, &dw_xchg))
- return exp;
- else
- return dw_aint_to_uint64(&dw_xchg);
-}
-
-#elif defined(ARCH_64)
-
-union {
- erts_smp_atomic_t pid_data;
- char align[ERTS_CACHE_LINE_SIZE];
-} last erts_align_attribute(ERTS_CACHE_LINE_SIZE);
-
-static ERTS_INLINE void
-last_pid_data_init_nob(Uint64 val)
-{
- erts_smp_atomic_init_nob(&last.pid_data, (erts_aint_t) val);
-}
-
-static ERTS_INLINE void
-last_pid_data_set_relb(Uint64 val)
-{
- erts_smp_atomic_set_relb(&last.pid_data, (erts_aint_t) val);
-}
-
-static ERTS_INLINE Uint64
-last_pid_data_read_nob(void)
-{
- return (Uint64) erts_smp_atomic_read_nob(&last.pid_data);
-}
-
-static ERTS_INLINE Uint64
-last_pid_data_read_acqb(void)
-{
- return (Uint64) erts_smp_atomic_read_acqb(&last.pid_data);
-}
-
-static ERTS_INLINE Uint64
-last_pid_data_cmpxchg_relb(Uint64 new, Uint64 exp)
-{
- return (Uint64) erts_smp_atomic_cmpxchg_relb(&last.pid_data,
- (erts_aint_t) new,
- (erts_aint_t) exp);
-}
-
-#else
-# error "Not 64-bit, nor 32-bit architecture..."
-#endif
-
-static ERTS_INLINE int
-last_pid_data_cmp(Uint64 lpd1, Uint64 lpd2)
-{
- Uint64 lpd1_wrap;
-
- if (lpd1 == lpd2)
- return 0;
-
- lpd1_wrap = lpd1 + (((Uint64) 1) << 63);
-
- if (lpd1 < lpd1_wrap)
- return (lpd1 < lpd2 && lpd2 < lpd1_wrap) ? -1 : 1;
- else
- return (lpd1_wrap <= lpd2 && lpd2 < lpd1) ? 1 : -1;
-}
-
-
-#define ERTS_PID_DATA_MASK__ ((1 << _PID_DATA_SIZE) - 1)
-
int erts_sched_compact_load;
Uint erts_no_schedulers;
-ErtsProcTab erts_proc erts_align_attribute(ERTS_CACHE_LINE_SIZE);
+ErtsPTab erts_proc erts_align_attribute(ERTS_CACHE_LINE_SIZE);
int erts_sched_thread_suggested_stack_size = -1;
@@ -371,8 +233,6 @@ erts_sched_stat_t erts_sched_stat;
static erts_tsd_key_t sched_data_key;
#endif
-erts_smp_rwmtx_t erts_proc_tab_rwmtx;
-
static erts_smp_atomic32_t function_calls;
#ifdef ERTS_SMP
@@ -409,29 +269,6 @@ struct erts_system_profile_flags_t erts_system_profile_flags;
#if ERTS_MAX_PROCESSES > 0x7fffffff
#error "Need to store process_count in another type"
#endif
-static erts_smp_atomic32_t process_count;
-
-typedef struct ErtsTermProcElement_ ErtsTermProcElement;
-struct ErtsTermProcElement_ {
- ErtsTermProcElement *next;
- ErtsTermProcElement *prev;
- int ix;
- union {
- struct {
- Eterm pid;
- Uint64 spawned;
- Uint64 exited;
- } process;
- struct {
- Uint64 interval;
- } bif_invocation;
- } u;
-};
-
-static struct {
- ErtsTermProcElement *start;
- ErtsTermProcElement *end;
-} saved_term_procs;
ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(misc_op_list,
ErtsMiscOpList,
@@ -493,8 +330,6 @@ do { \
* Local functions.
*/
-static void init_processes_bif(void);
-static void save_terminating_process(Process *p);
static void exec_misc_ops(ErtsRunQueue *);
static void print_function_from_pc(int to, void *to_arg, BeamInstr* x);
static int stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp,
@@ -562,39 +397,17 @@ erts_smp_lc_runq_is_locked(ErtsRunQueue *runq)
}
#endif
-static erts_interval_t *proc_interval;
-
-static void
-proc_interval_init(void)
-{
- proc_interval = erts_alloc_permanent_cache_aligned(
- ERTS_ALC_T_PROC_INTERVAL,
- sizeof(erts_interval_t));
- erts_smp_interval_init(proc_interval);
-}
-
-static ERTS_INLINE Uint64
-get_proc_interval(void)
-{
- return erts_smp_current_interval_nob(proc_interval);
-}
static ERTS_INLINE Uint64
ensure_later_proc_interval(Uint64 interval)
{
- return erts_smp_ensure_later_interval_nob(proc_interval, interval);
-}
-
-static ERTS_INLINE Uint64
-step_proc_interval(void)
-{
- return erts_smp_step_interval_nob(proc_interval);
+ return erts_smp_ensure_later_interval_nob(erts_ptab_interval(&erts_proc), interval);
}
Uint64
erts_get_proc_interval(void)
{
- return get_proc_interval();
+ return erts_smp_current_interval_nob(erts_ptab_interval(&erts_proc));
}
Uint64
@@ -606,7 +419,7 @@ erts_ensure_later_proc_interval(Uint64 interval)
Uint64
erts_step_proc_interval(void)
{
- return step_proc_interval();
+ return erts_smp_step_interval_nob(erts_ptab_interval(&erts_proc));
}
void
@@ -654,20 +467,18 @@ erts_pre_init_process(void)
#endif
}
+#ifdef ERTS_SMP
+static void
+release_process(void *vproc)
+{
+ erts_smp_proc_dec_refc((Process *) vproc);
+}
+#endif
+
/* initialize the scheduler */
void
-erts_init_process(int ncpu)
+erts_init_process(int ncpu, int proc_tab_size)
{
- int proc_tab_sz;
- int max_proc_bits;
- int proc_bits = ERTS_PROC_BITS;
- erts_smp_atomic_t *proc_entry;
- char *proc_tab_end;
- erts_smp_rwmtx_opt_t proc_tab_rwmtx_opts = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
- proc_tab_rwmtx_opts.type = ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ;
- proc_tab_rwmtx_opts.lived = ERTS_SMP_RWMTX_LONG_LIVED;
-
- proc_interval_init();
#ifdef ERTS_SMP
erts_disable_proc_not_running_opt = 0;
@@ -676,52 +487,16 @@ erts_init_process(int ncpu)
init_proclist_alloc();
- erts_smp_atomic32_init_nob(&process_count, 0);
-
- if (erts_use_r9_pids_ports)
- proc_bits = ERTS_R9_PROC_BITS;
-
- if (erts_proc.max > (1 << proc_bits))
- erts_proc.max = 1 << proc_bits;
-
- proc_tab_sz = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(erts_proc.max
- * sizeof(erts_smp_atomic_t));
- erts_proc.tab = erts_alloc(ERTS_ALC_T_PROC_TABLE, proc_tab_sz);
- proc_tab_end = ((char *) erts_proc.tab) + proc_tab_sz;
- proc_entry = erts_proc.tab;
- while (proc_tab_end > ((char *) proc_entry)) {
- erts_smp_atomic_init_nob(proc_entry, ERTS_AINT_NULL);
- proc_entry++;
- }
-
- erts_smp_rwmtx_init_opt(&erts_proc_tab_rwmtx,
- &proc_tab_rwmtx_opts,
- "proc_tab");
- last_pid_data_init_nob(~((Uint64) 0));
-
- max_proc_bits = erts_fit_in_bits_int32((Sint32) erts_proc.max - 1);
-
- erts_proc.tab_cache_lines = proc_tab_sz/ERTS_CACHE_LINE_SIZE;
- erts_proc.pix_per_cache_line = ERTS_CACHE_LINE_SIZE/sizeof(erts_smp_atomic_t);
- if ((erts_proc.max & (erts_proc.max - 1))
- | (erts_proc.pix_per_cache_line & (erts_proc.pix_per_cache_line - 1))) {
- /*
- * erts_proc.max or erts_proc.pix_per_cache_line
- * not a power of 2 :(
- */
- erts_proc.pix_cl_mask = 0;
- erts_proc.pix_cl_shift = 0;
- erts_proc.pix_cli_mask = 0;
- erts_proc.pix_cli_shift = 0;
- }
- else {
- ASSERT((erts_proc.tab_cache_lines
- & (erts_proc.tab_cache_lines - 1)) == 0);
- erts_proc.pix_cl_mask = erts_proc.tab_cache_lines-1;
- erts_proc.pix_cl_shift = erts_fit_in_bits_int32(erts_proc.pix_per_cache_line-1);
- erts_proc.pix_cli_shift = erts_fit_in_bits_int32(erts_proc.pix_cl_mask);
- erts_proc.pix_cli_mask = (1 << (max_proc_bits - erts_proc.pix_cli_shift)) - 1;
- }
+ erts_ptab_init_table(&erts_proc,
+ ERTS_ALC_T_PROC_TABLE,
+#ifdef ERTS_SMP
+ release_process,
+#else
+ NULL,
+#endif
+ (ErtsPTabElementCommon *) &erts_invalid_process.common,
+ proc_tab_size,
+ "process_table");
last_reductions = 0;
last_exact_reductions = 0;
@@ -732,7 +507,6 @@ void
erts_late_init_process(void)
{
int ix;
- init_processes_bif();
erts_smp_spinlock_init(&erts_sched_stat.lock, "sched_stat");
for (ix = 0; ix < ERTS_NO_PRIO_LEVELS; ix++) {
@@ -980,9 +754,9 @@ static ERTS_INLINE ErtsProcList *
proclist_create(Process *p)
{
ErtsProcList *plp = proclist_alloc();
- ensure_later_proc_interval(p->started_interval);
- plp->pid = p->id;
- plp->started_interval = p->started_interval;
+ ensure_later_proc_interval(p->common.u.alive.started_interval);
+ plp->pid = p->common.id;
+ plp->started_interval = p->common.u.alive.started_interval;
return plp;
}
@@ -992,12 +766,6 @@ proclist_destroy(ErtsProcList *plp)
proclist_free(plp);
}
-static ERTS_INLINE int
-proclist_same(ErtsProcList *plp, Process *p)
-{
- return plp->pid == p->id && plp->started_interval == p->started_interval;
-}
-
ErtsProcList *
erts_proclist_create(Process *p)
{
@@ -1010,12 +778,6 @@ erts_proclist_destroy(ErtsProcList *plp)
proclist_destroy(plp);
}
-int
-erts_proclist_same(ErtsProcList *plp, Process *p)
-{
- return proclist_same(plp, p);
-}
-
void *
erts_psd_set_init(Process *p, ErtsProcLocks plocks, int ix, void *data)
{
@@ -1776,37 +1538,32 @@ handle_reap_ports(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting)
unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_REAP_PORTS);
awdp->esdp->run_queue->halt_in_progress = 1;
if (erts_smp_atomic32_dec_read_acqb(&erts_halt_progress) == 0) {
- int i;
+ int i, max = erts_ptab_max(&erts_port);
erts_smp_atomic32_set_nob(&erts_halt_progress, 1);
- for (i = 0; i < erts_max_ports; i++) {
- Port *prt = &erts_port[i];
- erts_smp_port_state_lock(prt);
- if ((prt->status & (ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP
- | ERTS_PORT_SFLG_HALT))) {
- erts_smp_port_state_unlock(prt);
- continue;
- }
- /* We need to set the halt flag - get the port lock */
-#ifdef ERTS_SMP
- erts_smp_atomic_inc_nob(&prt->refc);
-#endif
- erts_smp_port_state_unlock(prt);
-#ifdef ERTS_SMP
- erts_smp_mtx_lock(prt->lock);
-#endif
- if ((prt->status & (ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP
- | ERTS_PORT_SFLG_HALT))) {
- erts_port_release(prt);
+ for (i = 0; i < max; i++) {
+ erts_aint32_t state;
+ Port *prt = erts_pix2port(i);
+ if (!prt)
continue;
- }
- erts_port_status_bor_set(prt, ERTS_PORT_SFLG_HALT);
- erts_smp_atomic32_inc_nob(&erts_halt_progress);
- if (prt->status & (ERTS_PORT_SFLG_EXITING
- | ERTS_PORT_SFLG_CLOSING)) {
- erts_port_release(prt);
+ state = erts_atomic32_read_acqb(&prt->state);
+ if (state & (ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP
+ | ERTS_PORT_SFLG_HALT))
continue;
+
+ /* We need to set the halt flag - get the port lock */
+
+ erts_smp_port_lock(prt);
+
+ state = erts_atomic32_read_nob(&prt->state);
+ if (!(state & (ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP
+ | ERTS_PORT_SFLG_HALT))) {
+ state = erts_atomic32_read_bor_relb(&prt->state,
+ ERTS_PORT_SFLG_HALT);
+ erts_smp_atomic32_inc_nob(&erts_halt_progress);
+ if (!(state & (ERTS_PORT_SFLG_EXITING|ERTS_PORT_SFLG_CLOSING)))
+ erts_deliver_port_exit(prt, prt->common.id, am_killed, 0);
}
- erts_do_exit_port(prt, prt->id, am_killed);
+
erts_port_release(prt);
}
if (erts_smp_atomic32_dec_read_nob(&erts_halt_progress) == 0) {
@@ -3183,12 +2940,12 @@ resume_run_queue(ErtsRunQueue *rq)
erts_smp_runq_lock(rq);
- (void) ERTS_RUNQ_FLGS_MASK_SET(rq,
- (ERTS_RUNQ_FLG_OUT_OF_WORK
- | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK
- | ERTS_RUNQ_FLG_SUSPENDED),
- (ERTS_RUNQ_FLG_OUT_OF_WORK
- | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK));
+ (void) ERTS_RUNQ_FLGS_READ_BSET(rq,
+ (ERTS_RUNQ_FLG_OUT_OF_WORK
+ | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK
+ | ERTS_RUNQ_FLG_SUSPENDED),
+ (ERTS_RUNQ_FLG_OUT_OF_WORK
+ | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK));
rq->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS;
for (pix = 0; pix < ERTS_NO_PROC_PRIO_LEVELS; pix++) {
@@ -4210,7 +3967,7 @@ erts_fprintf(stderr, "--------------------------------\n");
ERTS_DBG_CHK_FULL_REDS_HISTORY(rq);
rq->out_of_work_count = 0;
- (void) ERTS_RUNQ_FLGS_MASK_SET(rq, ERTS_RUNQ_FLGS_MIGRATION_INFO, flags);
+ (void) ERTS_RUNQ_FLGS_READ_BSET(rq, ERTS_RUNQ_FLGS_MIGRATION_INFO, flags);
rq->max_len = rq->len;
for (pix = 0; pix < ERTS_NO_PRIO_LEVELS; pix++) {
@@ -4896,6 +4653,11 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online)
erts_smp_atomic32_init_relb(&erts_halt_progress, -1);
erts_halt_code = 0;
+
+#if !defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
+ erts_lc_set_thread_name("scheduler 1");
+#endif
+
}
ErtsRunQueue *
@@ -5573,8 +5335,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all)
else if (on) { /* ------ BLOCK ------ */
if (schdlr_sspnd.msb.procs) {
plp = proclist_create(p);
- plp->next = schdlr_sspnd.msb.procs;
- schdlr_sspnd.msb.procs = plp;
+ erts_proclist_store_last(&schdlr_sspnd.msb.procs, plp);
p->flags |= F_HAVE_BLCKD_MSCHED;
ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.active) == 1);
ASSERT(p->scheduler_data->no == 1);
@@ -5659,8 +5420,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all)
~ERTS_SCHDLR_SSPND_CHNG_WAITER);
}
plp = proclist_create(p);
- plp->next = schdlr_sspnd.msb.procs;
- schdlr_sspnd.msb.procs = plp;
+ erts_proclist_store_last(&schdlr_sspnd.msb.procs, plp);
ASSERT(p->scheduler_data);
}
}
@@ -5671,20 +5431,16 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all)
}
else { /* ------ UNBLOCK ------ */
if (p->flags & F_HAVE_BLCKD_MSCHED) {
- ErtsProcList **plpp = &schdlr_sspnd.msb.procs;
- plp = schdlr_sspnd.msb.procs;
+ ErtsProcList *plp = erts_proclist_peek_first(schdlr_sspnd.msb.procs);
while (plp) {
- if (!proclist_same(plp, p)){
- plpp = &plp->next;
- plp = plp->next;
- }
- else {
- *plpp = plp->next;
- proclist_destroy(plp);
+ ErtsProcList *tmp_plp = plp;
+ plp = erts_proclist_peek_next(schdlr_sspnd.msb.procs, plp);
+ if (erts_proclist_same(tmp_plp, p)) {
+ erts_proclist_remove(&schdlr_sspnd.msb.procs, tmp_plp);
+ proclist_destroy(tmp_plp);
if (!all)
break;
- plp = *plpp;
}
}
}
@@ -5753,23 +5509,25 @@ erts_multi_scheduling_blockers(Process *p)
Eterm res = NIL;
erts_smp_mtx_lock(&schdlr_sspnd.mtx);
- if (schdlr_sspnd.msb.procs) {
+ if (!erts_proclist_is_empty(schdlr_sspnd.msb.procs)) {
Eterm *hp, *hp_end;
ErtsProcList *plp1, *plp2;
- Uint max_size;
- ASSERT(schdlr_sspnd.msb.procs);
- for (max_size = 0, plp1 = schdlr_sspnd.msb.procs;
+ Uint max_size = 0;
+
+ for (plp1 = erts_proclist_peek_first(schdlr_sspnd.msb.procs);
plp1;
- plp1 = plp1->next) {
+ plp1 = erts_proclist_peek_next(schdlr_sspnd.msb.procs, plp1)) {
max_size += 2;
}
ASSERT(max_size);
hp = HAlloc(p, max_size);
hp_end = hp + max_size;
- for (plp1 = schdlr_sspnd.msb.procs; plp1; plp1 = plp1->next) {
- for (plp2 = schdlr_sspnd.msb.procs;
+ for (plp1 = erts_proclist_peek_first(schdlr_sspnd.msb.procs);
+ plp1;
+ plp1 = erts_proclist_peek_next(schdlr_sspnd.msb.procs, plp1)) {
+ for (plp2 = erts_proclist_peek_first(schdlr_sspnd.msb.procs);
plp2->pid != plp1->pid;
- plp2 = plp2->next);
+ plp2 = erts_proclist_peek_next(schdlr_sspnd.msb.procs, plp2));
if (plp2 == plp1) {
res = CONS(hp, plp1->pid, res);
hp += 2;
@@ -6021,7 +5779,7 @@ handle_pend_sync_suspend(Process *suspendee,
ASSERT(is_nil(suspender->suspendee));
if (suspendee_alive) {
erts_suspend(suspendee, suspendee_locks, NULL);
- suspender->suspendee = suspendee->id;
+ suspender->suspendee = suspendee->common.id;
}
/* suspender is suspended waiting for suspendee to suspend;
resume suspender */
@@ -6042,7 +5800,7 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks,
ERTS_SMP_LC_ASSERT(c_p_locks & ERTS_PROC_LOCK_MAIN);
ERTS_SMP_LC_ASSERT(pid_locks & (ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS));
- if (c_p->id == pid)
+ if (c_p->common.id == pid)
return erts_pid2proc(c_p, c_p_locks, pid, pid_locks);
if (c_p_locks & ERTS_PROC_LOCK_STATUS)
@@ -6094,7 +5852,7 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks,
*/
if (!c_p->pending_suspenders) {
/* Mark rp pending for suspend by c_p */
- add_pend_suspend(rp, c_p->id, handle_pend_sync_suspend);
+ add_pend_suspend(rp, c_p->common.id, handle_pend_sync_suspend);
ASSERT(is_nil(c_p->suspendee));
/* Suspend c_p; when rp is suspended c_p will be resumed. */
@@ -6200,20 +5958,20 @@ handle_pend_bif_sync_suspend(Process *suspendee,
ASSERT(is_nil(suspender->suspendee));
if (!suspendee_alive)
erts_delete_suspend_monitor(&suspender->suspend_monitors,
- suspendee->id);
+ suspendee->common.id);
else {
#ifdef DEBUG
int res;
#endif
ErtsSuspendMonitor *smon;
smon = erts_lookup_suspend_monitor(suspender->suspend_monitors,
- suspendee->id);
+ suspendee->common.id);
#ifdef DEBUG
res =
#endif
do_bif_suspend_process(suspendee, smon, suspendee);
ASSERT(!smon || res != 0);
- suspender->suspendee = suspendee->id;
+ suspender->suspendee = suspendee->common.id;
}
/* suspender is suspended waiting for suspendee to suspend;
resume suspender */
@@ -6242,14 +6000,14 @@ handle_pend_bif_async_suspend(Process *suspendee,
ASSERT(is_nil(suspender->suspendee));
if (!suspendee_alive)
erts_delete_suspend_monitor(&suspender->suspend_monitors,
- suspendee->id);
+ suspendee->common.id);
else {
#ifdef DEBUG
int res;
#endif
ErtsSuspendMonitor *smon;
smon = erts_lookup_suspend_monitor(suspender->suspend_monitors,
- suspendee->id);
+ suspendee->common.id);
#ifdef DEBUG
res =
#endif
@@ -6294,7 +6052,7 @@ suspend_process_2(BIF_ALIST_2)
int unless_suspending = 0;
- if (BIF_P->id == BIF_ARG_1)
+ if (BIF_P->common.id == BIF_ARG_1)
goto badarg; /* We are not allowed to suspend ourselves */
if (is_not_nil(BIF_ARG_2)) {
@@ -6390,7 +6148,7 @@ suspend_process_2(BIF_ALIST_2)
if (!do_bif_suspend_process(BIF_P, smon, suspendee))
add_pend_suspend(suspendee,
- BIF_P->id,
+ BIF_P->common.id,
handle_pend_bif_async_suspend);
res = am_true;
@@ -6453,7 +6211,7 @@ suspend_process_2(BIF_ALIST_2)
else {
/* Mark suspendee pending for suspend by BIF_P */
add_pend_suspend(suspendee,
- BIF_P->id,
+ BIF_P->common.id,
handle_pend_bif_sync_suspend);
ASSERT(is_nil(BIF_P->suspendee));
@@ -6527,7 +6285,7 @@ resume_process_1(BIF_ALIST_1)
Process *suspendee;
int is_active;
- if (BIF_P->id == BIF_ARG_1)
+ if (BIF_P->common.id == BIF_ARG_1)
BIF_ERROR(BIF_P, BADARG);
erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK);
@@ -6652,7 +6410,8 @@ erts_process_status(Process *c_p, ErtsProcLocks c_p_locks,
for (i = 0; i < erts_no_schedulers; i++) {
esdp = ERTS_SCHEDULER_IX(i);
erts_smp_runq_lock(esdp->run_queue);
- if (esdp->free_process && esdp->free_process->id == rpid) {
+ if (esdp->free_process
+ && esdp->free_process->common.id == rpid) {
res = am_free;
erts_smp_runq_unlock(esdp->run_queue);
break;
@@ -6673,27 +6432,31 @@ erts_process_status(Process *c_p, ErtsProcLocks c_p_locks,
void
erts_suspend(Process* c_p, ErtsProcLocks c_p_locks, Port *busy_port)
{
-#ifdef DEBUG
- int res;
-#endif
+ int suspend;
+
ASSERT(c_p == erts_get_current_process());
ERTS_SMP_LC_ASSERT(c_p_locks == erts_proc_lc_my_proc_locks(c_p));
if (!(c_p_locks & ERTS_PROC_LOCK_STATUS))
erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_STATUS);
+ if (busy_port)
+ suspend = erts_save_suspend_process_on_port(busy_port, c_p);
+ else
+ suspend = 1;
+
+ if (suspend) {
#ifdef DEBUG
- res =
+ int res =
#endif
- suspend_process(c_p, c_p);
-
- ASSERT(res);
-
- if (busy_port)
- erts_wake_process_later(busy_port, c_p);
+ suspend_process(c_p, c_p);
+ ASSERT(res);
+ }
if (!(c_p_locks & ERTS_PROC_LOCK_STATUS))
erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS);
+ if (suspend && busy_port && erts_system_monitor_flags.busy_port)
+ monitor_generic(c_p, am_busy_port, busy_port->common.id);
}
void
@@ -6708,16 +6471,19 @@ erts_resume(Process* process, ErtsProcLocks process_locks)
}
int
-erts_resume_processes(ErtsProcList *plp)
+erts_resume_processes(ErtsProcList *list)
{
+ /* 'list' is expected to have been fetched (i.e. not a ring anymore) */
int nresumed = 0;
+ ErtsProcList *plp = list;
+
while (plp) {
Process *proc;
ErtsProcList *fplp;
ASSERT(is_internal_pid(plp->pid));
proc = erts_pid2proc(NULL, 0, plp->pid, ERTS_PROC_LOCK_STATUS);
if (proc) {
- if (proclist_same(plp, proc)) {
+ if (erts_proclist_same(plp, proc)) {
resume_process(proc);
nresumed++;
}
@@ -6870,9 +6636,8 @@ Process *schedule(Process *p, int calls)
state = erts_smp_atomic32_read_acqb(&p->state);
if (IS_TRACED(p)) {
- if (IS_TRACED_FL(p, F_TRACE_CALLS) && !(state & ERTS_PSFLG_FREE)) {
+ if (IS_TRACED_FL(p, F_TRACE_CALLS) && !(state & ERTS_PSFLG_FREE))
erts_schedule_time_break(p, ERTS_BP_CALL_TIME_SCHEDULE_OUT);
- }
if (state & (ERTS_PSFLG_FREE|ERTS_PSFLG_EXITING)) {
if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED_EXIT))
trace_sched(p, ((state & ERTS_PSFLG_FREE)
@@ -6885,7 +6650,7 @@ Process *schedule(Process *p, int calls)
else if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED_PROCS))
trace_virtual_sched(p, am_out);
}
- }
+ }
#ifdef ERTS_SMP
if (state & ERTS_PSFLG_PENDING_EXIT)
@@ -6949,9 +6714,8 @@ Process *schedule(Process *p, int calls)
#ifdef ERTS_SMP
{
ErtsProcList *pnd_xtrs = rq->procs.pending_exiters;
- rq->procs.pending_exiters = NULL;
-
- if (pnd_xtrs) {
+ if (erts_proclist_fetch(&pnd_xtrs, NULL)) {
+ rq->procs.pending_exiters = NULL;
erts_smp_runq_unlock(rq);
handle_pending_exiters(pnd_xtrs);
erts_smp_runq_lock(rq);
@@ -7247,7 +7011,7 @@ Process *schedule(Process *p, int calls)
erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
#ifdef ERTS_SMP
- if (is_not_nil(p->tracer_proc))
+ if (is_not_nil(ERTS_TRACER_PROC(p)))
erts_check_my_tracer_proc(p);
#endif
@@ -7450,70 +7214,6 @@ erts_get_exact_total_reductions(Process *c_p, Uint *redsp, Uint *diffp)
erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
}
-/*
- * erts_test_next_pid() is only used for testing.
- */
-Sint
-erts_test_next_pid(int set, Uint next)
-{
- Uint64 lpd;
- Sint res;
- Eterm pid_data;
- int first_pix = -1;
-
- erts_smp_rwmtx_rwlock(&erts_proc_tab_rwmtx);
-
- if (!set)
- lpd = last_pid_data_read_nob();
- else {
-
- lpd = (Uint64) next;
- pid_data = (Eterm) (lpd & ERTS_PID_DATA_MASK__);
- if (ERTS_INVALID_PID == make_internal_pid(pid_data)) {
- lpd += erts_proc.max;
- ASSERT(erts_pid_data2ix(pid_data)
- == erts_pid_data2ix(lpd & ERTS_PID_DATA_MASK__));
- }
- last_pid_data_set_relb(lpd);
- }
-
- while (1) {
- int pix;
- lpd++;
- pix = (int) (lpd % erts_proc.max);
- if (first_pix < 0)
- first_pix = pix;
- else if (pix == first_pix) {
- res = -1;
- break;
- }
- if (ERTS_AINT_NULL == erts_smp_atomic_read_nob(&erts_proc.tab[pix])) {
- pid_data = (Eterm) (lpd & ERTS_PID_DATA_MASK__);
- if (ERTS_INVALID_PID == make_internal_pid(pid_data)) {
- lpd += erts_proc.max;
- ASSERT(erts_pid_data2ix(pid_data)
- == erts_pid_data2ix(lpd & ERTS_PID_DATA_MASK__));
- }
- res = lpd & ERTS_PID_DATA_MASK__;
- break;
- }
- }
-
- erts_smp_rwmtx_rwunlock(&erts_proc_tab_rwmtx);
-
- return res;
-
-}
-
-Uint erts_process_count(void)
-{
- erts_aint32_t res = erts_smp_atomic32_read_nob(&process_count);
- if (res > erts_proc.max)
- return erts_proc.max;
- ASSERT(res >= 0);
- return (Uint) res;
-}
-
void
erts_free_proc(Process *p)
{
@@ -7523,132 +7223,66 @@ erts_free_proc(Process *p)
erts_free(ERTS_ALC_T_PROC, (void *) p);
}
+typedef struct {
+ Process *proc;
+ erts_aint32_t state;
+ ErtsRunQueue *run_queue;
+} ErtsEarlyProcInit;
+
+static void early_init_process_struct(void *varg, Eterm data)
+{
+ ErtsEarlyProcInit *arg = (ErtsEarlyProcInit *) varg;
+ Process *proc = arg->proc;
+
+ proc->common.id = make_internal_pid(data);
+ erts_smp_atomic32_init_relb(&proc->state, arg->state);
+
+#ifdef ERTS_SMP
+ RUNQ_SET_RQ(&proc->run_queue, arg->run_queue);
+
+ erts_proc_lock_init(proc); /* All locks locked */
+#endif
+
+}
+
/*
** Allocate process and find out where to place next process.
*/
static Process*
alloc_process(ErtsRunQueue *rq, erts_aint32_t state)
{
- int pix;
- Process* p;
- Uint64 lpd, exp_lpd;
- Eterm pid_data;
- erts_aint32_t proc_count;
-#ifdef DEBUG
- Eterm pid;
-#endif
-
- erts_smp_rwmtx_rlock(&erts_proc_tab_rwmtx);
-
- proc_count = erts_smp_atomic32_inc_read_acqb(&process_count);
- if (proc_count > erts_proc.max) {
- while (1) {
- erts_aint32_t act_proc_count;
-
- act_proc_count = erts_smp_atomic32_cmpxchg_relb(&process_count,
- proc_count-1,
- proc_count);
- if (act_proc_count == proc_count)
- goto system_limit;
- proc_count = act_proc_count;
- if (proc_count <= erts_proc.max)
- break;
- }
- }
+ ErtsEarlyProcInit init_arg;
+ Process *p;
- p = (Process*) erts_alloc_fnf(ERTS_ALC_T_PROC, sizeof(Process));
+ p = erts_alloc_fnf(ERTS_ALC_T_PROC, sizeof(Process));
if (!p)
- goto enomem;
-
- p->approx_started = erts_get_approx_time();
- p->started_interval = get_proc_interval();
-
- lpd = last_pid_data_read_acqb();
-
- /* Reserve slot */
- while (1) {
- lpd++;
- pix = erts_pid_data2ix((Eterm) (lpd & ERTS_PID_DATA_MASK__));
- if (erts_smp_atomic_read_nob(&erts_proc.tab[pix]) == ERTS_AINT_NULL) {
- erts_aint_t val;
- val = erts_smp_atomic_cmpxchg_relb(&erts_proc.tab[pix],
- ((erts_aint_t)
- ERTS_PROC_LOCK_BUSY),
- ERTS_AINT_NULL);
-
- if (ERTS_AINT_NULL == val)
- break;
- }
- }
-
- pid_data = (Eterm) lpd & ERTS_PID_DATA_MASK__;
+ return NULL;
- p->id = make_internal_pid(pid_data);
- if (p->id == ERTS_INVALID_PID) {
- /* Do not use the invalid pid; change serial */
- lpd += erts_proc.max;
- ASSERT(pix == erts_pid_data2ix((Eterm) (lpd & ERTS_PID_DATA_MASK__)));
- pid_data = (Eterm) lpd & ERTS_PID_DATA_MASK__;
- p->id = make_internal_pid(pid_data);
- ASSERT(p->id != ERTS_INVALID_PID);
- }
+ init_arg.proc = (Process *) p;
+ init_arg.run_queue = rq;
+ init_arg.state = state;
- exp_lpd = last_pid_data_read_nob();
+ ASSERT(((char *) p) == ((char *) &p->common));
- /* Move last pid data forward */
- while (1) {
- Uint64 act_lpd;
- if (last_pid_data_cmp(lpd, exp_lpd) < 0)
- break;
- act_lpd = last_pid_data_cmpxchg_relb(lpd, exp_lpd);
- if (act_lpd == exp_lpd)
- break;
- exp_lpd = act_lpd;
+ if (!erts_ptab_new_element(&erts_proc,
+ &p->common,
+ (void *) &init_arg,
+ early_init_process_struct)) {
+ erts_free(ERTS_ALC_T_PROC, p);
+ return NULL;
}
-#ifdef ERTS_SMP
- RUNQ_SET_RQ(&p->run_queue, rq);
-#endif
-
- erts_smp_atomic32_init_relb(&p->state, state);
-
-#ifdef DEBUG
- pid = p->id;
-#endif
-
-#ifdef ERTS_SMP
- erts_proc_lock_init(p); /* All locks locked */
-#endif
-
- /* Move into slot reserved */
-#ifdef DEBUG
- ASSERT(ERTS_PROC_LOCK_BUSY
- == (Process *) erts_smp_atomic_xchg_relb(&erts_proc.tab[pix],
- (erts_aint_t) p));
-#else
- erts_smp_atomic_set_relb(&erts_proc.tab[pix], (erts_aint_t) p);
-#endif
-
- ASSERT(internal_pid_serial(p->id) <= (erts_use_r9_pids_ports
- ? ERTS_MAX_PID_R9_SERIAL
- : ERTS_MAX_PID_SERIAL));
-
- erts_smp_rwmtx_runlock(&erts_proc_tab_rwmtx);
-
+ ASSERT(internal_pid_serial(p->common.id) <= ERTS_MAX_PID_SERIAL);
+
+ p->approx_started = erts_get_approx_time();
p->rcount = 0;
- ASSERT(p == (Process *)
- erts_smp_atomic_read_nob(
- &erts_proc.tab[internal_pid_index(pid)]));
-
- return p;
-enomem:
-system_limit:
-
- erts_smp_rwmtx_runlock(&erts_proc_tab_rwmtx);
- return NULL;
+ ASSERT(p == (Process *) (erts_ptab_pix2intptr_nob(
+ &erts_proc,
+ internal_pid_index(p->common.id))));
+ return p;
}
Eterm
@@ -7795,21 +7429,21 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
p->reds = 0;
#ifdef ERTS_SMP
- p->u.alive.ptimer = NULL;
+ p->common.u.alive.ptimer = NULL;
#else
- sys_memset(&p->u.alive.tm, 0, sizeof(ErlTimer));
+ sys_memset(&p->common.u.alive.tm, 0, sizeof(ErlTimer));
#endif
- p->reg = NULL;
- p->nlinks = NULL;
- p->monitors = NULL;
+ p->common.u.alive.reg = NULL;
+ ERTS_P_LINKS(p) = NULL;
+ ERTS_P_MONITORS(p) = NULL;
p->nodes_monitors = NULL;
p->suspend_monitors = NULL;
ASSERT(is_pid(parent->group_leader));
if (parent->group_leader == ERTS_INVALID_PID)
- p->group_leader = p->id;
+ p->group_leader = p->common.id;
else {
/* Needs to be done after the heap has been set up */
p->group_leader =
@@ -7818,18 +7452,18 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
: STORE_NC(&p->htop, &p->off_heap, parent->group_leader);
}
- erts_get_default_tracing(&p->trace_flags, &p->tracer_proc);
+ erts_get_default_tracing(&ERTS_TRACE_FLAGS(p), &ERTS_TRACER_PROC(p));
p->msg.first = NULL;
p->msg.last = &p->msg.first;
p->msg.save = &p->msg.first;
p->msg.len = 0;
#ifdef ERTS_SMP
- p->u.alive.msg_inq.first = NULL;
- p->u.alive.msg_inq.last = &p->u.alive.msg_inq.first;
- p->u.alive.msg_inq.len = 0;
+ p->msg_inq.first = NULL;
+ p->msg_inq.last = &p->msg_inq.first;
+ p->msg_inq.len = 0;
#endif
- p->bif_timers = NULL;
+ p->u.bif_timers = NULL;
p->mbuf = NULL;
p->mbuf_sz = 0;
p->psd = NULL;
@@ -7841,7 +7475,9 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
DT_UTAG(p) = NIL;
DT_UTAG_FLAGS(p) = 0;
#endif
- p->parent = parent->id == ERTS_INVALID_PID ? NIL : parent->id;
+ p->parent = (parent->common.id == ERTS_INVALID_PID
+ ? NIL
+ : parent->common.id);
INIT_HOLE_CHECK(p);
#ifdef DEBUG
@@ -7849,18 +7485,19 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
#endif
if (IS_TRACED(parent)) {
- if (parent->trace_flags & F_TRACE_SOS) {
- p->trace_flags |= (parent->trace_flags & TRACEE_FLAGS);
- p->tracer_proc = parent->tracer_proc;
+ if (ERTS_TRACE_FLAGS(parent) & F_TRACE_SOS) {
+ ERTS_TRACE_FLAGS(p) |= (ERTS_TRACE_FLAGS(parent) & TRACEE_FLAGS);
+ ERTS_TRACER_PROC(p) = ERTS_TRACER_PROC(parent);
}
if (ARE_TRACE_FLAGS_ON(parent, F_TRACE_PROCS)) {
- trace_proc_spawn(parent, p->id, mod, func, args);
+ trace_proc_spawn(parent, p->common.id, mod, func, args);
}
- if (parent->trace_flags & F_TRACE_SOS1) { /* Overrides TRACE_CHILDREN */
- p->trace_flags |= (parent->trace_flags & TRACEE_FLAGS);
- p->tracer_proc = parent->tracer_proc;
- p->trace_flags &= ~(F_TRACE_SOS1 | F_TRACE_SOS);
- parent->trace_flags &= ~(F_TRACE_SOS1 | F_TRACE_SOS);
+ if (ERTS_TRACE_FLAGS(parent) & F_TRACE_SOS1) {
+ /* Overrides TRACE_CHILDREN */
+ ERTS_TRACE_FLAGS(p) |= (ERTS_TRACE_FLAGS(parent) & TRACEE_FLAGS);
+ ERTS_TRACER_PROC(p) = ERTS_TRACER_PROC(parent);
+ ERTS_TRACE_FLAGS(p) &= ~(F_TRACE_SOS1 | F_TRACE_SOS);
+ ERTS_TRACE_FLAGS(parent) &= ~(F_TRACE_SOS1 | F_TRACE_SOS);
}
}
@@ -7873,27 +7510,27 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
int ret;
#endif
if (IS_TRACED_FL(parent, F_TRACE_PROCS)) {
- trace_proc(parent, parent, am_link, p->id);
+ trace_proc(parent, parent, am_link, p->common.id);
}
#ifdef DEBUG
- ret = erts_add_link(&(parent->nlinks), LINK_PID, p->id);
+ ret = erts_add_link(&ERTS_P_LINKS(parent), LINK_PID, p->common.id);
ASSERT(ret == 0);
- ret = erts_add_link(&(p->nlinks), LINK_PID, parent->id);
+ ret = erts_add_link(&ERTS_P_LINKS(p), LINK_PID, parent->common.id);
ASSERT(ret == 0);
#else
- erts_add_link(&(parent->nlinks), LINK_PID, p->id);
- erts_add_link(&(p->nlinks), LINK_PID, parent->id);
+ erts_add_link(&ERTS_P_LINKS(parent), LINK_PID, p->common.id);
+ erts_add_link(&ERTS_P_LINKS(p), LINK_PID, parent->common.id);
#endif
if (IS_TRACED(parent)) {
- if (parent->trace_flags & (F_TRACE_SOL|F_TRACE_SOL1)) {
- p->trace_flags |= (parent->trace_flags & TRACEE_FLAGS);
- p->tracer_proc = parent->tracer_proc; /* maybe steal */
+ if (ERTS_TRACE_FLAGS(parent) & (F_TRACE_SOL|F_TRACE_SOL1)) {
+ ERTS_TRACE_FLAGS(p) |= (ERTS_TRACE_FLAGS(parent)&TRACEE_FLAGS);
+ ERTS_TRACER_PROC(p) = ERTS_TRACER_PROC(parent); /*maybe steal*/
- if (parent->trace_flags & F_TRACE_SOL1) { /* maybe override */
- p ->trace_flags &= ~(F_TRACE_SOL1 | F_TRACE_SOL);
- parent->trace_flags &= ~(F_TRACE_SOL1 | F_TRACE_SOL);
+ if (ERTS_TRACE_FLAGS(parent) & F_TRACE_SOL1) {/*maybe override*/
+ ERTS_TRACE_FLAGS(p) &= ~(F_TRACE_SOL1 | F_TRACE_SOL);
+ ERTS_TRACE_FLAGS(parent) &= ~(F_TRACE_SOL1 | F_TRACE_SOL);
}
}
}
@@ -7906,8 +7543,8 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
Eterm mref;
mref = erts_make_ref(parent);
- erts_add_monitor(&(parent->monitors), MON_ORIGIN, mref, p->id, NIL);
- erts_add_monitor(&(p->monitors), MON_TARGET, mref, parent->id, NIL);
+ erts_add_monitor(&ERTS_P_MONITORS(parent), MON_ORIGIN, mref, p->common.id, NIL);
+ erts_add_monitor(&ERTS_P_MONITORS(p), MON_TARGET, mref, parent->common.id, NIL);
so->mref = mref;
}
@@ -7915,8 +7552,8 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
p->scheduler_data = NULL;
p->suspendee = NIL;
p->pending_suspenders = NULL;
- p->u.alive.pending_exit.reason = THE_NON_VALUE;
- p->u.alive.pending_exit.bp = NULL;
+ p->pending_exit.reason = THE_NON_VALUE;
+ p->pending_exit.bp = NULL;
#endif
#if !defined(NO_FPE_SIGNALS) || defined(HIPE)
@@ -7925,7 +7562,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL);
- res = p->id;
+ res = p->common.id;
/*
* Schedule process for execution.
@@ -7933,7 +7570,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
schedule_process(p, state, 0);
- VERBOSE(DEBUG_PROCESSES, ("Created a new process: %T\n",p->id));
+ VERBOSE(DEBUG_PROCESSES, ("Created a new process: %T\n",p->common.id));
#ifdef USE_VM_PROBES
if (DTRACE_ENABLED(process_spawn)) {
@@ -7968,10 +7605,10 @@ void erts_init_empty_process(Process *p)
p->min_heap_size = 0;
p->min_vheap_size = 0;
p->rcount = 0;
- p->id = ERTS_INVALID_PID;
+ p->common.id = ERTS_INVALID_PID;
p->reds = 0;
- p->tracer_proc = NIL;
- p->trace_flags = F_INITIAL_TRACE_FLAGS;
+ ERTS_TRACER_PROC(p) = NIL;
+ ERTS_TRACE_FLAGS(p) = F_INITIAL_TRACE_FLAGS;
p->group_leader = ERTS_INVALID_PID;
p->flags = 0;
p->fvalue = NIL;
@@ -7984,14 +7621,14 @@ void erts_init_empty_process(Process *p)
p->bin_old_vheap = 0;
p->bin_vheap_mature = 0;
#ifdef ERTS_SMP
- p->u.alive.ptimer = NULL;
+ p->common.u.alive.ptimer = NULL;
#else
- memset(&(p->u.alive.tm), 0, sizeof(ErlTimer));
+ memset(&(p->common.u.alive.tm), 0, sizeof(ErlTimer));
#endif
p->next = NULL;
p->off_heap.first = NULL;
p->off_heap.overhead = 0;
- p->reg = NULL;
+ p->common.u.alive.reg = NULL;
p->heap_sz = 0;
p->high_water = NULL;
p->old_hend = NULL;
@@ -8000,15 +7637,15 @@ void erts_init_empty_process(Process *p)
p->mbuf = NULL;
p->mbuf_sz = 0;
p->psd = NULL;
- p->monitors = NULL;
- p->nlinks = NULL; /* List of links */
+ ERTS_P_MONITORS(p) = NULL;
+ ERTS_P_LINKS(p) = NULL; /* List of links */
p->nodes_monitors = NULL;
p->suspend_monitors = NULL;
p->msg.first = NULL;
p->msg.last = &p->msg.first;
p->msg.save = &p->msg.first;
p->msg.len = 0;
- p->bif_timers = NULL;
+ p->u.bif_timers = NULL;
p->dictionary = NULL;
p->seq_trace_clock = 0;
p->seq_trace_lastcnt = 0;
@@ -8036,7 +7673,7 @@ void erts_init_empty_process(Process *p)
p->parent = NIL;
p->approx_started = 0;
- p->started_interval = 0;
+ p->common.u.alive.started_interval = 0;
#ifdef HIPE
hipe_init_process(&p->hipe);
@@ -8054,13 +7691,13 @@ void erts_init_empty_process(Process *p)
#ifdef ERTS_SMP
p->scheduler_data = NULL;
- p->u.alive.msg_inq.first = NULL;
- p->u.alive.msg_inq.last = &p->u.alive.msg_inq.first;
- p->u.alive.msg_inq.len = 0;
+ p->msg_inq.first = NULL;
+ p->msg_inq.last = &p->msg_inq.first;
+ p->msg_inq.len = 0;
p->suspendee = NIL;
p->pending_suspenders = NULL;
- p->u.alive.pending_exit.reason = THE_NON_VALUE;
- p->u.alive.pending_exit.bp = NULL;
+ p->pending_exit.reason = THE_NON_VALUE;
+ p->pending_exit.bp = NULL;
erts_proc_lock_init(p);
erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL);
RUNQ_SET_RQ(&p->run_queue, ERTS_RUNQ_IX(0));
@@ -8082,25 +7719,25 @@ erts_debug_verify_clean_empty_process(Process* p)
ASSERT(p->stop == NULL);
ASSERT(p->hend == NULL);
ASSERT(p->heap == NULL);
- ASSERT(p->id == ERTS_INVALID_PID);
- ASSERT(p->tracer_proc == NIL);
- ASSERT(p->trace_flags == F_INITIAL_TRACE_FLAGS);
+ ASSERT(p->common.id == ERTS_INVALID_PID);
+ ASSERT(ERTS_TRACER_PROC(p) == NIL);
+ ASSERT(ERTS_TRACE_FLAGS(p) == F_INITIAL_TRACE_FLAGS);
ASSERT(p->group_leader == ERTS_INVALID_PID);
ASSERT(p->next == NULL);
- ASSERT(p->reg == NULL);
+ ASSERT(p->common.u.alive.reg == NULL);
ASSERT(p->heap_sz == 0);
ASSERT(p->high_water == NULL);
ASSERT(p->old_hend == NULL);
ASSERT(p->old_htop == NULL);
ASSERT(p->old_heap == NULL);
- ASSERT(p->monitors == NULL);
- ASSERT(p->nlinks == NULL);
+ ASSERT(ERTS_P_MONITORS(p) == NULL);
+ ASSERT(ERTS_P_LINKS(p) == NULL);
ASSERT(p->nodes_monitors == NULL);
ASSERT(p->suspend_monitors == NULL);
ASSERT(p->msg.first == NULL);
ASSERT(p->msg.len == 0);
- ASSERT(p->bif_timers == NULL);
+ ASSERT(p->u.bif_timers == NULL);
ASSERT(p->dictionary == NULL);
ASSERT(p->catches == 0);
ASSERT(p->cp == NULL);
@@ -8110,12 +7747,12 @@ erts_debug_verify_clean_empty_process(Process* p)
ASSERT(p->parent == NIL);
#ifdef ERTS_SMP
- ASSERT(p->u.alive.msg_inq.first == NULL);
- ASSERT(p->u.alive.msg_inq.len == 0);
+ ASSERT(p->msg_inq.first == NULL);
+ ASSERT(p->msg_inq.len == 0);
ASSERT(p->suspendee == NIL);
ASSERT(p->pending_suspenders == NULL);
- ASSERT(p->u.alive.pending_exit.reason == THE_NON_VALUE);
- ASSERT(p->u.alive.pending_exit.bp == NULL);
+ ASSERT(p->pending_exit.reason == THE_NON_VALUE);
+ ASSERT(p->pending_exit.bp == NULL);
#endif
/* Thing that erts_cleanup_empty_process() cleans up */
@@ -8157,7 +7794,7 @@ delete_process(Process* p)
{
ErlMessage* mp;
- VERBOSE(DEBUG_PROCESSES, ("Removing process: %T\n",p->id));
+ VERBOSE(DEBUG_PROCESSES, ("Removing process: %T\n",p->common.id));
/* Cleanup psd */
@@ -8231,8 +7868,6 @@ delete_process(Process* p)
mp = next_mp;
}
- ASSERT(!p->monitors);
- ASSERT(!p->nlinks);
ASSERT(!p->nodes_monitors);
ASSERT(!p->suspend_monitors);
@@ -8296,7 +7931,7 @@ void
erts_handle_pending_exit(Process *c_p, ErtsProcLocks locks)
{
ErtsProcLocks xlocks;
- ASSERT(is_value(c_p->u.alive.pending_exit.reason));
+ ASSERT(is_value(c_p->pending_exit.reason));
ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == locks);
ERTS_SMP_LC_ASSERT(locks & ERTS_PROC_LOCK_MAIN);
ERTS_SMP_LC_ASSERT(!((ERTS_PSFLG_EXITING|ERTS_PSFLG_FREE)
@@ -8315,10 +7950,10 @@ erts_handle_pending_exit(Process *c_p, ErtsProcLocks locks)
set_proc_exiting(c_p,
erts_smp_atomic32_read_acqb(&c_p->state),
- c_p->u.alive.pending_exit.reason,
- c_p->u.alive.pending_exit.bp);
- c_p->u.alive.pending_exit.reason = THE_NON_VALUE;
- c_p->u.alive.pending_exit.bp = NULL;
+ c_p->pending_exit.reason,
+ c_p->pending_exit.bp);
+ c_p->pending_exit.reason = THE_NON_VALUE;
+ c_p->pending_exit.bp = NULL;
if (xlocks)
erts_smp_proc_unlock(c_p, xlocks);
@@ -8327,12 +7962,14 @@ erts_handle_pending_exit(Process *c_p, ErtsProcLocks locks)
static void
handle_pending_exiters(ErtsProcList *pnd_xtrs)
{
+ /* 'list' is expected to have been fetched (i.e. not a ring anymore) */
ErtsProcList *plp = pnd_xtrs;
- ErtsProcList *free_plp;
+
while (plp) {
+ ErtsProcList *free_plp;
Process *p = erts_pid2proc(NULL, 0, plp->pid, ERTS_PROC_LOCKS_ALL);
if (p) {
- if (proclist_same(plp, p)) {
+ if (erts_proclist_same(plp, p)) {
erts_aint32_t state = erts_smp_atomic32_read_acqb(&p->state);
if (!(state & ERTS_PSFLG_RUNNING)) {
ASSERT(state & ERTS_PSFLG_PENDING_EXIT);
@@ -8361,8 +7998,7 @@ save_pending_exiter(Process *p)
erts_smp_runq_lock(rq);
- plp->next = rq->procs.pending_exiters;
- rq->procs.pending_exiters = plp;
+ erts_proclist_store_last(&rq->procs.pending_exiters, plp);
erts_smp_runq_unlock(rq);
wake_scheduler(rq, 1);
@@ -8409,7 +8045,7 @@ send_exit_message(Process *to, ErtsProcLocks *to_locksp,
hp = bp->mem;
mess = copy_struct(exit_term, term_size, &hp, &bp->off_heap);
/* the trace token must in this case be updated by the caller */
- seq_trace_output(token, mess, SEQ_TRACE_SEND, to->id, NULL);
+ seq_trace_output(token, mess, SEQ_TRACE_SEND, to->common.id, NULL);
temp_token = copy_struct(token, sz_token, &hp, &bp->off_heap);
erts_queue_message(to, to_locksp, bp, mess, temp_token
#ifdef USE_VM_PROBES
@@ -8539,7 +8175,7 @@ send_exit_signal(Process *c_p, /* current process if and only
else if (reason != am_normal || (flags & ERTS_XSIG_FLG_NO_IGN_NORMAL)) {
#ifdef ERTS_SMP
if (!(state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT))) {
- ASSERT(!rp->u.alive.pending_exit.bp);
+ ASSERT(!rp->pending_exit.bp);
if (rp == c_p && (*rp_locks & ERTS_PROC_LOCK_MAIN)) {
/* Ensure that all locks on c_p are locked before
@@ -8589,7 +8225,7 @@ send_exit_signal(Process *c_p, /* current process if and only
set_pending_exit:
if (is_immed(rsn)) {
- rp->u.alive.pending_exit.reason = rsn;
+ rp->pending_exit.reason = rsn;
}
else {
Eterm *hp;
@@ -8597,11 +8233,11 @@ send_exit_signal(Process *c_p, /* current process if and only
ErlHeapFragment *bp = new_message_buffer(sz);
hp = &bp->mem[0];
- rp->u.alive.pending_exit.reason = copy_struct(rsn,
- sz,
- &hp,
- &bp->off_heap);
- rp->u.alive.pending_exit.bp = bp;
+ rp->pending_exit.reason = copy_struct(rsn,
+ sz,
+ &hp,
+ &bp->off_heap);
+ rp->pending_exit.bp = bp;
}
erts_smp_atomic32_read_bor_relb(&rp->state,
ERTS_PSFLG_PENDING_EXIT);
@@ -8700,7 +8336,7 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext)
if (!rp) {
goto done;
}
- rmon = erts_remove_monitor(&(rp->monitors),mon->ref);
+ rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), mon->ref);
erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
if (rmon == NULL) {
goto done;
@@ -8735,7 +8371,7 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext)
ASSERT(mon->type == MON_TARGET);
ASSERT(is_pid(mon->pid) || is_internal_port(mon->pid));
if (is_internal_port(mon->pid)) {
- Port *prt = erts_id2port(mon->pid, NULL, 0);
+ Port *prt = erts_id2port(mon->pid);
if (prt == NULL) {
goto done;
}
@@ -8751,13 +8387,13 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext)
goto done;
}
UseTmpHeapNoproc(3);
- rmon = erts_remove_monitor(&(rp->monitors),mon->ref);
+ rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), mon->ref);
if (rmon) {
erts_destroy_monitor(rmon);
watched = (is_atom(mon->name)
? TUPLE2(lhp, mon->name,
erts_this_dist_entry->sysname)
- : pcontext->p->id);
+ : pcontext->p->common.id);
erts_queue_monitor_message(rp, &rp_locks, mon->ref, am_process,
watched, pcontext->reason);
}
@@ -8822,21 +8458,22 @@ static void doit_exit_link(ErtsLink *lnk, void *vpcontext)
switch(lnk->type) {
case LINK_PID:
if(is_internal_port(item)) {
- Port *prt = erts_id2port(item, NULL, 0);
- if (prt) {
- rlnk = erts_remove_link(&prt->nlinks, p->id);
- if (rlnk)
- erts_destroy_link(rlnk);
- erts_do_exit_port(prt, p->id, reason);
- erts_port_release(prt);
- }
+ Port *prt = erts_port_lookup(item, ERTS_PORT_SFLGS_INVALID_LOOKUP);
+ if (prt)
+ erts_port_exit(NULL,
+ (ERTS_PORT_SIG_FLG_FORCE_SCHED
+ | ERTS_PORT_SIG_FLG_BROKEN_LINK),
+ prt,
+ p->common.id,
+ reason,
+ NULL);
}
else if(is_external_port(item)) {
erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
erts_dsprintf(dsbufp,
"Erroneous link between %T and external port %T "
"found\n",
- p->id,
+ p->common.id,
item);
erts_send_error_to_logger_nogl(dsbufp);
ASSERT(0); /* It isn't possible to setup such a link... */
@@ -8846,14 +8483,14 @@ static void doit_exit_link(ErtsLink *lnk, void *vpcontext)
| ERTS_PROC_LOCKS_XSIG_SEND);
rp = erts_pid2proc(NULL, 0, item, rp_locks);
if (rp) {
- rlnk = erts_remove_link(&(rp->nlinks), p->id);
+ rlnk = erts_remove_link(&ERTS_P_LINKS(rp), p->common.id);
/* If rlnk == NULL, we got unlinked while exiting,
i.e., do nothing... */
if (rlnk) {
int xres;
erts_destroy_link(rlnk);
xres = send_exit_signal(NULL,
- p->id,
+ p->common.id,
rp,
&rp_locks,
reason,
@@ -8865,7 +8502,7 @@ static void doit_exit_link(ErtsLink *lnk, void *vpcontext)
if (xres >= 0 && IS_TRACED_FL(rp, F_TRACE_PROCS)) {
/* We didn't exit the process and it is traced */
if (IS_TRACED_FL(rp, F_TRACE_PROCS)) {
- trace_proc(p, rp, am_getting_unlinked, p->id);
+ trace_proc(p, rp, am_getting_unlinked, p->common.id);
}
}
}
@@ -8879,12 +8516,12 @@ static void doit_exit_link(ErtsLink *lnk, void *vpcontext)
ErtsDSigData dsd;
int code;
ErtsDistLinkData dld;
- erts_remove_dist_link(&dld, p->id, item, dep);
+ erts_remove_dist_link(&dld, p->common.id, item, dep);
erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN);
code = erts_dsig_prepare(&dsd, dep, p, ERTS_DSP_NO_LOCK, 0);
if (code == ERTS_DSIG_PREP_CONNECTED) {
- code = erts_dsig_send_exit_tt(&dsd, p->id, item, reason,
- SEQ_TRACE_TOKEN(p));
+ code = erts_dsig_send_exit_tt(&dsd, p->common.id, item,
+ reason, SEQ_TRACE_TOKEN(p));
ASSERT(code == ERTS_DSIG_SEND_OK);
}
erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN);
@@ -8899,7 +8536,7 @@ static void doit_exit_link(ErtsLink *lnk, void *vpcontext)
/* dist entries have node links in a separate structure to
avoid confusion */
erts_smp_de_links_lock(dep);
- rlnk = erts_remove_link(&(dep->node_links), p->id);
+ rlnk = erts_remove_link(&(dep->node_links), p->common.id);
erts_smp_de_links_unlock(dep);
if (rlnk)
erts_destroy_link(rlnk);
@@ -8927,15 +8564,6 @@ resume_suspend_monitor(ErtsSuspendMonitor *smon, void *vc_p)
erts_destroy_suspend_monitor(smon);
}
-#ifdef ERTS_SMP
-static void
-proc_dec_refc(void *vproc)
-{
- erts_smp_proc_dec_refc((Process *) vproc);
-}
-#endif
-
-
/* this function fishishes a process and propagates exit messages - called
by process_main when a process dies */
void
@@ -8973,10 +8601,10 @@ erts_do_exit_process(Process* p, Eterm reason)
state = set_proc_exiting_state(p, erts_smp_atomic32_read_nob(&p->state));
if (state & ERTS_PSFLG_PENDING_EXIT) {
/* Process exited before pending exit was received... */
- p->u.alive.pending_exit.reason = THE_NON_VALUE;
- if (p->u.alive.pending_exit.bp) {
- free_message_buffer(p->u.alive.pending_exit.bp);
- p->u.alive.pending_exit.bp = NULL;
+ p->pending_exit.reason = THE_NON_VALUE;
+ if (p->pending_exit.bp) {
+ free_message_buffer(p->pending_exit.bp);
+ p->pending_exit.bp = NULL;
}
}
@@ -8993,23 +8621,25 @@ erts_do_exit_process(Process* p, Eterm reason)
trace_proc(p, p, am_exit, reason);
}
- erts_trace_check_exiting(p->id);
+ erts_trace_check_exiting(p->common.id);
- ASSERT((p->trace_flags & F_INITIAL_TRACE_FLAGS) == F_INITIAL_TRACE_FLAGS);
+ ASSERT((ERTS_TRACE_FLAGS(p) & F_INITIAL_TRACE_FLAGS)
+ == F_INITIAL_TRACE_FLAGS);
cancel_timer(p); /* Always cancel timer just in case */
- /*
- * The timer of this process can *not* be used anymore. The field used
- * for the timer is now used for misc exiting data.
- */
- p->u.exit_data = NULL;
-
- if (p->bif_timers)
+ if (p->u.bif_timers)
erts_cancel_bif_timers(p, ERTS_PROC_LOCKS_ALL);
erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL_MINOR);
+ /*
+ * The p->u.bif_timers of this process can *not* be used anymore;
+ * will be overwritten by misc termination data.
+ */
+ p->u.terminate = NULL;
+
+
erts_continue_exit_process(p);
}
@@ -9081,9 +8711,9 @@ erts_continue_exit_process(Process *p)
* The registered name *should* be the last "erlang resource" to
* cleanup.
*/
- if (p->reg) {
+ if (p->common.u.alive.reg) {
(void) erts_unregister_name(p, ERTS_PROC_LOCK_MAIN, NULL, THE_NON_VALUE);
- ASSERT(!p->reg);
+ ASSERT(!p->common.u.alive.reg);
}
erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR);
@@ -9097,22 +8727,18 @@ erts_continue_exit_process(Process *p)
yield_allowed = 0;
#endif
+ /*
+ * Note! The monitor and link fields will be overwritten
+ * by erts_ptab_delete_element() below.
+ */
+ mon = ERTS_P_MONITORS(p);
+ lnk = ERTS_P_LINKS(p);
+
{
- int maybe_save;
- int pix;
/* Do *not* use erts_get_runq_proc() */
ErtsRunQueue *rq;
rq = erts_get_runq_current(ERTS_GET_SCHEDULER_DATA_FROM_PROC(p));
- pix = internal_pid_index(p->id);
-
- erts_smp_rwmtx_rlock(&erts_proc_tab_rwmtx);
- maybe_save = saved_term_procs.end != NULL;
- if (maybe_save) {
- erts_smp_rwmtx_runlock(&erts_proc_tab_rwmtx);
- erts_smp_rwmtx_rwlock(&erts_proc_tab_rwmtx);
- }
-
erts_smp_runq_lock(rq);
#ifdef ERTS_SMP
@@ -9123,22 +8749,11 @@ erts_continue_exit_process(Process *p)
p->scheduler_data->current_process = NULL;
p->scheduler_data->free_process = p;
#endif
- /* Time of death! */
- erts_smp_atomic_set_relb(&erts_proc.tab[pix], ERTS_AINT_NULL);
- ASSERT(erts_smp_atomic32_read_nob(&process_count) > 0);
- erts_smp_atomic32_dec_relb(&process_count);
+ /* Time of death! */
+ erts_ptab_delete_element(&erts_proc, &p->common);
erts_smp_runq_unlock(rq);
-
- if (!maybe_save)
- erts_smp_rwmtx_runlock(&erts_proc_tab_rwmtx);
- else {
- if (saved_term_procs.end)
- save_terminating_process(p);
- erts_smp_rwmtx_rwunlock(&erts_proc_tab_rwmtx);
- }
-
}
/*
@@ -9148,12 +8763,6 @@ erts_continue_exit_process(Process *p)
* when the monitors and/or links hit.
*/
- mon = p->monitors;
- p->monitors = NULL; /* to avoid recursive deletion during traversal */
-
- lnk = p->nlinks;
- p->nlinks = NULL;
-
{
/* Inactivate and notify free */
erts_aint32_t n, e, a = erts_smp_atomic32_read_nob(&p->state);
@@ -9197,7 +8806,7 @@ erts_continue_exit_process(Process *p)
UseTmpHeap(4,p);
hp = &tmp_heap[0];
- exit_tuple = TUPLE3(hp, am_EXIT, p->id, reason);
+ exit_tuple = TUPLE3(hp, am_EXIT, p->common.id, reason);
exit_tuple_sz = size_object(exit_tuple);
@@ -9223,9 +8832,6 @@ erts_continue_exit_process(Process *p)
delete_process(p);
#ifdef ERTS_SMP
- erts_schedule_thr_prgr_later_op(proc_dec_refc,
- (void *) p,
- &p->u.release_data);
erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN);
ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p);
#endif
@@ -9277,9 +8883,9 @@ cancel_timer(Process* p)
ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p));
p->flags &= ~(F_INSLPQUEUE|F_TIMO);
#ifdef ERTS_SMP
- erts_cancel_smp_ptimer(p->u.alive.ptimer);
+ erts_cancel_smp_ptimer(p->common.u.alive.ptimer);
#else
- erts_cancel_timer(&p->u.alive.tm);
+ erts_cancel_timer(&p->common.u.alive.tm);
#endif
}
@@ -9300,12 +8906,12 @@ set_timer(Process* p, Uint timeout)
p->flags &= ~F_TIMO;
#ifdef ERTS_SMP
- erts_create_smp_ptimer(&p->u.alive.ptimer,
- p->id,
+ erts_create_smp_ptimer(&p->common.u.alive.ptimer,
+ p->common.id,
(ErlTimeoutProc) timeout_proc,
timeout);
#else
- erts_set_timer(&p->u.alive.tm,
+ erts_set_timer(&p->common.u.alive.tm,
(ErlTimeoutProc) timeout_proc,
NULL,
(void*) p,
@@ -9323,7 +8929,7 @@ erts_stack_dump(int to, void *to_arg, Process *p)
Eterm* sp;
int yreg = -1;
- if (p->trace_flags & F_SENSITIVE) {
+ if (ERTS_TRACE_FLAGS(p) & F_SENSITIVE) {
return;
}
erts_program_counter_info(to, to_arg, p);
@@ -9411,1068 +9017,6 @@ stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp, int yreg)
return yreg;
}
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
- * The processes/0 BIF implementation. *
-\* */
-
-
-#define ERTS_PROCESSES_BIF_TAB_INSPECT_INDICES_PER_RED 25
-#define ERTS_PROCESSES_BIF_TAB_CHUNK_SIZE 1000
-#define ERTS_PROCESSES_BIF_MIN_START_REDS \
- (ERTS_PROCESSES_BIF_TAB_CHUNK_SIZE \
- / ERTS_PROCESSES_BIF_TAB_INSPECT_INDICES_PER_RED)
-
-#define ERTS_PROCESSES_BIF_TAB_FREE_TERM_PROC_REDS 1
-
-#define ERTS_PROCESSES_BIF_INSPECT_TERM_PROC_PER_RED 10
-
-#define ERTS_PROCESSES_INSPECT_TERM_PROC_MAX_REDS \
- (ERTS_PROCESSES_BIF_TAB_CHUNK_SIZE \
- / ERTS_PROCESSES_BIF_TAB_INSPECT_INDICES_PER_RED)
-
-
-#define ERTS_PROCESSES_BIF_BUILD_RESULT_CONSES_PER_RED 75
-
-#define ERTS_PROCS_DBG_DO_TRACE 0
-
-#ifdef DEBUG
-# define ERTS_PROCESSES_BIF_DEBUGLEVEL 100
-#else
-# define ERTS_PROCESSES_BIF_DEBUGLEVEL 0
-#endif
-
-#define ERTS_PROCS_DBGLVL_CHK_HALLOC 1
-#define ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS 5
-#define ERTS_PROCS_DBGLVL_CHK_PIDS 10
-#define ERTS_PROCS_DBGLVL_CHK_TERM_PROC_LIST 20
-#define ERTS_PROCS_DBGLVL_CHK_RESLIST 20
-
-#if ERTS_PROCESSES_BIF_DEBUGLEVEL == 0
-# define ERTS_PROCS_ASSERT(EXP)
-#else
-# define ERTS_PROCS_ASSERT(EXP) \
- ((void) ((EXP) \
- ? 1 \
- : (debug_processes_assert_error(#EXP, __FILE__, __LINE__), 0)))
-#endif
-
-
-#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_HALLOC
-# define ERTS_PROCS_DBG_SAVE_HEAP_ALLOC(PBDP, HP, SZ) \
-do { \
- ERTS_PROCS_ASSERT(!(PBDP)->debug.heap); \
- ERTS_PROCS_ASSERT(!(PBDP)->debug.heap_size); \
- (PBDP)->debug.heap = (HP); \
- (PBDP)->debug.heap_size = (SZ); \
-} while (0)
-# define ERTS_PROCS_DBG_VERIFY_HEAP_ALLOC_USED(PBDP, HP) \
-do { \
- ERTS_PROCS_ASSERT((PBDP)->debug.heap); \
- ERTS_PROCS_ASSERT((PBDP)->debug.heap_size); \
- ERTS_PROCS_ASSERT((PBDP)->debug.heap + (PBDP)->debug.heap_size == (HP));\
- (PBDP)->debug.heap = NULL; \
- (PBDP)->debug.heap_size = 0; \
-} while (0)
-# define ERTS_PROCS_DBG_HEAP_ALLOC_INIT(PBDP) \
-do { \
- (PBDP)->debug.heap = NULL; \
- (PBDP)->debug.heap_size = 0; \
-} while (0)
-#else
-# define ERTS_PROCS_DBG_SAVE_HEAP_ALLOC(PBDP, HP, SZ)
-# define ERTS_PROCS_DBG_VERIFY_HEAP_ALLOC_USED(PBDP, HP)
-# define ERTS_PROCS_DBG_HEAP_ALLOC_INIT(PBDP)
-#endif
-
-#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_RESLIST
-# define ERTS_PROCS_DBG_CHK_RESLIST(R) debug_processes_check_res_list((R))
-#else
-# define ERTS_PROCS_DBG_CHK_RESLIST(R)
-#endif
-
-#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_PIDS
-# define ERTS_PROCS_DBG_SAVE_PIDS(PBDP) debug_processes_save_all_pids((PBDP))
-# define ERTS_PROCS_DBG_VERIFY_PIDS(PBDP) \
-do { \
- if (!(PBDP)->debug.correct_pids_verified) \
- debug_processes_verify_all_pids((PBDP)); \
-} while (0)
-# define ERTS_PROCS_DBG_CLEANUP_CHK_PIDS(PBDP) \
-do { \
- if ((PBDP)->debug.correct_pids) { \
- erts_free(ERTS_ALC_T_PROCS_PIDS, \
- (PBDP)->debug.correct_pids); \
- (PBDP)->debug.correct_pids = NULL; \
- } \
-} while(0)
-# define ERTS_PROCS_DBG_CHK_PIDS_INIT(PBDP) \
-do { \
- (PBDP)->debug.correct_pids_verified = 0; \
- (PBDP)->debug.correct_pids = NULL; \
-} while (0)
-#else
-# define ERTS_PROCS_DBG_SAVE_PIDS(PBDP)
-# define ERTS_PROCS_DBG_VERIFY_PIDS(PBDP)
-# define ERTS_PROCS_DBG_CLEANUP_CHK_PIDS(PBDP)
-# define ERTS_PROCS_DBG_CHK_PIDS_INIT(PBDP)
-#endif
-
-#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS
-# define ERTS_PROCS_DBG_CHK_PID_FOUND(PBDP, PID, IC) \
- debug_processes_check_found_pid((PBDP), (PID), (IC), 1)
-# define ERTS_PROCS_DBG_CHK_PID_NOT_FOUND(PBDP, PID, IC) \
- debug_processes_check_found_pid((PBDP), (PID), (IC), 0)
-#else
-# define ERTS_PROCS_DBG_CHK_PID_FOUND(PBDP, PID, IC)
-# define ERTS_PROCS_DBG_CHK_PID_NOT_FOUND(PBDP, PID, IC)
-#endif
-
-#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_TERM_PROC_LIST
-# define ERTS_PROCS_DBG_CHK_TPLIST() \
- debug_processes_check_term_proc_list()
-# define ERTS_PROCS_DBG_CHK_FREELIST(FL) \
- debug_processes_check_term_proc_free_list(FL)
-#else
-# define ERTS_PROCS_DBG_CHK_TPLIST()
-# define ERTS_PROCS_DBG_CHK_FREELIST(FL)
-#endif
-
-#if ERTS_PROCESSES_BIF_DEBUGLEVEL == 0
-#if ERTS_PROCS_DBG_DO_TRACE
-# define ERTS_PROCS_DBG_INIT(P, PBDP) (PBDP)->debug.caller = (P)->id
-# else
-# define ERTS_PROCS_DBG_INIT(P, PBDP)
-# endif
-# define ERTS_PROCS_DBG_CLEANUP(PBDP)
-#else
-# define ERTS_PROCS_DBG_INIT(P, PBDP) \
-do { \
- (PBDP)->debug.caller = (P)->id; \
- ERTS_PROCS_DBG_HEAP_ALLOC_INIT((PBDP)); \
- ERTS_PROCS_DBG_CHK_PIDS_INIT((PBDP)); \
-} while (0)
-# define ERTS_PROCS_DBG_CLEANUP(PBDP) \
-do { \
- ERTS_PROCS_DBG_CLEANUP_CHK_PIDS((PBDP)); \
-} while (0)
-#endif
-
-#if ERTS_PROCS_DBG_DO_TRACE
-# define ERTS_PROCS_DBG_TRACE(PID, FUNC, WHAT) \
- erts_fprintf(stderr, "%T %s:%d:%s(): %s\n", \
- (PID), __FILE__, __LINE__, #FUNC, #WHAT)
-#else
-# define ERTS_PROCS_DBG_TRACE(PID, FUNC, WHAT)
-#endif
-
-static Uint processes_bif_tab_chunks;
-static Export processes_trap_export;
-
-typedef struct {
- Uint64 interval;
-} ErtsProcessesBifChunkInfo;
-
-typedef enum {
- INITIALIZING,
- INSPECTING_TABLE,
- INSPECTING_TERMINATED_PROCESSES,
- BUILDING_RESULT,
- RETURN_RESULT
-} ErtsProcessesBifState;
-
-typedef struct {
- ErtsProcessesBifState state;
- Eterm caller;
- ErtsProcessesBifChunkInfo *chunk;
- int tix;
- int pid_ix;
- int pid_sz;
- Eterm *pid;
- ErtsTermProcElement *bif_invocation; /* Only used when > 1 chunk */
-
-#if ERTS_PROCESSES_BIF_DEBUGLEVEL != 0 || ERTS_PROCS_DBG_DO_TRACE
- struct {
- Eterm caller;
-#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS
- Uint64 *pid_started;
-#endif
-#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_HALLOC
- Eterm *heap;
- Uint heap_size;
-#endif
-#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_PIDS
- int correct_pids_verified;
- Eterm *correct_pids;
-#endif
- } debug;
-#endif
-
-} ErtsProcessesBifData;
-
-
-#if ERTS_PROCESSES_BIF_DEBUGLEVEL != 0
-static void debug_processes_assert_error(char* expr, char* file, int line);
-#endif
-#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_RESLIST
-static void debug_processes_check_res_list(Eterm list);
-#endif
-#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_PIDS
-static void debug_processes_save_all_pids(ErtsProcessesBifData *pbdp);
-static void debug_processes_verify_all_pids(ErtsProcessesBifData *pbdp);
-#endif
-#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS
-static void debug_processes_check_found_pid(ErtsProcessesBifData *pbdp,
- Eterm pid,
- Uint64 ic,
- int pid_should_be_found);
-#endif
-#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_TERM_PROC_LIST
-static void debug_processes_check_term_proc_list(void);
-static void debug_processes_check_term_proc_free_list(ErtsTermProcElement *tpep);
-#endif
-
-static void
-save_terminating_process(Process *p)
-{
- ErtsTermProcElement *tpep = erts_alloc(ERTS_ALC_T_PROCS_TPROC_EL,
- sizeof(ErtsTermProcElement));
- ERTS_PROCS_ASSERT(saved_term_procs.start && saved_term_procs.end);
- ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&erts_proc_tab_rwmtx));
-
- ERTS_PROCS_DBG_CHK_TPLIST();
-
- tpep->prev = saved_term_procs.end;
- tpep->next = NULL;
- tpep->ix = internal_pid_index(p->id);
- tpep->u.process.pid = p->id;
- tpep->u.process.spawned = p->started_interval;
- tpep->u.process.exited = get_proc_interval();
-
- saved_term_procs.end->next = tpep;
- saved_term_procs.end = tpep;
-
- ERTS_PROCS_DBG_CHK_TPLIST();
-
- ERTS_PROCS_ASSERT(tpep->prev->ix >= 0
- ? (tpep->u.process.exited
- >= tpep->prev->u.process.exited)
- : (tpep->u.process.exited
- >= tpep->prev->u.bif_invocation.interval));
-}
-
-static void
-cleanup_processes_bif_data(Binary *bp)
-{
- ErtsProcessesBifData *pbdp = ERTS_MAGIC_BIN_DATA(bp);
-
- ERTS_PROCS_DBG_TRACE(pbdp->debug.caller, cleanup_processes_bif_data, call);
-
- if (pbdp->state != INITIALIZING) {
-
- if (pbdp->chunk) {
- erts_free(ERTS_ALC_T_PROCS_CNKINF, pbdp->chunk);
- pbdp->chunk = NULL;
- }
- if (pbdp->pid) {
- erts_free(ERTS_ALC_T_PROCS_PIDS, pbdp->pid);
- pbdp->pid = NULL;
- }
-
-#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS
- if (pbdp->debug.pid_started) {
- erts_free(ERTS_ALC_T_PROCS_PIDS, pbdp->debug.pid_started);
- pbdp->debug.pid_started = NULL;
- }
-#endif
-
- if (pbdp->bif_invocation) {
- ErtsTermProcElement *tpep;
-
- erts_smp_rwmtx_rwlock(&erts_proc_tab_rwmtx);
-
- ERTS_PROCS_DBG_TRACE(pbdp->debug.caller,
- cleanup_processes_bif_data,
- term_proc_cleanup);
-
- tpep = pbdp->bif_invocation;
- pbdp->bif_invocation = NULL;
-
- ERTS_PROCS_DBG_CHK_TPLIST();
-
- if (tpep->prev) {
- /*
- * Only remove this bif invokation when we
- * have preceding invokations.
- */
- tpep->prev->next = tpep->next;
- if (tpep->next)
- tpep->next->prev = tpep->prev;
- else {
- /*
- * At the time of writing this branch cannot be
- * reached. I don't want to remove this code though
- * since it may be possible to reach this line
- * in the future if the cleanup order in
- * erts_do_exit_process() is changed. The ASSERT(0)
- * is only here to make us aware that the reorder
- * has happened. /rickard
- */
- ASSERT(0);
- saved_term_procs.end = tpep->prev;
- }
- erts_free(ERTS_ALC_T_PROCS_TPROC_EL, tpep);
- }
- else {
- /*
- * Free all elements until next bif invokation
- * is found.
- */
- ERTS_PROCS_ASSERT(saved_term_procs.start == tpep);
- do {
- ErtsTermProcElement *ftpep = tpep;
- tpep = tpep->next;
- erts_free(ERTS_ALC_T_PROCS_TPROC_EL, ftpep);
- } while (tpep && tpep->ix >= 0);
- saved_term_procs.start = tpep;
- if (tpep)
- tpep->prev = NULL;
- else
- saved_term_procs.end = NULL;
- }
-
- ERTS_PROCS_DBG_CHK_TPLIST();
-
- erts_smp_rwmtx_rwunlock(&erts_proc_tab_rwmtx);
-
- }
- }
-
- ERTS_PROCS_DBG_TRACE(pbdp->debug.caller,
- cleanup_processes_bif_data,
- return);
- ERTS_PROCS_DBG_CLEANUP(pbdp);
-}
-
-static int
-processes_bif_engine(Process *p, Eterm *res_accp, Binary *mbp)
-{
- ErtsProcessesBifData *pbdp = ERTS_MAGIC_BIN_DATA(mbp);
- int have_reds;
- int reds;
- int locked = 0;
-
- do {
- switch (pbdp->state) {
- case INITIALIZING:
- pbdp->chunk = erts_alloc(ERTS_ALC_T_PROCS_CNKINF,
- (sizeof(ErtsProcessesBifChunkInfo)
- * processes_bif_tab_chunks));
- pbdp->tix = 0;
- pbdp->pid_ix = 0;
-
- erts_smp_rwmtx_rwlock(&erts_proc_tab_rwmtx);
- locked = 1;
-
- ERTS_PROCS_DBG_TRACE(p->id, processes_bif_engine, init);
-
- pbdp->pid_sz = erts_process_count();
- pbdp->pid = erts_alloc(ERTS_ALC_T_PROCS_PIDS,
- sizeof(Eterm)*pbdp->pid_sz);
-
-#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS
- pbdp->debug.pid_started = erts_alloc(ERTS_ALC_T_PROCS_PIDS,
- sizeof(Uint64)*pbdp->pid_sz);
-#endif
-
- ERTS_PROCS_DBG_SAVE_PIDS(pbdp);
-
- if (processes_bif_tab_chunks == 1)
- pbdp->bif_invocation = NULL;
- else {
- /*
- * We will have to access the table multiple times
- * releasing the table lock in between chunks.
- */
- pbdp->bif_invocation = erts_alloc(ERTS_ALC_T_PROCS_TPROC_EL,
- sizeof(ErtsTermProcElement));
- pbdp->bif_invocation->ix = -1;
- pbdp->bif_invocation->u.bif_invocation.interval
- = step_proc_interval();
- ERTS_PROCS_DBG_CHK_TPLIST();
-
- pbdp->bif_invocation->next = NULL;
- if (saved_term_procs.end) {
- pbdp->bif_invocation->prev = saved_term_procs.end;
- saved_term_procs.end->next = pbdp->bif_invocation;
- ERTS_PROCS_ASSERT(saved_term_procs.start);
- }
- else {
- pbdp->bif_invocation->prev = NULL;
- saved_term_procs.start = pbdp->bif_invocation;
- }
- saved_term_procs.end = pbdp->bif_invocation;
-
- ERTS_PROCS_DBG_CHK_TPLIST();
-
- }
-
- pbdp->state = INSPECTING_TABLE;
- /* Fall through */
-
- case INSPECTING_TABLE: {
- int ix = pbdp->tix;
- int indices = ERTS_PROCESSES_BIF_TAB_CHUNK_SIZE;
- int cix = ix / ERTS_PROCESSES_BIF_TAB_CHUNK_SIZE;
- int end_ix = ix + indices;
- Uint64 *invocation_interval_p;
-
- invocation_interval_p
- = (pbdp->bif_invocation
- ? &pbdp->bif_invocation->u.bif_invocation.interval
- : NULL);
-
- ERTS_PROCS_ASSERT(is_nil(*res_accp));
- if (!locked) {
- erts_smp_rwmtx_rwlock(&erts_proc_tab_rwmtx);
- locked = 1;
- }
-
- ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&erts_proc_tab_rwmtx));
- ERTS_PROCS_DBG_TRACE(p->id, processes_bif_engine, insp_table);
-
- if (cix != 0)
- pbdp->chunk[cix].interval = step_proc_interval();
- else if (pbdp->bif_invocation)
- pbdp->chunk[0].interval = *invocation_interval_p;
- /* else: interval is irrelevant */
-
- if (end_ix >= erts_proc.max) {
- ERTS_PROCS_ASSERT(cix+1 == processes_bif_tab_chunks);
- end_ix = erts_proc.max;
- indices = end_ix - ix;
- /* What to do when done with this chunk */
- pbdp->state = (processes_bif_tab_chunks == 1
- ? BUILDING_RESULT
- : INSPECTING_TERMINATED_PROCESSES);
- }
-
- for (; ix < end_ix; ix++) {
- Process *rp = erts_pix2proc(ix);
- if (rp
- && (!invocation_interval_p
- || rp->started_interval < *invocation_interval_p)) {
- ERTS_PROCS_ASSERT(is_internal_pid(rp->id));
- pbdp->pid[pbdp->pid_ix] = rp->id;
-
-#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS
- pbdp->debug.pid_started[pbdp->pid_ix] = rp->started_interval;
-#endif
-
- pbdp->pid_ix++;
- ERTS_PROCS_ASSERT(pbdp->pid_ix <= pbdp->pid_sz);
- }
- }
-
- pbdp->tix = end_ix;
-
- erts_smp_rwmtx_rwunlock(&erts_proc_tab_rwmtx);
- locked = 0;
-
- reds = indices/ERTS_PROCESSES_BIF_TAB_INSPECT_INDICES_PER_RED;
- BUMP_REDS(p, reds);
-
- have_reds = ERTS_BIF_REDS_LEFT(p);
-
- if (have_reds && pbdp->state == INSPECTING_TABLE) {
- ix = pbdp->tix;
- indices = ERTS_PROCESSES_BIF_TAB_CHUNK_SIZE;
- end_ix = ix + indices;
- if (end_ix > erts_proc.max) {
- end_ix = erts_proc.max;
- indices = end_ix - ix;
- }
-
- reds = indices/ERTS_PROCESSES_BIF_TAB_INSPECT_INDICES_PER_RED;
-
- /* Pretend we have no reds left if we haven't got enough
- reductions to complete next chunk */
- if (reds > have_reds)
- have_reds = 0;
- }
-
- break;
- }
-
- case INSPECTING_TERMINATED_PROCESSES: {
- int i;
- int max_reds;
- int free_term_procs = 0;
- Uint64 invocation_interval;
- ErtsTermProcElement *tpep;
- ErtsTermProcElement *free_list = NULL;
-
- tpep = pbdp->bif_invocation;
- ERTS_PROCS_ASSERT(tpep);
- invocation_interval = tpep->u.bif_invocation.interval;
-
- max_reds = have_reds = ERTS_BIF_REDS_LEFT(p);
- if (max_reds > ERTS_PROCESSES_INSPECT_TERM_PROC_MAX_REDS)
- max_reds = ERTS_PROCESSES_INSPECT_TERM_PROC_MAX_REDS;
-
- reds = 0;
- erts_smp_rwmtx_rwlock(&erts_proc_tab_rwmtx);
- ERTS_PROCS_DBG_TRACE(p->id, processes_bif_engine, insp_term_procs);
-
- ERTS_PROCS_DBG_CHK_TPLIST();
-
- if (tpep->prev)
- tpep->prev->next = tpep->next;
- else {
- ERTS_PROCS_ASSERT(saved_term_procs.start == tpep);
- saved_term_procs.start = tpep->next;
-
- if (saved_term_procs.start && saved_term_procs.start->ix >= 0) {
- free_list = saved_term_procs.start;
- free_term_procs = 1;
- }
- }
-
- if (tpep->next)
- tpep->next->prev = tpep->prev;
- else
- saved_term_procs.end = tpep->prev;
-
- tpep = tpep->next;
-
- i = 0;
- while (reds < max_reds && tpep) {
- if (tpep->ix < 0) {
- if (free_term_procs) {
- ERTS_PROCS_ASSERT(free_list);
- ERTS_PROCS_ASSERT(tpep->prev);
-
- tpep->prev->next = NULL; /* end of free_list */
- saved_term_procs.start = tpep;
- tpep->prev = NULL;
- free_term_procs = 0;
- }
- }
- else {
- int cix = tpep->ix/ERTS_PROCESSES_BIF_TAB_CHUNK_SIZE;
- Uint64 chunk_interval = pbdp->chunk[cix].interval;
- Eterm pid = tpep->u.process.pid;
- ERTS_PROCS_ASSERT(is_internal_pid(pid));
-
- if (tpep->u.process.spawned < invocation_interval) {
- if (tpep->u.process.exited < chunk_interval) {
- ERTS_PROCS_DBG_CHK_PID_NOT_FOUND(pbdp,
- pid,
- tpep->u.process.spawned);
- pbdp->pid[pbdp->pid_ix] = pid;
-#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS
- pbdp->debug.pid_started[pbdp->pid_ix]
- = tpep->u.process.spawned;
-#endif
- pbdp->pid_ix++;
- ERTS_PROCS_ASSERT(pbdp->pid_ix <= pbdp->pid_sz);
- }
- else {
- ERTS_PROCS_DBG_CHK_PID_FOUND(pbdp,
- pid,
- tpep->u.process.spawned);
- }
- }
- else {
- ERTS_PROCS_DBG_CHK_PID_NOT_FOUND(pbdp,
- pid,
- tpep->u.process.spawned);
- }
-
- i++;
- if (i == ERTS_PROCESSES_BIF_INSPECT_TERM_PROC_PER_RED) {
- reds++;
- i = 0;
- }
- if (free_term_procs)
- reds += ERTS_PROCESSES_BIF_TAB_FREE_TERM_PROC_REDS;
- }
- tpep = tpep->next;
- }
-
- if (free_term_procs) {
- ERTS_PROCS_ASSERT(free_list);
- saved_term_procs.start = tpep;
- if (!tpep)
- saved_term_procs.end = NULL;
- else {
- ERTS_PROCS_ASSERT(tpep->prev);
- tpep->prev->next = NULL; /* end of free_list */
- tpep->prev = NULL;
- }
- }
-
- if (!tpep) {
- /* Done */
- ERTS_PROCS_ASSERT(pbdp->pid_ix == pbdp->pid_sz);
- pbdp->state = BUILDING_RESULT;
- pbdp->bif_invocation->next = free_list;
- free_list = pbdp->bif_invocation;
- pbdp->bif_invocation = NULL;
- }
- else {
- /* Link in bif_invocation again where we left off */
- pbdp->bif_invocation->prev = tpep->prev;
- pbdp->bif_invocation->next = tpep;
- tpep->prev = pbdp->bif_invocation;
- if (pbdp->bif_invocation->prev)
- pbdp->bif_invocation->prev->next = pbdp->bif_invocation;
- else {
- ERTS_PROCS_ASSERT(saved_term_procs.start == tpep);
- saved_term_procs.start = pbdp->bif_invocation;
- }
- }
-
- ERTS_PROCS_DBG_CHK_TPLIST();
- ERTS_PROCS_DBG_CHK_FREELIST(free_list);
- erts_smp_rwmtx_rwunlock(&erts_proc_tab_rwmtx);
-
- /*
- * We do the actual free of term proc structures now when we
- * have released the table lock instead of when we encountered
- * them. This since free() isn't for free and we don't want to
- * unnecessarily block other schedulers.
- */
- while (free_list) {
- tpep = free_list;
- free_list = tpep->next;
- erts_free(ERTS_ALC_T_PROCS_TPROC_EL, tpep);
- }
-
- have_reds -= reds;
- if (have_reds < 0)
- have_reds = 0;
- BUMP_REDS(p, reds);
- break;
- }
-
- case BUILDING_RESULT: {
- int conses, ix, min_ix;
- Eterm *hp;
- Eterm res = *res_accp;
-
- ERTS_PROCS_DBG_VERIFY_PIDS(pbdp);
- ERTS_PROCS_DBG_CHK_RESLIST(res);
-
- ERTS_PROCS_DBG_TRACE(p->id, processes_bif_engine, begin_build_res);
-
- have_reds = ERTS_BIF_REDS_LEFT(p);
- conses = ERTS_PROCESSES_BIF_BUILD_RESULT_CONSES_PER_RED*have_reds;
- min_ix = pbdp->pid_ix - conses;
- if (min_ix < 0) {
- min_ix = 0;
- conses = pbdp->pid_ix;
- }
-
- hp = HAlloc(p, conses*2);
- ERTS_PROCS_DBG_SAVE_HEAP_ALLOC(pbdp, hp, conses*2);
-
- for (ix = pbdp->pid_ix - 1; ix >= min_ix; ix--) {
- ERTS_PROCS_ASSERT(is_internal_pid(pbdp->pid[ix]));
- res = CONS(hp, pbdp->pid[ix], res);
- hp += 2;
- }
-
- ERTS_PROCS_DBG_VERIFY_HEAP_ALLOC_USED(pbdp, hp);
-
- pbdp->pid_ix = min_ix;
- if (min_ix == 0)
- pbdp->state = RETURN_RESULT;
- else {
- pbdp->pid_sz = min_ix;
- pbdp->pid = erts_realloc(ERTS_ALC_T_PROCS_PIDS,
- pbdp->pid,
- sizeof(Eterm)*pbdp->pid_sz);
-#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS
- pbdp->debug.pid_started = erts_realloc(ERTS_ALC_T_PROCS_PIDS,
- pbdp->debug.pid_started,
- (sizeof(Uint64)
- * pbdp->pid_sz));
-#endif
- }
- reds = conses/ERTS_PROCESSES_BIF_BUILD_RESULT_CONSES_PER_RED;
- BUMP_REDS(p, reds);
- have_reds -= reds;
-
- ERTS_PROCS_DBG_CHK_RESLIST(res);
- ERTS_PROCS_DBG_TRACE(p->id, processes_bif_engine, end_build_res);
- *res_accp = res;
- break;
- }
- case RETURN_RESULT:
- cleanup_processes_bif_data(mbp);
- return 1;
-
- default:
- erl_exit(ERTS_ABORT_EXIT,
- "erlang:processes/0: Invalid state: %d\n",
- (int) pbdp->state);
- }
-
-
- } while (have_reds || pbdp->state == RETURN_RESULT);
-
- return 0;
-}
-
-/*
- * processes_trap/2 is a hidden BIF that processes/0 traps to.
- */
-
-static BIF_RETTYPE processes_trap(BIF_ALIST_2)
-{
- Eterm res_acc;
- Binary *mbp;
-
- /*
- * This bif cannot be called from erlang code. It can only be
- * trapped to from processes/0; therefore, a bad argument
- * is a processes/0 internal error.
- */
-
- ERTS_PROCS_DBG_TRACE(BIF_P->id, processes_trap, call);
- ERTS_PROCS_ASSERT(is_nil(BIF_ARG_1) || is_list(BIF_ARG_1));
-
- res_acc = BIF_ARG_1;
-
- ERTS_PROCS_ASSERT(ERTS_TERM_IS_MAGIC_BINARY(BIF_ARG_2));
-
- mbp = ((ProcBin *) binary_val(BIF_ARG_2))->val;
-
- ERTS_PROCS_ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(mbp)
- == cleanup_processes_bif_data);
- ERTS_PROCS_ASSERT(
- ((ErtsProcessesBifData *) ERTS_MAGIC_BIN_DATA(mbp))->debug.caller
- == BIF_P->id);
-
- if (processes_bif_engine(BIF_P, &res_acc, mbp)) {
- ERTS_PROCS_DBG_TRACE(BIF_P->id, processes_trap, return);
- BIF_RET(res_acc);
- }
- else {
- ERTS_PROCS_DBG_TRACE(BIF_P->id, processes_trap, trap);
- ERTS_BIF_YIELD2(&processes_trap_export, BIF_P, res_acc, BIF_ARG_2);
- }
-}
-
-
-
-/*
- * The actual processes/0 BIF.
- */
-
-BIF_RETTYPE processes_0(BIF_ALIST_0)
-{
- /*
- * A requirement: The list of pids returned should be a consistent
- * snapshot of all processes existing at some point
- * in time during the execution of processes/0. Since
- * processes might terminate while processes/0 is
- * executing, we have to keep track of terminated
- * processes and add them to the result. We also
- * ignore processes created after processes/0 has
- * begun executing.
- */
- Eterm res_acc = NIL;
- Binary *mbp = erts_create_magic_binary(sizeof(ErtsProcessesBifData),
- cleanup_processes_bif_data);
- ErtsProcessesBifData *pbdp = ERTS_MAGIC_BIN_DATA(mbp);
-
- ERTS_PROCS_DBG_TRACE(BIF_P->id, processes_0, call);
- pbdp->state = INITIALIZING;
- ERTS_PROCS_DBG_INIT(BIF_P, pbdp);
-
- if (ERTS_BIF_REDS_LEFT(BIF_P) >= ERTS_PROCESSES_BIF_MIN_START_REDS
- && processes_bif_engine(BIF_P, &res_acc, mbp)) {
- erts_bin_free(mbp);
- ERTS_PROCS_DBG_CHK_RESLIST(res_acc);
- ERTS_PROCS_DBG_TRACE(BIF_P->id, processes_0, return);
- BIF_RET(res_acc);
- }
- else {
- Eterm *hp;
- Eterm magic_bin;
- ERTS_PROCS_DBG_CHK_RESLIST(res_acc);
- hp = HAlloc(BIF_P, PROC_BIN_SIZE);
- ERTS_PROCS_DBG_SAVE_HEAP_ALLOC(pbdp, hp, PROC_BIN_SIZE);
- magic_bin = erts_mk_magic_binary_term(&hp, &MSO(BIF_P), mbp);
- ERTS_PROCS_DBG_VERIFY_HEAP_ALLOC_USED(pbdp, hp);
- ERTS_PROCS_DBG_TRACE(BIF_P->id, processes_0, trap);
- ERTS_BIF_YIELD2(&processes_trap_export, BIF_P, res_acc, magic_bin);
- }
-}
-
-static void
-init_processes_bif(void)
-{
- saved_term_procs.start = NULL;
- saved_term_procs.end = NULL;
- processes_bif_tab_chunks = (((erts_proc.max - 1)
- / ERTS_PROCESSES_BIF_TAB_CHUNK_SIZE)
- + 1);
-
- /* processes_trap/2 is a hidden BIF that the processes/0 BIF traps to. */
- erts_init_trap_export(&processes_trap_export, am_erlang, am_processes_trap, 2,
- &processes_trap);
-
-}
-
-/*
- * Debug stuff
- */
-
-#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
-int
-erts_dbg_check_halloc_lock(Process *p)
-{
- if (ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p))
- return 1;
- if (p->id == ERTS_INVALID_PID)
- return 1;
- if (p->scheduler_data && p == p->scheduler_data->match_pseudo_process)
- return 1;
- if (erts_thr_progress_is_blocking())
- return 1;
- return 0;
-}
-#endif
-
-Eterm
-erts_debug_processes(Process *c_p)
-{
- /* This is the old processes/0 BIF. */
- int i;
- Uint need;
- Eterm res;
- Eterm* hp;
- Process *p;
- Eterm *hp_end;
-
- erts_smp_rwmtx_rwlock(&erts_proc_tab_rwmtx);
-
- res = NIL;
- need = erts_process_count() * 2;
- hp = HAlloc(c_p, need); /* we need two heap words for each pid */
- hp_end = hp + need;
-
- /* make the list by scanning bakward */
-
-
- for (i = erts_proc.max-1; i >= 0; i--) {
- p = erts_pix2proc(i);
- if (p) {
- res = CONS(hp, p->id, res);
- hp += 2;
- }
- }
-
- erts_smp_rwmtx_rwunlock(&erts_proc_tab_rwmtx);
-
- HRelease(c_p, hp_end, hp);
-
- return res;
-}
-
-Eterm
-erts_debug_processes_bif_info(Process *c_p)
-{
- ERTS_DECL_AM(processes_bif_info);
- Eterm elements[] = {
- AM_processes_bif_info,
- make_small((Uint) ERTS_PROCESSES_BIF_MIN_START_REDS),
- make_small((Uint) processes_bif_tab_chunks),
- make_small((Uint) ERTS_PROCESSES_BIF_TAB_CHUNK_SIZE),
- make_small((Uint) ERTS_PROCESSES_BIF_TAB_INSPECT_INDICES_PER_RED),
- make_small((Uint) ERTS_PROCESSES_BIF_TAB_FREE_TERM_PROC_REDS),
- make_small((Uint) ERTS_PROCESSES_BIF_INSPECT_TERM_PROC_PER_RED),
- make_small((Uint) ERTS_PROCESSES_INSPECT_TERM_PROC_MAX_REDS),
- make_small((Uint) ERTS_PROCESSES_BIF_BUILD_RESULT_CONSES_PER_RED),
- make_small((Uint) ERTS_PROCESSES_BIF_DEBUGLEVEL)
- };
- Uint sz = 0;
- Eterm *hp;
- (void) erts_bld_tuplev(NULL, &sz, sizeof(elements)/sizeof(Eterm), elements);
- hp = HAlloc(c_p, sz);
- return erts_bld_tuplev(&hp, NULL, sizeof(elements)/sizeof(Eterm), elements);
-}
-
-#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS
-static void
-debug_processes_check_found_pid(ErtsProcessesBifData *pbdp,
- Eterm pid,
- Uint64 ic,
- int pid_should_be_found)
-{
- int i;
- for (i = 0; i < pbdp->pid_ix; i++) {
- if (pbdp->pid[i] == pid && pbdp->debug.pid_started[i] == ic) {
- ERTS_PROCS_ASSERT(pid_should_be_found);
- return;
- }
- }
- ERTS_PROCS_ASSERT(!pid_should_be_found);
-}
-#endif
-
-#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_RESLIST
-static void
-debug_processes_check_res_list(Eterm list)
-{
- while (is_list(list)) {
- Eterm* consp = list_val(list);
- Eterm hd = CAR(consp);
- ERTS_PROCS_ASSERT(is_internal_pid(hd));
- list = CDR(consp);
- }
-
- ERTS_PROCS_ASSERT(is_nil(list));
-}
-#endif
-
-#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_PIDS
-
-static void
-debug_processes_save_all_pids(ErtsProcessesBifData *pbdp)
-{
- int ix, tix, cpix;
- pbdp->debug.correct_pids_verified = 0;
- pbdp->debug.correct_pids = erts_alloc(ERTS_ALC_T_PROCS_PIDS,
- sizeof(Eterm)*pbdp->pid_sz);
-
- for (tix = 0, cpix = 0; tix < erts_proc.max; tix++) {
- Process *rp = erts_pix2proc(tix);
- if (rp) {
- ERTS_PROCS_ASSERT(is_internal_pid(rp->id));
- pbdp->debug.correct_pids[cpix++] = rp->id;
- ERTS_PROCS_ASSERT(cpix <= pbdp->pid_sz);
- }
- }
- ERTS_PROCS_ASSERT(cpix == pbdp->pid_sz);
-
- for (ix = 0; ix < pbdp->pid_sz; ix++)
- pbdp->pid[ix] = make_small(ix);
-}
-
-static void
-debug_processes_verify_all_pids(ErtsProcessesBifData *pbdp)
-{
- int ix, cpix;
-
- ERTS_PROCS_ASSERT(pbdp->pid_ix == pbdp->pid_sz);
-
- for (ix = 0; ix < pbdp->pid_sz; ix++) {
- int found = 0;
- Eterm pid = pbdp->pid[ix];
- ERTS_PROCS_ASSERT(is_internal_pid(pid));
- for (cpix = ix; cpix < pbdp->pid_sz; cpix++) {
- if (pbdp->debug.correct_pids[cpix] == pid) {
- pbdp->debug.correct_pids[cpix] = NIL;
- found = 1;
- break;
- }
- }
- if (!found) {
- for (cpix = 0; cpix < ix; cpix++) {
- if (pbdp->debug.correct_pids[cpix] == pid) {
- pbdp->debug.correct_pids[cpix] = NIL;
- found = 1;
- break;
- }
- }
- }
- ERTS_PROCS_ASSERT(found);
- }
- pbdp->debug.correct_pids_verified = 1;
-
- erts_free(ERTS_ALC_T_PROCS_PIDS, pbdp->debug.correct_pids);
- pbdp->debug.correct_pids = NULL;
-}
-#endif /* ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_PIDS */
-
-#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_TERM_PROC_LIST
-static void
-debug_processes_check_term_proc_list(void)
-{
- ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&erts_proc_tab_rwmtx));
- if (!saved_term_procs.start)
- ERTS_PROCS_ASSERT(!saved_term_procs.end);
- else {
- Uint64 curr_interval = get_proc_interval();
- Uint64 *prev_x_interval_p = NULL;
- ErtsTermProcElement *tpep;
-
- for (tpep = saved_term_procs.start; tpep; tpep = tpep->next) {
- if (!tpep->prev)
- ERTS_PROCS_ASSERT(saved_term_procs.start == tpep);
- else
- ERTS_PROCS_ASSERT(tpep->prev->next == tpep);
- if (!tpep->next)
- ERTS_PROCS_ASSERT(saved_term_procs.end == tpep);
- else
- ERTS_PROCS_ASSERT(tpep->next->prev == tpep);
- if (tpep->ix < 0) {
- Uint64 interval = tpep->u.bif_invocation.interval;
- ERTS_PROCS_ASSERT(interval <= curr_interval);
- }
- else {
- Uint64 s_interval = tpep->u.process.spawned;
- Uint64 x_interval = tpep->u.process.exited;
-
- ERTS_PROCS_ASSERT(s_interval <= x_interval);
- if (prev_x_interval_p)
- ERTS_PROCS_ASSERT(*prev_x_interval_p <= x_interval);
- prev_x_interval_p = &tpep->u.process.exited;
- ERTS_PROCS_ASSERT(is_internal_pid(tpep->u.process.pid));
- ERTS_PROCS_ASSERT(tpep->ix
- == internal_pid_index(tpep->u.process.pid));
- }
- }
-
- }
-}
-
-static void
-debug_processes_check_term_proc_free_list(ErtsTermProcElement *free_list)
-{
- if (saved_term_procs.start) {
- ErtsTermProcElement *ftpep;
- ErtsTermProcElement *tpep;
-
- for (ftpep = free_list; ftpep; ftpep = ftpep->next) {
- for (tpep = saved_term_procs.start; tpep; tpep = tpep->next)
- ERTS_PROCS_ASSERT(ftpep != tpep);
- }
- }
-}
-
-#endif
-
-#if ERTS_PROCESSES_BIF_DEBUGLEVEL != 0
-
-static void
-debug_processes_assert_error(char* expr, char* file, int line)
-{
- fflush(stdout);
- erts_fprintf(stderr, "%s:%d: Assertion failed: %s\n", file, line, expr);
- fflush(stderr);
- abort();
-}
-
-#endif
-
-/* *\
- * End of the processes/0 BIF implementation. *
-\* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-
/*
* A nice system halt closing all open port goes as follows:
* 1) This function schedules the aux work ERTS_SSI_AUX_WORK_REAP_PORTS
@@ -10499,3 +9043,19 @@ void erl_halt(int code)
notify_reap_ports_relb();
}
}
+
+#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
+int
+erts_dbg_check_halloc_lock(Process *p)
+{
+ if (ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p))
+ return 1;
+ if (p->common.id == ERTS_INVALID_PID)
+ return 1;
+ if (p->scheduler_data && p == p->scheduler_data->match_pseudo_process)
+ return 1;
+ if (erts_thr_progress_is_blocking())
+ return 1;
+ return 0;
+}
+#endif