diff options
| author | Rickard Green <[email protected]> | 2012-12-07 00:28:50 +0100 |
|---|---|---|
| committer | Rickard Green <[email protected]> | 2012-12-07 00:28:50 +0100 |
| commit | 6027f852217f0014f1892fbbfa45c69e104da652 (patch) | |
| tree | bdb10abcf579b5607dc287b395fb15fa667b0512 /erts/emulator/beam/erl_process.c | |
| parent | d29c460c4ad1ca0cc2fb6a13a81b4ccc07516941 (diff) | |
| parent | 3f85767e086e08b70baee34d719df9a8bc8814f4 (diff) | |
| download | otp-6027f852217f0014f1892fbbfa45c69e104da652.tar.gz otp-6027f852217f0014f1892fbbfa45c69e104da652.tar.bz2 otp-6027f852217f0014f1892fbbfa45c69e104da652.zip | |
Merge branch 'rickard/port-optimizations/OTP-10336' into rickard/r16/port-optimizations/OTP-10336
* rickard/port-optimizations/OTP-10336:
Change annotate level for emacs-22 in cerl
Update etp-commands
Add documentation on communication in Erlang
Add support for busy port message queue
Add driver callback epilogue
Implement true asynchronous signaling between processes and ports
Add erl_drv_[send|output]_term
Move busy port flag
Use rwlock for driver list
Optimize management of port tasks
Improve configuration of process and port tables
Remove R9 compatibility features
Use ptab functionality also for ports
Prepare for use of ptab functionality also for ports
Atomic port state
Generalize process table implementation
Implement functionality for delaying thread progress from unmanaged threads
Conflicts:
erts/doc/src/erl_driver.xml
erts/doc/src/erlang.xml
erts/emulator/beam/beam_bif_load.c
erts/emulator/beam/beam_bp.c
erts/emulator/beam/beam_emu.c
erts/emulator/beam/bif.c
erts/emulator/beam/copy.c
erts/emulator/beam/erl_alloc.c
erts/emulator/beam/erl_alloc.types
erts/emulator/beam/erl_bif_info.c
erts/emulator/beam/erl_bif_port.c
erts/emulator/beam/erl_bif_trace.c
erts/emulator/beam/erl_init.c
erts/emulator/beam/erl_message.c
erts/emulator/beam/erl_port_task.c
erts/emulator/beam/erl_process.c
erts/emulator/beam/erl_process.h
erts/emulator/beam/erl_process_lock.c
erts/emulator/beam/erl_trace.c
erts/emulator/beam/export.h
erts/emulator/beam/global.h
erts/emulator/beam/io.c
erts/emulator/sys/unix/sys.c
erts/emulator/sys/vxworks/sys.c
erts/emulator/test/port_SUITE.erl
erts/etc/unix/cerl.src
erts/preloaded/ebin/erlang.beam
erts/preloaded/ebin/prim_inet.beam
erts/preloaded/src/prim_inet.erl
lib/hipe/cerl/erl_bif_types.erl
lib/kernel/doc/src/inet.xml
lib/kernel/src/inet.erl
Diffstat (limited to 'erts/emulator/beam/erl_process.c')
| -rw-r--r-- | erts/emulator/beam/erl_process.c | 2076 |
1 files changed, 318 insertions, 1758 deletions
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 61554780c4..aaca4b5f59 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -42,6 +42,7 @@ #include "erl_thr_queue.h" #include "erl_async.h" #include "dtrace-wrapper.h" +#include "erl_ptab.h" #define ERTS_DELAYED_WAKEUP_INFINITY (~(Uint64) 0) #define ERTS_DELAYED_WAKEUP_REDUCTIONS ((Uint64) CONTEXT_REDS/2) @@ -136,155 +137,16 @@ do { \ #define ERTS_EMPTY_RUNQ_PORTS(RQ) \ (RUNQ_READ_LEN(&(RQ)->ports.info.len) == 0 && (RQ)->misc.start == NULL) +const Process erts_invalid_process = {{ERTS_INVALID_PID}}; + extern BeamInstr beam_apply[]; extern BeamInstr beam_exit[]; extern BeamInstr beam_continue_exit[]; -#ifdef ARCH_32 - -union { - erts_smp_dw_atomic_t pid_data; - char align[ERTS_CACHE_LINE_SIZE]; -} last erts_align_attribute(ERTS_CACHE_LINE_SIZE); - - -static ERTS_INLINE Uint64 -dw_aint_to_uint64(erts_dw_aint_t *dw) -{ -#ifdef ETHR_SU_DW_NAINT_T__ - return (Uint64) dw->dw_sint; -#else - Uint64 res; - res = (Uint64) ((Uint32) dw->sint[ERTS_DW_AINT_HIGH_WORD]); - res <<= 32; - res |= (Uint64) ((Uint32) dw->sint[ERTS_DW_AINT_LOW_WORD]); - return res; -#endif -} - -static void -unint64_to_dw_aint(erts_dw_aint_t *dw, Uint64 val) -{ -#ifdef ETHR_SU_DW_NAINT_T__ - dw->dw_sint = (ETHR_SU_DW_NAINT_T__) val; -#else - dw->sint[ERTS_DW_AINT_LOW_WORD] = (erts_aint_t) (val & 0xffffffff); - dw->sint[ERTS_DW_AINT_HIGH_WORD] = (erts_aint_t) ((val >> 32) & 0xffffffff); -#endif -} - -static ERTS_INLINE void -last_pid_data_init_nob(Uint64 val) -{ - erts_dw_aint_t dw; - unint64_to_dw_aint(&dw, val); - erts_smp_dw_atomic_init_nob(&last.pid_data, &dw); -} - -static ERTS_INLINE void -last_pid_data_set_relb(Uint64 val) -{ - erts_dw_aint_t dw; - unint64_to_dw_aint(&dw, val); - erts_smp_dw_atomic_set_relb(&last.pid_data, &dw); -} - -static ERTS_INLINE Uint64 -last_pid_data_read_nob(void) -{ - erts_dw_aint_t dw; - erts_smp_dw_atomic_read_nob(&last.pid_data, &dw); - return dw_aint_to_uint64(&dw); -} - -static ERTS_INLINE Uint64 -last_pid_data_read_acqb(void) -{ - erts_dw_aint_t dw; - erts_smp_dw_atomic_read_acqb(&last.pid_data, &dw); - return dw_aint_to_uint64(&dw); -} - -static ERTS_INLINE Uint64 -last_pid_data_cmpxchg_relb(Uint64 new, Uint64 exp) -{ - erts_dw_aint_t dw_new, dw_xchg; - - unint64_to_dw_aint(&dw_new, new); - unint64_to_dw_aint(&dw_xchg, exp); - - if (erts_smp_dw_atomic_cmpxchg_relb(&last.pid_data, &dw_new, &dw_xchg)) - return exp; - else - return dw_aint_to_uint64(&dw_xchg); -} - -#elif defined(ARCH_64) - -union { - erts_smp_atomic_t pid_data; - char align[ERTS_CACHE_LINE_SIZE]; -} last erts_align_attribute(ERTS_CACHE_LINE_SIZE); - -static ERTS_INLINE void -last_pid_data_init_nob(Uint64 val) -{ - erts_smp_atomic_init_nob(&last.pid_data, (erts_aint_t) val); -} - -static ERTS_INLINE void -last_pid_data_set_relb(Uint64 val) -{ - erts_smp_atomic_set_relb(&last.pid_data, (erts_aint_t) val); -} - -static ERTS_INLINE Uint64 -last_pid_data_read_nob(void) -{ - return (Uint64) erts_smp_atomic_read_nob(&last.pid_data); -} - -static ERTS_INLINE Uint64 -last_pid_data_read_acqb(void) -{ - return (Uint64) erts_smp_atomic_read_acqb(&last.pid_data); -} - -static ERTS_INLINE Uint64 -last_pid_data_cmpxchg_relb(Uint64 new, Uint64 exp) -{ - return (Uint64) erts_smp_atomic_cmpxchg_relb(&last.pid_data, - (erts_aint_t) new, - (erts_aint_t) exp); -} - -#else -# error "Not 64-bit, nor 32-bit architecture..." -#endif - -static ERTS_INLINE int -last_pid_data_cmp(Uint64 lpd1, Uint64 lpd2) -{ - Uint64 lpd1_wrap; - - if (lpd1 == lpd2) - return 0; - - lpd1_wrap = lpd1 + (((Uint64) 1) << 63); - - if (lpd1 < lpd1_wrap) - return (lpd1 < lpd2 && lpd2 < lpd1_wrap) ? -1 : 1; - else - return (lpd1_wrap <= lpd2 && lpd2 < lpd1) ? 1 : -1; -} - - -#define ERTS_PID_DATA_MASK__ ((1 << _PID_DATA_SIZE) - 1) - int erts_sched_compact_load; Uint erts_no_schedulers; -ErtsProcTab erts_proc erts_align_attribute(ERTS_CACHE_LINE_SIZE); +ErtsPTab erts_proc erts_align_attribute(ERTS_CACHE_LINE_SIZE); int erts_sched_thread_suggested_stack_size = -1; @@ -371,8 +233,6 @@ erts_sched_stat_t erts_sched_stat; static erts_tsd_key_t sched_data_key; #endif -erts_smp_rwmtx_t erts_proc_tab_rwmtx; - static erts_smp_atomic32_t function_calls; #ifdef ERTS_SMP @@ -409,29 +269,6 @@ struct erts_system_profile_flags_t erts_system_profile_flags; #if ERTS_MAX_PROCESSES > 0x7fffffff #error "Need to store process_count in another type" #endif -static erts_smp_atomic32_t process_count; - -typedef struct ErtsTermProcElement_ ErtsTermProcElement; -struct ErtsTermProcElement_ { - ErtsTermProcElement *next; - ErtsTermProcElement *prev; - int ix; - union { - struct { - Eterm pid; - Uint64 spawned; - Uint64 exited; - } process; - struct { - Uint64 interval; - } bif_invocation; - } u; -}; - -static struct { - ErtsTermProcElement *start; - ErtsTermProcElement *end; -} saved_term_procs; ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(misc_op_list, ErtsMiscOpList, @@ -493,8 +330,6 @@ do { \ * Local functions. */ -static void init_processes_bif(void); -static void save_terminating_process(Process *p); static void exec_misc_ops(ErtsRunQueue *); static void print_function_from_pc(int to, void *to_arg, BeamInstr* x); static int stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp, @@ -562,39 +397,17 @@ erts_smp_lc_runq_is_locked(ErtsRunQueue *runq) } #endif -static erts_interval_t *proc_interval; - -static void -proc_interval_init(void) -{ - proc_interval = erts_alloc_permanent_cache_aligned( - ERTS_ALC_T_PROC_INTERVAL, - sizeof(erts_interval_t)); - erts_smp_interval_init(proc_interval); -} - -static ERTS_INLINE Uint64 -get_proc_interval(void) -{ - return erts_smp_current_interval_nob(proc_interval); -} static ERTS_INLINE Uint64 ensure_later_proc_interval(Uint64 interval) { - return erts_smp_ensure_later_interval_nob(proc_interval, interval); -} - -static ERTS_INLINE Uint64 -step_proc_interval(void) -{ - return erts_smp_step_interval_nob(proc_interval); + return erts_smp_ensure_later_interval_nob(erts_ptab_interval(&erts_proc), interval); } Uint64 erts_get_proc_interval(void) { - return get_proc_interval(); + return erts_smp_current_interval_nob(erts_ptab_interval(&erts_proc)); } Uint64 @@ -606,7 +419,7 @@ erts_ensure_later_proc_interval(Uint64 interval) Uint64 erts_step_proc_interval(void) { - return step_proc_interval(); + return erts_smp_step_interval_nob(erts_ptab_interval(&erts_proc)); } void @@ -654,20 +467,18 @@ erts_pre_init_process(void) #endif } +#ifdef ERTS_SMP +static void +release_process(void *vproc) +{ + erts_smp_proc_dec_refc((Process *) vproc); +} +#endif + /* initialize the scheduler */ void -erts_init_process(int ncpu) +erts_init_process(int ncpu, int proc_tab_size) { - int proc_tab_sz; - int max_proc_bits; - int proc_bits = ERTS_PROC_BITS; - erts_smp_atomic_t *proc_entry; - char *proc_tab_end; - erts_smp_rwmtx_opt_t proc_tab_rwmtx_opts = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; - proc_tab_rwmtx_opts.type = ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ; - proc_tab_rwmtx_opts.lived = ERTS_SMP_RWMTX_LONG_LIVED; - - proc_interval_init(); #ifdef ERTS_SMP erts_disable_proc_not_running_opt = 0; @@ -676,52 +487,16 @@ erts_init_process(int ncpu) init_proclist_alloc(); - erts_smp_atomic32_init_nob(&process_count, 0); - - if (erts_use_r9_pids_ports) - proc_bits = ERTS_R9_PROC_BITS; - - if (erts_proc.max > (1 << proc_bits)) - erts_proc.max = 1 << proc_bits; - - proc_tab_sz = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(erts_proc.max - * sizeof(erts_smp_atomic_t)); - erts_proc.tab = erts_alloc(ERTS_ALC_T_PROC_TABLE, proc_tab_sz); - proc_tab_end = ((char *) erts_proc.tab) + proc_tab_sz; - proc_entry = erts_proc.tab; - while (proc_tab_end > ((char *) proc_entry)) { - erts_smp_atomic_init_nob(proc_entry, ERTS_AINT_NULL); - proc_entry++; - } - - erts_smp_rwmtx_init_opt(&erts_proc_tab_rwmtx, - &proc_tab_rwmtx_opts, - "proc_tab"); - last_pid_data_init_nob(~((Uint64) 0)); - - max_proc_bits = erts_fit_in_bits_int32((Sint32) erts_proc.max - 1); - - erts_proc.tab_cache_lines = proc_tab_sz/ERTS_CACHE_LINE_SIZE; - erts_proc.pix_per_cache_line = ERTS_CACHE_LINE_SIZE/sizeof(erts_smp_atomic_t); - if ((erts_proc.max & (erts_proc.max - 1)) - | (erts_proc.pix_per_cache_line & (erts_proc.pix_per_cache_line - 1))) { - /* - * erts_proc.max or erts_proc.pix_per_cache_line - * not a power of 2 :( - */ - erts_proc.pix_cl_mask = 0; - erts_proc.pix_cl_shift = 0; - erts_proc.pix_cli_mask = 0; - erts_proc.pix_cli_shift = 0; - } - else { - ASSERT((erts_proc.tab_cache_lines - & (erts_proc.tab_cache_lines - 1)) == 0); - erts_proc.pix_cl_mask = erts_proc.tab_cache_lines-1; - erts_proc.pix_cl_shift = erts_fit_in_bits_int32(erts_proc.pix_per_cache_line-1); - erts_proc.pix_cli_shift = erts_fit_in_bits_int32(erts_proc.pix_cl_mask); - erts_proc.pix_cli_mask = (1 << (max_proc_bits - erts_proc.pix_cli_shift)) - 1; - } + erts_ptab_init_table(&erts_proc, + ERTS_ALC_T_PROC_TABLE, +#ifdef ERTS_SMP + release_process, +#else + NULL, +#endif + (ErtsPTabElementCommon *) &erts_invalid_process.common, + proc_tab_size, + "process_table"); last_reductions = 0; last_exact_reductions = 0; @@ -732,7 +507,6 @@ void erts_late_init_process(void) { int ix; - init_processes_bif(); erts_smp_spinlock_init(&erts_sched_stat.lock, "sched_stat"); for (ix = 0; ix < ERTS_NO_PRIO_LEVELS; ix++) { @@ -980,9 +754,9 @@ static ERTS_INLINE ErtsProcList * proclist_create(Process *p) { ErtsProcList *plp = proclist_alloc(); - ensure_later_proc_interval(p->started_interval); - plp->pid = p->id; - plp->started_interval = p->started_interval; + ensure_later_proc_interval(p->common.u.alive.started_interval); + plp->pid = p->common.id; + plp->started_interval = p->common.u.alive.started_interval; return plp; } @@ -992,12 +766,6 @@ proclist_destroy(ErtsProcList *plp) proclist_free(plp); } -static ERTS_INLINE int -proclist_same(ErtsProcList *plp, Process *p) -{ - return plp->pid == p->id && plp->started_interval == p->started_interval; -} - ErtsProcList * erts_proclist_create(Process *p) { @@ -1010,12 +778,6 @@ erts_proclist_destroy(ErtsProcList *plp) proclist_destroy(plp); } -int -erts_proclist_same(ErtsProcList *plp, Process *p) -{ - return proclist_same(plp, p); -} - void * erts_psd_set_init(Process *p, ErtsProcLocks plocks, int ix, void *data) { @@ -1776,37 +1538,32 @@ handle_reap_ports(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting) unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_REAP_PORTS); awdp->esdp->run_queue->halt_in_progress = 1; if (erts_smp_atomic32_dec_read_acqb(&erts_halt_progress) == 0) { - int i; + int i, max = erts_ptab_max(&erts_port); erts_smp_atomic32_set_nob(&erts_halt_progress, 1); - for (i = 0; i < erts_max_ports; i++) { - Port *prt = &erts_port[i]; - erts_smp_port_state_lock(prt); - if ((prt->status & (ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP - | ERTS_PORT_SFLG_HALT))) { - erts_smp_port_state_unlock(prt); - continue; - } - /* We need to set the halt flag - get the port lock */ -#ifdef ERTS_SMP - erts_smp_atomic_inc_nob(&prt->refc); -#endif - erts_smp_port_state_unlock(prt); -#ifdef ERTS_SMP - erts_smp_mtx_lock(prt->lock); -#endif - if ((prt->status & (ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP - | ERTS_PORT_SFLG_HALT))) { - erts_port_release(prt); + for (i = 0; i < max; i++) { + erts_aint32_t state; + Port *prt = erts_pix2port(i); + if (!prt) continue; - } - erts_port_status_bor_set(prt, ERTS_PORT_SFLG_HALT); - erts_smp_atomic32_inc_nob(&erts_halt_progress); - if (prt->status & (ERTS_PORT_SFLG_EXITING - | ERTS_PORT_SFLG_CLOSING)) { - erts_port_release(prt); + state = erts_atomic32_read_acqb(&prt->state); + if (state & (ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP + | ERTS_PORT_SFLG_HALT)) continue; + + /* We need to set the halt flag - get the port lock */ + + erts_smp_port_lock(prt); + + state = erts_atomic32_read_nob(&prt->state); + if (!(state & (ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP + | ERTS_PORT_SFLG_HALT))) { + state = erts_atomic32_read_bor_relb(&prt->state, + ERTS_PORT_SFLG_HALT); + erts_smp_atomic32_inc_nob(&erts_halt_progress); + if (!(state & (ERTS_PORT_SFLG_EXITING|ERTS_PORT_SFLG_CLOSING))) + erts_deliver_port_exit(prt, prt->common.id, am_killed, 0); } - erts_do_exit_port(prt, prt->id, am_killed); + erts_port_release(prt); } if (erts_smp_atomic32_dec_read_nob(&erts_halt_progress) == 0) { @@ -3183,12 +2940,12 @@ resume_run_queue(ErtsRunQueue *rq) erts_smp_runq_lock(rq); - (void) ERTS_RUNQ_FLGS_MASK_SET(rq, - (ERTS_RUNQ_FLG_OUT_OF_WORK - | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK - | ERTS_RUNQ_FLG_SUSPENDED), - (ERTS_RUNQ_FLG_OUT_OF_WORK - | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK)); + (void) ERTS_RUNQ_FLGS_READ_BSET(rq, + (ERTS_RUNQ_FLG_OUT_OF_WORK + | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK + | ERTS_RUNQ_FLG_SUSPENDED), + (ERTS_RUNQ_FLG_OUT_OF_WORK + | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK)); rq->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS; for (pix = 0; pix < ERTS_NO_PROC_PRIO_LEVELS; pix++) { @@ -4210,7 +3967,7 @@ erts_fprintf(stderr, "--------------------------------\n"); ERTS_DBG_CHK_FULL_REDS_HISTORY(rq); rq->out_of_work_count = 0; - (void) ERTS_RUNQ_FLGS_MASK_SET(rq, ERTS_RUNQ_FLGS_MIGRATION_INFO, flags); + (void) ERTS_RUNQ_FLGS_READ_BSET(rq, ERTS_RUNQ_FLGS_MIGRATION_INFO, flags); rq->max_len = rq->len; for (pix = 0; pix < ERTS_NO_PRIO_LEVELS; pix++) { @@ -4896,6 +4653,11 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online) erts_smp_atomic32_init_relb(&erts_halt_progress, -1); erts_halt_code = 0; + +#if !defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) + erts_lc_set_thread_name("scheduler 1"); +#endif + } ErtsRunQueue * @@ -5573,8 +5335,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) else if (on) { /* ------ BLOCK ------ */ if (schdlr_sspnd.msb.procs) { plp = proclist_create(p); - plp->next = schdlr_sspnd.msb.procs; - schdlr_sspnd.msb.procs = plp; + erts_proclist_store_last(&schdlr_sspnd.msb.procs, plp); p->flags |= F_HAVE_BLCKD_MSCHED; ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.active) == 1); ASSERT(p->scheduler_data->no == 1); @@ -5659,8 +5420,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) ~ERTS_SCHDLR_SSPND_CHNG_WAITER); } plp = proclist_create(p); - plp->next = schdlr_sspnd.msb.procs; - schdlr_sspnd.msb.procs = plp; + erts_proclist_store_last(&schdlr_sspnd.msb.procs, plp); ASSERT(p->scheduler_data); } } @@ -5671,20 +5431,16 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) } else { /* ------ UNBLOCK ------ */ if (p->flags & F_HAVE_BLCKD_MSCHED) { - ErtsProcList **plpp = &schdlr_sspnd.msb.procs; - plp = schdlr_sspnd.msb.procs; + ErtsProcList *plp = erts_proclist_peek_first(schdlr_sspnd.msb.procs); while (plp) { - if (!proclist_same(plp, p)){ - plpp = &plp->next; - plp = plp->next; - } - else { - *plpp = plp->next; - proclist_destroy(plp); + ErtsProcList *tmp_plp = plp; + plp = erts_proclist_peek_next(schdlr_sspnd.msb.procs, plp); + if (erts_proclist_same(tmp_plp, p)) { + erts_proclist_remove(&schdlr_sspnd.msb.procs, tmp_plp); + proclist_destroy(tmp_plp); if (!all) break; - plp = *plpp; } } } @@ -5753,23 +5509,25 @@ erts_multi_scheduling_blockers(Process *p) Eterm res = NIL; erts_smp_mtx_lock(&schdlr_sspnd.mtx); - if (schdlr_sspnd.msb.procs) { + if (!erts_proclist_is_empty(schdlr_sspnd.msb.procs)) { Eterm *hp, *hp_end; ErtsProcList *plp1, *plp2; - Uint max_size; - ASSERT(schdlr_sspnd.msb.procs); - for (max_size = 0, plp1 = schdlr_sspnd.msb.procs; + Uint max_size = 0; + + for (plp1 = erts_proclist_peek_first(schdlr_sspnd.msb.procs); plp1; - plp1 = plp1->next) { + plp1 = erts_proclist_peek_next(schdlr_sspnd.msb.procs, plp1)) { max_size += 2; } ASSERT(max_size); hp = HAlloc(p, max_size); hp_end = hp + max_size; - for (plp1 = schdlr_sspnd.msb.procs; plp1; plp1 = plp1->next) { - for (plp2 = schdlr_sspnd.msb.procs; + for (plp1 = erts_proclist_peek_first(schdlr_sspnd.msb.procs); + plp1; + plp1 = erts_proclist_peek_next(schdlr_sspnd.msb.procs, plp1)) { + for (plp2 = erts_proclist_peek_first(schdlr_sspnd.msb.procs); plp2->pid != plp1->pid; - plp2 = plp2->next); + plp2 = erts_proclist_peek_next(schdlr_sspnd.msb.procs, plp2)); if (plp2 == plp1) { res = CONS(hp, plp1->pid, res); hp += 2; @@ -6021,7 +5779,7 @@ handle_pend_sync_suspend(Process *suspendee, ASSERT(is_nil(suspender->suspendee)); if (suspendee_alive) { erts_suspend(suspendee, suspendee_locks, NULL); - suspender->suspendee = suspendee->id; + suspender->suspendee = suspendee->common.id; } /* suspender is suspended waiting for suspendee to suspend; resume suspender */ @@ -6042,7 +5800,7 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks, ERTS_SMP_LC_ASSERT(c_p_locks & ERTS_PROC_LOCK_MAIN); ERTS_SMP_LC_ASSERT(pid_locks & (ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS)); - if (c_p->id == pid) + if (c_p->common.id == pid) return erts_pid2proc(c_p, c_p_locks, pid, pid_locks); if (c_p_locks & ERTS_PROC_LOCK_STATUS) @@ -6094,7 +5852,7 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks, */ if (!c_p->pending_suspenders) { /* Mark rp pending for suspend by c_p */ - add_pend_suspend(rp, c_p->id, handle_pend_sync_suspend); + add_pend_suspend(rp, c_p->common.id, handle_pend_sync_suspend); ASSERT(is_nil(c_p->suspendee)); /* Suspend c_p; when rp is suspended c_p will be resumed. */ @@ -6200,20 +5958,20 @@ handle_pend_bif_sync_suspend(Process *suspendee, ASSERT(is_nil(suspender->suspendee)); if (!suspendee_alive) erts_delete_suspend_monitor(&suspender->suspend_monitors, - suspendee->id); + suspendee->common.id); else { #ifdef DEBUG int res; #endif ErtsSuspendMonitor *smon; smon = erts_lookup_suspend_monitor(suspender->suspend_monitors, - suspendee->id); + suspendee->common.id); #ifdef DEBUG res = #endif do_bif_suspend_process(suspendee, smon, suspendee); ASSERT(!smon || res != 0); - suspender->suspendee = suspendee->id; + suspender->suspendee = suspendee->common.id; } /* suspender is suspended waiting for suspendee to suspend; resume suspender */ @@ -6242,14 +6000,14 @@ handle_pend_bif_async_suspend(Process *suspendee, ASSERT(is_nil(suspender->suspendee)); if (!suspendee_alive) erts_delete_suspend_monitor(&suspender->suspend_monitors, - suspendee->id); + suspendee->common.id); else { #ifdef DEBUG int res; #endif ErtsSuspendMonitor *smon; smon = erts_lookup_suspend_monitor(suspender->suspend_monitors, - suspendee->id); + suspendee->common.id); #ifdef DEBUG res = #endif @@ -6294,7 +6052,7 @@ suspend_process_2(BIF_ALIST_2) int unless_suspending = 0; - if (BIF_P->id == BIF_ARG_1) + if (BIF_P->common.id == BIF_ARG_1) goto badarg; /* We are not allowed to suspend ourselves */ if (is_not_nil(BIF_ARG_2)) { @@ -6390,7 +6148,7 @@ suspend_process_2(BIF_ALIST_2) if (!do_bif_suspend_process(BIF_P, smon, suspendee)) add_pend_suspend(suspendee, - BIF_P->id, + BIF_P->common.id, handle_pend_bif_async_suspend); res = am_true; @@ -6453,7 +6211,7 @@ suspend_process_2(BIF_ALIST_2) else { /* Mark suspendee pending for suspend by BIF_P */ add_pend_suspend(suspendee, - BIF_P->id, + BIF_P->common.id, handle_pend_bif_sync_suspend); ASSERT(is_nil(BIF_P->suspendee)); @@ -6527,7 +6285,7 @@ resume_process_1(BIF_ALIST_1) Process *suspendee; int is_active; - if (BIF_P->id == BIF_ARG_1) + if (BIF_P->common.id == BIF_ARG_1) BIF_ERROR(BIF_P, BADARG); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK); @@ -6652,7 +6410,8 @@ erts_process_status(Process *c_p, ErtsProcLocks c_p_locks, for (i = 0; i < erts_no_schedulers; i++) { esdp = ERTS_SCHEDULER_IX(i); erts_smp_runq_lock(esdp->run_queue); - if (esdp->free_process && esdp->free_process->id == rpid) { + if (esdp->free_process + && esdp->free_process->common.id == rpid) { res = am_free; erts_smp_runq_unlock(esdp->run_queue); break; @@ -6673,27 +6432,31 @@ erts_process_status(Process *c_p, ErtsProcLocks c_p_locks, void erts_suspend(Process* c_p, ErtsProcLocks c_p_locks, Port *busy_port) { -#ifdef DEBUG - int res; -#endif + int suspend; + ASSERT(c_p == erts_get_current_process()); ERTS_SMP_LC_ASSERT(c_p_locks == erts_proc_lc_my_proc_locks(c_p)); if (!(c_p_locks & ERTS_PROC_LOCK_STATUS)) erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_STATUS); + if (busy_port) + suspend = erts_save_suspend_process_on_port(busy_port, c_p); + else + suspend = 1; + + if (suspend) { #ifdef DEBUG - res = + int res = #endif - suspend_process(c_p, c_p); - - ASSERT(res); - - if (busy_port) - erts_wake_process_later(busy_port, c_p); + suspend_process(c_p, c_p); + ASSERT(res); + } if (!(c_p_locks & ERTS_PROC_LOCK_STATUS)) erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS); + if (suspend && busy_port && erts_system_monitor_flags.busy_port) + monitor_generic(c_p, am_busy_port, busy_port->common.id); } void @@ -6708,16 +6471,19 @@ erts_resume(Process* process, ErtsProcLocks process_locks) } int -erts_resume_processes(ErtsProcList *plp) +erts_resume_processes(ErtsProcList *list) { + /* 'list' is expected to have been fetched (i.e. not a ring anymore) */ int nresumed = 0; + ErtsProcList *plp = list; + while (plp) { Process *proc; ErtsProcList *fplp; ASSERT(is_internal_pid(plp->pid)); proc = erts_pid2proc(NULL, 0, plp->pid, ERTS_PROC_LOCK_STATUS); if (proc) { - if (proclist_same(plp, proc)) { + if (erts_proclist_same(plp, proc)) { resume_process(proc); nresumed++; } @@ -6870,9 +6636,8 @@ Process *schedule(Process *p, int calls) state = erts_smp_atomic32_read_acqb(&p->state); if (IS_TRACED(p)) { - if (IS_TRACED_FL(p, F_TRACE_CALLS) && !(state & ERTS_PSFLG_FREE)) { + if (IS_TRACED_FL(p, F_TRACE_CALLS) && !(state & ERTS_PSFLG_FREE)) erts_schedule_time_break(p, ERTS_BP_CALL_TIME_SCHEDULE_OUT); - } if (state & (ERTS_PSFLG_FREE|ERTS_PSFLG_EXITING)) { if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED_EXIT)) trace_sched(p, ((state & ERTS_PSFLG_FREE) @@ -6885,7 +6650,7 @@ Process *schedule(Process *p, int calls) else if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED_PROCS)) trace_virtual_sched(p, am_out); } - } + } #ifdef ERTS_SMP if (state & ERTS_PSFLG_PENDING_EXIT) @@ -6949,9 +6714,8 @@ Process *schedule(Process *p, int calls) #ifdef ERTS_SMP { ErtsProcList *pnd_xtrs = rq->procs.pending_exiters; - rq->procs.pending_exiters = NULL; - - if (pnd_xtrs) { + if (erts_proclist_fetch(&pnd_xtrs, NULL)) { + rq->procs.pending_exiters = NULL; erts_smp_runq_unlock(rq); handle_pending_exiters(pnd_xtrs); erts_smp_runq_lock(rq); @@ -7247,7 +7011,7 @@ Process *schedule(Process *p, int calls) erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); #ifdef ERTS_SMP - if (is_not_nil(p->tracer_proc)) + if (is_not_nil(ERTS_TRACER_PROC(p))) erts_check_my_tracer_proc(p); #endif @@ -7450,70 +7214,6 @@ erts_get_exact_total_reductions(Process *c_p, Uint *redsp, Uint *diffp) erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); } -/* - * erts_test_next_pid() is only used for testing. - */ -Sint -erts_test_next_pid(int set, Uint next) -{ - Uint64 lpd; - Sint res; - Eterm pid_data; - int first_pix = -1; - - erts_smp_rwmtx_rwlock(&erts_proc_tab_rwmtx); - - if (!set) - lpd = last_pid_data_read_nob(); - else { - - lpd = (Uint64) next; - pid_data = (Eterm) (lpd & ERTS_PID_DATA_MASK__); - if (ERTS_INVALID_PID == make_internal_pid(pid_data)) { - lpd += erts_proc.max; - ASSERT(erts_pid_data2ix(pid_data) - == erts_pid_data2ix(lpd & ERTS_PID_DATA_MASK__)); - } - last_pid_data_set_relb(lpd); - } - - while (1) { - int pix; - lpd++; - pix = (int) (lpd % erts_proc.max); - if (first_pix < 0) - first_pix = pix; - else if (pix == first_pix) { - res = -1; - break; - } - if (ERTS_AINT_NULL == erts_smp_atomic_read_nob(&erts_proc.tab[pix])) { - pid_data = (Eterm) (lpd & ERTS_PID_DATA_MASK__); - if (ERTS_INVALID_PID == make_internal_pid(pid_data)) { - lpd += erts_proc.max; - ASSERT(erts_pid_data2ix(pid_data) - == erts_pid_data2ix(lpd & ERTS_PID_DATA_MASK__)); - } - res = lpd & ERTS_PID_DATA_MASK__; - break; - } - } - - erts_smp_rwmtx_rwunlock(&erts_proc_tab_rwmtx); - - return res; - -} - -Uint erts_process_count(void) -{ - erts_aint32_t res = erts_smp_atomic32_read_nob(&process_count); - if (res > erts_proc.max) - return erts_proc.max; - ASSERT(res >= 0); - return (Uint) res; -} - void erts_free_proc(Process *p) { @@ -7523,132 +7223,66 @@ erts_free_proc(Process *p) erts_free(ERTS_ALC_T_PROC, (void *) p); } +typedef struct { + Process *proc; + erts_aint32_t state; + ErtsRunQueue *run_queue; +} ErtsEarlyProcInit; + +static void early_init_process_struct(void *varg, Eterm data) +{ + ErtsEarlyProcInit *arg = (ErtsEarlyProcInit *) varg; + Process *proc = arg->proc; + + proc->common.id = make_internal_pid(data); + erts_smp_atomic32_init_relb(&proc->state, arg->state); + +#ifdef ERTS_SMP + RUNQ_SET_RQ(&proc->run_queue, arg->run_queue); + + erts_proc_lock_init(proc); /* All locks locked */ +#endif + +} + /* ** Allocate process and find out where to place next process. */ static Process* alloc_process(ErtsRunQueue *rq, erts_aint32_t state) { - int pix; - Process* p; - Uint64 lpd, exp_lpd; - Eterm pid_data; - erts_aint32_t proc_count; -#ifdef DEBUG - Eterm pid; -#endif - - erts_smp_rwmtx_rlock(&erts_proc_tab_rwmtx); - - proc_count = erts_smp_atomic32_inc_read_acqb(&process_count); - if (proc_count > erts_proc.max) { - while (1) { - erts_aint32_t act_proc_count; - - act_proc_count = erts_smp_atomic32_cmpxchg_relb(&process_count, - proc_count-1, - proc_count); - if (act_proc_count == proc_count) - goto system_limit; - proc_count = act_proc_count; - if (proc_count <= erts_proc.max) - break; - } - } + ErtsEarlyProcInit init_arg; + Process *p; - p = (Process*) erts_alloc_fnf(ERTS_ALC_T_PROC, sizeof(Process)); + p = erts_alloc_fnf(ERTS_ALC_T_PROC, sizeof(Process)); if (!p) - goto enomem; - - p->approx_started = erts_get_approx_time(); - p->started_interval = get_proc_interval(); - - lpd = last_pid_data_read_acqb(); - - /* Reserve slot */ - while (1) { - lpd++; - pix = erts_pid_data2ix((Eterm) (lpd & ERTS_PID_DATA_MASK__)); - if (erts_smp_atomic_read_nob(&erts_proc.tab[pix]) == ERTS_AINT_NULL) { - erts_aint_t val; - val = erts_smp_atomic_cmpxchg_relb(&erts_proc.tab[pix], - ((erts_aint_t) - ERTS_PROC_LOCK_BUSY), - ERTS_AINT_NULL); - - if (ERTS_AINT_NULL == val) - break; - } - } - - pid_data = (Eterm) lpd & ERTS_PID_DATA_MASK__; + return NULL; - p->id = make_internal_pid(pid_data); - if (p->id == ERTS_INVALID_PID) { - /* Do not use the invalid pid; change serial */ - lpd += erts_proc.max; - ASSERT(pix == erts_pid_data2ix((Eterm) (lpd & ERTS_PID_DATA_MASK__))); - pid_data = (Eterm) lpd & ERTS_PID_DATA_MASK__; - p->id = make_internal_pid(pid_data); - ASSERT(p->id != ERTS_INVALID_PID); - } + init_arg.proc = (Process *) p; + init_arg.run_queue = rq; + init_arg.state = state; - exp_lpd = last_pid_data_read_nob(); + ASSERT(((char *) p) == ((char *) &p->common)); - /* Move last pid data forward */ - while (1) { - Uint64 act_lpd; - if (last_pid_data_cmp(lpd, exp_lpd) < 0) - break; - act_lpd = last_pid_data_cmpxchg_relb(lpd, exp_lpd); - if (act_lpd == exp_lpd) - break; - exp_lpd = act_lpd; + if (!erts_ptab_new_element(&erts_proc, + &p->common, + (void *) &init_arg, + early_init_process_struct)) { + erts_free(ERTS_ALC_T_PROC, p); + return NULL; } -#ifdef ERTS_SMP - RUNQ_SET_RQ(&p->run_queue, rq); -#endif - - erts_smp_atomic32_init_relb(&p->state, state); - -#ifdef DEBUG - pid = p->id; -#endif - -#ifdef ERTS_SMP - erts_proc_lock_init(p); /* All locks locked */ -#endif - - /* Move into slot reserved */ -#ifdef DEBUG - ASSERT(ERTS_PROC_LOCK_BUSY - == (Process *) erts_smp_atomic_xchg_relb(&erts_proc.tab[pix], - (erts_aint_t) p)); -#else - erts_smp_atomic_set_relb(&erts_proc.tab[pix], (erts_aint_t) p); -#endif - - ASSERT(internal_pid_serial(p->id) <= (erts_use_r9_pids_ports - ? ERTS_MAX_PID_R9_SERIAL - : ERTS_MAX_PID_SERIAL)); - - erts_smp_rwmtx_runlock(&erts_proc_tab_rwmtx); - + ASSERT(internal_pid_serial(p->common.id) <= ERTS_MAX_PID_SERIAL); + + p->approx_started = erts_get_approx_time(); p->rcount = 0; - ASSERT(p == (Process *) - erts_smp_atomic_read_nob( - &erts_proc.tab[internal_pid_index(pid)])); - - return p; -enomem: -system_limit: - - erts_smp_rwmtx_runlock(&erts_proc_tab_rwmtx); - return NULL; + ASSERT(p == (Process *) (erts_ptab_pix2intptr_nob( + &erts_proc, + internal_pid_index(p->common.id)))); + return p; } Eterm @@ -7795,21 +7429,21 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->reds = 0; #ifdef ERTS_SMP - p->u.alive.ptimer = NULL; + p->common.u.alive.ptimer = NULL; #else - sys_memset(&p->u.alive.tm, 0, sizeof(ErlTimer)); + sys_memset(&p->common.u.alive.tm, 0, sizeof(ErlTimer)); #endif - p->reg = NULL; - p->nlinks = NULL; - p->monitors = NULL; + p->common.u.alive.reg = NULL; + ERTS_P_LINKS(p) = NULL; + ERTS_P_MONITORS(p) = NULL; p->nodes_monitors = NULL; p->suspend_monitors = NULL; ASSERT(is_pid(parent->group_leader)); if (parent->group_leader == ERTS_INVALID_PID) - p->group_leader = p->id; + p->group_leader = p->common.id; else { /* Needs to be done after the heap has been set up */ p->group_leader = @@ -7818,18 +7452,18 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). : STORE_NC(&p->htop, &p->off_heap, parent->group_leader); } - erts_get_default_tracing(&p->trace_flags, &p->tracer_proc); + erts_get_default_tracing(&ERTS_TRACE_FLAGS(p), &ERTS_TRACER_PROC(p)); p->msg.first = NULL; p->msg.last = &p->msg.first; p->msg.save = &p->msg.first; p->msg.len = 0; #ifdef ERTS_SMP - p->u.alive.msg_inq.first = NULL; - p->u.alive.msg_inq.last = &p->u.alive.msg_inq.first; - p->u.alive.msg_inq.len = 0; + p->msg_inq.first = NULL; + p->msg_inq.last = &p->msg_inq.first; + p->msg_inq.len = 0; #endif - p->bif_timers = NULL; + p->u.bif_timers = NULL; p->mbuf = NULL; p->mbuf_sz = 0; p->psd = NULL; @@ -7841,7 +7475,9 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). DT_UTAG(p) = NIL; DT_UTAG_FLAGS(p) = 0; #endif - p->parent = parent->id == ERTS_INVALID_PID ? NIL : parent->id; + p->parent = (parent->common.id == ERTS_INVALID_PID + ? NIL + : parent->common.id); INIT_HOLE_CHECK(p); #ifdef DEBUG @@ -7849,18 +7485,19 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). #endif if (IS_TRACED(parent)) { - if (parent->trace_flags & F_TRACE_SOS) { - p->trace_flags |= (parent->trace_flags & TRACEE_FLAGS); - p->tracer_proc = parent->tracer_proc; + if (ERTS_TRACE_FLAGS(parent) & F_TRACE_SOS) { + ERTS_TRACE_FLAGS(p) |= (ERTS_TRACE_FLAGS(parent) & TRACEE_FLAGS); + ERTS_TRACER_PROC(p) = ERTS_TRACER_PROC(parent); } if (ARE_TRACE_FLAGS_ON(parent, F_TRACE_PROCS)) { - trace_proc_spawn(parent, p->id, mod, func, args); + trace_proc_spawn(parent, p->common.id, mod, func, args); } - if (parent->trace_flags & F_TRACE_SOS1) { /* Overrides TRACE_CHILDREN */ - p->trace_flags |= (parent->trace_flags & TRACEE_FLAGS); - p->tracer_proc = parent->tracer_proc; - p->trace_flags &= ~(F_TRACE_SOS1 | F_TRACE_SOS); - parent->trace_flags &= ~(F_TRACE_SOS1 | F_TRACE_SOS); + if (ERTS_TRACE_FLAGS(parent) & F_TRACE_SOS1) { + /* Overrides TRACE_CHILDREN */ + ERTS_TRACE_FLAGS(p) |= (ERTS_TRACE_FLAGS(parent) & TRACEE_FLAGS); + ERTS_TRACER_PROC(p) = ERTS_TRACER_PROC(parent); + ERTS_TRACE_FLAGS(p) &= ~(F_TRACE_SOS1 | F_TRACE_SOS); + ERTS_TRACE_FLAGS(parent) &= ~(F_TRACE_SOS1 | F_TRACE_SOS); } } @@ -7873,27 +7510,27 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). int ret; #endif if (IS_TRACED_FL(parent, F_TRACE_PROCS)) { - trace_proc(parent, parent, am_link, p->id); + trace_proc(parent, parent, am_link, p->common.id); } #ifdef DEBUG - ret = erts_add_link(&(parent->nlinks), LINK_PID, p->id); + ret = erts_add_link(&ERTS_P_LINKS(parent), LINK_PID, p->common.id); ASSERT(ret == 0); - ret = erts_add_link(&(p->nlinks), LINK_PID, parent->id); + ret = erts_add_link(&ERTS_P_LINKS(p), LINK_PID, parent->common.id); ASSERT(ret == 0); #else - erts_add_link(&(parent->nlinks), LINK_PID, p->id); - erts_add_link(&(p->nlinks), LINK_PID, parent->id); + erts_add_link(&ERTS_P_LINKS(parent), LINK_PID, p->common.id); + erts_add_link(&ERTS_P_LINKS(p), LINK_PID, parent->common.id); #endif if (IS_TRACED(parent)) { - if (parent->trace_flags & (F_TRACE_SOL|F_TRACE_SOL1)) { - p->trace_flags |= (parent->trace_flags & TRACEE_FLAGS); - p->tracer_proc = parent->tracer_proc; /* maybe steal */ + if (ERTS_TRACE_FLAGS(parent) & (F_TRACE_SOL|F_TRACE_SOL1)) { + ERTS_TRACE_FLAGS(p) |= (ERTS_TRACE_FLAGS(parent)&TRACEE_FLAGS); + ERTS_TRACER_PROC(p) = ERTS_TRACER_PROC(parent); /*maybe steal*/ - if (parent->trace_flags & F_TRACE_SOL1) { /* maybe override */ - p ->trace_flags &= ~(F_TRACE_SOL1 | F_TRACE_SOL); - parent->trace_flags &= ~(F_TRACE_SOL1 | F_TRACE_SOL); + if (ERTS_TRACE_FLAGS(parent) & F_TRACE_SOL1) {/*maybe override*/ + ERTS_TRACE_FLAGS(p) &= ~(F_TRACE_SOL1 | F_TRACE_SOL); + ERTS_TRACE_FLAGS(parent) &= ~(F_TRACE_SOL1 | F_TRACE_SOL); } } } @@ -7906,8 +7543,8 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). Eterm mref; mref = erts_make_ref(parent); - erts_add_monitor(&(parent->monitors), MON_ORIGIN, mref, p->id, NIL); - erts_add_monitor(&(p->monitors), MON_TARGET, mref, parent->id, NIL); + erts_add_monitor(&ERTS_P_MONITORS(parent), MON_ORIGIN, mref, p->common.id, NIL); + erts_add_monitor(&ERTS_P_MONITORS(p), MON_TARGET, mref, parent->common.id, NIL); so->mref = mref; } @@ -7915,8 +7552,8 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->scheduler_data = NULL; p->suspendee = NIL; p->pending_suspenders = NULL; - p->u.alive.pending_exit.reason = THE_NON_VALUE; - p->u.alive.pending_exit.bp = NULL; + p->pending_exit.reason = THE_NON_VALUE; + p->pending_exit.bp = NULL; #endif #if !defined(NO_FPE_SIGNALS) || defined(HIPE) @@ -7925,7 +7562,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL); - res = p->id; + res = p->common.id; /* * Schedule process for execution. @@ -7933,7 +7570,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). schedule_process(p, state, 0); - VERBOSE(DEBUG_PROCESSES, ("Created a new process: %T\n",p->id)); + VERBOSE(DEBUG_PROCESSES, ("Created a new process: %T\n",p->common.id)); #ifdef USE_VM_PROBES if (DTRACE_ENABLED(process_spawn)) { @@ -7968,10 +7605,10 @@ void erts_init_empty_process(Process *p) p->min_heap_size = 0; p->min_vheap_size = 0; p->rcount = 0; - p->id = ERTS_INVALID_PID; + p->common.id = ERTS_INVALID_PID; p->reds = 0; - p->tracer_proc = NIL; - p->trace_flags = F_INITIAL_TRACE_FLAGS; + ERTS_TRACER_PROC(p) = NIL; + ERTS_TRACE_FLAGS(p) = F_INITIAL_TRACE_FLAGS; p->group_leader = ERTS_INVALID_PID; p->flags = 0; p->fvalue = NIL; @@ -7984,14 +7621,14 @@ void erts_init_empty_process(Process *p) p->bin_old_vheap = 0; p->bin_vheap_mature = 0; #ifdef ERTS_SMP - p->u.alive.ptimer = NULL; + p->common.u.alive.ptimer = NULL; #else - memset(&(p->u.alive.tm), 0, sizeof(ErlTimer)); + memset(&(p->common.u.alive.tm), 0, sizeof(ErlTimer)); #endif p->next = NULL; p->off_heap.first = NULL; p->off_heap.overhead = 0; - p->reg = NULL; + p->common.u.alive.reg = NULL; p->heap_sz = 0; p->high_water = NULL; p->old_hend = NULL; @@ -8000,15 +7637,15 @@ void erts_init_empty_process(Process *p) p->mbuf = NULL; p->mbuf_sz = 0; p->psd = NULL; - p->monitors = NULL; - p->nlinks = NULL; /* List of links */ + ERTS_P_MONITORS(p) = NULL; + ERTS_P_LINKS(p) = NULL; /* List of links */ p->nodes_monitors = NULL; p->suspend_monitors = NULL; p->msg.first = NULL; p->msg.last = &p->msg.first; p->msg.save = &p->msg.first; p->msg.len = 0; - p->bif_timers = NULL; + p->u.bif_timers = NULL; p->dictionary = NULL; p->seq_trace_clock = 0; p->seq_trace_lastcnt = 0; @@ -8036,7 +7673,7 @@ void erts_init_empty_process(Process *p) p->parent = NIL; p->approx_started = 0; - p->started_interval = 0; + p->common.u.alive.started_interval = 0; #ifdef HIPE hipe_init_process(&p->hipe); @@ -8054,13 +7691,13 @@ void erts_init_empty_process(Process *p) #ifdef ERTS_SMP p->scheduler_data = NULL; - p->u.alive.msg_inq.first = NULL; - p->u.alive.msg_inq.last = &p->u.alive.msg_inq.first; - p->u.alive.msg_inq.len = 0; + p->msg_inq.first = NULL; + p->msg_inq.last = &p->msg_inq.first; + p->msg_inq.len = 0; p->suspendee = NIL; p->pending_suspenders = NULL; - p->u.alive.pending_exit.reason = THE_NON_VALUE; - p->u.alive.pending_exit.bp = NULL; + p->pending_exit.reason = THE_NON_VALUE; + p->pending_exit.bp = NULL; erts_proc_lock_init(p); erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL); RUNQ_SET_RQ(&p->run_queue, ERTS_RUNQ_IX(0)); @@ -8082,25 +7719,25 @@ erts_debug_verify_clean_empty_process(Process* p) ASSERT(p->stop == NULL); ASSERT(p->hend == NULL); ASSERT(p->heap == NULL); - ASSERT(p->id == ERTS_INVALID_PID); - ASSERT(p->tracer_proc == NIL); - ASSERT(p->trace_flags == F_INITIAL_TRACE_FLAGS); + ASSERT(p->common.id == ERTS_INVALID_PID); + ASSERT(ERTS_TRACER_PROC(p) == NIL); + ASSERT(ERTS_TRACE_FLAGS(p) == F_INITIAL_TRACE_FLAGS); ASSERT(p->group_leader == ERTS_INVALID_PID); ASSERT(p->next == NULL); - ASSERT(p->reg == NULL); + ASSERT(p->common.u.alive.reg == NULL); ASSERT(p->heap_sz == 0); ASSERT(p->high_water == NULL); ASSERT(p->old_hend == NULL); ASSERT(p->old_htop == NULL); ASSERT(p->old_heap == NULL); - ASSERT(p->monitors == NULL); - ASSERT(p->nlinks == NULL); + ASSERT(ERTS_P_MONITORS(p) == NULL); + ASSERT(ERTS_P_LINKS(p) == NULL); ASSERT(p->nodes_monitors == NULL); ASSERT(p->suspend_monitors == NULL); ASSERT(p->msg.first == NULL); ASSERT(p->msg.len == 0); - ASSERT(p->bif_timers == NULL); + ASSERT(p->u.bif_timers == NULL); ASSERT(p->dictionary == NULL); ASSERT(p->catches == 0); ASSERT(p->cp == NULL); @@ -8110,12 +7747,12 @@ erts_debug_verify_clean_empty_process(Process* p) ASSERT(p->parent == NIL); #ifdef ERTS_SMP - ASSERT(p->u.alive.msg_inq.first == NULL); - ASSERT(p->u.alive.msg_inq.len == 0); + ASSERT(p->msg_inq.first == NULL); + ASSERT(p->msg_inq.len == 0); ASSERT(p->suspendee == NIL); ASSERT(p->pending_suspenders == NULL); - ASSERT(p->u.alive.pending_exit.reason == THE_NON_VALUE); - ASSERT(p->u.alive.pending_exit.bp == NULL); + ASSERT(p->pending_exit.reason == THE_NON_VALUE); + ASSERT(p->pending_exit.bp == NULL); #endif /* Thing that erts_cleanup_empty_process() cleans up */ @@ -8157,7 +7794,7 @@ delete_process(Process* p) { ErlMessage* mp; - VERBOSE(DEBUG_PROCESSES, ("Removing process: %T\n",p->id)); + VERBOSE(DEBUG_PROCESSES, ("Removing process: %T\n",p->common.id)); /* Cleanup psd */ @@ -8231,8 +7868,6 @@ delete_process(Process* p) mp = next_mp; } - ASSERT(!p->monitors); - ASSERT(!p->nlinks); ASSERT(!p->nodes_monitors); ASSERT(!p->suspend_monitors); @@ -8296,7 +7931,7 @@ void erts_handle_pending_exit(Process *c_p, ErtsProcLocks locks) { ErtsProcLocks xlocks; - ASSERT(is_value(c_p->u.alive.pending_exit.reason)); + ASSERT(is_value(c_p->pending_exit.reason)); ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == locks); ERTS_SMP_LC_ASSERT(locks & ERTS_PROC_LOCK_MAIN); ERTS_SMP_LC_ASSERT(!((ERTS_PSFLG_EXITING|ERTS_PSFLG_FREE) @@ -8315,10 +7950,10 @@ erts_handle_pending_exit(Process *c_p, ErtsProcLocks locks) set_proc_exiting(c_p, erts_smp_atomic32_read_acqb(&c_p->state), - c_p->u.alive.pending_exit.reason, - c_p->u.alive.pending_exit.bp); - c_p->u.alive.pending_exit.reason = THE_NON_VALUE; - c_p->u.alive.pending_exit.bp = NULL; + c_p->pending_exit.reason, + c_p->pending_exit.bp); + c_p->pending_exit.reason = THE_NON_VALUE; + c_p->pending_exit.bp = NULL; if (xlocks) erts_smp_proc_unlock(c_p, xlocks); @@ -8327,12 +7962,14 @@ erts_handle_pending_exit(Process *c_p, ErtsProcLocks locks) static void handle_pending_exiters(ErtsProcList *pnd_xtrs) { + /* 'list' is expected to have been fetched (i.e. not a ring anymore) */ ErtsProcList *plp = pnd_xtrs; - ErtsProcList *free_plp; + while (plp) { + ErtsProcList *free_plp; Process *p = erts_pid2proc(NULL, 0, plp->pid, ERTS_PROC_LOCKS_ALL); if (p) { - if (proclist_same(plp, p)) { + if (erts_proclist_same(plp, p)) { erts_aint32_t state = erts_smp_atomic32_read_acqb(&p->state); if (!(state & ERTS_PSFLG_RUNNING)) { ASSERT(state & ERTS_PSFLG_PENDING_EXIT); @@ -8361,8 +7998,7 @@ save_pending_exiter(Process *p) erts_smp_runq_lock(rq); - plp->next = rq->procs.pending_exiters; - rq->procs.pending_exiters = plp; + erts_proclist_store_last(&rq->procs.pending_exiters, plp); erts_smp_runq_unlock(rq); wake_scheduler(rq, 1); @@ -8409,7 +8045,7 @@ send_exit_message(Process *to, ErtsProcLocks *to_locksp, hp = bp->mem; mess = copy_struct(exit_term, term_size, &hp, &bp->off_heap); /* the trace token must in this case be updated by the caller */ - seq_trace_output(token, mess, SEQ_TRACE_SEND, to->id, NULL); + seq_trace_output(token, mess, SEQ_TRACE_SEND, to->common.id, NULL); temp_token = copy_struct(token, sz_token, &hp, &bp->off_heap); erts_queue_message(to, to_locksp, bp, mess, temp_token #ifdef USE_VM_PROBES @@ -8539,7 +8175,7 @@ send_exit_signal(Process *c_p, /* current process if and only else if (reason != am_normal || (flags & ERTS_XSIG_FLG_NO_IGN_NORMAL)) { #ifdef ERTS_SMP if (!(state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT))) { - ASSERT(!rp->u.alive.pending_exit.bp); + ASSERT(!rp->pending_exit.bp); if (rp == c_p && (*rp_locks & ERTS_PROC_LOCK_MAIN)) { /* Ensure that all locks on c_p are locked before @@ -8589,7 +8225,7 @@ send_exit_signal(Process *c_p, /* current process if and only set_pending_exit: if (is_immed(rsn)) { - rp->u.alive.pending_exit.reason = rsn; + rp->pending_exit.reason = rsn; } else { Eterm *hp; @@ -8597,11 +8233,11 @@ send_exit_signal(Process *c_p, /* current process if and only ErlHeapFragment *bp = new_message_buffer(sz); hp = &bp->mem[0]; - rp->u.alive.pending_exit.reason = copy_struct(rsn, - sz, - &hp, - &bp->off_heap); - rp->u.alive.pending_exit.bp = bp; + rp->pending_exit.reason = copy_struct(rsn, + sz, + &hp, + &bp->off_heap); + rp->pending_exit.bp = bp; } erts_smp_atomic32_read_bor_relb(&rp->state, ERTS_PSFLG_PENDING_EXIT); @@ -8700,7 +8336,7 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext) if (!rp) { goto done; } - rmon = erts_remove_monitor(&(rp->monitors),mon->ref); + rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), mon->ref); erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); if (rmon == NULL) { goto done; @@ -8735,7 +8371,7 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext) ASSERT(mon->type == MON_TARGET); ASSERT(is_pid(mon->pid) || is_internal_port(mon->pid)); if (is_internal_port(mon->pid)) { - Port *prt = erts_id2port(mon->pid, NULL, 0); + Port *prt = erts_id2port(mon->pid); if (prt == NULL) { goto done; } @@ -8751,13 +8387,13 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext) goto done; } UseTmpHeapNoproc(3); - rmon = erts_remove_monitor(&(rp->monitors),mon->ref); + rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), mon->ref); if (rmon) { erts_destroy_monitor(rmon); watched = (is_atom(mon->name) ? TUPLE2(lhp, mon->name, erts_this_dist_entry->sysname) - : pcontext->p->id); + : pcontext->p->common.id); erts_queue_monitor_message(rp, &rp_locks, mon->ref, am_process, watched, pcontext->reason); } @@ -8822,21 +8458,22 @@ static void doit_exit_link(ErtsLink *lnk, void *vpcontext) switch(lnk->type) { case LINK_PID: if(is_internal_port(item)) { - Port *prt = erts_id2port(item, NULL, 0); - if (prt) { - rlnk = erts_remove_link(&prt->nlinks, p->id); - if (rlnk) - erts_destroy_link(rlnk); - erts_do_exit_port(prt, p->id, reason); - erts_port_release(prt); - } + Port *prt = erts_port_lookup(item, ERTS_PORT_SFLGS_INVALID_LOOKUP); + if (prt) + erts_port_exit(NULL, + (ERTS_PORT_SIG_FLG_FORCE_SCHED + | ERTS_PORT_SIG_FLG_BROKEN_LINK), + prt, + p->common.id, + reason, + NULL); } else if(is_external_port(item)) { erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); erts_dsprintf(dsbufp, "Erroneous link between %T and external port %T " "found\n", - p->id, + p->common.id, item); erts_send_error_to_logger_nogl(dsbufp); ASSERT(0); /* It isn't possible to setup such a link... */ @@ -8846,14 +8483,14 @@ static void doit_exit_link(ErtsLink *lnk, void *vpcontext) | ERTS_PROC_LOCKS_XSIG_SEND); rp = erts_pid2proc(NULL, 0, item, rp_locks); if (rp) { - rlnk = erts_remove_link(&(rp->nlinks), p->id); + rlnk = erts_remove_link(&ERTS_P_LINKS(rp), p->common.id); /* If rlnk == NULL, we got unlinked while exiting, i.e., do nothing... */ if (rlnk) { int xres; erts_destroy_link(rlnk); xres = send_exit_signal(NULL, - p->id, + p->common.id, rp, &rp_locks, reason, @@ -8865,7 +8502,7 @@ static void doit_exit_link(ErtsLink *lnk, void *vpcontext) if (xres >= 0 && IS_TRACED_FL(rp, F_TRACE_PROCS)) { /* We didn't exit the process and it is traced */ if (IS_TRACED_FL(rp, F_TRACE_PROCS)) { - trace_proc(p, rp, am_getting_unlinked, p->id); + trace_proc(p, rp, am_getting_unlinked, p->common.id); } } } @@ -8879,12 +8516,12 @@ static void doit_exit_link(ErtsLink *lnk, void *vpcontext) ErtsDSigData dsd; int code; ErtsDistLinkData dld; - erts_remove_dist_link(&dld, p->id, item, dep); + erts_remove_dist_link(&dld, p->common.id, item, dep); erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); code = erts_dsig_prepare(&dsd, dep, p, ERTS_DSP_NO_LOCK, 0); if (code == ERTS_DSIG_PREP_CONNECTED) { - code = erts_dsig_send_exit_tt(&dsd, p->id, item, reason, - SEQ_TRACE_TOKEN(p)); + code = erts_dsig_send_exit_tt(&dsd, p->common.id, item, + reason, SEQ_TRACE_TOKEN(p)); ASSERT(code == ERTS_DSIG_SEND_OK); } erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN); @@ -8899,7 +8536,7 @@ static void doit_exit_link(ErtsLink *lnk, void *vpcontext) /* dist entries have node links in a separate structure to avoid confusion */ erts_smp_de_links_lock(dep); - rlnk = erts_remove_link(&(dep->node_links), p->id); + rlnk = erts_remove_link(&(dep->node_links), p->common.id); erts_smp_de_links_unlock(dep); if (rlnk) erts_destroy_link(rlnk); @@ -8927,15 +8564,6 @@ resume_suspend_monitor(ErtsSuspendMonitor *smon, void *vc_p) erts_destroy_suspend_monitor(smon); } -#ifdef ERTS_SMP -static void -proc_dec_refc(void *vproc) -{ - erts_smp_proc_dec_refc((Process *) vproc); -} -#endif - - /* this function fishishes a process and propagates exit messages - called by process_main when a process dies */ void @@ -8973,10 +8601,10 @@ erts_do_exit_process(Process* p, Eterm reason) state = set_proc_exiting_state(p, erts_smp_atomic32_read_nob(&p->state)); if (state & ERTS_PSFLG_PENDING_EXIT) { /* Process exited before pending exit was received... */ - p->u.alive.pending_exit.reason = THE_NON_VALUE; - if (p->u.alive.pending_exit.bp) { - free_message_buffer(p->u.alive.pending_exit.bp); - p->u.alive.pending_exit.bp = NULL; + p->pending_exit.reason = THE_NON_VALUE; + if (p->pending_exit.bp) { + free_message_buffer(p->pending_exit.bp); + p->pending_exit.bp = NULL; } } @@ -8993,23 +8621,25 @@ erts_do_exit_process(Process* p, Eterm reason) trace_proc(p, p, am_exit, reason); } - erts_trace_check_exiting(p->id); + erts_trace_check_exiting(p->common.id); - ASSERT((p->trace_flags & F_INITIAL_TRACE_FLAGS) == F_INITIAL_TRACE_FLAGS); + ASSERT((ERTS_TRACE_FLAGS(p) & F_INITIAL_TRACE_FLAGS) + == F_INITIAL_TRACE_FLAGS); cancel_timer(p); /* Always cancel timer just in case */ - /* - * The timer of this process can *not* be used anymore. The field used - * for the timer is now used for misc exiting data. - */ - p->u.exit_data = NULL; - - if (p->bif_timers) + if (p->u.bif_timers) erts_cancel_bif_timers(p, ERTS_PROC_LOCKS_ALL); erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL_MINOR); + /* + * The p->u.bif_timers of this process can *not* be used anymore; + * will be overwritten by misc termination data. + */ + p->u.terminate = NULL; + + erts_continue_exit_process(p); } @@ -9081,9 +8711,9 @@ erts_continue_exit_process(Process *p) * The registered name *should* be the last "erlang resource" to * cleanup. */ - if (p->reg) { + if (p->common.u.alive.reg) { (void) erts_unregister_name(p, ERTS_PROC_LOCK_MAIN, NULL, THE_NON_VALUE); - ASSERT(!p->reg); + ASSERT(!p->common.u.alive.reg); } erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR); @@ -9097,22 +8727,18 @@ erts_continue_exit_process(Process *p) yield_allowed = 0; #endif + /* + * Note! The monitor and link fields will be overwritten + * by erts_ptab_delete_element() below. + */ + mon = ERTS_P_MONITORS(p); + lnk = ERTS_P_LINKS(p); + { - int maybe_save; - int pix; /* Do *not* use erts_get_runq_proc() */ ErtsRunQueue *rq; rq = erts_get_runq_current(ERTS_GET_SCHEDULER_DATA_FROM_PROC(p)); - pix = internal_pid_index(p->id); - - erts_smp_rwmtx_rlock(&erts_proc_tab_rwmtx); - maybe_save = saved_term_procs.end != NULL; - if (maybe_save) { - erts_smp_rwmtx_runlock(&erts_proc_tab_rwmtx); - erts_smp_rwmtx_rwlock(&erts_proc_tab_rwmtx); - } - erts_smp_runq_lock(rq); #ifdef ERTS_SMP @@ -9123,22 +8749,11 @@ erts_continue_exit_process(Process *p) p->scheduler_data->current_process = NULL; p->scheduler_data->free_process = p; #endif - /* Time of death! */ - erts_smp_atomic_set_relb(&erts_proc.tab[pix], ERTS_AINT_NULL); - ASSERT(erts_smp_atomic32_read_nob(&process_count) > 0); - erts_smp_atomic32_dec_relb(&process_count); + /* Time of death! */ + erts_ptab_delete_element(&erts_proc, &p->common); erts_smp_runq_unlock(rq); - - if (!maybe_save) - erts_smp_rwmtx_runlock(&erts_proc_tab_rwmtx); - else { - if (saved_term_procs.end) - save_terminating_process(p); - erts_smp_rwmtx_rwunlock(&erts_proc_tab_rwmtx); - } - } /* @@ -9148,12 +8763,6 @@ erts_continue_exit_process(Process *p) * when the monitors and/or links hit. */ - mon = p->monitors; - p->monitors = NULL; /* to avoid recursive deletion during traversal */ - - lnk = p->nlinks; - p->nlinks = NULL; - { /* Inactivate and notify free */ erts_aint32_t n, e, a = erts_smp_atomic32_read_nob(&p->state); @@ -9197,7 +8806,7 @@ erts_continue_exit_process(Process *p) UseTmpHeap(4,p); hp = &tmp_heap[0]; - exit_tuple = TUPLE3(hp, am_EXIT, p->id, reason); + exit_tuple = TUPLE3(hp, am_EXIT, p->common.id, reason); exit_tuple_sz = size_object(exit_tuple); @@ -9223,9 +8832,6 @@ erts_continue_exit_process(Process *p) delete_process(p); #ifdef ERTS_SMP - erts_schedule_thr_prgr_later_op(proc_dec_refc, - (void *) p, - &p->u.release_data); erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); #endif @@ -9277,9 +8883,9 @@ cancel_timer(Process* p) ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p)); p->flags &= ~(F_INSLPQUEUE|F_TIMO); #ifdef ERTS_SMP - erts_cancel_smp_ptimer(p->u.alive.ptimer); + erts_cancel_smp_ptimer(p->common.u.alive.ptimer); #else - erts_cancel_timer(&p->u.alive.tm); + erts_cancel_timer(&p->common.u.alive.tm); #endif } @@ -9300,12 +8906,12 @@ set_timer(Process* p, Uint timeout) p->flags &= ~F_TIMO; #ifdef ERTS_SMP - erts_create_smp_ptimer(&p->u.alive.ptimer, - p->id, + erts_create_smp_ptimer(&p->common.u.alive.ptimer, + p->common.id, (ErlTimeoutProc) timeout_proc, timeout); #else - erts_set_timer(&p->u.alive.tm, + erts_set_timer(&p->common.u.alive.tm, (ErlTimeoutProc) timeout_proc, NULL, (void*) p, @@ -9323,7 +8929,7 @@ erts_stack_dump(int to, void *to_arg, Process *p) Eterm* sp; int yreg = -1; - if (p->trace_flags & F_SENSITIVE) { + if (ERTS_TRACE_FLAGS(p) & F_SENSITIVE) { return; } erts_program_counter_info(to, to_arg, p); @@ -9411,1068 +9017,6 @@ stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp, int yreg) return yreg; } -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ - * The processes/0 BIF implementation. * -\* */ - - -#define ERTS_PROCESSES_BIF_TAB_INSPECT_INDICES_PER_RED 25 -#define ERTS_PROCESSES_BIF_TAB_CHUNK_SIZE 1000 -#define ERTS_PROCESSES_BIF_MIN_START_REDS \ - (ERTS_PROCESSES_BIF_TAB_CHUNK_SIZE \ - / ERTS_PROCESSES_BIF_TAB_INSPECT_INDICES_PER_RED) - -#define ERTS_PROCESSES_BIF_TAB_FREE_TERM_PROC_REDS 1 - -#define ERTS_PROCESSES_BIF_INSPECT_TERM_PROC_PER_RED 10 - -#define ERTS_PROCESSES_INSPECT_TERM_PROC_MAX_REDS \ - (ERTS_PROCESSES_BIF_TAB_CHUNK_SIZE \ - / ERTS_PROCESSES_BIF_TAB_INSPECT_INDICES_PER_RED) - - -#define ERTS_PROCESSES_BIF_BUILD_RESULT_CONSES_PER_RED 75 - -#define ERTS_PROCS_DBG_DO_TRACE 0 - -#ifdef DEBUG -# define ERTS_PROCESSES_BIF_DEBUGLEVEL 100 -#else -# define ERTS_PROCESSES_BIF_DEBUGLEVEL 0 -#endif - -#define ERTS_PROCS_DBGLVL_CHK_HALLOC 1 -#define ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS 5 -#define ERTS_PROCS_DBGLVL_CHK_PIDS 10 -#define ERTS_PROCS_DBGLVL_CHK_TERM_PROC_LIST 20 -#define ERTS_PROCS_DBGLVL_CHK_RESLIST 20 - -#if ERTS_PROCESSES_BIF_DEBUGLEVEL == 0 -# define ERTS_PROCS_ASSERT(EXP) -#else -# define ERTS_PROCS_ASSERT(EXP) \ - ((void) ((EXP) \ - ? 1 \ - : (debug_processes_assert_error(#EXP, __FILE__, __LINE__), 0))) -#endif - - -#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_HALLOC -# define ERTS_PROCS_DBG_SAVE_HEAP_ALLOC(PBDP, HP, SZ) \ -do { \ - ERTS_PROCS_ASSERT(!(PBDP)->debug.heap); \ - ERTS_PROCS_ASSERT(!(PBDP)->debug.heap_size); \ - (PBDP)->debug.heap = (HP); \ - (PBDP)->debug.heap_size = (SZ); \ -} while (0) -# define ERTS_PROCS_DBG_VERIFY_HEAP_ALLOC_USED(PBDP, HP) \ -do { \ - ERTS_PROCS_ASSERT((PBDP)->debug.heap); \ - ERTS_PROCS_ASSERT((PBDP)->debug.heap_size); \ - ERTS_PROCS_ASSERT((PBDP)->debug.heap + (PBDP)->debug.heap_size == (HP));\ - (PBDP)->debug.heap = NULL; \ - (PBDP)->debug.heap_size = 0; \ -} while (0) -# define ERTS_PROCS_DBG_HEAP_ALLOC_INIT(PBDP) \ -do { \ - (PBDP)->debug.heap = NULL; \ - (PBDP)->debug.heap_size = 0; \ -} while (0) -#else -# define ERTS_PROCS_DBG_SAVE_HEAP_ALLOC(PBDP, HP, SZ) -# define ERTS_PROCS_DBG_VERIFY_HEAP_ALLOC_USED(PBDP, HP) -# define ERTS_PROCS_DBG_HEAP_ALLOC_INIT(PBDP) -#endif - -#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_RESLIST -# define ERTS_PROCS_DBG_CHK_RESLIST(R) debug_processes_check_res_list((R)) -#else -# define ERTS_PROCS_DBG_CHK_RESLIST(R) -#endif - -#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_PIDS -# define ERTS_PROCS_DBG_SAVE_PIDS(PBDP) debug_processes_save_all_pids((PBDP)) -# define ERTS_PROCS_DBG_VERIFY_PIDS(PBDP) \ -do { \ - if (!(PBDP)->debug.correct_pids_verified) \ - debug_processes_verify_all_pids((PBDP)); \ -} while (0) -# define ERTS_PROCS_DBG_CLEANUP_CHK_PIDS(PBDP) \ -do { \ - if ((PBDP)->debug.correct_pids) { \ - erts_free(ERTS_ALC_T_PROCS_PIDS, \ - (PBDP)->debug.correct_pids); \ - (PBDP)->debug.correct_pids = NULL; \ - } \ -} while(0) -# define ERTS_PROCS_DBG_CHK_PIDS_INIT(PBDP) \ -do { \ - (PBDP)->debug.correct_pids_verified = 0; \ - (PBDP)->debug.correct_pids = NULL; \ -} while (0) -#else -# define ERTS_PROCS_DBG_SAVE_PIDS(PBDP) -# define ERTS_PROCS_DBG_VERIFY_PIDS(PBDP) -# define ERTS_PROCS_DBG_CLEANUP_CHK_PIDS(PBDP) -# define ERTS_PROCS_DBG_CHK_PIDS_INIT(PBDP) -#endif - -#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS -# define ERTS_PROCS_DBG_CHK_PID_FOUND(PBDP, PID, IC) \ - debug_processes_check_found_pid((PBDP), (PID), (IC), 1) -# define ERTS_PROCS_DBG_CHK_PID_NOT_FOUND(PBDP, PID, IC) \ - debug_processes_check_found_pid((PBDP), (PID), (IC), 0) -#else -# define ERTS_PROCS_DBG_CHK_PID_FOUND(PBDP, PID, IC) -# define ERTS_PROCS_DBG_CHK_PID_NOT_FOUND(PBDP, PID, IC) -#endif - -#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_TERM_PROC_LIST -# define ERTS_PROCS_DBG_CHK_TPLIST() \ - debug_processes_check_term_proc_list() -# define ERTS_PROCS_DBG_CHK_FREELIST(FL) \ - debug_processes_check_term_proc_free_list(FL) -#else -# define ERTS_PROCS_DBG_CHK_TPLIST() -# define ERTS_PROCS_DBG_CHK_FREELIST(FL) -#endif - -#if ERTS_PROCESSES_BIF_DEBUGLEVEL == 0 -#if ERTS_PROCS_DBG_DO_TRACE -# define ERTS_PROCS_DBG_INIT(P, PBDP) (PBDP)->debug.caller = (P)->id -# else -# define ERTS_PROCS_DBG_INIT(P, PBDP) -# endif -# define ERTS_PROCS_DBG_CLEANUP(PBDP) -#else -# define ERTS_PROCS_DBG_INIT(P, PBDP) \ -do { \ - (PBDP)->debug.caller = (P)->id; \ - ERTS_PROCS_DBG_HEAP_ALLOC_INIT((PBDP)); \ - ERTS_PROCS_DBG_CHK_PIDS_INIT((PBDP)); \ -} while (0) -# define ERTS_PROCS_DBG_CLEANUP(PBDP) \ -do { \ - ERTS_PROCS_DBG_CLEANUP_CHK_PIDS((PBDP)); \ -} while (0) -#endif - -#if ERTS_PROCS_DBG_DO_TRACE -# define ERTS_PROCS_DBG_TRACE(PID, FUNC, WHAT) \ - erts_fprintf(stderr, "%T %s:%d:%s(): %s\n", \ - (PID), __FILE__, __LINE__, #FUNC, #WHAT) -#else -# define ERTS_PROCS_DBG_TRACE(PID, FUNC, WHAT) -#endif - -static Uint processes_bif_tab_chunks; -static Export processes_trap_export; - -typedef struct { - Uint64 interval; -} ErtsProcessesBifChunkInfo; - -typedef enum { - INITIALIZING, - INSPECTING_TABLE, - INSPECTING_TERMINATED_PROCESSES, - BUILDING_RESULT, - RETURN_RESULT -} ErtsProcessesBifState; - -typedef struct { - ErtsProcessesBifState state; - Eterm caller; - ErtsProcessesBifChunkInfo *chunk; - int tix; - int pid_ix; - int pid_sz; - Eterm *pid; - ErtsTermProcElement *bif_invocation; /* Only used when > 1 chunk */ - -#if ERTS_PROCESSES_BIF_DEBUGLEVEL != 0 || ERTS_PROCS_DBG_DO_TRACE - struct { - Eterm caller; -#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS - Uint64 *pid_started; -#endif -#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_HALLOC - Eterm *heap; - Uint heap_size; -#endif -#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_PIDS - int correct_pids_verified; - Eterm *correct_pids; -#endif - } debug; -#endif - -} ErtsProcessesBifData; - - -#if ERTS_PROCESSES_BIF_DEBUGLEVEL != 0 -static void debug_processes_assert_error(char* expr, char* file, int line); -#endif -#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_RESLIST -static void debug_processes_check_res_list(Eterm list); -#endif -#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_PIDS -static void debug_processes_save_all_pids(ErtsProcessesBifData *pbdp); -static void debug_processes_verify_all_pids(ErtsProcessesBifData *pbdp); -#endif -#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS -static void debug_processes_check_found_pid(ErtsProcessesBifData *pbdp, - Eterm pid, - Uint64 ic, - int pid_should_be_found); -#endif -#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_TERM_PROC_LIST -static void debug_processes_check_term_proc_list(void); -static void debug_processes_check_term_proc_free_list(ErtsTermProcElement *tpep); -#endif - -static void -save_terminating_process(Process *p) -{ - ErtsTermProcElement *tpep = erts_alloc(ERTS_ALC_T_PROCS_TPROC_EL, - sizeof(ErtsTermProcElement)); - ERTS_PROCS_ASSERT(saved_term_procs.start && saved_term_procs.end); - ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&erts_proc_tab_rwmtx)); - - ERTS_PROCS_DBG_CHK_TPLIST(); - - tpep->prev = saved_term_procs.end; - tpep->next = NULL; - tpep->ix = internal_pid_index(p->id); - tpep->u.process.pid = p->id; - tpep->u.process.spawned = p->started_interval; - tpep->u.process.exited = get_proc_interval(); - - saved_term_procs.end->next = tpep; - saved_term_procs.end = tpep; - - ERTS_PROCS_DBG_CHK_TPLIST(); - - ERTS_PROCS_ASSERT(tpep->prev->ix >= 0 - ? (tpep->u.process.exited - >= tpep->prev->u.process.exited) - : (tpep->u.process.exited - >= tpep->prev->u.bif_invocation.interval)); -} - -static void -cleanup_processes_bif_data(Binary *bp) -{ - ErtsProcessesBifData *pbdp = ERTS_MAGIC_BIN_DATA(bp); - - ERTS_PROCS_DBG_TRACE(pbdp->debug.caller, cleanup_processes_bif_data, call); - - if (pbdp->state != INITIALIZING) { - - if (pbdp->chunk) { - erts_free(ERTS_ALC_T_PROCS_CNKINF, pbdp->chunk); - pbdp->chunk = NULL; - } - if (pbdp->pid) { - erts_free(ERTS_ALC_T_PROCS_PIDS, pbdp->pid); - pbdp->pid = NULL; - } - -#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS - if (pbdp->debug.pid_started) { - erts_free(ERTS_ALC_T_PROCS_PIDS, pbdp->debug.pid_started); - pbdp->debug.pid_started = NULL; - } -#endif - - if (pbdp->bif_invocation) { - ErtsTermProcElement *tpep; - - erts_smp_rwmtx_rwlock(&erts_proc_tab_rwmtx); - - ERTS_PROCS_DBG_TRACE(pbdp->debug.caller, - cleanup_processes_bif_data, - term_proc_cleanup); - - tpep = pbdp->bif_invocation; - pbdp->bif_invocation = NULL; - - ERTS_PROCS_DBG_CHK_TPLIST(); - - if (tpep->prev) { - /* - * Only remove this bif invokation when we - * have preceding invokations. - */ - tpep->prev->next = tpep->next; - if (tpep->next) - tpep->next->prev = tpep->prev; - else { - /* - * At the time of writing this branch cannot be - * reached. I don't want to remove this code though - * since it may be possible to reach this line - * in the future if the cleanup order in - * erts_do_exit_process() is changed. The ASSERT(0) - * is only here to make us aware that the reorder - * has happened. /rickard - */ - ASSERT(0); - saved_term_procs.end = tpep->prev; - } - erts_free(ERTS_ALC_T_PROCS_TPROC_EL, tpep); - } - else { - /* - * Free all elements until next bif invokation - * is found. - */ - ERTS_PROCS_ASSERT(saved_term_procs.start == tpep); - do { - ErtsTermProcElement *ftpep = tpep; - tpep = tpep->next; - erts_free(ERTS_ALC_T_PROCS_TPROC_EL, ftpep); - } while (tpep && tpep->ix >= 0); - saved_term_procs.start = tpep; - if (tpep) - tpep->prev = NULL; - else - saved_term_procs.end = NULL; - } - - ERTS_PROCS_DBG_CHK_TPLIST(); - - erts_smp_rwmtx_rwunlock(&erts_proc_tab_rwmtx); - - } - } - - ERTS_PROCS_DBG_TRACE(pbdp->debug.caller, - cleanup_processes_bif_data, - return); - ERTS_PROCS_DBG_CLEANUP(pbdp); -} - -static int -processes_bif_engine(Process *p, Eterm *res_accp, Binary *mbp) -{ - ErtsProcessesBifData *pbdp = ERTS_MAGIC_BIN_DATA(mbp); - int have_reds; - int reds; - int locked = 0; - - do { - switch (pbdp->state) { - case INITIALIZING: - pbdp->chunk = erts_alloc(ERTS_ALC_T_PROCS_CNKINF, - (sizeof(ErtsProcessesBifChunkInfo) - * processes_bif_tab_chunks)); - pbdp->tix = 0; - pbdp->pid_ix = 0; - - erts_smp_rwmtx_rwlock(&erts_proc_tab_rwmtx); - locked = 1; - - ERTS_PROCS_DBG_TRACE(p->id, processes_bif_engine, init); - - pbdp->pid_sz = erts_process_count(); - pbdp->pid = erts_alloc(ERTS_ALC_T_PROCS_PIDS, - sizeof(Eterm)*pbdp->pid_sz); - -#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS - pbdp->debug.pid_started = erts_alloc(ERTS_ALC_T_PROCS_PIDS, - sizeof(Uint64)*pbdp->pid_sz); -#endif - - ERTS_PROCS_DBG_SAVE_PIDS(pbdp); - - if (processes_bif_tab_chunks == 1) - pbdp->bif_invocation = NULL; - else { - /* - * We will have to access the table multiple times - * releasing the table lock in between chunks. - */ - pbdp->bif_invocation = erts_alloc(ERTS_ALC_T_PROCS_TPROC_EL, - sizeof(ErtsTermProcElement)); - pbdp->bif_invocation->ix = -1; - pbdp->bif_invocation->u.bif_invocation.interval - = step_proc_interval(); - ERTS_PROCS_DBG_CHK_TPLIST(); - - pbdp->bif_invocation->next = NULL; - if (saved_term_procs.end) { - pbdp->bif_invocation->prev = saved_term_procs.end; - saved_term_procs.end->next = pbdp->bif_invocation; - ERTS_PROCS_ASSERT(saved_term_procs.start); - } - else { - pbdp->bif_invocation->prev = NULL; - saved_term_procs.start = pbdp->bif_invocation; - } - saved_term_procs.end = pbdp->bif_invocation; - - ERTS_PROCS_DBG_CHK_TPLIST(); - - } - - pbdp->state = INSPECTING_TABLE; - /* Fall through */ - - case INSPECTING_TABLE: { - int ix = pbdp->tix; - int indices = ERTS_PROCESSES_BIF_TAB_CHUNK_SIZE; - int cix = ix / ERTS_PROCESSES_BIF_TAB_CHUNK_SIZE; - int end_ix = ix + indices; - Uint64 *invocation_interval_p; - - invocation_interval_p - = (pbdp->bif_invocation - ? &pbdp->bif_invocation->u.bif_invocation.interval - : NULL); - - ERTS_PROCS_ASSERT(is_nil(*res_accp)); - if (!locked) { - erts_smp_rwmtx_rwlock(&erts_proc_tab_rwmtx); - locked = 1; - } - - ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&erts_proc_tab_rwmtx)); - ERTS_PROCS_DBG_TRACE(p->id, processes_bif_engine, insp_table); - - if (cix != 0) - pbdp->chunk[cix].interval = step_proc_interval(); - else if (pbdp->bif_invocation) - pbdp->chunk[0].interval = *invocation_interval_p; - /* else: interval is irrelevant */ - - if (end_ix >= erts_proc.max) { - ERTS_PROCS_ASSERT(cix+1 == processes_bif_tab_chunks); - end_ix = erts_proc.max; - indices = end_ix - ix; - /* What to do when done with this chunk */ - pbdp->state = (processes_bif_tab_chunks == 1 - ? BUILDING_RESULT - : INSPECTING_TERMINATED_PROCESSES); - } - - for (; ix < end_ix; ix++) { - Process *rp = erts_pix2proc(ix); - if (rp - && (!invocation_interval_p - || rp->started_interval < *invocation_interval_p)) { - ERTS_PROCS_ASSERT(is_internal_pid(rp->id)); - pbdp->pid[pbdp->pid_ix] = rp->id; - -#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS - pbdp->debug.pid_started[pbdp->pid_ix] = rp->started_interval; -#endif - - pbdp->pid_ix++; - ERTS_PROCS_ASSERT(pbdp->pid_ix <= pbdp->pid_sz); - } - } - - pbdp->tix = end_ix; - - erts_smp_rwmtx_rwunlock(&erts_proc_tab_rwmtx); - locked = 0; - - reds = indices/ERTS_PROCESSES_BIF_TAB_INSPECT_INDICES_PER_RED; - BUMP_REDS(p, reds); - - have_reds = ERTS_BIF_REDS_LEFT(p); - - if (have_reds && pbdp->state == INSPECTING_TABLE) { - ix = pbdp->tix; - indices = ERTS_PROCESSES_BIF_TAB_CHUNK_SIZE; - end_ix = ix + indices; - if (end_ix > erts_proc.max) { - end_ix = erts_proc.max; - indices = end_ix - ix; - } - - reds = indices/ERTS_PROCESSES_BIF_TAB_INSPECT_INDICES_PER_RED; - - /* Pretend we have no reds left if we haven't got enough - reductions to complete next chunk */ - if (reds > have_reds) - have_reds = 0; - } - - break; - } - - case INSPECTING_TERMINATED_PROCESSES: { - int i; - int max_reds; - int free_term_procs = 0; - Uint64 invocation_interval; - ErtsTermProcElement *tpep; - ErtsTermProcElement *free_list = NULL; - - tpep = pbdp->bif_invocation; - ERTS_PROCS_ASSERT(tpep); - invocation_interval = tpep->u.bif_invocation.interval; - - max_reds = have_reds = ERTS_BIF_REDS_LEFT(p); - if (max_reds > ERTS_PROCESSES_INSPECT_TERM_PROC_MAX_REDS) - max_reds = ERTS_PROCESSES_INSPECT_TERM_PROC_MAX_REDS; - - reds = 0; - erts_smp_rwmtx_rwlock(&erts_proc_tab_rwmtx); - ERTS_PROCS_DBG_TRACE(p->id, processes_bif_engine, insp_term_procs); - - ERTS_PROCS_DBG_CHK_TPLIST(); - - if (tpep->prev) - tpep->prev->next = tpep->next; - else { - ERTS_PROCS_ASSERT(saved_term_procs.start == tpep); - saved_term_procs.start = tpep->next; - - if (saved_term_procs.start && saved_term_procs.start->ix >= 0) { - free_list = saved_term_procs.start; - free_term_procs = 1; - } - } - - if (tpep->next) - tpep->next->prev = tpep->prev; - else - saved_term_procs.end = tpep->prev; - - tpep = tpep->next; - - i = 0; - while (reds < max_reds && tpep) { - if (tpep->ix < 0) { - if (free_term_procs) { - ERTS_PROCS_ASSERT(free_list); - ERTS_PROCS_ASSERT(tpep->prev); - - tpep->prev->next = NULL; /* end of free_list */ - saved_term_procs.start = tpep; - tpep->prev = NULL; - free_term_procs = 0; - } - } - else { - int cix = tpep->ix/ERTS_PROCESSES_BIF_TAB_CHUNK_SIZE; - Uint64 chunk_interval = pbdp->chunk[cix].interval; - Eterm pid = tpep->u.process.pid; - ERTS_PROCS_ASSERT(is_internal_pid(pid)); - - if (tpep->u.process.spawned < invocation_interval) { - if (tpep->u.process.exited < chunk_interval) { - ERTS_PROCS_DBG_CHK_PID_NOT_FOUND(pbdp, - pid, - tpep->u.process.spawned); - pbdp->pid[pbdp->pid_ix] = pid; -#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS - pbdp->debug.pid_started[pbdp->pid_ix] - = tpep->u.process.spawned; -#endif - pbdp->pid_ix++; - ERTS_PROCS_ASSERT(pbdp->pid_ix <= pbdp->pid_sz); - } - else { - ERTS_PROCS_DBG_CHK_PID_FOUND(pbdp, - pid, - tpep->u.process.spawned); - } - } - else { - ERTS_PROCS_DBG_CHK_PID_NOT_FOUND(pbdp, - pid, - tpep->u.process.spawned); - } - - i++; - if (i == ERTS_PROCESSES_BIF_INSPECT_TERM_PROC_PER_RED) { - reds++; - i = 0; - } - if (free_term_procs) - reds += ERTS_PROCESSES_BIF_TAB_FREE_TERM_PROC_REDS; - } - tpep = tpep->next; - } - - if (free_term_procs) { - ERTS_PROCS_ASSERT(free_list); - saved_term_procs.start = tpep; - if (!tpep) - saved_term_procs.end = NULL; - else { - ERTS_PROCS_ASSERT(tpep->prev); - tpep->prev->next = NULL; /* end of free_list */ - tpep->prev = NULL; - } - } - - if (!tpep) { - /* Done */ - ERTS_PROCS_ASSERT(pbdp->pid_ix == pbdp->pid_sz); - pbdp->state = BUILDING_RESULT; - pbdp->bif_invocation->next = free_list; - free_list = pbdp->bif_invocation; - pbdp->bif_invocation = NULL; - } - else { - /* Link in bif_invocation again where we left off */ - pbdp->bif_invocation->prev = tpep->prev; - pbdp->bif_invocation->next = tpep; - tpep->prev = pbdp->bif_invocation; - if (pbdp->bif_invocation->prev) - pbdp->bif_invocation->prev->next = pbdp->bif_invocation; - else { - ERTS_PROCS_ASSERT(saved_term_procs.start == tpep); - saved_term_procs.start = pbdp->bif_invocation; - } - } - - ERTS_PROCS_DBG_CHK_TPLIST(); - ERTS_PROCS_DBG_CHK_FREELIST(free_list); - erts_smp_rwmtx_rwunlock(&erts_proc_tab_rwmtx); - - /* - * We do the actual free of term proc structures now when we - * have released the table lock instead of when we encountered - * them. This since free() isn't for free and we don't want to - * unnecessarily block other schedulers. - */ - while (free_list) { - tpep = free_list; - free_list = tpep->next; - erts_free(ERTS_ALC_T_PROCS_TPROC_EL, tpep); - } - - have_reds -= reds; - if (have_reds < 0) - have_reds = 0; - BUMP_REDS(p, reds); - break; - } - - case BUILDING_RESULT: { - int conses, ix, min_ix; - Eterm *hp; - Eterm res = *res_accp; - - ERTS_PROCS_DBG_VERIFY_PIDS(pbdp); - ERTS_PROCS_DBG_CHK_RESLIST(res); - - ERTS_PROCS_DBG_TRACE(p->id, processes_bif_engine, begin_build_res); - - have_reds = ERTS_BIF_REDS_LEFT(p); - conses = ERTS_PROCESSES_BIF_BUILD_RESULT_CONSES_PER_RED*have_reds; - min_ix = pbdp->pid_ix - conses; - if (min_ix < 0) { - min_ix = 0; - conses = pbdp->pid_ix; - } - - hp = HAlloc(p, conses*2); - ERTS_PROCS_DBG_SAVE_HEAP_ALLOC(pbdp, hp, conses*2); - - for (ix = pbdp->pid_ix - 1; ix >= min_ix; ix--) { - ERTS_PROCS_ASSERT(is_internal_pid(pbdp->pid[ix])); - res = CONS(hp, pbdp->pid[ix], res); - hp += 2; - } - - ERTS_PROCS_DBG_VERIFY_HEAP_ALLOC_USED(pbdp, hp); - - pbdp->pid_ix = min_ix; - if (min_ix == 0) - pbdp->state = RETURN_RESULT; - else { - pbdp->pid_sz = min_ix; - pbdp->pid = erts_realloc(ERTS_ALC_T_PROCS_PIDS, - pbdp->pid, - sizeof(Eterm)*pbdp->pid_sz); -#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS - pbdp->debug.pid_started = erts_realloc(ERTS_ALC_T_PROCS_PIDS, - pbdp->debug.pid_started, - (sizeof(Uint64) - * pbdp->pid_sz)); -#endif - } - reds = conses/ERTS_PROCESSES_BIF_BUILD_RESULT_CONSES_PER_RED; - BUMP_REDS(p, reds); - have_reds -= reds; - - ERTS_PROCS_DBG_CHK_RESLIST(res); - ERTS_PROCS_DBG_TRACE(p->id, processes_bif_engine, end_build_res); - *res_accp = res; - break; - } - case RETURN_RESULT: - cleanup_processes_bif_data(mbp); - return 1; - - default: - erl_exit(ERTS_ABORT_EXIT, - "erlang:processes/0: Invalid state: %d\n", - (int) pbdp->state); - } - - - } while (have_reds || pbdp->state == RETURN_RESULT); - - return 0; -} - -/* - * processes_trap/2 is a hidden BIF that processes/0 traps to. - */ - -static BIF_RETTYPE processes_trap(BIF_ALIST_2) -{ - Eterm res_acc; - Binary *mbp; - - /* - * This bif cannot be called from erlang code. It can only be - * trapped to from processes/0; therefore, a bad argument - * is a processes/0 internal error. - */ - - ERTS_PROCS_DBG_TRACE(BIF_P->id, processes_trap, call); - ERTS_PROCS_ASSERT(is_nil(BIF_ARG_1) || is_list(BIF_ARG_1)); - - res_acc = BIF_ARG_1; - - ERTS_PROCS_ASSERT(ERTS_TERM_IS_MAGIC_BINARY(BIF_ARG_2)); - - mbp = ((ProcBin *) binary_val(BIF_ARG_2))->val; - - ERTS_PROCS_ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(mbp) - == cleanup_processes_bif_data); - ERTS_PROCS_ASSERT( - ((ErtsProcessesBifData *) ERTS_MAGIC_BIN_DATA(mbp))->debug.caller - == BIF_P->id); - - if (processes_bif_engine(BIF_P, &res_acc, mbp)) { - ERTS_PROCS_DBG_TRACE(BIF_P->id, processes_trap, return); - BIF_RET(res_acc); - } - else { - ERTS_PROCS_DBG_TRACE(BIF_P->id, processes_trap, trap); - ERTS_BIF_YIELD2(&processes_trap_export, BIF_P, res_acc, BIF_ARG_2); - } -} - - - -/* - * The actual processes/0 BIF. - */ - -BIF_RETTYPE processes_0(BIF_ALIST_0) -{ - /* - * A requirement: The list of pids returned should be a consistent - * snapshot of all processes existing at some point - * in time during the execution of processes/0. Since - * processes might terminate while processes/0 is - * executing, we have to keep track of terminated - * processes and add them to the result. We also - * ignore processes created after processes/0 has - * begun executing. - */ - Eterm res_acc = NIL; - Binary *mbp = erts_create_magic_binary(sizeof(ErtsProcessesBifData), - cleanup_processes_bif_data); - ErtsProcessesBifData *pbdp = ERTS_MAGIC_BIN_DATA(mbp); - - ERTS_PROCS_DBG_TRACE(BIF_P->id, processes_0, call); - pbdp->state = INITIALIZING; - ERTS_PROCS_DBG_INIT(BIF_P, pbdp); - - if (ERTS_BIF_REDS_LEFT(BIF_P) >= ERTS_PROCESSES_BIF_MIN_START_REDS - && processes_bif_engine(BIF_P, &res_acc, mbp)) { - erts_bin_free(mbp); - ERTS_PROCS_DBG_CHK_RESLIST(res_acc); - ERTS_PROCS_DBG_TRACE(BIF_P->id, processes_0, return); - BIF_RET(res_acc); - } - else { - Eterm *hp; - Eterm magic_bin; - ERTS_PROCS_DBG_CHK_RESLIST(res_acc); - hp = HAlloc(BIF_P, PROC_BIN_SIZE); - ERTS_PROCS_DBG_SAVE_HEAP_ALLOC(pbdp, hp, PROC_BIN_SIZE); - magic_bin = erts_mk_magic_binary_term(&hp, &MSO(BIF_P), mbp); - ERTS_PROCS_DBG_VERIFY_HEAP_ALLOC_USED(pbdp, hp); - ERTS_PROCS_DBG_TRACE(BIF_P->id, processes_0, trap); - ERTS_BIF_YIELD2(&processes_trap_export, BIF_P, res_acc, magic_bin); - } -} - -static void -init_processes_bif(void) -{ - saved_term_procs.start = NULL; - saved_term_procs.end = NULL; - processes_bif_tab_chunks = (((erts_proc.max - 1) - / ERTS_PROCESSES_BIF_TAB_CHUNK_SIZE) - + 1); - - /* processes_trap/2 is a hidden BIF that the processes/0 BIF traps to. */ - erts_init_trap_export(&processes_trap_export, am_erlang, am_processes_trap, 2, - &processes_trap); - -} - -/* - * Debug stuff - */ - -#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) -int -erts_dbg_check_halloc_lock(Process *p) -{ - if (ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p)) - return 1; - if (p->id == ERTS_INVALID_PID) - return 1; - if (p->scheduler_data && p == p->scheduler_data->match_pseudo_process) - return 1; - if (erts_thr_progress_is_blocking()) - return 1; - return 0; -} -#endif - -Eterm -erts_debug_processes(Process *c_p) -{ - /* This is the old processes/0 BIF. */ - int i; - Uint need; - Eterm res; - Eterm* hp; - Process *p; - Eterm *hp_end; - - erts_smp_rwmtx_rwlock(&erts_proc_tab_rwmtx); - - res = NIL; - need = erts_process_count() * 2; - hp = HAlloc(c_p, need); /* we need two heap words for each pid */ - hp_end = hp + need; - - /* make the list by scanning bakward */ - - - for (i = erts_proc.max-1; i >= 0; i--) { - p = erts_pix2proc(i); - if (p) { - res = CONS(hp, p->id, res); - hp += 2; - } - } - - erts_smp_rwmtx_rwunlock(&erts_proc_tab_rwmtx); - - HRelease(c_p, hp_end, hp); - - return res; -} - -Eterm -erts_debug_processes_bif_info(Process *c_p) -{ - ERTS_DECL_AM(processes_bif_info); - Eterm elements[] = { - AM_processes_bif_info, - make_small((Uint) ERTS_PROCESSES_BIF_MIN_START_REDS), - make_small((Uint) processes_bif_tab_chunks), - make_small((Uint) ERTS_PROCESSES_BIF_TAB_CHUNK_SIZE), - make_small((Uint) ERTS_PROCESSES_BIF_TAB_INSPECT_INDICES_PER_RED), - make_small((Uint) ERTS_PROCESSES_BIF_TAB_FREE_TERM_PROC_REDS), - make_small((Uint) ERTS_PROCESSES_BIF_INSPECT_TERM_PROC_PER_RED), - make_small((Uint) ERTS_PROCESSES_INSPECT_TERM_PROC_MAX_REDS), - make_small((Uint) ERTS_PROCESSES_BIF_BUILD_RESULT_CONSES_PER_RED), - make_small((Uint) ERTS_PROCESSES_BIF_DEBUGLEVEL) - }; - Uint sz = 0; - Eterm *hp; - (void) erts_bld_tuplev(NULL, &sz, sizeof(elements)/sizeof(Eterm), elements); - hp = HAlloc(c_p, sz); - return erts_bld_tuplev(&hp, NULL, sizeof(elements)/sizeof(Eterm), elements); -} - -#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS -static void -debug_processes_check_found_pid(ErtsProcessesBifData *pbdp, - Eterm pid, - Uint64 ic, - int pid_should_be_found) -{ - int i; - for (i = 0; i < pbdp->pid_ix; i++) { - if (pbdp->pid[i] == pid && pbdp->debug.pid_started[i] == ic) { - ERTS_PROCS_ASSERT(pid_should_be_found); - return; - } - } - ERTS_PROCS_ASSERT(!pid_should_be_found); -} -#endif - -#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_RESLIST -static void -debug_processes_check_res_list(Eterm list) -{ - while (is_list(list)) { - Eterm* consp = list_val(list); - Eterm hd = CAR(consp); - ERTS_PROCS_ASSERT(is_internal_pid(hd)); - list = CDR(consp); - } - - ERTS_PROCS_ASSERT(is_nil(list)); -} -#endif - -#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_PIDS - -static void -debug_processes_save_all_pids(ErtsProcessesBifData *pbdp) -{ - int ix, tix, cpix; - pbdp->debug.correct_pids_verified = 0; - pbdp->debug.correct_pids = erts_alloc(ERTS_ALC_T_PROCS_PIDS, - sizeof(Eterm)*pbdp->pid_sz); - - for (tix = 0, cpix = 0; tix < erts_proc.max; tix++) { - Process *rp = erts_pix2proc(tix); - if (rp) { - ERTS_PROCS_ASSERT(is_internal_pid(rp->id)); - pbdp->debug.correct_pids[cpix++] = rp->id; - ERTS_PROCS_ASSERT(cpix <= pbdp->pid_sz); - } - } - ERTS_PROCS_ASSERT(cpix == pbdp->pid_sz); - - for (ix = 0; ix < pbdp->pid_sz; ix++) - pbdp->pid[ix] = make_small(ix); -} - -static void -debug_processes_verify_all_pids(ErtsProcessesBifData *pbdp) -{ - int ix, cpix; - - ERTS_PROCS_ASSERT(pbdp->pid_ix == pbdp->pid_sz); - - for (ix = 0; ix < pbdp->pid_sz; ix++) { - int found = 0; - Eterm pid = pbdp->pid[ix]; - ERTS_PROCS_ASSERT(is_internal_pid(pid)); - for (cpix = ix; cpix < pbdp->pid_sz; cpix++) { - if (pbdp->debug.correct_pids[cpix] == pid) { - pbdp->debug.correct_pids[cpix] = NIL; - found = 1; - break; - } - } - if (!found) { - for (cpix = 0; cpix < ix; cpix++) { - if (pbdp->debug.correct_pids[cpix] == pid) { - pbdp->debug.correct_pids[cpix] = NIL; - found = 1; - break; - } - } - } - ERTS_PROCS_ASSERT(found); - } - pbdp->debug.correct_pids_verified = 1; - - erts_free(ERTS_ALC_T_PROCS_PIDS, pbdp->debug.correct_pids); - pbdp->debug.correct_pids = NULL; -} -#endif /* ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_PIDS */ - -#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_TERM_PROC_LIST -static void -debug_processes_check_term_proc_list(void) -{ - ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&erts_proc_tab_rwmtx)); - if (!saved_term_procs.start) - ERTS_PROCS_ASSERT(!saved_term_procs.end); - else { - Uint64 curr_interval = get_proc_interval(); - Uint64 *prev_x_interval_p = NULL; - ErtsTermProcElement *tpep; - - for (tpep = saved_term_procs.start; tpep; tpep = tpep->next) { - if (!tpep->prev) - ERTS_PROCS_ASSERT(saved_term_procs.start == tpep); - else - ERTS_PROCS_ASSERT(tpep->prev->next == tpep); - if (!tpep->next) - ERTS_PROCS_ASSERT(saved_term_procs.end == tpep); - else - ERTS_PROCS_ASSERT(tpep->next->prev == tpep); - if (tpep->ix < 0) { - Uint64 interval = tpep->u.bif_invocation.interval; - ERTS_PROCS_ASSERT(interval <= curr_interval); - } - else { - Uint64 s_interval = tpep->u.process.spawned; - Uint64 x_interval = tpep->u.process.exited; - - ERTS_PROCS_ASSERT(s_interval <= x_interval); - if (prev_x_interval_p) - ERTS_PROCS_ASSERT(*prev_x_interval_p <= x_interval); - prev_x_interval_p = &tpep->u.process.exited; - ERTS_PROCS_ASSERT(is_internal_pid(tpep->u.process.pid)); - ERTS_PROCS_ASSERT(tpep->ix - == internal_pid_index(tpep->u.process.pid)); - } - } - - } -} - -static void -debug_processes_check_term_proc_free_list(ErtsTermProcElement *free_list) -{ - if (saved_term_procs.start) { - ErtsTermProcElement *ftpep; - ErtsTermProcElement *tpep; - - for (ftpep = free_list; ftpep; ftpep = ftpep->next) { - for (tpep = saved_term_procs.start; tpep; tpep = tpep->next) - ERTS_PROCS_ASSERT(ftpep != tpep); - } - } -} - -#endif - -#if ERTS_PROCESSES_BIF_DEBUGLEVEL != 0 - -static void -debug_processes_assert_error(char* expr, char* file, int line) -{ - fflush(stdout); - erts_fprintf(stderr, "%s:%d: Assertion failed: %s\n", file, line, expr); - fflush(stderr); - abort(); -} - -#endif - -/* *\ - * End of the processes/0 BIF implementation. * -\* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - /* * A nice system halt closing all open port goes as follows: * 1) This function schedules the aux work ERTS_SSI_AUX_WORK_REAP_PORTS @@ -10499,3 +9043,19 @@ void erl_halt(int code) notify_reap_ports_relb(); } } + +#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) +int +erts_dbg_check_halloc_lock(Process *p) +{ + if (ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p)) + return 1; + if (p->common.id == ERTS_INVALID_PID) + return 1; + if (p->scheduler_data && p == p->scheduler_data->match_pseudo_process) + return 1; + if (erts_thr_progress_is_blocking()) + return 1; + return 0; +} +#endif |