diff options
Diffstat (limited to 'erts/emulator/beam/erl_process.c')
| -rw-r--r-- | erts/emulator/beam/erl_process.c | 9469 |
1 files changed, 9469 insertions, 0 deletions
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c new file mode 100644 index 0000000000..9960172366 --- /dev/null +++ b/erts/emulator/beam/erl_process.c @@ -0,0 +1,9469 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 1996-2009. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +#define ERL_PROCESS_C__ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include <stddef.h> /* offsetof() */ +#include <ctype.h> +#include "sys.h" +#include "erl_vm.h" +#include "global.h" +#include "erl_process.h" +#include "erl_nmgc.h" +#include "error.h" +#include "bif.h" +#include "erl_db.h" +#include "dist.h" +#include "beam_catches.h" +#include "erl_instrument.h" +#include "erl_threads.h" +#include "erl_binary.h" + +#define ERTS_RUNQ_CHECK_BALANCE_REDS_PER_SCHED (2000*CONTEXT_REDS) +#define ERTS_RUNQ_CALL_CHECK_BALANCE_REDS \ + (ERTS_RUNQ_CHECK_BALANCE_REDS_PER_SCHED/2) + +#define ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST (CONTEXT_REDS/10) + +#define ERTS_SCHED_SLEEP_SPINCOUNT 10000 + +#define ERTS_WAKEUP_OTHER_LIMIT (100*CONTEXT_REDS/2) +#define ERTS_WAKEUP_OTHER_DEC 10 +#define ERTS_WAKEUP_OTHER_FIXED_INC (CONTEXT_REDS/10) + +#define ERTS_MAX_CPU_TOPOLOGY_ID ((int) 0xffff) + +#if 0 || defined(DEBUG) +#define ERTS_FAKE_SCHED_BIND_PRINT_SORTED_CPU_DATA +#endif + +#if defined(DEBUG) && 0 +#define HARDDEBUG +#else +#undef HARDDEBUG +#endif + +#ifdef HARDDEBUG +#define HARDDEBUG_RUNQS +#endif + +#ifdef HIPE +#include "hipe_mode_switch.h" /* for hipe_init_process() */ +#include "hipe_signal.h" /* for hipe_thread_signal_init() */ +#endif + +#ifdef ERTS_ENABLE_LOCK_COUNT +#include "erl_lock_count.h" +#endif + +#define MAX_BIT (1 << PRIORITY_MAX) +#define HIGH_BIT (1 << PRIORITY_HIGH) +#define NORMAL_BIT (1 << PRIORITY_NORMAL) +#define LOW_BIT (1 << PRIORITY_LOW) + +#define ERTS_MAYBE_SAVE_TERMINATING_PROCESS(P) \ +do { \ + ERTS_SMP_LC_ASSERT(erts_lc_mtx_is_locked(&proc_tab_mtx)); \ + if (saved_term_procs.end) \ + save_terminating_process((P)); \ +} while (0) + +#define ERTS_EMPTY_RUNQ(RQ) \ + ((RQ)->len == 0 && (RQ)->misc.start == NULL) + +extern Eterm beam_apply[]; +extern Eterm beam_exit[]; +extern Eterm beam_continue_exit[]; + +static Sint p_last; +static Sint p_next; +static Sint p_serial; +static Uint p_serial_mask; +static Uint p_serial_shift; + +Uint erts_no_schedulers; +Uint erts_max_processes = ERTS_DEFAULT_MAX_PROCESSES; +Uint erts_process_tab_index_mask; + +int erts_sched_thread_suggested_stack_size = -1; + +#ifdef ERTS_ENABLE_LOCK_CHECK +ErtsLcPSDLocks erts_psd_required_locks[ERTS_PSD_SIZE]; +#endif + +#ifdef ERTS_SMP + +int erts_disable_proc_not_running_opt; + +#define ERTS_SCHED_CHANGING_ONLINE 1 +#define ERTS_SCHED_CHANGING_MULTI_SCHED 2 + +static struct { + erts_smp_mtx_t mtx; + erts_smp_cnd_t cnd; + int changing; + int online; + int curr_online; + int wait_curr_online; + erts_smp_atomic_t active; + struct { + erts_smp_atomic_t ongoing; + long wait_active; + ErtsProcList *procs; + } msb; /* Multi Scheduling Block */ +} schdlr_sspnd; + +static struct { + erts_smp_mtx_t update_mtx; + erts_smp_atomic_t active_runqs; + int last_active_runqs; + erts_smp_atomic_t used_runqs; + int forced_check_balance; + erts_smp_atomic_t checking_balance; + int halftime; + int full_reds_history_index; + struct { + int active_runqs; + int reds; + int max_len; + } prev_rise; + Uint n; +} balance_info; + +#define ERTS_BLNCE_SAVE_RISE(ACTIVE, MAX_LEN, REDS) \ +do { \ + balance_info.prev_rise.active_runqs = (ACTIVE); \ + balance_info.prev_rise.max_len = (MAX_LEN); \ + balance_info.prev_rise.reds = (REDS); \ +} while (0) + +#endif + +/* + * Cpu topology hierarchy. + */ +#define ERTS_TOPOLOGY_NODE 0 +#define ERTS_TOPOLOGY_PROCESSOR 1 +#define ERTS_TOPOLOGY_PROCESSOR_NODE 2 +#define ERTS_TOPOLOGY_CORE 3 +#define ERTS_TOPOLOGY_THREAD 4 +#define ERTS_TOPOLOGY_LOGICAL 5 + +#define ERTS_TOPOLOGY_MAX_DEPTH 6 + +typedef struct { + int bind_id; + int bound_id; +} ErtsCpuBindData; + +static ErtsCpuBindData *scheduler2cpu_map; +erts_smp_rwmtx_t erts_cpu_bind_rwmtx; + +typedef enum { + ERTS_CPU_BIND_SPREAD, + ERTS_CPU_BIND_PROCESSOR_SPREAD, + ERTS_CPU_BIND_THREAD_SPREAD, + ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD, + ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD, + ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD, + ERTS_CPU_BIND_NO_SPREAD, + ERTS_CPU_BIND_NONE +} ErtsCpuBindOrder; + +ErtsCpuBindOrder cpu_bind_order; + +static erts_cpu_topology_t *user_cpudata; +static int user_cpudata_size; +static erts_cpu_topology_t *system_cpudata; +static int system_cpudata_size; + +erts_sched_stat_t erts_sched_stat; + +ErtsRunQueue *erts_common_run_queue; + +#ifdef USE_THREADS +static erts_tsd_key_t sched_data_key; +#endif + +static erts_smp_mtx_t proc_tab_mtx; + +static erts_smp_atomic_t function_calls; + +#ifdef ERTS_SMP +static erts_smp_atomic_t doing_sys_schedule; +static erts_smp_atomic_t no_empty_run_queues; +#else /* !ERTS_SMP */ +ErtsSchedulerData *erts_scheduler_data; +#endif + +ErtsAlignedRunQueue *erts_aligned_run_queues; +Uint erts_no_run_queues; + +typedef struct { + ErtsSchedulerData esd; + char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsSchedulerData))]; +} ErtsAlignedSchedulerData; + +ErtsAlignedSchedulerData *erts_aligned_scheduler_data; + +#ifndef BM_COUNTERS +static int processes_busy; +#endif + +Process** process_tab; +static Uint last_reductions; +static Uint last_exact_reductions; +Uint erts_default_process_flags; +Eterm erts_system_monitor; +Eterm erts_system_monitor_msg_queue_len; +Eterm erts_system_monitor_long_gc; +Eterm erts_system_monitor_large_heap; +struct erts_system_monitor_flags_t erts_system_monitor_flags; + +/* system performance monitor */ +Eterm erts_system_profile; +struct erts_system_profile_flags_t erts_system_profile_flags; + +#ifdef HYBRID +Uint erts_num_active_procs; +Process** erts_active_procs; +#endif + +static erts_smp_atomic_t process_count; + +typedef struct ErtsTermProcElement_ ErtsTermProcElement; +struct ErtsTermProcElement_ { + ErtsTermProcElement *next; + ErtsTermProcElement *prev; + int ix; + union { + struct { + Eterm pid; + SysTimeval spawned; + SysTimeval exited; + } process; + struct { + SysTimeval time; + } bif_invocation; + } u; +}; + +static struct { + ErtsTermProcElement *start; + ErtsTermProcElement *end; +} saved_term_procs; + +ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(misc_op_list, + ErtsMiscOpList, + 10, + ERTS_ALC_T_MISC_OP_LIST) + +ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(proclist, + ErtsProcList, + 200, + ERTS_ALC_T_PROC_LIST) + +#define ERTS_RUNQ_IX(IX) (&erts_aligned_run_queues[(IX)].runq) +#define ERTS_SCHEDULER_IX(IX) (&erts_aligned_scheduler_data[(IX)].esd) + +#define ERTS_FOREACH_RUNQ(RQVAR, DO) \ +do { \ + ErtsRunQueue *RQVAR; \ + int ix__; \ + for (ix__ = 0; ix__ < erts_no_run_queues; ix__++) { \ + RQVAR = ERTS_RUNQ_IX(ix__); \ + erts_smp_runq_lock(RQVAR); \ + { DO; } \ + erts_smp_runq_unlock(RQVAR); \ + } \ +} while (0) + +#define ERTS_FOREACH_OP_RUNQ(RQVAR, DO) \ +do { \ + ErtsRunQueue *RQVAR; \ + int ix__; \ + ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&schdlr_sspnd.mtx)); \ + for (ix__ = 0; ix__ < schdlr_sspnd.online; ix__++) { \ + RQVAR = ERTS_RUNQ_IX(ix__); \ + erts_smp_runq_lock(RQVAR); \ + { DO; } \ + erts_smp_runq_unlock(RQVAR); \ + } \ +} while (0) + +#define ERTS_ATOMIC_FOREACH_RUNQ_X(RQVAR, DO, DOX) \ +do { \ + ErtsRunQueue *RQVAR; \ + int ix__; \ + for (ix__ = 0; ix__ < erts_no_run_queues; ix__++) { \ + RQVAR = ERTS_RUNQ_IX(ix__); \ + erts_smp_runq_lock(RQVAR); \ + { DO; } \ + } \ + { DOX; } \ + for (ix__ = 0; ix__ < erts_no_run_queues; ix__++) \ + erts_smp_runq_unlock(ERTS_RUNQ_IX(ix__)); \ +} while (0) + +#define ERTS_ATOMIC_FOREACH_RUNQ(RQVAR, DO) \ + ERTS_ATOMIC_FOREACH_RUNQ_X(RQVAR, DO, ) +/* + * Local functions. + */ + +static void init_processes_bif(void); +static void save_terminating_process(Process *p); +static void exec_misc_ops(ErtsRunQueue *); +static void print_function_from_pc(int to, void *to_arg, Eterm* x); +static int stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp, + int yreg); +#ifdef ERTS_SMP +static void handle_pending_exiters(ErtsProcList *); + +static void cpu_bind_order_sort(erts_cpu_topology_t *cpudata, + int size, + ErtsCpuBindOrder bind_order, + int mk_seq); +static void signal_schedulers_bind_change(erts_cpu_topology_t *cpudata, int size); + +#endif + +static void early_cpu_bind_init(void); +static void late_cpu_bind_init(void); + +#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) +int +erts_smp_lc_runq_is_locked(ErtsRunQueue *runq) +{ + return erts_smp_lc_mtx_is_locked(&runq->mtx); +} +#endif + +void +erts_pre_init_process(void) +{ +#ifdef USE_THREADS + erts_tsd_key_create(&sched_data_key); +#endif + +#ifdef ERTS_ENABLE_LOCK_CHECK + { + int ix; + + erts_psd_required_locks[ERTS_PSD_ERROR_HANDLER].get_locks + = ERTS_PSD_ERROR_HANDLER_BUF_GET_LOCKS; + erts_psd_required_locks[ERTS_PSD_ERROR_HANDLER].set_locks + = ERTS_PSD_ERROR_HANDLER_BUF_SET_LOCKS; + + erts_psd_required_locks[ERTS_PSD_SAVED_CALLS_BUF].get_locks + = ERTS_PSD_SAVED_CALLS_BUF_GET_LOCKS; + erts_psd_required_locks[ERTS_PSD_SAVED_CALLS_BUF].set_locks + = ERTS_PSD_SAVED_CALLS_BUF_SET_LOCKS; + + erts_psd_required_locks[ERTS_PSD_SCHED_ID].get_locks + = ERTS_PSD_SCHED_ID_GET_LOCKS; + erts_psd_required_locks[ERTS_PSD_SCHED_ID].set_locks + = ERTS_PSD_SCHED_ID_SET_LOCKS; + + erts_psd_required_locks[ERTS_PSD_DIST_ENTRY].get_locks + = ERTS_PSD_DIST_ENTRY_GET_LOCKS; + erts_psd_required_locks[ERTS_PSD_DIST_ENTRY].set_locks + = ERTS_PSD_DIST_ENTRY_GET_LOCKS; + + /* Check that we have locks for all entries */ + for (ix = 0; ix < ERTS_PSD_SIZE; ix++) { + ERTS_SMP_LC_ASSERT(erts_psd_required_locks[ix].get_locks); + ERTS_SMP_LC_ASSERT(erts_psd_required_locks[ix].set_locks); + } + } +#endif +} + +/* initialize the scheduler */ +void +erts_init_process(void) +{ + Uint proc_bits = ERTS_PROC_BITS; + +#ifdef ERTS_SMP + erts_disable_proc_not_running_opt = 0; + erts_init_proc_lock(); +#endif + + init_proclist_alloc(); + + erts_smp_atomic_init(&process_count, 0); + + if (erts_use_r9_pids_ports) { + proc_bits = ERTS_R9_PROC_BITS; + ASSERT(erts_max_processes <= (1 << ERTS_R9_PROC_BITS)); + } + + process_tab = (Process**) erts_alloc(ERTS_ALC_T_PROC_TABLE, + erts_max_processes*sizeof(Process*)); + sys_memzero(process_tab, erts_max_processes * sizeof(Process*)); +#ifdef HYBRID + erts_active_procs = (Process**) + erts_alloc(ERTS_ALC_T_ACTIVE_PROCS, + erts_max_processes * sizeof(Process*)); + erts_num_active_procs = 0; +#endif + + erts_smp_mtx_init(&proc_tab_mtx, "proc_tab"); + p_last = -1; + p_next = 0; + p_serial = 0; + + p_serial_shift = erts_fit_in_bits(erts_max_processes - 1); + p_serial_mask = ((~(~((Uint) 0) << proc_bits)) >> p_serial_shift); + erts_process_tab_index_mask = ~(~((Uint) 0) << p_serial_shift); +#ifndef BM_COUNTERS + processes_busy = 0; +#endif + last_reductions = 0; + last_exact_reductions = 0; + erts_default_process_flags = 0; +} + +void +erts_late_init_process(void) +{ + int ix; + init_processes_bif(); + + erts_smp_spinlock_init(&erts_sched_stat.lock, "sched_stat"); + for (ix = 0; ix < ERTS_NO_PRIO_LEVELS; ix++) { + Eterm atom; + char *atom_str; + switch (ix) { + case PRIORITY_MAX: + atom_str = "process_max"; + break; + case PRIORITY_HIGH: + atom_str = "process_high"; + break; + case PRIORITY_NORMAL: + atom_str = "process_normal"; + break; + case PRIORITY_LOW: + atom_str = "process_low"; + break; + case ERTS_PORT_PRIO_LEVEL: + atom_str = "port"; + break; + default: + atom_str = "bad_prio"; + ASSERT(!"bad prio"); + break; + } + atom = am_atom_put(atom_str, sys_strlen(atom_str)); + erts_sched_stat.prio[ix].name = atom; + erts_sched_stat.prio[ix].total_executed = 0; + erts_sched_stat.prio[ix].executed = 0; + erts_sched_stat.prio[ix].total_migrated = 0; + erts_sched_stat.prio[ix].migrated = 0; + } + +} + +static ERTS_INLINE ErtsProcList * +proclist_create(Process *p) +{ + ErtsProcList *plp = proclist_alloc(); + plp->pid = p->id; + plp->started = p->started; + return plp; +} + +static ERTS_INLINE void +proclist_destroy(ErtsProcList *plp) +{ + proclist_free(plp); +} + +static ERTS_INLINE int +proclist_same(ErtsProcList *plp, Process *p) +{ + return (plp->pid == p->id + && erts_cmp_timeval(&plp->started, &p->started) == 0); +} + +ErtsProcList * +erts_proclist_create(Process *p) +{ + return proclist_create(p); +} + +void +erts_proclist_destroy(ErtsProcList *plp) +{ + proclist_destroy(plp); +} + +int +erts_proclist_same(ErtsProcList *plp, Process *p) +{ + return proclist_same(plp, p); +} + +void * +erts_psd_set_init(Process *p, ErtsProcLocks plocks, int ix, void *data) +{ + void *old; + ErtsProcLocks xplocks; + int refc = 0; + ErtsPSD *psd = erts_alloc(ERTS_ALC_T_PSD, sizeof(ErtsPSD)); + int i; + for (i = 0; i < ERTS_PSD_SIZE; i++) + psd->data[i] = NULL; + + ERTS_SMP_LC_ASSERT(plocks); + ERTS_SMP_LC_ASSERT(plocks == erts_proc_lc_my_proc_locks(p)); + + xplocks = ERTS_PROC_LOCKS_ALL; + xplocks &= ~plocks; + if (xplocks && erts_smp_proc_trylock(p, xplocks) == EBUSY) { + if (xplocks & ERTS_PROC_LOCK_MAIN) { + erts_smp_proc_inc_refc(p); + erts_smp_proc_unlock(p, plocks); + erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL); + refc = 1; + } + else { + if (plocks & ERTS_PROC_LOCKS_ALL_MINOR) + erts_smp_proc_unlock(p, plocks & ERTS_PROC_LOCKS_ALL_MINOR); + erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR); + } + } + if (!p->psd) + p->psd = psd; + if (xplocks) + erts_smp_proc_unlock(p, xplocks); + if (refc) + erts_smp_proc_dec_refc(p); + ASSERT(p->psd); + if (p->psd != psd) + erts_free(ERTS_ALC_T_PSD, psd); + old = p->psd->data[ix]; + p->psd->data[ix] = data; + ERTS_SMP_LC_ASSERT(plocks == erts_proc_lc_my_proc_locks(p)); + return old; +} + +#ifdef ERTS_SMP + +static void +prepare_for_block(void *vrq) +{ + erts_smp_runq_unlock((ErtsRunQueue *) vrq); +} + +static void +resume_after_block(void *vrq) +{ + erts_smp_runq_lock((ErtsRunQueue *) vrq); +} + +#endif + +static ERTS_INLINE void +sched_waiting_sys(Uint no, ErtsRunQueue *rq) +{ + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); + ASSERT(rq->waiting >= 0); + rq->flags |= (ERTS_RUNQ_FLG_OUT_OF_WORK + | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK); + rq->waiting++; + rq->waiting *= -1; + rq->woken = 0; + if (erts_system_profile_flags.scheduler) + profile_scheduler(make_small(no), am_inactive); +} + +static ERTS_INLINE void +sched_active_sys(Uint no, ErtsRunQueue *rq) +{ + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); + ASSERT(rq->waiting < 0); + rq->waiting *= -1; + rq->waiting--; + if (erts_system_profile_flags.scheduler) + profile_scheduler(make_small(no), am_active); +} + +Uint +erts_active_schedulers(void) +{ + /* RRRRRRRRR */ + + Uint as = erts_no_schedulers; + + ERTS_ATOMIC_FOREACH_RUNQ(rq, as -= abs(rq->waiting)); + + ASSERT(as >= 0); + return as; +} + +#ifdef ERTS_SMP + +static ERTS_INLINE void +sched_waiting(Uint no, ErtsRunQueue *rq) +{ + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); + rq->flags |= (ERTS_RUNQ_FLG_OUT_OF_WORK + | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK); + if (rq->waiting < 0) + rq->waiting--; + else + rq->waiting++; + rq->woken = 0; + if (erts_system_profile_flags.scheduler) + profile_scheduler(make_small(no), am_inactive); +} + +static ERTS_INLINE void +sched_active(Uint no, ErtsRunQueue *rq) +{ + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); + if (rq->waiting < 0) + rq->waiting++; + else + rq->waiting--; + if (erts_system_profile_flags.scheduler) + profile_scheduler(make_small(no), am_active); +} + +static int ERTS_INLINE +ongoing_multi_scheduling_block(void) +{ + return erts_smp_atomic_read(&schdlr_sspnd.msb.ongoing) != 0; +} + +static ERTS_INLINE void +empty_runq(ErtsRunQueue *rq) +{ + long oifls = erts_smp_atomic_band(&rq->info_flags, ~ERTS_RUNQ_IFLG_NONEMPTY); + if (oifls & ERTS_RUNQ_IFLG_NONEMPTY) { +#ifdef DEBUG + long empty = erts_smp_atomic_read(&no_empty_run_queues); + ASSERT(0 <= empty && empty < erts_no_run_queues); +#endif + erts_smp_atomic_inc(&no_empty_run_queues); + } +} + +static ERTS_INLINE void +non_empty_runq(ErtsRunQueue *rq) +{ + long oifls = erts_smp_atomic_bor(&rq->info_flags, ERTS_RUNQ_IFLG_NONEMPTY); + if (!(oifls & ERTS_RUNQ_IFLG_NONEMPTY)) { +#ifdef DEBUG + long empty = erts_smp_atomic_read(&no_empty_run_queues); + ASSERT(0 < empty && empty <= erts_no_run_queues); +#endif + erts_smp_atomic_dec(&no_empty_run_queues); + } +} + +static ERTS_INLINE int +sched_spin_wake(ErtsRunQueue *rq) +{ +#if ERTS_SCHED_SLEEP_SPINCOUNT == 0 + return 0; +#else + long val; + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); + + val = erts_smp_atomic_read(&rq->spin_waiter); + ASSERT(val >= 0); + if (val != 0) { + erts_smp_atomic_inc(&rq->spin_wake); + return 1; + } + return 0; +#endif +} + +static ERTS_INLINE int +sched_spin_wake_all(ErtsRunQueue *rq) +{ +#if ERTS_SCHED_SLEEP_SPINCOUNT == 0 + return 0; +#else + long val; + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); + + val = erts_smp_atomic_read(&rq->spin_waiter); + ASSERT(val >= 0); + if (val != 0) + erts_smp_atomic_add(&rq->spin_wake, val); + return val; +#endif +} + +static void +sched_sys_wait(Uint no, ErtsRunQueue *rq) +{ + long dt; +#if ERTS_SCHED_SLEEP_SPINCOUNT != 0 + int val; + int spincount = ERTS_SCHED_SLEEP_SPINCOUNT; + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); + +#endif + + sched_waiting_sys(no, rq); + +#if ERTS_SCHED_SLEEP_SPINCOUNT != 0 + erts_smp_atomic_inc(&rq->spin_waiter); + erts_smp_runq_unlock(rq); + + erl_sys_schedule(1); /* Might give us something to do */ + + dt = do_time_read_and_reset(); + if (dt) bump_timer(dt); + + while (spincount-- > 0) { + val = erts_smp_atomic_read(&rq->spin_wake); + ASSERT(val >= 0); + if (val != 0) { + erts_smp_runq_lock(rq); + val = erts_smp_atomic_read(&rq->spin_wake); + ASSERT(val >= 0); + if (val != 0) + goto woken; + if (spincount == 0) + goto sleep; + erts_smp_runq_unlock(rq); + } + } + + erts_smp_runq_lock(rq); + val = erts_smp_atomic_read(&rq->spin_wake); + ASSERT(val >= 0); + if (val != 0) { + woken: + erts_smp_atomic_dec(&rq->spin_wake); + ASSERT(erts_smp_atomic_read(&rq->spin_wake) >= 0); + erts_smp_atomic_dec(&rq->spin_waiter); + ASSERT(erts_smp_atomic_read(&rq->spin_waiter) >= 0); + } + else { + sleep: + erts_smp_atomic_dec(&rq->spin_waiter); + ASSERT(erts_smp_atomic_read(&rq->spin_waiter) >= 0); + /* + * If we got new I/O tasks we aren't allowed to + * sleep in erl_sys_schedule(). + */ + if (!erts_port_task_have_outstanding_io_tasks()) { +#endif + + erts_sys_schedule_interrupt(0); + erts_smp_runq_unlock(rq); + + erl_sys_schedule(0); + + dt = do_time_read_and_reset(); + if (dt) bump_timer(dt); + + erts_smp_runq_lock(rq); + +#if ERTS_SCHED_SLEEP_SPINCOUNT != 0 + } + } +#endif + + sched_active_sys(no, rq); +} + +static void +sched_cnd_wait(Uint no, ErtsRunQueue *rq) +{ +#if ERTS_SCHED_SLEEP_SPINCOUNT != 0 + int val; + int spincount = ERTS_SCHED_SLEEP_SPINCOUNT; + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); +#endif + + sched_waiting(no, rq); + erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, + prepare_for_block, + resume_after_block, + (void *) rq); + +#if ERTS_SCHED_SLEEP_SPINCOUNT == 0 + erts_smp_cnd_wait(&rq->cnd, &rq->mtx); +#else + erts_smp_atomic_inc(&rq->spin_waiter); + erts_smp_mtx_unlock(&rq->mtx); + + while (spincount-- > 0) { + val = erts_smp_atomic_read(&rq->spin_wake); + ASSERT(val >= 0); + if (val != 0) { + erts_smp_mtx_lock(&rq->mtx); + val = erts_smp_atomic_read(&rq->spin_wake); + ASSERT(val >= 0); + if (val != 0) + goto woken; + if (spincount == 0) + goto sleep; + erts_smp_mtx_unlock(&rq->mtx); + } + } + + erts_smp_mtx_lock(&rq->mtx); + val = erts_smp_atomic_read(&rq->spin_wake); + ASSERT(val >= 0); + if (val == 0) { + sleep: + erts_smp_atomic_dec(&rq->spin_waiter); + ASSERT(erts_smp_atomic_read(&rq->spin_waiter) >= 0); + erts_smp_cnd_wait(&rq->cnd, &rq->mtx); + } + else { + woken: + erts_smp_atomic_dec(&rq->spin_wake); + ASSERT(erts_smp_atomic_read(&rq->spin_wake) >= 0); + erts_smp_atomic_dec(&rq->spin_waiter); + ASSERT(erts_smp_atomic_read(&rq->spin_waiter) >= 0); + } +#endif + + erts_smp_activity_end(ERTS_ACTIVITY_WAIT, + prepare_for_block, + resume_after_block, + (void *) rq); + + sched_active(no, rq); +} + +static void +wake_one_scheduler(void) +{ + ASSERT(erts_common_run_queue); + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(erts_common_run_queue)); + if (erts_common_run_queue->waiting) { + if (!sched_spin_wake(erts_common_run_queue)) { + if (erts_common_run_queue->waiting == -1) /* One scheduler waiting + and doing so in + sys_schedule */ + erts_sys_schedule_interrupt(1); + else + erts_smp_cnd_signal(&erts_common_run_queue->cnd); + } + } +} + +static void +wake_scheduler(ErtsRunQueue *rq, int incq) +{ + ASSERT(!erts_common_run_queue); + ASSERT(-1 <= rq->waiting && rq->waiting <= 1); + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); + if (rq->waiting && !rq->woken) { + if (!sched_spin_wake(rq)) { + if (rq->waiting < 0) + erts_sys_schedule_interrupt(1); + else + erts_smp_cnd_signal(&rq->cnd); + } + rq->woken = 1; + if (incq) + non_empty_runq(rq); + } +} + +static void +wake_all_schedulers(void) +{ + if (erts_common_run_queue) { + erts_smp_runq_lock(erts_common_run_queue); + if (erts_common_run_queue->waiting) { + if (erts_common_run_queue->waiting < 0) + erts_sys_schedule_interrupt(1); + sched_spin_wake_all(erts_common_run_queue); + erts_smp_cnd_broadcast(&erts_common_run_queue->cnd); + } + erts_smp_runq_unlock(erts_common_run_queue); + } + else { + int ix; + for (ix = 0; ix < erts_no_run_queues; ix++) { + ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); + erts_smp_runq_lock(rq); + wake_scheduler(rq, 0); + erts_smp_runq_unlock(rq); + } + } +} + +static ERTS_INLINE int +chk_wake_sched(ErtsRunQueue *crq, int ix, int activate) +{ + long iflgs; + ErtsRunQueue *wrq; + if (crq->ix == ix) + return 0; + wrq = ERTS_RUNQ_IX(ix); + iflgs = erts_smp_atomic_read(&wrq->info_flags); + if (!(iflgs & (ERTS_RUNQ_IFLG_SUSPENDED|ERTS_RUNQ_IFLG_NONEMPTY))) { + erts_smp_xrunq_lock(crq, wrq); + if (activate) { + if (ix == erts_smp_atomic_cmpxchg(&balance_info.active_runqs, ix+1, ix)) { + wrq->flags &= ~ERTS_RUNQ_FLG_INACTIVE; + } + } + wake_scheduler(wrq, 0); + erts_smp_xrunq_unlock(crq, wrq); + return 1; + } + return 0; +} + +static void +wake_scheduler_on_empty_runq(ErtsRunQueue *crq) +{ + int ix = crq->ix; + int stop_ix = ix; + int active_ix = erts_smp_atomic_read(&balance_info.active_runqs); + int balance_ix = erts_smp_atomic_read(&balance_info.used_runqs); + + if (active_ix > balance_ix) + active_ix = balance_ix; + + if (ix >= active_ix) + stop_ix = ix = active_ix; + + /* Try to wake a scheduler on an active run queue */ + while (1) { + ix--; + if (ix < 0) { + if (active_ix == stop_ix) + break; + ix = active_ix - 1; + } + if (ix == stop_ix) + break; + if (chk_wake_sched(crq, ix, 0)) + return; + } + + if (active_ix < balance_ix) { + /* Try to activate a new run queue and wake its scheduler */ + (void) chk_wake_sched(crq, active_ix, 1); + } +} + +#endif /* ERTS_SMP */ + +static ERTS_INLINE void +smp_notify_inc_runq(ErtsRunQueue *runq) +{ +#ifdef ERTS_SMP + if (erts_common_run_queue) + wake_one_scheduler(); + else + wake_scheduler(runq, 1); +#endif +} + +void +erts_smp_notify_inc_runq__(ErtsRunQueue *runq) +{ + smp_notify_inc_runq(runq); +} + +#ifdef ERTS_SMP + +ErtsRunQueue * +erts_prepare_emigrate(ErtsRunQueue *c_rq, ErtsRunQueueInfo *c_rqi, int prio) +{ + ASSERT(ERTS_CHK_RUNQ_FLG_EMIGRATE(c_rq->flags, prio)); + ASSERT(ERTS_CHK_RUNQ_FLG_EVACUATE(c_rq->flags, prio) + || c_rqi->len >= c_rqi->migrate.limit.this); + + while (1) { + ErtsRunQueue *n_rq = c_rqi->migrate.runq; + ERTS_DBG_VERIFY_VALID_RUNQP(n_rq); + erts_smp_xrunq_lock(c_rq, n_rq); + + /* + * erts_smp_xrunq_lock() may release lock on c_rq! We have + * to check that we still want to emigrate and emigrate + * to the same run queue as before. + */ + + if (ERTS_CHK_RUNQ_FLG_EMIGRATE(c_rq->flags, prio)) { + Uint32 force = (ERTS_CHK_RUNQ_FLG_EVACUATE(c_rq->flags, prio) + | (c_rq->flags & ERTS_RUNQ_FLG_INACTIVE)); + if (force || c_rqi->len > c_rqi->migrate.limit.this) { + ErtsRunQueueInfo *n_rqi; + /* We still want to emigrate */ + + if (n_rq != c_rqi->migrate.runq) { + /* Ahh... run queue changed; need to do it all over again... */ + erts_smp_runq_unlock(n_rq); + continue; + } + else { + + if (prio == ERTS_PORT_PRIO_LEVEL) + n_rqi = &n_rq->ports.info; + else + n_rqi = &n_rq->procs.prio_info[prio]; + + if (force || (n_rqi->len < c_rqi->migrate.limit.other)) { + /* emigrate ... */ + return n_rq; + } + } + } + } + + ASSERT(n_rq != c_rq); + erts_smp_runq_unlock(n_rq); + if (!(c_rq->flags & ERTS_RUNQ_FLG_INACTIVE)) { + /* No more emigrations to this runq */ + ERTS_UNSET_RUNQ_FLG_EMIGRATE(c_rq->flags, prio); + ERTS_DBG_SET_INVALID_RUNQP(c_rqi->migrate.runq, 0x3); + } + + return NULL; + } +} + +static void +immigrate(ErtsRunQueue *rq) +{ + int prio; + + ASSERT(rq->flags & ERTS_RUNQ_FLGS_IMMIGRATE_QMASK); + + for (prio = 0; prio < ERTS_NO_PRIO_LEVELS; prio++) { + if (ERTS_CHK_RUNQ_FLG_IMMIGRATE(rq->flags, prio)) { + ErtsRunQueueInfo *rqi = (prio == ERTS_PORT_PRIO_LEVEL + ? &rq->ports.info + : &rq->procs.prio_info[prio]); + ErtsRunQueue *from_rq = rqi->migrate.runq; + int rq_locked, from_rq_locked; + + ERTS_DBG_VERIFY_VALID_RUNQP(from_rq); + + rq_locked = 1; + from_rq_locked = 1; + erts_smp_xrunq_lock(rq, from_rq); + /* + * erts_smp_xrunq_lock() may release lock on rq! We have + * to check that we still want to immigrate from the same + * run queue as before. + */ + if (ERTS_CHK_RUNQ_FLG_IMMIGRATE(rq->flags, prio) + && from_rq == rqi->migrate.runq) { + ErtsRunQueueInfo *from_rqi = (prio == ERTS_PORT_PRIO_LEVEL + ? &from_rq->ports.info + : &from_rq->procs.prio_info[prio]); + if ((ERTS_CHK_RUNQ_FLG_EVACUATE(rq->flags, prio) + && ERTS_CHK_RUNQ_FLG_EVACUATE(from_rq->flags, prio) + && from_rqi->len) + || (from_rqi->len > rqi->migrate.limit.other + && rqi->len < rqi->migrate.limit.this)) { + if (prio == ERTS_PORT_PRIO_LEVEL) { + Port *prt = from_rq->ports.start; + if (prt) { + int prt_locked = 0; + (void) erts_port_migrate(prt, &prt_locked, + from_rq, &from_rq_locked, + rq, &rq_locked); + if (prt_locked) + erts_smp_port_unlock(prt); + } + } + else { + Process *proc; + ErtsRunPrioQueue *from_rpq; + from_rpq = (prio == PRIORITY_LOW + ? &from_rq->procs.prio[PRIORITY_NORMAL] + : &from_rq->procs.prio[prio]); + for (proc = from_rpq->first; proc; proc = proc->next) + if (proc->prio == prio && !proc->bound_runq) + break; + if (proc) { + ErtsProcLocks proc_locks = 0; + (void) erts_proc_migrate(proc, &proc_locks, + from_rq, &from_rq_locked, + rq, &rq_locked); + if (proc_locks) + erts_smp_proc_unlock(proc, proc_locks); + } + } + } + else { + ERTS_UNSET_RUNQ_FLG_IMMIGRATE(rq->flags, prio); + ERTS_DBG_SET_INVALID_RUNQP(rqi->migrate.runq, 0x1); + } + } + if (from_rq_locked) + erts_smp_runq_unlock(from_rq); + if (!rq_locked) + erts_smp_runq_lock(rq); + } + } +} + +static void +evacuate_run_queue(ErtsRunQueue *evac_rq, ErtsRunQueue *rq) +{ + Port *prt; + int prio; + int prt_locked = 0; + int rq_locked = 0; + int evac_rq_locked = 1; + + erts_smp_runq_lock(evac_rq); + + evac_rq->flags &= ~ERTS_RUNQ_FLGS_IMMIGRATE_QMASK; + evac_rq->flags |= (ERTS_RUNQ_FLGS_EMIGRATE_QMASK + | ERTS_RUNQ_FLGS_EVACUATE_QMASK + | ERTS_RUNQ_FLG_SUSPENDED); + + erts_smp_atomic_bor(&evac_rq->info_flags, ERTS_RUNQ_IFLG_SUSPENDED); + + /* + * Need to set up evacuation paths first since we + * may release the run queue lock on evac_rq + * when evacuating. + */ + evac_rq->misc.evac_runq = rq; + evac_rq->ports.info.migrate.runq = rq; + for (prio = 0; prio < ERTS_NO_PROC_PRIO_LEVELS; prio++) + evac_rq->procs.prio_info[prio].migrate.runq = rq; + + /* Evacuate scheduled misc ops */ + + if (evac_rq->misc.start) { + rq_locked = 1; + erts_smp_xrunq_lock(evac_rq, rq); + if (rq->misc.end) + rq->misc.end->next = evac_rq->misc.start; + else + rq->misc.start = evac_rq->misc.start; + rq->misc.end = evac_rq->misc.end; + evac_rq->misc.start = NULL; + evac_rq->misc.end = NULL; + } + + /* Evacuate scheduled ports */ + prt = evac_rq->ports.start; + while (prt) { + (void) erts_port_migrate(prt, &prt_locked, + evac_rq, &evac_rq_locked, + rq, &rq_locked); + if (prt_locked) + erts_smp_port_unlock(prt); + if (!evac_rq_locked) { + evac_rq_locked = 1; + erts_smp_runq_lock(evac_rq); + } + prt = evac_rq->ports.start; + } + + /* Evacuate scheduled processes */ + for (prio = 0; prio < ERTS_NO_PROC_PRIO_LEVELS; prio++) { + Process *proc; + + switch (prio) { + case PRIORITY_MAX: + case PRIORITY_HIGH: + case PRIORITY_NORMAL: + proc = evac_rq->procs.prio[prio].first; + while (proc) { + ErtsProcLocks proc_locks = 0; + + /* Bound processes are stuck... */ + while (proc->bound_runq) { + proc = proc->next; + if (!proc) + goto end_of_proc; + } + + (void) erts_proc_migrate(proc, &proc_locks, + evac_rq, &evac_rq_locked, + rq, &rq_locked); + if (proc_locks) + erts_smp_proc_unlock(proc, proc_locks); + if (!evac_rq_locked) { + erts_smp_runq_lock(evac_rq); + evac_rq_locked = 1; + } + + proc = evac_rq->procs.prio[prio].first; + } + + end_of_proc: + +#ifdef DEBUG + for (proc = evac_rq->procs.prio[prio].first; + proc; + proc = proc->next) { + ASSERT(proc->bound_runq); + } +#endif + break; + case PRIORITY_LOW: + break; + default: + ASSERT(!"Invalid process priority"); + break; + } + } + + if (rq_locked) + erts_smp_runq_unlock(rq); + + if (!evac_rq_locked) + erts_smp_runq_lock(evac_rq); + wake_scheduler(evac_rq, 0); + erts_smp_runq_unlock(evac_rq); +} + +static int +try_steal_task_from_victim(ErtsRunQueue *rq, int *rq_lockedp, ErtsRunQueue *vrq) +{ + Process *proc; + int vrq_locked; + + if (*rq_lockedp) + erts_smp_xrunq_lock(rq, vrq); + else + erts_smp_runq_lock(vrq); + vrq_locked = 1; + + ERTS_SMP_LC_CHK_RUNQ_LOCK(rq, *rq_lockedp); + ERTS_SMP_LC_CHK_RUNQ_LOCK(vrq, vrq_locked); + + /* + * Check for a runnable process to steal... + */ + + switch (vrq->flags & ERTS_RUNQ_FLGS_PROCS_QMASK) { + case MAX_BIT: + case MAX_BIT|HIGH_BIT: + case MAX_BIT|NORMAL_BIT: + case MAX_BIT|LOW_BIT: + case MAX_BIT|HIGH_BIT|NORMAL_BIT: + case MAX_BIT|HIGH_BIT|LOW_BIT: + case MAX_BIT|NORMAL_BIT|LOW_BIT: + case MAX_BIT|HIGH_BIT|NORMAL_BIT|LOW_BIT: + for (proc = vrq->procs.prio[PRIORITY_MAX].last; + proc; + proc = proc->prev) { + if (!proc->bound_runq) + break; + } + if (proc) + break; + case HIGH_BIT: + case HIGH_BIT|NORMAL_BIT: + case HIGH_BIT|LOW_BIT: + case HIGH_BIT|NORMAL_BIT|LOW_BIT: + for (proc = vrq->procs.prio[PRIORITY_HIGH].last; + proc; + proc = proc->prev) { + if (!proc->bound_runq) + break; + } + if (proc) + break; + case NORMAL_BIT: + case LOW_BIT: + case NORMAL_BIT|LOW_BIT: + for (proc = vrq->procs.prio[PRIORITY_NORMAL].last; + proc; + proc = proc->prev) { + if (!proc->bound_runq) + break; + } + if (proc) + break; + case 0: + proc = NULL; + break; + default: + ASSERT(!"Invalid queue mask"); + proc = NULL; + break; + } + + if (proc) { + ErtsProcLocks proc_locks = 0; + int res; + ErtsMigrateResult mres; + mres = erts_proc_migrate(proc, &proc_locks, + vrq, &vrq_locked, + rq, rq_lockedp); + if (proc_locks) + erts_smp_proc_unlock(proc, proc_locks); + res = !0; + switch (mres) { + case ERTS_MIGRATE_FAILED_RUNQ_SUSPENDED: + res = 0; + case ERTS_MIGRATE_SUCCESS: + if (vrq_locked) + erts_smp_runq_unlock(vrq); + return res; + default: /* Other failures */ + break; + } + } + + ERTS_SMP_LC_CHK_RUNQ_LOCK(rq, *rq_lockedp); + ERTS_SMP_LC_CHK_RUNQ_LOCK(vrq, vrq_locked); + + if (!vrq_locked) { + if (*rq_lockedp) + erts_smp_xrunq_lock(rq, vrq); + else + erts_smp_runq_lock(vrq); + vrq_locked = 1; + } + + ERTS_SMP_LC_CHK_RUNQ_LOCK(rq, *rq_lockedp); + ERTS_SMP_LC_CHK_RUNQ_LOCK(vrq, vrq_locked); + + /* + * Check for a runnable port to steal... + */ + + if (vrq->ports.info.len) { + Port *prt = vrq->ports.end; + int prt_locked = 0; + int res; + ErtsMigrateResult mres; + + mres = erts_port_migrate(prt, &prt_locked, + vrq, &vrq_locked, + rq, rq_lockedp); + if (prt_locked) + erts_smp_port_unlock(prt); + res = !0; + switch (mres) { + case ERTS_MIGRATE_FAILED_RUNQ_SUSPENDED: + res = 0; + case ERTS_MIGRATE_SUCCESS: + if (vrq_locked) + erts_smp_runq_unlock(vrq); + return res; + default: /* Other failures */ + break; + } + } + + if (vrq_locked) + erts_smp_runq_unlock(vrq); + + return 0; +} + + +static ERTS_INLINE int +check_possible_steal_victim(ErtsRunQueue *rq, int *rq_lockedp, int vix) +{ + ErtsRunQueue *vrq = ERTS_RUNQ_IX(vix); + long iflgs = erts_smp_atomic_read(&vrq->info_flags); + if (iflgs & ERTS_RUNQ_IFLG_NONEMPTY) + return try_steal_task_from_victim(rq, rq_lockedp, vrq); + else + return 0; +} + + +static int +try_steal_task(ErtsRunQueue *rq) +{ + int res, rq_locked, vix, active_rqs, blnc_rqs; + + if (erts_common_run_queue) + return 0; + + /* + * We are not allowed to steal jobs to this run queue + * if it is suspended. Note that it might get suspended + * at any time when we don't have the lock on the run + * queue. + */ + if (rq->flags & ERTS_RUNQ_FLG_SUSPENDED) + return 0; + + res = 0; + rq_locked = 1; + + ERTS_SMP_LC_CHK_RUNQ_LOCK(rq, rq_locked); + + active_rqs = erts_smp_atomic_read(&balance_info.active_runqs); + blnc_rqs = erts_smp_atomic_read(&balance_info.used_runqs); + + if (active_rqs > blnc_rqs) + active_rqs = blnc_rqs; + + if (rq->ix < active_rqs) { + + /* First try to steal from an inactive run queue... */ + if (active_rqs < blnc_rqs) { + int no = blnc_rqs - active_rqs; + int stop_ix = vix = active_rqs + rq->ix % no; + while (erts_smp_atomic_read(&no_empty_run_queues) < blnc_rqs) { + res = check_possible_steal_victim(rq, &rq_locked, vix); + if (res) + goto done; + vix++; + if (vix >= blnc_rqs) + vix = active_rqs; + if (vix == stop_ix) + break; + } + } + + vix = rq->ix; + + /* ... then try to steal a job from another active queue... */ + while (erts_smp_atomic_read(&no_empty_run_queues) < blnc_rqs) { + vix++; + if (vix >= active_rqs) + vix = 0; + if (vix == rq->ix) + break; + + res = check_possible_steal_victim(rq, &rq_locked, vix); + if (res) + goto done; + } + + } + + done: + + if (!rq_locked) + erts_smp_runq_lock(rq); + + if (!res) + res = !ERTS_EMPTY_RUNQ(rq); + + return res; +} + +#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN +void +erts_smp_notify_check_children_needed(void) +{ + int i; + for (i = 0; i < erts_no_schedulers; i++) { + erts_smp_runq_lock(ERTS_SCHEDULER_IX(i)->run_queue); + ERTS_SCHEDULER_IX(i)->check_children = 1; + if (!erts_common_run_queue) + wake_scheduler(ERTS_SCHEDULER_IX(i)->run_queue, 0); + erts_smp_runq_unlock(ERTS_SCHEDULER_IX(i)->run_queue); + } + if (ongoing_multi_scheduling_block()) { + /* Also blocked schedulers need to check children */ + erts_smp_mtx_lock(&schdlr_sspnd.mtx); + for (i = 0; i < erts_no_schedulers; i++) + ERTS_SCHEDULER_IX(i)->blocked_check_children = 1; + erts_smp_cnd_broadcast(&schdlr_sspnd.cnd); + erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + } + if (erts_common_run_queue) + wake_all_schedulers(); +} +#endif + +/* Run queue balancing */ + +typedef struct { + Uint32 flags; + struct { + int max_len; + int avail; + int reds; + int migration_limit; + int emigrate_to; + int immigrate_from; + } prio[ERTS_NO_PRIO_LEVELS]; + int reds; + int full_reds; + int full_reds_history_sum; + int full_reds_history_change; + int oowc; + int max_len; +} ErtsRunQueueBalance; +static ErtsRunQueueBalance *run_queue_info; + +typedef struct { + int qix; + int len; +} ErtsRunQueueCompare; +static ErtsRunQueueCompare *run_queue_compare; + +static int +rqc_len_cmp(const void *x, const void *y) +{ + return ((ErtsRunQueueCompare *) x)->len - ((ErtsRunQueueCompare *) y)->len; +} + +#define ERTS_PERCENT(X, Y) \ + ((Y) == 0 \ + ? ((X) == 0 ? 100 : INT_MAX) \ + : ((100*(X))/(Y))) + +#define ERTS_UPDATE_FULL_REDS(QIX, LAST_REDS) \ +do { \ + run_queue_info[(QIX)].full_reds \ + = run_queue_info[(QIX)].full_reds_history_sum; \ + run_queue_info[(QIX)].full_reds += (LAST_REDS); \ + run_queue_info[(QIX)].full_reds \ + >>= ERTS_FULL_REDS_HISTORY_AVG_SHFT; \ + run_queue_info[(QIX)].full_reds_history_sum \ + -= run_queue_info[(QIX)].full_reds_history_change; \ + run_queue_info[(QIX)].full_reds_history_sum += (LAST_REDS); \ + run_queue_info[(QIX)].full_reds_history_change = (LAST_REDS); \ +} while (0) + +#define ERTS_DBG_CHK_FULL_REDS_HISTORY(RQ) \ +do { \ + int sum__ = 0; \ + int rix__; \ + for (rix__ = 0; rix__ < ERTS_FULL_REDS_HISTORY_SIZE; rix__++) \ + sum__ += (RQ)->full_reds_history[rix__]; \ + ASSERT(sum__ == (RQ)->full_reds_history_sum); \ +} while (0); + +static void +check_balance(ErtsRunQueue *c_rq) +{ + ErtsRunQueueBalance avg = {0}; + Sint64 scheds_reds, full_scheds_reds; + int forced, active, current_active, oowc, half_full_scheds, full_scheds, + mmax_len, blnc_no_rqs, qix, pix, freds_hist_ix; + + if (erts_smp_atomic_xchg(&balance_info.checking_balance, 1)) { + c_rq->check_balance_reds = INT_MAX; + return; + } + + blnc_no_rqs = (int) erts_smp_atomic_read(&balance_info.used_runqs); + if (blnc_no_rqs == 1) { + c_rq->check_balance_reds = INT_MAX; + erts_smp_atomic_set(&balance_info.checking_balance, 0); + return; + } + + erts_smp_runq_unlock(c_rq); + + if (balance_info.halftime) { + balance_info.halftime = 0; + erts_smp_atomic_set(&balance_info.checking_balance, 0); + ERTS_FOREACH_RUNQ(rq, + { + if (rq->waiting) + rq->flags |= ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK; + else + rq->flags &= ~ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK; + rq->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS; + }); + + erts_smp_runq_lock(c_rq); + return; + } + + /* + * check_balance() is never called in more threads + * than one at a time, i.e., we will normally never + * get any conflicts on the balance_info.update_mtx. + * However, when blocking multi scheduling (which performance + * critical applications do *not* do) migration information + * is manipulated. Such updates of the migration information + * might clash with balancing. + */ + erts_smp_mtx_lock(&balance_info.update_mtx); + + forced = balance_info.forced_check_balance; + balance_info.forced_check_balance = 0; + + blnc_no_rqs = (int) erts_smp_atomic_read(&balance_info.used_runqs); + if (blnc_no_rqs == 1) { + erts_smp_mtx_unlock(&balance_info.update_mtx); + erts_smp_runq_lock(c_rq); + c_rq->check_balance_reds = INT_MAX; + erts_smp_atomic_set(&balance_info.checking_balance, 0); + return; + } + + freds_hist_ix = balance_info.full_reds_history_index; + balance_info.full_reds_history_index++; + if (balance_info.full_reds_history_index >= ERTS_FULL_REDS_HISTORY_SIZE) + balance_info.full_reds_history_index = 0; + + current_active = erts_smp_atomic_read(&balance_info.active_runqs); + + /* Read balance information for all run queues */ + for (qix = 0; qix < blnc_no_rqs; qix++) { + ErtsRunQueue *rq = ERTS_RUNQ_IX(qix); + erts_smp_runq_lock(rq); + + run_queue_info[qix].flags = rq->flags; + for (pix = 0; pix < ERTS_NO_PROC_PRIO_LEVELS; pix++) { + run_queue_info[qix].prio[pix].max_len + = rq->procs.prio_info[pix].max_len; + run_queue_info[qix].prio[pix].reds + = rq->procs.prio_info[pix].reds; + } + run_queue_info[qix].prio[ERTS_PORT_PRIO_LEVEL].max_len + = rq->ports.info.max_len; + run_queue_info[qix].prio[ERTS_PORT_PRIO_LEVEL].reds + = rq->ports.info.reds; + + run_queue_info[qix].full_reds_history_sum + = rq->full_reds_history_sum; + run_queue_info[qix].full_reds_history_change + = rq->full_reds_history[freds_hist_ix]; + + run_queue_info[qix].oowc = rq->out_of_work_count; + run_queue_info[qix].max_len = rq->max_len; + rq->check_balance_reds = INT_MAX; + + erts_smp_runq_unlock(rq); + } + + full_scheds = 0; + half_full_scheds = 0; + full_scheds_reds = 0; + scheds_reds = 0; + oowc = 0; + mmax_len = 0; + + /* Calculate availability for each priority in each run queues */ + for (qix = 0; qix < blnc_no_rqs; qix++) { + int treds = 0; + + if (run_queue_info[qix].flags & ERTS_RUNQ_FLG_OUT_OF_WORK) { + for (pix = 0; pix < ERTS_NO_PRIO_LEVELS; pix++) { + run_queue_info[qix].prio[pix].avail = 100; + treds += run_queue_info[qix].prio[pix].reds; + } + if (!(run_queue_info[qix].flags & ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK)) + half_full_scheds++; + ERTS_UPDATE_FULL_REDS(qix, ERTS_RUNQ_CHECK_BALANCE_REDS_PER_SCHED); + } + else { + ASSERT(!(run_queue_info[qix].flags & ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK)); + for (pix = 0; pix < ERTS_NO_PRIO_LEVELS; pix++) + treds += run_queue_info[qix].prio[pix].reds; + if (treds == 0) { + for (pix = 0; pix < ERTS_NO_PRIO_LEVELS; pix++) + run_queue_info[qix].prio[pix].avail = 0; + } + else { + int xreds = 0; + int procreds = treds; + procreds -= run_queue_info[qix].prio[ERTS_PORT_PRIO_LEVEL].reds; + + for (pix = 0; pix < ERTS_NO_PROC_PRIO_LEVELS; pix++) { + int av; + + if (xreds == 0) + av = 100; + else if (procreds == xreds) + av = 0; + else { + av = (100*(procreds - xreds)) / procreds; + if (av == 0) + av = 1; + } + run_queue_info[qix].prio[pix].avail = av; + if (pix < PRIORITY_NORMAL) /* ie., max or high */ + xreds += run_queue_info[qix].prio[pix].reds; + } + run_queue_info[qix].prio[ERTS_PORT_PRIO_LEVEL].avail = 100; + } + ERTS_UPDATE_FULL_REDS(qix, treds); + full_scheds_reds += run_queue_info[qix].full_reds; + full_scheds++; + half_full_scheds++; + } + run_queue_info[qix].reds = treds; + scheds_reds += treds; + oowc += run_queue_info[qix].oowc; + if (mmax_len < run_queue_info[qix].max_len) + mmax_len = run_queue_info[qix].max_len; + } + + if (!forced && half_full_scheds != blnc_no_rqs) { + int min = 1; + if (min < half_full_scheds) + min = half_full_scheds; + if (full_scheds) { + active = (scheds_reds - 1)/ERTS_RUNQ_CHECK_BALANCE_REDS_PER_SCHED+1; + } + else { + active = balance_info.last_active_runqs - 1; + } + + if (balance_info.last_active_runqs < current_active) { + ERTS_BLNCE_SAVE_RISE(current_active, mmax_len, scheds_reds); + active = current_active; + } + else if (active < balance_info.prev_rise.active_runqs) { + if (ERTS_PERCENT(mmax_len, + balance_info.prev_rise.max_len) >= 90 + && ERTS_PERCENT(scheds_reds, + balance_info.prev_rise.reds) >= 90) { + active = balance_info.prev_rise.active_runqs; + } + } + + if (active < min) + active = min; + else if (active > blnc_no_rqs) + active = blnc_no_rqs; + + if (active == blnc_no_rqs) + goto all_active; + + for (qix = 0; qix < active; qix++) { + run_queue_info[qix].flags = 0; + for (pix = 0; pix < ERTS_NO_PRIO_LEVELS; pix++) { + run_queue_info[qix].prio[pix].emigrate_to = -1; + run_queue_info[qix].prio[pix].immigrate_from = -1; + run_queue_info[qix].prio[pix].migration_limit = 0; + } + } + for (qix = active; qix < blnc_no_rqs; qix++) { + run_queue_info[qix].flags = ERTS_RUNQ_FLG_INACTIVE; + for (pix = 0; pix < ERTS_NO_PRIO_LEVELS; pix++) { + int tix = qix % active; + ERTS_SET_RUNQ_FLG_EMIGRATE(run_queue_info[qix].flags, pix); + run_queue_info[qix].prio[pix].emigrate_to = tix; + run_queue_info[qix].prio[pix].immigrate_from = -1; + run_queue_info[qix].prio[pix].migration_limit = 0; + } + } + } + else { + if (balance_info.last_active_runqs < current_active) + ERTS_BLNCE_SAVE_RISE(current_active, mmax_len, scheds_reds); + all_active: + + active = blnc_no_rqs; + + for (qix = 0; qix < blnc_no_rqs; qix++) { + + if (full_scheds_reds > 0) { + /* Calculate availability compared to other schedulers */ + if (!(run_queue_info[qix].flags & ERTS_RUNQ_FLG_OUT_OF_WORK)) { + Sint64 tmp = ((Sint64) run_queue_info[qix].full_reds + * (Sint64) full_scheds); + for (pix = 0; pix < ERTS_NO_PRIO_LEVELS; pix++) { + Sint64 avail = run_queue_info[qix].prio[pix].avail; + avail = (avail*tmp)/full_scheds_reds; + ASSERT(avail >= 0); + run_queue_info[qix].prio[pix].avail = (int) avail; + } + } + } + + /* Calculate average max length */ + for (pix = 0; pix < ERTS_NO_PRIO_LEVELS; pix++) { + run_queue_info[qix].prio[pix].emigrate_to = -1; + run_queue_info[qix].prio[pix].immigrate_from = -1; + avg.prio[pix].max_len += run_queue_info[qix].prio[pix].max_len; + avg.prio[pix].avail += run_queue_info[qix].prio[pix].avail; + } + + } + + for (pix = 0; pix < ERTS_NO_PRIO_LEVELS; pix++) { + int max_len = avg.prio[pix].max_len; + if (max_len != 0) { + int avail = avg.prio[pix].avail; + if (avail != 0) { + max_len = ((100*max_len - 1) / avail) + 1; + avg.prio[pix].max_len = max_len; + ASSERT(max_len >= 0); + } + } + } + + /* Calculate migration limits for all priority queues in all + run queues */ + for (qix = 0; qix < blnc_no_rqs; qix++) { + run_queue_info[qix].flags = 0; /* Reset for later use... */ + for (pix = 0; pix < ERTS_NO_PRIO_LEVELS; pix++) { + int limit; + if (avg.prio[pix].max_len == 0 + || run_queue_info[qix].prio[pix].avail == 0) + limit = 0; + else + limit = (((avg.prio[pix].max_len + * run_queue_info[qix].prio[pix].avail) - 1) + / 100 + 1); + run_queue_info[qix].prio[pix].migration_limit = limit; + } + } + + /* Setup migration paths for all priorities */ + for (pix = 0; pix < ERTS_NO_PRIO_LEVELS; pix++) { + int low = 0, high = 0; + for (qix = 0; qix < blnc_no_rqs; qix++) { + int len_diff = run_queue_info[qix].prio[pix].max_len; + len_diff -= run_queue_info[qix].prio[pix].migration_limit; +#ifdef DBG_PRINT +if (pix == 2) erts_fprintf(stderr, "%d ", len_diff); +#endif + run_queue_compare[qix].qix = qix; + run_queue_compare[qix].len = len_diff; + if (len_diff != 0) { + if (len_diff < 0) + low++; + else + high++; + } + } +#ifdef DBG_PRINT +if (pix == 2) erts_fprintf(stderr, "\n"); +#endif + if (low && high) { + int from_qix; + int to_qix; + int eof = 0; + int eot = 0; + int tix = 0; + int fix = blnc_no_rqs-1; + qsort(run_queue_compare, + blnc_no_rqs, + sizeof(ErtsRunQueueCompare), + rqc_len_cmp); + + while (1) { + if (run_queue_compare[fix].len <= 0) + eof = 1; + if (run_queue_compare[tix].len >= 0) + eot = 1; + if (eof || eot) + break; + from_qix = run_queue_compare[fix].qix; + to_qix = run_queue_compare[tix].qix; + if (run_queue_info[from_qix].prio[pix].avail == 0) { + ERTS_SET_RUNQ_FLG_EVACUATE(run_queue_info[from_qix].flags, + pix); + ERTS_SET_RUNQ_FLG_EVACUATE(run_queue_info[to_qix].flags, + pix); + } + ERTS_SET_RUNQ_FLG_EMIGRATE(run_queue_info[from_qix].flags, pix); + ERTS_SET_RUNQ_FLG_IMMIGRATE(run_queue_info[to_qix].flags, pix); + run_queue_info[from_qix].prio[pix].emigrate_to = to_qix; + run_queue_info[to_qix].prio[pix].immigrate_from = from_qix; + tix++; + fix--; + +#ifdef DBG_PRINT +if (pix == 2) erts_fprintf(stderr, "%d >--> %d\n", from_qix, to_qix); +#endif + } + + if (!eot && eof) { + if (fix < blnc_no_rqs-1) + fix++; + + if (run_queue_compare[fix].len > 0) { + int fix2 = -1; + while (tix < fix) { + if (run_queue_compare[tix].len >= 0) + break; + if (fix2 < fix) + fix2 = blnc_no_rqs-1; + from_qix = run_queue_compare[fix2].qix; + to_qix = run_queue_compare[tix].qix; + ASSERT(to_qix != from_qix); + if (run_queue_info[from_qix].prio[pix].avail == 0) + ERTS_SET_RUNQ_FLG_EVACUATE(run_queue_info[to_qix].flags, + pix); + ERTS_SET_RUNQ_FLG_IMMIGRATE(run_queue_info[to_qix].flags, pix); + run_queue_info[to_qix].prio[pix].immigrate_from = from_qix; + tix++; + fix2--; +#ifdef DBG_PRINT +if (pix == 2) erts_fprintf(stderr, "%d --> %d\n", from_qix, to_qix); +#endif + } + } + } + else if (!eof && eot) { + if (tix > 0) + tix--; + if (run_queue_compare[tix].len < 0) { + int tix2 = 0; + while (tix < fix) { + if (run_queue_compare[fix].len <= 0) + break; + if (tix2 > tix) + tix2 = 0; + from_qix = run_queue_compare[fix].qix; + to_qix = run_queue_compare[tix2].qix; + ASSERT(to_qix != from_qix); + if (run_queue_info[from_qix].prio[pix].avail == 0) + ERTS_SET_RUNQ_FLG_EVACUATE(run_queue_info[from_qix].flags, + pix); + ERTS_SET_RUNQ_FLG_EMIGRATE(run_queue_info[from_qix].flags, pix); + run_queue_info[from_qix].prio[pix].emigrate_to = to_qix; + fix--; + tix2++; +#ifdef DBG_PRINT +if (pix == 2) erts_fprintf(stderr, "%d >-- %d\n", from_qix, to_qix); +#endif + + } + } + } + } + } + +#ifdef DBG_PRINT +erts_fprintf(stderr, "--------------------------------\n"); +#endif + } + + balance_info.last_active_runqs = active; + erts_smp_atomic_set(&balance_info.active_runqs, active); + + balance_info.halftime = 1; + erts_smp_atomic_set(&balance_info.checking_balance, 0); + + /* Write migration paths and reset balance statistics in all queues */ + for (qix = 0; qix < blnc_no_rqs; qix++) { + int mqix; + Uint32 flags; + ErtsRunQueue *rq = ERTS_RUNQ_IX(qix); + ErtsRunQueueInfo *rqi; + flags = run_queue_info[qix].flags; + erts_smp_runq_lock(rq); + flags |= (rq->flags & ~ERTS_RUNQ_FLGS_MIGRATION_INFO); + ASSERT(!(flags & ERTS_RUNQ_FLG_OUT_OF_WORK)); + if (rq->waiting) + flags |= ERTS_RUNQ_FLG_OUT_OF_WORK; + + rq->full_reds_history_sum + = run_queue_info[qix].full_reds_history_sum; + rq->full_reds_history[freds_hist_ix] + = run_queue_info[qix].full_reds_history_change; + + ERTS_DBG_CHK_FULL_REDS_HISTORY(rq); + + rq->out_of_work_count = 0; + rq->flags = flags; + rq->max_len = rq->len; + for (pix = 0; pix < ERTS_NO_PRIO_LEVELS; pix++) { + rqi = (pix == ERTS_PORT_PRIO_LEVEL + ? &rq->ports.info + : &rq->procs.prio_info[pix]); + rqi->max_len = rqi->len; + rqi->reds = 0; + if (!(ERTS_CHK_RUNQ_FLG_EMIGRATE(flags, pix) + | ERTS_CHK_RUNQ_FLG_IMMIGRATE(flags, pix))) { + ASSERT(run_queue_info[qix].prio[pix].immigrate_from < 0); + ASSERT(run_queue_info[qix].prio[pix].emigrate_to < 0); +#ifdef DEBUG + rqi->migrate.limit.this = -1; + rqi->migrate.limit.other = -1; + ERTS_DBG_SET_INVALID_RUNQP(rqi->migrate.runq, 0x2); +#endif + + } + else if (ERTS_CHK_RUNQ_FLG_EMIGRATE(flags, pix)) { + ASSERT(!ERTS_CHK_RUNQ_FLG_IMMIGRATE(flags, pix)); + ASSERT(run_queue_info[qix].prio[pix].immigrate_from < 0); + ASSERT(run_queue_info[qix].prio[pix].emigrate_to >= 0); + + mqix = run_queue_info[qix].prio[pix].emigrate_to; + rqi->migrate.limit.this + = run_queue_info[qix].prio[pix].migration_limit; + rqi->migrate.limit.other + = run_queue_info[mqix].prio[pix].migration_limit; + rqi->migrate.runq = ERTS_RUNQ_IX(mqix); + } + else { + ASSERT(ERTS_CHK_RUNQ_FLG_IMMIGRATE(flags, pix)); + ASSERT(run_queue_info[qix].prio[pix].emigrate_to < 0); + ASSERT(run_queue_info[qix].prio[pix].immigrate_from >= 0); + + mqix = run_queue_info[qix].prio[pix].immigrate_from; + rqi->migrate.limit.this + = run_queue_info[qix].prio[pix].migration_limit; + rqi->migrate.limit.other + = run_queue_info[mqix].prio[pix].migration_limit; + rqi->migrate.runq = ERTS_RUNQ_IX(mqix); + } + } + + rq->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS; + erts_smp_runq_unlock(rq); + } + + balance_info.n++; + erts_smp_mtx_unlock(&balance_info.update_mtx); + + erts_smp_runq_lock(c_rq); +} + +#endif /* #ifdef ERTS_SMP */ + +Uint +erts_debug_nbalance(void) +{ +#ifdef ERTS_SMP + Uint n; + erts_smp_mtx_lock(&balance_info.update_mtx); + n = balance_info.n; + erts_smp_mtx_unlock(&balance_info.update_mtx); + return n; +#else + return 0; +#endif +} + +void +erts_early_init_scheduling(void) +{ + early_cpu_bind_init(); +} + +void +erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) +{ + int ix, n; + +#ifndef ERTS_SMP + mrq = 0; +#endif + + init_misc_op_list_alloc(); + + ASSERT(no_schedulers_online <= no_schedulers); + ASSERT(no_schedulers_online >= 1); + ASSERT(no_schedulers >= 1); + + /* Create and initialize run queues */ + + n = (int) (mrq ? no_schedulers : 1); + + erts_aligned_run_queues = erts_alloc(ERTS_ALC_T_RUNQS, + (sizeof(ErtsAlignedRunQueue)*(n+1))); + if ((((Uint) erts_aligned_run_queues) & ERTS_CACHE_LINE_MASK) == 0) + erts_aligned_run_queues = ((ErtsAlignedRunQueue *) + ((((Uint) erts_aligned_run_queues) + & ~ERTS_CACHE_LINE_MASK) + + ERTS_CACHE_LINE_SIZE)); + +#ifdef ERTS_SMP + erts_smp_atomic_init(&no_empty_run_queues, 0); +#endif + + for (ix = 0; ix < n; ix++) { + int pix, rix; + ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); + + rq->ix = ix; + erts_smp_atomic_init(&rq->info_flags, ERTS_RUNQ_IFLG_NONEMPTY); + + erts_smp_mtx_init(&rq->mtx, "run_queue"); + erts_smp_cnd_init(&rq->cnd); + + erts_smp_atomic_init(&rq->spin_waiter, 0); + erts_smp_atomic_init(&rq->spin_wake, 0); + + rq->waiting = 0; + rq->woken = 0; + rq->flags = !mrq ? ERTS_RUNQ_FLG_SHARED_RUNQ : 0; + rq->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS; + rq->full_reds_history_sum = 0; + for (rix = 0; rix < ERTS_FULL_REDS_HISTORY_SIZE; rix++) { + rq->full_reds_history_sum += ERTS_RUNQ_CHECK_BALANCE_REDS_PER_SCHED; + rq->full_reds_history[rix] = ERTS_RUNQ_CHECK_BALANCE_REDS_PER_SCHED; + } + rq->out_of_work_count = 0; + rq->max_len = 0; + rq->len = 0; + rq->wakeup_other = 0; + rq->wakeup_other_reds = 0; + + rq->procs.len = 0; + rq->procs.pending_exiters = NULL; + rq->procs.context_switches = 0; + rq->procs.reductions = 0; + + for (pix = 0; pix < ERTS_NO_PROC_PRIO_LEVELS; pix++) { + rq->procs.prio_info[pix].len = 0; + rq->procs.prio_info[pix].max_len = 0; + rq->procs.prio_info[pix].reds = 0; + rq->procs.prio_info[pix].migrate.limit.this = 0; + rq->procs.prio_info[pix].migrate.limit.other = 0; + ERTS_DBG_SET_INVALID_RUNQP(rq->procs.prio_info[pix].migrate.runq, + 0x0); + if (pix < ERTS_NO_PROC_PRIO_LEVELS - 1) { + rq->procs.prio[pix].first = NULL; + rq->procs.prio[pix].last = NULL; + } + } + + rq->misc.start = NULL; + rq->misc.end = NULL; + rq->misc.evac_runq = NULL; + + rq->ports.info.len = 0; + rq->ports.info.max_len = 0; + rq->ports.info.reds = 0; + rq->ports.info.migrate.limit.this = 0; + rq->ports.info.migrate.limit.other = 0; + rq->ports.info.migrate.runq = NULL; + rq->ports.start = NULL; + rq->ports.end = NULL; + } + + erts_common_run_queue = !mrq ? ERTS_RUNQ_IX(0) : NULL; + erts_no_run_queues = n; + +#ifdef ERTS_SMP + + if (erts_no_run_queues != 1) { + run_queue_info = erts_alloc(ERTS_ALC_T_RUNQ_BLNS, + (sizeof(ErtsRunQueueBalance) + * erts_no_run_queues)); + run_queue_compare = erts_alloc(ERTS_ALC_T_RUNQ_BLNS, + (sizeof(ErtsRunQueueCompare) + * erts_no_run_queues)); + } + +#endif + + /* Create and initialize scheduler specific data */ + + n = (int) no_schedulers; + erts_aligned_scheduler_data = erts_alloc(ERTS_ALC_T_SCHDLR_DATA, + (sizeof(ErtsAlignedSchedulerData) + *(n+1))); + if ((((Uint) erts_aligned_scheduler_data) & ERTS_CACHE_LINE_MASK) == 0) + erts_aligned_scheduler_data = ((ErtsAlignedSchedulerData *) + ((((Uint) erts_aligned_scheduler_data) + & ~ERTS_CACHE_LINE_MASK) + + ERTS_CACHE_LINE_SIZE)); + for (ix = 0; ix < n; ix++) { + ErtsSchedulerData *esdp = ERTS_SCHEDULER_IX(ix); +#ifdef ERTS_SMP + erts_bits_init_state(&esdp->erl_bits_state); + esdp->match_pseudo_process = NULL; + esdp->free_process = NULL; +#endif + esdp->no = (Uint) ix+1; + esdp->current_process = NULL; + esdp->current_port = NULL; + + esdp->virtual_reds = 0; + esdp->cpu_id = -1; + + erts_init_atom_cache_map(&esdp->atom_cache_map); + + if (erts_common_run_queue) { + esdp->run_queue = erts_common_run_queue; + esdp->run_queue->scheduler = NULL; + } + else { + esdp->run_queue = ERTS_RUNQ_IX(ix); + esdp->run_queue->scheduler = esdp; + } + +#ifdef ERTS_SMP +#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN + esdp->check_children = 0; + esdp->blocked_check_children = 0; +#endif + erts_smp_atomic_init(&esdp->suspended, 0); + erts_smp_atomic_init(&esdp->chk_cpu_bind, 0); +#endif + } + +#ifdef ERTS_SMP + erts_smp_mtx_init(&schdlr_sspnd.mtx, "schdlr_sspnd"); + erts_smp_cnd_init(&schdlr_sspnd.cnd); + + schdlr_sspnd.changing = 0; + schdlr_sspnd.online = no_schedulers_online; + schdlr_sspnd.curr_online = no_schedulers; + erts_smp_atomic_init(&schdlr_sspnd.msb.ongoing, 0); + erts_smp_atomic_init(&schdlr_sspnd.active, no_schedulers); + schdlr_sspnd.msb.procs = NULL; + erts_smp_atomic_set(&balance_info.used_runqs, + erts_common_run_queue ? 1 : no_schedulers_online); + erts_smp_atomic_init(&balance_info.active_runqs, no_schedulers); + balance_info.last_active_runqs = no_schedulers; + erts_smp_mtx_init(&balance_info.update_mtx, "migration_info_update"); + balance_info.forced_check_balance = 0; + balance_info.halftime = 1; + balance_info.full_reds_history_index = 0; + erts_smp_atomic_init(&balance_info.checking_balance, 0); + balance_info.prev_rise.active_runqs = 0; + balance_info.prev_rise.max_len = 0; + balance_info.prev_rise.reds = 0; + balance_info.n = 0; + + if (no_schedulers_online < no_schedulers) { + if (erts_common_run_queue) { + for (ix = no_schedulers_online; ix < no_schedulers; ix++) + erts_smp_atomic_set(&(ERTS_SCHEDULER_IX(ix)->suspended), 1); + } + else { + for (ix = no_schedulers_online; ix < erts_no_run_queues; ix++) + evacuate_run_queue(ERTS_RUNQ_IX(ix), + ERTS_RUNQ_IX(ix % no_schedulers_online)); + } + } + + schdlr_sspnd.wait_curr_online = no_schedulers_online; + schdlr_sspnd.curr_online *= 2; /* Boot strapping... */ + schdlr_sspnd.changing = ERTS_SCHED_CHANGING_ONLINE; + + erts_smp_atomic_init(&doing_sys_schedule, 0); + +#else /* !ERTS_SMP */ + { + ErtsSchedulerData *esdp; + esdp = ERTS_SCHEDULER_IX(0); + erts_scheduler_data = esdp; +#ifdef USE_THREADS + erts_tsd_set(sched_data_key, (void *) esdp); +#endif + } + erts_no_schedulers = 1; +#endif + + erts_smp_atomic_init(&function_calls, 0); + + /* init port tasks */ + erts_port_task_init(); + + late_cpu_bind_init(); +} + +ErtsRunQueue * +erts_schedid2runq(Uint id) +{ + int ix; + if (erts_common_run_queue) + return erts_common_run_queue; + ix = (int) id - 1; + ASSERT(0 <= ix && ix < erts_no_run_queues); + return ERTS_RUNQ_IX(ix); +} + +#ifdef USE_THREADS + +ErtsSchedulerData * +erts_get_scheduler_data(void) +{ + return (ErtsSchedulerData *) erts_tsd_get(sched_data_key); +} + +#endif + +static int remove_proc_from_runq(ErtsRunQueue *rq, Process *p, int to_inactive); + +static ERTS_INLINE void +suspend_process(ErtsRunQueue *rq, Process *p) +{ + ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); + p->rcount++; /* count number of suspend */ +#ifdef ERTS_SMP + ASSERT(!(p->runq_flags & ERTS_PROC_RUNQ_FLG_RUNNING) + || p == erts_get_current_process()); + ASSERT(p->status != P_RUNNING + || p->runq_flags & ERTS_PROC_RUNQ_FLG_RUNNING); + if (p->status_flags & ERTS_PROC_SFLG_PENDADD2SCHEDQ) + goto runable; +#endif + switch(p->status) { + case P_SUSPENDED: + break; + case P_RUNABLE: +#ifdef ERTS_SMP + runable: + if (!ERTS_PROC_PENDING_EXIT(p)) +#endif + remove_proc_from_runq(rq, p, 1); + /* else: + * leave process in schedq so it will discover the pending exit + */ + p->rstatus = P_RUNABLE; /* wakeup as runnable */ + break; + case P_RUNNING: + p->rstatus = P_RUNABLE; /* wakeup as runnable */ + break; + case P_WAITING: + p->rstatus = P_WAITING; /* wakeup as waiting */ + break; + case P_EXITING: + return; /* ignore this */ + case P_GARBING: + case P_FREE: + erl_exit(1, "bad state in suspend_process()\n"); + } + + if ((erts_system_profile_flags.runnable_procs) && (p->rcount == 1) && (p->status != P_WAITING)) { + profile_runnable_proc(p, am_inactive); + } + + p->status = P_SUSPENDED; + +} + +static ERTS_INLINE void +resume_process(Process *p) +{ + Uint32 *statusp; + ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); + switch (p->status) { + case P_SUSPENDED: + statusp = &p->status; + break; + case P_GARBING: + if (p->gcstatus == P_SUSPENDED) { + statusp = &p->gcstatus; + break; + } + /* Fall through */ + default: + return; + } + + ASSERT(p->rcount > 0); + + if (--p->rcount > 0) /* multiple suspend */ + return; + switch(p->rstatus) { + case P_RUNABLE: + *statusp = P_WAITING; /* make erts_add_to_runq work */ + erts_add_to_runq(p); + break; + case P_WAITING: + *statusp = P_WAITING; + break; + default: + erl_exit(1, "bad state in resume_process()\n"); + } + p->rstatus = P_FREE; +} + +#ifdef ERTS_SMP + +static void +susp_sched_prep_block(void *unused) +{ + erts_smp_mtx_unlock(&schdlr_sspnd.mtx); +} + +static void +susp_sched_resume_block(void *unused) +{ + erts_smp_mtx_lock(&schdlr_sspnd.mtx); +} + +static void +suspend_scheduler(ErtsSchedulerData *esdp) +{ + long no = (long) esdp->no; + ErtsRunQueue *rq = esdp->run_queue; + long active_schedulers; + int curr_online = 1; + int wake = 0; + + /* + * Schedulers may be suspended in two different ways: + * - A scheduler may be suspended since it is not online. + * All schedulers with scheduler ids greater than + * schdlr_sspnd.online are suspended. + * - Multi scheduling is blocked. All schedulers except the + * scheduler with scheduler id 1 are suspended. + * + * Regardless of why a scheduler is suspended, it ends up here. + */ + + ASSERT(no != 1); + + erts_smp_runq_unlock(esdp->run_queue); + + /* Unbind from cpu */ + erts_smp_rwmtx_rwlock(&erts_cpu_bind_rwmtx); + if (scheduler2cpu_map[esdp->no].bound_id >= 0 + && erts_unbind_from_cpu(erts_cpuinfo) == 0) { + esdp->cpu_id = scheduler2cpu_map[esdp->no].bound_id = -1; + } + erts_smp_rwmtx_rwunlock(&erts_cpu_bind_rwmtx); + + if (erts_system_profile_flags.scheduler) + profile_scheduler(make_small(esdp->no), am_inactive); + + erts_smp_mtx_lock(&schdlr_sspnd.mtx); + + active_schedulers = erts_smp_atomic_dectest(&schdlr_sspnd.active); + ASSERT(active_schedulers >= 1); + if (schdlr_sspnd.changing == ERTS_SCHED_CHANGING_MULTI_SCHED) { + if (active_schedulers == schdlr_sspnd.msb.wait_active) + wake = 1; + if (active_schedulers == 1) + schdlr_sspnd.changing = 0; + } + + while (1) { + +#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN + int check_children; + erts_smp_runq_lock(esdp->run_queue); + check_children = esdp->check_children; + esdp->check_children = 0; + erts_smp_runq_unlock(esdp->run_queue); + if (check_children) { + erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + erts_check_children(); + erts_smp_mtx_lock(&schdlr_sspnd.mtx); + } +#endif + + if (schdlr_sspnd.changing == ERTS_SCHED_CHANGING_ONLINE) { + int changed = 0; + if (no > schdlr_sspnd.online && curr_online) { + schdlr_sspnd.curr_online--; + curr_online = 0; + changed = 1; + } + else if (no <= schdlr_sspnd.online && !curr_online) { + schdlr_sspnd.curr_online++; + curr_online = 1; + changed = 1; + } + if (changed + && schdlr_sspnd.curr_online == schdlr_sspnd.wait_curr_online) + wake = 1; + if (schdlr_sspnd.online == schdlr_sspnd.curr_online) + schdlr_sspnd.changing = 0; + } + + if (wake) { + erts_smp_cnd_broadcast(&schdlr_sspnd.cnd); + wake = 0; + } + + + if (!(rq->flags & (ERTS_RUNQ_FLG_SHARED_RUNQ|ERTS_RUNQ_FLG_SUSPENDED))) + break; + if ((rq->flags & ERTS_RUNQ_FLG_SHARED_RUNQ) + && !erts_smp_atomic_read(&esdp->suspended)) + break; + + erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, + susp_sched_prep_block, + susp_sched_resume_block, + NULL); + while (1) { + +#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN + if (esdp->blocked_check_children) + break; +#endif + + erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); + + if (schdlr_sspnd.changing == ERTS_SCHED_CHANGING_ONLINE) + break; + + if (!(rq->flags & (ERTS_RUNQ_FLG_SHARED_RUNQ + | ERTS_RUNQ_FLG_SUSPENDED))) + break; + if ((rq->flags & ERTS_RUNQ_FLG_SHARED_RUNQ) + && !erts_smp_atomic_read(&esdp->suspended)) + break; + } + +#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN + esdp->blocked_check_children = 0; +#endif + + erts_smp_activity_end(ERTS_ACTIVITY_WAIT, + susp_sched_prep_block, + susp_sched_resume_block, + NULL); + } + + erts_smp_atomic_inc(&schdlr_sspnd.active); + + erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + + if (erts_system_profile_flags.scheduler) + profile_scheduler(make_small(esdp->no), am_active); + + erts_smp_runq_lock(esdp->run_queue); + non_empty_runq(esdp->run_queue); + + /* Make sure we check if we should bind to a cpu or not... */ + if (rq->flags & ERTS_RUNQ_FLG_SHARED_RUNQ) + erts_smp_atomic_set(&esdp->chk_cpu_bind, 1); + else + rq->flags |= ERTS_RUNQ_FLG_CHK_CPU_BIND; +} + +#define ERTS_RUNQ_RESET_SUSPEND_INFO(RQ, DBG_ID) \ +do { \ + int pix__; \ + (RQ)->misc.evac_runq = NULL; \ + (RQ)->ports.info.migrate.runq = NULL; \ + (RQ)->flags &= ~(ERTS_RUNQ_FLGS_IMMIGRATE_QMASK \ + | ERTS_RUNQ_FLGS_EMIGRATE_QMASK \ + | ERTS_RUNQ_FLGS_EVACUATE_QMASK \ + | ERTS_RUNQ_FLG_SUSPENDED); \ + (RQ)->flags |= (ERTS_RUNQ_FLG_OUT_OF_WORK \ + | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK); \ + (RQ)->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS; \ + erts_smp_atomic_band(&(RQ)->info_flags, ~ERTS_RUNQ_IFLG_SUSPENDED); \ + for (pix__ = 0; pix__ < ERTS_NO_PROC_PRIO_LEVELS; pix__++) { \ + (RQ)->procs.prio_info[pix__].max_len = 0; \ + (RQ)->procs.prio_info[pix__].reds = 0; \ + ERTS_DBG_SET_INVALID_RUNQP((RQ)->procs.prio_info[pix__].migrate.runq,\ + (DBG_ID)); \ + } \ + (RQ)->ports.info.max_len = 0; \ + (RQ)->ports.info.reds = 0; \ +} while (0) + +#define ERTS_RUNQ_RESET_MIGRATION_PATHS__(RQ) \ +do { \ + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked((RQ))); \ + (RQ)->misc.evac_runq = NULL; \ + (RQ)->ports.info.migrate.runq = NULL; \ + (RQ)->flags &= ~(ERTS_RUNQ_FLGS_IMMIGRATE_QMASK \ + | ERTS_RUNQ_FLGS_EMIGRATE_QMASK \ + | ERTS_RUNQ_FLGS_EVACUATE_QMASK); \ +} while (0) + +#ifdef DEBUG +#define ERTS_RUNQ_RESET_MIGRATION_PATHS(RQ, DBG_ID) \ +do { \ + int pix__; \ + ERTS_RUNQ_RESET_MIGRATION_PATHS__((RQ)); \ + for (pix__ = 0; pix__ < ERTS_NO_PROC_PRIO_LEVELS; pix__++) \ + ERTS_DBG_SET_INVALID_RUNQP((RQ)->procs.prio_info[pix__].migrate.runq,\ + (DBG_ID)); \ +} while (0) +#else +#define ERTS_RUNQ_RESET_MIGRATION_PATHS(RQ, DBG_ID) \ + ERTS_RUNQ_RESET_MIGRATION_PATHS__((RQ)) +#endif + +ErtsSchedSuspendResult +erts_schedulers_state(Uint *total, + Uint *online, + Uint *active, + int yield_allowed) +{ + int res; + erts_smp_mtx_lock(&schdlr_sspnd.mtx); + if (yield_allowed && schdlr_sspnd.changing) + res = ERTS_SCHDLR_SSPND_YIELD_RESTART; + else { + *active = *online = schdlr_sspnd.online; + if (ongoing_multi_scheduling_block()) + *active = 1; + res = ERTS_SCHDLR_SSPND_DONE; + } + erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + *total = erts_no_schedulers; + return res; +} + +ErtsSchedSuspendResult +erts_set_schedulers_online(Process *p, + ErtsProcLocks plocks, + Sint new_no, + Sint *old_no) +{ + int ix, res, no, have_unlocked_plocks; + + if (new_no < 1 || erts_no_schedulers < new_no) + return ERTS_SCHDLR_SSPND_EINVAL; + + erts_smp_mtx_lock(&schdlr_sspnd.mtx); + + have_unlocked_plocks = 0; + no = (int) new_no; + + if (schdlr_sspnd.changing) { + res = ERTS_SCHDLR_SSPND_YIELD_RESTART; + } + else { + int online = *old_no = schdlr_sspnd.online; + if (no == schdlr_sspnd.online) { + res = ERTS_SCHDLR_SSPND_DONE; + } + else { + schdlr_sspnd.changing = ERTS_SCHED_CHANGING_ONLINE; + schdlr_sspnd.online = no; + if (no > online) { + int ix; + schdlr_sspnd.wait_curr_online = no; + if (ongoing_multi_scheduling_block()) + /* No schedulers to resume */; + else if (erts_common_run_queue) { + for (ix = online; ix < no; ix++) + erts_smp_atomic_set(&ERTS_SCHEDULER_IX(ix)->suspended, + 0); + } + else { + if (plocks) { + have_unlocked_plocks = 1; + erts_smp_proc_unlock(p, plocks); + } + erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + erts_smp_mtx_lock(&balance_info.update_mtx); + for (ix = online; ix < no; ix++) { + ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); + erts_smp_runq_lock(rq); + ERTS_RUNQ_RESET_SUSPEND_INFO(rq, 0x5); + erts_smp_runq_unlock(rq); + } + /* + * Spread evacuation paths among all online + * run queues. + */ + for (ix = no; ix < erts_no_run_queues; ix++) { + ErtsRunQueue *from_rq = ERTS_RUNQ_IX(ix); + ErtsRunQueue *to_rq = ERTS_RUNQ_IX(ix % no); + evacuate_run_queue(from_rq, to_rq); + } + erts_smp_atomic_set(&balance_info.used_runqs, no); + erts_smp_mtx_unlock(&balance_info.update_mtx); + erts_smp_mtx_lock(&schdlr_sspnd.mtx); + } + erts_smp_cnd_broadcast(&schdlr_sspnd.cnd); + res = ERTS_SCHDLR_SSPND_DONE; + } + else /* if (no < online) */ { + if (p->scheduler_data->no <= no) { + res = ERTS_SCHDLR_SSPND_DONE; + schdlr_sspnd.wait_curr_online = no; + } + else { + /* + * Yield! Current process needs to migrate + * before bif returns. + */ + res = ERTS_SCHDLR_SSPND_YIELD_DONE; + schdlr_sspnd.wait_curr_online = no+1; + } + + if (ongoing_multi_scheduling_block()) + erts_smp_cnd_broadcast(&schdlr_sspnd.cnd); + else if (erts_common_run_queue) { + for (ix = no; ix < online; ix++) + erts_smp_atomic_set(&ERTS_SCHEDULER_IX(ix)->suspended, + 1); + wake_all_schedulers(); + } + else { + if (plocks) { + have_unlocked_plocks = 1; + erts_smp_proc_unlock(p, plocks); + } + erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + erts_smp_mtx_lock(&balance_info.update_mtx); + + for (ix = 0; ix < online; ix++) { + ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); + erts_smp_runq_lock(rq); + ERTS_RUNQ_RESET_MIGRATION_PATHS(rq, 0x6); + erts_smp_runq_unlock(rq); + } + /* + * Evacutation order important! Newly suspended run queues + * has to be evacuated last. + */ + for (ix = erts_no_run_queues-1; ix >= no; ix--) + evacuate_run_queue(ERTS_RUNQ_IX(ix), + ERTS_RUNQ_IX(ix % no)); + erts_smp_atomic_set(&balance_info.used_runqs, no); + erts_smp_mtx_unlock(&balance_info.update_mtx); + erts_smp_mtx_lock(&schdlr_sspnd.mtx); + ERTS_FOREACH_OP_RUNQ(rq, wake_scheduler(rq, 0)); + } + } + + erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, + susp_sched_prep_block, + susp_sched_resume_block, + NULL); + while (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) + erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); + erts_smp_activity_end(ERTS_ACTIVITY_WAIT, + susp_sched_prep_block, + susp_sched_resume_block, + NULL); + } + } + + erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + if (have_unlocked_plocks) + erts_smp_proc_lock(p, plocks); + + return res; +} + +ErtsSchedSuspendResult +erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) +{ + int ix, res, have_unlocked_plocks = 0; + ErtsProcList *plp; + + erts_smp_mtx_lock(&schdlr_sspnd.mtx); + if (on) { + if (schdlr_sspnd.changing) { + res = ERTS_SCHDLR_SSPND_YIELD_RESTART; /* Yield */ + } + else if (erts_is_multi_scheduling_blocked()) { + plp = proclist_create(p); + plp->next = schdlr_sspnd.msb.procs; + schdlr_sspnd.msb.procs = plp; + p->flags |= F_HAVE_BLCKD_MSCHED; + ASSERT(erts_smp_atomic_read(&schdlr_sspnd.active) == 1); + ASSERT(p->scheduler_data->no == 1); + res = 1; + } + else { + p->flags |= F_HAVE_BLCKD_MSCHED; + if (plocks) { + have_unlocked_plocks = 1; + erts_smp_proc_unlock(p, plocks); + } + erts_smp_atomic_set(&schdlr_sspnd.msb.ongoing, 1); + if (schdlr_sspnd.online == 1) { + res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; + ASSERT(erts_smp_atomic_read(&schdlr_sspnd.active) == 1); + ASSERT(p->scheduler_data->no == 1); + } + else { + schdlr_sspnd.changing = ERTS_SCHED_CHANGING_MULTI_SCHED; + if (p->scheduler_data->no == 1) { + res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; + schdlr_sspnd.msb.wait_active = 1; + } + else { + /* + * Yield! Current process needs to migrate + * before bif returns. + */ + res = ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED; + schdlr_sspnd.msb.wait_active = 2; + } + if (erts_common_run_queue) { + for (ix = 1; ix < schdlr_sspnd.online; ix++) + erts_smp_atomic_set(&ERTS_SCHEDULER_IX(ix)->suspended, 1); + wake_all_schedulers(); + } + else { + erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + erts_smp_mtx_lock(&balance_info.update_mtx); + erts_smp_atomic_set(&balance_info.used_runqs, 1); + for (ix = 0; ix < schdlr_sspnd.online; ix++) { + ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); + erts_smp_runq_lock(rq); + ERTS_RUNQ_RESET_MIGRATION_PATHS(rq, 0x7); + erts_smp_runq_unlock(rq); + } + /* + * Evacuate all activities in all other run queues + * into the first run queue. Note order is important, + * online run queues has to be evacuated last. + */ + for (ix = erts_no_run_queues-1; ix >= 1; ix--) + evacuate_run_queue(ERTS_RUNQ_IX(ix), ERTS_RUNQ_IX(0)); + erts_smp_mtx_unlock(&balance_info.update_mtx); + erts_smp_mtx_lock(&schdlr_sspnd.mtx); + } + erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, + susp_sched_prep_block, + susp_sched_resume_block, + NULL); + while (erts_smp_atomic_read(&schdlr_sspnd.active) + != schdlr_sspnd.msb.wait_active) + erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); + erts_smp_activity_end(ERTS_ACTIVITY_WAIT, + susp_sched_prep_block, + susp_sched_resume_block, + NULL); + } + plp = proclist_create(p); + plp->next = schdlr_sspnd.msb.procs; + schdlr_sspnd.msb.procs = plp; +#ifdef DEBUG + ERTS_FOREACH_RUNQ(srq, + { + if (srq != ERTS_RUNQ_IX(0)) { + ASSERT(ERTS_EMPTY_RUNQ(srq)); + ASSERT(srq->flags & ERTS_RUNQ_FLG_SUSPENDED); + } + }); +#endif + ASSERT(p->scheduler_data); + } + } + else if (!ongoing_multi_scheduling_block()) { + ASSERT(!schdlr_sspnd.msb.procs); + res = ERTS_SCHDLR_SSPND_DONE; + } + else { + if (p->flags & F_HAVE_BLCKD_MSCHED) { + ErtsProcList **plpp = &schdlr_sspnd.msb.procs; + plp = schdlr_sspnd.msb.procs; + + while (plp) { + if (!proclist_same(plp, p)){ + plpp = &plp->next; + plp = plp->next; + } + else { + *plpp = plp->next; + proclist_destroy(plp); + if (!all) + break; + plp = *plpp; + } + } + } + if (schdlr_sspnd.msb.procs) + res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; + else { + schdlr_sspnd.changing = ERTS_SCHED_CHANGING_MULTI_SCHED; +#ifdef DEBUG + ERTS_FOREACH_RUNQ(rq, + { + if (rq != p->scheduler_data->run_queue) { + if (!ERTS_EMPTY_RUNQ(rq)) { + Process *rp; + int pix; + ASSERT(rq->ports.info.len == 0); + for (pix = 0; pix < ERTS_NO_PROC_PRIO_LEVELS; pix++) { + for (rp = rq->procs.prio[pix].first; + rp; + rp = rp->next) { + ASSERT(rp->bound_runq); + } + } + } + + ASSERT(rq->flags & ERTS_RUNQ_FLG_SUSPENDED); + } + }); +#endif + p->flags &= ~F_HAVE_BLCKD_MSCHED; + erts_smp_atomic_set(&schdlr_sspnd.msb.ongoing, 0); + if (schdlr_sspnd.online == 1) + /* No schedulers to resume */; + else if (erts_common_run_queue) { + for (ix = 1; ix < schdlr_sspnd.online; ix++) + erts_smp_atomic_set(&ERTS_SCHEDULER_IX(ix)->suspended, 0); + wake_all_schedulers(); + } + else { + int online = schdlr_sspnd.online; + erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + if (plocks) { + have_unlocked_plocks = 1; + erts_smp_proc_unlock(p, plocks); + } + erts_smp_mtx_lock(&balance_info.update_mtx); + + /* Resume all online run queues */ + for (ix = 1; ix < online; ix++) { + ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); + erts_smp_runq_lock(rq); + ERTS_RUNQ_RESET_SUSPEND_INFO(rq, 0x4); + erts_smp_runq_unlock(rq); + } + + /* Spread evacuation paths among all online run queues */ + for (ix = online; ix < erts_no_run_queues; ix++) + evacuate_run_queue(ERTS_RUNQ_IX(ix), + ERTS_RUNQ_IX(ix % online)); + + erts_smp_atomic_set(&balance_info.used_runqs, online); + /* Make sure that we balance soon... */ + balance_info.forced_check_balance = 1; + erts_smp_runq_lock(ERTS_RUNQ_IX(0)); + ERTS_RUNQ_IX(0)->check_balance_reds = 0; + erts_smp_runq_unlock(ERTS_RUNQ_IX(0)); + erts_smp_mtx_unlock(&balance_info.update_mtx); + erts_smp_mtx_lock(&schdlr_sspnd.mtx); + } + erts_smp_cnd_broadcast(&schdlr_sspnd.cnd); + schdlr_sspnd.changing = 0; + res = ERTS_SCHDLR_SSPND_DONE; + } + } + + erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + if (have_unlocked_plocks) + erts_smp_proc_lock(p, plocks); + return res; +} + +#ifdef DEBUG +void +erts_dbg_multi_scheduling_return_trap(Process *p, Eterm return_value) +{ + if (return_value == am_blocked) { + long active = erts_smp_atomic_read(&schdlr_sspnd.active); + ASSERT(1 <= active && active <= 2); + ASSERT(ERTS_PROC_GET_SCHDATA(p)->no == 1); + } +} +#endif + +int +erts_is_multi_scheduling_blocked(void) +{ + return (erts_smp_atomic_read(&schdlr_sspnd.msb.ongoing) + && erts_smp_atomic_read(&schdlr_sspnd.active) == 1); +} + +Eterm +erts_multi_scheduling_blockers(Process *p) +{ + Eterm res = NIL; + + erts_smp_mtx_lock(&schdlr_sspnd.mtx); + if (erts_is_multi_scheduling_blocked()) { + Eterm *hp, *hp_end; + ErtsProcList *plp1, *plp2; + Uint max_size; + ASSERT(schdlr_sspnd.msb.procs); + for (max_size = 0, plp1 = schdlr_sspnd.msb.procs; + plp1; + plp1 = plp1->next) { + max_size += 2; + } + ASSERT(max_size); + hp = HAlloc(p, max_size); + hp_end = hp + max_size; + for (plp1 = schdlr_sspnd.msb.procs; plp1; plp1 = plp1->next) { + for (plp2 = schdlr_sspnd.msb.procs; + plp2->pid != plp1->pid; + plp2 = plp2->next); + if (plp2 == plp1) { + res = CONS(hp, plp1->pid, res); + hp += 2; + } + /* else: already in result list */ + } + HRelease(p, hp_end, hp); + } + erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + return res; +} + +static void * +sched_thread_func(void *vesdp) +{ +#ifdef ERTS_ENABLE_LOCK_CHECK + { + char buf[31]; + Uint no = ((ErtsSchedulerData *) vesdp)->no; + erts_snprintf(&buf[0], 31, "scheduler %bpu", no); + erts_lc_set_thread_name(&buf[0]); + } +#endif + erts_alloc_reg_scheduler_id(((ErtsSchedulerData *) vesdp)->no); + erts_tsd_set(sched_data_key, vesdp); +#ifdef ERTS_SMP + erts_proc_lock_prepare_proc_lock_waiter(); +#endif + erts_register_blockable_thread(); +#ifdef HIPE + hipe_thread_signal_init(); +#endif + erts_thread_init_float(); + erts_smp_mtx_lock(&schdlr_sspnd.mtx); + + ASSERT(schdlr_sspnd.changing == ERTS_SCHED_CHANGING_ONLINE); + + schdlr_sspnd.curr_online--; + + if (((ErtsSchedulerData *) vesdp)->no != 1) { + if (schdlr_sspnd.online == schdlr_sspnd.curr_online) { + schdlr_sspnd.changing = 0; + erts_smp_cnd_broadcast(&schdlr_sspnd.cnd); + } + } + else if (schdlr_sspnd.curr_online == schdlr_sspnd.wait_curr_online) + schdlr_sspnd.changing = 0; + else { + erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, + susp_sched_prep_block, + susp_sched_resume_block, + NULL); + while (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) + erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); + erts_smp_activity_end(ERTS_ACTIVITY_WAIT, + susp_sched_prep_block, + susp_sched_resume_block, + NULL); + ASSERT(!schdlr_sspnd.changing); + } + erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + + process_main(); + /* No schedulers should *ever* terminate */ + erl_exit(ERTS_ABORT_EXIT, "Scheduler thread number %bpu terminated\n", + ((ErtsSchedulerData *) vesdp)->no); + return NULL; +} + +void +erts_start_schedulers(void) +{ + int res = 0; + Uint actual = 0; + Uint wanted = erts_no_schedulers; + Uint wanted_no_schedulers = erts_no_schedulers; + ethr_thr_opts opts = ETHR_THR_OPTS_DEFAULT_INITER; + + opts.detached = 1; + opts.suggested_stack_size = erts_sched_thread_suggested_stack_size; + + if (wanted < 1) + wanted = 1; + if (wanted > ERTS_MAX_NO_OF_SCHEDULERS) { + wanted = ERTS_MAX_NO_OF_SCHEDULERS; + res = ENOTSUP; + } + + erts_block_system(0); + + while (actual < wanted) { + ErtsSchedulerData *esdp = ERTS_SCHEDULER_IX(actual); + actual++; + ASSERT(actual == esdp->no); +#ifdef ERTS_ENABLE_LOCK_COUNT + res = erts_lcnt_thr_create(&esdp->tid,sched_thread_func,(void*)esdp,&opts); +#else + res = ethr_thr_create(&esdp->tid,sched_thread_func,(void*)esdp,&opts); +#endif + if (res != 0) { + actual--; + break; + } + } + + erts_no_schedulers = actual; + erts_release_system(); + + if (actual < 1) + erl_exit(1, + "Failed to create any scheduler-threads: %s (%d)\n", + erl_errno_id(res), + res); + if (res != 0) { + erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); + ASSERT(actual != wanted_no_schedulers); + erts_dsprintf(dsbufp, + "Failed to create %bpu scheduler-threads (%s:%d); " + "only %bpu scheduler-thread%s created.\n", + wanted_no_schedulers, erl_errno_id(res), res, + actual, actual == 1 ? " was" : "s were"); + erts_send_error_to_logger_nogl(dsbufp); + } +} + +#endif /* ERTS_SMP */ + +static int +int_cmp(const void *vx, const void *vy) +{ + return *((int *) vx) - *((int *) vy); +} + +static int +cpu_spread_order_cmp(const void *vx, const void *vy) +{ + erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx; + erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy; + + if (x->thread != y->thread) + return x->thread - y->thread; + if (x->core != y->core) + return x->core - y->core; + if (x->processor_node != y->processor_node) + return x->processor_node - y->processor_node; + if (x->processor != y->processor) + return x->processor - y->processor; + if (x->node != y->node) + return x->node - y->node; + return 0; +} + +static int +cpu_processor_spread_order_cmp(const void *vx, const void *vy) +{ + erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx; + erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy; + + if (x->thread != y->thread) + return x->thread - y->thread; + if (x->processor_node != y->processor_node) + return x->processor_node - y->processor_node; + if (x->core != y->core) + return x->core - y->core; + if (x->node != y->node) + return x->node - y->node; + if (x->processor != y->processor) + return x->processor - y->processor; + return 0; +} + +static int +cpu_thread_spread_order_cmp(const void *vx, const void *vy) +{ + erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx; + erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy; + + if (x->thread != y->thread) + return x->thread - y->thread; + if (x->node != y->node) + return x->node - y->node; + if (x->processor != y->processor) + return x->processor - y->processor; + if (x->processor_node != y->processor_node) + return x->processor_node - y->processor_node; + if (x->core != y->core) + return x->core - y->core; + return 0; +} + +static int +cpu_thread_no_node_processor_spread_order_cmp(const void *vx, const void *vy) +{ + erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx; + erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy; + + if (x->thread != y->thread) + return x->thread - y->thread; + if (x->node != y->node) + return x->node - y->node; + if (x->core != y->core) + return x->core - y->core; + if (x->processor != y->processor) + return x->processor - y->processor; + return 0; +} + +static int +cpu_no_node_processor_spread_order_cmp(const void *vx, const void *vy) +{ + erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx; + erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy; + + if (x->node != y->node) + return x->node - y->node; + if (x->thread != y->thread) + return x->thread - y->thread; + if (x->core != y->core) + return x->core - y->core; + if (x->processor != y->processor) + return x->processor - y->processor; + return 0; +} + +static int +cpu_no_node_thread_spread_order_cmp(const void *vx, const void *vy) +{ + erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx; + erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy; + + if (x->node != y->node) + return x->node - y->node; + if (x->thread != y->thread) + return x->thread - y->thread; + if (x->processor != y->processor) + return x->processor - y->processor; + if (x->core != y->core) + return x->core - y->core; + return 0; +} + +static int +cpu_no_spread_order_cmp(const void *vx, const void *vy) +{ + erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx; + erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy; + + if (x->node != y->node) + return x->node - y->node; + if (x->processor != y->processor) + return x->processor - y->processor; + if (x->processor_node != y->processor_node) + return x->processor_node - y->processor_node; + if (x->core != y->core) + return x->core - y->core; + if (x->thread != y->thread) + return x->thread - y->thread; + return 0; +} + +static ERTS_INLINE void +make_cpudata_id_seq(erts_cpu_topology_t *cpudata, int size, int no_node) +{ + int ix; + int node = -1; + int processor = -1; + int processor_node = -1; + int processor_node_node = -1; + int core = -1; + int thread = -1; + int old_node = -1; + int old_processor = -1; + int old_processor_node = -1; + int old_core = -1; + int old_thread = -1; + + for (ix = 0; ix < size; ix++) { + if (!no_node || cpudata[ix].node >= 0) { + if (old_node == cpudata[ix].node) + cpudata[ix].node = node; + else { + old_node = cpudata[ix].node; + old_processor = processor = -1; + if (!no_node) + old_processor_node = processor_node = -1; + old_core = core = -1; + old_thread = thread = -1; + if (no_node || cpudata[ix].node >= 0) + cpudata[ix].node = ++node; + } + } + if (old_processor == cpudata[ix].processor) + cpudata[ix].processor = processor; + else { + old_processor = cpudata[ix].processor; + if (!no_node) + processor_node_node = old_processor_node = processor_node = -1; + old_core = core = -1; + old_thread = thread = -1; + cpudata[ix].processor = ++processor; + } + if (no_node && cpudata[ix].processor_node < 0) + old_processor_node = -1; + else { + if (old_processor_node == cpudata[ix].processor_node) { + if (no_node) + cpudata[ix].node = cpudata[ix].processor_node = node; + else { + if (processor_node_node >= 0) + cpudata[ix].node = processor_node_node; + cpudata[ix].processor_node = processor_node; + } + } + else { + old_processor_node = cpudata[ix].processor_node; + old_core = core = -1; + old_thread = thread = -1; + if (no_node) + cpudata[ix].node = cpudata[ix].processor_node = ++node; + else { + cpudata[ix].node = processor_node_node = ++node; + cpudata[ix].processor_node = ++processor_node; + } + } + } + if (!no_node && cpudata[ix].processor_node < 0) + cpudata[ix].processor_node = 0; + if (old_core == cpudata[ix].core) + cpudata[ix].core = core; + else { + old_core = cpudata[ix].core; + old_thread = thread = -1; + cpudata[ix].core = ++core; + } + if (old_thread == cpudata[ix].thread) + cpudata[ix].thread = thread; + else + old_thread = cpudata[ix].thread = ++thread; + } +} + +static void +cpu_bind_order_sort(erts_cpu_topology_t *cpudata, + int size, + ErtsCpuBindOrder bind_order, + int mk_seq) +{ + if (size > 1) { + int no_node = 0; + int (*cmp_func)(const void *, const void *); + switch (bind_order) { + case ERTS_CPU_BIND_SPREAD: + cmp_func = cpu_spread_order_cmp; + break; + case ERTS_CPU_BIND_PROCESSOR_SPREAD: + cmp_func = cpu_processor_spread_order_cmp; + break; + case ERTS_CPU_BIND_THREAD_SPREAD: + cmp_func = cpu_thread_spread_order_cmp; + break; + case ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD: + no_node = 1; + cmp_func = cpu_thread_no_node_processor_spread_order_cmp; + break; + case ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD: + no_node = 1; + cmp_func = cpu_no_node_processor_spread_order_cmp; + break; + case ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD: + no_node = 1; + cmp_func = cpu_no_node_thread_spread_order_cmp; + break; + case ERTS_CPU_BIND_NO_SPREAD: + cmp_func = cpu_no_spread_order_cmp; + break; + default: + cmp_func = NULL; + erl_exit(ERTS_ABORT_EXIT, + "Bad cpu bind type: %d\n", + (int) cpu_bind_order); + break; + } + + if (mk_seq) + make_cpudata_id_seq(cpudata, size, no_node); + + qsort(cpudata, size, sizeof(erts_cpu_topology_t), cmp_func); + } +} + +static int +processor_order_cmp(const void *vx, const void *vy) +{ + erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx; + erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy; + + if (x->processor != y->processor) + return x->processor - y->processor; + if (x->node != y->node) + return x->node - y->node; + if (x->processor_node != y->processor_node) + return x->processor_node - y->processor_node; + if (x->core != y->core) + return x->core - y->core; + if (x->thread != y->thread) + return x->thread - y->thread; + return 0; +} + +static void +check_cpu_bind(ErtsSchedulerData *esdp) +{ + int res; + int cpu_id; + erts_smp_runq_unlock(esdp->run_queue); + erts_smp_rwmtx_rwlock(&erts_cpu_bind_rwmtx); + cpu_id = scheduler2cpu_map[esdp->no].bind_id; + if (cpu_id >= 0 && cpu_id != scheduler2cpu_map[esdp->no].bound_id) { + res = erts_bind_to_cpu(erts_cpuinfo, cpu_id); + if (res == 0) + esdp->cpu_id = scheduler2cpu_map[esdp->no].bound_id = cpu_id; + else { + erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); + erts_dsprintf(dsbufp, "Scheduler %d failed to bind to cpu %d: %s\n", + (int) esdp->no, cpu_id, erl_errno_id(-res)); + erts_send_error_to_logger_nogl(dsbufp); + if (scheduler2cpu_map[esdp->no].bound_id >= 0) + goto unbind; + } + } + else if (cpu_id < 0 && scheduler2cpu_map[esdp->no].bound_id >= 0) { + unbind: + /* Get rid of old binding */ + res = erts_unbind_from_cpu(erts_cpuinfo); + if (res == 0) + esdp->cpu_id = scheduler2cpu_map[esdp->no].bound_id = -1; + else { + erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); + erts_dsprintf(dsbufp, "Scheduler %d failed to unbind from cpu %d: %s\n", + (int) esdp->no, cpu_id, erl_errno_id(-res)); + erts_send_error_to_logger_nogl(dsbufp); + } + } + erts_smp_runq_lock(esdp->run_queue); +#ifdef ERTS_SMP + if (erts_common_run_queue) + erts_smp_atomic_set(&esdp->chk_cpu_bind, 0); + else { + esdp->run_queue->flags &= ~ERTS_RUNQ_FLG_CHK_CPU_BIND; + } +#endif + erts_smp_rwmtx_rwunlock(&erts_cpu_bind_rwmtx); + +} + +static void +signal_schedulers_bind_change(erts_cpu_topology_t *cpudata, int size) +{ + int s_ix = 1; + int cpu_ix; + + if (cpu_bind_order != ERTS_CPU_BIND_NONE) { + + cpu_bind_order_sort(cpudata, size, cpu_bind_order, 1); + + for (cpu_ix = 0; cpu_ix < size && cpu_ix < erts_no_schedulers; cpu_ix++) + if (erts_is_cpu_available(erts_cpuinfo, cpudata[cpu_ix].logical)) + scheduler2cpu_map[s_ix++].bind_id = cpudata[cpu_ix].logical; + } + + if (s_ix <= erts_no_schedulers) + for (; s_ix <= erts_no_schedulers; s_ix++) + scheduler2cpu_map[s_ix].bind_id = -1; + +#ifdef ERTS_SMP + if (erts_common_run_queue) { + for (s_ix = 0; s_ix < erts_no_schedulers; s_ix++) + erts_smp_atomic_set(&ERTS_SCHEDULER_IX(s_ix)->chk_cpu_bind, 1); + wake_all_schedulers(); + } + else { + ERTS_FOREACH_RUNQ(rq, + { + rq->flags |= ERTS_RUNQ_FLG_CHK_CPU_BIND; + wake_scheduler(rq, 0); + }); + } +#else + check_cpu_bind(erts_get_scheduler_data()); +#endif +} + +int +erts_init_scheduler_bind_type(char *how) +{ + if (erts_bind_to_cpu(erts_cpuinfo, -1) == -ENOTSUP) + return ERTS_INIT_SCHED_BIND_TYPE_NOT_SUPPORTED; + + if (!system_cpudata && !user_cpudata) + return ERTS_INIT_SCHED_BIND_TYPE_ERROR_NO_CPU_TOPOLOGY; + + if (sys_strcmp(how, "s") == 0) + cpu_bind_order = ERTS_CPU_BIND_SPREAD; + else if (sys_strcmp(how, "ps") == 0) + cpu_bind_order = ERTS_CPU_BIND_PROCESSOR_SPREAD; + else if (sys_strcmp(how, "ts") == 0) + cpu_bind_order = ERTS_CPU_BIND_THREAD_SPREAD; + else if (sys_strcmp(how, "db") == 0 + || sys_strcmp(how, "tnnps") == 0) + cpu_bind_order = ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD; + else if (sys_strcmp(how, "nnps") == 0) + cpu_bind_order = ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD; + else if (sys_strcmp(how, "nnts") == 0) + cpu_bind_order = ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD; + else if (sys_strcmp(how, "ns") == 0) + cpu_bind_order = ERTS_CPU_BIND_NO_SPREAD; + else if (sys_strcmp(how, "u") == 0) + cpu_bind_order = ERTS_CPU_BIND_NONE; + else + return ERTS_INIT_SCHED_BIND_TYPE_ERROR_NO_BAD_TYPE; + + return ERTS_INIT_SCHED_BIND_TYPE_SUCCESS; +} + +typedef struct { + int *id; + int used; + int size; +} ErtsCpuTopIdSeq; + +typedef struct { + ErtsCpuTopIdSeq logical; + ErtsCpuTopIdSeq thread; + ErtsCpuTopIdSeq core; + ErtsCpuTopIdSeq processor_node; + ErtsCpuTopIdSeq processor; + ErtsCpuTopIdSeq node; +} ErtsCpuTopEntry; + +static void +init_cpu_top_entry(ErtsCpuTopEntry *cte) +{ + int size = 10; + cte->logical.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS, + sizeof(int)*size); + cte->logical.size = size; + cte->thread.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS, + sizeof(int)*size); + cte->thread.size = size; + cte->core.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS, + sizeof(int)*size); + cte->core.size = size; + cte->processor_node.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS, + sizeof(int)*size); + cte->processor_node.size = size; + cte->processor.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS, + sizeof(int)*size); + cte->processor.size = size; + cte->node.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS, + sizeof(int)*size); + cte->node.size = size; +} + +static void +destroy_cpu_top_entry(ErtsCpuTopEntry *cte) +{ + erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->logical.id); + erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->thread.id); + erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->core.id); + erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->processor_node.id); + erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->processor.id); + erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->node.id); +} + +static int +get_cput_value_or_range(int *v, int *vr, char **str) +{ + long l; + char *c = *str; + errno = 0; + if (!isdigit((unsigned char)*c)) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID; + l = strtol(c, &c, 10); + if (errno != 0 || l < 0 || ERTS_MAX_CPU_TOPOLOGY_ID < l) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID; + *v = (int) l; + if (*c == '-') { + c++; + if (!isdigit((unsigned char)*c)) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE; + l = strtol(c, &c, 10); + if (errno != 0 || l < 0 || ERTS_MAX_CPU_TOPOLOGY_ID < l) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE; + *vr = (int) l; + } + *str = c; + return ERTS_INIT_CPU_TOPOLOGY_OK; +} + +static int +get_cput_id_seq(ErtsCpuTopIdSeq *idseq, char **str) +{ + int ix = 0; + int need_size = 0; + char *c = *str; + + while (1) { + int res; + int val; + int nids; + int val_range = -1; + res = get_cput_value_or_range(&val, &val_range, &c); + if (res != ERTS_INIT_CPU_TOPOLOGY_OK) + return res; + if (val_range < 0 || val_range == val) + nids = 1; + else { + if (val_range > val) + nids = val_range - val + 1; + else + nids = val - val_range + 1; + } + need_size += nids; + if (need_size > idseq->size) { + idseq->size = need_size + 10; + idseq->id = erts_realloc(ERTS_ALC_T_TMP_CPU_IDS, + idseq->id, + sizeof(int)*idseq->size); + } + if (nids == 1) + idseq->id[ix++] = val; + else if (val_range > val) { + for (; val <= val_range; val++) + idseq->id[ix++] = val; + } + else { + for (; val >= val_range; val--) + idseq->id[ix++] = val; + } + if (*c != ',') + break; + c++; + } + *str = c; + idseq->used = ix; + return ERTS_INIT_CPU_TOPOLOGY_OK; +} + +static int +get_cput_entry(ErtsCpuTopEntry *cput, char **str) +{ + int h; + char *c = *str; + + cput->logical.used = 0; + cput->thread.id[0] = 0; + cput->thread.used = 1; + cput->core.id[0] = 0; + cput->core.used = 1; + cput->processor_node.id[0] = -1; + cput->processor_node.used = 1; + cput->processor.id[0] = 0; + cput->processor.used = 1; + cput->node.id[0] = -1; + cput->node.used = 1; + + h = ERTS_TOPOLOGY_MAX_DEPTH; + while (*c != ':' && *c != '\0') { + int res; + ErtsCpuTopIdSeq *idseqp; + switch (*c++) { + case 'L': + if (h <= ERTS_TOPOLOGY_LOGICAL) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY; + idseqp = &cput->logical; + h = ERTS_TOPOLOGY_LOGICAL; + break; + case 't': + case 'T': + if (h <= ERTS_TOPOLOGY_THREAD) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY; + idseqp = &cput->thread; + h = ERTS_TOPOLOGY_THREAD; + break; + case 'c': + case 'C': + if (h <= ERTS_TOPOLOGY_CORE) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY; + idseqp = &cput->core; + h = ERTS_TOPOLOGY_CORE; + break; + case 'p': + case 'P': + if (h <= ERTS_TOPOLOGY_PROCESSOR) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY; + idseqp = &cput->processor; + h = ERTS_TOPOLOGY_PROCESSOR; + break; + case 'n': + case 'N': + if (h <= ERTS_TOPOLOGY_PROCESSOR) { + do_node: + if (h <= ERTS_TOPOLOGY_NODE) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY; + idseqp = &cput->node; + h = ERTS_TOPOLOGY_NODE; + } + else { + int p_node = 0; + char *p_chk = c; + while (*p_chk != '\0' && *p_chk != ':') { + if (*p_chk == 'p' || *p_chk == 'P') { + p_node = 1; + break; + } + p_chk++; + } + if (!p_node) + goto do_node; + if (h <= ERTS_TOPOLOGY_PROCESSOR_NODE) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY; + idseqp = &cput->processor_node; + h = ERTS_TOPOLOGY_PROCESSOR_NODE; + } + break; + default: + return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_TYPE; + } + res = get_cput_id_seq(idseqp, &c); + if (res != ERTS_INIT_CPU_TOPOLOGY_OK) + return res; + } + + if (cput->logical.used < 1) + return ERTS_INIT_CPU_TOPOLOGY_MISSING_LID; + + if (*c == ':') { + c++; + } + + if (cput->thread.used != 1 + && cput->thread.used != cput->logical.used) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE; + if (cput->core.used != 1 + && cput->core.used != cput->logical.used) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE; + if (cput->processor_node.used != 1 + && cput->processor_node.used != cput->logical.used) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE; + if (cput->processor.used != 1 + && cput->processor.used != cput->logical.used) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE; + if (cput->node.used != 1 + && cput->node.used != cput->logical.used) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE; + + *str = c; + return ERTS_INIT_CPU_TOPOLOGY_OK; +} + +static int +verify_topology(erts_cpu_topology_t *cpudata, int size) +{ + if (size > 0) { + int *logical; + int node, processor, no_nodes, i; + + /* Verify logical ids */ + logical = erts_alloc(ERTS_ALC_T_TMP, sizeof(int)*size); + + for (i = 0; i < user_cpudata_size; i++) + logical[i] = user_cpudata[i].logical; + + qsort(logical, user_cpudata_size, sizeof(int), int_cmp); + for (i = 0; i < user_cpudata_size-1; i++) { + if (logical[i] == logical[i+1]) { + erts_free(ERTS_ALC_T_TMP, logical); + return ERTS_INIT_CPU_TOPOLOGY_NOT_UNIQUE_LIDS; + } + } + + erts_free(ERTS_ALC_T_TMP, logical); + + qsort(cpudata, size, sizeof(erts_cpu_topology_t), processor_order_cmp); + + /* Verify unique entities */ + + for (i = 1; i < user_cpudata_size; i++) { + if (user_cpudata[i-1].processor == user_cpudata[i].processor + && user_cpudata[i-1].node == user_cpudata[i].node + && (user_cpudata[i-1].processor_node + == user_cpudata[i].processor_node) + && user_cpudata[i-1].core == user_cpudata[i].core + && user_cpudata[i-1].thread == user_cpudata[i].thread) { + return ERTS_INIT_CPU_TOPOLOGY_NOT_UNIQUE_ENTITIES; + } + } + + /* Verify numa nodes */ + node = cpudata[0].node; + processor = cpudata[0].processor; + no_nodes = cpudata[0].node < 0 && cpudata[0].processor_node < 0; + for (i = 1; i < size; i++) { + if (no_nodes) { + if (cpudata[i].node >= 0 || cpudata[i].processor_node >= 0) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_NODES; + } + else { + if (cpudata[i].processor == processor && cpudata[i].node != node) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_NODES; + node = cpudata[i].node; + processor = cpudata[i].processor; + if (node >= 0 && cpudata[i].processor_node >= 0) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_NODES; + if (node < 0 && cpudata[i].processor_node < 0) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_NODES; + } + } + } + + return ERTS_INIT_CPU_TOPOLOGY_OK; +} + +int +erts_init_cpu_topology(char *topology_str) +{ + ErtsCpuTopEntry cput; + int need_size; + char *c; + int ix; + int error = ERTS_INIT_CPU_TOPOLOGY_OK; + + if (user_cpudata) + erts_free(ERTS_ALC_T_CPUDATA, user_cpudata); + user_cpudata_size = 10; + + user_cpudata = erts_alloc(ERTS_ALC_T_CPUDATA, + (sizeof(erts_cpu_topology_t) + * user_cpudata_size)); + + init_cpu_top_entry(&cput); + + ix = 0; + need_size = 0; + + c = topology_str; + if (*c == '\0') { + error = ERTS_INIT_CPU_TOPOLOGY_MISSING; + goto fail; + } + do { + int r; + error = get_cput_entry(&cput, &c); + if (error != ERTS_INIT_CPU_TOPOLOGY_OK) + goto fail; + need_size += cput.logical.used; + if (user_cpudata_size < need_size) { + user_cpudata_size = need_size + 10; + user_cpudata = erts_realloc(ERTS_ALC_T_CPUDATA, + user_cpudata, + (sizeof(erts_cpu_topology_t) + * user_cpudata_size)); + } + + ASSERT(cput.thread.used == 1 + || cput.thread.used == cput.logical.used); + ASSERT(cput.core.used == 1 + || cput.core.used == cput.logical.used); + ASSERT(cput.processor_node.used == 1 + || cput.processor_node.used == cput.logical.used); + ASSERT(cput.processor.used == 1 + || cput.processor.used == cput.logical.used); + ASSERT(cput.node.used == 1 + || cput.node.used == cput.logical.used); + + for (r = 0; r < cput.logical.used; r++) { + user_cpudata[ix].logical = cput.logical.id[r]; + user_cpudata[ix].thread = + cput.thread.id[cput.thread.used == 1 ? 0 : r]; + user_cpudata[ix].core = + cput.core.id[cput.core.used == 1 ? 0 : r]; + user_cpudata[ix].processor_node = + cput.processor_node.id[cput.processor_node.used == 1 ? 0 : r]; + user_cpudata[ix].processor = + cput.processor.id[cput.processor.used == 1 ? 0 : r]; + user_cpudata[ix].node = + cput.node.id[cput.node.used == 1 ? 0 : r]; + ix++; + } + } while (*c != '\0'); + + if (user_cpudata_size != ix) { + user_cpudata_size = ix; + user_cpudata = erts_realloc(ERTS_ALC_T_CPUDATA, + user_cpudata, + (sizeof(erts_cpu_topology_t) + * user_cpudata_size)); + } + + error = verify_topology(user_cpudata, user_cpudata_size); + if (error == ERTS_INIT_CPU_TOPOLOGY_OK) { + destroy_cpu_top_entry(&cput); + return ERTS_INIT_CPU_TOPOLOGY_OK; + } + + fail: + if (user_cpudata) + erts_free(ERTS_ALC_T_CPUDATA, user_cpudata); + user_cpudata_size = 0; + destroy_cpu_top_entry(&cput); + return error; +} + +#define ERTS_GET_CPU_TOPOLOGY_ERROR -1 +#define ERTS_GET_USED_CPU_TOPOLOGY 0 +#define ERTS_GET_DETECTED_CPU_TOPOLOGY 1 +#define ERTS_GET_DEFINED_CPU_TOPOLOGY 2 + +static Eterm get_cpu_topology_term(Process *c_p, int type); + +Eterm +erts_set_cpu_topology(Process *c_p, Eterm term) +{ + erts_cpu_topology_t *cpudata = NULL; + int cpudata_size = 0; + Eterm res; + + erts_smp_rwmtx_rwlock(&erts_cpu_bind_rwmtx); + res = get_cpu_topology_term(c_p, ERTS_GET_USED_CPU_TOPOLOGY); + if (term == am_undefined) { + if (user_cpudata) + erts_free(ERTS_ALC_T_CPUDATA, user_cpudata); + user_cpudata = NULL; + user_cpudata_size = 0; + + if (cpu_bind_order != ERTS_CPU_BIND_NONE && system_cpudata) { + cpudata_size = system_cpudata_size; + cpudata = erts_alloc(ERTS_ALC_T_TMP, + (sizeof(erts_cpu_topology_t) + * cpudata_size)); + + sys_memcpy((void *) cpudata, + (void *) system_cpudata, + sizeof(erts_cpu_topology_t)*cpudata_size); + } + } + else if (is_not_list(term)) { + error: + res = THE_NON_VALUE; + goto done; + } + else { + Eterm list = term; + int ix = 0; + + cpudata_size = 100; + cpudata = erts_alloc(ERTS_ALC_T_TMP, + (sizeof(erts_cpu_topology_t) + * cpudata_size)); + + while (is_list(list)) { + Eterm *lp = list_val(list); + Eterm cpu = CAR(lp); + Eterm* tp; + Sint id; + + if (is_not_tuple(cpu)) + goto error; + + tp = tuple_val(cpu); + + if (arityval(tp[0]) != 7 || tp[1] != am_cpu) + goto error; + + if (ix >= cpudata_size) { + cpudata_size += 100; + cpudata = erts_realloc(ERTS_ALC_T_TMP, + cpudata, + (sizeof(erts_cpu_topology_t) + * cpudata_size)); + } + + id = signed_val(tp[2]); + if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id) + goto error; + cpudata[ix].node = (int) id; + + id = signed_val(tp[3]); + if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id) + goto error; + cpudata[ix].processor = (int) id; + + id = signed_val(tp[4]); + if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id) + goto error; + cpudata[ix].processor_node = (int) id; + + id = signed_val(tp[5]); + if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id) + goto error; + cpudata[ix].core = (int) id; + + id = signed_val(tp[6]); + if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id) + goto error; + cpudata[ix].thread = (int) id; + + id = signed_val(tp[7]); + if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id) + goto error; + cpudata[ix].logical = (int) id; + + list = CDR(lp); + ix++; + } + + if (is_not_nil(list)) + goto error; + + cpudata_size = ix; + + if (ERTS_INIT_CPU_TOPOLOGY_OK != verify_topology(cpudata, cpudata_size)) + goto error; + + if (user_cpudata_size != cpudata_size) { + if (user_cpudata) + erts_free(ERTS_ALC_T_CPUDATA, user_cpudata); + user_cpudata = erts_alloc(ERTS_ALC_T_CPUDATA, + sizeof(erts_cpu_topology_t)*cpudata_size); + user_cpudata_size = cpudata_size; + } + + sys_memcpy((void *) user_cpudata, + (void *) cpudata, + sizeof(erts_cpu_topology_t)*cpudata_size); + } + + signal_schedulers_bind_change(cpudata, cpudata_size); + + done: + erts_smp_rwmtx_rwunlock(&erts_cpu_bind_rwmtx); + + if (cpudata) + erts_free(ERTS_ALC_T_TMP, cpudata); + + return res; +} + +static Eterm +bound_schedulers_term(ErtsCpuBindOrder order) +{ + switch (order) { + case ERTS_CPU_BIND_SPREAD: { + ERTS_DECL_AM(spread); + return AM_spread; + } + case ERTS_CPU_BIND_PROCESSOR_SPREAD: { + ERTS_DECL_AM(processor_spread); + return AM_processor_spread; + } + case ERTS_CPU_BIND_THREAD_SPREAD: { + ERTS_DECL_AM(thread_spread); + return AM_thread_spread; + } + case ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD: { + ERTS_DECL_AM(thread_no_node_processor_spread); + return AM_thread_no_node_processor_spread; + } + case ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD: { + ERTS_DECL_AM(no_node_processor_spread); + return AM_no_node_processor_spread; + } + case ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD: { + ERTS_DECL_AM(no_node_thread_spread); + return AM_no_node_thread_spread; + } + case ERTS_CPU_BIND_NO_SPREAD: { + ERTS_DECL_AM(no_spread); + return AM_no_spread; + } + case ERTS_CPU_BIND_NONE: { + ERTS_DECL_AM(unbound); + return AM_unbound; + } + default: + ASSERT(0); + return THE_NON_VALUE; + } +} + +Eterm +erts_bound_schedulers_term(Process *c_p) +{ + ErtsCpuBindOrder order; + erts_smp_rwmtx_rlock(&erts_cpu_bind_rwmtx); + order = cpu_bind_order; + erts_smp_rwmtx_runlock(&erts_cpu_bind_rwmtx); + return bound_schedulers_term(order); +} + +static void +create_tmp_cpu_topology_copy(erts_cpu_topology_t **cpudata, int *cpudata_size) +{ + if (user_cpudata) { + *cpudata_size = user_cpudata_size; + *cpudata = erts_alloc(ERTS_ALC_T_TMP, + (sizeof(erts_cpu_topology_t) + * (*cpudata_size))); + sys_memcpy((void *) *cpudata, + (void *) user_cpudata, + sizeof(erts_cpu_topology_t)*(*cpudata_size)); + } + else if (system_cpudata) { + *cpudata_size = system_cpudata_size; + *cpudata = erts_alloc(ERTS_ALC_T_TMP, + (sizeof(erts_cpu_topology_t) + * (*cpudata_size))); + sys_memcpy((void *) *cpudata, + (void *) system_cpudata, + sizeof(erts_cpu_topology_t)*(*cpudata_size)); + } + else { + *cpudata = NULL; + *cpudata_size = 0; + } +} + +static void +destroy_tmp_cpu_topology_copy(erts_cpu_topology_t *cpudata) +{ + if (cpudata) + erts_free(ERTS_ALC_T_TMP, cpudata); +} + +Eterm +erts_bind_schedulers(Process *c_p, Eterm how) +{ + Eterm res; + erts_cpu_topology_t *cpudata; + int cpudata_size; + ErtsCpuBindOrder old_cpu_bind_order; + + erts_smp_rwmtx_rwlock(&erts_cpu_bind_rwmtx); + + if (erts_bind_to_cpu(erts_cpuinfo, -1) == -ENOTSUP) { + ERTS_BIF_PREP_ERROR(res, c_p, EXC_NOTSUP); + } + else { + + old_cpu_bind_order = cpu_bind_order; + + if (ERTS_IS_ATOM_STR("spread", how)) + cpu_bind_order = ERTS_CPU_BIND_SPREAD; + else if (ERTS_IS_ATOM_STR("processor_spread", how)) + cpu_bind_order = ERTS_CPU_BIND_PROCESSOR_SPREAD; + else if (ERTS_IS_ATOM_STR("thread_spread", how)) + cpu_bind_order = ERTS_CPU_BIND_THREAD_SPREAD; + else if (ERTS_IS_ATOM_STR("default_bind", how) + || ERTS_IS_ATOM_STR("thread_no_node_processor_spread", how)) + cpu_bind_order = ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD; + else if (ERTS_IS_ATOM_STR("no_node_processor_spread", how)) + cpu_bind_order = ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD; + else if (ERTS_IS_ATOM_STR("no_node_thread_spread", how)) + cpu_bind_order = ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD; + else if (ERTS_IS_ATOM_STR("no_spread", how)) + cpu_bind_order = ERTS_CPU_BIND_NO_SPREAD; + else if (ERTS_IS_ATOM_STR("unbound", how)) + cpu_bind_order = ERTS_CPU_BIND_NONE; + else { + cpu_bind_order = old_cpu_bind_order; + ERTS_BIF_PREP_ERROR(res, c_p, BADARG); + goto done; + } + + create_tmp_cpu_topology_copy(&cpudata, &cpudata_size); + + if (!cpudata) { + cpu_bind_order = old_cpu_bind_order; + ERTS_BIF_PREP_ERROR(res, c_p, BADARG); + goto done; + } + + signal_schedulers_bind_change(cpudata, cpudata_size); + + destroy_tmp_cpu_topology_copy(cpudata); + + res = bound_schedulers_term(old_cpu_bind_order); + } + + done: + + erts_smp_rwmtx_rwunlock(&erts_cpu_bind_rwmtx); + + return res; +} + +Eterm +erts_fake_scheduler_bindings(Process *p, Eterm how) +{ + ErtsCpuBindOrder fake_cpu_bind_order; + erts_cpu_topology_t *cpudata; + int cpudata_size; + Eterm res; + + if (ERTS_IS_ATOM_STR("spread", how)) + fake_cpu_bind_order = ERTS_CPU_BIND_SPREAD; + else if (ERTS_IS_ATOM_STR("processor_spread", how)) + fake_cpu_bind_order = ERTS_CPU_BIND_PROCESSOR_SPREAD; + else if (ERTS_IS_ATOM_STR("thread_spread", how)) + fake_cpu_bind_order = ERTS_CPU_BIND_THREAD_SPREAD; + else if (ERTS_IS_ATOM_STR("default_bind", how) + || ERTS_IS_ATOM_STR("thread_no_node_processor_spread", how)) + fake_cpu_bind_order = ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD; + else if (ERTS_IS_ATOM_STR("no_node_processor_spread", how)) + fake_cpu_bind_order = ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD; + else if (ERTS_IS_ATOM_STR("no_node_thread_spread", how)) + fake_cpu_bind_order = ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD; + else if (ERTS_IS_ATOM_STR("no_spread", how)) + fake_cpu_bind_order = ERTS_CPU_BIND_NO_SPREAD; + else if (ERTS_IS_ATOM_STR("unbound", how)) + fake_cpu_bind_order = ERTS_CPU_BIND_NONE; + else { + ERTS_BIF_PREP_ERROR(res, p, BADARG); + return res; + } + + erts_smp_rwmtx_rlock(&erts_cpu_bind_rwmtx); + create_tmp_cpu_topology_copy(&cpudata, &cpudata_size); + erts_smp_rwmtx_runlock(&erts_cpu_bind_rwmtx); + + if (!cpudata || fake_cpu_bind_order == ERTS_CPU_BIND_NONE) + ERTS_BIF_PREP_RET(res, am_false); + else { + int i; + Eterm *hp; + + cpu_bind_order_sort(cpudata, cpudata_size, fake_cpu_bind_order, 1); + +#ifdef ERTS_FAKE_SCHED_BIND_PRINT_SORTED_CPU_DATA + + erts_fprintf(stderr, "node: "); + for (i = 0; i < cpudata_size; i++) + erts_fprintf(stderr, " %2d", cpudata[i].node); + erts_fprintf(stderr, "\n"); + erts_fprintf(stderr, "processor: "); + for (i = 0; i < cpudata_size; i++) + erts_fprintf(stderr, " %2d", cpudata[i].processor); + erts_fprintf(stderr, "\n"); + if (fake_cpu_bind_order != ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD + && fake_cpu_bind_order != ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD + && fake_cpu_bind_order != ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD) { + erts_fprintf(stderr, "processor_node:"); + for (i = 0; i < cpudata_size; i++) + erts_fprintf(stderr, " %2d", cpudata[i].processor_node); + erts_fprintf(stderr, "\n"); + } + erts_fprintf(stderr, "core: "); + for (i = 0; i < cpudata_size; i++) + erts_fprintf(stderr, " %2d", cpudata[i].core); + erts_fprintf(stderr, "\n"); + erts_fprintf(stderr, "thread: "); + for (i = 0; i < cpudata_size; i++) + erts_fprintf(stderr, " %2d", cpudata[i].thread); + erts_fprintf(stderr, "\n"); + erts_fprintf(stderr, "logical: "); + for (i = 0; i < cpudata_size; i++) + erts_fprintf(stderr, " %2d", cpudata[i].logical); + erts_fprintf(stderr, "\n"); +#endif + + hp = HAlloc(p, cpudata_size+1); + ERTS_BIF_PREP_RET(res, make_tuple(hp)); + *hp++ = make_arityval((Uint) cpudata_size); + for (i = 0; i < cpudata_size; i++) + *hp++ = make_small((Uint) cpudata[i].logical); + } + + destroy_tmp_cpu_topology_copy(cpudata); + + return res; +} + +Eterm +erts_get_schedulers_binds(Process *c_p) +{ + int ix; + ERTS_DECL_AM(unbound); + Eterm *hp = HAlloc(c_p, erts_no_schedulers+1); + Eterm res = make_tuple(hp); + + *(hp++) = make_arityval(erts_no_schedulers); + erts_smp_rwmtx_rlock(&erts_cpu_bind_rwmtx); + for (ix = 1; ix <= erts_no_schedulers; ix++) + *(hp++) = (scheduler2cpu_map[ix].bound_id >= 0 + ? make_small(scheduler2cpu_map[ix].bound_id) + : AM_unbound); + erts_smp_rwmtx_runlock(&erts_cpu_bind_rwmtx); + return res; +} + +static Eterm +bld_topology_term(Eterm **hpp, + Uint *hszp, + erts_cpu_topology_t *cpudata, + int size) +{ + Eterm res = NIL; + int i; + + if (size == 0) + return am_undefined; + + for (i = size-1; i >= 0; i--) { + res = erts_bld_cons(hpp, + hszp, + erts_bld_tuple(hpp, + hszp, + 7, + am_cpu, + make_small(cpudata[i].node), + make_small(cpudata[i].processor), + make_small(cpudata[i].processor_node), + make_small(cpudata[i].core), + make_small(cpudata[i].thread), + make_small(cpudata[i].logical)), + res); + } + return res; +} + +static Eterm +get_cpu_topology_term(Process *c_p, int type) +{ +#ifdef DEBUG + Eterm *hp_end; +#endif + Eterm *hp; + Uint hsz; + Eterm res = THE_NON_VALUE; + erts_cpu_topology_t *cpudata = NULL; + int size = 0; + + switch (type) { + case ERTS_GET_USED_CPU_TOPOLOGY: + if (user_cpudata) + goto defined; + else + goto detected; + case ERTS_GET_DETECTED_CPU_TOPOLOGY: + detected: + if (!system_cpudata) + res = am_undefined; + else { + size = system_cpudata_size; + cpudata = erts_alloc(ERTS_ALC_T_TMP, + (sizeof(erts_cpu_topology_t) + * size)); + sys_memcpy((void *) cpudata, + (void *) system_cpudata, + sizeof(erts_cpu_topology_t)*size); + } + break; + case ERTS_GET_DEFINED_CPU_TOPOLOGY: + defined: + if (!user_cpudata) + res = am_undefined; + else { + size = user_cpudata_size; + cpudata = user_cpudata; + } + break; + default: + erl_exit(ERTS_ABORT_EXIT, "Bad cpu topology type: %d\n", type); + break; + } + + if (res == am_undefined) { + ASSERT(!cpudata); + return res; + } + + hsz = 0; + + bld_topology_term(NULL, &hsz, + cpudata, size); + + hp = HAlloc(c_p, hsz); + +#ifdef DEBUG + hp_end = hp + hsz; +#endif + + res = bld_topology_term(&hp, NULL, + cpudata, size); + + ASSERT(hp_end == hp); + + if (cpudata && cpudata != system_cpudata && cpudata != user_cpudata) + erts_free(ERTS_ALC_T_TMP, cpudata); + + return res; +} + +Eterm +erts_get_cpu_topology_term(Process *c_p, Eterm which) +{ + Eterm res; + int type; + erts_smp_rwmtx_rlock(&erts_cpu_bind_rwmtx); + if (ERTS_IS_ATOM_STR("used", which)) + type = ERTS_GET_USED_CPU_TOPOLOGY; + else if (ERTS_IS_ATOM_STR("detected", which)) + type = ERTS_GET_DETECTED_CPU_TOPOLOGY; + else if (ERTS_IS_ATOM_STR("defined", which)) + type = ERTS_GET_DEFINED_CPU_TOPOLOGY; + else + type = ERTS_GET_CPU_TOPOLOGY_ERROR; + if (type == ERTS_GET_CPU_TOPOLOGY_ERROR) + res = THE_NON_VALUE; + else + res = get_cpu_topology_term(c_p, type); + erts_smp_rwmtx_runlock(&erts_cpu_bind_rwmtx); + return res; +} + +static void +early_cpu_bind_init(void) +{ + user_cpudata = NULL; + user_cpudata_size = 0; + + system_cpudata_size = erts_get_cpu_topology_size(erts_cpuinfo); + system_cpudata = erts_alloc(ERTS_ALC_T_CPUDATA, + (sizeof(erts_cpu_topology_t) + * system_cpudata_size)); + + cpu_bind_order = ERTS_CPU_BIND_NONE; + + if (!erts_get_cpu_topology(erts_cpuinfo, system_cpudata) + || ERTS_INIT_CPU_TOPOLOGY_OK != verify_topology(system_cpudata, + system_cpudata_size)) { + erts_free(ERTS_ALC_T_CPUDATA, system_cpudata); + system_cpudata = NULL; + system_cpudata_size = 0; + } +} + +static void +late_cpu_bind_init(void) +{ + int ix; + + erts_smp_rwmtx_init(&erts_cpu_bind_rwmtx, "cpu_bind"); + + scheduler2cpu_map = erts_alloc(ERTS_ALC_T_CPUDATA, + (sizeof(ErtsCpuBindData) + * (erts_no_schedulers+1))); + for (ix = 1; ix <= erts_no_schedulers; ix++) { + scheduler2cpu_map[ix].bind_id = -1; + scheduler2cpu_map[ix].bound_id = -1; + } + + if (cpu_bind_order != ERTS_CPU_BIND_NONE) { + erts_cpu_topology_t *cpudata; + int cpudata_size; + create_tmp_cpu_topology_copy(&cpudata, &cpudata_size); + ASSERT(cpudata); + signal_schedulers_bind_change(cpudata, cpudata_size); + destroy_tmp_cpu_topology_copy(cpudata); + } +} + +#ifdef ERTS_SMP + +static void +add_pend_suspend(Process *suspendee, + Eterm originator_pid, + void (*handle_func)(Process *, + ErtsProcLocks, + int, + Eterm)) +{ + ErtsPendingSuspend *psp = erts_alloc(ERTS_ALC_T_PEND_SUSPEND, + sizeof(ErtsPendingSuspend)); + psp->next = NULL; +#ifdef DEBUG +#ifdef ARCH_64 + psp->end = (ErtsPendingSuspend *) 0xdeaddeaddeaddead; +#else + psp->end = (ErtsPendingSuspend *) 0xdeaddead; +#endif +#endif + psp->pid = originator_pid; + psp->handle_func = handle_func; + + if (suspendee->pending_suspenders) + suspendee->pending_suspenders->end->next = psp; + else + suspendee->pending_suspenders = psp; + suspendee->pending_suspenders->end = psp; +} + +static void +handle_pending_suspend(Process *p, ErtsProcLocks p_locks) +{ + ErtsPendingSuspend *psp; + int is_alive = !ERTS_PROC_IS_EXITING(p); + + ERTS_SMP_LC_ASSERT(p_locks & ERTS_PROC_LOCK_STATUS); + + /* + * New pending suspenders might appear while we are processing + * (since we may release the status lock on p while processing). + */ + while (p->pending_suspenders) { + psp = p->pending_suspenders; + p->pending_suspenders = NULL; + while (psp) { + ErtsPendingSuspend *free_psp; + (*psp->handle_func)(p, p_locks, is_alive, psp->pid); + free_psp = psp; + psp = psp->next; + erts_free(ERTS_ALC_T_PEND_SUSPEND, (void *) free_psp); + } + } + +} + +static ERTS_INLINE void +cancel_suspend_of_suspendee(Process *p, ErtsProcLocks p_locks) +{ + if (is_not_nil(p->suspendee)) { + Process *rp; + if (!(p_locks & ERTS_PROC_LOCK_STATUS)) + erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + rp = erts_pid2proc(p, p_locks|ERTS_PROC_LOCK_STATUS, + p->suspendee, ERTS_PROC_LOCK_STATUS); + if (rp) { + erts_resume(rp, ERTS_PROC_LOCK_STATUS); + erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); + } + if (!(p_locks & ERTS_PROC_LOCK_STATUS)) + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + p->suspendee = NIL; + } +} + +static void +handle_pend_sync_suspend(Process *suspendee, + ErtsProcLocks suspendee_locks, + int suspendee_alive, + Eterm suspender_pid) +{ + Process *suspender; + + ERTS_SMP_LC_ASSERT(suspendee_locks & ERTS_PROC_LOCK_STATUS); + + suspender = erts_pid2proc(suspendee, + suspendee_locks, + suspender_pid, + ERTS_PROC_LOCK_STATUS); + if (suspender) { + ASSERT(is_nil(suspender->suspendee)); + if (suspendee_alive) { + ErtsRunQueue *rq = erts_get_runq_proc(suspendee); + erts_smp_runq_lock(rq); + suspend_process(rq, suspendee); + erts_smp_runq_unlock(rq); + suspender->suspendee = suspendee->id; + } + /* suspender is suspended waiting for suspendee to suspend; + resume suspender */ + resume_process(suspender); + erts_smp_proc_unlock(suspender, ERTS_PROC_LOCK_STATUS); + } +} + +/* + * Like erts_pid2proc() but: + * + * * At least ERTS_PROC_LOCK_MAIN have to be held on c_p. + * * At least ERTS_PROC_LOCK_MAIN have to be taken on pid. + * * It also waits for proc to be in a state != running and garbing. + * * If ERTS_PROC_LOCK_BUSY is returned, the calling process has to + * yield (ERTS_BIF_YIELD[0-3]()). c_p might in this case have been + * suspended. + */ + + +Process * +erts_pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks, + Eterm pid, ErtsProcLocks pid_locks) +{ + Process *rp; + int unlock_c_p_status; + + ERTS_SMP_LC_ASSERT(c_p_locks == erts_proc_lc_my_proc_locks(c_p)); + + ERTS_SMP_LC_ASSERT(c_p_locks & ERTS_PROC_LOCK_MAIN); + ERTS_SMP_LC_ASSERT(pid_locks & (ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS)); + + if (c_p->id == pid) + return erts_pid2proc(c_p, c_p_locks, pid, pid_locks); + + if (c_p_locks & ERTS_PROC_LOCK_STATUS) + unlock_c_p_status = 0; + else { + unlock_c_p_status = 1; + erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_STATUS); + } + + if (c_p->suspendee == pid) { + /* Process previously suspended by c_p (below)... */ + ErtsProcLocks rp_locks = pid_locks|ERTS_PROC_LOCK_STATUS; + rp = erts_pid2proc(c_p, c_p_locks|ERTS_PROC_LOCK_STATUS, pid, rp_locks); + c_p->suspendee = NIL; + ASSERT(c_p->flags & F_P2PNR_RESCHED); + c_p->flags &= ~F_P2PNR_RESCHED; + if (rp) + resume_process(rp); + } + else { + ErtsRunQueue *cp_rq, *rp_rq; + + rp = erts_pid2proc(c_p, c_p_locks|ERTS_PROC_LOCK_STATUS, + pid, ERTS_PROC_LOCK_STATUS); + + if (!rp) { + c_p->flags &= ~F_P2PNR_RESCHED; + goto done; + } + + ASSERT(!(c_p->flags & F_P2PNR_RESCHED)); + + cp_rq = erts_get_runq_proc(c_p); + rp_rq = erts_get_runq_proc(rp); + erts_smp_runqs_lock(cp_rq, rp_rq); + if (rp->runq_flags & ERTS_PROC_RUNQ_FLG_RUNNING) { + running: + /* Phiu... */ + + /* + * If we got pending suspenders and suspend ourselves waiting + * to suspend another process we might deadlock. + * In this case we have to yield, be suspended by + * someone else and then do it all over again. + */ + if (!c_p->pending_suspenders) { + /* Mark rp pending for suspend by c_p */ + add_pend_suspend(rp, c_p->id, handle_pend_sync_suspend); + ASSERT(is_nil(c_p->suspendee)); + + /* Suspend c_p; when rp is suspended c_p will be resumed. */ + suspend_process(cp_rq, c_p); + c_p->flags |= F_P2PNR_RESCHED; + } + /* Yield (caller is assumed to yield immediately in bif). */ + erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); + rp = ERTS_PROC_LOCK_BUSY; + } + else { + ErtsProcLocks need_locks = pid_locks & ~ERTS_PROC_LOCK_STATUS; + if (need_locks && erts_smp_proc_trylock(rp, need_locks) == EBUSY) { + erts_smp_runqs_unlock(cp_rq, rp_rq); + erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); + rp = erts_pid2proc(c_p, c_p_locks|ERTS_PROC_LOCK_STATUS, + pid, pid_locks|ERTS_PROC_LOCK_STATUS); + if (!rp) + goto done; + /* run-queues may have changed */ + cp_rq = erts_get_runq_proc(c_p); + rp_rq = erts_get_runq_proc(rp); + erts_smp_runqs_lock(cp_rq, rp_rq); + if (rp->runq_flags & ERTS_PROC_RUNQ_FLG_RUNNING) { + /* Ahh... */ + erts_smp_proc_unlock(rp, + pid_locks & ~ERTS_PROC_LOCK_STATUS); + goto running; + } + } + + /* rp is not running and we got the locks we want... */ + } + erts_smp_runqs_unlock(cp_rq, rp_rq); + } + + done: + if (rp && rp != ERTS_PROC_LOCK_BUSY && !(pid_locks & ERTS_PROC_LOCK_STATUS)) + erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); + if (unlock_c_p_status) + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS); + return rp; +} + +/* + * erts_pid2proc_nropt() is normally the same as + * erts_pid2proc_not_running(). However it is only + * to be used when 'not running' is a pure optimization, + * not a requirement. + */ + +Process * +erts_pid2proc_nropt(Process *c_p, ErtsProcLocks c_p_locks, + Eterm pid, ErtsProcLocks pid_locks) +{ + if (erts_disable_proc_not_running_opt) + return erts_pid2proc(c_p, c_p_locks, pid, pid_locks); + else + return erts_pid2proc_not_running(c_p, c_p_locks, pid, pid_locks); +} + +static ERTS_INLINE void +do_bif_suspend_process(ErtsSuspendMonitor *smon, + Process *suspendee, + ErtsRunQueue *locked_runq) +{ + ASSERT(suspendee); + ASSERT(!suspendee->is_exiting); + ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS + & erts_proc_lc_my_proc_locks(suspendee)); + if (smon) { + if (!smon->active) { + ErtsRunQueue *rq; + + if (locked_runq) + rq = locked_runq; + else { + rq = erts_get_runq_proc(suspendee); + erts_smp_runq_lock(rq); + } + + suspend_process(rq, suspendee); + + if (!locked_runq) + erts_smp_runq_unlock(rq); + } + smon->active += smon->pending; + ASSERT(smon->active); + smon->pending = 0; + } + +} + +static void +handle_pend_bif_sync_suspend(Process *suspendee, + ErtsProcLocks suspendee_locks, + int suspendee_alive, + Eterm suspender_pid) +{ + Process *suspender; + + ERTS_SMP_LC_ASSERT(suspendee_locks & ERTS_PROC_LOCK_STATUS); + + suspender = erts_pid2proc(suspendee, + suspendee_locks, + suspender_pid, + ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS); + if (suspender) { + ASSERT(is_nil(suspender->suspendee)); + if (!suspendee_alive) + erts_delete_suspend_monitor(&suspender->suspend_monitors, + suspendee->id); + else { + ErtsSuspendMonitor *smon; + smon = erts_lookup_suspend_monitor(suspender->suspend_monitors, + suspendee->id); + do_bif_suspend_process(smon, suspendee, NULL); + suspender->suspendee = suspendee->id; + } + /* suspender is suspended waiting for suspendee to suspend; + resume suspender */ + resume_process(suspender); + erts_smp_proc_unlock(suspender, + ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS); + } +} + +static void +handle_pend_bif_async_suspend(Process *suspendee, + ErtsProcLocks suspendee_locks, + int suspendee_alive, + Eterm suspender_pid) +{ + + Process *suspender; + + ERTS_SMP_LC_ASSERT(suspendee_locks & ERTS_PROC_LOCK_STATUS); + + suspender = erts_pid2proc(suspendee, + suspendee_locks, + suspender_pid, + ERTS_PROC_LOCK_LINK); + if (suspender) { + ASSERT(is_nil(suspender->suspendee)); + if (!suspendee_alive) + erts_delete_suspend_monitor(&suspender->suspend_monitors, + suspendee->id); + else { + ErtsSuspendMonitor *smon; + smon = erts_lookup_suspend_monitor(suspender->suspend_monitors, + suspendee->id); + do_bif_suspend_process(smon, suspendee, NULL); + } + erts_smp_proc_unlock(suspender, ERTS_PROC_LOCK_LINK); + } +} + +#endif /* ERTS_SMP */ + +/* + * The erlang:suspend_process/2 BIF + */ + +BIF_RETTYPE +suspend_process_2(BIF_ALIST_2) +{ + Eterm res; + Process* suspendee = NULL; + ErtsSuspendMonitor *smon; + ErtsProcLocks xlocks = (ErtsProcLocks) 0; + + /* Options and default values: */ + int asynchronous = 0; + int unless_suspending = 0; + + + if (BIF_P->id == BIF_ARG_1) + goto badarg; /* We are not allowed to suspend ourselves */ + + if (is_not_nil(BIF_ARG_2)) { + /* Parse option list */ + Eterm arg = BIF_ARG_2; + + while (is_list(arg)) { + Eterm *lp = list_val(arg); + arg = CAR(lp); + switch (arg) { + case am_unless_suspending: + unless_suspending = 1; + break; + case am_asynchronous: + asynchronous = 1; + break; + default: + goto badarg; + } + arg = CDR(lp); + } + if (is_not_nil(arg)) + goto badarg; + } + + xlocks = ERTS_PROC_LOCK_LINK | (asynchronous + ? (ErtsProcLocks) 0 + : ERTS_PROC_LOCK_STATUS); + + erts_smp_proc_lock(BIF_P, xlocks); + + suspendee = erts_pid2proc(BIF_P, + ERTS_PROC_LOCK_MAIN|xlocks, + BIF_ARG_1, + ERTS_PROC_LOCK_STATUS); + if (!suspendee) + goto no_suspendee; + + smon = erts_add_or_lookup_suspend_monitor(&BIF_P->suspend_monitors, + BIF_ARG_1); +#ifndef ERTS_SMP /* no ERTS_SMP */ + + /* This is really a piece of cake without SMP support... */ + if (!smon->active) { + suspend_process(erts_common_run_queue, suspendee); + smon->active++; + res = am_true; + } + else if (unless_suspending) + res = am_false; + else if (smon->active == INT_MAX) + goto system_limit; + else { + smon->active++; + res = am_true; + } + +#else /* ERTS_SMP */ + + /* ... but a little trickier with SMP support ... */ + + if (asynchronous) { + /* --- Asynchronous suspend begin ---------------------------------- */ + + ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_LINK + & erts_proc_lc_my_proc_locks(BIF_P)); + ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS + == erts_proc_lc_my_proc_locks(suspendee)); + + if (smon->active) { + smon->active += smon->pending; + smon->pending = 0; + if (unless_suspending) + res = am_false; + else if (smon->active == INT_MAX) + goto system_limit; + else { + smon->active++; + res = am_true; + } + /* done */ + } + else { + /* We havn't got any active suspends on the suspendee */ + if (smon->pending && unless_suspending) + res = am_false; + else { + ErtsRunQueue *rq; + if (smon->pending == INT_MAX) + goto system_limit; + + smon->pending++; + rq = erts_get_runq_proc(suspendee); + erts_smp_runq_lock(rq); + + if (suspendee->runq_flags & ERTS_PROC_RUNQ_FLG_RUNNING) + add_pend_suspend(suspendee, + BIF_P->id, + handle_pend_bif_async_suspend); + else + do_bif_suspend_process(smon, suspendee, rq); + erts_smp_runq_unlock(rq); + + res = am_true; + } + /* done */ + } + /* --- Asynchronous suspend end ------------------------------------ */ + } + else /* if (!asynchronous) */ { + /* --- Synchronous suspend begin ----------------------------------- */ + + ERTS_SMP_LC_ASSERT(((ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS) + & erts_proc_lc_my_proc_locks(BIF_P)) + == (ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS)); + ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS + == erts_proc_lc_my_proc_locks(suspendee)); + + if (BIF_P->suspendee == BIF_ARG_1) { + /* We are back after a yield and the suspendee + has been suspended on behalf of us. */ + ASSERT(smon->active >= 1); + BIF_P->suspendee = NIL; + res = (!unless_suspending || smon->active == 1 + ? am_true + : am_false); + /* done */ + } + else if (smon->active) { + if (unless_suspending) + res = am_false; + else { + smon->active++; + res = am_true; + } + /* done */ + } + else { + ErtsRunQueue *cp_rq, *s_rq; + /* We haven't got any active suspends on the suspendee */ + + /* + * If we have pending suspenders and suspend ourselves waiting + * to suspend another process, or suspend another process + * we might deadlock. In this case we have to yield, + * be suspended by someone else, and then do it all over again. + */ + if (BIF_P->pending_suspenders) + goto yield; + + if (!unless_suspending && smon->pending == INT_MAX) + goto system_limit; + if (!unless_suspending || smon->pending == 0) + smon->pending++; + + cp_rq = erts_get_runq_proc(BIF_P); + s_rq = erts_get_runq_proc(suspendee); + erts_smp_runqs_lock(cp_rq, s_rq); + if (!(suspendee->runq_flags & ERTS_PROC_RUNQ_FLG_RUNNING)) { + do_bif_suspend_process(smon, suspendee, s_rq); + erts_smp_runqs_unlock(cp_rq, s_rq); + res = (!unless_suspending || smon->active == 1 + ? am_true + : am_false); + /* done */ + } + else { + /* Mark suspendee pending for suspend by BIF_P */ + add_pend_suspend(suspendee, + BIF_P->id, + handle_pend_bif_sync_suspend); + + ASSERT(is_nil(BIF_P->suspendee)); + + /* + * Suspend BIF_P; when suspendee is suspended, BIF_P + * will be resumed and this BIF will be called again. + * This time with BIF_P->suspendee == BIF_ARG_1 (see + * above). + */ + suspend_process(cp_rq, BIF_P); + erts_smp_runqs_unlock(cp_rq, s_rq); + goto yield; + } + } + /* --- Synchronous suspend end ------------------------------------- */ + } + +#endif /* ERTS_SMP */ + + ASSERT(suspendee->status == P_SUSPENDED || (asynchronous && smon->pending)); + ASSERT(suspendee->status == P_SUSPENDED || !smon->active); + + erts_smp_proc_unlock(suspendee, ERTS_PROC_LOCK_STATUS); + erts_smp_proc_unlock(BIF_P, xlocks); + BIF_RET(res); + + system_limit: + ERTS_BIF_PREP_ERROR(res, BIF_P, SYSTEM_LIMIT); + goto do_return; + + no_suspendee: +#ifdef ERTS_SMP + BIF_P->suspendee = NIL; +#endif + erts_delete_suspend_monitor(&BIF_P->suspend_monitors, BIF_ARG_1); + + badarg: + ERTS_BIF_PREP_ERROR(res, BIF_P, BADARG); +#ifdef ERTS_SMP + goto do_return; + + yield: + ERTS_BIF_PREP_YIELD2(res, bif_export[BIF_suspend_process_2], + BIF_P, BIF_ARG_1, BIF_ARG_2); +#endif + + do_return: + if (suspendee) + erts_smp_proc_unlock(suspendee, ERTS_PROC_LOCK_STATUS); + if (xlocks) + erts_smp_proc_unlock(BIF_P, xlocks); + return res; + +} + + +/* + * The erlang:resume_process/1 BIF + */ + +BIF_RETTYPE +resume_process_1(BIF_ALIST_1) +{ + ErtsSuspendMonitor *smon; + Process *suspendee; + int is_active; + + if (BIF_P->id == BIF_ARG_1) + BIF_ERROR(BIF_P, BADARG); + + erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK); + smon = erts_lookup_suspend_monitor(BIF_P->suspend_monitors, BIF_ARG_1); + + if (!smon) { + /* No previous suspend or dead suspendee */ + goto error; + } + else if (smon->pending) { + smon->pending--; + ASSERT(smon->pending >= 0); + if (smon->active) { + smon->active += smon->pending; + smon->pending = 0; + } + is_active = smon->active; + } + else if (smon->active) { + smon->active--; + ASSERT(smon->pending >= 0); + is_active = 1; + } + else { + /* No previous suspend or dead suspendee */ + goto error; + } + + if (smon->active || smon->pending || !is_active) { + /* Leave the suspendee as it is; just verify that it is still alive */ + suspendee = erts_pid2proc(BIF_P, + ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_LINK, + BIF_ARG_1, + 0); + if (!suspendee) + goto no_suspendee; + + } + else { + /* Resume */ + suspendee = erts_pid2proc(BIF_P, + ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_LINK, + BIF_ARG_1, + ERTS_PROC_LOCK_STATUS); + if (!suspendee) + goto no_suspendee; + + ASSERT(suspendee->status == P_SUSPENDED + || (suspendee->status == P_GARBING + && suspendee->gcstatus == P_SUSPENDED)); + resume_process(suspendee); + + erts_smp_proc_unlock(suspendee, ERTS_PROC_LOCK_STATUS); + } + + if (!smon->active && !smon->pending) + erts_delete_suspend_monitor(&BIF_P->suspend_monitors, BIF_ARG_1); + + erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK); + + BIF_RET(am_true); + + no_suspendee: + /* cleanup */ + erts_delete_suspend_monitor(&BIF_P->suspend_monitors, BIF_ARG_1); + + error: + erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK); + BIF_ERROR(BIF_P, BADARG); +} + +Uint +erts_run_queues_len(Uint *qlen) +{ + int i = 0; + Uint len = 0; + ERTS_ATOMIC_FOREACH_RUNQ(rq, + { + if (qlen) + qlen[i++] = rq->procs.len; + len += rq->procs.len; + } + ); + return len; +} + +#ifdef HARDDEBUG_RUNQS +static void +check_procs_runq(ErtsRunQueue *runq, Process *p_in_q, Process *p_not_in_q) +{ + int len[ERTS_NO_PROC_PRIO_LEVELS] = {0}; + int tot_len; + int prioq, prio; + int found_p_in_q; + Process *p, *prevp; + + found_p_in_q = 0; + for (prioq = 0; prioq < ERTS_NO_PROC_PRIO_LEVELS - 1; prioq++) { + prevp = NULL; + for (p = runq->procs.prio[prioq].first; p; p = p->next) { + ASSERT(p != p_not_in_q); + if (p == p_in_q) + found_p_in_q = 1; + switch (p->prio) { + case PRIORITY_MAX: + case PRIORITY_HIGH: + case PRIORITY_NORMAL: + ASSERT(prioq == p->prio); + break; + case PRIORITY_LOW: + ASSERT(prioq == PRIORITY_NORMAL); + break; + default: + ASSERT(!"Bad prio on process"); + } + len[p->prio]++; + ASSERT(prevp == p->prev); + if (p->prev) { + ASSERT(p->prev->next == p); + } + else { + ASSERT(runq->procs.prio[prioq].first == p); + } + if (p->next) { + ASSERT(p->next->prev == p); + } + else { + ASSERT(runq->procs.prio[prioq].last == p); + } + ASSERT(p->run_queue == runq); + prevp = p; + } + } + + ASSERT(!p_in_q || found_p_in_q); + + tot_len = 0; + for (prio = 0; prio < ERTS_NO_PROC_PRIO_LEVELS; prio++) { + ASSERT(len[prio] == runq->procs.prio_info[prio].len); + if (len[prio]) { + ASSERT(runq->flags & (1 << prio)); + } + else { + ASSERT(!(runq->flags & (1 << prio))); + } + tot_len += len[prio]; + } + ASSERT(runq->procs.len == tot_len); +} +# define ERTS_DBG_CHK_PROCS_RUNQ(RQ) check_procs_runq((RQ), NULL, NULL) +# define ERTS_DBG_CHK_PROCS_RUNQ_PROC(RQ, P) check_procs_runq((RQ), (P), NULL) +# define ERTS_DBG_CHK_PROCS_RUNQ_NOPROC(RQ, P) check_procs_runq((RQ), NULL, (P)) +#else +# define ERTS_DBG_CHK_PROCS_RUNQ(RQ) +# define ERTS_DBG_CHK_PROCS_RUNQ_PROC(RQ, P) +# define ERTS_DBG_CHK_PROCS_RUNQ_NOPROC(RQ, P) +#endif + + +static ERTS_INLINE void +enqueue_process(ErtsRunQueue *runq, Process *p) +{ + ErtsRunPrioQueue *rpq; + ErtsRunQueueInfo *rqi; + + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq)); + ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); + + ASSERT(p->bound_runq || !(runq->flags & ERTS_RUNQ_FLG_SUSPENDED)); + + rqi = &runq->procs.prio_info[p->prio]; + rqi->len++; + if (rqi->max_len < rqi->len) + rqi->max_len = rqi->len; + + runq->procs.len++; + runq->len++; + if (runq->max_len < runq->len) + runq->max_len = runq->len; + + runq->flags |= (1 << p->prio); + + rpq = (p->prio == PRIORITY_LOW + ? &runq->procs.prio[PRIORITY_NORMAL] + : &runq->procs.prio[p->prio]); + + p->next = NULL; + p->prev = rpq->last; + if (rpq->last) + rpq->last->next = p; + else + rpq->first = p; + rpq->last = p; + + switch (p->status) { + case P_EXITING: + break; + case P_GARBING: + p->gcstatus = P_RUNABLE; + break; + default: + p->status = P_RUNABLE; + break; + } + +#ifdef ERTS_SMP + p->status_flags |= ERTS_PROC_SFLG_INRUNQ; +#endif + + ERTS_DBG_CHK_PROCS_RUNQ_PROC(runq, p); +} + + +static ERTS_INLINE int +dequeue_process(ErtsRunQueue *runq, Process *p) +{ + ErtsRunPrioQueue *rpq; + int res = 1; + + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq)); + ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); + + ERTS_DBG_CHK_PROCS_RUNQ(runq); + + rpq = &runq->procs.prio[p->prio == PRIORITY_LOW ? PRIORITY_NORMAL : p->prio]; + if (p->prev) { + p->prev->next = p->next; + } + else if (rpq->first == p) { + rpq->first = p->next; + } + else { + res = 0; + } + if (p->next) { + p->next->prev = p->prev; + } + else if (rpq->last == p) { + rpq->last = p->prev; + } + else { + ASSERT(res == 0); + } + + if (res) { + + if (--runq->procs.prio_info[p->prio].len == 0) + runq->flags &= ~(1 << p->prio); + runq->procs.len--; + runq->len--; + +#ifdef ERTS_SMP + p->status_flags &= ~ERTS_PROC_SFLG_INRUNQ; +#endif + } + + ERTS_DBG_CHK_PROCS_RUNQ_NOPROC(runq, p); + return res; +} + +/* schedule a process */ +static ERTS_INLINE void +internal_add_to_runq(ErtsRunQueue *runq, Process *p) +{ + Uint32 prev_status = p->status; + ErtsRunQueue *add_runq; +#ifdef ERTS_SMP + + ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq)); + + if (p->status_flags & ERTS_PROC_SFLG_INRUNQ) + return; + else if (p->runq_flags & ERTS_PROC_RUNQ_FLG_RUNNING) { + ASSERT(p->status != P_SUSPENDED); + ERTS_DBG_CHK_PROCS_RUNQ_NOPROC(runq, p); + p->status_flags |= ERTS_PROC_SFLG_PENDADD2SCHEDQ; + return; + } + ASSERT(!p->scheduler_data); +#endif + + ERTS_DBG_CHK_PROCS_RUNQ_NOPROC(runq, p); +#ifndef ERTS_SMP + /* Never schedule a suspended process (ok in smp case) */ + ASSERT(p->status != P_SUSPENDED); + add_runq = runq; + +#else + ASSERT(!p->bound_runq || p->bound_runq == p->run_queue); + if (p->bound_runq) { + if (p->bound_runq == runq) + add_runq = runq; + else { + add_runq = p->bound_runq; + erts_smp_xrunq_lock(runq, add_runq); + } + } + else { + add_runq = erts_check_emigration_need(runq, p->prio); + if (!add_runq) + add_runq = runq; + else /* Process emigrated */ + p->run_queue = add_runq; + } +#endif + + /* Enqueue the process */ + enqueue_process(add_runq, p); + + if ((erts_system_profile_flags.runnable_procs) + && (prev_status == P_WAITING + || prev_status == P_SUSPENDED)) { + profile_runnable_proc(p, am_active); + } + + smp_notify_inc_runq(add_runq); + + if (add_runq != runq) + erts_smp_runq_unlock(add_runq); +} + + +void +erts_add_to_runq(Process *p) +{ + ErtsRunQueue *runq = erts_get_runq_proc(p); + erts_smp_runq_lock(runq); + internal_add_to_runq(runq, p); + erts_smp_runq_unlock(runq); +} + +/* Possibly remove a scheduled process we need to suspend */ + +static int +remove_proc_from_runq(ErtsRunQueue *rq, Process *p, int to_inactive) +{ + int res; + + ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); + +#ifdef ERTS_SMP + if (p->status_flags & ERTS_PROC_SFLG_PENDADD2SCHEDQ) { + p->status_flags &= ~ERTS_PROC_SFLG_PENDADD2SCHEDQ; + ASSERT(!remove_proc_from_runq(rq, p, 0)); + return 1; + } +#endif + + res = dequeue_process(rq, p); + + if (res && erts_system_profile_flags.runnable_procs && to_inactive) + profile_runnable_proc(p, am_inactive); + +#ifdef ERTS_SMP + ASSERT(!(p->status_flags & ERTS_PROC_SFLG_INRUNQ)); +#endif + + return res; +} + +#ifdef ERTS_SMP + +ErtsMigrateResult +erts_proc_migrate(Process *p, ErtsProcLocks *plcks, + ErtsRunQueue *from_rq, int *from_locked, + ErtsRunQueue *to_rq, int *to_locked) +{ + ERTS_SMP_LC_ASSERT(*plcks == erts_proc_lc_my_proc_locks(p)); + ERTS_SMP_LC_ASSERT((ERTS_PROC_LOCK_STATUS & *plcks) + || from_locked); + ERTS_SMP_LC_CHK_RUNQ_LOCK(from_rq, *from_locked); + ERTS_SMP_LC_CHK_RUNQ_LOCK(to_rq, *to_locked); + + ASSERT(!erts_common_run_queue); + + /* + * If we have the lock on the run queue to migrate to, + * check that it isn't suspended. If it is suspended, + * we will refuse to migrate to it anyway. + */ + if (*to_locked && (to_rq->flags & ERTS_RUNQ_FLG_SUSPENDED)) + return ERTS_MIGRATE_FAILED_RUNQ_SUSPENDED; + + /* We need status lock on process and locks on both run queues */ + + if (!(ERTS_PROC_LOCK_STATUS & *plcks)) { + if (erts_smp_proc_trylock(p, ERTS_PROC_LOCK_STATUS) == EBUSY) { + ErtsProcLocks lcks = *plcks; + Eterm pid = p->id; + Process *proc = *plcks ? p : NULL; + + if (*from_locked) { + *from_locked = 0; + erts_smp_runq_unlock(from_rq); + } + if (*to_locked) { + *to_locked = 0; + erts_smp_runq_unlock(to_rq); + } + + proc = erts_pid2proc_opt(proc, + lcks, + pid, + lcks|ERTS_PROC_LOCK_STATUS, + ERTS_P2P_FLG_ALLOW_OTHER_X); + if (!proc) { + *plcks = 0; + return ERTS_MIGRATE_FAILED_NOT_IN_RUNQ; + } + ASSERT(proc == p); + } + *plcks |= ERTS_PROC_LOCK_STATUS; + } + + ASSERT(!p->bound_runq); + + ERTS_SMP_LC_CHK_RUNQ_LOCK(from_rq, *from_locked); + ERTS_SMP_LC_CHK_RUNQ_LOCK(to_rq, *to_locked); + + if (p->run_queue != from_rq) + return ERTS_MIGRATE_FAILED_RUNQ_CHANGED; + + if (!*from_locked || !*to_locked) { + if (from_rq < to_rq) { + if (!*to_locked) { + if (!*from_locked) + erts_smp_runq_lock(from_rq); + erts_smp_runq_lock(to_rq); + } + else if (erts_smp_runq_trylock(from_rq) == EBUSY) { + erts_smp_runq_unlock(to_rq); + erts_smp_runq_lock(from_rq); + erts_smp_runq_lock(to_rq); + } + } + else { + if (!*from_locked) { + if (!*to_locked) + erts_smp_runq_lock(to_rq); + erts_smp_runq_lock(from_rq); + } + else if (erts_smp_runq_trylock(to_rq) == EBUSY) { + erts_smp_runq_unlock(from_rq); + erts_smp_runq_lock(to_rq); + erts_smp_runq_lock(from_rq); + } + } + *to_locked = *from_locked = 1; + } + + ERTS_SMP_LC_CHK_RUNQ_LOCK(from_rq, *from_locked); + ERTS_SMP_LC_CHK_RUNQ_LOCK(to_rq, *to_locked); + + /* Ok we now got all locks we need; do it... */ + + /* Refuse to migrate to a suspended run queue */ + if (to_rq->flags & ERTS_RUNQ_FLG_SUSPENDED) + return ERTS_MIGRATE_FAILED_RUNQ_SUSPENDED; + + if ((p->runq_flags & ERTS_PROC_RUNQ_FLG_RUNNING) + || !(p->status_flags & ERTS_PROC_SFLG_INRUNQ)) + return ERTS_MIGRATE_FAILED_NOT_IN_RUNQ; + + dequeue_process(from_rq, p); + p->run_queue = to_rq; + enqueue_process(to_rq, p); + + return ERTS_MIGRATE_SUCCESS; +} +#endif /* ERTS_SMP */ + +Eterm +erts_process_status(Process *c_p, ErtsProcLocks c_p_locks, + Process *rp, Eterm rpid) +{ + Eterm res = am_undefined; + Process *p; + + if (rp) { + ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS + & erts_proc_lc_my_proc_locks(rp)); + p = rp; + } + else { + p = erts_pid2proc_opt(c_p, c_p_locks, + rpid, ERTS_PROC_LOCK_STATUS, + ERTS_P2P_FLG_ALLOW_OTHER_X); + } + + if (p) { + switch (p->status) { + case P_RUNABLE: + res = am_runnable; + break; + case P_WAITING: + res = am_waiting; + break; + case P_RUNNING: + res = am_running; + break; + case P_EXITING: + res = am_exiting; + break; + case P_GARBING: + res = am_garbage_collecting; + break; + case P_SUSPENDED: + res = am_suspended; + break; + case P_FREE: /* We cannot look up a process in P_FREE... */ + default: /* Not a valid status... */ + erl_exit(1, "Bad status (%b32u) found for process %T\n", + p->status, p->id); + break; + } + +#ifdef ERTS_SMP + if (!rp && (p != c_p || !(ERTS_PROC_LOCK_STATUS & c_p_locks))) + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + } + else { + int i; + ErtsSchedulerData *esdp; + + if (erts_common_run_queue) + erts_smp_runq_lock(erts_common_run_queue); + + for (i = 0; i < erts_no_schedulers; i++) { + esdp = ERTS_SCHEDULER_IX(i); + if (!erts_common_run_queue) + erts_smp_runq_lock(esdp->run_queue); + if (esdp->free_process && esdp->free_process->id == rpid) { + res = am_free; + if (!erts_common_run_queue) + erts_smp_runq_unlock(esdp->run_queue); + break; + } + if (!erts_common_run_queue) + erts_smp_runq_unlock(esdp->run_queue); + } + + if (erts_common_run_queue) + erts_smp_runq_unlock(erts_common_run_queue); +#endif + + } + + return res; +} + +/* +** Suspend a process +** If we are to suspend on a port the busy_port is the thing +** otherwise busy_port is NIL +*/ + +void +erts_suspend(Process* process, ErtsProcLocks process_locks, Port *busy_port) +{ + ErtsRunQueue *rq; + + ERTS_SMP_LC_ASSERT(process_locks == erts_proc_lc_my_proc_locks(process)); + if (!(process_locks & ERTS_PROC_LOCK_STATUS)) + erts_smp_proc_lock(process, ERTS_PROC_LOCK_STATUS); + + rq = erts_get_runq_proc(process); + + erts_smp_runq_lock(rq); + + suspend_process(rq, process); + + erts_smp_runq_unlock(rq); + + if (busy_port) + erts_wake_process_later(busy_port, process); + + if (!(process_locks & ERTS_PROC_LOCK_STATUS)) + erts_smp_proc_unlock(process, ERTS_PROC_LOCK_STATUS); + +} + +void +erts_resume(Process* process, ErtsProcLocks process_locks) +{ + ERTS_SMP_LC_ASSERT(process_locks == erts_proc_lc_my_proc_locks(process)); + if (!(process_locks & ERTS_PROC_LOCK_STATUS)) + erts_smp_proc_lock(process, ERTS_PROC_LOCK_STATUS); + resume_process(process); + if (!(process_locks & ERTS_PROC_LOCK_STATUS)) + erts_smp_proc_unlock(process, ERTS_PROC_LOCK_STATUS); +} + +int +erts_resume_processes(ErtsProcList *plp) +{ + int nresumed = 0; + while (plp) { + Process *proc; + ErtsProcList *fplp; + ASSERT(is_internal_pid(plp->pid)); + proc = erts_pid2proc(NULL, 0, plp->pid, ERTS_PROC_LOCK_STATUS); + if (proc) { + if (proclist_same(plp, proc)) { + resume_process(proc); + nresumed++; + } + erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_STATUS); + } + fplp = plp; + plp = plp->next; + proclist_destroy(fplp); + } + return nresumed; +} + +Eterm +erts_get_process_priority(Process *p) +{ + ErtsRunQueue *rq; + Eterm value; + ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); + rq = erts_get_runq_proc(p); + erts_smp_runq_lock(rq); + switch(p->prio) { + case PRIORITY_MAX: value = am_max; break; + case PRIORITY_HIGH: value = am_high; break; + case PRIORITY_NORMAL: value = am_normal; break; + case PRIORITY_LOW: value = am_low; break; + default: ASSERT(0); value = am_undefined; break; + } + erts_smp_runq_unlock(rq); + return value; +} + +Eterm +erts_set_process_priority(Process *p, Eterm new_value) +{ + ErtsRunQueue *rq; + Eterm old_value; + ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); + rq = erts_get_runq_proc(p); +#ifdef ERTS_SMP + ASSERT(!(p->status_flags & ERTS_PROC_SFLG_INRUNQ)); +#endif + erts_smp_runq_lock(rq); + switch(p->prio) { + case PRIORITY_MAX: old_value = am_max; break; + case PRIORITY_HIGH: old_value = am_high; break; + case PRIORITY_NORMAL: old_value = am_normal; break; + case PRIORITY_LOW: old_value = am_low; break; + default: ASSERT(0); old_value = am_undefined; break; + } + switch (new_value) { + case am_max: p->prio = PRIORITY_MAX; break; + case am_high: p->prio = PRIORITY_HIGH; break; + case am_normal: p->prio = PRIORITY_NORMAL; break; + case am_low: p->prio = PRIORITY_LOW; break; + default: old_value = THE_NON_VALUE; break; + } + erts_smp_runq_unlock(rq); + return old_value; +} + +#ifdef ERTS_SMP + +static ERTS_INLINE int +prepare_for_sys_schedule(void) +{ + while (!erts_port_task_have_outstanding_io_tasks() + && !erts_smp_atomic_xchg(&doing_sys_schedule, 1)) { + if (!erts_port_task_have_outstanding_io_tasks()) + return 1; + erts_smp_atomic_set(&doing_sys_schedule, 0); + } + return 0; +} + +#else + +static ERTS_INLINE int +prepare_for_sys_schedule(void) +{ + return !erts_port_task_have_outstanding_io_tasks(); +} + +#endif + +/* note that P_RUNNING is only set so that we don't try to remove +** running processes from the schedule queue if they exit - a running +** process not being in the schedule queue!! +** Schedule for up to INPUT_REDUCTIONS context switches, +** return 1 if more to do. +*/ + +/* + * schedule() is called from BEAM (process_main()) or HiPE + * (hipe_mode_switch()) when the current process is to be + * replaced by a new process. 'calls' is the number of reduction + * steps the current process consumed. + * schedule() returns the new process, and the new process' + * ->fcalls field is initialised with its allowable number of + * reduction steps. + * + * When no process is runnable, or when sufficiently many reduction + * steps have been made, schedule() calls erl_sys_schedule() to + * schedule system-level activities. + * + * We use the same queue for normal and low prio processes. + * We reschedule low prio processes a certain number of times + * so that normal processes get to run more frequently. + */ + +Process *schedule(Process *p, int calls) +{ + ErtsRunQueue *rq; + ErtsRunPrioQueue *rpq; + long dt; + ErtsSchedulerData *esdp; + int context_reds; + long fcalls; + int input_reductions; + int actual_reds; + int reds; + + if (ERTS_USE_MODIFIED_TIMING()) { + context_reds = ERTS_MODIFIED_TIMING_CONTEXT_REDS; + input_reductions = ERTS_MODIFIED_TIMING_INPUT_REDS; + } + else { + context_reds = CONTEXT_REDS; + input_reductions = INPUT_REDUCTIONS; + } + + ERTS_SMP_LC_ASSERT(!ERTS_LC_IS_BLOCKING); + + /* + * Clean up after the process being scheduled out. + */ + if (!p) { /* NULL in the very first schedule() call */ + esdp = erts_get_scheduler_data(); + rq = erts_get_runq_current(esdp); + ASSERT(esdp); + fcalls = erts_smp_atomic_read(&function_calls); + actual_reds = reds = 0; + erts_smp_runq_lock(rq); + } else { +#ifdef ERTS_SMP + ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); + esdp = p->scheduler_data; + ASSERT(esdp->current_process == p + || esdp->free_process == p); +#else + esdp = erts_scheduler_data; + ASSERT(esdp->current_process == p); +#endif + reds = actual_reds = calls - esdp->virtual_reds; + if (reds < ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST) + reds = ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST; + esdp->virtual_reds = 0; + + fcalls = erts_smp_atomic_addtest(&function_calls, reds); + ASSERT(esdp && esdp == erts_get_scheduler_data()); + + rq = erts_get_runq_current(esdp); + + p->reds += actual_reds; + + erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + + if ((erts_system_profile_flags.runnable_procs) + && (p->status == P_WAITING)) { + profile_runnable_proc(p, am_inactive); + } + + if (IS_TRACED(p)) { + switch (p->status) { + case P_EXITING: + if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED_EXIT)) + trace_sched(p, am_out_exiting); + break; + case P_FREE: + if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED_EXIT)) + trace_sched(p, am_out_exited); + break; + default: + if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED)) + trace_sched(p, am_out); + else if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED_PROCS)) + trace_virtual_sched(p, am_out); + break; + } + } + +#ifdef ERTS_SMP + if (ERTS_PROC_PENDING_EXIT(p)) { + erts_handle_pending_exit(p, + ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS); + p->status_flags |= ERTS_PROC_SFLG_PENDADD2SCHEDQ; + } + + if (p->pending_suspenders) { + handle_pending_suspend(p, + ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS); + ASSERT(!(p->status_flags & ERTS_PROC_SFLG_PENDADD2SCHEDQ) + || p->status != P_SUSPENDED); + } +#endif + erts_smp_runq_lock(rq); + + ERTS_PROC_REDUCTIONS_EXECUTED(rq, p->prio, reds, actual_reds); + + esdp->current_process = NULL; +#ifdef ERTS_SMP + p->scheduler_data = NULL; + p->runq_flags &= ~ERTS_PROC_RUNQ_FLG_RUNNING; + p->status_flags &= ~ERTS_PROC_SFLG_RUNNING; + + if (p->status_flags & ERTS_PROC_SFLG_PENDADD2SCHEDQ) { + p->status_flags &= ~ERTS_PROC_SFLG_PENDADD2SCHEDQ; + internal_add_to_runq(rq, p); + } +#endif + + + if (p->status == P_FREE) { +#ifdef ERTS_SMP + ASSERT(esdp->free_process == p); + esdp->free_process = NULL; + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS); + erts_smp_proc_dec_refc(p); +#else + erts_free_proc(p); +#endif + } else { + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS); + } + +#ifdef ERTS_SMP + { + ErtsProcList *pnd_xtrs = rq->procs.pending_exiters; + rq->procs.pending_exiters = NULL; + + if (pnd_xtrs) { + erts_smp_runq_unlock(rq); + handle_pending_exiters(pnd_xtrs); + erts_smp_runq_lock(rq); + } + + } + ASSERT(!esdp->free_process); +#endif + ASSERT(!esdp->current_process); + + ERTS_SMP_CHK_NO_PROC_LOCKS; + + dt = do_time_read_and_reset(); + if (dt) { + erts_smp_runq_unlock(rq); + bump_timer(dt); + erts_smp_runq_lock(rq); + } + BM_STOP_TIMER(system); + + } + + ERTS_SMP_LC_ASSERT(!ERTS_LC_IS_BLOCKING); + check_activities_to_run: { + +#ifdef ERTS_SMP + + if (!(rq->flags & ERTS_RUNQ_FLG_SHARED_RUNQ) + && rq->check_balance_reds <= 0) { + check_balance(rq); + } + + ERTS_SMP_LC_ASSERT(!ERTS_LC_IS_BLOCKING); + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); + + if (rq->flags & ERTS_RUNQ_FLGS_IMMIGRATE_QMASK) + immigrate(rq); + + continue_check_activities_to_run: + + if (rq->flags & (ERTS_RUNQ_FLG_SHARED_RUNQ + | ERTS_RUNQ_FLG_CHK_CPU_BIND + | ERTS_RUNQ_FLG_SUSPENDED)) { + if ((rq->flags & ERTS_RUNQ_FLG_SUSPENDED) + || erts_smp_atomic_read(&esdp->suspended)) { + suspend_scheduler(esdp); + } + if ((rq->flags & ERTS_RUNQ_FLG_CHK_CPU_BIND) + || erts_smp_atomic_read(&esdp->chk_cpu_bind)) { + check_cpu_bind(esdp); + } + } + +#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN + if (esdp->check_children) { + esdp->check_children = 0; + erts_smp_runq_unlock(rq); + erts_check_children(); + erts_smp_runq_lock(rq); + } +#endif + + erts_smp_chk_system_block(prepare_for_block, + resume_after_block, + (void *) rq); + + ERTS_SMP_LC_ASSERT(!ERTS_LC_IS_BLOCKING); + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); + +#endif + + ASSERT(rq->len == rq->procs.len + rq->ports.info.len); + +#ifndef ERTS_SMP + + if (rq->len == 0 && !rq->misc.start) + goto do_sys_schedule; + +#else /* ERTS_SMP */ + if (rq->len == 0 && !rq->misc.start) { + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); + + rq->wakeup_other = 0; + rq->wakeup_other_reds = 0; + + empty_runq(rq); + + if (rq->flags & (ERTS_RUNQ_FLG_SHARED_RUNQ + | ERTS_RUNQ_FLG_SUSPENDED)) { + if ((rq->flags & ERTS_RUNQ_FLG_SUSPENDED) + || erts_smp_atomic_read(&esdp->suspended)) { + non_empty_runq(rq); + goto continue_check_activities_to_run; + } + } + else if (!(rq->flags & ERTS_RUNQ_FLG_INACTIVE)) { + /* + * Check for ERTS_RUNQ_FLG_SUSPENDED has to be done + * after trying to steal a task. + */ + if (try_steal_task(rq) + || (rq->flags & ERTS_RUNQ_FLG_SUSPENDED)) { + non_empty_runq(rq); + goto continue_check_activities_to_run; + } + } + + if (prepare_for_sys_schedule()) { + erts_smp_atomic_set(&function_calls, 0); + fcalls = 0; + sched_sys_wait(esdp->no, rq); + erts_smp_atomic_set(&doing_sys_schedule, 0); + } + else { + /* If all schedulers are waiting, one of them *should* + be waiting in erl_sys_schedule() */ + sched_cnd_wait(esdp->no, rq); + } + + non_empty_runq(rq); + + goto check_activities_to_run; + } + else +#endif /* ERTS_SMP */ + if (fcalls > input_reductions && prepare_for_sys_schedule()) { + int runnable; + +#ifdef ERTS_SMP + runnable = 1; +#else + do_sys_schedule: + runnable = rq->len != 0; + if (!runnable) + sched_waiting_sys(esdp->no, rq); +#endif + + /* + * Schedule system-level activities. + */ + + erts_smp_atomic_set(&function_calls, 0); + fcalls = 0; + ASSERT(!erts_port_task_have_outstanding_io_tasks()); +#ifdef ERTS_SMP + /* erts_sys_schedule_interrupt(0); */ +#endif + erts_smp_runq_unlock(rq); + erl_sys_schedule(runnable); + dt = do_time_read_and_reset(); + if (dt) bump_timer(dt); +#ifdef ERTS_SMP + erts_smp_runq_lock(rq); + erts_smp_atomic_set(&doing_sys_schedule, 0); + goto continue_check_activities_to_run; +#else + if (!runnable) + sched_active_sys(esdp->no, rq); + goto check_activities_to_run; +#endif + } + + if (rq->misc.start) + exec_misc_ops(rq); + +#ifdef ERTS_SMP + { + int wo_reds = rq->wakeup_other_reds; + if (wo_reds) { + if (rq->len < 2) { + rq->wakeup_other -= ERTS_WAKEUP_OTHER_DEC*wo_reds; + if (rq->wakeup_other < 0) + rq->wakeup_other = 0; + } + else if (rq->wakeup_other < ERTS_WAKEUP_OTHER_LIMIT) + rq->wakeup_other += rq->len*wo_reds + ERTS_WAKEUP_OTHER_FIXED_INC; + else { + if (erts_common_run_queue) { + if (erts_common_run_queue->waiting) + wake_one_scheduler(); + } + else if (erts_smp_atomic_read(&no_empty_run_queues) != 0) { + wake_scheduler_on_empty_runq(rq); + rq->wakeup_other = 0; + } + rq->wakeup_other = 0; + } + } + rq->wakeup_other_reds = 0; + } +#endif + + /* + * Find a new port to run. + */ + + if (rq->ports.info.len) { + int have_outstanding_io; + have_outstanding_io = erts_port_task_execute(rq, &esdp->current_port); + if (have_outstanding_io && fcalls > 2*input_reductions) { + /* + * If we have performed more than 2*INPUT_REDUCTIONS since + * last call to erl_sys_schedule() and we still haven't + * handled all I/O tasks we stop running processes and + * focus completely on ports. + * + * One could argue that this is a strange behavior. The + * reason for doing it this way is that it is similar + * to the behavior before port tasks were introduced. + * We don't want to change the behavior too much, at + * least not at the time of writing. This behavior + * might change in the future. + * + * /rickard + */ + goto check_activities_to_run; + } + } + + /* + * Find a new process to run. + */ + pick_next_process: + + ERTS_DBG_CHK_PROCS_RUNQ(rq); + + switch (rq->flags & ERTS_RUNQ_FLGS_PROCS_QMASK) { + case MAX_BIT: + case MAX_BIT|HIGH_BIT: + case MAX_BIT|NORMAL_BIT: + case MAX_BIT|LOW_BIT: + case MAX_BIT|HIGH_BIT|NORMAL_BIT: + case MAX_BIT|HIGH_BIT|LOW_BIT: + case MAX_BIT|NORMAL_BIT|LOW_BIT: + case MAX_BIT|HIGH_BIT|NORMAL_BIT|LOW_BIT: + rpq = &rq->procs.prio[PRIORITY_MAX]; + break; + case HIGH_BIT: + case HIGH_BIT|NORMAL_BIT: + case HIGH_BIT|LOW_BIT: + case HIGH_BIT|NORMAL_BIT|LOW_BIT: + rpq = &rq->procs.prio[PRIORITY_HIGH]; + break; + case NORMAL_BIT: + rpq = &rq->procs.prio[PRIORITY_NORMAL]; + break; + case LOW_BIT: + rpq = &rq->procs.prio[PRIORITY_NORMAL]; + break; + case NORMAL_BIT|LOW_BIT: + rpq = &rq->procs.prio[PRIORITY_NORMAL]; + ASSERT(rpq->first != NULL); + p = rpq->first; + if (p->prio == PRIORITY_LOW) { + if (p == rpq->last || p->skipped >= RESCHEDULE_LOW-1) + p->skipped = 0; + else { + /* skip it */ + p->skipped++; + rpq->first = p->next; + rpq->first->prev = NULL; + rpq->last->next = p; + p->prev = rpq->last; + p->next = NULL; + rpq->last = p; + goto pick_next_process; + } + } + break; + case 0: /* No process at all */ + default: + ASSERT((rq->flags & ERTS_RUNQ_FLGS_PROCS_QMASK) == 0); + ASSERT(rq->procs.len == 0); + goto check_activities_to_run; + } + + BM_START_TIMER(system); + + /* + * Take the chosen process out of the queue. + */ + ASSERT(rpq->first); /* Wrong qmask in rq->flags? */ + p = rpq->first; +#ifdef ERTS_SMP + ERTS_SMP_LC_ASSERT(rq == p->run_queue); +#endif + rpq->first = p->next; + if (!rpq->first) + rpq->last = NULL; + else + rpq->first->prev = NULL; + + p->next = p->prev = NULL; + + if (--rq->procs.prio_info[p->prio].len == 0) + rq->flags &= ~(1 << p->prio); + ASSERT(rq->procs.len > 0); + rq->procs.len--; + ASSERT(rq->len > 0); + rq->len--; + + { + Uint32 ee_flgs = (ERTS_RUNQ_FLG_EVACUATE(p->prio) + | ERTS_RUNQ_FLG_EMIGRATE(p->prio)); + + if ((rq->flags & (ERTS_RUNQ_FLG_SUSPENDED|ee_flgs)) == ee_flgs) + ERTS_UNSET_RUNQ_FLG_EVACUATE(rq->flags, p->prio); + } + + ERTS_DBG_CHK_PROCS_RUNQ_NOPROC(rq, p); + + rq->procs.context_switches++; + + esdp->current_process = p; + +#ifdef ERTS_SMP + p->runq_flags |= ERTS_PROC_RUNQ_FLG_RUNNING; + erts_smp_runq_unlock(rq); + + ERTS_SMP_CHK_NO_PROC_LOCKS; + + erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS); + + if (erts_sched_stat.enabled) { + Uint old = ERTS_PROC_SCHED_ID(p, + (ERTS_PROC_LOCK_MAIN + | ERTS_PROC_LOCK_STATUS), + esdp->no); + int migrated = old && old != esdp->no; + + erts_smp_spin_lock(&erts_sched_stat.lock); + erts_sched_stat.prio[p->prio].total_executed++; + erts_sched_stat.prio[p->prio].executed++; + if (migrated) { + erts_sched_stat.prio[p->prio].total_migrated++; + erts_sched_stat.prio[p->prio].migrated++; + } + erts_smp_spin_unlock(&erts_sched_stat.lock); + } + + p->status_flags |= ERTS_PROC_SFLG_RUNNING; + p->status_flags &= ~ERTS_PROC_SFLG_INRUNQ; + if (ERTS_PROC_PENDING_EXIT(p)) { + erts_handle_pending_exit(p, + ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS); + } + ASSERT(!p->scheduler_data); + p->scheduler_data = esdp; + +#endif + ASSERT(p->status != P_SUSPENDED); /* Never run a suspended process */ + + ACTIVATE(p); + reds = context_reds; + + if (IS_TRACED(p)) { + switch (p->status) { + case P_EXITING: + if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED_EXIT)) + trace_sched(p, am_in_exiting); + break; + default: + if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED)) + trace_sched(p, am_in); + else if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED_PROCS)) + trace_virtual_sched(p, am_in); + break; + } + } + if (p->status != P_EXITING) + p->status = P_RUNNING; + + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + +#ifdef ERTS_SMP + if (is_not_nil(p->tracer_proc)) + erts_check_my_tracer_proc(p); +#endif + + if ((FLAGS(p) & F_FORCE_GC) || (MSO(p).overhead >= BIN_VHEAP_SZ(p))) { + reds -= erts_garbage_collect(p, 0, p->arg_reg, p->arity); + if (reds < 0) { + reds = 1; + } + } + + p->fcalls = reds; + ASSERT(IS_ACTIVE(p)); + ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); + return p; + } +} + +void +erts_sched_stat_modify(int what) +{ + int ix; + switch (what) { + case ERTS_SCHED_STAT_MODIFY_ENABLE: + erts_smp_block_system(0); + erts_sched_stat.enabled = 1; + erts_smp_release_system(); + break; + case ERTS_SCHED_STAT_MODIFY_DISABLE: + erts_smp_block_system(0); + erts_sched_stat.enabled = 1; + erts_smp_release_system(); + break; + case ERTS_SCHED_STAT_MODIFY_CLEAR: + erts_smp_spin_lock(&erts_sched_stat.lock); + for (ix = 0; ix < ERTS_NO_PRIO_LEVELS; ix++) { + erts_sched_stat.prio[ix].total_executed = 0; + erts_sched_stat.prio[ix].executed = 0; + erts_sched_stat.prio[ix].total_migrated = 0; + erts_sched_stat.prio[ix].migrated = 0; + } + erts_smp_spin_unlock(&erts_sched_stat.lock); + break; + } +} + +Eterm +erts_sched_stat_term(Process *p, int total) +{ + Uint sz; + Uint *hp; + Eterm prio[ERTS_NO_PRIO_LEVELS]; + Uint executed[ERTS_NO_PRIO_LEVELS]; + Uint migrated[ERTS_NO_PRIO_LEVELS]; + + erts_smp_spin_lock(&erts_sched_stat.lock); + if (total) { + int i; + for (i = 0; i < ERTS_NO_PRIO_LEVELS; i++) { + prio[i] = erts_sched_stat.prio[i].name; + executed[i] = erts_sched_stat.prio[i].total_executed; + migrated[i] = erts_sched_stat.prio[i].total_migrated; + } + } + else { + int i; + for (i = 0; i < ERTS_NO_PRIO_LEVELS; i++) { + prio[i] = erts_sched_stat.prio[i].name; + executed[i] = erts_sched_stat.prio[i].executed; + erts_sched_stat.prio[i].executed = 0; + migrated[i] = erts_sched_stat.prio[i].migrated; + erts_sched_stat.prio[i].migrated = 0; + } + } + erts_smp_spin_unlock(&erts_sched_stat.lock); + + sz = 0; + (void) erts_bld_atom_2uint_3tup_list(NULL, &sz, ERTS_NO_PRIO_LEVELS, + prio, executed, migrated); + hp = HAlloc(p, sz); + return erts_bld_atom_2uint_3tup_list(&hp, NULL, ERTS_NO_PRIO_LEVELS, + prio, executed, migrated); +} + +/* + * Scheduling of misc stuff + */ + +void +erts_schedule_misc_op(void (*func)(void *), void *arg) +{ + ErtsRunQueue *rq = erts_get_runq_current(NULL); + ErtsMiscOpList *molp = misc_op_list_alloc(); + + erts_smp_runq_lock(rq); + + while (rq->misc.evac_runq) { + ErtsRunQueue *tmp_rq = rq->misc.evac_runq; + erts_smp_runq_unlock(rq); + rq = tmp_rq; + erts_smp_runq_lock(rq); + } + + ASSERT(!(rq->flags & ERTS_RUNQ_FLG_SUSPENDED)); + + molp->next = NULL; + molp->func = func; + molp->arg = arg; + if (rq->misc.end) + rq->misc.end->next = molp; + else + rq->misc.start = molp; + rq->misc.end = molp; + smp_notify_inc_runq(rq); + erts_smp_runq_unlock(rq); +} + +static void +exec_misc_ops(ErtsRunQueue *rq) +{ + int i; + ErtsMiscOpList *molp = rq->misc.start; + ErtsMiscOpList *tmp_molp = molp; + + for (i = 0; i < ERTS_MAX_MISC_OPS-1; i++) { + if (!tmp_molp) + goto mtq; + tmp_molp = tmp_molp->next; + } + + if (!tmp_molp) { + mtq: + rq->misc.start = NULL; + rq->misc.end = NULL; + } + else { + rq->misc.start = tmp_molp->next; + tmp_molp->next = NULL; + if (!rq->misc.start) + rq->misc.end = NULL; + } + + erts_smp_runq_unlock(rq); + + while (molp) { + tmp_molp = molp; + (*molp->func)(molp->arg); + molp = molp->next; + misc_op_list_free(tmp_molp); + } + + erts_smp_runq_lock(rq); +} + +Uint +erts_get_total_context_switches(void) +{ + Uint res = 0; + ERTS_ATOMIC_FOREACH_RUNQ(rq, res += rq->procs.context_switches); + return res; +} + +void +erts_get_total_reductions(Uint *redsp, Uint *diffp) +{ + Uint reds = 0; + ERTS_ATOMIC_FOREACH_RUNQ_X(rq, + + reds += rq->procs.reductions, + + if (redsp) *redsp = reds; + if (diffp) *diffp = reds - last_reductions; + last_reductions = reds); +} + +void +erts_get_exact_total_reductions(Process *c_p, Uint *redsp, Uint *diffp) +{ + Uint reds = erts_current_reductions(c_p, c_p); + int ix; + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); + /* + * Wait for other schedulers to schedule out their processes + * and update 'reductions'. + */ + erts_smp_block_system(0); + for (reds = 0, ix = 0; ix < erts_no_run_queues; ix++) + reds += ERTS_RUNQ_IX(ix)->procs.reductions; + if (redsp) + *redsp = reds; + if (diffp) + *diffp = reds - last_exact_reductions; + last_exact_reductions = reds; + erts_smp_release_system(); + erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); +} + +/* + * erts_test_next_pid() is only used for testing. + */ +Sint +erts_test_next_pid(int set, Uint next) +{ + Sint res; + Sint p_prev; + + + erts_smp_mtx_lock(&proc_tab_mtx); + + if (!set) { + res = p_next < 0 ? -1 : (p_serial << p_serial_shift | p_next); + } + else { + + p_serial = (Sint) ((next >> p_serial_shift) & p_serial_mask); + p_next = (Sint) (erts_process_tab_index_mask & next); + + if (p_next >= erts_max_processes) { + p_next = 0; + p_serial++; + p_serial &= p_serial_mask; + } + + p_prev = p_next; + + do { + if (!process_tab[p_next]) + break; + p_next++; + if(p_next >= erts_max_processes) { + p_next = 0; + p_serial++; + p_serial &= p_serial_mask; + } + } while (p_prev != p_next); + + res = process_tab[p_next] ? -1 : (p_serial << p_serial_shift | p_next); + + } + + erts_smp_mtx_unlock(&proc_tab_mtx); + + return res; + +} + +Uint erts_process_count(void) +{ + long res = erts_smp_atomic_read(&process_count); + ASSERT(res >= 0); + return (Uint) res; +} + +void +erts_free_proc(Process *p) +{ +#if defined(ERTS_ENABLE_LOCK_COUNT) && defined(ERTS_SMP) + erts_lcnt_proc_lock_destroy(p); +#endif + erts_free(ERTS_ALC_T_PROC, (void *) p); +} + + +/* +** Allocate process and find out where to place next process. +*/ +static Process* +alloc_process(void) +{ +#ifdef ERTS_SMP + erts_pix_lock_t *pix_lock; +#endif + Process* p; + int p_prev; + + erts_smp_mtx_lock(&proc_tab_mtx); + + if (p_next == -1) { + p = NULL; + goto error; /* Process table full! */ + } + + p = (Process*) erts_alloc_fnf(ERTS_ALC_T_PROC, sizeof(Process)); + if (!p) + goto error; /* ENOMEM */ + + p_last = p_next; + + erts_get_emu_time(&p->started); + +#ifdef ERTS_SMP + pix_lock = ERTS_PIX2PIXLOCK(p_next); + erts_pix_lock(pix_lock); +#endif + ASSERT(!process_tab[p_next]); + + process_tab[p_next] = p; + erts_smp_atomic_inc(&process_count); + p->id = make_internal_pid(p_serial << p_serial_shift | p_next); + if (p->id == ERTS_INVALID_PID) { + /* Do not use the invalid pid; change serial */ + p_serial++; + p_serial &= p_serial_mask; + p->id = make_internal_pid(p_serial << p_serial_shift | p_next); + ASSERT(p->id != ERTS_INVALID_PID); + } + ASSERT(internal_pid_serial(p->id) <= (erts_use_r9_pids_ports + ? ERTS_MAX_PID_R9_SERIAL + : ERTS_MAX_PID_SERIAL)); + +#ifdef ERTS_SMP + erts_proc_lock_init(p); /* All locks locked */ + erts_pix_unlock(pix_lock); +#endif + + p->rstatus = P_FREE; + p->rcount = 0; + + /* + * set p_next to the next available slot + */ + + p_prev = p_next; + + while (1) { + p_next++; + if(p_next >= erts_max_processes) { + p_serial++; + p_serial &= p_serial_mask; + p_next = 0; + } + + if (p_prev == p_next) { + p_next = -1; + break; /* Table full! */ + } + + if (!process_tab[p_next]) + break; /* found a free slot */ + } + + error: + + erts_smp_mtx_unlock(&proc_tab_mtx); + + return p; + +} + +Eterm +erl_create_process(Process* parent, /* Parent of process (default group leader). */ + Eterm mod, /* Tagged atom for module. */ + Eterm func, /* Tagged atom for function. */ + Eterm args, /* Arguments for function (must be well-formed list). */ + ErlSpawnOpts* so) /* Options for spawn. */ +{ + ErtsRunQueue *rq; + Process *p; + Sint arity; /* Number of arguments. */ +#ifndef HYBRID + Uint arg_size; /* Size of arguments. */ +#endif + Uint sz; /* Needed words on heap. */ + Uint heap_need; /* Size needed on heap. */ + Eterm res = THE_NON_VALUE; + +#ifdef ERTS_SMP + erts_smp_proc_lock(parent, ERTS_PROC_LOCKS_ALL_MINOR); +#endif + +#ifdef HYBRID + /* + * Copy the arguments to the global heap + * Since global GC might occur we want to do this before adding the + * new process to the process_tab. + */ + BM_SWAP_TIMER(system,copy); + LAZY_COPY(parent,args); + BM_SWAP_TIMER(copy,system); + heap_need = 0; +#endif /* HYBRID */ + /* + * Check for errors. + */ + + if (is_not_atom(mod) || is_not_atom(func) || ((arity = list_length(args)) < 0)) { + so->error_code = BADARG; + goto error; + } + p = alloc_process(); /* All proc locks are locked by this thread + on success */ + if (!p) { + erts_send_error_to_logger_str(parent->group_leader, + "Too many processes\n"); + so->error_code = SYSTEM_LIMIT; + goto error; + } + + processes_busy++; + BM_COUNT(processes_spawned); + +#ifndef HYBRID + BM_SWAP_TIMER(system,size); + arg_size = size_object(args); + BM_SWAP_TIMER(size,system); + heap_need = arg_size; +#endif + + p->flags = erts_default_process_flags; + + /* Scheduler queue mutex should be locked when changeing + * prio. In this case we don't have to lock it, since + * noone except us has access to the process. + */ + if (so->flags & SPO_USE_ARGS) { + p->min_heap_size = so->min_heap_size; + p->prio = so->priority; + p->max_gen_gcs = so->max_gen_gcs; + } else { + p->min_heap_size = H_MIN_SIZE; + p->prio = PRIORITY_NORMAL; + p->max_gen_gcs = (Uint16) erts_smp_atomic_read(&erts_max_gen_gcs); + } + p->skipped = 0; + ASSERT(p->min_heap_size == erts_next_heap_size(p->min_heap_size, 0)); + + p->initial[INITIAL_MOD] = mod; + p->initial[INITIAL_FUN] = func; + p->initial[INITIAL_ARI] = (Uint) arity; + + /* + * Must initialize binary lists here before copying binaries to process. + */ + p->off_heap.mso = NULL; +#ifndef HYBRID /* FIND ME! */ + p->off_heap.funs = NULL; +#endif + p->off_heap.externals = NULL; + p->off_heap.overhead = 0; + + heap_need += + IS_CONST(parent->group_leader) ? 0 : NC_HEAP_SIZE(parent->group_leader); + + if (heap_need < p->min_heap_size) { + sz = heap_need = p->min_heap_size; + } else { + sz = erts_next_heap_size(heap_need, 0); + } + +#ifdef HIPE + hipe_init_process(&p->hipe); +#ifdef ERTS_SMP + hipe_init_process_smp(&p->hipe_smp); +#endif +#endif + + p->heap = (Eterm *) ERTS_HEAP_ALLOC(ERTS_ALC_T_HEAP, sizeof(Eterm)*sz); + p->old_hend = p->old_htop = p->old_heap = NULL; + p->high_water = p->heap; +#ifdef INCREMENTAL + p->scan_top = p->high_water; +#endif + p->gen_gcs = 0; + p->stop = p->hend = p->heap + sz; + p->htop = p->heap; + p->heap_sz = sz; + p->catches = 0; + + p->bin_vheap_sz = H_MIN_SIZE; + p->bin_old_vheap_sz = H_MIN_SIZE; + p->bin_old_vheap = 0; + + /* No need to initialize p->fcalls. */ + + p->current = p->initial+INITIAL_MOD; + + p->i = (Eterm *) beam_apply; + p->cp = (Eterm *) beam_apply+1; + + p->arg_reg = p->def_arg_reg; + p->max_arg_reg = sizeof(p->def_arg_reg)/sizeof(p->def_arg_reg[0]); + p->arg_reg[0] = mod; + p->arg_reg[1] = func; + BM_STOP_TIMER(system); + BM_MESSAGE(args,p,parent); + BM_START_TIMER(system); +#ifdef HYBRID + p->arg_reg[2] = args; +#ifdef INCREMENTAL + p->active = 0; + if (ptr_val(args) >= inc_fromspc && ptr_val(args) < inc_fromend) + INC_ACTIVATE(p); +#endif +#else + BM_SWAP_TIMER(system,copy); + p->arg_reg[2] = copy_struct(args, arg_size, &p->htop, &p->off_heap); + BM_MESSAGE_COPIED(arg_size); + BM_SWAP_TIMER(copy,system); +#endif + p->arity = 3; + + p->fvalue = NIL; + p->freason = EXC_NULL; + p->ftrace = NIL; + p->reds = 0; + +#ifdef ERTS_SMP + p->u.ptimer = NULL; +#else + sys_memset(&p->u.tm, 0, sizeof(ErlTimer)); +#endif + + p->reg = NULL; + p->nlinks = NULL; + p->monitors = NULL; + p->nodes_monitors = NULL; + p->suspend_monitors = NULL; + + ASSERT(is_pid(parent->group_leader)); + + if (parent->group_leader == ERTS_INVALID_PID) + p->group_leader = p->id; + else { + /* Needs to be done after the heap has been set up */ + p->group_leader = + IS_CONST(parent->group_leader) + ? parent->group_leader + : STORE_NC(&p->htop, &p->off_heap.externals, parent->group_leader); + } + + erts_get_default_tracing(&p->trace_flags, &p->tracer_proc); + + p->msg.first = NULL; + p->msg.last = &p->msg.first; + p->msg.save = &p->msg.first; + p->msg.len = 0; +#ifdef ERTS_SMP + p->msg_inq.first = NULL; + p->msg_inq.last = &p->msg_inq.first; + p->msg_inq.len = 0; + p->bound_runq = NULL; +#endif + p->bif_timers = NULL; + p->mbuf = NULL; + p->mbuf_sz = 0; + p->psd = NULL; + p->dictionary = NULL; + p->seq_trace_lastcnt = 0; + p->seq_trace_clock = 0; + SEQ_TRACE_TOKEN(p) = NIL; + p->parent = parent->id == ERTS_INVALID_PID ? NIL : parent->id; + +#ifdef HYBRID + p->rrma = NULL; + p->rrsrc = NULL; + p->nrr = 0; + p->rrsz = 0; +#endif + + INIT_HOLE_CHECK(p); +#ifdef DEBUG + p->last_old_htop = NULL; +#endif + + if (IS_TRACED(parent)) { + if (parent->trace_flags & F_TRACE_SOS) { + p->trace_flags |= (parent->trace_flags & TRACEE_FLAGS); + p->tracer_proc = parent->tracer_proc; + } + if (ARE_TRACE_FLAGS_ON(parent, F_TRACE_PROCS)) { + trace_proc_spawn(parent, p->id, mod, func, args); + } + if (parent->trace_flags & F_TRACE_SOS1) { /* Overrides TRACE_CHILDREN */ + p->trace_flags |= (parent->trace_flags & TRACEE_FLAGS); + p->tracer_proc = parent->tracer_proc; + p->trace_flags &= ~(F_TRACE_SOS1 | F_TRACE_SOS); + parent->trace_flags &= ~(F_TRACE_SOS1 | F_TRACE_SOS); + } + } + + /* + * Check if this process should be initially linked to its parent. + */ + + if (so->flags & SPO_LINK) { +#ifdef DEBUG + int ret; +#endif + if (IS_TRACED_FL(parent, F_TRACE_PROCS)) { + trace_proc(parent, parent, am_link, p->id); + } + +#ifdef DEBUG + ret = erts_add_link(&(parent->nlinks), LINK_PID, p->id); + ASSERT(ret == 0); + ret = erts_add_link(&(p->nlinks), LINK_PID, parent->id); + ASSERT(ret == 0); +#else + erts_add_link(&(parent->nlinks), LINK_PID, p->id); + erts_add_link(&(p->nlinks), LINK_PID, parent->id); +#endif + + if (IS_TRACED(parent)) { + if (parent->trace_flags & (F_TRACE_SOL|F_TRACE_SOL1)) { + p->trace_flags |= (parent->trace_flags & TRACEE_FLAGS); + p->tracer_proc = parent->tracer_proc; /* maybe steal */ + + if (parent->trace_flags & F_TRACE_SOL1) { /* maybe override */ + p ->trace_flags &= ~(F_TRACE_SOL1 | F_TRACE_SOL); + parent->trace_flags &= ~(F_TRACE_SOL1 | F_TRACE_SOL); + } + } + } + } + + /* + * Test whether this process should be initially monitored by its parent. + */ + if (so->flags & SPO_MONITOR) { + Eterm mref; + + mref = erts_make_ref(parent); + erts_add_monitor(&(parent->monitors), MON_ORIGIN, mref, p->id, NIL); + erts_add_monitor(&(p->monitors), MON_TARGET, mref, parent->id, NIL); + so->mref = mref; + } + +#ifdef HYBRID + /* + * Add process to the array of active processes. + */ + ACTIVATE(p); + p->active_index = erts_num_active_procs++; + erts_active_procs[p->active_index] = p; +#endif + +#ifdef ERTS_SMP + p->scheduler_data = NULL; + p->is_exiting = 0; + p->status_flags = 0; + p->runq_flags = 0; + p->suspendee = NIL; + p->pending_suspenders = NULL; + p->pending_exit.reason = THE_NON_VALUE; + p->pending_exit.bp = NULL; +#endif + +#if !defined(NO_FPE_SIGNALS) + p->fp_exception = 0; +#endif + + /* + * Schedule process for execution. + */ + + if (!((so->flags & SPO_USE_ARGS) && so->scheduler)) + rq = erts_get_runq_proc(parent); + else { + int ix = so->scheduler-1; + ASSERT(0 <= ix && ix < erts_no_run_queues); + rq = ERTS_RUNQ_IX(ix); + p->bound_runq = rq; + } + + erts_smp_runq_lock(rq); + +#ifdef ERTS_SMP + p->run_queue = rq; +#endif + + p->status = P_WAITING; + internal_add_to_runq(rq, p); + + erts_smp_runq_unlock(rq); + + res = p->id; + erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL); + + VERBOSE(DEBUG_PROCESSES, ("Created a new process: %T\n",p->id)); + + error: + + erts_smp_proc_unlock(parent, ERTS_PROC_LOCKS_ALL_MINOR); + + return res; +} + +/* + * Initiates a pseudo process that can be used + * for arithmetic BIFs. + */ + +void erts_init_empty_process(Process *p) +{ + p->htop = NULL; + p->stop = NULL; + p->hend = NULL; + p->heap = NULL; + p->gen_gcs = 0; + p->max_gen_gcs = 0; + p->min_heap_size = 0; + p->status = P_RUNABLE; + p->gcstatus = P_RUNABLE; + p->rstatus = P_RUNABLE; + p->rcount = 0; + p->id = ERTS_INVALID_PID; + p->prio = PRIORITY_NORMAL; + p->reds = 0; + p->tracer_proc = NIL; + p->trace_flags = F_INITIAL_TRACE_FLAGS; + p->group_leader = ERTS_INVALID_PID; + p->flags = 0; + p->fvalue = NIL; + p->freason = EXC_NULL; + p->ftrace = NIL; + p->fcalls = 0; + + p->bin_vheap_sz=H_MIN_SIZE; + p->bin_old_vheap_sz=H_MIN_SIZE; + p->bin_old_vheap = 0; +#ifdef ERTS_SMP + p->u.ptimer = NULL; + p->bound_runq = NULL; +#else + memset(&(p->u.tm), 0, sizeof(ErlTimer)); +#endif + p->next = NULL; + p->off_heap.mso = NULL; +#ifndef HYBRID /* FIND ME! */ + p->off_heap.funs = NULL; +#endif + p->off_heap.externals = NULL; + p->off_heap.overhead = 0; + p->reg = NULL; + p->heap_sz = 0; + p->high_water = NULL; +#ifdef INCREMENTAL + p->scan_top = NULL; +#endif + p->old_hend = NULL; + p->old_htop = NULL; + p->old_heap = NULL; + p->mbuf = NULL; + p->mbuf_sz = 0; + p->psd = NULL; + p->monitors = NULL; + p->nlinks = NULL; /* List of links */ + p->nodes_monitors = NULL; + p->suspend_monitors = NULL; + p->msg.first = NULL; + p->msg.last = &p->msg.first; + p->msg.save = &p->msg.first; + p->msg.len = 0; + p->bif_timers = NULL; + p->dictionary = NULL; + p->seq_trace_clock = 0; + p->seq_trace_lastcnt = 0; + p->seq_trace_token = NIL; + p->initial[0] = 0; + p->initial[1] = 0; + p->initial[2] = 0; + p->catches = 0; + p->cp = NULL; + p->i = NULL; + p->current = NULL; + + /* + * Saved x registers. + */ + p->arity = 0; + p->arg_reg = NULL; + p->max_arg_reg = 0; + p->def_arg_reg[0] = 0; + p->def_arg_reg[1] = 0; + p->def_arg_reg[2] = 0; + p->def_arg_reg[3] = 0; + p->def_arg_reg[4] = 0; + p->def_arg_reg[5] = 0; + + p->parent = NIL; + p->started.tv_sec = 0; + p->started.tv_usec = 0; + +#ifdef HIPE + hipe_init_process(&p->hipe); +#ifdef ERTS_SMP + hipe_init_process_smp(&p->hipe_smp); +#endif +#endif + + ACTIVATE(p); + +#ifdef HYBRID + p->rrma = NULL; + p->rrsrc = NULL; + p->nrr = 0; + p->rrsz = 0; +#endif + INIT_HOLE_CHECK(p); +#ifdef DEBUG + p->last_old_htop = NULL; +#endif + + +#ifdef ERTS_SMP + p->scheduler_data = NULL; + p->is_exiting = 0; + p->status_flags = 0; + p->runq_flags = 0; + p->msg_inq.first = NULL; + p->msg_inq.last = &p->msg_inq.first; + p->msg_inq.len = 0; + p->suspendee = NIL; + p->pending_suspenders = NULL; + p->pending_exit.reason = THE_NON_VALUE; + p->pending_exit.bp = NULL; + erts_proc_lock_init(p); + erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL); + p->run_queue = ERTS_RUNQ_IX(0); +#endif + +#if !defined(NO_FPE_SIGNALS) + p->fp_exception = 0; +#endif + +} + +#ifdef DEBUG + +void +erts_debug_verify_clean_empty_process(Process* p) +{ + /* Things that erts_cleanup_empty_process() will *not* cleanup... */ + ASSERT(p->htop == NULL); + ASSERT(p->stop == NULL); + ASSERT(p->hend == NULL); + ASSERT(p->heap == NULL); + ASSERT(p->id == ERTS_INVALID_PID); + ASSERT(p->tracer_proc == NIL); + ASSERT(p->trace_flags == F_INITIAL_TRACE_FLAGS); + ASSERT(p->group_leader == ERTS_INVALID_PID); + ASSERT(p->next == NULL); + ASSERT(p->reg == NULL); + ASSERT(p->heap_sz == 0); + ASSERT(p->high_water == NULL); +#ifdef INCREMENTAL + ASSERT(p->scan_top == NULL); +#endif + ASSERT(p->old_hend == NULL); + ASSERT(p->old_htop == NULL); + ASSERT(p->old_heap == NULL); + + ASSERT(p->monitors == NULL); + ASSERT(p->nlinks == NULL); + ASSERT(p->nodes_monitors == NULL); + ASSERT(p->suspend_monitors == NULL); + ASSERT(p->msg.first == NULL); + ASSERT(p->msg.len == 0); + ASSERT(p->bif_timers == NULL); + ASSERT(p->dictionary == NULL); + ASSERT(p->catches == 0); + ASSERT(p->cp == NULL); + ASSERT(p->i == NULL); + ASSERT(p->current == NULL); + + ASSERT(p->parent == NIL); + +#ifdef ERTS_SMP + ASSERT(p->msg_inq.first == NULL); + ASSERT(p->msg_inq.len == 0); + ASSERT(p->suspendee == NIL); + ASSERT(p->pending_suspenders == NULL); + ASSERT(p->pending_exit.reason == THE_NON_VALUE); + ASSERT(p->pending_exit.bp == NULL); +#endif + + /* Thing that erts_cleanup_empty_process() cleans up */ + + ASSERT(p->off_heap.mso == NULL); +#ifndef HYBRID /* FIND ME! */ + ASSERT(p->off_heap.funs == NULL); +#endif + ASSERT(p->off_heap.externals == NULL); + ASSERT(p->off_heap.overhead == 0); + + ASSERT(p->mbuf == NULL); +} + +#endif + +void +erts_cleanup_empty_process(Process* p) +{ + ErlHeapFragment* mbufp; + + /* We only check fields that are known to be used... */ + + erts_cleanup_offheap(&p->off_heap); + p->off_heap.mso = NULL; +#ifndef HYBRID /* FIND ME! */ + p->off_heap.funs = NULL; +#endif + p->off_heap.externals = NULL; + p->off_heap.overhead = 0; + + mbufp = p->mbuf; + while (mbufp) { + ErlHeapFragment *next = mbufp->next; + free_message_buffer(mbufp); + mbufp = next; + } + p->mbuf = NULL; +#if defined(ERTS_ENABLE_LOCK_COUNT) && defined(ERTS_SMP) + erts_lcnt_proc_lock_destroy(p); +#endif +#ifdef DEBUG + erts_debug_verify_clean_empty_process(p); +#endif +} + +/* + * p must be the currently executing process. + */ +static void +delete_process(Process* p) +{ + ErlMessage* mp; + ErlHeapFragment* bp; + + VERBOSE(DEBUG_PROCESSES, ("Removing process: %T\n",p->id)); + + /* Cleanup psd */ + + if (p->psd) + erts_free(ERTS_ALC_T_PSD, p->psd); + + /* Clean binaries and funs */ + erts_cleanup_offheap(&p->off_heap); + + /* + * The mso list should not be used anymore, but if it is, make sure that + * we'll notice. + */ + p->off_heap.mso = (void *) 0x8DEFFACD; + + if (p->arg_reg != p->def_arg_reg) { + erts_free(ERTS_ALC_T_ARG_REG, p->arg_reg); + } + + /* + * Release heaps. Clobber contents in DEBUG build. + */ + + +#ifdef DEBUG + sys_memset(p->heap, DEBUG_BAD_BYTE, p->heap_sz*sizeof(Eterm)); +#endif + +#ifdef HIPE + hipe_delete_process(&p->hipe); +#endif + + ERTS_HEAP_FREE(ERTS_ALC_T_HEAP, (void*) p->heap, p->heap_sz*sizeof(Eterm)); + if (p->old_heap != NULL) { + +#ifdef DEBUG + sys_memset(p->old_heap, DEBUG_BAD_BYTE, + (p->old_hend-p->old_heap)*sizeof(Eterm)); +#endif + ERTS_HEAP_FREE(ERTS_ALC_T_OLD_HEAP, + p->old_heap, + (p->old_hend-p->old_heap)*sizeof(Eterm)); + } + + /* + * Free all pending message buffers. + */ + bp = p->mbuf; + while (bp != NULL) { + ErlHeapFragment* next_bp = bp->next; + free_message_buffer(bp); + bp = next_bp; + } + + erts_erase_dicts(p); + + /* free all pending messages */ + mp = p->msg.first; + while(mp != NULL) { + ErlMessage* next_mp = mp->next; + if (mp->data.attached) { + if (is_value(mp->m[0])) + free_message_buffer(mp->data.heap_frag); + else { + if (is_not_nil(mp->m[1])) { + ErlHeapFragment *heap_frag; + heap_frag = (ErlHeapFragment *) mp->data.dist_ext->ext_endp; + erts_cleanup_offheap(&heap_frag->off_heap); + } + erts_free_dist_ext_copy(mp->data.dist_ext); + } + } + free_message(mp); + mp = next_mp; + } + + ASSERT(!p->monitors); + ASSERT(!p->nlinks); + ASSERT(!p->nodes_monitors); + ASSERT(!p->suspend_monitors); + + p->fvalue = NIL; + +#ifdef HYBRID + erts_active_procs[p->active_index] = + erts_active_procs[--erts_num_active_procs]; + erts_active_procs[p->active_index]->active_index = p->active_index; +#ifdef INCREMENTAL + if (INC_IS_ACTIVE(p)) + INC_DEACTIVATE(p); +#endif + + if (p->rrma != NULL) { + erts_free(ERTS_ALC_T_ROOTSET,p->rrma); + erts_free(ERTS_ALC_T_ROOTSET,p->rrsrc); + } +#endif + +} + +static ERTS_INLINE void +set_proc_exiting(Process *p, Eterm reason, ErlHeapFragment *bp) +{ +#ifdef ERTS_SMP + erts_pix_lock_t *pix_lock = ERTS_PID2PIXLOCK(p->id); + ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(p) == ERTS_PROC_LOCKS_ALL); + /* + * You are required to have all proc locks and the pix lock when going + * to status P_EXITING. This makes it is enough to take any lock when + * looking up a process (pid2proc()) to prevent the looked up process + * from exiting until the lock has been released. + */ + + erts_pix_lock(pix_lock); + p->is_exiting = 1; +#endif + p->status = P_EXITING; +#ifdef ERTS_SMP + erts_pix_unlock(pix_lock); +#endif + p->fvalue = reason; + if (bp) + erts_link_mbuf_to_proc(p, bp); + /* + * We used to set freason to EXC_EXIT here, but there is no need to + * save the stack trace since this process irreversibly is going to + * exit. + */ + p->freason = EXTAG_EXIT; + KILL_CATCHES(p); + cancel_timer(p); + p->i = (Eterm *) beam_exit; +} + + +#ifdef ERTS_SMP + +void +erts_handle_pending_exit(Process *c_p, ErtsProcLocks locks) +{ + ErtsProcLocks xlocks; + ASSERT(is_value(c_p->pending_exit.reason)); + ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == locks); + ERTS_SMP_LC_ASSERT(locks & ERTS_PROC_LOCK_MAIN); + ERTS_SMP_LC_ASSERT(c_p->status != P_EXITING); + ERTS_SMP_LC_ASSERT(c_p->status != P_FREE); + + /* Ensure that all locks on c_p are locked before proceeding... */ + if (locks == ERTS_PROC_LOCKS_ALL) + xlocks = 0; + else { + xlocks = ~locks & ERTS_PROC_LOCKS_ALL; + if (erts_smp_proc_trylock(c_p, xlocks) == EBUSY) { + erts_smp_proc_unlock(c_p, locks & ~ERTS_PROC_LOCK_MAIN); + erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); + } + } + + set_proc_exiting(c_p, c_p->pending_exit.reason, c_p->pending_exit.bp); + c_p->pending_exit.reason = THE_NON_VALUE; + c_p->pending_exit.bp = NULL; + + if (xlocks) + erts_smp_proc_unlock(c_p, xlocks); +} + +static void +handle_pending_exiters(ErtsProcList *pnd_xtrs) +{ + ErtsProcList *plp = pnd_xtrs; + ErtsProcList *free_plp; + while (plp) { + Process *p = erts_pid2proc(NULL, 0, plp->pid, ERTS_PROC_LOCKS_ALL); + if (p) { + if (proclist_same(plp, p) + && !(p->status_flags & ERTS_PROC_SFLG_RUNNING)) { + ASSERT(p->status_flags & ERTS_PROC_SFLG_INRUNQ); + ASSERT(ERTS_PROC_PENDING_EXIT(p)); + erts_handle_pending_exit(p, ERTS_PROC_LOCKS_ALL); + } + erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL); + } + free_plp = plp; + plp = plp->next; + proclist_destroy(free_plp); + } +} + +static void +save_pending_exiter(Process *p) +{ + ErtsProcList *plp; + ErtsRunQueue *rq; + + ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); + + rq = erts_get_runq_current(NULL); + + plp = proclist_create(p); + + erts_smp_runq_lock(rq); + + plp->next = rq->procs.pending_exiters; + rq->procs.pending_exiters = plp; + + erts_smp_runq_unlock(rq); + +} + +#endif + +/* + * This function delivers an EXIT message to a process + * which is trapping EXITs. + */ + +static ERTS_INLINE void +send_exit_message(Process *to, ErtsProcLocks *to_locksp, + Eterm exit_term, Uint term_size, Eterm token) +{ + if (token == NIL) { + Eterm* hp; + Eterm mess; + ErlHeapFragment* bp; + ErlOffHeap *ohp; + + hp = erts_alloc_message_heap(term_size, &bp, &ohp, to, to_locksp); + mess = copy_struct(exit_term, term_size, &hp, ohp); + erts_queue_message(to, to_locksp, bp, mess, NIL); + } else { + ErlHeapFragment* bp; + Eterm* hp; + Eterm mess; + Eterm temp_token; + Uint sz_token; + + ASSERT(is_tuple(token)); + sz_token = size_object(token); + bp = new_message_buffer(term_size+sz_token); + hp = bp->mem; + mess = copy_struct(exit_term, term_size, &hp, &bp->off_heap); + /* the trace token must in this case be updated by the caller */ + seq_trace_output(token, mess, SEQ_TRACE_SEND, to->id, NULL); + temp_token = copy_struct(token, sz_token, &hp, &bp->off_heap); + erts_queue_message(to, to_locksp, bp, mess, temp_token); + } +} + +/* + * + * *** Exit signal behavior *** + * + * Exit signals are asynchronous (truly asynchronous in the + * SMP emulator). When the signal is received the receiver receives an + * 'EXIT' message if it is trapping exits; otherwise, it will either + * ignore the signal if the exit reason is normal, or go into an + * exiting state (status P_EXITING). When a process has gone into the + * exiting state it will not execute any more Erlang code, but it might + * take a while before it actually exits. The exit signal is being + * received when the 'EXIT' message is put in the message queue, the + * signal is dropped, or when it changes state into exiting. The time it + * is in the exiting state before actually exiting is undefined (it + * might take a really long time under certain conditions). The + * receiver of the exit signal does not break links or trigger monitors + * until it actually exits. + * + * Exit signals and other signals, e.g. messages, have to be received + * by a receiver in the same order as sent by a sender. + * + * + * + * Exit signal implementation in the SMP emulator: + * + * If the receiver is trapping exits, the signal is transformed + * into an 'EXIT' message and sent as a normal message, if the + * reason is normal the signal is dropped; otherwise, the process + * is determined to be exited. The interesting case is when the + * process is to be exited and this is what is described below. + * + * If it is possible, the receiver is set in the exiting state straight + * away and we are done; otherwise, the sender places the exit reason + * in the pending_exit field of the process struct and if necessary + * adds the receiver to the run queue. It is typically not possible + * to set a scheduled process or a process which we cannot get all locks + * on without releasing locks on it in an exiting state straight away. + * + * The receiver will poll the pending_exit field when it reach certain + * places during it's execution. When it discovers the pending exit + * it will change state into the exiting state. If the receiver wasn't + * scheduled when the pending exit was set, the first scheduler that + * schedules a new process will set the receiving process in the exiting + * state just before it schedules next process. + * + * When the exit signal is placed in the pending_exit field, the signal + * is considered as being in transit on the Erlang level. The signal is + * actually in some kind of semi transit state, since we have already + * determined how it should be received. It will exit the process no + * matter what if it is received (the process may exit by itself before + * reception of the exit signal). The signal is received when it is + * discovered in the pending_exit field by the receiver. + * + * The receiver have to poll the pending_exit field at least before: + * - moving messages from the message in queue to the private message + * queue. This in order to preserve signal order. + * - unlink. Otherwise the process might get exited on a link that + * have been removed. + * - changing the trap_exit flag to true. This in order to simplify the + * implementation; otherwise, we would have to transform the signal + * into an 'EXIT' message when setting the trap_exit flag to true. We + * would also have to maintain a queue of exit signals in transit. + * - being scheduled in or out. + */ + +static ERTS_INLINE int +send_exit_signal(Process *c_p, /* current process if and only + if reason is stored on it */ + Eterm from, /* Id of sender of signal */ + Process *rp, /* receiving process */ + ErtsProcLocks *rp_locks,/* current locks on receiver */ + Eterm reason, /* exit reason */ + Eterm exit_tuple, /* Prebuild exit tuple + or THE_NON_VALUE */ + Uint exit_tuple_sz, /* Size of prebuilt exit tuple + (if exit_tuple != THE_NON_VALUE) */ + Eterm token, /* token */ + Process *token_update, /* token updater */ + Uint32 flags /* flags */ + ) +{ + Eterm rsn = reason == am_kill ? am_killed : reason; + + ERTS_SMP_LC_ASSERT(*rp_locks == erts_proc_lc_my_proc_locks(rp)); + ERTS_SMP_LC_ASSERT((*rp_locks & ERTS_PROC_LOCKS_XSIG_SEND) + == ERTS_PROC_LOCKS_XSIG_SEND); + + ASSERT(reason != THE_NON_VALUE); + + if (ERTS_PROC_IS_TRAPPING_EXITS(rp) + && (reason != am_kill || (flags & ERTS_XSIG_FLG_IGN_KILL))) { + if (is_not_nil(token) && token_update) + seq_trace_update_send(token_update); + if (is_value(exit_tuple)) + send_exit_message(rp, rp_locks, exit_tuple, exit_tuple_sz, token); + else + erts_deliver_exit_message(from, rp, rp_locks, rsn, token); + return 1; /* Receiver will get a message */ + } + else if (reason != am_normal || (flags & ERTS_XSIG_FLG_NO_IGN_NORMAL)) { +#ifdef ERTS_SMP + if (!ERTS_PROC_PENDING_EXIT(rp) && !rp->is_exiting) { + ASSERT(rp->status != P_EXITING); + ASSERT(rp->status != P_FREE); + ASSERT(!rp->pending_exit.bp); + + if (rp == c_p && (*rp_locks & ERTS_PROC_LOCK_MAIN)) { + /* Ensure that all locks on c_p are locked before + proceeding... */ + if (*rp_locks != ERTS_PROC_LOCKS_ALL) { + ErtsProcLocks need_locks = (~(*rp_locks) + & ERTS_PROC_LOCKS_ALL); + if (erts_smp_proc_trylock(c_p, need_locks) == EBUSY) { + erts_smp_proc_unlock(c_p, + *rp_locks & ~ERTS_PROC_LOCK_MAIN); + erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); + } + *rp_locks = ERTS_PROC_LOCKS_ALL; + } + set_proc_exiting(c_p, rsn, NULL); + } + else if (!(rp->status_flags & ERTS_PROC_SFLG_RUNNING)) { + /* Process not running ... */ + ErtsProcLocks need_locks = ~(*rp_locks) & ERTS_PROC_LOCKS_ALL; + if (need_locks + && erts_smp_proc_trylock(rp, need_locks) == EBUSY) { + /* ... but we havn't got all locks on it ... */ + save_pending_exiter(rp); + /* + * The pending exit will be discovered when next + * process is scheduled in + */ + goto set_pending_exit; + } + else { + /* ...and we have all locks on it... */ + *rp_locks = ERTS_PROC_LOCKS_ALL; + set_proc_exiting(rp, + (is_immed(rsn) + ? rsn + : copy_object(rsn, rp)), + NULL); + } + } + else { /* Process running... */ + + /* + * The pending exit will be discovered when the process + * is scheduled out if not discovered earlier. + */ + + set_pending_exit: + if (is_immed(rsn)) { + rp->pending_exit.reason = rsn; + } + else { + Eterm *hp; + Uint sz = size_object(rsn); + ErlHeapFragment *bp = new_message_buffer(sz); + + hp = &bp->mem[0]; + rp->pending_exit.reason = copy_struct(rsn, + sz, + &hp, + &bp->off_heap); + rp->pending_exit.bp = bp; + } + ASSERT(ERTS_PROC_PENDING_EXIT(rp)); + } + if (!(rp->status_flags + & (ERTS_PROC_SFLG_INRUNQ|ERTS_PROC_SFLG_RUNNING))) + erts_add_to_runq(rp); + } + /* else: + * + * The receiver already has a pending exit (or is exiting) + * so we drop this signal. + * + * NOTE: dropping this exit signal is based on the assumption + * that the receiver *will* exit; either on the pending + * exit or by itself before seeing the pending exit. + */ +#else /* !ERTS_SMP */ + if (c_p == rp) { + rp->status = P_EXITING; + c_p->fvalue = rsn; + } + else if (rp->status != P_EXITING) { /* No recursive process exits /PaN */ + Eterm old_status = rp->status; + set_proc_exiting(rp, + is_immed(rsn) ? rsn : copy_object(rsn, rp), + NULL); + ACTIVATE(rp); + if (old_status != P_RUNABLE && old_status != P_RUNNING) + erts_add_to_runq(rp); + } +#endif + return -1; /* Receiver will exit */ + } + + return 0; /* Receiver unaffected */ +} + + +int +erts_send_exit_signal(Process *c_p, + Eterm from, + Process *rp, + ErtsProcLocks *rp_locks, + Eterm reason, + Eterm token, + Process *token_update, + Uint32 flags) +{ + return send_exit_signal(c_p, + from, + rp, + rp_locks, + reason, + THE_NON_VALUE, + 0, + token, + token_update, + flags); +} + +typedef struct { + Eterm reason; + Process *p; +} ExitMonitorContext; + +static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext) +{ + ExitMonitorContext *pcontext = vpcontext; + DistEntry *dep; + ErtsMonitor *rmon; + Process *rp; + + if (mon->type == MON_ORIGIN) { + /* We are monitoring someone else, we need to demonitor that one.. */ + if (is_atom(mon->pid)) { /* remote by name */ + ASSERT(is_node_name_atom(mon->pid)); + dep = erts_sysname_to_connected_dist_entry(mon->pid); + if (dep) { + erts_smp_de_links_lock(dep); + rmon = erts_remove_monitor(&(dep->monitors), mon->ref); + erts_smp_de_links_unlock(dep); + if (rmon) { + ErtsDSigData dsd; + int code = erts_dsig_prepare(&dsd, dep, NULL, + ERTS_DSP_NO_LOCK, 0); + if (code == ERTS_DSIG_PREP_CONNECTED) { + code = erts_dsig_send_demonitor(&dsd, + rmon->pid, + mon->name, + mon->ref, + 1); + ASSERT(code == ERTS_DSIG_SEND_OK); + } + erts_destroy_monitor(rmon); + } + erts_deref_dist_entry(dep); + } + } else { + ASSERT(is_pid(mon->pid)); + if (is_internal_pid(mon->pid)) { /* local by pid or name */ + rp = erts_pid2proc(NULL, 0, mon->pid, ERTS_PROC_LOCK_LINK); + if (!rp) { + goto done; + } + rmon = erts_remove_monitor(&(rp->monitors),mon->ref); + erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); + if (rmon == NULL) { + goto done; + } + erts_destroy_monitor(rmon); + } else { /* remote by pid */ + ASSERT(is_external_pid(mon->pid)); + dep = external_pid_dist_entry(mon->pid); + ASSERT(dep != NULL); + if (dep) { + erts_smp_de_links_lock(dep); + rmon = erts_remove_monitor(&(dep->monitors), mon->ref); + erts_smp_de_links_unlock(dep); + if (rmon) { + ErtsDSigData dsd; + int code = erts_dsig_prepare(&dsd, dep, NULL, + ERTS_DSP_NO_LOCK, 0); + if (code == ERTS_DSIG_PREP_CONNECTED) { + code = erts_dsig_send_demonitor(&dsd, + rmon->pid, + mon->pid, + mon->ref, + 1); + ASSERT(code == ERTS_DSIG_SEND_OK); + } + erts_destroy_monitor(rmon); + } + } + } + } + } else { /* type == MON_TARGET */ + ASSERT(mon->type == MON_TARGET); + ASSERT(is_pid(mon->pid) || is_internal_port(mon->pid)); + if (is_internal_port(mon->pid)) { + Port *prt = erts_id2port(mon->pid, NULL, 0); + if (prt == NULL) { + goto done; + } + erts_fire_port_monitor(prt, mon->ref); + erts_port_release(prt); + } else if (is_internal_pid(mon->pid)) {/* local by name or pid */ + Eterm watched; + Eterm lhp[3]; + ErtsProcLocks rp_locks = (ERTS_PROC_LOCK_LINK + | ERTS_PROC_LOCKS_MSG_SEND); + rp = erts_pid2proc(NULL, 0, mon->pid, rp_locks); + if (rp == NULL) { + goto done; + } + rmon = erts_remove_monitor(&(rp->monitors),mon->ref); + if (rmon) { + erts_destroy_monitor(rmon); + watched = (is_atom(mon->name) + ? TUPLE2(lhp, mon->name, + erts_this_dist_entry->sysname) + : pcontext->p->id); + erts_queue_monitor_message(rp, &rp_locks, mon->ref, am_process, + watched, pcontext->reason); + } + /* else: demonitor while we exited, i.e. do nothing... */ + erts_smp_proc_unlock(rp, rp_locks); + } else { /* external by pid or name */ + ASSERT(is_external_pid(mon->pid)); + dep = external_pid_dist_entry(mon->pid); + ASSERT(dep != NULL); + if (dep) { + erts_smp_de_links_lock(dep); + rmon = erts_remove_monitor(&(dep->monitors), mon->ref); + erts_smp_de_links_unlock(dep); + if (rmon) { + ErtsDSigData dsd; + int code = erts_dsig_prepare(&dsd, dep, NULL, + ERTS_DSP_NO_LOCK, 0); + if (code == ERTS_DSIG_PREP_CONNECTED) { + code = erts_dsig_send_m_exit(&dsd, + mon->pid, + (rmon->name != NIL + ? rmon->name + : rmon->pid), + mon->ref, + pcontext->reason); + ASSERT(code == ERTS_DSIG_SEND_OK); + } + erts_destroy_monitor(rmon); + } + } + } + } + done: + /* As the monitors are previously removed from the process, + distribution operations will not cause monitors to disappear, + we can safely delete it. */ + + erts_destroy_monitor(mon); +} + +typedef struct { + Process *p; + Eterm reason; + Eterm exit_tuple; + Uint exit_tuple_sz; +} ExitLinkContext; + +static void doit_exit_link(ErtsLink *lnk, void *vpcontext) +{ + ExitLinkContext *pcontext = vpcontext; + /* Unpack context, it's readonly */ + Process *p = pcontext->p; + Eterm reason = pcontext->reason; + Eterm exit_tuple = pcontext->exit_tuple; + Uint exit_tuple_sz = pcontext->exit_tuple_sz; + Eterm item = lnk->pid; + ErtsLink *rlnk; + DistEntry *dep; + Process *rp; + + switch(lnk->type) { + case LINK_PID: + if(is_internal_port(item)) { + Port *prt = erts_id2port(item, NULL, 0); + if (prt) { + rlnk = erts_remove_link(&prt->nlinks, p->id); + if (rlnk) + erts_destroy_link(rlnk); + erts_do_exit_port(prt, p->id, reason); + erts_port_release(prt); + } + } + else if(is_external_port(item)) { + erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); + erts_dsprintf(dsbufp, + "Erroneous link between %T and external port %T " + "found\n", + p->id, + item); + erts_send_error_to_logger_nogl(dsbufp); + ASSERT(0); /* It isn't possible to setup such a link... */ + } + else if (is_internal_pid(item)) { + ErtsProcLocks rp_locks = (ERTS_PROC_LOCK_LINK + | ERTS_PROC_LOCKS_XSIG_SEND); + rp = erts_pid2proc(NULL, 0, item, rp_locks); + if (rp) { + rlnk = erts_remove_link(&(rp->nlinks), p->id); + /* If rlnk == NULL, we got unlinked while exiting, + i.e., do nothing... */ + if (rlnk) { + int xres; + erts_destroy_link(rlnk); + xres = send_exit_signal(NULL, + p->id, + rp, + &rp_locks, + reason, + exit_tuple, + exit_tuple_sz, + SEQ_TRACE_TOKEN(p), + p, + ERTS_XSIG_FLG_IGN_KILL); + if (xres >= 0 && IS_TRACED_FL(rp, F_TRACE_PROCS)) { + /* We didn't exit the process and it is traced */ + if (IS_TRACED_FL(rp, F_TRACE_PROCS)) { + trace_proc(p, rp, am_getting_unlinked, p->id); + } + } + } + ASSERT(rp != p); + erts_smp_proc_unlock(rp, rp_locks); + } + } + else if (is_external_pid(item)) { + dep = external_pid_dist_entry(item); + if(dep != erts_this_dist_entry) { + ErtsDSigData dsd; + int code; + ErtsDistLinkData dld; + erts_remove_dist_link(&dld, p->id, item, dep); + erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); + code = erts_dsig_prepare(&dsd, dep, p, ERTS_DSP_NO_LOCK, 0); + if (code == ERTS_DSIG_PREP_CONNECTED) { + code = erts_dsig_send_exit_tt(&dsd, p->id, item, reason, + SEQ_TRACE_TOKEN(p)); + ASSERT(code == ERTS_DSIG_SEND_OK); + } + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN); + erts_destroy_dist_link(&dld); + } + } + break; + case LINK_NODE: + ASSERT(is_node_name_atom(item)); + dep = erts_sysname_to_connected_dist_entry(item); + if(dep) { + /* dist entries have node links in a separate structure to + avoid confusion */ + erts_smp_de_links_lock(dep); + rlnk = erts_remove_link(&(dep->node_links), p->id); + erts_smp_de_links_unlock(dep); + if (rlnk) + erts_destroy_link(rlnk); + erts_deref_dist_entry(dep); + } else { +#ifndef ERTS_SMP + /* XXX Is this possible? Shouldn't this link + previously have been removed if the node + had previously been disconnected. */ + ASSERT(0); +#endif + /* This is possible when smp support has been enabled, + and dist port and process exits simultaneously. */ + } + break; + + default: + erl_exit(1, "bad type in link list\n"); + break; + } + erts_destroy_link(lnk); +} + +static void +resume_suspend_monitor(ErtsSuspendMonitor *smon, void *vc_p) +{ + Process *suspendee = erts_pid2proc((Process *) vc_p, ERTS_PROC_LOCK_MAIN, + smon->pid, ERTS_PROC_LOCK_STATUS); + if (suspendee) { + if (smon->active) + resume_process(suspendee); + erts_smp_proc_unlock(suspendee, ERTS_PROC_LOCK_STATUS); + } + erts_destroy_suspend_monitor(smon); +} + +static void +continue_exit_process(Process *p +#ifdef ERTS_SMP + , erts_pix_lock_t *pix_lock +#endif + ); + +/* this function fishishes a process and propagates exit messages - called + by process_main when a process dies */ +void +erts_do_exit_process(Process* p, Eterm reason) +{ +#ifdef ERTS_SMP + erts_pix_lock_t *pix_lock = ERTS_PID2PIXLOCK(p->id); +#endif + + p->arity = 0; /* No live registers */ + p->fvalue = reason; + +#ifdef ERTS_SMP + ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); + /* By locking all locks (main lock is already locked) when going + to status P_EXITING, it is enough to take any lock when + looking up a process (erts_pid2proc()) to prevent the looked up + process from exiting until the lock has been released. */ + erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR); +#endif + + if (erts_system_profile_flags.runnable_procs && (p->status != P_WAITING)) { + profile_runnable_proc(p, am_inactive); + } + +#ifdef ERTS_SMP + erts_pix_lock(pix_lock); + p->is_exiting = 1; +#endif + + p->status = P_EXITING; + +#ifdef ERTS_SMP + erts_pix_unlock(pix_lock); + + if (ERTS_PROC_PENDING_EXIT(p)) { + /* Process exited before pending exit was received... */ + p->pending_exit.reason = THE_NON_VALUE; + if (p->pending_exit.bp) { + free_message_buffer(p->pending_exit.bp); + p->pending_exit.bp = NULL; + } + } + + cancel_suspend_of_suspendee(p, ERTS_PROC_LOCKS_ALL); + + ERTS_SMP_MSGQ_MV_INQ2PRIVQ(p); +#endif + + if (IS_TRACED_FL(p,F_TRACE_PROCS)) + trace_proc(p, p, am_exit, reason); + + erts_trace_check_exiting(p->id); + + ASSERT((p->trace_flags & F_INITIAL_TRACE_FLAGS) == F_INITIAL_TRACE_FLAGS); + + cancel_timer(p); /* Always cancel timer just in case */ + + /* + * The timer of this process can *not* be used anymore. The field used + * for the timer is now used for misc exiting data. + */ + p->u.exit_data = NULL; + + if (p->bif_timers) + erts_cancel_bif_timers(p, ERTS_PROC_LOCKS_ALL); + +#ifdef ERTS_SMP + if (p->flags & F_HAVE_BLCKD_MSCHED) + erts_block_multi_scheduling(p, ERTS_PROC_LOCKS_ALL, 0, 1); +#endif + + erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL_MINOR); + +#ifdef ERTS_SMP + continue_exit_process(p, pix_lock); +#else + continue_exit_process(p); +#endif +} + +void +erts_continue_exit_process(Process *c_p) +{ +#ifdef ERTS_SMP + continue_exit_process(c_p, ERTS_PID2PIXLOCK(c_p->id)); +#else + continue_exit_process(c_p); +#endif +} + +static void +continue_exit_process(Process *p +#ifdef ERTS_SMP + , erts_pix_lock_t *pix_lock +#endif + ) +{ + ErtsLink* lnk; + ErtsMonitor *mon; + ErtsProcLocks curr_locks = ERTS_PROC_LOCK_MAIN; + Eterm reason = p->fvalue; + DistEntry *dep; + struct saved_calls *scb; +#ifdef DEBUG + int yield_allowed = 1; +#endif + + ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(p)); + +#ifdef DEBUG + erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + ASSERT(p->status == P_EXITING); + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); +#endif + + if (p->flags & F_USING_DB) { + if (erts_db_process_exiting(p, ERTS_PROC_LOCK_MAIN)) + goto yield; + p->flags &= ~F_USING_DB; + } + + if (p->flags & F_USING_DDLL) { + erts_ddll_proc_dead(p, ERTS_PROC_LOCK_MAIN); + p->flags &= ~F_USING_DDLL; + } + + if (p->nodes_monitors) { + erts_delete_nodes_monitors(p, ERTS_PROC_LOCK_MAIN); + p->nodes_monitors = NULL; + } + + + if (p->suspend_monitors) { + erts_sweep_suspend_monitors(p->suspend_monitors, + resume_suspend_monitor, + p); + p->suspend_monitors = NULL; + } + + /* + * The registered name *should* be the last "erlang resource" to + * cleanup. + */ + if (p->reg) { + (void) erts_unregister_name(p, ERTS_PROC_LOCK_MAIN, NULL, THE_NON_VALUE); + ASSERT(!p->reg); + } + + erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR); + curr_locks = ERTS_PROC_LOCKS_ALL; + + /* + * From this point on we are no longer allowed to yield + * this process. + */ +#ifdef DEBUG + yield_allowed = 0; +#endif + + { + int pix; + /* Do *not* use erts_get_runq_proc() */ + ErtsRunQueue *rq; + rq = erts_get_runq_current(ERTS_GET_SCHEDULER_DATA_FROM_PROC(p)); + + ASSERT(internal_pid_index(p->id) < erts_max_processes); + pix = internal_pid_index(p->id); + + erts_smp_mtx_lock(&proc_tab_mtx); + erts_smp_runq_lock(rq); + +#ifdef ERTS_SMP + erts_pix_lock(pix_lock); + + ASSERT(p->scheduler_data); + ASSERT(p->scheduler_data->current_process == p); + ASSERT(p->scheduler_data->free_process == NULL); + + p->scheduler_data->current_process = NULL; + p->scheduler_data->free_process = p; + p->status_flags = 0; +#endif + process_tab[pix] = NULL; /* Time of death! */ + ASSERT(erts_smp_atomic_read(&process_count) > 0); + erts_smp_atomic_dec(&process_count); + +#ifdef ERTS_SMP + erts_pix_unlock(pix_lock); +#endif + erts_smp_runq_unlock(rq); + + if (p_next < 0) { + if (p_last >= p_next) { + p_serial++; + p_serial &= p_serial_mask; + } + p_next = pix; + } + + ERTS_MAYBE_SAVE_TERMINATING_PROCESS(p); + + erts_smp_mtx_unlock(&proc_tab_mtx); + } + + /* + * All "erlang resources" have to be deallocated before this point, + * e.g. registered name, so monitoring and linked processes can + * be sure that all interesting resources have been deallocated + * when the monitors and/or links hit. + */ + + mon = p->monitors; + p->monitors = NULL; /* to avoid recursive deletion during traversal */ + + lnk = p->nlinks; + p->nlinks = NULL; + p->status = P_FREE; + dep = ((p->flags & F_DISTRIBUTION) + ? ERTS_PROC_SET_DIST_ENTRY(p, ERTS_PROC_LOCKS_ALL, NULL) + : NULL); + scb = ERTS_PROC_SET_SAVED_CALLS_BUF(p, ERTS_PROC_LOCKS_ALL, NULL); + + erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL); + processes_busy--; + + if (dep) { + erts_do_net_exits(dep, reason); + if(dep) + erts_deref_dist_entry(dep); + } + + /* + * Pre-build the EXIT tuple if there are any links. + */ + if (lnk) { + Eterm tmp_heap[4]; + Eterm exit_tuple; + Uint exit_tuple_sz; + Eterm* hp; + + hp = &tmp_heap[0]; + + exit_tuple = TUPLE3(hp, am_EXIT, p->id, reason); + + exit_tuple_sz = size_object(exit_tuple); + + { + ExitLinkContext context = {p, reason, exit_tuple, exit_tuple_sz}; + erts_sweep_links(lnk, &doit_exit_link, &context); + } + } + + { + ExitMonitorContext context = {reason, p}; + erts_sweep_monitors(mon,&doit_exit_monitor,&context); + } + + if (scb) + erts_free(ERTS_ALC_T_CALLS_BUF, (void *) scb); + + delete_process(p); + + erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); + ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); + + return; + + yield: + +#ifdef DEBUG + ASSERT(yield_allowed); +#endif + + ERTS_SMP_LC_ASSERT(curr_locks == erts_proc_lc_my_proc_locks(p)); + ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & curr_locks); + + ASSERT(p->status == P_EXITING); + + p->i = (Eterm *) beam_continue_exit; + + if (!(curr_locks & ERTS_PROC_LOCK_STATUS)) { + erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + curr_locks |= ERTS_PROC_LOCK_STATUS; + } + + erts_add_to_runq(p); + + if (curr_locks != ERTS_PROC_LOCK_MAIN) + erts_smp_proc_unlock(p, ~ERTS_PROC_LOCK_MAIN & curr_locks); + + ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(p)); + +} + +/* Callback for process timeout */ +static void +timeout_proc(Process* p) +{ + p->i = (Eterm *) p->def_arg_reg[0]; + p->flags |= F_TIMO; + p->flags &= ~F_INSLPQUEUE; + + if (p->status == P_WAITING) + erts_add_to_runq(p); + if (p->status == P_SUSPENDED) + p->rstatus = P_RUNABLE; /* MUST set resume status to runnable */ +} + + +void +cancel_timer(Process* p) +{ + ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p)); + p->flags &= ~(F_INSLPQUEUE|F_TIMO); +#ifdef ERTS_SMP + erts_cancel_smp_ptimer(p->u.ptimer); +#else + erl_cancel_timer(&p->u.tm); +#endif +} + +/* + * Insert a process into the time queue, with a timeout 'timeout' in ms. + */ +void +set_timer(Process* p, Uint timeout) +{ + ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p)); + + /* check for special case timeout=0 DONT ADD TO time queue */ + if (timeout == 0) { + p->flags |= F_TIMO; + return; + } + p->flags |= F_INSLPQUEUE; + p->flags &= ~F_TIMO; + +#ifdef ERTS_SMP + erts_create_smp_ptimer(&p->u.ptimer, + p->id, + (ErlTimeoutProc) timeout_proc, + timeout); +#else + erl_set_timer(&p->u.tm, + (ErlTimeoutProc) timeout_proc, + NULL, + (void*) p, + timeout); +#endif +} + +/* + * Stack dump functions follow. + */ + +void +erts_stack_dump(int to, void *to_arg, Process *p) +{ + Eterm* sp; + int yreg = -1; + + if (p->trace_flags & F_SENSITIVE) { + return; + } + erts_program_counter_info(to, to_arg, p); + for (sp = p->stop; sp < STACK_START(p); sp++) { + yreg = stack_element_dump(to, to_arg, p, sp, yreg); + } +} + +void +erts_program_counter_info(int to, void *to_arg, Process *p) +{ + int i; + + erts_print(to, to_arg, "Program counter: %p (", p->i); + print_function_from_pc(to, to_arg, p->i); + erts_print(to, to_arg, ")\n"); + erts_print(to, to_arg, "CP: %p (", p->cp); + print_function_from_pc(to, to_arg, p->cp); + erts_print(to, to_arg, ")\n"); + if (!((p->status == P_RUNNING) || (p->status == P_GARBING))) { + erts_print(to, to_arg, "arity = %d\n",p->arity); + if (!ERTS_IS_CRASH_DUMPING) { + /* + * Only print the arguments if we are not writing a + * crash dump file. The arguments cannot be interpreted + * by the crashdump_viewer application and will therefore + * only cause problems. + */ + for (i = 0; i < p->arity; i++) + erts_print(to, to_arg, " %T\n", p->arg_reg[i]); + } + } +} + +static void +print_function_from_pc(int to, void *to_arg, Eterm* x) +{ + Eterm* addr = find_function_from_pc(x); + if (addr == NULL) { + if (x == beam_exit) { + erts_print(to, to_arg, "<terminate process>"); + } else if (x == beam_continue_exit) { + erts_print(to, to_arg, "<continue terminate process>"); + } else if (x == beam_apply+1) { + erts_print(to, to_arg, "<terminate process normally>"); + } else if (x == 0) { + erts_print(to, to_arg, "invalid"); + } else { + erts_print(to, to_arg, "unknown function"); + } + } else { + erts_print(to, to_arg, "%T:%T/%d + %d", + addr[0], addr[1], addr[2], ((x-addr)-2) * sizeof(Eterm)); + } +} + +static int +stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp, int yreg) +{ + Eterm x = *sp; + + if (yreg < 0 || is_CP(x)) { + erts_print(to, to_arg, "\n%p ", sp); + } else { + char sbuf[16]; + sprintf(sbuf, "y(%d)", yreg); + erts_print(to, to_arg, "%-8s ", sbuf); + yreg++; + } + + if (is_CP(x)) { + erts_print(to, to_arg, "Return addr %p (", (Eterm *) x); + print_function_from_pc(to, to_arg, cp_val(x)); + erts_print(to, to_arg, ")\n"); + yreg = 0; + } else if is_catch(x) { + erts_print(to, to_arg, "Catch %p (", catch_pc(x)); + print_function_from_pc(to, to_arg, catch_pc(x)); + erts_print(to, to_arg, ")\n"); + } else { + erts_print(to, to_arg, "%T\n", x); + } + return yreg; +} + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ + * The processes/0 BIF implementation. * +\* */ + + +#define ERTS_PROCESSES_BIF_TAB_INSPECT_INDICES_PER_RED 25 +#define ERTS_PROCESSES_BIF_TAB_CHUNK_SIZE 1000 +#define ERTS_PROCESSES_BIF_MIN_START_REDS \ + (ERTS_PROCESSES_BIF_TAB_CHUNK_SIZE \ + / ERTS_PROCESSES_BIF_TAB_INSPECT_INDICES_PER_RED) + +#define ERTS_PROCESSES_BIF_TAB_FREE_TERM_PROC_REDS 1 + +#define ERTS_PROCESSES_BIF_INSPECT_TERM_PROC_PER_RED 10 + +#define ERTS_PROCESSES_INSPECT_TERM_PROC_MAX_REDS \ + (ERTS_PROCESSES_BIF_TAB_CHUNK_SIZE \ + / ERTS_PROCESSES_BIF_TAB_INSPECT_INDICES_PER_RED) + + +#define ERTS_PROCESSES_BIF_BUILD_RESULT_CONSES_PER_RED 75 + +#define ERTS_PROCS_DBG_DO_TRACE 0 + +#ifdef DEBUG +# define ERTS_PROCESSES_BIF_DEBUGLEVEL 100 +#else +# define ERTS_PROCESSES_BIF_DEBUGLEVEL 0 +#endif + +#define ERTS_PROCS_DBGLVL_CHK_HALLOC 1 +#define ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS 5 +#define ERTS_PROCS_DBGLVL_CHK_PIDS 10 +#define ERTS_PROCS_DBGLVL_CHK_TERM_PROC_LIST 20 +#define ERTS_PROCS_DBGLVL_CHK_RESLIST 20 + +#if ERTS_PROCESSES_BIF_DEBUGLEVEL == 0 +# define ERTS_PROCS_ASSERT(EXP) +#else +# define ERTS_PROCS_ASSERT(EXP) \ + ((void) ((EXP) \ + ? 1 \ + : (debug_processes_assert_error(#EXP, __FILE__, __LINE__), 0))) +#endif + + +#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_HALLOC +# define ERTS_PROCS_DBG_SAVE_HEAP_ALLOC(PBDP, HP, SZ) \ +do { \ + ERTS_PROCS_ASSERT(!(PBDP)->debug.heap); \ + ERTS_PROCS_ASSERT(!(PBDP)->debug.heap_size); \ + (PBDP)->debug.heap = (HP); \ + (PBDP)->debug.heap_size = (SZ); \ +} while (0) +# define ERTS_PROCS_DBG_VERIFY_HEAP_ALLOC_USED(PBDP, HP) \ +do { \ + ERTS_PROCS_ASSERT((PBDP)->debug.heap); \ + ERTS_PROCS_ASSERT((PBDP)->debug.heap_size); \ + ERTS_PROCS_ASSERT((PBDP)->debug.heap + (PBDP)->debug.heap_size == (HP));\ + (PBDP)->debug.heap = NULL; \ + (PBDP)->debug.heap_size = 0; \ +} while (0) +# define ERTS_PROCS_DBG_HEAP_ALLOC_INIT(PBDP) \ +do { \ + (PBDP)->debug.heap = NULL; \ + (PBDP)->debug.heap_size = 0; \ +} while (0) +#else +# define ERTS_PROCS_DBG_SAVE_HEAP_ALLOC(PBDP, HP, SZ) +# define ERTS_PROCS_DBG_VERIFY_HEAP_ALLOC_USED(PBDP, HP) +# define ERTS_PROCS_DBG_HEAP_ALLOC_INIT(PBDP) +#endif + +#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_RESLIST +# define ERTS_PROCS_DBG_CHK_RESLIST(R) debug_processes_check_res_list((R)) +#else +# define ERTS_PROCS_DBG_CHK_RESLIST(R) +#endif + +#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_PIDS +# define ERTS_PROCS_DBG_SAVE_PIDS(PBDP) debug_processes_save_all_pids((PBDP)) +# define ERTS_PROCS_DBG_VERIFY_PIDS(PBDP) \ +do { \ + if (!(PBDP)->debug.correct_pids_verified) \ + debug_processes_verify_all_pids((PBDP)); \ +} while (0) +# define ERTS_PROCS_DBG_CLEANUP_CHK_PIDS(PBDP) \ +do { \ + if ((PBDP)->debug.correct_pids) { \ + erts_free(ERTS_ALC_T_PROCS_PIDS, \ + (PBDP)->debug.correct_pids); \ + (PBDP)->debug.correct_pids = NULL; \ + } \ +} while(0) +# define ERTS_PROCS_DBG_CHK_PIDS_INIT(PBDP) \ +do { \ + (PBDP)->debug.correct_pids_verified = 0; \ + (PBDP)->debug.correct_pids = NULL; \ +} while (0) +#else +# define ERTS_PROCS_DBG_SAVE_PIDS(PBDP) +# define ERTS_PROCS_DBG_VERIFY_PIDS(PBDP) +# define ERTS_PROCS_DBG_CLEANUP_CHK_PIDS(PBDP) +# define ERTS_PROCS_DBG_CHK_PIDS_INIT(PBDP) +#endif + +#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS +# define ERTS_PROCS_DBG_CHK_PID_FOUND(PBDP, PID, TVP) \ + debug_processes_check_found_pid((PBDP), (PID), (TVP), 1) +# define ERTS_PROCS_DBG_CHK_PID_NOT_FOUND(PBDP, PID, TVP) \ + debug_processes_check_found_pid((PBDP), (PID), (TVP), 0) +#else +# define ERTS_PROCS_DBG_CHK_PID_FOUND(PBDP, PID, TVP) +# define ERTS_PROCS_DBG_CHK_PID_NOT_FOUND(PBDP, PID, TVP) +#endif + +#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_TERM_PROC_LIST +# define ERTS_PROCS_DBG_CHK_TPLIST() \ + debug_processes_check_term_proc_list() +# define ERTS_PROCS_DBG_CHK_FREELIST(FL) \ + debug_processes_check_term_proc_free_list(FL) +#else +# define ERTS_PROCS_DBG_CHK_TPLIST() +# define ERTS_PROCS_DBG_CHK_FREELIST(FL) +#endif + +#if ERTS_PROCESSES_BIF_DEBUGLEVEL == 0 +#if ERTS_PROCS_DBG_DO_TRACE +# define ERTS_PROCS_DBG_INIT(P, PBDP) (PBDP)->debug.caller = (P)->id +# else +# define ERTS_PROCS_DBG_INIT(P, PBDP) +# endif +# define ERTS_PROCS_DBG_CLEANUP(PBDP) +#else +# define ERTS_PROCS_DBG_INIT(P, PBDP) \ +do { \ + (PBDP)->debug.caller = (P)->id; \ + ERTS_PROCS_DBG_HEAP_ALLOC_INIT((PBDP)); \ + ERTS_PROCS_DBG_CHK_PIDS_INIT((PBDP)); \ +} while (0) +# define ERTS_PROCS_DBG_CLEANUP(PBDP) \ +do { \ + ERTS_PROCS_DBG_CLEANUP_CHK_PIDS((PBDP)); \ +} while (0) +#endif + +#if ERTS_PROCS_DBG_DO_TRACE +# define ERTS_PROCS_DBG_TRACE(PID, FUNC, WHAT) \ + erts_fprintf(stderr, "%T %s:%d:%s(): %s\n", \ + (PID), __FILE__, __LINE__, #FUNC, #WHAT) +#else +# define ERTS_PROCS_DBG_TRACE(PID, FUNC, WHAT) +#endif + +static Uint processes_bif_tab_chunks; +static Export processes_trap_export; + +typedef struct { + SysTimeval time; +} ErtsProcessesBifChunkInfo; + +typedef enum { + INITIALIZING, + INSPECTING_TABLE, + INSPECTING_TERMINATED_PROCESSES, + BUILDING_RESULT, + RETURN_RESULT +} ErtsProcessesBifState; + +typedef struct { + ErtsProcessesBifState state; + Eterm caller; + ErtsProcessesBifChunkInfo *chunk; + int tix; + int pid_ix; + int pid_sz; + Eterm *pid; + ErtsTermProcElement *bif_invocation; /* Only used when > 1 chunk */ + +#if ERTS_PROCESSES_BIF_DEBUGLEVEL != 0 || ERTS_PROCS_DBG_DO_TRACE + struct { + Eterm caller; +#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS + SysTimeval *pid_started; +#endif +#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_HALLOC + Eterm *heap; + Uint heap_size; +#endif +#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_PIDS + int correct_pids_verified; + Eterm *correct_pids; +#endif + } debug; +#endif + +} ErtsProcessesBifData; + + +#if ERTS_PROCESSES_BIF_DEBUGLEVEL != 0 +static void debug_processes_assert_error(char* expr, char* file, int line); +#endif +#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_RESLIST +static void debug_processes_check_res_list(Eterm list); +#endif +#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_PIDS +static void debug_processes_save_all_pids(ErtsProcessesBifData *pbdp); +static void debug_processes_verify_all_pids(ErtsProcessesBifData *pbdp); +#endif +#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS +static void debug_processes_check_found_pid(ErtsProcessesBifData *pbdp, + Eterm pid, + SysTimeval *started, + int pid_should_be_found); +#endif +#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_TERM_PROC_LIST +static SysTimeval debug_tv_start; +static void debug_processes_check_term_proc_list(void); +static void debug_processes_check_term_proc_free_list(ErtsTermProcElement *tpep); +#endif + +static void +save_terminating_process(Process *p) +{ + ErtsTermProcElement *tpep = erts_alloc(ERTS_ALC_T_PROCS_TPROC_EL, + sizeof(ErtsTermProcElement)); + ERTS_PROCS_ASSERT(saved_term_procs.start && saved_term_procs.end); + ERTS_SMP_LC_ASSERT(erts_lc_mtx_is_locked(&proc_tab_mtx)); + + ERTS_PROCS_DBG_CHK_TPLIST(); + + tpep->prev = saved_term_procs.end; + tpep->next = NULL; + tpep->ix = internal_pid_index(p->id); + tpep->u.process.pid = p->id; + tpep->u.process.spawned = p->started; + erts_get_emu_time(&tpep->u.process.exited); + + saved_term_procs.end->next = tpep; + saved_term_procs.end = tpep; + + ERTS_PROCS_DBG_CHK_TPLIST(); + + ERTS_PROCS_ASSERT((tpep->prev->ix >= 0 + ? erts_cmp_timeval(&tpep->u.process.exited, + &tpep->prev->u.process.exited) + : erts_cmp_timeval(&tpep->u.process.exited, + &tpep->prev->u.bif_invocation.time)) > 0); +} + +static void +cleanup_processes_bif_data(Binary *bp) +{ + ErtsProcessesBifData *pbdp = ERTS_MAGIC_BIN_DATA(bp); + + ERTS_PROCS_DBG_TRACE(pbdp->debug.caller, cleanup_processes_bif_data, call); + + if (pbdp->state != INITIALIZING) { + + if (pbdp->chunk) { + erts_free(ERTS_ALC_T_PROCS_CNKINF, pbdp->chunk); + pbdp->chunk = NULL; + } + if (pbdp->pid) { + erts_free(ERTS_ALC_T_PROCS_PIDS, pbdp->pid); + pbdp->pid = NULL; + } + +#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS + if (pbdp->debug.pid_started) { + erts_free(ERTS_ALC_T_PROCS_PIDS, pbdp->debug.pid_started); + pbdp->debug.pid_started = NULL; + } +#endif + + if (pbdp->bif_invocation) { + ErtsTermProcElement *tpep; + + erts_smp_mtx_lock(&proc_tab_mtx); + + ERTS_PROCS_DBG_TRACE(pbdp->debug.caller, + cleanup_processes_bif_data, + term_proc_cleanup); + + tpep = pbdp->bif_invocation; + pbdp->bif_invocation = NULL; + + ERTS_PROCS_DBG_CHK_TPLIST(); + + if (tpep->prev) { + /* + * Only remove this bif invokation when we + * have preceding invokations. + */ + tpep->prev->next = tpep->next; + if (tpep->next) + tpep->next->prev = tpep->prev; + else { + /* + * At the time of writing this branch cannot be + * reached. I don't want to remove this code though + * since it may be possible to reach this line + * in the future if the cleanup order in + * erts_do_exit_process() is changed. The ASSERT(0) + * is only here to make us aware that the reorder + * has happened. /rickard + */ + ASSERT(0); + saved_term_procs.end = tpep->prev; + } + erts_free(ERTS_ALC_T_PROCS_TPROC_EL, tpep); + } + else { + /* + * Free all elements until next bif invokation + * is found. + */ + ERTS_PROCS_ASSERT(saved_term_procs.start == tpep); + do { + ErtsTermProcElement *ftpep = tpep; + tpep = tpep->next; + erts_free(ERTS_ALC_T_PROCS_TPROC_EL, ftpep); + } while (tpep && tpep->ix >= 0); + saved_term_procs.start = tpep; + if (tpep) + tpep->prev = NULL; + else + saved_term_procs.end = NULL; + } + + ERTS_PROCS_DBG_CHK_TPLIST(); + + erts_smp_mtx_unlock(&proc_tab_mtx); + + } + } + + ERTS_PROCS_DBG_TRACE(pbdp->debug.caller, + cleanup_processes_bif_data, + return); + ERTS_PROCS_DBG_CLEANUP(pbdp); +} + +static int +processes_bif_engine(Process *p, Eterm *res_accp, Binary *mbp) +{ + ErtsProcessesBifData *pbdp = ERTS_MAGIC_BIN_DATA(mbp); + int have_reds; + int reds; + int locked = 0; + + do { + switch (pbdp->state) { + case INITIALIZING: + pbdp->chunk = erts_alloc(ERTS_ALC_T_PROCS_CNKINF, + (sizeof(ErtsProcessesBifChunkInfo) + * processes_bif_tab_chunks)); + pbdp->tix = 0; + pbdp->pid_ix = 0; + + erts_smp_mtx_lock(&proc_tab_mtx); + locked = 1; + + ERTS_PROCS_DBG_TRACE(p->id, processes_bif_engine, init); + + pbdp->pid_sz = erts_process_count(); + pbdp->pid = erts_alloc(ERTS_ALC_T_PROCS_PIDS, + sizeof(Eterm)*pbdp->pid_sz); + +#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS + pbdp->debug.pid_started = erts_alloc(ERTS_ALC_T_PROCS_PIDS, + sizeof(SysTimeval)*pbdp->pid_sz); +#endif + + ERTS_PROCS_DBG_SAVE_PIDS(pbdp); + + if (processes_bif_tab_chunks == 1) + pbdp->bif_invocation = NULL; + else { + /* + * We will have to access the table multiple times + * releasing the table lock in between chunks. + */ + pbdp->bif_invocation = erts_alloc(ERTS_ALC_T_PROCS_TPROC_EL, + sizeof(ErtsTermProcElement)); + pbdp->bif_invocation->ix = -1; + erts_get_emu_time(&pbdp->bif_invocation->u.bif_invocation.time); + ERTS_PROCS_DBG_CHK_TPLIST(); + + pbdp->bif_invocation->next = NULL; + if (saved_term_procs.end) { + pbdp->bif_invocation->prev = saved_term_procs.end; + saved_term_procs.end->next = pbdp->bif_invocation; + ERTS_PROCS_ASSERT(saved_term_procs.start); + } + else { + pbdp->bif_invocation->prev = NULL; + saved_term_procs.start = pbdp->bif_invocation; + } + saved_term_procs.end = pbdp->bif_invocation; + + ERTS_PROCS_DBG_CHK_TPLIST(); + + } + + pbdp->state = INSPECTING_TABLE; + /* Fall through */ + + case INSPECTING_TABLE: { + int ix = pbdp->tix; + int indices = ERTS_PROCESSES_BIF_TAB_CHUNK_SIZE; + int cix = ix / ERTS_PROCESSES_BIF_TAB_CHUNK_SIZE; + int end_ix = ix + indices; + SysTimeval *invocation_timep; + + invocation_timep = (pbdp->bif_invocation + ? &pbdp->bif_invocation->u.bif_invocation.time + : NULL); + + ERTS_PROCS_ASSERT(is_nil(*res_accp)); + if (!locked) { + erts_smp_mtx_lock(&proc_tab_mtx); + locked = 1; + } + + ERTS_SMP_LC_ASSERT(erts_lc_mtx_is_locked(&proc_tab_mtx)); + ERTS_PROCS_DBG_TRACE(p->id, processes_bif_engine, insp_table); + + if (cix != 0) + erts_get_emu_time(&pbdp->chunk[cix].time); + else if (pbdp->bif_invocation) + pbdp->chunk[0].time = *invocation_timep; + /* else: Time is irrelevant */ + + if (end_ix >= erts_max_processes) { + ERTS_PROCS_ASSERT(cix+1 == processes_bif_tab_chunks); + end_ix = erts_max_processes; + indices = end_ix - ix; + /* What to do when done with this chunk */ + pbdp->state = (processes_bif_tab_chunks == 1 + ? BUILDING_RESULT + : INSPECTING_TERMINATED_PROCESSES); + } + + for (; ix < end_ix; ix++) { + Process *rp = process_tab[ix]; + if (rp + && (!invocation_timep + || erts_cmp_timeval(&rp->started, + invocation_timep) < 0)) { + ERTS_PROCS_ASSERT(is_internal_pid(rp->id)); + pbdp->pid[pbdp->pid_ix] = rp->id; + +#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS + pbdp->debug.pid_started[pbdp->pid_ix] = rp->started; +#endif + + pbdp->pid_ix++; + ERTS_PROCS_ASSERT(pbdp->pid_ix <= pbdp->pid_sz); + } + } + + pbdp->tix = end_ix; + + erts_smp_mtx_unlock(&proc_tab_mtx); + locked = 0; + + reds = indices/ERTS_PROCESSES_BIF_TAB_INSPECT_INDICES_PER_RED; + BUMP_REDS(p, reds); + + have_reds = ERTS_BIF_REDS_LEFT(p); + + if (have_reds && pbdp->state == INSPECTING_TABLE) { + ix = pbdp->tix; + indices = ERTS_PROCESSES_BIF_TAB_CHUNK_SIZE; + end_ix = ix + indices; + if (end_ix > erts_max_processes) { + end_ix = erts_max_processes; + indices = end_ix - ix; + } + + reds = indices/ERTS_PROCESSES_BIF_TAB_INSPECT_INDICES_PER_RED; + + /* Pretend we have no reds left if we haven't got enough + reductions to complete next chunk */ + if (reds > have_reds) + have_reds = 0; + } + + break; + } + + case INSPECTING_TERMINATED_PROCESSES: { + int i; + int max_reds; + int free_term_procs = 0; + SysTimeval *invocation_timep; + ErtsTermProcElement *tpep; + ErtsTermProcElement *free_list = NULL; + + tpep = pbdp->bif_invocation; + ERTS_PROCS_ASSERT(tpep); + invocation_timep = &tpep->u.bif_invocation.time; + + max_reds = have_reds = ERTS_BIF_REDS_LEFT(p); + if (max_reds > ERTS_PROCESSES_INSPECT_TERM_PROC_MAX_REDS) + max_reds = ERTS_PROCESSES_INSPECT_TERM_PROC_MAX_REDS; + + reds = 0; + erts_smp_mtx_lock(&proc_tab_mtx); + ERTS_PROCS_DBG_TRACE(p->id, processes_bif_engine, insp_term_procs); + + ERTS_PROCS_DBG_CHK_TPLIST(); + + if (tpep->prev) + tpep->prev->next = tpep->next; + else { + ERTS_PROCS_ASSERT(saved_term_procs.start == tpep); + saved_term_procs.start = tpep->next; + + if (saved_term_procs.start && saved_term_procs.start->ix >= 0) { + free_list = saved_term_procs.start; + free_term_procs = 1; + } + } + + if (tpep->next) + tpep->next->prev = tpep->prev; + else + saved_term_procs.end = tpep->prev; + + tpep = tpep->next; + + i = 0; + while (reds < max_reds && tpep) { + if (tpep->ix < 0) { + if (free_term_procs) { + ERTS_PROCS_ASSERT(free_list); + ERTS_PROCS_ASSERT(tpep->prev); + + tpep->prev->next = NULL; /* end of free_list */ + saved_term_procs.start = tpep; + tpep->prev = NULL; + free_term_procs = 0; + } + } + else { + int cix = tpep->ix/ERTS_PROCESSES_BIF_TAB_CHUNK_SIZE; + SysTimeval *chunk_timep = &pbdp->chunk[cix].time; + Eterm pid = tpep->u.process.pid; + ERTS_PROCS_ASSERT(is_internal_pid(pid)); + + if (erts_cmp_timeval(&tpep->u.process.spawned, + invocation_timep) < 0) { + if (erts_cmp_timeval(&tpep->u.process.exited, + chunk_timep) < 0) { + ERTS_PROCS_DBG_CHK_PID_NOT_FOUND(pbdp, + pid, + &tpep->u.process.spawned); + pbdp->pid[pbdp->pid_ix] = pid; +#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS + pbdp->debug.pid_started[pbdp->pid_ix] = tpep->u.process.spawned; +#endif + pbdp->pid_ix++; + ERTS_PROCS_ASSERT(pbdp->pid_ix <= pbdp->pid_sz); + } + else { + ERTS_PROCS_DBG_CHK_PID_FOUND(pbdp, + pid, + &tpep->u.process.spawned); + } + } + else { + ERTS_PROCS_DBG_CHK_PID_NOT_FOUND(pbdp, + pid, + &tpep->u.process.spawned); + } + + i++; + if (i == ERTS_PROCESSES_BIF_INSPECT_TERM_PROC_PER_RED) { + reds++; + i = 0; + } + if (free_term_procs) + reds += ERTS_PROCESSES_BIF_TAB_FREE_TERM_PROC_REDS; + } + tpep = tpep->next; + } + + if (free_term_procs) { + ERTS_PROCS_ASSERT(free_list); + saved_term_procs.start = tpep; + if (!tpep) + saved_term_procs.end = NULL; + else { + ERTS_PROCS_ASSERT(tpep->prev); + tpep->prev->next = NULL; /* end of free_list */ + tpep->prev = NULL; + } + } + + if (!tpep) { + /* Done */ + ERTS_PROCS_ASSERT(pbdp->pid_ix == pbdp->pid_sz); + pbdp->state = BUILDING_RESULT; + pbdp->bif_invocation->next = free_list; + free_list = pbdp->bif_invocation; + pbdp->bif_invocation = NULL; + } + else { + /* Link in bif_invocation again where we left off */ + pbdp->bif_invocation->prev = tpep->prev; + pbdp->bif_invocation->next = tpep; + tpep->prev = pbdp->bif_invocation; + if (pbdp->bif_invocation->prev) + pbdp->bif_invocation->prev->next = pbdp->bif_invocation; + else { + ERTS_PROCS_ASSERT(saved_term_procs.start == tpep); + saved_term_procs.start = pbdp->bif_invocation; + } + } + + ERTS_PROCS_DBG_CHK_TPLIST(); + ERTS_PROCS_DBG_CHK_FREELIST(free_list); + erts_smp_mtx_unlock(&proc_tab_mtx); + + /* + * We do the actual free of term proc structures now when we + * have released the table lock instead of when we encountered + * them. This since free() isn't for free and we don't want to + * unnecessarily block other schedulers. + */ + while (free_list) { + tpep = free_list; + free_list = tpep->next; + erts_free(ERTS_ALC_T_PROCS_TPROC_EL, tpep); + } + + have_reds -= reds; + if (have_reds < 0) + have_reds = 0; + BUMP_REDS(p, reds); + break; + } + + case BUILDING_RESULT: { + int conses, ix, min_ix; + Eterm *hp; + Eterm res = *res_accp; + + ERTS_PROCS_DBG_VERIFY_PIDS(pbdp); + ERTS_PROCS_DBG_CHK_RESLIST(res); + + ERTS_PROCS_DBG_TRACE(p->id, processes_bif_engine, begin_build_res); + + have_reds = ERTS_BIF_REDS_LEFT(p); + conses = ERTS_PROCESSES_BIF_BUILD_RESULT_CONSES_PER_RED*have_reds; + min_ix = pbdp->pid_ix - conses; + if (min_ix < 0) { + min_ix = 0; + conses = pbdp->pid_ix; + } + + hp = HAlloc(p, conses*2); + ERTS_PROCS_DBG_SAVE_HEAP_ALLOC(pbdp, hp, conses*2); + + for (ix = pbdp->pid_ix - 1; ix >= min_ix; ix--) { + ERTS_PROCS_ASSERT(is_internal_pid(pbdp->pid[ix])); + res = CONS(hp, pbdp->pid[ix], res); + hp += 2; + } + + ERTS_PROCS_DBG_VERIFY_HEAP_ALLOC_USED(pbdp, hp); + + pbdp->pid_ix = min_ix; + if (min_ix == 0) + pbdp->state = RETURN_RESULT; + else { + pbdp->pid_sz = min_ix; + pbdp->pid = erts_realloc(ERTS_ALC_T_PROCS_PIDS, + pbdp->pid, + sizeof(Eterm)*pbdp->pid_sz); +#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS + pbdp->debug.pid_started = erts_realloc(ERTS_ALC_T_PROCS_PIDS, + pbdp->debug.pid_started, + sizeof(SysTimeval)*pbdp->pid_sz); +#endif + } + reds = conses/ERTS_PROCESSES_BIF_BUILD_RESULT_CONSES_PER_RED; + BUMP_REDS(p, reds); + have_reds -= reds; + + ERTS_PROCS_DBG_CHK_RESLIST(res); + ERTS_PROCS_DBG_TRACE(p->id, processes_bif_engine, end_build_res); + *res_accp = res; + break; + } + case RETURN_RESULT: + cleanup_processes_bif_data(mbp); + return 1; + + default: + erl_exit(ERTS_ABORT_EXIT, + "erlang:processes/0: Invalid state: %d\n", + (int) pbdp->state); + } + + + } while (have_reds || pbdp->state == RETURN_RESULT); + + return 0; +} + +/* + * processes_trap/2 is a hidden BIF that processes/0 traps to. + */ + +static BIF_RETTYPE processes_trap(BIF_ALIST_2) +{ + Eterm res_acc; + Binary *mbp; + + /* + * This bif cannot be called from erlang code. It can only be + * trapped to from processes/0; therefore, a bad argument + * is a processes/0 internal error. + */ + + ERTS_PROCS_DBG_TRACE(BIF_P->id, processes_trap, call); + ERTS_PROCS_ASSERT(is_nil(BIF_ARG_1) || is_list(BIF_ARG_1)); + + res_acc = BIF_ARG_1; + + ERTS_PROCS_ASSERT(ERTS_TERM_IS_MAGIC_BINARY(BIF_ARG_2)); + + mbp = ((ProcBin *) binary_val(BIF_ARG_2))->val; + + ERTS_PROCS_ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(mbp) + == cleanup_processes_bif_data); + ERTS_PROCS_ASSERT( + ((ErtsProcessesBifData *) ERTS_MAGIC_BIN_DATA(mbp))->debug.caller + == BIF_P->id); + + if (processes_bif_engine(BIF_P, &res_acc, mbp)) { + ERTS_PROCS_DBG_TRACE(BIF_P->id, processes_trap, return); + BIF_RET(res_acc); + } + else { + ERTS_PROCS_DBG_TRACE(BIF_P->id, processes_trap, trap); + ERTS_BIF_YIELD2(&processes_trap_export, BIF_P, res_acc, BIF_ARG_2); + } +} + + + +/* + * The actual processes/0 BIF. + */ + +BIF_RETTYPE processes_0(BIF_ALIST_0) +{ + /* + * A requirement: The list of pids returned should be a consistent + * snapshot of all processes existing at some point + * in time during the execution of processes/0. Since + * processes might terminate while processes/0 is + * executing, we have to keep track of terminated + * processes and add them to the result. We also + * ignore processes created after processes/0 has + * begun executing. + */ + Eterm res_acc = NIL; + Binary *mbp = erts_create_magic_binary(sizeof(ErtsProcessesBifData), + cleanup_processes_bif_data); + ErtsProcessesBifData *pbdp = ERTS_MAGIC_BIN_DATA(mbp); + + ERTS_PROCS_DBG_TRACE(BIF_P->id, processes_0, call); + pbdp->state = INITIALIZING; + ERTS_PROCS_DBG_INIT(BIF_P, pbdp); + + if (ERTS_BIF_REDS_LEFT(BIF_P) >= ERTS_PROCESSES_BIF_MIN_START_REDS + && processes_bif_engine(BIF_P, &res_acc, mbp)) { + erts_bin_free(mbp); + ERTS_PROCS_DBG_CHK_RESLIST(res_acc); + ERTS_PROCS_DBG_TRACE(BIF_P->id, processes_0, return); + BIF_RET(res_acc); + } + else { + Eterm *hp; + Eterm magic_bin; + ERTS_PROCS_DBG_CHK_RESLIST(res_acc); + hp = HAlloc(BIF_P, PROC_BIN_SIZE); + ERTS_PROCS_DBG_SAVE_HEAP_ALLOC(pbdp, hp, PROC_BIN_SIZE); + magic_bin = erts_mk_magic_binary_term(&hp, &MSO(BIF_P), mbp); + ERTS_PROCS_DBG_VERIFY_HEAP_ALLOC_USED(pbdp, hp); + ERTS_PROCS_DBG_TRACE(BIF_P->id, processes_0, trap); + ERTS_BIF_YIELD2(&processes_trap_export, BIF_P, res_acc, magic_bin); + } +} + +static void +init_processes_bif(void) +{ + saved_term_procs.start = NULL; + saved_term_procs.end = NULL; + processes_bif_tab_chunks = (((erts_max_processes - 1) + / ERTS_PROCESSES_BIF_TAB_CHUNK_SIZE) + + 1); + + /* processes_trap/2 is a hidden BIF that the processes/0 BIF traps to. */ + sys_memset((void *) &processes_trap_export, 0, sizeof(Export)); + processes_trap_export.address = &processes_trap_export.code[3]; + processes_trap_export.code[0] = am_erlang; + processes_trap_export.code[1] = am_processes_trap; + processes_trap_export.code[2] = 2; + processes_trap_export.code[3] = (Eterm) em_apply_bif; + processes_trap_export.code[4] = (Eterm) &processes_trap; + +#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_TERM_PROC_LIST + erts_get_emu_time(&debug_tv_start); +#endif + +} + +/* + * Debug stuff + */ + +Eterm +erts_debug_processes(Process *c_p) +{ + /* This is the old processes/0 BIF. */ + int i; + Uint need; + Eterm res; + Eterm* hp; + Process *p; +#ifdef DEBUG + Eterm *hp_end; +#endif + + erts_smp_mtx_lock(&proc_tab_mtx); + + res = NIL; + need = erts_process_count() * 2; + hp = HAlloc(c_p, need); /* we need two heap words for each pid */ +#ifdef DEBUG + hp_end = hp + need; +#endif + + /* make the list by scanning bakward */ + + + for (i = erts_max_processes-1; i >= 0; i--) { + if ((p = process_tab[i]) != NULL) { + res = CONS(hp, process_tab[i]->id, res); + hp += 2; + } + } + ASSERT(hp == hp_end); + + erts_smp_mtx_unlock(&proc_tab_mtx); + + return res; +} + +Eterm +erts_debug_processes_bif_info(Process *c_p) +{ + ERTS_DECL_AM(processes_bif_info); + Eterm elements[] = { + AM_processes_bif_info, + make_small((Uint) ERTS_PROCESSES_BIF_MIN_START_REDS), + make_small((Uint) processes_bif_tab_chunks), + make_small((Uint) ERTS_PROCESSES_BIF_TAB_CHUNK_SIZE), + make_small((Uint) ERTS_PROCESSES_BIF_TAB_INSPECT_INDICES_PER_RED), + make_small((Uint) ERTS_PROCESSES_BIF_TAB_FREE_TERM_PROC_REDS), + make_small((Uint) ERTS_PROCESSES_BIF_INSPECT_TERM_PROC_PER_RED), + make_small((Uint) ERTS_PROCESSES_INSPECT_TERM_PROC_MAX_REDS), + make_small((Uint) ERTS_PROCESSES_BIF_BUILD_RESULT_CONSES_PER_RED), + make_small((Uint) ERTS_PROCESSES_BIF_DEBUGLEVEL) + }; + Uint sz = 0; + Eterm *hp; + (void) erts_bld_tuplev(NULL, &sz, sizeof(elements)/sizeof(Eterm), elements); + hp = HAlloc(c_p, sz); + return erts_bld_tuplev(&hp, NULL, sizeof(elements)/sizeof(Eterm), elements); +} + +#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS +static void +debug_processes_check_found_pid(ErtsProcessesBifData *pbdp, + Eterm pid, + SysTimeval *tvp, + int pid_should_be_found) +{ + int i; + for (i = 0; i < pbdp->pid_ix; i++) { + if (pbdp->pid[i] == pid + && pbdp->debug.pid_started[i].tv_sec == tvp->tv_sec + && pbdp->debug.pid_started[i].tv_usec == tvp->tv_usec) { + ERTS_PROCS_ASSERT(pid_should_be_found); + return; + } + } + ERTS_PROCS_ASSERT(!pid_should_be_found); +} +#endif + +#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_RESLIST +static void +debug_processes_check_res_list(Eterm list) +{ + while (is_list(list)) { + Eterm* consp = list_val(list); + Eterm hd = CAR(consp); + ERTS_PROCS_ASSERT(is_internal_pid(hd)); + list = CDR(consp); + } + + ERTS_PROCS_ASSERT(is_nil(list)); +} +#endif + +#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_PIDS + +static void +debug_processes_save_all_pids(ErtsProcessesBifData *pbdp) +{ + int ix, tix, cpix; + pbdp->debug.correct_pids_verified = 0; + pbdp->debug.correct_pids = erts_alloc(ERTS_ALC_T_PROCS_PIDS, + sizeof(Eterm)*pbdp->pid_sz); + + for (tix = 0, cpix = 0; tix < erts_max_processes; tix++) { + Process *rp = process_tab[tix]; + if (rp) { + ERTS_PROCS_ASSERT(is_internal_pid(rp->id)); + pbdp->debug.correct_pids[cpix++] = rp->id; + ERTS_PROCS_ASSERT(cpix <= pbdp->pid_sz); + } + } + ERTS_PROCS_ASSERT(cpix == pbdp->pid_sz); + + for (ix = 0; ix < pbdp->pid_sz; ix++) + pbdp->pid[ix] = make_small(ix); +} + +static void +debug_processes_verify_all_pids(ErtsProcessesBifData *pbdp) +{ + int ix, cpix; + + ERTS_PROCS_ASSERT(pbdp->pid_ix == pbdp->pid_sz); + + for (ix = 0; ix < pbdp->pid_sz; ix++) { + int found = 0; + Eterm pid = pbdp->pid[ix]; + ERTS_PROCS_ASSERT(is_internal_pid(pid)); + for (cpix = ix; cpix < pbdp->pid_sz; cpix++) { + if (pbdp->debug.correct_pids[cpix] == pid) { + pbdp->debug.correct_pids[cpix] = NIL; + found = 1; + break; + } + } + if (!found) { + for (cpix = 0; cpix < ix; cpix++) { + if (pbdp->debug.correct_pids[cpix] == pid) { + pbdp->debug.correct_pids[cpix] = NIL; + found = 1; + break; + } + } + } + ERTS_PROCS_ASSERT(found); + } + pbdp->debug.correct_pids_verified = 1; + + erts_free(ERTS_ALC_T_PROCS_PIDS, pbdp->debug.correct_pids); + pbdp->debug.correct_pids = NULL; +} +#endif /* ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_PIDS */ + +#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_TERM_PROC_LIST +static void +debug_processes_check_term_proc_list(void) +{ + ERTS_SMP_LC_ASSERT(erts_lc_mtx_is_locked(&proc_tab_mtx)); + if (!saved_term_procs.start) + ERTS_PROCS_ASSERT(!saved_term_procs.end); + else { + SysTimeval tv_now; + SysTimeval *prev_xtvp = NULL; + ErtsTermProcElement *tpep; + erts_get_emu_time(&tv_now); + + for (tpep = saved_term_procs.start; tpep; tpep = tpep->next) { + if (!tpep->prev) + ERTS_PROCS_ASSERT(saved_term_procs.start == tpep); + else + ERTS_PROCS_ASSERT(tpep->prev->next == tpep); + if (!tpep->next) + ERTS_PROCS_ASSERT(saved_term_procs.end == tpep); + else + ERTS_PROCS_ASSERT(tpep->next->prev == tpep); + if (tpep->ix < 0) { + SysTimeval *tvp = &tpep->u.bif_invocation.time; + ERTS_PROCS_ASSERT(erts_cmp_timeval(&debug_tv_start, tvp) < 0 + && erts_cmp_timeval(tvp, &tv_now) < 0); + } + else { + SysTimeval *stvp = &tpep->u.process.spawned; + SysTimeval *xtvp = &tpep->u.process.exited; + + ERTS_PROCS_ASSERT(erts_cmp_timeval(&debug_tv_start, + stvp) < 0); + ERTS_PROCS_ASSERT(erts_cmp_timeval(stvp, xtvp) < 0); + if (prev_xtvp) + ERTS_PROCS_ASSERT(erts_cmp_timeval(prev_xtvp, xtvp) < 0); + prev_xtvp = xtvp; + ERTS_PROCS_ASSERT(is_internal_pid(tpep->u.process.pid)); + ERTS_PROCS_ASSERT(tpep->ix + == internal_pid_index(tpep->u.process.pid)); + } + } + + } +} + +static void +debug_processes_check_term_proc_free_list(ErtsTermProcElement *free_list) +{ + if (saved_term_procs.start) { + ErtsTermProcElement *ftpep; + ErtsTermProcElement *tpep; + + for (ftpep = free_list; ftpep; ftpep = ftpep->next) { + for (tpep = saved_term_procs.start; tpep; tpep = tpep->next) + ERTS_PROCS_ASSERT(ftpep != tpep); + } + } +} + +#endif + +#if ERTS_PROCESSES_BIF_DEBUGLEVEL != 0 + +static void +debug_processes_assert_error(char* expr, char* file, int line) +{ + fflush(stdout); + erts_fprintf(stderr, "%s:%d: Assertion failed: %s\n", file, line, expr); + fflush(stderr); + abort(); +} + +#endif + +/* *\ + * End of the processes/0 BIF implementation. * +\* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |