From 414f4fb8dc9a188f8148a1f92e5f9125108e170d Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Fri, 16 Dec 2011 13:02:09 +0100 Subject: Optimize process table access --- erts/doc/src/erl.xml | 4 +- erts/emulator/beam/beam_bp.c | 2 +- erts/emulator/beam/beam_emu.c | 4 - erts/emulator/beam/bif.c | 21 +- erts/emulator/beam/break.c | 21 +- erts/emulator/beam/dist.c | 25 +- erts/emulator/beam/erl_alloc.types | 1 + erts/emulator/beam/erl_bif_info.c | 22 +- erts/emulator/beam/erl_bif_timer.c | 4 +- erts/emulator/beam/erl_bif_trace.c | 17 +- erts/emulator/beam/erl_db.c | 14 +- erts/emulator/beam/erl_db_util.h | 2 +- erts/emulator/beam/erl_init.c | 9 +- erts/emulator/beam/erl_message.c | 4 +- erts/emulator/beam/erl_nif.c | 20 +- erts/emulator/beam/erl_node_container_utils.h | 43 +- erts/emulator/beam/erl_node_tables.c | 38 +- erts/emulator/beam/erl_process.c | 782 +++++++++++++++++--------- erts/emulator/beam/erl_process.h | 39 +- erts/emulator/beam/erl_process_dump.c | 7 +- erts/emulator/beam/erl_process_lock.c | 261 ++++++--- erts/emulator/beam/erl_process_lock.h | 287 +++------- erts/emulator/beam/erl_threads.h | 31 +- erts/emulator/beam/erl_time.h | 4 +- erts/emulator/beam/erl_time_sup.c | 85 +-- erts/emulator/beam/erl_trace.c | 33 +- erts/emulator/beam/global.h | 119 +++- erts/emulator/beam/io.c | 59 +- erts/emulator/beam/register.c | 7 +- erts/emulator/beam/sys.h | 45 +- erts/emulator/beam/utils.c | 294 +++++++++- erts/emulator/sys/win32/erl_win_sys.h | 5 + erts/emulator/sys/win32/sys_time.c | 6 +- 33 files changed, 1512 insertions(+), 803 deletions(-) diff --git a/erts/doc/src/erl.xml b/erts/doc/src/erl.xml index cfbc38f176..4ef3e089a6 100644 --- a/erts/doc/src/erl.xml +++ b/erts/doc/src/erl.xml @@ -590,7 +590,9 @@

Sets the maximum number of concurrent processes for this system. must be in the range 16..134217727. - Default is 32768.

+ Default is 32768.

+

NOTE: It is possible to choose any value in the range + but powers of 2 perform best.

 diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c index d772bea02f..f4e7119d4d 100644 --- a/erts/emulator/beam/beam_bp.c +++ b/erts/emulator/beam/beam_bp.c @@ -1255,7 +1255,7 @@ static int clear_function_break(Module *m, BeamInstr *pc, int bif, BeamInstr bre for (j = 0; j < bdt->hash[i].n; ++j) { item = &(bdt->hash[i].item[j]); if (item->pid != NIL) { - h_p = process_tab[internal_pid_index(item->pid)]; + h_p = erts_proc_lookup(item->pid); if (h_p) { pbt = ERTS_PROC_SET_CALL_TIME(h_p, ERTS_PROC_LOCK_MAIN, NULL); if (pbt) { diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 18a57931ae..5ca3e8060b 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -64,11 +64,7 @@ # define PROCESS_MAIN_CHK_LOCKS(P) \ do { \ if ((P)) { \ - erts_pix_lock_t *pix_lock__ = ERTS_PIX2PIXLOCK(internal_pid_index((P)->id));\ erts_proc_lc_chk_only_proc_main((P)); \ - erts_pix_lock(pix_lock__); \ - ASSERT(0 < (P)->lock.refc && (P)->lock.refc < erts_no_schedulers*5);\ - erts_pix_unlock(pix_lock__); \ } \ else \ erts_lc_check_exact(NULL, 0); \ diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 39d4582435..dd9488afa6 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -1403,9 +1403,8 @@ BIF_RETTYPE exit_2(BIF_ALIST_2) } else { rp_locks = ERTS_PROC_LOCKS_XSIG_SEND; - rp = erts_pid2proc_opt(BIF_P, ERTS_PROC_LOCK_MAIN, - BIF_ARG_1, rp_locks, - ERTS_P2P_FLG_SMP_INC_REFC); + rp = erts_pid2proc(BIF_P, ERTS_PROC_LOCK_MAIN, + BIF_ARG_1, rp_locks); if (!rp) { BIF_RET(am_true); } @@ -1427,8 +1426,6 @@ BIF_RETTYPE exit_2(BIF_ALIST_2) rp_locks &= ~ERTS_PROC_LOCK_MAIN; if (rp_locks) erts_smp_proc_unlock(rp, rp_locks); - if (rp != BIF_P) - erts_smp_proc_dec_refc(rp); #endif /* * We may have exited ourselves and may have to take action. @@ -1826,8 +1823,7 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend) { if (internal_pid_index(to) >= erts_max_processes) return SEND_BADARG; - rp = erts_pid2proc_opt(p, ERTS_PROC_LOCK_MAIN, - to, 0, ERTS_P2P_FLG_SMP_INC_REFC); + rp = erts_proc_lookup_raw(to); if (!rp) { ERTS_SMP_ASSERT_IS_NOT_EXITING(p); @@ -1865,7 +1861,7 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend) { } erts_whereis_name(p, ERTS_PROC_LOCK_MAIN, to, - &rp, 0, ERTS_P2P_FLG_SMP_INC_REFC, + &rp, 0, 0, &pt); if (pt) { @@ -2017,7 +2013,7 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend) { erts_whereis_name(p, ERTS_PROC_LOCK_MAIN, tp[1], - &rp, 0, ERTS_P2P_FLG_SMP_INC_REFC, + &rp, 0, 0, &pt); if (pt) { portid = pt->id; @@ -2076,7 +2072,6 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend) { p == rp ? (rp_locks & ~ERTS_PROC_LOCK_MAIN) : rp_locks); - erts_smp_proc_dec_refc(rp); return res; } } @@ -3486,7 +3481,7 @@ BIF_RETTYPE garbage_collect_1(BIF_ALIST_1) if (rp == ERTS_PROC_LOCK_BUSY) ERTS_BIF_YIELD1(bif_export[BIF_garbage_collect_1], BIF_P, BIF_ARG_1); #else - rp = erts_pid2proc(BIF_P, 0, BIF_ARG_1, 0); + rp = erts_proc_lookup(BIF_ARG_1); #endif if (!rp) BIF_RET(am_false); @@ -4232,8 +4227,8 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) erts_smp_thr_progress_block(); for (i = 0; i < erts_max_processes; i++) { - if (process_tab[i] != (Process*) 0) { - Process* p = process_tab[i]; + Process *p = erts_pix2proc(i); + if (p) { #ifdef USE_VM_PROBES p->seq_trace_token = (p->dt_utag != NIL) ? am_have_dt_utag : NIL; #else diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index 6f5020dc14..0a5fc26ee8 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -71,9 +71,10 @@ process_info(int to, void *to_arg) { int i; for (i = 0; i < erts_max_processes; i++) { - if ((process_tab[i] != NULL) && (process_tab[i]->i != ENULL)) { - if (process_tab[i]->status != P_EXITING) - print_process_info(to, to_arg, process_tab[i]); + Process *p = erts_pix2proc(i); + if (p && p->i != ENULL) { + if (p->status != P_EXITING) + print_process_info(to, to_arg, p); } } @@ -89,7 +90,8 @@ process_killer(void) erts_printf("\n\nProcess Information\n\n"); erts_printf("--------------------------------------------------\n"); for (i = erts_max_processes-1; i >= 0; i--) { - if (((rp = process_tab[i]) != NULL) && rp->i != ENULL) { + rp = erts_pix2proc(i); + if (rp && rp->i != ENULL) { int br; print_process_info(ERTS_PRINT_STDOUT, NULL, rp); erts_printf("(k)ill (n)ext (r)eturn:\n"); @@ -180,9 +182,9 @@ static void doit_print_monitor(ErtsMonitor *mon, void *vpcontext) void print_process_info(int to, void *to_arg, Process *p) { + time_t approx_started; int garbing = 0; int running = 0; - time_t tmp_t; struct saved_calls *scb; /* display the PID */ @@ -245,8 +247,8 @@ print_process_info(int to, void *to_arg, Process *p) } erts_print(to, to_arg, "Spawned by: %T\n", p->parent); - tmp_t = p->started.tv_sec; - erts_print(to, to_arg, "Started: %s", ctime(&tmp_t)); + approx_started = (time_t) p->approx_started; + erts_print(to, to_arg, "Started: %s", ctime(&approx_started)); ERTS_SMP_MSGQ_MV_INQ2PRIVQ(p); erts_print(to, to_arg, "Message queue length: %d\n", p->msg.len); @@ -616,7 +618,8 @@ bin_check(void) int i, printed = 0; for (i=0; i < erts_max_processes; i++) { - if ((rp = process_tab[i]) == NULL) + rp = erts_pix2proc(i); + if (!rp) continue; for (hdr = rp->off_heap.first; hdr; hdr = hdr->next) { if (hdr->thing_word == HEADER_PROC_BIN) { @@ -704,7 +707,7 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) erts_print_nif_taints(fd, NULL); erts_fdprintf(fd, "Atoms: %d\n", atom_table_size()); info(fd, NULL); /* General system info */ - if (process_tab != NULL) /* XXX true at init */ + if (erts_proc.tab) process_info(fd, NULL); /* Info about each process and port */ db_info(fd, NULL, 0); erts_print_bif_timer_info(fd, NULL); diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index 7c75c9fdb7..a7f11740f5 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -1312,7 +1312,7 @@ int erts_net_message(Port *prt, if (is_not_pid(from) || is_not_atom(to)){ goto invalid_message; } - rp = erts_whereis_process(NULL, 0, to, 0, ERTS_P2P_FLG_SMP_INC_REFC); + rp = erts_whereis_process(NULL, 0, to, 0, 0); if (rp) { Uint xsize = (type == DOP_REG_SEND ? 0 @@ -1338,7 +1338,6 @@ int erts_net_message(Port *prt, erts_queue_dist_message(rp, &locks, ede_copy, token); if (locks) erts_smp_proc_unlock(rp, locks); - erts_smp_proc_dec_refc(rp); } break; @@ -1364,7 +1363,7 @@ int erts_net_message(Port *prt, if (is_not_pid(to)) { goto invalid_message; } - rp = erts_pid2proc_opt(NULL, 0, to, 0, ERTS_P2P_FLG_SMP_INC_REFC); + rp = erts_proc_lookup(to); if (rp) { Uint xsize = type == DOP_SEND ? 0 : ERTS_HEAP_FRAG_SIZE(token_size); ErtsProcLocks locks = 0; @@ -1388,7 +1387,6 @@ int erts_net_message(Port *prt, erts_queue_dist_message(rp, &locks, ede_copy, token); if (locks) erts_smp_proc_unlock(rp, locks); - erts_smp_proc_dec_refc(rp); } break; @@ -1544,8 +1542,7 @@ int erts_net_message(Port *prt, if (is_not_pid(from) || is_not_internal_pid(to)) { goto invalid_message; } - rp = erts_pid2proc_opt(NULL, 0, to, rp_locks, - ERTS_P2P_FLG_SMP_INC_REFC); + rp = erts_pid2proc(NULL, 0, to, rp_locks); if (rp) { (void) erts_send_exit_signal(NULL, from, @@ -1556,7 +1553,6 @@ int erts_net_message(Port *prt, NULL, 0); erts_smp_proc_unlock(rp, rp_locks); - erts_smp_proc_dec_refc(rp); } break; } @@ -2246,7 +2242,7 @@ static void doit_print_monitor_info(ErtsMonitor *mon, void *vptdp) void *arg = ((struct print_to_data *) vptdp)->arg; Process *rp; ErtsMonitor *rmon; - rp = erts_pid2proc_unlocked(mon->pid); + rp = erts_proc_lookup(mon->pid); if (!rp || (rmon = erts_lookup_monitor(rp->monitors, mon->ref)) == NULL) { erts_print(to, arg, "Warning, stray monitor for: %T\n", mon->pid); } else if (mon->type == MON_ORIGIN) { @@ -2285,7 +2281,7 @@ static void doit_print_link_info2(ErtsLink *lnk, void *vpplc) static void doit_print_link_info(ErtsLink *lnk, void *vptdp) { - if (is_internal_pid(lnk->pid) && erts_pid2proc_unlocked(lnk->pid)) { + if (is_internal_pid(lnk->pid) && erts_proc_lookup(lnk->pid)) { PrintLinkContext plc = {(struct print_to_data *) vptdp, lnk->pid}; erts_doforall_links(ERTS_LINK_ROOT(lnk), &doit_print_link_info2, &plc); } @@ -2307,7 +2303,7 @@ static void doit_print_nodelink_info(ErtsLink *lnk, void *vpcontext) { PrintNodeLinkContext *pcontext = vpcontext; - if (is_internal_pid(lnk->pid) && erts_pid2proc_unlocked(lnk->pid)) + if (is_internal_pid(lnk->pid) && erts_proc_lookup(lnk->pid)) erts_print(pcontext->ptd.to, pcontext->ptd.arg, "Remote monitoring: %T %T\n", lnk->pid, pcontext->sysname); } @@ -2710,9 +2706,8 @@ BIF_RETTYPE dist_exit_3(BIF_ALIST_3) } else { lp_locks = ERTS_PROC_LOCKS_XSIG_SEND; - lp = erts_pid2proc_opt(BIF_P, ERTS_PROC_LOCK_MAIN, - local, lp_locks, - ERTS_P2P_FLG_SMP_INC_REFC); + lp = erts_pid2proc(BIF_P, ERTS_PROC_LOCK_MAIN, + local, lp_locks); if (!lp) { BIF_RET(am_true); /* ignore */ } @@ -2731,9 +2726,7 @@ BIF_RETTYPE dist_exit_3(BIF_ALIST_3) lp_locks &= ~ERTS_PROC_LOCK_MAIN; #endif erts_smp_proc_unlock(lp, lp_locks); - if (lp != BIF_P) - erts_smp_proc_dec_refc(lp); - else { + if (lp == BIF_P) { /* * We may have exited current process and may have to take action. */ diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index bba6b83ac6..b30e6dc018 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -263,6 +263,7 @@ type ZLIB STANDARD SYSTEM zlib type CPU_GRPS_MAP LONG_LIVED SYSTEM cpu_groups_map type AUX_WORK_TMO LONG_LIVED SYSTEM aux_work_timeouts type MISC_AUX_WORK_Q LONG_LIVED SYSTEM misc_aux_work_q +type PROC_INTERVAL LONG_LIVED SYSTEM process_interval +if threads_no_smp # Need thread safe allocs, but std_alloc and fix_alloc are not; diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index f889ccdb93..8117b1bfaa 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -3139,15 +3139,19 @@ BIF_RETTYPE is_process_alive_1(BIF_ALIST_1) if(internal_pid_index(BIF_ARG_1) >= erts_max_processes) BIF_ERROR(BIF_P, BADARG); - rp = erts_pid2proc(BIF_P, ERTS_PROC_LOCK_MAIN, - BIF_ARG_1, ERTS_PROC_LOCK_STATUS); + rp = erts_proc_lookup(BIF_ARG_1); if (!rp) { BIF_RET(am_false); } else { - int have_pending_exit = ERTS_PROC_PENDING_EXIT(rp); - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); - if (have_pending_exit) + int px; +#ifdef ERTS_SMP + px = (erts_atomic32_read_acqb(&rp->aflags) + & (ERTS_PROC_AFLG_PENDING_EXIT|ERTS_PROC_AFLG_EXITING)); +#else + px = 0; +#endif + if (px) ERTS_BIF_AWAIT_X_DATA_TRAP(BIF_P, BIF_ARG_1, am_false); else BIF_RET(am_true); @@ -3751,9 +3755,8 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) && is_internal_pid(tp[2])) { int xres; ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND; - Process *rp = erts_pid2proc_opt(BIF_P, ERTS_PROC_LOCK_MAIN, - tp[2], rp_locks, - ERTS_P2P_FLG_SMP_INC_REFC); + Process *rp = erts_pid2proc(BIF_P, ERTS_PROC_LOCK_MAIN, + tp[2], rp_locks); if (!rp) { DECL_AM(dead); BIF_RET(AM_dead); @@ -3779,7 +3782,6 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) rp_locks &= ~ERTS_PROC_LOCK_MAIN; #endif erts_smp_proc_unlock(rp, rp_locks); - erts_smp_proc_dec_refc(rp); if (xres > 1) { DECL_AM(message); BIF_RET(AM_message); @@ -3990,7 +3992,7 @@ static Eterm lcnt_build_lock_term(Eterm **hpp, Uint *szp, erts_lcnt_lock_t *lock id = am_atom_put(ltype, strlen(ltype)); } else if (lock->flag & ERTS_LCNT_LT_PROCLOCK) { /* use registered names as id's for process locks if available */ - proc = erts_pid2proc_unlocked(lock->id); + proc = erts_proc_lookup(lock->id); if (proc && proc->reg) { id = proc->reg->name; } else { diff --git a/erts/emulator/beam/erl_bif_timer.c b/erts/emulator/beam/erl_bif_timer.c index d806be0704..525b11f61c 100644 --- a/erts/emulator/beam/erl_bif_timer.c +++ b/erts/emulator/beam/erl_bif_timer.c @@ -324,10 +324,9 @@ bif_timer_timeout(ErtsBifTimer* btm) ASSERT(!erts_get_current_process()); if (btm->flags & BTM_FLG_BYNAME) - rp = erts_whereis_process(NULL,0,btm->receiver.name,0,ERTS_P2P_FLG_SMP_INC_REFC); + rp = erts_whereis_process(NULL, 0, btm->receiver.name, 0, 0); else { rp = btm->receiver.proc.ess; - erts_smp_proc_inc_refc(rp); unlink_proc(btm); } @@ -379,7 +378,6 @@ bif_timer_timeout(ErtsBifTimer* btm) #endif ); erts_smp_proc_unlock(rp, rp_locks); - erts_smp_proc_dec_refc(rp); } } diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c index 80f774523c..8fc8e363b1 100644 --- a/erts/emulator/beam/erl_bif_trace.c +++ b/erts/emulator/beam/erl_bif_trace.c @@ -653,8 +653,7 @@ Eterm trace_3(BIF_ALIST_3) if (procs || mods) { /* tracing of processes */ for (i = 0; i < erts_max_processes; i++) { - Process* tracee_p = process_tab[i]; - + Process* tracee_p = erts_pix2proc(i); if (! tracee_p) continue; if (tracer != NIL) { @@ -772,8 +771,7 @@ static int port_already_traced(Process *c_p, Port *tracee_port, Eterm tracer) return 1; } else if(is_internal_pid(tracee_port->tracer_proc)) { - Process *tracer_p = erts_pid2proc(c_p, ERTS_PROC_LOCK_MAIN, - tracee_port->tracer_proc, 0); + Process *tracer_p = erts_proc_lookup(tracee_port->tracer_proc); if (!tracer_p) { /* Current trace process now invalid * - discard it and approve the new. */ @@ -813,8 +811,7 @@ static int already_traced(Process *c_p, Process *tracee_p, Eterm tracer) return 1; } else if(is_internal_pid(tracee_p->tracer_proc)) { - Process *tracer_p = erts_pid2proc(c_p, ERTS_PROC_LOCK_MAIN, - tracee_p->tracer_proc, 0); + Process *tracer_p = erts_proc_lookup(tracee_p->tracer_proc); if (!tracer_p) { /* Current trace process now invalid * - discard it and approve the new. */ @@ -876,7 +873,7 @@ trace_info_pid(Process* p, Eterm pid_spec, Eterm key) } if (is_internal_pid(tracer)) { - if (!erts_pid2proc(p, ERTS_PROC_LOCK_MAIN, tracer, 0)) { + if (!erts_proc_lookup(tracer)) { reset_tracer: tracee->trace_flags &= ~TRACEE_FLAGS; trace_flags = tracee->trace_flags; @@ -2142,9 +2139,9 @@ BIF_RETTYPE system_profile_2(BIF_ALIST_2) /* Check if valid process, no locks are taken */ if (is_internal_pid(profiler)) { - if (internal_pid_index(profiler) >= erts_max_processes) goto error; - profiler_p = process_tab[internal_pid_index(profiler)]; - if (INVALID_PID(profiler_p, profiler)) goto error; + profiler_p = erts_proc_lookup(profiler); + if (!profiler_p) + goto error; } else if (is_internal_port(profiler)) { if (internal_port_index(profiler) >= erts_max_ports) goto error; profiler_port = &erts_port[internal_port_index(profiler)]; diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index 51bdf53823..2b8bcb5bf7 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -2850,7 +2850,7 @@ void init_db(void) else db_max_tabs = user_requested_db_max_tabs; - bits = erts_fit_in_bits(db_max_tabs-1); + bits = erts_fit_in_bits_int32(db_max_tabs-1); if (bits > SMALL_BITS) { erl_exit(1,"Max limit for ets tabled too high %u (max %u).", db_max_tabs, ((Uint)1)<started, &tb->common.heir_started) != 0) { + if (to_proc->started_interval != tb->common.heir_started_interval) { erts_smp_proc_unlock(to_proc, to_locks); return 0; /* heir dead and pid reused, table still mine */ } @@ -3520,14 +3520,14 @@ static void set_heir(Process* me, DbTable* tb, Eterm heir, UWord heir_data) return; } if (heir == me->id) { - tb->common.heir_started = me->started; + erts_ensure_later_proc_interval(me->started_interval); + tb->common.heir_started_interval = me->started_interval; } else { - Process* heir_proc= erts_pid2proc_opt(me, ERTS_PROC_LOCK_MAIN, heir, - 0, ERTS_P2P_FLG_SMP_INC_REFC); + Process* heir_proc= erts_proc_lookup(heir); if (heir_proc != NULL) { - tb->common.heir_started = heir_proc->started; - erts_smp_proc_dec_refc(heir_proc); + erts_ensure_later_proc_interval(heir_proc->started_interval); + tb->common.heir_started_interval = heir_proc->started_interval; } else { tb->common.heir = am_none; } diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h index 6a96e174e1..ff5982640d 100644 --- a/erts/emulator/beam/erl_db_util.h +++ b/erts/emulator/beam/erl_db_util.h @@ -219,7 +219,7 @@ typedef struct db_table_common { Eterm owner; /* Pid of the creator */ Eterm heir; /* Pid of the heir */ UWord heir_data; /* To send in ETS-TRANSFER (is_immed or (DbTerm*) */ - SysTimeval heir_started; /* To further identify the heir */ + Uint64 heir_started_interval; /* To further identify the heir */ Eterm the_name; /* an atom */ Eterm id; /* atom | integer */ DbTableMethod* meth; /* table methods */ diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index ca4385dd3a..ead7b40679 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -614,6 +614,7 @@ early_init(int *argc, char **argv) /* erts_printf_eterm_func = erts_printf_term; erts_disable_tolerant_timeofday = 0; display_items = 200; + erts_proc.max = ERTS_DEFAULT_MAX_PROCESSES; erts_backtrace_depth = DEFAULT_BACKTRACE_SIZE; erts_async_max_threads = 0; erts_async_thread_suggested_stack_size = ERTS_ASYNC_THREAD_MIN_STACK_SIZE; @@ -1160,7 +1161,7 @@ erl_start(int argc, char **argv) case 'P': /* set maximum number of processes */ Parg = get_arg(argv[i]+2, argv[i+1], &i); - erts_max_processes = atoi(Parg); + erts_proc.max = atoi(Parg); /* Check of result is delayed until later. This is because +R may be given after +P. */ break; @@ -1439,10 +1440,10 @@ erl_start(int argc, char **argv) } /* Delayed check of +P flag */ - if (erts_max_processes < ERTS_MIN_PROCESSES - || erts_max_processes > ERTS_MAX_PROCESSES + if (erts_proc.max < ERTS_MIN_PROCESSES + || erts_proc.max > ERTS_MAX_PROCESSES || (erts_use_r9_pids_ports - && erts_max_processes > ERTS_MAX_R9_PROCESSES)) { + && erts_proc.max > ERTS_MAX_R9_PROCESSES)) { erts_fprintf(stderr, "bad number of processes %s\n", Parg); erts_usage(); } diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index bd86e3ea9e..4a797fbeae 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -363,7 +363,7 @@ erts_queue_dist_message(Process *rcvr, } } - if (rcvr->is_exiting || ERTS_PROC_PENDING_EXIT(rcvr)) { + if (ERTS_PROC_IS_EXITING(rcvr) || ERTS_PROC_PENDING_EXIT(rcvr)) { /* Drop message if receiver is exiting or has a pending exit ... */ if (is_not_nil(token)) { ErlHeapFragment *heap_frag; @@ -482,7 +482,7 @@ erts_queue_message(Process* receiver, } } - if (receiver->is_exiting || ERTS_PROC_PENDING_EXIT(receiver)) { + if (ERTS_PROC_IS_EXITING(receiver) || ERTS_PROC_PENDING_EXIT(receiver)) { /* Drop message if receiver is exiting or has a pending * exit ... */ diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index b323bc7f69..6d0975f41f 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -311,6 +311,7 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, #endif Eterm receiver = to_pid->pid; int flush_me = 0; + int scheduler = erts_get_scheduler_id() != 0; if (env != NULL) { c_p = env->proc; @@ -330,8 +331,11 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, #if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP) rp_had_locks = rp_locks; #endif - rp = erts_pid2proc_opt(c_p, ERTS_PROC_LOCK_MAIN, - receiver, rp_locks, ERTS_P2P_FLG_SMP_INC_REFC); + + rp = (scheduler + ? erts_proc_lookup(receiver) + : erts_pid2proc_opt(c_p, ERTS_PROC_LOCK_MAIN, + receiver, rp_locks, ERTS_P2P_FLG_SMP_INC_REFC)); if (rp == NULL) { ASSERT(env == NULL || receiver != c_p->id); return 0; @@ -358,12 +362,12 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, , NIL #endif ); - if (rp_locks) { - ERTS_SMP_LC_ASSERT(rp_locks == (rp_had_locks | (ERTS_PROC_LOCK_MSGQ | - ERTS_PROC_LOCK_STATUS))); - erts_smp_proc_unlock(rp, (ERTS_PROC_LOCK_MSGQ | ERTS_PROC_LOCK_STATUS)); - } - erts_smp_proc_dec_refc(rp); + if (c_p == rp) + rp_locks &= ~ERTS_PROC_LOCK_MAIN; + if (rp_locks) + erts_smp_proc_unlock(rp, rp_locks); + if (!scheduler) + erts_smp_proc_dec_refc(rp); if (flush_me) { cache_env(env); } diff --git a/erts/emulator/beam/erl_node_container_utils.h b/erts/emulator/beam/erl_node_container_utils.h index 329a2204cc..7b4cb7b042 100644 --- a/erts/emulator/beam/erl_node_container_utils.h +++ b/erts/emulator/beam/erl_node_container_utils.h @@ -128,8 +128,47 @@ extern int erts_use_r9_pids_ports; * Pids * \* */ -#define internal_pid_index(x) (internal_pid_data((x)) \ - & erts_process_tab_index_mask) +#define erts_max_processes erts_proc.max + +typedef struct { + erts_smp_atomic_t *tab; + int max; + int tab_cache_lines; + int pix_per_cache_line; + int pix_cl_mask; + int pix_cl_shift; + int pix_cli_mask; + int pix_cli_shift; +} ErtsProcTab; + +extern ErtsProcTab erts_proc; + +ERTS_GLB_INLINE int erts_pid_data2ix(Eterm pid_data); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE int erts_pid_data2ix(Eterm pid_data) +{ + int n, pix; + + n = (int) pid_data; + if (erts_proc.pix_cl_mask) { + pix = ((n & erts_proc.pix_cl_mask) << erts_proc.pix_cl_shift); + pix += ((n >> erts_proc.pix_cli_shift) & erts_proc.pix_cli_mask); + } + else { + n %= erts_proc.max; + pix = n % erts_proc.tab_cache_lines; + pix *= erts_proc.pix_per_cache_line; + pix += n / erts_proc.tab_cache_lines; + } + ASSERT(0 <= pix && pix < erts_proc.max); + return pix; +} + +#endif + +#define internal_pid_index(x) erts_pid_data2ix(internal_pid_data((x))) #define internal_pid_node_name(x) (internal_pid_node((x))->sysname) #define external_pid_node_name(x) (external_pid_node((x))->sysname) diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c index 1481f66b55..16367c305d 100644 --- a/erts/emulator/beam/erl_node_tables.c +++ b/erts/emulator/beam/erl_node_tables.c @@ -1310,21 +1310,22 @@ setup_reference_table(void) UnUseTmpHeapNoproc(3); /* Insert all processes */ - for (i = 0; i < erts_max_processes; i++) - if (process_tab[i]) { + for (i = 0; i < erts_max_processes; i++) { + Process *proc = erts_pix2proc(i); + if (proc) { ErlMessage *msg; /* Insert Heap */ - insert_offheap(&(process_tab[i]->off_heap), + insert_offheap(&(proc->off_heap), HEAP_REF, - process_tab[i]->id); + proc->id); /* Insert message buffers */ - for(hfp = process_tab[i]->mbuf; hfp; hfp = hfp->next) + for(hfp = proc->mbuf; hfp; hfp = hfp->next) insert_offheap(&(hfp->off_heap), HEAP_REF, - process_tab[i]->id); + proc->id); /* Insert msg msg buffers */ - for (msg = process_tab[i]->msg.first; msg; msg = msg->next) { + for (msg = proc->msg.first; msg; msg = msg->next) { ErlHeapFragment *heap_frag = NULL; if (msg->data.attached) { if (is_value(ERL_MESSAGE_TERM(msg))) @@ -1332,7 +1333,7 @@ setup_reference_table(void) else { if (msg->data.dist_ext->dep) insert_dist_entry(msg->data.dist_ext->dep, - HEAP_REF, process_tab[i]->id, 0); + HEAP_REF, proc->id, 0); if (is_not_nil(ERL_MESSAGE_TOKEN(msg))) heap_frag = erts_dist_ext_trailer(msg->data.dist_ext); } @@ -1340,10 +1341,10 @@ setup_reference_table(void) if (heap_frag) insert_offheap(&(heap_frag->off_heap), HEAP_REF, - process_tab[i]->id); + proc->id); } #ifdef ERTS_SMP - for (msg = process_tab[i]->msg_inq.first; msg; msg = msg->next) { + for (msg = proc->msg_inq.first; msg; msg = msg->next) { ErlHeapFragment *heap_frag = NULL; if (msg->data.attached) { if (is_value(ERL_MESSAGE_TERM(msg))) @@ -1351,7 +1352,7 @@ setup_reference_table(void) else { if (msg->data.dist_ext->dep) insert_dist_entry(msg->data.dist_ext->dep, - HEAP_REF, process_tab[i]->id, 0); + HEAP_REF, proc->id, 0); if (is_not_nil(ERL_MESSAGE_TOKEN(msg))) heap_frag = erts_dist_ext_trailer(msg->data.dist_ext); } @@ -1359,21 +1360,22 @@ setup_reference_table(void) if (heap_frag) insert_offheap(&(heap_frag->off_heap), HEAP_REF, - process_tab[i]->id); + proc->id); } #endif /* Insert links */ - if(process_tab[i]->nlinks) - insert_links(process_tab[i]->nlinks, process_tab[i]->id); - if(process_tab[i]->monitors) - insert_monitors(process_tab[i]->monitors, process_tab[i]->id); + if(proc->nlinks) + insert_links(proc->nlinks, proc->id); + if(proc->monitors) + insert_monitors(proc->monitors, proc->id); /* Insert controller */ { - DistEntry *dep = ERTS_PROC_GET_DIST_ENTRY(process_tab[i]); + DistEntry *dep = ERTS_PROC_GET_DIST_ENTRY(proc); if (dep) - insert_dist_entry(dep, CTRL_REF, process_tab[i]->id, 0); + insert_dist_entry(dep, CTRL_REF, proc->id, 0); } } + } #ifdef ERTS_SMP erts_foreach_sys_msg_in_q(insert_sys_msg); diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 4f8e00f7f1..7174991ade 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -95,13 +95,6 @@ #define NORMAL_BIT (1 << PRIORITY_NORMAL) #define LOW_BIT (1 << PRIORITY_LOW) -#define ERTS_MAYBE_SAVE_TERMINATING_PROCESS(P) \ -do { \ - ERTS_SMP_LC_ASSERT(erts_lc_mtx_is_locked(&proc_tab_mtx)); \ - if (saved_term_procs.end) \ - save_terminating_process((P)); \ -} while (0) - #define ERTS_EMPTY_RUNQ(RQ) \ ((RQ)->len == 0 && (RQ)->misc.start == NULL) @@ -112,16 +105,151 @@ extern BeamInstr beam_apply[]; extern BeamInstr beam_exit[]; extern BeamInstr beam_continue_exit[]; -static Sint p_last; -static Sint p_next; -static Sint p_serial; -static Uint p_serial_mask; -static Uint p_serial_shift; +#ifdef ARCH_32 + +union { + erts_smp_dw_atomic_t pid_data; + char align[ERTS_CACHE_LINE_SIZE]; +} last erts_align_attribute(ERTS_CACHE_LINE_SIZE); + + +static ERTS_INLINE Uint64 +dw_aint_to_uint64(erts_dw_aint_t *dw) +{ +#ifdef ETHR_SU_DW_NAINT_T__ + return (Uint64) dw->dw_sint; +#else + Uint64 res; + res = (Uint64) ((Uint32) dw->sint[ERTS_DW_AINT_HIGH_WORD]); + res <<= 32; + res |= (Uint64) ((Uint32) dw->sint[ERTS_DW_AINT_LOW_WORD]); + return res; +#endif +} + +static void +unint64_to_dw_aint(erts_dw_aint_t *dw, Uint64 val) +{ +#ifdef ETHR_SU_DW_NAINT_T__ + dw->dw_sint = (ETHR_SU_DW_NAINT_T__) val; +#else + dw->sint[ERTS_DW_AINT_LOW_WORD] = (erts_aint_t) (val & 0xffffffff); + dw->sint[ERTS_DW_AINT_HIGH_WORD] = (erts_aint_t) ((val >> 32) & 0xffffffff); +#endif +} + +static ERTS_INLINE void +last_pid_data_init_nob(Uint64 val) +{ + erts_dw_aint_t dw; + unint64_to_dw_aint(&dw, val); + erts_smp_dw_atomic_init_nob(&last.pid_data, &dw); +} + +static ERTS_INLINE void +last_pid_data_set_relb(Uint64 val) +{ + erts_dw_aint_t dw; + unint64_to_dw_aint(&dw, val); + erts_smp_dw_atomic_set_relb(&last.pid_data, &dw); +} + +static ERTS_INLINE Uint64 +last_pid_data_read_nob(void) +{ + erts_dw_aint_t dw; + erts_smp_dw_atomic_read_nob(&last.pid_data, &dw); + return dw_aint_to_uint64(&dw); +} + +static ERTS_INLINE Uint64 +last_pid_data_read_acqb(void) +{ + erts_dw_aint_t dw; + erts_smp_dw_atomic_read_acqb(&last.pid_data, &dw); + return dw_aint_to_uint64(&dw); +} + +static ERTS_INLINE Uint64 +last_pid_data_cmpxchg_relb(Uint64 new, Uint64 exp) +{ + erts_dw_aint_t dw_new, dw_xchg; + + unint64_to_dw_aint(&dw_new, new); + unint64_to_dw_aint(&dw_xchg, exp); + + if (erts_smp_dw_atomic_cmpxchg_relb(&last.pid_data, &dw_new, &dw_xchg)) + return exp; + else + return dw_aint_to_uint64(&dw_xchg); +} + +#elif defined(ARCH_64) + +union { + erts_smp_atomic_t pid_data; + char align[ERTS_CACHE_LINE_SIZE]; +} last erts_align_attribute(ERTS_CACHE_LINE_SIZE); + +static ERTS_INLINE void +last_pid_data_init_nob(Uint64 val) +{ + erts_smp_atomic_init_nob(&last.pid_data, (erts_aint_t) val); +} + +static ERTS_INLINE void +last_pid_data_set_relb(Uint64 val) +{ + erts_smp_atomic_set_relb(&last.pid_data, (erts_aint_t) val); +} + +static ERTS_INLINE Uint64 +last_pid_data_read_nob(void) +{ + return (Uint64) erts_smp_atomic_read_nob(&last.pid_data); +} + +static ERTS_INLINE Uint64 +last_pid_data_read_acqb(void) +{ + return (Uint64) erts_smp_atomic_read_acqb(&last.pid_data); +} + +static ERTS_INLINE Uint64 +last_pid_data_cmpxchg_relb(Uint64 new, Uint64 exp) +{ + return (Uint64) erts_smp_atomic_cmpxchg_relb(&last.pid_data, + (erts_aint_t) new, + (erts_aint_t) exp); +} + +#else +# error "Not 64-bit, nor 32-bit architecture..." +#endif + +static ERTS_INLINE int +last_pid_data_cmp(Uint64 lpd1, Uint64 lpd2) +{ + Uint64 lpd1_wrap; + + if (lpd1 == lpd2) + return 0; + + lpd1_wrap = lpd1 + (((Uint64) 1) << 63); + + if (lpd1 < lpd1_wrap) + return (lpd1 < lpd2 && lpd2 < lpd1_wrap) ? -1 : 1; + else + return (lpd1_wrap <= lpd2 && lpd2 < lpd1) ? 1 : -1; +} + + +#define ERTS_PID_DATA_MASK__ ((1 << _PID_DATA_SIZE) - 1) int erts_sched_compact_load; Uint erts_no_schedulers; -Uint erts_max_processes = ERTS_DEFAULT_MAX_PROCESSES; -Uint erts_process_tab_index_mask; + +ErtsProcTab erts_proc erts_align_attribute(ERTS_CACHE_LINE_SIZE); static int wakeup_other_limit; @@ -204,7 +332,7 @@ erts_sched_stat_t erts_sched_stat; static erts_tsd_key_t sched_data_key; #endif -static erts_smp_mtx_t proc_tab_mtx; +erts_smp_rwmtx_t erts_proc_tab_rwmtx; static erts_smp_atomic32_t function_calls; @@ -227,7 +355,6 @@ typedef union { static ErtsAlignedSchedulerSleepInfo *aligned_sched_sleep_info; -Process** process_tab; static Uint last_reductions; static Uint last_exact_reductions; Uint erts_default_process_flags; @@ -258,11 +385,11 @@ struct ErtsTermProcElement_ { union { struct { Eterm pid; - SysTimeval spawned; - SysTimeval exited; + Uint64 spawned; + Uint64 exited; } process; struct { - SysTimeval time; + Uint64 interval; } bif_invocation; } u; }; @@ -399,6 +526,53 @@ erts_smp_lc_runq_is_locked(ErtsRunQueue *runq) } #endif +static erts_interval_t *proc_interval; + +static void +proc_interval_init(void) +{ + proc_interval = erts_alloc_permanent_cache_aligned( + ERTS_ALC_T_PROC_INTERVAL, + sizeof(erts_interval_t)); + erts_smp_interval_init(proc_interval); +} + +static ERTS_INLINE Uint64 +get_proc_interval(void) +{ + return erts_smp_current_interval_nob(proc_interval); +} + +static ERTS_INLINE Uint64 +ensure_later_proc_interval(Uint64 interval) +{ + return erts_smp_ensure_later_interval_nob(proc_interval, interval); +} + +static ERTS_INLINE Uint64 +step_proc_interval(void) +{ + return erts_smp_step_interval_nob(proc_interval); +} + +Uint64 +erts_get_proc_interval(void) +{ + return get_proc_interval(); +} + +Uint64 +erts_ensure_later_proc_interval(Uint64 interval) +{ + return ensure_later_proc_interval(interval); +} + +Uint64 +erts_step_proc_interval(void) +{ + return step_proc_interval(); +} + void erts_pre_init_process(void) { @@ -448,7 +622,16 @@ erts_pre_init_process(void) void erts_init_process(int ncpu) { - Uint proc_bits = ERTS_PROC_BITS; + int proc_tab_sz; + int max_proc_bits; + int proc_bits = ERTS_PROC_BITS; + erts_smp_atomic_t *proc_entry; + char *proc_tab_end; + erts_smp_rwmtx_opt_t proc_tab_rwmtx_opts = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; + proc_tab_rwmtx_opts.type = ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ; + proc_tab_rwmtx_opts.lived = ERTS_SMP_RWMTX_LONG_LIVED; + + proc_interval_init(); #ifdef ERTS_SMP erts_disable_proc_not_running_opt = 0; @@ -459,29 +642,57 @@ erts_init_process(int ncpu) erts_smp_atomic32_init_nob(&process_count, 0); - if (erts_use_r9_pids_ports) { + if (erts_use_r9_pids_ports) proc_bits = ERTS_R9_PROC_BITS; - ASSERT(erts_max_processes <= (1 << ERTS_R9_PROC_BITS)); - } - process_tab = (Process**) erts_alloc(ERTS_ALC_T_PROC_TABLE, - erts_max_processes*sizeof(Process*)); - sys_memzero(process_tab, erts_max_processes * sizeof(Process*)); + if (erts_proc.max > (1 << proc_bits)) + erts_proc.max = 1 << proc_bits; + + proc_tab_sz = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(erts_proc.max + * sizeof(erts_smp_atomic_t)); + erts_proc.tab = erts_alloc(ERTS_ALC_T_PROC_TABLE, proc_tab_sz); + proc_tab_end = ((char *) erts_proc.tab) + proc_tab_sz; + proc_entry = erts_proc.tab; + while (proc_tab_end > ((char *) proc_entry)) { + erts_smp_atomic_init_nob(proc_entry, ERTS_AINT_NULL); + proc_entry++; + } #ifdef HYBRID erts_active_procs = (Process**) erts_alloc(ERTS_ALC_T_ACTIVE_PROCS, - erts_max_processes * sizeof(Process*)); + erts_proc.max * sizeof(Process*)); erts_num_active_procs = 0; #endif - erts_smp_mtx_init(&proc_tab_mtx, "proc_tab"); - p_last = -1; - p_next = 0; - p_serial = 0; + erts_smp_rwmtx_init_opt(&erts_proc_tab_rwmtx, + &proc_tab_rwmtx_opts, + "proc_tab"); + last_pid_data_init_nob(~((Uint64) 0)); + + max_proc_bits = erts_fit_in_bits_int32((Sint32) erts_proc.max - 1); + + erts_proc.tab_cache_lines = proc_tab_sz/ERTS_CACHE_LINE_SIZE; + erts_proc.pix_per_cache_line = ERTS_CACHE_LINE_SIZE/sizeof(erts_smp_atomic_t); + if ((erts_proc.max & (erts_proc.max - 1)) + | (erts_proc.pix_per_cache_line & (erts_proc.pix_per_cache_line - 1))) { + /* + * erts_proc.max or erts_proc.pix_per_cache_line + * not a power of 2 :( + */ + erts_proc.pix_cl_mask = 0; + erts_proc.pix_cl_shift = 0; + erts_proc.pix_cli_mask = 0; + erts_proc.pix_cli_shift = 0; + } + else { + ASSERT((erts_proc.tab_cache_lines + & (erts_proc.tab_cache_lines - 1)) == 0); + erts_proc.pix_cl_mask = erts_proc.tab_cache_lines-1; + erts_proc.pix_cl_shift = erts_fit_in_bits_int32(erts_proc.pix_per_cache_line-1); + erts_proc.pix_cli_shift = erts_fit_in_bits_int32(erts_proc.pix_cl_mask); + erts_proc.pix_cli_mask = (1 << (max_proc_bits - erts_proc.pix_cli_shift)) - 1; + } - p_serial_shift = erts_fit_in_bits(erts_max_processes - 1); - p_serial_mask = ((~(~((Uint) 0) << proc_bits)) >> p_serial_shift); - erts_process_tab_index_mask = ~(~((Uint) 0) << p_serial_shift); last_reductions = 0; last_exact_reductions = 0; erts_default_process_flags = 0; @@ -739,8 +950,9 @@ static ERTS_INLINE ErtsProcList * proclist_create(Process *p) { ErtsProcList *plp = proclist_alloc(); + ensure_later_proc_interval(p->started_interval); plp->pid = p->id; - plp->started = p->started; + plp->started_interval = p->started_interval; return plp; } @@ -753,8 +965,7 @@ proclist_destroy(ErtsProcList *plp) static ERTS_INLINE int proclist_same(ErtsProcList *plp, Process *p) { - return (plp->pid == p->id - && erts_cmp_timeval(&plp->started, &p->started) == 0); + return plp->pid == p->id && plp->started_interval == p->started_interval; } ErtsProcList * @@ -5162,7 +5373,7 @@ do_bif_suspend_process(ErtsSuspendMonitor *smon, ErtsRunQueue *locked_runq) { ASSERT(suspendee); - ASSERT(!suspendee->is_exiting); + ASSERT(!ERTS_PROC_IS_EXITING(suspendee)); ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(suspendee)); if (smon) { @@ -6338,27 +6549,14 @@ Process *schedule(Process *p, int calls) #ifdef ERTS_SMP ASSERT(esdp->free_process == p); esdp->free_process = NULL; - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS); - erts_smp_proc_dec_refc(p); #else erts_free_proc(p); #endif - } else { - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS); } -#ifdef ERTS_SMP - { - ErtsProcList *pnd_xtrs = rq->procs.pending_exiters; - rq->procs.pending_exiters = NULL; + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS); - if (pnd_xtrs) { - erts_smp_runq_unlock(rq); - handle_pending_exiters(pnd_xtrs); - erts_smp_runq_lock(rq); - } - - } +#ifdef ERTS_SMP ASSERT(!esdp->free_process); #endif ASSERT(!esdp->current_process); @@ -6381,6 +6579,20 @@ Process *schedule(Process *p, int calls) #ifdef ERTS_SMP +#ifdef ERTS_SMP + { + ErtsProcList *pnd_xtrs = rq->procs.pending_exiters; + rq->procs.pending_exiters = NULL; + + if (pnd_xtrs) { + erts_smp_runq_unlock(rq); + handle_pending_exiters(pnd_xtrs); + erts_smp_runq_lock(rq); + } + + } +#endif + if (rq->check_balance_reds <= 0) check_balance(rq); @@ -6918,44 +7130,50 @@ erts_get_exact_total_reductions(Process *c_p, Uint *redsp, Uint *diffp) Sint erts_test_next_pid(int set, Uint next) { + Uint64 lpd; Sint res; - Sint p_prev; + Eterm pid_data; + int first_pix = -1; + erts_smp_rwmtx_rwlock(&erts_proc_tab_rwmtx); - erts_smp_mtx_lock(&proc_tab_mtx); - - if (!set) { - res = p_next < 0 ? -1 : (p_serial << p_serial_shift | p_next); - } + if (!set) + lpd = last_pid_data_read_nob(); else { - p_serial = (Sint) ((next >> p_serial_shift) & p_serial_mask); - p_next = (Sint) (erts_process_tab_index_mask & next); - - if (p_next >= erts_max_processes) { - p_next = 0; - p_serial++; - p_serial &= p_serial_mask; + lpd = (Uint64) next; + pid_data = (Eterm) (lpd & ERTS_PID_DATA_MASK__); + if (ERTS_INVALID_PID == make_internal_pid(pid_data)) { + lpd += erts_proc.max; + ASSERT(erts_pid_data2ix(pid_data) + == erts_pid_data2ix(lpd & ERTS_PID_DATA_MASK__)); } + last_pid_data_set_relb(lpd); + } - p_prev = p_next; - - do { - if (!process_tab[p_next]) - break; - p_next++; - if(p_next >= erts_max_processes) { - p_next = 0; - p_serial++; - p_serial &= p_serial_mask; + while (1) { + int pix; + lpd++; + pix = (int) (lpd % erts_proc.max); + if (first_pix < 0) + first_pix = pix; + else if (pix == first_pix) { + res = -1; + break; + } + if (ERTS_AINT_NULL == erts_smp_atomic_read_nob(&erts_proc.tab[pix])) { + pid_data = (Eterm) (lpd & ERTS_PID_DATA_MASK__); + if (ERTS_INVALID_PID == make_internal_pid(pid_data)) { + lpd += erts_proc.max; + ASSERT(erts_pid_data2ix(pid_data) + == erts_pid_data2ix(lpd & ERTS_PID_DATA_MASK__)); } - } while (p_prev != p_next); - - res = process_tab[p_next] ? -1 : (p_serial << p_serial_shift | p_next); - + res = lpd & ERTS_PID_DATA_MASK__; + break; + } } - erts_smp_mtx_unlock(&proc_tab_mtx); + erts_smp_rwmtx_rwunlock(&erts_proc_tab_rwmtx); return res; @@ -6964,6 +7182,8 @@ erts_test_next_pid(int set, Uint next) Uint erts_process_count(void) { erts_aint32_t res = erts_smp_atomic32_read_nob(&process_count); + if (res > erts_proc.max) + return erts_proc.max; ASSERT(res >= 0); return (Uint) res; } @@ -6977,91 +7197,126 @@ erts_free_proc(Process *p) erts_free(ERTS_ALC_T_PROC, (void *) p); } - /* ** Allocate process and find out where to place next process. */ static Process* alloc_process(void) { -#ifdef ERTS_SMP - erts_pix_lock_t *pix_lock; -#endif + int pix; Process* p; - int p_prev; + Uint64 lpd, exp_lpd; + Eterm pid_data; + erts_aint32_t proc_count; +#ifdef DEBUG + Eterm pid; +#endif - erts_smp_mtx_lock(&proc_tab_mtx); + erts_smp_rwmtx_rlock(&erts_proc_tab_rwmtx); - if (p_next == -1) { - p = NULL; - goto error; /* Process table full! */ + proc_count = erts_smp_atomic32_inc_read_acqb(&process_count); + if (proc_count > erts_proc.max) { + while (1) { + erts_aint32_t act_proc_count; + + act_proc_count = erts_smp_atomic32_cmpxchg_relb(&process_count, + proc_count-1, + proc_count); + if (act_proc_count == proc_count) + goto system_limit; + proc_count = act_proc_count; + if (proc_count <= erts_proc.max) + break; + } } p = (Process*) erts_alloc_fnf(ERTS_ALC_T_PROC, sizeof(Process)); if (!p) - goto error; /* ENOMEM */ + goto enomem; - p_last = p_next; + p->approx_started = erts_get_approx_time(); + p->started_interval = get_proc_interval(); - erts_get_emu_time(&p->started); + lpd = last_pid_data_read_acqb(); -#ifdef ERTS_SMP - pix_lock = ERTS_PIX2PIXLOCK(p_next); - erts_pix_lock(pix_lock); -#endif - ASSERT(!process_tab[p_next]); + /* Reserve slot */ + while (1) { + lpd++; + pix = erts_pid_data2ix((Eterm) (lpd & ERTS_PID_DATA_MASK__)); + if (erts_smp_atomic_read_nob(&erts_proc.tab[pix]) == ERTS_AINT_NULL) { + erts_aint_t val; + val = erts_smp_atomic_cmpxchg_relb(&erts_proc.tab[pix], + ((erts_aint_t) + ERTS_PROC_LOCK_BUSY), + ERTS_AINT_NULL); + + if (ERTS_AINT_NULL == val) + break; + } + } + + pid_data = (Eterm) lpd & ERTS_PID_DATA_MASK__; - process_tab[p_next] = p; - erts_smp_atomic32_inc_nob(&process_count); - p->id = make_internal_pid(p_serial << p_serial_shift | p_next); + p->id = make_internal_pid(pid_data); if (p->id == ERTS_INVALID_PID) { /* Do not use the invalid pid; change serial */ - p_serial++; - p_serial &= p_serial_mask; - p->id = make_internal_pid(p_serial << p_serial_shift | p_next); + lpd += erts_proc.max; + ASSERT(pix == erts_pid_data2ix((Eterm) (lpd & ERTS_PID_DATA_MASK__))); + pid_data = (Eterm) lpd & ERTS_PID_DATA_MASK__; + p->id = make_internal_pid(pid_data); ASSERT(p->id != ERTS_INVALID_PID); } - ASSERT(internal_pid_serial(p->id) <= (erts_use_r9_pids_ports - ? ERTS_MAX_PID_R9_SERIAL - : ERTS_MAX_PID_SERIAL)); -#ifdef ERTS_SMP - erts_proc_lock_init(p); /* All locks locked */ - erts_pix_unlock(pix_lock); -#endif + exp_lpd = last_pid_data_read_nob(); - p->rstatus = P_FREE; - p->rcount = 0; + /* Move last pid data forward */ + while (1) { + Uint64 act_lpd; + if (last_pid_data_cmp(lpd, exp_lpd) < 0) + break; + act_lpd = last_pid_data_cmpxchg_relb(lpd, exp_lpd); + if (act_lpd == exp_lpd) + break; + exp_lpd = act_lpd; + } - /* - * set p_next to the next available slot - */ +#ifdef DEBUG + pid = p->id; +#endif - p_prev = p_next; +#ifdef ERTS_SMP + erts_proc_lock_init(p); /* All locks locked */ +#endif - while (1) { - p_next++; - if(p_next >= erts_max_processes) { - p_serial++; - p_serial &= p_serial_mask; - p_next = 0; - } + /* Move into slot reserved */ +#ifdef DEBUG + ASSERT(ERTS_PROC_LOCK_BUSY + == (Process *) erts_smp_atomic_xchg_relb(&erts_proc.tab[pix], + (erts_aint_t) p)); +#else + erts_smp_atomic_set_relb(&erts_proc.tab[pix], (erts_aint_t) p); +#endif - if (p_prev == p_next) { - p_next = -1; - break; /* Table full! */ - } + ASSERT(internal_pid_serial(p->id) <= (erts_use_r9_pids_ports + ? ERTS_MAX_PID_R9_SERIAL + : ERTS_MAX_PID_SERIAL)); - if (!process_tab[p_next]) - break; /* found a free slot */ - } + erts_smp_rwmtx_runlock(&erts_proc_tab_rwmtx); - error: + p->rstatus = P_FREE; + p->rcount = 0; - erts_smp_mtx_unlock(&proc_tab_mtx); + ASSERT(p == (Process *) + erts_smp_atomic_read_nob( + &erts_proc.tab[internal_pid_index(pid)])); return p; +enomem: +system_limit: + + erts_smp_rwmtx_runlock(&erts_proc_tab_rwmtx); + return NULL; } Eterm @@ -7089,7 +7344,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). /* * Copy the arguments to the global heap * Since global GC might occur we want to do this before adding the - * new process to the process_tab. + * new process to the erts_proc.tab. */ BM_SWAP_TIMER(system,copy); LAZY_COPY(parent,args); @@ -7246,7 +7501,12 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). : STORE_NC(&p->htop, &p->off_heap, parent->group_leader); } +#if 1 + p->trace_flags = ~TRACEE_FLAGS; + p->tracer_proc = NIL; +#else erts_get_default_tracing(&p->trace_flags, &p->tracer_proc); +#endif p->msg.first = NULL; p->msg.last = &p->msg.first; @@ -7358,7 +7618,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). #ifdef ERTS_SMP p->scheduler_data = NULL; - p->is_exiting = 0; + erts_atomic32_init_nob(&p->aflags, 0); p->status_flags = 0; p->runq_flags = 0; p->suspendee = NIL; @@ -7510,8 +7770,8 @@ void erts_init_empty_process(Process *p) p->def_arg_reg[5] = 0; p->parent = NIL; - p->started.tv_sec = 0; - p->started.tv_usec = 0; + p->approx_started = 0; + p->started_interval = 0; #ifdef HIPE hipe_init_process(&p->hipe); @@ -7536,7 +7796,7 @@ void erts_init_empty_process(Process *p) #ifdef ERTS_SMP p->scheduler_data = NULL; - p->is_exiting = 0; + erts_atomic32_init_nob(&p->aflags, 0); p->status_flags = 0; p->runq_flags = 0; p->msg_inq.first = NULL; @@ -7757,7 +8017,7 @@ set_proc_exiting(Process *p, Eterm reason, ErlHeapFragment *bp) */ erts_pix_lock(pix_lock); - p->is_exiting = 1; + erts_atomic32_read_bor_relb(&p->aflags, ERTS_PROC_AFLG_EXITING); #endif p->status = P_EXITING; #ifdef ERTS_SMP @@ -7849,7 +8109,7 @@ save_pending_exiter(Process *p) rq->procs.pending_exiters = plp; erts_smp_runq_unlock(rq); - + wake_scheduler(rq, 1); } #endif @@ -8021,7 +8281,7 @@ send_exit_signal(Process *c_p, /* current process if and only } else if (reason != am_normal || (flags & ERTS_XSIG_FLG_NO_IGN_NORMAL)) { #ifdef ERTS_SMP - if (!ERTS_PROC_PENDING_EXIT(rp) && !rp->is_exiting) { + if (!ERTS_PROC_PENDING_EXIT(rp) && !ERTS_PROC_IS_EXITING(rp)) { ASSERT(rp->status != P_EXITING); ASSERT(rp->status != P_FREE); ASSERT(!rp->pending_exit.bp); @@ -8087,6 +8347,8 @@ send_exit_signal(Process *c_p, /* current process if and only &bp->off_heap); rp->pending_exit.bp = bp; } + erts_atomic32_read_bor_relb(&rp->aflags, + ERTS_PROC_AFLG_PENDING_EXIT); ASSERT(ERTS_PROC_PENDING_EXIT(rp)); } if (!(rp->status_flags @@ -8417,6 +8679,15 @@ resume_suspend_monitor(ErtsSuspendMonitor *smon, void *vc_p) erts_destroy_suspend_monitor(smon); } +#ifdef ERTS_SMP +static void +proc_dec_refc(void *vproc) +{ + erts_smp_proc_dec_refc((Process *) vproc); +} +#endif + + static void continue_exit_process(Process *p #ifdef ERTS_SMP @@ -8462,7 +8733,7 @@ erts_do_exit_process(Process* p, Eterm reason) #ifdef ERTS_SMP erts_pix_lock(pix_lock); - p->is_exiting = 1; + erts_atomic32_read_bor_relb(&p->aflags, ERTS_PROC_AFLG_EXITING); #endif p->status = P_EXITING; @@ -8619,20 +8890,24 @@ continue_exit_process(Process *p #endif { + int maybe_save; int pix; /* Do *not* use erts_get_runq_proc() */ ErtsRunQueue *rq; rq = erts_get_runq_current(ERTS_GET_SCHEDULER_DATA_FROM_PROC(p)); - ASSERT(internal_pid_index(p->id) < erts_max_processes); pix = internal_pid_index(p->id); - erts_smp_mtx_lock(&proc_tab_mtx); + erts_smp_rwmtx_rlock(&erts_proc_tab_rwmtx); + maybe_save = saved_term_procs.end != NULL; + if (maybe_save) { + erts_smp_rwmtx_runlock(&erts_proc_tab_rwmtx); + erts_smp_rwmtx_rwlock(&erts_proc_tab_rwmtx); + } + erts_smp_runq_lock(rq); #ifdef ERTS_SMP - erts_pix_lock(pix_lock); - ASSERT(p->scheduler_data); ASSERT(p->scheduler_data->current_process == p); ASSERT(p->scheduler_data->free_process == NULL); @@ -8641,26 +8916,22 @@ continue_exit_process(Process *p p->scheduler_data->free_process = p; p->status_flags = 0; #endif - process_tab[pix] = NULL; /* Time of death! */ + /* Time of death! */ + erts_smp_atomic_set_relb(&erts_proc.tab[pix], ERTS_AINT_NULL); + ASSERT(erts_smp_atomic32_read_nob(&process_count) > 0); - erts_smp_atomic32_dec_nob(&process_count); + erts_smp_atomic32_dec_relb(&process_count); -#ifdef ERTS_SMP - erts_pix_unlock(pix_lock); -#endif erts_smp_runq_unlock(rq); - if (p_next < 0) { - if (p_last >= p_next) { - p_serial++; - p_serial &= p_serial_mask; - } - p_next = pix; + if (!maybe_save) + erts_smp_rwmtx_runlock(&erts_proc_tab_rwmtx); + else { + if (saved_term_procs.end) + save_terminating_process(p); + erts_smp_rwmtx_rwunlock(&erts_proc_tab_rwmtx); } - ERTS_MAYBE_SAVE_TERMINATING_PROCESS(p); - - erts_smp_mtx_unlock(&proc_tab_mtx); } /* @@ -8730,8 +9001,25 @@ continue_exit_process(Process *p delete_process(p); +#ifdef ERTS_SMP + /* + * Each scheduler will decrease refc by one via misc aux work; + * we have one refc for reference from process table which we + * now want to remove, i.e. we increase refc with schedulers-1. + * + * Process struct wont be deallocated until (earliest) when + * all schedulers have decreased refc via misc aux work... + */ + if (erts_no_schedulers != 1) + erts_smp_proc_add_refc(p, (Sint32) erts_no_schedulers-1); + erts_schedule_multi_misc_aux_work(0, + erts_no_schedulers, + proc_dec_refc, + (void *) p); + erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); +#endif return; @@ -9041,13 +9329,13 @@ do { \ #endif #if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS -# define ERTS_PROCS_DBG_CHK_PID_FOUND(PBDP, PID, TVP) \ - debug_processes_check_found_pid((PBDP), (PID), (TVP), 1) -# define ERTS_PROCS_DBG_CHK_PID_NOT_FOUND(PBDP, PID, TVP) \ - debug_processes_check_found_pid((PBDP), (PID), (TVP), 0) +# define ERTS_PROCS_DBG_CHK_PID_FOUND(PBDP, PID, IC) \ + debug_processes_check_found_pid((PBDP), (PID), (IC), 1) +# define ERTS_PROCS_DBG_CHK_PID_NOT_FOUND(PBDP, PID, IC) \ + debug_processes_check_found_pid((PBDP), (PID), (IC), 0) #else -# define ERTS_PROCS_DBG_CHK_PID_FOUND(PBDP, PID, TVP) -# define ERTS_PROCS_DBG_CHK_PID_NOT_FOUND(PBDP, PID, TVP) +# define ERTS_PROCS_DBG_CHK_PID_FOUND(PBDP, PID, IC) +# define ERTS_PROCS_DBG_CHK_PID_NOT_FOUND(PBDP, PID, IC) #endif #if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_TERM_PROC_LIST @@ -9092,7 +9380,7 @@ static Uint processes_bif_tab_chunks; static Export processes_trap_export; typedef struct { - SysTimeval time; + Uint64 interval; } ErtsProcessesBifChunkInfo; typedef enum { @@ -9117,7 +9405,7 @@ typedef struct { struct { Eterm caller; #if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS - SysTimeval *pid_started; + Uint64 *pid_started; #endif #if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_HALLOC Eterm *heap; @@ -9146,11 +9434,10 @@ static void debug_processes_verify_all_pids(ErtsProcessesBifData *pbdp); #if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS static void debug_processes_check_found_pid(ErtsProcessesBifData *pbdp, Eterm pid, - SysTimeval *started, + Uint64 ic, int pid_should_be_found); #endif #if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_TERM_PROC_LIST -static SysTimeval debug_tv_start; static void debug_processes_check_term_proc_list(void); static void debug_processes_check_term_proc_free_list(ErtsTermProcElement *tpep); #endif @@ -9161,7 +9448,7 @@ save_terminating_process(Process *p) ErtsTermProcElement *tpep = erts_alloc(ERTS_ALC_T_PROCS_TPROC_EL, sizeof(ErtsTermProcElement)); ERTS_PROCS_ASSERT(saved_term_procs.start && saved_term_procs.end); - ERTS_SMP_LC_ASSERT(erts_lc_mtx_is_locked(&proc_tab_mtx)); + ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&erts_proc_tab_rwmtx)); ERTS_PROCS_DBG_CHK_TPLIST(); @@ -9169,19 +9456,19 @@ save_terminating_process(Process *p) tpep->next = NULL; tpep->ix = internal_pid_index(p->id); tpep->u.process.pid = p->id; - tpep->u.process.spawned = p->started; - erts_get_emu_time(&tpep->u.process.exited); + tpep->u.process.spawned = p->started_interval; + tpep->u.process.exited = get_proc_interval(); saved_term_procs.end->next = tpep; saved_term_procs.end = tpep; ERTS_PROCS_DBG_CHK_TPLIST(); - ERTS_PROCS_ASSERT((tpep->prev->ix >= 0 - ? erts_cmp_timeval(&tpep->u.process.exited, - &tpep->prev->u.process.exited) - : erts_cmp_timeval(&tpep->u.process.exited, - &tpep->prev->u.bif_invocation.time)) > 0); + ERTS_PROCS_ASSERT(tpep->prev->ix >= 0 + ? (tpep->u.process.exited + >= tpep->prev->u.process.exited) + : (tpep->u.process.exited + >= tpep->prev->u.bif_invocation.interval)); } static void @@ -9212,7 +9499,7 @@ cleanup_processes_bif_data(Binary *bp) if (pbdp->bif_invocation) { ErtsTermProcElement *tpep; - erts_smp_mtx_lock(&proc_tab_mtx); + erts_smp_rwmtx_rwlock(&erts_proc_tab_rwmtx); ERTS_PROCS_DBG_TRACE(pbdp->debug.caller, cleanup_processes_bif_data, @@ -9266,7 +9553,7 @@ cleanup_processes_bif_data(Binary *bp) ERTS_PROCS_DBG_CHK_TPLIST(); - erts_smp_mtx_unlock(&proc_tab_mtx); + erts_smp_rwmtx_rwunlock(&erts_proc_tab_rwmtx); } } @@ -9294,7 +9581,7 @@ processes_bif_engine(Process *p, Eterm *res_accp, Binary *mbp) pbdp->tix = 0; pbdp->pid_ix = 0; - erts_smp_mtx_lock(&proc_tab_mtx); + erts_smp_rwmtx_rwlock(&erts_proc_tab_rwmtx); locked = 1; ERTS_PROCS_DBG_TRACE(p->id, processes_bif_engine, init); @@ -9305,7 +9592,7 @@ processes_bif_engine(Process *p, Eterm *res_accp, Binary *mbp) #if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS pbdp->debug.pid_started = erts_alloc(ERTS_ALC_T_PROCS_PIDS, - sizeof(SysTimeval)*pbdp->pid_sz); + sizeof(Uint64)*pbdp->pid_sz); #endif ERTS_PROCS_DBG_SAVE_PIDS(pbdp); @@ -9320,7 +9607,8 @@ processes_bif_engine(Process *p, Eterm *res_accp, Binary *mbp) pbdp->bif_invocation = erts_alloc(ERTS_ALC_T_PROCS_TPROC_EL, sizeof(ErtsTermProcElement)); pbdp->bif_invocation->ix = -1; - erts_get_emu_time(&pbdp->bif_invocation->u.bif_invocation.time); + pbdp->bif_invocation->u.bif_invocation.interval + = step_proc_interval(); ERTS_PROCS_DBG_CHK_TPLIST(); pbdp->bif_invocation->next = NULL; @@ -9347,30 +9635,31 @@ processes_bif_engine(Process *p, Eterm *res_accp, Binary *mbp) int indices = ERTS_PROCESSES_BIF_TAB_CHUNK_SIZE; int cix = ix / ERTS_PROCESSES_BIF_TAB_CHUNK_SIZE; int end_ix = ix + indices; - SysTimeval *invocation_timep; + Uint64 *invocation_interval_p; - invocation_timep = (pbdp->bif_invocation - ? &pbdp->bif_invocation->u.bif_invocation.time - : NULL); + invocation_interval_p + = (pbdp->bif_invocation + ? &pbdp->bif_invocation->u.bif_invocation.interval + : NULL); ERTS_PROCS_ASSERT(is_nil(*res_accp)); if (!locked) { - erts_smp_mtx_lock(&proc_tab_mtx); + erts_smp_rwmtx_rwlock(&erts_proc_tab_rwmtx); locked = 1; } - ERTS_SMP_LC_ASSERT(erts_lc_mtx_is_locked(&proc_tab_mtx)); + ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&erts_proc_tab_rwmtx)); ERTS_PROCS_DBG_TRACE(p->id, processes_bif_engine, insp_table); if (cix != 0) - erts_get_emu_time(&pbdp->chunk[cix].time); + pbdp->chunk[cix].interval = step_proc_interval(); else if (pbdp->bif_invocation) - pbdp->chunk[0].time = *invocation_timep; - /* else: Time is irrelevant */ + pbdp->chunk[0].interval = *invocation_interval_p; + /* else: interval is irrelevant */ - if (end_ix >= erts_max_processes) { + if (end_ix >= erts_proc.max) { ERTS_PROCS_ASSERT(cix+1 == processes_bif_tab_chunks); - end_ix = erts_max_processes; + end_ix = erts_proc.max; indices = end_ix - ix; /* What to do when done with this chunk */ pbdp->state = (processes_bif_tab_chunks == 1 @@ -9379,16 +9668,15 @@ processes_bif_engine(Process *p, Eterm *res_accp, Binary *mbp) } for (; ix < end_ix; ix++) { - Process *rp = process_tab[ix]; + Process *rp = erts_pix2proc(ix); if (rp - && (!invocation_timep - || erts_cmp_timeval(&rp->started, - invocation_timep) < 0)) { + && (!invocation_interval_p + || rp->started_interval < *invocation_interval_p)) { ERTS_PROCS_ASSERT(is_internal_pid(rp->id)); pbdp->pid[pbdp->pid_ix] = rp->id; #if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS - pbdp->debug.pid_started[pbdp->pid_ix] = rp->started; + pbdp->debug.pid_started[pbdp->pid_ix] = rp->started_interval; #endif pbdp->pid_ix++; @@ -9398,7 +9686,7 @@ processes_bif_engine(Process *p, Eterm *res_accp, Binary *mbp) pbdp->tix = end_ix; - erts_smp_mtx_unlock(&proc_tab_mtx); + erts_smp_rwmtx_rwunlock(&erts_proc_tab_rwmtx); locked = 0; reds = indices/ERTS_PROCESSES_BIF_TAB_INSPECT_INDICES_PER_RED; @@ -9410,8 +9698,8 @@ processes_bif_engine(Process *p, Eterm *res_accp, Binary *mbp) ix = pbdp->tix; indices = ERTS_PROCESSES_BIF_TAB_CHUNK_SIZE; end_ix = ix + indices; - if (end_ix > erts_max_processes) { - end_ix = erts_max_processes; + if (end_ix > erts_proc.max) { + end_ix = erts_proc.max; indices = end_ix - ix; } @@ -9430,20 +9718,20 @@ processes_bif_engine(Process *p, Eterm *res_accp, Binary *mbp) int i; int max_reds; int free_term_procs = 0; - SysTimeval *invocation_timep; + Uint64 invocation_interval; ErtsTermProcElement *tpep; ErtsTermProcElement *free_list = NULL; tpep = pbdp->bif_invocation; ERTS_PROCS_ASSERT(tpep); - invocation_timep = &tpep->u.bif_invocation.time; + invocation_interval = tpep->u.bif_invocation.interval; max_reds = have_reds = ERTS_BIF_REDS_LEFT(p); if (max_reds > ERTS_PROCESSES_INSPECT_TERM_PROC_MAX_REDS) max_reds = ERTS_PROCESSES_INSPECT_TERM_PROC_MAX_REDS; reds = 0; - erts_smp_mtx_lock(&proc_tab_mtx); + erts_smp_rwmtx_rwlock(&erts_proc_tab_rwmtx); ERTS_PROCS_DBG_TRACE(p->id, processes_bif_engine, insp_term_procs); ERTS_PROCS_DBG_CHK_TPLIST(); @@ -9482,20 +9770,19 @@ processes_bif_engine(Process *p, Eterm *res_accp, Binary *mbp) } else { int cix = tpep->ix/ERTS_PROCESSES_BIF_TAB_CHUNK_SIZE; - SysTimeval *chunk_timep = &pbdp->chunk[cix].time; + Uint64 chunk_interval = pbdp->chunk[cix].interval; Eterm pid = tpep->u.process.pid; ERTS_PROCS_ASSERT(is_internal_pid(pid)); - if (erts_cmp_timeval(&tpep->u.process.spawned, - invocation_timep) < 0) { - if (erts_cmp_timeval(&tpep->u.process.exited, - chunk_timep) < 0) { + if (tpep->u.process.spawned < invocation_interval) { + if (tpep->u.process.exited < chunk_interval) { ERTS_PROCS_DBG_CHK_PID_NOT_FOUND(pbdp, pid, - &tpep->u.process.spawned); + tpep->u.process.spawned); pbdp->pid[pbdp->pid_ix] = pid; #if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS - pbdp->debug.pid_started[pbdp->pid_ix] = tpep->u.process.spawned; + pbdp->debug.pid_started[pbdp->pid_ix] + = tpep->u.process.spawned; #endif pbdp->pid_ix++; ERTS_PROCS_ASSERT(pbdp->pid_ix <= pbdp->pid_sz); @@ -9503,13 +9790,13 @@ processes_bif_engine(Process *p, Eterm *res_accp, Binary *mbp) else { ERTS_PROCS_DBG_CHK_PID_FOUND(pbdp, pid, - &tpep->u.process.spawned); + tpep->u.process.spawned); } } else { ERTS_PROCS_DBG_CHK_PID_NOT_FOUND(pbdp, pid, - &tpep->u.process.spawned); + tpep->u.process.spawned); } i++; @@ -9558,7 +9845,7 @@ processes_bif_engine(Process *p, Eterm *res_accp, Binary *mbp) ERTS_PROCS_DBG_CHK_TPLIST(); ERTS_PROCS_DBG_CHK_FREELIST(free_list); - erts_smp_mtx_unlock(&proc_tab_mtx); + erts_smp_rwmtx_rwunlock(&erts_proc_tab_rwmtx); /* * We do the actual free of term proc structures now when we @@ -9618,8 +9905,9 @@ processes_bif_engine(Process *p, Eterm *res_accp, Binary *mbp) sizeof(Eterm)*pbdp->pid_sz); #if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS pbdp->debug.pid_started = erts_realloc(ERTS_ALC_T_PROCS_PIDS, - pbdp->debug.pid_started, - sizeof(SysTimeval)*pbdp->pid_sz); + pbdp->debug.pid_started, + (sizeof(Uint64) + * pbdp->pid_sz)); #endif } reds = conses/ERTS_PROCESSES_BIF_BUILD_RESULT_CONSES_PER_RED; @@ -9739,7 +10027,7 @@ init_processes_bif(void) { saved_term_procs.start = NULL; saved_term_procs.end = NULL; - processes_bif_tab_chunks = (((erts_max_processes - 1) + processes_bif_tab_chunks = (((erts_proc.max - 1) / ERTS_PROCESSES_BIF_TAB_CHUNK_SIZE) + 1); @@ -9752,10 +10040,6 @@ init_processes_bif(void) processes_trap_export.code[3] = (BeamInstr) em_apply_bif; processes_trap_export.code[4] = (BeamInstr) &processes_trap; -#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_TERM_PROC_LIST - erts_get_emu_time(&debug_tv_start); -#endif - } /* @@ -9787,31 +10071,29 @@ erts_debug_processes(Process *c_p) Eterm res; Eterm* hp; Process *p; -#ifdef DEBUG Eterm *hp_end; -#endif - erts_smp_mtx_lock(&proc_tab_mtx); + erts_smp_rwmtx_rwlock(&erts_proc_tab_rwmtx); res = NIL; need = erts_process_count() * 2; hp = HAlloc(c_p, need); /* we need two heap words for each pid */ -#ifdef DEBUG hp_end = hp + need; -#endif /* make the list by scanning bakward */ - for (i = erts_max_processes-1; i >= 0; i--) { - if ((p = process_tab[i]) != NULL) { - res = CONS(hp, process_tab[i]->id, res); + for (i = erts_proc.max-1; i >= 0; i--) { + p = erts_pix2proc(i); + if (p) { + res = CONS(hp, p->id, res); hp += 2; } } - ASSERT(hp == hp_end); - erts_smp_mtx_unlock(&proc_tab_mtx); + erts_smp_rwmtx_rwunlock(&erts_proc_tab_rwmtx); + + HRelease(c_p, hp_end, hp); return res; } @@ -9843,14 +10125,12 @@ erts_debug_processes_bif_info(Process *c_p) static void debug_processes_check_found_pid(ErtsProcessesBifData *pbdp, Eterm pid, - SysTimeval *tvp, + Uint64 ic, int pid_should_be_found) { int i; for (i = 0; i < pbdp->pid_ix; i++) { - if (pbdp->pid[i] == pid - && pbdp->debug.pid_started[i].tv_sec == tvp->tv_sec - && pbdp->debug.pid_started[i].tv_usec == tvp->tv_usec) { + if (pbdp->pid[i] == pid && pbdp->debug.pid_started[i] == ic) { ERTS_PROCS_ASSERT(pid_should_be_found); return; } @@ -9884,8 +10164,8 @@ debug_processes_save_all_pids(ErtsProcessesBifData *pbdp) pbdp->debug.correct_pids = erts_alloc(ERTS_ALC_T_PROCS_PIDS, sizeof(Eterm)*pbdp->pid_sz); - for (tix = 0, cpix = 0; tix < erts_max_processes; tix++) { - Process *rp = process_tab[tix]; + for (tix = 0, cpix = 0; tix < erts_proc.max; tix++) { + Process *rp = erts_pix2proc(tix); if (rp) { ERTS_PROCS_ASSERT(is_internal_pid(rp->id)); pbdp->debug.correct_pids[cpix++] = rp->id; @@ -9938,14 +10218,13 @@ debug_processes_verify_all_pids(ErtsProcessesBifData *pbdp) static void debug_processes_check_term_proc_list(void) { - ERTS_SMP_LC_ASSERT(erts_lc_mtx_is_locked(&proc_tab_mtx)); + ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&erts_proc_tab_rwmtx)); if (!saved_term_procs.start) ERTS_PROCS_ASSERT(!saved_term_procs.end); else { - SysTimeval tv_now; - SysTimeval *prev_xtvp = NULL; + Uint64 curr_interval = get_proc_interval(); + Uint64 *prev_x_interval_p = NULL; ErtsTermProcElement *tpep; - erts_get_emu_time(&tv_now); for (tpep = saved_term_procs.start; tpep; tpep = tpep->next) { if (!tpep->prev) @@ -9957,20 +10236,17 @@ debug_processes_check_term_proc_list(void) else ERTS_PROCS_ASSERT(tpep->next->prev == tpep); if (tpep->ix < 0) { - SysTimeval *tvp = &tpep->u.bif_invocation.time; - ERTS_PROCS_ASSERT(erts_cmp_timeval(&debug_tv_start, tvp) < 0 - && erts_cmp_timeval(tvp, &tv_now) < 0); + Uint64 interval = tpep->u.bif_invocation.interval; + ERTS_PROCS_ASSERT(interval <= curr_interval); } else { - SysTimeval *stvp = &tpep->u.process.spawned; - SysTimeval *xtvp = &tpep->u.process.exited; + Uint64 s_interval = tpep->u.process.spawned; + Uint64 x_interval = tpep->u.process.exited; - ERTS_PROCS_ASSERT(erts_cmp_timeval(&debug_tv_start, - stvp) < 0); - ERTS_PROCS_ASSERT(erts_cmp_timeval(stvp, xtvp) < 0); - if (prev_xtvp) - ERTS_PROCS_ASSERT(erts_cmp_timeval(prev_xtvp, xtvp) < 0); - prev_xtvp = xtvp; + ERTS_PROCS_ASSERT(s_interval <= x_interval); + if (prev_x_interval_p) + ERTS_PROCS_ASSERT(*prev_x_interval_p <= x_interval); + prev_x_interval_p = &tpep->u.process.exited; ERTS_PROCS_ASSERT(is_internal_pid(tpep->u.process.pid)); ERTS_PROCS_ASSERT(tpep->ix == internal_pid_index(tpep->u.process.pid)); diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index cff0783bc4..d40f50c0af 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -290,7 +290,7 @@ struct ErtsSchedulerSleepInfo_ { typedef struct ErtsProcList_ ErtsProcList; struct ErtsProcList_ { Eterm pid; - SysTimeval started; + Uint64 started_interval; ErtsProcList* next; }; @@ -600,6 +600,8 @@ struct ErtsPendingSuspend_ { # define BIN_OLD_VHEAP(p) (p)->bin_old_vheap struct process { + Eterm id; /* The pid of this process + (need to be first in struct) */ /* All fields in the PCB that differs between different heap * architectures, have been moved to the end of this struct to * make sure that as few offsets as possible differ. Different @@ -646,7 +648,6 @@ struct process { Uint32 gcstatus; /* process gc STATE */ Uint32 rstatus; /* process resume STATE */ Uint32 rcount; /* suspend count */ - Eterm id; /* The pid of this process */ int prio; /* Priority of process */ int skipped; /* Times a low prio process has been rescheduled */ Uint reds; /* No of reductions for this process */ @@ -699,8 +700,8 @@ struct process { * Information mainly for post-mortem use (erl crash dump). */ Eterm parent; /* Pid of process that created this process. */ - SysTimeval started; /* Time when started. */ - + erts_approx_time_t approx_started; /* Time when started. */ + Uint64 started_interval; /* This is the place, where all fields that differs between memory * architectures, have gone to. @@ -735,8 +736,8 @@ struct process { #ifdef ERTS_SMP erts_proc_lock_t lock; + erts_atomic32_t aflags; ErtsSchedulerData *scheduler_data; - int is_exiting; Uint32 runq_flags; Uint32 status_flags; ErlMessageInQueue msg_inq; @@ -815,6 +816,10 @@ void erts_check_for_holes(Process* p); #define SEQ_TRACE_TOKEN(p) ((p)->seq_trace_token) +#define ERTS_PROC_AFLG_EXITING (((erts_aint_t) 1) << 0) +#define ERTS_PROC_AFLG_PENDING_EXIT (((erts_aint_t) 1) << 1) + + /* The sequential tracing token is a tuple of size 5: * * {Flags, Label, Serial, Sender} @@ -887,13 +892,12 @@ Eterm* erts_heap_alloc(Process* p, Uint need, Uint xtra); Eterm* erts_set_hole_marker(Eterm* ptr, Uint sz); #endif -extern Process** process_tab; +extern erts_smp_rwmtx_t erts_proc_tab_rwmtx; +extern erts_smp_atomic_t *erts_proc_tab; #ifdef HYBRID extern Uint erts_num_active_procs; extern Process** erts_active_procs; #endif -extern Uint erts_max_processes; -extern Uint erts_process_tab_index_mask; extern Uint erts_default_process_flags; extern erts_smp_rwmtx_t erts_cpu_bind_rwmtx; /* If any of the erts_system_monitor_* variables are set (enabled), @@ -1091,6 +1095,9 @@ void erts_early_init_scheduling(int); void erts_init_scheduling(int, int); Eterm erts_sched_wall_time_request(Process *c_p, int set, int enable); +Uint64 erts_get_proc_interval(void); +Uint64 erts_ensure_later_proc_interval(Uint64); +Uint64 erts_step_proc_interval(void); ErtsProcList *erts_proclist_create(Process *); void erts_proclist_destroy(ErtsProcList *); @@ -1607,23 +1614,17 @@ Process *erts_pid2proc_nropt(Process *c_p, ErtsProcLocks pid_locks); extern int erts_disable_proc_not_running_opt; + +#define ERTS_PROC_IS_EXITING(P) \ + (ERTS_PROC_AFLG_EXITING & erts_atomic32_read_acqb(&(P)->aflags)) + #ifdef DEBUG #define ERTS_SMP_ASSERT_IS_NOT_EXITING(P) \ - do { ASSERT(!(P)->is_exiting); } while (0) + do { ASSERT(!ERTS_PROC_IS_EXITING((P))); } while (0) #else #define ERTS_SMP_ASSERT_IS_NOT_EXITING(P) #endif -/* NOTE: At least one process lock has to be held on P! */ -#ifdef ERTS_ENABLE_LOCK_CHECK -#define ERTS_PROC_IS_EXITING(P) \ - (ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks((P)) != 0 \ - || erts_lc_pix_lock_is_locked(ERTS_PID2PIXLOCK((P)->id))),\ - (P)->is_exiting) -#else -#define ERTS_PROC_IS_EXITING(P) ((P)->is_exiting) -#endif - #else /* !ERTS_SMP */ #define ERTS_PROC_IS_EXITING(P) ((P)->status == P_EXITING) diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c index 3550f1396c..37cc717156 100644 --- a/erts/emulator/beam/erl_process_dump.c +++ b/erts/emulator/beam/erl_process_dump.c @@ -65,10 +65,9 @@ erts_deep_process_dump(int to, void *to_arg) all_binaries = NULL; for (i = 0; i < erts_max_processes; i++) { - if ((process_tab[i] != NULL) && (process_tab[i]->i != ENULL)) { - if (process_tab[i]->status != P_EXITING) { - Process* p = process_tab[i]; - + Process *p = erts_pix2proc(i); + if (p && p->i != ENULL) { + if (p->status != P_EXITING) { if (p->status != P_GARBING) { dump_process_info(to, to_arg, p); } diff --git a/erts/emulator/beam/erl_process_lock.c b/erts/emulator/beam/erl_process_lock.c index 3e52a8767f..b864340782 100644 --- a/erts/emulator/beam/erl_process_lock.c +++ b/erts/emulator/beam/erl_process_lock.c @@ -67,7 +67,7 @@ #include "erl_process.h" -const Process erts_proc_lock_busy; +const Process erts_proc_lock_busy = {ERTS_INVALID_PID}; #ifdef ERTS_SMP @@ -671,12 +671,11 @@ erts_proc_lock_prepare_proc_lock_waiter(void) */ static void -proc_safelock(Process *a_proc, - erts_pix_lock_t *a_pix_lck, +proc_safelock(int is_sched, + Process *a_proc, ErtsProcLocks a_have_locks, ErtsProcLocks a_need_locks, Process *b_proc, - erts_pix_lock_t *b_pix_lck, ErtsProcLocks b_have_locks, ErtsProcLocks b_need_locks) { @@ -684,7 +683,6 @@ proc_safelock(Process *a_proc, #ifdef ERTS_ENABLE_LOCK_CHECK Eterm pid1, pid2; #endif - erts_pix_lock_t *pix_lck1, *pix_lck2; ErtsProcLocks need_locks1, have_locks1, need_locks2, have_locks2; ErtsProcLocks unlock_mask; int lock_no, refc1 = 0, refc2 = 0; @@ -701,14 +699,12 @@ proc_safelock(Process *a_proc, #ifdef ERTS_ENABLE_LOCK_CHECK pid1 = a_proc->id; #endif - pix_lck1 = a_pix_lck; need_locks1 = a_need_locks; have_locks1 = a_have_locks; p2 = b_proc; #ifdef ERTS_ENABLE_LOCK_CHECK pid2 = b_proc->id; #endif - pix_lck2 = b_pix_lck; need_locks2 = b_need_locks; have_locks2 = b_have_locks; } @@ -717,14 +713,12 @@ proc_safelock(Process *a_proc, #ifdef ERTS_ENABLE_LOCK_CHECK pid1 = b_proc->id; #endif - pix_lck1 = b_pix_lck; need_locks1 = b_need_locks; have_locks1 = b_have_locks; p2 = a_proc; #ifdef ERTS_ENABLE_LOCK_CHECK pid2 = a_proc->id; #endif - pix_lck2 = a_pix_lck; need_locks2 = a_need_locks; have_locks2 = a_have_locks; } @@ -735,14 +729,12 @@ proc_safelock(Process *a_proc, #ifdef ERTS_ENABLE_LOCK_CHECK pid1 = a_proc->id; #endif - pix_lck1 = a_pix_lck; need_locks1 = a_need_locks | b_need_locks; have_locks1 = a_have_locks | b_have_locks; p2 = NULL; #ifdef ERTS_ENABLE_LOCK_CHECK pid2 = 0; #endif - pix_lck2 = NULL; need_locks2 = 0; have_locks2 = 0; } @@ -752,14 +744,12 @@ proc_safelock(Process *a_proc, #ifdef ERTS_ENABLE_LOCK_CHECK pid1 = b_proc->id; #endif - pix_lck1 = b_pix_lck; need_locks1 = b_need_locks; have_locks1 = b_have_locks; p2 = NULL; #ifdef ERTS_ENABLE_LOCK_CHECK pid2 = 0; #endif - pix_lck2 = NULL; need_locks2 = 0; have_locks2 = 0; #ifdef ERTS_ENABLE_LOCK_CHECK @@ -807,21 +797,21 @@ proc_safelock(Process *a_proc, if (unlock_locks) { have_locks1 &= ~unlock_locks; need_locks1 |= unlock_locks; - if (!have_locks1) { + if (!is_sched && !have_locks1) { refc1 = 1; erts_smp_proc_inc_refc(p1); } - erts_smp_proc_unlock__(p1, pix_lck1, unlock_locks); + erts_smp_proc_unlock(p1, unlock_locks); } unlock_locks = unlock_mask & have_locks2; if (unlock_locks) { have_locks2 &= ~unlock_locks; need_locks2 |= unlock_locks; - if (!have_locks2) { + if (!is_sched && !have_locks2) { refc2 = 1; erts_smp_proc_inc_refc(p2); } - erts_smp_proc_unlock__(p2, pix_lck2, unlock_locks); + erts_smp_proc_unlock(p2, unlock_locks); } } @@ -852,7 +842,7 @@ proc_safelock(Process *a_proc, if (need_locks2 & lock) lock_no--; locks = need_locks1 & lock_mask; - erts_smp_proc_lock__(p1, pix_lck1, locks); + erts_smp_proc_lock(p1, locks); have_locks1 |= locks; need_locks1 &= ~locks; } @@ -863,7 +853,7 @@ proc_safelock(Process *a_proc, lock = (1 << ++lock_no); } locks = need_locks2 & lock_mask; - erts_smp_proc_lock__(p2, pix_lck2, locks); + erts_smp_proc_lock(p2, locks); have_locks2 |= locks; need_locks2 &= ~locks; } @@ -898,10 +888,12 @@ proc_safelock(Process *a_proc, } #endif - if (refc1) - erts_smp_proc_dec_refc(p1); - if (refc2) - erts_smp_proc_dec_refc(p2); + if (!is_sched) { + if (refc1) + erts_smp_proc_dec_refc(p1); + if (refc2) + erts_smp_proc_dec_refc(p2); + } } void @@ -912,72 +904,185 @@ erts_proc_safelock(Process *a_proc, ErtsProcLocks b_have_locks, ErtsProcLocks b_need_locks) { - proc_safelock(a_proc, - a_proc ? ERTS_PID2PIXLOCK(a_proc->id) : NULL, + proc_safelock(erts_get_scheduler_id() != 0, + a_proc, a_have_locks, a_need_locks, b_proc, - b_proc ? ERTS_PID2PIXLOCK(b_proc->id) : NULL, b_have_locks, b_need_locks); } -/* - * erts_pid2proc_safelock() is called from erts_pid2proc_opt() when - * it wasn't possible to trylock all locks needed. - * c_p - current process - * c_p_have_locks - locks held on c_p - * pid - process id of process we are looking up - * proc - process struct of process we are looking - * up (both in and out argument) - * need_locks - all locks we need (including have_locks) - * pix_lock - pix lock for process we are looking up - * flags - option flags - */ -void -erts_pid2proc_safelock(Process *c_p, - ErtsProcLocks c_p_have_locks, - Process **proc, - ErtsProcLocks need_locks, - erts_pix_lock_t *pix_lock, - int flags) +Process * +erts_pid2proc_opt(Process *c_p, + ErtsProcLocks c_p_have_locks, + Eterm pid, + ErtsProcLocks pid_need_locks, + int flags) { - Process *p = *proc; - ERTS_LC_ASSERT(p->lock.refc > 0); - ERTS_LC_ASSERT(process_tab[internal_pid_index(p->id)] == p); - p->lock.refc++; - erts_pix_unlock(pix_lock); - - proc_safelock(c_p, - c_p ? ERTS_PID2PIXLOCK(c_p->id) : NULL, - c_p_have_locks, - c_p_have_locks, - p, - pix_lock, - 0, - need_locks); + Process *dec_refc_proc = NULL; + int need_ptl; + ErtsProcLocks need_locks; + Uint pix; + Process *proc; +#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT) + ErtsProcLocks lcnt_locks; +#endif - erts_pix_lock(pix_lock); +#ifdef ERTS_ENABLE_LOCK_CHECK + if (c_p) { + ErtsProcLocks might_unlock = c_p_have_locks & pid_need_locks; + if (might_unlock) + erts_proc_lc_might_unlock(c_p, might_unlock); + } +#endif - if (!p->is_exiting - || ((flags & ERTS_P2P_FLG_ALLOW_OTHER_X) - && process_tab[internal_pid_index(p->id)] == p)) { - ERTS_LC_ASSERT(p->lock.refc > 1); - p->lock.refc--; + if (is_not_internal_pid(pid)) + return NULL; + pix = internal_pid_index(pid); + + ERTS_LC_ASSERT((pid_need_locks & ERTS_PROC_LOCKS_ALL) == pid_need_locks); + need_locks = pid_need_locks; + + if (c_p && c_p->id == pid) { + ASSERT(c_p->id != ERTS_INVALID_PID); + ASSERT(c_p == erts_pix2proc(pix)); + + if (!(flags & ERTS_P2P_FLG_ALLOW_OTHER_X) + && ERTS_PROC_IS_EXITING(c_p)) + return NULL; + need_locks &= ~c_p_have_locks; + if (!need_locks) { + if (flags & ERTS_P2P_FLG_SMP_INC_REFC) + erts_smp_proc_inc_refc(c_p); + return c_p; + } } - else { - /* No proc. Note, we need to keep refc until after process unlock */ - erts_pix_unlock(pix_lock); - erts_smp_proc_unlock__(p, pix_lock, need_locks); - *proc = NULL; - erts_pix_lock(pix_lock); - ERTS_LC_ASSERT(p->lock.refc > 0); - if (--p->lock.refc == 0) { - erts_pix_unlock(pix_lock); - erts_free_proc(p); - erts_pix_lock(pix_lock); + + need_ptl = !erts_get_scheduler_id(); + + if (need_ptl) + erts_smp_rwmtx_rwlock(&erts_proc_tab_rwmtx); + + proc = (Process *) erts_smp_atomic_read_ddrb(&erts_proc.tab[pix]); + + if (proc) { + if (proc->id != pid) + proc = NULL; + else if (!need_locks) { + if (flags & ERTS_P2P_FLG_SMP_INC_REFC) + erts_smp_proc_inc_refc(proc); } + else { + int busy; + +#if ERTS_PROC_LOCK_OWN_IMPL +#ifdef ERTS_ENABLE_LOCK_COUNT + lcnt_locks = need_locks; + if (!(flags & ERTS_P2P_FLG_TRY_LOCK)) { + erts_lcnt_proc_lock(&proc->lock, need_locks); + } +#endif + +#ifdef ERTS_ENABLE_LOCK_CHECK + /* Make sure erts_pid2proc_safelock() is enough to handle + a potential lock order violation situation... */ + busy = erts_proc_lc_trylock_force_busy(proc, need_locks); + if (!busy) +#endif +#endif /* ERTS_PROC_LOCK_OWN_IMPL */ + { + /* Try a quick trylock to grab all the locks we need. */ + busy = (int) erts_smp_proc_raw_trylock__(proc, need_locks); + +#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_CHECK) + erts_proc_lc_trylock(proc, need_locks, !busy); +#endif +#ifdef ERTS_PROC_LOCK_DEBUG + if (!busy) + erts_proc_lock_op_debug(proc, need_locks, 1); +#endif + } + +#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT) + if (flags & ERTS_P2P_FLG_TRY_LOCK) + erts_lcnt_proc_trylock(&proc->lock, need_locks, + busy ? EBUSY : 0); +#endif + + if (!busy) { + if (flags & ERTS_P2P_FLG_SMP_INC_REFC) + erts_smp_proc_inc_refc(proc); + +#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT) + /* all is great */ + if (!(flags & ERTS_P2P_FLG_TRY_LOCK)) + erts_lcnt_proc_lock_post_x(&proc->lock, lcnt_locks, + __FILE__, __LINE__); +#endif + + } + else { + if (flags & ERTS_P2P_FLG_TRY_LOCK) + proc = ERTS_PROC_LOCK_BUSY; + else { + if (flags & ERTS_P2P_FLG_SMP_INC_REFC) + erts_smp_proc_inc_refc(proc); + +#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT) + erts_lcnt_proc_lock_unaquire(&proc->lock, lcnt_locks); +#endif + + if (need_ptl) { + erts_smp_proc_inc_refc(proc); + dec_refc_proc = proc; + erts_smp_rwmtx_rwunlock(&erts_proc_tab_rwmtx); + need_ptl = 0; + } + + proc_safelock(!need_ptl, + c_p, + c_p_have_locks, + c_p_have_locks, + proc, + 0, + need_locks); + } + } + } } + + if (need_ptl) + erts_smp_rwmtx_rwunlock(&erts_proc_tab_rwmtx); + + if (need_locks + && proc + && proc != ERTS_PROC_LOCK_BUSY + && (!(flags & ERTS_P2P_FLG_ALLOW_OTHER_X) + ? ERTS_PROC_IS_EXITING(proc) + : (proc + != (Process *) erts_smp_atomic_read_nob(&erts_proc.tab[pix])))) { + + erts_smp_proc_unlock(proc, need_locks); + + if (flags & ERTS_P2P_FLG_SMP_INC_REFC) + dec_refc_proc = proc; + proc = NULL; + + } + + if (dec_refc_proc) + erts_smp_proc_dec_refc(dec_refc_proc); + +#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_PROC_LOCK_DEBUG) + ERTS_LC_ASSERT(!proc + || proc == ERTS_PROC_LOCK_BUSY + || (pid_need_locks == + (ERTS_PROC_LOCK_FLGS_READ_(&proc->lock) + & pid_need_locks))); +#endif + + return proc; } void @@ -1017,7 +1122,7 @@ erts_proc_lock_init(Process *p) erts_lc_trylock(1, &p->lock.status.lc); #endif #endif - p->lock.refc = 1; + erts_atomic32_init_nob(&p->lock.refc, 1); #ifdef ERTS_PROC_LOCK_DEBUG { int i; @@ -1590,4 +1695,4 @@ check_queue(erts_proc_lock_t *lck) } #endif -#endif /* ERTS_SMP (the whole file) */ +#endif /* ERTS_SMP */ diff --git a/erts/emulator/beam/erl_process_lock.h b/erts/emulator/beam/erl_process_lock.h index ee341bd951..9981af7a8a 100644 --- a/erts/emulator/beam/erl_process_lock.h +++ b/erts/emulator/beam/erl_process_lock.h @@ -93,7 +93,7 @@ typedef struct erts_proc_lock_t_ { #else # error "no implementation" #endif - Sint32 refc; + erts_atomic32_t refc; #ifdef ERTS_PROC_LOCK_DEBUG erts_smp_atomic32_t locked[ERTS_PROC_LOCK_MAX_BIT+1]; #endif @@ -171,8 +171,6 @@ typedef struct erts_proc_lock_t_ { * are guaranteed that the process won't exit until the lock you are * holding has been released. Appart from all process locks also * the pix lock corresponding to the process has to be held. - * At the same time as status is changed to P_EXITING, also the - * field 'is_exiting' in the process structure is set to a value != 0. * * Lock order: * Process locks with low numeric values has to be locked before @@ -282,12 +280,8 @@ typedef struct { } u; } erts_pix_lock_t; -#define ERTS_PIX2PIXLOCKIX(PIX) \ - ((PIX) & ((1 << ERTS_PIX_LOCKS_BITS) - 1)) -#define ERTS_PIX2PIXLOCK(PIX) \ - (&erts_pix_locks[ERTS_PIX2PIXLOCKIX((PIX))]) #define ERTS_PID2PIXLOCK(PID) \ - ERTS_PIX2PIXLOCK(internal_pid_data((PID))) + (&erts_pix_locks[(internal_pid_data((PID)) & ((1 << ERTS_PIX_LOCKS_BITS) - 1))]) #if ERTS_PROC_LOCK_OWN_IMPL @@ -820,44 +814,36 @@ erts_smp_proc_trylock(Process *p, ErtsProcLocks locks) #endif } - ERTS_GLB_INLINE void erts_smp_proc_inc_refc(Process *p) { #ifdef ERTS_SMP - erts_pix_lock_t *pixlck = ERTS_PID2PIXLOCK(p->id); - erts_pix_lock(pixlck); - ERTS_LC_ASSERT(p->lock.refc > 0); - p->lock.refc++; - erts_pix_unlock(pixlck); +#ifdef ERTS_ENABLE_LOCK_CHECK + erts_aint32_t refc = erts_atomic32_inc_read_nob(&p->lock.refc); + ERTS_SMP_LC_ASSERT(refc > 1); +#else + erts_atomic32_inc_nob(&p->lock.refc); +#endif #endif } ERTS_GLB_INLINE void erts_smp_proc_dec_refc(Process *p) { #ifdef ERTS_SMP - Process *fp; - erts_pix_lock_t *pixlck = ERTS_PID2PIXLOCK(p->id); - erts_pix_lock(pixlck); - ERTS_LC_ASSERT(p->lock.refc > 0); - fp = --p->lock.refc == 0 ? p : NULL; - erts_pix_unlock(pixlck); - if (fp) - erts_free_proc(fp); + erts_aint32_t refc = erts_atomic32_dec_read_nob(&p->lock.refc); + ERTS_SMP_LC_ASSERT(refc >= 0); + if (refc == 0) + erts_free_proc(p); #endif } -ERTS_GLB_INLINE void erts_smp_proc_add_refc(Process *p, Sint32 refc) +ERTS_GLB_INLINE void erts_smp_proc_add_refc(Process *p, Sint32 add_refc) { #ifdef ERTS_SMP - Process *fp; - erts_pix_lock_t *pixlck = ERTS_PID2PIXLOCK(p->id); - erts_pix_lock(pixlck); - ERTS_LC_ASSERT(p->lock.refc > 0); - p->lock.refc += refc; - fp = p->lock.refc == 0 ? p : NULL; - erts_pix_unlock(pixlck); - if (fp) - erts_free_proc(fp); + erts_aint32_t refc = erts_atomic32_add_read_nob(&p->lock.refc, + (erts_aint32_t) add_refc); + ERTS_SMP_LC_ASSERT(refc >= 0); + if (refc == 0) + erts_free_proc(p); #endif } @@ -898,218 +884,75 @@ extern const Process erts_proc_lock_busy; #define erts_pid2proc(PROC, HL, PID, NL) \ erts_pid2proc_opt((PROC), (HL), (PID), (NL), 0) -ERTS_GLB_INLINE Process * -erts_pid2proc_opt(Process *, ErtsProcLocks, Eterm, ErtsProcLocks, int); -#ifdef ERTS_SMP -void -erts_pid2proc_safelock(Process *c_p, - ErtsProcLocks c_p_have_locks, - Process **proc, - ErtsProcLocks need_locks, - erts_pix_lock_t *pix_lock, - int flags); -ERTS_GLB_INLINE Process *erts_pid2proc_unlocked_opt(Eterm pid, int flags); -#define erts_pid2proc_unlocked(PID) erts_pid2proc_unlocked_opt((PID), 0) -#else -#define erts_pid2proc_unlocked_opt(PID, FLGS) \ - erts_pid2proc_opt(NULL, 0, (PID), 0, FLGS) -#define erts_pid2proc_unlocked(PID) erts_pid2proc_opt(NULL, 0, (PID), 0, 0) +ERTS_GLB_INLINE Process *erts_pix2proc(int ix); +ERTS_GLB_INLINE Process *erts_proc_lookup_raw(Eterm pid); +ERTS_GLB_INLINE Process *erts_proc_lookup(Eterm pid); + +#ifndef ERTS_SMP +ERTS_GLB_INLINE #endif +Process *erts_pid2proc_opt(Process *, ErtsProcLocks, Eterm, ErtsProcLocks, int); #if ERTS_GLB_INLINE_INCL_FUNC_DEF -ERTS_GLB_INLINE Process * -#ifdef ERTS_SMP -erts_pid2proc_unlocked_opt(Eterm pid, int flags) -#else -erts_pid2proc_opt(Process *c_p_unused, - ErtsProcLocks c_p_have_locks_unused, - Eterm pid, - ErtsProcLocks pid_need_locks_unused, - int flags) -#endif +ERTS_GLB_INLINE Process *erts_pix2proc(int ix) { - Uint pix; Process *proc; + ASSERT(0 <= ix && ix < erts_proc.max); + proc = (Process *) erts_smp_atomic_read_nob(&erts_proc.tab[ix]); + return proc == ERTS_PROC_LOCK_BUSY ? NULL : proc; +} + +ERTS_GLB_INLINE Process *erts_proc_lookup_raw(Eterm pid) +{ + Process *proc; + int pix; + + /* + * In SMP case: Only scheduler threads are allowed + * to use this function. Other threads need to + * atomicaly increment refc at lookup, i.e., use + * erts_pid2proc_opt() with ERTS_P2P_FLG_SMP_INC_REFC. + */ + ERTS_SMP_LC_ASSERT(erts_get_scheduler_id()); if (is_not_internal_pid(pid)) return NULL; pix = internal_pid_index(pid); - if(pix >= erts_max_processes) + + proc = (Process *) erts_smp_atomic_read_ddrb(&erts_proc.tab[pix]); + + if (proc && proc->id != pid) return NULL; - proc = process_tab[pix]; - if (proc) { - if (proc->id != pid - || (!(flags & ERTS_P2P_FLG_ALLOW_OTHER_X) - && proc->status == P_EXITING)) - proc = NULL; - } + return proc; } -#ifdef ERTS_SMP +ERTS_GLB_INLINE Process *erts_proc_lookup(Eterm pid) +{ + Process *proc = erts_proc_lookup_raw(pid); + if (proc && ERTS_PROC_IS_EXITING(proc)) + return NULL; + return proc; +} +#ifndef ERTS_SMP ERTS_GLB_INLINE Process * -erts_pid2proc_opt(Process *c_p, - ErtsProcLocks c_p_have_locks, +erts_pid2proc_opt(Process *c_p_unused, + ErtsProcLocks c_p_have_locks_unused, Eterm pid, - ErtsProcLocks pid_need_locks, + ErtsProcLocks pid_need_locks_unused, int flags) { - erts_pix_lock_t *pix_lock; - ErtsProcLocks need_locks; - Uint pix; - Process *proc; -#if ERTS_PROC_LOCK_OWN_IMPL -#ifdef ERTS_ENABLE_LOCK_COUNT - ErtsProcLocks lcnt_locks; -#endif -#endif /* ERTS_PROC_LOCK_OWN_IMPL */ - -#ifdef ERTS_ENABLE_LOCK_CHECK - if (c_p) { - ErtsProcLocks might_unlock = c_p_have_locks & pid_need_locks; - if (might_unlock) - erts_proc_lc_might_unlock(c_p, might_unlock); - } -#endif - - if (is_not_internal_pid(pid)) { - proc = NULL; - goto done; - } - pix = internal_pid_index(pid); - if(pix >= erts_max_processes) { - proc = NULL; - goto done; - } - - ERTS_LC_ASSERT((pid_need_locks & ERTS_PROC_LOCKS_ALL) == pid_need_locks); - need_locks = pid_need_locks; - - pix_lock = ERTS_PIX2PIXLOCK(pix); - - if (c_p && c_p->id == pid) { - ASSERT(c_p->id != ERTS_INVALID_PID); - ASSERT(c_p == process_tab[pix]); - if (!(flags & ERTS_P2P_FLG_ALLOW_OTHER_X) && c_p->is_exiting) { - proc = NULL; - goto done; - } - need_locks &= ~c_p_have_locks; - if (!need_locks) { - proc = c_p; - erts_pix_lock(pix_lock); - if (flags & ERTS_P2P_FLG_SMP_INC_REFC) - proc->lock.refc++; - erts_pix_unlock(pix_lock); - goto done; - } - } - - erts_pix_lock(pix_lock); - - proc = process_tab[pix]; - if (proc) { - if (proc->id != pid || (!(flags & ERTS_P2P_FLG_ALLOW_OTHER_X) - && ERTS_PROC_IS_EXITING(proc))) { - proc = NULL; - } - else if (!need_locks) { - if (flags & ERTS_P2P_FLG_SMP_INC_REFC) - proc->lock.refc++; - } - else { - int busy; - -#if ERTS_PROC_LOCK_OWN_IMPL -#ifdef ERTS_ENABLE_LOCK_COUNT - lcnt_locks = need_locks; - if (!(flags & ERTS_P2P_FLG_TRY_LOCK)) { - erts_lcnt_proc_lock(&proc->lock, need_locks); - } -#endif - -#ifdef ERTS_ENABLE_LOCK_CHECK - /* Make sure erts_pid2proc_safelock() is enough to handle - a potential lock order violation situation... */ - busy = erts_proc_lc_trylock_force_busy(proc, need_locks); - if (!busy) -#endif -#endif /* ERTS_PROC_LOCK_OWN_IMPL */ - { - /* Try a quick trylock to grab all the locks we need. */ - busy = (int) erts_smp_proc_raw_trylock__(proc, need_locks); -#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_CHECK) - erts_proc_lc_trylock(proc, need_locks, !busy); -#endif -#ifdef ERTS_PROC_LOCK_DEBUG - if (!busy) - erts_proc_lock_op_debug(proc, need_locks, 1); -#endif - } - -#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT) - if (flags & ERTS_P2P_FLG_TRY_LOCK) { - if (busy) { - erts_lcnt_proc_trylock(&proc->lock, need_locks, EBUSY); - } else { - erts_lcnt_proc_trylock(&proc->lock, need_locks, 0); - } - } -#endif - if (!busy) { - if (flags & ERTS_P2P_FLG_SMP_INC_REFC) - proc->lock.refc++; -#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT) - /* all is great */ - if (!(flags & ERTS_P2P_FLG_TRY_LOCK)) { - erts_lcnt_proc_lock_post_x(&proc->lock, lcnt_locks, __FILE__, __LINE__); - } -#endif - } - else { - if (flags & ERTS_P2P_FLG_TRY_LOCK) - proc = ERTS_PROC_LOCK_BUSY; - else { - if (flags & ERTS_P2P_FLG_SMP_INC_REFC) - proc->lock.refc++; -#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT) - erts_lcnt_proc_lock_unaquire(&proc->lock, lcnt_locks); -#endif - erts_pid2proc_safelock(c_p, - c_p_have_locks, - &proc, - pid_need_locks, - pix_lock, - flags); - if (proc && (flags & ERTS_P2P_FLG_SMP_INC_REFC)) - proc->lock.refc++; - } - } - } - } - - erts_pix_unlock(pix_lock); -#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_PROC_LOCK_DEBUG) - ERTS_LC_ASSERT(!proc - || proc == ERTS_PROC_LOCK_BUSY - || (pid_need_locks == - (ERTS_PROC_LOCK_FLGS_READ_(&proc->lock) - & pid_need_locks))); -#endif - - - done: - -#if ERTS_PROC_LOCK_ATOMIC_IMPL - ETHR_COMPILER_BARRIER; -#endif - + Process *proc = erts_proc_lookup_raw(pid); + if (!(flags & ERTS_P2P_FLG_ALLOW_OTHER_X) + && proc + && proc->status == P_EXITING) + return NULL; return proc; } -#endif /* ERTS_SMP */ +#endif /* !ERTS_SMP */ #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h index ee47c98009..17628286bc 100644 --- a/erts/emulator/beam/erl_threads.h +++ b/erts/emulator/beam/erl_threads.h @@ -258,10 +258,6 @@ #include "sys.h" -typedef struct { SWord sint[2]; } erts_no_dw_atomic_t; -typedef SWord erts_no_atomic_t; -typedef Sint32 erts_no_atomic32_t; - #ifdef USE_THREADS #define ETHR_TRY_INLINE_FUNCS @@ -411,12 +407,15 @@ typedef struct { typedef int erts_rwmtx_t; typedef int erts_tsd_key_t; typedef int erts_tse_t; -#define erts_dw_aint_t erts_no_dw_atomic_t -#define erts_dw_atomic_t erts_no_dw_atomic_t -#define erts_aint_t SWord -#define erts_atomic_t erts_no_atomic_t -#define erts_aint32_t Sint32 -#define erts_atomic32_t erts_no_atomic32_t + +typedef struct { SWord sint[2]; } erts_dw_aint_t; +typedef SWord erts_aint_t; +typedef Sint32 erts_aint32_t; + +#define erts_dw_atomic_t erts_dw_aint_t +#define erts_atomic_t erts_aint_t +#define erts_atomic32_t erts_aint32_t + #if __GNUC__ > 2 typedef struct { } erts_spinlock_t; typedef struct { } erts_rwlock_t; @@ -425,6 +424,14 @@ typedef struct { int gcc_is_buggy; } erts_spinlock_t; typedef struct { int gcc_is_buggy; } erts_rwlock_t; #endif +#ifdef WORDS_BIGENDIAN +#define ERTS_DW_AINT_LOW_WORD 1 +#define ERTS_DW_AINT_HIGH_WORD 0 +#else +#define ERTS_DW_AINT_LOW_WORD 0 +#define ERTS_DW_AINT_HIGH_WORD 1 +#endif + #define ERTS_MTX_INITER 0 #define ERTS_CND_INITER 0 #define ERTS_THR_INIT_DATA_DEF_INITER 0 @@ -433,6 +440,10 @@ typedef struct { int gcc_is_buggy; } erts_rwlock_t; #endif /* #ifdef USE_THREADS */ +#define erts_no_dw_atomic_t erts_dw_aint_t +#define erts_no_atomic_t erts_aint_t +#define erts_no_atomic32_t erts_aint32_t + #define ERTS_AINT_NULL ((erts_aint_t) NULL) #define ERTS_AINT_T_MAX (~(((erts_aint_t) 1) << (sizeof(erts_aint_t)*8-1))) diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h index 6c6e193818..4bbdcaa3e3 100644 --- a/erts/emulator/beam/erl_time.h +++ b/erts/emulator/beam/erl_time.h @@ -118,9 +118,11 @@ ERTS_GLB_INLINE void erts_do_time_add(erts_short_time_t elapsed) void erts_get_now_cpu(Uint* megasec, Uint* sec, Uint* microsec); #endif +typedef UWord erts_approx_time_t; +erts_approx_time_t erts_get_approx_time(void); + void erts_get_timeval(SysTimeval *tv); erts_time_t erts_get_time(void); -void erts_get_emu_time(SysTimeval *); ERTS_GLB_INLINE int erts_cmp_timeval(SysTimeval *t1p, SysTimeval *t2p); diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c index 1d0735aa99..c1b4408c06 100644 --- a/erts/emulator/beam/erl_time_sup.c +++ b/erts/emulator/beam/erl_time_sup.c @@ -91,6 +91,41 @@ static SysTimeval then; /* Used in get_now */ static SysTimeval last_emu_time; /* Used in erts_get_emu_time() */ SysTimeval erts_first_emu_time; /* Used in erts_get_emu_time() */ +union { + erts_smp_atomic_t time; + char align[ERTS_CACHE_LINE_SIZE]; +} approx erts_align_attribute(ERTS_CACHE_LINE_SIZE); + +static void +init_approx_time(void) +{ + erts_smp_atomic_init_nob(&approx.time, 0); +} + +static ERTS_INLINE erts_approx_time_t +get_approx_time(void) +{ + return (erts_approx_time_t) erts_smp_atomic_read_nob(&approx.time); +} + +static ERTS_INLINE void +update_approx_time(SysTimeval *tv) +{ + erts_approx_time_t new_secs = (erts_approx_time_t) tv->tv_sec; + erts_approx_time_t old_secs = get_approx_time(); + if (old_secs != new_secs) + erts_smp_atomic_set_nob(&approx.time, new_secs); +} + +/* + * erts_get_approx_time() returns an *approximate* time + * in seconds. NOTE that this time may jump backwards!!! + */ +erts_approx_time_t +erts_get_approx_time(void) +{ + return get_approx_time(); +} #ifdef HAVE_GETHRTIME @@ -398,6 +433,8 @@ erts_init_time_sup(void) { erts_smp_mtx_init(&erts_timeofday_mtx, "timeofday"); + init_approx_time(); + last_emu_time.tv_sec = 0; last_emu_time.tv_usec = 0; @@ -417,7 +454,7 @@ erts_init_time_sup(void) gtv = inittv; then.tv_sec = then.tv_usec = 0; - erts_get_emu_time(&erts_first_emu_time); + erts_deliver_time(); return CLOCK_RESOLUTION; } @@ -873,6 +910,8 @@ get_now(Uint* megasec, Uint* sec, Uint* microsec) *megasec = (Uint) (now.tv_sec / 1000000); *sec = (Uint) (now.tv_sec % 1000000); *microsec = (Uint) (now.tv_usec); + + update_approx_time(&now); } void @@ -885,6 +924,8 @@ get_sys_now(Uint* megasec, Uint* sec, Uint* microsec) *megasec = (Uint) (now.tv_sec / 1000000); *sec = (Uint) (now.tv_sec % 1000000); *microsec = (Uint) (now.tv_usec); + + update_approx_time(&now); } @@ -901,6 +942,8 @@ void erts_deliver_time(void) { do_erts_deliver_time(&now); erts_smp_mtx_unlock(&erts_timeofday_mtx); + + update_approx_time(&now); } /* get *real* time (not ticks) remaining until next timeout - if there @@ -949,6 +992,7 @@ void erts_get_timeval(SysTimeval *tv) erts_smp_mtx_lock(&erts_timeofday_mtx); get_tolerant_timeofday(tv); erts_smp_mtx_unlock(&erts_timeofday_mtx); + update_approx_time(tv); } erts_time_t @@ -961,7 +1005,9 @@ erts_get_time(void) get_tolerant_timeofday(&sys_tv); erts_smp_mtx_unlock(&erts_timeofday_mtx); - + + update_approx_time(&sys_tv); + return sys_tv.tv_sec; } @@ -977,38 +1023,3 @@ void erts_get_now_cpu(Uint* megasec, Uint* sec, Uint* microsec) { *sec = (Uint)(tp.tv_sec % 1000000); } #endif - - -/* - * erts_get_emu_time() is similar to get_now(). You will - * always get different times from erts_get_emu_time(), but they - * may equal a time from get_now(). - * - * erts_get_emu_time() is only used internally in the emulator in - * order to order emulator internal events. - */ - -void -erts_get_emu_time(SysTimeval *this_emu_time_p) -{ - erts_smp_mtx_lock(&erts_timeofday_mtx); - - get_tolerant_timeofday(this_emu_time_p); - - /* Make sure time is later than last */ - if (last_emu_time.tv_sec > this_emu_time_p->tv_sec || - (last_emu_time.tv_sec == this_emu_time_p->tv_sec - && last_emu_time.tv_usec >= this_emu_time_p->tv_usec)) { - *this_emu_time_p = last_emu_time; - this_emu_time_p->tv_usec++; - } - /* Check for carry from above + general reasonability */ - if (this_emu_time_p->tv_usec >= 1000000) { - this_emu_time_p->tv_usec = 0; - this_emu_time_p->tv_sec++; - } - - last_emu_time = *this_emu_time_p; - - erts_smp_mtx_unlock(&erts_timeofday_mtx); -} diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index 009ca1eb52..a321bc504b 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -151,8 +151,8 @@ do { (RES) = (TPID); } while(0) #define ERTS_TRACER_REF_TYPE Process * #define ERTS_GET_TRACER_REF(RES, TPID, TRACEE_FLGS) \ do { \ - (RES) = process_tab[internal_pid_index((TPID))]; \ - if (INVALID_PID((RES), (TPID)) || !((RES)->trace_flags & F_TRACER)) { \ + (RES) = erts_proc_lookup((TPID)); \ + if (!(RES) || !((RES)->trace_flags & F_TRACER)) { \ (TPID) = NIL; \ (TRACEE_FLGS) &= ~TRACEE_FLAGS; \ return; \ @@ -211,7 +211,7 @@ erts_set_system_seq_tracer(Process *c_p, ErtsProcLocks c_p_locks, Eterm new) Eterm old = THE_NON_VALUE; if (new != am_false) { - if (!erts_pid2proc(c_p, c_p_locks, new, 0) + if (!erts_proc_lookup(new) && !erts_is_valid_tracer_port(new)) { return old; } @@ -250,7 +250,7 @@ get_default_tracing(Uint *flagsp, Eterm *tracerp) if (is_nil(default_tracer)) { default_trace_flags &= ~TRACEE_FLAGS; } else if (is_internal_pid(default_tracer)) { - if (!erts_pid2proc(NULL, 0, default_tracer, 0)) { + if (!erts_proc_lookup(default_tracer)) { reset_tracer: default_trace_flags &= ~TRACEE_FLAGS; default_tracer = NIL; @@ -580,9 +580,10 @@ profile_send(Eterm from, Eterm message) { ASSERT(is_internal_pid(profiler) && internal_pid_index(profiler) < erts_max_processes); - profile_p = process_tab[internal_pid_index(profiler)]; + profile_p = erts_proc_lookup(profiler); - if (INVALID_PID(profile_p, profiler)) return; + if (!profile_p) + return; sz = size_object(message); hp = erts_alloc_message_heap(sz, &bp, &off_heap, profile_p, 0); @@ -874,7 +875,7 @@ trace_send(Process *p, Eterm to, Eterm msg) operation = am_send; if (is_internal_pid(to)) { - if (!erts_pid2proc(p, ERTS_PROC_LOCK_MAIN, to, 0)) + if (!erts_proc_lookup(to)) goto send_to_non_existing_process; } else if(is_external_pid(to) @@ -1116,8 +1117,8 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type, #ifndef ERTS_SMP - tracer = process_tab[internal_pid_index(seq_tracer)]; - if (INVALID_PID(tracer, tracer->id)) { + tracer = erts_proc_lookup(seq_tracer); + if (!tracer) { system_seq_tracer = am_false; return; /* no need to send anything */ } @@ -2451,10 +2452,9 @@ monitor_long_gc(Process *p, Uint time) { #ifndef ERTS_SMP ASSERT(is_internal_pid(system_monitor) && internal_pid_index(system_monitor) < erts_max_processes); - monitor_p = process_tab[internal_pid_index(system_monitor)]; - if (INVALID_PID(monitor_p, system_monitor) || p == monitor_p) { + monitor_p = erts_proc_lookup(system_monitor); + if (!monitor_p || p == monitor_p) return; - } #endif hsz = 0; @@ -2527,8 +2527,8 @@ monitor_large_heap(Process *p) { #ifndef ERTS_SMP ASSERT(is_internal_pid(system_monitor) && internal_pid_index(system_monitor) < erts_max_processes); - monitor_p = process_tab[internal_pid_index(system_monitor)]; - if (INVALID_PID(monitor_p, system_monitor) || p == monitor_p) { + monitor_p = erts_proc_lookup(system_monitor); + if (monitor_p || p == monitor_p) { return; } #endif @@ -2582,10 +2582,9 @@ monitor_generic(Process *p, Eterm type, Eterm spec) { #ifndef ERTS_SMP ASSERT(is_internal_pid(system_monitor) && internal_pid_index(system_monitor) < erts_max_processes); - monitor_p = process_tab[internal_pid_index(system_monitor)]; - if (INVALID_PID(monitor_p, system_monitor) || p == monitor_p) { + monitor_p = erts_proc_lookup(system_monitor); + if (!monitor_p || p == monitor_p) return; - } #endif hp = ERTS_ALLOC_SYSMSG_HEAP(5, &bp, &off_heap, monitor_p); diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index b000e2c5d4..ded9646800 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -1471,6 +1471,122 @@ void erl_drv_thr_init(void); /* utils.c */ +typedef struct { +#ifdef DEBUG + int smp_api; +#endif + union { + Uint64 not_atomic; +#ifdef ARCH_64 + erts_atomic_t atomic; +#else + erts_dw_atomic_t atomic; +#endif + } counter; +} erts_interval_t; + +void erts_interval_init(erts_interval_t *); +void erts_smp_interval_init(erts_interval_t *); +Uint64 erts_step_interval_nob(erts_interval_t *); +Uint64 erts_step_interval_relb(erts_interval_t *); +Uint64 erts_smp_step_interval_nob(erts_interval_t *); +Uint64 erts_smp_step_interval_relb(erts_interval_t *); +Uint64 erts_ensure_later_interval_nob(erts_interval_t *, Uint64); +Uint64 erts_ensure_later_interval_acqb(erts_interval_t *, Uint64); +Uint64 erts_smp_ensure_later_interval_nob(erts_interval_t *, Uint64); +Uint64 erts_smp_ensure_later_interval_acqb(erts_interval_t *, Uint64); +#ifdef ARCH_32 +ERTS_GLB_INLINE Uint64 erts_interval_dw_aint_to_val__(erts_dw_aint_t *); +#endif +ERTS_GLB_INLINE Uint64 erts_current_interval_nob__(erts_interval_t *); +ERTS_GLB_INLINE Uint64 erts_current_interval_acqb__(erts_interval_t *); +ERTS_GLB_INLINE Uint64 erts_current_interval_nob(erts_interval_t *); +ERTS_GLB_INLINE Uint64 erts_current_interval_acqb(erts_interval_t *); +ERTS_GLB_INLINE Uint64 erts_smp_current_interval_nob(erts_interval_t *); +ERTS_GLB_INLINE Uint64 erts_smp_current_interval_acqb(erts_interval_t *); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +#ifdef ARCH_32 + +ERTS_GLB_INLINE Uint64 +erts_interval_dw_aint_to_val__(erts_dw_aint_t *dw) +{ +#ifdef ETHR_SU_DW_NAINT_T__ + return (Uint64) dw->dw_sint; +#else + Uint64 res; + res = (Uint64) ((Uint32) dw->sint[ERTS_DW_AINT_HIGH_WORD]); + res <<= 32; + res |= (Uint64) ((Uint32) dw->sint[ERTS_DW_AINT_LOW_WORD]); + return res; +#endif +} + +#endif + +ERTS_GLB_INLINE Uint64 +erts_current_interval_nob__(erts_interval_t *icp) +{ +#ifdef ARCH_64 + return (Uint64) erts_atomic_read_nob(&icp->counter.atomic); +#else + erts_dw_aint_t dw; + erts_dw_atomic_read_nob(&icp->counter.atomic, &dw); + return erts_interval_dw_aint_to_val__(&dw); +#endif +} + +ERTS_GLB_INLINE Uint64 +erts_current_interval_acqb__(erts_interval_t *icp) +{ +#ifdef ARCH_64 + return (Uint64) erts_atomic_read_acqb(&icp->counter.atomic); +#else + erts_dw_aint_t dw; + erts_dw_atomic_read_acqb(&icp->counter.atomic, &dw); + return erts_interval_dw_aint_to_val__(&dw); +#endif +} + +ERTS_GLB_INLINE Uint64 +erts_current_interval_nob(erts_interval_t *icp) +{ + ASSERT(!icp->smp_api); + return erts_current_interval_nob__(icp); +} + +ERTS_GLB_INLINE Uint64 +erts_current_interval_acqb(erts_interval_t *icp) +{ + ASSERT(!icp->smp_api); + return erts_current_interval_acqb__(icp); +} + +ERTS_GLB_INLINE Uint64 +erts_smp_current_interval_nob(erts_interval_t *icp) +{ + ASSERT(icp->smp_api); +#ifdef ERTS_SMP + return erts_current_interval_nob__(icp); +#else + return icp->counter.not_atomic; +#endif +} + +ERTS_GLB_INLINE Uint64 +erts_smp_current_interval_acqb(erts_interval_t *icp) +{ + ASSERT(icp->smp_api); +#ifdef ERTS_SMP + return erts_current_interval_acqb__(icp); +#else + return icp->counter.not_atomic; +#endif +} + +#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ + /* * To be used to silence unused result warnings, but do not abuse it. */ @@ -1478,7 +1594,8 @@ void erts_silence_warn_unused_result(long unused); void erts_cleanup_offheap(ErlOffHeap *offheap); -Uint erts_fit_in_bits(Uint); +int erts_fit_in_bits_int64(Sint64); +int erts_fit_in_bits_int32(Sint32); int list_length(Eterm); Export* erts_find_function(Eterm, Eterm, unsigned int); int erts_is_builtin(Eterm, Eterm, int); diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index 8a2a43bebd..8d20178307 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -1319,7 +1319,7 @@ void init_io(void) erts_max_ports = ERTS_MAX_R9_PORTS; } - port_extra_shift = erts_fit_in_bits(erts_max_ports - 1); + port_extra_shift = erts_fit_in_bits_int32(erts_max_ports - 1); port_num_mask = (1 << ports_bits) - 1; erts_port_tab_index_mask = ~(~((Uint) 0) << port_extra_shift); @@ -1556,6 +1556,7 @@ deliver_result(Eterm sender, Eterm pid, Eterm res) { Process *rp; ErtsProcLocks rp_locks = 0; + int scheduler = erts_get_scheduler_id() != 0; ERTS_SMP_CHK_NO_PROC_LOCKS; @@ -1563,7 +1564,9 @@ deliver_result(Eterm sender, Eterm pid, Eterm res) && is_internal_pid(pid) && internal_pid_index(pid) < erts_max_processes); - rp = erts_pid2proc_opt(NULL, 0, pid, 0, ERTS_P2P_FLG_SMP_INC_REFC); + rp = (scheduler + ? erts_proc_lookup(pid) + : erts_pid2proc_opt(NULL, 0, pid, 0, ERTS_P2P_FLG_SMP_INC_REFC)); if (rp) { Eterm tuple; @@ -1581,7 +1584,9 @@ deliver_result(Eterm sender, Eterm pid, Eterm res) #endif ); erts_smp_proc_unlock(rp, rp_locks); - erts_smp_proc_dec_refc(rp); + if (!scheduler) + erts_smp_proc_dec_refc(rp); + } } @@ -1606,6 +1611,7 @@ static void deliver_read_message(Port* prt, Eterm to, ErlHeapFragment *bp; ErlOffHeap *ohp; ErtsProcLocks rp_locks = 0; + int scheduler = erts_get_scheduler_id() != 0; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); ERTS_SMP_CHK_NO_PROC_LOCKS; @@ -1620,7 +1626,10 @@ static void deliver_read_message(Port* prt, Eterm to, need += 2*len; } - rp = erts_pid2proc_opt(NULL, 0, to, 0, ERTS_P2P_FLG_SMP_INC_REFC); + rp = (scheduler + ? erts_proc_lookup(to) + : erts_pid2proc_opt(NULL, 0, to, 0, ERTS_P2P_FLG_SMP_INC_REFC)); + if (!rp) return; @@ -1673,8 +1682,10 @@ static void deliver_read_message(Port* prt, Eterm to, , NIL #endif ); - erts_smp_proc_unlock(rp, rp_locks); - erts_smp_proc_dec_refc(rp); + if (rp_locks) + erts_smp_proc_unlock(rp, rp_locks); + if (!scheduler) + erts_smp_proc_dec_refc(rp); } /* @@ -1741,6 +1752,7 @@ deliver_vec_message(Port* prt, /* Port */ ErlHeapFragment *bp; ErlOffHeap *ohp; ErtsProcLocks rp_locks = 0; + int scheduler = erts_get_scheduler_id() != 0; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); ERTS_SMP_CHK_NO_PROC_LOCKS; @@ -1749,7 +1761,10 @@ deliver_vec_message(Port* prt, /* Port */ * Check arguments for validity. */ - rp = erts_pid2proc_opt(NULL, 0, to, 0, ERTS_P2P_FLG_SMP_INC_REFC); + + rp = (scheduler + ? erts_proc_lookup(to) + : erts_pid2proc_opt(NULL, 0, to, 0, ERTS_P2P_FLG_SMP_INC_REFC)); if (!rp) return; @@ -1831,7 +1846,8 @@ deliver_vec_message(Port* prt, /* Port */ #endif ); erts_smp_proc_unlock(rp, rp_locks); - erts_smp_proc_dec_refc(rp); + if (!scheduler) + erts_smp_proc_dec_refc(rp); } @@ -2239,9 +2255,8 @@ void erts_port_command(Process *proc, { ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND; - Process* rp = erts_pid2proc_opt(NULL, 0, - port->connected, rp_locks, - ERTS_P2P_FLG_SMP_INC_REFC); + Process* rp = erts_pid2proc(NULL, 0, + port->connected, rp_locks); if (rp) { (void) erts_send_exit_signal(NULL, port->id, @@ -2252,7 +2267,6 @@ void erts_port_command(Process *proc, NULL, 0); erts_smp_proc_unlock(rp, rp_locks); - erts_smp_proc_dec_refc(rp); } } @@ -2803,13 +2817,17 @@ void driver_report_exit(int ix, int status) ErlHeapFragment *bp = NULL; ErlOffHeap *ohp; ErtsProcLocks rp_locks = 0; + int scheduler = erts_get_scheduler_id() != 0; ERTS_SMP_CHK_NO_PROC_LOCKS; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); pid = prt->connected; ASSERT(is_internal_pid(pid)); - rp = erts_pid2proc_opt(NULL, 0, pid, 0, ERTS_P2P_FLG_SMP_INC_REFC); + + rp = (scheduler + ? erts_proc_lookup(pid) + : erts_pid2proc_opt(NULL, 0, pid, 0, ERTS_P2P_FLG_SMP_INC_REFC)); if (!rp) return; @@ -2826,7 +2844,8 @@ void driver_report_exit(int ix, int status) ); erts_smp_proc_unlock(rp, rp_locks); - erts_smp_proc_dec_refc(rp); + if (!scheduler) + erts_smp_proc_dec_refc(rp); } @@ -2957,6 +2976,7 @@ driver_deliver_term(ErlDrvPort port, ErlOffHeap *ohp; ErtsProcLocks rp_locks = 0; struct b2t_states__ b2t; + int scheduler = 1; /* Silence erroneous warning... */ init_b2t_states(&b2t); @@ -3148,7 +3168,13 @@ driver_deliver_term(ErlDrvPort port, if (res <= 0) goto done; - rp = erts_pid2proc_opt(NULL, 0, to, rp_locks, ERTS_P2P_FLG_SMP_INC_REFC); + /* + * Increase refc on proc if done from a non-scheduler thread. + */ + scheduler = erts_get_scheduler_id() != 0; + rp = (scheduler + ? erts_proc_lookup(to) + : erts_pid2proc_opt(NULL, 0, to, 0, ERTS_P2P_FLG_SMP_INC_REFC)); if (!rp) { res = 0; goto done; @@ -3394,7 +3420,8 @@ driver_deliver_term(ErlDrvPort port, if (rp) { if (rp_locks) erts_smp_proc_unlock(rp, rp_locks); - erts_smp_proc_dec_refc(rp); + if (!scheduler) + erts_smp_proc_dec_refc(rp); } #endif cleanup_b2t_states(&b2t); diff --git a/erts/emulator/beam/register.c b/erts/emulator/beam/register.c index 26d64887d0..0ddae56322 100644 --- a/erts/emulator/beam/register.c +++ b/erts/emulator/beam/register.c @@ -96,12 +96,7 @@ reg_safe_write_lock(Process *c_p, ErtsProcLocks *c_p_locks) static ERTS_INLINE int is_proc_alive(Process *p) { - int res; - erts_pix_lock_t *pixlck = ERTS_PID2PIXLOCK(p->id); - erts_pix_lock(pixlck); - res = !p->is_exiting; - erts_pix_unlock(pixlck); - return res; + return !ERTS_PROC_IS_EXITING(p); } #endif diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index 7b2bb81f62..0d3b910278 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -39,6 +39,8 @@ #define ENABLE_CHILD_WAITER_THREAD 1 #endif +#define ERTS_I64_LITERAL(X) X##LL + #if defined (__WIN32__) # include "erl_win_sys.h" #elif defined (VXWORKS) @@ -91,14 +93,22 @@ typedef ERTS_SYS_FD_TYPE ErtsSysFdType; # endif #endif -#ifdef __GNUC__ -# if __GNUC__ < 3 && (__GNUC__ != 2 || __GNUC_MINOR__ < 96) -# define ERTS_LIKELY(BOOL) (BOOL) -# define ERTS_UNLIKELY(BOOL) (BOOL) -# else -# define ERTS_LIKELY(BOOL) __builtin_expect((BOOL), !0) -# define ERTS_UNLIKELY(BOOL) __builtin_expect((BOOL), 0) -# endif +#if !defined(__GNUC__) +# define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) 0 +#elif !defined(__GNUC_MINOR__) +# define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \ + ((__GNUC__ << 24) >= (((MAJ) << 24) | ((MIN) << 12) | (PL))) +#elif !defined(__GNUC_PATCHLEVEL__) +# define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \ + (((__GNUC__ << 24) | (__GNUC_MINOR__ << 12)) >= (((MAJ) << 24) | ((MIN) << 12) | (PL))) +#else +# define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \ + (((__GNUC__ << 24) | (__GNUC_MINOR__ << 12) | __GNUC_PATCHLEVEL__) >= (((MAJ) << 24) | ((MIN) << 12) | (PL))) +#endif + +#if ERTS_AT_LEAST_GCC_VSN__(2, 96, 0) +# define ERTS_LIKELY(BOOL) __builtin_expect((BOOL), !0) +# define ERTS_UNLIKELY(BOOL) __builtin_expect((BOOL), 0) #else # define ERTS_LIKELY(BOOL) (BOOL) # define ERTS_UNLIKELY(BOOL) (BOOL) @@ -178,6 +188,18 @@ int real_printf(const char *fmt, ...); # define printf real_printf #endif +#undef __deprecated +#if ERTS_AT_LEAST_GCC_VSN__(3, 0, 0) +# define __deprecated __attribute__((deprecated)) +#else +# define __deprecated +#endif +#if ERTS_AT_LEAST_GCC_VSN__(3, 0, 4) +# define erts_align_attribute(SZ) __attribute__ ((aligned (SZ))) +#else +# define erts_align_attribute(SZ) +#endif + /* In VC++, noreturn is a declspec that has to be before the types, * but in GNUC it is an att ribute to be placed between return type * and function name, hence __decl_noreturn __noreturn @@ -185,12 +207,6 @@ int real_printf(const char *fmt, ...); #if __GNUC__ # define __decl_noreturn # define __noreturn __attribute__((noreturn)) -# undef __deprecated -# if __GNUC__ >= 3 -# define __deprecated __attribute__((deprecated)) -# else -# define __deprecated -# endif #else # if defined(__WIN32__) && defined(_MSC_VER) # define __noreturn @@ -199,7 +215,6 @@ int real_printf(const char *fmt, ...); # define __noreturn # define __decl_noreturn # endif -# define __deprecated #endif /* diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index a36d15204e..245916bc0d 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -268,16 +268,42 @@ list_length(Eterm list) return i; } -Uint erts_fit_in_bits(Uint n) +static const struct { + Sint64 mask; + int bits; +} fib_data[] = {{ERTS_I64_LITERAL(0x2), 1}, + {ERTS_I64_LITERAL(0xc), 2}, + {ERTS_I64_LITERAL(0xf0), 4}, + {ERTS_I64_LITERAL(0xff00), 8}, + {ERTS_I64_LITERAL(0xffff0000), 16}, + {ERTS_I64_LITERAL(0xffffffff00000000), 32}}; + +static ERTS_INLINE int +fit_in_bits(Sint64 value, int start) { - Uint i; + int bits = 0; + int i; - i = 0; - while (n > 0) { - i++; - n >>= 1; - } - return i; + for (i = start; i >= 0; i--) { + if (value & fib_data[i].mask) { + value >>= fib_data[i].bits; + bits |= fib_data[i].bits; + } + } + + bits++; + + return bits; +} + +int erts_fit_in_bits_int64(Sint64 value) +{ + return fit_in_bits(value, 5); +} + +int erts_fit_in_bits_int32(Sint32 value) +{ + return fit_in_bits((Sint64) (Uint32) value, 4); } int @@ -3240,7 +3266,7 @@ ptimer_timeout(ErtsSmpPTimer *ptimer) ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS, ERTS_P2P_FLG_ALLOW_OTHER_X); if (p) { - if (!p->is_exiting + if (!ERTS_PROC_IS_EXITING(p) && !(ptimer->timer.flags & ERTS_PTMR_FLG_CANCELLED)) { ASSERT(*ptimer->timer.timer_ref == ptimer); *ptimer->timer.timer_ref = NULL; @@ -3458,6 +3484,254 @@ void erts_silence_warn_unused_result(long unused) } +/* + * Interval counts + */ +void +erts_interval_init(erts_interval_t *icp) +{ +#ifdef ARCH_64 + erts_atomic_init_nob(&icp->counter.atomic, 0); +#else + erts_dw_aint_t dw; +#ifdef ETHR_SU_DW_NAINT_T__ + dw.dw_sint = 0; +#else + dw.sint[ERTS_DW_AINT_HIGH_WORD] = 0; + dw.sint[ERTS_DW_AINT_LOW_WORD] = 0; +#endif + erts_dw_atomic_init_nob(&icp->counter.atomic, &dw); + +#endif +#ifdef DEBUG + icp->smp_api = 0; +#endif +} + +void +erts_smp_interval_init(erts_interval_t *icp) +{ +#ifdef ERTS_SMP + erts_interval_init(icp); +#else + icp->counter.not_atomic = 0; +#endif +#ifdef DEBUG + icp->smp_api = 1; +#endif +} + +static ERTS_INLINE Uint64 +step_interval_nob(erts_interval_t *icp) +{ +#ifdef ARCH_64 + return (Uint64) erts_atomic_inc_read_nob(&icp->counter.atomic); +#else + erts_dw_aint_t exp; + + erts_dw_atomic_read_nob(&icp->counter.atomic, &exp); + while (1) { + erts_dw_aint_t new = exp; + +#ifdef ETHR_SU_DW_NAINT_T__ + new.dw_sint++; +#else + new.sint[ERTS_DW_AINT_LOW_WORD]++; + if (new.sint[ERTS_DW_AINT_LOW_WORD] == 0) + new.sint[ERTS_DW_AINT_HIGH_WORD]++; +#endif + + if (erts_dw_atomic_cmpxchg_nob(&icp->counter.atomic, &new, &exp)) + return erts_interval_dw_aint_to_val__(&new); + + } +#endif +} + +static ERTS_INLINE Uint64 +step_interval_relb(erts_interval_t *icp) +{ +#ifdef ARCH_64 + return (Uint64) erts_atomic_inc_read_relb(&icp->counter.atomic); +#else + erts_dw_aint_t exp; + + erts_dw_atomic_read_nob(&icp->counter.atomic, &exp); + while (1) { + erts_dw_aint_t new = exp; + +#ifdef ETHR_SU_DW_NAINT_T__ + new.dw_sint++; +#else + new.sint[ERTS_DW_AINT_LOW_WORD]++; + if (new.sint[ERTS_DW_AINT_LOW_WORD] == 0) + new.sint[ERTS_DW_AINT_HIGH_WORD]++; +#endif + + if (erts_dw_atomic_cmpxchg_relb(&icp->counter.atomic, &new, &exp)) + return erts_interval_dw_aint_to_val__(&new); + + } +#endif +} + + +static ERTS_INLINE Uint64 +ensure_later_interval_nob(erts_interval_t *icp, Uint64 ic) +{ + Uint64 curr_ic; +#ifdef ARCH_64 + curr_ic = (Uint64) erts_atomic_read_nob(&icp->counter.atomic); + if (curr_ic > ic) + return curr_ic; + return (Uint64) erts_atomic_inc_read_nob(&icp->counter.atomic); +#else + erts_dw_aint_t exp; + + erts_dw_atomic_read_nob(&icp->counter.atomic, &exp); + curr_ic = erts_interval_dw_aint_to_val__(&exp); + if (curr_ic > ic) + return curr_ic; + + while (1) { + erts_dw_aint_t new = exp; + +#ifdef ETHR_SU_DW_NAINT_T__ + new.dw_sint++; +#else + new.sint[ERTS_DW_AINT_LOW_WORD]++; + if (new.sint[ERTS_DW_AINT_LOW_WORD] == 0) + new.sint[ERTS_DW_AINT_HIGH_WORD]++; +#endif + + if (erts_dw_atomic_cmpxchg_nob(&icp->counter.atomic, &new, &exp)) + return erts_interval_dw_aint_to_val__(&new); + + curr_ic = erts_interval_dw_aint_to_val__(&exp); + if (curr_ic > ic) + return curr_ic; + } +#endif +} + + +static ERTS_INLINE Uint64 +ensure_later_interval_acqb(erts_interval_t *icp, Uint64 ic) +{ + Uint64 curr_ic; +#ifdef ARCH_64 + curr_ic = (Uint64) erts_atomic_read_acqb(&icp->counter.atomic); + if (curr_ic > ic) + return curr_ic; + return (Uint64) erts_atomic_inc_read_acqb(&icp->counter.atomic); +#else + erts_dw_aint_t exp; + + erts_dw_atomic_read_acqb(&icp->counter.atomic, &exp); + curr_ic = erts_interval_dw_aint_to_val__(&exp); + if (curr_ic > ic) + return curr_ic; + + while (1) { + erts_dw_aint_t new = exp; + +#ifdef ETHR_SU_DW_NAINT_T__ + new.dw_sint++; +#else + new.sint[ERTS_DW_AINT_LOW_WORD]++; + if (new.sint[ERTS_DW_AINT_LOW_WORD] == 0) + new.sint[ERTS_DW_AINT_HIGH_WORD]++; +#endif + + if (erts_dw_atomic_cmpxchg_acqb(&icp->counter.atomic, &new, &exp)) + return erts_interval_dw_aint_to_val__(&new); + + curr_ic = erts_interval_dw_aint_to_val__(&exp); + if (curr_ic > ic) + return curr_ic; + } +#endif +} + +Uint64 +erts_step_interval_nob(erts_interval_t *icp) +{ + ASSERT(!icp->smp_api); + return step_interval_nob(icp); +} + +Uint64 +erts_step_interval_relb(erts_interval_t *icp) +{ + ASSERT(!icp->smp_api); + return step_interval_relb(icp); +} + +Uint64 +erts_smp_step_interval_nob(erts_interval_t *icp) +{ + ASSERT(icp->smp_api); +#ifdef ERTS_SMP + return step_interval_nob(icp); +#else + return ++icp->counter.not_atomic; +#endif +} + +Uint64 +erts_smp_step_interval_relb(erts_interval_t *icp) +{ + ASSERT(icp->smp_api); +#ifdef ERTS_SMP + return step_interval_relb(icp); +#else + return ++icp->counter.not_atomic; +#endif +} + +Uint64 +erts_ensure_later_interval_nob(erts_interval_t *icp, Uint64 ic) +{ + ASSERT(!icp->smp_api); + return ensure_later_interval_nob(icp, ic); +} + +Uint64 +erts_ensure_later_interval_acqb(erts_interval_t *icp, Uint64 ic) +{ + ASSERT(!icp->smp_api); + return ensure_later_interval_acqb(icp, ic); +} + +Uint64 +erts_smp_ensure_later_interval_nob(erts_interval_t *icp, Uint64 ic) +{ + ASSERT(icp->smp_api); +#ifdef ERTS_SMP + return ensure_later_interval_nob(icp, ic); +#else + if (icp->counter.not_atomic > ic) + return icp->counter.not_atomic; + else + return ++icp->counter.not_atomic; +#endif +} + +Uint64 +erts_smp_ensure_later_interval_acqb(erts_interval_t *icp, Uint64 ic) +{ + ASSERT(icp->smp_api); +#ifdef ERTS_SMP + return ensure_later_interval_acqb(icp, ic); +#else + if (icp->counter.not_atomic > ic) + return icp->counter.not_atomic; + else + return ++icp->counter.not_atomic; +#endif +} + + #ifdef DEBUG /* * Handy functions when using a debugger - don't use in the code! @@ -3490,7 +3764,7 @@ Process *p; void ppi(Eterm pid) { - pp(erts_pid2proc_unlocked(pid)); + pp(erts_proc_lookup(pid)); } void td(Eterm x) diff --git a/erts/emulator/sys/win32/erl_win_sys.h b/erts/emulator/sys/win32/erl_win_sys.h index 03298a6c54..21c52ca26d 100644 --- a/erts/emulator/sys/win32/erl_win_sys.h +++ b/erts/emulator/sys/win32/erl_win_sys.h @@ -90,6 +90,11 @@ #define strncasecmp _strnicmp +#ifndef __GNUC__ +# undef ERTS_I64_LITERAL +# define ERTS_I64_LITERAL(X) X##i64 +#endif + /* * Practial Windows specific macros. */ diff --git a/erts/emulator/sys/win32/sys_time.c b/erts/emulator/sys/win32/sys_time.c index b5123dc45d..2f2dfc8197 100644 --- a/erts/emulator/sys/win32/sys_time.c +++ b/erts/emulator/sys/win32/sys_time.c @@ -26,11 +26,7 @@ #include "sys.h" #include "assert.h" -#ifdef __GNUC__ -#define LL_LITERAL(X) X##LL -#else -#define LL_LITERAL(X) X##i64 -#endif +#define LL_LITERAL(X) ERTS_I64_LITERAL(X) /******************* Routines for time measurement *********************/ -- cgit v1.2.3