diff options
Diffstat (limited to 'erts/emulator')
-rw-r--r-- | erts/emulator/beam/atom.names | 4 | ||||
-rw-r--r-- | erts/emulator/beam/erl_bif_info.c | 34 | ||||
-rw-r--r-- | erts/emulator/beam/erl_process.c | 188 | ||||
-rw-r--r-- | erts/emulator/beam/erl_process.h | 23 | ||||
-rw-r--r-- | erts/emulator/test/statistics_SUITE.erl | 25 |
5 files changed, 202 insertions, 72 deletions
diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index 477a7676d6..a44d23b181 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -76,6 +76,7 @@ atom ac atom accessor atom active atom active_tasks +atom active_tasks_all atom alive atom all atom all_but_first @@ -567,6 +568,7 @@ atom return_to atom return_trace atom run_queue atom run_queue_lengths +atom run_queue_lengths_all atom runnable atom runnable_ports atom runnable_procs @@ -656,8 +658,10 @@ atom Times='*' atom timestamp atom total atom total_active_tasks +atom total_active_tasks_all atom total_heap_size atom total_run_queue_lengths +atom total_run_queue_lengths_all atom tpkt atom trace trace_ts traced atom trace_control_word diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 5fc70dfc02..e2773475b0 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -3450,9 +3450,15 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1) if (is_non_value(res)) BIF_RET(am_undefined); BIF_TRAP1(gather_sched_wall_time_res_trap, BIF_P, res); - } else if (BIF_ARG_1 == am_total_active_tasks - || BIF_ARG_1 == am_total_run_queue_lengths) { - Uint no = erts_run_queues_len(NULL, 0, BIF_ARG_1 == am_total_active_tasks); + } else if ((BIF_ARG_1 == am_total_active_tasks) + | (BIF_ARG_1 == am_total_run_queue_lengths) + | (BIF_ARG_1 == am_total_active_tasks_all) + | (BIF_ARG_1 == am_total_run_queue_lengths_all)) { + Uint no = erts_run_queues_len(NULL, 0, + ((BIF_ARG_1 == am_total_active_tasks) + | (BIF_ARG_1 == am_total_active_tasks_all)), + ((BIF_ARG_1 == am_total_active_tasks_all) + | (BIF_ARG_1 == am_total_run_queue_lengths_all))); if (IS_USMALL(0, no)) res = make_small(no); else { @@ -3460,13 +3466,21 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1) res = uint_to_big(no, hp); } BIF_RET(res); - } else if (BIF_ARG_1 == am_active_tasks - || BIF_ARG_1 == am_run_queue_lengths) { + } else if ((BIF_ARG_1 == am_active_tasks) + | (BIF_ARG_1 == am_run_queue_lengths) + | (BIF_ARG_1 == am_active_tasks_all) + | (BIF_ARG_1 == am_run_queue_lengths_all)) { Eterm res, *hp, **hpp; Uint sz, *szp; - int no_qs = erts_no_run_queues; + int incl_dirty_io = ((BIF_ARG_1 == am_active_tasks_all) + | (BIF_ARG_1 == am_run_queue_lengths_all)); + int no_qs = (erts_no_run_queues + ERTS_NUM_DIRTY_CPU_RUNQS + + (incl_dirty_io ? ERTS_NUM_DIRTY_IO_RUNQS : 0)); Uint *qszs = erts_alloc(ERTS_ALC_T_TMP,sizeof(Uint)*no_qs*2); - (void) erts_run_queues_len(qszs, 0, BIF_ARG_1 == am_active_tasks); + (void) erts_run_queues_len(qszs, 0, + ((BIF_ARG_1 == am_active_tasks) + | (BIF_ARG_1 == am_active_tasks_all)), + incl_dirty_io); sz = 0; szp = &sz; hpp = NULL; @@ -3539,7 +3553,7 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1) res = TUPLE2(hp, b1, b2); BIF_RET(res); } else if (BIF_ARG_1 == am_run_queue) { - res = erts_run_queues_len(NULL, 1, 0); + res = erts_run_queues_len(NULL, 1, 0, 0); BIF_RET(make_small(res)); } else if (BIF_ARG_1 == am_wall_clock) { UWord w1, w2; @@ -3557,9 +3571,9 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1) else if (ERTS_IS_ATOM_STR("run_queues", BIF_ARG_1)) { Eterm res, *hp, **hpp; Uint sz, *szp; - int no_qs = erts_no_run_queues; + int no_qs = erts_no_run_queues + ERTS_NUM_DIRTY_RUNQS; Uint *qszs = erts_alloc(ERTS_ALC_T_TMP,sizeof(Uint)*no_qs*2); - (void) erts_run_queues_len(qszs, 0, 0); + (void) erts_run_queues_len(qszs, 0, 0, 1); sz = 0; szp = &sz; hpp = NULL; diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index a56ef4aedb..6f96833b21 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -405,13 +405,59 @@ static erts_atomic_t runq_supervisor_sleeping; ErtsSchedulerData *erts_scheduler_data; #endif -ErtsAlignedRunQueue *erts_aligned_run_queues; -Uint erts_no_run_queues; +ErtsAlignedRunQueue * ERTS_WRITE_UNLIKELY(erts_aligned_run_queues); +Uint ERTS_WRITE_UNLIKELY(erts_no_run_queues); -ErtsAlignedSchedulerData *erts_aligned_scheduler_data; #ifdef ERTS_DIRTY_SCHEDULERS -ErtsAlignedSchedulerData *erts_aligned_dirty_cpu_scheduler_data; -ErtsAlignedSchedulerData *erts_aligned_dirty_io_scheduler_data; + +struct { + union { + erts_smp_atomic32_t active; + char align__[ERTS_CACHE_LINE_SIZE]; + } cpu; + union { + erts_smp_atomic32_t active; + char align__[ERTS_CACHE_LINE_SIZE]; + } io; +} dirty_count erts_align_attribute(ERTS_CACHE_LINE_SIZE); + +#endif + +static ERTS_INLINE void +dirty_active(ErtsSchedulerData *esdp, erts_aint32_t add) +{ +#ifdef ERTS_DIRTY_SCHEDULERS + erts_aint32_t val; + erts_smp_atomic32_t *ap; + switch (esdp->type) { + case ERTS_SCHED_DIRTY_CPU: + ap = &dirty_count.cpu.active; + break; + case ERTS_SCHED_DIRTY_IO: + ap = &dirty_count.io.active; + break; + default: + ap = NULL; + ERTS_INTERNAL_ERROR("Not a dirty scheduler"); + break; + } + + /* + * All updates done under run-queue lock, so + * no inc or dec needed... + */ + ERTS_SMP_ASSERT(erts_smp_lc_runq_is_locked(esdp->run_queue)); + + val = erts_smp_atomic32_read_nob(ap); + val += add; + erts_smp_atomic32_set_nob(ap, val); +#endif +} + +ErtsAlignedSchedulerData * ERTS_WRITE_UNLIKELY(erts_aligned_scheduler_data); +#ifdef ERTS_DIRTY_SCHEDULERS +ErtsAlignedSchedulerData * ERTS_WRITE_UNLIKELY(erts_aligned_dirty_cpu_scheduler_data); +ErtsAlignedSchedulerData * ERTS_WRITE_UNLIKELY(erts_aligned_dirty_io_scheduler_data); typedef union { Process dsp; char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(Process))]; @@ -539,22 +585,28 @@ do { \ } \ } while (0) -#define ERTS_ATOMIC_FOREACH_RUNQ_X(RQVAR, DO, DOX) \ +#define ERTS_ATOMIC_FOREACH_RUNQ_X(RQVAR, NRQS, DO, DOX) \ do { \ ErtsRunQueue *RQVAR; \ + int nrqs = (NRQS); \ int ix__; \ - for (ix__ = 0; ix__ < erts_no_run_queues; ix__++) { \ + for (ix__ = 0; ix__ < nrqs; ix__++) { \ RQVAR = ERTS_RUNQ_IX(ix__); \ erts_smp_runq_lock(RQVAR); \ { DO; } \ } \ { DOX; } \ - for (ix__ = 0; ix__ < erts_no_run_queues; ix__++) \ + for (ix__ = 0; ix__ < nrqs; ix__++) \ erts_smp_runq_unlock(ERTS_RUNQ_IX(ix__)); \ } while (0) -#define ERTS_ATOMIC_FOREACH_RUNQ(RQVAR, DO) \ - ERTS_ATOMIC_FOREACH_RUNQ_X(RQVAR, DO, ) +#define ERTS_ATOMIC_FOREACH_RUNQ(RQVAR, DO) \ + ERTS_ATOMIC_FOREACH_RUNQ_X(RQVAR, erts_no_run_queues + ERTS_NUM_DIRTY_RUNQS, DO, ) + +#define ERTS_ATOMIC_FOREACH_NORMAL_RUNQ(RQVAR, DO) \ + ERTS_ATOMIC_FOREACH_RUNQ_X(RQVAR, erts_no_run_queues, DO, ) + + /* * Local functions. */ @@ -2977,7 +3029,7 @@ erts_active_schedulers(void) { Uint as = erts_no_schedulers; - ERTS_ATOMIC_FOREACH_RUNQ(rq, as -= abs(rq->waiting)); + ERTS_ATOMIC_FOREACH_NORMAL_RUNQ(rq, as -= abs(rq->waiting)); return as; } @@ -3396,6 +3448,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) rq->sleepers.list->prev = ssi; rq->sleepers.list = ssi; erts_smp_spin_unlock(&rq->sleepers.lock); + dirty_active(esdp, -1); } #endif @@ -3737,6 +3790,9 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) sched_active_sys(esdp->no, rq); } + if (ERTS_SCHEDULER_IS_DIRTY(esdp)) + dirty_active(esdp, 1); + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); } @@ -6136,7 +6192,7 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online #endif ) { - int ix, n, no_ssi; + int ix, n, no_ssi, tot_rqs; char *daww_ptr; size_t daww_sz; size_t size_runqs; @@ -6169,26 +6225,19 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online /* Create and initialize run queues */ n = no_schedulers; - size_runqs = sizeof(ErtsAlignedRunQueue) * (n + ERTS_NUM_DIRTY_RUNQS); + tot_rqs = (n + ERTS_NUM_DIRTY_RUNQS); + size_runqs = sizeof(ErtsAlignedRunQueue) * tot_rqs; erts_aligned_run_queues = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_RUNQS, size_runqs); #ifdef ERTS_SMP -#ifdef ERTS_DIRTY_SCHEDULERS - erts_aligned_run_queues += ERTS_NUM_DIRTY_RUNQS; -#endif erts_smp_atomic32_init_nob(&no_empty_run_queues, 0); #endif erts_no_run_queues = n; - for (ix = -(ERTS_NUM_DIRTY_RUNQS); ix < n; ix++) { + for (ix = 0; ix < tot_rqs; ix++) { int pix, rix; -#ifdef ERTS_DIRTY_SCHEDULERS - ErtsRunQueue *rq = ERTS_RUNQ_IX_IS_DIRTY(ix) ? - ERTS_DIRTY_RUNQ_IX(ix) : ERTS_RUNQ_IX(ix); -#else ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); -#endif rq->ix = ix; @@ -6461,6 +6510,11 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online ERTS_SCHED_DIRTY_IO, no_dirty_io_schedulers); + erts_smp_atomic32_init_nob(&dirty_count.cpu.active, + (erts_aint32_t) no_dirty_cpu_schedulers); + erts_smp_atomic32_init_nob(&dirty_count.io.active, + (erts_aint32_t) no_dirty_io_schedulers); + #endif if (set_schdlr_sspnd_change_flags) @@ -7835,6 +7889,7 @@ suspend_scheduler(ErtsSchedulerData *esdp) #endif if (sched_type != ERTS_SCHED_NORMAL) { + dirty_active(esdp, -1); erts_smp_runq_unlock(esdp->run_queue); dirty_sched_wall_time_change(esdp, 0); } @@ -8143,7 +8198,9 @@ suspend_scheduler(ErtsSchedulerData *esdp) erts_smp_runq_lock(esdp->run_queue); non_empty_runq(esdp->run_queue); - if (sched_type == ERTS_SCHED_NORMAL) { + if (sched_type != ERTS_SCHED_NORMAL) + dirty_active(esdp, 1); + else { schedule_bound_processes(esdp->run_queue, &sbp); erts_sched_check_cpu_bind_post_suspend(esdp); @@ -9803,38 +9860,69 @@ erts_internal_is_process_executing_dirty_1(BIF_ALIST_1) BIF_RET(am_false); } +static ERTS_INLINE void +run_queues_len_aux(ErtsRunQueue *rq, Uint *tot_len, Uint *qlen, int *ip, int incl_active_sched, int locked) +{ + Sint rq_len; + + if (locked) + rq_len = (Sint) erts_smp_atomic32_read_dirty(&rq->len); + else + rq_len = (Sint) erts_smp_atomic32_read_nob(&rq->len); + ASSERT(rq_len >= 0); + + if (incl_active_sched) { +#ifdef ERTS_DIRTY_SCHEDULERS + if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) { + erts_aint32_t dcnt; + if (ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(rq)) { + dcnt = erts_smp_atomic32_read_nob(&dirty_count.cpu.active); + ASSERT(0 <= dcnt && dcnt <= erts_no_dirty_cpu_schedulers); + } + else { + ASSERT(ERTS_RUNQ_IS_DIRTY_IO_RUNQ(rq)); + dcnt = erts_smp_atomic32_read_nob(&dirty_count.io.active); + ASSERT(0 <= dcnt && dcnt <= erts_no_dirty_io_schedulers); + } + rq_len += (Sint) dcnt; + } + else +#endif + { + if (ERTS_RUNQ_FLGS_GET_NOB(rq) & ERTS_RUNQ_FLG_EXEC) + rq_len++; + } + } + if (qlen) + qlen[(*ip)++] = rq_len; + *tot_len += (Uint) rq_len; +} Uint -erts_run_queues_len(Uint *qlen, int atomic_queues_read, int incl_active_sched) +erts_run_queues_len(Uint *qlen, int atomic_queues_read, int incl_active_sched, + int incl_dirty_io) { - int i = 0; + int i = 0, j = 0; Uint len = 0; - if (atomic_queues_read) - ERTS_ATOMIC_FOREACH_RUNQ(rq, - { - Sint rq_len = (Sint) erts_smp_atomic32_read_dirty(&rq->len); - ASSERT(rq_len >= 0); - if (incl_active_sched - && (ERTS_RUNQ_FLGS_GET_NOB(rq) & ERTS_RUNQ_FLG_EXEC)) { - rq_len++; - } - if (qlen) - qlen[i++] = rq_len; - len += (Uint) rq_len; - } - ); + int no_rqs = erts_no_run_queues; + +#ifdef ERTS_DIRTY_SCHEDULERS + if (incl_dirty_io) + no_rqs += ERTS_NUM_DIRTY_RUNQS; + else + no_rqs += ERTS_NUM_DIRTY_CPU_RUNQS; +#endif + + if (atomic_queues_read) { + ERTS_ATOMIC_FOREACH_RUNQ_X(rq, no_rqs, + run_queues_len_aux(rq, &len, qlen, &j, + incl_active_sched, 1), + /* Nothing... */); + } else { - for (i = 0; i < erts_no_run_queues; i++) { + for (i = 0; i < no_rqs; i++) { ErtsRunQueue *rq = ERTS_RUNQ_IX(i); - Sint rq_len = (Sint) erts_smp_atomic32_read_nob(&rq->len); - ASSERT(rq_len >= 0); - if (incl_active_sched - && (ERTS_RUNQ_FLGS_GET_NOB(rq) & ERTS_RUNQ_FLG_EXEC)) { - rq_len++; - } - if (qlen) - qlen[i] = rq_len; - len += (Uint) rq_len; + run_queues_len_aux(rq, &len, qlen, &j, incl_active_sched, 0); } } @@ -12136,6 +12224,8 @@ erts_get_total_reductions(Uint *redsp, Uint *diffp) Uint reds = 0; ERTS_ATOMIC_FOREACH_RUNQ_X(rq, + erts_no_run_queues + ERTS_NUM_DIRTY_RUNQS, + reds += rq->procs.reductions, if (redsp) *redsp = reds; diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 6113c8aa05..aa88ff26e8 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -1541,24 +1541,29 @@ extern int erts_system_profile_ts_type; } while (0) #if defined(ERTS_DIRTY_SCHEDULERS) && defined(ERTS_SMP) -#define ERTS_NUM_DIRTY_RUNQS 2 +#define ERTS_NUM_DIRTY_CPU_RUNQS 1 +#define ERTS_NUM_DIRTY_IO_RUNQS 1 #else -#define ERTS_NUM_DIRTY_RUNQS 0 +#define ERTS_NUM_DIRTY_CPU_RUNQS 0 +#define ERTS_NUM_DIRTY_IO_RUNQS 0 #endif +#define ERTS_NUM_DIRTY_RUNQS (ERTS_NUM_DIRTY_CPU_RUNQS+ERTS_NUM_DIRTY_IO_RUNQS) + #define ERTS_RUNQ_IX(IX) \ - (ASSERT(0 <= (IX) && (IX) < erts_no_run_queues), \ + (ASSERT(0 <= (IX) && (IX) < erts_no_run_queues+ERTS_NUM_DIRTY_RUNQS), \ &erts_aligned_run_queues[(IX)].runq) #ifdef ERTS_DIRTY_SCHEDULERS #define ERTS_RUNQ_IX_IS_DIRTY(IX) \ - (-(ERTS_NUM_DIRTY_RUNQS) <= (IX) && (IX) < 0) + (ASSERT(0 <= (IX) && (IX) < erts_no_run_queues+ERTS_NUM_DIRTY_RUNQS), \ + (erts_no_run_queues <= (IX))) #define ERTS_DIRTY_RUNQ_IX(IX) \ (ASSERT(ERTS_RUNQ_IX_IS_DIRTY(IX)), \ &erts_aligned_run_queues[(IX)].runq) -#define ERTS_DIRTY_CPU_RUNQ (&erts_aligned_run_queues[-1].runq) -#define ERTS_DIRTY_IO_RUNQ (&erts_aligned_run_queues[-2].runq) -#define ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(RQ) ((RQ)->ix == -1) -#define ERTS_RUNQ_IS_DIRTY_IO_RUNQ(RQ) ((RQ)->ix == -2) +#define ERTS_DIRTY_CPU_RUNQ (&erts_aligned_run_queues[erts_no_run_queues].runq) +#define ERTS_DIRTY_IO_RUNQ (&erts_aligned_run_queues[erts_no_run_queues+1].runq) +#define ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(RQ) ((RQ) == ERTS_DIRTY_CPU_RUNQ) +#define ERTS_RUNQ_IS_DIRTY_IO_RUNQ(RQ) ((RQ) == ERTS_DIRTY_IO_RUNQ) #else #define ERTS_RUNQ_IX_IS_DIRTY(IX) 0 #endif @@ -1837,7 +1842,7 @@ Uint erts_active_schedulers(void); void erts_init_process(int, int, int); Eterm erts_process_state2status(erts_aint32_t); Eterm erts_process_status(Process *, Eterm); -Uint erts_run_queues_len(Uint *, int, int); +Uint erts_run_queues_len(Uint *, int, int, int); void erts_add_to_runq(Process *); Eterm erts_bound_schedulers_term(Process *c_p); Eterm erts_get_cpu_topology_term(Process *c_p, Eterm which); diff --git a/erts/emulator/test/statistics_SUITE.erl b/erts/emulator/test/statistics_SUITE.erl index 3057905f4c..65188b6e26 100644 --- a/erts/emulator/test/statistics_SUITE.erl +++ b/erts/emulator/test/statistics_SUITE.erl @@ -502,20 +502,37 @@ run_queues_lengths_active_tasks(Config) -> end, lists:seq(1,10)), + + TRQLs0 = statistics(total_run_queue_lengths), + TRQLAs0 = statistics(total_run_queue_lengths_all), TATs0 = statistics(total_active_tasks), + TATAs0 = statistics(total_active_tasks_all), true = is_integer(TRQLs0), true = is_integer(TATs0), true = TRQLs0 >= 0, + true = TRQLAs0 >= 0, true = TATs0 >= 11, + true = TATAs0 >= 11, NoScheds = erlang:system_info(schedulers), + {DefRqs, + AllRqs} = case erlang:system_info(dirty_cpu_schedulers) of + 0 -> {NoScheds, NoScheds}; + _ -> {NoScheds+1, NoScheds+2} + end, RQLs0 = statistics(run_queue_lengths), + RQLAs0 = statistics(run_queue_lengths_all), ATs0 = statistics(active_tasks), - NoScheds = length(RQLs0), - NoScheds = length(ATs0), + ATAs0 = statistics(active_tasks_all), + DefRqs = length(RQLs0), + AllRqs = length(RQLAs0), + DefRqs = length(ATs0), + AllRqs = length(ATAs0), true = lists:sum(RQLs0) >= 0, + true = lists:sum(RQLAs0) >= 0, true = lists:sum(ATs0) >= 11, + true = lists:sum(ATAs0) >= 11, SO = erlang:system_flag(schedulers_online, 1), @@ -531,8 +548,8 @@ run_queues_lengths_active_tasks(Config) -> RQLs1 = statistics(run_queue_lengths), ATs1 = statistics(active_tasks), - NoScheds = length(RQLs1), - NoScheds = length(ATs1), + DefRqs = length(RQLs1), + DefRqs = length(ATs1), TRQLs2 = lists:sum(RQLs1), TATs2 = lists:sum(ATs1), true = TRQLs2 >= 10, |