Implement statistics(scheduler_wall_time)

8 files changed, 327 insertions, 7 deletions

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index 71454b3e57..7be40976f6 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -95,6 +95,7 @@ atom atom
 atom atom_used
 atom attributes
 atom await_proc_exit
+atom await_sched_wall_time_modifications
 atom awaiting_load
 atom awaiting_unload
 atom backtrace backtrace_depth
@@ -239,6 +240,7 @@ atom generational
 atom get_seq_token
 atom get_tcw
 atom getenv
+atom gather_sched_wall_time_result
 atom getting_linked
 atom getting_unlinked
 atom global
@@ -554,6 +556,7 @@ atom waiting
 atom wall_clock
 atom warning
 atom warning_msg
+atom scheduler_wall_time
 atom wordsize
 atom write_concurrency
 atom xor
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 55f4798892..f8305944a4 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -43,6 +43,9 @@ static Export* set_cpu_topology_trap = NULL;
 static Export* await_proc_exit_trap = NULL;
 Export* erts_format_cpu_topology_trap = NULL;
 
+static Export *await_sched_wall_time_mod_trap;
+static erts_smp_atomic32_t sched_wall_time;
+
 #define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1)
 
 /*
@@ -4160,6 +4163,18 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2)
 	erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
 
 	BIF_RET(am_true);
+    } else if (BIF_ARG_1 == am_scheduler_wall_time) {
+	if (BIF_ARG_2 == am_true || BIF_ARG_2 == am_false) {
+	    erts_aint32_t new = BIF_ARG_2 == am_true ? 1 : 0;
+	    erts_aint32_t old = erts_smp_atomic32_xchg_nob(&sched_wall_time,
+							   new);
+	    Eterm ref = erts_sched_wall_time_request(BIF_P, 1, new);
+	    ASSERT(is_value(ref));
+	    BIF_TRAP2(await_sched_wall_time_mod_trap,
+		      BIF_P,
+		      ref,
+		      old ? am_true : am_false);
+	}
     } else if (ERTS_IS_ATOM_STR("scheduling_statistics", BIF_ARG_1)) {
 	int what;
 	if (ERTS_IS_ATOM_STR("disable", BIF_ARG_2))
@@ -4457,6 +4472,9 @@ void erts_init_bif(void)
 						    am_format_cpu_topology,
 						    1);
     await_proc_exit_trap = erts_export_put(am_erlang,am_await_proc_exit,3);
+    await_sched_wall_time_mod_trap
+	= erts_export_put(am_erlang, am_await_sched_wall_time_modifications, 2);
+    erts_smp_atomic32_init_nob(&sched_wall_time, 0);
 }
 
 #ifdef HARDDEBUG
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index 962db8b831..90a6c0cbee 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -367,6 +367,7 @@ type	EXPORT		LONG_LIVED_LOW	CODE		export_entry
 type	MONITOR_SH	STANDARD_LOW	PROCESSES	monitor_sh
 type	NLINK_SH	STANDARD_LOW	PROCESSES	nlink_sh
 type	AINFO_REQ	STANDARD_LOW	SYSTEM		alloc_info_request
+type	SCHED_WTIME_REQ	STANDARD_LOW	SYSTEM		sched_wall_time_request
 
 +else # "fullword"
 
@@ -383,6 +384,7 @@ type	EXPORT		LONG_LIVED	CODE		export_entry
 type	MONITOR_SH	FIXED_SIZE	PROCESSES	monitor_sh
 type	NLINK_SH	FIXED_SIZE	PROCESSES	nlink_sh
 type	AINFO_REQ	SHORT_LIVED	SYSTEM		alloc_info_request
+type	SCHED_WTIME_REQ	SHORT_LIVED	SYSTEM		sched_wall_time_request
 
 +endif
 
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 5a806777fe..ebd475f73a 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -57,6 +57,8 @@
 static Export* alloc_info_trap = NULL;
 static Export* alloc_sizes_trap = NULL;
 
+static Export *gather_sched_wall_time_res_trap;
+
 #define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1)
 
 /* Keep erts_system_version as a global variable for easy access from a core */
@@ -3180,7 +3182,12 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1)
     Eterm res;
     Eterm* hp;
 
-    if (BIF_ARG_1 == am_context_switches) {
+    if (BIF_ARG_1 == am_scheduler_wall_time) {
+	res = erts_sched_wall_time_request(BIF_P, 0, 0);
+	if (is_non_value(res))
+	    BIF_RET(am_undefined);
+	BIF_TRAP1(gather_sched_wall_time_res_trap, BIF_P, res);
+    } else if (BIF_ARG_1 == am_context_switches) {
 	Eterm cs = erts_make_integer(erts_get_total_context_switches(), BIF_P);
 	hp = HAlloc(BIF_P, 3);
 	res = TUPLE2(hp, cs, SMALL_ZERO);
@@ -4160,6 +4167,8 @@ erts_bif_info_init(void)
 
     alloc_info_trap = erts_export_put(am_erlang, am_alloc_info, 1);
     alloc_sizes_trap = erts_export_put(am_erlang, am_alloc_sizes, 1);
+    gather_sched_wall_time_res_trap
+	= erts_export_put(am_erlang, am_gather_sched_wall_time_result, 1);
     process_info_init();
     os_info_init();
 }
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index b8c6b64fc0..c6226cd9d0 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -537,6 +537,209 @@ erts_late_init_process(void)
 
 }
 
+static void
+init_sched_wall_time(ErtsSchedWallTime *swtp)
+{
+    swtp->enabled = 0;
+    swtp->start = 0;
+    swtp->working.total = 0;
+    swtp->working.start = 0;
+    swtp->working.currently = 0;
+}
+
+static ERTS_INLINE Uint64
+sched_wall_time_ts(void)
+{
+#ifdef HAVE_GETHRTIME
+    return (Uint64) sys_gethrtime();
+#else
+    Uint64 res;
+    SysTimeval tv;
+    sys_gettimeofday(&tv);
+    res = (Uint64) tv.tv_sec*1000000;
+    res += (Uint64) tv.tv_usec;
+    return res;
+#endif
+}
+
+static ERTS_INLINE void
+sched_wall_time_change(ErtsSchedulerData *esdp, int working)
+{
+    if (esdp->sched_wall_time.enabled) {
+	Uint64 ts = sched_wall_time_ts();
+	if (working) {
+#ifdef DEBUG
+	    ASSERT(!esdp->sched_wall_time.working.currently);
+	    esdp->sched_wall_time.working.currently = 1;
+#endif
+	    ts -= esdp->sched_wall_time.start;
+	    esdp->sched_wall_time.working.start = ts;
+	}
+	else {
+#ifdef DEBUG
+	    ASSERT(esdp->sched_wall_time.working.currently);
+	    esdp->sched_wall_time.working.currently = 0;
+#endif
+	    ts -= esdp->sched_wall_time.start;
+	    ts -= esdp->sched_wall_time.working.start;
+	    esdp->sched_wall_time.working.total += ts;
+	}
+    }
+}
+
+typedef struct {
+    int set;
+    int enable;
+    Process *proc;
+    Eterm ref;
+    Eterm ref_heap[REF_THING_SIZE];
+    Uint req_sched;
+    erts_smp_atomic32_t refc;
+} ErtsSchedWallTimeReq;
+
+#if !HALFWORD_HEAP
+ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(swtreq,
+				 ErtsSchedWallTimeReq,
+				 5,
+				 ERTS_ALC_T_SCHED_WTIME_REQ)
+#else
+static ERTS_INLINE ErtsSchedWallTimeReq *
+swtreq_alloc(void)
+{
+    return erts_alloc(ERTS_ALC_T_SCHED_WTIME_REQ,
+		      sizeof(ErtsSchedWallTimeReq));
+}
+
+static ERTS_INLINE void
+swtreq_free(ErtsSchedWallTimeReq *ptr)
+{
+    erts_free(ERTS_ALC_T_SCHED_WTIME_REQ, ptr);
+}
+#endif
+
+static void
+reply_sched_wall_time(void *vswtrp)
+{
+    Uint64 working = 0, total = 0;
+    ErtsSchedulerData *esdp = erts_get_scheduler_data();
+    ErtsSchedWallTimeReq *swtrp = (ErtsSchedWallTimeReq *) vswtrp;
+    ErtsProcLocks rp_locks = (swtrp->req_sched == esdp->no
+			      ? ERTS_PROC_LOCK_MAIN
+			      : 0);
+    Process *rp = swtrp->proc;
+    Eterm ref_copy = NIL, msg;
+    Eterm *hp = NULL;
+    Eterm **hpp;
+    Uint sz, *szp;
+    ErlOffHeap *ohp = NULL;
+    ErlHeapFragment *bp = NULL;
+
+    ASSERT(esdp);
+    
+    if (swtrp->set) {
+	if (!swtrp->enable && esdp->sched_wall_time.enabled)
+	    esdp->sched_wall_time.enabled = 0;
+	else if (swtrp->enable && !esdp->sched_wall_time.enabled) {
+	    Uint64 ts = sched_wall_time_ts();
+	    esdp->sched_wall_time.enabled = 1;
+	    esdp->sched_wall_time.start = ts;
+	    esdp->sched_wall_time.working.total = 0;
+	    esdp->sched_wall_time.working.start = 0;
+	    esdp->sched_wall_time.working.currently = 1;
+	}
+    }
+
+    if (esdp->sched_wall_time.enabled) {
+	Uint64 ts = sched_wall_time_ts();
+	ASSERT(esdp->sched_wall_time.working.currently);
+	ts -= esdp->sched_wall_time.start;
+	total = ts;
+	ts -= esdp->sched_wall_time.working.start;
+	working = esdp->sched_wall_time.working.total + ts;
+    }
+
+    sz = 0;
+    hpp = NULL;
+    szp = &sz;
+
+    while (1) {
+	if (hpp)
+	    ref_copy = STORE_NC(hpp, ohp, swtrp->ref);
+	else
+	    *szp += REF_THING_SIZE;
+
+	if (swtrp->set)
+	    msg = ref_copy;
+	else {
+	    msg = (!esdp->sched_wall_time.enabled
+		   ? am_notsup
+		   : erts_bld_tuple(hpp, szp, 3,
+				    make_small(esdp->no),
+				    erts_bld_uint64(hpp, szp, working),
+				    erts_bld_uint64(hpp, szp, total)));
+
+	    msg = erts_bld_tuple(hpp, szp, 2, ref_copy, msg);
+	}
+	if (hpp)
+	    break;
+
+	hp = erts_alloc_message_heap(sz, &bp, &ohp, rp, &rp_locks);
+	szp = NULL;
+	hpp = &hp;
+    }
+
+    erts_queue_message(rp, &rp_locks, bp, msg, NIL);
+
+    if (swtrp->req_sched == esdp->no)
+	rp_locks &= ~ERTS_PROC_LOCK_MAIN;
+ 
+    if (rp_locks)
+	erts_smp_proc_unlock(rp, rp_locks);
+
+    erts_smp_proc_dec_refc(rp);
+
+    if (erts_smp_atomic32_dec_read_nob(&swtrp->refc) == 0)
+	swtreq_free(vswtrp);
+}
+
+Eterm
+erts_sched_wall_time_request(Process *c_p, int set, int enable)
+{
+    ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(c_p);
+    Eterm ref;
+    ErtsSchedWallTimeReq *swtrp;
+    Eterm *hp;
+
+    if (!set && !esdp->sched_wall_time.enabled)
+	return THE_NON_VALUE;
+
+    swtrp = swtreq_alloc();
+    ref = erts_make_ref(c_p);
+    hp = &swtrp->ref_heap[0];
+
+    swtrp->set = set;
+    swtrp->enable = enable;
+    swtrp->proc = c_p;
+    swtrp->ref = STORE_NC(&hp, NULL, ref);
+    swtrp->req_sched = esdp->no;
+    erts_smp_atomic32_init_nob(&swtrp->refc,
+			       (erts_aint32_t) erts_no_schedulers);
+
+    erts_smp_proc_add_refc(c_p, (Sint32) erts_no_schedulers);
+
+#ifdef ERTS_SMP
+    if (erts_no_schedulers > 1)
+	erts_schedule_multi_misc_aux_work(1,
+					  erts_no_schedulers,
+					  reply_sched_wall_time,
+					  (void *) swtrp);
+#endif
+
+    reply_sched_wall_time((void *) swtrp);
+
+    return ref;
+}
+
 static ERTS_INLINE ErtsProcList *
 proclist_create(Process *p)
 {
@@ -1707,6 +1910,7 @@ aux_thread(void *unused)
 static void
 scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 {
+    int working = 1;
     ErtsSchedulerSleepInfo *ssi = esdp->ssi;
     int spincount;
     erts_aint32_t aux_work = 0;
@@ -1737,12 +1941,17 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 
     tse_wait:
 
+	if (thr_prgr_active != working)
+	    sched_wall_time_change(esdp, thr_prgr_active);
+
 	while (1) {
 
 	    aux_work = erts_atomic32_read_acqb(&ssi->aux_work);
 	    if (aux_work) {
-		if (!thr_prgr_active)
+		if (!thr_prgr_active) {
 		    erts_thr_progress_active(esdp, thr_prgr_active = 1);
+		    sched_wall_time_change(esdp, 1);
+		}
 		aux_work = handle_aux_work(&esdp->aux_work_data, aux_work);
 		if (aux_work && erts_thr_progress_update(esdp))
 		    erts_thr_progress_leader_update(esdp);
@@ -1751,8 +1960,10 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 	    if (aux_work)
 		flgs = erts_smp_atomic32_read_acqb(&ssi->flags);
 	    else {
-		if (thr_prgr_active)
+		if (thr_prgr_active) {
 		    erts_thr_progress_active(esdp, thr_prgr_active = 0);
+		    sched_wall_time_change(esdp, 0);
+		}
 		erts_thr_progress_prepare_wait(esdp);
 
 		flgs = sched_spin_wait(ssi, spincount);
@@ -1789,8 +2000,10 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 	if (flgs & ~ERTS_SSI_FLG_SUSPENDED)
 	    erts_smp_atomic32_read_band_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED);
 
-	if (!thr_prgr_active)
+	if (!thr_prgr_active) {
 	    erts_thr_progress_active(esdp, thr_prgr_active = 1);
+	    sched_wall_time_change(esdp, 1);
+	}
 
 	erts_smp_runq_lock(rq);
 	sched_active(esdp->no, rq);
@@ -1806,14 +2019,21 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 
 	sched_waiting_sys(esdp->no, rq);
 
+
 	erts_smp_runq_unlock(rq);
 
+	ASSERT(working);
+	sched_wall_time_change(esdp, working = 0);
+
 	spincount = ERTS_SCHED_SYS_SLEEP_SPINCOUNT;
 
 	while (spincount-- > 0) {
 
 	sys_poll_aux_work:
 
+	    if (working)
+		sched_wall_time_change(esdp, working = 0);
+
 	    ASSERT(!erts_port_task_have_outstanding_io_tasks());
 
 	    erl_sys_schedule(1); /* Might give us something to do */
@@ -1828,6 +2048,8 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 
 	    aux_work = erts_atomic32_read_acqb(&ssi->aux_work);
 	    if (aux_work) {
+		if (!working)
+		    sched_wall_time_change(esdp, working = 1);
 #ifdef ERTS_SMP
 		if (!thr_prgr_active)
 		    erts_thr_progress_active(esdp, thr_prgr_active = 1);
@@ -1920,6 +2142,9 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 
 	erts_smp_runq_unlock(rq);
 
+	if (working)
+	    sched_wall_time_change(esdp, working = 0);
+
 #ifdef ERTS_SMP
 	if (thr_prgr_active)
 	    erts_thr_progress_active(esdp, thr_prgr_active = 0);
@@ -1955,6 +2180,8 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 	if (flgs & ~ERTS_SSI_FLG_SUSPENDED)
 	    erts_smp_atomic32_read_band_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED);
 #endif
+	if (!working)
+	    sched_wall_time_change(esdp, working = 1);
 	sched_active_sys(esdp->no, rq);
     }
 
@@ -3430,9 +3657,14 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online)
 	esdp->run_queue->scheduler = esdp;
 
 	init_aux_work_data(&esdp->aux_work_data, esdp);
+	init_sched_wall_time(&esdp->sched_wall_time);
     }
 
     init_misc_aux_work();
+#if !HALFWORD_HEAP
+    init_swtreq_alloc();
+#endif
+
 
 #ifdef ERTS_SMP
 
@@ -3755,6 +3987,8 @@ suspend_scheduler(ErtsSchedulerData *esdp)
     if (erts_system_profile_flags.scheduler)
     	profile_scheduler(make_small(esdp->no), am_inactive);
 
+    sched_wall_time_change(esdp, 0);
+
     erts_smp_mtx_lock(&schdlr_sspnd.mtx);
 
     flgs = sched_prep_spin_suspended(ssi, ERTS_SSI_FLG_SUSPENDED);
@@ -3813,16 +4047,20 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 
 		aux_work = erts_atomic32_read_acqb(&ssi->aux_work);
 		if (aux_work) {
-		    if (!thr_prgr_active)
+		    if (!thr_prgr_active) {
 			erts_thr_progress_active(esdp, thr_prgr_active = 1);
+			sched_wall_time_change(esdp, 1);
+		    }
 		    aux_work = handle_aux_work(&esdp->aux_work_data, aux_work);
 		    if (aux_work && erts_thr_progress_update(esdp))
 			erts_thr_progress_leader_update(esdp);
 		}
 
 		if (!aux_work) {
-		    if (thr_prgr_active)
+		    if (thr_prgr_active) {
 			erts_thr_progress_active(esdp, thr_prgr_active = 0);
+			sched_wall_time_change(esdp, 0);
+		    }
 		    erts_thr_progress_prepare_wait(esdp);
 		    flgs = sched_spin_suspended(ssi,
 						ERTS_SCHED_SUSPEND_SLEEP_SPINCOUNT);
@@ -3877,8 +4115,10 @@ suspend_scheduler(ErtsSchedulerData *esdp)
     if (erts_system_profile_flags.scheduler)
     	profile_scheduler(make_small(esdp->no), am_active);
 
-    if (!thr_prgr_active)
+    if (!thr_prgr_active) {
 	erts_thr_progress_active(esdp, thr_prgr_active = 1);
+	sched_wall_time_change(esdp, 1);
+    }
 
     erts_smp_runq_lock(esdp->run_queue);
     non_empty_runq(esdp->run_queue);
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index a51b380bb0..173b7df69d 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -394,6 +394,16 @@ do {								\
 } while (0)
 
 typedef struct {
+    int enabled;
+    Uint64 start;
+    struct {
+	Uint64 total;
+	Uint64 start;
+	int currently;
+    } working;
+} ErtsSchedWallTime;
+
+typedef struct {
     int sched_id;
     ErtsSchedulerData *esdp;
     ErtsSchedulerSleepInfo *ssi;
@@ -457,6 +467,8 @@ struct ErtsSchedulerData_ {
 
     ErtsSchedAllocData alloc_data;
 
+    ErtsSchedWallTime sched_wall_time;
+
 #ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
     erts_alloc_verify_func_t verify_unused_temp_alloc;
     Allctr_t *verify_unused_temp_alloc_data;
@@ -1064,6 +1076,8 @@ void erts_late_init_process(void);
 void erts_early_init_scheduling(int);
 void erts_init_scheduling(int, int);
 
+Eterm erts_sched_wall_time_request(Process *c_p, int set, int enable);
+
 ErtsProcList *erts_proclist_create(Process *);
 void erts_proclist_destroy(ErtsProcList *);
 int erts_proclist_same(ErtsProcList *, Process *);
diff --git a/erts/preloaded/ebin/erlang.beam b/erts/preloaded/ebin/erlang.beam
index 9e369d5348..b78747bc84 100644
--- a/erts/preloaded/ebin/erlang.beam
+++ b/erts/preloaded/ebin/erlang.beam
diff --git a/erts/preloaded/src/erlang.erl b/erts/preloaded/src/erlang.erl
index 4affc9bffe..4cbff3f36e 100644
--- a/erts/preloaded/src/erlang.erl
+++ b/erts/preloaded/src/erlang.erl
@@ -43,6 +43,9 @@
 -export([memory/0, memory/1]).
 -export([alloc_info/1, alloc_sizes/1]).
 
+-export([gather_sched_wall_time_result/1,
+	 await_sched_wall_time_modifications/2]).
+
 -deprecated([hash/2]).
 
 % Get rid of autoimports of spawn to avoid clashes with ourselves.
@@ -1202,3 +1205,34 @@ receive_allocator(Ref, N, Acc) ->
 	{Ref, _, InfoList} ->
 	    receive_allocator(Ref, N-1, insert_info(InfoList, Acc))
     end.
+
+-spec await_sched_wall_time_modifications(Ref, Result) -> boolean() when
+      Ref :: reference(),
+      Result :: boolean().
+
+await_sched_wall_time_modifications(Ref, Result) ->
+    sched_wall_time(Ref, erlang:system_info(schedulers)),
+    Result.
+
+-spec gather_sched_wall_time_result(Ref) -> [{pos_integer(),
+					      non_neg_integer(),
+					      non_neg_integer()}] when
+      Ref :: reference().
+
+gather_sched_wall_time_result(Ref) when is_reference(Ref) ->
+    sched_wall_time(Ref, erlang:system_info(schedulers), []).
+
+sched_wall_time(_Ref, 0) ->
+    ok;
+sched_wall_time(Ref, N) ->
+    receive Ref -> sched_wall_time(Ref, N-1) end.
+
+sched_wall_time(_Ref, 0, Acc) ->
+    Acc;
+sched_wall_time(Ref, N, undefined) ->
+    receive {Ref, _} -> sched_wall_time(Ref, N-1, undefined) end;
+sched_wall_time(Ref, N, Acc) ->
+    receive
+	{Ref, undefined} -> sched_wall_time(Ref, N-1, undefined);
+	{Ref, SWT} -> sched_wall_time(Ref, N-1, [SWT|Acc])
+    end.