Optimized timer implementation

1 files changed, 49 insertions, 42 deletions

diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 743711cc3b..b1c30e7652 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -52,7 +52,7 @@ typedef struct process Process;
 #include "erl_node_container_utils.h"
 #include "erl_node_tables.h"
 #include "erl_monitors.h"
-#include "erl_bif_timer.h"
+#include "erl_hl_timer.h"
 #include "erl_time.h"
 #include "erl_atom_table.h"
 #include "external.h"
@@ -278,16 +278,18 @@ typedef enum {
 #define ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC	(((erts_aint32_t) 1) << 3)
 #define ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM	(((erts_aint32_t) 1) << 4)
 #define ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP	(((erts_aint32_t) 1) << 5)
-#define ERTS_SSI_AUX_WORK_ASYNC_READY		(((erts_aint32_t) 1) << 6)
-#define ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN	(((erts_aint32_t) 1) << 7)
-#define ERTS_SSI_AUX_WORK_MISC_THR_PRGR		(((erts_aint32_t) 1) << 8)
-#define ERTS_SSI_AUX_WORK_MISC			(((erts_aint32_t) 1) << 9)
-#define ERTS_SSI_AUX_WORK_CHECK_CHILDREN	(((erts_aint32_t) 1) << 10)
-#define ERTS_SSI_AUX_WORK_SET_TMO		(((erts_aint32_t) 1) << 11)
-#define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK	(((erts_aint32_t) 1) << 12)
-#define ERTS_SSI_AUX_WORK_REAP_PORTS		(((erts_aint32_t) 1) << 13)
-
-#define ERTS_SSI_AUX_WORK_MAX                                           14
+#define ERTS_SSI_AUX_WORK_CNCLD_TMRS		(((erts_aint32_t) 1) << 6)
+#define ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR	(((erts_aint32_t) 1) << 7)
+#define ERTS_SSI_AUX_WORK_ASYNC_READY		(((erts_aint32_t) 1) << 8)
+#define ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN	(((erts_aint32_t) 1) << 9)
+#define ERTS_SSI_AUX_WORK_MISC_THR_PRGR		(((erts_aint32_t) 1) << 10)
+#define ERTS_SSI_AUX_WORK_MISC			(((erts_aint32_t) 1) << 11)
+#define ERTS_SSI_AUX_WORK_CHECK_CHILDREN	(((erts_aint32_t) 1) << 12)
+#define ERTS_SSI_AUX_WORK_SET_TMO		(((erts_aint32_t) 1) << 13)
+#define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK	(((erts_aint32_t) 1) << 14)
+#define ERTS_SSI_AUX_WORK_REAP_PORTS		(((erts_aint32_t) 1) << 15)
+
+#define ERTS_SSI_AUX_WORK_MAX                                           16
 
 typedef struct ErtsSchedulerSleepInfo_ ErtsSchedulerSleepInfo;
 
@@ -463,19 +465,21 @@ typedef union {
 
 extern ErtsAlignedRunQueue *erts_aligned_run_queues;
 
-#define ERTS_PROC_REDUCTIONS_EXECUTED(RQ, PRIO, REDS, AREDS)	\
+#define ERTS_PROC_REDUCTIONS_EXECUTED(SD, RQ, PRIO, REDS, AREDS)\
 do {								\
     (RQ)->procs.reductions += (AREDS);				\
     (RQ)->procs.prio_info[(PRIO)].reds += (REDS);		\
     (RQ)->check_balance_reds -= (REDS);				\
     (RQ)->wakeup_other_reds += (AREDS);				\
+    (SD)->check_time_reds += (AREDS);				\
 } while (0)
 
-#define ERTS_PORT_REDUCTIONS_EXECUTED(RQ, REDS)			\
+#define ERTS_PORT_REDUCTIONS_EXECUTED(SD, RQ, REDS)		\
 do {								\
     (RQ)->ports.info.reds += (REDS);				\
     (RQ)->check_balance_reds -= (REDS);				\
     (RQ)->wakeup_other_reds += (REDS);				\
+    (SD)->check_time_reds += (REDS);				\
 } while (0)
 
 typedef struct {
@@ -516,6 +520,9 @@ typedef struct {
     } dd;
     struct {
 	ErtsThrPrgrVal thr_prgr;
+    } cncld_tmrs;
+    struct {
+	ErtsThrPrgrVal thr_prgr;
 	UWord size;
 	ErtsThrPrgrLaterOp *first;
 	ErtsThrPrgrLaterOp *last;
@@ -566,6 +573,7 @@ struct ErtsSchedulerData_ {
 
     ErtsTimerWheel *timer_wheel;
     ErtsNextTimeoutRef next_tmo_ref;
+    ErtsHLTimerService *timer_service;
 #ifdef ERTS_SMP
     ethr_tid tid;		/* Thread id */
     struct erl_bits_state erl_bits_state; /* erl_bits.c state */
@@ -592,6 +600,9 @@ struct ErtsSchedulerData_ {
     ErtsAuxWorkData aux_work_data;
     ErtsAtomCacheMap atom_cache_map;
 
+    ErtsMonotonicTime last_monotonic_time;
+    int check_time_reds;
+
     Uint32 thr_id;
     Uint64 unique;
     Uint64 ref;
@@ -913,10 +924,8 @@ struct process {
 
     ErlMessageQueue msg;	/* Message queue */
 
-    union {
-	ErtsBifTimer *bif_timers;	/* Bif timers aiming at this process */
-	void *terminate;
-    } u;
+    ErtsBifTimers *bif_timers;	/* Bif timers aiming at this process */
+    ErtsBifTimers *accessor_bif_timers;	/* Accessor bif timers */
 
     ProcDict  *dictionary;       /* Process dictionary, may be NULL */
 
@@ -927,9 +936,12 @@ struct process {
 #ifdef USE_VM_PROBES
     Eterm dt_utag;              /* Place to store the dynamc trace user tag */
     Uint dt_utag_flags;         /* flag field for the dt_utag */
-#endif       
-    BeamInstr initial[3];	/* Initial module(0), function(1), arity(2), often used instead
+#endif
+    union {
+	void *terminate;
+	BeamInstr initial[3];	/* Initial module(0), function(1), arity(2), often used instead
 				   of pointer to funcinfo instruction, hence the BeamInstr datatype */
+    } u;
     BeamInstr* current;		/* Current Erlang function, part of the funcinfo:
 				 * module(0), function(1), arity(2)
 				 * (module and functions are tagged atoms;
@@ -975,7 +987,6 @@ struct process {
     ErtsSchedulerData *scheduler_data;
     Eterm suspendee;
     ErtsPendingSuspend *pending_suspenders;
-    ErtsRunQueue *preferred_run_queue;
     erts_smp_atomic_t run_queue;
 #ifdef HIPE
     struct hipe_process_state_smp hipe_smp;
@@ -1085,15 +1096,14 @@ void erts_check_for_holes(Process* p);
 #define ERTS_PSFLG_RUNNING_SYS		ERTS_PSFLG_BIT(15)
 #define ERTS_PSFLG_PROXY		ERTS_PSFLG_BIT(16)
 #define ERTS_PSFLG_DELAYED_SYS		ERTS_PSFLG_BIT(17)
-#define ERTS_PSFLG_OFF_HEAP_MSGS	ERTS_PSFLG_BIT(18)
 #ifdef ERTS_DIRTY_SCHEDULERS
-#define ERTS_PSFLG_DIRTY_CPU_PROC	ERTS_PSFLG_BIT(19)
-#define ERTS_PSFLG_DIRTY_IO_PROC	ERTS_PSFLG_BIT(20)
-#define ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q	ERTS_PSFLG_BIT(21)
-#define ERTS_PSFLG_DIRTY_IO_PROC_IN_Q	ERTS_PSFLG_BIT(22)
-#define ERTS_PSFLG_MAX  (ERTS_PSFLGS_ZERO_BIT_OFFSET + 23)
+#define ERTS_PSFLG_DIRTY_CPU_PROC	ERTS_PSFLG_BIT(18)
+#define ERTS_PSFLG_DIRTY_IO_PROC	ERTS_PSFLG_BIT(19)
+#define ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q	ERTS_PSFLG_BIT(20)
+#define ERTS_PSFLG_DIRTY_IO_PROC_IN_Q	ERTS_PSFLG_BIT(21)
+#define ERTS_PSFLG_MAX  (ERTS_PSFLGS_ZERO_BIT_OFFSET + 22)
 #else
-#define ERTS_PSFLG_MAX  (ERTS_PSFLGS_ZERO_BIT_OFFSET + 19)
+#define ERTS_PSFLG_MAX  (ERTS_PSFLGS_ZERO_BIT_OFFSET + 18)
 #endif
 
 #define ERTS_PSFLGS_IN_PRQ_MASK 	(ERTS_PSFLG_IN_PRQ_MAX		\
@@ -1112,7 +1122,6 @@ void erts_check_for_holes(Process* p);
  * Static flags that do not change after process creation.
  */
 #define ERTS_STC_FLG_SYSTEM_PROC	(((Uint32) 1) << 0)
-#define ERTS_STC_FLG_PREFER_SCHED	(((Uint32) 1) << 1)
 
 /* The sequential tracing token is a tuple of size 5:
  *
@@ -1141,9 +1150,7 @@ void erts_check_for_holes(Process* p);
 #define SPO_LINK 1
 #define SPO_USE_ARGS 2
 #define SPO_MONITOR 4
-#define SPO_OFF_HEAP_MSGS 8
-#define SPO_SYSTEM_PROC 16
-#define SPO_PREFER_SCHED 32
+#define SPO_SYSTEM_PROC 8
 
 /*
  * The following struct contains options for a process to be spawned.
@@ -1231,7 +1238,6 @@ extern struct erts_system_profile_flags_t erts_system_profile_flags;
 #define F_P2PNR_RESCHED      (1 <<  9) /* Process has been rescheduled via erts_pid2proc_not_running() */
 #define F_FORCE_GC           (1 << 10) /* Force gc at process in-scheduling */
 #define F_DISABLE_GC         (1 << 11) /* Disable GC */
-#define F_OFF_HEAP_MSGS      (1 << 12)
 
 /* process trace_flags */
 #define F_SENSITIVE          (1 << 0)
@@ -1267,8 +1273,6 @@ extern struct erts_system_profile_flags_t erts_system_profile_flags;
 #  define F_INITIAL_TRACE_FLAGS 0
 #endif
 
-
-
 #define TRACEE_FLAGS ( F_TRACE_PROCS | F_TRACE_CALLS \
 		     | F_TRACE_SOS |  F_TRACE_SOS1| F_TRACE_RECEIVE  \
 		     | F_TRACE_SOL | F_TRACE_SOL1 | F_TRACE_SEND \
@@ -1302,12 +1306,14 @@ extern struct erts_system_profile_flags_t erts_system_profile_flags;
 #define ERTS_XSIG_FLG_IGN_KILL		(((Uint32) 1) << 0)
 #define ERTS_XSIG_FLG_NO_IGN_NORMAL	(((Uint32) 1) << 1)
 
-#define CANCEL_TIMER(p) \
-    do { \
-	if ((p)->flags & (F_INSLPQUEUE)) \
-	    cancel_timer(p); \
-	else \
-	    (p)->flags &= ~F_TIMO; \
+#define CANCEL_TIMER(P)					\
+    do {						\
+	if ((P)->flags & (F_INSLPQUEUE|F_TIMO)) {	\
+	    if ((P)->flags & F_INSLPQUEUE)		\
+		erts_cancel_proc_timer((P));		\
+	    else					\
+		(P)->flags &= ~F_TIMO;			\
+	}						\
     } while (0)
 
 #if defined(ERTS_DIRTY_SCHEDULERS) && defined(ERTS_SMP)
@@ -1594,6 +1600,9 @@ Eterm erts_multi_scheduling_blockers(Process *);
 void erts_start_schedulers(void);
 void erts_alloc_notify_delayed_dealloc(int);
 void erts_alloc_ensure_handle_delayed_dealloc_call(int);
+#ifdef ERTS_SMP
+void erts_notify_canceled_timer(ErtsSchedulerData *, int);
+#endif
 void erts_smp_notify_check_children_needed(void);
 #endif
 #if ERTS_USE_ASYNC_READY_Q
@@ -1628,8 +1637,6 @@ void erts_schedule_misc_op(void (*)(void *), void *);
 Eterm erl_create_process(Process*, Eterm, Eterm, Eterm, ErlSpawnOpts*);
 void erts_do_exit_process(Process*, Eterm);
 void erts_continue_exit_process(Process *);
-void set_timer(Process*, Uint);
-void cancel_timer(Process*);
 /* Begin System profile */
 Uint erts_runnable_process_count(void);
 /* End System profile */