Be less eager requesting wakeup for cleanup jobs

10 files changed, 177 insertions, 46 deletions

diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index d0afc8ed8d..98c2988323 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -264,9 +264,10 @@ static void schedule_free_dbtable(DbTable* tb)
      *               function has returned).
      */
     ASSERT(erts_refc_read(&tb->common.ref, 0) == 0);
-    erts_schedule_thr_prgr_later_op(free_dbtable,
-				    (void *) tb,
-				    &tb->release.data);
+    erts_schedule_thr_prgr_later_cleanup_op(free_dbtable,
+					    (void *) tb,
+					    &tb->release.data,
+					    sizeof(DbTable));
 }
 
 static ERTS_INLINE void db_init_lock(DbTable* tb, int use_frequent_read_lock,
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index e7e4030900..86adc5bc99 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -532,6 +532,8 @@ void erts_usage(void)
     erts_fprintf(stderr, "            see the erl(1) documentation for more info.\n");
     erts_fprintf(stderr, "-sws val    set scheduler wakeup strategy, valid values are:\n");
     erts_fprintf(stderr, "            default|legacy.\n");
+    erts_fprintf(stderr, "-swct val   set scheduler wake cleanup threshold, valid values are:\n");
+    erts_fprintf(stderr, "            very_lazy|lazy|medium|eager|very_eager.\n");
     erts_fprintf(stderr, "-swt val    set scheduler wakeup threshold, valid values are:\n");
     erts_fprintf(stderr, "            very_low|low|medium|high|very_high.\n");
     erts_fprintf(stderr, "-sss size   suggested stack size in kilo words for scheduler threads,\n");
@@ -1413,6 +1415,16 @@ erl_start(int argc, char **argv)
 		    erts_usage();
 		}
 	    }
+	    else if (has_prefix("wct", sub_param)) {
+		arg = get_arg(sub_param+3, argv[i+1], &i);
+		if (erts_sched_set_wake_cleanup_threshold(arg) != 0) {
+		    erts_fprintf(stderr, "scheduler wake cleanup threshold: %s\n",
+				 arg);
+		    erts_usage();
+		}
+		VERBOSE(DEBUG_SYSTEM,
+			("scheduler wake cleanup threshold: %s\n", arg));
+	    }
 	    else if (sys_strcmp("wt", sub_param) == 0) {
 		arg = get_arg(sub_param+2, argv[i+1], &i);
 		if (erts_sched_set_wakeup_other_thresold(arg) != 0) {
diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c
index ce045ec94e..23000391ec 100644
--- a/erts/emulator/beam/erl_port_task.c
+++ b/erts/emulator/beam/erl_port_task.c
@@ -151,9 +151,10 @@ static ERTS_INLINE void
 schedule_port_task_free(ErtsPortTask *ptp)
 {
 #ifdef ERTS_SMP
-    erts_schedule_thr_prgr_later_op(call_port_task_free,
-				    (void *) ptp,
-				    &ptp->u.release);
+    erts_schedule_thr_prgr_later_cleanup_op(call_port_task_free,
+					    (void *) ptp,
+					    &ptp->u.release,
+					    sizeof(ErtsPortTask));
 #else
     port_task_free(ptp);
 #endif
@@ -772,9 +773,10 @@ static void
 schedule_port_task_handle_list_free(ErtsPortTaskHandleList *pthlp)
 {
 #ifdef ERTS_SMP
-    erts_schedule_thr_prgr_later_op(free_port_task_handle_list,
-				    (void *) pthlp,
-				    &pthlp->u.release);
+    erts_schedule_thr_prgr_later_cleanup_op(free_port_task_handle_list,
+					    (void *) pthlp,
+					    &pthlp->u.release,
+					    sizeof(ErtsPortTaskHandleList));
 #else
     erts_free(ERTS_ALC_T_PT_HNDL_LIST, pthlp);
 #endif
@@ -1999,9 +2001,10 @@ begin_port_cleanup(Port *pp, ErtsPortTask **execqp, int *processing_busy_q_p)
      * Schedule cleanup of port structure...
      */
 #ifdef ERTS_SMP
-    erts_schedule_thr_prgr_later_op(release_port,
-				    (void *) pp,
-				    &pp->common.u.release);
+    erts_schedule_thr_prgr_later_cleanup_op(release_port,
+					    (void *) pp,
+					    &pp->common.u.release,
+					    sizeof(Port));
 #else
     pp->cleanup = 1;
 #endif
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 00247b387a..7e0884f55f 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -146,6 +146,14 @@ extern BeamInstr beam_continue_exit[];
 int erts_sched_compact_load;
 Uint erts_no_schedulers;
 
+#define ERTS_THR_PRGR_LATER_CLEANUP_OP_THRESHOLD_VERY_LAZY		(4*1024*1024)
+#define ERTS_THR_PRGR_LATER_CLEANUP_OP_THRESHOLD_LAZY			(512*1024)
+#define ERTS_THR_PRGR_LATER_CLEANUP_OP_THRESHOLD_MEDIUM			(64*1024)
+#define ERTS_THR_PRGR_LATER_CLEANUP_OP_THRESHOLD_EAGER			(16*1024)
+#define ERTS_THR_PRGR_LATER_CLEANUP_OP_THRESHOLD_VERY_EAGER		(1024)
+
+static UWord thr_prgr_later_cleanup_op_threshold = ERTS_THR_PRGR_LATER_CLEANUP_OP_THRESHOLD_MEDIUM;
+
 ErtsPTab erts_proc erts_align_attribute(ERTS_CACHE_LINE_SIZE);
 
 int erts_sched_thread_suggested_stack_size = -1;
@@ -496,6 +504,7 @@ erts_init_process(int ncpu, int proc_tab_size)
 #endif
 			 (ErtsPTabElementCommon *) &erts_invalid_process.common,
 			 proc_tab_size,
+			 sizeof(Process),
 			 "process_table");
 
     last_reductions = 0;
@@ -909,6 +918,53 @@ unset_aux_work_flags(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flgs)
 #ifdef ERTS_SMP
 
 static ERTS_INLINE void
+haw_chk_later_cleanup_op_wakeup(ErtsAuxWorkData *awdp, ErtsThrPrgrVal val)
+{
+    if (awdp->later_op.first
+	&& erts_thr_progress_cmp(val, awdp->later_op.thr_prgr) >= 0) {
+	awdp->later_op.size = thr_prgr_later_cleanup_op_threshold;
+    }
+}
+
+static ERTS_INLINE void
+haw_thr_prgr_wakeup(ErtsAuxWorkData *awdp, ErtsThrPrgrVal val)
+{
+    int cmp = erts_thr_progress_cmp(val, awdp->latest_wakeup);
+    if (cmp != 0) {
+	if (cmp > 0) {
+	    awdp->latest_wakeup = val;
+	    haw_chk_later_cleanup_op_wakeup(awdp, val);
+	}
+	erts_thr_progress_wakeup(awdp->esdp, val);
+    }
+}
+
+static ERTS_INLINE void
+haw_thr_prgr_soft_wakeup(ErtsAuxWorkData *awdp, ErtsThrPrgrVal val)
+{
+    if (erts_thr_progress_cmp(val, awdp->latest_wakeup) > 0) {
+	awdp->latest_wakeup = val;
+	haw_chk_later_cleanup_op_wakeup(awdp, val);
+	erts_thr_progress_wakeup(awdp->esdp, val);
+    }
+}
+
+static ERTS_INLINE void
+haw_thr_prgr_later_cleanup_op_wakeup(ErtsAuxWorkData *awdp, ErtsThrPrgrVal val, UWord size)
+{
+    if (erts_thr_progress_cmp(val, awdp->latest_wakeup) > 0) {
+	awdp->later_op.thr_prgr = val;
+	if (awdp->later_op.size > size)
+	    awdp->later_op.size -= size;
+	else {
+	    awdp->latest_wakeup = val;
+	    awdp->later_op.size = thr_prgr_later_cleanup_op_threshold;
+	    erts_thr_progress_wakeup(awdp->esdp, val);
+	}
+    }
+}
+
+static ERTS_INLINE void
 haw_thr_prgr_current_reset(ErtsAuxWorkData *awdp)
 {
     awdp->current_thr_prgr = ERTS_THR_PRGR_INVALID;
@@ -1066,8 +1122,7 @@ misc_aux_work_clean(ErtsThrQ_t *q,
     case ERTS_THR_Q_NEED_THR_PRGR:
 #ifdef ERTS_SMP
 	set_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_MISC_THR_PRGR);
-	erts_thr_progress_wakeup(awdp->esdp,
-				 erts_thr_q_need_thr_progress(q));
+	haw_thr_prgr_soft_wakeup(awdp, erts_thr_q_need_thr_progress(q));
 #endif
     case ERTS_THR_Q_CLEAN:
 	break;
@@ -1225,8 +1280,7 @@ handle_async_ready_clean(ErtsAuxWorkData *awdp,
 	return aux_work & ~ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN;
 #ifdef ERTS_SMP
     case ERTS_ASYNC_READY_NEED_THR_PRGR:
-	erts_thr_progress_wakeup(awdp->esdp,
-				 awdp->async_ready.thr_prgr);
+	haw_thr_prgr_soft_wakeup(awdp, awdp->async_ready.thr_prgr);
 	awdp->async_ready.need_thr_prgr = 1;
 	return aux_work & ~ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN;
 #endif
@@ -1300,7 +1354,7 @@ handle_delayed_dealloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waitin
 	awdp->dd.thr_prgr = wakeup;
 	set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD_THR_PRGR);
 	awdp->dd.thr_prgr = wakeup;
-	erts_thr_progress_wakeup(awdp->esdp, wakeup);
+	haw_thr_prgr_soft_wakeup(awdp, wakeup);
     }
     else if (awdp->dd.completed_callback) {
 	awdp->dd.completed_callback(awdp->dd.completed_arg);
@@ -1341,7 +1395,7 @@ handle_delayed_dealloc_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, i
 	if (wakeup == ERTS_THR_PRGR_INVALID)
 	    wakeup = erts_thr_progress_later(awdp->esdp);
 	awdp->dd.thr_prgr = wakeup;
-	erts_thr_progress_wakeup(awdp->esdp, wakeup);
+	haw_thr_prgr_soft_wakeup(awdp, wakeup);
     }
     else {
 	unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD_THR_PRGR);
@@ -1376,6 +1430,7 @@ handle_thr_prgr_later_op(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int wait
 	}
 	lop->func(lop->data);
 	if (!awdp->later_op.first) {
+	    awdp->later_op.size = thr_prgr_later_cleanup_op_threshold;
 	    awdp->later_op.last = NULL;
 	    unset_aux_work_flags(awdp->ssi,
 				 ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP);
@@ -1386,6 +1441,29 @@ handle_thr_prgr_later_op(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int wait
     return aux_work;
 }
 
+static ERTS_INLINE ErtsThrPrgrVal
+enqueue_later_op(ErtsSchedulerData *esdp,
+		 void (*later_func)(void *),
+		 void *later_data,
+		 ErtsThrPrgrLaterOp *lop)
+{
+    ErtsThrPrgrVal later = erts_thr_progress_later(esdp);
+    ASSERT(esdp);
+
+    lop->func = later_func;
+    lop->data = later_data;
+    lop->later = later;
+    lop->next = NULL;
+    if (!esdp->aux_work_data.later_op.last)
+	esdp->aux_work_data.later_op.first = lop;
+    else
+	esdp->aux_work_data.later_op.last->next = lop;
+    esdp->aux_work_data.later_op.last = lop;
+    set_aux_work_flags_wakeup_nob(esdp->ssi,
+				  ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP);
+    return later;
+}
+
 #endif /* ERTS_SMP */
 
 void
@@ -1396,31 +1474,24 @@ erts_schedule_thr_prgr_later_op(void (*later_func)(void *),
 #ifndef ERTS_SMP
     later_func(later_data);
 #else
-    ErtsSchedulerData *esdp;
-    ErtsAuxWorkData *awdp;
-    int request_wakeup = 1;
-
-    esdp = erts_get_scheduler_data();
-    ASSERT(esdp);
-    awdp = &esdp->aux_work_data;
+    ErtsSchedulerData *esdp = erts_get_scheduler_data();
+    ErtsThrPrgrVal later = enqueue_later_op(esdp, later_func, later_data, lop);
+    haw_thr_prgr_wakeup(&esdp->aux_work_data, later);
+#endif
+}
 
-    lop->func = later_func;
-    lop->data = later_data;
-    lop->later = erts_thr_progress_later(esdp);
-    lop->next = NULL;
-    if (!awdp->later_op.last)
-	awdp->later_op.first = lop;
-    else {
-	ErtsThrPrgrLaterOp *last = awdp->later_op.last;
-	last->next = lop;
-	if (erts_thr_progress_equal(last->later, lop->later))
-	    request_wakeup = 0;
-    }
-    awdp->later_op.last = lop;
-    set_aux_work_flags_wakeup_nob(awdp->ssi,
-				  ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP);
-    if (request_wakeup)
-	erts_thr_progress_wakeup(esdp, lop->later);
+void
+erts_schedule_thr_prgr_later_cleanup_op(void (*later_func)(void *),
+					void *later_data,
+					ErtsThrPrgrLaterOp *lop,
+					UWord size)
+{
+#ifndef ERTS_SMP
+    later_func(later_data);
+#else
+    ErtsSchedulerData *esdp = erts_get_scheduler_data();
+    ErtsThrPrgrVal later = enqueue_later_op(esdp, later_func, later_data, lop);
+    haw_thr_prgr_later_cleanup_op_wakeup(&esdp->aux_work_data, later, size);
 #endif
 }
 
@@ -4358,6 +4429,25 @@ erts_sched_set_busy_wait_threshold(char *str)
 
     return 0;
 }
+
+int
+erts_sched_set_wake_cleanup_threshold(char *str)
+{
+    if (sys_strcmp(str, "very_lazy") == 0)
+	thr_prgr_later_cleanup_op_threshold = ERTS_THR_PRGR_LATER_CLEANUP_OP_THRESHOLD_VERY_LAZY;
+    else if (sys_strcmp(str, "lazy") == 0)
+	thr_prgr_later_cleanup_op_threshold = ERTS_THR_PRGR_LATER_CLEANUP_OP_THRESHOLD_LAZY;
+    else if (sys_strcmp(str, "medium") == 0)
+	thr_prgr_later_cleanup_op_threshold = ERTS_THR_PRGR_LATER_CLEANUP_OP_THRESHOLD_MEDIUM;
+    else if (sys_strcmp(str, "eager") == 0)
+	thr_prgr_later_cleanup_op_threshold = ERTS_THR_PRGR_LATER_CLEANUP_OP_THRESHOLD_EAGER;
+    else if (sys_strcmp(str, "very_eager") == 0)
+	thr_prgr_later_cleanup_op_threshold = ERTS_THR_PRGR_LATER_CLEANUP_OP_THRESHOLD_VERY_EAGER;
+    else
+	return EINVAL;
+    return 0;
+}
+
 static void
 init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp)
 {
@@ -4365,10 +4455,13 @@ init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp)
     awdp->esdp = esdp;
     awdp->ssi = esdp ? esdp->ssi : NULL;
 #ifdef ERTS_SMP
+    awdp->latest_wakeup = ERTS_THR_PRGR_VAL_FIRST;
     awdp->misc.thr_prgr = ERTS_THR_PRGR_VAL_WAITING;
     awdp->dd.thr_prgr = ERTS_THR_PRGR_VAL_WAITING;
     awdp->dd.completed_callback = NULL;
     awdp->dd.completed_arg = NULL;
+    awdp->later_op.thr_prgr = ERTS_THR_PRGR_VAL_FIRST;
+    awdp->later_op.size = 0;
     awdp->later_op.first = NULL;
     awdp->later_op.last = NULL;
 #endif
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 6d1032c292..3d3579fa7e 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -430,6 +430,7 @@ typedef struct {
     ErtsSchedulerSleepInfo *ssi;
 #ifdef ERTS_SMP
     ErtsThrPrgrVal current_thr_prgr;
+    ErtsThrPrgrVal latest_wakeup;
 #endif
     struct {
 	int ix;
@@ -444,6 +445,8 @@ typedef struct {
 	void (*completed_arg)(void *);
     } dd;
     struct {
+	ErtsThrPrgrVal thr_prgr;
+	UWord size;
 	ErtsThrPrgrLaterOp *first;
 	ErtsThrPrgrLaterOp *last;
     } later_op;
@@ -1298,10 +1301,15 @@ ERTS_GLB_INLINE int erts_proclist_is_last(ErtsProcList *list,
 int erts_sched_set_wakeup_other_thresold(char *str);
 int erts_sched_set_wakeup_other_type(char *str);
 int erts_sched_set_busy_wait_threshold(char *str);
+int erts_sched_set_wake_cleanup_threshold(char *);
 
 void erts_schedule_thr_prgr_later_op(void (*)(void *),
 				     void *,
 				     ErtsThrPrgrLaterOp *);
+void erts_schedule_thr_prgr_later_cleanup_op(void (*)(void *),
+					     void *,
+					     ErtsThrPrgrLaterOp *,
+					     UWord);
 
 #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
 int erts_dbg_check_halloc_lock(Process *p);
diff --git a/erts/emulator/beam/erl_ptab.c b/erts/emulator/beam/erl_ptab.c
index 87beeafa1a..5bbc71c659 100644
--- a/erts/emulator/beam/erl_ptab.c
+++ b/erts/emulator/beam/erl_ptab.c
@@ -421,6 +421,7 @@ erts_ptab_init_table(ErtsPTab *ptab,
 		     void (*release_element)(void *),
 		     ErtsPTabElementCommon *invalid_element,
 		     int size,
+		     UWord element_size,
 		     char *name)
 {
     size_t tab_sz;
@@ -443,6 +444,7 @@ erts_ptab_init_table(ErtsPTab *ptab,
 	bits = erts_fit_in_bits_int32((Sint32) size - 1);
     }
 
+    ptab->r.o.element_size = element_size;
     ptab->r.o.max = size;
 
     tab_sz = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(size*sizeof(erts_smp_atomic_t));
@@ -670,9 +672,10 @@ erts_ptab_delete_element(ErtsPTab *ptab,
     }
 
     if (ptab->r.o.release_element)
-	erts_schedule_thr_prgr_later_op(ptab->r.o.release_element,
-					(void *) ptab_el,
-					&ptab_el->u.release);
+	erts_schedule_thr_prgr_later_cleanup_op(ptab->r.o.release_element,
+						(void *) ptab_el,
+						&ptab_el->u.release,
+						ptab->r.o.element_size);
 }
 
 /*
diff --git a/erts/emulator/beam/erl_ptab.h b/erts/emulator/beam/erl_ptab.h
index 8a130f42a3..7fa1251900 100644
--- a/erts/emulator/beam/erl_ptab.h
+++ b/erts/emulator/beam/erl_ptab.h
@@ -109,6 +109,7 @@ typedef struct {
     ErtsPTabElementCommon *invalid_element;
     Eterm invalid_data;
     void (*release_element)(void *);
+    UWord element_size;
 } ErtsPTabReadOnlyData;
 
 typedef struct {
@@ -177,6 +178,7 @@ void erts_ptab_init_table(ErtsPTab *ptab,
 			  void (*release_element)(void *),
 			  ErtsPTabElementCommon *invalid_element,
 			  int size,
+			  UWord element_size,
 			  char *name);
 int erts_ptab_new_element(ErtsPTab *ptab,
 			  ErtsPTabElementCommon *ptab_el,
diff --git a/erts/emulator/beam/erl_thr_progress.c b/erts/emulator/beam/erl_thr_progress.c
index 1292cb0678..cf5e3dc012 100644
--- a/erts/emulator/beam/erl_thr_progress.c
+++ b/erts/emulator/beam/erl_thr_progress.c
@@ -418,7 +418,7 @@ erts_thr_progress_pre_init(void)
 {
     intrnl = NULL;
     erts_tsd_key_create(&erts_thr_prgr_data_key__);
-    init_nob(&erts_thr_prgr__.current, 0);
+    init_nob(&erts_thr_prgr__.current, ERTS_THR_PRGR_VAL_FIRST);
 }
 
 void
diff --git a/erts/emulator/beam/erl_thr_progress.h b/erts/emulator/beam/erl_thr_progress.h
index 1416aa6166..1aeecf2b06 100644
--- a/erts/emulator/beam/erl_thr_progress.h
+++ b/erts/emulator/beam/erl_thr_progress.h
@@ -108,6 +108,8 @@ struct ErtsThrPrgrLaterOp_ {
 
 #ifdef ERTS_SMP
 
+/* ERTS_THR_PRGR_VAL_FIRST should only be used when initializing... */
+#define ERTS_THR_PRGR_VAL_FIRST ((ErtsThrPrgrVal) 0)
 #define ERTS_THR_PRGR_VAL_WAITING (~((ErtsThrPrgrVal) 0))
 #define ERTS_THR_PRGR_INVALID (~((ErtsThrPrgrVal) 0))
 
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 8c67f731f4..43918a7141 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -2651,11 +2651,17 @@ void erts_init_io(int port_tab_size,
 		  int port_tab_size_ignore_files)
 {
     ErlDrvEntry** dp;
+    UWord common_element_size;
     erts_smp_rwmtx_opt_t drv_list_rwmtx_opts = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
     drv_list_rwmtx_opts.type = ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ;
     drv_list_rwmtx_opts.lived = ERTS_SMP_RWMTX_LONG_LIVED;
 
+    common_element_size = ERTS_ALC_DATA_ALIGN_SIZE(sizeof(Port));
+    common_element_size += ERTS_ALC_DATA_ALIGN_SIZE(sizeof(ErtsPortTaskBusyPortQ));
+    common_element_size += 10; /* name */
 #ifdef ERTS_SMP
+    common_element_size += sizeof(erts_mtx_t);
+
     init_xports_list_alloc();
 #endif
 
@@ -2684,6 +2690,7 @@ void erts_init_io(int port_tab_size,
 			 NULL,
 			 (ErtsPTabElementCommon *) &erts_invalid_port.common,
 			 port_tab_size,
+			 common_element_size, /* Doesn't need to be excact */
 			 "port_table");
 
     erts_smp_atomic_init_nob(&erts_bytes_out, 0);