aboutsummaryrefslogtreecommitdiffstats
path: root/erts
diff options
context:
space:
mode:
Diffstat (limited to 'erts')
-rw-r--r--erts/doc/src/erl.xml21
-rw-r--r--erts/doc/src/notes.xml68
-rw-r--r--erts/emulator/beam/erl_db_hash.c63
-rw-r--r--erts/emulator/beam/erl_init.c32
-rw-r--r--erts/emulator/beam/erl_process.c337
-rw-r--r--erts/emulator/beam/erl_process.h4
-rw-r--r--erts/etc/common/erlexec.c2
7 files changed, 466 insertions, 61 deletions
diff --git a/erts/doc/src/erl.xml b/erts/doc/src/erl.xml
index 4ef3e089a6..16f5205b29 100644
--- a/erts/doc/src/erl.xml
+++ b/erts/doc/src/erl.xml
@@ -768,6 +768,16 @@
<seealso marker="erlang#system_info_scheduler_bindings">erlang:system_info(scheduler_bindings)</seealso>.
</p>
</item>
+ <tag><marker id="+sbwt"><c>+sbwt none|very_short|short|medium|long|very_long</c></marker></tag>
+ <item>
+ <p>Set scheduler busy wait threshold. Default is <c>medium</c>.
+ The threshold determines how long schedulers should busy
+ wait when running out of work before going to sleep.
+ </p>
+ <p><em>NOTE:</em> This flag may be removed or changed at any time
+ without prior notice.
+ </p>
+ </item>
<tag><marker id="+scl"><c>+scl true|false</c></marker></tag>
<item>
<p>Enable or disable scheduler compaction of load. By default
@@ -903,6 +913,17 @@
<p>For more information, see
<seealso marker="erlang#system_info_cpu_topology">erlang:system_info(cpu_topology)</seealso>.</p>
</item>
+ <tag><marker id="+sws"><c>+sws default|legacy|proposal</c></marker></tag>
+ <item>
+ <p>Set scheduler wakeup strategy. Default is <c>legacy</c> (has been
+ used since OTP-R13B). The <c>proposal</c> strategy is the currently
+ proposed strategy for OTP-R16. Note that the <c>proposal</c> strategy
+ might change during OTP-R15.
+ </p>
+ <p><em>NOTE:</em> This flag may be removed or changed at any time
+ without prior notice.
+ </p>
+ </item>
<tag><marker id="+swt"><c>+swt very_low|low|medium|high|very_high</c></marker></tag>
<item>
<p>Set scheduler wakeup threshold. Default is <c>medium</c>.
diff --git a/erts/doc/src/notes.xml b/erts/doc/src/notes.xml
index 028a2bbf3d..6b6a3374d1 100644
--- a/erts/doc/src/notes.xml
+++ b/erts/doc/src/notes.xml
@@ -30,6 +30,74 @@
</header>
<p>This document describes the changes made to the ERTS application.</p>
+<section><title>Erts 5.9.1.1</title>
+
+ <section><title>Fixed Bugs and Malfunctions</title>
+ <list>
+ <item>
+ <p>
+ If threads support for the runtime system had been
+ disabled at compile time (<c>--disable-threads</c> had
+ been passed to <c>configure</c>), and the <c>+A</c>
+ command line argument of <c>erl</c> was passed when
+ starting the runtime system, <seealso
+ marker="erl_driver#driver_system_info">driver_system_info()</seealso>
+ erroneously claimed that the runtime system had async
+ threads even though it had not.</p>
+ <p>
+ Due to this bug the file driver did not split tasks into
+ smaller chunks, but instead completed the whole task at
+ once, i.e., the scheduler got occupied with I/O for a
+ longer time than intended.</p>
+ <p>
+ Own Id: OTP-10059</p>
+ </item>
+ </list>
+ </section>
+
+
+ <section><title>Improvements and New Features</title>
+ <list>
+ <item>
+ <p>
+ A proposal for a new scheduler wakeup strategy has been
+ implemented. For more information see the documentation
+ of the <seealso marker="erl#+sws">+sws</seealso> command
+ line argument of <c>erl</c>.</p>
+ <p>
+ Own Id: OTP-10033 Aux Id: Seq12025 </p>
+ </item>
+ <item>
+ <p>
+ A switch for configuration of busy wait length for
+ scheduler threads has been added. For more information
+ see the documentation of the <seealso
+ marker="erl#+sbwt">+sbwt</seealso> command line argument
+ of <c>erl</c>.</p>
+ <p>
+ Own Id: OTP-10044 Aux Id: Seq11976 </p>
+ </item>
+ <item>
+ <p>
+ The extra memory barriers introduced by bug-fix OTP-9281
+ were unnecessarily used also on tables without the
+ <c>write_concurrency</c> option enabled. This could
+ unnecessarily degrade performance of ETS tables without
+ <c>write_concurrency</c> on some hardware (e.g. PowerPC)
+ while not effecting performance at all on other hardware
+ (e.g. x86/x86_64).</p>
+ <p>
+ OTP-9281 (R14B03): ETS tables using the
+ <c>write_concurrency</c> option could potentially get
+ into an internally inconsistent state.</p>
+ <p>
+ Own Id: OTP-10048 Aux Id: OTP-9281 </p>
+ </item>
+ </list>
+ </section>
+
+</section>
+
<section><title>Erts 5.9.1</title>
<section><title>Fixed Bugs and Malfunctions</title>
diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index c726be5fb4..2fea4671e1 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -105,7 +105,20 @@
#define NSEG_2 256 /* Size of second segment table */
#define NSEG_INC 128 /* Number of segments to grow after that */
-#define SEGTAB(tb) ((struct segment**)erts_smp_atomic_read_ddrb(&(tb)->segtab))
+#ifdef ERTS_SMP
+# define DB_USING_FINE_LOCKING(TB) (((TB))->common.type & DB_FINE_LOCKED)
+#else
+# define DB_USING_FINE_LOCKING(TB) 0
+#endif
+
+#ifdef ETHR_ORDERED_READ_DEPEND
+#define SEGTAB(tb) ((struct segment**) erts_smp_atomic_read_nob(&(tb)->segtab))
+#else
+#define SEGTAB(tb) \
+ (DB_USING_FINE_LOCKING(tb) \
+ ? ((struct segment**) erts_smp_atomic_read_ddrb(&(tb)->segtab)) \
+ : ((struct segment**) erts_smp_atomic_read_nob(&(tb)->segtab)))
+#endif
#define NACTIVE(tb) ((int)erts_smp_atomic_read_nob(&(tb)->nactive))
#define NITEMS(tb) ((int)erts_smp_atomic_read_nob(&(tb)->common.nitems))
@@ -122,7 +135,9 @@
*/
static ERTS_INLINE Uint hash_to_ix(DbTableHash* tb, HashValue hval)
{
- Uint mask = erts_smp_atomic_read_acqb(&tb->szm);
+ Uint mask = (DB_USING_FINE_LOCKING(tb)
+ ? erts_smp_atomic_read_acqb(&tb->szm)
+ : erts_smp_atomic_read_nob(&tb->szm));
Uint ix = hval & mask;
if (ix >= erts_smp_atomic_read_nob(&tb->nactive)) {
ix &= mask>>1;
@@ -319,7 +334,10 @@ struct ext_segment {
static ERTS_INLINE void SET_SEGTAB(DbTableHash* tb,
struct segment** segtab)
{
- erts_smp_atomic_set_wb(&tb->segtab, (erts_aint_t) segtab);
+ if (DB_USING_FINE_LOCKING(tb))
+ erts_smp_atomic_set_wb(&tb->segtab, (erts_aint_t) segtab);
+ else
+ erts_smp_atomic_set_nob(&tb->segtab, (erts_aint_t) segtab);
#ifdef VALGRIND
tb->top_ptr_to_segment_with_active_segtab = EXTSEG(segtab);
#endif
@@ -2501,6 +2519,28 @@ static Eterm build_term_list(Process* p, HashDbTerm* ptr1, HashDbTerm* ptr2,
return list;
}
+static ERTS_INLINE int
+begin_resizing(DbTableHash* tb)
+{
+ if (DB_USING_FINE_LOCKING(tb))
+ return !erts_smp_atomic_xchg_acqb(&tb->is_resizing, 1);
+ else {
+ if (erts_smp_atomic_read_nob(&tb->is_resizing))
+ return 0;
+ erts_smp_atomic_set_nob(&tb->is_resizing, 1);
+ return 1;
+ }
+}
+
+static ERTS_INLINE void
+done_resizing(DbTableHash* tb)
+{
+ if (DB_USING_FINE_LOCKING(tb))
+ erts_smp_atomic_set_relb(&tb->is_resizing, 0);
+ else
+ erts_smp_atomic_set_nob(&tb->is_resizing, 0);
+}
+
/* Grow table with one new bucket.
** Allocate new segment if needed.
*/
@@ -2513,9 +2553,8 @@ static void grow(DbTableHash* tb, int nactive)
int from_ix;
int szm;
- if (erts_smp_atomic_xchg_acqb(&tb->is_resizing, 1)) {
+ if (!begin_resizing(tb))
return; /* already in progress */
- }
if (NACTIVE(tb) != nactive) {
goto abort; /* already done (race) */
}
@@ -2547,9 +2586,12 @@ static void grow(DbTableHash* tb, int nactive)
}
erts_smp_atomic_inc_nob(&tb->nactive);
if (from_ix == 0) {
- erts_smp_atomic_set_relb(&tb->szm, szm);
+ if (DB_USING_FINE_LOCKING(tb))
+ erts_smp_atomic_set_relb(&tb->szm, szm);
+ else
+ erts_smp_atomic_set_nob(&tb->szm, szm);
}
- erts_smp_atomic_set_relb(&tb->is_resizing, 0);
+ done_resizing(tb);
/* Finally, let's split the bucket. We try to do it in a smart way
to keep link order and avoid unnecessary updates of next-pointers */
@@ -2581,7 +2623,7 @@ static void grow(DbTableHash* tb, int nactive)
return;
abort:
- erts_smp_atomic_set_relb(&tb->is_resizing, 0);
+ done_resizing(tb);
}
@@ -2590,9 +2632,8 @@ abort:
*/
static void shrink(DbTableHash* tb, int nactive)
{
- if (erts_smp_atomic_xchg_acqb(&tb->is_resizing, 1)) {
+ if (!begin_resizing(tb))
return; /* already in progress */
- }
if (NACTIVE(tb) == nactive) {
erts_smp_rwmtx_t* lck;
int src_ix = nactive - 1;
@@ -2639,7 +2680,7 @@ static void shrink(DbTableHash* tb, int nactive)
}
/*else already done */
- erts_smp_atomic_set_relb(&tb->is_resizing, 0);
+ done_resizing(tb);
}
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 51d3153a3b..f88d485448 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -568,10 +568,14 @@ void erts_usage(void)
erts_fprintf(stderr, "-rg amount set reader groups limit\n");
erts_fprintf(stderr, "-sbt type set scheduler bind type, valid types are:\n");
erts_fprintf(stderr, " u|ns|ts|ps|s|nnts|nnps|tnnps|db\n");
+ erts_fprintf(stderr, "-sbwt val set scheduler busy wait threshold, valid values are:\n");
+ erts_fprintf(stderr, " none|very_short|short|medium|long|very_long.\n");
erts_fprintf(stderr, "-scl bool enable/disable compaction of scheduler load,\n");
erts_fprintf(stderr, " see the erl(1) documentation for more info.\n");
erts_fprintf(stderr, "-sct cput set cpu topology,\n");
erts_fprintf(stderr, " see the erl(1) documentation for more info.\n");
+ erts_fprintf(stderr, "-sws val set scheduler wakeup strategy, valid values are:\n");
+ erts_fprintf(stderr, " default|legacy|proposal.\n");
erts_fprintf(stderr, "-swt val set scheduler wakeup threshold, valid values are:\n");
erts_fprintf(stderr, " very_low|low|medium|high|very_high.\n");
erts_fprintf(stderr, "-sss size suggested stack size in kilo words for scheduler threads,\n");
@@ -848,6 +852,10 @@ early_init(int *argc, char **argv) /*
}
}
+#ifndef USE_THREADS
+ erts_async_max_threads = 0;
+#endif
+
#ifdef ERTS_SMP
no_schedulers = schdlrs;
no_schedulers_online = schdlrs_onln;
@@ -1256,6 +1264,16 @@ erl_start(int argc, char **argv)
erts_usage();
}
}
+ else if (has_prefix("bwt", sub_param)) {
+ arg = get_arg(sub_param+3, argv[i+1], &i);
+ if (erts_sched_set_busy_wait_threshold(arg) != 0) {
+ erts_fprintf(stderr, "bad scheduler busy wait threshold: %s\n",
+ arg);
+ erts_usage();
+ }
+ VERBOSE(DEBUG_SYSTEM,
+ ("scheduler wakup threshold: %s\n", arg));
+ }
else if (has_prefix("cl", sub_param)) {
arg = get_arg(sub_param+2, argv[i+1], &i);
if (sys_strcmp("true", arg) == 0)
@@ -1316,13 +1334,23 @@ erl_start(int argc, char **argv)
erts_use_sender_punish = 0;
else if (sys_strcmp("wt", sub_param) == 0) {
arg = get_arg(sub_param+2, argv[i+1], &i);
- if (erts_sched_set_wakeup_limit(arg) != 0) {
+ if (erts_sched_set_wakeup_other_thresold(arg) != 0) {
erts_fprintf(stderr, "scheduler wakeup threshold: %s\n",
arg);
erts_usage();
}
VERBOSE(DEBUG_SYSTEM,
- ("scheduler wakup threshold: %s\n", arg));
+ ("scheduler wakeup threshold: %s\n", arg));
+ }
+ else if (sys_strcmp("ws", sub_param) == 0) {
+ arg = get_arg(sub_param+2, argv[i+1], &i);
+ if (erts_sched_set_wakeup_other_type(arg) != 0) {
+ erts_fprintf(stderr, "scheduler wakeup strategy: %s\n",
+ arg);
+ erts_usage();
+ }
+ VERBOSE(DEBUG_SYSTEM,
+ ("scheduler wakeup threshold: %s\n", arg));
}
else if (has_prefix("ss", sub_param)) {
/* suggested stack size (Kilo Words) for scheduler threads */
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index bb8edd2624..7c46a38c9b 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -52,21 +52,22 @@
#define ERTS_SCHED_SPIN_UNTIL_YIELD 100
-#define ERTS_SCHED_SYS_SLEEP_SPINCOUNT 10
+#define ERTS_SCHED_SYS_SLEEP_SPINCOUNT_VERY_LONG 40
+#define ERTS_SCHED_AUX_WORK_SLEEP_SPINCOUNT_FACT_VERY_LONG 1000
+#define ERTS_SCHED_SYS_SLEEP_SPINCOUNT_LONG 20
+#define ERTS_SCHED_AUX_WORK_SLEEP_SPINCOUNT_FACT_LONG 1000
+#define ERTS_SCHED_SYS_SLEEP_SPINCOUNT_MEDIUM 10
+#define ERTS_SCHED_AUX_WORK_SLEEP_SPINCOUNT_FACT_MEDIUM 1000
+#define ERTS_SCHED_SYS_SLEEP_SPINCOUNT_SHORT 10
+#define ERTS_SCHED_AUX_WORK_SLEEP_SPINCOUNT_FACT_SHORT 0
+#define ERTS_SCHED_SYS_SLEEP_SPINCOUNT_VERY_SHORT 5
+#define ERTS_SCHED_AUX_WORK_SLEEP_SPINCOUNT_FACT_VERY_SHORT 0
+#define ERTS_SCHED_SYS_SLEEP_SPINCOUNT_NONE 0
+#define ERTS_SCHED_AUX_WORK_SLEEP_SPINCOUNT_FACT_NONE 0
+
#define ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT 1000
-#define ERTS_SCHED_TSE_SLEEP_SPINCOUNT \
- (ERTS_SCHED_SYS_SLEEP_SPINCOUNT*ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT)
#define ERTS_SCHED_SUSPEND_SLEEP_SPINCOUNT 0
-#define ERTS_WAKEUP_OTHER_LIMIT_VERY_HIGH (200*CONTEXT_REDS)
-#define ERTS_WAKEUP_OTHER_LIMIT_HIGH (50*CONTEXT_REDS)
-#define ERTS_WAKEUP_OTHER_LIMIT_MEDIUM (10*CONTEXT_REDS)
-#define ERTS_WAKEUP_OTHER_LIMIT_LOW (CONTEXT_REDS)
-#define ERTS_WAKEUP_OTHER_LIMIT_VERY_LOW (CONTEXT_REDS/10)
-
-#define ERTS_WAKEUP_OTHER_DEC 10
-#define ERTS_WAKEUP_OTHER_FIXED_INC (CONTEXT_REDS/10)
-
#if 0 || defined(DEBUG)
#define ERTS_FAKE_SCHED_BIND_PRINT_SORTED_CPU_DATA
#endif
@@ -283,14 +284,18 @@ Uint erts_no_schedulers;
ErtsProcTab erts_proc erts_align_attribute(ERTS_CACHE_LINE_SIZE);
-static int wakeup_other_limit;
-
int erts_sched_thread_suggested_stack_size = -1;
#ifdef ERTS_ENABLE_LOCK_CHECK
ErtsLcPSDLocks erts_psd_required_locks[ERTS_PSD_SIZE];
#endif
+static struct {
+ int aux_work;
+ int tse;
+ int sys_schedule;
+} sched_busy_wait;
+
#ifdef ERTS_SMP
int erts_disable_proc_not_running_opt;
@@ -2333,7 +2338,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
erts_smp_runq_unlock(rq);
- spincount = ERTS_SCHED_TSE_SLEEP_SPINCOUNT;
+ spincount = sched_busy_wait.tse;
tse_wait:
@@ -2384,7 +2389,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
}
flgs = sched_prep_cont_spin_wait(ssi);
- spincount = ERTS_SCHED_TSE_SLEEP_SPINCOUNT;
+ spincount = sched_busy_wait.aux_work;
if (!(flgs & ERTS_SSI_FLG_WAITING)) {
ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING));
@@ -2421,7 +2426,9 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
ASSERT(working);
sched_wall_time_change(esdp, working = 0);
- spincount = ERTS_SCHED_SYS_SLEEP_SPINCOUNT;
+ spincount = sched_busy_wait.sys_schedule;
+ if (spincount == 0)
+ goto sys_aux_work;
while (spincount-- > 0) {
@@ -4188,31 +4195,285 @@ erts_debug_nbalance(void)
#endif
}
+/* Wakeup other schedulers */
+
+typedef enum {
+ ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_VERY_HIGH,
+ ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_HIGH,
+ ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_MEDIUM,
+ ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_LOW,
+ ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_VERY_LOW
+} ErtsSchedWakeupOtherThreshold;
+
+typedef enum {
+ ERTS_SCHED_WAKEUP_OTHER_TYPE_PROPOSAL,
+ ERTS_SCHED_WAKEUP_OTHER_TYPE_LEGACY
+} ErtsSchedWakeupOtherType;
+
+/* First proposal */
+
+#define ERTS_WAKEUP_OTHER_LIMIT_VERY_HIGH (200*CONTEXT_REDS)
+#define ERTS_WAKEUP_OTHER_LIMIT_HIGH (50*CONTEXT_REDS)
+#define ERTS_WAKEUP_OTHER_LIMIT_MEDIUM (10*CONTEXT_REDS)
+#define ERTS_WAKEUP_OTHER_LIMIT_LOW (CONTEXT_REDS)
+#define ERTS_WAKEUP_OTHER_LIMIT_VERY_LOW (CONTEXT_REDS/10)
+
+#define ERTS_WAKEUP_OTHER_DEC_SHIFT_VERY_HIGH 3
+#define ERTS_WAKEUP_OTHER_DEC_SHIFT_HIGH 1
+#define ERTS_WAKEUP_OTHER_DEC_SHIFT_MEDIUM 0
+#define ERTS_WAKEUP_OTHER_DEC_SHIFT_LOW -2
+#define ERTS_WAKEUP_OTHER_DEC_SHIFT_VERY_LOW -5
+
+#define ERTS_WAKEUP_OTHER_DEC_SHIFT 2
+#define ERTS_WAKEUP_OTHER_FIXED_INC (CONTEXT_REDS/10)
+
+/* To be legacy */
+
+#define ERTS_WAKEUP_OTHER_LIMIT_VERY_HIGH_LEGACY (200*CONTEXT_REDS)
+#define ERTS_WAKEUP_OTHER_LIMIT_HIGH_LEGACY (50*CONTEXT_REDS)
+#define ERTS_WAKEUP_OTHER_LIMIT_MEDIUM_LEGACY (10*CONTEXT_REDS)
+#define ERTS_WAKEUP_OTHER_LIMIT_LOW_LEGACY (CONTEXT_REDS)
+#define ERTS_WAKEUP_OTHER_LIMIT_VERY_LOW_LEGACY (CONTEXT_REDS/10)
+
+#define ERTS_WAKEUP_OTHER_DEC_LEGACY 10
+#define ERTS_WAKEUP_OTHER_FIXED_INC_LEGACY (CONTEXT_REDS/10)
+
+#ifdef ERTS_SMP
+
+static struct {
+ ErtsSchedWakeupOtherThreshold threshold;
+ ErtsSchedWakeupOtherType type;
+ int limit;
+ int dec_shift;
+ int dec_mask;
+ void (*check)(ErtsRunQueue *rq, Uint32 flags);
+} wakeup_other;
+
+static void
+wakeup_other_check(ErtsRunQueue *rq, Uint32 flags)
+{
+ int wo_reds = rq->wakeup_other_reds;
+ if (wo_reds) {
+ int left_len = rq->len - 1;
+ if (left_len < 1) {
+ int wo_reduce = wo_reds << wakeup_other.dec_shift;
+ wo_reduce &= wakeup_other.dec_mask;
+ rq->wakeup_other -= wo_reduce;
+ if (rq->wakeup_other < 0)
+ rq->wakeup_other = 0;
+ }
+ else {
+ rq->wakeup_other += (left_len*wo_reds
+ + ERTS_WAKEUP_OTHER_FIXED_INC);
+ if (rq->wakeup_other > wakeup_other.limit) {
+ int empty_rqs =
+ erts_smp_atomic32_read_acqb(&no_empty_run_queues);
+ if (flags & ERTS_RUNQ_FLG_PROTECTED)
+ ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_PROTECTED);
+ if (empty_rqs != 0)
+ wake_scheduler_on_empty_runq(rq);
+ rq->wakeup_other = 0;
+ }
+ }
+ rq->wakeup_other_reds = 0;
+ }
+}
+
+static void
+wakeup_other_set_limit(void)
+{
+ switch (wakeup_other.threshold) {
+ case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_VERY_HIGH:
+ wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_VERY_HIGH;
+ wakeup_other.dec_shift = ERTS_WAKEUP_OTHER_DEC_SHIFT_VERY_HIGH;
+ break;
+ case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_HIGH:
+ wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_HIGH;
+ wakeup_other.dec_shift = ERTS_WAKEUP_OTHER_DEC_SHIFT_HIGH;
+ break;
+ case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_MEDIUM:
+ wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_MEDIUM;
+ wakeup_other.dec_shift = ERTS_WAKEUP_OTHER_DEC_SHIFT_MEDIUM;
+ break;
+ case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_LOW:
+ wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_LOW;
+ wakeup_other.dec_shift = ERTS_WAKEUP_OTHER_DEC_SHIFT_LOW;
+ break;
+ case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_VERY_LOW:
+ wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_VERY_LOW;
+ wakeup_other.dec_shift = ERTS_WAKEUP_OTHER_DEC_SHIFT_VERY_LOW;
+ break;
+ }
+ if (wakeup_other.dec_shift < 0)
+ wakeup_other.dec_mask = (1 << (sizeof(wakeup_other.dec_mask)*8
+ + wakeup_other.dec_shift)) - 1;
+ else {
+ wakeup_other.dec_mask = 0;
+ wakeup_other.dec_mask = ~wakeup_other.dec_mask;
+ }
+}
+
+static void
+wakeup_other_check_legacy(ErtsRunQueue *rq, Uint32 flags)
+{
+ int wo_reds = rq->wakeup_other_reds;
+ if (wo_reds) {
+ erts_aint32_t len = rq->len;
+ if (len < 2) {
+ rq->wakeup_other -= ERTS_WAKEUP_OTHER_DEC_LEGACY*wo_reds;
+ if (rq->wakeup_other < 0)
+ rq->wakeup_other = 0;
+ }
+ else if (rq->wakeup_other < wakeup_other.limit)
+ rq->wakeup_other += len*wo_reds + ERTS_WAKEUP_OTHER_FIXED_INC_LEGACY;
+ else {
+ if (flags & ERTS_RUNQ_FLG_PROTECTED)
+ ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_PROTECTED);
+ if (erts_smp_atomic32_read_acqb(&no_empty_run_queues) != 0) {
+ wake_scheduler_on_empty_runq(rq);
+ rq->wakeup_other = 0;
+ }
+ rq->wakeup_other = 0;
+ }
+ }
+ rq->wakeup_other_reds = 0;
+}
+
+static void
+wakeup_other_set_limit_legacy(void)
+{
+ switch (wakeup_other.threshold) {
+ case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_VERY_HIGH:
+ wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_VERY_HIGH_LEGACY;
+ break;
+ case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_HIGH:
+ wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_HIGH_LEGACY;
+ break;
+ case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_MEDIUM:
+ wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_MEDIUM_LEGACY;
+ break;
+ case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_LOW:
+ wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_LOW_LEGACY;
+ break;
+ case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_VERY_LOW:
+ wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_VERY_LOW_LEGACY;
+ break;
+ }
+}
+
+static void
+set_wakeup_other_data(void)
+{
+ switch (wakeup_other.type) {
+ case ERTS_SCHED_WAKEUP_OTHER_TYPE_PROPOSAL:
+ wakeup_other.check = wakeup_other_check;
+ wakeup_other_set_limit();
+ break;
+ case ERTS_SCHED_WAKEUP_OTHER_TYPE_LEGACY:
+ wakeup_other.check = wakeup_other_check_legacy;
+ wakeup_other_set_limit_legacy();
+ break;
+ }
+}
+
+#endif
+
void
erts_early_init_scheduling(int no_schedulers)
{
aux_work_timeout_early_init(no_schedulers);
- wakeup_other_limit = ERTS_WAKEUP_OTHER_LIMIT_MEDIUM;
+#ifdef ERTS_SMP
+ wakeup_other.threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_MEDIUM;
+ wakeup_other.type = ERTS_SCHED_WAKEUP_OTHER_TYPE_LEGACY;
+#endif
+ sched_busy_wait.sys_schedule = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_MEDIUM;
+ sched_busy_wait.tse = (ERTS_SCHED_SYS_SLEEP_SPINCOUNT_MEDIUM
+ * ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT);
+ sched_busy_wait.aux_work = (ERTS_SCHED_SYS_SLEEP_SPINCOUNT_MEDIUM
+ * ERTS_SCHED_AUX_WORK_SLEEP_SPINCOUNT_FACT_MEDIUM);
}
int
-erts_sched_set_wakeup_limit(char *str)
+erts_sched_set_wakeup_other_thresold(char *str)
{
+ ErtsSchedWakeupOtherThreshold threshold;
if (sys_strcmp(str, "very_high") == 0)
- wakeup_other_limit = ERTS_WAKEUP_OTHER_LIMIT_VERY_HIGH;
+ threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_VERY_HIGH;
else if (sys_strcmp(str, "high") == 0)
- wakeup_other_limit = ERTS_WAKEUP_OTHER_LIMIT_HIGH;
+ threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_HIGH;
else if (sys_strcmp(str, "medium") == 0)
- wakeup_other_limit = ERTS_WAKEUP_OTHER_LIMIT_MEDIUM;
+ threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_MEDIUM;
else if (sys_strcmp(str, "low") == 0)
- wakeup_other_limit = ERTS_WAKEUP_OTHER_LIMIT_LOW;
+ threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_LOW;
else if (sys_strcmp(str, "very_low") == 0)
- wakeup_other_limit = ERTS_WAKEUP_OTHER_LIMIT_VERY_LOW;
+ threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_VERY_LOW;
else
return EINVAL;
+#ifdef ERTS_SMP
+ wakeup_other.threshold = threshold;
+ set_wakeup_other_data();
+#endif
return 0;
}
+int
+erts_sched_set_wakeup_other_type(char *str)
+{
+ ErtsSchedWakeupOtherType type;
+ if (sys_strcmp(str, "proposal") == 0)
+ type = ERTS_SCHED_WAKEUP_OTHER_TYPE_PROPOSAL;
+ else if (sys_strcmp(str, "default") == 0)
+ type = ERTS_SCHED_WAKEUP_OTHER_TYPE_LEGACY;
+ else if (sys_strcmp(str, "legacy") == 0)
+ type = ERTS_SCHED_WAKEUP_OTHER_TYPE_LEGACY;
+ else
+ return EINVAL;
+#ifdef ERTS_SMP
+ wakeup_other.type = type;
+#endif
+ return 0;
+}
+
+int
+erts_sched_set_busy_wait_threshold(char *str)
+{
+ int sys_sched;
+ int aux_work_fact;
+
+ if (sys_strcmp(str, "very_long") == 0) {
+ sys_sched = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_VERY_LONG;
+ aux_work_fact = ERTS_SCHED_AUX_WORK_SLEEP_SPINCOUNT_FACT_VERY_LONG;
+ }
+ else if (sys_strcmp(str, "long") == 0) {
+ sys_sched = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_LONG;
+ aux_work_fact = ERTS_SCHED_AUX_WORK_SLEEP_SPINCOUNT_FACT_LONG;
+ }
+ else if (sys_strcmp(str, "medium") == 0) {
+ sys_sched = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_MEDIUM;
+ aux_work_fact = ERTS_SCHED_AUX_WORK_SLEEP_SPINCOUNT_FACT_MEDIUM;
+ }
+ else if (sys_strcmp(str, "short") == 0) {
+ sys_sched = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_SHORT;
+ aux_work_fact = ERTS_SCHED_AUX_WORK_SLEEP_SPINCOUNT_FACT_SHORT;
+ }
+ else if (sys_strcmp(str, "very_short") == 0) {
+ sys_sched = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_VERY_SHORT;
+ aux_work_fact = ERTS_SCHED_AUX_WORK_SLEEP_SPINCOUNT_FACT_VERY_SHORT;
+ }
+ else if (sys_strcmp(str, "none") == 0) {
+ sys_sched = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_NONE;
+ aux_work_fact = ERTS_SCHED_AUX_WORK_SLEEP_SPINCOUNT_FACT_NONE;
+ }
+ else {
+ return EINVAL;
+ }
+
+ sched_busy_wait.sys_schedule = sys_sched;
+ sched_busy_wait.tse = sys_sched*ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT;
+ sched_busy_wait.aux_work = sys_sched*aux_work_fact;
+
+ return 0;
+}
static void
init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp)
{
@@ -4241,6 +4502,10 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online)
init_misc_op_list_alloc();
+#ifdef ERTS_SMP
+ set_wakeup_other_data();
+#endif
+
ASSERT(no_schedulers_online <= no_schedulers);
ASSERT(no_schedulers_online >= 1);
ASSERT(no_schedulers >= 1);
@@ -6681,29 +6946,7 @@ Process *schedule(Process *p, int calls)
exec_misc_ops(rq);
#ifdef ERTS_SMP
- {
- int wo_reds = rq->wakeup_other_reds;
- if (wo_reds) {
- erts_aint32_t len = rq->len;
- if (len < 2) {
- rq->wakeup_other -= ERTS_WAKEUP_OTHER_DEC*wo_reds;
- if (rq->wakeup_other < 0)
- rq->wakeup_other = 0;
- }
- else if (rq->wakeup_other < wakeup_other_limit)
- rq->wakeup_other += len*wo_reds + ERTS_WAKEUP_OTHER_FIXED_INC;
- else {
- if (flags & ERTS_RUNQ_FLG_PROTECTED)
- ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_PROTECTED);
- if (erts_smp_atomic32_read_acqb(&no_empty_run_queues) != 0) {
- wake_scheduler_on_empty_runq(rq);
- rq->wakeup_other = 0;
- }
- rq->wakeup_other = 0;
- }
- }
- rq->wakeup_other_reds = 0;
- }
+ wakeup_other.check(rq, flags);
#endif
/*
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 2ab7b30906..0798697350 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -1178,7 +1178,9 @@ ErtsProcList *erts_proclist_create(Process *);
void erts_proclist_destroy(ErtsProcList *);
int erts_proclist_same(ErtsProcList *, Process *);
-int erts_sched_set_wakeup_limit(char *str);
+int erts_sched_set_wakeup_other_thresold(char *str);
+int erts_sched_set_wakeup_other_type(char *str);
+int erts_sched_set_busy_wait_threshold(char *str);
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
int erts_dbg_check_halloc_lock(Process *p);
diff --git a/erts/etc/common/erlexec.c b/erts/etc/common/erlexec.c
index 19b3bb82ef..c9aee16def 100644
--- a/erts/etc/common/erlexec.c
+++ b/erts/etc/common/erlexec.c
@@ -121,9 +121,11 @@ static char *plusM_other_switches[] = {
/* +s arguments with values */
static char *pluss_val_switches[] = {
"bt",
+ "bwt",
"cl",
"ct",
"wt",
+ "ws",
"ss",
NULL
};