aboutsummaryrefslogtreecommitdiffstats
path: root/erts/emulator
diff options
context:
space:
mode:
Diffstat (limited to 'erts/emulator')
-rw-r--r--erts/emulator/beam/atom.names8
-rw-r--r--erts/emulator/beam/beam_bif_load.c73
-rw-r--r--erts/emulator/beam/beam_catches.c6
-rw-r--r--erts/emulator/beam/beam_emu.c14
-rw-r--r--erts/emulator/beam/beam_load.c17
-rw-r--r--erts/emulator/beam/beam_ranges.c152
-rw-r--r--erts/emulator/beam/bif.c108
-rw-r--r--erts/emulator/beam/bif.h25
-rw-r--r--erts/emulator/beam/bif.tab3
-rw-r--r--erts/emulator/beam/big.c26
-rw-r--r--erts/emulator/beam/big.h1
-rw-r--r--erts/emulator/beam/binary.c2
-rw-r--r--erts/emulator/beam/break.c12
-rw-r--r--erts/emulator/beam/code_ix.c4
-rw-r--r--erts/emulator/beam/code_ix.h2
-rw-r--r--erts/emulator/beam/copy.c44
-rw-r--r--erts/emulator/beam/dist.c10
-rw-r--r--erts/emulator/beam/erl_afit_alloc.c2
-rw-r--r--erts/emulator/beam/erl_alloc.c81
-rw-r--r--erts/emulator/beam/erl_alloc.types1
-rw-r--r--erts/emulator/beam/erl_alloc_util.c10
-rw-r--r--erts/emulator/beam/erl_ao_firstfit_alloc.c2
-rw-r--r--erts/emulator/beam/erl_bestfit_alloc.c2
-rw-r--r--erts/emulator/beam/erl_bif_ddll.c2
-rw-r--r--erts/emulator/beam/erl_bif_info.c200
-rw-r--r--erts/emulator/beam/erl_bif_port.c4
-rw-r--r--erts/emulator/beam/erl_cpu_topology.c10
-rw-r--r--erts/emulator/beam/erl_db.c14
-rw-r--r--erts/emulator/beam/erl_db_hash.c2
-rw-r--r--erts/emulator/beam/erl_db_tree.c2
-rw-r--r--erts/emulator/beam/erl_db_util.c18
-rw-r--r--erts/emulator/beam/erl_debug.c14
-rw-r--r--erts/emulator/beam/erl_drv_thread.c2
-rw-r--r--erts/emulator/beam/erl_fun.c2
-rw-r--r--erts/emulator/beam/erl_gc.c6
-rw-r--r--erts/emulator/beam/erl_goodfit_alloc.c4
-rw-r--r--erts/emulator/beam/erl_init.c38
-rw-r--r--erts/emulator/beam/erl_map.c40
-rw-r--r--erts/emulator/beam/erl_monitors.c2
-rw-r--r--erts/emulator/beam/erl_nif.c2
-rw-r--r--erts/emulator/beam/erl_node_tables.c4
-rw-r--r--erts/emulator/beam/erl_port_task.c4
-rw-r--r--erts/emulator/beam/erl_process.c2348
-rw-r--r--erts/emulator/beam/erl_process.h29
-rw-r--r--erts/emulator/beam/erl_process_dict.c8
-rw-r--r--erts/emulator/beam/erl_process_lock.h7
-rw-r--r--erts/emulator/beam/erl_ptab.c2
-rw-r--r--erts/emulator/beam/erl_term.c2
-rw-r--r--erts/emulator/beam/erl_thr_progress.c12
-rw-r--r--erts/emulator/beam/erl_thr_queue.c6
-rw-r--r--erts/emulator/beam/erl_time_sup.c6
-rw-r--r--erts/emulator/beam/erl_trace.c4
-rw-r--r--erts/emulator/beam/erl_unicode.c2
-rw-r--r--erts/emulator/beam/external.c18
-rw-r--r--erts/emulator/beam/global.h13
-rw-r--r--erts/emulator/beam/index.c4
-rw-r--r--erts/emulator/beam/io.c6
-rw-r--r--erts/emulator/beam/module.h1
-rw-r--r--erts/emulator/beam/register.c2
-rw-r--r--erts/emulator/beam/sys.h25
-rw-r--r--erts/emulator/beam/time.c2
-rw-r--r--erts/emulator/beam/utils.c20
-rw-r--r--erts/emulator/drivers/common/efile_drv.c15
-rw-r--r--erts/emulator/drivers/common/inet_drv.c12
-rw-r--r--erts/emulator/drivers/win32/win_con.c4
-rw-r--r--erts/emulator/hipe/hipe_bif0.c4
-rw-r--r--erts/emulator/hipe/hipe_mode_switch.c6
-rw-r--r--erts/emulator/hipe/hipe_native_bif.c2
-rw-r--r--erts/emulator/sys/common/erl_mmap.c14
-rw-r--r--erts/emulator/sys/common/erl_mmap.h4
-rw-r--r--erts/emulator/sys/common/erl_mseg.c5
-rw-r--r--erts/emulator/sys/common/erl_os_monotonic_time_extender.c2
-rw-r--r--erts/emulator/sys/unix/erl_child_setup.c1
-rw-r--r--erts/emulator/sys/unix/sys.c16
-rw-r--r--erts/emulator/sys/unix/sys_drivers.c16
-rw-r--r--erts/emulator/sys/unix/sys_float.c2
-rw-r--r--erts/emulator/sys/unix/sys_time.c24
-rw-r--r--erts/emulator/sys/win32/erl_poll.c10
-rw-r--r--erts/emulator/sys/win32/sys.c8
-rw-r--r--erts/emulator/sys/win32/sys_interrupt.c6
-rw-r--r--erts/emulator/sys/win32/sys_time.c6
-rw-r--r--erts/emulator/test/Makefile1
-rw-r--r--erts/emulator/test/distribution_SUITE.erl2
-rw-r--r--erts/emulator/test/estone_SUITE.erl4
-rw-r--r--erts/emulator/test/multi_load_SUITE.erl196
-rw-r--r--erts/emulator/test/nif_SUITE.erl4
-rw-r--r--erts/emulator/test/scheduler_SUITE.erl176
-rw-r--r--erts/emulator/valgrind/suppress.patched.3.6.02
-rw-r--r--erts/emulator/valgrind/suppress.standard2
89 files changed, 2257 insertions, 1779 deletions
diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index 6fb08ee896..169b071cd7 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -125,7 +125,9 @@ atom binary_longest_suffix_trap
atom binary_to_list_continue
atom binary_to_term_trap
atom block
+atom block_normal
atom blocked
+atom blocked_normal
atom bm
atom bnot
atom bor
@@ -190,7 +192,9 @@ atom dexit
atom depth
atom dgroup_leader
atom dictionary
+atom dirty_cpu
atom dirty_cpu_schedulers_online
+atom dirty_io
atom disable_trace
atom disabled
atom display_items
@@ -211,6 +215,7 @@ atom dsend
atom dsend_continue_trap
atom dunlink
atom duplicate_bag
+atom duplicated
atom dupnames
atom einval
atom elib_malloc
@@ -273,6 +278,7 @@ atom gather_gc_info_result
atom gather_io_bytes
atom gather_microstate_accounting_result
atom gather_sched_wall_time_result
+atom gather_system_check_result
atom getting_linked
atom getting_unlinked
atom global
@@ -303,6 +309,7 @@ atom index
atom infinity
atom info
atom info_msg
+atom init
atom initial_call
atom input
atom internal
@@ -610,6 +617,7 @@ atom use_stdio
atom used
atom utf8
atom unblock
+atom unblock_normal
atom uniq
atom unless_suspending
atom unloaded
diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index 1b4c022370..0b47fc3586 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -86,7 +86,7 @@ BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3)
ASSERT(modp->curr.num_breakpoints == 0);
}
- erts_start_staging_code_ix();
+ erts_start_staging_code_ix(1);
res = erts_make_stub_module(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3);
@@ -139,6 +139,25 @@ prepare_loading_2(BIF_ALIST_2)
BIF_RET(res);
}
+BIF_RETTYPE
+has_prepared_code_on_load_1(BIF_ALIST_1)
+{
+ Eterm res;
+ ProcBin* pb;
+
+ if (!ERTS_TERM_IS_MAGIC_BINARY(BIF_ARG_1)) {
+ error:
+ BIF_ERROR(BIF_P, BADARG);
+ }
+
+ pb = (ProcBin*) binary_val(BIF_ARG_1);
+ res = erts_has_code_on_load(pb->val);
+ if (res == NIL) {
+ goto error;
+ }
+ BIF_RET(res);
+}
+
struct m {
Binary* code;
Eterm module;
@@ -163,14 +182,13 @@ exception_list(Process* p, Eterm tag, struct m* mp, Sint exceptions)
Eterm* hp = HAlloc(p, 3 + 2*exceptions);
Eterm res = NIL;
- mp += exceptions - 1;
while (exceptions > 0) {
if (mp->exception) {
res = CONS(hp, mp->module, res);
hp += 2;
exceptions--;
}
- mp--;
+ mp++;
}
return TUPLE2(hp, tag, res);
}
@@ -202,8 +220,12 @@ finish_loading_1(BIF_ALIST_1)
n = erts_list_length(BIF_ARG_1);
if (n < 0) {
- ERTS_BIF_PREP_ERROR(res, BIF_P, BADARG);
- goto done;
+ badarg:
+ if (p) {
+ erts_free(ERTS_ALC_T_LOADER_TMP, p);
+ }
+ erts_release_code_write_permission();
+ BIF_ERROR(BIF_P, BADARG);
}
p = erts_alloc(ERTS_ALC_T_LOADER_TMP, n*sizeof(struct m));
@@ -218,29 +240,32 @@ finish_loading_1(BIF_ALIST_1)
ProcBin* pb;
if (!ERTS_TERM_IS_MAGIC_BINARY(term)) {
- ERTS_BIF_PREP_ERROR(res, BIF_P, BADARG);
- goto done;
+ goto badarg;
}
pb = (ProcBin*) binary_val(term);
p[i].code = pb->val;
p[i].module = erts_module_for_prepared_code(p[i].code);
if (p[i].module == NIL) {
- ERTS_BIF_PREP_ERROR(res, BIF_P, BADARG);
- goto done;
+ goto badarg;
}
BIF_ARG_1 = CDR(cons);
}
/*
* Since we cannot handle atomic loading of a group of modules
- * if one or more of them uses on_load, we will only allow one
- * element in the list. This limitation is intended to be
- * lifted in the future.
+ * if one or more of them uses on_load, we will only allow
+ * more than one element in the list if none of the modules
+ * have an on_load function.
*/
if (n > 1) {
- ERTS_BIF_PREP_ERROR(res, BIF_P, SYSTEM_LIMIT);
- goto done;
+ for (i = 0; i < n; i++) {
+ if (erts_has_code_on_load(p[i].code) == am_true) {
+ erts_free(ERTS_ALC_T_LOADER_TMP, p);
+ erts_release_code_write_permission();
+ BIF_ERROR(BIF_P, SYSTEM_LIMIT);
+ }
+ }
}
/*
@@ -252,11 +277,27 @@ finish_loading_1(BIF_ALIST_1)
*/
res = am_ok;
- erts_start_staging_code_ix();
+ erts_start_staging_code_ix(n);
for (i = 0; i < n; i++) {
p[i].modp = erts_put_module(p[i].module);
+ p[i].modp->seen = 0;
}
+
+ exceptions = 0;
+ for (i = 0; i < n; i++) {
+ p[i].exception = 0;
+ if (p[i].modp->seen) {
+ p[i].exception = 1;
+ exceptions++;
+ }
+ p[i].modp->seen = 1;
+ }
+ if (exceptions) {
+ res = exception_list(BIF_P, am_duplicated, p, exceptions);
+ goto done;
+ }
+
for (i = 0; i < n; i++) {
if (p[i].modp->curr.num_breakpoints > 0 ||
p[i].modp->curr.num_traced_exports > 0 ||
@@ -492,7 +533,7 @@ BIF_RETTYPE delete_module_1(BIF_ALIST_1)
}
{
- erts_start_staging_code_ix();
+ erts_start_staging_code_ix(0);
code_ix = erts_staging_code_ix();
modp = erts_get_module(BIF_ARG_1, code_ix);
if (!modp) {
diff --git a/erts/emulator/beam/beam_catches.c b/erts/emulator/beam/beam_catches.c
index c1fd17c65d..7a1f4901aa 100644
--- a/erts/emulator/beam/beam_catches.c
+++ b/erts/emulator/beam/beam_catches.c
@@ -143,7 +143,7 @@ BeamInstr *beam_catches_car(unsigned i)
struct bc_pool* p = &bccix[erts_active_code_ix()];
if (i >= p->tabsize ) {
- erl_exit(1, "beam_catches_delmod: index %#x is out of range\r\n", i);
+ erts_exit(ERTS_ERROR_EXIT, "beam_catches_delmod: index %#x is out of range\r\n", i);
}
return p->beam_catches[i].cp;
}
@@ -157,10 +157,10 @@ void beam_catches_delmod(unsigned head, BeamInstr *code, unsigned code_bytes,
ASSERT((code_ix == erts_active_code_ix()) != bccix[erts_staging_code_ix()].is_staging);
for(i = head; i != (unsigned)-1;) {
if (i >= p->tabsize) {
- erl_exit(1, "beam_catches_delmod: index %#x is out of range\r\n", i);
+ erts_exit(ERTS_ERROR_EXIT, "beam_catches_delmod: index %#x is out of range\r\n", i);
}
if( (char*)p->beam_catches[i].cp - (char*)code >= code_bytes ) {
- erl_exit(1,
+ erts_exit(ERTS_ERROR_EXIT,
"beam_catches_delmod: item %#x has cp %p which is not "
"in module's range [%p,%p[\r\n",
i, p->beam_catches[i].cp, code, ((char*)code + code_bytes));
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 10f132abfc..a390422040 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -4730,7 +4730,7 @@ do { \
#ifdef NO_FPE_SIGNALS
OpCase(fclearerror):
OpCase(i_fcheckerror):
- erl_exit(1, "fclearerror/i_fcheckerror without fpe signals (beam_emu)");
+ erts_exit(ERTS_ERROR_EXIT, "fclearerror/i_fcheckerror without fpe signals (beam_emu)");
# define ERTS_NO_FPE_CHECK_INIT ERTS_FP_CHECK_INIT
# define ERTS_NO_FPE_ERROR ERTS_FP_ERROR
#else
@@ -4885,7 +4885,7 @@ do { \
I = handle_error(c_p, I, reg, NULL);
goto post_error_handling;
default:
- erl_exit(1, "hipe_mode_switch: result %u\n", c_p->def_arg_reg[3]);
+ erts_exit(ERTS_ERROR_EXIT, "hipe_mode_switch: result %u\n", c_p->def_arg_reg[3]);
}
}
OpCase(hipe_call_count): {
@@ -4997,7 +4997,7 @@ do { \
OpCase(label_L):
OpCase(on_load):
OpCase(line_I):
- erl_exit(1, "meta op\n");
+ erts_exit(ERTS_ERROR_EXIT, "meta op\n");
/*
* One-time initialization of Beam emulator.
@@ -5051,7 +5051,7 @@ do { \
}
#ifdef NO_JUMP_TABLE
default:
- erl_exit(1, "unexpected op code %d\n",Go);
+ erts_exit(ERTS_ERROR_EXIT, "unexpected op code %d\n",Go);
}
#endif
return; /* Never executed */
@@ -5096,7 +5096,7 @@ translate_gc_bif(void* gcf)
} else if (gcf == erts_gc_binary_part_3) {
return binary_part_3;
} else {
- erl_exit(1, "bad gc bif");
+ erts_exit(ERTS_ERROR_EXIT, "bad gc bif");
}
}
@@ -5161,7 +5161,7 @@ handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf)
Eterm* hp;
Eterm Value = c_p->fvalue;
Eterm Args = am_true;
- c_p->i = pc; /* In case we call erl_exit(). */
+ c_p->i = pc; /* In case we call erts_exit(). */
ASSERT(c_p->freason != TRAP); /* Should have been handled earlier. */
@@ -5225,7 +5225,7 @@ handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf)
c_p->cp = 0; /* To avoid keeping stale references. */
return new_pc;
}
- if (c_p->catches > 0) erl_exit(1, "Catch not found");
+ if (c_p->catches > 0) erts_exit(ERTS_ERROR_EXIT, "Catch not found");
}
ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
terminate_proc(c_p, Value);
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 1511a4f935..f115df935f 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -888,6 +888,23 @@ erts_module_for_prepared_code(Binary* magic)
}
}
+/*
+ * Return a non-zero value if the module has an on_load function,
+ * or 0 if it does not.
+ */
+
+Eterm
+erts_has_code_on_load(Binary* magic)
+{
+ LoaderState* stp;
+
+ if (ERTS_MAGIC_BIN_DESTRUCTOR(magic) != loader_state_dtor) {
+ return NIL;
+ }
+ stp = ERTS_MAGIC_BIN_DATA(magic);
+ return stp->on_load ? am_true : am_false;
+}
+
static void
free_loader_state(Binary* magic)
{
diff --git a/erts/emulator/beam/beam_ranges.c b/erts/emulator/beam/beam_ranges.c
index 5a2b66727a..54c337ee72 100644
--- a/erts/emulator/beam/beam_ranges.c
+++ b/erts/emulator/beam/beam_ranges.c
@@ -53,6 +53,7 @@ struct ranges {
};
static struct ranges r[ERTS_NUM_CODE_IX];
static erts_smp_atomic_t mem_used;
+static Range* write_ptr;
#ifdef HARD_DEBUG
static void check_consistency(struct ranges* p)
@@ -72,6 +73,17 @@ static void check_consistency(struct ranges* p)
# define CHECK(r)
#endif /* HARD_DEBUG */
+static int
+rangecompare(Range* a, Range* b)
+{
+ if (a->start < b->start) {
+ return -1;
+ } else if (a->start == b->start) {
+ return 0;
+ } else {
+ return 1;
+ }
+}
void
erts_init_ranges(void)
@@ -88,45 +100,70 @@ erts_init_ranges(void)
}
void
-erts_start_staging_ranges(void)
+erts_start_staging_ranges(int num_new)
{
+ ErtsCodeIndex src = erts_active_code_ix();
ErtsCodeIndex dst = erts_staging_code_ix();
+ Sint need;
if (r[dst].modules) {
erts_smp_atomic_add_nob(&mem_used, -r[dst].allocated);
erts_free(ERTS_ALC_T_MODULE_REFS, r[dst].modules);
- r[dst].modules = NULL;
}
+
+ need = r[dst].allocated = r[src].n + num_new;
+ erts_smp_atomic_add_nob(&mem_used, need);
+ write_ptr = erts_alloc(ERTS_ALC_T_MODULE_REFS,
+ need * sizeof(Range));
+ r[dst].modules = write_ptr;
}
void
erts_end_staging_ranges(int commit)
{
- ErtsCodeIndex dst = erts_staging_code_ix();
-
- if (commit && r[dst].modules == NULL) {
+ if (commit) {
Sint i;
- Sint n;
-
- /* No modules added, just clone src and remove purged code. */
ErtsCodeIndex src = erts_active_code_ix();
+ ErtsCodeIndex dst = erts_staging_code_ix();
+ Range* mp;
+ Sint num_inserted;
- erts_smp_atomic_add_nob(&mem_used, r[src].n);
- r[dst].modules = erts_alloc(ERTS_ALC_T_MODULE_REFS,
- r[src].n * sizeof(Range));
- r[dst].allocated = r[src].n;
- n = 0;
+ mp = r[dst].modules;
+ num_inserted = write_ptr - mp;
for (i = 0; i < r[src].n; i++) {
Range* rp = r[src].modules+i;
if (rp->start < RANGE_END(rp)) {
/* Only insert a module that has not been purged. */
- r[dst].modules[n] = *rp;
- n++;
+ write_ptr->start = rp->start;
+ erts_smp_atomic_init_nob(&write_ptr->end,
+ (erts_aint_t)(RANGE_END(rp)));
+ write_ptr++;
+ }
+ }
+
+ /*
+ * There are num_inserted new range entries (unsorted) at the
+ * beginning of the modules array, followed by the old entries
+ * (sorted). We must now sort the entire array.
+ */
+
+ r[dst].n = write_ptr - mp;
+ if (num_inserted > 1) {
+ qsort(mp, r[dst].n, sizeof(Range),
+ (int (*)(const void *, const void *)) rangecompare);
+ } else if (num_inserted == 1) {
+ /* Sift the new range into place. This is faster than qsort(). */
+ Range t = mp[0];
+ for (i = 0; i < r[dst].n-1 && t.start > mp[i+1].start; i++) {
+ mp[i] = mp[i+1];
}
+ mp[i] = t;
}
- r[dst].n = n;
+ r[dst].modules = mp;
+ CHECK(&r[dst]);
erts_smp_atomic_set_nob(&r[dst].mid,
- (erts_aint_t) (r[dst].modules + n / 2));
+ (erts_aint_t) (r[dst].modules +
+ r[dst].n / 2));
}
}
@@ -135,82 +172,29 @@ erts_update_ranges(BeamInstr* code, Uint size)
{
ErtsCodeIndex dst = erts_staging_code_ix();
ErtsCodeIndex src = erts_active_code_ix();
- Sint i;
- Sint n;
- Sint need;
if (src == dst) {
ASSERT(!erts_initialized);
/*
- * During start-up of system, the indices are the same.
- * Handle this by faking a source area.
+ * During start-up of system, the indices are the same
+ * and erts_start_staging_ranges() has not been called.
*/
- src = (src+1) % ERTS_NUM_CODE_IX;
- if (r[src].modules) {
- erts_smp_atomic_add_nob(&mem_used, -r[src].allocated);
- erts_free(ERTS_ALC_T_MODULE_REFS, r[src].modules);
+ if (r[dst].modules == NULL) {
+ Sint need = 128;
+ erts_smp_atomic_add_nob(&mem_used, need);
+ r[dst].modules = erts_alloc(ERTS_ALC_T_MODULE_REFS,
+ need * sizeof(Range));
+ r[dst].allocated = need;
+ write_ptr = r[dst].modules;
}
- r[src] = r[dst];
- r[dst].modules = 0;
}
- CHECK(&r[src]);
-
- ASSERT(r[dst].modules == NULL);
- need = r[dst].allocated = r[src].n + 1;
- erts_smp_atomic_add_nob(&mem_used, need);
- r[dst].modules = (Range *) erts_alloc(ERTS_ALC_T_MODULE_REFS,
- need * sizeof(Range));
- n = 0;
- for (i = 0; i < r[src].n; i++) {
- Range* rp = r[src].modules+i;
- if (code < rp->start) {
- r[dst].modules[n].start = code;
- erts_smp_atomic_init_nob(&r[dst].modules[n].end,
- (erts_aint_t)(((byte *)code) + size));
- ASSERT(!n || RANGE_END(&r[dst].modules[n-1]) < code);
- n++;
- break;
- }
- if (rp->start < RANGE_END(rp)) {
- /* Only insert a module that has not been purged. */
- r[dst].modules[n].start = rp->start;
- erts_smp_atomic_init_nob(&r[dst].modules[n].end,
- (erts_aint_t)(RANGE_END(rp)));
- ASSERT(!n || RANGE_END(&r[dst].modules[n-1]) < rp->start);
- n++;
- }
- }
-
- while (i < r[src].n) {
- Range* rp = r[src].modules+i;
- if (rp->start < RANGE_END(rp)) {
- /* Only insert a module that has not been purged. */
- r[dst].modules[n].start = rp->start;
- erts_smp_atomic_init_nob(&r[dst].modules[n].end,
- (erts_aint_t)(RANGE_END(rp)));
- ASSERT(!n || RANGE_END(&r[dst].modules[n-1]) < rp->start);
- n++;
- }
- i++;
- }
-
- if (n == 0 || code > r[dst].modules[n-1].start) {
- r[dst].modules[n].start = code;
- erts_smp_atomic_init_nob(&r[dst].modules[n].end,
- (erts_aint_t)(((byte *)code) + size));
- ASSERT(!n || RANGE_END(&r[dst].modules[n-1]) < code);
- n++;
- }
-
- ASSERT(n <= r[src].n+1);
- r[dst].n = n;
- erts_smp_atomic_set_nob(&r[dst].mid,
- (erts_aint_t) (r[dst].modules + n / 2));
-
- CHECK(&r[dst]);
- CHECK(&r[src]);
+ ASSERT(r[dst].modules);
+ write_ptr->start = code;
+ erts_smp_atomic_init_nob(&(write_ptr->end),
+ (erts_aint_t)(((byte *)code) + size));
+ write_ptr++;
}
void
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 2654785ffc..37bb28c6f8 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -2242,7 +2242,7 @@ BIF_RETTYPE send_3(BIF_ALIST_3)
erts_dsend_export_trap_context(p, ctx));
break;
default:
- erl_exit(ERTS_ABORT_EXIT, "send_3 invalid result %d\n", (int)result);
+ erts_exit(ERTS_ABORT_EXIT, "send_3 invalid result %d\n", (int)result);
break;
}
@@ -2286,7 +2286,7 @@ static BIF_RETTYPE dsend_continue_trap_1(BIF_ALIST_1)
BIF_TRAP1(&dsend_continue_trap_export, BIF_P, BIF_ARG_1);
}
default:
- erl_exit(ERTS_ABORT_EXIT, "dsend_continue_trap invalid result %d\n", (int)result);
+ erts_exit(ERTS_ABORT_EXIT, "dsend_continue_trap invalid result %d\n", (int)result);
break;
}
ASSERT(! "Can not arrive here");
@@ -2360,7 +2360,7 @@ Eterm erl_send(Process *p, Eterm to, Eterm msg)
erts_dsend_export_trap_context(p, ctx));
break;
default:
- erl_exit(ERTS_ABORT_EXIT, "invalid send result %d\n", (int)result);
+ erts_exit(ERTS_ABORT_EXIT, "invalid send result %d\n", (int)result);
break;
}
@@ -3738,7 +3738,7 @@ BIF_RETTYPE erts_debug_display_1(BIF_ALIST_1)
erts_dsprintf_buf_t *dsbufp = erts_create_tmp_dsbuf(64);
pres = erts_dsprintf(dsbufp, "%.*T\n", INT_MAX, BIF_ARG_1);
if (pres < 0)
- erl_exit(1, "Failed to convert term to string: %d (%s)\n",
+ erts_exit(ERTS_ERROR_EXIT, "Failed to convert term to string: %d (%s)\n",
-pres, erl_errno_id(-pres));
hp = HAlloc(BIF_P, 2*dsbufp->str_len); /* we need length * 2 heap words */
res = buf_to_intlist(&hp, dsbufp->str, dsbufp->str_len, NIL);
@@ -3760,7 +3760,7 @@ BIF_RETTYPE display_string_1(BIF_ALIST_1)
}
str = (char *) erts_alloc(ERTS_ALC_T_TMP, sizeof(char)*(len + 1));
if (intlist_to_buf(string, str, len) != len)
- erl_exit(1, "%s:%d: Internal error\n", __FILE__, __LINE__);
+ erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error\n", __FILE__, __LINE__);
str[len] = '\0';
erts_fprintf(stderr, "%s", str);
erts_free(ERTS_ALC_T_TMP, (void *) str);
@@ -3780,7 +3780,7 @@ BIF_RETTYPE display_nl_0(BIF_ALIST_0)
BIF_RETTYPE halt_0(BIF_ALIST_0)
{
VERBOSE(DEBUG_SYSTEM,("System halted by BIF halt()\n"));
- erl_halt(0);
+ erts_halt(0);
ERTS_BIF_YIELD1(bif_export[BIF_halt_1], BIF_P, am_undefined);
}
@@ -3793,17 +3793,18 @@ static char halt_msg[HALT_MSG_SIZE];
/* ARGSUSED */
BIF_RETTYPE halt_1(BIF_ALIST_1)
{
- Sint code;
+ Uint code;
- if (is_small(BIF_ARG_1) && (code = signed_val(BIF_ARG_1)) >= 0) {
+ if (term_to_Uint_mask(BIF_ARG_1, &code)) {
+ int pos_int_code = (int) (code & INT_MAX);
VERBOSE(DEBUG_SYSTEM,("System halted by BIF halt(%T)\n", BIF_ARG_1));
- erl_halt((int)(- code));
+ erts_halt(pos_int_code);
ERTS_BIF_YIELD1(bif_export[BIF_halt_1], BIF_P, am_undefined);
}
else if (ERTS_IS_ATOM_STR("abort", BIF_ARG_1)) {
VERBOSE(DEBUG_SYSTEM,("System halted by BIF halt(%T)\n", BIF_ARG_1));
erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
- erl_exit(ERTS_ABORT_EXIT, "");
+ erts_exit(ERTS_ABORT_EXIT, "");
}
else if (is_string(BIF_ARG_1) || BIF_ARG_1 == NIL) {
Sint i;
@@ -3814,11 +3815,11 @@ BIF_RETTYPE halt_1(BIF_ALIST_1)
halt_msg[i] = '\0';
VERBOSE(DEBUG_SYSTEM,("System halted by BIF halt(%T)\n", BIF_ARG_1));
erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
- erl_exit(ERTS_DUMP_EXIT, "%s\n", halt_msg);
+ erts_exit(ERTS_DUMP_EXIT, "%s\n", halt_msg);
}
else
goto error;
- return NIL; /* Pedantic (lint does not know about erl_exit) */
+ return NIL; /* Pedantic (lint does not know about erts_exit) */
error:
BIF_ERROR(BIF_P, BADARG);
}
@@ -3829,7 +3830,7 @@ BIF_RETTYPE halt_1(BIF_ALIST_1)
/* ARGSUSED */
BIF_RETTYPE halt_2(BIF_ALIST_2)
{
- Sint code;
+ Uint code;
Eterm optlist = BIF_ARG_2;
int flush = 1;
@@ -3856,23 +3857,24 @@ BIF_RETTYPE halt_2(BIF_ALIST_2)
if (is_not_nil(optlist))
goto error;
- if (is_small(BIF_ARG_1) && (code = signed_val(BIF_ARG_1)) >= 0) {
+ if (term_to_Uint_mask(BIF_ARG_1, &code)) {
+ int pos_int_code = (int) (code & INT_MAX);
VERBOSE(DEBUG_SYSTEM,
("System halted by BIF halt(%T, %T)\n", BIF_ARG_1, BIF_ARG_2));
if (flush) {
- erl_halt((int)(- code));
+ erts_halt(pos_int_code);
ERTS_BIF_YIELD1(bif_export[BIF_halt_1], BIF_P, am_undefined);
}
else {
erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
- erl_exit((int)(- code), "");
+ erts_exit(pos_int_code, "");
}
}
else if (ERTS_IS_ATOM_STR("abort", BIF_ARG_1)) {
VERBOSE(DEBUG_SYSTEM,
("System halted by BIF halt(%T, %T)\n", BIF_ARG_1, BIF_ARG_2));
erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
- erl_exit(ERTS_ABORT_EXIT, "");
+ erts_exit(ERTS_ABORT_EXIT, "");
}
else if (is_string(BIF_ARG_1) || BIF_ARG_1 == NIL) {
Sint i;
@@ -3884,11 +3886,11 @@ BIF_RETTYPE halt_2(BIF_ALIST_2)
VERBOSE(DEBUG_SYSTEM,
("System halted by BIF halt(%T, %T)\n", BIF_ARG_1, BIF_ARG_2));
erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
- erl_exit(ERTS_DUMP_EXIT, "%s\n", halt_msg);
+ erts_exit(ERTS_DUMP_EXIT, "%s\n", halt_msg);
}
else
goto error;
- return NIL; /* Pedantic (lint does not know about erl_exit) */
+ return NIL; /* Pedantic (lint does not know about erts_exit) */
error:
BIF_ERROR(BIF_P, BADARG);
}
@@ -3944,7 +3946,7 @@ term2list_dsprintf(Process *p, Eterm term)
erts_dsprintf_buf_t *dsbufp = erts_create_tmp_dsbuf(64);
pres = erts_dsprintf(dsbufp, "%T", term);
if (pres < 0)
- erl_exit(1, "Failed to convert term to list: %d (%s)\n",
+ erts_exit(ERTS_ERROR_EXIT, "Failed to convert term to list: %d (%s)\n",
-pres, erl_errno_id(-pres));
hp = HAlloc(p, 2*dsbufp->str_len); /* we need length * 2 heap words */
res = buf_to_intlist(&hp, dsbufp->str, dsbufp->str_len, NIL);
@@ -4195,22 +4197,33 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2)
Sint n;
if (BIF_ARG_1 == am_multi_scheduling) {
- if (BIF_ARG_2 == am_block || BIF_ARG_2 == am_unblock) {
+ if (BIF_ARG_2 == am_block || BIF_ARG_2 == am_unblock
+ || BIF_ARG_2 == am_block_normal || BIF_ARG_2 == am_unblock_normal) {
#ifndef ERTS_SMP
BIF_RET(am_disabled);
#else
+ int block = (BIF_ARG_2 == am_block
+ || BIF_ARG_2 == am_block_normal);
+ int normal = (BIF_ARG_2 == am_block_normal
+ || BIF_ARG_2 == am_unblock_normal);
if (erts_no_schedulers == 1)
BIF_RET(am_disabled);
else {
switch (erts_block_multi_scheduling(BIF_P,
ERTS_PROC_LOCK_MAIN,
- BIF_ARG_2 == am_block,
+ block,
+ normal,
0)) {
case ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED:
BIF_RET(am_blocked);
+ case ERTS_SCHDLR_SSPND_DONE_NMSCHED_BLOCKED:
+ BIF_RET(am_blocked_normal);
case ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED:
ERTS_BIF_YIELD_RETURN_X(BIF_P, am_blocked,
am_multi_scheduling);
+ case ERTS_SCHDLR_SSPND_YIELD_DONE_NMSCHED_BLOCKED:
+ ERTS_BIF_YIELD_RETURN_X(BIF_P, am_blocked_normal,
+ am_multi_scheduling);
case ERTS_SCHDLR_SSPND_DONE:
BIF_RET(am_enabled);
case ERTS_SCHDLR_SSPND_YIELD_RESTART:
@@ -4243,11 +4256,7 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2)
switch (erts_set_schedulers_online(BIF_P,
ERTS_PROC_LOCK_MAIN,
signed_val(BIF_ARG_2),
- &old_no
-#ifdef ERTS_DIRTY_SCHEDULERS
- , 0
-#endif
- )) {
+ &old_no, 0)) {
case ERTS_SCHDLR_SSPND_DONE:
BIF_RET(make_small(old_no));
case ERTS_SCHDLR_SSPND_YIELD_RESTART:
@@ -4617,29 +4626,27 @@ BIF_RETTYPE erts_internal_cmp_term_2(BIF_ALIST_2) {
/*
* Processes doing yield on return in a bif ends up in bif_return_trap().
*/
-static BIF_RETTYPE bif_return_trap(
-#ifdef DEBUG
- BIF_ALIST_2
-#else
- BIF_ALIST_1
-#endif
- )
+static BIF_RETTYPE bif_return_trap(BIF_ALIST_2)
{
-#ifdef DEBUG
+ Eterm res = BIF_ARG_1;
+
switch (BIF_ARG_2) {
- case am_multi_scheduling:
#ifdef ERTS_SMP
- erts_dbg_multi_scheduling_return_trap(BIF_P, BIF_ARG_1);
-#endif
- break;
- case am_schedulers_online:
+ case am_multi_scheduling: {
+ int msb = erts_is_multi_scheduling_blocked();
+ if (msb > 0)
+ res = am_blocked;
+ else if (msb < 0)
+ res = am_blocked_normal;
+ else
+ ERTS_INTERNAL_ERROR("Unexpected multi scheduling block state");
break;
+ }
+#endif
default:
break;
}
-#endif
-
- BIF_RET(BIF_ARG_1);
+ BIF_RET(res);
}
/*
@@ -4744,17 +4751,12 @@ void erts_init_bif(void)
erts_smp_atomic_init_nob(&erts_dead_ports_ptr, (erts_aint_t) NULL);
/*
- * bif_return_trap/1 is a hidden BIF that bifs that need to
- * yield the calling process traps to. The only thing it does:
- * return the value passed as argument.
+ * bif_return_trap/2 is a hidden BIF that bifs that need to
+ * yield the calling process traps to.
*/
- erts_init_trap_export(&bif_return_trap_export, am_erlang, am_bif_return_trap,
-#ifdef DEBUG
- 2
-#else
- 1
-#endif
- , &bif_return_trap);
+ erts_init_trap_export(&bif_return_trap_export,
+ am_erlang, am_bif_return_trap, 2,
+ &bif_return_trap);
erts_await_result = erts_export_put(am_erts_internal,
am_await_result,
diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h
index a62eddf36b..46af552b39 100644
--- a/erts/emulator/beam/bif.h
+++ b/erts/emulator/beam/bif.h
@@ -338,37 +338,20 @@ do { \
} while(0)
extern Export bif_return_trap_export;
-#ifdef DEBUG
-#define ERTS_BIF_PREP_YIELD_RETURN_X(RET, P, VAL, DEBUG_VAL) \
+#define ERTS_BIF_PREP_YIELD_RETURN_X(RET, P, VAL, OP) \
do { \
ERTS_VBUMP_ALL_REDS(P); \
- ERTS_BIF_PREP_TRAP2(RET, &bif_return_trap_export, (P), (VAL), \
- (DEBUG_VAL)); \
+ ERTS_BIF_PREP_TRAP2(RET, &bif_return_trap_export, (P), (VAL), (OP));\
} while (0)
-#else
-#define ERTS_BIF_PREP_YIELD_RETURN_X(RET, P, VAL, DEBUG_VAL) \
-do { \
- ERTS_VBUMP_ALL_REDS(P); \
- ERTS_BIF_PREP_TRAP1(RET, &bif_return_trap_export, (P), (VAL)); \
-} while (0)
-#endif
#define ERTS_BIF_PREP_YIELD_RETURN(RET, P, VAL) \
ERTS_BIF_PREP_YIELD_RETURN_X(RET, (P), (VAL), am_undefined)
-#ifdef DEBUG
-#define ERTS_BIF_YIELD_RETURN_X(P, VAL, DEBUG_VAL) \
+#define ERTS_BIF_YIELD_RETURN_X(P, VAL, OP) \
do { \
ERTS_VBUMP_ALL_REDS(P); \
- BIF_TRAP2(&bif_return_trap_export, (P), (VAL), (DEBUG_VAL)); \
+ BIF_TRAP2(&bif_return_trap_export, (P), (VAL), (OP)); \
} while (0)
-#else
-#define ERTS_BIF_YIELD_RETURN_X(P, VAL, DEBUG_VAL) \
-do { \
- ERTS_VBUMP_ALL_REDS(P); \
- BIF_TRAP1(&bif_return_trap_export, (P), (VAL)); \
-} while (0)
-#endif
#define ERTS_BIF_RETURN_YIELD(P) ERTS_VBUMP_ALL_REDS((P))
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 4efc055aaf..a6670c10e8 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -174,6 +174,8 @@ bif erts_internal:perf_counter_unit/0
bif erts_internal:is_system_process/1
+bif erts_internal:system_check/1
+
# inet_db support
bif erlang:port_set_data/2
bif erlang:port_get_data/1
@@ -649,6 +651,7 @@ bif binary:split/2
bif binary:split/3
bif erts_debug:size_shared/1
bif erts_debug:copy_shared/1
+bif erlang:has_prepared_code_on_load/1
#
# Obsolete
diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c
index 11838e24ef..c738a9ecf5 100644
--- a/erts/emulator/beam/big.c
+++ b/erts/emulator/beam/big.c
@@ -2035,6 +2035,32 @@ term_to_Uint(Eterm term, Uint *up)
}
}
+/* same as term_to_Uint()
+ but also accept larger bignums by masking
+ */
+int
+term_to_Uint_mask(Eterm term, Uint *up)
+{
+ if (is_small(term)) {
+ Sint i = signed_val(term);
+ if (i < 0) {
+ *up = BADARG;
+ return 0;
+ }
+ *up = (Uint) i;
+ return 1;
+ } else if (is_big(term) && !big_sign(term)) {
+ ErtsDigit* xr = big_v(term);
+
+ ERTS_CT_ASSERT(sizeof(ErtsDigit) == sizeof(Uint));
+ *up = (Uint)*xr; /* just pick first word */
+ return 1;
+ } else {
+ *up = BADARG;
+ return 0;
+ }
+}
+
int
term_to_UWord(Eterm term, UWord *up)
{
diff --git a/erts/emulator/beam/big.h b/erts/emulator/beam/big.h
index 9c92de6b55..ee51e5d313 100644
--- a/erts/emulator/beam/big.h
+++ b/erts/emulator/beam/big.h
@@ -154,6 +154,7 @@ Eterm bytes_to_big(byte*, dsize_t, int, Eterm*);
byte* big_to_bytes(Eterm, byte*);
int term_to_Uint(Eterm, Uint*);
+int term_to_Uint_mask(Eterm, Uint*);
int term_to_UWord(Eterm, UWord*);
int term_to_Sint(Eterm, Sint*);
#if HAVE_INT64
diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c
index d3e481c7f9..3fd284b589 100644
--- a/erts/emulator/beam/binary.c
+++ b/erts/emulator/beam/binary.c
@@ -50,7 +50,7 @@ erts_init_binary(void)
if ((((UWord) &((Binary *) 0)->orig_bytes[0]) % ((UWord) 8)) != 0) {
/* I assume that any compiler should be able to optimize this
away. If not, this test is not very expensive... */
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"Internal error: Address of orig_bytes[0] of a Binary"
" is *not* 8-byte aligned\n");
}
diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index 2c8ecf04be..8c039ef132 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -104,7 +104,7 @@ process_killer(void)
erts_printf("(k)ill (n)ext (r)eturn:\n");
while(1) {
if ((j = sys_get_key(0)) <= 0)
- erl_exit(0, "");
+ erts_exit(0, "");
switch(j) {
case 'k': {
ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND;
@@ -493,7 +493,7 @@ do_break(void)
halt immediately if break is called */
mode = erts_read_env("ERL_CONSOLE_MODE");
if (mode && strcmp(mode, "window") != 0)
- erl_exit(0, "");
+ erts_exit(0, "");
erts_free_read_env(mode);
#endif /* __WIN32__ */
@@ -503,7 +503,7 @@ do_break(void)
while (1) {
if ((i = sys_get_key(0)) <= 0)
- erl_exit(0, "");
+ erts_exit(0, "");
switch (i) {
case 'q':
case 'a':
@@ -513,9 +513,9 @@ do_break(void)
* The usual reason for a read error is Ctrl-C. Treat this as
* 'a' to avoid infinite loop.
*/
- erl_exit(0, "");
+ erts_exit(0, "");
case 'A': /* Halt generating crash dump */
- erl_exit(1, "Crash dump requested by user");
+ erts_exit(ERTS_ERROR_EXIT, "Crash dump requested by user");
case 'c':
return;
case 'p':
@@ -785,7 +785,7 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
erts_fdprintf(fd, "Atoms: %d\n", atom_table_size());
#ifdef USE_THREADS
- /* We want to note which thread it was that called erl_exit */
+ /* We want to note which thread it was that called erts_exit */
if (erts_get_scheduler_data()) {
erts_fdprintf(fd, "Calling Thread: scheduler:%d\n",
erts_get_scheduler_data()->no);
diff --git a/erts/emulator/beam/code_ix.c b/erts/emulator/beam/code_ix.c
index 209d008d18..1b0968c55c 100644
--- a/erts/emulator/beam/code_ix.c
+++ b/erts/emulator/beam/code_ix.c
@@ -65,12 +65,12 @@ void erts_code_ix_init(void)
CIX_TRACE("init");
}
-void erts_start_staging_code_ix(void)
+void erts_start_staging_code_ix(int num_new)
{
beam_catches_start_staging();
export_start_staging();
module_start_staging();
- erts_start_staging_ranges();
+ erts_start_staging_ranges(num_new);
CIX_TRACE("start");
}
diff --git a/erts/emulator/beam/code_ix.h b/erts/emulator/beam/code_ix.h
index 5f00b409ef..7a66211a5b 100644
--- a/erts/emulator/beam/code_ix.h
+++ b/erts/emulator/beam/code_ix.h
@@ -100,7 +100,7 @@ void erts_release_code_write_permission(void);
* Must be followed by calls to either "end" and "commit" or "abort" before
* code write permission can be released.
*/
-void erts_start_staging_code_ix(void);
+void erts_start_staging_code_ix(int num_new);
/* End the staging.
* Preceded by "start" and must be followed by "commit".
diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index 67a96f6442..b38ebe066b 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -58,7 +58,7 @@ Eterm copy_object_x(Eterm obj, Process* to, Uint extra) {
res = copy_struct(obj, size, &hp, &to->off_heap);
#ifdef DEBUG
if (eq(obj, res) == 0) {
- erl_exit(ERTS_ABORT_EXIT, "copy not equal to source\n");
+ erts_exit(ERTS_ABORT_EXIT, "copy not equal to source\n");
}
#endif
return res;
@@ -174,7 +174,7 @@ Uint size_object(Eterm obj)
}
break;
default:
- erl_exit(ERTS_ABORT_EXIT, "size_object: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr));
+ erts_exit(ERTS_ABORT_EXIT, "size_object: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr));
}
break;
case SUB_BINARY_SUBTAG:
@@ -205,7 +205,7 @@ Uint size_object(Eterm obj)
}
break;
case BIN_MATCHSTATE_SUBTAG:
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"size_object: matchstate term not allowed");
default:
sum += thing_arityval(hdr) + 1;
@@ -223,7 +223,7 @@ Uint size_object(Eterm obj)
obj = ESTACK_POP(s);
break;
default:
- erl_exit(ERTS_ABORT_EXIT, "size_object: bad tag for %#x\n", obj);
+ erts_exit(ERTS_ABORT_EXIT, "size_object: bad tag for %#x\n", obj);
}
}
}
@@ -438,10 +438,10 @@ Uint size_shared(Eterm obj)
goto pop_next;
}
default:
- erl_exit(ERTS_ABORT_EXIT, "size_shared: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr));
+ erts_exit(ERTS_ABORT_EXIT, "size_shared: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr));
}
case BIN_MATCHSTATE_SUBTAG:
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"size_shared: matchstate term not allowed");
default:
sum += thing_arityval(hdr) + 1;
@@ -457,7 +457,7 @@ Uint size_shared(Eterm obj)
obj = EQUEUE_GET(s);
break;
default:
- erl_exit(ERTS_ABORT_EXIT, "size_shared: bad tag for %#x\n", obj);
+ erts_exit(ERTS_ABORT_EXIT, "size_shared: bad tag for %#x\n", obj);
}
}
@@ -569,7 +569,7 @@ cleanup:
goto cleanup_next;
}
default:
- erl_exit(ERTS_ABORT_EXIT, "size_shared: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr));
+ erts_exit(ERTS_ABORT_EXIT, "size_shared: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr));
}
default:
goto cleanup_next;
@@ -584,7 +584,7 @@ cleanup:
obj = EQUEUE_GET(s);
break;
default:
- erl_exit(ERTS_ABORT_EXIT, "size_shared: bad tag for %#x\n", obj);
+ erts_exit(ERTS_ABORT_EXIT, "size_shared: bad tag for %#x\n", obj);
}
}
@@ -643,7 +643,7 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint
goto L_copy_list;
case TAG_PRIMARY_BOXED: argp = &res; goto L_copy_boxed;
default:
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"%s, line %d: Internal error in copy_struct: 0x%08x\n",
__FILE__, __LINE__,obj);
}
@@ -691,7 +691,7 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint
case TAG_PRIMARY_IMMED1: *tailp = obj; goto L_copy;
case TAG_PRIMARY_BOXED: argp = tailp; goto L_copy_boxed;
default:
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"%s, line %d: Internal error in copy_struct: 0x%08x\n",
__FILE__, __LINE__,obj);
}
@@ -869,11 +869,11 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint
*argp = make_hashmap(tp);
break;
default:
- erl_exit(ERTS_ABORT_EXIT, "copy_struct: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr));
+ erts_exit(ERTS_ABORT_EXIT, "copy_struct: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr));
}
break;
case BIN_MATCHSTATE_SUBTAG:
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"copy_struct: matchstate term not allowed");
default:
i = thing_arityval(hdr)+1;
@@ -901,13 +901,13 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint
} else {
#ifdef DEBUG
if (htop != hbot)
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"Internal error in copy_struct() when copying %T:"
" htop=%p != hbot=%p (sz=%beu)\n",
org_obj, htop, hbot, org_sz);
#else
if (htop > hbot) {
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"Internal error in copy_struct(): htop, hbot overrun\n");
}
#endif
@@ -1246,10 +1246,10 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info)
goto pop_next;
}
default:
- erl_exit(ERTS_ABORT_EXIT, "copy_shared_calculate: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr));
+ erts_exit(ERTS_ABORT_EXIT, "copy_shared_calculate: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr));
}
case BIN_MATCHSTATE_SUBTAG:
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"size_shared: matchstate term not allowed");
default:
sum += thing_arityval(hdr) + 1;
@@ -1278,7 +1278,7 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info)
obj = EQUEUE_GET(s);
break;
default:
- erl_exit(ERTS_ABORT_EXIT, "[pid=%T] size_shared: bad tag for %#x\n", obj);
+ erts_exit(ERTS_ABORT_EXIT, "[pid=%T] size_shared: bad tag for %#x\n", obj);
}
}
}
@@ -1530,7 +1530,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info,
goto cleanup_next;
}
default:
- erl_exit(ERTS_ABORT_EXIT, "copy_shared_perform: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr));
+ erts_exit(ERTS_ABORT_EXIT, "copy_shared_perform: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr));
}
case REFC_BINARY_SUBTAG: {
ProcBin* pb = (ProcBin *) ptr;
@@ -1682,7 +1682,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info,
hscan += MAP_HEADER_ARITY(*hscan) + 1;
break;
default:
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"copy_shared_perform: bad hashmap type %d\n",
MAP_HEADER_TYPE(*hscan));
}
@@ -1711,7 +1711,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info,
ASSERT(resp < hp);
break;
default:
- erl_exit(ERTS_ABORT_EXIT, "size_shared: bad tag for %#x\n", obj);
+ erts_exit(ERTS_ABORT_EXIT, "size_shared: bad tag for %#x\n", obj);
}
}
@@ -1744,7 +1744,7 @@ all_clean:
if (eq(saved_obj, result) == 0) {
erts_fprintf(stderr, "original = %T\n", saved_obj);
erts_fprintf(stderr, "copy = %T\n", result);
- erl_exit(ERTS_ABORT_EXIT, "copy (shared) not equal to source\n");
+ erts_exit(ERTS_ABORT_EXIT, "copy (shared) not equal to source\n");
}
#endif
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index e31ef29562..fa385f105d 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -1942,7 +1942,7 @@ erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx)
goto done;
}
default:
- erl_exit(ERTS_ABORT_EXIT, "dsig_send invalid phase (%d)\n", (int)ctx->phase);
+ erts_exit(ERTS_ABORT_EXIT, "dsig_send invalid phase (%d)\n", (int)ctx->phase);
}
}
@@ -1963,7 +1963,7 @@ dist_port_command(Port *prt, ErtsDistOutputBuf *obuf)
ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
if (size > (Uint) INT_MAX)
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"Absurdly large distribution output data buffer "
"(%beu bytes) passed.\n",
size);
@@ -2003,7 +2003,7 @@ dist_port_commandv(Port *prt, ErtsDistOutputBuf *obuf)
ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
if (size > (Uint) INT_MAX)
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"Absurdly large distribution output data buffer "
"(%beu bytes) passed.\n",
size);
@@ -3370,7 +3370,7 @@ send_nodes_mon_msgs(Process *c_p, Eterm what, Eterm node, Eterm type, Eterm reas
continue;
break;
default:
- erl_exit(ERTS_ABORT_EXIT, "Bad node type found\n");
+ erts_exit(ERTS_ABORT_EXIT, "Bad node type found\n");
}
}
@@ -3679,7 +3679,7 @@ erts_processes_monitoring_nodes(Process *c_p)
case ERTS_NODES_MON_OPT_TYPES: type = am_all; break;
case ERTS_NODES_MON_OPT_TYPE_VISIBLE: type = am_visible; break;
case ERTS_NODES_MON_OPT_TYPE_HIDDEN: type = am_hidden; break;
- default: erl_exit(ERTS_ABORT_EXIT, "Bad node type found\n");
+ default: erts_exit(ERTS_ABORT_EXIT, "Bad node type found\n");
}
olist = erts_bld_cons(hpp, szp,
erts_bld_tuple(hpp, szp, 2,
diff --git a/erts/emulator/beam/erl_afit_alloc.c b/erts/emulator/beam/erl_afit_alloc.c
index 47dafa53c0..4b0541c10e 100644
--- a/erts/emulator/beam/erl_afit_alloc.c
+++ b/erts/emulator/beam/erl_afit_alloc.c
@@ -241,7 +241,7 @@ info_options(Allctr_t *allctr,
if (hpp || szp) {
if (!atoms_initialized)
- erl_exit(1, "%s:%d: Internal error: Atoms not initialized",
+ erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error: Atoms not initialized",
__FILE__, __LINE__);;
res = NIL;
diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 806f569c38..490e0c0915 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -165,6 +165,8 @@ enum allctr_type {
struct au_init {
int enable;
int thr_spec;
+ int disable_allowed;
+ int thr_spec_allowed;
int carrier_migration_allowed;
enum allctr_type atype;
struct {
@@ -217,7 +219,7 @@ typedef struct {
struct au_init test_alloc;
} erts_alc_hndl_args_init_t;
-#define ERTS_AU_INIT__ {0, 0, 1, GOODFIT, DEFAULT_ALLCTR_INIT, {1,1,1,1}}
+#define ERTS_AU_INIT__ {0, 0, 1, 1, 1, GOODFIT, DEFAULT_ALLCTR_INIT, {1,1,1,1}}
#define SET_DEFAULT_ALLOC_OPTS(IP) \
do { \
@@ -294,6 +296,9 @@ set_default_literal_alloc_opts(struct au_init *ip)
SET_DEFAULT_ALLOC_OPTS(ip);
ip->enable = 1;
ip->thr_spec = 0;
+ ip->disable_allowed = 0;
+ ip->thr_spec_allowed = 0;
+ ip->carrier_migration_allowed = 0;
ip->atype = BESTFIT;
ip->init.bf.ao = 1;
ip->init.util.ramv = 0;
@@ -665,11 +670,11 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
if (mlockall(MCL_CURRENT|MCL_FUTURE) != 0) {
int err = errno;
char *errstr = err ? strerror(err) : "unknown";
- erl_exit(-1, "Failed to lock physical memory: %s (%d)\n",
+ erts_exit(1, "Failed to lock physical memory: %s (%d)\n",
errstr, err);
}
#else
- erl_exit(-1, "Failed to lock physical memory: Not supported\n");
+ erts_exit(1, "Failed to lock physical memory: Not supported\n");
#endif
}
@@ -769,9 +774,6 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
#if HAVE_ERTS_MSEG
init.mseg.nos = erts_no_schedulers;
erts_mseg_init(&init.mseg);
-# if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
- erts_mmap_init(&erts_literal_mmapper, &init.mseg.literal_mmap);
-# endif
#endif
erts_alcu_init(&init.alloc_util);
@@ -810,13 +812,13 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) {
if (!erts_allctrs[i].alloc)
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"Missing alloc function for %s\n", ERTS_ALC_A2AD(i));
if (!erts_allctrs[i].realloc)
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"Missing realloc function for %s\n", ERTS_ALC_A2AD(i));
if (!erts_allctrs[i].free)
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"Missing free function for %s\n", ERTS_ALC_A2AD(i));
}
@@ -887,7 +889,7 @@ erts_alloc_late_init(void)
static void *
erts_realloc_fixed_size(ErtsAlcType_t type, void *extra, void *p, Uint size)
{
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"Attempt to reallocate a block of the fixed size type %s\n",
ERTS_ALC_T2TD(type));
}
@@ -969,7 +971,7 @@ set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init, int ncpu)
#endif
{
#ifdef ERTS_SMP
- erl_exit(ERTS_ABORT_EXIT, "%salloc is not thread safe\n",
+ erts_exit(ERTS_ABORT_EXIT, "%salloc is not thread safe\n",
init->init.util.name_prefix);
#else
af->alloc = erts_alcu_alloc;
@@ -1014,7 +1016,7 @@ start_au_allocator(ErtsAlcType_t alctr_n,
* tspec->size)
+ ERTS_CACHE_LINE_SIZE - 1));
if (!states)
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"Failed to allocate allocator states for %salloc\n",
init->init.util.name_prefix);
tspec->allctr = (Allctr_t **) states;
@@ -1042,7 +1044,7 @@ start_au_allocator(ErtsAlcType_t alctr_n,
(tot_fix_list_size
+ ERTS_CACHE_LINE_SIZE - 1));
if (!fix_lists)
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"Failed to allocate fix lists for %salloc\n",
init->init.util.name_prefix);
@@ -1116,7 +1118,7 @@ start_au_allocator(ErtsAlcType_t alctr_n,
}
if (!as)
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"Failed to start %salloc\n", init->init.util.name_prefix);
ASSERT(as == (void *) as0);
@@ -1352,9 +1354,17 @@ handle_au_arg(struct au_init *auip,
else
goto bad_switch;
break;
- case 'e':
- auip->enable = get_bool_value(sub_param+1, argv, ip);
+ case 'e': {
+ int e = get_bool_value(sub_param + 1, argv, ip);
+ if (!auip->disable_allowed && !e) {
+ if (!u_switch)
+ bad_value(param, sub_param + 1, "false");
+ else
+ ASSERT(auip->enable); /* ignore */
+ }
+ else auip->enable = e;
break;
+ }
case 'l':
if (has_prefix("lmbcs", sub_param)) {
auip->default_.lmbcs = 0;
@@ -1423,7 +1433,14 @@ handle_au_arg(struct au_init *auip,
case 't': {
int res = get_bool_value(sub_param+1, argv, ip);
if (res > 0) {
- auip->thr_spec = 1;
+ if (!auip->thr_spec_allowed) {
+ if (!u_switch)
+ bad_value(param, sub_param + 1, "true");
+ else
+ ASSERT(!auip->thr_spec); /* ignore */
+ }
+ else
+ auip->thr_spec = 1;
break;
}
else if (res == 0) {
@@ -1472,6 +1489,16 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
case 'B':
handle_au_arg(&init->binary_alloc, &argv[i][3], argv, &i, 0);
break;
+ case 'I':
+ if (has_prefix("scs", argv[i]+3)) {
+#if HAVE_ERTS_MSEG
+ init->mseg.literal_mmap.scs =
+#endif
+ get_mb_value(argv[i]+6, argv, &i);
+ }
+ else
+ handle_au_arg(&init->literal_alloc, &argv[i][3], argv, &i, 0);
+ break;
case 'D':
handle_au_arg(&init->std_alloc, &argv[i][3], argv, &i, 0);
break;
@@ -1908,7 +1935,7 @@ erts_alc_fatal_error(int error, int func, ErtsAlcType_t n, ...)
case ERTS_ALC_O_FREE: op_str = "free"; break;
default: op_str = "UNKNOWN"; break;
}
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"%s: %s operation not supported (memory type: \"%s\")\n",
allctr_str, op_str, t_str);
break;
@@ -1922,18 +1949,18 @@ erts_alc_fatal_error(int error, int func, ErtsAlcType_t n, ...)
va_start(argp, n);
size = va_arg(argp, Uint);
va_end(argp);
- erl_exit(-1,
+ erts_exit(1,
"%s: Cannot %s %lu bytes of memory (of type \"%s\").\n",
allctr_str, op, size, t_str);
break;
}
case ERTS_ALC_E_NOALLCTR:
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"erts_alloc: Unknown allocator type: %d\n",
ERTS_ALC_T2A(ERTS_ALC_N2T(n)));
break;
default:
- erl_exit(ERTS_ABORT_EXIT, "erts_alloc: Unknown error: %d\n", error);
+ erts_exit(ERTS_ABORT_EXIT, "erts_alloc: Unknown error: %d\n", error);
break;
}
}
@@ -3120,7 +3147,7 @@ reply_alloc_info(void *vair)
make_small(0), ainfo);
}
else {
- erl_exit(ERTS_ABORT_EXIT, "%s:%d: internal error\n",
+ erts_exit(ERTS_ABORT_EXIT, "%s:%d: internal error\n",
__FILE__, __LINE__);
}
}
@@ -3354,7 +3381,7 @@ void *safe_realloc(void *ptr, Uint sz)
* Keep alloc_SUITE_data/allocator_test.h updated if changes are made *
* to erts_alc_test() *
\* */
-#define ERTS_ALC_TEST_ABORT erl_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error\n")
+#define ERTS_ALC_TEST_ABORT erts_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error\n")
UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3)
{
@@ -3819,7 +3846,7 @@ check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func)
found_type = GET_TYPE_OF_PATTERN(pre_pattern);
if (pre_pattern != MK_PATTERN(n)) {
if ((FIXED_FENCE_PATTERN_MASK & pre_pattern) != FIXED_FENCE_PATTERN)
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"ERROR: Fence at beginning of memory block (p=0x%u) "
"clobbered.\n",
(UWord) ptr);
@@ -3836,12 +3863,12 @@ check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func)
char *op_str;
if ((FIXED_FENCE_PATTERN_MASK & post_pattern) != FIXED_FENCE_PATTERN)
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"ERROR: Fence at end of memory block (p=0x%u, sz=%u) "
"clobbered.\n",
(UWord) ptr, (UWord) sz);
if (found_type != GET_TYPE_OF_PATTERN(post_pattern))
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"ERROR: Fence around memory block (p=0x%u, sz=%u) "
"clobbered.\n",
(UWord) ptr, (UWord) sz);
@@ -3864,7 +3891,7 @@ check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func)
default: op_str = "???"; break;
}
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"ERROR: Memory block (p=0x%u, sz=%u) allocated as type \"%s\","
" but %s as type \"%s\".\n",
(UWord) ptr, (UWord) sz, ftype, op_str, otype);
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index 5f153ac0ab..2932adca84 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -362,6 +362,7 @@ type SCHED_WTIME_REQ SHORT_LIVED SYSTEM sched_wall_time_request
type GC_INFO_REQ SHORT_LIVED SYSTEM gc_info_request
type PORT_DATA_HEAP STANDARD SYSTEM port_data_heap
type MSACC DRIVER SYSTEM microstate_accounting
+type SYS_CHECK_REQ SHORT_LIVED SYSTEM system_check_request
#
# Types used by system specific code
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index 3230b6ef34..5e7dd7cce8 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -4718,7 +4718,7 @@ make_name_atoms(Allctr_t *allctr)
size_t prefix_len = strlen(allctr->name_prefix);
if (prefix_len > MAX_ATOM_CHARACTERS + sizeof(realloc) - 1)
- erl_exit(1,"Too long allocator name: %salloc\n",allctr->name_prefix);
+ erts_exit(ERTS_ERROR_EXIT,"Too long allocator name: %salloc\n",allctr->name_prefix);
memcpy((void *) buf, (void *) allctr->name_prefix, prefix_len);
@@ -5912,7 +5912,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
/* erts_alcu_start assumes that allctr has been zeroed */
if (((UWord)allctr & ERTS_CRR_ALCTR_FLG_MASK) != 0) {
- erl_exit(ERTS_ABORT_EXIT, "%s:%d:erts_alcu_start: Alignment error\n",
+ erts_exit(ERTS_ABORT_EXIT, "%s:%d:erts_alcu_start: Alignment error\n",
__FILE__, __LINE__);
}
@@ -6128,7 +6128,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
if (allctr->thread_safe)
erts_mtx_destroy(&allctr->mutex);
#endif
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"Failed to create main carrier for %salloc\n",
init->name_prefix);
}
@@ -6273,7 +6273,7 @@ erts_alcu_test(UWord op, UWord a1, UWord a2)
case 0x019: return (UWord) PREV_BLK((Block_t *) a1);
case 0x01a: return (UWord) IS_MBC_FIRST_BLK((Allctr_t*)a1, (Block_t *) a2);
case 0x01b: return (UWord) sizeof(Unit_t);
- case 0x01c: return (unsigned long) BLK_TO_MBC((Block_t*) a1);
+ case 0x01c: return (UWord) BLK_TO_MBC((Block_t*) a1);
case 0x01d: ((Allctr_t*) a1)->add_mbc((Allctr_t*)a1, (Carrier_t*)a2); break;
case 0x01e: ((Allctr_t*) a1)->remove_mbc((Allctr_t*)a1, (Carrier_t*)a2); break;
#ifdef ERTS_SMP
@@ -6344,7 +6344,7 @@ erts_alcu_verify_unused(Allctr_t *allctr)
if (no) {
UWord sz = allctr->sbcs.blocks.curr.size;
sz += allctr->mbcs.blocks.curr.size;
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"%salloc() used when expected to be unused!\n"
"Total amount of blocks allocated: %bpu\n"
"Total amount of bytes allocated: %bpu\n",
diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.c b/erts/emulator/beam/erl_ao_firstfit_alloc.c
index 19420af8ab..5cd067ff54 100644
--- a/erts/emulator/beam/erl_ao_firstfit_alloc.c
+++ b/erts/emulator/beam/erl_ao_firstfit_alloc.c
@@ -1035,7 +1035,7 @@ info_options(Allctr_t *allctr,
if (hpp || szp) {
if (!atoms_initialized)
- erl_exit(1, "%s:%d: Internal error: Atoms not initialized",
+ erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error: Atoms not initialized",
__FILE__, __LINE__);;
res = NIL;
diff --git a/erts/emulator/beam/erl_bestfit_alloc.c b/erts/emulator/beam/erl_bestfit_alloc.c
index fb853b65ab..f39a18ac88 100644
--- a/erts/emulator/beam/erl_bestfit_alloc.c
+++ b/erts/emulator/beam/erl_bestfit_alloc.c
@@ -940,7 +940,7 @@ info_options(Allctr_t *allctr,
if (hpp || szp) {
if (!atoms_initialized)
- erl_exit(1, "%s:%d: Internal error: Atoms not initialized",
+ erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error: Atoms not initialized",
__FILE__, __LINE__);;
res = NIL;
diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c
index 2b1d875bfe..f006856b6a 100644
--- a/erts/emulator/beam/erl_bif_ddll.c
+++ b/erts/emulator/beam/erl_bif_ddll.c
@@ -1309,7 +1309,7 @@ static Eterm notify_when_loaded(Process *p, Eterm name_term, char *name, ErtsPro
case ERL_DE_FORCE_RELOAD:
break;
default:
- erl_exit(1,"Internal error, unknown state %u in dynamic driver.", drv->handle->status);
+ erts_exit(ERTS_ERROR_EXIT,"Internal error, unknown state %u in dynamic driver.", drv->handle->status);
}
p->flags |= F_USING_DDLL;
r = add_monitor(p, drv->handle, ERL_DE_PROC_AWAIT_LOAD);
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index bc5c83e542..8c748c9bf7 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -66,6 +66,7 @@ static Export* gather_io_bytes_trap = NULL;
static Export *gather_sched_wall_time_res_trap;
static Export *gather_msacc_res_trap;
static Export *gather_gc_info_res_trap;
+static Export *gather_system_check_res_trap;
#define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1)
@@ -319,13 +320,11 @@ erts_print_system_version(int to, void *arg, Process *c_p)
char *ov = otp_version;
#ifdef ERTS_SMP
Uint total, online, active;
-#ifdef ERTS_DIRTY_SCHEDULERS
Uint dirty_cpu, dirty_cpu_onln, dirty_io;
- (void) erts_schedulers_state(&total, &online, &active, &dirty_cpu, &dirty_cpu_onln, &dirty_io, 0);
-#else
- (void) erts_schedulers_state(&total, &online, &active, NULL, NULL, NULL, 0);
-#endif
+ erts_schedulers_state(&total, &online, &active,
+ &dirty_cpu, &dirty_cpu_onln, NULL,
+ &dirty_io, NULL);
#endif
for (i = 0; i < sizeof(otp_version)-4; i++) {
if (ov[i] == '-' && ov[i+1] == 'r' && ov[i+2] == 'c')
@@ -931,7 +930,7 @@ BIF_RETTYPE process_info_1(BIF_ALIST_1)
case ERTS_PI_FAIL_TYPE_AWAIT_EXIT:
ERTS_BIF_AWAIT_X_DATA_TRAP(BIF_P, BIF_ARG_1, am_undefined);
default:
- erl_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error", __FILE__, __LINE__);
+ erts_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error", __FILE__, __LINE__);
}
}
@@ -971,7 +970,7 @@ BIF_RETTYPE process_info_2(BIF_ALIST_2)
case ERTS_PI_FAIL_TYPE_AWAIT_EXIT:
ERTS_BIF_AWAIT_X_DATA_TRAP(BIF_P, BIF_ARG_1, am_undefined);
default:
- erl_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error",
+ erts_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error",
__FILE__, __LINE__);
}
}
@@ -1726,7 +1725,7 @@ info_1_tuple(Process* BIF_P, /* Pointer to current process. */
return res;
buf = (char *) erts_alloc(ERTS_ALC_T_TMP, len+1);
if (intlist_to_buf(*tp, buf, len) != len)
- erl_exit(1, "%s:%d: Internal error\n", __FILE__, __LINE__);
+ erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error\n", __FILE__, __LINE__);
buf[len] = '\0';
res = erts_instr_dump_memory_map(buf) ? am_true : am_false;
erts_free(ERTS_ALC_T_TMP, (void *) buf);
@@ -1745,7 +1744,7 @@ info_1_tuple(Process* BIF_P, /* Pointer to current process. */
return res;
buf = (char *) erts_alloc(ERTS_ALC_T_TMP, len+1);
if (intlist_to_buf(tp[1], buf, len) != len)
- erl_exit(1, "%s:%d: Internal error\n", __FILE__, __LINE__);
+ erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error\n", __FILE__, __LINE__);
buf[len] = '\0';
res = erts_instr_dump_stat(buf, 1) ? am_true : am_false;
erts_free(ERTS_ALC_T_TMP, (void *) buf);
@@ -2043,12 +2042,18 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
#ifndef ERTS_SMP
BIF_RET(am_disabled);
#else
+#ifndef ERTS_DIRTY_SCHEDULERS
if (erts_no_schedulers == 1)
BIF_RET(am_disabled);
- else {
- BIF_RET(erts_is_multi_scheduling_blocked()
- ? am_blocked
- : am_enabled);
+ else
+#endif
+ {
+ int msb = erts_is_multi_scheduling_blocked();
+ BIF_RET(!msb
+ ? am_enabled
+ : (msb > 0
+ ? am_blocked
+ : am_blocked_normal));
}
#endif
} else if (BIF_ARG_1 == am_build_type) {
@@ -2517,77 +2522,120 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
res = TUPLE3(hp, make_small(1), make_small(1), make_small(1));
BIF_RET(res);
#else
+ Eterm *hp;
Uint total, online, active;
- switch (erts_schedulers_state(&total,
- &online,
- &active,
- NULL,
- NULL,
- NULL,
- 1)) {
- case ERTS_SCHDLR_SSPND_DONE: {
- Eterm *hp = HAlloc(BIF_P, 4);
- res = TUPLE3(hp,
- make_small(total),
- make_small(online),
- make_small(active));
- BIF_RET(res);
+ erts_schedulers_state(&total, &online, &active,
+ NULL, NULL, NULL, NULL, NULL);
+ hp = HAlloc(BIF_P, 4);
+ res = TUPLE3(hp,
+ make_small(total),
+ make_small(online),
+ make_small(active));
+ BIF_RET(res);
+#endif
+ } else if (ERTS_IS_ATOM_STR("schedulers_state", BIF_ARG_1)) {
+#ifndef ERTS_SMP
+ Eterm *hp = HAlloc(BIF_P, 4);
+ res = TUPLE3(hp, make_small(1), make_small(1), make_small(1));
+ BIF_RET(res);
+#else
+ Eterm *hp;
+ Uint total, online, active;
+ erts_schedulers_state(&total, &online, &active,
+ NULL, NULL, NULL, NULL, NULL);
+ hp = HAlloc(BIF_P, 4);
+ res = TUPLE3(hp,
+ make_small(total),
+ make_small(online),
+ make_small(active));
+ BIF_RET(res);
+#endif
+ } else if (ERTS_IS_ATOM_STR("all_schedulers_state", BIF_ARG_1)) {
+#ifndef ERTS_SMP
+ Eterm *hp = HAlloc(BIF_P, 2+5);
+ res = CONS(hp+5,
+ TUPLE4(hp,
+ am_normal,
+ make_small(1),
+ make_small(1),
+ make_small(1)),
+ NIL);
+ BIF_RET(res);
+#else
+ Eterm *hp, tpl;
+ Uint sz, total, online, active,
+ dirty_cpu_total, dirty_cpu_online, dirty_cpu_active,
+ dirty_io_total, dirty_io_active;
+ erts_schedulers_state(&total, &online, &active,
+ &dirty_cpu_total, &dirty_cpu_online, &dirty_cpu_active,
+ &dirty_io_total, &dirty_io_active);
+
+ sz = 2+5;
+ if (dirty_cpu_total)
+ sz += 2+5;
+ if (dirty_io_total)
+ sz += 2+5;
+
+ hp = HAlloc(BIF_P, sz);
+
+ res = NIL;
+ if (dirty_io_total) {
+ tpl = TUPLE4(hp,
+ am_dirty_io,
+ make_small(dirty_io_total),
+ make_small(dirty_io_total),
+ make_small(dirty_io_active));
+ hp += 5;
+ res = CONS(hp, tpl, res);
+ hp += 2;
}
- case ERTS_SCHDLR_SSPND_YIELD_RESTART:
- ERTS_VBUMP_ALL_REDS(BIF_P);
- BIF_TRAP1(bif_export[BIF_system_info_1],
- BIF_P, BIF_ARG_1);
- default:
- ASSERT(0);
- BIF_ERROR(BIF_P, EXC_INTERNAL_ERROR);
+ if (dirty_cpu_total) {
+ tpl = TUPLE4(hp,
+ am_dirty_cpu,
+ make_small(dirty_cpu_total),
+ make_small(dirty_cpu_online),
+ make_small(dirty_cpu_active));
+ hp += 5;
+ res = CONS(hp, tpl, res);
+ hp += 2;
}
+ tpl = TUPLE4(hp,
+ am_normal,
+ make_small(total),
+ make_small(online),
+ make_small(active));
+ hp += 5;
+ res = CONS(hp, tpl, res);
+ BIF_RET(res);
#endif
} else if (ERTS_IS_ATOM_STR("schedulers_online", BIF_ARG_1)) {
#ifndef ERTS_SMP
BIF_RET(make_small(1));
#else
- Uint total, online, active;
- switch (erts_schedulers_state(&total, &online, &active, NULL, NULL, NULL, 1)) {
- case ERTS_SCHDLR_SSPND_DONE:
- BIF_RET(make_small(online));
- case ERTS_SCHDLR_SSPND_YIELD_RESTART:
- ERTS_VBUMP_ALL_REDS(BIF_P);
- BIF_TRAP1(bif_export[BIF_system_info_1],
- BIF_P, BIF_ARG_1);
- default:
- ASSERT(0);
- BIF_ERROR(BIF_P, EXC_INTERNAL_ERROR);
- }
+ Uint online;
+ erts_schedulers_state(NULL, &online, NULL, NULL, NULL, NULL, NULL, NULL);
+ BIF_RET(make_small(online));
#endif
} else if (ERTS_IS_ATOM_STR("schedulers_active", BIF_ARG_1)) {
#ifndef ERTS_SMP
BIF_RET(make_small(1));
#else
- Uint total, online, active;
- switch (erts_schedulers_state(&total, &online, &active, NULL, NULL, NULL, 1)) {
- case ERTS_SCHDLR_SSPND_DONE:
- BIF_RET(make_small(active));
- case ERTS_SCHDLR_SSPND_YIELD_RESTART:
- ERTS_VBUMP_ALL_REDS(BIF_P);
- BIF_TRAP1(bif_export[BIF_system_info_1],
- BIF_P, BIF_ARG_1);
- default:
- ASSERT(0);
- BIF_ERROR(BIF_P, EXC_INTERNAL_ERROR);
- }
+ Uint active;
+ erts_schedulers_state(NULL, NULL, &active, NULL, NULL, NULL, NULL, NULL);
+ BIF_RET(make_small(active));
#endif
#if defined(ERTS_SMP) && defined(ERTS_DIRTY_SCHEDULERS)
} else if (ERTS_IS_ATOM_STR("dirty_cpu_schedulers", BIF_ARG_1)) {
Uint dirty_cpu;
- erts_schedulers_state(NULL, NULL, NULL, &dirty_cpu, NULL, NULL, 1);
+ erts_schedulers_state(NULL, NULL, NULL, &dirty_cpu, NULL, NULL, NULL, NULL);
BIF_RET(make_small(dirty_cpu));
} else if (ERTS_IS_ATOM_STR("dirty_cpu_schedulers_online", BIF_ARG_1)) {
Uint dirty_cpu_onln;
- erts_schedulers_state(NULL, NULL, NULL, NULL, &dirty_cpu_onln, NULL, 1);
+ erts_schedulers_state(NULL, NULL, NULL, NULL, &dirty_cpu_onln, NULL, NULL, NULL);
BIF_RET(make_small(dirty_cpu_onln));
} else if (ERTS_IS_ATOM_STR("dirty_io_schedulers", BIF_ARG_1)) {
Uint dirty_io;
- erts_schedulers_state(NULL, NULL, NULL, NULL, NULL, &dirty_io, 1);
+ erts_schedulers_state(NULL, NULL, NULL, NULL, NULL, NULL, &dirty_io, NULL);
BIF_RET(make_small(dirty_io));
#endif
} else if (ERTS_IS_ATOM_STR("run_queues", BIF_ARG_1)) {
@@ -2641,7 +2689,16 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
if (erts_no_schedulers == 1)
BIF_RET(NIL);
else
- BIF_RET(erts_multi_scheduling_blockers(BIF_P));
+ BIF_RET(erts_multi_scheduling_blockers(BIF_P, 0));
+#endif
+ } else if (ERTS_IS_ATOM_STR("normal_multi_scheduling_blockers", BIF_ARG_1)) {
+#ifndef ERTS_SMP
+ BIF_RET(NIL);
+#else
+ if (erts_no_schedulers == 1)
+ BIF_RET(NIL);
+ else
+ BIF_RET(erts_multi_scheduling_blockers(BIF_P, 1));
#endif
} else if (ERTS_IS_ATOM_STR("modified_timing_level", BIF_ARG_1)) {
BIF_RET(ERTS_USE_MODIFIED_TIMING()
@@ -3786,6 +3843,18 @@ BIF_RETTYPE erts_internal_is_system_process_1(BIF_ALIST_1)
BIF_ERROR(BIF_P, BADARG);
}
+BIF_RETTYPE erts_internal_system_check_1(BIF_ALIST_1)
+{
+ Eterm res;
+ if (ERTS_IS_ATOM_STR("schedulers", BIF_ARG_1)) {
+ res = erts_system_check_request(BIF_P);
+ if (is_non_value(res))
+ BIF_RET(am_undefined);
+ BIF_TRAP1(gather_system_check_res_trap, BIF_P, res);
+ }
+
+ BIF_ERROR(BIF_P, BADARG);
+}
static erts_smp_atomic_t hipe_test_reschedule_flag;
@@ -3800,7 +3869,7 @@ static void broken_halt_test(Eterm bif_arg_2)
#if defined(ERTS_HAVE_TRY_CATCH)
erts_get_scheduler_data()->run_queue = NULL;
#endif
- erl_exit(ERTS_DUMP_EXIT, "%T", bif_arg_2);
+ erts_exit(ERTS_DUMP_EXIT, "%T", bif_arg_2);
}
@@ -4045,7 +4114,7 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
BIF_RET(am_true);
}
else if (ERTS_IS_ATOM_STR("abort", BIF_ARG_1)) {
- erl_exit(ERTS_ABORT_EXIT, "%T\n", BIF_ARG_2);
+ erts_exit(ERTS_ABORT_EXIT, "%T\n", BIF_ARG_2);
}
else if (ERTS_IS_ATOM_STR("kill_dist_connection", BIF_ARG_1)) {
DistEntry *dep = erts_sysname_to_connected_dist_entry(BIF_ARG_2);
@@ -4407,7 +4476,8 @@ erts_bif_info_init(void)
= erts_export_put(am_erts_internal, am_gather_io_bytes, 2);
gather_msacc_res_trap
= erts_export_put(am_erts_internal, am_gather_microstate_accounting_result, 2);
-
+ gather_system_check_res_trap
+ = erts_export_put(am_erts_internal, am_gather_system_check_result, 1);
process_info_init();
os_info_init();
}
diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c
index 3acc1d7bbd..86bcf43678 100644
--- a/erts/emulator/beam/erl_bif_port.c
+++ b/erts/emulator/beam/erl_bif_port.c
@@ -823,7 +823,7 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump)
} else {
name_buf = (char *) erts_alloc(ERTS_ALC_T_TMP, i + 1);
if (intlist_to_buf(name, name_buf, i) != i)
- erl_exit(1, "%s:%d: Internal error\n", __FILE__, __LINE__);
+ erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error\n", __FILE__, __LINE__);
name_buf[i] = '\0';
}
driver = &vanilla_driver;
@@ -1201,7 +1201,7 @@ static Eterm http_bld_uri(struct packet_callback_args* pca,
return erts_bld_tuple(hpp, szp, 3, am_scheme, s1, s2);
default:
- erl_exit(1, "%s, line %d: type=%u\n", __FILE__, __LINE__, uri->type);
+ erts_exit(ERTS_ERROR_EXIT, "%s, line %d: type=%u\n", __FILE__, __LINE__, uri->type);
}
}
diff --git a/erts/emulator/beam/erl_cpu_topology.c b/erts/emulator/beam/erl_cpu_topology.c
index 8395f6ecc6..c263eebfcc 100644
--- a/erts/emulator/beam/erl_cpu_topology.c
+++ b/erts/emulator/beam/erl_cpu_topology.c
@@ -402,7 +402,7 @@ cpu_bind_order_sort(erts_cpu_topology_t *cpudata,
break;
default:
cmp_func = NULL;
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"Bad cpu bind type: %d\n",
(int) cpu_bind_order);
break;
@@ -1590,7 +1590,7 @@ get_cpu_topology_term(Process *c_p, int type)
}
break;
default:
- erl_exit(ERTS_ABORT_EXIT, "Bad cpu topology type: %d\n", type);
+ erts_exit(ERTS_ABORT_EXIT, "Bad cpu topology type: %d\n", type);
break;
}
@@ -1967,7 +1967,7 @@ cpu_group_insert(erts_cpu_groups_map_t *map,
ix = 0;
} while (ix != start);
- erl_exit(ERTS_ABORT_EXIT, "Reader groups map full\n");
+ erts_exit(ERTS_ABORT_EXIT, "Reader groups map full\n");
}
@@ -2290,7 +2290,7 @@ remove_cpu_groups(erts_cpu_groups_callback_t callback, void *arg)
prev_cgm = cgm;
}
- erl_exit(ERTS_ABORT_EXIT, "Cpu groups not found\n");
+ erts_exit(ERTS_ABORT_EXIT, "Cpu groups not found\n");
}
static int
@@ -2320,7 +2320,7 @@ cpu_groups_lookup(erts_cpu_groups_map_t *map,
ix = 0;
} while (ix != start);
- erl_exit(ERTS_ABORT_EXIT, "Logical cpu id %d not found\n", logical);
+ erts_exit(ERTS_ABORT_EXIT, "Logical cpu id %d not found\n", logical);
}
static void
diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index 55b8789e29..441dbdde4f 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -1295,7 +1295,7 @@ BIF_RETTYPE ets_rename_2(BIF_ALIST_2)
goto badarg;
if (!remove_named_tab(tb, 1))
- erl_exit(1,"Could not find named tab %s", tb->common.id);
+ erts_exit(ERTS_ERROR_EXIT,"Could not find named tab %s", tb->common.id);
tb->common.id = tb->common.the_name = BIF_ARG_2;
@@ -1580,7 +1580,7 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
BIF_P->common.id,
make_small(slot)),
0) != DB_ERROR_NONE) {
- erl_exit(1,"Could not update ets metadata.");
+ erts_exit(ERTS_ERROR_EXIT,"Could not update ets metadata.");
}
db_meta_unlock(meta_pid_to_tab, LCK_WRITE_REC);
@@ -2933,7 +2933,7 @@ void init_db(ErtsDbSpinCount db_spin_count)
bits = erts_fit_in_bits_int32(db_max_tabs-1);
if (bits > SMALL_BITS) {
- erl_exit(1,"Max limit for ets tabled too high %u (max %u).",
+ erts_exit(ERTS_ERROR_EXIT,"Max limit for ets tabled too high %u (max %u).",
db_max_tabs, ((Uint)1)<<SMALL_BITS);
}
meta_main_tab_slot_mask = (((Uint)1)<<bits) - 1;
@@ -2994,7 +2994,7 @@ void init_db(ErtsDbSpinCount db_spin_count)
db_init_lock(meta_pid_to_tab, "meta_pid_to_tab", "meta_pid_to_tab_FIX");*/
if (db_create_hash(NULL, meta_pid_to_tab) != DB_ERROR_NONE) {
- erl_exit(1,"Unable to create ets metadata tables.");
+ erts_exit(ERTS_ERROR_EXIT,"Unable to create ets metadata tables.");
}
erts_smp_atomic_set_nob(&init_tb.common.memory_size, 0);
@@ -3025,7 +3025,7 @@ void init_db(ErtsDbSpinCount db_spin_count)
db_init_lock(meta_pid_to_fixed_tab, "meta_pid_to_fixed_tab", "meta_pid_to_fixed_tab_FIX");*/
if (db_create_hash(NULL, meta_pid_to_fixed_tab) != DB_ERROR_NONE) {
- erl_exit(1,"Unable to create ets metadata tables.");
+ erts_exit(ERTS_ERROR_EXIT,"Unable to create ets metadata tables.");
}
/* Non visual BIF to trap to. */
@@ -3222,7 +3222,7 @@ retry:
* yielding.
*/
#define ERTS_DB_INTERNAL_ERROR(LSTR) \
- erl_exit(ERTS_ABORT_EXIT, "%s:%d:erts_db_process_exiting(): " LSTR "\n", \
+ erts_exit(ERTS_ABORT_EXIT, "%s:%d:erts_db_process_exiting(): " LSTR "\n", \
__FILE__, __LINE__)
int
@@ -3494,7 +3494,7 @@ static void fix_table_locked(Process* p, DbTable* tb)
make_small(tb->common.slot)),
0) != DB_ERROR_NONE) {
UnUseTmpHeap(3,p);
- erl_exit(1,"Could not insert ets metadata in safe_fixtable.");
+ erts_exit(ERTS_ERROR_EXIT,"Could not insert ets metadata in safe_fixtable.");
}
UnUseTmpHeap(3,p);
db_meta_unlock(meta_pid_to_fixed_tab, LCK_WRITE_REC);
diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index cff65f244d..240b79fc60 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -3022,7 +3022,7 @@ void db_check_table_hash(DbTable *tbl)
if ((list = BUCKET(tb,j)) != 0) {
while (list != 0) {
if (!is_tuple(make_tuple(list->dbterm.tpl))) {
- erl_exit(1, "Bad term in slot %d of ets table", j);
+ erts_exit(ERTS_ERROR_EXIT, "Bad term in slot %d of ets table", j);
}
list = list->next;
}
diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c
index 311b69d114..0f642f0867 100644
--- a/erts/emulator/beam/erl_db_tree.c
+++ b/erts/emulator/beam/erl_db_tree.c
@@ -1204,7 +1204,7 @@ static int db_select_count_continue_tree(Process *p,
tptr = tuple_val(continuation);
if (arityval(*tptr) != 5)
- erl_exit(1,"Internal error in ets:select_count/1");
+ erts_exit(ERTS_ERROR_EXIT,"Internal error in ets:select_count/1");
lastkey = tptr[2];
end_condition = tptr[3];
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index 399be6058c..973be58420 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -1862,11 +1862,11 @@ restart:
#ifdef DMC_DEBUG
if (*heap_fence != FENCE_PATTERN) {
- erl_exit(1, "Heap fence overwritten in db_prog_match after op "
+ erts_exit(ERTS_ERROR_EXIT, "Heap fence overwritten in db_prog_match after op "
"0x%08x, overwritten with 0x%08x.", save_op, *heap_fence);
}
if (*stack_fence != FENCE_PATTERN) {
- erl_exit(1, "Stack fence overwritten in db_prog_match after op "
+ erts_exit(ERTS_ERROR_EXIT, "Stack fence overwritten in db_prog_match after op "
"0x%08x, overwritten with 0x%08x.", save_op,
*stack_fence);
}
@@ -2495,7 +2495,7 @@ restart:
case matchHalt:
goto success;
default:
- erl_exit(1, "Internal error: unexpected opcode in match program.");
+ erts_exit(ERTS_ERROR_EXIT, "Internal error: unexpected opcode in match program.");
}
}
fail:
@@ -2512,11 +2512,11 @@ success:
#ifdef DMC_DEBUG
if (*heap_fence != FENCE_PATTERN) {
- erl_exit(1, "Heap fence overwritten in db_prog_match after op "
+ erts_exit(ERTS_ERROR_EXIT, "Heap fence overwritten in db_prog_match after op "
"0x%08x, overwritten with 0x%08x.", save_op, *heap_fence);
}
if (*stack_fence != FENCE_PATTERN) {
- erl_exit(1, "Stack fence overwritten in db_prog_match after op "
+ erts_exit(ERTS_ERROR_EXIT, "Stack fence overwritten in db_prog_match after op "
"0x%08x, overwritten with 0x%08x.", save_op,
*stack_fence);
}
@@ -3448,7 +3448,7 @@ static DMCRet dmc_one_term(DMCContext *context,
break;
}
default:
- erl_exit(1, "db_match_compile: "
+ erts_exit(ERTS_ERROR_EXIT, "db_match_compile: "
"Bad object on heap: 0x%bex\n", c);
}
return retOk;
@@ -4679,7 +4679,7 @@ static DMCRet dmc_fun(DMCContext *context,
DMC_PUSH(*text, matchCall3);
break;
default:
- erl_exit(1,"ets:match() internal error, "
+ erts_exit(ERTS_ERROR_EXIT,"ets:match() internal error, "
"guard with more than 3 arguments.");
}
DMC_PUSH(*text, (UWord) b->biff);
@@ -4959,7 +4959,7 @@ static Uint my_size_object(Eterm t)
tmp == am_const) {
sum += size_object(tuple_val(t)[2]);
} else {
- erl_exit(1,"Internal error, sizing unrecognized object in "
+ erts_exit(ERTS_ERROR_EXIT,"Internal error, sizing unrecognized object in "
"(d)ets:match compilation.");
}
break;
@@ -5004,7 +5004,7 @@ static Eterm my_copy_struct(Eterm t, Eterm **hp, ErlOffHeap* off_heap)
sz = size_object(b);
ret = copy_struct(b,sz,hp,off_heap);
} else {
- erl_exit(1, "Trying to constant-copy non constant expression "
+ erts_exit(ERTS_ERROR_EXIT, "Trying to constant-copy non constant expression "
"0x%bex in (d)ets:match compilation.", t);
}
} else {
diff --git a/erts/emulator/beam/erl_debug.c b/erts/emulator/beam/erl_debug.c
index 7b13cf2c83..2902c98864 100644
--- a/erts/emulator/beam/erl_debug.c
+++ b/erts/emulator/beam/erl_debug.c
@@ -255,14 +255,14 @@ void erts_check_stack(Process *p)
Eterm *stack_end = p->htop;
if (p->stop > stack_start)
- erl_exit(1,
+ erts_exit(ERTS_ERROR_EXIT,
"<%lu.%lu.%lu>: Stack underflow\n",
internal_pid_channel_no(p->common.id),
internal_pid_number(p->common.id),
internal_pid_serial(p->common.id));
if (p->stop < stack_end)
- erl_exit(1,
+ erts_exit(ERTS_ERROR_EXIT,
"<%lu.%lu.%lu>: Stack overflow\n",
internal_pid_channel_no(p->common.id),
internal_pid_number(p->common.id),
@@ -287,7 +287,7 @@ void erts_check_stack(Process *p)
if (in_mbuf)
continue;
- erl_exit(1,
+ erts_exit(ERTS_ERROR_EXIT,
"<%lu.%lu.%lu>: Wild stack pointer\n",
internal_pid_channel_no(p->common.id),
internal_pid_number(p->common.id),
@@ -374,7 +374,7 @@ void erts_check_memory(Process *p, Eterm *start, Eterm *end)
#ifdef DEBUG
if (hval == DEBUG_BAD_WORD) {
print_untagged_memory(start, end);
- erl_exit(1, "Uninitialized HAlloc'ed memory found @ 0x%0*lx!\n",
+ erts_exit(ERTS_ERROR_EXIT, "Uninitialized HAlloc'ed memory found @ 0x%0*lx!\n",
PTR_SIZE,(unsigned long)(pos - 1));
}
#endif
@@ -387,7 +387,7 @@ void erts_check_memory(Process *p, Eterm *start, Eterm *end)
if (verify_eterm(p,hval))
continue;
- erl_exit(1, "Wild pointer found @ 0x%0*lx!\n",
+ erts_exit(ERTS_ERROR_EXIT, "Wild pointer found @ 0x%0*lx!\n",
PTR_SIZE,(unsigned long)(pos - 1));
}
}
@@ -397,11 +397,11 @@ void verify_process(Process *p)
#define VERIFY_AREA(name,ptr,sz) { \
int n = (sz); \
while (n--) if(!verify_eterm(p,*(ptr+n))) \
- erl_exit(1,"Wild pointer found in " name " of %T!\n",p->common.id); }
+ erts_exit(ERTS_ERROR_EXIT,"Wild pointer found in " name " of %T!\n",p->common.id); }
#define VERIFY_ETERM(name,eterm) { \
if(!verify_eterm(p,eterm)) \
- erl_exit(1,"Wild pointer found in " name " of %T!\n",p->common.id); }
+ erts_exit(ERTS_ERROR_EXIT,"Wild pointer found in " name " of %T!\n",p->common.id); }
ErtsMessage* mp = p->msg.first;
diff --git a/erts/emulator/beam/erl_drv_thread.c b/erts/emulator/beam/erl_drv_thread.c
index e0404eb5c9..184f8e8931 100644
--- a/erts/emulator/beam/erl_drv_thread.c
+++ b/erts/emulator/beam/erl_drv_thread.c
@@ -43,7 +43,7 @@ fatal_error(int err, char *func)
else
estr = "Unknown error";
}
- erl_exit(ERTS_ABORT_EXIT, "Fatal error in %s: %s [%d]\n", func, estr, err);
+ erts_exit(ERTS_ABORT_EXIT, "Fatal error in %s: %s [%d]\n", func, estr, err);
}
#define ERL_DRV_TSD_KEYS_INC 10
diff --git a/erts/emulator/beam/erl_fun.c b/erts/emulator/beam/erl_fun.c
index cff476694c..ed555aa92f 100644
--- a/erts/emulator/beam/erl_fun.c
+++ b/erts/emulator/beam/erl_fun.c
@@ -188,7 +188,7 @@ erts_erase_fun_entry(ErlFunEntry* fe)
#endif
{
if (fe->address != unloaded_fun)
- erl_exit(1,
+ erts_exit(ERTS_ERROR_EXIT,
"Internal error: "
"Invalid reference count found on #Fun<%T.%d.%d>: "
" About to erase fun still referred by code.\n",
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 21b03ae8bd..0d125c5598 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -70,7 +70,7 @@
erts_fprintf(stderr, "stop=%p\n", (p)->stop); \
erts_fprintf(stderr, "htop=%p\n", (p)->htop); \
erts_fprintf(stderr, "heap=%p\n", (p)->heap); \
- erl_exit(ERTS_ABORT_EXIT, "%s, line %d: %T: Overrun stack and heap\n", \
+ erts_exit(ERTS_ABORT_EXIT, "%s, line %d: %T: Overrun stack and heap\n", \
__FILE__,__LINE__,(P)->common.id); \
}
@@ -289,7 +289,7 @@ erts_next_heap_size(Uint size, Uint offset)
low = mid + 1;
}
}
- erl_exit(1, "no next heap size found: %beu, offset %beu\n", size, offset);
+ erts_exit(ERTS_ERROR_EXIT, "no next heap size found: %beu, offset %beu\n", size, offset);
}
return 0;
}
@@ -3067,7 +3067,7 @@ within(Eterm *ptr, Process *p)
#define ERTS_CHK_OFFHEAP_ASSERT(EXP) \
do { \
if (!(EXP)) \
- erl_exit(ERTS_ABORT_EXIT, \
+ erts_exit(ERTS_ABORT_EXIT, \
"%s:%d: Assertion failed: %s\n", \
__FILE__, __LINE__, #EXP); \
} while (0)
diff --git a/erts/emulator/beam/erl_goodfit_alloc.c b/erts/emulator/beam/erl_goodfit_alloc.c
index f89f8723d9..9b4aad9d91 100644
--- a/erts/emulator/beam/erl_goodfit_alloc.c
+++ b/erts/emulator/beam/erl_goodfit_alloc.c
@@ -571,8 +571,8 @@ info_options(Allctr_t *allctr,
if (hpp || szp) {
if (!atoms_initialized)
- erl_exit(1, "%s:%d: Internal error: Atoms not initialized",
- __FILE__, __LINE__);;
+ erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error: Atoms not initialized",
+ __FILE__, __LINE__);
res = NIL;
add_2tup(hpp, szp, &res, am.as, am.gf);
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 67089df863..86c4eeb484 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -264,7 +264,7 @@ this_rel_num(void)
i++;
this_rel = atoi(&this_rel_str[i]);
if (this_rel < 1)
- erl_exit(-1, "Unexpected ERLANG_OTP_RELEASE format\n");
+ erts_exit(1, "Unexpected ERLANG_OTP_RELEASE format\n");
}
return this_rel;
}
@@ -412,7 +412,7 @@ erl_first_process_otp(char* modname, void* code, unsigned size, int argc, char**
start_mod = erts_atom_put((byte *) modname, sys_strlen(modname), ERTS_ATOM_ENC_LATIN1, 1);
if (erts_find_function(start_mod, am_start, 2,
erts_active_code_ix()) == NULL) {
- erl_exit(5, "No function %s:start/2\n", modname);
+ erts_exit(ERTS_ERROR_EXIT, "No function %s:start/2\n", modname);
}
/*
@@ -451,7 +451,7 @@ erl_system_process_otp(Eterm parent_pid, char* modname)
start_mod = erts_atom_put((byte *) modname, sys_strlen(modname), ERTS_ATOM_ENC_LATIN1, 1);
if (erts_find_function(start_mod, am_start, 0,
erts_active_code_ix()) == NULL) {
- erl_exit(5, "No function %s:start/0\n", modname);
+ erts_exit(ERTS_ERROR_EXIT, "No function %s:start/0\n", modname);
}
parent = erts_pid2proc(NULL, 0, parent_pid, ERTS_PROC_LOCK_MAIN);
@@ -532,12 +532,12 @@ load_preloaded(void)
length = preload_p[i].size;
module_name = erts_atom_put((byte *) name, sys_strlen(name), ERTS_ATOM_ENC_LATIN1, 1);
if ((code = sys_preload_begin(&preload_p[i])) == 0)
- erl_exit(1, "Failed to find preloaded code for module %s\n",
+ erts_exit(ERTS_ERROR_EXIT, "Failed to find preloaded code for module %s\n",
name);
res = erts_preload_module(NULL, 0, NIL, &module_name, code, length);
sys_preload_end(&preload_p[i]);
if (res != NIL)
- erl_exit(1,"Failed loading preloaded module %s (%T)\n",
+ erts_exit(ERTS_ERROR_EXIT,"Failed loading preloaded module %s (%T)\n",
name, res);
i++;
}
@@ -670,7 +670,7 @@ void erts_usage(void)
erts_fprintf(stderr, "Note that if the emulator is started with erlexec (typically\n");
erts_fprintf(stderr, "from the erl script), these flags should be specified with +.\n");
erts_fprintf(stderr, "\n\n");
- erl_exit(-1, "");
+ erts_exit(1, "");
}
#ifdef USE_THREADS
@@ -1472,7 +1472,7 @@ erl_start(int argc, char **argv)
}
erts_fprintf(stderr, "(" EMULATOR ") emulator version "
ERLANG_VERSION "\n");
- erl_exit(0, "");
+ erts_exit(0, "");
}
break;
@@ -2304,23 +2304,17 @@ system_cleanup(int flush_async)
}
static __decl_noreturn void __noreturn
-erl_exit_vv(int n, int flush_async, char *fmt, va_list args1, va_list args2)
+erts_exit_vv(int n, int flush_async, char *fmt, va_list args1, va_list args2)
{
- unsigned int an;
-
system_cleanup(flush_async);
save_statistics();
- if (n < 0)
- an = -(unsigned int)n;
- else
- an = n;
if (erts_mtrace_enabled)
- erts_mtrace_exit((Uint32) an);
+ erts_mtrace_exit((Uint32) n);
/* Produce an Erlang core dump if error */
- if (((n > 0 && erts_no_crash_dump == 0) || n == ERTS_DUMP_EXIT)
+ if (((n == ERTS_ERROR_EXIT && erts_no_crash_dump == 0) || n == ERTS_DUMP_EXIT)
&& erts_initialized) {
erl_crash_dump_v((char*) NULL, 0, fmt, args1);
}
@@ -2333,29 +2327,29 @@ erl_exit_vv(int n, int flush_async, char *fmt, va_list args1, va_list args2)
exit(0);
else if (n == ERTS_DUMP_EXIT)
ERTS_EXIT_AFTER_DUMP(1);
- else if (n > 0 || n == ERTS_ABORT_EXIT)
+ else if (n == ERTS_ERROR_EXIT || n == ERTS_ABORT_EXIT)
abort();
- exit(an);
+ exit(n);
}
/* Exit without flushing async threads */
-__decl_noreturn void __noreturn erl_exit(int n, char *fmt, ...)
+__decl_noreturn void __noreturn erts_exit(int n, char *fmt, ...)
{
va_list args1, args2;
va_start(args1, fmt);
va_start(args2, fmt);
- erl_exit_vv(n, 0, fmt, args1, args2);
+ erts_exit_vv(n, 0, fmt, args1, args2);
va_end(args2);
va_end(args1);
}
/* Exit after flushing async threads */
-__decl_noreturn void __noreturn erl_exit_flush_async(int n, char *fmt, ...)
+__decl_noreturn void __noreturn erts_flush_async_exit(int n, char *fmt, ...)
{
va_list args1, args2;
va_start(args1, fmt);
va_start(args2, fmt);
- erl_exit_vv(n, 1, fmt, args1, args2);
+ erts_exit_vv(n, 1, fmt, args1, args2);
va_end(args2);
va_end(args1);
}
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index d0ffb11e79..4b6931f848 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -1270,7 +1270,7 @@ recurse:
break;
}
default:
- erl_exit(ERTS_ABORT_EXIT, "bad header %ld\r\n", hdrA);
+ erts_exit(ERTS_ABORT_EXIT, "bad header %ld\r\n", hdrA);
}
}
@@ -1297,7 +1297,7 @@ recurse:
break;
}
default:
- erl_exit(ERTS_ABORT_EXIT, "bad header %ld\r\n", hdrB);
+ erts_exit(ERTS_ABORT_EXIT, "bad header %ld\r\n", hdrB);
}
}
}
@@ -1387,7 +1387,7 @@ resume_from_trap:
res = make_boxed(nhp);
break;
default:
- erl_exit(ERTS_ABORT_EXIT, "strange mix %d\r\n", sp->mix);
+ erts_exit(ERTS_ABORT_EXIT, "strange mix %d\r\n", sp->mix);
}
}
@@ -1882,7 +1882,7 @@ void hashmap_iterator_init(ErtsWStack* s, Eterm node, int reverse) {
sz = hashmap_bitcount(MAP_HEADER_VAL(hdr));
break;
default:
- erl_exit(ERTS_ABORT_EXIT, "bad header");
+ erts_exit(ERTS_ABORT_EXIT, "bad header");
}
WSTACK_PUSH3((*s), (UWord)THE_NON_VALUE, /* end marker */
@@ -1919,7 +1919,7 @@ Eterm* hashmap_iterator_next(ErtsWStack* s) {
ASSERT(sz < 17);
break;
default:
- erl_exit(ERTS_ABORT_EXIT, "bad header");
+ erts_exit(ERTS_ABORT_EXIT, "bad header");
}
idx++;
@@ -1969,7 +1969,7 @@ Eterm* hashmap_iterator_prev(ErtsWStack* s) {
ASSERT(sz < 17);
break;
default:
- erl_exit(1, "bad header");
+ erts_exit(ERTS_ERROR_EXIT, "bad header");
}
if (idx > sz)
@@ -2153,12 +2153,12 @@ int erts_hashmap_insert_down(Uint32 hx, Eterm key, Eterm node, Uint *sz,
size += HAMT_HEAD_BITMAP_SZ(n+1);
goto unroll;
default:
- erl_exit(1, "bad header tag %ld\r\n", hdr & _HEADER_MAP_SUBTAG_MASK);
+ erts_exit(ERTS_ERROR_EXIT, "bad header tag %ld\r\n", hdr & _HEADER_MAP_SUBTAG_MASK);
break;
}
break;
default:
- erl_exit(1, "bad primary tag %p\r\n", node);
+ erts_exit(ERTS_ERROR_EXIT, "bad primary tag %p\r\n", node);
break;
}
}
@@ -2273,12 +2273,12 @@ Eterm erts_hashmap_insert_up(Eterm *hp, Eterm key, Eterm value,
res = make_hashmap(nhp);
break;
default:
- erl_exit(1, "bad header tag %x\r\n", hdr & _HEADER_MAP_SUBTAG_MASK);
+ erts_exit(ERTS_ERROR_EXIT, "bad header tag %x\r\n", hdr & _HEADER_MAP_SUBTAG_MASK);
break;
}
break;
default:
- erl_exit(1, "bad primary tag %x\r\n", primary_tag(node));
+ erts_exit(ERTS_ERROR_EXIT, "bad primary tag %x\r\n", primary_tag(node));
break;
}
@@ -2396,12 +2396,12 @@ static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm map) {
/* not occupied */
goto not_found;
default:
- erl_exit(1, "bad header tag %ld\r\n", hdr & _HEADER_MAP_SUBTAG_MASK);
+ erts_exit(ERTS_ERROR_EXIT, "bad header tag %ld\r\n", hdr & _HEADER_MAP_SUBTAG_MASK);
break;
}
break;
default:
- erl_exit(1, "bad primary tag %p\r\n", node);
+ erts_exit(ERTS_ERROR_EXIT, "bad primary tag %p\r\n", node);
break;
}
}
@@ -2578,7 +2578,7 @@ unroll:
res = make_hashmap(nhp);
break;
default:
- erl_exit(1, "bad header tag %x\r\n", hdr & _HEADER_MAP_SUBTAG_MASK);
+ erts_exit(ERTS_ERROR_EXIT, "bad header tag %x\r\n", hdr & _HEADER_MAP_SUBTAG_MASK);
break;
}
} while(!ESTACK_ISEMPTY(stack));
@@ -2728,7 +2728,7 @@ BIF_RETTYPE erts_internal_term_type_1(BIF_ALIST_1) {
case MAP_HEADER_TAG_HAMT_NODE_BITMAP :
BIF_RET(ERTS_MAKE_AM("hashmap_node"));
default:
- erl_exit(ERTS_ABORT_EXIT, "term_type: bad map header type %d\n", MAP_HEADER_TYPE(hdr));
+ erts_exit(ERTS_ABORT_EXIT, "term_type: bad map header type %d\n", MAP_HEADER_TYPE(hdr));
}
case REFC_BINARY_SUBTAG:
BIF_RET(ERTS_MAKE_AM("refc_binary"));
@@ -2752,7 +2752,7 @@ BIF_RETTYPE erts_internal_term_type_1(BIF_ALIST_1) {
case FLOAT_SUBTAG:
BIF_RET(ERTS_MAKE_AM("hfloat"));
default:
- erl_exit(ERTS_ABORT_EXIT, "term_type: Invalid tag (0x%X)\n", hdr);
+ erts_exit(ERTS_ABORT_EXIT, "term_type: Invalid tag (0x%X)\n", hdr);
}
}
case TAG_PRIMARY_IMMED1:
@@ -2772,13 +2772,13 @@ BIF_RETTYPE erts_internal_term_type_1(BIF_ALIST_1) {
case _TAG_IMMED2_NIL:
BIF_RET(ERTS_MAKE_AM("nil"));
default:
- erl_exit(ERTS_ABORT_EXIT, "term_type: Invalid tag (0x%X)\n", obj);
+ erts_exit(ERTS_ABORT_EXIT, "term_type: Invalid tag (0x%X)\n", obj);
}
default:
- erl_exit(ERTS_ABORT_EXIT, "term_type: Invalid tag (0x%X)\n", obj);
+ erts_exit(ERTS_ABORT_EXIT, "term_type: Invalid tag (0x%X)\n", obj);
}
default:
- erl_exit(ERTS_ABORT_EXIT, "term_type: Invalid tag (0x%X)\n", obj);
+ erts_exit(ERTS_ABORT_EXIT, "term_type: Invalid tag (0x%X)\n", obj);
}
}
@@ -2811,7 +2811,7 @@ BIF_RETTYPE erts_internal_map_hashmap_children_1(BIF_ALIST_1) {
ptr += 2;
break;
default:
- erl_exit(1, "bad header\r\n");
+ erts_exit(ERTS_ERROR_EXIT, "bad header\r\n");
break;
}
ASSERT(sz < 17);
@@ -2889,7 +2889,7 @@ static Eterm hashmap_info(Process *p, Eterm node) {
}
break;
default:
- erl_exit(1, "bad header\r\n");
+ erts_exit(ERTS_ERROR_EXIT, "bad header\r\n");
break;
}
}
diff --git a/erts/emulator/beam/erl_monitors.c b/erts/emulator/beam/erl_monitors.c
index 7dfa01c8ac..bd899fa2f9 100644
--- a/erts/emulator/beam/erl_monitors.c
+++ b/erts/emulator/beam/erl_monitors.c
@@ -356,7 +356,7 @@ void erts_add_monitor(ErtsMonitor **root, Uint type, Eterm ref, Eterm pid,
tstack[tpos++] = this;
this = &((*this)->right);
} else { /* Equal key is an error for monitors */
- erl_exit(1,"Insertion of already present monitor!");
+ erts_exit(ERTS_ERROR_EXIT,"Insertion of already present monitor!");
break;
}
}
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 2204231748..8580bc2689 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -331,7 +331,7 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
#ifdef ERTS_SMP
c_p = NULL;
#else
- erl_exit(ERTS_ABORT_EXIT,"enif_send: env==NULL on non-SMP VM");
+ erts_exit(ERTS_ABORT_EXIT,"enif_send: env==NULL on non-SMP VM");
#endif
}
diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index 13f14adbab..8617f42d7b 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -1053,7 +1053,7 @@ insert_dist_entry(DistEntry *dist, int type, Eterm id, Uint creation)
}
if(!rdp)
- erl_exit(1,
+ erts_exit(ERTS_ERROR_EXIT,
"Reference to non-existing distribution table entry found!\n");
insert_dist_referrer(rdp, type, id, creation);
@@ -1114,7 +1114,7 @@ insert_node(ErlNode *node, int type, Eterm id)
}
if (!rnp)
- erl_exit(1, "Reference to non-existing node table entry found!\n");
+ erts_exit(ERTS_ERROR_EXIT, "Reference to non-existing node table entry found!\n");
insert_node_referrer(rnp, type, id);
}
diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c
index 197b328fe2..e85395e062 100644
--- a/erts/emulator/beam/erl_port_task.c
+++ b/erts/emulator/beam/erl_port_task.c
@@ -1786,7 +1786,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
reds = erts_dist_command(pp, CONTEXT_REDS - pp->reds);
break;
default:
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"Invalid port task type: %d\n",
(int) ptp->type);
break;
@@ -2051,7 +2051,7 @@ begin_port_cleanup(Port *pp, ErtsPortTask **execqp, int *processing_busy_q_p)
break;
}
default:
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"Invalid port task type: %d\n",
(int) ptp->type);
}
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index fd29c22810..a706ebf595 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -153,10 +153,8 @@ int ERTS_WRITE_UNLIKELY(erts_eager_check_io) = 1;
int ERTS_WRITE_UNLIKELY(erts_sched_compact_load);
int ERTS_WRITE_UNLIKELY(erts_sched_balance_util) = 0;
Uint ERTS_WRITE_UNLIKELY(erts_no_schedulers);
-#ifdef ERTS_DIRTY_SCHEDULERS
-Uint ERTS_WRITE_UNLIKELY(erts_no_dirty_cpu_schedulers);
-Uint ERTS_WRITE_UNLIKELY(erts_no_dirty_io_schedulers);
-#endif
+Uint ERTS_WRITE_UNLIKELY(erts_no_dirty_cpu_schedulers) = 0;
+Uint ERTS_WRITE_UNLIKELY(erts_no_dirty_io_schedulers) = 0;
static char *erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_NO_FLAGS] = {0};
int erts_aux_work_no_flags = ERTS_SSI_AUX_WORK_NO_FLAGS;
@@ -188,84 +186,176 @@ int erts_disable_proc_not_running_opt;
static ErtsAuxWorkData *aux_thread_aux_work_data;
-#define ERTS_SCHDLR_SSPND_CHNG_WAITER (((erts_aint32_t) 1) << 0)
+#define ERTS_SCHDLR_SSPND_CHNG_NMSB (((erts_aint32_t) 1) << 0)
#define ERTS_SCHDLR_SSPND_CHNG_MSB (((erts_aint32_t) 1) << 1)
#define ERTS_SCHDLR_SSPND_CHNG_ONLN (((erts_aint32_t) 1) << 2)
+#define ERTS_SCHDLR_SSPND_CHNG_DCPU_ONLN (((erts_aint32_t) 1) << 3)
-#ifndef DEBUG
+typedef enum {
+ ERTS_SCHED_NORMAL,
+ ERTS_SCHED_DIRTY_CPU,
+ ERTS_SCHED_DIRTY_IO
+} ErtsSchedType;
-#define ERTS_SCHDLR_SSPND_CHNG_SET(VAL, OLD_VAL) \
- erts_smp_atomic32_set_nob(&schdlr_sspnd.changing, (VAL))
+typedef struct {
+ int ongoing;
+ ErtsProcList *blckrs;
+ ErtsProcList *chngq;
+} ErtsMultiSchedulingBlock;
-#ifdef ERTS_DIRTY_SCHEDULERS
-#define ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET(VAL, OLD_VAL) \
- erts_smp_atomic32_set_nob(&schdlr_sspnd.dirty_cpu_changing, (VAL))
-#define ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET(VAL, OLD_VAL) \
- erts_smp_atomic32_set_nob(&schdlr_sspnd.dirty_io_changing, (VAL))
-#endif
+static struct {
+ erts_smp_mtx_t mtx;
+ Uint32 online;
+ Uint32 curr_online;
+ Uint32 active;
+ erts_smp_atomic32_t changing;
+ ErtsProcList *chngq;
+ Eterm changer;
+ ErtsMultiSchedulingBlock nmsb; /* Normal multi Scheduling Block */
+ ErtsMultiSchedulingBlock msb; /* Multi Scheduling Block */
+} schdlr_sspnd;
-#else
+#define ERTS_SCHDLR_SSPND_S_BITS 10
+#define ERTS_SCHDLR_SSPND_DCS_BITS 11
+#define ERTS_SCHDLR_SSPND_DIS_BITS 11
-#define ERTS_SCHDLR_SSPND_CHNG_SET(VAL, OLD_VAL) \
-do { \
- erts_aint32_t old_val__; \
- old_val__ = erts_smp_atomic32_xchg_nob(&schdlr_sspnd.changing, \
- (VAL)); \
- ASSERT(old_val__ == (OLD_VAL)); \
-} while (0)
+#define ERTS_SCHDLR_SSPND_S_MASK ((1 << ERTS_SCHDLR_SSPND_S_BITS)-1)
+#define ERTS_SCHDLR_SSPND_DCS_MASK ((1 << ERTS_SCHDLR_SSPND_DCS_BITS)-1)
+#define ERTS_SCHDLR_SSPND_DIS_MASK ((1 << ERTS_SCHDLR_SSPND_DIS_BITS)-1)
-#ifdef ERTS_DIRTY_SCHEDULERS
-#define ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET(VAL, OLD_VAL) \
-do { \
- erts_aint32_t old_val__; \
- old_val__ = erts_smp_atomic32_xchg_nob(&schdlr_sspnd.dirty_cpu_changing, \
- (VAL)); \
- ASSERT(old_val__ == (OLD_VAL)); \
-} while (0)
-#define ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET(VAL, OLD_VAL) \
-do { \
- erts_aint32_t old_val__; \
- old_val__ = erts_smp_atomic32_xchg_nob(&schdlr_sspnd.dirty_io_changing, \
- (VAL)); \
- ASSERT(old_val__ == (OLD_VAL)); \
-} while (0)
-#endif
+#define ERTS_SCHDLR_SSPND_S_SHIFT 0
+#define ERTS_SCHDLR_SSPND_DCS_SHIFT (ERTS_SCHDLR_SSPND_S_SHIFT \
+ + ERTS_SCHDLR_SSPND_S_BITS)
+#define ERTS_SCHDLR_SSPND_DIS_SHIFT (ERTS_SCHDLR_SSPND_DCS_SHIFT \
+ + ERTS_SCHDLR_SSPND_DCS_BITS)
+#if (ERTS_SCHDLR_SSPND_S_BITS \
+ + ERTS_SCHDLR_SSPND_DCS_BITS \
+ + ERTS_SCHDLR_SSPND_DIS_BITS) > 32
+# error Wont fit in Uint32
#endif
-
-static struct {
- erts_smp_mtx_t mtx;
- erts_smp_cnd_t cnd;
- int online;
- int curr_online;
- int wait_curr_online;
-#ifdef ERTS_DIRTY_SCHEDULERS
- int dirty_cpu_online;
- int dirty_cpu_curr_online;
- int dirty_cpu_wait_curr_online;
- int dirty_io_online;
- int dirty_io_curr_online;
- int dirty_io_wait_curr_online;
+#if (ERTS_MAX_NO_OF_SCHEDULERS-1) > ERTS_SCHDLR_SSPND_S_MASK
+# error Max no schedulers wont fit in its bit-field
#endif
- erts_smp_atomic32_t changing;
- erts_smp_atomic32_t active;
-#ifdef ERTS_DIRTY_SCHEDULERS
- erts_smp_atomic32_t dirty_cpu_changing;
- erts_smp_atomic32_t dirty_cpu_active;
- erts_smp_atomic32_t dirty_io_changing;
- erts_smp_atomic32_t dirty_io_active;
+#if ERTS_MAX_NO_OF_DIRTY_CPU_SCHEDULERS > ERTS_SCHDLR_SSPND_DCS_MASK
+# error Max no dirty cpu schedulers wont fit in its bit-field
#endif
- struct {
- int ongoing;
- long wait_active;
-#ifdef ERTS_DIRTY_SCHEDULERS
- long dirty_cpu_wait_active;
- long dirty_io_wait_active;
+#if ERTS_MAX_NO_OF_DIRTY_IO_SCHEDULERS > ERTS_SCHDLR_SSPND_DIS_MASK
+# error Max no dirty io schedulers wont fit in its bit-field
#endif
- ErtsProcList *procs;
- } msb; /* Multi Scheduling Block */
-} schdlr_sspnd;
+
+#define ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(S, DCS, DIS) \
+ ((((Uint32) (((S) & ERTS_SCHDLR_SSPND_S_MASK))-1) \
+ << ERTS_SCHDLR_SSPND_S_SHIFT) \
+ | ((((Uint32) ((DCS) & ERTS_SCHDLR_SSPND_DCS_MASK)) \
+ << ERTS_SCHDLR_SSPND_DCS_SHIFT)) \
+ | ((((Uint32) ((DIS) & ERTS_SCHDLR_SSPND_DIS_MASK)) \
+ << ERTS_SCHDLR_SSPND_DIS_SHIFT)))
+
+static void init_scheduler_suspend(void);
+
+static ERTS_INLINE Uint32
+schdlr_sspnd_get_nscheds(Uint32 *valp, ErtsSchedType type)
+{
+ Uint32 res = (Uint32) (*valp);
+ switch (type) {
+ case ERTS_SCHED_NORMAL:
+ res >>= ERTS_SCHDLR_SSPND_S_SHIFT;
+ res &= (Uint32) ERTS_SCHDLR_SSPND_S_MASK;
+ res++;
+ break;
+ case ERTS_SCHED_DIRTY_CPU:
+ res >>= ERTS_SCHDLR_SSPND_DCS_SHIFT;
+ res &= (Uint32) ERTS_SCHDLR_SSPND_DCS_MASK;
+ break;
+ case ERTS_SCHED_DIRTY_IO:
+ res >>= ERTS_SCHDLR_SSPND_DIS_SHIFT;
+ res &= (Uint32) ERTS_SCHDLR_SSPND_DIS_MASK;
+ break;
+ default:
+ ERTS_INTERNAL_ERROR("Invalid scheduler type");
+ return 0;
+ }
+
+ return res;
+}
+
+static ERTS_INLINE void
+schdlr_sspnd_dec_nscheds(Uint32 *valp, ErtsSchedType type)
+{
+ ASSERT(schdlr_sspnd_get_nscheds(valp, type) > 0);
+
+ switch (type) {
+ case ERTS_SCHED_NORMAL:
+ *valp -= ((Uint32) 1) << ERTS_SCHDLR_SSPND_S_SHIFT;
+ break;
+ case ERTS_SCHED_DIRTY_CPU:
+ *valp -= ((Uint32) 1) << ERTS_SCHDLR_SSPND_DCS_SHIFT;
+ break;
+ case ERTS_SCHED_DIRTY_IO:
+ *valp -= ((Uint32) 1) << ERTS_SCHDLR_SSPND_DIS_SHIFT;
+ break;
+ default:
+ ERTS_INTERNAL_ERROR("Invalid scheduler type");
+ }
+}
+
+static ERTS_INLINE void
+schdlr_sspnd_inc_nscheds(Uint32 *valp, ErtsSchedType type)
+{
+ switch (type) {
+ case ERTS_SCHED_NORMAL:
+ ASSERT(schdlr_sspnd_get_nscheds(valp, type)
+ < ERTS_MAX_NO_OF_SCHEDULERS-1);
+ *valp += ((Uint32) 1) << ERTS_SCHDLR_SSPND_S_SHIFT;
+ break;
+ case ERTS_SCHED_DIRTY_CPU:
+ ASSERT(schdlr_sspnd_get_nscheds(valp, type)
+ < ERTS_MAX_NO_OF_DIRTY_CPU_SCHEDULERS);
+ *valp += ((Uint32) 1) << ERTS_SCHDLR_SSPND_DCS_SHIFT;
+ break;
+ case ERTS_SCHED_DIRTY_IO:
+ ASSERT(schdlr_sspnd_get_nscheds(valp, type)
+ < ERTS_MAX_NO_OF_DIRTY_IO_SCHEDULERS);
+ *valp += ((Uint32) 1) << ERTS_SCHDLR_SSPND_DIS_SHIFT;
+ break;
+ default:
+ ERTS_INTERNAL_ERROR("Invalid scheduler type");
+ }
+}
+
+static ERTS_INLINE void
+schdlr_sspnd_set_nscheds(Uint32 *valp, ErtsSchedType type, Uint32 no)
+{
+ Uint32 val = *valp;
+
+ switch (type) {
+ case ERTS_SCHED_NORMAL:
+ ASSERT(no > 0);
+ val &= ~(((Uint32) ERTS_SCHDLR_SSPND_S_MASK)
+ << ERTS_SCHDLR_SSPND_S_SHIFT);
+ val |= (((no-1) & ((Uint32) ERTS_SCHDLR_SSPND_S_MASK))
+ << ERTS_SCHDLR_SSPND_S_SHIFT);
+ break;
+ case ERTS_SCHED_DIRTY_CPU:
+ val &= ~(((Uint32) ERTS_SCHDLR_SSPND_DCS_MASK)
+ << ERTS_SCHDLR_SSPND_DCS_SHIFT);
+ val |= ((no & ((Uint32) ERTS_SCHDLR_SSPND_DCS_MASK))
+ << ERTS_SCHDLR_SSPND_DCS_SHIFT);
+ break;
+ case ERTS_SCHED_DIRTY_IO:
+ val &= ~(((Uint32) ERTS_SCHDLR_SSPND_DIS_MASK)
+ << ERTS_SCHDLR_SSPND_DIS_SHIFT);
+ val |= ((no & ((Uint32) ERTS_SCHDLR_SSPND_DIS_MASK))
+ << ERTS_SCHDLR_SSPND_DIS_SHIFT);
+ break;
+ default:
+ ERTS_INTERNAL_ERROR("Invalid scheduler type");
+ }
+
+ *valp = val;
+}
static struct {
erts_smp_mtx_t update_mtx;
@@ -426,8 +516,10 @@ do { \
do { \
ErtsRunQueue *RQVAR; \
int ix__; \
+ int online__ = (int) schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, \
+ ERTS_SCHED_NORMAL); \
ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&schdlr_sspnd.mtx)); \
- for (ix__ = 0; ix__ < schdlr_sspnd.online; ix__++) { \
+ for (ix__ = 0; ix__ < online__; ix__++) { \
RQVAR = ERTS_RUNQ_IX(ix__); \
erts_smp_runq_lock(RQVAR); \
{ DO; } \
@@ -505,7 +597,7 @@ dbg_chk_aux_work_val(erts_aint32_t value)
valid |= ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED;
if (~valid & value)
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"Invalid aux_work value found: 0x%x\n",
~valid & value);
}
@@ -959,10 +1051,24 @@ typedef struct {
erts_smp_atomic32_t refc;
} ErtsSchedWallTimeReq;
+typedef struct {
+ Process *proc;
+ Eterm ref;
+ Eterm ref_heap[REF_THING_SIZE];
+ Uint req_sched;
+ erts_smp_atomic32_t refc;
+} ErtsSystemCheckReq;
+
+
ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(swtreq,
- ErtsSchedWallTimeReq,
- 5,
- ERTS_ALC_T_SCHED_WTIME_REQ)
+ ErtsSchedWallTimeReq,
+ 5,
+ ERTS_ALC_T_SCHED_WTIME_REQ)
+
+ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(screq,
+ ErtsSystemCheckReq,
+ 5,
+ ERTS_ALC_T_SYS_CHECK_REQ)
static void
reply_sched_wall_time(void *vswtrp)
@@ -1095,6 +1201,75 @@ erts_sched_wall_time_request(Process *c_p, int set, int enable)
return ref;
}
+static void
+reply_system_check(void *vscrp)
+{
+ ErtsSchedulerData *esdp = erts_get_scheduler_data();
+ ErtsSystemCheckReq *scrp = (ErtsSystemCheckReq *) vscrp;
+ ErtsProcLocks rp_locks = (scrp->req_sched == esdp->no ? ERTS_PROC_LOCK_MAIN : 0);
+ Process *rp = scrp->proc;
+ Eterm msg;
+ Eterm *hp = NULL;
+ Eterm **hpp;
+ Uint sz;
+ ErlOffHeap *ohp = NULL;
+ ErtsMessage *mp = NULL;
+
+ ASSERT(esdp);
+#ifdef ERTS_DIRTY_SCHEDULERS
+ ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp));
+#endif
+
+ sz = REF_THING_SIZE;
+ mp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp);
+ hpp = &hp;
+ msg = STORE_NC(hpp, ohp, scrp->ref);
+
+ erts_queue_message(rp, &rp_locks, mp, msg, NIL);
+
+ if (scrp->req_sched == esdp->no)
+ rp_locks &= ~ERTS_PROC_LOCK_MAIN;
+
+ if (rp_locks)
+ erts_smp_proc_unlock(rp, rp_locks);
+
+ erts_proc_dec_refc(rp);
+
+ if (erts_smp_atomic32_dec_read_nob(&scrp->refc) == 0)
+ screq_free(vscrp);
+}
+
+
+Eterm erts_system_check_request(Process *c_p) {
+ ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(c_p);
+ Eterm ref;
+ ErtsSystemCheckReq *scrp;
+ Eterm *hp;
+
+ scrp = screq_alloc();
+ ref = erts_make_ref(c_p);
+ hp = &scrp->ref_heap[0];
+
+ scrp->proc = c_p;
+ scrp->ref = STORE_NC(&hp, NULL, ref);
+ scrp->req_sched = esdp->no;
+ erts_smp_atomic32_init_nob(&scrp->refc, (erts_aint32_t) erts_no_schedulers);
+
+ erts_proc_add_refc(c_p, (Sint) erts_no_schedulers);
+
+#ifdef ERTS_SMP
+ if (erts_no_schedulers > 1)
+ erts_schedule_multi_misc_aux_work(1,
+ erts_no_schedulers,
+ reply_system_check,
+ (void *) scrp);
+#endif
+
+ reply_system_check((void *) scrp);
+
+ return ref;
+}
+
static ERTS_INLINE ErtsProcList *
proclist_create(Process *p)
{
@@ -1105,6 +1280,15 @@ proclist_create(Process *p)
return plp;
}
+static ERTS_INLINE ErtsProcList *
+proclist_copy(ErtsProcList *plp0)
+{
+ ErtsProcList *plp1 = proclist_alloc();
+ plp1->pid = plp0->pid;
+ plp1->started_interval = plp0->started_interval;
+ return plp1;
+}
+
static ERTS_INLINE void
proclist_destroy(ErtsProcList *plp)
{
@@ -1112,6 +1296,12 @@ proclist_destroy(ErtsProcList *plp)
}
ErtsProcList *
+erts_proclist_copy(ErtsProcList *plp)
+{
+ return proclist_copy(plp);
+}
+
+ErtsProcList *
erts_proclist_create(Process *p)
{
return proclist_create(p);
@@ -1189,7 +1379,7 @@ erts_sched_finish_poke(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flags)
case 0:
break;
default:
- erl_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error\n",
+ erts_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error\n",
__FILE__, __LINE__);
break;
}
@@ -2147,7 +2337,7 @@ handle_reap_ports(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting)
erts_port_release(prt);
}
if (erts_smp_atomic32_dec_read_nob(&erts_halt_progress) == 0) {
- erl_exit_flush_async(erts_halt_code, "");
+ erts_flush_async_exit(erts_halt_code, "");
}
}
return aux_work & ~ERTS_SSI_AUX_WORK_REAP_PORTS;
@@ -2549,13 +2739,6 @@ sched_active(Uint no, ErtsRunQueue *rq)
profile_scheduler(make_small(no), am_active);
}
-static int ERTS_INLINE
-ongoing_multi_scheduling_block(void)
-{
- ERTS_SMP_LC_ASSERT(erts_lc_mtx_is_locked(&schdlr_sspnd.mtx));
- return schdlr_sspnd.msb.ongoing;
-}
-
static ERTS_INLINE void
empty_runq_aux(ErtsRunQueue *rq, Uint32 old_flags)
{
@@ -2942,8 +3125,10 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
timeout_time = erts_check_next_timeout_time(esdp);
current_time = erts_get_monotonic_time(esdp);
do_timeout = (current_time >= timeout_time);
- } else
+ } else {
+ current_time = 0;
timeout_time = ERTS_MONOTONIC_TIME_MAX;
+ }
if (do_timeout) {
if (!thr_prgr_active) {
erts_thr_progress_active(esdp, thr_prgr_active = 1);
@@ -3665,7 +3850,7 @@ immigrate(ErtsRunQueue *c_rq, ErtsMigrationPath *mp)
prio = ERTS_PORT_PRIO_LEVEL;
break;
default:
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"%s:%d:%s(): Invalid immigrate queue mask",
__FILE__, __LINE__, __func__);
prio = 0;
@@ -3689,7 +3874,7 @@ immigrate(ErtsRunQueue *c_rq, ErtsMigrationPath *mp)
rq = erts_port_runq(prt);
if (rq) {
if (rq != c_rq)
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"%s:%d:%s(): Internal error",
__FILE__, __LINE__, __func__);
erts_enqueue_port(c_rq, prt);
@@ -3754,24 +3939,33 @@ static ERTS_INLINE void
resume_run_queue(ErtsRunQueue *rq)
{
int pix;
+ Uint32 oflgs;
erts_smp_runq_lock(rq);
- (void) ERTS_RUNQ_FLGS_READ_BSET(rq,
- (ERTS_RUNQ_FLG_OUT_OF_WORK
- | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK
- | ERTS_RUNQ_FLG_SUSPENDED),
- (ERTS_RUNQ_FLG_OUT_OF_WORK
- | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK));
+ oflgs = ERTS_RUNQ_FLGS_READ_BSET(rq,
+ (ERTS_RUNQ_FLG_OUT_OF_WORK
+ | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK
+ | ERTS_RUNQ_FLG_SUSPENDED),
+ (ERTS_RUNQ_FLG_OUT_OF_WORK
+ | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK));
+
+ if (oflgs & ERTS_RUNQ_FLG_SUSPENDED) {
+ erts_aint32_t len;
+
+ rq->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS;
+ for (pix = 0; pix < ERTS_NO_PROC_PRIO_LEVELS; pix++) {
+ len = erts_smp_atomic32_read_dirty(&rq->procs.prio_info[pix].len);
+ rq->procs.prio_info[pix].max_len = len;
+ rq->procs.prio_info[pix].reds = 0;
+ }
+ len = erts_smp_atomic32_read_dirty(&rq->ports.info.len);
+ rq->ports.info.max_len = len;
+ rq->ports.info.reds = 0;
+ len = erts_smp_atomic32_read_dirty(&rq->len);
+ rq->max_len = len;
- rq->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS;
- for (pix = 0; pix < ERTS_NO_PROC_PRIO_LEVELS; pix++) {
- rq->procs.prio_info[pix].max_len = 0;
- rq->procs.prio_info[pix].reds = 0;
}
- rq->ports.info.max_len = 0;
- rq->ports.info.reds = 0;
- rq->max_len = 0;
erts_smp_runq_unlock(rq);
@@ -3880,7 +4074,7 @@ evacuate_run_queue(ErtsRunQueue *rq,
prt_rq = erts_port_runq(prt);
if (prt_rq) {
if (prt_rq != to_rq)
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"%s:%d:%s() internal error\n",
__FILE__, __LINE__, __func__);
erts_enqueue_port(to_rq, prt);
@@ -4125,7 +4319,7 @@ no_procs:
return 0;
else {
if (prt_rq != rq)
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"%s:%d:%s() internal error\n",
__FILE__, __LINE__, __func__);
*rq_lockedp = 1;
@@ -5550,6 +5744,9 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
char *daww_ptr;
size_t daww_sz;
size_t size_runqs;
+#ifdef ERTS_SMP
+ erts_aint32_t set_schdlr_sspnd_change_flags;
+#endif
init_misc_op_list_alloc();
init_proc_sys_task_queues_alloc();
@@ -5776,6 +5973,7 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
init_misc_aux_work();
init_swtreq_alloc();
+ init_screq_alloc();
erts_atomic32_init_nob(&debug_wait_completed_count, 0); /* debug only */
debug_wait_completed_flags = 0;
@@ -5786,25 +5984,6 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
erts_alloc_permanent_cache_aligned(ERTS_ALC_T_SCHDLR_DATA,
sizeof(ErtsAuxWorkData));
- erts_smp_mtx_init(&schdlr_sspnd.mtx, "schdlr_sspnd");
- erts_smp_cnd_init(&schdlr_sspnd.cnd);
-
- erts_smp_atomic32_init_nob(&schdlr_sspnd.changing, 0);
- schdlr_sspnd.online = no_schedulers_online;
- schdlr_sspnd.curr_online = no_schedulers;
- schdlr_sspnd.msb.ongoing = 0;
- erts_smp_atomic32_init_nob(&schdlr_sspnd.active, no_schedulers);
-#ifdef ERTS_DIRTY_SCHEDULERS
- erts_smp_atomic32_init_nob(&schdlr_sspnd.dirty_cpu_changing, 0);
- schdlr_sspnd.dirty_cpu_online = no_dirty_cpu_schedulers_online;
- schdlr_sspnd.dirty_cpu_curr_online = no_dirty_cpu_schedulers;
- erts_smp_atomic32_init_nob(&schdlr_sspnd.dirty_cpu_active, no_dirty_cpu_schedulers);
- erts_smp_atomic32_init_nob(&schdlr_sspnd.dirty_io_changing, 0);
- schdlr_sspnd.dirty_io_online = no_dirty_io_schedulers;
- schdlr_sspnd.dirty_io_curr_online = no_dirty_io_schedulers;
- erts_smp_atomic32_init_nob(&schdlr_sspnd.dirty_io_active, no_dirty_io_schedulers);
-#endif
- schdlr_sspnd.msb.procs = NULL;
init_no_runqs(no_schedulers_online, no_schedulers_online);
balance_info.last_active_runqs = no_schedulers;
erts_smp_mtx_init(&balance_info.update_mtx, "migration_info_update");
@@ -5819,32 +5998,66 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
init_migration_paths();
- if (no_schedulers_online < no_schedulers) {
+ init_scheduler_suspend();
+
+ set_schdlr_sspnd_change_flags = 0;
+
+ schdlr_sspnd_set_nscheds(&schdlr_sspnd.online,
+ ERTS_SCHED_NORMAL,
+ no_schedulers_online);
+ schdlr_sspnd_set_nscheds(&schdlr_sspnd.curr_online,
+ ERTS_SCHED_NORMAL,
+ no_schedulers);
+ schdlr_sspnd_set_nscheds(&schdlr_sspnd.active,
+ ERTS_SCHED_NORMAL,
+ no_schedulers);
+
+ if (no_schedulers_online != no_schedulers) {
+ ASSERT(no_schedulers_online < no_schedulers);
+ set_schdlr_sspnd_change_flags |= ERTS_SCHDLR_SSPND_CHNG_ONLN;
+ schdlr_sspnd.changer = am_init;
change_no_used_runqs(no_schedulers_online);
for (ix = no_schedulers_online; ix < erts_no_run_queues; ix++)
suspend_run_queue(ERTS_RUNQ_IX(ix));
}
- schdlr_sspnd.wait_curr_online = no_schedulers_online;
- schdlr_sspnd.curr_online *= 2; /* Boot strapping... */
- ERTS_SCHDLR_SSPND_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN
- | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0);
#ifdef ERTS_DIRTY_SCHEDULERS
- schdlr_sspnd.dirty_cpu_wait_curr_online = no_dirty_cpu_schedulers_online;
- schdlr_sspnd.dirty_cpu_curr_online *= 2;
- ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN
- | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0);
- for (ix = no_dirty_cpu_schedulers_online; ix < no_dirty_cpu_schedulers; ix++) {
- ErtsSchedulerData* esdp = ERTS_DIRTY_CPU_SCHEDULER_IX(ix);
- erts_smp_atomic32_read_bor_nob(&esdp->ssi->flags, ERTS_SSI_FLG_SUSPENDED);
+
+ schdlr_sspnd_set_nscheds(&schdlr_sspnd.online,
+ ERTS_SCHED_DIRTY_CPU,
+ no_dirty_cpu_schedulers_online);
+ schdlr_sspnd_set_nscheds(&schdlr_sspnd.curr_online,
+ ERTS_SCHED_DIRTY_CPU,
+ no_dirty_cpu_schedulers);
+ schdlr_sspnd_set_nscheds(&schdlr_sspnd.active,
+ ERTS_SCHED_DIRTY_CPU,
+ no_dirty_cpu_schedulers);
+
+ if (no_dirty_cpu_schedulers_online != no_dirty_cpu_schedulers) {
+ ASSERT(no_dirty_cpu_schedulers_online < no_dirty_cpu_schedulers);
+ set_schdlr_sspnd_change_flags |= ERTS_SCHDLR_SSPND_CHNG_DCPU_ONLN;
+ for (ix = no_dirty_cpu_schedulers_online; ix < no_dirty_cpu_schedulers; ix++) {
+ ErtsSchedulerData* esdp = ERTS_DIRTY_CPU_SCHEDULER_IX(ix);
+ erts_smp_atomic32_read_bor_nob(&esdp->ssi->flags, ERTS_SSI_FLG_SUSPENDED);
+ }
}
- schdlr_sspnd.dirty_io_wait_curr_online = no_dirty_io_schedulers;
- schdlr_sspnd.dirty_io_curr_online *= 2;
- ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN
- | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0);
+ schdlr_sspnd_set_nscheds(&schdlr_sspnd.online,
+ ERTS_SCHED_DIRTY_IO,
+ no_dirty_io_schedulers);
+ schdlr_sspnd_set_nscheds(&schdlr_sspnd.curr_online,
+ ERTS_SCHED_DIRTY_IO,
+ no_dirty_io_schedulers);
+ schdlr_sspnd_set_nscheds(&schdlr_sspnd.active,
+ ERTS_SCHED_DIRTY_IO,
+ no_dirty_io_schedulers);
+
#endif
+ if (set_schdlr_sspnd_change_flags)
+ erts_smp_atomic32_set_nob(&schdlr_sspnd.changing,
+ set_schdlr_sspnd_change_flags);
+
erts_smp_atomic32_init_nob(&doing_sys_schedule, 0);
init_misc_aux_work();
@@ -5859,11 +6072,9 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
#endif
}
erts_no_schedulers = 1;
-#ifdef ERTS_DIRTY_SCHEDULERS
erts_no_dirty_cpu_schedulers = 0;
erts_no_dirty_io_schedulers = 0;
#endif
-#endif
erts_smp_atomic32_init_nob(&function_calls, 0);
@@ -6578,9 +6789,6 @@ suspend_process(Process *c_p, Process *p)
if (suspended) {
- ASSERT(!(ERTS_PSFLG_RUNNING & state)
- || p == erts_get_current_process());
-
if (suspended > 0 && erts_system_profile_flags.runnable_procs) {
/* 'state' is before our change... */
@@ -6625,19 +6833,6 @@ resume_process(Process *p, ErtsProcLocks locks)
add2runq(enqueue, enq_prio, p, state, NULL);
}
-int
-erts_get_max_no_executing_schedulers(void)
-{
-#ifdef ERTS_SMP
- if (erts_smp_atomic32_read_nob(&schdlr_sspnd.changing))
- return (int) erts_no_schedulers;
- ERTS_THR_MEMORY_BARRIER;
- return (int) erts_smp_atomic32_read_nob(&schdlr_sspnd.active);
-#else
- return 1;
-#endif
-}
-
#ifdef ERTS_SMP
static void
@@ -6740,42 +6935,82 @@ sched_set_suspended_sleeptype(ErtsSchedulerSleepInfo *ssi)
}
}
-#ifdef ERTS_DIRTY_SCHEDULERS
+static void
+init_scheduler_suspend(void)
+{
+ erts_smp_mtx_init(&schdlr_sspnd.mtx, "schdlr_sspnd");
+ schdlr_sspnd.online = ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0);
+ schdlr_sspnd.curr_online = ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0);
+ schdlr_sspnd.active = ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0);
+ erts_smp_atomic32_init_nob(&schdlr_sspnd.changing, 0);
+ schdlr_sspnd.chngq = NULL;
+ schdlr_sspnd.changer = am_false;
+ schdlr_sspnd.nmsb.ongoing = 0;
+ schdlr_sspnd.nmsb.blckrs = NULL;
+ schdlr_sspnd.nmsb.chngq = NULL;
+ schdlr_sspnd.msb.ongoing = 0;
+ schdlr_sspnd.msb.blckrs = NULL;
+ schdlr_sspnd.msb.chngq = NULL;
+}
+
+typedef struct {
+ struct {
+ Eterm chngr;
+ Eterm nxt;
+ } onln;
+ struct {
+ ErtsProcList *chngrs;
+ } msb;
+} ErtsSchdlrSspndResume;
+
+static void
+schdlr_sspnd_resume_proc(Eterm pid)
+{
+ Process *p = erts_pid2proc(NULL, 0, pid, ERTS_PROC_LOCK_STATUS);
+ if (p) {
+ resume_process(p, ERTS_PROC_LOCK_STATUS);
+ erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+ }
+}
+
+static ERTS_INLINE void
+schdlr_sspnd_resume_procs(ErtsSchedType sched_type,
+ ErtsSchdlrSspndResume *resume)
+{
+ if (is_internal_pid(resume->onln.chngr)) {
+ schdlr_sspnd_resume_proc(resume->onln.chngr);
+ resume->onln.chngr = NIL;
+ }
+ if (is_internal_pid(resume->onln.nxt)) {
+ schdlr_sspnd_resume_proc(resume->onln.nxt);
+ resume->onln.nxt = NIL;
+ }
+ while (resume->msb.chngrs) {
+ ErtsProcList *plp = resume->msb.chngrs;
+ resume->msb.chngrs = plp->next;
+ schdlr_sspnd_resume_proc(plp->pid);
+ proclist_destroy(plp);
+ }
+}
static void
suspend_scheduler(ErtsSchedulerData *esdp)
{
erts_aint32_t flgs;
erts_aint32_t changing;
-#ifdef ERTS_DIRTY_SCHEDULERS
- long no = (long) (ERTS_SCHEDULER_IS_DIRTY(esdp)
- ? ERTS_DIRTY_SCHEDULER_NO(esdp)
- : esdp->no);
-#else
- long no = (long) esdp->no;
-#endif
+ long no;
ErtsSchedulerSleepInfo *ssi = esdp->ssi;
- long active_schedulers;
int curr_online = 1;
- int wake = 0;
+ ErtsSchdlrSspndResume resume = {{NIL, NIL}, {NULL}};
erts_aint32_t aux_work;
int thr_prgr_active = 1;
ErtsStuckBoundProcesses sbp = {NULL, NULL};
- int* ss_onlinep;
- int* ss_curr_onlinep;
- int* ss_wait_curr_onlinep;
- long* ss_wait_activep;
- long ss_wait_active_target;
- erts_smp_atomic32_t* ss_changingp;
- erts_smp_atomic32_t* ss_activep;
+ ErtsSchedType sched_type;
+ erts_aint32_t online_flag;
/*
* Schedulers may be suspended in two different ways:
* - A scheduler may be suspended since it is not online.
- * All schedulers with scheduler ids greater than
- * schdlr_sspnd.online are suspended; same for dirty
- * schedulers and schdlr_sspnd.dirty_cpu_online and
- * schdlr_sspnd.dirty_io_online.
* - Multi scheduling is blocked. All schedulers except the
* scheduler with scheduler id 1 are suspended, and all
* dirty CPU and dirty I/O schedulers are suspended.
@@ -6783,27 +7018,43 @@ suspend_scheduler(ErtsSchedulerData *esdp)
* Regardless of why a scheduler is suspended, it ends up here.
*/
- ASSERT(ERTS_SCHEDULER_IS_DIRTY(esdp) || no != 1);
-
#ifdef ERTS_DIRTY_SCHEDULERS
if (ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+ no = ERTS_DIRTY_SCHEDULER_NO(esdp);
+ if (ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(esdp->run_queue)) {
+ online_flag = ERTS_SCHDLR_SSPND_CHNG_DCPU_ONLN;
+ sched_type = ERTS_SCHED_DIRTY_CPU;
+ }
+ else {
+ online_flag = 0;
+ sched_type = ERTS_SCHED_DIRTY_IO;
+ }
+ }
+ else
+#endif
+ {
+ online_flag = ERTS_SCHDLR_SSPND_CHNG_ONLN;
+ no = esdp->no;
+ sched_type = ERTS_SCHED_NORMAL;
+ }
+
+ ASSERT(sched_type != ERTS_SCHED_NORMAL || no != 1);
+
+ if (sched_type != ERTS_SCHED_NORMAL) {
if (erts_smp_mtx_trylock(&schdlr_sspnd.mtx) == EBUSY) {
erts_smp_runq_unlock(esdp->run_queue);
erts_smp_mtx_lock(&schdlr_sspnd.mtx);
erts_smp_runq_lock(esdp->run_queue);
}
- if (ongoing_multi_scheduling_block())
+ if (schdlr_sspnd.msb.ongoing)
evacuate_run_queue(esdp->run_queue, &sbp);
- } else
-#endif
+ erts_smp_runq_unlock(esdp->run_queue);
+ }
+ else {
evacuate_run_queue(esdp->run_queue, &sbp);
- erts_smp_runq_unlock(esdp->run_queue);
+ erts_smp_runq_unlock(esdp->run_queue);
-#ifdef ERTS_DIRTY_SCHEDULERS
- if (!ERTS_SCHEDULER_IS_DIRTY(esdp))
-#endif
- {
erts_sched_check_cpu_bind_prep_suspend(esdp);
if (erts_system_profile_flags.scheduler)
@@ -6817,318 +7068,115 @@ suspend_scheduler(ErtsSchedulerData *esdp)
flgs = sched_prep_spin_suspended(ssi, ERTS_SSI_FLG_SUSPENDED);
if (flgs & ERTS_SSI_FLG_SUSPENDED) {
-#ifdef ERTS_DIRTY_SCHEDULERS
- if (ERTS_SCHEDULER_IS_DIRTY(esdp)) {
- if (ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(esdp->run_queue)) {
- active_schedulers = erts_smp_atomic32_dec_read_nob(&schdlr_sspnd.dirty_cpu_active);
- ASSERT(active_schedulers >= 0);
- changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_changing);
- ss_onlinep = &schdlr_sspnd.dirty_cpu_online;
- ss_curr_onlinep = &schdlr_sspnd.dirty_cpu_curr_online;
- ss_wait_curr_onlinep = &schdlr_sspnd.dirty_cpu_wait_curr_online;
- ss_changingp = &schdlr_sspnd.dirty_cpu_changing;
- ss_wait_activep = &schdlr_sspnd.msb.dirty_cpu_wait_active;
- ss_activep = &schdlr_sspnd.dirty_cpu_active;
- } else {
- active_schedulers = erts_smp_atomic32_dec_read_nob(&schdlr_sspnd.dirty_io_active);
- ASSERT(active_schedulers >= 0);
- changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_io_changing);
- ss_onlinep = &schdlr_sspnd.dirty_io_online;
- ss_curr_onlinep = &schdlr_sspnd.dirty_io_curr_online;
- ss_wait_curr_onlinep = &schdlr_sspnd.dirty_io_wait_curr_online;
- ss_changingp = &schdlr_sspnd.dirty_io_changing;
- ss_wait_activep = &schdlr_sspnd.msb.dirty_io_wait_active;
- ss_activep = &schdlr_sspnd.dirty_io_active;
- }
- ss_wait_active_target = 0;
- }
- else
-#endif
- {
- active_schedulers = erts_smp_atomic32_dec_read_nob(&schdlr_sspnd.active);
- ASSERT(active_schedulers >= 1);
- changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
- ss_onlinep = &schdlr_sspnd.online;
- ss_curr_onlinep = &schdlr_sspnd.curr_online;
- ss_wait_curr_onlinep = &schdlr_sspnd.wait_curr_online;
- ss_changingp = &schdlr_sspnd.changing;
- ss_wait_activep = &schdlr_sspnd.msb.wait_active;
- ss_activep = &schdlr_sspnd.active;
- ss_wait_active_target = 1;
- }
- if (changing & ERTS_SCHDLR_SSPND_CHNG_MSB) {
- if (active_schedulers == *ss_wait_activep)
- wake = 1;
- if (active_schedulers == ss_wait_active_target) {
- changing = erts_smp_atomic32_read_band_nob(ss_changingp,
- ~ERTS_SCHDLR_SSPND_CHNG_MSB);
- changing &= ~ERTS_SCHDLR_SSPND_CHNG_MSB;
- }
- }
-
- while (1) {
- if (changing & ERTS_SCHDLR_SSPND_CHNG_ONLN) {
- int changed = 0;
- if (no > *ss_onlinep && curr_online) {
- (*ss_curr_onlinep)--;
- curr_online = 0;
- changed = 1;
- }
- else if (no <= *ss_onlinep && !curr_online) {
- (*ss_curr_onlinep)++;
- curr_online = 1;
- changed = 1;
- }
- if (changed
- && *ss_curr_onlinep == *ss_wait_curr_onlinep)
- wake = 1;
- if (*ss_onlinep == *ss_curr_onlinep) {
- changing = erts_smp_atomic32_read_band_nob(ss_changingp,
- ~ERTS_SCHDLR_SSPND_CHNG_ONLN);
- changing &= ~ERTS_SCHDLR_SSPND_CHNG_ONLN;
- }
- }
+ schdlr_sspnd_dec_nscheds(&schdlr_sspnd.active, sched_type);
- if (wake) {
- erts_smp_cnd_signal(&schdlr_sspnd.cnd);
- wake = 0;
- }
-
- if (curr_online && !ongoing_multi_scheduling_block()) {
- flgs = erts_smp_atomic32_read_acqb(&ssi->flags);
- if (!(flgs & ERTS_SSI_FLG_SUSPENDED))
- break;
- }
- erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
+ ASSERT(schdlr_sspnd_get_nscheds(&schdlr_sspnd.active,
+ ERTS_SCHED_NORMAL) >= 1);
- while (1) {
- erts_aint32_t qmask;
- erts_aint32_t flgs;
+ changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
- qmask = (ERTS_RUNQ_FLGS_GET(esdp->run_queue)
- & ERTS_RUNQ_FLGS_QMASK);
- aux_work = erts_atomic32_read_acqb(&ssi->aux_work);
- if (aux_work|qmask) {
- if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
- if (!thr_prgr_active) {
- erts_thr_progress_active(esdp, thr_prgr_active = 1);
- sched_wall_time_change(esdp, 1);
- }
- if (aux_work)
- aux_work = handle_aux_work(&esdp->aux_work_data,
- aux_work,
- 1);
+ while (1) {
- if (aux_work && erts_thr_progress_update(esdp))
- erts_thr_progress_leader_update(esdp);
+ if (changing & (ERTS_SCHDLR_SSPND_CHNG_NMSB
+ | ERTS_SCHDLR_SSPND_CHNG_MSB)) {
+ int i = 0;
+ ErtsMultiSchedulingBlock *msb[3] = {0};
+ if (changing & ERTS_SCHDLR_SSPND_CHNG_NMSB)
+ msb[i++] = &schdlr_sspnd.nmsb;
+ if (changing & ERTS_SCHDLR_SSPND_CHNG_MSB)
+ msb[i++] = &schdlr_sspnd.msb;
+
+ for (i = 0; msb[i]; i++) {
+ erts_aint32_t clr_flg = 0;
+
+ if (msb[i] == &schdlr_sspnd.nmsb
+ && schdlr_sspnd_get_nscheds(&schdlr_sspnd.active,
+ ERTS_SCHED_NORMAL) == 1) {
+ clr_flg = ERTS_SCHDLR_SSPND_CHNG_NMSB;
}
- if (qmask) {
-#ifdef ERTS_DIRTY_SCHEDULERS
- if (ERTS_SCHEDULER_IS_DIRTY(esdp)) {
- erts_smp_mtx_lock(&schdlr_sspnd.mtx);
- erts_smp_runq_lock(esdp->run_queue);
- if (ongoing_multi_scheduling_block())
- evacuate_run_queue(esdp->run_queue, &sbp);
- erts_smp_runq_unlock(esdp->run_queue);
- erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
- } else
-#endif
- {
- erts_smp_runq_lock(esdp->run_queue);
- evacuate_run_queue(esdp->run_queue, &sbp);
- erts_smp_runq_unlock(esdp->run_queue);
- }
+ else if (schdlr_sspnd.active
+ == ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0)) {
+ clr_flg = ERTS_SCHDLR_SSPND_CHNG_MSB;
}
- }
- if (!aux_work) {
-#ifdef ERTS_DIRTY_SCHEDULERS
- if (!ERTS_SCHEDULER_IS_DIRTY(esdp))
-#endif
- {
- if (thr_prgr_active) {
- erts_thr_progress_active(esdp, thr_prgr_active = 0);
- sched_wall_time_change(esdp, 0);
+ if (clr_flg) {
+ ErtsProcList *plp, *end_plp;
+ changing = erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing,
+ ~clr_flg);
+ changing &= ~clr_flg;
+ (void) erts_proclist_fetch(&msb[i]->chngq, &end_plp);
+ /* resume processes that initiated the multi scheduling block... */
+ plp = msb[i]->chngq;
+ while (plp) {
+ erts_proclist_store_last(&msb[i]->blckrs,
+ proclist_copy(plp));
+ plp = plp->next;
}
- erts_thr_progress_prepare_wait(esdp);
+ if (end_plp)
+ end_plp->next = resume.msb.chngrs;
+ resume.msb.chngrs = msb[i]->chngq;
+ msb[i]->chngq = NULL;
}
- flgs = sched_spin_suspended(ssi,
- ERTS_SCHED_SUSPEND_SLEEP_SPINCOUNT);
- if (flgs == (ERTS_SSI_FLG_SLEEPING
- | ERTS_SSI_FLG_WAITING
- | ERTS_SSI_FLG_SUSPENDED)) {
- flgs = sched_set_suspended_sleeptype(ssi);
- if (flgs == (ERTS_SSI_FLG_SLEEPING
- | ERTS_SSI_FLG_TSE_SLEEPING
- | ERTS_SSI_FLG_WAITING
- | ERTS_SSI_FLG_SUSPENDED)) {
- int res;
-
- do {
- res = erts_tse_twait(ssi->event, -1);
- } while (res == EINTR);
- }
- }
-#ifdef ERTS_DIRTY_SCHEDULERS
- if (!ERTS_SCHEDULER_IS_DIRTY(esdp))
-#endif
- erts_thr_progress_finalize_wait(esdp);
}
-
- flgs = sched_prep_spin_suspended(ssi, (ERTS_SSI_FLG_WAITING
- | ERTS_SSI_FLG_SUSPENDED));
- if (!(flgs & ERTS_SSI_FLG_SUSPENDED))
- break;
- changing = erts_smp_atomic32_read_nob(ss_changingp);
- if (changing & ~ERTS_SCHDLR_SSPND_CHNG_WAITER)
- break;
- }
-
- erts_smp_mtx_lock(&schdlr_sspnd.mtx);
- changing = erts_smp_atomic32_read_nob(ss_changingp);
- }
-
- active_schedulers = erts_smp_atomic32_inc_read_nob(ss_activep);
- changing = erts_smp_atomic32_read_nob(ss_changingp);
- if ((changing & ERTS_SCHDLR_SSPND_CHNG_MSB)
- && *ss_onlinep == active_schedulers) {
- erts_smp_atomic32_read_band_nob(ss_changingp,
- ~ERTS_SCHDLR_SSPND_CHNG_MSB);
- }
-
-#ifdef ERTS_DIRTY_SCHEDULERS
- if (!ERTS_SCHEDULER_IS_DIRTY(esdp))
-#endif
- ASSERT(no <= *ss_onlinep);
- ASSERT(!ongoing_multi_scheduling_block());
-
- }
-
- erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
-
- ASSERT(curr_online);
-
-#ifdef ERTS_DIRTY_SCHEDULERS
- if (!ERTS_SCHEDULER_IS_DIRTY(esdp))
-#endif
- {
- if (erts_system_profile_flags.scheduler)
- profile_scheduler(make_small(esdp->no), am_active);
-
- if (!thr_prgr_active) {
- erts_thr_progress_active(esdp, thr_prgr_active = 1);
- sched_wall_time_change(esdp, 1);
- }
- }
-
- if (!ERTS_SCHEDULER_IS_DIRTY(esdp))
- (void) erts_get_monotonic_time(esdp);
- erts_smp_runq_lock(esdp->run_queue);
- non_empty_runq(esdp->run_queue);
-
-#ifdef ERTS_DIRTY_SCHEDULERS
- if (!ERTS_SCHEDULER_IS_DIRTY(esdp))
-#endif
- {
- schedule_bound_processes(esdp->run_queue, &sbp);
-
- erts_sched_check_cpu_bind_post_suspend(esdp);
- }
-}
-
-#else /* !ERTS_DIRTY_SCHEDULERS */
-
-static void
-suspend_scheduler(ErtsSchedulerData *esdp)
-{
- erts_aint32_t flgs;
- erts_aint32_t changing;
- long no = (long) esdp->no;
- ErtsSchedulerSleepInfo *ssi = esdp->ssi;
- long active_schedulers;
- int curr_online = 1;
- int wake = 0;
- erts_aint32_t aux_work;
- int thr_prgr_active = 1;
- ErtsStuckBoundProcesses sbp = {NULL, NULL};
-
- /*
- * Schedulers may be suspended in two different ways:
- * - A scheduler may be suspended since it is not online.
- * All schedulers with scheduler ids greater than
- * schdlr_sspnd.online are suspended.
- * - Multi scheduling is blocked. All schedulers except the
- * scheduler with scheduler id 1 are suspended.
- *
- * Regardless of why a scheduler is suspended, it ends up here.
- */
-
- ASSERT(no != 1);
-
- evacuate_run_queue(esdp->run_queue, &sbp);
-
- erts_smp_runq_unlock(esdp->run_queue);
-
- erts_sched_check_cpu_bind_prep_suspend(esdp);
-
- if (erts_system_profile_flags.scheduler)
- profile_scheduler(make_small(esdp->no), am_inactive);
-
- sched_wall_time_change(esdp, 0);
-
- erts_smp_mtx_lock(&schdlr_sspnd.mtx);
-
- flgs = sched_prep_spin_suspended(ssi, ERTS_SSI_FLG_SUSPENDED);
- if (flgs & ERTS_SSI_FLG_SUSPENDED) {
-
- active_schedulers = erts_smp_atomic32_dec_read_nob(&schdlr_sspnd.active);
- ASSERT(active_schedulers >= 1);
- changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
- if (changing & ERTS_SCHDLR_SSPND_CHNG_MSB) {
- if (active_schedulers == schdlr_sspnd.msb.wait_active)
- wake = 1;
- if (active_schedulers == 1) {
- changing = erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing,
- ~ERTS_SCHDLR_SSPND_CHNG_MSB);
- changing &= ~ERTS_SCHDLR_SSPND_CHNG_MSB;
}
- }
- while (1) {
- if (changing & ERTS_SCHDLR_SSPND_CHNG_ONLN) {
+ if (changing & online_flag) {
int changed = 0;
- if (no > schdlr_sspnd.online && curr_online) {
- schdlr_sspnd.curr_online--;
+ Uint32 st_online;
+
+ st_online = schdlr_sspnd_get_nscheds(&schdlr_sspnd.online,
+ sched_type);
+ if (no > st_online && curr_online) {
+ schdlr_sspnd_dec_nscheds(&schdlr_sspnd.curr_online,
+ sched_type);
curr_online = 0;
changed = 1;
}
- else if (no <= schdlr_sspnd.online && !curr_online) {
- schdlr_sspnd.curr_online++;
+ else if (no <= st_online && !curr_online) {
+ schdlr_sspnd_inc_nscheds(&schdlr_sspnd.curr_online,
+ sched_type);
curr_online = 1;
changed = 1;
}
if (changed
- && schdlr_sspnd.curr_online == schdlr_sspnd.wait_curr_online)
- wake = 1;
- if (schdlr_sspnd.online == schdlr_sspnd.curr_online) {
+ && (schdlr_sspnd_get_nscheds(&schdlr_sspnd.online,
+ sched_type)
+ == schdlr_sspnd_get_nscheds(&schdlr_sspnd.curr_online,
+ sched_type))) {
+ ErtsProcList *plp;
changing = erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing,
- ~ERTS_SCHDLR_SSPND_CHNG_ONLN);
- changing &= ~ERTS_SCHDLR_SSPND_CHNG_ONLN;
+ ~online_flag);
+ changing &= ~online_flag;
+ if (sched_type == ERTS_SCHED_NORMAL) {
+ ASSERT(is_internal_pid(schdlr_sspnd.changer)
+ || schdlr_sspnd.changer == am_init);
+ /* resume process that initiated this change... */
+ resume.onln.chngr = schdlr_sspnd.changer;
+ plp = erts_proclist_peek_first(schdlr_sspnd.chngq);
+ if (!plp)
+ schdlr_sspnd.changer = am_false;
+ else {
+ schdlr_sspnd.changer = am_true; /* change right in transit */
+ /* resume process that is queued for next change... */
+ resume.onln.nxt = plp->pid;
+ ASSERT(is_internal_pid(resume.onln.nxt));
+ }
+ }
}
}
- if (wake) {
- erts_smp_cnd_signal(&schdlr_sspnd.cnd);
- wake = 0;
- }
-
- if (curr_online && !ongoing_multi_scheduling_block()) {
+ if (curr_online
+ && (sched_type == ERTS_SCHED_NORMAL
+ ? !(schdlr_sspnd.msb.ongoing|schdlr_sspnd.nmsb.ongoing)
+ : !schdlr_sspnd.msb.ongoing)) {
flgs = erts_smp_atomic32_read_acqb(&ssi->flags);
if (!(flgs & ERTS_SSI_FLG_SUSPENDED))
break;
}
erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
+ schdlr_sspnd_resume_procs(sched_type, &resume);
+
while (1) {
ErtsMonotonicTime current_time;
erts_aint32_t qmask;
@@ -7136,9 +7184,23 @@ suspend_scheduler(ErtsSchedulerData *esdp)
qmask = (ERTS_RUNQ_FLGS_GET(esdp->run_queue)
& ERTS_RUNQ_FLGS_QMASK);
- aux_work = erts_atomic32_read_acqb(&ssi->aux_work);
- if (aux_work|qmask) {
- if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+
+ if (sched_type != ERTS_SCHED_NORMAL) {
+ if (qmask) {
+ erts_smp_mtx_lock(&schdlr_sspnd.mtx);
+ erts_smp_runq_lock(esdp->run_queue);
+ if (schdlr_sspnd.msb.ongoing)
+ evacuate_run_queue(esdp->run_queue, &sbp);
+ erts_smp_runq_unlock(esdp->run_queue);
+ erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
+ }
+ aux_work = 0;
+ }
+ else {
+
+ aux_work = erts_atomic32_read_acqb(&ssi->aux_work);
+
+ if (aux_work|qmask) {
if (!thr_prgr_active) {
erts_thr_progress_active(esdp, thr_prgr_active = 1);
sched_wall_time_change(esdp, 1);
@@ -7147,45 +7209,53 @@ suspend_scheduler(ErtsSchedulerData *esdp)
aux_work = handle_aux_work(&esdp->aux_work_data,
aux_work,
1);
+
if (aux_work && erts_thr_progress_update(esdp))
erts_thr_progress_leader_update(esdp);
+ if (qmask) {
+ erts_smp_runq_lock(esdp->run_queue);
+ evacuate_run_queue(esdp->run_queue, &sbp);
+ erts_smp_runq_unlock(esdp->run_queue);
+ }
}
- if (qmask) {
- erts_smp_runq_lock(esdp->run_queue);
- evacuate_run_queue(esdp->run_queue, &sbp);
- erts_smp_runq_unlock(esdp->run_queue);
- }
+
}
if (aux_work) {
- if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
- current_time = erts_get_monotonic_time(esdp);
- if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) {
- if (!thr_prgr_active) {
- erts_thr_progress_active(esdp, thr_prgr_active = 1);
- sched_wall_time_change(esdp, 1);
- }
- erts_bump_timers(esdp->timer_wheel, current_time);
+ ASSERT(sched_type == ERTS_SCHED_NORMAL);
+ current_time = erts_get_monotonic_time(esdp);
+ if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) {
+ if (!thr_prgr_active) {
+ erts_thr_progress_active(esdp, thr_prgr_active = 1);
+ sched_wall_time_change(esdp, 1);
}
+ erts_bump_timers(esdp->timer_wheel, current_time);
}
}
else {
ErtsMonotonicTime timeout_time;
- int do_timeout = 0;
- if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+ int do_timeout;
+
+ if (sched_type == ERTS_SCHED_NORMAL) {
timeout_time = erts_check_next_timeout_time(esdp);
current_time = erts_get_monotonic_time(esdp);
do_timeout = (current_time >= timeout_time);
- } else
+ }
+ else {
timeout_time = ERTS_MONOTONIC_TIME_MAX;
+ current_time = 0;
+ do_timeout = 0;
+ }
+
if (do_timeout) {
+ ASSERT(sched_type == ERTS_SCHED_NORMAL);
if (!thr_prgr_active) {
erts_thr_progress_active(esdp, thr_prgr_active = 1);
sched_wall_time_change(esdp, 1);
}
}
else {
- if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+ if (sched_type == ERTS_SCHED_NORMAL) {
if (thr_prgr_active) {
erts_thr_progress_active(esdp, thr_prgr_active = 0);
sched_wall_time_change(esdp, 0);
@@ -7204,30 +7274,39 @@ suspend_scheduler(ErtsSchedulerData *esdp)
| ERTS_SSI_FLG_SUSPENDED)) {
int res;
- current_time = ERTS_SCHEDULER_IS_DIRTY(esdp) ? 0 :
- erts_get_monotonic_time(esdp);
+ if (sched_type == ERTS_SCHED_NORMAL)
+ current_time = erts_get_monotonic_time(esdp);
+ else
+ current_time = 0;
+
do {
Sint64 timeout;
if (current_time >= timeout_time)
break;
- if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+ if (sched_type != ERTS_SCHED_NORMAL)
+ timeout = -1;
+ else
timeout = ERTS_MONOTONIC_TO_NSEC(timeout_time
- current_time
- 1) + 1;
- } else
- timeout = -1;
res = erts_tse_twait(ssi->event, timeout);
- current_time = ERTS_SCHEDULER_IS_DIRTY(esdp) ? 0 :
- erts_get_monotonic_time(esdp);
+
+ if (sched_type == ERTS_SCHED_NORMAL)
+ current_time = erts_get_monotonic_time(esdp);
+ else
+ current_time = 0;
+
} while (res == EINTR);
}
}
- if (!ERTS_SCHEDULER_IS_DIRTY(esdp))
+ if (sched_type == ERTS_SCHED_NORMAL)
erts_thr_progress_finalize_wait(esdp);
}
- if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && current_time >= timeout_time)
+ if (current_time >= timeout_time) {
+ ASSERT(sched_type == ERTS_SCHED_NORMAL);
erts_bump_timers(esdp->timer_wheel, current_time);
+ }
}
flgs = sched_prep_spin_suspended(ssi, (ERTS_SSI_FLG_WAITING
@@ -7235,7 +7314,7 @@ suspend_scheduler(ErtsSchedulerData *esdp)
if (!(flgs & ERTS_SSI_FLG_SUSPENDED))
break;
changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
- if (changing & ~ERTS_SCHDLR_SSPND_CHNG_WAITER)
+ if (changing)
break;
}
@@ -7243,612 +7322,494 @@ suspend_scheduler(ErtsSchedulerData *esdp)
changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
}
- active_schedulers = erts_smp_atomic32_inc_read_nob(&schdlr_sspnd.active);
+ schdlr_sspnd_inc_nscheds(&schdlr_sspnd.active, sched_type);
changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
if ((changing & ERTS_SCHDLR_SSPND_CHNG_MSB)
- && schdlr_sspnd.online == active_schedulers) {
+ && schdlr_sspnd.online == schdlr_sspnd.active) {
erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing,
~ERTS_SCHDLR_SSPND_CHNG_MSB);
}
- ASSERT(no <= schdlr_sspnd.online);
- ASSERT(!ongoing_multi_scheduling_block());
-
+ ASSERT(no <= schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, sched_type));
+ ASSERT((sched_type == ERTS_SCHED_NORMAL
+ ? !(schdlr_sspnd.msb.ongoing|schdlr_sspnd.nmsb.ongoing)
+ : !schdlr_sspnd.msb.ongoing));
}
erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
+ ASSERT(!resume.msb.chngrs);
+ schdlr_sspnd_resume_procs(sched_type, &resume);
+
ASSERT(curr_online);
- if (erts_system_profile_flags.scheduler)
- profile_scheduler(make_small(esdp->no), am_active);
+ if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+ if (erts_system_profile_flags.scheduler)
+ profile_scheduler(make_small(esdp->no), am_active);
- if (!thr_prgr_active) {
- erts_thr_progress_active(esdp, thr_prgr_active = 1);
- sched_wall_time_change(esdp, 1);
+ if (!thr_prgr_active) {
+ erts_thr_progress_active(esdp, thr_prgr_active = 1);
+ sched_wall_time_change(esdp, 1);
+ }
}
+ if (sched_type == ERTS_SCHED_NORMAL)
+ (void) erts_get_monotonic_time(esdp);
erts_smp_runq_lock(esdp->run_queue);
non_empty_runq(esdp->run_queue);
- schedule_bound_processes(esdp->run_queue, &sbp);
+ if (sched_type == ERTS_SCHED_NORMAL) {
+ schedule_bound_processes(esdp->run_queue, &sbp);
- erts_sched_check_cpu_bind_post_suspend(esdp);
+ erts_sched_check_cpu_bind_post_suspend(esdp);
+ }
}
-#endif
-
-ErtsSchedSuspendResult
+void
erts_schedulers_state(Uint *total,
Uint *online,
Uint *active,
Uint *dirty_cpu,
Uint *dirty_cpu_online,
+ Uint *dirty_cpu_active,
Uint *dirty_io,
- int yield_allowed)
+ Uint *dirty_io_active)
{
- int res = ERTS_SCHDLR_SSPND_EINVAL;
- erts_aint32_t changing;
- erts_smp_mtx_lock(&schdlr_sspnd.mtx);
- changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
-#ifdef ERTS_DIRTY_SCHEDULERS
- changing |= (erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_changing)
- | erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_io_changing));
-#endif
- if (yield_allowed && (changing & ~ERTS_SCHDLR_SSPND_CHNG_WAITER))
- res = ERTS_SCHDLR_SSPND_YIELD_RESTART;
- else {
+ if (active || online || dirty_cpu_online
+ || dirty_cpu_active || dirty_io_active) {
+ erts_smp_mtx_lock(&schdlr_sspnd.mtx);
if (active)
- *active = schdlr_sspnd.online;
+ *active = schdlr_sspnd_get_nscheds(&schdlr_sspnd.active,
+ ERTS_SCHED_NORMAL);
if (online)
- *online = schdlr_sspnd.online;
- if (ongoing_multi_scheduling_block() && active)
- *active = 1;
-#ifdef ERTS_DIRTY_SCHEDULERS
+ *online = schdlr_sspnd_get_nscheds(&schdlr_sspnd.curr_online,
+ ERTS_SCHED_NORMAL);
+ if (dirty_cpu_active)
+ *dirty_cpu_active = schdlr_sspnd_get_nscheds(&schdlr_sspnd.active,
+ ERTS_SCHED_DIRTY_CPU);
if (dirty_cpu_online)
- *dirty_cpu_online = schdlr_sspnd.dirty_cpu_online;
-#endif
- res = ERTS_SCHDLR_SSPND_DONE;
+ *dirty_cpu_online = schdlr_sspnd_get_nscheds(&schdlr_sspnd.curr_online,
+ ERTS_SCHED_DIRTY_CPU);
+ if (dirty_io_active)
+ *dirty_io_active = schdlr_sspnd_get_nscheds(&schdlr_sspnd.active,
+ ERTS_SCHED_DIRTY_IO);
+ erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
}
- erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
+
if (total)
*total = erts_no_schedulers;
-#ifdef ERTS_DIRTY_SCHEDULERS
if (dirty_cpu)
*dirty_cpu = erts_no_dirty_cpu_schedulers;
if (dirty_io)
*dirty_io = erts_no_dirty_io_schedulers;
-#endif
- return res;
}
-#ifdef ERTS_DIRTY_SCHEDULERS
+static void
+abort_sched_onln_chng_waitq(Process *p)
+{
+ Eterm resume = NIL;
+
+ erts_smp_mtx_lock(&schdlr_sspnd.mtx);
+
+#ifdef DEBUG
+ {
+ int found_it = 0;
+ ErtsProcList *plp = erts_proclist_peek_first(schdlr_sspnd.chngq);
+ while (plp) {
+ if (erts_proclist_same(plp, p))
+ found_it++;
+ plp = erts_proclist_peek_next(schdlr_sspnd.chngq, plp);
+ }
+ ASSERT(found_it == !!(p->flags & F_SCHDLR_ONLN_WAITQ));
+ }
+#endif
+
+ if (p->flags & F_SCHDLR_ONLN_WAITQ) {
+ ErtsProcList *plp = NULL;
+
+ plp = erts_proclist_peek_first(schdlr_sspnd.chngq);
+ if (plp) {
+ if (erts_proclist_same(plp, p)
+ && schdlr_sspnd.changer == am_true) {
+ p->flags &= ~F_SCHDLR_ONLN_WAITQ;
+ /*
+ * Change right was in transit to us;
+ * transfer it to the next process by
+ * resuming it...
+ */
+ erts_proclist_remove(&schdlr_sspnd.chngq, plp);
+ proclist_destroy(plp);
+ plp = erts_proclist_peek_first(schdlr_sspnd.chngq);
+ if (plp)
+ resume = plp->pid;
+ else
+ schdlr_sspnd.changer = am_false;
+ }
+ else {
+ do {
+ if (erts_proclist_same(plp, p)) {
+ p->flags &= ~F_SCHDLR_ONLN_WAITQ;
+ erts_proclist_remove(&schdlr_sspnd.chngq, plp);
+ proclist_destroy(plp);
+ break;
+ }
+ plp = erts_proclist_peek_next(schdlr_sspnd.chngq, plp);
+ } while (plp);
+ }
+ }
+ }
+
+ erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
+
+ if (is_internal_pid(resume))
+ schdlr_sspnd_resume_proc(resume);
+}
ErtsSchedSuspendResult
erts_set_schedulers_online(Process *p,
ErtsProcLocks plocks,
Sint new_no,
- Sint *old_no
-#ifdef ERTS_DIRTY_SCHEDULERS
- , int dirty_only
-#endif
- )
+ Sint *old_no,
+ int dirty_only)
{
- ErtsSchedulerData *esdp;
- int ix, res = -1, no, have_unlocked_plocks, end_wait;
- erts_aint32_t changing = 0;
+ int resume_proc, ix, res = -1, no, have_unlocked_plocks;
+ erts_aint32_t changing = 0, change_flags;
+ int online, increase;
+ ErtsProcList *plp;
#ifdef ERTS_DIRTY_SCHEDULERS
- ErtsSchedulerSleepInfo* ssi;
- int dirty_no, change_dirty;
+ int dirty_no, change_dirty, dirty_online;
+#else
+ ASSERT(!dirty_only);
#endif
if (new_no < 1)
return ERTS_SCHDLR_SSPND_EINVAL;
-#ifdef ERTS_DIRTY_SCHEDULERS
else if (dirty_only && erts_no_dirty_cpu_schedulers < new_no)
return ERTS_SCHDLR_SSPND_EINVAL;
-#endif
else if (erts_no_schedulers < new_no)
return ERTS_SCHDLR_SSPND_EINVAL;
- esdp = ERTS_PROC_GET_SCHDATA(p);
- end_wait = 0;
+#ifdef ERTS_DIRTY_SCHEDULERS
+ if (dirty_only)
+ resume_proc = 0;
+ else
+#endif
+ {
+ resume_proc = 1;
+ /*
+ * If we suspend current process we need to suspend before
+ * requesting the change; otherwise, we got a resume/suspend
+ * race...
+ */
+ if (!(plocks & ERTS_PROC_LOCK_STATUS))
+ erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
+ suspend_process(p, p);
+ if (!(plocks & ERTS_PROC_LOCK_STATUS))
+ erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+ }
erts_smp_mtx_lock(&schdlr_sspnd.mtx);
+ change_flags = 0;
have_unlocked_plocks = 0;
no = (int) new_no;
#ifdef ERTS_DIRTY_SCHEDULERS
- ASSERT(schdlr_sspnd.dirty_cpu_online <= erts_no_dirty_cpu_schedulers);
+ if (!dirty_only)
+#endif
+ {
+ changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
+ if (changing & ERTS_SCHDLR_SSPND_CHNG_ONLN) {
+ enqueue_wait:
+ p->flags |= F_SCHDLR_ONLN_WAITQ;
+ plp = proclist_create(p);
+ erts_proclist_store_last(&schdlr_sspnd.chngq, plp);
+ resume_proc = 0;
+ res = ERTS_SCHDLR_SSPND_YIELD_RESTART;
+ goto done;
+ }
+ plp = erts_proclist_peek_first(schdlr_sspnd.chngq);
+ if (!plp) {
+ ASSERT(schdlr_sspnd.changer == am_false);
+ }
+ else {
+ ASSERT(schdlr_sspnd.changer == am_true);
+ if (!erts_proclist_same(plp, p))
+ goto enqueue_wait;
+ p->flags &= ~F_SCHDLR_ONLN_WAITQ;
+ erts_proclist_remove(&schdlr_sspnd.chngq, plp);
+ proclist_destroy(plp);
+ }
+ }
+
+ *old_no = online = schdlr_sspnd_get_nscheds(&schdlr_sspnd.online,
+ ERTS_SCHED_NORMAL);
+#ifndef ERTS_DIRTY_SCHEDULERS
+ if (no == online) {
+ res = ERTS_SCHDLR_SSPND_DONE;
+ goto done;
+ }
+#else /* ERTS_DIRTY_SCHEDULERS */
+ dirty_online = schdlr_sspnd_get_nscheds(&schdlr_sspnd.online,
+ ERTS_SCHED_DIRTY_CPU);
+ if (dirty_only)
+ *old_no = dirty_online;
+
+ ASSERT(dirty_online <= erts_no_dirty_cpu_schedulers);
+
if (dirty_only) {
- if (no > schdlr_sspnd.online) {
- erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
- return ERTS_SCHDLR_SSPND_EINVAL;
+ if (no > online) {
+ res = ERTS_SCHDLR_SSPND_EINVAL;
+ goto done;
}
dirty_no = no;
+ if (dirty_no == dirty_online) {
+ res = ERTS_SCHDLR_SSPND_DONE;
+ goto done;
+ }
+ change_dirty = 1;
} else {
/*
* Adjust the number of dirty CPU schedulers online relative to the
* adjustment made to the number of normal schedulers online.
*/
int total_pct = erts_no_dirty_cpu_schedulers*100/erts_no_schedulers;
- int onln_pct = no*total_pct/schdlr_sspnd.online;
- dirty_no = schdlr_sspnd.dirty_cpu_online*onln_pct/100;
+ int onln_pct = no*total_pct/online;
+ dirty_no = dirty_online*onln_pct/100;
if (dirty_no == 0)
dirty_no = 1;
ASSERT(dirty_no <= erts_no_dirty_cpu_schedulers);
- }
-#endif
- changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
-#ifdef ERTS_DIRTY_SCHEDULERS
- changing |= erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_changing);
-#endif
- if (changing) {
- res = ERTS_SCHDLR_SSPND_YIELD_RESTART;
- }
- else {
- int online = *old_no = schdlr_sspnd.online;
-#ifdef ERTS_DIRTY_SCHEDULERS
- int dirty_online = schdlr_sspnd.dirty_cpu_online;
- if (dirty_only) {
- *old_no = schdlr_sspnd.dirty_cpu_online;
- if (dirty_no == schdlr_sspnd.dirty_cpu_online) {
+ if (no != online)
+ change_dirty = (dirty_no != dirty_online);
+ else {
+ dirty_only = 1;
+ if (dirty_no == dirty_online) {
res = ERTS_SCHDLR_SSPND_DONE;
+ goto done;
}
change_dirty = 1;
- } else {
-#endif
- if (no == schdlr_sspnd.online) {
-#ifdef ERTS_DIRTY_SCHEDULERS
- dirty_only = 1;
- if (dirty_no == schdlr_sspnd.dirty_cpu_online)
-#endif
- res = ERTS_SCHDLR_SSPND_DONE;
-#ifdef ERTS_DIRTY_SCHEDULERS
- else
- change_dirty = 1;
-#endif
- }
-#ifdef ERTS_DIRTY_SCHEDULERS
- else
- change_dirty = (dirty_no != schdlr_sspnd.dirty_cpu_online);
}
-#endif
- if (res == -1)
- {
- int increase = (no > online);
-#ifdef ERTS_DIRTY_SCHEDULERS
- if (!dirty_only) {
-#endif
- ERTS_SCHDLR_SSPND_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN
- | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0);
- schdlr_sspnd.online = no;
-#ifdef ERTS_DIRTY_SCHEDULERS
- } else
- increase = (dirty_no > dirty_online);
- if (change_dirty) {
- ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN
- | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0);
- schdlr_sspnd.dirty_cpu_online = dirty_no;
- }
-#endif
- if (increase) {
- int ix;
-#ifdef ERTS_DIRTY_SCHEDULERS
- if (!dirty_only) {
-#endif
- schdlr_sspnd.wait_curr_online = no;
- if (ongoing_multi_scheduling_block()) {
- for (ix = online; ix < no; ix++)
- erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(ix));
- }
- else {
- if (plocks) {
- have_unlocked_plocks = 1;
- erts_smp_proc_unlock(p, plocks);
- }
- change_no_used_runqs(no);
+ }
+ if (change_dirty) {
+ change_flags |= ERTS_SCHDLR_SSPND_CHNG_DCPU_ONLN;
+ schdlr_sspnd_set_nscheds(&schdlr_sspnd.online,
+ ERTS_SCHED_DIRTY_CPU,
+ dirty_no);
+ }
+
+ if (dirty_only)
+ increase = (dirty_no > dirty_online);
+ else
+#endif /* ERTS_DIRTY_SCHEDULERS */
+ {
+ change_flags |= ERTS_SCHDLR_SSPND_CHNG_ONLN;
+ schdlr_sspnd_set_nscheds(&schdlr_sspnd.online,
+ ERTS_SCHED_NORMAL,
+ no);
+ increase = (no > online);
+ }
- for (ix = online; ix < no; ix++)
- resume_run_queue(ERTS_RUNQ_IX(ix));
+ erts_smp_atomic32_read_bor_nob(&schdlr_sspnd.changing, change_flags);
- for (ix = no; ix < erts_no_run_queues; ix++)
- suspend_run_queue(ERTS_RUNQ_IX(ix));
- }
+ res = ERTS_SCHDLR_SSPND_DONE;
+ if (increase) {
+ int ix;
#ifdef ERTS_DIRTY_SCHEDULERS
+ if (change_dirty) {
+ ErtsSchedulerSleepInfo* ssi;
+ if (schdlr_sspnd.msb.ongoing) {
+ for (ix = dirty_online; ix < dirty_no; ix++) {
+ ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix);
+ erts_sched_poke(ssi);
}
- if (change_dirty) {
- schdlr_sspnd.dirty_cpu_wait_curr_online = dirty_no;
- ASSERT(schdlr_sspnd.dirty_cpu_curr_online !=
- schdlr_sspnd.dirty_cpu_wait_curr_online);
- if (ongoing_multi_scheduling_block()) {
- for (ix = dirty_online; ix < dirty_no; ix++) {
- ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix);
- erts_sched_poke(ssi);
- }
- } else {
- for (ix = dirty_online; ix < dirty_no; ix++) {
- ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix);
- scheduler_ssi_resume_wake(ssi);
- erts_smp_atomic32_read_band_nob(&ssi->flags,
- ~ERTS_SSI_FLG_SUSPENDED);
- }
- wake_dirty_schedulers(ERTS_DIRTY_CPU_RUNQ, 0);
- }
+ } else {
+ for (ix = dirty_online; ix < dirty_no; ix++) {
+ ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix);
+ scheduler_ssi_resume_wake(ssi);
}
-#endif
- res = ERTS_SCHDLR_SSPND_DONE;
}
- else /* if (no < online) */ {
-#ifdef ERTS_DIRTY_SCHEDULERS
- if (change_dirty) {
- schdlr_sspnd.dirty_cpu_wait_curr_online = dirty_no;
- ASSERT(schdlr_sspnd.dirty_cpu_curr_online !=
- schdlr_sspnd.dirty_cpu_wait_curr_online);
- if (ongoing_multi_scheduling_block()) {
- for (ix = dirty_no; ix < dirty_online; ix++) {
- ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix);
- erts_sched_poke(ssi);
- }
- } else {
- for (ix = dirty_no; ix < dirty_online; ix++) {
- ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix);
- erts_smp_atomic32_read_bor_nob(&ssi->flags,
- ERTS_SSI_FLG_SUSPENDED);
- }
- wake_dirty_schedulers(ERTS_DIRTY_CPU_RUNQ, 0);
- }
- }
- if (dirty_only) {
- res = ERTS_SCHDLR_SSPND_DONE;
- }
- else
+ }
+ if (!dirty_only)
#endif
- {
- if (p->scheduler_data->no <= no) {
- res = ERTS_SCHDLR_SSPND_DONE;
- schdlr_sspnd.wait_curr_online = no;
- }
- else {
- /*
- * Yield! Current process needs to migrate
- * before bif returns.
- */
- res = ERTS_SCHDLR_SSPND_YIELD_DONE;
- schdlr_sspnd.wait_curr_online = no+1;
- }
-
- if (ongoing_multi_scheduling_block()) {
- for (ix = no; ix < online; ix++)
- erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(ix));
- }
- else {
- if (plocks) {
- have_unlocked_plocks = 1;
- erts_smp_proc_unlock(p, plocks);
- }
-
- change_no_used_runqs(no);
- for (ix = no; ix < erts_no_run_queues; ix++)
- suspend_run_queue(ERTS_RUNQ_IX(ix));
-
- for (ix = no; ix < online; ix++) {
- ErtsRunQueue *rq = ERTS_RUNQ_IX(ix);
- wake_scheduler(rq);
- }
- }
- }
+ {
+ if (schdlr_sspnd.msb.ongoing|schdlr_sspnd.nmsb.ongoing) {
+ for (ix = online; ix < no; ix++)
+ erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(ix));
}
-
-#ifdef ERTS_DIRTY_SCHEDULERS
- if (change_dirty) {
- while (schdlr_sspnd.dirty_cpu_curr_online != schdlr_sspnd.dirty_cpu_wait_curr_online)
- erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx);
- ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER);
- erts_smp_atomic32_read_band_nob(&schdlr_sspnd.dirty_cpu_changing,
- ~ERTS_SCHDLR_SSPND_CHNG_WAITER);
- }
- if (!dirty_only)
-#endif
- {
- if (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) {
- erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
- if (plocks && !have_unlocked_plocks) {
- have_unlocked_plocks = 1;
- erts_smp_proc_unlock(p, plocks);
- }
- erts_thr_progress_active(esdp, 0);
- erts_thr_progress_prepare_wait(esdp);
- end_wait = 1;
- erts_smp_mtx_lock(&schdlr_sspnd.mtx);
+ else {
+ if (plocks) {
+ have_unlocked_plocks = 1;
+ erts_smp_proc_unlock(p, plocks);
}
+ change_no_used_runqs(no);
- while (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online)
- erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx);
+ for (ix = online; ix < no; ix++)
+ resume_run_queue(ERTS_RUNQ_IX(ix));
- ASSERT(res != ERTS_SCHDLR_SSPND_DONE
- ? (ERTS_SCHDLR_SSPND_CHNG_WAITER
- & erts_smp_atomic32_read_nob(&schdlr_sspnd.changing))
- : (ERTS_SCHDLR_SSPND_CHNG_WAITER
- == erts_smp_atomic32_read_nob(&schdlr_sspnd.changing)));
- erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing,
- ~ERTS_SCHDLR_SSPND_CHNG_WAITER);
+ for (ix = no; ix < erts_no_run_queues; ix++)
+ suspend_run_queue(ERTS_RUNQ_IX(ix));
}
}
}
-
- erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
+ else /* if decrease */ {
#ifdef ERTS_DIRTY_SCHEDULERS
- ASSERT(schdlr_sspnd.dirty_cpu_online <= schdlr_sspnd.online);
- if (!dirty_only)
-#endif
- {
- if (end_wait) {
- erts_thr_progress_finalize_wait(esdp);
- erts_thr_progress_active(esdp, 1);
- }
- if (have_unlocked_plocks)
- erts_smp_proc_lock(p, plocks);
- }
-
- return res;
-}
-
-#else /* !ERTS_DIRTY_SCHEDULERS */
-
-ErtsSchedSuspendResult
-erts_set_schedulers_online(Process *p,
- ErtsProcLocks plocks,
- Sint new_no,
- Sint *old_no)
-{
- ErtsSchedulerData *esdp;
- int ix, res, no, have_unlocked_plocks, end_wait;
- erts_aint32_t changing;
-
- if (new_no < 1 || erts_no_schedulers < new_no)
- return ERTS_SCHDLR_SSPND_EINVAL;
-
- esdp = ERTS_PROC_GET_SCHDATA(p);
- end_wait = 0;
-
- erts_smp_mtx_lock(&schdlr_sspnd.mtx);
-
- have_unlocked_plocks = 0;
- no = (int) new_no;
-
- changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
- if (changing) {
- res = ERTS_SCHDLR_SSPND_YIELD_RESTART;
- }
- else {
- int online = *old_no = schdlr_sspnd.online;
- if (no == schdlr_sspnd.online) {
- res = ERTS_SCHDLR_SSPND_DONE;
- }
- else {
- ERTS_SCHDLR_SSPND_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN
- | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0);
- schdlr_sspnd.online = no;
- if (no > online) {
- int ix;
- schdlr_sspnd.wait_curr_online = no;
- if (ongoing_multi_scheduling_block()) {
- for (ix = online; ix < no; ix++)
- erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(ix));
+ if (change_dirty) {
+ ErtsSchedulerSleepInfo* ssi;
+ if (schdlr_sspnd.msb.ongoing) {
+ for (ix = dirty_no; ix < dirty_online; ix++) {
+ ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix);
+ erts_sched_poke(ssi);
}
- else {
- if (plocks) {
- have_unlocked_plocks = 1;
- erts_smp_proc_unlock(p, plocks);
- }
- change_no_used_runqs(no);
-
- for (ix = online; ix < no; ix++)
- resume_run_queue(ERTS_RUNQ_IX(ix));
-
- for (ix = no; ix < erts_no_run_queues; ix++)
- suspend_run_queue(ERTS_RUNQ_IX(ix));
+ } else {
+ for (ix = dirty_no; ix < dirty_online; ix++) {
+ ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix);
+ erts_smp_atomic32_read_bor_nob(&ssi->flags,
+ ERTS_SSI_FLG_SUSPENDED);
}
- res = ERTS_SCHDLR_SSPND_DONE;
+ wake_dirty_schedulers(ERTS_DIRTY_CPU_RUNQ, 0);
}
- else /* if (no < online) */ {
- if (p->scheduler_data->no <= no) {
- res = ERTS_SCHDLR_SSPND_DONE;
- schdlr_sspnd.wait_curr_online = no;
- }
- else {
- /*
- * Yield! Current process needs to migrate
- * before bif returns.
- */
- res = ERTS_SCHDLR_SSPND_YIELD_DONE;
- schdlr_sspnd.wait_curr_online = no+1;
- }
-
- if (ongoing_multi_scheduling_block()) {
- for (ix = no; ix < online; ix++)
- erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(ix));
- }
- else {
- if (plocks) {
- have_unlocked_plocks = 1;
- erts_smp_proc_unlock(p, plocks);
- }
-
- change_no_used_runqs(no);
- for (ix = no; ix < erts_no_run_queues; ix++)
- suspend_run_queue(ERTS_RUNQ_IX(ix));
-
- for (ix = no; ix < online; ix++) {
- ErtsRunQueue *rq = ERTS_RUNQ_IX(ix);
- wake_scheduler(rq);
- }
- }
+ }
+ if (!dirty_only)
+#endif
+ {
+ if (schdlr_sspnd.msb.ongoing|schdlr_sspnd.nmsb.ongoing) {
+ for (ix = no; ix < online; ix++)
+ erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(ix));
}
-
- if (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) {
- erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
- if (plocks && !have_unlocked_plocks) {
+ else {
+ if (plocks) {
have_unlocked_plocks = 1;
erts_smp_proc_unlock(p, plocks);
}
- erts_thr_progress_active(esdp, 0);
- erts_thr_progress_prepare_wait(esdp);
- end_wait = 1;
- erts_smp_mtx_lock(&schdlr_sspnd.mtx);
- }
-
- while (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online)
- erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx);
- ASSERT(res != ERTS_SCHDLR_SSPND_DONE
- ? (ERTS_SCHDLR_SSPND_CHNG_WAITER
- & erts_smp_atomic32_read_nob(&schdlr_sspnd.changing))
- : (ERTS_SCHDLR_SSPND_CHNG_WAITER
- == erts_smp_atomic32_read_nob(&schdlr_sspnd.changing)));
- erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing,
- ~ERTS_SCHDLR_SSPND_CHNG_WAITER);
+ change_no_used_runqs(no);
+ for (ix = no; ix < erts_no_run_queues; ix++)
+ suspend_run_queue(ERTS_RUNQ_IX(ix));
+ for (ix = no; ix < online; ix++) {
+ ErtsRunQueue *rq = ERTS_RUNQ_IX(ix);
+ wake_scheduler(rq);
+ }
+ }
}
}
- erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
- if (end_wait) {
- erts_thr_progress_finalize_wait(esdp);
- erts_thr_progress_active(esdp, 1);
+ if (change_flags & ERTS_SCHDLR_SSPND_CHNG_ONLN) {
+ /* Suspend and wait for requested change to complete... */
+ schdlr_sspnd.changer = p->common.id;
+ resume_proc = 0;
+ res = ERTS_SCHDLR_SSPND_YIELD_DONE;
}
+
+done:
+
+ ASSERT(schdlr_sspnd_get_nscheds(&schdlr_sspnd.online,
+ ERTS_SCHED_DIRTY_CPU)
+ <= schdlr_sspnd_get_nscheds(&schdlr_sspnd.online,
+ ERTS_SCHED_NORMAL));
+
+ erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
+
if (have_unlocked_plocks)
erts_smp_proc_lock(p, plocks);
+ if (resume_proc) {
+ if (!(plocks & ERTS_PROC_LOCK_STATUS))
+ erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
+ resume_process(p, plocks|ERTS_PROC_LOCK_STATUS);
+ if (!(plocks & ERTS_PROC_LOCK_STATUS))
+ erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+ }
+
return res;
}
-#endif
-
ErtsSchedSuspendResult
-erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all)
+erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal, int all)
{
- int ix, res, have_unlocked_plocks = 0, online;
- erts_aint32_t changing;
+ int resume_proc, ix, res, have_unlocked_plocks = 0;
ErtsProcList *plp;
#ifdef ERTS_DIRTY_SCHEDULERS
ErtsSchedulerSleepInfo* ssi;
#endif
+ ErtsMultiSchedulingBlock *msbp;
+ erts_aint32_t chng_flg;
+ int have_blckd_flg;
- erts_smp_mtx_lock(&schdlr_sspnd.mtx);
- changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
-#ifdef ERTS_DIRTY_SCHEDULERS
- changing |= (erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_changing)
- | erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_io_changing));
-#endif
- if (changing) {
- res = ERTS_SCHDLR_SSPND_YIELD_RESTART; /* Yield */
+ if (normal) {
+ chng_flg = ERTS_SCHDLR_SSPND_CHNG_NMSB;
+ have_blckd_flg = F_HAVE_BLCKD_NMSCHED;
+ msbp = &schdlr_sspnd.nmsb;
+ }
+ else {
+ chng_flg = ERTS_SCHDLR_SSPND_CHNG_MSB;
+ have_blckd_flg = F_HAVE_BLCKD_MSCHED;
+ msbp = &schdlr_sspnd.msb;
}
- else if (on) { /* ------ BLOCK ------ */
- if (schdlr_sspnd.msb.procs) {
+
+ /*
+ * If we suspend current process we need to suspend before
+ * requesting the change; otherwise, we got a resume/suspend
+ * race...
+ */
+ if (!on) {
+ /* We never suspend current process when unblocking... */
+ resume_proc = 0;
+ }
+ else {
+ resume_proc = 1;
+ if (!(plocks & ERTS_PROC_LOCK_STATUS))
+ erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
+ suspend_process(p, p);
+ if (!(plocks & ERTS_PROC_LOCK_STATUS))
+ erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+ }
+
+ erts_smp_mtx_lock(&schdlr_sspnd.mtx);
+ if (on) { /* ------ BLOCK ------ */
+ if (msbp->chngq) {
+ ASSERT(msbp->ongoing);
+ p->flags |= have_blckd_flg;
+ goto wait_until_msb;
+ }
+ else if (msbp->blckrs) {
+ ASSERT(msbp->ongoing);
plp = proclist_create(p);
- erts_proclist_store_last(&schdlr_sspnd.msb.procs, plp);
- p->flags |= F_HAVE_BLCKD_MSCHED;
- ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.active) == 1);
-#ifdef ERTS_DIRTY_SCHEDULERS
- ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_active) == 0);
- ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_io_active) == 0);
-#endif
+ erts_proclist_store_last(&msbp->blckrs, plp);
+ p->flags |= have_blckd_flg;
+ ASSERT(schdlr_sspnd.active == ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0));
ASSERT(p->scheduler_data->no == 1);
- res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED;
- } else {
- int online = schdlr_sspnd.online;
- p->flags |= F_HAVE_BLCKD_MSCHED;
+ if (schdlr_sspnd.msb.ongoing)
+ res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED;
+ else
+ res = ERTS_SCHDLR_SSPND_DONE_NMSCHED_BLOCKED;
+ }
+ else {
+ int online = (int) schdlr_sspnd_get_nscheds(&schdlr_sspnd.online,
+ ERTS_SCHED_NORMAL);
+ ASSERT(!msbp->ongoing);
+ p->flags |= have_blckd_flg;
if (plocks) {
have_unlocked_plocks = 1;
erts_smp_proc_unlock(p, plocks);
}
- ASSERT(!ongoing_multi_scheduling_block());
- schdlr_sspnd.msb.ongoing = 1;
- if (online == 1) {
- res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED;
- ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.active) == 1);
-#ifdef ERTS_DIRTY_SCHEDULERS
- ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_active) == 1);
- ASSERT(!(erts_smp_atomic32_read_nob(&ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(0)->flags)
- & ERTS_SSI_FLG_SUSPENDED));
- schdlr_sspnd.msb.dirty_cpu_wait_active = 0;
- ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_MSB
- | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0);
- ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(0);
- erts_smp_atomic32_read_bor_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED);
- wake_dirty_schedulers(ERTS_DIRTY_CPU_RUNQ, 0);
- while (erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_active)
- != schdlr_sspnd.msb.dirty_cpu_wait_active)
- erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx);
- ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER);
-
- schdlr_sspnd.msb.dirty_io_wait_active = 0;
- ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_MSB
- | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0);
- for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) {
- ssi = ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix);
- erts_smp_atomic32_read_bor_nob(&ssi->flags,
- ERTS_SSI_FLG_SUSPENDED);
- }
- wake_dirty_schedulers(ERTS_DIRTY_IO_RUNQ, 0);
- while (erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_io_active)
- != schdlr_sspnd.msb.dirty_io_wait_active)
- erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx);
- ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER);
-#endif
+ ASSERT(!msbp->ongoing);
+ msbp->ongoing = 1;
+ if (schdlr_sspnd.active == ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0)
+ || (normal && schdlr_sspnd_get_nscheds(&schdlr_sspnd.active,
+ ERTS_SCHED_NORMAL) == 1)) {
ASSERT(p->scheduler_data->no == 1);
+ plp = proclist_create(p);
+ erts_proclist_store_last(&msbp->blckrs, plp);
+ if (schdlr_sspnd.msb.ongoing)
+ res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED;
+ else
+ res = ERTS_SCHDLR_SSPND_DONE_NMSCHED_BLOCKED;
}
else {
- ERTS_SCHDLR_SSPND_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_MSB
- | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0);
- if (p->scheduler_data->no == 1) {
- res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED;
- schdlr_sspnd.msb.wait_active = 1;
- }
- else {
- /*
- * Yield! Current process needs to migrate
- * before bif returns.
- */
- res = ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED;
- schdlr_sspnd.msb.wait_active = 2;
- }
-
-#ifdef ERTS_DIRTY_SCHEDULERS
- schdlr_sspnd.msb.dirty_cpu_wait_active = 0;
- ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_MSB
- | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0);
- for (ix = 0; ix < erts_no_dirty_cpu_schedulers; ix++) {
- ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix);
- erts_smp_atomic32_read_bor_nob(&ssi->flags,
- ERTS_SSI_FLG_SUSPENDED);
- }
- wake_dirty_schedulers(ERTS_DIRTY_CPU_RUNQ, 0);
- while (erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_active)
- != schdlr_sspnd.msb.dirty_cpu_wait_active)
- erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx);
- ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER);
- ASSERT(schdlr_sspnd.dirty_cpu_curr_online == schdlr_sspnd.dirty_cpu_online);
-
- schdlr_sspnd.msb.dirty_io_wait_active = 0;
- ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_MSB
- | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0);
- for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) {
- ssi = ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix);
- erts_smp_atomic32_read_bor_nob(&ssi->flags,
- ERTS_SSI_FLG_SUSPENDED);
- }
- wake_dirty_schedulers(ERTS_DIRTY_IO_RUNQ, 0);
- while (erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_io_active)
- != schdlr_sspnd.msb.dirty_io_wait_active)
- erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx);
- ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER);
- ASSERT(schdlr_sspnd.dirty_io_curr_online == schdlr_sspnd.dirty_io_online);
-#endif
+ erts_smp_atomic32_read_bor_nob(&schdlr_sspnd.changing,
+ chng_flg);
change_no_used_runqs(1);
for (ix = 1; ix < erts_no_run_queues; ix++)
suspend_run_queue(ERTS_RUNQ_IX(ix));
@@ -7858,84 +7819,84 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all)
wake_scheduler(rq);
}
- if (erts_smp_atomic32_read_nob(&schdlr_sspnd.active)
- != schdlr_sspnd.msb.wait_active) {
- ErtsSchedulerData *esdp;
-
- erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
-
- if (plocks && !have_unlocked_plocks) {
- have_unlocked_plocks = 1;
- erts_smp_proc_unlock(p, plocks);
+#ifdef ERTS_DIRTY_SCHEDULERS
+ if (!normal) {
+ for (ix = 0; ix < erts_no_dirty_cpu_schedulers; ix++) {
+ ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix);
+ erts_smp_atomic32_read_bor_nob(&ssi->flags,
+ ERTS_SSI_FLG_SUSPENDED);
}
+ wake_dirty_schedulers(ERTS_DIRTY_CPU_RUNQ, 0);
- esdp = ERTS_PROC_GET_SCHDATA(p);
-
- erts_thr_progress_active(esdp, 0);
- erts_thr_progress_prepare_wait(esdp);
-
- erts_smp_mtx_lock(&schdlr_sspnd.mtx);
-
- while (erts_smp_atomic32_read_nob(&schdlr_sspnd.active)
- != schdlr_sspnd.msb.wait_active)
- erts_smp_cnd_wait(&schdlr_sspnd.cnd,
- &schdlr_sspnd.mtx);
-
- erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
-
- erts_thr_progress_active(esdp, 1);
- erts_thr_progress_finalize_wait(esdp);
+ for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) {
+ ssi = ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix);
+ erts_smp_atomic32_read_bor_nob(&ssi->flags,
+ ERTS_SSI_FLG_SUSPENDED);
+ }
+ wake_dirty_schedulers(ERTS_DIRTY_IO_RUNQ, 0);
+ }
+#endif
- erts_smp_mtx_lock(&schdlr_sspnd.mtx);
+ wait_until_msb:
- }
+ ASSERT(chng_flg & erts_smp_atomic32_read_nob(&schdlr_sspnd.changing));
- ASSERT(res != ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED
- ? (ERTS_SCHDLR_SSPND_CHNG_WAITER
- & erts_smp_atomic32_read_nob(&schdlr_sspnd.changing))
- : (ERTS_SCHDLR_SSPND_CHNG_WAITER
- == erts_smp_atomic32_read_nob(&schdlr_sspnd.changing)));
- erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing,
- ~ERTS_SCHDLR_SSPND_CHNG_WAITER);
+ plp = proclist_create(p);
+ erts_proclist_store_last(&msbp->chngq, plp);
+ resume_proc = 0;
+ if (schdlr_sspnd.msb.ongoing)
+ res = ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED;
+ else
+ res = ERTS_SCHDLR_SSPND_YIELD_DONE_NMSCHED_BLOCKED;
}
- plp = proclist_create(p);
- erts_proclist_store_last(&schdlr_sspnd.msb.procs, plp);
ASSERT(p->scheduler_data);
}
}
- else if (!ongoing_multi_scheduling_block()) {
- /* unblock not ongoing */
- ASSERT(!schdlr_sspnd.msb.procs);
- res = ERTS_SCHDLR_SSPND_DONE;
+ else if (!msbp->ongoing) {
+ ASSERT(!msbp->blckrs);
+ goto unblock_res;
}
else { /* ------ UNBLOCK ------ */
- if (p->flags & F_HAVE_BLCKD_MSCHED) {
- ErtsProcList *plp = erts_proclist_peek_first(schdlr_sspnd.msb.procs);
-
- while (plp) {
- ErtsProcList *tmp_plp = plp;
- plp = erts_proclist_peek_next(schdlr_sspnd.msb.procs, plp);
- if (erts_proclist_same(tmp_plp, p)) {
- erts_proclist_remove(&schdlr_sspnd.msb.procs, tmp_plp);
- proclist_destroy(tmp_plp);
- if (!all)
- break;
+ if (p->flags & have_blckd_flg) {
+ ErtsProcList *plps[2];
+ ErtsProcList *plp;
+ int limit = 0;
+
+ plps[limit++] = erts_proclist_peek_first(msbp->blckrs);
+ if (all)
+ plps[limit++] = erts_proclist_peek_first(msbp->chngq);
+
+ for (ix = 0; ix < limit; ix++) {
+ plp = plps[ix];
+ while (plp) {
+ ErtsProcList *tmp_plp = plp;
+ plp = erts_proclist_peek_next(msbp->blckrs, plp);
+ if (erts_proclist_same(tmp_plp, p)) {
+ erts_proclist_remove(&msbp->blckrs, tmp_plp);
+ proclist_destroy(tmp_plp);
+ if (!all)
+ break;
+ }
}
}
}
- if (schdlr_sspnd.msb.procs)
- res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED;
- else {
- ERTS_SCHDLR_SSPND_CHNG_SET(ERTS_SCHDLR_SSPND_CHNG_MSB, 0);
- p->flags &= ~F_HAVE_BLCKD_MSCHED;
- schdlr_sspnd.msb.ongoing = 0;
- if (schdlr_sspnd.online == 1) {
+ if (!msbp->blckrs && !msbp->chngq) {
+ int online = (int) schdlr_sspnd_get_nscheds(&schdlr_sspnd.online,
+ ERTS_SCHED_NORMAL);
+ erts_smp_atomic32_read_bor_nob(&schdlr_sspnd.changing,
+ chng_flg);
+ p->flags &= ~have_blckd_flg;
+ msbp->ongoing = 0;
+ if (online == 1) {
/* No normal schedulers to resume */
- ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.active) == 1);
- ERTS_SCHDLR_SSPND_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_MSB);
+ ASSERT(schdlr_sspnd_get_nscheds(&schdlr_sspnd.active,
+ ERTS_SCHED_NORMAL) == 1);
+#ifndef ERTS_DIRTY_SCHEDULERS
+ erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing,
+ ~chng_flg);
+#endif
}
- else {
- online = schdlr_sspnd.online;
+ else if (!(schdlr_sspnd.msb.ongoing|schdlr_sspnd.nmsb.ongoing)) {
if (plocks) {
have_unlocked_plocks = 1;
erts_smp_proc_unlock(p, plocks);
@@ -7951,83 +7912,91 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all)
suspend_run_queue(ERTS_RUNQ_IX(ix));
}
#ifdef ERTS_DIRTY_SCHEDULERS
- ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET(ERTS_SCHDLR_SSPND_CHNG_MSB, 0);
- schdlr_sspnd.msb.dirty_cpu_wait_active = schdlr_sspnd.dirty_cpu_online;
- for (ix = 0; ix < schdlr_sspnd.dirty_cpu_online; ix++) {
- ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix);
- scheduler_ssi_resume_wake(ssi);
- erts_smp_atomic32_read_band_nob(&ssi->flags,
- ~ERTS_SSI_FLG_SUSPENDED);
- }
- wake_dirty_schedulers(ERTS_DIRTY_CPU_RUNQ, 0);
-
- ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET(ERTS_SCHDLR_SSPND_CHNG_MSB, 0);
- schdlr_sspnd.msb.dirty_io_wait_active = erts_no_dirty_io_schedulers;
- for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) {
- ssi = ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix);
- scheduler_ssi_resume_wake(ssi);
- erts_smp_atomic32_read_band_nob(&ssi->flags,
- ~ERTS_SSI_FLG_SUSPENDED);
- }
- wake_dirty_schedulers(ERTS_DIRTY_IO_RUNQ, 0);
+ if (!normal) {
+ ASSERT(!schdlr_sspnd.msb.ongoing);
+ online = (int) schdlr_sspnd_get_nscheds(&schdlr_sspnd.online,
+ ERTS_SCHED_DIRTY_CPU);
+ for (ix = 0; ix < online; ix++) {
+ ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix);
+ scheduler_ssi_resume_wake(ssi);
+ }
+
+ for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) {
+ ssi = ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix);
+ scheduler_ssi_resume_wake(ssi);
+ }
+ }
#endif
- res = ERTS_SCHDLR_SSPND_DONE;
}
+
+ unblock_res:
+ if (schdlr_sspnd.msb.ongoing)
+ res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED;
+ else if (schdlr_sspnd.nmsb.ongoing)
+ res = ERTS_SCHDLR_SSPND_DONE_NMSCHED_BLOCKED;
+ else
+ res = ERTS_SCHDLR_SSPND_DONE;
}
erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
+
if (have_unlocked_plocks)
erts_smp_proc_lock(p, plocks);
- return res;
-}
-#ifdef DEBUG
-void
-erts_dbg_multi_scheduling_return_trap(Process *p, Eterm return_value)
-{
- if (return_value == am_blocked) {
- erts_aint32_t active = erts_smp_atomic32_read_nob(&schdlr_sspnd.active);
- ASSERT(1 <= active && active <= 2);
- ASSERT(ERTS_PROC_GET_SCHDATA(p)->no == 1);
+ if (resume_proc) {
+ if (!(plocks & ERTS_PROC_LOCK_STATUS))
+ erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
+ resume_process(p, plocks|ERTS_PROC_LOCK_STATUS);
+ if (!(plocks & ERTS_PROC_LOCK_STATUS))
+ erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
}
+
+ return res;
}
-#endif
int
erts_is_multi_scheduling_blocked(void)
{
int res;
erts_smp_mtx_lock(&schdlr_sspnd.mtx);
- res = schdlr_sspnd.msb.procs != NULL;
+ if (schdlr_sspnd.msb.blckrs)
+ res = 1;
+ else if (schdlr_sspnd.nmsb.blckrs)
+ res = -1;
+ else
+ res = 0;
erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
return res;
}
Eterm
-erts_multi_scheduling_blockers(Process *p)
+erts_multi_scheduling_blockers(Process *p, int normal)
{
Eterm res = NIL;
+ ErtsMultiSchedulingBlock *msbp;
+
+ msbp = normal ? &schdlr_sspnd.nmsb : &schdlr_sspnd.msb;
erts_smp_mtx_lock(&schdlr_sspnd.mtx);
- if (!erts_proclist_is_empty(schdlr_sspnd.msb.procs)) {
+ if (!erts_proclist_is_empty(msbp->blckrs)) {
Eterm *hp, *hp_end;
ErtsProcList *plp1, *plp2;
Uint max_size = 0;
- for (plp1 = erts_proclist_peek_first(schdlr_sspnd.msb.procs);
+ for (plp1 = erts_proclist_peek_first(msbp->blckrs);
plp1;
- plp1 = erts_proclist_peek_next(schdlr_sspnd.msb.procs, plp1)) {
+ plp1 = erts_proclist_peek_next(msbp->blckrs, plp1)) {
max_size += 2;
}
ASSERT(max_size);
hp = HAlloc(p, max_size);
hp_end = hp + max_size;
- for (plp1 = erts_proclist_peek_first(schdlr_sspnd.msb.procs);
+ for (plp1 = erts_proclist_peek_first(msbp->blckrs);
plp1;
- plp1 = erts_proclist_peek_next(schdlr_sspnd.msb.procs, plp1)) {
- for (plp2 = erts_proclist_peek_first(schdlr_sspnd.msb.procs);
+ plp1 = erts_proclist_peek_next(msbp->blckrs, plp1)) {
+ for (plp2 = erts_proclist_peek_first(msbp->blckrs);
plp2->pid != plp1->pid;
- plp2 = erts_proclist_peek_next(schdlr_sspnd.msb.procs, plp2));
+ plp2 = erts_proclist_peek_next(msbp->blckrs, plp2));
if (plp2 == plp1) {
res = CONS(hp, plp1->pid, res);
hp += 2;
@@ -8096,39 +8065,6 @@ sched_thread_func(void *vesdp)
#endif
erts_thread_init_float();
- if (no == 1) {
- erts_thr_progress_active(esdp, 0);
- erts_thr_progress_prepare_wait(esdp);
- }
-
- erts_smp_mtx_lock(&schdlr_sspnd.mtx);
-
- ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.changing)
- & ERTS_SCHDLR_SSPND_CHNG_ONLN);
-
- if (--schdlr_sspnd.curr_online == schdlr_sspnd.wait_curr_online) {
- erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing,
- ~ERTS_SCHDLR_SSPND_CHNG_ONLN);
- if (no != 1)
-#ifdef ERTS_DIRTY_SCHEDULERS
- erts_smp_cnd_broadcast(&schdlr_sspnd.cnd);
-#else
- erts_smp_cnd_signal(&schdlr_sspnd.cnd);
-#endif
- }
-
- if (no == 1) {
- while (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online)
- erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx);
- ERTS_SCHDLR_SSPND_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER);
- }
- erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
-
- if (no == 1) {
- erts_thr_progress_finalize_wait(esdp);
- erts_thr_progress_active(esdp, 1);
- }
-
#ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
esdp->verify_unused_temp_alloc
= erts_alloc_get_verify_unused_temp_alloc(
@@ -8138,7 +8074,7 @@ sched_thread_func(void *vesdp)
process_main();
/* No schedulers should *ever* terminate */
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"Scheduler thread number %beu terminated\n",
no);
return NULL;
@@ -8183,27 +8119,9 @@ sched_dirty_cpu_thread_func(void *vesdp)
#endif
erts_thread_init_float();
- erts_smp_mtx_lock(&schdlr_sspnd.mtx);
- ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_changing)
- & ERTS_SCHDLR_SSPND_CHNG_ONLN);
-
- if (--schdlr_sspnd.dirty_cpu_curr_online == schdlr_sspnd.dirty_cpu_wait_curr_online) {
- erts_smp_atomic32_read_band_nob(&schdlr_sspnd.dirty_cpu_changing,
- ~ERTS_SCHDLR_SSPND_CHNG_ONLN);
- if (no != 1)
- erts_smp_cnd_broadcast(&schdlr_sspnd.cnd);
- }
-
- if (no == 1) {
- while (schdlr_sspnd.dirty_cpu_curr_online != schdlr_sspnd.dirty_cpu_wait_curr_online)
- erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx);
- ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER);
- }
- erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
-
process_main();
/* No schedulers should *ever* terminate */
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"Dirty CPU scheduler thread number %beu terminated\n",
no);
return NULL;
@@ -8246,27 +8164,9 @@ sched_dirty_io_thread_func(void *vesdp)
#endif
erts_thread_init_float();
- erts_smp_mtx_lock(&schdlr_sspnd.mtx);
- ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_io_changing)
- & ERTS_SCHDLR_SSPND_CHNG_ONLN);
-
- if (--schdlr_sspnd.dirty_io_curr_online == schdlr_sspnd.dirty_io_wait_curr_online) {
- erts_smp_atomic32_read_band_nob(&schdlr_sspnd.dirty_io_changing,
- ~ERTS_SCHDLR_SSPND_CHNG_ONLN);
- if (no != 1)
- erts_smp_cnd_broadcast(&schdlr_sspnd.cnd);
- }
-
- if (no == 1) {
- while (schdlr_sspnd.dirty_io_curr_online != schdlr_sspnd.dirty_io_wait_curr_online)
- erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx);
- ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER);
- }
- erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
-
process_main();
/* No schedulers should *ever* terminate */
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"Dirty I/O scheduler thread number %beu terminated\n",
no);
return NULL;
@@ -8296,12 +8196,12 @@ erts_start_schedulers(void)
erts_snprintf(opts.name, 16, "runq_supervisor");
erts_atomic_init_nob(&runq_supervisor_sleeping, 0);
if (0 != ethr_event_init(&runq_supervision_event))
- erl_exit(1, "Failed to create run-queue supervision event\n");
+ erts_exit(ERTS_ERROR_EXIT, "Failed to create run-queue supervision event\n");
if (0 != ethr_thr_create(&runq_supervisor_tid,
runq_supervisor,
NULL,
&opts))
- erl_exit(1, "Failed to create run-queue supervision thread\n");
+ erts_exit(ERTS_ERROR_EXIT, "Failed to create run-queue supervision thread\n");
}
#endif
@@ -8339,14 +8239,14 @@ erts_start_schedulers(void)
erts_snprintf(opts.name, 16, "%d_dirty_cpu_scheduler", ix + 1);
res = ethr_thr_create(&esdp->tid,sched_dirty_cpu_thread_func,(void*)esdp,&opts);
if (res != 0)
- erl_exit(1, "Failed to create dirty cpu scheduler thread %d\n", ix);
+ erts_exit(ERTS_ERROR_EXIT, "Failed to create dirty cpu scheduler thread %d\n", ix);
}
for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) {
ErtsSchedulerData *esdp = ERTS_DIRTY_IO_SCHEDULER_IX(ix);
erts_snprintf(opts.name, 16, "%d_dirty_io_scheduler", ix + 1);
res = ethr_thr_create(&esdp->tid,sched_dirty_io_thread_func,(void*)esdp,&opts);
if (res != 0)
- erl_exit(1, "Failed to create dirty io scheduler thread %d\n", ix);
+ erts_exit(ERTS_ERROR_EXIT, "Failed to create dirty io scheduler thread %d\n", ix);
}
}
#endif
@@ -8358,10 +8258,10 @@ erts_start_schedulers(void)
res = ethr_thr_create(&aux_tid, aux_thread, NULL, &opts);
if (res != 0)
- erl_exit(1, "Failed to create aux thread\n");
+ erts_exit(ERTS_ERROR_EXIT, "Failed to create aux thread\n");
if (actual < 1)
- erl_exit(1,
+ erts_exit(ERTS_ERROR_EXIT,
"Failed to create any scheduler-threads: %s (%d)\n",
erl_errno_id(res),
res);
@@ -8515,9 +8415,8 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks,
resume_process(rp, rp_locks);
}
else {
-
rp = erts_pid2proc(c_p, c_p_locks|ERTS_PROC_LOCK_STATUS,
- pid, pid_locks|ERTS_PROC_LOCK_STATUS);
+ pid, ERTS_PROC_LOCK_STATUS);
if (!rp) {
c_p->flags &= ~F_P2PNR_RESCHED;
@@ -8526,40 +8425,84 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks,
ASSERT(!(c_p->flags & F_P2PNR_RESCHED));
- if (suspend) {
- if (suspend_process(c_p, rp))
- goto done;
- }
- else {
- if (!((ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS)
- & erts_smp_atomic32_read_acqb(&rp->state)))
- goto done;
+ /*
+ * Suspend the other process in order to prevent
+ * it from being selected for normal execution.
+ * This will however not prevent it from being
+ * selected for execution of a system task. If
+ * it is selected for execution of a system task
+ * we might be blocked for quite a while if the
+ * try-lock below fails. That is, there is room
+ * for improvement here...
+ */
- }
+ if (!suspend_process(c_p, rp)) {
+ /* Other process running */
- /* Other process running */
+ ASSERT(ERTS_PSFLG_RUNNING
+ & erts_smp_atomic32_read_nob(&rp->state));
- /*
- * If we got pending suspenders and suspend ourselves waiting
- * to suspend another process we might deadlock.
- * In this case we have to yield, be suspended by
- * someone else and then do it all over again.
- */
- if (!c_p->pending_suspenders) {
- /* Mark rp pending for suspend by c_p */
- add_pend_suspend(rp, c_p->common.id, handle_pend_sync_suspend);
- ASSERT(is_nil(c_p->suspendee));
+ running:
- /* Suspend c_p; when rp is suspended c_p will be resumed. */
- suspend_process(c_p, c_p);
- c_p->flags |= F_P2PNR_RESCHED;
+ /*
+ * If we got pending suspenders and suspend ourselves waiting
+ * to suspend another process we might deadlock.
+ * In this case we have to yield, be suspended by
+ * someone else and then do it all over again.
+ */
+ if (!c_p->pending_suspenders) {
+ /* Mark rp pending for suspend by c_p */
+ add_pend_suspend(rp, c_p->common.id, handle_pend_sync_suspend);
+ ASSERT(is_nil(c_p->suspendee));
+
+ /* Suspend c_p; when rp is suspended c_p will be resumed. */
+ suspend_process(c_p, c_p);
+ c_p->flags |= F_P2PNR_RESCHED;
+ }
+ /* Yield (caller is assumed to yield immediately in bif). */
+ erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS);
+ rp = ERTS_PROC_LOCK_BUSY;
+ }
+ else {
+ ErtsProcLocks need_locks = pid_locks & ~ERTS_PROC_LOCK_STATUS;
+ if (need_locks && erts_smp_proc_trylock(rp, need_locks) == EBUSY) {
+ if (ERTS_PSFLG_RUNNING_SYS
+ & erts_smp_atomic32_read_nob(&rp->state)) {
+ /* Executing system task... */
+ resume_process(rp, ERTS_PROC_LOCK_STATUS);
+ goto running;
+ }
+ erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS);
+ /*
+ * If we are unlucky, the process just got selected for
+ * execution of a system task. In this case we may be
+ * blocked here for quite a while... Execution of system
+ * tasks are fortunately quite rare events. We try to
+ * avoid this by checking if it is in a state executing
+ * system tasks (above), but it will not prevent all
+ * scenarios for a long block here...
+ */
+ rp = erts_pid2proc(c_p, c_p_locks|ERTS_PROC_LOCK_STATUS,
+ pid, pid_locks|ERTS_PROC_LOCK_STATUS);
+ if (!rp)
+ goto done;
+ }
+
+ /*
+ * The previous suspend has prevented the process
+ * from being selected for normal execution regardless
+ * of locks held or not held on it...
+ */
+ ASSERT(!(ERTS_PSFLG_RUNNING
+ & erts_smp_atomic32_read_nob(&rp->state)));
+
+ if (!suspend)
+ resume_process(rp, pid_locks|ERTS_PROC_LOCK_STATUS);
}
- /* Yield (caller is assumed to yield immediately in bif). */
- erts_smp_proc_unlock(rp, pid_locks|ERTS_PROC_LOCK_STATUS);
- rp = ERTS_PROC_LOCK_BUSY;
}
done:
+
if (rp && rp != ERTS_PROC_LOCK_BUSY && !(pid_locks & ERTS_PROC_LOCK_STATUS))
erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS);
if (unlock_c_p_status)
@@ -9493,11 +9436,11 @@ Process *schedule(Process *p, int calls)
if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
ErtsProcList *pnd_xtrs = rq->procs.pending_exiters;
if (erts_proclist_fetch(&pnd_xtrs, NULL)) {
- rq->procs.pending_exiters = NULL;
- erts_smp_runq_unlock(rq);
- handle_pending_exiters(pnd_xtrs);
- erts_smp_runq_lock(rq);
- }
+ rq->procs.pending_exiters = NULL;
+ erts_smp_runq_unlock(rq);
+ handle_pending_exiters(pnd_xtrs);
+ erts_smp_runq_lock(rq);
+ }
if (rq->check_balance_reds <= 0)
check_balance(rq);
@@ -12269,7 +12212,7 @@ static void doit_exit_link(ErtsLink *lnk, void *vpcontext)
break;
default:
- erl_exit(1, "bad type in link list\n");
+ erts_exit(ERTS_ERROR_EXIT, "bad type in link list\n");
break;
}
erts_destroy_link(lnk);
@@ -12310,7 +12253,7 @@ erts_do_exit_process(Process* p, Eterm reason)
#endif
if (p->static_flags & ERTS_STC_FLG_SYSTEM_PROC)
- erl_exit(ERTS_DUMP_EXIT, "System process %T terminated: %T\n",
+ erts_exit(ERTS_DUMP_EXIT, "System process %T terminated: %T\n",
p->common.id, reason);
#ifdef ERTS_SMP
@@ -12414,21 +12357,46 @@ erts_continue_exit_process(Process *p)
#endif
#ifdef ERTS_SMP
+ if (p->flags & F_SCHDLR_ONLN_WAITQ)
+ abort_sched_onln_chng_waitq(p);
+
if (p->flags & F_HAVE_BLCKD_MSCHED) {
ErtsSchedSuspendResult ssr;
- ssr = erts_block_multi_scheduling(p, ERTS_PROC_LOCK_MAIN, 0, 1);
+ ssr = erts_block_multi_scheduling(p, ERTS_PROC_LOCK_MAIN, 0, 0, 1);
+ switch (ssr) {
+ case ERTS_SCHDLR_SSPND_YIELD_RESTART:
+ goto yield;
+ case ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED:
+ case ERTS_SCHDLR_SSPND_DONE_NMSCHED_BLOCKED:
+ case ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED:
+ case ERTS_SCHDLR_SSPND_YIELD_DONE_NMSCHED_BLOCKED:
+ case ERTS_SCHDLR_SSPND_DONE:
+ case ERTS_SCHDLR_SSPND_YIELD_DONE:
+ p->flags &= ~F_HAVE_BLCKD_MSCHED;
+ break;
+ case ERTS_SCHDLR_SSPND_EINVAL:
+ default:
+ erts_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error: %d\n",
+ __FILE__, __LINE__, (int) ssr);
+ }
+ }
+ if (p->flags & F_HAVE_BLCKD_NMSCHED) {
+ ErtsSchedSuspendResult ssr;
+ ssr = erts_block_multi_scheduling(p, ERTS_PROC_LOCK_MAIN, 0, 1, 1);
switch (ssr) {
case ERTS_SCHDLR_SSPND_YIELD_RESTART:
goto yield;
case ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED:
+ case ERTS_SCHDLR_SSPND_DONE_NMSCHED_BLOCKED:
case ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED:
+ case ERTS_SCHDLR_SSPND_YIELD_DONE_NMSCHED_BLOCKED:
case ERTS_SCHDLR_SSPND_DONE:
case ERTS_SCHDLR_SSPND_YIELD_DONE:
p->flags &= ~F_HAVE_BLCKD_MSCHED;
break;
case ERTS_SCHDLR_SSPND_EINVAL:
default:
- erl_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error: %d\n",
+ erts_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error: %d\n",
__FILE__, __LINE__, (int) ssr);
}
}
@@ -12925,10 +12893,10 @@ erts_print_scheduler_info(int to, void *to_arg, ErtsSchedulerData *esdp) {
* The same global atomic is used as refcount.
*
* A BIF that calls this should make sure to schedule out to never come back:
- * erl_halt((int)(- code));
+ * erts_halt(code);
* ERTS_BIF_YIELD1(bif_export[BIF_erlang_halt_1], BIF_P, NIL);
*/
-void erl_halt(int code)
+void erts_halt(int code)
{
if (-1 == erts_smp_atomic32_cmpxchg_acqb(&erts_halt_progress,
erts_no_schedulers,
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index ef4aab7234..c88bd7056c 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -79,10 +79,8 @@ struct ErtsNodesMonitor_;
#define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT 0
#define ERTS_MAX_NO_OF_SCHEDULERS 1024
-#ifdef ERTS_DIRTY_SCHEDULERS
#define ERTS_MAX_NO_OF_DIRTY_CPU_SCHEDULERS ERTS_MAX_NO_OF_SCHEDULERS
#define ERTS_MAX_NO_OF_DIRTY_IO_SCHEDULERS ERTS_MAX_NO_OF_SCHEDULERS
-#endif
#define ERTS_DEFAULT_MAX_PROCESSES (1 << 18)
@@ -246,7 +244,9 @@ extern int erts_sched_thread_suggested_stack_size;
typedef enum {
ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED,
+ ERTS_SCHDLR_SSPND_DONE_NMSCHED_BLOCKED,
ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED,
+ ERTS_SCHDLR_SSPND_YIELD_DONE_NMSCHED_BLOCKED,
ERTS_SCHDLR_SSPND_DONE,
ERTS_SCHDLR_SSPND_YIELD_RESTART,
ERTS_SCHDLR_SSPND_YIELD_DONE,
@@ -1339,6 +1339,8 @@ extern int erts_system_profile_ts_type;
#define F_OFF_HEAP_MSGQ_CHNG (1 << 14) /* Off heap msg queue changing */
#define F_ABANDONED_HEAP_USE (1 << 15) /* Have usage of abandoned heap */
#define F_DELAY_GC (1 << 16) /* Similar to disable GC (see below) */
+#define F_SCHDLR_ONLN_WAITQ (1 << 17) /* Process enqueued waiting to change schedulers online */
+#define F_HAVE_BLCKD_NMSCHED (1 << 18) /* Process has blocked normal multi-scheduling */
/*
* F_DISABLE_GC and F_DELAY_GC are similar. Both will prevent
@@ -1523,6 +1525,7 @@ void erts_init_scheduling(int, int
int erts_set_gc_state(Process *c_p, int enable);
Eterm erts_sched_wall_time_request(Process *c_p, int set, int enable);
+Eterm erts_system_check_request(Process *c_p);
Eterm erts_gc_info_request(Process *c_p);
Uint64 erts_get_proc_interval(void);
Uint64 erts_ensure_later_proc_interval(Uint64);
@@ -1532,6 +1535,7 @@ int erts_setup_nif_gc(Process* proc, Eterm** objv, int* nobj); /* see erl_nif.c
void erts_destroy_nif_export(void *); /* see erl_nif.c */
ErtsProcList *erts_proclist_create(Process *);
+ErtsProcList *erts_proclist_copy(ErtsProcList *);
void erts_proclist_destroy(ErtsProcList *);
ERTS_GLB_INLINE int erts_proclist_same(ErtsProcList *, Process *);
@@ -1717,28 +1721,21 @@ void erts_schedule_complete_off_heap_message_queue_change(Eterm pid);
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
int erts_dbg_check_halloc_lock(Process *p);
#endif
-#ifdef DEBUG
-void erts_dbg_multi_scheduling_return_trap(Process *, Eterm);
-#endif
-int erts_get_max_no_executing_schedulers(void);
#if defined(ERTS_SMP) || defined(ERTS_DIRTY_SCHEDULERS)
-ErtsSchedSuspendResult
-erts_schedulers_state(Uint *, Uint *, Uint *, Uint *, Uint *, Uint *, int);
+void
+erts_schedulers_state(Uint *, Uint *, Uint *, Uint *, Uint *, Uint *, Uint *, Uint *);
#endif
#ifdef ERTS_SMP
ErtsSchedSuspendResult
erts_set_schedulers_online(Process *p,
ErtsProcLocks plocks,
Sint new_no,
- Sint *old_no
-#ifdef ERTS_DIRTY_SCHEDULERS
- , int dirty_only
-#endif
- );
+ Sint *old_no,
+ int dirty_only);
ErtsSchedSuspendResult
-erts_block_multi_scheduling(Process *, ErtsProcLocks, int, int);
+erts_block_multi_scheduling(Process *, ErtsProcLocks, int, int, int);
int erts_is_multi_scheduling_blocked(void);
-Eterm erts_multi_scheduling_blockers(Process *);
+Eterm erts_multi_scheduling_blockers(Process *, int);
void erts_start_schedulers(void);
void erts_alloc_notify_delayed_dealloc(int);
void erts_alloc_ensure_handle_delayed_dealloc_call(int);
@@ -2461,6 +2458,6 @@ erts_sched_poke(ErtsSchedulerSleepInfo *ssi)
#endif
-void erl_halt(int code);
+void erts_halt(int code);
extern erts_smp_atomic32_t erts_halt_progress;
extern int erts_halt_code;
diff --git a/erts/emulator/beam/erl_process_dict.c b/erts/emulator/beam/erl_process_dict.c
index 36d16f7f42..487b944fc0 100644
--- a/erts/emulator/beam/erl_process_dict.c
+++ b/erts/emulator/beam/erl_process_dict.c
@@ -392,7 +392,7 @@ static void pd_hash_erase(Process *p, Eterm id, Eterm *ret)
"display term found in line %d:\n"
"%T\n", p->common.id, __LINE__, old);
#endif
- erl_exit(1, "Damaged process dictionary found during erase/1.");
+ erts_exit(ERTS_ERROR_EXIT, "Damaged process dictionary found during erase/1.");
}
if ((range = HASH_RANGE(p->dictionary)) > INITIAL_SIZE &&
range / 2 > (p->dictionary->numElements)) {
@@ -452,7 +452,7 @@ Eterm pd_hash_get_with_hval(Process *p, Eterm bucket, Eterm id)
"display term found in line %d:\n"
"%T\n", p->common.id, __LINE__, bucket);
#endif
- erl_exit(1, "Damaged process dictionary found during get/1.");
+ erts_exit(ERTS_ERROR_EXIT, "Damaged process dictionary found during get/1.");
}
return am_undefined;
}
@@ -707,7 +707,7 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value)
"%T\n", p->common.id, __LINE__, old);
#endif
- erl_exit(1, "Damaged process dictionary found during put/2.");
+ erts_exit(ERTS_ERROR_EXIT, "Damaged process dictionary found during put/2.");
}
if (HASH_RANGE(p->dictionary) <= p->dictionary->numElements) {
grow(p);
@@ -1061,7 +1061,7 @@ static void pd_check(ProcDict *pd)
}
continue;
} else {
- erl_exit(1,
+ erts_exit(ERTS_ERROR_EXIT,
"Found tag 0x%08x in process dictionary at position %d",
(unsigned long) t, (int) i);
}
diff --git a/erts/emulator/beam/erl_process_lock.h b/erts/emulator/beam/erl_process_lock.h
index a64c993e8f..9c59301086 100644
--- a/erts/emulator/beam/erl_process_lock.h
+++ b/erts/emulator/beam/erl_process_lock.h
@@ -523,6 +523,10 @@ erts_smp_proc_lock__(Process *p,
ERTS_LC_ASSERT((locks & ~ERTS_PROC_LOCKS_ALL) == 0);
+#ifdef ERTS_ENABLE_LOCK_CHECK
+ erts_proc_lc_lock(p, locks, file, line);
+#endif
+
old_lflgs = erts_smp_proc_raw_trylock__(p, locks);
if (old_lflgs != 0) {
@@ -544,9 +548,6 @@ erts_smp_proc_lock__(Process *p,
#ifdef ERTS_ENABLE_LOCK_COUNT
erts_lcnt_proc_lock_post_x(&(p->lock), locks, file, line);
#endif
-#ifdef ERTS_ENABLE_LOCK_CHECK
- erts_proc_lc_lock(p, locks, file, line);
-#endif
#ifdef ERTS_PROC_LOCK_DEBUG
erts_proc_lock_op_debug(p, locks, 1);
diff --git a/erts/emulator/beam/erl_ptab.c b/erts/emulator/beam/erl_ptab.c
index f7997df051..9ed175fe01 100644
--- a/erts/emulator/beam/erl_ptab.c
+++ b/erts/emulator/beam/erl_ptab.c
@@ -1255,7 +1255,7 @@ ptab_list_bif_engine(Process *c_p, Eterm *res_accp, Binary *mbp)
return 1;
default:
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"%s:%d:ptab_list_bif_engine(): Invalid state: %d\n",
__FILE__, __LINE__, (int) ptlbdp->state);
}
diff --git a/erts/emulator/beam/erl_term.c b/erts/emulator/beam/erl_term.c
index e31d3a951d..d2343912b4 100644
--- a/erts/emulator/beam/erl_term.c
+++ b/erts/emulator/beam/erl_term.c
@@ -72,7 +72,7 @@ et_abort(const char *expr, const char *file, unsigned line)
* Prevent infinite loop.
*/
have_been_called = 1;
- erl_exit(1, "TYPE ASSERTION FAILED, file %s, line %u: %s\n", file, line, expr);
+ erts_exit(ERTS_ERROR_EXIT, "TYPE ASSERTION FAILED, file %s, line %u: %s\n", file, line, expr);
}
#else
erts_fprintf(stderr, "TYPE ASSERTION FAILED, file %s, line %u: %s\n", file, line, expr);
diff --git a/erts/emulator/beam/erl_thr_progress.c b/erts/emulator/beam/erl_thr_progress.c
index 7148b756e7..7b06fd840f 100644
--- a/erts/emulator/beam/erl_thr_progress.c
+++ b/erts/emulator/beam/erl_thr_progress.c
@@ -502,7 +502,7 @@ erts_thr_progress_register_unmanaged_thread(ErtsThrPrgrCallbacks *callbacks)
if (tpd) {
if (!tpd->is_temporary)
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"%s:%d:%s(): Double register of thread\n",
__FILE__, __LINE__, __func__);
is_blocking = tpd->is_blocking;
@@ -524,7 +524,7 @@ erts_thr_progress_register_unmanaged_thread(ErtsThrPrgrCallbacks *callbacks)
#endif
ASSERT(tpd->id >= 0);
if (tpd->id >= intrnl->unmanaged.no)
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"%s:%d:%s(): Too many unmanaged registered threads\n",
__FILE__, __LINE__, __func__);
@@ -547,7 +547,7 @@ erts_thr_progress_register_managed_thread(ErtsSchedulerData *esdp,
if (tpd) {
if (!tpd->is_temporary)
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"%s:%d:%s(): Double register of thread\n",
__FILE__, __LINE__, __func__);
is_blocking = tpd->is_blocking;
@@ -568,7 +568,7 @@ erts_thr_progress_register_managed_thread(ErtsSchedulerData *esdp,
tpd->id = erts_atomic32_inc_read_nob(&intrnl->misc.data.managed_id);
ASSERT(tpd->id >= 0);
if (tpd->id >= intrnl->managed.no)
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"%s:%d:%s(): Too many managed registered threads\n",
__FILE__, __LINE__, __func__);
@@ -1033,7 +1033,7 @@ has_reached_wakeup(ErtsThrPrgrVal wakeup)
limit += 1;
if (!erts_thr_progress_has_passed__(limit, wakeup))
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"Invalid wakeup request value found:"
" current=%b64u, wakeup=%b64u, limit=%b64u",
current, wakeup, limit);
@@ -1102,7 +1102,7 @@ request_wakeup_managed(ErtsThrPrgrData *tpd, ErtsThrPrgrVal value)
ix = erts_atomic32_inc_read_nob(&mwd->len) - 1;
#if ERTS_THR_PRGR_DBG_CHK_WAKEUP_REQUEST_VALUE
if (ix >= intrnl->managed.no)
- erl_exit(ERTS_ABORT_EXIT, "Internal error: Too many wakeup requests\n");
+ erts_exit(ERTS_ABORT_EXIT, "Internal error: Too many wakeup requests\n");
#endif
mwd->id[ix] = tpd->id;
diff --git a/erts/emulator/beam/erl_thr_queue.c b/erts/emulator/beam/erl_thr_queue.c
index 3a91ca9dbe..7ff456b915 100644
--- a/erts/emulator/beam/erl_thr_queue.c
+++ b/erts/emulator/beam/erl_thr_queue.c
@@ -224,7 +224,7 @@ ErtsThrQCleanState_t
erts_thr_q_destroy(ErtsThrQ_t *q)
{
if (!q->q.blk)
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"Trying to destroy not created thread queue\n");
return erts_thr_q_finalize(q);
}
@@ -589,7 +589,7 @@ enqueue(ErtsThrQ_t *q, void *data, ErtsThrQElement_t *this)
#if ERTS_THR_Q_DBG_CHK_DATA
if (!data)
- erl_exit(ERTS_ABORT_EXIT, "Missing data in enqueue\n");
+ erts_exit(ERTS_ABORT_EXIT, "Missing data in enqueue\n");
#endif
ASSERT(!q->q.finalizing);
@@ -771,7 +771,7 @@ erts_thr_q_dequeue(ErtsThrQ_t *q)
#if ERTS_THR_Q_DBG_CHK_DATA
head->data.ptr = NULL;
if (!res)
- erl_exit(ERTS_ABORT_EXIT, "Missing data in dequeue\n");
+ erts_exit(ERTS_ABORT_EXIT, "Missing data in dequeue\n");
#endif
clean(q,
(q->head.deq_fini.automatic
diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c
index 98159fdf72..f3f528eaf6 100644
--- a/erts/emulator/beam/erl_time_sup.c
+++ b/erts/emulator/beam/erl_time_sup.c
@@ -289,7 +289,7 @@ read_corrected_time(int os_drift_corrected)
ci = time_sup.inf.c.parmon.cdata.insts.curr;
else {
if (os_mtime < time_sup.inf.c.parmon.cdata.insts.prev.os_mtime)
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"OS monotonic time stepped backwards\n");
ci = time_sup.inf.c.parmon.cdata.insts.prev;
}
@@ -381,7 +381,7 @@ check_time_correction(void *vesdp)
erts_smp_rwmtx_runlock(&time_sup.inf.c.parmon.rwmtx);
if (os_mtime < ci.os_mtime)
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"OS monotonic time stepped backwards\n");
erl_mtime = calc_corrected_erl_mtime(os_mtime, &ci, &mdiff,
@@ -798,7 +798,7 @@ finalize_corrected_time_offset(ErtsSystemTime *stimep)
erts_smp_rwmtx_runlock(&time_sup.inf.c.parmon.rwmtx);
if (os_mtime < ci.os_mtime)
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"OS monotonic time stepped backwards\n");
return calc_corrected_erl_mtime(os_mtime, &ci, NULL,
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index 25393369a7..41876b8c62 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -661,7 +661,7 @@ WRITE_SYS_MSG_TO_PORT(Eterm unused_to,
erts_encode_ext(message, &ptr);
if (!(ptr <= buffer+size)) {
- erl_exit(1, "Internal error in do_send_to_port: %d\n", ptr-buffer);
+ erts_exit(ERTS_ERROR_EXIT, "Internal error in do_send_to_port: %d\n", ptr-buffer);
}
#ifndef ERTS_SMP
@@ -1286,7 +1286,7 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type,
case SEQ_TRACE_PRINT: type_atom = am_print; break;
case SEQ_TRACE_RECEIVE: type_atom = am_receive; break;
default:
- erl_exit(1, "invalid type in seq_trace_output_generic: %d:\n", type);
+ erts_exit(ERTS_ERROR_EXIT, "invalid type in seq_trace_output_generic: %d:\n", type);
return; /* To avoid warning */
}
diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c
index 551717139d..36d85b0a22 100644
--- a/erts/emulator/beam/erl_unicode.c
+++ b/erts/emulator/beam/erl_unicode.c
@@ -2507,7 +2507,7 @@ void erts_copy_utf8_to_utf16_little(byte *target, byte *bytes, int num_chars)
((Uint) (bytes[3] & ((byte) 0x3F)));
bytes += 4;
} else {
- erl_exit(1,"Internal unicode error in prim_file:internal_name2native/1");
+ erts_exit(ERTS_ERROR_EXIT,"Internal unicode error in prim_file:internal_name2native/1");
}
*target++ = (byte) (unipoint & 0xFF);
*target++ = (byte) ((unipoint >> 8) & 0xFF);
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 58010dcb72..aac1490f0c 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -570,7 +570,7 @@ void erts_encode_ext(Eterm term, byte **ext)
*ep++ = VERSION_MAGIC;
ep = enc_term(NULL, term, ep, TERM_TO_BINARY_DFLAGS, NULL);
if (!ep)
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"%s:%d:erts_encode_ext(): Internal data structure error\n",
__FILE__, __LINE__);
*ext = ep;
@@ -1559,7 +1559,7 @@ static BIF_RETTYPE binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binar
b2t_destroy_context(ctx);
if (ctx->u.dc.factory.hp > ctx->u.dc.factory.hp_end) {
- erl_exit(1, ":%s, line %d: heap overrun by %d words(s)\n",
+ erts_exit(ERTS_ERROR_EXIT, ":%s, line %d: heap overrun by %d words(s)\n",
__FILE__, __LINE__, ctx->u.dc.factory.hp - ctx->u.dc.factory.hp_end);
}
erts_factory_close(&ctx->u.dc.factory);
@@ -1708,12 +1708,12 @@ erts_term_to_binary_simple(Process* p, Eterm Term, Uint size, int level, Uint fl
if ((endp = enc_term(NULL, Term, bytes, flags, NULL))
== NULL) {
- erl_exit(1, "%s, line %d: bad term: %x\n",
+ erts_exit(ERTS_ERROR_EXIT, "%s, line %d: bad term: %x\n",
__FILE__, __LINE__, Term);
}
real_size = endp - bytes;
if (real_size > size) {
- erl_exit(1, "%s, line %d: buffer overflow: %d word(s)\n",
+ erts_exit(ERTS_ERROR_EXIT, "%s, line %d: buffer overflow: %d word(s)\n",
__FILE__, __LINE__, real_size - size);
}
@@ -1753,12 +1753,12 @@ erts_term_to_binary_simple(Process* p, Eterm Term, Uint size, int level, Uint fl
bytes[0] = VERSION_MAGIC;
if ((endp = enc_term(NULL, Term, bytes+1, flags, NULL))
== NULL) {
- erl_exit(1, "%s, line %d: bad term: %x\n",
+ erts_exit(ERTS_ERROR_EXIT, "%s, line %d: bad term: %x\n",
__FILE__, __LINE__, Term);
}
real_size = endp - bytes;
if (real_size > size) {
- erl_exit(1, "%s, line %d: buffer overflow: %d word(s)\n",
+ erts_exit(ERTS_ERROR_EXIT, "%s, line %d: buffer overflow: %d word(s)\n",
__FILE__, __LINE__, endp - (bytes + size));
}
return erts_realloc_binary(bin, real_size);
@@ -2631,7 +2631,7 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
ASSERT(node_sz < 17);
break;
default:
- erl_exit(1, "bad header\r\n");
+ erts_exit(ERTS_ERROR_EXIT, "bad header\r\n");
}
ptr++;
@@ -4086,7 +4086,7 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj,
ASSERT(node_sz < 17);
break;
default:
- erl_exit(1, "bad header\r\n");
+ erts_exit(ERTS_ERROR_EXIT, "bad header\r\n");
}
ptr++;
@@ -4174,7 +4174,7 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj,
break;
default:
- erl_exit(1,"Internal data structure error (in encode_size_struct2)%x\n",
+ erts_exit(ERTS_ERROR_EXIT,"Internal data structure error (in encode_size_struct2)%x\n",
obj);
}
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 2a8bdb6ee3..d7edf451be 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -425,7 +425,7 @@ void erl_grow_estack(ErtsEStack*, Uint need);
#define ESTACK_CHANGE_ALLOCATOR(s,t) \
do { \
if ((s).start != ESTK_DEF_STACK(s)) { \
- erl_exit(1, "Internal error - trying to change allocator " \
+ erts_exit(ERTS_ERROR_EXIT, "Internal error - trying to change allocator " \
"type of active estack\n"); \
} \
(s).alloc_type = (t); \
@@ -586,7 +586,7 @@ do { \
#define WSTACK_CHANGE_ALLOCATOR(s,t) \
do { \
if (s.wstart != WSTK_DEF_STACK(s)) { \
- erl_exit(1, "Internal error - trying to change allocator " \
+ erts_exit(ERTS_ERROR_EXIT, "Internal error - trying to change allocator " \
"type of active wstack\n"); \
} \
s.alloc_type = (t); \
@@ -778,7 +778,7 @@ ErtsPStack s = { (byte*)PSTK_DEF_STACK(s), /* pstart */ \
#define PSTACK_CHANGE_ALLOCATOR(s,t) \
do { \
if (s.pstart != (byte*)PSTK_DEF_STACK(s)) { \
- erl_exit(1, "Internal error - trying to change allocator " \
+ erts_exit(ERTS_ERROR_EXIT, "Internal error - trying to change allocator " \
"type of active pstack\n"); \
} \
s.alloc_type = (t); \
@@ -1008,6 +1008,7 @@ typedef struct {
Binary* erts_alloc_loader_state(void);
Eterm erts_module_for_prepared_code(Binary* magic);
+Eterm erts_has_code_on_load(Binary* magic);
Eterm erts_prepare_loading(Binary* loader_state, Process *c_p,
Eterm group_leader, Eterm* modp,
byte* code, Uint size);
@@ -1026,7 +1027,7 @@ Eterm erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info);
/* beam_ranges.c */
void erts_init_ranges(void);
-void erts_start_staging_ranges(void);
+void erts_start_staging_ranges(int num_new);
void erts_end_staging_ranges(int commit);
void erts_update_ranges(BeamInstr* code, Uint size);
void erts_remove_from_ranges(BeamInstr* code);
@@ -1047,8 +1048,8 @@ double erts_get_positive_zero_float(void);
/* config.c */
-__decl_noreturn void __noreturn erl_exit(int n, char*, ...);
-__decl_noreturn void __noreturn erl_exit_flush_async(int n, char*, ...);
+__decl_noreturn void __noreturn erts_exit(int n, char*, ...);
+__decl_noreturn void __noreturn erts_flush_async_exit(int n, char*, ...);
void erl_error(char*, va_list);
/* This controls whether sharing-preserving copy is used by Erlang */
diff --git a/erts/emulator/beam/index.c b/erts/emulator/beam/index.c
index 06d0b5123d..5f6ea14732 100644
--- a/erts/emulator/beam/index.c
+++ b/erts/emulator/beam/index.c
@@ -84,7 +84,7 @@ index_put_entry(IndexTable* t, void* tmpl)
Uint sz;
if (ix >= t->limit) {
/* A core dump is unnecessary */
- erl_exit(ERTS_DUMP_EXIT, "no more index entries in %s (max=%d)\n",
+ erts_exit(ERTS_DUMP_EXIT, "no more index entries in %s (max=%d)\n",
t->htable.name, t->limit);
}
sz = INDEX_PAGE_SIZE*sizeof(IndexSlot*);
@@ -123,7 +123,7 @@ void erts_index_merge(Hash* src, IndexTable* dst)
ix = dst->entries++;
if (ix >= dst->size) {
if (ix >= dst->limit) {
- erl_exit(1, "no more index entries in %s (max=%d)\n",
+ erts_exit(ERTS_ERROR_EXIT, "no more index entries in %s (max=%d)\n",
dst->htable.name, dst->limit);
}
sz = INDEX_PAGE_SIZE*sizeof(IndexSlot*);
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 41b0fcdd1b..797e4cdadc 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -3587,7 +3587,7 @@ terminate_port(Port *prt)
if ((state & ERTS_PORT_SFLG_HALT)
&& (erts_smp_atomic32_dec_read_nob(&erts_halt_progress) == 0)) {
erts_port_release(prt); /* We will exit and never return */
- erl_exit_flush_async(erts_halt_code, "");
+ erts_flush_async_exit(erts_halt_code, "");
}
if (is_internal_port(send_closed_port_id))
deliver_result(send_closed_port_id, connected_id, am_closed);
@@ -5018,7 +5018,7 @@ void erts_raw_port_command(Port* p, byte* buf, Uint len)
ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(p));
if (len > (Uint) INT_MAX)
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"Absurdly large data buffer (%beu bytes) passed to"
"output callback of %s driver.\n",
len,
@@ -7396,7 +7396,7 @@ driver_system_info(ErlDrvSysInfo *sip, size_t si_size)
* of ErlDrvSysInfo (introduced in driver version 1.0).
*/
if (!sip || si_size < ERL_DRV_SYS_INFO_SIZE(smp_support))
- erl_exit(1,
+ erts_exit(ERTS_ERROR_EXIT,
"driver_system_info(%p, %ld) called with invalid arguments\n",
sip, si_size);
diff --git a/erts/emulator/beam/module.h b/erts/emulator/beam/module.h
index e66d628ca9..b7468b0926 100644
--- a/erts/emulator/beam/module.h
+++ b/erts/emulator/beam/module.h
@@ -37,6 +37,7 @@ struct erl_module_instance {
typedef struct erl_module {
IndexSlot slot; /* Must be located at top of struct! */
int module; /* Atom index for module (not tagged). */
+ int seen; /* Used by finish_loading() */
struct erl_module_instance curr;
struct erl_module_instance old; /* protected by "old_code" rwlock */
diff --git a/erts/emulator/beam/register.c b/erts/emulator/beam/register.c
index fdb6cbc813..ebc35b0c4d 100644
--- a/erts/emulator/beam/register.c
+++ b/erts/emulator/beam/register.c
@@ -125,7 +125,7 @@ static RegProc* reg_alloc(RegProc *tmpl)
{
RegProc* obj = (RegProc*) erts_alloc(ERTS_ALC_T_REG_PROC, sizeof(RegProc));
if (!obj) {
- erl_exit(1, "Can't allocate %d bytes of memory\n", sizeof(RegProc));
+ erts_exit(ERTS_ERROR_EXIT, "Can't allocate %d bytes of memory\n", sizeof(RegProc));
}
obj->name = tmpl->name;
obj->p = tmpl->p;
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 068d636a40..03b9088adc 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -580,15 +580,16 @@ static unsigned long zero_value = 0, one_value = 1;
# endif /* !__WIN32__ */
#endif /* WANT_NONBLOCKING */
-__decl_noreturn void __noreturn erl_exit(int n, char*, ...);
+__decl_noreturn void __noreturn erts_exit(int n, char*, ...);
-/* Some special erl_exit() codes: */
-#define ERTS_INTR_EXIT INT_MIN /* called from signal handler */
-#define ERTS_ABORT_EXIT (INT_MIN + 1) /* no crash dump; only abort() */
-#define ERTS_DUMP_EXIT (INT_MIN + 2) /* crash dump; then exit() */
+/* Some special erts_exit() codes: */
+#define ERTS_INTR_EXIT -1 /* called from signal handler */
+#define ERTS_ABORT_EXIT -2 /* no crash dump; only abort() */
+#define ERTS_DUMP_EXIT -3 /* crash dump; then exit() */
+#define ERTS_ERROR_EXIT -4 /* crash dump; then abort() */
#define ERTS_INTERNAL_ERROR(What) \
- erl_exit(ERTS_ABORT_EXIT, "%s:%d:%s(): Internal error: %s\n", \
+ erts_exit(ERTS_ABORT_EXIT, "%s:%d:%s(): Internal error: %s\n", \
__FILE__, __LINE__, __func__, What)
Eterm erts_check_io_info(void *p);
@@ -903,7 +904,7 @@ erts_refc_inc(erts_refc_t *refcp, erts_aint_t min_val)
#ifdef ERTS_REFC_DEBUG
erts_aint_t val = erts_smp_atomic_inc_read_nob((erts_smp_atomic_t *) refcp);
if (val < min_val)
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"erts_refc_inc(): Bad refc found (refc=%ld < %ld)!\n",
val, min_val);
#else
@@ -917,7 +918,7 @@ erts_refc_inctest(erts_refc_t *refcp, erts_aint_t min_val)
erts_aint_t val = erts_smp_atomic_inc_read_nob((erts_smp_atomic_t *) refcp);
#ifdef ERTS_REFC_DEBUG
if (val < min_val)
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"erts_refc_inctest(): Bad refc found (refc=%ld < %ld)!\n",
val, min_val);
#endif
@@ -930,7 +931,7 @@ erts_refc_dec(erts_refc_t *refcp, erts_aint_t min_val)
#ifdef ERTS_REFC_DEBUG
erts_aint_t val = erts_smp_atomic_dec_read_nob((erts_smp_atomic_t *) refcp);
if (val < min_val)
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"erts_refc_dec(): Bad refc found (refc=%ld < %ld)!\n",
val, min_val);
#else
@@ -944,7 +945,7 @@ erts_refc_dectest(erts_refc_t *refcp, erts_aint_t min_val)
erts_aint_t val = erts_smp_atomic_dec_read_nob((erts_smp_atomic_t *) refcp);
#ifdef ERTS_REFC_DEBUG
if (val < min_val)
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"erts_refc_dectest(): Bad refc found (refc=%ld < %ld)!\n",
val, min_val);
#endif
@@ -957,7 +958,7 @@ erts_refc_add(erts_refc_t *refcp, erts_aint_t diff, erts_aint_t min_val)
#ifdef ERTS_REFC_DEBUG
erts_aint_t val = erts_smp_atomic_add_read_nob((erts_smp_atomic_t *) refcp, diff);
if (val < min_val)
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"erts_refc_add(%ld): Bad refc found (refc=%ld < %ld)!\n",
diff, val, min_val);
#else
@@ -971,7 +972,7 @@ erts_refc_read(erts_refc_t *refcp, erts_aint_t min_val)
erts_aint_t val = erts_smp_atomic_read_nob((erts_smp_atomic_t *) refcp);
#ifdef ERTS_REFC_DEBUG
if (val < min_val)
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"erts_refc_read(): Bad refc found (refc=%ld < %ld)!\n",
val, min_val);
#endif
diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c
index 48cb39333a..0ce4f443f3 100644
--- a/erts/emulator/beam/time.c
+++ b/erts/emulator/beam/time.c
@@ -567,7 +567,7 @@ erts_init_time(int time_correction, ErtsTimeWarpMode time_warp_mode)
itime = erts_init_time_sup(time_correction, time_warp_mode);
#ifdef TIW_ITIME_IS_CONSTANT
if (itime != TIW_ITIME) {
- erl_exit(ERTS_ABORT_EXIT, "timer resolution mismatch %d != %d", itime, TIW_ITIME);
+ erts_exit(ERTS_ABORT_EXIT, "timer resolution mismatch %d != %d", itime, TIW_ITIME);
}
#else
tiw_itime = itime;
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 3ba02023a1..66fcfcf6ce 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -1069,7 +1069,7 @@ tail_recur:
}
default:
- erl_exit(1, "Invalid tag in make_hash(0x%X,0x%X)\n", term, op);
+ erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash(0x%X,0x%X)\n", term, op);
return 0;
}
if (WSTACK_ISEMPTY(stack)) break;
@@ -1342,7 +1342,7 @@ make_hash2(Eterm term)
i = hashmap_bitcount(MAP_HEADER_VAL(hdr));
break;
default:
- erl_exit(1, "bad header");
+ erts_exit(ERTS_ERROR_EXIT, "bad header");
}
while (i) {
if (is_list(*ptr)) {
@@ -1505,7 +1505,7 @@ make_hash2(Eterm term)
break;
default:
- erl_exit(1, "Invalid tag in make_hash2(0x%X)\n", term);
+ erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term);
}
}
break;
@@ -1536,7 +1536,7 @@ make_hash2(Eterm term)
UINT32_HASH(NIL_DEF, HCONST_2);
goto hash2_common;
default:
- erl_exit(1, "Invalid tag in make_hash2(0x%X)\n", term);
+ erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term);
}
case _TAG_IMMED1_SMALL:
{
@@ -1552,7 +1552,7 @@ make_hash2(Eterm term)
}
break;
default:
- erl_exit(1, "Invalid tag in make_hash2(0x%X)\n", term);
+ erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term);
hash2_common:
/* Uint32 hash always has the hash value of the previous term,
@@ -1747,7 +1747,7 @@ make_internal_hash(Eterm term)
i = hashmap_bitcount(MAP_HEADER_VAL(hdr));
break;
default:
- erl_exit(1, "bad header");
+ erts_exit(ERTS_ERROR_EXIT, "bad header");
}
while (i) {
if (is_list(*ptr)) {
@@ -1923,7 +1923,7 @@ make_internal_hash(Eterm term)
goto pop_next;
}
default:
- erl_exit(1, "Invalid tag in make_hash2(0x%X)\n", term);
+ erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term);
}
}
break;
@@ -1936,7 +1936,7 @@ make_internal_hash(Eterm term)
goto pop_next;
default:
- erl_exit(1, "Invalid tag in make_hash2(0x%X)\n", term);
+ erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term);
pop_next:
if (ESTACK_ISEMPTY(s)) {
@@ -2219,7 +2219,7 @@ tail_recur:
}
default:
- erl_exit(1, "Invalid tag in make_broken_hash\n");
+ erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_broken_hash\n");
return 0;
}
if (WSTACK_ISEMPTY(stack)) break;
@@ -2893,7 +2893,7 @@ tailrecur_ne:
ASSERT(sz > 0 && sz < 17);
break;
default:
- erl_exit(1, "Unknown hashmap subsubtag\n");
+ erts_exit(ERTS_ERROR_EXIT, "Unknown hashmap subsubtag\n");
}
goto term_array;
}
diff --git a/erts/emulator/drivers/common/efile_drv.c b/erts/emulator/drivers/common/efile_drv.c
index c5c780dce5..3088dfd572 100644
--- a/erts/emulator/drivers/common/efile_drv.c
+++ b/erts/emulator/drivers/common/efile_drv.c
@@ -119,8 +119,6 @@
#include "dtrace-wrapper.h"
-void erl_exit(int n, char *fmt, ...);
-
static ErlDrvSysInfo sys_info;
/* For explanation of this var, see comment for same var in erl_async.c */
@@ -509,21 +507,10 @@ struct t_data
static void *ef_safe_alloc(Uint s)
{
void *p = EF_ALLOC(s);
- if (!p) erl_exit(1, "efile drv: Can't allocate %lu bytes of memory\n", (unsigned long)s);
- return p;
-}
-
-#if 0 /* Currently not used */
-
-static void *ef_safe_realloc(void *op, Uint s)
-{
- void *p = EF_REALLOC(op, s);
- if (!p) erl_exit(1, "efile drv: Can't reallocate %lu bytes of memory\n", (unsigned long)s);
+ if (!p) erts_exit(ERTS_ERROR_EXIT, "efile drv: Can't allocate %lu bytes of memory\n", (unsigned long)s);
return p;
}
-#endif
-
/*********************************************************************
* ErlIOVec manipulation functions.
*/
diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c
index f910b5955a..b219669ce5 100644
--- a/erts/emulator/drivers/common/inet_drv.c
+++ b/erts/emulator/drivers/common/inet_drv.c
@@ -1345,7 +1345,7 @@ static const struct in6_addr in6addr_loopback =
#endif /* HAVE_IN6 */
/* XXX: is this a driver interface function ??? */
-void erl_exit(int n, char*, ...);
+void erts_exit(int n, char*, ...);
/*
* Malloc wrapper,
@@ -1358,7 +1358,7 @@ void erl_exit(int n, char*, ...);
static void *alloc_wrapper(ErlDrvSizeT size){
void *ret = driver_alloc(size);
if(ret == NULL)
- erl_exit(1,"Out of virtual memory in malloc (%s)", __FILE__);
+ erts_exit(ERTS_ERROR_EXIT,"Out of virtual memory in malloc (%s)", __FILE__);
return ret;
}
#define ALLOC(X) alloc_wrapper(X)
@@ -1366,7 +1366,7 @@ static void *alloc_wrapper(ErlDrvSizeT size){
static void *realloc_wrapper(void *current, ErlDrvSizeT size){
void *ret = driver_realloc(current,size);
if(ret == NULL)
- erl_exit(1,"Out of virtual memory in realloc (%s)", __FILE__);
+ erts_exit(ERTS_ERROR_EXIT,"Out of virtual memory in realloc (%s)", __FILE__);
return ret;
}
#define REALLOC(X,Y) realloc_wrapper(X,Y)
@@ -1627,7 +1627,7 @@ check_double_release(InetDrvBufStk *bs, ErlDrvBinary* buf)
int i;
for (i = 0; i < bs->buf.pos; ++i) {
if (bs->buf.stk[i] == buf) {
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"Multiple buffer release in inet_drv, this "
"is a bug, save the core and send it to "
@@ -6803,7 +6803,7 @@ static ErlDrvSSizeT inet_fill_opts(inet_descriptor* desc,
do { \
ErlDrvSizeT new_need = ((Ptr) - (*dest)) + (Size); \
if (new_need > dest_used) { \
- erl_exit(1,"Internal error in inet_drv, " \
+ erts_exit(ERTS_ERROR_EXIT,"Internal error in inet_drv, " \
"miscalculated buffer size"); \
} \
dest_used = new_need; \
@@ -7180,7 +7180,7 @@ static ErlDrvSSizeT sctp_fill_opts(inet_descriptor* desc,
do { \
int need; \
if ((Index) > spec_allocated) { \
- erl_exit(1,"Internal error in inet_drv, " \
+ erts_exit(ERTS_ERROR_EXIT,"Internal error in inet_drv, " \
"miscalculated buffer size"); \
} \
need = (Index) + (N); \
diff --git a/erts/emulator/drivers/win32/win_con.c b/erts/emulator/drivers/win32/win_con.c
index 0d63d46698..7fe708dc7b 100644
--- a/erts/emulator/drivers/win32/win_con.c
+++ b/erts/emulator/drivers/win32/win_con.c
@@ -279,7 +279,7 @@ ConInit(void)
}
/*
- ConNormalExit() is called from erl_exit() when the emulator
+ ConNormalExit() is called from erts_exit() when the emulator
is stopping. If the exit has not been initiated by this
console thread (WM_DESTROY or ID_BREAK), the function must
invoke the console thread to save the user preferences.
@@ -529,7 +529,7 @@ ConThreadInit(LPVOID param)
/*
PostQuitMessage() results in WM_QUIT which makes GetMessage()
return 0 (which stops the main loop). Before we return from
- the console thread, the ctrl_handler is called to do erl_exit.
+ the console thread, the ctrl_handler is called to do erts_exit.
*/
(*ctrl_handler)(CTRL_CLOSE_EVENT);
return msg.wParam;
diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c
index 6f495b8825..70354aa9af 100644
--- a/erts/emulator/hipe/hipe_bif0.c
+++ b/erts/emulator/hipe/hipe_bif0.c
@@ -406,7 +406,7 @@ BIF_RETTYPE hipe_bifs_enter_code_2(BIF_ALIST_2)
nrcallees = arityval(tuple_val(BIF_ARG_2)[0]);
else
nrcallees = 0;
- erl_exit(1, "%s: failed to allocate %lu bytes and %lu trampolines\r\n",
+ erts_exit(ERTS_ERROR_EXIT, "%s: failed to allocate %lu bytes and %lu trampolines\r\n",
__func__, (unsigned long)nrbytes, (unsigned long)nrcallees);
}
memcpy(address, bytes, nrbytes);
@@ -1322,7 +1322,7 @@ static void *hipe_make_stub(Eterm m, Eterm f, unsigned int arity, int is_remote)
export_entry = erts_export_get_or_make_stub(m, f, arity);
StubAddress = hipe_make_native_stub(export_entry, arity);
if (!StubAddress)
- erl_exit(1, "hipe_make_stub: code allocation failed\r\n");
+ erts_exit(ERTS_ERROR_EXIT, "hipe_make_stub: code allocation failed\r\n");
return StubAddress;
}
diff --git a/erts/emulator/hipe/hipe_mode_switch.c b/erts/emulator/hipe/hipe_mode_switch.c
index 976180cc30..804c458b6c 100644
--- a/erts/emulator/hipe/hipe_mode_switch.c
+++ b/erts/emulator/hipe/hipe_mode_switch.c
@@ -108,7 +108,7 @@ static const char *code_str(unsigned code)
static void __noreturn
hipe_abort(const char *expr, const char *file, unsigned line)
{
- erl_exit(1, "ASSERTION FAILED, file %s, line %u: %s\r\n", file, line, expr);
+ erts_exit(ERTS_ERROR_EXIT, "ASSERTION FAILED, file %s, line %u: %s\r\n", file, line, expr);
}
#define HIPE_ASSERT3(expr, file, line) \
@@ -316,7 +316,7 @@ Process *hipe_mode_switch(Process *p, unsigned cmd, Eterm reg[])
break;
}
default:
- erl_exit(1, "hipe_mode_switch: cmd %#x\r\n", cmd);
+ erts_exit(ERTS_ERROR_EXIT, "hipe_mode_switch: cmd %#x\r\n", cmd);
}
do_return_from_native:
DPRINTF("result == %#x (%s)", result, code_str(result));
@@ -560,7 +560,7 @@ Process *hipe_mode_switch(Process *p, unsigned cmd, Eterm reg[])
goto do_throw_to_native;
}
default:
- erl_exit(1, "hipe_mode_switch: result %#x\r\n", result);
+ erts_exit(ERTS_ERROR_EXIT, "hipe_mode_switch: result %#x\r\n", result);
}
HIPE_CHECK_PCB(p);
p->def_arg_reg[3] = result;
diff --git a/erts/emulator/hipe/hipe_native_bif.c b/erts/emulator/hipe/hipe_native_bif.c
index 16cc70234d..0ec417999b 100644
--- a/erts/emulator/hipe/hipe_native_bif.c
+++ b/erts/emulator/hipe/hipe_native_bif.c
@@ -180,7 +180,7 @@ void hipe_fclearerror_error(Process *p)
#if !defined(NO_FPE_SIGNALS)
erts_fp_check_init_error(&p->fp_exception);
#else
- erl_exit(ERTS_ABORT_EXIT, "Emulated FPE not cleared by HiPE");
+ erts_exit(ERTS_ABORT_EXIT, "Emulated FPE not cleared by HiPE");
#endif
}
diff --git a/erts/emulator/sys/common/erl_mmap.c b/erts/emulator/sys/common/erl_mmap.c
index 03ca080c14..889d6f3868 100644
--- a/erts/emulator/sys/common/erl_mmap.c
+++ b/erts/emulator/sys/common/erl_mmap.c
@@ -1341,7 +1341,7 @@ os_unreserve_physical(char *ptr, UWord size)
void *res = mmap((void *) ptr, (size_t) size, ERTS_MMAP_UNRESERVE_PROT,
ERTS_MMAP_UNRESERVE_FLAGS, ERTS_MMAP_FD, 0);
if (res == (void *) MAP_FAILED)
- erl_exit(ERTS_ABORT_EXIT, "Failed to unreserve memory");
+ erts_exit(ERTS_ABORT_EXIT, "Failed to unreserve memory");
}
static void *
@@ -2136,7 +2136,7 @@ erts_mmap_init(ErtsMemMapper* mm, ErtsMMapInit *init)
#endif
erts_page_inv_mask = pagesize - 1;
if (pagesize & erts_page_inv_mask)
- erl_exit(-1, "erts_mmap: Invalid pagesize: %bpu\n",
+ erts_exit(1, "erts_mmap: Invalid pagesize: %bpu\n",
pagesize);
ERTS_MMAP_OP_RINGBUF_INIT();
@@ -2148,7 +2148,7 @@ erts_mmap_init(ErtsMemMapper* mm, ErtsMMapInit *init)
#if HAVE_MMAP && !defined(MAP_ANON)
mm->mmap_fd = open("/dev/zero", O_RDWR);
if (mm->mmap_fd < 0)
- erl_exit(-1, "erts_mmap: Failed to open /dev/zero\n");
+ erts_exit(1, "erts_mmap: Failed to open /dev/zero\n");
#endif
erts_smp_mtx_init(&mm->mtx, "erts_mmap");
@@ -2165,7 +2165,7 @@ erts_mmap_init(ErtsMemMapper* mm, ErtsMMapInit *init)
sz = end - ptr;
start = os_mmap_virtual(ptr, sz);
if (!start || start > ptr || start >= end)
- erl_exit(-1,
+ erts_exit(1,
"erts_mmap: Failed to create virtual range for super carrier\n");
sz = start - ptr;
if (sz)
@@ -2203,7 +2203,7 @@ erts_mmap_init(ErtsMemMapper* mm, ErtsMMapInit *init)
start = os_mmap(NULL, sz, 1);
}
if (!start)
- erl_exit(-1,
+ erts_exit(1,
"erts_mmap: Failed to create super carrier of size %bpu MB\n",
init->scs/1024/1024);
end = start + sz;
@@ -2250,7 +2250,7 @@ erts_mmap_init(ErtsMemMapper* mm, ErtsMMapInit *init)
if ((desc_size
+ ERTS_SUPERALIGNED_SIZE
+ ERTS_PAGEALIGNED_SIZE) > end - start)
- erl_exit(-1, "erts_mmap: No space for segments in super carrier\n");
+ erts_exit(1, "erts_mmap: No space for segments in super carrier\n");
mm->sa.bot = start;
mm->sa.bot += desc_size;
@@ -2288,7 +2288,7 @@ erts_mmap_init(ErtsMemMapper* mm, ErtsMMapInit *init)
*/
#ifdef ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION
if (virtual_map && !os_reserve_physical(start, mm->sa.bot - start))
- erl_exit(-1, "erts_mmap: Failed to reserve physical memory for descriptors\n");
+ erts_exit(1, "erts_mmap: Failed to reserve physical memory for descriptors\n");
#endif
mm->desc.unused_start = start;
mm->desc.unused_end = mm->sa.bot;
diff --git a/erts/emulator/sys/common/erl_mmap.h b/erts/emulator/sys/common/erl_mmap.h
index 61d912fd28..67e131b53e 100644
--- a/erts/emulator/sys/common/erl_mmap.h
+++ b/erts/emulator/sys/common/erl_mmap.h
@@ -50,8 +50,10 @@ typedef struct {
#define ERTS_MMAP_INIT_DEFAULT_INITER \
{{NULL, NULL}, {NULL, NULL}, 0, 1, (1 << 16), 1}
+#define ERTS_LITERAL_VIRTUAL_AREA_SIZE (UWORD_CONSTANT(1)*1024*1024*1024)
+
#define ERTS_MMAP_INIT_LITERAL_INITER \
- {{NULL, NULL}, {NULL, NULL}, 1024*1024*1024, 1, (1 << 16), 0}
+ {{NULL, NULL}, {NULL, NULL}, ERTS_LITERAL_VIRTUAL_AREA_SIZE, 1, (1 << 16), 0}
typedef struct ErtsMemMapper_ ErtsMemMapper;
diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c
index 20695899eb..286130bece 100644
--- a/erts/emulator/sys/common/erl_mseg.c
+++ b/erts/emulator/sys/common/erl_mseg.c
@@ -1403,9 +1403,12 @@ erts_mseg_init(ErtsMsegInit_t *init)
erts_mtx_init(&init_atoms_mutex, "mseg_init_atoms");
erts_mmap_init(&erts_dflt_mmapper, &init->dflt_mmap);
+#if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
+ erts_mmap_init(&erts_literal_mmapper, &init->literal_mmap);
+#endif
if (!IS_2POW(GET_PAGE_SIZE))
- erl_exit(ERTS_ABORT_EXIT, "erts_mseg: Unexpected page_size %beu\n", GET_PAGE_SIZE);
+ erts_exit(ERTS_ABORT_EXIT, "erts_mseg: Unexpected page_size %beu\n", GET_PAGE_SIZE);
ASSERT((MSEG_ALIGNED_SIZE % GET_PAGE_SIZE) == 0);
diff --git a/erts/emulator/sys/common/erl_os_monotonic_time_extender.c b/erts/emulator/sys/common/erl_os_monotonic_time_extender.c
index b79485241f..367b22d989 100644
--- a/erts/emulator/sys/common/erl_os_monotonic_time_extender.c
+++ b/erts/emulator/sys/common/erl_os_monotonic_time_extender.c
@@ -44,7 +44,7 @@ static void *os_monotonic_time_extender(void *vstatep)
erts_milli_sleep(sleep_time);
}
- erl_exit(ERTS_ABORT_EXIT, "os_monotonic_time_extender thread terminating");
+ erts_exit(ERTS_ABORT_EXIT, "os_monotonic_time_extender thread terminating");
return NULL;
}
diff --git a/erts/emulator/sys/unix/erl_child_setup.c b/erts/emulator/sys/unix/erl_child_setup.c
index 4e61530cf1..8dd14bbac6 100644
--- a/erts/emulator/sys/unix/erl_child_setup.c
+++ b/erts/emulator/sys/unix/erl_child_setup.c
@@ -61,6 +61,7 @@
#include "erl_driver.h"
#include "sys_uds.h"
#include "hash.h"
+#include "erl_term.h"
#include "erl_child_setup.h"
#define SET_CLOEXEC(fd) fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | FD_CLOEXEC)
diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c
index 2ad5f3b4d5..0a18e54389 100644
--- a/erts/emulator/sys/unix/sys.c
+++ b/erts/emulator/sys/unix/sys.c
@@ -631,7 +631,7 @@ break_requested(void)
fprintf(stderr,"break!\n");
#endif
if (ERTS_BREAK_REQUESTED)
- erl_exit(ERTS_INTR_EXIT, "");
+ erts_exit(ERTS_INTR_EXIT, "");
ERTS_SET_BREAK_REQUESTED;
ERTS_CHK_IO_AS_INTR(); /* Make sure we don't sleep in poll */
@@ -670,7 +670,7 @@ sigusr1_exit(void)
}
prepare_crash_dump(secs);
- erl_exit(1, "Received SIGUSR1\n");
+ erts_exit(ERTS_ERROR_EXIT, "Received SIGUSR1\n");
}
#ifdef ETHR_UNUSABLE_SIGUSRX
@@ -726,7 +726,7 @@ static RETSIGTYPE suspend_signal(int signum)
static void
quit_requested(void)
{
- erl_exit(ERTS_INTR_EXIT, "");
+ erts_exit(ERTS_INTR_EXIT, "");
}
#if (defined(SIG_SIGSET) || defined(SIG_SIGNAL))
@@ -1252,7 +1252,7 @@ signal_dispatcher_thread_func(void *unused)
if (res < 0) {
if (errno == EINTR)
continue;
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"signal-dispatcher thread got unexpected error: %s (%d)\n",
erl_errno_id(errno),
errno);
@@ -1289,7 +1289,7 @@ signal_dispatcher_thread_func(void *unused)
sigusr1_exit();
break;
default:
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"signal-dispatcher thread received unknown "
"signal notification: '%c'\n",
buf[i]);
@@ -1308,7 +1308,7 @@ init_smp_sig_notify(void)
thr_opts.name = "sys_sig_dispatcher";
if (pipe(sig_notify_fds) < 0) {
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"Failed to create signal-dispatcher pipe: %s (%d)\n",
erl_errno_id(errno),
errno);
@@ -1324,7 +1324,7 @@ init_smp_sig_notify(void)
static void
init_smp_sig_suspend(void) {
if (pipe(sig_suspend_fds) < 0) {
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"Failed to create sig_suspend pipe: %s (%d)\n",
erl_errno_id(errno),
errno);
@@ -1340,7 +1340,7 @@ static void initialize_darwin_main_thread_pipes(void)
{
if (pipe(erts_darwin_main_thread_pipe) < 0 ||
pipe(erts_darwin_main_thread_result_pipe) < 0) {
- erl_exit(1,"Fatal error initializing Darwin main thread stealing");
+ erts_exit(ERTS_ERROR_EXIT,"Fatal error initializing Darwin main thread stealing");
}
}
diff --git a/erts/emulator/sys/unix/sys_drivers.c b/erts/emulator/sys/unix/sys_drivers.c
index 2a7cd91265..82fea1b330 100644
--- a/erts/emulator/sys/unix/sys_drivers.c
+++ b/erts/emulator/sys/unix/sys_drivers.c
@@ -1665,14 +1665,14 @@ static ErlDrvData forker_start(ErlDrvPort port_num, char* name,
res = erts_sys_getenv_raw("BINDIR", bindir, &bindirsz);
if (res != 0) {
if (res < 0)
- erl_exit(-1,
+ erts_exit(1,
"Environment variable BINDIR is not set\n");
if (res > 0)
- erl_exit(-1,
+ erts_exit(1,
"Value of environment variable BINDIR is too large\n");
}
if (bindir[0] != DIR_SEPARATOR_CHAR)
- erl_exit(-1,
+ erts_exit(1,
"Environment variable BINDIR does not contain an"
" absolute path\n");
csp_path_sz = (strlen(bindir)
@@ -1686,7 +1686,7 @@ static ErlDrvData forker_start(ErlDrvPort port_num, char* name,
DIR_SEPARATOR_CHAR,
CHILD_SETUP_PROG_NAME);
if (socketpair(AF_UNIX, SOCK_STREAM, 0, fds) < 0) {
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"Could not open unix domain socket in spawn_init: %d\n",
errno);
}
@@ -1754,11 +1754,11 @@ static void forker_ready_input(ErlDrvData e, ErlDrvEvent fd)
if ((res = read(fd, proto, sizeof(*proto))) < 0) {
if (errno == ERRNO_BLOCK)
return;
- erl_exit(ERTS_DUMP_EXIT, "Failed to read from erl_child_setup: %d\n", errno);
+ erts_exit(ERTS_DUMP_EXIT, "Failed to read from erl_child_setup: %d\n", errno);
}
if (res == 0)
- erl_exit(ERTS_DUMP_EXIT, "erl_child_setup closed\n");
+ erts_exit(ERTS_DUMP_EXIT, "erl_child_setup closed\n");
ASSERT(res == sizeof(*proto));
@@ -1814,7 +1814,7 @@ static void forker_ready_output(ErlDrvData e, ErlDrvEvent fd)
proto->u.start.fds, 3, 0) < 0) {
if (errno == ERRNO_BLOCK)
return;
- erl_exit(ERTS_DUMP_EXIT, "Failed to write to erl_child_setup: %d\n", errno);
+ erts_exit(ERTS_DUMP_EXIT, "Failed to write to erl_child_setup: %d\n", errno);
}
#ifndef FORKER_PROTO_START_ACK
close(proto->u.start.fds[0]);
@@ -1846,7 +1846,7 @@ static ErlDrvSSizeT forker_control(ErlDrvData e, unsigned int cmd, char *buf,
driver_select(port_num, forker_fd, ERL_DRV_WRITE|ERL_DRV_USE, 1);
return 0;
}
- erl_exit(ERTS_DUMP_EXIT, "Failed to write to erl_child_setup: %d\n", errno);
+ erts_exit(ERTS_DUMP_EXIT, "Failed to write to erl_child_setup: %d\n", errno);
}
#ifndef FORKER_PROTO_START_ACK
diff --git a/erts/emulator/sys/unix/sys_float.c b/erts/emulator/sys/unix/sys_float.c
index 1ef9e5eef7..8fe7e599e5 100644
--- a/erts/emulator/sys/unix/sys_float.c
+++ b/erts/emulator/sys/unix/sys_float.c
@@ -90,7 +90,7 @@ void erts_fp_check_init_error(volatile unsigned long *fpexnp)
snprintf(buf, sizeof buf, "ERTS_FP_CHECK_INIT at %p: detected unhandled FPE at %p\r\n",
__builtin_return_address(0), (void*)*fpexnp);
if (write(2, buf, strlen(buf)) <= 0)
- erl_exit(ERTS_ABORT_EXIT, "%s", buf);
+ erts_exit(ERTS_ABORT_EXIT, "%s", buf);
*fpexnp = 0;
#if defined(__i386__) || defined(__x86_64__)
erts_restore_fpu();
diff --git a/erts/emulator/sys/unix/sys_time.c b/erts/emulator/sys/unix/sys_time.c
index 03d39c7ce6..8ce02506ab 100644
--- a/erts/emulator/sys/unix/sys_time.c
+++ b/erts/emulator/sys/unix/sys_time.c
@@ -344,7 +344,7 @@ sys_init_time(ErtsSysInitTimeResult *init_resp)
* times() (CLK_TCK), the resolution is always one millisecond..
*/
if ((erts_sys_time_data__.r.o.ticks_per_sec = TICKS_PER_SEC()) < 0)
- erl_exit(ERTS_ABORT_EXIT, "Can't get clock ticks/sec\n");
+ erts_exit(ERTS_ABORT_EXIT, "Can't get clock ticks/sec\n");
#if defined(OS_MONOTONIC_TIME_USING_TIMES)
#if ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT
@@ -454,7 +454,7 @@ posix_clock_gettime(clockid_t id, char *name)
if (clock_gettime(id, &ts) != 0) {
int err = errno;
char *errstr = err ? strerror(err) : "unknown";
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"clock_gettime(%s, _) failed: %s (%d)\n",
name, errstr, err);
}
@@ -499,13 +499,13 @@ posix_clock_gettime_times(clockid_t mid, char *mname,
if (mres != 0) {
char *errstr = merr ? strerror(merr) : "unknown";
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"clock_gettime(%s, _) failed: %s (%d)\n",
mname, errstr, merr);
}
if (sres != 0) {
char *errstr = serr ? strerror(serr) : "unknown";
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"clock_gettime(%s, _) failed: %s (%d)\n",
sname, errstr, serr);
}
@@ -678,7 +678,7 @@ mach_clocks_init(void)
clck_srv_p = &internal_state.r.o.mach.clock.monotonic.srv;
kret = host_get_clock_service(host, id, clck_srv_p);
if (kret != KERN_SUCCESS) {
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"host_get_clock_service(_, %s, _) failed\n",
name);
}
@@ -690,7 +690,7 @@ mach_clocks_init(void)
clck_srv_p = &internal_state.r.o.mach.clock.wall.srv;
kret = host_get_clock_service(host, id, clck_srv_p);
if (kret != KERN_SUCCESS) {
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"host_get_clock_service(_, %s, _) failed\n",
name);
}
@@ -699,7 +699,7 @@ mach_clocks_init(void)
if (atexit(mach_clocks_fini) != 0) {
int err = errno;
char *errstr = err ? strerror(err) : "unknown";
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"Failed to register mach_clocks_fini() "
"for call at exit: %s (%d)\n",
errstr, err);
@@ -721,7 +721,7 @@ mach_clock_getres(ErtsMachClock *clk)
(clock_attr_t) attr,
&cnt);
if (kret != KERN_SUCCESS || cnt != 1) {
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"clock_get_attributes(%s, _) failed\n",
clk->name);
}
@@ -739,7 +739,7 @@ mach_clock_get_time(ErtsMachClock *clk)
kret = clock_get_time(clk->srv, &time_spec);
if (kret != KERN_SUCCESS)
- erl_exit(ERTS_ABORT_EXIT, "clock_get_time(%s, _) failed\n", clk->name);
+ erts_exit(ERTS_ABORT_EXIT, "clock_get_time(%s, _) failed\n", clk->name);
return ERTS_TimeSpec2Sint64(&time_spec);
}
@@ -785,11 +785,11 @@ erts_os_times(ErtsMonotonicTime *mtimep, ErtsSystemTime *stimep)
&sys_time_spec);
if (mkret != KERN_SUCCESS)
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"clock_get_time(%s, _) failed\n",
internal_state.r.o.mach.clock.monotonic.name);
if (skret != KERN_SUCCESS)
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"clock_get_time(%s, _) failed\n",
internal_state.r.o.mach.clock.wall.name);
@@ -854,7 +854,7 @@ erts_os_system_time(void)
if (gettimeofday(&tv, NULL) != 0) {
int err = errno;
char *errstr = err ? strerror(err) : "unknown";
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"gettimeofday(_, NULL) failed: %s (%d)\n",
errstr, err);
}
diff --git a/erts/emulator/sys/win32/erl_poll.c b/erts/emulator/sys/win32/erl_poll.c
index 660ded297a..9c1bef5f4f 100644
--- a/erts/emulator/sys/win32/erl_poll.c
+++ b/erts/emulator/sys/win32/erl_poll.c
@@ -437,7 +437,7 @@ wakeup_cause(ErtsPollSet ps)
break;
default:
res = 0;
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"%s:%d: Internal error: Invalid wakeup_state=%d\n",
__FILE__, __LINE__, (int) wakeup_state);
}
@@ -577,7 +577,7 @@ static void signal_standby(ErtsPollSet ps)
--(ps->standby_wait_counter);
if (ps->standby_wait_counter < 0) {
LeaveCriticalSection(&(ps->standby_crit));
- erl_exit(1,"Standby signalled by more threads than expected");
+ erts_exit(ERTS_ERROR_EXIT,"Standby signalled by more threads than expected");
}
if (!(ps->standby_wait_counter)) {
SetEvent(ps->standby_wait_event);
@@ -739,7 +739,7 @@ static void *break_waiter(void *param)
erts_mtx_unlock(&break_waiter_lock);
break;
default:
- erl_exit(1,"Unexpected event in break_waiter");
+ erts_exit(ERTS_ERROR_EXIT,"Unexpected event in break_waiter");
}
}
}
@@ -1158,7 +1158,7 @@ int erts_poll_wait(ErtsPollSet ps,
HARDDEBUGF(("Oups!"));
/* Oups, got signalled before we took the lock, can't reset */
if(!is_io_ready(ps)) {
- erl_exit(1,"Internal error: "
+ erts_exit(ERTS_ERROR_EXIT,"Internal error: "
"Inconsistent io structures in erl_poll.\n");
}
START_WAITER(ps,w);
@@ -1219,7 +1219,7 @@ int erts_poll_wait(ErtsPollSet ps,
ERTS_SET_BREAK_REQUESTED;
break;
case BREAK_WAITER_GOT_HALT:
- erl_exit(0,"");
+ erts_exit(0,"");
break;
default:
break;
diff --git a/erts/emulator/sys/win32/sys.c b/erts/emulator/sys/win32/sys.c
index 76ce25916a..ea7523ddc2 100644
--- a/erts/emulator/sys/win32/sys.c
+++ b/erts/emulator/sys/win32/sys.c
@@ -38,7 +38,7 @@
void erts_sys_init_float(void);
void erl_start(int, char**);
-void erl_exit(int n, char*, ...);
+void erts_exit(int n, char*, ...);
void erl_error(char*, va_list);
void erl_crash_dump(char*, int, char*, ...);
@@ -200,7 +200,7 @@ erts_sys_misc_mem_sz(void)
*/
void sys_tty_reset(int exit_code)
{
- if (exit_code > 0)
+ if (exit_code == ERTS_ERROR_EXIT)
ConWaitForExit();
else
ConNormalExit();
@@ -3108,13 +3108,13 @@ check_supported_os_version(void)
|| int_os_version.dwMajorVersion < major
|| (int_os_version.dwMajorVersion == major
&& int_os_version.dwMinorVersion < minor))
- erl_exit(-1,
+ erts_exit(1,
"Windows version not supported "
"(min required: winnt %d.%d)\n",
major, minor);
}
#else
- erl_exit(-1,
+ erts_exit(1,
"Windows version not supported "
"(min required: win %d.%d)\n",
nt_major, nt_minor);
diff --git a/erts/emulator/sys/win32/sys_interrupt.c b/erts/emulator/sys/win32/sys_interrupt.c
index d6178de03c..a89211fa8e 100644
--- a/erts/emulator/sys/win32/sys_interrupt.c
+++ b/erts/emulator/sys/win32/sys_interrupt.c
@@ -81,7 +81,7 @@ BOOL WINAPI ctrl_handler_ignore_break(DWORD dwCtrlType)
return TRUE;
/* else pour through... */
case CTRL_CLOSE_EVENT:
- erl_exit(0, "");
+ erts_exit(0, "");
break;
}
return TRUE;
@@ -106,7 +106,7 @@ BOOL WINAPI ctrl_handler_replace_intr(DWORD dwCtrlType)
/* else pour through... */
case CTRL_CLOSE_EVENT:
case CTRL_SHUTDOWN_EVENT:
- erl_exit(0, "");
+ erts_exit(0, "");
break;
}
return TRUE;
@@ -133,7 +133,7 @@ BOOL WINAPI ctrl_handler(DWORD dwCtrlType)
return TRUE;
/* else pour through... */
case CTRL_CLOSE_EVENT:
- erl_exit(0, "");
+ erts_exit(0, "");
break;
}
return TRUE;
diff --git a/erts/emulator/sys/win32/sys_time.c b/erts/emulator/sys/win32/sys_time.c
index 3b4fd26d63..9a9d90610d 100644
--- a/erts/emulator/sys/win32/sys_time.c
+++ b/erts/emulator/sys/win32/sys_time.c
@@ -147,7 +147,7 @@ os_monotonic_time_qpc(void)
LARGE_INTEGER pc;
if (!(*internal_state.r.o.pQueryPerformanceCounter)(&pc))
- erl_exit(ERTS_ABORT_EXIT, "QueryPerformanceCounter() failed\n");
+ erts_exit(ERTS_ABORT_EXIT, "QueryPerformanceCounter() failed\n");
return (ErtsMonotonicTime) pc.QuadPart;
}
@@ -164,7 +164,7 @@ os_times_qpc(ErtsMonotonicTime *mtimep, ErtsSystemTime *stimep)
GetSystemTime(&st);
if (!qpcr)
- erl_exit(ERTS_ABORT_EXIT, "QueryPerformanceCounter() failed\n");
+ erts_exit(ERTS_ABORT_EXIT, "QueryPerformanceCounter() failed\n");
*mtimep = (ErtsMonotonicTime) pc.QuadPart;
@@ -251,7 +251,7 @@ sys_hrtime_qpc(void)
LARGE_INTEGER pc;
if (!(*internal_state.r.o.pQueryPerformanceCounter)(&pc))
- erl_exit(ERTS_ABORT_EXIT, "QueryPerformanceCounter() failed\n");
+ erts_exit(ERTS_ABORT_EXIT, "QueryPerformanceCounter() failed\n");
ASSERT(pc.QuadPart > 0);
diff --git a/erts/emulator/test/Makefile b/erts/emulator/test/Makefile
index 987a655c3d..318db4b45e 100644
--- a/erts/emulator/test/Makefile
+++ b/erts/emulator/test/Makefile
@@ -74,6 +74,7 @@ MODULES= \
match_spec_SUITE \
module_info_SUITE \
monitor_SUITE \
+ multi_load_SUITE \
nested_SUITE \
nif_SUITE \
node_container_SUITE \
diff --git a/erts/emulator/test/distribution_SUITE.erl b/erts/emulator/test/distribution_SUITE.erl
index 7a33dbbb36..d562f79bb7 100644
--- a/erts/emulator/test/distribution_SUITE.erl
+++ b/erts/emulator/test/distribution_SUITE.erl
@@ -164,7 +164,7 @@ bulk_send(Terms, BinSize) ->
?line stop_node(Node),
?line test_server:timetrap_cancel(Dog),
- {comment, integer_to_list(trunc(Size/1024/Elapsed+0.5)) ++ " K/s"}.
+ {comment, integer_to_list(trunc(Size/1024/max(1,Elapsed)+0.5)) ++ " K/s"}.
bulk_sendsend(Terms, BinSize) ->
{Rate1, MonitorCount1} = bulk_sendsend2(Terms, BinSize, 5),
diff --git a/erts/emulator/test/estone_SUITE.erl b/erts/emulator/test/estone_SUITE.erl
index 7be55eca8d..7710424894 100644
--- a/erts/emulator/test/estone_SUITE.erl
+++ b/erts/emulator/test/estone_SUITE.erl
@@ -382,11 +382,11 @@ apply_micro(M) ->
{weight_percentage, M#micro.weight},
{loops, M#micro.loops},
{microsecs,MicroSecs},
- {estones, (M#micro.weight * M#micro.weight * ?STONEFACTOR) div MicroSecs},
+ {estones, (M#micro.weight * M#micro.weight * ?STONEFACTOR) div max(1,MicroSecs)},
{gcs, GC1 - GC0},
{kilo_word_reclaimed, (Words1 - Words0) div 1000},
{kilo_reductions, Reds div 1000},
- {gc_intensity, gci(Elapsed, GC1 - GC0, Words1 - Words0)}].
+ {gc_intensity, gci(max(1,Elapsed), GC1 - GC0, Words1 - Words0)}].
monotonic_time() ->
try erlang:monotonic_time() catch error:undef -> erlang:now() end.
diff --git a/erts/emulator/test/multi_load_SUITE.erl b/erts/emulator/test/multi_load_SUITE.erl
new file mode 100644
index 0000000000..784b239116
--- /dev/null
+++ b/erts/emulator/test/multi_load_SUITE.erl
@@ -0,0 +1,196 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 1999-2012. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(multi_load_SUITE).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1,
+ init_per_group/2,end_per_group/2,
+ many/1,on_load/1,errors/1]).
+
+-include_lib("common_test/include/ct.hrl").
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() ->
+ [many,on_load,errors].
+
+groups() ->
+ [].
+
+init_per_suite(Config) ->
+ Config.
+
+end_per_suite(_Config) ->
+ ok.
+
+init_per_group(_GroupName, Config) ->
+ Config.
+
+end_per_group(_GroupName, Config) ->
+ Config.
+
+many(_Config) ->
+ Ms = make_modules(100, fun many_module/1),
+
+ io:put_chars("Light load\n"
+ "=========="),
+ many_measure(Ms),
+
+ _ = [spawn_link(fun many_worker/0) || _ <- lists:seq(1, 8)],
+ erlang:yield(),
+ io:put_chars("Heavy load\n"
+ "=========="),
+ many_measure(Ms),
+
+ ok.
+
+many_module(M) ->
+ ["-module("++M++").",
+ "-compile(export_all).",
+ "f1() -> ok.",
+ "f2() -> ok.",
+ "f3() -> ok.",
+ "f4() -> ok."].
+
+many_measure(Ms) ->
+ many_purge(Ms),
+ MsPrep1 = prepare_modules(Ms),
+ Us1 = ms(fun() -> many_load_seq(MsPrep1) end),
+ many_try_call(Ms),
+ many_purge(Ms),
+ MsPrep2 = prepare_modules(Ms),
+ Us2 = ms(fun() -> many_load_par(MsPrep2) end),
+ many_try_call(Ms),
+ io:format("# modules: ~9w\n"
+ "Sequential: ~9w µs\n"
+ "Parallel: ~9w µs\n"
+ "Ratio: ~9w\n",
+ [length(Ms),Us1,Us2,round(Us1/Us2)]),
+ ok.
+
+many_load_seq(Ms) ->
+ [erlang:finish_loading([M]) || M <- Ms],
+ ok.
+
+many_load_par(Ms) ->
+ erlang:finish_loading(Ms).
+
+many_purge(Ms) ->
+ _ = [catch erlang:purge_module(M) || {M,_} <- Ms],
+ ok.
+
+many_try_call(Ms) ->
+ _ = [begin
+ ok = M:f1(),
+ ok = M:f2(),
+ ok = M:f3(),
+ ok = M:f4()
+ end || {M,_} <- Ms],
+ ok.
+
+many_worker() ->
+ many_worker(lists:seq(1, 100)).
+
+many_worker(L) ->
+ N0 = length(L),
+ N1 = N0 * N0 * N0,
+ N2 = N1 div (N0 * N0),
+ N3 = N2 + 10,
+ _ = N3 - 10,
+ many_worker(L).
+
+
+on_load(_Config) ->
+ On = make_modules(2, fun on_load_module/1),
+ OnPrep = prepare_modules(On),
+ {'EXIT',{system_limit,_}} = (catch erlang:finish_loading(OnPrep)),
+
+ Normal = make_modules(1, fun on_load_normal/1),
+ Mixed = Normal ++ tl(On),
+ MixedPrep = prepare_modules(Mixed),
+ {'EXIT',{system_limit,_}} = (catch erlang:finish_loading(MixedPrep)),
+
+ [false,true] = [erlang:has_prepared_code_on_load(Code) ||
+ Code <- MixedPrep],
+ {'EXIT',{badarg,_}} = (catch erlang:has_prepared_code_on_load(<<1,2,3>>)),
+ Magic = ets:match_spec_compile([{'_',[true],['$_']}]),
+ {'EXIT',{badarg,_}} = (catch erlang:has_prepared_code_on_load(Magic)),
+
+ SingleOnPrep = tl(OnPrep),
+ {on_load,[OnLoadMod]} = erlang:finish_loading(SingleOnPrep),
+ ok = erlang:call_on_load_function(OnLoadMod),
+
+ ok.
+
+on_load_module(M) ->
+ ["-module("++M++").",
+ "-on_load(f/0).",
+ "f() -> ok."].
+
+on_load_normal(M) ->
+ ["-module("++M++")."].
+
+
+errors(_Config) ->
+ finish_loading_badarg(x),
+ finish_loading_badarg([x|y]),
+ finish_loading_badarg([x]),
+ finish_loading_badarg([<<>>]),
+
+ Mods = make_modules(2, fun errors_module/1),
+ Ms = lists:sort([M || {M,_} <- Mods]),
+ Prep = prepare_modules(Mods),
+ {duplicated,Dups} = erlang:finish_loading(Prep ++ Prep),
+ Ms = lists:sort(Dups),
+ ok.
+
+finish_loading_badarg(Arg) ->
+ {'EXIT',{badarg,[{erlang,finish_loading,[Arg],_}|_]}} =
+ (catch erlang:finish_loading(Arg)).
+
+errors_module(M) ->
+ ["-module("++M++").",
+ "-export([f/0]).",
+ "f() -> ok."].
+
+%%%
+%%% Common utilities
+%%%
+
+ms(Fun) ->
+ {Ms,ok} = timer:tc(Fun),
+ Ms.
+
+make_modules(0, _) ->
+ [];
+make_modules(N, Fun) ->
+ U = erlang:unique_integer([positive]),
+ M0 = "m__" ++ integer_to_list(N) ++ "_" ++ integer_to_list(U),
+ Contents = Fun(M0),
+ Forms = [make_form(S) || S <- Contents],
+ {ok,M,Code} = compile:forms(Forms),
+ [{M,Code}|make_modules(N-1, Fun)].
+
+make_form(S) ->
+ {ok,Toks,_} = erl_scan:string(S),
+ {ok,Form} = erl_parse:parse_form(Toks),
+ Form.
+
+prepare_modules(Ms) ->
+ [erlang:prepare_loading(M, Code) || {M,Code} <- Ms].
diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl
index 7ce37b04b3..c6c617f97f 100644
--- a/erts/emulator/test/nif_SUITE.erl
+++ b/erts/emulator/test/nif_SUITE.erl
@@ -1711,7 +1711,9 @@ tmpmem() ->
false -> undefined;
MemInfo ->
MSBCS = lists:foldl(
- fun ({instance, _, L}, Acc) ->
+ fun ({instance, 0, _}, Acc) ->
+ Acc; % Ignore instance 0
+ ({instance, _, L}, Acc) ->
{value,{_,MBCS}} = lists:keysearch(mbcs, 1, L),
{value,{_,SBCS}} = lists:keysearch(sbcs, 1, L),
[MBCS,SBCS | Acc]
diff --git a/erts/emulator/test/scheduler_SUITE.erl b/erts/emulator/test/scheduler_SUITE.erl
index 1f284228db..f998e7d4d2 100644
--- a/erts/emulator/test/scheduler_SUITE.erl
+++ b/erts/emulator/test/scheduler_SUITE.erl
@@ -54,6 +54,7 @@
sct_cmd/1,
sbt_cmd/1,
scheduler_threads/1,
+ scheduler_suspend_basic/1,
scheduler_suspend/1,
dirty_scheduler_threads/1,
dirty_scheduler_exit/1,
@@ -69,8 +70,10 @@ all() ->
[equal, few_low, many_low, equal_with_part_time_high,
equal_with_part_time_max,
equal_and_high_with_part_time_max, equal_with_high,
- equal_with_high_max, bound_process,
- {group, scheduler_bind}, scheduler_threads, scheduler_suspend,
+ equal_with_high_max,
+ bound_process,
+ {group, scheduler_bind}, scheduler_threads,
+ scheduler_suspend_basic, scheduler_suspend,
dirty_scheduler_threads, dirty_scheduler_exit,
reader_groups].
@@ -1131,6 +1134,7 @@ dirty_schedulers_online_test(true) ->
dirty_schedulers_online_smp_test(erlang:system_info(schedulers_online)).
dirty_schedulers_online_smp_test(SchedOnln) when SchedOnln < 4 -> ok;
dirty_schedulers_online_smp_test(SchedOnln) ->
+ receive after 500 -> ok end,
DirtyCPUSchedOnln = erlang:system_info(dirty_cpu_schedulers_online),
SchedOnln = DirtyCPUSchedOnln,
HalfSchedOnln = SchedOnln div 2,
@@ -1139,9 +1143,11 @@ dirty_schedulers_online_smp_test(SchedOnln) ->
HalfDirtyCPUSchedOnln = erlang:system_flag(schedulers_online, SchedOnln),
DirtyCPUSchedOnln = erlang:system_flag(dirty_cpu_schedulers_online,
HalfDirtyCPUSchedOnln),
+ receive after 500 -> ok end,
HalfDirtyCPUSchedOnln = erlang:system_info(dirty_cpu_schedulers_online),
QrtrDirtyCPUSchedOnln = HalfDirtyCPUSchedOnln div 2,
SchedOnln = erlang:system_flag(schedulers_online, HalfSchedOnln),
+ receive after 500 -> ok end,
QrtrDirtyCPUSchedOnln = erlang:system_info(dirty_cpu_schedulers_online),
ok.
@@ -1214,6 +1220,113 @@ wait_dse([Pid|Pids]) ->
dirty_sleeper() ->
erlang:nif_error({error,?MODULE}).
+scheduler_suspend_basic(Config) when is_list(Config) ->
+ case erlang:system_info(multi_scheduling) of
+ disabled ->
+ {skip, "Nothing to test"};
+ _ ->
+ Onln = erlang:system_info(schedulers_online),
+ try
+ scheduler_suspend_basic_test()
+ after
+ erlang:system_flag(schedulers_online, Onln)
+ end
+ end.
+
+scheduler_suspend_basic_test() ->
+ %% The receives after setting scheduler states are there
+ %% since the operation is not fully synchronous. For example,
+ %% we do not wait for dirty cpu schedulers online to complete
+ %% before returning from erlang:system_flag(schedulers_online, _).
+
+ erlang:system_flag(schedulers_online,
+ erlang:system_info(schedulers)),
+ try
+ erlang:system_flag(dirty_cpu_schedulers_online,
+ erlang:system_info(dirty_cpu_schedulers)),
+ receive after 500 -> ok end
+ catch
+ _ : _ ->
+ ok
+ end,
+
+ S0 = sched_state(),
+ io:format("~p~n", [S0]),
+ {{normal,NTot0,NOnln0,NAct0},
+ {dirty_cpu,DCTot0,DCOnln0,DCAct0},
+ {dirty_io,DITot0,DIOnln0,DIAct0}} = S0,
+ enabled = erlang:system_info(multi_scheduling),
+
+ DCOne = case DCTot0 of
+ 0 -> 0;
+ _ -> 1
+ end,
+
+ blocked_normal = erlang:system_flag(multi_scheduling, block_normal),
+ blocked_normal = erlang:system_info(multi_scheduling),
+ {{normal,NTot0,NOnln0,1},
+ {dirty_cpu,DCTot0,DCOnln0,DCAct0},
+ {dirty_io,DITot0,DIOnln0,DIAct0}} = sched_state(),
+
+ NOnln0 = erlang:system_flag(schedulers_online, 1),
+ receive after 500 -> ok end,
+ {{normal,NTot0,1,1},
+ {dirty_cpu,DCTot0,DCOne,DCOne},
+ {dirty_io,DITot0,DIOnln0,DIAct0}} = sched_state(),
+
+ 1 = erlang:system_flag(schedulers_online, NOnln0),
+ receive after 500 -> ok end,
+ {{normal,NTot0,NOnln0,1},
+ {dirty_cpu,DCTot0,DCOnln0,DCAct0},
+ {dirty_io,DITot0,DIOnln0,DIAct0}} = sched_state(),
+
+ blocked = erlang:system_flag(multi_scheduling, block),
+ blocked = erlang:system_info(multi_scheduling),
+ receive after 500 -> ok end,
+ {{normal,NTot0,NOnln0,1},
+ {dirty_cpu,DCTot0,DCOnln0,0},
+ {dirty_io,DITot0,DIOnln0,0}} = sched_state(),
+
+ NOnln0 = erlang:system_flag(schedulers_online, 1),
+ receive after 500 -> ok end,
+ {{normal,NTot0,1,1},
+ {dirty_cpu,DCTot0,DCOne,0},
+ {dirty_io,DITot0,DIOnln0,0}} = sched_state(),
+
+ 1 = erlang:system_flag(schedulers_online, NOnln0),
+ receive after 500 -> ok end,
+ {{normal,NTot0,NOnln0,1},
+ {dirty_cpu,DCTot0,DCOnln0,0},
+ {dirty_io,DITot0,DIOnln0,0}} = sched_state(),
+
+ blocked = erlang:system_flag(multi_scheduling, unblock_normal),
+ blocked = erlang:system_info(multi_scheduling),
+ {{normal,NTot0,NOnln0,1},
+ {dirty_cpu,DCTot0,DCOnln0,0},
+ {dirty_io,DITot0,DIOnln0,0}} = sched_state(),
+
+ enabled = erlang:system_flag(multi_scheduling, unblock),
+ enabled = erlang:system_info(multi_scheduling),
+ receive after 500 -> ok end,
+ {{normal,NTot0,NOnln0,NAct0},
+ {dirty_cpu,DCTot0,DCOnln0,DCAct0},
+ {dirty_io,DITot0,DIOnln0,DIAct0}} = sched_state(),
+
+ NOnln0 = erlang:system_flag(schedulers_online, 1),
+ receive after 500 -> ok end,
+ {{normal,NTot0,1,1},
+ {dirty_cpu,DCTot0,DCOne,DCOne},
+ {dirty_io,DITot0,DIOnln0,DIAct0}} = sched_state(),
+
+ 1 = erlang:system_flag(schedulers_online, NOnln0),
+ receive after 500 -> ok end,
+ {{normal,NTot0,NOnln0,NAct0},
+ {dirty_cpu,DCTot0,DCOnln0,DCAct0},
+ {dirty_io,DITot0,DIOnln0,DIAct0}} = sched_state(),
+
+ ok.
+
+
scheduler_suspend(Config) when is_list(Config) ->
?line Dog = ?t:timetrap(?t:minutes(5)),
?line lists:foreach(fun (S) -> scheduler_suspend_test(Config, S) end,
@@ -1238,12 +1351,17 @@ scheduler_suspend_test(Config, Schedulers) ->
?line [ok] = mcall(Node, [fun () -> sst0_loop(300) end]),
?line [ok] = mcall(Node, [fun () -> sst1_loop(300) end]),
?line [ok] = mcall(Node, [fun () -> sst2_loop(300) end]),
- ?line [ok, ok, ok, ok, ok] = mcall(Node,
- [fun () -> sst0_loop(200) end,
- fun () -> sst1_loop(200) end,
- fun () -> sst2_loop(200) end,
- fun () -> sst2_loop(200) end,
- fun () -> sst3_loop(Sched, 200) end]),
+ ?line [ok] = mcall(Node, [fun () -> sst4_loop(300) end]),
+ ?line [ok] = mcall(Node, [fun () -> sst5_loop(300) end]),
+ ?line [ok, ok, ok, ok,
+ ok, ok, ok] = mcall(Node,
+ [fun () -> sst0_loop(200) end,
+ fun () -> sst1_loop(200) end,
+ fun () -> sst2_loop(200) end,
+ fun () -> sst2_loop(200) end,
+ fun () -> sst3_loop(Sched, 200) end,
+ fun () -> sst4_loop(200) end,
+ fun () -> sst5_loop(200) end]),
?line [SState] = mcall(Node, [fun () ->
case Sched == SchedOnln of
false ->
@@ -1320,6 +1438,20 @@ sst3_loop_with_dirty_schedulers(S, DS, N) ->
erlang:system_flag(dirty_cpu_schedulers_online, DS),
sst3_loop_with_dirty_schedulers(S, DS, N-1).
+sst4_loop(0) ->
+ ok;
+sst4_loop(N) ->
+ erlang:system_flag(multi_scheduling, block_normal),
+ erlang:system_flag(multi_scheduling, unblock_normal),
+ sst4_loop(N-1).
+
+sst5_loop(0) ->
+ ok;
+sst5_loop(N) ->
+ erlang:system_flag(multi_scheduling, block_normal),
+ erlang:system_flag(multi_scheduling, unblock_normal),
+ sst5_loop(N-1).
+
reader_groups(Config) when is_list(Config) ->
%% White box testing. These results are correct, but other results
%% could be too...
@@ -1598,6 +1730,34 @@ reader_groups_map(CPUT, Groups) ->
%% Utils
%%
+sched_state() ->
+ sched_state(erlang:system_info(all_schedulers_state),
+ undefined,
+ {dirty_cpu,0,0,0},
+ {dirty_io,0,0,0}).
+
+
+sched_state([], N, DC, DI) ->
+ try
+ chk_basic(N),
+ chk_basic(DC),
+ chk_basic(DI),
+ {N, DC, DI}
+ catch
+ _ : _ ->
+ ?t:fail({inconsisten_scheduler_state, {N, DC, DI}})
+ end;
+sched_state([{normal, _, _, _} = S | Rest], _S, DC, DI) ->
+ sched_state(Rest, S, DC, DI);
+sched_state([{dirty_cpu, _, _, _} = DC | Rest], S, _DC, DI) ->
+ sched_state(Rest, S, DC, DI);
+sched_state([{dirty_io, _, _, _} = DI | Rest], S, DC, _DI) ->
+ sched_state(Rest, S, DC, DI).
+
+chk_basic({_Type, Tot, Onln, Act}) ->
+ true = Tot >= Onln,
+ true = Onln >= Act.
+
l(Id) ->
{logical, Id}.
diff --git a/erts/emulator/valgrind/suppress.patched.3.6.0 b/erts/emulator/valgrind/suppress.patched.3.6.0
index 16cecf2dba..fcde4a0123 100644
--- a/erts/emulator/valgrind/suppress.patched.3.6.0
+++ b/erts/emulator/valgrind/suppress.patched.3.6.0
@@ -368,7 +368,7 @@ Memcheck:Addr4
...
fun:erts_print_scheduler_info
...
-fun:erl_exit
+fun:erts_exit
fun:broken_halt_test
fun:erts_debug_set_internal_state_2
fun:process_main
diff --git a/erts/emulator/valgrind/suppress.standard b/erts/emulator/valgrind/suppress.standard
index a1f3f82364..bb07c92fc1 100644
--- a/erts/emulator/valgrind/suppress.standard
+++ b/erts/emulator/valgrind/suppress.standard
@@ -336,7 +336,7 @@ Memcheck:Addr4
...
fun:erts_print_scheduler_info
...
-fun:erl_exit
+fun:erts_exit
fun:broken_halt_test
fun:erts_debug_set_internal_state_2
fun:process_main