aboutsummaryrefslogtreecommitdiffstats
path: root/erts
diff options
context:
space:
mode:
Diffstat (limited to 'erts')
-rw-r--r--erts/doc/src/erl_nif.xml20
-rw-r--r--erts/emulator/beam/beam_emu.c298
-rw-r--r--erts/emulator/beam/erl_gc.c24
-rw-r--r--erts/emulator/beam/erl_message.c4
-rw-r--r--erts/emulator/beam/erl_nif.c97
-rw-r--r--erts/emulator/beam/erl_process.c113
-rw-r--r--erts/emulator/beam/erl_process.h25
-rw-r--r--erts/emulator/beam/global.h4
-rw-r--r--erts/emulator/beam/utils.c18
-rw-r--r--erts/emulator/hipe/hipe_mode_switch.c2
-rw-r--r--erts/emulator/test/dirty_nif_SUITE.erl129
-rw-r--r--erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c24
-rw-r--r--erts/preloaded/src/erts.app.src2
13 files changed, 534 insertions, 226 deletions
diff --git a/erts/doc/src/erl_nif.xml b/erts/doc/src/erl_nif.xml
index 57e047af08..4efd155b09 100644
--- a/erts/doc/src/erl_nif.xml
+++ b/erts/doc/src/erl_nif.xml
@@ -322,13 +322,13 @@ ok
<p>The
<seealso marker="#enif_consume_timeslice">enif_consume_timeslice()</seealso>
- function can be used to inform the runtime system about the lenght of the
+ function can be used to inform the runtime system about the length of the
NIF call. It should typically always be used unless the NIF executes very
quickly.</p>
- <p>If the NIF call is too lenghty one needs to handle this in one of the
+ <p>If the NIF call is too lengthy one needs to handle this in one of the
following ways in order to avoid degraded responsiveness, scheduler load
- balancing problems, and other strange behaviours:</p>
+ balancing problems, and other strange behaviors:</p>
<taglist>
<tag>Yielding NIF</tag>
@@ -410,14 +410,14 @@ ok
<seealso marker="erlang#system_flag_multi_scheduling"><c>erlang:system_flag(multi_scheduling,
block)</c></seealso>, might also take a very long time to
complete. This since all ongoing dirty operations on all
- dirty schedulers need to complete before the the block
+ dirty schedulers need to complete before the block
operation can complete.
</p>
<p>
A lot of operations communicating with a process executing a
dirty NIF can, however, complete while it is executing the
- dirty NIF. For example, retreiving information about it via
+ dirty NIF. For example, retrieving information about it via
<c>process_info()</c>, setting its group leader,
register/unregister its name, etc.
</p>
@@ -425,10 +425,10 @@ ok
<p>
Termination of a process executing a dirty NIF can only be
completed up to a certain point while it is executing the
- dirty NIF. All Erlang resources such as registered names,
- ETS tables, etc will be released. All links and monitors
+ dirty NIF. All Erlang resources such as its registered name,
+ its ETS tables, etc will be released. All links and monitors
will be triggered. The actual execution of the NIF will
- however <em>not</em> be stopped. The NIF can safely contiue
+ however <em>not</em> be stopped. The NIF can safely continue
execution, allocate heap memory, etc, but it is of course better
to stop executing as soon as possible. The NIF can check
whether current process is alive or not using
@@ -450,8 +450,8 @@ ok
collect a process in order to determine if it has
references to the module, a process executing a dirty
NIF might delay purging for a very long time. Delaying
- a purge operatin implies delaying <em>all</em> code
- loding operations which might cause severe problems for
+ a purge operation implies delaying <em>all</em> code
+ loading operations which might cause severe problems for
the system as a whole.
</p>
</item>
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index f8f2e29c95..0a59f8785c 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -1323,11 +1323,7 @@ void process_main(void)
if (start_time != 0) {
Sint64 diff = erts_timestamp_millis() - start_time;
- if (diff > 0 && (Uint) diff > erts_system_monitor_long_schedule
-#if defined(ERTS_SMP) && defined(ERTS_DIRTY_SCHEDULERS)
- && !ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(c_p))
-#endif
- ) {
+ if (diff > 0 && (Uint) diff > erts_system_monitor_long_schedule) {
BeamInstr *inptr = find_function_from_pc(start_time_i);
BeamInstr *outptr = find_function_from_pc(c_p->i);
monitor_long_schedule_proc(c_p,inptr,outptr,(Uint) diff);
@@ -1337,7 +1333,7 @@ void process_main(void)
PROCESS_MAIN_CHK_LOCKS(c_p);
ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
- c_p = schedule(c_p, reds_used);
+ c_p = erts_schedule(NULL, c_p, reds_used);
ASSERT(!(c_p->flags & F_HIPE_MODE));
ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
start_time = 0;
@@ -3559,12 +3555,10 @@ do { \
typedef Eterm NifF(struct enif_environment_t*, int argc, Eterm argv[]);
NifF* fp = vbf = (NifF*) I[1];
struct enif_environment_t env;
-#ifdef ERTS_DIRTY_SCHEDULERS
- if (!c_p->scheduler_data)
- live_hf_end = ERTS_INVALID_HFRAG_PTR; /* On dirty scheduler */
- else
+#ifdef ERTS_SMP
+ ASSERT(c_p->scheduler_data);
#endif
- live_hf_end = c_p->mbuf;
+ live_hf_end = c_p->mbuf;
erts_pre_nif(&env, c_p, (struct erl_module_nif*)I[2], NULL);
nif_bif_result = (*fp)(&env, bif_nif_arity, reg);
if (env.exception_thrown)
@@ -3574,10 +3568,7 @@ do { \
PROCESS_MAIN_CHK_LOCKS(c_p);
ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR);
- if (env.exiting) {
- ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
- goto do_schedule;
- }
+ ASSERT(!env.exiting);
ASSERT(!ERTS_PROC_IS_EXITING(c_p));
}
@@ -5162,6 +5153,283 @@ do { \
}
}
+/*
+ * erts_dirty_process_main() is what dirty schedulers execute. Since they handle
+ * only NIF calls they do not need to be able to execute all BEAM
+ * instructions.
+ */
+void erts_dirty_process_main(ErtsSchedulerData *esdp)
+{
+#ifdef ERTS_DIRTY_SCHEDULERS
+ Process* c_p = NULL;
+ ErtsMonotonicTime start_time;
+#ifdef DEBUG
+ ERTS_DECLARE_DUMMY(Eterm pid);
+#endif
+
+ /* Pointer to X registers: x(1)..x(N); reg[0] is used when doing GC,
+ * in all other cases x0 is used.
+ */
+ register Eterm* reg REG_xregs = NULL;
+
+ /*
+ * Top of heap (next free location); grows upwards.
+ */
+ register Eterm* HTOP REG_htop = NULL;
+
+ /* Stack pointer. Grows downwards; points
+ * to last item pushed (normally a saved
+ * continuation pointer).
+ */
+ register Eterm* E REG_stop = NULL;
+
+ /*
+ * Pointer to next threaded instruction.
+ */
+ register BeamInstr *I REG_I = NULL;
+
+ ERTS_MSACC_DECLARE_CACHE_X() /* a cached value of the tsd pointer for msacc */
+
+ /*
+ * start_time always positive for dirty CPU schedulers,
+ * and negative for dirty I/O schedulers.
+ */
+
+ if (ERTS_SCHEDULER_IS_DIRTY_CPU(esdp)) {
+ start_time = erts_get_monotonic_time(NULL);
+ ASSERT(start_time >= 0);
+ }
+ else {
+ start_time = ERTS_SINT64_MIN;
+ ASSERT(start_time < 0);
+ }
+
+ goto do_dirty_schedule;
+
+ context_switch:
+ c_p->arity = I[-1];
+ c_p->current = I-3; /* Pointer to Mod, Func, Arity */
+
+ {
+ int reds_used;
+ Eterm* argp;
+ int i;
+
+ /*
+ * Make sure that there is enough room for the argument registers to be saved.
+ */
+ if (c_p->arity > c_p->max_arg_reg) {
+ /*
+ * Yes, this is an expensive operation, but you only pay it the first
+ * time you call a function with more than 6 arguments which is
+ * scheduled out. This is better than paying for 26 words of wasted
+ * space for most processes which never call functions with more than
+ * 6 arguments.
+ */
+ Uint size = c_p->arity * sizeof(c_p->arg_reg[0]);
+ if (c_p->arg_reg != c_p->def_arg_reg) {
+ c_p->arg_reg = (Eterm *) erts_realloc(ERTS_ALC_T_ARG_REG,
+ (void *) c_p->arg_reg,
+ size);
+ } else {
+ c_p->arg_reg = (Eterm *) erts_alloc(ERTS_ALC_T_ARG_REG, size);
+ }
+ c_p->max_arg_reg = c_p->arity;
+ }
+
+ /*
+ * Save the argument registers and everything else.
+ */
+
+ argp = c_p->arg_reg;
+ for (i = c_p->arity - 1; i >= 0; i--) {
+ argp[i] = reg[i];
+ }
+ SWAPOUT;
+ c_p->i = I;
+
+ do_dirty_schedule:
+
+ if (start_time < 0) {
+ /*
+ * Dirty I/O scheduler:
+ * One reduction consumed regardless of
+ * time spent in the dirty NIF.
+ */
+ reds_used = esdp->virtual_reds + 1;
+ }
+ else {
+ /*
+ * Dirty CPU scheduler:
+ * Currently two reductions consumed per
+ * micro second spent in the dirty NIF.
+ */
+ ErtsMonotonicTime time;
+ time = erts_get_monotonic_time(esdp);
+ time -= start_time;
+ time = ERTS_MONOTONIC_TO_USEC(time);
+ time *= (CONTEXT_REDS-1)/1000 + 1;
+ ASSERT(time >= 0);
+ if (time == 0)
+ time = 1; /* At least one reduction */
+ time += esdp->virtual_reds;
+ reds_used = time > INT_MAX ? INT_MAX : (int) time;
+ }
+
+ PROCESS_MAIN_CHK_LOCKS(c_p);
+ ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
+ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+ c_p = erts_schedule(esdp, c_p, reds_used);
+
+ if (start_time >= 0) {
+ start_time = erts_get_monotonic_time(esdp);
+ ASSERT(start_time >= 0);
+ }
+ }
+
+ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+#ifdef DEBUG
+ pid = c_p->common.id; /* Save for debugging purposes */
+#endif
+ ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
+ PROCESS_MAIN_CHK_LOCKS(c_p);
+
+ ASSERT(!(c_p->flags & F_HIPE_MODE));
+ ERTS_MSACC_UPDATE_CACHE_X();
+
+ reg = esdp->x_reg_array;
+ {
+ Eterm* argp;
+ int i;
+
+ argp = c_p->arg_reg;
+ for (i = c_p->arity - 1; i >= 0; i--) {
+ reg[i] = argp[i];
+ CHECK_TERM(reg[i]);
+ }
+
+ /*
+ * We put the original reduction count in the process structure, to reduce
+ * the code size (referencing a field in a struct through a pointer stored
+ * in a register gives smaller code than referencing a global variable).
+ */
+
+ I = c_p->i;
+
+ ASSERT(BeamOp(op_call_nif) == (BeamInstr *) *I);
+
+ if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)
+ && (ERTS_TRACE_FLAGS(c_p) & F_SENSITIVE) == 0) {
+ c_p->fcalls = REDS_IN(c_p) = 0;
+ }
+
+ SWAPIN;
+
+#ifdef USE_VM_PROBES
+ if (DTRACE_ENABLED(process_scheduled)) {
+ DTRACE_CHARBUF(process_buf, DTRACE_TERM_BUF_SIZE);
+ DTRACE_CHARBUF(fun_buf, DTRACE_TERM_BUF_SIZE);
+ dtrace_proc_str(c_p, process_buf);
+
+ if (ERTS_PROC_IS_EXITING(c_p)) {
+ strcpy(fun_buf, "<exiting>");
+ } else {
+ BeamInstr *fptr = find_function_from_pc(c_p->i);
+ if (fptr) {
+ dtrace_fun_decode(c_p, (Eterm)fptr[0],
+ (Eterm)fptr[1], (Uint)fptr[2],
+ NULL, fun_buf);
+ } else {
+ erts_snprintf(fun_buf, sizeof(DTRACE_CHARBUF_NAME(fun_buf)),
+ "<unknown/%p>", next);
+ }
+ }
+
+ DTRACE2(process_scheduled, process_buf, fun_buf);
+ }
+#endif
+ }
+
+ {
+ Eterm nif_bif_result;
+ Eterm bif_nif_arity;
+
+ {
+ /*
+ * call_nif is always first instruction in function:
+ *
+ * I[-3]: Module
+ * I[-2]: Function
+ * I[-1]: Arity
+ * I[0]: &&call_nif
+ * I[1]: Function pointer to NIF function
+ * I[2]: Pointer to erl_module_nif
+ * I[3]: Function pointer to dirty NIF
+ */
+ BifFunction vbf;
+
+ ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_NIF);
+
+ DTRACE_NIF_ENTRY(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]);
+ c_p->current = I-3; /* current and vbf set to please handle_error */
+ SWAPOUT;
+ PROCESS_MAIN_CHK_LOCKS(c_p);
+ bif_nif_arity = I[-1];
+ ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
+
+ ASSERT(!ERTS_PROC_IS_EXITING(c_p));
+ {
+ typedef Eterm NifF(struct enif_environment_t*, int argc, Eterm argv[]);
+ NifF* fp = vbf = (NifF*) I[1];
+ struct enif_environment_t env;
+ ASSERT(!c_p->scheduler_data);
+ erts_pre_dirty_nif(esdp, &env, c_p, (struct erl_module_nif*)I[2], NULL);
+ nif_bif_result = (*fp)(&env, bif_nif_arity, reg);
+ if (env.exception_thrown)
+ nif_bif_result = THE_NON_VALUE;
+ erts_post_nif(&env);
+ PROCESS_MAIN_CHK_LOCKS(c_p);
+ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+ ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR);
+ if (env.exiting) {
+ ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
+ goto do_dirty_schedule;
+ }
+ ASSERT(!ERTS_PROC_IS_EXITING(c_p));
+ }
+ DTRACE_NIF_RETURN(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]);
+ ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
+ ERTS_HOLE_CHECK(c_p);
+ if (ERTS_IS_GC_DESIRED(c_p)) {
+ nif_bif_result = erts_gc_after_bif_call(c_p,
+ nif_bif_result,
+ reg, bif_nif_arity);
+ }
+ SWAPIN; /* There might have been a garbage collection. */
+ if (is_value(nif_bif_result)) {
+ r(0) = nif_bif_result;
+ CHECK_TERM(r(0));
+ I = c_p->cp;
+ c_p->cp = 0;
+ Goto(*I);
+ } else if (c_p->freason == TRAP) {
+ I = c_p->i;
+ ASSERT(!(c_p->flags & F_HIBERNATE_SCHED));
+ goto context_switch;
+ }
+ I = handle_error(c_p, c_p->cp, reg, vbf);
+ }
+ }
+ if (I == 0) {
+ goto do_dirty_schedule;
+ } else {
+ ASSERT(!is_value(r(0)));
+ SWAPIN;
+ goto context_switch;
+ }
+#endif /* ERTS_DIRTY_SCHEDULERS */
+}
+
static BifFunction
translate_gc_bif(void* gcf)
{
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index c7bbbd5ca0..d0d74bbf44 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -1183,22 +1183,14 @@ minor_collection(Process* p, ErlHeapFragment *live_hf_end,
adjust_size = p->htop - p->heap;
}
- goto done;
}
+ else if (need_after > HEAP_SIZE(p)) {
+ grow_new_heap(p, next_heap_size(p, need_after, 0), objv, nobj);
+ adjust_size = p->htop - p->heap;
+ }
+ /*else: The heap size turned out to be just right. We are done. */
- if (HEAP_SIZE(p) >= need_after) {
- /*
- * The heap size turned out to be just right. We are done.
- */
- goto done;
- }
-
- grow_new_heap(p, next_heap_size(p, need_after, 0), objv, nobj);
- adjust_size = p->htop - p->heap;
-
- done:
ASSERT(HEAP_SIZE(p) == next_heap_size(p, HEAP_SIZE(p), 0));
- ASSERT(MBUF(p) == NULL);
/* The heap usage during GC should be larger than what we end up
after a GC, even if we grow it. If this assertion is not true
@@ -1591,6 +1583,9 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end,
HIGH_WATER(p) = HEAP_TOP(p);
+#ifdef HARDDEBUG
+ disallow_heap_frag_ref_in_heap(p);
+#endif
remove_message_buffers(p);
if (p->flags & F_ON_HEAP_MSGQ)
@@ -1603,9 +1598,6 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end,
adjusted = adjust_after_fullsweep(p, need, objv, nobj);
-#ifdef HARDDEBUG
- disallow_heap_frag_ref_in_heap(p);
-#endif
ErtsGcQuickSanityCheck(p);
return gc_cost(size_after, adjusted ? size_after : 0);
diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index ac7b9d6606..71ab92937d 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -1345,7 +1345,7 @@ erts_prep_msgq_for_inspection(Process *c_p, Process *rp,
mpp = i == 0 ? &rp->msg.first : &mip[i-1].msgp->next;
- ASSERT((*mpp)->next == bad_mp);
+ ASSERT(*mpp == bad_mp);
erts_msgq_update_internal_pointers(&rp->msg, mpp, &bad_mp->next);
@@ -1725,7 +1725,7 @@ void erts_factory_trim_and_close(ErtsHeapFactory* factory,
case FACTORY_MESSAGE: {
ErtsMessage *mp = factory->message;
if (mp->data.attached == ERTS_MSG_COMBINED_HFRAG) {
- if (!mp->hfrag.next) {
+ if (!factory->heap_frags) {
Uint sz = factory->hp - factory->hp_start;
mp = erts_shrink_message(mp, sz, brefs, brefs_size);
factory->message = mp;
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 2bbb8e3c91..f1c35cd5a5 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -178,9 +178,6 @@ static ERTS_INLINE void ensure_heap(ErlNifEnv* env, size_t may_need)
void erts_pre_nif(ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif,
Process* tracee)
{
-#ifdef ERTS_DIRTY_SCHEDULERS
- ErtsSchedulerData *esdp;
-#endif
env->mod_nif = mod_nif;
env->proc = p;
env->hp = HEAP_TOP(p);
@@ -193,57 +190,65 @@ void erts_pre_nif(ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif,
ASSERT(p->common.id != ERTS_INVALID_PID);
-#ifdef ERTS_DIRTY_SCHEDULERS
- esdp = erts_get_scheduler_data();
- ASSERT(esdp);
+#if defined(DEBUG) && defined(ERTS_DIRTY_SCHEDULERS)
+ {
+ ErtsSchedulerData *esdp = erts_get_scheduler_data();
+ ASSERT(esdp);
- if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
-#ifdef DEBUG
- erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state);
+ if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+ erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state);
- ASSERT(p->scheduler_data == esdp);
- ASSERT((state & (ERTS_PSFLG_RUNNING
- | ERTS_PSFLG_RUNNING_SYS))
- && !(state & (ERTS_PSFLG_DIRTY_RUNNING
- | ERTS_PSFLG_DIRTY_RUNNING_SYS)));
+ ASSERT(p->scheduler_data == esdp);
+ ASSERT((state & (ERTS_PSFLG_RUNNING
+ | ERTS_PSFLG_RUNNING_SYS))
+ && !(state & (ERTS_PSFLG_DIRTY_RUNNING
+ | ERTS_PSFLG_DIRTY_RUNNING_SYS)));
+ }
+ }
#endif
+}
- }
- else {
- Process *sproc;
+void erts_pre_dirty_nif(ErtsSchedulerData *esdp,
+ ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif,
+ Process* tracee)
+{
+#ifdef ERTS_DIRTY_SCHEDULERS
+ Process *sproc;
#ifdef DEBUG
- erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state);
+ erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state);
- ASSERT(!p->scheduler_data);
- ASSERT((state & ERTS_PSFLG_DIRTY_RUNNING)
- && !(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS)));
+ ASSERT(!p->scheduler_data);
+ ASSERT((state & ERTS_PSFLG_DIRTY_RUNNING)
+ && !(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS)));
+ ASSERT(esdp);
#endif
- sproc = esdp->dirty_shadow_process;
- ASSERT(sproc);
- ASSERT(sproc->static_flags & ERTS_STC_FLG_SHADOW_PROC);
- ASSERT(erts_smp_atomic32_read_nob(&sproc->state)
- == (ERTS_PSFLG_ACTIVE
- | ERTS_PSFLG_DIRTY_RUNNING
- | ERTS_PSFLG_PROXY));
-
- sproc->next = p;
- sproc->common.id = p->common.id;
- sproc->htop = p->htop;
- sproc->stop = p->stop;
- sproc->hend = p->hend;
- sproc->heap = p->heap;
- sproc->abandoned_heap = p->abandoned_heap;
- sproc->heap_sz = p->heap_sz;
- sproc->high_water = p->high_water;
- sproc->old_hend = p->old_hend;
- sproc->old_htop = p->old_htop;
- sproc->old_heap = p->old_heap;
- sproc->mbuf = NULL;
- sproc->mbuf_sz = 0;
- ERTS_INIT_OFF_HEAP(&sproc->off_heap);
- env->proc = sproc;
- }
+ erts_pre_nif(env, p, mod_nif, tracee);
+
+ sproc = esdp->dirty_shadow_process;
+ ASSERT(sproc);
+ ASSERT(sproc->static_flags & ERTS_STC_FLG_SHADOW_PROC);
+ ASSERT(erts_smp_atomic32_read_nob(&sproc->state)
+ == (ERTS_PSFLG_ACTIVE
+ | ERTS_PSFLG_DIRTY_RUNNING
+ | ERTS_PSFLG_PROXY));
+
+ sproc->next = p;
+ sproc->common.id = p->common.id;
+ sproc->htop = p->htop;
+ sproc->stop = p->stop;
+ sproc->hend = p->hend;
+ sproc->heap = p->heap;
+ sproc->abandoned_heap = p->abandoned_heap;
+ sproc->heap_sz = p->heap_sz;
+ sproc->high_water = p->high_water;
+ sproc->old_hend = p->old_hend;
+ sproc->old_htop = p->old_htop;
+ sproc->old_heap = p->old_heap;
+ sproc->mbuf = NULL;
+ sproc->mbuf_sz = 0;
+ ERTS_INIT_OFF_HEAP(&sproc->off_heap);
+ env->proc = sproc;
#endif
}
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index f8cbe60e76..c0b1d7246c 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -8197,7 +8197,7 @@ sched_dirty_cpu_thread_func(void *vesdp)
#endif
erts_thread_init_float();
- process_main();
+ erts_dirty_process_main(esdp);
/* No schedulers should *ever* terminate */
erts_exit(ERTS_ABORT_EXIT,
"Dirty CPU scheduler thread number %beu terminated\n",
@@ -8242,7 +8242,7 @@ sched_dirty_io_thread_func(void *vesdp)
#endif
erts_thread_init_float();
- process_main();
+ erts_dirty_process_main(esdp);
/* No schedulers should *ever* terminate */
erts_exit(ERTS_ABORT_EXIT,
"Dirty I/O scheduler thread number %beu terminated\n",
@@ -9377,77 +9377,6 @@ scheduler_gc_proc(Process *c_p, int reds_left)
return reds;
}
-static ERTS_INLINE void
-clean_dirty_start(Process *p)
-{
-#if defined(ERTS_DIRTY_SCHEDULERS) && !defined(ARCH_64)
- void *ptr = ERTS_PROC_SET_DIRTY_CPU_START(p, NULL);
- if (ptr)
- erts_free(ERTS_ALC_T_DIRTY_START, ptr);
-#endif
-}
-
-static ERTS_INLINE void
-save_dirty_start(ErtsSchedulerData *esdp, Process *c_p)
-{
-#ifdef ERTS_DIRTY_SCHEDULERS
- if (ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(esdp->run_queue)) {
- ErtsMonotonicTime time = erts_get_monotonic_time(esdp);
-#ifdef ARCH_64
- ERTS_PROC_SET_DIRTY_CPU_START(c_p, (void *) time);
-#else
- ErtsMonotonicTime *stimep;
-
- stimep = (ErtsMonotonicTime *) ERTS_PROC_GET_DIRTY_CPU_START(c_p);
- if (!stimep) {
- stimep = erts_alloc(ERTS_ALC_T_DIRTY_START,
- sizeof(ErtsMonotonicTime));
- ERTS_PROC_SET_DIRTY_CPU_START(c_p, (void *) stimep);
- }
- *stimep = time;
-#endif
- }
-#endif
-}
-
-static ERTS_INLINE int
-get_dirty_reds(ErtsSchedulerData *esdp, Process *c_p)
-{
-
-#ifndef ERTS_DIRTY_SCHEDULERS
- return -1;
-#else
- ErtsMonotonicTime stime, time;
-
- if (!ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(esdp->run_queue))
- return 1;
-
-#ifdef ARCH_64
- stime = (ErtsMonotonicTime) ERTS_PROC_GET_DIRTY_CPU_START(c_p);
-#else
- {
- ErtsMonotonicTime *stimep;
- stimep = (ErtsMonotonicTime *) ERTS_PROC_GET_DIRTY_CPU_START(c_p);
- ASSERT(stimep);
- stime = *stimep;
- }
-#endif
-
- time = erts_get_monotonic_time(esdp);
-
- ASSERT(stime && stime < time);
-
- time -= stime;
- time = ERTS_MONOTONIC_TO_USEC(time);
- time *= 2;
-
- if (time > INT_MAX)
- return INT_MAX;
- return (int) time;
-#endif
-
-}
-
/*
* schedule() is called from BEAM (process_main()) or HiPE
* (hipe_mode_switch()) when the current process is to be
@@ -9466,11 +9395,10 @@ get_dirty_reds(ErtsSchedulerData *esdp, Process *c_p)
* so that normal processes get to run more frequently.
*/
-Process *schedule(Process *p, int calls)
+Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
{
Process *proxy_p = NULL;
ErtsRunQueue *rq;
- ErtsSchedulerData *esdp;
int context_reds;
int fcalls;
int input_reductions;
@@ -9507,8 +9435,19 @@ Process *schedule(Process *p, int calls)
* Clean up after the process being scheduled out.
*/
if (!p) { /* NULL in the very first schedule() call */
+#ifdef ERTS_DIRTY_SCHEDULERS
+ is_normal_sched = !esdp;
+ if (is_normal_sched) {
+ esdp = erts_get_scheduler_data();
+ ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp));
+ }
+ else {
+ ASSERT(ERTS_SCHEDULER_IS_DIRTY(esdp));
+ }
+#else
esdp = erts_get_scheduler_data();
- is_normal_sched = !ERTS_SCHEDULER_IS_DIRTY(esdp);
+ is_normal_sched = 1;
+#endif
rq = erts_get_runq_current(esdp);
ASSERT(esdp);
fcalls = (int) erts_smp_atomic32_read_acqb(&function_calls);
@@ -9517,12 +9456,12 @@ Process *schedule(Process *p, int calls)
} else {
#ifdef ERTS_SMP
#ifdef ERTS_DIRTY_SCHEDULERS
- esdp = p->scheduler_data;
- is_normal_sched = esdp != NULL;
- if (is_normal_sched)
+ is_normal_sched = !esdp;
+ if (is_normal_sched) {
+ esdp = p->scheduler_data;
ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp));
+ }
else {
- esdp = erts_get_scheduler_data();
ASSERT(ERTS_SCHEDULER_IS_DIRTY(esdp));
}
#else
@@ -9541,10 +9480,7 @@ Process *schedule(Process *p, int calls)
ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p);
- if (is_normal_sched)
- reds = actual_reds = calls - esdp->virtual_reds;
- else
- reds = actual_reds = get_dirty_reds(esdp, p);
+ reds = actual_reds = calls - esdp->virtual_reds;
ASSERT(actual_reds >= 0);
if (reds < ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST)
@@ -9994,17 +9930,10 @@ Process *schedule(Process *p, int calls)
calls = 0;
reds = context_reds;
-#ifdef ERTS_SMP
-
erts_smp_runq_unlock(rq);
-#endif /* ERTS_SMP */
-
}
- if (!is_normal_sched)
- save_dirty_start(esdp, p);
-
#ifdef ERTS_SMP
if (flags & ERTS_RUNQ_FLG_PROTECTED)
@@ -11745,8 +11674,6 @@ delete_process(Process* p)
if (nif_export)
erts_destroy_nif_export(nif_export);
- clean_dirty_start(p);
-
/* Cleanup psd */
psd = (ErtsPSD *) erts_smp_atomic_read_nob(&p->psd);
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index b44ac442aa..7569fd81bd 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -810,25 +810,13 @@ erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi)
#define ERTS_PSD_DELAYED_GC_TASK_QS 4
#define ERTS_PSD_NIF_TRAP_EXPORT 5
#define ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF 6
-#define ERTS_PSD_DIRTY_CPU_START 7
-#define ERTS_PSD_SIZE 8
+#define ERTS_PSD_SIZE 7
-#if !defined(HIPE) && !defined(ERTS_DIRTY_SCHEDULERS)
+#if !defined(HIPE)
# undef ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF
-# undef ERTS_PSD_DIRTY_CPU_START
# undef ERTS_PSD_SIZE
# define ERTS_PSD_SIZE 6
-#elif !defined(HIPE)
-# undef ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF
-# undef ERTS_PSD_DIRTY_CPU_START
-# undef ERTS_PSD_SIZE
-# define ERTS_PSD_DIRTY_CPU_START 6
-# define ERTS_PSD_SIZE 7
-#elif !defined(ERTS_DIRTY_SCHEDULERS)
-# undef ERTS_PSD_DIRTY_CPU_START
-# undef ERTS_PSD_SIZE
-# define ERTS_PSD_SIZE 7
#endif
typedef struct {
@@ -1831,7 +1819,7 @@ Eterm erts_get_schedulers_binds(Process *c_p);
Eterm erts_set_cpu_topology(Process *c_p, Eterm term);
Eterm erts_bind_schedulers(Process *c_p, Eterm how);
ErtsRunQueue *erts_schedid2runq(Uint);
-Process *schedule(Process*, int);
+Process *erts_schedule(ErtsSchedulerData *, Process*, int);
void erts_schedule_misc_op(void (*)(void *), void *);
Eterm erl_create_process(Process*, Eterm, Eterm, Eterm, ErlSpawnOpts*);
void erts_do_exit_process(Process*, Eterm);
@@ -2061,13 +2049,6 @@ erts_psd_set(Process *p, int ix, void *data)
((struct saved_calls *) erts_psd_set((P), ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF, (void *) (SCB)))
#endif
-#ifdef ERTS_DIRTY_SCHEDULERS
-#define ERTS_PROC_GET_DIRTY_CPU_START(P) \
- ((void *) erts_psd_get((P), ERTS_PSD_DIRTY_CPU_START))
-#define ERTS_PROC_SET_DIRTY_CPU_START(P, DCS) \
- ((void *) erts_psd_set((P), ERTS_PSD_DIRTY_CPU_START, (void *) (DCS)))
-#endif
-
ERTS_GLB_INLINE Eterm erts_proc_get_error_handler(Process *p);
ERTS_GLB_INLINE Eterm erts_proc_set_error_handler(Process *p, Eterm handler);
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index b76b9cd874..f3d4ac56cd 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -62,6 +62,9 @@ struct enif_environment_t /* ErlNifEnv */
extern void erts_pre_nif(struct enif_environment_t*, Process*,
struct erl_module_nif*, Process* tracee);
extern void erts_post_nif(struct enif_environment_t* env);
+extern void erts_pre_dirty_nif(ErtsSchedulerData *,
+ struct enif_environment_t*, Process*,
+ struct erl_module_nif*, Process* tracee);
extern Eterm erts_nif_taints(Process* p);
extern void erts_print_nif_taints(int to, void* to_arg);
void erts_unload_nif(struct erl_module_nif* nif);
@@ -1152,6 +1155,7 @@ void print_pass_through(int, byte*, int);
int catchlevel(Process*);
void init_emulator(void);
void process_main(void);
+void erts_dirty_process_main(ErtsSchedulerData *);
Eterm build_stacktrace(Process* c_p, Eterm exc);
Eterm expand_error_value(Process* c_p, Uint freason, Eterm Value);
void erts_save_stacktrace(Process* p, struct StackTrace* s, int depth);
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index f0418446a8..675fafa726 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -2210,7 +2210,9 @@ do_allocate_logger_message(Eterm gleader, Eterm **hp, ErlOffHeap **ohp,
#ifndef ERTS_SMP
#ifdef USE_THREADS
- if (erts_get_scheduler_data()) /* Must be scheduler thread */
+ if (!erts_get_scheduler_data()) /* Must be scheduler thread */
+ *p = NULL;
+ else
#endif
{
*p = erts_whereis_process(NULL, 0, am_error_logger, 0, 0);
@@ -2226,18 +2228,10 @@ do_allocate_logger_message(Eterm gleader, Eterm **hp, ErlOffHeap **ohp,
}
/* So we have an error logger, lets build the message */
- if (sz <= HeapWordsLeft(*p)) {
- *ohp = &MSO(*p);
- *hp = HEAP_TOP(*p);
- HEAP_TOP(*p) += sz;
- } else {
-#endif
- *bp = new_message_buffer(sz);
- *ohp = &(*bp)->off_heap;
- *hp = (*bp)->mem;
-#ifndef ERTS_SMP
- }
#endif
+ *bp = new_message_buffer(sz);
+ *ohp = &(*bp)->off_heap;
+ *hp = (*bp)->mem;
return (is_nil(gleader)
? am_noproc
diff --git a/erts/emulator/hipe/hipe_mode_switch.c b/erts/emulator/hipe/hipe_mode_switch.c
index 884331e969..ed95045292 100644
--- a/erts/emulator/hipe/hipe_mode_switch.c
+++ b/erts/emulator/hipe/hipe_mode_switch.c
@@ -547,7 +547,7 @@ Process *hipe_mode_switch(Process *p, unsigned cmd, Eterm reg[])
p->flags &= ~F_HIPE_MODE;
ERTS_SMP_UNREQ_PROC_MAIN_LOCK(p);
- p = schedule(p, reds_in - p->fcalls);
+ p = erts_schedule(NULL, p, reds_in - p->fcalls);
ERTS_SMP_REQ_PROC_MAIN_LOCK(p);
ASSERT(!(p->flags & F_HIPE_MODE));
#ifdef ERTS_SMP
diff --git a/erts/emulator/test/dirty_nif_SUITE.erl b/erts/emulator/test/dirty_nif_SUITE.erl
index c3afbc0803..83b098a704 100644
--- a/erts/emulator/test/dirty_nif_SUITE.erl
+++ b/erts/emulator/test/dirty_nif_SUITE.erl
@@ -32,19 +32,23 @@
dirty_nif/1, dirty_nif_send/1,
dirty_nif_exception/1, call_dirty_nif_exception/1,
dirty_scheduler_exit/1, dirty_call_while_terminated/1,
- dirty_heap_access/1]).
+ dirty_heap_access/1, dirty_process_info/1,
+ dirty_process_register/1, dirty_process_trace/1]).
-define(nif_stub,nif_stub_error(?LINE)).
suite() -> [{ct_hooks,[ts_install_cth]}].
-all() ->
+all() ->
[dirty_nif,
dirty_nif_send,
dirty_nif_exception,
dirty_scheduler_exit,
dirty_call_while_terminated,
- dirty_heap_access].
+ dirty_heap_access,
+ dirty_process_info,
+ dirty_process_register,
+ dirty_process_trace].
init_per_suite(Config) ->
try erlang:system_info(dirty_cpu_schedulers) of
@@ -187,7 +191,7 @@ dirty_call_while_terminated(Config) when is_list(Config) ->
blipp:blupp(Bin)
end,
[monitor,link]),
- receive {dirty_alive, Pid} -> ok end,
+ receive {dirty_alive, _Pid} -> ok end,
{value, {BinAddr, 4711, 2}} = lists:keysearch(4711, 2,
element(2,
process_info(self(),
@@ -241,7 +245,7 @@ dirty_heap_access(Config) when is_list(Config) ->
end),
{N, R} = access_dirty_heap(Dirty, RGL, 0, 0),
receive
- {Pid, Res} ->
+ {_Pid, Res} ->
1000 = length(Res),
lists:foreach(fun (X) -> Ref = X end, Res)
end,
@@ -269,12 +273,123 @@ access_dirty_heap(Dirty, RGL, N, R) ->
end)
end.
+%% These tests verify that processes that access a process executing a
+%% dirty NIF where the main lock is needed for that access do not get
+%% blocked. Each test passes its pid to dirty_sleeper, which sends a
+%% 'ready' message when it's running on a dirty scheduler and just before
+%% it starts a 6 second sleep. When it receives the message, it verifies
+%% that access to the dirty process is as it expects. After the dirty
+%% process finishes its 6 second sleep but before it returns from the dirty
+%% scheduler, it sends a 'done' message. If the tester already received
+%% that message, the test fails because it means attempting to access the
+%% dirty process waited for that process to return to a regular scheduler,
+%% so verify that we haven't received that message, and also verify that
+%% the dirty process is still alive immediately after accessing it.
+dirty_process_info(Config) when is_list(Config) ->
+ access_dirty_process(
+ Config,
+ fun() -> ok end,
+ fun(NifPid) ->
+ PI = process_info(NifPid),
+ {current_function,{?MODULE,dirty_sleeper,1}} =
+ lists:keyfind(current_function, 1, PI),
+ ok
+ end,
+ fun(_) -> ok end).
+
+dirty_process_register(Config) when is_list(Config) ->
+ access_dirty_process(
+ Config,
+ fun() -> ok end,
+ fun(NifPid) ->
+ register(test_dirty_process_register, NifPid),
+ NifPid = whereis(test_dirty_process_register),
+ unregister(test_dirty_process_register),
+ false = lists:member(test_dirty_process_register,
+ registered()),
+ ok
+ end,
+ fun(_) -> ok end).
+
+dirty_process_trace(Config) when is_list(Config) ->
+ access_dirty_process(
+ Config,
+ fun() ->
+ erlang:trace_pattern({?MODULE,dirty_sleeper,1},
+ [{'_',[],[{return_trace}]}],
+ [local,meta]),
+ ok
+ end,
+ fun(NifPid) ->
+ erlang:trace(NifPid, true, [call,timestamp]),
+ ok
+ end,
+ fun(NifPid) ->
+ receive
+ done ->
+ receive
+ {trace_ts,NifPid,call,{?MODULE,dirty_sleeper,_},_} ->
+ ok
+ after
+ 0 ->
+ error(missing_trace_call_message)
+ end,
+ receive
+ {trace_ts,NifPid,return_from,{?MODULE,dirty_sleeper,1},
+ ok,_} ->
+ ok
+ after
+ 100 ->
+ error(missing_trace_return_message)
+ end
+ after
+ 6500 ->
+ error(missing_done_message)
+ end,
+ ok
+ end).
+
%%
%% Internal...
%%
+access_dirty_process(Config, Start, Test, Finish) ->
+ {ok, Node} = start_node(Config, ""),
+ [ok] = mcall(Node,
+ [fun() ->
+ Path = ?config(data_dir, Config),
+ Lib = atom_to_list(?MODULE),
+ ok = erlang:load_nif(filename:join(Path,Lib), []),
+ ok = test_dirty_process_access(Start, Test, Finish)
+ end]),
+ stop_node(Node),
+ ok.
+
+test_dirty_process_access(Start, Test, Finish) ->
+ ok = Start(),
+ Self = self(),
+ NifPid = spawn_link(fun() ->
+ ok = dirty_sleeper(Self)
+ end),
+ ok = receive
+ ready ->
+ ok = Test(NifPid),
+ receive
+ done ->
+ error(dirty_process_info_blocked)
+ after
+ 0 ->
+ true = erlang:is_process_alive(NifPid),
+ ok
+ end
+ after
+ 3000 ->
+ error(timeout)
+ end,
+ ok = Finish(NifPid).
+
receive_any() ->
- receive M -> M end.
+ receive M -> M end.
start_node(Config) ->
start_node(Config, "").
@@ -314,13 +429,13 @@ mcall(Node, Funs) ->
%% The NIFs:
lib_loaded() -> false.
-call_nif_schedule(_,_) -> ?nif_stub.
call_dirty_nif(_,_,_) -> ?nif_stub.
send_from_dirty_nif(_) -> ?nif_stub.
call_dirty_nif_exception(_) -> ?nif_stub.
call_dirty_nif_zero_args() -> ?nif_stub.
dirty_call_while_terminated_nif(_) -> ?nif_stub.
dirty_sleeper() -> ?nif_stub.
+dirty_sleeper(_) -> ?nif_stub.
dirty_heap_access_nif(_) -> ?nif_stub.
nif_stub_error(Line) ->
diff --git a/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c b/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c
index 2013c88167..e38bececde 100644
--- a/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c
+++ b/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c
@@ -19,7 +19,9 @@
*/
#include "erl_nif.h"
#include <assert.h>
-#ifndef __WIN32__
+#ifdef __WIN32__
+#include <windows.h>
+#else
#include <unistd.h>
#endif
@@ -146,12 +148,31 @@ static ERL_NIF_TERM call_dirty_nif_zero_args(ErlNifEnv* env, int argc, const ERL
static ERL_NIF_TERM
dirty_sleeper(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
{
+ ErlNifPid pid;
+ ErlNifEnv* msg_env = NULL;
+
assert(enif_is_on_dirty_scheduler(env));
+
+ /* If we get a pid argument, it indicates a process involved in the
+ test wants a message from us. Prior to the sleep we send a 'ready'
+ message, and then after the sleep, send a 'done' message. */
+ if (argc == 1 && enif_get_local_pid(env, argv[0], &pid)) {
+ msg_env = enif_alloc_env();
+ enif_send(env, &pid, msg_env, enif_make_atom(msg_env, "ready"));
+ }
+
#ifdef __WIN32__
Sleep(6000);
#else
sleep(6);
#endif
+
+ if (argc == 1) {
+ assert(msg_env != NULL);
+ enif_send(env, &pid, msg_env, enif_make_atom(msg_env, "done"));
+ enif_free_env(msg_env);
+ }
+
return enif_make_atom(env, "ok");
}
@@ -216,6 +237,7 @@ static ErlNifFunc nif_funcs[] =
{"call_dirty_nif_exception", 1, call_dirty_nif_exception, ERL_NIF_DIRTY_JOB_IO_BOUND},
{"call_dirty_nif_zero_args", 0, call_dirty_nif_zero_args, ERL_NIF_DIRTY_JOB_CPU_BOUND},
{"dirty_sleeper", 0, dirty_sleeper, ERL_NIF_DIRTY_JOB_IO_BOUND},
+ {"dirty_sleeper", 1, dirty_sleeper, ERL_NIF_DIRTY_JOB_CPU_BOUND},
{"dirty_call_while_terminated_nif", 1, dirty_call_while_terminated_nif, ERL_NIF_DIRTY_JOB_CPU_BOUND},
{"dirty_heap_access_nif", 1, dirty_heap_access_nif, ERL_NIF_DIRTY_JOB_CPU_BOUND}
};
diff --git a/erts/preloaded/src/erts.app.src b/erts/preloaded/src/erts.app.src
index 98e0224a5f..e18da28905 100644
--- a/erts/preloaded/src/erts.app.src
+++ b/erts/preloaded/src/erts.app.src
@@ -37,7 +37,7 @@
{registered, []},
{applications, []},
{env, []},
- {runtime_dependencies, ["stdlib-2.5", "kernel-4.0", "sasl-2.4"]}
+ {runtime_dependencies, ["stdlib-3.0", "kernel-5.0", "sasl-3.0"]}
]}.
%% vim: ft=erlang
generated by cgit v1.2.3 (git 2.25.1) at 2024-12-20 04:47:35 +0000