aboutsummaryrefslogtreecommitdiffstats
path: root/erts/emulator/beam/beam_emu.c
diff options
context:
space:
mode:
Diffstat (limited to 'erts/emulator/beam/beam_emu.c')
-rw-r--r--erts/emulator/beam/beam_emu.c191
1 files changed, 151 insertions, 40 deletions
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 72526bca5e..f8f2e29c95 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -64,18 +64,21 @@
# ifdef ERTS_SMP
# define PROCESS_MAIN_CHK_LOCKS(P) \
do { \
- if ((P)) { \
+ if ((P)) \
erts_proc_lc_chk_only_proc_main((P)); \
- } \
- else \
- erts_lc_check_exact(NULL, 0); \
- ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking()); \
+ ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking()); \
+} while (0)
+# define ERTS_SMP_REQ_PROC_MAIN_LOCK(P) \
+do { \
+ if ((P)) \
+ erts_proc_lc_require_lock((P), ERTS_PROC_LOCK_MAIN, \
+ __FILE__, __LINE__); \
+} while (0)
+# define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P) \
+do { \
+ if ((P)) \
+ erts_proc_lc_unrequire_lock((P), ERTS_PROC_LOCK_MAIN); \
} while (0)
-# define ERTS_SMP_REQ_PROC_MAIN_LOCK(P) \
- if ((P)) erts_proc_lc_require_lock((P), ERTS_PROC_LOCK_MAIN,\
- __FILE__, __LINE__)
-# define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P) \
- if ((P)) erts_proc_lc_unrequire_lock((P), ERTS_PROC_LOCK_MAIN)
# else
# define ERTS_SMP_REQ_PROC_MAIN_LOCK(P)
# define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P)
@@ -308,7 +311,8 @@ void** beam_ops;
if (E - HTOP < (needed + (HeapNeed))) { \
SWAPOUT; \
PROCESS_MAIN_CHK_LOCKS(c_p); \
- FCALLS -= erts_garbage_collect_nobump(c_p, needed + (HeapNeed), reg, (M)); \
+ FCALLS -= erts_garbage_collect_nobump(c_p, needed + (HeapNeed), \
+ reg, (M), FCALLS); \
ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \
PROCESS_MAIN_CHK_LOCKS(c_p); \
SWAPIN; \
@@ -360,7 +364,7 @@ void** beam_ops;
if ((E - HTOP < need) || (MSO(c_p).overhead + (VNh) >= BIN_VHEAP_SZ(c_p))) {\
SWAPOUT; \
PROCESS_MAIN_CHK_LOCKS(c_p); \
- FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live)); \
+ FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live), FCALLS); \
ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \
PROCESS_MAIN_CHK_LOCKS(c_p); \
SWAPIN; \
@@ -381,7 +385,7 @@ void** beam_ops;
if (E - HTOP < need) { \
SWAPOUT; \
PROCESS_MAIN_CHK_LOCKS(c_p); \
- FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live));\
+ FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live), FCALLS); \
ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \
PROCESS_MAIN_CHK_LOCKS(c_p); \
SWAPIN; \
@@ -402,7 +406,7 @@ void** beam_ops;
SWAPOUT; \
reg[Live] = Extra; \
PROCESS_MAIN_CHK_LOCKS(c_p); \
- FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live)+1); \
+ FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live)+1, FCALLS); \
ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \
PROCESS_MAIN_CHK_LOCKS(c_p); \
Extra = reg[Live]; \
@@ -1196,6 +1200,25 @@ init_emulator(void)
#define DTRACE_NIF_RETURN(p, m, f, a) do {} while (0)
#endif /* USE_VM_PROBES */
+#ifdef DEBUG
+#define ERTS_DBG_CHK_REDS(P, FC) \
+ do { \
+ if (ERTS_PROC_GET_SAVED_CALLS_BUF((P))) { \
+ ASSERT(FC <= 0); \
+ ASSERT(erts_proc_sched_data(c_p)->virtual_reds \
+ <= 0 - (FC)); \
+ } \
+ else { \
+ ASSERT(FC <= CONTEXT_REDS); \
+ ASSERT(erts_proc_sched_data(c_p)->virtual_reds \
+ <= CONTEXT_REDS - (FC)); \
+ } \
+} while (0)
+#else
+#define ERTS_DBG_CHK_REDS(P, FC)
+#endif
+
+
/*
* process_main() is called twice:
* The first call performs some initialisation, including exporting
@@ -1290,14 +1313,19 @@ void process_main(void)
goto do_schedule1;
do_schedule:
- reds_used = REDS_IN(c_p) - FCALLS;
+ ASSERT(c_p->debug_reds_in == REDS_IN(c_p));
+ if (!ERTS_PROC_GET_SAVED_CALLS_BUF(c_p))
+ reds_used = REDS_IN(c_p) - FCALLS;
+ else
+ reds_used = REDS_IN(c_p) - (CONTEXT_REDS + FCALLS);
+ ASSERT(reds_used >= 0);
do_schedule1:
if (start_time != 0) {
Sint64 diff = erts_timestamp_millis() - start_time;
if (diff > 0 && (Uint) diff > erts_system_monitor_long_schedule
-#ifdef ERTS_DIRTY_SCHEDULERS
- && !ERTS_SCHEDULER_IS_DIRTY(c_p->scheduler_data)
+#if defined(ERTS_SMP) && defined(ERTS_DIRTY_SCHEDULERS)
+ && !ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(c_p))
#endif
) {
BeamInstr *inptr = find_function_from_pc(start_time_i);
@@ -1310,6 +1338,7 @@ void process_main(void)
ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
c_p = schedule(c_p, reds_used);
+ ASSERT(!(c_p->flags & F_HIPE_MODE));
ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
start_time = 0;
#ifdef DEBUG
@@ -1325,8 +1354,8 @@ void process_main(void)
start_time_i = c_p->i;
}
- reg = ERTS_PROC_GET_SCHDATA(c_p)->x_reg_array;
- freg = ERTS_PROC_GET_SCHDATA(c_p)->f_reg_array;
+ reg = erts_proc_sched_data(c_p)->x_reg_array;
+ freg = erts_proc_sched_data(c_p)->f_reg_array;
ERL_BITS_RELOAD_STATEP(c_p);
{
int reds;
@@ -1348,16 +1377,21 @@ void process_main(void)
SET_I(c_p->i);
- reds = c_p->fcalls;
- if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)
- && (ERTS_TRACE_FLAGS(c_p) & F_SENSITIVE) == 0) {
- neg_o_reds = -reds;
- FCALLS = REDS_IN(c_p) = 0;
+ REDS_IN(c_p) = reds = c_p->fcalls;
+#ifdef DEBUG
+ c_p->debug_reds_in = reds;
+#endif
+
+ if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) {
+ neg_o_reds = -CONTEXT_REDS;
+ FCALLS = neg_o_reds + reds;
} else {
neg_o_reds = 0;
- FCALLS = REDS_IN(c_p) = reds;
+ FCALLS = reds;
}
+ ERTS_DBG_CHK_REDS(c_p, FCALLS);
+
next = (BeamInstr *) *I;
SWAPIN;
ASSERT(VALID_INSTR(next));
@@ -1673,6 +1707,14 @@ void process_main(void)
BeamInstr *next;
Eterm result;
+ if (!(FCALLS > 0 || FCALLS > neg_o_reds)) {
+ /* If we have run out of reductions, we do a context
+ switch before calling the bif */
+ c_p->arity = 2;
+ c_p->current = NULL;
+ goto context_switch3;
+ }
+
PRE_BIF_SWAPOUT(c_p);
c_p->fcalls = FCALLS - 1;
result = erl_send(c_p, r(0), x(1));
@@ -1770,7 +1812,7 @@ void process_main(void)
if (E - HTOP < 3) {
SWAPOUT;
PROCESS_MAIN_CHK_LOCKS(c_p);
- FCALLS -= erts_garbage_collect_nobump(c_p, 3, reg+2, 1);
+ FCALLS -= erts_garbage_collect_nobump(c_p, 3, reg+2, 1, FCALLS);
ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
PROCESS_MAIN_CHK_LOCKS(c_p);
SWAPIN;
@@ -1865,6 +1907,7 @@ void process_main(void)
c_p->flags |= F_DELAY_GC;
loop_rec__:
+
PROCESS_MAIN_CHK_LOCKS(c_p);
msgp = PEEK_MESSAGE(c_p);
@@ -2007,6 +2050,8 @@ void process_main(void)
ERTS_VBUMP_LEAVE_REDS_INTERNAL(c_p, 5, FCALLS);
}
+ ERTS_DBG_CHK_REDS(c_p, FCALLS);
+
ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
PROCESS_MAIN_CHK_LOCKS(c_p);
NextPF(0, next);
@@ -2166,7 +2211,7 @@ void process_main(void)
PreFetch(0, next);
if (IS_TRACED_FL(c_p, F_TRACE_RECEIVE)) {
- trace_receive(c_p, am_timeout);
+ trace_receive(c_p, am_clock_service, am_timeout, NULL);
}
if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) {
save_calls(c_p, &exp_timeout);
@@ -2535,6 +2580,7 @@ do { \
GetArg1(2, tmp_reg[0]);
bf = (BifFunction) Arg(1);
+ ERTS_DBG_CHK_REDS(c_p, FCALLS);
c_p->fcalls = FCALLS;
PROCESS_MAIN_CHK_LOCKS(c_p);
ASSERT(!ERTS_PROC_IS_EXITING(c_p));
@@ -2544,6 +2590,7 @@ do { \
PROCESS_MAIN_CHK_LOCKS(c_p);
ERTS_HOLE_CHECK(c_p);
FCALLS = c_p->fcalls;
+ ERTS_DBG_CHK_REDS(c_p, FCALLS);
if (is_value(result)) {
StoreBifResult(3, result);
}
@@ -2564,6 +2611,7 @@ do { \
GetArg1(1, tmp_reg[0]);
bf = (BifFunction) Arg(0);
+ ERTS_DBG_CHK_REDS(c_p, FCALLS);
c_p->fcalls = FCALLS;
PROCESS_MAIN_CHK_LOCKS(c_p);
ASSERT(!ERTS_PROC_IS_EXITING(c_p));
@@ -2573,6 +2621,7 @@ do { \
PROCESS_MAIN_CHK_LOCKS(c_p);
ERTS_HOLE_CHECK(c_p);
FCALLS = c_p->fcalls;
+ ERTS_DBG_CHK_REDS(c_p, FCALLS);
if (is_value(result)) {
StoreBifResult(2, result);
}
@@ -2591,6 +2640,7 @@ do { \
GetArg1(2, x(live));
bf = (GcBifFunction) Arg(1);
+ ERTS_DBG_CHK_REDS(c_p, FCALLS);
c_p->fcalls = FCALLS;
SWAPOUT;
PROCESS_MAIN_CHK_LOCKS(c_p);
@@ -2602,6 +2652,7 @@ do { \
SWAPIN;
ERTS_HOLE_CHECK(c_p);
FCALLS = c_p->fcalls;
+ ERTS_DBG_CHK_REDS(c_p, FCALLS);
if (is_value(result)) {
StoreBifResult(4, result);
}
@@ -2630,6 +2681,7 @@ do { \
*/
live++;
bf = (GcBifFunction) Arg(1);
+ ERTS_DBG_CHK_REDS(c_p, FCALLS);
c_p->fcalls = FCALLS;
SWAPOUT;
PROCESS_MAIN_CHK_LOCKS(c_p);
@@ -2641,6 +2693,7 @@ do { \
SWAPIN;
ERTS_HOLE_CHECK(c_p);
FCALLS = c_p->fcalls;
+ ERTS_DBG_CHK_REDS(c_p, FCALLS);
if (is_value(result)) {
StoreBifResult(5, result);
}
@@ -2671,6 +2724,7 @@ do { \
*/
live += 2;
bf = (GcBifFunction) Arg(1);
+ ERTS_DBG_CHK_REDS(c_p, FCALLS);
c_p->fcalls = FCALLS;
SWAPOUT;
PROCESS_MAIN_CHK_LOCKS(c_p);
@@ -2682,6 +2736,7 @@ do { \
SWAPIN;
ERTS_HOLE_CHECK(c_p);
FCALLS = c_p->fcalls;
+ ERTS_DBG_CHK_REDS(c_p, FCALLS);
if (is_value(result)) {
StoreBifResult(5, result);
}
@@ -2708,6 +2763,7 @@ do { \
GetArg2(2, tmp_reg[0], tmp_reg[1]);
bf = (BifFunction) Arg(1);
+ ERTS_DBG_CHK_REDS(c_p, FCALLS);
c_p->fcalls = FCALLS;
PROCESS_MAIN_CHK_LOCKS(c_p);
ASSERT(!ERTS_PROC_IS_EXITING(c_p));
@@ -2717,6 +2773,7 @@ do { \
PROCESS_MAIN_CHK_LOCKS(c_p);
ERTS_HOLE_CHECK(c_p);
FCALLS = c_p->fcalls;
+ ERTS_DBG_CHK_REDS(c_p, FCALLS);
if (is_value(result)) {
StoreBifResult(4, result);
}
@@ -2764,6 +2821,15 @@ do { \
BeamInstr *next;
ErlHeapFragment *live_hf_end;
+
+ if (!((FCALLS - 1) > 0 || (FCALLS-1) > neg_o_reds)) {
+ /* If we have run out of reductions, we do a context
+ switch before calling the bif */
+ c_p->arity = ((Export *)Arg(0))->code[2];
+ c_p->current = ((Export *)Arg(0))->code;
+ goto context_switch3;
+ }
+
if (ERTS_MSACC_IS_ENABLED_CACHED_X()) {
if (GET_BIF_MODULE(Arg(0)) == am_ets) {
ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_ETS);
@@ -2775,6 +2841,7 @@ do { \
bf = GET_BIF_ADDRESS(Arg(0));
PRE_BIF_SWAPOUT(c_p);
+ ERTS_DBG_CHK_REDS(c_p, FCALLS);
c_p->fcalls = FCALLS - 1;
if (FCALLS <= 0) {
save_calls(c_p, (Export *) Arg(0));
@@ -2796,6 +2863,7 @@ do { \
PROCESS_MAIN_CHK_LOCKS(c_p);
HTOP = HEAP_TOP(c_p);
FCALLS = c_p->fcalls;
+ ERTS_DBG_CHK_REDS(c_p, FCALLS);
/* We have to update the cache if we are enabled in order
to make sure no book keeping is done after we disabled
msacc. We don't always do this as it is quite expensive. */
@@ -3290,10 +3358,19 @@ do { \
context_switch2: /* Entry for fun calls. */
c_p->current = I-3; /* Pointer to Mod, Func, Arity */
+ context_switch3:
+
{
Eterm* argp;
int i;
+ if (erts_smp_atomic32_read_nob(&c_p->state) & ERTS_PSFLG_EXITING) {
+ c_p->i = beam_exit;
+ c_p->arity = 0;
+ c_p->current = NULL;
+ goto do_schedule;
+ }
+
/*
* Make sure that there is enough room for the argument registers to be saved.
*/
@@ -3324,8 +3401,13 @@ do { \
* (beacuse the code for the Dispatch() macro becomes shorter that way).
*/
- reds_used = REDS_IN(c_p) - FCALLS + 1;
-
+ ASSERT(c_p->debug_reds_in == REDS_IN(c_p));
+ if (!ERTS_PROC_GET_SAVED_CALLS_BUF(c_p))
+ reds_used = REDS_IN(c_p) - FCALLS;
+ else
+ reds_used = REDS_IN(c_p) - (CONTEXT_REDS + FCALLS);
+ ASSERT(reds_used >= 0);
+
/*
* Save the argument registers and everything else.
*/
@@ -3456,6 +3538,12 @@ do { \
BifFunction vbf;
ErlHeapFragment *live_hf_end;
+ if (!((FCALLS - 1) > 0 || (FCALLS - 1) > neg_o_reds)) {
+ /* If we have run out of reductions, we do a context
+ switch before calling the nif */
+ goto context_switch;
+ }
+
ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_NIF);
DTRACE_NIF_ENTRY(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]);
@@ -3471,18 +3559,27 @@ do { \
typedef Eterm NifF(struct enif_environment_t*, int argc, Eterm argv[]);
NifF* fp = vbf = (NifF*) I[1];
struct enif_environment_t env;
+#ifdef ERTS_DIRTY_SCHEDULERS
+ if (!c_p->scheduler_data)
+ live_hf_end = ERTS_INVALID_HFRAG_PTR; /* On dirty scheduler */
+ else
+#endif
+ live_hf_end = c_p->mbuf;
erts_pre_nif(&env, c_p, (struct erl_module_nif*)I[2], NULL);
- live_hf_end = c_p->mbuf;
nif_bif_result = (*fp)(&env, bif_nif_arity, reg);
if (env.exception_thrown)
nif_bif_result = THE_NON_VALUE;
erts_post_nif(&env);
- }
- ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(nif_bif_result));
- PROCESS_MAIN_CHK_LOCKS(c_p);
- ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
- ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR);
+ PROCESS_MAIN_CHK_LOCKS(c_p);
+ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+ ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR);
+ if (env.exiting) {
+ ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
+ goto do_schedule;
+ }
+ ASSERT(!ERTS_PROC_IS_EXITING(c_p));
+ }
DTRACE_NIF_RETURN(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]);
goto apply_bif_or_nif_epilogue;
@@ -3498,6 +3595,13 @@ do { \
* code[3]: &&apply_bif
* code[4]: Function pointer to BIF function
*/
+
+ if (!((FCALLS - 1) > 0 || (FCALLS - 1) > neg_o_reds)) {
+ /* If we have run out of reductions, we do a context
+ switch before calling the bif */
+ goto context_switch;
+ }
+
if (ERTS_MSACC_IS_ENABLED_CACHED_X()) {
if ((Eterm)I[-3] == am_ets) {
ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_ETS);
@@ -3514,6 +3618,7 @@ do { \
DTRACE_BIF_ENTRY(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]);
SWAPOUT;
+ ERTS_DBG_CHK_REDS(c_p, FCALLS - 1);
c_p->fcalls = FCALLS - 1;
vbf = (BifFunction) Arg(0);
PROCESS_MAIN_CHK_LOCKS(c_p);
@@ -3549,6 +3654,7 @@ do { \
}
SWAPIN; /* There might have been a garbage collection. */
FCALLS = c_p->fcalls;
+ ERTS_DBG_CHK_REDS(c_p, FCALLS);
if (is_value(nif_bif_result)) {
r(0) = nif_bif_result;
CHECK_TERM(r(0));
@@ -4611,9 +4717,9 @@ do { \
OpCase(i_generic_breakpoint): {
BeamInstr real_I;
ASSERT(I[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI));
- SWAPOUT;
+ HEAVY_SWAPOUT;
real_I = erts_generic_breakpoint(c_p, I, reg);
- SWAPIN;
+ HEAVY_SWAPIN;
ASSERT(VALID_INSTR(real_I));
Goto(real_I);
}
@@ -4790,6 +4896,7 @@ do { \
{
#define HIPE_MODE_SWITCH(Cmd) \
SWAPOUT; \
+ ERTS_DBG_CHK_REDS(c_p, FCALLS); \
c_p->fcalls = FCALLS; \
c_p->def_arg_reg[4] = -neg_o_reds; \
c_p = hipe_mode_switch(c_p, Cmd, reg); \
@@ -4828,13 +4935,17 @@ do { \
#undef HIPE_MODE_SWITCH
L_post_hipe_mode_switch:
- reg = ERTS_PROC_GET_SCHDATA(c_p)->x_reg_array;
- freg = ERTS_PROC_GET_SCHDATA(c_p)->f_reg_array;
+#ifdef DEBUG
+ pid = c_p->common.id; /* may have switched process... */
+#endif
+ reg = erts_proc_sched_data(c_p)->x_reg_array;
+ freg = erts_proc_sched_data(c_p)->f_reg_array;
ERL_BITS_RELOAD_STATEP(c_p);
/* XXX: this abuse of def_arg_reg[] is horrid! */
neg_o_reds = -c_p->def_arg_reg[4];
FCALLS = c_p->fcalls;
SWAPIN;
+ ERTS_DBG_CHK_REDS(c_p, FCALLS);
switch( c_p->def_arg_reg[3] ) {
case HIPE_MODE_SWITCH_RES_RETURN:
ASSERT(is_value(reg[0]));
@@ -6839,7 +6950,7 @@ erts_current_reductions(Process *current, Process *p)
if (current != p) {
return 0;
} else if (current->fcalls < 0 && ERTS_PROC_GET_SAVED_CALLS_BUF(current)) {
- return -current->fcalls;
+ return current->fcalls + CONTEXT_REDS;
} else {
return REDS_IN(current) - current->fcalls;
}
generated by cgit v1.2.3 (git 2.25.1) at 2024-08-10 05:36:35 +0000