aboutsummaryrefslogtreecommitdiffstats
path: root/erts/emulator/beam
diff options
context:
space:
mode:
Diffstat (limited to 'erts/emulator/beam')
-rw-r--r--erts/emulator/beam/beam_bif_load.c12
-rw-r--r--erts/emulator/beam/beam_emu.c94
-rw-r--r--erts/emulator/beam/bif.c48
-rw-r--r--erts/emulator/beam/bif.h1
-rw-r--r--erts/emulator/beam/erl_gc.c40
-rw-r--r--erts/emulator/beam/erl_gc.h2
-rw-r--r--erts/emulator/beam/erl_init.c10
-rw-r--r--erts/emulator/beam/erl_process.c62
-rw-r--r--erts/emulator/beam/erl_process.h19
-rw-r--r--erts/emulator/beam/erl_trace.c22
-rw-r--r--erts/emulator/beam/global.h2
11 files changed, 231 insertions, 81 deletions
diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index 87508dcf5f..801218774b 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -38,7 +38,7 @@
#include "erl_thr_progress.h"
static void set_default_trace_pattern(Eterm module);
-static Eterm check_process_code(Process* rp, Module* modp, Uint flags, int *redsp);
+static Eterm check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls);
static void delete_code(Module* modp);
static void decrement_refc(BeamCodeHeader*);
static int any_heap_ref_ptrs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size);
@@ -467,7 +467,7 @@ check_old_code_1(BIF_ALIST_1)
}
Eterm
-erts_check_process_code(Process *c_p, Eterm module, Uint flags, int *redsp)
+erts_check_process_code(Process *c_p, Eterm module, Uint flags, int *redsp, int fcalls)
{
Module* modp;
Eterm res;
@@ -483,7 +483,7 @@ erts_check_process_code(Process *c_p, Eterm module, Uint flags, int *redsp)
return am_false;
erts_rlock_old_code(code_ix);
res = (!modp->old.code_hdr ? am_false :
- check_process_code(c_p, modp, flags, redsp));
+ check_process_code(c_p, modp, flags, redsp, fcalls));
erts_runlock_old_code(code_ix);
return res;
@@ -506,7 +506,7 @@ BIF_RETTYPE erts_internal_check_process_code_2(BIF_ALIST_2)
goto badarg;
}
- res = erts_check_process_code(BIF_P, BIF_ARG_1, flags, &reds);
+ res = erts_check_process_code(BIF_P, BIF_ARG_1, flags, &reds, BIF_P->fcalls);
ASSERT(is_value(res));
@@ -753,7 +753,7 @@ check_mod_funs(Process *p, ErlOffHeap *off_heap, char *area, size_t area_size)
static Eterm
-check_process_code(Process* rp, Module* modp, Uint flags, int *redsp)
+check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls)
{
BeamInstr* start;
char* literals;
@@ -955,7 +955,7 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp)
if (need_gc & ERTS_ORDINARY_GC__) {
FLAGS(rp) |= F_NEED_FULLSWEEP;
- *redsp += erts_garbage_collect_nobump(rp, 0, rp->arg_reg, rp->arity);
+ *redsp += erts_garbage_collect_nobump(rp, 0, rp->arg_reg, rp->arity, fcalls);
done_gc |= ERTS_ORDINARY_GC__;
}
if (need_gc & ERTS_LITERAL_GC__) {
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 72526bca5e..59112e1e43 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -308,7 +308,8 @@ void** beam_ops;
if (E - HTOP < (needed + (HeapNeed))) { \
SWAPOUT; \
PROCESS_MAIN_CHK_LOCKS(c_p); \
- FCALLS -= erts_garbage_collect_nobump(c_p, needed + (HeapNeed), reg, (M)); \
+ FCALLS -= erts_garbage_collect_nobump(c_p, needed + (HeapNeed), \
+ reg, (M), FCALLS); \
ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \
PROCESS_MAIN_CHK_LOCKS(c_p); \
SWAPIN; \
@@ -360,7 +361,7 @@ void** beam_ops;
if ((E - HTOP < need) || (MSO(c_p).overhead + (VNh) >= BIN_VHEAP_SZ(c_p))) {\
SWAPOUT; \
PROCESS_MAIN_CHK_LOCKS(c_p); \
- FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live)); \
+ FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live), FCALLS); \
ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \
PROCESS_MAIN_CHK_LOCKS(c_p); \
SWAPIN; \
@@ -381,7 +382,7 @@ void** beam_ops;
if (E - HTOP < need) { \
SWAPOUT; \
PROCESS_MAIN_CHK_LOCKS(c_p); \
- FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live));\
+ FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live), FCALLS); \
ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \
PROCESS_MAIN_CHK_LOCKS(c_p); \
SWAPIN; \
@@ -402,7 +403,7 @@ void** beam_ops;
SWAPOUT; \
reg[Live] = Extra; \
PROCESS_MAIN_CHK_LOCKS(c_p); \
- FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live)+1); \
+ FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live)+1, FCALLS); \
ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \
PROCESS_MAIN_CHK_LOCKS(c_p); \
Extra = reg[Live]; \
@@ -1196,6 +1197,25 @@ init_emulator(void)
#define DTRACE_NIF_RETURN(p, m, f, a) do {} while (0)
#endif /* USE_VM_PROBES */
+#ifdef DEBUG
+#define ERTS_DBG_CHK_REDS(P, FC) \
+ do { \
+ if (ERTS_PROC_GET_SAVED_CALLS_BUF((P))) { \
+ ASSERT(FC <= 0); \
+ ASSERT(ERTS_PROC_GET_SCHDATA(c_p)->virtual_reds \
+ <= 0 - (FC)); \
+ } \
+ else { \
+ ASSERT(FC <= CONTEXT_REDS); \
+ ASSERT(ERTS_PROC_GET_SCHDATA(c_p)->virtual_reds \
+ <= CONTEXT_REDS - (FC)); \
+ } \
+} while (0)
+#else
+#define ERTS_DBG_CHK_REDS(P, FC)
+#endif
+
+
/*
* process_main() is called twice:
* The first call performs some initialisation, including exporting
@@ -1290,7 +1310,12 @@ void process_main(void)
goto do_schedule1;
do_schedule:
- reds_used = REDS_IN(c_p) - FCALLS;
+ ASSERT(c_p->debug_reds_in == REDS_IN(c_p));
+ if (!ERTS_PROC_GET_SAVED_CALLS_BUF(c_p))
+ reds_used = REDS_IN(c_p) - FCALLS;
+ else
+ reds_used = REDS_IN(c_p) - (CONTEXT_REDS + FCALLS);
+ ASSERT(reds_used >= 0);
do_schedule1:
if (start_time != 0) {
@@ -1310,6 +1335,7 @@ void process_main(void)
ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
c_p = schedule(c_p, reds_used);
+ ASSERT(!(c_p->flags & F_HIPE_MODE));
ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
start_time = 0;
#ifdef DEBUG
@@ -1348,16 +1374,21 @@ void process_main(void)
SET_I(c_p->i);
- reds = c_p->fcalls;
- if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)
- && (ERTS_TRACE_FLAGS(c_p) & F_SENSITIVE) == 0) {
- neg_o_reds = -reds;
- FCALLS = REDS_IN(c_p) = 0;
+ REDS_IN(c_p) = reds = c_p->fcalls;
+#ifdef DEBUG
+ c_p->debug_reds_in = reds;
+#endif
+
+ if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) {
+ neg_o_reds = -CONTEXT_REDS;
+ FCALLS = neg_o_reds + reds;
} else {
neg_o_reds = 0;
- FCALLS = REDS_IN(c_p) = reds;
+ FCALLS = reds;
}
+ ERTS_DBG_CHK_REDS(c_p, FCALLS);
+
next = (BeamInstr *) *I;
SWAPIN;
ASSERT(VALID_INSTR(next));
@@ -1770,7 +1801,7 @@ void process_main(void)
if (E - HTOP < 3) {
SWAPOUT;
PROCESS_MAIN_CHK_LOCKS(c_p);
- FCALLS -= erts_garbage_collect_nobump(c_p, 3, reg+2, 1);
+ FCALLS -= erts_garbage_collect_nobump(c_p, 3, reg+2, 1, FCALLS);
ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
PROCESS_MAIN_CHK_LOCKS(c_p);
SWAPIN;
@@ -1865,6 +1896,7 @@ void process_main(void)
c_p->flags |= F_DELAY_GC;
loop_rec__:
+
PROCESS_MAIN_CHK_LOCKS(c_p);
msgp = PEEK_MESSAGE(c_p);
@@ -2007,6 +2039,8 @@ void process_main(void)
ERTS_VBUMP_LEAVE_REDS_INTERNAL(c_p, 5, FCALLS);
}
+ ERTS_DBG_CHK_REDS(c_p, FCALLS);
+
ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
PROCESS_MAIN_CHK_LOCKS(c_p);
NextPF(0, next);
@@ -2535,6 +2569,7 @@ do { \
GetArg1(2, tmp_reg[0]);
bf = (BifFunction) Arg(1);
+ ERTS_DBG_CHK_REDS(c_p, FCALLS);
c_p->fcalls = FCALLS;
PROCESS_MAIN_CHK_LOCKS(c_p);
ASSERT(!ERTS_PROC_IS_EXITING(c_p));
@@ -2544,6 +2579,7 @@ do { \
PROCESS_MAIN_CHK_LOCKS(c_p);
ERTS_HOLE_CHECK(c_p);
FCALLS = c_p->fcalls;
+ ERTS_DBG_CHK_REDS(c_p, FCALLS);
if (is_value(result)) {
StoreBifResult(3, result);
}
@@ -2564,6 +2600,7 @@ do { \
GetArg1(1, tmp_reg[0]);
bf = (BifFunction) Arg(0);
+ ERTS_DBG_CHK_REDS(c_p, FCALLS);
c_p->fcalls = FCALLS;
PROCESS_MAIN_CHK_LOCKS(c_p);
ASSERT(!ERTS_PROC_IS_EXITING(c_p));
@@ -2573,6 +2610,7 @@ do { \
PROCESS_MAIN_CHK_LOCKS(c_p);
ERTS_HOLE_CHECK(c_p);
FCALLS = c_p->fcalls;
+ ERTS_DBG_CHK_REDS(c_p, FCALLS);
if (is_value(result)) {
StoreBifResult(2, result);
}
@@ -2591,6 +2629,7 @@ do { \
GetArg1(2, x(live));
bf = (GcBifFunction) Arg(1);
+ ERTS_DBG_CHK_REDS(c_p, FCALLS);
c_p->fcalls = FCALLS;
SWAPOUT;
PROCESS_MAIN_CHK_LOCKS(c_p);
@@ -2602,6 +2641,7 @@ do { \
SWAPIN;
ERTS_HOLE_CHECK(c_p);
FCALLS = c_p->fcalls;
+ ERTS_DBG_CHK_REDS(c_p, FCALLS);
if (is_value(result)) {
StoreBifResult(4, result);
}
@@ -2630,6 +2670,7 @@ do { \
*/
live++;
bf = (GcBifFunction) Arg(1);
+ ERTS_DBG_CHK_REDS(c_p, FCALLS);
c_p->fcalls = FCALLS;
SWAPOUT;
PROCESS_MAIN_CHK_LOCKS(c_p);
@@ -2641,6 +2682,7 @@ do { \
SWAPIN;
ERTS_HOLE_CHECK(c_p);
FCALLS = c_p->fcalls;
+ ERTS_DBG_CHK_REDS(c_p, FCALLS);
if (is_value(result)) {
StoreBifResult(5, result);
}
@@ -2671,6 +2713,7 @@ do { \
*/
live += 2;
bf = (GcBifFunction) Arg(1);
+ ERTS_DBG_CHK_REDS(c_p, FCALLS);
c_p->fcalls = FCALLS;
SWAPOUT;
PROCESS_MAIN_CHK_LOCKS(c_p);
@@ -2682,6 +2725,7 @@ do { \
SWAPIN;
ERTS_HOLE_CHECK(c_p);
FCALLS = c_p->fcalls;
+ ERTS_DBG_CHK_REDS(c_p, FCALLS);
if (is_value(result)) {
StoreBifResult(5, result);
}
@@ -2708,6 +2752,7 @@ do { \
GetArg2(2, tmp_reg[0], tmp_reg[1]);
bf = (BifFunction) Arg(1);
+ ERTS_DBG_CHK_REDS(c_p, FCALLS);
c_p->fcalls = FCALLS;
PROCESS_MAIN_CHK_LOCKS(c_p);
ASSERT(!ERTS_PROC_IS_EXITING(c_p));
@@ -2717,6 +2762,7 @@ do { \
PROCESS_MAIN_CHK_LOCKS(c_p);
ERTS_HOLE_CHECK(c_p);
FCALLS = c_p->fcalls;
+ ERTS_DBG_CHK_REDS(c_p, FCALLS);
if (is_value(result)) {
StoreBifResult(4, result);
}
@@ -2775,6 +2821,7 @@ do { \
bf = GET_BIF_ADDRESS(Arg(0));
PRE_BIF_SWAPOUT(c_p);
+ ERTS_DBG_CHK_REDS(c_p, FCALLS);
c_p->fcalls = FCALLS - 1;
if (FCALLS <= 0) {
save_calls(c_p, (Export *) Arg(0));
@@ -2796,6 +2843,7 @@ do { \
PROCESS_MAIN_CHK_LOCKS(c_p);
HTOP = HEAP_TOP(c_p);
FCALLS = c_p->fcalls;
+ ERTS_DBG_CHK_REDS(c_p, FCALLS);
/* We have to update the cache if we are enabled in order
to make sure no book keeping is done after we disabled
msacc. We don't always do this as it is quite expensive. */
@@ -3324,8 +3372,13 @@ do { \
* (beacuse the code for the Dispatch() macro becomes shorter that way).
*/
- reds_used = REDS_IN(c_p) - FCALLS + 1;
-
+ ASSERT(c_p->debug_reds_in == REDS_IN(c_p));
+ if (!ERTS_PROC_GET_SAVED_CALLS_BUF(c_p))
+ reds_used = REDS_IN(c_p) - FCALLS;
+ else
+ reds_used = REDS_IN(c_p) - (CONTEXT_REDS + FCALLS);
+ ASSERT(reds_used >= 0);
+
/*
* Save the argument registers and everything else.
*/
@@ -3514,6 +3567,7 @@ do { \
DTRACE_BIF_ENTRY(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]);
SWAPOUT;
+ ERTS_DBG_CHK_REDS(c_p, FCALLS - 1);
c_p->fcalls = FCALLS - 1;
vbf = (BifFunction) Arg(0);
PROCESS_MAIN_CHK_LOCKS(c_p);
@@ -3549,6 +3603,7 @@ do { \
}
SWAPIN; /* There might have been a garbage collection. */
FCALLS = c_p->fcalls;
+ ERTS_DBG_CHK_REDS(c_p, FCALLS);
if (is_value(nif_bif_result)) {
r(0) = nif_bif_result;
CHECK_TERM(r(0));
@@ -4611,9 +4666,9 @@ do { \
OpCase(i_generic_breakpoint): {
BeamInstr real_I;
ASSERT(I[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI));
- SWAPOUT;
+ HEAVY_SWAPOUT;
real_I = erts_generic_breakpoint(c_p, I, reg);
- SWAPIN;
+ HEAVY_SWAPIN;
ASSERT(VALID_INSTR(real_I));
Goto(real_I);
}
@@ -4790,6 +4845,7 @@ do { \
{
#define HIPE_MODE_SWITCH(Cmd) \
SWAPOUT; \
+ ERTS_DBG_CHK_REDS(c_p, FCALLS); \
c_p->fcalls = FCALLS; \
c_p->def_arg_reg[4] = -neg_o_reds; \
c_p = hipe_mode_switch(c_p, Cmd, reg); \
@@ -4828,6 +4884,9 @@ do { \
#undef HIPE_MODE_SWITCH
L_post_hipe_mode_switch:
+#ifdef DEBUG
+ pid = c_p->common.id; /* may have switched process... */
+#endif
reg = ERTS_PROC_GET_SCHDATA(c_p)->x_reg_array;
freg = ERTS_PROC_GET_SCHDATA(c_p)->f_reg_array;
ERL_BITS_RELOAD_STATEP(c_p);
@@ -4835,6 +4894,7 @@ do { \
neg_o_reds = -c_p->def_arg_reg[4];
FCALLS = c_p->fcalls;
SWAPIN;
+ ERTS_DBG_CHK_REDS(c_p, FCALLS);
switch( c_p->def_arg_reg[3] ) {
case HIPE_MODE_SWITCH_RES_RETURN:
ASSERT(is_value(reg[0]));
@@ -6839,7 +6899,7 @@ erts_current_reductions(Process *current, Process *p)
if (current != p) {
return 0;
} else if (current->fcalls < 0 && ERTS_PROC_GET_SAVED_CALLS_BUF(current)) {
- return -current->fcalls;
+ return current->fcalls + CONTEXT_REDS;
} else {
return REDS_IN(current) - current->fcalls;
}
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index ed5b2983dd..fa5c51ef88 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -1569,7 +1569,32 @@ static BIF_RETTYPE process_flag_aux(Process *BIF_P,
scb->n = 0;
}
- scb = ERTS_PROC_SET_SAVED_CALLS_BUF(rp, scb);
+#ifdef HIPE
+ if (rp->flags & F_HIPE_MODE) {
+ ASSERT(!ERTS_PROC_GET_SAVED_CALLS_BUF(rp));
+ scb = ERTS_PROC_SET_SUSPENDED_SAVED_CALLS_BUF(rp, scb);
+ }
+ else
+#endif
+ {
+#ifdef HIPE
+ ASSERT(!ERTS_PROC_GET_SUSPENDED_SAVED_CALLS_BUF(rp));
+#endif
+ scb = ERTS_PROC_SET_SAVED_CALLS_BUF(rp, scb);
+ if (rp == BIF_P && ((scb && i == 0) || (!scb && i != 0))) {
+ /* Adjust fcalls to match save calls setting... */
+ if (i == 0)
+ BIF_P->fcalls += CONTEXT_REDS; /* disabled it */
+ else
+ BIF_P->fcalls -= CONTEXT_REDS; /* enabled it */
+
+ /*
+ * Make sure we reschedule immediately so the
+ * change take effect at once.
+ */
+ ERTS_VBUMP_ALL_REDS(BIF_P);
+ }
+ }
if (!scb)
old_value = make_small(0);
@@ -1578,12 +1603,7 @@ static BIF_RETTYPE process_flag_aux(Process *BIF_P,
erts_free(ERTS_ALC_T_CALLS_BUF, (void *) scb);
}
- /* Make sure the process in question is rescheduled
- immediately, if it's us, so the call saving takes effect. */
- if (rp == BIF_P)
- BIF_RET2(old_value, CONTEXT_REDS);
- else
- BIF_RET(old_value);
+ BIF_RET(old_value);
}
error:
@@ -1775,10 +1795,18 @@ BIF_RETTYPE process_flag_3(BIF_ALIST_3)
Process *rp;
Eterm res;
- if ((rp = erts_pid2proc(BIF_P, ERTS_PROC_LOCK_MAIN,
- BIF_ARG_1, ERTS_PROC_LOCK_MAIN)) == NULL) {
+#ifdef ERTS_SMP
+ rp = erts_pid2proc_not_running(BIF_P, ERTS_PROC_LOCK_MAIN,
+ BIF_ARG_1, ERTS_PROC_LOCK_MAIN);
+ if (rp == ERTS_PROC_LOCK_BUSY)
+ ERTS_BIF_YIELD3(bif_export[BIF_process_flag_3], BIF_P,
+ BIF_ARG_1, BIF_ARG_2, BIF_ARG_3);
+#else
+ rp = erts_proc_lookup(BIF_ARG_1);
+#endif
+
+ if (!rp)
BIF_ERROR(BIF_P, BADARG);
- }
res = process_flag_aux(BIF_P, rp, BIF_ARG_2, BIF_ARG_3);
diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h
index 546e3830b9..5d751dd67d 100644
--- a/erts/emulator/beam/bif.h
+++ b/erts/emulator/beam/bif.h
@@ -52,6 +52,7 @@ extern Export *erts_convert_time_unit_trap;
(p)->fcalls = 0; \
else \
(p)->fcalls = -CONTEXT_REDS; \
+ ASSERT(ERTS_BIF_REDS_LEFT((p)) == 0); \
} while(0)
#define ERTS_VBUMP_ALL_REDS_INTERNAL(p, fcalls) \
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 4698458521..5a180d7e48 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -116,7 +116,7 @@ static Eterm *full_sweep_heaps(Process *p,
char *oh, Uint oh_size,
Eterm *objv, int nobj);
static int garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
- int need, Eterm* objv, int nobj);
+ int need, Eterm* objv, int nobj, int fcalls);
static int major_collection(Process* p, ErlHeapFragment *live_hf_end,
int need, Eterm* objv, int nobj, Uint *recl);
static int minor_collection(Process* p, ErlHeapFragment *live_hf_end,
@@ -392,15 +392,15 @@ erts_gc_after_bif_call_lhf(Process* p, ErlHeapFragment *live_hf_end,
regs = ERTS_PROC_GET_SCHDATA(p)->x_reg_array;
}
#endif
- cost = garbage_collect(p, live_hf_end, 0, regs, p->arity);
+ cost = garbage_collect(p, live_hf_end, 0, regs, p->arity, p->fcalls);
} else {
- cost = garbage_collect(p, live_hf_end, 0, regs, arity);
+ cost = garbage_collect(p, live_hf_end, 0, regs, arity, p->fcalls);
}
} else {
Eterm val[1];
val[0] = result;
- cost = garbage_collect(p, live_hf_end, 0, val, 1);
+ cost = garbage_collect(p, live_hf_end, 0, val, 1, p->fcalls);
result = val[0];
}
BUMP_REDS(p, cost);
@@ -431,7 +431,7 @@ static ERTS_INLINE void reset_active_writer(Process *p)
#define ERTS_ABANDON_HEAP_COST 10
static int
-delay_garbage_collection(Process *p, ErlHeapFragment *live_hf_end, int need)
+delay_garbage_collection(Process *p, ErlHeapFragment *live_hf_end, int need, int fcalls)
{
ErlHeapFragment *hfrag;
Eterm *orig_heap, *orig_hend, *orig_htop, *orig_stop;
@@ -506,12 +506,16 @@ delay_garbage_collection(Process *p, ErlHeapFragment *live_hf_end, int need)
/* Make sure that we do a proper GC as soon as possible... */
p->flags |= F_FORCE_GC;
- reds_left = ERTS_BIF_REDS_LEFT(p);
+ reds_left = ERTS_REDS_LEFT(p, fcalls);
+ ASSERT(CONTEXT_REDS - reds_left >= ERTS_PROC_GET_SCHDATA(p)->virtual_reds);
+
if (reds_left > ERTS_ABANDON_HEAP_COST) {
int vreds = reds_left - ERTS_ABANDON_HEAP_COST;
- ERTS_VBUMP_REDS(p, vreds);
+ ERTS_PROC_GET_SCHDATA((p))->virtual_reds += vreds;
}
- return ERTS_ABANDON_HEAP_COST;
+
+ ASSERT(CONTEXT_REDS >= ERTS_PROC_GET_SCHDATA(p)->virtual_reds);
+ return reds_left;
}
static ERTS_FORCE_INLINE Uint
@@ -570,7 +574,7 @@ young_gen_usage(Process *p)
*/
static int
garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
- int need, Eterm* objv, int nobj)
+ int need, Eterm* objv, int nobj, int fcalls)
{
Uint reclaimed_now = 0;
int reds;
@@ -581,8 +585,11 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE);
#endif
+ ASSERT(CONTEXT_REDS - ERTS_REDS_LEFT(p, fcalls)
+ >= ERTS_PROC_GET_SCHDATA(p)->virtual_reds);
+
if (p->flags & (F_DISABLE_GC|F_DELAY_GC))
- return delay_garbage_collection(p, live_hf_end, need);
+ return delay_garbage_collection(p, live_hf_end, need, fcalls);
if (p->abandoned_heap)
live_hf_end = ERTS_INVALID_HFRAG_PTR;
@@ -700,16 +707,23 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
}
int
-erts_garbage_collect_nobump(Process* p, int need, Eterm* objv, int nobj)
+erts_garbage_collect_nobump(Process* p, int need, Eterm* objv, int nobj, int fcalls)
{
- return garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj);
+ int reds = garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj, fcalls);
+ int reds_left = ERTS_REDS_LEFT(p, fcalls);
+ if (reds > reds_left)
+ reds = reds_left;
+ ASSERT(CONTEXT_REDS - (reds_left - reds) >= ERTS_PROC_GET_SCHDATA(p)->virtual_reds);
+ return reds;
}
void
erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj)
{
- int reds = garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj);
+ int reds = garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj, p->fcalls);
BUMP_REDS(p, reds);
+ ASSERT(CONTEXT_REDS - ERTS_BIF_REDS_LEFT(p)
+ >= ERTS_PROC_GET_SCHDATA(p)->virtual_reds);
}
/*
diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h
index 40b7c5d12c..8a6ff99990 100644
--- a/erts/emulator/beam/erl_gc.h
+++ b/erts/emulator/beam/erl_gc.h
@@ -139,7 +139,7 @@ Eterm erts_process_gc_info(struct process*, Uint *, Eterm **);
void erts_gc_info(ErtsGCInfo *gcip);
void erts_init_gc(void);
-int erts_garbage_collect_nobump(struct process*, int, Eterm*, int);
+int erts_garbage_collect_nobump(struct process*, int, Eterm*, int, int);
void erts_garbage_collect(struct process*, int, Eterm*, int);
void erts_garbage_collect_hibernate(struct process* p);
Eterm erts_gc_after_bif_call_lhf(struct process* p, ErlHeapFragment *live_hf_end,
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index dec9bdfedc..4d27530866 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -206,15 +206,15 @@ int erts_no_line_info = 0; /* -L: Don't load line information */
ErtsModifiedTimings erts_modified_timings[] = {
/* 0 */ {make_small(0), CONTEXT_REDS, INPUT_REDUCTIONS},
- /* 1 */ {make_small(0), 2*CONTEXT_REDS, 2*INPUT_REDUCTIONS},
+ /* 1 */ {make_small(0), (3*CONTEXT_REDS)/4, 2*INPUT_REDUCTIONS},
/* 2 */ {make_small(0), CONTEXT_REDS/2, INPUT_REDUCTIONS/2},
- /* 3 */ {make_small(0), 3*CONTEXT_REDS, 3*INPUT_REDUCTIONS},
+ /* 3 */ {make_small(0), (7*CONTEXT_REDS)/8, 3*INPUT_REDUCTIONS},
/* 4 */ {make_small(0), CONTEXT_REDS/3, 3*INPUT_REDUCTIONS},
- /* 5 */ {make_small(0), 4*CONTEXT_REDS, INPUT_REDUCTIONS/2},
+ /* 5 */ {make_small(0), (10*CONTEXT_REDS)/11, INPUT_REDUCTIONS/2},
/* 6 */ {make_small(1), CONTEXT_REDS/4, 2*INPUT_REDUCTIONS},
- /* 7 */ {make_small(1), 5*CONTEXT_REDS, INPUT_REDUCTIONS/3},
+ /* 7 */ {make_small(1), (5*CONTEXT_REDS)/7, INPUT_REDUCTIONS/3},
/* 8 */ {make_small(10), CONTEXT_REDS/5, 3*INPUT_REDUCTIONS},
- /* 9 */ {make_small(10), 6*CONTEXT_REDS, INPUT_REDUCTIONS/4}
+ /* 9 */ {make_small(10), (6*CONTEXT_REDS)/7, INPUT_REDUCTIONS/4}
};
#define ERTS_MODIFIED_TIMING_LEVELS \
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index d485affa3b..c6c3fd64a1 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -9240,6 +9240,21 @@ erts_set_process_priority(Process *p, Eterm value)
}
}
+static int
+scheduler_gc_proc(Process *c_p, int reds_left)
+{
+ int fcalls, reds;
+ if (!ERTS_PROC_GET_SAVED_CALLS_BUF(c_p))
+ fcalls = reds_left;
+ else
+ fcalls = reds_left - CONTEXT_REDS;
+ reds = erts_garbage_collect_nobump(c_p, 0, c_p->arg_reg, c_p->arity, fcalls);
+ ASSERT(reds_left >= reds);
+ return reds;
+}
+
+
+
/*
* schedule() is called from BEAM (process_main()) or HiPE
* (hipe_mode_switch()) when the current process is to be
@@ -9318,6 +9333,7 @@ Process *schedule(Process *p, int calls)
#endif
reds = actual_reds = calls - esdp->virtual_reds;
+ ASSERT(actual_reds >= 0);
if (reds < ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST)
reds = ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST;
esdp->virtual_reds = 0;
@@ -9740,7 +9756,7 @@ Process *schedule(Process *p, int calls)
esdp->current_process = p;
-
+ calls = 0;
reds = context_reds;
#ifdef ERTS_SMP
@@ -9819,9 +9835,7 @@ Process *schedule(Process *p, int calls)
/* Migrate to dirty scheduler... */
sunlock_sched_out_proc:
erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
- p->fcalls = reds;
goto sched_out_proc;
-
}
}
else {
@@ -9856,8 +9870,6 @@ Process *schedule(Process *p, int calls)
#endif /* ERTS_SMP */
- p->fcalls = reds;
-
erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
if (IS_TRACED(p)) {
@@ -9890,14 +9902,15 @@ Process *schedule(Process *p, int calls)
* not allowed to execute system tasks.
*/
if (!(p->flags & F_DELAY_GC)) {
- reds -= execute_sys_tasks(p, &state, reds);
+ int cost = execute_sys_tasks(p, &state, reds);
+ calls += cost;
+ reds -= cost;
if (reds <= 0
#ifdef ERTS_DIRTY_SCHEDULERS
|| ERTS_SCHEDULER_IS_DIRTY(esdp)
|| (state & ERTS_PSFLGS_DIRTY_WORK)
#endif
) {
- p->fcalls = reds;
goto sched_out_proc;
}
}
@@ -9910,8 +9923,9 @@ Process *schedule(Process *p, int calls)
if (((state & (ERTS_PSFLG_SUSPENDED
| ERTS_PSFLG_ACTIVE)) != ERTS_PSFLG_ACTIVE)
- && !(state & ERTS_PSFLG_EXITING))
+ && !(state & ERTS_PSFLG_EXITING)) {
goto sched_out_proc;
+ }
n = e = state;
n &= ~ERTS_PSFLG_RUNNING_SYS;
@@ -9930,11 +9944,11 @@ Process *schedule(Process *p, int calls)
if (ERTS_IS_GC_DESIRED(p)) {
if (!(state & ERTS_PSFLG_EXITING) && !(p->flags & (F_DELAY_GC|F_DISABLE_GC))) {
- reds -= erts_garbage_collect_nobump(p, 0, p->arg_reg, p->arity);
- if (reds <= 0) {
- p->fcalls = reds;
+ int cost = scheduler_gc_proc(p, reds);
+ calls += cost;
+ reds -= cost;
+ if (reds <= 0)
goto sched_out_proc;
- }
}
}
@@ -9942,6 +9956,8 @@ Process *schedule(Process *p, int calls)
free_proxy_proc(proxy_p);
proxy_p = NULL;
}
+
+ p->fcalls = reds;
ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p);
@@ -10211,21 +10227,24 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds)
else {
if (!garbage_collected) {
FLAGS(c_p) |= F_NEED_FULLSWEEP;
- reds -= erts_garbage_collect_nobump(c_p,
- 0,
- c_p->arg_reg,
- c_p->arity);
+ reds -= scheduler_gc_proc(c_p, reds);
garbage_collected = 1;
}
st_res = am_true;
}
break;
case ERTS_PSTT_CPC: {
+ int fcalls;
int cpc_reds = 0;
+ if (!ERTS_PROC_GET_SAVED_CALLS_BUF(c_p))
+ fcalls = reds;
+ else
+ fcalls = reds - CONTEXT_REDS;
st_res = erts_check_process_code(c_p,
st->arg[0],
unsigned_val(st->arg[1]),
- &cpc_reds);
+ &cpc_reds,
+ fcalls);
reds -= cpc_reds;
if (is_non_value(st_res)) {
/* Needed gc, but gc was disabled */
@@ -10257,6 +10276,9 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds)
*statep = state;
+ if (in_reds < reds)
+ return in_reds;
+
return in_reds - reds;
}
@@ -11482,8 +11504,10 @@ delete_process(Process* p)
psd = (ErtsPSD *) erts_smp_atomic_read_nob(&p->psd);
- if (psd)
+ if (psd) {
+ erts_smp_atomic_set_nob(&p->psd, (erts_aint_t) NULL); /* Reduction counting depends on this... */
erts_free(ERTS_ALC_T_PSD, psd);
+ }
/* Clean binaries and funs */
erts_cleanup_offheap(&p->off_heap);
@@ -12577,6 +12601,8 @@ erts_continue_exit_process(Process *p)
dep = (p->flags & F_DISTRIBUTION) ? erts_this_dist_entry : NULL;
scb = ERTS_PROC_SET_SAVED_CALLS_BUF(p, NULL);
+ if (scb)
+ p->fcalls += CONTEXT_REDS; /* Reduction counting depends on this... */
pbt = ERTS_PROC_SET_CALL_TIME(p, NULL);
nif_export = ERTS_PROC_SET_NIF_TRAP_EXPORT(p, NULL);
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 61acf5924b..8c7dfc8d2f 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -805,8 +805,15 @@ erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi)
#define ERTS_PSD_CALL_TIME_BP 3
#define ERTS_PSD_DELAYED_GC_TASK_QS 4
#define ERTS_PSD_NIF_TRAP_EXPORT 5
+#ifdef HIPE
+#define ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF 6
+#endif
+#ifdef HIPE
+#define ERTS_PSD_SIZE 7
+#else
#define ERTS_PSD_SIZE 6
+#endif
typedef struct {
void *data[ERTS_PSD_SIZE];
@@ -1071,6 +1078,10 @@ struct process {
Uint space_verified; /* Avoid HAlloc forcing heap fragments when */
Eterm* space_verified_from; /* we rely on available heap space (TestHeap) */
#endif
+
+#ifdef DEBUG
+ Uint debug_reds_in;
+#endif
};
extern const Process erts_invalid_process;
@@ -1342,6 +1353,7 @@ extern int erts_system_profile_ts_type;
#define F_DELAY_GC (1 << 16) /* Similar to disable GC (see below) */
#define F_SCHDLR_ONLN_WAITQ (1 << 17) /* Process enqueued waiting to change schedulers online */
#define F_HAVE_BLCKD_NMSCHED (1 << 18) /* Process has blocked normal multi-scheduling */
+#define F_HIPE_MODE (1 << 19)
/*
* F_DISABLE_GC and F_DELAY_GC are similar. Both will prevent
@@ -1999,6 +2011,13 @@ erts_psd_set(Process *p, int ix, void *data)
#define ERTS_PROC_SET_NIF_TRAP_EXPORT(P, NTE) \
erts_psd_set((P), ERTS_PSD_NIF_TRAP_EXPORT, (void *) (NTE))
+#ifdef HIPE
+#define ERTS_PROC_GET_SUSPENDED_SAVED_CALLS_BUF(P) \
+ ((struct saved_calls *) erts_psd_get((P), ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF))
+#define ERTS_PROC_SET_SUSPENDED_SAVED_CALLS_BUF(P, SCB) \
+ ((struct saved_calls *) erts_psd_set((P), ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF, (void *) (SCB)))
+#endif
+
ERTS_GLB_INLINE Eterm erts_proc_get_error_handler(Process *p);
ERTS_GLB_INLINE Eterm erts_proc_set_error_handler(Process *p, Eterm handler);
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index bd88769dfc..00e7cb23ba 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -1282,16 +1282,18 @@ trace_proc_spawn(Process *p, Eterm what, Eterm pid,
void save_calls(Process *p, Export *e)
{
- struct saved_calls *scb = ERTS_PROC_GET_SAVED_CALLS_BUF(p);
- if (scb) {
- Export **ct = &scb->ct[0];
- int len = scb->len;
-
- ct[scb->cur] = e;
- if (++scb->cur >= len)
- scb->cur = 0;
- if (scb->n < len)
- scb->n++;
+ if ((ERTS_TRACE_FLAGS(p) & F_SENSITIVE) == 0) {
+ struct saved_calls *scb = ERTS_PROC_GET_SAVED_CALLS_BUF(p);
+ if (scb) {
+ Export **ct = &scb->ct[0];
+ int len = scb->len;
+
+ ct[scb->cur] = e;
+ if (++scb->cur >= len)
+ scb->cur = 0;
+ if (scb->n < len)
+ scb->n++;
+ }
}
}
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index a7bc990deb..49eec44053 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -997,7 +997,7 @@ Eterm erl_is_function(Process* p, Eterm arg1, Eterm arg2);
#define ERTS_CPC_ALLOW_GC (1 << 0)
#define ERTS_CPC_COPY_LITERALS (1 << 1)
#define ERTS_CPC_ALL (ERTS_CPC_ALLOW_GC | ERTS_CPC_COPY_LITERALS)
-Eterm erts_check_process_code(Process *c_p, Eterm module, Uint flags, int *redsp);
+Eterm erts_check_process_code(Process *c_p, Eterm module, Uint flags, int *redsp, int fcalls);
typedef struct {
Eterm *ptr;
generated by cgit v1.2.3 (git 2.25.1) at 2024-12-20 05:03:37 +0000