aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--erts/emulator/Makefile.in1
-rw-r--r--erts/emulator/beam/arith_instrs.tab62
-rw-r--r--erts/emulator/beam/beam_emu.c30
-rw-r--r--erts/emulator/beam/bif.c6
-rw-r--r--erts/emulator/beam/bif_instrs.tab20
-rw-r--r--erts/emulator/beam/global.h2
-rw-r--r--erts/emulator/beam/instrs.tab27
-rw-r--r--erts/emulator/beam/macros.tab16
-rw-r--r--erts/emulator/beam/msg_instrs.tab12
-rw-r--r--erts/emulator/beam/ops.tab23
-rw-r--r--erts/emulator/beam/trace_instrs.tab2
-rw-r--r--erts/emulator/hipe/hipe_ops.tab3
-rwxr-xr-xerts/emulator/utils/beam_makeops36
13 files changed, 154 insertions, 86 deletions
diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in
index bc7eb72221..cdc14a0c8b 100644
--- a/erts/emulator/Makefile.in
+++ b/erts/emulator/Makefile.in
@@ -545,6 +545,7 @@ OPCODE_TABLES += hipe/hipe_ops.tab hipe/hipe_instrs.tab
endif
$(TTF_DIR)/beam_cold.h \
+$(TTF_DIR)/beam_warm.h \
$(TTF_DIR)/beam_hot.h \
$(TTF_DIR)/beam_opcodes.c \
$(TTF_DIR)/beam_opcodes.h \
diff --git a/erts/emulator/beam/arith_instrs.tab b/erts/emulator/beam/arith_instrs.tab
index 67cd7c6a2a..7c9cd47e28 100644
--- a/erts/emulator/beam/arith_instrs.tab
+++ b/erts/emulator/beam/arith_instrs.tab
@@ -28,7 +28,7 @@ OUTLINED_ARITH_2(Fail, Live, Name, BIF, Op1, Op2, Dst) {
result = erts_gc_$Name (c_p, reg, live);
HEAVY_SWAPIN;
ERTS_HOLE_CHECK(c_p);
- if (is_value(result)) {
+ if (ERTS_LIKELY(is_value(result))) {
$REFRESH_GEN_DEST();
$Dst = result;
$NEXT0();
@@ -49,10 +49,10 @@ plus.fetch(Op1, Op2) {
}
plus.execute(Fail, Live, Dst) {
- if (is_both_small(PlusOp1, PlusOp2)) {
+ if (ERTS_LIKELY(is_both_small(PlusOp1, PlusOp2))) {
Sint i = signed_val(PlusOp1) + signed_val(PlusOp2);
ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i));
- if (MY_IS_SSMALL(i)) {
+ if (ERTS_LIKELY(MY_IS_SSMALL(i))) {
$Dst = make_small(i);
$NEXT0();
}
@@ -72,10 +72,10 @@ minus.fetch(Op1, Op2) {
}
minus.execute(Fail, Live, Dst) {
- if (is_both_small(MinusOp1, MinusOp2)) {
+ if (ERTS_LIKELY(is_both_small(MinusOp1, MinusOp2))) {
Sint i = signed_val(MinusOp1) - signed_val(MinusOp2);
ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i));
- if (MY_IS_SSMALL(i)) {
+ if (ERTS_LIKELY(MY_IS_SSMALL(i))) {
$Dst = make_small(i);
$NEXT0();
}
@@ -98,10 +98,10 @@ increment.fetch(Src) {
increment.execute(IncrementVal, Live, Dst) {
increment_val = $IncrementVal;
- if (is_small(increment_reg_val)) {
+ if (ERTS_LIKELY(is_small(increment_reg_val))) {
Sint i = signed_val(increment_reg_val) + increment_val;
ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i));
- if (MY_IS_SSMALL(i)) {
+ if (ERTS_LIKELY(MY_IS_SSMALL(i))) {
$Dst = make_small(i);
$NEXT0();
}
@@ -113,7 +113,7 @@ increment.execute(IncrementVal, Live, Dst) {
result = erts_gc_mixed_plus(c_p, reg, live);
HEAVY_SWAPIN;
ERTS_HOLE_CHECK(c_p);
- if (is_value(result)) {
+ if (ERTS_LIKELY(is_value(result))) {
$REFRESH_GEN_DEST();
$Dst = result;
$NEXT0();
@@ -137,12 +137,12 @@ i_m_div(Fail, Live, Op1, Op2, Dst) {
i_int_div(Fail, Live, Op1, Op2, Dst) {
Eterm op1 = $Op1;
Eterm op2 = $Op2;
- if (op2 == SMALL_ZERO) {
+ if (ERTS_UNLIKELY(op2 == SMALL_ZERO)) {
c_p->freason = BADARITH;
$BIF_ERROR_ARITY_2($Fail, BIF_intdiv_2, op1, op2);
- } else if (is_both_small(op1, op2)) {
+ } else if (ERTS_LIKELY(is_both_small(op1, op2))) {
Sint ires = signed_val(op1) / signed_val(op2);
- if (MY_IS_SSMALL(ires)) {
+ if (ERTS_LIKELY(MY_IS_SSMALL(ires))) {
$Dst = make_small(ires);
$NEXT0();
}
@@ -162,15 +162,15 @@ rem.fetch(Src1, Src2) {
}
rem.execute(Fail, Live, Dst) {
- if (RemOp2 == SMALL_ZERO) {
- c_p->freason = BADARITH;
- $BIF_ERROR_ARITY_2($Fail, BIF_rem_2, RemOp1, RemOp2);
- } else if (is_both_small(RemOp1, RemOp2)) {
- $Dst = make_small(signed_val(RemOp1) % signed_val(RemOp2));
- $NEXT0();
- } else {
- $OUTLINED_ARITH_2($Fail, $Live, int_rem, BIF_rem_2, RemOp1, RemOp2, $Dst);
- }
+ if (ERTS_UNLIKELY(RemOp2 == SMALL_ZERO)) {
+ c_p->freason = BADARITH;
+ $BIF_ERROR_ARITY_2($Fail, BIF_rem_2, RemOp1, RemOp2);
+ } else if (ERTS_LIKELY(is_both_small(RemOp1, RemOp2))) {
+ $Dst = make_small(signed_val(RemOp1) % signed_val(RemOp2));
+ $NEXT0();
+ } else {
+ $OUTLINED_ARITH_2($Fail, $Live, int_rem, BIF_rem_2, RemOp1, RemOp2, $Dst);
+ }
}
i_band := band.fetch.execute;
@@ -185,7 +185,7 @@ band.fetch(Src1, Src2) {
}
band.execute(Fail, Live, Dst) {
- if (is_both_small(BandOp1, BandOp2)) {
+ if (ERTS_LIKELY(is_both_small(BandOp1, BandOp2))) {
/*
* No need to untag -- TAG & TAG == TAG.
*/
@@ -196,7 +196,7 @@ band.execute(Fail, Live, Dst) {
}
i_bor(Fail, Live, Src1, Src2, Dst) {
- if (is_both_small($Src1, $Src2)) {
+ if (ERTS_LIKELY(is_both_small($Src1, $Src2))) {
/*
* No need to untag -- TAG | TAG == TAG.
*/
@@ -207,7 +207,7 @@ i_bor(Fail, Live, Src1, Src2, Dst) {
}
i_bxor(Fail, Live, Src1, Src2, Dst) {
- if (is_both_small($Src1, $Src2)) {
+ if (ERTS_LIKELY(is_both_small($Src1, $Src2))) {
/*
* TAG ^ TAG == 0.
*
@@ -232,7 +232,7 @@ shift.setup_bsr(Src1, Src2) {
Op1 = $Src1;
Op2 = $Src2;
shift_left_count = 0;
- if (is_small(Op2)) {
+ if (ERTS_LIKELY(is_small(Op2))) {
shift_left_count = -signed_val(Op2);
} else if (is_big(Op2)) {
/*
@@ -248,7 +248,7 @@ shift.setup_bsl(Src1, Src2) {
Op1 = $Src1;
Op2 = $Src2;
shift_left_count = 0;
- if (is_small(Op2)) {
+ if (ERTS_LIKELY(is_small(Op2))) {
shift_left_count = signed_val(Op2);
} else if (is_big(Op2)) {
if (bignum_header_is_neg(*big_val(Op2))) {
@@ -271,10 +271,10 @@ shift.setup_bsl(Src1, Src2) {
shift.execute(Fail, Live, Dst) {
Uint big_words_needed;
- if (is_small(Op1)) {
+ if (ERTS_LIKELY(is_small(Op1))) {
Sint int_res = signed_val(Op1);
- if (shift_left_count == 0 || int_res == 0) {
- if (is_not_integer(Op2)) {
+ if (ERTS_UNLIKELY(shift_left_count == 0 || int_res == 0)) {
+ if (ERTS_UNLIKELY(is_not_integer(Op2))) {
goto shift_error;
}
if (int_res == 0) {
@@ -343,7 +343,7 @@ shift.execute(Fail, Live, Dst) {
HTOP += bignum_header_arity(*HTOP) + 1;
}
HEAP_SPACE_VERIFIED(0);
- if (is_nil(Op1)) {
+ if (ERTS_UNLIKELY(is_nil(Op1))) {
/*
* This result must have been only slighty larger
* than allowed since it wasn't caught by the
@@ -381,7 +381,7 @@ shift.execute(Fail, Live, Dst) {
i_int_bnot(Fail, Src, Live, Dst) {
Eterm bnot_val = $Src;
- if (is_small(bnot_val)) {
+ if (ERTS_LIKELY(is_small(bnot_val))) {
bnot_val = make_small(~signed_val(bnot_val));
} else {
Uint live = $Live;
@@ -390,7 +390,7 @@ i_int_bnot(Fail, Src, Live, Dst) {
bnot_val = erts_gc_bnot(c_p, reg, live);
HEAVY_SWAPIN;
ERTS_HOLE_CHECK(c_p);
- if (is_nil(bnot_val)) {
+ if (ERTS_UNLIKELY(is_nil(bnot_val))) {
$BIF_ERROR_ARITY_1($Fail, BIF_bnot_1, reg[live]);
}
$REFRESH_GEN_DEST();
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 2c37dc42b3..7ef0772703 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -392,9 +392,8 @@ static BeamInstr* call_error_handler(Process* p, ErtsCodeMFA* mfa,
Eterm* reg, Eterm func) NOINLINE;
static BeamInstr* fixed_apply(Process* p, Eterm* reg, Uint arity,
BeamInstr *I, Uint offs) NOINLINE;
-static BeamInstr* apply(Process* p, Eterm module, Eterm function,
- Eterm args, Eterm* reg,
- BeamInstr *I, Uint offs) NOINLINE;
+static BeamInstr* apply(Process* p, Eterm* reg,
+ BeamInstr *I, Uint offs) NOINLINE;
static BeamInstr* call_fun(Process* p, int arity,
Eterm* reg, Eterm args) NOINLINE;
static BeamInstr* apply_fun(Process* p, Eterm fun,
@@ -439,6 +438,12 @@ init_emulator(void)
# define REG_stop asm("%l3")
# define REG_I asm("%l4")
# define REG_fcalls asm("%l5")
+#elif defined(__GNUC__) && defined(__amd64__) && !defined(DEBUG)
+# define REG_xregs asm("%r12")
+# define REG_htop
+# define REG_stop asm("%r13")
+# define REG_I asm("%rbx")
+# define REG_fcalls asm("%r14")
#else
# define REG_xregs
# define REG_htop
@@ -870,6 +875,8 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
goto do_schedule1;
}
+#include "beam_warm.h"
+
OpCase(normal_exit): {
SWAPOUT;
c_p->freason = EXC_NORMAL;
@@ -2174,13 +2181,14 @@ apply_bif_error_adjustment(Process *p, Export *ep,
}
static BeamInstr*
-apply(
-Process* p, Eterm module, Eterm function, Eterm args, Eterm* reg,
-BeamInstr *I, Uint stack_offset)
+apply(Process* p, Eterm* reg, BeamInstr *I, Uint stack_offset)
{
int arity;
Export* ep;
Eterm tmp;
+ Eterm module = reg[0];
+ Eterm function = reg[1];
+ Eterm args = reg[2];
/*
* Check the arguments which should be of the form apply(Module,
@@ -2297,8 +2305,9 @@ fixed_apply(Process* p, Eterm* reg, Uint arity,
if (is_not_atom(module)) goto error;
/* Handle apply of apply/3... */
- if (module == am_erlang && function == am_apply && arity == 3)
- return apply(p, reg[0], reg[1], reg[2], reg, I, stack_offset);
+ if (module == am_erlang && function == am_apply && arity == 3) {
+ return apply(p, reg, I, stack_offset);
+ }
/*
* Get the index into the export table, or failing that the export
@@ -2319,10 +2328,13 @@ fixed_apply(Process* p, Eterm* reg, Uint arity,
}
int
-erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* reg)
+erts_hibernate(Process* c_p, Eterm* reg)
{
int arity;
Eterm tmp;
+ Eterm module = reg[0];
+ Eterm function = reg[1];
+ Eterm args = reg[2];
if (is_not_atom(module) || is_not_atom(function)) {
/*
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index cbbc2e88a1..ded6c6f1a4 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -1250,7 +1250,11 @@ BIF_RETTYPE hibernate_3(BIF_ALIST_3)
*/
Eterm reg[3];
- if (erts_hibernate(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, reg)) {
+ reg[0] = BIF_ARG_1;
+ reg[1] = BIF_ARG_2;
+ reg[2] = BIF_ARG_3;
+
+ if (erts_hibernate(BIF_P, reg)) {
/*
* If hibernate succeeded, TRAP. The process will be wait in a
* hibernated state if its state is inactive (!ERTS_PSFLG_ACTIVE);
diff --git a/erts/emulator/beam/bif_instrs.tab b/erts/emulator/beam/bif_instrs.tab
index 5aa0523e06..3c95113907 100644
--- a/erts/emulator/beam/bif_instrs.tab
+++ b/erts/emulator/beam/bif_instrs.tab
@@ -45,7 +45,7 @@ CALL_GUARD_BIF(BF, TmpReg, Dst) {
ERTS_HOLE_CHECK(c_p);
FCALLS = c_p->fcalls;
ERTS_DBG_CHK_REDS(c_p, FCALLS);
- if (is_value(result)) {
+ if (ERTS_LIKELY(is_value(result))) {
$Dst = result;
$NEXT0();
}
@@ -145,12 +145,12 @@ i_gc_bif1(Fail, Bif, Src, Live, Dst) {
ERTS_HOLE_CHECK(c_p);
FCALLS = c_p->fcalls;
ERTS_DBG_CHK_REDS(c_p, FCALLS);
- if (is_value(result)) {
+ if (ERTS_LIKELY(is_value(result))) {
$REFRESH_GEN_DEST();
$Dst = result;
$NEXT0();
}
- if ($Fail != 0) { /* Handle error in guard. */
+ if (ERTS_LIKELY($Fail != 0)) { /* Handle error in guard. */
$NEXT($Fail);
}
@@ -195,13 +195,13 @@ i_gc_bif2(Fail, Bif, Live, Src1, Src2, Dst) {
ERTS_HOLE_CHECK(c_p);
FCALLS = c_p->fcalls;
ERTS_DBG_CHK_REDS(c_p, FCALLS);
- if (is_value(result)) {
+ if (ERTS_LIKELY(is_value(result))) {
$REFRESH_GEN_DEST();
$Dst = result;
$NEXT0();
}
- if ($Fail != 0) { /* Handle error in guard. */
+ if (ERTS_LIKELY($Fail != 0)) { /* Handle error in guard. */
$NEXT($Fail);
}
@@ -249,14 +249,14 @@ i_gc_bif3(Fail, Bif, Live, Src2, Src3, Dst) {
ERTS_HOLE_CHECK(c_p);
FCALLS = c_p->fcalls;
ERTS_DBG_CHK_REDS(c_p, FCALLS);
- if (is_value(result)) {
+ if (ERTS_LIKELY(is_value(result))) {
$REFRESH_GEN_DEST();
$Dst = result;
$NEXT0();
}
/* Handle error in guard. */
- if ($Fail != 0) {
+ if (ERTS_LIKELY($Fail != 0)) {
$NEXT($Fail);
}
@@ -325,7 +325,7 @@ call_bif(Exp) {
ERTS_MSACC_UPDATE_CACHE_X();
}
ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR);
- if (is_value(result)) {
+ if (ERTS_LIKELY(is_value(result))) {
r(0) = result;
CHECK_TERM(r(0));
$NEXT0();
@@ -370,7 +370,7 @@ send() {
PROCESS_MAIN_CHK_LOCKS(c_p);
HTOP = HEAP_TOP(c_p);
FCALLS = c_p->fcalls;
- if (is_value(result)) {
+ if (ERTS_LIKELY(is_value(result))) {
r(0) = result;
CHECK_TERM(r(0));
} else if (c_p->freason == TRAP) {
@@ -520,7 +520,7 @@ nif_bif.epilogue() {
SWAPIN; /* There might have been a garbage collection. */
FCALLS = c_p->fcalls;
ERTS_DBG_CHK_REDS(c_p, FCALLS);
- if (is_value(nif_bif_result)) {
+ if (ERTS_LIKELY(is_value(nif_bif_result))) {
r(0) = nif_bif_result;
CHECK_TERM(r(0));
SET_I(c_p->cp);
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 27a6202bb7..c72ae32f2c 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1386,7 +1386,7 @@ Uint erts_current_reductions(Process* current, Process *p);
int erts_print_system_version(fmtfn_t to, void *arg, Process *c_p);
-int erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* reg);
+int erts_hibernate(Process* c_p, Eterm* reg);
ERTS_GLB_FORCE_INLINE int erts_is_literal(Eterm tptr, Eterm *ptr);
diff --git a/erts/emulator/beam/instrs.tab b/erts/emulator/beam/instrs.tab
index 1af01e53bd..9c3e615cea 100644
--- a/erts/emulator/beam/instrs.tab
+++ b/erts/emulator/beam/instrs.tab
@@ -159,7 +159,7 @@ i_move_call_ext_last(Dest, StackOffset, Src) {
APPLY(I, Deallocate) {
//| -no_next
HEAVY_SWAPOUT;
- next = apply(c_p, r(0), x(1), x(2), reg, $I, $Deallocate);
+ next = apply(c_p, reg, $I, $Deallocate);
HEAVY_SWAPIN;
}
@@ -171,7 +171,7 @@ HANDLE_APPLY_ERROR() {
i_apply() {
BeamInstr *next;
$APPLY(NULL, 0);
- if (next != NULL) {
+ if (ERTS_LIKELY(next != NULL)) {
$i_call(next);
}
$HANDLE_APPLY_ERROR();
@@ -180,7 +180,7 @@ i_apply() {
i_apply_last(Deallocate) {
BeamInstr *next;
$APPLY(I, $Deallocate);
- if (next != NULL) {
+ if (ERTS_LIKELY(next != NULL)) {
$i_call_last(next, $Deallocate);
}
$HANDLE_APPLY_ERROR();
@@ -189,7 +189,7 @@ i_apply_last(Deallocate) {
i_apply_only() {
BeamInstr *next;
$APPLY(I, 0);
- if (next != NULL) {
+ if (ERTS_LIKELY(next != NULL)) {
$i_call_only(next);
}
$HANDLE_APPLY_ERROR();
@@ -205,7 +205,7 @@ FIXED_APPLY(Arity, I, Deallocate) {
apply(Arity) {
BeamInstr *next;
$FIXED_APPLY($Arity, NULL, 0);
- if (next != NULL) {
+ if (ERTS_LIKELY(next != NULL)) {
$i_call(next);
}
$HANDLE_APPLY_ERROR();
@@ -214,7 +214,7 @@ apply(Arity) {
apply_last(Arity, Deallocate) {
BeamInstr *next;
$FIXED_APPLY($Arity, I, $Deallocate);
- if (next != NULL) {
+ if (ERTS_LIKELY(next != NULL)) {
$i_call_last(next, $Deallocate);
}
$HANDLE_APPLY_ERROR();
@@ -238,7 +238,7 @@ DISPATCH_FUN(I) {
i_apply_fun() {
BeamInstr *next;
$APPLY_FUN();
- if (next != NULL) {
+ if (ERTS_LIKELY(next != NULL)) {
SET_CP(c_p, $NEXT_INSTRUCTION);
$DISPATCH_FUN(next);
}
@@ -248,7 +248,7 @@ i_apply_fun() {
i_apply_fun_last(Deallocate) {
BeamInstr *next;
$APPLY_FUN();
- if (next != NULL) {
+ if (ERTS_LIKELY(next != NULL)) {
$deallocate($Deallocate);
$DISPATCH_FUN(next);
}
@@ -258,7 +258,7 @@ i_apply_fun_last(Deallocate) {
i_apply_fun_only() {
BeamInstr *next;
$APPLY_FUN();
- if (next != NULL) {
+ if (ERTS_LIKELY(next != NULL)) {
$DISPATCH_FUN(next);
}
$HANDLE_APPLY_FUN_ERROR();
@@ -274,7 +274,7 @@ CALL_FUN(Fun) {
i_call_fun(Fun) {
BeamInstr *next;
$CALL_FUN($Fun);
- if (next != NULL) {
+ if (ERTS_LIKELY(next != NULL)) {
SET_CP(c_p, $NEXT_INSTRUCTION);
$DISPATCH_FUN(next);
}
@@ -284,7 +284,7 @@ i_call_fun(Fun) {
i_call_fun_last(Fun, Deallocate) {
BeamInstr *next;
$CALL_FUN($Fun);
- if (next != NULL) {
+ if (ERTS_LIKELY(next != NULL)) {
$deallocate($Deallocate);
$DISPATCH_FUN(next);
}
@@ -378,7 +378,7 @@ element_group.fetch(Src) {
element_group.execute(Fail, Index, Dst) {
element_index = $Index;
- if (is_small(element_index) && is_tuple(element_tuple)) {
+ if (ERTS_LIKELY(is_small(element_index) && is_tuple(element_tuple))) {
Eterm* tp = tuple_val(element_tuple);
if ((signed_val(element_index) >= 1) &&
@@ -402,7 +402,7 @@ fast_element_group.fetch(Src) {
}
fast_element_group.execute(Fail, Index, Dst) {
- if (is_tuple(fast_element_tuple)) {
+ if (ERTS_LIKELY(is_tuple(fast_element_tuple))) {
Eterm* tp = tuple_val(fast_element_tuple);
Eterm pos = $Index; /* Untagged integer >= 1 */
if (pos <= arityval(*tp)) {
@@ -906,5 +906,6 @@ i_raise() {
c_p->freason = PRIMARY_EXCEPTION(s->freason);
}
goto find_func_info;
+ //| -no_next
}
diff --git a/erts/emulator/beam/macros.tab b/erts/emulator/beam/macros.tab
index 41dc761e90..bac96be7d3 100644
--- a/erts/emulator/beam/macros.tab
+++ b/erts/emulator/beam/macros.tab
@@ -42,7 +42,7 @@ JUMP(Fail) {
GC_TEST(Ns, Nh, Live) {
Uint need = $Nh + $Ns;
- if (E - HTOP < need) {
+ if (ERTS_UNLIKELY(E - HTOP < need)) {
SWAPOUT;
PROCESS_MAIN_CHK_LOCKS(c_p);
FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, $Live, FCALLS);
@@ -55,7 +55,7 @@ GC_TEST(Ns, Nh, Live) {
GC_TEST_PRESERVE(NeedHeap, Live, PreserveTerm) {
Uint need = $NeedHeap;
- if (E - HTOP < need) {
+ if (ERTS_UNLIKELY(E - HTOP < need)) {
SWAPOUT;
reg[$Live] = $PreserveTerm;
PROCESS_MAIN_CHK_LOCKS(c_p);
@@ -99,7 +99,13 @@ FAIL_BODY() {
FAIL_HEAD_OR_BODY(Fail) {
//| -no_prefetch
- if ($Fail) {
+
+ /*
+ * In a correctly working program, we expect failures in
+ * guards to be more likely than failures in bodies.
+ */
+
+ if (ERTS_LIKELY($Fail)) {
$FAIL($Fail);
}
goto find_func_info;
@@ -122,7 +128,7 @@ SYSTEM_LIMIT(Fail) {
BIF_ERROR_ARITY_1(Fail, BIF, Op1) {
//| -no_prefetch
- if ($Fail) {
+ if (ERTS_LIKELY($Fail)) {
$FAIL($Fail);
}
reg[0] = $Op1;
@@ -133,7 +139,7 @@ BIF_ERROR_ARITY_1(Fail, BIF, Op1) {
BIF_ERROR_ARITY_2(Fail, BIF, Op1, Op2) {
//| -no_prefetch
- if ($Fail) {
+ if (ERTS_LIKELY($Fail)) {
$FAIL($Fail);
}
reg[0] = $Op1;
diff --git a/erts/emulator/beam/msg_instrs.tab b/erts/emulator/beam/msg_instrs.tab
index 509143268b..093d48c64c 100644
--- a/erts/emulator/beam/msg_instrs.tab
+++ b/erts/emulator/beam/msg_instrs.tab
@@ -66,7 +66,9 @@ i_recv_set() {
if (c_p->msg.mark == (BeamInstr *) ($NEXT_INSTRUCTION)) {
c_p->msg.save = c_p->msg.saved_last;
}
- /* Fall through to the loop_rec/2 instruction */
+ SET_I($NEXT_INSTRUCTION);
+ goto loop_rec_top__;
+ //| -no_next
}
i_loop_rec(Dest) {
@@ -79,6 +81,10 @@ i_loop_rec(Dest) {
ErtsMessage* msgp;
+ /* Entry point from recv_set */
+ loop_rec_top__:
+ ;
+
/*
* We need to disable GC while matching messages
* in the queue. This since messages with data outside
@@ -87,7 +93,8 @@ i_loop_rec(Dest) {
ASSERT(!(c_p->flags & F_DELAY_GC));
c_p->flags |= F_DELAY_GC;
-loop_rec__:
+ /* Entry point from loop_rec_end */
+ loop_rec__:
PROCESS_MAIN_CHK_LOCKS(c_p);
@@ -285,6 +292,7 @@ timeout() {
TIMEOUT_VALUE() {
c_p->freason = EXC_TIMEOUT_VALUE;
goto find_func_info;
+ //| -no_next
}
i_wait_error_locked() {
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index b6e995fdbe..d848c1cceb 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -59,6 +59,7 @@ put_tuple u==0 d => too_old_compiler
# All the other instructions.
#
+%cold
label L
i_func_info I a a I
int_code_end
@@ -68,6 +69,7 @@ i_debug_breakpoint
i_return_time_trace
i_return_to_trace
i_yield
+%hot
return
@@ -217,6 +219,11 @@ is_number Fail Literal=q => move Literal x | is_number Fail x
jump f
+#
+# Expection rasing instructions. Infrequently executed.
+#
+
+%cold
case_end NotInX=cy => move NotInX x | case_end x
badmatch NotInX=cy => move NotInX x | badmatch x
@@ -238,6 +245,12 @@ i_raise
badarg j
system_limit j
+%hot
+
+#
+# Move instructions.
+#
+
move C=cxy x==0 | jump Lbl => move_jump Lbl C
move_jump f ncxy
@@ -404,8 +417,10 @@ wait_timeout_unlocked s f
wait_timeout_locked_int I f
wait_timeout_locked s f
+%cold
i_wait_error
i_wait_error_locked
+%hot
send
@@ -515,6 +530,7 @@ put_list s s d
# Some more only used by the emulator
#
+%cold
normal_exit
continue_exit
apply_bif
@@ -522,6 +538,7 @@ call_nif
call_error_handler
error_action_code
return_trace
+%hot
#
# Instruction transformations & folded instructions.
@@ -939,9 +956,11 @@ i_apply_fun
i_apply_fun_last P
i_apply_fun_only
+%cold
i_hibernate
i_perf_counter
+%hot
call_bif e
@@ -1040,7 +1059,7 @@ func_info M F A => i_func_info u M F A
# New bit syntax matching (R11B).
# ================================================================
-%cold
+%warm
bs_start_match2 Fail=f ica X Y D => jump Fail
bs_start_match2 Fail Bin X Y D => i_bs_start_match2 Bin Fail X Y D
i_bs_start_match2 xy f I I x
@@ -1138,7 +1157,7 @@ i_bs_validate_unicode_retract j s s
#
# Constructing binaries
#
-%cold
+%warm
bs_init2 Fail Sz Words Regs Flags Dst | binary_too_big(Sz) => system_limit Fail
diff --git a/erts/emulator/beam/trace_instrs.tab b/erts/emulator/beam/trace_instrs.tab
index dfd1d16d58..c71f2ef003 100644
--- a/erts/emulator/beam/trace_instrs.tab
+++ b/erts/emulator/beam/trace_instrs.tab
@@ -102,7 +102,7 @@ i_yield() {
i_hibernate() {
HEAVY_SWAPOUT;
- if (erts_hibernate(c_p, r(0), x(1), x(2), reg)) {
+ if (erts_hibernate(c_p, reg)) {
FCALLS = c_p->fcalls;
c_p->flags &= ~F_HIBERNATE_SCHED;
goto do_schedule;
diff --git a/erts/emulator/hipe/hipe_ops.tab b/erts/emulator/hipe/hipe_ops.tab
index 96e4c0da91..19a3820a6a 100644
--- a/erts/emulator/hipe/hipe_ops.tab
+++ b/erts/emulator/hipe/hipe_ops.tab
@@ -23,4 +23,7 @@ hipe_trap_call_closure
hipe_trap_return
hipe_trap_throw
hipe_trap_resume
+
+%cold
hipe_call_count
+%hot
diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops
index 6c54ab3421..bcc92472d4 100755
--- a/erts/emulator/utils/beam_makeops
+++ b/erts/emulator/utils/beam_makeops
@@ -20,13 +20,16 @@
#
use strict;
use vars qw($BEAM_FORMAT_NUMBER);
+use constant COLD => 0;
+use constant WARM => 1;
+use constant HOT => 2;
$BEAM_FORMAT_NUMBER = undef;
my $target = \&emulator_output;
my $outdir = "."; # Directory for output files.
my $verbose = 0;
-my $hot = 1;
+my $hotness = 1;
my $num_file_opcodes = 0;
my $wordsize = 32;
my %defs; # Defines (from command line).
@@ -344,13 +347,16 @@ while (<>) {
}
#
- # Handle %hot/%cold.
+ # Handle %hot, %warm, and %cold.
#
if (/^\%hot/) {
- $hot = 1;
+ $hotness = HOT;
next;
+ } elsif (/^\%warm/) {
+ $hotness = WARM;
+ next;
} elsif (/^\%cold/) {
- $hot = 0;
+ $hotness = COLD;
next;
}
@@ -438,7 +444,7 @@ while (<>) {
if (defined $gen_opnum{$name,$arity} and $obsolete[$gen_opnum{$name,$arity}]) {
error("specific instructions may not be specified for obsolete instructions");
}
- save_specific_ops($name, $arity, $hot, @args);
+ save_specific_ops($name, $arity, $hotness, @args);
if (defined $op_num) {
error("specific instructions must not be numbered");
} elsif (!defined($gen_arity{$name}) && !defined($unnumbered{$name,$arity})) {
@@ -801,12 +807,17 @@ sub emulator_output {
$name = "$outdir/beam_hot.h";
open(STDOUT, ">$name") || die "Failed to open $name for writing: $!\n";
comment('C');
- print_code(1);
+ print_code(HOT);
+
+ $name = "$outdir/beam_warm.h";
+ open(STDOUT, ">$name") || die "Failed to open $name for writing: $!\n";
+ comment('C');
+ print_code(WARM);
$name = "$outdir/beam_cold.h";
open(STDOUT, ">$name") || die "Failed to open $name for writing: $!\n";
comment('C');
- print_code(0);
+ print_code(COLD);
}
sub init_item {
@@ -1039,14 +1050,16 @@ sub combine_micro_instructions {
# Now generate code for each group.
foreach my $group (sort keys %groups) {
- my($code,@labels) = combine_instruction_group($group, @{$groups{$group}});
- push @generated_code, [1,$code,@labels];
+ my($hotness,$code,@labels) =
+ combine_instruction_group($group, @{$groups{$group}});
+ push @generated_code, [$hotness,$code,@labels];
}
}
sub combine_instruction_group {
my($group,@in_instrs) = @_;
my $gcode = ''; # Code for the entire group.
+ my $group_hotness = COLD;
# Get code for the head of the group (if any).
my $head_name = "$group.head";
@@ -1075,7 +1088,8 @@ sub combine_instruction_group {
error("no $specific_key instruction")
unless defined $specific_op_ref;
foreach my $specific_op (@$specific_op_ref) {
- my($name, $hot, @args) = @{$specific_op};
+ my($name, $hotness, @args) = @{$specific_op};
+ $group_hotness = $hotness unless $group_hotness >= $hotness;
my $offset = 0;
my @rest = @args;
my @new_subs;
@@ -1182,7 +1196,7 @@ sub combine_instruction_group {
$offset = $order_to_offset{$slots[$i+1]} if $i < $#slots;
}
- ("{\n$gcode\n}\n\n",@opcase_labels);
+ ($group_hotness,"{\n$gcode\n}\n\n",@opcase_labels);
}
sub micro_label {
generated by cgit v1.2.3 (git 2.25.1) at 2024-12-18 18:27:50 +0000