aboutsummaryrefslogtreecommitdiffstats
path: root/erts/emulator/beam/beam_emu.c
diff options
context:
space:
mode:
Diffstat (limited to 'erts/emulator/beam/beam_emu.c')
-rw-r--r--erts/emulator/beam/beam_emu.c237
1 files changed, 133 insertions, 104 deletions
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 2c37dc42b3..60d0008d8f 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -50,15 +50,16 @@
#if defined(NO_JUMP_TABLE)
# define OpCase(OpCode) case op_##OpCode
# define CountCase(OpCode) case op_count_##OpCode
-# define OpCode(OpCode) ((Uint*)op_##OpCode)
-# define Goto(Rel) {Go = (int)(UWord)(Rel); goto emulator_loop;}
-# define LabelAddr(Addr) &&##Addr
+# define IsOpCode(InstrWord, OpCode) (BeamCodeAddr(InstrWord) == (BeamInstr)op_##OpCode)
+# define Goto(Rel) {Go = BeamCodeAddr(Rel); goto emulator_loop;}
+# define GotoPF(Rel) Goto(Rel)
#else
# define OpCase(OpCode) lb_##OpCode
# define CountCase(OpCode) lb_count_##OpCode
-# define Goto(Rel) goto *((void *)Rel)
-# define LabelAddr(Label) &&Label
-# define OpCode(OpCode) (&&lb_##OpCode)
+# define IsOpCode(InstrWord, OpCode) (BeamCodeAddr(InstrWord) == (BeamInstr)&&lb_##OpCode)
+# define Goto(Rel) goto *((void *)BeamCodeAddr(Rel))
+# define GotoPF(Rel) goto *((void *)Rel)
+# define LabelAddr(Label) &&Label
#endif
#ifdef ERTS_ENABLE_LOCK_CHECK
@@ -107,9 +108,7 @@ do { \
# define CHECK_ARGS(T)
#endif
-#ifndef MAX
-#define MAX(x, y) (((x) > (y)) ? (x) : (y))
-#endif
+#define CHECK_ALIGNED(Dst) ASSERT((((Uint)&Dst) & (sizeof(Uint)-1)) == 0)
#define GET_BIF_MODULE(p) (p->info.mfa.module)
#define GET_BIF_FUNCTION(p) (p->info.mfa.function)
@@ -134,11 +133,11 @@ do { \
/* We don't check the range if an ordinary switch is used */
#ifdef NO_JUMP_TABLE
-#define VALID_INSTR(IP) ((UWord)(IP) < (NUMBER_OF_OPCODES*2+10))
+# define VALID_INSTR(IP) (BeamCodeAddr(IP) < (NUMBER_OF_OPCODES*2+10))
#else
-#define VALID_INSTR(IP) \
- ((SWord)LabelAddr(emulator_loop) <= (SWord)(IP) && \
- (SWord)(IP) < (SWord)LabelAddr(end_emulator_loop))
+# define VALID_INSTR(IP) \
+ ((BeamInstr)LabelAddr(emulator_loop) <= BeamCodeAddr(IP) && \
+ BeamCodeAddr(IP) < (BeamInstr)LabelAddr(end_emulator_loop))
#endif /* NO_JUMP_TABLE */
#define SET_CP(p, ip) \
@@ -163,11 +162,6 @@ BeamInstr beam_apply[2];
BeamInstr beam_exit[1];
BeamInstr beam_continue_exit[1];
-BeamInstr* em_call_error_handler;
-BeamInstr* em_apply_bif;
-BeamInstr* em_call_nif;
-BeamInstr* em_call_bif_e;
-
/* NOTE These should be the only variables containing trace instructions.
** Sometimes tests are form the instruction value, and sometimes
@@ -239,16 +233,21 @@ void** beam_ops;
ERTS_UNREQ_PROC_MAIN_LOCK((P))
#define db(N) (N)
+#define fb(N) ((Sint)(Sint32)(N))
+#define jb(N) ((Sint)(Sint32)(N))
#define tb(N) (N)
-#define xb(N) (*(Eterm *) (((unsigned char *)reg) + (N)))
-#define yb(N) (*(Eterm *) (((unsigned char *)E) + (N)))
+#define xb(N) (*ADD_BYTE_OFFSET(reg, N))
+#define yb(N) (*ADD_BYTE_OFFSET(E, N))
#define Sb(N) (*REG_TARGET_PTR(N))
#define lb(N) (*(double *) (((unsigned char *)&(freg[0].fd)) + (N)))
#define Qb(N) (N)
#define Ib(N) (N)
+
#define x(N) reg[N]
#define y(N) E[N]
#define r(N) x(N)
+#define Q(N) (N*sizeof(Eterm *))
+#define l(N) (freg[N].fd)
/*
* Check that we haven't used the reductions and jump to function pointed to by
@@ -257,8 +256,8 @@ void** beam_ops;
#define DispatchMacro() \
do { \
- BeamInstr* dis_next; \
- dis_next = (BeamInstr *) *I; \
+ BeamInstr dis_next; \
+ dis_next = *I; \
CHECK_ARGS(I); \
if (FCALLS > 0 || FCALLS > neg_o_reds) { \
FCALLS--; \
@@ -266,12 +265,12 @@ void** beam_ops;
} else { \
goto context_switch; \
} \
- } while (0)
+ } while (0) \
#define DispatchMacroFun() \
do { \
- BeamInstr* dis_next; \
- dis_next = (BeamInstr *) *I; \
+ BeamInstr dis_next; \
+ dis_next = *I; \
CHECK_ARGS(I); \
if (FCALLS > 0 || FCALLS > neg_o_reds) { \
FCALLS--; \
@@ -281,23 +280,23 @@ void** beam_ops;
} \
} while (0)
-#define DispatchMacrox() \
- do { \
- if (FCALLS > 0) { \
- Eterm* dis_next; \
- SET_I(((Export *) Arg(0))->addressv[erts_active_code_ix()]); \
- dis_next = (Eterm *) *I; \
- FCALLS--; \
- CHECK_ARGS(I); \
- Goto(dis_next); \
- } else if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p) \
- && FCALLS > neg_o_reds) { \
- goto save_calls1; \
- } else { \
- SET_I(((Export *) Arg(0))->addressv[erts_active_code_ix()]); \
- CHECK_ARGS(I); \
- goto context_switch; \
- } \
+#define DispatchMacrox() \
+ do { \
+ if (FCALLS > 0) { \
+ BeamInstr dis_next; \
+ SET_I(((Export *) Arg(0))->addressv[erts_active_code_ix()]); \
+ dis_next = *I; \
+ FCALLS--; \
+ CHECK_ARGS(I); \
+ Goto(dis_next); \
+ } else if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p) \
+ && FCALLS > neg_o_reds) { \
+ goto save_calls1; \
+ } else { \
+ SET_I(((Export *) Arg(0))->addressv[erts_active_code_ix()]); \
+ CHECK_ARGS(I); \
+ goto context_switch; \
+ } \
} while (0)
#ifdef DEBUG
@@ -384,6 +383,7 @@ do { \
* The following functions are called directly by process_main().
* Don't inline them.
*/
+static void init_emulator_finish(void) NOINLINE;
static ErtsCodeMFA *ubif2mfa(void* uf) NOINLINE;
static ErtsCodeMFA *gcbif2mfa(void* gcf) NOINLINE;
static BeamInstr* handle_error(Process* c_p, BeamInstr* pc,
@@ -392,9 +392,8 @@ static BeamInstr* call_error_handler(Process* p, ErtsCodeMFA* mfa,
Eterm* reg, Eterm func) NOINLINE;
static BeamInstr* fixed_apply(Process* p, Eterm* reg, Uint arity,
BeamInstr *I, Uint offs) NOINLINE;
-static BeamInstr* apply(Process* p, Eterm module, Eterm function,
- Eterm args, Eterm* reg,
- BeamInstr *I, Uint offs) NOINLINE;
+static BeamInstr* apply(Process* p, Eterm* reg,
+ BeamInstr *I, Uint offs) NOINLINE;
static BeamInstr* call_fun(Process* p, int arity,
Eterm* reg, Eterm args) NOINLINE;
static BeamInstr* apply_fun(Process* p, Eterm fun,
@@ -439,6 +438,12 @@ init_emulator(void)
# define REG_stop asm("%l3")
# define REG_I asm("%l4")
# define REG_fcalls asm("%l5")
+#elif defined(__GNUC__) && defined(__amd64__) && !defined(DEBUG)
+# define REG_xregs asm("%r12")
+# define REG_htop
+# define REG_stop asm("%r13")
+# define REG_I asm("%rbx")
+# define REG_fcalls asm("%r14")
#else
# define REG_xregs
# define REG_htop
@@ -624,7 +629,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
#ifndef NO_JUMP_TABLE
static void* opcodes[] = { DEFINE_OPCODES };
#else
- int Go;
+ register BeamInstr Go;
#endif
#endif
@@ -644,7 +649,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
* Note: c_p->arity must be set to reflect the number of useful terms in
* c_p->arg_reg before calling the scheduler.
*/
- if (!init_done) {
+ if (ERTS_UNLIKELY(!init_done)) {
/* This should only be reached during the init phase when only the main
* process is running. I.e. there is no race for init_done.
*/
@@ -700,7 +705,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
{
int reds;
Eterm* argp;
- BeamInstr *next;
+ BeamInstr next;
int i;
argp = c_p->arg_reg;
@@ -732,7 +737,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
ERTS_DBG_CHK_REDS(c_p, FCALLS);
- next = (BeamInstr *) *I;
+ next = *I;
SWAPIN;
ASSERT(VALID_INSTR(next));
@@ -870,6 +875,8 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
goto do_schedule1;
}
+#include "beam_warm.h"
+
OpCase(normal_exit): {
SWAPOUT;
c_p->freason = EXC_NORMAL;
@@ -958,9 +965,6 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
init_emulator:
{
- int i;
- Export* ep;
-
#ifndef NO_JUMP_TABLE
#ifdef ERTS_OPCODE_COUNTER_SUPPORT
#ifdef DEBUG
@@ -972,35 +976,8 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
beam_ops = opcodes;
#endif /* ERTS_OPCODE_COUNTER_SUPPORT */
#endif /* NO_JUMP_TABLE */
-
- em_call_error_handler = OpCode(call_error_handler);
- em_apply_bif = OpCode(apply_bif);
- em_call_nif = OpCode(call_nif);
- em_call_bif_e = OpCode(call_bif_e);
-
- beam_apply[0] = (BeamInstr) OpCode(i_apply);
- beam_apply[1] = (BeamInstr) OpCode(normal_exit);
- beam_exit[0] = (BeamInstr) OpCode(error_action_code);
- beam_continue_exit[0] = (BeamInstr) OpCode(continue_exit);
- beam_return_to_trace[0] = (BeamInstr) OpCode(i_return_to_trace);
- beam_return_trace[0] = (BeamInstr) OpCode(return_trace);
- beam_exception_trace[0] = (BeamInstr) OpCode(return_trace); /* UGLY */
- beam_return_time_trace[0] = (BeamInstr) OpCode(i_return_time_trace);
-
- /*
- * Enter all BIFs into the export table.
- */
- for (i = 0; i < BIF_SIZE; i++) {
- ep = erts_export_put(bif_table[i].module,
- bif_table[i].name,
- bif_table[i].arity);
- bif_export[i] = ep;
- ep->beam[0] = (BeamInstr) OpCode(apply_bif);
- ep->beam[1] = (BeamInstr) bif_table[i].f;
- /* XXX: set func info for bifs */
- ep->info.op = (BeamInstr) BeamOp(op_i_func_info_IaaI);
- }
+ init_emulator_finish();
return;
}
#ifdef NO_JUMP_TABLE
@@ -1012,26 +989,71 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
save_calls1:
{
- Eterm* dis_next;
+ BeamInstr dis_next;
save_calls(c_p, (Export *) Arg(0));
SET_I(((Export *) Arg(0))->addressv[erts_active_code_ix()]);
- dis_next = (Eterm *) *I;
+ dis_next = *I;
FCALLS--;
Goto(dis_next);
}
}
/*
+ * One-time initialization of emulator. Does not need to be
+ * in process_main().
+ */
+static void
+init_emulator_finish(void)
+{
+ int i;
+ Export* ep;
+
+#if defined(ARCH_64) && defined(CODE_MODEL_SMALL)
+ for (i = 0; i < NUMBER_OF_OPCODES; i++) {
+ BeamInstr instr = BeamOpCodeAddr(i);
+ if (instr >= (1ull << 32)) {
+ erts_exit(ERTS_ERROR_EXIT,
+ "This run-time was supposed be compiled with all code below 2Gb,\n"
+ "but the instruction '%s' is located at %016lx.\n",
+ opc[i].name, instr);
+ }
+ }
+#endif
+
+ beam_apply[0] = BeamOpCodeAddr(op_i_apply);
+ beam_apply[1] = BeamOpCodeAddr(op_normal_exit);
+ beam_exit[0] = BeamOpCodeAddr(op_error_action_code);
+ beam_continue_exit[0] = BeamOpCodeAddr(op_continue_exit);
+ beam_return_to_trace[0] = BeamOpCodeAddr(op_i_return_to_trace);
+ beam_return_trace[0] = BeamOpCodeAddr(op_return_trace);
+ beam_exception_trace[0] = BeamOpCodeAddr(op_return_trace); /* UGLY */
+ beam_return_time_trace[0] = BeamOpCodeAddr(op_i_return_time_trace);
+
+ /*
+ * Enter all BIFs into the export table.
+ */
+ for (i = 0; i < BIF_SIZE; i++) {
+ ep = erts_export_put(bif_table[i].module,
+ bif_table[i].name,
+ bif_table[i].arity);
+ bif_export[i] = ep;
+ ep->beam[0] = BeamOpCodeAddr(op_apply_bif);
+ ep->beam[1] = (BeamInstr) bif_table[i].f;
+ /* XXX: set func info for bifs */
+ ep->info.op = BeamOpCodeAddr(op_i_func_info_IaaI);
+ }
+}
+
+/*
* erts_dirty_process_main() is what dirty schedulers execute. Since they handle
* only NIF calls they do not need to be able to execute all BEAM
* instructions.
*/
void erts_dirty_process_main(ErtsSchedulerData *esdp)
{
-#ifdef ERTS_DIRTY_SCHEDULERS
Process* c_p = NULL;
ErtsMonotonicTime start_time;
#ifdef DEBUG
@@ -1246,12 +1268,12 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp)
ERTS_UNREQ_PROC_MAIN_LOCK(c_p);
ASSERT(!ERTS_PROC_IS_EXITING(c_p));
- if (em_apply_bif == (BeamInstr *) *I) {
+ if (BeamIsOpCode(*I, op_apply_bif)) {
exiting = erts_call_dirty_bif(esdp, c_p, I, reg);
}
else {
- ASSERT(em_call_nif == (BeamInstr *) *I);
- exiting = erts_call_dirty_nif(esdp, c_p, I, reg);
+ ASSERT(BeamIsOpCode(*I, op_call_nif));
+ exiting = erts_call_dirty_nif(esdp, c_p, I, reg);
}
ASSERT(!(c_p->flags & F_HIBERNATE_SCHED));
@@ -1270,7 +1292,6 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp)
I = c_p->i;
goto context_switch;
}
-#endif /* ERTS_DIRTY_SCHEDULERS */
}
static ErtsCodeMFA *
@@ -2087,8 +2108,8 @@ apply_bif_error_adjustment(Process *p, Export *ep,
* and apply_last_IP.
*/
if (I
- && ep->beam[0] == (BeamInstr) em_apply_bif
- && (ep == bif_export[BIF_error_1]
+ && BeamIsOpCode(ep->beam[0], op_apply_bif)
+ && (ep == bif_export[BIF_error_1]
|| ep == bif_export[BIF_error_2]
|| ep == bif_export[BIF_exit_1]
|| ep == bif_export[BIF_throw_1])) {
@@ -2174,13 +2195,14 @@ apply_bif_error_adjustment(Process *p, Export *ep,
}
static BeamInstr*
-apply(
-Process* p, Eterm module, Eterm function, Eterm args, Eterm* reg,
-BeamInstr *I, Uint stack_offset)
+apply(Process* p, Eterm* reg, BeamInstr *I, Uint stack_offset)
{
int arity;
Export* ep;
Eterm tmp;
+ Eterm module = reg[0];
+ Eterm function = reg[1];
+ Eterm args = reg[2];
/*
* Check the arguments which should be of the form apply(Module,
@@ -2297,8 +2319,9 @@ fixed_apply(Process* p, Eterm* reg, Uint arity,
if (is_not_atom(module)) goto error;
/* Handle apply of apply/3... */
- if (module == am_erlang && function == am_apply && arity == 3)
- return apply(p, reg[0], reg[1], reg[2], reg, I, stack_offset);
+ if (module == am_erlang && function == am_apply && arity == 3) {
+ return apply(p, reg, I, stack_offset);
+ }
/*
* Get the index into the export table, or failing that the export
@@ -2319,10 +2342,13 @@ fixed_apply(Process* p, Eterm* reg, Uint arity,
}
int
-erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* reg)
+erts_hibernate(Process* c_p, Eterm* reg)
{
int arity;
Eterm tmp;
+ Eterm module = reg[0];
+ Eterm function = reg[1];
+ Eterm args = reg[2];
if (is_not_atom(module) || is_not_atom(function)) {
/*
@@ -3173,13 +3199,16 @@ erts_is_builtin(Eterm Mod, Eterm Name, int arity)
Export e;
Export* ep;
- if (Mod == am_erlang && Name == am_apply && arity == 3) {
- /*
- * Special case. apply/3 is built-in (implemented in C),
- * but implemented in a different way than all other
- * BIFs.
- */
- return 1;
+ if (Mod == am_erlang) {
+ /*
+ * Special case for built-in functions that are implemented
+ * as instructions as opposed to SNIFs.
+ */
+ if (Name == am_apply && (arity == 2 || arity == 3)) {
+ return 1;
+ } else if (Name == am_yield && arity == 0) {
+ return 1;
+ }
}
e.info.mfa.module = Mod;
@@ -3189,8 +3218,8 @@ erts_is_builtin(Eterm Mod, Eterm Name, int arity)
if ((ep = export_get(&e)) == NULL) {
return 0;
}
- return ep->addressv[erts_active_code_ix()] == ep->beam
- && (ep->beam[0] == (BeamInstr) em_apply_bif);
+ return ep->addressv[erts_active_code_ix()] == ep->beam &&
+ BeamIsOpCode(ep->beam[0], op_apply_bif);
}
generated by cgit v1.2.3 (git 2.25.1) at 2024-08-29 15:35:37 +0000