/* * %CopyrightBegin% * * Copyright Ericsson AB 2001-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * * %CopyrightEnd% */ /* * hipe_mode_switch.c */ #ifdef HAVE_CONFIG_H #include "config.h" #endif #include "sys.h" #include "erl_vm.h" #include "global.h" #include "erl_process.h" #include "beam_load.h" /* which includes beam_opcodes.h */ #include "beam_catches.h" #include "hipe_mode_switch.h" #include "bif.h" #include "error.h" #include "hipe_stack.h" #include "hipe_bif0.h" /* hipe_mfa_info_table_init() */ #if defined(ERTS_ENABLE_LOCK_CHECK) # define ERTS_REQ_PROC_MAIN_LOCK(P) \ if ((P)) erts_proc_lc_require_lock((P), ERTS_PROC_LOCK_MAIN, \ __FILE__, __LINE__) # define ERTS_UNREQ_PROC_MAIN_LOCK(P) \ if ((P)) erts_proc_lc_unrequire_lock((P), ERTS_PROC_LOCK_MAIN) #else # define ERTS_REQ_PROC_MAIN_LOCK(P) # define ERTS_UNREQ_PROC_MAIN_LOCK(P) #endif /* * Internal debug support. * #define HIPE_DEBUG to the desired debug level: * 0 no checks * 1 check PCB consistency at mode-switches * 2 log commands and results at mode-switches * 3 log commands, results, and PCB contents at mode-switches * * TODO: check PCB consistency at native BIF calls */ int hipe_modeswitch_debug = 0; extern BeamInstr beam_exit[]; #define HIPE_DEBUG 0 #if HIPE_DEBUG > 1 /* include DPRINTF() logging */ #define DPRINTF(fmt, args...) \ do { \ if (hipe_modeswitch_debug > 0) { \ printf("%s, line %u: " fmt "\r\n", __FUNCTION__, __LINE__ , ##args); \ fflush(stdout); \ } \ } while (0) static const char *code_str(unsigned code) { static const char *cmd_str[] = { "call from beam", "return from beam", "throw from beam", "resume from beam", "return to beam", "call to beam", "throw to beam", "suspend to beam", "wait from native", "wait_timeout from native", "trap from native", "call closure from beam", "call closure to beam", }; unsigned cmd = code & 0xFF; if (cmd < (sizeof(cmd_str)/sizeof(cmd_str[0]))) return cmd_str[cmd]; else return "???"; } #else /* HIPE_DEBUG > 1 */ #define DPRINTF(fmt, args...) do{}while(0) #endif /* HIPE_DEBUG > 1 */ #if HIPE_DEBUG > 0 /* include HIPE_ASSERT and PCB checking */ static void __noreturn hipe_abort(const char *expr, const char *file, unsigned line) { erts_exit(ERTS_ERROR_EXIT, "ASSERTION FAILED, file %s, line %u: %s\r\n", file, line, expr); } #define HIPE_ASSERT3(expr, file, line) \ do { \ if (!(expr)) \ hipe_abort(#expr, file, line); \ } while (0) #define HIPE_ASSERT(expr) HIPE_ASSERT3(expr, __FILE__, __LINE__) void hipe_check_pcb(Process *p, const char *file, unsigned line) { #if HIPE_DEBUG > 2 if (hipe_modeswitch_debug > 0) { printf("%s, line %u: p %p = {htop %p, stop %p, nstack %p, nsp %p, nstend %p}\r\n", file, line, p, p->htop, p->stop, p->hipe.nstack, p->hipe.nsp, p->hipe.nstend); } #endif HIPE_ASSERT3(p != NULL, file, line); HIPE_ASSERT3(p->htop <= p->stop, file, line); HIPE_ASSERT3(p->hipe.nstack <= p->hipe.nstend, file, line); HIPE_ASSERT3(p->hipe.nsp >= p->hipe.nstack, file, line); HIPE_ASSERT3(p->hipe.nsp <= p->hipe.nstend, file, line); } #define HIPE_CHECK_PCB(P) hipe_check_pcb((P), __FILE__, __LINE__) #else /* HIPE_DEBUG > 0 */ #define HIPE_ASSERT(expr) do{}while(0) #define HIPE_CHECK_PCB(P) do{}while(0) #endif /* HIPE_DEBUG > 0 */ /* ensure that at least nwords words are available on the native stack */ #if defined(__sparc__) #include "hipe_sparc_glue.h" #elif defined(__i386__) #include "hipe_x86_glue.h" #elif defined(__x86_64__) #include "hipe_amd64_glue.h" #elif defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) #include "hipe_ppc_glue.h" #elif defined(__arm__) #include "hipe_arm_glue.h" #endif Uint hipe_beam_pc_return[1]; /* needed in hipe_debug.c */ Uint hipe_beam_pc_throw[1]; /* needed in hipe_debug.c */ Uint hipe_beam_pc_resume[1]; /* needed by hipe_set_timeout() */ Eterm hipe_beam_catch_throw; void hipe_mode_switch_init(void) { hipe_arch_glue_init(); hipe_beam_pc_return[0] = BeamOpCodeAddr(op_hipe_trap_return); hipe_beam_pc_throw[0] = BeamOpCodeAddr(op_hipe_trap_throw); hipe_beam_pc_resume[0] = BeamOpCodeAddr(op_hipe_trap_resume); hipe_beam_catch_throw = make_catch(beam_catches_cons(hipe_beam_pc_throw, BEAM_CATCHES_NIL)); hipe_mfa_info_table_init(); } void hipe_set_call_trap(ErtsCodeInfo* ci, void *nfun, int is_closure) { BeamInstr* bfun = erts_codeinfo_to_code(ci); HIPE_ASSERT(ci->op == BeamOpCode(op_i_func_info_IaaI)); bfun[0] = is_closure ? BeamOpCodeAddr(op_hipe_trap_call_closure) : BeamOpCodeAddr(op_hipe_trap_call); ci->u.ncallee = (void (*)(void)) nfun; } static __inline__ void hipe_push_beam_trap_frame(Process *p, Eterm reg[], unsigned arity) { if (&p->stop[1] < p->hend && p->stop[1] == hipe_beam_catch_throw) { /* Trap frame already reserved */ ASSERT(p->stop[0] == NIL); } else { ASSERT(!(p->flags & F_DISABLE_GC)); if ((p->stop - 2) < p->htop) { DPRINTF("calling gc to increase BEAM stack size"); erts_garbage_collect(p, 2, reg, arity); ASSERT(!((p->stop - 2) < p->htop)); } p->stop -= 2; p->stop[1] = hipe_beam_catch_throw; } p->stop[0] = make_cp(p->cp); ++p->catches; p->cp = hipe_beam_pc_return; } static __inline__ void hipe_pop_beam_trap_frame(Process *p) { ASSERT(p->stop[1] == hipe_beam_catch_throw); p->cp = cp_val(p->stop[0]); --p->catches; p->stop += 2; } Process *hipe_mode_switch(Process *p, unsigned cmd, Eterm reg[]) { unsigned result; Eterm reds_in = p->def_arg_reg[5]; /* * Process is in the normal case scheduled out when reduction * count reach zero. When "save calls" is enabled reduction * count is subtracted with CONTEXT_REDS, i.e. initial reduction * count will be zero or less and process is scheduled out * when -CONTEXT_REDS is reached. * * HiPE does not support the "save calls" feature, so we switch * to using a positive reduction counter when executing in * hipe mode, but need to restore the "save calls" when * returning to beam. We also need to hide the save calls buffer * from BIFs. We do that by moving the saved calls buf to * suspended saved calls buf. * * Beam has initial reduction count in stored in p->def_arg_reg[5]. * * Beam expects -neg_o_reds to be found in p->def_arg_reg[4] * on return to beam. */ { struct saved_calls *scb = ERTS_PROC_SET_SAVED_CALLS_BUF(p, NULL); if (scb) { reds_in += CONTEXT_REDS; p->fcalls += CONTEXT_REDS; ERTS_PROC_SET_SUSPENDED_SAVED_CALLS_BUF(p, scb); } } p->flags |= F_HIPE_MODE; /* inform bifs where we are comming from... */ p->i = NULL; /* Set current_function to undefined. stdlib hibernate tests rely on it. */ p->current = NULL; DPRINTF("cmd == %#x (%s)", cmd, code_str(cmd)); HIPE_CHECK_PCB(p); p->arity = 0; switch (cmd & 0xFF) { case HIPE_MODE_SWITCH_CMD_CALL: { /* BEAM calls a native code function */ unsigned arity = cmd >> 8; /* p->hipe.u.ncallee set in beam_emu */ if (p->cp == hipe_beam_pc_return) { /* Native called BEAM, which now tailcalls native. */ hipe_pop_beam_trap_frame(p); result = hipe_tailcall_to_native(p, arity, reg); break; } DPRINTF("calling %#lx/%u", (long)p->hipe.u.ncallee, arity); result = hipe_call_to_native(p, arity, reg); break; } case HIPE_MODE_SWITCH_CMD_CALL_CLOSURE: { /* BEAM calls a native code closure */ unsigned arity = cmd >> 8; /* #formals + #fvs (closure not counted) */ Eterm fun; ErlFunThing *funp; /* drop the fvs, move the closure, correct arity */ fun = reg[arity]; HIPE_ASSERT(is_fun(fun)); funp = (ErlFunThing*)fun_val(fun); HIPE_ASSERT(funp->num_free <= arity); arity -= funp->num_free; /* arity == #formals */ reg[arity] = fun; ++arity; /* correct for having added the closure */ /* HIPE_ASSERT(p->hipe.u.ncallee == (void(*)(void))funp->native_address); */ /* just like a normal call from now on */ /* p->hipe.u.ncallee set in beam_emu */ if (p->cp == hipe_beam_pc_return) { /* Native called BEAM, which now tailcalls native. */ hipe_pop_beam_trap_frame(p); result = hipe_tailcall_to_native(p, arity, reg); break; } DPRINTF("calling %#lx/%u", (long)p->hipe.u.ncallee, arity); result = hipe_call_to_native(p, arity, reg); break; } case HIPE_MODE_SWITCH_CMD_THROW: { /* BEAM just executed hipe_beam_pc_throw[] */ /* Native called BEAM, which now throws an exception back to native. */ DPRINTF("beam throws freason %#lx fvalue %#lx", p->freason, p->fvalue); hipe_pop_beam_trap_frame(p); do_throw_to_native: p->def_arg_reg[0] = exception_tag[GET_EXC_CLASS(p->freason)]; hipe_find_handler(p); result = hipe_throw_to_native(p); break; } case HIPE_MODE_SWITCH_CMD_RETURN: { /* BEAM just executed hipe_beam_pc_return[] */ /* Native called BEAM, which now returns back to native. */ /* pop trap frame off estack */ hipe_pop_beam_trap_frame(p); p->def_arg_reg[0] = reg[0]; result = hipe_return_to_native(p); break; } do_resume: case HIPE_MODE_SWITCH_CMD_RESUME: { /* BEAM just executed hipe_beam_pc_resume[] */ /* BEAM called native, which suspended. */ if (p->flags & F_TIMO) { /* XXX: The process will immediately execute 'clear_timeout', repeating these two statements. Remove them? */ p->flags &= ~F_TIMO; JOIN_MESSAGE(p); p->def_arg_reg[0] = 0; /* make_small(0)? */ } else p->def_arg_reg[0] = 1; /* make_small(1)? */ result = hipe_return_to_native(p); break; } default: erts_exit(ERTS_ERROR_EXIT, "hipe_mode_switch: cmd %#x\r\n", cmd); } do_return_from_native: DPRINTF("result == %#x (%s)", result, code_str(result)); HIPE_CHECK_PCB(p); switch (result) { case HIPE_MODE_SWITCH_RES_RETURN: { hipe_return_from_native(p); reg[0] = p->def_arg_reg[0]; DPRINTF("returning with r(0) == %#lx", reg[0]); break; } case HIPE_MODE_SWITCH_RES_THROW: { DPRINTF("native throws freason %#lx fvalue %#lx", p->freason, p->fvalue); hipe_throw_from_native(p); break; } case HIPE_MODE_SWITCH_RES_TRAP: { /* * Native code called a BIF, which "failed" with a TRAP to BEAM. * Prior to returning, the BIF stored (see BIF_TRAP): * the callee's address in p->i * the callee's parameters in reg[0..2] * the callee's arity in p->arity (for BEAM gc purposes) * * We need to remove the BIF's parameters from the native * stack: to this end hipe_${ARCH}_glue.S stores the BIF's * arity in p->hipe.narity. * * If the BIF emptied the stack (typically hibernate), p->hipe.nstack * is NULL and there is no need to get rid of stacked parameters. */ unsigned int i, is_recursive = 0; if (p->hipe.nstack != NULL) { ASSERT(p->hipe.nsp != NULL); is_recursive = hipe_trap_from_native_is_recursive(p); } else { /* Some architectures (risc) need this re-reset of nsp as the * BIF wrapper do not detect stack change and causes an obsolete * stack pointer to be saved in p->hipe.nsp before return to us. */ p->hipe.nsp = NULL; } /* Schedule next process if current process was hibernated or is waiting for messages */ if (p->flags & F_HIBERNATE_SCHED) { p->flags &= ~F_HIBERNATE_SCHED; goto do_schedule; } if (!(erts_atomic32_read_acqb(&p->state) & ERTS_PSFLG_ACTIVE)) { for (i = 0; i < p->arity; ++i) p->arg_reg[i] = reg[i]; goto do_schedule; } if (is_recursive) hipe_push_beam_trap_frame(p, reg, p->arity); result = HIPE_MODE_SWITCH_RES_CALL_BEAM; break; } case HIPE_MODE_SWITCH_RES_CALL_EXPORTED: { /* Native code calls or tailcalls BEAM. * * p->hipe.u.callee_exp is the callee's export entry * p->arity is the callee's arity * p->def_arg_reg[] contains the register parameters * p->hipe.nsp[] contains the stacked parameters */ if (hipe_call_from_native_is_recursive(p, reg)) { /* BEAM called native, which now calls BEAM */ hipe_push_beam_trap_frame(p, reg, p->arity); } break; } case HIPE_MODE_SWITCH_RES_CALL_CLOSURE: { /* Native code calls or tailcalls a closure in BEAM * * In native code a call to a closure of arity n looks like * F(A1, ..., AN, Closure), * The BEAM code for a closure expects to get: * F(A1, ..., AN, FV1, ..., FVM, Closure) * (Where Ai is argument i and FVj is free variable j) * * p->hipe.u.closure contains the closure * p->def_arg_reg[] contains the register parameters * p->hipe.nsp[] contains the stacked parameters */ ErlFunThing *closure; unsigned num_free, arity, i, is_recursive; HIPE_ASSERT(is_fun(p->hipe.u.closure)); closure = (ErlFunThing*)fun_val(p->hipe.u.closure); num_free = closure->num_free; arity = closure->fe->arity; /* Store the arity in p->arity for the stack popping. */ /* Note: we already have the closure so only need to move arity values to reg[]. However, there are arity+1 parameters in the native code state that need to be removed. */ p->arity = arity+1; /* +1 for the closure */ /* Get parameters, don't do GC just yet. */ is_recursive = hipe_call_from_native_is_recursive(p, reg); if ((Sint)closure->fe->address[-1] < 0) { /* Unloaded. Let beam_emu.c:call_fun() deal with it. */ result = HIPE_MODE_SWITCH_RES_CALL_CLOSURE; } else { /* The BEAM code is present. Prepare to call it. */ /* Append the free vars after the actual parameters. */ for (i = 0; i < num_free; ++i) reg[arity+i] = closure->env[i]; /* Update arity to reflect the new parameters. */ arity += i; /* Make a call to the closure's BEAM code. */ p->i = closure->fe->address; /* Change result code to the faster plain CALL type. */ result = HIPE_MODE_SWITCH_RES_CALL_BEAM; } /* Append the closure as the last parameter. Don't increment arity. */ reg[arity] = p->hipe.u.closure; if (is_recursive) { /* BEAM called native, which now calls BEAM. Need to put a trap-frame on the beam stack. This may cause GC, which is safe now that the arguments, free vars, and most importantly the closure, all are in reg[]. */ hipe_push_beam_trap_frame(p, reg, arity+1); } break; } case HIPE_MODE_SWITCH_RES_SUSPEND: { p->i = hipe_beam_pc_resume; p->arity = 0; goto do_schedule; } case HIPE_MODE_SWITCH_RES_WAIT: case HIPE_MODE_SWITCH_RES_WAIT_TIMEOUT: { /* same semantics, different debug trace messages */ /* XXX: BEAM has different entries for the locked and unlocked cases. HiPE doesn't, so we must check dynamically. */ if (p->hipe_smp.have_receive_locks) p->hipe_smp.have_receive_locks = 0; else erts_proc_lock(p, ERTS_PROC_LOCKS_MSG_RECEIVE); p->i = hipe_beam_pc_resume; p->arity = 0; erts_atomic32_read_band_relb(&p->state, ~ERTS_PSFLG_ACTIVE); erts_proc_unlock(p, ERTS_PROC_LOCKS_MSG_RECEIVE); do_schedule: { struct saved_calls *scb; scb = ERTS_PROC_SET_SUSPENDED_SAVED_CALLS_BUF(p, NULL); if (scb) ERTS_PROC_SET_SAVED_CALLS_BUF(p, scb); /* The process may have died while it was executing, if so we return out from native code to the interpreter */ if (erts_atomic32_read_nob(&p->state) & ERTS_PSFLG_EXITING) p->i = beam_exit; #ifdef DEBUG ASSERT(p->debug_reds_in == reds_in); #endif p->flags &= ~F_HIPE_MODE; ERTS_UNREQ_PROC_MAIN_LOCK(p); p = erts_schedule(NULL, p, reds_in - p->fcalls); ERTS_REQ_PROC_MAIN_LOCK(p); ASSERT(!(p->flags & F_HIPE_MODE)); p->hipe_smp.have_receive_locks = 0; reg = p->scheduler_data->x_reg_array; } { Eterm *argp; int i; argp = p->arg_reg; for (i = p->arity; --i >= 0;) reg[i] = argp[i]; } { struct saved_calls *scb; reds_in = p->fcalls; p->def_arg_reg[5] = reds_in; #ifdef DEBUG p->debug_reds_in = reds_in; #endif if (p->i == hipe_beam_pc_resume) { p->flags |= F_HIPE_MODE; /* inform bifs where we are comming from... */ scb = ERTS_PROC_SET_SAVED_CALLS_BUF(p, NULL); if (scb) ERTS_PROC_SET_SUSPENDED_SAVED_CALLS_BUF(p, scb); p->i = NULL; p->arity = 0; goto do_resume; } scb = ERTS_PROC_GET_SAVED_CALLS_BUF(p); if (!scb) p->def_arg_reg[4] = 0; else { p->def_arg_reg[4] = CONTEXT_REDS; p->fcalls = -CONTEXT_REDS + reds_in; } } HIPE_CHECK_PCB(p); result = HIPE_MODE_SWITCH_RES_CALL_BEAM; p->def_arg_reg[3] = result; return p; } case HIPE_MODE_SWITCH_RES_APPLY: { Eterm mfa[3], args; unsigned int arity; void *address; hipe_pop_params(p, 3, &mfa[0]); /* Unroll the arglist onto reg[]. */ args = mfa[2]; arity = 0; while (is_list(args)) { if (arity < 255) { reg[arity++] = CAR(list_val(args)); args = CDR(list_val(args)); } else goto do_apply_fail; } if (is_not_nil(args)) goto do_apply_fail; /* find a native code entry point for {M,F,A} for a remote call */ address = hipe_get_remote_na(mfa[0], mfa[1], arity); if (!address) goto do_apply_fail; p->hipe.u.ncallee = (void(*)(void)) address; result = hipe_tailcall_to_native(p, arity, reg); goto do_return_from_native; do_apply_fail: p->freason = BADARG; goto do_throw_to_native; } default: erts_exit(ERTS_ERROR_EXIT, "hipe_mode_switch: result %#x\r\n", result); } { struct saved_calls *scb = ERTS_PROC_SET_SUSPENDED_SAVED_CALLS_BUF(p, NULL); if (!scb) p->def_arg_reg[4] = 0; else { p->def_arg_reg[4] = CONTEXT_REDS; p->fcalls -= CONTEXT_REDS; ERTS_PROC_SET_SAVED_CALLS_BUF(p, scb); } } HIPE_CHECK_PCB(p); p->def_arg_reg[3] = result; #if NR_ARG_REGS > 5 /* * When NR_ARG_REGS > 5, we need to protect the process' input * reduction count (which BEAM stores in def_arg_reg[5]) from * being clobbered by the arch glue code. */ p->def_arg_reg[5] = reds_in; #endif p->flags &= ~F_HIPE_MODE; return p; } #define HIPE_INITIAL_NSTACK_SIZE 128 /* PRE: size is zero or a power of two */ static unsigned hipe_next_nstack_size(unsigned size) { return size ? size * 2 : HIPE_INITIAL_NSTACK_SIZE; } #if 0 && defined(HIPE_NSTACK_GROWS_UP) void hipe_inc_nstack(Process *p) { Eterm *old_nstack = p->hipe.nstack; unsigned old_size = p->hipe.nstend - old_nstack; unsigned new_size = hipe_next_nstack_size(old_size); Eterm *new_nstack = erts_realloc(ERTS_ALC_T_HIPE, (char *) old_nstack, new_size*sizeof(Eterm)); p->hipe.nstend = new_nstack + new_size; if (new_nstack != old_nstack) { p->hipe.nsp = new_nstack + (p->hipe.nsp - old_nstack); p->hipe.nstack = new_nstack; if (p->hipe.nstgraylim) p->hipe.nstgraylim = new_nstack + (p->hipe.nstgraylim - old_nstack); if (p->hipe.nstblacklim) p->hipe.nstblacklim = new_nstack + (p->hipe.nstblacklim - old_nstack); } } #endif #if defined(HIPE_NSTACK_GROWS_DOWN) void hipe_inc_nstack(Process *p) { unsigned old_size = p->hipe.nstend - p->hipe.nstack; unsigned new_size = hipe_next_nstack_size(old_size); Eterm *new_nstack = erts_alloc(ERTS_ALC_T_HIPE_STK, new_size*sizeof(Eterm)); unsigned used_size = p->hipe.nstend - p->hipe.nsp; if (used_size) sys_memcpy(new_nstack+new_size-used_size, p->hipe.nsp, used_size*sizeof(Eterm)); if (p->hipe.nstgraylim) p->hipe.nstgraylim = new_nstack + new_size - (p->hipe.nstend - p->hipe.nstgraylim); if (p->hipe.nstblacklim) p->hipe.nstblacklim = new_nstack + new_size - (p->hipe.nstend - p->hipe.nstblacklim); if (p->hipe.nstack) erts_free(ERTS_ALC_T_HIPE_STK, p->hipe.nstack); p->hipe.nstack = new_nstack; p->hipe.nstend = new_nstack + new_size; p->hipe.nsp = new_nstack + new_size - used_size; } #endif void hipe_empty_nstack(Process *p) { if (p->hipe.nstack) { erts_free(ERTS_ALC_T_HIPE_STK, p->hipe.nstack); } p->hipe.nstgraylim = NULL; p->hipe.nsp = NULL; p->hipe.nstack = NULL; p->hipe.nstend = NULL; } void hipe_set_closure_stub(ErlFunEntry *fe) { fe->native_address = (Eterm*) hipe_closure_stub_address(fe->arity); } Eterm hipe_build_stacktrace(Process *p, struct StackTrace *s) { int depth, i; Uint heap_size; Eterm *hp, *hp_end, mfa, m, f, head, *next_p, next; const void *ra; unsigned int a; depth = s->depth; if (depth < 1) return NIL; heap_size = 7 * depth; /* each [{M,F,A,[]}|_] is 2+5 == 7 words */ hp = HAlloc(p, heap_size); hp_end = hp + heap_size; head = NIL; next_p = &head; for (i = 0; i < depth; ++i) { ra = (const void*)s->trace[i]; if (!hipe_find_mfa_from_ra(ra, &m, &f, &a)) continue; mfa = TUPLE4(hp, m, f, make_small(a), NIL); hp += 5; next = CONS(hp, mfa, NIL); *next_p = next; next_p = &CDR(list_val(next)); hp += 2; } HRelease(p, hp_end, hp); return head; }