diff options
Diffstat (limited to 'erts/emulator/beam/beam_emu.c')
| -rw-r--r-- | erts/emulator/beam/beam_emu.c | 3610 |
1 files changed, 33 insertions, 3577 deletions
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index a5a462944b..ff3c725bbc 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -62,17 +62,17 @@ #endif #ifdef ERTS_ENABLE_LOCK_CHECK -# define PROCESS_MAIN_CHK_LOCKS(P) \ -do { \ - if ((P)) \ - erts_proc_lc_chk_only_proc_main((P)); \ - ERTS_LC_ASSERT(!erts_thr_progress_is_blocking()); \ +# define PROCESS_MAIN_CHK_LOCKS(P) \ +do { \ + if ((P)) \ + erts_proc_lc_chk_only_proc_main((P)); \ + ERTS_LC_ASSERT(!erts_thr_progress_is_blocking()); \ } while (0) # define ERTS_REQ_PROC_MAIN_LOCK(P) \ -do { \ - if ((P)) \ - erts_proc_lc_require_lock((P), ERTS_PROC_LOCK_MAIN, \ - __FILE__, __LINE__); \ +do { \ + if ((P)) \ + erts_proc_lc_require_lock((P), ERTS_PROC_LOCK_MAIN, \ + __FILE__, __LINE__); \ } while (0) # define ERTS_UNREQ_PROC_MAIN_LOCK(P) \ do { \ @@ -156,63 +156,8 @@ do { \ #define REG_TARGET_PTR(Target) (((Target) & 1) ? &yb(Target-1) : &xb(Target)) #define REG_TARGET(Target) (*REG_TARGET_PTR(Target)) -/* - * Store a result into a register given a destination descriptor. - */ - -#define StoreResult(Result, DestDesc) \ - do { \ - Eterm stb_reg; \ - stb_reg = (DestDesc); \ - CHECK_TERM(Result); \ - REG_TARGET(stb_reg) = (Result); \ - } while (0) - -/* - * Store a result into a register and execute the next instruction. - * Dst points to the word with a destination descriptor, which MUST - * be just before the next instruction. - */ - -#define StoreBifResult(Dst, Result) \ - do { \ - BeamInstr* stb_next; \ - Eterm stb_reg; \ - stb_reg = Arg(Dst); \ - I += (Dst) + 2; \ - stb_next = (BeamInstr *) *I; \ - CHECK_TERM(Result); \ - REG_TARGET(stb_reg) = (Result); \ - Goto(stb_next); \ - } while (0) - -#define ClauseFail() goto jump_f - -#define SAVE_CP(X) \ - do { \ - *(X) = make_cp(c_p->cp); \ - c_p->cp = 0; \ - } while(0) - -#define RESTORE_CP(X) SET_CP(c_p, (BeamInstr *) cp_val(*(X))) - #define ISCATCHEND(instr) ((Eterm *) *(instr) == OpCode(catch_end_y)) -#define BIF_ERROR_ARITY_1(Op1, BIF) \ - if (Arg(0) != 0) goto jump_f; \ - reg[0] = Op1; \ - SWAPOUT; \ - I = handle_error(c_p, I, reg, &bif_export[BIF]->info.mfa); \ - goto post_error_handling - -#define BIF_ERROR_ARITY_2(Op1, Op2, BIF) \ - if (Arg(0) != 0) goto jump_f; \ - reg[0] = Op1; \ - reg[1] = Op2; \ - SWAPOUT; \ - I = handle_error(c_p, I, reg, &bif_export[BIF]->info.mfa); \ - goto post_error_handling - /* * Special Beam instructions. */ @@ -306,63 +251,6 @@ void** beam_ops; #define y(N) E[N] #define r(N) x(N) -#define TestBinVHeap(VNh, Nh, Live) \ - do { \ - unsigned need = (Nh); \ - if ((E - HTOP < need) || (MSO(c_p).overhead + (VNh) >= BIN_VHEAP_SZ(c_p))) {\ - SWAPOUT; \ - PROCESS_MAIN_CHK_LOCKS(c_p); \ - FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live), FCALLS); \ - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \ - PROCESS_MAIN_CHK_LOCKS(c_p); \ - SWAPIN; \ - } \ - HEAP_SPACE_VERIFIED(need); \ - } while (0) - - - -/* - * Check if Nh words of heap are available; if not, do a garbage collection. - * Live is number of active argument registers to be preserved. - */ - -#define TestHeap(Nh, Live) \ - do { \ - unsigned need = (Nh); \ - if (E - HTOP < need) { \ - SWAPOUT; \ - PROCESS_MAIN_CHK_LOCKS(c_p); \ - FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live), FCALLS); \ - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \ - PROCESS_MAIN_CHK_LOCKS(c_p); \ - SWAPIN; \ - } \ - HEAP_SPACE_VERIFIED(need); \ - } while (0) - -/* - * Check if Nh words of heap are available; if not, do a garbage collection. - * Live is number of active argument registers to be preserved. - * Takes special care to preserve Extra if a garbage collection occurs. - */ - -#define TestHeapPreserve(Nh, Live, Extra) \ - do { \ - unsigned need = (Nh); \ - if (E - HTOP < need) { \ - SWAPOUT; \ - reg[Live] = Extra; \ - PROCESS_MAIN_CHK_LOCKS(c_p); \ - FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live)+1, FCALLS); \ - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \ - PROCESS_MAIN_CHK_LOCKS(c_p); \ - Extra = reg[Live]; \ - SWAPIN; \ - } \ - HEAP_SPACE_VERIFIED(need); \ - } while (0) - /* * Check that we haven't used the reductions and jump to function pointed to by * the I register. If we are out of reductions, do a context switch. @@ -435,12 +323,6 @@ void** beam_ops; ASSERT(VALID_INSTR(*I)); \ Goto(*I) -#define PreFetch(N, Dst) do { Dst = (BeamInstr *) *(I + N + 1); } while (0) -#define NextPF(N, Dst) \ - I += N + 1; \ - ASSERT(VALID_INSTR(Dst)); \ - Goto(Dst) - #define GetR(pos, tr) \ do { \ tr = Arg(pos); \ @@ -456,13 +338,20 @@ void** beam_ops; CHECK_TERM(tr); \ } while (0) -#define GetArg1(N, Dst) GetR((N), Dst) - -#define GetArg2(N, Dst1, Dst2) \ - do { \ - GetR(N, Dst1); \ - GetR((N)+1, Dst2); \ - } while (0) +#define PUT_TERM_REG(term, desc) \ +do { \ + switch (loader_tag(desc)) { \ + case LOADER_X_REG: \ + x(loader_x_reg_index(desc)) = (term); \ + break; \ + case LOADER_Y_REG: \ + y(loader_y_reg_index(desc)) = (term); \ + break; \ + default: \ + ASSERT(0); \ + break; \ + } \ +} while(0) #define DispatchReturn \ do { \ @@ -485,51 +374,6 @@ do { \ # define BEAM_IS_TUPLE(Src) is_boxed(Src) #endif -#if defined(ARCH_64) -#define BsSafeMul(A, B, Fail, Target) \ - do { Uint64 _res = (A) * (B); \ - if (_res / B != A) { Fail; } \ - Target = _res; \ - } while (0) -#else -#define BsSafeMul(A, B, Fail, Target) \ - do { Uint64 _res = (Uint64)(A) * (Uint64)(B); \ - if ((_res >> (8*sizeof(Uint))) != 0) { Fail; } \ - Target = _res; \ - } while (0) -#endif - -#define BsGetFieldSize(Bits, Unit, Fail, Target) \ - do { \ - Sint _signed_size; Uint _uint_size; \ - Uint temp_bits; \ - if (is_small(Bits)) { \ - _signed_size = signed_val(Bits); \ - if (_signed_size < 0) { Fail; } \ - _uint_size = (Uint) _signed_size; \ - } else { \ - if (!term_to_Uint(Bits, &temp_bits)) { Fail; } \ - _uint_size = temp_bits; \ - } \ - BsSafeMul(_uint_size, Unit, Fail, Target); \ - } while (0) - -#define BsGetUncheckedFieldSize(Bits, Unit, Fail, Target) \ - do { \ - Sint _signed_size; Uint _uint_size; \ - Uint temp_bits; \ - if (is_small(Bits)) { \ - _signed_size = signed_val(Bits); \ - if (_signed_size < 0) { Fail; } \ - _uint_size = (Uint) _signed_size; \ - } else { \ - if (!term_to_Uint(Bits, &temp_bits)) { Fail; } \ - _uint_size = (Uint) temp_bits; \ - } \ - Target = _uint_size * Unit; \ - } while (0) - - /* * process_main() is already huge, so we want to avoid inlining * into it. Especially functions that are seldom used. @@ -718,6 +562,13 @@ init_emulator(void) #define ERTS_DBG_CHK_REDS(P, FC) #endif +#ifdef NO_FPE_SIGNALS +# define ERTS_NO_FPE_CHECK_INIT ERTS_FP_CHECK_INIT +# define ERTS_NO_FPE_ERROR ERTS_FP_ERROR +#else +# define ERTS_NO_FPE_CHECK_INIT(p) +# define ERTS_NO_FPE_ERROR(p, a, b) +#endif /* * process_main() is called twice: @@ -781,8 +632,6 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array) #endif #endif - Eterm pt_arity; /* Used by do_put_tuple */ - Uint64 start_time = 0; /* Monitor long schedule */ BeamInstr* start_time_i = NULL; @@ -923,1849 +772,9 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array) #ifdef NO_JUMP_TABLE switch (Go) { #endif -#include "beam_hot.h" - - { - Eterm increment_reg_val; - Eterm increment_val; - Uint live; - Eterm result; - - OpCase(i_increment_rIId): - increment_reg_val = x(0); - I--; - goto do_increment; - - OpCase(i_increment_xIId): - increment_reg_val = xb(Arg(0)); - goto do_increment; - - OpCase(i_increment_yIId): - increment_reg_val = yb(Arg(0)); - goto do_increment; - - do_increment: - increment_val = Arg(1); - if (is_small(increment_reg_val)) { - Sint i = signed_val(increment_reg_val) + increment_val; - ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i)); - if (MY_IS_SSMALL(i)) { - result = make_small(i); - StoreBifResult(3, result); - } - } - - live = Arg(2); - HEAVY_SWAPOUT; - reg[live] = increment_reg_val; - reg[live+1] = make_small(increment_val); - result = erts_gc_mixed_plus(c_p, reg, live); - HEAVY_SWAPIN; - ERTS_HOLE_CHECK(c_p); - if (is_value(result)) { - StoreBifResult(3, result); - } - ASSERT(c_p->freason != BADMATCH || is_value(c_p->fvalue)); - goto find_func_info; - } - -#define DO_OUTLINED_ARITH_2(name, Op1, Op2, BIF)\ - do { \ - Eterm result; \ - Uint live = Arg(1); \ - \ - HEAVY_SWAPOUT; \ - reg[live] = Op1; \ - reg[live+1] = Op2; \ - result = erts_gc_##name(c_p, reg, live); \ - HEAVY_SWAPIN; \ - ERTS_HOLE_CHECK(c_p); \ - if (is_value(result)) { \ - StoreBifResult(4, result); \ - } \ - BIF_ERROR_ARITY_2(reg[live], reg[live+1], BIF);\ - } while (0) - - { - Eterm PlusOp1, PlusOp2; - Eterm result; - - OpCase(i_plus_jIxxd): - PlusOp1 = xb(Arg(2)); - PlusOp2 = xb(Arg(3)); - goto do_plus; - - OpCase(i_plus_jIxyd): - PlusOp1 = xb(Arg(2)); - PlusOp2 = yb(Arg(3)); - goto do_plus; - - OpCase(i_plus_jIssd): - GetArg2(2, PlusOp1, PlusOp2); - goto do_plus; - - do_plus: - if (is_both_small(PlusOp1, PlusOp2)) { - Sint i = signed_val(PlusOp1) + signed_val(PlusOp2); - ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i)); - if (MY_IS_SSMALL(i)) { - result = make_small(i); - StoreBifResult(4, result); - } - } - DO_OUTLINED_ARITH_2(mixed_plus, PlusOp1, PlusOp2, BIF_splus_2); - } - - { - Eterm MinusOp1, MinusOp2; - Eterm result; - - OpCase(i_minus_jIxxd): - MinusOp1 = xb(Arg(2)); - MinusOp2 = xb(Arg(3)); - goto do_minus; - - OpCase(i_minus_jIssd): - GetArg2(2, MinusOp1, MinusOp2); - goto do_minus; - - do_minus: - if (is_both_small(MinusOp1, MinusOp2)) { - Sint i = signed_val(MinusOp1) - signed_val(MinusOp2); - ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i)); - if (MY_IS_SSMALL(i)) { - result = make_small(i); - StoreBifResult(4, result); - } - } - DO_OUTLINED_ARITH_2(mixed_minus, MinusOp1, MinusOp2, BIF_sminus_2); - } - - { - Eterm is_eq_exact_lit_val; - - OpCase(i_is_eq_exact_literal_fxc): - is_eq_exact_lit_val = xb(Arg(1)); - goto do_is_eq_exact_literal; - - OpCase(i_is_eq_exact_literal_fyc): - is_eq_exact_lit_val = yb(Arg(1)); - goto do_is_eq_exact_literal; - - do_is_eq_exact_literal: - if (!eq(Arg(2), is_eq_exact_lit_val)) { - ClauseFail(); - } - Next(3); - } - - { - Eterm is_ne_exact_lit_val; - - OpCase(i_is_ne_exact_literal_fxc): - is_ne_exact_lit_val = xb(Arg(1)); - goto do_is_ne_exact_literal; - - OpCase(i_is_ne_exact_literal_fyc): - is_ne_exact_lit_val = yb(Arg(1)); - goto do_is_ne_exact_literal; - - do_is_ne_exact_literal: - if (eq(Arg(2), is_ne_exact_lit_val)) { - ClauseFail(); - } - Next(3); - } - - OpCase(i_move_call_ext_last_ePc): { - r(0) = Arg(2); - } - /* FALL THROUGH */ - OpCase(i_call_ext_last_eP): - RESTORE_CP(E); - E = ADD_BYTE_OFFSET(E, Arg(1)); - - /* - * Note: The pointer to the export entry is never NULL; if the module - * is not loaded, it points to code which will invoke the error handler - * (see lb_call_error_handler below). - */ - DTRACE_GLOBAL_CALL_FROM_EXPORT(c_p, Arg(0)); - Dispatchx(); - - OpCase(i_move_call_ext_ce): { - r(0) = Arg(0); - I++; - } - /* FALL THROUGH */ - OpCase(i_call_ext_e): - SET_CP(c_p, I+2); - DTRACE_GLOBAL_CALL_FROM_EXPORT(c_p, Arg(0)); - Dispatchx(); - - OpCase(i_move_call_ext_only_ec): { - r(0) = Arg(1); - } - /* FALL THROUGH */ - OpCase(i_call_ext_only_e): - DTRACE_GLOBAL_CALL_FROM_EXPORT(c_p, Arg(0)); - Dispatchx(); - - OpCase(return): { - SET_I(c_p->cp); - DTRACE_RETURN_FROM_PC(c_p); - /* - * We must clear the CP to make sure that a stale value do not - * create a false module dependcy preventing code upgrading. - * It also means that we can use the CP in stack backtraces. - */ - c_p->cp = 0; - CHECK_TERM(r(0)); - HEAP_SPACE_VERIFIED(0); - DispatchReturn; - } - - /* - * Send is almost a standard call-BIF with two arguments, except for: - * 1) It cannot be traced. - * 2) There is no pointer to the send_2 function stored in - * the instruction. - */ - - OpCase(send): { - BeamInstr *next; - Eterm result; - - if (!(FCALLS > 0 || FCALLS > neg_o_reds)) { - /* If we have run out of reductions, we do a context - switch before calling the bif */ - c_p->arity = 2; - c_p->current = NULL; - goto context_switch3; - } - - PRE_BIF_SWAPOUT(c_p); - c_p->fcalls = FCALLS - 1; - result = erl_send(c_p, r(0), x(1)); - PreFetch(0, next); - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - ERTS_REQ_PROC_MAIN_LOCK(c_p); - PROCESS_MAIN_CHK_LOCKS(c_p); - HTOP = HEAP_TOP(c_p); - FCALLS = c_p->fcalls; - if (is_value(result)) { - r(0) = result; - CHECK_TERM(r(0)); - NextPF(0, next); - } else if (c_p->freason == TRAP) { - SET_CP(c_p, I+1); - SET_I(c_p->i); - SWAPIN; - Dispatch(); - } - goto find_func_info; - } - - { - Eterm element_index; - Eterm element_tuple; - - OpCase(i_element_jxsd): - element_tuple = xb(Arg(1)); - goto do_element; - - OpCase(i_element_jysd): - element_tuple = yb(Arg(1)); - goto do_element; - - do_element: - GetArg1(2, element_index); - if (is_small(element_index) && is_tuple(element_tuple)) { - Eterm* tp = tuple_val(element_tuple); - - if ((signed_val(element_index) >= 1) && - (signed_val(element_index) <= arityval(*tp))) { - Eterm result = tp[signed_val(element_index)]; - StoreBifResult(3, result); - } - } - c_p->freason = BADARG; - BIF_ERROR_ARITY_2(element_index, element_tuple, BIF_element_2); - } - - OpCase(badarg_j): - badarg: - c_p->freason = BADARG; - goto lb_Cl_error; - - { - Eterm fast_element_tuple; - - OpCase(i_fast_element_jxId): - fast_element_tuple = xb(Arg(1)); - goto do_fast_element; - - OpCase(i_fast_element_jyId): - fast_element_tuple = yb(Arg(1)); - goto do_fast_element; - - do_fast_element: - if (is_tuple(fast_element_tuple)) { - Eterm* tp = tuple_val(fast_element_tuple); - Eterm pos = Arg(2); /* Untagged integer >= 1 */ - if (pos <= arityval(*tp)) { - Eterm result = tp[pos]; - StoreBifResult(3, result); - } - } - c_p->freason = BADARG; - BIF_ERROR_ARITY_2(make_small(Arg(2)), fast_element_tuple, BIF_element_2); - } - - OpCase(catch_yf): - c_p->catches++; - yb(Arg(0)) = Arg(1); - Next(2); - - OpCase(catch_end_y): { - c_p->catches--; - make_blank(yb(Arg(0))); - if (is_non_value(r(0))) { - c_p->fvalue = NIL; - if (x(1) == am_throw) { - r(0) = x(2); - } else { - if (x(1) == am_error) { - SWAPOUT; - x(2) = add_stacktrace(c_p, x(2), x(3)); - SWAPIN; - } - /* only x(2) is included in the rootset here */ - if (E - HTOP < 3) { - SWAPOUT; - PROCESS_MAIN_CHK_LOCKS(c_p); - FCALLS -= erts_garbage_collect_nobump(c_p, 3, reg+2, 1, FCALLS); - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - PROCESS_MAIN_CHK_LOCKS(c_p); - SWAPIN; - } - r(0) = TUPLE2(HTOP, am_EXIT, x(2)); - HTOP += 3; - } - } - CHECK_TERM(r(0)); - Next(1); - } - - OpCase(try_end_y): { - c_p->catches--; - make_blank(yb(Arg(0))); - if (is_non_value(r(0))) { - c_p->fvalue = NIL; - r(0) = x(1); - x(1) = x(2); - x(2) = x(3); - } - Next(1); - } - - /* - * Skeleton for receive statement: - * - * recv_mark L1 Optional - * call make_ref/monitor Optional - * ... - * recv_set L1 Optional - * L1: <-------------------+ - * <-----------+ | - * | | - * loop_rec L2 ------+---+ | - * ... | | | - * remove_message | | | - * jump L3 | | | - * ... | | | - * loop_rec_end L1 --+ | | - * L2: <---------------+ | - * wait L1 -----------------+ or wait_timeout - * timeout - * - * L3: Code after receive... - * - * - */ - - OpCase(recv_mark_f): { - /* - * Save the current position in message buffer and the - * the label for the loop_rec/2 instruction for the - * the receive statement. - */ - c_p->msg.mark = (BeamInstr *) Arg(0); - c_p->msg.saved_last = c_p->msg.last; - Next(1); - } - - OpCase(i_recv_set): { - /* - * If the mark is valid (points to the loop_rec/2 - * instruction that follows), we know that the saved - * position points to the first message that could - * possibly be matched out. - * - * If the mark is invalid, we do nothing, meaning that - * we will look through all messages in the message queue. - */ - if (c_p->msg.mark == (BeamInstr *) (I+1)) { - c_p->msg.save = c_p->msg.saved_last; - } - I++; - /* Fall through to the loop_rec/2 instruction */ - } - - /* - * Pick up the next message and place it in x(0). - * If no message, jump to a wait or wait_timeout instruction. - */ - OpCase(i_loop_rec_f): - { - BeamInstr *next; - ErtsMessage* msgp; - - /* - * We need to disable GC while matching messages - * in the queue. This since messages with data outside - * the heap will be corrupted by a GC. - */ - ASSERT(!(c_p->flags & F_DELAY_GC)); - c_p->flags |= F_DELAY_GC; - - loop_rec__: - - PROCESS_MAIN_CHK_LOCKS(c_p); - - msgp = PEEK_MESSAGE(c_p); - - if (!msgp) { - erts_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); - /* Make sure messages wont pass exit signals... */ - if (ERTS_PROC_PENDING_EXIT(c_p)) { - erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); - SWAPOUT; - c_p->flags &= ~F_DELAY_GC; - c_p->arity = 0; - goto do_schedule; /* Will be rescheduled for exit */ - } - ERTS_MSGQ_MV_INQ2PRIVQ(c_p); - msgp = PEEK_MESSAGE(c_p); - if (msgp) - erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); - else - { - c_p->flags &= ~F_DELAY_GC; - SET_I((BeamInstr *) Arg(0)); - Goto(*I); /* Jump to a wait or wait_timeout instruction */ - } - } - if (is_non_value(ERL_MESSAGE_TERM(msgp))) { - SWAPOUT; /* erts_decode_dist_message() may write to heap... */ - if (!erts_decode_dist_message(c_p, ERTS_PROC_LOCK_MAIN, msgp, 0)) { - /* - * A corrupt distribution message that we weren't able to decode; - * remove it... - */ - /* No swapin should be needed */ - ASSERT(HTOP == c_p->htop && E == c_p->stop); - /* TODO: Add DTrace probe for this bad message situation? */ - UNLINK_MESSAGE(c_p, msgp); - msgp->next = NULL; - erts_cleanup_messages(msgp); - goto loop_rec__; - } - SWAPIN; - } - PreFetch(1, next); - r(0) = ERL_MESSAGE_TERM(msgp); - NextPF(1, next); - } - - /* - * Remove a (matched) message from the message queue. - */ - OpCase(remove_message): { - BeamInstr *next; - ErtsMessage* msgp; - PROCESS_MAIN_CHK_LOCKS(c_p); - - ERTS_CHK_MBUF_SZ(c_p); - - PreFetch(0, next); - msgp = PEEK_MESSAGE(c_p); - - if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) { - save_calls(c_p, &exp_receive); - } - if (ERL_MESSAGE_TOKEN(msgp) == NIL) { -#ifdef USE_VM_PROBES - if (DT_UTAG(c_p) != NIL) { - if (DT_UTAG_FLAGS(c_p) & DT_UTAG_PERMANENT) { - SEQ_TRACE_TOKEN(c_p) = am_have_dt_utag; - } else { - DT_UTAG(c_p) = NIL; - SEQ_TRACE_TOKEN(c_p) = NIL; - } - } else { -#endif - SEQ_TRACE_TOKEN(c_p) = NIL; -#ifdef USE_VM_PROBES - } - DT_UTAG_FLAGS(c_p) &= ~DT_UTAG_SPREADING; -#endif - } else if (ERL_MESSAGE_TOKEN(msgp) != am_undefined) { - Eterm msg; - SEQ_TRACE_TOKEN(c_p) = ERL_MESSAGE_TOKEN(msgp); -#ifdef USE_VM_PROBES - if (ERL_MESSAGE_TOKEN(msgp) == am_have_dt_utag) { - if (DT_UTAG(c_p) == NIL) { - DT_UTAG(c_p) = ERL_MESSAGE_DT_UTAG(msgp); - } - DT_UTAG_FLAGS(c_p) |= DT_UTAG_SPREADING; - } else { -#endif - ASSERT(is_tuple(SEQ_TRACE_TOKEN(c_p))); - ASSERT(SEQ_TRACE_TOKEN_ARITY(c_p) == 5); - ASSERT(is_small(SEQ_TRACE_TOKEN_SERIAL(c_p))); - ASSERT(is_small(SEQ_TRACE_TOKEN_LASTCNT(c_p))); - ASSERT(is_small(SEQ_TRACE_TOKEN_FLAGS(c_p))); - ASSERT(is_pid(SEQ_TRACE_TOKEN_SENDER(c_p))); - c_p->seq_trace_lastcnt = unsigned_val(SEQ_TRACE_TOKEN_SERIAL(c_p)); - if (c_p->seq_trace_clock < unsigned_val(SEQ_TRACE_TOKEN_SERIAL(c_p))) { - c_p->seq_trace_clock = unsigned_val(SEQ_TRACE_TOKEN_SERIAL(c_p)); - } - msg = ERL_MESSAGE_TERM(msgp); - seq_trace_output(SEQ_TRACE_TOKEN(c_p), msg, SEQ_TRACE_RECEIVE, - c_p->common.id, c_p); -#ifdef USE_VM_PROBES - } -#endif - } -#ifdef USE_VM_PROBES - if (DTRACE_ENABLED(message_receive)) { - Eterm token2 = NIL; - DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE); - Sint tok_label = 0; - Sint tok_lastcnt = 0; - Sint tok_serial = 0; - - dtrace_proc_str(c_p, receiver_name); - token2 = SEQ_TRACE_TOKEN(c_p); - if (have_seqtrace(token2)) { - tok_label = signed_val(SEQ_TRACE_T_LABEL(token2)); - tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token2)); - tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token2)); - } - DTRACE6(message_receive, - receiver_name, size_object(ERL_MESSAGE_TERM(msgp)), - c_p->msg.len - 1, tok_label, tok_lastcnt, tok_serial); - } -#endif - UNLINK_MESSAGE(c_p, msgp); - JOIN_MESSAGE(c_p); - CANCEL_TIMER(c_p); - - erts_save_message_in_proc(c_p, msgp); - c_p->flags &= ~F_DELAY_GC; - - if (ERTS_IS_GC_DESIRED_INTERNAL(c_p, HTOP, E)) { - /* - * We want to GC soon but we leave a few - * reductions giving the message some time - * to turn into garbage. - */ - ERTS_VBUMP_LEAVE_REDS_INTERNAL(c_p, 5, FCALLS); - } - - ERTS_DBG_CHK_REDS(c_p, FCALLS); - ERTS_CHK_MBUF_SZ(c_p); - - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - PROCESS_MAIN_CHK_LOCKS(c_p); - NextPF(0, next); - } - - /* - * Advance the save pointer to the next message (the current - * message didn't match), then jump to the loop_rec instruction. - */ - OpCase(loop_rec_end_f): { - - ASSERT(c_p->flags & F_DELAY_GC); - - SET_I((BeamInstr *) Arg(0)); - SAVE_MESSAGE(c_p); - if (FCALLS > 0 || FCALLS > neg_o_reds) { - FCALLS--; - goto loop_rec__; - } - - c_p->flags &= ~F_DELAY_GC; - c_p->i = I; - SWAPOUT; - c_p->arity = 0; - c_p->current = NULL; - goto do_schedule; - } - /* - * Prepare to wait for a message or a timeout, whichever occurs first. - * - * Note: In order to keep the compatibility between 32 and 64 bits - * emulators, only timeout values that can be represented in 32 bits - * (unsigned) or less are allowed. - */ - - - OpCase(i_wait_timeout_fs): { - erts_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); - - /* Fall through */ - } - OpCase(i_wait_timeout_locked_fs): { - Eterm timeout_value; - - /* - * If we have already set the timer, we must NOT set it again. Therefore, - * we must test the F_INSLPQUEUE flag as well as the F_TIMO flag. - */ - if (c_p->flags & (F_INSLPQUEUE | F_TIMO)) { - goto wait2; - } - GetArg1(1, timeout_value); - if (timeout_value != make_small(0)) { - - if (timeout_value == am_infinity) - c_p->flags |= F_TIMO; - else { - int tres = erts_set_proc_timer_term(c_p, timeout_value); - if (tres == 0) { - /* - * The timer routiner will set c_p->i to the value in - * c_p->def_arg_reg[0]. Note that it is safe to use this - * location because there are no living x registers in - * a receive statement. - */ - BeamInstr** pi = (BeamInstr**) c_p->def_arg_reg; - *pi = I+3; - } - else { /* Wrong time */ - OpCase(i_wait_error_locked): { - erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); - /* Fall through */ - } - OpCase(i_wait_error): { - c_p->freason = EXC_TIMEOUT_VALUE; - goto find_func_info; - } - } - } - - /* - * Prepare to wait indefinitely for a new message to arrive - * (or the time set above if falling through from above). - * - * When a new message arrives, control will be transferred - * the loop_rec instruction (at label L1). In case of - * of timeout, control will be transferred to the timeout - * instruction following the wait_timeout instruction. - */ - - OpCase(wait_locked_f): - OpCase(wait_f): - - wait2: { - c_p->i = (BeamInstr *) Arg(0); /* L1 */ - SWAPOUT; - c_p->arity = 0; - - if (!ERTS_PTMR_IS_TIMED_OUT(c_p)) - erts_atomic32_read_band_relb(&c_p->state, - ~ERTS_PSFLG_ACTIVE); - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); - c_p->current = NULL; - goto do_schedule; - } - OpCase(wait_unlocked_f): { - erts_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); - goto wait2; - } - } - erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); - Next(2); - } - - OpCase(i_wait_timeout_fI): { - erts_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); - } - - OpCase(i_wait_timeout_locked_fI): - { - /* - * If we have already set the timer, we must NOT set it again. Therefore, - * we must test the F_INSLPQUEUE flag as well as the F_TIMO flag. - */ - if ((c_p->flags & (F_INSLPQUEUE | F_TIMO)) == 0) { - BeamInstr** p = (BeamInstr **) c_p->def_arg_reg; - *p = I+3; - erts_set_proc_timer_uword(c_p, Arg(1)); - } - goto wait2; - } - - /* - * A timeout has occurred. Reset the save pointer so that the next - * receive statement will examine the first message first. - */ - OpCase(timeout_locked): { - erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); - } - OpCase(timeout): { - BeamInstr *next; - - PreFetch(0, next); - if (IS_TRACED_FL(c_p, F_TRACE_RECEIVE)) { - trace_receive(c_p, am_clock_service, am_timeout, NULL); - } - if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) { - save_calls(c_p, &exp_timeout); - } - c_p->flags &= ~F_TIMO; - JOIN_MESSAGE(c_p); - NextPF(0, next); - } - - { - Eterm select_val2; - - OpCase(i_select_tuple_arity2_yfAAff): - select_val2 = yb(Arg(0)); - goto do_select_tuple_arity2; - - OpCase(i_select_tuple_arity2_xfAAff): - select_val2 = xb(Arg(0)); - goto do_select_tuple_arity2; - - do_select_tuple_arity2: - if (is_not_tuple(select_val2)) { - goto select_val2_fail; - } - select_val2 = *tuple_val(select_val2); - goto do_select_val2; - - OpCase(i_select_val2_yfccff): - select_val2 = yb(Arg(0)); - goto do_select_val2; - - OpCase(i_select_val2_xfccff): - select_val2 = xb(Arg(0)); - goto do_select_val2; - - do_select_val2: - if (select_val2 == Arg(2)) { - I += 3; - } else if (select_val2 == Arg(3)) { - I += 4; - } - - select_val2_fail: - SET_I((BeamInstr *) Arg(1)); - Goto(*I); - } - - { - Eterm select_val; - - OpCase(i_select_tuple_arity_xfI): - select_val = xb(Arg(0)); - goto do_select_tuple_arity; - - OpCase(i_select_tuple_arity_yfI): - select_val = yb(Arg(0)); - goto do_select_tuple_arity; - - do_select_tuple_arity: - if (is_tuple(select_val)) { - select_val = *tuple_val(select_val); - goto do_linear_search; - } - SET_I((BeamInstr *) Arg(1)); - Goto(*I); - - OpCase(i_select_val_lins_xfI): - select_val = xb(Arg(0)); - goto do_linear_search; - - OpCase(i_select_val_lins_yfI): - select_val = yb(Arg(0)); - goto do_linear_search; - - do_linear_search: { - BeamInstr *vs = &Arg(3); - int ix = 0; - - for(;;) { - if (vs[ix+0] >= select_val) { ix += 0; break; } - if (vs[ix+1] >= select_val) { ix += 1; break; } - ix += 2; - } - - if (vs[ix] == select_val) { - I += ix + Arg(2) + 2; - } - - SET_I((BeamInstr *) Arg(1)); - Goto(*I); - } - - OpCase(i_select_val_bins_xfI): - select_val = xb(Arg(0)); - goto do_binary_search; - - OpCase(i_select_val_bins_yfI): - select_val = yb(Arg(0)); - goto do_binary_search; - - do_binary_search: - { - struct Pairs { - BeamInstr val; - BeamInstr* addr; - }; - struct Pairs* low; - struct Pairs* high; - struct Pairs* mid; - int bdiff; /* int not long because the arrays aren't that large */ - - low = (struct Pairs *) &Arg(3); - high = low + Arg(2); - - /* The pointer subtraction (high-low) below must produce - * a signed result, because high could be < low. That - * requires the compiler to insert quite a bit of code. - * - * However, high will be > low so the result will be - * positive. We can use that knowledge to optimise the - * entire sequence, from the initial comparison to the - * computation of mid. - * - * -- Mikael Pettersson, Acumem AB - * - * Original loop control code: - * - * while (low < high) { - * mid = low + (high-low) / 2; - * - */ - while ((bdiff = (int)((char*)high - (char*)low)) > 0) { - unsigned int boffset = ((unsigned int)bdiff >> 1) & ~(sizeof(struct Pairs)-1); - - mid = (struct Pairs*)((char*)low + boffset); - if (select_val < mid->val) { - high = mid; - } else if (select_val > mid->val) { - low = mid + 1; - } else { - SET_I(mid->addr); - Goto(*I); - } - } - SET_I((BeamInstr *) Arg(1)); - Goto(*I); - } - } - - { - Eterm jump_on_val_zero_index; - - OpCase(i_jump_on_val_zero_yfI): - jump_on_val_zero_index = yb(Arg(0)); - goto do_jump_on_val_zero_index; - - OpCase(i_jump_on_val_zero_xfI): - jump_on_val_zero_index = xb(Arg(0)); - goto do_jump_on_val_zero_index; - - do_jump_on_val_zero_index: - if (is_small(jump_on_val_zero_index)) { - jump_on_val_zero_index = signed_val(jump_on_val_zero_index); - if (jump_on_val_zero_index < Arg(2)) { - SET_I((BeamInstr *) (&Arg(3))[jump_on_val_zero_index]); - Goto(*I); - } - } - SET_I((BeamInstr *) Arg(1)); - Goto(*I); - } - - { - Eterm jump_on_val_index; - - - OpCase(i_jump_on_val_yfII): - jump_on_val_index = yb(Arg(0)); - goto do_jump_on_val_index; - - OpCase(i_jump_on_val_xfII): - jump_on_val_index = xb(Arg(0)); - goto do_jump_on_val_index; - - do_jump_on_val_index: - if (is_small(jump_on_val_index)) { - jump_on_val_index = (Uint) (signed_val(jump_on_val_index) - Arg(3)); - if (jump_on_val_index < Arg(2)) { - SET_I((BeamInstr *) (&Arg(4))[jump_on_val_index]); - Goto(*I); - } - } - SET_I((BeamInstr *) Arg(1)); - Goto(*I); - } - - do_put_tuple: { - Eterm* hp = HTOP; - - *hp++ = make_arityval(pt_arity); - - do { - Eterm term = *I++; - switch (loader_tag(term)) { - case LOADER_X_REG: - *hp++ = x(loader_x_reg_index(term)); - break; - case LOADER_Y_REG: - *hp++ = y(loader_y_reg_index(term)); - break; - default: - *hp++ = term; - break; - } - } while (--pt_arity != 0); - HTOP = hp; - Goto(*I); - } - - OpCase(new_map_dII): { - Eterm res; - - HEAVY_SWAPOUT; - res = new_map(c_p, reg, I-1); - HEAVY_SWAPIN; - StoreResult(res, Arg(0)); - Next(3+Arg(2)); - } - - OpCase(i_new_small_map_lit_dIq): { - Eterm res; - Uint n; - - HEAVY_SWAPOUT; - res = new_small_map_lit(c_p, reg, &n, I-1); - HEAVY_SWAPIN; - StoreResult(res, Arg(0)); - Next(3+n); - } - -#define PUT_TERM_REG(term, desc) \ -do { \ - switch (loader_tag(desc)) { \ - case LOADER_X_REG: \ - x(loader_x_reg_index(desc)) = (term); \ - break; \ - case LOADER_Y_REG: \ - y(loader_y_reg_index(desc)) = (term); \ - break; \ - default: \ - ASSERT(0); \ - break; \ - } \ -} while(0) - - OpCase(i_get_map_elements_fsI): { - Eterm map; - BeamInstr *fs; - Uint sz, n; - - GetArg1(1, map); - - /* this instruction assumes Arg1 is a map, - * i.e. that it follows a test is_map if needed. - */ - - n = (Uint)Arg(2) / 3; - fs = &Arg(3); /* pattern fields and target registers */ - - if (is_flatmap(map)) { - flatmap_t *mp; - Eterm *ks; - Eterm *vs; - - mp = (flatmap_t *)flatmap_val(map); - sz = flatmap_get_size(mp); - - if (sz == 0) { - ClauseFail(); - } - - ks = flatmap_get_keys(mp); - vs = flatmap_get_values(mp); - - while(sz) { - if (EQ((Eterm) fs[0], *ks)) { - PUT_TERM_REG(*vs, fs[1]); - n--; - fs += 3; - /* no more values to fetch, we are done */ - if (n == 0) { - I = fs; - Next(-1); - } - } - ks++, sz--, vs++; - } - - ClauseFail(); - } else { - const Eterm *v; - Uint32 hx; - ASSERT(is_hashmap(map)); - while(n--) { - hx = fs[2]; - ASSERT(hx == hashmap_make_hash((Eterm)fs[0])); - if ((v = erts_hashmap_get(hx, (Eterm)fs[0], map)) == NULL) { - ClauseFail(); - } - PUT_TERM_REG(*v, fs[1]); - fs += 3; - } - I = fs; - Next(-1); - } - } -#undef PUT_TERM_REG - - OpCase(update_map_assoc_jsdII): { - Eterm res; - Eterm map; - - GetArg1(1, map); - HEAVY_SWAPOUT; - res = update_map_assoc(c_p, reg, map, I); - HEAVY_SWAPIN; - if (is_value(res)) { - StoreResult(res, Arg(2)); - Next(5+Arg(4)); - } else { - /* - * This can only happen if the code was compiled - * with the compiler in OTP 17. - */ - c_p->freason = BADMAP; - c_p->fvalue = map; - goto lb_Cl_error; - } - } - - OpCase(update_map_exact_jsdII): { - Eterm res; - Eterm map; - - GetArg1(1, map); - HEAVY_SWAPOUT; - res = update_map_exact(c_p, reg, map, I); - HEAVY_SWAPIN; - if (is_value(res)) { - StoreResult(res, Arg(2)); - Next(5+Arg(4)); - } else { - goto lb_Cl_error; - } - } - - - /* - * All guards with zero arguments have special instructions: - * self/0 - * node/0 - * - * All other guard BIFs take one or two arguments. - */ - - /* - * Guard BIF in head. On failure, ignore the error and jump - * to the code for the next clause. We don't support tracing - * of guard BIFs. - */ - - OpCase(bif1_fbsd): - { - ErtsBifFunc bf; - Eterm tmp_reg[1]; - Eterm result; - - GetArg1(2, tmp_reg[0]); - bf = (BifFunction) Arg(1); - ERTS_DBG_CHK_REDS(c_p, FCALLS); - c_p->fcalls = FCALLS; - PROCESS_MAIN_CHK_LOCKS(c_p); - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - ERTS_CHK_MBUF_SZ(c_p); - result = (*bf)(c_p, tmp_reg, I); - ERTS_CHK_MBUF_SZ(c_p); - ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - PROCESS_MAIN_CHK_LOCKS(c_p); - ERTS_HOLE_CHECK(c_p); - FCALLS = c_p->fcalls; - ERTS_DBG_CHK_REDS(c_p, FCALLS); - if (is_value(result)) { - StoreBifResult(3, result); - } - SET_I((BeamInstr *) Arg(0)); - Goto(*I); - } - - /* - * Guard BIF in body. It can fail like any BIF. No trace support. - */ - - OpCase(bif1_body_bsd): - { - ErtsBifFunc bf; - - Eterm tmp_reg[1]; - Eterm result; - - GetArg1(1, tmp_reg[0]); - bf = (ErtsBifFunc) Arg(0); - ERTS_DBG_CHK_REDS(c_p, FCALLS); - c_p->fcalls = FCALLS; - PROCESS_MAIN_CHK_LOCKS(c_p); - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - ERTS_CHK_MBUF_SZ(c_p); - result = (*bf)(c_p, tmp_reg, I); - ERTS_CHK_MBUF_SZ(c_p); - ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - PROCESS_MAIN_CHK_LOCKS(c_p); - ERTS_HOLE_CHECK(c_p); - FCALLS = c_p->fcalls; - ERTS_DBG_CHK_REDS(c_p, FCALLS); - if (is_value(result)) { - StoreBifResult(2, result); - } - reg[0] = tmp_reg[0]; - SWAPOUT; - I = handle_error(c_p, I, reg, ubif2mfa((void *) bf)); - goto post_error_handling; - } - - OpCase(i_gc_bif1_jIsId): - { - typedef Eterm (*GcBifFunction)(Process*, Eterm*, Uint); - GcBifFunction bf; - Eterm result; - Uint live = (Uint) Arg(3); - - GetArg1(2, x(live)); - bf = (GcBifFunction) Arg(1); - ERTS_DBG_CHK_REDS(c_p, FCALLS); - c_p->fcalls = FCALLS; - SWAPOUT; - PROCESS_MAIN_CHK_LOCKS(c_p); - ERTS_UNREQ_PROC_MAIN_LOCK(c_p); - ERTS_CHK_MBUF_SZ(c_p); - result = (*bf)(c_p, reg, live); - ERTS_CHK_MBUF_SZ(c_p); - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - ERTS_REQ_PROC_MAIN_LOCK(c_p); - PROCESS_MAIN_CHK_LOCKS(c_p); - SWAPIN; - ERTS_HOLE_CHECK(c_p); - FCALLS = c_p->fcalls; - ERTS_DBG_CHK_REDS(c_p, FCALLS); - if (is_value(result)) { - StoreBifResult(4, result); - } - if (Arg(0) != 0) { - SET_I((BeamInstr *) Arg(0)); - Goto(*I); - } - x(0) = x(live); - I = handle_error(c_p, I, reg, gcbif2mfa((void *) bf)); - goto post_error_handling; - } - - OpCase(i_gc_bif2_jIIssd): /* Note, one less parameter than the i_gc_bif1 - and i_gc_bif3 */ - { - typedef Eterm (*GcBifFunction)(Process*, Eterm*, Uint); - GcBifFunction bf; - Eterm result; - Uint live = (Uint) Arg(2); - - GetArg2(3, x(live), x(live+1)); - /* - * XXX This calling convention does not make sense. 'live' - * should point out the first argument, not the second - * (i.e. 'live' should not be incremented below). - */ - live++; - bf = (GcBifFunction) Arg(1); - ERTS_DBG_CHK_REDS(c_p, FCALLS); - c_p->fcalls = FCALLS; - SWAPOUT; - PROCESS_MAIN_CHK_LOCKS(c_p); - ERTS_UNREQ_PROC_MAIN_LOCK(c_p); - ERTS_CHK_MBUF_SZ(c_p); - result = (*bf)(c_p, reg, live); - ERTS_CHK_MBUF_SZ(c_p); - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - ERTS_REQ_PROC_MAIN_LOCK(c_p); - PROCESS_MAIN_CHK_LOCKS(c_p); - SWAPIN; - ERTS_HOLE_CHECK(c_p); - FCALLS = c_p->fcalls; - ERTS_DBG_CHK_REDS(c_p, FCALLS); - if (is_value(result)) { - StoreBifResult(5, result); - } - if (Arg(0) != 0) { - SET_I((BeamInstr *) Arg(0)); - Goto(*I); - } - live--; - x(0) = x(live); - x(1) = x(live+1); - I = handle_error(c_p, I, reg, gcbif2mfa((void *) bf)); - goto post_error_handling; - } - - OpCase(i_gc_bif3_jIIssd): - { - typedef Eterm (*GcBifFunction)(Process*, Eterm*, Uint); - GcBifFunction bf; - Eterm result; - Uint live = (Uint) Arg(2); - - x(live) = x(SCRATCH_X_REG); - GetArg2(3, x(live+1), x(live+2)); - /* - * XXX This calling convention does not make sense. 'live' - * should point out the first argument, not the third - * (i.e. 'live' should not be incremented below). - */ - live += 2; - bf = (GcBifFunction) Arg(1); - ERTS_DBG_CHK_REDS(c_p, FCALLS); - c_p->fcalls = FCALLS; - SWAPOUT; - PROCESS_MAIN_CHK_LOCKS(c_p); - ERTS_UNREQ_PROC_MAIN_LOCK(c_p); - ERTS_CHK_MBUF_SZ(c_p); - result = (*bf)(c_p, reg, live); - ERTS_CHK_MBUF_SZ(c_p); - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - ERTS_REQ_PROC_MAIN_LOCK(c_p); - PROCESS_MAIN_CHK_LOCKS(c_p); - SWAPIN; - ERTS_HOLE_CHECK(c_p); - FCALLS = c_p->fcalls; - ERTS_DBG_CHK_REDS(c_p, FCALLS); - if (is_value(result)) { - StoreBifResult(5, result); - } - if (Arg(0) != 0) { - SET_I((BeamInstr *) Arg(0)); - Goto(*I); - } - live -= 2; - x(0) = x(live); - x(1) = x(live+1); - x(2) = x(live+2); - I = handle_error(c_p, I, reg, gcbif2mfa((void *) bf)); - goto post_error_handling; - } - - /* - * Guards bifs and, or, xor in guards. - */ - OpCase(i_bif2_fbssd): - { - Eterm tmp_reg[2]; - ErtsBifFunc bf; - Eterm result; - - GetArg2(2, tmp_reg[0], tmp_reg[1]); - bf = (ErtsBifFunc) Arg(1); - ERTS_DBG_CHK_REDS(c_p, FCALLS); - c_p->fcalls = FCALLS; - PROCESS_MAIN_CHK_LOCKS(c_p); - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - ERTS_CHK_MBUF_SZ(c_p); - result = (*bf)(c_p, tmp_reg, I); - ERTS_CHK_MBUF_SZ(c_p); - ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - PROCESS_MAIN_CHK_LOCKS(c_p); - ERTS_HOLE_CHECK(c_p); - FCALLS = c_p->fcalls; - ERTS_DBG_CHK_REDS(c_p, FCALLS); - if (is_value(result)) { - StoreBifResult(4, result); - } - SET_I((BeamInstr *) Arg(0)); - Goto(*I); - } - - /* - * Guards bifs and, or, xor, relational operators in body. - */ - OpCase(i_bif2_body_bssd): - { - Eterm tmp_reg[2]; - ErtsBifFunc bf; - Eterm result; - - GetArg2(1, tmp_reg[0], tmp_reg[1]); - bf = (ErtsBifFunc) Arg(0); - PROCESS_MAIN_CHK_LOCKS(c_p); - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - ERTS_CHK_MBUF_SZ(c_p); - result = (*bf)(c_p, tmp_reg, I); - ERTS_CHK_MBUF_SZ(c_p); - ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - PROCESS_MAIN_CHK_LOCKS(c_p); - ERTS_HOLE_CHECK(c_p); - if (is_value(result)) { - ASSERT(!is_CP(result)); - StoreBifResult(3, result); - } - reg[0] = tmp_reg[0]; - reg[1] = tmp_reg[1]; - SWAPOUT; - I = handle_error(c_p, I, reg, ubif2mfa((void *) bf)); - goto post_error_handling; - } - - /* - * The most general BIF call. The BIF may build any amount of data - * on the heap. The result is always returned in r(0). - */ - OpCase(call_bif_e): - { - ErtsBifFunc bf; - Eterm result; - BeamInstr *next; - ErlHeapFragment *live_hf_end; - Export *export = (Export*)Arg(0); - - - if (!((FCALLS - 1) > 0 || (FCALLS-1) > neg_o_reds)) { - /* If we have run out of reductions, we do a context - switch before calling the bif */ - c_p->arity = GET_BIF_ARITY(export); - c_p->current = &export->info.mfa; - goto context_switch3; - } - - ERTS_MSACC_SET_BIF_STATE_CACHED_X( - GET_BIF_MODULE(export), GET_BIF_ADDRESS(export)); - - bf = GET_BIF_ADDRESS(export); - - PRE_BIF_SWAPOUT(c_p); - ERTS_DBG_CHK_REDS(c_p, FCALLS); - c_p->fcalls = FCALLS - 1; - if (FCALLS <= 0) { - save_calls(c_p, export); - } - PreFetch(1, next); - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - live_hf_end = c_p->mbuf; - ERTS_CHK_MBUF_SZ(c_p); - result = (*bf)(c_p, reg, I); - ERTS_CHK_MBUF_SZ(c_p); - ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - ERTS_HOLE_CHECK(c_p); - ERTS_REQ_PROC_MAIN_LOCK(c_p); - if (ERTS_IS_GC_DESIRED(c_p)) { - Uint arity = GET_BIF_ARITY(export); - result = erts_gc_after_bif_call_lhf(c_p, live_hf_end, result, reg, arity); - E = c_p->stop; - } - PROCESS_MAIN_CHK_LOCKS(c_p); - HTOP = HEAP_TOP(c_p); - FCALLS = c_p->fcalls; - ERTS_DBG_CHK_REDS(c_p, FCALLS); - /* We have to update the cache if we are enabled in order - to make sure no book keeping is done after we disabled - msacc. We don't always do this as it is quite expensive. */ - if (ERTS_MSACC_IS_ENABLED_CACHED_X()) - ERTS_MSACC_UPDATE_CACHE_X(); - ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR); - if (is_value(result)) { - r(0) = result; - CHECK_TERM(r(0)); - NextPF(1, next); - } else if (c_p->freason == TRAP) { - SET_CP(c_p, I+2); - SET_I(c_p->i); - SWAPIN; - Dispatch(); - } - - /* - * Error handling. SWAPOUT is not needed because it was done above. - */ - ASSERT(c_p->stop == E); - I = handle_error(c_p, I, reg, &export->info.mfa); - goto post_error_handling; - } - - /* - * Arithmetic operations. - */ - - OpCase(i_times_jIssd): - { - Eterm Op1, Op2; - GetArg2(2, Op1, Op2); - DO_OUTLINED_ARITH_2(mixed_times, Op1, Op2, BIF_stimes_2); - } - - OpCase(i_m_div_jIssd): - { - Eterm Op1, Op2; - GetArg2(2, Op1, Op2); - DO_OUTLINED_ARITH_2(mixed_div, Op1, Op2, BIF_div_2); - } - - OpCase(i_int_div_jIssd): - { - Eterm Op1, Op2; - - GetArg2(2, Op1, Op2); - if (Op2 == SMALL_ZERO) { - c_p->freason = BADARITH; - BIF_ERROR_ARITY_2(Op1, Op2, BIF_intdiv_2); - } else if (is_both_small(Op1, Op2)) { - Sint ires = signed_val(Op1) / signed_val(Op2); - if (MY_IS_SSMALL(ires)) { - Eterm result = make_small(ires); - StoreBifResult(4, result); - } - } - DO_OUTLINED_ARITH_2(int_div, Op1, Op2, BIF_intdiv_2); - } - - { - Eterm RemOp1, RemOp2; - - OpCase(i_rem_jIxxd): - RemOp1 = xb(Arg(2)); - RemOp2 = xb(Arg(3)); - goto do_rem; - - OpCase(i_rem_jIssd): - GetArg2(2, RemOp1, RemOp2); - goto do_rem; - - do_rem: - if (RemOp2 == SMALL_ZERO) { - c_p->freason = BADARITH; - BIF_ERROR_ARITY_2(RemOp1, RemOp2, BIF_rem_2); - } else if (is_both_small(RemOp1, RemOp2)) { - Eterm result = make_small(signed_val(RemOp1) % signed_val(RemOp2)); - StoreBifResult(4, result); - } else { - DO_OUTLINED_ARITH_2(int_rem, RemOp1, RemOp2, BIF_rem_2); - } - } - - { - Eterm BandOp1, BandOp2; - - OpCase(i_band_jIxcd): - BandOp1 = xb(Arg(2)); - BandOp2 = Arg(3); - goto do_band; - - OpCase(i_band_jIssd): - GetArg2(2, BandOp1, BandOp2); - goto do_band; - - do_band: - if (is_both_small(BandOp1, BandOp2)) { - /* - * No need to untag -- TAG & TAG == TAG. - */ - Eterm result = BandOp1 & BandOp2; - StoreBifResult(4, result); - } - DO_OUTLINED_ARITH_2(band, BandOp1, BandOp2, BIF_band_2); - } - - /* - * An error occurred in an arithmetic operation or test that could - * appear either in a head or in a body. - * In a head, execution should continue at failure address in Arg(0). - * In a body, Arg(0) == 0 and an exception should be raised. - */ - lb_Cl_error: { - if (Arg(0) != 0) { - OpCase(jump_f): { - jump_f: - SET_I((BeamInstr *) Arg(0)); - Goto(*I); - } - } - ASSERT(c_p->freason != BADMATCH || is_value(c_p->fvalue)); - goto find_func_info; - } - - OpCase(i_bor_jIssd): - { - Eterm Op1, Op2; - - GetArg2(2, Op1, Op2); - if (is_both_small(Op1, Op2)) { - /* - * No need to untag -- TAG | TAG == TAG. - */ - Eterm result = Op1 | Op2; - StoreBifResult(4, result); - } - DO_OUTLINED_ARITH_2(bor, Op1, Op2, BIF_bor_2); - } - - OpCase(i_bxor_jIssd): - { - Eterm Op1, Op2; - - GetArg2(2, Op1, Op2); - if (is_both_small(Op1, Op2)) { - /* - * TAG ^ TAG == 0. - * - * Therefore, we perform the XOR operation on the tagged values, - * and OR in the tag bits. - */ - Eterm result = (Op1 ^ Op2) | make_small(0); - StoreBifResult(4, result); - } - DO_OUTLINED_ARITH_2(bxor, Op1, Op2, BIF_bxor_2); - } - - { - Eterm Op1, Op2; - Sint i; - Sint ires; - Eterm* bigp; - Eterm tmp_big[2]; - - OpCase(i_bsr_jIssd): - GetArg2(2, Op1, Op2); - if (is_small(Op2)) { - i = -signed_val(Op2); - if (is_small(Op1)) { - goto small_shift; - } else if (is_big(Op1)) { - if (i == 0) { - StoreBifResult(4, Op1); - } - ires = big_size(Op1); - goto big_shift; - } - } else if (is_big(Op2)) { - /* - * N bsr NegativeBigNum == N bsl MAX_SMALL - * N bsr PositiveBigNum == N bsl MIN_SMALL - */ - Op2 = make_small(bignum_header_is_neg(*big_val(Op2)) ? - MAX_SMALL : MIN_SMALL); - goto do_bsl; - } - c_p->freason = BADARITH; - BIF_ERROR_ARITY_2(Op1, Op2, BIF_bsr_2); - - OpCase(i_bsl_jIssd): - GetArg2(2, Op1, Op2); - do_bsl: - if (is_small(Op2)) { - i = signed_val(Op2); - - if (is_small(Op1)) { - small_shift: - ires = signed_val(Op1); - - if (i == 0 || ires == 0) { - StoreBifResult(4, Op1); - } else if (i < 0) { /* Right shift */ - i = -i; - if (i >= SMALL_BITS-1) { - Op1 = (ires < 0) ? SMALL_MINUS_ONE : SMALL_ZERO; - } else { - Op1 = make_small(ires >> i); - } - StoreBifResult(4, Op1); - } else if (i < SMALL_BITS-1) { /* Left shift */ - if ((ires > 0 && ((~(Uint)0 << ((SMALL_BITS-1)-i)) & ires) == 0) || - ((~(Uint)0 << ((SMALL_BITS-1)-i)) & ~ires) == 0) { - Op1 = make_small(ires << i); - StoreBifResult(4, Op1); - } - } - ires = 1; /* big_size(small_to_big(Op1)) */ - - big_shift: - if (i > 0) { /* Left shift. */ - ires += (i / D_EXP); - } else { /* Right shift. */ - if (ires <= (-i / D_EXP)) - ires = 3; /* ??? */ - else - ires -= (-i / D_EXP); - } - { - ires = BIG_NEED_SIZE(ires+1); - /* - * Slightly conservative check the size to avoid - * allocating huge amounts of memory for bignums that - * clearly would overflow the arity in the header - * word. - */ - if (ires-8 > BIG_ARITY_MAX) { - c_p->freason = SYSTEM_LIMIT; - goto lb_Cl_error; - } - TestHeapPreserve(ires+1, Arg(1), Op1); - if (is_small(Op1)) { - Op1 = small_to_big(signed_val(Op1), tmp_big); - } - bigp = HTOP; - Op1 = big_lshift(Op1, i, bigp); - if (is_big(Op1)) { - HTOP += bignum_header_arity(*HTOP) + 1; - } - HEAP_SPACE_VERIFIED(0); - if (is_nil(Op1)) { - /* - * This result must have been only slight larger - * than allowed since it wasn't caught by the - * previous test. - */ - c_p->freason = SYSTEM_LIMIT; - goto lb_Cl_error; - } - ERTS_HOLE_CHECK(c_p); - StoreBifResult(4, Op1); - } - } else if (is_big(Op1)) { - if (i == 0) { - StoreBifResult(4, Op1); - } - ires = big_size(Op1); - goto big_shift; - } - } else if (is_big(Op2)) { - if (bignum_header_is_neg(*big_val(Op2))) { - /* - * N bsl NegativeBigNum is either 0 or -1, depending on - * the sign of N. Since we don't believe this case - * is common, do the calculation with the minimum - * amount of code. - */ - Op2 = make_small(MIN_SMALL); - goto do_bsl; - } else if (is_small(Op1) || is_big(Op1)) { - /* - * N bsl PositiveBigNum is too large to represent. - */ - c_p->freason = SYSTEM_LIMIT; - goto lb_Cl_error; - } - /* Fall through if the left argument is not an integer. */ - } - c_p->freason = BADARITH; - BIF_ERROR_ARITY_2(Op1, Op2, BIF_bsl_2); - } - - OpCase(i_int_bnot_jsId): - { - Eterm bnot_val; - - GetArg1(1, bnot_val); - if (is_small(bnot_val)) { - bnot_val = make_small(~signed_val(bnot_val)); - } else { - Uint live = Arg(2); - HEAVY_SWAPOUT; - reg[live] = bnot_val; - bnot_val = erts_gc_bnot(c_p, reg, live); - HEAVY_SWAPIN; - ERTS_HOLE_CHECK(c_p); - if (is_nil(bnot_val)) { - BIF_ERROR_ARITY_1(reg[live], BIF_bnot_1); - } - } - StoreBifResult(3, bnot_val); - } - - OpCase(i_apply): { - BeamInstr *next; - HEAVY_SWAPOUT; - next = apply(c_p, r(0), x(1), x(2), reg, NULL, 0); - HEAVY_SWAPIN; - if (next != NULL) { - SET_CP(c_p, I+1); - SET_I(next); - Dispatch(); - } - I = handle_error(c_p, I, reg, &bif_export[BIF_apply_3]->info.mfa); - goto post_error_handling; - } - - OpCase(i_apply_last_P): { - BeamInstr *next; - HEAVY_SWAPOUT; - next = apply(c_p, r(0), x(1), x(2), reg, I, Arg(0)); - HEAVY_SWAPIN; - if (next != NULL) { - SET_CP(c_p, (BeamInstr *) E[0]); - E = ADD_BYTE_OFFSET(E, Arg(0)); - SET_I(next); - Dispatch(); - } - I = handle_error(c_p, I, reg, &bif_export[BIF_apply_3]->info.mfa); - goto post_error_handling; - } - - OpCase(i_apply_only): { - BeamInstr *next; - HEAVY_SWAPOUT; - next = apply(c_p, r(0), x(1), x(2), reg, I, 0); - HEAVY_SWAPIN; - if (next != NULL) { - SET_I(next); - Dispatch(); - } - I = handle_error(c_p, I, reg, &bif_export[BIF_apply_3]->info.mfa); - goto post_error_handling; - } - - OpCase(apply_I): { - BeamInstr *next; - - HEAVY_SWAPOUT; - next = fixed_apply(c_p, reg, Arg(0), NULL, 0); - HEAVY_SWAPIN; - if (next != NULL) { - SET_CP(c_p, I+2); - SET_I(next); - Dispatch(); - } - I = handle_error(c_p, I, reg, &bif_export[BIF_apply_3]->info.mfa); - goto post_error_handling; - } - - OpCase(apply_last_IP): { - BeamInstr *next; - - HEAVY_SWAPOUT; - next = fixed_apply(c_p, reg, Arg(0), I, Arg(1)); - HEAVY_SWAPIN; - if (next != NULL) { - SET_CP(c_p, (BeamInstr *) E[0]); - E = ADD_BYTE_OFFSET(E, Arg(1)); - SET_I(next); - Dispatch(); - } - I = handle_error(c_p, I, reg, &bif_export[BIF_apply_3]->info.mfa); - goto post_error_handling; - } - - OpCase(i_apply_fun): { - BeamInstr *next; - - HEAVY_SWAPOUT; - next = apply_fun(c_p, r(0), x(1), reg); - HEAVY_SWAPIN; - if (next != NULL) { - SET_CP(c_p, I+1); - SET_I(next); - Dispatchfun(); - } - goto find_func_info; - } - - OpCase(i_apply_fun_last_P): { - BeamInstr *next; - - HEAVY_SWAPOUT; - next = apply_fun(c_p, r(0), x(1), reg); - HEAVY_SWAPIN; - if (next != NULL) { - SET_CP(c_p, (BeamInstr *) E[0]); - E = ADD_BYTE_OFFSET(E, Arg(0)); - SET_I(next); - Dispatchfun(); - } - goto find_func_info; - } - - OpCase(i_apply_fun_only): { - BeamInstr *next; - - HEAVY_SWAPOUT; - next = apply_fun(c_p, r(0), x(1), reg); - HEAVY_SWAPIN; - if (next != NULL) { - SET_I(next); - Dispatchfun(); - } - goto find_func_info; - } - - OpCase(i_call_fun_I): { - BeamInstr *next; - - HEAVY_SWAPOUT; - next = call_fun(c_p, Arg(0), reg, THE_NON_VALUE); - HEAVY_SWAPIN; - if (next != NULL) { - SET_CP(c_p, I+2); - SET_I(next); - Dispatchfun(); - } - goto find_func_info; - } - - OpCase(i_call_fun_last_IP): { - BeamInstr *next; - - HEAVY_SWAPOUT; - next = call_fun(c_p, Arg(0), reg, THE_NON_VALUE); - HEAVY_SWAPIN; - if (next != NULL) { - SET_CP(c_p, (BeamInstr *) E[0]); - E = ADD_BYTE_OFFSET(E, Arg(1)); - SET_I(next); - Dispatchfun(); - } - goto find_func_info; - } +#include "beam_hot.h" +#include "beam_instrs.h" #ifdef DEBUG /* @@ -2869,7 +878,7 @@ do { \ OpCase(normal_exit): { SWAPOUT; c_p->freason = EXC_NORMAL; - c_p->arity = 0; /* In case this process will never be garbed again. */ + c_p->arity = 0; /* In case this process will ever be garbed again. */ ERTS_UNREQ_PROC_MAIN_LOCK(c_p); erts_do_exit_process(c_p, am_normal); ERTS_REQ_PROC_MAIN_LOCK(c_p); @@ -2883,32 +892,6 @@ do { \ goto do_schedule; } - OpCase(i_raise): { - Eterm raise_trace = x(2); - Eterm raise_value = x(1); - struct StackTrace *s; - - c_p->fvalue = raise_value; - c_p->ftrace = raise_trace; - s = get_trace_from_exc(raise_trace); - if (s == NULL) { - c_p->freason = EXC_ERROR; - } else { - c_p->freason = PRIMARY_EXCEPTION(s->freason); - } - goto find_func_info; - } - - { - Eterm badmatch_val; - - OpCase(badmatch_x): - badmatch_val = xb(Arg(0)); - c_p->fvalue = badmatch_val; - c_p->freason = BADMATCH; - } - /* Fall through here */ - find_func_info: { SWAPOUT; I = handle_error(c_p, I, reg, NULL); @@ -2949,172 +932,6 @@ do { \ } } - { - Eterm nif_bif_result; - Eterm bif_nif_arity; - - OpCase(call_nif): - { - /* - * call_nif is always first instruction in function: - * - * I[-3]: Module - * I[-2]: Function - * I[-1]: Arity - * I[0]: &&call_nif - * I[1]: Function pointer to NIF function - * I[2]: Pointer to erl_module_nif - * I[3]: Function pointer to dirty NIF - * - * This layout is determined by the NifExport struct - */ - BifFunction vbf; - ErlHeapFragment *live_hf_end; - ErtsCodeMFA *codemfa; - - ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_NIF); - - codemfa = erts_code_to_codemfa(I); - - c_p->current = codemfa; /* current and vbf set to please handle_error */ - - DTRACE_NIF_ENTRY(c_p, codemfa); - - HEAVY_SWAPOUT; - - PROCESS_MAIN_CHK_LOCKS(c_p); - bif_nif_arity = codemfa->arity; - ERTS_UNREQ_PROC_MAIN_LOCK(c_p); - - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - { - typedef Eterm NifF(struct enif_environment_t*, int argc, Eterm argv[]); - NifF* fp = vbf = (NifF*) I[1]; - struct enif_environment_t env; - ASSERT(c_p->scheduler_data); - live_hf_end = c_p->mbuf; - ERTS_CHK_MBUF_SZ(c_p); - erts_pre_nif(&env, c_p, (struct erl_module_nif*)I[2], NULL); - nif_bif_result = (*fp)(&env, bif_nif_arity, reg); - if (env.exception_thrown) - nif_bif_result = THE_NON_VALUE; - erts_post_nif(&env); - ERTS_CHK_MBUF_SZ(c_p); - - PROCESS_MAIN_CHK_LOCKS(c_p); - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR); - ASSERT(!env.exiting); - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - } - - DTRACE_NIF_RETURN(c_p, codemfa); - goto apply_bif_or_nif_epilogue; - - OpCase(apply_bif): - /* - * At this point, I points to the code[0] in the export entry for - * the BIF: - * - * code[-3]: Module - * code[-2]: Function - * code[-1]: Arity - * code[0]: &&apply_bif - * code[1]: Function pointer to BIF function - */ - - if (!((FCALLS - 1) > 0 || (FCALLS - 1) > neg_o_reds)) { - /* If we have run out of reductions, we do a context - switch before calling the bif */ - goto context_switch; - } - - codemfa = erts_code_to_codemfa(I); - - ERTS_MSACC_SET_BIF_STATE_CACHED_X(codemfa->module, (BifFunction)Arg(0)); - - - /* In case we apply process_info/1,2 or load_nif/1 */ - c_p->current = codemfa; - c_p->i = I; /* In case we apply check_process_code/2. */ - c_p->arity = 0; /* To allow garbage collection on ourselves - * (check_process_code/2). - */ - DTRACE_BIF_ENTRY(c_p, codemfa); - - SWAPOUT; - ERTS_DBG_CHK_REDS(c_p, FCALLS - 1); - c_p->fcalls = FCALLS - 1; - vbf = (BifFunction) Arg(0); - PROCESS_MAIN_CHK_LOCKS(c_p); - bif_nif_arity = codemfa->arity; - ASSERT(bif_nif_arity <= 4); - ERTS_UNREQ_PROC_MAIN_LOCK(c_p); - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - { - ErtsBifFunc bf = vbf; - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - live_hf_end = c_p->mbuf; - ERTS_CHK_MBUF_SZ(c_p); - nif_bif_result = (*bf)(c_p, reg, I); - ERTS_CHK_MBUF_SZ(c_p); - ASSERT(!ERTS_PROC_IS_EXITING(c_p) || - is_non_value(nif_bif_result)); - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - PROCESS_MAIN_CHK_LOCKS(c_p); - } - /* We have to update the cache if we are enabled in order - to make sure no book keeping is done after we disabled - msacc. We don't always do this as it is quite expensive. */ - if (ERTS_MSACC_IS_ENABLED_CACHED_X()) - ERTS_MSACC_UPDATE_CACHE_X(); - ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR); - DTRACE_BIF_RETURN(c_p, codemfa); - - apply_bif_or_nif_epilogue: - ERTS_REQ_PROC_MAIN_LOCK(c_p); - ERTS_HOLE_CHECK(c_p); - if (ERTS_IS_GC_DESIRED(c_p)) { - nif_bif_result = erts_gc_after_bif_call_lhf(c_p, live_hf_end, - nif_bif_result, - reg, bif_nif_arity); - } - SWAPIN; /* There might have been a garbage collection. */ - FCALLS = c_p->fcalls; - ERTS_DBG_CHK_REDS(c_p, FCALLS); - if (is_value(nif_bif_result)) { - r(0) = nif_bif_result; - CHECK_TERM(r(0)); - SET_I(c_p->cp); - c_p->cp = 0; - Goto(*I); - } else if (c_p->freason == TRAP) { - SET_I(c_p->i); - if (c_p->flags & F_HIBERNATE_SCHED) { - c_p->flags &= ~F_HIBERNATE_SCHED; - goto do_schedule; - } - Dispatch(); - } - I = handle_error(c_p, c_p->cp, reg, c_p->current); - goto post_error_handling; - } - } - - { - Eterm case_end_val; - - OpCase(case_end_x): - case_end_val = xb(Arg(0)); - c_p->fvalue = case_end_val; - c_p->freason = EXC_CASE_CLAUSE; - goto find_func_info; - } - - OpCase(if_end): - c_p->freason = EXC_IF_CLAUSE; - goto find_func_info; - OpCase(i_func_info_IaaI): { ErtsCodeInfo *ci = (ErtsCodeInfo*)I; c_p->freason = EXC_FUNCTION_CLAUSE; @@ -3122,1368 +939,8 @@ do { \ goto handle_error; } - OpCase(try_case_end_s): - { - Eterm try_case_end_val; - GetArg1(0, try_case_end_val); - c_p->fvalue = try_case_end_val; - c_p->freason = EXC_TRY_CLAUSE; - goto find_func_info; - } - - /* - * Construction of binaries using new instructions. - */ - { - Eterm new_binary; - Eterm num_bits_term; - Uint num_bits; - Uint alloc; - Uint num_bytes; - - OpCase(i_bs_init_bits_heap_IIIx): { - num_bits = Arg(0); - alloc = Arg(1); - I++; - goto do_bs_init_bits_known; - } - - OpCase(i_bs_init_bits_IIx): { - num_bits = Arg(0); - alloc = 0; - goto do_bs_init_bits_known; - } - - OpCase(i_bs_init_bits_fail_heap_sIjIx): { - GetArg1(0, num_bits_term); - alloc = Arg(1); - I += 2; - goto do_bs_init_bits; - } - - OpCase(i_bs_init_bits_fail_yjIx): { - num_bits_term = yb(Arg(0)); - I++; - alloc = 0; - goto do_bs_init_bits; - } - OpCase(i_bs_init_bits_fail_xjIx): { - num_bits_term = xb(Arg(0)); - I++; - alloc = 0; - /* FALL THROUGH */ - } - - /* num_bits_term = Term for number of bits to build (small/big) - * alloc = Number of words to allocate on heap - * Operands: Fail Live Dst - */ - - do_bs_init_bits: - if (is_small(num_bits_term)) { - Sint size = signed_val(num_bits_term); - if (size < 0) { - goto badarg; - } - num_bits = (Uint) size; - } else { - Uint bits; - - if (!term_to_Uint(num_bits_term, &bits)) { - c_p->freason = bits; - goto lb_Cl_error; - - } - num_bits = (Eterm) bits; - } - - /* num_bits = Number of bits to build - * alloc = Number of extra words to allocate on heap - * Operands: NotUsed Live Dst - */ - do_bs_init_bits_known: - num_bytes = ((Uint64)num_bits+(Uint64)7) >> 3; - if (num_bits & 7) { - alloc += ERL_SUB_BIN_SIZE; - } - if (num_bytes <= ERL_ONHEAP_BIN_LIMIT) { - alloc += heap_bin_size(num_bytes); - } else { - alloc += PROC_BIN_SIZE; - } - TestHeap(alloc, Arg(1)); - - /* num_bits = Number of bits to build - * num_bytes = Number of bytes to allocate in the binary - * alloc = Total number of words to allocate on heap - * Operands: NotUsed NotUsed Dst - */ - if (num_bytes <= ERL_ONHEAP_BIN_LIMIT) { - ErlHeapBin* hb; - - erts_bin_offset = 0; - erts_writable_bin = 0; - hb = (ErlHeapBin *) HTOP; - HTOP += heap_bin_size(num_bytes); - hb->thing_word = header_heap_bin(num_bytes); - hb->size = num_bytes; - erts_current_bin = (byte *) hb->data; - new_binary = make_binary(hb); - - do_bits_sub_bin: - if (num_bits & 7) { - ErlSubBin* sb; - - sb = (ErlSubBin *) HTOP; - HTOP += ERL_SUB_BIN_SIZE; - sb->thing_word = HEADER_SUB_BIN; - sb->size = num_bytes - 1; - sb->bitsize = num_bits & 7; - sb->offs = 0; - sb->bitoffs = 0; - sb->is_writable = 0; - sb->orig = new_binary; - new_binary = make_binary(sb); - } - HEAP_SPACE_VERIFIED(0); - xb(Arg(2)) = new_binary; - Next(3); - } else { - Binary* bptr; - ProcBin* pb; - - erts_bin_offset = 0; - erts_writable_bin = 0; - - /* - * Allocate the binary struct itself. - */ - bptr = erts_bin_nrml_alloc(num_bytes); - erts_current_bin = (byte *) bptr->orig_bytes; - - /* - * Now allocate the ProcBin on the heap. - */ - pb = (ProcBin *) HTOP; - HTOP += PROC_BIN_SIZE; - pb->thing_word = HEADER_PROC_BIN; - pb->size = num_bytes; - pb->next = MSO(c_p).first; - MSO(c_p).first = (struct erl_off_heap_header*) pb; - pb->val = bptr; - pb->bytes = (byte*) bptr->orig_bytes; - pb->flags = 0; - OH_OVERHEAD(&(MSO(c_p)), pb->size / sizeof(Eterm)); - new_binary = make_binary(pb); - goto do_bits_sub_bin; - } - } - - { - Eterm BsOp1, BsOp2; - - OpCase(i_bs_init_fail_heap_sIjIx): { - GetArg1(0, BsOp1); - BsOp2 = Arg(1); - I += 2; - goto do_bs_init; - } - - OpCase(i_bs_init_fail_yjIx): { - BsOp1 = yb(Arg(0)); - BsOp2 = 0; - I++; - goto do_bs_init; - } - - OpCase(i_bs_init_fail_xjIx): { - BsOp1 = xb(Arg(0)); - BsOp2 = 0; - I++; - } - /* FALL THROUGH */ - do_bs_init: - if (is_small(BsOp1)) { - Sint size = signed_val(BsOp1); - if (size < 0) { - goto badarg; - } - BsOp1 = (Eterm) size; - } else { - Uint bytes; - - if (!term_to_Uint(BsOp1, &bytes)) { - c_p->freason = bytes; - goto lb_Cl_error; - } - if ((bytes >> (8*sizeof(Uint)-3)) != 0) { - goto system_limit; - } - BsOp1 = (Eterm) bytes; - } - if (BsOp1 <= ERL_ONHEAP_BIN_LIMIT) { - goto do_heap_bin_alloc; - } else { - goto do_proc_bin_alloc; - } - - - OpCase(i_bs_init_heap_IIIx): { - BsOp1 = Arg(0); - BsOp2 = Arg(1); - I++; - goto do_proc_bin_alloc; - } - - OpCase(i_bs_init_IIx): { - BsOp1 = Arg(0); - BsOp2 = 0; - } - /* FALL THROUGH */ - do_proc_bin_alloc: { - Binary* bptr; - ProcBin* pb; - - erts_bin_offset = 0; - erts_writable_bin = 0; - TestBinVHeap(BsOp1 / sizeof(Eterm), - BsOp2 + PROC_BIN_SIZE + ERL_SUB_BIN_SIZE, Arg(1)); - - /* - * Allocate the binary struct itself. - */ - bptr = erts_bin_nrml_alloc(BsOp1); - erts_current_bin = (byte *) bptr->orig_bytes; - - /* - * Now allocate the ProcBin on the heap. - */ - pb = (ProcBin *) HTOP; - HTOP += PROC_BIN_SIZE; - pb->thing_word = HEADER_PROC_BIN; - pb->size = BsOp1; - pb->next = MSO(c_p).first; - MSO(c_p).first = (struct erl_off_heap_header*) pb; - pb->val = bptr; - pb->bytes = (byte*) bptr->orig_bytes; - pb->flags = 0; - - OH_OVERHEAD(&(MSO(c_p)), BsOp1 / sizeof(Eterm)); - - xb(Arg(2)) = make_binary(pb); - Next(3); - } - - OpCase(i_bs_init_heap_bin_heap_IIIx): { - BsOp1 = Arg(0); - BsOp2 = Arg(1); - I++; - goto do_heap_bin_alloc; - } - - OpCase(i_bs_init_heap_bin_IIx): { - BsOp1 = Arg(0); - BsOp2 = 0; - } - /* Fall through */ - do_heap_bin_alloc: - { - ErlHeapBin* hb; - Uint bin_need; - - bin_need = heap_bin_size(BsOp1); - erts_bin_offset = 0; - erts_writable_bin = 0; - TestHeap(bin_need+BsOp2+ERL_SUB_BIN_SIZE, Arg(1)); - hb = (ErlHeapBin *) HTOP; - HTOP += bin_need; - hb->thing_word = header_heap_bin(BsOp1); - hb->size = BsOp1; - erts_current_bin = (byte *) hb->data; - BsOp1 = make_binary(hb); - xb(Arg(2)) = BsOp1; - Next(3); - } - } - - OpCase(bs_add_jssIx): { - Eterm Op1, Op2; - Uint Unit = Arg(3); - - GetArg2(1, Op1, Op2); - if (is_both_small(Op1, Op2)) { - Sint Arg1 = signed_val(Op1); - Sint Arg2 = signed_val(Op2); - - if (Arg1 >= 0 && Arg2 >= 0) { - BsSafeMul(Arg2, Unit, goto system_limit, Op1); - Op1 += Arg1; - - store_bs_add_result: - if (Op1 <= MAX_SMALL) { - Op1 = make_small(Op1); - } else { - /* - * May generate a heap fragment, but in this - * particular case it is OK, since the value will be - * stored into an x register (the GC will scan x - * registers for references to heap fragments) and - * there is no risk that value can be stored into a - * location that is not scanned for heap-fragment - * references (such as the heap). - */ - SWAPOUT; - Op1 = erts_make_integer(Op1, c_p); - HTOP = HEAP_TOP(c_p); - } - xb(Arg(4)) = Op1; - Next(5); - } - goto badarg; - } else { - Uint a; - Uint b; - Uint c; - - /* - * Now we know that one of the arguments is - * not a small. We must convert both arguments - * to Uints and check for errors at the same time. - * - * Error checking is tricky. - * - * If one of the arguments is not numeric or - * not positive, the error reason is BADARG. - * - * Otherwise if both arguments are numeric, - * but at least one argument does not fit in - * an Uint, the reason is SYSTEM_LIMIT. - */ - - if (!term_to_Uint(Op1, &a)) { - if (a == BADARG) { - goto badarg; - } - if (!term_to_Uint(Op2, &b)) { - c_p->freason = b; - goto lb_Cl_error; - } - goto system_limit; - } else if (!term_to_Uint(Op2, &b)) { - c_p->freason = b; - goto lb_Cl_error; - } - - /* - * The arguments are now correct and stored in a and b. - */ - - BsSafeMul(b, Unit, goto system_limit, c); - Op1 = a + c; - if (Op1 < a) { - /* - * If the result is less than one of the - * arguments, there must have been an overflow. - */ - goto system_limit; - } - goto store_bs_add_result; - } - /* No fallthrough */ - ASSERT(0); - } - - OpCase(bs_put_string_II): - { - BeamInstr *next; - PreFetch(2, next); - erts_new_bs_put_string(ERL_BITS_ARGS_2((byte *) Arg(1), Arg(0))); - NextPF(2, next); - } - - /* - * x(SCRATCH_X_REG); - * Operands: Fail ExtraHeap Live Unit Size Dst - */ - - OpCase(i_bs_append_jIIIsx): { - Uint live = Arg(2); - Uint res; - Eterm Size; - - GetArg1(4, Size); - HEAVY_SWAPOUT; - reg[live] = x(SCRATCH_X_REG); - res = erts_bs_append(c_p, reg, live, Size, Arg(1), Arg(3)); - HEAVY_SWAPIN; - if (is_non_value(res)) { - /* c_p->freason is already set (may be either BADARG or SYSTEM_LIMIT). */ - goto lb_Cl_error; - } - xb(Arg(5)) = res; - Next(6); - } - - /* - * Operands: Fail Size Src Unit Dst - */ - OpCase(i_bs_private_append_jIssx): { - Eterm res; - Eterm Size, Src; - - GetArg2(2, Size, Src); - res = erts_bs_private_append(c_p, Src, Size, Arg(1)); - if (is_non_value(res)) { - /* c_p->freason is already set (may be either BADARG or SYSTEM_LIMIT). */ - goto lb_Cl_error; - } - xb(Arg(4)) = res; - Next(5); - } - - OpCase(bs_init_writable): { - HEAVY_SWAPOUT; - r(0) = erts_bs_init_writable(c_p, r(0)); - HEAVY_SWAPIN; - Next(0); - } - - /* - * Calculate the number of bytes needed to encode the source - * operarand to UTF-8. If the source operand is invalid (e.g. wrong - * type or range) we return a nonsense integer result (0 or 4). We - * can get away with that because we KNOW that bs_put_utf8 will do - * full error checking. - */ - OpCase(i_bs_utf8_size_sx): { - Eterm arg; - Eterm result; - - GetArg1(0, arg); - if (arg < make_small(0x80UL)) { - result = make_small(1); - } else if (arg < make_small(0x800UL)) { - result = make_small(2); - } else if (arg < make_small(0x10000UL)) { - result = make_small(3); - } else { - result = make_small(4); - } - xb(Arg(1)) = result; - Next(2); - } - - OpCase(i_bs_put_utf8_js): { - Eterm arg; - - GetArg1(1, arg); - if (!erts_bs_put_utf8(ERL_BITS_ARGS_1(arg))) { - goto badarg; - } - Next(2); - } - - /* - * Calculate the number of bytes needed to encode the source - * operarand to UTF-8. If the source operand is invalid (e.g. wrong - * type or range) we return a nonsense integer result (2 or 4). We - * can get away with that because we KNOW that bs_put_utf16 will do - * full error checking. - */ - - OpCase(i_bs_utf16_size_sx): { - Eterm arg; - Eterm result = make_small(2); - - GetArg1(0, arg); - if (arg >= make_small(0x10000UL)) { - result = make_small(4); - } - xb(Arg(1)) = result; - Next(2); - } - - OpCase(bs_put_utf16_jIs): { - Eterm arg; - - GetArg1(2, arg); - if (!erts_bs_put_utf16(ERL_BITS_ARGS_2(arg, Arg(1)))) { - goto badarg; - } - Next(3); - } - - /* - * Only used for validating a value about to be stored in a binary. - */ - OpCase(i_bs_validate_unicode_js): { - Eterm val; - - GetArg1(1, val); - - /* - * There is no need to untag the integer, but it IS necessary - * to make sure it is small (if the term is a bignum, it could - * slip through the test, and there is no further test that - * would catch it, since bit syntax construction silently masks - * too big numbers). - */ - if (is_not_small(val) || val > make_small(0x10FFFFUL) || - (make_small(0xD800UL) <= val && val <= make_small(0xDFFFUL))) { - goto badarg; - } - Next(2); - } - - /* - * Only used for validating a value matched out. - */ - OpCase(i_bs_validate_unicode_retract_jss): { - Eterm i; /* Integer to validate */ - - /* - * There is no need to untag the integer, but it IS necessary - * to make sure it is small (a bignum pointer could fall in - * the valid range). - */ - - GetArg1(1, i); - if (is_not_small(i) || i > make_small(0x10FFFFUL) || - (make_small(0xD800UL) <= i && i <= make_small(0xDFFFUL))) { - Eterm ms; /* Match context */ - ErlBinMatchBuffer* mb; - - GetArg1(2, ms); - mb = ms_matchbuffer(ms); - mb->offset -= 32; - goto badarg; - } - Next(3); - } - - /* - * Matching of binaries. - */ - - { - Eterm header; - BeamInstr *next; - Uint slots; - Eterm context; - - do_start_match: - slots = Arg(2); - if (!is_boxed(context)) { - ClauseFail(); - } - PreFetch(4, next); - header = *boxed_val(context); - if (header_is_bin_matchstate(header)) { - ErlBinMatchState* ms = (ErlBinMatchState *) boxed_val(context); - Uint actual_slots = HEADER_NUM_SLOTS(header); - ms->save_offset[0] = ms->mb.offset; - if (actual_slots < slots) { - ErlBinMatchState* dst; - Uint live = Arg(1); - Uint wordsneeded = ERL_BIN_MATCHSTATE_SIZE(slots); - - TestHeapPreserve(wordsneeded, live, context); - ms = (ErlBinMatchState *) boxed_val(context); - dst = (ErlBinMatchState *) HTOP; - *dst = *ms; - *HTOP = HEADER_BIN_MATCHSTATE(slots); - HTOP += wordsneeded; - HEAP_SPACE_VERIFIED(0); - xb(Arg(3)) = make_matchstate(dst); - } - } else if (is_binary_header(header)) { - Eterm result; - Uint live = Arg(1); - Uint wordsneeded = ERL_BIN_MATCHSTATE_SIZE(slots); - TestHeapPreserve(wordsneeded, live, context); - HEAP_TOP(c_p) = HTOP; -#ifdef DEBUG - c_p->stop = E; /* Needed for checking in HeapOnlyAlloc(). */ -#endif - result = erts_bs_start_match_2(c_p, context, slots); - HTOP = HEAP_TOP(c_p); - HEAP_SPACE_VERIFIED(0); - if (is_non_value(result)) { - ClauseFail(); - } else { - xb(Arg(3)) = result; - } - } else { - ClauseFail(); - } - NextPF(4, next); - - OpCase(i_bs_start_match2_xfIIx): { - context = xb(Arg(0)); - I++; - goto do_start_match; - } - OpCase(i_bs_start_match2_yfIIx): { - context = yb(Arg(0)); - I++; - goto do_start_match; - } - } - - OpCase(bs_test_zero_tail2_fx): { - BeamInstr *next; - ErlBinMatchBuffer *_mb; - - PreFetch(2, next); - _mb = (ErlBinMatchBuffer*) ms_matchbuffer(xb(Arg(1))); - if (_mb->size != _mb->offset) { - ClauseFail(); - } - NextPF(2, next); - } - - OpCase(bs_test_tail_imm2_fxI): { - BeamInstr *next; - ErlBinMatchBuffer *_mb; - PreFetch(3, next); - _mb = ms_matchbuffer(xb(Arg(1))); - if (_mb->size - _mb->offset != Arg(2)) { - ClauseFail(); - } - NextPF(3, next); - } - - OpCase(bs_test_unit_fxI): { - BeamInstr *next; - ErlBinMatchBuffer *_mb; - PreFetch(3, next); - _mb = ms_matchbuffer(xb(Arg(1))); - if ((_mb->size - _mb->offset) % Arg(2)) { - ClauseFail(); - } - NextPF(3, next); - } - - OpCase(bs_test_unit8_fx): { - BeamInstr *next; - ErlBinMatchBuffer *_mb; - PreFetch(2, next); - _mb = ms_matchbuffer(xb(Arg(1))); - if ((_mb->size - _mb->offset) & 7) { - ClauseFail(); - } - NextPF(2, next); - } - - { - Eterm bs_get_integer8_context; - - OpCase(i_bs_get_integer_8_xfx): { - ErlBinMatchBuffer *_mb; - Eterm _result; - bs_get_integer8_context = xb(Arg(0)); - I++; - _mb = ms_matchbuffer(bs_get_integer8_context); - if (_mb->size - _mb->offset < 8) { - ClauseFail(); - } - if (BIT_OFFSET(_mb->offset) != 0) { - _result = erts_bs_get_integer_2(c_p, 8, 0, _mb); - } else { - _result = make_small(_mb->base[BYTE_OFFSET(_mb->offset)]); - _mb->offset += 8; - } - xb(Arg(1)) = _result; - Next(2); - } - } - - { - Eterm bs_get_integer_16_context; - - OpCase(i_bs_get_integer_16_xfx): - bs_get_integer_16_context = xb(Arg(0)); - I++; - - { - ErlBinMatchBuffer *_mb; - Eterm _result; - _mb = ms_matchbuffer(bs_get_integer_16_context); - if (_mb->size - _mb->offset < 16) { - ClauseFail(); - } - if (BIT_OFFSET(_mb->offset) != 0) { - _result = erts_bs_get_integer_2(c_p, 16, 0, _mb); - } else { - _result = make_small(get_int16(_mb->base+BYTE_OFFSET(_mb->offset))); - _mb->offset += 16; - } - xb(Arg(1)) = _result; - Next(2); - } - } - - { - Eterm bs_get_integer_32_context; - - OpCase(i_bs_get_integer_32_xfIx): - bs_get_integer_32_context = xb(Arg(0)); - I++; - - { - ErlBinMatchBuffer *_mb; - Uint32 _integer; - Eterm _result; - _mb = ms_matchbuffer(bs_get_integer_32_context); - if (_mb->size - _mb->offset < 32) { ClauseFail(); } - if (BIT_OFFSET(_mb->offset) != 0) { - _integer = erts_bs_get_unaligned_uint32(_mb); - } else { - _integer = get_int32(_mb->base + _mb->offset/8); - } - _mb->offset += 32; -#if !defined(ARCH_64) - if (IS_USMALL(0, _integer)) { -#endif - _result = make_small(_integer); -#if !defined(ARCH_64) - } else { - TestHeap(BIG_UINT_HEAP_SIZE, Arg(1)); - _result = uint_to_big((Uint) _integer, HTOP); - HTOP += BIG_UINT_HEAP_SIZE; - HEAP_SPACE_VERIFIED(0); - } -#endif - xb(Arg(2)) = _result; - Next(3); - } - } - - { - Eterm Ms, Sz; - - /* Operands: x(Reg) Size Live Fail Flags Dst */ - OpCase(i_bs_get_integer_imm_xIIfIx): { - Uint wordsneeded; - Ms = xb(Arg(0)); - Sz = Arg(1); - wordsneeded = 1+WSIZE(NBYTES(Sz)); - TestHeapPreserve(wordsneeded, Arg(2), Ms); - I += 3; - /* Operands: Fail Flags Dst */ - goto do_bs_get_integer_imm; - } - - /* Operands: x(Reg) Size Fail Flags Dst */ - OpCase(i_bs_get_integer_small_imm_xIfIx): { - Ms = xb(Arg(0)); - Sz = Arg(1); - I += 2; - /* Operands: Fail Flags Dst */ - goto do_bs_get_integer_imm; - } - - /* - * Ms = match context - * Sz = size of field - * Operands: Fail Flags Dst - */ - do_bs_get_integer_imm: { - ErlBinMatchBuffer* mb; - Eterm result; - - mb = ms_matchbuffer(Ms); - LIGHT_SWAPOUT; - result = erts_bs_get_integer_2(c_p, Sz, Arg(1), mb); - LIGHT_SWAPIN; - HEAP_SPACE_VERIFIED(0); - if (is_non_value(result)) { - ClauseFail(); - } - xb(Arg(2)) = result; - Next(3); - } - } - - /* - * Operands: Fail Live FlagsAndUnit Ms Sz Dst - */ - OpCase(i_bs_get_integer_fIIssx): { - Uint flags; - Uint size; - Eterm Ms; - Eterm Sz; - ErlBinMatchBuffer* mb; - Eterm result; - - flags = Arg(2); - GetArg2(3, Ms, Sz); - BsGetFieldSize(Sz, (flags >> 3), ClauseFail(), size); - if (size >= SMALL_BITS) { - Uint wordsneeded; - /* Check bits size before potential gc. - * We do not want a gc and then realize we don't need - * the allocated space (i.e. if the op fails). - * - * Remember to re-acquire the matchbuffer after gc. - */ - - mb = ms_matchbuffer(Ms); - if (mb->size - mb->offset < size) { - ClauseFail(); - } - wordsneeded = 1+WSIZE(NBYTES((Uint) size)); - TestHeapPreserve(wordsneeded, Arg(1), Ms); - } - mb = ms_matchbuffer(Ms); - LIGHT_SWAPOUT; - result = erts_bs_get_integer_2(c_p, size, flags, mb); - LIGHT_SWAPIN; - HEAP_SPACE_VERIFIED(0); - if (is_non_value(result)) { - ClauseFail(); - } - xb(Arg(5)) = result; - Next(6); - } - - { - Eterm get_utf8_context; - - /* Operands: MatchContext Fail Dst */ - OpCase(i_bs_get_utf8_xfx): { - get_utf8_context = xb(Arg(0)); - I++; - } - - /* - * get_utf8_context = match_context - * Operands: Fail Dst - */ - - { - Eterm result = erts_bs_get_utf8(ms_matchbuffer(get_utf8_context)); - if (is_non_value(result)) { - ClauseFail(); - } - xb(Arg(1)) = result; - Next(2); - } - } - - { - Eterm get_utf16_context; - - /* Operands: MatchContext Fail Flags Dst */ - OpCase(i_bs_get_utf16_xfIx): { - get_utf16_context = xb(Arg(0)); - I++; - } - - /* - * get_utf16_context = match_context - * Operands: Fail Flags Dst - */ - { - Eterm result = erts_bs_get_utf16(ms_matchbuffer(get_utf16_context), - Arg(1)); - if (is_non_value(result)) { - ClauseFail(); - } - xb(Arg(2)) = result; - Next(3); - } - } - - { - Eterm context_to_binary_context; - ErlBinMatchBuffer* mb; - ErlSubBin* sb; - Uint size; - Uint offs; - Uint orig; - Uint hole_size; - - OpCase(bs_context_to_binary_x): - context_to_binary_context = xb(Arg(0)); - I--; - - if (is_boxed(context_to_binary_context) && - header_is_bin_matchstate(*boxed_val(context_to_binary_context))) { - ErlBinMatchState* ms; - ms = (ErlBinMatchState *) boxed_val(context_to_binary_context); - mb = &ms->mb; - offs = ms->save_offset[0]; - size = mb->size - offs; - goto do_bs_get_binary_all_reuse_common; - } - Next(2); - - OpCase(i_bs_get_binary_all_reuse_xfI): { - context_to_binary_context = xb(Arg(0)); - I++; - } - - mb = ms_matchbuffer(context_to_binary_context); - size = mb->size - mb->offset; - if (size % Arg(1) != 0) { - ClauseFail(); - } - offs = mb->offset; - - do_bs_get_binary_all_reuse_common: - orig = mb->orig; - sb = (ErlSubBin *) boxed_val(context_to_binary_context); - hole_size = 1 + header_arity(sb->thing_word) - ERL_SUB_BIN_SIZE; - sb->thing_word = HEADER_SUB_BIN; - sb->size = BYTE_OFFSET(size); - sb->bitsize = BIT_OFFSET(size); - sb->offs = BYTE_OFFSET(offs); - sb->bitoffs = BIT_OFFSET(offs); - sb->is_writable = 0; - sb->orig = orig; - if (hole_size) { - sb[1].thing_word = make_pos_bignum_header(hole_size-1); - } - Next(2); - } - - { - Eterm match_string_context; - - OpCase(i_bs_match_string_xfII): { - match_string_context = xb(Arg(0)); - I++; - } - - { - BeamInstr *next; - byte* bytes; - Uint bits; - ErlBinMatchBuffer* mb; - Uint offs; - - PreFetch(3, next); - bits = Arg(1); - bytes = (byte *) Arg(2); - mb = ms_matchbuffer(match_string_context); - if (mb->size - mb->offset < bits) { - ClauseFail(); - } - offs = mb->offset & 7; - if (offs == 0 && (bits & 7) == 0) { - if (sys_memcmp(bytes, mb->base+(mb->offset>>3), bits>>3)) { - ClauseFail(); - } - } else if (erts_cmp_bits(bytes, 0, mb->base+(mb->offset>>3), mb->offset & 7, bits)) { - ClauseFail(); - } - mb->offset += bits; - NextPF(3, next); - } - } - - OpCase(i_bs_save2_xI): { - BeamInstr *next; - ErlBinMatchState *_ms; - PreFetch(2, next); - _ms = (ErlBinMatchState*) boxed_val((Eterm) xb(Arg(0))); - _ms->save_offset[Arg(1)] = _ms->mb.offset; - NextPF(2, next); - } - - OpCase(i_bs_restore2_xI): { - BeamInstr *next; - ErlBinMatchState *_ms; - PreFetch(2, next); - _ms = (ErlBinMatchState*) boxed_val((Eterm) xb(Arg(0))); - _ms->mb.offset = _ms->save_offset[Arg(1)]; - NextPF(2, next); - } - #include "beam_cold.h" - /* - * Trace and debugging support. - */ - - OpCase(return_trace): { - ErtsCodeMFA* mfa = (ErtsCodeMFA *)(E[0]); - - SWAPOUT; /* Needed for shared heap */ - ERTS_UNREQ_PROC_MAIN_LOCK(c_p); - erts_trace_return(c_p, mfa, r(0), ERTS_TRACER_FROM_ETERM(E+1)/* tracer */); - ERTS_REQ_PROC_MAIN_LOCK(c_p); - SWAPIN; - c_p->cp = NULL; - SET_I((BeamInstr *) cp_val(E[2])); - E += 3; - Goto(*I); - } - - OpCase(i_generic_breakpoint): { - BeamInstr real_I; - HEAVY_SWAPOUT; - real_I = erts_generic_breakpoint(c_p, erts_code_to_codeinfo(I), reg); - HEAVY_SWAPIN; - ASSERT(VALID_INSTR(real_I)); - Goto(real_I); - } - - OpCase(i_return_time_trace): { - BeamInstr *pc = (BeamInstr *) (UWord) E[0]; - SWAPOUT; - erts_trace_time_return(c_p, erts_code_to_codeinfo(pc)); - SWAPIN; - c_p->cp = NULL; - SET_I((BeamInstr *) cp_val(E[1])); - E += 2; - Goto(*I); - } - - OpCase(i_return_to_trace): { - if (IS_TRACED_FL(c_p, F_TRACE_RETURN_TO)) { - Uint *cpp = (Uint*) E; - for(;;) { - ASSERT(is_CP(*cpp)); - if (*cp_val(*cpp) == (BeamInstr) OpCode(return_trace)) { - do ++cpp; while(is_not_CP(*cpp)); - cpp += 2; - } else if (*cp_val(*cpp) == (BeamInstr) OpCode(i_return_to_trace)) { - do ++cpp; while(is_not_CP(*cpp)); - } else break; - } - SWAPOUT; /* Needed for shared heap */ - ERTS_UNREQ_PROC_MAIN_LOCK(c_p); - erts_trace_return_to(c_p, cp_val(*cpp)); - ERTS_REQ_PROC_MAIN_LOCK(c_p); - SWAPIN; - } - c_p->cp = NULL; - SET_I((BeamInstr *) cp_val(E[0])); - E += 1; - Goto(*I); - } - - /* - * New floating point instructions. - */ - - OpCase(fmove_ql): { - Eterm fr = Arg(1); - BeamInstr *next; - - PreFetch(2, next); - GET_DOUBLE(Arg(0), *(FloatDef*)ADD_BYTE_OFFSET(freg, fr)); - NextPF(2, next); - } - - OpCase(fmove_dl): { - Eterm targ1; - Eterm fr = Arg(1); - BeamInstr *next; - - PreFetch(2, next); - targ1 = REG_TARGET(Arg(0)); - /* Arg(0) == HEADER_FLONUM */ - GET_DOUBLE(targ1, *(FloatDef*)ADD_BYTE_OFFSET(freg, fr)); - NextPF(2, next); - } - - OpCase(fmove_ld): { - Eterm fr = Arg(0); - Eterm dest = make_float(HTOP); - - PUT_DOUBLE(*(FloatDef*)ADD_BYTE_OFFSET(freg, fr), HTOP); - HTOP += FLOAT_SIZE_OBJECT; - StoreBifResult(1, dest); - } - - OpCase(fconv_dl): { - Eterm targ1; - Eterm fr = Arg(1); - BeamInstr *next; - - targ1 = REG_TARGET(Arg(0)); - PreFetch(2, next); - if (is_small(targ1)) { - lb(fr) = (double) signed_val(targ1); - } else if (is_big(targ1)) { - if (big_to_double(targ1, &lb(fr)) < 0) { - goto fbadarith; - } - } else if (is_float(targ1)) { - GET_DOUBLE(targ1, *(FloatDef*)ADD_BYTE_OFFSET(freg, fr)); - } else { - goto fbadarith; - } - NextPF(2, next); - } - -#ifdef NO_FPE_SIGNALS - OpCase(fclearerror): - OpCase(i_fcheckerror): - erts_exit(ERTS_ERROR_EXIT, "fclearerror/i_fcheckerror without fpe signals (beam_emu)"); -# define ERTS_NO_FPE_CHECK_INIT ERTS_FP_CHECK_INIT -# define ERTS_NO_FPE_ERROR ERTS_FP_ERROR -#else -# define ERTS_NO_FPE_CHECK_INIT(p) -# define ERTS_NO_FPE_ERROR(p, a, b) - - OpCase(fclearerror): { - BeamInstr *next; - - PreFetch(0, next); - ERTS_FP_CHECK_INIT(c_p); - NextPF(0, next); - } - - OpCase(i_fcheckerror): { - BeamInstr *next; - - PreFetch(0, next); - ERTS_FP_ERROR(c_p, freg[0].fd, goto fbadarith); - NextPF(0, next); - } -#endif - - - OpCase(i_fadd_lll): { - BeamInstr *next; - - PreFetch(3, next); - ERTS_NO_FPE_CHECK_INIT(c_p); - lb(Arg(2)) = lb(Arg(0)) + lb(Arg(1)); - ERTS_NO_FPE_ERROR(c_p, lb(Arg(2)), goto fbadarith); - NextPF(3, next); - } - OpCase(i_fsub_lll): { - BeamInstr *next; - - PreFetch(3, next); - ERTS_NO_FPE_CHECK_INIT(c_p); - lb(Arg(2)) = lb(Arg(0)) - lb(Arg(1)); - ERTS_NO_FPE_ERROR(c_p, lb(Arg(2)), goto fbadarith); - NextPF(3, next); - } - OpCase(i_fmul_lll): { - BeamInstr *next; - - PreFetch(3, next); - ERTS_NO_FPE_CHECK_INIT(c_p); - lb(Arg(2)) = lb(Arg(0)) * lb(Arg(1)); - ERTS_NO_FPE_ERROR(c_p, lb(Arg(2)), goto fbadarith); - NextPF(3, next); - } - OpCase(i_fdiv_lll): { - BeamInstr *next; - - PreFetch(3, next); - ERTS_NO_FPE_CHECK_INIT(c_p); - lb(Arg(2)) = lb(Arg(0)) / lb(Arg(1)); - ERTS_NO_FPE_ERROR(c_p, lb(Arg(2)), goto fbadarith); - NextPF(3, next); - } - OpCase(i_fnegate_ll): { - BeamInstr *next; - - PreFetch(2, next); - ERTS_NO_FPE_CHECK_INIT(c_p); - lb(Arg(1)) = -lb(Arg(0)); - ERTS_NO_FPE_ERROR(c_p, lb(Arg(1)), goto fbadarith); - NextPF(2, next); - - fbadarith: - c_p->freason = BADARITH; - goto find_func_info; - } - -#ifdef HIPE - { -#define HIPE_MODE_SWITCH(Cmd) \ - SWAPOUT; \ - ERTS_DBG_CHK_REDS(c_p, FCALLS); \ - c_p->fcalls = FCALLS; \ - c_p->def_arg_reg[4] = -neg_o_reds; \ - c_p = hipe_mode_switch(c_p, Cmd, reg); \ - goto L_post_hipe_mode_switch - - OpCase(hipe_trap_call): { - /* - * I[-5]: &&lb_i_func_info_IaaI - * I[-4]: Native code callee (inserted by HiPE) - * I[-3]: Module (tagged atom) - * I[-2]: Function (tagged atom) - * I[-1]: Arity (untagged integer) - * I[ 0]: &&lb_hipe_trap_call - * ... remainder of original BEAM code - */ - ErtsCodeInfo *ci = erts_code_to_codeinfo(I); - ASSERT(ci->op == (Uint) OpCode(i_func_info_IaaI)); - c_p->hipe.u.ncallee = ci->u.ncallee; - ++hipe_trap_count; - HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_CALL | (ci->mfa.arity << 8)); - } - OpCase(hipe_trap_call_closure): { - ErtsCodeInfo *ci = erts_code_to_codeinfo(I); - ASSERT(ci->op == (Uint) OpCode(i_func_info_IaaI)); - c_p->hipe.u.ncallee = ci->u.ncallee; - ++hipe_trap_count; - HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_CALL_CLOSURE | (ci->mfa.arity << 8)); - } - OpCase(hipe_trap_return): { - HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_RETURN); - } - OpCase(hipe_trap_throw): { - HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_THROW); - } - OpCase(hipe_trap_resume): { - HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_RESUME); - } -#undef HIPE_MODE_SWITCH - - L_post_hipe_mode_switch: -#ifdef DEBUG - pid = c_p->common.id; /* may have switched process... */ -#endif - reg = erts_proc_sched_data(c_p)->x_reg_array; - freg = erts_proc_sched_data(c_p)->f_reg_array; - ERL_BITS_RELOAD_STATEP(c_p); - /* XXX: this abuse of def_arg_reg[] is horrid! */ - neg_o_reds = -c_p->def_arg_reg[4]; - FCALLS = c_p->fcalls; - SWAPIN; - ERTS_DBG_CHK_REDS(c_p, FCALLS); - switch( c_p->def_arg_reg[3] ) { - case HIPE_MODE_SWITCH_RES_RETURN: - ASSERT(is_value(reg[0])); - SET_I(c_p->cp); - c_p->cp = 0; - Goto(*I); - case HIPE_MODE_SWITCH_RES_CALL_EXPORTED: - c_p->i = c_p->hipe.u.callee_exp->addressv[erts_active_code_ix()]; - /*fall through*/ - case HIPE_MODE_SWITCH_RES_CALL_BEAM: - SET_I(c_p->i); - Dispatch(); - case HIPE_MODE_SWITCH_RES_CALL_CLOSURE: - /* This can be used to call any function value, but currently it's - only used to call closures referring to unloaded modules. */ - { - BeamInstr *next; - - next = call_fun(c_p, c_p->arity - 1, reg, THE_NON_VALUE); - HEAVY_SWAPIN; - if (next != NULL) { - SET_I(next); - Dispatchfun(); - } - goto find_func_info; - } - case HIPE_MODE_SWITCH_RES_THROW: - c_p->cp = NULL; - I = handle_error(c_p, I, reg, NULL); - goto post_error_handling; - default: - erts_exit(ERTS_ERROR_EXIT, "hipe_mode_switch: result %u\n", c_p->def_arg_reg[3]); - } - } - OpCase(hipe_call_count): { - /* - * I[-5]: &&lb_i_func_info_IaaI - * I[-4]: pointer to struct hipe_call_count (inserted by HiPE) - * I[-3]: Module (tagged atom) - * I[-2]: Function (tagged atom) - * I[-1]: Arity (untagged integer) - * I[ 0]: &&lb_hipe_call_count - * ... remainder of original BEAM code - */ - ErtsCodeInfo *ci = erts_code_to_codeinfo(I); - struct hipe_call_count *hcc = ci->u.hcc; - ASSERT(ci->op == (Uint) OpCode(i_func_info_IaaI)); - ASSERT(hcc != NULL); - ASSERT(VALID_INSTR(hcc->opcode)); - ++(hcc->count); - Goto(hcc->opcode); - } -#endif /* HIPE */ - - OpCase(i_yield): - { - /* This is safe as long as REDS_IN(c_p) is never stored - * in c_p->arg_reg[0]. It is currently stored in c_p->def_arg_reg[5], - * which may be c_p->arg_reg[5], which is close, but no banana. - */ - c_p->arg_reg[0] = am_true; - c_p->arity = 1; /* One living register (the 'true' return value) */ - SWAPOUT; - c_p->i = I + 1; /* Next instruction */ - c_p->current = NULL; - goto do_schedule; - } - - OpCase(i_hibernate): { - HEAVY_SWAPOUT; - if (erts_hibernate(c_p, r(0), x(1), x(2), reg)) { - FCALLS = c_p->fcalls; - c_p->flags &= ~F_HIBERNATE_SCHED; - goto do_schedule; - } else { - HEAVY_SWAPIN; - I = handle_error(c_p, I, reg, &bif_export[BIF_hibernate_3]->info.mfa); - goto post_error_handling; - } - } - - /* This is optimised as an instruction because - it has to be very very fast */ - OpCase(i_perf_counter): { - BeamInstr* next; - ErtsSysPerfCounter ts; - PreFetch(0, next); - - ts = erts_sys_perf_counter(); - - if (IS_SSMALL(ts)) { - r(0) = make_small((Sint)ts); - } else { - TestHeap(ERTS_SINT64_HEAP_SIZE(ts),0); - r(0) = make_big(HTOP); -#if defined(ARCH_32) - if (ts >= (((Uint64) 1) << 32)) { - *HTOP = make_pos_bignum_header(2); - BIG_DIGIT(HTOP, 0) = (Uint) (ts & ((Uint) 0xffffffff)); - BIG_DIGIT(HTOP, 1) = (Uint) ((ts >> 32) & ((Uint) 0xffffffff)); - HTOP += 3; - } - else -#endif - { - *HTOP = make_pos_bignum_header(1); - BIG_DIGIT(HTOP, 0) = (Uint) ts; - HTOP += 2; - } - } - NextPF(0, next); - } - - OpCase(i_debug_breakpoint): { - HEAVY_SWAPOUT; - I = call_error_handler(c_p, erts_code_to_codemfa(I), reg, am_breakpoint); - HEAVY_SWAPIN; - if (I) { - Goto(*I); - } - goto handle_error; - } - - - OpCase(system_limit_j): - system_limit: - c_p->freason = SYSTEM_LIMIT; - goto lb_Cl_error; - - #ifdef ERTS_OPCODE_COUNTER_SUPPORT DEFINE_COUNTING_LABELS; #endif @@ -5872,7 +2329,6 @@ erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* re int arity; Eterm tmp; - if (is_not_atom(module) || is_not_atom(function)) { /* * No need to test args here -- done below. |