diff options
Diffstat (limited to 'erts/emulator/beam/beam_emu.c')
-rw-r--r-- | erts/emulator/beam/beam_emu.c | 3421 |
1 files changed, 1871 insertions, 1550 deletions
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 8b409e139b..bc83699951 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2014. All Rights Reserved. + * Copyright Ericsson AB 1996-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -38,6 +38,7 @@ #include "beam_bp.h" #include "beam_catches.h" #include "erl_thr_progress.h" +#include "erl_nfunc_sched.h" #ifdef HIPE #include "hipe_mode_switch.h" #include "hipe_bif1.h" @@ -64,18 +65,21 @@ # ifdef ERTS_SMP # define PROCESS_MAIN_CHK_LOCKS(P) \ do { \ - if ((P)) { \ + if ((P)) \ erts_proc_lc_chk_only_proc_main((P)); \ - } \ - else \ - erts_lc_check_exact(NULL, 0); \ - ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking()); \ + ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking()); \ +} while (0) +# define ERTS_SMP_REQ_PROC_MAIN_LOCK(P) \ +do { \ + if ((P)) \ + erts_proc_lc_require_lock((P), ERTS_PROC_LOCK_MAIN, \ + __FILE__, __LINE__); \ +} while (0) +# define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P) \ +do { \ + if ((P)) \ + erts_proc_lc_unrequire_lock((P), ERTS_PROC_LOCK_MAIN); \ } while (0) -# define ERTS_SMP_REQ_PROC_MAIN_LOCK(P) \ - if ((P)) erts_proc_lc_require_lock((P), ERTS_PROC_LOCK_MAIN,\ - __FILE__, __LINE__) -# define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P) \ - if ((P)) erts_proc_lc_unrequire_lock((P), ERTS_PROC_LOCK_MAIN) # else # define ERTS_SMP_REQ_PROC_MAIN_LOCK(P) # define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P) @@ -99,10 +103,7 @@ do { \ do { \ int i_; \ int Arity_ = PC[-1]; \ - if (Arity_ > 0) { \ - CHECK_TERM(r(0)); \ - } \ - for (i_ = 1; i_ < Arity_; i_++) { \ + for (i_ = 0; i_ < Arity_; i_++) { \ CHECK_TERM(x(i_)); \ } \ } while (0) @@ -116,13 +117,16 @@ do { \ #define MAX(x, y) (((x) > (y)) ? (x) : (y)) #endif -#define GET_BIF_ADDRESS(p) ((BifFunction) (((Export *) p)->code[4])) +#define GET_BIF_MODULE(p) (p->info.mfa.module) +#define GET_BIF_FUNCTION(p) (p->info.mfa.function) +#define GET_BIF_ARITY(p) (p->info.mfa.arity) +#define GET_BIF_ADDRESS(p) ((BifFunction) (p->beam[1])) #define TermWords(t) (((t) / (sizeof(BeamInstr)/sizeof(Eterm))) + !!((t) % (sizeof(BeamInstr)/sizeof(Eterm)))) /* * We reuse some of fields in the save area in the process structure. - * This is safe to do, since this space is only activly used when + * This is safe to do, since this space is only actively used when * the process is switched out. */ #define REDS_IN(p) ((p)->def_arg_reg[5]) @@ -151,51 +155,41 @@ do { \ ASSERT(VALID_INSTR(* (Eterm *)(ip))); \ I = (ip) -#define FetchArgs(S1, S2) tmp_arg1 = (S1); tmp_arg2 = (S2) +/* + * Register target (X or Y register). + */ + +#define REG_TARGET_PTR(Target) (((Target) & 1) ? &yb(Target-1) : &xb(Target)) +#define REG_TARGET(Target) (*REG_TARGET_PTR(Target)) /* * Store a result into a register given a destination descriptor. */ -#define StoreResult(Result, DestDesc) \ - do { \ - Eterm stb_reg; \ - stb_reg = (DestDesc); \ - CHECK_TERM(Result); \ - switch (beam_reg_tag(stb_reg)) { \ - case R_REG_DEF: \ - r(0) = (Result); break; \ - case X_REG_DEF: \ - xb(x_reg_offset(stb_reg)) = (Result); break; \ - default: \ - yb(y_reg_offset(stb_reg)) = (Result); break; \ - } \ +#define StoreResult(Result, DestDesc) \ + do { \ + Eterm stb_reg; \ + stb_reg = (DestDesc); \ + CHECK_TERM(Result); \ + REG_TARGET(stb_reg) = (Result); \ } while (0) -#define StoreSimpleDest(Src, Dest) Dest = (Src) - /* * Store a result into a register and execute the next instruction. * Dst points to the word with a destination descriptor, which MUST * be just before the next instruction. */ -#define StoreBifResult(Dst, Result) \ - do { \ - BeamInstr* stb_next; \ - Eterm stb_reg; \ - stb_reg = Arg(Dst); \ - I += (Dst) + 2; \ - stb_next = (BeamInstr *) *I; \ - CHECK_TERM(Result); \ - switch (beam_reg_tag(stb_reg)) { \ - case R_REG_DEF: \ - r(0) = (Result); Goto(stb_next); \ - case X_REG_DEF: \ - xb(x_reg_offset(stb_reg)) = (Result); Goto(stb_next); \ - default: \ - yb(y_reg_offset(stb_reg)) = (Result); Goto(stb_next); \ - } \ +#define StoreBifResult(Dst, Result) \ + do { \ + BeamInstr* stb_next; \ + Eterm stb_reg; \ + stb_reg = Arg(Dst); \ + I += (Dst) + 2; \ + stb_next = (BeamInstr *) *I; \ + CHECK_TERM(Result); \ + REG_TARGET(stb_reg) = (Result); \ + Goto(stb_next); \ } while (0) #define ClauseFail() goto jump_f @@ -221,11 +215,12 @@ BeamInstr beam_continue_exit[1]; BeamInstr* em_call_error_handler; BeamInstr* em_apply_bif; BeamInstr* em_call_nif; +BeamInstr* em_call_bif_e; /* NOTE These should be the only variables containing trace instructions. ** Sometimes tests are form the instruction value, and sometimes -** for the refering variable (one of these), and rouge references +** for the referring variable (one of these), and rouge references ** will most likely cause chaos. */ BeamInstr beam_return_to_trace[1]; /* OpCode(i_return_to_trace) */ @@ -250,6 +245,14 @@ void** beam_ops; HEAP_TOP(c_p) = HTOP; \ c_p->stop = E +#define HEAVY_SWAPIN \ + SWAPIN; \ + FCALLS = c_p->fcalls + +#define HEAVY_SWAPOUT \ + SWAPOUT; \ + c_p->fcalls = FCALLS + /* * Use LIGHT_SWAPOUT when the called function * will call HeapOnlyAlloc() (and never HAlloc()). @@ -293,7 +296,7 @@ void** beam_ops; #define Ib(N) (N) #define x(N) reg[N] #define y(N) E[N] -#define r(N) x##N +#define r(N) x(N) /* * Makes sure that there are StackNeed + HeapNeed + 1 words available @@ -309,12 +312,11 @@ void** beam_ops; needed = (StackNeed) + 1; \ if (E - HTOP < (needed + (HeapNeed))) { \ SWAPOUT; \ - reg[0] = r(0); \ PROCESS_MAIN_CHK_LOCKS(c_p); \ - FCALLS -= erts_garbage_collect(c_p, needed + (HeapNeed), reg, (M)); \ + FCALLS -= erts_garbage_collect_nobump(c_p, needed + (HeapNeed), \ + reg, (M), FCALLS); \ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \ PROCESS_MAIN_CHK_LOCKS(c_p); \ - r(0) = reg[0]; \ SWAPIN; \ } \ E -= needed; \ @@ -363,12 +365,10 @@ void** beam_ops; unsigned need = (Nh); \ if ((E - HTOP < need) || (MSO(c_p).overhead + (VNh) >= BIN_VHEAP_SZ(c_p))) {\ SWAPOUT; \ - reg[0] = r(0); \ PROCESS_MAIN_CHK_LOCKS(c_p); \ - FCALLS -= erts_garbage_collect(c_p, need, reg, (Live)); \ + FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live), FCALLS); \ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \ PROCESS_MAIN_CHK_LOCKS(c_p); \ - r(0) = reg[0]; \ SWAPIN; \ } \ HEAP_SPACE_VERIFIED(need); \ @@ -386,12 +386,10 @@ void** beam_ops; unsigned need = (Nh); \ if (E - HTOP < need) { \ SWAPOUT; \ - reg[0] = r(0); \ PROCESS_MAIN_CHK_LOCKS(c_p); \ - FCALLS -= erts_garbage_collect(c_p, need, reg, (Live)); \ + FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live), FCALLS); \ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \ PROCESS_MAIN_CHK_LOCKS(c_p); \ - r(0) = reg[0]; \ SWAPIN; \ } \ HEAP_SPACE_VERIFIED(need); \ @@ -408,15 +406,11 @@ void** beam_ops; unsigned need = (Nh); \ if (E - HTOP < need) { \ SWAPOUT; \ - reg[0] = r(0); \ reg[Live] = Extra; \ PROCESS_MAIN_CHK_LOCKS(c_p); \ - FCALLS -= erts_garbage_collect(c_p, need, reg, (Live)+1); \ + FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live)+1, FCALLS); \ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \ PROCESS_MAIN_CHK_LOCKS(c_p); \ - if (Live > 0) { \ - r(0) = reg[0]; \ - } \ Extra = reg[Live]; \ SWAPIN; \ } \ @@ -426,7 +420,7 @@ void** beam_ops; #define TestHeapPutList(Need, Reg) \ do { \ TestHeap((Need), 1); \ - PutList(Reg, r(0), r(0), StoreSimpleDest); \ + PutList(Reg, r(0), r(0)); \ CHECK_TERM(r(0)); \ } while (0) @@ -438,10 +432,9 @@ void** beam_ops; #define MakeFun(FunP, NumFree) \ do { \ - SWAPOUT; \ - reg[0] = r(0); \ + HEAVY_SWAPOUT; \ r(0) = new_fun(c_p, reg, (ErlFunEntry *) FunP, NumFree); \ - SWAPIN; \ + HEAVY_SWAPIN; \ } while (0) #define PutTuple(Dst, Arity) \ @@ -531,15 +524,19 @@ void** beam_ops; ASSERT(VALID_INSTR(Dst)); \ Goto(Dst) -#define GetR(pos, tr) \ - do { \ - tr = Arg(pos); \ - switch (beam_reg_tag(tr)) { \ - case R_REG_DEF: tr = r(0); break; \ - case X_REG_DEF: tr = xb(x_reg_offset(tr)); break; \ - case Y_REG_DEF: ASSERT(y_reg_offset(tr) >= 1); tr = yb(y_reg_offset(tr)); break; \ - } \ - CHECK_TERM(tr); \ +#define GetR(pos, tr) \ + do { \ + tr = Arg(pos); \ + switch (loader_tag(tr)) { \ + case LOADER_X_REG: \ + tr = x(loader_x_reg_index(tr)); \ + break; \ + case LOADER_Y_REG: \ + ASSERT(loader_y_reg_index(tr) >= 1); \ + tr = y(loader_y_reg_index(tr)); \ + break; \ + } \ + CHECK_TERM(tr); \ } while (0) #define GetArg1(N, Dst) GetR((N), Dst) @@ -550,62 +547,140 @@ void** beam_ops; GetR((N)+1, Dst2); \ } while (0) -#define PutList(H, T, Dst, Store) \ - do { \ - HTOP[0] = (H); HTOP[1] = (T); \ - Store(make_list(HTOP), Dst); \ - HTOP += 2; \ +#define PutList(H, T, Dst) \ + do { \ + HTOP[0] = (H); HTOP[1] = (T); \ + Dst = make_list(HTOP); \ + HTOP += 2; \ } while (0) -#define Move(Src, Dst, Store) \ - do { \ - Eterm term = (Src); \ - Store(term, Dst); \ - } while (0) +#define Swap(R1, R2) \ + do { \ + Eterm V = R1; \ + R1 = R2; \ + R2 = V; \ + } while (0) + +#define SwapTemp(R1, R2, Tmp) \ + do { \ + Eterm V = R1; \ + R1 = R2; \ + R2 = Tmp = V; \ + } while (0) + +#define Move(Src, Dst) Dst = (Src) + +#define Move2Par(S1, D1, S2, D2) \ + do { \ + Eterm V1, V2; \ + V1 = (S1); V2 = (S2); D1 = V1; D2 = V2; \ + } while (0) + +#define MoveShift(Src, SD, D) \ + do { \ + Eterm V; \ + V = Src; D = SD; SD = V; \ + } while (0) + +#define MoveDup(Src, D1, D2) \ + do { \ + D1 = D2 = (Src); \ + } while (0) -#define Move2(S1, D1, S2, D2) D1 = (S1); D2 = (S2) #define Move3(S1, D1, S2, D2, S3, D3) D1 = (S1); D2 = (S2); D3 = (S3) -#define MoveGenDest(src, dstp) \ - if ((dstp) == NULL) { r(0) = (src); } else { *(dstp) = src; } +#define MoveWindow3(S1, S2, S3, D) \ + do { \ + Eterm xt0, xt1, xt2; \ + Eterm *y = &D; \ + xt0 = S1; \ + xt1 = S2; \ + xt2 = S3; \ + y[0] = xt0; \ + y[1] = xt1; \ + y[2] = xt2; \ + } while (0) -#define MoveReturn(Src, Dest) \ - (Dest) = (Src); \ - I = c_p->cp; \ - ASSERT(VALID_INSTR(*c_p->cp)); \ - c_p->cp = 0; \ - CHECK_TERM(r(0)); \ - Goto(*I) +#define MoveWindow4(S1, S2, S3, S4, D) \ + do { \ + Eterm xt0, xt1, xt2, xt3; \ + Eterm *y = &D; \ + xt0 = S1; \ + xt1 = S2; \ + xt2 = S3; \ + xt3 = S4; \ + y[0] = xt0; \ + y[1] = xt1; \ + y[2] = xt2; \ + y[3] = xt3; \ + } while (0) + +#define MoveWindow5(S1, S2, S3, S4, S5, D) \ + do { \ + Eterm xt0, xt1, xt2, xt3, xt4; \ + Eterm *y = &D; \ + xt0 = S1; \ + xt1 = S2; \ + xt2 = S3; \ + xt3 = S4; \ + xt4 = S5; \ + y[0] = xt0; \ + y[1] = xt1; \ + y[2] = xt2; \ + y[3] = xt3; \ + y[4] = xt4; \ + } while (0) + +#define DispatchReturn \ +do { \ + if (FCALLS > 0 || FCALLS > neg_o_reds) { \ + FCALLS--; \ + Goto(*I); \ + } \ + else { \ + c_p->current = NULL; \ + c_p->arity = 1; \ + goto context_switch3; \ + } \ +} while (0) + +#define MoveReturn(Src) \ + x(0) = (Src); \ + I = c_p->cp; \ + ASSERT(VALID_INSTR(*c_p->cp)); \ + c_p->cp = 0; \ + CHECK_TERM(r(0)); \ + DispatchReturn #define DeallocateReturn(Deallocate) \ do { \ int words_to_pop = (Deallocate); \ - SET_I((BeamInstr *) cp_val(*E)); \ + SET_I((BeamInstr *) cp_val(*E)); \ E = ADD_BYTE_OFFSET(E, words_to_pop); \ CHECK_TERM(r(0)); \ - Goto(*I); \ + DispatchReturn; \ } while (0) -#define MoveDeallocateReturn(Src, Dest, Deallocate) \ - (Dest) = (Src); \ +#define MoveDeallocateReturn(Src, Deallocate) \ + x(0) = (Src); \ DeallocateReturn(Deallocate) -#define MoveCall(Src, Dest, CallDest, Size) \ - (Dest) = (Src); \ +#define MoveCall(Src, CallDest, Size) \ + x(0) = (Src); \ SET_CP(c_p, I+Size+1); \ - SET_I((BeamInstr *) CallDest); \ + SET_I((BeamInstr *) CallDest); \ Dispatch(); -#define MoveCallLast(Src, Dest, CallDest, Deallocate) \ - (Dest) = (Src); \ - RESTORE_CP(E); \ - E = ADD_BYTE_OFFSET(E, (Deallocate)); \ - SET_I((BeamInstr *) CallDest); \ +#define MoveCallLast(Src, CallDest, Deallocate) \ + x(0) = (Src); \ + RESTORE_CP(E); \ + E = ADD_BYTE_OFFSET(E, (Deallocate)); \ + SET_I((BeamInstr *) CallDest); \ Dispatch(); -#define MoveCallOnly(Src, Dest, CallDest) \ - (Dest) = (Src); \ - SET_I((BeamInstr *) CallDest); \ +#define MoveCallOnly(Src, CallDest) \ + x(0) = (Src); \ + SET_I((BeamInstr *) CallDest); \ Dispatch(); #define MoveJump(Src) \ @@ -613,60 +688,69 @@ void** beam_ops; SET_I((BeamInstr *) Arg(0)); \ Goto(*I); -#define GetList(Src, H, T) do { \ - Eterm* tmp_ptr = list_val(Src); \ - H = CAR(tmp_ptr); \ - T = CDR(tmp_ptr); } while (0) - -#define GetTupleElement(Src, Element, Dest) \ - do { \ - tmp_arg1 = (Eterm) COMPRESS_POINTER(((unsigned char *) tuple_val(Src)) + \ - (Element)); \ - (Dest) = (*(Eterm *) EXPAND_POINTER(tmp_arg1)); \ +#define GetList(Src, H, T) \ + do { \ + Eterm* tmp_ptr = list_val(Src); \ + Eterm hd, tl; \ + hd = CAR(tmp_ptr); \ + tl = CDR(tmp_ptr); \ + H = hd; T = tl; \ } while (0) -#define ExtractNextElement(Dest) \ - tmp_arg1 += sizeof(Eterm); \ - (Dest) = (* (Eterm *) (((unsigned char *) EXPAND_POINTER(tmp_arg1)))) - -#define ExtractNextElement2(Dest) \ - do { \ - Eterm* ene_dstp = &(Dest); \ - ene_dstp[0] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[1]; \ - ene_dstp[1] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[2]; \ - tmp_arg1 += sizeof(Eterm) + sizeof(Eterm); \ +#define GetTupleElement(Src, Element, Dest) \ + do { \ + Eterm* src; \ + src = ADD_BYTE_OFFSET(tuple_val(Src), (Element)); \ + (Dest) = *src; \ } while (0) -#define ExtractNextElement3(Dest) \ - do { \ - Eterm* ene_dstp = &(Dest); \ - ene_dstp[0] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[1]; \ - ene_dstp[1] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[2]; \ - ene_dstp[2] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[3]; \ - tmp_arg1 += 3*sizeof(Eterm); \ +#define GetTupleElement2(Src, Element, Dest) \ + do { \ + Eterm* src; \ + Eterm* dst; \ + Eterm E1, E2; \ + src = ADD_BYTE_OFFSET(tuple_val(Src), (Element)); \ + dst = &(Dest); \ + E1 = src[0]; \ + E2 = src[1]; \ + dst[0] = E1; \ + dst[1] = E2; \ } while (0) -#define ExtractNextElement4(Dest) \ - do { \ - Eterm* ene_dstp = &(Dest); \ - ene_dstp[0] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[1]; \ - ene_dstp[1] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[2]; \ - ene_dstp[2] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[3]; \ - ene_dstp[3] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[4]; \ - tmp_arg1 += 4*sizeof(Eterm); \ +#define GetTupleElement2Y(Src, Element, D1, D2) \ + do { \ + Eterm* src; \ + Eterm E1, E2; \ + src = ADD_BYTE_OFFSET(tuple_val(Src), (Element)); \ + E1 = src[0]; \ + E2 = src[1]; \ + D1 = E1; \ + D2 = E2; \ } while (0) -#define ExtractElement(Element, Dest) \ - do { \ - tmp_arg1 += (Element); \ - (Dest) = (* (Eterm *) EXPAND_POINTER(tmp_arg1)); \ +#define GetTupleElement3(Src, Element, Dest) \ + do { \ + Eterm* src; \ + Eterm* dst; \ + Eterm E1, E2, E3; \ + src = ADD_BYTE_OFFSET(tuple_val(Src), (Element)); \ + dst = &(Dest); \ + E1 = src[0]; \ + E2 = src[1]; \ + E3 = src[2]; \ + dst[0] = E1; \ + dst[1] = E2; \ + dst[2] = E3; \ } while (0) #define EqualImmed(X, Y, Action) if (X != Y) { Action; } #define NotEqualImmed(X, Y, Action) if (X == Y) { Action; } #define EqualExact(X, Y, Action) if (!EQ(X,Y)) { Action; } -#define IsLessThan(X, Y, Action) if (CMP_GE(X, Y)) { Action; } -#define IsGreaterEqual(X, Y, Action) if (CMP_LT(X, Y)) { Action; } +#define NotEqualExact(X, Y, Action) if (EQ(X,Y)) { Action; } +#define Equal(X, Y, Action) CMP_EQ_ACTION(X,Y,Action) +#define NotEqual(X, Y, Action) CMP_NE_ACTION(X,Y,Action) +#define IsLessThan(X, Y, Action) CMP_LT_ACTION(X,Y,Action) +#define IsGreaterEqual(X, Y, Action) CMP_GE_ACTION(X,Y,Action) #define IsFloat(Src, Fail) if (is_not_float(Src)) { Fail; } @@ -690,18 +774,26 @@ void** beam_ops; if (is_not_list(Src)) { Fail; } \ A(Need, Alive) -#define IsNonemptyListTestHeap(Src, Need, Alive, Fail) \ - if (is_not_list(Src)) { Fail; } \ +#define IsNonemptyListTestHeap(Need, Alive, Fail) \ + if (is_not_list(x(0))) { Fail; } \ TestHeap(Need, Alive) +#define IsNonemptyListGetList(Src, H, T, Fail) \ + if (is_not_list(Src)) { \ + Fail; \ + } else { \ + Eterm* tmp_ptr = list_val(Src); \ + Eterm hd, tl; \ + hd = CAR(tmp_ptr); \ + tl = CDR(tmp_ptr); \ + H = hd; T = tl; \ + } + #define IsTuple(X, Action) if (is_not_tuple(X)) Action -#define IsArity(Pointer, Arity, Fail) \ - if (*(Eterm *) \ - EXPAND_POINTER(tmp_arg1 = (Eterm) \ - COMPRESS_POINTER(tuple_val(Pointer))) != (Arity)) \ - { \ - Fail; \ +#define IsArity(Pointer, Arity, Fail) \ + if (*tuple_val(Pointer) != (Arity)) { \ + Fail; \ } #define IsMap(Src, Fail) if (!is_map(Src)) { Fail; } @@ -738,14 +830,29 @@ void** beam_ops; } \ } while (0) -#define IsTupleOfArity(Src, Arity, Fail) \ - do { \ - if (is_not_tuple(Src) || \ - *(Eterm *) \ - EXPAND_POINTER(tmp_arg1 = \ - (Eterm) COMPRESS_POINTER(tuple_val(Src))) != Arity) { \ - Fail; \ - } \ +#ifdef DEBUG +#define IsTupleOfArity(Src, Arityval, Fail) \ + do { \ + if (!(is_tuple(Src) && *tuple_val(Src) == Arityval)) { \ + Fail; \ + } \ + } while (0) +#else +#define IsTupleOfArity(Src, Arityval, Fail) \ + do { \ + if (!(is_boxed(Src) && *tuple_val(Src) == Arityval)) { \ + Fail; \ + } \ + } while (0) +#endif + +#define IsTaggedTuple(Src,Arityval,Tag,Fail) \ + do { \ + if (!(is_tuple(Src) && \ + (tuple_val(Src))[0] == Arityval && \ + (tuple_val(Src))[1] == Tag)) { \ + Fail; \ + } \ } while (0) #define IsBoolean(X, Fail) if ((X) != am_true && (X) != am_false) { Fail; } @@ -756,7 +863,7 @@ void** beam_ops; #define IsBitstring(Src, Fail) \ if (is_not_binary(Src)) { Fail; } -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) #define BsSafeMul(A, B, Fail, Target) \ do { Uint64 _res = (A) * (B); \ if (_res / B != A) { Fail; } \ @@ -773,6 +880,7 @@ void** beam_ops; #define BsGetFieldSize(Bits, Unit, Fail, Target) \ do { \ Sint _signed_size; Uint _uint_size; \ + Uint temp_bits; \ if (is_small(Bits)) { \ _signed_size = signed_val(Bits); \ if (_signed_size < 0) { Fail; } \ @@ -787,6 +895,7 @@ void** beam_ops; #define BsGetUncheckedFieldSize(Bits, Unit, Fail, Target) \ do { \ Sint _signed_size; Uint _uint_size; \ + Uint temp_bits; \ if (is_small(Bits)) { \ _signed_size = signed_val(Bits); \ if (_signed_size < 0) { Fail; } \ @@ -798,7 +907,7 @@ void** beam_ops; Target = _uint_size * Unit; \ } while (0) -#define BsGetFloat2(Ms, Live, Sz, Flags, Dst, Store, Fail) \ +#define BsGetFloat2(Ms, Live, Sz, Flags, Dst, Fail) \ do { \ ErlBinMatchBuffer *_mb; \ Eterm _result; Sint _size; \ @@ -809,12 +918,12 @@ void** beam_ops; LIGHT_SWAPOUT; \ _result = erts_bs_get_float_2(c_p, _size, (Flags), _mb); \ LIGHT_SWAPIN; \ - HEAP_SPACE_VERIFIED(0); \ + HEAP_SPACE_VERIFIED(0); \ if (is_non_value(_result)) { Fail; } \ - else { Store(_result, Dst); } \ + else { Dst = _result; } \ } while (0) -#define BsGetBinaryImm_2(Ms, Live, Sz, Flags, Dst, Store, Fail) \ +#define BsGetBinaryImm_2(Ms, Live, Sz, Flags, Dst, Fail) \ do { \ ErlBinMatchBuffer *_mb; \ Eterm _result; \ @@ -823,12 +932,12 @@ void** beam_ops; LIGHT_SWAPOUT; \ _result = erts_bs_get_binary_2(c_p, (Sz), (Flags), _mb); \ LIGHT_SWAPIN; \ - HEAP_SPACE_VERIFIED(0); \ + HEAP_SPACE_VERIFIED(0); \ if (is_non_value(_result)) { Fail; } \ - else { Store(_result, Dst); } \ + else { Dst = _result; } \ } while (0) -#define BsGetBinary_2(Ms, Live, Sz, Flags, Dst, Store, Fail) \ +#define BsGetBinary_2(Ms, Live, Sz, Flags, Dst, Fail) \ do { \ ErlBinMatchBuffer *_mb; \ Eterm _result; Uint _size; \ @@ -838,27 +947,27 @@ void** beam_ops; LIGHT_SWAPOUT; \ _result = erts_bs_get_binary_2(c_p, _size, (Flags), _mb); \ LIGHT_SWAPIN; \ - HEAP_SPACE_VERIFIED(0); \ + HEAP_SPACE_VERIFIED(0); \ if (is_non_value(_result)) { Fail; } \ - else { Store(_result, Dst); } \ + else { Dst = _result; } \ } while (0) -#define BsGetBinaryAll_2(Ms, Live, Unit, Dst, Store, Fail) \ - do { \ - ErlBinMatchBuffer *_mb; \ - Eterm _result; \ - TestHeap(ERL_SUB_BIN_SIZE, Live); \ - _mb = ms_matchbuffer(Ms); \ - if (((_mb->size - _mb->offset) % Unit) == 0) { \ - LIGHT_SWAPOUT; \ - _result = erts_bs_get_binary_all_2(c_p, _mb); \ - LIGHT_SWAPIN; \ - HEAP_SPACE_VERIFIED(0); \ - ASSERT(is_value(_result)); \ - Store(_result, Dst); \ - } else { \ - HEAP_SPACE_VERIFIED(0); \ - Fail; } \ +#define BsGetBinaryAll_2(Ms, Live, Unit, Dst, Fail) \ + do { \ + ErlBinMatchBuffer *_mb; \ + Eterm _result; \ + TestHeap(ERL_SUB_BIN_SIZE, Live); \ + _mb = ms_matchbuffer(Ms); \ + if (((_mb->size - _mb->offset) % Unit) == 0) { \ + LIGHT_SWAPOUT; \ + _result = erts_bs_get_binary_all_2(c_p, _mb); \ + LIGHT_SWAPIN; \ + HEAP_SPACE_VERIFIED(0); \ + ASSERT(is_value(_result)); \ + Dst = _result; \ + } else { \ + HEAP_SPACE_VERIFIED(0); \ + Fail; } \ } while (0) #define BsSkipBits2(Ms, Bits, Unit, Fail) \ @@ -953,14 +1062,17 @@ void** beam_ops; * The following functions are called directly by process_main(). * Don't inline them. */ -static BifFunction translate_gc_bif(void* gcf) NOINLINE; +static ErtsCodeMFA *ubif2mfa(void* uf) NOINLINE; +static ErtsCodeMFA *gcbif2mfa(void* gcf) NOINLINE; static BeamInstr* handle_error(Process* c_p, BeamInstr* pc, - Eterm* reg, BifFunction bf) NOINLINE; -static BeamInstr* call_error_handler(Process* p, BeamInstr* ip, + Eterm* reg, ErtsCodeMFA* bif_mfa) NOINLINE; +static BeamInstr* call_error_handler(Process* p, ErtsCodeMFA* mfa, Eterm* reg, Eterm func) NOINLINE; -static BeamInstr* fixed_apply(Process* p, Eterm* reg, Uint arity) NOINLINE; +static BeamInstr* fixed_apply(Process* p, Eterm* reg, Uint arity, + BeamInstr *I, Uint offs) NOINLINE; static BeamInstr* apply(Process* p, Eterm module, Eterm function, - Eterm args, Eterm* reg) NOINLINE; + Eterm args, Eterm* reg, + BeamInstr *I, Uint offs) NOINLINE; static BeamInstr* call_fun(Process* p, int arity, Eterm* reg, Eterm args) NOINLINE; static BeamInstr* apply_fun(Process* p, Eterm fun, @@ -982,14 +1094,14 @@ static BeamInstr* next_catch(Process* c_p, Eterm *reg); static void terminate_proc(Process* c_p, Eterm Value); static Eterm add_stacktrace(Process* c_p, Eterm Value, Eterm exc); static void save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, - BifFunction bf, Eterm args); + ErtsCodeMFA *bif_mfa, Eterm args); static struct StackTrace * get_trace_from_exc(Eterm exc); static Eterm make_arglist(Process* c_p, Eterm* reg, int a); void init_emulator(void) { - process_main(); + process_main(0, 0); } /* @@ -998,23 +1110,17 @@ init_emulator(void) */ #if defined(__GNUC__) && defined(sparc) && !defined(DEBUG) -# define REG_x0 asm("%l0") # define REG_xregs asm("%l1") # define REG_htop asm("%l2") # define REG_stop asm("%l3") # define REG_I asm("%l4") # define REG_fcalls asm("%l5") -# define REG_tmp_arg1 asm("%l6") -# define REG_tmp_arg2 asm("%l7") #else -# define REG_x0 # define REG_xregs # define REG_htop # define REG_stop # define REG_I # define REG_fcalls -# define REG_tmp_arg1 -# define REG_tmp_arg2 #endif #ifdef USE_VM_PROBES @@ -1023,107 +1129,119 @@ init_emulator(void) #ifdef USE_VM_CALL_PROBES -#define DTRACE_LOCAL_CALL(p, m, f, a) \ +#define DTRACE_LOCAL_CALL(p, mfa) \ if (DTRACE_ENABLED(local_function_entry)) { \ DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ - DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE); \ + DTRACE_CHARBUF(mfa_buf, DTRACE_TERM_BUF_SIZE); \ int depth = STACK_START(p) - STACK_TOP(p); \ - dtrace_fun_decode(p, m, f, a, \ - process_name, mfa); \ - DTRACE3(local_function_entry, process_name, mfa, depth); \ + dtrace_fun_decode(p, mfa, process_name, mfa_buf); \ + DTRACE3(local_function_entry, process_name, mfa_buf, depth); \ } -#define DTRACE_GLOBAL_CALL(p, m, f, a) \ +#define DTRACE_GLOBAL_CALL(p, mfa) \ if (DTRACE_ENABLED(global_function_entry)) { \ DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ - DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE); \ + DTRACE_CHARBUF(mfa_buf, DTRACE_TERM_BUF_SIZE); \ int depth = STACK_START(p) - STACK_TOP(p); \ - dtrace_fun_decode(p, m, f, a, \ - process_name, mfa); \ - DTRACE3(global_function_entry, process_name, mfa, depth); \ + dtrace_fun_decode(p, mfa, process_name, mfa_buf); \ + DTRACE3(global_function_entry, process_name, mfa_buf, depth); \ } -#define DTRACE_RETURN(p, m, f, a) \ +#define DTRACE_RETURN(p, mfa) \ if (DTRACE_ENABLED(function_return)) { \ DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ - DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE); \ + DTRACE_CHARBUF(mfa_buf, DTRACE_TERM_BUF_SIZE); \ int depth = STACK_START(p) - STACK_TOP(p); \ - dtrace_fun_decode(p, m, f, a, \ - process_name, mfa); \ - DTRACE3(function_return, process_name, mfa, depth); \ + dtrace_fun_decode(p, mfa, process_name, mfa_buf); \ + DTRACE3(function_return, process_name, mfa_buf, depth); \ } -#define DTRACE_BIF_ENTRY(p, m, f, a) \ - if (DTRACE_ENABLED(bif_entry)) { \ - DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ - DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE); \ - dtrace_fun_decode(p, m, f, a, \ - process_name, mfa); \ - DTRACE2(bif_entry, process_name, mfa); \ +#define DTRACE_BIF_ENTRY(p, mfa) \ + if (DTRACE_ENABLED(bif_entry)) { \ + DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ + DTRACE_CHARBUF(mfa_buf, DTRACE_TERM_BUF_SIZE); \ + dtrace_fun_decode(p, mfa, process_name, mfa_buf); \ + DTRACE2(bif_entry, process_name, mfa_buf); \ } -#define DTRACE_BIF_RETURN(p, m, f, a) \ - if (DTRACE_ENABLED(bif_return)) { \ - DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ - DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE); \ - dtrace_fun_decode(p, m, f, a, \ - process_name, mfa); \ - DTRACE2(bif_return, process_name, mfa); \ +#define DTRACE_BIF_RETURN(p, mfa) \ + if (DTRACE_ENABLED(bif_return)) { \ + DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ + DTRACE_CHARBUF(mfa_buf, DTRACE_TERM_BUF_SIZE); \ + dtrace_fun_decode(p, mfa, process_name, mfa_buf); \ + DTRACE2(bif_return, process_name, mfa_buf); \ } -#define DTRACE_NIF_ENTRY(p, m, f, a) \ - if (DTRACE_ENABLED(nif_entry)) { \ - DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ - DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE); \ - dtrace_fun_decode(p, m, f, a, \ - process_name, mfa); \ - DTRACE2(nif_entry, process_name, mfa); \ +#define DTRACE_NIF_ENTRY(p, mfa) \ + if (DTRACE_ENABLED(nif_entry)) { \ + DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ + DTRACE_CHARBUF(mfa_buf, DTRACE_TERM_BUF_SIZE); \ + dtrace_fun_decode(p, mfa, process_name, mfa_buf); \ + DTRACE2(nif_entry, process_name, mfa_buf); \ } -#define DTRACE_NIF_RETURN(p, m, f, a) \ - if (DTRACE_ENABLED(nif_return)) { \ - DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ - DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE); \ - dtrace_fun_decode(p, m, f, a, \ - process_name, mfa); \ - DTRACE2(nif_return, process_name, mfa); \ +#define DTRACE_NIF_RETURN(p, mfa) \ + if (DTRACE_ENABLED(nif_return)) { \ + DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ + DTRACE_CHARBUF(mfa_buf, DTRACE_TERM_BUF_SIZE); \ + dtrace_fun_decode(p, mfa, process_name, mfa_buf); \ + DTRACE2(nif_return, process_name, mfa_buf); \ } #define DTRACE_GLOBAL_CALL_FROM_EXPORT(p,e) \ do { \ if (DTRACE_ENABLED(global_function_entry)) { \ BeamInstr* fp = (BeamInstr *) (((Export *) (e))->addressv[erts_active_code_ix()]); \ - DTRACE_GLOBAL_CALL((p), (Eterm)fp[-3], (Eterm)fp[-2], fp[-1]); \ + DTRACE_GLOBAL_CALL((p), erts_code_to_codemfa(fp)); \ } \ } while(0) #define DTRACE_RETURN_FROM_PC(p) \ do { \ - BeamInstr* fp; \ - if (DTRACE_ENABLED(function_return) && (fp = find_function_from_pc((p)->cp))) { \ - DTRACE_RETURN((p), (Eterm)fp[0], (Eterm)fp[1], (Uint)fp[2]); \ + ErtsCodeMFA* cmfa; \ + if (DTRACE_ENABLED(function_return) && (cmfa = find_function_from_pc((p)->cp))) { \ + DTRACE_RETURN((p), cmfa); \ } \ } while(0) #else /* USE_VM_PROBES */ -#define DTRACE_LOCAL_CALL(p, m, f, a) do {} while (0) -#define DTRACE_GLOBAL_CALL(p, m, f, a) do {} while (0) +#define DTRACE_LOCAL_CALL(p, mfa) do {} while (0) +#define DTRACE_GLOBAL_CALL(p, mfa) do {} while (0) #define DTRACE_GLOBAL_CALL_FROM_EXPORT(p, e) do {} while (0) -#define DTRACE_RETURN(p, m, f, a) do {} while (0) +#define DTRACE_RETURN(p, mfa) do {} while (0) #define DTRACE_RETURN_FROM_PC(p) do {} while (0) -#define DTRACE_BIF_ENTRY(p, m, f, a) do {} while (0) -#define DTRACE_BIF_RETURN(p, m, f, a) do {} while (0) -#define DTRACE_NIF_ENTRY(p, m, f, a) do {} while (0) -#define DTRACE_NIF_RETURN(p, m, f, a) do {} while (0) +#define DTRACE_BIF_ENTRY(p, mfa) do {} while (0) +#define DTRACE_BIF_RETURN(p, mfa) do {} while (0) +#define DTRACE_NIF_ENTRY(p, mfa) do {} while (0) +#define DTRACE_NIF_RETURN(p, mfa) do {} while (0) #endif /* USE_VM_PROBES */ +#ifdef DEBUG +#define ERTS_DBG_CHK_REDS(P, FC) \ + do { \ + if (ERTS_PROC_GET_SAVED_CALLS_BUF((P))) { \ + ASSERT(FC <= 0); \ + ASSERT(erts_proc_sched_data(c_p)->virtual_reds \ + <= 0 - (FC)); \ + } \ + else { \ + ASSERT(FC <= CONTEXT_REDS); \ + ASSERT(erts_proc_sched_data(c_p)->virtual_reds \ + <= CONTEXT_REDS - (FC)); \ + } \ +} while (0) +#else +#define ERTS_DBG_CHK_REDS(P, FC) +#endif + + /* * process_main() is called twice: * The first call performs some initialisation, including exporting * the instructions' C labels to the loader. * The second call starts execution of BEAM code. This call never returns. */ -void process_main(void) +void process_main(Eterm * x_reg_array, FloatDef* f_reg_array) { static int init_done = 0; Process* c_p = NULL; @@ -1132,15 +1250,10 @@ void process_main(void) ERTS_DECLARE_DUMMY(Eterm pid); #endif - /* - * X register zero; also called r(0) - */ - register Eterm x0 REG_x0 = NIL; - /* Pointer to X registers: x(1)..x(N); reg[0] is used when doing GC, * in all other cases x0 is used. */ - register Eterm* reg REG_xregs = NULL; + register Eterm* reg REG_xregs = x_reg_array; /* * Top of heap (next free location); grows upwards. @@ -1164,29 +1277,16 @@ void process_main(void) register Sint FCALLS REG_fcalls = 0; /* - * Temporaries used for picking up arguments for instructions. - */ - register Eterm tmp_arg1 REG_tmp_arg1 = NIL; - register Eterm tmp_arg2 REG_tmp_arg2 = NIL; -#if HEAP_ON_C_STACK - Eterm tmp_big[2]; /* Temporary buffer for small bignums if HEAP_ON_C_STACK. */ -#else - Eterm *tmp_big; /* Temporary buffer for small bignums if !HEAP_ON_C_STACK. */ -#endif - - /* * X registers and floating point registers are located in * scheduler specific data. */ - register FloatDef *freg; + register FloatDef *freg = f_reg_array; /* * For keeping the negative old value of 'reds' when call saving is active. */ int neg_o_reds = 0; - Eterm (*arith_func)(Process* p, Eterm* reg, Uint live); - #ifdef ERTS_OPCODE_COUNTER_SUPPORT static void* counting_opcodes[] = { DEFINE_COUNTING_OPCODES }; #else @@ -1197,13 +1297,13 @@ void process_main(void) #endif #endif - Uint temp_bits; /* Temporary used by BsSkipBits2 & BsGetInteger2 */ - Eterm pt_arity; /* Used by do_put_tuple */ Uint64 start_time = 0; /* Monitor long schedule */ BeamInstr* start_time_i = NULL; + ERTS_MSACC_DECLARE_CACHE_X() /* a cached value of the tsd pointer for msacc */ + ERL_BITS_DECLARE_STATEP; /* Has to be last declaration */ @@ -1229,29 +1329,29 @@ void process_main(void) goto do_schedule1; do_schedule: - reds_used = REDS_IN(c_p) - FCALLS; + ASSERT(c_p->arity < 6); + ASSERT(c_p->debug_reds_in == REDS_IN(c_p)); + if (!ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) + reds_used = REDS_IN(c_p) - FCALLS; + else + reds_used = REDS_IN(c_p) - (CONTEXT_REDS + FCALLS); + ASSERT(reds_used >= 0); do_schedule1: if (start_time != 0) { Sint64 diff = erts_timestamp_millis() - start_time; - if (diff > 0 && (Uint) diff > erts_system_monitor_long_schedule -#ifdef ERTS_DIRTY_SCHEDULERS - && !ERTS_SCHEDULER_IS_DIRTY(c_p->scheduler_data) -#endif - ) { - BeamInstr *inptr = find_function_from_pc(start_time_i); - BeamInstr *outptr = find_function_from_pc(c_p->i); + if (diff > 0 && (Uint) diff > erts_system_monitor_long_schedule) { + ErtsCodeMFA *inptr = find_function_from_pc(start_time_i); + ErtsCodeMFA *outptr = find_function_from_pc(c_p->i); monitor_long_schedule_proc(c_p,inptr,outptr,(Uint) diff); } } PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); -#if HALFWORD_HEAP - ASSERT(erts_get_scheduler_data()->num_tmp_heap_used == 0); -#endif ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - c_p = schedule(c_p, reds_used); + c_p = erts_schedule(NULL, c_p, reds_used); + ASSERT(!(c_p->flags & F_HIPE_MODE)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); start_time = 0; #ifdef DEBUG @@ -1260,16 +1360,13 @@ void process_main(void) ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); + ERTS_MSACC_UPDATE_CACHE_X(); + if (erts_system_monitor_long_schedule != 0) { start_time = erts_timestamp_millis(); start_time_i = c_p->i; } - reg = ERTS_PROC_GET_SCHDATA(c_p)->x_reg_array; - freg = ERTS_PROC_GET_SCHDATA(c_p)->f_reg_array; -#if !HEAP_ON_C_STACK - tmp_big = ERTS_PROC_GET_SCHDATA(c_p)->beam_emu_tmp_heap; -#endif ERL_BITS_RELOAD_STATEP(c_p); { int reds; @@ -1278,7 +1375,7 @@ void process_main(void) int i; argp = c_p->arg_reg; - for (i = c_p->arity - 1; i > 0; i--) { + for (i = c_p->arity - 1; i >= 0; i--) { reg[i] = argp[i]; CHECK_TERM(reg[i]); } @@ -1291,23 +1388,22 @@ void process_main(void) SET_I(c_p->i); - reds = c_p->fcalls; - if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p) - && (ERTS_TRACE_FLAGS(c_p) & F_SENSITIVE) == 0) { - neg_o_reds = -reds; - FCALLS = REDS_IN(c_p) = 0; + REDS_IN(c_p) = reds = c_p->fcalls; +#ifdef DEBUG + c_p->debug_reds_in = reds; +#endif + + if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) { + neg_o_reds = -CONTEXT_REDS; + FCALLS = neg_o_reds + reds; } else { neg_o_reds = 0; - FCALLS = REDS_IN(c_p) = reds; + FCALLS = reds; } + ERTS_DBG_CHK_REDS(c_p, FCALLS); + next = (BeamInstr *) *I; - r(0) = c_p->arg_reg[0]; -#ifdef HARDDEBUG - if (c_p->arity > 0) { - CHECK_TERM(r(0)); - } -#endif SWAPIN; ASSERT(VALID_INSTR(next)); @@ -1320,10 +1416,9 @@ void process_main(void) if (ERTS_PROC_IS_EXITING(c_p)) { strcpy(fun_buf, "<exiting>"); } else { - BeamInstr *fptr = find_function_from_pc(c_p->i); - if (fptr) { - dtrace_fun_decode(c_p, (Eterm)fptr[0], - (Eterm)fptr[1], (Uint)fptr[2], + ErtsCodeMFA *cmfa = find_function_from_pc(c_p->i); + if (cmfa) { + dtrace_fun_decode(c_p, cmfa, NULL, fun_buf); } else { erts_snprintf(fun_buf, sizeof(DTRACE_CHARBUF_NAME(fun_buf)), @@ -1346,26 +1441,24 @@ void process_main(void) #endif #include "beam_hot.h" -#define STORE_ARITH_RESULT(res) StoreBifResult(2, (res)); -#define ARITH_FUNC(name) erts_gc_##name - { Eterm increment_reg_val; Eterm increment_val; Uint live; Eterm result; - OpCase(i_increment_yIId): - increment_reg_val = yb(Arg(0)); + OpCase(i_increment_rIId): + increment_reg_val = x(0); + I--; goto do_increment; OpCase(i_increment_xIId): increment_reg_val = xb(Arg(0)); goto do_increment; - OpCase(i_increment_rIId): - increment_reg_val = r(0); - I--; + OpCase(i_increment_yIId): + increment_reg_val = yb(Arg(0)); + goto do_increment; do_increment: increment_val = Arg(1); @@ -1374,239 +1467,143 @@ void process_main(void) ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i)); if (MY_IS_SSMALL(i)) { result = make_small(i); - store_result: StoreBifResult(3, result); } } live = Arg(2); - SWAPOUT; - reg[0] = r(0); + HEAVY_SWAPOUT; reg[live] = increment_reg_val; reg[live+1] = make_small(increment_val); result = erts_gc_mixed_plus(c_p, reg, live); - r(0) = reg[0]; - SWAPIN; + HEAVY_SWAPIN; ERTS_HOLE_CHECK(c_p); if (is_value(result)) { - goto store_result; + StoreBifResult(3, result); } ASSERT(c_p->freason != BADMATCH || is_value(c_p->fvalue)); goto find_func_info; } -#define DO_BIG_ARITH(Func,Arg1,Arg2) \ - do { \ - Uint live = Arg(1); \ - SWAPOUT; \ - reg[0] = r(0); \ - reg[live] = (Arg1); \ - reg[live+1] = (Arg2); \ - result = (Func)(c_p, reg, live); \ - r(0) = reg[0]; \ - SWAPIN; \ - ERTS_HOLE_CHECK(c_p); \ - if (is_value(result)) { \ - StoreBifResult(4,result); \ - } \ - goto lb_Cl_error; \ - } while(0) +#define DO_OUTLINED_ARITH_2(name, Op1, Op2) \ + do { \ + Eterm result; \ + Uint live = Arg(1); \ + \ + HEAVY_SWAPOUT; \ + reg[live] = Op1; \ + reg[live+1] = Op2; \ + result = erts_gc_##name(c_p, reg, live); \ + HEAVY_SWAPIN; \ + ERTS_HOLE_CHECK(c_p); \ + if (is_value(result)) { \ + StoreBifResult(4, result); \ + } \ + goto lb_Cl_error; \ + } while (0) - OpCase(i_plus_jIxxd): { + Eterm PlusOp1, PlusOp2; Eterm result; - if (is_both_small(xb(Arg(2)), xb(Arg(3)))) { - Sint i = signed_val(xb(Arg(2))) + signed_val(xb(Arg(3))); + OpCase(i_plus_jIxxd): + PlusOp1 = xb(Arg(2)); + PlusOp2 = xb(Arg(3)); + goto do_plus; + + OpCase(i_plus_jIxyd): + PlusOp1 = xb(Arg(2)); + PlusOp2 = yb(Arg(3)); + goto do_plus; + + OpCase(i_plus_jIssd): + GetArg2(2, PlusOp1, PlusOp2); + goto do_plus; + + do_plus: + if (is_both_small(PlusOp1, PlusOp2)) { + Sint i = signed_val(PlusOp1) + signed_val(PlusOp2); ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i)); if (MY_IS_SSMALL(i)) { result = make_small(i); StoreBifResult(4, result); } } - DO_BIG_ARITH(ARITH_FUNC(mixed_plus), xb(Arg(2)), xb(Arg(3))); + DO_OUTLINED_ARITH_2(mixed_plus, PlusOp1, PlusOp2); } - OpCase(i_plus_jId): { + Eterm MinusOp1, MinusOp2; Eterm result; - if (is_both_small(tmp_arg1, tmp_arg2)) { - Sint i = signed_val(tmp_arg1) + signed_val(tmp_arg2); - ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i)); - if (MY_IS_SSMALL(i)) { - result = make_small(i); - STORE_ARITH_RESULT(result); - } - } - arith_func = ARITH_FUNC(mixed_plus); - goto do_big_arith2; - } - OpCase(i_minus_jIxxd): - { - Eterm result; + MinusOp1 = xb(Arg(2)); + MinusOp2 = xb(Arg(3)); + goto do_minus; - if (is_both_small(xb(Arg(2)), xb(Arg(3)))) { - Sint i = signed_val(xb(Arg(2))) - signed_val(xb(Arg(3))); - ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i)); - if (MY_IS_SSMALL(i)) { - result = make_small(i); - StoreBifResult(4, result); - } - } - DO_BIG_ARITH(ARITH_FUNC(mixed_minus), xb(Arg(2)), xb(Arg(3))); - } + OpCase(i_minus_jIssd): + GetArg2(2, MinusOp1, MinusOp2); + goto do_minus; - OpCase(i_minus_jId): - { - Eterm result; - - if (is_both_small(tmp_arg1, tmp_arg2)) { - Sint i = signed_val(tmp_arg1) - signed_val(tmp_arg2); + do_minus: + if (is_both_small(MinusOp1, MinusOp2)) { + Sint i = signed_val(MinusOp1) - signed_val(MinusOp2); ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i)); if (MY_IS_SSMALL(i)) { result = make_small(i); - STORE_ARITH_RESULT(result); + StoreBifResult(4, result); } } - arith_func = ARITH_FUNC(mixed_minus); - goto do_big_arith2; + DO_OUTLINED_ARITH_2(mixed_minus, MinusOp1, MinusOp2); } - OpCase(i_is_lt_f): - if (CMP_GE(tmp_arg1, tmp_arg2)) { - ClauseFail(); - } - Next(1); - - OpCase(i_is_ge_f): - if (CMP_LT(tmp_arg1, tmp_arg2)) { - ClauseFail(); - } - Next(1); - - OpCase(i_is_eq_f): - if (CMP_NE(tmp_arg1, tmp_arg2)) { - ClauseFail(); - } - Next(1); - - OpCase(i_is_ne_f): - if (CMP_EQ(tmp_arg1, tmp_arg2)) { - ClauseFail(); - } - Next(1); - - OpCase(i_is_eq_exact_f): - if (!EQ(tmp_arg1, tmp_arg2)) { - ClauseFail(); - } - Next(1); - { Eterm is_eq_exact_lit_val; - OpCase(i_is_eq_exact_literal_xfc): - is_eq_exact_lit_val = xb(Arg(0)); - I++; + OpCase(i_is_eq_exact_literal_fxc): + is_eq_exact_lit_val = xb(Arg(1)); goto do_is_eq_exact_literal; - OpCase(i_is_eq_exact_literal_yfc): - is_eq_exact_lit_val = yb(Arg(0)); - I++; + OpCase(i_is_eq_exact_literal_fyc): + is_eq_exact_lit_val = yb(Arg(1)); goto do_is_eq_exact_literal; - OpCase(i_is_eq_exact_literal_rfc): - is_eq_exact_lit_val = r(0); - do_is_eq_exact_literal: - if (!eq(Arg(1), is_eq_exact_lit_val)) { + if (!eq(Arg(2), is_eq_exact_lit_val)) { ClauseFail(); } - Next(2); + Next(3); } { Eterm is_ne_exact_lit_val; - OpCase(i_is_ne_exact_literal_xfc): - is_ne_exact_lit_val = xb(Arg(0)); - I++; + OpCase(i_is_ne_exact_literal_fxc): + is_ne_exact_lit_val = xb(Arg(1)); goto do_is_ne_exact_literal; - OpCase(i_is_ne_exact_literal_yfc): - is_ne_exact_lit_val = yb(Arg(0)); - I++; + OpCase(i_is_ne_exact_literal_fyc): + is_ne_exact_lit_val = yb(Arg(1)); goto do_is_ne_exact_literal; - OpCase(i_is_ne_exact_literal_rfc): - is_ne_exact_lit_val = r(0); - do_is_ne_exact_literal: - if (eq(Arg(1), is_ne_exact_lit_val)) { + if (eq(Arg(2), is_ne_exact_lit_val)) { ClauseFail(); } - Next(2); + Next(3); } - OpCase(move_window3_xxxy): { - BeamInstr *next; - Eterm xt0, xt1, xt2; - Eterm *y = (Eterm *)(((unsigned char *)E) + (Arg(3))); - PreFetch(4, next); - xt0 = xb(Arg(0)); - xt1 = xb(Arg(1)); - xt2 = xb(Arg(2)); - y[0] = xt0; - y[1] = xt1; - y[2] = xt2; - NextPF(4, next); - } - OpCase(move_window4_xxxxy): { - BeamInstr *next; - Eterm xt0, xt1, xt2, xt3; - Eterm *y = (Eterm *)(((unsigned char *)E) + (Arg(4))); - PreFetch(5, next); - xt0 = xb(Arg(0)); - xt1 = xb(Arg(1)); - xt2 = xb(Arg(2)); - xt3 = xb(Arg(3)); - y[0] = xt0; - y[1] = xt1; - y[2] = xt2; - y[3] = xt3; - NextPF(5, next); - } - OpCase(move_window5_xxxxxy): { - BeamInstr *next; - Eterm xt0, xt1, xt2, xt3, xt4; - Eterm *y = (Eterm *)(((unsigned char *)E) + (Arg(5))); - PreFetch(6, next); - xt0 = xb(Arg(0)); - xt1 = xb(Arg(1)); - xt2 = xb(Arg(2)); - xt3 = xb(Arg(3)); - xt4 = xb(Arg(4)); - y[0] = xt0; - y[1] = xt1; - y[2] = xt2; - y[3] = xt3; - y[4] = xt4; - NextPF(6, next); - } - - OpCase(i_move_call_only_fcr): { + OpCase(i_move_call_only_fc): { r(0) = Arg(1); } /* FALL THROUGH */ OpCase(i_call_only_f): { SET_I((BeamInstr *) Arg(0)); - DTRACE_LOCAL_CALL(c_p, (Eterm)I[-3], (Eterm)I[-2], I[-1]); + DTRACE_LOCAL_CALL(c_p, erts_code_to_codemfa(I)); Dispatch(); } - OpCase(i_move_call_last_fPcr): { + OpCase(i_move_call_last_fPc): { r(0) = Arg(2); } /* FALL THROUGH */ @@ -1614,11 +1611,11 @@ void process_main(void) RESTORE_CP(E); E = ADD_BYTE_OFFSET(E, Arg(1)); SET_I((BeamInstr *) Arg(0)); - DTRACE_LOCAL_CALL(c_p, (Eterm)I[-3], (Eterm)I[-2], I[-1]); + DTRACE_LOCAL_CALL(c_p, erts_code_to_codemfa(I)); Dispatch(); } - OpCase(i_move_call_crf): { + OpCase(i_move_call_cf): { r(0) = Arg(0); I++; } @@ -1626,11 +1623,11 @@ void process_main(void) OpCase(i_call_f): { SET_CP(c_p, I+2); SET_I((BeamInstr *) Arg(0)); - DTRACE_LOCAL_CALL(c_p, (Eterm)I[-3], (Eterm)I[-2], I[-1]); + DTRACE_LOCAL_CALL(c_p, erts_code_to_codemfa(I)); Dispatch(); } - OpCase(i_move_call_ext_last_ePcr): { + OpCase(i_move_call_ext_last_ePc): { r(0) = Arg(2); } /* FALL THROUGH */ @@ -1646,7 +1643,7 @@ void process_main(void) DTRACE_GLOBAL_CALL_FROM_EXPORT(c_p, Arg(0)); Dispatchx(); - OpCase(i_move_call_ext_cre): { + OpCase(i_move_call_ext_ce): { r(0) = Arg(0); I++; } @@ -1656,7 +1653,7 @@ void process_main(void) DTRACE_GLOBAL_CALL_FROM_EXPORT(c_p, Arg(0)); Dispatchx(); - OpCase(i_move_call_ext_only_ecr): { + OpCase(i_move_call_ext_only_ec): { r(0) = Arg(1); } /* FALL THROUGH */ @@ -1706,7 +1703,7 @@ void process_main(void) c_p->cp = 0; CHECK_TERM(r(0)); HEAP_SPACE_VERIFIED(0); - Goto(*I); + DispatchReturn; } /* @@ -1720,19 +1717,21 @@ void process_main(void) BeamInstr *next; Eterm result; + if (!(FCALLS > 0 || FCALLS > neg_o_reds)) { + /* If we have run out of reductions, we do a context + switch before calling the bif */ + c_p->arity = 2; + c_p->current = NULL; + goto context_switch3; + } + PRE_BIF_SWAPOUT(c_p); c_p->fcalls = FCALLS - 1; - reg[0] = r(0); result = erl_send(c_p, r(0), x(1)); PreFetch(0, next); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); - if (c_p->mbuf || MSO(c_p).overhead >= BIN_VHEAP_SZ(c_p)) { - result = erts_gc_after_bif_call(c_p, result, reg, 2); - r(0) = reg[0]; - E = c_p->stop; - } HTOP = HEAP_TOP(c_p); FCALLS = c_p->fcalls; if (is_value(result)) { @@ -1743,7 +1742,6 @@ void process_main(void) SET_CP(c_p, I+1); SET_I(c_p->i); SWAPIN; - r(0) = reg[0]; Dispatch(); } goto find_func_info; @@ -1753,29 +1751,23 @@ void process_main(void) Eterm element_index; Eterm element_tuple; - OpCase(i_element_xjsd): - element_tuple = xb(Arg(0)); - I++; + OpCase(i_element_jxsd): + element_tuple = xb(Arg(1)); goto do_element; - OpCase(i_element_yjsd): - element_tuple = yb(Arg(0)); - I++; + OpCase(i_element_jysd): + element_tuple = yb(Arg(1)); goto do_element; - OpCase(i_element_rjsd): - element_tuple = r(0); - /* Fall through */ - do_element: - GetArg1(1, element_index); + GetArg1(2, element_index); if (is_small(element_index) && is_tuple(element_tuple)) { Eterm* tp = tuple_val(element_tuple); if ((signed_val(element_index) >= 1) && (signed_val(element_index) <= arityval(*tp))) { Eterm result = tp[signed_val(element_index)]; - StoreBifResult(2, result); + StoreBifResult(3, result); } } } @@ -1789,29 +1781,24 @@ void process_main(void) { Eterm fast_element_tuple; - OpCase(i_fast_element_rjId): - fast_element_tuple = r(0); + OpCase(i_fast_element_jxId): + fast_element_tuple = xb(Arg(1)); + goto do_fast_element; + + OpCase(i_fast_element_jyId): + fast_element_tuple = yb(Arg(1)); + goto do_fast_element; do_fast_element: if (is_tuple(fast_element_tuple)) { Eterm* tp = tuple_val(fast_element_tuple); - Eterm pos = Arg(1); /* Untagged integer >= 1 */ + Eterm pos = Arg(2); /* Untagged integer >= 1 */ if (pos <= arityval(*tp)) { Eterm result = tp[pos]; - StoreBifResult(2, result); + StoreBifResult(3, result); } } goto badarg; - - OpCase(i_fast_element_xjId): - fast_element_tuple = xb(Arg(0)); - I++; - goto do_fast_element; - - OpCase(i_fast_element_yjId): - fast_element_tuple = yb(Arg(0)); - I++; - goto do_fast_element; } OpCase(catch_yf): @@ -1823,6 +1810,7 @@ void process_main(void) c_p->catches--; make_blank(yb(Arg(0))); if (is_non_value(r(0))) { + c_p->fvalue = NIL; if (x(1) == am_throw) { r(0) = x(2); } else { @@ -1832,11 +1820,10 @@ void process_main(void) SWAPIN; } /* only x(2) is included in the rootset here */ - if (E - HTOP < 3 || c_p->mbuf) { /* Force GC in case add_stacktrace() - * created heap fragments */ + if (E - HTOP < 3) { SWAPOUT; PROCESS_MAIN_CHK_LOCKS(c_p); - FCALLS -= erts_garbage_collect(c_p, 3, reg+2, 1); + FCALLS -= erts_garbage_collect_nobump(c_p, 3, reg+2, 1, FCALLS); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); SWAPIN; @@ -1853,6 +1840,7 @@ void process_main(void) c_p->catches--; make_blank(yb(Arg(0))); if (is_non_value(r(0))) { + c_p->fvalue = NIL; r(0) = x(1); x(1) = x(2); x(2) = x(3); @@ -1917,10 +1905,18 @@ void process_main(void) * Pick up the next message and place it in x(0). * If no message, jump to a wait or wait_timeout instruction. */ - OpCase(i_loop_rec_fr): + OpCase(i_loop_rec_f): { BeamInstr *next; - ErlMessage* msgp; + ErtsMessage* msgp; + + /* + * We need to disable GC while matching messages + * in the queue. This since messages with data outside + * the heap will be corrupted by a GC. + */ + ASSERT(!(c_p->flags & F_DELAY_GC)); + c_p->flags |= F_DELAY_GC; loop_rec__: @@ -1935,6 +1931,8 @@ void process_main(void) if (ERTS_PROC_PENDING_EXIT(c_p)) { erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); SWAPOUT; + c_p->flags &= ~F_DELAY_GC; + c_p->arity = 0; goto do_schedule; /* Will be rescheduled for exit */ } ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p); @@ -1944,32 +1942,27 @@ void process_main(void) else #endif { + c_p->flags &= ~F_DELAY_GC; SET_I((BeamInstr *) Arg(0)); Goto(*I); /* Jump to a wait or wait_timeout instruction */ } } - ErtsMoveMsgAttachmentIntoProc(msgp, c_p, E, HTOP, FCALLS, - { - SWAPOUT; - reg[0] = r(0); - PROCESS_MAIN_CHK_LOCKS(c_p); - }, - { - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - PROCESS_MAIN_CHK_LOCKS(c_p); - r(0) = reg[0]; - SWAPIN; - }); if (is_non_value(ERL_MESSAGE_TERM(msgp))) { - /* - * A corrupt distribution message that we weren't able to decode; - * remove it... - */ - ASSERT(!msgp->data.attached); - /* TODO: Add DTrace probe for this bad message situation? */ - UNLINK_MESSAGE(c_p, msgp); - free_message(msgp); - goto loop_rec__; + SWAPOUT; /* erts_decode_dist_message() may write to heap... */ + if (!erts_decode_dist_message(c_p, ERTS_PROC_LOCK_MAIN, msgp, 0)) { + /* + * A corrupt distribution message that we weren't able to decode; + * remove it... + */ + /* No swapin should be needed */ + ASSERT(HTOP == c_p->htop && E == c_p->stop); + /* TODO: Add DTrace probe for this bad message situation? */ + UNLINK_MESSAGE(c_p, msgp); + msgp->next = NULL; + erts_cleanup_messages(msgp); + goto loop_rec__; + } + SWAPIN; } PreFetch(1, next); r(0) = ERL_MESSAGE_TERM(msgp); @@ -1981,10 +1974,11 @@ void process_main(void) */ OpCase(remove_message): { BeamInstr *next; - ErlMessage* msgp; - + ErtsMessage* msgp; PROCESS_MAIN_CHK_LOCKS(c_p); + ERTS_CHK_MBUF_SZ(c_p); + PreFetch(0, next); msgp = PEEK_MESSAGE(c_p); @@ -1996,20 +1990,7 @@ void process_main(void) if (DT_UTAG(c_p) != NIL) { if (DT_UTAG_FLAGS(c_p) & DT_UTAG_PERMANENT) { SEQ_TRACE_TOKEN(c_p) = am_have_dt_utag; -#ifdef DTRACE_TAG_HARDDEBUG - if (DT_UTAG_FLAGS(c_p) & DT_UTAG_SPREADING) - erts_fprintf(stderr, - "Dtrace -> (%T) stop spreading " - "tag %T with message %T\r\n", - c_p->common.id,DT_UTAG(c_p),ERL_MESSAGE_TERM(msgp)); -#endif } else { -#ifdef DTRACE_TAG_HARDDEBUG - erts_fprintf(stderr, - "Dtrace -> (%T) kill tag %T with " - "message %T\r\n", - c_p->common.id,DT_UTAG(c_p),ERL_MESSAGE_TERM(msgp)); -#endif DT_UTAG(c_p) = NIL; SEQ_TRACE_TOKEN(c_p) = NIL; } @@ -2029,12 +2010,6 @@ void process_main(void) DT_UTAG(c_p) = ERL_MESSAGE_DT_UTAG(msgp); } DT_UTAG_FLAGS(c_p) |= DT_UTAG_SPREADING; -#ifdef DTRACE_TAG_HARDDEBUG - erts_fprintf(stderr, - "Dtrace -> (%T) receive tag (%T) " - "with message %T\r\n", - c_p->common.id, DT_UTAG(c_p), ERL_MESSAGE_TERM(msgp)); -#endif } else { #endif ASSERT(is_tuple(SEQ_TRACE_TOKEN(c_p))); @@ -2064,7 +2039,7 @@ void process_main(void) dtrace_proc_str(c_p, receiver_name); token2 = SEQ_TRACE_TOKEN(c_p); - if (token2 != NIL && token2 != am_have_dt_utag) { + if (have_seqtrace(token2)) { tok_label = signed_val(SEQ_TRACE_T_LABEL(token2)); tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token2)); tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token2)); @@ -2077,11 +2052,24 @@ void process_main(void) UNLINK_MESSAGE(c_p, msgp); JOIN_MESSAGE(c_p); CANCEL_TIMER(c_p); - free_message(msgp); + + erts_save_message_in_proc(c_p, msgp); + c_p->flags &= ~F_DELAY_GC; + + if (ERTS_IS_GC_DESIRED_INTERNAL(c_p, HTOP, E)) { + /* + * We want to GC soon but we leave a few + * reductions giving the message some time + * to turn into garbage. + */ + ERTS_VBUMP_LEAVE_REDS_INTERNAL(c_p, 5, FCALLS); + } + + ERTS_DBG_CHK_REDS(c_p, FCALLS); + ERTS_CHK_MBUF_SZ(c_p); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); - NextPF(0, next); } @@ -2090,9 +2078,22 @@ void process_main(void) * message didn't match), then jump to the loop_rec instruction. */ OpCase(loop_rec_end_f): { + + ASSERT(c_p->flags & F_DELAY_GC); + SET_I((BeamInstr *) Arg(0)); SAVE_MESSAGE(c_p); - goto loop_rec__; + if (FCALLS > 0 || FCALLS > neg_o_reds) { + FCALLS--; + goto loop_rec__; + } + + c_p->flags &= ~F_DELAY_GC; + c_p->i = I; + SWAPOUT; + c_p->arity = 0; + c_p->current = NULL; + goto do_schedule; } /* * Prepare to wait for a message or a timeout, whichever occurs first. @@ -2131,8 +2132,6 @@ void process_main(void) * c_p->def_arg_reg[0]. Note that it is safe to use this * location because there are no living x registers in * a receive statement. - * Note that for the halfword emulator, the two first elements - * of the array are used. */ BeamInstr** pi = (BeamInstr**) c_p->def_arg_reg; *pi = I+3; @@ -2163,10 +2162,30 @@ void process_main(void) OpCase(wait_f): wait2: { +#ifndef ERTS_SMP + if (ERTS_PROC_IS_EXITING(c_p)) { + /* + * I non smp case: + * + * Currently executing process might be sent an exit + * signal if it is traced by a port that it also is + * linked to, and the port terminates during the + * trace. In this case we do *not* want to clear + * the active flag, which will make the process hang + * in limbo forever. + */ + SWAPOUT; + c_p->arity = 0; + goto do_schedule; + } +#endif c_p->i = (BeamInstr *) Arg(0); /* L1 */ SWAPOUT; c_p->arity = 0; - erts_smp_atomic32_read_band_relb(&c_p->state, ~ERTS_PSFLG_ACTIVE); + + if (!ERTS_PTMR_IS_TIMED_OUT(c_p)) + erts_smp_atomic32_read_band_relb(&c_p->state, + ~ERTS_PSFLG_ACTIVE); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); c_p->current = NULL; @@ -2212,7 +2231,7 @@ void process_main(void) PreFetch(0, next); if (IS_TRACED_FL(c_p, F_TRACE_RECEIVE)) { - trace_receive(c_p, am_timeout); + trace_receive(c_p, am_clock_service, am_timeout, NULL); } if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) { save_calls(c_p, &exp_timeout); @@ -2233,10 +2252,6 @@ void process_main(void) select_val2 = xb(Arg(0)); goto do_select_tuple_arity2; - OpCase(i_select_tuple_arity2_rfAAff): - select_val2 = r(0); - I--; - do_select_tuple_arity2: if (is_not_tuple(select_val2)) { goto select_val2_fail; @@ -2252,10 +2267,6 @@ void process_main(void) select_val2 = xb(Arg(0)); goto do_select_val2; - OpCase(i_select_val2_rfccff): - select_val2 = r(0); - I--; - do_select_val2: if (select_val2 == Arg(2)) { I += 3; @@ -2279,10 +2290,6 @@ void process_main(void) select_val = yb(Arg(0)); goto do_select_tuple_arity; - OpCase(i_select_tuple_arity_rfI): - select_val = r(0); - I--; - do_select_tuple_arity: if (is_tuple(select_val)) { select_val = *tuple_val(select_val); @@ -2299,10 +2306,6 @@ void process_main(void) select_val = yb(Arg(0)); goto do_linear_search; - OpCase(i_select_val_lins_rfI): - select_val = r(0); - I--; - do_linear_search: { BeamInstr *vs = &Arg(3); int ix = 0; @@ -2329,10 +2332,6 @@ void process_main(void) select_val = yb(Arg(0)); goto do_binary_search; - OpCase(i_select_val_bins_rfI): - select_val = r(0); - I--; - do_binary_search: { struct Pairs { @@ -2393,10 +2392,6 @@ void process_main(void) jump_on_val_zero_index = xb(Arg(0)); goto do_jump_on_val_zero_index; - OpCase(i_jump_on_val_zero_rfI): - jump_on_val_zero_index = r(0); - I--; - do_jump_on_val_zero_index: if (is_small(jump_on_val_zero_index)) { jump_on_val_zero_index = signed_val(jump_on_val_zero_index); @@ -2421,10 +2416,6 @@ void process_main(void) jump_on_val_index = xb(Arg(0)); goto do_jump_on_val_index; - OpCase(i_jump_on_val_rfII): - jump_on_val_index = r(0); - I--; - do_jump_on_val_index: if (is_small(jump_on_val_index)) { jump_on_val_index = (Uint) (signed_val(jump_on_val_index) - Arg(3)); @@ -2444,15 +2435,12 @@ void process_main(void) do { Eterm term = *I++; - switch (term & _TAG_IMMED1_MASK) { - case (R_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER: - *hp++ = r(0); - break; - case (X_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER: - *hp++ = x(term >> _TAG_IMMED1_SIZE); + switch (loader_tag(term)) { + case LOADER_X_REG: + *hp++ = x(loader_x_reg_index(term)); break; - case (Y_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER: - *hp++ = y(term >> _TAG_IMMED1_SIZE); + case LOADER_Y_REG: + *hp++ = y(loader_y_reg_index(term)); break; default: *hp++ = term; @@ -2466,31 +2454,26 @@ void process_main(void) OpCase(new_map_dII): { Eterm res; - x(0) = r(0); - SWAPOUT; + HEAVY_SWAPOUT; res = new_map(c_p, reg, I-1); - SWAPIN; - r(0) = x(0); + HEAVY_SWAPIN; StoreResult(res, Arg(0)); Next(3+Arg(2)); } -#define PUT_TERM_REG(term, desc) \ -do { \ - switch ((desc) & _TAG_IMMED1_MASK) { \ - case (R_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER: \ - r(0) = (term); \ - break; \ - case (X_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER: \ - x((desc) >> _TAG_IMMED1_SIZE) = (term); \ - break; \ - case (Y_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER: \ - y((desc) >> _TAG_IMMED1_SIZE) = (term); \ - break; \ - default: \ - ASSERT(0); \ - break; \ - } \ +#define PUT_TERM_REG(term, desc) \ +do { \ + switch (loader_tag(desc)) { \ + case LOADER_X_REG: \ + x(loader_x_reg_index(desc)) = (term); \ + break; \ + case LOADER_Y_REG: \ + y(loader_y_reg_index(desc)) = (term); \ + break; \ + default: \ + ASSERT(0); \ + break; \ + } \ } while(0) OpCase(i_get_map_elements_fsI): { @@ -2561,12 +2544,10 @@ do { \ Eterm map; GetArg1(1, map); - x(0) = r(0); - SWAPOUT; + HEAVY_SWAPOUT; res = update_map_assoc(c_p, reg, map, I); - SWAPIN; + HEAVY_SWAPIN; if (is_value(res)) { - r(0) = x(0); StoreResult(res, Arg(2)); Next(5+Arg(4)); } else { @@ -2585,12 +2566,10 @@ do { \ Eterm map; GetArg1(1, map); - x(0) = r(0); - SWAPOUT; + HEAVY_SWAPOUT; res = update_map_exact(c_p, reg, map, I); - SWAPIN; + HEAVY_SWAPIN; if (is_value(res)) { - r(0) = x(0); StoreResult(res, Arg(2)); Next(5+Arg(4)); } else { @@ -2615,21 +2594,25 @@ do { \ OpCase(bif1_fbsd): { - Eterm (*bf)(Process*, Eterm*); + ErtsBifFunc bf; Eterm tmp_reg[1]; Eterm result; GetArg1(2, tmp_reg[0]); bf = (BifFunction) Arg(1); + ERTS_DBG_CHK_REDS(c_p, FCALLS); c_p->fcalls = FCALLS; PROCESS_MAIN_CHK_LOCKS(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - result = (*bf)(c_p, tmp_reg); + ERTS_CHK_MBUF_SZ(c_p); + result = (*bf)(c_p, tmp_reg, I); + ERTS_CHK_MBUF_SZ(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_HOLE_CHECK(c_p); FCALLS = c_p->fcalls; + ERTS_DBG_CHK_REDS(c_p, FCALLS); if (is_value(result)) { StoreBifResult(3, result); } @@ -2643,28 +2626,32 @@ do { \ OpCase(bif1_body_bsd): { - Eterm (*bf)(Process*, Eterm*); + ErtsBifFunc bf; Eterm tmp_reg[1]; Eterm result; GetArg1(1, tmp_reg[0]); - bf = (BifFunction) Arg(0); + bf = (ErtsBifFunc) Arg(0); + ERTS_DBG_CHK_REDS(c_p, FCALLS); c_p->fcalls = FCALLS; PROCESS_MAIN_CHK_LOCKS(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - result = (*bf)(c_p, tmp_reg); + ERTS_CHK_MBUF_SZ(c_p); + result = (*bf)(c_p, tmp_reg, I); + ERTS_CHK_MBUF_SZ(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_HOLE_CHECK(c_p); FCALLS = c_p->fcalls; + ERTS_DBG_CHK_REDS(c_p, FCALLS); if (is_value(result)) { StoreBifResult(2, result); } reg[0] = tmp_reg[0]; SWAPOUT; - I = handle_error(c_p, I, reg, bf); + I = handle_error(c_p, I, reg, ubif2mfa((void *) bf)); goto post_error_handling; } @@ -2672,26 +2659,26 @@ do { \ { typedef Eterm (*GcBifFunction)(Process*, Eterm*, Uint); GcBifFunction bf; - Eterm arg; Eterm result; Uint live = (Uint) Arg(3); - GetArg1(2, arg); - reg[0] = r(0); - reg[live] = arg; + GetArg1(2, x(live)); bf = (GcBifFunction) Arg(1); + ERTS_DBG_CHK_REDS(c_p, FCALLS); c_p->fcalls = FCALLS; SWAPOUT; PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); + ERTS_CHK_MBUF_SZ(c_p); result = (*bf)(c_p, reg, live); + ERTS_CHK_MBUF_SZ(c_p); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); SWAPIN; - r(0) = reg[0]; ERTS_HOLE_CHECK(c_p); FCALLS = c_p->fcalls; + ERTS_DBG_CHK_REDS(c_p, FCALLS); if (is_value(result)) { StoreBifResult(4, result); } @@ -2699,12 +2686,12 @@ do { \ SET_I((BeamInstr *) Arg(0)); Goto(*I); } - reg[0] = arg; - I = handle_error(c_p, I, reg, translate_gc_bif((void *) bf)); + x(0) = x(live); + I = handle_error(c_p, I, reg, gcbif2mfa((void *) bf)); goto post_error_handling; } - OpCase(i_gc_bif2_jIId): /* Note, one less parameter than the i_gc_bif1 + OpCase(i_gc_bif2_jIIssd): /* Note, one less parameter than the i_gc_bif1 and i_gc_bif3 */ { typedef Eterm (*GcBifFunction)(Process*, Eterm*, Uint); @@ -2712,96 +2699,115 @@ do { \ Eterm result; Uint live = (Uint) Arg(2); - reg[0] = r(0); - reg[live++] = tmp_arg1; - reg[live] = tmp_arg2; + GetArg2(3, x(live), x(live+1)); + /* + * XXX This calling convention does not make sense. 'live' + * should point out the first argument, not the second + * (i.e. 'live' should not be incremented below). + */ + live++; bf = (GcBifFunction) Arg(1); + ERTS_DBG_CHK_REDS(c_p, FCALLS); c_p->fcalls = FCALLS; SWAPOUT; PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); + ERTS_CHK_MBUF_SZ(c_p); result = (*bf)(c_p, reg, live); + ERTS_CHK_MBUF_SZ(c_p); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); SWAPIN; - r(0) = reg[0]; ERTS_HOLE_CHECK(c_p); FCALLS = c_p->fcalls; + ERTS_DBG_CHK_REDS(c_p, FCALLS); if (is_value(result)) { - StoreBifResult(3, result); + StoreBifResult(5, result); } if (Arg(0) != 0) { SET_I((BeamInstr *) Arg(0)); Goto(*I); } - reg[0] = tmp_arg1; - reg[1] = tmp_arg2; - I = handle_error(c_p, I, reg, translate_gc_bif((void *) bf)); + live--; + x(0) = x(live); + x(1) = x(live+1); + I = handle_error(c_p, I, reg, gcbif2mfa((void *) bf)); goto post_error_handling; } - OpCase(i_gc_bif3_jIsId): + OpCase(i_gc_bif3_jIIssd): { typedef Eterm (*GcBifFunction)(Process*, Eterm*, Uint); GcBifFunction bf; - Eterm arg; Eterm result; - Uint live = (Uint) Arg(3); + Uint live = (Uint) Arg(2); - GetArg1(2, arg); - reg[0] = r(0); - reg[live++] = arg; - reg[live++] = tmp_arg1; - reg[live] = tmp_arg2; + x(live) = x(SCRATCH_X_REG); + GetArg2(3, x(live+1), x(live+2)); + /* + * XXX This calling convention does not make sense. 'live' + * should point out the first argument, not the third + * (i.e. 'live' should not be incremented below). + */ + live += 2; bf = (GcBifFunction) Arg(1); + ERTS_DBG_CHK_REDS(c_p, FCALLS); c_p->fcalls = FCALLS; SWAPOUT; PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); + ERTS_CHK_MBUF_SZ(c_p); result = (*bf)(c_p, reg, live); + ERTS_CHK_MBUF_SZ(c_p); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); SWAPIN; - r(0) = reg[0]; ERTS_HOLE_CHECK(c_p); FCALLS = c_p->fcalls; + ERTS_DBG_CHK_REDS(c_p, FCALLS); if (is_value(result)) { - StoreBifResult(4, result); + StoreBifResult(5, result); } if (Arg(0) != 0) { SET_I((BeamInstr *) Arg(0)); Goto(*I); } - reg[0] = arg; - reg[1] = tmp_arg1; - reg[2] = tmp_arg2; - I = handle_error(c_p, I, reg, translate_gc_bif((void *) bf)); + live -= 2; + x(0) = x(live); + x(1) = x(live+1); + x(2) = x(live+2); + I = handle_error(c_p, I, reg, gcbif2mfa((void *) bf)); goto post_error_handling; } /* * Guards bifs and, or, xor in guards. */ - OpCase(i_bif2_fbd): + OpCase(i_bif2_fbssd): { - Eterm tmp_reg[2] = {tmp_arg1, tmp_arg2}; - Eterm (*bf)(Process*, Eterm*); + Eterm tmp_reg[2]; + ErtsBifFunc bf; Eterm result; - bf = (BifFunction) Arg(1); + GetArg2(2, tmp_reg[0], tmp_reg[1]); + bf = (ErtsBifFunc) Arg(1); + ERTS_DBG_CHK_REDS(c_p, FCALLS); c_p->fcalls = FCALLS; PROCESS_MAIN_CHK_LOCKS(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - result = (*bf)(c_p, tmp_reg); + ERTS_CHK_MBUF_SZ(c_p); + result = (*bf)(c_p, tmp_reg, I); + ERTS_CHK_MBUF_SZ(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_HOLE_CHECK(c_p); FCALLS = c_p->fcalls; + ERTS_DBG_CHK_REDS(c_p, FCALLS); if (is_value(result)) { - StoreBifResult(2, result); + StoreBifResult(4, result); } SET_I((BeamInstr *) Arg(0)); Goto(*I); @@ -2810,28 +2816,31 @@ do { \ /* * Guards bifs and, or, xor, relational operators in body. */ - OpCase(i_bif2_body_bd): + OpCase(i_bif2_body_bssd): { - Eterm tmp_reg[2] = {tmp_arg1, tmp_arg2}; - Eterm (*bf)(Process*, Eterm*); + Eterm tmp_reg[2]; + ErtsBifFunc bf; Eterm result; - bf = (BifFunction) Arg(0); + GetArg2(1, tmp_reg[0], tmp_reg[1]); + bf = (ErtsBifFunc) Arg(0); PROCESS_MAIN_CHK_LOCKS(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - result = (*bf)(c_p, tmp_reg); + ERTS_CHK_MBUF_SZ(c_p); + result = (*bf)(c_p, tmp_reg, I); + ERTS_CHK_MBUF_SZ(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_HOLE_CHECK(c_p); if (is_value(result)) { ASSERT(!is_CP(result)); - StoreBifResult(1, result); + StoreBifResult(3, result); } - reg[0] = tmp_arg1; - reg[1] = tmp_arg2; + reg[0] = tmp_reg[0]; + reg[1] = tmp_reg[1]; SWAPOUT; - I = handle_error(c_p, I, reg, bf); + I = handle_error(c_p, I, reg, ubif2mfa((void *) bf)); goto post_error_handling; } @@ -2841,32 +2850,58 @@ do { \ */ OpCase(call_bif_e): { - Eterm (*bf)(Process*, Eterm*, BeamInstr*) = GET_BIF_ADDRESS(Arg(0)); + ErtsBifFunc bf; Eterm result; BeamInstr *next; + ErlHeapFragment *live_hf_end; + Export *export = (Export*)Arg(0); + + + if (!((FCALLS - 1) > 0 || (FCALLS-1) > neg_o_reds)) { + /* If we have run out of reductions, we do a context + switch before calling the bif */ + c_p->arity = GET_BIF_ARITY(export); + c_p->current = &export->info.mfa; + goto context_switch3; + } + + ERTS_MSACC_SET_BIF_STATE_CACHED_X( + GET_BIF_MODULE(export), GET_BIF_ADDRESS(export)); + + bf = GET_BIF_ADDRESS(export); PRE_BIF_SWAPOUT(c_p); + ERTS_DBG_CHK_REDS(c_p, FCALLS); c_p->fcalls = FCALLS - 1; if (FCALLS <= 0) { - save_calls(c_p, (Export *) Arg(0)); + save_calls(c_p, export); } PreFetch(1, next); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - reg[0] = r(0); + live_hf_end = c_p->mbuf; + ERTS_CHK_MBUF_SZ(c_p); result = (*bf)(c_p, reg, I); + ERTS_CHK_MBUF_SZ(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); ERTS_HOLE_CHECK(c_p); ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); - PROCESS_MAIN_CHK_LOCKS(c_p); - if (c_p->mbuf || MSO(c_p).overhead >= BIN_VHEAP_SZ(c_p)) { - Uint arity = ((Export *)Arg(0))->code[2]; - result = erts_gc_after_bif_call(c_p, result, reg, arity); + if (ERTS_IS_GC_DESIRED(c_p)) { + Uint arity = GET_BIF_ARITY(export); + result = erts_gc_after_bif_call_lhf(c_p, live_hf_end, result, reg, arity); E = c_p->stop; } + PROCESS_MAIN_CHK_LOCKS(c_p); HTOP = HEAP_TOP(c_p); FCALLS = c_p->fcalls; + ERTS_DBG_CHK_REDS(c_p, FCALLS); + /* We have to update the cache if we are enabled in order + to make sure no book keeping is done after we disabled + msacc. We don't always do this as it is quite expensive. */ + if (ERTS_MSACC_IS_ENABLED_CACHED_X()) + ERTS_MSACC_UPDATE_CACHE_X(); + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR); if (is_value(result)) { r(0) = result; CHECK_TERM(r(0)); @@ -2875,7 +2910,6 @@ do { \ SET_CP(c_p, I+2); SET_I(c_p->i); SWAPIN; - r(0) = reg[0]; Dispatch(); } @@ -2883,7 +2917,7 @@ do { \ * Error handling. SWAPOUT is not needed because it was done above. */ ASSERT(c_p->stop == E); - I = handle_error(c_p, I, reg, bf); + I = handle_error(c_p, I, reg, &export->info.mfa); goto post_error_handling; } @@ -2891,115 +2925,85 @@ do { \ * Arithmetic operations. */ - OpCase(i_times_jId): + OpCase(i_times_jIssd): { - arith_func = ARITH_FUNC(mixed_times); - goto do_big_arith2; + Eterm Op1, Op2; + GetArg2(2, Op1, Op2); + DO_OUTLINED_ARITH_2(mixed_times, Op1, Op2); } - OpCase(i_m_div_jId): + OpCase(i_m_div_jIssd): { - arith_func = ARITH_FUNC(mixed_div); - goto do_big_arith2; + Eterm Op1, Op2; + GetArg2(2, Op1, Op2); + DO_OUTLINED_ARITH_2(mixed_div, Op1, Op2); } - OpCase(i_int_div_jId): + OpCase(i_int_div_jIssd): { - Eterm result; + Eterm Op1, Op2; - if (tmp_arg2 == SMALL_ZERO) { + GetArg2(2, Op1, Op2); + if (Op2 == SMALL_ZERO) { goto badarith; - } else if (is_both_small(tmp_arg1, tmp_arg2)) { - Sint ires = signed_val(tmp_arg1) / signed_val(tmp_arg2); + } else if (is_both_small(Op1, Op2)) { + Sint ires = signed_val(Op1) / signed_val(Op2); if (MY_IS_SSMALL(ires)) { - result = make_small(ires); - STORE_ARITH_RESULT(result); + Eterm result = make_small(ires); + StoreBifResult(4, result); } } - arith_func = ARITH_FUNC(int_div); - goto do_big_arith2; + DO_OUTLINED_ARITH_2(int_div, Op1, Op2); } - OpCase(i_rem_jIxxd): { - Eterm result; + Eterm RemOp1, RemOp2; - if (xb(Arg(3)) == SMALL_ZERO) { - goto badarith; - } else if (is_both_small(xb(Arg(2)), xb(Arg(3)))) { - result = make_small(signed_val(xb(Arg(2))) % signed_val(xb(Arg(3)))); + OpCase(i_rem_jIxxd): + RemOp1 = xb(Arg(2)); + RemOp2 = xb(Arg(3)); + goto do_rem; + + OpCase(i_rem_jIssd): + GetArg2(2, RemOp1, RemOp2); + goto do_rem; + + do_rem: + if (RemOp2 == SMALL_ZERO) { + goto badarith; + } else if (is_both_small(RemOp1, RemOp2)) { + Eterm result = make_small(signed_val(RemOp1) % signed_val(RemOp2)); StoreBifResult(4, result); + } else { + DO_OUTLINED_ARITH_2(int_rem, RemOp1, RemOp2); } - DO_BIG_ARITH(ARITH_FUNC(int_rem),xb(Arg(2)),xb(Arg(3))); } - OpCase(i_rem_jId): { - Eterm result; - - if (tmp_arg2 == SMALL_ZERO) { - goto badarith; - } else if (is_both_small(tmp_arg1, tmp_arg2)) { - result = make_small(signed_val(tmp_arg1) % signed_val(tmp_arg2)); - STORE_ARITH_RESULT(result); - } else { - arith_func = ARITH_FUNC(int_rem); - goto do_big_arith2; - } - } + Eterm BandOp1, BandOp2; OpCase(i_band_jIxcd): - { - Eterm result; + BandOp1 = xb(Arg(2)); + BandOp2 = Arg(3); + goto do_band; + + OpCase(i_band_jIssd): + GetArg2(2, BandOp1, BandOp2); + goto do_band; - if (is_both_small(xb(Arg(2)), Arg(3))) { + do_band: + if (is_both_small(BandOp1, BandOp2)) { /* * No need to untag -- TAG & TAG == TAG. */ - result = xb(Arg(2)) & Arg(3); + Eterm result = BandOp1 & BandOp2; StoreBifResult(4, result); } - DO_BIG_ARITH(ARITH_FUNC(band),xb(Arg(2)),Arg(3)); - } - - OpCase(i_band_jId): - { - Eterm result; - - if (is_both_small(tmp_arg1, tmp_arg2)) { - /* - * No need to untag -- TAG & TAG == TAG. - */ - result = tmp_arg1 & tmp_arg2; - STORE_ARITH_RESULT(result); - } - arith_func = ARITH_FUNC(band); - goto do_big_arith2; - } - -#undef DO_BIG_ARITH - - do_big_arith2: - { - Eterm result; - Uint live = Arg(1); - - SWAPOUT; - reg[0] = r(0); - reg[live] = tmp_arg1; - reg[live+1] = tmp_arg2; - result = arith_func(c_p, reg, live); - r(0) = reg[0]; - SWAPIN; - ERTS_HOLE_CHECK(c_p); - if (is_value(result)) { - STORE_ARITH_RESULT(result); - } - goto lb_Cl_error; + DO_OUTLINED_ARITH_2(band, BandOp1, BandOp2); } /* - * An error occured in an arithmetic operation or test that could + * An error occurred in an arithmetic operation or test that could * appear either in a head or in a body. * In a head, execution should continue at failure address in Arg(0). * In a body, Arg(0) == 0 and an exception should be raised. @@ -3016,97 +3020,103 @@ do { \ goto find_func_info; } - OpCase(i_bor_jId): + OpCase(i_bor_jIssd): { - Eterm result; + Eterm Op1, Op2; - if (is_both_small(tmp_arg1, tmp_arg2)) { + GetArg2(2, Op1, Op2); + if (is_both_small(Op1, Op2)) { /* * No need to untag -- TAG | TAG == TAG. */ - result = tmp_arg1 | tmp_arg2; - STORE_ARITH_RESULT(result); + Eterm result = Op1 | Op2; + StoreBifResult(4, result); } - arith_func = ARITH_FUNC(bor); - goto do_big_arith2; + DO_OUTLINED_ARITH_2(bor, Op1, Op2); } - OpCase(i_bxor_jId): + OpCase(i_bxor_jIssd): { - Eterm result; + Eterm Op1, Op2; - if (is_both_small(tmp_arg1, tmp_arg2)) { + GetArg2(2, Op1, Op2); + if (is_both_small(Op1, Op2)) { /* - * We could extract the tag from one argument, but a tag extraction - * could mean a shift. Therefore, play it safe here. + * TAG ^ TAG == 0. + * + * Therefore, we perform the XOR operation on the tagged values, + * and OR in the tag bits. */ - result = make_small(signed_val(tmp_arg1) ^ signed_val(tmp_arg2)); - STORE_ARITH_RESULT(result); + Eterm result = (Op1 ^ Op2) | make_small(0); + StoreBifResult(4, result); } - arith_func = ARITH_FUNC(bxor); - goto do_big_arith2; + DO_OUTLINED_ARITH_2(bxor, Op1, Op2); } { + Eterm Op1, Op2; Sint i; Sint ires; Eterm* bigp; + Eterm tmp_big[2]; - OpCase(i_bsr_jId): - if (is_small(tmp_arg2)) { - i = -signed_val(tmp_arg2); - if (is_small(tmp_arg1)) { + OpCase(i_bsr_jIssd): + GetArg2(2, Op1, Op2); + if (is_small(Op2)) { + i = -signed_val(Op2); + if (is_small(Op1)) { goto small_shift; - } else if (is_big(tmp_arg1)) { + } else if (is_big(Op1)) { if (i == 0) { - StoreBifResult(2, tmp_arg1); + StoreBifResult(4, Op1); } + ires = big_size(Op1); goto big_shift; } - } else if (is_big(tmp_arg2)) { + } else if (is_big(Op2)) { /* * N bsr NegativeBigNum == N bsl MAX_SMALL * N bsr PositiveBigNum == N bsl MIN_SMALL */ - tmp_arg2 = make_small(bignum_header_is_neg(*big_val(tmp_arg2)) ? + Op2 = make_small(bignum_header_is_neg(*big_val(Op2)) ? MAX_SMALL : MIN_SMALL); goto do_bsl; } goto badarith; - OpCase(i_bsl_jId): + OpCase(i_bsl_jIssd): + GetArg2(2, Op1, Op2); do_bsl: - if (is_small(tmp_arg2)) { - i = signed_val(tmp_arg2); + if (is_small(Op2)) { + i = signed_val(Op2); - if (is_small(tmp_arg1)) { + if (is_small(Op1)) { small_shift: - ires = signed_val(tmp_arg1); + ires = signed_val(Op1); if (i == 0 || ires == 0) { - StoreBifResult(2, tmp_arg1); + StoreBifResult(4, Op1); } else if (i < 0) { /* Right shift */ i = -i; if (i >= SMALL_BITS-1) { - tmp_arg1 = (ires < 0) ? SMALL_MINUS_ONE : SMALL_ZERO; + Op1 = (ires < 0) ? SMALL_MINUS_ONE : SMALL_ZERO; } else { - tmp_arg1 = make_small(ires >> i); + Op1 = make_small(ires >> i); } - StoreBifResult(2, tmp_arg1); + StoreBifResult(4, Op1); } else if (i < SMALL_BITS-1) { /* Left shift */ if ((ires > 0 && ((~(Uint)0 << ((SMALL_BITS-1)-i)) & ires) == 0) || ((~(Uint)0 << ((SMALL_BITS-1)-i)) & ~ires) == 0) { - tmp_arg1 = make_small(ires << i); - StoreBifResult(2, tmp_arg1); + Op1 = make_small(ires << i); + StoreBifResult(4, Op1); } } - tmp_arg1 = small_to_big(ires, tmp_big); + ires = 1; /* big_size(small_to_big(Op1)) */ big_shift: if (i > 0) { /* Left shift. */ - ires = big_size(tmp_arg1) + (i / D_EXP); + ires += (i / D_EXP); } else { /* Right shift. */ - ires = big_size(tmp_arg1); if (ires <= (-i / D_EXP)) ires = 3; /* ??? */ else @@ -3124,14 +3134,17 @@ do { \ c_p->freason = SYSTEM_LIMIT; goto lb_Cl_error; } - TestHeapPreserve(ires+1, Arg(1), tmp_arg1); + TestHeapPreserve(ires+1, Arg(1), Op1); + if (is_small(Op1)) { + Op1 = small_to_big(signed_val(Op1), tmp_big); + } bigp = HTOP; - tmp_arg1 = big_lshift(tmp_arg1, i, bigp); - if (is_big(tmp_arg1)) { + Op1 = big_lshift(Op1, i, bigp); + if (is_big(Op1)) { HTOP += bignum_header_arity(*HTOP) + 1; } HEAP_SPACE_VERIFIED(0); - if (is_nil(tmp_arg1)) { + if (is_nil(Op1)) { /* * This result must have been only slight larger * than allowed since it wasn't caught by the @@ -3141,25 +3154,26 @@ do { \ goto lb_Cl_error; } ERTS_HOLE_CHECK(c_p); - StoreBifResult(2, tmp_arg1); + StoreBifResult(4, Op1); } - } else if (is_big(tmp_arg1)) { + } else if (is_big(Op1)) { if (i == 0) { - StoreBifResult(2, tmp_arg1); + StoreBifResult(4, Op1); } + ires = big_size(Op1); goto big_shift; } - } else if (is_big(tmp_arg2)) { - if (bignum_header_is_neg(*big_val(tmp_arg2))) { + } else if (is_big(Op2)) { + if (bignum_header_is_neg(*big_val(Op2))) { /* * N bsl NegativeBigNum is either 0 or -1, depending on * the sign of N. Since we don't believe this case * is common, do the calculation with the minimum * amount of code. */ - tmp_arg2 = make_small(MIN_SMALL); + Op2 = make_small(MIN_SMALL); goto do_bsl; - } else if (is_small(tmp_arg1) || is_big(tmp_arg1)) { + } else if (is_small(Op1) || is_big(Op1)) { /* * N bsl PositiveBigNum is too large to represent. */ @@ -3183,12 +3197,10 @@ do { \ bnot_val = make_small(~signed_val(bnot_val)); } else { Uint live = Arg(2); - SWAPOUT; - reg[0] = r(0); + HEAVY_SWAPOUT; reg[live] = bnot_val; bnot_val = erts_gc_bnot(c_p, reg, live); - r(0) = reg[0]; - SWAPIN; + HEAVY_SWAPIN; ERTS_HOLE_CHECK(c_p); if (is_nil(bnot_val)) { goto lb_Cl_error; @@ -3203,92 +3215,84 @@ do { \ OpCase(i_apply): { BeamInstr *next; - SWAPOUT; - next = apply(c_p, r(0), x(1), x(2), reg); - SWAPIN; + HEAVY_SWAPOUT; + next = apply(c_p, r(0), x(1), x(2), reg, NULL, 0); + HEAVY_SWAPIN; if (next != NULL) { - r(0) = reg[0]; SET_CP(c_p, I+1); SET_I(next); Dispatch(); } - I = handle_error(c_p, I, reg, apply_3); + I = handle_error(c_p, I, reg, &bif_export[BIF_apply_3]->info.mfa); goto post_error_handling; } OpCase(i_apply_last_P): { BeamInstr *next; - SWAPOUT; - next = apply(c_p, r(0), x(1), x(2), reg); - SWAPIN; + HEAVY_SWAPOUT; + next = apply(c_p, r(0), x(1), x(2), reg, I, Arg(0)); + HEAVY_SWAPIN; if (next != NULL) { - r(0) = reg[0]; - SET_CP(c_p, (BeamInstr *) EXPAND_POINTER(E[0])); + SET_CP(c_p, (BeamInstr *) E[0]); E = ADD_BYTE_OFFSET(E, Arg(0)); SET_I(next); Dispatch(); } - I = handle_error(c_p, I, reg, apply_3); + I = handle_error(c_p, I, reg, &bif_export[BIF_apply_3]->info.mfa); goto post_error_handling; } OpCase(i_apply_only): { BeamInstr *next; - SWAPOUT; - next = apply(c_p, r(0), x(1), x(2), reg); - SWAPIN; + HEAVY_SWAPOUT; + next = apply(c_p, r(0), x(1), x(2), reg, I, 0); + HEAVY_SWAPIN; if (next != NULL) { - r(0) = reg[0]; SET_I(next); Dispatch(); } - I = handle_error(c_p, I, reg, apply_3); + I = handle_error(c_p, I, reg, &bif_export[BIF_apply_3]->info.mfa); goto post_error_handling; } OpCase(apply_I): { BeamInstr *next; - reg[0] = r(0); - SWAPOUT; - next = fixed_apply(c_p, reg, Arg(0)); - SWAPIN; + HEAVY_SWAPOUT; + next = fixed_apply(c_p, reg, Arg(0), NULL, 0); + HEAVY_SWAPIN; if (next != NULL) { - r(0) = reg[0]; SET_CP(c_p, I+2); SET_I(next); Dispatch(); } - I = handle_error(c_p, I, reg, apply_3); + I = handle_error(c_p, I, reg, &bif_export[BIF_apply_3]->info.mfa); goto post_error_handling; } OpCase(apply_last_IP): { BeamInstr *next; - reg[0] = r(0); - SWAPOUT; - next = fixed_apply(c_p, reg, Arg(0)); - SWAPIN; + HEAVY_SWAPOUT; + next = fixed_apply(c_p, reg, Arg(0), I, Arg(1)); + HEAVY_SWAPIN; if (next != NULL) { - r(0) = reg[0]; - SET_CP(c_p, (BeamInstr *) EXPAND_POINTER(E[0])); + SET_CP(c_p, (BeamInstr *) E[0]); E = ADD_BYTE_OFFSET(E, Arg(1)); SET_I(next); Dispatch(); } - I = handle_error(c_p, I, reg, apply_3); + I = handle_error(c_p, I, reg, &bif_export[BIF_apply_3]->info.mfa); goto post_error_handling; } OpCase(i_apply_fun): { BeamInstr *next; - SWAPOUT; + HEAVY_SWAPOUT; next = apply_fun(c_p, r(0), x(1), reg); - SWAPIN; + HEAVY_SWAPIN; if (next != NULL) { - r(0) = reg[0]; SET_CP(c_p, I+1); SET_I(next); Dispatchfun(); @@ -3299,12 +3303,11 @@ do { \ OpCase(i_apply_fun_last_P): { BeamInstr *next; - SWAPOUT; + HEAVY_SWAPOUT; next = apply_fun(c_p, r(0), x(1), reg); - SWAPIN; + HEAVY_SWAPIN; if (next != NULL) { - r(0) = reg[0]; - SET_CP(c_p, (BeamInstr *) EXPAND_POINTER(E[0])); + SET_CP(c_p, (BeamInstr *) E[0]); E = ADD_BYTE_OFFSET(E, Arg(0)); SET_I(next); Dispatchfun(); @@ -3315,11 +3318,10 @@ do { \ OpCase(i_apply_fun_only): { BeamInstr *next; - SWAPOUT; + HEAVY_SWAPOUT; next = apply_fun(c_p, r(0), x(1), reg); - SWAPIN; + HEAVY_SWAPIN; if (next != NULL) { - r(0) = reg[0]; SET_I(next); Dispatchfun(); } @@ -3329,13 +3331,10 @@ do { \ OpCase(i_call_fun_I): { BeamInstr *next; - SWAPOUT; - reg[0] = r(0); - + HEAVY_SWAPOUT; next = call_fun(c_p, Arg(0), reg, THE_NON_VALUE); - SWAPIN; + HEAVY_SWAPIN; if (next != NULL) { - r(0) = reg[0]; SET_CP(c_p, I+2); SET_I(next); Dispatchfun(); @@ -3346,13 +3345,11 @@ do { \ OpCase(i_call_fun_last_IP): { BeamInstr *next; - SWAPOUT; - reg[0] = r(0); + HEAVY_SWAPOUT; next = call_fun(c_p, Arg(0), reg, THE_NON_VALUE); - SWAPIN; + HEAVY_SWAPIN; if (next != NULL) { - r(0) = reg[0]; - SET_CP(c_p, (BeamInstr *) EXPAND_POINTER(E[0])); + SET_CP(c_p, (BeamInstr *) E[0]); E = ADD_BYTE_OFFSET(E, Arg(1)); SET_I(next); Dispatchfun(); @@ -3386,19 +3383,29 @@ do { \ * called from I[-3], I[-2], and I[-1] respectively. */ context_switch_fun: - c_p->arity = I[-1] + 1; + /* Add one for the environment of the fun */ + c_p->arity = erts_code_to_codemfa(I)->arity + 1; goto context_switch2; context_switch: - c_p->arity = I[-1]; + c_p->arity = erts_code_to_codemfa(I)->arity; + + context_switch2: /* Entry for fun calls. */ + c_p->current = erts_code_to_codemfa(I); - context_switch2: /* Entry for fun calls. */ - c_p->current = I-3; /* Pointer to Mod, Func, Arity */ + context_switch3: { Eterm* argp; int i; + if (erts_smp_atomic32_read_nob(&c_p->state) & ERTS_PSFLG_EXITING) { + c_p->i = beam_exit; + c_p->arity = 0; + c_p->current = NULL; + goto do_schedule; + } + /* * Make sure that there is enough room for the argument registers to be saved. */ @@ -3429,17 +3436,21 @@ do { \ * (beacuse the code for the Dispatch() macro becomes shorter that way). */ - reds_used = REDS_IN(c_p) - FCALLS + 1; - + ASSERT(c_p->debug_reds_in == REDS_IN(c_p)); + if (!ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) + reds_used = REDS_IN(c_p) - FCALLS; + else + reds_used = REDS_IN(c_p) - (CONTEXT_REDS + FCALLS); + ASSERT(reds_used >= 0); + /* * Save the argument registers and everything else. */ argp = c_p->arg_reg; - for (i = c_p->arity - 1; i > 0; i--) { + for (i = c_p->arity - 1; i >= 0; i--) { argp[i] = reg[i]; } - c_p->arg_reg[0] = r(0); SWAPOUT; c_p->i = I; goto do_schedule1; @@ -3452,23 +3463,18 @@ do { \ Eterm* p; PreFetch(3, next); - GetArg2(0, element, tuple); + GetArg1(0, element); + tuple = REG_TARGET(Arg(1)); ASSERT(is_tuple(tuple)); p = (Eterm *) ((unsigned char *) tuple_val(tuple) + Arg(2)); *p = element; NextPF(3, next); } - OpCase(i_is_ne_exact_f): - if (EQ(tmp_arg1, tmp_arg2)) { - ClauseFail(); - } - Next(1); - OpCase(normal_exit): { SWAPOUT; c_p->freason = EXC_NORMAL; - c_p->arity = 0; /* In case this process will ever be garbed again. */ + c_p->arity = 0; /* In case this process will never be garbed again. */ ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); erts_do_exit_process(c_p, am_normal); ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); @@ -3482,48 +3488,18 @@ do { \ goto do_schedule; } - OpCase(raise_ss): { - /* This was not done very well in R10-0; then, we passed the tag in - the first argument and hoped that the existing c_p->ftrace was - still correct. But the ftrace-object already includes the tag - (or rather, the freason). Now, we pass the original ftrace in - the first argument. We also handle atom tags in the first - argument for backwards compatibility. - */ - Eterm raise_val1; - Eterm raise_val2; - GetArg2(0, raise_val1, raise_val2); - c_p->fvalue = raise_val2; - if (c_p->freason == EXC_NULL) { - /* a safety check for the R10-0 case; should not happen */ - c_p->ftrace = NIL; - c_p->freason = EXC_ERROR; - } - /* for R10-0 code, keep existing c_p->ftrace and hope it's correct */ - switch (raise_val1) { - case am_throw: - c_p->freason = EXC_THROWN & ~EXF_SAVETRACE; - break; - case am_error: - c_p->freason = EXC_ERROR & ~EXF_SAVETRACE; - break; - case am_exit: - c_p->freason = EXC_EXIT & ~EXF_SAVETRACE; - break; - default: - {/* R10-1 and later - XXX note: should do sanity check on given trace if it can be - passed from a user! Currently only expecting generated calls. - */ - struct StackTrace *s; - c_p->ftrace = raise_val1; - s = get_trace_from_exc(raise_val1); - if (s == NULL) { - c_p->freason = EXC_ERROR; - } else { - c_p->freason = PRIMARY_EXCEPTION(s->freason); - } - } + OpCase(i_raise): { + Eterm raise_trace = x(2); + Eterm raise_value = x(1); + struct StackTrace *s; + + c_p->fvalue = raise_value; + c_p->ftrace = raise_trace; + s = get_trace_from_exc(raise_trace); + if (s == NULL) { + c_p->freason = EXC_ERROR; + } else { + c_p->freason = PRIMARY_EXCEPTION(s->freason); } goto find_func_info; } @@ -3531,25 +3507,14 @@ do { \ { Eterm badmatch_val; - OpCase(badmatch_y): - badmatch_val = yb(Arg(0)); - goto do_badmatch; - OpCase(badmatch_x): badmatch_val = xb(Arg(0)); - goto do_badmatch; - - OpCase(badmatch_r): - badmatch_val = r(0); - - do_badmatch: c_p->fvalue = badmatch_val; c_p->freason = BADMATCH; } /* Fall through here */ find_func_info: { - reg[0] = r(0); SWAPOUT; I = handle_error(c_p, I, reg, NULL); goto post_error_handling; @@ -3566,11 +3531,10 @@ do { \ * code[3]: &&call_error_handler * code[4]: Not used */ - SWAPOUT; - reg[0] = r(0); - I = call_error_handler(c_p, I-3, reg, am_undefined_function); - r(0) = reg[0]; - SWAPIN; + HEAVY_SWAPOUT; + I = call_error_handler(c_p, erts_code_to_codemfa(I), + reg, am_undefined_function); + HEAVY_SWAPIN; if (I) { Goto(*I); } @@ -3578,18 +3542,13 @@ do { \ /* Fall through */ OpCase(error_action_code): { handle_error: - reg[0] = r(0); SWAPOUT; I = handle_error(c_p, NULL, reg, NULL); post_error_handling: if (I == 0) { goto do_schedule; } else { - r(0) = reg[0]; ASSERT(!is_value(r(0))); - if (c_p->mbuf) { - erts_garbage_collect(c_p, 0, reg+1, 3); - } SWAPIN; Goto(*I); } @@ -3611,15 +3570,25 @@ do { \ * I[1]: Function pointer to NIF function * I[2]: Pointer to erl_module_nif * I[3]: Function pointer to dirty NIF + * + * This layout is determined by the NifExport struct */ BifFunction vbf; + ErlHeapFragment *live_hf_end; + ErtsCodeMFA *codemfa; + + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_NIF); + + codemfa = erts_code_to_codemfa(I); + + c_p->current = codemfa; /* current and vbf set to please handle_error */ + + DTRACE_NIF_ENTRY(c_p, codemfa); + + HEAVY_SWAPOUT; - DTRACE_NIF_ENTRY(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]); - c_p->current = I-3; /* current and vbf set to please handle_error */ - SWAPOUT; - c_p->fcalls = FCALLS - 1; PROCESS_MAIN_CHK_LOCKS(c_p); - bif_nif_arity = I[-1]; + bif_nif_arity = codemfa->arity; ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); @@ -3627,68 +3596,99 @@ do { \ typedef Eterm NifF(struct enif_environment_t*, int argc, Eterm argv[]); NifF* fp = vbf = (NifF*) I[1]; struct enif_environment_t env; - erts_pre_nif(&env, c_p, (struct erl_module_nif*)I[2]); - reg[0] = r(0); +#ifdef ERTS_SMP + ASSERT(c_p->scheduler_data); +#endif + live_hf_end = c_p->mbuf; + ERTS_CHK_MBUF_SZ(c_p); + erts_pre_nif(&env, c_p, (struct erl_module_nif*)I[2], NULL); nif_bif_result = (*fp)(&env, bif_nif_arity, reg); if (env.exception_thrown) nif_bif_result = THE_NON_VALUE; erts_post_nif(&env); + ERTS_CHK_MBUF_SZ(c_p); + + PROCESS_MAIN_CHK_LOCKS(c_p); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR); + ASSERT(!env.exiting); + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); } - ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(nif_bif_result)); - PROCESS_MAIN_CHK_LOCKS(c_p); - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - DTRACE_NIF_RETURN(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]); + DTRACE_NIF_RETURN(c_p, codemfa); goto apply_bif_or_nif_epilogue; OpCase(apply_bif): /* - * At this point, I points to the code[3] in the export entry for + * At this point, I points to the code[0] in the export entry for * the BIF: * - * code[0]: Module - * code[1]: Function - * code[2]: Arity - * code[3]: &&apply_bif - * code[4]: Function pointer to BIF function + * code[-3]: Module + * code[-2]: Function + * code[-1]: Arity + * code[0]: &&apply_bif + * code[1]: Function pointer to BIF function */ - c_p->current = I-3; /* In case we apply process_info/1,2 or load_nif/1 */ + if (!((FCALLS - 1) > 0 || (FCALLS - 1) > neg_o_reds)) { + /* If we have run out of reductions, we do a context + switch before calling the bif */ + goto context_switch; + } + + codemfa = erts_code_to_codemfa(I); + + ERTS_MSACC_SET_BIF_STATE_CACHED_X(codemfa->module, (BifFunction)Arg(0)); + + + /* In case we apply process_info/1,2 or load_nif/1 */ + c_p->current = codemfa; c_p->i = I; /* In case we apply check_process_code/2. */ c_p->arity = 0; /* To allow garbage collection on ourselves * (check_process_code/2). */ - DTRACE_BIF_ENTRY(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]); + DTRACE_BIF_ENTRY(c_p, codemfa); SWAPOUT; + ERTS_DBG_CHK_REDS(c_p, FCALLS - 1); c_p->fcalls = FCALLS - 1; vbf = (BifFunction) Arg(0); PROCESS_MAIN_CHK_LOCKS(c_p); - bif_nif_arity = I[-1]; + bif_nif_arity = codemfa->arity; ASSERT(bif_nif_arity <= 4); ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); - reg[0] = r(0); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); { - Eterm (*bf)(Process*, Eterm*, BeamInstr*) = vbf; + ErtsBifFunc bf = vbf; ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + live_hf_end = c_p->mbuf; + ERTS_CHK_MBUF_SZ(c_p); nif_bif_result = (*bf)(c_p, reg, I); + ERTS_CHK_MBUF_SZ(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(nif_bif_result)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); } - - DTRACE_BIF_RETURN(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]); + /* We have to update the cache if we are enabled in order + to make sure no book keeping is done after we disabled + msacc. We don't always do this as it is quite expensive. */ + if (ERTS_MSACC_IS_ENABLED_CACHED_X()) + ERTS_MSACC_UPDATE_CACHE_X(); + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR); + DTRACE_BIF_RETURN(c_p, codemfa); apply_bif_or_nif_epilogue: ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); ERTS_HOLE_CHECK(c_p); - if (c_p->mbuf) { - nif_bif_result = erts_gc_after_bif_call(c_p, nif_bif_result, - reg, bif_nif_arity); + if (ERTS_IS_GC_DESIRED(c_p)) { + nif_bif_result = erts_gc_after_bif_call_lhf(c_p, live_hf_end, + nif_bif_result, + reg, bif_nif_arity); } SWAPIN; /* There might have been a garbage collection. */ FCALLS = c_p->fcalls; + ERTS_DBG_CHK_REDS(c_p, FCALLS); if (is_value(nif_bif_result)) { r(0) = nif_bif_result; CHECK_TERM(r(0)); @@ -3697,14 +3697,13 @@ do { \ Goto(*I); } else if (c_p->freason == TRAP) { SET_I(c_p->i); - r(0) = reg[0]; if (c_p->flags & F_HIBERNATE_SCHED) { c_p->flags &= ~F_HIBERNATE_SCHED; goto do_schedule; } Dispatch(); } - I = handle_error(c_p, c_p->cp, reg, vbf); + I = handle_error(c_p, c_p->cp, reg, c_p->current); goto post_error_handling; } } @@ -3719,21 +3718,21 @@ do { \ StoreBifResult(1, result); } + OpCase(i_get_hash_cId): + { + Eterm arg; + Eterm result; + + GetArg1(0, arg); + result = erts_pd_hash_get_with_hx(c_p, Arg(1), arg); + StoreBifResult(2, result); + } + { Eterm case_end_val; OpCase(case_end_x): case_end_val = xb(Arg(0)); - goto do_case_end; - - OpCase(case_end_y): - case_end_val = yb(Arg(0)); - goto do_case_end; - - OpCase(case_end_r): - case_end_val = r(0); - - do_case_end: c_p->fvalue = case_end_val; c_p->freason = EXC_CASE_CLAUSE; goto find_func_info; @@ -3744,8 +3743,9 @@ do { \ goto find_func_info; OpCase(i_func_info_IaaI): { + ErtsCodeInfo *ci = (ErtsCodeInfo*)I; c_p->freason = EXC_FUNCTION_CLAUSE; - c_p->current = I + 2; + c_p->current = &ci->mfa; goto handle_error; } @@ -3781,19 +3781,13 @@ do { \ goto do_bs_init_bits_known; } - OpCase(i_bs_init_bits_fail_heap_IjId): { - /* tmp_arg1 was fetched by an i_fetch instruction */ - num_bits_term = tmp_arg1; - alloc = Arg(0); - I++; + OpCase(i_bs_init_bits_fail_heap_sIjId): { + GetArg1(0, num_bits_term); + alloc = Arg(1); + I += 2; goto do_bs_init_bits; } - OpCase(i_bs_init_bits_fail_rjId): { - num_bits_term = r(0); - alloc = 0; - goto do_bs_init_bits; - } OpCase(i_bs_init_bits_fail_yjId): { num_bits_term = yb(Arg(0)); I++; @@ -3891,7 +3885,6 @@ do { \ * Allocate the binary struct itself. */ bptr = erts_bin_nrml_alloc(num_bytes); - erts_refc_init(&bptr->refc, 1); erts_current_bin = (byte *) bptr->orig_bytes; /* @@ -3913,52 +3906,48 @@ do { \ } { - OpCase(i_bs_init_fail_heap_IjId): { - /* tmp_arg1 was fetched by an i_fetch instruction */ - tmp_arg2 = Arg(0); - I++; - goto do_bs_init; - } + Eterm BsOp1, BsOp2; - OpCase(i_bs_init_fail_rjId): { - tmp_arg1 = r(0); - tmp_arg2 = 0; + OpCase(i_bs_init_fail_heap_sIjId): { + GetArg1(0, BsOp1); + BsOp2 = Arg(1); + I += 2; goto do_bs_init; } OpCase(i_bs_init_fail_yjId): { - tmp_arg1 = yb(Arg(0)); - tmp_arg2 = 0; + BsOp1 = yb(Arg(0)); + BsOp2 = 0; I++; goto do_bs_init; } OpCase(i_bs_init_fail_xjId): { - tmp_arg1 = xb(Arg(0)); - tmp_arg2 = 0; + BsOp1 = xb(Arg(0)); + BsOp2 = 0; I++; } /* FALL THROUGH */ do_bs_init: - if (is_small(tmp_arg1)) { - Sint size = signed_val(tmp_arg1); + if (is_small(BsOp1)) { + Sint size = signed_val(BsOp1); if (size < 0) { goto badarg; } - tmp_arg1 = (Eterm) size; + BsOp1 = (Eterm) size; } else { Uint bytes; - if (!term_to_Uint(tmp_arg1, &bytes)) { + if (!term_to_Uint(BsOp1, &bytes)) { c_p->freason = bytes; goto lb_Cl_error; } if ((bytes >> (8*sizeof(Uint)-3)) != 0) { goto system_limit; } - tmp_arg1 = (Eterm) bytes; + BsOp1 = (Eterm) bytes; } - if (tmp_arg1 <= ERL_ONHEAP_BIN_LIMIT) { + if (BsOp1 <= ERL_ONHEAP_BIN_LIMIT) { goto do_heap_bin_alloc; } else { goto do_proc_bin_alloc; @@ -3966,15 +3955,15 @@ do { \ OpCase(i_bs_init_heap_IIId): { - tmp_arg1 = Arg(0); - tmp_arg2 = Arg(1); + BsOp1 = Arg(0); + BsOp2 = Arg(1); I++; goto do_proc_bin_alloc; } OpCase(i_bs_init_IId): { - tmp_arg1 = Arg(0); - tmp_arg2 = 0; + BsOp1 = Arg(0); + BsOp2 = 0; } /* FALL THROUGH */ do_proc_bin_alloc: { @@ -3983,14 +3972,13 @@ do { \ erts_bin_offset = 0; erts_writable_bin = 0; - TestBinVHeap(tmp_arg1 / sizeof(Eterm), - tmp_arg2 + PROC_BIN_SIZE + ERL_SUB_BIN_SIZE, Arg(1)); + TestBinVHeap(BsOp1 / sizeof(Eterm), + BsOp2 + PROC_BIN_SIZE + ERL_SUB_BIN_SIZE, Arg(1)); /* * Allocate the binary struct itself. */ - bptr = erts_bin_nrml_alloc(tmp_arg1); - erts_refc_init(&bptr->refc, 1); + bptr = erts_bin_nrml_alloc(BsOp1); erts_current_bin = (byte *) bptr->orig_bytes; /* @@ -3999,28 +3987,28 @@ do { \ pb = (ProcBin *) HTOP; HTOP += PROC_BIN_SIZE; pb->thing_word = HEADER_PROC_BIN; - pb->size = tmp_arg1; + pb->size = BsOp1; pb->next = MSO(c_p).first; MSO(c_p).first = (struct erl_off_heap_header*) pb; pb->val = bptr; pb->bytes = (byte*) bptr->orig_bytes; pb->flags = 0; - OH_OVERHEAD(&(MSO(c_p)), tmp_arg1 / sizeof(Eterm)); + OH_OVERHEAD(&(MSO(c_p)), BsOp1 / sizeof(Eterm)); StoreBifResult(2, make_binary(pb)); } OpCase(i_bs_init_heap_bin_heap_IIId): { - tmp_arg1 = Arg(0); - tmp_arg2 = Arg(1); + BsOp1 = Arg(0); + BsOp2 = Arg(1); I++; goto do_heap_bin_alloc; } OpCase(i_bs_init_heap_bin_IId): { - tmp_arg1 = Arg(0); - tmp_arg2 = 0; + BsOp1 = Arg(0); + BsOp2 = 0; } /* Fall through */ do_heap_bin_alloc: @@ -4028,33 +4016,36 @@ do { \ ErlHeapBin* hb; Uint bin_need; - bin_need = heap_bin_size(tmp_arg1); + bin_need = heap_bin_size(BsOp1); erts_bin_offset = 0; erts_writable_bin = 0; - TestHeap(bin_need+tmp_arg2+ERL_SUB_BIN_SIZE, Arg(1)); + TestHeap(bin_need+BsOp2+ERL_SUB_BIN_SIZE, Arg(1)); hb = (ErlHeapBin *) HTOP; HTOP += bin_need; - hb->thing_word = header_heap_bin(tmp_arg1); - hb->size = tmp_arg1; + hb->thing_word = header_heap_bin(BsOp1); + hb->size = BsOp1; erts_current_bin = (byte *) hb->data; - tmp_arg1 = make_binary(hb); - StoreBifResult(2, tmp_arg1); + BsOp1 = make_binary(hb); + StoreBifResult(2, BsOp1); } } - OpCase(i_bs_add_jId): { - Uint Unit = Arg(1); - if (is_both_small(tmp_arg1, tmp_arg2)) { - Sint Arg1 = signed_val(tmp_arg1); - Sint Arg2 = signed_val(tmp_arg2); + OpCase(bs_add_jssId): { + Eterm Op1, Op2; + Uint Unit = Arg(3); + + GetArg2(1, Op1, Op2); + if (is_both_small(Op1, Op2)) { + Sint Arg1 = signed_val(Op1); + Sint Arg2 = signed_val(Op2); if (Arg1 >= 0 && Arg2 >= 0) { - BsSafeMul(Arg2, Unit, goto system_limit, tmp_arg1); - tmp_arg1 += Arg1; + BsSafeMul(Arg2, Unit, goto system_limit, Op1); + Op1 += Arg1; store_bs_add_result: - if (MY_IS_SSMALL((Sint) tmp_arg1)) { - tmp_arg1 = make_small(tmp_arg1); + if (Op1 <= MAX_SMALL) { + Op1 = make_small(Op1); } else { /* * May generate a heap fragment, but in this @@ -4066,10 +4057,10 @@ do { \ * references (such as the heap). */ SWAPOUT; - tmp_arg1 = erts_make_integer(tmp_arg1, c_p); + Op1 = erts_make_integer(Op1, c_p); HTOP = HEAP_TOP(c_p); } - StoreBifResult(2, tmp_arg1); + StoreBifResult(4, Op1); } goto badarg; } else { @@ -4092,16 +4083,16 @@ do { \ * an Uint, the reason is SYSTEM_LIMIT. */ - if (!term_to_Uint(tmp_arg1, &a)) { + if (!term_to_Uint(Op1, &a)) { if (a == BADARG) { goto badarg; } - if (!term_to_Uint(tmp_arg2, &b)) { + if (!term_to_Uint(Op2, &b)) { c_p->freason = b; goto lb_Cl_error; } goto system_limit; - } else if (!term_to_Uint(tmp_arg2, &b)) { + } else if (!term_to_Uint(Op2, &b)) { c_p->freason = b; goto lb_Cl_error; } @@ -4111,8 +4102,8 @@ do { \ */ BsSafeMul(b, Unit, goto system_limit, c); - tmp_arg1 = a + c; - if (tmp_arg1 < a) { + Op1 = a + c; + if (Op1 < a) { /* * If the result is less than one of the * arguments, there must have been an overflow. @@ -4134,52 +4125,47 @@ do { \ } /* - * tmp_arg1 = Number of bytes to build - * tmp_arg2 = Source binary - * Operands: Fail ExtraHeap Live Unit Dst + * x(SCRATCH_X_REG); + * Operands: Fail ExtraHeap Live Unit Size Dst */ - OpCase(i_bs_append_jIIId): { + OpCase(i_bs_append_jIIIsd): { Uint live = Arg(2); Uint res; + Eterm Size; - SWAPOUT; - reg[0] = r(0); - reg[live] = tmp_arg2; - res = erts_bs_append(c_p, reg, live, tmp_arg1, Arg(1), Arg(3)); - r(0) = reg[0]; - SWAPIN; + GetArg1(4, Size); + HEAVY_SWAPOUT; + reg[live] = x(SCRATCH_X_REG); + res = erts_bs_append(c_p, reg, live, Size, Arg(1), Arg(3)); + HEAVY_SWAPIN; if (is_non_value(res)) { /* c_p->freason is already set (may be either BADARG or SYSTEM_LIMIT). */ goto lb_Cl_error; } - StoreBifResult(4, res); + StoreBifResult(5, res); } /* - * tmp_arg1 = Number of bytes to build - * tmp_arg2 = Source binary - * Operands: Fail Unit Dst + * Operands: Fail Size Src Unit Dst */ - OpCase(i_bs_private_append_jId): { + OpCase(i_bs_private_append_jIssd): { Eterm res; + Eterm Size, Src; - res = erts_bs_private_append(c_p, tmp_arg2, tmp_arg1, Arg(1)); + GetArg2(2, Size, Src); + res = erts_bs_private_append(c_p, Src, Size, Arg(1)); if (is_non_value(res)) { /* c_p->freason is already set (may be either BADARG or SYSTEM_LIMIT). */ goto lb_Cl_error; } - StoreBifResult(2, res); + StoreBifResult(4, res); } - /* - * tmp_arg1 = Initial size of writable binary - * Operands: Live Dst - */ OpCase(bs_init_writable): { - SWAPOUT; + HEAVY_SWAPOUT; r(0) = erts_bs_init_writable(c_p, r(0)); - SWAPIN; + HEAVY_SWAPIN; Next(0); } @@ -4236,7 +4222,7 @@ do { \ StoreBifResult(1, result); } - OpCase(i_bs_put_utf16_jIs): { + OpCase(bs_put_utf16_jIs): { Eterm arg; GetArg1(2, arg); @@ -4270,26 +4256,29 @@ do { \ /* * Only used for validating a value matched out. - * - * tmp_arg1 = Integer to validate - * tmp_arg2 = Match context */ - OpCase(i_bs_validate_unicode_retract_j): { - /* - * There is no need to untag the integer, but it IS necessary - * to make sure it is small (a bignum pointer could fall in - * the valid range). - */ - if (is_not_small(tmp_arg1) || tmp_arg1 > make_small(0x10FFFFUL) || - (make_small(0xD800UL) <= tmp_arg1 && - tmp_arg1 <= make_small(0xDFFFUL))) { - ErlBinMatchBuffer *mb = ms_matchbuffer(tmp_arg2); + OpCase(i_bs_validate_unicode_retract_jss): { + Eterm i; /* Integer to validate */ - mb->offset -= 32; - goto badarg; - } - Next(1); - } + /* + * There is no need to untag the integer, but it IS necessary + * to make sure it is small (a bignum pointer could fall in + * the valid range). + */ + + GetArg1(1, i); + if (is_not_small(i) || i > make_small(0x10FFFFUL) || + (make_small(0xD800UL) <= i && i <= make_small(0xDFFFUL))) { + Eterm ms; /* Match context */ + ErlBinMatchBuffer* mb; + + GetArg1(2, ms); + mb = ms_matchbuffer(ms); + mb->offset -= 32; + goto badarg; + } + Next(3); + } /* * Matching of binaries. @@ -4301,9 +4290,6 @@ do { \ Uint slots; Eterm context; - OpCase(i_bs_start_match2_rfIId): { - context = r(0); - do_start_match: slots = Arg(2); if (!is_boxed(context)) { @@ -4350,7 +4336,7 @@ do { \ ClauseFail(); } NextPF(4, next); - } + OpCase(i_bs_start_match2_xfIId): { context = xb(Arg(0)); I++; @@ -4363,18 +4349,6 @@ do { \ } } - OpCase(bs_test_zero_tail2_fr): { - BeamInstr *next; - ErlBinMatchBuffer *_mb; - - PreFetch(1, next); - _mb = (ErlBinMatchBuffer*) ms_matchbuffer(r(0)); - if (_mb->size != _mb->offset) { - ClauseFail(); - } - NextPF(1, next); - } - OpCase(bs_test_zero_tail2_fx): { BeamInstr *next; ErlBinMatchBuffer *_mb; @@ -4387,16 +4361,6 @@ do { \ NextPF(2, next); } - OpCase(bs_test_tail_imm2_frI): { - BeamInstr *next; - ErlBinMatchBuffer *_mb; - PreFetch(2, next); - _mb = ms_matchbuffer(r(0)); - if (_mb->size - _mb->offset != Arg(1)) { - ClauseFail(); - } - NextPF(2, next); - } OpCase(bs_test_tail_imm2_fxI): { BeamInstr *next; ErlBinMatchBuffer *_mb; @@ -4408,16 +4372,6 @@ do { \ NextPF(3, next); } - OpCase(bs_test_unit_frI): { - BeamInstr *next; - ErlBinMatchBuffer *_mb; - PreFetch(2, next); - _mb = ms_matchbuffer(r(0)); - if ((_mb->size - _mb->offset) % Arg(1)) { - ClauseFail(); - } - NextPF(2, next); - } OpCase(bs_test_unit_fxI): { BeamInstr *next; ErlBinMatchBuffer *_mb; @@ -4429,16 +4383,6 @@ do { \ NextPF(3, next); } - OpCase(bs_test_unit8_fr): { - BeamInstr *next; - ErlBinMatchBuffer *_mb; - PreFetch(1, next); - _mb = ms_matchbuffer(r(0)); - if ((_mb->size - _mb->offset) & 7) { - ClauseFail(); - } - NextPF(1, next); - } OpCase(bs_test_unit8_fx): { BeamInstr *next; ErlBinMatchBuffer *_mb; @@ -4453,19 +4397,11 @@ do { \ { Eterm bs_get_integer8_context; - OpCase(i_bs_get_integer_8_rfd): { - bs_get_integer8_context = r(0); - goto do_bs_get_integer_8; - } - OpCase(i_bs_get_integer_8_xfd): { - bs_get_integer8_context = xb(Arg(0)); - I++; - } - - do_bs_get_integer_8: { ErlBinMatchBuffer *_mb; Eterm _result; + bs_get_integer8_context = xb(Arg(0)); + I++; _mb = ms_matchbuffer(bs_get_integer8_context); if (_mb->size - _mb->offset < 8) { ClauseFail(); @@ -4483,15 +4419,10 @@ do { \ { Eterm bs_get_integer_16_context; - OpCase(i_bs_get_integer_16_rfd): - bs_get_integer_16_context = r(0); - goto do_bs_get_integer_16; - OpCase(i_bs_get_integer_16_xfd): bs_get_integer_16_context = xb(Arg(0)); I++; - do_bs_get_integer_16: { ErlBinMatchBuffer *_mb; Eterm _result; @@ -4512,17 +4443,10 @@ do { \ { Eterm bs_get_integer_32_context; - OpCase(i_bs_get_integer_32_rfId): - bs_get_integer_32_context = r(0); - goto do_bs_get_integer_32; - - OpCase(i_bs_get_integer_32_xfId): bs_get_integer_32_context = xb(Arg(0)); I++; - - do_bs_get_integer_32: { ErlBinMatchBuffer *_mb; Uint32 _integer; @@ -4535,11 +4459,11 @@ do { \ _integer = get_int32(_mb->base + _mb->offset/8); } _mb->offset += 32; -#if !defined(ARCH_64) || HALFWORD_HEAP +#if !defined(ARCH_64) if (IS_USMALL(0, _integer)) { #endif _result = make_small(_integer); -#if !defined(ARCH_64) || HALFWORD_HEAP +#if !defined(ARCH_64) } else { TestHeap(BIG_UINT_HEAP_SIZE, Arg(1)); _result = uint_to_big((Uint) _integer, HTOP); @@ -4551,103 +4475,82 @@ do { \ } } - /* Operands: Size Live Fail Flags Dst */ - OpCase(i_bs_get_integer_imm_rIIfId): { - tmp_arg1 = r(0); - /* Operands: Size Live Fail Flags Dst */ - goto do_bs_get_integer_imm_test_heap; - } + { + Eterm Ms, Sz; - /* Operands: x(Reg) Size Live Fail Flags Dst */ + /* Operands: x(Reg) Size Live Fail Flags Dst */ OpCase(i_bs_get_integer_imm_xIIfId): { - tmp_arg1 = xb(Arg(0)); - I++; - /* Operands: Size Live Fail Flags Dst */ - goto do_bs_get_integer_imm_test_heap; - } - - /* - * tmp_arg1 = match context - * Operands: Size Live Fail Flags Dst - */ - do_bs_get_integer_imm_test_heap: { - Uint wordsneeded; - tmp_arg2 = Arg(0); - wordsneeded = 1+WSIZE(NBYTES(tmp_arg2)); - TestHeapPreserve(wordsneeded, Arg(1), tmp_arg1); - I += 2; - /* Operands: Fail Flags Dst */ - goto do_bs_get_integer_imm; - } - - /* Operands: Size Fail Flags Dst */ - OpCase(i_bs_get_integer_small_imm_rIfId): { - tmp_arg1 = r(0); - tmp_arg2 = Arg(0); - I++; - /* Operands: Fail Flags Dst */ - goto do_bs_get_integer_imm; - } + Uint wordsneeded; + Ms = xb(Arg(0)); + Sz = Arg(1); + wordsneeded = 1+WSIZE(NBYTES(Sz)); + TestHeapPreserve(wordsneeded, Arg(2), Ms); + I += 3; + /* Operands: Fail Flags Dst */ + goto do_bs_get_integer_imm; + } - /* Operands: x(Reg) Size Fail Flags Dst */ + /* Operands: x(Reg) Size Fail Flags Dst */ OpCase(i_bs_get_integer_small_imm_xIfId): { - tmp_arg1 = xb(Arg(0)); - tmp_arg2 = Arg(1); - I += 2; - /* Operands: Fail Flags Dst */ - goto do_bs_get_integer_imm; - } + Ms = xb(Arg(0)); + Sz = Arg(1); + I += 2; + /* Operands: Fail Flags Dst */ + goto do_bs_get_integer_imm; + } - /* - * tmp_arg1 = match context - * tmp_arg2 = size of field - * Operands: Fail Flags Dst - */ + /* + * Ms = match context + * Sz = size of field + * Operands: Fail Flags Dst + */ do_bs_get_integer_imm: { - ErlBinMatchBuffer* mb; - Eterm result; + ErlBinMatchBuffer* mb; + Eterm result; - mb = ms_matchbuffer(tmp_arg1); - LIGHT_SWAPOUT; - result = erts_bs_get_integer_2(c_p, tmp_arg2, Arg(1), mb); - LIGHT_SWAPIN; - HEAP_SPACE_VERIFIED(0); - if (is_non_value(result)) { - ClauseFail(); + mb = ms_matchbuffer(Ms); + LIGHT_SWAPOUT; + result = erts_bs_get_integer_2(c_p, Sz, Arg(1), mb); + LIGHT_SWAPIN; + HEAP_SPACE_VERIFIED(0); + if (is_non_value(result)) { + ClauseFail(); + } + StoreBifResult(2, result); } - StoreBifResult(2, result); } /* - * tmp_arg1 = Match context - * tmp_arg2 = Size field - * Operands: Fail Live FlagsAndUnit Dst + * Operands: Fail Live FlagsAndUnit Ms Sz Dst */ - OpCase(i_bs_get_integer_fIId): { + OpCase(i_bs_get_integer_fIIssd): { Uint flags; Uint size; + Eterm Ms; + Eterm Sz; ErlBinMatchBuffer* mb; Eterm result; flags = Arg(2); - BsGetFieldSize(tmp_arg2, (flags >> 3), ClauseFail(), size); + GetArg2(3, Ms, Sz); + BsGetFieldSize(Sz, (flags >> 3), ClauseFail(), size); if (size >= SMALL_BITS) { Uint wordsneeded; - /* check bits size before potential gc. + /* Check bits size before potential gc. * We do not want a gc and then realize we don't need - * the allocated space (i.e. if the op fails) + * the allocated space (i.e. if the op fails). * - * remember to reacquire the matchbuffer after gc. + * Remember to re-acquire the matchbuffer after gc. */ - mb = ms_matchbuffer(tmp_arg1); + mb = ms_matchbuffer(Ms); if (mb->size - mb->offset < size) { ClauseFail(); } wordsneeded = 1+WSIZE(NBYTES((Uint) size)); - TestHeapPreserve(wordsneeded, Arg(1), tmp_arg1); + TestHeapPreserve(wordsneeded, Arg(1), Ms); } - mb = ms_matchbuffer(tmp_arg1); + mb = ms_matchbuffer(Ms); LIGHT_SWAPOUT; result = erts_bs_get_integer_2(c_p, size, flags, mb); LIGHT_SWAPIN; @@ -4655,18 +4558,13 @@ do { \ if (is_non_value(result)) { ClauseFail(); } - StoreBifResult(3, result); + StoreBifResult(5, result); } { Eterm get_utf8_context; /* Operands: MatchContext Fail Dst */ - OpCase(i_bs_get_utf8_rfd): { - get_utf8_context = r(0); - goto do_bs_get_utf8; - } - OpCase(i_bs_get_utf8_xfd): { get_utf8_context = xb(Arg(0)); I++; @@ -4677,7 +4575,7 @@ do { \ * Operands: Fail Dst */ - do_bs_get_utf8: { + { Eterm result = erts_bs_get_utf8(ms_matchbuffer(get_utf8_context)); if (is_non_value(result)) { ClauseFail(); @@ -4690,12 +4588,7 @@ do { \ Eterm get_utf16_context; /* Operands: MatchContext Fail Flags Dst */ - OpCase(i_bs_get_utf16_rfId): { - get_utf16_context = r(0); - goto do_bs_get_utf16; - } - - OpCase(i_bs_get_utf16_xfId): { + OpCase(i_bs_get_utf16_xfId): { get_utf16_context = xb(Arg(0)); I++; } @@ -4704,7 +4597,7 @@ do { \ * get_utf16_context = match_context * Operands: Fail Flags Dst */ - do_bs_get_utf16: { + { Eterm result = erts_bs_get_utf16(ms_matchbuffer(get_utf16_context), Arg(1)); if (is_non_value(result)) { @@ -4723,26 +4616,10 @@ do { \ Uint orig; Uint hole_size; - OpCase(bs_context_to_binary_r): { - context_to_binary_context = x0; - I -= 2; - goto do_context_to_binary; - } - - /* Unfortunately, inlining can generate this instruction. */ - OpCase(bs_context_to_binary_y): { - context_to_binary_context = yb(Arg(0)); - goto do_context_to_binary0; - } - - OpCase(bs_context_to_binary_x): { + OpCase(bs_context_to_binary_x): context_to_binary_context = xb(Arg(0)); - - do_context_to_binary0: I--; - } - do_context_to_binary: if (is_boxed(context_to_binary_context) && header_is_bin_matchstate(*boxed_val(context_to_binary_context))) { ErlBinMatchState* ms; @@ -4754,17 +4631,11 @@ do { \ } Next(2); - OpCase(i_bs_get_binary_all_reuse_rfI): { - context_to_binary_context = x0; - goto do_bs_get_binary_all_reuse; - } - OpCase(i_bs_get_binary_all_reuse_xfI): { context_to_binary_context = xb(Arg(0)); I++; } - do_bs_get_binary_all_reuse: mb = ms_matchbuffer(context_to_binary_context); size = mb->size - mb->offset; if (size % Arg(1) != 0) { @@ -4792,16 +4663,11 @@ do { \ { Eterm match_string_context; - OpCase(i_bs_match_string_rfII): { - match_string_context = r(0); - goto do_bs_match_string; - } OpCase(i_bs_match_string_xfII): { match_string_context = xb(Arg(0)); I++; } - do_bs_match_string: { BeamInstr *next; byte* bytes; @@ -4829,14 +4695,6 @@ do { \ } } - OpCase(i_bs_save2_rI): { - BeamInstr *next; - ErlBinMatchState *_ms; - PreFetch(1, next); - _ms = (ErlBinMatchState*) boxed_val((Eterm) r(0)); - _ms->save_offset[Arg(0)] = _ms->mb.offset; - NextPF(1, next); - } OpCase(i_bs_save2_xI): { BeamInstr *next; ErlBinMatchState *_ms; @@ -4846,14 +4704,6 @@ do { \ NextPF(2, next); } - OpCase(i_bs_restore2_rI): { - BeamInstr *next; - ErlBinMatchState *_ms; - PreFetch(1, next); - _ms = (ErlBinMatchState*) boxed_val((Eterm) r(0)); - _ms->mb.offset = _ms->save_offset[Arg(0)]; - NextPF(1, next); - } OpCase(i_bs_restore2_xI): { BeamInstr *next; ErlBinMatchState *_ms; @@ -4884,11 +4734,11 @@ do { \ */ OpCase(return_trace): { - BeamInstr* code = (BeamInstr *) (UWord) E[0]; + ErtsCodeMFA* mfa = (ErtsCodeMFA *)(E[0]); SWAPOUT; /* Needed for shared heap */ ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); - erts_trace_return(c_p, code, r(0), E+1/*Process tracer*/); + erts_trace_return(c_p, mfa, r(0), ERTS_TRACER_FROM_ETERM(E+1)/* tracer */); ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); SWAPIN; c_p->cp = NULL; @@ -4899,12 +4749,9 @@ do { \ OpCase(i_generic_breakpoint): { BeamInstr real_I; - ASSERT(I[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); - SWAPOUT; - reg[0] = r(0); - real_I = erts_generic_breakpoint(c_p, I, reg); - r(0) = reg[0]; - SWAPIN; + HEAVY_SWAPOUT; + real_I = erts_generic_breakpoint(c_p, erts_code_to_codeinfo(I), reg); + HEAVY_SWAPIN; ASSERT(VALID_INSTR(real_I)); Goto(real_I); } @@ -4912,7 +4759,7 @@ do { \ OpCase(i_return_time_trace): { BeamInstr *pc = (BeamInstr *) (UWord) E[0]; SWAPOUT; - erts_trace_time_return(c_p, pc); + erts_trace_time_return(c_p, erts_code_to_codeinfo(pc)); SWAPIN; c_p->cp = NULL; SET_I((BeamInstr *) cp_val(E[1])); @@ -4963,7 +4810,7 @@ do { \ BeamInstr *next; PreFetch(2, next); - GetR(0, targ1); + targ1 = REG_TARGET(Arg(0)); /* Arg(0) == HEADER_FLONUM */ GET_DOUBLE(targ1, *(FloatDef*)ADD_BYTE_OFFSET(freg, fr)); NextPF(2, next); @@ -4983,7 +4830,7 @@ do { \ Eterm fr = Arg(1); BeamInstr *next; - GetR(0, targ1); + targ1 = REG_TARGET(Arg(0)); PreFetch(2, next); if (is_small(targ1)) { fb(fr) = (double) signed_val(targ1); @@ -5002,7 +4849,7 @@ do { \ #ifdef NO_FPE_SIGNALS OpCase(fclearerror): OpCase(i_fcheckerror): - erl_exit(1, "fclearerror/i_fcheckerror without fpe signals (beam_emu)"); + erts_exit(ERTS_ERROR_EXIT, "fclearerror/i_fcheckerror without fpe signals (beam_emu)"); # define ERTS_NO_FPE_CHECK_INIT ERTS_FP_CHECK_INIT # define ERTS_NO_FPE_ERROR ERTS_FP_ERROR #else @@ -5079,7 +4926,13 @@ do { \ #ifdef HIPE { - unsigned cmd; +#define HIPE_MODE_SWITCH(Cmd) \ + SWAPOUT; \ + ERTS_DBG_CHK_REDS(c_p, FCALLS); \ + c_p->fcalls = FCALLS; \ + c_p->def_arg_reg[4] = -neg_o_reds; \ + c_p = hipe_mode_switch(c_p, Cmd, reg); \ + goto L_post_hipe_mode_switch OpCase(hipe_trap_call): { /* @@ -5091,54 +4944,53 @@ do { \ * I[ 0]: &&lb_hipe_trap_call * ... remainder of original BEAM code */ - ASSERT(I[-5] == (Uint) OpCode(i_func_info_IaaI)); - c_p->hipe.u.ncallee = (void(*)(void)) I[-4]; - cmd = HIPE_MODE_SWITCH_CMD_CALL | (I[-1] << 8); + ErtsCodeInfo *ci = erts_code_to_codeinfo(I); + ASSERT(ci->op == (Uint) OpCode(i_func_info_IaaI)); + c_p->hipe.u.ncallee = ci->u.ncallee; ++hipe_trap_count; - goto L_hipe_mode_switch; + HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_CALL | (ci->mfa.arity << 8)); } OpCase(hipe_trap_call_closure): { - ASSERT(I[-5] == (Uint) OpCode(i_func_info_IaaI)); - c_p->hipe.u.ncallee = (void(*)(void)) I[-4]; - cmd = HIPE_MODE_SWITCH_CMD_CALL_CLOSURE | (I[-1] << 8); + ErtsCodeInfo *ci = erts_code_to_codeinfo(I); + ASSERT(ci->op == (Uint) OpCode(i_func_info_IaaI)); + c_p->hipe.u.ncallee = ci->u.ncallee; ++hipe_trap_count; - goto L_hipe_mode_switch; + HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_CALL_CLOSURE | (ci->mfa.arity << 8)); } OpCase(hipe_trap_return): { - cmd = HIPE_MODE_SWITCH_CMD_RETURN; - goto L_hipe_mode_switch; + HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_RETURN); } OpCase(hipe_trap_throw): { - cmd = HIPE_MODE_SWITCH_CMD_THROW; - goto L_hipe_mode_switch; + HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_THROW); } OpCase(hipe_trap_resume): { - cmd = HIPE_MODE_SWITCH_CMD_RESUME; - goto L_hipe_mode_switch; + HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_RESUME); } - L_hipe_mode_switch: - /* XXX: this abuse of def_arg_reg[] is horrid! */ - SWAPOUT; - c_p->fcalls = FCALLS; - c_p->def_arg_reg[4] = -neg_o_reds; - reg[0] = r(0); - c_p = hipe_mode_switch(c_p, cmd, reg); - reg = ERTS_PROC_GET_SCHDATA(c_p)->x_reg_array; - freg = ERTS_PROC_GET_SCHDATA(c_p)->f_reg_array; +#undef HIPE_MODE_SWITCH + + L_post_hipe_mode_switch: +#ifdef DEBUG + pid = c_p->common.id; /* may have switched process... */ +#endif + reg = erts_proc_sched_data(c_p)->x_reg_array; + freg = erts_proc_sched_data(c_p)->f_reg_array; ERL_BITS_RELOAD_STATEP(c_p); + /* XXX: this abuse of def_arg_reg[] is horrid! */ neg_o_reds = -c_p->def_arg_reg[4]; FCALLS = c_p->fcalls; SWAPIN; - switch( c_p->def_arg_reg[3] ) { /* Halfword wont work with hipe yet! */ + ERTS_DBG_CHK_REDS(c_p, FCALLS); + switch( c_p->def_arg_reg[3] ) { case HIPE_MODE_SWITCH_RES_RETURN: ASSERT(is_value(reg[0])); - MoveReturn(reg[0], r(0)); + SET_I(c_p->cp); + c_p->cp = 0; + Goto(*I); case HIPE_MODE_SWITCH_RES_CALL_EXPORTED: c_p->i = c_p->hipe.u.callee_exp->addressv[erts_active_code_ix()]; /*fall through*/ case HIPE_MODE_SWITCH_RES_CALL_BEAM: SET_I(c_p->i); - r(0) = reg[0]; Dispatch(); case HIPE_MODE_SWITCH_RES_CALL_CLOSURE: /* This can be used to call any function value, but currently it's @@ -5147,9 +4999,8 @@ do { \ BeamInstr *next; next = call_fun(c_p, c_p->arity - 1, reg, THE_NON_VALUE); - SWAPIN; + HEAVY_SWAPIN; if (next != NULL) { - r(0) = reg[0]; SET_I(next); Dispatchfun(); } @@ -5160,7 +5011,7 @@ do { \ I = handle_error(c_p, I, reg, NULL); goto post_error_handling; default: - erl_exit(1, "hipe_mode_switch: result %u\n", c_p->def_arg_reg[3]); + erts_exit(ERTS_ERROR_EXIT, "hipe_mode_switch: result %u\n", c_p->def_arg_reg[3]); } } OpCase(hipe_call_count): { @@ -5173,8 +5024,9 @@ do { \ * I[ 0]: &&lb_hipe_call_count * ... remainder of original BEAM code */ - struct hipe_call_count *hcc = (struct hipe_call_count*)I[-4]; - ASSERT(I[-5] == (Uint) OpCode(i_func_info_IaaI)); + ErtsCodeInfo *ci = erts_code_to_codeinfo(I); + struct hipe_call_count *hcc = ci->u.hcc; + ASSERT(ci->op == (Uint) OpCode(i_func_info_IaaI)); ASSERT(hcc != NULL); ASSERT(VALID_INSTR(hcc->opcode)); ++(hcc->count); @@ -5197,22 +5049,54 @@ do { \ } OpCase(i_hibernate): { - SWAPOUT; + HEAVY_SWAPOUT; if (erts_hibernate(c_p, r(0), x(1), x(2), reg)) { + FCALLS = c_p->fcalls; c_p->flags &= ~F_HIBERNATE_SCHED; goto do_schedule; } else { - I = handle_error(c_p, I, reg, hibernate_3); + HEAVY_SWAPIN; + I = handle_error(c_p, I, reg, &bif_export[BIF_hibernate_3]->info.mfa); goto post_error_handling; } } + /* This is optimised as an instruction because + it has to be very very fast */ + OpCase(i_perf_counter): { + BeamInstr* next; + ErtsSysPerfCounter ts; + PreFetch(0, next); + + ts = erts_sys_perf_counter(); + + if (IS_SSMALL(ts)) { + r(0) = make_small((Sint)ts); + } else { + TestHeap(ERTS_SINT64_HEAP_SIZE(ts),0); + r(0) = make_big(HTOP); +#if defined(ARCH_32) + if (ts >= (((Uint64) 1) << 32)) { + *HTOP = make_pos_bignum_header(2); + BIG_DIGIT(HTOP, 0) = (Uint) (ts & ((Uint) 0xffffffff)); + BIG_DIGIT(HTOP, 1) = (Uint) ((ts >> 32) & ((Uint) 0xffffffff)); + HTOP += 3; + } + else +#endif + { + *HTOP = make_pos_bignum_header(1); + BIG_DIGIT(HTOP, 0) = (Uint) ts; + HTOP += 2; + } + } + NextPF(0, next); + } + OpCase(i_debug_breakpoint): { - SWAPOUT; - reg[0] = r(0); - I = call_error_handler(c_p, I-3, reg, am_breakpoint); - r(0) = reg[0]; - SWAPIN; + HEAVY_SWAPOUT; + I = call_error_handler(c_p, erts_code_to_codemfa(I), reg, am_breakpoint); + HEAVY_SWAPIN; if (I) { Goto(*I); } @@ -5240,7 +5124,7 @@ do { \ OpCase(label_L): OpCase(on_load): OpCase(line_I): - erl_exit(1, "meta op\n"); + erts_exit(ERTS_ERROR_EXIT, "meta op\n"); /* * One-time initialization of Beam emulator. @@ -5266,6 +5150,7 @@ do { \ em_call_error_handler = OpCode(call_error_handler); em_apply_bif = OpCode(apply_bif); em_call_nif = OpCode(call_nif); + em_call_bif_e = OpCode(call_bif_e); beam_apply[0] = (BeamInstr) OpCode(i_apply); beam_apply[1] = (BeamInstr) OpCode(normal_exit); @@ -5284,17 +5169,17 @@ do { \ bif_table[i].name, bif_table[i].arity); bif_export[i] = ep; - ep->code[3] = (BeamInstr) OpCode(apply_bif); - ep->code[4] = (BeamInstr) bif_table[i].f; + ep->beam[0] = (BeamInstr) OpCode(apply_bif); + ep->beam[1] = (BeamInstr) bif_table[i].f; /* XXX: set func info for bifs */ - ep->fake_op_func_info_for_hipe[0] = (BeamInstr) BeamOp(op_i_func_info_IaaI); + ep->info.op = (BeamInstr) BeamOp(op_i_func_info_IaaI); } return; } #ifdef NO_JUMP_TABLE default: - erl_exit(1, "unexpected op code %d\n",Go); + erts_exit(ERTS_ERROR_EXIT, "unexpected op code %d\n",Go); } #endif return; /* Never executed */ @@ -5313,34 +5198,277 @@ do { \ } } -static BifFunction -translate_gc_bif(void* gcf) +/* + * erts_dirty_process_main() is what dirty schedulers execute. Since they handle + * only NIF calls they do not need to be able to execute all BEAM + * instructions. + */ +void erts_dirty_process_main(ErtsSchedulerData *esdp) { - if (gcf == erts_gc_length_1) { - return length_1; - } else if (gcf == erts_gc_size_1) { - return size_1; - } else if (gcf == erts_gc_bit_size_1) { - return bit_size_1; - } else if (gcf == erts_gc_byte_size_1) { - return byte_size_1; - } else if (gcf == erts_gc_map_size_1) { - return map_size_1; - } else if (gcf == erts_gc_abs_1) { - return abs_1; - } else if (gcf == erts_gc_float_1) { - return float_1; - } else if (gcf == erts_gc_round_1) { - return round_1; - } else if (gcf == erts_gc_trunc_1) { - return round_1; - } else if (gcf == erts_gc_binary_part_2) { - return binary_part_2; - } else if (gcf == erts_gc_binary_part_3) { - return binary_part_3; - } else { - erl_exit(1, "bad gc bif"); +#ifdef ERTS_DIRTY_SCHEDULERS + Process* c_p = NULL; + ErtsMonotonicTime start_time; +#ifdef DEBUG + ERTS_DECLARE_DUMMY(Eterm pid); +#endif + + /* Pointer to X registers: x(1)..x(N); reg[0] is used when doing GC, + * in all other cases x0 is used. + */ + register Eterm* reg REG_xregs = NULL; + + /* + * Top of heap (next free location); grows upwards. + */ + register Eterm* HTOP REG_htop = NULL; + + /* Stack pointer. Grows downwards; points + * to last item pushed (normally a saved + * continuation pointer). + */ + register Eterm* E REG_stop = NULL; + + /* + * Pointer to next threaded instruction. + */ + register BeamInstr *I REG_I = NULL; + + ERTS_MSACC_DECLARE_CACHE_X() /* a cached value of the tsd pointer for msacc */ + + /* + * start_time always positive for dirty CPU schedulers, + * and negative for dirty I/O schedulers. + */ + + if (ERTS_SCHEDULER_IS_DIRTY_CPU(esdp)) { + start_time = erts_get_monotonic_time(NULL); + ASSERT(start_time >= 0); + } + else { + start_time = ERTS_SINT64_MIN; + ASSERT(start_time < 0); } + + goto do_dirty_schedule; + + context_switch: + c_p->current = erts_code_to_codemfa(I); /* Pointer to Mod, Func, Arity */ + c_p->arity = c_p->current->arity; + + { + int reds_used; + Eterm* argp; + int i; + + /* + * Make sure that there is enough room for the argument registers to be saved. + */ + if (c_p->arity > c_p->max_arg_reg) { + /* + * Yes, this is an expensive operation, but you only pay it the first + * time you call a function with more than 6 arguments which is + * scheduled out. This is better than paying for 26 words of wasted + * space for most processes which never call functions with more than + * 6 arguments. + */ + Uint size = c_p->arity * sizeof(c_p->arg_reg[0]); + if (c_p->arg_reg != c_p->def_arg_reg) { + c_p->arg_reg = (Eterm *) erts_realloc(ERTS_ALC_T_ARG_REG, + (void *) c_p->arg_reg, + size); + } else { + c_p->arg_reg = (Eterm *) erts_alloc(ERTS_ALC_T_ARG_REG, size); + } + c_p->max_arg_reg = c_p->arity; + } + + /* + * Save the argument registers and everything else. + */ + + argp = c_p->arg_reg; + for (i = c_p->arity - 1; i >= 0; i--) { + argp[i] = reg[i]; + } + SWAPOUT; + c_p->i = I; + + do_dirty_schedule: + + if (start_time < 0) { + /* + * Dirty I/O scheduler: + * One reduction consumed regardless of + * time spent in the dirty NIF. + */ + reds_used = esdp->virtual_reds + 1; + } + else { + /* + * Dirty CPU scheduler: + * Reductions based on time consumed by + * the dirty NIF. + */ + Sint64 treds; + treds = erts_time2reds(start_time, + erts_get_monotonic_time(esdp)); + treds += esdp->virtual_reds; + reds_used = treds > INT_MAX ? INT_MAX : (int) treds; + } + + PROCESS_MAIN_CHK_LOCKS(c_p); + ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + c_p = erts_schedule(esdp, c_p, reds_used); + + if (start_time >= 0) { + start_time = erts_get_monotonic_time(esdp); + ASSERT(start_time >= 0); + } + } + + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); +#ifdef DEBUG + pid = c_p->common.id; /* Save for debugging purposes */ +#endif + ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); + PROCESS_MAIN_CHK_LOCKS(c_p); + + ASSERT(!(c_p->flags & F_HIPE_MODE)); + ERTS_MSACC_UPDATE_CACHE_X(); + + /* + * Set fcalls even though we ignore it, so we don't + * confuse code accessing it... + */ + if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) + c_p->fcalls = 0; + else + c_p->fcalls = CONTEXT_REDS; + + if (erts_smp_atomic32_read_nob(&c_p->state) & ERTS_PSFLG_DIRTY_RUNNING_SYS) { + erts_execute_dirty_system_task(c_p); + goto do_dirty_schedule; + } + else { + ErtsCodeMFA *codemfa; + Eterm* argp; + int i, exiting; + + reg = esdp->x_reg_array; + + argp = c_p->arg_reg; + for (i = c_p->arity - 1; i >= 0; i--) { + reg[i] = argp[i]; + CHECK_TERM(reg[i]); + } + + /* + * We put the original reduction count in the process structure, to reduce + * the code size (referencing a field in a struct through a pointer stored + * in a register gives smaller code than referencing a global variable). + */ + + I = c_p->i; + + SWAPIN; + +#ifdef USE_VM_PROBES + if (DTRACE_ENABLED(process_scheduled)) { + DTRACE_CHARBUF(process_buf, DTRACE_TERM_BUF_SIZE); + DTRACE_CHARBUF(fun_buf, DTRACE_TERM_BUF_SIZE); + dtrace_proc_str(c_p, process_buf); + + if (ERTS_PROC_IS_EXITING(c_p)) { + strcpy(fun_buf, "<exiting>"); + } else { + ErtsCodeMFA *cmfa = find_function_from_pc(c_p->i); + if (cmfa) { + dtrace_fun_decode(c_p, cmfa, NULL, fun_buf); + } else { + erts_snprintf(fun_buf, sizeof(DTRACE_CHARBUF_NAME(fun_buf)), + "<unknown/%p>", *I); + } + } + + DTRACE2(process_scheduled, process_buf, fun_buf); + } +#endif + + /* + * call_nif is always first instruction in function: + * + * I[-3]: Module + * I[-2]: Function + * I[-1]: Arity + * I[0]: &&call_nif + * I[1]: Function pointer to NIF function + * I[2]: Pointer to erl_module_nif + * I[3]: Function pointer to dirty NIF + * + * This layout is determined by the NifExport struct + */ + + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_NIF); + + codemfa = erts_code_to_codemfa(I); + + DTRACE_NIF_ENTRY(c_p, codemfa); + c_p->current = codemfa; + SWAPOUT; + PROCESS_MAIN_CHK_LOCKS(c_p); + ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); + + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + if (em_apply_bif == (BeamInstr *) *I) { + exiting = erts_call_dirty_bif(esdp, c_p, I, reg); + } + else { + ASSERT(em_call_nif == (BeamInstr *) *I); + exiting = erts_call_dirty_nif(esdp, c_p, I, reg); + } + + ASSERT(!(c_p->flags & F_HIBERNATE_SCHED)); + + PROCESS_MAIN_CHK_LOCKS(c_p); + ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR); + if (exiting) + goto do_dirty_schedule; + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + + DTRACE_NIF_RETURN(c_p, codemfa); + ERTS_HOLE_CHECK(c_p); + SWAPIN; + I = c_p->i; + goto context_switch; + } +#endif /* ERTS_DIRTY_SCHEDULERS */ +} + +static ErtsCodeMFA * +gcbif2mfa(void* gcf) +{ + int i; + for (i = 0; erts_gc_bifs[i].bif; i++) { + if (erts_gc_bifs[i].gc_bif == gcf) + return &bif_export[erts_gc_bifs[i].exp_ix]->info.mfa; + } + erts_exit(ERTS_ERROR_EXIT, "bad gc bif"); + return NULL; +} + +static ErtsCodeMFA * +ubif2mfa(void* uf) +{ + int i; + for (i = 0; erts_u_bifs[i].bif; i++) { + if (erts_u_bifs[i].bif == uf) + return &bif_export[erts_u_bifs[i].exp_ix]->info.mfa; + } + erts_exit(ERTS_ERROR_EXIT, "bad u bif"); + return NULL; } /* @@ -5399,15 +5527,27 @@ Eterm error_atom[NUMBER_EXIT_CODES] = { */ static BeamInstr* -handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf) +handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, ErtsCodeMFA *bif_mfa) { Eterm* hp; Eterm Value = c_p->fvalue; Eterm Args = am_true; - c_p->i = pc; /* In case we call erl_exit(). */ ASSERT(c_p->freason != TRAP); /* Should have been handled earlier. */ + if (c_p->freason & EXF_RESTORE_NIF) + erts_nif_export_restore_error(c_p, &pc, reg, &bif_mfa); + +#ifdef DEBUG + if (bif_mfa) { + /* Verify that bif_mfa does not point into our nif export */ + NifExport *nep = ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p); + ASSERT(!nep || !ErtsInArea(bif_mfa, (char *)nep, sizeof(NifExport))); + } +#endif + + c_p->i = pc; /* In case we call erts_exit(). */ + /* * Check if we have an arglist for the top level call. If so, this * is encoded in Value, so we have to dig out the real Value as well @@ -5430,7 +5570,7 @@ handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf) * more modular. */ if (c_p->freason & EXF_SAVETRACE) { - save_stacktrace(c_p, pc, reg, bf, Args); + save_stacktrace(c_p, pc, reg, bif_mfa, Args); } /* @@ -5468,7 +5608,7 @@ handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf) c_p->cp = 0; /* To avoid keeping stale references. */ return new_pc; } - if (c_p->catches > 0) erl_exit(1, "Catch not found"); + if (c_p->catches > 0) erts_exit(ERTS_ERROR_EXIT, "Catch not found"); } ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); terminate_proc(c_p, Value); @@ -5500,8 +5640,10 @@ next_catch(Process* c_p, Eterm *reg) { /* Can not follow cp here - code may be unloaded */ BeamInstr *cpp = c_p->cp; if (cpp == beam_exception_trace) { - erts_trace_exception(c_p, cp_val(ptr[0]), - reg[1], reg[2], ptr+1); + ErtsCodeMFA *mfa = (ErtsCodeMFA*)cp_val(ptr[0]); + erts_trace_exception(c_p, mfa, + reg[1], reg[2], + ERTS_TRACER_FROM_ETERM(ptr+1)); /* Skip return_trace parameters */ ptr += 2; } else if (cpp == beam_return_trace) { @@ -5527,8 +5669,10 @@ next_catch(Process* c_p, Eterm *reg) { if (is_catch(*ptr) && active_catches) goto found_catch; } if (cp_val(*prev) == beam_exception_trace) { - erts_trace_exception(c_p, cp_val(ptr[0]), - reg[1], reg[2], ptr+1); + ErtsCodeMFA *mfa = (ErtsCodeMFA*)cp_val(ptr[0]); + erts_trace_exception(c_p, mfa, + reg[1], reg[2], + ERTS_TRACER_FROM_ETERM(ptr+1)); } /* Skip return_trace parameters */ ptr += 2; @@ -5701,11 +5845,12 @@ expand_error_value(Process* c_p, Uint freason, Eterm Value) { */ static void -save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf, - Eterm args) { +save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, + ErtsCodeMFA *bif_mfa, Eterm args) { struct StackTrace* s; int sz; int depth = erts_backtrace_depth; /* max depth (never negative) */ + if (depth > 0) { /* There will always be a current function */ depth --; @@ -5721,33 +5866,30 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf, s->depth = 0; /* - * If the failure was in a BIF other than 'error', 'exit' or - * 'throw', find the bif-table index and save the argument + * If the failure was in a BIF other than 'error/1', 'error/2', + * 'exit/1' or 'throw/1', save BIF-MFA and save the argument * registers by consing up an arglist. */ - if (bf != NULL && bf != error_1 && bf != error_2 && - bf != exit_1 && bf != throw_1) { - int i; - int a = 0; - for (i = 0; i < BIF_SIZE; i++) { - if (bf == bif_table[i].f || bf == bif_table[i].traced) { - Export *ep = bif_export[i]; - s->current = ep->code; - a = bif_table[i].arity; + if (bif_mfa) { + if (bif_mfa->module == am_erlang) { + switch (bif_mfa->function) { + case am_error: + if (bif_mfa->arity == 1 || bif_mfa->arity == 2) + goto non_bif_stacktrace; + break; + case am_exit: + if (bif_mfa->arity == 1) + goto non_bif_stacktrace; + break; + case am_throw: + if (bif_mfa->arity == 1) + goto non_bif_stacktrace; + break; + default: break; } } - if (i >= BIF_SIZE) { - /* - * The Bif does not really exist (no BIF entry). It is a - * TRAP and traps are called through apply_bif, which also - * sets c_p->current (luckily). - * OR it is a NIF called by call_nif where current is also set. - */ - ASSERT(c_p->current); - s->current = c_p->current; - a = s->current[2]; - } + s->current = bif_mfa; /* Save first stack entry */ ASSERT(pc); if (depth > 0) { @@ -5760,8 +5902,11 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf, depth--; } s->pc = NULL; - args = make_arglist(c_p, reg, a); /* Overwrite CAR(c_p->ftrace) */ + args = make_arglist(c_p, reg, bif_mfa->arity); /* Overwrite CAR(c_p->ftrace) */ } else { + + non_bif_stacktrace: + s->current = c_p->current; /* * For a function_clause error, the arguments are in the beam @@ -5771,7 +5916,7 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf, (GET_EXC_INDEX(EXC_FUNCTION_CLAUSE)) ) { int a; ASSERT(s->current); - a = s->current[2]; + a = s->current->arity; args = make_arglist(c_p, reg, a); /* Overwrite CAR(c_p->ftrace) */ /* Save first stack entry */ ASSERT(c_p->cp); @@ -5822,12 +5967,30 @@ erts_save_stacktrace(Process* p, struct StackTrace* s, int depth) p->cp) { /* Cannot follow cp here - code may be unloaded */ BeamInstr *cpp = p->cp; + int trace_cp; if (cpp == beam_exception_trace || cpp == beam_return_trace) { /* Skip return_trace parameters */ ptr += 2; + trace_cp = 1; } else if (cpp == beam_return_to_trace) { /* Skip return_to_trace parameters */ ptr += 1; + trace_cp = 1; + } + else { + trace_cp = 0; + } + if (trace_cp && s->pc == cpp) { + /* + * If process 'cp' points to a return/exception trace + * instruction and 'cp' has been saved as 'pc' in + * stacktrace, we need to update 'pc' in stacktrace + * with the actual 'cp' located on the top of the + * stack; otherwise, we will lose the top stackframe + * when building the stack trace. + */ + ASSERT(is_CP(p->stop[0])); + s->pc = cp_val(p->stop[0]); } } while (ptr < STACK_START(p) && depth > 0) { @@ -5942,17 +6105,20 @@ build_stacktrace(Process* c_p, Eterm exc) { erts_lookup_function_info(&fi, s->pc, 1); } else if (GET_EXC_INDEX(s->freason) == GET_EXC_INDEX(EXC_FUNCTION_CLAUSE)) { - erts_lookup_function_info(&fi, s->current, 1); + erts_lookup_function_info(&fi, erts_codemfa_to_code(s->current), 1); } else { erts_set_current_function(&fi, s->current); } + depth = s->depth; /* - * If fi.current is still NULL, default to the initial function + * If fi.current is still NULL, and we have no + * stack at all, default to the initial function * (e.g. spawn_link(erlang, abs, [1])). */ - if (fi.current == NULL) { - erts_set_current_function(&fi, c_p->u.initial); + if (fi.mfa == NULL) { + if (depth <= 0) + erts_set_current_function(&fi, &c_p->u.initial); args = am_true; /* Just in case */ } else { args = get_args_from_exc(exc); @@ -5962,13 +6128,12 @@ build_stacktrace(Process* c_p, Eterm exc) { * Look up all saved continuation pointers and calculate * needed heap space. */ - depth = s->depth; stk = stkp = (FunctionInfo *) erts_alloc(ERTS_ALC_T_TMP, depth*sizeof(FunctionInfo)); - heap_size = fi.needed + 2; + heap_size = fi.mfa ? fi.needed + 2 : 0; for (i = 0; i < depth; i++) { erts_lookup_function_info(stkp, s->trace[i], 1); - if (stkp->current) { + if (stkp->mfa) { heap_size += stkp->needed + 2; stkp++; } @@ -5984,15 +6149,17 @@ build_stacktrace(Process* c_p, Eterm exc) { res = CONS(hp, mfa, res); hp += 2; } - hp = erts_build_mfa_item(&fi, hp, args, &mfa); - res = CONS(hp, mfa, res); + if (fi.mfa) { + hp = erts_build_mfa_item(&fi, hp, args, &mfa); + res = CONS(hp, mfa, res); + } erts_free(ERTS_ALC_T_TMP, (void *) stk); return res; } static BeamInstr* -call_error_handler(Process* p, BeamInstr* fi, Eterm* reg, Eterm func) +call_error_handler(Process* p, ErtsCodeMFA* mfa, Eterm* reg, Eterm func) { Eterm* hp; Export* ep; @@ -6001,13 +6168,14 @@ call_error_handler(Process* p, BeamInstr* fi, Eterm* reg, Eterm func) Uint sz; int i; + DBG_TRACE_MFA_P(mfa, "call_error_handler"); /* * Search for the error_handler module. */ ep = erts_find_function(erts_proc_get_error_handler(p), func, 3, erts_active_code_ix()); if (ep == NULL) { /* No error handler */ - p->current = fi; + p->current = mfa; p->freason = EXC_UNDEF; return 0; } @@ -6016,7 +6184,7 @@ call_error_handler(Process* p, BeamInstr* fi, Eterm* reg, Eterm func) * Create a list with all arguments in the x registers. */ - arity = fi[2]; + arity = mfa->arity; sz = 2 * arity; if (HeapWordsLeft(p) < sz) { erts_garbage_collect(p, sz, reg, arity); @@ -6032,8 +6200,8 @@ call_error_handler(Process* p, BeamInstr* fi, Eterm* reg, Eterm func) /* * Set up registers for call to error_handler:<func>/3. */ - reg[0] = fi[0]; - reg[1] = fi[1]; + reg[0] = mfa->module; + reg[1] = mfa->function; reg[2] = args; return ep->addressv[erts_active_code_ix()]; } @@ -6082,8 +6250,107 @@ apply_setup_error_handler(Process* p, Eterm module, Eterm function, Uint arity, return ep; } +static ERTS_INLINE void +apply_bif_error_adjustment(Process *p, Export *ep, + Eterm *reg, Uint arity, + BeamInstr *I, Uint stack_offset) +{ + /* + * I is only set when the apply is a tail call, i.e., + * from the instructions i_apply_only, i_apply_last_P, + * and apply_last_IP. + */ + if (I + && ep->beam[0] == (BeamInstr) em_apply_bif + && (ep == bif_export[BIF_error_1] + || ep == bif_export[BIF_error_2] + || ep == bif_export[BIF_exit_1] + || ep == bif_export[BIF_throw_1])) { + /* + * We are about to tail apply one of the BIFs + * erlang:error/1, erlang:error/2, erlang:exit/1, + * or erlang:throw/1. Error handling of these BIFs is + * special! + * + * We need 'p->cp' to point into the calling + * function when handling the error after the BIF has + * been applied. This in order to get the topmost + * stackframe correct. Without the following adjustment, + * 'p->cp' will point into the function that called + * current function when handling the error. We add a + * dummy stackframe in order to achieve this. + * + * Note that these BIFs unconditionally will cause + * an exception to be raised. That is, our modifications + * of 'p->cp' as well as the stack will be corrected by + * the error handling code. + * + * If we find an exception/return-to trace continuation + * pointer as the topmost continuation pointer, we do not + * need to do anything since the information already will + * be available for generation of the stacktrace. + */ + int apply_only = stack_offset == 0; + BeamInstr *cpp; + + if (apply_only) { + ASSERT(p->cp != NULL); + cpp = p->cp; + } + else { + ASSERT(is_CP(p->stop[0])); + cpp = cp_val(p->stop[0]); + } + + if (cpp != beam_exception_trace + && cpp != beam_return_trace + && cpp != beam_return_to_trace) { + Uint need = stack_offset /* bytes */ / sizeof(Eterm); + if (need == 0) + need = 1; /* i_apply_only */ + if (p->stop - p->htop < need) + erts_garbage_collect(p, (int) need, reg, arity+1); + p->stop -= need; + + if (apply_only) { + /* + * Called from the i_apply_only instruction. + * + * 'p->cp' contains continuation pointer pointing + * into the function that called current function. + * We push that continuation pointer onto the stack, + * and set 'p->cp' to point into current function. + */ + + p->stop[0] = make_cp(p->cp); + p->cp = I; + } + else { + /* + * Called from an i_apply_last_p, or apply_last_IP, + * instruction. + * + * Calling instruction will after we return read + * a continuation pointer from the stack and write + * it to 'p->cp', and then remove the topmost + * stackframe of size 'stack_offset'. + * + * We have sized the dummy-stackframe so that it + * will be removed by the instruction we currently + * are executing, and leave the stackframe that + * normally would have been removed intact. + * + */ + p->stop[0] = make_cp(I); + } + } + } +} + static BeamInstr* -apply(Process* p, Eterm module, Eterm function, Eterm args, Eterm* reg) +apply( +Process* p, Eterm module, Eterm function, Eterm args, Eterm* reg, +BeamInstr *I, Uint stack_offset) { int arity; Export* ep; @@ -6108,21 +6375,54 @@ apply(Process* p, Eterm module, Eterm function, Eterm args, Eterm* reg) return 0; } - /* The module argument may be either an atom or an abstract module - * (currently implemented using tuples, but this might change). - */ - this = THE_NON_VALUE; - if (is_not_atom(module)) { - Eterm* tp; + while (1) { + Eterm m, f, a; + /* The module argument may be either an atom or an abstract module + * (currently implemented using tuples, but this might change). + */ + this = THE_NON_VALUE; + if (is_not_atom(module)) { + Eterm* tp; + + if (is_not_tuple(module)) goto error; + tp = tuple_val(module); + if (arityval(tp[0]) < 1) goto error; + this = module; + module = tp[1]; + if (is_not_atom(module)) goto error; + } - if (is_not_tuple(module)) goto error; - tp = tuple_val(module); - if (arityval(tp[0]) < 1) goto error; - this = module; - module = tp[1]; - if (is_not_atom(module)) goto error; + if (module != am_erlang || function != am_apply) + break; + + /* Adjust for multiple apply of apply/3... */ + + a = args; + if (is_list(a)) { + Eterm *consp = list_val(a); + m = CAR(consp); + a = CDR(consp); + if (is_list(a)) { + consp = list_val(a); + f = CAR(consp); + a = CDR(consp); + if (is_list(a)) { + consp = list_val(a); + a = CAR(consp); + if (is_nil(CDR(consp))) { + /* erlang:apply/3 */ + module = m; + function = f; + args = a; + if (is_not_atom(f)) + goto error; + continue; + } + } + } + } + break; /* != erlang:apply/3 */ } - /* * Walk down the 3rd parameter of apply (the argument list) and copy * the parameters to the x registers (reg[]). If the module argument @@ -6160,12 +6460,14 @@ apply(Process* p, Eterm module, Eterm function, Eterm args, Eterm* reg) } else if (ERTS_PROC_GET_SAVED_CALLS_BUF(p)) { save_calls(p, ep); } + apply_bif_error_adjustment(p, ep, reg, arity, I, stack_offset); DTRACE_GLOBAL_CALL_FROM_EXPORT(p, ep); return ep->addressv[erts_active_code_ix()]; } static BeamInstr* -fixed_apply(Process* p, Eterm* reg, Uint arity) +fixed_apply(Process* p, Eterm* reg, Uint arity, + BeamInstr *I, Uint stack_offset) { Export* ep; Eterm module; @@ -6195,6 +6497,10 @@ fixed_apply(Process* p, Eterm* reg, Uint arity) if (is_not_atom(module)) goto error; ++arity; } + + /* Handle apply of apply/3... */ + if (module == am_erlang && function == am_apply && arity == 3) + return apply(p, reg[0], reg[1], reg[2], reg, I, stack_offset); /* * Get the index into the export table, or failing that the export @@ -6209,6 +6515,7 @@ fixed_apply(Process* p, Eterm* reg, Uint arity) } else if (ERTS_PROC_GET_SAVED_CALLS_BUF(p)) { save_calls(p, ep); } + apply_bif_error_adjustment(p, ep, reg, arity, I, stack_offset); DTRACE_GLOBAL_CALL_FROM_EXPORT(p, ep); return ep->addressv[erts_active_code_ix()]; } @@ -6219,6 +6526,23 @@ erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* re int arity; Eterm tmp; +#ifndef ERTS_SMP + if (ERTS_PROC_IS_EXITING(c_p)) { + /* + * I non smp case: + * + * Currently executing process might be sent an exit + * signal if it is traced by a port that it also is + * linked to, and the port terminates during the + * trace. In this case we do *not* want to clear + * the active flag, which will make the process hang + * in limbo forever. Get out of here and terminate + * the process... + */ + return -1; + } +#endif + if (is_not_atom(module) || is_not_atom(function)) { /* * No need to test args here -- done below. @@ -6261,11 +6585,11 @@ erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* re #ifdef USE_VM_PROBES if (DTRACE_ENABLED(process_hibernate)) { + ErtsCodeMFA cmfa = { module, function, arity}; DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); - DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE); - dtrace_fun_decode(c_p, module, function, arity, - process_name, mfa); - DTRACE2(process_hibernate, process_name, mfa); + DTRACE_CHARBUF(mfa_buf, DTRACE_TERM_BUF_SIZE); + dtrace_fun_decode(c_p, &cmfa, process_name, mfa_buf); + DTRACE2(process_hibernate, process_name, mfa_buf); } #endif /* @@ -6295,15 +6619,24 @@ erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* re ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS); -#ifdef ERTS_SMP - ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p); +#ifndef ERTS_SMP + if (ERTS_PROC_IS_EXITING(c_p)) { + /* + * See comment in the beginning of the function... + * + * This second test is needed since gc might be traced. + */ + return -1; + } +#else /* ERTS_SMP */ + ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p); if (!c_p->msg.len) #endif erts_smp_atomic32_read_band_relb(&c_p->state, ~ERTS_PSFLG_ACTIVE); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); } erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS); - c_p->current = bif_export[BIF_hibernate_3]->code; + c_p->current = &bif_export[BIF_hibernate_3]->info.mfa; c_p->flags |= F_HIBERNATE_SCHED; /* Needed also when woken! */ return 1; } @@ -6326,21 +6659,15 @@ call_fun(Process* p, /* Current process. */ if (is_fun_header(hdr)) { ErlFunThing* funp = (ErlFunThing *) fun_val(fun); - ErlFunEntry* fe; - BeamInstr* code_ptr; + ErlFunEntry* fe = funp->fe; + BeamInstr* code_ptr = fe->address; Eterm* var_ptr; - int actual_arity; - unsigned num_free; - - fe = funp->fe; - num_free = funp->num_free; - code_ptr = fe->address; - actual_arity = (int) code_ptr[-1]; + unsigned num_free = funp->num_free; + ErtsCodeMFA *mfa = erts_code_to_codemfa(code_ptr); + int actual_arity = mfa->arity; if (actual_arity == arity+num_free) { - DTRACE_LOCAL_CALL(p, (Eterm)code_ptr[-3], - (Eterm)code_ptr[-2], - code_ptr[-1]); + DTRACE_LOCAL_CALL(p, mfa); if (num_free == 0) { return code_ptr; } else { @@ -6395,34 +6722,49 @@ call_fun(Process* p, /* Current process. */ * representation (the module has never been loaded), * or the module defining the fun has been unloaded. */ + module = fe->module; - if ((modp = erts_get_module(module, code_ix)) != NULL - && modp->curr.code != NULL) { + + ERTS_SMP_READ_MEMORY_BARRIER; + if (fe->pend_purge_address) { /* - * There is a module loaded, but obviously the fun is not - * defined in it. We must not call the error_handler - * (or we will get into an infinite loop). + * The system is currently trying to purge the + * module containing this fun. Suspend the process + * and let it try again when the purge operation is + * done (may succeed or not). */ - goto badfun; + ep = erts_suspend_process_on_pending_purge_lambda(p, fe); + ASSERT(ep); } + else { + if ((modp = erts_get_module(module, code_ix)) != NULL + && modp->curr.code_hdr != NULL) { + /* + * There is a module loaded, but obviously the fun is not + * defined in it. We must not call the error_handler + * (or we will get into an infinite loop). + */ + goto badfun; + } - /* - * No current code for this module. Call the error_handler module - * to attempt loading the module. - */ + /* + * No current code for this module. Call the error_handler module + * to attempt loading the module. + */ - ep = erts_find_function(erts_proc_get_error_handler(p), - am_undefined_lambda, 3, code_ix); - if (ep == NULL) { /* No error handler */ - p->current = NULL; - p->freason = EXC_UNDEF; - return NULL; + ep = erts_find_function(erts_proc_get_error_handler(p), + am_undefined_lambda, 3, code_ix); + if (ep == NULL) { /* No error handler */ + p->current = NULL; + p->freason = EXC_UNDEF; + return NULL; + } } reg[0] = module; reg[1] = fun; reg[2] = args; reg[3] = NIL; - return ep->addressv[erts_active_code_ix()]; + return ep->addressv[code_ix]; } } } else if (is_export_header(hdr)) { @@ -6430,10 +6772,10 @@ call_fun(Process* p, /* Current process. */ int actual_arity; ep = *((Export **) (export_val(fun) + 1)); - actual_arity = (int) ep->code[2]; + actual_arity = ep->info.mfa.arity; if (arity == actual_arity) { - DTRACE_GLOBAL_CALL(p, ep->code[0], ep->code[1], (Uint)ep->code[2]); + DTRACE_GLOBAL_CALL(p, &ep->info.mfa); return ep->addressv[erts_active_code_ix()]; } else { /* @@ -6487,7 +6829,7 @@ apply_fun(Process* p, Eterm fun, Eterm args, Eterm* reg) } if (is_not_nil(tmp)) { /* Must be well-formed list */ - p->freason = EXC_UNDEF; + p->freason = EXC_BADARG; return NULL; } reg[arity] = fun; @@ -6514,16 +6856,13 @@ new_fun(Process* p, Eterm* reg, ErlFunEntry* fe, int num_free) p->htop = hp + needed; funp = (ErlFunThing *) hp; hp = funp->env; - erts_refc_inc(&fe->refc, 2); + erts_smp_refc_inc(&fe->refc, 2); funp->thing_word = HEADER_FUN; funp->next = MSO(p).first; MSO(p).first = (struct erl_off_heap_header*) funp; funp->fe = fe; funp->num_free = num_free; funp->creator = p->common.id; -#ifdef HIPE - funp->native_address = fe->native_address; -#endif funp->arity = (int)fe->address[-1] - num_free; for (i = 0; i < num_free; i++) { *hp++ = reg[i]; @@ -6602,23 +6941,20 @@ static Eterm get_map_element_hash(Eterm map, Eterm key, Uint32 hx) return vs ? *vs : THE_NON_VALUE; } -#define GET_TERM(term, dest) \ -do { \ - Eterm src = (Eterm)(term); \ - switch (src & _TAG_IMMED1_MASK) { \ - case (R_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER: \ - dest = x(0); \ - break; \ - case (X_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER: \ - dest = x(src >> _TAG_IMMED1_SIZE); \ - break; \ - case (Y_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER: \ - dest = y(src >> _TAG_IMMED1_SIZE); \ - break; \ - default: \ - dest = src; \ - break; \ - } \ +#define GET_TERM(term, dest) \ +do { \ + Eterm src = (Eterm)(term); \ + switch (loader_tag(src)) { \ + case LOADER_X_REG: \ + dest = x(loader_x_reg_index(src)); \ + break; \ + case LOADER_Y_REG: \ + dest = y(loader_y_reg_index(src)); \ + break; \ + default: \ + dest = src; \ + break; \ + } \ } while(0) @@ -6657,13 +6993,6 @@ new_map(Process* p, Eterm* reg, BeamInstr* I) erts_factory_proc_init(&factory, p); res = erts_hashmap_from_array(&factory, thp, n/2, 0); erts_factory_close(&factory); - if (p->mbuf) { - Uint live = Arg(2); - reg[live] = res; - erts_garbage_collect(p, 0, reg, live+1); - res = reg[live]; - E = p->stop; - } return res; } @@ -6729,13 +7058,6 @@ update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I) hx = hashmap_make_hash(new_key); res = erts_hashmap_insert(p, hx, new_key, val, res, 0); - if (p->mbuf) { - Uint live = Arg(3); - reg[live] = res; - erts_garbage_collect(p, 0, reg, live+1); - res = reg[live]; - E = p->stop; - } new_p += 2; } @@ -6894,13 +7216,11 @@ update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I) /* The expensive case, need to build a hashmap */ if (n > MAP_SMALL_MAP_LIMIT) { - res = erts_hashmap_from_ks_and_vs(p,flatmap_get_keys(mp),flatmap_get_values(mp),n); - if (p->mbuf) { - Uint live = Arg(3); - reg[live] = res; - erts_garbage_collect(p, 0, reg, live+1); - res = reg[live]; - } + ErtsHeapFactory factory; + erts_factory_proc_init(&factory, p); + res = erts_hashmap_from_ks_and_vs(&factory,flatmap_get_keys(mp), + flatmap_get_values(mp),n); + erts_factory_close(&factory); } return res; } @@ -6956,14 +7276,6 @@ update_map_exact(Process* p, Eterm* reg, Eterm map, BeamInstr* I) return res; } - if (p->mbuf) { - Uint live = Arg(3); - reg[live] = res; - erts_garbage_collect(p, 0, reg, live+1); - res = reg[live]; - E = p->stop; - } - new_p += 2; } return res; @@ -7072,15 +7384,24 @@ erts_is_builtin(Eterm Mod, Eterm Name, int arity) Export e; Export* ep; - e.code[0] = Mod; - e.code[1] = Name; - e.code[2] = arity; + if (Mod == am_erlang && Name == am_apply && arity == 3) { + /* + * Special case. apply/3 is built-in (implemented in C), + * but implemented in a different way than all other + * BIFs. + */ + return 1; + } + + e.info.mfa.module = Mod; + e.info.mfa.function = Name; + e.info.mfa.arity = arity; if ((ep = export_get(&e)) == NULL) { return 0; } - return ep->addressv[erts_active_code_ix()] == ep->code+3 - && (ep->code[3] == (BeamInstr) em_apply_bif); + return ep->addressv[erts_active_code_ix()] == ep->beam + && (ep->beam[0] == (BeamInstr) em_apply_bif); } @@ -7095,7 +7416,7 @@ erts_current_reductions(Process *current, Process *p) if (current != p) { return 0; } else if (current->fcalls < 0 && ERTS_PROC_GET_SAVED_CALLS_BUF(current)) { - return -current->fcalls; + return current->fcalls + CONTEXT_REDS; } else { return REDS_IN(current) - current->fcalls; } |