diff options
Diffstat (limited to 'erts/emulator/beam/instrs.tab')
| -rw-r--r-- | erts/emulator/beam/instrs.tab | 167 |
1 files changed, 123 insertions, 44 deletions
diff --git a/erts/emulator/beam/instrs.tab b/erts/emulator/beam/instrs.tab index bc8c1189a8..156de67716 100644 --- a/erts/emulator/beam/instrs.tab +++ b/erts/emulator/beam/instrs.tab @@ -19,7 +19,12 @@ // %CopyrightEnd% // -// Stack manipulation instructions +// +// Stack manipulation instructions follow. +// +// See the comment for AH() in macros.tab for information about +// the layout of stack frames. +// allocate(NeedStack, Live) { $AH($NeedStack, 0, $Live); @@ -58,22 +63,84 @@ allocate_heap_zero(NeedStack, NeedHeap, Live) { deallocate(Deallocate) { //| -no_prefetch - SET_CP(c_p, (BeamInstr *) cp_val(*E)); E = ADD_BYTE_OFFSET(E, $Deallocate); } +deallocate_return0 := dealloc_ret.n0.execute; +deallocate_return1 := dealloc_ret.n1.execute; +deallocate_return2 := dealloc_ret.n2.execute; +deallocate_return3 := dealloc_ret.n3.execute; +deallocate_return4 := dealloc_ret.n4.execute; + +dealloc_ret.head() { + Uint num_bytes; +} + +dealloc_ret.n0() { + num_bytes = (0+1) * sizeof(Eterm); +} + +dealloc_ret.n1() { + num_bytes = (1+1) * sizeof(Eterm); +} + +dealloc_ret.n2() { + num_bytes = (2+1) * sizeof(Eterm); +} + +dealloc_ret.n3() { + num_bytes = (3+1) * sizeof(Eterm); +} + +dealloc_ret.n4() { + num_bytes = (4+1) * sizeof(Eterm); +} + +dealloc_ret.execute() { + //| -no_next + + /* + * Micro-benchmarks showed that the deallocate_return instruction + * became slower when the continuation pointer was moved from + * the process struct to the stack. The reason seems to be read + * dependencies, i.e. that the CPU cannot figure out beforehand + * from which position on the stack the continuation pointer + * should be fetched. + * + * Making sure that num_bytes is always initialized with a + * constant value seems to restore the lost speed. + */ + + E = ADD_BYTE_OFFSET(E, num_bytes); + $RETURN(); + CHECK_TERM(x(0)); + DispatchReturn; +} + deallocate_return(Deallocate) { //| -no_next - int words_to_pop = $Deallocate; - SET_I((BeamInstr *) cp_val(*E)); - E = ADD_BYTE_OFFSET(E, words_to_pop); + Uint bytes_to_pop = $Deallocate; + E = ADD_BYTE_OFFSET(E, bytes_to_pop); + $RETURN(); CHECK_TERM(x(0)); DispatchReturn; } move_deallocate_return(Src, Deallocate) { - x(0) = $Src; - $deallocate_return($Deallocate); + //| -no_next + + /* + * Explicitly do reads first to mitigate the impact of read + * dependencies. + */ + + Uint bytes_to_pop = $Deallocate; + Eterm src = $Src; + E = ADD_BYTE_OFFSET(E, bytes_to_pop); + x(0) = src; + $RETURN(); + CHECK_TERM(x(0)); + DispatchReturn; } // Call instructions @@ -93,14 +160,16 @@ DISPATCH_ABS(CallDest) { } i_call(CallDest) { - SET_CP(c_p, $NEXT_INSTRUCTION); + $SAVE_CONTINUATION_POINTER($NEXT_INSTRUCTION); $DISPATCH_REL($CallDest); } move_call(Src, CallDest) { - x(0) = $Src; - SET_CP(c_p, $NEXT_INSTRUCTION); - $DISPATCH_REL($CallDest); + Eterm call_dest = $CallDest; + Eterm src = $Src; + $SAVE_CONTINUATION_POINTER($NEXT_INSTRUCTION); + x(0) = src; + $DISPATCH_REL(call_dest); } i_call_last(CallDest, Deallocate) { @@ -109,8 +178,11 @@ i_call_last(CallDest, Deallocate) { } move_call_last(Src, CallDest, Deallocate) { - x(0) = $Src; - $i_call_last($CallDest, $Deallocate); + Eterm call_dest = $CallDest; + Eterm src = $Src; + $deallocate($Deallocate); + x(0) = src; + $DISPATCH_REL(call_dest); } i_call_only(CallDest) { @@ -118,8 +190,10 @@ i_call_only(CallDest) { } move_call_only(Src, CallDest) { - x(0) = $Src; - $i_call_only($CallDest); + Eterm call_dest = $CallDest; + Eterm src = $Src; + x(0) = src; + $DISPATCH_REL(call_dest); } DISPATCHX(Dest) { @@ -131,22 +205,27 @@ DISPATCHX(Dest) { } i_call_ext(Dest) { - SET_CP(c_p, $NEXT_INSTRUCTION); + $SAVE_CONTINUATION_POINTER($NEXT_INSTRUCTION); $DISPATCHX($Dest); } -i_move_call_ext(Src, Dest) { - x(0) = $Src; - $i_call_ext($Dest); +i_move_call_ext(Src, CallDest) { + Eterm call_dest = $CallDest; + Eterm src = $Src; + $SAVE_CONTINUATION_POINTER($NEXT_INSTRUCTION); + x(0) = src; + $DISPATCHX(call_dest); } i_call_ext_only(Dest) { $DISPATCHX($Dest); } -i_move_call_ext_only(Dest, Src) { - x(0) = $Src; - $i_call_ext_only($Dest); +i_move_call_ext_only(CallDest, Src) { + Eterm call_dest = $CallDest; + Eterm src = $Src; + x(0) = src; + $DISPATCHX(call_dest); } i_call_ext_last(Dest, Deallocate) { @@ -154,9 +233,12 @@ i_call_ext_last(Dest, Deallocate) { $DISPATCHX($Dest); } -i_move_call_ext_last(Dest, StackOffset, Src) { - x(0) = $Src; - $i_call_ext_last($Dest, $StackOffset); +i_move_call_ext_last(CallDest, Deallocate, Src) { + Eterm call_dest = $CallDest; + Eterm src = $Src; + $deallocate($Deallocate); + x(0) = src; + $DISPATCHX(call_dest); } APPLY(I, Deallocate, Next) { @@ -175,7 +257,7 @@ i_apply() { BeamInstr *next; $APPLY(NULL, 0, next); if (ERTS_LIKELY(next != NULL)) { - SET_CP(c_p, $NEXT_INSTRUCTION); + $SAVE_CONTINUATION_POINTER($NEXT_INSTRUCTION); $DISPATCH_ABS(next); } $HANDLE_APPLY_ERROR(); @@ -211,7 +293,7 @@ apply(Arity) { BeamInstr *next; $FIXED_APPLY($Arity, NULL, 0, next); if (ERTS_LIKELY(next != NULL)) { - SET_CP(c_p, $NEXT_INSTRUCTION); + $SAVE_CONTINUATION_POINTER($NEXT_INSTRUCTION); $DISPATCH_ABS(next); } $HANDLE_APPLY_ERROR(); @@ -247,7 +329,7 @@ i_apply_fun() { BeamInstr *next; $APPLY_FUN(next); if (ERTS_LIKELY(next != NULL)) { - SET_CP(c_p, $NEXT_INSTRUCTION); + $SAVE_CONTINUATION_POINTER($NEXT_INSTRUCTION); $DISPATCH_FUN(next); } $HANDLE_APPLY_FUN_ERROR(); @@ -283,7 +365,7 @@ i_call_fun(Fun) { BeamInstr *next; $CALL_FUN($Fun, next); if (ERTS_LIKELY(next != NULL)) { - SET_CP(c_p, $NEXT_INSTRUCTION); + $SAVE_CONTINUATION_POINTER($NEXT_INSTRUCTION); $DISPATCH_FUN(next); } $HANDLE_APPLY_FUN_ERROR(); @@ -301,15 +383,8 @@ i_call_fun_last(Fun, Deallocate) { return() { //| -no_next - SET_I(c_p->cp); + $RETURN(); DTRACE_RETURN_FROM_PC(c_p); - - /* - * We must clear the CP to make sure that a stale value do not - * create a false module dependcy preventing code upgrading. - * It also means that we can use the CP in stack backtraces. - */ - c_p->cp = 0; CHECK_TERM(r(0)); HEAP_SPACE_VERIFIED(0); DispatchReturn; @@ -478,16 +553,21 @@ i_make_fun(FunP, NumFree) { } move_trim(Src, Dst, Words) { - Uint cp = E[0]; $Dst = $Src; - E += $Words; - E[0] = cp; + $i_trim($Words); } i_trim(Words) { - Uint cp = E[0]; E += $Words; - E[0] = cp; + + /* + * Clear the reserved location for the continuation pointer at + * E[0]. This is not strictly necessary for correctness, but if a + * GC is triggered before E[0] is overwritten by another + * continuation pointer the now dead term at E[0] would be + * retained by the GC. + */ + E[0] = NIL; } move(Src, Dst) { @@ -599,8 +679,7 @@ move_window5(S1, S2, S3, S4, S5, D) { move_return(Src) { //| -no_next x(0) = $Src; - SET_I(c_p->cp); - c_p->cp = 0; + $RETURN(); DispatchReturn; } |