aboutsummaryrefslogtreecommitdiffstats
path: root/erts/emulator/beam/beam_emu.c
diff options
context:
space:
mode:
Diffstat (limited to 'erts/emulator/beam/beam_emu.c')
-rw-r--r--erts/emulator/beam/beam_emu.c3328
1 files changed, 2150 insertions, 1178 deletions
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 5781009f58..4716460a6b 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -1,18 +1,19 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 1996-2013. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2016. All Rights Reserved.
*
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
*
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*
* %CopyrightEnd%
*/
@@ -31,6 +32,7 @@
#include "big.h"
#include "beam_load.h"
#include "erl_binary.h"
+#include "erl_map.h"
#include "erl_bits.h"
#include "dist.h"
#include "beam_bp.h"
@@ -48,7 +50,7 @@
# define OpCase(OpCode) case op_##OpCode
# define CountCase(OpCode) case op_count_##OpCode
# define OpCode(OpCode) ((Uint*)op_##OpCode)
-# define Goto(Rel) {Go = (int)(Rel); goto emulator_loop;}
+# define Goto(Rel) {Go = (int)(UWord)(Rel); goto emulator_loop;}
# define LabelAddr(Addr) &&##Addr
#else
# define OpCase(OpCode) lb_##OpCode
@@ -62,17 +64,21 @@
# ifdef ERTS_SMP
# define PROCESS_MAIN_CHK_LOCKS(P) \
do { \
- if ((P)) { \
+ if ((P)) \
erts_proc_lc_chk_only_proc_main((P)); \
- } \
- else \
- erts_lc_check_exact(NULL, 0); \
- ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking()); \
+ ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking()); \
+} while (0)
+# define ERTS_SMP_REQ_PROC_MAIN_LOCK(P) \
+do { \
+ if ((P)) \
+ erts_proc_lc_require_lock((P), ERTS_PROC_LOCK_MAIN, \
+ __FILE__, __LINE__); \
+} while (0)
+# define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P) \
+do { \
+ if ((P)) \
+ erts_proc_lc_unrequire_lock((P), ERTS_PROC_LOCK_MAIN); \
} while (0)
-# define ERTS_SMP_REQ_PROC_MAIN_LOCK(P) \
- if ((P)) erts_proc_lc_require_lock((P), ERTS_PROC_LOCK_MAIN)
-# define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P) \
- if ((P)) erts_proc_lc_unrequire_lock((P), ERTS_PROC_LOCK_MAIN)
# else
# define ERTS_SMP_REQ_PROC_MAIN_LOCK(P)
# define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P)
@@ -96,10 +102,7 @@ do { \
do { \
int i_; \
int Arity_ = PC[-1]; \
- if (Arity_ > 0) { \
- CHECK_TERM(r(0)); \
- } \
- for (i_ = 1; i_ < Arity_; i_++) { \
+ for (i_ = 0; i_ < Arity_; i_++) { \
CHECK_TERM(x(i_)); \
} \
} while (0)
@@ -113,6 +116,9 @@ do { \
#define MAX(x, y) (((x) > (y)) ? (x) : (y))
#endif
+#define GET_BIF_MODULE(p) ((Eterm) (((Export *) p)->code[0]))
+#define GET_BIF_FUNCTION(p) ((Eterm) (((Export *) p)->code[1]))
+#define GET_BIF_ARITY(p) ((Eterm) (((Export *) p)->code[2]))
#define GET_BIF_ADDRESS(p) ((BifFunction) (((Export *) p)->code[4]))
#define TermWords(t) (((t) / (sizeof(BeamInstr)/sizeof(Eterm))) + !!((t) % (sizeof(BeamInstr)/sizeof(Eterm))))
@@ -133,7 +139,7 @@ do { \
/* We don't check the range if an ordinary switch is used */
#ifdef NO_JUMP_TABLE
-#define VALID_INSTR(IP) (0 <= (int)(IP) && ((int)(IP) < (NUMBER_OF_OPCODES*2+10)))
+#define VALID_INSTR(IP) ((UWord)(IP) < (NUMBER_OF_OPCODES*2+10))
#else
#define VALID_INSTR(IP) \
((SWord)LabelAddr(emulator_loop) <= (SWord)(IP) && \
@@ -148,25 +154,21 @@ do { \
ASSERT(VALID_INSTR(* (Eterm *)(ip))); \
I = (ip)
-#define FetchArgs(S1, S2) tmp_arg1 = (S1); tmp_arg2 = (S2)
+/*
+ * Register target (X or Y register).
+ */
+#define REG_TARGET(Target) (*(((Target) & 1) ? &yb(Target-1) : &xb(Target)))
/*
* Store a result into a register given a destination descriptor.
*/
-#define StoreResult(Result, DestDesc) \
- do { \
- Eterm stb_reg; \
- stb_reg = (DestDesc); \
- CHECK_TERM(Result); \
- switch (beam_reg_tag(stb_reg)) { \
- case R_REG_DEF: \
- r(0) = (Result); break; \
- case X_REG_DEF: \
- xb(x_reg_offset(stb_reg)) = (Result); break; \
- default: \
- yb(y_reg_offset(stb_reg)) = (Result); break; \
- } \
+#define StoreResult(Result, DestDesc) \
+ do { \
+ Eterm stb_reg; \
+ stb_reg = (DestDesc); \
+ CHECK_TERM(Result); \
+ REG_TARGET(stb_reg) = (Result); \
} while (0)
#define StoreSimpleDest(Src, Dest) Dest = (Src)
@@ -177,22 +179,16 @@ do { \
* be just before the next instruction.
*/
-#define StoreBifResult(Dst, Result) \
- do { \
- BeamInstr* stb_next; \
- Eterm stb_reg; \
- stb_reg = Arg(Dst); \
- I += (Dst) + 2; \
- stb_next = (BeamInstr *) *I; \
- CHECK_TERM(Result); \
- switch (beam_reg_tag(stb_reg)) { \
- case R_REG_DEF: \
- r(0) = (Result); Goto(stb_next); \
- case X_REG_DEF: \
- xb(x_reg_offset(stb_reg)) = (Result); Goto(stb_next); \
- default: \
- yb(y_reg_offset(stb_reg)) = (Result); Goto(stb_next); \
- } \
+#define StoreBifResult(Dst, Result) \
+ do { \
+ BeamInstr* stb_next; \
+ Eterm stb_reg; \
+ stb_reg = Arg(Dst); \
+ I += (Dst) + 2; \
+ stb_next = (BeamInstr *) *I; \
+ CHECK_TERM(Result); \
+ REG_TARGET(stb_reg) = (Result); \
+ Goto(stb_next); \
} while (0)
#define ClauseFail() goto jump_f
@@ -217,6 +213,7 @@ BeamInstr beam_continue_exit[1];
BeamInstr* em_call_error_handler;
BeamInstr* em_apply_bif;
+BeamInstr* em_call_nif;
/* NOTE These should be the only variables containing trace instructions.
@@ -238,10 +235,6 @@ BeamInstr beam_return_time_trace[1]; /* OpCode(i_return_time_trace) */
void** beam_ops;
#endif
-#ifndef ERTS_SMP /* Not supported with smp emulator */
-extern int count_instructions;
-#endif
-
#define SWAPIN \
HTOP = HEAP_TOP(c_p); \
E = c_p->stop
@@ -250,6 +243,14 @@ extern int count_instructions;
HEAP_TOP(c_p) = HTOP; \
c_p->stop = E
+#define HEAVY_SWAPIN \
+ SWAPIN; \
+ FCALLS = c_p->fcalls
+
+#define HEAVY_SWAPOUT \
+ SWAPOUT; \
+ c_p->fcalls = FCALLS
+
/*
* Use LIGHT_SWAPOUT when the called function
* will call HeapOnlyAlloc() (and never HAlloc()).
@@ -293,7 +294,7 @@ extern int count_instructions;
#define Ib(N) (N)
#define x(N) reg[N]
#define y(N) E[N]
-#define r(N) x##N
+#define r(N) x(N)
/*
* Makes sure that there are StackNeed + HeapNeed + 1 words available
@@ -309,12 +310,11 @@ extern int count_instructions;
needed = (StackNeed) + 1; \
if (E - HTOP < (needed + (HeapNeed))) { \
SWAPOUT; \
- reg[0] = r(0); \
PROCESS_MAIN_CHK_LOCKS(c_p); \
- FCALLS -= erts_garbage_collect(c_p, needed + (HeapNeed), reg, (M)); \
+ FCALLS -= erts_garbage_collect_nobump(c_p, needed + (HeapNeed), \
+ reg, (M), FCALLS); \
ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \
PROCESS_MAIN_CHK_LOCKS(c_p); \
- r(0) = reg[0]; \
SWAPIN; \
} \
E -= needed; \
@@ -363,12 +363,10 @@ extern int count_instructions;
unsigned need = (Nh); \
if ((E - HTOP < need) || (MSO(c_p).overhead + (VNh) >= BIN_VHEAP_SZ(c_p))) {\
SWAPOUT; \
- reg[0] = r(0); \
PROCESS_MAIN_CHK_LOCKS(c_p); \
- FCALLS -= erts_garbage_collect(c_p, need, reg, (Live)); \
+ FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live), FCALLS); \
ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \
PROCESS_MAIN_CHK_LOCKS(c_p); \
- r(0) = reg[0]; \
SWAPIN; \
} \
HEAP_SPACE_VERIFIED(need); \
@@ -386,12 +384,10 @@ extern int count_instructions;
unsigned need = (Nh); \
if (E - HTOP < need) { \
SWAPOUT; \
- reg[0] = r(0); \
PROCESS_MAIN_CHK_LOCKS(c_p); \
- FCALLS -= erts_garbage_collect(c_p, need, reg, (Live)); \
+ FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live), FCALLS); \
ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \
PROCESS_MAIN_CHK_LOCKS(c_p); \
- r(0) = reg[0]; \
SWAPIN; \
} \
HEAP_SPACE_VERIFIED(need); \
@@ -408,15 +404,11 @@ extern int count_instructions;
unsigned need = (Nh); \
if (E - HTOP < need) { \
SWAPOUT; \
- reg[0] = r(0); \
reg[Live] = Extra; \
PROCESS_MAIN_CHK_LOCKS(c_p); \
- FCALLS -= erts_garbage_collect(c_p, need, reg, (Live)+1); \
+ FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live)+1, FCALLS); \
ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \
PROCESS_MAIN_CHK_LOCKS(c_p); \
- if (Live > 0) { \
- r(0) = reg[0]; \
- } \
Extra = reg[Live]; \
SWAPIN; \
} \
@@ -438,10 +430,9 @@ extern int count_instructions;
#define MakeFun(FunP, NumFree) \
do { \
- SWAPOUT; \
- reg[0] = r(0); \
+ HEAVY_SWAPOUT; \
r(0) = new_fun(c_p, reg, (ErlFunEntry *) FunP, NumFree); \
- SWAPIN; \
+ HEAVY_SWAPIN; \
} while (0)
#define PutTuple(Dst, Arity) \
@@ -531,15 +522,19 @@ extern int count_instructions;
ASSERT(VALID_INSTR(Dst)); \
Goto(Dst)
-#define GetR(pos, tr) \
- do { \
- tr = Arg(pos); \
- switch (beam_reg_tag(tr)) { \
- case R_REG_DEF: tr = r(0); break; \
- case X_REG_DEF: tr = xb(x_reg_offset(tr)); break; \
- case Y_REG_DEF: ASSERT(y_reg_offset(tr) >= 1); tr = yb(y_reg_offset(tr)); break; \
- } \
- CHECK_TERM(tr); \
+#define GetR(pos, tr) \
+ do { \
+ tr = Arg(pos); \
+ switch (loader_tag(tr)) { \
+ case LOADER_X_REG: \
+ tr = x(loader_x_reg_index(tr)); \
+ break; \
+ case LOADER_Y_REG: \
+ ASSERT(loader_y_reg_index(tr) >= 1); \
+ tr = y(loader_y_reg_index(tr)); \
+ break; \
+ } \
+ CHECK_TERM(tr); \
} while (0)
#define GetArg1(N, Dst) GetR((N), Dst)
@@ -557,23 +552,93 @@ extern int count_instructions;
HTOP += 2; \
} while (0)
+#define Swap(R1, R2) \
+ do { \
+ Eterm V = R1; \
+ R1 = R2; \
+ R2 = V; \
+ } while (0)
+
+#define SwapTemp(R1, R2, Tmp) \
+ do { \
+ Eterm V = R1; \
+ R1 = R2; \
+ R2 = Tmp = V; \
+ } while (0)
+
#define Move(Src, Dst, Store) \
do { \
Eterm term = (Src); \
Store(term, Dst); \
} while (0)
-#define Move2(src1, dst1, src2, dst2) dst1 = (src1); dst2 = (src2)
+#define Move2Par(S1, D1, S2, D2) \
+ do { \
+ Eterm V1, V2; \
+ V1 = (S1); V2 = (S2); D1 = V1; D2 = V2; \
+ } while (0)
+
+#define MoveShift(Src, SD, D) \
+ do { \
+ Eterm V; \
+ V = Src; D = SD; SD = V; \
+ } while (0)
+
+#define MoveDup(Src, D1, D2) \
+ do { \
+ D1 = D2 = (Src); \
+ } while (0)
+
+#define Move3(S1, D1, S2, D2, S3, D3) D1 = (S1); D2 = (S2); D3 = (S3)
-#define MoveGenDest(src, dstp) \
- if ((dstp) == NULL) { r(0) = (src); } else { *(dstp) = src; }
+#define MoveWindow3(S1, S2, S3, D) \
+ do { \
+ Eterm xt0, xt1, xt2; \
+ Eterm *y = &D; \
+ xt0 = S1; \
+ xt1 = S2; \
+ xt2 = S3; \
+ y[0] = xt0; \
+ y[1] = xt1; \
+ y[2] = xt2; \
+ } while (0)
-#define MoveReturn(Src, Dest) \
- (Dest) = (Src); \
- I = c_p->cp; \
- ASSERT(VALID_INSTR(*c_p->cp)); \
- c_p->cp = 0; \
- CHECK_TERM(r(0)); \
+#define MoveWindow4(S1, S2, S3, S4, D) \
+ do { \
+ Eterm xt0, xt1, xt2, xt3; \
+ Eterm *y = &D; \
+ xt0 = S1; \
+ xt1 = S2; \
+ xt2 = S3; \
+ xt3 = S4; \
+ y[0] = xt0; \
+ y[1] = xt1; \
+ y[2] = xt2; \
+ y[3] = xt3; \
+ } while (0)
+
+#define MoveWindow5(S1, S2, S3, S4, S5, D) \
+ do { \
+ Eterm xt0, xt1, xt2, xt3, xt4; \
+ Eterm *y = &D; \
+ xt0 = S1; \
+ xt1 = S2; \
+ xt2 = S3; \
+ xt3 = S4; \
+ xt4 = S5; \
+ y[0] = xt0; \
+ y[1] = xt1; \
+ y[2] = xt2; \
+ y[3] = xt3; \
+ y[4] = xt4; \
+ } while (0)
+
+#define MoveReturn(Src) \
+ x(0) = (Src); \
+ I = c_p->cp; \
+ ASSERT(VALID_INSTR(*c_p->cp)); \
+ c_p->cp = 0; \
+ CHECK_TERM(r(0)); \
Goto(*I)
#define DeallocateReturn(Deallocate) \
@@ -585,26 +650,26 @@ extern int count_instructions;
Goto(*I); \
} while (0)
-#define MoveDeallocateReturn(Src, Dest, Deallocate) \
- (Dest) = (Src); \
+#define MoveDeallocateReturn(Src, Deallocate) \
+ x(0) = (Src); \
DeallocateReturn(Deallocate)
-#define MoveCall(Src, Dest, CallDest, Size) \
- (Dest) = (Src); \
+#define MoveCall(Src, CallDest, Size) \
+ x(0) = (Src); \
SET_CP(c_p, I+Size+1); \
- SET_I((BeamInstr *) CallDest); \
+ SET_I((BeamInstr *) CallDest); \
Dispatch();
-#define MoveCallLast(Src, Dest, CallDest, Deallocate) \
- (Dest) = (Src); \
- RESTORE_CP(E); \
- E = ADD_BYTE_OFFSET(E, (Deallocate)); \
- SET_I((BeamInstr *) CallDest); \
+#define MoveCallLast(Src, CallDest, Deallocate) \
+ x(0) = (Src); \
+ RESTORE_CP(E); \
+ E = ADD_BYTE_OFFSET(E, (Deallocate)); \
+ SET_I((BeamInstr *) CallDest); \
Dispatch();
-#define MoveCallOnly(Src, Dest, CallDest) \
- (Dest) = (Src); \
- SET_I((BeamInstr *) CallDest); \
+#define MoveCallOnly(Src, CallDest) \
+ x(0) = (Src); \
+ SET_I((BeamInstr *) CallDest); \
Dispatch();
#define MoveJump(Src) \
@@ -612,57 +677,69 @@ extern int count_instructions;
SET_I((BeamInstr *) Arg(0)); \
Goto(*I);
-#define GetList(Src, H, T) do { \
- Eterm* tmp_ptr = list_val(Src); \
- H = CAR(tmp_ptr); \
- T = CDR(tmp_ptr); } while (0)
-
-#define GetTupleElement(Src, Element, Dest) \
- do { \
- tmp_arg1 = (Eterm) COMPRESS_POINTER(((unsigned char *) tuple_val(Src)) + \
- (Element)); \
- (Dest) = (*(Eterm *) EXPAND_POINTER(tmp_arg1)); \
+#define GetList(Src, H, T) \
+ do { \
+ Eterm* tmp_ptr = list_val(Src); \
+ Eterm hd, tl; \
+ hd = CAR(tmp_ptr); \
+ tl = CDR(tmp_ptr); \
+ H = hd; T = tl; \
} while (0)
-#define ExtractNextElement(Dest) \
- tmp_arg1 += sizeof(Eterm); \
- (Dest) = (* (Eterm *) (((unsigned char *) EXPAND_POINTER(tmp_arg1))))
-
-#define ExtractNextElement2(Dest) \
- do { \
- Eterm* ene_dstp = &(Dest); \
- ene_dstp[0] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[1]; \
- ene_dstp[1] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[2]; \
- tmp_arg1 += sizeof(Eterm) + sizeof(Eterm); \
+#define GetTupleElement(Src, Element, Dest) \
+ do { \
+ Eterm* src; \
+ src = ADD_BYTE_OFFSET(tuple_val(Src), (Element)); \
+ (Dest) = *src; \
} while (0)
-#define ExtractNextElement3(Dest) \
- do { \
- Eterm* ene_dstp = &(Dest); \
- ene_dstp[0] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[1]; \
- ene_dstp[1] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[2]; \
- ene_dstp[2] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[3]; \
- tmp_arg1 += 3*sizeof(Eterm); \
+#define GetTupleElement2(Src, Element, Dest) \
+ do { \
+ Eterm* src; \
+ Eterm* dst; \
+ Eterm E1, E2; \
+ src = ADD_BYTE_OFFSET(tuple_val(Src), (Element)); \
+ dst = &(Dest); \
+ E1 = src[0]; \
+ E2 = src[1]; \
+ dst[0] = E1; \
+ dst[1] = E2; \
} while (0)
-#define ExtractNextElement4(Dest) \
- do { \
- Eterm* ene_dstp = &(Dest); \
- ene_dstp[0] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[1]; \
- ene_dstp[1] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[2]; \
- ene_dstp[2] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[3]; \
- ene_dstp[3] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[4]; \
- tmp_arg1 += 4*sizeof(Eterm); \
+#define GetTupleElement2Y(Src, Element, D1, D2) \
+ do { \
+ Eterm* src; \
+ Eterm E1, E2; \
+ src = ADD_BYTE_OFFSET(tuple_val(Src), (Element)); \
+ E1 = src[0]; \
+ E2 = src[1]; \
+ D1 = E1; \
+ D2 = E2; \
} while (0)
-#define ExtractElement(Element, Dest) \
- do { \
- tmp_arg1 += (Element); \
- (Dest) = (* (Eterm *) EXPAND_POINTER(tmp_arg1)); \
+#define GetTupleElement3(Src, Element, Dest) \
+ do { \
+ Eterm* src; \
+ Eterm* dst; \
+ Eterm E1, E2, E3; \
+ src = ADD_BYTE_OFFSET(tuple_val(Src), (Element)); \
+ dst = &(Dest); \
+ E1 = src[0]; \
+ E2 = src[1]; \
+ E3 = src[2]; \
+ dst[0] = E1; \
+ dst[1] = E2; \
+ dst[2] = E3; \
} while (0)
#define EqualImmed(X, Y, Action) if (X != Y) { Action; }
#define NotEqualImmed(X, Y, Action) if (X == Y) { Action; }
+#define EqualExact(X, Y, Action) if (!EQ(X,Y)) { Action; }
+#define NotEqualExact(X, Y, Action) if (EQ(X,Y)) { Action; }
+#define Equal(X, Y, Action) CMP_EQ_ACTION(X,Y,Action)
+#define NotEqual(X, Y, Action) CMP_NE_ACTION(X,Y,Action)
+#define IsLessThan(X, Y, Action) CMP_LT_ACTION(X,Y,Action)
+#define IsGreaterEqual(X, Y, Action) CMP_GE_ACTION(X,Y,Action)
#define IsFloat(Src, Fail) if (is_not_float(Src)) { Fail; }
@@ -686,20 +763,48 @@ extern int count_instructions;
if (is_not_list(Src)) { Fail; } \
A(Need, Alive)
-#define IsNonemptyListTestHeap(Src, Need, Alive, Fail) \
- if (is_not_list(Src)) { Fail; } \
+#define IsNonemptyListTestHeap(Need, Alive, Fail) \
+ if (is_not_list(x(0))) { Fail; } \
TestHeap(Need, Alive)
+#define IsNonemptyListGetList(Src, H, T, Fail) \
+ if (is_not_list(Src)) { \
+ Fail; \
+ } else { \
+ Eterm* tmp_ptr = list_val(Src); \
+ Eterm hd, tl; \
+ hd = CAR(tmp_ptr); \
+ tl = CDR(tmp_ptr); \
+ H = hd; T = tl; \
+ }
+
#define IsTuple(X, Action) if (is_not_tuple(X)) Action
-#define IsArity(Pointer, Arity, Fail) \
- if (*(Eterm *) \
- EXPAND_POINTER(tmp_arg1 = (Eterm) \
- COMPRESS_POINTER(tuple_val(Pointer))) != (Arity)) \
- { \
- Fail; \
+#define IsArity(Pointer, Arity, Fail) \
+ if (*tuple_val(Pointer) != (Arity)) { \
+ Fail; \
}
+#define IsMap(Src, Fail) if (!is_map(Src)) { Fail; }
+
+#define GetMapElement(Src, Key, Dst, Fail) \
+ do { \
+ Eterm _res = get_map_element(Src, Key); \
+ if (is_non_value(_res)) { \
+ Fail; \
+ } \
+ Dst = _res; \
+ } while (0)
+
+#define GetMapElementHash(Src, Key, Hx, Dst, Fail) \
+ do { \
+ Eterm _res = get_map_element_hash(Src, Key, Hx); \
+ if (is_non_value(_res)) { \
+ Fail; \
+ } \
+ Dst = _res; \
+ } while (0)
+
#define IsFunction(X, Action) \
do { \
if ( !(is_any_fun(X)) ) { \
@@ -714,15 +819,21 @@ extern int count_instructions;
} \
} while (0)
-#define IsTupleOfArity(Src, Arity, Fail) \
- do { \
- if (is_not_tuple(Src) || \
- *(Eterm *) \
- EXPAND_POINTER(tmp_arg1 = \
- (Eterm) COMPRESS_POINTER(tuple_val(Src))) != Arity) { \
- Fail; \
- } \
+#ifdef DEBUG
+#define IsTupleOfArity(Src, Arityval, Fail) \
+ do { \
+ if (!(is_tuple(Src) && *tuple_val(Src) == Arityval)) { \
+ Fail; \
+ } \
+ } while (0)
+#else
+#define IsTupleOfArity(Src, Arityval, Fail) \
+ do { \
+ if (!(is_boxed(Src) && *tuple_val(Src) == Arityval)) { \
+ Fail; \
+ } \
} while (0)
+#endif
#define IsBoolean(X, Fail) if ((X) != am_true && (X) != am_false) { Fail; }
@@ -732,7 +843,7 @@ extern int count_instructions;
#define IsBitstring(Src, Fail) \
if (is_not_binary(Src)) { Fail; }
-#if defined(ARCH_64) && !HALFWORD_HEAP
+#if defined(ARCH_64)
#define BsSafeMul(A, B, Fail, Target) \
do { Uint64 _res = (A) * (B); \
if (_res / B != A) { Fail; } \
@@ -749,6 +860,7 @@ extern int count_instructions;
#define BsGetFieldSize(Bits, Unit, Fail, Target) \
do { \
Sint _signed_size; Uint _uint_size; \
+ Uint temp_bits; \
if (is_small(Bits)) { \
_signed_size = signed_val(Bits); \
if (_signed_size < 0) { Fail; } \
@@ -763,6 +875,7 @@ extern int count_instructions;
#define BsGetUncheckedFieldSize(Bits, Unit, Fail, Target) \
do { \
Sint _signed_size; Uint _uint_size; \
+ Uint temp_bits; \
if (is_small(Bits)) { \
_signed_size = signed_val(Bits); \
if (_signed_size < 0) { Fail; } \
@@ -943,7 +1056,13 @@ static BeamInstr* apply_fun(Process* p, Eterm fun,
Eterm args, Eterm* reg) NOINLINE;
static Eterm new_fun(Process* p, Eterm* reg,
ErlFunEntry* fe, int num_free) NOINLINE;
-
+static Eterm new_map(Process* p, Eterm* reg, BeamInstr* I) NOINLINE;
+static Eterm update_map_assoc(Process* p, Eterm* reg,
+ Eterm map, BeamInstr* I) NOINLINE;
+static Eterm update_map_exact(Process* p, Eterm* reg,
+ Eterm map, BeamInstr* I) NOINLINE;
+static Eterm get_map_element(Eterm map, Eterm key);
+static Eterm get_map_element_hash(Eterm map, Eterm key, Uint32 hx);
/*
* Functions not directly called by process_main(). OK to inline.
@@ -968,23 +1087,17 @@ init_emulator(void)
*/
#if defined(__GNUC__) && defined(sparc) && !defined(DEBUG)
-# define REG_x0 asm("%l0")
# define REG_xregs asm("%l1")
# define REG_htop asm("%l2")
# define REG_stop asm("%l3")
# define REG_I asm("%l4")
# define REG_fcalls asm("%l5")
-# define REG_tmp_arg1 asm("%l6")
-# define REG_tmp_arg2 asm("%l7")
#else
-# define REG_x0
# define REG_xregs
# define REG_htop
# define REG_stop
# define REG_I
# define REG_fcalls
-# define REG_tmp_arg1
-# define REG_tmp_arg2
#endif
#ifdef USE_VM_PROBES
@@ -1059,17 +1172,52 @@ init_emulator(void)
DTRACE2(nif_return, process_name, mfa); \
}
+#define DTRACE_GLOBAL_CALL_FROM_EXPORT(p,e) \
+ do { \
+ if (DTRACE_ENABLED(global_function_entry)) { \
+ BeamInstr* fp = (BeamInstr *) (((Export *) (e))->addressv[erts_active_code_ix()]); \
+ DTRACE_GLOBAL_CALL((p), (Eterm)fp[-3], (Eterm)fp[-2], fp[-1]); \
+ } \
+ } while(0)
+
+#define DTRACE_RETURN_FROM_PC(p) \
+ do { \
+ BeamInstr* fp; \
+ if (DTRACE_ENABLED(function_return) && (fp = find_function_from_pc((p)->cp))) { \
+ DTRACE_RETURN((p), (Eterm)fp[0], (Eterm)fp[1], (Uint)fp[2]); \
+ } \
+ } while(0)
+
#else /* USE_VM_PROBES */
+#define DTRACE_LOCAL_CALL(p, m, f, a) do {} while (0)
+#define DTRACE_GLOBAL_CALL(p, m, f, a) do {} while (0)
+#define DTRACE_GLOBAL_CALL_FROM_EXPORT(p, e) do {} while (0)
+#define DTRACE_RETURN(p, m, f, a) do {} while (0)
+#define DTRACE_RETURN_FROM_PC(p) do {} while (0)
+#define DTRACE_BIF_ENTRY(p, m, f, a) do {} while (0)
+#define DTRACE_BIF_RETURN(p, m, f, a) do {} while (0)
+#define DTRACE_NIF_ENTRY(p, m, f, a) do {} while (0)
+#define DTRACE_NIF_RETURN(p, m, f, a) do {} while (0)
+#endif /* USE_VM_PROBES */
-#define DTRACE_LOCAL_CALL(p, m, f, a) do {} while (0)
-#define DTRACE_GLOBAL_CALL(p, m, f, a) do {} while (0)
-#define DTRACE_RETURN(p, m, f, a) do {} while (0)
-#define DTRACE_BIF_ENTRY(p, m, f, a) do {} while (0)
-#define DTRACE_BIF_RETURN(p, m, f, a) do {} while (0)
-#define DTRACE_NIF_ENTRY(p, m, f, a) do {} while (0)
-#define DTRACE_NIF_RETURN(p, m, f, a) do {} while (0)
+#ifdef DEBUG
+#define ERTS_DBG_CHK_REDS(P, FC) \
+ do { \
+ if (ERTS_PROC_GET_SAVED_CALLS_BUF((P))) { \
+ ASSERT(FC <= 0); \
+ ASSERT(erts_proc_sched_data(c_p)->virtual_reds \
+ <= 0 - (FC)); \
+ } \
+ else { \
+ ASSERT(FC <= CONTEXT_REDS); \
+ ASSERT(erts_proc_sched_data(c_p)->virtual_reds \
+ <= CONTEXT_REDS - (FC)); \
+ } \
+} while (0)
+#else
+#define ERTS_DBG_CHK_REDS(P, FC)
+#endif
-#endif /* USE_VM_PROBES */
/*
* process_main() is called twice:
@@ -1086,11 +1234,6 @@ void process_main(void)
ERTS_DECLARE_DUMMY(Eterm pid);
#endif
- /*
- * X register zero; also called r(0)
- */
- register Eterm x0 REG_x0 = NIL;
-
/* Pointer to X registers: x(1)..x(N); reg[0] is used when doing GC,
* in all other cases x0 is used.
*/
@@ -1118,17 +1261,6 @@ void process_main(void)
register Sint FCALLS REG_fcalls = 0;
/*
- * Temporaries used for picking up arguments for instructions.
- */
- register Eterm tmp_arg1 REG_tmp_arg1 = NIL;
- register Eterm tmp_arg2 REG_tmp_arg2 = NIL;
-#if HEAP_ON_C_STACK
- Eterm tmp_big[2]; /* Temporary buffer for small bignums if HEAP_ON_C_STACK. */
-#else
- Eterm *tmp_big; /* Temporary buffer for small bignums if !HEAP_ON_C_STACK. */
-#endif
-
- /*
* X registers and floating point registers are located in
* scheduler specific data.
*/
@@ -1139,24 +1271,23 @@ void process_main(void)
*/
int neg_o_reds = 0;
- Eterm (*arith_func)(Process* p, Eterm* reg, Uint live);
-
-#ifndef NO_JUMP_TABLE
- static void* opcodes[] = { DEFINE_OPCODES };
#ifdef ERTS_OPCODE_COUNTER_SUPPORT
static void* counting_opcodes[] = { DEFINE_COUNTING_OPCODES };
-#endif
+#else
+#ifndef NO_JUMP_TABLE
+ static void* opcodes[] = { DEFINE_OPCODES };
#else
int Go;
#endif
-
- Uint temp_bits; /* Temporary used by BsSkipBits2 & BsGetInteger2 */
+#endif
Eterm pt_arity; /* Used by do_put_tuple */
Uint64 start_time = 0; /* Monitor long schedule */
BeamInstr* start_time_i = NULL;
+ ERTS_MSACC_DECLARE_CACHE_X() /* a cached value of the tsd pointer for msacc */
+
ERL_BITS_DECLARE_STATEP; /* Has to be last declaration */
@@ -1169,15 +1300,25 @@ void process_main(void)
* c_p->arg_reg before calling the scheduler.
*/
if (!init_done) {
+ /* This should only be reached during the init phase when only the main
+ * process is running. I.e. there is no race for init_done.
+ */
init_done = 1;
goto init_emulator;
}
+
c_p = NULL;
reds_used = 0;
+
goto do_schedule1;
do_schedule:
- reds_used = REDS_IN(c_p) - FCALLS;
+ ASSERT(c_p->debug_reds_in == REDS_IN(c_p));
+ if (!ERTS_PROC_GET_SAVED_CALLS_BUF(c_p))
+ reds_used = REDS_IN(c_p) - FCALLS;
+ else
+ reds_used = REDS_IN(c_p) - (CONTEXT_REDS + FCALLS);
+ ASSERT(reds_used >= 0);
do_schedule1:
if (start_time != 0) {
@@ -1191,11 +1332,9 @@ void process_main(void)
PROCESS_MAIN_CHK_LOCKS(c_p);
ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
-#if HALFWORD_HEAP
- ASSERT(erts_get_scheduler_data()->num_tmp_heap_used == 0);
-#endif
ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
- c_p = schedule(c_p, reds_used);
+ c_p = erts_schedule(NULL, c_p, reds_used);
+ ASSERT(!(c_p->flags & F_HIPE_MODE));
ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
start_time = 0;
#ifdef DEBUG
@@ -1204,16 +1343,15 @@ void process_main(void)
ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
PROCESS_MAIN_CHK_LOCKS(c_p);
+ ERTS_MSACC_UPDATE_CACHE_X();
+
if (erts_system_monitor_long_schedule != 0) {
start_time = erts_timestamp_millis();
start_time_i = c_p->i;
}
- reg = ERTS_PROC_GET_SCHDATA(c_p)->x_reg_array;
- freg = ERTS_PROC_GET_SCHDATA(c_p)->f_reg_array;
-#if !HEAP_ON_C_STACK
- tmp_big = ERTS_PROC_GET_SCHDATA(c_p)->beam_emu_tmp_heap;
-#endif
+ reg = erts_proc_sched_data(c_p)->x_reg_array;
+ freg = erts_proc_sched_data(c_p)->f_reg_array;
ERL_BITS_RELOAD_STATEP(c_p);
{
int reds;
@@ -1222,7 +1360,7 @@ void process_main(void)
int i;
argp = c_p->arg_reg;
- for (i = c_p->arity - 1; i > 0; i--) {
+ for (i = c_p->arity - 1; i >= 0; i--) {
reg[i] = argp[i];
CHECK_TERM(reg[i]);
}
@@ -1235,23 +1373,22 @@ void process_main(void)
SET_I(c_p->i);
- reds = c_p->fcalls;
- if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)
- && (ERTS_TRACE_FLAGS(c_p) & F_SENSITIVE) == 0) {
- neg_o_reds = -reds;
- FCALLS = REDS_IN(c_p) = 0;
+ REDS_IN(c_p) = reds = c_p->fcalls;
+#ifdef DEBUG
+ c_p->debug_reds_in = reds;
+#endif
+
+ if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) {
+ neg_o_reds = -CONTEXT_REDS;
+ FCALLS = neg_o_reds + reds;
} else {
neg_o_reds = 0;
- FCALLS = REDS_IN(c_p) = reds;
+ FCALLS = reds;
}
+ ERTS_DBG_CHK_REDS(c_p, FCALLS);
+
next = (BeamInstr *) *I;
- r(0) = c_p->arg_reg[0];
-#ifdef HARDDEBUG
- if (c_p->arity > 0) {
- CHECK_TERM(r(0));
- }
-#endif
SWAPIN;
ASSERT(VALID_INSTR(next));
@@ -1270,7 +1407,7 @@ void process_main(void)
(Eterm)fptr[1], (Uint)fptr[2],
NULL, fun_buf);
} else {
- erts_snprintf(fun_buf, sizeof(fun_buf),
+ erts_snprintf(fun_buf, sizeof(DTRACE_CHARBUF_NAME(fun_buf)),
"<unknown/%p>", next);
}
}
@@ -1290,26 +1427,24 @@ void process_main(void)
#endif
#include "beam_hot.h"
-#define STORE_ARITH_RESULT(res) StoreBifResult(2, (res));
-#define ARITH_FUNC(name) erts_gc_##name
-
{
Eterm increment_reg_val;
Eterm increment_val;
Uint live;
Eterm result;
- OpCase(i_increment_yIId):
- increment_reg_val = yb(Arg(0));
+ OpCase(i_increment_rIId):
+ increment_reg_val = x(0);
+ I--;
goto do_increment;
OpCase(i_increment_xIId):
increment_reg_val = xb(Arg(0));
goto do_increment;
- OpCase(i_increment_rIId):
- increment_reg_val = r(0);
- I--;
+ OpCase(i_increment_yIId):
+ increment_reg_val = yb(Arg(0));
+ goto do_increment;
do_increment:
increment_val = Arg(1);
@@ -1318,137 +1453,133 @@ void process_main(void)
ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i));
if (MY_IS_SSMALL(i)) {
result = make_small(i);
- store_result:
StoreBifResult(3, result);
}
}
live = Arg(2);
- SWAPOUT;
- reg[0] = r(0);
+ HEAVY_SWAPOUT;
reg[live] = increment_reg_val;
reg[live+1] = make_small(increment_val);
result = erts_gc_mixed_plus(c_p, reg, live);
- r(0) = reg[0];
- SWAPIN;
+ HEAVY_SWAPIN;
ERTS_HOLE_CHECK(c_p);
if (is_value(result)) {
- goto store_result;
+ StoreBifResult(3, result);
}
ASSERT(c_p->freason != BADMATCH || is_value(c_p->fvalue));
goto find_func_info;
}
-
- OpCase(i_plus_jId):
+
+#define DO_OUTLINED_ARITH_2(name, Op1, Op2) \
+ do { \
+ Eterm result; \
+ Uint live = Arg(1); \
+ \
+ HEAVY_SWAPOUT; \
+ reg[live] = Op1; \
+ reg[live+1] = Op2; \
+ result = erts_gc_##name(c_p, reg, live); \
+ HEAVY_SWAPIN; \
+ ERTS_HOLE_CHECK(c_p); \
+ if (is_value(result)) { \
+ StoreBifResult(4, result); \
+ } \
+ goto lb_Cl_error; \
+ } while (0)
+
{
+ Eterm PlusOp1, PlusOp2;
Eterm result;
- if (is_both_small(tmp_arg1, tmp_arg2)) {
- Sint i = signed_val(tmp_arg1) + signed_val(tmp_arg2);
+ OpCase(i_plus_jIxxd):
+ PlusOp1 = xb(Arg(2));
+ PlusOp2 = xb(Arg(3));
+ goto do_plus;
+
+ OpCase(i_plus_jIxyd):
+ PlusOp1 = xb(Arg(2));
+ PlusOp2 = yb(Arg(3));
+ goto do_plus;
+
+ OpCase(i_plus_jIssd):
+ GetArg2(2, PlusOp1, PlusOp2);
+ goto do_plus;
+
+ do_plus:
+ if (is_both_small(PlusOp1, PlusOp2)) {
+ Sint i = signed_val(PlusOp1) + signed_val(PlusOp2);
ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i));
if (MY_IS_SSMALL(i)) {
result = make_small(i);
- STORE_ARITH_RESULT(result);
+ StoreBifResult(4, result);
}
-
}
- arith_func = ARITH_FUNC(mixed_plus);
- goto do_big_arith2;
+ DO_OUTLINED_ARITH_2(mixed_plus, PlusOp1, PlusOp2);
}
- OpCase(i_minus_jId):
{
+ Eterm MinusOp1, MinusOp2;
Eterm result;
- if (is_both_small(tmp_arg1, tmp_arg2)) {
- Sint i = signed_val(tmp_arg1) - signed_val(tmp_arg2);
+ OpCase(i_minus_jIxxd):
+ MinusOp1 = xb(Arg(2));
+ MinusOp2 = xb(Arg(3));
+ goto do_minus;
+
+ OpCase(i_minus_jIssd):
+ GetArg2(2, MinusOp1, MinusOp2);
+ goto do_minus;
+
+ do_minus:
+ if (is_both_small(MinusOp1, MinusOp2)) {
+ Sint i = signed_val(MinusOp1) - signed_val(MinusOp2);
ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i));
if (MY_IS_SSMALL(i)) {
result = make_small(i);
- STORE_ARITH_RESULT(result);
+ StoreBifResult(4, result);
}
}
- arith_func = ARITH_FUNC(mixed_minus);
- goto do_big_arith2;
+ DO_OUTLINED_ARITH_2(mixed_minus, MinusOp1, MinusOp2);
}
- OpCase(i_is_lt_f):
- if (CMP_GE(tmp_arg1, tmp_arg2)) {
- ClauseFail();
- }
- Next(1);
-
- OpCase(i_is_ge_f):
- if (CMP_LT(tmp_arg1, tmp_arg2)) {
- ClauseFail();
- }
- Next(1);
-
- OpCase(i_is_eq_f):
- if (CMP_NE(tmp_arg1, tmp_arg2)) {
- ClauseFail();
- }
- Next(1);
-
- OpCase(i_is_ne_f):
- if (CMP_EQ(tmp_arg1, tmp_arg2)) {
- ClauseFail();
- }
- Next(1);
-
- OpCase(i_is_eq_exact_f):
- if (!EQ(tmp_arg1, tmp_arg2)) {
- ClauseFail();
- }
- Next(1);
-
{
Eterm is_eq_exact_lit_val;
- OpCase(i_is_eq_exact_literal_xfc):
- is_eq_exact_lit_val = xb(Arg(0));
- I++;
+ OpCase(i_is_eq_exact_literal_fxc):
+ is_eq_exact_lit_val = xb(Arg(1));
goto do_is_eq_exact_literal;
- OpCase(i_is_eq_exact_literal_yfc):
- is_eq_exact_lit_val = yb(Arg(0));
- I++;
+ OpCase(i_is_eq_exact_literal_fyc):
+ is_eq_exact_lit_val = yb(Arg(1));
goto do_is_eq_exact_literal;
- OpCase(i_is_eq_exact_literal_rfc):
- is_eq_exact_lit_val = r(0);
-
do_is_eq_exact_literal:
- if (!eq(Arg(1), is_eq_exact_lit_val)) {
+ if (!eq(Arg(2), is_eq_exact_lit_val)) {
ClauseFail();
}
- Next(2);
+ Next(3);
}
{
Eterm is_ne_exact_lit_val;
- OpCase(i_is_ne_exact_literal_xfc):
- is_ne_exact_lit_val = xb(Arg(0));
- I++;
+ OpCase(i_is_ne_exact_literal_fxc):
+ is_ne_exact_lit_val = xb(Arg(1));
goto do_is_ne_exact_literal;
- OpCase(i_is_ne_exact_literal_yfc):
- is_ne_exact_lit_val = yb(Arg(0));
- I++;
+ OpCase(i_is_ne_exact_literal_fyc):
+ is_ne_exact_lit_val = yb(Arg(1));
goto do_is_ne_exact_literal;
- OpCase(i_is_ne_exact_literal_rfc):
- is_ne_exact_lit_val = r(0);
-
do_is_ne_exact_literal:
- if (eq(Arg(1), is_ne_exact_lit_val)) {
+ if (eq(Arg(2), is_ne_exact_lit_val)) {
ClauseFail();
}
- Next(2);
+ Next(3);
}
- OpCase(i_move_call_only_fcr): {
+ OpCase(i_move_call_only_fc): {
r(0) = Arg(1);
}
/* FALL THROUGH */
@@ -1458,7 +1589,7 @@ void process_main(void)
Dispatch();
}
- OpCase(i_move_call_last_fPcr): {
+ OpCase(i_move_call_last_fPc): {
r(0) = Arg(2);
}
/* FALL THROUGH */
@@ -1470,7 +1601,7 @@ void process_main(void)
Dispatch();
}
- OpCase(i_move_call_crf): {
+ OpCase(i_move_call_cf): {
r(0) = Arg(0);
I++;
}
@@ -1482,7 +1613,7 @@ void process_main(void)
Dispatch();
}
- OpCase(i_move_call_ext_last_ePcr): {
+ OpCase(i_move_call_ext_last_ePc): {
r(0) = Arg(2);
}
/* FALL THROUGH */
@@ -1495,40 +1626,25 @@ void process_main(void)
* is not loaded, it points to code which will invoke the error handler
* (see lb_call_error_handler below).
*/
-#ifdef USE_VM_CALL_PROBES
- if (DTRACE_ENABLED(global_function_entry)) {
- BeamInstr* fp = (BeamInstr *) (((Export *) Arg(0))->addressv[erts_active_code_ix()]);
- DTRACE_GLOBAL_CALL(c_p, (Eterm)fp[-3], (Eterm)fp[-2], fp[-1]);
- }
-#endif
+ DTRACE_GLOBAL_CALL_FROM_EXPORT(c_p, Arg(0));
Dispatchx();
- OpCase(i_move_call_ext_cre): {
+ OpCase(i_move_call_ext_ce): {
r(0) = Arg(0);
I++;
}
/* FALL THROUGH */
OpCase(i_call_ext_e):
SET_CP(c_p, I+2);
-#ifdef USE_VM_CALL_PROBES
- if (DTRACE_ENABLED(global_function_entry)) {
- BeamInstr* fp = (BeamInstr *) (((Export *) Arg(0))->addressv[erts_active_code_ix()]);
- DTRACE_GLOBAL_CALL(c_p, (Eterm)fp[-3], (Eterm)fp[-2], fp[-1]);
- }
-#endif
+ DTRACE_GLOBAL_CALL_FROM_EXPORT(c_p, Arg(0));
Dispatchx();
- OpCase(i_move_call_ext_only_ecr): {
+ OpCase(i_move_call_ext_only_ec): {
r(0) = Arg(1);
}
/* FALL THROUGH */
OpCase(i_call_ext_only_e):
-#ifdef USE_VM_CALL_PROBES
- if (DTRACE_ENABLED(global_function_entry)) {
- BeamInstr* fp = (BeamInstr *) (((Export *) Arg(0))->addressv[erts_active_code_ix()]);
- DTRACE_GLOBAL_CALL(c_p, (Eterm)fp[-3], (Eterm)fp[-2], fp[-1]);
- }
-#endif
+ DTRACE_GLOBAL_CALL_FROM_EXPORT(c_p, Arg(0));
Dispatchx();
OpCase(init_y): {
@@ -1562,18 +1678,9 @@ void process_main(void)
Next(1);
}
-
OpCase(return): {
-#ifdef USE_VM_CALL_PROBES
- BeamInstr* fptr;
-#endif
SET_I(c_p->cp);
-
-#ifdef USE_VM_CALL_PROBES
- if (DTRACE_ENABLED(function_return) && (fptr = find_function_from_pc(c_p->cp))) {
- DTRACE_RETURN(c_p, (Eterm)fptr[0], (Eterm)fptr[1], (Uint)fptr[2]);
- }
-#endif
+ DTRACE_RETURN_FROM_PC(c_p);
/*
* We must clear the CP to make sure that a stale value do not
* create a false module dependcy preventing code upgrading.
@@ -1596,19 +1703,21 @@ void process_main(void)
BeamInstr *next;
Eterm result;
+ if (!(FCALLS > 0 || FCALLS > neg_o_reds)) {
+ /* If we have run out of reductions, we do a context
+ switch before calling the bif */
+ c_p->arity = 2;
+ c_p->current = NULL;
+ goto context_switch3;
+ }
+
PRE_BIF_SWAPOUT(c_p);
c_p->fcalls = FCALLS - 1;
- reg[0] = r(0);
result = erl_send(c_p, r(0), x(1));
PreFetch(0, next);
ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
PROCESS_MAIN_CHK_LOCKS(c_p);
- if (c_p->mbuf || MSO(c_p).overhead >= BIN_VHEAP_SZ(c_p)) {
- result = erts_gc_after_bif_call(c_p, result, reg, 2);
- r(0) = reg[0];
- E = c_p->stop;
- }
HTOP = HEAP_TOP(c_p);
FCALLS = c_p->fcalls;
if (is_value(result)) {
@@ -1619,7 +1728,6 @@ void process_main(void)
SET_CP(c_p, I+1);
SET_I(c_p->i);
SWAPIN;
- r(0) = reg[0];
Dispatch();
}
goto find_func_info;
@@ -1629,29 +1737,23 @@ void process_main(void)
Eterm element_index;
Eterm element_tuple;
- OpCase(i_element_xjsd):
- element_tuple = xb(Arg(0));
- I++;
+ OpCase(i_element_jxsd):
+ element_tuple = xb(Arg(1));
goto do_element;
- OpCase(i_element_yjsd):
- element_tuple = yb(Arg(0));
- I++;
+ OpCase(i_element_jysd):
+ element_tuple = yb(Arg(1));
goto do_element;
- OpCase(i_element_rjsd):
- element_tuple = r(0);
- /* Fall through */
-
do_element:
- GetArg1(1, element_index);
+ GetArg1(2, element_index);
if (is_small(element_index) && is_tuple(element_tuple)) {
Eterm* tp = tuple_val(element_tuple);
if ((signed_val(element_index) >= 1) &&
(signed_val(element_index) <= arityval(*tp))) {
Eterm result = tp[signed_val(element_index)];
- StoreBifResult(2, result);
+ StoreBifResult(3, result);
}
}
}
@@ -1665,29 +1767,24 @@ void process_main(void)
{
Eterm fast_element_tuple;
- OpCase(i_fast_element_rjId):
- fast_element_tuple = r(0);
+ OpCase(i_fast_element_jxId):
+ fast_element_tuple = xb(Arg(1));
+ goto do_fast_element;
+
+ OpCase(i_fast_element_jyId):
+ fast_element_tuple = yb(Arg(1));
+ goto do_fast_element;
do_fast_element:
if (is_tuple(fast_element_tuple)) {
Eterm* tp = tuple_val(fast_element_tuple);
- Eterm pos = Arg(1); /* Untagged integer >= 1 */
+ Eterm pos = Arg(2); /* Untagged integer >= 1 */
if (pos <= arityval(*tp)) {
Eterm result = tp[pos];
- StoreBifResult(2, result);
+ StoreBifResult(3, result);
}
}
goto badarg;
-
- OpCase(i_fast_element_xjId):
- fast_element_tuple = xb(Arg(0));
- I++;
- goto do_fast_element;
-
- OpCase(i_fast_element_yjId):
- fast_element_tuple = yb(Arg(0));
- I++;
- goto do_fast_element;
}
OpCase(catch_yf):
@@ -1708,11 +1805,10 @@ void process_main(void)
SWAPIN;
}
/* only x(2) is included in the rootset here */
- if (E - HTOP < 3 || c_p->mbuf) { /* Force GC in case add_stacktrace()
- * created heap fragments */
+ if (E - HTOP < 3) {
SWAPOUT;
PROCESS_MAIN_CHK_LOCKS(c_p);
- FCALLS -= erts_garbage_collect(c_p, 3, reg+2, 1);
+ FCALLS -= erts_garbage_collect_nobump(c_p, 3, reg+2, 1, FCALLS);
ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
PROCESS_MAIN_CHK_LOCKS(c_p);
SWAPIN;
@@ -1793,10 +1889,18 @@ void process_main(void)
* Pick up the next message and place it in x(0).
* If no message, jump to a wait or wait_timeout instruction.
*/
- OpCase(i_loop_rec_fr):
+ OpCase(i_loop_rec_f):
{
BeamInstr *next;
- ErlMessage* msgp;
+ ErtsMessage* msgp;
+
+ /*
+ * We need to disable GC while matching messages
+ * in the queue. This since messages with data outside
+ * the heap will be corrupted by a GC.
+ */
+ ASSERT(!(c_p->flags & F_DELAY_GC));
+ c_p->flags |= F_DELAY_GC;
loop_rec__:
@@ -1811,6 +1915,7 @@ void process_main(void)
if (ERTS_PROC_PENDING_EXIT(c_p)) {
erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
SWAPOUT;
+ c_p->flags &= ~F_DELAY_GC;
goto do_schedule; /* Will be rescheduled for exit */
}
ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p);
@@ -1820,32 +1925,27 @@ void process_main(void)
else
#endif
{
+ c_p->flags &= ~F_DELAY_GC;
SET_I((BeamInstr *) Arg(0));
Goto(*I); /* Jump to a wait or wait_timeout instruction */
}
}
- ErtsMoveMsgAttachmentIntoProc(msgp, c_p, E, HTOP, FCALLS,
- {
- SWAPOUT;
- reg[0] = r(0);
- PROCESS_MAIN_CHK_LOCKS(c_p);
- },
- {
- ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
- PROCESS_MAIN_CHK_LOCKS(c_p);
- r(0) = reg[0];
- SWAPIN;
- });
if (is_non_value(ERL_MESSAGE_TERM(msgp))) {
- /*
- * A corrupt distribution message that we weren't able to decode;
- * remove it...
- */
- ASSERT(!msgp->data.attached);
- /* TODO: Add DTrace probe for this bad message situation? */
- UNLINK_MESSAGE(c_p, msgp);
- free_message(msgp);
- goto loop_rec__;
+ SWAPOUT; /* erts_decode_dist_message() may write to heap... */
+ if (!erts_decode_dist_message(c_p, ERTS_PROC_LOCK_MAIN, msgp, 0)) {
+ /*
+ * A corrupt distribution message that we weren't able to decode;
+ * remove it...
+ */
+ /* No swapin should be needed */
+ ASSERT(HTOP == c_p->htop && E == c_p->stop);
+ /* TODO: Add DTrace probe for this bad message situation? */
+ UNLINK_MESSAGE(c_p, msgp);
+ msgp->next = NULL;
+ erts_cleanup_messages(msgp);
+ goto loop_rec__;
+ }
+ SWAPIN;
}
PreFetch(1, next);
r(0) = ERL_MESSAGE_TERM(msgp);
@@ -1857,8 +1957,7 @@ void process_main(void)
*/
OpCase(remove_message): {
BeamInstr *next;
- ErlMessage* msgp;
-
+ ErtsMessage* msgp;
PROCESS_MAIN_CHK_LOCKS(c_p);
PreFetch(0, next);
@@ -1872,20 +1971,7 @@ void process_main(void)
if (DT_UTAG(c_p) != NIL) {
if (DT_UTAG_FLAGS(c_p) & DT_UTAG_PERMANENT) {
SEQ_TRACE_TOKEN(c_p) = am_have_dt_utag;
-#ifdef DTRACE_TAG_HARDDEBUG
- if (DT_UTAG_FLAGS(c_p) & DT_UTAG_SPREADING)
- erts_fprintf(stderr,
- "Dtrace -> (%T) stop spreading "
- "tag %T with message %T\r\n",
- c_p->common.id,DT_UTAG(c_p),ERL_MESSAGE_TERM(msgp));
-#endif
} else {
-#ifdef DTRACE_TAG_HARDDEBUG
- erts_fprintf(stderr,
- "Dtrace -> (%T) kill tag %T with "
- "message %T\r\n",
- c_p->common.id,DT_UTAG(c_p),ERL_MESSAGE_TERM(msgp));
-#endif
DT_UTAG(c_p) = NIL;
SEQ_TRACE_TOKEN(c_p) = NIL;
}
@@ -1905,12 +1991,6 @@ void process_main(void)
DT_UTAG(c_p) = ERL_MESSAGE_DT_UTAG(msgp);
}
DT_UTAG_FLAGS(c_p) |= DT_UTAG_SPREADING;
-#ifdef DTRACE_TAG_HARDDEBUG
- erts_fprintf(stderr,
- "Dtrace -> (%T) receive tag (%T) "
- "with message %T\r\n",
- c_p->common.id, DT_UTAG(c_p), ERL_MESSAGE_TERM(msgp));
-#endif
} else {
#endif
ASSERT(is_tuple(SEQ_TRACE_TOKEN(c_p)));
@@ -1940,7 +2020,7 @@ void process_main(void)
dtrace_proc_str(c_p, receiver_name);
token2 = SEQ_TRACE_TOKEN(c_p);
- if (token2 != NIL && token2 != am_have_dt_utag) {
+ if (have_seqtrace(token2)) {
tok_label = signed_val(SEQ_TRACE_T_LABEL(token2));
tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token2));
tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token2));
@@ -1953,11 +2033,23 @@ void process_main(void)
UNLINK_MESSAGE(c_p, msgp);
JOIN_MESSAGE(c_p);
CANCEL_TIMER(c_p);
- free_message(msgp);
+
+ erts_save_message_in_proc(c_p, msgp);
+ c_p->flags &= ~F_DELAY_GC;
+
+ if (ERTS_IS_GC_DESIRED_INTERNAL(c_p, HTOP, E)) {
+ /*
+ * We want to GC soon but we leave a few
+ * reductions giving the message some time
+ * to turn into garbage.
+ */
+ ERTS_VBUMP_LEAVE_REDS_INTERNAL(c_p, 5, FCALLS);
+ }
+
+ ERTS_DBG_CHK_REDS(c_p, FCALLS);
ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
PROCESS_MAIN_CHK_LOCKS(c_p);
-
NextPF(0, next);
}
@@ -1966,9 +2058,22 @@ void process_main(void)
* message didn't match), then jump to the loop_rec instruction.
*/
OpCase(loop_rec_end_f): {
+
+ ASSERT(c_p->flags & F_DELAY_GC);
+
SET_I((BeamInstr *) Arg(0));
SAVE_MESSAGE(c_p);
- goto loop_rec__;
+ if (FCALLS > 0 || FCALLS > neg_o_reds) {
+ FCALLS--;
+ goto loop_rec__;
+ }
+
+ c_p->flags &= ~F_DELAY_GC;
+ c_p->i = I;
+ SWAPOUT;
+ c_p->arity = 0;
+ c_p->current = NULL;
+ goto do_schedule;
}
/*
* Prepare to wait for a message or a timeout, whichever occurs first.
@@ -1996,44 +2101,30 @@ void process_main(void)
}
GetArg1(1, timeout_value);
if (timeout_value != make_small(0)) {
-#if !defined(ARCH_64) || HALFWORD_HEAP
- Uint time_val;
-#endif
- if (is_small(timeout_value) && signed_val(timeout_value) > 0 &&
-#if defined(ARCH_64) && !HALFWORD_HEAP
- ((unsigned_val(timeout_value) >> 32) == 0)
-#else
- 1
-#endif
- ) {
- /*
- * The timer routiner will set c_p->i to the value in
- * c_p->def_arg_reg[0]. Note that it is safe to use this
- * location because there are no living x registers in
- * a receive statement.
- * Note that for the halfword emulator, the two first elements
- * of the array are used.
- */
- BeamInstr** pi = (BeamInstr**) c_p->def_arg_reg;
- *pi = I+3;
- set_timer(c_p, unsigned_val(timeout_value));
- } else if (timeout_value == am_infinity) {
+ if (timeout_value == am_infinity)
c_p->flags |= F_TIMO;
-#if !defined(ARCH_64) || HALFWORD_HEAP
- } else if (term_to_Uint(timeout_value, &time_val)) {
- BeamInstr** pi = (BeamInstr**) c_p->def_arg_reg;
- *pi = I+3;
- set_timer(c_p, time_val);
-#endif
- } else { /* Wrong time */
- OpCase(i_wait_error_locked): {
- erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
- /* Fall through */
+ else {
+ int tres = erts_set_proc_timer_term(c_p, timeout_value);
+ if (tres == 0) {
+ /*
+ * The timer routiner will set c_p->i to the value in
+ * c_p->def_arg_reg[0]. Note that it is safe to use this
+ * location because there are no living x registers in
+ * a receive statement.
+ */
+ BeamInstr** pi = (BeamInstr**) c_p->def_arg_reg;
+ *pi = I+3;
}
+ else { /* Wrong time */
+ OpCase(i_wait_error_locked): {
+ erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
+ /* Fall through */
+ }
OpCase(i_wait_error): {
- c_p->freason = EXC_TIMEOUT_VALUE;
- goto find_func_info;
+ c_p->freason = EXC_TIMEOUT_VALUE;
+ goto find_func_info;
+ }
}
}
@@ -2051,6 +2142,22 @@ void process_main(void)
OpCase(wait_f):
wait2: {
+#ifndef ERTS_SMP
+ if (ERTS_PROC_IS_EXITING(c_p)) {
+ /*
+ * I non smp case:
+ *
+ * Currently executing process might be sent an exit
+ * signal if it is traced by a port that it also is
+ * linked to, and the port terminates during the
+ * trace. In this case we do *not* want to clear
+ * the active flag, which will make the process hang
+ * in limbo forever.
+ */
+ SWAPOUT;
+ goto do_schedule;
+ }
+#endif
c_p->i = (BeamInstr *) Arg(0); /* L1 */
SWAPOUT;
c_p->arity = 0;
@@ -2082,7 +2189,7 @@ void process_main(void)
if ((c_p->flags & (F_INSLPQUEUE | F_TIMO)) == 0) {
BeamInstr** p = (BeamInstr **) c_p->def_arg_reg;
*p = I+3;
- set_timer(c_p, Arg(1));
+ erts_set_proc_timer_uword(c_p, Arg(1));
}
goto wait2;
}
@@ -2100,7 +2207,7 @@ void process_main(void)
PreFetch(0, next);
if (IS_TRACED_FL(c_p, F_TRACE_RECEIVE)) {
- trace_receive(c_p, am_timeout);
+ trace_receive(c_p, am_clock_service, am_timeout, NULL);
}
if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) {
save_calls(c_p, &exp_timeout);
@@ -2110,22 +2217,17 @@ void process_main(void)
NextPF(0, next);
}
-
{
Eterm select_val2;
- OpCase(i_select_tuple_arity2_yfAfAf):
+ OpCase(i_select_tuple_arity2_yfAAff):
select_val2 = yb(Arg(0));
goto do_select_tuple_arity2;
- OpCase(i_select_tuple_arity2_xfAfAf):
+ OpCase(i_select_tuple_arity2_xfAAff):
select_val2 = xb(Arg(0));
goto do_select_tuple_arity2;
- OpCase(i_select_tuple_arity2_rfAfAf):
- select_val2 = r(0);
- I--;
-
do_select_tuple_arity2:
if (is_not_tuple(select_val2)) {
goto select_val2_fail;
@@ -2133,22 +2235,18 @@ void process_main(void)
select_val2 = *tuple_val(select_val2);
goto do_select_val2;
- OpCase(i_select_val2_yfcfcf):
+ OpCase(i_select_val2_yfccff):
select_val2 = yb(Arg(0));
goto do_select_val2;
- OpCase(i_select_val2_xfcfcf):
+ OpCase(i_select_val2_xfccff):
select_val2 = xb(Arg(0));
goto do_select_val2;
- OpCase(i_select_val2_rfcfcf):
- select_val2 = r(0);
- I--;
-
do_select_val2:
if (select_val2 == Arg(2)) {
- I += 2;
- } else if (select_val2 == Arg(4)) {
+ I += 3;
+ } else if (select_val2 == Arg(3)) {
I += 4;
}
@@ -2168,30 +2266,48 @@ void process_main(void)
select_val = yb(Arg(0));
goto do_select_tuple_arity;
- OpCase(i_select_tuple_arity_rfI):
- select_val = r(0);
- I--;
-
do_select_tuple_arity:
if (is_tuple(select_val)) {
select_val = *tuple_val(select_val);
- goto do_binary_search;
+ goto do_linear_search;
+ }
+ SET_I((BeamInstr *) Arg(1));
+ Goto(*I);
+
+ OpCase(i_select_val_lins_xfI):
+ select_val = xb(Arg(0));
+ goto do_linear_search;
+
+ OpCase(i_select_val_lins_yfI):
+ select_val = yb(Arg(0));
+ goto do_linear_search;
+
+ do_linear_search: {
+ BeamInstr *vs = &Arg(3);
+ int ix = 0;
+
+ for(;;) {
+ if (vs[ix+0] >= select_val) { ix += 0; break; }
+ if (vs[ix+1] >= select_val) { ix += 1; break; }
+ ix += 2;
}
+
+ if (vs[ix] == select_val) {
+ I += ix + Arg(2) + 2;
+ }
+
SET_I((BeamInstr *) Arg(1));
Goto(*I);
+ }
- OpCase(i_select_val_xfI):
+ OpCase(i_select_val_bins_xfI):
select_val = xb(Arg(0));
goto do_binary_search;
- OpCase(i_select_val_yfI):
+ OpCase(i_select_val_bins_yfI):
select_val = yb(Arg(0));
goto do_binary_search;
- OpCase(i_select_val_rfI):
- select_val = r(0);
- I--;
-
do_binary_search:
{
struct Pairs {
@@ -2252,10 +2368,6 @@ void process_main(void)
jump_on_val_zero_index = xb(Arg(0));
goto do_jump_on_val_zero_index;
- OpCase(i_jump_on_val_zero_rfI):
- jump_on_val_zero_index = r(0);
- I--;
-
do_jump_on_val_zero_index:
if (is_small(jump_on_val_zero_index)) {
jump_on_val_zero_index = signed_val(jump_on_val_zero_index);
@@ -2280,10 +2392,6 @@ void process_main(void)
jump_on_val_index = xb(Arg(0));
goto do_jump_on_val_index;
- OpCase(i_jump_on_val_rfII):
- jump_on_val_index = r(0);
- I--;
-
do_jump_on_val_index:
if (is_small(jump_on_val_index)) {
jump_on_val_index = (Uint) (signed_val(jump_on_val_index) - Arg(3));
@@ -2303,15 +2411,12 @@ void process_main(void)
do {
Eterm term = *I++;
- switch (term & _TAG_IMMED1_MASK) {
- case (R_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER:
- *hp++ = r(0);
- break;
- case (X_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER:
- *hp++ = x(term >> _TAG_IMMED1_SIZE);
+ switch (loader_tag(term)) {
+ case LOADER_X_REG:
+ *hp++ = x(loader_x_reg_index(term));
break;
- case (Y_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER:
- *hp++ = y(term >> _TAG_IMMED1_SIZE);
+ case LOADER_Y_REG:
+ *hp++ = y(loader_y_reg_index(term));
break;
default:
*hp++ = term;
@@ -2322,6 +2427,133 @@ void process_main(void)
Goto(*I);
}
+ OpCase(new_map_dII): {
+ Eterm res;
+
+ HEAVY_SWAPOUT;
+ res = new_map(c_p, reg, I-1);
+ HEAVY_SWAPIN;
+ StoreResult(res, Arg(0));
+ Next(3+Arg(2));
+ }
+
+#define PUT_TERM_REG(term, desc) \
+do { \
+ switch (loader_tag(desc)) { \
+ case LOADER_X_REG: \
+ x(loader_x_reg_index(desc)) = (term); \
+ break; \
+ case LOADER_Y_REG: \
+ y(loader_y_reg_index(desc)) = (term); \
+ break; \
+ default: \
+ ASSERT(0); \
+ break; \
+ } \
+} while(0)
+
+ OpCase(i_get_map_elements_fsI): {
+ Eterm map;
+ BeamInstr *fs;
+ Uint sz, n;
+
+ GetArg1(1, map);
+
+ /* this instruction assumes Arg1 is a map,
+ * i.e. that it follows a test is_map if needed.
+ */
+
+ n = (Uint)Arg(2) / 3;
+ fs = &Arg(3); /* pattern fields and target registers */
+
+ if (is_flatmap(map)) {
+ flatmap_t *mp;
+ Eterm *ks;
+ Eterm *vs;
+
+ mp = (flatmap_t *)flatmap_val(map);
+ sz = flatmap_get_size(mp);
+
+ if (sz == 0) {
+ ClauseFail();
+ }
+
+ ks = flatmap_get_keys(mp);
+ vs = flatmap_get_values(mp);
+
+ while(sz) {
+ if (EQ((Eterm) fs[0], *ks)) {
+ PUT_TERM_REG(*vs, fs[1]);
+ n--;
+ fs += 3;
+ /* no more values to fetch, we are done */
+ if (n == 0) {
+ I = fs;
+ Next(-1);
+ }
+ }
+ ks++, sz--, vs++;
+ }
+
+ ClauseFail();
+ } else {
+ const Eterm *v;
+ Uint32 hx;
+ ASSERT(is_hashmap(map));
+ while(n--) {
+ hx = fs[2];
+ ASSERT(hx == hashmap_make_hash((Eterm)fs[0]));
+ if ((v = erts_hashmap_get(hx, (Eterm)fs[0], map)) == NULL) {
+ ClauseFail();
+ }
+ PUT_TERM_REG(*v, fs[1]);
+ fs += 3;
+ }
+ I = fs;
+ Next(-1);
+ }
+ }
+#undef PUT_TERM_REG
+
+ OpCase(update_map_assoc_jsdII): {
+ Eterm res;
+ Eterm map;
+
+ GetArg1(1, map);
+ HEAVY_SWAPOUT;
+ res = update_map_assoc(c_p, reg, map, I);
+ HEAVY_SWAPIN;
+ if (is_value(res)) {
+ StoreResult(res, Arg(2));
+ Next(5+Arg(4));
+ } else {
+ /*
+ * This can only happen if the code was compiled
+ * with the compiler in OTP 17.
+ */
+ c_p->freason = BADMAP;
+ c_p->fvalue = map;
+ goto lb_Cl_error;
+ }
+ }
+
+ OpCase(update_map_exact_jsdII): {
+ Eterm res;
+ Eterm map;
+
+ GetArg1(1, map);
+ HEAVY_SWAPOUT;
+ res = update_map_exact(c_p, reg, map, I);
+ HEAVY_SWAPIN;
+ if (is_value(res)) {
+ StoreResult(res, Arg(2));
+ Next(5+Arg(4));
+ } else {
+ goto lb_Cl_error;
+ }
+ }
+
+
/*
* All guards with zero arguments have special instructions:
* self/0
@@ -2344,6 +2576,7 @@ void process_main(void)
GetArg1(2, tmp_reg[0]);
bf = (BifFunction) Arg(1);
+ ERTS_DBG_CHK_REDS(c_p, FCALLS);
c_p->fcalls = FCALLS;
PROCESS_MAIN_CHK_LOCKS(c_p);
ASSERT(!ERTS_PROC_IS_EXITING(c_p));
@@ -2353,6 +2586,7 @@ void process_main(void)
PROCESS_MAIN_CHK_LOCKS(c_p);
ERTS_HOLE_CHECK(c_p);
FCALLS = c_p->fcalls;
+ ERTS_DBG_CHK_REDS(c_p, FCALLS);
if (is_value(result)) {
StoreBifResult(3, result);
}
@@ -2373,6 +2607,7 @@ void process_main(void)
GetArg1(1, tmp_reg[0]);
bf = (BifFunction) Arg(0);
+ ERTS_DBG_CHK_REDS(c_p, FCALLS);
c_p->fcalls = FCALLS;
PROCESS_MAIN_CHK_LOCKS(c_p);
ASSERT(!ERTS_PROC_IS_EXITING(c_p));
@@ -2382,6 +2617,7 @@ void process_main(void)
PROCESS_MAIN_CHK_LOCKS(c_p);
ERTS_HOLE_CHECK(c_p);
FCALLS = c_p->fcalls;
+ ERTS_DBG_CHK_REDS(c_p, FCALLS);
if (is_value(result)) {
StoreBifResult(2, result);
}
@@ -2395,14 +2631,12 @@ void process_main(void)
{
typedef Eterm (*GcBifFunction)(Process*, Eterm*, Uint);
GcBifFunction bf;
- Eterm arg;
Eterm result;
Uint live = (Uint) Arg(3);
- GetArg1(2, arg);
- reg[0] = r(0);
- reg[live] = arg;
+ GetArg1(2, x(live));
bf = (GcBifFunction) Arg(1);
+ ERTS_DBG_CHK_REDS(c_p, FCALLS);
c_p->fcalls = FCALLS;
SWAPOUT;
PROCESS_MAIN_CHK_LOCKS(c_p);
@@ -2412,9 +2646,9 @@ void process_main(void)
ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
PROCESS_MAIN_CHK_LOCKS(c_p);
SWAPIN;
- r(0) = reg[0];
ERTS_HOLE_CHECK(c_p);
FCALLS = c_p->fcalls;
+ ERTS_DBG_CHK_REDS(c_p, FCALLS);
if (is_value(result)) {
StoreBifResult(4, result);
}
@@ -2422,12 +2656,12 @@ void process_main(void)
SET_I((BeamInstr *) Arg(0));
Goto(*I);
}
- reg[0] = arg;
+ x(0) = x(live);
I = handle_error(c_p, I, reg, translate_gc_bif((void *) bf));
goto post_error_handling;
}
- OpCase(i_gc_bif2_jIId): /* Note, one less parameter than the i_gc_bif1
+ OpCase(i_gc_bif2_jIIssd): /* Note, one less parameter than the i_gc_bif1
and i_gc_bif3 */
{
typedef Eterm (*GcBifFunction)(Process*, Eterm*, Uint);
@@ -2435,10 +2669,15 @@ void process_main(void)
Eterm result;
Uint live = (Uint) Arg(2);
- reg[0] = r(0);
- reg[live++] = tmp_arg1;
- reg[live] = tmp_arg2;
+ GetArg2(3, x(live), x(live+1));
+ /*
+ * XXX This calling convention does not make sense. 'live'
+ * should point out the first argument, not the second
+ * (i.e. 'live' should not be incremented below).
+ */
+ live++;
bf = (GcBifFunction) Arg(1);
+ ERTS_DBG_CHK_REDS(c_p, FCALLS);
c_p->fcalls = FCALLS;
SWAPOUT;
PROCESS_MAIN_CHK_LOCKS(c_p);
@@ -2448,36 +2687,40 @@ void process_main(void)
ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
PROCESS_MAIN_CHK_LOCKS(c_p);
SWAPIN;
- r(0) = reg[0];
ERTS_HOLE_CHECK(c_p);
FCALLS = c_p->fcalls;
+ ERTS_DBG_CHK_REDS(c_p, FCALLS);
if (is_value(result)) {
- StoreBifResult(3, result);
+ StoreBifResult(5, result);
}
if (Arg(0) != 0) {
SET_I((BeamInstr *) Arg(0));
Goto(*I);
}
- reg[0] = tmp_arg1;
- reg[1] = tmp_arg2;
+ live--;
+ x(0) = x(live);
+ x(1) = x(live+1);
I = handle_error(c_p, I, reg, translate_gc_bif((void *) bf));
goto post_error_handling;
}
- OpCase(i_gc_bif3_jIsId):
+ OpCase(i_gc_bif3_jIIssd):
{
typedef Eterm (*GcBifFunction)(Process*, Eterm*, Uint);
GcBifFunction bf;
- Eterm arg;
Eterm result;
- Uint live = (Uint) Arg(3);
+ Uint live = (Uint) Arg(2);
- GetArg1(2, arg);
- reg[0] = r(0);
- reg[live++] = arg;
- reg[live++] = tmp_arg1;
- reg[live] = tmp_arg2;
+ x(live) = x(SCRATCH_X_REG);
+ GetArg2(3, x(live+1), x(live+2));
+ /*
+ * XXX This calling convention does not make sense. 'live'
+ * should point out the first argument, not the third
+ * (i.e. 'live' should not be incremented below).
+ */
+ live += 2;
bf = (GcBifFunction) Arg(1);
+ ERTS_DBG_CHK_REDS(c_p, FCALLS);
c_p->fcalls = FCALLS;
SWAPOUT;
PROCESS_MAIN_CHK_LOCKS(c_p);
@@ -2487,19 +2730,20 @@ void process_main(void)
ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
PROCESS_MAIN_CHK_LOCKS(c_p);
SWAPIN;
- r(0) = reg[0];
ERTS_HOLE_CHECK(c_p);
FCALLS = c_p->fcalls;
+ ERTS_DBG_CHK_REDS(c_p, FCALLS);
if (is_value(result)) {
- StoreBifResult(4, result);
+ StoreBifResult(5, result);
}
if (Arg(0) != 0) {
SET_I((BeamInstr *) Arg(0));
Goto(*I);
}
- reg[0] = arg;
- reg[1] = tmp_arg1;
- reg[2] = tmp_arg2;
+ live -= 2;
+ x(0) = x(live);
+ x(1) = x(live+1);
+ x(2) = x(live+2);
I = handle_error(c_p, I, reg, translate_gc_bif((void *) bf));
goto post_error_handling;
}
@@ -2507,13 +2751,15 @@ void process_main(void)
/*
* Guards bifs and, or, xor in guards.
*/
- OpCase(i_bif2_fbd):
+ OpCase(i_bif2_fbssd):
{
- Eterm tmp_reg[2] = {tmp_arg1, tmp_arg2};
+ Eterm tmp_reg[2];
Eterm (*bf)(Process*, Eterm*);
Eterm result;
+ GetArg2(2, tmp_reg[0], tmp_reg[1]);
bf = (BifFunction) Arg(1);
+ ERTS_DBG_CHK_REDS(c_p, FCALLS);
c_p->fcalls = FCALLS;
PROCESS_MAIN_CHK_LOCKS(c_p);
ASSERT(!ERTS_PROC_IS_EXITING(c_p));
@@ -2523,8 +2769,9 @@ void process_main(void)
PROCESS_MAIN_CHK_LOCKS(c_p);
ERTS_HOLE_CHECK(c_p);
FCALLS = c_p->fcalls;
+ ERTS_DBG_CHK_REDS(c_p, FCALLS);
if (is_value(result)) {
- StoreBifResult(2, result);
+ StoreBifResult(4, result);
}
SET_I((BeamInstr *) Arg(0));
Goto(*I);
@@ -2533,12 +2780,13 @@ void process_main(void)
/*
* Guards bifs and, or, xor, relational operators in body.
*/
- OpCase(i_bif2_body_bd):
+ OpCase(i_bif2_body_bssd):
{
- Eterm tmp_reg[2] = {tmp_arg1, tmp_arg2};
+ Eterm tmp_reg[2];
Eterm (*bf)(Process*, Eterm*);
Eterm result;
+ GetArg2(1, tmp_reg[0], tmp_reg[1]);
bf = (BifFunction) Arg(0);
PROCESS_MAIN_CHK_LOCKS(c_p);
ASSERT(!ERTS_PROC_IS_EXITING(c_p));
@@ -2549,10 +2797,10 @@ void process_main(void)
ERTS_HOLE_CHECK(c_p);
if (is_value(result)) {
ASSERT(!is_CP(result));
- StoreBifResult(1, result);
+ StoreBifResult(3, result);
}
- reg[0] = tmp_arg1;
- reg[1] = tmp_arg2;
+ reg[0] = tmp_reg[0];
+ reg[1] = tmp_reg[1];
SWAPOUT;
I = handle_error(c_p, I, reg, bf);
goto post_error_handling;
@@ -2564,31 +2812,60 @@ void process_main(void)
*/
OpCase(call_bif_e):
{
- Eterm (*bf)(Process*, Eterm*, BeamInstr*) = GET_BIF_ADDRESS(Arg(0));
+ Eterm (*bf)(Process*, Eterm*, BeamInstr*);
Eterm result;
BeamInstr *next;
+ ErlHeapFragment *live_hf_end;
+
+
+ if (!((FCALLS - 1) > 0 || (FCALLS-1) > neg_o_reds)) {
+ /* If we have run out of reductions, we do a context
+ switch before calling the bif */
+ c_p->arity = ((Export *)Arg(0))->code[2];
+ c_p->current = ((Export *)Arg(0))->code;
+ goto context_switch3;
+ }
+
+ if (ERTS_MSACC_IS_ENABLED_CACHED_X()) {
+ if (GET_BIF_MODULE(Arg(0)) == am_ets) {
+ ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_ETS);
+ } else {
+ ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_BIF);
+ }
+ }
+
+ bf = GET_BIF_ADDRESS(Arg(0));
PRE_BIF_SWAPOUT(c_p);
+ ERTS_DBG_CHK_REDS(c_p, FCALLS);
c_p->fcalls = FCALLS - 1;
if (FCALLS <= 0) {
save_calls(c_p, (Export *) Arg(0));
}
PreFetch(1, next);
ASSERT(!ERTS_PROC_IS_EXITING(c_p));
- reg[0] = r(0);
+ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+ live_hf_end = c_p->mbuf;
result = (*bf)(c_p, reg, I);
ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
ERTS_HOLE_CHECK(c_p);
ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
- PROCESS_MAIN_CHK_LOCKS(c_p);
- if (c_p->mbuf || MSO(c_p).overhead >= BIN_VHEAP_SZ(c_p)) {
+ if (ERTS_IS_GC_DESIRED(c_p)) {
Uint arity = ((Export *)Arg(0))->code[2];
- result = erts_gc_after_bif_call(c_p, result, reg, arity);
+ result = erts_gc_after_bif_call_lhf(c_p, live_hf_end, result, reg, arity);
E = c_p->stop;
}
+ PROCESS_MAIN_CHK_LOCKS(c_p);
HTOP = HEAP_TOP(c_p);
FCALLS = c_p->fcalls;
+ ERTS_DBG_CHK_REDS(c_p, FCALLS);
+ /* We have to update the cache if we are enabled in order
+ to make sure no book keeping is done after we disabled
+ msacc. We don't always do this as it is quite expensive. */
+ if (ERTS_MSACC_IS_ENABLED_CACHED_X())
+ ERTS_MSACC_UPDATE_CACHE_X();
+ ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR);
if (is_value(result)) {
r(0) = result;
CHECK_TERM(r(0));
@@ -2597,7 +2874,6 @@ void process_main(void)
SET_CP(c_p, I+2);
SET_I(c_p->i);
SWAPIN;
- r(0) = reg[0];
Dispatch();
}
@@ -2613,82 +2889,81 @@ void process_main(void)
* Arithmetic operations.
*/
- OpCase(i_times_jId):
+ OpCase(i_times_jIssd):
{
- arith_func = ARITH_FUNC(mixed_times);
- goto do_big_arith2;
+ Eterm Op1, Op2;
+ GetArg2(2, Op1, Op2);
+ DO_OUTLINED_ARITH_2(mixed_times, Op1, Op2);
}
- OpCase(i_m_div_jId):
+ OpCase(i_m_div_jIssd):
{
- arith_func = ARITH_FUNC(mixed_div);
- goto do_big_arith2;
+ Eterm Op1, Op2;
+ GetArg2(2, Op1, Op2);
+ DO_OUTLINED_ARITH_2(mixed_div, Op1, Op2);
}
- OpCase(i_int_div_jId):
+ OpCase(i_int_div_jIssd):
{
- Eterm result;
+ Eterm Op1, Op2;
- if (tmp_arg2 == SMALL_ZERO) {
+ GetArg2(2, Op1, Op2);
+ if (Op2 == SMALL_ZERO) {
goto badarith;
- } else if (is_both_small(tmp_arg1, tmp_arg2)) {
- Sint ires = signed_val(tmp_arg1) / signed_val(tmp_arg2);
+ } else if (is_both_small(Op1, Op2)) {
+ Sint ires = signed_val(Op1) / signed_val(Op2);
if (MY_IS_SSMALL(ires)) {
- result = make_small(ires);
- STORE_ARITH_RESULT(result);
+ Eterm result = make_small(ires);
+ StoreBifResult(4, result);
}
}
- arith_func = ARITH_FUNC(int_div);
- goto do_big_arith2;
+ DO_OUTLINED_ARITH_2(int_div, Op1, Op2);
}
- OpCase(i_rem_jId):
{
- Eterm result;
-
- if (tmp_arg2 == SMALL_ZERO) {
- goto badarith;
- } else if (is_both_small(tmp_arg1, tmp_arg2)) {
- result = make_small(signed_val(tmp_arg1) % signed_val(tmp_arg2));
- STORE_ARITH_RESULT(result);
+ Eterm RemOp1, RemOp2;
+
+ OpCase(i_rem_jIxxd):
+ RemOp1 = xb(Arg(2));
+ RemOp2 = xb(Arg(3));
+ goto do_rem;
+
+ OpCase(i_rem_jIssd):
+ GetArg2(2, RemOp1, RemOp2);
+ goto do_rem;
+
+ do_rem:
+ if (RemOp2 == SMALL_ZERO) {
+ goto badarith;
+ } else if (is_both_small(RemOp1, RemOp2)) {
+ Eterm result = make_small(signed_val(RemOp1) % signed_val(RemOp2));
+ StoreBifResult(4, result);
} else {
- arith_func = ARITH_FUNC(int_rem);
- goto do_big_arith2;
- }
- }
-
- OpCase(i_band_jId):
- {
- Eterm result;
-
- if (is_both_small(tmp_arg1, tmp_arg2)) {
- /*
- * No need to untag -- TAG & TAG == TAG.
- */
- result = tmp_arg1 & tmp_arg2;
- STORE_ARITH_RESULT(result);
+ DO_OUTLINED_ARITH_2(int_rem, RemOp1, RemOp2);
}
- arith_func = ARITH_FUNC(band);
- goto do_big_arith2;
}
- do_big_arith2:
{
- Eterm result;
- Uint live = Arg(1);
-
- SWAPOUT;
- reg[0] = r(0);
- reg[live] = tmp_arg1;
- reg[live+1] = tmp_arg2;
- result = arith_func(c_p, reg, live);
- r(0) = reg[0];
- SWAPIN;
- ERTS_HOLE_CHECK(c_p);
- if (is_value(result)) {
- STORE_ARITH_RESULT(result);
+ Eterm BandOp1, BandOp2;
+
+ OpCase(i_band_jIxcd):
+ BandOp1 = xb(Arg(2));
+ BandOp2 = Arg(3);
+ goto do_band;
+
+ OpCase(i_band_jIssd):
+ GetArg2(2, BandOp1, BandOp2);
+ goto do_band;
+
+ do_band:
+ if (is_both_small(BandOp1, BandOp2)) {
+ /*
+ * No need to untag -- TAG & TAG == TAG.
+ */
+ Eterm result = BandOp1 & BandOp2;
+ StoreBifResult(4, result);
}
- goto lb_Cl_error;
+ DO_OUTLINED_ARITH_2(band, BandOp1, BandOp2);
}
/*
@@ -2709,97 +2984,105 @@ void process_main(void)
goto find_func_info;
}
- OpCase(i_bor_jId):
+ OpCase(i_bor_jIssd):
{
- Eterm result;
+ Eterm Op1, Op2;
- if (is_both_small(tmp_arg1, tmp_arg2)) {
+ GetArg2(2, Op1, Op2);
+ if (is_both_small(Op1, Op2)) {
/*
* No need to untag -- TAG | TAG == TAG.
*/
- result = tmp_arg1 | tmp_arg2;
- STORE_ARITH_RESULT(result);
+ Eterm result = Op1 | Op2;
+ StoreBifResult(4, result);
}
- arith_func = ARITH_FUNC(bor);
- goto do_big_arith2;
+ DO_OUTLINED_ARITH_2(bor, Op1, Op2);
}
- OpCase(i_bxor_jId):
+ OpCase(i_bxor_jIssd):
{
- Eterm result;
+ Eterm Op1, Op2;
- if (is_both_small(tmp_arg1, tmp_arg2)) {
+ GetArg2(2, Op1, Op2);
+ if (is_both_small(Op1, Op2)) {
/*
* We could extract the tag from one argument, but a tag extraction
* could mean a shift. Therefore, play it safe here.
*/
- result = make_small(signed_val(tmp_arg1) ^ signed_val(tmp_arg2));
- STORE_ARITH_RESULT(result);
+ Eterm result = make_small(signed_val(Op1) ^ signed_val(Op2));
+ StoreBifResult(4, result);
}
- arith_func = ARITH_FUNC(bxor);
- goto do_big_arith2;
+ DO_OUTLINED_ARITH_2(bxor, Op1, Op2);
}
{
+ Eterm Op1, Op2;
Sint i;
Sint ires;
Eterm* bigp;
+ Eterm tmp_big[2];
- OpCase(i_bsr_jId):
- if (is_small(tmp_arg2)) {
- i = -signed_val(tmp_arg2);
- if (is_small(tmp_arg1)) {
+ OpCase(i_bsr_jIssd):
+ GetArg2(2, Op1, Op2);
+ if (is_small(Op2)) {
+ i = -signed_val(Op2);
+ if (is_small(Op1)) {
goto small_shift;
- } else if (is_big(tmp_arg1)) {
+ } else if (is_big(Op1)) {
if (i == 0) {
- StoreBifResult(2, tmp_arg1);
+ StoreBifResult(4, Op1);
}
goto big_shift;
}
- } else if (is_big(tmp_arg2)) {
+ } else if (is_big(Op2)) {
/*
* N bsr NegativeBigNum == N bsl MAX_SMALL
* N bsr PositiveBigNum == N bsl MIN_SMALL
*/
- tmp_arg2 = make_small(bignum_header_is_neg(*big_val(tmp_arg2)) ?
+ Op2 = make_small(bignum_header_is_neg(*big_val(Op2)) ?
MAX_SMALL : MIN_SMALL);
goto do_bsl;
}
goto badarith;
- OpCase(i_bsl_jId):
+ OpCase(i_bsl_jIssd):
+ GetArg2(2, Op1, Op2);
+
do_bsl:
- if (is_small(tmp_arg2)) {
- i = signed_val(tmp_arg2);
+ if (is_small(Op2)) {
+ i = signed_val(Op2);
- if (is_small(tmp_arg1)) {
+ if (is_small(Op1)) {
small_shift:
- ires = signed_val(tmp_arg1);
+ ires = signed_val(Op1);
if (i == 0 || ires == 0) {
- StoreBifResult(2, tmp_arg1);
+ StoreBifResult(4, Op1);
} else if (i < 0) { /* Right shift */
i = -i;
if (i >= SMALL_BITS-1) {
- tmp_arg1 = (ires < 0) ? SMALL_MINUS_ONE : SMALL_ZERO;
+ Op1 = (ires < 0) ? SMALL_MINUS_ONE : SMALL_ZERO;
} else {
- tmp_arg1 = make_small(ires >> i);
+ Op1 = make_small(ires >> i);
}
- StoreBifResult(2, tmp_arg1);
+ StoreBifResult(4, Op1);
} else if (i < SMALL_BITS-1) { /* Left shift */
if ((ires > 0 && ((~(Uint)0 << ((SMALL_BITS-1)-i)) & ires) == 0) ||
((~(Uint)0 << ((SMALL_BITS-1)-i)) & ~ires) == 0) {
- tmp_arg1 = make_small(ires << i);
- StoreBifResult(2, tmp_arg1);
+ Op1 = make_small(ires << i);
+ StoreBifResult(4, Op1);
}
}
- tmp_arg1 = small_to_big(ires, tmp_big);
+ Op1 = small_to_big(ires, tmp_big);
+#ifdef TAG_LITERAL_PTR
+ Op1 |= TAG_LITERAL_PTR;
+#endif
big_shift:
if (i > 0) { /* Left shift. */
- ires = big_size(tmp_arg1) + (i / D_EXP);
+ ires = big_size(Op1) + (i / D_EXP);
} else { /* Right shift. */
- ires = big_size(tmp_arg1);
+ ires = big_size(Op1);
if (ires <= (-i / D_EXP))
ires = 3; /* ??? */
else
@@ -2817,14 +3100,14 @@ void process_main(void)
c_p->freason = SYSTEM_LIMIT;
goto lb_Cl_error;
}
- TestHeapPreserve(ires+1, Arg(1), tmp_arg1);
+ TestHeapPreserve(ires+1, Arg(1), Op1);
bigp = HTOP;
- tmp_arg1 = big_lshift(tmp_arg1, i, bigp);
- if (is_big(tmp_arg1)) {
+ Op1 = big_lshift(Op1, i, bigp);
+ if (is_big(Op1)) {
HTOP += bignum_header_arity(*HTOP) + 1;
}
HEAP_SPACE_VERIFIED(0);
- if (is_nil(tmp_arg1)) {
+ if (is_nil(Op1)) {
/*
* This result must have been only slight larger
* than allowed since it wasn't caught by the
@@ -2834,25 +3117,25 @@ void process_main(void)
goto lb_Cl_error;
}
ERTS_HOLE_CHECK(c_p);
- StoreBifResult(2, tmp_arg1);
+ StoreBifResult(4, Op1);
}
- } else if (is_big(tmp_arg1)) {
+ } else if (is_big(Op1)) {
if (i == 0) {
- StoreBifResult(2, tmp_arg1);
+ StoreBifResult(4, Op1);
}
goto big_shift;
}
- } else if (is_big(tmp_arg2)) {
- if (bignum_header_is_neg(*big_val(tmp_arg2))) {
+ } else if (is_big(Op2)) {
+ if (bignum_header_is_neg(*big_val(Op2))) {
/*
* N bsl NegativeBigNum is either 0 or -1, depending on
* the sign of N. Since we don't believe this case
* is common, do the calculation with the minimum
* amount of code.
*/
- tmp_arg2 = make_small(MIN_SMALL);
+ Op2 = make_small(MIN_SMALL);
goto do_bsl;
- } else if (is_small(tmp_arg1) || is_big(tmp_arg1)) {
+ } else if (is_small(Op1) || is_big(Op1)) {
/*
* N bsl PositiveBigNum is too large to represent.
*/
@@ -2876,12 +3159,10 @@ void process_main(void)
bnot_val = make_small(~signed_val(bnot_val));
} else {
Uint live = Arg(2);
- SWAPOUT;
- reg[0] = r(0);
+ HEAVY_SWAPOUT;
reg[live] = bnot_val;
bnot_val = erts_gc_bnot(c_p, reg, live);
- r(0) = reg[0];
- SWAPIN;
+ HEAVY_SWAPIN;
ERTS_HOLE_CHECK(c_p);
if (is_nil(bnot_val)) {
goto lb_Cl_error;
@@ -2896,11 +3177,10 @@ void process_main(void)
OpCase(i_apply): {
BeamInstr *next;
- SWAPOUT;
+ HEAVY_SWAPOUT;
next = apply(c_p, r(0), x(1), x(2), reg);
- SWAPIN;
+ HEAVY_SWAPIN;
if (next != NULL) {
- r(0) = reg[0];
SET_CP(c_p, I+1);
SET_I(next);
Dispatch();
@@ -2911,12 +3191,11 @@ void process_main(void)
OpCase(i_apply_last_P): {
BeamInstr *next;
- SWAPOUT;
+ HEAVY_SWAPOUT;
next = apply(c_p, r(0), x(1), x(2), reg);
- SWAPIN;
+ HEAVY_SWAPIN;
if (next != NULL) {
- r(0) = reg[0];
- SET_CP(c_p, (BeamInstr *) EXPAND_POINTER(E[0]));
+ SET_CP(c_p, (BeamInstr *) E[0]);
E = ADD_BYTE_OFFSET(E, Arg(0));
SET_I(next);
Dispatch();
@@ -2927,11 +3206,10 @@ void process_main(void)
OpCase(i_apply_only): {
BeamInstr *next;
- SWAPOUT;
+ HEAVY_SWAPOUT;
next = apply(c_p, r(0), x(1), x(2), reg);
- SWAPIN;
+ HEAVY_SWAPIN;
if (next != NULL) {
- r(0) = reg[0];
SET_I(next);
Dispatch();
}
@@ -2942,12 +3220,10 @@ void process_main(void)
OpCase(apply_I): {
BeamInstr *next;
- reg[0] = r(0);
- SWAPOUT;
+ HEAVY_SWAPOUT;
next = fixed_apply(c_p, reg, Arg(0));
- SWAPIN;
+ HEAVY_SWAPIN;
if (next != NULL) {
- r(0) = reg[0];
SET_CP(c_p, I+2);
SET_I(next);
Dispatch();
@@ -2959,13 +3235,11 @@ void process_main(void)
OpCase(apply_last_IP): {
BeamInstr *next;
- reg[0] = r(0);
- SWAPOUT;
+ HEAVY_SWAPOUT;
next = fixed_apply(c_p, reg, Arg(0));
- SWAPIN;
+ HEAVY_SWAPIN;
if (next != NULL) {
- r(0) = reg[0];
- SET_CP(c_p, (BeamInstr *) EXPAND_POINTER(E[0]));
+ SET_CP(c_p, (BeamInstr *) E[0]);
E = ADD_BYTE_OFFSET(E, Arg(1));
SET_I(next);
Dispatch();
@@ -2977,11 +3251,10 @@ void process_main(void)
OpCase(i_apply_fun): {
BeamInstr *next;
- SWAPOUT;
+ HEAVY_SWAPOUT;
next = apply_fun(c_p, r(0), x(1), reg);
- SWAPIN;
+ HEAVY_SWAPIN;
if (next != NULL) {
- r(0) = reg[0];
SET_CP(c_p, I+1);
SET_I(next);
Dispatchfun();
@@ -2992,12 +3265,11 @@ void process_main(void)
OpCase(i_apply_fun_last_P): {
BeamInstr *next;
- SWAPOUT;
+ HEAVY_SWAPOUT;
next = apply_fun(c_p, r(0), x(1), reg);
- SWAPIN;
+ HEAVY_SWAPIN;
if (next != NULL) {
- r(0) = reg[0];
- SET_CP(c_p, (BeamInstr *) EXPAND_POINTER(E[0]));
+ SET_CP(c_p, (BeamInstr *) E[0]);
E = ADD_BYTE_OFFSET(E, Arg(0));
SET_I(next);
Dispatchfun();
@@ -3008,11 +3280,10 @@ void process_main(void)
OpCase(i_apply_fun_only): {
BeamInstr *next;
- SWAPOUT;
+ HEAVY_SWAPOUT;
next = apply_fun(c_p, r(0), x(1), reg);
- SWAPIN;
+ HEAVY_SWAPIN;
if (next != NULL) {
- r(0) = reg[0];
SET_I(next);
Dispatchfun();
}
@@ -3022,13 +3293,10 @@ void process_main(void)
OpCase(i_call_fun_I): {
BeamInstr *next;
- SWAPOUT;
- reg[0] = r(0);
-
+ HEAVY_SWAPOUT;
next = call_fun(c_p, Arg(0), reg, THE_NON_VALUE);
- SWAPIN;
+ HEAVY_SWAPIN;
if (next != NULL) {
- r(0) = reg[0];
SET_CP(c_p, I+2);
SET_I(next);
Dispatchfun();
@@ -3039,13 +3307,11 @@ void process_main(void)
OpCase(i_call_fun_last_IP): {
BeamInstr *next;
- SWAPOUT;
- reg[0] = r(0);
+ HEAVY_SWAPOUT;
next = call_fun(c_p, Arg(0), reg, THE_NON_VALUE);
- SWAPIN;
+ HEAVY_SWAPIN;
if (next != NULL) {
- r(0) = reg[0];
- SET_CP(c_p, (BeamInstr *) EXPAND_POINTER(E[0]));
+ SET_CP(c_p, (BeamInstr *) E[0]);
E = ADD_BYTE_OFFSET(E, Arg(1));
SET_I(next);
Dispatchfun();
@@ -3088,10 +3354,19 @@ void process_main(void)
context_switch2: /* Entry for fun calls. */
c_p->current = I-3; /* Pointer to Mod, Func, Arity */
+ context_switch3:
+
{
Eterm* argp;
int i;
+ if (erts_smp_atomic32_read_nob(&c_p->state) & ERTS_PSFLG_EXITING) {
+ c_p->i = beam_exit;
+ c_p->arity = 0;
+ c_p->current = NULL;
+ goto do_schedule;
+ }
+
/*
* Make sure that there is enough room for the argument registers to be saved.
*/
@@ -3122,17 +3397,21 @@ void process_main(void)
* (beacuse the code for the Dispatch() macro becomes shorter that way).
*/
- reds_used = REDS_IN(c_p) - FCALLS + 1;
-
+ ASSERT(c_p->debug_reds_in == REDS_IN(c_p));
+ if (!ERTS_PROC_GET_SAVED_CALLS_BUF(c_p))
+ reds_used = REDS_IN(c_p) - FCALLS;
+ else
+ reds_used = REDS_IN(c_p) - (CONTEXT_REDS + FCALLS);
+ ASSERT(reds_used >= 0);
+
/*
* Save the argument registers and everything else.
*/
argp = c_p->arg_reg;
- for (i = c_p->arity - 1; i > 0; i--) {
+ for (i = c_p->arity - 1; i >= 0; i--) {
argp[i] = reg[i];
}
- c_p->arg_reg[0] = r(0);
SWAPOUT;
c_p->i = I;
goto do_schedule1;
@@ -3145,23 +3424,18 @@ void process_main(void)
Eterm* p;
PreFetch(3, next);
- GetArg2(0, element, tuple);
+ GetArg1(0, element);
+ tuple = REG_TARGET(Arg(1));
ASSERT(is_tuple(tuple));
p = (Eterm *) ((unsigned char *) tuple_val(tuple) + Arg(2));
*p = element;
NextPF(3, next);
}
- OpCase(i_is_ne_exact_f):
- if (EQ(tmp_arg1, tmp_arg2)) {
- ClauseFail();
- }
- Next(1);
-
OpCase(normal_exit): {
SWAPOUT;
c_p->freason = EXC_NORMAL;
- c_p->arity = 0; /* In case this process will ever be garbed again. */
+ c_p->arity = 0; /* In case this process will never be garbed again. */
ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
erts_do_exit_process(c_p, am_normal);
ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
@@ -3175,48 +3449,18 @@ void process_main(void)
goto do_schedule;
}
- OpCase(raise_ss): {
- /* This was not done very well in R10-0; then, we passed the tag in
- the first argument and hoped that the existing c_p->ftrace was
- still correct. But the ftrace-object already includes the tag
- (or rather, the freason). Now, we pass the original ftrace in
- the first argument. We also handle atom tags in the first
- argument for backwards compatibility.
- */
- Eterm raise_val1;
- Eterm raise_val2;
- GetArg2(0, raise_val1, raise_val2);
- c_p->fvalue = raise_val2;
- if (c_p->freason == EXC_NULL) {
- /* a safety check for the R10-0 case; should not happen */
- c_p->ftrace = NIL;
- c_p->freason = EXC_ERROR;
- }
- /* for R10-0 code, keep existing c_p->ftrace and hope it's correct */
- switch (raise_val1) {
- case am_throw:
- c_p->freason = EXC_THROWN & ~EXF_SAVETRACE;
- break;
- case am_error:
- c_p->freason = EXC_ERROR & ~EXF_SAVETRACE;
- break;
- case am_exit:
- c_p->freason = EXC_EXIT & ~EXF_SAVETRACE;
- break;
- default:
- {/* R10-1 and later
- XXX note: should do sanity check on given trace if it can be
- passed from a user! Currently only expecting generated calls.
- */
- struct StackTrace *s;
- c_p->ftrace = raise_val1;
- s = get_trace_from_exc(raise_val1);
- if (s == NULL) {
- c_p->freason = EXC_ERROR;
- } else {
- c_p->freason = PRIMARY_EXCEPTION(s->freason);
- }
- }
+ OpCase(i_raise): {
+ Eterm raise_trace = x(2);
+ Eterm raise_value = x(1);
+ struct StackTrace *s;
+
+ c_p->fvalue = raise_value;
+ c_p->ftrace = raise_trace;
+ s = get_trace_from_exc(raise_trace);
+ if (s == NULL) {
+ c_p->freason = EXC_ERROR;
+ } else {
+ c_p->freason = PRIMARY_EXCEPTION(s->freason);
}
goto find_func_info;
}
@@ -3224,25 +3468,14 @@ void process_main(void)
{
Eterm badmatch_val;
- OpCase(badmatch_y):
- badmatch_val = yb(Arg(0));
- goto do_badmatch;
-
OpCase(badmatch_x):
badmatch_val = xb(Arg(0));
- goto do_badmatch;
-
- OpCase(badmatch_r):
- badmatch_val = r(0);
-
- do_badmatch:
c_p->fvalue = badmatch_val;
c_p->freason = BADMATCH;
}
/* Fall through here */
find_func_info: {
- reg[0] = r(0);
SWAPOUT;
I = handle_error(c_p, I, reg, NULL);
goto post_error_handling;
@@ -3259,11 +3492,9 @@ void process_main(void)
* code[3]: &&call_error_handler
* code[4]: Not used
*/
- SWAPOUT;
- reg[0] = r(0);
+ HEAVY_SWAPOUT;
I = call_error_handler(c_p, I-3, reg, am_undefined_function);
- r(0) = reg[0];
- SWAPIN;
+ HEAVY_SWAPIN;
if (I) {
Goto(*I);
}
@@ -3271,18 +3502,13 @@ void process_main(void)
/* Fall through */
OpCase(error_action_code): {
handle_error:
- reg[0] = r(0);
SWAPOUT;
I = handle_error(c_p, NULL, reg, NULL);
post_error_handling:
if (I == 0) {
goto do_schedule;
} else {
- r(0) = reg[0];
ASSERT(!is_value(r(0)));
- if (c_p->mbuf) {
- erts_garbage_collect(c_p, 0, reg+1, 3);
- }
SWAPIN;
Goto(*I);
}
@@ -3303,8 +3529,18 @@ void process_main(void)
* I[0]: &&call_nif
* I[1]: Function pointer to NIF function
* I[2]: Pointer to erl_module_nif
+ * I[3]: Function pointer to dirty NIF
*/
BifFunction vbf;
+ ErlHeapFragment *live_hf_end;
+
+ if (!((FCALLS - 1) > 0 || (FCALLS - 1) > neg_o_reds)) {
+ /* If we have run out of reductions, we do a context
+ switch before calling the nif */
+ goto context_switch;
+ }
+
+ ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_NIF);
DTRACE_NIF_ENTRY(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]);
c_p->current = I-3; /* current and vbf set to please handle_error */
@@ -3319,14 +3555,22 @@ void process_main(void)
typedef Eterm NifF(struct enif_environment_t*, int argc, Eterm argv[]);
NifF* fp = vbf = (NifF*) I[1];
struct enif_environment_t env;
- erts_pre_nif(&env, c_p, (struct erl_module_nif*)I[2]);
- reg[0] = r(0);
+#ifdef ERTS_SMP
+ ASSERT(c_p->scheduler_data);
+#endif
+ live_hf_end = c_p->mbuf;
+ erts_pre_nif(&env, c_p, (struct erl_module_nif*)I[2], NULL);
nif_bif_result = (*fp)(&env, bif_nif_arity, reg);
+ if (env.exception_thrown)
+ nif_bif_result = THE_NON_VALUE;
erts_post_nif(&env);
+
+ PROCESS_MAIN_CHK_LOCKS(c_p);
+ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+ ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR);
+ ASSERT(!env.exiting);
+ ASSERT(!ERTS_PROC_IS_EXITING(c_p));
}
- ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(nif_bif_result));
- PROCESS_MAIN_CHK_LOCKS(c_p);
- ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
DTRACE_NIF_RETURN(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]);
goto apply_bif_or_nif_epilogue;
@@ -3343,6 +3587,20 @@ void process_main(void)
* code[4]: Function pointer to BIF function
*/
+ if (!((FCALLS - 1) > 0 || (FCALLS - 1) > neg_o_reds)) {
+ /* If we have run out of reductions, we do a context
+ switch before calling the bif */
+ goto context_switch;
+ }
+
+ if (ERTS_MSACC_IS_ENABLED_CACHED_X()) {
+ if ((Eterm)I[-3] == am_ets) {
+ ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_ETS);
+ } else {
+ ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_BIF);
+ }
+ }
+
c_p->current = I-3; /* In case we apply process_info/1,2 or load_nif/1 */
c_p->i = I; /* In case we apply check_process_code/2. */
c_p->arity = 0; /* To allow garbage collection on ourselves
@@ -3351,34 +3609,43 @@ void process_main(void)
DTRACE_BIF_ENTRY(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]);
SWAPOUT;
+ ERTS_DBG_CHK_REDS(c_p, FCALLS - 1);
c_p->fcalls = FCALLS - 1;
vbf = (BifFunction) Arg(0);
PROCESS_MAIN_CHK_LOCKS(c_p);
bif_nif_arity = I[-1];
- ASSERT(bif_nif_arity <= 3);
+ ASSERT(bif_nif_arity <= 4);
ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
- reg[0] = r(0);
+ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
{
Eterm (*bf)(Process*, Eterm*, BeamInstr*) = vbf;
ASSERT(!ERTS_PROC_IS_EXITING(c_p));
+ live_hf_end = c_p->mbuf;
nif_bif_result = (*bf)(c_p, reg, I);
ASSERT(!ERTS_PROC_IS_EXITING(c_p) ||
is_non_value(nif_bif_result));
ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
PROCESS_MAIN_CHK_LOCKS(c_p);
}
-
+ /* We have to update the cache if we are enabled in order
+ to make sure no book keeping is done after we disabled
+ msacc. We don't always do this as it is quite expensive. */
+ if (ERTS_MSACC_IS_ENABLED_CACHED_X())
+ ERTS_MSACC_UPDATE_CACHE_X();
+ ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR);
DTRACE_BIF_RETURN(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]);
apply_bif_or_nif_epilogue:
ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
ERTS_HOLE_CHECK(c_p);
- if (c_p->mbuf) {
- nif_bif_result = erts_gc_after_bif_call(c_p, nif_bif_result,
- reg, bif_nif_arity);
+ if (ERTS_IS_GC_DESIRED(c_p)) {
+ nif_bif_result = erts_gc_after_bif_call_lhf(c_p, live_hf_end,
+ nif_bif_result,
+ reg, bif_nif_arity);
}
SWAPIN; /* There might have been a garbage collection. */
FCALLS = c_p->fcalls;
+ ERTS_DBG_CHK_REDS(c_p, FCALLS);
if (is_value(nif_bif_result)) {
r(0) = nif_bif_result;
CHECK_TERM(r(0));
@@ -3387,7 +3654,6 @@ void process_main(void)
Goto(*I);
} else if (c_p->freason == TRAP) {
SET_I(c_p->i);
- r(0) = reg[0];
if (c_p->flags & F_HIBERNATE_SCHED) {
c_p->flags &= ~F_HIBERNATE_SCHED;
goto do_schedule;
@@ -3409,21 +3675,21 @@ void process_main(void)
StoreBifResult(1, result);
}
+ OpCase(i_get_hash_cId):
+ {
+ Eterm arg;
+ Eterm result;
+
+ GetArg1(0, arg);
+ result = erts_pd_hash_get_with_hx(c_p, Arg(1), arg);
+ StoreBifResult(2, result);
+ }
+
{
Eterm case_end_val;
OpCase(case_end_x):
case_end_val = xb(Arg(0));
- goto do_case_end;
-
- OpCase(case_end_y):
- case_end_val = yb(Arg(0));
- goto do_case_end;
-
- OpCase(case_end_r):
- case_end_val = r(0);
-
- do_case_end:
c_p->fvalue = case_end_val;
c_p->freason = EXC_CASE_CLAUSE;
goto find_func_info;
@@ -3471,19 +3737,13 @@ void process_main(void)
goto do_bs_init_bits_known;
}
- OpCase(i_bs_init_bits_fail_heap_IjId): {
- /* tmp_arg1 was fetched by an i_fetch instruction */
- num_bits_term = tmp_arg1;
- alloc = Arg(0);
- I++;
+ OpCase(i_bs_init_bits_fail_heap_sIjId): {
+ GetArg1(0, num_bits_term);
+ alloc = Arg(1);
+ I += 2;
goto do_bs_init_bits;
}
- OpCase(i_bs_init_bits_fail_rjId): {
- num_bits_term = r(0);
- alloc = 0;
- goto do_bs_init_bits;
- }
OpCase(i_bs_init_bits_fail_yjId): {
num_bits_term = yb(Arg(0));
I++;
@@ -3581,8 +3841,6 @@ void process_main(void)
* Allocate the binary struct itself.
*/
bptr = erts_bin_nrml_alloc(num_bytes);
- bptr->flags = 0;
- bptr->orig_size = num_bytes;
erts_refc_init(&bptr->refc, 1);
erts_current_bin = (byte *) bptr->orig_bytes;
@@ -3605,52 +3863,48 @@ void process_main(void)
}
{
- OpCase(i_bs_init_fail_heap_IjId): {
- /* tmp_arg1 was fetched by an i_fetch instruction */
- tmp_arg2 = Arg(0);
- I++;
- goto do_bs_init;
- }
+ Eterm BsOp1, BsOp2;
- OpCase(i_bs_init_fail_rjId): {
- tmp_arg1 = r(0);
- tmp_arg2 = 0;
+ OpCase(i_bs_init_fail_heap_sIjId): {
+ GetArg1(0, BsOp1);
+ BsOp2 = Arg(1);
+ I += 2;
goto do_bs_init;
}
OpCase(i_bs_init_fail_yjId): {
- tmp_arg1 = yb(Arg(0));
- tmp_arg2 = 0;
+ BsOp1 = yb(Arg(0));
+ BsOp2 = 0;
I++;
goto do_bs_init;
}
OpCase(i_bs_init_fail_xjId): {
- tmp_arg1 = xb(Arg(0));
- tmp_arg2 = 0;
+ BsOp1 = xb(Arg(0));
+ BsOp2 = 0;
I++;
}
/* FALL THROUGH */
do_bs_init:
- if (is_small(tmp_arg1)) {
- Sint size = signed_val(tmp_arg1);
+ if (is_small(BsOp1)) {
+ Sint size = signed_val(BsOp1);
if (size < 0) {
goto badarg;
}
- tmp_arg1 = (Eterm) size;
+ BsOp1 = (Eterm) size;
} else {
Uint bytes;
- if (!term_to_Uint(tmp_arg1, &bytes)) {
+ if (!term_to_Uint(BsOp1, &bytes)) {
c_p->freason = bytes;
goto lb_Cl_error;
}
if ((bytes >> (8*sizeof(Uint)-3)) != 0) {
goto system_limit;
}
- tmp_arg1 = (Eterm) bytes;
+ BsOp1 = (Eterm) bytes;
}
- if (tmp_arg1 <= ERL_ONHEAP_BIN_LIMIT) {
+ if (BsOp1 <= ERL_ONHEAP_BIN_LIMIT) {
goto do_heap_bin_alloc;
} else {
goto do_proc_bin_alloc;
@@ -3658,15 +3912,15 @@ void process_main(void)
OpCase(i_bs_init_heap_IIId): {
- tmp_arg1 = Arg(0);
- tmp_arg2 = Arg(1);
+ BsOp1 = Arg(0);
+ BsOp2 = Arg(1);
I++;
goto do_proc_bin_alloc;
}
OpCase(i_bs_init_IId): {
- tmp_arg1 = Arg(0);
- tmp_arg2 = 0;
+ BsOp1 = Arg(0);
+ BsOp2 = 0;
}
/* FALL THROUGH */
do_proc_bin_alloc: {
@@ -3675,15 +3929,13 @@ void process_main(void)
erts_bin_offset = 0;
erts_writable_bin = 0;
- TestBinVHeap(tmp_arg1 / sizeof(Eterm),
- tmp_arg2 + PROC_BIN_SIZE + ERL_SUB_BIN_SIZE, Arg(1));
+ TestBinVHeap(BsOp1 / sizeof(Eterm),
+ BsOp2 + PROC_BIN_SIZE + ERL_SUB_BIN_SIZE, Arg(1));
/*
* Allocate the binary struct itself.
*/
- bptr = erts_bin_nrml_alloc(tmp_arg1);
- bptr->flags = 0;
- bptr->orig_size = tmp_arg1;
+ bptr = erts_bin_nrml_alloc(BsOp1);
erts_refc_init(&bptr->refc, 1);
erts_current_bin = (byte *) bptr->orig_bytes;
@@ -3693,28 +3945,28 @@ void process_main(void)
pb = (ProcBin *) HTOP;
HTOP += PROC_BIN_SIZE;
pb->thing_word = HEADER_PROC_BIN;
- pb->size = tmp_arg1;
+ pb->size = BsOp1;
pb->next = MSO(c_p).first;
MSO(c_p).first = (struct erl_off_heap_header*) pb;
pb->val = bptr;
pb->bytes = (byte*) bptr->orig_bytes;
pb->flags = 0;
- OH_OVERHEAD(&(MSO(c_p)), tmp_arg1 / sizeof(Eterm));
+ OH_OVERHEAD(&(MSO(c_p)), BsOp1 / sizeof(Eterm));
StoreBifResult(2, make_binary(pb));
}
OpCase(i_bs_init_heap_bin_heap_IIId): {
- tmp_arg1 = Arg(0);
- tmp_arg2 = Arg(1);
+ BsOp1 = Arg(0);
+ BsOp2 = Arg(1);
I++;
goto do_heap_bin_alloc;
}
OpCase(i_bs_init_heap_bin_IId): {
- tmp_arg1 = Arg(0);
- tmp_arg2 = 0;
+ BsOp1 = Arg(0);
+ BsOp2 = 0;
}
/* Fall through */
do_heap_bin_alloc:
@@ -3722,33 +3974,36 @@ void process_main(void)
ErlHeapBin* hb;
Uint bin_need;
- bin_need = heap_bin_size(tmp_arg1);
+ bin_need = heap_bin_size(BsOp1);
erts_bin_offset = 0;
erts_writable_bin = 0;
- TestHeap(bin_need+tmp_arg2+ERL_SUB_BIN_SIZE, Arg(1));
+ TestHeap(bin_need+BsOp2+ERL_SUB_BIN_SIZE, Arg(1));
hb = (ErlHeapBin *) HTOP;
HTOP += bin_need;
- hb->thing_word = header_heap_bin(tmp_arg1);
- hb->size = tmp_arg1;
+ hb->thing_word = header_heap_bin(BsOp1);
+ hb->size = BsOp1;
erts_current_bin = (byte *) hb->data;
- tmp_arg1 = make_binary(hb);
- StoreBifResult(2, tmp_arg1);
+ BsOp1 = make_binary(hb);
+ StoreBifResult(2, BsOp1);
}
}
- OpCase(i_bs_add_jId): {
- Uint Unit = Arg(1);
- if (is_both_small(tmp_arg1, tmp_arg2)) {
- Sint Arg1 = signed_val(tmp_arg1);
- Sint Arg2 = signed_val(tmp_arg2);
+ OpCase(bs_add_jssId): {
+ Eterm Op1, Op2;
+ Uint Unit = Arg(3);
+
+ GetArg2(1, Op1, Op2);
+ if (is_both_small(Op1, Op2)) {
+ Sint Arg1 = signed_val(Op1);
+ Sint Arg2 = signed_val(Op2);
if (Arg1 >= 0 && Arg2 >= 0) {
- BsSafeMul(Arg2, Unit, goto system_limit, tmp_arg1);
- tmp_arg1 += Arg1;
+ BsSafeMul(Arg2, Unit, goto system_limit, Op1);
+ Op1 += Arg1;
store_bs_add_result:
- if (MY_IS_SSMALL((Sint) tmp_arg1)) {
- tmp_arg1 = make_small(tmp_arg1);
+ if (Op1 <= MAX_SMALL) {
+ Op1 = make_small(Op1);
} else {
/*
* May generate a heap fragment, but in this
@@ -3760,10 +4015,10 @@ void process_main(void)
* references (such as the heap).
*/
SWAPOUT;
- tmp_arg1 = erts_make_integer(tmp_arg1, c_p);
+ Op1 = erts_make_integer(Op1, c_p);
HTOP = HEAP_TOP(c_p);
}
- StoreBifResult(2, tmp_arg1);
+ StoreBifResult(4, Op1);
}
goto badarg;
} else {
@@ -3786,16 +4041,16 @@ void process_main(void)
* an Uint, the reason is SYSTEM_LIMIT.
*/
- if (!term_to_Uint(tmp_arg1, &a)) {
+ if (!term_to_Uint(Op1, &a)) {
if (a == BADARG) {
goto badarg;
}
- if (!term_to_Uint(tmp_arg2, &b)) {
+ if (!term_to_Uint(Op2, &b)) {
c_p->freason = b;
goto lb_Cl_error;
}
goto system_limit;
- } else if (!term_to_Uint(tmp_arg2, &b)) {
+ } else if (!term_to_Uint(Op2, &b)) {
c_p->freason = b;
goto lb_Cl_error;
}
@@ -3805,8 +4060,8 @@ void process_main(void)
*/
BsSafeMul(b, Unit, goto system_limit, c);
- tmp_arg1 = a + c;
- if (tmp_arg1 < a) {
+ Op1 = a + c;
+ if (Op1 < a) {
/*
* If the result is less than one of the
* arguments, there must have been an overflow.
@@ -3828,52 +4083,47 @@ void process_main(void)
}
/*
- * tmp_arg1 = Number of bytes to build
- * tmp_arg2 = Source binary
- * Operands: Fail ExtraHeap Live Unit Dst
+ * x(SCRATCH_X_REG);
+ * Operands: Fail ExtraHeap Live Unit Size Dst
*/
- OpCase(i_bs_append_jIIId): {
+ OpCase(i_bs_append_jIIIsd): {
Uint live = Arg(2);
Uint res;
+ Eterm Size;
- SWAPOUT;
- reg[0] = r(0);
- reg[live] = tmp_arg2;
- res = erts_bs_append(c_p, reg, live, tmp_arg1, Arg(1), Arg(3));
- r(0) = reg[0];
- SWAPIN;
+ GetArg1(4, Size);
+ HEAVY_SWAPOUT;
+ reg[live] = x(SCRATCH_X_REG);
+ res = erts_bs_append(c_p, reg, live, Size, Arg(1), Arg(3));
+ HEAVY_SWAPIN;
if (is_non_value(res)) {
/* c_p->freason is already set (may be either BADARG or SYSTEM_LIMIT). */
goto lb_Cl_error;
}
- StoreBifResult(4, res);
+ StoreBifResult(5, res);
}
/*
- * tmp_arg1 = Number of bytes to build
- * tmp_arg2 = Source binary
- * Operands: Fail Unit Dst
+ * Operands: Fail Size Src Unit Dst
*/
- OpCase(i_bs_private_append_jId): {
+ OpCase(i_bs_private_append_jIssd): {
Eterm res;
+ Eterm Size, Src;
- res = erts_bs_private_append(c_p, tmp_arg2, tmp_arg1, Arg(1));
+ GetArg2(2, Size, Src);
+ res = erts_bs_private_append(c_p, Src, Size, Arg(1));
if (is_non_value(res)) {
/* c_p->freason is already set (may be either BADARG or SYSTEM_LIMIT). */
goto lb_Cl_error;
}
- StoreBifResult(2, res);
+ StoreBifResult(4, res);
}
- /*
- * tmp_arg1 = Initial size of writable binary
- * Operands: Live Dst
- */
OpCase(bs_init_writable): {
- SWAPOUT;
+ HEAVY_SWAPOUT;
r(0) = erts_bs_init_writable(c_p, r(0));
- SWAPIN;
+ HEAVY_SWAPIN;
Next(0);
}
@@ -3930,7 +4180,7 @@ void process_main(void)
StoreBifResult(1, result);
}
- OpCase(i_bs_put_utf16_jIs): {
+ OpCase(bs_put_utf16_jIs): {
Eterm arg;
GetArg1(2, arg);
@@ -3964,26 +4214,29 @@ void process_main(void)
/*
* Only used for validating a value matched out.
- *
- * tmp_arg1 = Integer to validate
- * tmp_arg2 = Match context
*/
- OpCase(i_bs_validate_unicode_retract_j): {
- /*
- * There is no need to untag the integer, but it IS necessary
- * to make sure it is small (a bignum pointer could fall in
- * the valid range).
- */
- if (is_not_small(tmp_arg1) || tmp_arg1 > make_small(0x10FFFFUL) ||
- (make_small(0xD800UL) <= tmp_arg1 &&
- tmp_arg1 <= make_small(0xDFFFUL))) {
- ErlBinMatchBuffer *mb = ms_matchbuffer(tmp_arg2);
+ OpCase(i_bs_validate_unicode_retract_jss): {
+ Eterm i; /* Integer to validate */
- mb->offset -= 32;
- goto badarg;
- }
- Next(1);
- }
+ /*
+ * There is no need to untag the integer, but it IS necessary
+ * to make sure it is small (a bignum pointer could fall in
+ * the valid range).
+ */
+
+ GetArg1(1, i);
+ if (is_not_small(i) || i > make_small(0x10FFFFUL) ||
+ (make_small(0xD800UL) <= i && i <= make_small(0xDFFFUL))) {
+ Eterm ms; /* Match context */
+ ErlBinMatchBuffer* mb;
+
+ GetArg1(2, ms);
+ mb = ms_matchbuffer(ms);
+ mb->offset -= 32;
+ goto badarg;
+ }
+ Next(3);
+ }
/*
* Matching of binaries.
@@ -3995,9 +4248,6 @@ void process_main(void)
Uint slots;
Eterm context;
- OpCase(i_bs_start_match2_rfIId): {
- context = r(0);
-
do_start_match:
slots = Arg(2);
if (!is_boxed(context)) {
@@ -4044,7 +4294,7 @@ void process_main(void)
ClauseFail();
}
NextPF(4, next);
- }
+
OpCase(i_bs_start_match2_xfIId): {
context = xb(Arg(0));
I++;
@@ -4057,18 +4307,6 @@ void process_main(void)
}
}
- OpCase(bs_test_zero_tail2_fr): {
- BeamInstr *next;
- ErlBinMatchBuffer *_mb;
-
- PreFetch(1, next);
- _mb = (ErlBinMatchBuffer*) ms_matchbuffer(r(0));
- if (_mb->size != _mb->offset) {
- ClauseFail();
- }
- NextPF(1, next);
- }
-
OpCase(bs_test_zero_tail2_fx): {
BeamInstr *next;
ErlBinMatchBuffer *_mb;
@@ -4081,16 +4319,6 @@ void process_main(void)
NextPF(2, next);
}
- OpCase(bs_test_tail_imm2_frI): {
- BeamInstr *next;
- ErlBinMatchBuffer *_mb;
- PreFetch(2, next);
- _mb = ms_matchbuffer(r(0));
- if (_mb->size - _mb->offset != Arg(1)) {
- ClauseFail();
- }
- NextPF(2, next);
- }
OpCase(bs_test_tail_imm2_fxI): {
BeamInstr *next;
ErlBinMatchBuffer *_mb;
@@ -4102,16 +4330,6 @@ void process_main(void)
NextPF(3, next);
}
- OpCase(bs_test_unit_frI): {
- BeamInstr *next;
- ErlBinMatchBuffer *_mb;
- PreFetch(2, next);
- _mb = ms_matchbuffer(r(0));
- if ((_mb->size - _mb->offset) % Arg(1)) {
- ClauseFail();
- }
- NextPF(2, next);
- }
OpCase(bs_test_unit_fxI): {
BeamInstr *next;
ErlBinMatchBuffer *_mb;
@@ -4123,16 +4341,6 @@ void process_main(void)
NextPF(3, next);
}
- OpCase(bs_test_unit8_fr): {
- BeamInstr *next;
- ErlBinMatchBuffer *_mb;
- PreFetch(1, next);
- _mb = ms_matchbuffer(r(0));
- if ((_mb->size - _mb->offset) & 7) {
- ClauseFail();
- }
- NextPF(1, next);
- }
OpCase(bs_test_unit8_fx): {
BeamInstr *next;
ErlBinMatchBuffer *_mb;
@@ -4147,19 +4355,11 @@ void process_main(void)
{
Eterm bs_get_integer8_context;
- OpCase(i_bs_get_integer_8_rfd): {
- bs_get_integer8_context = r(0);
- goto do_bs_get_integer_8;
- }
-
OpCase(i_bs_get_integer_8_xfd): {
- bs_get_integer8_context = xb(Arg(0));
- I++;
- }
-
- do_bs_get_integer_8: {
ErlBinMatchBuffer *_mb;
Eterm _result;
+ bs_get_integer8_context = xb(Arg(0));
+ I++;
_mb = ms_matchbuffer(bs_get_integer8_context);
if (_mb->size - _mb->offset < 8) {
ClauseFail();
@@ -4177,15 +4377,10 @@ void process_main(void)
{
Eterm bs_get_integer_16_context;
- OpCase(i_bs_get_integer_16_rfd):
- bs_get_integer_16_context = r(0);
- goto do_bs_get_integer_16;
-
OpCase(i_bs_get_integer_16_xfd):
bs_get_integer_16_context = xb(Arg(0));
I++;
- do_bs_get_integer_16:
{
ErlBinMatchBuffer *_mb;
Eterm _result;
@@ -4206,17 +4401,10 @@ void process_main(void)
{
Eterm bs_get_integer_32_context;
- OpCase(i_bs_get_integer_32_rfId):
- bs_get_integer_32_context = r(0);
- goto do_bs_get_integer_32;
-
-
OpCase(i_bs_get_integer_32_xfId):
bs_get_integer_32_context = xb(Arg(0));
I++;
-
- do_bs_get_integer_32:
{
ErlBinMatchBuffer *_mb;
Uint32 _integer;
@@ -4229,11 +4417,11 @@ void process_main(void)
_integer = get_int32(_mb->base + _mb->offset/8);
}
_mb->offset += 32;
-#if !defined(ARCH_64) || HALFWORD_HEAP
+#if !defined(ARCH_64)
if (IS_USMALL(0, _integer)) {
#endif
_result = make_small(_integer);
-#if !defined(ARCH_64) || HALFWORD_HEAP
+#if !defined(ARCH_64)
} else {
TestHeap(BIG_UINT_HEAP_SIZE, Arg(1));
_result = uint_to_big((Uint) _integer, HTOP);
@@ -4245,91 +4433,82 @@ void process_main(void)
}
}
- /* Operands: Size Live Fail Flags Dst */
- OpCase(i_bs_get_integer_imm_rIIfId): {
- tmp_arg1 = r(0);
- /* Operands: Size Live Fail Flags Dst */
- goto do_bs_get_integer_imm_test_heap;
- }
+ {
+ Eterm Ms, Sz;
- /* Operands: x(Reg) Size Live Fail Flags Dst */
+ /* Operands: x(Reg) Size Live Fail Flags Dst */
OpCase(i_bs_get_integer_imm_xIIfId): {
- tmp_arg1 = xb(Arg(0));
- I++;
- /* Operands: Size Live Fail Flags Dst */
- goto do_bs_get_integer_imm_test_heap;
- }
-
- /*
- * tmp_arg1 = match context
- * Operands: Size Live Fail Flags Dst
- */
- do_bs_get_integer_imm_test_heap: {
- Uint wordsneeded;
- tmp_arg2 = Arg(0);
- wordsneeded = 1+WSIZE(NBYTES(tmp_arg2));
- TestHeapPreserve(wordsneeded, Arg(1), tmp_arg1);
- I += 2;
- /* Operands: Fail Flags Dst */
- goto do_bs_get_integer_imm;
- }
-
- /* Operands: Size Fail Flags Dst */
- OpCase(i_bs_get_integer_small_imm_rIfId): {
- tmp_arg1 = r(0);
- tmp_arg2 = Arg(0);
- I++;
- /* Operands: Fail Flags Dst */
- goto do_bs_get_integer_imm;
- }
+ Uint wordsneeded;
+ Ms = xb(Arg(0));
+ Sz = Arg(1);
+ wordsneeded = 1+WSIZE(NBYTES(Sz));
+ TestHeapPreserve(wordsneeded, Arg(2), Ms);
+ I += 3;
+ /* Operands: Fail Flags Dst */
+ goto do_bs_get_integer_imm;
+ }
- /* Operands: x(Reg) Size Fail Flags Dst */
+ /* Operands: x(Reg) Size Fail Flags Dst */
OpCase(i_bs_get_integer_small_imm_xIfId): {
- tmp_arg1 = xb(Arg(0));
- tmp_arg2 = Arg(1);
- I += 2;
- /* Operands: Fail Flags Dst */
- goto do_bs_get_integer_imm;
- }
+ Ms = xb(Arg(0));
+ Sz = Arg(1);
+ I += 2;
+ /* Operands: Fail Flags Dst */
+ goto do_bs_get_integer_imm;
+ }
- /*
- * tmp_arg1 = match context
- * tmp_arg2 = size of field
- * Operands: Fail Flags Dst
- */
+ /*
+ * Ms = match context
+ * Sz = size of field
+ * Operands: Fail Flags Dst
+ */
do_bs_get_integer_imm: {
- ErlBinMatchBuffer* mb;
- Eterm result;
+ ErlBinMatchBuffer* mb;
+ Eterm result;
- mb = ms_matchbuffer(tmp_arg1);
- LIGHT_SWAPOUT;
- result = erts_bs_get_integer_2(c_p, tmp_arg2, Arg(1), mb);
- LIGHT_SWAPIN;
- HEAP_SPACE_VERIFIED(0);
- if (is_non_value(result)) {
- ClauseFail();
+ mb = ms_matchbuffer(Ms);
+ LIGHT_SWAPOUT;
+ result = erts_bs_get_integer_2(c_p, Sz, Arg(1), mb);
+ LIGHT_SWAPIN;
+ HEAP_SPACE_VERIFIED(0);
+ if (is_non_value(result)) {
+ ClauseFail();
+ }
+ StoreBifResult(2, result);
}
- StoreBifResult(2, result);
}
/*
- * tmp_arg1 = Match context
- * tmp_arg2 = Size field
- * Operands: Fail Live FlagsAndUnit Dst
+ * Operands: Fail Live FlagsAndUnit Ms Sz Dst
*/
- OpCase(i_bs_get_integer_fIId): {
+ OpCase(i_bs_get_integer_fIIssd): {
Uint flags;
Uint size;
+ Eterm Ms;
+ Eterm Sz;
ErlBinMatchBuffer* mb;
Eterm result;
flags = Arg(2);
- BsGetFieldSize(tmp_arg2, (flags >> 3), ClauseFail(), size);
+ GetArg2(3, Ms, Sz);
+ BsGetFieldSize(Sz, (flags >> 3), ClauseFail(), size);
if (size >= SMALL_BITS) {
- Uint wordsneeded = 1+WSIZE(NBYTES((Uint) size));
- TestHeapPreserve(wordsneeded, Arg(1), tmp_arg1);
+ Uint wordsneeded;
+ /* Check bits size before potential gc.
+ * We do not want a gc and then realize we don't need
+ * the allocated space (i.e. if the op fails).
+ *
+ * Remember to re-acquire the matchbuffer after gc.
+ */
+
+ mb = ms_matchbuffer(Ms);
+ if (mb->size - mb->offset < size) {
+ ClauseFail();
+ }
+ wordsneeded = 1+WSIZE(NBYTES((Uint) size));
+ TestHeapPreserve(wordsneeded, Arg(1), Ms);
}
- mb = ms_matchbuffer(tmp_arg1);
+ mb = ms_matchbuffer(Ms);
LIGHT_SWAPOUT;
result = erts_bs_get_integer_2(c_p, size, flags, mb);
LIGHT_SWAPIN;
@@ -4337,18 +4516,13 @@ void process_main(void)
if (is_non_value(result)) {
ClauseFail();
}
- StoreBifResult(3, result);
+ StoreBifResult(5, result);
}
{
Eterm get_utf8_context;
/* Operands: MatchContext Fail Dst */
- OpCase(i_bs_get_utf8_rfd): {
- get_utf8_context = r(0);
- goto do_bs_get_utf8;
- }
-
OpCase(i_bs_get_utf8_xfd): {
get_utf8_context = xb(Arg(0));
I++;
@@ -4359,7 +4533,7 @@ void process_main(void)
* Operands: Fail Dst
*/
- do_bs_get_utf8: {
+ {
Eterm result = erts_bs_get_utf8(ms_matchbuffer(get_utf8_context));
if (is_non_value(result)) {
ClauseFail();
@@ -4372,12 +4546,7 @@ void process_main(void)
Eterm get_utf16_context;
/* Operands: MatchContext Fail Flags Dst */
- OpCase(i_bs_get_utf16_rfId): {
- get_utf16_context = r(0);
- goto do_bs_get_utf16;
- }
-
- OpCase(i_bs_get_utf16_xfId): {
+ OpCase(i_bs_get_utf16_xfId): {
get_utf16_context = xb(Arg(0));
I++;
}
@@ -4386,7 +4555,7 @@ void process_main(void)
* get_utf16_context = match_context
* Operands: Fail Flags Dst
*/
- do_bs_get_utf16: {
+ {
Eterm result = erts_bs_get_utf16(ms_matchbuffer(get_utf16_context),
Arg(1));
if (is_non_value(result)) {
@@ -4405,26 +4574,10 @@ void process_main(void)
Uint orig;
Uint hole_size;
- OpCase(bs_context_to_binary_r): {
- context_to_binary_context = x0;
- I -= 2;
- goto do_context_to_binary;
- }
-
- /* Unfortunately, inlining can generate this instruction. */
- OpCase(bs_context_to_binary_y): {
- context_to_binary_context = yb(Arg(0));
- goto do_context_to_binary0;
- }
-
- OpCase(bs_context_to_binary_x): {
+ OpCase(bs_context_to_binary_x):
context_to_binary_context = xb(Arg(0));
-
- do_context_to_binary0:
I--;
- }
- do_context_to_binary:
if (is_boxed(context_to_binary_context) &&
header_is_bin_matchstate(*boxed_val(context_to_binary_context))) {
ErlBinMatchState* ms;
@@ -4436,17 +4589,11 @@ void process_main(void)
}
Next(2);
- OpCase(i_bs_get_binary_all_reuse_rfI): {
- context_to_binary_context = x0;
- goto do_bs_get_binary_all_reuse;
- }
-
OpCase(i_bs_get_binary_all_reuse_xfI): {
context_to_binary_context = xb(Arg(0));
I++;
}
- do_bs_get_binary_all_reuse:
mb = ms_matchbuffer(context_to_binary_context);
size = mb->size - mb->offset;
if (size % Arg(1) != 0) {
@@ -4474,16 +4621,11 @@ void process_main(void)
{
Eterm match_string_context;
- OpCase(i_bs_match_string_rfII): {
- match_string_context = r(0);
- goto do_bs_match_string;
- }
OpCase(i_bs_match_string_xfII): {
match_string_context = xb(Arg(0));
I++;
}
- do_bs_match_string:
{
BeamInstr *next;
byte* bytes;
@@ -4511,14 +4653,6 @@ void process_main(void)
}
}
- OpCase(i_bs_save2_rI): {
- BeamInstr *next;
- ErlBinMatchState *_ms;
- PreFetch(1, next);
- _ms = (ErlBinMatchState*) boxed_val((Eterm) r(0));
- _ms->save_offset[Arg(0)] = _ms->mb.offset;
- NextPF(1, next);
- }
OpCase(i_bs_save2_xI): {
BeamInstr *next;
ErlBinMatchState *_ms;
@@ -4528,14 +4662,6 @@ void process_main(void)
NextPF(2, next);
}
- OpCase(i_bs_restore2_rI): {
- BeamInstr *next;
- ErlBinMatchState *_ms;
- PreFetch(1, next);
- _ms = (ErlBinMatchState*) boxed_val((Eterm) r(0));
- _ms->mb.offset = _ms->save_offset[Arg(0)];
- NextPF(1, next);
- }
OpCase(i_bs_restore2_xI): {
BeamInstr *next;
ErlBinMatchState *_ms;
@@ -4570,7 +4696,7 @@ void process_main(void)
SWAPOUT; /* Needed for shared heap */
ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
- erts_trace_return(c_p, code, r(0), E+1/*Process tracer*/);
+ erts_trace_return(c_p, code, r(0), ERTS_TRACER_FROM_ETERM(E+1)/* tracer */);
ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
SWAPIN;
c_p->cp = NULL;
@@ -4582,11 +4708,9 @@ void process_main(void)
OpCase(i_generic_breakpoint): {
BeamInstr real_I;
ASSERT(I[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI));
- SWAPOUT;
- reg[0] = r(0);
+ HEAVY_SWAPOUT;
real_I = erts_generic_breakpoint(c_p, I, reg);
- r(0) = reg[0];
- SWAPIN;
+ HEAVY_SWAPIN;
ASSERT(VALID_INSTR(real_I));
Goto(real_I);
}
@@ -4645,7 +4769,7 @@ void process_main(void)
BeamInstr *next;
PreFetch(2, next);
- GetR(0, targ1);
+ targ1 = REG_TARGET(Arg(0));
/* Arg(0) == HEADER_FLONUM */
GET_DOUBLE(targ1, *(FloatDef*)ADD_BYTE_OFFSET(freg, fr));
NextPF(2, next);
@@ -4665,7 +4789,7 @@ void process_main(void)
Eterm fr = Arg(1);
BeamInstr *next;
- GetR(0, targ1);
+ targ1 = REG_TARGET(Arg(0));
PreFetch(2, next);
if (is_small(targ1)) {
fb(fr) = (double) signed_val(targ1);
@@ -4684,7 +4808,7 @@ void process_main(void)
#ifdef NO_FPE_SIGNALS
OpCase(fclearerror):
OpCase(i_fcheckerror):
- erl_exit(1, "fclearerror/i_fcheckerror without fpe signals (beam_emu)");
+ erts_exit(ERTS_ERROR_EXIT, "fclearerror/i_fcheckerror without fpe signals (beam_emu)");
# define ERTS_NO_FPE_CHECK_INIT ERTS_FP_CHECK_INIT
# define ERTS_NO_FPE_ERROR ERTS_FP_ERROR
#else
@@ -4761,7 +4885,13 @@ void process_main(void)
#ifdef HIPE
{
- unsigned cmd;
+#define HIPE_MODE_SWITCH(Cmd) \
+ SWAPOUT; \
+ ERTS_DBG_CHK_REDS(c_p, FCALLS); \
+ c_p->fcalls = FCALLS; \
+ c_p->def_arg_reg[4] = -neg_o_reds; \
+ c_p = hipe_mode_switch(c_p, Cmd, reg); \
+ goto L_post_hipe_mode_switch
OpCase(hipe_trap_call): {
/*
@@ -4774,50 +4904,50 @@ void process_main(void)
* ... remainder of original BEAM code
*/
ASSERT(I[-5] == (Uint) OpCode(i_func_info_IaaI));
- c_p->hipe.ncallee = (void(*)(void)) I[-4];
- cmd = HIPE_MODE_SWITCH_CMD_CALL | (I[-1] << 8);
+ c_p->hipe.u.ncallee = (void(*)(void)) I[-4];
++hipe_trap_count;
- goto L_hipe_mode_switch;
+ HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_CALL | (I[-1] << 8));
}
OpCase(hipe_trap_call_closure): {
ASSERT(I[-5] == (Uint) OpCode(i_func_info_IaaI));
- c_p->hipe.ncallee = (void(*)(void)) I[-4];
- cmd = HIPE_MODE_SWITCH_CMD_CALL_CLOSURE | (I[-1] << 8);
+ c_p->hipe.u.ncallee = (void(*)(void)) I[-4];
++hipe_trap_count;
- goto L_hipe_mode_switch;
+ HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_CALL_CLOSURE | (I[-1] << 8));
}
OpCase(hipe_trap_return): {
- cmd = HIPE_MODE_SWITCH_CMD_RETURN;
- goto L_hipe_mode_switch;
+ HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_RETURN);
}
OpCase(hipe_trap_throw): {
- cmd = HIPE_MODE_SWITCH_CMD_THROW;
- goto L_hipe_mode_switch;
+ HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_THROW);
}
OpCase(hipe_trap_resume): {
- cmd = HIPE_MODE_SWITCH_CMD_RESUME;
- goto L_hipe_mode_switch;
+ HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_RESUME);
}
- L_hipe_mode_switch:
- /* XXX: this abuse of def_arg_reg[] is horrid! */
- SWAPOUT;
- c_p->fcalls = FCALLS;
- c_p->def_arg_reg[4] = -neg_o_reds;
- reg[0] = r(0);
- c_p = hipe_mode_switch(c_p, cmd, reg);
- reg = ERTS_PROC_GET_SCHDATA(c_p)->x_reg_array;
- freg = ERTS_PROC_GET_SCHDATA(c_p)->f_reg_array;
+#undef HIPE_MODE_SWITCH
+
+ L_post_hipe_mode_switch:
+#ifdef DEBUG
+ pid = c_p->common.id; /* may have switched process... */
+#endif
+ reg = erts_proc_sched_data(c_p)->x_reg_array;
+ freg = erts_proc_sched_data(c_p)->f_reg_array;
ERL_BITS_RELOAD_STATEP(c_p);
+ /* XXX: this abuse of def_arg_reg[] is horrid! */
neg_o_reds = -c_p->def_arg_reg[4];
FCALLS = c_p->fcalls;
SWAPIN;
- switch( c_p->def_arg_reg[3] ) { /* Halfword wont work with hipe yet! */
+ ERTS_DBG_CHK_REDS(c_p, FCALLS);
+ switch( c_p->def_arg_reg[3] ) {
case HIPE_MODE_SWITCH_RES_RETURN:
ASSERT(is_value(reg[0]));
- MoveReturn(reg[0], r(0));
- case HIPE_MODE_SWITCH_RES_CALL:
+ SET_I(c_p->cp);
+ c_p->cp = 0;
+ Goto(*I);
+ case HIPE_MODE_SWITCH_RES_CALL_EXPORTED:
+ c_p->i = c_p->hipe.u.callee_exp->addressv[erts_active_code_ix()];
+ /*fall through*/
+ case HIPE_MODE_SWITCH_RES_CALL_BEAM:
SET_I(c_p->i);
- r(0) = reg[0];
Dispatch();
case HIPE_MODE_SWITCH_RES_CALL_CLOSURE:
/* This can be used to call any function value, but currently it's
@@ -4826,9 +4956,8 @@ void process_main(void)
BeamInstr *next;
next = call_fun(c_p, c_p->arity - 1, reg, THE_NON_VALUE);
- SWAPIN;
+ HEAVY_SWAPIN;
if (next != NULL) {
- r(0) = reg[0];
SET_I(next);
Dispatchfun();
}
@@ -4839,7 +4968,7 @@ void process_main(void)
I = handle_error(c_p, I, reg, NULL);
goto post_error_handling;
default:
- erl_exit(1, "hipe_mode_switch: result %u\n", c_p->def_arg_reg[3]);
+ erts_exit(ERTS_ERROR_EXIT, "hipe_mode_switch: result %u\n", c_p->def_arg_reg[3]);
}
}
OpCase(hipe_call_count): {
@@ -4876,22 +5005,54 @@ void process_main(void)
}
OpCase(i_hibernate): {
- SWAPOUT;
+ HEAVY_SWAPOUT;
if (erts_hibernate(c_p, r(0), x(1), x(2), reg)) {
+ FCALLS = c_p->fcalls;
c_p->flags &= ~F_HIBERNATE_SCHED;
goto do_schedule;
} else {
+ HEAVY_SWAPIN;
I = handle_error(c_p, I, reg, hibernate_3);
goto post_error_handling;
}
}
+ /* This is optimised as an instruction because
+ it has to be very very fast */
+ OpCase(i_perf_counter): {
+ BeamInstr* next;
+ ErtsSysPerfCounter ts;
+ PreFetch(0, next);
+
+ ts = erts_sys_perf_counter();
+
+ if (IS_SSMALL(ts)) {
+ r(0) = make_small((Sint)ts);
+ } else {
+ TestHeap(ERTS_SINT64_HEAP_SIZE(ts),0);
+ r(0) = make_big(HTOP);
+#if defined(ARCH_32) || HALFWORD_HEAP
+ if (ts >= (((Uint64) 1) << 32)) {
+ *HTOP = make_pos_bignum_header(2);
+ BIG_DIGIT(HTOP, 0) = (Uint) (ts & ((Uint) 0xffffffff));
+ BIG_DIGIT(HTOP, 1) = (Uint) ((ts >> 32) & ((Uint) 0xffffffff));
+ HTOP += 3;
+ }
+ else
+#endif
+ {
+ *HTOP = make_pos_bignum_header(1);
+ BIG_DIGIT(HTOP, 0) = (Uint) ts;
+ HTOP += 2;
+ }
+ }
+ NextPF(0, next);
+ }
+
OpCase(i_debug_breakpoint): {
- SWAPOUT;
- reg[0] = r(0);
+ HEAVY_SWAPOUT;
I = call_error_handler(c_p, I-3, reg, am_breakpoint);
- r(0) = reg[0];
- SWAPIN;
+ HEAVY_SWAPIN;
if (I) {
Goto(*I);
}
@@ -4919,7 +5080,7 @@ void process_main(void)
OpCase(label_L):
OpCase(on_load):
OpCase(line_I):
- erl_exit(1, "meta op\n");
+ erts_exit(ERTS_ERROR_EXIT, "meta op\n");
/*
* One-time initialization of Beam emulator.
@@ -4932,26 +5093,19 @@ void process_main(void)
#ifndef NO_JUMP_TABLE
#ifdef ERTS_OPCODE_COUNTER_SUPPORT
-
- /* Are tables correctly generated by beam_makeops? */
- ASSERT(sizeof(counting_opcodes) == sizeof(opcodes));
-
- if (count_instructions) {
#ifdef DEBUG
- counting_opcodes[op_catch_end_y] = LabelAddr(lb_catch_end_y);
+ counting_opcodes[op_catch_end_y] = LabelAddr(lb_catch_end_y);
#endif
- counting_opcodes[op_i_func_info_IaaI] = LabelAddr(lb_i_func_info_IaaI);
- beam_ops = counting_opcodes;
- }
- else
-#endif /* #ifndef ERTS_OPCODE_COUNTER_SUPPORT */
- {
- beam_ops = opcodes;
- }
+ counting_opcodes[op_i_func_info_IaaI] = LabelAddr(lb_i_func_info_IaaI);
+ beam_ops = counting_opcodes;
+#else /* #ifndef ERTS_OPCODE_COUNTER_SUPPORT */
+ beam_ops = opcodes;
+#endif /* ERTS_OPCODE_COUNTER_SUPPORT */
#endif /* NO_JUMP_TABLE */
em_call_error_handler = OpCode(call_error_handler);
em_apply_bif = OpCode(apply_bif);
+ em_call_nif = OpCode(call_nif);
beam_apply[0] = (BeamInstr) OpCode(i_apply);
beam_apply[1] = (BeamInstr) OpCode(normal_exit);
@@ -4980,7 +5134,7 @@ void process_main(void)
}
#ifdef NO_JUMP_TABLE
default:
- erl_exit(1, "unexpected op code %d\n",Go);
+ erts_exit(ERTS_ERROR_EXIT, "unexpected op code %d\n",Go);
}
#endif
return; /* Never executed */
@@ -4999,6 +5153,278 @@ void process_main(void)
}
}
+/*
+ * erts_dirty_process_main() is what dirty schedulers execute. Since they handle
+ * only NIF calls they do not need to be able to execute all BEAM
+ * instructions.
+ */
+void erts_dirty_process_main(ErtsSchedulerData *esdp)
+{
+#ifdef ERTS_DIRTY_SCHEDULERS
+ Process* c_p = NULL;
+ ErtsMonotonicTime start_time;
+#ifdef DEBUG
+ ERTS_DECLARE_DUMMY(Eterm pid);
+#endif
+
+ /* Pointer to X registers: x(1)..x(N); reg[0] is used when doing GC,
+ * in all other cases x0 is used.
+ */
+ register Eterm* reg REG_xregs = NULL;
+
+ /*
+ * Top of heap (next free location); grows upwards.
+ */
+ register Eterm* HTOP REG_htop = NULL;
+
+ /* Stack pointer. Grows downwards; points
+ * to last item pushed (normally a saved
+ * continuation pointer).
+ */
+ register Eterm* E REG_stop = NULL;
+
+ /*
+ * Pointer to next threaded instruction.
+ */
+ register BeamInstr *I REG_I = NULL;
+
+ ERTS_MSACC_DECLARE_CACHE_X() /* a cached value of the tsd pointer for msacc */
+
+ /*
+ * start_time always positive for dirty CPU schedulers,
+ * and negative for dirty I/O schedulers.
+ */
+
+ if (ERTS_SCHEDULER_IS_DIRTY_CPU(esdp)) {
+ start_time = erts_get_monotonic_time(NULL);
+ ASSERT(start_time >= 0);
+ }
+ else {
+ start_time = ERTS_SINT64_MIN;
+ ASSERT(start_time < 0);
+ }
+
+ goto do_dirty_schedule;
+
+ context_switch:
+ c_p->arity = I[-1];
+ c_p->current = I-3; /* Pointer to Mod, Func, Arity */
+
+ {
+ int reds_used;
+ Eterm* argp;
+ int i;
+
+ /*
+ * Make sure that there is enough room for the argument registers to be saved.
+ */
+ if (c_p->arity > c_p->max_arg_reg) {
+ /*
+ * Yes, this is an expensive operation, but you only pay it the first
+ * time you call a function with more than 6 arguments which is
+ * scheduled out. This is better than paying for 26 words of wasted
+ * space for most processes which never call functions with more than
+ * 6 arguments.
+ */
+ Uint size = c_p->arity * sizeof(c_p->arg_reg[0]);
+ if (c_p->arg_reg != c_p->def_arg_reg) {
+ c_p->arg_reg = (Eterm *) erts_realloc(ERTS_ALC_T_ARG_REG,
+ (void *) c_p->arg_reg,
+ size);
+ } else {
+ c_p->arg_reg = (Eterm *) erts_alloc(ERTS_ALC_T_ARG_REG, size);
+ }
+ c_p->max_arg_reg = c_p->arity;
+ }
+
+ /*
+ * Save the argument registers and everything else.
+ */
+
+ argp = c_p->arg_reg;
+ for (i = c_p->arity - 1; i >= 0; i--) {
+ argp[i] = reg[i];
+ }
+ SWAPOUT;
+ c_p->i = I;
+
+ do_dirty_schedule:
+
+ if (start_time < 0) {
+ /*
+ * Dirty I/O scheduler:
+ * One reduction consumed regardless of
+ * time spent in the dirty NIF.
+ */
+ reds_used = esdp->virtual_reds + 1;
+ }
+ else {
+ /*
+ * Dirty CPU scheduler:
+ * Currently two reductions consumed per
+ * micro second spent in the dirty NIF.
+ */
+ ErtsMonotonicTime time;
+ time = erts_get_monotonic_time(esdp);
+ time -= start_time;
+ time = ERTS_MONOTONIC_TO_USEC(time);
+ time *= (CONTEXT_REDS-1)/1000 + 1;
+ ASSERT(time >= 0);
+ if (time == 0)
+ time = 1; /* At least one reduction */
+ time += esdp->virtual_reds;
+ reds_used = time > INT_MAX ? INT_MAX : (int) time;
+ }
+
+ PROCESS_MAIN_CHK_LOCKS(c_p);
+ ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
+ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+ c_p = erts_schedule(esdp, c_p, reds_used);
+
+ if (start_time >= 0) {
+ start_time = erts_get_monotonic_time(esdp);
+ ASSERT(start_time >= 0);
+ }
+ }
+
+ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+#ifdef DEBUG
+ pid = c_p->common.id; /* Save for debugging purposes */
+#endif
+ ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
+ PROCESS_MAIN_CHK_LOCKS(c_p);
+
+ ASSERT(!(c_p->flags & F_HIPE_MODE));
+ ERTS_MSACC_UPDATE_CACHE_X();
+
+ reg = esdp->x_reg_array;
+ {
+ Eterm* argp;
+ int i;
+
+ argp = c_p->arg_reg;
+ for (i = c_p->arity - 1; i >= 0; i--) {
+ reg[i] = argp[i];
+ CHECK_TERM(reg[i]);
+ }
+
+ /*
+ * We put the original reduction count in the process structure, to reduce
+ * the code size (referencing a field in a struct through a pointer stored
+ * in a register gives smaller code than referencing a global variable).
+ */
+
+ I = c_p->i;
+
+ ASSERT(BeamOp(op_call_nif) == (BeamInstr *) *I);
+
+ /*
+ * Set fcalls even though we ignore it, so we don't
+ * confuse code accessing it...
+ */
+ if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p))
+ c_p->fcalls = 0;
+ else
+ c_p->fcalls = CONTEXT_REDS;
+
+ SWAPIN;
+
+#ifdef USE_VM_PROBES
+ if (DTRACE_ENABLED(process_scheduled)) {
+ DTRACE_CHARBUF(process_buf, DTRACE_TERM_BUF_SIZE);
+ DTRACE_CHARBUF(fun_buf, DTRACE_TERM_BUF_SIZE);
+ dtrace_proc_str(c_p, process_buf);
+
+ if (ERTS_PROC_IS_EXITING(c_p)) {
+ strcpy(fun_buf, "<exiting>");
+ } else {
+ BeamInstr *fptr = find_function_from_pc(c_p->i);
+ if (fptr) {
+ dtrace_fun_decode(c_p, (Eterm)fptr[0],
+ (Eterm)fptr[1], (Uint)fptr[2],
+ NULL, fun_buf);
+ } else {
+ erts_snprintf(fun_buf, sizeof(DTRACE_CHARBUF_NAME(fun_buf)),
+ "<unknown/%p>", *I);
+ }
+ }
+
+ DTRACE2(process_scheduled, process_buf, fun_buf);
+ }
+#endif
+ }
+
+ {
+#ifdef DEBUG
+ Eterm result;
+#endif
+ Eterm arity;
+
+ {
+ /*
+ * call_nif is always first instruction in function:
+ *
+ * I[-3]: Module
+ * I[-2]: Function
+ * I[-1]: Arity
+ * I[0]: &&call_nif
+ * I[1]: Function pointer to NIF function
+ * I[2]: Pointer to erl_module_nif
+ * I[3]: Function pointer to dirty NIF
+ */
+ BifFunction vbf;
+
+ ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_NIF);
+
+ DTRACE_NIF_ENTRY(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]);
+ c_p->current = I-3; /* current and vbf set to please handle_error */
+ SWAPOUT;
+ PROCESS_MAIN_CHK_LOCKS(c_p);
+ arity = I[-1];
+ ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
+
+ ASSERT(!ERTS_PROC_IS_EXITING(c_p));
+ {
+ typedef Eterm NifF(struct enif_environment_t*, int argc, Eterm argv[]);
+ NifF* fp = vbf = (NifF*) I[1];
+ struct enif_environment_t env;
+ ASSERT(!c_p->scheduler_data);
+
+ erts_pre_dirty_nif(esdp, &env, c_p,
+ (struct erl_module_nif*)I[2], NULL);
+
+#ifdef DEBUG
+ result =
+#else
+ (void)
+#endif
+ (*fp)(&env, arity, reg);
+
+ erts_post_nif(&env);
+
+ ASSERT(!is_value(result));
+ ASSERT(c_p->freason == TRAP);
+ ASSERT(!(c_p->flags & F_HIBERNATE_SCHED));
+
+ PROCESS_MAIN_CHK_LOCKS(c_p);
+ ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
+ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+ ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR);
+ if (env.exiting)
+ goto do_dirty_schedule;
+ ASSERT(!ERTS_PROC_IS_EXITING(c_p));
+ }
+
+ DTRACE_NIF_RETURN(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]);
+ ERTS_HOLE_CHECK(c_p);
+ SWAPIN;
+ I = c_p->i;
+ goto context_switch;
+ }
+ }
+#endif /* ERTS_DIRTY_SCHEDULERS */
+}
+
static BifFunction
translate_gc_bif(void* gcf)
{
@@ -5010,6 +5436,8 @@ translate_gc_bif(void* gcf)
return bit_size_1;
} else if (gcf == erts_gc_byte_size_1) {
return byte_size_1;
+ } else if (gcf == erts_gc_map_size_1) {
+ return map_size_1;
} else if (gcf == erts_gc_abs_1) {
return abs_1;
} else if (gcf == erts_gc_float_1) {
@@ -5023,7 +5451,7 @@ translate_gc_bif(void* gcf)
} else if (gcf == erts_gc_binary_part_3) {
return binary_part_3;
} else {
- erl_exit(1, "bad gc bif");
+ erts_exit(ERTS_ERROR_EXIT, "bad gc bif");
}
}
@@ -5057,7 +5485,9 @@ Eterm error_atom[NUMBER_EXIT_CODES] = {
am_notalive, /* 14 */
am_system_limit, /* 15 */
am_try_clause, /* 16 */
- am_notsup /* 17 */
+ am_notsup, /* 17 */
+ am_badmap, /* 18 */
+ am_badkey, /* 19 */
};
/*
@@ -5086,7 +5516,7 @@ handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf)
Eterm* hp;
Eterm Value = c_p->fvalue;
Eterm Args = am_true;
- c_p->i = pc; /* In case we call erl_exit(). */
+ c_p->i = pc; /* In case we call erts_exit(). */
ASSERT(c_p->freason != TRAP); /* Should have been handled earlier. */
@@ -5150,7 +5580,7 @@ handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf)
c_p->cp = 0; /* To avoid keeping stale references. */
return new_pc;
}
- if (c_p->catches > 0) erl_exit(1, "Catch not found");
+ if (c_p->catches > 0) erts_exit(ERTS_ERROR_EXIT, "Catch not found");
}
ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
terminate_proc(c_p, Value);
@@ -5183,7 +5613,8 @@ next_catch(Process* c_p, Eterm *reg) {
BeamInstr *cpp = c_p->cp;
if (cpp == beam_exception_trace) {
erts_trace_exception(c_p, cp_val(ptr[0]),
- reg[1], reg[2], ptr+1);
+ reg[1], reg[2],
+ ERTS_TRACER_FROM_ETERM(ptr+1));
/* Skip return_trace parameters */
ptr += 2;
} else if (cpp == beam_return_trace) {
@@ -5210,7 +5641,8 @@ next_catch(Process* c_p, Eterm *reg) {
}
if (cp_val(*prev) == beam_exception_trace) {
erts_trace_exception(c_p, cp_val(ptr[0]),
- reg[1], reg[2], ptr+1);
+ reg[1], reg[2],
+ ERTS_TRACER_FROM_ETERM(ptr+1));
}
/* Skip return_trace parameters */
ptr += 2;
@@ -5260,18 +5692,35 @@ next_catch(Process* c_p, Eterm *reg) {
static void
terminate_proc(Process* c_p, Eterm Value)
{
+ Eterm *hp;
+ Eterm Args = NIL;
+
/* Add a stacktrace if this is an error. */
if (GET_EXC_CLASS(c_p->freason) == EXTAG_ERROR) {
Value = add_stacktrace(c_p, Value, c_p->ftrace);
}
/* EXF_LOG is a primary exception flag */
if (c_p->freason & EXF_LOG) {
+ int alive = erts_is_alive;
erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
- erts_dsprintf(dsbufp, "Error in process %T ", c_p->common.id);
- if (erts_is_alive)
- erts_dsprintf(dsbufp, "on node %T ", erts_this_node->sysname);
- erts_dsprintf(dsbufp,"with exit value: %0.*T\n", display_items, Value);
- erts_send_error_to_logger(c_p->group_leader, dsbufp);
+
+ /* Build the format message */
+ erts_dsprintf(dsbufp, "Error in process ~p ");
+ if (alive)
+ erts_dsprintf(dsbufp, "on node ~p ");
+ erts_dsprintf(dsbufp, "with exit value:~n~p~n");
+
+ /* Build the args in reverse order */
+ hp = HAlloc(c_p, 2);
+ Args = CONS(hp, Value, Args);
+ if (alive) {
+ hp = HAlloc(c_p, 2);
+ Args = CONS(hp, erts_this_node->sysname, Args);
+ }
+ hp = HAlloc(c_p, 2);
+ Args = CONS(hp, c_p->common.id, Args);
+
+ erts_send_error_term_to_logger(c_p->group_leader, dsbufp, Args);
}
/*
* If we use a shared heap, the process will be garbage-collected.
@@ -5313,6 +5762,8 @@ expand_error_value(Process* c_p, Uint freason, Eterm Value) {
case (GET_EXC_INDEX(EXC_TRY_CLAUSE)):
case (GET_EXC_INDEX(EXC_BADFUN)):
case (GET_EXC_INDEX(EXC_BADARITY)):
+ case (GET_EXC_INDEX(EXC_BADMAP)):
+ case (GET_EXC_INDEX(EXC_BADKEY)):
/* Some common exceptions: value -> {atom, value} */
ASSERT(is_value(Value));
hp = HAlloc(c_p, 3);
@@ -5615,7 +6066,7 @@ build_stacktrace(Process* c_p, Eterm exc) {
* (e.g. spawn_link(erlang, abs, [1])).
*/
if (fi.current == NULL) {
- erts_set_current_function(&fi, c_p->initial);
+ erts_set_current_function(&fi, c_p->u.initial);
args = am_true; /* Just in case */
} else {
args = get_args_from_exc(exc);
@@ -5654,7 +6105,6 @@ build_stacktrace(Process* c_p, Eterm exc) {
return res;
}
-
static BeamInstr*
call_error_handler(Process* p, BeamInstr* fi, Eterm* reg, Eterm func)
{
@@ -5702,7 +6152,6 @@ call_error_handler(Process* p, BeamInstr* fi, Eterm* reg, Eterm func)
return ep->addressv[erts_active_code_ix()];
}
-
static Export*
apply_setup_error_handler(Process* p, Eterm module, Eterm function, Uint arity, Eterm* reg)
{
@@ -5825,13 +6274,7 @@ apply(Process* p, Eterm module, Eterm function, Eterm args, Eterm* reg)
} else if (ERTS_PROC_GET_SAVED_CALLS_BUF(p)) {
save_calls(p, ep);
}
-
-#ifdef USE_VM_CALL_PROBES
- if (DTRACE_ENABLED(global_function_entry)) {
- BeamInstr *fptr = (BeamInstr *) ep->addressv[erts_active_code_ix()];
- DTRACE_GLOBAL_CALL(p, (Eterm)fptr[-3], (Eterm)fptr[-2], (Uint)fptr[-1]);
- }
-#endif
+ DTRACE_GLOBAL_CALL_FROM_EXPORT(p, ep);
return ep->addressv[erts_active_code_ix()];
}
@@ -5880,13 +6323,7 @@ fixed_apply(Process* p, Eterm* reg, Uint arity)
} else if (ERTS_PROC_GET_SAVED_CALLS_BUF(p)) {
save_calls(p, ep);
}
-
-#ifdef USE_VM_CALL_PROBES
- if (DTRACE_ENABLED(global_function_entry)) {
- BeamInstr *fptr = (BeamInstr *) ep->addressv[erts_active_code_ix()];
- DTRACE_GLOBAL_CALL(p, (Eterm)fptr[-3], (Eterm)fptr[-2], (Uint)fptr[-1]);
- }
-#endif
+ DTRACE_GLOBAL_CALL_FROM_EXPORT(p, ep);
return ep->addressv[erts_active_code_ix()];
}
@@ -5896,6 +6333,23 @@ erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* re
int arity;
Eterm tmp;
+#ifndef ERTS_SMP
+ if (ERTS_PROC_IS_EXITING(c_p)) {
+ /*
+ * I non smp case:
+ *
+ * Currently executing process might be sent an exit
+ * signal if it is traced by a port that it also is
+ * linked to, and the port terminates during the
+ * trace. In this case we do *not* want to clear
+ * the active flag, which will make the process hang
+ * in limbo forever. Get out of here and terminate
+ * the process...
+ */
+ return -1;
+ }
+#endif
+
if (is_not_atom(module) || is_not_atom(function)) {
/*
* No need to test args here -- done below.
@@ -5972,8 +6426,17 @@ erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* re
ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
PROCESS_MAIN_CHK_LOCKS(c_p);
erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
-#ifdef ERTS_SMP
- ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p);
+#ifndef ERTS_SMP
+ if (ERTS_PROC_IS_EXITING(c_p)) {
+ /*
+ * See comment in the begining of the function...
+ *
+ * This second test is needed since gc might be traced.
+ */
+ return -1;
+ }
+#else /* ERTS_SMP */
+ ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p);
if (!c_p->msg.len)
#endif
erts_smp_atomic32_read_band_relb(&c_p->state, ~ERTS_PSFLG_ACTIVE);
@@ -6074,7 +6537,7 @@ call_fun(Process* p, /* Current process. */
*/
module = fe->module;
if ((modp = erts_get_module(module, code_ix)) != NULL
- && modp->curr.code != NULL) {
+ && modp->curr.code_hdr != NULL) {
/*
* There is a module loaded, but obviously the fun is not
* defined in it. We must not call the error_handler
@@ -6208,7 +6671,498 @@ new_fun(Process* p, Eterm* reg, ErlFunEntry* fe, int num_free)
return make_fun(funp);
}
-
+static Eterm get_map_element(Eterm map, Eterm key)
+{
+ Uint32 hx;
+ const Eterm *vs;
+ if (is_flatmap(map)) {
+ flatmap_t *mp;
+ Eterm *ks;
+ Uint i;
+ Uint n;
+
+ mp = (flatmap_t *)flatmap_val(map);
+ ks = flatmap_get_keys(mp);
+ vs = flatmap_get_values(mp);
+ n = flatmap_get_size(mp);
+ if (is_immed(key)) {
+ for (i = 0; i < n; i++) {
+ if (ks[i] == key) {
+ return vs[i];
+ }
+ }
+ } else {
+ for (i = 0; i < n; i++) {
+ if (EQ(ks[i], key)) {
+ return vs[i];
+ }
+ }
+ }
+ return THE_NON_VALUE;
+ }
+ ASSERT(is_hashmap(map));
+ hx = hashmap_make_hash(key);
+ vs = erts_hashmap_get(hx,key,map);
+ return vs ? *vs : THE_NON_VALUE;
+}
+
+static Eterm get_map_element_hash(Eterm map, Eterm key, Uint32 hx)
+{
+ const Eterm *vs;
+
+ if (is_flatmap(map)) {
+ flatmap_t *mp;
+ Eterm *ks;
+ Uint i;
+ Uint n;
+
+ mp = (flatmap_t *)flatmap_val(map);
+ ks = flatmap_get_keys(mp);
+ vs = flatmap_get_values(mp);
+ n = flatmap_get_size(mp);
+ if (is_immed(key)) {
+ for (i = 0; i < n; i++) {
+ if (ks[i] == key) {
+ return vs[i];
+ }
+ }
+ } else {
+ for (i = 0; i < n; i++) {
+ if (EQ(ks[i], key)) {
+ return vs[i];
+ }
+ }
+ }
+ return THE_NON_VALUE;
+ }
+
+ ASSERT(is_hashmap(map));
+ ASSERT(hx == hashmap_make_hash(key));
+ vs = erts_hashmap_get(hx, key, map);
+ return vs ? *vs : THE_NON_VALUE;
+}
+
+#define GET_TERM(term, dest) \
+do { \
+ Eterm src = (Eterm)(term); \
+ switch (loader_tag(src)) { \
+ case LOADER_X_REG: \
+ dest = x(loader_x_reg_index(src)); \
+ break; \
+ case LOADER_Y_REG: \
+ dest = y(loader_y_reg_index(src)); \
+ break; \
+ default: \
+ dest = src; \
+ break; \
+ } \
+} while(0)
+
+
+static Eterm
+new_map(Process* p, Eterm* reg, BeamInstr* I)
+{
+ Uint n = Arg(3);
+ Uint i;
+ Uint need = n + 1 /* hdr */ + 1 /*size*/ + 1 /* ptr */ + 1 /* arity */;
+ Eterm keys;
+ Eterm *mhp,*thp;
+ Eterm *E;
+ BeamInstr *ptr;
+ flatmap_t *mp;
+ ErtsHeapFactory factory;
+
+ ptr = &Arg(4);
+
+ if (n > 2*MAP_SMALL_MAP_LIMIT) {
+ Eterm res;
+ if (HeapWordsLeft(p) < n) {
+ erts_garbage_collect(p, n, reg, Arg(2));
+ }
+
+ mhp = p->htop;
+ thp = p->htop;
+ E = p->stop;
+
+ for (i = 0; i < n/2; i++) {
+ GET_TERM(*ptr++, *mhp++);
+ GET_TERM(*ptr++, *mhp++);
+ }
+
+ p->htop = mhp;
+
+ erts_factory_proc_init(&factory, p);
+ res = erts_hashmap_from_array(&factory, thp, n/2, 0);
+ erts_factory_close(&factory);
+ return res;
+ }
+
+ if (HeapWordsLeft(p) < need) {
+ erts_garbage_collect(p, need, reg, Arg(2));
+ }
+
+ thp = p->htop;
+ mhp = thp + 1 + n/2;
+ E = p->stop;
+ keys = make_tuple(thp);
+ *thp++ = make_arityval(n/2);
+
+ mp = (flatmap_t *)mhp; mhp += MAP_HEADER_FLATMAP_SZ;
+ mp->thing_word = MAP_HEADER_FLATMAP;
+ mp->size = n/2;
+ mp->keys = keys;
+
+ for (i = 0; i < n/2; i++) {
+ GET_TERM(*ptr++, *thp++);
+ GET_TERM(*ptr++, *mhp++);
+ }
+ p->htop = mhp;
+ return make_flatmap(mp);
+}
+
+static Eterm
+update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
+{
+ Uint n;
+ Uint num_old;
+ Uint num_updates;
+ Uint need;
+ flatmap_t *old_mp, *mp;
+ Eterm res;
+ Eterm* hp;
+ Eterm* E;
+ Eterm* old_keys;
+ Eterm* old_vals;
+ BeamInstr* new_p;
+ Eterm new_key;
+ Eterm* kp;
+
+ new_p = &Arg(5);
+ num_updates = Arg(4) / 2;
+
+ if (is_not_flatmap(map)) {
+ Uint32 hx;
+ Eterm val;
+
+ /* apparently the compiler does not emit is_map instructions,
+ * bad compiler */
+
+ if (is_not_hashmap(map))
+ return THE_NON_VALUE;
+
+ res = map;
+ E = p->stop;
+ while(num_updates--) {
+ /* assoc can't fail */
+ GET_TERM(new_p[0], new_key);
+ GET_TERM(new_p[1], val);
+ hx = hashmap_make_hash(new_key);
+
+ res = erts_hashmap_insert(p, hx, new_key, val, res, 0);
+
+ new_p += 2;
+ }
+ return res;
+ }
+
+ old_mp = (flatmap_t *) flatmap_val(map);
+ num_old = flatmap_get_size(old_mp);
+
+ /*
+ * If the old map is empty, create a new map.
+ */
+
+ if (num_old == 0) {
+ return new_map(p, reg, I+1);
+ }
+
+ /*
+ * Allocate heap space for the worst case (i.e. all keys in the
+ * update list are new).
+ */
+
+ need = 2*(num_old+num_updates) + 1 + MAP_HEADER_FLATMAP_SZ;
+ if (HeapWordsLeft(p) < need) {
+ Uint live = Arg(3);
+ reg[live] = map;
+ erts_garbage_collect(p, need, reg, live+1);
+ map = reg[live];
+ old_mp = (flatmap_t *)flatmap_val(map);
+ }
+
+ /*
+ * Build the skeleton for the map, ready to be filled in.
+ *
+ * +-----------------------------------+
+ * | (Space for aritvyal for keys) | <-----------+
+ * +-----------------------------------+ |
+ * | (Space for key 1) | | <-- kp
+ * +-----------------------------------+ |
+ * . |
+ * . |
+ * . |
+ * +-----------------------------------+ |
+ * | (Space for last key) | |
+ * +-----------------------------------+ |
+ * | MAP_HEADER | |
+ * +-----------------------------------+ |
+ * | (Space for number of keys/values) | |
+ * +-----------------------------------+ |
+ * | Boxed tuple pointer >----------------+
+ * +-----------------------------------+
+ * | (Space for value 1) | <-- hp
+ * +-----------------------------------+
+ */
+
+ E = p->stop;
+ kp = p->htop + 1; /* Point to first key */
+ hp = kp + num_old + num_updates;
+
+ res = make_flatmap(hp);
+ mp = (flatmap_t *)hp;
+ hp += MAP_HEADER_FLATMAP_SZ;
+ mp->thing_word = MAP_HEADER_FLATMAP;
+ mp->keys = make_tuple(kp-1);
+
+ old_vals = flatmap_get_values(old_mp);
+ old_keys = flatmap_get_keys(old_mp);
+
+ GET_TERM(*new_p, new_key);
+ n = num_updates;
+
+ /*
+ * Fill in keys and values, until we run out of either updates
+ * or old values and keys.
+ */
+
+ for (;;) {
+ Eterm key;
+ Sint c;
+
+ ASSERT(kp < (Eterm *)mp);
+ key = *old_keys;
+ if ((c = CMP_TERM(key, new_key)) < 0) {
+ /* Copy old key and value */
+ *kp++ = key;
+ *hp++ = *old_vals;
+ old_keys++, old_vals++, num_old--;
+ } else { /* Replace or insert new */
+ GET_TERM(new_p[1], *hp++);
+ if (c > 0) { /* If new new key */
+ *kp++ = new_key;
+ } else { /* If replacement */
+ *kp++ = key;
+ old_keys++, old_vals++, num_old--;
+ }
+ n--;
+ if (n == 0) {
+ break;
+ } else {
+ new_p += 2;
+ GET_TERM(*new_p, new_key);
+ }
+ }
+ if (num_old == 0) {
+ break;
+ }
+ }
+
+ /*
+ * At this point, we have run out of either old keys and values,
+ * or the update list. In other words, at least of one n and
+ * num_old must be zero.
+ */
+
+ if (n > 0) {
+ /*
+ * All old keys and values have been copied, but there
+ * are still new keys and values in the update list that
+ * must be copied.
+ */
+ ASSERT(num_old == 0);
+ while (n-- > 0) {
+ GET_TERM(new_p[0], *kp++);
+ GET_TERM(new_p[1], *hp++);
+ new_p += 2;
+ }
+ } else {
+ /*
+ * All updates are now done. We may still have old
+ * keys and values that we must copy.
+ */
+ ASSERT(n == 0);
+ while (num_old-- > 0) {
+ ASSERT(kp < (Eterm *)mp);
+ *kp++ = *old_keys++;
+ *hp++ = *old_vals++;
+ }
+ }
+
+ /*
+ * Calculate how many values that are unused at the end of the
+ * key tuple and fill it out with a bignum header.
+ */
+ if ((n = (Eterm *)mp - kp) > 0) {
+ *kp = make_pos_bignum_header(n-1);
+ }
+
+ /*
+ * Fill in the size of the map in both the key tuple and in the map.
+ */
+
+ n = kp - p->htop - 1; /* Actual number of keys/values */
+ *p->htop = make_arityval(n);
+ p->htop = hp;
+ mp->size = n;
+
+ /* The expensive case, need to build a hashmap */
+ if (n > MAP_SMALL_MAP_LIMIT) {
+ res = erts_hashmap_from_ks_and_vs(p,flatmap_get_keys(mp),flatmap_get_values(mp),n);
+ }
+ return res;
+}
+
+/*
+ * Update values for keys that already exist in the map.
+ */
+
+static Eterm
+update_map_exact(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
+{
+ Uint n;
+ Uint i;
+ Uint num_old;
+ Uint need;
+ flatmap_t *old_mp, *mp;
+ Eterm res;
+ Eterm* hp;
+ Eterm* E;
+ Eterm* old_keys;
+ Eterm* old_vals;
+ BeamInstr* new_p;
+ Eterm new_key;
+
+ new_p = &Arg(5);
+ n = Arg(4) / 2; /* Number of values to be updated */
+ ASSERT(n > 0);
+
+ if (is_not_flatmap(map)) {
+ Uint32 hx;
+ Eterm val;
+
+ /* apparently the compiler does not emit is_map instructions,
+ * bad compiler */
+
+ if (is_not_hashmap(map)) {
+ p->freason = BADMAP;
+ p->fvalue = map;
+ return THE_NON_VALUE;
+ }
+
+ res = map;
+ E = p->stop;
+ while(n--) {
+ GET_TERM(new_p[0], new_key);
+ GET_TERM(new_p[1], val);
+ hx = hashmap_make_hash(new_key);
+
+ res = erts_hashmap_insert(p, hx, new_key, val, res, 1);
+ if (is_non_value(res)) {
+ p->fvalue = new_key;
+ p->freason = BADKEY;
+ return res;
+ }
+
+ new_p += 2;
+ }
+ return res;
+ }
+
+ old_mp = (flatmap_t *) flatmap_val(map);
+ num_old = flatmap_get_size(old_mp);
+
+ /*
+ * If the old map is empty, fail.
+ */
+
+ if (num_old == 0) {
+ E = p->stop;
+ p->freason = BADKEY;
+ GET_TERM(new_p[0], p->fvalue);
+ return THE_NON_VALUE;
+ }
+
+ /*
+ * Allocate the exact heap space needed.
+ */
+
+ need = num_old + MAP_HEADER_FLATMAP_SZ;
+ if (HeapWordsLeft(p) < need) {
+ Uint live = Arg(3);
+ reg[live] = map;
+ erts_garbage_collect(p, need, reg, live+1);
+ map = reg[live];
+ old_mp = (flatmap_t *)flatmap_val(map);
+ }
+
+ /*
+ * Update map, keeping the old key tuple.
+ */
+
+ hp = p->htop;
+ E = p->stop;
+
+ old_vals = flatmap_get_values(old_mp);
+ old_keys = flatmap_get_keys(old_mp);
+
+ res = make_flatmap(hp);
+ mp = (flatmap_t *)hp;
+ hp += MAP_HEADER_FLATMAP_SZ;
+ mp->thing_word = MAP_HEADER_FLATMAP;
+ mp->size = num_old;
+ mp->keys = old_mp->keys;
+
+ /* Get array of key/value pairs to be updated */
+ GET_TERM(*new_p, new_key);
+
+ /* Update all values */
+ for (i = 0; i < num_old; i++) {
+ if (!EQ(*old_keys, new_key)) {
+ /* Not same keys */
+ *hp++ = *old_vals;
+ } else {
+ GET_TERM(new_p[1], *hp);
+ hp++;
+ n--;
+ if (n == 0) {
+ /*
+ * All updates done. Copy remaining values
+ * and return the result.
+ */
+ for (i++, old_vals++; i < num_old; i++) {
+ *hp++ = *old_vals++;
+ }
+ ASSERT(hp == p->htop + need);
+ p->htop = hp;
+ return res;
+ } else {
+ new_p += 2;
+ GET_TERM(*new_p, new_key);
+ }
+ }
+ old_vals++, old_keys++;
+ }
+
+ /*
+ * Updates left. That means that at least one the keys in the
+ * update list did not previously exist.
+ */
+ ASSERT(hp == p->htop + need);
+ p->freason = BADKEY;
+ p->fvalue = new_key;
+ return THE_NON_VALUE;
+}
+#undef GET_TERM
int catchlevel(Process *p)
{
@@ -6227,6 +7181,15 @@ erts_is_builtin(Eterm Mod, Eterm Name, int arity)
Export e;
Export* ep;
+ if (Mod == am_erlang && Name == am_apply && arity == 3) {
+ /*
+ * Special case. apply/3 is built-in (implemented in C),
+ * but implemented in a different way than all other
+ * BIFs.
+ */
+ return 1;
+ }
+
e.code[0] = Mod;
e.code[1] = Name;
e.code[2] = arity;
@@ -6250,9 +7213,18 @@ erts_current_reductions(Process *current, Process *p)
if (current != p) {
return 0;
} else if (current->fcalls < 0 && ERTS_PROC_GET_SAVED_CALLS_BUF(current)) {
- return -current->fcalls;
+ return current->fcalls + CONTEXT_REDS;
} else {
return REDS_IN(current) - current->fcalls;
}
}
+int
+erts_beam_jump_table(void)
+{
+#if defined(NO_JUMP_TABLE)
+ return 0;
+#else
+ return 1;
+#endif
+}