aboutsummaryrefslogtreecommitdiffstats
path: root/erts/emulator/beam
diff options
context:
space:
mode:
Diffstat (limited to 'erts/emulator/beam')
-rw-r--r--erts/emulator/beam/beam_debug.c119
-rw-r--r--erts/emulator/beam/beam_emu.c1280
-rw-r--r--erts/emulator/beam/beam_load.c554
-rw-r--r--erts/emulator/beam/bif.c5
-rw-r--r--erts/emulator/beam/bif.tab1
-rw-r--r--erts/emulator/beam/big.c8
-rw-r--r--erts/emulator/beam/big.h8
-rw-r--r--erts/emulator/beam/break.c2
-rw-r--r--erts/emulator/beam/copy.c113
-rw-r--r--erts/emulator/beam/dist.c159
-rw-r--r--erts/emulator/beam/dist.h3
-rw-r--r--erts/emulator/beam/erl_alloc.c31
-rw-r--r--erts/emulator/beam/erl_alloc.h7
-rw-r--r--erts/emulator/beam/erl_alloc.types5
-rw-r--r--erts/emulator/beam/erl_alloc_util.c34
-rw-r--r--erts/emulator/beam/erl_alloc_util.h5
-rw-r--r--erts/emulator/beam/erl_bif_lists.c4
-rw-r--r--erts/emulator/beam/erl_bif_timer.c15
-rw-r--r--erts/emulator/beam/erl_binary.h19
-rw-r--r--erts/emulator/beam/erl_bits.c14
-rw-r--r--erts/emulator/beam/erl_cpu_topology.c20
-rw-r--r--erts/emulator/beam/erl_db.c189
-rw-r--r--erts/emulator/beam/erl_db_hash.c50
-rw-r--r--erts/emulator/beam/erl_db_tree.c336
-rw-r--r--erts/emulator/beam/erl_db_util.c773
-rw-r--r--erts/emulator/beam/erl_db_util.h76
-rw-r--r--erts/emulator/beam/erl_drv_thread.c60
-rw-r--r--erts/emulator/beam/erl_init.c4
-rw-r--r--erts/emulator/beam/erl_lock_check.c6
-rw-r--r--erts/emulator/beam/erl_monitors.c18
-rw-r--r--erts/emulator/beam/erl_nif.c54
-rw-r--r--erts/emulator/beam/erl_nif.h14
-rw-r--r--erts/emulator/beam/erl_node_container_utils.h17
-rw-r--r--erts/emulator/beam/erl_process.c177
-rw-r--r--erts/emulator/beam/erl_process.h43
-rw-r--r--erts/emulator/beam/erl_process_dump.c240
-rw-r--r--erts/emulator/beam/erl_term.c49
-rw-r--r--erts/emulator/beam/erl_term.h174
-rw-r--r--erts/emulator/beam/erl_threads.h67
-rw-r--r--erts/emulator/beam/erl_time.h64
-rw-r--r--erts/emulator/beam/erl_time_sup.c19
-rw-r--r--erts/emulator/beam/erl_trace.c6
-rw-r--r--erts/emulator/beam/erl_vm.h9
-rw-r--r--erts/emulator/beam/global.h135
-rw-r--r--erts/emulator/beam/io.c12
-rw-r--r--erts/emulator/beam/ops.tab269
-rw-r--r--erts/emulator/beam/sys.h36
-rw-r--r--erts/emulator/beam/time.c286
-rw-r--r--erts/emulator/beam/utils.c296
49 files changed, 3610 insertions, 2275 deletions
diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index b0bf14b94f..2855241b91 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -48,7 +48,6 @@
void dbg_bt(Process* p, Eterm* sp);
void dbg_where(BeamInstr* addr, Eterm x0, Eterm* reg);
-static void print_big(int to, void *to_arg, Eterm* addr);
static int print_op(int to, void *to_arg, int op, int size, BeamInstr* addr);
Eterm
erts_debug_same_2(Process* p, Eterm term1, Eterm term2)
@@ -157,6 +156,25 @@ void debug_dump_code(BeamInstr *I, int num)
}
#endif
+BIF_RETTYPE
+erts_debug_instructions_0(BIF_ALIST_0)
+{
+ int i = 0;
+ Uint needed = num_instructions * 2;
+ Eterm* hp;
+ Eterm res = NIL;
+
+ for (i = 0; i < num_instructions; i++) {
+ needed += 2*strlen(opc[i].name);
+ }
+ hp = HAlloc(BIF_P, needed);
+ for (i = num_instructions-1; i >= 0; i--) {
+ Eterm s = erts_bld_string_n(&hp, 0, opc[i].name, strlen(opc[i].name));
+ res = erts_bld_cons(&hp, 0, s, res);
+ }
+ return res;
+}
+
Eterm
erts_debug_disassemble_1(Process* p, Eterm addr)
{
@@ -312,6 +330,7 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr)
BeamInstr packed = 0; /* Accumulator for packed operations. */
BeamInstr args[8]; /* Arguments for this instruction. */
BeamInstr* ap; /* Pointer to arguments. */
+ BeamInstr* unpacked; /* Unpacked arguments */
start_prog = opc[op].pack;
@@ -360,6 +379,12 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr)
*ap++ = packed & BEAM_LOOSE_MASK;
packed >>= BEAM_LOOSE_SHIFT;
break;
+#ifdef ARCH_64
+ case 'w': /* Shift 32 steps */
+ *ap++ = packed & BEAM_WIDE_MASK;
+ packed >>= BEAM_WIDE_SHIFT;
+ break;
+#endif
case 'p':
*sp++ = *--ap;
break;
@@ -386,7 +411,7 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr)
break;
case 'x': /* x(N) */
if (reg_index(ap[0]) == 0) {
- erts_print(to, to_arg, "X[0]");
+ erts_print(to, to_arg, "x[0]");
} else {
erts_print(to, to_arg, "x(%d)", reg_index(ap[0]));
}
@@ -506,6 +531,7 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr)
ap++;
break;
case 'P': /* Byte offset into tuple (see beam_load.c) */
+ case 'Q': /* Like 'P', but packable */
erts_print(to, to_arg, "%d", (*ap / sizeof(Eterm)) - 1);
ap++;
break;
@@ -526,9 +552,12 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr)
* Print more information about certain instructions.
*/
+ unpacked = ap;
ap = addr + size;
switch (op) {
- case op_i_select_val_sfI:
+ case op_i_select_val_rfI:
+ case op_i_select_val_xfI:
+ case op_i_select_val_yfI:
{
int n = ap[-1];
@@ -540,7 +569,24 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr)
}
}
break;
- case op_i_jump_on_val_sfII:
+ case op_i_select_tuple_arity_rfI:
+ case op_i_select_tuple_arity_xfI:
+ case op_i_select_tuple_arity_yfI:
+ {
+ int n = ap[-1];
+
+ while (n > 0) {
+ Uint arity = arityval(ap[0]);
+ erts_print(to, to_arg, " {%d} f(" HEXF ")", arity, ap[1]);
+ ap += 2;
+ size += 2;
+ n--;
+ }
+ }
+ break;
+ case op_i_jump_on_val_rfII:
+ case op_i_jump_on_val_xfII:
+ case op_i_jump_on_val_yfII:
{
int n;
for (n = ap[-2]; n > 0; n--) {
@@ -550,39 +596,46 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr)
}
}
break;
- case op_i_select_big_sf:
- while (ap[0]) {
- Eterm *bigp = (Eterm *) ap;
- int arity = thing_arityval(*bigp);
- print_big(to, to_arg, bigp);
- size += TermWords(arity+1);
- ap += TermWords(arity+1);
- erts_print(to, to_arg, " f(" HEXF ") ", ap[0]);
- ap++;
- size++;
+ case op_i_jump_on_val_zero_rfI:
+ case op_i_jump_on_val_zero_xfI:
+ case op_i_jump_on_val_zero_yfI:
+ {
+ int n;
+ for (n = ap[-1]; n > 0; n--) {
+ erts_print(to, to_arg, "f(" HEXF ") ", ap[0]);
+ ap++;
+ size++;
+ }
+ }
+ break;
+ case op_i_put_tuple_rI:
+ case op_i_put_tuple_xI:
+ case op_i_put_tuple_yI:
+ {
+ int n = unpacked[-1];
+
+ while (n > 0) {
+ if (!is_header(ap[0])) {
+ erts_print(to, to_arg, " %T", (Eterm) ap[0]);
+ } else {
+ switch ((ap[0] >> 2) & 0x03) {
+ case R_REG_DEF:
+ erts_print(to, to_arg, " x(0)");
+ break;
+ case X_REG_DEF:
+ erts_print(to, to_arg, " x(%d)", ap[0] >> 4);
+ break;
+ case Y_REG_DEF:
+ erts_print(to, to_arg, " y(%d)", ap[0] >> 4);
+ break;
+ }
+ }
+ ap++, size++, n--;
+ }
}
- ap++;
- size++;
break;
}
erts_print(to, to_arg, "\n");
return size;
}
-
-static void
-print_big(int to, void *to_arg, Eterm* addr)
-{
- int i;
- int k;
-
- i = BIG_SIZE(addr);
- if (BIG_SIGN(addr))
- erts_print(to, to_arg, "-#integer(%d) = {", i);
- else
- erts_print(to, to_arg, "#integer(%d) = {", i);
- erts_print(to, to_arg, "0x%x", BIG_DIGIT(addr, 0));
- for (k = 1; k < i; k++)
- erts_print(to, to_arg, ",0x%x", BIG_DIGIT(addr, k));
- erts_print(to, to_arg, "}");
-}
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 8a0e12dd4f..a4fb454481 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 1996-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -321,6 +321,7 @@ extern int count_instructions;
# define POST_BIF_GC_SWAPIN_0(_p, _res) \
ERTS_SMP_REQ_PROC_MAIN_LOCK((_p)); \
PROCESS_MAIN_CHK_LOCKS((_p)); \
+ ERTS_VERIFY_UNUSED_TEMP_ALLOC((_p)); \
if (((_p)->mbuf) || (MSO(_p).overhead >= BIN_VHEAP_SZ(_p)) ) { \
_res = erts_gc_after_bif_call((_p), (_res), NULL, 0); \
E = (_p)->stop; \
@@ -328,6 +329,7 @@ extern int count_instructions;
HTOP = HEAP_TOP((_p))
# define POST_BIF_GC_SWAPIN(_p, _res, _regs, _arity) \
+ ERTS_VERIFY_UNUSED_TEMP_ALLOC((_p)); \
ERTS_SMP_REQ_PROC_MAIN_LOCK((_p)); \
PROCESS_MAIN_CHK_LOCKS((_p)); \
if (((_p)->mbuf) || (MSO(_p).overhead >= BIN_VHEAP_SZ(_p)) ) { \
@@ -344,6 +346,8 @@ extern int count_instructions;
#define xb(N) (*(Eterm *) (((unsigned char *)reg) + (N)))
#define yb(N) (*(Eterm *) (((unsigned char *)E) + (N)))
#define fb(N) (*(double *) (((unsigned char *)&(freg[0].fd)) + (N)))
+#define Qb(N) (N)
+#define Ib(N) (N)
#define x(N) reg[N]
#define y(N) E[N]
#define r(N) x##N
@@ -365,6 +369,7 @@ extern int count_instructions;
reg[0] = r(0); \
PROCESS_MAIN_CHK_LOCKS(c_p); \
FCALLS -= erts_garbage_collect(c_p, needed + (HeapNeed), reg, (M)); \
+ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \
PROCESS_MAIN_CHK_LOCKS(c_p); \
r(0) = reg[0]; \
SWAPIN; \
@@ -418,6 +423,7 @@ extern int count_instructions;
reg[0] = r(0); \
PROCESS_MAIN_CHK_LOCKS(c_p); \
FCALLS -= erts_garbage_collect(c_p, need, reg, (Live)); \
+ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \
PROCESS_MAIN_CHK_LOCKS(c_p); \
r(0) = reg[0]; \
SWAPIN; \
@@ -440,6 +446,7 @@ extern int count_instructions;
reg[0] = r(0); \
PROCESS_MAIN_CHK_LOCKS(c_p); \
FCALLS -= erts_garbage_collect(c_p, need, reg, (Live)); \
+ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \
PROCESS_MAIN_CHK_LOCKS(c_p); \
r(0) = reg[0]; \
SWAPIN; \
@@ -462,6 +469,7 @@ extern int count_instructions;
reg[Live] = Extra; \
PROCESS_MAIN_CHK_LOCKS(c_p); \
FCALLS -= erts_garbage_collect(c_p, need, reg, (Live)+1); \
+ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \
PROCESS_MAIN_CHK_LOCKS(c_p); \
if (Live > 0) { \
r(0) = reg[0]; \
@@ -472,6 +480,13 @@ extern int count_instructions;
HEAP_SPACE_VERIFIED(need); \
} while (0)
+#define TestHeapPutList(Need, Reg) \
+ do { \
+ TestHeap((Need), 1); \
+ PutList(Reg, r(0), r(0), StoreSimpleDest); \
+ CHECK_TERM(r(0)); \
+ } while (0)
+
#ifdef HYBRID
#ifdef INCREMENTAL
#define TestGlobalHeap(Nh, Live, hp) \
@@ -516,6 +531,11 @@ extern int count_instructions;
SWAPIN; \
} while (0)
+#define PutTuple(Dst, Arity) \
+ do { \
+ Dst = make_tuple(HTOP); \
+ pt_arity = (Arity); \
+ } while (0)
/*
* Check that we haven't used the reductions and jump to function pointed to by
@@ -674,6 +694,11 @@ extern int count_instructions;
SET_I((BeamInstr *) CallDest); \
Dispatch();
+#define MoveJump(Src) \
+ r(0) = (Src); \
+ SET_I((BeamInstr *) Arg(0)); \
+ Goto(*I);
+
#define GetList(Src, H, T) do { \
Eterm* tmp_ptr = list_val(Src); \
H = CAR(tmp_ptr); \
@@ -723,16 +748,8 @@ extern int count_instructions;
(Dest) = (* (Eterm *) EXPAND_POINTER(tmp_arg1)); \
} while (0)
-#define PutTuple(Arity, Src, Dest) \
- ASSERT(is_arity_value(Arity)); \
- Dest = make_tuple(HTOP); \
- HTOP[0] = (Arity); \
- HTOP[1] = (Src); \
- HTOP += 2
-
-#define Put(Word) *HTOP++ = (Word)
-
#define EqualImmed(X, Y, Action) if (X != Y) { Action; }
+#define NotEqualImmed(X, Y, Action) if (X == Y) { Action; }
#define IsFloat(Src, Fail) if (is_not_float(Src)) { Fail; }
@@ -984,8 +1001,41 @@ extern int count_instructions;
#define IsPid(Src, Fail) if (is_not_pid(Src)) { Fail; }
#define IsRef(Src, Fail) if (is_not_ref(Src)) { Fail; }
-static BifFunction translate_gc_bif(void* gcf);
-static BeamInstr* handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf);
+/*
+ * process_main() is already huge, so we want to avoid inlining
+ * into it. Especially functions that are seldom used.
+ */
+#ifdef __GNUC__
+# define NOINLINE __attribute__((__noinline__))
+#else
+# define NOINLINE
+#endif
+
+/*
+ * The following functions are called directly by process_main().
+ * Don't inline them.
+ */
+static BifFunction translate_gc_bif(void* gcf) NOINLINE;
+static BeamInstr* handle_error(Process* c_p, BeamInstr* pc,
+ Eterm* reg, BifFunction bf) NOINLINE;
+static BeamInstr* call_error_handler(Process* p, BeamInstr* ip,
+ Eterm* reg, Eterm func) NOINLINE;
+static BeamInstr* fixed_apply(Process* p, Eterm* reg, Uint arity) NOINLINE;
+static BeamInstr* apply(Process* p, Eterm module, Eterm function,
+ Eterm args, Eterm* reg) NOINLINE;
+static int hibernate(Process* c_p, Eterm module, Eterm function,
+ Eterm args, Eterm* reg) NOINLINE;
+static BeamInstr* call_fun(Process* p, int arity,
+ Eterm* reg, Eterm args) NOINLINE;
+static BeamInstr* apply_fun(Process* p, Eterm fun,
+ Eterm args, Eterm* reg) NOINLINE;
+static Eterm new_fun(Process* p, Eterm* reg,
+ ErlFunEntry* fe, int num_free) NOINLINE;
+
+
+/*
+ * Functions not directly called by process_main(). OK to inline.
+ */
static BeamInstr* next_catch(Process* c_p, Eterm *reg);
static void terminate_proc(Process* c_p, Eterm Value);
static Eterm add_stacktrace(Process* c_p, Eterm Value, Eterm exc);
@@ -993,16 +1043,6 @@ static void save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg,
BifFunction bf, Eterm args);
static struct StackTrace * get_trace_from_exc(Eterm exc);
static Eterm make_arglist(Process* c_p, Eterm* reg, int a);
-static Eterm call_error_handler(Process* p, BeamInstr* ip, Eterm* reg);
-static Eterm call_breakpoint_handler(Process* p, BeamInstr* fi, Eterm* reg);
-static BeamInstr* fixed_apply(Process* p, Eterm* reg, Uint arity);
-static BeamInstr* apply(Process* p, Eterm module, Eterm function,
- Eterm args, Eterm* reg);
-static int hibernate(Process* c_p, Eterm module, Eterm function,
- Eterm args, Eterm* reg);
-static BeamInstr* call_fun(Process* p, int arity, Eterm* reg, Eterm args);
-static BeamInstr* apply_fun(Process* p, Eterm fun, Eterm args, Eterm* reg);
-static Eterm new_fun(Process* p, Eterm* reg, ErlFunEntry* fe, int num_free);
#if defined(VXWORKS)
static int init_done;
@@ -1146,6 +1186,8 @@ void process_main(void)
Uint temp_bits; /* Temporary used by BsSkipBits2 & BsGetInteger2 */
+ Eterm pt_arity; /* Used by do_put_tuple */
+
ERL_BITS_DECLARE_STATEP; /* Has to be last declaration */
@@ -1178,7 +1220,12 @@ void process_main(void)
do_schedule1:
PROCESS_MAIN_CHK_LOCKS(c_p);
ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
+#if HALFWORD_HEAP
+ ASSERT(erts_get_scheduler_data()->num_tmp_heap_used == 0);
+#endif
+ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
c_p = schedule(c_p, reds_used);
+ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
#ifdef DEBUG
pid = c_p->id;
#endif
@@ -1246,6 +1293,52 @@ void process_main(void)
#define STORE_ARITH_RESULT(res) StoreBifResult(2, (res));
#define ARITH_FUNC(name) erts_gc_##name
+ {
+ Eterm increment_reg_val;
+ Eterm increment_val;
+ Uint live;
+ Eterm result;
+
+ OpCase(i_increment_yIId):
+ increment_reg_val = yb(Arg(0));
+ goto do_increment;
+
+ OpCase(i_increment_xIId):
+ increment_reg_val = xb(Arg(0));
+ goto do_increment;
+
+ OpCase(i_increment_rIId):
+ increment_reg_val = r(0);
+ I--;
+
+ do_increment:
+ increment_val = Arg(1);
+ if (is_small(increment_reg_val)) {
+ Sint i = signed_val(increment_reg_val) + increment_val;
+ ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i));
+ if (MY_IS_SSMALL(i)) {
+ result = make_small(i);
+ store_result:
+ StoreBifResult(3, result);
+ }
+ }
+
+ live = Arg(2);
+ SWAPOUT;
+ reg[0] = r(0);
+ reg[live] = increment_reg_val;
+ reg[live+1] = make_small(increment_val);
+ result = erts_gc_mixed_plus(c_p, reg, live);
+ r(0) = reg[0];
+ SWAPIN;
+ ERTS_HOLE_CHECK(c_p);
+ if (is_value(result)) {
+ goto store_result;
+ }
+ ASSERT(c_p->freason != BADMATCH || is_value(c_p->fvalue));
+ goto find_func_info;
+ }
+
OpCase(i_plus_jId):
{
Eterm result;
@@ -1309,6 +1402,52 @@ void process_main(void)
}
Next(1);
+ {
+ Eterm is_eq_exact_lit_val;
+
+ OpCase(i_is_eq_exact_literal_xfc):
+ is_eq_exact_lit_val = xb(Arg(0));
+ I++;
+ goto do_is_eq_exact_literal;
+
+ OpCase(i_is_eq_exact_literal_yfc):
+ is_eq_exact_lit_val = yb(Arg(0));
+ I++;
+ goto do_is_eq_exact_literal;
+
+ OpCase(i_is_eq_exact_literal_rfc):
+ is_eq_exact_lit_val = r(0);
+
+ do_is_eq_exact_literal:
+ if (!eq(Arg(1), is_eq_exact_lit_val)) {
+ ClauseFail();
+ }
+ Next(2);
+ }
+
+ {
+ Eterm is_ne_exact_lit_val;
+
+ OpCase(i_is_ne_exact_literal_xfc):
+ is_ne_exact_lit_val = xb(Arg(0));
+ I++;
+ goto do_is_ne_exact_literal;
+
+ OpCase(i_is_ne_exact_literal_yfc):
+ is_ne_exact_lit_val = yb(Arg(0));
+ I++;
+ goto do_is_ne_exact_literal;
+
+ OpCase(i_is_ne_exact_literal_rfc):
+ is_ne_exact_lit_val = r(0);
+
+ do_is_ne_exact_literal:
+ if (eq(Arg(1), is_ne_exact_lit_val)) {
+ ClauseFail();
+ }
+ Next(2);
+ }
+
OpCase(i_move_call_only_fcr): {
r(0) = Arg(1);
}
@@ -1392,6 +1531,17 @@ void process_main(void)
NextPF(1, next);
}
+ OpCase(move_x1_c): {
+ x(1) = Arg(0);
+ Next(1);
+ }
+
+ OpCase(move_x2_c): {
+ x(2) = Arg(0);
+ Next(1);
+ }
+
+
OpCase(return): {
SET_I(c_p->cp);
/*
@@ -1405,16 +1555,6 @@ void process_main(void)
Goto(*I);
}
- OpCase(test_heap_1_put_list_Iy): {
- BeamInstr *next;
-
- PreFetch(2, next);
- TestHeap(Arg(0), 1);
- PutList(yb(Arg(1)), r(0), r(0), StoreSimpleDest);
- CHECK_TERM(r(0));
- NextPF(2, next);
- }
-
/*
* Send is almost a standard call-BIF with two arguments, except for:
* 1) It cannot be traced.
@@ -1447,24 +1587,36 @@ void process_main(void)
goto find_func_info;
}
- OpCase(i_element_jssd): {
- Eterm index;
- Eterm tuple;
-
- /*
- * Inlined version of element/2 for speed.
- */
- GetArg2(1, index, tuple);
- if (is_small(index) && is_tuple(tuple)) {
- Eterm* tp = tuple_val(tuple);
-
- if ((signed_val(index) >= 1) &&
- (signed_val(index) <= arityval(*tp))) {
- Eterm result = tp[signed_val(index)];
- StoreBifResult(3, result);
- }
- }
- }
+ {
+ Eterm element_index;
+ Eterm element_tuple;
+
+ OpCase(i_element_xjsd):
+ element_tuple = xb(Arg(0));
+ I++;
+ goto do_element;
+
+ OpCase(i_element_yjsd):
+ element_tuple = yb(Arg(0));
+ I++;
+ goto do_element;
+
+ OpCase(i_element_rjsd):
+ element_tuple = r(0);
+ /* Fall through */
+
+ do_element:
+ GetArg1(1, element_index);
+ if (is_small(element_index) && is_tuple(element_tuple)) {
+ Eterm* tp = tuple_val(element_tuple);
+
+ if ((signed_val(element_index) >= 1) &&
+ (signed_val(element_index) <= arityval(*tp))) {
+ Eterm result = tp[signed_val(element_index)];
+ StoreBifResult(2, result);
+ }
+ }
+ }
/* Fall through */
OpCase(badarg_j):
@@ -1472,24 +1624,32 @@ void process_main(void)
c_p->freason = BADARG;
goto lb_Cl_error;
- OpCase(i_fast_element_jIsd): {
- Eterm tuple;
-
- /*
- * Inlined version of element/2 for even more speed.
- * The first argument is an untagged integer >= 1.
- * The second argument is guaranteed to be a register operand.
- */
- GetArg1(2, tuple);
- if (is_tuple(tuple)) {
- Eterm* tp = tuple_val(tuple);
- tmp_arg2 = Arg(1);
- if (tmp_arg2 <= arityval(*tp)) {
- Eterm result = tp[tmp_arg2];
- StoreBifResult(3, result);
- }
- }
+ {
+ Eterm fast_element_tuple;
+
+ OpCase(i_fast_element_rjId):
+ fast_element_tuple = r(0);
+
+ do_fast_element:
+ if (is_tuple(fast_element_tuple)) {
+ Eterm* tp = tuple_val(fast_element_tuple);
+ Eterm pos = Arg(1); /* Untagged integer >= 1 */
+ if (pos <= arityval(*tp)) {
+ Eterm result = tp[pos];
+ StoreBifResult(2, result);
+ }
+ }
goto badarg;
+
+ OpCase(i_fast_element_xjId):
+ fast_element_tuple = xb(Arg(0));
+ I++;
+ goto do_fast_element;
+
+ OpCase(i_fast_element_yjId):
+ fast_element_tuple = yb(Arg(0));
+ I++;
+ goto do_fast_element;
}
OpCase(catch_yf):
@@ -1515,6 +1675,7 @@ void process_main(void)
SWAPOUT;
PROCESS_MAIN_CHK_LOCKS(c_p);
FCALLS -= erts_garbage_collect(c_p, 3, reg+2, 1);
+ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
PROCESS_MAIN_CHK_LOCKS(c_p);
SWAPIN;
}
@@ -1633,6 +1794,7 @@ void process_main(void)
PROCESS_MAIN_CHK_LOCKS(c_p);
},
{
+ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
PROCESS_MAIN_CHK_LOCKS(c_p);
r(0) = reg[0];
SWAPIN;
@@ -1691,6 +1853,7 @@ void process_main(void)
CANCEL_TIMER(c_p);
free_message(msgp);
+ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
PROCESS_MAIN_CHK_LOCKS(c_p);
NextPF(0, next);
@@ -1842,8 +2005,87 @@ void process_main(void)
NextPF(0, next);
}
- OpCase(i_select_val_sfI):
- GetArg1(0, tmp_arg1);
+
+ {
+ Eterm select_val2;
+
+ OpCase(i_select_tuple_arity2_yfAfAf):
+ select_val2 = yb(Arg(0));
+ goto do_select_tuple_arity2;
+
+ OpCase(i_select_tuple_arity2_xfAfAf):
+ select_val2 = xb(Arg(0));
+ goto do_select_tuple_arity2;
+
+ OpCase(i_select_tuple_arity2_rfAfAf):
+ select_val2 = r(0);
+ I--;
+
+ do_select_tuple_arity2:
+ if (is_not_tuple(select_val2)) {
+ goto select_val2_fail;
+ }
+ select_val2 = *tuple_val(select_val2);
+ goto do_select_val2;
+
+ OpCase(i_select_val2_yfcfcf):
+ select_val2 = yb(Arg(0));
+ goto do_select_val2;
+
+ OpCase(i_select_val2_xfcfcf):
+ select_val2 = xb(Arg(0));
+ goto do_select_val2;
+
+ OpCase(i_select_val2_rfcfcf):
+ select_val2 = r(0);
+ I--;
+
+ do_select_val2:
+ if (select_val2 == Arg(2)) {
+ I += 2;
+ } else if (select_val2 == Arg(4)) {
+ I += 4;
+ }
+
+ select_val2_fail:
+ SET_I((BeamInstr *) Arg(1));
+ Goto(*I);
+ }
+
+ {
+ Eterm select_val;
+
+ OpCase(i_select_tuple_arity_xfI):
+ select_val = xb(Arg(0));
+ goto do_select_tuple_arity;
+
+ OpCase(i_select_tuple_arity_yfI):
+ select_val = yb(Arg(0));
+ goto do_select_tuple_arity;
+
+ OpCase(i_select_tuple_arity_rfI):
+ select_val = r(0);
+ I--;
+
+ do_select_tuple_arity:
+ if (is_tuple(select_val)) {
+ select_val = *tuple_val(select_val);
+ goto do_binary_search;
+ }
+ SET_I((BeamInstr *) Arg(1));
+ Goto(*I);
+
+ OpCase(i_select_val_xfI):
+ select_val = xb(Arg(0));
+ goto do_binary_search;
+
+ OpCase(i_select_val_yfI):
+ select_val = yb(Arg(0));
+ goto do_binary_search;
+
+ OpCase(i_select_val_rfI):
+ select_val = r(0);
+ I--;
do_binary_search:
{
@@ -1880,9 +2122,9 @@ void process_main(void)
unsigned int boffset = ((unsigned int)bdiff >> 1) & ~(sizeof(struct Pairs)-1);
mid = (struct Pairs*)((char*)low + boffset);
- if (tmp_arg1 < mid->val) {
+ if (select_val < mid->val) {
high = mid;
- } else if (tmp_arg1 > mid->val) {
+ } else if (select_val > mid->val) {
low = mid + 1;
} else {
SET_I(mid->addr);
@@ -1892,16 +2134,28 @@ void process_main(void)
SET_I((BeamInstr *) Arg(1));
Goto(*I);
}
+ }
- OpCase(i_jump_on_val_zero_sfI):
{
- Eterm index;
-
- GetArg1(0, index);
- if (is_small(index)) {
- index = signed_val(index);
- if (index < Arg(2)) {
- SET_I((BeamInstr *) (&Arg(3))[index]);
+ Eterm jump_on_val_zero_index;
+
+ OpCase(i_jump_on_val_zero_yfI):
+ jump_on_val_zero_index = yb(Arg(0));
+ goto do_jump_on_val_zero_index;
+
+ OpCase(i_jump_on_val_zero_xfI):
+ jump_on_val_zero_index = xb(Arg(0));
+ goto do_jump_on_val_zero_index;
+
+ OpCase(i_jump_on_val_zero_rfI):
+ jump_on_val_zero_index = r(0);
+ I--;
+
+ do_jump_on_val_zero_index:
+ if (is_small(jump_on_val_zero_index)) {
+ jump_on_val_zero_index = signed_val(jump_on_val_zero_index);
+ if (jump_on_val_zero_index < Arg(2)) {
+ SET_I((BeamInstr *) (&Arg(3))[jump_on_val_zero_index]);
Goto(*I);
}
}
@@ -1909,15 +2163,27 @@ void process_main(void)
Goto(*I);
}
- OpCase(i_jump_on_val_sfII):
{
- Eterm index;
+ Eterm jump_on_val_index;
- GetArg1(0, index);
- if (is_small(index)) {
- index = (Uint) (signed_val(index) - Arg(3));
- if (index < Arg(2)) {
- SET_I((BeamInstr *) (&Arg(4))[index]);
+
+ OpCase(i_jump_on_val_yfII):
+ jump_on_val_index = yb(Arg(0));
+ goto do_jump_on_val_index;
+
+ OpCase(i_jump_on_val_xfII):
+ jump_on_val_index = xb(Arg(0));
+ goto do_jump_on_val_index;
+
+ OpCase(i_jump_on_val_rfII):
+ jump_on_val_index = r(0);
+ I--;
+
+ do_jump_on_val_index:
+ if (is_small(jump_on_val_index)) {
+ jump_on_val_index = (Uint) (signed_val(jump_on_val_index) - Arg(3));
+ if (jump_on_val_index < Arg(2)) {
+ SET_I((BeamInstr *) (&Arg(4))[jump_on_val_index]);
Goto(*I);
}
}
@@ -1925,6 +2191,32 @@ void process_main(void)
Goto(*I);
}
+ do_put_tuple: {
+ Eterm* hp = HTOP;
+
+ *hp++ = make_arityval(pt_arity);
+
+ do {
+ Eterm term = *I++;
+ switch (term & _TAG_IMMED1_MASK) {
+ case (R_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER:
+ *hp++ = r(0);
+ break;
+ case (X_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER:
+ *hp++ = x(term >> _TAG_IMMED1_SIZE);
+ break;
+ case (Y_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER:
+ *hp++ = y(term >> _TAG_IMMED1_SIZE);
+ break;
+ default:
+ *hp++ = term;
+ break;
+ }
+ } while (--pt_arity != 0);
+ HTOP = hp;
+ Goto(*I);
+ }
+
/*
* All guards with zero arguments have special instructions:
* self/0
@@ -1952,6 +2244,7 @@ void process_main(void)
ASSERT(!ERTS_PROC_IS_EXITING(c_p));
result = (*bf)(c_p, arg);
ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
+ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
PROCESS_MAIN_CHK_LOCKS(c_p);
ERTS_HOLE_CHECK(c_p);
FCALLS = c_p->fcalls;
@@ -1980,6 +2273,7 @@ void process_main(void)
ASSERT(!ERTS_PROC_IS_EXITING(c_p));
result = (*bf)(c_p, arg);
ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
+ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
PROCESS_MAIN_CHK_LOCKS(c_p);
ERTS_HOLE_CHECK(c_p);
FCALLS = c_p->fcalls;
@@ -2009,6 +2303,7 @@ void process_main(void)
PROCESS_MAIN_CHK_LOCKS(c_p);
ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
result = (*bf)(c_p, reg, live);
+ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
PROCESS_MAIN_CHK_LOCKS(c_p);
SWAPIN;
@@ -2044,6 +2339,7 @@ void process_main(void)
PROCESS_MAIN_CHK_LOCKS(c_p);
ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
result = (*bf)(c_p, reg, live);
+ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
PROCESS_MAIN_CHK_LOCKS(c_p);
SWAPIN;
@@ -2082,6 +2378,7 @@ void process_main(void)
PROCESS_MAIN_CHK_LOCKS(c_p);
ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
result = (*bf)(c_p, reg, live);
+ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
PROCESS_MAIN_CHK_LOCKS(c_p);
SWAPIN;
@@ -2116,6 +2413,7 @@ void process_main(void)
ASSERT(!ERTS_PROC_IS_EXITING(c_p));
result = (*bf)(c_p, tmp_arg1, tmp_arg2);
ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
+ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
PROCESS_MAIN_CHK_LOCKS(c_p);
ERTS_HOLE_CHECK(c_p);
FCALLS = c_p->fcalls;
@@ -2139,6 +2437,7 @@ void process_main(void)
ASSERT(!ERTS_PROC_IS_EXITING(c_p));
result = (*bf)(c_p, tmp_arg1, tmp_arg2);
ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
+ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
PROCESS_MAIN_CHK_LOCKS(c_p);
ERTS_HOLE_CHECK(c_p);
if (is_value(result)) {
@@ -2562,23 +2861,25 @@ void process_main(void)
OpCase(i_int_bnot_jsId):
{
- GetArg1(1, tmp_arg1);
- if (is_small(tmp_arg1)) {
- tmp_arg1 = make_small(~signed_val(tmp_arg1));
+ Eterm bnot_val;
+
+ GetArg1(1, bnot_val);
+ if (is_small(bnot_val)) {
+ bnot_val = make_small(~signed_val(bnot_val));
} else {
Uint live = Arg(2);
SWAPOUT;
reg[0] = r(0);
- reg[live] = tmp_arg1;
- tmp_arg1 = erts_gc_bnot(c_p, reg, live);
+ reg[live] = bnot_val;
+ bnot_val = erts_gc_bnot(c_p, reg, live);
r(0) = reg[0];
SWAPIN;
ERTS_HOLE_CHECK(c_p);
- if (is_nil(tmp_arg1)) {
+ if (is_nil(bnot_val)) {
goto lb_Cl_error;
}
}
- StoreBifResult(3, tmp_arg1);
+ StoreBifResult(3, bnot_val);
}
badarith:
@@ -2833,121 +3134,6 @@ void process_main(void)
goto do_schedule1;
}
- OpCase(i_select_tuple_arity_sfI):
- {
- GetArg1(0, tmp_arg1);
-
- if (is_tuple(tmp_arg1)) {
- tmp_arg1 = *tuple_val(tmp_arg1);
- goto do_binary_search;
- }
- SET_I((BeamInstr *) Arg(1));
- Goto(*I);
- }
-
- OpCase(i_select_big_sf):
- {
- Eterm* bigp;
- Uint arity;
- Eterm* given;
- Uint given_arity;
- Uint given_size;
-
- GetArg1(0, tmp_arg1);
- if (is_big(tmp_arg1)) {
-
- /*
- * The loader has sorted the bignumbers in descending order
- * on the arity word. Therefore, we know that the search
- * has failed as soon as we encounter an arity word less than
- * the arity word of the given number. There is a zero word
- * (less than any valid arity word) stored after the last bignumber.
- */
-
- given = big_val(tmp_arg1);
- given_arity = given[0];
- given_size = thing_arityval(given_arity);
- bigp = (Eterm *) &Arg(2);
- while ((arity = bigp[0]) > given_arity) {
- bigp += (TermWords(thing_arityval(arity) + 1) + 1) * (sizeof(BeamInstr)/sizeof(Eterm));
- }
- while (bigp[0] == given_arity) {
- if (memcmp(bigp+1, given+1, sizeof(Eterm)*given_size) == 0) {
- BeamInstr *tmp =
- ((BeamInstr *) (UWord) bigp) + TermWords(given_size + 1);
- SET_I((BeamInstr *) *tmp);
- Goto(*I);
- }
- bigp += (TermWords(thing_arityval(arity) + 1) + 1) * (sizeof(BeamInstr)/sizeof(Eterm));
- }
- }
-
- /*
- * Failed.
- */
-
- SET_I((BeamInstr *) Arg(1));
- Goto(*I);
- }
-
-#if defined(ARCH_64) && !HALFWORD_HEAP
- OpCase(i_select_float_sfI):
- {
- Uint f;
- int n;
- struct ValLabel {
- Uint f;
- BeamInstr* addr;
- };
- struct ValLabel* ptr;
-
- GetArg1(0, tmp_arg1);
- ASSERT(is_float(tmp_arg1));
- f = float_val(tmp_arg1)[1];
- n = Arg(2);
- ptr = (struct ValLabel *) &Arg(3);
- while (n-- > 0) {
- if (ptr->f == f) {
- SET_I(ptr->addr);
- Goto(*I);
- }
- ptr++;
- }
- SET_I((Eterm *) Arg(1));
- Goto(*I);
- }
-#else
- OpCase(i_select_float_sfI):
- {
- Uint fpart1;
- Uint fpart2;
- int n;
- struct ValLabel {
- Uint fpart1;
- Uint fpart2;
- BeamInstr* addr;
- };
- struct ValLabel* ptr;
-
- GetArg1(0, tmp_arg1);
- ASSERT(is_float(tmp_arg1));
- fpart1 = float_val(tmp_arg1)[1];
- fpart2 = float_val(tmp_arg1)[2];
-
- n = Arg(2);
- ptr = (struct ValLabel *) &Arg(3);
- while (n-- > 0) {
- if (ptr->fpart1 == fpart1 && ptr->fpart2 == fpart2) {
- SET_I(ptr->addr);
- Goto(*I);
- }
- ptr++;
- }
- SET_I((BeamInstr *) Arg(1));
- Goto(*I);
- }
-#endif
-
OpCase(set_tuple_element_sdP): {
Eterm element;
Eterm tuple;
@@ -2993,15 +3179,17 @@ void process_main(void)
the first argument. We also handle atom tags in the first
argument for backwards compatibility.
*/
- GetArg2(0, tmp_arg1, tmp_arg2);
- c_p->fvalue = tmp_arg2;
+ Eterm raise_val1;
+ Eterm raise_val2;
+ GetArg2(0, raise_val1, raise_val2);
+ c_p->fvalue = raise_val2;
if (c_p->freason == EXC_NULL) {
/* a safety check for the R10-0 case; should not happen */
c_p->ftrace = NIL;
c_p->freason = EXC_ERROR;
}
/* for R10-0 code, keep existing c_p->ftrace and hope it's correct */
- switch (tmp_arg1) {
+ switch (raise_val1) {
case am_throw:
c_p->freason = EXC_THROWN & ~EXF_SAVETRACE;
break;
@@ -3017,8 +3205,8 @@ void process_main(void)
passed from a user! Currently only expecting generated calls.
*/
struct StackTrace *s;
- c_p->ftrace = tmp_arg1;
- s = get_trace_from_exc(tmp_arg1);
+ c_p->ftrace = raise_val1;
+ s = get_trace_from_exc(raise_val1);
if (s == NULL) {
c_p->freason = EXC_ERROR;
} else {
@@ -3029,11 +3217,24 @@ void process_main(void)
goto find_func_info;
}
- OpCase(badmatch_s): {
- GetArg1(0, tmp_arg1);
- c_p->fvalue = tmp_arg1;
- c_p->freason = BADMATCH;
- }
+ {
+ Eterm badmatch_val;
+
+ OpCase(badmatch_y):
+ badmatch_val = yb(Arg(0));
+ goto do_badmatch;
+
+ OpCase(badmatch_x):
+ badmatch_val = xb(Arg(0));
+ goto do_badmatch;
+
+ OpCase(badmatch_r):
+ badmatch_val = r(0);
+
+ do_badmatch:
+ c_p->fvalue = badmatch_val;
+ c_p->freason = BADMATCH;
+ }
/* Fall through here */
find_func_info: {
@@ -3056,12 +3257,11 @@ void process_main(void)
*/
SWAPOUT;
reg[0] = r(0);
- tmp_arg1 = call_error_handler(c_p, I-3, reg);
+ I = call_error_handler(c_p, I-3, reg, am_undefined_function);
r(0) = reg[0];
SWAPIN;
- if (tmp_arg1) {
- SET_I(c_p->i);
- Dispatch();
+ if (I) {
+ Goto(*I);
}
/* Fall through */
@@ -3084,128 +3284,149 @@ void process_main(void)
}
}
- OpCase(call_nif):
- {
- /*
- * call_nif is always first instruction in function:
- *
- * I[-3]: Module
- * I[-2]: Function
- * I[-1]: Arity
- * I[0]: &&call_nif
- * I[1]: Function pointer to NIF function
- * I[2]: Pointer to erl_module_nif
- */
- BifFunction vbf;
-
- c_p->current = I-3; /* current and vbf set to please handle_error */
- SWAPOUT;
- c_p->fcalls = FCALLS - 1;
- PROCESS_MAIN_CHK_LOCKS(c_p);
- tmp_arg2 = I[-1];
- ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
+ {
+ Eterm nif_bif_result;
+ Eterm bif_nif_arity;
- ASSERT(!ERTS_PROC_IS_EXITING(c_p));
- {
- typedef Eterm NifF(struct enif_environment_t*, int argc, Eterm argv[]);
- NifF* fp = vbf = (NifF*) I[1];
- struct enif_environment_t env;
- erts_pre_nif(&env, c_p, (struct erl_module_nif*)I[2]);
- reg[0] = r(0);
- tmp_arg1 = (*fp)(&env, tmp_arg2, reg);
- erts_post_nif(&env);
- }
- ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(tmp_arg1));
- PROCESS_MAIN_CHK_LOCKS(c_p);
- goto apply_bif_or_nif_epilogue;
-
- OpCase(apply_bif):
- /*
- * At this point, I points to the code[3] in the export entry for
- * the BIF:
- *
- * code[0]: Module
- * code[1]: Function
- * code[2]: Arity
- * code[3]: &&apply_bif
- * code[4]: Function pointer to BIF function
- */
+ OpCase(call_nif):
+ {
+ /*
+ * call_nif is always first instruction in function:
+ *
+ * I[-3]: Module
+ * I[-2]: Function
+ * I[-1]: Arity
+ * I[0]: &&call_nif
+ * I[1]: Function pointer to NIF function
+ * I[2]: Pointer to erl_module_nif
+ */
+ BifFunction vbf;
- c_p->current = I-3; /* In case we apply process_info/1,2 or load_nif/1 */
- c_p->i = I; /* In case we apply check_process_code/2. */
- c_p->arity = 0; /* To allow garbage collection on ourselves
- * (check_process_code/2).
- */
- SWAPOUT;
- c_p->fcalls = FCALLS - 1;
- vbf = (BifFunction) Arg(0);
- PROCESS_MAIN_CHK_LOCKS(c_p);
- tmp_arg2 = I[-1];
- ASSERT(tmp_arg2 <= 3);
- ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
- switch (tmp_arg2) {
- case 3:
+ c_p->current = I-3; /* current and vbf set to please handle_error */
+ SWAPOUT;
+ c_p->fcalls = FCALLS - 1;
+ PROCESS_MAIN_CHK_LOCKS(c_p);
+ bif_nif_arity = I[-1];
+ ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
+ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+
+ ASSERT(!ERTS_PROC_IS_EXITING(c_p));
{
- Eterm (*bf)(Process*, Eterm, Eterm, Eterm, BeamInstr*) = vbf;
- ASSERT(!ERTS_PROC_IS_EXITING(c_p));
- tmp_arg1 = (*bf)(c_p, r(0), x(1), x(2), I);
- ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(tmp_arg1));
- PROCESS_MAIN_CHK_LOCKS(c_p);
+ typedef Eterm NifF(struct enif_environment_t*, int argc, Eterm argv[]);
+ NifF* fp = vbf = (NifF*) I[1];
+ struct enif_environment_t env;
+ erts_pre_nif(&env, c_p, (struct erl_module_nif*)I[2]);
+ reg[0] = r(0);
+ nif_bif_result = (*fp)(&env, bif_nif_arity, reg);
+ erts_post_nif(&env);
}
- break;
- case 2:
- {
- Eterm (*bf)(Process*, Eterm, Eterm, BeamInstr*) = vbf;
- ASSERT(!ERTS_PROC_IS_EXITING(c_p));
- tmp_arg1 = (*bf)(c_p, r(0), x(1), I);
- ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(tmp_arg1));
- PROCESS_MAIN_CHK_LOCKS(c_p);
+ ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(nif_bif_result));
+ PROCESS_MAIN_CHK_LOCKS(c_p);
+ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+ goto apply_bif_or_nif_epilogue;
+
+ OpCase(apply_bif):
+ /*
+ * At this point, I points to the code[3] in the export entry for
+ * the BIF:
+ *
+ * code[0]: Module
+ * code[1]: Function
+ * code[2]: Arity
+ * code[3]: &&apply_bif
+ * code[4]: Function pointer to BIF function
+ */
+
+ c_p->current = I-3; /* In case we apply process_info/1,2 or load_nif/1 */
+ c_p->i = I; /* In case we apply check_process_code/2. */
+ c_p->arity = 0; /* To allow garbage collection on ourselves
+ * (check_process_code/2).
+ */
+ SWAPOUT;
+ c_p->fcalls = FCALLS - 1;
+ vbf = (BifFunction) Arg(0);
+ PROCESS_MAIN_CHK_LOCKS(c_p);
+ bif_nif_arity = I[-1];
+ ASSERT(bif_nif_arity <= 3);
+ ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
+ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+ switch (bif_nif_arity) {
+ case 3:
+ {
+ Eterm (*bf)(Process*, Eterm, Eterm, Eterm, BeamInstr*) = vbf;
+ ASSERT(!ERTS_PROC_IS_EXITING(c_p));
+ nif_bif_result = (*bf)(c_p, r(0), x(1), x(2), I);
+ ASSERT(!ERTS_PROC_IS_EXITING(c_p) ||
+ is_non_value(nif_bif_result));
+ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+ PROCESS_MAIN_CHK_LOCKS(c_p);
+ }
+ break;
+ case 2:
+ {
+ Eterm (*bf)(Process*, Eterm, Eterm, BeamInstr*) = vbf;
+ ASSERT(!ERTS_PROC_IS_EXITING(c_p));
+ nif_bif_result = (*bf)(c_p, r(0), x(1), I);
+ ASSERT(!ERTS_PROC_IS_EXITING(c_p) ||
+ is_non_value(nif_bif_result));
+ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+ PROCESS_MAIN_CHK_LOCKS(c_p);
+ }
+ break;
+ case 1:
+ {
+ Eterm (*bf)(Process*, Eterm, BeamInstr*) = vbf;
+ ASSERT(!ERTS_PROC_IS_EXITING(c_p));
+ nif_bif_result = (*bf)(c_p, r(0), I);
+ ASSERT(!ERTS_PROC_IS_EXITING(c_p) ||
+ is_non_value(nif_bif_result));
+ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+ PROCESS_MAIN_CHK_LOCKS(c_p);
+ }
+ break;
+ case 0:
+ {
+ Eterm (*bf)(Process*, BeamInstr*) = vbf;
+ ASSERT(!ERTS_PROC_IS_EXITING(c_p));
+ nif_bif_result = (*bf)(c_p, I);
+ ASSERT(!ERTS_PROC_IS_EXITING(c_p) ||
+ is_non_value(nif_bif_result));
+ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+ PROCESS_MAIN_CHK_LOCKS(c_p);
+ break;
+ }
+ default:
+ erl_exit(1, "apply_bif: invalid arity: %u\n",
+ bif_nif_arity);
}
- break;
- case 1:
- {
- Eterm (*bf)(Process*, Eterm, BeamInstr*) = vbf;
- ASSERT(!ERTS_PROC_IS_EXITING(c_p));
- tmp_arg1 = (*bf)(c_p, r(0), I);
- ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(tmp_arg1));
- PROCESS_MAIN_CHK_LOCKS(c_p);
+
+ apply_bif_or_nif_epilogue:
+ ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
+ ERTS_HOLE_CHECK(c_p);
+ if (c_p->mbuf) {
+ reg[0] = r(0);
+ nif_bif_result = erts_gc_after_bif_call(c_p, nif_bif_result,
+ reg, bif_nif_arity);
+ r(0) = reg[0];
}
- break;
- case 0:
- {
- Eterm (*bf)(Process*, BeamInstr*) = vbf;
- ASSERT(!ERTS_PROC_IS_EXITING(c_p));
- tmp_arg1 = (*bf)(c_p, I);
- ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(tmp_arg1));
- PROCESS_MAIN_CHK_LOCKS(c_p);
- break;
+ SWAPIN; /* There might have been a garbage collection. */
+ FCALLS = c_p->fcalls;
+ if (is_value(nif_bif_result)) {
+ r(0) = nif_bif_result;
+ CHECK_TERM(r(0));
+ SET_I(c_p->cp);
+ Goto(*I);
+ } else if (c_p->freason == TRAP) {
+ SET_I(*((BeamInstr **) (UWord) ((c_p)->def_arg_reg + 3)));
+ r(0) = c_p->def_arg_reg[0];
+ x(1) = c_p->def_arg_reg[1];
+ x(2) = c_p->def_arg_reg[2];
+ Dispatch();
}
- }
-apply_bif_or_nif_epilogue:
- ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
- ERTS_HOLE_CHECK(c_p);
- if (c_p->mbuf) {
reg[0] = r(0);
- tmp_arg1 = erts_gc_after_bif_call(c_p, tmp_arg1, reg, tmp_arg2);
- r(0) = reg[0];
+ I = handle_error(c_p, c_p->cp, reg, vbf);
+ goto post_error_handling;
}
- SWAPIN; /* There might have been a garbage collection. */
- FCALLS = c_p->fcalls;
- if (is_value(tmp_arg1)) {
- r(0) = tmp_arg1;
- CHECK_TERM(r(0));
- SET_I(c_p->cp);
- Goto(*I);
- } else if (c_p->freason == TRAP) {
- SET_I(*((BeamInstr **) (UWord) ((c_p)->def_arg_reg + 3)));
- r(0) = c_p->def_arg_reg[0];
- x(1) = c_p->def_arg_reg[1];
- x(2) = c_p->def_arg_reg[2];
- Dispatch();
- }
- reg[0] = r(0);
- I = handle_error(c_p, c_p->cp, reg, vbf);
- goto post_error_handling;
}
OpCase(i_get_sd):
@@ -3218,11 +3439,25 @@ apply_bif_or_nif_epilogue:
StoreBifResult(1, result);
}
- OpCase(case_end_s):
- GetArg1(0, tmp_arg1);
- c_p->fvalue = tmp_arg1;
- c_p->freason = EXC_CASE_CLAUSE;
- goto find_func_info;
+ {
+ Eterm case_end_val;
+
+ OpCase(case_end_x):
+ case_end_val = xb(Arg(0));
+ goto do_case_end;
+
+ OpCase(case_end_y):
+ case_end_val = yb(Arg(0));
+ goto do_case_end;
+
+ OpCase(case_end_r):
+ case_end_val = r(0);
+
+ do_case_end:
+ c_p->fvalue = case_end_val;
+ c_p->freason = EXC_CASE_CLAUSE;
+ goto find_func_info;
+ }
OpCase(if_end):
c_p->freason = EXC_IF_CLAUSE;
@@ -3235,10 +3470,13 @@ apply_bif_or_nif_epilogue:
}
OpCase(try_case_end_s):
- GetArg1(0, tmp_arg1);
- c_p->fvalue = tmp_arg1;
- c_p->freason = EXC_TRY_CLAUSE;
- goto find_func_info;
+ {
+ Eterm try_case_end_val;
+ GetArg1(0, try_case_end_val);
+ c_p->fvalue = try_case_end_val;
+ c_p->freason = EXC_TRY_CLAUSE;
+ goto find_func_info;
+ }
/*
* Construction of binaries using new instructions.
@@ -3786,19 +4024,20 @@ apply_bif_or_nif_epilogue:
Eterm header;
BeamInstr *next;
Uint slots;
+ Eterm context;
OpCase(i_bs_start_match2_rfIId): {
- tmp_arg1 = r(0);
+ context = r(0);
do_start_match:
slots = Arg(2);
- if (!is_boxed(tmp_arg1)) {
+ if (!is_boxed(context)) {
ClauseFail();
}
PreFetch(4, next);
- header = *boxed_val(tmp_arg1);
+ header = *boxed_val(context);
if (header_is_bin_matchstate(header)) {
- ErlBinMatchState* ms = (ErlBinMatchState *) boxed_val(tmp_arg1);
+ ErlBinMatchState* ms = (ErlBinMatchState *) boxed_val(context);
Uint actual_slots = HEADER_NUM_SLOTS(header);
ms->save_offset[0] = ms->mb.offset;
if (actual_slots < slots) {
@@ -3806,8 +4045,8 @@ apply_bif_or_nif_epilogue:
Uint live = Arg(1);
Uint wordsneeded = ERL_BIN_MATCHSTATE_SIZE(slots);
- TestHeapPreserve(wordsneeded, live, tmp_arg1);
- ms = (ErlBinMatchState *) boxed_val(tmp_arg1);
+ TestHeapPreserve(wordsneeded, live, context);
+ ms = (ErlBinMatchState *) boxed_val(context);
dst = (ErlBinMatchState *) HTOP;
*dst = *ms;
*HTOP = HEADER_BIN_MATCHSTATE(slots);
@@ -3819,12 +4058,12 @@ apply_bif_or_nif_epilogue:
Eterm result;
Uint live = Arg(1);
Uint wordsneeded = ERL_BIN_MATCHSTATE_SIZE(slots);
- TestHeapPreserve(wordsneeded, live, tmp_arg1);
+ TestHeapPreserve(wordsneeded, live, context);
HEAP_TOP(c_p) = HTOP;
#ifdef DEBUG
c_p->stop = E; /* Needed for checking in HeapOnlyAlloc(). */
#endif
- result = erts_bs_start_match_2(c_p, tmp_arg1, slots);
+ result = erts_bs_start_match_2(c_p, context, slots);
HTOP = HEAP_TOP(c_p);
HEAP_SPACE_VERIFIED(0);
if (is_non_value(result)) {
@@ -3838,12 +4077,12 @@ apply_bif_or_nif_epilogue:
NextPF(4, next);
}
OpCase(i_bs_start_match2_xfIId): {
- tmp_arg1 = xb(Arg(0));
+ context = xb(Arg(0));
I++;
goto do_start_match;
}
OpCase(i_bs_start_match2_yfIId): {
- tmp_arg1 = yb(Arg(0));
+ context = yb(Arg(0));
I++;
goto do_start_match;
}
@@ -3936,93 +4175,105 @@ apply_bif_or_nif_epilogue:
NextPF(2, next);
}
+ {
+ Eterm bs_get_integer8_context;
+
OpCase(i_bs_get_integer_8_rfd): {
- tmp_arg1 = r(0);
- goto do_bs_get_integer_8;
- }
+ bs_get_integer8_context = r(0);
+ goto do_bs_get_integer_8;
+ }
OpCase(i_bs_get_integer_8_xfd): {
- tmp_arg1 = xb(Arg(0));
- I++;
- }
+ bs_get_integer8_context = xb(Arg(0));
+ I++;
+ }
do_bs_get_integer_8: {
- ErlBinMatchBuffer *_mb;
- Eterm _result;
- _mb = ms_matchbuffer(tmp_arg1);
- if (_mb->size - _mb->offset < 8) {
- ClauseFail();
- }
- if (BIT_OFFSET(_mb->offset) != 0) {
- _result = erts_bs_get_integer_2(c_p, 8, 0, _mb);
- } else {
- _result = make_small(_mb->base[BYTE_OFFSET(_mb->offset)]);
- _mb->offset += 8;
+ ErlBinMatchBuffer *_mb;
+ Eterm _result;
+ _mb = ms_matchbuffer(bs_get_integer8_context);
+ if (_mb->size - _mb->offset < 8) {
+ ClauseFail();
+ }
+ if (BIT_OFFSET(_mb->offset) != 0) {
+ _result = erts_bs_get_integer_2(c_p, 8, 0, _mb);
+ } else {
+ _result = make_small(_mb->base[BYTE_OFFSET(_mb->offset)]);
+ _mb->offset += 8;
+ }
+ StoreBifResult(1, _result);
}
- StoreBifResult(1, _result);
}
- OpCase(i_bs_get_integer_16_rfd): {
- tmp_arg1 = r(0);
+ {
+ Eterm bs_get_integer_16_context;
+
+ OpCase(i_bs_get_integer_16_rfd):
+ bs_get_integer_16_context = r(0);
goto do_bs_get_integer_16;
- }
- OpCase(i_bs_get_integer_16_xfd): {
- tmp_arg1 = xb(Arg(0));
+ OpCase(i_bs_get_integer_16_xfd):
+ bs_get_integer_16_context = xb(Arg(0));
I++;
- }
- do_bs_get_integer_16: {
- ErlBinMatchBuffer *_mb;
- Eterm _result;
- _mb = ms_matchbuffer(tmp_arg1);
- if (_mb->size - _mb->offset < 16) {
- ClauseFail();
- }
- if (BIT_OFFSET(_mb->offset) != 0) {
- _result = erts_bs_get_integer_2(c_p, 16, 0, _mb);
- } else {
- _result = make_small(get_int16(_mb->base+BYTE_OFFSET(_mb->offset)));
- _mb->offset += 16;
+ do_bs_get_integer_16:
+ {
+ ErlBinMatchBuffer *_mb;
+ Eterm _result;
+ _mb = ms_matchbuffer(bs_get_integer_16_context);
+ if (_mb->size - _mb->offset < 16) {
+ ClauseFail();
+ }
+ if (BIT_OFFSET(_mb->offset) != 0) {
+ _result = erts_bs_get_integer_2(c_p, 16, 0, _mb);
+ } else {
+ _result = make_small(get_int16(_mb->base+BYTE_OFFSET(_mb->offset)));
+ _mb->offset += 16;
+ }
+ StoreBifResult(1, _result);
}
- StoreBifResult(1, _result);
}
- OpCase(i_bs_get_integer_32_rfId): {
- tmp_arg1 = r(0);
+ {
+ Eterm bs_get_integer_32_context;
+
+ OpCase(i_bs_get_integer_32_rfId):
+ bs_get_integer_32_context = r(0);
goto do_bs_get_integer_32;
- }
+
- OpCase(i_bs_get_integer_32_xfId): {
- tmp_arg1 = xb(Arg(0));
+ OpCase(i_bs_get_integer_32_xfId):
+ bs_get_integer_32_context = xb(Arg(0));
I++;
- }
- do_bs_get_integer_32: {
- ErlBinMatchBuffer *_mb;
- Uint32 _integer;
- Eterm _result;
- _mb = ms_matchbuffer(tmp_arg1);
- if (_mb->size - _mb->offset < 32) { ClauseFail(); }
- if (BIT_OFFSET(_mb->offset) != 0) {
- _integer = erts_bs_get_unaligned_uint32(_mb);
- } else {
- _integer = get_int32(_mb->base + _mb->offset/8);
- }
- _mb->offset += 32;
+
+ do_bs_get_integer_32:
+ {
+ ErlBinMatchBuffer *_mb;
+ Uint32 _integer;
+ Eterm _result;
+ _mb = ms_matchbuffer(bs_get_integer_32_context);
+ if (_mb->size - _mb->offset < 32) { ClauseFail(); }
+ if (BIT_OFFSET(_mb->offset) != 0) {
+ _integer = erts_bs_get_unaligned_uint32(_mb);
+ } else {
+ _integer = get_int32(_mb->base + _mb->offset/8);
+ }
+ _mb->offset += 32;
#if !defined(ARCH_64) || HALFWORD_HEAP
- if (IS_USMALL(0, _integer)) {
+ if (IS_USMALL(0, _integer)) {
#endif
- _result = make_small(_integer);
+ _result = make_small(_integer);
#if !defined(ARCH_64) || HALFWORD_HEAP
- } else {
- TestHeap(BIG_UINT_HEAP_SIZE, Arg(1));
- _result = uint_to_big((Uint) _integer, HTOP);
- HTOP += BIG_UINT_HEAP_SIZE;
- HEAP_SPACE_VERIFIED(0);
- }
+ } else {
+ TestHeap(BIG_UINT_HEAP_SIZE, Arg(1));
+ _result = uint_to_big((Uint) _integer, HTOP);
+ HTOP += BIG_UINT_HEAP_SIZE;
+ HEAP_SPACE_VERIFIED(0);
+ }
#endif
- StoreBifResult(2, _result);
+ StoreBifResult(2, _result);
+ }
}
/* Operands: Size Live Fail Flags Dst */
@@ -4120,54 +4371,64 @@ apply_bif_or_nif_epilogue:
StoreBifResult(3, result);
}
- /* Operands: MatchContext Fail Dst */
+ {
+ Eterm get_utf8_context;
+
+ /* Operands: MatchContext Fail Dst */
OpCase(i_bs_get_utf8_rfd): {
- tmp_arg1 = r(0);
- goto do_bs_get_utf8;
- }
+ get_utf8_context = r(0);
+ goto do_bs_get_utf8;
+ }
OpCase(i_bs_get_utf8_xfd): {
- tmp_arg1 = xb(Arg(0));
- I++;
- }
+ get_utf8_context = xb(Arg(0));
+ I++;
+ }
- /*
- * tmp_arg1 = match_context
- * Operands: Fail Dst
- */
+ /*
+ * get_utf8_context = match_context
+ * Operands: Fail Dst
+ */
- do_bs_get_utf8: {
- Eterm result = erts_bs_get_utf8(ms_matchbuffer(tmp_arg1));
- if (is_non_value(result)) {
- ClauseFail();
+ do_bs_get_utf8: {
+ Eterm result = erts_bs_get_utf8(ms_matchbuffer(get_utf8_context));
+ if (is_non_value(result)) {
+ ClauseFail();
+ }
+ StoreBifResult(1, result);
}
- StoreBifResult(1, result);
}
- /* Operands: MatchContext Fail Flags Dst */
+ {
+ Eterm get_utf16_context;
+
+ /* Operands: MatchContext Fail Flags Dst */
OpCase(i_bs_get_utf16_rfId): {
- tmp_arg1 = r(0);
- goto do_bs_get_utf16;
- }
+ get_utf16_context = r(0);
+ goto do_bs_get_utf16;
+ }
OpCase(i_bs_get_utf16_xfId): {
- tmp_arg1 = xb(Arg(0));
- I++;
- }
+ get_utf16_context = xb(Arg(0));
+ I++;
+ }
- /*
- * tmp_arg1 = match_context
- * Operands: Fail Flags Dst
- */
- do_bs_get_utf16: {
- Eterm result = erts_bs_get_utf16(ms_matchbuffer(tmp_arg1), Arg(1));
- if (is_non_value(result)) {
- ClauseFail();
+ /*
+ * get_utf16_context = match_context
+ * Operands: Fail Flags Dst
+ */
+ do_bs_get_utf16: {
+ Eterm result = erts_bs_get_utf16(ms_matchbuffer(get_utf16_context),
+ Arg(1));
+ if (is_non_value(result)) {
+ ClauseFail();
+ }
+ StoreBifResult(2, result);
}
- StoreBifResult(2, result);
}
{
+ Eterm context_to_binary_context;
ErlBinMatchBuffer* mb;
ErlSubBin* sb;
Uint size;
@@ -4176,27 +4437,29 @@ apply_bif_or_nif_epilogue:
Uint hole_size;
OpCase(bs_context_to_binary_r): {
- tmp_arg1 = x0;
+ context_to_binary_context = x0;
I -= 2;
goto do_context_to_binary;
}
/* Unfortunately, inlining can generate this instruction. */
OpCase(bs_context_to_binary_y): {
- tmp_arg1 = yb(Arg(0));
+ context_to_binary_context = yb(Arg(0));
goto do_context_to_binary0;
}
OpCase(bs_context_to_binary_x): {
- tmp_arg1 = xb(Arg(0));
+ context_to_binary_context = xb(Arg(0));
do_context_to_binary0:
I--;
}
do_context_to_binary:
- if (is_boxed(tmp_arg1) && header_is_bin_matchstate(*boxed_val(tmp_arg1))) {
- ErlBinMatchState* ms = (ErlBinMatchState *) boxed_val(tmp_arg1);
+ if (is_boxed(context_to_binary_context) &&
+ header_is_bin_matchstate(*boxed_val(context_to_binary_context))) {
+ ErlBinMatchState* ms;
+ ms = (ErlBinMatchState *) boxed_val(context_to_binary_context);
mb = &ms->mb;
offs = ms->save_offset[0];
size = mb->size - offs;
@@ -4205,17 +4468,17 @@ apply_bif_or_nif_epilogue:
Next(2);
OpCase(i_bs_get_binary_all_reuse_rfI): {
- tmp_arg1 = x0;
+ context_to_binary_context = x0;
goto do_bs_get_binary_all_reuse;
}
OpCase(i_bs_get_binary_all_reuse_xfI): {
- tmp_arg1 = xb(Arg(0));
+ context_to_binary_context = xb(Arg(0));
I++;
}
do_bs_get_binary_all_reuse:
- mb = ms_matchbuffer(tmp_arg1);
+ mb = ms_matchbuffer(context_to_binary_context);
size = mb->size - mb->offset;
if (size % Arg(1) != 0) {
ClauseFail();
@@ -4224,7 +4487,7 @@ apply_bif_or_nif_epilogue:
do_bs_get_binary_all_reuse_common:
orig = mb->orig;
- sb = (ErlSubBin *) boxed_val(tmp_arg1);
+ sb = (ErlSubBin *) boxed_val(context_to_binary_context);
hole_size = 1 + header_arity(sb->thing_word) - ERL_SUB_BIN_SIZE;
sb->thing_word = HEADER_SUB_BIN;
sb->size = BYTE_OFFSET(size);
@@ -4240,12 +4503,14 @@ apply_bif_or_nif_epilogue:
}
{
+ Eterm match_string_context;
+
OpCase(i_bs_match_string_rfII): {
- tmp_arg1 = r(0);
+ match_string_context = r(0);
goto do_bs_match_string;
}
OpCase(i_bs_match_string_xfII): {
- tmp_arg1 = xb(Arg(0));
+ match_string_context = xb(Arg(0));
I++;
}
@@ -4260,7 +4525,7 @@ apply_bif_or_nif_epilogue:
PreFetch(3, next);
bits = Arg(1);
bytes = (byte *) Arg(2);
- mb = ms_matchbuffer(tmp_arg1);
+ mb = ms_matchbuffer(match_string_context);
if (mb->size - mb->offset < bits) {
ClauseFail();
}
@@ -4353,6 +4618,7 @@ apply_bif_or_nif_epilogue:
ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
flags = erts_call_trace(c_p, ep->code, ep->match_prog_set, reg,
0, &c_p->tracer_proc);
+ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
PROCESS_MAIN_CHK_LOCKS(c_p);
ASSERT(!ERTS_PROC_IS_EXITING(c_p));
@@ -4364,6 +4630,7 @@ apply_bif_or_nif_epilogue:
/* SWAPOUT, SWAPIN was done and r(0) was saved above */
PROCESS_MAIN_CHK_LOCKS(c_p);
FCALLS -= erts_garbage_collect(c_p, 3, reg, ep->code[2]);
+ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
PROCESS_MAIN_CHK_LOCKS(c_p);
r(0) = reg[0];
SWAPIN;
@@ -4453,6 +4720,7 @@ apply_bif_or_nif_epilogue:
reg[0] = r(0);
PROCESS_MAIN_CHK_LOCKS(c_p);
FCALLS -= erts_garbage_collect(c_p, 2, reg, I[-1]);
+ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
PROCESS_MAIN_CHK_LOCKS(c_p);
r(0) = reg[0];
}
@@ -4556,6 +4824,7 @@ apply_bif_or_nif_epilogue:
/* SWAPOUT was done and r(0) was saved above */
PROCESS_MAIN_CHK_LOCKS(c_p);
FCALLS -= erts_garbage_collect(c_p, need, reg, I[-1]);
+ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
PROCESS_MAIN_CHK_LOCKS(c_p);
r(0) = reg[0];
SWAPIN;
@@ -4723,7 +4992,7 @@ apply_bif_or_nif_epilogue:
NextPF(2, next);
}
- OpCase(fmove_new_ld): {
+ OpCase(fmove_ld): {
Eterm fr = Arg(0);
Eterm dest = make_float(HTOP);
@@ -4753,11 +5022,6 @@ apply_bif_or_nif_epilogue:
NextPF(2, next);
}
- /*
- * Old allocating fmove.
- */
-
-
#ifdef NO_FPE_SIGNALS
OpCase(fclearerror):
OpCase(i_fcheckerror):
@@ -4969,12 +5233,11 @@ apply_bif_or_nif_epilogue:
OpCase(i_debug_breakpoint): {
SWAPOUT;
reg[0] = r(0);
- tmp_arg1 = call_breakpoint_handler(c_p, I-3, reg);
+ I = call_error_handler(c_p, I-3, reg, am_breakpoint);
r(0) = reg[0];
SWAPIN;
- if (tmp_arg1) {
- SET_I(c_p->i);
- Dispatch();
+ if (I) {
+ Goto(*I);
}
goto no_error_handler;
}
@@ -5633,9 +5896,6 @@ build_stacktrace(Process* c_p, Eterm exc) {
Eterm args;
int depth;
BeamInstr* current;
-#if HALFWORD_HEAP
- BeamInstr current_buff[3];
-#endif
Eterm Where = NIL;
Eterm *next_p = &Where;
@@ -5665,14 +5925,7 @@ build_stacktrace(Process* c_p, Eterm exc) {
* (e.g. spawn_link(erlang, abs, [1])).
*/
if (current == NULL) {
-#if HALFWORD_HEAP
- current = current_buff;
- current[0] = (BeamInstr) c_p->initial[0];
- current[1] = (BeamInstr) c_p->initial[1];
- current[2] = (BeamInstr) c_p->initial[2];
-#else
current = c_p->initial;
-#endif
args = am_true; /* Just in case */
} else {
args = get_args_from_exc(exc);
@@ -5724,8 +5977,8 @@ build_stacktrace(Process* c_p, Eterm exc) {
}
-static Eterm
-call_error_handler(Process* p, BeamInstr* fi, Eterm* reg)
+static BeamInstr*
+call_error_handler(Process* p, BeamInstr* fi, Eterm* reg, Eterm func)
{
Eterm* hp;
Export* ep;
@@ -5737,62 +5990,12 @@ call_error_handler(Process* p, BeamInstr* fi, Eterm* reg)
/*
* Search for the error_handler module.
*/
- ep = erts_find_function(erts_proc_get_error_handler(p),
- am_undefined_function, 3);
- if (ep == NULL) { /* No error handler */
- p->current = fi;
- p->freason = EXC_UNDEF;
- return 0;
- }
- p->i = ep->address;
-
- /*
- * Create a list with all arguments in the x registers.
- */
-
- arity = fi[2];
- sz = 2 * arity;
- if (HeapWordsLeft(p) < sz) {
- erts_garbage_collect(p, sz, reg, arity);
- }
- hp = HEAP_TOP(p);
- HEAP_TOP(p) += sz;
- args = NIL;
- for (i = arity-1; i >= 0; i--) {
- args = CONS(hp, reg[i], args);
- hp += 2;
- }
-
- /*
- * Set up registers for call to error_handler:undefined_function/3.
- */
- reg[0] = fi[0];
- reg[1] = fi[1];
- reg[2] = args;
- return 1;
-}
-
-static Eterm
-call_breakpoint_handler(Process* p, BeamInstr* fi, Eterm* reg)
-{
- Eterm* hp;
- Export* ep;
- int arity;
- Eterm args;
- Uint sz;
- int i;
-
- /*
- * Search for error handler module.
- */
- ep = erts_find_function(erts_proc_get_error_handler(p),
- am_breakpoint, 3);
+ ep = erts_find_function(erts_proc_get_error_handler(p), func, 3);
if (ep == NULL) { /* No error handler */
p->current = fi;
p->freason = EXC_UNDEF;
return 0;
}
- p->i = ep->address;
/*
* Create a list with all arguments in the x registers.
@@ -5812,15 +6015,14 @@ call_breakpoint_handler(Process* p, BeamInstr* fi, Eterm* reg)
}
/*
- * Set up registers for call to error_handler:breakpoint/3.
+ * Set up registers for call to error_handler:<func>/3.
*/
reg[0] = fi[0];
reg[1] = fi[1];
reg[2] = args;
- return 1;
+ return ep->address;
}
-
static Export*
apply_setup_error_handler(Process* p, Eterm module, Eterm function, Uint arity, Eterm* reg)
@@ -6069,6 +6271,7 @@ hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* reg)
c_p->fvalue = NIL;
PROCESS_MAIN_CHK_LOCKS(c_p);
erts_garbage_collect_hibernate(c_p);
+ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
PROCESS_MAIN_CHK_LOCKS(c_p);
erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
ASSERT(!ERTS_PROC_IS_EXITING(c_p));
@@ -6314,6 +6517,7 @@ new_fun(Process* p, Eterm* reg, ErlFunEntry* fe, int num_free)
if (HEAP_LIMIT(p) - HEAP_TOP(p) <= needed) {
PROCESS_MAIN_CHK_LOCKS(p);
erts_garbage_collect(p, needed, reg, num_free);
+ ERTS_VERIFY_UNUSED_TEMP_ALLOC(p);
PROCESS_MAIN_CHK_LOCKS(p);
}
hp = p->htop;
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index df5602b040..788cb4209c 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 1996-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -89,13 +89,12 @@ typedef struct {
} Label;
/*
- * Type for a operand for a generic instruction.
+ * Type for an operand for a generic instruction.
*/
typedef struct {
unsigned type; /* Type of operand. */
- BeamInstr val; /* Value of operand. */
- Uint bigarity; /* Arity for bignumbers (only). */
+ BeamInstr val; /* Value of operand. */
} GenOpArg;
/*
@@ -326,11 +325,6 @@ typedef struct {
Literal* literals; /* Array of literals. */
LiteralPatch* literal_patches; /* Operands that need to be patched. */
Uint total_literal_size; /* Total heap size for all literals. */
-
- /*
- * Floating point.
- */
- int new_float_instructions; /* New allocation scheme for floating point. */
} LoaderState;
typedef struct {
@@ -476,12 +470,14 @@ static int read_code_header(LoaderState* stp);
static int load_code(LoaderState* stp);
static GenOp* gen_element(LoaderState* stp, GenOpArg Fail, GenOpArg Index,
GenOpArg Tuple, GenOpArg Dst);
-static GenOp* gen_split_values(LoaderState* stp, GenOpArg S, GenOpArg Fail,
+static GenOp* gen_split_values(LoaderState* stp, GenOpArg S,
+ GenOpArg TypeFail, GenOpArg Fail,
GenOpArg Size, GenOpArg* Rest);
static GenOp* gen_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail,
GenOpArg Size, GenOpArg* Rest);
-static GenOp* gen_select_big(LoaderState* stp, GenOpArg S, GenOpArg Fail,
- GenOpArg Size, GenOpArg* Rest);
+static GenOp* gen_select_literals(LoaderState* stp, GenOpArg S,
+ GenOpArg Fail, GenOpArg Size,
+ GenOpArg* Rest);
static GenOp* const_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail,
GenOpArg Size, GenOpArg* Rest);
static GenOp* gen_func_info(LoaderState* stp, GenOpArg mod, GenOpArg Func,
@@ -818,7 +814,6 @@ init_state(LoaderState* stp)
stp->total_literal_size = 0;
stp->literal_patches = 0;
stp->string_patches = 0;
- stp->new_float_instructions = 0;
stp->may_load_nif = 0;
stp->on_load = 0;
}
@@ -1618,7 +1613,6 @@ load_code(LoaderState* stp)
BeamInstr val;
BeamInstr words = 0;
- stp->new_float_instructions = 1;
GetTagAndValue(stp, tag, n);
VerifyTag(stp, tag, TAG_u);
while (n-- > 0) {
@@ -1772,7 +1766,7 @@ load_code(LoaderState* stp)
}
stp->specific_op = specific;
- CodeNeed(opc[stp->specific_op].sz+2); /* Extra margin for packing */
+ CodeNeed(opc[stp->specific_op].sz+16); /* Extra margin for packing */
code[ci++] = BeamOpCode(stp->specific_op);
}
@@ -1936,7 +1930,8 @@ load_code(LoaderState* stp)
}
code[ci++] = (BeamInstr) stp->import[i].bf;
break;
- case 'P': /* Byte offset into tuple */
+ case 'P': /* Byte offset into tuple or stack */
+ case 'Q': /* Like 'P', but packable */
VerifyTag(stp, tag, TAG_u);
tmp = tmp_op->a[arg].val;
code[ci++] = (BeamInstr) ((tmp_op->a[arg].val+1) * sizeof(Eterm));
@@ -1957,84 +1952,6 @@ load_code(LoaderState* stp)
}
/*
- * Load any list arguments using the primitive tags.
- */
-
- for ( ; arg < tmp_op->arity; arg++) {
- switch (tmp_op->a[arg].type) {
- case TAG_i:
- CodeNeed(1);
- code[ci++] = make_small(tmp_op->a[arg].val);
- break;
- case TAG_u:
- case TAG_a:
- case TAG_v:
- CodeNeed(1);
- code[ci++] = tmp_op->a[arg].val;
- break;
- case TAG_f:
- CodeNeed(1);
- code[ci] = stp->labels[tmp_op->a[arg].val].patches;
- stp->labels[tmp_op->a[arg].val].patches = ci;
- ci++;
- break;
- case TAG_q:
- {
- Eterm lit;
-
- lit = stp->literals[tmp_op->a[arg].val].term;
- if (is_big(lit)) {
- Eterm* bigp;
- Eterm *tmp;
- Uint size;
- Uint term_size;
-
- bigp = big_val(lit);
- term_size = bignum_header_arity(*bigp);
- size = TermWords(term_size + 1);
- CodeNeed(size);
- tmp = (Eterm *) (code + ci);
- *tmp++ = *bigp++;
- while (term_size-- > 0) {
- *tmp++ = *bigp++;
- }
- ci +=size;
- } else if (is_float(lit)) {
-#if defined(ARCH_64) && !HALFWORD_HEAP
- CodeNeed(1);
- code[ci++] = float_val(stp->literals[tmp_op->a[arg].val].term)[1];
-#elif HALFWORD_HEAP
- Eterm* fptr;
- Uint size;
- Eterm *tmp;
-
- fptr = float_val(stp->literals[tmp_op->a[arg].val].term)+1;
- size = TermWords(2);
- CodeNeed(size);
- tmp = (Eterm *) (code + ci);
- *tmp++ = *fptr++;
- *tmp = *fptr;
- ci += size;
-#else
- Eterm* fptr;
-
- fptr = float_val(stp->literals[tmp_op->a[arg].val].term)+1;
- CodeNeed(2);
- code[ci++] = *fptr++;
- code[ci++] = *fptr;
-#endif
- } else {
- LoadError0(stp, "literal is neither float nor big");
- }
- }
- break;
- default:
- LoadError1(stp, "unsupported primitive type '%c'",
- tag_to_letter[tmp_op->a[arg].type]);
- }
- }
-
- /*
* The packing engine.
*/
if (opc[stp->specific_op].pack[0]) {
@@ -2057,6 +1974,11 @@ load_code(LoaderState* stp)
case '6': /* Shift 16 steps */
packed = (packed << BEAM_LOOSE_SHIFT) | code[--ci];
break;
+#ifdef ARCH_64
+ case 'w': /* Shift 32 steps */
+ packed = (packed << BEAM_WIDE_SHIFT) | code[--ci];
+ break;
+#endif
case 'p': /* Put instruction (from stack). */
code[ci++] = *--sp;
break;
@@ -2072,6 +1994,58 @@ load_code(LoaderState* stp)
}
/*
+ * Load any list arguments using the primitive tags.
+ */
+
+ for ( ; arg < tmp_op->arity; arg++) {
+ switch (tmp_op->a[arg].type) {
+ case TAG_i:
+ CodeNeed(1);
+ code[ci++] = make_small(tmp_op->a[arg].val);
+ break;
+ case TAG_u:
+ case TAG_a:
+ case TAG_v:
+ CodeNeed(1);
+ code[ci++] = tmp_op->a[arg].val;
+ break;
+ case TAG_f:
+ CodeNeed(1);
+ code[ci] = stp->labels[tmp_op->a[arg].val].patches;
+ stp->labels[tmp_op->a[arg].val].patches = ci;
+ ci++;
+ break;
+ case TAG_r:
+ CodeNeed(1);
+ code[ci++] = (R_REG_DEF << _TAG_PRIMARY_SIZE) |
+ TAG_PRIMARY_HEADER;
+ break;
+ case TAG_x:
+ CodeNeed(1);
+ code[ci++] = (tmp_op->a[arg].val << _TAG_IMMED1_SIZE) |
+ (X_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER;
+ break;
+ case TAG_y:
+ CodeNeed(1);
+ code[ci++] = (tmp_op->a[arg].val << _TAG_IMMED1_SIZE) |
+ (Y_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER;
+ break;
+ case TAG_n:
+ CodeNeed(1);
+ code[ci++] = NIL;
+ break;
+ case TAG_q:
+ CodeNeed(1);
+ new_literal_patch(stp, ci);
+ code[ci++] = tmp_op->a[arg].val;
+ break;
+ default:
+ LoadError1(stp, "unsupported primitive type '%c'",
+ tag_to_letter[tmp_op->a[arg].type]);
+ }
+ }
+
+ /*
* Handle a few special cases.
*/
switch (stp->specific_op) {
@@ -2239,11 +2213,12 @@ use_jump_tab(LoaderState* stp, GenOpArg Size, GenOpArg* Rest)
}
/*
- * Predicate to test whether all values in a table are big numbers.
+ * Predicate to test whether all values in a table are either
+ * floats or bignums.
*/
static int
-all_values_are_big(LoaderState* stp, GenOpArg Size, GenOpArg* Rest)
+floats_or_bignums(LoaderState* stp, GenOpArg Size, GenOpArg* Rest)
{
int i;
@@ -2255,9 +2230,6 @@ all_values_are_big(LoaderState* stp, GenOpArg Size, GenOpArg* Rest)
if (Rest[i].type != TAG_q) {
return 0;
}
- if (is_not_big(stp->literals[Rest[i].val].term)) {
- return 0;
- }
if (Rest[i+1].type != TAG_f) {
return 0;
}
@@ -2317,6 +2289,14 @@ mixed_types(LoaderState* stp, GenOpArg Size, GenOpArg* Rest)
return 0;
}
+static int
+same_label(LoaderState* stp, GenOpArg Target, GenOpArg Label)
+{
+ return Target.type = TAG_f && Label.type == TAG_u &&
+ Target.val == Label.val;
+}
+
+
/*
* Generate an instruction for element/2.
*/
@@ -2328,23 +2308,23 @@ gen_element(LoaderState* stp, GenOpArg Fail, GenOpArg Index,
GenOp* op;
NEW_GENOP(stp, op);
- op->op = genop_i_element_4;
op->arity = 4;
- op->a[0] = Fail;
- op->a[1] = Index;
- op->a[2] = Tuple;
- op->a[3] = Dst;
op->next = NULL;
- /*
- * If safe, generate a faster instruction.
- */
-
if (Index.type == TAG_i && Index.val > 0 &&
(Tuple.type == TAG_r || Tuple.type == TAG_x || Tuple.type == TAG_y)) {
op->op = genop_i_fast_element_4;
- op->a[1].type = TAG_u;
- op->a[1].val = Index.val;
+ op->a[0] = Tuple;
+ op->a[1] = Fail;
+ op->a[2].type = TAG_u;
+ op->a[2].val = Index.val;
+ op->a[3] = Dst;
+ } else {
+ op->op = genop_i_element_4;
+ op->a[0] = Tuple;
+ op->a[1] = Fail;
+ op->a[2] = Index;
+ op->a[3] = Dst;
}
return op;
@@ -2595,8 +2575,6 @@ binary_too_big_bits(LoaderState* stp, GenOpArg Size)
return Size.type == TAG_u && (((Size.val+7)/8) >> (8*sizeof(Uint)-3) != 0);
}
-#define new_float_allocation(Stp) ((Stp)->new_float_instructions)
-
static GenOp*
gen_put_binary(LoaderState* stp, GenOpArg Fail,GenOpArg Size,
GenOpArg Unit, GenOpArg Flags, GenOpArg Src)
@@ -2809,6 +2787,52 @@ gen_skip_bits2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms,
return op;
}
+static GenOp*
+gen_increment(LoaderState* stp, GenOpArg Reg, GenOpArg Integer,
+ GenOpArg Live, GenOpArg Dst)
+{
+ GenOp* op;
+
+ NEW_GENOP(stp, op);
+ op->op = genop_i_increment_4;
+ op->arity = 4;
+ op->next = NULL;
+ op->a[0] = Reg;
+ op->a[1].type = TAG_u;
+ op->a[1].val = Integer.val;
+ op->a[2] = Live;
+ op->a[3] = Dst;
+ return op;
+}
+
+static GenOp*
+gen_increment_from_minus(LoaderState* stp, GenOpArg Reg, GenOpArg Integer,
+ GenOpArg Live, GenOpArg Dst)
+{
+ GenOp* op;
+
+ NEW_GENOP(stp, op);
+ op->op = genop_i_increment_4;
+ op->arity = 4;
+ op->next = NULL;
+ op->a[0] = Reg;
+ op->a[1].type = TAG_u;
+ op->a[1].val = -Integer.val;
+ op->a[2] = Live;
+ op->a[3] = Dst;
+ return op;
+}
+
+/*
+ * Test whether the negation of the given number is small.
+ */
+static int
+negation_is_small(LoaderState* stp, GenOpArg Int)
+{
+ return Int.type == TAG_i && IS_SSMALL(-Int.val);
+}
+
+
static int
smp(LoaderState* stp)
{
@@ -3000,6 +3024,21 @@ gen_select_tuple_arity(LoaderState* stp, GenOpArg S, GenOpArg Fail,
ASSERT(op->a[i].val < op->a[i+2].val);
}
#endif
+
+ /*
+ * Use a special-cased instruction if there are only two values.
+ */
+ if (size == 2) {
+ op->op = genop_i_select_tuple_arity2_6;
+ op->arity--;
+ op->a[2].type = TAG_u;
+ op->a[2].val = arityval(op->a[3].val);
+ op->a[3] = op->a[4];
+ op->a[4].type = TAG_u;
+ op->a[4].val = arityval(op->a[5].val);
+ op->a[5] = op->a[6];
+ }
+
return op;
}
@@ -3009,18 +3048,24 @@ gen_select_tuple_arity(LoaderState* stp, GenOpArg S, GenOpArg Fail,
*/
static GenOp*
-gen_split_values(LoaderState* stp, GenOpArg S, GenOpArg Fail,
- GenOpArg Size, GenOpArg* Rest)
+gen_split_values(LoaderState* stp, GenOpArg S, GenOpArg TypeFail,
+ GenOpArg Fail, GenOpArg Size, GenOpArg* Rest)
{
GenOp* op1;
GenOp* op2;
GenOp* label;
- Uint type;
+ GenOp* is_integer;
int i;
ASSERT(Size.val >= 2 && Size.val % 2 == 0);
+ NEW_GENOP(stp, is_integer);
+ is_integer->op = genop_is_integer_2;
+ is_integer->arity = 2;
+ is_integer->a[0] = TypeFail;
+ is_integer->a[1] = S;
+
NEW_GENOP(stp, label);
label->op = genop_label_1;
label->arity = 1;
@@ -3046,15 +3091,13 @@ gen_split_values(LoaderState* stp, GenOpArg S, GenOpArg Fail,
op2->a[2].type = TAG_u;
op2->a[2].val = 0;
- op1->next = label;
- label->next = op2;
- op2->next = NULL;
-
- type = Rest[0].type;
+ /*
+ * Split the list.
+ */
ASSERT(Size.type == TAG_u);
for (i = 0; i < Size.val; i += 2) {
- GenOp* op = (Rest[i].type == type) ? op1 : op2;
+ GenOp* op = (Rest[i].type == TAG_q) ? op2 : op1;
int dst = 3 + op->a[2].val;
ASSERT(Rest[i+1].type == TAG_f);
@@ -3063,13 +3106,36 @@ gen_split_values(LoaderState* stp, GenOpArg S, GenOpArg Fail,
op->arity += 2;
op->a[2].val += 2;
}
+ ASSERT(op1->a[2].val > 0);
+ ASSERT(op2->a[2].val > 0);
/*
- * None of the instructions should have zero elements in the list.
+ * Order the instruction sequence appropriately.
*/
- ASSERT(op1->a[2].val > 0);
- ASSERT(op2->a[2].val > 0);
+ if (TypeFail.val == Fail.val) {
+ /*
+ * select_val L1 S ... (small numbers)
+ * label L1
+ * is_integer Fail S
+ * select_val Fail S ... (bignums)
+ */
+ op1->next = label;
+ label->next = is_integer;
+ is_integer->next = op2;
+ } else {
+ /*
+ * is_integer TypeFail S
+ * select_val L1 S ... (small numbers)
+ * label L1
+ * select_val Fail S ... (bignums)
+ */
+ is_integer->next = op1;
+ op1->next = label;
+ label->next = op2;
+ op1 = is_integer;
+ }
+ op2->next = NULL;
return op1;
}
@@ -3091,6 +3157,29 @@ gen_jump_tab(LoaderState* stp, GenOpArg S, GenOpArg Fail, GenOpArg Size, GenOpAr
ASSERT(Size.val >= 2 && Size.val % 2 == 0);
/*
+ * If there is only one choice, don't generate a jump table.
+ */
+ if (Size.val == 2) {
+ GenOp* jump;
+
+ NEW_GENOP(stp, op);
+ op->arity = 3;
+ op->op = genop_is_ne_exact_3;
+ op->a[0] = Rest[1];
+ op->a[1] = S;
+ op->a[2] = Rest[0];
+
+ NEW_GENOP(stp, jump);
+ jump->next = NULL;
+ jump->arity = 1;
+ jump->op = genop_jump_1;
+ jump->a[0] = Fail;
+
+ op->next = jump;
+ return op;
+ }
+
+ /*
* Calculate the minimum and maximum values and size of jump table.
*/
@@ -3162,8 +3251,9 @@ genopargcompare(GenOpArg* a, GenOpArg* b)
}
/*
- * Generate a select_val instruction. We know that a jump table is not suitable,
- * and that all values are of the same type (integer, atoms, floats; never bignums).
+ * Generate a select_val instruction. We know that a jump table
+ * is not suitable, and that all values are of the same type
+ * (integer or atoms).
*/
static GenOp*
@@ -3177,12 +3267,7 @@ gen_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail,
NEW_GENOP(stp, op);
op->next = NULL;
- if (Rest[0].type != TAG_q) {
- op->op = genop_i_select_val_3;
- } else {
- ASSERT(is_float(stp->literals[Rest[0].val].term));
- op->op = genop_i_select_float_3;
- }
+ op->op = genop_i_select_val_3;
GENOP_ARITY(op, arity);
op->a[0] = S;
op->a[1] = Fail;
@@ -3204,19 +3289,19 @@ gen_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail,
}
#endif
- return op;
-}
-
-/*
- * Compare function for qsort().
- */
+ /*
+ * Use a special-cased instruction if there are only two values.
+ */
+ if (size == 2) {
+ op->op = genop_i_select_val2_6;
+ op->arity--;
+ op->a[2] = op->a[3];
+ op->a[3] = op->a[4];
+ op->a[4] = op->a[5];
+ op->a[5] = op->a[6];
+ }
-static int
-genbigcompare(GenOpArg* a, GenOpArg* b)
-{
- int val = (int)(b->bigarity - a->bigarity);
-
- return val != 0 ? val : ((int) (a->val - b->val));
+ return op;
}
/*
@@ -3224,37 +3309,35 @@ genbigcompare(GenOpArg* a, GenOpArg* b)
*/
static GenOp*
-gen_select_big(LoaderState* stp, GenOpArg S, GenOpArg Fail,
+gen_select_literals(LoaderState* stp, GenOpArg S, GenOpArg Fail,
GenOpArg Size, GenOpArg* Rest)
{
GenOp* op;
- int arity = Size.val + 2 + 1;
- int size = Size.val / 2;
+ GenOp* jump;
+ GenOp** prev_next = &op;
+
int i;
- NEW_GENOP(stp, op);
- op->next = NULL;
- op->op = genop_i_select_big_2;
- GENOP_ARITY(op, arity);
- op->a[0] = S;
- op->a[1] = Fail;
for (i = 0; i < Size.val; i += 2) {
+ GenOp* op;
ASSERT(Rest[i].type == TAG_q);
- op->a[i+2] = Rest[i];
- op->a[i+2].bigarity = *big_val(stp->literals[op->a[i+2].val].term);
- op->a[i+3] = Rest[i+1];
- }
- ASSERT(i+2 == arity-1);
- op->a[arity-1].type = TAG_u;
- op->a[arity-1].val = 0;
-
- /*
- * Sort the values in descending arity order.
- */
-
- qsort(op->a+2, size, 2*sizeof(GenOpArg),
- (int (*)(const void *, const void *)) genbigcompare);
+ NEW_GENOP(stp, op);
+ op->op = genop_is_ne_exact_3;
+ op->arity = 3;
+ op->a[0] = Rest[i+1];
+ op->a[1] = S;
+ op->a[2] = Rest[i];
+ *prev_next = op;
+ prev_next = &op->next;
+ }
+
+ NEW_GENOP(stp, jump);
+ jump->next = NULL;
+ jump->op = genop_jump_1;
+ jump->arity = 1;
+ jump->a[0] = Fail;
+ *prev_next = jump;
return op;
}
@@ -3272,7 +3355,6 @@ const_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail,
int i;
ASSERT(Size.type == TAG_u);
- ASSERT(S.type == TAG_q);
NEW_GENOP(stp, op);
op->next = NULL;
@@ -3283,18 +3365,32 @@ const_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail,
* Search for a literal matching the controlling expression.
*/
- if (S.type == TAG_q) {
- Eterm expr = stp->literals[S.val].term;
- for (i = 0; i < Size.val; i += 2) {
- if (Rest[i].type == TAG_q) {
- Eterm term = stp->literals[Rest[i].val].term;
- if (eq(term, expr)) {
- ASSERT(Rest[i+1].type == TAG_f);
- op->a[0] = Rest[i+1];
- return op;
+ switch (S.type) {
+ case TAG_q:
+ {
+ Eterm expr = stp->literals[S.val].term;
+ for (i = 0; i < Size.val; i += 2) {
+ if (Rest[i].type == TAG_q) {
+ Eterm term = stp->literals[Rest[i].val].term;
+ if (eq(term, expr)) {
+ ASSERT(Rest[i+1].type == TAG_f);
+ op->a[0] = Rest[i+1];
+ return op;
+ }
}
}
}
+ break;
+ case TAG_i:
+ case TAG_a:
+ for (i = 0; i < Size.val; i += 2) {
+ if (Rest[i].val == S.val && Rest[i].type == S.type) {
+ ASSERT(Rest[i+1].type == TAG_f);
+ op->a[0] = Rest[i+1];
+ return op;
+ }
+ }
+ break;
}
/*
@@ -3477,6 +3573,56 @@ gen_guard_bif3(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif,
return op;
}
+static GenOp*
+tuple_append_put5(LoaderState* stp, GenOpArg Arity, GenOpArg Dst,
+ GenOpArg* Puts, GenOpArg S1, GenOpArg S2, GenOpArg S3,
+ GenOpArg S4, GenOpArg S5)
+{
+ GenOp* op;
+ int arity = Arity.val; /* Arity of tuple, not the instruction */
+ int i;
+
+ NEW_GENOP(stp, op);
+ op->next = NULL;
+ GENOP_ARITY(op, arity+2+5);
+ op->op = genop_i_put_tuple_2;
+ op->a[0] = Dst;
+ op->a[1].type = TAG_u;
+ op->a[1].val = arity + 5;
+ for (i = 0; i < arity; i++) {
+ op->a[i+2] = Puts[i];
+ }
+ op->a[arity+2] = S1;
+ op->a[arity+3] = S2;
+ op->a[arity+4] = S3;
+ op->a[arity+5] = S4;
+ op->a[arity+6] = S5;
+ return op;
+}
+
+static GenOp*
+tuple_append_put(LoaderState* stp, GenOpArg Arity, GenOpArg Dst,
+ GenOpArg* Puts, GenOpArg S)
+{
+ GenOp* op;
+ int arity = Arity.val; /* Arity of tuple, not the instruction */
+ int i;
+
+ NEW_GENOP(stp, op);
+ op->next = NULL;
+ GENOP_ARITY(op, arity+2+1);
+ op->op = genop_i_put_tuple_2;
+ op->a[0] = Dst;
+ op->a[1].type = TAG_u;
+ op->a[1].val = arity + 1;
+ for (i = 0; i < arity; i++) {
+ op->a[i+2] = Puts[i];
+ }
+ op->a[arity+2] = S;
+ return op;
+}
+
+
/*
* Freeze the code in memory, move the string table into place,
@@ -3624,25 +3770,32 @@ freeze_code(LoaderState* stp)
CHKBLK(ERTS_ALC_T_CODE,code);
if (compile_size) {
byte* compile_info = str_table + strtab_size + attr_size;
- CHKBLK(ERTS_ALC_T_CODE,code);
+ CHKBLK(ERTS_ALC_T_CODE,code);
sys_memcpy(compile_info, stp->chunks[COMPILE_CHUNK].start,
stp->chunks[COMPILE_CHUNK].size);
- CHKBLK(ERTS_ALC_T_CODE,code);
+
+ CHKBLK(ERTS_ALC_T_CODE,code);
code[MI_COMPILE_PTR] = (BeamInstr) compile_info;
- CHKBLK(ERTS_ALC_T_CODE,code);
+ CHKBLK(ERTS_ALC_T_CODE,code);
code[MI_COMPILE_SIZE] = (BeamInstr) stp->chunks[COMPILE_CHUNK].size;
- CHKBLK(ERTS_ALC_T_CODE,code);
+ CHKBLK(ERTS_ALC_T_CODE,code);
decoded_size = erts_decode_ext_size(compile_info, compile_size, 0);
- CHKBLK(ERTS_ALC_T_CODE,code);
+ CHKBLK(ERTS_ALC_T_CODE,code);
if (decoded_size < 0) {
LoadError0(stp, "bad external term representation of compilation information");
}
- CHKBLK(ERTS_ALC_T_CODE,code);
+ CHKBLK(ERTS_ALC_T_CODE,code);
code[MI_COMPILE_SIZE_ON_HEAP] = decoded_size;
}
CHKBLK(ERTS_ALC_T_CODE,code);
/*
+ * Make sure that we have not overflowed the allocated code space.
+ */
+ ASSERT(str_table + strtab_size + attr_size + compile_size ==
+ ((byte *) code) + size);
+
+ /*
* Go through all i_new_bs_put_strings instructions, restore the pointer to
* the instruction and convert string offsets to pointers (to the
* FIRST character).
@@ -3876,11 +4029,23 @@ transform_engine(LoaderState* st)
if (i == 0)
goto restart;
break;
+#if defined(TOP_is_eq)
case TOP_is_eq:
ASSERT(ap < instr->arity);
if (*pc++ != instr->a[ap].val)
goto restart;
break;
+#endif
+ case TOP_is_type_eq:
+ mask = *pc++;
+
+ ASSERT(ap < instr->arity);
+ ASSERT(instr->a[ap].type < BEAM_NUM_TAGS);
+ if (((1 << instr->a[ap].type) & mask) == 0)
+ goto restart;
+ if (*pc++ != instr->a[ap].val)
+ goto restart;
+ break;
case TOP_is_same_var:
ASSERT(ap < instr->arity);
i = *pc++;
@@ -4001,14 +4166,17 @@ transform_engine(LoaderState* st)
case TOP_rest_args:
{
int n = *pc++;
+ int formal_arity = gen_opc[instr->op].arity;
+ int num_vars = n + (instr->arity - formal_arity);
+ int j = formal_arity;
+
var = erts_alloc(ERTS_ALC_T_LOADER_TMP,
- instr->arity * sizeof(GenOpArg));
+ num_vars * sizeof(GenOpArg));
for (i = 0; i < n; i++) {
var[i] = def_vars[i];
}
- while (i < instr->arity) {
- var[i] = instr->a[i];
- i++;
+ while (i < num_vars) {
+ var[i++] = instr->a[j++];
}
}
break;
@@ -5315,6 +5483,9 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
if (state.lambdas != state.def_lambdas) {
erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.lambdas);
}
+ erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.labels);
+ erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.atom);
+ erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.export);
if (bin != NULL) {
driver_free_binary(bin);
}
@@ -5326,9 +5497,18 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
if (code != NULL) {
erts_free(ERTS_ALC_T_CODE, code);
}
+ if (state.labels != NULL) {
+ erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.labels);
+ }
if (state.lambdas != state.def_lambdas) {
erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.lambdas);
}
+ if (state.atom != NULL) {
+ erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.atom);
+ }
+ if (state.export != NULL) {
+ erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.export);
+ }
if (bin != NULL) {
driver_free_binary(bin);
}
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index d4a43f6e5f..bb237e378a 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -1351,9 +1351,10 @@ BIF_RETTYPE exit_2(BIF_ALIST_2)
#ifdef ERTS_SMP
if (rp == BIF_P)
rp_locks &= ~ERTS_PROC_LOCK_MAIN;
- else
+ if (rp_locks)
+ erts_smp_proc_unlock(rp, rp_locks);
+ if (rp != BIF_P)
erts_smp_proc_dec_refc(rp);
- erts_smp_proc_unlock(rp, rp_locks);
#endif
/*
* We may have exited ourselves and may have to take action.
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 60b4b1946b..d9dd80fa8b 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -660,6 +660,7 @@ bif erts_debug:display/1
bif 'erl.system.debug':display/1 ebif_erts_debug_display_1
bif erts_debug:dist_ext_to_term/2
bif 'erl.system.debug':dist_ext_to_term/2 ebif_erts_debug_dist_ext_to_term_2
+bif erts_debug:instructions/0
#
# Monitor testing bif's...
diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c
index ff15d834ab..f47f5a9c0c 100644
--- a/erts/emulator/beam/big.c
+++ b/erts/emulator/beam/big.c
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 1996-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -1558,7 +1558,7 @@ Eterm erts_sint64_to_big(Sint64 x, Eterm **hpp)
** Convert a bignum to a double float
*/
int
-big_to_double(Eterm x, double* resp)
+big_to_double(Wterm x, double* resp)
{
double d = 0.0;
Eterm* xp = big_val(x);
@@ -1725,7 +1725,7 @@ static Eterm big_norm(Eterm *x, dsize_t xl, short sign)
/*
** Compare bignums
*/
-int big_comp(Eterm x, Eterm y)
+int big_comp(Wterm x, Wterm y)
{
Eterm* xp = big_val(x);
Eterm* yp = big_val(y);
@@ -2060,7 +2060,7 @@ static Eterm B_plus_minus(ErtsDigit *x, dsize_t xl, short xsgn,
/*
** Add bignums
*/
-Eterm big_plus(Eterm x, Eterm y, Eterm *r)
+Eterm big_plus(Wterm x, Wterm y, Eterm *r)
{
Eterm* xp = big_val(x);
Eterm* yp = big_val(y);
diff --git a/erts/emulator/beam/big.h b/erts/emulator/beam/big.h
index 25466cd3c2..f28a390aea 100644
--- a/erts/emulator/beam/big.h
+++ b/erts/emulator/beam/big.h
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 1996-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -120,7 +120,7 @@ char *erts_big_to_string(Eterm x, char *buf, Uint buf_sz);
Eterm small_times(Sint, Sint, Eterm*);
-Eterm big_plus(Eterm, Eterm, Eterm*);
+Eterm big_plus(Wterm, Wterm, Eterm*);
Eterm big_minus(Eterm, Eterm, Eterm*);
Eterm big_times(Eterm, Eterm, Eterm*);
Eterm big_div(Eterm, Eterm, Eterm*);
@@ -137,9 +137,9 @@ Eterm big_bxor(Eterm, Eterm, Eterm*);
Eterm big_bnot(Eterm, Eterm*);
Eterm big_lshift(Eterm, Sint, Eterm*);
-int big_comp (Eterm, Eterm);
+int big_comp (Wterm, Wterm);
int big_ucomp (Eterm, Eterm);
-int big_to_double(Eterm x, double* resp);
+int big_to_double(Wterm x, double* resp);
Eterm small_to_big(Sint, Eterm*);
Eterm uint_to_big(Uint, Eterm*);
Eterm uword_to_big(UWord, Eterm*);
diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index fd2ec91e65..d255cf3558 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -558,7 +558,7 @@ do_break(void)
#endif
#ifdef DEBUG
case 't':
- p_slpq();
+ erts_p_slpq();
return;
case 'b':
bin_check();
diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index 8bee47232e..243e8973cf 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 1996-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -72,8 +72,11 @@ copy_object(Eterm obj, Process* to)
* Return the "flat" size of the object.
*/
-Uint
-size_object(Eterm obj)
+#if HALFWORD_HEAP
+Uint size_object_rel(Eterm obj, Eterm* base)
+#else
+Uint size_object(Eterm obj)
+#endif
{
Uint sum = 0;
Eterm* ptr;
@@ -84,7 +87,7 @@ size_object(Eterm obj)
switch (primary_tag(obj)) {
case TAG_PRIMARY_LIST:
sum += 2;
- ptr = list_val(obj);
+ ptr = list_val_rel(obj,base);
obj = *ptr++;
if (!IS_CONST(obj)) {
ESTACK_PUSH(s, obj);
@@ -93,11 +96,11 @@ size_object(Eterm obj)
break;
case TAG_PRIMARY_BOXED:
{
- Eterm hdr = *boxed_val(obj);
+ Eterm hdr = *boxed_val_rel(obj,base);
ASSERT(is_header(hdr));
switch (hdr & _TAG_HEADER_MASK) {
case ARITYVAL_SUBTAG:
- ptr = tuple_val(obj);
+ ptr = tuple_val_rel(obj,base);
arity = header_arity(hdr);
sum += arity + 1;
if (arity == 0) { /* Empty tuple -- unusual. */
@@ -113,7 +116,7 @@ size_object(Eterm obj)
break;
case FUN_SUBTAG:
{
- Eterm* bptr = fun_val(obj);
+ Eterm* bptr = fun_val_rel(obj,base);
ErlFunThing* funp = (ErlFunThing *) bptr;
unsigned eterms = 1 /* creator */ + funp->num_free;
unsigned sz = thing_arityval(hdr);
@@ -136,7 +139,7 @@ size_object(Eterm obj)
Uint bitoffs;
Uint extra_bytes;
Eterm hdr;
- ERTS_GET_REAL_BIN(obj, real_bin, offset, bitoffs, bitsize);
+ ERTS_GET_REAL_BIN_REL(obj, real_bin, offset, bitoffs, bitsize, base);
if ((bitsize + bitoffs) > 8) {
sum += ERL_SUB_BIN_SIZE;
extra_bytes = 2;
@@ -146,11 +149,11 @@ size_object(Eterm obj)
} else {
extra_bytes = 0;
}
- hdr = *binary_val(real_bin);
+ hdr = *binary_val_rel(real_bin,base);
if (thing_subtag(hdr) == REFC_BINARY_SUBTAG) {
sum += PROC_BIN_SIZE;
} else {
- sum += heap_bin_size(binary_size(obj)+extra_bytes);
+ sum += heap_bin_size(binary_size_rel(obj,base)+extra_bytes);
}
goto pop_next;
}
@@ -181,8 +184,12 @@ size_object(Eterm obj)
/*
* Copy a structure to a heap.
*/
-Eterm
-copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
+#if HALFWORD_HEAP
+Eterm copy_struct_rel(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap,
+ Eterm* src_base, Eterm* dst_base)
+#else
+Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
+#endif
{
char* hstart;
Uint hsize;
@@ -214,7 +221,10 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
/* Copy the object onto the heap */
switch (primary_tag(obj)) {
- case TAG_PRIMARY_LIST: argp = &res; goto L_copy_list;
+ case TAG_PRIMARY_LIST:
+ argp = &res;
+ objp = list_val_rel(obj,src_base);
+ goto L_copy_list;
case TAG_PRIMARY_BOXED: argp = &res; goto L_copy_boxed;
default:
erl_exit(ERTS_ABORT_EXIT,
@@ -231,32 +241,46 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
hp++;
break;
case TAG_PRIMARY_LIST:
- objp = list_val(obj);
+ objp = list_val_rel(obj,src_base);
+ #if !HALFWORD_HEAP || defined(DEBUG)
if (in_area(objp,hstart,hsize)) {
+ ASSERT(!HALFWORD_HEAP);
hp++;
break;
}
+ #endif
argp = hp++;
/* Fall through */
L_copy_list:
tailp = argp;
- while (is_list(obj)) {
- objp = list_val(obj);
+ for (;;) {
tp = tailp;
- elem = *objp;
+ elem = CAR(objp);
if (IS_CONST(elem)) {
- *(hbot-2) = elem;
- tailp = hbot-1;
hbot -= 2;
+ CAR(hbot) = elem;
+ tailp = &CDR(hbot);
}
else {
- *htop = elem;
- tailp = htop+1;
+ CAR(htop) = elem;
+ #if HALFWORD_HEAP
+ CDR(htop) = CDR(objp);
+ *tailp = make_list_rel(htop,dst_base);
+ htop += 2;
+ goto L_copy;
+ #else
+ tailp = &CDR(htop);
htop += 2;
+ #endif
+ }
+ ASSERT(!HALFWORD_HEAP || tp < hp || tp >= hbot);
+ *tp = make_list_rel(tailp - 1, dst_base);
+ obj = CDR(objp);
+ if (!is_list(obj)) {
+ break;
}
- *tp = make_list(tailp - 1);
- obj = *(objp+1);
+ objp = list_val_rel(obj,src_base);
}
switch (primary_tag(obj)) {
case TAG_PRIMARY_IMMED1: *tailp = obj; goto L_copy;
@@ -268,21 +292,24 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
}
case TAG_PRIMARY_BOXED:
- if (in_area(boxed_val(obj),hstart,hsize)) {
+ #if !HALFWORD_HEAP || defined(DEBUG)
+ if (in_area(boxed_val_rel(obj,src_base),hstart,hsize)) {
+ ASSERT(!HALFWORD_HEAP);
hp++;
break;
}
+ #endif
argp = hp++;
L_copy_boxed:
- objp = boxed_val(obj);
+ objp = boxed_val_rel(obj, src_base);
hdr = *objp;
switch (hdr & _TAG_HEADER_MASK) {
case ARITYVAL_SUBTAG:
{
int const_flag = 1; /* assume constant tuple */
i = arityval(hdr);
- *argp = make_tuple(htop);
+ *argp = make_tuple_rel(htop, dst_base);
tp = htop; /* tp is pointer to new arity value */
*htop++ = *objp++; /* copy arity value */
while (i--) {
@@ -311,7 +338,7 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
while (i--) {
*tp++ = *objp++;
}
- *argp = make_binary(hbot);
+ *argp = make_binary_rel(hbot, dst_base);
pb = (ProcBin*) hbot;
erts_refc_inc(&pb->val->refc, 2);
pb->next = off_heap->first;
@@ -338,7 +365,7 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
extra_bytes = 0;
}
real_size = size+extra_bytes;
- objp = binary_val(real_bin);
+ objp = binary_val_rel(real_bin,src_base);
if (thing_subtag(*objp) == HEAP_BINARY_SUBTAG) {
ErlHeapBin* from = (ErlHeapBin *) objp;
ErlHeapBin* to;
@@ -368,7 +395,7 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
off_heap->first = (struct erl_off_heap_header*) to;
OH_OVERHEAD(off_heap, to->size / sizeof(Eterm));
}
- *argp = make_binary(hbot);
+ *argp = make_binary_rel(hbot, dst_base);
if (extra_bytes != 0) {
ErlSubBin* res;
hbot -= ERL_SUB_BIN_SIZE;
@@ -380,7 +407,7 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
res->offs = 0;
res->is_writable = 0;
res->orig = *argp;
- *argp = make_binary(hbot);
+ *argp = make_binary_rel(hbot, dst_base);
}
break;
}
@@ -400,7 +427,7 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
off_heap->first = (struct erl_off_heap_header*) funp;
erts_refc_inc(&funp->fe->refc, 2);
#endif
- *argp = make_fun(tp);
+ *argp = make_fun_rel(tp, dst_base);
}
break;
case EXTERNAL_PID_SUBTAG:
@@ -420,7 +447,7 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
off_heap->first = (struct erl_off_heap_header*)etp;
erts_refc_inc(&etp->node->refc, 2);
- *argp = make_external(tp);
+ *argp = make_external_rel(tp, dst_base);
}
break;
case BIN_MATCHSTATE_SUBTAG:
@@ -430,7 +457,7 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
i = thing_arityval(hdr)+1;
hbot -= i;
tp = hbot;
- *argp = make_boxed(hbot);
+ *argp = make_boxed_rel(hbot, dst_base);
while (i--) {
*tp++ = *objp++;
}
@@ -885,12 +912,21 @@ Eterm copy_struct_lazy(Process *from, Eterm orig, Uint offs)
*
* NOTE: Assumes that term is a tuple (ptr is an untagged tuple ptr).
*/
-Eterm
-copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
+#if HALFWORD_HEAP
+Eterm copy_shallow_rel(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap,
+ Eterm* src_base)
+#else
+Eterm copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
+#endif
{
Eterm* tp = ptr;
Eterm* hp = *hpp;
- Sint offs = hp - tp;
+ const Eterm res = make_tuple(hp);
+#if HALFWORD_HEAP
+ const Sint offs = COMPRESS_POINTER(hp - (tp - src_base));
+#else
+ const Sint offs = (hp - tp) * sizeof(Eterm);
+#endif
while (sz--) {
Eterm val = *tp++;
@@ -901,7 +937,7 @@ copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
break;
case TAG_PRIMARY_LIST:
case TAG_PRIMARY_BOXED:
- *hp++ = offset_ptr(val, offs);
+ *hp++ = byte_offset_ptr(val, offs);
break;
case TAG_PRIMARY_HEADER:
*hp++ = val;
@@ -958,7 +994,8 @@ copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
}
}
*hpp = hp;
- return make_tuple(ptr + offs);
+
+ return res;
}
/* Move all terms in heap fragments into heap. The terms must be guaranteed to
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 694460d702..02910fad90 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -917,6 +917,7 @@ int erts_net_message(Port *prt,
Eterm token_size;
ErtsMonitor *mon;
ErtsLink *lnk;
+ Uint tuple_arity;
int res;
#ifdef ERTS_DIST_MSG_DBG
int orig_len = len;
@@ -1003,29 +1004,23 @@ int erts_net_message(Port *prt,
#endif
if (is_not_tuple(arg) ||
- (tuple = tuple_val(arg), arityval(*tuple) < 1) ||
+ (tuple = tuple_val(arg), (tuple_arity = arityval(*tuple)) < 1) ||
is_not_small(tuple[1])) {
- erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
- erts_dsprintf(dsbufp, "Invalid distribution message: %.200T", arg);
- erts_send_error_to_logger_nogl(dsbufp);
- goto data_error;
+ goto invalid_message;
}
token_size = 0;
switch (type = unsigned_val(tuple[1])) {
case DOP_LINK:
+ if (tuple_arity != 3) {
+ goto invalid_message;
+ }
from = tuple[2];
to = tuple[3]; /* local proc to link to */
if (is_not_pid(from) || is_not_pid(to)) {
- erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
- PURIFY_MSG("data error");
- erts_dsprintf(dsbufp,
- "Invalid DOP_LINK distribution message: %.200T",
- arg);
- erts_send_error_to_logger_nogl(dsbufp);
- goto data_error;
+ goto invalid_message;
}
rp = erts_pid2proc_opt(NULL, 0,
@@ -1064,8 +1059,14 @@ int erts_net_message(Port *prt,
case DOP_UNLINK: {
ErtsDistLinkData dld;
+ if (tuple_arity != 3) {
+ goto invalid_message;
+ }
from = tuple[2];
to = tuple[3];
+ if (is_not_pid(from) || is_not_pid(to)) {
+ goto invalid_message;
+ }
rp = erts_pid2proc_opt(NULL, 0,
to, ERTS_PROC_LOCK_LINK,
@@ -1092,11 +1093,19 @@ int erts_net_message(Port *prt,
/* A remote process wants to monitor us, we get:
{DOP_MONITOR_P, Remote pid, local pid or name, ref} */
Eterm name;
+
+ if (tuple_arity != 4) {
+ goto invalid_message;
+ }
watcher = tuple[2];
watched = tuple[3]; /* local proc to monitor */
ref = tuple[4];
+ if (is_not_ref(ref)) {
+ goto invalid_message;
+ }
+
if (is_atom(watched)) {
name = watched;
rp = erts_whereis_process(NULL, 0,
@@ -1138,10 +1147,17 @@ int erts_net_message(Port *prt,
We get {DOP_DEMONITOR_P, Remote pid, Local pid or name, ref},
We need only the ref of course */
+ if (tuple_arity != 4) {
+ goto invalid_message;
+ }
/* watcher = tuple[2]; */
/* watched = tuple[3]; May be an atom in case of monitor name */
ref = tuple[4];
+ if(is_not_ref(ref)) {
+ goto invalid_message;
+ }
+
erts_smp_de_links_lock(dep);
mon = erts_remove_monitor(&(dep->monitors),ref);
erts_smp_de_links_unlock(dep);
@@ -1166,10 +1182,11 @@ int erts_net_message(Port *prt,
erts_destroy_monitor(mon);
break;
- case DOP_NODE_LINK: /* XXX never sent ?? */
- break;
-
case DOP_REG_SEND_TT:
+ if (tuple_arity != 5) {
+ goto invalid_message;
+ }
+
token_size = size_object(tuple[5]);
/* Fall through ... */
case DOP_REG_SEND:
@@ -1180,12 +1197,19 @@ int erts_net_message(Port *prt,
* There is intentionally no testing of the cookie (it is always '')
* from R9B and onwards.
*/
+ if (type != DOP_REG_SEND_TT && tuple_arity != 4) {
+ goto invalid_message;
+ }
+
#ifdef ERTS_DIST_MSG_DBG
dist_msg_dbg(&ede, "MSG", buf, orig_len);
#endif
from = tuple[2];
to = tuple[4];
+ if (is_not_pid(from) || is_not_atom(to)){
+ goto invalid_message;
+ }
rp = erts_whereis_process(NULL, 0, to, 0, ERTS_P2P_FLG_SMP_INC_REFC);
if (rp) {
Uint xsize = (type == DOP_REG_SEND
@@ -1217,6 +1241,10 @@ int erts_net_message(Port *prt,
break;
case DOP_SEND_TT:
+ if (tuple_arity != 4) {
+ goto invalid_message;
+ }
+
token_size = size_object(tuple[4]);
/* Fall through ... */
case DOP_SEND:
@@ -1227,8 +1255,13 @@ int erts_net_message(Port *prt,
#ifdef ERTS_DIST_MSG_DBG
dist_msg_dbg(&ede, "MSG", buf, orig_len);
#endif
-
+ if (type != DOP_SEND_TT && tuple_arity != 3) {
+ goto invalid_message;
+ }
to = tuple[3];
+ if (is_not_pid(to)) {
+ goto invalid_message;
+ }
rp = erts_pid2proc_opt(NULL, 0, to, 0, ERTS_P2P_FLG_SMP_INC_REFC);
if (rp) {
Uint xsize = type == DOP_SEND ? 0 : ERTS_HEAP_FRAG_SIZE(token_size);
@@ -1266,11 +1299,19 @@ int erts_net_message(Port *prt,
Eterm sysname;
ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_MSG_SEND|ERTS_PROC_LOCK_LINK;
+ if (tuple_arity != 5) {
+ goto invalid_message;
+ }
+
/* watched = tuple[2]; */ /* remote proc which died */
/* watcher = tuple[3]; */
ref = tuple[4];
reason = tuple[5];
+ if(is_not_ref(ref)) {
+ goto invalid_message;
+ }
+
erts_smp_de_links_lock(dep);
sysname = dep->sysname;
mon = erts_remove_monitor(&(dep->monitors), ref);
@@ -1317,24 +1358,25 @@ int erts_net_message(Port *prt,
ErtsProcLocks rp_locks = ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCKS_XSIG_SEND;
/* 'from', which 'to' is linked to, died */
if (type == DOP_EXIT) {
- from = tuple[2];
- to = tuple[3];
- reason = tuple[4];
- token = NIL;
+ if (tuple_arity != 4) {
+ goto invalid_message;
+ }
+
+ from = tuple[2];
+ to = tuple[3];
+ reason = tuple[4];
+ token = NIL;
} else {
- from = tuple[2];
- to = tuple[3];
- token = tuple[4];
- reason = tuple[5];
+ if (tuple_arity != 5) {
+ goto invalid_message;
+ }
+ from = tuple[2];
+ to = tuple[3];
+ token = tuple[4];
+ reason = tuple[5];
}
- if (is_not_internal_pid(to)) {
- erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
- PURIFY_MSG("data error");
- erts_dsprintf(dsbufp,
- "Invalid DOP_EXIT distribution message: %.200T",
- arg);
- erts_send_error_to_logger_nogl(dsbufp);
- goto data_error;
+ if (is_not_pid(from) || is_not_internal_pid(to)) {
+ goto invalid_message;
}
rp = erts_pid2proc(NULL, 0, to, rp_locks);
@@ -1381,15 +1423,24 @@ int erts_net_message(Port *prt,
ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND;
/* 'from' is send an exit signal to 'to' */
if (type == DOP_EXIT2) {
- from = tuple[2];
- to = tuple[3];
- reason = tuple[4];
- token = NIL;
+ if (tuple_arity != 4) {
+ goto invalid_message;
+ }
+ from = tuple[2];
+ to = tuple[3];
+ reason = tuple[4];
+ token = NIL;
} else {
- from = tuple[2];
- to = tuple[3];
- token = tuple[4];
- reason = tuple[5];
+ if (tuple_arity != 5) {
+ goto invalid_message;
+ }
+ from = tuple[2];
+ to = tuple[3];
+ token = tuple[4];
+ reason = tuple[5];
+ }
+ if (is_not_pid(from) || is_not_internal_pid(to)) {
+ goto invalid_message;
}
rp = erts_pid2proc_opt(NULL, 0, to, rp_locks,
ERTS_P2P_FLG_SMP_INC_REFC);
@@ -1408,10 +1459,14 @@ int erts_net_message(Port *prt,
break;
}
case DOP_GROUP_LEADER:
+ if (tuple_arity != 3) {
+ goto invalid_message;
+ }
from = tuple[2]; /* Group leader */
to = tuple[3]; /* new member */
- if (is_not_pid(from))
- break;
+ if (is_not_pid(from) || is_not_pid(to)) {
+ goto invalid_message;
+ }
rp = erts_pid2proc(NULL, 0, to, ERTS_PROC_LOCK_MAIN);
if (!rp)
@@ -1420,16 +1475,8 @@ int erts_net_message(Port *prt,
erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MAIN);
break;
- default: {
- erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
- erts_dsprintf(dsbufp,
- "Illegal value in distribution dispatch switch: "
- "%.200T",
- arg);
- erts_send_error_to_logger_nogl(dsbufp);
- PURIFY_MSG("data error");
- goto data_error;
- }
+ default:
+ goto invalid_message;
}
erts_cleanup_offheap(&off_heap);
@@ -1441,8 +1488,14 @@ int erts_net_message(Port *prt,
UnUseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE);
ERTS_SMP_CHK_NO_PROC_LOCKS;
return 0;
-
+ invalid_message:
+ {
+ erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
+ erts_dsprintf(dsbufp, "Invalid distribution message: %.200T", arg);
+ erts_send_error_to_logger_nogl(dsbufp);
+ }
data_error:
+ PURIFY_MSG("data error");
erts_cleanup_offheap(&off_heap);
#ifndef HYBRID /* FIND ME! */
if (ctl != ctl_default) {
diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h
index 9ccc3e5ba9..695a4fc3fe 100644
--- a/erts/emulator/beam/dist.h
+++ b/erts/emulator/beam/dist.h
@@ -52,7 +52,7 @@
#define DOP_SEND 2
#define DOP_EXIT 3
#define DOP_UNLINK 4
-#define DOP_NODE_LINK 5
+/* Ancient DOP_NODE_LINK (5) was here, can be reused */
#define DOP_REG_SEND 6
#define DOP_GROUP_LEADER 7
#define DOP_EXIT2 8
@@ -69,7 +69,6 @@
/* distribution trap functions */
extern Export* dsend2_trap;
extern Export* dsend3_trap;
-/*extern Export* dsend_nosuspend_trap;*/
extern Export* dlink_trap;
extern Export* dunlink_trap;
extern Export* dmonitor_node_trap;
diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 7793f60f4f..775f4435a9 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2002-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2002-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -1411,6 +1411,33 @@ void erts_alloc_reg_scheduler_id(Uint id)
erts_tsd_set(thr_ix_key, (void *)(long) ix);
}
+static void
+no_verify(Allctr_t *allctr)
+{
+
+}
+
+erts_alloc_verify_func_t
+erts_alloc_get_verify_unused_temp_alloc(Allctr_t **allctr)
+{
+ if (erts_allctrs_info[ERTS_ALC_A_TEMPORARY].alloc_util
+ && erts_allctrs_info[ERTS_ALC_A_TEMPORARY].thr_spec) {
+ ErtsAllocatorThrSpec_t *tspec;
+ tspec = &erts_allctr_thr_spec[ERTS_ALC_A_TEMPORARY];
+ if (!tspec->all_thr_safe) {
+ int ix = erts_alc_get_thr_ix();
+
+ if (ix < tspec->size) {
+ *allctr = tspec->allctr[ix];
+ return erts_alcu_verify_unused;
+ }
+ }
+ }
+
+ *allctr = NULL;
+ return no_verify;
+}
+
__decl_noreturn void
erts_alc_fatal_error(int error, int func, ErtsAlcType_t n, ...)
{
@@ -1568,7 +1595,6 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
Eterm atoms[sizeof(size)/sizeof(Uint)];
Uint *uintps[sizeof(size)/sizeof(Uint)];
Eterm euints[sizeof(size)/sizeof(Uint)];
- int need_atom;
int want_tot_or_sys;
int length;
Eterm res = THE_NON_VALUE;
@@ -1756,7 +1782,6 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
/* Calculate values needed... */
want_tot_or_sys = want.total || want.system;
- need_atom = ERTS_MEM_NEED_ALL_ALCU || want.atom;
if (ERTS_MEM_NEED_ALL_ALCU) {
size.total = 0;
diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h
index 3e96c76dbf..17ff9c3b03 100644
--- a/erts/emulator/beam/erl_alloc.h
+++ b/erts/emulator/beam/erl_alloc.h
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2002-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2002-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -236,6 +236,11 @@ void *erts_realloc_fnf(ErtsAlcType_t type, void *ptr, Uint size)
#endif /* #if ERTS_ALC_DO_INLINE || defined(ERTS_ALC_INTERNAL__) */
+typedef void (*erts_alloc_verify_func_t)(Allctr_t *);
+
+erts_alloc_verify_func_t
+erts_alloc_get_verify_unused_temp_alloc(Allctr_t **allctr);
+
#ifndef ERTS_CACHE_LINE_SIZE
/* Assume a cache line size of 64 bytes */
# define ERTS_CACHE_LINE_SIZE ((UWord) 64)
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index 408ffd12f7..ca71798917 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -1,7 +1,7 @@
#
# %CopyrightBegin%
#
-# Copyright Ericsson AB 2003-2010. All Rights Reserved.
+# Copyright Ericsson AB 2003-2011. All Rights Reserved.
#
# The contents of this file are subject to the Erlang Public License,
# Version 1.1, (the "License"); you may not use this file except in
@@ -193,6 +193,7 @@ type DB_FIXATION SHORT_LIVED ETS db_fixation
type DB_FIX_DEL SHORT_LIVED ETS fixed_del
type DB_TABLES LONG_LIVED ETS db_tabs
type DB_NTAB_ENT STANDARD ETS db_named_table_entry
+type DB_HEIR_DATA STANDARD ETS db_heir_data
type DB_TMP TEMPORARY ETS db_tmp
type DB_MC_STK TEMPORARY ETS db_mc_stack
type DB_MS_PSDO_PROC LONG_LIVED ETS db_match_pseudo_proc
@@ -263,6 +264,8 @@ type XPORTS_LIST SHORT_LIVED SYSTEM extra_port_list
type PROC_LCK_WTR LONG_LIVED SYSTEM proc_lock_waiter
type PROC_LCK_QS LONG_LIVED SYSTEM proc_lock_queues
type RUNQ_BLNS LONG_LIVED SYSTEM run_queue_balancing
+type MISC_AUX_WORK_Q LONG_LIVED SYSTEM misc_aux_work_q
+type MISC_AUX_WORK SHORT_LIVED SYSTEM misc_aux_work
+endif
#
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index 8b184899c9..1394b7e829 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2002-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2002-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -3368,6 +3368,38 @@ erts_alcu_test(unsigned long op, unsigned long a1, unsigned long a2)
* Debug functions *
\* */
+void
+erts_alcu_verify_unused(Allctr_t *allctr)
+{
+ UWord no;
+
+ no = allctr->sbcs.curr_mseg.no;
+ no += allctr->sbcs.curr_sys_alloc.no;
+ no += allctr->mbcs.blocks.curr.no;
+
+ if (no) {
+ UWord sz = allctr->sbcs.blocks.curr.size;
+ sz += allctr->mbcs.blocks.curr.size;
+ erl_exit(ERTS_ABORT_EXIT,
+ "%salloc() used when expected to be unused!\n"
+ "Total amount of blocks allocated: %bpu\n"
+ "Total amount of bytes allocated: %bpu\n",
+ allctr->name_prefix, no, sz);
+ }
+}
+
+void
+erts_alcu_verify_unused_ts(Allctr_t *allctr)
+{
+#ifdef USE_THREADS
+ erts_mtx_lock(&allctr->mutex);
+#endif
+ erts_alcu_verify_unused(allctr);
+#ifdef USE_THREADS
+ erts_mtx_unlock(&allctr->mutex);
+#endif
+}
+
#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
static void
diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h
index f2b951bca6..d296081714 100644
--- a/erts/emulator/beam/erl_alloc_util.h
+++ b/erts/emulator/beam/erl_alloc_util.h
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2002-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2002-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -333,6 +333,9 @@ struct Allctr_t_ {
int erts_alcu_start(Allctr_t *, AllctrInit_t *);
void erts_alcu_stop(Allctr_t *);
+void erts_alcu_verify_unused(Allctr_t *);
+void erts_alcu_verify_unused_ts(Allctr_t *allctr);
+
unsigned long erts_alcu_test(unsigned long, unsigned long, unsigned long);
diff --git a/erts/emulator/beam/erl_bif_lists.c b/erts/emulator/beam/erl_bif_lists.c
index ce13469801..47c48e74d6 100644
--- a/erts/emulator/beam/erl_bif_lists.c
+++ b/erts/emulator/beam/erl_bif_lists.c
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 1999-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -378,7 +378,7 @@ keyfind(int Bif, Process* p, Eterm Key, Eterm Pos, Eterm List)
Eterm *tuple_ptr = tuple_val(term);
if (pos <= arityval(*tuple_ptr)) {
Eterm element = tuple_ptr[pos];
- if (cmp(Key, element) == 0) {
+ if (CMP(Key, element) == 0) {
return term;
}
}
diff --git a/erts/emulator/beam/erl_bif_timer.c b/erts/emulator/beam/erl_bif_timer.c
index 4ae2f6ebf4..db771bd216 100644
--- a/erts/emulator/beam/erl_bif_timer.c
+++ b/erts/emulator/beam/erl_bif_timer.c
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2005-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -478,7 +478,7 @@ setup_bif_timer(Uint32 xflags,
tab_insert(btm);
ASSERT(btm == tab_find(ref));
btm->tm.active = 0; /* MUST be initalized */
- erl_set_timer(&btm->tm,
+ erts_set_timer(&btm->tm,
(ErlTimeoutProc) bif_timer_timeout,
(ErlCancelProc) bif_timer_cleanup,
(void *) btm,
@@ -550,7 +550,7 @@ BIF_RETTYPE cancel_timer_1(BIF_ALIST_1)
res = am_false;
}
else {
- Uint left = time_left(&btm->tm);
+ Uint left = erts_time_left(&btm->tm);
if (!(btm->flags & BTM_FLG_BYNAME)) {
erts_smp_proc_lock(btm->receiver.proc.ess, ERTS_PROC_LOCK_MSGQ);
unlink_proc(btm);
@@ -558,7 +558,7 @@ BIF_RETTYPE cancel_timer_1(BIF_ALIST_1)
}
tab_remove(btm);
ASSERT(!tab_find(BIF_ARG_1));
- erl_cancel_timer(&btm->tm);
+ erts_cancel_timer(&btm->tm);
erts_smp_btm_rwunlock();
res = erts_make_integer(left, BIF_P);
}
@@ -587,7 +587,7 @@ BIF_RETTYPE read_timer_1(BIF_ALIST_1)
res = am_false;
}
else {
- Uint left = time_left(&btm->tm);
+ Uint left = erts_time_left(&btm->tm);
res = erts_make_integer(left, BIF_P);
}
@@ -613,7 +613,8 @@ erts_print_bif_timer_info(int to, void *to_arg)
: btm->receiver.proc.ess->id);
erts_print(to, to_arg, "=timer:%T\n", receiver);
erts_print(to, to_arg, "Message: %T\n", btm->message);
- erts_print(to, to_arg, "Time left: %d ms\n", time_left(&btm->tm));
+ erts_print(to, to_arg, "Time left: %u ms\n",
+ erts_time_left(&btm->tm));
}
}
@@ -640,7 +641,7 @@ erts_cancel_bif_timers(Process *p, ErtsProcLocks plocks)
tab_remove(btm);
tmp_btm = btm;
btm = btm->receiver.proc.next;
- erl_cancel_timer(&tmp_btm->tm);
+ erts_cancel_timer(&tmp_btm->tm);
}
p->bif_timers = NULL;
diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h
index bdf0fe23fc..506c4813fa 100644
--- a/erts/emulator/beam/erl_binary.h
+++ b/erts/emulator/beam/erl_binary.h
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2000-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2000-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -71,6 +71,7 @@ typedef struct erl_heap_bin {
*/
#define binary_size(Bin) (binary_val(Bin)[1])
+#define binary_size_rel(Bin,BasePtr) (binary_val_rel(Bin,BasePtr)[1])
#define binary_bitsize(Bin) \
((*binary_val(Bin) == HEADER_SUB_BIN) ? \
@@ -93,9 +94,12 @@ typedef struct erl_heap_bin {
* Bitsize: output variable (Uint)
*/
-#define ERTS_GET_BINARY_BYTES(Bin,Bytep,Bitoffs,Bitsize) \
+#define ERTS_GET_BINARY_BYTES(Bin,Bytep,Bitoffs,Bitsize) \
+ ERTS_GET_BINARY_BYTES_REL(Bin,Bytep,Bitoffs,Bitsize,NULL)
+
+#define ERTS_GET_BINARY_BYTES_REL(Bin,Bytep,Bitoffs,Bitsize,BasePtr) \
do { \
- Eterm* _real_bin = binary_val(Bin); \
+ Eterm* _real_bin = binary_val_rel(Bin,BasePtr); \
Uint _offs = 0; \
Bitoffs = Bitsize = 0; \
if (*_real_bin == HEADER_SUB_BIN) { \
@@ -103,7 +107,7 @@ do { \
_offs = _sb->offs; \
Bitoffs = _sb->bitoffs; \
Bitsize = _sb->bitsize; \
- _real_bin = binary_val(_sb->orig); \
+ _real_bin = binary_val_rel(_sb->orig,BasePtr); \
} \
if (*_real_bin == HEADER_PROC_BIN) { \
Bytep = ((ProcBin *) _real_bin)->bytes + _offs; \
@@ -125,9 +129,12 @@ do { \
* BitSize: Extra bit size (Uint)
*/
-#define ERTS_GET_REAL_BIN(Bin, RealBin, ByteOffset, BitOffset, BitSize) \
+#define ERTS_GET_REAL_BIN(Bin, RealBin, ByteOffset, BitOffset, BitSize) \
+ ERTS_GET_REAL_BIN_REL(Bin, RealBin, ByteOffset, BitOffset, BitSize, NULL)
+
+#define ERTS_GET_REAL_BIN_REL(Bin, RealBin, ByteOffset, BitOffset, BitSize, BasePtr) \
do { \
- ErlSubBin* _sb = (ErlSubBin *) binary_val(Bin); \
+ ErlSubBin* _sb = (ErlSubBin *) binary_val_rel(Bin,BasePtr); \
if (_sb->thing_word == HEADER_SUB_BIN) { \
RealBin = _sb->orig; \
ByteOffset = _sb->offs; \
diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c
index 88d2c06246..6f8a7436d5 100644
--- a/erts/emulator/beam/erl_bits.c
+++ b/erts/emulator/beam/erl_bits.c
@@ -555,10 +555,11 @@ fmt_int(byte *buf, Uint sz, Eterm val, Uint size, Uint flags)
{
unsigned long offs;
- ASSERT(size != 0);
offs = BIT_OFFSET(size);
if (is_small(val)) {
Sint v = signed_val(val);
+
+ ASSERT(size != 0); /* Tested by caller */
if (flags & BSF_LITTLE) { /* Little endian */
sz--;
COPY_VAL(buf,1,v,sz);
@@ -578,6 +579,9 @@ fmt_int(byte *buf, Uint sz, Eterm val, Uint size, Uint flags)
ErtsDigit* dp = big_v(val);
int n = MIN(sz,ds);
+ if (size == 0) {
+ return 0;
+ }
if (flags & BSF_LITTLE) {
sz -= n; /* pad with this amount */
if (sign) {
@@ -729,15 +733,13 @@ erts_new_bs_put_integer(ERL_BITS_PROTO_3(Eterm arg, Uint num_bits, unsigned flag
Uint b;
byte *iptr;
- if (num_bits == 0) {
- return 1;
- }
-
bit_offset = BIT_OFFSET(bin_offset);
if (is_small(arg)) {
Uint rbits = 8 - bit_offset;
- if (bit_offset + num_bits <= 8) {
+ if (num_bits == 0) {
+ return 1;
+ } else if (bit_offset + num_bits <= 8) {
/*
* All bits are in the same byte.
*/
diff --git a/erts/emulator/beam/erl_cpu_topology.c b/erts/emulator/beam/erl_cpu_topology.c
index 8a6b4d8d6c..bcf8bcf270 100644
--- a/erts/emulator/beam/erl_cpu_topology.c
+++ b/erts/emulator/beam/erl_cpu_topology.c
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2010. All Rights Reserved.
+ * Copyright Ericsson AB 2010-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -630,13 +630,13 @@ write_schedulers_bind_change(erts_cpu_topology_t *cpudata, int size)
int
erts_init_scheduler_bind_type_string(char *how)
{
- if (erts_bind_to_cpu(cpuinfo, -1) == -ENOTSUP)
+ if (sys_strcmp(how, "u") == 0)
+ cpu_bind_order = ERTS_CPU_BIND_NONE;
+ else if (erts_bind_to_cpu(cpuinfo, -1) == -ENOTSUP)
return ERTS_INIT_SCHED_BIND_TYPE_NOT_SUPPORTED;
-
- if (!system_cpudata && !user_cpudata)
+ else if (!system_cpudata && !user_cpudata)
return ERTS_INIT_SCHED_BIND_TYPE_ERROR_NO_CPU_TOPOLOGY;
-
- if (sys_strcmp(how, "db") == 0)
+ else if (sys_strcmp(how, "db") == 0)
cpu_bind_order = ERTS_CPU_BIND_DEFAULT_BIND;
else if (sys_strcmp(how, "s") == 0)
cpu_bind_order = ERTS_CPU_BIND_SPREAD;
@@ -652,11 +652,8 @@ erts_init_scheduler_bind_type_string(char *how)
cpu_bind_order = ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD;
else if (sys_strcmp(how, "ns") == 0)
cpu_bind_order = ERTS_CPU_BIND_NO_SPREAD;
- else if (sys_strcmp(how, "u") == 0)
- cpu_bind_order = ERTS_CPU_BIND_NONE;
else
return ERTS_INIT_SCHED_BIND_TYPE_ERROR_NO_BAD_TYPE;
-
return ERTS_INIT_SCHED_BIND_TYPE_SUCCESS;
}
@@ -724,6 +721,11 @@ erts_bind_schedulers(Process *c_p, Eterm how)
erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx);
if (erts_bind_to_cpu(cpuinfo, -1) == -ENOTSUP) {
+ if (cpu_bind_order == ERTS_CPU_BIND_NONE
+ && ERTS_IS_ATOM_STR("unbound", how)) {
+ res = bound_schedulers_term(ERTS_CPU_BIND_NONE);
+ goto done;
+ }
ERTS_BIF_PREP_ERROR(res, c_p, EXC_NOTSUP);
}
else {
diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index 65565cf3c7..5b74240cc3 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 1996-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -219,47 +219,68 @@ Export ets_select_continue_exp;
* Static traps
*/
static Export ets_delete_continue_exp;
-
-static ERTS_INLINE DbTable* db_ref(DbTable* tb, db_lock_kind_t kind)
-{
- if (tb != NULL && kind != LCK_READ) {
- erts_refc_inc(&tb->common.ref, 2);
- }
- return tb;
-}
-
-static ERTS_INLINE DbTable* db_unref(DbTable* tb, db_lock_kind_t kind)
+
+static void
+free_dbtable(DbTable* tb)
{
- if (kind != LCK_READ && !erts_refc_dectest(&tb->common.ref, 0)) {
#ifdef HARDDEBUG
if (erts_smp_atomic_read(&tb->common.memory_size) != sizeof(DbTable)) {
- erts_fprintf(stderr, "ets: db_unref memory remain=%ld fix=%x\n",
- erts_smp_atomic_read(&tb->common.memory_size)-sizeof(DbTable),
+ erts_fprintf(stderr, "ets: free_dbtable memory remain=%ld fix=%x\n",
+ erts_smp_atomic_read(&tb->common.memory_size)-sizeof(DbTable),
tb->common.fixations);
}
- erts_fprintf(stderr, "ets: db_unref(%T) deleted!!!\r\n",
+ erts_fprintf(stderr, "ets: free_dbtable(%T) deleted!!!\r\n",
tb->common.id);
- erts_fprintf(stderr, "ets: db_unref: meta_pid_to_tab common.memory_size = %ld\n",
+ erts_fprintf(stderr, "ets: free_dbtable: meta_pid_to_tab common.memory_size = %ld\n",
erts_smp_atomic_read(&meta_pid_to_tab->common.memory_size));
print_table(ERTS_PRINT_STDOUT, NULL, 1, meta_pid_to_tab);
- erts_fprintf(stderr, "ets: db_unref: meta_pid_to_fixed_tab common.memory_size = %ld\n",
+ erts_fprintf(stderr, "ets: free_dbtable: meta_pid_to_fixed_tab common.memory_size = %ld\n",
erts_smp_atomic_read(&meta_pid_to_fixed_tab->common.memory_size));
print_table(ERTS_PRINT_STDOUT, NULL, 1, meta_pid_to_fixed_tab);
-
#endif
#ifdef ERTS_SMP
erts_smp_rwmtx_destroy(&tb->common.rwlock);
erts_smp_mtx_destroy(&tb->common.fixlock);
#endif
ASSERT(is_immed(tb->common.heir_data));
- erts_db_free(ERTS_ALC_T_DB_TABLE, tb, (void *) tb, sizeof(DbTable));
+ erts_db_free(ERTS_ALC_T_DB_TABLE, tb, (void *) tb, sizeof(DbTable));
ERTS_ETS_MISC_MEM_ADD(-sizeof(DbTable));
- return NULL;
- }
- return tb;
+}
+
+#ifdef ERTS_SMP
+static void
+chk_free_dbtable(void *vtb)
+{
+ DbTable * tb = (DbTable *) vtb;
+ ERTS_THR_MEMORY_BARRIER;
+ if (erts_refc_dectest(&tb->common.ref, 0) == 0)
+ free_dbtable(tb);
+}
+#endif
+
+static void schedule_free_dbtable(DbTable* tb)
+{
+ /*
+ * NON-SMP case: Caller is *not* allowed to access the *tb
+ * structure after this function has returned!
+ * SMP case: Caller is allowed to access the *tb structure
+ * until the bif has returned (we typically
+ * need to unlock the table lock after this
+ * function has returned).
+ */
+#ifdef ERTS_SMP
+ int scheds = erts_get_max_no_executing_schedulers();
+ ASSERT(scheds >= 1);
+ ASSERT(erts_refc_read(&tb->common.ref, 0) == 0);
+ erts_refc_init(&tb->common.ref, scheds);
+ ERTS_THR_MEMORY_BARRIER;
+ erts_smp_schedule_misc_aux_work(0, scheds, chk_free_dbtable, tb);
+#else
+ free_dbtable(tb);
+#endif
}
static ERTS_INLINE void db_init_lock(DbTable* tb, int use_frequent_read_lock,
@@ -270,8 +291,6 @@ static ERTS_INLINE void db_init_lock(DbTable* tb, int use_frequent_read_lock,
if (use_frequent_read_lock)
rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
#endif
- erts_refc_init(&tb->common.ref, 1);
- erts_refc_init(&tb->common.fixref, 0);
#ifdef ERTS_SMP
erts_smp_rwmtx_init_opt_x(&tb->common.rwlock, &rwmtx_opt,
rwname, tb->common.the_name);
@@ -280,7 +299,7 @@ static ERTS_INLINE void db_init_lock(DbTable* tb, int use_frequent_read_lock,
#endif
}
-static ERTS_INLINE void db_lock_take_over_ref(DbTable* tb, db_lock_kind_t kind)
+static ERTS_INLINE void db_lock(DbTable* tb, db_lock_kind_t kind)
{
#ifdef ERTS_SMP
ASSERT(tb != meta_pid_to_tab && tb != meta_pid_to_fixed_tab);
@@ -308,16 +327,13 @@ static ERTS_INLINE void db_lock_take_over_ref(DbTable* tb, db_lock_kind_t kind)
#endif
}
-static ERTS_INLINE void db_lock(DbTable* tb, db_lock_kind_t kind)
-{
- (void) db_ref(tb, kind);
-#ifdef ERTS_SMP
- db_lock_take_over_ref(tb, kind);
-#endif
-}
-
static ERTS_INLINE void db_unlock(DbTable* tb, db_lock_kind_t kind)
{
+ /*
+ * In NON-SMP case tb may refer to an already deallocated
+ * DbTable structure. That is, ONLY the SMP case is allowed
+ * to follow the tb pointer!
+ */
#ifdef ERTS_SMP
ASSERT(tb != meta_pid_to_tab && tb != meta_pid_to_fixed_tab);
@@ -344,7 +360,6 @@ static ERTS_INLINE void db_unlock(DbTable* tb, db_lock_kind_t kind)
}
}
#endif
- (void) db_unref(tb, kind); /* May delete table... */
}
@@ -371,6 +386,13 @@ DbTable* db_get_table_aux(Process *p,
DbTable *tb = NULL;
erts_smp_rwmtx_t *mtl = NULL;
+ /*
+ * IMPORTANT: Only scheduler threads are allowed
+ * to access tables. Memory management
+ * depend on it.
+ */
+ ASSERT(erts_get_scheduler_data());
+
if (is_small(id)) {
Uint slot = unsigned_val(id) & meta_main_tab_slot_mask;
if (!meta_already_locked) {
@@ -384,12 +406,8 @@ DbTable* db_get_table_aux(Process *p,
|| erts_lc_rwmtx_is_rwlocked(test_mtl));
}
#endif
- if (slot < db_max_tabs && IS_SLOT_ALIVE(slot)) {
- /* SMP: inc to prevent race, between unlock of meta_main_tab_lock
- * and the table locking outside the meta_main_tab_lock
- */
- tb = db_ref(meta_main_tab[slot].u.tb, kind);
- }
+ if (slot < db_max_tabs && IS_SLOT_ALIVE(slot))
+ tb = meta_main_tab[slot].u.tb;
}
else if (is_atom(id)) {
struct meta_name_tab_entry* bucket = meta_name_tab_bucket(id,&mtl);
@@ -403,16 +421,15 @@ DbTable* db_get_table_aux(Process *p,
if (bucket->pu.tb != NULL) {
if (is_atom(bucket->u.name_atom)) { /* single */
- if (bucket->u.name_atom == id) {
- tb = db_ref(bucket->pu.tb, kind);
- }
+ if (bucket->u.name_atom == id)
+ tb = bucket->pu.tb;
}
else { /* multi */
Uint cnt = unsigned_val(bucket->u.mcnt);
Uint i;
for (i=0; i<cnt; i++) {
if (bucket->pu.mvec[i].u.name_atom == id) {
- tb = db_ref(bucket->pu.mvec[i].pu.tb, kind);
+ tb = bucket->pu.mvec[i].pu.tb;
break;
}
}
@@ -420,7 +437,7 @@ DbTable* db_get_table_aux(Process *p,
}
}
if (tb) {
- db_lock_take_over_ref(tb, kind);
+ db_lock(tb, kind);
if (tb->common.id != id
|| ((tb->common.status & what) == 0 && p->id != tb->common.owner)) {
db_unlock(tb, kind);
@@ -594,11 +611,11 @@ done:
*/
static ERTS_INLINE void local_fix_table(DbTable* tb)
{
- erts_refc_inc(&tb->common.fixref, 1);
+ erts_refc_inc(&tb->common.ref, 1);
}
static ERTS_INLINE void local_unfix_table(DbTable* tb)
{
- if (erts_refc_dectest(&tb->common.fixref, 0) == 0) {
+ if (erts_refc_dectest(&tb->common.ref, 0) == 0) {
ASSERT(IS_HASH_TABLE(tb->common.status));
db_unfix_table_hash(&(tb->hash));
}
@@ -893,7 +910,8 @@ BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3)
Eterm upop;
Eterm* tpl;
Sint position;
- Eterm incr, warp, oldcnt;
+ Eterm incr, warp;
+ Wterm oldcnt;
if (is_not_list(iter)) {
goto finalize;
@@ -968,7 +986,7 @@ BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3)
Eterm* tpl = tuple_val(CAR(list_val(iter)));
Sint position = signed_val(tpl[1]);
Eterm incr = tpl[2];
- Eterm oldcnt = handle.dbterm->tpl[position];
+ Wterm oldcnt = db_do_read_element(&handle,position);
Eterm newcnt = db_add_counter(&htop, oldcnt, incr);
if (newcnt == NIL) {
@@ -981,9 +999,9 @@ BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3)
if (arityval(*tpl) == 4) { /* Maybe warp it */
Eterm threshold = tpl[3];
- if ((cmp(incr,make_small(0)) < 0) ? /* negative increment? */
- (cmp(newcnt,threshold) < 0) : /* if negative, check if below */
- (cmp(newcnt,threshold) > 0)) { /* else check if above threshold */
+ if ((CMP(incr,make_small(0)) < 0) ? /* negative increment? */
+ (CMP(newcnt,threshold) < 0) : /* if negative, check if below */
+ (CMP(newcnt,threshold) > 0)) { /* else check if above threshold */
newcnt = tpl[4];
}
@@ -1414,6 +1432,7 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
tb->common.type = status & ERTS_ETS_TABLE_TYPES;
/* Note, 'type' is *read only* from now on... */
#endif
+ erts_refc_init(&tb->common.ref, 0);
db_init_lock(tb, status & (DB_FINE_LOCKED|DB_FREQ_READ),
"db_tab", "db_tab_fix");
tb->common.keypos = keypos;
@@ -1436,8 +1455,7 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
"** Too many db tables **\n");
free_heir_data(tb);
tb->common.meth->db_free_table(tb);
- erts_db_free(ERTS_ALC_T_DB_TABLE, tb, (void *) tb, sizeof(DbTable));
- ERTS_ETS_MISC_MEM_ADD(-sizeof(DbTable));
+ free_dbtable(tb);
BIF_ERROR(BIF_P, SYSTEM_LIMIT);
}
@@ -1471,9 +1489,10 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
free_slot(slot);
erts_smp_rwmtx_rwunlock(mmtl);
- db_lock_take_over_ref(tb,LCK_WRITE);
+ db_lock(tb,LCK_WRITE);
free_heir_data(tb);
tb->common.meth->db_free_table(tb);
+ schedule_free_dbtable(tb);
db_unlock(tb,LCK_WRITE);
BIF_ERROR(BIF_P, BADARG);
}
@@ -2694,7 +2713,6 @@ BIF_RETTYPE ets_match_spec_run_r_3(BIF_ALIST_3)
Binary *mp;
Eterm res;
Uint32 dummy;
- Uint sz;
if (!(is_list(BIF_ARG_1) || BIF_ARG_1 == NIL) || !is_binary(BIF_ARG_2)) {
error:
@@ -2719,11 +2737,10 @@ BIF_RETTYPE ets_match_spec_run_r_3(BIF_ALIST_3)
BIF_TRAP3(bif_export[BIF_ets_match_spec_run_r_3],
BIF_P,lst,BIF_ARG_2,ret);
}
- res = db_prog_match(BIF_P, mp, CAR(list_val(lst)), NULL, 0, &dummy);
+ res = db_prog_match(BIF_P, mp, CAR(list_val(lst)), NULL, NULL, 0,
+ ERTS_PAM_COPY_RESULT, &dummy);
if (is_value(res)) {
- sz = size_object(res);
- hp = HAlloc(BIF_P, sz + 2);
- res = copy_struct(res, sz, &hp, &MSO(BIF_P));
+ hp = HAlloc(BIF_P, 2);
ret = CONS(hp,res,ret);
/*hp += 2;*/
}
@@ -2845,8 +2862,7 @@ void init_db(void)
meta_pid_to_tab->common.meth = &db_hash;
meta_pid_to_tab->common.compress = 0;
- erts_refc_init(&meta_pid_to_tab->common.ref, 1);
- erts_refc_init(&meta_pid_to_tab->common.fixref, 0);
+ erts_refc_init(&meta_pid_to_tab->common.ref, 0);
/* Neither rwlock or fixlock used
db_init_lock(meta_pid_to_tab, "meta_pid_to_tab", "meta_pid_to_tab_FIX");*/
@@ -2878,8 +2894,7 @@ void init_db(void)
meta_pid_to_fixed_tab->common.meth = &db_hash;
meta_pid_to_fixed_tab->common.compress = 0;
- erts_refc_init(&meta_pid_to_fixed_tab->common.ref, 1);
- erts_refc_init(&meta_pid_to_fixed_tab->common.fixref, 0);
+ erts_refc_init(&meta_pid_to_fixed_tab->common.ref, 0);
/* Neither rwlock or fixlock used
db_init_lock(meta_pid_to_fixed_tab, "meta_pid_to_fixed_tab", "meta_pid_to_fixed_tab_FIX");*/
@@ -3037,12 +3052,10 @@ retry:
to_pid, to_locks,
ERTS_P2P_FLG_TRY_LOCK);
if (to_proc == ERTS_PROC_LOCK_BUSY) {
- db_ref(tb, LCK_NONE); /* while unlocked */
db_unlock(tb,LCK_WRITE);
to_proc = erts_pid2proc(p, ERTS_PROC_LOCK_MAIN,
to_pid, to_locks);
db_lock(tb,LCK_WRITE);
- tb = db_unref(tb, LCK_NONE);
ASSERT(tb != NULL);
if (tb->common.owner != p->id) {
@@ -3153,13 +3166,13 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks)
erts_smp_rwmtx_t *mmtl = get_meta_main_tab_lock(ix);
erts_smp_rwmtx_rlock(mmtl);
if (!IS_SLOT_FREE(ix)) {
- tb = db_ref(GET_ANY_SLOT_TAB(ix), LCK_WRITE);
+ tb = GET_ANY_SLOT_TAB(ix);
ASSERT(tb);
}
erts_smp_rwmtx_runlock(mmtl);
if (tb) {
int do_yield;
- db_lock_take_over_ref(tb, LCK_WRITE);
+ db_lock(tb, LCK_WRITE);
/* Ownership may have changed since
we looked up the table. */
if (tb->common.owner != pid) {
@@ -3241,7 +3254,7 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks)
erts_smp_rwmtx_t *mmtl = get_meta_main_tab_lock(ix);
erts_smp_rwmtx_rlock(mmtl);
if (IS_SLOT_ALIVE(ix)) {
- tb = db_ref(meta_main_tab[ix].u.tb, LCK_WRITE_REC);
+ tb = meta_main_tab[ix].u.tb;
ASSERT(tb);
}
erts_smp_rwmtx_runlock(mmtl);
@@ -3249,7 +3262,7 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks)
int reds;
DbFixation** pp;
- db_lock_take_over_ref(tb, LCK_WRITE_REC);
+ db_lock(tb, LCK_WRITE_REC);
#ifdef ERTS_SMP
erts_smp_mtx_lock(&tb->common.fixlock);
#endif
@@ -3260,7 +3273,7 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks)
if ((*pp)->pid == pid) {
DbFixation* fix = *pp;
erts_aint_t diff = -((erts_aint_t) fix->counter);
- erts_refc_add(&tb->common.fixref,diff,0);
+ erts_refc_add(&tb->common.ref,diff,0);
*pp = fix->next;
erts_db_free(ERTS_ALC_T_DB_FIXATION,
tb, fix, sizeof(DbFixation));
@@ -3335,7 +3348,7 @@ static void fix_table_locked(Process* p, DbTable* tb)
#ifdef ERTS_SMP
erts_smp_mtx_lock(&tb->common.fixlock);
#endif
- erts_refc_inc(&tb->common.fixref,1);
+ erts_refc_inc(&tb->common.ref,1);
fix = tb->common.fixations;
if (fix == NULL) {
get_now(&(tb->common.megasec),
@@ -3389,7 +3402,7 @@ static void unfix_table_locked(Process* p, DbTable* tb,
for (pp = &tb->common.fixations; *pp != NULL; pp = &(*pp)->next) {
if ((*pp)->pid == p->id) {
DbFixation* fix = *pp;
- erts_refc_dec(&tb->common.fixref,0);
+ erts_refc_dec(&tb->common.ref,0);
--(fix->counter);
ASSERT(fix->counter >= 0);
if (fix->counter > 0) {
@@ -3419,7 +3432,6 @@ unlocked:
#ifdef ERTS_SMP
if (*kind_p == LCK_READ && tb->common.is_thread_safe) {
/* Must have write lock while purging pseudo-deleted (OTP-8166) */
- db_ref(tb, LCK_WRITE); /* LCK_WRITE need it, but not LCK_READ */
erts_smp_rwmtx_runlock(&tb->common.rwlock);
erts_smp_rwmtx_rwlock(&tb->common.rwlock);
*kind_p = LCK_WRITE;
@@ -3438,6 +3450,8 @@ static void free_fixations_locked(DbTable *tb)
fix = tb->common.fixations;
while (fix != NULL) {
+ erts_aint_t diff = -((erts_aint_t) fix->counter);
+ erts_refc_add(&tb->common.ref,diff,0);
next_fix = fix->next;
db_meta_lock(meta_pid_to_fixed_tab, LCK_WRITE_REC);
db_erase_bag_exact2(meta_pid_to_fixed_tab,
@@ -3475,11 +3489,24 @@ static void set_heir(Process* me, DbTable* tb, Eterm heir, UWord heir_data)
if (!is_immed(heir_data)) {
DeclareTmpHeap(tmp,2,me);
+ Eterm wrap_tpl;
+ int size;
+ DbTerm* dbterm;
+ Eterm* top;
+ ErlOffHeap tmp_offheap;
UseTmpHeap(2,me);
- /* Make a dummy 1-tuple around data to use db_get_term() */
- heir_data = (UWord) db_store_term(&tb->common, NULL, 0,
- TUPLE1(tmp,heir_data));
+ /* Make a dummy 1-tuple around data to use DbTerm */
+ wrap_tpl = TUPLE1(tmp,heir_data);
+ size = size_object(wrap_tpl);
+ dbterm = erts_db_alloc(ERTS_ALC_T_DB_HEIR_DATA, (DbTable *)tb,
+ (sizeof(DbTerm) + sizeof(Eterm)*(size-1)));
+ dbterm->size = size;
+ top = dbterm->tpl;
+ tmp_offheap.first = NULL;
+ copy_struct(wrap_tpl, size, &top, &tmp_offheap);
+ dbterm->first_oh = tmp_offheap.first;
+ heir_data = (UWord)dbterm;
UnUseTmpHeap(2,me);
ASSERT(!is_immed(heir_data));
}
@@ -3491,7 +3518,7 @@ static void free_heir_data(DbTable* tb)
if (tb->common.heir != am_none && !is_immed(tb->common.heir_data)) {
DbTerm* p = (DbTerm*) tb->common.heir_data;
db_cleanup_offheap_comp(p);
- erts_db_free(ERTS_ALC_T_DB_TERM, tb, (void *)p,
+ erts_db_free(ERTS_ALC_T_DB_HEIR_DATA, tb, (void *)p,
sizeof(DbTerm) + (p->size-1)*sizeof(Eterm));
}
#ifdef DEBUG
@@ -3561,10 +3588,6 @@ static int free_table_cont(Process *p,
mmtl = get_meta_main_tab_lock(tb->common.slot);
#ifdef ERTS_SMP
if (erts_smp_rwmtx_tryrwlock(mmtl) == EBUSY) {
- /*
- * We keep our increased refc over this op in order to
- * prevent the table from disapearing.
- */
erts_smp_rwmtx_rwunlock(&tb->common.rwlock);
erts_smp_rwmtx_rwlock(mmtl);
erts_smp_rwmtx_rwlock(&tb->common.rwlock);
@@ -3579,7 +3602,7 @@ static int free_table_cont(Process *p,
make_small(tb->common.slot));
db_meta_unlock(meta_pid_to_tab, LCK_WRITE_REC);
}
- db_unref(tb, LCK_NONE);
+ schedule_free_dbtable(tb);
BUMP_REDS(p, 100);
return 0;
}
diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 1e50fee554..9ef990cc4f 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 1998-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1998-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -258,12 +258,6 @@ static ERTS_INLINE Sint next_slot_w(DbTableHash* tb, Uint ix,
}
-/*
- * tplp is an untagged pointer to a tuple we know is large enough
- * and dth is a pointer to a DbTableHash.
- */
-#define GETKEY(dth, tplp) (*((tplp) + (dth)->common.keypos))
-
/*
* Some special binary flags
*/
@@ -434,6 +428,9 @@ static ERTS_INLINE void try_shrink(DbTableHash* tb)
}
}
+#define EQ_REL(x,y,y_base) \
+ (is_same(x,NULL,y,y_base) || (is_not_both_immed((x),(y)) && eq_rel((x),NULL,(y),y_base)))
+
/* Is this a live object (not pseodo-deleted) with the specified key?
*/
static ERTS_INLINE int has_live_key(DbTableHash* tb, HashDbTerm* b,
@@ -443,7 +440,7 @@ static ERTS_INLINE int has_live_key(DbTableHash* tb, HashDbTerm* b,
else {
Eterm itemKey = GETKEY(tb, b->dbterm.tpl);
ASSERT(!is_header(itemKey));
- return EQ(key,itemKey);
+ return EQ_REL(key, itemKey, b->dbterm.tpl);
}
}
@@ -456,7 +453,7 @@ static ERTS_INLINE int has_key(DbTableHash* tb, HashDbTerm* b,
else {
Eterm itemKey = GETKEY(tb, b->dbterm.tpl);
ASSERT(!is_header(itemKey));
- return EQ(key,itemKey);
+ return EQ_REL(key, itemKey, b->dbterm.tpl);
}
}
@@ -696,9 +693,7 @@ static int db_first_hash(Process *p, DbTable *tbl, Eterm *ret)
}
}
if (list != NULL) {
- Eterm key = GETKEY(tb, list->dbterm.tpl);
-
- COPY_OBJECT(key, p, ret);
+ *ret = db_copy_key(p, tbl, &list->dbterm);
RUNLOCK_HASH(lck);
}
else {
@@ -746,7 +741,7 @@ static int db_next_hash(Process *p, DbTable *tbl, Eterm key, Eterm *ret)
*ret = am_EOT;
}
else {
- COPY_OBJECT(GETKEY(tb, b->dbterm.tpl), p, ret);
+ *ret = db_copy_key(p, tbl, &b->dbterm);
RUNLOCK_HASH(lck);
}
return DB_ERROR_NONE;
@@ -1308,8 +1303,8 @@ static int db_select_continue_hash(Process *p,
}
for(;;) {
if (current->hvalue != INVALID_HASH &&
- (match_res = db_prog_match_and_copy(&tb->common, p, mp, all_objects,
- &current->dbterm, &hp, 2),
+ (match_res = db_match_dbterm(&tb->common, p, mp, all_objects,
+ &current->dbterm, &hp, 2),
is_value(match_res))) {
match_list = CONS(hp, match_res, match_list);
@@ -1473,8 +1468,8 @@ static int db_select_chunk_hash(Process *p, DbTable *tbl,
for(;;) {
if (current != NULL) {
if (current->hvalue != INVALID_HASH) {
- match_res = db_prog_match_and_copy(&tb->common, p, mpi.mp, 0,
- &current->dbterm, &hp, 2);
+ match_res = db_match_dbterm(&tb->common, p, mpi.mp, 0,
+ &current->dbterm, &hp, 2);
if (is_value(match_res)) {
match_list = CONS(hp, match_res, match_list);
++got;
@@ -1639,8 +1634,8 @@ static int db_select_count_hash(Process *p,
for(;;) {
if (current != NULL) {
if (current->hvalue != INVALID_HASH) {
- if (db_prog_match_and_copy(&tb->common, p, mpi.mp, 0,
- &current->dbterm, NULL,0) == am_true) {
+ if (db_match_dbterm(&tb->common, p, mpi.mp, 0,
+ &current->dbterm, NULL,0) == am_true) {
++got;
}
--num_left;
@@ -1788,8 +1783,8 @@ static int db_select_delete_hash(Process *p,
}
else {
int did_erase = 0;
- if (db_prog_match_and_copy(&tb->common, p, mpi.mp, 0,
- &(*current)->dbterm, NULL, 0) == am_true) {
+ if (db_match_dbterm(&tb->common, p, mpi.mp, 0,
+ &(*current)->dbterm, NULL, 0) == am_true) {
if (NFIXED(tb) > fixated_by_me) { /* fixated by others? */
if (slot_ix != last_pseudo_delete) {
add_fixed_deletion(tb, slot_ix);
@@ -1899,8 +1894,8 @@ static int db_select_delete_continue_hash(Process *p,
}
else {
int did_erase = 0;
- if (db_prog_match_and_copy(&tb->common, p, mp, 0,
- &(*current)->dbterm, NULL, 0) == am_true) {
+ if (db_match_dbterm(&tb->common, p, mp, 0,
+ &(*current)->dbterm, NULL, 0) == am_true) {
if (NFIXED(tb) > fixated_by_me) { /* fixated by others? */
if (slot_ix != last_pseudo_delete) {
add_fixed_deletion(tb, slot_ix);
@@ -1999,8 +1994,8 @@ static int db_select_count_continue_hash(Process *p,
current = current->next;
continue;
}
- if (db_prog_match_and_copy(&tb->common, p, mp, 0, &current->dbterm,
- NULL, 0) == am_true) {
+ if (db_match_dbterm(&tb->common, p, mp, 0, &current->dbterm,
+ NULL, 0) == am_true) {
++got;
}
--num_left;
@@ -2179,7 +2174,7 @@ static int analyze_pattern(DbTableHash *tb, Eterm pattern,
HashValue hval = NIL;
int num_heads = 0;
int i;
-
+
mpi->lists = mpi->dlists;
mpi->num_lists = 0;
mpi->key_given = 1;
@@ -2695,6 +2690,9 @@ static int db_lookup_dbterm_hash(DbTable *tbl, Eterm key, DbUpdateHandle* handle
handle->dbterm = &b->dbterm;
handle->mustResize = 0;
handle->new_size = b->dbterm.size;
+ #if HALFWORD_HEAP
+ handle->abs_vec = NULL;
+ #endif
handle->lck = lck;
/* KEEP hval WLOCKED, db_finalize_dbterm_hash will WUNLOCK */
return 1;
diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c
index 8108494fc5..6cdbec3213 100644
--- a/erts/emulator/beam/erl_db_tree.c
+++ b/erts/emulator/beam/erl_db_tree.c
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 1998-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1998-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -48,9 +48,6 @@
#include "erl_db_tree.h"
-
-
-#define GETKEY(dtt, tplp) (*((tplp) + (dtt)->common.keypos))
#define GETKEY_WITH_POS(Keypos, Tplp) (*((Tplp) + Keypos))
#define NITEMS(tb) ((int)erts_smp_atomic_read(&(tb)->common.nitems))
@@ -282,7 +279,7 @@ struct select_delete_context {
/*
** Forward declarations
*/
-static TreeDbTerm *linkout_tree(DbTableTree *tb, Eterm key);
+static TreeDbTerm *linkout_tree(DbTableTree *tb, Eterm key, Eterm* key_base);
static TreeDbTerm *linkout_object_tree(DbTableTree *tb,
Eterm object);
static int do_free_tree_cont(DbTableTree *tb, int num_left);
@@ -293,15 +290,15 @@ static int delsub(TreeDbTerm **this);
static TreeDbTerm *slot_search(Process *p, DbTableTree *tb, Sint slot);
static TreeDbTerm *find_node(DbTableTree *tb, Eterm key);
static TreeDbTerm **find_node2(DbTableTree *tb, Eterm key);
-static TreeDbTerm *find_next(DbTableTree *tb, DbTreeStack*, Eterm key);
-static TreeDbTerm *find_prev(DbTableTree *tb, DbTreeStack*, Eterm key);
+static TreeDbTerm *find_next(DbTableTree *tb, DbTreeStack*, Eterm key, Eterm* kbase);
+static TreeDbTerm *find_prev(DbTableTree *tb, DbTreeStack*, Eterm key, Eterm* kbase);
static TreeDbTerm *find_next_from_pb_key(DbTableTree *tb, DbTreeStack*,
Eterm key);
static TreeDbTerm *find_prev_from_pb_key(DbTableTree *tb, DbTreeStack*,
Eterm key);
static void traverse_backwards(DbTableTree *tb,
DbTreeStack*,
- Eterm lastkey,
+ Eterm lastkey, Eterm* lk_base,
int (*doit)(DbTableTree *tb,
TreeDbTerm *,
void *,
@@ -309,7 +306,7 @@ static void traverse_backwards(DbTableTree *tb,
void *context);
static void traverse_forward(DbTableTree *tb,
DbTreeStack*,
- Eterm lastkey,
+ Eterm lastkey, Eterm* lk_base,
int (*doit)(DbTableTree *tb,
TreeDbTerm *,
void *,
@@ -317,8 +314,8 @@ static void traverse_forward(DbTableTree *tb,
void *context);
static int key_given(DbTableTree *tb, Eterm pattern, TreeDbTerm **ret,
Eterm *partly_bound_key);
-static Sint cmp_partly_bound(Eterm partly_bound_key, Eterm bound_key);
-static Sint do_cmp_partly_bound(Eterm a, Eterm b, int *done);
+static Sint cmp_partly_bound(Eterm partly_bound_key, Eterm bound_key, Eterm* bk_base);
+static Sint do_cmp_partly_bound(Eterm a, Eterm b, Eterm* b_base, int *done);
static int analyze_pattern(DbTableTree *tb, Eterm pattern,
struct mp_info *mpi);
@@ -338,7 +335,6 @@ static int doit_select_delete(DbTableTree *tb,
TreeDbTerm *this,
void *ptr,
int forward);
-static void do_dump_tree(int to, void *to_arg, TreeDbTerm *t);
static int partly_bound_can_match_lesser(Eterm partly_bound_1,
Eterm partly_bound_2);
@@ -492,9 +488,6 @@ static int db_first_tree(Process *p, DbTable *tbl, Eterm *ret)
DbTableTree *tb = &tbl->tree;
DbTreeStack* stack;
TreeDbTerm *this;
- Eterm e;
- Eterm *hp;
- Uint sz;
if (( this = tb->root ) == NULL) {
*ret = am_EOT;
@@ -513,13 +506,7 @@ static int db_first_tree(Process *p, DbTable *tbl, Eterm *ret)
stack->slot = 1;
release_stack(tb,stack);
}
- e = GETKEY(tb, this->dbterm.tpl);
- sz = size_object(e);
-
- hp = HAlloc(p, sz);
-
- *ret = copy_struct(e,sz,&hp,&MSO(p));
-
+ *ret = db_copy_key(p, tbl, &this->dbterm);
return DB_ERROR_NONE;
}
@@ -528,26 +515,17 @@ static int db_next_tree(Process *p, DbTable *tbl, Eterm key, Eterm *ret)
DbTableTree *tb = &tbl->tree;
DbTreeStack* stack;
TreeDbTerm *this;
- Eterm e;
- Eterm *hp;
- Uint sz;
if (is_atom(key) && key == am_EOT)
return DB_ERROR_BADKEY;
stack = get_any_stack(tb);
- this = find_next(tb, stack, key);
+ this = find_next(tb, stack, key, NULL);
release_stack(tb,stack);
if (this == NULL) {
*ret = am_EOT;
return DB_ERROR_NONE;
}
- e = GETKEY(tb, this->dbterm.tpl);
- sz = size_object(e);
-
- hp = HAlloc(p, sz);
-
- *ret = copy_struct(e,sz,&hp,&MSO(p));
-
+ *ret = db_copy_key(p, tbl, &this->dbterm);
return DB_ERROR_NONE;
}
@@ -556,9 +534,6 @@ static int db_last_tree(Process *p, DbTable *tbl, Eterm *ret)
DbTableTree *tb = &tbl->tree;
TreeDbTerm *this;
DbTreeStack* stack;
- Eterm e;
- Eterm *hp;
- Uint sz;
if (( this = tb->root ) == NULL) {
*ret = am_EOT;
@@ -577,13 +552,7 @@ static int db_last_tree(Process *p, DbTable *tbl, Eterm *ret)
stack->slot = NITEMS(tb);
release_stack(tb,stack);
}
- e = GETKEY(tb, this->dbterm.tpl);
- sz = size_object(e);
-
- hp = HAlloc(p, sz);
-
- *ret = copy_struct(e,sz,&hp,&MSO(p));
-
+ *ret = db_copy_key(p, tbl, &this->dbterm);
return DB_ERROR_NONE;
}
@@ -592,27 +561,33 @@ static int db_prev_tree(Process *p, DbTable *tbl, Eterm key, Eterm *ret)
DbTableTree *tb = &tbl->tree;
TreeDbTerm *this;
DbTreeStack* stack;
- Eterm e;
- Eterm *hp;
- Uint sz;
if (is_atom(key) && key == am_EOT)
return DB_ERROR_BADKEY;
stack = get_any_stack(tb);
- this = find_prev(tb, stack, key);
+ this = find_prev(tb, stack, key, NULL);
release_stack(tb,stack);
if (this == NULL) {
*ret = am_EOT;
return DB_ERROR_NONE;
}
- e = GETKEY(tb, this->dbterm.tpl);
- sz = size_object(e);
+ *ret = db_copy_key(p, tbl, &this->dbterm);
+ return DB_ERROR_NONE;
+}
- hp = HAlloc(p, sz);
+static ERTS_INLINE int cmp_key(DbTableTree* tb, Eterm key, Eterm* key_base,
+ TreeDbTerm* obj)
+{
+ return cmp_rel(key, key_base,
+ GETKEY(tb,obj->dbterm.tpl), obj->dbterm.tpl);
+}
- *ret = copy_struct(e,sz,&hp,&MSO(p));
-
- return DB_ERROR_NONE;
+static ERTS_INLINE int cmp_key_eq(DbTableTree* tb, Eterm key, Eterm* key_base,
+ TreeDbTerm* obj)
+{
+ Eterm obj_key = GETKEY(tb,obj->dbterm.tpl);
+ return is_same(key, key_base, obj_key, obj->dbterm.tpl)
+ || cmp_rel(key, key_base, obj_key, obj->dbterm.tpl) == 0;
}
static int db_put_tree(DbTable *tbl, Eterm obj, int key_clash_fail)
@@ -646,8 +621,8 @@ static int db_put_tree(DbTable *tbl, Eterm obj, int key_clash_fail)
(*this)->balance = 0;
(*this)->left = (*this)->right = NULL;
break;
- } else if ((c = cmp(key,GETKEY(tb,(*this)->dbterm.tpl))) < 0) {
- /* go left */
+ } else if ((c = cmp_key(tb, key, NULL, *this)) < 0) {
+ /* go lefts */
dstack[dpos++] = DIR_LEFT;
tstack[tpos++] = this;
this = &((*this)->left);
@@ -801,7 +776,7 @@ static int db_erase_tree(DbTable *tbl, Eterm key, Eterm *ret)
*ret = am_true;
- if ((res = linkout_tree(tb, key)) != NULL) {
+ if ((res = linkout_tree(tb, key, NULL)) != NULL) {
free_term(tb, res);
}
return DB_ERROR_NONE;
@@ -993,15 +968,15 @@ static int db_select_continue_tree(Process *p,
stack = get_any_stack(tb);
if (chunk_size) {
if (reverse) {
- traverse_backwards(tb, stack, lastkey, &doit_select_chunk, &sc);
+ traverse_backwards(tb, stack, lastkey, NULL, &doit_select_chunk, &sc);
} else {
- traverse_forward(tb, stack, lastkey, &doit_select_chunk, &sc);
+ traverse_forward(tb, stack, lastkey, NULL, &doit_select_chunk, &sc);
}
} else {
if (reverse) {
- traverse_forward(tb, stack, lastkey, &doit_select, &sc);
+ traverse_forward(tb, stack, lastkey, NULL, &doit_select, &sc);
} else {
- traverse_backwards(tb, stack, lastkey, &doit_select, &sc);
+ traverse_backwards(tb, stack, lastkey, NULL, &doit_select, &sc);
}
}
release_stack(tb,stack);
@@ -1026,10 +1001,9 @@ static int db_select_continue_tree(Process *p,
}
key = GETKEY(tb, sc.lastobj);
-
- sz = size_object(key);
+ sz = size_object_rel(key,sc.lastobj);
hp = HAlloc(p, 9 + sz);
- key = copy_struct(key, sz, &hp, &MSO(p));
+ key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL);
continuation = TUPLE8
(hp,
tptr[1],
@@ -1050,8 +1024,8 @@ static int db_select_continue_tree(Process *p,
key = GETKEY(tb, sc.lastobj);
if (chunk_size) {
if (end_condition != NIL &&
- ((!reverse && cmp_partly_bound(end_condition,key) < 0) ||
- (reverse && cmp_partly_bound(end_condition,key) > 0))) {
+ ((!reverse && cmp_partly_bound(end_condition,key,sc.lastobj) < 0) ||
+ (reverse && cmp_partly_bound(end_condition,key,sc.lastobj) > 0))) {
/* done anyway */
if (!sc.got) {
RET_TO_BIF(am_EOT, DB_ERROR_NONE);
@@ -1063,16 +1037,16 @@ static int db_select_continue_tree(Process *p,
}
} else {
if (end_condition != NIL &&
- ((!reverse && cmp_partly_bound(end_condition,key) > 0) ||
- (reverse && cmp_partly_bound(end_condition,key) < 0))) {
+ ((!reverse && cmp_partly_bound(end_condition,key,sc.lastobj) > 0) ||
+ (reverse && cmp_partly_bound(end_condition,key,sc.lastobj) < 0))) {
/* done anyway */
RET_TO_BIF(sc.accum,DB_ERROR_NONE);
}
}
/* Not done yet, let's trap. */
- sz = size_object(key);
+ sz = size_object_rel(key,sc.lastobj);
hp = HAlloc(p, 9 + sz);
- key = copy_struct(key, sz, &hp, &MSO(p));
+ key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL);
continuation = TUPLE8
(hp,
tptr[1],
@@ -1099,6 +1073,7 @@ static int db_select_tree(Process *p, DbTable *tbl,
struct select_context sc;
struct mp_info mpi;
Eterm lastkey = THE_NON_VALUE;
+ Eterm* lk_base = NULL;
Eterm key;
Eterm continuation;
unsigned sz;
@@ -1140,7 +1115,7 @@ static int db_select_tree(Process *p, DbTable *tbl,
sc.all_objects = mpi.all_objects;
if (!mpi.got_partial && mpi.some_limitation &&
- cmp(mpi.least,mpi.most) == 0) {
+ CMP(mpi.least,mpi.most) == 0) {
doit_select(tb,mpi.save_term,&sc,0 /* direction doesn't matter */);
RET_TO_BIF(sc.accum,DB_ERROR_NONE);
}
@@ -1150,20 +1125,20 @@ static int db_select_tree(Process *p, DbTable *tbl,
if (mpi.some_limitation) {
if ((this = find_prev_from_pb_key(tb, stack, mpi.least)) != NULL) {
lastkey = GETKEY(tb, this->dbterm.tpl);
+ lk_base = this->dbterm.tpl;
}
sc.end_condition = mpi.most;
}
-
- traverse_forward(tb, stack, lastkey, &doit_select, &sc);
+ traverse_forward(tb, stack, lastkey, lk_base, &doit_select, &sc);
} else {
if (mpi.some_limitation) {
if ((this = find_next_from_pb_key(tb, stack, mpi.most)) != NULL) {
lastkey = GETKEY(tb, this->dbterm.tpl);
+ lk_base = this->dbterm.tpl;
}
sc.end_condition = mpi.least;
}
-
- traverse_backwards(tb, stack, lastkey, &doit_select, &sc);
+ traverse_backwards(tb, stack, lastkey, lk_base, &doit_select, &sc);
}
release_stack(tb,stack);
#ifdef HARDDEBUG
@@ -1176,9 +1151,9 @@ static int db_select_tree(Process *p, DbTable *tbl,
}
key = GETKEY(tb, sc.lastobj);
- sz = size_object(key);
+ sz = size_object_rel(key, sc.lastobj);
hp = HAlloc(p, 9 + sz + PROC_BIN_SIZE);
- key = copy_struct(key, sz, &hp, &MSO(p));
+ key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL);
if (mpi.all_objects)
(mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS;
mpb=db_make_mp_binary(p,mpi.mp,&hp);
@@ -1259,7 +1234,7 @@ static int db_select_count_continue_tree(Process *p,
}
stack = get_any_stack(tb);
- traverse_backwards(tb, stack, lastkey, &doit_select_count, &sc);
+ traverse_backwards(tb, stack, lastkey, NULL, &doit_select_count, &sc);
release_stack(tb,stack);
BUMP_REDS(p, 1000 - sc.max);
@@ -1269,12 +1244,12 @@ static int db_select_count_continue_tree(Process *p,
}
key = GETKEY(tb, sc.lastobj);
if (end_condition != NIL &&
- (cmp_partly_bound(end_condition,key) > 0)) {
+ (cmp_partly_bound(end_condition,key,sc.lastobj) > 0)) {
/* done anyway */
RET_TO_BIF(make_small(sc.got),DB_ERROR_NONE);
}
/* Not done yet, let's trap. */
- sz = size_object(key);
+ sz = size_object_rel(key, sc.lastobj);
if (IS_USMALL(0, sc.got)) {
hp = HAlloc(p, sz + 6);
egot = make_small(sc.got);
@@ -1284,7 +1259,7 @@ static int db_select_count_continue_tree(Process *p,
egot = uint_to_big(sc.got, hp);
hp += BIG_UINT_HEAP_SIZE;
}
- key = copy_struct(key, sz, &hp, &MSO(p));
+ key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL);
continuation = TUPLE5
(hp,
tptr[1],
@@ -1307,6 +1282,7 @@ static int db_select_count_tree(Process *p, DbTable *tbl,
struct select_count_context sc;
struct mp_info mpi;
Eterm lastkey = THE_NON_VALUE;
+ Eterm* lk_base = NULL;
Eterm key;
Eterm continuation;
unsigned sz;
@@ -1347,7 +1323,7 @@ static int db_select_count_tree(Process *p, DbTable *tbl,
sc.all_objects = mpi.all_objects;
if (!mpi.got_partial && mpi.some_limitation &&
- cmp(mpi.least,mpi.most) == 0) {
+ CMP(mpi.least,mpi.most) == 0) {
doit_select_count(tb,mpi.save_term,&sc,0 /* dummy */);
RET_TO_BIF(erts_make_integer(sc.got,p),DB_ERROR_NONE);
}
@@ -1356,11 +1332,12 @@ static int db_select_count_tree(Process *p, DbTable *tbl,
if (mpi.some_limitation) {
if ((this = find_next_from_pb_key(tb, stack, mpi.most)) != NULL) {
lastkey = GETKEY(tb, this->dbterm.tpl);
+ lk_base = this->dbterm.tpl;
}
sc.end_condition = mpi.least;
}
- traverse_backwards(tb, stack, lastkey, &doit_select_count, &sc);
+ traverse_backwards(tb, stack, lastkey, lk_base, &doit_select_count, &sc);
release_stack(tb,stack);
BUMP_REDS(p, 1000 - sc.max);
if (sc.max > 0) {
@@ -1368,7 +1345,7 @@ static int db_select_count_tree(Process *p, DbTable *tbl,
}
key = GETKEY(tb, sc.lastobj);
- sz = size_object(key);
+ sz = size_object_rel(key, sc.lastobj);
if (IS_USMALL(0, sc.got)) {
hp = HAlloc(p, sz + PROC_BIN_SIZE + 6);
egot = make_small(sc.got);
@@ -1378,7 +1355,7 @@ static int db_select_count_tree(Process *p, DbTable *tbl,
egot = uint_to_big(sc.got, hp);
hp += BIG_UINT_HEAP_SIZE;
}
- key = copy_struct(key, sz, &hp, &MSO(p));
+ key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL);
if (mpi.all_objects)
(mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS;
mpb = db_make_mp_binary(p,mpi.mp,&hp);
@@ -1409,6 +1386,7 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl,
struct select_context sc;
struct mp_info mpi;
Eterm lastkey = THE_NON_VALUE;
+ Eterm* lk_base = NULL;
Eterm key;
Eterm continuation;
unsigned sz;
@@ -1450,7 +1428,7 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl,
sc.all_objects = mpi.all_objects;
if (!mpi.got_partial && mpi.some_limitation &&
- cmp(mpi.least,mpi.most) == 0) {
+ CMP(mpi.least,mpi.most) == 0) {
doit_select(tb,mpi.save_term,&sc, 0 /* direction doesn't matter */);
if (sc.accum != NIL) {
hp=HAlloc(p, 3);
@@ -1465,20 +1443,20 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl,
if (mpi.some_limitation) {
if ((this = find_next_from_pb_key(tb, stack, mpi.most)) != NULL) {
lastkey = GETKEY(tb, this->dbterm.tpl);
+ lk_base = this->dbterm.tpl;
}
sc.end_condition = mpi.least;
}
-
- traverse_backwards(tb, stack, lastkey, &doit_select_chunk, &sc);
+ traverse_backwards(tb, stack, lastkey, lk_base, &doit_select_chunk, &sc);
} else {
if (mpi.some_limitation) {
if ((this = find_prev_from_pb_key(tb, stack, mpi.least)) != NULL) {
lastkey = GETKEY(tb, this->dbterm.tpl);
+ lk_base = this->dbterm.tpl;
}
sc.end_condition = mpi.most;
}
-
- traverse_forward(tb, stack, lastkey, &doit_select_chunk, &sc);
+ traverse_forward(tb, stack, lastkey, lk_base, &doit_select_chunk, &sc);
}
release_stack(tb,stack);
@@ -1503,9 +1481,9 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl,
}
key = GETKEY(tb, sc.lastobj);
- sz = size_object(key);
+ sz = size_object_rel(key, sc.lastobj);
hp = HAlloc(p, 9 + sz + PROC_BIN_SIZE);
- key = copy_struct(key, sz, &hp, &MSO(p));
+ key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL);
if (mpi.all_objects)
(mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS;
mpb = db_make_mp_binary(p,mpi.mp,&hp);
@@ -1528,9 +1506,9 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl,
}
key = GETKEY(tb, sc.lastobj);
- sz = size_object(key);
+ sz = size_object_rel(key, sc.lastobj);
hp = HAlloc(p, 9 + sz + PROC_BIN_SIZE);
- key = copy_struct(key, sz, &hp, &MSO(p));
+ key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL);
if (mpi.all_objects)
(mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS;
@@ -1606,7 +1584,7 @@ static int db_select_delete_continue_tree(Process *p,
sc.keypos = tb->common.keypos;
ASSERT(!erts_smp_atomic_read(&tb->is_stack_busy));
- traverse_backwards(tb, &tb->static_stack, lastkey, &doit_select_delete, &sc);
+ traverse_backwards(tb, &tb->static_stack, lastkey, NULL, &doit_select_delete, &sc);
BUMP_REDS(p, 1000 - sc.max);
@@ -1615,11 +1593,11 @@ static int db_select_delete_continue_tree(Process *p,
}
key = GETKEY(tb, (sc.lastterm)->dbterm.tpl);
if (end_condition != NIL &&
- cmp_partly_bound(end_condition,key) > 0) { /* done anyway */
+ cmp_partly_bound(end_condition,key,sc.lastterm->dbterm.tpl) > 0) { /* done anyway */
RET_TO_BIF(erts_make_integer(sc.accum,p),DB_ERROR_NONE);
}
/* Not done yet, let's trap. */
- sz = size_object(key);
+ sz = size_object_rel(key, sc.lastterm->dbterm.tpl);
if (IS_USMALL(0, sc.accum)) {
hp = HAlloc(p, sz + 6);
eaccsum = make_small(sc.accum);
@@ -1629,7 +1607,7 @@ static int db_select_delete_continue_tree(Process *p,
eaccsum = uint_to_big(sc.accum, hp);
hp += BIG_UINT_HEAP_SIZE;
}
- key = copy_struct(key, sz, &hp, &MSO(p));
+ key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastterm->dbterm.tpl, NULL);
continuation = TUPLE5
(hp,
tptr[1],
@@ -1650,6 +1628,7 @@ static int db_select_delete_tree(Process *p, DbTable *tbl,
struct select_delete_context sc;
struct mp_info mpi;
Eterm lastkey = THE_NON_VALUE;
+ Eterm* lk_base = NULL;
Eterm key;
Eterm continuation;
unsigned sz;
@@ -1693,7 +1672,7 @@ static int db_select_delete_tree(Process *p, DbTable *tbl,
sc.mp = mpi.mp;
if (!mpi.got_partial && mpi.some_limitation &&
- cmp(mpi.least,mpi.most) == 0) {
+ CMP(mpi.least,mpi.most) == 0) {
doit_select_delete(tb,mpi.save_term,&sc, 0 /* direction doesn't
matter */);
RET_TO_BIF(erts_make_integer(sc.accum,p),DB_ERROR_NONE);
@@ -1702,11 +1681,12 @@ static int db_select_delete_tree(Process *p, DbTable *tbl,
if (mpi.some_limitation) {
if ((this = find_next_from_pb_key(tb, &tb->static_stack, mpi.most)) != NULL) {
lastkey = GETKEY(tb, this->dbterm.tpl);
+ lk_base = this->dbterm.tpl;
}
sc.end_condition = mpi.least;
}
- traverse_backwards(tb, &tb->static_stack, lastkey, &doit_select_delete, &sc);
+ traverse_backwards(tb, &tb->static_stack, lastkey, lk_base, &doit_select_delete, &sc);
BUMP_REDS(p, 1000 - sc.max);
if (sc.max > 0) {
@@ -1714,7 +1694,7 @@ static int db_select_delete_tree(Process *p, DbTable *tbl,
}
key = GETKEY(tb, (sc.lastterm)->dbterm.tpl);
- sz = size_object(key);
+ sz = size_object_rel(key, sc.lastterm->dbterm.tpl);
if (IS_USMALL(0, sc.accum)) {
hp = HAlloc(p, sz + PROC_BIN_SIZE + 6);
eaccsum = make_small(sc.accum);
@@ -1724,7 +1704,7 @@ static int db_select_delete_tree(Process *p, DbTable *tbl,
eaccsum = uint_to_big(sc.accum, hp);
hp += BIG_UINT_HEAP_SIZE;
}
- key = copy_struct(key, sz, &hp, &MSO(p));
+ key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastterm->dbterm.tpl, NULL);
mpb = db_make_mp_binary(p,mpi.mp,&hp);
continuation = TUPLE5
@@ -1766,7 +1746,6 @@ static void db_print_tree(int to, void *to_arg,
"------------------------------------------------\n");
#else
erts_print(to, to_arg, "Ordered set (AVL tree), Elements: %d\n", NITEMS(tb));
- do_dump_tree(to, to_arg, tb->root);
#endif
}
@@ -1842,7 +1821,7 @@ do_db_tree_foreach_offheap(TreeDbTerm *tdbt,
}
static TreeDbTerm *linkout_tree(DbTableTree *tb,
- Eterm key)
+ Eterm key, Eterm* key_base)
{
TreeDbTerm **tstack[STACK_NEED];
int tpos = 0;
@@ -1865,7 +1844,7 @@ static TreeDbTerm *linkout_tree(DbTableTree *tb,
for (;;) {
if (!*this) { /* Failure */
return NULL;
- } else if ((c = cmp(key,GETKEY(tb,(*this)->dbterm.tpl))) < 0) {
+ } else if ((c = cmp_key(tb, key, key_base, *this)) < 0) {
dstack[dpos++] = DIR_LEFT;
tstack[tpos++] = this;
this = &((*this)->left);
@@ -1929,7 +1908,7 @@ static TreeDbTerm *linkout_object_tree(DbTableTree *tb,
for (;;) {
if (!*this) { /* Failure */
return NULL;
- } else if ((c = cmp(key,GETKEY(tb,(*this)->dbterm.tpl))) < 0) {
+ } else if ((c = cmp_key(tb,key,NULL,*this)) < 0) {
dstack[dpos++] = DIR_LEFT;
tstack[tpos++] = this;
this = &((*this)->left);
@@ -2082,15 +2061,6 @@ static int analyze_pattern(DbTableTree *tb, Eterm pattern,
return DB_ERROR_NONE;
}
-static void do_dump_tree(int to, void *to_arg, TreeDbTerm *t)
-{
- if (t != NULL) {
- do_dump_tree(to, to_arg, t->left);
- erts_print(to, to_arg, "%T\n", make_tuple(t->dbterm.tpl));
- do_dump_tree(to, to_arg, t->right);
- }
-}
-
static int do_free_tree_cont(DbTableTree *tb, int num_left)
{
TreeDbTerm *root;
@@ -2324,14 +2294,15 @@ done:
* Find next and previous in sort order
*/
-static TreeDbTerm *find_next(DbTableTree *tb, DbTreeStack* stack, Eterm key)
+static TreeDbTerm *find_next(DbTableTree *tb, DbTreeStack* stack,
+ Eterm key, Eterm* key_base)
{
TreeDbTerm *this;
TreeDbTerm *tmp;
Sint c;
if(( this = TOP_NODE(stack)) != NULL) {
- if (!CMP_EQ(GETKEY(tb, this->dbterm.tpl),key)) {
+ if (!cmp_key_eq(tb,key,key_base,this)) {
/* Start from the beginning */
stack->pos = stack->slot = 0;
}
@@ -2341,14 +2312,14 @@ static TreeDbTerm *find_next(DbTableTree *tb, DbTreeStack* stack, Eterm key)
return NULL;
for (;;) {
PUSH_NODE(stack, this);
- if (( c = cmp(GETKEY(tb, this->dbterm.tpl),key) ) < 0) {
+ if (( c = cmp_key(tb,key,key_base,this) ) > 0) {
if (this->right == NULL) /* We are at the previos
and the element does
not exist */
break;
else
this = this->right;
- } else if (c > 0) {
+ } else if (c < 0) {
if (this->left == NULL) /* Done */
return this;
else
@@ -2381,14 +2352,15 @@ static TreeDbTerm *find_next(DbTableTree *tb, DbTreeStack* stack, Eterm key)
return this;
}
-static TreeDbTerm *find_prev(DbTableTree *tb, DbTreeStack* stack, Eterm key)
+static TreeDbTerm *find_prev(DbTableTree *tb, DbTreeStack* stack,
+ Eterm key, Eterm* key_base)
{
TreeDbTerm *this;
TreeDbTerm *tmp;
Sint c;
if(( this = TOP_NODE(stack)) != NULL) {
- if (!CMP_EQ(GETKEY(tb, this->dbterm.tpl),key)) {
+ if (!cmp_key_eq(tb,key,key_base,this)) {
/* Start from the beginning */
stack->pos = stack->slot = 0;
}
@@ -2398,14 +2370,14 @@ static TreeDbTerm *find_prev(DbTableTree *tb, DbTreeStack* stack, Eterm key)
return NULL;
for (;;) {
PUSH_NODE(stack, this);
- if (( c = cmp(GETKEY(tb, this->dbterm.tpl),key) ) > 0) {
+ if (( c = cmp_key(tb,key,key_base,this) ) < 0) {
if (this->left == NULL) /* We are at the next
and the element does
not exist */
break;
else
this = this->left;
- } else if (c < 0) {
+ } else if (c > 0) {
if (this->right == NULL) /* Done */
return this;
else
@@ -2451,7 +2423,8 @@ static TreeDbTerm *find_next_from_pb_key(DbTableTree *tb, DbTreeStack* stack,
return NULL;
for (;;) {
PUSH_NODE(stack, this);
- if (( c = cmp_partly_bound(key,GETKEY(tb, this->dbterm.tpl)) ) >= 0) {
+ if (( c = cmp_partly_bound(key,GETKEY(tb, this->dbterm.tpl),
+ this->dbterm.tpl) ) >= 0) {
if (this->right == NULL) {
do {
tmp = POP_NODE(stack);
@@ -2484,7 +2457,8 @@ static TreeDbTerm *find_prev_from_pb_key(DbTableTree *tb, DbTreeStack* stack,
return NULL;
for (;;) {
PUSH_NODE(stack, this);
- if (( c = cmp_partly_bound(key,GETKEY(tb, this->dbterm.tpl)) ) <= 0) {
+ if (( c = cmp_partly_bound(key,GETKEY(tb, this->dbterm.tpl),
+ this->dbterm.tpl) ) <= 0) {
if (this->left == NULL) {
do {
tmp = POP_NODE(stack);
@@ -2514,12 +2488,11 @@ static TreeDbTerm *find_node(DbTableTree *tb, Eterm key)
Sint res;
DbTreeStack* stack = get_static_stack(tb);
- if(!stack || EMPTY_NODE(stack)
- || !CMP_EQ(GETKEY(tb, ( this = TOP_NODE(stack) )->dbterm.tpl), key)) {
+ if(!stack || EMPTY_NODE(stack)
+ || !cmp_key_eq(tb, key, NULL, (this=TOP_NODE(stack)))) {
this = tb->root;
- while (this != NULL &&
- ( res = cmp(key, GETKEY(tb, this->dbterm.tpl)) ) != 0) {
+ while (this != NULL && (res = cmp_key(tb,key,NULL,this)) != 0) {
if (res < 0)
this = this->left;
else
@@ -2541,8 +2514,7 @@ static TreeDbTerm **find_node2(DbTableTree *tb, Eterm key)
Sint res;
this = &tb->root;
- while ((*this) != NULL &&
- ( res = cmp(key, GETKEY(tb, (*this)->dbterm.tpl)) ) != 0) {
+ while ((*this) != NULL && (res = cmp_key(tb, key, NULL, *this)) != 0) {
if (res < 0)
this = &((*this)->left);
else
@@ -2565,6 +2537,9 @@ static int db_lookup_dbterm_tree(DbTable *tbl, Eterm key, DbUpdateHandle* handle
handle->mustResize = 0;
handle->bp = (void**) pp;
handle->new_size = (*pp)->dbterm.size;
+#if HALFWORD_HEAP
+ handle->abs_vec = NULL;
+#endif
return 1;
}
@@ -2589,7 +2564,7 @@ static void db_finalize_dbterm_tree(DbUpdateHandle* handle)
*/
static void traverse_backwards(DbTableTree *tb,
DbTreeStack* stack,
- Eterm lastkey,
+ Eterm lastkey, Eterm* lk_base,
int (*doit)(DbTableTree *,
TreeDbTerm *,
void *,
@@ -2608,15 +2583,16 @@ static void traverse_backwards(DbTableTree *tb,
this = this->right;
}
this = TOP_NODE(stack);
- next = find_prev(tb, stack, GETKEY(tb, this->dbterm.tpl));
+ next = find_prev(tb, stack, GETKEY(tb, this->dbterm.tpl),
+ this->dbterm.tpl);
if (!((*doit)(tb, this, context, 0)))
return;
} else {
- next = find_prev(tb, stack, lastkey);
+ next = find_prev(tb, stack, lastkey, lk_base);
}
while ((this = next) != NULL) {
- next = find_prev(tb, stack, GETKEY(tb, this->dbterm.tpl));
+ next = find_prev(tb, stack, GETKEY(tb, this->dbterm.tpl), this->dbterm.tpl);
if (!((*doit)(tb, this, context, 0)))
return;
}
@@ -2627,7 +2603,7 @@ static void traverse_backwards(DbTableTree *tb,
*/
static void traverse_forward(DbTableTree *tb,
DbTreeStack* stack,
- Eterm lastkey,
+ Eterm lastkey, Eterm* lk_base,
int (*doit)(DbTableTree *,
TreeDbTerm *,
void *,
@@ -2646,15 +2622,15 @@ static void traverse_forward(DbTableTree *tb,
this = this->left;
}
this = TOP_NODE(stack);
- next = find_next(tb, stack, GETKEY(tb, this->dbterm.tpl));
+ next = find_next(tb, stack, GETKEY(tb, this->dbterm.tpl), this->dbterm.tpl);
if (!((*doit)(tb, this, context, 1)))
return;
} else {
- next = find_next(tb, stack, lastkey);
+ next = find_next(tb, stack, lastkey, lk_base);
}
while ((this = next) != NULL) {
- next = find_next(tb, stack, GETKEY(tb, this->dbterm.tpl));
+ next = find_next(tb, stack, GETKEY(tb, this->dbterm.tpl), this->dbterm.tpl);
if (!((*doit)(tb, this, context, 1)))
return;
}
@@ -2680,7 +2656,7 @@ static int key_given(DbTableTree *tb, Eterm pattern, TreeDbTerm **ret,
if (( this = find_node(tb, key) ) == NULL) {
return -1;
}
- *ret = this;
+ *ret = this;
return 1;
} else if (partly_bound != NULL && key != am_Underscore &&
db_is_variable(key) < 0)
@@ -2691,7 +2667,7 @@ static int key_given(DbTableTree *tb, Eterm pattern, TreeDbTerm **ret,
-static Sint do_cmp_partly_bound(Eterm a, Eterm b, int *done)
+static Sint do_cmp_partly_bound(Eterm a, Eterm b, Eterm* b_base, int *done)
{
Eterm* aa;
Eterm* bb;
@@ -2705,44 +2681,44 @@ static Sint do_cmp_partly_bound(Eterm a, Eterm b, int *done)
*done = 1;
return 0;
}
- if (a == b)
+ if (is_same(a,NULL,b,b_base))
return 0;
switch (a & _TAG_PRIMARY_MASK) {
case TAG_PRIMARY_LIST:
if (!is_list(b)) {
- return cmp(a,b);
+ return cmp_rel(a,NULL,b,b_base);
}
aa = list_val(a);
- bb = list_val(b);
+ bb = list_val_rel(b,b_base);
while (1) {
- if ((j = do_cmp_partly_bound(*aa++, *bb++, done)) != 0 || *done)
+ if ((j = do_cmp_partly_bound(*aa++, *bb++, b_base, done)) != 0 || *done)
return j;
if (*aa==*bb)
return 0;
if (is_not_list(*aa) || is_not_list(*bb))
- return do_cmp_partly_bound(*aa, *bb, done);
+ return do_cmp_partly_bound(*aa, *bb, b_base, done);
aa = list_val(*aa);
- bb = list_val(*bb);
+ bb = list_val_rel(*bb,b_base);
}
case TAG_PRIMARY_BOXED:
if ((b & _TAG_PRIMARY_MASK) != TAG_PRIMARY_BOXED) {
- return cmp(a,b);
+ return cmp_rel(a,NULL,b,b_base);
}
a_hdr = ((*boxed_val(a)) & _TAG_HEADER_MASK) >> _TAG_PRIMARY_SIZE;
- b_hdr = ((*boxed_val(b)) & _TAG_HEADER_MASK) >> _TAG_PRIMARY_SIZE;
+ b_hdr = ((*boxed_val_rel(b,b_base)) & _TAG_HEADER_MASK) >> _TAG_PRIMARY_SIZE;
if (a_hdr != b_hdr) {
- return cmp(a, b);
+ return cmp_rel(a, NULL, b, b_base);
}
if (a_hdr == (_TAG_HEADER_ARITYVAL >> _TAG_PRIMARY_SIZE)) {
aa = tuple_val(a);
- bb = tuple_val(b);
+ bb = tuple_val_rel(b, b_base);
/* compare the arities */
i = arityval(*aa); /* get the arity*/
if (i < arityval(*bb)) return(-1);
if (i > arityval(*bb)) return(1);
while (i--) {
- if ((j = do_cmp_partly_bound(*++aa, *++bb, done)) != 0
+ if ((j = do_cmp_partly_bound(*++aa, *++bb, b_base, done)) != 0
|| *done)
return j;
}
@@ -2750,14 +2726,14 @@ static Sint do_cmp_partly_bound(Eterm a, Eterm b, int *done)
}
/* Drop through */
default:
- return cmp(a, b);
+ return cmp_rel(a, NULL, b, b_base);
}
}
-static Sint cmp_partly_bound(Eterm partly_bound_key, Eterm bound_key)
+static Sint cmp_partly_bound(Eterm partly_bound_key, Eterm bound_key, Eterm* bk_base)
{
int done = 0;
- Sint ret = do_cmp_partly_bound(partly_bound_key, bound_key, &done);
+ Sint ret = do_cmp_partly_bound(partly_bound_key, bound_key, bk_base, &done);
#ifdef HARDDEBUG
erts_fprintf(stderr,"\ncmp_partly_bound: %T", partly_bound_key);
if (ret < 0)
@@ -2766,7 +2742,7 @@ static Sint cmp_partly_bound(Eterm partly_bound_key, Eterm bound_key)
erts_fprintf(stderr," > ");
else
erts_fprintf(stderr," == ");
- erts_fprintf(stderr,"%T\n",bound_key);
+ erts_fprintf(stderr,"%T\n",bound_key); // HALFWORD BUG: printing rterm
#endif
return ret;
}
@@ -2853,7 +2829,7 @@ static int do_partly_bound_can_match_lesser(Eterm a, Eterm b,
if (not_eq_tags(a,b)) {
*done = 1;
- return (cmp(a, b) < 0) ? 1 : 0;
+ return (CMP(a, b) < 0) ? 1 : 0;
}
/* we now know that tags are the same */
@@ -2889,7 +2865,7 @@ static int do_partly_bound_can_match_lesser(Eterm a, Eterm b,
bb = list_val(*bb);
}
default:
- if((i = cmp(a, b)) != 0) {
+ if((i = CMP(a, b)) != 0) {
*done = 1;
}
return (i < 0) ? 1 : 0;
@@ -2924,7 +2900,7 @@ static int do_partly_bound_can_match_greater(Eterm a, Eterm b,
if (not_eq_tags(a,b)) {
*done = 1;
- return (cmp(a, b) > 0) ? 1 : 0;
+ return (CMP(a, b) > 0) ? 1 : 0;
}
/* we now know that tags are the same */
@@ -2960,7 +2936,7 @@ static int do_partly_bound_can_match_greater(Eterm a, Eterm b,
bb = list_val(*bb);
}
default:
- if((i = cmp(a, b)) != 0) {
+ if((i = CMP(a, b)) != 0) {
*done = 1;
}
return (i > 0) ? 1 : 0;
@@ -2983,16 +2959,16 @@ static int doit_select(DbTableTree *tb, TreeDbTerm *this, void *ptr,
if (sc->end_condition != NIL &&
((forward &&
cmp_partly_bound(sc->end_condition,
- GETKEY_WITH_POS(sc->keypos,
- this->dbterm.tpl)) < 0) ||
+ GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl),
+ this->dbterm.tpl) < 0) ||
(!forward &&
cmp_partly_bound(sc->end_condition,
- GETKEY_WITH_POS(sc->keypos,
- this->dbterm.tpl)) > 0))) {
+ GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl),
+ this->dbterm.tpl) > 0))) {
return 0;
}
- ret = db_prog_match_and_copy(&tb->common,sc->p,sc->mp,sc->all_objects,
- &this->dbterm, &hp, 2);
+ ret = db_match_dbterm(&tb->common,sc->p,sc->mp,sc->all_objects,
+ &this->dbterm, &hp, 2);
if (is_value(ret)) {
sc->accum = CONS(hp, ret, sc->accum);
}
@@ -3020,12 +2996,12 @@ static int doit_select_count(DbTableTree *tb, TreeDbTerm *this, void *ptr,
/* Always backwards traversing */
if (sc->end_condition != NIL &&
(cmp_partly_bound(sc->end_condition,
- GETKEY_WITH_POS(sc->keypos,
- this->dbterm.tpl)) > 0)) {
+ GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl),
+ this->dbterm.tpl) > 0)) {
return 0;
}
- ret = db_prog_match_and_copy(&tb->common, sc->p, sc->mp, 0,
- &this->dbterm, NULL, 0);
+ ret = db_match_dbterm(&tb->common, sc->p, sc->mp, 0,
+ &this->dbterm, NULL, 0);
if (ret == am_true) {
++(sc->got);
}
@@ -3047,17 +3023,17 @@ static int doit_select_chunk(DbTableTree *tb, TreeDbTerm *this, void *ptr,
if (sc->end_condition != NIL &&
((forward &&
cmp_partly_bound(sc->end_condition,
- GETKEY_WITH_POS(sc->keypos,
- this->dbterm.tpl)) < 0) ||
+ GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl),
+ this->dbterm.tpl) < 0) ||
(!forward &&
cmp_partly_bound(sc->end_condition,
- GETKEY_WITH_POS(sc->keypos,
- this->dbterm.tpl)) > 0))) {
+ GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl),
+ this->dbterm.tpl) > 0))) {
return 0;
}
- ret = db_prog_match_and_copy(&tb->common, sc->p, sc->mp, sc->all_objects,
- &this->dbterm, &hp, 2);
+ ret = db_match_dbterm(&tb->common, sc->p, sc->mp, sc->all_objects,
+ &this->dbterm, &hp, 2);
if (is_value(ret)) {
++(sc->got);
sc->accum = CONS(hp, ret, sc->accum);
@@ -3090,14 +3066,14 @@ static int doit_select_delete(DbTableTree *tb, TreeDbTerm *this, void *ptr,
if (sc->end_condition != NIL &&
cmp_partly_bound(sc->end_condition,
- GETKEY_WITH_POS(sc->keypos,
- this->dbterm.tpl)) > 0)
+ GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl),
+ this->dbterm.tpl) > 0)
return 0;
- ret = db_prog_match_and_copy(&tb->common, sc->p, sc->mp, 0,
- &this->dbterm, NULL, 0);
+ ret = db_match_dbterm(&tb->common, sc->p, sc->mp, 0,
+ &this->dbterm, NULL, 0);
if (ret == am_true) {
key = GETKEY(sc->tb, this->dbterm.tpl);
- linkout_tree(sc->tb, key);
+ linkout_tree(sc->tb, key, this->dbterm.tpl);
sc->erase_lastterm = 1;
++sc->accum;
}
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index 2852fb93fe..d3e31da413 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 1998-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1998-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -57,6 +57,7 @@
DBIF_TABLE_GUARD | DBIF_TABLE_BODY | DBIF_TRACE_GUARD | DBIF_TRACE_BODY
+#define HEAP_XTRA 100
/*
** Some convenience macros for stacks (DMC == db_match_compile)
@@ -229,6 +230,11 @@ typedef enum {
matchCall2,
matchCall3,
matchPushV,
+#if HALFWORD_HEAP
+ matchPushVGuard, /* First guard-only variable reference */
+#endif
+ matchPushVResult, /* First variable reference in result, or (if HALFWORD)
+ in guard if also referenced in result */
matchPushExpr, /* Push the whole expression we're matching ('$_') */
matchPushArrayAsList, /* Only when parameter is an Array and
not an erlang term (DCOMP_TRACE) */
@@ -292,11 +298,19 @@ DMC_DECLARE_STACK_TYPE(unsigned);
** Data about the heap during compilation
*/
+typedef struct DMCVariable {
+ int is_bound;
+ int is_in_body;
+#if HALFWORD_HEAP
+ int first_guard_label; /* to maybe change from PushVGuard to PushVResult */
+#endif
+} DMCVariable;
+
typedef struct DMCHeap {
int size;
- unsigned def[DMC_DEFAULT_SIZE];
- unsigned *data;
- int used;
+ DMCVariable vars_def[DMC_DEFAULT_SIZE];
+ DMCVariable* vars;
+ int vars_used;
} DMCHeap;
/*
@@ -323,7 +337,6 @@ typedef struct dmc_context {
Eterm *bodyexpr;
int num_match;
int current_match;
- int eheap_need;
Uint cflags;
int is_guard; /* 1 if in guard, 0 if in body */
int special; /* 1 if the head in the match was a single expression */
@@ -346,9 +359,22 @@ typedef struct dmc_context {
#define ERTS_DEFAULT_MS_HEAP_SIZE 128
+/* Runtime info about a $-variable
+*/
+typedef struct MatchVariable {
+ Eterm term;
+#ifdef DEBUG
+ Process* proc;
+ Eterm* base;
+#endif
+} MatchVariable;
+
typedef struct {
Process process;
- Eterm *heap;
+ union {
+ Eterm* heap;
+ MatchVariable* variables; /* first on "heap" */
+ }u;
Eterm default_heap[ERTS_DEFAULT_MS_HEAP_SIZE];
} ErtsMatchPseudoProcess;
@@ -371,10 +397,10 @@ cleanup_match_pseudo_process(ErtsMatchPseudoProcess *mpsp, int keep_heap)
}
#endif
if (!keep_heap) {
- if (mpsp->heap != &mpsp->default_heap[0]) {
+ if (mpsp->u.heap != mpsp->default_heap) {
/* Have to be done *after* call to erts_cleanup_empty_process() */
- erts_free(ERTS_ALC_T_DB_MS_RUN_HEAP, (void *) mpsp->heap);
- mpsp->heap = &mpsp->default_heap[0];
+ erts_free(ERTS_ALC_T_DB_MS_RUN_HEAP, (void *) mpsp->u.heap);
+ mpsp->u.heap = mpsp->default_heap;
}
#ifdef DEBUG
else {
@@ -398,7 +424,7 @@ create_match_pseudo_process(void)
mpsp = (ErtsMatchPseudoProcess *)erts_alloc(ERTS_ALC_T_DB_MS_PSDO_PROC,
sizeof(ErtsMatchPseudoProcess));
erts_init_empty_process(&mpsp->process);
- mpsp->heap = &mpsp->default_heap[0];
+ mpsp->u.heap = mpsp->default_heap;
return mpsp;
}
@@ -422,11 +448,11 @@ get_match_pseudo_process(Process *c_p, Uint heap_size)
mpsp = match_pseudo_process;
cleanup_match_pseudo_process(mpsp, 0);
#endif
- if (heap_size > ERTS_DEFAULT_MS_HEAP_SIZE)
- mpsp->heap = (Eterm *) erts_alloc(ERTS_ALC_T_DB_MS_RUN_HEAP,
- heap_size*sizeof(Uint));
+ if (heap_size > ERTS_DEFAULT_MS_HEAP_SIZE*sizeof(Eterm)) {
+ mpsp->u.heap = (Eterm*) erts_alloc(ERTS_ALC_T_DB_MS_RUN_HEAP, heap_size);
+ }
else {
- ASSERT(mpsp->heap == &mpsp->default_heap[0]);
+ ASSERT(mpsp->u.heap == mpsp->default_heap);
}
return mpsp;
}
@@ -467,23 +493,6 @@ erts_match_set_release_result(Process* c_p)
static erts_smp_atomic_t trace_control_word;
-
-Eterm
-erts_ets_copy_object(Eterm obj, Process* to)
-{
- Uint size = size_object(obj);
- Eterm* hp = HAlloc(to, size);
- Eterm res;
-
- res = copy_struct(obj, size, &hp, &MSO(to));
-#ifdef DEBUG
- if (eq(obj, res) == 0) {
- erl_exit(1, "copy not equal to source\n");
- }
-#endif
- return res;
-}
-
/* This needs to be here, before the bif table... */
static Eterm db_set_trace_control_word_fake_1(Process *p, Eterm val);
@@ -869,9 +878,9 @@ static DMCRet dmc_one_term(DMCContext *context,
#ifdef DMC_DEBUG
static int test_disassemble_next = 0;
-static void db_match_dis(Binary *prog);
+void db_match_dis(Binary *prog);
#define TRACE erts_fprintf(stderr,"Trace: %s:%d\n",__FILE__,__LINE__)
-#define FENCE_PATTERN_SIZE 1
+#define FENCE_PATTERN_SIZE (1*sizeof(Uint))
#define FENCE_PATTERN 0xDEADBEEFUL
#else
#define TRACE /* Nothing */
@@ -1179,14 +1188,14 @@ done:
}
Eterm erts_match_set_run(Process *p, Binary *mpsp,
- Eterm *args, int num_args,
+ Eterm *args, int num_args,
+ enum erts_pam_run_flags in_flags,
Uint32 *return_flags)
{
Eterm ret;
- ret = db_prog_match(p, mpsp,
- NIL, args,
- num_args, return_flags);
+ ret = db_prog_match(p, mpsp, NIL, NULL, args, num_args,
+ in_flags, return_flags);
#if defined(HARDDEBUG)
if (is_non_value(ret)) {
erts_fprintf(stderr, "Failed\n");
@@ -1210,9 +1219,9 @@ static Eterm erts_match_set_run_ets(Process *p, Binary *mpsp,
{
Eterm ret;
- ret = db_prog_match(p, mpsp,
- args, NULL,
- num_args, return_flags);
+ ret = db_prog_match(p, mpsp, args, NULL, NULL, num_args,
+ ERTS_PAM_CONTIGUOUS_TUPLE | ERTS_PAM_COPY_RESULT,
+ return_flags);
#if defined(HARDDEBUG)
if (is_non_value(ret)) {
erts_fprintf(stderr, "Failed\n");
@@ -1280,7 +1289,6 @@ Binary *db_match_compile(Eterm *matchexpr,
int structure_checked;
DMCRet res;
int current_try_label;
- Uint max_eheap_need;
Binary *bp = NULL;
unsigned clause_start;
@@ -1293,27 +1301,24 @@ Binary *db_match_compile(Eterm *matchexpr,
context.matchexpr = matchexpr;
context.guardexpr = guards;
context.bodyexpr = body;
- context.eheap_need = 0;
context.err_info = err_info;
context.cflags = flags;
heap.size = DMC_DEFAULT_SIZE;
- heap.data = heap.def;
+ heap.vars = heap.vars_def;
/*
** Compile the match expression
*/
restart:
- heap.used = 0;
- max_eheap_need = 0;
+ heap.vars_used = 0;
for (context.current_match = 0;
context.current_match < num_progs;
++context.current_match) { /* This loop is long,
too long */
- memset(heap.data, 0, heap.size * sizeof(*heap.data));
+ memset(heap.vars, 0, heap.size * sizeof(*heap.vars));
t = context.matchexpr[context.current_match];
context.stack_used = 0;
- context.eheap_need = 0;
structure_checked = 0;
if (context.current_match < num_progs - 1) {
DMC_PUSH(text,matchTryMeElse);
@@ -1485,10 +1490,6 @@ restart:
if (current_try_label >= 0) {
DMC_POKE(text, current_try_label, DMC_STACK_NUM(text));
}
- /* So, how much eheap did this part of the match program need? */
- if (context.eheap_need > max_eheap_need) {
- max_eheap_need = context.eheap_need;
- }
} /* for (context.current_match = 0 ...) */
@@ -1524,16 +1525,13 @@ restart:
ret->saved_program_buf = NULL;
ret->saved_program = NIL;
ret->term_save = context.save;
- ret->num_bindings = heap.used;
+ ret->num_bindings = heap.vars_used;
ret->single_variable = context.special;
sys_memcpy(ret->text, DMC_STACK_DATA(text),
DMC_STACK_NUM(text) * sizeof(UWord));
- ret->heap_size = ((heap.used * sizeof(Eterm)) +
- (max_eheap_need * sizeof(Eterm)) +
- (context.stack_need * sizeof(Eterm *)) +
- (3 * (FENCE_PATTERN_SIZE * sizeof(Eterm *))));
- ret->eheap_offset = heap.used + FENCE_PATTERN_SIZE;
- ret->stack_offset = ret->eheap_offset + max_eheap_need + FENCE_PATTERN_SIZE;
+ ret->stack_offset = heap.vars_used*sizeof(MatchVariable) + FENCE_PATTERN_SIZE;
+ ret->heap_size = ret->stack_offset + context.stack_need * sizeof(Eterm*) + FENCE_PATTERN_SIZE;
+
#ifdef DMC_DEBUG
ret->prog_end = ret->text + DMC_STACK_NUM(text);
#endif
@@ -1551,8 +1549,8 @@ error: /* Here is were we land when compilation failed. */
DMC_FREE(text);
if (context.copy != NULL)
free_message_buffer(context.copy);
- if (heap.data != heap.def)
- erts_free(ERTS_ALC_T_DB_MS_CMPL_HEAP, (void *) heap.data);
+ if (heap.vars != heap.vars_def)
+ erts_free(ERTS_ALC_T_DB_MS_CMPL_HEAP, (void *) heap.vars);
return bp;
}
@@ -1597,7 +1595,7 @@ erts_match_prog_foreach_offheap(Binary *bprog,
*/
static Eterm dpm_array_to_list(Process *psp, Eterm *arr, int arity)
{
- Eterm *hp = HAlloc(psp, arity * 2);
+ Eterm *hp = HAllocX(psp, arity * 2, HEAP_XTRA);
Eterm ret = NIL;
while (--arity >= 0) {
ret = CONS(hp, arr[arity], ret);
@@ -1606,15 +1604,82 @@ static Eterm dpm_array_to_list(Process *psp, Eterm *arr, int arity)
return ret;
}
+
+#if HALFWORD_HEAP
+struct heap_checkpoint_t
+{
+ Process *p;
+ Eterm* htop;
+ ErlHeapFragment* mbuf;
+ unsigned used_size;
+ ErlOffHeap off_heap;
+};
+
+static void heap_checkpoint_init(Process* p, struct heap_checkpoint_t* hcp)
+{
+ hcp->p = p;
+ hcp->htop = HEAP_TOP(p);
+ hcp->mbuf = MBUF(p);
+ hcp->used_size = hcp->mbuf ? hcp->mbuf->used_size : 0;
+ hcp->off_heap = MSO(p);
+}
+
+static void heap_checkpoint_revert(struct heap_checkpoint_t* hcp)
+{
+ struct erl_off_heap_header* oh = MSO(hcp->p).first;
+
+ if (oh != hcp->off_heap.first) {
+ ASSERT(oh != NULL);
+ if (hcp->off_heap.first) {
+ while (oh->next != hcp->off_heap.first) {
+ oh = oh->next;
+ }
+ oh->next = NULL;
+ }
+ erts_cleanup_offheap(&MSO(hcp->p));
+ MSO(hcp->p) = hcp->off_heap;
+ }
+ if (MBUF(hcp->p) != hcp->mbuf) {
+ ErlHeapFragment* hf = MBUF(hcp->p);
+ ASSERT(hf != NULL);
+ if (hcp->mbuf) {
+ while (hf->next != hcp->mbuf) {
+ hf = hf->next;
+ }
+ hf->next = NULL;
+ }
+ free_message_buffer(MBUF(hcp->p));
+ MBUF(hcp->p) = hcp->mbuf;
+ }
+ if (hcp->mbuf != NULL && hcp->mbuf->used_size != hcp->used_size) {
+ hcp->mbuf->used_size = hcp->used_size;
+ }
+ HEAP_TOP(hcp->p) = hcp->htop;
+}
+#endif /* HALFWORD_HEAP */
+
+static ERTS_INLINE Eterm copy_object_rel(Process* p, Eterm term, Eterm* base)
+{
+ if (!is_immed(term)) {
+ Uint sz = size_object_rel(term, base);
+ Eterm* top = HAllocX(p, sz, HEAP_XTRA);
+ return copy_struct_rel(term, sz, &top, &MSO(p), base, NULL);
+ }
+ return term;
+}
+
+
/*
** Execution of the match program, this is Pam.
** May return THE_NON_VALUE, which is a bailout.
-** the para meter 'arity' is only used if 'term' is actually an array,
+** the parameter 'arity' is only used if 'term' is actually an array,
** i.e. 'DCOMP_TRACE' was specified
*/
-Eterm db_prog_match(Process *c_p, Binary *bprog, Eterm term,
+Eterm db_prog_match(Process *c_p, Binary *bprog,
+ Eterm term, Eterm* base,
Eterm *termp,
int arity,
+ enum erts_pam_run_flags in_flags,
Uint32 *return_flags)
{
MatchProg *prog = Binary2MatchProg(bprog);
@@ -1623,7 +1688,7 @@ Eterm db_prog_match(Process *c_p, Binary *bprog, Eterm term,
Eterm t;
Eterm **sp;
Eterm *esp;
- Eterm *hp;
+ MatchVariable* variables;
BeamInstr *cp;
UWord *pc = prog->text;
Eterm *ehp;
@@ -1633,19 +1698,24 @@ Eterm db_prog_match(Process *c_p, Binary *bprog, Eterm term,
unsigned do_catch;
ErtsMatchPseudoProcess *mpsp;
Process *psp;
+ Process* build_proc;
Process *tmpp;
Process *current_scheduled;
ErtsSchedulerData *esdp;
Eterm (*bif)(Process*, ...);
int fail_label;
int atomic_trace;
+#if HALFWORD_HEAP
+ struct heap_checkpoint_t c_p_checkpoint = {};
+#endif
#ifdef DMC_DEBUG
Uint *heap_fence;
- Uint *eheap_fence;
Uint *stack_fence;
Uint save_op;
#endif /* DMC_DEBUG */
+ ASSERT(base==NULL || HALFWORD_HEAP);
+
mpsp = get_match_pseudo_process(c_p, prog->heap_size);
psp = &mpsp->process;
@@ -1655,7 +1725,6 @@ Eterm db_prog_match(Process *c_p, Binary *bprog, Eterm term,
esdp = ERTS_GET_SCHEDULER_DATA_FROM_PROC(c_p);
ASSERT(esdp != NULL);
current_scheduled = esdp->current_process;
- esdp->current_process = psp;
/* SMP: psp->scheduler_data is set by get_match_pseudo_process */
atomic_trace = 0;
@@ -1678,11 +1747,9 @@ Eterm db_prog_match(Process *c_p, Binary *bprog, Eterm term,
#ifdef DMC_DEBUG
save_op = 0;
- heap_fence = (Uint *) mpsp->heap + prog->eheap_offset - 1;
- eheap_fence = (Uint *) mpsp->heap + prog->stack_offset - 1;
- stack_fence = (Uint *) mpsp->heap + prog->heap_size - 1;
+ heap_fence = (Eterm*)((char*) mpsp->u.heap + prog->stack_offset) - 1;
+ stack_fence = (Eterm*)((char*) mpsp->u.heap + prog->heap_size) - 1;
*heap_fence = FENCE_PATTERN;
- *eheap_fence = FENCE_PATTERN;
*stack_fence = FENCE_PATTERN;
#endif /* DMC_DEBUG */
@@ -1696,36 +1763,48 @@ Eterm db_prog_match(Process *c_p, Binary *bprog, Eterm term,
*return_flags = 0U;
+ variables = mpsp->u.variables;
+#if HALFWORD_HEAP
+ c_p_checkpoint.p = NULL;
+#endif
+
restart:
ep = &term;
- esp = mpsp->heap + prog->stack_offset;
+ esp = (Eterm*)((char*)mpsp->u.heap + prog->stack_offset);
sp = (Eterm **) esp;
- hp = mpsp->heap;
- ehp = mpsp->heap + prog->eheap_offset;
ret = am_true;
do_catch = 0;
fail_label = -1;
+ build_proc = psp;
+ esdp->current_process = psp;
+ ASSERT_HALFWORD(!c_p_checkpoint.p);
+
+#ifdef DEBUG
+ ASSERT(variables == mpsp->u.variables);
+ for (i=0; i<prog->num_bindings; i++) {
+ variables[i].term = THE_NON_VALUE;
+ variables[i].proc = NULL;
+ variables[i].base = base;
+ }
+#endif
for (;;) {
-#ifdef DMC_DEBUG
+
+ #ifdef DMC_DEBUG
if (*heap_fence != FENCE_PATTERN) {
erl_exit(1, "Heap fence overwritten in db_prog_match after op "
"0x%08x, overwritten with 0x%08x.", save_op, *heap_fence);
}
- if (*eheap_fence != FENCE_PATTERN) {
- erl_exit(1, "Eheap fence overwritten in db_prog_match after op "
- "0x%08x, overwritten with 0x%08x.", save_op,
- *eheap_fence);
- }
if (*stack_fence != FENCE_PATTERN) {
erl_exit(1, "Stack fence overwritten in db_prog_match after op "
"0x%08x, overwritten with 0x%08x.", save_op,
*stack_fence);
}
save_op = *pc;
-#endif
+ #endif
switch (*pc++) {
case matchTryMeElse:
+ ASSERT(fail_label == -1);
fail_label = *pc++;
break;
case matchArray: /* only when DCOMP_TRACE, is always first
@@ -1736,13 +1815,14 @@ restart:
ep = termp;
break;
case matchArrayBind: /* When the array size is unknown. */
+ ASSERT(termp);
n = *pc++;
- hp[n] = dpm_array_to_list(psp, termp, arity);
+ variables[n].term = dpm_array_to_list(psp, termp, arity);
break;
case matchTuple: /* *ep is a tuple of arity n */
- if (!is_tuple(*ep))
+ if (!is_tuple_rel(*ep,base))
FAIL();
- ep = tuple_val(*ep);
+ ep = tuple_val_rel(*ep,base);
n = *pc++;
if (arityval(*ep) != n)
FAIL();
@@ -1750,9 +1830,9 @@ restart:
break;
case matchPushT: /* *ep is a tuple of arity n,
push ptr to first element */
- if (!is_tuple(*ep))
+ if (!is_tuple_rel(*ep,base))
FAIL();
- tp = tuple_val(*ep);
+ tp = tuple_val_rel(*ep,base);
n = *pc++;
if (arityval(*tp) != n)
FAIL();
@@ -1762,12 +1842,12 @@ restart:
case matchList:
if (!is_list(*ep))
FAIL();
- ep = list_val(*ep);
+ ep = list_val_rel(*ep,base);
break;
case matchPushL:
if (!is_list(*ep))
FAIL();
- *sp++ = list_val(*ep);
+ *sp++ = list_val_rel(*ep,base);
++ep;
break;
case matchPop:
@@ -1775,41 +1855,44 @@ restart:
break;
case matchBind:
n = *pc++;
- hp[n] = *ep++;
+ variables[n].term = *ep++;
break;
case matchCmp:
n = *pc++;
- if (!eq(hp[n],*ep))
+ if (!eq_rel(variables[n].term, base, *ep, base))
FAIL();
++ep;
break;
case matchEqBin:
t = (Eterm) *pc++;
- if (!eq(*ep,t))
+ if (!eq_rel(t,NULL,*ep,base))
FAIL();
++ep;
break;
case matchEqFloat:
- if (!is_float(*ep))
+ if (!is_float_rel(*ep,base))
FAIL();
- if (memcmp(float_val(*ep) + 1, pc, sizeof(double)))
+ if (memcmp(float_val_rel(*ep,base) + 1, pc, sizeof(double)))
FAIL();
pc += TermWords(2);
++ep;
break;
- case matchEqRef:
- if (!is_ref(*ep))
+ case matchEqRef: {
+ Eterm* epc = (Eterm*)pc;
+ if (!is_ref_rel(*ep,base))
FAIL();
- if (!eq(*ep, make_internal_ref((Uint *) pc)))
+ if (!eq_rel(make_internal_ref_rel(epc, epc), epc, *ep, base)) {
FAIL();
- i = thing_arityval(*((Uint *) pc));
+ }
+ i = thing_arityval(*epc);
pc += TermWords(i+1);
++ep;
break;
+ }
case matchEqBig:
- if (!is_big(*ep))
+ if (!is_big_rel(*ep,base))
FAIL();
- tp = big_val(*ep);
+ tp = big_val_rel(*ep,base);
{
Eterm *epc = (Eterm *) pc;
if (*tp != *epc)
@@ -1825,7 +1908,8 @@ restart:
++ep;
break;
case matchEq:
- t = (Eterm) *pc++;
+ t = (Eterm) *pc++;
+ ASSERT(is_immed(t));
if (t != *ep++)
FAIL();
break;
@@ -1833,25 +1917,32 @@ restart:
++ep;
break;
/*
- * Here comes guard instructions
+ * Here comes guard & body instructions
*/
case matchPushC: /* Push constant */
- *esp++ = *pc++;
+ if ((in_flags & ERTS_PAM_COPY_RESULT)
+ && do_catch && !is_immed(*pc)) {
+ *esp++ = copy_object(*pc++, c_p);
+ }
+ else {
+ *esp++ = *pc++;
+ }
break;
case matchConsA:
- ehp[1] = *--esp;
- ehp[0] = esp[-1];
+ ehp = HAllocX(build_proc, 2, HEAP_XTRA);
+ CDR(ehp) = *--esp;
+ CAR(ehp) = esp[-1];
esp[-1] = make_list(ehp);
- ehp += 2;
break;
case matchConsB:
- ehp[0] = *--esp;
- ehp[1] = esp[-1];
+ ehp = HAllocX(build_proc, 2, HEAP_XTRA);
+ CAR(ehp) = *--esp;
+ CDR(ehp) = esp[-1];
esp[-1] = make_list(ehp);
- ehp += 2;
break;
case matchMkTuple:
n = *pc++;
+ ehp = HAllocX(build_proc, n+1, HEAP_XTRA);
t = make_tuple(ehp);
*ehp++ = make_arityval(n);
while (n--) {
@@ -1861,7 +1952,7 @@ restart:
break;
case matchCall0:
bif = (Eterm (*)(Process*, ...)) *pc++;
- t = (*bif)(psp);
+ t = (*bif)(build_proc);
if (is_non_value(t)) {
if (do_catch)
t = FAIL_TERM;
@@ -1872,7 +1963,7 @@ restart:
break;
case matchCall1:
bif = (Eterm (*)(Process*, ...)) *pc++;
- t = (*bif)(psp, esp[-1]);
+ t = (*bif)(build_proc, esp[-1]);
if (is_non_value(t)) {
if (do_catch)
t = FAIL_TERM;
@@ -1883,7 +1974,7 @@ restart:
break;
case matchCall2:
bif = (Eterm (*)(Process*, ...)) *pc++;
- t = (*bif)(psp, esp[-1], esp[-2]);
+ t = (*bif)(build_proc, esp[-1], esp[-2]);
if (is_non_value(t)) {
if (do_catch)
t = FAIL_TERM;
@@ -1895,7 +1986,7 @@ restart:
break;
case matchCall3:
bif = (Eterm (*)(Process*, ...)) *pc++;
- t = (*bif)(psp, esp[-1], esp[-2], esp[-3]);
+ t = (*bif)(build_proc, esp[-1], esp[-2], esp[-3]);
if (is_non_value(t)) {
if (do_catch)
t = FAIL_TERM;
@@ -1905,15 +1996,73 @@ restart:
esp -= 2;
esp[-1] = t;
break;
+
+ #if HALFWORD_HEAP
+ case matchPushVGuard:
+ if (!base) goto case_matchPushV;
+ /* Build NULL-based copy on pseudo heap for easy disposal */
+ n = *pc++;
+ ASSERT(is_value(variables[n].term));
+ ASSERT(!variables[n].proc);
+ variables[n].term = copy_object_rel(psp, variables[n].term, base);
+ *esp++ = variables[n].term;
+ #ifdef DEBUG
+ variables[n].proc = psp;
+ variables[n].base = NULL;
+ #endif
+ break;
+ #endif
+ case matchPushVResult:
+ if (!(in_flags & ERTS_PAM_COPY_RESULT)) goto case_matchPushV;
+
+ /* Build (NULL-based) copy on callers heap */
+ #if HALFWORD_HEAP
+ if (!do_catch && !c_p_checkpoint.p) {
+ heap_checkpoint_init(c_p, &c_p_checkpoint);
+ }
+ #endif
+ n = *pc++;
+ ASSERT(is_value(variables[n].term));
+ ASSERT(!variables[n].proc);
+ variables[n].term = copy_object_rel(c_p, variables[n].term, base);
+ *esp++ = variables[n].term;
+ #ifdef DEBUG
+ variables[n].proc = c_p;
+ variables[n].base = NULL;
+ #endif
+ break;
case matchPushV:
- *esp++ = hp[*pc++];
+ case_matchPushV:
+ n = *pc++;
+ ASSERT(is_value(variables[n].term));
+ ASSERT(!variables[n].base);
+ *esp++ = variables[n].term;
break;
case matchPushExpr:
- *esp++ = term;
+ if (in_flags & ERTS_PAM_COPY_RESULT) {
+ Uint sz;
+ Eterm* top;
+ sz = size_object_rel(term, base);
+ top = HAllocX(build_proc, sz, HEAP_XTRA);
+ if (in_flags & ERTS_PAM_CONTIGUOUS_TUPLE) {
+ ASSERT(is_tuple_rel(term,base));
+ *esp++ = copy_shallow_rel(tuple_val_rel(term,base), sz,
+ &top, &MSO(build_proc), base);
+ }
+ else {
+ *esp++ = copy_struct_rel(term, sz, &top, &MSO(build_proc),
+ base, NULL);
+ }
+ }
+ else {
+ *esp = term;
+ }
break;
case matchPushArrayAsList:
+ ASSERT_HALFWORD(base == NULL);
n = arity; /* Only happens when 'term' is an array */
tp = termp;
+ ehp = HAllocX(build_proc, n*2, HEAP_XTRA);
*esp++ = make_list(ehp);
while (n--) {
*ehp++ = *tp++;
@@ -1926,7 +2075,8 @@ restart:
break;
case matchPushArrayAsListU:
/* This instruction is NOT efficient. */
- *esp++ = dpm_array_to_list(psp, termp, arity);
+ ASSERT_HALFWORD(base == NULL);
+ *esp++ = dpm_array_to_list(build_proc, termp, arity);
break;
case matchTrue:
if (*--esp != am_true)
@@ -2012,7 +2162,7 @@ restart:
case matchProcessDump: {
erts_dsprintf_buf_t *dsbufp = erts_create_tmp_dsbuf(0);
print_process_info(ERTS_PRINT_DSBUF, (void *) dsbufp, c_p);
- *esp++ = new_binary(psp, (byte *)dsbufp->str, (int)dsbufp->str_len);
+ *esp++ = new_binary(build_proc, (byte *)dsbufp->str, (int)dsbufp->str_len);
erts_destroy_tmp_dsbuf(dsbufp);
break;
}
@@ -2056,29 +2206,24 @@ restart:
if (SEQ_TRACE_TOKEN(c_p) == NIL)
*esp++ = NIL;
else {
+ Eterm sender = SEQ_TRACE_TOKEN_SENDER(c_p);
+ Uint sender_sz = is_immed(sender) ? 0 : size_object(sender);
+ ehp = HAllocX(build_proc, 6 + sender_sz, HEAP_XTRA);
+ if (sender_sz) {
+ sender = copy_struct(sender, sender_sz, &ehp, &MSO(build_proc));
+ }
*esp++ = make_tuple(ehp);
ehp[0] = make_arityval(5);
ehp[1] = SEQ_TRACE_TOKEN_FLAGS(c_p);
ehp[2] = SEQ_TRACE_TOKEN_LABEL(c_p);
ehp[3] = SEQ_TRACE_TOKEN_SERIAL(c_p);
- ehp[4] = SEQ_TRACE_TOKEN_SENDER(c_p);
+ ehp[4] = sender;
ehp[5] = SEQ_TRACE_TOKEN_LASTCNT(c_p);
ASSERT(SEQ_TRACE_TOKEN_ARITY(c_p) == 5);
ASSERT(is_immed(ehp[1]));
ASSERT(is_immed(ehp[2]));
ASSERT(is_immed(ehp[3]));
ASSERT(is_immed(ehp[5]));
- if(!is_immed(ehp[4])) {
- Eterm *sender = &ehp[4];
- ehp += 6;
- *sender = copy_struct(*sender,
- size_object(*sender),
- &ehp,
- &MSO(psp));
- }
- else
- ehp += 6;
-
}
break;
case matchEnableTrace:
@@ -2127,12 +2272,12 @@ restart:
if (!(c_p->cp) || !(cp = find_function_from_pc(c_p->cp))) {
*esp++ = am_undefined;
} else {
+ ehp = HAllocX(build_proc, 4, HEAP_XTRA);
*esp++ = make_tuple(ehp);
ehp[0] = make_arityval(3);
ehp[1] = cp[0];
ehp[2] = cp[1];
ehp[3] = make_small((Uint) cp[2]);
- ehp += 4;
}
break;
case matchSilent:
@@ -2209,8 +2354,12 @@ restart:
}
}
break;
- case matchCatch:
+ case matchCatch: /* Match success, now build result */
do_catch = 1;
+ if (in_flags & ERTS_PAM_COPY_RESULT) {
+ build_proc = c_p;
+ esdp->current_process = c_p;
+ }
break;
case matchHalt:
goto success;
@@ -2219,9 +2368,16 @@ restart:
}
}
fail:
+#if HALFWORD_HEAP
+ if (c_p_checkpoint.p) {
+ /* Dispose garbage built by guards on caller heap */
+ heap_checkpoint_revert(&c_p_checkpoint);
+ c_p_checkpoint.p = NULL;
+ }
+#endif
*return_flags = 0U;
- if (fail_label >= 0) { /* We failed during a "TryMeElse",
- lets restart, with the next match
+ if (fail_label >= 0) { /* We failed during a "TryMeElse",
+ lets restart, with the next match
program */
pc = (prog->text) + fail_label;
cleanup_match_pseudo_process(mpsp, 1);
@@ -2235,11 +2391,6 @@ success:
erl_exit(1, "Heap fence overwritten in db_prog_match after op "
"0x%08x, overwritten with 0x%08x.", save_op, *heap_fence);
}
- if (*eheap_fence != FENCE_PATTERN) {
- erl_exit(1, "Eheap fence overwritten in db_prog_match after op "
- "0x%08x, overwritten with 0x%08x.", save_op,
- *eheap_fence);
- }
if (*stack_fence != FENCE_PATTERN) {
erl_exit(1, "Stack fence overwritten in db_prog_match after op "
"0x%08x, overwritten with 0x%08x.", save_op,
@@ -2250,6 +2401,7 @@ success:
esdp->current_process = current_scheduled;
END_ATOMIC_TRACE(c_p);
+
return ret;
#undef FAIL
#undef FAIL_TERM
@@ -2261,7 +2413,8 @@ success:
/*
* Convert a match program to a "magic" binary to return up to erlang
*/
-Eterm db_make_mp_binary(Process *p, Binary *mp, Eterm **hpp) {
+Eterm db_make_mp_binary(Process *p, Binary *mp, Eterm **hpp)
+{
return erts_mk_magic_binary_term(hpp, &MSO(p), mp);
}
@@ -2327,13 +2480,13 @@ void db_free_dmc_err_info(DMCErrInfo *ei){
** Store bignum in *hpp and increase *hpp accordingly.
** *hpp is assumed to be large enough to hold the result.
*/
-Eterm db_add_counter(Eterm** hpp, Eterm counter, Eterm incr)
+Eterm db_add_counter(Eterm** hpp, Wterm counter, Eterm incr)
{
DeclareTmpHeapNoproc(big_tmp,2);
Eterm res;
Sint ires;
- Eterm arg1;
- Eterm arg2;
+ Wterm arg1;
+ Wterm arg2;
if (is_both_small(counter,incr)) {
ires = signed_val(counter) + signed_val(incr);
@@ -2374,6 +2527,34 @@ Eterm db_add_counter(Eterm** hpp, Eterm counter, Eterm incr)
}
}
+/* Must be called to read elements after db_lookup_dbterm.
+** Will decompress if needed.
+** HEALFWORD_HEAP:
+** Will convert from relative to Wterm format if needed.
+** (but only on top level, tuples and lists will still contain rterms)
+*/
+Wterm db_do_read_element(DbUpdateHandle* handle, Sint position)
+{
+ Eterm elem = handle->dbterm->tpl[position];
+ if (!is_header(elem)) {
+#if HALFWORD_HEAP
+ if (!is_immed(elem)
+ && !handle->tb->common.compress
+ && !(handle->abs_vec && handle->abs_vec[position])) {
+ return rterm2wterm(elem, handle->dbterm->tpl);
+ }
+#endif
+ return elem;
+ }
+
+ ASSERT(((DbTableCommon*)handle->tb)->compress);
+ ASSERT(!handle->mustResize);
+ handle->dbterm = db_alloc_tmp_uncompressed(&handle->tb->common,
+ handle->dbterm);
+ handle->mustResize = 1;
+ return handle->dbterm->tpl[position];
+}
+
/*
** Update one element:
** handle: Initialized by db_lookup_dbterm()
@@ -2390,14 +2571,17 @@ void db_do_update_element(DbUpdateHandle* handle,
Eterm* oldp;
Uint newval_sz;
Uint oldval_sz;
+#if HALFWORD_HEAP
+ Eterm* old_base;
+#endif
if (is_both_immed(newval,oldval)) {
handle->dbterm->tpl[position] = newval;
-#ifdef DEBUG_CLONE
+ #ifdef DEBUG_CLONE
if (handle->dbterm->debug_clone) {
handle->dbterm->debug_clone[position] = newval;
}
-#endif
+ #endif
return;
}
if (!handle->mustResize) {
@@ -2406,49 +2590,78 @@ void db_do_update_element(DbUpdateHandle* handle,
handle->dbterm);
handle->mustResize = 1;
oldval = handle->dbterm->tpl[position];
+ #if HALFWORD_HEAP
+ old_base = NULL;
+ #endif
}
- else if (is_boxed(newval)) {
- newp = boxed_val(newval);
- switch (*newp & _TAG_HEADER_MASK) {
- case _TAG_HEADER_POS_BIG:
- case _TAG_HEADER_NEG_BIG:
- case _TAG_HEADER_FLOAT:
- case _TAG_HEADER_HEAP_BIN:
- newval_sz = header_arity(*newp) + 1;
- if (is_boxed(oldval)) {
- oldp = boxed_val(oldval);
- switch (*oldp & _TAG_HEADER_MASK) {
- case _TAG_HEADER_POS_BIG:
- case _TAG_HEADER_NEG_BIG:
- case _TAG_HEADER_FLOAT:
- case _TAG_HEADER_HEAP_BIN:
- oldval_sz = header_arity(*oldp) + 1;
- if (oldval_sz == newval_sz) {
- /* "self contained" terms of same size, do memcpy */
- sys_memcpy(oldp, newp, newval_sz*sizeof(Eterm));
- return;
+ else {
+ #if HALFWORD_HEAP
+ ASSERT(!handle->abs_vec);
+ old_base = handle->dbterm->tpl;
+ #endif
+ if (is_boxed(newval)) {
+ newp = boxed_val(newval);
+ switch (*newp & _TAG_HEADER_MASK) {
+ case _TAG_HEADER_POS_BIG:
+ case _TAG_HEADER_NEG_BIG:
+ case _TAG_HEADER_FLOAT:
+ case _TAG_HEADER_HEAP_BIN:
+ newval_sz = header_arity(*newp) + 1;
+ if (is_boxed(oldval)) {
+ oldp = boxed_val_rel(oldval,old_base);
+ switch (*oldp & _TAG_HEADER_MASK) {
+ case _TAG_HEADER_POS_BIG:
+ case _TAG_HEADER_NEG_BIG:
+ case _TAG_HEADER_FLOAT:
+ case _TAG_HEADER_HEAP_BIN:
+ oldval_sz = header_arity(*oldp) + 1;
+ if (oldval_sz == newval_sz) {
+ /* "self contained" terms of same size, do memcpy */
+ sys_memcpy(oldp, newp, newval_sz*sizeof(Eterm));
+ return;
+ }
+ goto both_size_set;
}
- goto both_size_set;
}
+ goto new_size_set;
}
- goto new_size_set;
}
}
}
+#if HALFWORD_HEAP
+ else {
+ old_base = (handle->tb->common.compress
+ || (handle->abs_vec && handle->abs_vec[position])) ?
+ NULL : handle->dbterm->tpl;
+ }
+#endif
/* Not possible for simple memcpy or dbterm is already non-contiguous, */
/* need to realloc... */
newval_sz = is_immed(newval) ? 0 : size_object(newval);
new_size_set:
-
- oldval_sz = is_immed(oldval) ? 0 : size_object(oldval);
+
+ oldval_sz = is_immed(oldval) ? 0 : size_object_rel(oldval,old_base);
both_size_set:
handle->new_size = handle->new_size - oldval_sz + newval_sz;
- /* write new value in old dbterm, finalize will make a flat copy */
+ /* write new value in old dbterm, finalize will make a flat copy */
handle->dbterm->tpl[position] = newval;
handle->mustResize = 1;
+
+#if HALFWORD_HEAP
+ if (old_base && newval_sz > 0) {
+ ASSERT(!handle->tb->common.compress);
+ if (!handle->abs_vec) {
+ int i = header_arity(handle->dbterm->tpl[0]);
+ handle->abs_vec = erts_alloc(ERTS_ALC_T_TMP, (i+1)*sizeof(char));
+ sys_memset(handle->abs_vec, 0, i+1);
+ /* abs_vec[0] not used */
+ }
+ handle->abs_vec[position] = 1;
+ }
+#endif
}
static ERTS_INLINE byte* db_realloc_term(DbTableCommon* tb, void* old,
@@ -2547,7 +2760,7 @@ static void* copy_to_comp(DbTableCommon* tb, Eterm obj, DbTerm* dest,
tpl[arity + 1] = alloc_size;
tmp_offheap.first = NULL;
- tpl[tb->keypos] = copy_struct(key, size_object(key), &top.ep, &tmp_offheap);
+ tpl[tb->keypos] = copy_struct_rel(key, size_object(key), &top.ep, &tmp_offheap, NULL, tpl);
dest->first_oh = tmp_offheap.first;
for (i=1; i<=arity; i++) {
if (i != tb->keypos) {
@@ -2566,7 +2779,7 @@ static void* copy_to_comp(DbTableCommon* tb, Eterm obj, DbTerm* dest,
Eterm* dbg_top = erts_alloc(ERTS_ALC_T_DB_TERM, dest->size * sizeof(Eterm));
dest->debug_clone = dbg_top;
tmp_offheap.first = dest->first_oh;
- copy_struct(obj, dest->size, &dbg_top, &tmp_offheap);
+ copy_struct_rel(obj, dest->size, &dbg_top, &tmp_offheap, NULL, dbg_top);
dest->first_oh = tmp_offheap.first;
ASSERT(dbg_top == dest->debug_clone + dest->size);
}
@@ -2613,7 +2826,7 @@ void* db_store_term(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj)
newp->size = size;
top = newp->tpl;
tmp_offheap.first = NULL;
- copy_struct(obj, size, &top, &tmp_offheap);
+ copy_struct_rel(obj, size, &top, &tmp_offheap, NULL, top);
newp->first_oh = tmp_offheap.first;
#ifdef DEBUG_CLONE
newp->debug_clone = NULL;
@@ -2652,7 +2865,7 @@ void* db_store_term_comp(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj)
top = copy_to_comp(tb, obj, newp, new_sz);
ASSERT(top <= basep + new_sz);
- // SVERK: realloc?
+ /* ToDo: Maybe realloc if ((basep+new_sz) - top) > WASTED_SPACE_LIMIT */
return basep;
}
@@ -2673,6 +2886,9 @@ void db_finalize_resize(DbUpdateHandle* handle, Uint offset)
*(handle->bp) = newp;
newDbTerm = (DbTerm*) (newp + offset);
newDbTerm->size = handle->new_size;
+#ifdef DEBUG_CLONE
+ newDbTerm->debug_clone = NULL;
+#endif
/* make a flat copy */
@@ -2682,17 +2898,38 @@ void db_finalize_resize(DbUpdateHandle* handle, Uint offset)
db_free_tmp_uncompressed(handle->dbterm);
}
else {
- Eterm* top;
ErlOffHeap tmp_offheap;
+ Eterm* tpl = handle->dbterm->tpl;
+ Eterm* top = newDbTerm->tpl;
+
tmp_offheap.first = NULL;
- top = newDbTerm->tpl;
- copy_struct(make_tuple(handle->dbterm->tpl), handle->new_size,
- &top, &tmp_offheap);
- newDbTerm->first_oh = tmp_offheap.first;
-#ifdef DEBUG_CLONE
- newDbTerm->debug_clone = NULL;
-#endif
- ASSERT((byte*)top <= (newp + alloc_sz));
+
+ #if HALFWORD_HEAP
+ if (handle->abs_vec) {
+ int i, arity = header_arity(handle->dbterm->tpl[0]);
+
+ top[0] = tpl[0];
+ top += arity + 1;
+ for (i=1; i<=arity; i++) {
+ Eterm* src_base = handle->abs_vec[i] ? NULL : tpl;
+
+ newDbTerm->tpl[i] = copy_struct_rel(tpl[i],
+ size_object_rel(tpl[i],src_base),
+ &top, &tmp_offheap, src_base,
+ newDbTerm->tpl);
+ }
+ newDbTerm->first_oh = tmp_offheap.first;
+ ASSERT((byte*)top <= (newp + alloc_sz));
+ erts_free(ERTS_ALC_T_TMP, handle->abs_vec);
+ }
+ else
+ #endif /* HALFWORD_HEAP */
+ {
+ copy_struct_rel(make_tuple_rel(tpl,tpl), handle->new_size, &top,
+ &tmp_offheap, tpl, top);
+ newDbTerm->first_oh = tmp_offheap.first;
+ ASSERT((byte*)top == (newp + alloc_sz));
+ }
}
}
@@ -2705,9 +2942,9 @@ Eterm db_copy_from_comp(DbTableCommon* tb, DbTerm* bp, Eterm** hpp,
hp[0] = bp->tpl[0];
*hpp += arity + 1;
- hp[tb->keypos] = copy_struct(bp->tpl[tb->keypos],
- size_object(bp->tpl[tb->keypos]),
- hpp, off_heap);
+ hp[tb->keypos] = copy_struct_rel(bp->tpl[tb->keypos],
+ size_object_rel(bp->tpl[tb->keypos], bp->tpl),
+ hpp, off_heap, bp->tpl, NULL);
for (i=arity; i>0; i--) {
if (i != tb->keypos) {
if (is_immed(bp->tpl[i])) {
@@ -2721,7 +2958,7 @@ Eterm db_copy_from_comp(DbTableCommon* tb, DbTerm* bp, Eterm** hpp,
}
ASSERT((*hpp - hp) <= bp->size);
#ifdef DEBUG_CLONE
- ASSERT(eq(make_tuple(hp),make_tuple(bp->debug_clone)));
+ ASSERT(eq_rel(make_tuple(hp),make_tuple(bp->debug_clone),bp->debug_clone));
#endif
return make_tuple(hp);
}
@@ -2744,14 +2981,14 @@ Eterm db_copy_element_from_ets(DbTableCommon* tb, Process* p,
hp += extra;
HRelease(p, endp, hp);
#ifdef DEBUG_CLONE
- ASSERT(eq(copy,obj->debug_clone[pos]));
+ ASSERT(eq_rel(copy, obj->debug_clone[pos], obj->debug_clone));
#endif
return copy;
}
else {
- Uint sz = size_object(obj->tpl[pos]);
+ Uint sz = size_object_rel(obj->tpl[pos], obj->tpl);
*hpp = HAlloc(p, sz + extra);
- return copy_struct(obj->tpl[pos], sz, hpp, &MSO(p));
+ return copy_struct_rel(obj->tpl[pos], sz, hpp, &MSO(p), obj->tpl, NULL);
}
}
@@ -2765,7 +3002,7 @@ void db_cleanup_offheap_comp(DbTerm* obj)
ProcBin tmp;
for (u.hdr = obj->first_oh; u.hdr; u.hdr = u.hdr->next) {
- if ((UWord)u.voidp % sizeof(UWord) != 0) { /* unaligned ptr */
+ if ((UWord)u.voidp % sizeof(Uint) != 0) { /* unaligned ptr */
sys_memcpy(&tmp, u.voidp, sizeof(tmp));
/* Warning, must pass (void*)-variable to memcpy. Otherwise it will
cause Bus error on Sparc due to false compile time assumptions
@@ -2961,7 +3198,7 @@ static DMCRet dmc_one_term(DMCContext *context,
** Ouch, big integer in match variable.
*/
Eterm *save_hp;
- ASSERT(heap->data == heap->def);
+ ASSERT(heap->vars == heap->vars_def);
sz = sz2 = sz3 = 0;
for (j = 0; j < context->num_match; ++j) {
sz += size_object(context->matchexpr[j]);
@@ -2999,24 +3236,23 @@ static DMCRet dmc_one_term(DMCContext *context,
may be atoms that changed */
context->matchexpr[j] = context->copy->mem[j];
}
- heap->data = erts_alloc(ERTS_ALC_T_DB_MS_CMPL_HEAP,
- heap->size*sizeof(unsigned));
- sys_memset(heap->data, 0,
- heap->size * sizeof(unsigned));
+ heap->vars = erts_alloc(ERTS_ALC_T_DB_MS_CMPL_HEAP,
+ heap->size*sizeof(DMCVariable));
+ sys_memset(heap->vars, 0, heap->size * sizeof(DMCVariable));
DMC_CLEAR(*stack);
/*DMC_PUSH(*stack,NIL);*/
DMC_CLEAR(*text);
return retRestart;
}
- if (heap->data[n]) { /* already bound ? */
+ if (heap->vars[n].is_bound) {
DMC_PUSH(*text,matchCmp);
DMC_PUSH(*text,n);
} else { /* Not bound, bind! */
- if (n >= heap->used)
- heap->used = n + 1;
+ if (n >= heap->vars_used)
+ heap->vars_used = n + 1;
DMC_PUSH(*text,matchBind);
DMC_PUSH(*text,n);
- heap->data[n] = 1;
+ heap->vars[n].is_bound = 1;
}
} else if (c == am_Underscore) {
DMC_PUSH(*text, matchSkip);
@@ -3041,6 +3277,8 @@ static DMCRet dmc_one_term(DMCContext *context,
DMC_PUSH(*stack, c);
break;
case (_TAG_HEADER_REF >> _TAG_PRIMARY_SIZE):
+ {
+ Eterm* ref_val = internal_ref_val(c);
DMC_PUSH(*text, matchEqRef);
#if HALFWORD_HEAP
{
@@ -3048,25 +3286,27 @@ static DMCRet dmc_one_term(DMCContext *context,
UWord u;
Uint t[2];
} fiddle;
- ASSERT(thing_arityval(*internal_ref_val(c)) == 3);
- fiddle.t[0] = *internal_ref_val(c);
- fiddle.t[1] = (Uint) internal_ref_val(c)[1];
+ ASSERT(thing_arityval(ref_val[0]) == 3);
+ fiddle.t[0] = ref_val[0];
+ fiddle.t[1] = ref_val[1];
DMC_PUSH(*text, fiddle.u);
- fiddle.t[0] = (Uint) internal_ref_val(c)[2];
- fiddle.t[1] = (Uint) internal_ref_val(c)[3];
+ fiddle.t[0] = ref_val[2];
+ fiddle.t[1] = ref_val[3];
DMC_PUSH(*text, fiddle.u);
}
#else
- n = thing_arityval(*internal_ref_val(c));
- DMC_PUSH(*text, *internal_ref_val(c));
- for (i = 1; i <= n; ++i) {
- DMC_PUSH(*text, (Uint) internal_ref_val(c)[i]);
+ n = thing_arityval(ref_val[0]);
+ for (i = 0; i <= n; ++i) {
+ DMC_PUSH(*text, ref_val[i]);
}
#endif
break;
+ }
case (_TAG_HEADER_POS_BIG >> _TAG_PRIMARY_SIZE):
case (_TAG_HEADER_NEG_BIG >> _TAG_PRIMARY_SIZE):
- n = thing_arityval(*big_val(c));
+ {
+ Eterm* bval = big_val(c);
+ n = thing_arityval(bval[0]);
DMC_PUSH(*text, matchEqBig);
#if HALFWORD_HEAP
{
@@ -3075,13 +3315,13 @@ static DMCRet dmc_one_term(DMCContext *context,
Uint t[2];
} fiddle;
ASSERT(n >= 1);
- fiddle.t[0] = *big_val(c);
- fiddle.t[1] = big_val(c)[1];
+ fiddle.t[0] = bval[0];
+ fiddle.t[1] = bval[1];
DMC_PUSH(*text, fiddle.u);
for (i = 2; i <= n; ++i) {
- fiddle.t[0] = big_val(c)[i];
+ fiddle.t[0] = bval[i];
if (++i <= n) {
- fiddle.t[1] = big_val(c)[i];
+ fiddle.t[1] = bval[i];
} else {
fiddle.t[1] = (Uint) 0;
}
@@ -3089,12 +3329,12 @@ static DMCRet dmc_one_term(DMCContext *context,
}
}
#else
- DMC_PUSH(*text, *big_val(c));
- for (i = 1; i <= n; ++i) {
- DMC_PUSH(*text, (Uint) big_val(c)[i]);
+ for (i = 0; i <= n; ++i) {
+ DMC_PUSH(*text, (Uint) bval[i]);
}
#endif
break;
+ }
case (_TAG_HEADER_FLOAT >> _TAG_PRIMARY_SIZE):
DMC_PUSH(*text,matchEqFloat);
#if HALFWORD_HEAP
@@ -3227,7 +3467,6 @@ static DMCRet dmc_list(DMCContext *context,
DMC_PUSH(*text, matchConsB);
}
--context->stack_used; /* Two objects on stack becomes one */
- context->eheap_need += 2;
return retOk;
}
@@ -3286,7 +3525,6 @@ static DMCRet dmc_tuple(DMCContext *context,
DMC_PUSH(*text, matchMkTuple);
DMC_PUSH(*text, nelems);
context->stack_used -= (nelems - 1);
- context->eheap_need += (nelems + 1);
*constant = 0;
return retOk;
}
@@ -3304,9 +3542,6 @@ static DMCRet dmc_whole_expression(DMCContext *context,
} else {
ASSERT(is_tuple(context->matchexpr
[context->current_match]));
- context->eheap_need +=
- arityval(*(tuple_val(context->matchexpr
- [context->current_match]))) * 2;
DMC_PUSH(*text, matchPushArrayAsList);
}
} else {
@@ -3319,6 +3554,41 @@ static DMCRet dmc_whole_expression(DMCContext *context,
return retOk;
}
+/* Figure out which PushV instruction to use.
+*/
+static void dmc_add_pushv_variant(DMCContext *context, DMCHeap *heap,
+ DMC_STACK_TYPE(UWord) *text, Uint n)
+{
+ DMCVariable* v = &heap->vars[n];
+ MatchOps instr = matchPushV;
+
+ ASSERT(n < heap->vars_used && v->is_bound);
+ if (context->is_guard) {
+ #if HALFWORD_HEAP
+ if (!v->first_guard_label) {
+ v->first_guard_label = DMC_STACK_NUM(*text);
+ ASSERT(v->first_guard_label);
+ instr = matchPushVGuard; /* may be changed to PushVResult below */
+ }
+ #endif
+ }
+ else { /* body */
+ #if HALFWORD_HEAP
+ if (v->first_guard_label) {
+ /* Avoid double-copy, copy to result heap at first encounter in guard */
+ DMC_POKE(*text, v->first_guard_label, matchPushVResult);
+ v->is_in_body = 1;
+ }
+ #endif
+ if (!v->is_in_body) {
+ instr = matchPushVResult;
+ v->is_in_body = 1;
+ }
+ }
+ DMC_PUSH(*text, instr);
+ DMC_PUSH(*text, n);
+}
+
static DMCRet dmc_variable(DMCContext *context,
DMCHeap *heap,
DMC_STACK_TYPE(UWord) *text,
@@ -3326,13 +3596,13 @@ static DMCRet dmc_variable(DMCContext *context,
int *constant)
{
Uint n = db_is_variable(t);
- ASSERT(n >= 0);
- if (n >= heap->used)
- RETURN_VAR_ERROR("Variable $%d is unbound.", n, context, *constant);
- if (heap->data[n] == 0U)
+
+ if (n >= heap->vars_used || !heap->vars[n].is_bound) {
RETURN_VAR_ERROR("Variable $%d is unbound.", n, context, *constant);
- DMC_PUSH(*text, matchPushV);
- DMC_PUSH(*text, n);
+ }
+
+ dmc_add_pushv_variant(context, heap, text, n);
+
++context->stack_used;
if (context->stack_used > context->stack_need)
context->stack_need = context->stack_used;
@@ -3351,10 +3621,9 @@ static DMCRet dmc_all_bindings(DMCContext *context,
DMC_PUSH(*text, matchPushC);
DMC_PUSH(*text, NIL);
- for (i = heap->used - 1; i >= 0; --i) {
- if (heap->data[i]) {
- DMC_PUSH(*text, matchPushV);
- DMC_PUSH(*text, i);
+ for (i = heap->vars_used - 1; i >= 0; --i) {
+ if (heap->vars[i].is_bound) {
+ dmc_add_pushv_variant(context, heap, text, i);
DMC_PUSH(*text, matchConsB);
heap_used += 2;
}
@@ -3362,7 +3631,6 @@ static DMCRet dmc_all_bindings(DMCContext *context,
++context->stack_used;
if ((context->stack_used + 1) > context->stack_need)
context->stack_need = (context->stack_used + 1);
- context->eheap_need += heap_used;
*constant = 0;
return retOk;
}
@@ -3765,10 +4033,6 @@ static DMCRet dmc_get_seq_token(DMCContext *context,
*constant = 0;
DMC_PUSH(*text, matchGetSeqToken);
- context->eheap_need += (6 /* A 5-tuple is built */
- + EXTERNAL_THING_HEAD_SIZE + 2 /* Sender can
- be an external
- pid */);
if (++context->stack_used > context->stack_need)
context->stack_need = context->stack_used;
return retOk;
@@ -4065,7 +4329,6 @@ static DMCRet dmc_caller(DMCContext *context,
}
*constant = 0;
DMC_PUSH(*text, matchCaller); /* Creates binary */
- context->eheap_need += 4; /* A 3-tuple is built */
if (++context->stack_used > context->stack_need)
context->stack_need = context->stack_used;
return retOk;
@@ -4663,7 +4926,8 @@ static Eterm match_spec_test(Process *p, Eterm against, Eterm spec, int trace)
}
save_cp = p->cp;
p->cp = NULL;
- res = erts_match_set_run(p, mps, arr, n, &ret_flags);
+ res = erts_match_set_run(p, mps, arr, n,
+ ERTS_PAM_COPY_RESULT, &ret_flags);
p->cp = save_cp;
} else {
n = 0;
@@ -4676,11 +4940,10 @@ static Eterm match_spec_test(Process *p, Eterm against, Eterm spec, int trace)
if (is_non_value(res)) {
res = am_false;
}
- sz = size_object(res);
+ sz = 0;
if (ret_flags & MATCH_SET_EXCEPTION_TRACE) sz += 2;
if (ret_flags & MATCH_SET_RETURN_TRACE) sz += 2;
hp = HAlloc(p, 5 + sz);
- res = copy_struct(res, sz, &hp, &MSO(p));
flg = NIL;
if (ret_flags & MATCH_SET_EXCEPTION_TRACE) {
flg = CONS(hp, am_exception_trace, flg);
@@ -4735,28 +4998,24 @@ void db_free_tmp_uncompressed(DbTerm* obj)
erts_free(ERTS_ALC_T_TMP, obj);
}
-Eterm db_prog_match_and_copy(DbTableCommon* tb, Process* c_p, Binary* bprog,
+Eterm db_match_dbterm(DbTableCommon* tb, Process* c_p, Binary* bprog,
int all, DbTerm* obj, Eterm** hpp, Uint extra)
{
Uint32 dummy;
+ Eterm* base;
Eterm res;
if (tb->compress) {
obj = db_alloc_tmp_uncompressed(tb, obj);
+ base = NULL;
}
+ else base = HALFWORD_HEAP ? obj->tpl : NULL;
- res = db_prog_match(c_p, bprog, make_tuple(obj->tpl), NULL, 0, &dummy);
+ res = db_prog_match(c_p, bprog, make_tuple_rel(obj->tpl,base), base, NULL, 0,
+ ERTS_PAM_COPY_RESULT|ERTS_PAM_CONTIGUOUS_TUPLE, &dummy);
if (is_value(res) && hpp!=NULL) {
- if (all) {
- *hpp = HAlloc(c_p, obj->size + extra);
- res = copy_shallow(obj->tpl, obj->size, hpp, &MSO(c_p));
- }
- else {
- Uint sz = size_object(res);
- *hpp = HAlloc(c_p, sz + extra);
- res = copy_struct(res, sz, hpp, &MSO(c_p));
- }
+ *hpp = HAlloc(c_p, extra);
}
if (tb->compress) {
@@ -4767,10 +5026,11 @@ Eterm db_prog_match_and_copy(DbTableCommon* tb, Process* c_p, Binary* bprog,
#ifdef DMC_DEBUG
+
/*
** Disassemble match program
*/
-static void db_match_dis(Binary *bp)
+void db_match_dis(Binary *bp)
{
MatchProg *prog = Binary2MatchProg(bp);
UWord *t = prog->text;
@@ -4985,6 +5245,18 @@ static void db_match_dis(Binary *bp)
++t;
erts_printf("PushV\t%bpu\n", n);
break;
+ #if HALFWORD_HEAP
+ case matchPushVGuard:
+ n = (Uint) *++t;
+ ++t;
+ erts_printf("PushVGuard\t%bpu\n", n);
+ break;
+ #endif
+ case matchPushVResult:
+ n = (Uint) *++t;
+ ++t;
+ erts_printf("PushVResult\t%bpu\n", n);
+ break;
case matchTrue:
++t;
erts_printf("True\n");
@@ -5095,7 +5367,6 @@ static void db_match_dis(Binary *bp)
erts_printf("}\n");
erts_printf("num_bindings: %d\n", prog->num_bindings);
erts_printf("heap_size: %bpu\n", prog->heap_size);
- erts_printf("eheap_offset: %bpu\n", prog->eheap_offset);
erts_printf("stack_offset: %bpu\n", prog->stack_offset);
erts_printf("text: 0x%08x\n", (unsigned long) prog->text);
erts_printf("stack_size: %d (words)\n", prog->heap_size-prog->stack_offset);
diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h
index 10ba755e80..bb1751d309 100644
--- a/erts/emulator/beam/erl_db_util.h
+++ b/erts/emulator/beam/erl_db_util.h
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 1998-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1998-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -86,6 +86,9 @@ typedef struct {
Uint new_size;
int mustResize;
void* lck;
+#if HALFWORD_HEAP
+ unsigned char* abs_vec; /* [i] true if dbterm->tpl[i] is absolute Eterm */
+#endif
} DbUpdateHandle;
@@ -206,8 +209,7 @@ typedef struct db_fixation {
*/
typedef struct db_table_common {
- erts_refc_t ref;
- erts_refc_t fixref; /* fixation counter */
+ erts_refc_t ref; /* fixation counter and delete counter */
#ifdef ERTS_SMP
erts_smp_rwmtx_t rwlock; /* rw lock on table */
erts_smp_mtx_t fixlock; /* Protects fixations,megasec,sec,microsec */
@@ -253,10 +255,17 @@ typedef struct db_table_common {
(DB_BAG | DB_SET | DB_DUPLICATE_BAG)))
#define IS_TREE_TABLE(Status) (!!((Status) & \
DB_ORDERED_SET))
-#define NFIXED(T) (erts_refc_read(&(T)->common.fixref,0))
+#define NFIXED(T) (erts_refc_read(&(T)->common.ref,0))
#define IS_FIXED(T) (NFIXED(T) != 0)
-Eterm erts_ets_copy_object(Eterm, Process*);
+/*
+ * tplp is an untagged pointer to a tuple we know is large enough
+ * and dth is a pointer to a DbTableHash.
+ */
+#define GETKEY(dth, tplp) (*((tplp) + ((DbTableCommon*)(dth))->keypos))
+
+
+ERTS_GLB_INLINE Eterm db_copy_key(Process* p, DbTable* tb, DbTerm* obj);
Eterm db_copy_from_comp(DbTableCommon* tb, DbTerm* bp, Eterm** hpp,
ErlOffHeap* off_heap);
int db_eq_comp(DbTableCommon* tb, Eterm a, DbTerm* b);
@@ -265,9 +274,23 @@ DbTerm* db_alloc_tmp_uncompressed(DbTableCommon* tb, DbTerm* org);
ERTS_GLB_INLINE Eterm db_copy_object_from_ets(DbTableCommon* tb, DbTerm* bp,
Eterm** hpp, ErlOffHeap* off_heap);
ERTS_GLB_INLINE int db_eq(DbTableCommon* tb, Eterm a, DbTerm* b);
-ERTS_GLB_INLINE Eterm db_do_read_element(DbUpdateHandle* handle, Sint position);
+Wterm db_do_read_element(DbUpdateHandle* handle, Sint position);
#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+ERTS_GLB_INLINE Eterm db_copy_key(Process* p, DbTable* tb, DbTerm* obj)
+{
+ Eterm key = GETKEY(tb, obj->tpl);
+ if IS_CONST(key) return key;
+ else {
+ Uint size = size_object_rel(key, obj->tpl);
+ Eterm* hp = HAlloc(p, size);
+ Eterm res = copy_struct_rel(key, size, &hp, &MSO(p), obj->tpl, NULL);
+ ASSERT(eq_rel(res,NULL,key,obj->tpl));
+ return res;
+ }
+}
+
ERTS_GLB_INLINE Eterm db_copy_object_from_ets(DbTableCommon* tb, DbTerm* bp,
Eterm** hpp, ErlOffHeap* off_heap)
{
@@ -275,49 +298,27 @@ ERTS_GLB_INLINE Eterm db_copy_object_from_ets(DbTableCommon* tb, DbTerm* bp,
return db_copy_from_comp(tb, bp, hpp, off_heap);
}
else {
- return copy_shallow(bp->tpl, bp->size, hpp, off_heap);
+ return copy_shallow_rel(bp->tpl, bp->size, hpp, off_heap, bp->tpl);
}
}
ERTS_GLB_INLINE int db_eq(DbTableCommon* tb, Eterm a, DbTerm* b)
{
if (!tb->compress) {
- return eq(a, make_tuple(b->tpl));
+ return eq_rel(a, NULL, make_tuple_rel(b->tpl,b->tpl), b->tpl);
}
else {
return db_eq_comp(tb, a, b);
}
}
-/* Must be called to read elements after db_lookup_dbterm.
-** Will decompress if needed. */
-ERTS_GLB_INLINE Eterm db_do_read_element(DbUpdateHandle* handle, Sint position)
-{
- Eterm elem = handle->dbterm->tpl[position];
- if (!is_header(elem)) {
- return elem;
- }
- ASSERT(((DbTableCommon*)handle->tb)->compress);
- ASSERT(!handle->mustResize);
- handle->dbterm = db_alloc_tmp_uncompressed((DbTableCommon*)handle->tb, handle->dbterm);
- handle->mustResize = 1;
- return handle->dbterm->tpl[position];
-}
-
#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
-/* optimised version of copy_object (normal case? atomic object) */
-#define COPY_OBJECT(obj, p, objp) \
- if (IS_CONST(obj)) { *(objp) = (obj); } \
- else { *objp = erts_ets_copy_object(obj, p); }
#define DB_READ (DB_PROTECTED|DB_PUBLIC)
#define DB_WRITE DB_PUBLIC
#define DB_INFO (DB_PROTECTED|DB_PUBLIC|DB_PRIVATE)
-/* tb is an DbTableCommon and obj is an Eterm (tagged) */
-#define TERM_GETKEY(tb, obj) db_getkey((tb)->common.keypos, (obj))
-
#define ONLY_WRITER(P,T) (((T)->common.status & (DB_PRIVATE|DB_PROTECTED)) \
&& (T)->common.owner == (P)->id)
@@ -338,12 +339,11 @@ Eterm db_copy_element_from_ets(DbTableCommon* tb, Process* p, DbTerm* obj,
Uint pos, Eterm** hpp, Uint extra);
int db_has_variable(Eterm obj);
int db_is_variable(Eterm obj);
-Eterm db_do_read_element(DbUpdateHandle* handle, Sint position);
void db_do_update_element(DbUpdateHandle* handle,
Sint position,
Eterm newval);
void db_finalize_resize(DbUpdateHandle* handle, Uint offset);
-Eterm db_add_counter(Eterm** hpp, Eterm counter, Eterm incr);
+Eterm db_add_counter(Eterm** hpp, Wterm counter, Eterm incr);
Eterm db_match_set_lint(Process *p, Eterm matchexpr, Uint flags);
Binary *db_match_set_compile(Process *p, Eterm matchexpr,
Uint flags);
@@ -360,7 +360,6 @@ typedef struct match_prog {
struct erl_heap_fragment *saved_program_buf;
Eterm saved_program;
Uint heap_size; /* size of: heap + eheap + stack */
- Uint eheap_offset;
Uint stack_offset;
#ifdef DMC_DEBUG
UWord* prog_end; /* End of program */
@@ -424,11 +423,16 @@ Binary *db_match_compile(Eterm *matchexpr, Eterm *guards,
Eterm *body, int num_matches,
Uint flags,
DMCErrInfo *err_info);
-Eterm db_prog_match_and_copy(DbTableCommon* tb, Process* c_p, Binary* bprog,
- int all, DbTerm* obj, Eterm** hpp, Uint extra);
/* Returns newly allocated MatchProg binary with refc == 0*/
-Eterm db_prog_match(Process *p, Binary *prog, Eterm term, Eterm *termp, int arity,
+
+Eterm db_match_dbterm(DbTableCommon* tb, Process* c_p, Binary* bprog,
+ int all, DbTerm* obj, Eterm** hpp, Uint extra);
+
+Eterm db_prog_match(Process *p, Binary *prog, Eterm term, Eterm* base,
+ Eterm *termp, int arity,
+ enum erts_pam_run_flags in_flags,
Uint32 *return_flags /* Zeroed on enter */);
+
/* returns DB_ERROR_NONE if matches, 1 if not matches and some db error on
error. */
DMCErrInfo *db_new_dmc_err_info(void);
diff --git a/erts/emulator/beam/erl_drv_thread.c b/erts/emulator/beam/erl_drv_thread.c
index 17b08a71d4..39bbe9633b 100644
--- a/erts/emulator/beam/erl_drv_thread.c
+++ b/erts/emulator/beam/erl_drv_thread.c
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2007-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2007-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -24,6 +24,10 @@
#include "global.h"
#include <string.h>
+#if defined(__APPLE__) && defined(__MACH__) && !defined(__DARWIN__)
+#define __DARWIN__ 1
+#endif
+
#define ERL_DRV_THR_OPTS_SIZE(LAST_FIELD) \
(((size_t) &((ErlDrvThreadOpts *) 0)->LAST_FIELD) \
+ sizeof(((ErlDrvThreadOpts *) 0)->LAST_FIELD))
@@ -692,3 +696,57 @@ erl_drv_thread_join(ErlDrvTid tid, void **respp)
#endif
}
+#if defined(__DARWIN__) && defined(USE_THREADS) && defined(ERTS_SMP)
+extern int erts_darwin_main_thread_pipe[2];
+extern int erts_darwin_main_thread_result_pipe[2];
+
+
+int
+erl_drv_stolen_main_thread_join(ErlDrvTid tid, void **respp)
+{
+ void *dummy;
+ void **x;
+ if (respp == NULL)
+ x = &dummy;
+ else
+ x = respp;
+ read(erts_darwin_main_thread_result_pipe[0],x,sizeof(void *));
+ return 0;
+}
+
+int
+erl_drv_steal_main_thread(char *name,
+ ErlDrvTid *tid,
+ void* (*func)(void*),
+ void* arg,
+ ErlDrvThreadOpts *opts)
+{
+ char buff[sizeof(void* (*)(void*)) + sizeof(void *)];
+ int buff_sz = sizeof(void* (*)(void*)) + sizeof(void *);
+ /*struct ErlDrvTid_ *dtid;
+
+ dtid = erts_alloc_fnf(ERTS_ALC_T_DRV_TID,
+ (sizeof(struct ErlDrvTid_)
+ + (name ? sys_strlen(name) + 1 : 0)));
+ if (!dtid)
+ return ENOMEM;
+ memset(dtid,0,sizeof(ErlDrvTid_));
+ dtid->tid = (void * ) -1;
+ dtid->drv_thr = 1;
+ dtid->func = func;
+ dtid->arg = arg;
+ dtid->tsd = NULL;
+ dtid->tsd_len = 0;
+ dtid->name = no_name;
+ *tid = (ErlDrvTid) dtid;
+ */
+ *tid = NULL;
+ /* Ignore options and name... */
+
+ memcpy(buff,&func,sizeof(void* (*)(void*)));
+ memcpy(buff + sizeof(void* (*)(void*)),&arg,sizeof(void *));
+ write(erts_darwin_main_thread_pipe[1],buff,buff_sz);
+ return 0;
+}
+
+#endif
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index f4e0717d30..0a57eb6d88 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -250,7 +250,7 @@ erl_init(int ncpu)
erts_init_monitors();
erts_init_gc();
- init_time();
+ erts_init_time();
erts_init_sys_common_misc();
erts_init_process(ncpu);
erts_init_scheduling(use_multi_run_queue,
@@ -289,7 +289,7 @@ erl_init(int ncpu)
erts_delay_trap = erts_export_put(am_erlang, am_delay_trap, 2);
erts_late_init_process();
#if HAVE_ERTS_MSEG
- erts_mseg_late_init(); /* Must be after timer (init_time()) and thread
+ erts_mseg_late_init(); /* Must be after timer (erts_init_time()) and thread
initializations */
#endif
#ifdef HIPE
diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c
index 7a6aaa6bbe..9e18997890 100644
--- a/erts/emulator/beam/erl_lock_check.c
+++ b/erts/emulator/beam/erl_lock_check.c
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2005-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -155,7 +155,7 @@ static erts_lc_lock_order_t erts_lock_order[] = {
{ "alcu_allocator", "index" },
{ "alcu_delayed_free", "index" },
{ "mseg", NULL },
-#ifdef HALFWORD_HEAP
+#if HALFWORD_HEAP
{ "pmmap", NULL },
#endif
#ifdef ERTS_SMP
@@ -177,6 +177,8 @@ static erts_lc_lock_order_t erts_lock_order[] = {
{ "async_id", NULL },
{ "pix_lock", "address" },
{ "run_queues_lists", NULL },
+ { "misc_aux_work_queue", "index" },
+ { "misc_aux_work_pre_alloc_lock", "address" },
{ "sched_stat", NULL },
{ "run_queue_sleep_list", "address" },
#endif
diff --git a/erts/emulator/beam/erl_monitors.c b/erts/emulator/beam/erl_monitors.c
index d873c7a701..9751b5d77c 100644
--- a/erts/emulator/beam/erl_monitors.c
+++ b/erts/emulator/beam/erl_monitors.c
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2004-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -85,7 +85,7 @@ static ERTS_INLINE int cmp_mon_ref(Eterm ref1, Eterm ref2)
if (is_ref_thing_header(*b2)) {
return 1;
}
- return cmp(ref1,ref2);
+ return CMP(ref1,ref2);
}
#define CP_LINK_VAL(To, Hp, From) \
@@ -380,7 +380,7 @@ int erts_add_link(ErtsLink **root, Uint type, Eterm pid)
state = 1;
*this = create_link(type,pid);
break;
- } else if ((c = cmp(pid,(*this)->pid)) < 0) {
+ } else if ((c = CMP(pid,(*this)->pid)) < 0) {
/* go left */
dstack[dpos++] = DIR_LEFT;
tstack[tpos++] = this;
@@ -415,7 +415,7 @@ erts_add_or_lookup_suspend_monitor(ErtsSuspendMonitor **root, Eterm pid)
state = 1;
res = *this = create_suspend_monitor(pid);
break;
- } else if ((c = cmp(pid,(*this)->pid)) < 0) {
+ } else if ((c = CMP(pid,(*this)->pid)) < 0) {
/* go left */
dstack[dpos++] = DIR_LEFT;
tstack[tpos++] = this;
@@ -453,7 +453,7 @@ ErtsLink *erts_add_or_lookup_link(ErtsLink **root, Uint type, Eterm pid)
*this = create_link(type,pid);
ret = *this;
break;
- } else if ((c = cmp(pid,(*this)->pid)) < 0) {
+ } else if ((c = CMP(pid,(*this)->pid)) < 0) {
/* go left */
dstack[dpos++] = DIR_LEFT;
tstack[tpos++] = this;
@@ -663,7 +663,7 @@ ErtsLink *erts_remove_link(ErtsLink **root, Eterm pid)
for (;;) {
if (!*this) { /* Failure */
return NULL;
- } else if ((c = cmp(pid,(*this)->pid)) < 0) {
+ } else if ((c = CMP(pid,(*this)->pid)) < 0) {
dstack[dpos++] = DIR_LEFT;
tstack[tpos++] = this;
this = &((*this)->left);
@@ -715,7 +715,7 @@ erts_delete_suspend_monitor(ErtsSuspendMonitor **root, Eterm pid)
for (;;) {
if (!*this) { /* Nothing found */
return;
- } else if ((c = cmp(pid,(*this)->pid)) < 0) {
+ } else if ((c = CMP(pid,(*this)->pid)) < 0) {
dstack[dpos++] = DIR_LEFT;
tstack[tpos++] = this;
this = &((*this)->left);
@@ -771,7 +771,7 @@ ErtsLink *erts_lookup_link(ErtsLink *root, Eterm pid)
Sint c;
for (;;) {
- if (root == NULL || (c = cmp(pid,root->pid)) == 0) {
+ if (root == NULL || (c = CMP(pid,root->pid)) == 0) {
return root;
} else if (c < 0) {
root = root->left;
@@ -787,7 +787,7 @@ erts_lookup_suspend_monitor(ErtsSuspendMonitor *root, Eterm pid)
Sint c;
for (;;) {
- if (root == NULL || (c = cmp(pid,root->pid)) == 0) {
+ if (root == NULL || (c = CMP(pid,root->pid)) == 0) {
return root;
} else if (c < 0) {
root = root->left;
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index a680097c2d..135c6b0ccc 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2009-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2009-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -81,7 +81,6 @@ static ERTS_INLINE Eterm* alloc_heap(ErlNifEnv* env, unsigned need)
static Eterm* alloc_heap_heavy(ErlNifEnv* env, unsigned need, Eterm* hp)
{
- unsigned frag_sz;
env->hp = hp;
if (env->heap_frag == NULL) {
ASSERT(HEAP_LIMIT(env->proc) == env->hp_end);
@@ -91,11 +90,11 @@ static Eterm* alloc_heap_heavy(ErlNifEnv* env, unsigned need, Eterm* hp)
env->heap_frag->used_size = hp - env->heap_frag->mem;
ASSERT(env->heap_frag->used_size <= env->heap_frag->alloc_size);
}
- frag_sz = need + MIN_HEAP_FRAG_SZ;
- hp = erts_heap_alloc(env->proc, frag_sz);
- env->hp = hp + need;
- env->hp_end = hp + frag_sz;
+ hp = erts_heap_alloc(env->proc, need, MIN_HEAP_FRAG_SZ);
env->heap_frag = MBUF(env->proc);
+ env->hp = hp + need;
+ env->hp_end = env->heap_frag->mem + env->heap_frag->alloc_size;
+
return hp;
}
@@ -574,7 +573,7 @@ int enif_is_identical(Eterm lhs, Eterm rhs)
int enif_compare(Eterm lhs, Eterm rhs)
{
- return cmp(lhs,rhs);
+ return CMP(lhs,rhs);
}
int enif_get_tuple(ErlNifEnv* env, Eterm tpl, int* arity, const Eterm** array)
@@ -939,21 +938,26 @@ ERL_NIF_TERM enif_make_list_cell(ErlNifEnv* env, Eterm car, Eterm cdr)
ERL_NIF_TERM enif_make_list(ErlNifEnv* env, unsigned cnt, ...)
{
- Eterm* hp = alloc_heap(env,cnt*2);
- Eterm ret = make_list(hp);
- Eterm* last = &ret;
- va_list ap;
-
- va_start(ap,cnt);
- while (cnt--) {
- *last = make_list(hp);
- *hp = va_arg(ap,Eterm);
- last = ++hp;
- ++hp;
+ if (cnt == 0) {
+ return NIL;
+ }
+ else {
+ Eterm* hp = alloc_heap(env,cnt*2);
+ Eterm ret = make_list(hp);
+ Eterm* last = &ret;
+ va_list ap;
+
+ va_start(ap,cnt);
+ while (cnt--) {
+ *last = make_list(hp);
+ *hp = va_arg(ap,Eterm);
+ last = ++hp;
+ ++hp;
+ }
+ va_end(ap);
+ *last = NIL;
+ return ret;
}
- va_end(ap);
- *last = NIL;
- return ret;
}
ERL_NIF_TERM enif_make_list_from_array(ErlNifEnv* env, const ERL_NIF_TERM arr[], unsigned cnt)
@@ -1474,7 +1478,13 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
ret = load_nif_error(BIF_P, bad_lib, "Library version (%d.%d) not compatible (with %d.%d).",
entry->major, entry->minor, ERL_NIF_MAJOR_VERSION, ERL_NIF_MINOR_VERSION);
- }
+ }
+ else if (entry->minor >= 1
+ && sys_strcmp(entry->vm_variant, ERL_NIF_VM_VARIANT) != 0) {
+ ret = load_nif_error(BIF_P, bad_lib, "Library (%s) not compiled for "
+ "this vm variant (%s).",
+ entry->vm_variant, ERL_NIF_VM_VARIANT);
+ }
else if (!erts_is_atom_str((char*)entry->name, mod_atom)) {
ret = load_nif_error(BIF_P, bad_lib, "Library module name '%s' does not"
" match calling module '%T'", entry->name, mod_atom);
diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h
index ee3a7cd5f4..8050b3640a 100644
--- a/erts/emulator/beam/erl_nif.h
+++ b/erts/emulator/beam/erl_nif.h
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2009-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2009-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -30,9 +30,10 @@
** 0.1: R13B03
** 1.0: R13B04
** 2.0: R14A
+** 2.1: R14B02 "vm_variant"
*/
#define ERL_NIF_MAJOR_VERSION 2
-#define ERL_NIF_MINOR_VERSION 0
+#define ERL_NIF_MINOR_VERSION 1
#include <stdlib.h>
@@ -80,8 +81,10 @@ typedef long long ErlNifSInt64;
#endif
#ifdef HALFWORD_HEAP_EMULATOR
+# define ERL_NIF_VM_VARIANT "beam.halfword"
typedef unsigned int ERL_NIF_TERM;
#else
+# define ERL_NIF_VM_VARIANT "beam.vanilla"
typedef unsigned long ERL_NIF_TERM;
#endif
@@ -105,7 +108,8 @@ typedef struct enif_entry_t
int (*load) (ErlNifEnv*, void** priv_data, ERL_NIF_TERM load_info);
int (*reload) (ErlNifEnv*, void** priv_data, ERL_NIF_TERM load_info);
int (*upgrade)(ErlNifEnv*, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info);
- void (*unload) (ErlNifEnv*, void* priv_data);
+ void (*unload) (ErlNifEnv*, void* priv_data);
+ const char* vm_variant;
}ErlNifEntry;
@@ -198,6 +202,7 @@ extern TWinDynNifCallbacks WinDynNifCallbacks;
#define ERL_NIF_INIT(NAME, FUNCS, LOAD, RELOAD, UPGRADE, UNLOAD) \
ERL_NIF_INIT_PROLOGUE \
ERL_NIF_INIT_GLOB \
+ERL_NIF_INIT_DECL(NAME); \
ERL_NIF_INIT_DECL(NAME) \
{ \
static ErlNifEntry entry = \
@@ -207,7 +212,8 @@ ERL_NIF_INIT_DECL(NAME) \
#NAME, \
sizeof(FUNCS) / sizeof(*FUNCS), \
FUNCS, \
- LOAD, RELOAD, UPGRADE, UNLOAD \
+ LOAD, RELOAD, UPGRADE, UNLOAD, \
+ ERL_NIF_VM_VARIANT \
}; \
ERL_NIF_INIT_BODY; \
return &entry; \
diff --git a/erts/emulator/beam/erl_node_container_utils.h b/erts/emulator/beam/erl_node_container_utils.h
index ae1316eba2..2c67e781e0 100644
--- a/erts/emulator/beam/erl_node_container_utils.h
+++ b/erts/emulator/beam/erl_node_container_utils.h
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2001-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -255,19 +255,32 @@ extern int erts_use_r9_pids_ports;
#define internal_ref_no_of_numbers(x) \
(internal_ref_data((x))[0])
+#define internal_thing_ref_no_of_numbers(thing) \
+ (internal_thing_ref_data(thing)[0])
#define internal_ref_numbers(x) \
(&internal_ref_data((x))[1])
+#define internal_thing_ref_numbers(thing) \
+ (&internal_thing_ref_data(thing)[1])
#define external_ref_no_of_numbers(x) \
(external_ref_data((x))[0])
+#define external_thing_ref_no_of_numbers(thing) \
+ (external_thing_ref_data(thing)[0])
#define external_ref_numbers(x) \
(&external_ref_data((x))[1])
+#define external_thing_ref_numbers(thing) \
+ (&external_thing_ref_data(thing)[1])
+
#else
#define internal_ref_no_of_numbers(x) (internal_ref_data_words((x)))
+#define internal_thing_ref_no_of_numbers(t) (internal_thing_ref_data_words(t))
#define internal_ref_numbers(x) (internal_ref_data((x)))
+#define internal_thing_ref_numbers(t) (internal_thing_ref_data(t))
#define external_ref_no_of_numbers(x) (external_ref_data_words((x)))
+#define external_thing_ref_no_of_numbers(t) (external_thing_ref_data_words((t)))
#define external_ref_numbers(x) (external_ref_data((x)))
+#define external_thing_ref_numbers(t) (external_thing_ref_data((t)))
#endif
@@ -311,6 +324,8 @@ extern int erts_use_r9_pids_ports;
: external_ref_channel_no((x)))
#define is_ref(x) (is_internal_ref((x)) \
|| is_external_ref((x)))
+#define is_ref_rel(x,Base) (is_internal_ref_rel((x),Base) \
+ || is_external_ref_rel((x),Base))
#define is_not_ref(x) (!is_ref(x))
#endif
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 77ee1d6ac5..4d6e982325 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 1996-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -590,6 +590,122 @@ erts_sched_finish_poke(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flags)
}
}
+typedef struct erts_misc_aux_work_t_ erts_misc_aux_work_t;
+struct erts_misc_aux_work_t_ {
+ erts_misc_aux_work_t *next;
+ void (*func)(void *);
+ void *arg;
+};
+
+typedef struct {
+ erts_smp_mtx_t mtx;
+ erts_misc_aux_work_t *first;
+ erts_misc_aux_work_t *last;
+} erts_misc_aux_work_q_t;
+
+typedef union {
+ erts_misc_aux_work_q_t data;
+ char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_misc_aux_work_q_t))];
+} erts_algnd_misc_aux_work_q_t;
+
+static erts_algnd_misc_aux_work_q_t *misc_aux_work_queues;
+
+ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(misc_aux_work,
+ erts_misc_aux_work_t,
+ 200,
+ ERTS_ALC_T_MISC_AUX_WORK)
+
+static void
+init_misc_aux_work(void)
+{
+ int ix;
+
+ init_misc_aux_work_alloc();
+
+ misc_aux_work_queues = erts_alloc(ERTS_ALC_T_MISC_AUX_WORK_Q,
+ (sizeof(erts_algnd_misc_aux_work_q_t)
+ *(erts_no_schedulers+1)));
+ if ((((UWord) misc_aux_work_queues) & ERTS_CACHE_LINE_MASK) != 0)
+ misc_aux_work_queues = ((erts_algnd_misc_aux_work_q_t *)
+ ((((UWord) misc_aux_work_queues)
+ & ~ERTS_CACHE_LINE_MASK)
+ + ERTS_CACHE_LINE_SIZE));
+
+ for (ix = 0; ix < erts_no_schedulers; ix++) {
+ erts_smp_mtx_init_x(&misc_aux_work_queues[ix].data.mtx,
+ "misc_aux_work_queue",
+ make_small(ix + 1));
+ misc_aux_work_queues[ix].data.first = NULL;
+ misc_aux_work_queues[ix].data.last = NULL;
+ }
+}
+
+static void
+handle_misc_aux_work(ErtsSchedulerData *esdp)
+{
+ int ix = (int) esdp->no - 1;
+ erts_misc_aux_work_t *mawp;
+
+ erts_smp_mtx_lock(&misc_aux_work_queues[ix].data.mtx);
+ mawp = misc_aux_work_queues[ix].data.first;
+ misc_aux_work_queues[ix].data.first = NULL;
+ misc_aux_work_queues[ix].data.last = NULL;
+ erts_smp_mtx_unlock(&misc_aux_work_queues[ix].data.mtx);
+
+ while (mawp) {
+ erts_misc_aux_work_t *free_mawp;
+ mawp->func(mawp->arg);
+ free_mawp = mawp;
+ mawp = mawp->next;
+ misc_aux_work_free(free_mawp);
+ }
+}
+
+void
+erts_smp_schedule_misc_aux_work(int ignore_self,
+ int max_sched,
+ void (*func)(void *),
+ void *arg)
+{
+ int ix, ignore_ix = -1;
+
+ if (ignore_self) {
+ ErtsSchedulerData *esdp = erts_get_scheduler_data();
+ if (esdp)
+ ignore_ix = (int) esdp->no - 1;
+ }
+
+ ASSERT(0 <= max_sched && max_sched <= erts_no_schedulers);
+
+ for (ix = 0; ix < max_sched; ix++) {
+ erts_aint32_t aux_work;
+ erts_misc_aux_work_t *mawp;
+ ErtsSchedulerSleepInfo *ssi;
+ if (ix == ignore_ix)
+ continue;
+
+ mawp = misc_aux_work_alloc();
+
+ mawp->func = func;
+ mawp->arg = arg;
+ mawp->next = NULL;
+
+ erts_smp_mtx_lock(&misc_aux_work_queues[ix].data.mtx);
+ if (!misc_aux_work_queues[ix].data.last)
+ misc_aux_work_queues[ix].data.first = mawp;
+ else
+ misc_aux_work_queues[ix].data.last->next = mawp;
+ misc_aux_work_queues[ix].data.last = mawp;
+ erts_smp_mtx_unlock(&misc_aux_work_queues[ix].data.mtx);
+
+ ssi = ERTS_SCHED_SLEEP_INFO_IX(ix);
+ aux_work = erts_smp_atomic32_bor(&ssi->aux_work,
+ ERTS_SSI_AUX_WORK_MISC);
+ if ((aux_work & ERTS_SSI_AUX_WORK_MISC) == 0)
+ erts_sched_poke(ssi);
+ }
+}
+
#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN
void
erts_smp_notify_check_children_needed(void)
@@ -615,6 +731,12 @@ blockable_aux_work(ErtsSchedulerData *esdp,
erts_aint32_t aux_work)
{
if (aux_work & ERTS_SSI_BLOCKABLE_AUX_WORK_MASK) {
+ if (aux_work & ERTS_SSI_AUX_WORK_MISC) {
+ aux_work = erts_smp_atomic32_band(&ssi->aux_work,
+ ~ERTS_SSI_AUX_WORK_MISC);
+ aux_work &= ~ERTS_SSI_AUX_WORK_MISC;
+ handle_misc_aux_work(esdp);
+ }
#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN
if (aux_work & ERTS_SSI_AUX_WORK_CHECK_CHILDREN) {
aux_work = erts_smp_atomic32_band(&ssi->aux_work,
@@ -997,8 +1119,8 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
erl_sys_schedule(1); /* Might give us something to do */
- dt = do_time_read_and_reset();
- if (dt) bump_timer(dt);
+ dt = erts_do_time_read_and_reset();
+ if (dt) erts_bump_timer(dt);
sys_aux_work:
@@ -1094,8 +1216,8 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
erl_sys_schedule(0);
- dt = do_time_read_and_reset();
- if (dt) bump_timer(dt);
+ dt = erts_do_time_read_and_reset();
+ if (dt) erts_bump_timer(dt);
flgs = sched_prep_cont_spin_wait(ssi);
if (flgs & ERTS_SSI_FLG_WAITING)
@@ -2612,6 +2734,8 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online)
erts_smp_atomic32_init(&doing_sys_schedule, 0);
+ init_misc_aux_work();
+
#else /* !ERTS_SMP */
{
ErtsSchedulerData *esdp;
@@ -2628,6 +2752,15 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online)
/* init port tasks */
erts_port_task_init();
+
+#ifndef ERTS_SMP
+#ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
+ erts_scheduler_data->verify_unused_temp_alloc
+ = erts_alloc_get_verify_unused_temp_alloc(
+ &erts_scheduler_data->verify_unused_temp_alloc_data);
+ ERTS_VERIFY_UNUSED_TEMP_ALLOC(NULL);
+#endif
+#endif
}
ErtsRunQueue *
@@ -2739,6 +2872,19 @@ resume_process(Process *p)
p->rstatus = P_FREE;
}
+int
+erts_get_max_no_executing_schedulers(void)
+{
+#ifdef ERTS_SMP
+ if (erts_smp_atomic32_read(&schdlr_sspnd.changing))
+ return (int) erts_no_schedulers;
+ ERTS_THR_MEMORY_BARRIER;
+ return (int) erts_smp_atomic32_read(&schdlr_sspnd.active);
+#else
+ return 1;
+#endif
+}
+
#ifdef ERTS_SMP
static void
@@ -3574,6 +3720,13 @@ sched_thread_func(void *vesdp)
}
erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
+#ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
+ ((ErtsSchedulerData *) vesdp)->verify_unused_temp_alloc
+ = erts_alloc_get_verify_unused_temp_alloc(
+ &((ErtsSchedulerData *) vesdp)->verify_unused_temp_alloc_data);
+ ERTS_VERIFY_UNUSED_TEMP_ALLOC(NULL);
+#endif
+
process_main();
/* No schedulers should *ever* terminate */
erl_exit(ERTS_ABORT_EXIT, "Scheduler thread number %bpu terminated\n",
@@ -5070,10 +5223,10 @@ Process *schedule(Process *p, int calls)
ERTS_SMP_CHK_NO_PROC_LOCKS;
- dt = do_time_read_and_reset();
+ dt = erts_do_time_read_and_reset();
if (dt) {
erts_smp_runq_unlock(rq);
- bump_timer(dt);
+ erts_bump_timer(dt);
erts_smp_runq_lock(rq);
}
BM_STOP_TIMER(system);
@@ -5214,8 +5367,8 @@ Process *schedule(Process *p, int calls)
#endif
erts_smp_runq_unlock(rq);
erl_sys_schedule(runnable);
- dt = do_time_read_and_reset();
- if (dt) bump_timer(dt);
+ dt = erts_do_time_read_and_reset();
+ if (dt) erts_bump_timer(dt);
#ifdef ERTS_SMP
erts_smp_runq_lock(rq);
erts_smp_atomic32_set(&doing_sys_schedule, 0);
@@ -6931,11 +7084,11 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext)
DeclareTmpHeapNoproc(lhp,3);
ErtsProcLocks rp_locks = (ERTS_PROC_LOCK_LINK
| ERTS_PROC_LOCKS_MSG_SEND);
- UseTmpHeapNoproc(3);
rp = erts_pid2proc(NULL, 0, mon->pid, rp_locks);
if (rp == NULL) {
goto done;
}
+ UseTmpHeapNoproc(3);
rmon = erts_remove_monitor(&(rp->monitors),mon->ref);
if (rmon) {
erts_destroy_monitor(rmon);
@@ -7476,7 +7629,7 @@ cancel_timer(Process* p)
#ifdef ERTS_SMP
erts_cancel_smp_ptimer(p->u.ptimer);
#else
- erl_cancel_timer(&p->u.tm);
+ erts_cancel_timer(&p->u.tm);
#endif
}
@@ -7502,7 +7655,7 @@ set_timer(Process* p, Uint timeout)
(ErlTimeoutProc) timeout_proc,
timeout);
#else
- erl_set_timer(&p->u.tm,
+ erts_set_timer(&p->u.tm,
(ErlTimeoutProc) timeout_proc,
NULL,
(void*) p,
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 8e32121a68..e871a9834a 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 1996-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -28,6 +28,12 @@
#define ERTS_INCLUDE_SCHEDULER_INTERNALS
#endif
+/* #define ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC */
+
+#if !defined(ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC) && defined(DEBUG)
+# define ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
+#endif
+
typedef struct process Process;
#include "sys.h"
@@ -236,16 +242,14 @@ typedef enum {
| ERTS_SSI_FLG_WAITING \
| ERTS_SSI_FLG_SUSPENDED)
-
-#if !defined(ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK) \
- && defined(ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN)
#define ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK
-#endif
#define ERTS_SSI_AUX_WORK_CHECK_CHILDREN (((erts_aint32_t) 1) << 0)
+#define ERTS_SSI_AUX_WORK_MISC (((erts_aint32_t) 1) << 1)
#define ERTS_SSI_BLOCKABLE_AUX_WORK_MASK \
- (ERTS_SSI_AUX_WORK_CHECK_CHILDREN)
+ (ERTS_SSI_AUX_WORK_CHECK_CHILDREN \
+ | ERTS_SSI_AUX_WORK_MISC)
#define ERTS_SSI_NONBLOCKABLE_AUX_WORK_MASK \
(0)
@@ -423,6 +427,11 @@ struct ErtsSchedulerData_ {
/* NOTE: These fields are modified under held mutexes by other threads */
erts_smp_atomic32_t chk_cpu_bind; /* Only used when common run queue */
#endif
+
+#ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
+ erts_alloc_verify_func_t verify_unused_temp_alloc;
+ Allctr_t *verify_unused_temp_alloc_data;
+#endif
};
typedef union {
@@ -826,7 +835,7 @@ ERTS_GLB_INLINE void erts_heap_frag_shrink(Process* p, Eterm* hp)
}
#endif /* inline */
-Eterm* erts_heap_alloc(Process* p, Uint need);
+Eterm* erts_heap_alloc(Process* p, Uint need, Uint xtra);
#ifdef CHECK_FOR_HOLES
Eterm* erts_set_hole_marker(Eterm* ptr, Uint sz);
#endif
@@ -1034,6 +1043,7 @@ int erts_sched_set_wakeup_limit(char *str);
#ifdef DEBUG
void erts_dbg_multi_scheduling_return_trap(Process *, Eterm);
#endif
+int erts_get_max_no_executing_schedulers(void);
#ifdef ERTS_SMP
ErtsSchedSuspendResult
erts_schedulers_state(Uint *, Uint *, Uint *, int);
@@ -1048,6 +1058,11 @@ int erts_is_multi_scheduling_blocked(void);
Eterm erts_multi_scheduling_blockers(Process *);
void erts_start_schedulers(void);
void erts_smp_notify_check_children_needed(void);
+void
+erts_smp_schedule_misc_aux_work(int ignore_self,
+ int max_sched,
+ void (*func)(void *),
+ void *arg);
#endif
void erts_sched_notify_check_cpu_bind(void);
Uint erts_active_schedulers(void);
@@ -1141,6 +1156,20 @@ Uint erts_debug_nbalance(void);
# define ERTS_PROC_GET_SCHDATA(PROC) (erts_scheduler_data)
#endif
+#ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
+# define ERTS_VERIFY_UNUSED_TEMP_ALLOC(P) \
+do { \
+ ErtsSchedulerData *esdp__ = ((P) \
+ ? ERTS_PROC_GET_SCHDATA((Process *) (P)) \
+ : erts_get_scheduler_data()); \
+ if (esdp__) \
+ esdp__->verify_unused_temp_alloc( \
+ esdp__->verify_unused_temp_alloc_data); \
+} while (0)
+#else
+# define ERTS_VERIFY_UNUSED_TEMP_ALLOC(ESDP)
+#endif
+
#if defined(ERTS_SMP) || defined(USE_THREADS)
ErtsSchedulerData *erts_get_scheduler_data(void);
#else
diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c
index 7a7042abe4..68fda01597 100644
--- a/erts/emulator/beam/erl_process_dump.c
+++ b/erts/emulator/beam/erl_process_dump.c
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2003-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2003-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -261,139 +261,139 @@ print_function_from_pc(int to, void *to_arg, BeamInstr* x)
static void
heap_dump(int to, void *to_arg, Eterm x)
{
+ DeclareTmpHeapNoproc(last,1);
+ Eterm* next = last;
Eterm* ptr;
- Eterm last = OUR_NIL;
- Eterm* next = &last;
if (is_immed(x) || is_CP(x)) {
return;
}
-
- again:
- if (x == OUR_NIL) { /* We are done. */
- return;
- } if (is_CP(x)) {
- next = (Eterm *) EXPAND_POINTER(x);
- } else if (is_list(x)) {
- ptr = list_val(x);
- if (ptr[0] != OUR_NIL) {
- erts_print(to, to_arg, ADDR_FMT ":l", ptr);
- dump_element(to, to_arg, ptr[0]);
- erts_putc(to, to_arg, '|');
- dump_element(to, to_arg, ptr[1]);
- erts_putc(to, to_arg, '\n');
- if (is_immed(ptr[1])) {
- ptr[1] = make_small(0);
- }
- x = ptr[0];
- ptr[0] = (Eterm) COMPRESS_POINTER(next);
- next = ptr + 1;
- goto again;
- }
- } else if (is_boxed(x)) {
- Eterm hdr;
-
- ptr = boxed_val(x);
- hdr = *ptr;
- if (hdr != OUR_NIL) { /* If not visited */
- erts_print(to, to_arg, ADDR_FMT ":", ptr);
- if (is_arity_value(hdr)) {
- Uint i;
- Uint arity = arityval(hdr);
-
- erts_print(to, to_arg, "t" WORD_FMT ":", arity);
- for (i = 1; i <= arity; i++) {
- dump_element(to, to_arg, ptr[i]);
- if (is_immed(ptr[i])) {
- ptr[i] = make_small(0);
- }
- if (i < arity) {
- erts_putc(to, to_arg, ',');
- }
- }
+ UseTmpHeapNoproc(1);
+ *last = OUR_NIL;
+
+ while (x != OUR_NIL) {
+ if (is_CP(x)) {
+ next = (Eterm *) EXPAND_POINTER(x);
+ } else if (is_list(x)) {
+ ptr = list_val(x);
+ if (ptr[0] != OUR_NIL) {
+ erts_print(to, to_arg, ADDR_FMT ":l", ptr);
+ dump_element(to, to_arg, ptr[0]);
+ erts_putc(to, to_arg, '|');
+ dump_element(to, to_arg, ptr[1]);
erts_putc(to, to_arg, '\n');
- if (arity == 0) {
- ptr[0] = OUR_NIL;
- } else {
- x = ptr[arity];
- ptr[0] = (Eterm) COMPRESS_POINTER(next);
- next = ptr + arity - 1;
- goto again;
+ if (is_immed(ptr[1])) {
+ ptr[1] = make_small(0);
}
- } else if (hdr == HEADER_FLONUM) {
- FloatDef f;
- char sbuf[31];
- int i;
-
- GET_DOUBLE_DATA((ptr+1), f);
- i = sys_double_to_chars(f.fd, (char*) sbuf);
- sys_memset(sbuf+i, 0, 31-i);
- erts_print(to, to_arg, "F%X:%s\n", i, sbuf);
- *ptr = OUR_NIL;
- } else if (_is_bignum_header(hdr)) {
- erts_print(to, to_arg, "B%T\n", x);
- *ptr = OUR_NIL;
- } else if (is_binary_header(hdr)) {
- Uint tag = thing_subtag(hdr);
- Uint size = binary_size(x);
- Uint i;
-
- if (tag == HEAP_BINARY_SUBTAG) {
- byte* p;
-
- erts_print(to, to_arg, "Yh%X:", size);
- p = binary_bytes(x);
- for (i = 0; i < size; i++) {
- erts_print(to, to_arg, "%02X", p[i]);
+ x = ptr[0];
+ ptr[0] = (Eterm) COMPRESS_POINTER(next);
+ next = ptr + 1;
+ continue;
+ }
+ } else if (is_boxed(x)) {
+ Eterm hdr;
+
+ ptr = boxed_val(x);
+ hdr = *ptr;
+ if (hdr != OUR_NIL) { /* If not visited */
+ erts_print(to, to_arg, ADDR_FMT ":", ptr);
+ if (is_arity_value(hdr)) {
+ Uint i;
+ Uint arity = arityval(hdr);
+
+ erts_print(to, to_arg, "t" WORD_FMT ":", arity);
+ for (i = 1; i <= arity; i++) {
+ dump_element(to, to_arg, ptr[i]);
+ if (is_immed(ptr[i])) {
+ ptr[i] = make_small(0);
+ }
+ if (i < arity) {
+ erts_putc(to, to_arg, ',');
+ }
}
- } else if (tag == REFC_BINARY_SUBTAG) {
- ProcBin* pb = (ProcBin *) binary_val(x);
- Binary* val = pb->val;
-
- if (erts_smp_atomic_xchg(&val->refc, 0) != 0) {
- val->flags = (UWord) all_binaries;
- all_binaries = val;
+ erts_putc(to, to_arg, '\n');
+ if (arity == 0) {
+ ptr[0] = OUR_NIL;
+ } else {
+ x = ptr[arity];
+ ptr[0] = (Eterm) COMPRESS_POINTER(next);
+ next = ptr + arity - 1;
+ continue;
}
- erts_print(to, to_arg, "Yc%X:%X:%X", val,
- pb->bytes - (byte *)val->orig_bytes,
- size);
- } else if (tag == SUB_BINARY_SUBTAG) {
- ErlSubBin* Sb = (ErlSubBin *) binary_val(x);
- Eterm* real_bin = binary_val(Sb->orig);
- void* val;
-
- if (thing_subtag(*real_bin) == REFC_BINARY_SUBTAG) {
- ProcBin* pb = (ProcBin *) real_bin;
- val = pb->val;
- } else { /* Heap binary */
- val = real_bin;
+ } else if (hdr == HEADER_FLONUM) {
+ FloatDef f;
+ char sbuf[31];
+ int i;
+
+ GET_DOUBLE_DATA((ptr+1), f);
+ i = sys_double_to_chars(f.fd, (char*) sbuf);
+ sys_memset(sbuf+i, 0, 31-i);
+ erts_print(to, to_arg, "F%X:%s\n", i, sbuf);
+ *ptr = OUR_NIL;
+ } else if (_is_bignum_header(hdr)) {
+ erts_print(to, to_arg, "B%T\n", x);
+ *ptr = OUR_NIL;
+ } else if (is_binary_header(hdr)) {
+ Uint tag = thing_subtag(hdr);
+ Uint size = binary_size(x);
+ Uint i;
+
+ if (tag == HEAP_BINARY_SUBTAG) {
+ byte* p;
+
+ erts_print(to, to_arg, "Yh%X:", size);
+ p = binary_bytes(x);
+ for (i = 0; i < size; i++) {
+ erts_print(to, to_arg, "%02X", p[i]);
+ }
+ } else if (tag == REFC_BINARY_SUBTAG) {
+ ProcBin* pb = (ProcBin *) binary_val(x);
+ Binary* val = pb->val;
+
+ if (erts_smp_atomic_xchg(&val->refc, 0) != 0) {
+ val->flags = (UWord) all_binaries;
+ all_binaries = val;
+ }
+ erts_print(to, to_arg, "Yc%X:%X:%X", val,
+ pb->bytes - (byte *)val->orig_bytes,
+ size);
+ } else if (tag == SUB_BINARY_SUBTAG) {
+ ErlSubBin* Sb = (ErlSubBin *) binary_val(x);
+ Eterm* real_bin = binary_val(Sb->orig);
+ void* val;
+
+ if (thing_subtag(*real_bin) == REFC_BINARY_SUBTAG) {
+ ProcBin* pb = (ProcBin *) real_bin;
+ val = pb->val;
+ } else { /* Heap binary */
+ val = real_bin;
+ }
+ erts_print(to, to_arg, "Ys%X:%X:%X", val, Sb->offs, size);
}
- erts_print(to, to_arg, "Ys%X:%X:%X", val, Sb->offs, size);
+ erts_putc(to, to_arg, '\n');
+ *ptr = OUR_NIL;
+ } else if (is_external_pid_header(hdr)) {
+ erts_print(to, to_arg, "P%T\n", x);
+ *ptr = OUR_NIL;
+ } else if (is_external_port_header(hdr)) {
+ erts_print(to, to_arg, "p<%bpu.%bpu>\n",
+ port_channel_no(x), port_number(x));
+ *ptr = OUR_NIL;
+ } else {
+ /*
+ * All other we dump in the external term format.
+ */
+ dump_externally(to, to_arg, x);
+ erts_putc(to, to_arg, '\n');
+ *ptr = OUR_NIL;
}
- erts_putc(to, to_arg, '\n');
- *ptr = OUR_NIL;
- } else if (is_external_pid_header(hdr)) {
- erts_print(to, to_arg, "P%T\n", x);
- *ptr = OUR_NIL;
- } else if (is_external_port_header(hdr)) {
- erts_print(to, to_arg, "p<%bpu.%bpu>\n",
- port_channel_no(x), port_number(x));
- *ptr = OUR_NIL;
- } else {
- /*
- * All other we dump in the external term format.
- */
- dump_externally(to, to_arg, x);
- erts_putc(to, to_arg, '\n');
- *ptr = OUR_NIL;
}
}
+ x = *next;
+ *next = OUR_NIL;
+ next--;
}
-
- x = *next;
- *next = OUR_NIL;
- next--;
- goto again;
+ UnUseTmpHeapNoproc(1);
}
static void
diff --git a/erts/emulator/beam/erl_term.c b/erts/emulator/beam/erl_term.c
index c6458a0e45..f77e8b798f 100644
--- a/erts/emulator/beam/erl_term.c
+++ b/erts/emulator/beam/erl_term.c
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2000-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2000-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -58,9 +58,9 @@ do { \
#endif
#if ET_DEBUG
-unsigned tag_val_def_debug(Eterm x, const char *file, unsigned line)
+unsigned tag_val_def_debug(Wterm x, const char *file, unsigned line)
#else
-unsigned tag_val_def(Eterm x)
+unsigned tag_val_def(Wterm x)
#define file __FILE__
#define line __LINE__
#endif
@@ -125,10 +125,10 @@ FUNTY checked_##FUN(ARGTY x, const char *file, unsigned line) \
ET_DEFINE_CHECKED(Eterm,make_boxed,Eterm*,_is_taggable_pointer);
ET_DEFINE_CHECKED(int,is_boxed,Eterm,!is_header);
-ET_DEFINE_CHECKED(Eterm*,boxed_val,Eterm,_boxed_precond);
+ET_DEFINE_CHECKED(Eterm*,boxed_val,Wterm,_boxed_precond);
ET_DEFINE_CHECKED(Eterm,make_list,Eterm*,_is_taggable_pointer);
ET_DEFINE_CHECKED(int,is_not_list,Eterm,!is_header);
-ET_DEFINE_CHECKED(Eterm*,list_val,Eterm,_list_precond);
+ET_DEFINE_CHECKED(Eterm*,list_val,Wterm,_list_precond);
ET_DEFINE_CHECKED(Uint,unsigned_val,Eterm,is_small);
ET_DEFINE_CHECKED(Sint,signed_val,Eterm,is_small);
ET_DEFINE_CHECKED(Uint,atom_val,Eterm,is_atom);
@@ -136,34 +136,35 @@ ET_DEFINE_CHECKED(Uint,header_arity,Eterm,is_header);
ET_DEFINE_CHECKED(Uint,arityval,Eterm,is_arity_value);
ET_DEFINE_CHECKED(Uint,thing_arityval,Eterm,is_thing);
ET_DEFINE_CHECKED(Uint,thing_subtag,Eterm,is_thing);
-ET_DEFINE_CHECKED(Eterm*,binary_val,Eterm,is_binary);
-ET_DEFINE_CHECKED(Eterm*,fun_val,Eterm,is_fun);
+ET_DEFINE_CHECKED(Eterm*,binary_val,Wterm,is_binary);
+ET_DEFINE_CHECKED(Eterm*,fun_val,Wterm,is_fun);
ET_DEFINE_CHECKED(int,bignum_header_is_neg,Eterm,_is_bignum_header);
ET_DEFINE_CHECKED(Eterm,bignum_header_neg,Eterm,_is_bignum_header);
ET_DEFINE_CHECKED(Uint,bignum_header_arity,Eterm,_is_bignum_header);
-ET_DEFINE_CHECKED(Eterm*,big_val,Eterm,is_big);
-ET_DEFINE_CHECKED(Eterm*,float_val,Eterm,is_float);
-ET_DEFINE_CHECKED(Eterm*,tuple_val,Eterm,is_tuple);
+ET_DEFINE_CHECKED(Eterm*,big_val,Wterm,is_big);
+ET_DEFINE_CHECKED(Eterm*,float_val,Wterm,is_float);
+ET_DEFINE_CHECKED(Eterm*,tuple_val,Wterm,is_tuple);
ET_DEFINE_CHECKED(Uint,internal_pid_data,Eterm,is_internal_pid);
ET_DEFINE_CHECKED(struct erl_node_*,internal_pid_node,Eterm,is_internal_pid);
ET_DEFINE_CHECKED(Uint,internal_port_data,Eterm,is_internal_port);
ET_DEFINE_CHECKED(struct erl_node_*,internal_port_node,Eterm,is_internal_port);
-ET_DEFINE_CHECKED(Eterm*,internal_ref_val,Eterm,is_internal_ref);
-ET_DEFINE_CHECKED(Uint,internal_ref_data_words,Eterm,is_internal_ref);
-ET_DEFINE_CHECKED(Uint32*,internal_ref_data,Eterm,is_internal_ref);
+ET_DEFINE_CHECKED(Eterm*,internal_ref_val,Wterm,is_internal_ref);
+ET_DEFINE_CHECKED(Uint,internal_ref_data_words,Wterm,is_internal_ref);
+ET_DEFINE_CHECKED(Uint32*,internal_ref_data,Wterm,is_internal_ref);
ET_DEFINE_CHECKED(struct erl_node_*,internal_ref_node,Eterm,is_internal_ref);
-ET_DEFINE_CHECKED(Eterm*,external_val,Eterm,is_external);
-ET_DEFINE_CHECKED(Uint,external_data_words,Eterm,is_external);
-ET_DEFINE_CHECKED(Uint,external_pid_data_words,Eterm,is_external_pid);
-ET_DEFINE_CHECKED(Uint,external_pid_data,Eterm,is_external_pid);
-ET_DEFINE_CHECKED(struct erl_node_*,external_pid_node,Eterm,is_external_pid);
-ET_DEFINE_CHECKED(Uint,external_port_data_words,Eterm,is_external_port);
-ET_DEFINE_CHECKED(Uint,external_port_data,Eterm,is_external_port);
-ET_DEFINE_CHECKED(struct erl_node_*,external_port_node,Eterm,is_external_port);
-ET_DEFINE_CHECKED(Uint,external_ref_data_words,Eterm,is_external_ref);
-ET_DEFINE_CHECKED(Uint32*,external_ref_data,Eterm,is_external_ref);
+ET_DEFINE_CHECKED(Eterm*,external_val,Wterm,is_external);
+ET_DEFINE_CHECKED(Uint,external_data_words,Wterm,is_external);
+ET_DEFINE_CHECKED(Uint,external_pid_data_words,Wterm,is_external_pid);
+ET_DEFINE_CHECKED(Uint,external_pid_data,Wterm,is_external_pid);
+ET_DEFINE_CHECKED(struct erl_node_*,external_pid_node,Wterm,is_external_pid);
+ET_DEFINE_CHECKED(Uint,external_port_data_words,Wterm,is_external_port);
+ET_DEFINE_CHECKED(Uint,external_port_data,Wterm,is_external_port);
+ET_DEFINE_CHECKED(struct erl_node_*,external_port_node,Wterm,is_external_port);
+ET_DEFINE_CHECKED(Uint,external_ref_data_words,Wterm,is_external_ref);
+ET_DEFINE_CHECKED(Uint32*,external_ref_data,Wterm,is_external_ref);
ET_DEFINE_CHECKED(struct erl_node_*,external_ref_node,Eterm,is_external_ref);
-ET_DEFINE_CHECKED(Eterm*,export_val,Eterm,is_export);
+ET_DEFINE_CHECKED(Eterm*,export_val,Wterm,is_export);
+ET_DEFINE_CHECKED(Uint,external_thing_data_words,ExternalThing*,is_thing_ptr);
ET_DEFINE_CHECKED(Eterm,make_cp,UWord *,_is_taggable_pointer);
ET_DEFINE_CHECKED(UWord *,cp_val,Eterm,is_CP);
diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h
index b8e4473141..1d75fa313c 100644
--- a/erts/emulator/beam/erl_term.h
+++ b/erts/emulator/beam/erl_term.h
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2000-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2000-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -22,6 +22,8 @@
#include "sys.h" /* defines HALFWORD_HEAP */
+typedef UWord Wterm; /* Full word terms */
+
#if HALFWORD_HEAP
# define HEAP_ON_C_STACK 0
# if HALFWORD_ASSERT
@@ -193,7 +195,7 @@ struct erl_node_; /* Declared in erl_node_tables.h */
#endif
#define _is_aligned(x) (((Uint)(x) & 0x3) == 0)
#define _unchecked_make_boxed(x) ((Uint) COMPRESS_POINTER(x) + TAG_PRIMARY_BOXED)
-_ET_DECLARE_CHECKED(Eterm,make_boxed,Eterm*);
+_ET_DECLARE_CHECKED(Eterm,make_boxed,Eterm*)
#define make_boxed(x) _ET_APPLY(make_boxed,(x))
#if 1
#define _is_not_boxed(x) ((x) & (_TAG_PRIMARY_MASK-TAG_PRIMARY_BOXED))
@@ -204,12 +206,12 @@ _ET_DECLARE_CHECKED(int,is_boxed,Eterm)
#define is_boxed(x) (((x) & _TAG_PRIMARY_MASK) == TAG_PRIMARY_BOXED)
#endif
#define _unchecked_boxed_val(x) ((Eterm*) EXPAND_POINTER(((x) - TAG_PRIMARY_BOXED)))
-_ET_DECLARE_CHECKED(Eterm*,boxed_val,Eterm);
+_ET_DECLARE_CHECKED(Eterm*,boxed_val,Wterm)
#define boxed_val(x) _ET_APPLY(boxed_val,(x))
/* cons cell ("list") access methods */
#define _unchecked_make_list(x) ((Uint) COMPRESS_POINTER(x) + TAG_PRIMARY_LIST)
-_ET_DECLARE_CHECKED(Eterm,make_list,Eterm*);
+_ET_DECLARE_CHECKED(Eterm,make_list,Eterm*)
#define make_list(x) _ET_APPLY(make_list,(x))
#if 1
#define _unchecked_is_not_list(x) ((x) & (_TAG_PRIMARY_MASK-TAG_PRIMARY_LIST))
@@ -226,7 +228,7 @@ _ET_DECLARE_CHECKED(int,is_not_list,Eterm)
#define _list_precond(x) (is_list(x))
#endif
#define _unchecked_list_val(x) ((Eterm*) EXPAND_POINTER((x) - TAG_PRIMARY_LIST))
-_ET_DECLARE_CHECKED(Eterm*,list_val,Eterm);
+_ET_DECLARE_CHECKED(Eterm*,list_val,Wterm)
#define list_val(x) _ET_APPLY(list_val,(x))
#define CONS(hp, car, cdr) \
@@ -240,6 +242,8 @@ _ET_DECLARE_CHECKED(Eterm*,list_val,Eterm);
#define ptr_val(x) _unchecked_ptr_val((x)) /*XXX*/
#define _unchecked_offset_ptr(x,offs) ((x)+((offs)*sizeof(Eterm)))
#define offset_ptr(x,offs) _unchecked_offset_ptr(x,offs) /*XXX*/
+#define _unchecked_byte_offset_ptr(x,byte_offs) ((x)+(offs))
+#define byte_offset_ptr(x,offs) _unchecked_byte_offset_ptr(x,offs) /*XXX*/
/* fixnum ("small") access methods */
#if defined(ARCH_64) && !HALFWORD_HEAP
@@ -305,6 +309,7 @@ _ET_DECLARE_CHECKED(Uint,arityval,Eterm)
/* thing access methods */
#define is_thing(x) (is_header((x)) && header_is_thing((x)))
+#define is_thing_ptr(t) (is_thing((t)->header))
#define _unchecked_thing_arityval(x) _unchecked_header_arity((x))
_ET_DECLARE_CHECKED(Uint,thing_arityval,Eterm)
#define thing_arityval(x) _ET_APPLY(thing_arityval,(x))
@@ -339,7 +344,7 @@ _ET_DECLARE_CHECKED(Uint,thing_subtag,Eterm)
#define is_binary(x) (is_boxed((x)) && is_binary_header(*boxed_val((x))))
#define is_not_binary(x) (!is_binary((x)))
#define _unchecked_binary_val(x) _unchecked_boxed_val((x))
-_ET_DECLARE_CHECKED(Eterm*,binary_val,Eterm)
+_ET_DECLARE_CHECKED(Eterm*,binary_val,Wterm)
#define binary_val(x) _ET_APPLY(binary_val,(x))
/* process binaries stuff (special case of binaries) */
@@ -356,7 +361,7 @@ _ET_DECLARE_CHECKED(Eterm*,binary_val,Eterm)
#define is_fun(x) (is_boxed((x)) && is_fun_header(*boxed_val((x))))
#define is_not_fun(x) (!is_fun((x)))
#define _unchecked_fun_val(x) _unchecked_boxed_val((x))
-_ET_DECLARE_CHECKED(Eterm*,fun_val,Eterm)
+_ET_DECLARE_CHECKED(Eterm*,fun_val,Wterm)
#define fun_val(x) _ET_APPLY(fun_val,(x))
/* export access methods */
@@ -364,7 +369,7 @@ _ET_DECLARE_CHECKED(Eterm*,fun_val,Eterm)
#define is_export(x) (is_boxed((x)) && is_export_header(*boxed_val((x))))
#define is_not_export(x) (!is_export((x)))
#define _unchecked_export_val(x) _unchecked_boxed_val(x)
-_ET_DECLARE_CHECKED(Eterm*,export_val,Eterm)
+_ET_DECLARE_CHECKED(Eterm*,export_val,Wterm)
#define export_val(x) _ET_APPLY(export_val,(x))
#define is_export_header(x) ((x) == HEADER_EXPORT)
#if HALFWORD_HEAP
@@ -391,7 +396,7 @@ _ET_DECLARE_CHECKED(Uint,bignum_header_arity,Eterm)
#define is_big(x) (is_boxed((x)) && _is_bignum_header(*boxed_val((x))))
#define is_not_big(x) (!is_big((x)))
#define _unchecked_big_val(x) _unchecked_boxed_val((x))
-_ET_DECLARE_CHECKED(Eterm*,big_val,Eterm)
+_ET_DECLARE_CHECKED(Eterm*,big_val,Wterm)
#define big_val(x) _ET_APPLY(big_val,(x))
/* flonum ("float") access methods */
@@ -404,7 +409,7 @@ _ET_DECLARE_CHECKED(Eterm*,big_val,Eterm)
#define is_float(x) (is_boxed((x)) && *boxed_val((x)) == HEADER_FLONUM)
#define is_not_float(x) (!is_float(x))
#define _unchecked_float_val(x) _unchecked_boxed_val((x))
-_ET_DECLARE_CHECKED(Eterm*,float_val,Eterm)
+_ET_DECLARE_CHECKED(Eterm*,float_val,Wterm)
#define float_val(x) _ET_APPLY(float_val,(x))
/* Float definition for byte and word access */
@@ -422,15 +427,16 @@ typedef union float_def
} FloatDef;
#if defined(ARCH_64) && !HALFWORD_HEAP
-#define GET_DOUBLE(x, f) (f).fdw = *(float_val(x)+1)
+
+#define FLOAT_VAL_GET_DOUBLE(fval, f) (f).fdw = *((fval)+1)
#define PUT_DOUBLE(f, x) *(x) = HEADER_FLONUM, \
*((x)+1) = (f).fdw
#define GET_DOUBLE_DATA(p, f) (f).fdw = *((Uint *) (p))
#define PUT_DOUBLE_DATA(f,p) *((Uint *) (p)) = (f).fdw
#else
-#define GET_DOUBLE(x, f) (f).fw[0] = *(float_val(x)+1), \
- (f).fw[1] = *(float_val(x)+2)
+#define FLOAT_VAL_GET_DOUBLE(fval, f) (f).fw[0] = *((fval)+1), \
+ (f).fw[1] = *((fval)+2)
#define PUT_DOUBLE(f, x) *(x) = HEADER_FLONUM, \
*((x)+1) = (f).fw[0], \
@@ -440,6 +446,9 @@ typedef union float_def
#define PUT_DOUBLE_DATA(f,p) *((Uint *) (p)) = (f).fw[0],\
*(((Uint *) (p))+1) = (f).fw[1]
#endif
+
+#define GET_DOUBLE(x, f) FLOAT_VAL_GET_DOUBLE(float_val(x), f)
+
#define DOUBLE_DATA_WORDS (sizeof(ieee754_8)/sizeof(Eterm))
#define FLOAT_SIZE_OBJECT (DOUBLE_DATA_WORDS+1)
@@ -451,7 +460,7 @@ typedef union float_def
(is_boxed((x)) && *boxed_val((x)) == make_arityval((a)))
#define is_not_tuple_arity(x, a) (!is_tuple_arity((x),(a)))
#define _unchecked_tuple_val(x) _unchecked_boxed_val(x)
-_ET_DECLARE_CHECKED(Eterm*,tuple_val,Eterm)
+_ET_DECLARE_CHECKED(Eterm*,tuple_val,Wterm)
#define tuple_val(x) _ET_APPLY(tuple_val,(x))
#define TUPLE0(t) \
@@ -790,21 +799,24 @@ do { \
((RefThing*) internal_ref_val(x))
#define is_internal_ref(x) \
- (_unchecked_is_boxed((x)) && is_ref_thing_header(*boxed_val((x))))
+ (_unchecked_is_boxed((x)) && is_ref_thing_header(*boxed_val((x))))
+
#define is_not_internal_ref(x) \
(!is_internal_ref((x)))
#define _unchecked_internal_ref_val(x) _unchecked_boxed_val((x))
-_ET_DECLARE_CHECKED(Eterm*,internal_ref_val,Eterm)
+_ET_DECLARE_CHECKED(Eterm*,internal_ref_val,Wterm)
#define internal_ref_val(x) _ET_APPLY(internal_ref_val,(x))
+#define internal_thing_ref_data_words(t) (thing_arityval(*(Eterm*)(t)))
#define _unchecked_internal_ref_data_words(x) \
(_unchecked_thing_arityval(*_unchecked_internal_ref_val(x)))
-_ET_DECLARE_CHECKED(Uint,internal_ref_data_words,Eterm)
+_ET_DECLARE_CHECKED(Uint,internal_ref_data_words,Wterm)
#define internal_ref_data_words(x) _ET_APPLY(internal_ref_data_words,(x))
-#define _unchecked_internal_ref_data(x) (_unchecked_ref_thing_ptr(x)->data.ui32)
-_ET_DECLARE_CHECKED(Uint32*,internal_ref_data,Eterm)
+#define internal_thing_ref_data(thing) ((thing)->data.ui32)
+#define _unchecked_internal_ref_data(x) (internal_thing_ref_data(_unchecked_ref_thing_ptr(x)))
+_ET_DECLARE_CHECKED(Uint32*,internal_ref_data,Wterm)
#define internal_ref_data(x) _ET_APPLY(internal_ref_data,(x))
#define _unchecked_internal_ref_node(x) erts_this_node
@@ -889,14 +901,14 @@ typedef struct external_thing_ {
#define is_external_header(x) \
(((x) & (_TAG_HEADER_MASK-_BINARY_XXX_MASK)) == _TAG_HEADER_EXTERNAL_PID)
-#define is_external(x) \
- (is_boxed((x)) && is_external_header(*boxed_val((x))))
+#define is_external(x) (is_boxed((x)) && is_external_header(*boxed_val((x))))
+
#define is_external_pid(x) \
(is_boxed((x)) && is_external_pid_header(*boxed_val((x))))
#define is_external_port(x) \
- (is_boxed((x)) && is_external_port_header(*boxed_val((x))))
-#define is_external_ref(x) \
- (_unchecked_is_boxed((x)) && is_external_ref_header(*boxed_val((x))))
+ (is_boxed((x)) && is_external_port_header(*boxed_val((x))))
+
+#define is_external_ref(x) (_unchecked_is_boxed((x)) && is_external_ref_header(*boxed_val((x))))
#define _unchecked_is_external(x) \
(_unchecked_is_boxed((x)) && is_external_header(*_unchecked_boxed_val((x))))
@@ -914,17 +926,21 @@ typedef struct external_thing_ {
#define make_external_ref make_external
#define _unchecked_external_val(x) _unchecked_boxed_val((x))
-_ET_DECLARE_CHECKED(Eterm*,external_val,Eterm)
+_ET_DECLARE_CHECKED(Eterm*,external_val,Wterm)
#define external_val(x) _ET_APPLY(external_val,(x))
#define external_thing_ptr(x) ((ExternalThing *) external_val((x)))
#define _unchecked_external_thing_ptr(x) \
((ExternalThing *) _unchecked_external_val((x)))
+#define _unchecked_external_thing_data_words(thing) \
+ (_unchecked_thing_arityval((thing)->header) + (1 - EXTERNAL_THING_HEAD_SIZE))
+_ET_DECLARE_CHECKED(Uint,external_thing_data_words,ExternalThing*)
+#define external_thing_data_words(thing) _ET_APPLY(external_thing_data_words,(thing))
+
#define _unchecked_external_data_words(x) \
- (_unchecked_thing_arityval(_unchecked_external_thing_ptr((x))->header) \
- + (1 - EXTERNAL_THING_HEAD_SIZE))
-_ET_DECLARE_CHECKED(Uint,external_data_words,Eterm)
+ _unchecked_external_thing_data_words(_unchecked_external_thing_ptr((x)))
+_ET_DECLARE_CHECKED(Uint,external_data_words,Wterm)
#define external_data_words(x) _ET_APPLY(external_data_words,(x))
#define _unchecked_external_data(x) (_unchecked_external_thing_ptr((x))->data.ui)
@@ -935,15 +951,15 @@ _ET_DECLARE_CHECKED(Uint,external_data_words,Eterm)
#define _unchecked_external_pid_data_words(x) \
_unchecked_external_data_words((x))
-_ET_DECLARE_CHECKED(Uint,external_pid_data_words,Eterm)
+_ET_DECLARE_CHECKED(Uint,external_pid_data_words,Wterm)
#define external_pid_data_words(x) _ET_APPLY(external_pid_data_words,(x))
#define _unchecked_external_pid_data(x) _unchecked_external_data((x))[0]
-_ET_DECLARE_CHECKED(Uint,external_pid_data,Eterm)
+_ET_DECLARE_CHECKED(Uint,external_pid_data,Wterm)
#define external_pid_data(x) _ET_APPLY(external_pid_data,(x))
#define _unchecked_external_pid_node(x) _unchecked_external_node((x))
-_ET_DECLARE_CHECKED(struct erl_node_*,external_pid_node,Eterm)
+_ET_DECLARE_CHECKED(struct erl_node_*,external_pid_node,Wterm)
#define external_pid_node(x) _ET_APPLY(external_pid_node,(x))
#define external_pid_number(x) _GET_PID_NUM(external_pid_data((x)))
@@ -951,27 +967,29 @@ _ET_DECLARE_CHECKED(struct erl_node_*,external_pid_node,Eterm)
#define _unchecked_external_port_data_words(x) \
_unchecked_external_data_words((x))
-_ET_DECLARE_CHECKED(Uint,external_port_data_words,Eterm)
+_ET_DECLARE_CHECKED(Uint,external_port_data_words,Wterm)
#define external_port_data_words(x) _ET_APPLY(external_port_data_words,(x))
#define _unchecked_external_port_data(x) _unchecked_external_data((x))[0]
-_ET_DECLARE_CHECKED(Uint,external_port_data,Eterm)
+_ET_DECLARE_CHECKED(Uint,external_port_data,Wterm)
#define external_port_data(x) _ET_APPLY(external_port_data,(x))
#define _unchecked_external_port_node(x) _unchecked_external_node((x))
-_ET_DECLARE_CHECKED(struct erl_node_*,external_port_node,Eterm)
+_ET_DECLARE_CHECKED(struct erl_node_*,external_port_node,Wterm)
#define external_port_node(x) _ET_APPLY(external_port_node,(x))
#define external_port_number(x) _GET_PORT_NUM(external_port_data((x)))
#define _unchecked_external_ref_data_words(x) \
_unchecked_external_data_words((x))
-_ET_DECLARE_CHECKED(Uint,external_ref_data_words,Eterm)
+_ET_DECLARE_CHECKED(Uint,external_ref_data_words,Wterm)
#define external_ref_data_words(x) _ET_APPLY(external_ref_data_words,(x))
+#define external_thing_ref_data_words(thing) external_thing_data_words(thing)
#define _unchecked_external_ref_data(x) (_unchecked_external_thing_ptr((x))->data.ui32)
-_ET_DECLARE_CHECKED(Uint32*,external_ref_data,Eterm)
+_ET_DECLARE_CHECKED(Uint32*,external_ref_data,Wterm)
#define external_ref_data(x) _ET_APPLY(external_ref_data,(x))
+#define external_thing_ref_data(thing) ((thing)->data.ui32)
#define _unchecked_external_ref_node(x) _unchecked_external_node((x))
_ET_DECLARE_CHECKED(struct erl_node_*,external_ref_node,Eterm)
@@ -995,14 +1013,14 @@ _ET_DECLARE_CHECKED(struct erl_node_*,external_ref_node,Eterm)
#endif
#define _unchecked_make_cp(x) ((Eterm) COMPRESS_POINTER(x))
-_ET_DECLARE_CHECKED(Eterm,make_cp,BeamInstr*);
+_ET_DECLARE_CHECKED(Eterm,make_cp,BeamInstr*)
#define make_cp(x) _ET_APPLY(make_cp,(x))
#define is_not_CP(x) ((x) & _CPMASK)
#define is_CP(x) (!is_not_CP(x))
#define _unchecked_cp_val(x) ((BeamInstr*) EXPAND_POINTER(x))
-_ET_DECLARE_CHECKED(BeamInstr*,cp_val,Eterm);
+_ET_DECLARE_CHECKED(BeamInstr*,cp_val,Eterm)
#define cp_val(x) _ET_APPLY(cp_val,(x))
#define make_catch(x) (((x) << _TAG_IMMED2_SIZE) | _TAG_IMMED2_CATCH)
@@ -1083,10 +1101,10 @@ _ET_DECLARE_CHECKED(Uint,y_reg_index,Uint)
#define SMALL_DEF 0xf
#if ET_DEBUG
-extern unsigned tag_val_def_debug(Eterm, const char*, unsigned);
+extern unsigned tag_val_def_debug(Wterm, const char*, unsigned);
#define tag_val_def(x) tag_val_def_debug((x),__FILE__,__LINE__)
#else
-extern unsigned tag_val_def(Eterm);
+extern unsigned tag_val_def(Wterm);
#endif
#define not_eq_tags(X,Y) (tag_val_def((X)) ^ tag_val_def((Y)))
@@ -1102,5 +1120,81 @@ extern unsigned tag_val_def(Eterm);
#define FLOAT_BIG _NUMBER_CODE(FLOAT_DEF,BIG_DEF)
#define FLOAT_FLOAT _NUMBER_CODE(FLOAT_DEF,FLOAT_DEF)
+#if HALFWORD_HEAP
+#define ptr2rel(PTR,BASE) ((Eterm*)((char*)(PTR) - (char*)(BASE)))
+#define rterm2wterm(REL,BASE) ((Wterm)(REL) + (Wterm)(BASE))
+
+#else /* HALFWORD_HEAP */
+
+#define ptr2rel(PTR,BASE) (PTR)
+#define rterm2wterm(REL,BASE) (REL)
+
+#endif /* !HALFWORD_HEAP */
+
+#define make_list_rel(PTR, BASE) make_list(ptr2rel(PTR,BASE))
+#define make_boxed_rel(PTR, BASE) make_boxed(ptr2rel(PTR,BASE))
+#define make_fun_rel make_boxed_rel
+#define make_binary_rel make_boxed_rel
+#define make_tuple_rel make_boxed_rel
+#define make_external_rel make_boxed_rel
+#define make_internal_ref_rel make_boxed_rel
+
+#define binary_val_rel(RTERM, BASE) binary_val(rterm2wterm(RTERM, BASE))
+#define list_val_rel(RTERM, BASE) list_val(rterm2wterm(RTERM, BASE))
+#define boxed_val_rel(RTERM, BASE) boxed_val(rterm2wterm(RTERM, BASE))
+#define tuple_val_rel(RTERM, BASE) tuple_val(rterm2wterm(RTERM, BASE))
+#define export_val_rel(RTERM, BASE) export_val(rterm2wterm(RTERM, BASE))
+#define fun_val_rel(RTERM, BASE) fun_val(rterm2wterm(RTERM, BASE))
+#define big_val_rel(RTERM,BASE) big_val(rterm2wterm(RTERM,BASE))
+#define float_val_rel(RTERM,BASE) float_val(rterm2wterm(RTERM,BASE))
+#define internal_ref_val_rel(RTERM,BASE) internal_ref_val(rterm2wterm(RTERM,BASE))
+
+#define external_thing_ptr_rel(RTERM, BASE) external_thing_ptr(rterm2wterm(RTERM, BASE))
+#define external_data_words_rel(RTERM,BASE) external_data_words(rterm2wterm(RTERM,BASE))
+
+#define external_port_node_rel(RTERM,BASE) external_port_node(rterm2wterm(RTERM,BASE))
+#define external_port_data_rel(RTERM,BASE) external_port_data(rterm2wterm(RTERM,BASE))
+
+#define is_external_pid_rel(RTERM,BASE) is_external_pid(rterm2wterm(RTERM,BASE))
+#define external_pid_node_rel(RTERM,BASE) external_pid_node(rterm2wterm(RTERM,BASE))
+#define external_pid_data_rel(RTERM,BASE) external_pid_data(rterm2wterm(RTERM,BASE))
+
+#define is_binary_rel(RTERM,BASE) is_binary(rterm2wterm(RTERM,BASE))
+#define is_float_rel(RTERM,BASE) is_float(rterm2wterm(RTERM,BASE))
+#define is_fun_rel(RTERM,BASE) is_fun(rterm2wterm(RTERM,BASE))
+#define is_big_rel(RTERM,BASE) is_big(rterm2wterm(RTERM,BASE))
+#define is_export_rel(RTERM,BASE) is_export(rterm2wterm(RTERM,BASE))
+#define is_tuple_rel(RTERM,BASE) is_tuple(rterm2wterm(RTERM,BASE))
+
+#define GET_DOUBLE_REL(RTERM, f, BASE) GET_DOUBLE(rterm2wterm(RTERM,BASE), f)
+
+#define ref_thing_ptr_rel(RTERM,BASE) ref_thing_ptr(rterm2wterm(RTERM,BASE))
+#define is_internal_ref_rel(RTERM,BASE) is_internal_ref(rterm2wterm(RTERM,BASE))
+#define is_external_rel(RTERM,BASE) is_external(rterm2wterm(RTERM,BASE))
+#define is_external_port_rel(RTERM,BASE) is_external_port(rterm2wterm(RTERM,BASE))
+#define is_external_ref_rel(RTERM,BASE) is_external_ref(rterm2wterm(RTERM,BASE))
+
+#define external_node_rel(RTERM,BASE) external_node(rterm2wterm(RTERM,BASE))
+
+
+#if HALFWORD_HEAP
+ERTS_GLB_INLINE int is_same(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base);
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+ERTS_GLB_INLINE int is_same(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base)
+{
+ /* If bases differ, assume a and b are on different "heaps",
+ ie can only be same if immed */
+ ASSERT(a_base == b_base || is_immed(a) || is_immed(b)
+ || rterm2wterm(a,a_base) != rterm2wterm(b,b_base));
+
+ return a == b && (a_base == b_base || is_immed(a));
+}
+#endif
+
+#else /* !HALFWORD_HEAP */
+#define is_same(A,A_BASE,B,B_BASE) ((A)==(B))
+#endif
+
#endif /* __ERL_TERM_H */
diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h
index 84a20b51f2..8c9cace0c5 100644
--- a/erts/emulator/beam/erl_threads.h
+++ b/erts/emulator/beam/erl_threads.h
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2001-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -36,6 +36,16 @@
#include "erl_lock_count.h"
#include "erl_term.h"
+#if defined(__GLIBC__) && (__GLIBC__ << 16) + __GLIBC_MINOR__ < (2 << 16) + 4
+/*
+ * pthread_mutex_destroy() may return EBUSY when it shouldn't :( We have
+ * only seen this bug in glibc versions before 2.4. Note that condition
+ * variables, rwmutexes, spinlocks, and rwspinlocks also may be effected by
+ * this bug since these implementations may use mutexes internally.
+ */
+# define ERTS_THR_HAVE_BUSY_DESTROY_BUG
+#endif
+
#define ERTS_THR_MEMORY_BARRIER ETHR_MEMORY_BARRIER
#ifdef ERTS_ENABLE_LOCK_COUNT
@@ -175,6 +185,9 @@ typedef struct { int gcc_is_buggy; } erts_rwlock_t;
#endif /* #ifdef USE_THREADS */
#define ERTS_AINT_T_MAX (~(((erts_aint_t) 1) << (sizeof(erts_aint_t)*8-1)))
+#define ERTS_AINT_T_MIN ((((erts_aint_t) 1) << (sizeof(erts_aint_t)*8-1)))
+#define ERTS_AINT32_T_MAX (~(((erts_aint32_t) 1) << (sizeof(erts_aint32_t)*8-1)))
+#define ERTS_AINT32_T_MIN ((((erts_aint32_t) 1) << (sizeof(erts_aint32_t)*8-1)))
ERTS_GLB_INLINE void erts_thr_init(erts_thr_init_data_t *id);
ERTS_GLB_INLINE void erts_thr_late_init(erts_thr_late_init_data_t *id);
@@ -551,8 +564,16 @@ erts_mtx_destroy(erts_mtx_t *mtx)
erts_lcnt_destroy_lock(&mtx->lcnt);
#endif
res = ethr_mutex_destroy(&mtx->mtx);
- if (res)
+ if (res != 0) {
+#ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG
+ if (res == EBUSY) {
+ char *warn = "Ignoring busy mutex destroy. "
+ "Most likely a bug in pthread implementation.";
+ erts_send_warning_to_logger_str_nogl(warn);
+ }
+#endif
erts_thr_fatal_error(res, "destroy mutex");
+ }
#endif
}
@@ -647,8 +668,16 @@ erts_cnd_destroy(erts_cnd_t *cnd)
{
#ifdef USE_THREADS
int res = ethr_cond_destroy(cnd);
- if (res)
+ if (res != 0) {
+#ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG
+ if (res == EBUSY) {
+ char *warn = "Ignoring busy cond destroy. "
+ "Most likely a bug in pthread implementation.";
+ erts_send_warning_to_logger_str_nogl(warn);
+ }
+#endif
erts_thr_fatal_error(res, "destroy condition variable");
+ }
#endif
}
@@ -774,8 +803,16 @@ erts_rwmtx_destroy(erts_rwmtx_t *rwmtx)
erts_lcnt_destroy_lock(&rwmtx->lcnt);
#endif
res = ethr_rwmutex_destroy(&rwmtx->rwmtx);
- if (res != 0)
+ if (res != 0) {
+#ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG
+ if (res == EBUSY) {
+ char *warn = "Ignoring busy rwmutex destroy. "
+ "Most likely a bug in pthread implementation.";
+ erts_send_warning_to_logger_str_nogl(warn);
+ }
+#endif
erts_thr_fatal_error(res, "destroy rwmutex");
+ }
#endif
}
@@ -1452,8 +1489,16 @@ erts_spinlock_destroy(erts_spinlock_t *lock)
erts_lcnt_destroy_lock(&lock->lcnt);
#endif
res = ethr_spinlock_destroy(&lock->slck);
- if (res)
- erts_thr_fatal_error(res, "destroy spinlock");
+ if (res != 0) {
+#ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG
+ if (res == EBUSY) {
+ char *warn = "Ignoring busy spinlock destroy. "
+ "Most likely a bug in pthread implementation.";
+ erts_send_warning_to_logger_str_nogl(warn);
+ }
+#endif
+ erts_thr_fatal_error(res, "destroy rwlock");
+ }
#else
(void)lock;
#endif
@@ -1562,8 +1607,16 @@ erts_rwlock_destroy(erts_rwlock_t *lock)
erts_lcnt_destroy_lock(&lock->lcnt);
#endif
res = ethr_rwlock_destroy(&lock->rwlck);
- if (res)
+ if (res != 0) {
+#ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG
+ if (res == EBUSY) {
+ char *warn = "Ignoring busy rwlock destroy. "
+ "Most likely a bug in pthread implementation.";
+ erts_send_warning_to_logger_str_nogl(warn);
+ }
+#endif
erts_thr_fatal_error(res, "destroy rwlock");
+ }
#else
(void)lock;
#endif
diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h
index 6f6b971d34..93d8ea4cb4 100644
--- a/erts/emulator/beam/erl_time.h
+++ b/erts/emulator/beam/erl_time.h
@@ -20,11 +20,15 @@
#ifndef ERL_TIME_H__
#define ERL_TIME_H__
+extern erts_smp_atomic_t do_time; /* set at clock interrupt */
+extern SysTimeval erts_first_emu_time;
+
/*
** Timer entry:
*/
typedef struct erl_timer {
struct erl_timer* next; /* next entry tiw slot or chain */
+ struct erl_timer* prev; /* prev entry tiw slot or chain */
Uint slot; /* slot in timer wheel */
Uint count; /* number of loops remaining */
int active; /* 1=activated, 0=deactivated */
@@ -39,7 +43,6 @@ typedef void (*ErlTimeoutProc)(void*);
typedef void (*ErlCancelProc)(void*);
#ifdef ERTS_SMP
-
/*
* Process and port timer
*/
@@ -61,7 +64,66 @@ void erts_create_smp_ptimer(ErtsSmpPTimer **timer_ref,
ErlTimeoutProc timeout_func,
Uint timeout);
void erts_cancel_smp_ptimer(ErtsSmpPTimer *ptimer);
+#endif
+
+/* timer-wheel api */
+void erts_init_time(void);
+void erts_set_timer(ErlTimer*, ErlTimeoutProc, ErlCancelProc, void*, Uint);
+void erts_cancel_timer(ErlTimer*);
+void erts_bump_timer(erts_aint_t);
+Uint erts_timer_wheel_memory_size(void);
+Uint erts_time_left(ErlTimer *);
+erts_aint_t erts_next_time(void);
+
+#ifdef DEBUG
+void erts_p_slpq(void);
#endif
+ERTS_GLB_INLINE erts_aint_t erts_do_time_read_and_reset(void);
+ERTS_GLB_INLINE void erts_do_time_add(long);
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+ERTS_GLB_INLINE erts_aint_t erts_do_time_read_and_reset(void) { return erts_smp_atomic_xchg(&do_time, 0L); }
+ERTS_GLB_INLINE void erts_do_time_add(long elapsed) { erts_smp_atomic_add(&do_time, elapsed); }
+
+#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
+
+
+/* time_sup */
+
+#if (defined(HAVE_GETHRVTIME) || defined(HAVE_CLOCK_GETTIME))
+# ifndef HAVE_ERTS_NOW_CPU
+# define HAVE_ERTS_NOW_CPU
+# ifdef HAVE_GETHRVTIME
+# define erts_start_now_cpu() sys_start_hrvtime()
+# define erts_stop_now_cpu() sys_stop_hrvtime()
+# endif
+# endif
+void erts_get_now_cpu(Uint* megasec, Uint* sec, Uint* microsec);
#endif
+
+void erts_get_timeval(SysTimeval *tv);
+long erts_get_time(void);
+void erts_get_emu_time(SysTimeval *);
+
+ERTS_GLB_INLINE int erts_cmp_timeval(SysTimeval *t1p, SysTimeval *t2p);
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+ERTS_GLB_INLINE int
+erts_cmp_timeval(SysTimeval *t1p, SysTimeval *t2p)
+{
+ if (t1p->tv_sec == t2p->tv_sec) {
+ if (t1p->tv_usec < t2p->tv_usec)
+ return -1;
+ else if (t1p->tv_usec > t2p->tv_usec)
+ return 1;
+ return 0;
+ }
+ return t1p->tv_sec < t2p->tv_sec ? -1 : 1;
+}
+
+#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
+#endif /* ERL_TIME_H__ */
diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c
index 7b8706ea13..ca4b54188e 100644
--- a/erts/emulator/beam/erl_time_sup.c
+++ b/erts/emulator/beam/erl_time_sup.c
@@ -358,10 +358,6 @@ static int clock_resolution;
** instead of something like select.
*/
-#if defined(ERTS_TIMER_THREAD)
-static ERTS_INLINE void init_erts_deliver_time(const SysTimeval *inittv) { }
-static ERTS_INLINE void do_erts_deliver_time(const SysTimeval *current) { }
-#else
static SysTimeval last_delivered;
static void init_erts_deliver_time(const SysTimeval *inittv)
@@ -389,11 +385,10 @@ static void do_erts_deliver_time(const SysTimeval *current)
this by simply pretend as if the time stood still. :) */
if (elapsed > 0) {
- do_time_add(elapsed);
+ erts_do_time_add(elapsed);
last_delivered = cur_time;
}
}
-#endif
int
erts_init_time_sup(void)
@@ -786,7 +781,6 @@ get_sys_now(Uint* megasec, Uint* sec, Uint* microsec)
to a struct timeval representing current time (to save
a gettimeofday() where possible) or NULL */
-#if !defined(ERTS_TIMER_THREAD)
void erts_deliver_time(void) {
SysTimeval now;
@@ -797,7 +791,6 @@ void erts_deliver_time(void) {
erts_smp_mtx_unlock(&erts_timeofday_mtx);
}
-#endif
/* get *real* time (not ticks) remaining until next timeout - if there
isn't one, give a "long" time, that is guaranteed
@@ -806,14 +799,12 @@ void erts_deliver_time(void) {
void erts_time_remaining(SysTimeval *rem_time)
{
int ticks;
-#if !defined(ERTS_TIMER_THREAD)
SysTimeval cur_time;
-#endif
long elapsed;
- /* next_time() returns no of ticks to next timeout or -1 if none */
+ /* erts_next_time() returns no of ticks to next timeout or -1 if none */
- if ((ticks = next_time()) == -1) {
+ if ((ticks = erts_next_time()) == -1) {
/* timer queue empty */
/* this will cause at most 100000000 ticks */
rem_time->tv_sec = 100000;
@@ -822,9 +813,6 @@ void erts_time_remaining(SysTimeval *rem_time)
/* next timeout after ticks ticks */
ticks *= CLOCK_RESOLUTION;
-#if defined(ERTS_TIMER_THREAD)
- elapsed = 0;
-#else
erts_smp_mtx_lock(&erts_timeofday_mtx);
get_tolerant_timeofday(&cur_time);
@@ -839,7 +827,6 @@ void erts_time_remaining(SysTimeval *rem_time)
rem_time->tv_sec = rem_time->tv_usec = 0;
return;
}
-#endif
rem_time->tv_sec = (ticks - elapsed) / 1000;
rem_time->tv_usec = 1000 * ((ticks - elapsed) % 1000);
}
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index 3043bb1e8c..c0397ca6c3 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 1999-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -1668,7 +1668,7 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec,
return_flags = 0;
if (match_spec) {
pam_result = erts_match_set_run(p, match_spec, args, arity,
- &return_flags);
+ ERTS_PAM_TMP_RESULT, &return_flags);
if (is_non_value(pam_result)) {
erts_match_set_release_result(p);
#if !HEAP_ON_C_STACK
@@ -1815,7 +1815,7 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec,
return_flags = 0;
if (match_spec) {
pam_result = erts_match_set_run(p, match_spec, args, arity,
- &return_flags);
+ ERTS_PAM_TMP_RESULT, &return_flags);
if (is_non_value(pam_result)) {
erts_match_set_release_result(p);
UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h
index 48fa99934e..e7fd144ec3 100644
--- a/erts/emulator/beam/erl_vm.h
+++ b/erts/emulator/beam/erl_vm.h
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 1996-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -47,7 +47,7 @@
#define SEQ_TRACE 1
#define CONTEXT_REDS 2000 /* Swap process out after this number */
-#define MAX_ARG 256 /* Max number of arguments allowed */
+#define MAX_ARG 255 /* Max number of arguments allowed */
#define MAX_REG 1024 /* Max number of x(N) registers used */
/* Scheduler stores data for temporary heaps if
@@ -120,14 +120,15 @@
* Allocate heap memory, first on the ordinary heap;
* failing that, in a heap fragment.
*/
-#define HAlloc(p, sz) \
+#define HAllocX(p, sz, xtra) \
(ASSERT_EXPR((sz) >= 0), \
ErtsHAllocLockCheck(p), \
(IS_FORCE_HEAP_FRAGS || (((HEAP_LIMIT(p) - HEAP_TOP(p)) < (sz))) \
- ? erts_heap_alloc((p),(sz)) \
+ ? erts_heap_alloc((p),(sz),(xtra)) \
: (INIT_HEAP_MEM(p,sz), \
HEAP_TOP(p) = HEAP_TOP(p) + (sz), HEAP_TOP(p) - (sz))))
+#define HAlloc(P, SZ) HAllocX(P,SZ,0)
#define HRelease(p, endp, ptr) \
if ((ptr) == (endp)) { \
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index c948af14ae..bd540eaaa6 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 1996-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -890,9 +890,31 @@ void erl_error(char*, va_list);
/* copy.c */
void init_copy(void);
Eterm copy_object(Eterm, Process*);
+
+#if HALFWORD_HEAP
+Uint size_object_rel(Eterm, Eterm*);
+# define size_object(A) size_object_rel(A,NULL)
+
+Eterm copy_struct_rel(Eterm, Uint, Eterm**, ErlOffHeap*, Eterm* src_base, Eterm* dst_base);
+# define copy_struct(OBJ,SZ,HPP,OH) copy_struct_rel(OBJ,SZ,HPP,OH, NULL,NULL)
+
+Eterm copy_shallow_rel(Eterm*, Uint, Eterm**, ErlOffHeap*, Eterm* src_base);
+# define copy_shallow(A,B,C,D) copy_shallow_rel(A,B,C,D,NULL)
+
+#else /* !HALFWORD_HEAP */
+
Uint size_object(Eterm);
+# define size_object_rel(A,B) size_object(A)
+
Eterm copy_struct(Eterm, Uint, Eterm**, ErlOffHeap*);
+# define copy_struct_rel(OBJ,SZ,HPP,OH, SB,DB) copy_struct(OBJ,SZ,HPP,OH)
+
Eterm copy_shallow(Eterm*, Uint, Eterm**, ErlOffHeap*);
+# define copy_shallow_rel(A,B,C,D, BASE) copy_shallow(A,B,C,D)
+
+#endif
+
+
void move_multi_frags(Eterm** hpp, ErlOffHeap*, ErlHeapFragment* first,
Eterm* refs, unsigned nrefs);
@@ -1425,84 +1447,6 @@ void erl_drv_thr_init(void);
/* time.c */
-ERTS_GLB_INLINE erts_aint_t do_time_read_and_reset(void);
-#ifdef ERTS_TIMER_THREAD
-ERTS_GLB_INLINE int next_time(void);
-ERTS_GLB_INLINE void bump_timer(erts_aint_t);
-#else
-erts_aint_t next_time(void);
-void bump_timer(erts_aint_t);
-extern erts_smp_atomic_t do_time; /* set at clock interrupt */
-ERTS_GLB_INLINE void do_time_add(erts_aint_t);
-#endif
-
-#if ERTS_GLB_INLINE_INCL_FUNC_DEF
-
-#ifdef ERTS_TIMER_THREAD
-ERTS_GLB_INLINE erts_aint_t do_time_read_and_reset(void) { return 0; }
-ERTS_GLB_INLINE erts_aint_t next_time(void) { return -1; }
-ERTS_GLB_INLINE void bump_timer(erts_aint_t ignore) { }
-#else
-ERTS_GLB_INLINE erts_aint_t do_time_read_and_reset(void)
-{
- return erts_smp_atomic_xchg(&do_time, 0L);
-}
-ERTS_GLB_INLINE void do_time_add(erts_aint_t elapsed)
-{
- erts_smp_atomic_add(&do_time, elapsed);
-}
-#endif
-
-#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
-
-void init_time(void);
-void erl_set_timer(ErlTimer*, ErlTimeoutProc, ErlCancelProc, void*, Uint);
-void erl_cancel_timer(ErlTimer*);
-Uint time_left(ErlTimer *);
-
-Uint erts_timer_wheel_memory_size(void);
-
-#if (defined(HAVE_GETHRVTIME) || defined(HAVE_CLOCK_GETTIME))
-# ifndef HAVE_ERTS_NOW_CPU
-# define HAVE_ERTS_NOW_CPU
-# ifdef HAVE_GETHRVTIME
-# define erts_start_now_cpu() sys_start_hrvtime()
-# define erts_stop_now_cpu() sys_stop_hrvtime()
-# endif
-# endif
-void erts_get_now_cpu(Uint* megasec, Uint* sec, Uint* microsec);
-#endif
-
-void erts_get_timeval(SysTimeval *tv);
-long erts_get_time(void);
-
-extern SysTimeval erts_first_emu_time;
-
-void erts_get_emu_time(SysTimeval *);
-
-ERTS_GLB_INLINE int erts_cmp_timeval(SysTimeval *t1p, SysTimeval *t2p);
-
-#if ERTS_GLB_INLINE_INCL_FUNC_DEF
-
-ERTS_GLB_INLINE int
-erts_cmp_timeval(SysTimeval *t1p, SysTimeval *t2p)
-{
- if (t1p->tv_sec == t2p->tv_sec) {
- if (t1p->tv_usec < t2p->tv_usec)
- return -1;
- else if (t1p->tv_usec > t2p->tv_usec)
- return 1;
- return 0;
- }
- return t1p->tv_sec < t2p->tv_sec ? -1 : 1;
-}
-
-#endif
-
-#ifdef DEBUG
-void p_slpq(void);
-#endif
-
/* utils.c */
/*
@@ -1561,16 +1505,30 @@ void erts_init_utils_mem(void);
erts_dsprintf_buf_t *erts_create_tmp_dsbuf(Uint);
void erts_destroy_tmp_dsbuf(erts_dsprintf_buf_t *);
+#if HALFWORD_HEAP
+int eq_rel(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base);
+# define eq(A,B) eq_rel(A,NULL,B,NULL)
+#else
int eq(Eterm, Eterm);
+# define eq_rel(A,A_BASE,B,B_BASE) eq(A,B)
+#endif
+
#define EQ(x,y) (((x) == (y)) || (is_not_both_immed((x),(y)) && eq((x),(y))))
+#if HALFWORD_HEAP
+Sint cmp_rel(Eterm, Eterm*, Eterm, Eterm*);
+#define CMP(A,B) cmp_rel(A,NULL,B,NULL)
+#else
Sint cmp(Eterm, Eterm);
-#define cmp_lt(a,b) (cmp((a),(b)) < 0)
-#define cmp_le(a,b) (cmp((a),(b)) <= 0)
-#define cmp_eq(a,b) (cmp((a),(b)) == 0)
-#define cmp_ne(a,b) (cmp((a),(b)) != 0)
-#define cmp_ge(a,b) (cmp((a),(b)) >= 0)
-#define cmp_gt(a,b) (cmp((a),(b)) > 0)
+#define cmp_rel(A,A_BASE,B,B_BASE) cmp(A,B)
+#define CMP(A,B) cmp(A,B)
+#endif
+#define cmp_lt(a,b) (CMP((a),(b)) < 0)
+#define cmp_le(a,b) (CMP((a),(b)) <= 0)
+#define cmp_eq(a,b) (CMP((a),(b)) == 0)
+#define cmp_ne(a,b) (CMP((a),(b)) != 0)
+#define cmp_ge(a,b) (CMP((a),(b)) >= 0)
+#define cmp_gt(a,b) (CMP((a),(b)) > 0)
#define CMP_LT(a,b) ((a) != (b) && cmp_lt((a),(b)))
#define CMP_GE(a,b) ((a) == (b) || cmp_ge((a),(b)))
@@ -1799,8 +1757,15 @@ do { \
extern Binary *erts_match_set_compile(Process *p, Eterm matchexpr);
Eterm erts_match_set_lint(Process *p, Eterm matchexpr);
extern void erts_match_set_release_result(Process* p);
+
+enum erts_pam_run_flags {
+ ERTS_PAM_TMP_RESULT=0,
+ ERTS_PAM_COPY_RESULT=1,
+ ERTS_PAM_CONTIGUOUS_TUPLE=2
+};
extern Eterm erts_match_set_run(Process *p, Binary *mpsp,
Eterm *args, int num_args,
+ enum erts_pam_run_flags in_flags,
Uint32 *return_flags);
extern Eterm erts_match_set_get_source(Binary *mpsp);
extern void erts_match_prog_foreach_offheap(Binary *b,
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index f6c6a01fb2..f21a96c754 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -670,7 +670,7 @@ erts_open_driver(erts_driver_t* driver, /* Pointer to driver. */
#ifdef ERTS_SMP
erts_cancel_smp_ptimer(port->ptimer);
#else
- erl_cancel_timer(&(port->tm));
+ erts_cancel_timer(&(port->tm));
#endif
stopq(port);
kill_port(port);
@@ -1839,7 +1839,7 @@ terminate_port(Port *prt)
#ifdef ERTS_SMP
erts_cancel_smp_ptimer(prt->ptimer);
#else
- erl_cancel_timer(&prt->tm);
+ erts_cancel_timer(&prt->tm);
#endif
drv = prt->drv_ptr;
@@ -4068,7 +4068,7 @@ drv_cancel_timer(Port *prt)
#ifdef ERTS_SMP
erts_cancel_smp_ptimer(prt->ptimer);
#else
- erl_cancel_timer(&prt->tm);
+ erts_cancel_timer(&prt->tm);
#endif
if (erts_port_task_is_scheduled(&prt->timeout_task))
erts_port_task_abort(prt->id, &prt->timeout_task);
@@ -4092,7 +4092,7 @@ int driver_set_timer(ErlDrvPort ix, UWord t)
(ErlTimeoutProc) schedule_port_timeout,
t);
#else
- erl_set_timer(&prt->tm,
+ erts_set_timer(&prt->tm,
(ErlTimeoutProc) schedule_port_timeout,
NULL,
prt,
@@ -4123,9 +4123,9 @@ driver_read_timer(ErlDrvPort ix, unsigned long* t)
return -1;
ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
#ifdef ERTS_SMP
- *t = prt->ptimer ? time_left(&prt->ptimer->timer.tm) : 0;
+ *t = prt->ptimer ? erts_time_left(&prt->ptimer->timer.tm) : 0;
#else
- *t = time_left(&prt->tm);
+ *t = erts_time_left(&prt->tm);
#endif
return 0;
}
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index a2439d5582..e861f97e7a 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -101,16 +101,16 @@ return
%macro: test_heap TestHeap -pack
allocate t t
-allocate_heap I I I
+allocate_heap t I t
deallocate I
init y
allocate_zero t t
-allocate_heap_zero I I I
+allocate_heap_zero t I t
trim N Remaining => i_trim N
i_trim I
-test_heap I I
+test_heap I t
allocate_heap S u==0 R => allocate S R
allocate_heap_zero S u==0 R => allocate_zero S R
@@ -124,7 +124,7 @@ init Y1 | init Y2 => init2 Y1 Y2
# Selecting values
-select_val S=q Fail=f Size=u Rest=* => const_select_val(S, Fail, Size, Rest)
+select_val S=aiq Fail=f Size=u Rest=* => const_select_val(S, Fail, Size, Rest)
select_val S=s Fail=f Size=u Rest=* | use_jump_tab(Size, Rest) => \
gen_jump_tab(S, Fail, Size, Rest)
@@ -132,34 +132,59 @@ select_val S=s Fail=f Size=u Rest=* | use_jump_tab(Size, Rest) => \
is_integer Fail=f S | select_val S=s Fail=f Size=u Rest=* | use_jump_tab(Size, Rest) => \
gen_jump_tab(S, Fail, Size, Rest)
+is_integer TypeFail=f S | select_val S=s Fail=f Size=u Rest=* | \
+ mixed_types(Size, Rest) => \
+ gen_split_values(S, TypeFail, Fail, Size, Rest)
+
select_val S=s Fail=f Size=u Rest=* | mixed_types(Size, Rest) => \
- gen_split_values(S, Fail, Size, Rest)
+ gen_split_values(S, Fail, Fail, Size, Rest)
-is_integer Fail=f S | select_val S=s Fail=f Size=u Rest=* | \
+is_integer Fail=f S | select_val S=d Fail=f Size=u Rest=* | \
fixed_size_values(Size, Rest) => gen_select_val(S, Fail, Size, Rest)
-is_atom Fail=f S | select_val S=s Fail=f Size=u Rest=* | \
+is_atom Fail=f S | select_val S=d Fail=f Size=u Rest=* | \
fixed_size_values(Size, Rest) => gen_select_val(S, Fail, Size, Rest)
-select_val S=s Fail=f Size=u Rest=* | fixed_size_values(Size, Rest) => \
- gen_select_val(S, Fail, Size, Rest)
+select_val S=s Fail=f Size=u Rest=* | floats_or_bignums(Size, Rest) => \
+ gen_select_literals(S, Fail, Size, Rest)
-select_val S=s Fail=f Size=u Rest=* | all_values_are_big(Size, Rest) => \
- gen_select_big(S, Fail, Size, Rest)
+select_val S=d Fail=f Size=u Rest=* | fixed_size_values(Size, Rest) => \
+ gen_select_val(S, Fail, Size, Rest)
-is_tuple Fail=f S | select_tuple_arity S=s Fail=f Size=u Rest=* => \
+is_tuple Fail=f S | select_tuple_arity S=d Fail=f Size=u Rest=* => \
gen_select_tuple_arity(S, Fail, Size, Rest)
-select_tuple_arity S=s Fail=f Size=u Rest=* => \
+select_tuple_arity S=d Fail=f Size=u Rest=* => \
gen_select_tuple_arity(S, Fail, Size, Rest)
-i_select_val s f I
-i_select_tuple_arity s f I
-i_select_big s f
-i_select_float s f I
+i_select_val r f I
+i_select_val x f I
+i_select_val y f I
+
+i_select_val2 r f c f c f
+i_select_val2 x f c f c f
+i_select_val2 y f c f c f
+
+i_select_tuple_arity2 r f A f A f
+i_select_tuple_arity2 x f A f A f
+i_select_tuple_arity2 y f A f A f
+
+i_select_tuple_arity r f I
+i_select_tuple_arity x f I
+i_select_tuple_arity y f I
+
+i_jump_on_val_zero r f I
+i_jump_on_val_zero x f I
+i_jump_on_val_zero y f I
+
+i_jump_on_val r f I I
+i_jump_on_val x f I I
+i_jump_on_val y f I I
-i_jump_on_val_zero s f I
-i_jump_on_val s f I I
+jump Target | label Lbl | same_label(Target, Lbl) => label Lbl
+
+is_ne_exact L1 S1 S2 | jump Fail | label L2 | same_label(L1, L2) => \
+ is_eq_exact Fail S1 S2 | label L2
%macro: get_list GetList -pack
get_list x x x
@@ -234,11 +259,17 @@ is_number Fail Literal=q => move Literal x | is_number Fail x
jump f
-case_end Literal=q => move Literal x | case_end x
-badmatch Literal=q => move Literal x | badmatch x
+case_end Literal=cq => move Literal x | case_end x
+badmatch Literal=cq => move Literal x | badmatch x
+
+case_end r
+case_end x
+case_end y
+
+badmatch r
+badmatch x
+badmatch y
-case_end s
-badmatch s
if_end
raise s s
@@ -248,12 +279,33 @@ system_limit j
move R R =>
+move C=cxy r | jump Lbl => move_jump Lbl C
+
+%macro: move_jump MoveJump -nonext
+move_jump f n
+move_jump f c
+move_jump f x
+move_jump f y
+
move X1=x Y1=y | move X2=x Y2=y => move2 X1 Y1 X2 Y2
move Y1=y X1=x | move Y2=y X2=x => move2 Y1 X1 Y2 X2
+move X1=x X2=x | move X3=x X4=x => move2 X1 X2 X3 X4
+
+move C=aiq X=x==1 => move_x1 C
+move C=aiq X=x==2 => move_x2 C
+
+move_x1 c
+move_x2 c
%macro: move2 Move2 -pack
move2 x y x y
move2 y x y x
+move2 x x x x
+
+# The compiler almost never generates a "move Literal y(Y)" instruction,
+# so let's cheat if we encounter one.
+move S=n D=y => init D
+move S=c D=y => move S x | move x D
%macro:move Move -pack -gen_dest
move x x
@@ -265,15 +317,10 @@ move r x
move r y
move c r
move c x
-move c y
move n x
move n r
move y y
-%cold
-move s d
-%hot
-
# Receive operations.
loop_rec Fail Src | smp_mark_target_label(Fail) => i_loop_rec Fail Src
@@ -306,55 +353,78 @@ i_wait_error_locked
send
#
-# Comparisions.
+# Optimized comparisons with one immediate/literal operand.
+#
+
+is_eq_exact Lbl R=rxy C=ian => i_is_eq_exact_immed Lbl R C
+is_eq_exact Lbl R=rxy C=q => i_is_eq_exact_literal R Lbl C
+
+is_ne_exact Lbl R=rxy C=ian => i_is_ne_exact_immed Lbl R C
+is_ne_exact Lbl R=rxy C=q => i_is_ne_exact_literal R Lbl C
+
+%macro: i_is_eq_exact_immed EqualImmed -fail_action
+i_is_eq_exact_immed f r c
+i_is_eq_exact_immed f x c
+i_is_eq_exact_immed f y c
+
+i_is_eq_exact_literal r f c
+i_is_eq_exact_literal x f c
+i_is_eq_exact_literal y f c
+
+%macro: i_is_ne_exact_immed NotEqualImmed -fail_action
+i_is_ne_exact_immed f r c
+i_is_ne_exact_immed f x c
+i_is_ne_exact_immed f y c
+
+i_is_ne_exact_literal r f c
+i_is_ne_exact_literal x f c
+i_is_ne_exact_literal y f c
+
+#
+# All other comparisons.
#
-is_eq_exact Lbl=f R=rxy C=ian => i_is_eq_immed Lbl R C
-is_eq Lbl=f R=rxy C=an => i_is_eq_immed Lbl R C
+is_eq_exact Lbl S1 S2 => i_fetch S1 S2 | i_is_eq_exact Lbl
+is_ne_exact Lbl S1 S2 => i_fetch S1 S2 | i_is_ne_exact Lbl
is_ge Lbl S1 S2 => i_fetch S1 S2 | i_is_ge Lbl
is_lt Lbl S1 S2 => i_fetch S1 S2 | i_is_lt Lbl
is_eq Lbl S1 S2 => i_fetch S1 S2 | i_is_eq Lbl
is_ne Lbl S1 S2 => i_fetch S1 S2 | i_is_ne Lbl
-is_eq_exact Lbl=f S1 S2 => i_fetch S1 S2 | i_is_eq_exact Lbl
-is_ne_exact Lbl S1 S2 => i_fetch S1 S2 | i_is_ne_exact Lbl
-
+i_is_eq_exact f
+i_is_ne_exact f
i_is_lt f
i_is_ge f
i_is_eq f
i_is_ne f
-i_is_eq_exact f
-i_is_ne_exact f
-
-%macro: i_is_eq_immed EqualImmed -fail_action
-i_is_eq_immed f r c
-i_is_eq_immed f x c
-i_is_eq_immed f y c
#
# Putting things.
#
-put_tuple Arity Dst | put V => i_put_tuple Arity V Dst
+put_tuple Arity Dst => i_put_tuple Dst u
-%macro: i_put_tuple PutTuple -pack
-i_put_tuple A x x
-i_put_tuple A y x
-i_put_tuple A r x
-i_put_tuple A n x
-i_put_tuple A c x
-i_put_tuple A x y
-i_put_tuple A x r
-i_put_tuple A y r
-i_put_tuple A n r
-i_put_tuple A c r
+i_put_tuple Dst Arity Puts=* | put S1 | put S2 | \
+ put S3 | put S4 | put S5 => \
+ tuple_append_put5(Arity, Dst, Puts, S1, S2, S3, S4, S5)
-%cold
-i_put_tuple A r y
-i_put_tuple A y y
-i_put_tuple A c y
-%hot
+i_put_tuple Dst Arity Puts=* | put S => \
+ tuple_append_put(Arity, Dst, Puts, S)
+
+i_put_tuple/2
+
+%macro:i_put_tuple PutTuple -pack -goto:do_put_tuple
+i_put_tuple r I
+i_put_tuple x I
+i_put_tuple y I
+
+#
+# The instruction "put_list Const [] Dst" will not be generated by
+# the current BEAM compiler. But until R15A, play it safe by handling
+# that instruction with the following transformation.
+#
+put_list Const=c n Dst => move Const x | put_list x n Dst
%macro:put_list PutList -pack -gen_dest
@@ -362,10 +432,8 @@ put_list x n x
put_list y n x
put_list x x x
put_list y x x
-put_list c n x
put_list x x r
put_list y r r
-put_list c n r
put_list y y x
put_list x y x
@@ -376,6 +444,13 @@ put_list y y r
put_list y r x
put_list r n x
+put_list x r x
+put_list x y r
+put_list y x r
+put_list y x x
+
+put_list x r r
+
# put_list SrcReg Constant Dst
put_list r c r
put_list r c x
@@ -403,17 +478,9 @@ put_list c y x
put_list c y y
%cold
-put_list x r r
put_list s s d
%hot
-%macro: put Put
-put x
-put r
-put y
-put c
-put n
-
%macro: i_fetch FetchArgs -pack
i_fetch c c
i_fetch c r
@@ -464,19 +531,20 @@ move_return n r
move S r | deallocate D | return => move_deallocate_return S r D
-%macro: move_deallocate_return MoveDeallocateReturn -nonext
-move_deallocate_return x r P
-move_deallocate_return y r P
-move_deallocate_return c r P
-move_deallocate_return n r P
+%macro: move_deallocate_return MoveDeallocateReturn -pack -nonext
+move_deallocate_return x r Q
+move_deallocate_return y r Q
+move_deallocate_return c r Q
+move_deallocate_return n r Q
deallocate D | return => deallocate_return D
%macro: deallocate_return DeallocateReturn -nonext
-deallocate_return P
+deallocate_return Q
test_heap Need u==1 | put_list Y=y r r => test_heap_1_put_list Need Y
+%macro: test_heap_1_put_list TestHeapPutList -pack
test_heap_1_put_list I y
# Test tuple & arity (head)
@@ -576,14 +644,14 @@ is_list f y
is_nonempty_list Fail=f S=rx | allocate Need Rs => is_nonempty_list_allocate Fail S Need Rs
-%macro:is_nonempty_list_allocate IsNonemptyListAllocate -fail_action
-is_nonempty_list_allocate f x I I
-is_nonempty_list_allocate f r I I
+%macro:is_nonempty_list_allocate IsNonemptyListAllocate -fail_action -pack
+is_nonempty_list_allocate f x I t
+is_nonempty_list_allocate f r I t
is_nonempty_list F=f r | test_heap I1 I2 => is_non_empty_list_test_heap F r I1 I2
-%macro: is_non_empty_list_test_heap IsNonemptyListTestHeap -fail_action
-is_non_empty_list_test_heap f r I I
+%macro: is_non_empty_list_test_heap IsNonemptyListTestHeap -fail_action -pack
+is_non_empty_list_test_heap f r I t
%macro: is_nonempty_list IsNonemptyList -fail_action
is_nonempty_list f x
@@ -912,8 +980,13 @@ node x
node y
%hot
-i_fast_element j I s d
-i_element j s s d
+i_fast_element r j I d
+i_fast_element x j I d
+i_fast_element y j I d
+
+i_element r j s d
+i_element x j s d
+i_element y j s d
bif1 f b s d
bif1_body b s d
@@ -940,11 +1013,11 @@ move S r | call_last Ar P=f D => move_call_last S r P D
i_move_call_last f P c r
-%macro:move_call_last MoveCallLast -arg_f -nonext
+%macro:move_call_last MoveCallLast -arg_f -nonext -pack
move_call_last/4
-move_call_last x r f P
-move_call_last y r f P
+move_call_last x r f Q
+move_call_last y r f Q
move S=c r | call_only Ar P=f => i_move_call_only P S r
move S=x r | call_only Ar P=f => move_call_only S r P
@@ -1307,6 +1380,8 @@ fconv Arg=iqan Dst=l => move Arg x | fconv x Dst
fmove q l
fmove d l
+fmove l d
+
fconv d l
i_fadd l l l
@@ -1322,12 +1397,6 @@ fcheckerror p => i_fcheckerror
i_fcheckerror
fclearerror
-fmove FR=l Dst=d | new_float_allocation() => fmove_new FR Dst
-
-# The new instruction for moving a float out of a floating point register.
-# (No allocation.)
-fmove_new l d
-
#
# New apply instructions in R10B.
#
@@ -1336,7 +1405,21 @@ apply I
apply_last I P
#
-# New GCing arithmetic instructions.
+# Optimize addition and subtraction of small literals using
+# the i_increment/4 instruction (in bodies, not in guards).
+#
+
+gc_bif2 p Live u$bif:erlang:splus/2 Int=i Reg=d Dst => \
+ gen_increment(Reg, Int, Live, Dst)
+gc_bif2 p Live u$bif:erlang:splus/2 Reg=d Int=i Dst => \
+ gen_increment(Reg, Int, Live, Dst)
+
+gc_bif2 p Live u$bif:erlang:sminus/2 Reg=d Int=i Dst | \
+ negation_is_small(Int) => \
+ gen_increment_from_minus(Reg, Int, Live, Dst)
+
+#
+# GCing arithmetic instructions.
#
gc_bif2 Fail I u$bif:erlang:splus/2 S1 S2 Dst=d => i_fetch S1 S2 | i_plus Fail I Dst
@@ -1359,6 +1442,10 @@ gc_bif1 Fail I u$bif:erlang:bnot/1 Src Dst=d => i_int_bnot Fail Src I Dst
gc_bif1 Fail I u$bif:erlang:sminus/1 Src Dst=d => i_fetch i Src | i_minus Fail I Dst
gc_bif1 Fail I u$bif:erlang:splus/1 Src Dst=d => i_fetch i Src | i_plus Fail I Dst
+i_increment r I I d
+i_increment x I I d
+i_increment y I I d
+
i_plus j I d
i_minus j I d
i_times j I d
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 5a4dad0a28..ff828ae889 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 1996-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -39,13 +39,6 @@
#define ENABLE_CHILD_WAITER_THREAD 1
#endif
-/* The ERTS_TIMER_TREAD #define must be visible to the
- erl_${OS}_sys.h #include files: it controls whether
- certain optional facilities should be defined or not. */
-#if defined(ERTS_SMP) && 0
-#define ERTS_TIMER_THREAD
-#endif
-
#if defined (__WIN32__)
# include "erl_win_sys.h"
#elif defined (VXWORKS)
@@ -232,14 +225,14 @@ int real_printf(const char *fmt, ...);
#else
#error Neither 32 nor 64 bit architecture
#endif
-#ifdef ARCH_64
-# ifdef HALFWORD_HEAP_EMULATOR
+#if defined(ARCH_64) && defined(HALFWORD_HEAP_EMULATOR)
# define HALFWORD_HEAP 1
# define HALFWORD_ASSERT 0
-# else
+# define ASSERT_HALFWORD(COND) ASSERT(COND)
+#else
# define HALFWORD_HEAP 0
# define HALFWORD_ASSERT 0
-# endif
+# define ASSERT_HALFWORD(COND)
#endif
#if SIZEOF_VOID_P != SIZEOF_SIZE_T
@@ -338,12 +331,16 @@ typedef unsigned char byte;
(((size_t) 8) - (((size_t) (X)) & ((size_t) 7)))
#include "erl_lock_check.h"
+
+/* needed by erl_smp.h */
+int erts_send_warning_to_logger_str_nogl(char *);
+
#include "erl_smp.h"
#ifdef ERTS_WANT_BREAK_HANDLING
# ifdef ERTS_SMP
-extern erts_smp_atomic_t erts_break_requested;
-# define ERTS_BREAK_REQUESTED ((int) erts_smp_atomic_read(&erts_break_requested))
+extern erts_smp_atomic32_t erts_break_requested;
+# define ERTS_BREAK_REQUESTED ((int) erts_smp_atomic32_read(&erts_break_requested))
# else
extern volatile int erts_break_requested;
# define ERTS_BREAK_REQUESTED erts_break_requested
@@ -356,8 +353,8 @@ void erts_do_break_handling(void);
# define ERTS_GOT_SIGUSR1 0
# else
# ifdef ERTS_SMP
-extern erts_smp_atomic_t erts_got_sigusr1;
-# define ERTS_GOT_SIGUSR1 ((int) erts_smp_atomic_read(&erts_got_sigusr1))
+extern erts_smp_atomic32_t erts_got_sigusr1;
+# define ERTS_GOT_SIGUSR1 ((int) erts_smp_atomic32_read(&erts_got_sigusr1))
# else
extern volatile int erts_got_sigusr1;
# define ERTS_GOT_SIGUSR1 erts_got_sigusr1
@@ -524,7 +521,8 @@ int erts_send_info_to_logger_nogl(erts_dsprintf_buf_t *);
int erts_send_warning_to_logger_nogl(erts_dsprintf_buf_t *);
int erts_send_error_to_logger_nogl(erts_dsprintf_buf_t *);
int erts_send_info_to_logger_str_nogl(char *);
-int erts_send_warning_to_logger_str_nogl(char *);
+/* needed by erl_smp.h (declared above)
+ int erts_send_warning_to_logger_str_nogl(char *); */
int erts_send_error_to_logger_str_nogl(char *);
typedef struct preload {
@@ -562,11 +560,7 @@ extern char *erts_default_arg0;
extern char os_type[];
extern int sys_init_time(void);
-#if defined(ERTS_TIMER_THREAD)
-#define erts_deliver_time()
-#else
extern void erts_deliver_time(void);
-#endif
extern void erts_time_remaining(SysTimeval *);
extern int erts_init_time_sup(void);
extern void erts_sys_init_float(void);
diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c
index 9cb6ea34ef..c65cc37fc6 100644
--- a/erts/emulator/beam/time.c
+++ b/erts/emulator/beam/time.c
@@ -99,76 +99,18 @@ static erts_smp_mtx_t tiw_lock;
static ErlTimer** tiw; /* the timing wheel, allocated in init_time() */
static Uint tiw_pos; /* current position in wheel */
static Uint tiw_nto; /* number of timeouts in wheel */
+static Uint tiw_min;
+static ErlTimer *tiw_min_ptr;
/* END tiw_lock protected variables */
/* Actual interval time chosen by sys_init_time() */
static int itime; /* Constant after init */
-#if defined(ERTS_TIMER_THREAD)
-static SysTimeval time_start; /* start of current time interval */
-static erts_aint_t ticks_end; /* time_start+ticks_end == time_wakeup */
-static erts_aint_t ticks_latest; /* delta from time_start at latest time update*/
-
-static ERTS_INLINE erts_aint_t time_gettimeofday(SysTimeval *now)
-{
- erts_aint_t elapsed;
-
- erts_get_timeval(now);
- now->tv_usec = 1000 * (now->tv_usec / 1000); /* ms resolution */
- elapsed = (1000 * (now->tv_sec - time_start.tv_sec) +
- (now->tv_usec - time_start.tv_usec) / 1000);
- // elapsed /= CLOCK_RESOLUTION;
- return elapsed;
-}
-
-static erts_aint_t do_time_update(void)
-{
- SysTimeval now;
- erts_aint_t elapsed;
-
- elapsed = time_gettimeofday(&now);
- ticks_latest = elapsed;
- return elapsed;
-}
-
-static ERTS_INLINE erts_aint_t do_time_read(void)
-{
- return ticks_latest;
-}
-
-static erts_aint_t do_time_reset(void)
-{
- SysTimeval now;
- erts_aint_t elapsed;
-
- elapsed = time_gettimeofday(&now);
- time_start = now;
- ticks_end = LONG_MAX;
- ticks_latest = 0;
- return elapsed;
-}
-
-static ERTS_INLINE void do_time_init(void)
-{
- (void)do_time_reset();
-}
-
-#else
erts_smp_atomic_t do_time; /* set at clock interrupt */
-static ERTS_INLINE erts_aint_t do_time_read(void)
-{
- return erts_smp_atomic_read(&do_time);
-}
-static ERTS_INLINE erts_aint_t do_time_update(void)
-{
- return do_time_read();
-}
-static ERTS_INLINE void do_time_init(void)
-{
- erts_smp_atomic_init(&do_time, (erts_aint_t) 0);
-}
-#endif
+static ERTS_INLINE erts_aint_t do_time_read(void) { return erts_smp_atomic_read(&do_time); }
+static ERTS_INLINE erts_aint_t do_time_update(void) { return do_time_read(); }
+static ERTS_INLINE void do_time_init(void) { erts_smp_atomic_init(&do_time, 0L); }
/* get the time (in units of itime) to the next timeout,
or -1 if there are no timeouts */
@@ -182,6 +124,12 @@ static erts_aint_t next_time_internal(void) /* PRE: tiw_lock taken by caller */
if (tiw_nto == 0)
return -1; /* no timeouts in wheel */
+
+ if (tiw_min_ptr) {
+ min = tiw_min;
+ dt = do_time_read();
+ return ((min >= dt) ? (min - dt) : 0);
+ }
/* start going through wheel to find next timeout */
tm = nto = 0;
@@ -194,11 +142,17 @@ static erts_aint_t next_time_internal(void) /* PRE: tiw_lock taken by caller */
if (p->count == 0) {
/* found next timeout */
dt = do_time_read();
+ /* p->count is zero */
+ tiw_min_ptr = p;
+ tiw_min = tm;
return ((tm >= dt) ? (tm - dt) : 0);
} else {
/* keep shortest time in 'min' */
- if (tm + p->count*TIW_SIZE < min)
+ if (tm + p->count*TIW_SIZE < min) {
min = tm + p->count*TIW_SIZE;
+ tiw_min_ptr = p;
+ tiw_min = min;
+ }
}
p = p->next;
}
@@ -211,9 +165,33 @@ static erts_aint_t next_time_internal(void) /* PRE: tiw_lock taken by caller */
return ((min >= dt) ? (min - dt) : 0);
}
-#if !defined(ERTS_TIMER_THREAD)
+static void remove_timer(ErlTimer *p) {
+ /* first */
+ if (!p->prev) {
+ tiw[p->slot] = p->next;
+ if(p->next)
+ p->next->prev = NULL;
+ } else {
+ p->prev->next = p->next;
+ }
+
+ /* last */
+ if (!p->next) {
+ if (p->prev)
+ p->prev->next = NULL;
+ } else {
+ p->next->prev = p->prev;
+ }
+
+ p->next = NULL;
+ p->prev = NULL;
+ /* Make sure cancel callback isn't called */
+ p->active = 0;
+ tiw_nto--;
+}
+
/* Private export to erl_time_sup.c */
-erts_aint_t next_time(void)
+erts_aint_t erts_next_time(void)
{
erts_aint_t ret;
@@ -223,7 +201,6 @@ erts_aint_t next_time(void)
erts_smp_mtx_unlock(&tiw_lock);
return ret;
}
-#endif
static ERTS_INLINE void bump_timer_internal(erts_aint_t dt) /* PRE: tiw_lock is write-locked */
{
@@ -251,12 +228,16 @@ static ERTS_INLINE void bump_timer_internal(erts_aint_t dt) /* PRE: tiw_lock is
if (tiw_pos == keep_pos) count--;
prev = &tiw[tiw_pos];
while ((p = *prev) != NULL) {
+ ASSERT( p != p->next);
if (p->count < count) { /* we have a timeout */
- *prev = p->next; /* Remove from list */
- tiw_nto--;
- p->next = NULL;
- p->active = 0; /* Make sure cancel callback
- isn't called */
+ /* remove min time */
+ if (tiw_min_ptr == p) {
+ tiw_min_ptr = NULL;
+ tiw_min = 0;
+ }
+
+ /* Remove from list */
+ remove_timer(p);
*timeout_tail = p; /* Insert in timeout queue */
timeout_tail = &p->next;
}
@@ -270,6 +251,8 @@ static ERTS_INLINE void bump_timer_internal(erts_aint_t dt) /* PRE: tiw_lock is
dtime--;
}
tiw_pos = keep_pos;
+ if (tiw_min_ptr)
+ tiw_min -= dt;
erts_smp_mtx_unlock(&tiw_lock);
@@ -284,24 +267,17 @@ static ERTS_INLINE void bump_timer_internal(erts_aint_t dt) /* PRE: tiw_lock is
* callback is called.
*/
p->next = NULL;
+ p->prev = NULL;
p->slot = 0;
(*p->timeout)(p->arg);
}
}
-#if defined(ERTS_TIMER_THREAD)
-static void timer_thread_bump_timer(void)
-{
- erts_smp_mtx_lock(&tiw_lock);
- bump_timer_internal(do_time_reset());
-}
-#else
-void bump_timer(erts_aint_t dt) /* dt is value from do_time */
+void erts_bump_timer(erts_aint_t dt) /* dt is value from do_time */
{
erts_smp_mtx_lock(&tiw_lock);
bump_timer_internal(dt);
}
-#endif
Uint
erts_timer_wheel_memory_size(void)
@@ -309,82 +285,10 @@ erts_timer_wheel_memory_size(void)
return (Uint) TIW_SIZE * sizeof(ErlTimer*);
}
-#if defined(ERTS_TIMER_THREAD)
-static struct erts_iwait *timer_thread_iwait;
-
-static int timer_thread_setup_delay(SysTimeval *rem_time)
-{
- erts_aint_t elapsed;
- erts_aint_t ticks;
-
- erts_smp_mtx_lock(&tiw_lock);
- elapsed = do_time_update();
- ticks = next_time_internal();
- if (ticks == -1) /* timer queue empty */
- ticks = 100*1000*1000;
- if (elapsed > ticks)
- elapsed = ticks;
- ticks -= elapsed;
- //ticks *= CLOCK_RESOLUTION;
- rem_time->tv_sec = ticks / 1000;
- rem_time->tv_usec = 1000 * (ticks % 1000);
- ticks_end = ticks;
- erts_smp_mtx_unlock(&tiw_lock);
- return ticks;
-}
-
-static void *timer_thread_start(void *ignore)
-{
- SysTimeval delay;
-
-#ifdef ERTS_ENABLE_LOCK_CHECK
- erts_lc_set_thread_name("timer");
-#endif
- erts_register_blockable_thread();
-
- for(;;) {
- if (timer_thread_setup_delay(&delay)) {
- erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL);
- ASSERT_NO_LOCKED_LOCKS;
- erts_iwait_wait(timer_thread_iwait, &delay);
- ASSERT_NO_LOCKED_LOCKS;
- erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL);
- }
- else
- erts_smp_chk_system_block(NULL, NULL, NULL);
- timer_thread_bump_timer();
- ASSERT_NO_LOCKED_LOCKS;
- }
- /*NOTREACHED*/
- return NULL;
-}
-
-static ERTS_INLINE void timer_thread_post_insert(Uint ticks)
-{
- if ((Sint)ticks < ticks_end)
- erts_iwait_interrupt(timer_thread_iwait);
-}
-
-static void timer_thread_init(void)
-{
- erts_thr_opts_t opts = ERTS_THR_OPTS_DEFAULT_INITER;
- erts_tid_t tid;
-
- opts->detached = 1;
-
- timer_thread_iwait = erts_iwait_init();
- erts_thr_create(&tid, timer_thread_start, NULL, &opts);
-}
-
-#else
-static ERTS_INLINE void timer_thread_post_insert(Uint ticks) { }
-static ERTS_INLINE void timer_thread_init(void) { }
-#endif
-
/* this routine links the time cells into a free list at the start
and sets the time queue as empty */
void
-init_time(void)
+erts_init_time(void)
{
int i;
@@ -400,10 +304,13 @@ init_time(void)
tiw[i] = NULL;
do_time_init();
tiw_pos = tiw_nto = 0;
-
- timer_thread_init();
+ tiw_min_ptr = NULL;
+ tiw_min = 0;
}
+
+
+
/*
** Insert a process into the time queue, with a timeout 't'
*/
@@ -433,16 +340,31 @@ insert_timer(ErlTimer* p, Uint t)
/* insert at head of list at slot */
p->next = tiw[tm];
+ p->prev = NULL;
+ if (p->next != NULL)
+ p->next->prev = p;
tiw[tm] = p;
- tiw_nto++;
- timer_thread_post_insert(ticks);
+
+ /* insert min time */
+ if ((tiw_nto == 0) || ((tiw_min_ptr != NULL) && (ticks < tiw_min))) {
+ tiw_min = ticks;
+ tiw_min_ptr = p;
+ }
+ if ((tiw_min_ptr == p) && (ticks > tiw_min)) {
+ /* some other timer might be 'min' now */
+ tiw_min = 0;
+ tiw_min_ptr = NULL;
+ }
+
+ tiw_nto++;
}
void
-erl_set_timer(ErlTimer* p, ErlTimeoutProc timeout, ErlCancelProc cancel,
+erts_set_timer(ErlTimer* p, ErlTimeoutProc timeout, ErlCancelProc cancel,
void* arg, Uint t)
{
+
erts_deliver_time();
erts_smp_mtx_lock(&tiw_lock);
if (p->active) { /* XXX assert ? */
@@ -455,42 +377,34 @@ erl_set_timer(ErlTimer* p, ErlTimeoutProc timeout, ErlCancelProc cancel,
p->active = 1;
insert_timer(p, t);
erts_smp_mtx_unlock(&tiw_lock);
-#if defined(ERTS_SMP) && !defined(ERTS_TIMER_THREAD)
+#if defined(ERTS_SMP)
if (t <= (Uint) LONG_MAX)
erts_sys_schedule_interrupt_timed(1, (long) t);
#endif
}
void
-erl_cancel_timer(ErlTimer* p)
+erts_cancel_timer(ErlTimer* p)
{
- ErlTimer *tp;
- ErlTimer **prev;
-
erts_smp_mtx_lock(&tiw_lock);
if (!p->active) { /* allow repeated cancel (drivers) */
erts_smp_mtx_unlock(&tiw_lock);
return;
}
- /* find p in linked list at slot p->slot and remove it */
- prev = &tiw[p->slot];
- while ((tp = *prev) != NULL) {
- if (tp == p) {
- *prev = p->next; /* Remove from list */
- tiw_nto--;
- p->next = NULL;
- p->slot = p->count = 0;
- p->active = 0;
- if (p->cancel != NULL) {
- erts_smp_mtx_unlock(&tiw_lock);
- (*p->cancel)(p->arg);
- } else {
- erts_smp_mtx_unlock(&tiw_lock);
- }
- return;
- } else {
- prev = &tp->next;
- }
+
+ /* is it the 'min' timer, remove min */
+ if (p == tiw_min_ptr) {
+ tiw_min_ptr = NULL;
+ tiw_min = 0;
+ }
+
+ remove_timer(p);
+ p->slot = p->count = 0;
+
+ if (p->cancel != NULL) {
+ erts_smp_mtx_unlock(&tiw_lock);
+ (*p->cancel)(p->arg);
+ return;
}
erts_smp_mtx_unlock(&tiw_lock);
}
@@ -502,7 +416,7 @@ erl_cancel_timer(ErlTimer* p)
immediately if it hadn't been cancelled).
*/
Uint
-time_left(ErlTimer *p)
+erts_time_left(ErlTimer *p)
{
Uint left;
erts_aint_t dt;
@@ -530,8 +444,7 @@ time_left(ErlTimer *p)
}
#ifdef DEBUG
-
-void p_slpq()
+void erts_p_slpq()
{
int i;
ErlTimer* p;
@@ -560,5 +473,4 @@ void p_slpq()
erts_smp_mtx_unlock(&tiw_lock);
}
-
#endif /* DEBUG */
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 2bf283d9ec..f531d1430b 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 1996-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -59,13 +59,6 @@
/* profile_scheduler mini message queue */
-#ifdef ERTS_TIMER_THREAD
-/* A timer thread is not welcomed with this lock violation work around.
- * - Bj�rn-Egil
- */
-#error Timer thread may not be enabled due to lock violation.
-#endif
-
typedef struct {
Uint scheduler_id;
Uint no_schedulers;
@@ -98,7 +91,7 @@ dispatch_profile_msg_q(profile_sched_msg_q *psmq)
Eterm*
-erts_heap_alloc(Process* p, Uint need)
+erts_heap_alloc(Process* p, Uint need, Uint xtra)
{
ErlHeapFragment* bp;
Eterm* htop;
@@ -124,7 +117,7 @@ erts_heap_alloc(Process* p, Uint need)
p->space_verified_from = NULL;
#endif /* FORCE_HEAP_FRAGS */
- n = need;
+ n = need + xtra;
bp = MBUF(p);
if (bp != NULL && need <= (bp->alloc_size - bp->used_size)) {
Eterm* ret = bp->mem + bp->used_size;
@@ -160,7 +153,7 @@ erts_heap_alloc(Process* p, Uint need)
bp->next = MBUF(p);
MBUF(p) = bp;
bp->alloc_size = n;
- bp->used_size = n;
+ bp->used_size = need;
MBUF_SIZE(p) += n;
bp->off_heap.first = NULL;
bp->off_heap.overhead = 0;
@@ -1901,34 +1894,36 @@ erts_destroy_tmp_dsbuf(erts_dsprintf_buf_t *dsbufp)
erts_free(ERTS_ALC_T_TMP_DSBUF, (void *) dsbufp);
}
-
/* eq and cmp are written as separate functions a eq is a little faster */
/*
* Test for equality of two terms.
* Returns 0 if not equal, or a non-zero value otherwise.
*/
-
+#if HALFWORD_HEAP
+int eq_rel(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base)
+#else
int eq(Eterm a, Eterm b)
+#endif
{
DECLARE_WSTACK(stack);
Sint sz;
Eterm* aa;
- Eterm* bb;
+ Eterm* bb;
tailrecur:
- if (a == b) goto pop_next;
+ if (is_same(a, a_base, b, b_base)) goto pop_next;
tailrecur_ne:
switch (primary_tag(a)) {
case TAG_PRIMARY_LIST:
if (is_list(b)) {
- Eterm* aval = list_val(a);
- Eterm* bval = list_val(b);
+ Eterm* aval = list_val_rel(a, a_base);
+ Eterm* bval = list_val_rel(b, b_base);
while (1) {
Eterm atmp = CAR(aval);
Eterm btmp = CAR(bval);
- if (atmp != btmp) {
+ if (!is_same(atmp,a_base,btmp,b_base)) {
WSTACK_PUSH2(stack,(UWord) CDR(bval),(UWord) CDR(aval));
a = atmp;
b = btmp;
@@ -1936,7 +1931,7 @@ tailrecur_ne:
}
atmp = CDR(aval);
btmp = CDR(bval);
- if (atmp == btmp) {
+ if (is_same(atmp,a_base,btmp,b_base)) {
goto pop_next;
}
if (is_not_list(atmp) || is_not_list(btmp)) {
@@ -1944,22 +1939,22 @@ tailrecur_ne:
b = btmp;
goto tailrecur_ne;
}
- aval = list_val(atmp);
- bval = list_val(btmp);
+ aval = list_val_rel(atmp, a_base);
+ bval = list_val_rel(btmp, b_base);
}
}
break; /* not equal */
case TAG_PRIMARY_BOXED:
{
- Eterm hdr = *boxed_val(a);
+ Eterm hdr = *boxed_val_rel(a,a_base);
switch (hdr & _TAG_HEADER_MASK) {
case ARITYVAL_SUBTAG:
{
- aa = tuple_val(a);
- if (!is_boxed(b) || *boxed_val(b) != *aa)
+ aa = tuple_val_rel(a, a_base);
+ if (!is_boxed(b) || *boxed_val_rel(b,b_base) != *aa)
goto not_equal;
- bb = tuple_val(b);
+ bb = tuple_val_rel(b,b_base);
if ((sz = arityval(*aa)) == 0) goto pop_next;
++aa;
++bb;
@@ -1978,16 +1973,16 @@ tailrecur_ne:
Uint a_bitoffs;
Uint b_bitoffs;
- if (is_not_binary(b)) {
+ if (!is_binary_rel(b,b_base)) {
goto not_equal;
}
- a_size = binary_size(a);
- b_size = binary_size(b);
+ a_size = binary_size_rel(a,a_base);
+ b_size = binary_size_rel(b,b_base);
if (a_size != b_size) {
goto not_equal;
}
- ERTS_GET_BINARY_BYTES(a, a_ptr, a_bitoffs, a_bitsize);
- ERTS_GET_BINARY_BYTES(b, b_ptr, b_bitoffs, b_bitsize);
+ ERTS_GET_BINARY_BYTES_REL(a, a_ptr, a_bitoffs, a_bitsize, a_base);
+ ERTS_GET_BINARY_BYTES_REL(b, b_ptr, b_bitoffs, b_bitsize, b_base);
if ((a_bitsize | b_bitsize | a_bitoffs | b_bitoffs) == 0) {
if (sys_memcmp(a_ptr, b_ptr, a_size) == 0) goto pop_next;
} else if (a_bitsize == b_bitsize) {
@@ -1998,9 +1993,9 @@ tailrecur_ne:
}
case EXPORT_SUBTAG:
{
- if (is_export(b)) {
- Export* a_exp = *((Export **) (export_val(a) + 1));
- Export* b_exp = *((Export **) (export_val(b) + 1));
+ if (is_export_rel(b,b_base)) {
+ Export* a_exp = *((Export **) (export_val_rel(a,a_base) + 1));
+ Export* b_exp = *((Export **) (export_val_rel(b,b_base) + 1));
if (a_exp == b_exp) goto pop_next;
}
break; /* not equal */
@@ -2010,10 +2005,10 @@ tailrecur_ne:
ErlFunThing* f1;
ErlFunThing* f2;
- if (is_not_fun(b))
+ if (!is_fun_rel(b,b_base))
goto not_equal;
- f1 = (ErlFunThing *) fun_val(a);
- f2 = (ErlFunThing *) fun_val(b);
+ f1 = (ErlFunThing *) fun_val_rel(a,a_base);
+ f2 = (ErlFunThing *) fun_val_rel(b,b_base);
if (f1->fe->module != f2->fe->module ||
f1->fe->old_index != f2->fe->old_index ||
f1->fe->old_uniq != f2->fe->old_uniq ||
@@ -2031,15 +2026,15 @@ tailrecur_ne:
ExternalThing *ap;
ExternalThing *bp;
- if(is_not_external(b))
+ if(!is_external_rel(b,b_base))
goto not_equal;
- ap = external_thing_ptr(a);
- bp = external_thing_ptr(b);
+ ap = external_thing_ptr_rel(a,a_base);
+ bp = external_thing_ptr_rel(b,b_base);
if(ap->header == bp->header && ap->node == bp->node) {
- ASSERT(1 == external_data_words(a));
- ASSERT(1 == external_data_words(b));
+ ASSERT(1 == external_data_words_rel(a,a_base));
+ ASSERT(1 == external_data_words_rel(b,b_base));
if (ap->data.ui[0] == bp->data.ui[0]) goto pop_next;
}
@@ -2057,27 +2052,36 @@ tailrecur_ne:
Uint alen;
Uint blen;
Uint i;
+ ExternalThing* athing;
+ ExternalThing* bthing;
- if(is_not_external_ref(b))
+ if(!is_external_ref_rel(b,b_base))
goto not_equal;
- if(external_node(a) != external_node(b))
+ athing = external_thing_ptr_rel(a,a_base);
+ bthing = external_thing_ptr_rel(b,b_base);
+
+ if(athing->node != bthing->node)
goto not_equal;
- anum = external_ref_numbers(a);
- bnum = external_ref_numbers(b);
- alen = external_ref_no_of_numbers(a);
- blen = external_ref_no_of_numbers(b);
+ anum = external_thing_ref_numbers(athing);
+ bnum = external_thing_ref_numbers(bthing);
+ alen = external_thing_ref_no_of_numbers(athing);
+ blen = external_thing_ref_no_of_numbers(bthing);
goto ref_common;
case REF_SUBTAG:
-
- if (is_not_internal_ref(b))
+ if (!is_internal_ref_rel(b,b_base))
goto not_equal;
- alen = internal_ref_no_of_numbers(a);
- blen = internal_ref_no_of_numbers(b);
- anum = internal_ref_numbers(a);
- bnum = internal_ref_numbers(b);
+
+ {
+ RefThing* athing = ref_thing_ptr_rel(a,a_base);
+ RefThing* bthing = ref_thing_ptr_rel(b,b_base);
+ alen = internal_thing_ref_no_of_numbers(athing);
+ blen = internal_thing_ref_no_of_numbers(bthing);
+ anum = internal_thing_ref_numbers(athing);
+ bnum = internal_thing_ref_numbers(bthing);
+ }
ref_common:
ASSERT(alen > 0 && blen > 0);
@@ -2122,10 +2126,10 @@ tailrecur_ne:
{
int i;
- if (is_not_big(b))
+ if (!is_big_rel(b,b_base))
goto not_equal;
- aa = big_val(a); /* get pointer to thing */
- bb = big_val(b);
+ aa = big_val_rel(a,a_base);
+ bb = big_val_rel(b,b_base);
if (*aa != *bb)
goto not_equal;
i = BIG_ARITY(aa);
@@ -2140,9 +2144,9 @@ tailrecur_ne:
FloatDef af;
FloatDef bf;
- if (is_float(b)) {
- GET_DOUBLE(a, af);
- GET_DOUBLE(b, bf);
+ if (is_float_rel(b,b_base)) {
+ GET_DOUBLE_REL(a, af, a_base);
+ GET_DOUBLE_REL(b, bf, b_base);
if (af.fd == bf.fd) goto pop_next;
}
break; /* not equal */
@@ -2161,7 +2165,7 @@ term_array: /* arrays in 'aa' and 'bb', length in 'sz' */
Eterm* bp = bb;
Sint i = sz;
for (;;) {
- if (*ap != *bp) break;
+ if (!is_same(*ap,a_base,*bp,b_base)) break;
if (--i == 0) goto pop_next;
++ap;
++bp;
@@ -2250,7 +2254,11 @@ static int cmp_atoms(Eterm a, Eterm b)
bb->name+3, bb->len-3);
}
+#if HALFWORD_HEAP
+Sint cmp_rel(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base)
+#else
Sint cmp(Eterm a, Eterm b)
+#endif
{
DECLARE_WSTACK(stack);
Eterm* aa;
@@ -2284,7 +2292,7 @@ Sint cmp(Eterm a, Eterm b)
tailrecur:
- if (a == b) { /* Equal values or pointers. */
+ if (is_same(a,a_base,b,b_base)) { /* Equal values or pointers. */
goto pop_next;
}
tailrecur_ne:
@@ -2310,9 +2318,9 @@ tailrecur_ne:
if (is_internal_port(b)) {
bnode = erts_this_node;
bdata = internal_port_data(b);
- } else if (is_external_port(b)) {
- bnode = external_port_node(b);
- bdata = external_port_data(b);
+ } else if (is_external_port_rel(b,b_base)) {
+ bnode = external_port_node_rel(b,b_base);
+ bdata = external_port_data_rel(b,b_base);
} else {
a_tag = PORT_DEF;
goto mixed_types;
@@ -2328,9 +2336,9 @@ tailrecur_ne:
if (is_internal_pid(b)) {
bnode = erts_this_node;
bdata = internal_pid_data(b);
- } else if (is_external_pid(b)) {
- bnode = external_pid_node(b);
- bdata = external_pid_data(b);
+ } else if (is_external_pid_rel(b,b_base)) {
+ bnode = external_pid_node_rel(b,b_base);
+ bdata = external_pid_data_rel(b,b_base);
} else {
a_tag = PID_DEF;
goto mixed_types;
@@ -2363,12 +2371,12 @@ tailrecur_ne:
a_tag = LIST_DEF;
goto mixed_types;
}
- aa = list_val(a);
- bb = list_val(b);
+ aa = list_val_rel(a,a_base);
+ bb = list_val_rel(b,b_base);
while (1) {
Eterm atmp = CAR(aa);
Eterm btmp = CAR(bb);
- if (atmp != btmp) {
+ if (!is_same(atmp,a_base,btmp,b_base)) {
WSTACK_PUSH2(stack,(UWord) CDR(bb),(UWord) CDR(aa));
a = atmp;
b = btmp;
@@ -2376,7 +2384,7 @@ tailrecur_ne:
}
atmp = CDR(aa);
btmp = CDR(bb);
- if (atmp == btmp) {
+ if (is_same(atmp,a_base,btmp,b_base)) {
goto pop_next;
}
if (is_not_list(atmp) || is_not_list(btmp)) {
@@ -2384,20 +2392,20 @@ tailrecur_ne:
b = btmp;
goto tailrecur_ne;
}
- aa = list_val(atmp);
- bb = list_val(btmp);
+ aa = list_val_rel(atmp,a_base);
+ bb = list_val_rel(btmp,b_base);
}
case TAG_PRIMARY_BOXED:
{
- Eterm ahdr = *boxed_val(a);
+ Eterm ahdr = *boxed_val_rel(a,a_base);
switch ((ahdr & _TAG_HEADER_MASK) >> _TAG_PRIMARY_SIZE) {
case (_TAG_HEADER_ARITYVAL >> _TAG_PRIMARY_SIZE):
- if (is_not_tuple(b)) {
+ if (!is_tuple_rel(b,b_base)) {
a_tag = TUPLE_DEF;
goto mixed_types;
}
- aa = tuple_val(a);
- bb = tuple_val(b);
+ aa = tuple_val_rel(a,a_base);
+ bb = tuple_val_rel(b,b_base);
/* compare the arities */
i = arityval(ahdr); /* get the arity*/
if (i != arityval(*bb)) {
@@ -2411,31 +2419,31 @@ tailrecur_ne:
goto term_array;
case (_TAG_HEADER_FLOAT >> _TAG_PRIMARY_SIZE):
- if (is_not_float(b)) {
+ if (!is_float_rel(b,b_base)) {
a_tag = FLOAT_DEF;
goto mixed_types;
} else {
FloatDef af;
FloatDef bf;
- GET_DOUBLE(a, af);
- GET_DOUBLE(b, bf);
+ GET_DOUBLE_REL(a, af, a_base);
+ GET_DOUBLE_REL(b, bf, b_base);
ON_CMP_GOTO(float_comp(af.fd, bf.fd));
}
case (_TAG_HEADER_POS_BIG >> _TAG_PRIMARY_SIZE):
case (_TAG_HEADER_NEG_BIG >> _TAG_PRIMARY_SIZE):
- if (is_not_big(b)) {
+ if (!is_big_rel(b,b_base)) {
a_tag = BIG_DEF;
goto mixed_types;
}
- ON_CMP_GOTO(big_comp(a, b));
+ ON_CMP_GOTO(big_comp(rterm2wterm(a,a_base), rterm2wterm(b,b_base)));
case (_TAG_HEADER_EXPORT >> _TAG_PRIMARY_SIZE):
- if (is_not_export(b)) {
+ if (!is_export_rel(b,b_base)) {
a_tag = EXPORT_DEF;
goto mixed_types;
} else {
- Export* a_exp = *((Export **) (export_val(a) + 1));
- Export* b_exp = *((Export **) (export_val(b) + 1));
+ Export* a_exp = *((Export **) (export_val_rel(a,a_base) + 1));
+ Export* b_exp = *((Export **) (export_val_rel(b,b_base) + 1));
if ((j = cmp_atoms(a_exp->code[0], b_exp->code[0])) != 0) {
RETURN_NEQ(j);
@@ -2447,12 +2455,12 @@ tailrecur_ne:
}
break;
case (_TAG_HEADER_FUN >> _TAG_PRIMARY_SIZE):
- if (is_not_fun(b)) {
+ if (!is_fun_rel(b,b_base)) {
a_tag = FUN_DEF;
goto mixed_types;
} else {
- ErlFunThing* f1 = (ErlFunThing *) fun_val(a);
- ErlFunThing* f2 = (ErlFunThing *) fun_val(b);
+ ErlFunThing* f1 = (ErlFunThing *) fun_val_rel(a,a_base);
+ ErlFunThing* f2 = (ErlFunThing *) fun_val_rel(b,b_base);
Sint diff;
diff = cmpbytes(atom_tab(atom_val(f1->fe->module))->name,
@@ -2484,51 +2492,57 @@ tailrecur_ne:
if (is_internal_pid(b)) {
bnode = erts_this_node;
bdata = internal_pid_data(b);
- } else if (is_external_pid(b)) {
- bnode = external_pid_node(b);
- bdata = external_pid_data(b);
+ } else if (is_external_pid_rel(b,b_base)) {
+ bnode = external_pid_node_rel(b,b_base);
+ bdata = external_pid_data_rel(b,b_base);
} else {
a_tag = EXTERNAL_PID_DEF;
goto mixed_types;
}
- anode = external_pid_node(a);
- adata = external_pid_data(a);
+ anode = external_pid_node_rel(a,a_base);
+ adata = external_pid_data_rel(a,a_base);
goto pid_common;
case (_TAG_HEADER_EXTERNAL_PORT >> _TAG_PRIMARY_SIZE):
if (is_internal_port(b)) {
bnode = erts_this_node;
bdata = internal_port_data(b);
- } else if (is_external_port(b)) {
- bnode = external_port_node(b);
- bdata = external_port_data(b);
+ } else if (is_external_port_rel(b,b_base)) {
+ bnode = external_port_node_rel(b,b_base);
+ bdata = external_port_data_rel(b,b_base);
} else {
a_tag = EXTERNAL_PORT_DEF;
goto mixed_types;
}
- anode = external_port_node(a);
- adata = external_port_data(a);
+ anode = external_port_node_rel(a,a_base);
+ adata = external_port_data_rel(a,a_base);
goto port_common;
case (_TAG_HEADER_REF >> _TAG_PRIMARY_SIZE):
/*
* Note! When comparing refs we need to compare ref numbers
* (32-bit words), *not* ref data words.
*/
+
- if (is_internal_ref(b)) {
+ if (is_internal_ref_rel(b,b_base)) {
+ RefThing* bthing = ref_thing_ptr_rel(b,b_base);
bnode = erts_this_node;
- bnum = internal_ref_numbers(b);
- blen = internal_ref_no_of_numbers(b);
- } else if(is_external_ref(b)) {
- bnode = external_ref_node(b);
- bnum = external_ref_numbers(b);
- blen = external_ref_no_of_numbers(b);
+ bnum = internal_thing_ref_numbers(bthing);
+ blen = internal_thing_ref_no_of_numbers(bthing);
+ } else if(is_external_ref_rel(b,b_base)) {
+ ExternalThing* bthing = external_thing_ptr_rel(b,b_base);
+ bnode = bthing->node;
+ bnum = external_thing_ref_numbers(bthing);
+ blen = external_thing_ref_no_of_numbers(bthing);
} else {
a_tag = REF_DEF;
goto mixed_types;
}
- anode = erts_this_node;
- anum = internal_ref_numbers(a);
- alen = internal_ref_no_of_numbers(a);
+ {
+ RefThing* athing = ref_thing_ptr_rel(a,a_base);
+ anode = erts_this_node;
+ anum = internal_thing_ref_numbers(athing);
+ alen = internal_thing_ref_no_of_numbers(athing);
+ }
ref_common:
CMP_NODES(anode, bnode);
@@ -2557,31 +2571,36 @@ tailrecur_ne:
RETURN_NEQ((Sint32) (anum[i] - bnum[i]));
goto pop_next;
case (_TAG_HEADER_EXTERNAL_REF >> _TAG_PRIMARY_SIZE):
- if (is_internal_ref(b)) {
+ if (is_internal_ref_rel(b,b_base)) {
+ RefThing* bthing = ref_thing_ptr_rel(b,b_base);
bnode = erts_this_node;
- bnum = internal_ref_numbers(b);
- blen = internal_ref_no_of_numbers(b);
- } else if (is_external_ref(b)) {
- bnode = external_ref_node(b);
- bnum = external_ref_numbers(b);
- blen = external_ref_no_of_numbers(b);
+ bnum = internal_thing_ref_numbers(bthing);
+ blen = internal_thing_ref_no_of_numbers(bthing);
+ } else if (is_external_ref_rel(b,b_base)) {
+ ExternalThing* bthing = external_thing_ptr_rel(b,b_base);
+ bnode = bthing->node;
+ bnum = external_thing_ref_numbers(bthing);
+ blen = external_thing_ref_no_of_numbers(bthing);
} else {
a_tag = EXTERNAL_REF_DEF;
goto mixed_types;
}
- anode = external_ref_node(a);
- anum = external_ref_numbers(a);
- alen = external_ref_no_of_numbers(a);
+ {
+ ExternalThing* athing = external_thing_ptr_rel(a,a_base);
+ anode = athing->node;
+ anum = external_thing_ref_numbers(athing);
+ alen = external_thing_ref_no_of_numbers(athing);
+ }
goto ref_common;
default:
/* Must be a binary */
- ASSERT(is_binary(a));
- if (is_not_binary(b)) {
+ ASSERT(is_binary_rel(a,a_base));
+ if (!is_binary_rel(b,b_base)) {
a_tag = BINARY_DEF;
goto mixed_types;
} else {
- Uint a_size = binary_size(a);
- Uint b_size = binary_size(b);
+ Uint a_size = binary_size_rel(a,a_base);
+ Uint b_size = binary_size_rel(b,b_base);
Uint a_bitsize;
Uint b_bitsize;
Uint a_bitoffs;
@@ -2590,8 +2609,8 @@ tailrecur_ne:
int cmp;
byte* a_ptr;
byte* b_ptr;
- ERTS_GET_BINARY_BYTES(a, a_ptr, a_bitoffs, a_bitsize);
- ERTS_GET_BINARY_BYTES(b, b_ptr, b_bitoffs, b_bitsize);
+ ERTS_GET_BINARY_BYTES_REL(a, a_ptr, a_bitoffs, a_bitsize, a_base);
+ ERTS_GET_BINARY_BYTES_REL(b, b_ptr, b_bitoffs, b_bitsize, b_base);
if ((a_bitsize | b_bitsize | a_bitoffs | b_bitoffs) == 0) {
min_size = (a_size < b_size) ? a_size : b_size;
if ((cmp = sys_memcmp(a_ptr, b_ptr, min_size)) != 0) {
@@ -2618,7 +2637,6 @@ tailrecur_ne:
*/
mixed_types:
- b_tag = tag_val_def(b);
{
FloatDef f1, f2;
@@ -2628,39 +2646,47 @@ tailrecur_ne:
#else
Eterm *big_buf = erts_get_scheduler_data()->cmp_tmp_heap;
#endif
+#if HALFWORD_HEAP
+ Wterm aw = is_immed(a) ? a : rterm2wterm(a,a_base);
+ Wterm bw = is_immed(b) ? b : rterm2wterm(b,b_base);
+#else
+ Eterm aw = a;
+ Eterm bw = b;
+#endif
+ b_tag = tag_val_def(bw);
switch(_NUMBER_CODE(a_tag, b_tag)) {
case SMALL_BIG:
big = small_to_big(signed_val(a), big_buf);
- j = big_comp(big, b);
+ j = big_comp(big, bw);
break;
case SMALL_FLOAT:
f1.fd = signed_val(a);
- GET_DOUBLE(b, f2);
+ GET_DOUBLE(bw, f2);
j = float_comp(f1.fd, f2.fd);
break;
case BIG_SMALL:
big = small_to_big(signed_val(b), big_buf);
- j = big_comp(a, big);
+ j = big_comp(aw, big);
break;
case BIG_FLOAT:
- if (big_to_double(a, &f1.fd) < 0) {
+ if (big_to_double(aw, &f1.fd) < 0) {
j = big_sign(a) ? -1 : 1;
} else {
- GET_DOUBLE(b, f2);
+ GET_DOUBLE(bw, f2);
j = float_comp(f1.fd, f2.fd);
}
break;
case FLOAT_SMALL:
- GET_DOUBLE(a, f1);
+ GET_DOUBLE(aw, f1);
f2.fd = signed_val(b);
j = float_comp(f1.fd, f2.fd);
break;
case FLOAT_BIG:
- if (big_to_double(b, &f2.fd) < 0) {
+ if (big_to_double(bw, &f2.fd) < 0) {
j = big_sign(b) ? 1 : -1;
} else {
- GET_DOUBLE(a, f1);
+ GET_DOUBLE(aw, f1);
j = float_comp(f1.fd, f2.fd);
}
break;
@@ -3183,7 +3209,7 @@ erts_create_smp_ptimer(ErtsSmpPTimer **timer_ref,
*timer_ref = res;
- erl_set_timer(&res->timer.tm,
+ erts_set_timer(&res->timer.tm,
(ErlTimeoutProc) ptimer_timeout,
(ErlCancelProc) ptimer_cancelled,
(void*) res,
@@ -3197,7 +3223,7 @@ erts_cancel_smp_ptimer(ErtsSmpPTimer *ptimer)
ASSERT(*ptimer->timer.timer_ref == ptimer);
*ptimer->timer.timer_ref = NULL;
ptimer->timer.flags |= ERTS_PTMR_FLG_CANCELLED;
- erl_cancel_timer(&ptimer->timer.tm);
+ erts_cancel_timer(&ptimer->timer.tm);
}
}