diff options
Diffstat (limited to 'erts/emulator/beam')
66 files changed, 1710 insertions, 3459 deletions
diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index 7a50b24818..967cf013f0 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -320,6 +320,7 @@ atom ldflags atom Le='=<' atom lf atom line +atom line_delimiter atom line_length atom linked_in_driver atom links @@ -436,13 +437,6 @@ atom orelse atom os_pid atom os_type atom os_version -atom ose_bg_proc -atom ose_int_proc -atom ose_phantom -atom ose_pri_proc -atom ose_process_prio -atom ose_process_type -atom ose_ti_proc atom out atom out_exited atom out_exiting diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 0e192b1ebd..11508a1b39 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -913,7 +913,7 @@ any_heap_ref_ptrs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size) switch (primary_tag(val)) { case TAG_PRIMARY_BOXED: case TAG_PRIMARY_LIST: - if (in_area(EXPAND_POINTER(val), mod_start, mod_size)) { + if (in_area(val, mod_start, mod_size)) { return 1; } break; @@ -933,7 +933,7 @@ any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size) switch (primary_tag(val)) { case TAG_PRIMARY_BOXED: case TAG_PRIMARY_LIST: - if (in_area(EXPAND_POINTER(val), mod_start, mod_size)) { + if (in_area(val, mod_start, mod_size)) { return 1; } break; @@ -943,7 +943,7 @@ any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size) if (header_is_bin_matchstate(val)) { ErlBinMatchState *ms = (ErlBinMatchState*) p; ErlBinMatchBuffer *mb = &(ms->mb); - if (in_area(EXPAND_POINTER(mb->orig), mod_start, mod_size)) { + if (in_area(mb->orig, mod_start, mod_size)) { return 1; } } diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c index c774a70d4c..0e921d761c 100644 --- a/erts/emulator/beam/beam_debug.c +++ b/erts/emulator/beam/beam_debug.c @@ -79,7 +79,7 @@ erts_debug_breakpoint_2(BIF_ALIST_2) { Process* p = BIF_P; Eterm MFA = BIF_ARG_1; - Eterm bool = BIF_ARG_2; + Eterm boolean = BIF_ARG_2; Eterm* tp; Eterm mfa[3]; int i; @@ -87,7 +87,7 @@ erts_debug_breakpoint_2(BIF_ALIST_2) Eterm res; BpFunctions f; - if (bool != am_true && bool != am_false) + if (boolean != am_true && boolean != am_false) goto error; if (is_not_tuple(MFA)) { @@ -124,7 +124,7 @@ erts_debug_breakpoint_2(BIF_ALIST_2) erts_smp_thr_progress_block(); erts_bp_match_functions(&f, mfa, specified); - if (bool == am_true) { + if (boolean == am_true) { erts_set_debug_break(&f); erts_install_breakpoints(&f); erts_commit_staged_bp(); @@ -432,39 +432,33 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) while (*sign) { switch (*sign) { case 'r': /* x(0) */ - erts_print(to, to_arg, "x(0)"); + erts_print(to, to_arg, "r(0)"); break; case 'x': /* x(N) */ - if (reg_index(ap[0]) == 0) { - erts_print(to, to_arg, "x[0]"); - } else { - erts_print(to, to_arg, "x(%d)", reg_index(ap[0])); + { + Uint n = ap[0] / sizeof(Eterm); + erts_print(to, to_arg, "x(%d)", n); + ap++; } - ap++; break; case 'y': /* y(N) */ - erts_print(to, to_arg, "y(%d)", reg_index(ap[0]) - CP_SIZE); - ap++; + { + Uint n = ap[0] / sizeof(Eterm) - CP_SIZE; + erts_print(to, to_arg, "y(%d)", n); + ap++; + } break; case 'n': /* Nil */ erts_print(to, to_arg, "[]"); break; case 's': /* Any source (tagged constant or register) */ - tag = beam_reg_tag(*ap); - if (tag == X_REG_DEF) { - if (reg_index(*ap) == 0) { - erts_print(to, to_arg, "x[0]"); - } else { - erts_print(to, to_arg, "x(%d)", reg_index(*ap)); - } + tag = loader_tag(*ap); + if (tag == LOADER_X_REG) { + erts_print(to, to_arg, "x(%d)", loader_x_reg_index(*ap)); ap++; break; - } else if (tag == Y_REG_DEF) { - erts_print(to, to_arg, "y(%d)", reg_index(*ap) - CP_SIZE); - ap++; - break; - } else if (tag == R_REG_DEF) { - erts_print(to, to_arg, "x(0)"); + } else if (tag == LOADER_Y_REG) { + erts_print(to, to_arg, "y(%d)", loader_y_reg_index(*ap) - CP_SIZE); ap++; break; } @@ -481,20 +475,12 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) ap++; break; case 'd': /* Destination (x(0), x(N), y(N)) */ - switch (beam_reg_tag(*ap)) { - case X_REG_DEF: - if (reg_index(*ap) == 0) { - erts_print(to, to_arg, "x[0]"); - } else { - erts_print(to, to_arg, "x(%d)", reg_index(*ap)); - } - break; - case Y_REG_DEF: - erts_print(to, to_arg, "y(%d)", reg_index(*ap) - CP_SIZE); - break; - case R_REG_DEF: - erts_print(to, to_arg, "x(0)"); - break; + if (*ap & 1) { + erts_print(to, to_arg, "y(%d)", + *ap / sizeof(Eterm) - CP_SIZE); + } else { + erts_print(to, to_arg, "x(%d)", + *ap / sizeof(Eterm)); } ap++; break; @@ -561,7 +547,7 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) ap++; break; case 'l': /* fr(N) */ - erts_print(to, to_arg, "fr(%d)", reg_index(ap[0])); + erts_print(to, to_arg, "fr(%d)", loader_reg_index(ap[0])); ap++; break; default: @@ -580,7 +566,6 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) unpacked = ap; ap = addr + size; switch (op) { - case op_i_select_val_lins_rfI: case op_i_select_val_lins_xfI: case op_i_select_val_lins_yfI: { @@ -600,7 +585,6 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) } } break; - case op_i_select_val_bins_rfI: case op_i_select_val_bins_xfI: case op_i_select_val_bins_yfI: { @@ -614,7 +598,6 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) } } break; - case op_i_select_tuple_arity_rfI: case op_i_select_tuple_arity_xfI: case op_i_select_tuple_arity_yfI: { @@ -639,7 +622,6 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) } } break; - case op_i_jump_on_val_rfII: case op_i_jump_on_val_xfII: case op_i_jump_on_val_yfII: { @@ -651,7 +633,6 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) } } break; - case op_i_jump_on_val_zero_rfI: case op_i_jump_on_val_zero_xfI: case op_i_jump_on_val_zero_yfI: { @@ -663,7 +644,6 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) } } break; - case op_i_put_tuple_rI: case op_i_put_tuple_xI: case op_i_put_tuple_yI: case op_new_map_dII: @@ -673,20 +653,16 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) int n = unpacked[-1]; while (n > 0) { - if (!is_header(ap[0])) { + switch (loader_tag(ap[0])) { + case LOADER_X_REG: + erts_print(to, to_arg, " x(%d)", loader_x_reg_index(ap[0])); + break; + case LOADER_Y_REG: + erts_print(to, to_arg, " x(%d)", loader_y_reg_index(ap[0])); + break; + default: erts_print(to, to_arg, " %T", (Eterm) ap[0]); - } else { - switch ((ap[0] >> 2) & 0x03) { - case R_REG_DEF: - erts_print(to, to_arg, " x(0)"); - break; - case X_REG_DEF: - erts_print(to, to_arg, " x(%d)", ap[0] >> 4); - break; - case Y_REG_DEF: - erts_print(to, to_arg, " y(%d)", ap[0] >> 4); - break; - } + break; } ap++, size++, n--; } @@ -699,18 +675,16 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) while (n > 0) { if (n % 3 == 1) { erts_print(to, to_arg, " %X", ap[0]); - } else if (!is_header(ap[0])) { - erts_print(to, to_arg, " %T", (Eterm) ap[0]); } else { - switch ((ap[0] >> 2) & 0x03) { - case R_REG_DEF: - erts_print(to, to_arg, " x(0)"); + switch (loader_tag(ap[0])) { + case LOADER_X_REG: + erts_print(to, to_arg, " x(%d)", loader_x_reg_index(ap[0])); break; - case X_REG_DEF: - erts_print(to, to_arg, " x(%d)", ap[0] >> 4); + case LOADER_Y_REG: + erts_print(to, to_arg, " y(%d)", loader_y_reg_index(ap[0])); break; - case Y_REG_DEF: - erts_print(to, to_arg, " y(%d)", ap[0] >> 4); + default: + erts_print(to, to_arg, " %T", (Eterm) ap[0]); break; } } diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 38def5d89f..faf496a030 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -99,10 +99,7 @@ do { \ do { \ int i_; \ int Arity_ = PC[-1]; \ - if (Arity_ > 0) { \ - CHECK_TERM(r(0)); \ - } \ - for (i_ = 1; i_ < Arity_; i_++) { \ + for (i_ = 0; i_ < Arity_; i_++) { \ CHECK_TERM(x(i_)); \ } \ } while (0) @@ -151,25 +148,21 @@ do { \ ASSERT(VALID_INSTR(* (Eterm *)(ip))); \ I = (ip) -#define FetchArgs(S1, S2) tmp_arg1 = (S1); tmp_arg2 = (S2) +/* + * Register target (X or Y register). + */ +#define REG_TARGET(Target) (*(((Target) & 1) ? &yb(Target-1) : &xb(Target))) /* * Store a result into a register given a destination descriptor. */ -#define StoreResult(Result, DestDesc) \ - do { \ - Eterm stb_reg; \ - stb_reg = (DestDesc); \ - CHECK_TERM(Result); \ - switch (beam_reg_tag(stb_reg)) { \ - case R_REG_DEF: \ - r(0) = (Result); break; \ - case X_REG_DEF: \ - xb(x_reg_offset(stb_reg)) = (Result); break; \ - default: \ - yb(y_reg_offset(stb_reg)) = (Result); break; \ - } \ +#define StoreResult(Result, DestDesc) \ + do { \ + Eterm stb_reg; \ + stb_reg = (DestDesc); \ + CHECK_TERM(Result); \ + REG_TARGET(stb_reg) = (Result); \ } while (0) #define StoreSimpleDest(Src, Dest) Dest = (Src) @@ -180,22 +173,16 @@ do { \ * be just before the next instruction. */ -#define StoreBifResult(Dst, Result) \ - do { \ - BeamInstr* stb_next; \ - Eterm stb_reg; \ - stb_reg = Arg(Dst); \ - I += (Dst) + 2; \ - stb_next = (BeamInstr *) *I; \ - CHECK_TERM(Result); \ - switch (beam_reg_tag(stb_reg)) { \ - case R_REG_DEF: \ - r(0) = (Result); Goto(stb_next); \ - case X_REG_DEF: \ - xb(x_reg_offset(stb_reg)) = (Result); Goto(stb_next); \ - default: \ - yb(y_reg_offset(stb_reg)) = (Result); Goto(stb_next); \ - } \ +#define StoreBifResult(Dst, Result) \ + do { \ + BeamInstr* stb_next; \ + Eterm stb_reg; \ + stb_reg = Arg(Dst); \ + I += (Dst) + 2; \ + stb_next = (BeamInstr *) *I; \ + CHECK_TERM(Result); \ + REG_TARGET(stb_reg) = (Result); \ + Goto(stb_next); \ } while (0) #define ClauseFail() goto jump_f @@ -293,7 +280,7 @@ void** beam_ops; #define Ib(N) (N) #define x(N) reg[N] #define y(N) E[N] -#define r(N) x##N +#define r(N) x(N) /* * Makes sure that there are StackNeed + HeapNeed + 1 words available @@ -309,12 +296,10 @@ void** beam_ops; needed = (StackNeed) + 1; \ if (E - HTOP < (needed + (HeapNeed))) { \ SWAPOUT; \ - reg[0] = r(0); \ PROCESS_MAIN_CHK_LOCKS(c_p); \ FCALLS -= erts_garbage_collect(c_p, needed + (HeapNeed), reg, (M)); \ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \ PROCESS_MAIN_CHK_LOCKS(c_p); \ - r(0) = reg[0]; \ SWAPIN; \ } \ E -= needed; \ @@ -363,12 +348,10 @@ void** beam_ops; unsigned need = (Nh); \ if ((E - HTOP < need) || (MSO(c_p).overhead + (VNh) >= BIN_VHEAP_SZ(c_p))) {\ SWAPOUT; \ - reg[0] = r(0); \ PROCESS_MAIN_CHK_LOCKS(c_p); \ FCALLS -= erts_garbage_collect(c_p, need, reg, (Live)); \ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \ PROCESS_MAIN_CHK_LOCKS(c_p); \ - r(0) = reg[0]; \ SWAPIN; \ } \ HEAP_SPACE_VERIFIED(need); \ @@ -386,12 +369,10 @@ void** beam_ops; unsigned need = (Nh); \ if (E - HTOP < need) { \ SWAPOUT; \ - reg[0] = r(0); \ PROCESS_MAIN_CHK_LOCKS(c_p); \ FCALLS -= erts_garbage_collect(c_p, need, reg, (Live)); \ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \ PROCESS_MAIN_CHK_LOCKS(c_p); \ - r(0) = reg[0]; \ SWAPIN; \ } \ HEAP_SPACE_VERIFIED(need); \ @@ -408,15 +389,11 @@ void** beam_ops; unsigned need = (Nh); \ if (E - HTOP < need) { \ SWAPOUT; \ - reg[0] = r(0); \ reg[Live] = Extra; \ PROCESS_MAIN_CHK_LOCKS(c_p); \ FCALLS -= erts_garbage_collect(c_p, need, reg, (Live)+1); \ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \ PROCESS_MAIN_CHK_LOCKS(c_p); \ - if (Live > 0) { \ - r(0) = reg[0]; \ - } \ Extra = reg[Live]; \ SWAPIN; \ } \ @@ -439,7 +416,6 @@ void** beam_ops; #define MakeFun(FunP, NumFree) \ do { \ SWAPOUT; \ - reg[0] = r(0); \ r(0) = new_fun(c_p, reg, (ErlFunEntry *) FunP, NumFree); \ SWAPIN; \ } while (0) @@ -531,15 +507,19 @@ void** beam_ops; ASSERT(VALID_INSTR(Dst)); \ Goto(Dst) -#define GetR(pos, tr) \ - do { \ - tr = Arg(pos); \ - switch (beam_reg_tag(tr)) { \ - case R_REG_DEF: tr = r(0); break; \ - case X_REG_DEF: tr = xb(x_reg_offset(tr)); break; \ - case Y_REG_DEF: ASSERT(y_reg_offset(tr) >= 1); tr = yb(y_reg_offset(tr)); break; \ - } \ - CHECK_TERM(tr); \ +#define GetR(pos, tr) \ + do { \ + tr = Arg(pos); \ + switch (loader_tag(tr)) { \ + case LOADER_X_REG: \ + tr = x(loader_x_reg_index(tr)); \ + break; \ + case LOADER_Y_REG: \ + ASSERT(loader_y_reg_index(tr) >= 1); \ + tr = y(loader_y_reg_index(tr)); \ + break; \ + } \ + CHECK_TERM(tr); \ } while (0) #define GetArg1(N, Dst) GetR((N), Dst) @@ -557,24 +537,93 @@ void** beam_ops; HTOP += 2; \ } while (0) +#define Swap(R1, R2) \ + do { \ + Eterm V = R1; \ + R1 = R2; \ + R2 = V; \ + } while (0) + +#define SwapTemp(R1, R2, Tmp) \ + do { \ + Eterm V = R1; \ + R1 = R2; \ + R2 = Tmp = V; \ + } while (0) + #define Move(Src, Dst, Store) \ do { \ Eterm term = (Src); \ Store(term, Dst); \ } while (0) -#define Move2(S1, D1, S2, D2) D1 = (S1); D2 = (S2) +#define Move2Par(S1, D1, S2, D2) \ + do { \ + Eterm V1, V2; \ + V1 = (S1); V2 = (S2); D1 = V1; D2 = V2; \ + } while (0) + +#define MoveShift(Src, SD, D) \ + do { \ + Eterm V; \ + V = Src; D = SD; SD = V; \ + } while (0) + +#define MoveDup(Src, D1, D2) \ + do { \ + D1 = D2 = (Src); \ + } while (0) + #define Move3(S1, D1, S2, D2, S3, D3) D1 = (S1); D2 = (S2); D3 = (S3) -#define MoveGenDest(src, dstp) \ - if ((dstp) == NULL) { r(0) = (src); } else { *(dstp) = src; } +#define MoveWindow3(S1, S2, S3, D) \ + do { \ + Eterm xt0, xt1, xt2; \ + Eterm *y = &D; \ + xt0 = S1; \ + xt1 = S2; \ + xt2 = S3; \ + y[0] = xt0; \ + y[1] = xt1; \ + y[2] = xt2; \ + } while (0) + +#define MoveWindow4(S1, S2, S3, S4, D) \ + do { \ + Eterm xt0, xt1, xt2, xt3; \ + Eterm *y = &D; \ + xt0 = S1; \ + xt1 = S2; \ + xt2 = S3; \ + xt3 = S4; \ + y[0] = xt0; \ + y[1] = xt1; \ + y[2] = xt2; \ + y[3] = xt3; \ + } while (0) -#define MoveReturn(Src, Dest) \ - (Dest) = (Src); \ - I = c_p->cp; \ - ASSERT(VALID_INSTR(*c_p->cp)); \ - c_p->cp = 0; \ - CHECK_TERM(r(0)); \ +#define MoveWindow5(S1, S2, S3, S4, S5, D) \ + do { \ + Eterm xt0, xt1, xt2, xt3, xt4; \ + Eterm *y = &D; \ + xt0 = S1; \ + xt1 = S2; \ + xt2 = S3; \ + xt3 = S4; \ + xt4 = S5; \ + y[0] = xt0; \ + y[1] = xt1; \ + y[2] = xt2; \ + y[3] = xt3; \ + y[4] = xt4; \ + } while (0) + +#define MoveReturn(Src) \ + x(0) = (Src); \ + I = c_p->cp; \ + ASSERT(VALID_INSTR(*c_p->cp)); \ + c_p->cp = 0; \ + CHECK_TERM(r(0)); \ Goto(*I) #define DeallocateReturn(Deallocate) \ @@ -586,26 +635,26 @@ void** beam_ops; Goto(*I); \ } while (0) -#define MoveDeallocateReturn(Src, Dest, Deallocate) \ - (Dest) = (Src); \ +#define MoveDeallocateReturn(Src, Deallocate) \ + x(0) = (Src); \ DeallocateReturn(Deallocate) -#define MoveCall(Src, Dest, CallDest, Size) \ - (Dest) = (Src); \ +#define MoveCall(Src, CallDest, Size) \ + x(0) = (Src); \ SET_CP(c_p, I+Size+1); \ - SET_I((BeamInstr *) CallDest); \ + SET_I((BeamInstr *) CallDest); \ Dispatch(); -#define MoveCallLast(Src, Dest, CallDest, Deallocate) \ - (Dest) = (Src); \ - RESTORE_CP(E); \ - E = ADD_BYTE_OFFSET(E, (Deallocate)); \ - SET_I((BeamInstr *) CallDest); \ +#define MoveCallLast(Src, CallDest, Deallocate) \ + x(0) = (Src); \ + RESTORE_CP(E); \ + E = ADD_BYTE_OFFSET(E, (Deallocate)); \ + SET_I((BeamInstr *) CallDest); \ Dispatch(); -#define MoveCallOnly(Src, Dest, CallDest) \ - (Dest) = (Src); \ - SET_I((BeamInstr *) CallDest); \ +#define MoveCallOnly(Src, CallDest) \ + x(0) = (Src); \ + SET_I((BeamInstr *) CallDest); \ Dispatch(); #define MoveJump(Src) \ @@ -613,58 +662,56 @@ void** beam_ops; SET_I((BeamInstr *) Arg(0)); \ Goto(*I); -#define GetList(Src, H, T) do { \ - Eterm* tmp_ptr = list_val(Src); \ - H = CAR(tmp_ptr); \ - T = CDR(tmp_ptr); } while (0) - -#define GetTupleElement(Src, Element, Dest) \ - do { \ - tmp_arg1 = (Eterm) COMPRESS_POINTER(((unsigned char *) tuple_val(Src)) + \ - (Element)); \ - (Dest) = (*(Eterm *) EXPAND_POINTER(tmp_arg1)); \ - } while (0) - -#define ExtractNextElement(Dest) \ - tmp_arg1 += sizeof(Eterm); \ - (Dest) = (* (Eterm *) (((unsigned char *) EXPAND_POINTER(tmp_arg1)))) - -#define ExtractNextElement2(Dest) \ - do { \ - Eterm* ene_dstp = &(Dest); \ - ene_dstp[0] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[1]; \ - ene_dstp[1] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[2]; \ - tmp_arg1 += sizeof(Eterm) + sizeof(Eterm); \ +#define GetList(Src, H, T) \ + do { \ + Eterm* tmp_ptr = list_val(Src); \ + Eterm hd, tl; \ + hd = CAR(tmp_ptr); \ + tl = CDR(tmp_ptr); \ + H = hd; T = tl; \ } while (0) -#define ExtractNextElement3(Dest) \ - do { \ - Eterm* ene_dstp = &(Dest); \ - ene_dstp[0] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[1]; \ - ene_dstp[1] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[2]; \ - ene_dstp[2] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[3]; \ - tmp_arg1 += 3*sizeof(Eterm); \ +#define GetTupleElement(Src, Element, Dest) \ + do { \ + Eterm* src; \ + src = ADD_BYTE_OFFSET(tuple_val(Src), (Element)); \ + (Dest) = *src; \ } while (0) -#define ExtractNextElement4(Dest) \ - do { \ - Eterm* ene_dstp = &(Dest); \ - ene_dstp[0] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[1]; \ - ene_dstp[1] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[2]; \ - ene_dstp[2] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[3]; \ - ene_dstp[3] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[4]; \ - tmp_arg1 += 4*sizeof(Eterm); \ +#define GetTupleElement2(Src, Element, Dest) \ + do { \ + Eterm* src; \ + Eterm* dst; \ + Eterm E1, E2; \ + src = ADD_BYTE_OFFSET(tuple_val(Src), (Element)); \ + dst = &(Dest); \ + E1 = src[0]; \ + E2 = src[1]; \ + dst[0] = E1; \ + dst[1] = E2; \ } while (0) -#define ExtractElement(Element, Dest) \ - do { \ - tmp_arg1 += (Element); \ - (Dest) = (* (Eterm *) EXPAND_POINTER(tmp_arg1)); \ +#define GetTupleElement3(Src, Element, Dest) \ + do { \ + Eterm* src; \ + Eterm* dst; \ + Eterm E1, E2, E3; \ + src = ADD_BYTE_OFFSET(tuple_val(Src), (Element)); \ + dst = &(Dest); \ + E1 = src[0]; \ + E2 = src[1]; \ + E3 = src[2]; \ + dst[0] = E1; \ + dst[1] = E2; \ + dst[2] = E3; \ } while (0) #define EqualImmed(X, Y, Action) if (X != Y) { Action; } #define NotEqualImmed(X, Y, Action) if (X == Y) { Action; } #define EqualExact(X, Y, Action) if (!EQ(X,Y)) { Action; } +#define NotEqualExact(X, Y, Action) if (EQ(X,Y)) { Action; } +#define Equal(X, Y, Action) if (!CMP_EQ(X,Y)) { Action; } +#define NotEqual(X, Y, Action) if (!CMP_NE(X,Y)) { Action; } #define IsLessThan(X, Y, Action) if (CMP_GE(X, Y)) { Action; } #define IsGreaterEqual(X, Y, Action) if (CMP_LT(X, Y)) { Action; } @@ -690,18 +737,26 @@ void** beam_ops; if (is_not_list(Src)) { Fail; } \ A(Need, Alive) -#define IsNonemptyListTestHeap(Src, Need, Alive, Fail) \ - if (is_not_list(Src)) { Fail; } \ +#define IsNonemptyListTestHeap(Need, Alive, Fail) \ + if (is_not_list(x(0))) { Fail; } \ TestHeap(Need, Alive) +#define IsNonemptyListGetList(Src, H, T, Fail) \ + if (is_not_list(Src)) { \ + Fail; \ + } else { \ + Eterm* tmp_ptr = list_val(Src); \ + Eterm hd, tl; \ + hd = CAR(tmp_ptr); \ + tl = CDR(tmp_ptr); \ + H = hd; T = tl; \ + } + #define IsTuple(X, Action) if (is_not_tuple(X)) Action -#define IsArity(Pointer, Arity, Fail) \ - if (*(Eterm *) \ - EXPAND_POINTER(tmp_arg1 = (Eterm) \ - COMPRESS_POINTER(tuple_val(Pointer))) != (Arity)) \ - { \ - Fail; \ +#define IsArity(Pointer, Arity, Fail) \ + if (*tuple_val(Pointer) != (Arity)) { \ + Fail; \ } #define IsMap(Src, Fail) if (!is_map(Src)) { Fail; } @@ -738,15 +793,21 @@ void** beam_ops; } \ } while (0) -#define IsTupleOfArity(Src, Arity, Fail) \ - do { \ - if (is_not_tuple(Src) || \ - *(Eterm *) \ - EXPAND_POINTER(tmp_arg1 = \ - (Eterm) COMPRESS_POINTER(tuple_val(Src))) != Arity) { \ - Fail; \ - } \ +#ifdef DEBUG +#define IsTupleOfArity(Src, Arityval, Fail) \ + do { \ + if (!(is_tuple(Src) && *tuple_val(Src) == Arityval)) { \ + Fail; \ + } \ } while (0) +#else +#define IsTupleOfArity(Src, Arityval, Fail) \ + do { \ + if (!(is_boxed(Src) && *tuple_val(Src) == Arityval)) { \ + Fail; \ + } \ + } while (0) +#endif #define IsBoolean(X, Fail) if ((X) != am_true && (X) != am_false) { Fail; } @@ -756,7 +817,7 @@ void** beam_ops; #define IsBitstring(Src, Fail) \ if (is_not_binary(Src)) { Fail; } -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) #define BsSafeMul(A, B, Fail, Target) \ do { Uint64 _res = (A) * (B); \ if (_res / B != A) { Fail; } \ @@ -773,6 +834,7 @@ void** beam_ops; #define BsGetFieldSize(Bits, Unit, Fail, Target) \ do { \ Sint _signed_size; Uint _uint_size; \ + Uint temp_bits; \ if (is_small(Bits)) { \ _signed_size = signed_val(Bits); \ if (_signed_size < 0) { Fail; } \ @@ -787,6 +849,7 @@ void** beam_ops; #define BsGetUncheckedFieldSize(Bits, Unit, Fail, Target) \ do { \ Sint _signed_size; Uint _uint_size; \ + Uint temp_bits; \ if (is_small(Bits)) { \ _signed_size = signed_val(Bits); \ if (_signed_size < 0) { Fail; } \ @@ -998,23 +1061,17 @@ init_emulator(void) */ #if defined(__GNUC__) && defined(sparc) && !defined(DEBUG) -# define REG_x0 asm("%l0") # define REG_xregs asm("%l1") # define REG_htop asm("%l2") # define REG_stop asm("%l3") # define REG_I asm("%l4") # define REG_fcalls asm("%l5") -# define REG_tmp_arg1 asm("%l6") -# define REG_tmp_arg2 asm("%l7") #else -# define REG_x0 # define REG_xregs # define REG_htop # define REG_stop # define REG_I # define REG_fcalls -# define REG_tmp_arg1 -# define REG_tmp_arg2 #endif #ifdef USE_VM_PROBES @@ -1132,11 +1189,6 @@ void process_main(void) ERTS_DECLARE_DUMMY(Eterm pid); #endif - /* - * X register zero; also called r(0) - */ - register Eterm x0 REG_x0 = NIL; - /* Pointer to X registers: x(1)..x(N); reg[0] is used when doing GC, * in all other cases x0 is used. */ @@ -1164,17 +1216,6 @@ void process_main(void) register Sint FCALLS REG_fcalls = 0; /* - * Temporaries used for picking up arguments for instructions. - */ - register Eterm tmp_arg1 REG_tmp_arg1 = NIL; - register Eterm tmp_arg2 REG_tmp_arg2 = NIL; -#if HEAP_ON_C_STACK - Eterm tmp_big[2]; /* Temporary buffer for small bignums if HEAP_ON_C_STACK. */ -#else - Eterm *tmp_big; /* Temporary buffer for small bignums if !HEAP_ON_C_STACK. */ -#endif - - /* * X registers and floating point registers are located in * scheduler specific data. */ @@ -1185,8 +1226,6 @@ void process_main(void) */ int neg_o_reds = 0; - Eterm (*arith_func)(Process* p, Eterm* reg, Uint live); - #ifdef ERTS_OPCODE_COUNTER_SUPPORT static void* counting_opcodes[] = { DEFINE_COUNTING_OPCODES }; #else @@ -1197,8 +1236,6 @@ void process_main(void) #endif #endif - Uint temp_bits; /* Temporary used by BsSkipBits2 & BsGetInteger2 */ - Eterm pt_arity; /* Used by do_put_tuple */ Uint64 start_time = 0; /* Monitor long schedule */ @@ -1247,9 +1284,6 @@ void process_main(void) PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); -#if HALFWORD_HEAP - ASSERT(erts_get_scheduler_data()->num_tmp_heap_used == 0); -#endif ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); c_p = schedule(c_p, reds_used); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); @@ -1267,9 +1301,6 @@ void process_main(void) reg = ERTS_PROC_GET_SCHDATA(c_p)->x_reg_array; freg = ERTS_PROC_GET_SCHDATA(c_p)->f_reg_array; -#if !HEAP_ON_C_STACK - tmp_big = ERTS_PROC_GET_SCHDATA(c_p)->beam_emu_tmp_heap; -#endif ERL_BITS_RELOAD_STATEP(c_p); { int reds; @@ -1278,7 +1309,7 @@ void process_main(void) int i; argp = c_p->arg_reg; - for (i = c_p->arity - 1; i > 0; i--) { + for (i = c_p->arity - 1; i >= 0; i--) { reg[i] = argp[i]; CHECK_TERM(reg[i]); } @@ -1302,12 +1333,6 @@ void process_main(void) } next = (BeamInstr *) *I; - r(0) = c_p->arg_reg[0]; -#ifdef HARDDEBUG - if (c_p->arity > 0) { - CHECK_TERM(r(0)); - } -#endif SWAPIN; ASSERT(VALID_INSTR(next)); @@ -1346,26 +1371,24 @@ void process_main(void) #endif #include "beam_hot.h" -#define STORE_ARITH_RESULT(res) StoreBifResult(2, (res)); -#define ARITH_FUNC(name) erts_gc_##name - { Eterm increment_reg_val; Eterm increment_val; Uint live; Eterm result; - OpCase(i_increment_yIId): - increment_reg_val = yb(Arg(0)); + OpCase(i_increment_rIId): + increment_reg_val = x(0); + I--; goto do_increment; OpCase(i_increment_xIId): increment_reg_val = xb(Arg(0)); goto do_increment; - OpCase(i_increment_rIId): - increment_reg_val = r(0); - I--; + OpCase(i_increment_yIId): + increment_reg_val = yb(Arg(0)); + goto do_increment; do_increment: increment_val = Arg(1); @@ -1374,229 +1397,133 @@ void process_main(void) ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i)); if (MY_IS_SSMALL(i)) { result = make_small(i); - store_result: StoreBifResult(3, result); } } live = Arg(2); SWAPOUT; - reg[0] = r(0); reg[live] = increment_reg_val; reg[live+1] = make_small(increment_val); result = erts_gc_mixed_plus(c_p, reg, live); - r(0) = reg[0]; SWAPIN; ERTS_HOLE_CHECK(c_p); if (is_value(result)) { - goto store_result; + StoreBifResult(3, result); } ASSERT(c_p->freason != BADMATCH || is_value(c_p->fvalue)); goto find_func_info; } -#define DO_BIG_ARITH(Func,Arg1,Arg2) \ - do { \ - Uint live = Arg(1); \ - SWAPOUT; \ - reg[0] = r(0); \ - reg[live] = (Arg1); \ - reg[live+1] = (Arg2); \ - result = (Func)(c_p, reg, live); \ - r(0) = reg[0]; \ - SWAPIN; \ - ERTS_HOLE_CHECK(c_p); \ - if (is_value(result)) { \ - StoreBifResult(4,result); \ - } \ - goto lb_Cl_error; \ - } while(0) +#define DO_OUTLINED_ARITH_2(name, Op1, Op2) \ + do { \ + Eterm result; \ + Uint live = Arg(1); \ + \ + SWAPOUT; \ + reg[live] = Op1; \ + reg[live+1] = Op2; \ + result = erts_gc_##name(c_p, reg, live); \ + SWAPIN; \ + ERTS_HOLE_CHECK(c_p); \ + if (is_value(result)) { \ + StoreBifResult(4, result); \ + } \ + goto lb_Cl_error; \ + } while (0) - OpCase(i_plus_jIxxd): { + Eterm PlusOp1, PlusOp2; Eterm result; - if (is_both_small(xb(Arg(2)), xb(Arg(3)))) { - Sint i = signed_val(xb(Arg(2))) + signed_val(xb(Arg(3))); + OpCase(i_plus_jIxxd): + PlusOp1 = xb(Arg(2)); + PlusOp2 = xb(Arg(3)); + goto do_plus; + + OpCase(i_plus_jIxyd): + PlusOp1 = xb(Arg(2)); + PlusOp2 = yb(Arg(3)); + goto do_plus; + + OpCase(i_plus_jIssd): + GetArg2(2, PlusOp1, PlusOp2); + goto do_plus; + + do_plus: + if (is_both_small(PlusOp1, PlusOp2)) { + Sint i = signed_val(PlusOp1) + signed_val(PlusOp2); ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i)); if (MY_IS_SSMALL(i)) { result = make_small(i); StoreBifResult(4, result); } } - DO_BIG_ARITH(ARITH_FUNC(mixed_plus), xb(Arg(2)), xb(Arg(3))); + DO_OUTLINED_ARITH_2(mixed_plus, PlusOp1, PlusOp2); } - OpCase(i_plus_jId): { + Eterm MinusOp1, MinusOp2; Eterm result; - if (is_both_small(tmp_arg1, tmp_arg2)) { - Sint i = signed_val(tmp_arg1) + signed_val(tmp_arg2); - ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i)); - if (MY_IS_SSMALL(i)) { - result = make_small(i); - STORE_ARITH_RESULT(result); - } - } - arith_func = ARITH_FUNC(mixed_plus); - goto do_big_arith2; - } - OpCase(i_minus_jIxxd): - { - Eterm result; - - if (is_both_small(xb(Arg(2)), xb(Arg(3)))) { - Sint i = signed_val(xb(Arg(2))) - signed_val(xb(Arg(3))); - ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i)); - if (MY_IS_SSMALL(i)) { - result = make_small(i); - StoreBifResult(4, result); - } - } - DO_BIG_ARITH(ARITH_FUNC(mixed_minus), xb(Arg(2)), xb(Arg(3))); - } + MinusOp1 = xb(Arg(2)); + MinusOp2 = xb(Arg(3)); + goto do_minus; - OpCase(i_minus_jId): - { - Eterm result; + OpCase(i_minus_jIssd): + GetArg2(2, MinusOp1, MinusOp2); + goto do_minus; - if (is_both_small(tmp_arg1, tmp_arg2)) { - Sint i = signed_val(tmp_arg1) - signed_val(tmp_arg2); + do_minus: + if (is_both_small(MinusOp1, MinusOp2)) { + Sint i = signed_val(MinusOp1) - signed_val(MinusOp2); ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i)); if (MY_IS_SSMALL(i)) { result = make_small(i); - STORE_ARITH_RESULT(result); + StoreBifResult(4, result); } } - arith_func = ARITH_FUNC(mixed_minus); - goto do_big_arith2; + DO_OUTLINED_ARITH_2(mixed_minus, MinusOp1, MinusOp2); } - OpCase(i_is_lt_f): - if (CMP_GE(tmp_arg1, tmp_arg2)) { - ClauseFail(); - } - Next(1); - - OpCase(i_is_ge_f): - if (CMP_LT(tmp_arg1, tmp_arg2)) { - ClauseFail(); - } - Next(1); - - OpCase(i_is_eq_f): - if (CMP_NE(tmp_arg1, tmp_arg2)) { - ClauseFail(); - } - Next(1); - - OpCase(i_is_ne_f): - if (CMP_EQ(tmp_arg1, tmp_arg2)) { - ClauseFail(); - } - Next(1); - - OpCase(i_is_eq_exact_f): - if (!EQ(tmp_arg1, tmp_arg2)) { - ClauseFail(); - } - Next(1); - { Eterm is_eq_exact_lit_val; - OpCase(i_is_eq_exact_literal_xfc): - is_eq_exact_lit_val = xb(Arg(0)); - I++; + OpCase(i_is_eq_exact_literal_fxc): + is_eq_exact_lit_val = xb(Arg(1)); goto do_is_eq_exact_literal; - OpCase(i_is_eq_exact_literal_yfc): - is_eq_exact_lit_val = yb(Arg(0)); - I++; + OpCase(i_is_eq_exact_literal_fyc): + is_eq_exact_lit_val = yb(Arg(1)); goto do_is_eq_exact_literal; - OpCase(i_is_eq_exact_literal_rfc): - is_eq_exact_lit_val = r(0); - do_is_eq_exact_literal: - if (!eq(Arg(1), is_eq_exact_lit_val)) { + if (!eq(Arg(2), is_eq_exact_lit_val)) { ClauseFail(); } - Next(2); + Next(3); } { Eterm is_ne_exact_lit_val; - OpCase(i_is_ne_exact_literal_xfc): - is_ne_exact_lit_val = xb(Arg(0)); - I++; + OpCase(i_is_ne_exact_literal_fxc): + is_ne_exact_lit_val = xb(Arg(1)); goto do_is_ne_exact_literal; - OpCase(i_is_ne_exact_literal_yfc): - is_ne_exact_lit_val = yb(Arg(0)); - I++; + OpCase(i_is_ne_exact_literal_fyc): + is_ne_exact_lit_val = yb(Arg(1)); goto do_is_ne_exact_literal; - OpCase(i_is_ne_exact_literal_rfc): - is_ne_exact_lit_val = r(0); - do_is_ne_exact_literal: - if (eq(Arg(1), is_ne_exact_lit_val)) { + if (eq(Arg(2), is_ne_exact_lit_val)) { ClauseFail(); } - Next(2); + Next(3); } - OpCase(move_window3_xxxy): { - BeamInstr *next; - Eterm xt0, xt1, xt2; - Eterm *y = (Eterm *)(((unsigned char *)E) + (Arg(3))); - PreFetch(4, next); - xt0 = xb(Arg(0)); - xt1 = xb(Arg(1)); - xt2 = xb(Arg(2)); - y[0] = xt0; - y[1] = xt1; - y[2] = xt2; - NextPF(4, next); - } - OpCase(move_window4_xxxxy): { - BeamInstr *next; - Eterm xt0, xt1, xt2, xt3; - Eterm *y = (Eterm *)(((unsigned char *)E) + (Arg(4))); - PreFetch(5, next); - xt0 = xb(Arg(0)); - xt1 = xb(Arg(1)); - xt2 = xb(Arg(2)); - xt3 = xb(Arg(3)); - y[0] = xt0; - y[1] = xt1; - y[2] = xt2; - y[3] = xt3; - NextPF(5, next); - } - OpCase(move_window5_xxxxxy): { - BeamInstr *next; - Eterm xt0, xt1, xt2, xt3, xt4; - Eterm *y = (Eterm *)(((unsigned char *)E) + (Arg(5))); - PreFetch(6, next); - xt0 = xb(Arg(0)); - xt1 = xb(Arg(1)); - xt2 = xb(Arg(2)); - xt3 = xb(Arg(3)); - xt4 = xb(Arg(4)); - y[0] = xt0; - y[1] = xt1; - y[2] = xt2; - y[3] = xt3; - y[4] = xt4; - NextPF(6, next); - } - - OpCase(i_move_call_only_fcr): { + OpCase(i_move_call_only_fc): { r(0) = Arg(1); } /* FALL THROUGH */ @@ -1606,7 +1533,7 @@ void process_main(void) Dispatch(); } - OpCase(i_move_call_last_fPcr): { + OpCase(i_move_call_last_fPc): { r(0) = Arg(2); } /* FALL THROUGH */ @@ -1618,7 +1545,7 @@ void process_main(void) Dispatch(); } - OpCase(i_move_call_crf): { + OpCase(i_move_call_cf): { r(0) = Arg(0); I++; } @@ -1630,7 +1557,7 @@ void process_main(void) Dispatch(); } - OpCase(i_move_call_ext_last_ePcr): { + OpCase(i_move_call_ext_last_ePc): { r(0) = Arg(2); } /* FALL THROUGH */ @@ -1646,7 +1573,7 @@ void process_main(void) DTRACE_GLOBAL_CALL_FROM_EXPORT(c_p, Arg(0)); Dispatchx(); - OpCase(i_move_call_ext_cre): { + OpCase(i_move_call_ext_ce): { r(0) = Arg(0); I++; } @@ -1656,7 +1583,7 @@ void process_main(void) DTRACE_GLOBAL_CALL_FROM_EXPORT(c_p, Arg(0)); Dispatchx(); - OpCase(i_move_call_ext_only_ecr): { + OpCase(i_move_call_ext_only_ec): { r(0) = Arg(1); } /* FALL THROUGH */ @@ -1722,7 +1649,6 @@ void process_main(void) PRE_BIF_SWAPOUT(c_p); c_p->fcalls = FCALLS - 1; - reg[0] = r(0); result = erl_send(c_p, r(0), x(1)); PreFetch(0, next); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); @@ -1730,7 +1656,6 @@ void process_main(void) PROCESS_MAIN_CHK_LOCKS(c_p); if (c_p->mbuf || MSO(c_p).overhead >= BIN_VHEAP_SZ(c_p)) { result = erts_gc_after_bif_call(c_p, result, reg, 2); - r(0) = reg[0]; E = c_p->stop; } HTOP = HEAP_TOP(c_p); @@ -1743,7 +1668,6 @@ void process_main(void) SET_CP(c_p, I+1); SET_I(c_p->i); SWAPIN; - r(0) = reg[0]; Dispatch(); } goto find_func_info; @@ -1753,29 +1677,23 @@ void process_main(void) Eterm element_index; Eterm element_tuple; - OpCase(i_element_xjsd): - element_tuple = xb(Arg(0)); - I++; + OpCase(i_element_jxsd): + element_tuple = xb(Arg(1)); goto do_element; - OpCase(i_element_yjsd): - element_tuple = yb(Arg(0)); - I++; + OpCase(i_element_jysd): + element_tuple = yb(Arg(1)); goto do_element; - OpCase(i_element_rjsd): - element_tuple = r(0); - /* Fall through */ - do_element: - GetArg1(1, element_index); + GetArg1(2, element_index); if (is_small(element_index) && is_tuple(element_tuple)) { Eterm* tp = tuple_val(element_tuple); if ((signed_val(element_index) >= 1) && (signed_val(element_index) <= arityval(*tp))) { Eterm result = tp[signed_val(element_index)]; - StoreBifResult(2, result); + StoreBifResult(3, result); } } } @@ -1789,29 +1707,24 @@ void process_main(void) { Eterm fast_element_tuple; - OpCase(i_fast_element_rjId): - fast_element_tuple = r(0); + OpCase(i_fast_element_jxId): + fast_element_tuple = xb(Arg(1)); + goto do_fast_element; + + OpCase(i_fast_element_jyId): + fast_element_tuple = yb(Arg(1)); + goto do_fast_element; do_fast_element: if (is_tuple(fast_element_tuple)) { Eterm* tp = tuple_val(fast_element_tuple); - Eterm pos = Arg(1); /* Untagged integer >= 1 */ + Eterm pos = Arg(2); /* Untagged integer >= 1 */ if (pos <= arityval(*tp)) { Eterm result = tp[pos]; - StoreBifResult(2, result); + StoreBifResult(3, result); } } goto badarg; - - OpCase(i_fast_element_xjId): - fast_element_tuple = xb(Arg(0)); - I++; - goto do_fast_element; - - OpCase(i_fast_element_yjId): - fast_element_tuple = yb(Arg(0)); - I++; - goto do_fast_element; } OpCase(catch_yf): @@ -1917,7 +1830,7 @@ void process_main(void) * Pick up the next message and place it in x(0). * If no message, jump to a wait or wait_timeout instruction. */ - OpCase(i_loop_rec_fr): + OpCase(i_loop_rec_f): { BeamInstr *next; ErlMessage* msgp; @@ -1951,13 +1864,11 @@ void process_main(void) ErtsMoveMsgAttachmentIntoProc(msgp, c_p, E, HTOP, FCALLS, { SWAPOUT; - reg[0] = r(0); PROCESS_MAIN_CHK_LOCKS(c_p); }, { ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); - r(0) = reg[0]; SWAPIN; }); if (is_non_value(ERL_MESSAGE_TERM(msgp))) { @@ -2131,8 +2042,6 @@ void process_main(void) * c_p->def_arg_reg[0]. Note that it is safe to use this * location because there are no living x registers in * a receive statement. - * Note that for the halfword emulator, the two first elements - * of the array are used. */ BeamInstr** pi = (BeamInstr**) c_p->def_arg_reg; *pi = I+3; @@ -2249,10 +2158,6 @@ void process_main(void) select_val2 = xb(Arg(0)); goto do_select_tuple_arity2; - OpCase(i_select_tuple_arity2_rfAAff): - select_val2 = r(0); - I--; - do_select_tuple_arity2: if (is_not_tuple(select_val2)) { goto select_val2_fail; @@ -2268,10 +2173,6 @@ void process_main(void) select_val2 = xb(Arg(0)); goto do_select_val2; - OpCase(i_select_val2_rfccff): - select_val2 = r(0); - I--; - do_select_val2: if (select_val2 == Arg(2)) { I += 3; @@ -2295,10 +2196,6 @@ void process_main(void) select_val = yb(Arg(0)); goto do_select_tuple_arity; - OpCase(i_select_tuple_arity_rfI): - select_val = r(0); - I--; - do_select_tuple_arity: if (is_tuple(select_val)) { select_val = *tuple_val(select_val); @@ -2315,10 +2212,6 @@ void process_main(void) select_val = yb(Arg(0)); goto do_linear_search; - OpCase(i_select_val_lins_rfI): - select_val = r(0); - I--; - do_linear_search: { BeamInstr *vs = &Arg(3); int ix = 0; @@ -2345,10 +2238,6 @@ void process_main(void) select_val = yb(Arg(0)); goto do_binary_search; - OpCase(i_select_val_bins_rfI): - select_val = r(0); - I--; - do_binary_search: { struct Pairs { @@ -2409,10 +2298,6 @@ void process_main(void) jump_on_val_zero_index = xb(Arg(0)); goto do_jump_on_val_zero_index; - OpCase(i_jump_on_val_zero_rfI): - jump_on_val_zero_index = r(0); - I--; - do_jump_on_val_zero_index: if (is_small(jump_on_val_zero_index)) { jump_on_val_zero_index = signed_val(jump_on_val_zero_index); @@ -2437,10 +2322,6 @@ void process_main(void) jump_on_val_index = xb(Arg(0)); goto do_jump_on_val_index; - OpCase(i_jump_on_val_rfII): - jump_on_val_index = r(0); - I--; - do_jump_on_val_index: if (is_small(jump_on_val_index)) { jump_on_val_index = (Uint) (signed_val(jump_on_val_index) - Arg(3)); @@ -2460,15 +2341,12 @@ void process_main(void) do { Eterm term = *I++; - switch (term & _TAG_IMMED1_MASK) { - case (R_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER: - *hp++ = r(0); - break; - case (X_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER: - *hp++ = x(term >> _TAG_IMMED1_SIZE); + switch (loader_tag(term)) { + case LOADER_X_REG: + *hp++ = x(loader_x_reg_index(term)); break; - case (Y_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER: - *hp++ = y(term >> _TAG_IMMED1_SIZE); + case LOADER_Y_REG: + *hp++ = y(loader_y_reg_index(term)); break; default: *hp++ = term; @@ -2482,31 +2360,26 @@ void process_main(void) OpCase(new_map_dII): { Eterm res; - x(0) = r(0); SWAPOUT; res = new_map(c_p, reg, I-1); SWAPIN; - r(0) = x(0); StoreResult(res, Arg(0)); Next(3+Arg(2)); } -#define PUT_TERM_REG(term, desc) \ -do { \ - switch ((desc) & _TAG_IMMED1_MASK) { \ - case (R_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER: \ - r(0) = (term); \ - break; \ - case (X_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER: \ - x((desc) >> _TAG_IMMED1_SIZE) = (term); \ - break; \ - case (Y_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER: \ - y((desc) >> _TAG_IMMED1_SIZE) = (term); \ - break; \ - default: \ - ASSERT(0); \ - break; \ - } \ +#define PUT_TERM_REG(term, desc) \ +do { \ + switch (loader_tag(desc)) { \ + case LOADER_X_REG: \ + x(loader_x_reg_index(desc)) = (term); \ + break; \ + case LOADER_Y_REG: \ + y(loader_y_reg_index(desc)) = (term); \ + break; \ + default: \ + ASSERT(0); \ + break; \ + } \ } while(0) OpCase(i_get_map_elements_fsI): { @@ -2577,12 +2450,10 @@ do { \ Eterm map; GetArg1(1, map); - x(0) = r(0); SWAPOUT; res = update_map_assoc(c_p, reg, map, I); SWAPIN; if (is_value(res)) { - r(0) = x(0); StoreResult(res, Arg(2)); Next(5+Arg(4)); } else { @@ -2601,12 +2472,10 @@ do { \ Eterm map; GetArg1(1, map); - x(0) = r(0); SWAPOUT; res = update_map_exact(c_p, reg, map, I); SWAPIN; if (is_value(res)) { - r(0) = x(0); StoreResult(res, Arg(2)); Next(5+Arg(4)); } else { @@ -2688,13 +2557,10 @@ do { \ { typedef Eterm (*GcBifFunction)(Process*, Eterm*, Uint); GcBifFunction bf; - Eterm arg; Eterm result; Uint live = (Uint) Arg(3); - GetArg1(2, arg); - reg[0] = r(0); - reg[live] = arg; + GetArg1(2, x(live)); bf = (GcBifFunction) Arg(1); c_p->fcalls = FCALLS; SWAPOUT; @@ -2705,7 +2571,6 @@ do { \ ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); SWAPIN; - r(0) = reg[0]; ERTS_HOLE_CHECK(c_p); FCALLS = c_p->fcalls; if (is_value(result)) { @@ -2715,12 +2580,12 @@ do { \ SET_I((BeamInstr *) Arg(0)); Goto(*I); } - reg[0] = arg; + x(0) = x(live); I = handle_error(c_p, I, reg, translate_gc_bif((void *) bf)); goto post_error_handling; } - OpCase(i_gc_bif2_jIId): /* Note, one less parameter than the i_gc_bif1 + OpCase(i_gc_bif2_jIIssd): /* Note, one less parameter than the i_gc_bif1 and i_gc_bif3 */ { typedef Eterm (*GcBifFunction)(Process*, Eterm*, Uint); @@ -2728,9 +2593,13 @@ do { \ Eterm result; Uint live = (Uint) Arg(2); - reg[0] = r(0); - reg[live++] = tmp_arg1; - reg[live] = tmp_arg2; + GetArg2(3, x(live), x(live+1)); + /* + * XXX This calling convention does not make sense. 'live' + * should point out the first argument, not the second + * (i.e. 'live' should not be incremented below). + */ + live++; bf = (GcBifFunction) Arg(1); c_p->fcalls = FCALLS; SWAPOUT; @@ -2741,35 +2610,37 @@ do { \ ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); SWAPIN; - r(0) = reg[0]; ERTS_HOLE_CHECK(c_p); FCALLS = c_p->fcalls; if (is_value(result)) { - StoreBifResult(3, result); + StoreBifResult(5, result); } if (Arg(0) != 0) { SET_I((BeamInstr *) Arg(0)); Goto(*I); } - reg[0] = tmp_arg1; - reg[1] = tmp_arg2; + live--; + x(0) = x(live); + x(1) = x(live+1); I = handle_error(c_p, I, reg, translate_gc_bif((void *) bf)); goto post_error_handling; } - OpCase(i_gc_bif3_jIsId): + OpCase(i_gc_bif3_jIIssd): { typedef Eterm (*GcBifFunction)(Process*, Eterm*, Uint); GcBifFunction bf; - Eterm arg; Eterm result; - Uint live = (Uint) Arg(3); + Uint live = (Uint) Arg(2); - GetArg1(2, arg); - reg[0] = r(0); - reg[live++] = arg; - reg[live++] = tmp_arg1; - reg[live] = tmp_arg2; + x(live) = x(SCRATCH_X_REG); + GetArg2(3, x(live+1), x(live+2)); + /* + * XXX This calling convention does not make sense. 'live' + * should point out the first argument, not the third + * (i.e. 'live' should not be incremented below). + */ + live += 2; bf = (GcBifFunction) Arg(1); c_p->fcalls = FCALLS; SWAPOUT; @@ -2780,19 +2651,19 @@ do { \ ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); SWAPIN; - r(0) = reg[0]; ERTS_HOLE_CHECK(c_p); FCALLS = c_p->fcalls; if (is_value(result)) { - StoreBifResult(4, result); + StoreBifResult(5, result); } if (Arg(0) != 0) { SET_I((BeamInstr *) Arg(0)); Goto(*I); } - reg[0] = arg; - reg[1] = tmp_arg1; - reg[2] = tmp_arg2; + live -= 2; + x(0) = x(live); + x(1) = x(live+1); + x(2) = x(live+2); I = handle_error(c_p, I, reg, translate_gc_bif((void *) bf)); goto post_error_handling; } @@ -2800,12 +2671,13 @@ do { \ /* * Guards bifs and, or, xor in guards. */ - OpCase(i_bif2_fbd): + OpCase(i_bif2_fbssd): { - Eterm tmp_reg[2] = {tmp_arg1, tmp_arg2}; + Eterm tmp_reg[2]; Eterm (*bf)(Process*, Eterm*); Eterm result; + GetArg2(2, tmp_reg[0], tmp_reg[1]); bf = (BifFunction) Arg(1); c_p->fcalls = FCALLS; PROCESS_MAIN_CHK_LOCKS(c_p); @@ -2817,7 +2689,7 @@ do { \ ERTS_HOLE_CHECK(c_p); FCALLS = c_p->fcalls; if (is_value(result)) { - StoreBifResult(2, result); + StoreBifResult(4, result); } SET_I((BeamInstr *) Arg(0)); Goto(*I); @@ -2826,12 +2698,13 @@ do { \ /* * Guards bifs and, or, xor, relational operators in body. */ - OpCase(i_bif2_body_bd): + OpCase(i_bif2_body_bssd): { - Eterm tmp_reg[2] = {tmp_arg1, tmp_arg2}; + Eterm tmp_reg[2]; Eterm (*bf)(Process*, Eterm*); Eterm result; + GetArg2(1, tmp_reg[0], tmp_reg[1]); bf = (BifFunction) Arg(0); PROCESS_MAIN_CHK_LOCKS(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); @@ -2842,10 +2715,10 @@ do { \ ERTS_HOLE_CHECK(c_p); if (is_value(result)) { ASSERT(!is_CP(result)); - StoreBifResult(1, result); + StoreBifResult(3, result); } - reg[0] = tmp_arg1; - reg[1] = tmp_arg2; + reg[0] = tmp_reg[0]; + reg[1] = tmp_reg[1]; SWAPOUT; I = handle_error(c_p, I, reg, bf); goto post_error_handling; @@ -2869,7 +2742,6 @@ do { \ PreFetch(1, next); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - reg[0] = r(0); result = (*bf)(c_p, reg, I); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); @@ -2891,7 +2763,6 @@ do { \ SET_CP(c_p, I+2); SET_I(c_p->i); SWAPIN; - r(0) = reg[0]; Dispatch(); } @@ -2907,111 +2778,81 @@ do { \ * Arithmetic operations. */ - OpCase(i_times_jId): + OpCase(i_times_jIssd): { - arith_func = ARITH_FUNC(mixed_times); - goto do_big_arith2; + Eterm Op1, Op2; + GetArg2(2, Op1, Op2); + DO_OUTLINED_ARITH_2(mixed_times, Op1, Op2); } - OpCase(i_m_div_jId): + OpCase(i_m_div_jIssd): { - arith_func = ARITH_FUNC(mixed_div); - goto do_big_arith2; + Eterm Op1, Op2; + GetArg2(2, Op1, Op2); + DO_OUTLINED_ARITH_2(mixed_div, Op1, Op2); } - OpCase(i_int_div_jId): + OpCase(i_int_div_jIssd): { - Eterm result; + Eterm Op1, Op2; - if (tmp_arg2 == SMALL_ZERO) { + GetArg2(2, Op1, Op2); + if (Op2 == SMALL_ZERO) { goto badarith; - } else if (is_both_small(tmp_arg1, tmp_arg2)) { - Sint ires = signed_val(tmp_arg1) / signed_val(tmp_arg2); + } else if (is_both_small(Op1, Op2)) { + Sint ires = signed_val(Op1) / signed_val(Op2); if (MY_IS_SSMALL(ires)) { - result = make_small(ires); - STORE_ARITH_RESULT(result); + Eterm result = make_small(ires); + StoreBifResult(4, result); } } - arith_func = ARITH_FUNC(int_div); - goto do_big_arith2; + DO_OUTLINED_ARITH_2(int_div, Op1, Op2); } - OpCase(i_rem_jIxxd): { - Eterm result; + Eterm RemOp1, RemOp2; - if (xb(Arg(3)) == SMALL_ZERO) { - goto badarith; - } else if (is_both_small(xb(Arg(2)), xb(Arg(3)))) { - result = make_small(signed_val(xb(Arg(2))) % signed_val(xb(Arg(3)))); + OpCase(i_rem_jIxxd): + RemOp1 = xb(Arg(2)); + RemOp2 = xb(Arg(3)); + goto do_rem; + + OpCase(i_rem_jIssd): + GetArg2(2, RemOp1, RemOp2); + goto do_rem; + + do_rem: + if (RemOp2 == SMALL_ZERO) { + goto badarith; + } else if (is_both_small(RemOp1, RemOp2)) { + Eterm result = make_small(signed_val(RemOp1) % signed_val(RemOp2)); StoreBifResult(4, result); + } else { + DO_OUTLINED_ARITH_2(int_rem, RemOp1, RemOp2); } - DO_BIG_ARITH(ARITH_FUNC(int_rem),xb(Arg(2)),xb(Arg(3))); } - OpCase(i_rem_jId): { - Eterm result; - - if (tmp_arg2 == SMALL_ZERO) { - goto badarith; - } else if (is_both_small(tmp_arg1, tmp_arg2)) { - result = make_small(signed_val(tmp_arg1) % signed_val(tmp_arg2)); - STORE_ARITH_RESULT(result); - } else { - arith_func = ARITH_FUNC(int_rem); - goto do_big_arith2; - } - } + Eterm BandOp1, BandOp2; OpCase(i_band_jIxcd): - { - Eterm result; + BandOp1 = xb(Arg(2)); + BandOp2 = Arg(3); + goto do_band; + + OpCase(i_band_jIssd): + GetArg2(2, BandOp1, BandOp2); + goto do_band; - if (is_both_small(xb(Arg(2)), Arg(3))) { + do_band: + if (is_both_small(BandOp1, BandOp2)) { /* * No need to untag -- TAG & TAG == TAG. */ - result = xb(Arg(2)) & Arg(3); + Eterm result = BandOp1 & BandOp2; StoreBifResult(4, result); } - DO_BIG_ARITH(ARITH_FUNC(band),xb(Arg(2)),Arg(3)); - } - - OpCase(i_band_jId): - { - Eterm result; - - if (is_both_small(tmp_arg1, tmp_arg2)) { - /* - * No need to untag -- TAG & TAG == TAG. - */ - result = tmp_arg1 & tmp_arg2; - STORE_ARITH_RESULT(result); - } - arith_func = ARITH_FUNC(band); - goto do_big_arith2; - } - -#undef DO_BIG_ARITH - - do_big_arith2: - { - Eterm result; - Uint live = Arg(1); - - SWAPOUT; - reg[0] = r(0); - reg[live] = tmp_arg1; - reg[live+1] = tmp_arg2; - result = arith_func(c_p, reg, live); - r(0) = reg[0]; - SWAPIN; - ERTS_HOLE_CHECK(c_p); - if (is_value(result)) { - STORE_ARITH_RESULT(result); - } - goto lb_Cl_error; + DO_OUTLINED_ARITH_2(band, BandOp1, BandOp2); } /* @@ -3032,97 +2873,102 @@ do { \ goto find_func_info; } - OpCase(i_bor_jId): + OpCase(i_bor_jIssd): { - Eterm result; + Eterm Op1, Op2; - if (is_both_small(tmp_arg1, tmp_arg2)) { + GetArg2(2, Op1, Op2); + if (is_both_small(Op1, Op2)) { /* * No need to untag -- TAG | TAG == TAG. */ - result = tmp_arg1 | tmp_arg2; - STORE_ARITH_RESULT(result); + Eterm result = Op1 | Op2; + StoreBifResult(4, result); } - arith_func = ARITH_FUNC(bor); - goto do_big_arith2; + DO_OUTLINED_ARITH_2(bor, Op1, Op2); } - OpCase(i_bxor_jId): + OpCase(i_bxor_jIssd): { - Eterm result; + Eterm Op1, Op2; - if (is_both_small(tmp_arg1, tmp_arg2)) { + GetArg2(2, Op1, Op2); + if (is_both_small(Op1, Op2)) { /* * We could extract the tag from one argument, but a tag extraction * could mean a shift. Therefore, play it safe here. */ - result = make_small(signed_val(tmp_arg1) ^ signed_val(tmp_arg2)); - STORE_ARITH_RESULT(result); + Eterm result = make_small(signed_val(Op1) ^ signed_val(Op2)); + StoreBifResult(4, result); } - arith_func = ARITH_FUNC(bxor); - goto do_big_arith2; + DO_OUTLINED_ARITH_2(bxor, Op1, Op2); } { + Eterm Op1, Op2; Sint i; Sint ires; Eterm* bigp; + Eterm tmp_big[2]; - OpCase(i_bsr_jId): - if (is_small(tmp_arg2)) { - i = -signed_val(tmp_arg2); - if (is_small(tmp_arg1)) { + OpCase(i_bsr_jIssd): + GetArg2(2, Op1, Op2); + if (is_small(Op2)) { + i = -signed_val(Op2); + if (is_small(Op1)) { goto small_shift; - } else if (is_big(tmp_arg1)) { + } else if (is_big(Op1)) { if (i == 0) { - StoreBifResult(2, tmp_arg1); + StoreBifResult(4, Op1); } goto big_shift; } - } else if (is_big(tmp_arg2)) { + } else if (is_big(Op2)) { /* * N bsr NegativeBigNum == N bsl MAX_SMALL * N bsr PositiveBigNum == N bsl MIN_SMALL */ - tmp_arg2 = make_small(bignum_header_is_neg(*big_val(tmp_arg2)) ? + Op2 = make_small(bignum_header_is_neg(*big_val(Op2)) ? MAX_SMALL : MIN_SMALL); goto do_bsl; } goto badarith; - OpCase(i_bsl_jId): + OpCase(i_bsl_jIssd): + GetArg2(2, Op1, Op2); + do_bsl: - if (is_small(tmp_arg2)) { - i = signed_val(tmp_arg2); + if (is_small(Op2)) { + i = signed_val(Op2); - if (is_small(tmp_arg1)) { + if (is_small(Op1)) { small_shift: - ires = signed_val(tmp_arg1); + ires = signed_val(Op1); if (i == 0 || ires == 0) { - StoreBifResult(2, tmp_arg1); + StoreBifResult(4, Op1); } else if (i < 0) { /* Right shift */ i = -i; if (i >= SMALL_BITS-1) { - tmp_arg1 = (ires < 0) ? SMALL_MINUS_ONE : SMALL_ZERO; + Op1 = (ires < 0) ? SMALL_MINUS_ONE : SMALL_ZERO; } else { - tmp_arg1 = make_small(ires >> i); + Op1 = make_small(ires >> i); } - StoreBifResult(2, tmp_arg1); + StoreBifResult(4, Op1); } else if (i < SMALL_BITS-1) { /* Left shift */ if ((ires > 0 && ((~(Uint)0 << ((SMALL_BITS-1)-i)) & ires) == 0) || ((~(Uint)0 << ((SMALL_BITS-1)-i)) & ~ires) == 0) { - tmp_arg1 = make_small(ires << i); - StoreBifResult(2, tmp_arg1); + Op1 = make_small(ires << i); + StoreBifResult(4, Op1); } } - tmp_arg1 = small_to_big(ires, tmp_big); + Op1 = small_to_big(ires, tmp_big); big_shift: if (i > 0) { /* Left shift. */ - ires = big_size(tmp_arg1) + (i / D_EXP); + ires = big_size(Op1) + (i / D_EXP); } else { /* Right shift. */ - ires = big_size(tmp_arg1); + ires = big_size(Op1); if (ires <= (-i / D_EXP)) ires = 3; /* ??? */ else @@ -3140,14 +2986,14 @@ do { \ c_p->freason = SYSTEM_LIMIT; goto lb_Cl_error; } - TestHeapPreserve(ires+1, Arg(1), tmp_arg1); + TestHeapPreserve(ires+1, Arg(1), Op1); bigp = HTOP; - tmp_arg1 = big_lshift(tmp_arg1, i, bigp); - if (is_big(tmp_arg1)) { + Op1 = big_lshift(Op1, i, bigp); + if (is_big(Op1)) { HTOP += bignum_header_arity(*HTOP) + 1; } HEAP_SPACE_VERIFIED(0); - if (is_nil(tmp_arg1)) { + if (is_nil(Op1)) { /* * This result must have been only slight larger * than allowed since it wasn't caught by the @@ -3157,25 +3003,25 @@ do { \ goto lb_Cl_error; } ERTS_HOLE_CHECK(c_p); - StoreBifResult(2, tmp_arg1); + StoreBifResult(4, Op1); } - } else if (is_big(tmp_arg1)) { + } else if (is_big(Op1)) { if (i == 0) { - StoreBifResult(2, tmp_arg1); + StoreBifResult(4, Op1); } goto big_shift; } - } else if (is_big(tmp_arg2)) { - if (bignum_header_is_neg(*big_val(tmp_arg2))) { + } else if (is_big(Op2)) { + if (bignum_header_is_neg(*big_val(Op2))) { /* * N bsl NegativeBigNum is either 0 or -1, depending on * the sign of N. Since we don't believe this case * is common, do the calculation with the minimum * amount of code. */ - tmp_arg2 = make_small(MIN_SMALL); + Op2 = make_small(MIN_SMALL); goto do_bsl; - } else if (is_small(tmp_arg1) || is_big(tmp_arg1)) { + } else if (is_small(Op1) || is_big(Op1)) { /* * N bsl PositiveBigNum is too large to represent. */ @@ -3200,10 +3046,8 @@ do { \ } else { Uint live = Arg(2); SWAPOUT; - reg[0] = r(0); reg[live] = bnot_val; bnot_val = erts_gc_bnot(c_p, reg, live); - r(0) = reg[0]; SWAPIN; ERTS_HOLE_CHECK(c_p); if (is_nil(bnot_val)) { @@ -3223,7 +3067,6 @@ do { \ next = apply(c_p, r(0), x(1), x(2), reg); SWAPIN; if (next != NULL) { - r(0) = reg[0]; SET_CP(c_p, I+1); SET_I(next); Dispatch(); @@ -3238,8 +3081,7 @@ do { \ next = apply(c_p, r(0), x(1), x(2), reg); SWAPIN; if (next != NULL) { - r(0) = reg[0]; - SET_CP(c_p, (BeamInstr *) EXPAND_POINTER(E[0])); + SET_CP(c_p, (BeamInstr *) E[0]); E = ADD_BYTE_OFFSET(E, Arg(0)); SET_I(next); Dispatch(); @@ -3254,7 +3096,6 @@ do { \ next = apply(c_p, r(0), x(1), x(2), reg); SWAPIN; if (next != NULL) { - r(0) = reg[0]; SET_I(next); Dispatch(); } @@ -3265,12 +3106,10 @@ do { \ OpCase(apply_I): { BeamInstr *next; - reg[0] = r(0); SWAPOUT; next = fixed_apply(c_p, reg, Arg(0)); SWAPIN; if (next != NULL) { - r(0) = reg[0]; SET_CP(c_p, I+2); SET_I(next); Dispatch(); @@ -3282,13 +3121,11 @@ do { \ OpCase(apply_last_IP): { BeamInstr *next; - reg[0] = r(0); SWAPOUT; next = fixed_apply(c_p, reg, Arg(0)); SWAPIN; if (next != NULL) { - r(0) = reg[0]; - SET_CP(c_p, (BeamInstr *) EXPAND_POINTER(E[0])); + SET_CP(c_p, (BeamInstr *) E[0]); E = ADD_BYTE_OFFSET(E, Arg(1)); SET_I(next); Dispatch(); @@ -3304,7 +3141,6 @@ do { \ next = apply_fun(c_p, r(0), x(1), reg); SWAPIN; if (next != NULL) { - r(0) = reg[0]; SET_CP(c_p, I+1); SET_I(next); Dispatchfun(); @@ -3319,8 +3155,7 @@ do { \ next = apply_fun(c_p, r(0), x(1), reg); SWAPIN; if (next != NULL) { - r(0) = reg[0]; - SET_CP(c_p, (BeamInstr *) EXPAND_POINTER(E[0])); + SET_CP(c_p, (BeamInstr *) E[0]); E = ADD_BYTE_OFFSET(E, Arg(0)); SET_I(next); Dispatchfun(); @@ -3335,7 +3170,6 @@ do { \ next = apply_fun(c_p, r(0), x(1), reg); SWAPIN; if (next != NULL) { - r(0) = reg[0]; SET_I(next); Dispatchfun(); } @@ -3346,12 +3180,9 @@ do { \ BeamInstr *next; SWAPOUT; - reg[0] = r(0); - next = call_fun(c_p, Arg(0), reg, THE_NON_VALUE); SWAPIN; if (next != NULL) { - r(0) = reg[0]; SET_CP(c_p, I+2); SET_I(next); Dispatchfun(); @@ -3363,12 +3194,10 @@ do { \ BeamInstr *next; SWAPOUT; - reg[0] = r(0); next = call_fun(c_p, Arg(0), reg, THE_NON_VALUE); SWAPIN; if (next != NULL) { - r(0) = reg[0]; - SET_CP(c_p, (BeamInstr *) EXPAND_POINTER(E[0])); + SET_CP(c_p, (BeamInstr *) E[0]); E = ADD_BYTE_OFFSET(E, Arg(1)); SET_I(next); Dispatchfun(); @@ -3452,10 +3281,9 @@ do { \ */ argp = c_p->arg_reg; - for (i = c_p->arity - 1; i > 0; i--) { + for (i = c_p->arity - 1; i >= 0; i--) { argp[i] = reg[i]; } - c_p->arg_reg[0] = r(0); SWAPOUT; c_p->i = I; goto do_schedule1; @@ -3468,19 +3296,14 @@ do { \ Eterm* p; PreFetch(3, next); - GetArg2(0, element, tuple); + GetArg1(0, element); + tuple = REG_TARGET(Arg(1)); ASSERT(is_tuple(tuple)); p = (Eterm *) ((unsigned char *) tuple_val(tuple) + Arg(2)); *p = element; NextPF(3, next); } - OpCase(i_is_ne_exact_f): - if (EQ(tmp_arg1, tmp_arg2)) { - ClauseFail(); - } - Next(1); - OpCase(normal_exit): { SWAPOUT; c_p->freason = EXC_NORMAL; @@ -3547,25 +3370,14 @@ do { \ { Eterm badmatch_val; - OpCase(badmatch_y): - badmatch_val = yb(Arg(0)); - goto do_badmatch; - OpCase(badmatch_x): badmatch_val = xb(Arg(0)); - goto do_badmatch; - - OpCase(badmatch_r): - badmatch_val = r(0); - - do_badmatch: c_p->fvalue = badmatch_val; c_p->freason = BADMATCH; } /* Fall through here */ find_func_info: { - reg[0] = r(0); SWAPOUT; I = handle_error(c_p, I, reg, NULL); goto post_error_handling; @@ -3583,9 +3395,7 @@ do { \ * code[4]: Not used */ SWAPOUT; - reg[0] = r(0); I = call_error_handler(c_p, I-3, reg, am_undefined_function); - r(0) = reg[0]; SWAPIN; if (I) { Goto(*I); @@ -3594,14 +3404,12 @@ do { \ /* Fall through */ OpCase(error_action_code): { handle_error: - reg[0] = r(0); SWAPOUT; I = handle_error(c_p, NULL, reg, NULL); post_error_handling: if (I == 0) { goto do_schedule; } else { - r(0) = reg[0]; ASSERT(!is_value(r(0))); if (c_p->mbuf) { erts_garbage_collect(c_p, 0, reg+1, 3); @@ -3644,7 +3452,6 @@ do { \ NifF* fp = vbf = (NifF*) I[1]; struct enif_environment_t env; erts_pre_nif(&env, c_p, (struct erl_module_nif*)I[2]); - reg[0] = r(0); nif_bif_result = (*fp)(&env, bif_nif_arity, reg); if (env.exception_thrown) nif_bif_result = THE_NON_VALUE; @@ -3683,7 +3490,7 @@ do { \ bif_nif_arity = I[-1]; ASSERT(bif_nif_arity <= 4); ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); - reg[0] = r(0); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); { Eterm (*bf)(Process*, Eterm*, BeamInstr*) = vbf; ASSERT(!ERTS_PROC_IS_EXITING(c_p)); @@ -3713,7 +3520,6 @@ do { \ Goto(*I); } else if (c_p->freason == TRAP) { SET_I(c_p->i); - r(0) = reg[0]; if (c_p->flags & F_HIBERNATE_SCHED) { c_p->flags &= ~F_HIBERNATE_SCHED; goto do_schedule; @@ -3740,16 +3546,6 @@ do { \ OpCase(case_end_x): case_end_val = xb(Arg(0)); - goto do_case_end; - - OpCase(case_end_y): - case_end_val = yb(Arg(0)); - goto do_case_end; - - OpCase(case_end_r): - case_end_val = r(0); - - do_case_end: c_p->fvalue = case_end_val; c_p->freason = EXC_CASE_CLAUSE; goto find_func_info; @@ -3797,19 +3593,13 @@ do { \ goto do_bs_init_bits_known; } - OpCase(i_bs_init_bits_fail_heap_IjId): { - /* tmp_arg1 was fetched by an i_fetch instruction */ - num_bits_term = tmp_arg1; - alloc = Arg(0); - I++; + OpCase(i_bs_init_bits_fail_heap_sIjId): { + GetArg1(0, num_bits_term); + alloc = Arg(1); + I += 2; goto do_bs_init_bits; } - OpCase(i_bs_init_bits_fail_rjId): { - num_bits_term = r(0); - alloc = 0; - goto do_bs_init_bits; - } OpCase(i_bs_init_bits_fail_yjId): { num_bits_term = yb(Arg(0)); I++; @@ -3929,52 +3719,48 @@ do { \ } { - OpCase(i_bs_init_fail_heap_IjId): { - /* tmp_arg1 was fetched by an i_fetch instruction */ - tmp_arg2 = Arg(0); - I++; - goto do_bs_init; - } + Eterm BsOp1, BsOp2; - OpCase(i_bs_init_fail_rjId): { - tmp_arg1 = r(0); - tmp_arg2 = 0; + OpCase(i_bs_init_fail_heap_sIjId): { + GetArg1(0, BsOp1); + BsOp2 = Arg(1); + I += 2; goto do_bs_init; } OpCase(i_bs_init_fail_yjId): { - tmp_arg1 = yb(Arg(0)); - tmp_arg2 = 0; + BsOp1 = yb(Arg(0)); + BsOp2 = 0; I++; goto do_bs_init; } OpCase(i_bs_init_fail_xjId): { - tmp_arg1 = xb(Arg(0)); - tmp_arg2 = 0; + BsOp1 = xb(Arg(0)); + BsOp2 = 0; I++; } /* FALL THROUGH */ do_bs_init: - if (is_small(tmp_arg1)) { - Sint size = signed_val(tmp_arg1); + if (is_small(BsOp1)) { + Sint size = signed_val(BsOp1); if (size < 0) { goto badarg; } - tmp_arg1 = (Eterm) size; + BsOp1 = (Eterm) size; } else { Uint bytes; - if (!term_to_Uint(tmp_arg1, &bytes)) { + if (!term_to_Uint(BsOp1, &bytes)) { c_p->freason = bytes; goto lb_Cl_error; } if ((bytes >> (8*sizeof(Uint)-3)) != 0) { goto system_limit; } - tmp_arg1 = (Eterm) bytes; + BsOp1 = (Eterm) bytes; } - if (tmp_arg1 <= ERL_ONHEAP_BIN_LIMIT) { + if (BsOp1 <= ERL_ONHEAP_BIN_LIMIT) { goto do_heap_bin_alloc; } else { goto do_proc_bin_alloc; @@ -3982,15 +3768,15 @@ do { \ OpCase(i_bs_init_heap_IIId): { - tmp_arg1 = Arg(0); - tmp_arg2 = Arg(1); + BsOp1 = Arg(0); + BsOp2 = Arg(1); I++; goto do_proc_bin_alloc; } OpCase(i_bs_init_IId): { - tmp_arg1 = Arg(0); - tmp_arg2 = 0; + BsOp1 = Arg(0); + BsOp2 = 0; } /* FALL THROUGH */ do_proc_bin_alloc: { @@ -3999,13 +3785,13 @@ do { \ erts_bin_offset = 0; erts_writable_bin = 0; - TestBinVHeap(tmp_arg1 / sizeof(Eterm), - tmp_arg2 + PROC_BIN_SIZE + ERL_SUB_BIN_SIZE, Arg(1)); + TestBinVHeap(BsOp1 / sizeof(Eterm), + BsOp2 + PROC_BIN_SIZE + ERL_SUB_BIN_SIZE, Arg(1)); /* * Allocate the binary struct itself. */ - bptr = erts_bin_nrml_alloc(tmp_arg1); + bptr = erts_bin_nrml_alloc(BsOp1); erts_refc_init(&bptr->refc, 1); erts_current_bin = (byte *) bptr->orig_bytes; @@ -4015,28 +3801,28 @@ do { \ pb = (ProcBin *) HTOP; HTOP += PROC_BIN_SIZE; pb->thing_word = HEADER_PROC_BIN; - pb->size = tmp_arg1; + pb->size = BsOp1; pb->next = MSO(c_p).first; MSO(c_p).first = (struct erl_off_heap_header*) pb; pb->val = bptr; pb->bytes = (byte*) bptr->orig_bytes; pb->flags = 0; - OH_OVERHEAD(&(MSO(c_p)), tmp_arg1 / sizeof(Eterm)); + OH_OVERHEAD(&(MSO(c_p)), BsOp1 / sizeof(Eterm)); StoreBifResult(2, make_binary(pb)); } OpCase(i_bs_init_heap_bin_heap_IIId): { - tmp_arg1 = Arg(0); - tmp_arg2 = Arg(1); + BsOp1 = Arg(0); + BsOp2 = Arg(1); I++; goto do_heap_bin_alloc; } OpCase(i_bs_init_heap_bin_IId): { - tmp_arg1 = Arg(0); - tmp_arg2 = 0; + BsOp1 = Arg(0); + BsOp2 = 0; } /* Fall through */ do_heap_bin_alloc: @@ -4044,33 +3830,36 @@ do { \ ErlHeapBin* hb; Uint bin_need; - bin_need = heap_bin_size(tmp_arg1); + bin_need = heap_bin_size(BsOp1); erts_bin_offset = 0; erts_writable_bin = 0; - TestHeap(bin_need+tmp_arg2+ERL_SUB_BIN_SIZE, Arg(1)); + TestHeap(bin_need+BsOp2+ERL_SUB_BIN_SIZE, Arg(1)); hb = (ErlHeapBin *) HTOP; HTOP += bin_need; - hb->thing_word = header_heap_bin(tmp_arg1); - hb->size = tmp_arg1; + hb->thing_word = header_heap_bin(BsOp1); + hb->size = BsOp1; erts_current_bin = (byte *) hb->data; - tmp_arg1 = make_binary(hb); - StoreBifResult(2, tmp_arg1); + BsOp1 = make_binary(hb); + StoreBifResult(2, BsOp1); } } - OpCase(i_bs_add_jId): { - Uint Unit = Arg(1); - if (is_both_small(tmp_arg1, tmp_arg2)) { - Sint Arg1 = signed_val(tmp_arg1); - Sint Arg2 = signed_val(tmp_arg2); + OpCase(bs_add_jssId): { + Eterm Op1, Op2; + Uint Unit = Arg(3); + + GetArg2(1, Op1, Op2); + if (is_both_small(Op1, Op2)) { + Sint Arg1 = signed_val(Op1); + Sint Arg2 = signed_val(Op2); if (Arg1 >= 0 && Arg2 >= 0) { - BsSafeMul(Arg2, Unit, goto system_limit, tmp_arg1); - tmp_arg1 += Arg1; + BsSafeMul(Arg2, Unit, goto system_limit, Op1); + Op1 += Arg1; store_bs_add_result: - if (MY_IS_SSMALL((Sint) tmp_arg1)) { - tmp_arg1 = make_small(tmp_arg1); + if (MY_IS_SSMALL((Sint) Op1)) { + Op1 = make_small(Op1); } else { /* * May generate a heap fragment, but in this @@ -4082,10 +3871,10 @@ do { \ * references (such as the heap). */ SWAPOUT; - tmp_arg1 = erts_make_integer(tmp_arg1, c_p); + Op1 = erts_make_integer(Op1, c_p); HTOP = HEAP_TOP(c_p); } - StoreBifResult(2, tmp_arg1); + StoreBifResult(4, Op1); } goto badarg; } else { @@ -4108,16 +3897,16 @@ do { \ * an Uint, the reason is SYSTEM_LIMIT. */ - if (!term_to_Uint(tmp_arg1, &a)) { + if (!term_to_Uint(Op1, &a)) { if (a == BADARG) { goto badarg; } - if (!term_to_Uint(tmp_arg2, &b)) { + if (!term_to_Uint(Op2, &b)) { c_p->freason = b; goto lb_Cl_error; } goto system_limit; - } else if (!term_to_Uint(tmp_arg2, &b)) { + } else if (!term_to_Uint(Op2, &b)) { c_p->freason = b; goto lb_Cl_error; } @@ -4127,8 +3916,8 @@ do { \ */ BsSafeMul(b, Unit, goto system_limit, c); - tmp_arg1 = a + c; - if (tmp_arg1 < a) { + Op1 = a + c; + if (Op1 < a) { /* * If the result is less than one of the * arguments, there must have been an overflow. @@ -4150,48 +3939,43 @@ do { \ } /* - * tmp_arg1 = Number of bytes to build - * tmp_arg2 = Source binary - * Operands: Fail ExtraHeap Live Unit Dst + * x(SCRATCH_X_REG); + * Operands: Fail ExtraHeap Live Unit Size Dst */ - OpCase(i_bs_append_jIIId): { + OpCase(i_bs_append_jIIIsd): { Uint live = Arg(2); Uint res; + Eterm Size; + GetArg1(4, Size); SWAPOUT; - reg[0] = r(0); - reg[live] = tmp_arg2; - res = erts_bs_append(c_p, reg, live, tmp_arg1, Arg(1), Arg(3)); - r(0) = reg[0]; + reg[live] = x(SCRATCH_X_REG); + res = erts_bs_append(c_p, reg, live, Size, Arg(1), Arg(3)); SWAPIN; if (is_non_value(res)) { /* c_p->freason is already set (may be either BADARG or SYSTEM_LIMIT). */ goto lb_Cl_error; } - StoreBifResult(4, res); + StoreBifResult(5, res); } /* - * tmp_arg1 = Number of bytes to build - * tmp_arg2 = Source binary - * Operands: Fail Unit Dst + * Operands: Fail Size Src Unit Dst */ - OpCase(i_bs_private_append_jId): { + OpCase(i_bs_private_append_jIssd): { Eterm res; + Eterm Size, Src; - res = erts_bs_private_append(c_p, tmp_arg2, tmp_arg1, Arg(1)); + GetArg2(2, Size, Src); + res = erts_bs_private_append(c_p, Src, Size, Arg(1)); if (is_non_value(res)) { /* c_p->freason is already set (may be either BADARG or SYSTEM_LIMIT). */ goto lb_Cl_error; } - StoreBifResult(2, res); + StoreBifResult(4, res); } - /* - * tmp_arg1 = Initial size of writable binary - * Operands: Live Dst - */ OpCase(bs_init_writable): { SWAPOUT; r(0) = erts_bs_init_writable(c_p, r(0)); @@ -4286,26 +4070,29 @@ do { \ /* * Only used for validating a value matched out. - * - * tmp_arg1 = Integer to validate - * tmp_arg2 = Match context */ - OpCase(i_bs_validate_unicode_retract_j): { - /* - * There is no need to untag the integer, but it IS necessary - * to make sure it is small (a bignum pointer could fall in - * the valid range). - */ - if (is_not_small(tmp_arg1) || tmp_arg1 > make_small(0x10FFFFUL) || - (make_small(0xD800UL) <= tmp_arg1 && - tmp_arg1 <= make_small(0xDFFFUL))) { - ErlBinMatchBuffer *mb = ms_matchbuffer(tmp_arg2); + OpCase(i_bs_validate_unicode_retract_jss): { + Eterm i; /* Integer to validate */ - mb->offset -= 32; - goto badarg; - } - Next(1); - } + /* + * There is no need to untag the integer, but it IS necessary + * to make sure it is small (a bignum pointer could fall in + * the valid range). + */ + + GetArg1(1, i); + if (is_not_small(i) || i > make_small(0x10FFFFUL) || + (make_small(0xD800UL) <= i && i <= make_small(0xDFFFUL))) { + Eterm ms; /* Match context */ + ErlBinMatchBuffer* mb; + + GetArg1(2, ms); + mb = ms_matchbuffer(ms); + mb->offset -= 32; + goto badarg; + } + Next(3); + } /* * Matching of binaries. @@ -4317,9 +4104,6 @@ do { \ Uint slots; Eterm context; - OpCase(i_bs_start_match2_rfIId): { - context = r(0); - do_start_match: slots = Arg(2); if (!is_boxed(context)) { @@ -4366,7 +4150,7 @@ do { \ ClauseFail(); } NextPF(4, next); - } + OpCase(i_bs_start_match2_xfIId): { context = xb(Arg(0)); I++; @@ -4379,18 +4163,6 @@ do { \ } } - OpCase(bs_test_zero_tail2_fr): { - BeamInstr *next; - ErlBinMatchBuffer *_mb; - - PreFetch(1, next); - _mb = (ErlBinMatchBuffer*) ms_matchbuffer(r(0)); - if (_mb->size != _mb->offset) { - ClauseFail(); - } - NextPF(1, next); - } - OpCase(bs_test_zero_tail2_fx): { BeamInstr *next; ErlBinMatchBuffer *_mb; @@ -4403,16 +4175,6 @@ do { \ NextPF(2, next); } - OpCase(bs_test_tail_imm2_frI): { - BeamInstr *next; - ErlBinMatchBuffer *_mb; - PreFetch(2, next); - _mb = ms_matchbuffer(r(0)); - if (_mb->size - _mb->offset != Arg(1)) { - ClauseFail(); - } - NextPF(2, next); - } OpCase(bs_test_tail_imm2_fxI): { BeamInstr *next; ErlBinMatchBuffer *_mb; @@ -4424,16 +4186,6 @@ do { \ NextPF(3, next); } - OpCase(bs_test_unit_frI): { - BeamInstr *next; - ErlBinMatchBuffer *_mb; - PreFetch(2, next); - _mb = ms_matchbuffer(r(0)); - if ((_mb->size - _mb->offset) % Arg(1)) { - ClauseFail(); - } - NextPF(2, next); - } OpCase(bs_test_unit_fxI): { BeamInstr *next; ErlBinMatchBuffer *_mb; @@ -4445,16 +4197,6 @@ do { \ NextPF(3, next); } - OpCase(bs_test_unit8_fr): { - BeamInstr *next; - ErlBinMatchBuffer *_mb; - PreFetch(1, next); - _mb = ms_matchbuffer(r(0)); - if ((_mb->size - _mb->offset) & 7) { - ClauseFail(); - } - NextPF(1, next); - } OpCase(bs_test_unit8_fx): { BeamInstr *next; ErlBinMatchBuffer *_mb; @@ -4469,19 +4211,11 @@ do { \ { Eterm bs_get_integer8_context; - OpCase(i_bs_get_integer_8_rfd): { - bs_get_integer8_context = r(0); - goto do_bs_get_integer_8; - } - OpCase(i_bs_get_integer_8_xfd): { - bs_get_integer8_context = xb(Arg(0)); - I++; - } - - do_bs_get_integer_8: { ErlBinMatchBuffer *_mb; Eterm _result; + bs_get_integer8_context = xb(Arg(0)); + I++; _mb = ms_matchbuffer(bs_get_integer8_context); if (_mb->size - _mb->offset < 8) { ClauseFail(); @@ -4499,15 +4233,10 @@ do { \ { Eterm bs_get_integer_16_context; - OpCase(i_bs_get_integer_16_rfd): - bs_get_integer_16_context = r(0); - goto do_bs_get_integer_16; - OpCase(i_bs_get_integer_16_xfd): bs_get_integer_16_context = xb(Arg(0)); I++; - do_bs_get_integer_16: { ErlBinMatchBuffer *_mb; Eterm _result; @@ -4528,17 +4257,10 @@ do { \ { Eterm bs_get_integer_32_context; - OpCase(i_bs_get_integer_32_rfId): - bs_get_integer_32_context = r(0); - goto do_bs_get_integer_32; - - OpCase(i_bs_get_integer_32_xfId): bs_get_integer_32_context = xb(Arg(0)); I++; - - do_bs_get_integer_32: { ErlBinMatchBuffer *_mb; Uint32 _integer; @@ -4551,11 +4273,11 @@ do { \ _integer = get_int32(_mb->base + _mb->offset/8); } _mb->offset += 32; -#if !defined(ARCH_64) || HALFWORD_HEAP +#if !defined(ARCH_64) if (IS_USMALL(0, _integer)) { #endif _result = make_small(_integer); -#if !defined(ARCH_64) || HALFWORD_HEAP +#if !defined(ARCH_64) } else { TestHeap(BIG_UINT_HEAP_SIZE, Arg(1)); _result = uint_to_big((Uint) _integer, HTOP); @@ -4567,103 +4289,82 @@ do { \ } } - /* Operands: Size Live Fail Flags Dst */ - OpCase(i_bs_get_integer_imm_rIIfId): { - tmp_arg1 = r(0); - /* Operands: Size Live Fail Flags Dst */ - goto do_bs_get_integer_imm_test_heap; - } + { + Eterm Ms, Sz; - /* Operands: x(Reg) Size Live Fail Flags Dst */ + /* Operands: x(Reg) Size Live Fail Flags Dst */ OpCase(i_bs_get_integer_imm_xIIfId): { - tmp_arg1 = xb(Arg(0)); - I++; - /* Operands: Size Live Fail Flags Dst */ - goto do_bs_get_integer_imm_test_heap; - } - - /* - * tmp_arg1 = match context - * Operands: Size Live Fail Flags Dst - */ - do_bs_get_integer_imm_test_heap: { - Uint wordsneeded; - tmp_arg2 = Arg(0); - wordsneeded = 1+WSIZE(NBYTES(tmp_arg2)); - TestHeapPreserve(wordsneeded, Arg(1), tmp_arg1); - I += 2; - /* Operands: Fail Flags Dst */ - goto do_bs_get_integer_imm; - } - - /* Operands: Size Fail Flags Dst */ - OpCase(i_bs_get_integer_small_imm_rIfId): { - tmp_arg1 = r(0); - tmp_arg2 = Arg(0); - I++; - /* Operands: Fail Flags Dst */ - goto do_bs_get_integer_imm; - } + Uint wordsneeded; + Ms = xb(Arg(0)); + Sz = Arg(1); + wordsneeded = 1+WSIZE(NBYTES(Sz)); + TestHeapPreserve(wordsneeded, Arg(2), Ms); + I += 3; + /* Operands: Fail Flags Dst */ + goto do_bs_get_integer_imm; + } - /* Operands: x(Reg) Size Fail Flags Dst */ + /* Operands: x(Reg) Size Fail Flags Dst */ OpCase(i_bs_get_integer_small_imm_xIfId): { - tmp_arg1 = xb(Arg(0)); - tmp_arg2 = Arg(1); - I += 2; - /* Operands: Fail Flags Dst */ - goto do_bs_get_integer_imm; - } + Ms = xb(Arg(0)); + Sz = Arg(1); + I += 2; + /* Operands: Fail Flags Dst */ + goto do_bs_get_integer_imm; + } - /* - * tmp_arg1 = match context - * tmp_arg2 = size of field - * Operands: Fail Flags Dst - */ + /* + * Ms = match context + * Sz = size of field + * Operands: Fail Flags Dst + */ do_bs_get_integer_imm: { - ErlBinMatchBuffer* mb; - Eterm result; + ErlBinMatchBuffer* mb; + Eterm result; - mb = ms_matchbuffer(tmp_arg1); - LIGHT_SWAPOUT; - result = erts_bs_get_integer_2(c_p, tmp_arg2, Arg(1), mb); - LIGHT_SWAPIN; - HEAP_SPACE_VERIFIED(0); - if (is_non_value(result)) { - ClauseFail(); + mb = ms_matchbuffer(Ms); + LIGHT_SWAPOUT; + result = erts_bs_get_integer_2(c_p, Sz, Arg(1), mb); + LIGHT_SWAPIN; + HEAP_SPACE_VERIFIED(0); + if (is_non_value(result)) { + ClauseFail(); + } + StoreBifResult(2, result); } - StoreBifResult(2, result); } /* - * tmp_arg1 = Match context - * tmp_arg2 = Size field - * Operands: Fail Live FlagsAndUnit Dst + * Operands: Fail Live FlagsAndUnit Ms Sz Dst */ - OpCase(i_bs_get_integer_fIId): { + OpCase(i_bs_get_integer_fIIssd): { Uint flags; Uint size; + Eterm Ms; + Eterm Sz; ErlBinMatchBuffer* mb; Eterm result; flags = Arg(2); - BsGetFieldSize(tmp_arg2, (flags >> 3), ClauseFail(), size); + GetArg2(3, Ms, Sz); + BsGetFieldSize(Sz, (flags >> 3), ClauseFail(), size); if (size >= SMALL_BITS) { Uint wordsneeded; - /* check bits size before potential gc. + /* Check bits size before potential gc. * We do not want a gc and then realize we don't need - * the allocated space (i.e. if the op fails) + * the allocated space (i.e. if the op fails). * - * remember to reacquire the matchbuffer after gc. + * Remember to re-acquire the matchbuffer after gc. */ - mb = ms_matchbuffer(tmp_arg1); + mb = ms_matchbuffer(Ms); if (mb->size - mb->offset < size) { ClauseFail(); } wordsneeded = 1+WSIZE(NBYTES((Uint) size)); - TestHeapPreserve(wordsneeded, Arg(1), tmp_arg1); + TestHeapPreserve(wordsneeded, Arg(1), Ms); } - mb = ms_matchbuffer(tmp_arg1); + mb = ms_matchbuffer(Ms); LIGHT_SWAPOUT; result = erts_bs_get_integer_2(c_p, size, flags, mb); LIGHT_SWAPIN; @@ -4671,18 +4372,13 @@ do { \ if (is_non_value(result)) { ClauseFail(); } - StoreBifResult(3, result); + StoreBifResult(5, result); } { Eterm get_utf8_context; /* Operands: MatchContext Fail Dst */ - OpCase(i_bs_get_utf8_rfd): { - get_utf8_context = r(0); - goto do_bs_get_utf8; - } - OpCase(i_bs_get_utf8_xfd): { get_utf8_context = xb(Arg(0)); I++; @@ -4693,7 +4389,7 @@ do { \ * Operands: Fail Dst */ - do_bs_get_utf8: { + { Eterm result = erts_bs_get_utf8(ms_matchbuffer(get_utf8_context)); if (is_non_value(result)) { ClauseFail(); @@ -4706,12 +4402,7 @@ do { \ Eterm get_utf16_context; /* Operands: MatchContext Fail Flags Dst */ - OpCase(i_bs_get_utf16_rfId): { - get_utf16_context = r(0); - goto do_bs_get_utf16; - } - - OpCase(i_bs_get_utf16_xfId): { + OpCase(i_bs_get_utf16_xfId): { get_utf16_context = xb(Arg(0)); I++; } @@ -4720,7 +4411,7 @@ do { \ * get_utf16_context = match_context * Operands: Fail Flags Dst */ - do_bs_get_utf16: { + { Eterm result = erts_bs_get_utf16(ms_matchbuffer(get_utf16_context), Arg(1)); if (is_non_value(result)) { @@ -4739,26 +4430,10 @@ do { \ Uint orig; Uint hole_size; - OpCase(bs_context_to_binary_r): { - context_to_binary_context = x0; - I -= 2; - goto do_context_to_binary; - } - - /* Unfortunately, inlining can generate this instruction. */ - OpCase(bs_context_to_binary_y): { - context_to_binary_context = yb(Arg(0)); - goto do_context_to_binary0; - } - - OpCase(bs_context_to_binary_x): { + OpCase(bs_context_to_binary_x): context_to_binary_context = xb(Arg(0)); - - do_context_to_binary0: I--; - } - do_context_to_binary: if (is_boxed(context_to_binary_context) && header_is_bin_matchstate(*boxed_val(context_to_binary_context))) { ErlBinMatchState* ms; @@ -4770,17 +4445,11 @@ do { \ } Next(2); - OpCase(i_bs_get_binary_all_reuse_rfI): { - context_to_binary_context = x0; - goto do_bs_get_binary_all_reuse; - } - OpCase(i_bs_get_binary_all_reuse_xfI): { context_to_binary_context = xb(Arg(0)); I++; } - do_bs_get_binary_all_reuse: mb = ms_matchbuffer(context_to_binary_context); size = mb->size - mb->offset; if (size % Arg(1) != 0) { @@ -4808,16 +4477,11 @@ do { \ { Eterm match_string_context; - OpCase(i_bs_match_string_rfII): { - match_string_context = r(0); - goto do_bs_match_string; - } OpCase(i_bs_match_string_xfII): { match_string_context = xb(Arg(0)); I++; } - do_bs_match_string: { BeamInstr *next; byte* bytes; @@ -4845,14 +4509,6 @@ do { \ } } - OpCase(i_bs_save2_rI): { - BeamInstr *next; - ErlBinMatchState *_ms; - PreFetch(1, next); - _ms = (ErlBinMatchState*) boxed_val((Eterm) r(0)); - _ms->save_offset[Arg(0)] = _ms->mb.offset; - NextPF(1, next); - } OpCase(i_bs_save2_xI): { BeamInstr *next; ErlBinMatchState *_ms; @@ -4862,14 +4518,6 @@ do { \ NextPF(2, next); } - OpCase(i_bs_restore2_rI): { - BeamInstr *next; - ErlBinMatchState *_ms; - PreFetch(1, next); - _ms = (ErlBinMatchState*) boxed_val((Eterm) r(0)); - _ms->mb.offset = _ms->save_offset[Arg(0)]; - NextPF(1, next); - } OpCase(i_bs_restore2_xI): { BeamInstr *next; ErlBinMatchState *_ms; @@ -4917,9 +4565,7 @@ do { \ BeamInstr real_I; ASSERT(I[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); SWAPOUT; - reg[0] = r(0); real_I = erts_generic_breakpoint(c_p, I, reg); - r(0) = reg[0]; SWAPIN; ASSERT(VALID_INSTR(real_I)); Goto(real_I); @@ -4979,7 +4625,7 @@ do { \ BeamInstr *next; PreFetch(2, next); - GetR(0, targ1); + targ1 = REG_TARGET(Arg(0)); /* Arg(0) == HEADER_FLONUM */ GET_DOUBLE(targ1, *(FloatDef*)ADD_BYTE_OFFSET(freg, fr)); NextPF(2, next); @@ -4999,7 +4645,7 @@ do { \ Eterm fr = Arg(1); BeamInstr *next; - GetR(0, targ1); + targ1 = REG_TARGET(Arg(0)); PreFetch(2, next); if (is_small(targ1)) { fb(fr) = (double) signed_val(targ1); @@ -5095,7 +4741,12 @@ do { \ #ifdef HIPE { - unsigned cmd; +#define HIPE_MODE_SWITCH(Cmd) \ + SWAPOUT; \ + c_p->fcalls = FCALLS; \ + c_p->def_arg_reg[4] = -neg_o_reds; \ + c_p = hipe_mode_switch(c_p, Cmd, reg); \ + goto L_post_hipe_mode_switch OpCase(hipe_trap_call): { /* @@ -5109,52 +4760,45 @@ do { \ */ ASSERT(I[-5] == (Uint) OpCode(i_func_info_IaaI)); c_p->hipe.u.ncallee = (void(*)(void)) I[-4]; - cmd = HIPE_MODE_SWITCH_CMD_CALL | (I[-1] << 8); ++hipe_trap_count; - goto L_hipe_mode_switch; + HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_CALL | (I[-1] << 8)); } OpCase(hipe_trap_call_closure): { ASSERT(I[-5] == (Uint) OpCode(i_func_info_IaaI)); c_p->hipe.u.ncallee = (void(*)(void)) I[-4]; - cmd = HIPE_MODE_SWITCH_CMD_CALL_CLOSURE | (I[-1] << 8); ++hipe_trap_count; - goto L_hipe_mode_switch; + HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_CALL_CLOSURE | (I[-1] << 8)); } OpCase(hipe_trap_return): { - cmd = HIPE_MODE_SWITCH_CMD_RETURN; - goto L_hipe_mode_switch; + HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_RETURN); } OpCase(hipe_trap_throw): { - cmd = HIPE_MODE_SWITCH_CMD_THROW; - goto L_hipe_mode_switch; + HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_THROW); } OpCase(hipe_trap_resume): { - cmd = HIPE_MODE_SWITCH_CMD_RESUME; - goto L_hipe_mode_switch; + HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_RESUME); } - L_hipe_mode_switch: - /* XXX: this abuse of def_arg_reg[] is horrid! */ - SWAPOUT; - c_p->fcalls = FCALLS; - c_p->def_arg_reg[4] = -neg_o_reds; - reg[0] = r(0); - c_p = hipe_mode_switch(c_p, cmd, reg); +#undef HIPE_MODE_SWITCH + + L_post_hipe_mode_switch: reg = ERTS_PROC_GET_SCHDATA(c_p)->x_reg_array; freg = ERTS_PROC_GET_SCHDATA(c_p)->f_reg_array; ERL_BITS_RELOAD_STATEP(c_p); + /* XXX: this abuse of def_arg_reg[] is horrid! */ neg_o_reds = -c_p->def_arg_reg[4]; FCALLS = c_p->fcalls; SWAPIN; - switch( c_p->def_arg_reg[3] ) { /* Halfword wont work with hipe yet! */ + switch( c_p->def_arg_reg[3] ) { case HIPE_MODE_SWITCH_RES_RETURN: ASSERT(is_value(reg[0])); - MoveReturn(reg[0], r(0)); + SET_I(c_p->cp); + c_p->cp = 0; + Goto(*I); case HIPE_MODE_SWITCH_RES_CALL_EXPORTED: c_p->i = c_p->hipe.u.callee_exp->addressv[erts_active_code_ix()]; /*fall through*/ case HIPE_MODE_SWITCH_RES_CALL_BEAM: SET_I(c_p->i); - r(0) = reg[0]; Dispatch(); case HIPE_MODE_SWITCH_RES_CALL_CLOSURE: /* This can be used to call any function value, but currently it's @@ -5165,7 +4809,6 @@ do { \ next = call_fun(c_p, c_p->arity - 1, reg, THE_NON_VALUE); SWAPIN; if (next != NULL) { - r(0) = reg[0]; SET_I(next); Dispatchfun(); } @@ -5225,9 +4868,7 @@ do { \ OpCase(i_debug_breakpoint): { SWAPOUT; - reg[0] = r(0); I = call_error_handler(c_p, I-3, reg, am_breakpoint); - r(0) = reg[0]; SWAPIN; if (I) { Goto(*I); @@ -6644,23 +6285,20 @@ static Eterm get_map_element_hash(Eterm map, Eterm key, Uint32 hx) return vs ? *vs : THE_NON_VALUE; } -#define GET_TERM(term, dest) \ -do { \ - Eterm src = (Eterm)(term); \ - switch (src & _TAG_IMMED1_MASK) { \ - case (R_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER: \ - dest = x(0); \ - break; \ - case (X_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER: \ - dest = x(src >> _TAG_IMMED1_SIZE); \ - break; \ - case (Y_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER: \ - dest = y(src >> _TAG_IMMED1_SIZE); \ - break; \ - default: \ - dest = src; \ - break; \ - } \ +#define GET_TERM(term, dest) \ +do { \ + Eterm src = (Eterm)(term); \ + switch (loader_tag(src)) { \ + case LOADER_X_REG: \ + dest = x(loader_x_reg_index(src)); \ + break; \ + case LOADER_Y_REG: \ + dest = y(loader_y_reg_index(src)); \ + break; \ + default: \ + dest = src; \ + break; \ + } \ } while(0) @@ -7114,6 +6752,15 @@ erts_is_builtin(Eterm Mod, Eterm Name, int arity) Export e; Export* ep; + if (Mod == am_erlang && Name == am_apply && arity == 3) { + /* + * Special case. apply/3 is built-in (implemented in C), + * but implemented in a different way than all other + * BIFs. + */ + return 1; + } + e.code[0] = Mod; e.code[1] = Name; e.code[2] = arity; diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index b70e5b9a2d..a9d47eb7b0 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -1847,9 +1847,7 @@ load_code(LoaderState* stp) case TAG_o: break; case TAG_x: - if (last_op->a[arg].val == 0) { - last_op->a[arg].type = TAG_r; - } else if (last_op->a[arg].val >= MAX_REG) { + if (last_op->a[arg].val >= MAX_REG) { LoadError1(stp, "invalid x register number: %u", last_op->a[arg].val); } @@ -1894,15 +1892,14 @@ load_code(LoaderState* stp) */ { Eterm* hp; -/* XXX:PaN - Halfword should use ARCH_64 variant instead */ -#if !defined(ARCH_64) || HALFWORD_HEAP +#if !defined(ARCH_64) Uint high, low; # endif last_op->a[arg].val = new_literal(stp, &hp, FLOAT_SIZE_OBJECT); hp[0] = HEADER_FLONUM; last_op->a[arg].type = TAG_q; -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) GetInt(stp, 8, hp[1]); # else GetInt(stp, 4, high); @@ -2056,7 +2053,42 @@ load_code(LoaderState* stp) if (((opc[specific].mask[0] & mask[0]) == mask[0]) && ((opc[specific].mask[1] & mask[1]) == mask[1]) && ((opc[specific].mask[2] & mask[2]) == mask[2])) { - break; + + if (!opc[specific].involves_r) { + break; /* No complications - match */ + } + + /* + * The specific operation uses the 'r' operand, + * which is shorthand for x(0). Now things + * get complicated. First we must check whether + * all operands that should be of type 'r' use + * x(0) (as opposed to some other X register). + */ + for (arg = 0; arg < arity; arg++) { + if (opc[specific].involves_r & (1 << arg) && + tmp_op->a[arg].type == TAG_x) { + if (tmp_op->a[arg].val != 0) { + break; /* Other X register than 0 */ + } + } + } + + if (arg == arity) { + /* + * All 'r' operands use x(0) in the generic + * operation. That means a match. Now we + * will need to rewrite the generic instruction + * to actually use 'r' instead of 'x(0)'. + */ + for (arg = 0; arg < arity; arg++) { + if (opc[specific].involves_r & (1 << arg) && + tmp_op->a[arg].type == TAG_x) { + tmp_op->a[arg].type = TAG_r; + } + } + break; /* Match */ + } } specific++; } @@ -2167,14 +2199,11 @@ load_code(LoaderState* stp) break; case 's': /* Any source (tagged constant or register) */ switch (tag) { - case TAG_r: - code[ci++] = make_rreg(); - break; case TAG_x: - code[ci++] = make_xreg(tmp_op->a[arg].val); + code[ci++] = make_loader_x_reg(tmp_op->a[arg].val); break; case TAG_y: - code[ci++] = make_yreg(tmp_op->a[arg].val); + code[ci++] = make_loader_y_reg(tmp_op->a[arg].val); break; case TAG_i: code[ci++] = (BeamInstr) make_small((Uint)tmp_op->a[arg].val); @@ -2185,6 +2214,10 @@ load_code(LoaderState* stp) case TAG_n: code[ci++] = NIL; break; + case TAG_q: + new_literal_patch(stp, ci); + code[ci++] = tmp_op->a[arg].val; + break; default: LoadError1(stp, "bad tag %d for general source", tmp_op->a[arg].type); @@ -2193,14 +2226,11 @@ load_code(LoaderState* stp) break; case 'd': /* Destination (x(0), x(N), y(N) */ switch (tag) { - case TAG_r: - code[ci++] = make_rreg(); - break; case TAG_x: - code[ci++] = make_xreg(tmp_op->a[arg].val); + code[ci++] = tmp_op->a[arg].val * sizeof(Eterm); break; case TAG_y: - code[ci++] = make_yreg(tmp_op->a[arg].val); + code[ci++] = tmp_op->a[arg].val * sizeof(Eterm) + 1; break; default: LoadError1(stp, "bad tag %d for destination", @@ -2357,20 +2387,13 @@ load_code(LoaderState* stp) stp->labels[tmp_op->a[arg].val].patches = ci; ci++; break; - case TAG_r: - CodeNeed(1); - code[ci++] = (R_REG_DEF << _TAG_PRIMARY_SIZE) | - TAG_PRIMARY_HEADER; - break; case TAG_x: CodeNeed(1); - code[ci++] = (tmp_op->a[arg].val << _TAG_IMMED1_SIZE) | - (X_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER; + code[ci++] = make_loader_x_reg(tmp_op->a[arg].val); break; case TAG_y: CodeNeed(1); - code[ci++] = (tmp_op->a[arg].val << _TAG_IMMED1_SIZE) | - (Y_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER; + code[ci++] = make_loader_y_reg(tmp_op->a[arg].val); break; case TAG_n: CodeNeed(1); @@ -2458,7 +2481,6 @@ load_code(LoaderState* stp) stp->on_load = ci; break; case op_bs_put_string_II: - case op_i_bs_match_string_rfII: case op_i_bs_match_string_xfII: new_string_patch(stp, ci-1); break; @@ -2680,6 +2702,12 @@ same_label(LoaderState* stp, GenOpArg Target, GenOpArg Label) Target.val == Label.val; } +static int +is_killed(LoaderState* stp, GenOpArg Reg, GenOpArg Live) +{ + return Reg.type == TAG_x && Live.type == TAG_u && + Live.val <= Reg.val; +} /* * Generate an instruction for element/2. @@ -2696,17 +2724,17 @@ gen_element(LoaderState* stp, GenOpArg Fail, GenOpArg Index, op->next = NULL; if (Index.type == TAG_i && Index.val > 0 && - (Tuple.type == TAG_r || Tuple.type == TAG_x || Tuple.type == TAG_y)) { + (Tuple.type == TAG_x || Tuple.type == TAG_y)) { op->op = genop_i_fast_element_4; - op->a[0] = Tuple; - op->a[1] = Fail; + op->a[0] = Fail; + op->a[1] = Tuple; op->a[2].type = TAG_u; op->a[2].val = Index.val; op->a[3] = Dst; } else { op->op = genop_i_element_4; - op->a[0] = Tuple; - op->a[1] = Fail; + op->a[0] = Fail; + op->a[1] = Tuple; op->a[2] = Index; op->a[3] = Dst; } @@ -2832,23 +2860,16 @@ gen_get_integer2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Live, goto generic; } } else { - GenOp* op2; - NEW_GENOP(stp, op2); - - op->op = genop_i_fetch_2; - op->arity = 2; - op->a[0] = Ms; - op->a[1] = Size; - op->next = op2; - - op2->op = genop_i_bs_get_integer_4; - op2->arity = 4; - op2->a[0] = Fail; - op2->a[1] = Live; - op2->a[2].type = TAG_u; - op2->a[2].val = (Unit.val << 3) | Flags.val; - op2->a[3] = Dst; - op2->next = NULL; + op->op = genop_i_bs_get_integer_6; + op->arity = 6; + op->a[0] = Fail; + op->a[1] = Live; + op->a[2].type = TAG_u; + op->a[2].val = (Unit.val << 3) | Flags.val; + op->a[3] = Ms; + op->a[4] = Size; + op->a[5] = Dst; + op->next = NULL; return op; } op->next = NULL; @@ -3272,14 +3293,14 @@ gen_literal_timeout(LoaderState* stp, GenOpArg Fail, GenOpArg Time) op->a[1].type = TAG_u; if (Time.type == TAG_i && (timeout = Time.val) >= 0 && -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) (timeout >> 32) == 0 #else 1 #endif ) { op->a[1].val = timeout; -#if !defined(ARCH_64) || HALFWORD_HEAP +#if !defined(ARCH_64) } else if (Time.type == TAG_q) { Eterm big; @@ -3296,7 +3317,7 @@ gen_literal_timeout(LoaderState* stp, GenOpArg Fail, GenOpArg Time) } #endif } else { -#if !defined(ARCH_64) || HALFWORD_HEAP +#if !defined(ARCH_64) error: #endif op->op = genop_i_wait_error_0; @@ -3319,14 +3340,14 @@ gen_literal_timeout_locked(LoaderState* stp, GenOpArg Fail, GenOpArg Time) op->a[1].type = TAG_u; if (Time.type == TAG_i && (timeout = Time.val) >= 0 && -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) (timeout >> 32) == 0 #else 1 #endif ) { op->a[1].val = timeout; -#if !defined(ARCH_64) || HALFWORD_HEAP +#if !defined(ARCH_64) } else if (Time.type == TAG_q) { Eterm big; @@ -3343,7 +3364,7 @@ gen_literal_timeout_locked(LoaderState* stp, GenOpArg Fail, GenOpArg Time) } #endif } else { -#if !defined(ARCH_64) || HALFWORD_HEAP +#if !defined(ARCH_64) error: #endif op->op = genop_i_wait_error_locked_0; @@ -3899,9 +3920,7 @@ gen_make_fun2(LoaderState* stp, GenOpArg idx) /* * Rewrite gc_bifs with one parameter (the common case). Utilized * in ops.tab to rewrite instructions calling bif's in guards - * to use a garbage collecting implementation. The instructions - * are sometimes once again rewritten to handle literals (putting the - * parameter in the mostly unused r[0] before the instruction is executed). + * to use a garbage collecting implementation. */ static GenOp* gen_guard_bif1(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, @@ -3956,10 +3975,6 @@ gen_guard_bif1(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, /* * This is used by the ops.tab rule that rewrites gc_bifs with two parameters. - * The instruction returned is then again rewritten to an i_load instruction - * followed by i_gc_bif2_jIId, to handle literals properly. - * As opposed to the i_gc_bif1_jIsId, the instruction i_gc_bif2_jIId is - * always rewritten, regardless of if there actually are any literals. */ static GenOp* gen_guard_bif2(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, @@ -3986,23 +4001,19 @@ gen_guard_bif2(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, op->a[2].val = stp->import[Bif.val].arity; return op; } - op->op = genop_ii_gc_bif2_6; + op->op = genop_i_gc_bif2_6; op->arity = 6; op->a[0] = Fail; op->a[1].type = TAG_u; - op->a[2] = S1; - op->a[3] = S2; - op->a[4] = Live; + op->a[2] = Live; + op->a[3] = S1; + op->a[4] = S2; op->a[5] = Dst; return op; } /* * This is used by the ops.tab rule that rewrites gc_bifs with three parameters. - * The instruction returned is then again rewritten to a move instruction that - * uses r[0] for temp storage, followed by an i_load instruction, - * followed by i_gc_bif3_jIsId, to handle literals properly. Rewriting - * always occur, as with the gc_bif2 counterpart. */ static GenOp* gen_guard_bif3(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, @@ -4033,10 +4044,10 @@ gen_guard_bif3(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, op->arity = 7; op->a[0] = Fail; op->a[1].type = TAG_u; - op->a[2] = S1; - op->a[3] = S2; - op->a[4] = S3; - op->a[5] = Live; + op->a[2] = Live; + op->a[3] = S1; + op->a[4] = S2; + op->a[5] = S3; op->a[6] = Dst; op->next = NULL; return op; @@ -4323,7 +4334,7 @@ gen_has_map_fields(LoaderState* stp, GenOpArg Fail, GenOpArg Src, for (i = 0; i < n; i++) { op->a[3+2*i] = Rest[i]; op->a[3+2*i+1].type = TAG_x; - op->a[3+2*i+1].val = 0; /* x(0); normally not used */ + op->a[3+2*i+1].val = SCRATCH_X_REG; /* Ignore result */ } return op; } @@ -4801,7 +4812,8 @@ transform_engine(LoaderState* st) if (var[i].type != instr->a[ap].type) goto restart; switch (var[i].type) { - case TAG_r: case TAG_n: break; + case TAG_n: + break; default: if (var[i].val != instr->a[ap].val) goto restart; @@ -5868,7 +5880,7 @@ make_stub(BeamInstr* fp, Eterm mod, Eterm func, Uint arity, Uint native, BeamIns fp[4] = arity; #ifdef HIPE if (native) { - fp[5] = BeamOpCode(op_move_return_nr); + fp[5] = BeamOpCode(op_move_return_n); hipe_mfa_save_orig_beam_op(mod, func, arity, fp+5); } #endif @@ -6304,7 +6316,7 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) #ifdef HIPE op = (Eterm) BeamOpCode(op_hipe_trap_call); /* Might be changed later. */ #else - op = (Eterm) BeamOpCode(op_move_return_nr); + op = (Eterm) BeamOpCode(op_move_return_n); #endif fp = make_stub(fp, Mod, func, arity, (Uint)native_address, op); } diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h index d5af634fad..eedb5ee4cd 100644 --- a/erts/emulator/beam/beam_load.h +++ b/erts/emulator/beam/beam_load.h @@ -52,6 +52,7 @@ extern BeamInstr* em_call_error_handler; extern BeamInstr* em_apply_bif; extern BeamInstr* em_call_nif; + /* * The following variables keep a sorted list of address ranges for * each module. It allows us to quickly find a function given an diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 4e3a1cef69..453ac74c90 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -616,7 +616,7 @@ erts_queue_monitor_message(Process *p, } static BIF_RETTYPE -local_pid_monitor(Process *p, Eterm target, Eterm mon_ref, int bool) +local_pid_monitor(Process *p, Eterm target, Eterm mon_ref, int boolean) { BIF_RETTYPE ret; Process *rp; @@ -634,7 +634,7 @@ local_pid_monitor(Process *p, Eterm target, Eterm mon_ref, int bool) if (!rp) { erts_smp_proc_unlock(p, ERTS_PROC_LOCK_LINK); p_locks &= ~ERTS_PROC_LOCK_LINK; - if (bool) + if (boolean) ret = am_false; else erts_queue_monitor_message(p, &p_locks, @@ -643,7 +643,7 @@ local_pid_monitor(Process *p, Eterm target, Eterm mon_ref, int bool) else { ASSERT(rp != p); - if (bool) + if (boolean) ret = am_true; erts_add_monitor(&ERTS_P_MONITORS(p), MON_ORIGIN, mon_ref, target, NIL); @@ -4117,16 +4117,9 @@ BIF_RETTYPE make_fun_3(BIF_ALIST_3) if (arity < 0) { goto error; } -#if HALFWORD_HEAP - hp = HAlloc(BIF_P, 3); - hp[0] = HEADER_EXPORT; - /* Yes, May be misaligned, but X86_64 will fix it... */ - *((Export **) (hp+1)) = erts_export_get_or_make_stub(BIF_ARG_1, BIF_ARG_2, (Uint) arity); -#else hp = HAlloc(BIF_P, 2); hp[0] = HEADER_EXPORT; hp[1] = (Eterm) erts_export_get_or_make_stub(BIF_ARG_1, BIF_ARG_2, (Uint) arity); -#endif BIF_RET(make_export(hp)); } @@ -4631,7 +4624,7 @@ BIF_RETTYPE hash_2(BIF_ALIST_2) if ((range = signed_val(BIF_ARG_2)) <= 0) { /* [1..MAX_SMALL] */ BIF_ERROR(BIF_P, BADARG); } -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) if (range > ((1L << 27) - 1)) BIF_ERROR(BIF_P, BADARG); #endif @@ -4703,7 +4696,7 @@ BIF_RETTYPE phash2_2(BIF_ALIST_2) /* * Return either a small or a big. Use the heap for bigs if there is room. */ -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) BIF_RET(make_small(final_hash)); #else if (IS_USMALL(0, final_hash)) { diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c index 044bf6a34e..02d37e24df 100644 --- a/erts/emulator/beam/big.c +++ b/erts/emulator/beam/big.c @@ -1487,20 +1487,8 @@ Eterm uint_to_big(Uint x, Eterm *y) Eterm uword_to_big(UWord x, Eterm *y) { -#if HALFWORD_HEAP - Uint upper = x >> 32; - Uint lower = x & 0xFFFFFFFFUL; - if (upper == 0) { - *y = make_pos_bignum_header(1); - } else { - *y = make_pos_bignum_header(2); - BIG_DIGIT(y, 1) = upper; - } - BIG_DIGIT(y, 0) = lower; -#else *y = make_pos_bignum_header(1); BIG_DIGIT(y, 0) = x; -#endif return make_big(y); } @@ -1525,7 +1513,7 @@ Eterm small_to_big(Sint x, Eterm *y) Eterm erts_uint64_to_big(Uint64 x, Eterm **hpp) { Eterm *hp = *hpp; -#if defined(ARCH_32) || HALFWORD_HEAP +#if defined(ARCH_32) if (x >= (((Uint64) 1) << 32)) { *hp = make_pos_bignum_header(2); BIG_DIGIT(hp, 0) = (Uint) (x & ((Uint) 0xffffffff)); @@ -1555,7 +1543,7 @@ Eterm erts_sint64_to_big(Sint64 x, Eterm **hpp) neg = 1; ux = -(Uint64)x; } -#if defined(ARCH_32) || HALFWORD_HEAP +#if defined(ARCH_32) if (ux >= (((Uint64) 1) << 32)) { if (neg) *hp = make_neg_bignum_header(2); @@ -1588,7 +1576,7 @@ erts_uint64_array_to_big(Uint **hpp, int neg, int len, Uint64 *array) pot_digits = digits = 0; for (i = 0; i < len; i++) { -#if defined(ARCH_32) || HALFWORD_HEAP +#if defined(ARCH_32) Uint low_val = array[i] & ((Uint) 0xffffffff); Uint high_val = (array[i] >> 32) & ((Uint) 0xffffffff); BIG_DIGIT(headerp, pot_digits) = low_val; @@ -1651,8 +1639,6 @@ big_to_double(Wterm x, double* resp) /* * Logic has been copied from erl_bif_guard.c and slightly * modified to use a static instead of dynamic heap - * - * HALFWORD: Return relative term with 'heap' as base. */ Eterm double_to_big(double x, Eterm *heap, Uint hsz) @@ -1683,7 +1669,7 @@ double_to_big(double x, Eterm *heap, Uint hsz) sz = BIG_NEED_SIZE(ds); /* number of words including arity */ hp = heap; - res = make_big_rel(hp, heap); + res = make_big(hp); xp = (ErtsDigit*) (hp + 1); ASSERT(ds < hsz); @@ -2618,6 +2604,9 @@ Eterm erts_chars_to_integer(Process *BIF_P, char *bytes, size--; } + if (size == 0) + goto bytebuf_to_integer_1_error; + if (size < SMALL_DIGITS && base <= 10) { /* * * Take shortcut if we know that all chars are '0' < b < '9' and diff --git a/erts/emulator/beam/big.h b/erts/emulator/beam/big.h index 4aa9724ae3..94f9bce10e 100644 --- a/erts/emulator/beam/big.h +++ b/erts/emulator/beam/big.h @@ -35,7 +35,7 @@ typedef Uint ErtsDigit; -#if ((SIZEOF_VOID_P == 4) || HALFWORD_HEAP) && defined(SIZEOF_LONG_LONG) && (SIZEOF_LONG_LONG == 8) +#if (SIZEOF_VOID_P == 4) && defined(SIZEOF_LONG_LONG) && (SIZEOF_LONG_LONG == 8) /* Assume 32-bit machine with long long support */ typedef Uint64 ErtsDoubleDigit; typedef Uint16 ErtsHalfDigit; @@ -90,13 +90,9 @@ typedef Uint dsize_t; /* Vector size type */ #define BIG_UINT_HEAP_SIZE (1 + 1) /* always, since sizeof(Uint) <= sizeof(Eterm) */ -#if HALFWORD_HEAP -#define BIG_UWORD_HEAP_SIZE(UW) (((UW) >> (sizeof(Uint) * 8)) ? 3 : 2) -#else #define BIG_UWORD_HEAP_SIZE(UW) BIG_UINT_HEAP_SIZE -#endif -#if defined(ARCH_32) || HALFWORD_HEAP +#if defined(ARCH_32) #define ERTS_UINT64_BIG_HEAP_SIZE__(X) \ ((X) >= (((Uint64) 1) << 32) ? (1 + 2) : (1 + 1)) @@ -178,4 +174,3 @@ Eterm erts_sint64_to_big(Sint64, Eterm **); Eterm erts_chars_to_integer(Process *, char*, Uint, const int); #endif - diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index 64c8bc5e58..4ce9d24479 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -536,7 +536,9 @@ do_break(void) erts_printf("Erlang (%s) emulator version " ERLANG_VERSION "\n", EMULATOR); +#if ERTS_SAVED_COMPILE_TIME erts_printf("Compiled on " ERLANG_COMPILE_DATE "\n"); +#endif return; case 'd': distribution_info(ERTS_PRINT_STDOUT, NULL); @@ -774,7 +776,9 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) } erts_fdprintf(fd, "System version: "); erts_print_system_version(fd, NULL, NULL); +#if ERTS_SAVED_COMPILE_TIME erts_fdprintf(fd, "%s\n", "Compiled: " ERLANG_COMPILE_DATE); +#endif erts_fdprintf(fd, "Taints: "); erts_print_nif_taints(fd, NULL); diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index 8849dadd00..ec769c3b49 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -40,40 +40,37 @@ static void move_one_frag(Eterm** hpp, ErlHeapFragment*, ErlOffHeap*); /* * Copy object "obj" to process p. */ -Eterm -copy_object(Eterm obj, Process* to) -{ - Uint size = size_object(obj); - Eterm* hp = HAlloc(to, size); - Eterm res; +Eterm copy_object_x(Eterm obj, Process* to, Uint extra) { + if (!is_immed(obj)) { + Uint size = size_object(obj); + Eterm* hp = HAllocX(to, size, extra); + Eterm res; #ifdef USE_VM_PROBES - if (DTRACE_ENABLED(copy_object)) { - DTRACE_CHARBUF(proc_name, 64); + if (DTRACE_ENABLED(copy_object)) { + DTRACE_CHARBUF(proc_name, 64); - erts_snprintf(proc_name, sizeof(DTRACE_CHARBUF_NAME(proc_name)), - "%T", to->common.id); - DTRACE2(copy_object, proc_name, size); - } + erts_snprintf(proc_name, sizeof(DTRACE_CHARBUF_NAME(proc_name)), + "%T", to->common.id); + DTRACE2(copy_object, proc_name, size); + } #endif - res = copy_struct(obj, size, &hp, &to->off_heap); + res = copy_struct(obj, size, &hp, &to->off_heap); #ifdef DEBUG - if (eq(obj, res) == 0) { - erl_exit(ERTS_ABORT_EXIT, "copy not equal to source\n"); - } + if (eq(obj, res) == 0) { + erl_exit(ERTS_ABORT_EXIT, "copy not equal to source\n"); + } #endif - return res; + return res; + } + return obj; } /* * Return the "flat" size of the object. */ -#if HALFWORD_HEAP -Uint size_object_rel(Eterm obj, Eterm* base) -#else Uint size_object(Eterm obj) -#endif { Uint sum = 0; Eterm* ptr; @@ -84,7 +81,7 @@ Uint size_object(Eterm obj) switch (primary_tag(obj)) { case TAG_PRIMARY_LIST: sum += 2; - ptr = list_val_rel(obj,base); + ptr = list_val(obj); obj = *ptr++; if (!IS_CONST(obj)) { ESTACK_PUSH(s, obj); @@ -93,11 +90,11 @@ Uint size_object(Eterm obj) break; case TAG_PRIMARY_BOXED: { - Eterm hdr = *boxed_val_rel(obj,base); + Eterm hdr = *boxed_val(obj); ASSERT(is_header(hdr)); switch (hdr & _TAG_HEADER_MASK) { case ARITYVAL_SUBTAG: - ptr = tuple_val_rel(obj,base); + ptr = tuple_val(obj); arity = header_arity(hdr); sum += arity + 1; if (arity == 0) { /* Empty tuple -- unusual. */ @@ -113,7 +110,7 @@ Uint size_object(Eterm obj) break; case FUN_SUBTAG: { - Eterm* bptr = fun_val_rel(obj,base); + Eterm* bptr = fun_val(obj); ErlFunThing* funp = (ErlFunThing *) bptr; unsigned eterms = 1 /* creator */ + funp->num_free; unsigned sz = thing_arityval(hdr); @@ -134,7 +131,7 @@ Uint size_object(Eterm obj) { Uint n; flatmap_t *mp; - mp = (flatmap_t*)flatmap_val_rel(obj,base); + mp = (flatmap_t*)flatmap_val(obj); ptr = (Eterm *)mp; n = flatmap_get_size(mp) + 1; sum += n + 2; @@ -153,7 +150,7 @@ Uint size_object(Eterm obj) { Eterm *head; Uint sz; - head = hashmap_val_rel(obj, base); + head = hashmap_val(obj); sz = hashmap_bitcount(MAP_HEADER_VAL(hdr)); sum += 1 + sz + header_arity(hdr); head += 1 + header_arity(hdr); @@ -182,7 +179,7 @@ Uint size_object(Eterm obj) Uint bitoffs; Uint extra_bytes; Eterm hdr; - ERTS_GET_REAL_BIN_REL(obj, real_bin, offset, bitoffs, bitsize, base); + ERTS_GET_REAL_BIN(obj, real_bin, offset, bitoffs, bitsize); if ((bitsize + bitoffs) > 8) { sum += ERL_SUB_BIN_SIZE; extra_bytes = 2; @@ -192,11 +189,11 @@ Uint size_object(Eterm obj) } else { extra_bytes = 0; } - hdr = *binary_val_rel(real_bin,base); + hdr = *binary_val(real_bin); if (thing_subtag(hdr) == REFC_BINARY_SUBTAG) { sum += PROC_BIN_SIZE; } else { - sum += heap_bin_size(binary_size_rel(obj,base)+extra_bytes); + sum += heap_bin_size(binary_size(obj)+extra_bytes); } goto pop_next; } @@ -227,12 +224,7 @@ Uint size_object(Eterm obj) /* * Copy a structure to a heap. */ -#if HALFWORD_HEAP -Eterm copy_struct_rel(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, - Eterm* src_base, Eterm* dst_base) -#else Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) -#endif { char* hstart; Uint hsize; @@ -268,7 +260,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) switch (primary_tag(obj)) { case TAG_PRIMARY_LIST: argp = &res; - objp = list_val_rel(obj,src_base); + objp = list_val(obj); goto L_copy_list; case TAG_PRIMARY_BOXED: argp = &res; goto L_copy_boxed; default: @@ -286,14 +278,11 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) hp++; break; case TAG_PRIMARY_LIST: - objp = list_val_rel(obj,src_base); - #if !HALFWORD_HEAP || defined(DEBUG) + objp = list_val(obj); if (in_area(objp,hstart,hsize)) { - ASSERT(!HALFWORD_HEAP); hp++; break; } - #endif argp = hp++; /* Fall through */ @@ -309,23 +298,15 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) } else { CAR(htop) = elem; - #if HALFWORD_HEAP - CDR(htop) = CDR(objp); - *tailp = make_list_rel(htop,dst_base); - htop += 2; - goto L_copy; - #else tailp = &CDR(htop); htop += 2; - #endif } - ASSERT(!HALFWORD_HEAP || tp < hp || tp >= hbot); - *tp = make_list_rel(tailp - 1, dst_base); + *tp = make_list(tailp - 1); obj = CDR(objp); if (!is_list(obj)) { break; } - objp = list_val_rel(obj,src_base); + objp = list_val(obj); } switch (primary_tag(obj)) { case TAG_PRIMARY_IMMED1: *tailp = obj; goto L_copy; @@ -337,24 +318,21 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) } case TAG_PRIMARY_BOXED: - #if !HALFWORD_HEAP || defined(DEBUG) - if (in_area(boxed_val_rel(obj,src_base),hstart,hsize)) { - ASSERT(!HALFWORD_HEAP); + if (in_area(boxed_val(obj),hstart,hsize)) { hp++; break; } - #endif argp = hp++; L_copy_boxed: - objp = boxed_val_rel(obj, src_base); + objp = boxed_val(obj); hdr = *objp; switch (hdr & _TAG_HEADER_MASK) { case ARITYVAL_SUBTAG: { int const_flag = 1; /* assume constant tuple */ i = arityval(hdr); - *argp = make_tuple_rel(htop, dst_base); + *argp = make_tuple(htop); tp = htop; /* tp is pointer to new arity value */ *htop++ = *objp++; /* copy arity value */ while (i--) { @@ -383,7 +361,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) while (i--) { *tp++ = *objp++; } - *argp = make_binary_rel(hbot, dst_base); + *argp = make_binary(hbot); pb = (ProcBin*) hbot; erts_refc_inc(&pb->val->refc, 2); pb->next = off_heap->first; @@ -410,7 +388,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) extra_bytes = 0; } real_size = size+extra_bytes; - objp = binary_val_rel(real_bin,src_base); + objp = binary_val(real_bin); if (thing_subtag(*objp) == HEAP_BINARY_SUBTAG) { ErlHeapBin* from = (ErlHeapBin *) objp; ErlHeapBin* to; @@ -440,7 +418,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) off_heap->first = (struct erl_off_heap_header*) to; OH_OVERHEAD(off_heap, to->size / sizeof(Eterm)); } - *argp = make_binary_rel(hbot, dst_base); + *argp = make_binary(hbot); if (extra_bytes != 0) { ErlSubBin* res; hbot -= ERL_SUB_BIN_SIZE; @@ -452,7 +430,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) res->offs = 0; res->is_writable = 0; res->orig = *argp; - *argp = make_binary_rel(hbot, dst_base); + *argp = make_binary(hbot); } break; } @@ -470,7 +448,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) funp->next = off_heap->first; off_heap->first = (struct erl_off_heap_header*) funp; erts_refc_inc(&funp->fe->refc, 2); - *argp = make_fun_rel(tp, dst_base); + *argp = make_fun(tp); } break; case EXTERNAL_PID_SUBTAG: @@ -490,7 +468,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) off_heap->first = (struct erl_off_heap_header*)etp; erts_refc_inc(&etp->node->refc, 2); - *argp = make_external_rel(tp, dst_base); + *argp = make_external(tp); } break; case MAP_SUBTAG: @@ -498,7 +476,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) switch (MAP_HEADER_TYPE(hdr)) { case MAP_HEADER_TAG_FLATMAP_HEAD : i = flatmap_get_size(objp) + 3; - *argp = make_flatmap_rel(htop, dst_base); + *argp = make_flatmap(htop); while (i--) { *htop++ = *objp++; } @@ -509,7 +487,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) case MAP_HEADER_TAG_HAMT_NODE_BITMAP : i = 1 + hashmap_bitcount(MAP_HEADER_VAL(hdr)); while (i--) { *htop++ = *objp++; } - *argp = make_hashmap_rel(tp, dst_base); + *argp = make_hashmap(tp); break; default: erl_exit(ERTS_ABORT_EXIT, "copy_struct: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr)); @@ -522,7 +500,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) i = thing_arityval(hdr)+1; hbot -= i; tp = hbot; - *argp = make_boxed_rel(hbot, dst_base); + *argp = make_boxed(hbot); while (i--) { *tp++ = *objp++; } @@ -563,21 +541,12 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) * * NOTE: Assumes that term is a tuple (ptr is an untagged tuple ptr). */ -#if HALFWORD_HEAP -Eterm copy_shallow_rel(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, - Eterm* src_base) -#else Eterm copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) -#endif { Eterm* tp = ptr; Eterm* hp = *hpp; const Eterm res = make_tuple(hp); -#if HALFWORD_HEAP - const Sint offs = COMPRESS_POINTER(hp - (tp - src_base)); -#else const Sint offs = (hp - tp) * sizeof(Eterm); -#endif while (sz--) { Eterm val = *tp++; diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index 23897a49ae..10dd6cf206 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -45,6 +45,8 @@ #include "erl_thr_progress.h" #include "dtrace-wrapper.h" +#define DIST_CTL_DEFAULT_SIZE 64 + /* Turn this on to get printouts of all distribution messages * which go on the line */ @@ -66,9 +68,13 @@ static void bw(byte *buf, ErlDrvSizeT sz) static void dist_msg_dbg(ErtsDistExternal *edep, char *what, byte *buf, int sz) { + ErtsHeapFactory factory; + DeclareTmpHeapNoproc(ctl_default,DIST_CTL_DEFAULT_SIZE); + Eterm* ctl = ctl_default; byte *extp = edep->extp; Eterm msg; - Sint size = erts_decode_dist_ext_size(edep); + Sint ctl_len; + Sint size = ctl_len = erts_decode_dist_ext_size(edep); if (size < 0) { erts_fprintf(stderr, "DIST MSG DEBUG: erts_decode_dist_ext_size(%s) failed:\n", @@ -76,10 +82,9 @@ dist_msg_dbg(ErtsDistExternal *edep, char *what, byte *buf, int sz) bw(buf, sz); } else { - Eterm *hp; ErlHeapFragment *mbuf = new_message_buffer(size); - hp = mbuf->mem; - msg = erts_decode_dist_ext(&hp, &mbuf->off_heap, edep); + erts_factory_static_init(&factory, ctl, ctl_len, &mbuf->off_heap); + msg = erts_decode_dist_ext(&factory, edep); if (is_value(msg)) erts_fprintf(stderr, " %s: %T\n", what, msg); else { @@ -732,19 +737,11 @@ Eterm erts_dsend_export_trap_context(Process* p, ErtsSendContext* ctx) Binary* ctx_bin = erts_create_magic_binary(sizeof(struct exported_ctx), erts_dsend_context_dtor); struct exported_ctx* dst = ERTS_MAGIC_BIN_DATA(ctx_bin); - Uint ctl_size = !HALFWORD_HEAP ? 0 : (arityval(ctx->ctl_heap[0]) + 1); - Eterm* hp = HAlloc(p, ctl_size + PROC_BIN_SIZE); + Eterm* hp = HAlloc(p, PROC_BIN_SIZE); sys_memcpy(&dst->ctx, ctx, sizeof(ErtsSendContext)); ASSERT(ctx->dss.ctl == make_tuple(ctx->ctl_heap)); -#if !HALFWORD_HEAP dst->ctx.dss.ctl = make_tuple(dst->ctx.ctl_heap); -#else - /* Must put control tuple in low mem */ - sys_memcpy(hp, ctx->ctl_heap, ctl_size*sizeof(Eterm)); - dst->ctx.dss.ctl = make_tuple(hp); - hp += ctl_size; -#endif if (ctx->dss.acmp) { sys_memcpy(&dst->acm, ctx->dss.acmp, sizeof(ErtsAtomCacheMap)); dst->ctx.dss.acmp = &dst->acm; @@ -1136,7 +1133,6 @@ int erts_net_message(Port *prt, byte *buf, ErlDrvSizeT len) { -#define DIST_CTL_DEFAULT_SIZE 64 ErtsDistExternal ede; byte *t; Sint ctl_len; @@ -1790,8 +1786,8 @@ erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx) #ifdef ERTS_DIST_MSG_DBG erts_fprintf(stderr, ">>%s CTL: %T\n", ctx->pass_through_size ? "P" : " ", ctx->ctl); - if (is_value(msg)) - erts_fprintf(stderr, " MSG: %T\n", msg); + if (is_value(ctx->msg)) + erts_fprintf(stderr, " MSG: %T\n", ctx->msg); #endif ctx->data_size = ctx->pass_through_size; @@ -2061,9 +2057,9 @@ dist_port_commandv(Port *prt, ErtsDistOutputBuf *obuf) } -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) #define ERTS_PORT_REDS_MASK__ 0x003fffffffffffffL -#elif defined(ARCH_32) || HALFWORD_HEAP +#elif defined(ARCH_32) #define ERTS_PORT_REDS_MASK__ 0x003fffff #else # error "Ohh come on ... !?!" diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index 55c164bf11..d68f22d573 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -123,10 +123,6 @@ typedef union { static ErtsAllocatorState_t std_alloc_state; static ErtsAllocatorState_t ll_alloc_state; -#if HALFWORD_HEAP -static ErtsAllocatorState_t std_low_alloc_state; -static ErtsAllocatorState_t ll_low_alloc_state; -#endif static ErtsAllocatorState_t sl_alloc_state; static ErtsAllocatorState_t temp_alloc_state; static ErtsAllocatorState_t eheap_alloc_state; @@ -150,24 +146,10 @@ typedef struct { #define ERTS_ALC_INFO_A_MSEG_ALLOC (ERTS_ALC_A_MAX + 2) #define ERTS_ALC_INFO_A_MAX ERTS_ALC_INFO_A_MSEG_ALLOC -#if !HALFWORD_HEAP ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(aireq, - ErtsAllocInfoReq, - 5, - ERTS_ALC_T_AINFO_REQ) -#else -static ERTS_INLINE ErtsAllocInfoReq * -aireq_alloc(void) -{ - return erts_alloc(ERTS_ALC_T_AINFO_REQ, sizeof(ErtsAllocInfoReq)); -} - -static ERTS_INLINE void -aireq_free(ErtsAllocInfoReq *ptr) -{ - erts_free(ERTS_ALC_T_AINFO_REQ, ptr); -} -#endif + ErtsAllocInfoReq, + 5, + ERTS_ALC_T_AINFO_REQ) ErtsAlcType_t erts_fix_core_allocator_ix; @@ -229,10 +211,6 @@ typedef struct { struct au_init ets_alloc; struct au_init driver_alloc; struct au_init fix_alloc; -#if HALFWORD_HEAP - struct au_init std_low_alloc; - struct au_init ll_low_alloc; -#endif } erts_alc_hndl_args_init_t; #define ERTS_AU_INIT__ {0, 0, 1, GOODFIT, DEFAULT_ALLCTR_INIT, {1,1,1,1}} @@ -259,10 +237,6 @@ set_default_sl_alloc_opts(struct au_init *ip) #endif ip->init.util.ts = ERTS_ALC_MTA_SHORT_LIVED; ip->init.util.rsbcst = 80; -#if HALFWORD_HEAP - ip->init.util.force = 1; - ip->init.util.low_mem = 1; -#endif ip->init.util.acul = ERTS_ALC_DEFAULT_ACUL; } @@ -328,10 +302,6 @@ set_default_temp_alloc_opts(struct au_init *ip) ip->init.util.ts = ERTS_ALC_MTA_TEMPORARY; ip->init.util.rsbcst = 90; ip->init.util.rmbcmt = 100; -#if HALFWORD_HEAP - ip->init.util.force = 1; - ip->init.util.low_mem = 1; -#endif } static void @@ -350,10 +320,6 @@ set_default_eheap_alloc_opts(struct au_init *ip) #endif ip->init.util.ts = ERTS_ALC_MTA_EHEAP; ip->init.util.rsbcst = 50; -#if HALFWORD_HEAP - ip->init.util.force = 1; - ip->init.util.low_mem = 1; -#endif ip->init.util.acul = ERTS_ALC_DEFAULT_ACUL_EHEAP_ALLOC; } @@ -560,12 +526,10 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_PROC)] = sizeof(Process); -#if !HALFWORD_HEAP fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MONITOR_SH)] = ERTS_MONITOR_SH_SIZE * sizeof(Uint); fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_NLINK_SH)] = ERTS_LINK_SH_SIZE * sizeof(Uint); -#endif fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_DRV_EV_D_STATE)] = sizeof(ErtsDrvEventDataState); fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_DRV_SEL_D_STATE)] @@ -745,24 +709,6 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) erts_allctrs[ERTS_ALC_A_SYSTEM].free = erts_sys_free; erts_allctrs_info[ERTS_ALC_A_SYSTEM].enabled = 1; -#if HALFWORD_HEAP - /* Init low memory variants by cloning */ - init.std_low_alloc = init.std_alloc; - init.std_low_alloc.init.util.name_prefix = "std_low_"; - init.std_low_alloc.init.util.alloc_no = ERTS_ALC_A_STANDARD_LOW; - init.std_low_alloc.init.util.force = 1; - init.std_low_alloc.init.util.low_mem = 1; - - init.ll_low_alloc = init.ll_alloc; - init.ll_low_alloc.init.util.name_prefix = "ll_low_"; - init.ll_low_alloc.init.util.alloc_no = ERTS_ALC_A_LONG_LIVED_LOW; - init.ll_low_alloc.init.util.force = 1; - init.ll_low_alloc.init.util.low_mem = 1; - - set_au_allocator(ERTS_ALC_A_STANDARD_LOW, &init.std_low_alloc, ncpu); - set_au_allocator(ERTS_ALC_A_LONG_LIVED_LOW, &init.ll_low_alloc, ncpu); -#endif /* HALFWORD */ - set_au_allocator(ERTS_ALC_A_TEMPORARY, &init.temp_alloc, ncpu); set_au_allocator(ERTS_ALC_A_SHORT_LIVED, &init.sl_alloc, ncpu); set_au_allocator(ERTS_ALC_A_STANDARD, &init.std_alloc, ncpu); @@ -805,14 +751,6 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) start_au_allocator(ERTS_ALC_A_LONG_LIVED, &init.ll_alloc, &ll_alloc_state); -#if HALFWORD_HEAP - start_au_allocator(ERTS_ALC_A_LONG_LIVED_LOW, - &init.ll_low_alloc, - &ll_low_alloc_state); - start_au_allocator(ERTS_ALC_A_STANDARD_LOW, - &init.std_low_alloc, - &std_low_alloc_state); -#endif start_au_allocator(ERTS_ALC_A_EHEAP, &init.eheap_alloc, &eheap_alloc_state); @@ -836,9 +774,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) erts_mtrace_install_wrapper_functions(); extra_block_size += erts_instr_init(init.instr.stat, init.instr.map); -#if !HALFWORD_HEAP init_aireq_alloc(); -#endif #ifdef DEBUG extra_block_size += install_debug_functions(); @@ -1238,9 +1174,6 @@ get_acul_value(struct au_init *auip, char *param_end, char** argv, int* ip) if (sys_strcmp(value, "de") == 0) { switch (auip->init.util.alloc_no) { case ERTS_ALC_A_LONG_LIVED: -#if HALFWORD_HEAP - case ERTS_ALC_A_LONG_LIVED_LOW: -#endif return ERTS_ALC_DEFAULT_ENABLED_ACUL_LL_ALLOC; case ERTS_ALC_A_EHEAP: return ERTS_ALC_DEFAULT_ENABLED_ACUL_EHEAP_ALLOC; @@ -1957,48 +1890,6 @@ alcu_size(ErtsAlcType_t ai, ErtsAlcUFixInfo_t *fi, int fisz) return res; } -#if HALFWORD_HEAP -static ERTS_INLINE int -alcu_is_low(ErtsAlcType_t ai) -{ - int is_low = 0; - ASSERT(erts_allctrs_info[ai].enabled); - ASSERT(erts_allctrs_info[ai].alloc_util); - - if (!erts_allctrs_info[ai].thr_spec) { - Allctr_t *allctr = erts_allctrs_info[ai].extra; - is_low = allctr->mseg_opt.low_mem; - } - else { - ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[ai]; - int i; -# ifdef DEBUG - int found_one = 0; -# endif - - ASSERT(tspec->enabled); - - for (i = tspec->size - 1; i >= 0; i--) { - Allctr_t *allctr = tspec->allctr[i]; - if (allctr) { -# ifdef DEBUG - if (!found_one) { - is_low = allctr->mseg_opt.low_mem; - found_one = 1; - } - else ASSERT(is_low == allctr->mseg_opt.low_mem); -# else - is_low = allctr->mseg_opt.low_mem; - break; -# endif - } - } - ASSERT(found_one); - } - return is_low; -} -#endif /* HALFWORD */ - static ERTS_INLINE void add_fix_values(UWord *ap, UWord *up, ErtsAlcUFixInfo_t *fi, ErtsAlcType_t type) { @@ -2028,9 +1919,6 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) int code; int ets; int maximum; -#if HALFWORD_HEAP - int low; -#endif } want = {0}; struct { UWord total; @@ -2043,9 +1931,6 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) UWord code; UWord ets; UWord maximum; -#if HALFWORD_HEAP - UWord low; -#endif } size = {0}; Eterm atoms[sizeof(size)/sizeof(UWord)]; UWord *uintps[sizeof(size)/sizeof(UWord)]; @@ -2104,11 +1989,6 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) atoms[length] = am_maximum; uintps[length++] = &size.maximum; } -#if HALFWORD_HEAP - want.low = 1; - atoms[length] = am_low; - uintps[length++] = &size.low; -#endif } else { DeclareTmpHeapNoproc(tmp_heap,2); @@ -2202,15 +2082,6 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) return am_badarg; } break; -#if HALFWORD_HEAP - case am_low: - if (!want.low) { - want.low = 1; - atoms[length] = am_low; - uintps[length++] = &size.low; - } - break; -#endif default: UnUseTmpHeapNoproc(2); return am_badarg; @@ -2288,11 +2159,6 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) if (save) *save = asz; size.total += asz; -#if HALFWORD_HEAP - if (alcu_is_low(ai)) { - size.low += asz; - } -#endif } } } @@ -2319,7 +2185,6 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) &size.processes_used, fi, ERTS_ALC_T_PROC); -#if !HALFWORD_HEAP add_fix_values(&size.processes, &size.processes_used, fi, @@ -2329,7 +2194,6 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) &size.processes_used, fi, ERTS_ALC_T_NLINK_SH); -#endif add_fix_values(&size.processes, &size.processes_used, fi, diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index b1d511ab78..4804fb407d 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -87,11 +87,6 @@ allocator EHEAP true eheap_alloc allocator ETS true ets_alloc allocator FIXED_SIZE true fix_alloc -+if halfword -allocator LONG_LIVED_LOW true ll_low_alloc -allocator STANDARD_LOW true std_low_alloc -+endif - +else # Non smp build allocator TEMPORARY false temp_alloc @@ -102,11 +97,6 @@ allocator EHEAP false eheap_alloc allocator ETS false ets_alloc allocator FIXED_SIZE false fix_alloc -+if halfword -allocator LONG_LIVED_LOW false ll_low_alloc -allocator STANDARD_LOW false std_low_alloc -+endif - +endif allocator BINARY true binary_alloc @@ -349,29 +339,6 @@ type SSB SHORT_LIVED PROCESSES ssb +endif - -+if halfword - -type DDLL_PROCESS STANDARD_LOW SYSTEM ddll_processes -type MONITOR_LH STANDARD_LOW PROCESSES monitor_lh -type NLINK_LH STANDARD_LOW PROCESSES nlink_lh -type CODE LONG_LIVED_LOW CODE code -type DB_HEIR_DATA STANDARD_LOW ETS db_heir_data -type DB_MS_PSDO_PROC LONG_LIVED_LOW ETS db_match_pseudo_proc -type SCHDLR_DATA LONG_LIVED_LOW SYSTEM scheduler_data -type LL_TEMP_TERM LONG_LIVED_LOW SYSTEM ll_temp_term - -type NIF_TRAP_EXPORT STANDARD_LOW CODE nif_trap_export_entry -type EXPORT LONG_LIVED_LOW CODE export_entry -type MONITOR_SH STANDARD_LOW PROCESSES monitor_sh -type NLINK_SH STANDARD_LOW PROCESSES nlink_sh -type AINFO_REQ STANDARD_LOW SYSTEM alloc_info_request -type SCHED_WTIME_REQ STANDARD_LOW SYSTEM sched_wall_time_request -type GC_INFO_REQ STANDARD_LOW SYSTEM gc_info_request -type PORT_DATA_HEAP STANDARD_LOW SYSTEM port_data_heap - -+else # "fullword" - type DDLL_PROCESS STANDARD SYSTEM ddll_processes type MONITOR_LH STANDARD PROCESSES monitor_lh type NLINK_LH STANDARD PROCESSES nlink_lh @@ -390,9 +357,6 @@ type SCHED_WTIME_REQ SHORT_LIVED SYSTEM sched_wall_time_request type GC_INFO_REQ SHORT_LIVED SYSTEM gc_info_request type PORT_DATA_HEAP STANDARD SYSTEM port_data_heap -+endif - - # # Types used by system specific code # @@ -430,21 +394,6 @@ type SYS_WRITE_BUF BINARY SYSTEM sys_write_buf +endif -+if ose - -type SYS_READ_BUF TEMPORARY SYSTEM sys_read_buf -type FD_TAB LONG_LIVED SYSTEM fd_tab -type FD_ENTRY_BUF STANDARD SYSTEM fd_entry_buf -type FD_SIG_LIST SHORT_LIVED SYSTEM fd_sig_list -type DRV_EV STANDARD SYSTEM driver_event -type CS_PROG_PATH LONG_LIVED SYSTEM cs_prog_path -type ENVIRONMENT TEMPORARY SYSTEM environment -type PUTENV_STR SYSTEM SYSTEM putenv_string -type PRT_REP_EXIT STANDARD SYSTEM port_report_exit - -+endif - - +if win32 type DRV_DATA_BUF SYSTEM SYSTEM drv_data_buf diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index 236ee35d18..db4c30b9eb 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -2074,7 +2074,7 @@ mbc_alloc_block(Allctr_t *allctr, Uint size, Uint *blk_szp) if (!blk) { blk = create_carrier(allctr, get_blk_sz, CFLG_MBC); -#if !HALFWORD_HEAP && !ERTS_SUPER_ALIGNED_MSEG_ONLY +#if !ERTS_SUPER_ALIGNED_MSEG_ONLY if (!blk) { /* Emergency! We couldn't create the carrier as we wanted. Try to place it in a sys_alloced sbc. */ @@ -3511,8 +3511,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) int is_mseg = 0; #endif - if (HALFWORD_HEAP - || (ERTS_SUPER_ALIGNED_MSEG_ONLY && (flags & CFLG_MBC)) + if ((ERTS_SUPER_ALIGNED_MSEG_ONLY && (flags & CFLG_MBC)) || !allow_sys_alloc_carriers) { flags |= CFLG_FORCE_MSEG; flags &= ~CFLG_FORCE_SYS_ALLOC; @@ -3749,11 +3748,6 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) DEBUG_SAVE_ALIGNMENT(new_crr); return new_blk; } -#if HALFWORD_HEAP - /* Old carrier unchanged; restore stat */ - STAT_MSEG_SBC_ALLOC(allctr, old_crr_sz, old_blk_sz); - return NULL; -#endif create_flags |= CFLG_FORCE_SYS_ALLOC; /* since mseg_realloc() failed */ } @@ -3942,9 +3936,6 @@ static struct { Eterm e; Eterm t; Eterm ramv; -#if HALFWORD_HEAP - Eterm low; -#endif Eterm sbct; #if HAVE_ERTS_MSEG Eterm asbcst; @@ -4035,9 +4026,6 @@ init_atoms(Allctr_t *allctr) AM_INIT(e); AM_INIT(t); AM_INIT(ramv); -#if HALFWORD_HEAP - AM_INIT(low); -#endif AM_INIT(sbct); #if HAVE_ERTS_MSEG AM_INIT(asbcst); @@ -4643,9 +4631,6 @@ info_options(Allctr_t *allctr, "option e: true\n" "option t: %s\n" "option ramv: %s\n" -#if HALFWORD_HEAP - "option low: %s\n" -#endif "option sbct: %beu\n" #if HAVE_ERTS_MSEG "option asbcst: %bpu\n" @@ -4664,9 +4649,6 @@ info_options(Allctr_t *allctr, "option acul: %d\n", topt, allctr->ramv ? "true" : "false", -#if HALFWORD_HEAP - allctr->mseg_opt.low_mem ? "true" : "false", -#endif allctr->sbc_threshold, #if HAVE_ERTS_MSEG allctr->mseg_opt.abs_shrink_th, @@ -4729,9 +4711,6 @@ info_options(Allctr_t *allctr, add_2tup(hpp, szp, &res, am.sbct, bld_uint(hpp, szp, allctr->sbc_threshold)); -#if HALFWORD_HEAP - add_2tup(hpp, szp, &res, am.low, allctr->mseg_opt.low_mem ? am_true : am_false); -#endif add_2tup(hpp, szp, &res, am.ramv, allctr->ramv ? am_true : am_false); add_2tup(hpp, szp, &res, am.t, (allctr->t ? am_true : am_false)); add_2tup(hpp, szp, &res, am.e, am_true); @@ -5416,11 +5395,7 @@ do_erts_alcu_realloc(ErtsAlcType_t type, Block_t *new_blk; if(IS_SBC_BLK(blk)) { do_carrier_resize: -#if HALFWORD_HEAP - new_blk = resize_carrier(allctr, blk, size, CFLG_SBC | CFLG_FORCE_MSEG); -#else new_blk = resize_carrier(allctr, blk, size, CFLG_SBC); -#endif res = new_blk ? BLK2UMEM(new_blk) : NULL; } else if (alcu_flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE) @@ -5714,11 +5689,8 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) #ifdef ERTS_SMP if (init->tspec || init->tpref) allctr->mseg_opt.sched_spec = 1; -#endif -# if HALFWORD_HEAP - allctr->mseg_opt.low_mem = init->low_mem; -# endif -#endif +#endif /* ERTS_SMP */ +#endif /* HAVE_ERTS_MSEG */ allctr->name_prefix = init->name_prefix; if (!allctr->name_prefix) @@ -5875,7 +5847,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) CFLG_MBC | CFLG_FORCE_SIZE | CFLG_NO_CPOOL -#if !HALFWORD_HEAP && !ERTS_SUPER_ALIGNED_MSEG_ONLY +#if !ERTS_SUPER_ALIGNED_MSEG_ONLY | CFLG_FORCE_SYS_ALLOC #endif | CFLG_MAIN_CARRIER); diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index df1f0aa65a..792f8c63ac 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -45,7 +45,6 @@ typedef struct { int tspec; int tpref; int ramv; - int low_mem; /* HALFWORD only */ UWord sbct; UWord asbcst; UWord rsbcst; @@ -90,7 +89,6 @@ typedef struct { 0, /* (bool) tspec: thread specific */\ 0, /* (bool) tpref: thread preferred */\ 0, /* (bool) ramv: realloc always moves */\ - 0, /* (bool) low_mem: HALFWORD only */\ 512*1024, /* (bytes) sbct: sbc threshold */\ 2*1024*2024, /* (amount) asbcst: abs sbc shrink threshold */\ 20, /* (%) rsbcst: rel sbc shrink threshold */\ @@ -125,7 +123,6 @@ typedef struct { 0, /* (bool) tspec: thread specific */\ 0, /* (bool) tpref: thread preferred */\ 0, /* (bool) ramv: realloc always moves */\ - 0, /* (bool) low_mem: HALFWORD only */\ 64*1024, /* (bytes) sbct: sbc threshold */\ 2*1024*2024, /* (amount) asbcst: abs sbc shrink threshold */\ 20, /* (%) rsbcst: rel sbc shrink threshold */\ diff --git a/erts/emulator/beam/erl_arith.c b/erts/emulator/beam/erl_arith.c index b8c5ef9b09..3671025d22 100644 --- a/erts/emulator/beam/erl_arith.c +++ b/erts/emulator/beam/erl_arith.c @@ -42,15 +42,8 @@ # define MAX(x, y) (((x) > (y)) ? (x) : (y)) #endif -#if !HEAP_ON_C_STACK -# define DECLARE_TMP(VariableName,N,P) \ - Eterm *VariableName = ((ERTS_PROC_GET_SCHDATA(P)->erl_arith_tmp_heap) + (2 * N)) -#else -# define DECLARE_TMP(VariableName,N,P) \ - Eterm VariableName[2] -#endif -# define ARG_IS_NOT_TMP(Arg,Tmp) ((Arg) != make_big((Tmp))) - +#define DECLARE_TMP(VariableName,N,P) Eterm VariableName[2] +#define ARG_IS_NOT_TMP(Arg,Tmp) ((Arg) != make_big((Tmp))) static Eterm shift(Process* p, Eterm arg1, Eterm arg2, int right); diff --git a/erts/emulator/beam/erl_async.c b/erts/emulator/beam/erl_async.c index f071898046..be0bc0cfec 100644 --- a/erts/emulator/beam/erl_async.c +++ b/erts/emulator/beam/erl_async.c @@ -167,7 +167,6 @@ async_ready_q(Uint sched_id) #endif - void erts_init_async(void) { diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c index b1ebf0327e..b9640e211d 100644 --- a/erts/emulator/beam/erl_bif_binary.c +++ b/erts/emulator/beam/erl_bif_binary.c @@ -2786,7 +2786,6 @@ BIF_RETTYPE binary_referenced_byte_size_1(BIF_ALIST_1) } pb = (ProcBin *) binary_val(bin); if (pb->thing_word == HEADER_PROC_BIN) { - /* XXX:PaN - Halfword - orig_size is a long, we should handle that */ res = erts_make_integer((Uint) pb->val->orig_size, BIF_P); } else { /* heap binary */ res = erts_make_integer((Uint) ((ErlHeapBin *) pb)->size, BIF_P); @@ -2804,7 +2803,7 @@ BIF_RETTYPE binary_referenced_byte_size_1(BIF_ALIST_1) #endif static int get_need(Uint u) { -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) if (u > 0xFFFFFFFFUL) { if (u > 0xFFFFFFFFFFFFUL) { if (u > 0xFFFFFFFFFFFFFFUL) { diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index b44382cde8..9a132ee007 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -73,9 +73,6 @@ static char otp_version[] = ERLANG_OTP_VERSION; static char erts_system_version[] = ("Erlang/OTP " ERLANG_OTP_RELEASE "%s" " [erts-" ERLANG_VERSION "]" -#if !HEAP_ON_C_STACK && !HALFWORD_HEAP - " [no-c-stack-objects]" -#endif #ifndef OTP_RELEASE #ifdef ERLANG_GIT_VERSION " [source-" ERLANG_GIT_VERSION "]" @@ -84,12 +81,8 @@ static char erts_system_version[] = ("Erlang/OTP " ERLANG_OTP_RELEASE #endif #endif #ifdef ARCH_64 -#if HALFWORD_HEAP - " [64-bit halfword]" -#else " [64-bit]" #endif -#endif #ifdef ERTS_SMP " [smp:%beu:%beu]" #endif @@ -671,18 +664,12 @@ static Eterm pi_1_keys[] = { #define ERTS_PI_1_NO_OF_KEYS (sizeof(pi_1_keys)/sizeof(Eterm)) static Eterm pi_1_keys_list; -#if HEAP_ON_C_STACK static Eterm pi_1_keys_list_heap[2*ERTS_PI_1_NO_OF_KEYS]; -#endif static void process_info_init(void) { -#if HEAP_ON_C_STACK Eterm *hp = &pi_1_keys_list_heap[0]; -#else - Eterm *hp = erts_alloc(ERTS_ALC_T_LL_TEMP_TERM,sizeof(Eterm)*2*ERTS_PI_1_NO_OF_KEYS); -#endif int i; pi_1_keys_list = NIL; diff --git a/erts/emulator/beam/erl_bif_op.c b/erts/emulator/beam/erl_bif_op.c index c9192fc420..d53a9e11ca 100644 --- a/erts/emulator/beam/erl_bif_op.c +++ b/erts/emulator/beam/erl_bif_op.c @@ -258,7 +258,7 @@ Eterm erl_is_function(Process* p, Eterm arg1, Eterm arg2) BIF_RET(am_true); } } else if (is_export(arg1)) { - Export* exp = (Export *) EXPAND_POINTER((export_val(arg1))[1]); + Export* exp = (Export *) (export_val(arg1)[1]); if (exp->code[2] == (Uint) arity) { BIF_RET(am_true); diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c index 3ff54c7a60..e47d7bcbbb 100644 --- a/erts/emulator/beam/erl_bif_port.c +++ b/erts/emulator/beam/erl_bif_port.c @@ -1329,7 +1329,8 @@ BIF_RETTYPE decode_packet_3(BIF_ALIST_3) ErlSubBin* rest; Eterm res; Eterm options; - int code; + int code; + char delimiter = '\n'; if (!is_binary(BIF_ARG_2) || (!is_list(BIF_ARG_3) && !is_nil(BIF_ARG_3))) { @@ -1370,6 +1371,11 @@ BIF_RETTYPE decode_packet_3(BIF_ALIST_3) case am_line_length: trunc_len = val; goto next_option; + case am_line_delimiter: + if (type == TCP_PB_LINE_LF && val >= 0 && val <= 255) { + delimiter = (char)val; + goto next_option; + } } } } @@ -1390,7 +1396,7 @@ BIF_RETTYPE decode_packet_3(BIF_ALIST_3) pca.aligned_ptr = bin_ptr; } packet_sz = packet_get_length(type, (char*)pca.aligned_ptr, pca.bin_sz, - max_plen, trunc_len, &http_state); + max_plen, trunc_len, delimiter, &http_state); if (!(packet_sz > 0 && packet_sz <= pca.bin_sz)) { if (packet_sz < 0) { goto error; diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c index 86951f32b0..7f7cd376ac 100644 --- a/erts/emulator/beam/erl_bif_re.c +++ b/erts/emulator/beam/erl_bif_re.c @@ -100,7 +100,7 @@ Sint erts_re_set_loop_limit(Sint limit) static int term_to_int(Eterm term, int *sp) { -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) if (is_small(term)) { Uint x = signed_val(term); @@ -151,7 +151,7 @@ static int term_to_int(Eterm term, int *sp) static Eterm make_signed_integer(int x, Process *p) { -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) return make_small(x); #else Eterm* hp; diff --git a/erts/emulator/beam/erl_bif_unique.c b/erts/emulator/beam/erl_bif_unique.c index 5eca09c5a6..c4a39b8897 100644 --- a/erts/emulator/beam/erl_bif_unique.c +++ b/erts/emulator/beam/erl_bif_unique.c @@ -338,7 +338,7 @@ static struct { } w; } raw_unique_monotonic_integer erts_align_attribute(ERTS_CACHE_LINE_SIZE); -#if defined(ARCH_32) || HALFWORD_HEAP +#if defined(ARCH_32) # define ERTS_UNIQUE_MONOTONIC_OFFSET ERTS_SINT64_MIN #else # define ERTS_UNIQUE_MONOTONIC_OFFSET MIN_SMALL @@ -368,7 +368,7 @@ get_unique_monotonic_integer_heap_size(Uint64 raw, int positive) Sint64 value = ((Sint64) raw) + ERTS_UNIQUE_MONOTONIC_OFFSET; if (IS_SSMALL(value)) return 0; -#if defined(ARCH_32) || HALFWORD_HEAP +#if defined(ARCH_32) return ERTS_SINT64_HEAP_SIZE(value); #else return ERTS_UINT64_HEAP_SIZE((Uint64) value); @@ -393,7 +393,7 @@ make_unique_monotonic_integer_value(Eterm *hp, Uint hsz, Uint64 raw, int positiv if (hsz == 0) res = make_small(value); else { -#if defined(ARCH_32) || HALFWORD_HEAP +#if defined(ARCH_32) res = erts_sint64_to_big(value, &hp); #else res = erts_uint64_to_big((Uint64) value, &hp); diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h index ea01bf08f0..e181b5555d 100644 --- a/erts/emulator/beam/erl_binary.h +++ b/erts/emulator/beam/erl_binary.h @@ -72,7 +72,6 @@ typedef struct erl_heap_bin { */ #define binary_size(Bin) (binary_val(Bin)[1]) -#define binary_size_rel(Bin,BasePtr) (binary_val_rel(Bin,BasePtr)[1]) #define binary_bitsize(Bin) \ ((*binary_val(Bin) == HEADER_SUB_BIN) ? \ @@ -95,12 +94,9 @@ typedef struct erl_heap_bin { * Bitsize: output variable (Uint) */ -#define ERTS_GET_BINARY_BYTES(Bin,Bytep,Bitoffs,Bitsize) \ - ERTS_GET_BINARY_BYTES_REL(Bin,Bytep,Bitoffs,Bitsize,NULL) - -#define ERTS_GET_BINARY_BYTES_REL(Bin,Bytep,Bitoffs,Bitsize,BasePtr) \ +#define ERTS_GET_BINARY_BYTES(Bin,Bytep,Bitoffs,Bitsize) \ do { \ - Eterm* _real_bin = binary_val_rel(Bin,BasePtr); \ + Eterm* _real_bin = binary_val(Bin); \ Uint _offs = 0; \ Bitoffs = Bitsize = 0; \ if (*_real_bin == HEADER_SUB_BIN) { \ @@ -108,7 +104,7 @@ do { \ _offs = _sb->offs; \ Bitoffs = _sb->bitoffs; \ Bitsize = _sb->bitsize; \ - _real_bin = binary_val_rel(_sb->orig,BasePtr); \ + _real_bin = binary_val(_sb->orig); \ } \ if (*_real_bin == HEADER_PROC_BIN) { \ Bytep = ((ProcBin *) _real_bin)->bytes + _offs; \ @@ -131,11 +127,8 @@ do { \ */ #define ERTS_GET_REAL_BIN(Bin, RealBin, ByteOffset, BitOffset, BitSize) \ - ERTS_GET_REAL_BIN_REL(Bin, RealBin, ByteOffset, BitOffset, BitSize, NULL) - -#define ERTS_GET_REAL_BIN_REL(Bin, RealBin, ByteOffset, BitOffset, BitSize, BasePtr) \ do { \ - ErlSubBin* _sb = (ErlSubBin *) binary_val_rel(Bin,BasePtr); \ + ErlSubBin* _sb = (ErlSubBin *) binary_val(Bin); \ if (_sb->thing_word == HEADER_SUB_BIN) { \ RealBin = _sb->orig; \ ByteOffset = _sb->offs; \ diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c index 01734c55d7..11d83686a3 100644 --- a/erts/emulator/beam/erl_bits.c +++ b/erts/emulator/beam/erl_bits.c @@ -282,7 +282,7 @@ erts_bs_get_integer_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuff * Simply shift whole bytes into the result. */ switch (BYTE_OFFSET(n)) { -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) case 7: w = (w << 8) | *bp++; case 6: w = (w << 8) | *bp++; case 5: w = (w << 8) | *bp++; @@ -387,7 +387,7 @@ erts_bs_get_integer_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuff case 3: v32 = LSB[0] + (LSB[1]<<8) + (LSB[2]<<16); goto big_small; -#if !defined(ARCH_64) || HALFWORD_HEAP +#if !defined(ARCH_64) case 4: v32 = (LSB[0] + (LSB[1]<<8) + (LSB[2]<<16) + (LSB[3]<<24)); if (!IS_USMALL(sgn, v32)) { diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index 878ee32b47..9ec14ab5ae 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -1774,15 +1774,9 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1) * (it looks like an continuation pointer), but that is will crash the * emulator if this BIF is call traced. */ -#if HALFWORD_HEAP - Eterm *hp = HAlloc(BIF_P, 3); - hp[0] = make_pos_bignum_header(2); - *((UWord *) (UWord) (hp+1)) = (UWord) tb; -#else Eterm *hp = HAlloc(BIF_P, 2); hp[0] = make_pos_bignum_header(1); hp[1] = (Eterm) tb; -#endif BIF_TRAP1(&ets_delete_continue_exp, BIF_P, make_big(hp)); } else { @@ -2840,7 +2834,7 @@ BIF_RETTYPE ets_match_spec_run_r_3(BIF_ALIST_3) BIF_TRAP3(bif_export[BIF_ets_match_spec_run_r_3], BIF_P,lst,BIF_ARG_2,ret); } - res = db_prog_match(BIF_P, mp, CAR(list_val(lst)), NULL, NULL, 0, + res = db_prog_match(BIF_P, mp, CAR(list_val(lst)), NULL, 0, ERTS_PAM_COPY_RESULT, &dummy); if (is_value(res)) { hp = HAlloc(BIF_P, 2); @@ -3652,11 +3646,8 @@ static BIF_RETTYPE ets_delete_trap(BIF_ALIST_1) Eterm* ptr = big_val(cont); DbTable *tb = *((DbTable **) (UWord) (ptr + 1)); -#if HALFWORD_HEAP - ASSERT(*ptr == make_pos_bignum_header(2)); -#else ASSERT(*ptr == make_pos_bignum_header(1)); -#endif + db_lock(tb, LCK_WRITE); trap = free_table_cont(p, tb, 0, 1); db_unlock(tb, LCK_WRITE); diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 98a2e2842a..cff65f244d 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -469,9 +469,6 @@ static ERTS_INLINE void try_shrink(DbTableHash* tb) } } -#define EQ_REL(x,y,y_base) \ - (is_same(x,NULL,y,y_base) || (is_not_both_immed((x),(y)) && eq_rel((x),NULL,(y),y_base))) - /* Is this a live object (not pseodo-deleted) with the specified key? */ static ERTS_INLINE int has_live_key(DbTableHash* tb, HashDbTerm* b, @@ -481,7 +478,7 @@ static ERTS_INLINE int has_live_key(DbTableHash* tb, HashDbTerm* b, else { Eterm itemKey = GETKEY(tb, b->dbterm.tpl); ASSERT(!is_header(itemKey)); - return EQ_REL(key, itemKey, b->dbterm.tpl); + return EQ(key, itemKey); } } @@ -494,7 +491,7 @@ static ERTS_INLINE int has_key(DbTableHash* tb, HashDbTerm* b, else { Eterm itemKey = GETKEY(tb, b->dbterm.tpl); ASSERT(!is_header(itemKey)); - return EQ_REL(key, itemKey, b->dbterm.tpl); + return EQ(key, itemKey); } } @@ -2204,11 +2201,11 @@ static void db_print_hash(int to, void *to_arg, int show, DbTable *tbl) erts_print(to, to_arg, "*"); if (tb->common.compress) { Eterm key = GETKEY(tb, list->dbterm.tpl); - erts_print(to, to_arg, "key=%R", key, list->dbterm.tpl); + erts_print(to, to_arg, "key=%T", key); } else { - Eterm obj = make_tuple_rel(list->dbterm.tpl,list->dbterm.tpl); - erts_print(to, to_arg, "%R", obj, list->dbterm.tpl); + Eterm obj = make_tuple(list->dbterm.tpl); + erts_print(to, to_arg, "%T", obj); } if (list->next != 0) erts_print(to, to_arg, ","); @@ -2899,9 +2896,6 @@ Ldone: handle->dbterm = &b->dbterm; handle->flags = flags; handle->new_size = b->dbterm.size; -#if HALFWORD_HEAP - handle->abs_vec = NULL; -#endif handle->lck = lck; return 1; } diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c index 465aa566ad..311b69d114 100644 --- a/erts/emulator/beam/erl_db_tree.c +++ b/erts/emulator/beam/erl_db_tree.c @@ -280,7 +280,7 @@ struct select_delete_context { /* ** Forward declarations */ -static TreeDbTerm *linkout_tree(DbTableTree *tb, Eterm key, Eterm* key_base); +static TreeDbTerm *linkout_tree(DbTableTree *tb, Eterm key); static TreeDbTerm *linkout_object_tree(DbTableTree *tb, Eterm object); static int do_free_tree_cont(DbTableTree *tb, int num_left); @@ -291,15 +291,15 @@ static int delsub(TreeDbTerm **this); static TreeDbTerm *slot_search(Process *p, DbTableTree *tb, Sint slot); static TreeDbTerm *find_node(DbTableTree *tb, Eterm key); static TreeDbTerm **find_node2(DbTableTree *tb, Eterm key); -static TreeDbTerm *find_next(DbTableTree *tb, DbTreeStack*, Eterm key, Eterm* kbase); -static TreeDbTerm *find_prev(DbTableTree *tb, DbTreeStack*, Eterm key, Eterm* kbase); +static TreeDbTerm *find_next(DbTableTree *tb, DbTreeStack*, Eterm key); +static TreeDbTerm *find_prev(DbTableTree *tb, DbTreeStack*, Eterm key); static TreeDbTerm *find_next_from_pb_key(DbTableTree *tb, DbTreeStack*, Eterm key); static TreeDbTerm *find_prev_from_pb_key(DbTableTree *tb, DbTreeStack*, Eterm key); static void traverse_backwards(DbTableTree *tb, DbTreeStack*, - Eterm lastkey, Eterm* lk_base, + Eterm lastkey, int (*doit)(DbTableTree *tb, TreeDbTerm *, void *, @@ -307,7 +307,7 @@ static void traverse_backwards(DbTableTree *tb, void *context); static void traverse_forward(DbTableTree *tb, DbTreeStack*, - Eterm lastkey, Eterm* lk_base, + Eterm lastkey, int (*doit)(DbTableTree *tb, TreeDbTerm *, void *, @@ -315,8 +315,8 @@ static void traverse_forward(DbTableTree *tb, void *context); static int key_given(DbTableTree *tb, Eterm pattern, TreeDbTerm **ret, Eterm *partly_bound_key); -static Sint cmp_partly_bound(Eterm partly_bound_key, Eterm bound_key, Eterm* bk_base); -static Sint do_cmp_partly_bound(Eterm a, Eterm b, Eterm* b_base, int *done); +static Sint cmp_partly_bound(Eterm partly_bound_key, Eterm bound_key); +static Sint do_cmp_partly_bound(Eterm a, Eterm b, int *done); static int analyze_pattern(DbTableTree *tb, Eterm pattern, struct mp_info *mpi); @@ -517,7 +517,7 @@ static int db_next_tree(Process *p, DbTable *tbl, Eterm key, Eterm *ret) if (is_atom(key) && key == am_EOT) return DB_ERROR_BADKEY; stack = get_any_stack(tb); - this = find_next(tb, stack, key, NULL); + this = find_next(tb, stack, key); release_stack(tb,stack); if (this == NULL) { *ret = am_EOT; @@ -563,7 +563,7 @@ static int db_prev_tree(Process *p, DbTable *tbl, Eterm key, Eterm *ret) if (is_atom(key) && key == am_EOT) return DB_ERROR_BADKEY; stack = get_any_stack(tb); - this = find_prev(tb, stack, key, NULL); + this = find_prev(tb, stack, key); release_stack(tb,stack); if (this == NULL) { *ret = am_EOT; @@ -573,19 +573,13 @@ static int db_prev_tree(Process *p, DbTable *tbl, Eterm key, Eterm *ret) return DB_ERROR_NONE; } -static ERTS_INLINE Sint cmp_key(DbTableTree* tb, Eterm key, Eterm* key_base, - TreeDbTerm* obj) -{ - return cmp_rel(key, key_base, - GETKEY(tb,obj->dbterm.tpl), obj->dbterm.tpl); +static ERTS_INLINE Sint cmp_key(DbTableTree* tb, Eterm key, TreeDbTerm* obj) { + return CMP(key, GETKEY(tb,obj->dbterm.tpl)); } -static ERTS_INLINE int cmp_key_eq(DbTableTree* tb, Eterm key, Eterm* key_base, - TreeDbTerm* obj) -{ +static ERTS_INLINE int cmp_key_eq(DbTableTree* tb, Eterm key, TreeDbTerm* obj) { Eterm obj_key = GETKEY(tb,obj->dbterm.tpl); - return is_same(key, key_base, obj_key, obj->dbterm.tpl) - || cmp_rel(key, key_base, obj_key, obj->dbterm.tpl) == 0; + return is_same(key, obj_key) || CMP(key, obj_key) == 0; } static int db_put_tree(DbTable *tbl, Eterm obj, int key_clash_fail) @@ -619,7 +613,7 @@ static int db_put_tree(DbTable *tbl, Eterm obj, int key_clash_fail) (*this)->balance = 0; (*this)->left = (*this)->right = NULL; break; - } else if ((c = cmp_key(tb, key, NULL, *this)) < 0) { + } else if ((c = cmp_key(tb, key, *this)) < 0) { /* go lefts */ dstack[dpos++] = DIR_LEFT; tstack[tpos++] = this; @@ -774,7 +768,7 @@ static int db_erase_tree(DbTable *tbl, Eterm key, Eterm *ret) *ret = am_true; - if ((res = linkout_tree(tb, key, NULL)) != NULL) { + if ((res = linkout_tree(tb, key)) != NULL) { free_term(tb, res); } return DB_ERROR_NONE; @@ -970,15 +964,15 @@ static int db_select_continue_tree(Process *p, stack = get_any_stack(tb); if (chunk_size) { if (reverse) { - traverse_backwards(tb, stack, lastkey, NULL, &doit_select_chunk, &sc); + traverse_backwards(tb, stack, lastkey, &doit_select_chunk, &sc); } else { - traverse_forward(tb, stack, lastkey, NULL, &doit_select_chunk, &sc); + traverse_forward(tb, stack, lastkey, &doit_select_chunk, &sc); } } else { if (reverse) { - traverse_forward(tb, stack, lastkey, NULL, &doit_select, &sc); + traverse_forward(tb, stack, lastkey, &doit_select, &sc); } else { - traverse_backwards(tb, stack, lastkey, NULL, &doit_select, &sc); + traverse_backwards(tb, stack, lastkey, &doit_select, &sc); } } release_stack(tb,stack); @@ -1003,9 +997,9 @@ static int db_select_continue_tree(Process *p, } key = GETKEY(tb, sc.lastobj); - sz = size_object_rel(key,sc.lastobj); + sz = size_object(key); hp = HAlloc(p, 9 + sz); - key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL); + key = copy_struct(key, sz, &hp, &MSO(p)); continuation = TUPLE8 (hp, tptr[1], @@ -1026,8 +1020,8 @@ static int db_select_continue_tree(Process *p, key = GETKEY(tb, sc.lastobj); if (chunk_size) { if (end_condition != NIL && - ((!reverse && cmp_partly_bound(end_condition,key,sc.lastobj) < 0) || - (reverse && cmp_partly_bound(end_condition,key,sc.lastobj) > 0))) { + ((!reverse && cmp_partly_bound(end_condition,key) < 0) || + (reverse && cmp_partly_bound(end_condition,key) > 0))) { /* done anyway */ if (!sc.got) { RET_TO_BIF(am_EOT, DB_ERROR_NONE); @@ -1039,16 +1033,16 @@ static int db_select_continue_tree(Process *p, } } else { if (end_condition != NIL && - ((!reverse && cmp_partly_bound(end_condition,key,sc.lastobj) > 0) || - (reverse && cmp_partly_bound(end_condition,key,sc.lastobj) < 0))) { + ((!reverse && cmp_partly_bound(end_condition,key) > 0) || + (reverse && cmp_partly_bound(end_condition,key) < 0))) { /* done anyway */ RET_TO_BIF(sc.accum,DB_ERROR_NONE); } } /* Not done yet, let's trap. */ - sz = size_object_rel(key,sc.lastobj); + sz = size_object(key); hp = HAlloc(p, 9 + sz); - key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL); + key = copy_struct(key, sz, &hp, &MSO(p)); continuation = TUPLE8 (hp, tptr[1], @@ -1075,7 +1069,6 @@ static int db_select_tree(Process *p, DbTable *tbl, struct select_context sc; struct mp_info mpi; Eterm lastkey = THE_NON_VALUE; - Eterm* lk_base = NULL; Eterm key; Eterm continuation; unsigned sz; @@ -1127,20 +1120,18 @@ static int db_select_tree(Process *p, DbTable *tbl, if (mpi.some_limitation) { if ((this = find_prev_from_pb_key(tb, stack, mpi.least)) != NULL) { lastkey = GETKEY(tb, this->dbterm.tpl); - lk_base = this->dbterm.tpl; } sc.end_condition = mpi.most; } - traverse_forward(tb, stack, lastkey, lk_base, &doit_select, &sc); + traverse_forward(tb, stack, lastkey, &doit_select, &sc); } else { if (mpi.some_limitation) { if ((this = find_next_from_pb_key(tb, stack, mpi.most)) != NULL) { lastkey = GETKEY(tb, this->dbterm.tpl); - lk_base = this->dbterm.tpl; } sc.end_condition = mpi.least; } - traverse_backwards(tb, stack, lastkey, lk_base, &doit_select, &sc); + traverse_backwards(tb, stack, lastkey, &doit_select, &sc); } release_stack(tb,stack); #ifdef HARDDEBUG @@ -1153,9 +1144,9 @@ static int db_select_tree(Process *p, DbTable *tbl, } key = GETKEY(tb, sc.lastobj); - sz = size_object_rel(key, sc.lastobj); + sz = size_object(key); hp = HAlloc(p, 9 + sz + PROC_BIN_SIZE); - key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL); + key = copy_struct(key, sz, &hp, &MSO(p)); if (mpi.all_objects) (mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS; mpb=db_make_mp_binary(p,mpi.mp,&hp); @@ -1236,7 +1227,7 @@ static int db_select_count_continue_tree(Process *p, } stack = get_any_stack(tb); - traverse_backwards(tb, stack, lastkey, NULL, &doit_select_count, &sc); + traverse_backwards(tb, stack, lastkey, &doit_select_count, &sc); release_stack(tb,stack); BUMP_REDS(p, 1000 - sc.max); @@ -1246,12 +1237,12 @@ static int db_select_count_continue_tree(Process *p, } key = GETKEY(tb, sc.lastobj); if (end_condition != NIL && - (cmp_partly_bound(end_condition,key,sc.lastobj) > 0)) { + (cmp_partly_bound(end_condition,key) > 0)) { /* done anyway */ RET_TO_BIF(make_small(sc.got),DB_ERROR_NONE); } /* Not done yet, let's trap. */ - sz = size_object_rel(key, sc.lastobj); + sz = size_object(key); if (IS_USMALL(0, sc.got)) { hp = HAlloc(p, sz + 6); egot = make_small(sc.got); @@ -1261,7 +1252,7 @@ static int db_select_count_continue_tree(Process *p, egot = uint_to_big(sc.got, hp); hp += BIG_UINT_HEAP_SIZE; } - key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL); + key = copy_struct(key, sz, &hp, &MSO(p)); continuation = TUPLE5 (hp, tptr[1], @@ -1284,7 +1275,6 @@ static int db_select_count_tree(Process *p, DbTable *tbl, struct select_count_context sc; struct mp_info mpi; Eterm lastkey = THE_NON_VALUE; - Eterm* lk_base = NULL; Eterm key; Eterm continuation; unsigned sz; @@ -1334,12 +1324,11 @@ static int db_select_count_tree(Process *p, DbTable *tbl, if (mpi.some_limitation) { if ((this = find_next_from_pb_key(tb, stack, mpi.most)) != NULL) { lastkey = GETKEY(tb, this->dbterm.tpl); - lk_base = this->dbterm.tpl; } sc.end_condition = mpi.least; } - traverse_backwards(tb, stack, lastkey, lk_base, &doit_select_count, &sc); + traverse_backwards(tb, stack, lastkey, &doit_select_count, &sc); release_stack(tb,stack); BUMP_REDS(p, 1000 - sc.max); if (sc.max > 0) { @@ -1347,7 +1336,7 @@ static int db_select_count_tree(Process *p, DbTable *tbl, } key = GETKEY(tb, sc.lastobj); - sz = size_object_rel(key, sc.lastobj); + sz = size_object(key); if (IS_USMALL(0, sc.got)) { hp = HAlloc(p, sz + PROC_BIN_SIZE + 6); egot = make_small(sc.got); @@ -1357,7 +1346,7 @@ static int db_select_count_tree(Process *p, DbTable *tbl, egot = uint_to_big(sc.got, hp); hp += BIG_UINT_HEAP_SIZE; } - key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL); + key = copy_struct(key, sz, &hp, &MSO(p)); if (mpi.all_objects) (mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS; mpb = db_make_mp_binary(p,mpi.mp,&hp); @@ -1388,7 +1377,6 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl, struct select_context sc; struct mp_info mpi; Eterm lastkey = THE_NON_VALUE; - Eterm* lk_base = NULL; Eterm key; Eterm continuation; unsigned sz; @@ -1445,20 +1433,18 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl, if (mpi.some_limitation) { if ((this = find_next_from_pb_key(tb, stack, mpi.most)) != NULL) { lastkey = GETKEY(tb, this->dbterm.tpl); - lk_base = this->dbterm.tpl; } sc.end_condition = mpi.least; } - traverse_backwards(tb, stack, lastkey, lk_base, &doit_select_chunk, &sc); + traverse_backwards(tb, stack, lastkey, &doit_select_chunk, &sc); } else { if (mpi.some_limitation) { if ((this = find_prev_from_pb_key(tb, stack, mpi.least)) != NULL) { lastkey = GETKEY(tb, this->dbterm.tpl); - lk_base = this->dbterm.tpl; } sc.end_condition = mpi.most; } - traverse_forward(tb, stack, lastkey, lk_base, &doit_select_chunk, &sc); + traverse_forward(tb, stack, lastkey, &doit_select_chunk, &sc); } release_stack(tb,stack); @@ -1483,9 +1469,9 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl, } key = GETKEY(tb, sc.lastobj); - sz = size_object_rel(key, sc.lastobj); + sz = size_object(key); hp = HAlloc(p, 9 + sz + PROC_BIN_SIZE); - key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL); + key = copy_struct(key, sz, &hp, &MSO(p)); if (mpi.all_objects) (mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS; mpb = db_make_mp_binary(p,mpi.mp,&hp); @@ -1508,9 +1494,9 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl, } key = GETKEY(tb, sc.lastobj); - sz = size_object_rel(key, sc.lastobj); + sz = size_object(key); hp = HAlloc(p, 9 + sz + PROC_BIN_SIZE); - key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL); + key = copy_struct(key, sz, &hp, &MSO(p)); if (mpi.all_objects) (mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS; @@ -1586,7 +1572,7 @@ static int db_select_delete_continue_tree(Process *p, sc.keypos = tb->common.keypos; ASSERT(!erts_smp_atomic_read_nob(&tb->is_stack_busy)); - traverse_backwards(tb, &tb->static_stack, lastkey, NULL, &doit_select_delete, &sc); + traverse_backwards(tb, &tb->static_stack, lastkey, &doit_select_delete, &sc); BUMP_REDS(p, 1000 - sc.max); @@ -1595,11 +1581,11 @@ static int db_select_delete_continue_tree(Process *p, } key = GETKEY(tb, (sc.lastterm)->dbterm.tpl); if (end_condition != NIL && - cmp_partly_bound(end_condition,key,sc.lastterm->dbterm.tpl) > 0) { /* done anyway */ + cmp_partly_bound(end_condition,key) > 0) { /* done anyway */ RET_TO_BIF(erts_make_integer(sc.accum,p),DB_ERROR_NONE); } /* Not done yet, let's trap. */ - sz = size_object_rel(key, sc.lastterm->dbterm.tpl); + sz = size_object(key); if (IS_USMALL(0, sc.accum)) { hp = HAlloc(p, sz + 6); eaccsum = make_small(sc.accum); @@ -1609,7 +1595,7 @@ static int db_select_delete_continue_tree(Process *p, eaccsum = uint_to_big(sc.accum, hp); hp += BIG_UINT_HEAP_SIZE; } - key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastterm->dbterm.tpl, NULL); + key = copy_struct(key, sz, &hp, &MSO(p)); continuation = TUPLE5 (hp, tptr[1], @@ -1630,7 +1616,6 @@ static int db_select_delete_tree(Process *p, DbTable *tbl, struct select_delete_context sc; struct mp_info mpi; Eterm lastkey = THE_NON_VALUE; - Eterm* lk_base = NULL; Eterm key; Eterm continuation; unsigned sz; @@ -1683,12 +1668,11 @@ static int db_select_delete_tree(Process *p, DbTable *tbl, if (mpi.some_limitation) { if ((this = find_next_from_pb_key(tb, &tb->static_stack, mpi.most)) != NULL) { lastkey = GETKEY(tb, this->dbterm.tpl); - lk_base = this->dbterm.tpl; } sc.end_condition = mpi.least; } - traverse_backwards(tb, &tb->static_stack, lastkey, lk_base, &doit_select_delete, &sc); + traverse_backwards(tb, &tb->static_stack, lastkey, &doit_select_delete, &sc); BUMP_REDS(p, 1000 - sc.max); if (sc.max > 0) { @@ -1696,7 +1680,7 @@ static int db_select_delete_tree(Process *p, DbTable *tbl, } key = GETKEY(tb, (sc.lastterm)->dbterm.tpl); - sz = size_object_rel(key, sc.lastterm->dbterm.tpl); + sz = size_object(key); if (IS_USMALL(0, sc.accum)) { hp = HAlloc(p, sz + PROC_BIN_SIZE + 6); eaccsum = make_small(sc.accum); @@ -1706,7 +1690,7 @@ static int db_select_delete_tree(Process *p, DbTable *tbl, eaccsum = uint_to_big(sc.accum, hp); hp += BIG_UINT_HEAP_SIZE; } - key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastterm->dbterm.tpl, NULL); + key = copy_struct(key, sz, &hp, &MSO(p)); mpb = db_make_mp_binary(p,mpi.mp,&hp); continuation = TUPLE5 @@ -1734,7 +1718,7 @@ static int db_take_tree(Process *p, DbTable *tbl, Eterm key, Eterm *ret) TreeDbTerm *this; *ret = NIL; - this = linkout_tree(tb, key, NULL); + this = linkout_tree(tb, key); if (this) { Eterm copy, *hp, *hend; @@ -1845,9 +1829,7 @@ do_db_tree_foreach_offheap(TreeDbTerm *tdbt, do_db_tree_foreach_offheap(tdbt->right, func, arg); } -static TreeDbTerm *linkout_tree(DbTableTree *tb, - Eterm key, Eterm* key_base) -{ +static TreeDbTerm *linkout_tree(DbTableTree *tb, Eterm key) { TreeDbTerm **tstack[STACK_NEED]; int tpos = 0; int dstack[STACK_NEED+1]; @@ -1869,7 +1851,7 @@ static TreeDbTerm *linkout_tree(DbTableTree *tb, for (;;) { if (!*this) { /* Failure */ return NULL; - } else if ((c = cmp_key(tb, key, key_base, *this)) < 0) { + } else if ((c = cmp_key(tb, key, *this)) < 0) { dstack[dpos++] = DIR_LEFT; tstack[tpos++] = this; this = &((*this)->left); @@ -1933,7 +1915,7 @@ static TreeDbTerm *linkout_object_tree(DbTableTree *tb, for (;;) { if (!*this) { /* Failure */ return NULL; - } else if ((c = cmp_key(tb,key,NULL,*this)) < 0) { + } else if ((c = cmp_key(tb,key,*this)) < 0) { dstack[dpos++] = DIR_LEFT; tstack[tpos++] = this; this = &((*this)->left); @@ -2319,15 +2301,13 @@ done: * Find next and previous in sort order */ -static TreeDbTerm *find_next(DbTableTree *tb, DbTreeStack* stack, - Eterm key, Eterm* key_base) -{ +static TreeDbTerm *find_next(DbTableTree *tb, DbTreeStack* stack, Eterm key) { TreeDbTerm *this; TreeDbTerm *tmp; Sint c; if(( this = TOP_NODE(stack)) != NULL) { - if (!cmp_key_eq(tb,key,key_base,this)) { + if (!cmp_key_eq(tb,key,this)) { /* Start from the beginning */ stack->pos = stack->slot = 0; } @@ -2337,7 +2317,7 @@ static TreeDbTerm *find_next(DbTableTree *tb, DbTreeStack* stack, return NULL; for (;;) { PUSH_NODE(stack, this); - if (( c = cmp_key(tb,key,key_base,this) ) > 0) { + if (( c = cmp_key(tb,key,this) ) > 0) { if (this->right == NULL) /* We are at the previos and the element does not exist */ @@ -2377,15 +2357,13 @@ static TreeDbTerm *find_next(DbTableTree *tb, DbTreeStack* stack, return this; } -static TreeDbTerm *find_prev(DbTableTree *tb, DbTreeStack* stack, - Eterm key, Eterm* key_base) -{ +static TreeDbTerm *find_prev(DbTableTree *tb, DbTreeStack* stack, Eterm key) { TreeDbTerm *this; TreeDbTerm *tmp; Sint c; if(( this = TOP_NODE(stack)) != NULL) { - if (!cmp_key_eq(tb,key,key_base,this)) { + if (!cmp_key_eq(tb,key,this)) { /* Start from the beginning */ stack->pos = stack->slot = 0; } @@ -2395,7 +2373,7 @@ static TreeDbTerm *find_prev(DbTableTree *tb, DbTreeStack* stack, return NULL; for (;;) { PUSH_NODE(stack, this); - if (( c = cmp_key(tb,key,key_base,this) ) < 0) { + if (( c = cmp_key(tb,key,this) ) < 0) { if (this->left == NULL) /* We are at the next and the element does not exist */ @@ -2448,8 +2426,7 @@ static TreeDbTerm *find_next_from_pb_key(DbTableTree *tb, DbTreeStack* stack, return NULL; for (;;) { PUSH_NODE(stack, this); - if (( c = cmp_partly_bound(key,GETKEY(tb, this->dbterm.tpl), - this->dbterm.tpl) ) >= 0) { + if (( c = cmp_partly_bound(key,GETKEY(tb, this->dbterm.tpl))) >= 0) { if (this->right == NULL) { do { tmp = POP_NODE(stack); @@ -2482,8 +2459,7 @@ static TreeDbTerm *find_prev_from_pb_key(DbTableTree *tb, DbTreeStack* stack, return NULL; for (;;) { PUSH_NODE(stack, this); - if (( c = cmp_partly_bound(key,GETKEY(tb, this->dbterm.tpl), - this->dbterm.tpl) ) <= 0) { + if (( c = cmp_partly_bound(key,GETKEY(tb, this->dbterm.tpl))) <= 0) { if (this->left == NULL) { do { tmp = POP_NODE(stack); @@ -2514,10 +2490,10 @@ static TreeDbTerm *find_node(DbTableTree *tb, Eterm key) DbTreeStack* stack = get_static_stack(tb); if(!stack || EMPTY_NODE(stack) - || !cmp_key_eq(tb, key, NULL, (this=TOP_NODE(stack)))) { + || !cmp_key_eq(tb, key, (this=TOP_NODE(stack)))) { this = tb->root; - while (this != NULL && (res = cmp_key(tb,key,NULL,this)) != 0) { + while (this != NULL && (res = cmp_key(tb,key,this)) != 0) { if (res < 0) this = this->left; else @@ -2539,7 +2515,7 @@ static TreeDbTerm **find_node2(DbTableTree *tb, Eterm key) Sint res; this = &tb->root; - while ((*this) != NULL && (res = cmp_key(tb, key, NULL, *this)) != 0) { + while ((*this) != NULL && (res = cmp_key(tb, key, *this)) != 0) { if (res < 0) this = &((*this)->left); else @@ -2589,9 +2565,6 @@ db_lookup_dbterm_tree(Process *p, DbTable *tbl, Eterm key, Eterm obj, handle->flags = flags; handle->bp = (void**) pp; handle->new_size = (*pp)->dbterm.size; -#if HALFWORD_HEAP - handle->abs_vec = NULL; -#endif return 1; } @@ -2622,7 +2595,7 @@ db_finalize_dbterm_tree(int cret, DbUpdateHandle *handle) */ static void traverse_backwards(DbTableTree *tb, DbTreeStack* stack, - Eterm lastkey, Eterm* lk_base, + Eterm lastkey, int (*doit)(DbTableTree *, TreeDbTerm *, void *, @@ -2641,16 +2614,15 @@ static void traverse_backwards(DbTableTree *tb, this = this->right; } this = TOP_NODE(stack); - next = find_prev(tb, stack, GETKEY(tb, this->dbterm.tpl), - this->dbterm.tpl); + next = find_prev(tb, stack, GETKEY(tb, this->dbterm.tpl)); if (!((*doit)(tb, this, context, 0))) return; } else { - next = find_prev(tb, stack, lastkey, lk_base); + next = find_prev(tb, stack, lastkey); } while ((this = next) != NULL) { - next = find_prev(tb, stack, GETKEY(tb, this->dbterm.tpl), this->dbterm.tpl); + next = find_prev(tb, stack, GETKEY(tb, this->dbterm.tpl)); if (!((*doit)(tb, this, context, 0))) return; } @@ -2661,7 +2633,7 @@ static void traverse_backwards(DbTableTree *tb, */ static void traverse_forward(DbTableTree *tb, DbTreeStack* stack, - Eterm lastkey, Eterm* lk_base, + Eterm lastkey, int (*doit)(DbTableTree *, TreeDbTerm *, void *, @@ -2680,15 +2652,15 @@ static void traverse_forward(DbTableTree *tb, this = this->left; } this = TOP_NODE(stack); - next = find_next(tb, stack, GETKEY(tb, this->dbterm.tpl), this->dbterm.tpl); + next = find_next(tb, stack, GETKEY(tb, this->dbterm.tpl)); if (!((*doit)(tb, this, context, 1))) return; } else { - next = find_next(tb, stack, lastkey, lk_base); + next = find_next(tb, stack, lastkey); } while ((this = next) != NULL) { - next = find_next(tb, stack, GETKEY(tb, this->dbterm.tpl), this->dbterm.tpl); + next = find_next(tb, stack, GETKEY(tb, this->dbterm.tpl)); if (!((*doit)(tb, this, context, 1))) return; } @@ -2725,7 +2697,7 @@ static int key_given(DbTableTree *tb, Eterm pattern, TreeDbTerm **ret, -static Sint do_cmp_partly_bound(Eterm a, Eterm b, Eterm* b_base, int *done) +static Sint do_cmp_partly_bound(Eterm a, Eterm b, int *done) { Eterm* aa; Eterm* bb; @@ -2739,44 +2711,44 @@ static Sint do_cmp_partly_bound(Eterm a, Eterm b, Eterm* b_base, int *done) *done = 1; return 0; } - if (is_same(a,NULL,b,b_base)) + if (is_same(a,b)) return 0; switch (a & _TAG_PRIMARY_MASK) { case TAG_PRIMARY_LIST: if (!is_list(b)) { - return cmp_rel(a,NULL,b,b_base); + return CMP(a,b); } aa = list_val(a); - bb = list_val_rel(b,b_base); + bb = list_val(b); while (1) { - if ((j = do_cmp_partly_bound(*aa++, *bb++, b_base, done)) != 0 || *done) + if ((j = do_cmp_partly_bound(*aa++, *bb++, done)) != 0 || *done) return j; - if (is_same(*aa, NULL, *bb, b_base)) + if (is_same(*aa, *bb)) return 0; if (is_not_list(*aa) || is_not_list(*bb)) - return do_cmp_partly_bound(*aa, *bb, b_base, done); + return do_cmp_partly_bound(*aa, *bb, done); aa = list_val(*aa); - bb = list_val_rel(*bb,b_base); + bb = list_val(*bb); } case TAG_PRIMARY_BOXED: if ((b & _TAG_PRIMARY_MASK) != TAG_PRIMARY_BOXED) { - return cmp_rel(a,NULL,b,b_base); + return CMP(a,b); } a_hdr = ((*boxed_val(a)) & _TAG_HEADER_MASK) >> _TAG_PRIMARY_SIZE; - b_hdr = ((*boxed_val_rel(b,b_base)) & _TAG_HEADER_MASK) >> _TAG_PRIMARY_SIZE; + b_hdr = ((*boxed_val(b)) & _TAG_HEADER_MASK) >> _TAG_PRIMARY_SIZE; if (a_hdr != b_hdr) { - return cmp_rel(a, NULL, b, b_base); + return CMP(a,b); } if (a_hdr == (_TAG_HEADER_ARITYVAL >> _TAG_PRIMARY_SIZE)) { aa = tuple_val(a); - bb = tuple_val_rel(b, b_base); + bb = tuple_val(b); /* compare the arities */ i = arityval(*aa); /* get the arity*/ if (i < arityval(*bb)) return(-1); if (i > arityval(*bb)) return(1); while (i--) { - if ((j = do_cmp_partly_bound(*++aa, *++bb, b_base, done)) != 0 + if ((j = do_cmp_partly_bound(*++aa, *++bb, done)) != 0 || *done) return j; } @@ -2784,14 +2756,13 @@ static Sint do_cmp_partly_bound(Eterm a, Eterm b, Eterm* b_base, int *done) } /* Drop through */ default: - return cmp_rel(a, NULL, b, b_base); + return CMP(a,b); } } -static Sint cmp_partly_bound(Eterm partly_bound_key, Eterm bound_key, Eterm* bk_base) -{ +static Sint cmp_partly_bound(Eterm partly_bound_key, Eterm bound_key) { int done = 0; - Sint ret = do_cmp_partly_bound(partly_bound_key, bound_key, bk_base, &done); + Sint ret = do_cmp_partly_bound(partly_bound_key, bound_key, &done); #ifdef HARDDEBUG erts_fprintf(stderr,"\ncmp_partly_bound: %T", partly_bound_key); if (ret < 0) @@ -2800,7 +2771,7 @@ static Sint cmp_partly_bound(Eterm partly_bound_key, Eterm bound_key, Eterm* bk_ erts_fprintf(stderr," > "); else erts_fprintf(stderr," == "); - erts_fprintf(stderr,"%R\n", bound_key, bk_base); + erts_fprintf(stderr,"%T\n", bound_key); #endif return ret; } @@ -3017,12 +2988,10 @@ static int doit_select(DbTableTree *tb, TreeDbTerm *this, void *ptr, if (sc->end_condition != NIL && ((forward && cmp_partly_bound(sc->end_condition, - GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl), - this->dbterm.tpl) < 0) || + GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl)) < 0) || (!forward && cmp_partly_bound(sc->end_condition, - GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl), - this->dbterm.tpl) > 0))) { + GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl)) > 0))) { return 0; } ret = db_match_dbterm(&tb->common,sc->p,sc->mp,sc->all_objects, @@ -3054,8 +3023,7 @@ static int doit_select_count(DbTableTree *tb, TreeDbTerm *this, void *ptr, /* Always backwards traversing */ if (sc->end_condition != NIL && (cmp_partly_bound(sc->end_condition, - GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl), - this->dbterm.tpl) > 0)) { + GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl)) > 0)) { return 0; } ret = db_match_dbterm(&tb->common, sc->p, sc->mp, 0, @@ -3081,12 +3049,10 @@ static int doit_select_chunk(DbTableTree *tb, TreeDbTerm *this, void *ptr, if (sc->end_condition != NIL && ((forward && cmp_partly_bound(sc->end_condition, - GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl), - this->dbterm.tpl) < 0) || + GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl)) < 0) || (!forward && cmp_partly_bound(sc->end_condition, - GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl), - this->dbterm.tpl) > 0))) { + GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl)) > 0))) { return 0; } @@ -3124,14 +3090,13 @@ static int doit_select_delete(DbTableTree *tb, TreeDbTerm *this, void *ptr, if (sc->end_condition != NIL && cmp_partly_bound(sc->end_condition, - GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl), - this->dbterm.tpl) > 0) + GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl)) > 0) return 0; ret = db_match_dbterm(&tb->common, sc->p, sc->mp, 0, &this->dbterm, NULL, 0); if (ret == am_true) { key = GETKEY(sc->tb, this->dbterm.tpl); - linkout_tree(sc->tb, key, this->dbterm.tpl); + linkout_tree(sc->tb, key); sc->erase_lastterm = 1; ++sc->accum; } @@ -3157,11 +3122,10 @@ static void do_dump_tree2(DbTableTree* tb, int to, void *to_arg, int show, } else { prefix = ""; - term = make_tuple_rel(t->dbterm.tpl,t->dbterm.tpl); + term = make_tuple(t->dbterm.tpl); } - erts_print(to, to_arg, "%*s%s%R (addr = %p, bal = %d)\n", - offset, "", prefix, term, t->dbterm.tpl, - t, t->balance); + erts_print(to, to_arg, "%*s%s%T (addr = %p, bal = %d)\n", + offset, "", prefix, term, t, t->balance); } do_dump_tree2(tb, to, to_arg, show, t->left, offset + 4); } diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index dab357a079..0bf9558ac9 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -239,11 +239,7 @@ typedef enum { matchCall2, matchCall3, matchPushV, -#if HALFWORD_HEAP - matchPushVGuard, /* First guard-only variable reference */ -#endif - matchPushVResult, /* First variable reference in result, or (if HALFWORD) - in guard if also referenced in result */ + matchPushVResult, /* First variable reference in result */ matchPushExpr, /* Push the whole expression we're matching ('$_') */ matchPushArrayAsList, /* Only when parameter is an Array and not an erlang term (DCOMP_TRACE) */ @@ -310,9 +306,6 @@ DMC_DECLARE_STACK_TYPE(unsigned); typedef struct DMCVariable { int is_bound; int is_in_body; -#if HALFWORD_HEAP - int first_guard_label; /* to maybe change from PushVGuard to PushVResult */ -#endif } DMCVariable; typedef struct DMCHeap { @@ -374,7 +367,6 @@ typedef struct MatchVariable { Eterm term; #ifdef DEBUG Process* proc; - Eterm* base; #endif } MatchVariable; @@ -415,7 +407,7 @@ cleanup_match_pseudo_process(ErtsMatchPseudoProcess *mpsp, int keep_heap) else { int i; for (i = 0; i < ERTS_DEFAULT_MS_HEAP_SIZE; i++) { -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) mpsp->default_heap[i] = (Eterm) 0xdeadbeefdeadbeef; #else mpsp->default_heap[i] = (Eterm) 0xdeadbeef; @@ -1240,7 +1232,7 @@ Eterm erts_match_set_run(Process *p, Binary *mpsp, { Eterm ret; - ret = db_prog_match(p, mpsp, NIL, NULL, args, num_args, + ret = db_prog_match(p, mpsp, NIL, args, num_args, in_flags, return_flags); #if defined(HARDDEBUG) if (is_non_value(ret)) { @@ -1265,7 +1257,7 @@ static Eterm erts_match_set_run_ets(Process *p, Binary *mpsp, { Eterm ret; - ret = db_prog_match(p, mpsp, args, NULL, NULL, num_args, + ret = db_prog_match(p, mpsp, args, NULL, num_args, ERTS_PAM_COPY_RESULT, return_flags); #if defined(HARDDEBUG) @@ -1755,71 +1747,6 @@ static Eterm dpm_array_to_list(Process *psp, Eterm *arr, int arity) return ret; } - -#if HALFWORD_HEAP -struct heap_checkpoint_t -{ - Process *p; - Eterm* htop; - ErlHeapFragment* mbuf; - unsigned used_size; - ErlOffHeap off_heap; -}; - -static void heap_checkpoint_init(Process* p, struct heap_checkpoint_t* hcp) -{ - hcp->p = p; - hcp->htop = HEAP_TOP(p); - hcp->mbuf = MBUF(p); - hcp->used_size = hcp->mbuf ? hcp->mbuf->used_size : 0; - hcp->off_heap = MSO(p); -} - -static void heap_checkpoint_revert(struct heap_checkpoint_t* hcp) -{ - struct erl_off_heap_header* oh = MSO(hcp->p).first; - - if (oh != hcp->off_heap.first) { - ASSERT(oh != NULL); - if (hcp->off_heap.first) { - while (oh->next != hcp->off_heap.first) { - oh = oh->next; - } - oh->next = NULL; - } - erts_cleanup_offheap(&MSO(hcp->p)); - MSO(hcp->p) = hcp->off_heap; - } - if (MBUF(hcp->p) != hcp->mbuf) { - ErlHeapFragment* hf = MBUF(hcp->p); - ASSERT(hf != NULL); - if (hcp->mbuf) { - while (hf->next != hcp->mbuf) { - hf = hf->next; - } - hf->next = NULL; - } - free_message_buffer(MBUF(hcp->p)); - MBUF(hcp->p) = hcp->mbuf; - } - if (hcp->mbuf != NULL && hcp->mbuf->used_size != hcp->used_size) { - hcp->mbuf->used_size = hcp->used_size; - } - HEAP_TOP(hcp->p) = hcp->htop; -} -#endif /* HALFWORD_HEAP */ - -static ERTS_INLINE Eterm copy_object_rel(Process* p, Eterm term, Eterm* base) -{ - if (!is_immed(term)) { - Uint sz = size_object_rel(term, base); - Eterm* top = HAllocX(p, sz, HEAP_XTRA); - return copy_struct_rel(term, sz, &top, &MSO(p), base, NULL); - } - return term; -} - - /* ** Execution of the match program, this is Pam. ** May return THE_NON_VALUE, which is a bailout. @@ -1827,7 +1754,7 @@ static ERTS_INLINE Eterm copy_object_rel(Process* p, Eterm term, Eterm* base) ** i.e. 'DCOMP_TRACE' was specified */ Eterm db_prog_match(Process *c_p, Binary *bprog, - Eterm term, Eterm* base, + Eterm term, Eterm *termp, int arity, enum erts_pam_run_flags in_flags, @@ -1855,17 +1782,12 @@ Eterm db_prog_match(Process *c_p, Binary *bprog, Eterm bif_args[3]; int fail_label; int atomic_trace; -#if HALFWORD_HEAP - struct heap_checkpoint_t c_p_checkpoint = {}; -#endif #ifdef DMC_DEBUG Uint *heap_fence; Uint *stack_fence; Uint save_op; #endif /* DMC_DEBUG */ - ASSERT(base==NULL || HALFWORD_HEAP); - mpsp = get_match_pseudo_process(c_p, prog->heap_size); psp = &mpsp->process; @@ -1916,11 +1838,7 @@ Eterm db_prog_match(Process *c_p, Binary *bprog, do_catch != 0 */ *return_flags = 0U; - variables = mpsp->u.variables; -#if HALFWORD_HEAP - c_p_checkpoint.p = NULL; -#endif restart: ep = &term; @@ -1931,14 +1849,12 @@ restart: fail_label = -1; build_proc = psp; esdp->current_process = psp; - ASSERT_HALFWORD(!c_p_checkpoint.p); #ifdef DEBUG ASSERT(variables == mpsp->u.variables); for (i=0; i<prog->num_bindings; i++) { variables[i].term = THE_NON_VALUE; variables[i].proc = NULL; - variables[i].base = base; } #endif @@ -1974,9 +1890,9 @@ restart: variables[n].term = dpm_array_to_list(psp, termp, arity); break; case matchTuple: /* *ep is a tuple of arity n */ - if (!is_tuple_rel(*ep,base)) + if (!is_tuple(*ep)) FAIL(); - ep = tuple_val_rel(*ep,base); + ep = tuple_val(*ep); n = *pc++; if (arityval(*ep) != n) FAIL(); @@ -1984,9 +1900,9 @@ restart: break; case matchPushT: /* *ep is a tuple of arity n, push ptr to first element */ - if (!is_tuple_rel(*ep,base)) + if (!is_tuple(*ep)) FAIL(); - tp = tuple_val_rel(*ep,base); + tp = tuple_val(*ep); n = *pc++; if (arityval(*tp) != n) FAIL(); @@ -1996,51 +1912,51 @@ restart: case matchList: if (!is_list(*ep)) FAIL(); - ep = list_val_rel(*ep,base); + ep = list_val(*ep); break; case matchPushL: if (!is_list(*ep)) FAIL(); - *sp++ = list_val_rel(*ep,base); + *sp++ = list_val(*ep); ++ep; break; case matchMap: - if (!is_map_rel(*ep, base)) { + if (!is_map(*ep)) { FAIL(); } n = *pc++; - if (is_flatmap_rel(*ep,base)) { - if (flatmap_get_size(flatmap_val_rel(*ep, base)) < n) { + if (is_flatmap(*ep)) { + if (flatmap_get_size(flatmap_val(*ep)) < n) { FAIL(); } } else { - ASSERT(is_hashmap_rel(*ep,base)); - if (hashmap_size_rel(*ep, base) < n) { + ASSERT(is_hashmap(*ep)); + if (hashmap_size(*ep) < n) { FAIL(); } } - ep = flatmap_val_rel(*ep, base); + ep = flatmap_val(*ep); break; case matchPushM: - if (!is_map_rel(*ep, base)) { + if (!is_map(*ep)) { FAIL(); } n = *pc++; - if (is_flatmap_rel(*ep,base)) { - if (flatmap_get_size(flatmap_val_rel(*ep, base)) < n) { + if (is_flatmap(*ep)) { + if (flatmap_get_size(flatmap_val(*ep)) < n) { FAIL(); } } else { - ASSERT(is_hashmap_rel(*ep,base)); - if (hashmap_size_rel(*ep, base) < n) { + ASSERT(is_hashmap(*ep)); + if (hashmap_size(*ep) < n) { FAIL(); } } - *sp++ = flatmap_val_rel(*ep++, base); + *sp++ = flatmap_val(*ep++); break; case matchKey: t = (Eterm) *pc++; - tp = erts_maps_get_rel(t, make_boxed_rel(ep, base), base); + tp = erts_maps_get(t, make_boxed(ep)); if (!tp) { FAIL(); } @@ -2061,29 +1977,29 @@ restart: break; case matchCmp: n = *pc++; - if (!eq_rel(variables[n].term, base, *ep, base)) + if (!EQ(variables[n].term, *ep)) FAIL(); ++ep; break; case matchEqBin: t = (Eterm) *pc++; - if (!eq_rel(t,NULL,*ep,base)) + if (!EQ(t,*ep)) FAIL(); ++ep; break; case matchEqFloat: - if (!is_float_rel(*ep,base)) + if (!is_float(*ep)) FAIL(); - if (memcmp(float_val_rel(*ep,base) + 1, pc, sizeof(double))) + if (memcmp(float_val(*ep) + 1, pc, sizeof(double))) FAIL(); pc += TermWords(2); ++ep; break; case matchEqRef: { Eterm* epc = (Eterm*)pc; - if (!is_ref_rel(*ep,base)) + if (!is_ref(*ep)) FAIL(); - if (!eq_rel(make_internal_ref_rel(epc, epc), epc, *ep, base)) { + if (!EQ(make_internal_ref(epc), *ep)) { FAIL(); } i = thing_arityval(*epc); @@ -2092,9 +2008,9 @@ restart: break; } case matchEqBig: - if (!is_big_rel(*ep,base)) + if (!is_big(*ep)) FAIL(); - tp = big_val_rel(*ep,base); + tp = big_val(*ep); { Eterm *epc = (Eterm *) pc; if (*tp != *epc) @@ -2236,62 +2152,36 @@ restart: esp -= 2; esp[-1] = t; break; - - #if HALFWORD_HEAP - case matchPushVGuard: - if (!base) goto case_matchPushV; - /* Build NULL-based copy on pseudo heap for easy disposal */ - n = *pc++; - ASSERT(is_value(variables[n].term)); - ASSERT(!variables[n].proc); - variables[n].term = copy_object_rel(psp, variables[n].term, base); - *esp++ = variables[n].term; - #ifdef DEBUG - variables[n].proc = psp; - variables[n].base = NULL; - #endif - break; - #endif case matchPushVResult: if (!(in_flags & ERTS_PAM_COPY_RESULT)) goto case_matchPushV; - - /* Build (NULL-based) copy on callers heap */ - #if HALFWORD_HEAP - if (!do_catch && !c_p_checkpoint.p) { - heap_checkpoint_init(c_p, &c_p_checkpoint); - } - #endif + /* Build copy on callers heap */ n = *pc++; ASSERT(is_value(variables[n].term)); ASSERT(!variables[n].proc); - variables[n].term = copy_object_rel(c_p, variables[n].term, base); + variables[n].term = copy_object_x(variables[n].term, c_p, HEAP_XTRA); *esp++ = variables[n].term; #ifdef DEBUG variables[n].proc = c_p; - variables[n].base = NULL; #endif break; case matchPushV: case_matchPushV: n = *pc++; ASSERT(is_value(variables[n].term)); - ASSERT(!variables[n].base); *esp++ = variables[n].term; break; case matchPushExpr: if (in_flags & ERTS_PAM_COPY_RESULT) { Uint sz; Eterm* top; - sz = size_object_rel(term, base); + sz = size_object(term); top = HAllocX(build_proc, sz, HEAP_XTRA); if (in_flags & ERTS_PAM_CONTIGUOUS_TUPLE) { - ASSERT(is_tuple_rel(term,base)); - *esp++ = copy_shallow_rel(tuple_val_rel(term,base), sz, - &top, &MSO(build_proc), base); + ASSERT(is_tuple(term)); + *esp++ = copy_shallow(tuple_val(term), sz, &top, &MSO(build_proc)); } else { - *esp++ = copy_struct_rel(term, sz, &top, &MSO(build_proc), - base, NULL); + *esp++ = copy_struct(term, sz, &top, &MSO(build_proc)); } } else { @@ -2299,7 +2189,6 @@ restart: } break; case matchPushArrayAsList: - ASSERT_HALFWORD(base == NULL); n = arity; /* Only happens when 'term' is an array */ tp = termp; ehp = HAllocX(build_proc, n*2, HEAP_XTRA); @@ -2315,7 +2204,6 @@ restart: break; case matchPushArrayAsListU: /* This instruction is NOT efficient. */ - ASSERT_HALFWORD(base == NULL); *esp++ = dpm_array_to_list(build_proc, termp, arity); break; case matchTrue: @@ -2619,13 +2507,6 @@ restart: } } fail: -#if HALFWORD_HEAP - if (c_p_checkpoint.p) { - /* Dispose garbage built by guards on caller heap */ - heap_checkpoint_revert(&c_p_checkpoint); - c_p_checkpoint.p = NULL; - } -#endif *return_flags = 0U; if (fail_label >= 0) { /* We failed during a "TryMeElse", lets restart, with the next match @@ -2788,13 +2669,6 @@ Wterm db_do_read_element(DbUpdateHandle* handle, Sint position) { Eterm elem = handle->dbterm->tpl[position]; if (!is_header(elem)) { -#if HALFWORD_HEAP - if (!is_immed(elem) - && !handle->tb->common.compress - && !(handle->abs_vec && handle->abs_vec[position])) { - return rterm2wterm(elem, handle->dbterm->tpl); - } -#endif return elem; } @@ -2822,9 +2696,6 @@ void db_do_update_element(DbUpdateHandle* handle, Eterm* oldp; Uint newval_sz; Uint oldval_sz; -#if HALFWORD_HEAP - Eterm* old_base; -#endif if (is_both_immed(newval,oldval)) { handle->dbterm->tpl[position] = newval; @@ -2841,15 +2712,8 @@ void db_do_update_element(DbUpdateHandle* handle, handle->dbterm); handle->flags |= DB_MUST_RESIZE; oldval = handle->dbterm->tpl[position]; - #if HALFWORD_HEAP - old_base = NULL; - #endif } else { - #if HALFWORD_HEAP - ASSERT(!handle->abs_vec); - old_base = handle->dbterm->tpl; - #endif if (is_boxed(newval)) { newp = boxed_val(newval); switch (*newp & _TAG_HEADER_MASK) { @@ -2859,7 +2723,7 @@ void db_do_update_element(DbUpdateHandle* handle, case _TAG_HEADER_HEAP_BIN: newval_sz = header_arity(*newp) + 1; if (is_boxed(oldval)) { - oldp = boxed_val_rel(oldval,old_base); + oldp = boxed_val(oldval); switch (*oldp & _TAG_HEADER_MASK) { case _TAG_HEADER_POS_BIG: case _TAG_HEADER_NEG_BIG: @@ -2879,20 +2743,13 @@ void db_do_update_element(DbUpdateHandle* handle, } } } -#if HALFWORD_HEAP - else { - old_base = (handle->tb->common.compress - || (handle->abs_vec && handle->abs_vec[position])) ? - NULL : handle->dbterm->tpl; - } -#endif /* Not possible for simple memcpy or dbterm is already non-contiguous, */ /* need to realloc... */ newval_sz = is_immed(newval) ? 0 : size_object(newval); new_size_set: - oldval_sz = is_immed(oldval) ? 0 : size_object_rel(oldval,old_base); + oldval_sz = is_immed(oldval) ? 0 : size_object(oldval); both_size_set: handle->new_size = handle->new_size - oldval_sz + newval_sz; @@ -2900,19 +2757,6 @@ both_size_set: /* write new value in old dbterm, finalize will make a flat copy */ handle->dbterm->tpl[position] = newval; handle->flags |= DB_MUST_RESIZE; - -#if HALFWORD_HEAP - if (old_base && newval_sz > 0) { - ASSERT(!handle->tb->common.compress); - if (!handle->abs_vec) { - int i = header_arity(handle->dbterm->tpl[0]); - handle->abs_vec = erts_alloc(ERTS_ALC_T_TMP, (i+1)*sizeof(char)); - sys_memset(handle->abs_vec, 0, i+1); - /* abs_vec[0] not used */ - } - handle->abs_vec[position] = 1; - } -#endif } static ERTS_INLINE byte* db_realloc_term(DbTableCommon* tb, void* old, @@ -3011,7 +2855,7 @@ static void* copy_to_comp(DbTableCommon* tb, Eterm obj, DbTerm* dest, tpl[arity + 1] = alloc_size; tmp_offheap.first = NULL; - tpl[tb->keypos] = copy_struct_rel(key, size_object(key), &top.ep, &tmp_offheap, NULL, tpl); + tpl[tb->keypos] = copy_struct(key, size_object(key), &top.ep, &tmp_offheap); dest->first_oh = tmp_offheap.first; for (i=1; i<=arity; i++) { if (i != tb->keypos) { @@ -3030,7 +2874,7 @@ static void* copy_to_comp(DbTableCommon* tb, Eterm obj, DbTerm* dest, Eterm* dbg_top = erts_alloc(ERTS_ALC_T_DB_TERM, dest->size * sizeof(Eterm)); dest->debug_clone = dbg_top; tmp_offheap.first = dest->first_oh; - copy_struct_rel(obj, dest->size, &dbg_top, &tmp_offheap, NULL, dbg_top); + copy_struct(obj, dest->size, &dbg_top, &tmp_offheap); dest->first_oh = tmp_offheap.first; ASSERT(dbg_top == dest->debug_clone + dest->size); } @@ -3077,7 +2921,7 @@ void* db_store_term(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj) newp->size = size; top = newp->tpl; tmp_offheap.first = NULL; - copy_struct_rel(obj, size, &top, &tmp_offheap, NULL, top); + copy_struct(obj, size, &top, &tmp_offheap); newp->first_oh = tmp_offheap.first; #ifdef DEBUG_CLONE newp->debug_clone = NULL; @@ -3160,29 +3004,8 @@ void db_finalize_resize(DbUpdateHandle* handle, Uint offset) tmp_offheap.first = NULL; - #if HALFWORD_HEAP - if (handle->abs_vec) { - int i, arity = header_arity(handle->dbterm->tpl[0]); - - top[0] = tpl[0]; - top += arity + 1; - for (i=1; i<=arity; i++) { - Eterm* src_base = handle->abs_vec[i] ? NULL : tpl; - - newDbTerm->tpl[i] = copy_struct_rel(tpl[i], - size_object_rel(tpl[i],src_base), - &top, &tmp_offheap, src_base, - newDbTerm->tpl); - } - newDbTerm->first_oh = tmp_offheap.first; - ASSERT((byte*)top <= (newp + alloc_sz)); - erts_free(ERTS_ALC_T_TMP, handle->abs_vec); - } - else - #endif /* HALFWORD_HEAP */ { - copy_struct_rel(make_tuple_rel(tpl,tpl), handle->new_size, &top, - &tmp_offheap, tpl, top); + copy_struct(make_tuple(tpl), handle->new_size, &top, &tmp_offheap); newDbTerm->first_oh = tmp_offheap.first; ASSERT((byte*)top == (newp + alloc_sz)); } @@ -3199,9 +3022,9 @@ Eterm db_copy_from_comp(DbTableCommon* tb, DbTerm* bp, Eterm** hpp, hp[0] = bp->tpl[0]; *hpp += arity + 1; - hp[tb->keypos] = copy_struct_rel(bp->tpl[tb->keypos], - size_object_rel(bp->tpl[tb->keypos], bp->tpl), - hpp, off_heap, bp->tpl, NULL); + hp[tb->keypos] = copy_struct(bp->tpl[tb->keypos], + size_object(bp->tpl[tb->keypos]), + hpp, off_heap); erts_factory_static_init(&factory, *hpp, bp->size - (arity+1), off_heap); @@ -3221,7 +3044,7 @@ Eterm db_copy_from_comp(DbTableCommon* tb, DbTerm* bp, Eterm** hpp, ASSERT((*hpp - hp) <= bp->size); #ifdef DEBUG_CLONE - ASSERT(eq_rel(make_tuple(hp),NULL,make_tuple(bp->debug_clone),bp->debug_clone)); + ASSERT(EQ(make_tuple(hp),make_tuple(bp->debug_clone))); #endif return make_tuple(hp); } @@ -3245,14 +3068,14 @@ Eterm db_copy_element_from_ets(DbTableCommon* tb, Process* p, *hpp = erts_produce_heap(&factory, extra, 0); erts_factory_close(&factory); #ifdef DEBUG_CLONE - ASSERT(eq_rel(copy, NULL, obj->debug_clone[pos], obj->debug_clone)); + ASSERT(EQ(copy, obj->debug_clone[pos])); #endif return copy; } else { - Uint sz = size_object_rel(obj->tpl[pos], obj->tpl); + Uint sz = size_object(obj->tpl[pos]); *hpp = HAlloc(p, sz + extra); - return copy_struct_rel(obj->tpl[pos], sz, hpp, &MSO(p), obj->tpl, NULL); + return copy_struct(obj->tpl[pos], sz, hpp, &MSO(p)); } } @@ -3599,26 +3422,10 @@ static DMCRet dmc_one_term(DMCContext *context, { Eterm* ref_val = internal_ref_val(c); DMC_PUSH(*text, matchEqRef); -#if HALFWORD_HEAP - { - union { - UWord u; - Uint t[2]; - } fiddle; - ASSERT(thing_arityval(ref_val[0]) == 3); - fiddle.t[0] = ref_val[0]; - fiddle.t[1] = ref_val[1]; - DMC_PUSH(*text, fiddle.u); - fiddle.t[0] = ref_val[2]; - fiddle.t[1] = ref_val[3]; - DMC_PUSH(*text, fiddle.u); - } -#else n = thing_arityval(ref_val[0]); for (i = 0; i <= n; ++i) { DMC_PUSH(*text, ref_val[i]); } -#endif break; } case (_TAG_HEADER_POS_BIG >> _TAG_PRIMARY_SIZE): @@ -3627,53 +3434,19 @@ static DMCRet dmc_one_term(DMCContext *context, Eterm* bval = big_val(c); n = thing_arityval(bval[0]); DMC_PUSH(*text, matchEqBig); -#if HALFWORD_HEAP - { - union { - UWord u; - Uint t[2]; - } fiddle; - ASSERT(n >= 1); - fiddle.t[0] = bval[0]; - fiddle.t[1] = bval[1]; - DMC_PUSH(*text, fiddle.u); - for (i = 2; i <= n; ++i) { - fiddle.t[0] = bval[i]; - if (++i <= n) { - fiddle.t[1] = bval[i]; - } else { - fiddle.t[1] = (Uint) 0; - } - DMC_PUSH(*text, fiddle.u); - } - } -#else for (i = 0; i <= n; ++i) { DMC_PUSH(*text, (Uint) bval[i]); } -#endif break; } case (_TAG_HEADER_FLOAT >> _TAG_PRIMARY_SIZE): DMC_PUSH(*text,matchEqFloat); -#if HALFWORD_HEAP - { - union { - UWord u; - Uint t[2]; - } fiddle; - fiddle.t[0] = float_val(c)[1]; - fiddle.t[1] = float_val(c)[2]; - DMC_PUSH(*text, fiddle.u); - } -#else DMC_PUSH(*text, (Uint) float_val(c)[1]); #ifdef ARCH_64 DMC_PUSH(*text, (Uint) 0); #else DMC_PUSH(*text, (Uint) float_val(c)[2]); #endif -#endif break; default: /* BINARY, FUN, VECTOR, or EXTERNAL */ DMC_PUSH(*text, matchEqBin); @@ -3997,24 +3770,8 @@ static void dmc_add_pushv_variant(DMCContext *context, DMCHeap *heap, MatchOps instr = matchPushV; ASSERT(n < heap->vars_used && v->is_bound); - if (context->is_guard) { - #if HALFWORD_HEAP - if (!v->first_guard_label) { - v->first_guard_label = DMC_STACK_NUM(*text); - ASSERT(v->first_guard_label); - instr = matchPushVGuard; /* may be changed to PushVResult below */ - } - #endif - } - else { /* body */ - #if HALFWORD_HEAP - if (v->first_guard_label) { - /* Avoid double-copy, copy to result heap at first encounter in guard */ - DMC_POKE(*text, v->first_guard_label, matchPushVResult); - v->is_in_body = 1; - } - #endif - if (!v->is_in_body) { + if (!context->is_guard) { + if(!v->is_in_body) { instr = matchPushVResult; v->is_in_body = 1; } @@ -5443,16 +5200,13 @@ Eterm db_match_dbterm(DbTableCommon* tb, Process* c_p, Binary* bprog, int all, DbTerm* obj, Eterm** hpp, Uint extra) { Uint32 dummy; - Eterm* base; Eterm res; if (tb->compress) { obj = db_alloc_tmp_uncompressed(tb, obj); - base = NULL; } - else base = HALFWORD_HEAP ? obj->tpl : NULL; - res = db_prog_match(c_p, bprog, make_tuple_rel(obj->tpl,base), base, NULL, 0, + res = db_prog_match(c_p, bprog, make_tuple(obj->tpl), NULL, 0, ERTS_PAM_COPY_RESULT|ERTS_PAM_CONTIGUOUS_TUPLE, &dummy); if (is_value(res) && hpp!=NULL) { @@ -5573,7 +5327,7 @@ void db_match_dis(Binary *bp) first = 0; else erts_printf(", "); -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) erts_printf("0x%016bex", rt->data.ui[ri]); #else erts_printf("0x%08bex", rt->data.ui[ri]); @@ -5597,7 +5351,7 @@ void db_match_dis(Binary *bp) first = 0; else erts_printf(", "); -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) erts_printf("0x%016bex", *et); #else erts_printf("0x%08bex", *et); @@ -5720,13 +5474,6 @@ void db_match_dis(Binary *bp) ++t; erts_printf("PushV\t%beu\n", n); break; - #if HALFWORD_HEAP - case matchPushVGuard: - n = (Uint) *++t; - ++t; - erts_printf("PushVGuard\t%beu\n", n); - break; - #endif case matchPushVResult: n = (Uint) *++t; ++t; diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h index 1ccdc0305b..c0966655cd 100644 --- a/erts/emulator/beam/erl_db_util.h +++ b/erts/emulator/beam/erl_db_util.h @@ -90,9 +90,6 @@ typedef struct { Uint new_size; int flags; void* lck; -#if HALFWORD_HEAP - unsigned char* abs_vec; /* [i] true if dbterm->tpl[i] is absolute Eterm */ -#endif } DbUpdateHandle; @@ -287,10 +284,10 @@ ERTS_GLB_INLINE Eterm db_copy_key(Process* p, DbTable* tb, DbTerm* obj) Eterm key = GETKEY(tb, obj->tpl); if IS_CONST(key) return key; else { - Uint size = size_object_rel(key, obj->tpl); + Uint size = size_object(key); Eterm* hp = HAlloc(p, size); - Eterm res = copy_struct_rel(key, size, &hp, &MSO(p), obj->tpl, NULL); - ASSERT(eq_rel(res,NULL,key,obj->tpl)); + Eterm res = copy_struct(key, size, &hp, &MSO(p)); + ASSERT(EQ(res,key)); return res; } } @@ -302,14 +299,14 @@ ERTS_GLB_INLINE Eterm db_copy_object_from_ets(DbTableCommon* tb, DbTerm* bp, return db_copy_from_comp(tb, bp, hpp, off_heap); } else { - return copy_shallow_rel(bp->tpl, bp->size, hpp, off_heap, bp->tpl); + return copy_shallow(bp->tpl, bp->size, hpp, off_heap); } } ERTS_GLB_INLINE int db_eq(DbTableCommon* tb, Eterm a, DbTerm* b) { if (!tb->compress) { - return eq_rel(a, NULL, make_tuple_rel(b->tpl,b->tpl), b->tpl); + return EQ(a, make_tuple(b->tpl)); } else { return db_eq_comp(tb, a, b); @@ -435,7 +432,7 @@ Binary *db_match_compile(Eterm *matchexpr, Eterm *guards, Eterm db_match_dbterm(DbTableCommon* tb, Process* c_p, Binary* bprog, int all, DbTerm* obj, Eterm** hpp, Uint extra); -Eterm db_prog_match(Process *p, Binary *prog, Eterm term, Eterm* base, +Eterm db_prog_match(Process *p, Binary *prog, Eterm term, Eterm *termp, int arity, enum erts_pam_run_flags in_flags, Uint32 *return_flags /* Zeroed on enter */); diff --git a/erts/emulator/beam/erl_debug.c b/erts/emulator/beam/erl_debug.c index 2dcfb79f00..fae38728e4 100644 --- a/erts/emulator/beam/erl_debug.c +++ b/erts/emulator/beam/erl_debug.c @@ -631,29 +631,4 @@ void print_memory_info(Process *p) } erts_printf("+-----------------%s-%s-%s-%s-+\n",dashes,dashes,dashes,dashes); } -#if !HEAP_ON_C_STACK && defined(DEBUG) -Eterm *erts_debug_allocate_tmp_heap(int size, Process *p) -{ - ErtsSchedulerData *sd = ((p == NULL) ? erts_get_scheduler_data() : ERTS_PROC_GET_SCHDATA(p)); - int offset = sd->num_tmp_heap_used; - - ASSERT(offset+size <= TMP_HEAP_SIZE); - return (sd->tmp_heap)+offset; -} -void erts_debug_use_tmp_heap(int size, Process *p) -{ - ErtsSchedulerData *sd = ((p == NULL) ? erts_get_scheduler_data() : ERTS_PROC_GET_SCHDATA(p)); - - sd->num_tmp_heap_used += size; - ASSERT(sd->num_tmp_heap_used <= TMP_HEAP_SIZE); -} -void erts_debug_unuse_tmp_heap(int size, Process *p) -{ - ErtsSchedulerData *sd = ((p == NULL) ? erts_get_scheduler_data() : ERTS_PROC_GET_SCHDATA(p)); - - sd->num_tmp_heap_used -= size; - ASSERT(sd->num_tmp_heap_used >= 0); -} #endif -#endif - diff --git a/erts/emulator/beam/erl_debug.h b/erts/emulator/beam/erl_debug.h index 4905e64f07..f4259e7dae 100644 --- a/erts/emulator/beam/erl_debug.h +++ b/erts/emulator/beam/erl_debug.h @@ -92,10 +92,4 @@ extern void print_tagged_memory(Eterm *start, Eterm *end); extern void print_untagged_memory(Eterm *start, Eterm *end); extern void print_memory(Process *p); extern void print_memory_info(Process *p); -#if defined(DEBUG) && !HEAP_ON_C_STACK -extern Eterm *erts_debug_allocate_tmp_heap(int, Process *); -extern void erts_debug_use_tmp_heap(int, Process *); -extern void erts_debug_unuse_tmp_heap(int, Process *); -#endif - #endif /* _ERL_DEBUG_H_ */ diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h index 6b406d069c..e71b87803b 100644 --- a/erts/emulator/beam/erl_driver.h +++ b/erts/emulator/beam/erl_driver.h @@ -57,10 +57,6 @@ # define SIZEOF_LONG_LONG_SAVED__ SIZEOF_LONG_LONG # undef SIZEOF_LONG_LONG #endif -#ifdef HALFWORD_HEAP_EMULATOR -# define HALFWORD_HEAP_EMULATOR_SAVED__ HALFWORD_HEAP_EMULATOR -# undef HALFWORD_HEAP_EMULATOR -#endif #include "erl_int_sizes_config.h" #if defined(SIZEOF_CHAR_SAVED__) && SIZEOF_CHAR_SAVED__ != SIZEOF_CHAR # error SIZEOF_CHAR mismatch @@ -78,11 +74,6 @@ # error SIZEOF_LONG_LONG mismatch #endif -/* This is OK to override by the NIF/driver implementor */ -#if defined(HALFWORD_HEAP_EMULATOR_SAVED__) && !defined(HALFWORD_HEAP_EMULATOR) -#define HALFWORD_HEAP_EMULATOR HALFWORD_HEAP_EMULATOR_SAVED__ -#endif - #include "erl_drv_nif.h" #include <stdlib.h> @@ -699,16 +690,6 @@ EXTERN char *driver_dl_error(void); EXTERN int erl_drv_putenv(char *key, char *value); EXTERN int erl_drv_getenv(char *key, char *value, size_t *value_size); -#ifdef __OSE__ -typedef ErlDrvUInt ErlDrvOseEventId; -EXTERN union SIGNAL *erl_drv_ose_get_signal(ErlDrvEvent ev); -EXTERN ErlDrvEvent erl_drv_ose_event_alloc(SIGSELECT sig, ErlDrvOseEventId handle, - ErlDrvOseEventId (*resolve_signal)(union SIGNAL *sig), void *extra); -EXTERN void erl_drv_ose_event_free(ErlDrvEvent ev); -EXTERN void erl_drv_ose_event_fetch(ErlDrvEvent ev, SIGSELECT *sig, - ErlDrvOseEventId *handle, void **extra); -#endif - #endif /* !ERL_DRIVER_TYPES_ONLY */ #ifdef WIN32_DYNAMIC_ERL_DRIVER diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index d2604f1595..29579f74e6 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -128,7 +128,7 @@ static void disallow_heap_frag_ref_in_old_heap(Process* p); static void disallow_heap_frag_ref(Process* p, Eterm* n_htop, Eterm* objv, int nobj); #endif -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) # define MAX_HEAP_SIZES 154 #else # define MAX_HEAP_SIZES 59 @@ -147,26 +147,10 @@ typedef struct { erts_smp_atomic32_t refc; } ErtsGCInfoReq; -#if !HALFWORD_HEAP ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(gcireq, - ErtsGCInfoReq, - 5, - ERTS_ALC_T_GC_INFO_REQ) -#else -static ERTS_INLINE ErtsGCInfoReq * -gcireq_alloc(void) -{ - return erts_alloc(ERTS_ALC_T_GC_INFO_REQ, - sizeof(ErtsGCInfoReq)); -} - -static ERTS_INLINE void -gcireq_free(ErtsGCInfoReq *ptr) -{ - erts_free(ERTS_ALC_T_GC_INFO_REQ, ptr); -} -#endif - + ErtsGCInfoReq, + 5, + ERTS_ALC_T_GC_INFO_REQ) /* * Initialize GC global data. */ @@ -208,7 +192,7 @@ erts_init_gc(void) } - /* for 32 bit we want max_heap_size to be MAX(32bit) / 4 [words] (and halfword) + /* for 32 bit we want max_heap_size to be MAX(32bit) / 4 [words] * for 64 bit we want max_heap_size to be MAX(52bit) / 8 [words] */ @@ -232,10 +216,7 @@ erts_init_gc(void) init_gc_info(&esdp->gc_info); } -#if !HALFWORD_HEAP init_gcireq_alloc(); -#endif - } /* @@ -867,7 +848,6 @@ minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) if (OLD_HEAP(p) && ((mature <= OLD_HEND(p) - OLD_HTOP(p)) && - ((BIN_VHEAP_MATURE(p) < ( BIN_OLD_VHEAP_SZ(p) - BIN_OLD_VHEAP(p)))) && ((BIN_OLD_VHEAP_SZ(p) > BIN_OLD_VHEAP(p))) ) ) { ErlMessage *msgp; Uint size_after; @@ -1681,7 +1661,7 @@ disallow_heap_frag_ref_in_old_heap(Process* p) val = *hp++; switch (primary_tag(val)) { case TAG_PRIMARY_BOXED: - ptr = (Eterm *) EXPAND_POINTER(val); + ptr = (Eterm *) val; if (!in_area(ptr, old_heap, old_heap_size)) { if (in_area(ptr, new_heap, new_heap_size)) { abort(); @@ -1694,7 +1674,7 @@ disallow_heap_frag_ref_in_old_heap(Process* p) } break; case TAG_PRIMARY_LIST: - ptr = (Eterm *) EXPAND_POINTER(val); + ptr = (Eterm *) val; if (!in_area(ptr, old_heap, old_heap_size)) { if (in_area(ptr, new_heap, new_heap_size)) { abort(); @@ -2392,7 +2372,6 @@ sweep_off_heap(Process *p, int fullsweep) } BIN_VHEAP_SZ(p) = next_vheap_size(p, bin_vheap, BIN_VHEAP_SZ(p)); MSO(p).overhead = bin_vheap; - BIN_VHEAP_MATURE(p) = bin_vheap; /* * If we got any shrink candidates, check them out. diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index d9c3b0dcf4..5c209a4af2 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -91,11 +91,6 @@ const int etp_arch_bits = 32; #else # error "Not 64-bit, nor 32-bit arch" #endif -#if HALFWORD_HEAP -const int etp_halfword = 1; -#else -const int etp_halfword = 0; -#endif #ifdef HIPE const int etp_hipe = 1; #else diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c index 3b3b247020..84bee976ff 100644 --- a/erts/emulator/beam/erl_lock_check.c +++ b/erts/emulator/beam/erl_lock_check.c @@ -156,9 +156,6 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "instr", NULL }, { "alcu_allocator", "index" }, { "mseg", NULL }, -#if HALFWORD_HEAP - { "pmmap", NULL }, -#endif #ifdef ERTS_SMP { "port_task_pre_alloc_lock", "address" }, { "proclist_pre_alloc_lock", "address" }, diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c index ff2a355309..29b3024644 100644 --- a/erts/emulator/beam/erl_map.c +++ b/erts/emulator/beam/erl_map.c @@ -172,26 +172,22 @@ BIF_RETTYPE maps_to_list_1(BIF_ALIST_1) { */ const Eterm * -#if HALFWORD_HEAP -erts_maps_get_rel(Eterm key, Eterm map, Eterm *map_base) -#else erts_maps_get(Eterm key, Eterm map) -#endif { Uint32 hx; - if (is_flatmap_rel(map, map_base)) { + if (is_flatmap(map)) { Eterm *ks, *vs; flatmap_t *mp; Uint n, i; - mp = (flatmap_t *)flatmap_val_rel(map, map_base); + mp = (flatmap_t *)flatmap_val(map); n = flatmap_get_size(mp); if (n == 0) { return NULL; } - ks = (Eterm *)tuple_val_rel(mp->keys, map_base) + 1; + ks = (Eterm *)tuple_val(mp->keys) + 1; vs = flatmap_get_values(mp); if (is_immed(key)) { @@ -203,16 +199,16 @@ erts_maps_get(Eterm key, Eterm map) } for (i = 0; i < n; i++) { - if (eq_rel(ks[i], map_base, key, NULL)) { + if (EQ(ks[i], key)) { return &vs[i]; } } return NULL; } - ASSERT(is_hashmap_rel(map, map_base)); + ASSERT(is_hashmap(map)); hx = hashmap_make_hash(key); - return erts_hashmap_get_rel(hx, key, map, map_base); + return erts_hashmap_get(hx, key, map); } BIF_RETTYPE maps_find_2(BIF_ALIST_2) { @@ -1998,11 +1994,7 @@ Eterm* hashmap_iterator_prev(ErtsWStack* s) { } const Eterm * -#if HALFWORD_HEAP -erts_hashmap_get_rel(Uint32 hx, Eterm key, Eterm node, Eterm *map_base) -#else erts_hashmap_get(Uint32 hx, Eterm key, Eterm node) -#endif { Eterm *ptr, hdr, *res; Uint ix, lvl = 0; @@ -2011,7 +2003,7 @@ erts_hashmap_get(Uint32 hx, Eterm key, Eterm node) UseTmpHeapNoproc(2); ASSERT(is_boxed(node)); - ptr = boxed_val_rel(node, map_base); + ptr = boxed_val(node); hdr = *ptr; ASSERT(is_header(hdr)); ASSERT(is_hashmap_header_head(hdr)); @@ -2032,15 +2024,15 @@ erts_hashmap_get(Uint32 hx, Eterm key, Eterm node) node = ptr[ix+1]; if (is_list(node)) { /* LEAF NODE [K|V] */ - ptr = list_val_rel(node,map_base); - res = eq_rel(CAR(ptr), map_base, key, NULL) ? &(CDR(ptr)) : NULL; + ptr = list_val(node); + res = EQ(CAR(ptr), key) ? &(CDR(ptr)) : NULL; break; } hx = hashmap_shift_hash(th,hx,lvl,key); ASSERT(is_boxed(node)); - ptr = boxed_val_rel(node, map_base); + ptr = boxed_val(node); hdr = *ptr; ASSERT(is_header(hdr)); ASSERT(!is_hashmap_header_head(hdr)); diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h index c391de3f11..be6f791a4e 100644 --- a/erts/emulator/beam/erl_map.h +++ b/erts/emulator/beam/erl_map.h @@ -57,7 +57,6 @@ typedef struct flatmap_s { #define hashmap_size(x) (((hashmap_head_t*) hashmap_val(x))->size) -#define hashmap_size_rel(RTERM, BASE) hashmap_size(rterm2wterm(RTERM, BASE)) #define hashmap_make_hash(Key) make_internal_hash(Key) #define hashmap_restore_hash(Heap,Lvl,Key) \ @@ -104,23 +103,9 @@ Eterm erts_hashmap_from_array(ErtsHeapFactory*, Eterm *leafs, Uint n, int rejec Eterm erts_hashmap_from_ks_and_vs_extra(Process *p, Eterm *ks, Eterm *vs, Uint n, Eterm k, Eterm v); -const Eterm * -#if HALFWORD_HEAP -erts_maps_get_rel(Eterm key, Eterm map, Eterm *map_base); -# define erts_maps_get(A, B) erts_maps_get_rel(A, B, NULL) -#else -erts_maps_get(Eterm key, Eterm map); -# define erts_maps_get_rel(A, B, B_BASE) erts_maps_get(A, B) -#endif +const Eterm *erts_maps_get(Eterm key, Eterm map); -const Eterm * -#if HALFWORD_HEAP -erts_hashmap_get_rel(Uint32 hx, Eterm key, Eterm node, Eterm *map_base); -# define erts_hashmap_get(Hx, K, M) erts_hashmap_get_rel(Hx, K, M, NULL) -#else -erts_hashmap_get(Uint32 hx, Eterm key, Eterm map); -# define erts_hashmap_get_rel(Hx, K, M, M_BASE) erts_hashmap_get(Hx, K, M) -#endif +const Eterm *erts_hashmap_get(Uint32 hx, Eterm key, Eterm map); /* hamt nodes v2.0 * diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index fbdf3fb0e2..f37b430d27 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -32,9 +32,6 @@ struct external_thing_; struct erl_off_heap_header { Eterm thing_word; Uint size; -#if HALFWORD_HEAP - void* dummy_ptr_padding__; -#endif struct erl_off_heap_header* next; }; diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h index 7d880126f8..5da6bb877a 100644 --- a/erts/emulator/beam/erl_nif.h +++ b/erts/emulator/beam/erl_nif.h @@ -86,10 +86,6 @@ # define SIZEOF_LONG_LONG_SAVED__ SIZEOF_LONG_LONG # undef SIZEOF_LONG_LONG #endif -#ifdef HALFWORD_HEAP_EMULATOR -# define HALFWORD_HEAP_EMULATOR_SAVED__ HALFWORD_HEAP_EMULATOR -# undef HALFWORD_HEAP_EMULATOR -#endif #include "erl_int_sizes_config.h" #ifdef __cplusplus @@ -109,16 +105,11 @@ typedef long long ErlNifSInt64; #error No 64-bit integer type #endif -#ifdef HALFWORD_HEAP_EMULATOR -# define ERL_NIF_VM_VARIANT "beam.halfword" -typedef unsigned int ERL_NIF_TERM; -#else -# define ERL_NIF_VM_VARIANT "beam.vanilla" -# if SIZEOF_LONG == SIZEOF_VOID_P +#define ERL_NIF_VM_VARIANT "beam.vanilla" +#if SIZEOF_LONG == SIZEOF_VOID_P typedef unsigned long ERL_NIF_TERM; -# elif SIZEOF_LONG_LONG == SIZEOF_VOID_P +#elif SIZEOF_LONG_LONG == SIZEOF_VOID_P typedef unsigned long long ERL_NIF_TERM; -# endif #endif typedef ERL_NIF_TERM ERL_NIF_UINT; diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h index f93152c921..2f2180e1aa 100644 --- a/erts/emulator/beam/erl_nif_api_funcs.h +++ b/erts/emulator/beam/erl_nif_api_funcs.h @@ -543,7 +543,7 @@ static ERL_NIF_INLINE ERL_NIF_TERM enif_make_list9(ErlNifEnv* env, #ifndef enif_make_pid -# define enif_make_pid(ENV, PID) ((const ERL_NIF_TERM)((PID)->pid)) +# define enif_make_pid(ENV, PID) ((void)(ENV),(const ERL_NIF_TERM)((PID)->pid)) #if SIZEOF_LONG == 8 # define enif_get_int64 enif_get_long diff --git a/erts/emulator/beam/erl_node_container_utils.h b/erts/emulator/beam/erl_node_container_utils.h index 211b1a0090..b6a3531e28 100644 --- a/erts/emulator/beam/erl_node_container_utils.h +++ b/erts/emulator/beam/erl_node_container_utils.h @@ -255,7 +255,7 @@ extern ErtsPTab erts_port; * Refs * \* */ -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) #define internal_ref_no_of_numbers(x) \ (internal_ref_data((x))[0]) @@ -328,8 +328,6 @@ extern ErtsPTab erts_port; : external_ref_channel_no((x))) #define is_ref(x) (is_internal_ref((x)) \ || is_external_ref((x))) -#define is_ref_rel(x,Base) (is_internal_ref_rel((x),Base) \ - || is_external_ref_rel((x),Base)) #define is_not_ref(x) (!is_ref(x)) #endif diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c index 2fb790b953..62a44f7129 100644 --- a/erts/emulator/beam/erl_node_tables.c +++ b/erts/emulator/beam/erl_node_tables.c @@ -1207,23 +1207,11 @@ insert_offheap(ErlOffHeap *oh, int type, Eterm id) break; } if (insert_bin) { -#if HALFWORD_HEAP - UWord val = (UWord) u.pb->val; - DeclareTmpHeapNoproc(id_heap,BIG_UINT_HEAP_SIZE*2); /* extra place allocated */ -#else DeclareTmpHeapNoproc(id_heap,BIG_UINT_HEAP_SIZE); -#endif Uint *hp = &id_heap[0]; InsertedBin *nib; -#if HALFWORD_HEAP - int actual_need = BIG_UWORD_HEAP_SIZE(val); - ASSERT(actual_need <= (BIG_UINT_HEAP_SIZE*2)); - UseTmpHeapNoproc(actual_need); - a.id = erts_bld_uword(&hp, NULL, (UWord) val); -#else UseTmpHeapNoproc(BIG_UINT_HEAP_SIZE); a.id = erts_bld_uint(&hp, NULL, (Uint) u.pb->val); -#endif erts_match_prog_foreach_offheap(u.pb->val, insert_offheap2, (void *) &a); @@ -1231,11 +1219,7 @@ insert_offheap(ErlOffHeap *oh, int type, Eterm id) nib->bin_val = u.pb->val; nib->next = inserted_bins; inserted_bins = nib; -#if HALFWORD_HEAP - UnUseTmpHeapNoproc(actual_need); -#else UnUseTmpHeapNoproc(BIG_UINT_HEAP_SIZE); -#endif } } break; diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h index 694ac84232..64278d2ea0 100644 --- a/erts/emulator/beam/erl_node_tables.h +++ b/erts/emulator/beam/erl_node_tables.h @@ -66,7 +66,7 @@ #define ERTS_DE_QFLGS_ALL (ERTS_DE_QFLG_BUSY \ | ERTS_DE_QFLG_EXIT) -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) #define ERTS_DIST_OUTPUT_BUF_DBG_PATTERN ((Uint) 0xf713f713f713f713UL) #else #define ERTS_DIST_OUTPUT_BUF_DBG_PATTERN ((Uint) 0xf713f713) diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c index 5c38db1cbc..2c09834d19 100644 --- a/erts/emulator/beam/erl_port_task.c +++ b/erts/emulator/beam/erl_port_task.c @@ -1734,7 +1734,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) reds = ERTS_PORT_REDS_INPUT; ASSERT((state & ERTS_PORT_SFLGS_DEAD) == 0); DTRACE_DRIVER(driver_ready_input, pp); - /* NOTE some windows/ose drivers use ->ready_input + /* NOTE some windows drivers use ->ready_input for input and output */ (*pp->drv_ptr->ready_input)((ErlDrvData) pp->drv_data, ptp->u.alive.td.io.event); diff --git a/erts/emulator/beam/erl_printf_term.c b/erts/emulator/beam/erl_printf_term.c index 2917d58932..45ba4371dc 100644 --- a/erts/emulator/beam/erl_printf_term.c +++ b/erts/emulator/beam/erl_printf_term.c @@ -117,13 +117,12 @@ do { \ /* return 0 if list is not a non-empty flat list of printable characters */ static int -is_printable_string(Eterm list, Eterm* base) -{ +is_printable_string(Eterm list) { int len = 0; int c; while(is_list(list)) { - Eterm* consp = list_val_rel(list, base); + Eterm* consp = list_val(list); Eterm hd = CAR(consp); if (!is_byte(hd)) @@ -260,9 +259,7 @@ static char *format_binary(Uint16 x, char *b) { #endif static int -print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount, - Eterm* obj_base) /* ignored if !HALFWORD_HEAP */ -{ +print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount) { DECLARE_WSTACK(s); int res; int i; @@ -308,7 +305,7 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount, obj = (Eterm) popped.word; L_print_one_cons: { - Eterm* cons = list_val_rel(obj, obj_base); + Eterm* cons = list_val(obj); Eterm tl; obj = CAR(cons); @@ -344,11 +341,7 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount, PRINT_CHAR(res, fn, arg, '>'); goto L_done; } -#if HALFWORD_HEAP - wobj = is_immed(obj) ? (Wterm)obj : rterm2wterm(obj, obj_base); -#else wobj = (Wterm)obj; -#endif switch (tag_val_def(wobj)) { case NIL_DEF: PRINT_STRING(res, fn, arg, "[]"); @@ -424,10 +417,10 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount, PRINT_CHAR(res, fn, arg, '>'); break; case LIST_DEF: - if (is_printable_string(obj, obj_base)) { + if (is_printable_string(obj)) { int c; PRINT_CHAR(res, fn, arg, '"'); - nobj = list_val_rel(obj, obj_base); + nobj = list_val(obj); while (1) { if ((*dcount)-- <= 0) goto L_done; @@ -441,7 +434,7 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount, } if (is_not_list(*nobj)) break; - nobj = list_val_rel(*nobj, obj_base); + nobj = list_val(*nobj); } PRINT_CHAR(res, fn, arg, '"'); } else { @@ -469,11 +462,11 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount, case BINARY_DEF: { byte* bytep; - Uint bytesize = binary_size_rel(obj,obj_base); + Uint bytesize = binary_size(obj); Uint bitoffs; Uint bitsize; byte octet; - ERTS_GET_BINARY_BYTES_REL(obj, bytep, bitoffs, bitsize, obj_base); + ERTS_GET_BINARY_BYTES(obj, bytep, bitoffs, bitsize); if (bitsize || !bytesize || !is_printable_ascii(bytep, bytesize, bitoffs)) { @@ -648,13 +641,11 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount, int -erts_printf_term(fmtfn_t fn, void* arg, ErlPfEterm term, long precision, - ErlPfEterm* term_base) -{ +erts_printf_term(fmtfn_t fn, void* arg, ErlPfEterm term, long precision) { int res; ERTS_CT_ASSERT(sizeof(ErlPfEterm) == sizeof(Eterm)); - res = print_term(fn, arg, (Eterm)term, &precision, (Eterm*)term_base); + res = print_term(fn, arg, (Eterm)term, &precision); if (res < 0) return res; if (precision <= 0) diff --git a/erts/emulator/beam/erl_printf_term.h b/erts/emulator/beam/erl_printf_term.h index 87618a36d7..4618457f27 100644 --- a/erts/emulator/beam/erl_printf_term.h +++ b/erts/emulator/beam/erl_printf_term.h @@ -22,6 +22,5 @@ #define ERL_PRINTF_TERM_H__ #include "erl_printf_format.h" -int erts_printf_term(fmtfn_t fn, void* arg, ErlPfEterm term, long precision, - ErlPfEterm* term_base); +int erts_printf_term(fmtfn_t fn, void* arg, ErlPfEterm term, long precision); #endif diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 7135c0475e..15a6d5d651 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -58,11 +58,7 @@ #define ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST (CONTEXT_REDS/10) -#ifndef ERTS_SCHED_MIN_SPIN #define ERTS_SCHED_SPIN_UNTIL_YIELD 100 -#else -#define ERTS_SCHED_SPIN_UNTIL_YIELD 1 -#endif #define ERTS_SCHED_SYS_SLEEP_SPINCOUNT_VERY_LONG 40 #define ERTS_SCHED_AUX_WORK_SLEEP_SPINCOUNT_FACT_VERY_LONG 1000 @@ -119,7 +115,7 @@ #define RUNQ_SET_RQ(X, RQ) erts_smp_atomic_set_nob((X), (erts_aint_t) (RQ)) #ifdef DEBUG -# if defined(ARCH_64) && !HALFWORD_HEAP +# if defined(ARCH_64) # define ERTS_DBG_SET_INVALID_RUNQP(RQP, N) \ (RUNQ_SET_RQ((RQP), (0xdeadbeefdead0003LL | ((N) << 4))) # define ERTS_DBG_VERIFY_VALID_RUNQP(RQP) \ @@ -152,12 +148,7 @@ extern BeamInstr beam_apply[]; extern BeamInstr beam_exit[]; extern BeamInstr beam_continue_exit[]; -#ifdef __OSE__ -/* Eager check I/O not supported on OSE yet. */ -int erts_eager_check_io = 0; -#else int erts_eager_check_io = 1; -#endif int erts_sched_compact_load; int erts_sched_balance_util = 0; Uint erts_no_schedulers; @@ -971,25 +962,10 @@ typedef struct { erts_smp_atomic32_t refc; } ErtsSchedWallTimeReq; -#if !HALFWORD_HEAP ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(swtreq, - ErtsSchedWallTimeReq, - 5, - ERTS_ALC_T_SCHED_WTIME_REQ) -#else -static ERTS_INLINE ErtsSchedWallTimeReq * -swtreq_alloc(void) -{ - return erts_alloc(ERTS_ALC_T_SCHED_WTIME_REQ, - sizeof(ErtsSchedWallTimeReq)); -} - -static ERTS_INLINE void -swtreq_free(ErtsSchedWallTimeReq *ptr) -{ - erts_free(ERTS_ALC_T_SCHED_WTIME_REQ, ptr); -} -#endif + ErtsSchedWallTimeReq, + 5, + ERTS_ALC_T_SCHED_WTIME_REQ) static void reply_sched_wall_time(void *vswtrp) @@ -2536,19 +2512,10 @@ try_set_sys_scheduling(void) #endif static ERTS_INLINE int -prepare_for_sys_schedule(ErtsSchedulerData *esdp, int non_blocking) +prepare_for_sys_schedule(int non_blocking) { if (non_blocking && erts_eager_check_io) { #ifdef ERTS_SMP -#ifdef ERTS_SCHED_ONLY_POLL_SCHED_1 - if (esdp->no != 1) { - /* If we are not scheduler 1 and ERTS_SCHED_ONLY_POLL_SCHED_1 is used - then we make sure to wake scheduler 1 */ - ErtsRunQueue *rq = ERTS_RUNQ_IX(0); - wake_scheduler(rq); - return 0; - } -#endif return try_set_sys_scheduling(); #else return 1; @@ -2558,16 +2525,6 @@ prepare_for_sys_schedule(ErtsSchedulerData *esdp, int non_blocking) #ifdef ERTS_SMP while (!erts_port_task_have_outstanding_io_tasks() && try_set_sys_scheduling()) { -#ifdef ERTS_SCHED_ONLY_POLL_SCHED_1 - if (esdp->no != 1) { - /* If we are not scheduler 1 and ERTS_SCHED_ONLY_POLL_SCHED_1 is used - then we make sure to wake scheduler 1 */ - ErtsRunQueue *rq = ERTS_RUNQ_IX(0); - clear_sys_scheduling(); - wake_scheduler(rq); - return 0; - } -#endif if (!erts_port_task_have_outstanding_io_tasks()) return 1; clear_sys_scheduling(); @@ -2891,8 +2848,6 @@ aux_thread(void *unused) erts_thr_progress_active(NULL, thr_prgr_active = 0); erts_thr_progress_prepare_wait(NULL); - ERTS_SCHED_FAIR_YIELD(); - flgs = sched_spin_wait(ssi, 0); if (flgs & ERTS_SSI_FLG_SLEEPING) { @@ -2960,7 +2915,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) * be waiting in erl_sys_schedule() */ - if (ERTS_SCHEDULER_IS_DIRTY(esdp) || !prepare_for_sys_schedule(esdp, 0)) { + if (ERTS_SCHEDULER_IS_DIRTY(esdp) || !prepare_for_sys_schedule(0)) { sched_waiting(esdp->no, rq); @@ -3025,8 +2980,6 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) erts_thr_progress_prepare_wait(esdp); } - ERTS_SCHED_FAIR_YIELD(); - flgs = sched_spin_wait(ssi, spincount); if (flgs & ERTS_SSI_FLG_SLEEPING) { ASSERT(flgs & ERTS_SSI_FLG_WAITING); @@ -3097,13 +3050,8 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) #ifdef ERTS_DIRTY_SCHEDULERS ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp)); #endif - -#ifdef ERTS_SCHED_ONLY_POLL_SCHED_1 - ASSERT(esdp->no == 1); -#endif sched_waiting_sys(esdp->no, rq); - erts_smp_runq_unlock(rq); ASSERT(working); @@ -3173,7 +3121,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) * Got to check that we still got I/O tasks; otherwise * we have to continue checking for I/O... */ - if (!prepare_for_sys_schedule(esdp, 0)) { + if (!prepare_for_sys_schedule(0)) { spincount *= ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT; goto tse_wait; } @@ -3195,7 +3143,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) * Got to check that we still got I/O tasks; otherwise * we have to wait in erl_sys_schedule() after all... */ - if (!prepare_for_sys_schedule(esdp, 0)) { + if (!prepare_for_sys_schedule(0)) { /* * Not allowed to wait in erl_sys_schedule; * do tse wait instead... @@ -5316,17 +5264,11 @@ erts_early_init_scheduling(int no_schedulers) wakeup_other.threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_MEDIUM; wakeup_other.type = ERTS_SCHED_WAKEUP_OTHER_TYPE_DEFAULT; #endif -#ifndef ERTS_SCHED_MIN_SPIN sched_busy_wait.sys_schedule = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_MEDIUM; sched_busy_wait.tse = (ERTS_SCHED_SYS_SLEEP_SPINCOUNT_MEDIUM * ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT); sched_busy_wait.aux_work = (ERTS_SCHED_SYS_SLEEP_SPINCOUNT_MEDIUM * ERTS_SCHED_AUX_WORK_SLEEP_SPINCOUNT_FACT_MEDIUM); -#else - sched_busy_wait.sys_schedule = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_NONE; - sched_busy_wait.tse = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_NONE; - sched_busy_wait.aux_work = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_NONE; -#endif } int @@ -5510,9 +5452,6 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num, esdp->f_reg_array = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_BEAM_REGISTER, MAX_REG * sizeof(FloatDef)); -#if !HEAP_ON_C_STACK - esdp->num_tmp_heap_used = 0; -#endif #ifdef ERTS_DIRTY_SCHEDULERS if (ERTS_RUNQ_IX_IS_DIRTY(runq->ix)) { esdp->no = 0; @@ -5797,9 +5736,7 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online #endif init_misc_aux_work(); -#if !HALFWORD_HEAP init_swtreq_alloc(); -#endif erts_atomic32_init_nob(&debug_wait_completed_count, 0); /* debug only */ debug_wait_completed_flags = 0; @@ -7938,11 +7875,16 @@ sched_thread_func(void *vesdp) ErtsThrPrgrCallbacks callbacks; ErtsSchedulerData *esdp = vesdp; Uint no = esdp->no; +#ifdef ERTS_SMP + erts_tse_t *tse; +#endif erts_sched_init_time_sup(esdp); #ifdef ERTS_SMP - ERTS_SCHED_SLEEP_INFO_IX(no - 1)->event = erts_tse_fetch(); + tse = erts_tse_fetch(); + erts_tse_prepare_timed(tse); + ERTS_SCHED_SLEEP_INFO_IX(no - 1)->event = tse; callbacks.arg = (void *) esdp->ssi; callbacks.wakeup = thr_prgr_wakeup; callbacks.prepare_wait = thr_prgr_prep_wait; @@ -8204,18 +8146,12 @@ erts_start_schedulers(void) erts_snprintf(opts.name, 16, "%lu_scheduler", actual + 1); -#ifdef __OSE__ - /* This should be done in the bind strategy */ - opts.coreNo = (actual+1) % ose_num_cpus(); -#endif - res = ethr_thr_create(&esdp->tid, sched_thread_func, (void*)esdp, &opts); if (res != 0) { break; } } - erts_no_schedulers = actual; #ifdef ERTS_DIRTY_SCHEDULERS @@ -8244,10 +8180,6 @@ erts_start_schedulers(void) erts_snprintf(opts.name, 16, "aux"); -#ifdef __OSE__ - opts.coreNo = 0; -#endif /* __OSE__ */ - res = ethr_thr_create(&aux_tid, aux_thread, NULL, &opts); if (res != 0) erl_exit(1, "Failed to create aux thread\n"); @@ -8267,7 +8199,6 @@ erts_start_schedulers(void) actual, actual == 1 ? " was" : "s were"); erts_send_error_to_logger_nogl(dsbufp); } - } #endif /* ERTS_SMP */ @@ -8286,7 +8217,7 @@ add_pend_suspend(Process *suspendee, sizeof(ErtsPendingSuspend)); psp->next = NULL; #ifdef DEBUG -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) psp->end = (ErtsPendingSuspend *) 0xdeaddeaddeaddead; #else psp->end = (ErtsPendingSuspend *) 0xdeaddead; @@ -9412,12 +9343,10 @@ Process *schedule(Process *p, int calls) int leader_update = ERTS_SCHEDULER_IS_DIRTY(esdp) ? 0 : erts_thr_progress_update(esdp); aux_work = erts_atomic32_read_acqb(&esdp->ssi->aux_work); - if (aux_work | leader_update | ERTS_SCHED_FAIR) { + if (aux_work | leader_update) { erts_smp_runq_unlock(rq); if (leader_update) erts_thr_progress_leader_update(esdp); - else if (ERTS_SCHED_FAIR) - ERTS_SCHED_FAIR_YIELD(); if (aux_work) handle_aux_work(&esdp->aux_work_data, aux_work, 0); erts_smp_runq_lock(rq); @@ -9493,7 +9422,7 @@ Process *schedule(Process *p, int calls) } else if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && (fcalls > input_reductions && - prepare_for_sys_schedule(esdp, !0))) { + prepare_for_sys_schedule(!0))) { ErtsMonotonicTime current_time; /* * Schedule system-level activities. @@ -10905,7 +10834,6 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->bin_vheap_sz = p->min_vheap_size; p->bin_old_vheap_sz = p->min_vheap_size; p->bin_old_vheap = 0; - p->bin_vheap_mature = 0; p->sys_task_qs = NULL; @@ -11125,7 +11053,6 @@ void erts_init_empty_process(Process *p) p->bin_old_vheap_sz = BIN_VH_MIN_SIZE; p->bin_old_vheap = 0; p->sys_task_qs = NULL; - p->bin_vheap_mature = 0; ERTS_PTMR_INIT(p); p->next = NULL; p->off_heap.first = NULL; @@ -12542,7 +12469,7 @@ stack_element_dump(int to, void *to_arg, Eterm* sp, int yreg) } if (is_CP(x)) { - erts_print(to, to_arg, "Return addr %p (", (Eterm *) EXPAND_POINTER(x)); + erts_print(to, to_arg, "Return addr %p (", (Eterm *) x); print_function_from_pc(to, to_arg, cp_val(x)); erts_print(to, to_arg, ")\n"); yreg = 0; diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 20ffe7ea7c..e7c5614b9c 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -632,12 +632,6 @@ struct ErtsSchedulerData_ { Process *free_process; ErtsThrPrgrData thr_progress_data; #endif -#if !HEAP_ON_C_STACK - Eterm tmp_heap[TMP_HEAP_SIZE]; - int num_tmp_heap_used; - Eterm beam_emu_tmp_heap[BEAM_EMU_TMP_HEAP_SIZE]; - Eterm erl_arith_tmp_heap[ERL_ARITH_TMP_HEAP_SIZE]; -#endif ErtsSchedulerSleepInfo *ssi; Process *current_process; Uint no; /* Scheduler number for normal schedulers */ @@ -691,13 +685,6 @@ extern ErtsAlignedSchedulerData *erts_aligned_dirty_io_scheduler_data; extern ErtsSchedulerData *erts_scheduler_data; #endif -#ifdef ERTS_SCHED_FAIR -#define ERTS_SCHED_FAIR_YIELD() ETHR_YIELD() -#else -#define ERTS_SCHED_FAIR 0 -#define ERTS_SCHED_FAIR_YIELD() -#endif - #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) int erts_smp_lc_runq_is_locked(ErtsRunQueue *); #endif @@ -911,7 +898,6 @@ struct ErtsPendingSuspend_ { # define MIN_VHEAP_SIZE(p) (p)->min_vheap_size # define BIN_VHEAP_SZ(p) (p)->bin_vheap_sz -# define BIN_VHEAP_MATURE(p) (p)->bin_vheap_mature # define BIN_OLD_VHEAP_SZ(p) (p)->bin_old_vheap_sz # define BIN_OLD_VHEAP(p) (p)->bin_old_vheap @@ -1030,7 +1016,6 @@ struct process { ErtsPSD *psd; /* Rarely used process specific data */ Uint64 bin_vheap_sz; /* Virtual heap block size for binaries */ - Uint64 bin_vheap_mature; /* Virtual heap block size for binaries */ Uint64 bin_old_vheap_sz; /* Virtual old heap block size for binaries */ Uint64 bin_old_vheap; /* Virtual old heap size for binaries */ diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c index 25f0b1ed38..3b8ae11e94 100644 --- a/erts/emulator/beam/erl_process_dump.c +++ b/erts/emulator/beam/erl_process_dump.c @@ -365,7 +365,7 @@ heap_dump(int to, void *to_arg, Eterm x) while (x != OUR_NIL) { if (is_CP(x)) { - next = (Eterm *) EXPAND_POINTER(x); + next = (Eterm *) x; } else if (is_list(x)) { ptr = list_val(x); if (ptr[0] != OUR_NIL) { @@ -378,7 +378,7 @@ heap_dump(int to, void *to_arg, Eterm x) ptr[1] = make_small(0); } x = ptr[0]; - ptr[0] = (Eterm) COMPRESS_POINTER(next); + ptr[0] = (Eterm) next; next = ptr + 1; continue; } @@ -408,7 +408,7 @@ heap_dump(int to, void *to_arg, Eterm x) ptr[0] = OUR_NIL; } else { x = ptr[arity]; - ptr[0] = (Eterm) COMPRESS_POINTER(next); + ptr[0] = (Eterm) next; next = ptr + arity - 1; continue; } diff --git a/erts/emulator/beam/erl_process_lock.h b/erts/emulator/beam/erl_process_lock.h index 788348e613..a64c993e8f 100644 --- a/erts/emulator/beam/erl_process_lock.h +++ b/erts/emulator/beam/erl_process_lock.h @@ -854,9 +854,6 @@ ERTS_GLB_INLINE void erts_proc_dec_refc(Process *p) #endif if (!referred) { ASSERT(ERTS_PROC_IS_EXITING(p)); - ASSERT(ERTS_AINT_NULL - == erts_ptab_pix2intptr_ddrb(&erts_proc, - internal_pid_index(p->common.id))); erts_free_proc(p); } } @@ -872,9 +869,6 @@ ERTS_GLB_INLINE void erts_proc_add_refc(Process *p, Sint add_refc) #endif if (!referred) { ASSERT(ERTS_PROC_IS_EXITING(p)); - ASSERT(ERTS_AINT_NULL - == erts_ptab_pix2intptr_ddrb(&erts_proc, - internal_pid_index(p->common.id))); erts_free_proc(p); } } diff --git a/erts/emulator/beam/erl_term.c b/erts/emulator/beam/erl_term.c index e5050bfaa5..ed302fa2fa 100644 --- a/erts/emulator/beam/erl_term.c +++ b/erts/emulator/beam/erl_term.c @@ -174,9 +174,7 @@ ET_DEFINE_CHECKED(Uint,external_thing_data_words,ExternalThing*,is_thing_ptr); ET_DEFINE_CHECKED(Eterm,make_cp,UWord *,_is_taggable_pointer); ET_DEFINE_CHECKED(UWord *,cp_val,Eterm,is_CP); ET_DEFINE_CHECKED(Uint,catch_val,Eterm,is_catch); -ET_DEFINE_CHECKED(Uint,x_reg_offset,Uint,_is_xreg); -ET_DEFINE_CHECKED(Uint,y_reg_offset,Uint,_is_yreg); -ET_DEFINE_CHECKED(Uint,x_reg_index,Uint,_is_xreg); -ET_DEFINE_CHECKED(Uint,y_reg_index,Uint,_is_yreg); +ET_DEFINE_CHECKED(Uint,loader_x_reg_index,Uint,_is_loader_x_reg); +ET_DEFINE_CHECKED(Uint,loader_y_reg_index,Uint,_is_loader_y_reg); #endif /* ET_DEBUG */ diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h index d5e91d80d9..01ef567dcd 100644 --- a/erts/emulator/beam/erl_term.h +++ b/erts/emulator/beam/erl_term.h @@ -21,34 +21,8 @@ #ifndef __ERL_TERM_H #define __ERL_TERM_H -#include "sys.h" /* defines HALFWORD_HEAP */ - typedef UWord Wterm; /* Full word terms */ -#if HALFWORD_HEAP -# define HEAP_ON_C_STACK 0 -# if HALFWORD_ASSERT -# ifdef ET_DEBUG -# undef ET_DEBUG -# endif -# define ET_DEBUG 1 -# endif -# if 1 -# define CHECK_POINTER_MASK 0xFFFFFFFF00000000UL -# define COMPRESS_POINTER(APointer) ((Eterm) (UWord) (APointer)) -# define EXPAND_POINTER(AnEterm) ((UWord) (AnEterm)) -# else -# define CHECK_POINTER_MASK 0x0UL -# define COMPRESS_POINTER(AnUint) (AnUint) -# define EXPAND_POINTER(APointer) (APointer) -# endif -#else -# define HEAP_ON_C_STACK 1 -# define CHECK_POINTER_MASK 0x0UL -# define COMPRESS_POINTER(AnUint) (AnUint) -# define EXPAND_POINTER(APointer) (APointer) -#endif - struct erl_node_; /* Declared in erl_node_tables.h */ /* @@ -190,15 +164,12 @@ struct erl_node_; /* Declared in erl_node_tables.h */ /* boxed object access methods */ -#if HALFWORD_HEAP -#define _is_taggable_pointer(x) (((UWord)(x) & (CHECK_POINTER_MASK | 0x3)) == 0) -#define _boxed_precond(x) (is_boxed(x)) -#else + #define _is_taggable_pointer(x) (((Uint)(x) & 0x3) == 0) #define _boxed_precond(x) (is_boxed(x)) -#endif + #define _is_aligned(x) (((Uint)(x) & 0x3) == 0) -#define _unchecked_make_boxed(x) ((Uint) COMPRESS_POINTER(x) + TAG_PRIMARY_BOXED) +#define _unchecked_make_boxed(x) ((Uint)(x) + TAG_PRIMARY_BOXED) _ET_DECLARE_CHECKED(Eterm,make_boxed,const Eterm*) #define make_boxed(x) _ET_APPLY(make_boxed,(x)) #if 1 @@ -209,12 +180,12 @@ _ET_DECLARE_CHECKED(int,is_boxed,Eterm) #else #define is_boxed(x) (((x) & _TAG_PRIMARY_MASK) == TAG_PRIMARY_BOXED) #endif -#define _unchecked_boxed_val(x) ((Eterm*) EXPAND_POINTER(((x) - TAG_PRIMARY_BOXED))) +#define _unchecked_boxed_val(x) ((Eterm*) ((x) - TAG_PRIMARY_BOXED)) _ET_DECLARE_CHECKED(Eterm*,boxed_val,Wterm) #define boxed_val(x) _ET_APPLY(boxed_val,(x)) /* cons cell ("list") access methods */ -#define _unchecked_make_list(x) ((Uint) COMPRESS_POINTER(x) + TAG_PRIMARY_LIST) +#define _unchecked_make_list(x) ((Uint)(x) + TAG_PRIMARY_LIST) _ET_DECLARE_CHECKED(Eterm,make_list,const Eterm*) #define make_list(x) _ET_APPLY(make_list,(x)) #if 1 @@ -226,12 +197,8 @@ _ET_DECLARE_CHECKED(int,is_not_list,Eterm) #define is_list(x) (((x) & _TAG_PRIMARY_MASK) == TAG_PRIMARY_LIST) #define is_not_list(x) (!is_list((x))) #endif -#if HALFWORD_HEAP #define _list_precond(x) (is_list(x)) -#else -#define _list_precond(x) (is_list(x)) -#endif -#define _unchecked_list_val(x) ((Eterm*) EXPAND_POINTER((x) - TAG_PRIMARY_LIST)) +#define _unchecked_list_val(x) ((Eterm*) ((x) - TAG_PRIMARY_LIST)) _ET_DECLARE_CHECKED(Eterm*,list_val,Wterm) #define list_val(x) _ET_APPLY(list_val,(x)) @@ -242,7 +209,7 @@ _ET_DECLARE_CHECKED(Eterm*,list_val,Wterm) #define CDR(x) ((x)[1]) /* generic tagged pointer (boxed or list) access methods */ -#define _unchecked_ptr_val(x) ((Eterm*) EXPAND_POINTER((x) & ~((Uint) 0x3))) +#define _unchecked_ptr_val(x) ((Eterm*) ((x) & ~((Uint) 0x3))) #define ptr_val(x) _unchecked_ptr_val((x)) /*XXX*/ #define _unchecked_offset_ptr(x,offs) ((x)+((offs)*sizeof(Eterm))) #define offset_ptr(x,offs) _unchecked_offset_ptr(x,offs) /*XXX*/ @@ -250,7 +217,7 @@ _ET_DECLARE_CHECKED(Eterm*,list_val,Wterm) #define byte_offset_ptr(x,offs) _unchecked_byte_offset_ptr(x,offs) /*XXX*/ /* fixnum ("small") access methods */ -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) #define SMALL_BITS (64-4) #define SMALL_DIGITS (17) #else @@ -396,11 +363,7 @@ _ET_DECLARE_CHECKED(Eterm*,fun_val,Wterm) _ET_DECLARE_CHECKED(Eterm*,export_val,Wterm) #define export_val(x) _ET_APPLY(export_val,(x)) #define is_export_header(x) ((x) == HEADER_EXPORT) -#if HALFWORD_HEAP -#define HEADER_EXPORT _make_header(2,_TAG_HEADER_EXPORT) -#else #define HEADER_EXPORT _make_header(1,_TAG_HEADER_EXPORT) -#endif /* bignum access methods */ #define make_pos_bignum_header(sz) _make_header((sz),_TAG_HEADER_POS_BIG) @@ -424,7 +387,7 @@ _ET_DECLARE_CHECKED(Eterm*,big_val,Wterm) #define big_val(x) _ET_APPLY(big_val,(x)) /* flonum ("float") access methods */ -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) #define HEADER_FLONUM _make_header(1,_TAG_HEADER_FLOAT) #else #define HEADER_FLONUM _make_header(2,_TAG_HEADER_FLOAT) @@ -445,12 +408,12 @@ typedef union float_def byte fb[sizeof(ieee754_8)]; Uint16 fs[sizeof(ieee754_8) / sizeof(Uint16)]; Uint32 fw[sizeof(ieee754_8) / sizeof(Uint32)]; -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) Uint fdw; #endif } FloatDef; -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) #define FLOAT_VAL_GET_DOUBLE(fval, f) (f).fdw = *((fval)+1) @@ -727,7 +690,7 @@ _ET_DECLARE_CHECKED(struct erl_node_*,internal_port_node,Eterm) #define ERTS_MAX_REF_NUMBERS 3 #define ERTS_REF_NUMBERS ERTS_MAX_REF_NUMBERS -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) # define ERTS_REF_WORDS (ERTS_REF_NUMBERS/2 + 1) # define ERTS_REF_32BIT_WORDS (ERTS_REF_NUMBERS+1) #else @@ -749,7 +712,7 @@ typedef struct { #define make_ref_thing_header(DW) \ _make_header((DW)+REF_THING_HEAD_SIZE-1,_TAG_HEADER_REF) -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) /* * Ref layout on a 64-bit little endian machine: @@ -1009,27 +972,20 @@ _ET_DECLARE_CHECKED(struct erl_node_*,external_ref_node,Eterm) #define MAP_HEADER_VAL(Hdr) (((Hdr) >> (_HEADER_ARITY_OFFS + MAP_HEADER_TAG_SZ + MAP_HEADER_ARITY_SZ)) & (0xffff)) #define make_hashmap(x) make_boxed((Eterm*)(x)) -#define make_hashmap_rel make_boxed_rel #define is_hashmap(x) (is_boxed((x)) && is_hashmap_header(*boxed_val((x)))) #define is_not_hashmap(x) (!is_hashmap(x)) -#define is_hashmap_rel(RTERM,BASE) is_hashmap(rterm2wterm(RTERM,BASE)) #define is_hashmap_header(x) (((x) & (_HEADER_MAP_HASHMAP_HEAD_MASK)) == HAMT_SUBTAG_HEAD_ARRAY) #define hashmap_val(x) _unchecked_boxed_val((x)) -#define hashmap_val_rel(RTERM, BASE) hashmap_val(rterm2wterm(RTERM, BASE)) #define make_flatmap(x) make_boxed((Eterm*)(x)) -#define make_flatmap_rel(x, BASE) make_boxed_rel((Eterm*)(x),(BASE)) #define is_flatmap(x) (is_boxed((x)) && is_flatmap_header(*boxed_val((x)))) -#define is_flatmap_rel(RTERM,BASE) is_flatmap(rterm2wterm(RTERM,BASE)) #define is_not_flatmap(x) (!is_flatmap((x))) #define is_flatmap_header(x) (((x) & (_HEADER_MAP_SUBTAG_MASK)) == HAMT_SUBTAG_HEAD_FLATMAP) #define flatmap_val(x) (_unchecked_boxed_val((x))) -#define flatmap_val_rel(RTERM, BASE) flatmap_val(rterm2wterm(RTERM, BASE)) #define is_map_header(x) (((x) & (_TAG_HEADER_MASK)) == _TAG_HEADER_MAP) #define is_map(x) (is_boxed((x)) && is_map_header(*boxed_val(x))) #define is_not_map(x) (!is_map(x)) -#define is_map_rel(RTERM,BASE) is_map(rterm2wterm(RTERM,BASE)) /* number tests */ @@ -1048,14 +1004,14 @@ _ET_DECLARE_CHECKED(struct erl_node_*,external_ref_node,Eterm) #error "fix yer arch, like" #endif -#define _unchecked_make_cp(x) ((Eterm) COMPRESS_POINTER(x)) +#define _unchecked_make_cp(x) ((Eterm)(x)) _ET_DECLARE_CHECKED(Eterm,make_cp,BeamInstr*) #define make_cp(x) _ET_APPLY(make_cp,(x)) #define is_not_CP(x) ((x) & _CPMASK) #define is_CP(x) (!is_not_CP(x)) -#define _unchecked_cp_val(x) ((BeamInstr*) EXPAND_POINTER(x)) +#define _unchecked_cp_val(x) ((BeamInstr*) (x)) _ET_DECLARE_CHECKED(BeamInstr*,cp_val,Eterm) #define cp_val(x) _ET_APPLY(cp_val,(x)) @@ -1071,44 +1027,40 @@ _ET_DECLARE_CHECKED(Uint,catch_val,Eterm) /* * Overloaded tags. * - * SMALL = 15 - * ATOM/NIL=7 + * In the loader, we want to tag a term in a way so that it can + * be any literal (atom/integer/float/tuple/list/binary) or a + * register. * - * Note that the two least significant bits in SMALL/ATOM/NIL always are 3; - * thus, we can distinguish register from literals by looking at only these - * two bits. + * We can achive that by overloading the PID and PORT tags to + * mean X and Y registers. That works because there are no + * pid or port literals. */ -#define X_REG_DEF 0 -#define Y_REG_DEF 1 -#define R_REG_DEF 2 - -#define beam_reg_tag(x) ((x) & 3) +#define _LOADER_TAG_XREG _TAG_IMMED1_PID +#define _LOADER_TAG_YREG _TAG_IMMED1_PORT +#define _LOADER_TAG_SIZE _TAG_IMMED1_SIZE +#define _LOADER_MASK _TAG_IMMED1_MASK -#define make_rreg() R_REG_DEF -#define make_xreg(ix) (((ix) * sizeof(Eterm)) | X_REG_DEF) -#define make_yreg(ix) (((ix) * sizeof(Eterm)) | Y_REG_DEF) +#define LOADER_X_REG _LOADER_TAG_XREG +#define LOADER_Y_REG _LOADER_TAG_YREG -#define _is_xreg(x) (beam_reg_tag(x) == X_REG_DEF) -#define _is_yreg(x) (beam_reg_tag(x) == Y_REG_DEF) +#define make_loader_x_reg(R) (((R) << _LOADER_TAG_SIZE) | _LOADER_TAG_XREG) +#define make_loader_y_reg(R) (((R) << _LOADER_TAG_SIZE) | _LOADER_TAG_YREG) -#define _unchecked_x_reg_offset(R) ((R) - X_REG_DEF) -_ET_DECLARE_CHECKED(Uint,x_reg_offset,Uint) -#define x_reg_offset(R) _ET_APPLY(x_reg_offset,(R)) +#define loader_reg_index(R) ((R) >> _LOADER_TAG_SIZE) -#define _unchecked_y_reg_offset(R) ((R) - Y_REG_DEF) -_ET_DECLARE_CHECKED(Uint,y_reg_offset,Uint) -#define y_reg_offset(R) _ET_APPLY(y_reg_offset,(R)) +#define loader_tag(T) ((T) & _LOADER_MASK) -#define reg_index(R) ((R) / sizeof(Eterm)) +#define _is_loader_x_reg(x) (loader_tag(x) == _LOADER_TAG_XREG) +#define _is_loader_y_reg(x) (loader_tag(x) == _LOADER_TAG_YREG) -#define _unchecked_x_reg_index(R) ((R) >> 2) -_ET_DECLARE_CHECKED(Uint,x_reg_index,Uint) -#define x_reg_index(R) _ET_APPLY(x_reg_index,(R)) +#define _unchecked_loader_x_reg_index(R) ((R) >> _LOADER_TAG_SIZE) +_ET_DECLARE_CHECKED(Uint,loader_x_reg_index,Uint) +#define loader_x_reg_index(R) _ET_APPLY(loader_x_reg_index,(R)) -#define _unchecked_y_reg_index(R) ((R) >> 2) -_ET_DECLARE_CHECKED(Uint,y_reg_index,Uint) -#define y_reg_index(R) _ET_APPLY(y_reg_index,(R)) +#define _unchecked_loader_y_reg_index(R) ((R) >> _LOADER_TAG_SIZE) +_ET_DECLARE_CHECKED(Uint,loader_y_reg_index,Uint) +#define loader_y_reg_index(R) _ET_APPLY(loader_y_reg_index,(R)) /* * Backwards compatibility definitions: @@ -1160,82 +1112,7 @@ extern unsigned tag_val_def(Wterm); #define FLOAT_BIG _NUMBER_CODE(FLOAT_DEF,BIG_DEF) #define FLOAT_FLOAT _NUMBER_CODE(FLOAT_DEF,FLOAT_DEF) -#if HALFWORD_HEAP -#define ptr2rel(PTR,BASE) ((Eterm*)((char*)(PTR) - (char*)(BASE))) -#define rterm2wterm(REL,BASE) ((Wterm)(REL) + (Wterm)(BASE)) - -#else /* HALFWORD_HEAP */ - -#define ptr2rel(PTR,BASE) (PTR) -#define rterm2wterm(REL,BASE) (REL) - -#endif /* !HALFWORD_HEAP */ - -#define make_list_rel(PTR, BASE) make_list(ptr2rel(PTR,BASE)) -#define make_boxed_rel(PTR, BASE) make_boxed(ptr2rel(PTR,BASE)) -#define make_fun_rel make_boxed_rel -#define make_binary_rel make_boxed_rel -#define make_tuple_rel make_boxed_rel -#define make_external_rel make_boxed_rel -#define make_internal_ref_rel make_boxed_rel -#define make_big_rel make_boxed_rel - -#define binary_val_rel(RTERM, BASE) binary_val(rterm2wterm(RTERM, BASE)) -#define list_val_rel(RTERM, BASE) list_val(rterm2wterm(RTERM, BASE)) -#define boxed_val_rel(RTERM, BASE) boxed_val(rterm2wterm(RTERM, BASE)) -#define tuple_val_rel(RTERM, BASE) tuple_val(rterm2wterm(RTERM, BASE)) -#define export_val_rel(RTERM, BASE) export_val(rterm2wterm(RTERM, BASE)) -#define fun_val_rel(RTERM, BASE) fun_val(rterm2wterm(RTERM, BASE)) -#define big_val_rel(RTERM,BASE) big_val(rterm2wterm(RTERM,BASE)) -#define float_val_rel(RTERM,BASE) float_val(rterm2wterm(RTERM,BASE)) -#define internal_ref_val_rel(RTERM,BASE) internal_ref_val(rterm2wterm(RTERM,BASE)) - -#define external_thing_ptr_rel(RTERM, BASE) external_thing_ptr(rterm2wterm(RTERM, BASE)) -#define external_data_words_rel(RTERM,BASE) external_data_words(rterm2wterm(RTERM,BASE)) - -#define external_port_node_rel(RTERM,BASE) external_port_node(rterm2wterm(RTERM,BASE)) -#define external_port_data_rel(RTERM,BASE) external_port_data(rterm2wterm(RTERM,BASE)) - -#define is_external_pid_rel(RTERM,BASE) is_external_pid(rterm2wterm(RTERM,BASE)) -#define external_pid_node_rel(RTERM,BASE) external_pid_node(rterm2wterm(RTERM,BASE)) -#define external_pid_data_rel(RTERM,BASE) external_pid_data(rterm2wterm(RTERM,BASE)) - -#define is_binary_rel(RTERM,BASE) is_binary(rterm2wterm(RTERM,BASE)) -#define is_float_rel(RTERM,BASE) is_float(rterm2wterm(RTERM,BASE)) -#define is_fun_rel(RTERM,BASE) is_fun(rterm2wterm(RTERM,BASE)) -#define is_big_rel(RTERM,BASE) is_big(rterm2wterm(RTERM,BASE)) -#define is_export_rel(RTERM,BASE) is_export(rterm2wterm(RTERM,BASE)) -#define is_tuple_rel(RTERM,BASE) is_tuple(rterm2wterm(RTERM,BASE)) - -#define GET_DOUBLE_REL(RTERM, f, BASE) GET_DOUBLE(rterm2wterm(RTERM,BASE), f) - -#define ref_thing_ptr_rel(RTERM,BASE) ref_thing_ptr(rterm2wterm(RTERM,BASE)) -#define is_internal_ref_rel(RTERM,BASE) is_internal_ref(rterm2wterm(RTERM,BASE)) -#define is_external_rel(RTERM,BASE) is_external(rterm2wterm(RTERM,BASE)) -#define is_external_port_rel(RTERM,BASE) is_external_port(rterm2wterm(RTERM,BASE)) -#define is_external_ref_rel(RTERM,BASE) is_external_ref(rterm2wterm(RTERM,BASE)) - -#define external_node_rel(RTERM,BASE) external_node(rterm2wterm(RTERM,BASE)) - - -#if HALFWORD_HEAP -ERTS_GLB_INLINE int is_same(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base); - -#if ERTS_GLB_INLINE_INCL_FUNC_DEF -ERTS_GLB_INLINE int is_same(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base) -{ - /* If bases differ, assume a and b are on different "heaps", - ie can only be same if immed */ - ASSERT(a_base == b_base || is_immed(a) || is_immed(b) - || rterm2wterm(a,a_base) != rterm2wterm(b,b_base)); - - return a == b && (a_base == b_base || is_immed(a)); -} -#endif - -#else /* !HALFWORD_HEAP */ -#define is_same(A,A_BASE,B,B_BASE) ((A)==(B)) -#endif +#define is_same(A,B) ((A)==(B)) #endif /* __ERL_TERM_H */ diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h index 5347979372..34f91e2ec8 100644 --- a/erts/emulator/beam/erl_threads.h +++ b/erts/emulator/beam/erl_threads.h @@ -649,6 +649,7 @@ ERTS_GLB_INLINE void erts_tsd_set(erts_tsd_key_t key, void *value); ERTS_GLB_INLINE void * erts_tsd_get(erts_tsd_key_t key); ERTS_GLB_INLINE erts_tse_t *erts_tse_fetch(void); ERTS_GLB_INLINE void erts_tse_return(erts_tse_t *ep); +ERTS_GLB_INLINE void erts_tse_prepare_timed(erts_tse_t *ep); ERTS_GLB_INLINE void erts_tse_set(erts_tse_t *ep); ERTS_GLB_INLINE void erts_tse_reset(erts_tse_t *ep); ERTS_GLB_INLINE int erts_tse_wait(erts_tse_t *ep); @@ -3461,6 +3462,15 @@ ERTS_GLB_INLINE void erts_tse_return(erts_tse_t *ep) #endif } +ERTS_GLB_INLINE void erts_tse_prepare_timed(erts_tse_t *ep) +{ +#ifdef USE_THREADS + int res = ethr_event_prepare_timed(&((ethr_ts_event *) ep)->event); + if (res != 0) + erts_thr_fatal_error(res, "prepare timed"); +#endif +} + ERTS_GLB_INLINE void erts_tse_set(erts_tse_t *ep) { #ifdef USE_THREADS diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index e1b03a057f..e9dd96efc4 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -1677,12 +1677,7 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, args = transformed_args; if (is_internal_port(*tracer_pid)) { -#if HEAP_ON_C_STACK Eterm local_heap[64+MAX_ARG]; -#else - Eterm *local_heap = erts_alloc(ERTS_ALC_T_TEMP_TERM, - sizeof(Eterm)*(64+MAX_ARG)); -#endif hp = local_heap; if (!erts_is_valid_tracer_port(*tracer_pid)) { @@ -1697,9 +1692,6 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, if (is_not_nil(tracee)) erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL_MINOR); #endif -#if !HEAP_ON_C_STACK - erts_free(ERTS_ALC_T_TEMP_TERM,local_heap); -#endif UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); return 0; } @@ -1727,9 +1719,6 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, ERTS_PAM_TMP_RESULT, &return_flags); if (is_non_value(pam_result)) { erts_match_set_release_result(p); -#if !HEAP_ON_C_STACK - erts_free(ERTS_ALC_T_TEMP_TERM,local_heap); -#endif UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); return 0; } @@ -1738,9 +1727,6 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, /* Meta trace */ if (pam_result == am_false) { erts_match_set_release_result(p); -#if !HEAP_ON_C_STACK - erts_free(ERTS_ALC_T_TEMP_TERM,local_heap); -#endif UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); return return_flags; } @@ -1748,17 +1734,11 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, /* Non-meta trace */ if (*tracee_flags & F_TRACE_SILENT) { erts_match_set_release_result(p); -#if !HEAP_ON_C_STACK - erts_free(ERTS_ALC_T_TEMP_TERM,local_heap); -#endif UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); return 0; } if (pam_result == am_false) { erts_match_set_release_result(p); -#if !HEAP_ON_C_STACK - erts_free(ERTS_ALC_T_TEMP_TERM,local_heap); -#endif UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); return return_flags; } @@ -1802,9 +1782,6 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, send_to_port(p, mess, tracer_pid, tracee_flags); erts_smp_mtx_unlock(&smq_mtx); erts_match_set_release_result(p); -#if !HEAP_ON_C_STACK - erts_free(ERTS_ALC_T_TEMP_TERM,local_heap); -#endif UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); return *tracer_pid == NIL ? 0 : return_flags; @@ -1823,7 +1800,6 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, #ifdef DEBUG Eterm* limit; #endif - ASSERT(is_internal_pid(*tracer_pid)); tracer = erts_pid2proc(p, ERTS_PROC_LOCK_MAIN, @@ -3297,8 +3273,6 @@ sys_msg_dispatcher_func(void *unused) if (erts_thr_progress_update(NULL)) erts_thr_progress_leader_update(NULL); - ERTS_SCHED_FAIR_YIELD(); - #ifdef DEBUG_PRINTOUTS print_msg_type(smqp); #endif @@ -3453,9 +3427,6 @@ static void init_sys_msg_dispatcher(void) { erts_smp_thr_opts_t thr_opts = ERTS_SMP_THR_OPTS_DEFAULT_INITER; -#ifdef __OSE__ - thr_opts.coreNo = 0; -#endif thr_opts.detached = 1; thr_opts.name = "sys_msg_dispatcher"; init_smq_element_alloc(); @@ -3463,7 +3434,6 @@ init_sys_msg_dispatcher(void) sys_message_queue_end = NULL; erts_smp_cnd_init(&smq_cnd); erts_smp_mtx_init(&smq_mtx, "sys_msg_q"); - erts_smp_thr_create(&sys_msg_dispatcher_tid, sys_msg_dispatcher_func, NULL, diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h index 6dab3bf297..4058d63eaf 100644 --- a/erts/emulator/beam/erl_utils.h +++ b/erts/emulator/beam/erl_utils.h @@ -157,32 +157,15 @@ void erts_init_utils_mem(void); erts_dsprintf_buf_t *erts_create_tmp_dsbuf(Uint); void erts_destroy_tmp_dsbuf(erts_dsprintf_buf_t *); -#if HALFWORD_HEAP -int eq_rel(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base); -# define eq(A,B) eq_rel(A,NULL,B,NULL) -#else int eq(Eterm, Eterm); -# define eq_rel(A,A_BASE,B,B_BASE) eq(A,B) -#endif #define EQ(x,y) (((x) == (y)) || (is_not_both_immed((x),(y)) && eq((x),(y)))) -#if HALFWORD_HEAP -Sint erts_cmp_rel_opt(Eterm, Eterm*, Eterm, Eterm*, int, int); -#define cmp_rel(A,A_BASE,B,B_BASE) erts_cmp_rel_opt(A,A_BASE,B,B_BASE,0,0) -#define cmp_rel_term(A,A_BASE,B,B_BASE) erts_cmp_rel_opt(A,A_BASE,B,B_BASE,1,0) -#define CMP(A,B) erts_cmp_rel_opt(A,NULL,B,NULL,0,0) -#define CMP_TERM(A,B) erts_cmp_rel_opt(A,NULL,B,NULL,1,0) -#define CMP_EQ_ONLY(A,B) erts_cmp_rel_opt(A,NULL,B,NULL,0,1) -#else Sint erts_cmp(Eterm, Eterm, int, int); Sint cmp(Eterm a, Eterm b); -#define cmp_rel(A,A_BASE,B,B_BASE) erts_cmp(A,B,0,0) -#define cmp_rel_term(A,A_BASE,B,B_BASE) erts_cmp(A,B,1,0) #define CMP(A,B) erts_cmp(A,B,0,0) #define CMP_TERM(A,B) erts_cmp(A,B,1,0) #define CMP_EQ_ONLY(A,B) erts_cmp(A,B,0,1) -#endif #define cmp_lt(a,b) (CMP((a),(b)) < 0) #define cmp_le(a,b) (CMP((a),(b)) <= 0) diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h index 8948ca2ea9..98f27a1725 100644 --- a/erts/emulator/beam/erl_vm.h +++ b/erts/emulator/beam/erl_vm.h @@ -40,14 +40,6 @@ #define MAX_ARG 255 /* Max number of arguments allowed */ #define MAX_REG 1024 /* Max number of x(N) registers used */ -/* Scheduler stores data for temporary heaps if - !HEAP_ON_C_STACK. Macros (*TmpHeap*) in global.h selects if we put temporary - heap data on the C stack or if we use the buffers in the scheduler data. */ -#define TMP_HEAP_SIZE 128 /* Number of Eterm in the schedulers - small heap for transient heap data */ -#define ERL_ARITH_TMP_HEAP_SIZE 4 /* as does erl_arith... */ -#define BEAM_EMU_TMP_HEAP_SIZE 2 /* and beam_emu... */ - /* * The new arithmetic operations need some extra X registers in the register array. * so does the gc_bif's (i_gc_bif3 need 3 extra). @@ -117,12 +109,8 @@ #define HeapWordsLeft(p) (HEAP_LIMIT(p) - HEAP_TOP(p)) #if defined(DEBUG) || defined(CHECK_FOR_HOLES) -#if HALFWORD_HEAP -# define ERTS_HOLE_MARKER (0xdeadbeef) -#else # define ERTS_HOLE_MARKER (((0xdeadbeef << 24) << 8) | 0xdeadbeef) -#endif -#endif +#endif /* egil: 32-bit ? */ /* * Allocate heap memory on the ordinary heap, NEVER in a heap @@ -153,6 +141,7 @@ typedef struct op_entry { char* name; /* Name of instruction. */ Uint32 mask[3]; /* Signature mask. */ + unsigned involves_r; /* Needs special attention when matching. */ int sz; /* Number of loaded words. */ char* pack; /* Instructions for packing engine. */ char* sign; /* Signature string. */ diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index c6d7e3fcc5..c3e93d1ad2 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -2339,10 +2339,6 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Eterm val; FloatDef f; Sint r = 0; -#if HALFWORD_HEAP - UWord wobj; -#endif - if (ctx) { WSTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK); @@ -2362,11 +2358,8 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, outer_loop: while (!WSTACK_ISEMPTY(s)) { -#if HALFWORD_HEAP - obj = (Eterm) (wobj = WSTACK_POP(s)); -#else obj = WSTACK_POP(s); -#endif + switch (val = WSTACK_POP(s)) { case ENC_TERM: break; @@ -2384,11 +2377,7 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, break; case ENC_PATCH_FUN_SIZE: { -#if HALFWORD_HEAP - byte* size_p = (byte *) wobj; -#else byte* size_p = (byte *) obj; -#endif put_int32(ep - size_p, size_p); } goto outer_loop; @@ -2435,21 +2424,13 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, case ENC_LAST_ARRAY_ELEMENT: /* obj is the tuple */ { -#if HALFWORD_HEAP - Eterm* ptr = (Eterm *) wobj; -#else Eterm* ptr = (Eterm *) obj; -#endif obj = *ptr; } break; default: /* ENC_LAST_ARRAY_ELEMENT+1 and upwards */ { -#if HALFWORD_HEAP - Eterm* ptr = (Eterm *) wobj; -#else Eterm* ptr = (Eterm *) obj; -#endif obj = *ptr++; WSTACK_PUSH2(s, val-1, (UWord)ptr); } @@ -2977,7 +2958,7 @@ dec_term(ErtsDistExternal *edep, case B2TDecodeList: objp = next - 2; while (n > 0) { - objp[0] = (Eterm) COMPRESS_POINTER(next); + objp[0] = (Eterm) next; objp[1] = make_list(next); next = objp; objp -= 2; @@ -2988,7 +2969,7 @@ dec_term(ErtsDistExternal *edep, case B2TDecodeTuple: objp = next - 1; while (n-- > 0) { - objp[0] = (Eterm) COMPRESS_POINTER(next); + objp[0] = (Eterm) next; next = objp; objp--; } @@ -3041,7 +3022,7 @@ dec_term(ErtsDistExternal *edep, while (next != NULL) { objp = next; - next = (Eterm *) EXPAND_POINTER(*objp); + next = (Eterm *) *objp; switch (*ep++) { case INTEGER_EXT: @@ -3049,7 +3030,7 @@ dec_term(ErtsDistExternal *edep, Sint sn = get_int32(ep); ep += 4; -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) *objp = make_small(sn); #else if (MY_IS_SSMALL(sn)) { @@ -3172,7 +3153,7 @@ dec_term_atom_common: reds -= n; } while (n-- > 0) { - objp[0] = (Eterm) COMPRESS_POINTER(next); + objp[0] = (Eterm) next; next = objp; objp--; } @@ -3190,8 +3171,8 @@ dec_term_atom_common: *objp = make_list(hp); hp += 2 * n; objp = hp - 2; - objp[0] = (Eterm) COMPRESS_POINTER((objp+1)); - objp[1] = (Eterm) COMPRESS_POINTER(next); + objp[0] = (Eterm) (objp+1); + objp[1] = (Eterm) next; next = objp; objp -= 2; n--; @@ -3204,7 +3185,7 @@ dec_term_atom_common: reds -= n; } while (n > 0) { - objp[0] = (Eterm) COMPRESS_POINTER(next); + objp[0] = (Eterm) next; objp[1] = make_list(next); next = objp; objp -= 2; @@ -3371,7 +3352,7 @@ dec_term_atom_common: RefThing *rtp = (RefThing *) hp; ref_num = (Uint32 *) (hp + REF_THING_HEAD_SIZE); -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) hp += REF_THING_HEAD_SIZE + ref_words/2 + 1; rtp->header = make_ref_thing_header(ref_words/2 + 1); #else @@ -3382,13 +3363,13 @@ dec_term_atom_common: } else { ExternalThing *etp = (ExternalThing *) hp; -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) hp += EXTERNAL_THING_HEAD_SIZE + ref_words/2 + 1; #else hp += EXTERNAL_THING_HEAD_SIZE + ref_words; #endif -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) etp->header = make_external_ref_header(ref_words/2 + 1); #else etp->header = make_external_ref_header(ref_words); @@ -3401,7 +3382,7 @@ dec_term_atom_common: ref_num = &(etp->data.ui32[0]); } -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) *(ref_num++) = ref_words /* 32-bit arity */; #endif ref_num[0] = r0; @@ -3409,7 +3390,7 @@ dec_term_atom_common: ref_num[i] = get_int32(ep); ep += 4; } -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) if ((1 + ref_words) % 2) ref_num[ref_words] = 0; #endif @@ -3561,12 +3542,7 @@ dec_term_atom_common: } *objp = make_export(hp); *hp++ = HEADER_EXPORT; -#if HALFWORD_HEAP - *((UWord *) (UWord) hp) = (UWord) erts_export_get_or_make_stub(mod, name, arity); - hp += 2; -#else *hp++ = (Eterm) erts_export_get_or_make_stub(mod, name, arity); -#endif break; } break; @@ -3600,7 +3576,7 @@ dec_term_atom_common: * The list of maps is for later validation. */ - mp->thing_word = (Eterm) COMPRESS_POINTER(maps_list); + mp->thing_word = (Eterm) maps_list; maps_list = (Eterm *) mp; mp->size = size; @@ -3608,8 +3584,8 @@ dec_term_atom_common: *objp = make_flatmap(mp); for (n = size; n; n--) { - *vptr = (Eterm) COMPRESS_POINTER(next); - *kptr = (Eterm) COMPRESS_POINTER(vptr); + *vptr = (Eterm) next; + *kptr = (Eterm) vptr; next = kptr; vptr--; kptr--; @@ -3623,8 +3599,8 @@ dec_term_atom_common: hamt->leaf_array = hp; for (n = size; n; n--) { - CDR(hp) = (Eterm) COMPRESS_POINTER(next); - CAR(hp) = (Eterm) COMPRESS_POINTER(&CDR(hp)); + CDR(hp) = (Eterm) next; + CAR(hp) = (Eterm) &CDR(hp); next = &CAR(hp); hp += 2; } @@ -3701,11 +3677,11 @@ dec_term_atom_common: /* Environment */ for (i = num_free-1; i >= 0; i--) { - funp->env[i] = (Eterm) COMPRESS_POINTER(next); + funp->env[i] = (Eterm) next; next = funp->env + i; } /* Creator */ - funp->creator = (Eterm) COMPRESS_POINTER(next); + funp->creator = (Eterm) next; next = &(funp->creator); break; } @@ -3774,7 +3750,7 @@ dec_term_atom_common: /* Environment */ for (i = num_free-1; i >= 0; i--) { - funp->env[i] = (Eterm) COMPRESS_POINTER(next); + funp->env[i] = (Eterm) next; next = funp->env + i; } break; @@ -3870,11 +3846,11 @@ dec_term_atom_common: */ while (maps_list) { - next = (Eterm *)(EXPAND_POINTER(*maps_list)); + next = (Eterm *) *maps_list; *maps_list = MAP_HEADER_FLATMAP; if (!erts_validate_and_sort_flatmap((flatmap_t*)maps_list)) goto error; - maps_list = next; + maps_list = next; } ASSERT(hp <= factory->hp_end @@ -3900,7 +3876,7 @@ dec_term_atom_common: PSTACK_DESTROY(hamt_array); } - ASSERT((Eterm*)EXPAND_POINTER(*dbg_resultp) != NULL); + ASSERT((Eterm*)*dbg_resultp != NULL); if (ctx) { ctx->state = B2TDone; @@ -4192,11 +4168,7 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, case EXPORT_DEF: { Export* ep = *((Export **) (export_val(obj) + 1)); -#if HALFWORD_HEAP - result += 2; -#else result += 1; -#endif result += encode_size_struct2(acmp, ep->code[0], dflags); result += encode_size_struct2(acmp, ep->code[1], dflags); result += encode_size_struct2(acmp, make_small(ep->code[2]), dflags); @@ -4311,7 +4283,7 @@ init_done: switch (tag) { case INTEGER_EXT: SKIP(4); -#if !defined(ARCH_64) || HALFWORD_HEAP +#if !defined(ARCH_64) heap_size += BIG_UINT_HEAP_SIZE; #endif break; @@ -4400,7 +4372,7 @@ init_done: ep += 2; atom_extra_skip = 1 + 4*id_words; /* In case it is an external ref */ -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) heap_size += EXTERNAL_THING_HEAD_SIZE + id_words/2 + 1; #else heap_size += EXTERNAL_THING_HEAD_SIZE + id_words; @@ -4486,11 +4458,7 @@ init_done: break; case EXPORT_EXT: terms += 3; -#if HALFWORD_HEAP - heap_size += 3; -#else heap_size += 2; -#endif break; case NEW_FUN_EXT: { diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index ec9296d034..b4d02dd1dd 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -310,9 +310,6 @@ typedef union { typedef struct proc_bin { Eterm thing_word; /* Subtag REFC_BINARY_SUBTAG. */ Uint size; /* Binary size in bytes. */ -#if HALFWORD_HEAP - void* dummy_ptr_padding__; -#endif struct erl_off_heap_header *next; Binary *val; /* Pointer to Binary structure. */ byte *bytes; /* Pointer to the actual data bytes. */ @@ -957,31 +954,12 @@ __decl_noreturn void __noreturn erl_exit_flush_async(int n, char*, ...); void erl_error(char*, va_list); /* copy.c */ -Eterm copy_object(Eterm, Process*); - -#if HALFWORD_HEAP -Uint size_object_rel(Eterm, Eterm*); -# define size_object(A) size_object_rel(A,NULL) - -Eterm copy_struct_rel(Eterm, Uint, Eterm**, ErlOffHeap*, Eterm* src_base, Eterm* dst_base); -# define copy_struct(OBJ,SZ,HPP,OH) copy_struct_rel(OBJ,SZ,HPP,OH, NULL,NULL) - -Eterm copy_shallow_rel(Eterm*, Uint, Eterm**, ErlOffHeap*, Eterm* src_base); -# define copy_shallow(A,B,C,D) copy_shallow_rel(A,B,C,D,NULL) - -#else /* !HALFWORD_HEAP */ +Eterm copy_object_x(Eterm, Process*, Uint); +#define copy_object(Term, Proc) copy_object_x(Term,Proc,0) Uint size_object(Eterm); -# define size_object_rel(A,B) size_object(A) - Eterm copy_struct(Eterm, Uint, Eterm**, ErlOffHeap*); -# define copy_struct_rel(OBJ,SZ,HPP,OH, SB,DB) copy_struct(OBJ,SZ,HPP,OH) - Eterm copy_shallow(Eterm*, Uint, Eterm**, ErlOffHeap*); -# define copy_shallow_rel(A,B,C,D, BASE) copy_shallow(A,B,C,D) - -#endif - void move_multi_frags(Eterm** hpp, ErlOffHeap*, ErlHeapFragment* first, Eterm* refs, unsigned nrefs); @@ -1180,7 +1158,7 @@ void bin_write(int, void*, byte*, size_t); int intlist_to_buf(Eterm, char*, int); /* most callers pass plain char*'s */ struct Sint_buf { -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) char s[22]; #else char s[12]; @@ -1466,69 +1444,16 @@ erts_alloc_message_heap(Uint size, #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ -#if !HEAP_ON_C_STACK -# if defined(DEBUG) -# define DeclareTmpHeap(VariableName,Size,Process) \ - Eterm *VariableName = erts_debug_allocate_tmp_heap(Size,Process) -# define DeclareTypedTmpHeap(Type,VariableName,Process) \ - Type *VariableName = (Type *) erts_debug_allocate_tmp_heap(sizeof(Type)/sizeof(Eterm),Process) -# define DeclareTmpHeapNoproc(VariableName,Size) \ - Eterm *VariableName = erts_debug_allocate_tmp_heap(Size,NULL) -# define UseTmpHeap(Size,Proc) \ - do { \ - erts_debug_use_tmp_heap((Size),(Proc)); \ - } while (0) -# define UnUseTmpHeap(Size,Proc) \ - do { \ - erts_debug_unuse_tmp_heap((Size),(Proc)); \ - } while (0) -# define UseTmpHeapNoproc(Size) \ - do { \ - erts_debug_use_tmp_heap(Size,NULL); \ - } while (0) -# define UnUseTmpHeapNoproc(Size) \ - do { \ - erts_debug_unuse_tmp_heap(Size,NULL); \ - } while (0) -# else -# define DeclareTmpHeap(VariableName,Size,Process) \ - Eterm *VariableName = (ERTS_PROC_GET_SCHDATA(Process)->tmp_heap)+(ERTS_PROC_GET_SCHDATA(Process)->num_tmp_heap_used) -# define DeclareTypedTmpHeap(Type,VariableName,Process) \ - Type *VariableName = (Type *) (ERTS_PROC_GET_SCHDATA(Process)->tmp_heap)+(ERTS_PROC_GET_SCHDATA(Process)->num_tmp_heap_used) -# define DeclareTmpHeapNoproc(VariableName,Size) \ - Eterm *VariableName = (erts_get_scheduler_data()->tmp_heap)+(erts_get_scheduler_data()->num_tmp_heap_used) -# define UseTmpHeap(Size,Proc) \ - do { \ - ERTS_PROC_GET_SCHDATA(Proc)->num_tmp_heap_used += (Size); \ - } while (0) -# define UnUseTmpHeap(Size,Proc) \ - do { \ - ERTS_PROC_GET_SCHDATA(Proc)->num_tmp_heap_used -= (Size); \ - } while (0) -# define UseTmpHeapNoproc(Size) \ - do { \ - erts_get_scheduler_data()->num_tmp_heap_used += (Size); \ - } while (0) -# define UnUseTmpHeapNoproc(Size) \ - do { \ - erts_get_scheduler_data()->num_tmp_heap_used -= (Size); \ - } while (0) - - -# endif - -#else -# define DeclareTmpHeap(VariableName,Size,Process) \ +#define DeclareTmpHeap(VariableName,Size,Process) \ Eterm VariableName[Size] -# define DeclareTypedTmpHeap(Type,VariableName,Process) \ +#define DeclareTypedTmpHeap(Type,VariableName,Process) \ Type VariableName[1] -# define DeclareTmpHeapNoproc(VariableName,Size) \ +#define DeclareTmpHeapNoproc(VariableName,Size) \ Eterm VariableName[Size] -# define UseTmpHeap(Size,Proc) /* Nothing */ -# define UnUseTmpHeap(Size,Proc) /* Nothing */ -# define UseTmpHeapNoproc(Size) /* Nothing */ -# define UnUseTmpHeapNoproc(Size) /* Nothing */ -#endif /* HEAP_ON_C_STACK */ +#define UseTmpHeap(Size,Proc) /* Nothing */ +#define UnUseTmpHeap(Size,Proc) /* Nothing */ +#define UseTmpHeapNoproc(Size) /* Nothing */ +#define UnUseTmpHeapNoproc(Size) /* Nothing */ ERTS_GLB_INLINE void dtrace_pid_str(Eterm pid, char *process_buf); ERTS_GLB_INLINE void dtrace_proc_str(Process *process, char *process_buf); diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index 900616c981..fdd26fcc4b 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -52,9 +52,7 @@ #include "erl_hl_timer.h" extern ErlDrvEntry fd_driver_entry; -#ifndef __OSE__ extern ErlDrvEntry vanilla_driver_entry; -#endif extern ErlDrvEntry spawn_driver_entry; extern ErlDrvEntry *driver_tab[]; /* table of static drivers, only used during initialization */ @@ -1105,7 +1103,7 @@ io_list_vec_len(Eterm obj, int* vsize, Uint* csize, Uint p_v_size = 0; Uint p_c_size = 0; Uint p_in_clist = 0; - Uint total; /* Uint due to halfword emulator */ + Uint total; goto L_jump_start; /* avoid a push */ @@ -2794,9 +2792,7 @@ void erts_init_io(int port_tab_size, erts_smp_rwmtx_rwlock(&erts_driver_list_lock); init_driver(&fd_driver, &fd_driver_entry, NULL); -#ifndef __OSE__ init_driver(&vanilla_driver, &vanilla_driver_entry, NULL); -#endif init_driver(&spawn_driver, &spawn_driver_entry, NULL); erts_init_static_drivers(); for (dp = driver_tab; *dp != NULL; dp++) @@ -4091,10 +4087,10 @@ erts_port_control(Process* c_p, binp = NULL; if (is_binary(data) && binary_bitoffset(data) == 0) { - Eterm *ebinp = binary_val_rel(data, NULL); + Eterm *ebinp = binary_val(data); ASSERT(!tmp_alloced); if (*ebinp == HEADER_SUB_BIN) - ebinp = binary_val_rel(((ErlSubBin *) ebinp)->orig, NULL); + ebinp = binary_val(((ErlSubBin *) ebinp)->orig); if (*ebinp != HEADER_PROC_BIN) copy = 1; else { @@ -5248,25 +5244,17 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) break; case ERL_DRV_INT: /* signed int argument */ ERTS_DDT_CHK_ENOUGH_ARGS(1); -#if HALFWORD_HEAP - erts_bld_sint64(NULL, &need, (Sint64)ptr[0]); -#else /* check for bignum */ if (!IS_SSMALL((Sint)ptr[0])) need += BIG_UINT_HEAP_SIZE; /* use small_to_big */ -#endif ptr++; depth++; break; case ERL_DRV_UINT: /* unsigned int argument */ ERTS_DDT_CHK_ENOUGH_ARGS(1); -#if HALFWORD_HEAP - erts_bld_uint64(NULL, &need, (Uint64)ptr[0]); -#else /* check for bignum */ if (!IS_USMALL(0, (Uint)ptr[0])) need += BIG_UINT_HEAP_SIZE; /* use small_to_big */ -#endif ptr++; depth++; break; @@ -5465,10 +5453,6 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) break; case ERL_DRV_INT: /* signed int argument */ -#if HALFWORD_HEAP - erts_reserve_heap(&factory, BIG_NEED_SIZE(2)); - mess = erts_bld_sint64(&factory.hp, NULL, (Sint64)ptr[0]); -#else erts_reserve_heap(&factory, BIG_UINT_HEAP_SIZE); if (IS_SSMALL((Sint)ptr[0])) mess = make_small((Sint)ptr[0]); @@ -5476,15 +5460,10 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) mess = small_to_big((Sint)ptr[0], factory.hp); factory.hp += BIG_UINT_HEAP_SIZE; } -#endif ptr++; break; case ERL_DRV_UINT: /* unsigned int argument */ -#if HALFWORD_HEAP - erts_reserve_heap(&factory, BIG_NEED_FOR_BITS(64)); - mess = erts_bld_uint64(&factory.hp, NULL, (Uint64)ptr[0]); -#else erts_reserve_heap(&factory, BIG_UINT_HEAP_SIZE); if (IS_USMALL(0, (Uint)ptr[0])) mess = make_small((Uint)ptr[0]); @@ -5492,7 +5471,6 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) mess = uint_to_big((Uint)ptr[0], factory.hp); factory.hp += BIG_UINT_HEAP_SIZE; } -#endif ptr++; break; @@ -6809,7 +6787,7 @@ int driver_monitor_process(ErlDrvPort drvport, { Port *prt; int ret; -#if !HEAP_ON_C_STACK || (defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)) +#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) ErtsSchedulerData *sched = erts_get_scheduler_data(); #endif @@ -6820,16 +6798,6 @@ int driver_monitor_process(ErlDrvPort drvport, /* Now (in SMP) we should have either the port lock (if we have a scheduler) or the port data lock (if we're a driver thread) */ ERTS_SMP_LC_ASSERT((sched != NULL || prt->port_data_lock)); - -#if !HEAP_ON_C_STACK - if (!sched) { - /* Need a separate allocation for the ref :( */ - Eterm *buf = erts_alloc(ERTS_ALC_T_TEMP_TERM, - sizeof(Eterm)*REF_THING_SIZE); - ret = do_driver_monitor_process(prt,buf,process,monitor); - erts_free(ERTS_ALC_T_TEMP_TERM,buf); - } else -#endif { DeclareTmpHeapNoproc(buf,REF_THING_SIZE); UseTmpHeapNoproc(REF_THING_SIZE); @@ -6882,7 +6850,7 @@ int driver_demonitor_process(ErlDrvPort drvport, { Port *prt; int ret; -#if !HEAP_ON_C_STACK || (defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)) +#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) ErtsSchedulerData *sched = erts_get_scheduler_data(); #endif @@ -6893,15 +6861,6 @@ int driver_demonitor_process(ErlDrvPort drvport, /* Now we should have either the port lock (if we have a scheduler) or the port data lock (if we're a driver thread) */ ERTS_SMP_LC_ASSERT((sched != NULL || prt->port_data_lock)); -#if !HEAP_ON_C_STACK - if (!sched) { - /* Need a separate allocation for the ref :( */ - Eterm *buf = erts_alloc(ERTS_ALC_T_TEMP_TERM, - sizeof(Eterm)*REF_THING_SIZE); - ret = do_driver_demonitor_process(prt,buf,monitor); - erts_free(ERTS_ALC_T_TEMP_TERM,buf); - } else -#endif { DeclareTmpHeapNoproc(buf,REF_THING_SIZE); UseTmpHeapNoproc(REF_THING_SIZE); @@ -6937,7 +6896,7 @@ ErlDrvTermData driver_get_monitored_process(ErlDrvPort drvport, { Port *prt; ErlDrvTermData ret; -#if !HEAP_ON_C_STACK || (defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)) +#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) ErtsSchedulerData *sched = erts_get_scheduler_data(); #endif @@ -6948,16 +6907,6 @@ ErlDrvTermData driver_get_monitored_process(ErlDrvPort drvport, /* Now we should have either the port lock (if we have a scheduler) or the port data lock (if we're a driver thread) */ ERTS_SMP_LC_ASSERT((sched != NULL || prt->port_data_lock)); - -#if !HEAP_ON_C_STACK - if (!sched) { - /* Need a separate allocation for the ref :( */ - Eterm *buf = erts_alloc(ERTS_ALC_T_TEMP_TERM, - sizeof(Eterm)*REF_THING_SIZE); - ret = do_driver_get_monitored_process(prt,buf,monitor); - erts_free(ERTS_ALC_T_TEMP_TERM,buf); - } else -#endif { DeclareTmpHeapNoproc(buf,REF_THING_SIZE); UseTmpHeapNoproc(REF_THING_SIZE); diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab index 1d32e72247..46fefb88af 100644 --- a/erts/emulator/beam/ops.tab +++ b/erts/emulator/beam/ops.tab @@ -39,8 +39,8 @@ too_old_compiler | never() => # necessary.) Since the instructions don't work correctly in R12B, simply # refuse to load the module. -func_info M=a a==am_module_info A=u==0 | label L | move n r => too_old_compiler -func_info M=a a==am_module_info A=u==1 | label L | move n r => too_old_compiler +func_info M=a a==am_module_info A=u==0 | label L | move n x==0 => too_old_compiler +func_info M=a a==am_module_info A=u==1 | label L | move n x==0 => too_old_compiler # The undocumented and unsupported guard BIF is_constant/1 was removed # in R13. The is_constant/2 operation is marked as obsolete in genop.tab, @@ -76,17 +76,6 @@ return # with the following call instruction, we need to make sure that # there is no line/1 instruction between the move and the call. # - -move S r | line Loc | call_ext Ar Func => \ - line Loc | move S r | call_ext Ar Func -move S r | line Loc | call_ext_last Ar Func=u$is_bif D => \ - line Loc | move S r | call_ext_last Ar Func D -move S r | line Loc | call_ext_only Ar Func=u$is_bif => \ - line Loc | move S r | call_ext_only Ar Func -move S r | line Loc | call Ar Func => \ - line Loc | move S r | call Ar Func - -# # A tail-recursive call to an external function (non-BIF) will # never be saved on the stack, so there is no reason to keep # the line instruction. (The compiler did not remove the line @@ -94,10 +83,14 @@ move S r | line Loc | call Ar Func => \ # BIFs and ordinary Erlang functions.) # -line Loc | call_ext_last Ar Func=u$is_not_bif D => \ - call_ext_last Ar Func D -line Loc | call_ext_only Ar Func=u$is_not_bif => \ - call_ext_only Ar Func +move S X0=x==0 | line Loc | call_ext Ar Func => \ + line Loc | move S X0 | call_ext Ar Func +move S X0=x==0 | line Loc | call_ext_last Ar Func=u$is_not_bif D => \ + move S X0 | call_ext_last Ar Func D +move S X0=x==0 | line Loc | call_ext_only Ar Func=u$is_not_bif => \ + move S X0 | call_ext_only Ar Func +move S X0=x==0 | line Loc | call Ar Func => \ + line Loc | move S X0 | call Ar Func line Loc | func_info M F A => func_info M F A | line Loc @@ -167,31 +160,24 @@ is_tuple Fail=f S | select_tuple_arity S=d Fail=f Size=u Rest=* => \ select_tuple_arity S=d Fail=f Size=u Rest=* => \ gen_select_tuple_arity(S, Fail, Size, Rest) -i_select_val_bins r f I i_select_val_bins x f I i_select_val_bins y f I -i_select_val_lins r f I i_select_val_lins x f I i_select_val_lins y f I -i_select_val2 r f c c f f i_select_val2 x f c c f f i_select_val2 y f c c f f -i_select_tuple_arity r f I i_select_tuple_arity x f I i_select_tuple_arity y f I -i_select_tuple_arity2 r f A A f f i_select_tuple_arity2 x f A A f f i_select_tuple_arity2 y f A A f f -i_jump_on_val_zero r f I i_jump_on_val_zero x f I i_jump_on_val_zero y f I -i_jump_on_val r f I I i_jump_on_val x f I I i_jump_on_val y f I I @@ -203,30 +189,21 @@ is_ne_exact L1 S1 S2 | jump Fail | label L2 | same_label(L1, L2) => \ %macro: get_list GetList -pack get_list x x x get_list x x y -get_list x x r get_list x y x get_list x y y -get_list x y r -get_list x r x -get_list x r y get_list y x x get_list y x y -get_list y x r get_list y y x get_list y y y -get_list y y r -get_list y r x -get_list y r y +# The following get_list instructions using x(0) are frequently used. get_list r x x +get_list r r y +get_list x r x get_list r x y -get_list r x r -get_list r y x -get_list r y y get_list r y r -get_list r r x -get_list r r y +get_list r x r # Old-style catch. catch y f @@ -237,33 +214,31 @@ try Y F => catch Y F try_case Y => try_end Y try_end y -try_case_end Literal=q => move Literal x | try_case_end x try_case_end s # Destructive set tuple element -set_tuple_element Lit=q Tuple Pos => move Lit x | set_tuple_element x Tuple Pos set_tuple_element s d P # Get tuple element %macro: i_get_tuple_element GetTupleElement -pack i_get_tuple_element x P x -i_get_tuple_element r P x i_get_tuple_element y P x -i_get_tuple_element x P r -i_get_tuple_element y P r %cold -i_get_tuple_element r P r i_get_tuple_element x P y -i_get_tuple_element r P y i_get_tuple_element y P y %hot +%macro: i_get_tuple_element2 GetTupleElement2 -pack +i_get_tuple_element2 x P x + +%macro: i_get_tuple_element3 GetTupleElement3 -pack +i_get_tuple_element3 x P x + %macro: is_number IsNumber -fail_action %cold -is_number f r is_number f x is_number f y %hot @@ -273,16 +248,12 @@ is_number Fail Literal=q => move Literal x | is_number Fail x jump f -case_end Literal=cq => move Literal x | case_end x -badmatch Literal=cq => move Literal x | badmatch x +case_end NotInX=cy => move NotInX x | case_end x +badmatch NotInX=cy => move NotInX x | badmatch x -case_end r case_end x -case_end y -badmatch r badmatch x -badmatch y if_end raise s s @@ -291,7 +262,7 @@ raise s s badarg j system_limit j -move C=cxy r | jump Lbl => move_jump Lbl C +move C=cxy x==0 | jump Lbl => move_jump Lbl C %macro: move_jump MoveJump -nonext move_jump f n @@ -309,8 +280,6 @@ move_window/6 # x -> y -move S1=r S2=y | move X1=x Y1=y => move2 S1 S2 X1 Y1 - move X1=x Y1=y | move X2=x Y2=y | move X3=x Y3=y | succ(Y1,Y2) | succ(Y2,Y3) => \ move_window X1 X2 X3 Y1 Y3 @@ -323,24 +292,76 @@ move_window X1=x X2=x X3=x X4=x Y1=y Y4=y | move X5=x Y5=y | succ(Y4,Y5) => \ move_window X1=x X2=x X3=x Y1=y Y3=y => move_window3 X1 X2 X3 Y1 move_window X1=x X2=x X3=x X4=x Y1=y Y4=y => move_window4 X1 X2 X3 X4 Y1 +%macro: move_window3 MoveWindow3 -pack +%macro: move_window4 MoveWindow4 -pack +%macro: move_window5 MoveWindow5 -pack + move_window3 x x x y move_window4 x x x x y move_window5 x x x x x y -move X1=x Y1=y | move X2=x Y2=y => move2 X1 Y1 X2 Y2 -move Y1=y X1=x | move Y2=y X2=x => move2 Y1 X1 Y2 X2 -move X1=x X2=x | move X3=x X4=x => move2 X1 X2 X3 X4 +# Swap registers. +move R1=x Tmp=x | move R2=xy R1 | move Tmp R2 => swap_temp R1 R2 Tmp + +swap_temp R1 R2 Tmp | line Loc | apply Live | is_killed(Tmp, Live) => \ + swap R1 R2 | line Loc | apply Live + +swap_temp R1 R2 Tmp | line Loc | call Live Addr | is_killed(Tmp, Live) => \ + swap R1 R2 | line Loc | call Live Addr +swap_temp R1 R2 Tmp | call_only Live Addr | \ + is_killed(Tmp, Live) => swap R1 R2 | call_only Live Addr +swap_temp R1 R2 Tmp | call_last Live Addr D | \ + is_killed(Tmp, Live) => swap R1 R2 | call_last Live Addr D -move S1=x S2=r | move S3=x S4=x => move2 S1 S2 S3 S4 -move S1=x S2=r | move X1=x Y1=y => move2 S1 S2 X1 Y1 -move S1=y S2=r | move X1=x Y1=y => move2 S1 S2 X1 Y1 +swap_temp R1 R2 Tmp | line Loc | call_ext Live Addr | is_killed(Tmp, Live) => \ + swap R1 R2 | line Loc | call_ext Live Addr +swap_temp R1 R2 Tmp | line Loc | call_ext_only Live Addr | \ + is_killed(Tmp, Live) => swap R1 R2 | line Loc | call_ext_only Live Addr +swap_temp R1 R2 Tmp | line Loc | call_ext_last Live Addr D | \ + is_killed(Tmp, Live) => swap R1 R2 | line Loc | call_ext_last Live Addr D -move Y1=y X1=x | move S1=r D1=x => move2 Y1 X1 S1 D1 -move S1=r D1=x | move Y1=y X1=x => move2 S1 D1 Y1 X1 +%macro: swap_temp SwapTemp -pack +swap_temp x x x +swap_temp x y x -move2 X1=x Y1=y X2=x Y2=y | move X3=x Y3=y => move3 X1 Y1 X2 Y2 X3 Y3 -move2 Y1=y X1=x Y2=y X2=x | move Y3=y X3=x => move3 Y1 X1 Y2 X2 Y3 X3 -move2 X1=x X2=x X3=x X4=x | move X5=x X6=x => move3 X1 X2 X3 X4 X5 X6 +%macro: swap Swap -pack +swap x x +swap x y + +move Src=x D1=x | move Src=x D2=x => move_dup Src D1 D2 +move Src=x SD=x | move SD=x D=x => move_dup Src SD D +move Src=x D1=x | move Src=x D2=y => move_dup Src D1 D2 +move Src=y SD=x | move SD=x D=y => move_dup Src SD D +move Src=x SD=x | move SD=x D=y => move_dup Src SD D +move Src=y SD=x | move SD=x D=x => move_dup Src SD D + +move SD=x D=x | move Src=xy SD=x => move_shift Src SD D +move SD=y D=x | move Src=x SD=y => move_shift Src SD D +move SD=x D=y | move Src=x SD=x => move_shift Src SD D + +# The transformations above guarantee that the source for +# the second move is not the same as the destination for +# the first move. That means that we can do the moves in +# parallel (fetch both values, then store them) which could +# be faster. + +move X1=x Y1=y | move X2=x Y2=y => move2_par X1 Y1 X2 Y2 +move Y1=y X1=x | move Y2=y X2=x => move2_par Y1 X1 Y2 X2 + +move X1=x X2=x | move X3=x X4=x => move2_par X1 X2 X3 X4 + +move X1=x X2=x | move X3=x Y1=y => move2_par X1 X2 X3 Y1 + +move S1=x S2=x | move X1=x Y1=y => move2_par S1 S2 X1 Y1 + +move S1=y S2=x | move X1=x Y1=y => move2_par S1 S2 X1 Y1 + +move Y1=y X1=x | move S1=x D1=x => move2_par Y1 X1 S1 D1 +move S1=x D1=x | move Y1=y X1=x => move2_par S1 D1 Y1 X1 + +move2_par X1=x Y1=y X2=x Y2=y | move X3=x Y3=y => move3 X1 Y1 X2 Y2 X3 Y3 +move2_par Y1=y X1=x Y2=y X2=x | move Y3=y X3=x => move3 Y1 X1 Y2 X2 Y3 X3 +move2_par X1=x X2=x X3=x X4=x | move X5=x X6=x => move3 X1 X2 X3 X4 X5 X6 move C=aiq X=x==1 => move_x1 C move C=aiq X=x==2 => move_x2 C @@ -348,21 +369,32 @@ move C=aiq X=x==2 => move_x2 C move_x1 c move_x2 c -%macro: move2 Move2 -pack -move2 x y x y -move2 y x y x -move2 x x x x +%macro: move_shift MoveShift -pack +move_shift x x x +move_shift y x x +move_shift x y x +move_shift x x y + +%macro: move_dup MoveDup -pack +move_dup x x x +move_dup x x y +move_dup y x x +move_dup y x y -move2 x r x x +%macro: move2_par Move2Par -pack -move2 x r x y -move2 r y x y -move2 y r x y +move2_par x y x y +move2_par y x y x +move2_par x x x x -move2 r x y x -move2 y x r x +move2_par x x x y -%macro: move3 Move3 +move2_par y x x y + +move2_par x x y x +move2_par y x x x + +%macro: move3 Move3 -pack move3 x y x y x y move3 y x y x y x move3 x x x x x x @@ -375,20 +407,22 @@ move S=c D=y => move S x | move x D %macro:move Move -pack -gen_dest move x x move x y -move x r move y x -move y r -move r x -move r y -move c r move c x move n x -move n r move y y +# The following move instructions using x(0) are frequently used. + +move x r +move r x +move y r +move c r +move r y + # Receive operations. -loop_rec Fail Src | smp_mark_target_label(Fail) => i_loop_rec Fail Src +loop_rec Fail x==0 | smp_mark_target_label(Fail) => i_loop_rec Fail label L | wait_timeout Fail Src | smp_already_locked(L) => label L | i_wait_timeout_locked Fail Src wait_timeout Fail Src => i_wait_timeout Fail Src @@ -403,7 +437,7 @@ label L | timeout | smp_already_locked(L) => label L | timeout_locked remove_message timeout timeout_locked -i_loop_rec f r +i_loop_rec f loop_rec_end f wait f wait_locked f @@ -421,93 +455,57 @@ send # Optimized comparisons with one immediate/literal operand. # -is_eq_exact Lbl R=rxy C=ian => i_is_eq_exact_immed Lbl R C -is_eq_exact Lbl R=rxy C=q => i_is_eq_exact_literal R Lbl C +is_eq_exact Lbl R=xy C=ian => i_is_eq_exact_immed Lbl R C +is_eq_exact Lbl R=xy C=q => i_is_eq_exact_literal Lbl R C -is_ne_exact Lbl R=rxy C=ian => i_is_ne_exact_immed Lbl R C -is_ne_exact Lbl R=rxy C=q => i_is_ne_exact_literal R Lbl C +is_ne_exact Lbl R=xy C=ian => i_is_ne_exact_immed Lbl R C +is_ne_exact Lbl R=xy C=q => i_is_ne_exact_literal Lbl R C %macro: i_is_eq_exact_immed EqualImmed -fail_action i_is_eq_exact_immed f r c i_is_eq_exact_immed f x c i_is_eq_exact_immed f y c -i_is_eq_exact_literal r f c -i_is_eq_exact_literal x f c -i_is_eq_exact_literal y f c +i_is_eq_exact_literal f x c +i_is_eq_exact_literal f y c %macro: i_is_ne_exact_immed NotEqualImmed -fail_action -i_is_ne_exact_immed f r c i_is_ne_exact_immed f x c i_is_ne_exact_immed f y c -i_is_ne_exact_literal r f c -i_is_ne_exact_literal x f c -i_is_ne_exact_literal y f c - -# -# Common Compare Specializations -# We don't do all of them since we want -# to keep the instruction set small-ish -# - -is_eq_exact Lbl S1=xy S2=r => is_eq_exact Lbl S2 S1 -is_eq_exact Lbl S1=rx S2=xy => i_is_eq_exact_spec Lbl S1 S2 -%macro: i_is_eq_exact_spec EqualExact -fail_action - -i_is_eq_exact_spec f x x -i_is_eq_exact_spec f x y -i_is_eq_exact_spec f r x -i_is_eq_exact_spec f r y -%cold -i_is_eq_exact_spec f r r -%hot +i_is_ne_exact_literal f x c +i_is_ne_exact_literal f y c -is_lt Lbl S1=rxc S2=rxc => i_is_lt_spec Lbl S1 S2 +is_eq_exact Lbl Y=y X=x => is_eq_exact Lbl X Y +%macro: is_eq_exact EqualExact -fail_action -pack +is_eq_exact f x x +is_eq_exact f x y +is_eq_exact f s s -%macro: i_is_lt_spec IsLessThan -fail_action - -i_is_lt_spec f x x -i_is_lt_spec f x r -i_is_lt_spec f x c -i_is_lt_spec f r x -i_is_lt_spec f r c -i_is_lt_spec f c x -i_is_lt_spec f c r +%macro: is_lt IsLessThan -fail_action +is_lt f x x +is_lt f x c +is_lt f c x %cold -i_is_lt_spec f r r -i_is_lt_spec f c c +is_lt f s s %hot -is_ge Lbl S1=xc S2=xc => i_is_ge_spec Lbl S1 S2 - -%macro: i_is_ge_spec IsGreaterEqual -fail_action - -i_is_ge_spec f x x -i_is_ge_spec f x c -i_is_ge_spec f c x +%macro: is_ge IsGreaterEqual -fail_action +is_ge f x x +is_ge f x c +is_ge f c x %cold -i_is_ge_spec f c c +is_ge f s s %hot -# -# All other comparisons. -# +%macro: is_ne_exact NotEqualExact -fail_action +is_ne_exact f s s -is_eq_exact Lbl S1 S2 => i_fetch S1 S2 | i_is_eq_exact Lbl -is_ne_exact Lbl S1 S2 => i_fetch S1 S2 | i_is_ne_exact Lbl +%macro: is_eq Equal -fail_action +is_eq f s s -is_lt Lbl S1 S2 => i_fetch S1 S2 | i_is_lt Lbl -is_ge Lbl S1 S2 => i_fetch S1 S2 | i_is_ge Lbl -is_eq Lbl S1 S2 => i_fetch S1 S2 | i_is_eq Lbl -is_ne Lbl S1 S2 => i_fetch S1 S2 | i_is_ne Lbl - -i_is_eq_exact f -i_is_ne_exact f -i_is_lt f -i_is_ge f -i_is_eq f -i_is_ne f +%macro: is_ne NotEqual -fail_action +is_ne f s s # # Putting things. @@ -525,7 +523,6 @@ i_put_tuple Dst Arity Puts=* | put S => \ i_put_tuple/2 %macro:i_put_tuple PutTuple -pack -goto:do_put_tuple -i_put_tuple r I i_put_tuple x I i_put_tuple y I @@ -542,74 +539,36 @@ put_list x n x put_list y n x put_list x x x put_list y x x -put_list x x r -put_list y r r put_list y y x put_list x y x -put_list r x x -put_list r y x -put_list r x r -put_list y y r -put_list y r x -put_list r n x -put_list x r x -put_list x y r -put_list y x r put_list y x x -put_list x r r - # put_list SrcReg Constant Dst -put_list r c r -put_list r c x -put_list r c y -put_list x c r put_list x c x put_list x c y -put_list y c r put_list y c x -put_list y c y # put_list Constant SrcReg Dst -put_list c r r -put_list c r x -put_list c r y -put_list c x r put_list c x x -put_list c x y - -put_list c y r put_list c y x -put_list c y y -%cold -put_list s s d -%hot +# The following put_list instructions using x(0) are frequently used. -%macro: i_fetch FetchArgs -pack -i_fetch c r -i_fetch c x -i_fetch c y -i_fetch r c -i_fetch r x -i_fetch r y -i_fetch x c -i_fetch x r -i_fetch x x -i_fetch x y -i_fetch y c -i_fetch y r -i_fetch y x -i_fetch y y +put_list y r r +put_list x r r +put_list r n r +put_list r n x +put_list r x x +put_list r x r +put_list x x r %cold -i_fetch c c -i_fetch s s +put_list s s d %hot # @@ -631,27 +590,27 @@ return_trace # Note: There is no 'move_return y r', since there never are any y registers # when we do move_return (if we have y registers, we must do move_deallocate_return). -move S r | return => move_return S r +move S x==0 | return => move_return S %macro: move_return MoveReturn -nonext -move_return x r -move_return c r -move_return n r +move_return x +move_return c +move_return n -move S r | deallocate D | return => move_deallocate_return S r D +move S x==0 | deallocate D | return => move_deallocate_return S D %macro: move_deallocate_return MoveDeallocateReturn -pack -nonext -move_deallocate_return x r Q -move_deallocate_return y r Q -move_deallocate_return c r Q -move_deallocate_return n r Q +move_deallocate_return x Q +move_deallocate_return y Q +move_deallocate_return c Q +move_deallocate_return n Q deallocate D | return => deallocate_return D %macro: deallocate_return DeallocateReturn -nonext deallocate_return Q -test_heap Need u==1 | put_list Y=y r r => test_heap_1_put_list Need Y +test_heap Need u==1 | put_list Y=y x==0 x==0 => test_heap_1_put_list Need Y %macro: test_heap_1_put_list TestHeapPutList -pack test_heap_1_put_list I y @@ -660,18 +619,18 @@ test_heap_1_put_list I y is_tuple Fail Literal=q => move Literal x | is_tuple Fail x is_tuple Fail=f c => jump Fail -is_tuple Fail=f S=rxy | test_arity Fail=f S=rxy Arity => is_tuple_of_arity Fail S Arity +is_tuple Fail=f S=xy | test_arity Fail=f S=xy Arity => is_tuple_of_arity Fail S Arity %macro:is_tuple_of_arity IsTupleOfArity -fail_action +is_tuple_of_arity f r A is_tuple_of_arity f x A is_tuple_of_arity f y A -is_tuple_of_arity f r A %macro: is_tuple IsTuple -fail_action +is_tuple f r is_tuple f x is_tuple f y -is_tuple f r test_arity Fail Literal=q Arity => move Literal x | test_arity Fail x Arity test_arity Fail=f c Arity => jump Fail @@ -679,95 +638,63 @@ test_arity Fail=f c Arity => jump Fail %macro: test_arity IsArity -fail_action test_arity f x A test_arity f y A -test_arity f r A - -is_tuple_of_arity Fail=f Reg Arity | get_tuple_element Reg P=u==0 Dst=xy => \ - is_tuple_of_arity Fail Reg Arity | extract_next_element Dst | original_reg Reg P - -test_arity Fail Reg Arity | get_tuple_element Reg P=u==0 Dst=xy => \ - test_arity Fail Reg Arity | extract_next_element Dst | original_reg Reg P - -original_reg Reg P1 | get_tuple_element Reg P2 Dst=xy | succ(P1, P2) => \ - extract_next_element Dst | original_reg Reg P2 - -get_tuple_element Reg P Dst => i_get_tuple_element Reg P Dst | original_reg Reg P - -original_reg Reg Pos => - -original_reg/2 - -extract_next_element D1=xy | original_reg Reg P1 | get_tuple_element Reg P2 D2=xy | \ -succ(P1, P2) | succ(D1, D2) => \ - extract_next_element2 D1 | original_reg Reg P2 - -extract_next_element2 D1=xy | original_reg Reg P1 | get_tuple_element Reg P2 D2=xy | \ -succ(P1, P2) | succ2(D1, D2) => \ - extract_next_element3 D1 | original_reg Reg P2 - -#extract_next_element3 D1=xy | original_reg Reg P1 | get_tuple_element Reg P2 D2=xy | \ -#succ(P1, P2) | succ3(D1, D2) => \ -# extract_next_element4 D1 | original_reg Reg P2 - -%macro: extract_next_element ExtractNextElement -pack -extract_next_element x -extract_next_element y -%macro: extract_next_element2 ExtractNextElement2 -pack -extract_next_element2 x -extract_next_element2 y +get_tuple_element Reg=x P1 D1=x | get_tuple_element Reg=x P2 D2=x | \ + get_tuple_element Reg=x P3 D3=x | \ + succ(P1, P2) | succ(P2, P3) | \ + succ(D1, D2) | succ(D2, D3) => i_get_tuple_element3 Reg P1 D1 -%macro: extract_next_element3 ExtractNextElement3 -pack -extract_next_element3 x -extract_next_element3 y +get_tuple_element Reg=x P1 D1=x | get_tuple_element Reg=x P2 D2=x | \ + succ(P1, P2) | succ(D1, D2) => i_get_tuple_element2 Reg P1 D1 -#%macro: extract_next_element4 ExtractNextElement4 -pack -#extract_next_element4 x -#extract_next_element4 y +get_tuple_element Reg P Dst => i_get_tuple_element Reg P Dst is_integer Fail=f i => is_integer Fail=f an => jump Fail is_integer Fail Literal=q => move Literal x | is_integer Fail x -is_integer Fail=f S=rx | allocate Need Regs => is_integer_allocate Fail S Need Regs +is_integer Fail=f S=x | allocate Need Regs => is_integer_allocate Fail S Need Regs %macro: is_integer_allocate IsIntegerAllocate -fail_action is_integer_allocate f x I I -is_integer_allocate f r I I %macro: is_integer IsInteger -fail_action is_integer f x is_integer f y -is_integer f r is_list Fail=f n => is_list Fail Literal=q => move Literal x | is_list Fail x is_list Fail=f c => jump Fail %macro: is_list IsList -fail_action -is_list f r is_list f x %cold is_list f y %hot -is_nonempty_list Fail=f S=rx | allocate Need Rs => is_nonempty_list_allocate Fail S Need Rs +is_nonempty_list Fail=f S=x | allocate Need Rs => is_nonempty_list_allocate Fail S Need Rs %macro:is_nonempty_list_allocate IsNonemptyListAllocate -fail_action -pack -is_nonempty_list_allocate f x I t is_nonempty_list_allocate f r I t +is_nonempty_list_allocate f x I t -is_nonempty_list F=f r | test_heap I1 I2 => is_non_empty_list_test_heap F r I1 I2 +is_nonempty_list F=f x==0 | test_heap I1 I2 => is_non_empty_list_test_heap F I1 I2 %macro: is_non_empty_list_test_heap IsNonemptyListTestHeap -fail_action -pack -is_non_empty_list_test_heap f r I t +is_non_empty_list_test_heap f I t + +is_nonempty_list Fail=f S=x | get_list S D1=x D2=x => \ + is_nonempty_list_get_list Fail S D1 D2 + +%macro: is_nonempty_list_get_list IsNonemptyListGetList -fail_action -pack +is_nonempty_list_get_list f r x x +is_nonempty_list_get_list f x x x %macro: is_nonempty_list IsNonemptyList -fail_action is_nonempty_list f x is_nonempty_list f y -is_nonempty_list f r %macro: is_atom IsAtom -fail_action is_atom f x -is_atom f r %cold is_atom f y %hot @@ -775,7 +702,6 @@ is_atom Fail=f a => is_atom Fail=f niq => jump Fail %macro: is_float IsFloat -fail_action -is_float f r is_float f x %cold is_float f y @@ -789,12 +715,10 @@ is_nil Fail=f qia => jump Fail %macro: is_nil IsNil -fail_action is_nil f x is_nil f y -is_nil f r is_binary Fail Literal=q => move Literal x | is_binary Fail x is_binary Fail=f c => jump Fail %macro: is_binary IsBinary -fail_action -is_binary f r is_binary f x %cold is_binary f y @@ -806,7 +730,6 @@ is_bitstr Fail Term => is_bitstring Fail Term is_bitstring Fail Literal=q => move Literal x | is_bitstring Fail x is_bitstring Fail=f c => jump Fail %macro: is_bitstring IsBitstring -fail_action -is_bitstring f r is_bitstring f x %cold is_bitstring f y @@ -814,7 +737,6 @@ is_bitstring f y is_reference Fail=f cq => jump Fail %macro: is_reference IsRef -fail_action -is_reference f r is_reference f x %cold is_reference f y @@ -822,7 +744,6 @@ is_reference f y is_pid Fail=f cq => jump Fail %macro: is_pid IsPid -fail_action -is_pid f r is_pid f x %cold is_pid f y @@ -830,7 +751,6 @@ is_pid f y is_port Fail=f cq => jump Fail %macro: is_port IsPort -fail_action -is_port f r is_port f x %cold is_port f y @@ -842,14 +762,12 @@ is_boolean Fail=f ac => jump Fail %cold %macro: is_boolean IsBoolean -fail_action -is_boolean f r is_boolean f x is_boolean f y %hot is_function2 Fail=f acq Arity => jump Fail is_function2 Fail=f Fun a => jump Fail -is_function2 Fail Fun Literal=q => move Literal x | is_function2 Fail Fun x is_function2 f s s %macro: is_function2 IsFunction2 -fail_action @@ -989,76 +907,76 @@ call_ext_only u==3 u$func:erlang:hibernate/3 => i_hibernate %unless USE_VM_PROBES call_ext Arity u$func:erlang:dt_get_tag/0 => \ - move a=am_undefined r + move a=am_undefined x=0 call_ext_last Arity u$func:erlang:dt_get_tag/0 D => \ - move a=am_undefined r | deallocate D | return + move a=am_undefined x=0 | deallocate D | return call_ext_only Arity u$func:erlang:dt_get_tag/0 => \ - move a=am_undefined r | return - -move Any r | call_ext Arity u$func:erlang:dt_put_tag/1 => \ - move a=am_undefined r -move Any r | call_ext_last Arity u$func:erlang:dt_put_tag/1 D => \ - move a=am_undefined r | deallocate D | return -move Any r | call_ext_only Arity u$func:erlang:dt_put_tag/1 => \ - move a=am_undefined r | return + move a=am_undefined x=0 | return + +move Any x==0 | call_ext Arity u$func:erlang:dt_put_tag/1 => \ + move a=am_undefined x=0 +move Any x==0 | call_ext_last Arity u$func:erlang:dt_put_tag/1 D => \ + move a=am_undefined x=0 | deallocate D | return +move Any x==0 | call_ext_only Arity u$func:erlang:dt_put_tag/1 => \ + move a=am_undefined x=0 | return call_ext Arity u$func:erlang:dt_put_tag/1 => \ - move a=am_undefined r + move a=am_undefined x=0 call_ext_last Arity u$func:erlang:dt_put_tag/1 D => \ - move a=am_undefined r | deallocate D | return + move a=am_undefined x=0 | deallocate D | return call_ext_only Arity u$func:erlang:dt_put_tag/1 => \ - move a=am_undefined r | return + move a=am_undefined x=0 | return call_ext Arity u$func:erlang:dt_get_tag_data/0 => \ - move a=am_undefined r + move a=am_undefined x=0 call_ext_last Arity u$func:erlang:dt_get_tag_data/0 D => \ - move a=am_undefined r | deallocate D | return + move a=am_undefined x=0 | deallocate D | return call_ext_only Arity u$func:erlang:dt_get_tag_data/0 => \ - move a=am_undefined r | return - -move Any r | call_ext Arity u$func:erlang:dt_spread_tag/1 => \ - move a=am_true r -move Any r | call_ext_last Arity u$func:erlang:dt_spread_tag/1 D => \ - move a=am_true r | deallocate D | return -move Any r | call_ext_only Arity u$func:erlang:dt_spread_tag/1 => \ - move a=am_true r | return + move a=am_undefined x=0 | return + +move Any x==0 | call_ext Arity u$func:erlang:dt_spread_tag/1 => \ + move a=am_true x=0 +move Any x==0 | call_ext_last Arity u$func:erlang:dt_spread_tag/1 D => \ + move a=am_true x=0 | deallocate D | return +move Any x==0 | call_ext_only Arity u$func:erlang:dt_spread_tag/1 => \ + move a=am_true x=0 | return call_ext Arity u$func:erlang:dt_spread_tag/1 => \ - move a=am_true r + move a=am_true x=0 call_ext_last Arity u$func:erlang:dt_spread_tag/1 D => \ - move a=am_true r | deallocate D | return + move a=am_true x=0 | deallocate D | return call_ext_only Arity u$func:erlang:dt_spread_tag/1 => \ - move a=am_true r | return - -move Any r | call_ext Arity u$func:erlang:dt_restore_tag/1 => \ - move a=am_true r -move Any r | call_ext_last Arity u$func:erlang:dt_restore_tag/1 D => \ - move a=am_true r | deallocate D | return -move Any r | call_ext_only Arity u$func:erlang:dt_restore_tag/1 => \ - move a=am_true r | return + move a=am_true x=0 | return + +move Any x==0 | call_ext Arity u$func:erlang:dt_restore_tag/1 => \ + move a=am_true x=0 +move Any x==0 | call_ext_last Arity u$func:erlang:dt_restore_tag/1 D => \ + move a=am_true x=0 | deallocate D | return +move Any x==0 | call_ext_only Arity u$func:erlang:dt_restore_tag/1 => \ + move a=am_true x=0 | return call_ext Arity u$func:erlang:dt_restore_tag/1 => \ - move a=am_true r + move a=am_true x=0 call_ext_last Arity u$func:erlang:dt_restore_tag/1 D => \ - move a=am_true r | deallocate D | return + move a=am_true x=0 | deallocate D | return call_ext_only Arity u$func:erlang:dt_restore_tag/1 => \ - move a=am_true r | return - -move Any r | call_ext Arity u$func:erlang:dt_prepend_vm_tag_data/1 => \ - move Any r -move Any r | call_ext_last Arity u$func:erlang:dt_prepend_vm_tag_data/1 D => \ - move Any r | deallocate D | return -move Any r | call_ext_only Arity u$func:erlang:dt_prepend_vm_tag_data/1 => \ - move Any r | return + move a=am_true x=0 | return + +move Any x==0 | call_ext Arity u$func:erlang:dt_prepend_vm_tag_data/1 => \ + move Any x=0 +move Any x==0 | call_ext_last Arity u$func:erlang:dt_prepend_vm_tag_data/1 D => \ + move Any x=0 | deallocate D | return +move Any x==0 | call_ext_only Arity u$func:erlang:dt_prepend_vm_tag_data/1 => \ + move Any x=0 | return call_ext Arity u$func:erlang:dt_prepend_vm_tag_data/1 => call_ext_last Arity u$func:erlang:dt_prepend_vm_tag_data/1 D => \ deallocate D | return call_ext_only Arity u$func:erlang:dt_prepend_vm_tag_data/1 => \ return -move Any r | call_ext Arity u$func:erlang:dt_append_vm_tag_data/1 => \ - move Any r -move Any r | call_ext_last Arity u$func:erlang:dt_append_vm_tag_data/1 D => \ - move Any r | deallocate D | return -move Any r | call_ext_only Arity u$func:erlang:dt_append_vm_tag_data/1 => \ - move Any r | return +move Any x==0 | call_ext Arity u$func:erlang:dt_append_vm_tag_data/1 => \ + move Any x=0 +move Any x==0 | call_ext_last Arity u$func:erlang:dt_append_vm_tag_data/1 D => \ + move Any x=0 | deallocate D | return +move Any x==0 | call_ext_only Arity u$func:erlang:dt_append_vm_tag_data/1 => \ + move Any x=0 | return call_ext Arity u$func:erlang:dt_append_vm_tag_data/1 => call_ext_last Arity u$func:erlang:dt_append_vm_tag_data/1 D => \ deallocate D | return @@ -1066,7 +984,7 @@ call_ext_only Arity u$func:erlang:dt_append_vm_tag_data/1 => \ return # Can happen after one of the transformations above. -move Discarded r | move Something r => move Something r +move Discarded x==0 | move Something x==0 => move Something x=0 %endif @@ -1091,9 +1009,9 @@ call_ext_only Ar=u Bif=u$is_bif => \ # with call instructions. # -move S=c r | call_ext Ar=u Func=u$is_not_bif => i_move_call_ext S r Func -move S=c r | call_ext_last Ar=u Func=u$is_not_bif D => i_move_call_ext_last Func D S r -move S=c r | call_ext_only Ar=u Func=u$is_not_bif => i_move_call_ext_only Func S r +move S=c x==0 | call_ext Ar=u Func=u$is_not_bif => i_move_call_ext S Func +move S=c x==0 | call_ext_last Ar=u Func=u$is_not_bif D => i_move_call_ext_last Func D S +move S=c x==0 | call_ext_only Ar=u Func=u$is_not_bif => i_move_call_ext_only Func S call_ext Ar Func => i_call_ext Func call_ext_last Ar Func D => i_call_ext_last Func D @@ -1120,78 +1038,71 @@ bif0 u$bif:erlang:node/0 Dst=d => node Dst bif1 Fail Bif=u$bif:erlang:get/1 Src=s Dst=d => i_get Src Dst -bif2 Jump=j u$bif:erlang:element/2 S1=s S2=rxy Dst=d => gen_element(Jump, S1, S2, Dst) +bif2 Jump=j u$bif:erlang:element/2 S1=s S2=xy Dst=d => gen_element(Jump, S1, S2, Dst) -bif1 Fail Bif Literal=q Dst => move Literal x | bif1 Fail Bif x Dst bif1 p Bif S1 Dst => bif1_body Bif S1 Dst -bif1_body Bif Literal=q Dst => move Literal x | bif1_body Bif x Dst - -bif2 p Bif S1 S2 Dst => i_fetch S1 S2 | i_bif2_body Bif Dst -bif2 Fail Bif S1 S2 Dst => i_fetch S1 S2 | i_bif2 Fail Bif Dst +bif2 p Bif S1 S2 Dst => i_bif2_body Bif S1 S2 Dst +bif2 Fail Bif S1 S2 Dst => i_bif2 Fail Bif S1 S2 Dst i_get s d %macro: self Self -self r self x self y %macro: node Node -node r node x %cold node y %hot -i_fast_element r j I d -i_fast_element x j I d -i_fast_element y j I d +i_fast_element j x I d +i_fast_element j y I d -i_element r j s d -i_element x j s d -i_element y j s d +i_element j x s d +i_element j y s d bif1 f b s d bif1_body b s d -i_bif2 f b d -i_bif2_body b d +i_bif2 f b s s d +i_bif2_body b s s d # # Internal calls. # -move S=c r | call Ar P=f => i_move_call S r P -move S=s r | call Ar P=f => move_call S r P +move S=c x==0 | call Ar P=f => i_move_call S P +move S=s x==0 | call Ar P=f => move_call S P -i_move_call c r f +i_move_call c f %macro:move_call MoveCall -arg_f -size -nonext -move_call/3 +move_call/2 -move_call x r f -move_call y r f +move_call x f +move_call y f -move S=c r | call_last Ar P=f D => i_move_call_last P D S r -move S r | call_last Ar P=f D => move_call_last S r P D +move S=c x==0 | call_last Ar P=f D => i_move_call_last P D S +move S x==0 | call_last Ar P=f D => move_call_last S P D -i_move_call_last f P c r +i_move_call_last f P c %macro:move_call_last MoveCallLast -arg_f -nonext -pack -move_call_last/4 -move_call_last x r f Q -move_call_last y r f Q +move_call_last/3 +move_call_last x f Q +move_call_last y f Q -move S=c r | call_only Ar P=f => i_move_call_only P S r -move S=x r | call_only Ar P=f => move_call_only S r P +move S=c x==0 | call_only Ar P=f => i_move_call_only P S +move S=x x==0 | call_only Ar P=f => move_call_only S P -i_move_call_only f c r +i_move_call_only f c %macro:move_call_only MoveCallOnly -arg_f -nonext -move_call_only/3 +move_call_only/2 -move_call_only x r f +move_call_only x f call Ar Func => i_call Func call_last Ar Func D => i_call_last Func D @@ -1205,9 +1116,9 @@ i_call_ext e i_call_ext_last e P i_call_ext_only e -i_move_call_ext c r e -i_move_call_ext_last e P c r -i_move_call_ext_only e c r +i_move_call_ext c e +i_move_call_ext_last e P c +i_move_call_ext_only e c # Fun calls. @@ -1227,7 +1138,6 @@ i_make_fun I t %macro: is_function IsFunction -fail_action is_function f x is_function f y -is_function f r is_function Fail=f c => jump Fail func_info M F A => i_func_info u M F A @@ -1239,131 +1149,105 @@ func_info M F A => i_func_info u M F A %cold bs_start_match2 Fail=f ica X Y D => jump Fail bs_start_match2 Fail Bin X Y D => i_bs_start_match2 Bin Fail X Y D -i_bs_start_match2 r f I I d i_bs_start_match2 x f I I d i_bs_start_match2 y f I I d bs_save2 Reg Index => gen_bs_save(Reg, Index) -i_bs_save2 r I i_bs_save2 x I bs_restore2 Reg Index => gen_bs_restore(Reg, Index) -i_bs_restore2 r I i_bs_restore2 x I # Matching integers bs_match_string Fail Ms Bits Val => i_bs_match_string Ms Fail Bits Val -i_bs_match_string r f I I i_bs_match_string x f I I # Fetching integers from binaries. -bs_get_integer2 Fail=f Ms=rx Live=u Sz=sq Unit=u Flags=u Dst=d => \ +bs_get_integer2 Fail=f Ms=x Live=u Sz=sq Unit=u Flags=u Dst=d => \ gen_get_integer2(Fail, Ms, Live, Sz, Unit, Flags, Dst) -i_bs_get_integer_small_imm r I f I d i_bs_get_integer_small_imm x I f I d -i_bs_get_integer_imm r I I f I d i_bs_get_integer_imm x I I f I d -i_bs_get_integer f I I d -i_bs_get_integer_8 r f d +i_bs_get_integer f I I s s d i_bs_get_integer_8 x f d -i_bs_get_integer_16 r f d i_bs_get_integer_16 x f d -i_bs_get_integer_32 r f I d i_bs_get_integer_32 x f I d # Fetching binaries from binaries. -bs_get_binary2 Fail=f Ms=rx Live=u Sz=sq Unit=u Flags=u Dst=d => \ +bs_get_binary2 Fail=f Ms=x Live=u Sz=sq Unit=u Flags=u Dst=d => \ gen_get_binary2(Fail, Ms, Live, Sz, Unit, Flags, Dst) %macro: i_bs_get_binary_imm2 BsGetBinaryImm_2 -fail_action -gen_dest %macro: i_bs_get_binary2 BsGetBinary_2 -fail_action -gen_dest %macro: i_bs_get_binary_all2 BsGetBinaryAll_2 -fail_action -gen_dest -i_bs_get_binary_imm2 f r I I I d i_bs_get_binary_imm2 f x I I I d -i_bs_get_binary2 f r I s I d i_bs_get_binary2 f x I s I d -i_bs_get_binary_all2 f r I I d i_bs_get_binary_all2 f x I I d -i_bs_get_binary_all_reuse r f I i_bs_get_binary_all_reuse x f I # Fetching float from binaries. -bs_get_float2 Fail=f Ms=rx Live=u Sz=s Unit=u Flags=u Dst=d => \ +bs_get_float2 Fail=f Ms=x Live=u Sz=s Unit=u Flags=u Dst=d => \ gen_get_float2(Fail, Ms, Live, Sz, Unit, Flags, Dst) -bs_get_float2 Fail=f Ms=rx Live=u Sz=q Unit=u Flags=u Dst=d => jump Fail +bs_get_float2 Fail=f Ms=x Live=u Sz=q Unit=u Flags=u Dst=d => jump Fail %macro: i_bs_get_float2 BsGetFloat2 -fail_action -gen_dest -i_bs_get_float2 f r I s I d i_bs_get_float2 f x I s I d # Miscellanous -bs_skip_bits2 Fail=f Ms=rx Sz=s Unit=u Flags=u => \ - gen_skip_bits2(Fail, Ms, Sz, Unit, Flags) -bs_skip_bits2 Fail=f Ms=rx Sz=q Unit=u Flags=u => \ +bs_skip_bits2 Fail=f Ms=x Sz=sq Unit=u Flags=u => \ gen_skip_bits2(Fail, Ms, Sz, Unit, Flags) %macro: i_bs_skip_bits_imm2 BsSkipBitsImm2 -fail_action -i_bs_skip_bits_imm2 f r I i_bs_skip_bits_imm2 f x I %macro: i_bs_skip_bits2 BsSkipBits2 -fail_action -i_bs_skip_bits2 f r x I -i_bs_skip_bits2 f r y I i_bs_skip_bits2 f x x I -i_bs_skip_bits2 f x r I i_bs_skip_bits2 f x y I %macro: i_bs_skip_bits_all2 BsSkipBitsAll2 -fail_action -i_bs_skip_bits_all2 f r I i_bs_skip_bits_all2 f x I -bs_test_tail2 Fail=f Ms=rx Bits=u==0 => bs_test_zero_tail2 Fail Ms -bs_test_tail2 Fail=f Ms=rx Bits=u => bs_test_tail_imm2 Fail Ms Bits -bs_test_zero_tail2 f r +bs_test_tail2 Fail=f Ms=x Bits=u==0 => bs_test_zero_tail2 Fail Ms +bs_test_tail2 Fail=f Ms=x Bits=u => bs_test_tail_imm2 Fail Ms Bits bs_test_zero_tail2 f x -bs_test_tail_imm2 f r I bs_test_tail_imm2 f x I bs_test_unit F Ms Unit=u==8 => bs_test_unit8 F Ms -bs_test_unit f r I bs_test_unit f x I -bs_test_unit8 f r bs_test_unit8 f x -bs_context_to_binary r +# An y register operand for bs_context_to_binary is rare, +# but can happen because of inlining. + +bs_context_to_binary Y=y => move Y x | bs_context_to_binary x + bs_context_to_binary x -bs_context_to_binary y # # Utf8/utf16/utf32 support. (R12B-5) # -bs_get_utf8 Fail=f Ms=rx u u Dst=d => i_bs_get_utf8 Ms Fail Dst -i_bs_get_utf8 r f d +bs_get_utf8 Fail=f Ms=x u u Dst=d => i_bs_get_utf8 Ms Fail Dst i_bs_get_utf8 x f d -bs_skip_utf8 Fail=f Ms=rx u u => i_bs_get_utf8 Ms Fail x +bs_skip_utf8 Fail=f Ms=x u u => i_bs_get_utf8 Ms Fail x -bs_get_utf16 Fail=f Ms=rx u Flags=u Dst=d => i_bs_get_utf16 Ms Fail Flags Dst -bs_skip_utf16 Fail=f Ms=rx u Flags=u => i_bs_get_utf16 Ms Fail Flags x +bs_get_utf16 Fail=f Ms=x u Flags=u Dst=d => i_bs_get_utf16 Ms Fail Flags Dst +bs_skip_utf16 Fail=f Ms=x u Flags=u => i_bs_get_utf16 Ms Fail Flags x -i_bs_get_utf16 r f I d i_bs_get_utf16 x f I d -bs_get_utf32 Fail=f Ms=rx Live=u Flags=u Dst=d => \ +bs_get_utf32 Fail=f Ms=x Live=u Flags=u Dst=d => \ bs_get_integer2 Fail Ms Live i=32 u=1 Flags Dst | \ - i_fetch Dst Ms | \ - i_bs_validate_unicode_retract Fail -bs_skip_utf32 Fail=f Ms=rx Live=u Flags=u => \ + i_bs_validate_unicode_retract Fail Dst Ms +bs_skip_utf32 Fail=f Ms=x Live=u Flags=u => \ bs_get_integer2 Fail Ms Live i=32 u=1 Flags x | \ - i_fetch x Ms | \ - i_bs_validate_unicode_retract Fail + i_bs_validate_unicode_retract Fail x Ms -i_bs_validate_unicode_retract j +i_bs_validate_unicode_retract j s s %hot # @@ -1385,13 +1269,12 @@ bs_init2 Fail Sz=u Words Regs Flags Dst => \ bs_init2 Fail Sz Words=u==0 Regs Flags Dst => \ i_bs_init_fail Sz Fail Regs Dst bs_init2 Fail Sz Words Regs Flags Dst => \ - i_fetch Sz r | i_bs_init_fail_heap Words Fail Regs Dst + i_bs_init_fail_heap Sz Words Fail Regs Dst -i_bs_init_fail r j I d i_bs_init_fail x j I d i_bs_init_fail y j I d -i_bs_init_fail_heap I j I d +i_bs_init_fail_heap s I j I d i_bs_init I I d i_bs_init_heap_bin I I d @@ -1408,39 +1291,35 @@ bs_init_bits Fail Sz=u Words Regs Flags Dst => i_bs_init_bits_heap Sz Words Reg bs_init_bits Fail Sz Words=u==0 Regs Flags Dst => \ i_bs_init_bits_fail Sz Fail Regs Dst bs_init_bits Fail Sz Words Regs Flags Dst => \ - i_fetch Sz r | i_bs_init_bits_fail_heap Words Fail Regs Dst + i_bs_init_bits_fail_heap Sz Words Fail Regs Dst -i_bs_init_bits_fail r j I d i_bs_init_bits_fail x j I d i_bs_init_bits_fail y j I d -i_bs_init_bits_fail_heap I j I d +i_bs_init_bits_fail_heap s I j I d i_bs_init_bits I I d i_bs_init_bits_heap I I I d bs_add Fail S1=i==0 S2 Unit=u==1 D => move S2 D -bs_add Fail S1 S2 Unit D => i_fetch S1 S2 | i_bs_add Fail Unit D -i_bs_add j I d +bs_add j s s I d bs_append Fail Size Extra Live Unit Bin Flags Dst => \ - i_fetch Size Bin | i_bs_append Fail Extra Live Unit Dst + move Bin x | i_bs_append Fail Extra Live Unit Size Dst bs_private_append Fail Size Unit Bin Flags Dst => \ - i_fetch Size Bin | i_bs_private_append Fail Unit Dst + i_bs_private_append Fail Unit Size Bin Dst bs_init_writable -i_bs_append j I I I d -i_bs_private_append j I d +i_bs_append j I I I s d +i_bs_private_append j I s s d # # Storing integers into binaries. # -bs_put_integer Fail=j Sz=s Unit=u Flags=u Literal=q => \ - move Literal x | bs_put_integer Fail Sz Unit Flags x bs_put_integer Fail=j Sz=sq Unit=u Flags=u Src=s => \ gen_put_integer(Fail, Sz, Unit, Flags, Src) @@ -1454,32 +1333,22 @@ i_new_bs_put_integer_imm j I I s # Utf8/utf16/utf32 support. (R12B-5) # -bs_utf8_size Fail Literal=q Dst=d => \ - move Literal x | bs_utf8_size Fail x Dst bs_utf8_size j Src=s Dst=d => i_bs_utf8_size Src Dst i_bs_utf8_size s d -bs_utf16_size Fail Literal=q Dst=d => \ - move Literal x | bs_utf16_size Fail x Dst bs_utf16_size j Src=s Dst=d => i_bs_utf16_size Src Dst i_bs_utf16_size s d -bs_put_utf8 Fail=j Flags=u Literal=q => \ - move Literal x | bs_put_utf8 Fail Flags x bs_put_utf8 Fail u Src=s => i_bs_put_utf8 Fail Src i_bs_put_utf8 j s -bs_put_utf16 Fail=j Flags=u Literal=q => \ - move Literal x | bs_put_utf16 Fail Flags x bs_put_utf16 Fail Flags=u Src=s => i_bs_put_utf16 Fail Flags Src i_bs_put_utf16 j I s -bs_put_utf32 Fail=j Flags=u Literal=q => \ - move Literal x | bs_put_utf32 Fail Flags x bs_put_utf32 Fail=j Flags=u Src=s => \ i_bs_validate_unicode Fail Src | bs_put_integer Fail i=32 u=1 Flags Src @@ -1490,9 +1359,6 @@ i_bs_validate_unicode j s # bs_put_float Fail Sz=q Unit Flags Val => badarg Fail -bs_put_float Fail=j Sz Unit=u Flags=u Literal=q => \ - move Literal x | bs_put_float Fail Sz Unit Flags x - bs_put_float Fail=j Sz=s Unit=u Flags=u Src=s => \ gen_put_float(Fail, Sz, Unit, Flags, Src) @@ -1506,8 +1372,6 @@ i_new_bs_put_float_imm j I I s # Storing binaries into binaries. # -bs_put_binary Fail Sz Unit Flags Literal=q => \ - move Literal x | bs_put_binary Fail Sz Unit Flags x bs_put_binary Fail=j Sz=s Unit=u Flags=u Src=s => \ gen_put_binary(Fail, Sz, Unit, Flags, Src) @@ -1600,7 +1464,6 @@ is_map Fail Lit=q | literal_is_map(Lit) => is_map Fail cq => jump Fail %macro: is_map IsMap -fail_action -is_map f r is_map f x is_map f y @@ -1611,101 +1474,120 @@ has_map_fields Fail Src Size Rest=* => \ ## Transform get_map_elements(s) #{ K1 := V1, K2 := V2 } -get_map_elements Fail Src=rxy Size=u==2 Rest=* => \ +get_map_elements Fail Src=xy Size=u==2 Rest=* => \ gen_get_map_element(Fail, Src, Size, Rest) get_map_elements Fail Src Size Rest=* | map_key_sort(Size, Rest) => \ gen_get_map_elements(Fail, Src, Size, Rest) i_get_map_elements f s I -i_get_map_element Fail Src=rxy Key=ry Dst => \ +i_get_map_element Fail Src=xy Key=y Dst => \ move Key x | i_get_map_element Fail Src x Dst %macro: i_get_map_element_hash GetMapElementHash -fail_action -i_get_map_element_hash f r c I r -i_get_map_element_hash f x c I r -i_get_map_element_hash f y c I r -i_get_map_element_hash f r c I x i_get_map_element_hash f x c I x i_get_map_element_hash f y c I x -i_get_map_element_hash f r c I y i_get_map_element_hash f x c I y i_get_map_element_hash f y c I y %macro: i_get_map_element GetMapElement -fail_action -i_get_map_element f r x r -i_get_map_element f x x r -i_get_map_element f y x r -i_get_map_element f r x x i_get_map_element f x x x i_get_map_element f y x x -i_get_map_element f r x y i_get_map_element f x x y i_get_map_element f y x y # +# Convert the plus operations to a generic plus instruction. +# +gen_plus/5 +gen_minus/5 + +gc_bif1 Fail Live u$bif:erlang:splus/1 Src Dst => \ + gen_plus Fail Live Src i Dst +gc_bif2 Fail Live u$bif:erlang:splus/2 S1 S2 Dst => \ + gen_plus Fail Live S1 S2 Dst + +gc_bif1 Fail Live u$bif:erlang:sminus/1 Src Dst => \ + gen_minus Fail Live i Src Dst +gc_bif2 Fail Live u$bif:erlang:sminus/2 S1 S2 Dst => \ + gen_minus Fail Live S1 S2 Dst + +# # Optimize addition and subtraction of small literals using # the i_increment/4 instruction (in bodies, not in guards). # -gc_bif2 p Live u$bif:erlang:splus/2 Int=i Reg=d Dst => \ +gen_plus p Live Int=i Reg=d Dst => \ gen_increment(Reg, Int, Live, Dst) -gc_bif2 p Live u$bif:erlang:splus/2 Reg=d Int=i Dst => \ +gen_plus p Live Reg=d Int=i Dst => \ gen_increment(Reg, Int, Live, Dst) -gc_bif2 p Live u$bif:erlang:sminus/2 Reg=d Int=i Dst | \ - negation_is_small(Int) => \ +gen_minus p Live Reg=d Int=i Dst | negation_is_small(Int) => \ gen_increment_from_minus(Reg, Int, Live, Dst) # # GCing arithmetic instructions. # -gc_bif2 Fail I u$bif:erlang:splus/2 S1=x S2=x Dst=d => i_plus Fail I S1 S2 Dst -gc_bif2 Fail I u$bif:erlang:splus/2 S1 S2 Dst=d => i_fetch S1 S2 | i_plus Fail I Dst -gc_bif2 Fail I u$bif:erlang:sminus/2 S1=x S2=x Dst=d => i_minus Fail I S1 S2 Dst -gc_bif2 Fail I u$bif:erlang:sminus/2 S1 S2 Dst=d => i_fetch S1 S2 | i_minus Fail I Dst -gc_bif2 Fail I u$bif:erlang:stimes/2 S1 S2 Dst=d => i_fetch S1 S2 | i_times Fail I Dst -gc_bif2 Fail I u$bif:erlang:div/2 S1 S2 Dst=d => i_fetch S1 S2 | i_m_div Fail I Dst +gen_plus Fail Live Y=y X=x Dst => i_plus Fail Live X Y Dst +gen_plus Fail Live S1 S2 Dst => i_plus Fail Live S1 S2 Dst -gc_bif2 Fail I u$bif:erlang:intdiv/2 S1 S2 Dst=d => i_fetch S1 S2 | i_int_div Fail I Dst -gc_bif2 Fail I u$bif:erlang:rem/2 S1=x S2=x Dst=d => i_rem Fail I S1 S2 Dst -gc_bif2 Fail I u$bif:erlang:rem/2 S1 S2 Dst=d => i_fetch S1 S2 | i_rem Fail I Dst +gen_minus Fail Live S1 S2 Dst => i_minus Fail Live S1 S2 Dst -gc_bif2 Fail I u$bif:erlang:bsl/2 S1 S2 Dst=d => i_fetch S1 S2 | i_bsl Fail I Dst -gc_bif2 Fail I u$bif:erlang:bsr/2 S1 S2 Dst=d => i_fetch S1 S2 | i_bsr Fail I Dst +gc_bif2 Fail Live u$bif:erlang:stimes/2 S1 S2 Dst => \ + i_times Fail Live S1 S2 Dst -gc_bif2 Fail I u$bif:erlang:band/2 S1=x S2=c Dst=d => i_band Fail I S1 S2 Dst -gc_bif2 Fail I u$bif:erlang:band/2 S1 S2 Dst=d => i_fetch S1 S2 | i_band Fail I Dst -gc_bif2 Fail I u$bif:erlang:bor/2 S1 S2 Dst=d => i_fetch S1 S2 | i_bor Fail I Dst -gc_bif2 Fail I u$bif:erlang:bxor/2 S1 S2 Dst=d => i_fetch S1 S2 | i_bxor Fail I Dst +gc_bif2 Fail Live u$bif:erlang:div/2 S1 S2 Dst => \ + i_m_div Fail Live S1 S2 Dst +gc_bif2 Fail Live u$bif:erlang:intdiv/2 S1 S2 Dst => \ + i_int_div Fail Live S1 S2 Dst -gc_bif1 Fail I u$bif:erlang:bnot/1 Src Dst=d => i_int_bnot Fail Src I Dst +gc_bif2 Fail Live u$bif:erlang:rem/2 S1 S2 Dst => \ + i_rem Fail Live S1 S2 Dst + +gc_bif2 Fail Live u$bif:erlang:bsl/2 S1 S2 Dst => \ + i_bsl Fail Live S1 S2 Dst +gc_bif2 Fail Live u$bif:erlang:bsr/2 S1 S2 Dst => \ + i_bsr Fail Live S1 S2 Dst + +gc_bif2 Fail Live u$bif:erlang:band/2 S1 S2 Dst => \ + i_band Fail Live S1 S2 Dst + +gc_bif2 Fail Live u$bif:erlang:bor/2 S1 S2 Dst => \ + i_bor Fail Live S1 S2 Dst -gc_bif1 Fail I u$bif:erlang:sminus/1 Src Dst=d => i_fetch i Src | i_minus Fail I Dst -gc_bif1 Fail I u$bif:erlang:splus/1 Src Dst=d => i_fetch i Src | i_plus Fail I Dst +gc_bif2 Fail Live u$bif:erlang:bxor/2 S1 S2 Dst => \ + i_bxor Fail Live S1 S2 Dst + +gc_bif1 Fail I u$bif:erlang:bnot/1 Src Dst=d => i_int_bnot Fail Src I Dst i_increment r I I d i_increment x I I d i_increment y I I d i_plus j I x x d -i_plus j I d +i_plus j I x y d +i_plus j I s s d + i_minus j I x x d -i_minus j I d -i_times j I d -i_m_div j I d -i_int_div j I d +i_minus j I s s d + +i_times j I s s d + +i_m_div j I s s d +i_int_div j I s s d + i_rem j I x x d -i_rem j I d +i_rem j I s s d -i_bsl j I d -i_bsr j I d +i_bsl j I s s d +i_bsr j I s s d i_band j I x c d -i_band j I d -i_bor j I d -i_bxor j I d +i_band j I s s d + +i_bor j I s s d +i_bxor j I s s d i_int_bnot j s I d @@ -1731,21 +1613,18 @@ gc_bif2 Fail I Bif S1 S2 Dst => \ gc_bif3 Fail I Bif S1 S2 S3 Dst => \ gen_guard_bif3(Fail, I, Bif, S1, S2, S3, Dst) -i_gc_bif1 Fail Bif V=q Live D => move V x | i_gc_bif1 Fail Bif x Live D - i_gc_bif1 j I s I d -ii_gc_bif2/6 - -ii_gc_bif2 Fail Bif S1 S2 Live D => i_fetch S1 S2 | i_gc_bif2 Fail Bif Live D - -i_gc_bif2 j I I d +i_gc_bif2 j I I s s d ii_gc_bif3/7 -ii_gc_bif3 Fail Bif S1 S2 S3 Live D => move S1 x | i_fetch S2 S3 | i_gc_bif3 Fail Bif x Live D +# A specific instruction can only have 6 operands, so we must +# pass one of the arguments in an x register. +ii_gc_bif3 Fail Bif Live S1 S2 S3 Dst => \ + move S1 x | i_gc_bif3 Fail Bif Live S2 S3 Dst -i_gc_bif3 j I s I d +i_gc_bif3 j I I s s d # # The following instruction is specially handled in beam_load.c diff --git a/erts/emulator/beam/packet_parser.c b/erts/emulator/beam/packet_parser.c index 2dd421a9e9..a737a86f14 100644 --- a/erts/emulator/beam/packet_parser.c +++ b/erts/emulator/beam/packet_parser.c @@ -256,6 +256,7 @@ int packet_get_length(enum PacketParseType htype, const char* ptr, unsigned n, /* Bytes read so far */ unsigned max_plen, /* Max packet length, 0=no limit */ unsigned trunc_len, /* Truncate (lines) if longer, 0=no limit */ + char delimiter, /* Line delimiting character */ int* statep) /* Protocol specific state */ { unsigned hlen, plen; @@ -299,9 +300,9 @@ int packet_get_length(enum PacketParseType htype, goto remain; case TCP_PB_LINE_LF: { - /* TCP_PB_LINE_LF: [Data ... \n] */ + /* TCP_PB_LINE_LF: [Data ... Delimiter] */ const char* ptr2; - if ((ptr2 = memchr(ptr, '\n', n)) == NULL) { + if ((ptr2 = memchr(ptr, delimiter, n)) == NULL) { if (n > max_plen && max_plen != 0) { /* packet full */ DEBUGF((" => packet full (no NL)=%d\r\n", n)); goto error; diff --git a/erts/emulator/beam/packet_parser.h b/erts/emulator/beam/packet_parser.h index ff158ff8b8..717d905fad 100644 --- a/erts/emulator/beam/packet_parser.h +++ b/erts/emulator/beam/packet_parser.h @@ -105,7 +105,8 @@ int packet_get_length(enum PacketParseType htype, const char* ptr, unsigned n, /* Bytes read so far */ unsigned max_plen, /* Packet max length, 0=no limit */ unsigned trunc_len, /* Truncate (lines) if longer, 0=no limit */ - int* statep); /* Internal protocol state */ + char delimiter, /* Line delimiting character */ + int* statep); /* Internal protocol state */ ERTS_GLB_INLINE void packet_get_body(enum PacketParseType htype, diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index bb871b05ba..34011147d9 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -74,9 +74,7 @@ #if defined (__WIN32__) # include "erl_win_sys.h" -#elif defined (__OSE__) -# include "erl_ose_sys.h" -#else +#else # include "erl_unix_sys.h" #ifndef UNIX # define UNIX 1 @@ -285,62 +283,11 @@ __decl_noreturn void __noreturn erl_assert_error(const char* expr, const char *f #else #error Neither 32 nor 64 bit architecture #endif -#if defined(ARCH_64) && defined(HALFWORD_HEAP_EMULATOR) -# define HALFWORD_HEAP 1 -# define HALFWORD_ASSERT 0 -# define ASSERT_HALFWORD(COND) ASSERT(COND) -# undef ERTS_SIZEOF_TERM -# define ERTS_SIZEOF_TERM 4 -#else -# define HALFWORD_HEAP 0 -# define HALFWORD_ASSERT 0 -# define ASSERT_HALFWORD(COND) -#endif #if SIZEOF_VOID_P != SIZEOF_SIZE_T #error sizeof(void*) != sizeof(size_t) #endif -#if HALFWORD_HEAP - -#if SIZEOF_INT == 4 -typedef unsigned int Eterm; -typedef unsigned int Uint; -typedef int Sint; -#define ERTS_UINT_MAX UINT_MAX -#define ERTS_SIZEOF_ETERM SIZEOF_INT -#define ErtsStrToSint strtol -#else -#error Found no appropriate type to use for 'Eterm', 'Uint' and 'Sint' -#endif - -#if SIZEOF_VOID_P == SIZEOF_LONG -typedef unsigned long UWord; -typedef long SWord; -#define SWORD_CONSTANT(Const) Const##L -#define UWORD_CONSTANT(Const) Const##UL -#define ERTS_UWORD_MAX ULONG_MAX -#define ERTS_SWORD_MAX LONG_MAX -#elif SIZEOF_VOID_P == SIZEOF_INT -typedef unsigned int UWord; -typedef int SWord; -#define SWORD_CONSTANT(Const) Const -#define UWORD_CONSTANT(Const) Const##U -#define ERTS_UWORD_MAX UINT_MAX -#define ERTS_SWORD_MAX INT_MAX -#elif SIZEOF_VOID_P == SIZEOF_LONG_LONG -typedef unsigned long long UWord; -typedef long long SWord; -#define SWORD_CONSTANT(Const) Const##LL -#define UWORD_CONSTANT(Const) Const##ULL -#define ERTS_UWORD_MAX ULLONG_MAX -#define ERTS_SWORD_MAX LLONG_MAX -#else -#error Found no appropriate type to use for 'Eterm', 'Uint' and 'Sint' -#endif - -#else /* !HALFWORD_HEAP */ - #if SIZEOF_VOID_P == SIZEOF_LONG typedef unsigned long Eterm; typedef unsigned long Uint; @@ -383,8 +330,6 @@ typedef Uint UWord; typedef Sint SWord; #define ERTS_UINT_MAX ERTS_UWORD_MAX -#endif /* HALFWORD_HEAP */ - typedef UWord BeamInstr; #ifndef HAVE_INT64 diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index e9d7c91ac9..5fe4091adc 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -897,7 +897,7 @@ tail_recur: Uint y2 = y1 < 0 ? -(Uint)y1 : y1; UINT32_HASH_STEP(y2, FUNNY_NUMBER2); -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) if (y2 >> 32) UINT32_HASH_STEP(y2 >> 32, FUNNY_NUMBER2); #endif @@ -1018,7 +1018,7 @@ tail_recur: } d = BIG_DIGIT(ptr, k); k = sizeof(ErtsDigit); -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) if (!(d >> 32)) k /= 2; #endif @@ -1988,7 +1988,7 @@ tail_recur: (atom_tab(atom_val(term))->slot.bucket.hvalue); break; case SMALL_DEF: -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) { Sint y1 = signed_val(term); Uint y2 = y1 < 0 ? -(Uint)y1 : y1; @@ -2597,11 +2597,7 @@ erts_destroy_tmp_dsbuf(erts_dsprintf_buf_t *dsbufp) * Test for equality of two terms. * Returns 0 if not equal, or a non-zero value otherwise. */ -#if HALFWORD_HEAP -int eq_rel(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base) -#else int eq(Eterm a, Eterm b) -#endif { DECLARE_WSTACK(stack); Sint sz; @@ -2609,18 +2605,18 @@ int eq(Eterm a, Eterm b) Eterm* bb; tailrecur: - if (is_same(a, a_base, b, b_base)) goto pop_next; + if (is_same(a, b)) goto pop_next; tailrecur_ne: switch (primary_tag(a)) { case TAG_PRIMARY_LIST: if (is_list(b)) { - Eterm* aval = list_val_rel(a, a_base); - Eterm* bval = list_val_rel(b, b_base); + Eterm* aval = list_val(a); + Eterm* bval = list_val(b); while (1) { Eterm atmp = CAR(aval); Eterm btmp = CAR(bval); - if (!is_same(atmp,a_base,btmp,b_base)) { + if (!is_same(atmp,btmp)) { WSTACK_PUSH2(stack,(UWord) CDR(bval),(UWord) CDR(aval)); a = atmp; b = btmp; @@ -2628,7 +2624,7 @@ tailrecur_ne: } atmp = CDR(aval); btmp = CDR(bval); - if (is_same(atmp,a_base,btmp,b_base)) { + if (is_same(atmp,btmp)) { goto pop_next; } if (is_not_list(atmp) || is_not_list(btmp)) { @@ -2636,22 +2632,22 @@ tailrecur_ne: b = btmp; goto tailrecur_ne; } - aval = list_val_rel(atmp, a_base); - bval = list_val_rel(btmp, b_base); + aval = list_val(atmp); + bval = list_val(btmp); } } break; /* not equal */ case TAG_PRIMARY_BOXED: { - Eterm hdr = *boxed_val_rel(a,a_base); + Eterm hdr = *boxed_val(a); switch (hdr & _TAG_HEADER_MASK) { case ARITYVAL_SUBTAG: { - aa = tuple_val_rel(a, a_base); - if (!is_boxed(b) || *boxed_val_rel(b,b_base) != *aa) + aa = tuple_val(a); + if (!is_boxed(b) || *boxed_val(b) != *aa) goto not_equal; - bb = tuple_val_rel(b,b_base); + bb = tuple_val(b); if ((sz = arityval(*aa)) == 0) goto pop_next; ++aa; ++bb; @@ -2670,16 +2666,16 @@ tailrecur_ne: Uint a_bitoffs; Uint b_bitoffs; - if (!is_binary_rel(b,b_base)) { + if (!is_binary(b)) { goto not_equal; } - a_size = binary_size_rel(a,a_base); - b_size = binary_size_rel(b,b_base); + a_size = binary_size(a); + b_size = binary_size(b); if (a_size != b_size) { goto not_equal; } - ERTS_GET_BINARY_BYTES_REL(a, a_ptr, a_bitoffs, a_bitsize, a_base); - ERTS_GET_BINARY_BYTES_REL(b, b_ptr, b_bitoffs, b_bitsize, b_base); + ERTS_GET_BINARY_BYTES(a, a_ptr, a_bitoffs, a_bitsize); + ERTS_GET_BINARY_BYTES(b, b_ptr, b_bitoffs, b_bitsize); if ((a_bitsize | b_bitsize | a_bitoffs | b_bitoffs) == 0) { if (sys_memcmp(a_ptr, b_ptr, a_size) == 0) goto pop_next; } else if (a_bitsize == b_bitsize) { @@ -2690,9 +2686,9 @@ tailrecur_ne: } case EXPORT_SUBTAG: { - if (is_export_rel(b,b_base)) { - Export* a_exp = *((Export **) (export_val_rel(a,a_base) + 1)); - Export* b_exp = *((Export **) (export_val_rel(b,b_base) + 1)); + if (is_export(b)) { + Export* a_exp = *((Export **) (export_val(a) + 1)); + Export* b_exp = *((Export **) (export_val(b) + 1)); if (a_exp == b_exp) goto pop_next; } break; /* not equal */ @@ -2702,10 +2698,10 @@ tailrecur_ne: ErlFunThing* f1; ErlFunThing* f2; - if (!is_fun_rel(b,b_base)) + if (!is_fun(b)) goto not_equal; - f1 = (ErlFunThing *) fun_val_rel(a,a_base); - f2 = (ErlFunThing *) fun_val_rel(b,b_base); + f1 = (ErlFunThing *) fun_val(a); + f2 = (ErlFunThing *) fun_val(b); if (f1->fe->module != f2->fe->module || f1->fe->old_index != f2->fe->old_index || f1->fe->old_uniq != f2->fe->old_uniq || @@ -2723,15 +2719,15 @@ tailrecur_ne: ExternalThing *ap; ExternalThing *bp; - if(!is_external_rel(b,b_base)) + if(!is_external(b)) goto not_equal; - ap = external_thing_ptr_rel(a,a_base); - bp = external_thing_ptr_rel(b,b_base); + ap = external_thing_ptr(a); + bp = external_thing_ptr(b); if(ap->header == bp->header && ap->node == bp->node) { - ASSERT(1 == external_data_words_rel(a,a_base)); - ASSERT(1 == external_data_words_rel(b,b_base)); + ASSERT(1 == external_data_words(a)); + ASSERT(1 == external_data_words(b)); if (ap->data.ui[0] == bp->data.ui[0]) goto pop_next; } @@ -2752,11 +2748,11 @@ tailrecur_ne: ExternalThing* athing; ExternalThing* bthing; - if(!is_external_ref_rel(b,b_base)) + if(!is_external_ref(b)) goto not_equal; - athing = external_thing_ptr_rel(a,a_base); - bthing = external_thing_ptr_rel(b,b_base); + athing = external_thing_ptr(a); + bthing = external_thing_ptr(b); if(athing->node != bthing->node) goto not_equal; @@ -2768,12 +2764,12 @@ tailrecur_ne: goto ref_common; case REF_SUBTAG: - if (!is_internal_ref_rel(b,b_base)) + if (!is_internal_ref(b)) goto not_equal; { - RefThing* athing = ref_thing_ptr_rel(a,a_base); - RefThing* bthing = ref_thing_ptr_rel(b,b_base); + RefThing* athing = ref_thing_ptr(a); + RefThing* bthing = ref_thing_ptr(b); alen = internal_thing_ref_no_of_numbers(athing); blen = internal_thing_ref_no_of_numbers(bthing); anum = internal_thing_ref_numbers(athing); @@ -2823,10 +2819,10 @@ tailrecur_ne: { int i; - if (!is_big_rel(b,b_base)) + if (!is_big(b)) goto not_equal; - aa = big_val_rel(a,a_base); - bb = big_val_rel(b,b_base); + aa = big_val(a); + bb = big_val(b); if (*aa != *bb) goto not_equal; i = BIG_ARITY(aa); @@ -2841,19 +2837,19 @@ tailrecur_ne: FloatDef af; FloatDef bf; - if (is_float_rel(b,b_base)) { - GET_DOUBLE_REL(a, af, a_base); - GET_DOUBLE_REL(b, bf, b_base); + if (is_float(b)) { + GET_DOUBLE(a, af); + GET_DOUBLE(b, bf); if (af.fd == bf.fd) goto pop_next; } break; /* not equal */ } case MAP_SUBTAG: - if (is_flatmap_rel(a, a_base)) { - aa = flatmap_val_rel(a, a_base); - if (!is_boxed(b) || *boxed_val_rel(b,b_base) != *aa) + if (is_flatmap(a)) { + aa = flatmap_val(a); + if (!is_boxed(b) || *boxed_val(b) != *aa) goto not_equal; - bb = flatmap_val_rel(b,b_base); + bb = flatmap_val(b); sz = flatmap_get_size((flatmap_t*)aa); if (sz != flatmap_get_size((flatmap_t*)bb)) goto not_equal; @@ -2865,11 +2861,11 @@ tailrecur_ne: goto term_array; } else { - if (!is_boxed(b) || *boxed_val_rel(b,b_base) != hdr) + if (!is_boxed(b) || *boxed_val(b) != hdr) goto not_equal; - aa = hashmap_val_rel(a, a_base) + 1; - bb = hashmap_val_rel(b, b_base) + 1; + aa = hashmap_val(a) + 1; + bb = hashmap_val(b) + 1; switch (hdr & _HEADER_MAP_SUBTAG_MASK) { case HAMT_SUBTAG_HEAD_ARRAY: aa++; bb++; @@ -2902,7 +2898,7 @@ term_array: /* arrays in 'aa' and 'bb', length in 'sz' */ Eterm* bp = bb; Sint i = sz; for (;;) { - if (!is_same(*ap,a_base,*bp,b_base)) break; + if (!is_same(*ap,*bp)) break; if (--i == 0) goto pop_next; ++ap; ++bp; @@ -2991,7 +2987,6 @@ static int cmp_atoms(Eterm a, Eterm b) bb->name+3, bb->len-3); } -#if !HALFWORD_HEAP /* cmp(Eterm a, Eterm b) * For compatibility with HiPE - arith-based compare. */ @@ -2999,39 +2994,22 @@ Sint cmp(Eterm a, Eterm b) { return erts_cmp(a, b, 0, 0); } -#endif -#if HALFWORD_HEAP -static Sint erts_cmp_compound_rel_opt(Eterm a, Eterm* a_base, - Eterm b, Eterm* b_base, - int exact, int eq_only); -#else static Sint erts_cmp_compound(Eterm a, Eterm b, int exact, int eq_only); -#endif -#if HALFWORD_HEAP -Sint erts_cmp_rel_opt(Eterm a, Eterm* a_base, - Eterm b, Eterm* b_base, - int exact, int eq_only) -#else Sint erts_cmp(Eterm a, Eterm b, int exact, int eq_only) -#endif { if (is_atom(a) && is_atom(b)) { return cmp_atoms(a, b); } else if (is_both_small(a, b)) { return (signed_val(a) - signed_val(b)); - } else if (is_float_rel(a, a_base) && is_float_rel(b, b_base)) { + } else if (is_float(a) && is_float(b)) { FloatDef af, bf; - GET_DOUBLE_REL(a, af, a_base); - GET_DOUBLE_REL(b, bf, b_base); + GET_DOUBLE(a, af); + GET_DOUBLE(b, bf); return float_comp(af.fd, bf.fd); } -#if HALFWORD_HEAP - return erts_cmp_compound_rel_opt(a,a_base,b,b_base,exact,eq_only); -#else return erts_cmp_compound(a,b,exact,eq_only); -#endif } @@ -3039,13 +3017,7 @@ Sint erts_cmp(Eterm a, Eterm b, int exact, int eq_only) * exact = 1 -> term-based compare * exact = 0 -> arith-based compare */ -#if HALFWORD_HEAP -static Sint erts_cmp_compound_rel_opt(Eterm a, Eterm* a_base, - Eterm b, Eterm* b_base, - int exact, int eq_only) -#else static Sint erts_cmp_compound(Eterm a, Eterm b, int exact, int eq_only) -#endif { #define PSTACK_TYPE struct erts_cmp_hashmap_state struct erts_cmp_hashmap_state { @@ -3112,7 +3084,7 @@ static Sint erts_cmp_compound(Eterm a, Eterm b, int exact, int eq_only) bodyrecur: j = 0; tailrecur: - if (is_same(a,a_base,b,b_base)) { /* Equal values or pointers. */ + if (is_same(a,b)) { /* Equal values or pointers. */ goto pop_next; } tailrecur_ne: @@ -3138,9 +3110,9 @@ tailrecur_ne: if (is_internal_port(b)) { bnode = erts_this_node; bdata = internal_port_data(b); - } else if (is_external_port_rel(b,b_base)) { - bnode = external_port_node_rel(b,b_base); - bdata = external_port_data_rel(b,b_base); + } else if (is_external_port(b)) { + bnode = external_port_node(b); + bdata = external_port_data(b); } else { a_tag = PORT_DEF; goto mixed_types; @@ -3156,9 +3128,9 @@ tailrecur_ne: if (is_internal_pid(b)) { bnode = erts_this_node; bdata = internal_pid_data(b); - } else if (is_external_pid_rel(b,b_base)) { - bnode = external_pid_node_rel(b,b_base); - bdata = external_pid_data_rel(b,b_base); + } else if (is_external_pid(b)) { + bnode = external_pid_node(b); + bdata = external_pid_data(b); } else { a_tag = PID_DEF; goto mixed_types; @@ -3191,12 +3163,12 @@ tailrecur_ne: a_tag = LIST_DEF; goto mixed_types; } - aa = list_val_rel(a,a_base); - bb = list_val_rel(b,b_base); + aa = list_val(a); + bb = list_val(b); while (1) { Eterm atmp = CAR(aa); Eterm btmp = CAR(bb); - if (!is_same(atmp,a_base,btmp,b_base)) { + if (!is_same(atmp,btmp)) { WSTACK_PUSH2(stack,(UWord) CDR(bb),(UWord) CDR(aa)); a = atmp; b = btmp; @@ -3204,7 +3176,7 @@ tailrecur_ne: } atmp = CDR(aa); btmp = CDR(bb); - if (is_same(atmp,a_base,btmp,b_base)) { + if (is_same(atmp,btmp)) { goto pop_next; } if (is_not_list(atmp) || is_not_list(btmp)) { @@ -3212,20 +3184,20 @@ tailrecur_ne: b = btmp; goto tailrecur_ne; } - aa = list_val_rel(atmp,a_base); - bb = list_val_rel(btmp,b_base); + aa = list_val(atmp); + bb = list_val(btmp); } case TAG_PRIMARY_BOXED: { - Eterm ahdr = *boxed_val_rel(a,a_base); + Eterm ahdr = *boxed_val(a); switch ((ahdr & _TAG_HEADER_MASK) >> _TAG_PRIMARY_SIZE) { case (_TAG_HEADER_ARITYVAL >> _TAG_PRIMARY_SIZE): - if (!is_tuple_rel(b,b_base)) { + if (!is_tuple(b)) { a_tag = TUPLE_DEF; goto mixed_types; } - aa = tuple_val_rel(a,a_base); - bb = tuple_val_rel(b,b_base); + aa = tuple_val(a); + bb = tuple_val(b); /* compare the arities */ i = arityval(ahdr); /* get the arity*/ if (i != arityval(*bb)) { @@ -3241,18 +3213,18 @@ tailrecur_ne: { struct erts_cmp_hashmap_state* sp; if (is_flatmap_header(ahdr)) { - if (!is_flatmap_rel(b,b_base)) { - if (is_hashmap_rel(b,b_base)) { - aa = (Eterm *)flatmap_val_rel(a,a_base); - i = flatmap_get_size((flatmap_t*)aa) - hashmap_size_rel(b,b_base); + if (!is_flatmap(b)) { + if (is_hashmap(b)) { + aa = (Eterm *)flatmap_val(a); + i = flatmap_get_size((flatmap_t*)aa) - hashmap_size(b); ASSERT(i != 0); RETURN_NEQ(i); } a_tag = MAP_DEF; goto mixed_types; } - aa = (Eterm *)flatmap_val_rel(a,a_base); - bb = (Eterm *)flatmap_val_rel(b,b_base); + aa = (Eterm *)flatmap_val(a); + bb = (Eterm *)flatmap_val(b); i = flatmap_get_size((flatmap_t*)aa); if (i != flatmap_get_size((flatmap_t*)bb)) { @@ -3279,21 +3251,21 @@ tailrecur_ne: goto bodyrecur; } } - if (!is_hashmap_rel(b,b_base)) { - if (is_flatmap_rel(b,b_base)) { - bb = (Eterm *)flatmap_val_rel(b,b_base); - i = hashmap_size_rel(a,a_base) - flatmap_get_size((flatmap_t*)bb); + if (!is_hashmap(b)) { + if (is_flatmap(b)) { + bb = (Eterm *)flatmap_val(b); + i = hashmap_size(a) - flatmap_get_size((flatmap_t*)bb); ASSERT(i != 0); RETURN_NEQ(i); } a_tag = MAP_DEF; goto mixed_types; } - i = hashmap_size_rel(a,a_base) - hashmap_size_rel(b,b_base); + i = hashmap_size(a) - hashmap_size(b); if (i) { RETURN_NEQ(i); } - if (hashmap_size_rel(a,a_base) == 0) { + if (hashmap_size(a) == 0) { goto pop_next; } @@ -3328,31 +3300,31 @@ tailrecur_ne: goto bodyrecur; } case (_TAG_HEADER_FLOAT >> _TAG_PRIMARY_SIZE): - if (!is_float_rel(b,b_base)) { + if (!is_float(b)) { a_tag = FLOAT_DEF; goto mixed_types; } else { FloatDef af; FloatDef bf; - GET_DOUBLE_REL(a, af, a_base); - GET_DOUBLE_REL(b, bf, b_base); + GET_DOUBLE(a, af); + GET_DOUBLE(b, bf); ON_CMP_GOTO(float_comp(af.fd, bf.fd)); } case (_TAG_HEADER_POS_BIG >> _TAG_PRIMARY_SIZE): case (_TAG_HEADER_NEG_BIG >> _TAG_PRIMARY_SIZE): - if (!is_big_rel(b,b_base)) { + if (!is_big(b)) { a_tag = BIG_DEF; goto mixed_types; } - ON_CMP_GOTO(big_comp(rterm2wterm(a,a_base), rterm2wterm(b,b_base))); + ON_CMP_GOTO(big_comp(a, b)); case (_TAG_HEADER_EXPORT >> _TAG_PRIMARY_SIZE): - if (!is_export_rel(b,b_base)) { + if (!is_export(b)) { a_tag = EXPORT_DEF; goto mixed_types; } else { - Export* a_exp = *((Export **) (export_val_rel(a,a_base) + 1)); - Export* b_exp = *((Export **) (export_val_rel(b,b_base) + 1)); + Export* a_exp = *((Export **) (export_val(a) + 1)); + Export* b_exp = *((Export **) (export_val(b) + 1)); if ((j = cmp_atoms(a_exp->code[0], b_exp->code[0])) != 0) { RETURN_NEQ(j); @@ -3364,12 +3336,12 @@ tailrecur_ne: } break; case (_TAG_HEADER_FUN >> _TAG_PRIMARY_SIZE): - if (!is_fun_rel(b,b_base)) { + if (!is_fun(b)) { a_tag = FUN_DEF; goto mixed_types; } else { - ErlFunThing* f1 = (ErlFunThing *) fun_val_rel(a,a_base); - ErlFunThing* f2 = (ErlFunThing *) fun_val_rel(b,b_base); + ErlFunThing* f1 = (ErlFunThing *) fun_val(a); + ErlFunThing* f2 = (ErlFunThing *) fun_val(b); Sint diff; diff = cmpbytes(atom_tab(atom_val(f1->fe->module))->name, @@ -3401,29 +3373,29 @@ tailrecur_ne: if (is_internal_pid(b)) { bnode = erts_this_node; bdata = internal_pid_data(b); - } else if (is_external_pid_rel(b,b_base)) { - bnode = external_pid_node_rel(b,b_base); - bdata = external_pid_data_rel(b,b_base); + } else if (is_external_pid(b)) { + bnode = external_pid_node(b); + bdata = external_pid_data(b); } else { a_tag = EXTERNAL_PID_DEF; goto mixed_types; } - anode = external_pid_node_rel(a,a_base); - adata = external_pid_data_rel(a,a_base); + anode = external_pid_node(a); + adata = external_pid_data(a); goto pid_common; case (_TAG_HEADER_EXTERNAL_PORT >> _TAG_PRIMARY_SIZE): if (is_internal_port(b)) { bnode = erts_this_node; bdata = internal_port_data(b); - } else if (is_external_port_rel(b,b_base)) { - bnode = external_port_node_rel(b,b_base); - bdata = external_port_data_rel(b,b_base); + } else if (is_external_port(b)) { + bnode = external_port_node(b); + bdata = external_port_data(b); } else { a_tag = EXTERNAL_PORT_DEF; goto mixed_types; } - anode = external_port_node_rel(a,a_base); - adata = external_port_data_rel(a,a_base); + anode = external_port_node(a); + adata = external_port_data(a); goto port_common; case (_TAG_HEADER_REF >> _TAG_PRIMARY_SIZE): /* @@ -3431,14 +3403,13 @@ tailrecur_ne: * (32-bit words), *not* ref data words. */ - - if (is_internal_ref_rel(b,b_base)) { - RefThing* bthing = ref_thing_ptr_rel(b,b_base); + if (is_internal_ref(b)) { + RefThing* bthing = ref_thing_ptr(b); bnode = erts_this_node; bnum = internal_thing_ref_numbers(bthing); blen = internal_thing_ref_no_of_numbers(bthing); - } else if(is_external_ref_rel(b,b_base)) { - ExternalThing* bthing = external_thing_ptr_rel(b,b_base); + } else if(is_external_ref(b)) { + ExternalThing* bthing = external_thing_ptr(b); bnode = bthing->node; bnum = external_thing_ref_numbers(bthing); blen = external_thing_ref_no_of_numbers(bthing); @@ -3447,7 +3418,7 @@ tailrecur_ne: goto mixed_types; } { - RefThing* athing = ref_thing_ptr_rel(a,a_base); + RefThing* athing = ref_thing_ptr(a); anode = erts_this_node; anum = internal_thing_ref_numbers(athing); alen = internal_thing_ref_no_of_numbers(athing); @@ -3480,13 +3451,13 @@ tailrecur_ne: RETURN_NEQ((Sint32) (anum[i] - bnum[i])); goto pop_next; case (_TAG_HEADER_EXTERNAL_REF >> _TAG_PRIMARY_SIZE): - if (is_internal_ref_rel(b,b_base)) { - RefThing* bthing = ref_thing_ptr_rel(b,b_base); + if (is_internal_ref(b)) { + RefThing* bthing = ref_thing_ptr(b); bnode = erts_this_node; bnum = internal_thing_ref_numbers(bthing); blen = internal_thing_ref_no_of_numbers(bthing); - } else if (is_external_ref_rel(b,b_base)) { - ExternalThing* bthing = external_thing_ptr_rel(b,b_base); + } else if (is_external_ref(b)) { + ExternalThing* bthing = external_thing_ptr(b); bnode = bthing->node; bnum = external_thing_ref_numbers(bthing); blen = external_thing_ref_no_of_numbers(bthing); @@ -3495,7 +3466,7 @@ tailrecur_ne: goto mixed_types; } { - ExternalThing* athing = external_thing_ptr_rel(a,a_base); + ExternalThing* athing = external_thing_ptr(a); anode = athing->node; anum = external_thing_ref_numbers(athing); alen = external_thing_ref_no_of_numbers(athing); @@ -3503,13 +3474,13 @@ tailrecur_ne: goto ref_common; default: /* Must be a binary */ - ASSERT(is_binary_rel(a,a_base)); - if (!is_binary_rel(b,b_base)) { + ASSERT(is_binary(a)); + if (!is_binary(b)) { a_tag = BINARY_DEF; goto mixed_types; } else { - Uint a_size = binary_size_rel(a,a_base); - Uint b_size = binary_size_rel(b,b_base); + Uint a_size = binary_size(a); + Uint b_size = binary_size(b); Uint a_bitsize; Uint b_bitsize; Uint a_bitoffs; @@ -3518,8 +3489,8 @@ tailrecur_ne: int cmp; byte* a_ptr; byte* b_ptr; - ERTS_GET_BINARY_BYTES_REL(a, a_ptr, a_bitoffs, a_bitsize, a_base); - ERTS_GET_BINARY_BYTES_REL(b, b_ptr, b_bitoffs, b_bitsize, b_base); + ERTS_GET_BINARY_BYTES(a, a_ptr, a_bitoffs, a_bitsize); + ERTS_GET_BINARY_BYTES(b, b_ptr, b_bitoffs, b_bitsize); if ((a_bitsize | b_bitsize | a_bitoffs | b_bitoffs) == 0) { min_size = (a_size < b_size) ? a_size : b_size; if ((cmp = sys_memcmp(a_ptr, b_ptr, min_size)) != 0) { @@ -3550,13 +3521,8 @@ tailrecur_ne: { FloatDef f1, f2; Eterm big; -#if HALFWORD_HEAP - Wterm aw = is_immed(a) ? a : rterm2wterm(a,a_base); - Wterm bw = is_immed(b) ? b : rterm2wterm(b,b_base); -#else Eterm aw = a; Eterm bw = b; -#endif #define MAX_LOSSLESS_FLOAT ((double)((1LL << 53) - 2)) #define MIN_LOSSLESS_FLOAT ((double)(((1LL << 53) - 2)*-1)) #define BIG_ARITY_FLOAT_MAX (1024 / D_EXP) /* arity of max float as a bignum */ @@ -3628,7 +3594,7 @@ tailrecur_ne: } } else { big = double_to_big(f2.fd, big_buf, sizeof(big_buf)/sizeof(Eterm)); - j = big_comp(aw, rterm2wterm(big,big_buf)); + j = big_comp(aw, big); } if (_NUMBER_CODE(a_tag, b_tag) == FLOAT_BIG) { j = -j; @@ -3676,7 +3642,7 @@ term_array: /* arrays in 'aa' and 'bb', length in 'i' */ while (--i) { a = *aa++; b = *bb++; - if (!is_same(a,a_base, b,b_base)) { + if (!is_same(a, b)) { if (is_atom(a) && is_atom(b)) { if ((j = cmp_atoms(a, b)) != 0) { goto not_equal; @@ -4368,7 +4334,7 @@ iolist_size(const int yield_support, ErtsIOListState *state, Eterm obj, ErlDrvSi { int res, init_yield_count, yield_count; Eterm* objp; - Uint size = (Uint) *sizep; /* Intentionally Uint due to halfword heap */ + Uint size = (Uint) *sizep; DECLARE_ESTACK(s); if (!yield_support) |