diff options
Diffstat (limited to 'erts/emulator/beam')
40 files changed, 1185 insertions, 729 deletions
diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index 65cc5d0ceb..5ec1409adf 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -107,7 +107,7 @@ atom await_sched_wall_time_modifications atom awaiting_load atom awaiting_unload atom backtrace backtrace_depth -atom badarg badarith badarity badfile badmatch badsig badfun +atom badarg badarith badarity badfile badfun badkey badmap badmatch badsig atom bag atom band atom big diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index fe0fa22751..500a98195b 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -39,12 +39,9 @@ static void set_default_trace_pattern(Eterm module); static Eterm check_process_code(Process* rp, Module* modp, int allow_gc, int *redsp); static void delete_code(Module* modp); static void decrement_refc(BeamInstr* code); -static int is_native(BeamInstr* code); static int any_heap_ref_ptrs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size); static int any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size); - - BIF_RETTYPE code_is_module_native_1(BIF_ALIST_1) { Module* modp; @@ -59,8 +56,8 @@ BIF_RETTYPE code_is_module_native_1(BIF_ALIST_1) return am_undefined; } erts_rlock_old_code(code_ix); - res = ((modp->curr.code && is_native(modp->curr.code)) || - (modp->old.code != 0 && is_native(modp->old.code))) ? + res = (erts_is_module_native(modp->curr.code) || + erts_is_module_native(modp->old.code)) ? am_true : am_false; erts_runlock_old_code(code_ix); return res; @@ -1106,25 +1103,3 @@ beam_make_current_old(Process *c_p, ErtsProcLocks c_p_locks, Eterm module) } return NIL; } - -static int -is_native(BeamInstr* code) -{ - Uint i, num_functions = code[MI_NUM_FUNCTIONS]; - - /* Check NativeAdress of first real function in module - */ - for (i=0; i<num_functions; i++) { - BeamInstr* func_info = (BeamInstr *) code[MI_FUNCTIONS+i]; - Eterm name = (Eterm) func_info[3]; - - if (is_atom(name)) { - return func_info[1] != 0; - } - else ASSERT(is_nil(name)); /* ignore BIF stubs */ - } - /* Not a single non-BIF function? */ - return 0; -} - - diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c index 6bb987985d..0367ca8aba 100644 --- a/erts/emulator/beam/beam_debug.c +++ b/erts/emulator/beam/beam_debug.c @@ -661,10 +661,9 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) case op_i_put_tuple_rI: case op_i_put_tuple_xI: case op_i_put_tuple_yI: - case op_new_map_jdII: + case op_new_map_dII: case op_update_map_assoc_jsdII: case op_update_map_exact_jsdII: - case op_i_has_map_fields_fsI: case op_i_get_map_elements_fsI: { int n = unpacked[-1]; diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 4b2ab72a29..9fe02c3724 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -562,7 +562,8 @@ void** beam_ops; Store(term, Dst); \ } while (0) -#define Move2(src1, dst1, src2, dst2) dst1 = (src1); dst2 = (src2) +#define Move2(S1, D1, S2, D2) D1 = (S1); D2 = (S2) +#define Move3(S1, D1, S2, D2, S3, D3) D1 = (S1); D2 = (S2); D3 = (S3) #define MoveGenDest(src, dstp) \ if ((dstp) == NULL) { r(0) = (src); } else { *(dstp) = src; } @@ -662,6 +663,9 @@ void** beam_ops; #define EqualImmed(X, Y, Action) if (X != Y) { Action; } #define NotEqualImmed(X, Y, Action) if (X == Y) { Action; } +#define EqualExact(X, Y, Action) if (!EQ(X,Y)) { Action; } +#define IsLessThan(X, Y, Action) if (CMP_GE(X, Y)) { Action; } +#define IsGreaterEqual(X, Y, Action) if (CMP_LT(X, Y)) { Action; } #define IsFloat(Src, Fail) if (is_not_float(Src)) { Fail; } @@ -701,8 +705,6 @@ void** beam_ops; #define IsMap(Src, Fail) if (!is_map(Src)) { Fail; } -#define HasMapField(Src, Key, Fail) if (has_not_map_field(Src, Key)) { Fail; } - #define GetMapElement(Src, Key, Dst, Fail) \ do { \ Eterm _res = get_map_element(Src, Key); \ @@ -712,6 +714,15 @@ void** beam_ops; Dst = _res; \ } while (0) +#define GetMapElementHash(Src, Key, Hx, Dst, Fail) \ + do { \ + Eterm _res = get_map_element_hash(Src, Key, Hx); \ + if (is_non_value(_res)) { \ + Fail; \ + } \ + Dst = _res; \ + } while (0) + #define IsFunction(X, Action) \ do { \ if ( !(is_any_fun(X)) ) { \ @@ -960,8 +971,8 @@ static Eterm update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I) NOINLINE; static Eterm update_map_exact(Process* p, Eterm* reg, Eterm map, BeamInstr* I) NOINLINE; -static int has_not_map_field(Eterm map, Eterm key); static Eterm get_map_element(Eterm map, Eterm key); +static Eterm get_map_element_hash(Eterm map, Eterm key, Uint32 hx); /* * Functions not directly called by process_main(). OK to inline. @@ -1077,16 +1088,32 @@ init_emulator(void) DTRACE2(nif_return, process_name, mfa); \ } -#else /* USE_VM_PROBES */ - -#define DTRACE_LOCAL_CALL(p, m, f, a) do {} while (0) -#define DTRACE_GLOBAL_CALL(p, m, f, a) do {} while (0) -#define DTRACE_RETURN(p, m, f, a) do {} while (0) -#define DTRACE_BIF_ENTRY(p, m, f, a) do {} while (0) -#define DTRACE_BIF_RETURN(p, m, f, a) do {} while (0) -#define DTRACE_NIF_ENTRY(p, m, f, a) do {} while (0) -#define DTRACE_NIF_RETURN(p, m, f, a) do {} while (0) +#define DTRACE_GLOBAL_CALL_FROM_EXPORT(p,e) \ + do { \ + if (DTRACE_ENABLED(global_function_entry)) { \ + BeamInstr* fp = (BeamInstr *) (((Export *) (e))->addressv[erts_active_code_ix()]); \ + DTRACE_GLOBAL_CALL((p), (Eterm)fp[-3], (Eterm)fp[-2], fp[-1]); \ + } \ + } while(0) + +#define DTRACE_RETURN_FROM_PC(p) \ + do { \ + BeamInstr* fp; \ + if (DTRACE_ENABLED(function_return) && (fp = find_function_from_pc((p)->cp))) { \ + DTRACE_RETURN((p), (Eterm)fp[0], (Eterm)fp[1], (Uint)fp[2]); \ + } \ + } while(0) +#else /* USE_VM_PROBES */ +#define DTRACE_LOCAL_CALL(p, m, f, a) do {} while (0) +#define DTRACE_GLOBAL_CALL(p, m, f, a) do {} while (0) +#define DTRACE_GLOBAL_CALL_FROM_EXPORT(p, e) do {} while (0) +#define DTRACE_RETURN(p, m, f, a) do {} while (0) +#define DTRACE_RETURN_FROM_PC(p) do {} while (0) +#define DTRACE_BIF_ENTRY(p, m, f, a) do {} while (0) +#define DTRACE_BIF_RETURN(p, m, f, a) do {} while (0) +#define DTRACE_NIF_ENTRY(p, m, f, a) do {} while (0) +#define DTRACE_NIF_RETURN(p, m, f, a) do {} while (0) #endif /* USE_VM_PROBES */ /* @@ -1366,7 +1393,39 @@ void process_main(void) ASSERT(c_p->freason != BADMATCH || is_value(c_p->fvalue)); goto find_func_info; } - + +#define DO_BIG_ARITH(Func,Arg1,Arg2) \ + do { \ + Uint live = Arg(1); \ + SWAPOUT; \ + reg[0] = r(0); \ + reg[live] = (Arg1); \ + reg[live+1] = (Arg2); \ + result = (Func)(c_p, reg, live); \ + r(0) = reg[0]; \ + SWAPIN; \ + ERTS_HOLE_CHECK(c_p); \ + if (is_value(result)) { \ + StoreBifResult(4,result); \ + } \ + goto lb_Cl_error; \ + } while(0) + + OpCase(i_plus_jIxxd): + { + Eterm result; + + if (is_both_small(xb(Arg(2)), xb(Arg(3)))) { + Sint i = signed_val(xb(Arg(2))) + signed_val(xb(Arg(3))); + ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i)); + if (MY_IS_SSMALL(i)) { + result = make_small(i); + StoreBifResult(4, result); + } + } + DO_BIG_ARITH(ARITH_FUNC(mixed_plus), xb(Arg(2)), xb(Arg(3))); + } + OpCase(i_plus_jId): { Eterm result; @@ -1378,12 +1437,26 @@ void process_main(void) result = make_small(i); STORE_ARITH_RESULT(result); } - } arith_func = ARITH_FUNC(mixed_plus); goto do_big_arith2; } + OpCase(i_minus_jIxxd): + { + Eterm result; + + if (is_both_small(xb(Arg(2)), xb(Arg(3)))) { + Sint i = signed_val(xb(Arg(2))) - signed_val(xb(Arg(3))); + ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i)); + if (MY_IS_SSMALL(i)) { + result = make_small(i); + StoreBifResult(4, result); + } + } + DO_BIG_ARITH(ARITH_FUNC(mixed_minus), xb(Arg(2)), xb(Arg(3))); + } + OpCase(i_minus_jId): { Eterm result; @@ -1476,6 +1549,52 @@ void process_main(void) Next(2); } + OpCase(move_window3_xxxy): { + BeamInstr *next; + Eterm xt0, xt1, xt2; + Eterm *y = (Eterm *)(((unsigned char *)E) + (Arg(3))); + PreFetch(4, next); + xt0 = xb(Arg(0)); + xt1 = xb(Arg(1)); + xt2 = xb(Arg(2)); + y[0] = xt0; + y[1] = xt1; + y[2] = xt2; + NextPF(4, next); + } + OpCase(move_window4_xxxxy): { + BeamInstr *next; + Eterm xt0, xt1, xt2, xt3; + Eterm *y = (Eterm *)(((unsigned char *)E) + (Arg(4))); + PreFetch(5, next); + xt0 = xb(Arg(0)); + xt1 = xb(Arg(1)); + xt2 = xb(Arg(2)); + xt3 = xb(Arg(3)); + y[0] = xt0; + y[1] = xt1; + y[2] = xt2; + y[3] = xt3; + NextPF(5, next); + } + OpCase(move_window5_xxxxxy): { + BeamInstr *next; + Eterm xt0, xt1, xt2, xt3, xt4; + Eterm *y = (Eterm *)(((unsigned char *)E) + (Arg(5))); + PreFetch(6, next); + xt0 = xb(Arg(0)); + xt1 = xb(Arg(1)); + xt2 = xb(Arg(2)); + xt3 = xb(Arg(3)); + xt4 = xb(Arg(4)); + y[0] = xt0; + y[1] = xt1; + y[2] = xt2; + y[3] = xt3; + y[4] = xt4; + NextPF(6, next); + } + OpCase(i_move_call_only_fcr): { r(0) = Arg(1); } @@ -1523,12 +1642,7 @@ void process_main(void) * is not loaded, it points to code which will invoke the error handler * (see lb_call_error_handler below). */ -#ifdef USE_VM_CALL_PROBES - if (DTRACE_ENABLED(global_function_entry)) { - BeamInstr* fp = (BeamInstr *) (((Export *) Arg(0))->addressv[erts_active_code_ix()]); - DTRACE_GLOBAL_CALL(c_p, (Eterm)fp[-3], (Eterm)fp[-2], fp[-1]); - } -#endif + DTRACE_GLOBAL_CALL_FROM_EXPORT(c_p, Arg(0)); Dispatchx(); OpCase(i_move_call_ext_cre): { @@ -1538,12 +1652,7 @@ void process_main(void) /* FALL THROUGH */ OpCase(i_call_ext_e): SET_CP(c_p, I+2); -#ifdef USE_VM_CALL_PROBES - if (DTRACE_ENABLED(global_function_entry)) { - BeamInstr* fp = (BeamInstr *) (((Export *) Arg(0))->addressv[erts_active_code_ix()]); - DTRACE_GLOBAL_CALL(c_p, (Eterm)fp[-3], (Eterm)fp[-2], fp[-1]); - } -#endif + DTRACE_GLOBAL_CALL_FROM_EXPORT(c_p, Arg(0)); Dispatchx(); OpCase(i_move_call_ext_only_ecr): { @@ -1551,12 +1660,7 @@ void process_main(void) } /* FALL THROUGH */ OpCase(i_call_ext_only_e): -#ifdef USE_VM_CALL_PROBES - if (DTRACE_ENABLED(global_function_entry)) { - BeamInstr* fp = (BeamInstr *) (((Export *) Arg(0))->addressv[erts_active_code_ix()]); - DTRACE_GLOBAL_CALL(c_p, (Eterm)fp[-3], (Eterm)fp[-2], fp[-1]); - } -#endif + DTRACE_GLOBAL_CALL_FROM_EXPORT(c_p, Arg(0)); Dispatchx(); OpCase(init_y): { @@ -1590,18 +1694,9 @@ void process_main(void) Next(1); } - OpCase(return): { -#ifdef USE_VM_CALL_PROBES - BeamInstr* fptr; -#endif SET_I(c_p->cp); - -#ifdef USE_VM_CALL_PROBES - if (DTRACE_ENABLED(function_return) && (fptr = find_function_from_pc(c_p->cp))) { - DTRACE_RETURN(c_p, (Eterm)fptr[0], (Eterm)fptr[1], (Uint)fptr[2]); - } -#endif + DTRACE_RETURN_FROM_PC(c_p); /* * We must clear the CP to make sure that a stale value do not * create a false module dependcy preventing code upgrading. @@ -2367,77 +2462,16 @@ void process_main(void) Goto(*I); } - OpCase(new_map_jdII): { + OpCase(new_map_dII): { Eterm res; x(0) = r(0); SWAPOUT; - res = new_map(c_p, reg, I); + res = new_map(c_p, reg, I-1); SWAPIN; r(0) = x(0); - StoreResult(res, Arg(1)); - Next(4+Arg(3)); - } - - OpCase(i_has_map_fields_fsI): { - flatmap_t* mp; - Eterm map; - Eterm field; - Eterm *ks; - BeamInstr* fs; - Uint sz,n; - - GetArg1(1, map); - n = (Uint)Arg(2); - fs = &Arg(3); /* pattern fields */ - - /* get term from field? */ - if (is_hashmap(map)) { - Uint32 hx; - while(n--) { - field = *fs++; - hx = hashmap_make_hash(field); - if (!erts_hashmap_get(hx,field,map)) { - SET_I((BeamInstr *) Arg(0)); - goto has_map_fields_fail; - } - } - goto has_map_fields_ok; - } - - ASSERT(is_flatmap(map)); - - mp = (flatmap_t *)flatmap_val(map); - sz = flatmap_get_size(mp); - - if (sz == 0) { - SET_I((BeamInstr *) Arg(0)); - goto has_map_fields_fail; - } - - ks = flatmap_get_keys(mp); - - ASSERT(n>0); - - while(sz) { - field = (Eterm)*fs; - if (EQ(field,*ks)) { - n--; - fs++; - if (n == 0) break; - } - ks++; sz--; - } - - if (n) { - SET_I((BeamInstr *) Arg(0)); - goto has_map_fields_fail; - } -has_map_fields_ok: - I += 4 + Arg(2); -has_map_fields_fail: - ASSERT(VALID_INSTR(*I)); - Goto(*I); + StoreResult(res, Arg(0)); + Next(3+Arg(2)); } #define PUT_TERM_REG(term, desc) \ @@ -2469,7 +2503,7 @@ do { \ * i.e. that it follows a test is_map if needed. */ - n = (Uint)Arg(2) / 2; + n = (Uint)Arg(2) / 3; fs = &Arg(3); /* pattern fields and target registers */ if (is_flatmap(map)) { @@ -2481,49 +2515,43 @@ do { \ sz = flatmap_get_size(mp); if (sz == 0) { - SET_I((BeamInstr *) Arg(0)); - goto get_map_elements_fail; + ClauseFail(); } ks = flatmap_get_keys(mp); vs = flatmap_get_values(mp); while(sz) { - if (EQ((Eterm)*fs,*ks)) { + if (EQ((Eterm) fs[0], *ks)) { PUT_TERM_REG(*vs, fs[1]); n--; - fs += 2; + fs += 3; /* no more values to fetch, we are done */ - if (n == 0) break; + if (n == 0) { + I = fs; + Next(-1); + } } - ks++; sz--; - vs++; + ks++, sz--, vs++; } - if (n) { - SET_I((BeamInstr *) Arg(0)); - goto get_map_elements_fail; - } + ClauseFail(); } else { const Eterm *v; Uint32 hx; ASSERT(is_hashmap(map)); while(n--) { - hx = hashmap_make_hash((Eterm)*fs); - if ((v = erts_hashmap_get(hx,(Eterm)*fs, map)) == NULL) { - SET_I((BeamInstr *) Arg(0)); - goto get_map_elements_fail; + hx = fs[2]; + ASSERT(hx == hashmap_make_hash((Eterm)fs[0])); + if ((v = erts_hashmap_get(hx, (Eterm)fs[0], map)) == NULL) { + ClauseFail(); } PUT_TERM_REG(*v, fs[1]); - fs += 2; + fs += 3; } + I = fs; + Next(-1); } - - - I += 4 + Arg(2); -get_map_elements_fail: - ASSERT(VALID_INSTR(*I)); - Goto(*I); } #undef PUT_TERM_REG @@ -2541,7 +2569,13 @@ get_map_elements_fail: StoreResult(res, Arg(2)); Next(5+Arg(4)); } else { - goto badarg; + /* + * This can only happen if the code was compiled + * with the compiler in OTP 17. + */ + c_p->freason = BADMAP; + c_p->fvalue = map; + goto lb_Cl_error; } } @@ -2559,7 +2593,7 @@ get_map_elements_fail: StoreResult(res, Arg(2)); Next(5+Arg(4)); } else { - goto badarg; + goto lb_Cl_error; } } @@ -2885,6 +2919,19 @@ get_map_elements_fail: goto do_big_arith2; } + OpCase(i_rem_jIxxd): + { + Eterm result; + + if (xb(Arg(3)) == SMALL_ZERO) { + goto badarith; + } else if (is_both_small(xb(Arg(2)), xb(Arg(3)))) { + result = make_small(signed_val(xb(Arg(2))) % signed_val(xb(Arg(3)))); + StoreBifResult(4, result); + } + DO_BIG_ARITH(ARITH_FUNC(int_rem),xb(Arg(2)),xb(Arg(3))); + } + OpCase(i_rem_jId): { Eterm result; @@ -2900,6 +2947,20 @@ get_map_elements_fail: } } + OpCase(i_band_jIxcd): + { + Eterm result; + + if (is_both_small(xb(Arg(2)), Arg(3))) { + /* + * No need to untag -- TAG & TAG == TAG. + */ + result = xb(Arg(2)) & Arg(3); + StoreBifResult(4, result); + } + DO_BIG_ARITH(ARITH_FUNC(band),xb(Arg(2)),Arg(3)); + } + OpCase(i_band_jId): { Eterm result; @@ -2915,6 +2976,8 @@ get_map_elements_fail: goto do_big_arith2; } +#undef DO_BIG_ARITH + do_big_arith2: { Eterm result; @@ -3566,6 +3629,8 @@ get_map_elements_fail: erts_pre_nif(&env, c_p, (struct erl_module_nif*)I[2]); reg[0] = r(0); nif_bif_result = (*fp)(&env, bif_nif_arity, reg); + if (env.exception_thrown) + nif_bif_result = THE_NON_VALUE; erts_post_nif(&env); } ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(nif_bif_result)); @@ -5187,8 +5252,6 @@ get_map_elements_fail: #ifndef NO_JUMP_TABLE #ifdef ERTS_OPCODE_COUNTER_SUPPORT - /* Are tables correctly generated by beam_makeops? */ - ERTS_CT_ASSERT(sizeof(counting_opcodes) == sizeof(opcodes)); #ifdef DEBUG counting_opcodes[op_catch_end_y] = LabelAddr(lb_catch_end_y); #endif @@ -5309,7 +5372,9 @@ Eterm error_atom[NUMBER_EXIT_CODES] = { am_notalive, /* 14 */ am_system_limit, /* 15 */ am_try_clause, /* 16 */ - am_notsup /* 17 */ + am_notsup, /* 17 */ + am_badmap, /* 18 */ + am_badkey, /* 19 */ }; /* @@ -5565,6 +5630,8 @@ expand_error_value(Process* c_p, Uint freason, Eterm Value) { case (GET_EXC_INDEX(EXC_TRY_CLAUSE)): case (GET_EXC_INDEX(EXC_BADFUN)): case (GET_EXC_INDEX(EXC_BADARITY)): + case (GET_EXC_INDEX(EXC_BADMAP)): + case (GET_EXC_INDEX(EXC_BADKEY)): /* Some common exceptions: value -> {atom, value} */ ASSERT(is_value(Value)); hp = HAlloc(c_p, 3); @@ -6075,13 +6142,7 @@ apply(Process* p, Eterm module, Eterm function, Eterm args, Eterm* reg) } else if (ERTS_PROC_GET_SAVED_CALLS_BUF(p)) { save_calls(p, ep); } - -#ifdef USE_VM_CALL_PROBES - if (DTRACE_ENABLED(global_function_entry)) { - BeamInstr *fptr = (BeamInstr *) ep->addressv[erts_active_code_ix()]; - DTRACE_GLOBAL_CALL(p, (Eterm)fptr[-3], (Eterm)fptr[-2], (Uint)fptr[-1]); - } -#endif + DTRACE_GLOBAL_CALL_FROM_EXPORT(p, ep); return ep->addressv[erts_active_code_ix()]; } @@ -6130,13 +6191,7 @@ fixed_apply(Process* p, Eterm* reg, Uint arity) } else if (ERTS_PROC_GET_SAVED_CALLS_BUF(p)) { save_calls(p, ep); } - -#ifdef USE_VM_CALL_PROBES - if (DTRACE_ENABLED(global_function_entry)) { - BeamInstr *fptr = (BeamInstr *) ep->addressv[erts_active_code_ix()]; - DTRACE_GLOBAL_CALL(p, (Eterm)fptr[-3], (Eterm)fptr[-2], (Uint)fptr[-1]); - } -#endif + DTRACE_GLOBAL_CALL_FROM_EXPORT(p, ep); return ep->addressv[erts_active_code_ix()]; } @@ -6458,42 +6513,45 @@ new_fun(Process* p, Eterm* reg, ErlFunEntry* fe, int num_free) return make_fun(funp); } -static int has_not_map_field(Eterm map, Eterm key) +static Eterm get_map_element(Eterm map, Eterm key) { Uint32 hx; + const Eterm *vs; if (is_flatmap(map)) { - flatmap_t* mp; - Eterm* keys; + flatmap_t *mp; + Eterm *ks; Uint i; Uint n; - mp = (flatmap_t *)flatmap_val(map); - keys = flatmap_get_keys(mp); - n = flatmap_get_size(mp); + mp = (flatmap_t *)flatmap_val(map); + ks = flatmap_get_keys(mp); + vs = flatmap_get_values(mp); + n = flatmap_get_size(mp); if (is_immed(key)) { for (i = 0; i < n; i++) { - if (keys[i] == key) { - return 0; + if (ks[i] == key) { + return vs[i]; } } } else { - for (i = 0; i < n; i++) { - if (EQ(keys[i], key)) { - return 0; + for (i = 0; i < n; i++) { + if (EQ(ks[i], key)) { + return vs[i]; } } } - return 1; + return THE_NON_VALUE; } ASSERT(is_hashmap(map)); hx = hashmap_make_hash(key); - return erts_hashmap_get(hx,key,map) ? 0 : 1; + vs = erts_hashmap_get(hx,key,map); + return vs ? *vs : THE_NON_VALUE; } -static Eterm get_map_element(Eterm map, Eterm key) +static Eterm get_map_element_hash(Eterm map, Eterm key, Uint32 hx) { - Uint32 hx; const Eterm *vs; + if (is_flatmap(map)) { flatmap_t *mp; Eterm *ks; @@ -6519,9 +6577,10 @@ static Eterm get_map_element(Eterm map, Eterm key) } return THE_NON_VALUE; } + ASSERT(is_hashmap(map)); - hx = hashmap_make_hash(key); - vs = erts_hashmap_get(hx,key,map); + ASSERT(hx == hashmap_make_hash(key)); + vs = erts_hashmap_get(hx, key, map); return vs ? *vs : THE_NON_VALUE; } @@ -6561,6 +6620,7 @@ new_map(Process* p, Eterm* reg, BeamInstr* I) ptr = &Arg(4); if (n > 2*MAP_SMALL_MAP_LIMIT) { + Eterm res; if (HeapWordsLeft(p) < n) { erts_garbage_collect(p, n, reg, Arg(2)); } @@ -6577,7 +6637,15 @@ new_map(Process* p, Eterm* reg, BeamInstr* I) p->htop = mhp; factory.p = p; - return erts_hashmap_from_array(&factory, thp, n/2, 0); + res = erts_hashmap_from_array(&factory, thp, n/2, 0); + if (p->mbuf) { + Uint live = Arg(2); + reg[live] = res; + erts_garbage_collect(p, 0, reg, live+1); + res = reg[live]; + E = p->stop; + } + return res; } if (HeapWordsLeft(p) < need) { @@ -6647,10 +6715,9 @@ update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I) reg[live] = res; erts_garbage_collect(p, 0, reg, live+1); res = reg[live]; + E = p->stop; } - E = p->stop; - new_p += 2; } return res; @@ -6850,30 +6917,34 @@ update_map_exact(Process* p, Eterm* reg, Eterm map, BeamInstr* I) /* apparently the compiler does not emit is_map instructions, * bad compiler */ - if (is_not_hashmap(map)) + if (is_not_hashmap(map)) { + p->freason = BADMAP; + p->fvalue = map; return THE_NON_VALUE; + } res = map; E = p->stop; while(n--) { - /* assoc can't fail */ GET_TERM(new_p[0], new_key); GET_TERM(new_p[1], val); hx = hashmap_make_hash(new_key); res = erts_hashmap_insert(p, hx, new_key, val, res, 1); - if (is_non_value(res)) + if (is_non_value(res)) { + p->fvalue = new_key; + p->freason = BADKEY; return res; + } if (p->mbuf) { Uint live = Arg(3); reg[live] = res; erts_garbage_collect(p, 0, reg, live+1); res = reg[live]; + E = p->stop; } - E = p->stop; - new_p += 2; } return res; @@ -6883,10 +6954,13 @@ update_map_exact(Process* p, Eterm* reg, Eterm map, BeamInstr* I) num_old = flatmap_get_size(old_mp); /* - * If the old map is empty, create a new map. + * If the old map is empty, fail. */ if (num_old == 0) { + E = p->stop; + p->freason = BADKEY; + GET_TERM(new_p[0], p->fvalue); return THE_NON_VALUE; } @@ -6956,6 +7030,8 @@ update_map_exact(Process* p, Eterm* reg, Eterm map, BeamInstr* I) * update list did not previously exist. */ ASSERT(hp == p->htop + need); + p->freason = BADKEY; + p->fvalue = new_key; return THE_NON_VALUE; } #undef GET_TERM diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 02689e5b19..0d40201934 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -36,6 +36,7 @@ #include "beam_catches.h" #include "erl_binary.h" #include "erl_zlib.h" +#include "erl_map.h" #ifdef HIPE #include "hipe_bif0.h" @@ -529,6 +530,7 @@ static Eterm functions_in_module(Process* p, Eterm mod); static Eterm attributes_for_module(Process* p, Eterm mod); static Eterm compilation_info_for_module(Process* p, Eterm mod); static Eterm md5_of_module(Process* p, Eterm mod); +static Eterm has_native(Process* p, Eterm mod); static Eterm native_addresses(Process* p, Eterm mod); int patch_funentries(Eterm Patchlist); int patch(Eterm Addresses, Uint fe); @@ -3171,7 +3173,11 @@ gen_increment_from_minus(LoaderState* stp, GenOpArg Reg, GenOpArg Integer, static int negation_is_small(LoaderState* stp, GenOpArg Int) { - return Int.type == TAG_i && IS_SSMALL(-Int.val); + /* Check for the rare case of overflow in BeamInstr (UWord) -> Sint + * Cast to the correct type before using IS_SSMALL (Sint) */ + return Int.type == TAG_i && + !(Int.val & ~((((BeamInstr)1) << ((sizeof(Sint)*8)-1))-1)) && + IS_SSMALL(-((Sint)Int.val)); } @@ -4051,8 +4057,139 @@ tuple_append_put(LoaderState* stp, GenOpArg Arity, GenOpArg Dst, } /* + * Predicate to test whether the given literal is a map. + */ + +static int +literal_is_map(LoaderState* stp, GenOpArg Lit) +{ + Eterm term; + + ASSERT(Lit.type == TAG_q); + term = stp->literals[Lit.val].term; + return is_map(term); +} + +/* + * Predicate to test whether the given literal is an empty map. + */ + +static int +is_empty_map(LoaderState* stp, GenOpArg Lit) +{ + Eterm term; + + if (Lit.type != TAG_q) { + return 0; + } + term = stp->literals[Lit.val].term; + return is_flatmap(term) && flatmap_get_size(flatmap_val(term)) == 0; +} + +/* + * Pseudo predicate map_key_sort that will sort the Rest operand for + * map instructions as a side effect. + */ + +typedef struct SortGenOpArg { + Eterm term; /* Term to use for comparing */ + GenOpArg arg; /* Original data */ +} SortGenOpArg; + +static int +genopargtermcompare(SortGenOpArg* a, SortGenOpArg* b) +{ + return CMP_TERM(a->term, b->term); +} + +static int +map_key_sort(LoaderState* stp, GenOpArg Size, GenOpArg* Rest) +{ + SortGenOpArg* t; + unsigned size = Size.val; + unsigned i; + + if (size == 2) { + return 1; /* Already sorted. */ + } + + + t = (SortGenOpArg *) erts_alloc(ERTS_ALC_T_TMP, size*sizeof(SortGenOpArg)); + + /* + * Copy original data and sort keys to a temporary array. + */ + for (i = 0; i < size; i += 2) { + t[i].arg = Rest[i]; + switch (Rest[i].type) { + case TAG_a: + t[i].term = Rest[i].val; + ASSERT(is_atom(t[i].term)); + break; + case TAG_i: + t[i].term = make_small(Rest[i].val); + break; + case TAG_n: + t[i].term = NIL; + break; + case TAG_q: + t[i].term = stp->literals[Rest[i].val].term; + break; + default: + /* + * Not a literal key. Not allowed. Only a single + * variable key is allowed in each map instruction. + */ + erts_free(ERTS_ALC_T_TMP, (void *) t); + return 0; + } +#ifdef DEBUG + t[i+1].term = THE_NON_VALUE; +#endif + t[i+1].arg = Rest[i+1]; + } + + /* + * Sort the temporary array. + */ + qsort((void *) t, size / 2, 2 * sizeof(SortGenOpArg), + (int (*)(const void *, const void *)) genopargtermcompare); + + /* + * Copy back the sorted, original data. + */ + for (i = 0; i < size; i++) { + Rest[i] = t[i].arg; + } + + erts_free(ERTS_ALC_T_TMP, (void *) t); + return 1; +} + +static int +hash_genop_arg(LoaderState* stp, GenOpArg Key, Uint32* hx) +{ + switch (Key.type) { + case TAG_a: + *hx = hashmap_make_hash(Key.val); + return 1; + case TAG_i: + *hx = hashmap_make_hash(make_small(Key.val)); + return 1; + case TAG_n: + *hx = hashmap_make_hash(NIL); + return 1; + case TAG_q: + *hx = hashmap_make_hash(stp->literals[Key.val].term); + return 1; + default: + return 0; + } +} + +/* * Replace a get_map_elements with one key to an instruction with one - * element + * element. */ static GenOp* @@ -4060,37 +4197,99 @@ gen_get_map_element(LoaderState* stp, GenOpArg Fail, GenOpArg Src, GenOpArg Size, GenOpArg* Rest) { GenOp* op; + GenOpArg Key; + Uint32 hx = 0; ASSERT(Size.type == TAG_u); NEW_GENOP(stp, op); op->next = NULL; - op->op = genop_get_map_element_4; - op->arity = 4; - op->a[0] = Fail; op->a[1] = Src; op->a[2] = Rest[0]; - op->a[3] = Rest[1]; + + Key = Rest[0]; + if (hash_genop_arg(stp, Key, &hx)) { + op->arity = 5; + op->op = genop_i_get_map_element_hash_5; + op->a[3].type = TAG_u; + op->a[3].val = (BeamInstr) hx; + op->a[4] = Rest[1]; + } else { + op->arity = 4; + op->op = genop_i_get_map_element_4; + op->a[3] = Rest[1]; + } return op; } static GenOp* -gen_has_map_field(LoaderState* stp, GenOpArg Fail, GenOpArg Src, - GenOpArg Size, GenOpArg* Rest) +gen_get_map_elements(LoaderState* stp, GenOpArg Fail, GenOpArg Src, + GenOpArg Size, GenOpArg* Rest) { GenOp* op; + Uint32 hx; + Uint i; + GenOpArg* dst; +#ifdef DEBUG + int good_hash; +#endif ASSERT(Size.type == TAG_u); NEW_GENOP(stp, op); + op->op = genop_i_get_map_elements_3; + GENOP_ARITY(op, 3 + 3*(Size.val/2)); op->next = NULL; - op->op = genop_has_map_field_3; - op->arity = 4; + op->a[0] = Fail; + op->a[1] = Src; + op->a[2].type = TAG_u; + op->a[2].val = 3*(Size.val/2); + + dst = op->a+3; + for (i = 0; i < Size.val / 2; i++) { + dst[0] = Rest[2*i]; + dst[1] = Rest[2*i+1]; +#ifdef DEBUG + good_hash = +#endif + hash_genop_arg(stp, dst[0], &hx); +#ifdef DEBUG + ASSERT(good_hash); +#endif + dst[2].type = TAG_u; + dst[2].val = (BeamInstr) hx; + dst += 3; + } + return op; +} + +static GenOp* +gen_has_map_fields(LoaderState* stp, GenOpArg Fail, GenOpArg Src, + GenOpArg Size, GenOpArg* Rest) +{ + GenOp* op; + Uint i; + Uint n; + + ASSERT(Size.type == TAG_u); + n = Size.val; + + NEW_GENOP(stp, op); + GENOP_ARITY(op, 3 + 2*n); + op->next = NULL; + op->op = genop_get_map_elements_3; op->a[0] = Fail; op->a[1] = Src; - op->a[2] = Rest[0]; + op->a[2].type = TAG_u; + op->a[2].val = 2*n; + + for (i = 0; i < n; i++) { + op->a[3+2*i] = Rest[i]; + op->a[3+2*i+1].type = TAG_x; + op->a[3+2*i+1].val = 0; /* x(0); normally not used */ + } return op; } @@ -5210,6 +5409,9 @@ erts_module_info_0(Process* p, Eterm module) list = CONS(hp, tup, list) BUILD_INFO(am_md5); +#ifdef HIPE + BUILD_INFO(am_native); +#endif BUILD_INFO(am_compile); BUILD_INFO(am_attributes); BUILD_INFO(am_exports); @@ -5235,6 +5437,8 @@ erts_module_info_1(Process* p, Eterm module, Eterm what) return compilation_info_for_module(p, module); } else if (what == am_native_addresses) { return native_addresses(p, module); + } else if (what == am_native) { + return has_native(p, module); } return THE_NON_VALUE; } @@ -5295,6 +5499,53 @@ functions_in_module(Process* p, /* Process whose heap to use. */ } /* + * Returns 'true' if mod has any native compiled functions, otherwise 'false' + */ + +static Eterm +has_native(Process* p, Eterm mod) +{ + Eterm result = am_false; +#ifdef HIPE + Module* modp; + + if (is_not_atom(mod)) { + return THE_NON_VALUE; + } + + modp = erts_get_module(mod, erts_active_code_ix()); + if (modp == NULL) { + return THE_NON_VALUE; + } + + if (erts_is_module_native(modp->curr.code)) { + result = am_true; + } +#endif + return result; +} + +int +erts_is_module_native(BeamInstr* code) +{ + Uint i, num_functions; + + /* Check NativeAdress of first real function in module */ + if (code != NULL) { + num_functions = code[MI_NUM_FUNCTIONS]; + for (i=0; i<num_functions; i++) { + BeamInstr* func_info = (BeamInstr *) code[MI_FUNCTIONS+i]; + Eterm name = (Eterm) func_info[3]; + if (is_atom(name)) { + return func_info[1] != 0; + } + else ASSERT(is_nil(name)); /* ignore BIF stubs */ + } + } + return 0; +} + +/* * Builds a list of all functions including native addresses. * [{Name,Arity,NativeAddress},...] * @@ -5497,7 +5748,11 @@ md5_of_module(Process* p, /* Process whose heap to use. */ return THE_NON_VALUE; } code = modp->curr.code; - res = new_binary(p, (byte *) code[MI_MD5_PTR], MD5_SIZE); + if (code[MI_MD5_PTR] != 0) { + res = new_binary(p, (byte *) code[MI_MD5_PTR], MD5_SIZE); + } else { + res = am_undefined; + } return res; } @@ -5970,6 +6225,7 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) LoaderState* stp; BeamInstr Funcs; BeamInstr Patchlist; + Eterm MD5Bin; Eterm* tp; BeamInstr* code = NULL; BeamInstr* ptrs; @@ -5998,12 +6254,15 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) goto error; } tp = tuple_val(Info); - if (tp[0] != make_arityval(2)) { + if (tp[0] != make_arityval(3)) { goto error; } Funcs = tp[1]; - Patchlist = tp[2]; - + Patchlist = tp[2]; + MD5Bin = tp[3]; + if (is_not_binary(MD5Bin) || (binary_size(MD5Bin) != MD5_SIZE)) { + goto error; + } if ((n = erts_list_length(Funcs)) < 0) { goto error; } @@ -6053,6 +6312,7 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) code_size = ((WORDS_PER_FUNCTION+1)*n + MI_FUNCTIONS + 2) * sizeof(BeamInstr); code_size += stp->chunks[ATTR_CHUNK].size; code_size += stp->chunks[COMPILE_CHUNK].size; + code_size += MD5_SIZE; code = erts_alloc_fnf(ERTS_ALC_T_CODE, code_size); if (!code) { goto error; @@ -6159,6 +6419,15 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) if (info == NULL) { goto error; } + { + byte *tmp = NULL; + byte *md5 = NULL; + if ((md5 = erts_get_aligned_binary_bytes(MD5Bin, &tmp)) != NULL) { + sys_memcpy(info, md5, MD5_SIZE); + code[MI_MD5_PTR] = (BeamInstr) info; + } + erts_free_aligned_binary_bytes(tmp); + } /* * Insert the module in the module table. diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h index 0e3ca0bdb0..46b0c60ab0 100644 --- a/erts/emulator/beam/beam_load.h +++ b/erts/emulator/beam/beam_load.h @@ -23,10 +23,10 @@ #include "beam_opcodes.h" #include "erl_process.h" +int erts_is_module_native(BeamInstr* code); Eterm beam_make_current_old(Process *c_p, ErtsProcLocks c_p_locks, Eterm module); - typedef struct gen_op_entry { char* name; int arity; diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 0c9a1d079c..2b782f4484 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -611,11 +611,7 @@ erts_queue_monitor_message(Process *p, ref_copy = copy_struct(ref, ref_size, &hp, ohp); tup = TUPLE5(hp, am_DOWN, ref_copy, type, item_copy, reason_copy); - erts_queue_message(p, p_locksp, bp, tup, NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(p, p_locksp, bp, tup, NIL); } static BIF_RETTYPE @@ -2911,7 +2907,7 @@ static int do_list_to_integer(Process *p, Eterm orig_list, Uint ui = 0; int skip = 0; int neg = 0; - int n = 0; + Sint n = 0; int m; int lg2; Eterm res; @@ -2991,7 +2987,9 @@ static int do_list_to_integer(Process *p, Eterm orig_list, else i = (Sint)ui; res = make_small(i); } else { - lg2 = (n+1)*230/69+1; + /* Convert from log10 to log2 by multiplying with 1/log10(2)=3.3219 + which we round up to (3 + 1/3) */ + lg2 = (n+1)*3 + (n+1)/3 + 1; m = (lg2+D_EXP-1)/D_EXP; /* number of digits */ m = BIG_NEED_SIZE(m); /* number of words + thing */ diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index bfecac1612..142fcb3c00 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -388,11 +388,7 @@ static void doit_node_link_net_exits(ErtsLink *lnk, void *vnecp) Eterm tup; Eterm *hp = erts_alloc_message_heap(3,&bp,&ohp,rp,&rp_locks); tup = TUPLE2(hp, am_nodedown, name); - erts_queue_message(rp, &rp_locks, bp, tup, NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(rp, &rp_locks, bp, tup, NIL); } erts_smp_proc_unlock(rp, rp_locks); } @@ -712,7 +708,7 @@ void erts_dsend_context_dtor(Binary* ctx_bin) ErtsSendContext* ctx = ERTS_MAGIC_BIN_DATA(ctx_bin); switch (ctx->dss.phase) { case ERTS_DSIG_SEND_PHASE_MSG_SIZE: - DESTROY_SAVED_ESTACK(&ctx->dss.u.sc.estack); + DESTROY_SAVED_WSTACK(&ctx->dss.u.sc.wstack); break; case ERTS_DSIG_SEND_PHASE_MSG_ENCODE: DESTROY_SAVED_WSTACK(&ctx->dss.u.ec.wstack); @@ -1800,7 +1796,7 @@ erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx) erts_encode_dist_ext_size(ctx->ctl, ctx->flags, ctx->acmp, &ctx->data_size); if (is_value(ctx->msg)) { - ctx->u.sc.estack.start = NULL; + ctx->u.sc.wstack.wstart = NULL; ctx->u.sc.flags = ctx->flags; ctx->u.sc.level = 0; ctx->phase = ERTS_DSIG_SEND_PHASE_MSG_SIZE; @@ -3325,11 +3321,7 @@ send_nodes_mon_msg(Process *rp, } ASSERT(hend == hp); - erts_queue_message(rp, rp_locksp, bp, msg, NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(rp, rp_locksp, bp, msg, NIL); } static void diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h index 2a2ba0c83f..cd2cc0ef4a 100644 --- a/erts/emulator/beam/dist.h +++ b/erts/emulator/beam/dist.h @@ -282,7 +282,7 @@ typedef struct TTBSizeContext_ { int level; Uint result; Eterm obj; - ErtsEStack estack; + ErtsWStack wstack; } TTBSizeContext; typedef struct TTBEncodeContext_ { diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index 71902c2f9f..b832626e7b 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -3189,11 +3189,7 @@ reply_alloc_info(void *vair) HRelease(rp, hp_end, hp); } - erts_queue_message(rp, &rp_locks, bp, msg, NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(rp, &rp_locks, bp, msg, NIL); if (air->req_sched == sched_id) rp_locks &= ~ERTS_PROC_LOCK_MAIN; diff --git a/erts/emulator/beam/erl_arith.c b/erts/emulator/beam/erl_arith.c index 5150a8a507..47d516534f 100644 --- a/erts/emulator/beam/erl_arith.c +++ b/erts/emulator/beam/erl_arith.c @@ -2048,3 +2048,8 @@ Eterm erts_gc_bnot(Process* p, Eterm* reg, Uint live) } return result; } + +/* Needed to remove compiler optimization */ +double erts_get_positive_zero_float() { + return 0.0f; +} diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c index fc4f819f56..7b35edc9c4 100644 --- a/erts/emulator/beam/erl_bif_ddll.c +++ b/erts/emulator/beam/erl_bif_ddll.c @@ -1731,11 +1731,7 @@ static void notify_proc(Process *proc, Eterm ref, Eterm driver_name, Eterm type, hp += REF_THING_SIZE; mess = TUPLE5(hp,type,r,am_driver,driver_name,tag); } - erts_queue_message(proc, &rp_locks, bp, mess, am_undefined -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(proc, &rp_locks, bp, mess, am_undefined); erts_smp_proc_unlock(proc, rp_locks); ERTS_SMP_CHK_NO_PROC_LOCKS; } diff --git a/erts/emulator/beam/erl_bif_guard.c b/erts/emulator/beam/erl_bif_guard.c index e7d84ebda1..6226ec2d04 100644 --- a/erts/emulator/beam/erl_bif_guard.c +++ b/erts/emulator/beam/erl_bif_guard.c @@ -459,25 +459,25 @@ Eterm erts_gc_byte_size_1(Process* p, Eterm* reg, Uint live) Eterm erts_gc_map_size_1(Process* p, Eterm* reg, Uint live) { Eterm arg = reg[live]; - Eterm* hp; - Uint size; if (is_flatmap(arg)) { flatmap_t *mp = (flatmap_t*)flatmap_val(arg); - size = flatmap_get_size(mp); + return make_small(flatmap_get_size(mp)); } else if (is_hashmap(arg)) { + Eterm* hp; + Uint size; size = hashmap_size(arg); - } else { - BIF_ERROR(p, BADARG); - } - if (IS_USMALL(0, size)) { - return make_small(size); - } - if (ERTS_NEED_GC(p, BIG_UINT_HEAP_SIZE)) { - erts_garbage_collect(p, BIG_UINT_HEAP_SIZE, reg, live); - } - hp = p->htop; - p->htop += BIG_UINT_HEAP_SIZE; - return uint_to_big(size, hp); + if (IS_USMALL(0, size)) { + return make_small(size); + } + if (ERTS_NEED_GC(p, BIG_UINT_HEAP_SIZE)) { + erts_garbage_collect(p, BIG_UINT_HEAP_SIZE, reg, live); + } + hp = p->htop; + p->htop += BIG_UINT_HEAP_SIZE; + return uint_to_big(size, hp); + } + p->fvalue = arg; + BIF_ERROR(p, BADMAP); } Eterm erts_gc_abs_1(Process* p, Eterm* reg, Uint live) diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index e30687fca5..f74aea80a7 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -3749,6 +3749,20 @@ BIF_RETTYPE erts_internal_is_system_process_1(BIF_ALIST_1) static erts_smp_atomic_t hipe_test_reschedule_flag; +#if defined(VALGRIND) && defined(__GNUC__) +/* Force noinline for valgrind suppression */ +static void broken_halt_test(Eterm bif_arg_2) __attribute__((noinline)); +#endif + +static void broken_halt_test(Eterm bif_arg_2) +{ + /* Ugly ugly code used by bif_SUITE:erlang_halt/1 */ +#if defined(ERTS_HAVE_TRY_CATCH) + erts_get_scheduler_data()->run_queue = NULL; +#endif + erl_exit(ERTS_DUMP_EXIT, "%T", bif_arg_2); +} + BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) { @@ -4042,11 +4056,7 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) } } else if (ERTS_IS_ATOM_STR("broken_halt", BIF_ARG_1)) { - /* Ugly ugly code used by bif_SUITE:erlang_halt/1 */ -#if defined(ERTS_HAVE_TRY_CATCH) - erts_get_scheduler_data()->run_queue = NULL; -#endif - erl_exit(ERTS_DUMP_EXIT, "%T", BIF_ARG_2); + broken_halt_test(BIF_ARG_2); } else if (ERTS_IS_ATOM_STR("unique_monotonic_integer_state", BIF_ARG_1)) { int res = erts_debug_set_unique_monotonic_integer_state(BIF_ARG_2); diff --git a/erts/emulator/beam/erl_bif_lists.c b/erts/emulator/beam/erl_bif_lists.c index 820ed2385d..e006d57124 100644 --- a/erts/emulator/beam/erl_bif_lists.c +++ b/erts/emulator/beam/erl_bif_lists.c @@ -390,7 +390,7 @@ keyfind(int Bif, Process* p, Eterm Key, Eterm Pos, Eterm List) Eterm *tuple_ptr = tuple_val(term); if (pos <= arityval(*tuple_ptr)) { Eterm element = tuple_ptr[pos]; - if (CMP(Key, element) == 0) { + if (CMP_EQ(Key, element)) { return term; } } diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c index 5cc0a23dc9..b8ae93fa58 100644 --- a/erts/emulator/beam/erl_bits.c +++ b/erts/emulator/beam/erl_bits.c @@ -165,6 +165,26 @@ erts_bs_start_match_2(Process *p, Eterm Binary, Uint Max) return make_matchstate(ms); } +#ifdef DEBUG +# define CHECK_MATCH_BUFFER(MB) check_match_buffer(MB) + +static void check_match_buffer(ErlBinMatchBuffer* mb) +{ + Eterm realbin; + Uint byteoffs; + byte* bytes, bitoffs, bitsz; + ProcBin* pb; + ERTS_GET_REAL_BIN(mb->orig, realbin, byteoffs, bitoffs, bitsz); + bytes = binary_bytes(realbin) + byteoffs; + ERTS_ASSERT(mb->base >= bytes && mb->base <= (bytes + binary_size(mb->orig))); + pb = (ProcBin *) boxed_val(realbin); + if (pb->thing_word == HEADER_PROC_BIN) + ERTS_ASSERT(pb->flags == 0); +} +#else +# define CHECK_MATCH_BUFFER(MB) +#endif + Eterm erts_bs_get_integer_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuffer* mb) { @@ -185,6 +205,7 @@ erts_bs_get_integer_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuff return SMALL_ZERO; } + CHECK_MATCH_BUFFER(mb); if (mb->size - mb->offset < num_bits) { /* Asked for too many bits. */ return THE_NON_VALUE; } @@ -425,6 +446,7 @@ erts_bs_get_binary_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuffe { ErlSubBin* sb; + CHECK_MATCH_BUFFER(mb); if (mb->size - mb->offset < num_bits) { /* Asked for too many bits. */ return THE_NON_VALUE; } @@ -456,6 +478,7 @@ erts_bs_get_float_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuffer byte* fptr; FloatDef f; + CHECK_MATCH_BUFFER(mb); if (num_bits == 0) { f.fd = 0.0; hp = HeapOnlyAlloc(p, FLOAT_SIZE_OBJECT); @@ -509,6 +532,8 @@ erts_bs_get_binary_all_2(Process *p, ErlBinMatchBuffer* mb) { ErlSubBin* sb; Uint size; + + CHECK_MATCH_BUFFER(mb); size = mb->size-mb->offset; sb = (ErlSubBin *) HeapOnlyAlloc(p, ERL_SUB_BIN_SIZE); sb->thing_word = HEADER_SUB_BIN; @@ -1595,6 +1620,7 @@ erts_bs_get_unaligned_uint32(ErlBinMatchBuffer* mb) byte* LSB; byte* MSB; + CHECK_MATCH_BUFFER(mb); ASSERT((mb->offset & 7) != 0); ASSERT(mb->size - mb->offset >= 32); @@ -1654,6 +1680,8 @@ erts_bs_get_utf8(ErlBinMatchBuffer* mb) 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,9,9,9,9,9,9,9,9 }; + CHECK_MATCH_BUFFER(mb); + if ((remaining_bits = mb->size - mb->offset) < 8) { return THE_NON_VALUE; } @@ -1738,6 +1766,7 @@ erts_bs_get_utf16(ErlBinMatchBuffer* mb, Uint flags) return THE_NON_VALUE; } + CHECK_MATCH_BUFFER(mb); /* * Set up the pointer to the source bytes. */ diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 045c8ae135..383ee7c430 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -2471,10 +2471,10 @@ static int alloc_seg(DbTableHash *tb) */ static int free_seg(DbTableHash *tb, int free_records) { - int seg_ix = (tb->nslots >> SEGSZ_EXP) - 1; + const int seg_ix = (tb->nslots >> SEGSZ_EXP) - 1; + struct segment** const segtab = SEGTAB(tb); + struct ext_segment* const top = (struct ext_segment*) segtab[seg_ix]; int bytes; - struct segment** segtab = SEGTAB(tb); - struct ext_segment* top = (struct ext_segment*) segtab[seg_ix]; int nrecords = 0; ASSERT(top != NULL); @@ -2537,7 +2537,7 @@ static int free_seg(DbTableHash *tb, int free_records) (void*)top, bytes); #ifdef DEBUG if (seg_ix > 0) { - if (seg_ix < tb->nsegs) SEGTAB(tb)[seg_ix] = NULL; + segtab[seg_ix] = NULL; } else { SET_SEGTAB(tb, NULL); } diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c index 577da35b75..d90af46659 100644 --- a/erts/emulator/beam/erl_db_tree.c +++ b/erts/emulator/beam/erl_db_tree.c @@ -1116,7 +1116,7 @@ static int db_select_tree(Process *p, DbTable *tbl, sc.all_objects = mpi.all_objects; if (!mpi.got_partial && mpi.some_limitation && - CMP(mpi.least,mpi.most) == 0) { + CMP_EQ(mpi.least,mpi.most)) { doit_select(tb,mpi.save_term,&sc,0 /* direction doesn't matter */); RET_TO_BIF(sc.accum,DB_ERROR_NONE); } @@ -1324,7 +1324,7 @@ static int db_select_count_tree(Process *p, DbTable *tbl, sc.all_objects = mpi.all_objects; if (!mpi.got_partial && mpi.some_limitation && - CMP(mpi.least,mpi.most) == 0) { + CMP_EQ(mpi.least,mpi.most)) { doit_select_count(tb,mpi.save_term,&sc,0 /* dummy */); RET_TO_BIF(erts_make_integer(sc.got,p),DB_ERROR_NONE); } @@ -1429,7 +1429,7 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl, sc.all_objects = mpi.all_objects; if (!mpi.got_partial && mpi.some_limitation && - CMP(mpi.least,mpi.most) == 0) { + CMP_EQ(mpi.least,mpi.most)) { doit_select(tb,mpi.save_term,&sc, 0 /* direction doesn't matter */); if (sc.accum != NIL) { hp=HAlloc(p, 3); @@ -1673,7 +1673,7 @@ static int db_select_delete_tree(Process *p, DbTable *tbl, sc.mp = mpi.mp; if (!mpi.got_partial && mpi.some_limitation && - CMP(mpi.least,mpi.most) == 0) { + CMP_EQ(mpi.least,mpi.most)) { doit_select_delete(tb,mpi.save_term,&sc, 0 /* direction doesn't matter */); RET_TO_BIF(erts_make_integer(sc.accum,p),DB_ERROR_NONE); diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index 0bf562d937..0fb1c397c9 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -2153,8 +2153,8 @@ restart: break; case matchMkFlatMap: n = *pc++; - ehp = HAllocX(build_proc, 1 + MAP_HEADER_FLATMAP_SZ + n, HEAP_XTRA); - t = *ehp++ = *--esp; + ehp = HAllocX(build_proc, MAP_HEADER_FLATMAP_SZ + n, HEAP_XTRA); + t = *--esp; { flatmap_t *m = (flatmap_t *)ehp; m->thing_word = MAP_HEADER_FLATMAP; diff --git a/erts/emulator/beam/erl_drv_thread.c b/erts/emulator/beam/erl_drv_thread.c index 31b05d22af..240faa823d 100644 --- a/erts/emulator/beam/erl_drv_thread.c +++ b/erts/emulator/beam/erl_drv_thread.c @@ -604,10 +604,12 @@ erl_drv_thread_create(char *name, ethr_thr_opts ethr_opts = ETHR_THR_OPTS_DEFAULT_INITER; ethr_thr_opts *use_opts; - if (!opts) + if (!opts && !name) use_opts = NULL; else { - ethr_opts.suggested_stack_size = opts->suggested_stack_size; + if(opts) + ethr_opts.suggested_stack_size = opts->suggested_stack_size; + ethr_opts.name = name; use_opts = ðr_opts; } diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 6a986b31ed..1785fc27be 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -2646,11 +2646,7 @@ reply_gc_info(void *vgcirp) hpp = &hp; } - erts_queue_message(rp, &rp_locks, bp, msg, NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(rp, &rp_locks, bp, msg, NIL); if (gcirp->req_sched == esdp->no) rp_locks &= ~ERTS_PROC_LOCK_MAIN; diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c index 2245799819..cf92ded415 100644 --- a/erts/emulator/beam/erl_hl_timer.c +++ b/erts/emulator/beam/erl_hl_timer.c @@ -1149,15 +1149,8 @@ hlt_bif_timer_timeout(ErtsHLTimer *tmr, Uint32 roflgs) * the middle of tree destruction). */ if (!ERTS_PROC_IS_EXITING(proc)) { - erts_queue_message(proc, - &proc_locks, - tmr->btm.bp, - tmr->btm.message, - NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(proc, &proc_locks, tmr->btm.bp, + tmr->btm.message, NIL); erts_smp_proc_unlock(proc, ERTS_PROC_LOCKS_MSG_SEND); proc_locks &= ~ERTS_PROC_LOCKS_MSG_SEND; tmr->btm.bp = NULL; @@ -1886,12 +1879,7 @@ access_sched_local_btm(Process *c_p, Eterm pid, ERTS_HLT_ASSERT(hp + 4 == hp_end); } - erts_queue_message(proc, &proc_locks, bp, - msg, NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(proc, &proc_locks, bp, msg, NIL); if (c_p) proc_locks &= ~ERTS_PROC_LOCK_MAIN; @@ -2016,12 +2004,7 @@ try_access_sched_remote_btm(ErtsSchedulerData *esdp, msg = TUPLE3(hp, tag, tref, res); - erts_queue_message(c_p, &proc_locks, bp, - msg, NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(c_p, &proc_locks, bp, msg, NIL); proc_locks &= ~ERTS_PROC_LOCK_MAIN; if (proc_locks) diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index 9769f36e42..988ff0e2b5 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -120,6 +120,8 @@ const int etp_big_endian = 1; #else const int etp_big_endian = 0; #endif +const Eterm etp_the_non_value = THE_NON_VALUE; + /* * Note about VxWorks: All variables must be initialized by executable code, * not by an initializer. Otherwise a new instance of the emulator will @@ -1956,11 +1958,6 @@ erl_start(int argc, char **argv) goto time_correction_false; else if (sys_strcmp(argv[i]+2, "true") == 0) goto time_correction_true; -#ifdef ERTS_OPCODE_COUNTER_SUPPORT - else if (argv[i][2] == 'i') { /* -ci: undcoumented option*/ - count_instructions = 1; - } -#endif else if (argv[i][2] == '\0') { if (i + 1 >= argc) goto time_correction_false; diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c index ab40f47919..bb2a2bcdf9 100644 --- a/erts/emulator/beam/erl_map.c +++ b/erts/emulator/beam/erl_map.c @@ -89,7 +89,7 @@ static Eterm flatmap_from_validated_list(Process *p, Eterm list, Uint size); static Eterm hashmap_from_validated_list(Process *p, Eterm list, Uint size); static Eterm hashmap_from_unsorted_array(ErtsHeapFactory*, hxnode_t *hxns, Uint n, int reject_dupkeys); static Eterm hashmap_from_sorted_unique_array(ErtsHeapFactory*, hxnode_t *hxns, Uint n, int is_root); -static Eterm hashmap_from_chunked_array(ErtsHeapFactory*, hxnode_t *hxns, Uint n, int is_root); +static Eterm hashmap_from_chunked_array(ErtsHeapFactory*, hxnode_t *hxns, Uint n, Uint size, int is_root); static Eterm hashmap_info(Process *p, Eterm node); static Eterm hashmap_bld_tuple_uint(Uint **hpp, Uint *szp, Uint n, Uint nums[]); static int hxnodecmp(hxnode_t* a, hxnode_t* b); @@ -102,14 +102,8 @@ static int hxnodecmpkey(hxnode_t* a, hxnode_t* b); BIF_RETTYPE map_size_1(BIF_ALIST_1) { if (is_flatmap(BIF_ARG_1)) { - Eterm *hp; - Uint hsz = 0; flatmap_t *mp = (flatmap_t*)flatmap_val(BIF_ARG_1); - Uint n = flatmap_get_size(mp); - - erts_bld_uint(NULL, &hsz, n); - hp = HAlloc(BIF_P, hsz); - BIF_RET(erts_bld_uint(&hp, NULL, n)); + BIF_RET(make_small(flatmap_get_size(mp))); } else if (is_hashmap(BIF_ARG_1)) { Eterm *head, *hp, res; Uint size, hsz=0; @@ -122,7 +116,8 @@ BIF_RETTYPE map_size_1(BIF_ALIST_1) { BIF_RET(res); } - BIF_ERROR(BIF_P, BADARG); + BIF_P->fvalue = BIF_ARG_1; + BIF_ERROR(BIF_P, BADMAP); } /* maps:to_list/1 */ @@ -150,7 +145,8 @@ BIF_RETTYPE maps_to_list_1(BIF_ALIST_1) { return hashmap_to_list(BIF_P, BIF_ARG_1); } - BIF_ERROR(BIF_P, BADARG); + BIF_P->fvalue = BIF_ARG_1; + BIF_ERROR(BIF_P, BADMAP); } /* maps:find/2 @@ -217,34 +213,29 @@ BIF_RETTYPE maps_find_2(BIF_ALIST_2) { } BIF_RET(am_error); } - BIF_ERROR(BIF_P, BADARG); + BIF_P->fvalue = BIF_ARG_2; + BIF_ERROR(BIF_P, BADMAP); } /* maps:get/2 * return value if key *matches* a key in the map - * exception bad_key if none matches + * exception badkey if none matches */ BIF_RETTYPE maps_get_2(BIF_ALIST_2) { if (is_map(BIF_ARG_2)) { - Eterm *hp; - Eterm error; const Eterm *value; - char *s_error; value = erts_maps_get(BIF_ARG_1, BIF_ARG_2); if (value) { BIF_RET(*value); } - s_error = "bad_key"; - error = am_atom_put(s_error, sys_strlen(s_error)); - - hp = HAlloc(BIF_P, 3); - BIF_P->fvalue = TUPLE2(hp, error, BIF_ARG_1); - BIF_ERROR(BIF_P, EXC_ERROR_2); + BIF_P->fvalue = BIF_ARG_1; + BIF_ERROR(BIF_P, BADKEY); } - BIF_ERROR(BIF_P, BADARG); + BIF_P->fvalue = BIF_ARG_2; + BIF_ERROR(BIF_P, BADMAP); } /* maps:from_list/1 @@ -392,12 +383,11 @@ static Eterm hashmap_from_validated_list(Process *p, Eterm list, Uint size) { Eterm item = list; Eterm *hp; Eterm *kv, res; - Eterm tmp[2]; Uint32 sw, hx; Uint ix = 0; hxnode_t *hxns; ErtsHeapFactory factory; - + DeclareTmpHeap(tmp,2,p); ASSERT(size > 0); hp = HAlloc(p, (2 * size)); @@ -405,6 +395,7 @@ static Eterm hashmap_from_validated_list(Process *p, Eterm list, Uint size) { /* create tmp hx values and leaf ptrs */ hxns = (hxnode_t *)erts_alloc(ERTS_ALC_T_TMP, size * sizeof(hxnode_t)); + UseTmpHeap(2,p); while(is_list(item)) { res = CAR(list_val(item)); kv = tuple_val(res); @@ -417,6 +408,7 @@ static Eterm hashmap_from_validated_list(Process *p, Eterm list, Uint size) { ix++; item = CDR(list_val(item)); } + UnUseTmpHeap(2,p); factory.p = p; res = hashmap_from_unsorted_array(&factory, hxns, size, 0); @@ -567,14 +559,14 @@ static Eterm hashmap_from_unsorted_array(ErtsHeapFactory* factory, while(ix < jx) { lx = ix; - while(ix < jx && EQ(CAR(list_val(hxns[ix].val)), CAR(list_val(hxns[lx].val)))) { + while(++ix < jx && EQ(CAR(list_val(hxns[ix].val)), + CAR(list_val(hxns[lx].val)))) { if (reject_dupkeys) return THE_NON_VALUE; if (hxns[ix].i > hxns[lx].i) { lx = ix; } - ix++; } hxns[cx].hx = hxns[lx].hx; hxns[cx].val = hxns[lx].val; @@ -625,9 +617,9 @@ static Eterm hashmap_from_sorted_unique_array(ErtsHeapFactory* factory, Uint i,ix,jx,elems; Uint32 sw, hx; Eterm val; - Eterm th[2]; hxnode_t *tmp; - + DeclareTmpHeapNoproc(th,2); + UseTmpHeapNoproc(2); ASSERT(lvl < 32); ix = 0; elems = 1; @@ -661,23 +653,52 @@ static Eterm hashmap_from_sorted_unique_array(ErtsHeapFactory* factory, ix++; } - res = hashmap_from_chunked_array(factory, hxns, elems, !lvl); + res = hashmap_from_chunked_array(factory, hxns, elems, n, !lvl); ERTS_FACTORY_HOLE_CHECK(factory); + UnUseTmpHeapNoproc(2); return res; } #define HALLOC_EXTRA 200 -static Eterm hashmap_from_chunked_array(ErtsHeapFactory *factory, - hxnode_t *hxns, Uint n, int is_root) { +static Eterm hashmap_from_chunked_array(ErtsHeapFactory *factory, hxnode_t *hxns, Uint n, + Uint size, int is_root) { Uint ix, d, dn, dc, slot, elems; Uint32 v, vp, vn, hdr; Uint bp, sz; DECLARE_ESTACK(stack); Eterm res = NIL, *hp = NULL, *nhp; - ASSERT(n > 1); + + /* if we get here with only one element then + * we have eight levels of collisions + */ + + if (n == 1) { + res = hxns[0].val; + v = hxns[0].hx; + for (d = 7; d > 0; d--) { + slot = maskval(v,d); + hp = erts_produce_heap(factory, HAMT_NODE_BITMAP_SZ(1), HALLOC_EXTRA); + hp[0] = MAP_HEADER_HAMT_NODE_BITMAP(1 << slot); + hp[1] = res; + res = make_hashmap(hp); + } + + slot = maskval(v,0); + hp = erts_produce_heap(factory, (is_root ? 3 : 2), 0); + + if (is_root) { + hp[0] = MAP_HEADER_HAMT_HEAD_BITMAP(1 << slot); + hp[1] = size; + hp[2] = res; + } else { + hp[0] = MAP_HEADER_HAMT_NODE_BITMAP(1 << slot); + hp[1] = res; + } + return make_hashmap(hp); + } /* push initial nodes on the stack, * this is the starting depth */ @@ -840,7 +861,7 @@ static Eterm hashmap_from_chunked_array(ErtsHeapFactory *factory, if (is_root) { *hp++ = (hdr == 0xffff) ? MAP_HEADER_HAMT_HEAD_ARRAY : MAP_HEADER_HAMT_HEAD_BITMAP(hdr); - *hp++ = n; + *hp++ = size; } else { *hp++ = MAP_HEADER_HAMT_NODE_BITMAP(hdr); } @@ -858,7 +879,12 @@ static Eterm hashmap_from_chunked_array(ErtsHeapFactory *factory, #undef HALLOC_EXTRA static int hxnodecmpkey(hxnode_t *a, hxnode_t *b) { - return CMP_TERM(CAR(list_val(a->val)), CAR(list_val(b->val))); + Sint c = CMP_TERM(CAR(list_val(a->val)), CAR(list_val(b->val))); +#if ERTS_SIZEOF_ETERM <= SIZEOF_INT + return c; +#else + return c > 0 ? 1 : (c < 0 ? -1 : 0); +#endif } static int hxnodecmp(hxnode_t *a, hxnode_t *b) { @@ -876,7 +902,8 @@ BIF_RETTYPE maps_is_key_2(BIF_ALIST_2) { if (is_map(BIF_ARG_2)) { BIF_RET(erts_maps_get(BIF_ARG_1, BIF_ARG_2) ? am_true : am_false); } - BIF_ERROR(BIF_P, BADARG); + BIF_P->fvalue = BIF_ARG_2; + BIF_ERROR(BIF_P, BADMAP); } /* maps:keys/1 */ @@ -904,7 +931,8 @@ BIF_RETTYPE maps_keys_1(BIF_ALIST_1) { } else if (is_hashmap(BIF_ARG_1)) { BIF_RET(hashmap_keys(BIF_P, BIF_ARG_1)); } - BIF_ERROR(BIF_P, BADARG); + BIF_P->fvalue = BIF_ARG_1; + BIF_ERROR(BIF_P, BADMAP); } /* maps:merge/2 */ @@ -916,6 +944,7 @@ BIF_RETTYPE maps_merge_2(BIF_ALIST_2) { /* Will always become a tree */ BIF_RET(map_merge_mixed(BIF_P, BIF_ARG_1, BIF_ARG_2, 0)); } + BIF_P->fvalue = BIF_ARG_2; } else if (is_hashmap(BIF_ARG_1)) { if (is_hashmap(BIF_ARG_2)) { BIF_RET(hashmap_merge(BIF_P, BIF_ARG_1, BIF_ARG_2)); @@ -923,8 +952,11 @@ BIF_RETTYPE maps_merge_2(BIF_ALIST_2) { /* Will always become a tree */ BIF_RET(map_merge_mixed(BIF_P, BIF_ARG_2, BIF_ARG_1, 1)); } + BIF_P->fvalue = BIF_ARG_2; + } else { + BIF_P->fvalue = BIF_ARG_1; } - BIF_ERROR(BIF_P, BADARG); + BIF_ERROR(BIF_P, BADMAP); } static Eterm flatmap_merge(Process *p, Eterm nodeA, Eterm nodeB) { @@ -1099,12 +1131,13 @@ static Eterm hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB) { struct HashmapMergePStackType* sp = PSTACK_PUSH(s); Eterm *hp, *nhp; Eterm hdrA, hdrB; - Eterm th[2]; Uint32 ahx, bhx; Uint size; /* total key-value counter */ int keepA = 0; - unsigned lvl = 0; + unsigned int lvl = 0; + DeclareTmpHeap(th,2,p); Eterm res = THE_NON_VALUE; + UseTmpHeap(2,p); /* * Strategy: Do depth-first traversal of both trees (at the same time) @@ -1297,6 +1330,7 @@ recurse: res = make_boxed(nhp); } PSTACK_DESTROY(s); + UnUseTmpHeap(2,p); return res; } @@ -1315,8 +1349,9 @@ static int hash_cmp(Uint32 ha, Uint32 hb) int hashmap_key_hash_cmp(Eterm* ap, Eterm* bp) { - Eterm th[2]; - unsigned lvl = 0; + unsigned int lvl = 0; + DeclareTmpHeapNoproc(th,2); + UseTmpHeapNoproc(2); if (ap && bp) { ASSERT(CMP_TERM(CAR(ap), CAR(bp)) != 0); @@ -1324,11 +1359,14 @@ int hashmap_key_hash_cmp(Eterm* ap, Eterm* bp) Uint32 ha = hashmap_restore_hash(th, lvl, CAR(ap)); Uint32 hb = hashmap_restore_hash(th, lvl, CAR(bp)); int cmp = hash_cmp(ha, hb); - if (cmp) + if (cmp) { + UnUseTmpHeapNoproc(2); return cmp; + } lvl += 8; } } + UnUseTmpHeapNoproc(2); return ap ? -1 : 1; } @@ -1357,7 +1395,8 @@ BIF_RETTYPE maps_put_3(BIF_ALIST_3) { if (is_map(BIF_ARG_3)) { BIF_RET(erts_maps_put(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3)); } - BIF_ERROR(BIF_P, BADARG); + BIF_P->fvalue = BIF_ARG_3; + BIF_ERROR(BIF_P, BADMAP); } /* maps:remove/3 */ @@ -1451,7 +1490,8 @@ BIF_RETTYPE maps_remove_2(BIF_ALIST_2) { BIF_RET(res); } } - BIF_ERROR(BIF_P, BADARG); + BIF_P->fvalue = BIF_ARG_2; + BIF_ERROR(BIF_P, BADMAP); } int erts_maps_update(Process *p, Eterm key, Eterm value, Eterm map, Eterm *res) { @@ -1647,13 +1687,17 @@ Eterm erts_maps_put(Process *p, Eterm key, Eterm value, Eterm map) { /* maps:update/3 */ BIF_RETTYPE maps_update_3(BIF_ALIST_3) { - if (is_map(BIF_ARG_3)) { + if (is_not_map(BIF_ARG_3)) { + BIF_P->fvalue = BIF_ARG_3; + BIF_ERROR(BIF_P, BADMAP); + } else { Eterm res; if (erts_maps_update(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, &res)) { BIF_RET(res); } + BIF_P->fvalue = BIF_ARG_1; + BIF_ERROR(BIF_P, BADKEY); } - BIF_ERROR(BIF_P, BADARG); } @@ -1682,7 +1726,8 @@ BIF_RETTYPE maps_values_1(BIF_ALIST_1) { } else if (is_hashmap(BIF_ARG_1)) { BIF_RET(hashmap_values(BIF_P, BIF_ARG_1)); } - BIF_ERROR(BIF_P, BADARG); + BIF_P->fvalue = BIF_ARG_1; + BIF_ERROR(BIF_P, BADMAP); } static Eterm hashmap_to_list(Process *p, Eterm node) { @@ -1901,13 +1946,14 @@ int erts_hashmap_insert_down(Uint32 hx, Eterm key, Eterm node, Uint *sz, Uint *update_size, ErtsEStack *sp, int is_update) { Eterm *ptr; Eterm hdr, ckey; - Eterm th[2]; Uint32 ix, cix, bp, hval, chx; Uint slot, lvl = 0, clvl; Uint size = 0, n = 0; + DeclareTmpHeapNoproc(th,2); *update_size = 1; + UseTmpHeapNoproc(2); for (;;) { switch(primary_tag(node)) { case TAG_PRIMARY_LIST: /* LEAF NODE [K|V] */ @@ -1918,6 +1964,7 @@ int erts_hashmap_insert_down(Uint32 hx, Eterm key, Eterm node, Uint *sz, goto unroll; } if (is_update) { + UnUseTmpHeapNoproc(2); return 0; } goto insert_subnodes; @@ -1937,26 +1984,32 @@ int erts_hashmap_insert_down(Uint32 hx, Eterm key, Eterm node, Uint *sz, case HAMT_SUBTAG_NODE_BITMAP: hval = MAP_HEADER_VAL(hdr); ix = hashmap_index(hx); - bp = 1 << ix; - slot = hashmap_bitcount(hval & (bp - 1)); - n = hashmap_bitcount(hval); + bp = 1 << ix; + if (hval == 0xffff) { + slot = ix; + n = 16; + } else { + slot = hashmap_bitcount(hval & (bp - 1)); + n = hashmap_bitcount(hval); + } + + ESTACK_PUSH4(*sp, n, bp, slot, node); + + if (!(bp & hval)) { /* not occupied */ + if (is_update) { + UnUseTmpHeapNoproc(2); + return 0; + } + size += HAMT_NODE_BITMAP_SZ(n+1); + goto unroll; + } + + hx = hashmap_shift_hash(th,hx,lvl,key); + node = ptr[slot+1]; + ASSERT(HAMT_NODE_BITMAP_SZ(n) <= 17); + size += HAMT_NODE_BITMAP_SZ(n); + break; - ESTACK_PUSH4(*sp, n, bp, slot, node); - - /* occupied */ - if (bp & hval) { - hx = hashmap_shift_hash(th,hx,lvl,key); - node = ptr[slot+1]; - ASSERT(HAMT_NODE_BITMAP_SZ(n) <= 17); - size += HAMT_NODE_BITMAP_SZ(n); - break; - } - /* not occupied */ - if (is_update) { - return 0; - } - size += HAMT_NODE_BITMAP_SZ(n+1); - goto unroll; case HAMT_SUBTAG_HEAD_BITMAP: hval = MAP_HEADER_VAL(hdr); ix = hashmap_index(hx); @@ -1976,6 +2029,7 @@ int erts_hashmap_insert_down(Uint32 hx, Eterm key, Eterm node, Uint *sz, } /* not occupied */ if (is_update) { + UnUseTmpHeapNoproc(2); return 0; } size += HAMT_HEAD_BITMAP_SZ(n+1); @@ -2009,6 +2063,7 @@ insert_subnodes: unroll: *sz = size + /* res cons */ 2; + UnUseTmpHeapNoproc(2); return 1; } @@ -2151,13 +2206,14 @@ static Eterm hashmap_values(Process* p, Eterm node) { static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm map) { Eterm *hp = NULL, *nhp = NULL, *hp_end = NULL; - Eterm th[2]; Eterm *ptr; Eterm hdr, res = map, node = map; Uint32 ix, bp, hval; Uint slot, lvl = 0; Uint size = 0, n = 0; DECLARE_ESTACK(stack); + DeclareTmpHeapNoproc(th,2); + UseTmpHeapNoproc(2); for (;;) { switch(primary_tag(node)) { @@ -2183,21 +2239,25 @@ static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm map) { hval = MAP_HEADER_VAL(hdr); ix = hashmap_index(hx); bp = 1 << ix; - slot = hashmap_bitcount(hval & (bp - 1)); - n = hashmap_bitcount(hval); + if (hval == 0xffff) { + slot = ix; + n = 16; + } else if (bp & hval) { + slot = hashmap_bitcount(hval & (bp - 1)); + n = hashmap_bitcount(hval); + } else { + /* not occupied */ + goto not_found; + } ESTACK_PUSH4(stack, n, bp, slot, node); - /* occupied */ - if (bp & hval) { - hx = hashmap_shift_hash(th,hx,lvl,key); - node = ptr[slot+1]; - ASSERT(HAMT_NODE_BITMAP_SZ(n) <= 17); - size += HAMT_NODE_BITMAP_SZ(n); - break; - } - /* not occupied */ - goto not_found; + hx = hashmap_shift_hash(th,hx,lvl,key); + node = ptr[slot+1]; + ASSERT(HAMT_NODE_BITMAP_SZ(n) <= 17); + size += HAMT_NODE_BITMAP_SZ(n); + break; + case HAMT_SUBTAG_HEAD_BITMAP: hval = MAP_HEADER_VAL(hdr); ix = hashmap_index(hx); @@ -2268,6 +2328,7 @@ unroll: erts_validate_and_sort_flatmap(mp); DESTROY_WSTACK(wstack); + UnUseTmpHeapNoproc(2); return make_flatmap(mp); } @@ -2408,6 +2469,7 @@ not_found: DESTROY_ESTACK(stack); ERTS_VERIFY_UNUSED_TEMP_ALLOC(p); ERTS_HOLE_CHECK(p); + UnUseTmpHeapNoproc(2); return res; } @@ -2488,8 +2550,12 @@ Uint hashmap_over_estimated_heap_size(Uint k) BIF_RETTYPE erts_debug_map_info_1(BIF_ALIST_1) { if (is_hashmap(BIF_ARG_1)) { BIF_RET(hashmap_info(BIF_P,BIF_ARG_1)); + } else if (is_flatmap(BIF_ARG_1)) { + BIF_ERROR(BIF_P, BADARG); + } else { + BIF_P->fvalue = BIF_ARG_1; + BIF_ERROR(BIF_P, BADMAP); } - BIF_ERROR(BIF_P, BADARG); } /* @@ -2502,8 +2568,12 @@ BIF_RETTYPE erts_internal_map_to_tuple_keys_1(BIF_ALIST_1) { if (is_flatmap(BIF_ARG_1)) { flatmap_t *mp = (flatmap_t*)flatmap_val(BIF_ARG_1); BIF_RET(mp->keys); + } else if (is_hashmap(BIF_ARG_1)) { + BIF_ERROR(BIF_P, BADARG); + } else { + BIF_P->fvalue = BIF_ARG_1; + BIF_ERROR(BIF_P, BADMAP); } - BIF_ERROR(BIF_P, BADARG); } /* @@ -2531,7 +2601,8 @@ BIF_RETTYPE erts_internal_map_type_1(BIF_ALIST_1) { erl_exit(1, "bad header"); } } - BIF_ERROR(BIF_P, BADARG); + BIF_P->fvalue = BIF_ARG_1; + BIF_ERROR(BIF_P, BADMAP); } /* @@ -2571,8 +2642,12 @@ BIF_RETTYPE erts_internal_map_hashmap_children_1(BIF_ALIST_1) { hp = HAlloc(BIF_P, 2*sz); while(sz--) { res = CONS(hp, *ptr++, res); hp += 2; } BIF_RET(res); + } else if (is_flatmap(BIF_ARG_1)) { + BIF_ERROR(BIF_P, BADARG); + } else { + BIF_P->fvalue = BIF_ARG_1; + BIF_ERROR(BIF_P, BADMAP); } - BIF_ERROR(BIF_P, BADARG); } diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index 43a03c793e..ccfc2e6458 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -369,11 +369,7 @@ erts_queue_dist_message(Process *rcvr, tok_label, tok_lastcnt, tok_serial); } #endif - erts_queue_message(rcvr, rcvr_locks, mbuf, msg, token -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(rcvr, rcvr_locks, mbuf, msg, token); } else { /* Enqueue message on external format */ @@ -563,15 +559,15 @@ queue_message(Process *c_p, } void -erts_queue_message(Process* receiver, - ErtsProcLocks *receiver_locks, - ErlHeapFragment* bp, - Eterm message, - Eterm seq_trace_token #ifdef USE_VM_PROBES - , Eterm dt_utag +erts_queue_message_probe(Process* receiver, ErtsProcLocks *receiver_locks, + ErlHeapFragment* bp, + Eterm message, Eterm seq_trace_token, Eterm dt_utag) +#else +erts_queue_message(Process* receiver, ErtsProcLocks *receiver_locks, + ErlHeapFragment* bp, + Eterm message, Eterm seq_trace_token) #endif - ) { queue_message(NULL, receiver, @@ -1117,11 +1113,7 @@ erts_deliver_exit_message(Eterm from, Process *to, ErtsProcLocks *to_locksp, /* the trace token must in this case be updated by the caller */ seq_trace_output(token, save, SEQ_TRACE_SEND, to->common.id, NULL); temptoken = copy_struct(token, sz_token, &hp, &bp->off_heap); - erts_queue_message(to, to_locksp, bp, save, temptoken -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(to, to_locksp, bp, save, temptoken); } else { ErlOffHeap *ohp; sz_reason = size_object(reason); @@ -1138,11 +1130,7 @@ erts_deliver_exit_message(Eterm from, Process *to, ErtsProcLocks *to_locksp, ? from : copy_struct(from, sz_from, &hp, ohp)); save = TUPLE3(hp, am_EXIT, from_copy, mess); - erts_queue_message(to, to_locksp, bp, save, NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(to, to_locksp, bp, save, NIL); } } diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index 6b8c3cebc7..1e1dafee90 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -248,11 +248,17 @@ ErlHeapFragment* erts_resize_message_buffer(ErlHeapFragment *, Uint, Eterm *, Uint); void free_message_buffer(ErlHeapFragment *); void erts_queue_dist_message(Process*, ErtsProcLocks*, ErtsDistExternal *, Eterm); -void erts_queue_message(Process*, ErtsProcLocks*, ErlHeapFragment*, Eterm, Eterm #ifdef USE_VM_PROBES - , Eterm dt_utag +void erts_queue_message_probe(Process*, ErtsProcLocks*, ErlHeapFragment*, + Eterm message, Eterm seq_trace_token, Eterm dt_utag); +#define erts_queue_message(RP,RL,BP,Msg,SEQ) \ + erts_queue_message_probe((RP),(RL),(BP),(Msg),(SEQ),NIL) +#else +void erts_queue_message(Process*, ErtsProcLocks*, ErlHeapFragment*, + Eterm message, Eterm seq_trace_token); +#define erts_queue_message_probe(RP,RL,BP,Msg,SEQ,TAG) \ + erts_queue_message((RP),(RL),(BP),(Msg),(SEQ)) #endif -); void erts_deliver_exit_message(Eterm, Process*, ErtsProcLocks *, Eterm, Eterm); Sint erts_send_message(Process*, Process*, ErtsProcLocks*, Eterm, unsigned); void erts_link_mbuf_to_proc(Process *proc, ErlHeapFragment *bp); diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 62f018bd41..426a00304e 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -128,6 +128,7 @@ void erts_pre_nif(ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif) env->heap_frag = NULL; env->fpe_was_unmasked = erts_block_fpe(); env->tmp_obj_list = NULL; + env->exception_thrown = 0; } static void pre_nif_noproc(ErlNifEnv* env, struct erl_module_nif* mod_nif) @@ -357,11 +358,7 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, if (flush_me) { flush_env(env); /* Needed for ERTS_HOLE_CHECK */ } - erts_queue_message(rp, &rp_locks, frags, msg, am_undefined -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(rp, &rp_locks, frags, msg, am_undefined); if (c_p == rp) rp_locks &= ~ERTS_PROC_LOCK_MAIN; if (rp_locks) @@ -740,9 +737,15 @@ Eterm enif_make_sub_binary(ErlNifEnv* env, ERL_NIF_TERM bin_term, Eterm enif_make_badarg(ErlNifEnv* env) { + env->exception_thrown = 1; BIF_ERROR(env->proc, BADARG); } +int enif_has_pending_exception(ErlNifEnv* env) +{ + return env->exception_thrown; +} + int enif_get_atom(ErlNifEnv* env, Eterm atom, char* buf, unsigned len, ErlNifCharEncoding encoding) { @@ -962,8 +965,12 @@ ERL_NIF_TERM enif_make_uint64(ErlNifEnv* env, ErlNifUInt64 i) ERL_NIF_TERM enif_make_double(ErlNifEnv* env, double d) { - Eterm* hp = alloc_heap(env,FLOAT_SIZE_OBJECT); + Eterm* hp; FloatDef f; + + if (!erts_isfinite(d)) + return enif_make_badarg(env); + hp = alloc_heap(env,FLOAT_SIZE_OBJECT); f.fd = d; PUT_DOUBLE(f, hp); return make_float(hp); @@ -976,6 +983,8 @@ ERL_NIF_TERM enif_make_atom(ErlNifEnv* env, const char* name) ERL_NIF_TERM enif_make_atom_len(ErlNifEnv* env, const char* name, size_t len) { + if (len > MAX_ATOM_CHARACTERS) + return enif_make_badarg(env); return erts_atom_put((byte*)name, len, ERTS_ATOM_ENC_LATIN1, 1); } @@ -989,6 +998,8 @@ int enif_make_existing_atom_len(ErlNifEnv* env, const char* name, size_t len, ERL_NIF_TERM* atom, ErlNifCharEncoding encoding) { ASSERT(encoding == ERL_NIF_LATIN1); + if (len > MAX_ATOM_CHARACTERS) + return 0; return erts_atom_get(name, len, atom, ERTS_ATOM_ENC_LATIN1); } @@ -1752,14 +1763,13 @@ execute_dirty_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) ASSERT(ep); if (ep->fp) fp = NULL; - if (is_non_value(result)) { + if (is_non_value(result) || env->exception_thrown) { if (proc->freason != TRAP) { - ASSERT(proc->freason == BADARG); return init_nif_sched_data(env, dirty_nif_exception, fp, 0, argc, argv); } else { if (ep->fp == NULL) restore_nif_mfa(proc, ep, 1); - return result; + return THE_NON_VALUE; } } else diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h index 9b2b90c82d..c4fdfd4187 100644 --- a/erts/emulator/beam/erl_nif.h +++ b/erts/emulator/beam/erl_nif.h @@ -46,9 +46,10 @@ ** remove enif_schedule_dirty_nif, enif_schedule_dirty_nif_finalizer, enif_dirty_nif_finalizer ** add ErlNifEntry options ** add ErlNifFunc flags +** 2.8: 18.0 add enif_has_pending_exception */ #define ERL_NIF_MAJOR_VERSION 2 -#define ERL_NIF_MINOR_VERSION 7 +#define ERL_NIF_MINOR_VERSION 8 /* * The emulator will refuse to load a nif-lib with a major version diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h index 630cefae93..bdcbb32c46 100644 --- a/erts/emulator/beam/erl_nif_api_funcs.h +++ b/erts/emulator/beam/erl_nif_api_funcs.h @@ -156,6 +156,7 @@ ERL_NIF_API_FUNC_DECL(int, enif_map_iterator_next, (ErlNifEnv *env, ErlNifMapIte ERL_NIF_API_FUNC_DECL(int, enif_map_iterator_prev, (ErlNifEnv *env, ErlNifMapIterator *iter)); ERL_NIF_API_FUNC_DECL(int, enif_map_iterator_get_pair, (ErlNifEnv *env, ErlNifMapIterator *iter, ERL_NIF_TERM *key, ERL_NIF_TERM *value)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_schedule_nif,(ErlNifEnv*,const char*,int,ERL_NIF_TERM (*)(ErlNifEnv*,int,const ERL_NIF_TERM[]),int,const ERL_NIF_TERM[])); +ERL_NIF_API_FUNC_DECL(int, enif_has_pending_exception, (ErlNifEnv *env)); /* ** ADD NEW ENTRIES HERE (before this comment) !!! @@ -305,6 +306,7 @@ ERL_NIF_API_FUNC_DECL(int,enif_is_on_dirty_scheduler,(ErlNifEnv*)); # define enif_map_iterator_prev ERL_NIF_API_FUNC_MACRO(enif_map_iterator_prev) # define enif_map_iterator_get_pair ERL_NIF_API_FUNC_MACRO(enif_map_iterator_get_pair) # define enif_schedule_nif ERL_NIF_API_FUNC_MACRO(enif_schedule_nif) +# define enif_has_pending_exception ERL_NIF_API_FUNC_MACRO(enif_has_pending_exception) /* ** ADD NEW ENTRIES HERE (before this comment) diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 588bc75a43..af8db519d4 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -1027,11 +1027,7 @@ reply_sched_wall_time(void *vswtrp) hpp = &hp; } - erts_queue_message(rp, &rp_locks, bp, msg, NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(rp, &rp_locks, bp, msg, NIL); if (swtrp->req_sched == esdp->no) rp_locks &= ~ERTS_PROC_LOCK_MAIN; @@ -5026,9 +5022,11 @@ wakeup_other_check(ErtsRunQueue *rq, Uint32 flags) + ERTS_WAKEUP_OTHER_FIXED_INC); if (rq->wakeup_other > wakeup_other.limit) { #ifdef ERTS_DIRTY_SCHEDULERS - if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix) && rq->waiting) - wake_dirty_schedulers(rq, 1); - else + if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) { + if (rq->waiting) { + wake_dirty_schedulers(rq, 1); + } + } else #endif { int empty_rqs = @@ -9715,15 +9713,7 @@ notify_sys_task_executed(Process *c_p, ErtsProcSysTask *st, Eterm st_result) ASSERT(hp_start + hsz == hp); #endif - erts_queue_message(rp, - &rp_locks, - bp, - msg, - NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(rp, &rp_locks, bp, msg, NIL); if (c_p == rp) rp_locks &= ~ERTS_PROC_LOCK_MAIN; @@ -11413,11 +11403,7 @@ send_exit_message(Process *to, ErtsProcLocks *to_locksp, hp = erts_alloc_message_heap(term_size, &bp, &ohp, to, to_locksp); mess = copy_struct(exit_term, term_size, &hp, ohp); - erts_queue_message(to, to_locksp, bp, mess, NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(to, to_locksp, bp, mess, NIL); } else { ErlHeapFragment* bp; Eterm* hp; @@ -11433,11 +11419,7 @@ send_exit_message(Process *to, ErtsProcLocks *to_locksp, /* the trace token must in this case be updated by the caller */ seq_trace_output(token, mess, SEQ_TRACE_SEND, to->common.id, NULL); temp_token = copy_struct(token, sz_token, &hp, &bp->off_heap); - erts_queue_message(to, to_locksp, bp, mess, temp_token -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(to, to_locksp, bp, mess, temp_token); } } diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h index 095aa54ddd..602aab46dc 100644 --- a/erts/emulator/beam/erl_term.h +++ b/erts/emulator/beam/erl_term.h @@ -298,7 +298,6 @@ _ET_DECLARE_CHECKED(Uint,atom_val,Eterm) /* header (arityval or thing) access methods */ #define _make_header(sz,tag) ((Uint)(((Uint)(sz) << _HEADER_ARITY_OFFS) + (tag))) #define is_header(x) (((x) & _TAG_PRIMARY_MASK) == TAG_PRIMARY_HEADER) -//#define _unchecked_header_arity(x) ((x) >> _HEADER_ARITY_OFFS) #define _unchecked_header_arity(x) \ (is_map_header(x) ? MAP_HEADER_ARITY(x) : ((x) >> _HEADER_ARITY_OFFS)) _ET_DECLARE_CHECKED(Uint,header_arity,Eterm) @@ -1028,6 +1027,7 @@ _ET_DECLARE_CHECKED(struct erl_node_*,external_ref_node,Eterm) #define is_map_header(x) (((x) & (_TAG_HEADER_MASK)) == _TAG_HEADER_MAP) #define is_map(x) (is_boxed((x)) && is_map_header(*boxed_val(x))) +#define is_not_map(x) (!is_map(x)) #define is_map_rel(RTERM,BASE) is_map(rterm2wterm(RTERM,BASE)) /* number tests */ diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c index 1139cb9c97..e550c999b8 100644 --- a/erts/emulator/beam/erl_time_sup.c +++ b/erts/emulator/beam/erl_time_sup.c @@ -1929,11 +1929,7 @@ send_time_offset_changed_notifications(void *new_offsetp) *patch_refp = ref; ASSERT(hsz == size_object(message_template)); message = copy_struct(message_template, hsz, &hp, ohp); - erts_queue_message(rp, &rp_locks, bp, message, NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(rp, &rp_locks, bp, message, NIL); } erts_smp_proc_unlock(rp, rp_locks); } diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index 2f9969b0e7..aaecc5a02e 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -130,14 +130,9 @@ do { \ enqueue_sys_msg_unlocked(SYS_MSG_TYPE_TRACE, (FPID), (TPID), (MSG), (BP)); \ } while(0) #else -#ifdef USE_VM_PROBES -#define ERTS_ENQ_TRACE_MSG(FPID, TPROC, MSG, BP) \ - erts_queue_message((TPROC), NULL, (BP), (MSG), NIL, NIL) -#else #define ERTS_ENQ_TRACE_MSG(FPID, TPROC, MSG, BP) \ erts_queue_message((TPROC), NULL, (BP), (MSG), NIL) #endif -#endif /* * NOTE that the ERTS_GET_TRACER_REF() returns from the function (!!!) @@ -636,11 +631,7 @@ profile_send(Eterm from, Eterm message) { hp = erts_alloc_message_heap(sz, &bp, &off_heap, profile_p, 0); msg = copy_struct(message, sz, &hp, &bp->off_heap); - erts_queue_message(profile_p, NULL, bp, msg, NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(profile_p, NULL, bp, msg, NIL); } } @@ -1240,11 +1231,8 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type, enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SEQTRACE, NIL, NIL, mess, bp); erts_smp_mtx_unlock(&smq_mtx); #else - erts_queue_message(tracer, NULL, bp, mess, NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); /* trace_token must be NIL here */ + /* trace_token must be NIL here */ + erts_queue_message(tracer, NULL, bp, mess, NIL); #endif } } @@ -2343,11 +2331,7 @@ monitor_long_schedule_proc(Process *p, BeamInstr *in_fp, BeamInstr *out_fp, Uint #ifdef ERTS_SMP enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, p->common.id, NIL, msg, bp); #else - erts_queue_message(monitor_p, NULL, bp, msg, NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(monitor_p, NULL, bp, msg, NIL); #endif } void @@ -2408,11 +2392,7 @@ monitor_long_schedule_port(Port *pp, ErtsPortTaskType type, Uint time) #ifdef ERTS_SMP enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, pp->common.id, NIL, msg, bp); #else - erts_queue_message(monitor_p, NULL, bp, msg, NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(monitor_p, NULL, bp, msg, NIL); #endif } @@ -2483,11 +2463,7 @@ monitor_long_gc(Process *p, Uint time) { #ifdef ERTS_SMP enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, p->common.id, NIL, msg, bp); #else - erts_queue_message(monitor_p, NULL, bp, msg, NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(monitor_p, NULL, bp, msg, NIL); #endif } @@ -2558,11 +2534,7 @@ monitor_large_heap(Process *p) { #ifdef ERTS_SMP enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, p->common.id, NIL, msg, bp); #else - erts_queue_message(monitor_p, NULL, bp, msg, NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(monitor_p, NULL, bp, msg, NIL); #endif } @@ -2590,11 +2562,7 @@ monitor_generic(Process *p, Eterm type, Eterm spec) { #ifdef ERTS_SMP enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, p->common.id, NIL, msg, bp); #else - erts_queue_message(monitor_p, NULL, bp, msg, NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(monitor_p, NULL, bp, msg, NIL); #endif } @@ -3389,11 +3357,7 @@ sys_msg_dispatcher_func(void *unused) } else { queue_proc_msg: - erts_queue_message(proc,&proc_locks,smqp->bp,smqp->msg,NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(proc,&proc_locks,smqp->bp,smqp->msg,NIL); #ifdef DEBUG_PRINTOUTS erts_fprintf(stderr, "delivered\n"); #endif diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h index 7cb8972e29..6a28105cb9 100644 --- a/erts/emulator/beam/erl_utils.h +++ b/erts/emulator/beam/erl_utils.h @@ -167,24 +167,26 @@ int eq(Eterm, Eterm); #define EQ(x,y) (((x) == (y)) || (is_not_both_immed((x),(y)) && eq((x),(y)))) #if HALFWORD_HEAP -Sint erts_cmp_rel_opt(Eterm, Eterm*, Eterm, Eterm*, int); -#define cmp_rel(A,A_BASE,B,B_BASE) erts_cmp_rel_opt(A,A_BASE,B,B_BASE,0) -#define cmp_rel_term(A,A_BASE,B,B_BASE) erts_cmp_rel_opt(A,A_BASE,B,B_BASE,1) -#define CMP(A,B) erts_cmp_rel_opt(A,NULL,B,NULL,0) -#define CMP_TERM(A,B) erts_cmp_rel_opt(A,NULL,B,NULL,1) +Sint erts_cmp_rel_opt(Eterm, Eterm*, Eterm, Eterm*, int, int); +#define cmp_rel(A,A_BASE,B,B_BASE) erts_cmp_rel_opt(A,A_BASE,B,B_BASE,0,0) +#define cmp_rel_term(A,A_BASE,B,B_BASE) erts_cmp_rel_opt(A,A_BASE,B,B_BASE,1,0) +#define CMP(A,B) erts_cmp_rel_opt(A,NULL,B,NULL,0,0) +#define CMP_TERM(A,B) erts_cmp_rel_opt(A,NULL,B,NULL,1,0) +#define CMP_EQ_ONLY(A,B) erts_cmp_rel_opt(A,NULL,B,NULL,0,1) #else -Sint cmp(Eterm, Eterm); -Sint erts_cmp(Eterm, Eterm, int); -#define cmp_rel(A,A_BASE,B,B_BASE) erts_cmp(A,B,0) -#define cmp_rel_term(A,A_BASE,B,B_BASE) erts_cmp(A,B,1) -#define CMP(A,B) erts_cmp(A,B,0) -#define CMP_TERM(A,B) erts_cmp(A,B,1) +Sint erts_cmp(Eterm, Eterm, int, int); +Sint cmp(Eterm a, Eterm b); +#define cmp_rel(A,A_BASE,B,B_BASE) erts_cmp(A,B,0,0) +#define cmp_rel_term(A,A_BASE,B,B_BASE) erts_cmp(A,B,1,0) +#define CMP(A,B) erts_cmp(A,B,0,0) +#define CMP_TERM(A,B) erts_cmp(A,B,1,0) +#define CMP_EQ_ONLY(A,B) erts_cmp(A,B,0,1) #endif #define cmp_lt(a,b) (CMP((a),(b)) < 0) #define cmp_le(a,b) (CMP((a),(b)) <= 0) -#define cmp_eq(a,b) (CMP((a),(b)) == 0) -#define cmp_ne(a,b) (CMP((a),(b)) != 0) +#define cmp_eq(a,b) (CMP_EQ_ONLY((a),(b)) == 0) +#define cmp_ne(a,b) (CMP_EQ_ONLY((a),(b)) != 0) #define cmp_ge(a,b) (CMP((a),(b)) >= 0) #define cmp_gt(a,b) (CMP((a),(b)) > 0) diff --git a/erts/emulator/beam/error.h b/erts/emulator/beam/error.h index ddc2c1396d..e63967adb6 100644 --- a/erts/emulator/beam/error.h +++ b/erts/emulator/beam/error.h @@ -140,7 +140,13 @@ #define EXC_NOTSUP ((17 << 8) | EXC_ERROR) /* Not supported */ -#define NUMBER_EXIT_CODES 18 /* The number of exit code indices */ +#define EXC_BADMAP ((18 << 8) | EXC_ERROR) + /* Bad map */ + +#define EXC_BADKEY ((19 << 8) | EXC_ERROR) + /* Bad key in map */ + +#define NUMBER_EXIT_CODES 20 /* The number of exit code indices */ /* * Internal pseudo-error codes. @@ -152,6 +158,8 @@ */ #define BADARG EXC_BADARG #define BADARITH EXC_BADARITH +#define BADKEY EXC_BADKEY +#define BADMAP EXC_BADMAP #define BADMATCH EXC_BADMATCH #define SYSTEM_LIMIT EXC_SYSTEM_LIMIT diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 2a9189b51e..fe48298155 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -1779,7 +1779,7 @@ static void ttb_context_destructor(Binary *context_bin) context->alive = 0; switch (context->state) { case TTBSize: - DESTROY_SAVED_ESTACK(&context->s.sc.estack); + DESTROY_SAVED_WSTACK(&context->s.sc.wstack); break; case TTBEncode: DESTROY_SAVED_WSTACK(&context->s.ec.wstack); @@ -1847,7 +1847,7 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla /* Setup enough to get started */ context->state = TTBSize; context->alive = 1; - context->s.sc.estack.start = NULL; + context->s.sc.wstack.wstart = NULL; context->s.sc.flags = flags; context->s.sc.level = level; } else { @@ -3962,51 +3962,35 @@ static int encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags, Sint *reds, Uint *res) { - DECLARE_ESTACK(s); + DECLARE_WSTACK(s); Uint m, i, arity; Uint result = 0; Sint r = 0; if (ctx) { - ESTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK); + WSTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK); r = *reds; - if (ctx->estack.start) { /* restore saved stack */ - ESTACK_RESTORE(s, &ctx->estack); + if (ctx->wstack.wstart) { /* restore saved stack */ + WSTACK_RESTORE(s, &ctx->wstack); result = ctx->result; obj = ctx->obj; } } - goto L_jump_start; +#define LIST_TAIL_OP ((0 << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER) +#define TERM_ARRAY_OP(N) (((N) << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER) +#define TERM_ARRAY_OP_DEC(OP) ((OP) - (1 << _TAG_PRIMARY_SIZE)) + + + for (;;) { + ASSERT(!is_header(obj)); - outer_loop: - while (!ESTACK_ISEMPTY(s)) { - obj = ESTACK_POP(s); - handle_popped_obj: - if (is_list(obj)) { - Eterm* cons = list_val(obj); - Eterm tl; - - tl = CDR(cons); - obj = CAR(cons); - ESTACK_PUSH(s, tl); - } else if (is_nil(obj)) { - result++; - goto outer_loop; - } else { - /* - * Other term (in the tail of a non-proper list or - * in a fun's environment). - */ - } - - L_jump_start: if (ctx && --r == 0) { *reds = r; ctx->obj = obj; ctx->result = result; - ESTACK_SAVE(s, &ctx->estack); + WSTACK_SAVE(s, &ctx->wstack); return -1; } switch (tag_val_def(obj)) { @@ -4089,70 +4073,43 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, result += m + 2 + 1; } else { result += 5; - goto handle_popped_obj; + WSTACK_PUSH2(s, (UWord)CDR(list_val(obj)), (UWord)LIST_TAIL_OP); + obj = CAR(list_val(obj)); + continue; /* big loop */ } break; case TUPLE_DEF: { Eterm* ptr = tuple_val(obj); - Uint i; arity = arityval(*ptr); if (arity <= 0xff) { result += 1 + 1; } else { result += 1 + 4; } - for (i = 1; i <= arity; ++i) { - if (is_list(ptr[i])) { - if ((m = is_string(obj)) && (m < MAX_STRING_LEN)) { - result += m + 2 + 1; - } else { - result += 5; - } - } - ESTACK_PUSH(s,ptr[i]); + if (arity > 1) { + WSTACK_PUSH2(s, (UWord) (ptr + 2), + (UWord) TERM_ARRAY_OP(arity-1)); } - goto outer_loop; + else if (arity == 0) { + break; + } + obj = ptr[1]; + continue; /* big loop */ } - break; case MAP_DEF: if (is_flatmap(obj)) { flatmap_t *mp = (flatmap_t*)flatmap_val(obj); Uint size = flatmap_get_size(mp); - Uint i; - Eterm *ptr; result += 1 + 4; /* tag + 4 bytes size */ - /* push values first */ - ptr = flatmap_get_values(mp); - i = size; - while(i--) { - if (is_list(*ptr)) { - if ((m = is_string(*ptr)) && (m < MAX_STRING_LEN)) { - result += m + 2 + 1; - } else { - result += 5; - } - } - ESTACK_PUSH(s,*ptr); - ++ptr; + if (size) { + WSTACK_PUSH4(s, (UWord) flatmap_get_values(mp), + (UWord) TERM_ARRAY_OP(size), + (UWord) flatmap_get_keys(mp), + (UWord) TERM_ARRAY_OP(size)); } - - ptr = flatmap_get_keys(mp); - i = size; - while(i--) { - if (is_list(*ptr)) { - if ((m = is_string(*ptr)) && (m < MAX_STRING_LEN)) { - result += m + 2 + 1; - } else { - result += 5; - } - } - ESTACK_PUSH(s,*ptr); - ++ptr; - } - goto outer_loop; } else { Eterm *ptr; Eterm hdr; @@ -4164,8 +4121,12 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, case HAMT_SUBTAG_HEAD_ARRAY: ptr++; node_sz = 16; + result += 1 + 4; /* tag + 4 bytes size */ break; - case HAMT_SUBTAG_HEAD_BITMAP: ptr++; + case HAMT_SUBTAG_HEAD_BITMAP: + ptr++; + result += 1 + 4; /* tag + 4 bytes size */ + /*fall through*/ case HAMT_SUBTAG_NODE_BITMAP: node_sz = hashmap_bitcount(MAP_HEADER_VAL(hdr)); ASSERT(node_sz < 17); @@ -4175,11 +4136,16 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, } ptr++; - ESTACK_RESERVE(s, node_sz); + WSTACK_RESERVE(s, node_sz*2); while(node_sz--) { - ESTACK_FAST_PUSH(s, *ptr++); + if (is_list(*ptr)) { + WSTACK_FAST_PUSH(s, CAR(list_val(*ptr))); + WSTACK_FAST_PUSH(s, CDR(list_val(*ptr))); + } else { + WSTACK_FAST_PUSH(s, *ptr); + } + ptr++; } - result += 1 + 4; /* tag + 4 bytes size */ } break; case FLOAT_DEF: @@ -4232,25 +4198,13 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, result += 2 * (1 + 4); /* Index + Uniq */ result += 1 + (funp->num_free < 0x100 ? 1 : 4); } - for (i = 1; i < funp->num_free; i++) { - obj = funp->env[i]; - - if (is_not_list(obj)) { - /* Push any non-list terms on the stack */ - ESTACK_PUSH(s, obj); - } else { - /* Lists must be handled specially. */ - if ((m = is_string(obj)) && (m < MAX_STRING_LEN)) { - result += m + 2 + 1; - } else { - result += 5; - ESTACK_PUSH(s, obj); - } - } + if (funp->num_free > 1) { + WSTACK_PUSH2(s, (UWord) (funp->env + 1), + (UWord) TERM_ARRAY_OP(funp->num_free-1)); } if (funp->num_free != 0) { obj = funp->env[0]; - goto L_jump_start; + continue; /* big loop */ } break; } @@ -4273,11 +4227,40 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, erl_exit(1,"Internal data structure error (in encode_size_struct2)%x\n", obj); } + + if (WSTACK_ISEMPTY(s)) { + break; + } + obj = (Eterm) WSTACK_POP(s); + + if (is_header(obj)) { + switch (obj) { + case LIST_TAIL_OP: + obj = (Eterm) WSTACK_POP(s); + if (is_list(obj)) { + Eterm* cons = list_val(obj); + + WSTACK_PUSH2(s, (UWord)CDR(cons), (UWord)LIST_TAIL_OP); + obj = CAR(cons); + } + break; + + case TERM_ARRAY_OP(1): + obj = *(Eterm*)WSTACK_POP(s); + break; + default: { /* TERM_ARRAY_OP(N) when N > 1 */ + Eterm* ptr = (Eterm*) WSTACK_POP(s); + WSTACK_PUSH2(s, (UWord) (ptr+1), + (UWord) TERM_ARRAY_OP_DEC(obj)); + obj = *ptr; + } + } + } } - DESTROY_ESTACK(s); + WSTACK_DESTROY(s); if (ctx) { - ASSERT(ctx->estack.start == NULL); + ASSERT(ctx->wstack.wstart == NULL); *reds = r; } *res = result; diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 44d4ef18dd..340c7033ab 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -52,6 +52,7 @@ struct enif_environment_t /* ErlNifEnv */ ErlHeapFragment* heap_frag; int fpe_was_unmasked; struct enif_tmp_obj_t* tmp_obj_list; + int exception_thrown; /* boolean */ }; extern void erts_pre_nif(struct enif_environment_t*, Process*, struct erl_module_nif*); @@ -866,6 +867,9 @@ void print_process_info(int, void *, Process*); void info(int, void *); void loaded(int, void *); +/* erl_arith.c */ +double erts_get_positive_zero_float(void); + /* config.c */ __decl_noreturn void __noreturn erl_exit(int n, char*, ...); diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index 3e5e97ab15..ccc7da265e 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -1419,15 +1419,7 @@ queue_port_sched_op_reply(Process *rp, bp = erts_resize_message_buffer(bp, used_h_size, &msg, 1); } - erts_queue_message(rp, - rp_locksp, - bp, - msg, - NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(rp, rp_locksp, bp, msg, NIL); } static void @@ -3076,11 +3068,7 @@ deliver_result(Eterm sender, Eterm pid, Eterm res) hp = erts_alloc_message_heap(sz_res + 3, &bp, &ohp, rp, &rp_locks); res = copy_struct(res, sz_res, &hp, ohp); tuple = TUPLE2(hp, sender, res); - erts_queue_message(rp, &rp_locks, bp, tuple, NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(rp, &rp_locks, bp, tuple, NIL); if (rp_locks) erts_smp_proc_unlock(rp, rp_locks); @@ -3176,11 +3164,7 @@ static void deliver_read_message(Port* prt, erts_aint32_t state, Eterm to, tuple = TUPLE2(hp, prt->common.id, tuple); hp += 3; - erts_queue_message(rp, &rp_locks, bp, tuple, am_undefined -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(rp, &rp_locks, bp, tuple, am_undefined); if (rp_locks) erts_smp_proc_unlock(rp, rp_locks); if (!scheduler) @@ -3347,11 +3331,7 @@ deliver_vec_message(Port* prt, /* Port */ tuple = TUPLE2(hp, prt->common.id, tuple); hp += 3; - erts_queue_message(rp, &rp_locks, bp, tuple, am_undefined -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(rp, &rp_locks, bp, tuple, am_undefined); erts_smp_proc_unlock(rp, rp_locks); if (!scheduler) erts_proc_dec_refc(rp); @@ -5030,11 +5010,7 @@ void driver_report_exit(ErlDrvPort ix, int status) hp += 3; tuple = TUPLE2(hp, prt->common.id, tuple); - erts_queue_message(rp, &rp_locks, bp, tuple, am_undefined -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(rp, &rp_locks, bp, tuple, am_undefined); erts_smp_proc_unlock(rp, rp_locks); if (!scheduler) @@ -5552,7 +5528,9 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) mess = make_float(hp); f.fd = *((double *) ptr[0]); - PUT_DOUBLE(f, hp); + if (!erts_isfinite(f.fd)) + ERTS_DDT_FAIL; + PUT_DOUBLE(f, hp); hp += FLOAT_SIZE_OBJECT; ptr++; break; @@ -5634,11 +5612,7 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) HRelease(rp, hp_end, hp); } /* send message */ - erts_queue_message(rp, &rp_locks, bp, mess, am_undefined -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(rp, &rp_locks, bp, mess, am_undefined); } else { if (b2t.ix > b2t.used) diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab index d3649080dc..ece038131e 100644 --- a/erts/emulator/beam/ops.tab +++ b/erts/emulator/beam/ops.tab @@ -298,10 +298,49 @@ move_jump f c move_jump f x move_jump f y + +# Movement to and from the stack is common +# Try to pack as much as we can into one instruction + +# Window move +move_window/5 +move_window/6 + +# x -> y + +move S1=r S2=y | move X1=x Y1=y => move2 S1 S2 X1 Y1 + +move X1=x Y1=y | move X2=x Y2=y | move X3=x Y3=y | succ(Y1,Y2) | succ(Y2,Y3) => \ + move_window X1 X2 X3 Y1 Y3 + +move_window X1=x X2=x X3=x Y1=y Y3=y | move X4=x Y4=y | succ(Y3,Y4) => \ + move_window X1 X2 X3 X4 Y1 Y4 + +move_window X1=x X2=x X3=x X4=x Y1=y Y4=y | move X5=x Y5=y | succ(Y4,Y5) => \ + move_window5 X1 X2 X3 X4 X5 Y1 + +move_window X1=x X2=x X3=x Y1=y Y3=y => move_window3 X1 X2 X3 Y1 +move_window X1=x X2=x X3=x X4=x Y1=y Y4=y => move_window4 X1 X2 X3 X4 Y1 + +move_window3 x x x y +move_window4 x x x x y +move_window5 x x x x x y + move X1=x Y1=y | move X2=x Y2=y => move2 X1 Y1 X2 Y2 move Y1=y X1=x | move Y2=y X2=x => move2 Y1 X1 Y2 X2 move X1=x X2=x | move X3=x X4=x => move2 X1 X2 X3 X4 +move S1=x S2=r | move S3=x S4=x => move2 S1 S2 S3 S4 +move S1=x S2=r | move X1=x Y1=y => move2 S1 S2 X1 Y1 +move S1=y S2=r | move X1=x Y1=y => move2 S1 S2 X1 Y1 + +move Y1=y X1=x | move S1=r D1=x => move2 Y1 X1 S1 D1 +move S1=r D1=x | move Y1=y X1=x => move2 S1 D1 Y1 X1 + +move2 X1=x Y1=y X2=x Y2=y | move X3=x Y3=y => move3 X1 Y1 X2 Y2 X3 Y3 +move2 Y1=y X1=x Y2=y X2=x | move Y3=y X3=x => move3 Y1 X1 Y2 X2 Y3 X3 +move2 X1=x X2=x X3=x X4=x | move X5=x X6=x => move3 X1 X2 X3 X4 X5 X6 + move C=aiq X=x==1 => move_x1 C move C=aiq X=x==2 => move_x2 C @@ -313,6 +352,20 @@ move2 x y x y move2 y x y x move2 x x x x +move2 x r x x + +move2 x r x y +move2 r y x y +move2 y r x y + +move2 r x y x +move2 y x r x + +%macro: move3 Move3 +move3 x y x y x y +move3 y x y x y x +move3 x x x x x x + # The compiler almost never generates a "move Literal y(Y)" instruction, # so let's cheat if we encounter one. move S=n D=y => init D @@ -392,14 +445,59 @@ i_is_ne_exact_literal x f c i_is_ne_exact_literal y f c # +# Common Compare Specializations +# We don't do all of them since we want +# to keep the instruction set small-ish +# + +is_eq_exact Lbl S1=xy S2=r => is_eq_exact Lbl S2 S1 +is_eq_exact Lbl S1=rx S2=xy => i_is_eq_exact_spec Lbl S1 S2 +%macro: i_is_eq_exact_spec EqualExact -fail_action + +i_is_eq_exact_spec f x x +i_is_eq_exact_spec f x y +i_is_eq_exact_spec f r x +i_is_eq_exact_spec f r y +%cold +i_is_eq_exact_spec f r r +%hot + +is_lt Lbl S1=rxc S2=rxc => i_is_lt_spec Lbl S1 S2 + +%macro: i_is_lt_spec IsLessThan -fail_action + +i_is_lt_spec f x x +i_is_lt_spec f x r +i_is_lt_spec f x c +i_is_lt_spec f r x +i_is_lt_spec f r c +i_is_lt_spec f c x +i_is_lt_spec f c r +%cold +i_is_lt_spec f r r +i_is_lt_spec f c c +%hot + +is_ge Lbl S1=xc S2=xc => i_is_ge_spec Lbl S1 S2 + +%macro: i_is_ge_spec IsGreaterEqual -fail_action + +i_is_ge_spec f x x +i_is_ge_spec f x c +i_is_ge_spec f c x +%cold +i_is_ge_spec f c c +%hot + +# # All other comparisons. # is_eq_exact Lbl S1 S2 => i_fetch S1 S2 | i_is_eq_exact Lbl is_ne_exact Lbl S1 S2 => i_fetch S1 S2 | i_is_ne_exact Lbl -is_ge Lbl S1 S2 => i_fetch S1 S2 | i_is_ge Lbl is_lt Lbl S1 S2 => i_fetch S1 S2 | i_is_lt Lbl +is_ge Lbl S1 S2 => i_fetch S1 S2 | i_is_ge Lbl is_eq Lbl S1 S2 => i_fetch S1 S2 | i_is_eq Lbl is_ne Lbl S1 S2 => i_fetch S1 S2 | i_is_ne Lbl @@ -493,7 +591,6 @@ put_list s s d %hot %macro: i_fetch FetchArgs -pack -i_fetch c c i_fetch c r i_fetch c x i_fetch c y @@ -510,6 +607,7 @@ i_fetch y x i_fetch y y %cold +i_fetch c c i_fetch s s %hot @@ -1473,79 +1571,67 @@ apply_last I P # Map instructions in R17. # -put_map_assoc F n Dst Live Size Rest=* => new_map F Dst Live Size Rest -put_map_assoc F Src=s Dst Live Size Rest=* => \ +sorted_put_map_assoc/5 +put_map_assoc F Map Dst Live Size Rest=* | map_key_sort(Size, Rest) => \ + sorted_put_map_assoc F Map Dst Live Size Rest + +sorted_put_map_exact/5 +put_map_exact F Map Dst Live Size Rest=* | map_key_sort(Size, Rest) => \ + sorted_put_map_exact F Map Dst Live Size Rest + +sorted_put_map_assoc j Map Dst Live Size Rest=* | is_empty_map(Map) => \ + new_map Dst Live Size Rest +sorted_put_map_assoc F Src=s Dst Live Size Rest=* => \ update_map_assoc F Src Dst Live Size Rest -put_map_assoc F Src Dst Live Size Rest=* => \ +sorted_put_map_assoc F Src Dst Live Size Rest=* => \ move Src x | update_map_assoc F x Dst Live Size Rest -put_map_exact F n Dst Live Size Rest=* => new_map F Dst Live Size Rest -put_map_exact F Src=s Dst Live Size Rest=* => \ + +sorted_put_map_exact F Src=s Dst Live Size Rest=* => \ update_map_exact F Src Dst Live Size Rest -put_map_exact F Src Dst Live Size Rest=* => \ +sorted_put_map_exact F Src Dst Live Size Rest=* => \ move Src x | update_map_exact F x Dst Live Size Rest -new_map j d I I +new_map d I I update_map_assoc j s d I I update_map_exact j s d I I -is_map Fail Literal=q => move Literal x | is_map Fail x -is_map Fail c => jump Fail +is_map Fail Lit=q | literal_is_map(Lit) => +is_map Fail cq => jump Fail %macro: is_map IsMap -fail_action is_map f r is_map f x is_map f y -## Transform has_map_field(s) #{ K1 := _, K2 := _ } - -has_map_field/3 +## Transform has_map_fields #{ K1 := _, K2 := _ } to has_map_elements -has_map_fields Fail Src Size=u==1 Rest=* => gen_has_map_field(Fail,Src,Size,Rest) -has_map_fields Fail Src Size Rest=* => i_has_map_fields Fail Src Size Rest - -i_has_map_fields f s I - -has_map_field Fail Src=rxy Key=arxy => i_has_map_field Fail Src Key -has_map_field Fail Src Key => move Key x | i_has_map_field Fail Src x - -%macro: i_has_map_field HasMapField -fail_action -i_has_map_field f r a -i_has_map_field f x a -i_has_map_field f y a -i_has_map_field f r r -i_has_map_field f x r -i_has_map_field f y r -i_has_map_field f r x -i_has_map_field f x x -i_has_map_field f y x -i_has_map_field f r y -i_has_map_field f x y -i_has_map_field f y y +has_map_fields Fail Src Size Rest=* => \ + gen_has_map_fields(Fail, Src, Size, Rest) ## Transform get_map_elements(s) #{ K1 := V1, K2 := V2 } -get_map_element/4 - -get_map_elements Fail Src=rxy Size=u==2 Rest=* => gen_get_map_element(Fail,Src,Size,Rest) -get_map_elements Fail Src Size Rest=* => i_get_map_elements Fail Src Size Rest +get_map_elements Fail Src=rxy Size=u==2 Rest=* => \ + gen_get_map_element(Fail, Src, Size, Rest) +get_map_elements Fail Src Size Rest=* | map_key_sort(Size, Rest) => \ + gen_get_map_elements(Fail, Src, Size, Rest) i_get_map_elements f s I -get_map_element Fail Src=rxy Key=ax Dst => i_get_map_element Fail Src Key Dst -get_map_element Fail Src=rxy Key=rycq Dst => \ +i_get_map_element Fail Src=rxy Key=ry Dst => \ move Key x | i_get_map_element Fail Src x Dst -get_map_element Fail Src Key Dst => jump Fail + +%macro: i_get_map_element_hash GetMapElementHash -fail_action +i_get_map_element_hash f r c I r +i_get_map_element_hash f x c I r +i_get_map_element_hash f y c I r +i_get_map_element_hash f r c I x +i_get_map_element_hash f x c I x +i_get_map_element_hash f y c I x +i_get_map_element_hash f r c I y +i_get_map_element_hash f x c I y +i_get_map_element_hash f y c I y %macro: i_get_map_element GetMapElement -fail_action -i_get_map_element f r a r -i_get_map_element f x a r -i_get_map_element f y a r -i_get_map_element f r a x -i_get_map_element f x a x -i_get_map_element f y a x -i_get_map_element f r a y -i_get_map_element f x a y -i_get_map_element f y a y i_get_map_element f r x r i_get_map_element f x x r i_get_map_element f y x r @@ -1574,17 +1660,21 @@ gc_bif2 p Live u$bif:erlang:sminus/2 Reg=d Int=i Dst | \ # GCing arithmetic instructions. # +gc_bif2 Fail I u$bif:erlang:splus/2 S1=x S2=x Dst=d => i_plus Fail I S1 S2 Dst gc_bif2 Fail I u$bif:erlang:splus/2 S1 S2 Dst=d => i_fetch S1 S2 | i_plus Fail I Dst +gc_bif2 Fail I u$bif:erlang:sminus/2 S1=x S2=x Dst=d => i_minus Fail I S1 S2 Dst gc_bif2 Fail I u$bif:erlang:sminus/2 S1 S2 Dst=d => i_fetch S1 S2 | i_minus Fail I Dst gc_bif2 Fail I u$bif:erlang:stimes/2 S1 S2 Dst=d => i_fetch S1 S2 | i_times Fail I Dst gc_bif2 Fail I u$bif:erlang:div/2 S1 S2 Dst=d => i_fetch S1 S2 | i_m_div Fail I Dst gc_bif2 Fail I u$bif:erlang:intdiv/2 S1 S2 Dst=d => i_fetch S1 S2 | i_int_div Fail I Dst +gc_bif2 Fail I u$bif:erlang:rem/2 S1=x S2=x Dst=d => i_rem Fail I S1 S2 Dst gc_bif2 Fail I u$bif:erlang:rem/2 S1 S2 Dst=d => i_fetch S1 S2 | i_rem Fail I Dst gc_bif2 Fail I u$bif:erlang:bsl/2 S1 S2 Dst=d => i_fetch S1 S2 | i_bsl Fail I Dst gc_bif2 Fail I u$bif:erlang:bsr/2 S1 S2 Dst=d => i_fetch S1 S2 | i_bsr Fail I Dst +gc_bif2 Fail I u$bif:erlang:band/2 S1=x S2=c Dst=d => i_band Fail I S1 S2 Dst gc_bif2 Fail I u$bif:erlang:band/2 S1 S2 Dst=d => i_fetch S1 S2 | i_band Fail I Dst gc_bif2 Fail I u$bif:erlang:bor/2 S1 S2 Dst=d => i_fetch S1 S2 | i_bor Fail I Dst gc_bif2 Fail I u$bif:erlang:bxor/2 S1 S2 Dst=d => i_fetch S1 S2 | i_bxor Fail I Dst @@ -1598,16 +1688,20 @@ i_increment r I I d i_increment x I I d i_increment y I I d +i_plus j I x x d i_plus j I d +i_minus j I x x d i_minus j I d i_times j I d i_m_div j I d i_int_div j I d +i_rem j I x x d i_rem j I d i_bsl j I d i_bsr j I d +i_band j I x c d i_band j I d i_bor j I d i_bxor j I d diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index 4cbc4b0caf..e21e916de5 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -957,12 +957,15 @@ tail_recur: UINT32_HASH_RET(external_ref_numbers(term)[0],FUNNY_NUMBER9,FUNNY_NUMBER10); case FLOAT_DEF: { - FloatDef ff; - GET_DOUBLE(term, ff); - hash = hash*FUNNY_NUMBER6 + (ff.fw[0] ^ ff.fw[1]); - break; + FloatDef ff; + GET_DOUBLE(term, ff); + if (ff.fd == 0.0f) { + /* ensure positive 0.0 */ + ff.fd = erts_get_positive_zero_float(); + } + hash = hash*FUNNY_NUMBER6 + (ff.fw[0] ^ ff.fw[1]); + break; } - case MAKE_HASH_CDR_PRE_OP: term = (Eterm) WSTACK_POP(stack); if (is_not_list(term)) { @@ -1476,6 +1479,10 @@ make_hash2(Eterm term) { FloatDef ff; GET_DOUBLE(term, ff); + if (ff.fd == 0.0f) { + /* ensure positive 0.0 */ + ff.fd = erts_get_positive_zero_float(); + } #if defined(WORDS_BIGENDIAN) || defined(DOUBLE_MIDDLE_ENDIAN) UINT32_HASH_2(ff.fw[0], ff.fw[1], HCONST_12); #else @@ -1616,7 +1623,6 @@ make_internal_hash(Eterm term) Eterm tmp; DECLARE_ESTACK(s); - UseTmpHeapNoproc(2); hash = 0; for (;;) { switch (primary_tag(term)) { @@ -1707,6 +1713,7 @@ make_internal_hash(Eterm term) } goto pop_next; } + case HAMT_SUBTAG_HEAD_ARRAY: case HAMT_SUBTAG_HEAD_BITMAP: size = *ptr++; UINT32_HASH(size, HCONST_16); @@ -1895,10 +1902,13 @@ make_internal_hash(Eterm term) { FloatDef ff; GET_DOUBLE(term, ff); + if (ff.fd == 0.0f) { + /* ensure positive 0.0 */ + ff.fd = erts_get_positive_zero_float(); + } UINT32_HASH_2(ff.fw[0], ff.fw[1], HCONST_12); goto pop_next; } - default: erl_exit(1, "Invalid tag in make_hash2(0x%X)\n", term); } @@ -1918,7 +1928,6 @@ make_internal_hash(Eterm term) pop_next: if (ESTACK_ISEMPTY(s)) { DESTROY_ESTACK(s); - UnUseTmpHeapNoproc(2); return hash; } @@ -2080,12 +2089,15 @@ tail_recur: break; case FLOAT_DEF: { - FloatDef ff; - GET_DOUBLE(term, ff); - hash = hash*FUNNY_NUMBER6 + (ff.fw[0] ^ ff.fw[1]); + FloatDef ff; + GET_DOUBLE(term, ff); + if (ff.fd == 0.0f) { + /* ensure positive 0.0 */ + ff.fd = erts_get_positive_zero_float(); + } + hash = hash*FUNNY_NUMBER6 + (ff.fw[0] ^ ff.fw[1]); } break; - case MAKE_HASH_CDR_PRE_OP: term = (Eterm) WSTACK_POP(stack); if (is_not_list(term)) { @@ -2299,11 +2311,7 @@ static int do_send_to_logger(Eterm tag, Eterm gleader, char *buf, int len) erts_queue_error_logger_message(from, tuple3, bp); } #else - erts_queue_message(p, NULL /* only used for smp build */, bp, tuple3, NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(p, NULL /* only used for smp build */, bp, tuple3, NIL); #endif return 0; } @@ -2912,18 +2920,54 @@ static int cmp_atoms(Eterm a, Eterm b) */ Sint cmp(Eterm a, Eterm b) { - return erts_cmp(a, b, 0); + return erts_cmp(a, b, 0, 0); } #endif +#if HALFWORD_HEAP +static Sint erts_cmp_compound_rel_opt(Eterm a, Eterm* a_base, + Eterm b, Eterm* b_base, + int exact, int eq_only); +#else +static Sint erts_cmp_compound(Eterm a, Eterm b, int exact, int eq_only); +#endif + +#if HALFWORD_HEAP +Sint erts_cmp_rel_opt(Eterm a, Eterm* a_base, + Eterm b, Eterm* b_base, + int exact, int eq_only) +#else +Sint erts_cmp(Eterm a, Eterm b, int exact, int eq_only) +#endif +{ + if (is_atom(a) && is_atom(b)) { + return cmp_atoms(a, b); + } else if (is_both_small(a, b)) { + return (signed_val(a) - signed_val(b)); + } else if (is_float_rel(a, a_base) && is_float_rel(b, b_base)) { + FloatDef af, bf; + GET_DOUBLE_REL(a, af, a_base); + GET_DOUBLE_REL(b, bf, b_base); + return float_comp(af.fd, bf.fd); + } +#if HALFWORD_HEAP + return erts_cmp_compound_rel_opt(a,a_base,b,b_base,exact,eq_only); +#else + return erts_cmp_compound(a,b,exact,eq_only); +#endif +} + + /* erts_cmp(Eterm a, Eterm b, int exact) * exact = 1 -> term-based compare * exact = 0 -> arith-based compare */ #if HALFWORD_HEAP -Sint erts_cmp_rel_opt(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base, int exact) +static Sint erts_cmp_compound_rel_opt(Eterm a, Eterm* a_base, + Eterm b, Eterm* b_base, + int exact, int eq_only) #else -Sint erts_cmp(Eterm a, Eterm b, int exact) +static Sint erts_cmp_compound(Eterm a, Eterm b, int exact, int eq_only) #endif { #define PSTACK_TYPE struct erts_cmp_hashmap_state @@ -3743,7 +3787,7 @@ pop_next: return 0; not_equal: - if (!PSTACK_IS_EMPTY(hmap_stack)) { + if (!PSTACK_IS_EMPTY(hmap_stack) && !eq_only) { WSTACK_ROLLBACK(stack, PSTACK_TOP(hmap_stack)->wstack_rollback); goto pop_next; } |