diff options
Diffstat (limited to 'erts/emulator')
83 files changed, 3947 insertions, 1092 deletions
diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in index c9076a9d64..b658e79378 100644 --- a/erts/emulator/Makefile.in +++ b/erts/emulator/Makefile.in @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 1996-2010. All Rights Reserved. +# Copyright Ericsson AB 1996-2011. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in @@ -742,7 +742,7 @@ RUN_OBJS = \ $(OBJDIR)/erl_bif_re.o $(OBJDIR)/erl_unicode.o \ $(OBJDIR)/packet_parser.o $(OBJDIR)/safe_hash.o \ $(OBJDIR)/erl_zlib.o $(OBJDIR)/erl_nif.o \ - $(OBJDIR)/erl_bif_binary.o + $(OBJDIR)/erl_bif_binary.o $(OBJDIR)/erl_ao_firstfit_alloc.o ifeq ($(TARGET),win32) DRV_OBJS = \ diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index 327620772f..68d64fb7b0 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 1996-2010. All Rights Reserved. +# Copyright Ericsson AB 1996-2011. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in @@ -76,6 +76,7 @@ atom allocator_sizes atom alloc_util_allocators atom allow_passive_connect atom already_loaded +atom amd64 atom anchored atom and atom andalso diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 1ca405961f..2561d7a630 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2010. All Rights Reserved. + * Copyright Ericsson AB 1999-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -142,7 +142,7 @@ BIF_RETTYPE code_is_module_native_1(BIF_ALIST_1) if ((modp = erts_get_module(BIF_ARG_1)) == NULL) { return am_undefined; } - return (is_native(modp->code) || + return ((modp->code && is_native(modp->code)) || (modp->old_code != 0 && is_native(modp->old_code))) ? am_true : am_false; } @@ -162,6 +162,23 @@ BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3) return res; } +BIF_RETTYPE +check_old_code_1(BIF_ALIST_1) +{ + Module* modp; + + if (is_not_atom(BIF_ARG_1)) { + BIF_ERROR(BIF_P, BADARG); + } + modp = erts_get_module(BIF_ARG_1); + if (modp == NULL) { /* Doesn't exist. */ + BIF_RET(am_false); + } else if (modp->old_code == NULL) { /* No old code. */ + BIF_RET(am_false); + } + BIF_RET(am_true); +} + Eterm check_process_code_2(BIF_ALIST_2) { @@ -175,8 +192,19 @@ check_process_code_2(BIF_ALIST_2) Eterm res; if (internal_pid_index(BIF_ARG_1) >= erts_max_processes) goto error; - rp = erts_pid2proc_not_running(BIF_P, ERTS_PROC_LOCK_MAIN, - BIF_ARG_1, ERTS_PROC_LOCK_MAIN); + modp = erts_get_module(BIF_ARG_2); + if (modp == NULL) { /* Doesn't exist. */ + return am_false; + } else if (modp->old_code == NULL) { /* No old code. */ + return am_false; + } + +#ifdef ERTS_SMP + rp = erts_pid2proc_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, + BIF_ARG_1, ERTS_PROC_LOCK_MAIN); +#else + rp = erts_pid2proc(BIF_P, 0, BIF_ARG_1, 0); +#endif if (!rp) { BIF_RET(am_false); } @@ -184,11 +212,12 @@ check_process_code_2(BIF_ALIST_2) ERTS_BIF_YIELD2(bif_export[BIF_check_process_code_2], BIF_P, BIF_ARG_1, BIF_ARG_2); } - modp = erts_get_module(BIF_ARG_2); res = check_process_code(rp, modp); #ifdef ERTS_SMP - if (BIF_P != rp) + if (BIF_P != rp) { + erts_resume(rp, ERTS_PROC_LOCK_MAIN); erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MAIN); + } #endif BIF_RET(res); } @@ -406,11 +435,6 @@ check_process_code(Process* rp, Module* modp) #endif #define INSIDE(a) (start <= (a) && (a) < end) - if (modp == NULL) { /* Doesn't exist. */ - return am_false; - } else if (modp->old_code == NULL) { /* No old code. */ - return am_false; - } /* * Pick up limits for the module. diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 32ea8588d2..937b3d9e53 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -1911,13 +1911,15 @@ void process_main(void) * Note that for the halfword emulator, the two first elements * of the array are used. */ - *((BeamInstr **) (UWord) c_p->def_arg_reg) = I+3; + BeamInstr** pi = (BeamInstr**) c_p->def_arg_reg; + *pi = I+3; set_timer(c_p, unsigned_val(timeout_value)); } else if (timeout_value == am_infinity) { c_p->flags |= F_TIMO; #if !defined(ARCH_64) || HALFWORD_HEAP } else if (term_to_Uint(timeout_value, &time_val)) { - *((BeamInstr **) (UWord) c_p->def_arg_reg) = I+3; + BeamInstr** pi = (BeamInstr**) c_p->def_arg_reg; + *pi = I+3; set_timer(c_p, time_val); #endif } else { /* Wrong time */ @@ -1974,7 +1976,8 @@ void process_main(void) * we must test the F_INSLPQUEUE flag as well as the F_TIMO flag. */ if ((c_p->flags & (F_INSLPQUEUE | F_TIMO)) == 0) { - *((BeamInstr **) (UWord) c_p->def_arg_reg) = I+3; + BeamInstr** p = (BeamInstr **) c_p->def_arg_reg; + *p = I+3; set_timer(c_p, Arg(1)); } goto wait2; @@ -3558,7 +3561,7 @@ void process_main(void) * Operands: NotUsed Live Dst */ do_bs_init_bits_known: - num_bytes = (num_bits+7) >> 3; + num_bytes = ((Uint64)num_bits+(Uint64)7) >> 3; if (num_bits & 7) { alloc += ERL_SUB_BIN_SIZE; } @@ -5322,7 +5325,7 @@ void process_main(void) ep->code[3] = (BeamInstr) OpCode(apply_bif); ep->code[4] = (BeamInstr) bif_table[i].f; /* XXX: set func info for bifs */ - ((BeamInstr*)ep->code + 3)[-5] = (BeamInstr) BeamOp(op_i_func_info_IaaI); + ep->fake_op_func_info_for_hipe[0] = (BeamInstr) BeamOp(op_i_func_info_IaaI); } return; diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 57fe25453d..eb10ae59a8 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -332,20 +332,22 @@ typedef struct { Eterm* func_tab[1]; /* Pointers to each function. */ } LoadedCode; -#define GetTagAndValue(Stp, Tag, Val) \ - do { \ - BeamInstr __w; \ - GetByte(Stp, __w); \ - Tag = __w & 0x07; \ - if ((__w & 0x08) == 0) { \ - Val = __w >> 4; \ - } else if ((__w & 0x10) == 0) { \ - Val = ((__w >> 5) << 8); \ - GetByte(Stp, __w); \ - Val |= __w; \ - } else { \ - if (!get_int_val(Stp, __w, &(Val))) goto load_error; \ - } \ +#define GetTagAndValue(Stp, Tag, Val) \ + do { \ + BeamInstr __w; \ + GetByte(Stp, __w); \ + Tag = __w & 0x07; \ + if ((__w & 0x08) == 0) { \ + Val = __w >> 4; \ + } else if ((__w & 0x10) == 0) { \ + Val = ((__w >> 5) << 8); \ + GetByte(Stp, __w); \ + Val |= __w; \ + } else { \ + int __res = get_tag_and_value(Stp, __w, (Tag), &(Val)); \ + if (__res < 0) goto load_error; \ + Tag = (unsigned) __res; \ + } \ } while (0) @@ -489,8 +491,8 @@ static void load_printf(int line, LoaderState* context, char *fmt, ...); static int transform_engine(LoaderState* st); static void id_to_string(Uint id, char* s); static void new_genop(LoaderState* stp); -static int get_int_val(LoaderState* stp, Uint len_code, BeamInstr* result); -static int get_erlang_integer(LoaderState* stp, Uint len_code, BeamInstr* result); +static int get_tag_and_value(LoaderState* stp, Uint len_code, + unsigned tag, BeamInstr* result); static int new_label(LoaderState* stp); static void new_literal_patch(LoaderState* stp, int pos); static void new_string_patch(LoaderState* stp, int pos); @@ -1470,46 +1472,15 @@ load_code(LoaderState* stp) last_op->arity = 0; ASSERT(arity <= MAX_OPARGS); -#define GetValue(Stp, First, Val) \ - do { \ - if (((First) & 0x08) == 0) { \ - Val = (First) >> 4; \ - } else if (((First) & 0x10) == 0) { \ - BeamInstr __w; \ - GetByte(Stp, __w); \ - Val = (((First) >> 5) << 8) | __w; \ - } else { \ - if (!get_int_val(Stp, (First), &(Val))) goto load_error; \ - } \ - } while (0) - for (arg = 0; arg < arity; arg++) { - BeamInstr first; - - GetByte(stp, first); - last_op->a[arg].type = first & 0x07; + GetTagAndValue(stp, last_op->a[arg].type, last_op->a[arg].val); switch (last_op->a[arg].type) { case TAG_i: - if ((first & 0x08) == 0) { - last_op->a[arg].val = first >> 4; - } else if ((first & 0x10) == 0) { - BeamInstr w; - GetByte(stp, w); - ASSERT(first < 0x800); - last_op->a[arg].val = ((first >> 5) << 8) | w; - } else { - int i = get_erlang_integer(stp, first, &(last_op->a[arg].val)); - if (i < 0) { - goto load_error; - } - last_op->a[arg].type = i; - } - break; case TAG_u: - GetValue(stp, first, last_op->a[arg].val); + case TAG_q: + case TAG_o: break; case TAG_x: - GetValue(stp, first, last_op->a[arg].val); if (last_op->a[arg].val == 0) { last_op->a[arg].type = TAG_r; } else if (last_op->a[arg].val >= MAX_REG) { @@ -1518,7 +1489,6 @@ load_code(LoaderState* stp) } break; case TAG_y: - GetValue(stp, first, last_op->a[arg].val); if (last_op->a[arg].val >= MAX_REG) { LoadError1(stp, "invalid y register number: %u", last_op->a[arg].val); @@ -1526,7 +1496,6 @@ load_code(LoaderState* stp) last_op->a[arg].val += CP_SIZE; break; case TAG_a: - GetValue(stp, first, last_op->a[arg].val); if (last_op->a[arg].val == 0) { last_op->a[arg].type = TAG_n; } else if (last_op->a[arg].val >= stp->num_atoms) { @@ -1536,7 +1505,6 @@ load_code(LoaderState* stp) } break; case TAG_f: - GetValue(stp, first, last_op->a[arg].val); if (last_op->a[arg].val == 0) { last_op->a[arg].type = TAG_p; } else if (last_op->a[arg].val >= stp->num_labels) { @@ -1544,7 +1512,6 @@ load_code(LoaderState* stp) } break; case TAG_h: - GetValue(stp, first, last_op->a[arg].val); if (last_op->a[arg].val > 65535) { LoadError1(stp, "invalid range for character data type: %u", last_op->a[arg].val); @@ -1552,11 +1519,9 @@ load_code(LoaderState* stp) break; case TAG_z: { - BeamInstr ext_tag; unsigned tag; - GetValue(stp, first, ext_tag); - switch (ext_tag) { + switch (last_op->a[arg].val) { case 0: /* Floating point number */ { Eterm* hp; @@ -1648,7 +1613,8 @@ load_code(LoaderState* stp) break; } default: - LoadError1(stp, "invalid extended tag %d", ext_tag); + LoadError1(stp, "invalid extended tag %d", + last_op->a[arg].val); break; } } @@ -1659,7 +1625,6 @@ load_code(LoaderState* stp) } last_op->arity++; } -#undef GetValue ASSERT(arity == last_op->arity); @@ -2562,13 +2527,8 @@ should_gen_heap_bin(LoaderState* stp, GenOpArg Src) static int binary_too_big(LoaderState* stp, GenOpArg Size) { - return Size.type == TAG_u && ((Size.val >> (8*sizeof(Uint)-3)) != 0); -} - -static int -binary_too_big_bits(LoaderState* stp, GenOpArg Size) -{ - return Size.type == TAG_u && (((Size.val+7)/8) >> (8*sizeof(Uint)-3) != 0); + return Size.type == TAG_o || + (Size.type == TAG_u && ((Size.val >> (8*sizeof(Uint)-3)) != 0)); } static GenOp* @@ -4317,41 +4277,9 @@ load_printf(int line, LoaderState* context, char *fmt,...) erts_send_error_to_logger(context->group_leader, dsbufp); } - -static int -get_int_val(LoaderState* stp, Uint len_code, BeamInstr* result) -{ - Uint count; - Uint val; - - len_code >>= 5; - ASSERT(len_code < 8); - if (len_code == 7) { - LoadError0(stp, "can't load integers bigger than 8 bytes yet\n"); - } - count = len_code + 2; - if (count == 5) { - Uint msb; - GetByte(stp, msb); - if (msb == 0) { - count--; - } - GetInt(stp, 4, *result); - } else if (count <= 4) { - GetInt(stp, count, val); - *result = ((val << 8*(sizeof(val)-count)) >> 8*(sizeof(val)-count)); - } else { - LoadError1(stp, "too big integer; %d bytes\n", count); - } - return 1; - - load_error: - return 0; -} - - static int -get_erlang_integer(LoaderState* stp, Uint len_code, BeamInstr* result) +get_tag_and_value(LoaderState* stp, Uint len_code, + unsigned tag, BeamInstr* result) { Uint count; Sint val; @@ -4371,17 +4299,62 @@ get_erlang_integer(LoaderState* stp, Uint len_code, BeamInstr* result) if (len_code < 7) { count = len_code + 2; } else { - Uint tag; + unsigned sztag; UWord len_word; ASSERT(len_code == 7); - GetTagAndValue(stp, tag, len_word); - VerifyTag(stp, TAG_u, tag); + GetTagAndValue(stp, sztag, len_word); + VerifyTag(stp, sztag, TAG_u); count = len_word + 9; } /* - * Handle values up to the size of an int, meaning either a small or bignum. + * The value for tags except TAG_i must be an unsigned integer + * fitting in an Uint. If it does not fit, we'll indicate overflow + * by changing the tag to TAG_o. + */ + + if (tag != TAG_i) { + if (count == sizeof(Uint)+1) { + Uint msb; + + /* + * The encoded value has one more byte than an Uint. + * It will still fit in an Uint if the most significant + * byte is 0. + */ + GetByte(stp, msb); + GetInt(stp, sizeof(Uint), *result); + if (msb != 0) { + /* Overflow: Negative or too big. */ + return TAG_o; + } + } else if (count == sizeof(Uint)) { + /* + * The value must be positive (or the encoded value would + * have been one byte longer). + */ + GetInt(stp, count, *result); + } else if (count < sizeof(Uint)) { + GetInt(stp, count, *result); + + /* + * If the sign bit is set, the value is negative + * (not allowed). + */ + if (*result & ((Uint)1 << (count*8-1))) { + return TAG_o; + } + } else { + GetInt(stp, count, *result); + return TAG_o; + } + return tag; + } + + /* + * TAG_i: First handle values up to the size of an Uint (i.e. either + * a small or a bignum). */ if (count <= sizeof(val)) { diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index b3325d635b..68b3350d7f 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -2168,20 +2168,146 @@ BIF_RETTYPE tl_1(BIF_ALIST_1) /**********************************************************************/ /* return the size of an I/O list */ -BIF_RETTYPE iolist_size_1(BIF_ALIST_1) +static Eterm +accumulate(Eterm acc, Uint size) { - Sint size = io_list_len(BIF_ARG_1); + if (is_non_value(acc)) { + /* + * There is no pre-existing accumulator. Allocate a + * bignum buffer with one extra word to be used if + * the bignum grows in the future. + */ + Eterm* hp = (Eterm *) erts_alloc(ERTS_ALC_T_TEMP_TERM, + (BIG_UINT_HEAP_SIZE+1) * + sizeof(Eterm)); + return uint_to_big(size, hp); + } else { + Eterm* big; + int need_heap; - if (size == -1) { - BIF_ERROR(BIF_P, BADARG); - } else if (IS_USMALL(0, (Uint) size)) { - BIF_RET(make_small(size)); + /* + * Add 'size' to 'acc' in place. There is always one + * extra word allocated in case the bignum grows by one word. + */ + big = big_val(acc); + need_heap = BIG_NEED_SIZE(BIG_SIZE(big)); + acc = big_plus_small(acc, size, big); + if (BIG_NEED_SIZE(big_size(acc)) > need_heap) { + /* + * The extra word has been consumed. Grow the + * allocation by one word. + */ + big = (Eterm *) erts_realloc(ERTS_ALC_T_TEMP_TERM, + big_val(acc), + (need_heap+1) * sizeof(Eterm)); + acc = make_big(big); + } + return acc; + } +} + +static Eterm +consolidate(Process* p, Eterm acc, Uint size) +{ + Eterm* hp; + + if (is_non_value(acc)) { + return erts_make_integer(size, p); } else { - Eterm* hp = HAlloc(BIF_P, BIG_UINT_HEAP_SIZE); - BIF_RET(uint_to_big(size, hp)); + Eterm* big; + Uint sz; + Eterm res; + + acc = accumulate(acc, size); + big = big_val(acc); + sz = BIG_NEED_SIZE(BIG_SIZE(big)); + hp = HAlloc(p, sz); + res = make_big(hp); + while (sz--) { + *hp++ = *big++; + } + erts_free(ERTS_ALC_T_TEMP_TERM, (void *) big_val(acc)); + return res; } } +BIF_RETTYPE iolist_size_1(BIF_ALIST_1) +{ + Eterm obj, hd; + Eterm* objp; + Uint size = 0; + Uint cur_size; + Uint new_size; + Eterm acc = THE_NON_VALUE; + DECLARE_ESTACK(s); + + obj = BIF_ARG_1; + goto L_again; + + while (!ESTACK_ISEMPTY(s)) { + obj = ESTACK_POP(s); + L_again: + if (is_list(obj)) { + L_iter_list: + objp = list_val(obj); + hd = CAR(objp); + obj = CDR(objp); + /* Head */ + if (is_byte(hd)) { + size++; + if (size == 0) { + acc = accumulate(acc, (Uint) -1); + size = 1; + } + } else if (is_binary(hd) && binary_bitsize(hd) == 0) { + cur_size = binary_size(hd); + if ((new_size = size + cur_size) >= size) { + size = new_size; + } else { + acc = accumulate(acc, size); + size = cur_size; + } + } else if (is_list(hd)) { + ESTACK_PUSH(s, obj); + obj = hd; + goto L_iter_list; + } else if (is_not_nil(hd)) { + goto L_type_error; + } + /* Tail */ + if (is_list(obj)) { + goto L_iter_list; + } else if (is_binary(obj) && binary_bitsize(obj) == 0) { + cur_size = binary_size(obj); + if ((new_size = size + cur_size) >= size) { + size = new_size; + } else { + acc = accumulate(acc, size); + size = cur_size; + } + } else if (is_not_nil(obj)) { + goto L_type_error; + } + } else if (is_binary(obj) && binary_bitsize(obj) == 0) { + cur_size = binary_size(obj); + if ((new_size = size + cur_size) >= size) { + size = new_size; + } else { + acc = accumulate(acc, size); + size = cur_size; + } + } else if (is_not_nil(obj)) { + goto L_type_error; + } + } + + DESTROY_ESTACK(s); + BIF_RET(consolidate(BIF_P, acc, size)); + + L_type_error: + DESTROY_ESTACK(s); + BIF_ERROR(BIF_P, BADARG); +} /**********************************************************************/ @@ -3215,20 +3341,32 @@ BIF_RETTYPE garbage_collect_1(BIF_ALIST_1) BIF_ERROR(BIF_P, BADARG); } - rp = erts_pid2proc_not_running(BIF_P, ERTS_PROC_LOCK_MAIN, + if (BIF_P->id == BIF_ARG_1) + rp = BIF_P; + else { +#ifdef ERTS_SMP + rp = erts_pid2proc_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, BIF_ARG_1, ERTS_PROC_LOCK_MAIN); - if (!rp) - BIF_RET(am_false); - if (rp == ERTS_PROC_LOCK_BUSY) - ERTS_BIF_YIELD1(bif_export[BIF_garbage_collect_1], BIF_P, BIF_ARG_1); + if (rp == ERTS_PROC_LOCK_BUSY) + ERTS_BIF_YIELD1(bif_export[BIF_garbage_collect_1], BIF_P, BIF_ARG_1); +#else + rp = erts_pid2proc(BIF_P, 0, BIF_ARG_1, 0); +#endif + if (!rp) + BIF_RET(am_false); + } /* The GC cost is taken for the process executing this BIF. */ FLAGS(rp) |= F_NEED_FULLSWEEP; reds = erts_garbage_collect(rp, 0, rp->arg_reg, rp->arity); - if (BIF_P != rp) +#ifdef ERTS_SMP + if (BIF_P != rp) { + erts_resume(rp, ERTS_PROC_LOCK_MAIN); erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MAIN); + } +#endif BIF_RET2(am_true, reds); } @@ -3270,6 +3408,7 @@ BIF_RETTYPE ports_0(BIF_ALIST_0) Eterm* dead_ports; int alive, dead; Uint32 next_ss; + int i; /* To get a consistent snapshot... * We add alive ports from start of the buffer @@ -3281,21 +3420,18 @@ BIF_RETTYPE ports_0(BIF_ALIST_0) erts_smp_atomic_set(&erts_dead_ports_ptr, (erts_aint_t) (port_buf + erts_max_ports)); - next_ss = erts_smp_atomic_inctest(&erts_ports_snapshot); + next_ss = erts_smp_atomic32_inctest(&erts_ports_snapshot); - if (erts_smp_atomic_read(&erts_ports_alive) > 0) { - erts_aint_t i; - for (i = erts_max_ports-1; i >= 0; i--) { - Port* prt = &erts_port[i]; - erts_smp_port_state_lock(prt); - if (!(prt->status & ERTS_PORT_SFLGS_DEAD) - && prt->snapshot != next_ss) { - ASSERT(prt->snapshot == next_ss - 1); - *pp++ = prt->id; - prt->snapshot = next_ss; /* Consumed by this snapshot */ - } - erts_smp_port_state_unlock(prt); + for (i = erts_max_ports-1; i >= 0; i--) { + Port* prt = &erts_port[i]; + erts_smp_port_state_lock(prt); + if (!(prt->status & ERTS_PORT_SFLGS_DEAD) + && prt->snapshot != next_ss) { + ASSERT(prt->snapshot == next_ss - 1); + *pp++ = prt->id; + prt->snapshot = next_ss; /* Consumed by this snapshot */ } + erts_smp_port_state_unlock(prt); } dead_ports = (Eterm*)erts_smp_atomic_xchg(&erts_dead_ports_ptr, diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index d9dd80fa8b..831c0b1ce6 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -87,6 +87,8 @@ bif erlang:exit/2 bif 'erl.lang.proc':signal/2 ebif_signal_2 exit_2 bif erlang:external_size/1 bif 'erl.lang.term':external_size/1 ebif_external_size_1 +bif erlang:external_size/2 +bif 'erl.lang.term':external_size/2 ebif_external_size_2 ubif erlang:float/1 ubif 'erl.lang.number':to_float/1 ebif_to_float_1 float_1 bif erlang:float_to_list/1 @@ -802,6 +804,12 @@ bif prim_file:internal_name2native/1 bif prim_file:internal_native2name/1 bif prim_file:internal_normalize_utf8/1 bif file:native_name_encoding/0 + +# +# New in R14B04. +# +bif erlang:check_old_code/1 + # # Obsolete # diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c index f47f5a9c0c..d18de9ae5d 100644 --- a/erts/emulator/beam/big.c +++ b/erts/emulator/beam/big.c @@ -1588,7 +1588,7 @@ big_to_double(Wterm x, double* resp) /* ** Estimate the number of decimal digits (include sign) */ -int big_decimal_estimate(Eterm x) +int big_decimal_estimate(Wterm x) { Eterm* xp = big_val(x); int lg = I_lg(BIG_V(xp), BIG_SIZE(xp)); @@ -1602,7 +1602,7 @@ int big_decimal_estimate(Eterm x) ** Convert a bignum into a string of decimal numbers */ -static void write_big(Eterm x, void (*write_func)(void *, char), void *arg) +static void write_big(Wterm x, void (*write_func)(void *, char), void *arg) { Eterm* xp = big_val(x); ErtsDigit* dx = BIG_V(xp); @@ -1681,7 +1681,7 @@ write_string(void *arg, char c) *(--(*((char **) arg))) = c; } -char *erts_big_to_string(Eterm x, char *buf, Uint buf_sz) +char *erts_big_to_string(Wterm x, char *buf, Uint buf_sz) { char *big_str = buf + buf_sz - 1; *big_str = '\0'; diff --git a/erts/emulator/beam/big.h b/erts/emulator/beam/big.h index f28a390aea..2afc37004f 100644 --- a/erts/emulator/beam/big.h +++ b/erts/emulator/beam/big.h @@ -114,9 +114,9 @@ typedef Uint dsize_t; /* Vector size type */ #endif -int big_decimal_estimate(Eterm); +int big_decimal_estimate(Wterm); Eterm erts_big_to_list(Eterm, Eterm**); -char *erts_big_to_string(Eterm x, char *buf, Uint buf_sz); +char *erts_big_to_string(Wterm x, char *buf, Uint buf_sz); Eterm small_times(Sint, Sint, Eterm*); diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c index 9486602633..1fb39c6c67 100644 --- a/erts/emulator/beam/binary.c +++ b/erts/emulator/beam/binary.c @@ -32,11 +32,11 @@ #include "erl_bits.h" #ifdef DEBUG -static int list_to_bitstr_buf(Eterm obj, char* buf, int len); +static int list_to_bitstr_buf(Eterm obj, char* buf, Uint len); #else static int list_to_bitstr_buf(Eterm obj, char* buf); #endif -static Sint bitstr_list_len(Eterm obj); +static int bitstr_list_len(Eterm obj, Uint* num_bytes); void erts_init_binary(void) @@ -355,21 +355,24 @@ BIF_RETTYPE bitstring_to_list_1(BIF_ALIST_1) BIF_RETTYPE erts_list_to_binary_bif(Process *p, Eterm arg) { Eterm bin; - int i; + Uint size; int offset; byte* bytes; + if (is_nil(arg)) { BIF_RET(new_binary(p,(byte*)"",0)); } if (is_not_list(arg)) { goto error; } - if ((i = io_list_len(arg)) < 0) { - goto error; + switch (erts_iolist_size(arg, &size)) { + case ERTS_IOLIST_OVERFLOW: BIF_ERROR(p, SYSTEM_LIMIT); + case ERTS_IOLIST_TYPE: goto error; + default: ; } - bin = new_binary(p, (byte *)NULL, i); + bin = new_binary(p, (byte *)NULL, size); bytes = binary_bytes(bin); - offset = io_list_to_buf(arg, (char*) bytes, i); + offset = io_list_to_buf(arg, (char*) bytes, size); ASSERT(offset == 0); BIF_RET(bin); @@ -396,7 +399,8 @@ BIF_RETTYPE iolist_to_binary_1(BIF_ALIST_1) BIF_RETTYPE list_to_bitstring_1(BIF_ALIST_1) { Eterm bin; - int i,offset; + Uint sz; + int offset; byte* bytes; ErlSubBin* sb1; Eterm* hp; @@ -405,15 +409,19 @@ BIF_RETTYPE list_to_bitstring_1(BIF_ALIST_1) BIF_RET(new_binary(BIF_P,(byte*)"",0)); } if (is_not_list(BIF_ARG_1)) { - goto error; + error: + BIF_ERROR(BIF_P, BADARG); } - if ((i = bitstr_list_len(BIF_ARG_1)) < 0) { + switch (bitstr_list_len(BIF_ARG_1, &sz)) { + case ERTS_IOLIST_TYPE: goto error; + case ERTS_IOLIST_OVERFLOW: + BIF_ERROR(BIF_P, SYSTEM_LIMIT); } - bin = new_binary(BIF_P, (byte *)NULL, i); + bin = new_binary(BIF_P, (byte *)NULL, sz); bytes = binary_bytes(bin); #ifdef DEBUG - offset = list_to_bitstr_buf(BIF_ARG_1, (char*) bytes, i); + offset = list_to_bitstr_buf(BIF_ARG_1, (char*) bytes, sz); #else offset = list_to_bitstr_buf(BIF_ARG_1, (char*) bytes); #endif @@ -422,20 +430,16 @@ BIF_RETTYPE list_to_bitstring_1(BIF_ALIST_1) hp = HAlloc(BIF_P, ERL_SUB_BIN_SIZE); sb1 = (ErlSubBin *) hp; sb1->thing_word = HEADER_SUB_BIN; - sb1->size = i-1; + sb1->size = sz-1; sb1->offs = 0; sb1->orig = bin; sb1->bitoffs = 0; sb1->bitsize = offset; sb1->is_writable = 0; - hp += ERL_SUB_BIN_SIZE; bin = make_binary(sb1); } BIF_RET(bin); - - error: - BIF_ERROR(BIF_P, BADARG); } BIF_RETTYPE split_binary_2(BIF_ALIST_2) @@ -499,7 +503,7 @@ BIF_RETTYPE split_binary_2(BIF_ALIST_2) */ static int #ifdef DEBUG -list_to_bitstr_buf(Eterm obj, char* buf, int len) +list_to_bitstr_buf(Eterm obj, char* buf, Uint len) #else list_to_bitstr_buf(Eterm obj, char* buf) #endif @@ -602,8 +606,8 @@ list_to_bitstr_buf(Eterm obj, char* buf) return offset; } -static Sint -bitstr_list_len(Eterm obj) +static int +bitstr_list_len(Eterm obj, Uint* num_bytes) { Eterm* objp; Uint len = 0; @@ -611,6 +615,26 @@ bitstr_list_len(Eterm obj) DECLARE_ESTACK(s); goto L_again; +#define SAFE_ADD(Var, Val) \ + do { \ + Uint valvar = (Val); \ + Var += valvar; \ + if (Var < valvar) { \ + goto L_overflow_error; \ + } \ + } while (0) + +#define SAFE_ADD_BITSIZE(Var, Bin) \ + do { \ + if (*binary_val(Bin) == HEADER_SUB_BIN) { \ + Uint valvar = ((ErlSubBin *) binary_val(Bin))->bitsize; \ + Var += valvar; \ + if (Var < valvar) { \ + goto L_overflow_error; \ + } \ + } \ + } while (0) + while (!ESTACK_ISEMPTY(s)) { obj = ESTACK_POP(s); L_again: @@ -621,9 +645,12 @@ bitstr_list_len(Eterm obj) obj = CAR(objp); if (is_byte(obj)) { len++; + if (len == 0) { + goto L_overflow_error; + } } else if (is_binary(obj)) { - len += binary_size(obj); - offs += binary_bitsize(obj); + SAFE_ADD(len, binary_size(obj)); + SAFE_ADD_BITSIZE(offs, obj); } else if (is_list(obj)) { ESTACK_PUSH(s, CDR(objp)); goto L_iter_list; /* on head */ @@ -635,24 +662,44 @@ bitstr_list_len(Eterm obj) if (is_list(obj)) goto L_iter_list; /* on tail */ else if (is_binary(obj)) { - len += binary_size(obj); - offs += binary_bitsize(obj); + SAFE_ADD(len, binary_size(obj)); + SAFE_ADD_BITSIZE(offs, obj); } else if (is_not_nil(obj)) { goto L_type_error; } } else if (is_binary(obj)) { - len += binary_size(obj); - offs += binary_bitsize(obj); + SAFE_ADD(len, binary_size(obj)); + SAFE_ADD_BITSIZE(offs, obj); } else if (is_not_nil(obj)) { goto L_type_error; } } +#undef SAFE_ADD +#undef SAFE_ADD_BITSIZE DESTROY_ESTACK(s); - return (Sint) (len + (offs/8) + ((offs % 8) != 0)); + + /* + * Make sure that the number of bits in the bitstring will fit + * in an Uint to ensure that the binary can be matched using + * the binary syntax. + */ + if (len << 3 < len) { + goto L_overflow_error; + } + len += (offs >> 3) + ((offs & 7) != 0); + if (len << 3 < len) { + goto L_overflow_error; + } + *num_bytes = len; + return ERTS_IOLIST_OK; L_type_error: DESTROY_ESTACK(s); - return (Sint) -1; + return ERTS_IOLIST_TYPE; + + L_overflow_error: + DESTROY_ESTACK(s); + return ERTS_IOLIST_OVERFLOW; } diff --git a/erts/emulator/beam/erl_afit_alloc.c b/erts/emulator/beam/erl_afit_alloc.c index e8b594bb47..d397cd8848 100644 --- a/erts/emulator/beam/erl_afit_alloc.c +++ b/erts/emulator/beam/erl_afit_alloc.c @@ -43,9 +43,9 @@ /* Prototypes of callback functions */ static Block_t * get_free_block (Allctr_t *, Uint, - Block_t *, Uint); -static void link_free_block (Allctr_t *, Block_t *); -static void unlink_free_block (Allctr_t *, Block_t *); + Block_t *, Uint, Uint32); +static void link_free_block (Allctr_t *, Block_t *, Uint32); +static void unlink_free_block (Allctr_t *, Block_t *, Uint32); static Eterm info_options (Allctr_t *, char *, int *, @@ -72,6 +72,8 @@ erts_afalc_start(AFAllctr_t *afallctr, is a struct). */ Allctr_t *allctr = (Allctr_t *) afallctr; + init->sbmbct = 0; /* Small mbc not supported by afit */ + sys_memcpy((void *) afallctr, (void *) &nulled_state, sizeof(AFAllctr_t)); allctr->mbc_header_size = sizeof(Carrier_t); @@ -105,7 +107,8 @@ erts_afalc_start(AFAllctr_t *afallctr, } static Block_t * -get_free_block(Allctr_t *allctr, Uint size, Block_t *cand_blk, Uint cand_size) +get_free_block(Allctr_t *allctr, Uint size, Block_t *cand_blk, Uint cand_size, + Uint32 flags) { AFAllctr_t *afallctr = (AFAllctr_t *) allctr; @@ -123,7 +126,7 @@ get_free_block(Allctr_t *allctr, Uint size, Block_t *cand_blk, Uint cand_size) } static void -link_free_block(Allctr_t *allctr, Block_t *block) +link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) { AFFreeBlock_t *blk = (AFFreeBlock_t *) block; AFAllctr_t *afallctr = (AFAllctr_t *) allctr; @@ -144,7 +147,7 @@ link_free_block(Allctr_t *allctr, Block_t *block) } static void -unlink_free_block(Allctr_t *allctr, Block_t *block) +unlink_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) { AFFreeBlock_t *blk = (AFFreeBlock_t *) block; AFAllctr_t *afallctr = (AFAllctr_t *) allctr; diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index 673eac7fea..bbc8a445a7 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -50,6 +50,9 @@ #include "erl_bestfit_alloc.h" #define GET_ERL_AF_ALLOC_IMPL #include "erl_afit_alloc.h" +#define GET_ERL_AOFF_ALLOC_IMPL +#include "erl_ao_firstfit_alloc.h" + #define ERTS_ALC_DEFAULT_MAX_THR_PREF 16 @@ -85,11 +88,19 @@ typedef union { char align_bfa[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(BFAllctr_t))]; AFAllctr_t afa; char align_afa[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(AFAllctr_t))]; + AOFFAllctr_t aoffa; + char align_aoffa[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(AOFFAllctr_t))]; } ErtsAllocatorState_t; -static ErtsAllocatorState_t sl_alloc_state; +static ErtsAllocatorState_t sbmbc_alloc_state; static ErtsAllocatorState_t std_alloc_state; static ErtsAllocatorState_t ll_alloc_state; +#if HALFWORD_HEAP +static ErtsAllocatorState_t sbmbc_low_alloc_state; +static ErtsAllocatorState_t std_low_alloc_state; +static ErtsAllocatorState_t ll_low_alloc_state; +#endif +static ErtsAllocatorState_t sl_alloc_state; static ErtsAllocatorState_t temp_alloc_state; static ErtsAllocatorState_t eheap_alloc_state; static ErtsAllocatorState_t binary_alloc_state; @@ -116,7 +127,8 @@ static void *fix_core_alloc(Uint size) enum allctr_type { GOODFIT, BESTFIT, - AFIT + AFIT, + AOFIRSTFIT }; struct au_init { @@ -128,6 +140,7 @@ struct au_init { GFAllctrInit_t gf; BFAllctrInit_t bf; AFAllctrInit_t af; + AOFFAllctrInit_t aoff; } init; struct { int mmbcs; @@ -141,7 +154,8 @@ struct au_init { ERTS_DEFAULT_ALLCTR_INIT, \ ERTS_DEFAULT_GF_ALLCTR_INIT, \ ERTS_DEFAULT_BF_ALLCTR_INIT, \ - ERTS_DEFAULT_AF_ALLCTR_INIT \ + ERTS_DEFAULT_AF_ALLCTR_INIT, \ + ERTS_DEFAULT_AOFF_ALLCTR_INIT \ } typedef struct { @@ -158,6 +172,7 @@ typedef struct { char *mtrace; char *nodename; } instr; + struct au_init sbmbc_alloc; struct au_init sl_alloc; struct au_init std_alloc; struct au_init ll_alloc; @@ -166,6 +181,11 @@ typedef struct { struct au_init binary_alloc; struct au_init ets_alloc; struct au_init driver_alloc; +#if HALFWORD_HEAP + struct au_init sbmbc_low_alloc; + struct au_init std_low_alloc; + struct au_init ll_low_alloc; +#endif } erts_alc_hndl_args_init_t; #define ERTS_AU_INIT__ {0, 0, GOODFIT, DEFAULT_ALLCTR_INIT, {1,1,1,1}} @@ -177,6 +197,34 @@ do { \ } while (0) static void +set_default_sbmbc_alloc_opts(struct au_init *ip) +{ + SET_DEFAULT_ALLOC_OPTS(ip); + ip->enable = 0; + ip->thr_spec = 0; + ip->atype = BESTFIT; + ip->init.bf.ao = 1; + ip->init.util.ramv = 0; + ip->init.util.mmsbc = 0; + ip->init.util.mmmbc = 500; + ip->init.util.sbct = ~((UWord) 0); + ip->init.util.name_prefix = "sbmbc_"; + ip->init.util.alloc_no = ERTS_ALC_A_SBMBC; +#ifndef SMALL_MEMORY + ip->init.util.mmbcs = 2*1024*1024; /* Main carrier size */ +#else + ip->init.util.mmbcs = 1*1024*1024; /* Main carrier size */ +#endif + ip->init.util.ts = ERTS_ALC_MTA_SBMBC; + ip->init.util.asbcst = 0; + ip->init.util.rsbcst = 0; + ip->init.util.rsbcmt = 0; + ip->init.util.rmbcmt = 0; + ip->init.util.sbmbct = 0; + ip->init.util.sbmbcs = 0; +} + +static void set_default_sl_alloc_opts(struct au_init *ip) { SET_DEFAULT_ALLOC_OPTS(ip); @@ -193,6 +241,11 @@ set_default_sl_alloc_opts(struct au_init *ip) #endif ip->init.util.ts = ERTS_ALC_MTA_SHORT_LIVED; ip->init.util.rsbcst = 80; +#if HALFWORD_HEAP + ip->init.util.force = 1; + ip->init.util.low_mem = 1; +#endif + } static void @@ -237,6 +290,8 @@ set_default_ll_alloc_opts(struct au_init *ip) ip->init.util.rsbcst = 0; ip->init.util.rsbcmt = 0; ip->init.util.rmbcmt = 0; + ip->init.util.sbmbct = 0; + ip->init.util.sbmbcs = 0; } static void @@ -256,6 +311,10 @@ set_default_temp_alloc_opts(struct au_init *ip) ip->init.util.ts = ERTS_ALC_MTA_TEMPORARY; ip->init.util.rsbcst = 90; ip->init.util.rmbcmt = 100; +#if HALFWORD_HEAP + ip->init.util.force = 1; + ip->init.util.low_mem = 1; +#endif } static void @@ -275,6 +334,10 @@ set_default_eheap_alloc_opts(struct au_init *ip) #endif ip->init.util.ts = ERTS_ALC_MTA_EHEAP; ip->init.util.rsbcst = 50; +#if HALFWORD_HEAP + ip->init.util.force = 1; + ip->init.util.low_mem = 1; +#endif } static void @@ -418,10 +481,13 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) hdbg_init(); #endif + erts_have_sbmbc_alloc = 0; + erts_sys_alloc_init(); init_thr_ix(erts_no_schedulers); erts_init_utils_mem(); + set_default_sbmbc_alloc_opts(&init.sbmbc_alloc); set_default_sl_alloc_opts(&init.sl_alloc); set_default_std_alloc_opts(&init.std_alloc); set_default_ll_alloc_opts(&init.ll_alloc); @@ -435,6 +501,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) handle_args(argc, argv, &init); if (erts_no_schedulers <= 1) { + init.sbmbc_alloc.thr_spec = 0; init.sl_alloc.thr_spec = 0; init.std_alloc.thr_spec = 0; init.ll_alloc.thr_spec = 0; @@ -446,6 +513,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) if (init.erts_alloc_config) { /* Adjust flags that erts_alloc_config won't like */ + init.sbmbc_alloc.thr_spec = 0; init.temp_alloc.thr_spec = 0; init.sl_alloc.thr_spec = 0; init.std_alloc.thr_spec = 0; @@ -462,6 +530,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) init.temp_alloc.thr_spec = erts_no_schedulers; /* Others must use thread preferred interface */ + adjust_tpref(&init.sbmbc_alloc, erts_no_schedulers); adjust_tpref(&init.sl_alloc, erts_no_schedulers); adjust_tpref(&init.std_alloc, erts_no_schedulers); adjust_tpref(&init.ll_alloc, erts_no_schedulers); @@ -479,6 +548,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) * The following allocators cannot be run with afit strategy. * Make sure they don't... */ + refuse_af_strategy(&init.sbmbc_alloc); refuse_af_strategy(&init.sl_alloc); refuse_af_strategy(&init.std_alloc); refuse_af_strategy(&init.ll_alloc); @@ -500,6 +570,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) erts_afalc_init(); erts_bfalc_init(); erts_gfalc_init(); + erts_aoffalc_init(); for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) { erts_allctrs[i].alloc = NULL; @@ -531,7 +602,32 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) erts_allctrs[ERTS_ALC_A_SYSTEM].free = erts_sys_free; erts_allctrs_info[ERTS_ALC_A_SYSTEM].enabled = 1; +#if HALFWORD_HEAP + /* Init low memory variants by cloning */ + init.sbmbc_low_alloc = init.sbmbc_alloc; + init.sbmbc_low_alloc.init.util.name_prefix = "sbmbc_low_"; + init.sbmbc_low_alloc.init.util.alloc_no = ERTS_ALC_A_SBMBC_LOW; + init.sbmbc_low_alloc.init.util.low_mem = 1; + + init.std_low_alloc = init.std_alloc; + init.std_low_alloc.init.util.name_prefix = "std_low_"; + init.std_low_alloc.init.util.alloc_no = ERTS_ALC_A_STANDARD_LOW; + init.std_low_alloc.init.util.force = 1; + init.std_low_alloc.init.util.low_mem = 1; + + init.ll_low_alloc = init.ll_alloc; + init.ll_low_alloc.init.util.name_prefix = "ll_low_"; + init.ll_low_alloc.init.util.alloc_no = ERTS_ALC_A_LONG_LIVED_LOW; + init.ll_low_alloc.init.util.force = 1; + init.ll_low_alloc.init.util.low_mem = 1; + + set_au_allocator(ERTS_ALC_A_SBMBC_LOW, &init.sbmbc_low_alloc); + set_au_allocator(ERTS_ALC_A_STANDARD_LOW, &init.std_low_alloc); + set_au_allocator(ERTS_ALC_A_LONG_LIVED_LOW, &init.ll_low_alloc); +#endif /* HALFWORD */ + set_au_allocator(ERTS_ALC_A_TEMPORARY, &init.temp_alloc); + set_au_allocator(ERTS_ALC_A_SBMBC, &init.sbmbc_alloc); set_au_allocator(ERTS_ALC_A_SHORT_LIVED, &init.sl_alloc); set_au_allocator(ERTS_ALC_A_STANDARD, &init.std_alloc); set_au_allocator(ERTS_ALC_A_LONG_LIVED, &init.ll_alloc); @@ -561,6 +657,20 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) erts_mtrace_init(init.instr.mtrace, init.instr.nodename); + /* sbmbc_alloc() needs to be started first */ + start_au_allocator(ERTS_ALC_A_SBMBC, + &init.sbmbc_alloc, + &sbmbc_alloc_state); +#if HALFWORD_HEAP + start_au_allocator(ERTS_ALC_A_SBMBC_LOW, + &init.sbmbc_low_alloc, + &sbmbc_low_alloc_state); + erts_have_sbmbc_alloc = (init.sbmbc_alloc.enable + && init.sbmbc_low_alloc.enable); +#else + erts_have_sbmbc_alloc = init.sbmbc_alloc.enable; +#endif + start_au_allocator(ERTS_ALC_A_TEMPORARY, &init.temp_alloc, &temp_alloc_state); @@ -576,7 +686,14 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) start_au_allocator(ERTS_ALC_A_LONG_LIVED, &init.ll_alloc, &ll_alloc_state); - +#if HALFWORD_HEAP + start_au_allocator(ERTS_ALC_A_LONG_LIVED_LOW, + &init.ll_low_alloc, + &ll_low_alloc_state); + start_au_allocator(ERTS_ALC_A_STANDARD_LOW, + &init.std_low_alloc, + &std_low_alloc_state); +#endif start_au_allocator(ERTS_ALC_A_EHEAP, &init.eheap_alloc, &eheap_alloc_state); @@ -612,11 +729,9 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) erts_set_fix_size(ERTS_ALC_T_PROC, sizeof(Process)); erts_set_fix_size(ERTS_ALC_T_DB_TABLE, sizeof(DbTable)); erts_set_fix_size(ERTS_ALC_T_ATOM, sizeof(Atom)); - erts_set_fix_size(ERTS_ALC_T_EXPORT, sizeof(Export)); + erts_set_fix_size(ERTS_ALC_T_MODULE, sizeof(Module)); erts_set_fix_size(ERTS_ALC_T_REG_PROC, sizeof(RegProc)); - erts_set_fix_size(ERTS_ALC_T_MONITOR_SH, ERTS_MONITOR_SH_SIZE*sizeof(Uint)); - erts_set_fix_size(ERTS_ALC_T_NLINK_SH, ERTS_LINK_SH_SIZE*sizeof(Uint)); erts_set_fix_size(ERTS_ALC_T_FUN_ENTRY, sizeof(ErlFunEntry)); #ifdef ERTS_ALC_T_DRV_EV_D_STATE erts_set_fix_size(ERTS_ALC_T_DRV_EV_D_STATE, @@ -626,6 +741,11 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) erts_set_fix_size(ERTS_ALC_T_DRV_SEL_D_STATE, sizeof(ErtsDrvSelectDataState)); #endif +#if !HALFWORD_HEAP + erts_set_fix_size(ERTS_ALC_T_EXPORT, sizeof(Export)); + erts_set_fix_size(ERTS_ALC_T_MONITOR_SH, ERTS_MONITOR_SH_SIZE*sizeof(Uint)); + erts_set_fix_size(ERTS_ALC_T_NLINK_SH, ERTS_LINK_SH_SIZE*sizeof(Uint)); +#endif #endif #endif @@ -638,6 +758,12 @@ set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init) ErtsAllocatorInfo_t *ai = &erts_allctrs_info[alctr_n]; ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[alctr_n]; + /* + * Some allocators are forced on if halfword heap is used. + */ + if (init->init.util.force) + init->enable = 1; + if (!init->enable) { af->alloc = erts_sys_alloc; af->realloc = erts_sys_realloc; @@ -786,6 +912,12 @@ start_au_allocator(ErtsAlcType_t alctr_n, &init->init.af, &init->init.util); break; + case AOFIRSTFIT: + as = (void *) erts_aoffalc_start((AOFFAllctr_t *) as0, + &init->init.aoff, + &init->init.util); + break; + default: as = NULL; ASSERT(0); @@ -896,6 +1028,20 @@ get_kb_value(char *param_end, char** argv, int* ip) } static Uint +get_byte_value(char *param_end, char** argv, int* ip) +{ + Sint tmp; + char *rest; + char *param = argv[*ip]+1; + char *value = get_value(param_end, argv, ip); + errno = 0; + tmp = (Sint) strtol(value, &rest, 10); + if (errno != 0 || rest == value || tmp < 0) + bad_value(param, param_end, value); + return (Uint) tmp; +} + +static Uint get_amount_value(char *param_end, char** argv, int* ip) { Sint tmp; @@ -966,6 +1112,9 @@ handle_au_arg(struct au_init *auip, else if (strcmp("af", alg) == 0) { auip->atype = AFIT; } + else if (strcmp("aoff", alg) == 0) { + auip->atype = AOFIRSTFIT; + } else { bad_value(param, sub_param + 1, alg); } @@ -1034,6 +1183,12 @@ handle_au_arg(struct au_init *auip, if(has_prefix("sbct", sub_param)) { auip->init.util.sbct = get_kb_value(sub_param + 4, argv, ip); } + else if (has_prefix("sbmbcs", sub_param)) { + auip->init.util.sbmbcs = get_byte_value(sub_param + 6, argv, ip); + } + else if (has_prefix("sbmbct", sub_param)) { + auip->init.util.sbmbct = get_byte_value(sub_param + 6, argv, ip); + } else if (has_prefix("smbcs", sub_param)) { auip->default_.smbcs = 0; auip->init.util.smbcs = get_kb_value(sub_param + 5, argv, ip); @@ -1072,6 +1227,7 @@ static void handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) { struct au_init *aui[] = { + &init->sbmbc_alloc, &init->binary_alloc, &init->std_alloc, &init->ets_alloc, @@ -1099,6 +1255,9 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) case 'B': handle_au_arg(&init->binary_alloc, &argv[i][3], argv, &i); break; + case 'C': + handle_au_arg(&init->sbmbc_alloc, &argv[i][3], argv, &i); + break; case 'D': handle_au_arg(&init->std_alloc, &argv[i][3], argv, &i); break; @@ -1348,14 +1507,6 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) argv[j++] = argv[i]; } *argc = j; -#if HALFWORD_HEAP - /* If halfword heap, silently ignore any disabling of internal - allocators */ - for (i = 0; i < aui_sz; ++i) - aui[i]->enable = 1; -#endif - - } static char *type_no_str(ErtsAlcType_t n) @@ -1528,10 +1679,10 @@ erts_realloc_n_enomem(ErtsAlcType_t n, void *ptr, Uint size) erts_alc_fatal_error(ERTS_ALC_E_NOMEM, ERTS_ALC_O_REALLOC, n, size); } -static ERTS_INLINE Uint +static ERTS_INLINE UWord alcu_size(ErtsAlcType_t ai) { - Uint res = 0; + UWord res = 0; ASSERT(erts_allctrs_info[ai].enabled); ASSERT(erts_allctrs_info[ai].alloc_util); @@ -1563,6 +1714,49 @@ alcu_size(ErtsAlcType_t ai) return res; } +#if HALFWORD_HEAP +static ERTS_INLINE int +alcu_is_low(ErtsAlcType_t ai) +{ + int is_low = 0; + ASSERT(erts_allctrs_info[ai].enabled); + ASSERT(erts_allctrs_info[ai].alloc_util); + + if (!erts_allctrs_info[ai].thr_spec) { + Allctr_t *allctr = erts_allctrs_info[ai].extra; + is_low = allctr->mseg_opt.low_mem; + } + else { + ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[ai]; + int i; +# ifdef DEBUG + int found_one = 0; +# endif + + ASSERT(tspec->all_thr_safe); + ASSERT(tspec->enabled); + + for (i = tspec->size - 1; i >= 0; i--) { + Allctr_t *allctr = tspec->allctr[i]; + if (allctr) { +# ifdef DEBUG + if (!found_one) { + is_low = allctr->mseg_opt.low_mem; + found_one = 1; + } + else ASSERT(is_low == allctr->mseg_opt.low_mem); +# else + is_low = allctr->mseg_opt.low_mem; + break; +# endif + } + } + ASSERT(found_one); + } + return is_low; +} +#endif /* HALFWORD */ + Eterm erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) { @@ -1579,22 +1773,28 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) int code; int ets; int maximum; +#if HALFWORD_HEAP + int low; +#endif } want = {0}; struct { - Uint total; - Uint processes; - Uint processes_used; - Uint system; - Uint atom; - Uint atom_used; - Uint binary; - Uint code; - Uint ets; - Uint maximum; + UWord total; + UWord processes; + UWord processes_used; + UWord system; + UWord atom; + UWord atom_used; + UWord binary; + UWord code; + UWord ets; + UWord maximum; +#if HALFWORD_HEAP + UWord low; +#endif } size = {0}; - Eterm atoms[sizeof(size)/sizeof(Uint)]; - Uint *uintps[sizeof(size)/sizeof(Uint)]; - Eterm euints[sizeof(size)/sizeof(Uint)]; + Eterm atoms[sizeof(size)/sizeof(UWord)]; + UWord *uintps[sizeof(size)/sizeof(UWord)]; + Eterm euints[sizeof(size)/sizeof(UWord)]; int want_tot_or_sys; int length; Eterm res = THE_NON_VALUE; @@ -1646,7 +1846,11 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) atoms[length] = am_maximum; uintps[length++] = &size.maximum; } - +#if HALFWORD_HEAP + want.low = 1; + atoms[length] = am_low; + uintps[length++] = &size.low; +#endif } else { DeclareTmpHeapNoproc(tmp_heap,2); @@ -1740,6 +1944,15 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) return am_badarg; } break; +#if HALFWORD_HEAP + case am_low: + if (!want.low) { + want.low = 1; + atoms[length] = am_low; + uintps[length++] = &size.low; + } + break; +#endif default: UnUseTmpHeapNoproc(2); return am_badarg; @@ -1751,12 +1964,16 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) return am_badarg; } - /* All alloc_util allocators *have* to be enabled */ + /* All alloc_util allocators except sbmbc_alloc *have* to be enabled */ for (ai = ERTS_ALC_A_MIN; ai <= ERTS_ALC_A_MAX; ai++) { switch (ai) { case ERTS_ALC_A_SYSTEM: case ERTS_ALC_A_FIXED_SIZE: + case ERTS_ALC_A_SBMBC: +#if HALFWORD_HEAP + case ERTS_ALC_A_SBMBC_LOW: +#endif break; default: if (!erts_allctrs_info[ai].enabled @@ -1769,7 +1986,7 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) ASSERT(length <= sizeof(atoms)/sizeof(Eterm)); ASSERT(length <= sizeof(euints)/sizeof(Eterm)); - ASSERT(length <= sizeof(uintps)/sizeof(Uint)); + ASSERT(length <= sizeof(uintps)/sizeof(UWord)); if (proc) { @@ -1788,14 +2005,20 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) for (ai = ERTS_ALC_A_MIN; ai <= ERTS_ALC_A_MAX; ai++) { if (erts_allctrs_info[ai].alloc_util) { - Uint *save; - Uint asz; + UWord *save; + UWord asz; switch (ai) { case ERTS_ALC_A_TEMPORARY: /* * Often not thread safe and usually never * contain any allocated memory. */ + case ERTS_ALC_A_SBMBC: + /* Included in other allocators */ +#if HALFWORD_HEAP + case ERTS_ALC_A_SBMBC_LOW: + /* Included in other allocators */ +#endif continue; case ERTS_ALC_A_EHEAP: save = &size.processes; @@ -1814,6 +2037,11 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) if (save) *save = asz; size.total += asz; +#if HALFWORD_HEAP + if (alcu_is_low(ai)) { + size.low += asz; + } +#endif } } } @@ -1821,7 +2049,7 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) if (want_tot_or_sys || want.processes || want.processes_used) { - Uint tmp; + UWord tmp; if (ERTS_MEM_NEED_ALL_ALCU) tmp = size.processes; @@ -1836,6 +2064,9 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) size.processes = size.processes_used = tmp; +#if HALFWORD_HEAP + /* BUG: We ignore link and monitor memory */ +#else erts_fix_info(ERTS_ALC_T_NLINK_SH, &efi); size.processes += efi.total; size.processes_used += efi.used; @@ -1843,6 +2074,7 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) erts_fix_info(ERTS_ALC_T_MONITOR_SH, &efi); size.processes += efi.total; size.processes_used += efi.used; +#endif erts_fix_info(ERTS_ALC_T_PROC, &efi); size.processes += efi.total; @@ -1879,8 +2111,12 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) erts_fix_info(ERTS_ALC_T_MODULE, &efi); size.code += efi.used; size.code += export_table_sz(); +#if HALFWORD_HEAP + size.code += export_list_size() * sizeof(Export); +#else erts_fix_info(ERTS_ALC_T_EXPORT, &efi); size.code += efi.used; +#endif size.code += erts_fun_table_sz(); erts_fix_info(ERTS_ALC_T_FUN_ENTRY, &efi); size.code += efi.used; @@ -1913,7 +2149,7 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) /* Print result... */ erts_print(to, arg, "=memory\n"); for (i = 0; i < length; i++) - erts_print(to, arg, "%T: %beu\n", atoms[i], *uintps[i]); + erts_print(to, arg, "%T: %bpu\n", atoms[i], *uintps[i]); } if (proc) { @@ -1926,9 +2162,9 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) if (only_one_value) { ASSERT(length == 1); hsz = 0; - erts_bld_uint(NULL, &hsz, *uintps[0]); + erts_bld_uword(NULL, &hsz, *uintps[0]); hp = hsz ? HAlloc((Process *) proc, hsz) : NULL; - res = erts_bld_uint(&hp, NULL, *uintps[0]); + res = erts_bld_uword(&hp, NULL, *uintps[0]); } else { Uint **hpp = NULL; @@ -1938,7 +2174,7 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) while (1) { int i; for (i = 0; i < length; i++) - euints[i] = erts_bld_uint(hpp, hszp, *uintps[i]); + euints[i] = erts_bld_uword(hpp, hszp, *uintps[i]); res = erts_bld_2tup_list(hpp, hszp, length, atoms, euints); if (hpp) break; @@ -2764,6 +3000,7 @@ unsigned long erts_alc_test(unsigned long op, case 0x2: return erts_bfalc_test(op, a1, a2); case 0x3: return erts_afalc_test(op, a1, a2); case 0x4: return erts_mseg_test(op, a1, a2, a3); + case 0x5: return erts_aoffalc_test(op, a1, a2); case 0xf: switch (op) { case 0xf00: @@ -2807,6 +3044,7 @@ unsigned long erts_alc_test(unsigned long op, init.atype = GOODFIT; init.init.util.name_prefix = (char *) a1; init.init.util.ts = a2 ? 1 : 0; + init.init.util.sbmbct = 0; if ((char **) a3) { char **argv = (char **) a3; @@ -2842,6 +3080,14 @@ unsigned long erts_alc_test(unsigned long op, &init.init.af, &init.init.util); break; + case AOFIRSTFIT: + allctr = erts_aoffalc_start((AOFFAllctr_t *) + erts_alloc(ERTS_ALC_T_UNDEF, + sizeof(AOFFAllctr_t)), + &init.init.aoff, + &init.init.util); + break; + default: ASSERT(0); allctr = NULL; diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h index ce792d4d17..c35a60da22 100644 --- a/erts/emulator/beam/erl_alloc.h +++ b/erts/emulator/beam/erl_alloc.h @@ -99,6 +99,14 @@ unsigned long erts_alc_test(unsigned long, #define ERTS_ALC_MIN_LONG_LIVED_TIME (10*60*1000) +#if HALFWORD_HEAP +#define ERTS_IS_SBMBC_ALLOCATOR_NO__(NO) \ + ((NO) == ERTS_ALC_A_SBMBC || (NO) == ERTS_ALC_A_SBMBC_LOW) +#else +#define ERTS_IS_SBMBC_ALLOCATOR_NO__(NO) \ + ((NO) == ERTS_ALC_A_SBMBC) +#endif + typedef struct { int alloc_util; int enabled; diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index ca71798917..eda0831441 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -65,6 +65,11 @@ allocator SYSTEM true sys_alloc +allocator SBMBC true sbmbc_alloc ++if halfword +allocator SBMBC_LOW true sbmbc_low_alloc ++endif + +if smp allocator TEMPORARY true temp_alloc @@ -75,6 +80,11 @@ allocator EHEAP true eheap_alloc allocator ETS true ets_alloc allocator FIXED_SIZE true fix_alloc ++if halfword +allocator LONG_LIVED_LOW true ll_low_alloc +allocator STANDARD_LOW true std_low_alloc ++endif + +else # Non smp build allocator TEMPORARY false temp_alloc @@ -85,12 +95,18 @@ allocator EHEAP false eheap_alloc allocator ETS false ets_alloc allocator FIXED_SIZE false fix_alloc ++if halfword +allocator LONG_LIVED_LOW false ll_low_alloc +allocator STANDARD_LOW false std_low_alloc ++endif + +endif allocator BINARY true binary_alloc allocator DRIVER true driver_alloc + # --- Class declarations ----------------------------------------------------- # # Syntax: class <CLASS> <DESCRIPTION> @@ -123,16 +139,12 @@ class SYSTEM system_data # # <TYPE> <ALLOCATOR> <CLASS> <DESCRIPTION> +type SBMBC SBMBC SYSTEM small_block_mbc type PROC FIXED_SIZE PROCESSES proc type ATOM FIXED_SIZE ATOM atom_entry -type EXPORT FIXED_SIZE CODE export_entry type MODULE FIXED_SIZE CODE module_entry type REG_PROC FIXED_SIZE PROCESSES reg_proc type LINK_LH STANDARD PROCESSES link_lh -type MONITOR_SH FIXED_SIZE PROCESSES monitor_sh -type MONITOR_LH STANDARD PROCESSES monitor_lh -type NLINK_SH FIXED_SIZE PROCESSES nlink_sh -type NLINK_LH STANDARD PROCESSES nlink_lh type SUSPEND_MON STANDARD PROCESSES suspend_monitor type PEND_SUSPEND SHORT_LIVED PROCESSES pending_suspend type PROC_LIST SHORT_LIVED PROCESSES proc_list @@ -175,7 +187,6 @@ type DRIVER STANDARD SYSTEM driver type NIF DRIVER SYSTEM nif_internal type BINARY BINARY BINARIES binary type NBIF_TABLE SYSTEM SYSTEM nbif_tab -type CODE LONG_LIVED CODE code type ARG_REG STANDARD PROCESSES arg_reg type PROC_DICT STANDARD PROCESSES proc_dict type CALLS_BUF STANDARD PROCESSES calls_buf @@ -193,10 +204,8 @@ type DB_FIXATION SHORT_LIVED ETS db_fixation type DB_FIX_DEL SHORT_LIVED ETS fixed_del type DB_TABLES LONG_LIVED ETS db_tabs type DB_NTAB_ENT STANDARD ETS db_named_table_entry -type DB_HEIR_DATA STANDARD ETS db_heir_data type DB_TMP TEMPORARY ETS db_tmp type DB_MC_STK TEMPORARY ETS db_mc_stack -type DB_MS_PSDO_PROC LONG_LIVED ETS db_match_pseudo_proc type DB_MS_RUN_HEAP SHORT_LIVED ETS db_match_spec_run_heap type DB_MS_CMPL_HEAP TEMPORARY ETS db_match_spec_cmpl_heap type DB_SEG ETS ETS db_segment @@ -213,10 +222,8 @@ type LOGGER_DSBUF TEMPORARY SYSTEM logger_dsbuf type TMP_DSBUF TEMPORARY SYSTEM tmp_dsbuf type INFO_DSBUF SYSTEM SYSTEM info_dsbuf # INFO_DSBUF have to use the SYSTEM allocator; otherwise, a deadlock might occur -type SCHDLR_DATA LONG_LIVED SYSTEM scheduler_data type SCHDLR_SLP_INFO LONG_LIVED SYSTEM scheduler_sleep_info type RUNQS LONG_LIVED SYSTEM run_queues -type DDLL_PROCESS STANDARD SYSTEM ddll_processes type DDLL_HANDLE STANDARD SYSTEM ddll_handle type DDLL_ERRCODES LONG_LIVED SYSTEM ddll_errcodes type DDLL_TMP_BUF TEMPORARY SYSTEM ddll_tmp_buf @@ -327,13 +334,46 @@ type SSB SHORT_LIVED PROCESSES ssb +endif ++if halfword + +type SBMBC_LOW SBMBC_LOW SYSTEM small_block_mbc_low +type DDLL_PROCESS STANDARD_LOW SYSTEM ddll_processes +type MONITOR_LH STANDARD_LOW PROCESSES monitor_lh +type NLINK_LH STANDARD_LOW PROCESSES nlink_lh +type CODE LONG_LIVED_LOW CODE code +type DB_HEIR_DATA STANDARD_LOW ETS db_heir_data +type DB_MS_PSDO_PROC LONG_LIVED_LOW ETS db_match_pseudo_proc +type SCHDLR_DATA LONG_LIVED_LOW SYSTEM scheduler_data +type LL_TEMP_TERM LONG_LIVED_LOW SYSTEM ll_temp_term + +# no FIXED_SIZE for low memory +type EXPORT STANDARD_LOW CODE export_entry +type MONITOR_SH STANDARD_LOW PROCESSES monitor_sh +type NLINK_SH STANDARD_LOW PROCESSES nlink_sh + ++else # "fullword" + +type DDLL_PROCESS STANDARD SYSTEM ddll_processes +type MONITOR_LH STANDARD PROCESSES monitor_lh +type NLINK_LH STANDARD PROCESSES nlink_lh +type CODE LONG_LIVED CODE code +type DB_HEIR_DATA STANDARD ETS db_heir_data +type DB_MS_PSDO_PROC LONG_LIVED ETS db_match_pseudo_proc +type SCHDLR_DATA LONG_LIVED SYSTEM scheduler_data +type LL_TEMP_TERM LONG_LIVED SYSTEM ll_temp_term + +type EXPORT FIXED_SIZE CODE export_entry +type MONITOR_SH FIXED_SIZE PROCESSES monitor_sh +type NLINK_SH FIXED_SIZE PROCESSES nlink_sh + ++endif + + # # Types used by system specific code # type TEMP_TERM TEMPORARY SYSTEM temp_term -type LL_TEMP_TERM LONG_LIVED SYSTEM ll_temp_term - type DRV_TAB LONG_LIVED SYSTEM drv_tab type DRV_EV_STATE LONG_LIVED SYSTEM driver_event_state type DRV_EV_D_STATE FIXED_SIZE SYSTEM driver_event_data_state diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index 84c72439a3..d51ed0c36d 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -66,6 +66,7 @@ static int atoms_initialized = 0; static int initialized = 0; +int erts_have_sbmbc_alloc; #if HAVE_ERTS_MSEG @@ -85,8 +86,6 @@ static int initialized = 0; #undef ASSERT #define ASSERT ASSERT_EXPR -#define ERTS_ALCU_FLG_FAIL_REALLOC_MOVE ((UWord) 1) - #if 0 /* Can be useful for debugging */ #define MBC_REALLOC_ALWAYS_MOVES @@ -275,14 +274,26 @@ static void check_blk_carrier(Allctr_t *, Block_t *); #ifdef DEBUG #define DEBUG_CHECK_CARRIER_NO_SZ(AP) \ - ASSERT(((AP)->sbcs.curr_mseg.no && (AP)->sbcs.curr_mseg.size) \ - || (!(AP)->sbcs.curr_mseg.no && !(AP)->sbcs.curr_mseg.size));\ - ASSERT(((AP)->sbcs.curr_sys_alloc.no && (AP)->sbcs.curr_sys_alloc.size)\ - || (!(AP)->sbcs.curr_sys_alloc.no && !(AP)->sbcs.curr_sys_alloc.size));\ - ASSERT(((AP)->mbcs.curr_mseg.no && (AP)->mbcs.curr_mseg.size) \ - || (!(AP)->mbcs.curr_mseg.no && !(AP)->mbcs.curr_mseg.size));\ - ASSERT(((AP)->mbcs.curr_sys_alloc.no && (AP)->mbcs.curr_sys_alloc.size)\ - || (!(AP)->mbcs.curr_sys_alloc.no && !(AP)->mbcs.curr_sys_alloc.size)) + ASSERT(((AP)->sbcs.curr.norm.mseg.no \ + && (AP)->sbcs.curr.norm.mseg.size) \ + || (!(AP)->sbcs.curr.norm.mseg.no \ + && !(AP)->sbcs.curr.norm.mseg.size)); \ + ASSERT(((AP)->sbcs.curr.norm.sys_alloc.no \ + && (AP)->sbcs.curr.norm.sys_alloc.size) \ + || (!(AP)->sbcs.curr.norm.sys_alloc.no \ + && !(AP)->sbcs.curr.norm.sys_alloc.size)); \ + ASSERT(((AP)->mbcs.curr.norm.mseg.no \ + && (AP)->mbcs.curr.norm.mseg.size) \ + || (!(AP)->mbcs.curr.norm.mseg.no \ + && !(AP)->mbcs.curr.norm.mseg.size)); \ + ASSERT(((AP)->mbcs.curr.norm.sys_alloc.no \ + && (AP)->mbcs.curr.norm.sys_alloc.size) \ + || (!(AP)->mbcs.curr.norm.sys_alloc.no \ + && !(AP)->mbcs.curr.norm.sys_alloc.size)); \ + ASSERT(((AP)->sbmbcs.curr.small_block.no \ + && (AP)->sbmbcs.curr.small_block.size) \ + || (!(AP)->sbmbcs.curr.small_block.no \ + && !(AP)->sbmbcs.curr.small_block.size)) #else #define DEBUG_CHECK_CARRIER_NO_SZ(AP) @@ -292,27 +303,27 @@ static void check_blk_carrier(Allctr_t *, Block_t *); (AP)->sbcs.blocks.curr.size += (BSZ); \ if ((AP)->sbcs.blocks.max.size < (AP)->sbcs.blocks.curr.size) \ (AP)->sbcs.blocks.max.size = (AP)->sbcs.blocks.curr.size; \ - if ((AP)->sbcs.max.no < ((AP)->sbcs.curr_mseg.no \ - + (AP)->sbcs.curr_sys_alloc.no)) \ - (AP)->sbcs.max.no = ((AP)->sbcs.curr_mseg.no \ - + (AP)->sbcs.curr_sys_alloc.no); \ - if ((AP)->sbcs.max.size < ((AP)->sbcs.curr_mseg.size \ - + (AP)->sbcs.curr_sys_alloc.size)) \ - (AP)->sbcs.max.size = ((AP)->sbcs.curr_mseg.size \ - + (AP)->sbcs.curr_sys_alloc.size) + if ((AP)->sbcs.max.no < ((AP)->sbcs.curr.norm.mseg.no \ + + (AP)->sbcs.curr.norm.sys_alloc.no)) \ + (AP)->sbcs.max.no = ((AP)->sbcs.curr.norm.mseg.no \ + + (AP)->sbcs.curr.norm.sys_alloc.no); \ + if ((AP)->sbcs.max.size < ((AP)->sbcs.curr.norm.mseg.size \ + + (AP)->sbcs.curr.norm.sys_alloc.size)) \ + (AP)->sbcs.max.size = ((AP)->sbcs.curr.norm.mseg.size \ + + (AP)->sbcs.curr.norm.sys_alloc.size) #define STAT_MSEG_SBC_ALLOC(AP, CSZ, BSZ) \ do { \ - (AP)->sbcs.curr_mseg.no++; \ - (AP)->sbcs.curr_mseg.size += (CSZ); \ + (AP)->sbcs.curr.norm.mseg.no++; \ + (AP)->sbcs.curr.norm.mseg.size += (CSZ); \ STAT_SBC_ALLOC((AP), (BSZ)); \ DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ } while (0) #define STAT_SYS_ALLOC_SBC_ALLOC(AP, CSZ, BSZ) \ do { \ - (AP)->sbcs.curr_sys_alloc.no++; \ - (AP)->sbcs.curr_sys_alloc.size += (CSZ); \ + (AP)->sbcs.curr.norm.sys_alloc.no++; \ + (AP)->sbcs.curr.norm.sys_alloc.size += (CSZ); \ STAT_SBC_ALLOC((AP), (BSZ)); \ DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ } while (0) @@ -324,85 +335,111 @@ do { \ #define STAT_MSEG_SBC_FREE(AP, CSZ, BSZ) \ do { \ - ASSERT((AP)->sbcs.curr_mseg.no > 0); \ - (AP)->sbcs.curr_mseg.no--; \ - ASSERT((AP)->sbcs.curr_mseg.size >= (CSZ)); \ - (AP)->sbcs.curr_mseg.size -= (CSZ); \ + ASSERT((AP)->sbcs.curr.norm.mseg.no > 0); \ + (AP)->sbcs.curr.norm.mseg.no--; \ + ASSERT((AP)->sbcs.curr.norm.mseg.size >= (CSZ)); \ + (AP)->sbcs.curr.norm.mseg.size -= (CSZ); \ STAT_SBC_FREE((AP), (BSZ)); \ DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ } while (0) #define STAT_SYS_ALLOC_SBC_FREE(AP, CSZ, BSZ) \ do { \ - ASSERT((AP)->sbcs.curr_sys_alloc.no > 0); \ - (AP)->sbcs.curr_sys_alloc.no--; \ - ASSERT((AP)->sbcs.curr_sys_alloc.size >= (CSZ)); \ - (AP)->sbcs.curr_sys_alloc.size -= (CSZ); \ + ASSERT((AP)->sbcs.curr.norm.sys_alloc.no > 0); \ + (AP)->sbcs.curr.norm.sys_alloc.no--; \ + ASSERT((AP)->sbcs.curr.norm.sys_alloc.size >= (CSZ)); \ + (AP)->sbcs.curr.norm.sys_alloc.size -= (CSZ); \ STAT_SBC_FREE((AP), (BSZ)); \ DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ } while (0) #define STAT_MBC_ALLOC(AP) \ - if ((AP)->mbcs.max.no < ((AP)->mbcs.curr_mseg.no \ - + (AP)->mbcs.curr_sys_alloc.no)) \ - (AP)->mbcs.max.no = ((AP)->mbcs.curr_mseg.no \ - + (AP)->mbcs.curr_sys_alloc.no); \ - if ((AP)->mbcs.max.size < ((AP)->mbcs.curr_mseg.size \ - + (AP)->mbcs.curr_sys_alloc.size)) \ - (AP)->mbcs.max.size = ((AP)->mbcs.curr_mseg.size \ - + (AP)->mbcs.curr_sys_alloc.size) + if ((AP)->mbcs.max.no < ((AP)->mbcs.curr.norm.mseg.no \ + + (AP)->mbcs.curr.norm.sys_alloc.no)) \ + (AP)->mbcs.max.no = ((AP)->mbcs.curr.norm.mseg.no \ + + (AP)->mbcs.curr.norm.sys_alloc.no); \ + if ((AP)->mbcs.max.size < ((AP)->mbcs.curr.norm.mseg.size \ + + (AP)->mbcs.curr.norm.sys_alloc.size)) \ + (AP)->mbcs.max.size = ((AP)->mbcs.curr.norm.mseg.size \ + + (AP)->mbcs.curr.norm.sys_alloc.size) + +#define STAT_SBMBC_ALLOC(AP, CSZ) \ +do { \ + (AP)->sbmbcs.curr.small_block.no++; \ + (AP)->sbmbcs.curr.small_block.size += (CSZ); \ + if ((AP)->sbmbcs.max.no < (AP)->sbmbcs.curr.small_block.no) \ + (AP)->sbmbcs.max.no = (AP)->sbmbcs.curr.small_block.no; \ + if ((AP)->sbmbcs.max.size < (AP)->sbmbcs.curr.small_block.size) \ + (AP)->sbmbcs.max.size = (AP)->sbmbcs.curr.small_block.size; \ + DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ +} while (0) #define STAT_MSEG_MBC_ALLOC(AP, CSZ) \ do { \ - (AP)->mbcs.curr_mseg.no++; \ - (AP)->mbcs.curr_mseg.size += (CSZ); \ + (AP)->mbcs.curr.norm.mseg.no++; \ + (AP)->mbcs.curr.norm.mseg.size += (CSZ); \ STAT_MBC_ALLOC((AP)); \ DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ } while (0) #define STAT_SYS_ALLOC_MBC_ALLOC(AP, CSZ) \ do { \ - (AP)->mbcs.curr_sys_alloc.no++; \ - (AP)->mbcs.curr_sys_alloc.size += (CSZ); \ + (AP)->mbcs.curr.norm.sys_alloc.no++; \ + (AP)->mbcs.curr.norm.sys_alloc.size += (CSZ); \ STAT_MBC_ALLOC((AP)); \ DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ } while (0) +#define STAT_SBMBC_FREE(AP, CSZ) \ +do { \ + ASSERT((AP)->sbmbcs.curr.small_block.no > 0); \ + (AP)->sbmbcs.curr.small_block.no--; \ + ASSERT((AP)->sbmbcs.curr.small_block.size >= (CSZ)); \ + (AP)->sbmbcs.curr.small_block.size -= (CSZ); \ + DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ +} while (0) + #define STAT_MSEG_MBC_FREE(AP, CSZ) \ do { \ - ASSERT((AP)->mbcs.curr_mseg.no > 0); \ - (AP)->mbcs.curr_mseg.no--; \ - ASSERT((AP)->mbcs.curr_mseg.size >= (CSZ)); \ - (AP)->mbcs.curr_mseg.size -= (CSZ); \ + ASSERT((AP)->mbcs.curr.norm.mseg.no > 0); \ + (AP)->mbcs.curr.norm.mseg.no--; \ + ASSERT((AP)->mbcs.curr.norm.mseg.size >= (CSZ)); \ + (AP)->mbcs.curr.norm.mseg.size -= (CSZ); \ DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ } while (0) #define STAT_SYS_ALLOC_MBC_FREE(AP, CSZ) \ do { \ - ASSERT((AP)->mbcs.curr_sys_alloc.no > 0); \ - (AP)->mbcs.curr_sys_alloc.no--; \ - ASSERT((AP)->mbcs.curr_sys_alloc.size >= (CSZ)); \ - (AP)->mbcs.curr_sys_alloc.size -= (CSZ); \ + ASSERT((AP)->mbcs.curr.norm.sys_alloc.no > 0); \ + (AP)->mbcs.curr.norm.sys_alloc.no--; \ + ASSERT((AP)->mbcs.curr.norm.sys_alloc.size >= (CSZ)); \ + (AP)->mbcs.curr.norm.sys_alloc.size -= (CSZ); \ DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ } while (0) -#define STAT_MBC_BLK_ALLOC(AP, BSZ) \ +#define STAT_MBC_BLK_ALLOC(AP, BSZ, FLGS) \ do { \ - (AP)->mbcs.blocks.curr.no++; \ - if ((AP)->mbcs.blocks.max.no < (AP)->mbcs.blocks.curr.no) \ - (AP)->mbcs.blocks.max.no = (AP)->mbcs.blocks.curr.no; \ - (AP)->mbcs.blocks.curr.size += (BSZ); \ - if ((AP)->mbcs.blocks.max.size < (AP)->mbcs.blocks.curr.size) \ - (AP)->mbcs.blocks.max.size = (AP)->mbcs.blocks.curr.size; \ + CarriersStats_t *cstats__ = (((FLGS) & ERTS_ALCU_FLG_SBMBC) \ + ? &(AP)->sbmbcs \ + : &(AP)->mbcs); \ + cstats__->blocks.curr.no++; \ + if (cstats__->blocks.max.no < cstats__->blocks.curr.no) \ + cstats__->blocks.max.no = cstats__->blocks.curr.no; \ + cstats__->blocks.curr.size += (BSZ); \ + if (cstats__->blocks.max.size < cstats__->blocks.curr.size) \ + cstats__->blocks.max.size = cstats__->blocks.curr.size; \ } while (0) -#define STAT_MBC_BLK_FREE(AP, BSZ) \ +#define STAT_MBC_BLK_FREE(AP, BSZ, FLGS) \ do { \ - ASSERT((AP)->mbcs.blocks.curr.no > 0); \ - (AP)->mbcs.blocks.curr.no--; \ - ASSERT((AP)->mbcs.blocks.curr.size >= (BSZ)); \ - (AP)->mbcs.blocks.curr.size -= (BSZ); \ + CarriersStats_t *cstats__ = (((FLGS) & ERTS_ALCU_FLG_SBMBC) \ + ? &(AP)->sbmbcs \ + : &(AP)->mbcs); \ + ASSERT(cstats__->blocks.curr.no > 0); \ + cstats__->blocks.curr.no--; \ + ASSERT(cstats__->blocks.curr.size >= (BSZ)); \ + cstats__->blocks.curr.size -= (BSZ); \ } while (0) /* Debug stuff... */ @@ -410,7 +447,7 @@ do { \ static UWord carrier_alignment; #define DEBUG_SAVE_ALIGNMENT(C) \ do { \ - UWord algnmnt__ = sizeof(Unit_t) - (((UWord) (C)) % sizeof(Unit_t)); \ + UWord algnmnt__ = sizeof(Unit_t) - (((UWord) (C)) % sizeof(Unit_t));\ carrier_alignment = MIN(carrier_alignment, algnmnt__); \ ASSERT(((UWord) (C)) % sizeof(UWord) == 0); \ } while (0) @@ -524,8 +561,8 @@ static Uint get_next_mbc_size(Allctr_t *allctr) { Uint size; - int cs = (allctr->mbcs.curr_mseg.no - + allctr->mbcs.curr_sys_alloc.no + int cs = (allctr->mbcs.curr.norm.mseg.no + + allctr->mbcs.curr.norm.sys_alloc.no - (allctr->main_carrier ? 1 : 0)); ASSERT(cs >= 0); @@ -609,7 +646,8 @@ unlink_carrier(CarrierList_t *cl, Carrier_t *crr) } } - +static Block_t *create_sbmbc(Allctr_t *allctr, Uint umem_sz); +static void destroy_sbmbc(Allctr_t *allctr, Block_t *blk); static Block_t *create_carrier(Allctr_t *, Uint, UWord); static void destroy_carrier(Allctr_t *, Block_t *); @@ -619,39 +657,57 @@ static void destroy_carrier(Allctr_t *, Block_t *); * block in a sbc. */ static ERTS_INLINE void * -mbc_alloc_block(Allctr_t *allctr, Uint size, Uint *blk_szp) +mbc_alloc_block(Allctr_t *allctr, Uint size, Uint *blk_szp, Uint32 *alcu_flgsp) { Block_t *blk; + Uint get_blk_sz; + Uint sbmbct; ASSERT(size); ASSERT(size < allctr->sbc_threshold); - *blk_szp = UMEMSZ2BLKSZ(allctr, size); + *blk_szp = get_blk_sz = UMEMSZ2BLKSZ(allctr, size); - blk = (*allctr->get_free_block)(allctr, *blk_szp, NULL, 0); + sbmbct = allctr->sbmbc_threshold; + if (sbmbct) { + if (get_blk_sz < sbmbct) { + *alcu_flgsp |= ERTS_ALCU_FLG_SBMBC; + if (get_blk_sz + allctr->min_block_size > sbmbct) { + /* Since we use block size to determine if blocks are + located in sbmbc or not... */ + get_blk_sz += allctr->min_block_size; + } + } + } + + blk = (*allctr->get_free_block)(allctr, get_blk_sz, NULL, 0, *alcu_flgsp); -#if HALFWORD_HEAP if (!blk) { - blk = create_carrier(allctr, *blk_szp, CFLG_MBC|CFLG_FORCE_MSEG); - } + if ((*alcu_flgsp) & ERTS_ALCU_FLG_SBMBC) + blk = create_sbmbc(allctr, get_blk_sz); + else { +#if HALFWORD_HEAP + blk = create_carrier(allctr, get_blk_sz, CFLG_MBC|CFLG_FORCE_MSEG); #else - if (!blk) { - blk = create_carrier(allctr, *blk_szp, CFLG_MBC); - if (!blk) { - /* Emergency! We couldn't create the carrier as we wanted. - Try to place it in a sys_alloced sbc. */ - blk = create_carrier(allctr, - size, - CFLG_SBC|CFLG_FORCE_SIZE|CFLG_FORCE_SYS_ALLOC); + blk = create_carrier(allctr, get_blk_sz, CFLG_MBC); + if (!blk) { + /* Emergency! We couldn't create the carrier as we wanted. + Try to place it in a sys_alloced sbc. */ + blk = create_carrier(allctr, + size, + (CFLG_SBC + | CFLG_FORCE_SIZE + | CFLG_FORCE_SYS_ALLOC)); + } +#endif } } -#endif #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG if (IS_MBC_BLK(blk)) { - (*allctr->link_free_block)(allctr, blk); + (*allctr->link_free_block)(allctr, blk, *alcu_flgsp); HARD_CHECK_BLK_CARRIER(allctr, blk); - (*allctr->unlink_free_block)(allctr, blk); + (*allctr->unlink_free_block)(allctr, blk, *alcu_flgsp); } #endif @@ -664,7 +720,8 @@ mbc_alloc_finalize(Allctr_t *allctr, Uint org_blk_sz, UWord flags, Uint want_blk_sz, - int valid_blk_info) + int valid_blk_info, + Uint32 alcu_flgs) { Uint blk_sz; Uint nxt_blk_sz; @@ -700,7 +757,7 @@ mbc_alloc_finalize(Allctr_t *allctr, SET_PREV_BLK_FREE(nxt_nxt_blk); } } - (*allctr->link_free_block)(allctr, nxt_blk); + (*allctr->link_free_block)(allctr, nxt_blk, alcu_flgs); ASSERT(IS_NOT_LAST_BLK(blk)); ASSERT(IS_FREE_BLK(nxt_blk)); @@ -741,7 +798,7 @@ mbc_alloc_finalize(Allctr_t *allctr, : IS_NOT_LAST_BLK(blk)); } - STAT_MBC_BLK_ALLOC(allctr, blk_sz); + STAT_MBC_BLK_ALLOC(allctr, blk_sz, alcu_flgs); ASSERT(IS_ALLOCED_BLK(blk)); ASSERT(blk_sz == BLK_SZ(blk)); @@ -761,7 +818,8 @@ mbc_alloc(Allctr_t *allctr, Uint size) { Block_t *blk; Uint blk_sz; - blk = mbc_alloc_block(allctr, size, &blk_sz); + Uint32 alcu_flgs = 0; + blk = mbc_alloc_block(allctr, size, &blk_sz, &alcu_flgs); if (!blk) return NULL; if (IS_MBC_BLK(blk)) @@ -770,7 +828,8 @@ mbc_alloc(Allctr_t *allctr, Uint size) BLK_SZ(blk), GET_BLK_HDR_FLGS(blk), blk_sz, - 1); + 1, + alcu_flgs); return BLK2UMEM(blk); } @@ -779,6 +838,7 @@ mbc_free(Allctr_t *allctr, void *p) { Uint is_first_blk; Uint is_last_blk; + Uint32 alcu_flgs = 0; Uint blk_sz; Block_t *blk; Block_t *nxt_blk; @@ -788,13 +848,15 @@ mbc_free(Allctr_t *allctr, void *p) blk = UMEM2BLK(p); blk_sz = BLK_SZ(blk); + if (blk_sz < allctr->sbmbc_threshold) + alcu_flgs |= ERTS_ALCU_FLG_SBMBC; ASSERT(IS_MBC_BLK(blk)); ASSERT(blk_sz >= allctr->min_block_size); HARD_CHECK_BLK_CARRIER(allctr, blk); - STAT_MBC_BLK_FREE(allctr, blk_sz); + STAT_MBC_BLK_FREE(allctr, blk_sz, alcu_flgs); is_first_blk = IS_FIRST_BLK(blk); is_last_blk = IS_LAST_BLK(blk); @@ -802,7 +864,7 @@ mbc_free(Allctr_t *allctr, void *p) if (!is_first_blk && IS_PREV_BLK_FREE(blk)) { /* Coalesce with previous block... */ blk = PREV_BLK(blk); - (*allctr->unlink_free_block)(allctr, blk); + (*allctr->unlink_free_block)(allctr, blk, alcu_flgs); blk_sz += BLK_SZ(blk); is_first_blk = IS_FIRST_BLK(blk); @@ -818,7 +880,7 @@ mbc_free(Allctr_t *allctr, void *p) nxt_blk = NXT_BLK(blk); if (IS_FREE_BLK(nxt_blk)) { /* Coalesce with next block... */ - (*allctr->unlink_free_block)(allctr, nxt_blk); + (*allctr->unlink_free_block)(allctr, nxt_blk, alcu_flgs); blk_sz += BLK_SZ(nxt_blk); SET_BLK_SZ(blk, blk_sz); @@ -850,16 +912,20 @@ mbc_free(Allctr_t *allctr, void *p) if (is_first_blk && is_last_blk - && allctr->main_carrier != FBLK2MBC(allctr, blk)) - destroy_carrier(allctr, blk); + && allctr->main_carrier != FBLK2MBC(allctr, blk)) { + if (alcu_flgs & ERTS_ALCU_FLG_SBMBC) + destroy_sbmbc(allctr, blk); + else + destroy_carrier(allctr, blk); + } else { - (*allctr->link_free_block)(allctr, blk); + (*allctr->link_free_block)(allctr, blk, alcu_flgs); HARD_CHECK_BLK_CARRIER(allctr, blk); } } static void * -mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs) +mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) { void *new_p; Uint old_blk_sz; @@ -867,7 +933,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs) #ifndef MBC_REALLOC_ALWAYS_MOVES Block_t *new_blk, *cand_blk; Uint cand_blk_sz; - Uint blk_sz; + Uint blk_sz, get_blk_sz; Block_t *nxt_blk; Uint nxt_blk_sz; Uint is_last_blk; @@ -883,10 +949,16 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs) ASSERT(old_blk_sz >= allctr->min_block_size); #ifdef MBC_REALLOC_ALWAYS_MOVES - if (flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE) + if (alcu_flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE) return NULL; #else /* !MBC_REALLOC_ALWAYS_MOVES */ - blk_sz = UMEMSZ2BLKSZ(allctr, size); + get_blk_sz = blk_sz = UMEMSZ2BLKSZ(allctr, size); + if ((alcu_flgs & ERTS_ALCU_FLG_SBMBC) + && (blk_sz + allctr->min_block_size > allctr->sbmbc_threshold)) { + /* Since we use block size to determine if blocks are + located in sbmbc or not... */ + get_blk_sz = blk_sz + allctr->min_block_size; + } ASSERT(IS_ALLOCED_BLK(blk)); ASSERT(IS_MBC_BLK(blk)); @@ -901,6 +973,9 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs) Uint diff_sz_val = old_blk_sz - blk_sz; Uint old_blk_sz_val = old_blk_sz; + if (get_blk_sz >= old_blk_sz) + return p; + if (diff_sz_val >= (~((Uint) 0) / 100)) { /* div both by 128 */ old_blk_sz_val >>= 7; @@ -909,7 +984,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs) /* Avoid fragmentation by moving the block if it is shrunk much */ if (100*diff_sz_val > allctr->mbc_move_threshold*old_blk_sz_val) { - if (flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE) + if (alcu_flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE) return NULL; cand_blk_sz = old_blk_sz; @@ -926,9 +1001,10 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs) } new_blk = (*allctr->get_free_block)(allctr, - blk_sz, + get_blk_sz, cand_blk, - cand_blk_sz); + cand_blk_sz, + alcu_flgs); if (new_blk || cand_blk != blk) goto move_into_new_blk; @@ -952,8 +1028,8 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs) nxt_blk_sz, SBH_THIS_FREE|SBH_PREV_ALLOCED|SBH_NOT_LAST_BLK); - STAT_MBC_BLK_FREE(allctr, old_blk_sz); - STAT_MBC_BLK_ALLOC(allctr, blk_sz); + STAT_MBC_BLK_FREE(allctr, old_blk_sz, alcu_flgs); + STAT_MBC_BLK_ALLOC(allctr, blk_sz, alcu_flgs); ASSERT(BLK_SZ(blk) >= allctr->min_block_size); @@ -964,7 +1040,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs) if (IS_FREE_BLK(nxt_nxt_blk)) { /* Coalesce with next free block... */ nxt_blk_sz += BLK_SZ(nxt_nxt_blk); - (*allctr->unlink_free_block)(allctr, nxt_nxt_blk); + (*allctr->unlink_free_block)(allctr, nxt_nxt_blk, alcu_flgs); SET_BLK_SZ(nxt_blk, nxt_blk_sz); is_last_blk = IS_LAST_BLK(nxt_nxt_blk); @@ -979,7 +1055,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs) } } - (*allctr->link_free_block)(allctr, nxt_blk); + (*allctr->link_free_block)(allctr, nxt_blk, alcu_flgs); ASSERT(IS_ALLOCED_BLK(blk)); @@ -1009,12 +1085,12 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs) if (!is_last_blk) { nxt_blk = NXT_BLK(blk); nxt_blk_sz = BLK_SZ(nxt_blk); - if (IS_FREE_BLK(nxt_blk) && blk_sz <= old_blk_sz + nxt_blk_sz) { + if (IS_FREE_BLK(nxt_blk) && get_blk_sz <= old_blk_sz + nxt_blk_sz) { /* Grow into next block... */ HARD_CHECK_BLK_CARRIER(allctr, blk); - (*allctr->unlink_free_block)(allctr, nxt_blk); + (*allctr->unlink_free_block)(allctr, nxt_blk, alcu_flgs); nxt_blk_sz -= blk_sz - old_blk_sz; is_last_blk = IS_LAST_BLK(nxt_blk); @@ -1051,13 +1127,13 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs) else SET_BLK_SZ_FTR(nxt_blk, nxt_blk_sz); - (*allctr->link_free_block)(allctr, nxt_blk); + (*allctr->link_free_block)(allctr, nxt_blk, alcu_flgs); ASSERT(IS_FREE_BLK(nxt_blk)); } - STAT_MBC_BLK_FREE(allctr, old_blk_sz); - STAT_MBC_BLK_ALLOC(allctr, blk_sz); + STAT_MBC_BLK_FREE(allctr, old_blk_sz, alcu_flgs); + STAT_MBC_BLK_ALLOC(allctr, blk_sz, alcu_flgs); ASSERT(IS_ALLOCED_BLK(blk)); @@ -1088,7 +1164,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs) } } - if (flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE) + if (alcu_flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE) return NULL; /* Need to grow in another block */ @@ -1108,7 +1184,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs) } } - if (cand_blk_sz < blk_sz) { + if (cand_blk_sz < get_blk_sz) { /* We wont fit in cand_blk get a new one */ #endif /* !MBC_REALLOC_ALWAYS_MOVES */ @@ -1127,9 +1203,10 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs) /* We will at least fit in cand_blk */ new_blk = (*allctr->get_free_block)(allctr, - blk_sz, + get_blk_sz, cand_blk, - cand_blk_sz); + cand_blk_sz, + alcu_flgs); move_into_new_blk: /* * new_blk, and cand_blk have to be correctly set @@ -1142,7 +1219,8 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs) BLK_SZ(new_blk), GET_BLK_HDR_FLGS(new_blk), blk_sz, - 1); + 1, + alcu_flgs); new_p = BLK2UMEM(new_blk); sys_memcpy(new_p, p, MIN(size, old_blk_sz - ABLK_HDR_SZ)); mbc_free(allctr, p); @@ -1164,7 +1242,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs) HARD_CHECK_BLK_CARRIER(allctr, blk); - (*allctr->unlink_free_block)(allctr, new_blk); /* prev */ + (*allctr->unlink_free_block)(allctr, new_blk, alcu_flgs); /* prev */ if (is_last_blk) new_blk_flgs |= LAST_BLK_HDR_FLG; @@ -1173,7 +1251,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs) if (IS_FREE_BLK(nxt_blk)) { new_blk_flgs |= GET_LAST_BLK_HDR_FLG(nxt_blk); new_blk_sz += BLK_SZ(nxt_blk); - (*allctr->unlink_free_block)(allctr, nxt_blk); + (*allctr->unlink_free_block)(allctr, nxt_blk, alcu_flgs); } } @@ -1196,9 +1274,10 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs) new_blk_sz, new_blk_flgs, blk_sz, - 0); + 0, + alcu_flgs); - STAT_MBC_BLK_FREE(allctr, old_blk_sz); + STAT_MBC_BLK_FREE(allctr, old_blk_sz, alcu_flgs); return new_p; } @@ -1243,6 +1322,100 @@ do { \ #define CHECK_1BLK_CARRIER(A, SBC, MSEGED, C, CSZ, B, BSZ) #endif +static Block_t * +create_sbmbc(Allctr_t *allctr, Uint umem_sz) +{ + Block_t *blk; + Uint blk_sz; + Uint crr_sz = allctr->sbmbc_size; + Carrier_t *crr; + +#if HALFWORD_HEAP + if (allctr->mseg_opt.low_mem) + crr = erts_alloc(ERTS_ALC_T_SBMBC_LOW, crr_sz); + else +#endif + crr = erts_alloc(ERTS_ALC_T_SBMBC, crr_sz); + + INC_CC(allctr->calls.sbmbc_alloc); + SET_CARRIER_HDR(crr, crr_sz, SCH_SYS_ALLOC|SCH_MBC); + + blk = MBC2FBLK(allctr, crr); + +#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG + if (allctr->mbc_header_size % sizeof(Unit_t) == 0) + crr_sz -= sizeof(UWord); +#endif + + blk_sz = UNIT_FLOOR(crr_sz - allctr->mbc_header_size); + + SET_MBC_BLK_FTR(((UWord *) blk)[-1]); + SET_BLK_HDR(blk, blk_sz, SBH_THIS_FREE|SBH_PREV_FREE|SBH_LAST_BLK); + +#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG + *((Carrier_t **) NXT_BLK(blk)) = crr; +#endif + + link_carrier(&allctr->sbmbc_list, crr); + +#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG + if (allctr->mbc_header_size % sizeof(Unit_t) == 0) + crr_sz += sizeof(UWord); +#endif + + STAT_SBMBC_ALLOC(allctr, crr_sz); + CHECK_1BLK_CARRIER(allctr, 0, 0, crr, crr_sz, blk, blk_sz); +#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG + if (allctr->mbc_header_size % sizeof(Unit_t) == 0) + crr_sz -= sizeof(UWord); +#endif + if (allctr->creating_mbc) + (*allctr->creating_mbc)(allctr, crr, ERTS_ALCU_FLG_SBMBC); + + DEBUG_SAVE_ALIGNMENT(crr); + return blk; +} + +static void +destroy_sbmbc(Allctr_t *allctr, Block_t *blk) +{ + Uint crr_sz; + Carrier_t *crr; + + ASSERT(IS_FIRST_BLK(blk)); + + ASSERT(IS_MBC_BLK(blk)); + + crr = FBLK2MBC(allctr, blk); + crr_sz = CARRIER_SZ(crr); + +#ifdef DEBUG + if (!allctr->stopped) { + ASSERT(IS_LAST_BLK(blk)); + +#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG + (*allctr->link_free_block)(allctr, blk, ERTS_ALCU_FLG_SBMBC); + HARD_CHECK_BLK_CARRIER(allctr, blk); + (*allctr->unlink_free_block)(allctr, blk, ERTS_ALCU_FLG_SBMBC); +#endif + } +#endif + + STAT_SBMBC_FREE(allctr, crr_sz); + + unlink_carrier(&allctr->sbmbc_list, crr); + if (allctr->destroying_mbc) + (*allctr->destroying_mbc)(allctr, crr, ERTS_ALCU_FLG_SBMBC); + + INC_CC(allctr->calls.sbmbc_free); + +#if HALFWORD_HEAP + if (allctr->mseg_opt.low_mem) + erts_free(ERTS_ALC_T_SBMBC_LOW, crr); + else +#endif + erts_free(ERTS_ALC_T_SBMBC, crr); +} static Block_t * create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) @@ -1271,11 +1444,11 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) if (erts_mseg_no() >= max_mseg_carriers) goto try_sys_alloc; if (flags & CFLG_SBC) { - if (allctr->sbcs.curr_mseg.no >= allctr->max_mseg_sbcs) + if (allctr->sbcs.curr.norm.mseg.no >= allctr->max_mseg_sbcs) goto try_sys_alloc; } else { - if (allctr->mbcs.curr_mseg.no >= allctr->max_mseg_mbcs) + if (allctr->mbcs.curr.norm.mseg.no >= allctr->max_mseg_mbcs) goto try_sys_alloc; } @@ -1289,7 +1462,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) if (crr_sz < allctr->mbc_header_size + blk_sz) crr_sz = allctr->mbc_header_size + blk_sz; #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG - if (sizeof(Unit_t) == sizeof(UWord)) + if (allctr->mbc_header_size % sizeof(Unit_t) == 0) crr_sz += sizeof(UWord); #endif } @@ -1330,7 +1503,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) && bcrr_sz < allctr->smallest_mbc_size) bcrr_sz = allctr->smallest_mbc_size; #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG - if (sizeof(Unit_t) == sizeof(UWord)) + if (allctr->mbc_header_size % sizeof(Unit_t) == 0) bcrr_sz += sizeof(UWord); #endif @@ -1385,7 +1558,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) blk = MBC2FBLK(allctr, crr); #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG - if (sizeof(Unit_t) == sizeof(UWord)) + if (allctr->mbc_header_size % sizeof(Unit_t) == 0) crr_sz -= sizeof(UWord); #endif @@ -1406,16 +1579,16 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) link_carrier(&allctr->mbc_list, crr); #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG - if (sizeof(Unit_t) == sizeof(UWord)) + if (allctr->mbc_header_size % sizeof(Unit_t) == 0) crr_sz += sizeof(UWord); #endif CHECK_1BLK_CARRIER(allctr, 0, is_mseg, crr, crr_sz, blk, blk_sz); #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG - if (sizeof(Unit_t) == sizeof(UWord)) + if (allctr->mbc_header_size % sizeof(Unit_t) == 0) crr_sz -= sizeof(UWord); #endif if (allctr->creating_mbc) - (*allctr->creating_mbc)(allctr, crr); + (*allctr->creating_mbc)(allctr, crr, 0); } @@ -1595,9 +1768,9 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk) ASSERT(IS_LAST_BLK(blk)); #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG - (*allctr->link_free_block)(allctr, blk); + (*allctr->link_free_block)(allctr, blk, 0); HARD_CHECK_BLK_CARRIER(allctr, blk); - (*allctr->unlink_free_block)(allctr, blk); + (*allctr->unlink_free_block)(allctr, blk, 0); #endif } #endif @@ -1614,7 +1787,7 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk) unlink_carrier(&allctr->mbc_list, crr); if (allctr->destroying_mbc) - (*allctr->destroying_mbc)(allctr, crr); + (*allctr->destroying_mbc)(allctr, crr, 0); } @@ -1639,6 +1812,9 @@ static struct { Eterm e; Eterm t; Eterm ramv; +#if HALFWORD_HEAP + Eterm low; +#endif Eterm sbct; #if HAVE_ERTS_MSEG Eterm asbcst; @@ -1655,12 +1831,15 @@ static struct { Eterm lmbcs; Eterm smbcs; Eterm mbcgs; + Eterm sbmbcs; + Eterm sbmbct; #if HAVE_ERTS_MSEG Eterm mmc; #endif Eterm ycs; + /* Eterm sbmbcs; */ Eterm mbcs; Eterm sbcs; Eterm sys_alloc_carriers_size; @@ -1685,6 +1864,8 @@ static struct { Eterm mseg_dealloc; Eterm mseg_realloc; #endif + Eterm sbmbc_alloc; + Eterm sbmbc_free; #ifdef DEBUG Eterm end_of_atoms; #endif @@ -1724,6 +1905,9 @@ init_atoms(Allctr_t *allctr) AM_INIT(e); AM_INIT(t); AM_INIT(ramv); +#if HALFWORD_HEAP + AM_INIT(low); +#endif AM_INIT(sbct); #if HAVE_ERTS_MSEG AM_INIT(asbcst); @@ -1740,12 +1924,15 @@ init_atoms(Allctr_t *allctr) AM_INIT(lmbcs); AM_INIT(smbcs); AM_INIT(mbcgs); + AM_INIT(sbmbcs); + AM_INIT(sbmbct); #if HAVE_ERTS_MSEG AM_INIT(mmc); #endif AM_INIT(ycs); + /*AM_INIT(sbmbcs);*/ AM_INIT(mbcs); AM_INIT(sbcs); AM_INIT(sys_alloc_carriers_size); @@ -1770,6 +1957,8 @@ init_atoms(Allctr_t *allctr) AM_INIT(mseg_dealloc); AM_INIT(mseg_realloc); #endif + AM_INIT(sbmbc_free); + AM_INIT(sbmbc_alloc); #ifdef DEBUG for (atom = (Eterm *) &am; atom < &am.end_of_atoms; atom++) { @@ -1863,7 +2052,9 @@ sz_info_carriers(Allctr_t *allctr, Uint *szp) { Eterm res = THE_NON_VALUE; - Uint curr_size = cs->curr_mseg.size + cs->curr_sys_alloc.size; + Uint curr_size = (cs == &allctr->sbmbcs + ? cs->curr.small_block.size + : cs->curr.norm.mseg.size + cs->curr.norm.sys_alloc.size); if (print_to_p) { int to = *print_to_p; @@ -1911,8 +2102,17 @@ info_carriers(Allctr_t *allctr, Uint *szp) { Eterm res = THE_NON_VALUE; - Uint curr_no = cs->curr_mseg.no + cs->curr_sys_alloc.no; - Uint curr_size = cs->curr_mseg.size + cs->curr_sys_alloc.size; + Uint curr_no, curr_size; + int small_block = cs == &allctr->sbmbcs; + + if (small_block) { + curr_no = cs->curr.small_block.no; + curr_size = cs->curr.small_block.size; + } + else { + curr_no = cs->curr.norm.mseg.no + cs->curr.norm.sys_alloc.no; + curr_size = cs->curr.norm.mseg.size + cs->curr.norm.sys_alloc.size; + } if (print_to_p) { int to = *print_to_p; @@ -1938,18 +2138,20 @@ info_carriers(Allctr_t *allctr, curr_no, cs->max.no, cs->max_ever.no); + if (!small_block) { #if HAVE_ERTS_MSEG - erts_print(to, - arg, - "%smseg carriers: %bpu\n", - prefix, - cs->curr_mseg.no); -#endif - erts_print(to, - arg, - "%ssys_alloc carriers: %bpu\n", - prefix, - cs->curr_sys_alloc.no); + erts_print(to, + arg, + "%smseg carriers: %bpu\n", + prefix, + cs->curr.norm.mseg.no); +#endif + erts_print(to, + arg, + "%ssys_alloc carriers: %bpu\n", + prefix, + cs->curr.norm.sys_alloc.no); + } erts_print(to, arg, "%scarriers size: %beu %bpu %bpu\n", @@ -1957,43 +2159,49 @@ info_carriers(Allctr_t *allctr, curr_size, cs->max.size, cs->max_ever.size); + if (!small_block) { #if HAVE_ERTS_MSEG - erts_print(to, - arg, - "%smseg carriers size: %bpu\n", - prefix, - cs->curr_mseg.size); -#endif - erts_print(to, - arg, - "%ssys_alloc carriers size: %bpu\n", - prefix, - cs->curr_sys_alloc.size); + erts_print(to, + arg, + "%smseg carriers size: %bpu\n", + prefix, + cs->curr.norm.mseg.size); +#endif + erts_print(to, + arg, + "%ssys_alloc carriers size: %bpu\n", + prefix, + cs->curr.norm.sys_alloc.size); + } } if (hpp || szp) { res = NIL; - add_2tup(hpp, szp, &res, - am.sys_alloc_carriers_size, - bld_unstable_uint(hpp, szp, cs->curr_sys_alloc.size)); + if (!small_block) { + add_2tup(hpp, szp, &res, + am.sys_alloc_carriers_size, + bld_unstable_uint(hpp, szp, cs->curr.norm.sys_alloc.size)); #if HAVE_ERTS_MSEG - add_2tup(hpp, szp, &res, - am.mseg_alloc_carriers_size, - bld_unstable_uint(hpp, szp, cs->curr_mseg.size)); + add_2tup(hpp, szp, &res, + am.mseg_alloc_carriers_size, + bld_unstable_uint(hpp, szp, cs->curr.norm.mseg.size)); #endif + } add_4tup(hpp, szp, &res, am.carriers_size, bld_unstable_uint(hpp, szp, curr_size), bld_unstable_uint(hpp, szp, cs->max.size), bld_unstable_uint(hpp, szp, cs->max_ever.size)); - add_2tup(hpp, szp, &res, - am.sys_alloc_carriers, - bld_unstable_uint(hpp, szp, cs->curr_sys_alloc.no)); + if (!small_block) { + add_2tup(hpp, szp, &res, + am.sys_alloc_carriers, + bld_unstable_uint(hpp, szp, cs->curr.norm.sys_alloc.no)); #if HAVE_ERTS_MSEG - add_2tup(hpp, szp, &res, - am.mseg_alloc_carriers, - bld_unstable_uint(hpp, szp, cs->curr_mseg.no)); + add_2tup(hpp, szp, &res, + am.mseg_alloc_carriers, + bld_unstable_uint(hpp, szp, cs->curr.norm.mseg.no)); #endif + } add_4tup(hpp, szp, &res, am.carriers, bld_unstable_uint(hpp, szp, curr_no), @@ -2071,6 +2279,9 @@ info_calls(Allctr_t *allctr, PRINT_CC_5(to, arg, prefix, "free", allctr->calls.this_free); PRINT_CC_5(to, arg, prefix, "realloc", allctr->calls.this_realloc); + PRINT_CC_4(to, arg, "sbmbc_alloc", allctr->calls.sbmbc_alloc); + PRINT_CC_4(to, arg, "sbmbc_free", allctr->calls.sbmbc_free); + #if HAVE_ERTS_MSEG PRINT_CC_4(to, arg, "mseg_alloc", allctr->calls.mseg_alloc); PRINT_CC_4(to, arg, "mseg_dealloc", allctr->calls.mseg_dealloc); @@ -2122,6 +2333,14 @@ info_calls(Allctr_t *allctr, bld_unstable_uint(hpp, szp, allctr->calls.mseg_alloc.no)); #endif add_3tup(hpp, szp, &res, + am.sbmbc_free, + bld_unstable_uint(hpp, szp, allctr->calls.sbmbc_free.giga_no), + bld_unstable_uint(hpp, szp, allctr->calls.sbmbc_free.no)); + add_3tup(hpp, szp, &res, + am.sbmbc_alloc, + bld_unstable_uint(hpp, szp, allctr->calls.sbmbc_alloc.giga_no), + bld_unstable_uint(hpp, szp, allctr->calls.sbmbc_alloc.no)); + add_3tup(hpp, szp, &res, allctr->name.realloc, bld_unstable_uint(hpp, szp, allctr->calls.this_realloc.giga_no), bld_unstable_uint(hpp, szp, allctr->calls.this_realloc.no)); @@ -2168,6 +2387,9 @@ info_options(Allctr_t *allctr, "option e: true\n" "option t: %s\n" "option ramv: %s\n" +#if HALFWORD_HEAP + "option low: %s\n" +#endif "option sbct: %beu\n" #if HAVE_ERTS_MSEG "option asbcst: %bpu\n" @@ -2182,9 +2404,14 @@ info_options(Allctr_t *allctr, #endif "option lmbcs: %beu\n" "option smbcs: %beu\n" - "option mbcgs: %beu\n", + "option mbcgs: %beu\n" + "option sbmbcs: %beu\n" + "option sbmbct: %beu\n", topt, allctr->ramv ? "true" : "false", +#if HALFWORD_HEAP + allctr->mseg_opt.low_mem ? "true" : "false", +#endif allctr->sbc_threshold, #if HAVE_ERTS_MSEG allctr->mseg_opt.abs_shrink_th, @@ -2199,7 +2426,9 @@ info_options(Allctr_t *allctr, #endif allctr->largest_mbc_size, allctr->smallest_mbc_size, - allctr->mbc_growth_stages); + allctr->mbc_growth_stages, + allctr->sbmbc_size, + allctr->sbmbc_threshold); } res = (*allctr->info_options)(allctr, "option ", print_to_p, print_to_arg, @@ -2207,6 +2436,12 @@ info_options(Allctr_t *allctr, if (hpp || szp) { add_2tup(hpp, szp, &res, + am.sbmbct, + bld_uint(hpp, szp, allctr->sbmbc_threshold)); + add_2tup(hpp, szp, &res, + am.sbmbcs, + bld_uint(hpp, szp, allctr->sbmbc_size)); + add_2tup(hpp, szp, &res, am.mbcgs, bld_uint(hpp, szp, allctr->mbc_growth_stages)); add_2tup(hpp, szp, &res, @@ -2243,6 +2478,9 @@ info_options(Allctr_t *allctr, add_2tup(hpp, szp, &res, am.sbct, bld_uint(hpp, szp, allctr->sbc_threshold)); +#if HALFWORD_HEAP + add_2tup(hpp, szp, &res, am.low, allctr->mseg_opt.low_mem ? am_true : am_false); +#endif add_2tup(hpp, szp, &res, am.ramv, allctr->ramv ? am_true : am_false); add_2tup(hpp, szp, &res, am.t, (allctr->t ? bld_uint(hpp, szp, (Uint) allctr->t) @@ -2270,10 +2508,10 @@ update_max_ever_values(CarriersStats_t *cs) static ERTS_INLINE void reset_max_values(CarriersStats_t *cs) { - cs->max.no = cs->curr_mseg.no + cs->curr_sys_alloc.no; - cs->max.size = cs->curr_mseg.size + cs->curr_sys_alloc.size; - cs->blocks.max.no = cs->blocks.curr.no; - cs->blocks.max.size = cs->blocks.curr.size; + cs->max.no = cs->curr.norm.mseg.no + cs->curr.norm.sys_alloc.no; + cs->max.size = cs->curr.norm.mseg.size + cs->curr.norm.sys_alloc.size; + cs->blocks.max.no = cs->blocks.curr.no; + cs->blocks.max.size = cs->blocks.curr.size; } @@ -2352,7 +2590,7 @@ erts_alcu_sz_info(Allctr_t *allctr, Uint **hpp, Uint *szp) { - Eterm res, mbcs, sbcs; + Eterm res, sbmbcs, mbcs, sbcs; res = THE_NON_VALUE; @@ -2374,24 +2612,29 @@ erts_alcu_sz_info(Allctr_t *allctr, /* Update sbc values not continously updated */ allctr->sbcs.blocks.curr.no - = allctr->sbcs.curr_mseg.no + allctr->sbcs.curr_sys_alloc.no; + = allctr->sbcs.curr.norm.mseg.no + allctr->sbcs.curr.norm.sys_alloc.no; allctr->sbcs.blocks.max.no = allctr->sbcs.max.no; + update_max_ever_values(&allctr->sbmbcs); update_max_ever_values(&allctr->mbcs); update_max_ever_values(&allctr->sbcs); - mbcs = sz_info_carriers(allctr, &allctr->mbcs, "mbcs ", print_to_p, - print_to_arg, hpp, szp); - sbcs = sz_info_carriers(allctr, &allctr->sbcs, "sbcs ", print_to_p, - print_to_arg, hpp, szp); + sbmbcs = sz_info_carriers(allctr, &allctr->sbmbcs, "sbmbcs ", print_to_p, + print_to_arg, hpp, szp); + mbcs = sz_info_carriers(allctr, &allctr->mbcs, "mbcs ", print_to_p, + print_to_arg, hpp, szp); + sbcs = sz_info_carriers(allctr, &allctr->sbcs, "sbcs ", print_to_p, + print_to_arg, hpp, szp); if (hpp || szp) { res = NIL; add_2tup(hpp, szp, &res, am.sbcs, sbcs); add_2tup(hpp, szp, &res, am.mbcs, mbcs); + add_2tup(hpp, szp, &res, am.sbmbcs, sbmbcs); } if (begin_max_period) { + reset_max_values(&allctr->sbmbcs); reset_max_values(&allctr->mbcs); reset_max_values(&allctr->sbcs); } @@ -2413,7 +2656,7 @@ erts_alcu_info(Allctr_t *allctr, Uint **hpp, Uint *szp) { - Eterm res, sett, mbcs, sbcs, calls; + Eterm res, sett, sbmbcs, mbcs, sbcs, calls; res = THE_NON_VALUE; @@ -2435,9 +2678,10 @@ erts_alcu_info(Allctr_t *allctr, /* Update sbc values not continously updated */ allctr->sbcs.blocks.curr.no - = allctr->sbcs.curr_mseg.no + allctr->sbcs.curr_sys_alloc.no; + = allctr->sbcs.curr.norm.mseg.no + allctr->sbcs.curr.norm.sys_alloc.no; allctr->sbcs.blocks.max.no = allctr->sbcs.max.no; + update_max_ever_values(&allctr->sbmbcs); update_max_ever_values(&allctr->mbcs); update_max_ever_values(&allctr->sbcs); @@ -2449,11 +2693,13 @@ erts_alcu_info(Allctr_t *allctr, ERTS_ALCU_VSN_STR); } - sett = info_options(allctr, print_to_p, print_to_arg, hpp, szp); - mbcs = info_carriers(allctr, &allctr->mbcs, "mbcs ", print_to_p, - print_to_arg, hpp, szp); - sbcs = info_carriers(allctr, &allctr->sbcs, "sbcs ", print_to_p, - print_to_arg, hpp, szp); + sett = info_options(allctr, print_to_p, print_to_arg, hpp, szp); + sbmbcs = info_carriers(allctr, &allctr->sbmbcs, "sbmbcs ", print_to_p, + print_to_arg, hpp, szp); + mbcs = info_carriers(allctr, &allctr->mbcs, "mbcs ", print_to_p, + print_to_arg, hpp, szp); + sbcs = info_carriers(allctr, &allctr->sbcs, "sbcs ", print_to_p, + print_to_arg, hpp, szp); calls = info_calls(allctr, print_to_p, print_to_arg, hpp, szp); if (hpp || szp) { @@ -2462,6 +2708,7 @@ erts_alcu_info(Allctr_t *allctr, add_2tup(hpp, szp, &res, am.calls, calls); add_2tup(hpp, szp, &res, am.sbcs, sbcs); add_2tup(hpp, szp, &res, am.mbcs, mbcs); + add_2tup(hpp, szp, &res, am.sbmbcs, sbmbcs); add_2tup(hpp, szp, &res, am.options, sett); add_3tup(hpp, szp, &res, am.versions, @@ -2470,6 +2717,7 @@ erts_alcu_info(Allctr_t *allctr, } if (begin_max_period) { + reset_max_values(&allctr->sbmbcs); reset_max_values(&allctr->mbcs); reset_max_values(&allctr->sbcs); } @@ -2493,12 +2741,14 @@ erts_alcu_current_size(Allctr_t *allctr, AllctrSize_t *size) erts_mtx_lock(&allctr->mutex); #endif - size->carriers = allctr->mbcs.curr_mseg.size; - size->carriers += allctr->mbcs.curr_sys_alloc.size; - size->carriers += allctr->sbcs.curr_mseg.size; - size->carriers += allctr->sbcs.curr_sys_alloc.size; + size->carriers = allctr->mbcs.curr.norm.mseg.size; + size->carriers += allctr->mbcs.curr.norm.sys_alloc.size; + size->carriers += allctr->sbmbcs.curr.small_block.size; + size->carriers += allctr->sbcs.curr.norm.mseg.size; + size->carriers += allctr->sbcs.curr.norm.sys_alloc.size; size->blocks = allctr->mbcs.blocks.curr.size; + size->blocks += allctr->sbmbcs.blocks.curr.size; size->blocks += allctr->sbcs.blocks.curr.size; #ifdef USE_THREADS @@ -2710,7 +2960,7 @@ do_erts_alcu_realloc(ErtsAlcType_t type, void *extra, void *p, Uint size, - UWord flgs) + Uint32 alcu_flgs) { Allctr_t *allctr = (Allctr_t *) extra; Block_t *blk; @@ -2743,9 +2993,32 @@ do_erts_alcu_realloc(ErtsAlcType_t type, blk = UMEM2BLK(p); + if (allctr->sbmbc_threshold > 0) { + Uint old_sz, new_sz, lim; + lim = allctr->sbmbc_threshold; + old_sz = BLK_SZ(blk); + new_sz = UMEMSZ2BLKSZ(allctr, size); + if ((old_sz < lim && lim <= new_sz) + || (new_sz < lim && lim <= old_sz)) { + /* *Need* to move it... */ + + INC_CC(allctr->calls.this_realloc); + res = do_erts_alcu_alloc(type, extra, size); + DEC_CC(allctr->calls.this_alloc); + + sys_memcpy(res, p, MIN(size, old_sz - ABLK_HDR_SZ)); + + do_erts_alcu_free(type, extra, p); + DEC_CC(allctr->calls.this_free); + return res; + } + if (old_sz < lim) + alcu_flgs |= ERTS_ALCU_FLG_SBMBC; + } + if (size < allctr->sbc_threshold) { if (IS_MBC_BLK(blk)) - res = mbc_realloc(allctr, p, size, flgs); + res = mbc_realloc(allctr, p, size, alcu_flgs); else { Uint used_sz = allctr->sbc_header_size + ABLK_HDR_SZ + size; Uint crr_sz; @@ -2776,7 +3049,7 @@ do_erts_alcu_realloc(ErtsAlcType_t type, if (100*diff_sz_val < allctr->sbc_move_threshold*crr_sz_val) /* Data won't be copied into a new carrier... */ goto do_carrier_resize; - else if (flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE) + else if (alcu_flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE) return NULL; res = mbc_alloc(allctr, size); @@ -2799,7 +3072,7 @@ do_erts_alcu_realloc(ErtsAlcType_t type, #endif res = new_blk ? BLK2UMEM(new_blk) : NULL; } - else if (flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE) + else if (alcu_flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE) return NULL; else { #if HALFWORD_HEAP @@ -3105,13 +3378,12 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) goto error; #if HAVE_ERTS_MSEG - { - ErtsMsegOpt_t mseg_opt = ERTS_MSEG_DEFAULT_OPT_INITIALIZER; - - sys_memcpy((void *) &allctr->mseg_opt, - (void *) &mseg_opt, - sizeof(ErtsMsegOpt_t)); - } + sys_memcpy((void *) &allctr->mseg_opt, + (void *) &erts_mseg_default_opt, + sizeof(ErtsMsegOpt_t)); +# if HALFWORD_HEAP + allctr->mseg_opt.low_mem = init->low_mem; +# endif #endif allctr->name_prefix = init->name_prefix; @@ -3160,6 +3432,26 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->min_block_size = UNIT_CEILING(allctr->min_block_size + sizeof(UWord)); + + allctr->sbmbc_threshold = init->sbmbct; + + if (!erts_have_sbmbc_alloc + || ERTS_IS_SBMBC_ALLOCATOR_NO__(allctr->alloc_no)) + allctr->sbmbc_threshold = 0; + + if (!allctr->sbmbc_threshold) + allctr->sbmbc_size = 0; + else { + Uint min_size; + allctr->sbmbc_size = init->sbmbcs; + min_size = allctr->sbmbc_threshold; + min_size += allctr->min_block_size; + min_size += allctr->mbc_header_size; + if (allctr->sbmbc_size < min_size) + allctr->sbmbc_size = min_size; + } + + #if HAVE_ERTS_MSEG if (allctr->mseg_opt.abs_shrink_th > ~((UWord) 0) / 100) allctr->mseg_opt.abs_shrink_th = ~((UWord) 0) / 100; @@ -3171,12 +3463,16 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) #ifdef ERTS_ENABLE_LOCK_COUNT erts_mtx_init_x_opt(&allctr->mutex, - "alcu_allocator", - make_small(allctr->alloc_no), - ERTS_LCNT_LT_ALLOC); + ERTS_IS_SBMBC_ALLOCATOR_NO__(allctr->alloc_no) + ? "sbmbc_alloc" + : "alcu_allocator", + make_small(allctr->alloc_no), + ERTS_LCNT_LT_ALLOC); #else erts_mtx_init_x(&allctr->mutex, - "alcu_allocator", + ERTS_IS_SBMBC_ALLOCATOR_NO__(allctr->alloc_no) + ? "sbmbc_alloc" + : "alcu_allocator", make_small(allctr->alloc_no)); #endif /*ERTS_ENABLE_LOCK_COUNT*/ @@ -3246,7 +3542,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) if (!blk) goto error; - (*allctr->link_free_block)(allctr, blk); + (*allctr->link_free_block)(allctr, blk, 0); HARD_CHECK_BLK_CARRIER(allctr, blk); @@ -3276,6 +3572,8 @@ erts_alcu_stop(Allctr_t *allctr) destroy_carrier(allctr, SBC2BLK(allctr, allctr->sbc_list.first)); while (allctr->mbc_list.first) destroy_carrier(allctr, MBC2FBLK(allctr, allctr->mbc_list.first)); + while (allctr->sbmbc_list.first) + destroy_sbmbc(allctr, MBC2FBLK(allctr, allctr->sbmbc_list.first)); #ifdef USE_THREADS if (allctr->thread_safe) @@ -3373,13 +3671,15 @@ erts_alcu_verify_unused(Allctr_t *allctr) { UWord no; - no = allctr->sbcs.curr_mseg.no; - no += allctr->sbcs.curr_sys_alloc.no; + no = allctr->sbcs.curr.norm.mseg.no; + no += allctr->sbcs.curr.norm.sys_alloc.no; no += allctr->mbcs.blocks.curr.no; + no += allctr->sbmbcs.blocks.curr.no; if (no) { UWord sz = allctr->sbcs.blocks.curr.size; sz += allctr->mbcs.blocks.curr.size; + sz += allctr->sbmbcs.blocks.curr.size; erl_exit(ERTS_ABORT_EXIT, "%salloc() used when expected to be unused!\n" "Total amount of blocks allocated: %bpu\n" @@ -3478,7 +3778,7 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk) (*allctr->check_block)(allctr, blk, (int) is_free_blk); if (IS_LAST_BLK(blk)) { - carrier_end = ((char *) NXT_BLK(blk)) + sizeof(UWord); + carrier_end = ((char *) NXT_BLK(blk)); mbc = *((Carrier_t **) NXT_BLK(blk)); prev_blk = NULL; blk = MBC2FBLK(allctr, mbc); @@ -3493,9 +3793,9 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk) ASSERT(IS_MB_CARRIER(mbc)); ASSERT((((char *) mbc) + allctr->mbc_header_size - + tot_blk_sz - + sizeof(UWord)) == carrier_end); - ASSERT(((char *) mbc) + CARRIER_SZ(mbc) == carrier_end); + + tot_blk_sz) == carrier_end); + ASSERT(((char *) mbc) + CARRIER_SZ(mbc) - sizeof(Unit_t) <= carrier_end + && carrier_end <= ((char *) mbc) + CARRIER_SZ(mbc)); if (allctr->check_mbc) (*allctr->check_mbc)(allctr, mbc); @@ -3509,6 +3809,7 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk) cl = &allctr->mbc_list; } +#if 0 /* FIXIT sbmbc */ if (cl->first == crr) { ASSERT(!crr->prev); } @@ -3523,6 +3824,7 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk) ASSERT(crr->next); ASSERT(crr->next->prev == crr); } +#endif } #endif diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index d296081714..fed4d3dbe6 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -34,10 +34,12 @@ typedef struct { typedef struct { char *name_prefix; ErtsAlcType_t alloc_no; + int force; int ts; int tspec; int tpref; int ramv; + int low_mem; /* HALFWORD only */ UWord sbct; UWord asbcst; UWord rsbcst; @@ -49,6 +51,8 @@ typedef struct { UWord lmbcs; UWord smbcs; UWord mbcgs; + UWord sbmbct; + UWord sbmbcs; } AllctrInit_t; typedef struct { @@ -66,10 +70,12 @@ typedef struct { #define ERTS_DEFAULT_ALLCTR_INIT { \ NULL, \ ERTS_ALC_A_INVALID, /* (number) alloc_no: allocator number */\ + 0, /* (bool) force: force enabled */\ 1, /* (bool) ts: thread safe */\ 0, /* (bool) tspec: thread specific */\ 0, /* (bool) tpref: thread preferred */\ 0, /* (bool) ramv: realloc always moves */\ + 0, /* (bool) low_mem: HALFWORD only */\ 512*1024, /* (bytes) sbct: sbc threshold */\ 2*1024*2024, /* (amount) asbcst: abs sbc shrink threshold */\ 20, /* (%) rsbcst: rel sbc shrink threshold */\ @@ -80,7 +86,9 @@ typedef struct { 10, /* (amount) mmmbc: max mseg mbcs */\ 10*1024*1024, /* (bytes) lmbcs: largest mbc size */\ 1024*1024, /* (bytes) smbcs: smallest mbc size */\ - 10 /* (amount) mbcgs: mbc growth stages */\ + 10, /* (amount) mbcgs: mbc growth stages */\ + 256, /* (bytes) sbmbct: small block mbc threshold */\ + 8*1024 /* (bytes) sbmbcs: small block mbc size */\ } #else /* if SMALL_MEMORY */ @@ -93,10 +101,12 @@ typedef struct { #define ERTS_DEFAULT_ALLCTR_INIT { \ NULL, \ ERTS_ALC_A_INVALID, /* (number) alloc_no: allocator number */\ + 0, /* (bool) force: force enabled */\ 1, /* (bool) ts: thread safe */\ 0, /* (bool) tspec: thread specific */\ 0, /* (bool) tpref: thread preferred */\ 0, /* (bool) ramv: realloc always moves */\ + 0, /* (bool) low_mem: HALFWORD only */\ 64*1024, /* (bytes) sbct: sbc threshold */\ 2*1024*2024, /* (amount) asbcst: abs sbc shrink threshold */\ 20, /* (%) rsbcst: rel sbc shrink threshold */\ @@ -106,7 +116,9 @@ typedef struct { 10, /* (amount) mmmbc: max mseg mbcs */\ 1024*1024, /* (bytes) lmbcs: largest mbc size */\ 128*1024, /* (bytes) smbcs: smallest mbc size */\ - 10 /* (amount) mbcgs: mbc growth stages */\ + 10, /* (amount) mbcgs: mbc growth stages */\ + 256, /* (bytes) sbmbct: small block mbc threshold */\ + 8*1024 /* (bytes) sbmbcs: small block mbc size */\ } #endif @@ -141,6 +153,9 @@ void erts_alcu_current_size(Allctr_t *, AllctrSize_t *); #if defined(GET_ERL_ALLOC_UTIL_IMPL) && !defined(ERL_ALLOC_UTIL_IMPL__) #define ERL_ALLOC_UTIL_IMPL__ +#define ERTS_ALCU_FLG_FAIL_REALLOC_MOVE (((Uint32) 1) << 0) +#define ERTS_ALCU_FLG_SBMBC (((Uint32) 1) << 1) + #ifdef USE_THREADS #define ERL_THREADS_EMU_INTERNAL__ #include "erl_threads.h" @@ -185,6 +200,8 @@ void erts_alcu_current_size(Allctr_t *, AllctrSize_t *); #define CARRIER_SZ(C) \ ((C)->chdr & SZ_MASK) +extern int erts_have_sbmbc_alloc; + typedef union {char c[8]; long l; double d;} Unit_t; typedef struct Carrier_t_ Carrier_t; @@ -213,8 +230,13 @@ typedef struct { } StatValues_t; typedef struct { - StatValues_t curr_mseg; - StatValues_t curr_sys_alloc; + union { + struct { + StatValues_t mseg; + StatValues_t sys_alloc; + } norm; + StatValues_t small_block; + } curr; StatValues_t max; StatValues_t max_ever; struct { @@ -254,6 +276,8 @@ struct Allctr_t_ { Uint largest_mbc_size; Uint smallest_mbc_size; Uint mbc_growth_stages; + Uint sbmbc_threshold; + Uint sbmbc_size; #if HAVE_ERTS_MSEG ErtsMsegOpt_t mseg_opt; #endif @@ -266,6 +290,7 @@ struct Allctr_t_ { Uint min_block_size; /* Carriers */ + CarrierList_t sbmbc_list; CarrierList_t mbc_list; CarrierList_t sbc_list; @@ -274,15 +299,15 @@ struct Allctr_t_ { /* Callback functions (first 4 are mandatory) */ Block_t * (*get_free_block) (Allctr_t *, Uint, - Block_t *, Uint); - void (*link_free_block) (Allctr_t *, Block_t *); - void (*unlink_free_block) (Allctr_t *, Block_t *); + Block_t *, Uint, Uint32); + void (*link_free_block) (Allctr_t *, Block_t *, Uint32); + void (*unlink_free_block) (Allctr_t *, Block_t *, Uint32); Eterm (*info_options) (Allctr_t *, char *, int *, void *, Uint **, Uint *); Uint (*get_next_mbc_size) (Allctr_t *); - void (*creating_mbc) (Allctr_t *, Carrier_t *); - void (*destroying_mbc) (Allctr_t *, Carrier_t *); + void (*creating_mbc) (Allctr_t *, Carrier_t *, Uint32); + void (*destroying_mbc) (Allctr_t *, Carrier_t *, Uint32); void (*init_atoms) (void); #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG @@ -309,6 +334,8 @@ struct Allctr_t_ { CallCounter_t this_alloc; CallCounter_t this_free; CallCounter_t this_realloc; + CallCounter_t sbmbc_alloc; + CallCounter_t sbmbc_free; CallCounter_t mseg_alloc; CallCounter_t mseg_dealloc; CallCounter_t mseg_realloc; @@ -319,6 +346,7 @@ struct Allctr_t_ { CarriersStats_t sbcs; CarriersStats_t mbcs; + CarriersStats_t sbmbcs; #ifdef DEBUG #ifdef USE_THREADS diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.c b/erts/emulator/beam/erl_ao_firstfit_alloc.c new file mode 100644 index 0000000000..002852cdad --- /dev/null +++ b/erts/emulator/beam/erl_ao_firstfit_alloc.c @@ -0,0 +1,972 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2003-2009. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + + +/* + * Description: An "address order first fit" allocator + * based on a Red-Black (binary search) Tree. The search, + * insert, and delete operations are all O(log n) operations + * on a Red-Black Tree. + * Red-Black Trees are described in "Introduction to Algorithms", + * by Thomas H. Cormen, Charles E. Leiserson, and Ronald L. Riverest. + * + * This module is a callback-module for erl_alloc_util.c + * + * Algorithm: The tree nodes are free-blocks ordered in address order. + * Every node also keeps the size of the largest block in its + * sub-tree ('max_size'). By that we can start from root and keep + * left (for low addresses) while dismissing entire sub-trees with + * too small blocks. + * + * Authors: Rickard Green/Sverker Eriksson + */ + + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif +#include "global.h" +#define GET_ERL_AOFF_ALLOC_IMPL +#include "erl_ao_firstfit_alloc.h" + +#ifdef DEBUG +#if 0 +#define HARD_DEBUG +#endif +#else +#undef HARD_DEBUG +#endif + +#define MIN_MBC_SZ (16*1024) +#define MIN_MBC_FIRST_FREE_SZ (4*1024) + +#define TREE_NODE_FLG (((Uint) 1) << 0) +#define RED_FLG (((Uint) 1) << 1) +#ifdef HARD_DEBUG +# define LEFT_VISITED_FLG (((Uint) 1) << 2) +# define RIGHT_VISITED_FLG (((Uint) 1) << 3) +#endif + +#define IS_RED(N) (((AOFF_RBTree_t *) (N)) \ + && ((AOFF_RBTree_t *) (N))->flags & RED_FLG) +#define IS_BLACK(N) (!IS_RED(((AOFF_RBTree_t *) (N)))) + +#define SET_RED(N) (((AOFF_RBTree_t *) (N))->flags |= RED_FLG) +#define SET_BLACK(N) (((AOFF_RBTree_t *) (N))->flags &= ~RED_FLG) + +#undef ASSERT +#define ASSERT ASSERT_EXPR + +#if 1 +#define RBT_ASSERT ASSERT +#else +#define RBT_ASSERT(x) +#endif + + +/* Types... */ +typedef struct AOFF_RBTree_t_ AOFF_RBTree_t; + +struct AOFF_RBTree_t_ { + Block_t hdr; + Uint flags; + AOFF_RBTree_t *parent; + AOFF_RBTree_t *left; + AOFF_RBTree_t *right; + Uint max_sz; /* of all blocks in this sub-tree */ +}; + +#ifdef HARD_DEBUG +static AOFF_RBTree_t * check_tree(AOFF_RBTree_t* root, Uint); +#endif + + +/* Calculate 'max_size' of tree node x by only looking at the direct children + * of x and x itself. + */ +static ERTS_INLINE Uint node_max_size(AOFF_RBTree_t *x) +{ + Uint sz = BLK_SZ(x); + if (x->left && x->left->max_sz > sz) { + sz = x->left->max_sz; + } + if (x->right && x->right->max_sz > sz) { + sz = x->right->max_sz; + } + return sz; +} + +/* Set new possibly lower 'max_size' of node and propagate change toward root +*/ +static ERTS_INLINE void lower_max_size(AOFF_RBTree_t *node, + AOFF_RBTree_t* stop_at) +{ + AOFF_RBTree_t* x = node; + Uint old_max = x->max_sz; + Uint new_max = node_max_size(x); + + if (new_max < old_max) { + x->max_sz = new_max; + while ((x=x->parent) != stop_at && x->max_sz == old_max) { + x->max_sz = node_max_size(x); + } + ASSERT(x == stop_at || x->max_sz > old_max); + } + else ASSERT(new_max == old_max); +} + + +/* Prototypes of callback functions */ +static Block_t* aoff_get_free_block(Allctr_t *, Uint, Block_t *, Uint, Uint32 flags); +static void aoff_link_free_block(Allctr_t *, Block_t*, Uint32 flags); +static void aoff_unlink_free_block(Allctr_t *allctr, Block_t *del, Uint32 flags); + +static Eterm info_options(Allctr_t *, char *, int *, void *, Uint **, Uint *); +static void init_atoms(void); + + + +#ifdef DEBUG + +/* Destroy all tree fields */ +#define DESTROY_TREE_NODE(N) \ + sys_memset((void *) (((Block_t *) (N)) + 1), \ + 0xff, \ + (sizeof(AOFF_RBTree_t) - sizeof(Block_t))) + +#else + +#define DESTROY_TREE_NODE(N) + +#endif + + +static int atoms_initialized = 0; + +void +erts_aoffalc_init(void) +{ + atoms_initialized = 0; +} + +Allctr_t * +erts_aoffalc_start(AOFFAllctr_t *alc, + AOFFAllctrInit_t* aoffinit, + AllctrInit_t *init) +{ + AOFFAllctr_t nulled_state = {{0}}; + /* {{0}} is used instead of {0}, in order to avoid (an incorrect) gcc + warning. gcc warns if {0} is used as initializer of a struct when + the first member is a struct (not if, for example, the third member + is a struct). */ + Allctr_t *allctr = (Allctr_t *) alc; + + sys_memcpy((void *) alc, (void *) &nulled_state, sizeof(AOFFAllctr_t)); + + allctr->mbc_header_size = sizeof(Carrier_t); + allctr->min_mbc_size = MIN_MBC_SZ; + allctr->min_mbc_first_free_size = MIN_MBC_FIRST_FREE_SZ; + allctr->min_block_size = sizeof(AOFF_RBTree_t); + + allctr->vsn_str = ERTS_ALC_AOFF_ALLOC_VSN_STR; + + + /* Callback functions */ + + allctr->get_free_block = aoff_get_free_block; + allctr->link_free_block = aoff_link_free_block; + allctr->unlink_free_block = aoff_unlink_free_block; + allctr->info_options = info_options; + + allctr->get_next_mbc_size = NULL; + allctr->creating_mbc = NULL; + allctr->destroying_mbc = NULL; + allctr->init_atoms = init_atoms; + +#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG + allctr->check_block = NULL; + allctr->check_mbc = NULL; +#endif + + allctr->atoms_initialized = 0; + + if (!erts_alcu_start(allctr, init)) + return NULL; + + return allctr; +} + +/* + * Red-Black Tree operations needed + */ + +static ERTS_INLINE void +left_rotate(AOFF_RBTree_t **root, AOFF_RBTree_t *x) +{ + AOFF_RBTree_t *y = x->right; + x->right = y->left; + if (y->left) + y->left->parent = x; + y->parent = x->parent; + if (!y->parent) { + RBT_ASSERT(*root == x); + *root = y; + } + else if (x == x->parent->left) + x->parent->left = y; + else { + RBT_ASSERT(x == x->parent->right); + x->parent->right = y; + } + y->left = x; + x->parent = y; + + y->max_sz = x->max_sz; + x->max_sz = node_max_size(x); + ASSERT(y->max_sz >= x->max_sz); +} + +static ERTS_INLINE void +right_rotate(AOFF_RBTree_t **root, AOFF_RBTree_t *x) +{ + AOFF_RBTree_t *y = x->left; + x->left = y->right; + if (y->right) + y->right->parent = x; + y->parent = x->parent; + if (!y->parent) { + RBT_ASSERT(*root == x); + *root = y; + } + else if (x == x->parent->right) + x->parent->right = y; + else { + RBT_ASSERT(x == x->parent->left); + x->parent->left = y; + } + y->right = x; + x->parent = y; + y->max_sz = x->max_sz; + x->max_sz = node_max_size(x); + ASSERT(y->max_sz >= x->max_sz); +} + + +/* + * Replace node x with node y + * NOTE: block header of y is not changed + */ +static ERTS_INLINE void +replace(AOFF_RBTree_t **root, AOFF_RBTree_t *x, AOFF_RBTree_t *y) +{ + + if (!x->parent) { + RBT_ASSERT(*root == x); + *root = y; + } + else if (x == x->parent->left) + x->parent->left = y; + else { + RBT_ASSERT(x == x->parent->right); + x->parent->right = y; + } + if (x->left) { + RBT_ASSERT(x->left->parent == x); + x->left->parent = y; + } + if (x->right) { + RBT_ASSERT(x->right->parent == x); + x->right->parent = y; + } + + y->flags = x->flags; + y->parent = x->parent; + y->right = x->right; + y->left = x->left; + + y->max_sz = x->max_sz; + lower_max_size(y, NULL); + DESTROY_TREE_NODE(x); +} + +static void +tree_insert_fixup(AOFF_RBTree_t** root, AOFF_RBTree_t *blk) +{ + AOFF_RBTree_t *x = blk, *y; + + /* + * Rearrange the tree so that it satisfies the Red-Black Tree properties + */ + + RBT_ASSERT(x != *root && IS_RED(x->parent)); + do { + + /* + * x and its parent are both red. Move the red pair up the tree + * until we get to the root or until we can separate them. + */ + + RBT_ASSERT(IS_RED(x)); + RBT_ASSERT(IS_BLACK(x->parent->parent)); + RBT_ASSERT(x->parent->parent); + + if (x->parent == x->parent->parent->left) { + y = x->parent->parent->right; + if (IS_RED(y)) { + SET_BLACK(y); + x = x->parent; + SET_BLACK(x); + x = x->parent; + SET_RED(x); + } + else { + + if (x == x->parent->right) { + x = x->parent; + left_rotate(root, x); + } + + RBT_ASSERT(x == x->parent->parent->left->left); + RBT_ASSERT(IS_RED(x)); + RBT_ASSERT(IS_RED(x->parent)); + RBT_ASSERT(IS_BLACK(x->parent->parent)); + RBT_ASSERT(IS_BLACK(y)); + + SET_BLACK(x->parent); + SET_RED(x->parent->parent); + right_rotate(root, x->parent->parent); + + RBT_ASSERT(x == x->parent->left); + RBT_ASSERT(IS_RED(x)); + RBT_ASSERT(IS_RED(x->parent->right)); + RBT_ASSERT(IS_BLACK(x->parent)); + break; + } + } + else { + RBT_ASSERT(x->parent == x->parent->parent->right); + y = x->parent->parent->left; + if (IS_RED(y)) { + SET_BLACK(y); + x = x->parent; + SET_BLACK(x); + x = x->parent; + SET_RED(x); + } + else { + + if (x == x->parent->left) { + x = x->parent; + right_rotate(root, x); + } + + RBT_ASSERT(x == x->parent->parent->right->right); + RBT_ASSERT(IS_RED(x)); + RBT_ASSERT(IS_RED(x->parent)); + RBT_ASSERT(IS_BLACK(x->parent->parent)); + RBT_ASSERT(IS_BLACK(y)); + + SET_BLACK(x->parent); + SET_RED(x->parent->parent); + left_rotate(root, x->parent->parent); + + RBT_ASSERT(x == x->parent->right); + RBT_ASSERT(IS_RED(x)); + RBT_ASSERT(IS_RED(x->parent->left)); + RBT_ASSERT(IS_BLACK(x->parent)); + break; + } + } + } while (x != *root && IS_RED(x->parent)); + + SET_BLACK(*root); +} + +static void +aoff_unlink_free_block(Allctr_t *allctr, Block_t *del, Uint32 flags) +{ + AOFFAllctr_t *alc = (AOFFAllctr_t *) allctr; + AOFF_RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC) + ? &alc->sbmbc_root : &alc->mbc_root); + Uint spliced_is_black; + AOFF_RBTree_t *x, *y, *z = (AOFF_RBTree_t *) del; + AOFF_RBTree_t null_x; /* null_x is used to get the fixup started when we + splice out a node without children. */ + + null_x.parent = NULL; + +#ifdef HARD_DEBUG + check_tree(*root, 0); +#endif + + /* Remove node from tree... */ + + /* Find node to splice out */ + if (!z->left || !z->right) + y = z; + else + /* Set y to z:s successor */ + for(y = z->right; y->left; y = y->left); + /* splice out y */ + x = y->left ? y->left : y->right; + spliced_is_black = IS_BLACK(y); + if (x) { + x->parent = y->parent; + } + else if (spliced_is_black) { + x = &null_x; + x->flags = 0; + SET_BLACK(x); + x->right = x->left = NULL; + x->max_sz = 0; + x->parent = y->parent; + y->left = x; + } + + if (!y->parent) { + RBT_ASSERT(*root == y); + *root = x; + } + else { + if (y == y->parent->left) { + y->parent->left = x; + } + else { + RBT_ASSERT(y == y->parent->right); + y->parent->right = x; + } + if (y->parent != z) { + lower_max_size(y->parent, (y==z ? NULL : z)); + } + } + if (y != z) { + /* We spliced out the successor of z; replace z by the successor */ + replace(root, z, y); + } + + if (spliced_is_black) { + /* We removed a black node which makes the resulting tree + violate the Red-Black Tree properties. Fixup tree... */ + + while (IS_BLACK(x) && x->parent) { + + /* + * x has an "extra black" which we move up the tree + * until we reach the root or until we can get rid of it. + * + * y is the sibbling of x + */ + + if (x == x->parent->left) { + y = x->parent->right; + RBT_ASSERT(y); + if (IS_RED(y)) { + RBT_ASSERT(y->right); + RBT_ASSERT(y->left); + SET_BLACK(y); + RBT_ASSERT(IS_BLACK(x->parent)); + SET_RED(x->parent); + left_rotate(root, x->parent); + y = x->parent->right; + } + RBT_ASSERT(y); + RBT_ASSERT(IS_BLACK(y)); + if (IS_BLACK(y->left) && IS_BLACK(y->right)) { + SET_RED(y); + x = x->parent; + } + else { + if (IS_BLACK(y->right)) { + SET_BLACK(y->left); + SET_RED(y); + right_rotate(root, y); + y = x->parent->right; + } + RBT_ASSERT(y); + if (IS_RED(x->parent)) { + + SET_BLACK(x->parent); + SET_RED(y); + } + RBT_ASSERT(y->right); + SET_BLACK(y->right); + left_rotate(root, x->parent); + x = *root; + break; + } + } + else { + RBT_ASSERT(x == x->parent->right); + y = x->parent->left; + RBT_ASSERT(y); + if (IS_RED(y)) { + RBT_ASSERT(y->right); + RBT_ASSERT(y->left); + SET_BLACK(y); + RBT_ASSERT(IS_BLACK(x->parent)); + SET_RED(x->parent); + right_rotate(root, x->parent); + y = x->parent->left; + } + RBT_ASSERT(y); + RBT_ASSERT(IS_BLACK(y)); + if (IS_BLACK(y->right) && IS_BLACK(y->left)) { + SET_RED(y); + x = x->parent; + } + else { + if (IS_BLACK(y->left)) { + SET_BLACK(y->right); + SET_RED(y); + left_rotate(root, y); + y = x->parent->left; + } + RBT_ASSERT(y); + if (IS_RED(x->parent)) { + SET_BLACK(x->parent); + SET_RED(y); + } + RBT_ASSERT(y->left); + SET_BLACK(y->left); + right_rotate(root, x->parent); + x = *root; + break; + } + } + } + SET_BLACK(x); + + if (null_x.parent) { + if (null_x.parent->left == &null_x) + null_x.parent->left = NULL; + else { + RBT_ASSERT(null_x.parent->right == &null_x); + null_x.parent->right = NULL; + } + RBT_ASSERT(!null_x.left); + RBT_ASSERT(!null_x.right); + } + else if (*root == &null_x) { + *root = NULL; + RBT_ASSERT(!null_x.left); + RBT_ASSERT(!null_x.right); + } + } + + DESTROY_TREE_NODE(del); + +#ifdef HARD_DEBUG + check_tree(*root, 0); +#endif +} + +static void +aoff_link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) +{ + AOFFAllctr_t *alc = (AOFFAllctr_t *) allctr; + AOFF_RBTree_t *blk = (AOFF_RBTree_t *) block; + AOFF_RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC) + ? &alc->sbmbc_root : &alc->mbc_root); + Uint blk_sz = BLK_SZ(blk); + +#ifdef HARD_DEBUG + check_tree(*root, 0); +#endif + + blk->flags = 0; + blk->left = NULL; + blk->right = NULL; + blk->max_sz = blk_sz; + + if (!*root) { + blk->parent = NULL; + SET_BLACK(blk); + *root = blk; + } + else { + AOFF_RBTree_t *x = *root; + while (1) { + if (x->max_sz < blk_sz) { + x->max_sz = blk_sz; + } + if (blk < x) { + if (!x->left) { + blk->parent = x; + x->left = blk; + break; + } + x = x->left; + } + else { + if (!x->right) { + blk->parent = x; + x->right = blk; + break; + } + x = x->right; + } + + } + + /* Insert block into size tree */ + RBT_ASSERT(blk->parent); + + SET_RED(blk); + if (IS_RED(blk->parent)) + tree_insert_fixup(root, blk); + } + +#ifdef HARD_DEBUG + check_tree(*root, 0); +#endif +} + +static Block_t * +aoff_get_free_block(Allctr_t *allctr, Uint size, + Block_t *cand_blk, Uint cand_size, Uint32 flags) +{ + AOFFAllctr_t *alc = (AOFFAllctr_t *) allctr; + AOFF_RBTree_t *x = ((flags & ERTS_ALCU_FLG_SBMBC) + ? alc->sbmbc_root : alc->mbc_root); + AOFF_RBTree_t *blk = NULL; +#ifdef HARD_DEBUG + AOFF_RBTree_t* dbg_blk = check_tree(x, size); +#endif + + ASSERT(!cand_blk || cand_size >= size); + + while (x) { + if (x->left && x->left->max_sz >= size) { + x = x->left; + } + else if (BLK_SZ(x) >= size) { + blk = x; + break; + } + else { + x = x->right; + } + } + +#ifdef HARD_DEBUG + ASSERT(blk == dbg_blk); +#endif + + if (!blk) + return NULL; + + if (cand_blk && cand_blk < &blk->hdr) { + return NULL; /* cand_blk was better */ + } + + aoff_unlink_free_block(allctr, (Block_t *) blk, flags); + + return (Block_t *) blk; +} + + +/* + * info_options() + */ + +static struct { + Eterm as; + Eterm aoff; +#ifdef DEBUG + Eterm end_of_atoms; +#endif +} am; + +static void ERTS_INLINE atom_init(Eterm *atom, char *name) +{ + *atom = am_atom_put(name, strlen(name)); +} +#define AM_INIT(AM) atom_init(&am.AM, #AM) + +static void +init_atoms(void) +{ +#ifdef DEBUG + Eterm *atom; +#endif + + if (atoms_initialized) + return; + +#ifdef DEBUG + for (atom = (Eterm *) &am; atom <= &am.end_of_atoms; atom++) { + *atom = THE_NON_VALUE; + } +#endif + AM_INIT(as); + AM_INIT(aoff); + +#ifdef DEBUG + for (atom = (Eterm *) &am; atom < &am.end_of_atoms; atom++) { + ASSERT(*atom != THE_NON_VALUE); + } +#endif + + atoms_initialized = 1; +} + + +#define bld_uint erts_bld_uint +#define bld_cons erts_bld_cons +#define bld_tuple erts_bld_tuple + +static ERTS_INLINE void +add_2tup(Uint **hpp, Uint *szp, Eterm *lp, Eterm el1, Eterm el2) +{ + *lp = bld_cons(hpp, szp, bld_tuple(hpp, szp, 2, el1, el2), *lp); +} + +static Eterm +info_options(Allctr_t *allctr, + char *prefix, + int *print_to_p, + void *print_to_arg, + Uint **hpp, + Uint *szp) +{ + Eterm res = THE_NON_VALUE; + + if (print_to_p) { + erts_print(*print_to_p, + print_to_arg, + "%sas: %s\n", + prefix, + "aoff"); + } + + if (hpp || szp) { + + if (!atoms_initialized) + erl_exit(1, "%s:%d: Internal error: Atoms not initialized", + __FILE__, __LINE__);; + + res = NIL; + add_2tup(hpp, szp, &res, am.as, am.aoff); + } + + return res; +} + + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ + * NOTE: erts_aoffalc_test() is only supposed to be used for testing. * + * * + * Keep alloc_SUITE_data/allocator_test.h updated if changes are made * + * to erts_aoffalc_test() * +\* */ + +unsigned long +erts_aoffalc_test(unsigned long op, unsigned long a1, unsigned long a2) +{ + switch (op) { + case 0x500: return (unsigned long) 0; /* IS_AOBF */ + case 0x501: return (unsigned long) ((AOFFAllctr_t *) a1)->mbc_root; + case 0x502: return (unsigned long) ((AOFF_RBTree_t *) a1)->parent; + case 0x503: return (unsigned long) ((AOFF_RBTree_t *) a1)->left; + case 0x504: return (unsigned long) ((AOFF_RBTree_t *) a1)->right; + case 0x506: return (unsigned long) IS_BLACK((AOFF_RBTree_t *) a1); + case 0x508: return (unsigned long) 1; /* IS_AOFF */ + case 0x509: return (unsigned long) ((AOFF_RBTree_t *) a1)->max_sz; + default: ASSERT(0); return ~((unsigned long) 0); + } +} + + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ + * Debug functions * +\* */ + + +#ifdef HARD_DEBUG + +#define IS_LEFT_VISITED(FB) ((FB)->flags & LEFT_VISITED_FLG) +#define IS_RIGHT_VISITED(FB) ((FB)->flags & RIGHT_VISITED_FLG) + +#define SET_LEFT_VISITED(FB) ((FB)->flags |= LEFT_VISITED_FLG) +#define SET_RIGHT_VISITED(FB) ((FB)->flags |= RIGHT_VISITED_FLG) + +#define UNSET_LEFT_VISITED(FB) ((FB)->flags &= ~LEFT_VISITED_FLG) +#define UNSET_RIGHT_VISITED(FB) ((FB)->flags &= ~RIGHT_VISITED_FLG) + + +#if 0 +# define PRINT_TREE +#else +# undef PRINT_TREE +#endif + +#ifdef PRINT_TREE +static void print_tree(AOFF_RBTree_t*); +#endif + +/* + * Checks that the order between parent and children are correct, + * and that the Red-Black Tree properies are satisfied. if size > 0, + * check_tree() returns the node that satisfies "address order first fit" + * + * The Red-Black Tree properies are: + * 1. Every node is either red or black. + * 2. Every leaf (NIL) is black. + * 3. If a node is red, then both its children are black. + * 4. Every simple path from a node to a descendant leaf + * contains the same number of black nodes. + * + * + own.max_size == MAX(own.size, left.max_size, right.max_size) + */ + +static AOFF_RBTree_t * +check_tree(AOFF_RBTree_t* root, Uint size) +{ + AOFF_RBTree_t *res = NULL; + Sint blacks; + Sint curr_blacks; + AOFF_RBTree_t *x; + +#ifdef PRINT_TREE + print_tree(root); +#endif + + if (!root) + return res; + + x = root; + ASSERT(IS_BLACK(x)); + ASSERT(!x->parent); + curr_blacks = 1; + blacks = -1; + + while (x) { + if (!IS_LEFT_VISITED(x)) { + SET_LEFT_VISITED(x); + if (x->left) { + x = x->left; + if (IS_BLACK(x)) + curr_blacks++; + continue; + } + else { + if (blacks < 0) + blacks = curr_blacks; + ASSERT(blacks == curr_blacks); + } + } + + if (!IS_RIGHT_VISITED(x)) { + SET_RIGHT_VISITED(x); + if (x->right) { + x = x->right; + if (IS_BLACK(x)) + curr_blacks++; + continue; + } + else { + if (blacks < 0) + blacks = curr_blacks; + ASSERT(blacks == curr_blacks); + } + } + + + if (IS_RED(x)) { + ASSERT(IS_BLACK(x->right)); + ASSERT(IS_BLACK(x->left)); + } + + ASSERT(x->parent || x == root); + + if (x->left) { + ASSERT(x->left->parent == x); + ASSERT(x->left < x); + ASSERT(x->left->max_sz <= x->max_sz); + } + + if (x->right) { + ASSERT(x->right->parent == x); + ASSERT(x->right > x); + ASSERT(x->right->max_sz <= x->max_sz); + } + ASSERT(x->max_sz >= BLK_SZ(x)); + ASSERT(x->max_sz == BLK_SZ(x) + || x->max_sz == (x->left ? x->left->max_sz : 0) + || x->max_sz == (x->right ? x->right->max_sz : 0)); + + if (size && BLK_SZ(x) >= size) { + if (!res || x < res) { + res = x; + } + } + + UNSET_LEFT_VISITED(x); + UNSET_RIGHT_VISITED(x); + if (IS_BLACK(x)) + curr_blacks--; + x = x->parent; + + } + + ASSERT(curr_blacks == 0); + + UNSET_LEFT_VISITED(root); + UNSET_RIGHT_VISITED(root); + + return res; + +} + + +#ifdef PRINT_TREE +#define INDENT_STEP 2 + +#include <stdio.h> + +static void +print_tree_aux(AOFF_RBTree_t *x, int indent) +{ + int i; + + if (x) { + print_tree_aux(x->right, indent + INDENT_STEP); + for (i = 0; i < indent; i++) { + putc(' ', stderr); + } + fprintf(stderr, "%s: sz=%lu addr=0x%lx max_size=%lu\r\n", + IS_BLACK(x) ? "BLACK" : "RED", + BLK_SZ(x), (Uint)x, x->max_sz); + print_tree_aux(x->left, indent + INDENT_STEP); + } +} + + +static void +print_tree(AOFF_RBTree_t* root) +{ + fprintf(stderr, " --- AOFF tree begin ---\r\n"); + print_tree_aux(root, 0); + fprintf(stderr, " --- AOFF tree end ---\r\n"); +} + +#endif /* PRINT_TREE */ + +#endif /* HARD_DEBUG */ + diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.h b/erts/emulator/beam/erl_ao_firstfit_alloc.h new file mode 100644 index 0000000000..0bf0ec8cee --- /dev/null +++ b/erts/emulator/beam/erl_ao_firstfit_alloc.h @@ -0,0 +1,60 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2003-2009. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + + +#ifndef ERL_AO_FIRSTFIT_ALLOC__ +#define ERL_AO_FIRSTFIT_ALLOC__ + +#include "erl_alloc_util.h" + +#define ERTS_ALC_AOFF_ALLOC_VSN_STR "0.9" + +typedef struct AOFFAllctr_t_ AOFFAllctr_t; + +typedef struct { + int dummy; +} AOFFAllctrInit_t; + +#define ERTS_DEFAULT_AOFF_ALLCTR_INIT {0/*dummy*/} + +void erts_aoffalc_init(void); +Allctr_t *erts_aoffalc_start(AOFFAllctr_t *, AOFFAllctrInit_t*, AllctrInit_t *); + +#endif /* #ifndef ERL_AO_FIRSTFIT_ALLOC__ */ + + + +#if defined(GET_ERL_AOFF_ALLOC_IMPL) && !defined(ERL_AOFF_ALLOC_IMPL__) +#define ERL_AOFF_ALLOC_IMPL__ + +#define GET_ERL_ALLOC_UTIL_IMPL +#include "erl_alloc_util.h" + + +struct AOFFAllctr_t_ { + Allctr_t allctr; /* Has to be first! */ + + struct AOFF_RBTree_t_* mbc_root; + struct AOFF_RBTree_t_* sbmbc_root; +}; + +unsigned long erts_aoffalc_test(unsigned long, unsigned long, unsigned long); + +#endif /* #if defined(GET_ERL_AOFF_ALLOC_IMPL) + && !defined(ERL_AOFF_ALLOC_IMPL__) */ diff --git a/erts/emulator/beam/erl_async.c b/erts/emulator/beam/erl_async.c index 12c7631448..91b64411d4 100644 --- a/erts/emulator/beam/erl_async.c +++ b/erts/emulator/beam/erl_async.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2000-2010. All Rights Reserved. + * Copyright Ericsson AB 2000-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -251,6 +251,7 @@ static int async_del(long id) erts_free(ERTS_ALC_T_ASYNC, a); return 1; } + a = a->next; } erts_mtx_unlock(&async_q[i].mtx); } diff --git a/erts/emulator/beam/erl_bestfit_alloc.c b/erts/emulator/beam/erl_bestfit_alloc.c index 3035e5df16..5e3032ddaa 100644 --- a/erts/emulator/beam/erl_bestfit_alloc.c +++ b/erts/emulator/beam/erl_bestfit_alloc.c @@ -84,24 +84,24 @@ #ifdef HARD_DEBUG -static RBTree_t * check_tree(BFAllctr_t *, Uint); +static RBTree_t * check_tree(RBTree_t, int, Uint); #endif -static void tree_delete(Allctr_t *allctr, Block_t *del); +static void tree_delete(Allctr_t *allctr, Block_t *del, Uint32 flags); /* Prototypes of callback functions */ /* "address order best fit" specific callback functions */ static Block_t * aobf_get_free_block (Allctr_t *, Uint, - Block_t *, Uint); -static void aobf_link_free_block (Allctr_t *, Block_t *); + Block_t *, Uint, Uint32); +static void aobf_link_free_block (Allctr_t *, Block_t *, Uint32); #define aobf_unlink_free_block tree_delete /* "best fit" specific callback functions */ static Block_t * bf_get_free_block (Allctr_t *, Uint, - Block_t *, Uint); -static void bf_link_free_block (Allctr_t *, Block_t *); -static ERTS_INLINE void bf_unlink_free_block (Allctr_t *, Block_t *); + Block_t *, Uint, Uint32); +static void bf_link_free_block (Allctr_t *, Block_t *, Uint32); +static ERTS_INLINE void bf_unlink_free_block (Allctr_t *, Block_t *, Uint32); static Eterm info_options (Allctr_t *, char *, int *, @@ -303,7 +303,7 @@ replace(RBTree_t **root, RBTree_t *x, RBTree_t *y) } static void -tree_insert_fixup(BFAllctr_t *bfallctr, RBTree_t *blk) +tree_insert_fixup(RBTree_t **root, RBTree_t *blk) { RBTree_t *x = blk, *y; @@ -311,7 +311,7 @@ tree_insert_fixup(BFAllctr_t *bfallctr, RBTree_t *blk) * Rearrange the tree so that it satisfies the Red-Black Tree properties */ - RBT_ASSERT(x != bfallctr->root && IS_RED(x->parent)); + RBT_ASSERT(x != *root && IS_RED(x->parent)); do { /* @@ -336,7 +336,7 @@ tree_insert_fixup(BFAllctr_t *bfallctr, RBTree_t *blk) if (x == x->parent->right) { x = x->parent; - left_rotate(&bfallctr->root, x); + left_rotate(root, x); } RBT_ASSERT(x == x->parent->parent->left->left); @@ -347,7 +347,7 @@ tree_insert_fixup(BFAllctr_t *bfallctr, RBTree_t *blk) SET_BLACK(x->parent); SET_RED(x->parent->parent); - right_rotate(&bfallctr->root, x->parent->parent); + right_rotate(root, x->parent->parent); RBT_ASSERT(x == x->parent->left); RBT_ASSERT(IS_RED(x)); @@ -370,7 +370,7 @@ tree_insert_fixup(BFAllctr_t *bfallctr, RBTree_t *blk) if (x == x->parent->left) { x = x->parent; - right_rotate(&bfallctr->root, x); + right_rotate(root, x); } RBT_ASSERT(x == x->parent->parent->right->right); @@ -381,7 +381,7 @@ tree_insert_fixup(BFAllctr_t *bfallctr, RBTree_t *blk) SET_BLACK(x->parent); SET_RED(x->parent->parent); - left_rotate(&bfallctr->root, x->parent->parent); + left_rotate(root, x->parent->parent); RBT_ASSERT(x == x->parent->right); RBT_ASSERT(IS_RED(x)); @@ -390,9 +390,9 @@ tree_insert_fixup(BFAllctr_t *bfallctr, RBTree_t *blk) break; } } - } while (x != bfallctr->root && IS_RED(x->parent)); + } while (x != *root && IS_RED(x->parent)); - SET_BLACK(bfallctr->root); + SET_BLACK(*root); } @@ -402,18 +402,22 @@ tree_insert_fixup(BFAllctr_t *bfallctr, RBTree_t *blk) * callback function in the address order case. */ static void -tree_delete(Allctr_t *allctr, Block_t *del) +tree_delete(Allctr_t *allctr, Block_t *del, Uint32 flags) { BFAllctr_t *bfallctr = (BFAllctr_t *) allctr; Uint spliced_is_black; + RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC) + ? &bfallctr->sbmbc_root + : &bfallctr->mbc_root); RBTree_t *x, *y, *z = (RBTree_t *) del; RBTree_t null_x; /* null_x is used to get the fixup started when we splice out a node without children. */ null_x.parent = NULL; + #ifdef HARD_DEBUG - check_tree(bfallctr, 0); + check_tree(*root, bfallctr->address_order, 0); #endif /* Remove node from tree... */ @@ -440,8 +444,8 @@ tree_delete(Allctr_t *allctr, Block_t *del) } if (!y->parent) { - RBT_ASSERT(bfallctr->root == y); - bfallctr->root = x; + RBT_ASSERT(*root == y); + *root = x; } else if (y == y->parent->left) y->parent->left = x; @@ -451,7 +455,7 @@ tree_delete(Allctr_t *allctr, Block_t *del) } if (y != z) { /* We spliced out the successor of z; replace z by the successor */ - replace(&bfallctr->root, z, y); + replace(root, z, y); } if (spliced_is_black) { @@ -476,7 +480,7 @@ tree_delete(Allctr_t *allctr, Block_t *del) SET_BLACK(y); RBT_ASSERT(IS_BLACK(x->parent)); SET_RED(x->parent); - left_rotate(&bfallctr->root, x->parent); + left_rotate(root, x->parent); y = x->parent->right; } RBT_ASSERT(y); @@ -489,7 +493,7 @@ tree_delete(Allctr_t *allctr, Block_t *del) if (IS_BLACK(y->right)) { SET_BLACK(y->left); SET_RED(y); - right_rotate(&bfallctr->root, y); + right_rotate(root, y); y = x->parent->right; } RBT_ASSERT(y); @@ -500,8 +504,8 @@ tree_delete(Allctr_t *allctr, Block_t *del) } RBT_ASSERT(y->right); SET_BLACK(y->right); - left_rotate(&bfallctr->root, x->parent); - x = bfallctr->root; + left_rotate(root, x->parent); + x = *root; break; } } @@ -515,7 +519,7 @@ tree_delete(Allctr_t *allctr, Block_t *del) SET_BLACK(y); RBT_ASSERT(IS_BLACK(x->parent)); SET_RED(x->parent); - right_rotate(&bfallctr->root, x->parent); + right_rotate(root, x->parent); y = x->parent->left; } RBT_ASSERT(y); @@ -528,7 +532,7 @@ tree_delete(Allctr_t *allctr, Block_t *del) if (IS_BLACK(y->left)) { SET_BLACK(y->right); SET_RED(y); - left_rotate(&bfallctr->root, y); + left_rotate(root, y); y = x->parent->left; } RBT_ASSERT(y); @@ -538,8 +542,8 @@ tree_delete(Allctr_t *allctr, Block_t *del) } RBT_ASSERT(y->left); SET_BLACK(y->left); - right_rotate(&bfallctr->root, x->parent); - x = bfallctr->root; + right_rotate(root, x->parent); + x = *root; break; } } @@ -556,8 +560,8 @@ tree_delete(Allctr_t *allctr, Block_t *del) RBT_ASSERT(!null_x.left); RBT_ASSERT(!null_x.right); } - else if (bfallctr->root == &null_x) { - bfallctr->root = NULL; + else if (*root == &null_x) { + *root = NULL; RBT_ASSERT(!null_x.left); RBT_ASSERT(!null_x.right); } @@ -567,7 +571,7 @@ tree_delete(Allctr_t *allctr, Block_t *del) DESTROY_TREE_NODE(del); #ifdef HARD_DEBUG - check_tree(bfallctr, 0); + check_tree(root, bfallctr->address_order, 0); #endif } @@ -577,23 +581,28 @@ tree_delete(Allctr_t *allctr, Block_t *del) \* */ static void -aobf_link_free_block(Allctr_t *allctr, Block_t *block) +aobf_link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) { BFAllctr_t *bfallctr = (BFAllctr_t *) allctr; + RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC) + ? &bfallctr->sbmbc_root + : &bfallctr->mbc_root); RBTree_t *blk = (RBTree_t *) block; Uint blk_sz = BLK_SZ(blk); + + blk->flags = 0; blk->left = NULL; blk->right = NULL; - if (!bfallctr->root) { + if (!*root) { blk->parent = NULL; SET_BLACK(blk); - bfallctr->root = blk; + *root = blk; } else { - RBTree_t *x = bfallctr->root; + RBTree_t *x = *root; while (1) { Uint size; @@ -623,28 +632,32 @@ aobf_link_free_block(Allctr_t *allctr, Block_t *block) SET_RED(blk); if (IS_RED(blk->parent)) - tree_insert_fixup(bfallctr, blk); + tree_insert_fixup(root, blk); } #ifdef HARD_DEBUG - check_tree(bfallctr, 0); + check_tree(root, 1, 0); #endif } #if 0 /* tree_delete() is directly used instead */ static void -aobf_unlink_free_block(Allctr_t *allctr, Block_t *block) +aobf_unlink_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) { - tree_delete(allctr, block); + tree_delete(allctr, block, flags); } #endif static Block_t * aobf_get_free_block(Allctr_t *allctr, Uint size, - Block_t *cand_blk, Uint cand_size) + Block_t *cand_blk, Uint cand_size, + Uint32 flags) { BFAllctr_t *bfallctr = (BFAllctr_t *) allctr; - RBTree_t *x = bfallctr->root; + RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC) + ? &bfallctr->sbmbc_root + : &bfallctr->mbc_root); + RBTree_t *x = *root; RBTree_t *blk = NULL; Uint blk_sz; @@ -665,7 +678,7 @@ aobf_get_free_block(Allctr_t *allctr, Uint size, return NULL; #ifdef HARD_DEBUG - ASSERT(blk == check_tree(bfallctr, size)); + ASSERT(blk == check_tree(root, 1, size)); #endif if (cand_blk) { @@ -676,7 +689,7 @@ aobf_get_free_block(Allctr_t *allctr, Uint size, return NULL; /* cand_blk was better */ } - aobf_unlink_free_block(allctr, (Block_t *) blk); + aobf_unlink_free_block(allctr, (Block_t *) blk, flags); return (Block_t *) blk; } @@ -687,9 +700,12 @@ aobf_get_free_block(Allctr_t *allctr, Uint size, \* */ static void -bf_link_free_block(Allctr_t *allctr, Block_t *block) +bf_link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) { BFAllctr_t *bfallctr = (BFAllctr_t *) allctr; + RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC) + ? &bfallctr->sbmbc_root + : &bfallctr->mbc_root); RBTree_t *blk = (RBTree_t *) block; Uint blk_sz = BLK_SZ(blk); @@ -700,13 +716,13 @@ bf_link_free_block(Allctr_t *allctr, Block_t *block) blk->left = NULL; blk->right = NULL; - if (!bfallctr->root) { + if (!*root) { blk->parent = NULL; SET_BLACK(blk); - bfallctr->root = blk; + *root = blk; } else { - RBTree_t *x = bfallctr->root; + RBTree_t *x = *root; while (1) { Uint size; @@ -745,7 +761,7 @@ bf_link_free_block(Allctr_t *allctr, Block_t *block) SET_RED(blk); if (IS_RED(blk->parent)) - tree_insert_fixup(bfallctr, blk); + tree_insert_fixup(root, blk); } @@ -753,14 +769,17 @@ bf_link_free_block(Allctr_t *allctr, Block_t *block) LIST_NEXT(blk) = NULL; #ifdef HARD_DEBUG - check_tree(bfallctr, 0); + check_tree(root, 0, 0); #endif } static ERTS_INLINE void -bf_unlink_free_block(Allctr_t *allctr, Block_t *block) +bf_unlink_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) { BFAllctr_t *bfallctr = (BFAllctr_t *) allctr; + RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC) + ? &bfallctr->sbmbc_root + : &bfallctr->mbc_root); RBTree_t *x = (RBTree_t *) block; if (IS_LIST_ELEM(x)) { @@ -778,9 +797,9 @@ bf_unlink_free_block(Allctr_t *allctr, Block_t *block) ASSERT(IS_LIST_ELEM(LIST_NEXT(x))); #ifdef HARD_DEBUG - check_tree(bfallctr, 0); + check_tree(root, 0, 0); #endif - replace(&bfallctr->root, x, LIST_NEXT(x)); + replace(root, x, LIST_NEXT(x)); #ifdef HARD_DEBUG check_tree(bfallctr, 0); @@ -788,7 +807,7 @@ bf_unlink_free_block(Allctr_t *allctr, Block_t *block) } else { /* Remove from tree */ - tree_delete(allctr, block); + tree_delete(allctr, block, flags); } DESTROY_LIST_ELEM(x); @@ -797,10 +816,14 @@ bf_unlink_free_block(Allctr_t *allctr, Block_t *block) static Block_t * bf_get_free_block(Allctr_t *allctr, Uint size, - Block_t *cand_blk, Uint cand_size) + Block_t *cand_blk, Uint cand_size, + Uint32 flags) { BFAllctr_t *bfallctr = (BFAllctr_t *) allctr; - RBTree_t *x = bfallctr->root; + RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC) + ? &bfallctr->sbmbc_root + : &bfallctr->mbc_root); + RBTree_t *x = *root; RBTree_t *blk = NULL; Uint blk_sz; @@ -827,7 +850,7 @@ bf_get_free_block(Allctr_t *allctr, Uint size, #ifdef HARD_DEBUG { - RBTree_t *ct_blk = check_tree(bfallctr, size); + RBTree_t *ct_blk = check_tree(root, 0, size); ASSERT(BLK_SZ(ct_blk) == BLK_SZ(blk)); } #endif @@ -839,7 +862,7 @@ bf_get_free_block(Allctr_t *allctr, Uint size, the tree node */ blk = LIST_NEXT(blk) ? LIST_NEXT(blk) : blk; - bf_unlink_free_block(allctr, (Block_t *) blk); + bf_unlink_free_block(allctr, (Block_t *) blk, flags); return (Block_t *) blk; } @@ -949,13 +972,14 @@ erts_bfalc_test(unsigned long op, unsigned long a1, unsigned long a2) { switch (op) { case 0x200: return (unsigned long) ((BFAllctr_t *) a1)->address_order; - case 0x201: return (unsigned long) ((BFAllctr_t *) a1)->root; + case 0x201: return (unsigned long) ((BFAllctr_t *) a1)->mbc_root; case 0x202: return (unsigned long) ((RBTree_t *) a1)->parent; case 0x203: return (unsigned long) ((RBTree_t *) a1)->left; case 0x204: return (unsigned long) ((RBTree_t *) a1)->right; case 0x205: return (unsigned long) ((RBTreeList_t *) a1)->next; case 0x206: return (unsigned long) IS_BLACK((RBTree_t *) a1); case 0x207: return (unsigned long) IS_TREE_NODE((RBTree_t *) a1); + case 0x208: return (unsigned long) 0; /* IS_AOFF */ default: ASSERT(0); return ~((unsigned long) 0); } } @@ -985,7 +1009,7 @@ erts_bfalc_test(unsigned long op, unsigned long a1, unsigned long a2) #endif #ifdef PRINT_TREE -static void print_tree(BFAllctr_t *); +static void print_tree(RBTree_t *, int); #endif /* @@ -1003,7 +1027,7 @@ static void print_tree(BFAllctr_t *); */ static RBTree_t * -check_tree(BFAllctr_t *bfallctr, Uint size) +check_tree(RBTree_t *root, int ao, Uint size) { RBTree_t *res = NULL; Sint blacks; @@ -1011,13 +1035,13 @@ check_tree(BFAllctr_t *bfallctr, Uint size) RBTree_t *x; #ifdef PRINT_TREE - print_tree(bfallctr); + print_tree(root, ao); #endif - if (!bfallctr->root) + if (!root) return res; - x = bfallctr->root; + x = root; ASSERT(IS_BLACK(x)); ASSERT(!x->parent); curr_blacks = 1; @@ -1060,11 +1084,11 @@ check_tree(BFAllctr_t *bfallctr, Uint size) ASSERT(IS_BLACK(x->left)); } - ASSERT(x->parent || x == bfallctr->root); + ASSERT(x->parent || x == root); if (x->left) { ASSERT(x->left->parent == x); - if (bfallctr->address_order) { + if (ao) { ASSERT(BLK_SZ(x->left) < BLK_SZ(x) || (BLK_SZ(x->left) == BLK_SZ(x) && x->left < x)); } @@ -1076,7 +1100,7 @@ check_tree(BFAllctr_t *bfallctr, Uint size) if (x->right) { ASSERT(x->right->parent == x); - if (bfallctr->address_order) { + if (ao) { ASSERT(BLK_SZ(x->right) > BLK_SZ(x) || (BLK_SZ(x->right) == BLK_SZ(x) && x->right > x)); } @@ -1087,7 +1111,7 @@ check_tree(BFAllctr_t *bfallctr, Uint size) } if (size && BLK_SZ(x) >= size) { - if (bfallctr->address_order) { + if (ao) { if (!res || BLK_SZ(x) < BLK_SZ(res) || (BLK_SZ(x) == BLK_SZ(res) && x < res)) @@ -1109,8 +1133,8 @@ check_tree(BFAllctr_t *bfallctr, Uint size) ASSERT(curr_blacks == 0); - UNSET_LEFT_VISITED(bfallctr->root); - UNSET_RIGHT_VISITED(bfallctr->root); + UNSET_LEFT_VISITED(root); + UNSET_RIGHT_VISITED(root); return res; @@ -1148,11 +1172,11 @@ print_tree_aux(RBTree_t *x, int indent) static void -print_tree(BFAllctr_t *bfallctr) +print_tree(RBTree_t *root, int ao) { - char *type = bfallctr->address_order ? "Size-Adress" : "Size"; + char *type = ao ? "Size-Adress" : "Size"; fprintf(stderr, " --- %s tree begin ---\r\n", type); - print_tree_aux(bfallctr->root, 0); + print_tree_aux(root, 0); fprintf(stderr, " --- %s tree end ---\r\n", type); } diff --git a/erts/emulator/beam/erl_bestfit_alloc.h b/erts/emulator/beam/erl_bestfit_alloc.h index cb35e21e57..faa2d9742e 100644 --- a/erts/emulator/beam/erl_bestfit_alloc.h +++ b/erts/emulator/beam/erl_bestfit_alloc.h @@ -54,7 +54,8 @@ typedef struct RBTree_t_ RBTree_t; struct BFAllctr_t_ { Allctr_t allctr; /* Has to be first! */ - RBTree_t * root; + RBTree_t * mbc_root; + RBTree_t * sbmbc_root; int address_order; }; diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c index c9cdcb87a6..9631fb50db 100644 --- a/erts/emulator/beam/erl_bif_ddll.c +++ b/erts/emulator/beam/erl_bif_ddll.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2006-2010. All Rights Reserved. + * Copyright Ericsson AB 2006-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -146,7 +146,7 @@ BIF_RETTYPE erl_ddll_try_load_3(Process *p, Eterm path_term, Eterm name_term, Eterm options) { char *path = NULL; - int path_len; + Uint path_len; char *name = NULL; DE_Handle *dh; erts_driver_t *drv; @@ -221,9 +221,7 @@ BIF_RETTYPE erl_ddll_try_load_3(Process *p, Eterm path_term, goto error; } - path_len = io_list_len(path_term); - - if (path_len <= 0) { + if (erts_iolist_size(path_term, &path_len)) { goto error; } path = erts_alloc(ERTS_ALC_T_DDLL_TMP_BUF, path_len + 1 /* might need path separator */ + sys_strlen(name) + 1); @@ -1878,7 +1876,7 @@ static Eterm mkatom(char *str) static char *pick_list_or_atom(Eterm name_term) { char *name = NULL; - int name_len; + Uint name_len; if (is_atom(name_term)) { Atom *ap = atom_tab(atom_val(name_term)); if (ap->len == 0) { @@ -1890,8 +1888,7 @@ static char *pick_list_or_atom(Eterm name_term) memcpy(name,ap->name,ap->len); name[ap->len] = '\0'; } else { - name_len = io_list_len(name_term); - if (name_len <= 0) { + if (erts_iolist_size(name_term, &name_len)) { goto error; } name = erts_alloc(ERTS_ALC_T_DDLL_TMP_BUF, name_len + 1); diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index e50fc18e64..f264bf44df 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -1732,14 +1732,14 @@ info_1_tuple(Process* BIF_P, /* Pointer to current process. */ # define ERTS_ERROR_CHECKER_PRINTF_XML VALGRIND_PRINTF_XML # endif #endif - int buf_size = 8*1024; /* Try with 8KB first */ + Uint buf_size = 8*1024; /* Try with 8KB first */ char *buf = erts_alloc(ERTS_ALC_T_TMP, buf_size); int r = io_list_to_buf(*tp, (char*) buf, buf_size - 1); if (r < 0) { erts_free(ERTS_ALC_T_TMP, (void *) buf); - buf_size = io_list_len(*tp); - if (buf_size < 0) + if (erts_iolist_size(*tp, &buf_size)) { goto badarg; + } buf_size++; buf = erts_alloc(ERTS_ALC_T_TMP, buf_size); r = io_list_to_buf(*tp, (char*) buf, buf_size - 1); diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c index fbc92b9730..3fd35dd963 100644 --- a/erts/emulator/beam/erl_bif_port.c +++ b/erts/emulator/beam/erl_bif_port.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2010. All Rights Reserved. + * Copyright Ericsson AB 2001-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -996,6 +996,7 @@ static byte* convert_environment(Process* p, Eterm env) Eterm* hp; Uint heap_size; int n; + Uint size; byte* bytes; if ((n = list_length(env)) < 0) { @@ -1039,15 +1040,15 @@ static byte* convert_environment(Process* p, Eterm env) if (is_not_nil(env)) { goto done; } - if ((n = io_list_len(all)) < 0) { + if (erts_iolist_size(all, &size)) { goto done; } /* * Put the result in a binary (no risk for a memory leak that way). */ - (void) erts_new_heap_binary(p, NULL, n, &bytes); - io_list_to_buf(all, (char*)bytes, n); + (void) erts_new_heap_binary(p, NULL, size, &bytes); + io_list_to_buf(all, (char*)bytes, size); done: erts_free(ERTS_ALC_T_TMP, temp_heap); diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c index d4a8a3aaa7..26891c4348 100644 --- a/erts/emulator/beam/erl_bif_re.c +++ b/erts/emulator/beam/erl_bif_re.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2008-2010. All Rights Reserved. + * Copyright Ericsson AB 2008-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -417,7 +417,7 @@ build_compile_result(Process *p, Eterm error_tag, pcre *result, int errcode, con BIF_RETTYPE re_compile_2(BIF_ALIST_2) { - int slen; + Uint slen; char *expr; pcre *result; int errcode = 0; @@ -444,7 +444,7 @@ re_compile_2(BIF_ALIST_2) BIF_TRAP2(ucompile_trap_exportp, BIF_P, BIF_ARG_1, BIF_ARG_2); } - if ((slen = io_list_len(BIF_ARG_1)) < 0) { + if (erts_iolist_size(BIF_ARG_1, &slen)) { BIF_ERROR(BIF_P,BADARG); } expr = erts_alloc(ERTS_ALC_T_RE_TMP_BUF, slen + 1); @@ -795,8 +795,8 @@ build_capture(Eterm capture_spec[CAPSPEC_SIZE], const pcre *code) memcpy(tmpb,ap->name,ap->len); tmpb[ap->len] = '\0'; } else { - int slen = io_list_len(val); - if (slen < 0) { + Uint slen; + if (erts_iolist_size(val, &slen)) { goto error; } if ((slen + 1) > tmpbsiz) { @@ -851,7 +851,7 @@ re_run_3(BIF_ALIST_3) const pcre *code_tmp; RestartContext restart; byte *temp_alloc = NULL; - int slength; + Uint slength; int startoffset = 0; int options = 0, comp_options = 0; int ovsize; @@ -875,7 +875,7 @@ re_run_3(BIF_ALIST_3) if (is_not_tuple(BIF_ARG_2) || (arityval(*tuple_val(BIF_ARG_2)) != 4)) { if (is_binary(BIF_ARG_2) || is_list(BIF_ARG_2) || is_nil(BIF_ARG_2)) { /* Compile from textual RE */ - int slen; + Uint slen; char *expr; pcre *result; int errcode = 0; @@ -889,7 +889,7 @@ re_run_3(BIF_ALIST_3) BIF_TRAP3(urun_trap_exportp, BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); } - if ((slen = io_list_len(BIF_ARG_2)) < 0) { + if (erts_iolist_size(BIF_ARG_2, &slen)) { BIF_ERROR(BIF_P,BADARG); } @@ -1027,7 +1027,7 @@ re_run_3(BIF_ALIST_3) restart.flags |= RESTART_FLAG_SUBJECT_IN_BINARY; } else { handle_iolist: - if ((slength = io_list_len(BIF_ARG_1)) < 0) { + if (erts_iolist_size(BIF_ARG_1, &slength)) { erts_free(ERTS_ALC_T_RE_SUBJECT, restart.ovector); erts_free(ERTS_ALC_T_RE_SUBJECT, restart.code); if (restart.ret_info != NULL) { diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c index 0174e5fc43..e56084b9cb 100644 --- a/erts/emulator/beam/erl_bits.c +++ b/erts/emulator/beam/erl_bits.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2010. All Rights Reserved. + * Copyright Ericsson AB 1999-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in diff --git a/erts/emulator/beam/erl_bits.h b/erts/emulator/beam/erl_bits.h index 0f67733fa4..3309ea706b 100644 --- a/erts/emulator/beam/erl_bits.h +++ b/erts/emulator/beam/erl_bits.h @@ -150,7 +150,7 @@ void erts_bits_destroy_state(ERL_BITS_PROTO_0); * NBYTES(x) returns the number of bytes needed to store x bits. */ -#define NBYTES(x) (((x) + 7) >> 3) +#define NBYTES(x) (((Uint64)(x) + (Uint64) 7) >> 3) #define BYTE_OFFSET(ofs) ((Uint) (ofs) >> 3) #define BIT_OFFSET(ofs) ((ofs) & 7) diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index e0a6aa05c6..eb50f56502 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -338,13 +338,13 @@ static ERTS_INLINE void db_unlock(DbTable* tb, db_lock_kind_t kind) ASSERT(tb != meta_pid_to_tab && tb != meta_pid_to_fixed_tab); if (tb->common.type & DB_FINE_LOCKED) { - if (tb->common.is_thread_safe) { - ASSERT(kind == LCK_WRITE); + if (kind == LCK_WRITE) { + ASSERT(tb->common.is_thread_safe); tb->common.is_thread_safe = 0; erts_smp_rwmtx_rwunlock(&tb->common.rwlock); } else { - ASSERT(kind != LCK_WRITE); + ASSERT(!tb->common.is_thread_safe); erts_smp_rwmtx_runlock(&tb->common.rwlock); } } @@ -543,9 +543,9 @@ static int remove_named_tab(DbTable *tb, int have_lock) * We keep our increased refc over this op in order to * prevent the table from disapearing. */ - erts_smp_rwmtx_rwunlock(&tb->common.rwlock); + db_unlock(tb, LCK_WRITE); erts_smp_rwmtx_rwlock(rwlock); - erts_smp_rwmtx_rwlock(&tb->common.rwlock); + db_lock(tb, LCK_WRITE); } #endif @@ -1650,24 +1650,6 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1) tb->common.status &= ~(DB_PROTECTED|DB_PUBLIC|DB_PRIVATE); tb->common.status |= DB_DELETE; - mmtl = get_meta_main_tab_lock(tb->common.slot); -#ifdef ERTS_SMP - if (erts_smp_rwmtx_tryrwlock(mmtl) == EBUSY) { - /* - * We keep our increased refc over this op in order to - * prevent the table from disapearing. - */ - erts_smp_rwmtx_rwunlock(&tb->common.rwlock); - erts_smp_rwmtx_rwlock(mmtl); - erts_smp_rwmtx_rwlock(&tb->common.rwlock); - } -#endif - /* We must keep the slot, to be found by db_proc_dead() if process dies */ - MARK_SLOT_DEAD(tb->common.slot); - erts_smp_rwmtx_rwunlock(mmtl); - if (is_atom(tb->common.id)) - remove_named_tab(tb, 0); - if (tb->common.owner != BIF_P->id) { DeclareTmpHeap(meta_tuple,3,BIF_P); @@ -1691,6 +1673,25 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1) db_meta_unlock(meta_pid_to_tab, LCK_WRITE_REC); UnUseTmpHeap(3,BIF_P); } + + mmtl = get_meta_main_tab_lock(tb->common.slot); +#ifdef ERTS_SMP + if (erts_smp_rwmtx_tryrwlock(mmtl) == EBUSY) { + /* + * We keep our increased refc over this op in order to + * prevent the table from disapearing. + */ + db_unlock(tb, LCK_WRITE); + erts_smp_rwmtx_rwlock(mmtl); + db_lock(tb, LCK_WRITE); + } +#endif + /* We must keep the slot, to be found by db_proc_dead() if process dies */ + MARK_SLOT_DEAD(tb->common.slot); + erts_smp_rwmtx_rwunlock(mmtl); + if (is_atom(tb->common.id)) + remove_named_tab(tb, 0); + /* disable inheritance */ free_heir_data(tb); tb->common.heir = am_none; diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 9ef990cc4f..fdc82c8b88 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -105,7 +105,7 @@ #define NSEG_2 256 /* Size of second segment table */ #define NSEG_INC 128 /* Number of segments to grow after that */ -#define SEGTAB(tb) ((struct segment**)erts_smp_atomic_read(&(tb)->segtab)) +#define SEGTAB(tb) ((struct segment**)erts_smp_atomic_read_acqb(&(tb)->segtab)) #define NACTIVE(tb) ((int)erts_smp_atomic_read(&(tb)->nactive)) #define NITEMS(tb) ((int)erts_smp_atomic_read(&(tb)->common.nitems)) @@ -122,8 +122,8 @@ */ static ERTS_INLINE Uint hash_to_ix(DbTableHash* tb, HashValue hval) { - Uint mask = erts_smp_atomic_read(&tb->szm); - Uint ix = hval & mask; + Uint mask = erts_smp_atomic_read_acqb(&tb->szm); + Uint ix = hval & mask; if (ix >= erts_smp_atomic_read(&tb->nactive)) { ix &= mask>>1; ASSERT(ix < erts_smp_atomic_read(&tb->nactive)); @@ -668,6 +668,7 @@ int db_create_hash(Process *p, DbTable *tbl) else { /* coarse locking */ tb->locks = NULL; } + ERTS_THR_MEMORY_BARRIER; #endif /* ERST_SMP */ return DB_ERROR_NONE; } @@ -2085,7 +2086,14 @@ static void db_print_hash(int to, void *to_arg, int show, DbTable *tbl) while(list != 0) { if (list->hvalue == INVALID_HASH) erts_print(to, to_arg, "*"); - erts_print(to, to_arg, "%T", make_tuple(list->dbterm.tpl)); + if (tb->common.compress) { + Eterm key = GETKEY(tb, list->dbterm.tpl); + erts_print(to, to_arg, "key=%R", key, list->dbterm.tpl); + } + else { + Eterm obj = make_tuple_rel(list->dbterm.tpl,list->dbterm.tpl); + erts_print(to, to_arg, "%R", obj, list->dbterm.tpl); + } if (list->next != 0) erts_print(to, to_arg, ","); list = list->next; @@ -2342,7 +2350,7 @@ static int alloc_seg(DbTableHash *tb) struct ext_segment* eseg; eseg = (struct ext_segment*) SEGTAB(tb)[seg_ix-1]; MY_ASSERT(eseg!=NULL && eseg->s.is_ext_segment); - erts_smp_atomic_set(&tb->segtab, (erts_aint_t) eseg->segtab); + erts_smp_atomic_set_relb(&tb->segtab, (erts_aint_t) eseg->segtab); tb->nsegs = eseg->nsegs; } ASSERT(seg_ix < tb->nsegs); @@ -2414,7 +2422,7 @@ static int free_seg(DbTableHash *tb, int free_records) MY_ASSERT(newtop->s.is_ext_segment); if (newtop->prev_segtab != NULL) { /* Time to use a smaller segtab */ - erts_smp_atomic_set(&tb->segtab, (erts_aint_t)newtop->prev_segtab); + erts_smp_atomic_set_relb(&tb->segtab, (erts_aint_t)newtop->prev_segtab); tb->nsegs = seg_ix; ASSERT(tb->nsegs == EXTSEG(SEGTAB(tb))->nsegs); } @@ -2431,7 +2439,7 @@ static int free_seg(DbTableHash *tb, int free_records) if (seg_ix > 0) { if (seg_ix < tb->nsegs) SEGTAB(tb)[seg_ix] = NULL; } else { - erts_smp_atomic_set(&tb->segtab, (erts_aint_t)NULL); + erts_smp_atomic_set_relb(&tb->segtab, (erts_aint_t)NULL); } #endif tb->nslots -= SEGSZ; @@ -2526,9 +2534,9 @@ static void grow(DbTableHash* tb, int nactive) } erts_smp_atomic_inc(&tb->nactive); if (from_ix == 0) { - erts_smp_atomic_set(&tb->szm, szm); + erts_smp_atomic_set_relb(&tb->szm, szm); } - erts_smp_atomic_set(&tb->is_resizing, 0); + erts_smp_atomic_set_relb(&tb->is_resizing, 0); /* Finally, let's split the bucket. We try to do it in a smart way to keep link order and avoid unnecessary updates of next-pointers */ @@ -2560,7 +2568,7 @@ static void grow(DbTableHash* tb, int nactive) return; abort: - erts_smp_atomic_set(&tb->is_resizing, 0); + erts_smp_atomic_set_relb(&tb->is_resizing, 0); } @@ -2604,7 +2612,7 @@ static void shrink(DbTableHash* tb, int nactive) erts_smp_atomic_set(&tb->nactive, src_ix); if (dst_ix == 0) { - erts_smp_atomic_set(&tb->szm, low_szm); + erts_smp_atomic_set_relb(&tb->szm, low_szm); } WUNLOCK_HASH(lck); @@ -2618,7 +2626,7 @@ static void shrink(DbTableHash* tb, int nactive) } /*else already done */ - erts_smp_atomic_set(&tb->is_resizing, 0); + erts_smp_atomic_set_relb(&tb->is_resizing, 0); } diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c index a59c0c258d..9a0ba3a418 100644 --- a/erts/emulator/beam/erl_db_tree.c +++ b/erts/emulator/beam/erl_db_tree.c @@ -111,7 +111,7 @@ static void release_stack(DbTableTree* tb, DbTreeStack* stack) { if (stack == &tb->static_stack) { ASSERT(erts_smp_atomic_read(&tb->is_stack_busy) == 1); - erts_smp_atomic_set(&tb->is_stack_busy, 0); + erts_smp_atomic_set_relb(&tb->is_stack_busy, 0); } else { erts_db_free(ERTS_ALC_T_DB_STK, (DbTable *) tb, @@ -179,7 +179,7 @@ static ERTS_INLINE TreeDbTerm* replace_dbterm(DbTableTree *tb, TreeDbTerm* old, static TreeDbTerm *traverse_until(TreeDbTerm *t, int *current, int to); static void check_slot_pos(DbTableTree *tb); static void check_saved_stack(DbTableTree *tb); -static int check_table_tree(TreeDbTerm *t); +static int check_table_tree(DbTableTree* tb, TreeDbTerm *t); #define TREE_DEBUG #endif @@ -194,8 +194,8 @@ static int check_table_tree(TreeDbTerm *t); ** Debugging dump */ -static void do_dump_tree2(int to, void *to_arg, int show, TreeDbTerm *t, - int offset); +static void do_dump_tree2(DbTableTree*, int to, void *to_arg, int show, + TreeDbTerm *t, int offset); #else @@ -1730,6 +1730,7 @@ static int db_select_delete_tree(Process *p, DbTable *tbl, ** Other interface routines (not directly coupled to one bif) */ + /* Display tree contents (for dump) */ static void db_print_tree(int to, void *to_arg, int show, @@ -1740,7 +1741,7 @@ static void db_print_tree(int to, void *to_arg, if (show) erts_print(to, to_arg, "\nTree data dump:\n" "------------------------------------------------\n"); - do_dump_tree2(to, to_arg, show, tb->root, 0); + do_dump_tree2(&tbl->tree, to, to_arg, show, tb->root, 0); if (show) erts_print(to, to_arg, "\n" "------------------------------------------------\n"); @@ -2694,7 +2695,7 @@ static Sint do_cmp_partly_bound(Eterm a, Eterm b, Eterm* b_base, int *done) while (1) { if ((j = do_cmp_partly_bound(*aa++, *bb++, b_base, done)) != 0 || *done) return j; - if (*aa==*bb) + if (is_same(*aa, NULL, *bb, b_base)) return 0; if (is_not_list(*aa) || is_not_list(*bb)) return do_cmp_partly_bound(*aa, *bb, b_base, done); @@ -2742,7 +2743,7 @@ static Sint cmp_partly_bound(Eterm partly_bound_key, Eterm bound_key, Eterm* bk_ erts_fprintf(stderr," > "); else erts_fprintf(stderr," == "); - erts_fprintf(stderr,"%T\n",bound_key); // HALFWORD BUG: printing rterm + erts_fprintf(stderr,"%R\n", bound_key, bk_base); #endif return ret; } @@ -3084,19 +3085,28 @@ static int doit_select_delete(DbTableTree *tb, TreeDbTerm *this, void *ptr, } #ifdef TREE_DEBUG -static void do_dump_tree2(int to, void *to_arg, int show, TreeDbTerm *t, - int offset) +static void do_dump_tree2(DbTableTree* tb, int to, void *to_arg, int show, + TreeDbTerm *t, int offset) { if (t == NULL) - return 0; - do_dump_tree2(to, to_arg, show, t->right, offset + 4); + return; + do_dump_tree2(tb, to, to_arg, show, t->right, offset + 4); if (show) { - erts_print(to, to_arg, "%*s%T (addr = %p, bal = %d)\n" - offset, "", make_tuple(t->dbterm.tpl), + const char* prefix; + Eterm term; + if (tb->common.compress) { + prefix = "key="; + term = GETKEY(tb, t->dbterm.tpl); + } + else { + prefix = ""; + term = make_tuple_rel(t->dbterm.tpl,t->dbterm.tpl); + } + erts_print(to, to_arg, "%*s%s%R (addr = %p, bal = %d)\n", + offset, "", prefix, term, t->dbterm.tpl, t, t->balance); } - do_dump_tree2(to, to_arg, show, t->left, offset + 4); - return sum; + do_dump_tree2(tb, to, to_arg, show, t->left, offset + 4); } #endif @@ -3106,7 +3116,7 @@ static void do_dump_tree2(int to, void *to_arg, int show, TreeDbTerm *t, void db_check_table_tree(DbTable *tbl) { DbTableTree *tb = &tbl->tree; - check_table_tree(tb->root); + check_table_tree(tb, tb->root); check_saved_stack(tb); check_slot_pos(tb); } @@ -3137,7 +3147,7 @@ static void check_slot_pos(DbTableTree *tb) "element position %d is really 0x%08X, when stack says " "it's 0x%08X\n", tb->stack.slot, t, tb->stack.array[tb->stack.pos - 1]); - do_dump_tree2(ERTS_PRINT_STDERR, NULL, 1, tb->root, 0); + do_dump_tree2(tb, ERTS_PRINT_STDERR, NULL, 1, tb->root, 0); } } @@ -3152,14 +3162,14 @@ static void check_saved_stack(DbTableTree *tb) if (t != stack->array[0]) { erts_fprintf(stderr,"tb->stack[0] is 0x%08X, should be 0x%08X\n", stack->array[0], t); - do_dump_tree2(ERTS_PRINT_STDERR, NULL, 1, tb->root, 0); + do_dump_tree2(tb, ERTS_PRINT_STDERR, NULL, 1, tb->root, 0); return; } while (n < stack->pos) { if (t == NULL) { erts_fprintf(stderr, "NULL pointer in tree when stack not empty," " stack depth is %d\n", n); - do_dump_tree2(ERTS_PRINT_STDERR, NULL, 1, tb->root, 0); + do_dump_tree2(tb, ERTS_PRINT_STDERR, NULL, 1, tb->root, 0); return; } n++; @@ -3173,28 +3183,26 @@ static void check_saved_stack(DbTableTree *tb) "represent child pointer in tree!" "(left == 0x%08X, right == 0x%08X\n", n, tb->stack[n], t->left, t->right); - do_dump_tree2(ERTS_PRINT_STDERR, NULL, 1, tb->root, 0); + do_dump_tree2(tb, ERTS_PRINT_STDERR, NULL, 1, tb->root, 0); return; } } } } -static int check_table_tree(TreeDbTerm *t) +static int check_table_tree(DbTableTree* tb, TreeDbTerm *t) { int lh, rh; if (t == NULL) return 0; - lh = check_table_tree(t->left); - rh = check_table_tree(t->right); + lh = check_table_tree(tb, t->left); + rh = check_table_tree(tb, t->right); if ((rh - lh) != t->balance) { erts_fprintf(stderr, "Invalid tree balance for this node:\n"); - erts_fprintf(stderr,"balance = %d, left = 0x%08X, right = 0x%08X\n" - "data = %T", - t->balance, t->left, t->right, - make_tuple(t->dbterm.tpl)); + erts_fprintf(stderr,"balance = %d, left = 0x%08X, right = 0x%08X\n", + t->balance, t->left, t->right); erts_fprintf(stderr,"\nDump:\n---------------------------------\n"); - do_dump_tree2(ERTS_PRINT_STDERR, NULL, 1, t, 0); + do_dump_tree2(tb, ERTS_PRINT_STDERR, NULL, 1, t, 0); erts_fprintf(stderr,"\n---------------------------------\n"); } return ((rh > lh) ? rh : lh) + 1; diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index c3b074f782..e5be1f253a 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -1731,6 +1731,7 @@ Eterm db_prog_match(Process *c_p, Binary *bprog, #define BEGIN_ATOMIC_TRACE(p) \ do { \ if (! atomic_trace) { \ + erts_refc_inc(&bprog->refc, 2); \ erts_smp_proc_unlock((p), ERTS_PROC_LOCK_MAIN); \ erts_smp_block_system(0); \ atomic_trace = !0; \ @@ -1741,6 +1742,9 @@ Eterm db_prog_match(Process *c_p, Binary *bprog, if (atomic_trace) { \ erts_smp_release_system(); \ erts_smp_proc_lock((p), ERTS_PROC_LOCK_MAIN); \ + if (erts_refc_dectest(&bprog->refc, 0) == 0) {\ + erts_bin_free(bprog); \ + } \ atomic_trace = 0; \ } \ } while (0) diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h index 069b71ab0d..401967a8de 100644 --- a/erts/emulator/beam/erl_driver.h +++ b/erts/emulator/beam/erl_driver.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2010. All Rights Reserved. + * Copyright Ericsson AB 1999-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in diff --git a/erts/emulator/beam/erl_drv_thread.c b/erts/emulator/beam/erl_drv_thread.c index 39bbe9633b..dc578f6d2a 100644 --- a/erts/emulator/beam/erl_drv_thread.c +++ b/erts/emulator/beam/erl_drv_thread.c @@ -700,6 +700,13 @@ erl_drv_thread_join(ErlDrvTid tid, void **respp) extern int erts_darwin_main_thread_pipe[2]; extern int erts_darwin_main_thread_result_pipe[2]; +int erl_drv_stolen_main_thread_join(ErlDrvTid tid, void **respp); +int erl_drv_steal_main_thread(char *name, + ErlDrvTid *dtid, + void* (*func)(void*), + void* arg, + ErlDrvThreadOpts *opts); + int erl_drv_stolen_main_thread_join(ErlDrvTid tid, void **respp) diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 5edcd667e7..e3445bcdc5 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -100,14 +100,14 @@ static Uint combined_message_size(Process* p); static void remove_message_buffers(Process* p); static int major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl); static int minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl); -static void do_minor(Process *p, int new_sz, Eterm* objv, int nobj); +static void do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj); static Eterm* sweep_rootset(Rootset *rootset, Eterm* htop, char* src, Uint src_size); static Eterm* sweep_one_area(Eterm* n_hp, Eterm* n_htop, char* src, Uint src_size); static Eterm* sweep_one_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, char* src, Uint src_size); static Eterm* collect_heap_frags(Process* p, Eterm* heap, Eterm* htop, Eterm* objv, int nobj); -static Uint adjust_after_fullsweep(Process *p, int size_before, +static Uint adjust_after_fullsweep(Process *p, Uint size_before, int need, Eterm *objv, int nobj); static void shrink_new_heap(Process *p, Uint new_sz, Eterm *objv, int nobj); static void grow_new_heap(Process *p, Uint new_sz, Eterm* objv, int nobj); @@ -441,7 +441,15 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj) p->last_old_htop = p->old_htop; #endif - return ((int) (HEAP_TOP(p) - HEAP_START(p)) / 10); + /* FIXME: This function should really return an Sint, i.e., a possibly + 64 bit wide signed integer, but that requires updating all the code + that calls it. For now, we just return INT_MAX if the result is too + large for an int. */ + { + Sint result = (HEAP_TOP(p) - HEAP_START(p)) / 10; + if (result >= INT_MAX) return INT_MAX; + else return (int) result; + } } /* @@ -599,7 +607,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, Uint lit_size) char* area; Uint area_size; Eterm* old_htop; - int n; + Uint n; /* * Set GC state. @@ -731,7 +739,7 @@ minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) * This improved Estone by more than 1200 estones on my computer * (Ultra Sparc 10). */ - size_t new_sz = erts_next_heap_size(HEAP_TOP(p) - HEAP_START(p), 1); + Uint new_sz = erts_next_heap_size(HEAP_TOP(p) - HEAP_START(p), 1); /* Create new, empty old_heap */ n_old = (Eterm *) ERTS_HEAP_ALLOC(ERTS_ALC_T_OLD_HEAP, @@ -871,12 +879,12 @@ minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) #endif /* HIPE */ static void -do_minor(Process *p, int new_sz, Eterm* objv, int nobj) +do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj) { Rootset rootset; /* Rootset for GC (stack, dictionary, etc). */ Roots* roots; Eterm* n_htop; - int n; + Uint n; Eterm* ptr; Eterm val; Eterm gval; @@ -1079,14 +1087,14 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) { Rootset rootset; Roots* roots; - int size_before; + Uint size_before; Eterm* n_heap; Eterm* n_htop; char* src = (char *) HEAP_START(p); Uint src_size = (char *) HEAP_TOP(p) - src; char* oh = (char *) OLD_HEAP(p); Uint oh_size = (char *) OLD_HTOP(p) - oh; - int n; + Uint n; Uint new_sz; Uint fragments = MBUF_SIZE(p) + combined_message_size(p); ErlMessage *msgp; @@ -1312,10 +1320,10 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) } static Uint -adjust_after_fullsweep(Process *p, int size_before, int need, Eterm *objv, int nobj) +adjust_after_fullsweep(Process *p, Uint size_before, int need, Eterm *objv, int nobj) { - int wanted, sz, size_after, need_after; - int stack_size = STACK_SZ_ON_HEAP(p); + Uint wanted, sz, size_after, need_after; + Uint stack_size = STACK_SZ_ON_HEAP(p); Uint reclaimed_now; size_after = (HEAP_TOP(p) - HEAP_START(p)); @@ -1915,8 +1923,8 @@ static void grow_new_heap(Process *p, Uint new_sz, Eterm* objv, int nobj) { Eterm* new_heap; - int heap_size = HEAP_TOP(p) - HEAP_START(p); - int stack_size = p->hend - p->stop; + Uint heap_size = HEAP_TOP(p) - HEAP_START(p); + Uint stack_size = p->hend - p->stop; Sint offs; ASSERT(HEAP_SIZE(p) < new_sz); @@ -1954,10 +1962,10 @@ static void shrink_new_heap(Process *p, Uint new_sz, Eterm *objv, int nobj) { Eterm* new_heap; - int heap_size = HEAP_TOP(p) - HEAP_START(p); + Uint heap_size = HEAP_TOP(p) - HEAP_START(p); Sint offs; - int stack_size = p->hend - p->stop; + Uint stack_size = p->hend - p->stop; ASSERT(new_sz < p->heap_sz); sys_memmove(p->heap + new_sz - stack_size, p->stop, stack_size * diff --git a/erts/emulator/beam/erl_goodfit_alloc.c b/erts/emulator/beam/erl_goodfit_alloc.c index 76b206d76f..1cc508ac5a 100644 --- a/erts/emulator/beam/erl_goodfit_alloc.c +++ b/erts/emulator/beam/erl_goodfit_alloc.c @@ -163,10 +163,10 @@ BKT_MIN_SZ(GFAllctr_t *gfallctr, int ix) /* Prototypes of callback functions */ static Block_t * get_free_block (Allctr_t *, Uint, - Block_t *, Uint); -static void link_free_block (Allctr_t *, Block_t *); -static void unlink_free_block (Allctr_t *, Block_t *); -static void update_last_aux_mbc (Allctr_t *, Carrier_t *); + Block_t *, Uint, Uint32); +static void link_free_block (Allctr_t *, Block_t *, Uint32); +static void unlink_free_block (Allctr_t *, Block_t *, Uint32); +static void update_last_aux_mbc (Allctr_t *, Carrier_t *, Uint32); static Eterm info_options (Allctr_t *, char *, int *, void *, Uint **, Uint *); static void init_atoms (void); @@ -197,6 +197,8 @@ erts_gfalc_start(GFAllctr_t *gfallctr, is a struct). */ Allctr_t *allctr = (Allctr_t *) gfallctr; + init->sbmbct = 0; /* Small mbc not yet supported by goodfit */ + sys_memcpy((void *) gfallctr, (void *) &nulled_state, sizeof(GFAllctr_t)); allctr->mbc_header_size = sizeof(Carrier_t); @@ -379,7 +381,7 @@ search_bucket(Allctr_t *allctr, int ix, Uint size) static Block_t * get_free_block(Allctr_t *allctr, Uint size, - Block_t *cand_blk, Uint cand_size) + Block_t *cand_blk, Uint cand_size, Uint32 flags) { GFAllctr_t *gfallctr = (GFAllctr_t *) allctr; int unsafe_bi, min_bi; @@ -398,7 +400,7 @@ get_free_block(Allctr_t *allctr, Uint size, if (blk) { if (cand_blk && cand_size <= BLK_SZ(blk)) return NULL; /* cand_blk was better */ - unlink_free_block(allctr, blk); + unlink_free_block(allctr, blk, flags); return blk; } if (min_bi < NO_OF_BKTS - 1) { @@ -418,14 +420,14 @@ get_free_block(Allctr_t *allctr, Uint size, ASSERT(blk); if (cand_blk && cand_size <= BLK_SZ(blk)) return NULL; /* cand_blk was better */ - unlink_free_block(allctr, blk); + unlink_free_block(allctr, blk, flags); return blk; } static void -link_free_block(Allctr_t *allctr, Block_t *block) +link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) { GFAllctr_t *gfallctr = (GFAllctr_t *) allctr; GFFreeBlock_t *blk = (GFFreeBlock_t *) block; @@ -446,7 +448,7 @@ link_free_block(Allctr_t *allctr, Block_t *block) } static void -unlink_free_block(Allctr_t *allctr, Block_t *block) +unlink_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) { GFAllctr_t *gfallctr = (GFAllctr_t *) allctr; GFFreeBlock_t *blk = (GFFreeBlock_t *) block; @@ -467,7 +469,7 @@ unlink_free_block(Allctr_t *allctr, Block_t *block) } static void -update_last_aux_mbc(Allctr_t *allctr, Carrier_t *mbc) +update_last_aux_mbc(Allctr_t *allctr, Carrier_t *mbc, Uint32 flags) { GFAllctr_t *gfallctr = (GFAllctr_t *) allctr; diff --git a/erts/emulator/beam/erl_instrument.c b/erts/emulator/beam/erl_instrument.c index f3f3c22933..04ea004ef7 100644 --- a/erts/emulator/beam/erl_instrument.c +++ b/erts/emulator/beam/erl_instrument.c @@ -1186,6 +1186,8 @@ erts_instr_init(int stat, int map_stat) sys_memzero((void *) stats->n, sizeof(Stat_t)*(ERTS_ALC_N_MAX+1)); for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) { + if (ERTS_IS_SBMBC_ALLOCATOR_NO__(i)) + continue; if (erts_allctrs_info[i].enabled) stats->ap[i] = &stats->a[i]; else @@ -1199,6 +1201,8 @@ erts_instr_init(int stat, int map_stat) erts_instr_memory_map = 1; erts_instr_stat = 1; for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) { + if (ERTS_IS_SBMBC_ALLOCATOR_NO__(i)) + continue; erts_allctrs[i].alloc = map_stat_alloc; erts_allctrs[i].realloc = map_stat_realloc; erts_allctrs[i].free = map_stat_free; @@ -1209,6 +1213,8 @@ erts_instr_init(int stat, int map_stat) else { erts_instr_stat = 1; for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) { + if (ERTS_IS_SBMBC_ALLOCATOR_NO__(i)) + continue; erts_allctrs[i].alloc = stat_alloc; erts_allctrs[i].realloc = stat_realloc; erts_allctrs[i].free = stat_free; diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c index 9e18997890..587d82f2bb 100644 --- a/erts/emulator/beam/erl_lock_check.c +++ b/erts/emulator/beam/erl_lock_check.c @@ -153,6 +153,7 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "instr", NULL }, { "fix_alloc", "index" }, { "alcu_allocator", "index" }, + { "sbmbc_alloc", "index" }, { "alcu_delayed_free", "index" }, { "mseg", NULL }, #if HALFWORD_HEAP @@ -240,7 +241,7 @@ typedef struct erts_lc_locked_lock_t_ erts_lc_locked_lock_t; struct erts_lc_locked_lock_t_ { erts_lc_locked_lock_t *next; erts_lc_locked_lock_t *prev; - Eterm extra; + UWord extra; Sint16 id; Uint16 flags; }; @@ -441,12 +442,12 @@ new_locked_lock(erts_lc_lock_t *lck, Uint16 op_flags) } static void -print_lock2(char *prefix, Sint16 id, Eterm extra, Uint16 flags, char *suffix) +print_lock2(char *prefix, Sint16 id, Wterm extra, Uint16 flags, char *suffix) { char *lname = (0 <= id && id < ERTS_LOCK_ORDER_SIZE ? erts_lock_order[id].name : "unknown"); - if (is_boxed(extra)) + if (is_not_immed(extra)) erts_fprintf(stderr, "%s'%s:%p%s'%s%s", prefix, @@ -1260,7 +1261,8 @@ erts_lc_init_lock(erts_lc_lock_t *lck, char *name, Uint16 flags) { lck->id = erts_lc_get_lock_order_id(name); - lck->extra = make_boxed(&lck->extra); + lck->extra = &lck->extra; + ASSERT(is_not_immed(lck->extra)); lck->flags = flags; lck->inited = ERTS_LC_INITITALIZED; } @@ -1270,6 +1272,7 @@ erts_lc_init_lock_x(erts_lc_lock_t *lck, char *name, Uint16 flags, Eterm extra) { lck->id = erts_lc_get_lock_order_id(name); lck->extra = extra; + ASSERT(is_immed(lck->extra)); lck->flags = flags; lck->inited = ERTS_LC_INITITALIZED; } diff --git a/erts/emulator/beam/erl_lock_check.h b/erts/emulator/beam/erl_lock_check.h index cdb06d4458..b67f36fa06 100644 --- a/erts/emulator/beam/erl_lock_check.h +++ b/erts/emulator/beam/erl_lock_check.h @@ -39,7 +39,7 @@ typedef struct { int inited; Sint16 id; Uint16 flags; - Eterm extra; + UWord extra; } erts_lc_lock_t; #define ERTS_LC_INITITALIZED 0x7f7f7f7f diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 8b48444904..6e7ac43676 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -472,7 +472,7 @@ static void tmp_alloc_dtor(struct enif_tmp_obj_t* obj) int enif_inspect_iolist_as_binary(ErlNifEnv* env, Eterm term, ErlNifBinary* bin) { struct enif_tmp_obj_t* tobj; - int sz; + Uint sz; if (is_binary(term)) { return enif_inspect_binary(env,term,bin); } @@ -483,7 +483,7 @@ int enif_inspect_iolist_as_binary(ErlNifEnv* env, Eterm term, ErlNifBinary* bin) bin->ref_bin = NULL; return 1; } - if ((sz = io_list_len(term)) < 0) { + if (erts_iolist_size(term, &sz)) { return 0; } @@ -578,7 +578,15 @@ int enif_is_identical(Eterm lhs, Eterm rhs) int enif_compare(Eterm lhs, Eterm rhs) { - return CMP(lhs,rhs); + Sint result = CMP(lhs,rhs); + + if (result < 0) { + return -1; + } else if (result > 0) { + return 1; + } + + return result; } int enif_get_tuple(ErlNifEnv* env, Eterm tpl, int* arity, const Eterm** array) @@ -833,8 +841,11 @@ ERL_NIF_TERM enif_make_uint(ErlNifEnv* env, unsigned i) ERL_NIF_TERM enif_make_long(ErlNifEnv* env, long i) { + if (IS_SSMALL(i)) { + return make_small(i); + } #if SIZEOF_LONG == ERTS_SIZEOF_ETERM - return IS_SSMALL(i) ? make_small(i) : small_to_big(i, alloc_heap(env,2)); + return small_to_big(i, alloc_heap(env,2)); #elif SIZEOF_LONG == 8 ensure_heap(env,3); return erts_sint64_to_big(i, &env->hp); @@ -843,8 +854,11 @@ ERL_NIF_TERM enif_make_long(ErlNifEnv* env, long i) ERL_NIF_TERM enif_make_ulong(ErlNifEnv* env, unsigned long i) { + if (IS_USMALL(0,i)) { + return make_small(i); + } #if SIZEOF_LONG == ERTS_SIZEOF_ETERM - return IS_USMALL(0,i) ? make_small(i) : uint_to_big(i,alloc_heap(env,2)); + return uint_to_big(i,alloc_heap(env,2)); #elif SIZEOF_LONG == 8 ensure_heap(env,3); return erts_uint64_to_big(i, &env->hp); diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c index 1b07024ca1..e6b55c45e4 100644 --- a/erts/emulator/beam/erl_port_task.c +++ b/erts/emulator/beam/erl_port_task.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2006-2010. All Rights Reserved. + * Copyright Ericsson AB 2006-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -658,8 +658,6 @@ erts_port_task_free_port(Port *pp) when scheduled out... */ ErtsPortTask *ptp = port_task_alloc(); erts_smp_port_state_lock(pp); - ASSERT(erts_smp_atomic_read(&erts_ports_alive) > 0); - erts_smp_atomic_dec(&erts_ports_alive); pp->status &= ~ERTS_PORT_SFLG_CLOSING; pp->status |= ERTS_PORT_SFLG_FREE_SCHEDULED; erts_may_save_closed_port(pp); @@ -681,7 +679,6 @@ erts_port_task_free_port(Port *pp) port_is_dequeued = 1; } erts_smp_port_state_lock(pp); - erts_smp_atomic_dec(&erts_ports_alive); pp->status &= ~ERTS_PORT_SFLG_CLOSING; pp->status |= ERTS_PORT_SFLG_FREE_SCHEDULED; erts_may_save_closed_port(pp); diff --git a/erts/emulator/beam/erl_printf_term.c b/erts/emulator/beam/erl_printf_term.c index b71404fd27..34da9cab84 100644 --- a/erts/emulator/beam/erl_printf_term.c +++ b/erts/emulator/beam/erl_printf_term.c @@ -114,13 +114,13 @@ do { \ /* return 0 if list is not a non-empty flat list of printable characters */ static int -is_printable_string(Eterm list) +is_printable_string(Eterm list, Eterm* base) { int len = 0; int c; while(is_list(list)) { - Eterm* consp = list_val(list); + Eterm* consp = list_val_rel(list, base); Eterm hd = CAR(consp); if (!is_byte(hd)) @@ -226,17 +226,20 @@ static int print_atom_name(fmtfn_t fn, void* arg, Eterm atom, long *dcount) #define PRT_LAST_ARRAY_ELEMENT ((Eterm) 7) /* Note! Must be last... */ static int -print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount) +print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount, + Eterm* obj_base) /* ignored if !HALFWORD_HEAP */ { DECLARE_WSTACK(s); int res; int i; Eterm val; Uint32 *ref_num; + union { + UWord word; + Eterm* ptr; + }popped; Eterm* nobj; -#if HALFWORD_HEAP - UWord wobj; -#endif + Wterm wobj; res = 0; @@ -258,18 +261,17 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount) PRINT_CHAR(res, fn, arg, '}'); goto L_outer_loop; default: -#if HALFWORD_HEAP - obj = (Eterm) (wobj = WSTACK_POP(s)); -#else - obj = WSTACK_POP(s); -#endif + popped.word = WSTACK_POP(s); + switch (val) { case PRT_TERM: + obj = (Eterm) popped.word; break; case PRT_ONE_CONS: + obj = (Eterm) popped.word; L_print_one_cons: { - Eterm* cons = list_val(obj); + Eterm* cons = list_val_rel(obj, obj_base); Eterm tl; obj = CAR(cons); @@ -288,27 +290,13 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount) } break; case PRT_LAST_ARRAY_ELEMENT: - { -#if HALFWORD_HEAP - Eterm* ptr = (Eterm *) wobj; -#else - Eterm* ptr = (Eterm *) obj; -#endif - obj = *ptr; - } + obj = *popped.ptr; break; default: /* PRT_LAST_ARRAY_ELEMENT+1 and upwards */ - { -#if HALFWORD_HEAP - Eterm* ptr = (Eterm *) wobj; -#else - Eterm* ptr = (Eterm *) obj; -#endif - obj = *ptr++; - WSTACK_PUSH(s, (UWord) ptr); - WSTACK_PUSH(s, val-1); - WSTACK_PUSH(s, PRT_COMMA); - } + obj = *popped.ptr; + WSTACK_PUSH(s, (UWord) (popped.ptr + 1)); + WSTACK_PUSH(s, val-1); + WSTACK_PUSH(s, PRT_COMMA); break; } break; @@ -325,8 +313,12 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount) PRINT_CHAR(res, fn, arg, '>'); goto L_done; } - - switch (tag_val_def(obj)) { +#if HALFWORD_HEAP + wobj = is_immed(obj) ? (Wterm)obj : rterm2wterm(obj, obj_base); +#else + wobj = (Wterm)obj; +#endif + switch (tag_val_def(wobj)) { case NIL_DEF: PRINT_STRING(res, fn, arg, "[]"); break; @@ -348,13 +340,13 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount) int print_res; char def_buf[64]; char *buf, *big_str; - Uint sz = (Uint) big_decimal_estimate(obj); + Uint sz = (Uint) big_decimal_estimate(wobj); sz++; if (sz <= 64) buf = &def_buf[0]; else buf = erts_alloc(ERTS_ALC_T_TMP, sz); - big_str = erts_big_to_string(obj, buf, sz); + big_str = erts_big_to_string(wobj, buf, sz); print_res = erts_printf_string(fn, arg, big_str); if (buf != &def_buf[0]) erts_free(ERTS_ALC_T_TMP, (void *) buf); @@ -369,9 +361,9 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount) case EXTERNAL_REF_DEF: PRINT_STRING(res, fn, arg, "#Ref<"); PRINT_ULONG(res, fn, arg, 'u', 0, 1, - (unsigned long) ref_channel_no(obj)); - ref_num = ref_numbers(obj); - for (i = ref_no_of_numbers(obj)-1; i >= 0; i--) { + (unsigned long) ref_channel_no(wobj)); + ref_num = ref_numbers(wobj); + for (i = ref_no_of_numbers(wobj)-1; i >= 0; i--) { PRINT_CHAR(res, fn, arg, '.'); PRINT_ULONG(res, fn, arg, 'u', 0, 1, (unsigned long) ref_num[i]); } @@ -381,30 +373,30 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount) case EXTERNAL_PID_DEF: PRINT_CHAR(res, fn, arg, '<'); PRINT_ULONG(res, fn, arg, 'u', 0, 1, - (unsigned long) pid_channel_no(obj)); + (unsigned long) pid_channel_no(wobj)); PRINT_CHAR(res, fn, arg, '.'); PRINT_ULONG(res, fn, arg, 'u', 0, 1, - (unsigned long) pid_number(obj)); + (unsigned long) pid_number(wobj)); PRINT_CHAR(res, fn, arg, '.'); PRINT_ULONG(res, fn, arg, 'u', 0, 1, - (unsigned long) pid_serial(obj)); + (unsigned long) pid_serial(wobj)); PRINT_CHAR(res, fn, arg, '>'); break; case PORT_DEF: case EXTERNAL_PORT_DEF: PRINT_STRING(res, fn, arg, "#Port<"); PRINT_ULONG(res, fn, arg, 'u', 0, 1, - (unsigned long) port_channel_no(obj)); + (unsigned long) port_channel_no(wobj)); PRINT_CHAR(res, fn, arg, '.'); PRINT_ULONG(res, fn, arg, 'u', 0, 1, - (unsigned long) port_number(obj)); + (unsigned long) port_number(wobj)); PRINT_CHAR(res, fn, arg, '>'); break; case LIST_DEF: - if (is_printable_string(obj)) { + if (is_printable_string(obj, obj_base)) { int c; PRINT_CHAR(res, fn, arg, '"'); - nobj = list_val(obj); + nobj = list_val_rel(obj, obj_base); while (1) { if ((*dcount)-- <= 0) goto L_done; @@ -418,7 +410,7 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount) } if (is_not_list(*nobj)) break; - nobj = list_val(*nobj); + nobj = list_val_rel(*nobj, obj_base); } PRINT_CHAR(res, fn, arg, '"'); } else { @@ -428,7 +420,7 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount) } break; case TUPLE_DEF: - nobj = tuple_val(obj); /* pointer to arity */ + nobj = tuple_val(wobj); /* pointer to arity */ i = arityval(*nobj); /* arity */ PRINT_CHAR(res, fn, arg, '{'); WSTACK_PUSH(s,PRT_CLOSE_TUPLE); @@ -440,13 +432,13 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount) break; case FLOAT_DEF: { FloatDef ff; - GET_DOUBLE(obj, ff); + GET_DOUBLE(wobj, ff); PRINT_DOUBLE(res, fn, arg, 'e', 6, 0, ff.fd); } break; case BINARY_DEF: { - ProcBin* pb = (ProcBin *) binary_val(obj); + ProcBin* pb = (ProcBin *) binary_val(wobj); if (pb->size == 1) PRINT_STRING(res, fn, arg, "<<1 byte>>"); else { @@ -458,7 +450,7 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount) break; case EXPORT_DEF: { - Export* ep = *((Export **) (export_val(obj) + 1)); + Export* ep = *((Export **) (export_val(wobj) + 1)); Atom* module = atom_tab(atom_val(ep->code[0])); Atom* name = atom_tab(atom_val(ep->code[1])); @@ -474,7 +466,7 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount) break; case FUN_DEF: { - ErlFunThing *funp = (ErlFunThing *) fun_val(obj); + ErlFunThing *funp = (ErlFunThing *) fun_val(wobj); Atom *ap = atom_tab(atom_val(funp->fe->module)); PRINT_STRING(res, fn, arg, "#Fun<"); @@ -490,7 +482,7 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount) break; default: PRINT_STRING(res, fn, arg, "<unknown:"); - PRINT_POINTER(res, fn, arg, (UWord) obj); + PRINT_POINTER(res, fn, arg, wobj); PRINT_CHAR(res, fn, arg, '>'); break; } @@ -503,9 +495,10 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount) } int -erts_printf_term(fmtfn_t fn, void* arg, unsigned long term, long precision) +erts_printf_term(fmtfn_t fn, void* arg, unsigned long term, long precision, + unsigned long* term_base) { - int res = print_term(fn, arg, (Uint) term, &precision); + int res = print_term(fn, arg, (Eterm)term, &precision, (Eterm*)term_base); if (res < 0) return res; if (precision <= 0) diff --git a/erts/emulator/beam/erl_printf_term.h b/erts/emulator/beam/erl_printf_term.h index 4f76028396..a48a3de34c 100644 --- a/erts/emulator/beam/erl_printf_term.h +++ b/erts/emulator/beam/erl_printf_term.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2009. All Rights Reserved. + * Copyright Ericsson AB 2005-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -21,6 +21,6 @@ #define ERL_PRINTF_TERM_H__ #include "erl_printf_format.h" -int erts_printf_term(fmtfn_t fn, void* arg, unsigned long term, long precision); - +int erts_printf_term(fmtfn_t fn, void* arg, unsigned long term, long precision, + unsigned long* term_base); #endif diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 31f23d3978..d8aed63544 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -166,9 +166,8 @@ static struct { static struct { erts_smp_mtx_t update_mtx; - erts_smp_atomic32_t active_runqs; + erts_smp_atomic32_t no_runqs; int last_active_runqs; - erts_smp_atomic32_t used_runqs; int forced_check_balance; erts_smp_atomic32_t checking_balance; int halftime; @@ -965,7 +964,7 @@ sched_spin_wait(ErtsSchedulerSleepInfo *ssi, int spincount) erts_aint32_t flgs; do { - flgs = erts_smp_atomic32_read(&ssi->flags); + flgs = erts_smp_atomic32_read_acqb(&ssi->flags); if ((flgs & (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING)) != (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING)) { break; @@ -1114,7 +1113,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) else { erts_aint_t dt; - erts_smp_atomic32_set(&function_calls, 0); + erts_smp_atomic32_set_relb(&function_calls, 0); *fcalls = 0; sched_waiting_sys(esdp->no, rq); @@ -1147,7 +1146,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) nonblockable_aux_work(esdp, ssi, aux_work); #endif - flgs = erts_smp_atomic32_read(&ssi->flags); + flgs = erts_smp_atomic32_read_acqb(&ssi->flags); if (!(flgs & ERTS_SSI_FLG_WAITING)) { ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); goto sys_woken; @@ -1333,6 +1332,76 @@ wake_all_schedulers(void) } } +#define ERTS_NO_USED_RUNQS_SHIFT 16 +#define ERTS_NO_RUNQS_MASK 0xffff + +#if ERTS_MAX_NO_OF_SCHEDULERS > ERTS_NO_RUNQS_MASK +# error "Too large amount of schedulers allowed" +#endif + +static ERTS_INLINE void +init_no_runqs(int active, int used) +{ + erts_aint32_t no_runqs = (erts_aint32_t) (active & ERTS_NO_RUNQS_MASK); + no_runqs |= (erts_aint32_t) ((used & ERTS_NO_RUNQS_MASK) << ERTS_NO_USED_RUNQS_SHIFT); + erts_smp_atomic32_init(&balance_info.no_runqs, no_runqs); +} + +static ERTS_INLINE void +get_no_runqs(int *active, int *used) +{ + erts_aint32_t no_runqs = erts_smp_atomic32_read(&balance_info.no_runqs); + if (active) + *active = (int) (no_runqs & ERTS_NO_RUNQS_MASK); + if (used) + *used = (int) ((no_runqs >> ERTS_NO_USED_RUNQS_SHIFT) & ERTS_NO_RUNQS_MASK); +} + +static ERTS_INLINE void +set_no_used_runqs(int used) +{ + erts_aint32_t exp = erts_smp_atomic32_read(&balance_info.no_runqs); + while (1) { + erts_aint32_t act, new; + new = (used << ERTS_NO_USED_RUNQS_SHIFT) | (exp & ERTS_NO_RUNQS_MASK); + act = erts_smp_atomic32_cmpxchg(&balance_info.no_runqs, new, exp); + if (act == exp) + break; + exp = act; + } +} + +static ERTS_INLINE void +set_no_active_runqs(int active) +{ + erts_aint32_t exp = erts_smp_atomic32_read(&balance_info.no_runqs); + while (1) { + erts_aint32_t act, new; + new = (exp & (ERTS_NO_RUNQS_MASK << ERTS_NO_USED_RUNQS_SHIFT)) | active; + act = erts_smp_atomic32_cmpxchg(&balance_info.no_runqs, new, exp); + if (act == exp) + break; + exp = act; + } +} + +static ERTS_INLINE int +try_inc_no_active_runqs(int active) +{ + erts_aint32_t exp = erts_smp_atomic32_read(&balance_info.no_runqs); + if (((exp >> ERTS_NO_USED_RUNQS_SHIFT) & ERTS_NO_RUNQS_MASK) < active) + return 0; + if ((exp & ERTS_NO_RUNQS_MASK) + 1 == active) { + erts_aint32_t new, act; + new = (exp & ~ERTS_NO_RUNQS_MASK) | active; + act = erts_smp_atomic32_cmpxchg(&balance_info.no_runqs, new, exp); + if (act == exp) + return 1; + } + return 0; +} + + static ERTS_INLINE int chk_wake_sched(ErtsRunQueue *crq, int ix, int activate) { @@ -1344,9 +1413,7 @@ chk_wake_sched(ErtsRunQueue *crq, int ix, int activate) iflgs = erts_smp_atomic32_read(&wrq->info_flags); if (!(iflgs & (ERTS_RUNQ_IFLG_SUSPENDED|ERTS_RUNQ_IFLG_NONEMPTY))) { if (activate) { - if (ix == erts_smp_atomic32_cmpxchg(&balance_info.active_runqs, - ix+1, - ix)) { + if (try_inc_no_active_runqs(ix+1)) { erts_smp_xrunq_lock(crq, wrq); wrq->flags &= ~ERTS_RUNQ_FLG_INACTIVE; erts_smp_xrunq_unlock(crq, wrq); @@ -1363,8 +1430,9 @@ wake_scheduler_on_empty_runq(ErtsRunQueue *crq) { int ix = crq->ix; int stop_ix = ix; - int active_ix = erts_smp_atomic32_read(&balance_info.active_runqs); - int balance_ix = erts_smp_atomic32_read(&balance_info.used_runqs); + int active_ix, balance_ix; + + get_no_runqs(&active_ix, &balance_ix); if (active_ix > balance_ix) active_ix = balance_ix; @@ -1416,7 +1484,7 @@ erts_sched_notify_check_cpu_bind(void) int ix; if (erts_common_run_queue) { for (ix = 0; ix < erts_no_schedulers; ix++) - erts_smp_atomic32_set(&ERTS_SCHEDULER_IX(ix)->chk_cpu_bind, 1); + erts_smp_atomic32_set_relb(&ERTS_SCHEDULER_IX(ix)->chk_cpu_bind, 1); wake_all_schedulers(); } else { @@ -1871,8 +1939,7 @@ try_steal_task(ErtsRunQueue *rq) ERTS_SMP_LC_CHK_RUNQ_LOCK(rq, rq_locked); - active_rqs = erts_smp_atomic32_read(&balance_info.active_runqs); - blnc_rqs = erts_smp_atomic32_read(&balance_info.used_runqs); + get_no_runqs(&active_rqs, &blnc_rqs); if (active_rqs > blnc_rqs) active_rqs = blnc_rqs; @@ -1883,7 +1950,7 @@ try_steal_task(ErtsRunQueue *rq) if (active_rqs < blnc_rqs) { int no = blnc_rqs - active_rqs; int stop_ix = vix = active_rqs + rq->ix % no; - while (erts_smp_atomic32_read(&no_empty_run_queues) < blnc_rqs) { + while (erts_smp_atomic32_read_acqb(&no_empty_run_queues) < blnc_rqs) { res = check_possible_steal_victim(rq, &rq_locked, vix); if (res) goto done; @@ -1898,7 +1965,7 @@ try_steal_task(ErtsRunQueue *rq) vix = rq->ix; /* ... then try to steal a job from another active queue... */ - while (erts_smp_atomic32_read(&no_empty_run_queues) < blnc_rqs) { + while (erts_smp_atomic32_read_acqb(&no_empty_run_queues) < blnc_rqs) { vix++; if (vix >= active_rqs) vix = 0; @@ -1999,7 +2066,7 @@ check_balance(ErtsRunQueue *c_rq) return; } - blnc_no_rqs = (int) erts_smp_atomic32_read(&balance_info.used_runqs); + get_no_runqs(NULL, &blnc_no_rqs); if (blnc_no_rqs == 1) { c_rq->check_balance_reds = INT_MAX; erts_smp_atomic32_set(&balance_info.checking_balance, 0); @@ -2038,7 +2105,8 @@ check_balance(ErtsRunQueue *c_rq) forced = balance_info.forced_check_balance; balance_info.forced_check_balance = 0; - blnc_no_rqs = (int) erts_smp_atomic32_read(&balance_info.used_runqs); + get_no_runqs(¤t_active, &blnc_no_rqs); + if (blnc_no_rqs == 1) { erts_smp_mtx_unlock(&balance_info.update_mtx); erts_smp_runq_lock(c_rq); @@ -2052,8 +2120,6 @@ check_balance(ErtsRunQueue *c_rq) if (balance_info.full_reds_history_index >= ERTS_FULL_REDS_HISTORY_SIZE) balance_info.full_reds_history_index = 0; - current_active = erts_smp_atomic32_read(&balance_info.active_runqs); - /* Read balance information for all run queues */ for (qix = 0; qix < blnc_no_rqs; qix++) { ErtsRunQueue *rq = ERTS_RUNQ_IX(qix); @@ -2387,7 +2453,7 @@ erts_fprintf(stderr, "--------------------------------\n"); } balance_info.last_active_runqs = active; - erts_smp_atomic32_set(&balance_info.active_runqs, active); + set_no_active_runqs(active); balance_info.halftime = 1; erts_smp_atomic32_set(&balance_info.checking_balance, 0); @@ -2695,9 +2761,8 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) erts_smp_atomic32_init(&schdlr_sspnd.msb.ongoing, 0); erts_smp_atomic32_init(&schdlr_sspnd.active, no_schedulers); schdlr_sspnd.msb.procs = NULL; - erts_smp_atomic32_set(&balance_info.used_runqs, - erts_common_run_queue ? 1 : no_schedulers_online); - erts_smp_atomic32_init(&balance_info.active_runqs, no_schedulers); + init_no_runqs(no_schedulers, + erts_common_run_queue ? 1 : no_schedulers_online); balance_info.last_active_runqs = no_schedulers; erts_smp_mtx_init(&balance_info.update_mtx, "migration_info_update"); balance_info.forced_check_balance = 0; @@ -2939,7 +3004,7 @@ sched_spin_suspended(ErtsSchedulerSleepInfo *ssi, int spincount) erts_aint32_t flgs; do { - flgs = erts_smp_atomic32_read(&ssi->flags); + flgs = erts_smp_atomic32_read_acqb(&ssi->flags); if ((flgs & (ERTS_SSI_FLG_SLEEPING | ERTS_SSI_FLG_WAITING | ERTS_SSI_FLG_SUSPENDED)) @@ -3068,7 +3133,7 @@ suspend_scheduler(ErtsSchedulerData *esdp) wake = 0; } - flgs = erts_smp_atomic32_read(&ssi->flags); + flgs = erts_smp_atomic32_read_acqb(&ssi->flags); if (!(flgs & ERTS_SSI_FLG_SUSPENDED)) break; erts_smp_mtx_unlock(&schdlr_sspnd.mtx); @@ -3292,7 +3357,7 @@ erts_set_schedulers_online(Process *p, ErtsRunQueue *to_rq = ERTS_RUNQ_IX(ix % no); evacuate_run_queue(from_rq, to_rq); } - erts_smp_atomic32_set(&balance_info.used_runqs, no); + set_no_used_runqs(no); erts_smp_mtx_unlock(&balance_info.update_mtx); erts_smp_mtx_lock(&schdlr_sspnd.mtx); } @@ -3346,7 +3411,7 @@ erts_set_schedulers_online(Process *p, for (ix = erts_no_run_queues-1; ix >= no; ix--) evacuate_run_queue(ERTS_RUNQ_IX(ix), ERTS_RUNQ_IX(ix % no)); - erts_smp_atomic32_set(&balance_info.used_runqs, no); + set_no_used_runqs(no); erts_smp_mtx_unlock(&balance_info.update_mtx); erts_smp_mtx_lock(&schdlr_sspnd.mtx); for (ix = no; ix < online; ix++) { @@ -3443,7 +3508,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) else { erts_smp_mtx_unlock(&schdlr_sspnd.mtx); erts_smp_mtx_lock(&balance_info.update_mtx); - erts_smp_atomic32_set(&balance_info.used_runqs, 1); + set_no_used_runqs(1); for (ix = 0; ix < online; ix++) { ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); erts_smp_runq_lock(rq); @@ -3580,7 +3645,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) evacuate_run_queue(ERTS_RUNQ_IX(ix), ERTS_RUNQ_IX(ix % online)); - erts_smp_atomic32_set(&balance_info.used_runqs, online); + set_no_used_runqs(online); /* Make sure that we balance soon... */ balance_info.forced_check_balance = 1; erts_smp_runq_lock(ERTS_RUNQ_IX(0)); @@ -3887,21 +3952,9 @@ handle_pend_sync_suspend(Process *suspendee, } } -/* - * Like erts_pid2proc() but: - * - * * At least ERTS_PROC_LOCK_MAIN have to be held on c_p. - * * At least ERTS_PROC_LOCK_MAIN have to be taken on pid. - * * It also waits for proc to be in a state != running and garbing. - * * If ERTS_PROC_LOCK_BUSY is returned, the calling process has to - * yield (ERTS_BIF_YIELD[0-3]()). c_p might in this case have been - * suspended. - */ - - -Process * -erts_pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks, - Eterm pid, ErtsProcLocks pid_locks) +static Process * +pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks, + Eterm pid, ErtsProcLocks pid_locks, int suspend) { Process *rp; int unlock_c_p_status; @@ -3928,7 +3981,7 @@ erts_pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks, c_p->suspendee = NIL; ASSERT(c_p->flags & F_P2PNR_RESCHED); c_p->flags &= ~F_P2PNR_RESCHED; - if (rp) + if (!suspend && rp) resume_process(rp); } else { @@ -3992,6 +4045,8 @@ erts_pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks, } /* rp is not running and we got the locks we want... */ + if (suspend) + suspend_process(rp_rq, rp); } erts_smp_runqs_unlock(cp_rq, rp_rq); } @@ -4004,6 +4059,35 @@ erts_pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks, return rp; } + +/* + * Like erts_pid2proc() but: + * + * * At least ERTS_PROC_LOCK_MAIN have to be held on c_p. + * * At least ERTS_PROC_LOCK_MAIN have to be taken on pid. + * * It also waits for proc to be in a state != running and garbing. + * * If ERTS_PROC_LOCK_BUSY is returned, the calling process has to + * yield (ERTS_BIF_YIELD[0-3]()). c_p might in this case have been + * suspended. + */ +Process * +erts_pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks, + Eterm pid, ErtsProcLocks pid_locks) +{ + return pid2proc_not_running(c_p, c_p_locks, pid, pid_locks, 0); +} + +/* + * Like erts_pid2proc_not_running(), but hands over the process + * in a suspended state unless (c_p is looked up). + */ +Process * +erts_pid2proc_suspend(Process *c_p, ErtsProcLocks c_p_locks, + Eterm pid, ErtsProcLocks pid_locks) +{ + return pid2proc_not_running(c_p, c_p_locks, pid, pid_locks, 1); +} + /* * erts_pid2proc_nropt() is normally the same as * erts_pid2proc_not_running(). However it is only @@ -4117,6 +4201,21 @@ handle_pend_bif_async_suspend(Process *suspendee, } } +#else + +/* + * Non-smp version of erts_pid2proc_suspend(). + */ +Process * +erts_pid2proc_suspend(Process *c_p, ErtsProcLocks c_p_locks, + Eterm pid, ErtsProcLocks pid_locks) +{ + Process *rp = erts_pid2proc(c_p, c_p_locks, pid, pid_locks); + if (rp) + erts_suspend(rp, pid_locks, NULL); + return rp; +} + #endif /* ERTS_SMP */ /* @@ -4650,7 +4749,7 @@ internal_add_to_runq(ErtsRunQueue *runq, Process *p) if (p->status_flags & ERTS_PROC_SFLG_INRUNQ) return NULL; else if (p->runq_flags & ERTS_PROC_RUNQ_FLG_RUNNING) { - ASSERT(p->rcount == 0); + ASSERT(ERTS_PROC_IS_EXITING(p) || p->rcount == 0); ERTS_DBG_CHK_PROCS_RUNQ_NOPROC(runq, p); p->status_flags |= ERTS_PROC_SFLG_PENDADD2SCHEDQ; return NULL; @@ -4661,7 +4760,7 @@ internal_add_to_runq(ErtsRunQueue *runq, Process *p) ERTS_DBG_CHK_PROCS_RUNQ_NOPROC(runq, p); #ifndef ERTS_SMP /* Never schedule a suspended process (ok in smp case) */ - ASSERT(p->rcount == 0); + ASSERT(ERTS_PROC_IS_EXITING(p) || p->rcount == 0); add_runq = runq; #else ASSERT(!p->bound_runq || p->bound_runq == p->run_queue); @@ -5097,7 +5196,7 @@ Process *schedule(Process *p, int calls) esdp = erts_get_scheduler_data(); rq = erts_get_runq_current(esdp); ASSERT(esdp); - fcalls = (int) erts_smp_atomic32_read(&function_calls); + fcalls = (int) erts_smp_atomic32_read_acqb(&function_calls); actual_reds = reds = 0; erts_smp_runq_lock(rq); } else { @@ -5248,14 +5347,14 @@ Process *schedule(Process *p, int calls) | ERTS_RUNQ_FLG_CHK_CPU_BIND | ERTS_RUNQ_FLG_SUSPENDED)) { if ((rq->flags & ERTS_RUNQ_FLG_SUSPENDED) - || (erts_smp_atomic32_read(&esdp->ssi->flags) + || (erts_smp_atomic32_read_acqb(&esdp->ssi->flags) & ERTS_SSI_FLG_SUSPENDED)) { ASSERT(erts_smp_atomic32_read(&esdp->ssi->flags) & ERTS_SSI_FLG_SUSPENDED); suspend_scheduler(esdp); } if ((rq->flags & ERTS_RUNQ_FLG_CHK_CPU_BIND) - || erts_smp_atomic32_read(&esdp->chk_cpu_bind)) { + || erts_smp_atomic32_read_acqb(&esdp->chk_cpu_bind)) { erts_sched_check_cpu_bind(esdp); } } @@ -5306,7 +5405,7 @@ Process *schedule(Process *p, int calls) if (rq->flags & (ERTS_RUNQ_FLG_SHARED_RUNQ | ERTS_RUNQ_FLG_SUSPENDED)) { if ((rq->flags & ERTS_RUNQ_FLG_SUSPENDED) - || (erts_smp_atomic32_read(&esdp->ssi->flags) + || (erts_smp_atomic32_read_acqb(&esdp->ssi->flags) & ERTS_SSI_FLG_SUSPENDED)) { ASSERT(erts_smp_atomic32_read(&esdp->ssi->flags) & ERTS_SSI_FLG_SUSPENDED); @@ -5350,7 +5449,7 @@ Process *schedule(Process *p, int calls) * Schedule system-level activities. */ - erts_smp_atomic32_set(&function_calls, 0); + erts_smp_atomic32_set_relb(&function_calls, 0); fcalls = 0; ASSERT(!erts_port_task_have_outstanding_io_tasks()); @@ -5392,7 +5491,7 @@ Process *schedule(Process *p, int calls) if (erts_common_run_queue->waiting) wake_scheduler(erts_common_run_queue, 0, 1); } - else if (erts_smp_atomic32_read(&no_empty_run_queues) != 0) { + else if (erts_smp_atomic32_read_acqb(&no_empty_run_queues) != 0) { wake_scheduler_on_empty_runq(rq); rq->wakeup_other = 0; } @@ -5683,10 +5782,13 @@ erts_sched_stat_term(Process *p, int total) void erts_schedule_misc_op(void (*func)(void *), void *arg) { - ErtsRunQueue *rq = erts_get_runq_current(NULL); + ErtsRunQueue *rq; ErtsMiscOpList *molp = misc_op_list_alloc(); + ErtsSchedulerData *esdp = erts_get_scheduler_data(); - if (!rq) { + if (esdp) { + rq = esdp->run_queue; + } else { /* * This can only happen when the sys msg dispatcher * thread schedules misc ops (this happens *very* @@ -7603,7 +7705,8 @@ continue_exit_process(Process *p static void timeout_proc(Process* p) { - p->i = *((BeamInstr **) (UWord) p->def_arg_reg); + BeamInstr** pi = (BeamInstr **) p->def_arg_reg; + p->i = *pi; p->flags |= F_TIMO; p->flags &= ~F_INSLPQUEUE; diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 334ae5573f..296acc7367 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -1535,6 +1535,10 @@ erts_get_atom_cache_map(Process *c_p) } #endif +Process *erts_pid2proc_suspend(Process *, + ErtsProcLocks, + Eterm, + ErtsProcLocks); #ifdef ERTS_SMP Process *erts_pid2proc_not_running(Process *, diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index c0397ca6c3..8833137112 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -1538,8 +1538,7 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, Eterm tracee; #endif Eterm transformed_args[MAX_ARG]; - DeclareTmpHeap(sub_bin_heap_et,ERL_SUB_BIN_SIZE,p); - ErlSubBin *sub_bin_heap = (ErlSubBin *) sub_bin_heap_et; + DeclareTypedTmpHeap(ErlSubBin,sub_bin_heap,p); ASSERT(tracer_pid); if (*tracer_pid == am_true) { @@ -1600,21 +1599,20 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, if (is_boxed(arg) && header_is_bin_matchstate(*boxed_val(arg))) { ErlBinMatchState* ms = (ErlBinMatchState *) boxed_val(arg); ErlBinMatchBuffer* mb = &ms->mb; - ErlSubBin* sb = sub_bin_heap; Uint bit_size; ASSERT(sub_bin_heap->thing_word == 0); /* At most one of match context */ bit_size = mb->size - mb->offset; - sb->thing_word = HEADER_SUB_BIN; - sb->size = BYTE_OFFSET(bit_size); - sb->bitsize = BIT_OFFSET(bit_size); - sb->offs = BYTE_OFFSET(mb->offset); - sb->bitoffs = BIT_OFFSET(mb->offset); - sb->is_writable = 0; - sb->orig = mb->orig; - - arg = make_binary(sb); + sub_bin_heap->thing_word = HEADER_SUB_BIN; + sub_bin_heap->size = BYTE_OFFSET(bit_size); + sub_bin_heap->bitsize = BIT_OFFSET(bit_size); + sub_bin_heap->offs = BYTE_OFFSET(mb->offset); + sub_bin_heap->bitoffs = BIT_OFFSET(mb->offset); + sub_bin_heap->is_writable = 0; + sub_bin_heap->orig = mb->orig; + + arg = make_binary(sub_bin_heap); } transformed_args[i] = arg; } diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c index dacf228e92..158eb361a4 100644 --- a/erts/emulator/beam/erl_unicode.c +++ b/erts/emulator/beam/erl_unicode.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2008-2010. All Rights Reserved. + * Copyright Ericsson AB 2008-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 1a102f7187..6953e7fe7d 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -459,6 +459,12 @@ Uint erts_encode_ext_size(Eterm term) + 1 /* VERSION_MAGIC */; } +Uint erts_encode_ext_size_2(Eterm term, unsigned dflags) +{ + return encode_size_struct2(NULL, term, TERM_TO_BINARY_DFLAGS|dflags) + + 1 /* VERSION_MAGIC */; +} + Uint erts_encode_ext_size_ets(Eterm term) { return encode_size_struct2(NULL, term, TERM_TO_BINARY_DFLAGS|DFLAGS_INTERNAL_TAGS); @@ -1262,6 +1268,49 @@ external_size_1(Process* p, Eterm Term) } Eterm +external_size_2(Process* p, Eterm Term, Eterm Flags) +{ + Uint size; + Uint flags = TERM_TO_BINARY_DFLAGS; + + while (is_list(Flags)) { + Eterm arg = CAR(list_val(Flags)); + Eterm* tp; + + if (is_tuple(arg) && *(tp = tuple_val(arg)) == make_arityval(2)) { + if (tp[1] == am_minor_version && is_small(tp[2])) { + switch (signed_val(tp[2])) { + case 0: + break; + case 1: + flags |= DFLAG_NEW_FLOATS; + break; + default: + goto error; + } + } else { + goto error; + } + } else { + error: + BIF_ERROR(p, BADARG); + } + Flags = CDR(list_val(Flags)); + } + if (is_not_nil(Flags)) { + goto error; + } + + size = erts_encode_ext_size_2(Term, flags); + if (IS_USMALL(0, size)) { + BIF_RET(make_small(size)); + } else { + Eterm* hp = HAlloc(p, BIG_UINT_HEAP_SIZE); + BIF_RET(uint_to_big(size, hp)); + } +} + +Eterm erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags) { Uint size; diff --git a/erts/emulator/beam/external.h b/erts/emulator/beam/external.h index d8287b96a4..72fe74baf2 100644 --- a/erts/emulator/beam/external.h +++ b/erts/emulator/beam/external.h @@ -160,6 +160,7 @@ Uint erts_encode_dist_ext_size(Eterm, Uint32, ErtsAtomCacheMap *); void erts_encode_dist_ext(Eterm, byte **, Uint32, ErtsAtomCacheMap *); Uint erts_encode_ext_size(Eterm); +Uint erts_encode_ext_size_2(Eterm, unsigned); Uint erts_encode_ext_size_ets(Eterm); void erts_encode_ext(Eterm, byte **); byte* erts_encode_ext_ets(Eterm, byte *, struct erl_off_heap_header** ext_off_heap); diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 96da894d90..499bdd77ba 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -183,7 +183,7 @@ struct port { process to get (line oriented I/O)*/ Uint32 status; /* Status and type flags */ int control_flags; /* Flags for port_control() */ - Uint32 snapshot; /* Next snapshot that port should be part of */ + erts_aint32_t snapshot; /* Next snapshot that port should be part of */ struct reg_proc *reg; ErlDrvPDL port_data_lock; @@ -527,11 +527,10 @@ union erl_off_heap_ptr { /* arrays that get malloced at startup */ extern Port* erts_port; -extern erts_smp_atomic_t erts_ports_alive; extern Uint erts_max_ports; extern Uint erts_port_tab_index_mask; -extern erts_smp_atomic_t erts_ports_snapshot; +extern erts_smp_atomic32_t erts_ports_snapshot; extern erts_smp_atomic_t erts_dead_ports_ptr; ERTS_GLB_INLINE void erts_may_save_closed_port(Port *prt); @@ -541,12 +540,12 @@ ERTS_GLB_INLINE void erts_may_save_closed_port(Port *prt); ERTS_GLB_INLINE void erts_may_save_closed_port(Port *prt) { ERTS_SMP_LC_ASSERT(erts_smp_lc_spinlock_is_locked(&prt->state_lck)); - if (prt->snapshot != erts_smp_atomic_read(&erts_ports_snapshot)) { + if (prt->snapshot != erts_smp_atomic32_read_acqb(&erts_ports_snapshot)) { /* Dead ports are added from the end of the snapshot buffer */ Eterm* tombstone = (Eterm*) erts_smp_atomic_addtest(&erts_dead_ports_ptr, -(erts_aint_t)sizeof(Eterm)); ASSERT(tombstone+1 != NULL); - ASSERT(prt->snapshot == (Uint32) erts_smp_atomic_read(&erts_ports_snapshot) - 1); + ASSERT(prt->snapshot == erts_smp_atomic32_read(&erts_ports_snapshot) - 1); *tombstone = prt->id; } /*else no ongoing snapshot or port was already included or created after snapshot */ @@ -1653,10 +1652,14 @@ struct Sint_buf { }; char* Sint_to_buf(Sint, struct Sint_buf*); +#define ERTS_IOLIST_OK 0 +#define ERTS_IOLIST_OVERFLOW 1 +#define ERTS_IOLIST_TYPE 2 + Eterm buf_to_intlist(Eterm**, char*, int, Eterm); /* most callers pass plain char*'s */ int io_list_to_buf(Eterm, char*, int); int io_list_to_buf2(Eterm, char*, int); -int io_list_len(Eterm); +int erts_iolist_size(Eterm, Uint *); int is_string(Eterm); void erl_at_exit(void (*) (void*), void*); Eterm collect_memory(Process *); @@ -1890,6 +1893,8 @@ erts_alloc_message_heap(Uint size, # if defined(DEBUG) # define DeclareTmpHeap(VariableName,Size,Process) \ Eterm *VariableName = erts_debug_allocate_tmp_heap(Size,Process) +# define DeclareTypedTmpHeap(Type,VariableName,Process) \ + Type *VariableName = (Type *) erts_debug_allocate_tmp_heap(sizeof(Type)/sizeof(Eterm),Process) # define DeclareTmpHeapNoproc(VariableName,Size) \ Eterm *VariableName = erts_debug_allocate_tmp_heap(Size,NULL) # define UseTmpHeap(Size,Proc) \ @@ -1911,6 +1916,8 @@ erts_alloc_message_heap(Uint size, # else # define DeclareTmpHeap(VariableName,Size,Process) \ Eterm *VariableName = (ERTS_PROC_GET_SCHDATA(Process)->tmp_heap)+(ERTS_PROC_GET_SCHDATA(Process)->num_tmp_heap_used) +# define DeclareTypedTmpHeap(Type,VariableName,Process) \ + Type *VariableName = (Type *) (ERTS_PROC_GET_SCHDATA(Process)->tmp_heap)+(ERTS_PROC_GET_SCHDATA(Process)->num_tmp_heap_used) # define DeclareTmpHeapNoproc(VariableName,Size) \ Eterm *VariableName = (erts_get_scheduler_data()->tmp_heap)+(erts_get_scheduler_data()->num_tmp_heap_used) # define UseTmpHeap(Size,Proc) \ @@ -1936,6 +1943,8 @@ erts_alloc_message_heap(Uint size, #else # define DeclareTmpHeap(VariableName,Size,Process) \ Eterm VariableName[Size] +# define DeclareTypedTmpHeap(Type,VariableName,Process) \ + Type VariableName[1] # define DeclareTmpHeapNoproc(VariableName,Size) \ Eterm VariableName[Size] # define UseTmpHeap(Size,Proc) /* Nothing */ diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index f619c6f88b..df5f8b22a3 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -56,7 +56,6 @@ static erts_smp_tsd_key_t driver_list_last_error_key; /* Save last DDLL error o per thread basis (for BC interfaces) */ Port* erts_port; /* The port table */ -erts_smp_atomic_t erts_ports_alive; erts_smp_atomic_t erts_bytes_out; /* No bytes sent out of the system */ erts_smp_atomic_t erts_bytes_in; /* No bytes gotten into the system */ @@ -82,6 +81,9 @@ static void driver_monitor_unlock_pdl(Port *p); #define DRV_MONITOR_UNLOCK_PDL(Port) /* nothing */ #endif +#define ERL_SMALL_IO_BIN_LIMIT (4*ERL_ONHEAP_BIN_LIMIT) +#define SMALL_WRITE_VEC 16 + static ERTS_INLINE ErlIOQueue* drvport2ioq(ErlDrvPort drvport) { @@ -190,7 +192,7 @@ typedef struct line_buf_context { static erts_smp_spinlock_t get_free_port_lck; static Uint last_port_num; static Uint port_num_mask; -erts_smp_atomic_t erts_ports_snapshot; /* Identifies the _next_ snapshot (not the ongoing) */ +erts_smp_atomic32_t erts_ports_snapshot; /* Identifies the _next_ snapshot (not the ongoing) */ static ERTS_INLINE void @@ -421,10 +423,9 @@ setup_port(Port* prt, Eterm pid, erts_driver_t *driver, new_name = (char*) erts_alloc(ERTS_ALC_T_PORT_NAME, sys_strlen(name)+1); sys_strcpy(new_name, name); erts_smp_runq_lock(runq); - erts_smp_atomic_inc(&erts_ports_alive); erts_smp_port_state_lock(prt); prt->status = ERTS_PORT_SFLG_CONNECTED | xstatus; - prt->snapshot = (Uint32) erts_smp_atomic_read(&erts_ports_snapshot); + prt->snapshot = erts_smp_atomic32_read(&erts_ports_snapshot); old_name = prt->name; prt->name = new_name; #ifdef ERTS_SMP @@ -954,13 +955,14 @@ do { \ int _bitoffs; \ int _bitsize; \ ERTS_GET_REAL_BIN(obj, _real, _offset, _bitoffs, _bitsize); \ - ASSERT(_bitsize == 0); \ + if (_bitsize != 0) goto L_type_error; \ if (thing_subtag(*binary_val(_real)) == REFC_BINARY_SUBTAG && \ _bitoffs == 0) { \ b_size += _size; \ + if (b_size < _size) goto L_overflow_error; \ in_clist = 0; \ v_size++; \ - if (_size >= bin_limit) { \ + if (_size >= ERL_SMALL_IO_BIN_LIMIT) { \ p_in_clist = 0; \ p_v_size++; \ } else { \ @@ -972,6 +974,7 @@ do { \ } \ } else { \ c_size += _size; \ + if (c_size < _size) goto L_overflow_error; \ if (!in_clist) { \ in_clist = 1; \ v_size++; \ @@ -986,29 +989,30 @@ do { \ /* -** Size of a io list in bytes -** return -1 if error -** returns: - Total size of io list -** vsize - SysIOVec size needed for a writev -** csize - Number of bytes not in binary (in the common binary) -** pvsize - SysIOVec size needed if packing small binaries -** pcsize - Number of bytes in the common binary if packing -*/ + * Returns 0 if successful and a non-zero value otherwise. + * + * Return values through pointers: + * *vsize - SysIOVec size needed for a writev + * *csize - Number of bytes not in binary (in the common binary) + * *pvsize - SysIOVec size needed if packing small binaries + * *pcsize - Number of bytes in the common binary if packing + * *total_size - Total size of iolist in bytes + */ static int -io_list_vec_len(Eterm obj, int* vsize, int* csize, - int bin_limit, /* small binaries limit */ - int * pvsize, int * pcsize) +io_list_vec_len(Eterm obj, Uint* vsize, Uint* csize, + Uint* pvsize, Uint* pcsize, Uint* total_size) { DECLARE_ESTACK(s); Eterm* objp; - int v_size = 0; - int c_size = 0; - int b_size = 0; - int in_clist = 0; - int p_v_size = 0; - int p_c_size = 0; - int p_in_clist = 0; + Uint v_size = 0; + Uint c_size = 0; + Uint b_size = 0; + Uint in_clist = 0; + Uint p_v_size = 0; + Uint p_c_size = 0; + Uint p_in_clist = 0; + Uint total; goto L_jump_start; /* avoid a push */ @@ -1022,6 +1026,9 @@ io_list_vec_len(Eterm obj, int* vsize, int* csize, if (is_byte(obj)) { c_size++; + if (c_size == 0) { + goto L_overflow_error; + } if (!in_clist) { in_clist = 1; v_size++; @@ -1061,32 +1068,31 @@ io_list_vec_len(Eterm obj, int* vsize, int* csize, } } + total = c_size + b_size; + if (total < c_size) { + goto L_overflow_error; + } + *total_size = total; + DESTROY_ESTACK(s); - if (vsize != NULL) - *vsize = v_size; - if (csize != NULL) - *csize = c_size; - if (pvsize != NULL) - *pvsize = p_v_size; - if (pcsize != NULL) - *pcsize = p_c_size; - return c_size + b_size; + *vsize = v_size; + *csize = c_size; + *pvsize = p_v_size; + *pcsize = p_c_size; + return 0; L_type_error: + L_overflow_error: DESTROY_ESTACK(s); - return -1; + return 1; } -#define ERL_SMALL_IO_BIN_LIMIT (4*ERL_ONHEAP_BIN_LIMIT) -#define SMALL_WRITE_VEC 16 - - /* write data to a port */ int erts_write_to_port(Eterm caller_id, Port *p, Eterm list) { char *buf; erts_driver_t *drv = p->drv_ptr; - int size; + Uint size; int fpe_was_unmasked; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(p)); @@ -1094,10 +1100,10 @@ int erts_write_to_port(Eterm caller_id, Port *p, Eterm list) p->caller = caller_id; if (drv->outputv != NULL) { - int vsize; - int csize; - int pvsize; - int pcsize; + Uint vsize; + Uint csize; + Uint pvsize; + Uint pcsize; int blimit; SysIOVec iv[SMALL_WRITE_VEC]; ErlDrvBinary* bv[SMALL_WRITE_VEC]; @@ -1106,9 +1112,8 @@ int erts_write_to_port(Eterm caller_id, Port *p, Eterm list) ErlDrvBinary* cbin; ErlIOVec ev; - if ((size = io_list_vec_len(list, &vsize, &csize, - ERL_SMALL_IO_BIN_LIMIT, - &pvsize, &pcsize)) < 0) { + if (io_list_vec_len(list, &vsize, &csize, + &pvsize, &pcsize, &size)) { goto bad_value; } /* To pack or not to pack (small binaries) ...? */ @@ -1183,7 +1188,7 @@ int erts_write_to_port(Eterm caller_id, Port *p, Eterm list) else { ASSERT(r == -1); /* Overflow */ erts_free(ERTS_ALC_T_TMP, buf); - if ((size = io_list_len(list)) < 0) { + if (erts_iolist_size(list, &size)) { goto bad_value; } @@ -1274,7 +1279,6 @@ void init_io(void) erts_smp_atomic_init(&erts_bytes_out, 0); erts_smp_atomic_init(&erts_bytes_in, 0); - erts_smp_atomic_init(&erts_ports_alive, 0); for (i = 0; i < erts_max_ports; i++) { erts_port_task_init_sched(&erts_port[i].sched); @@ -1296,7 +1300,7 @@ void init_io(void) erts_port[i].port_data_lock = NULL; } - erts_smp_atomic_init(&erts_ports_snapshot, (erts_aint_t) 0); + erts_smp_atomic32_init(&erts_ports_snapshot, (erts_aint32_t) 0); last_port_num = 0; erts_smp_spinlock_init(&get_free_port_lck, "get_free_port"); @@ -2147,7 +2151,7 @@ erts_port_control(Process* p, Port* prt, Uint command, Eterm iolist) byte* to_port = NULL; /* Buffer to write to port. */ /* Initialization is for shutting up warning about use before set. */ - int to_len = 0; /* Length of buffer. */ + Uint to_len = 0; /* Length of buffer. */ int must_free = 0; /* True if the buffer should be freed. */ char port_result[ERL_ONHEAP_BIN_LIMIT]; /* Default buffer for result from port. */ char* port_resp; /* Pointer to result buffer. */ @@ -2192,7 +2196,7 @@ erts_port_control(Process* p, Port* prt, Uint command, Eterm iolist) } else { ASSERT(r == -1); /* Overflow */ erts_free(ERTS_ALC_T_TMP, (void *) to_port); - if ((to_len = io_list_len(iolist)) < 0) { /* Type error */ + if (erts_iolist_size(iolist, &to_len)) { /* Type error */ return THE_NON_VALUE; } must_free = 1; diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab index 6caa1e0b2d..304ce22ef2 100644 --- a/erts/emulator/beam/ops.tab +++ b/erts/emulator/beam/ops.tab @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 1997-2010. All Rights Reserved. +# Copyright Ericsson AB 1997-2011. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in @@ -1236,7 +1236,7 @@ i_bs_init_heap I I I d i_bs_init_heap_bin_heap I I I d -bs_init_bits Fail Sz Words Regs Flags Dst | binary_too_big_bits(Sz) => system_limit Fail +bs_init_bits Fail Sz=o Words Regs Flags Dst => system_limit Fail bs_init_bits Fail Sz=u Words=u==0 Regs Flags Dst => i_bs_init_bits Sz Regs Dst bs_init_bits Fail Sz=u Words Regs Flags Dst => i_bs_init_bits_heap Sz Words Regs Dst diff --git a/erts/emulator/beam/safe_hash.c b/erts/emulator/beam/safe_hash.c index 21d6ce9304..4c54e19cdb 100644 --- a/erts/emulator/beam/safe_hash.c +++ b/erts/emulator/beam/safe_hash.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2008-2009. All Rights Reserved. + * Copyright Ericsson AB 2008-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -99,7 +99,7 @@ static void rehash(SafeHash* h, int grow_limit) erts_free(h->type, (void *) old_tab); } /*else already done */ - erts_smp_atomic_set(&h->is_rehashing, 0); + erts_smp_atomic_set_relb(&h->is_rehashing, 0); } diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index 6b4f3b3b36..a17de717bc 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -3021,13 +3021,25 @@ int io_list_to_buf(Eterm obj, char* buf, int len) return -1; } -int io_list_len(Eterm obj) +/* + * Return 0 if successful, and non-zero if unsuccessful. + */ +int erts_iolist_size(Eterm obj, Uint* sizep) { Eterm* objp; - Sint len = 0; + Uint size = 0; DECLARE_ESTACK(s); goto L_again; +#define SAFE_ADD(Var, Val) \ + do { \ + Uint valvar = (Val); \ + Var += valvar; \ + if (Var < valvar) { \ + goto L_overflow_error; \ + } \ + } while (0) + while (!ESTACK_ISEMPTY(s)) { obj = ESTACK_POP(s); L_again: @@ -3037,9 +3049,12 @@ int io_list_len(Eterm obj) /* Head */ obj = CAR(objp); if (is_byte(obj)) { - len++; + size++; + if (size == 0) { + goto L_overflow_error; + } } else if (is_binary(obj) && binary_bitsize(obj) == 0) { - len += binary_size(obj); + SAFE_ADD(size, binary_size(obj)); } else if (is_list(obj)) { ESTACK_PUSH(s, CDR(objp)); goto L_iter_list; /* on head */ @@ -3051,23 +3066,29 @@ int io_list_len(Eterm obj) if (is_list(obj)) goto L_iter_list; /* on tail */ else if (is_binary(obj) && binary_bitsize(obj) == 0) { - len += binary_size(obj); + SAFE_ADD(size, binary_size(obj)); } else if (is_not_nil(obj)) { goto L_type_error; } } else if (is_binary(obj) && binary_bitsize(obj) == 0) { /* Tail was binary */ - len += binary_size(obj); + SAFE_ADD(size, binary_size(obj)); } else if (is_not_nil(obj)) { goto L_type_error; } } +#undef SAFE_ADD DESTROY_ESTACK(s); - return len; + *sizep = size; + return ERTS_IOLIST_OK; + + L_overflow_error: + DESTROY_ESTACK(s); + return ERTS_IOLIST_OVERFLOW; L_type_error: DESTROY_ESTACK(s); - return -1; + return ERTS_IOLIST_TYPE; } /* return 0 if item is not a non-empty flat list of bytes */ diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c index aec456151c..ebc4469a23 100644 --- a/erts/emulator/drivers/common/inet_drv.c +++ b/erts/emulator/drivers/common/inet_drv.c @@ -1948,7 +1948,7 @@ static int http_response_inetdrv(void *arg, int major, int minor, tcp_descriptor* desc = (tcp_descriptor*) arg; int i = 0; ErlDrvTermData spec[27]; - ErlDrvTermData caller; + ErlDrvTermData caller = ERL_DRV_NIL; if (desc->inet.active == INET_PASSIVE) { /* {inet_async,S,Ref,{ok,{http_response,Version,Status,Phrase}}} */ @@ -2041,7 +2041,7 @@ http_request_inetdrv(void* arg, const http_atom_t* meth, const char* meth_ptr, tcp_descriptor* desc = (tcp_descriptor*) arg; int i = 0; ErlDrvTermData spec[43]; - ErlDrvTermData caller; + ErlDrvTermData caller = ERL_DRV_NIL; if (desc->inet.active == INET_PASSIVE) { /* {inet_async, S, Ref, {ok,{http_request,Meth,Uri,Version}}} */ @@ -2092,7 +2092,7 @@ http_header_inetdrv(void* arg, const http_atom_t* name, const char* name_ptr, tcp_descriptor* desc = (tcp_descriptor*) arg; int i = 0; ErlDrvTermData spec[26]; - ErlDrvTermData caller; + ErlDrvTermData caller = ERL_DRV_NIL; if (desc->inet.active == INET_PASSIVE) { /* {inet_async,S,Ref,{ok,{http_header,Bit,Name,IValue,Value}} */ @@ -2221,7 +2221,7 @@ int ssl_tls_inetdrv(void* arg, unsigned type, unsigned major, unsigned minor, tcp_descriptor* desc = (tcp_descriptor*) arg; int i = 0; ErlDrvTermData spec[28]; - ErlDrvTermData caller; + ErlDrvTermData caller = ERL_DRV_NIL; ErlDrvBinary* bin; int ret; @@ -3709,6 +3709,8 @@ static int inet_ctl_fdopen(inet_descriptor* desc, int domain, int type, /* check that it is a socket and that the socket is bound */ if (IS_SOCKET_ERROR(sock_name(s, (struct sockaddr*) &name, &sz))) return ctl_error(sock_errno(), rbuf, rsize); + if (name.sa.sa_family != domain) + return ctl_error(EINVAL, rbuf, rsize); desc->s = s; if ((desc->event = sock_create_event(desc)) == INVALID_EVENT) return ctl_error(sock_errno(), rbuf, rsize); @@ -7980,11 +7982,11 @@ static int tcp_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf, int len, timeout = get_int32(buf); if (desc->inet.state == TCP_STATE_ACCEPTING) { - unsigned long time_left; - int oid; - ErlDrvTermData ocaller; - int oreq; - unsigned otimeout; + unsigned long time_left = 0; + int oid = 0; + ErlDrvTermData ocaller = ERL_DRV_NIL; + int oreq = 0; + unsigned otimeout = 0; ErlDrvTermData caller = driver_caller(desc->inet.port); MultiTimerData *mtd = NULL,*omtd = NULL; ErlDrvMonitor monitor, omonitor; @@ -9739,7 +9741,7 @@ static int packet_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf, int len, if (desc->active || (len != 8)) return ctl_error(EINVAL, rbuf, rsize); timeout = get_int32(buf); - /* The 2nd arg, Length(4), is ignored for both UDP ans SCTP protocols, + /* The 2nd arg, Length(4), is ignored for both UDP and SCTP protocols, since they are msg-oriented. */ if (enq_async(desc, tbuf, PACKET_REQ_RECV) < 0) diff --git a/erts/emulator/hipe/hipe_bif2.c b/erts/emulator/hipe/hipe_bif2.c index 19cd32c68f..2660f74a82 100644 --- a/erts/emulator/hipe/hipe_bif2.c +++ b/erts/emulator/hipe/hipe_bif2.c @@ -1,9 +1,8 @@ /* * %CopyrightBegin% - - * + * * Copyright Ericsson AB 2001-2011. All Rights Reserved. - * + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the @@ -37,14 +36,25 @@ #include "hipe_arch.h" #include "hipe_stack.h" -BIF_RETTYPE hipe_bifs_show_estack_1(BIF_ALIST_1) +static void proc_unlock(Process* c_p, Process* rp) { + ErtsProcLocks locks = ERTS_PROC_LOCKS_ALL; + if (rp == c_p) { + locks &= ~ERTS_PROC_LOCK_MAIN; + } + if (rp && locks) { + erts_smp_proc_unlock(rp, locks); + } +} + +BIF_RETTYPE hipe_bifs_show_estack_1(BIF_ALIST_1) +{ Process *rp = erts_pid2proc(BIF_P, ERTS_PROC_LOCK_MAIN, BIF_ARG_1, ERTS_PROC_LOCKS_ALL); if (!rp) BIF_ERROR(BIF_P, BADARG); hipe_print_estack(rp); - erts_smp_proc_unlock(rp, ERTS_PROC_LOCKS_ALL); + proc_unlock(BIF_P, rp); BIF_RET(am_true); } @@ -55,7 +65,7 @@ BIF_RETTYPE hipe_bifs_show_heap_1(BIF_ALIST_1) if (!rp) BIF_ERROR(BIF_P, BADARG); hipe_print_heap(rp); - erts_smp_proc_unlock(rp, ERTS_PROC_LOCKS_ALL); + proc_unlock(BIF_P, rp); BIF_RET(am_true); } @@ -66,7 +76,7 @@ BIF_RETTYPE hipe_bifs_show_nstack_1(BIF_ALIST_1) if (!rp) BIF_ERROR(BIF_P, BADARG); hipe_print_nstack(rp); - erts_smp_proc_unlock(rp, ERTS_PROC_LOCKS_ALL); + proc_unlock(BIF_P, rp); BIF_RET(am_true); } @@ -82,7 +92,7 @@ BIF_RETTYPE hipe_bifs_show_pcb_1(BIF_ALIST_1) if (!rp) BIF_ERROR(BIF_P, BADARG); hipe_print_pcb(rp); - erts_smp_proc_unlock(rp, ERTS_PROC_LOCKS_ALL); + proc_unlock(BIF_P, rp); BIF_RET(am_true); } diff --git a/erts/emulator/hipe/hipe_bif64.c b/erts/emulator/hipe/hipe_bif64.c index 73784fc1e8..baaf5af2cd 100644 --- a/erts/emulator/hipe/hipe_bif64.c +++ b/erts/emulator/hipe/hipe_bif64.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2010. All Rights Reserved. + * Copyright Ericsson AB 2001-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in diff --git a/erts/emulator/hipe/hipe_bif64.h b/erts/emulator/hipe/hipe_bif64.h index 9fd6b79605..6d494886ec 100644 --- a/erts/emulator/hipe/hipe_bif64.h +++ b/erts/emulator/hipe/hipe_bif64.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2009. All Rights Reserved. + * Copyright Ericsson AB 2001-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in diff --git a/erts/emulator/hipe/hipe_bif64.tab b/erts/emulator/hipe/hipe_bif64.tab index 6e663cfd35..228318af39 100644 --- a/erts/emulator/hipe/hipe_bif64.tab +++ b/erts/emulator/hipe/hipe_bif64.tab @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 2004-2009. All Rights Reserved. +# Copyright Ericsson AB 2004-2011. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in diff --git a/erts/emulator/hipe/hipe_mkliterals.c b/erts/emulator/hipe/hipe_mkliterals.c index 650861b54b..bced90785d 100644 --- a/erts/emulator/hipe/hipe_mkliterals.c +++ b/erts/emulator/hipe/hipe_mkliterals.c @@ -587,9 +587,9 @@ static void print_params(FILE *fp, void (*print_param)(FILE*,const struct rts_pa (*print_param)(fp, &rts_params[i]); } -static int do_c(FILE *fp) +static int do_c(FILE *fp, const char* this_exe) { - fprintf(fp, "/* File: hipe_literals.h, generated by hipe_mkliterals */\n"); + fprintf(fp, "/* File: hipe_literals.h, generated by %s */\n", this_exe); fprintf(fp, "#ifndef __HIPE_LITERALS_H__\n"); fprintf(fp, "#define __HIPE_LITERALS_H__\n\n"); print_literals(fp, c_define_literal); @@ -603,9 +603,9 @@ static int do_c(FILE *fp) return 0; } -static int do_e(FILE *fp) +static int do_e(FILE *fp, const char* this_exe) { - fprintf(fp, "%%%% File: hipe_literals.hrl, generated by hipe_mkliterals"); + fprintf(fp, "%%%% File: hipe_literals.hrl, generated by %s", this_exe); fprintf(fp, "\n\n"); print_literals(fp, e_define_literal); fprintf(fp, "\n"); @@ -622,9 +622,9 @@ int main(int argc, const char **argv) compute_crc(); if (argc == 2) { if (strcmp(argv[1], "-c") == 0) - return do_c(stdout); + return do_c(stdout, argv[0]); if (strcmp(argv[1], "-e") == 0) - return do_e(stdout); + return do_e(stdout, argv[0]); } fprintf(stderr, "usage: %s [-c | -e] > output-file\n", argv[0]); return 1; diff --git a/erts/emulator/hipe/hipe_mode_switch.c b/erts/emulator/hipe/hipe_mode_switch.c index 16f8fb1347..e3e8367b62 100644 --- a/erts/emulator/hipe/hipe_mode_switch.c +++ b/erts/emulator/hipe/hipe_mode_switch.c @@ -346,7 +346,12 @@ Process *hipe_mode_switch(Process *p, unsigned cmd, Eterm reg[]) p->arity = callee_arity; } - /* If process is in P_WAITING state, we schedule the next process */ + /* Schedule next process if current process was hibernated or is waiting + for messages */ + if (p->flags & F_HIBERNATE_SCHED) { + p->flags &= ~F_HIBERNATE_SCHED; + goto do_schedule; + } if (p->status == P_WAITING) { goto do_schedule; } diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c index 218bd79584..cd4de21d65 100644 --- a/erts/emulator/sys/common/erl_check_io.c +++ b/erts/emulator/sys/common/erl_check_io.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2006-2009. All Rights Reserved. + * Copyright Ericsson AB 2006-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -1137,6 +1137,11 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait) restart: +#ifdef ERTS_BREAK_REQUESTED + if (ERTS_BREAK_REQUESTED) + erts_do_break_handling(); +#endif + /* Figure out timeout value */ if (do_wait) { erts_time_remaining(&wait_time); diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c index ffa3a6328c..eaef6680dd 100644 --- a/erts/emulator/sys/common/erl_mseg.c +++ b/erts/emulator/sys/common/erl_mseg.c @@ -134,7 +134,16 @@ static int mmap_fd; #define CAN_PARTLY_DESTROY 0 #endif -static const ErtsMsegOpt_t default_opt = ERTS_MSEG_DEFAULT_OPT_INITIALIZER; +const ErtsMsegOpt_t erts_mseg_default_opt = { + 1, /* Use cache */ + 1, /* Preserv data */ + 0, /* Absolute shrink threshold */ + 0 /* Relative shrink threshold */ +#if HALFWORD_HEAP + ,0 /* need low memory */ +#endif +}; + typedef struct cache_desc_t_ { void *seg; @@ -605,18 +614,10 @@ mseg_clear_cache(MemKind* mk) INC_CC(clear_cache); } -static ERTS_INLINE MemKind* type2mk(ErtsAlcType_t atype) +static ERTS_INLINE MemKind* memkind(const ErtsMsegOpt_t *opt) { #if HALFWORD_HEAP - switch (atype) { - case ERTS_ALC_A_ETS: - case ERTS_ALC_A_BINARY: - case ERTS_ALC_A_FIXED_SIZE: - case ERTS_ALC_A_DRIVER: - return &hi_mem; - default: - return &low_mem; - } + return opt->low_mem ? &low_mem : &hi_mem; #else return &the_mem; #endif @@ -628,7 +629,7 @@ mseg_alloc(ErtsAlcType_t atype, Uint *size_p, const ErtsMsegOpt_t *opt) Uint max, min, diff_size, size; cache_desc_t *cd, *cand_cd; void *seg; - MemKind* mk = type2mk(atype); + MemKind* mk = memkind(opt); INC_CC(alloc); @@ -742,7 +743,7 @@ static void mseg_dealloc(ErtsAlcType_t atype, void *seg, Uint size, const ErtsMsegOpt_t *opt) { - MemKind* mk = type2mk(atype); + MemKind* mk = memkind(opt); cache_desc_t *cd; ERTS_MSEG_DEALLOC_STAT(mk,size); @@ -800,7 +801,7 @@ static void * mseg_realloc(ErtsAlcType_t atype, void *seg, Uint old_size, Uint *new_size_p, const ErtsMsegOpt_t *opt) { - MemKind* mk = type2mk(atype); + MemKind* mk = memkind(opt); void *new_seg; Uint new_size; @@ -1372,7 +1373,7 @@ erts_mseg_alloc_opt(ErtsAlcType_t atype, Uint *size_p, const ErtsMsegOpt_t *opt) void * erts_mseg_alloc(ErtsAlcType_t atype, Uint *size_p) { - return erts_mseg_alloc_opt(atype, size_p, &default_opt); + return erts_mseg_alloc_opt(atype, size_p, &erts_mseg_default_opt); } void @@ -1387,7 +1388,7 @@ erts_mseg_dealloc_opt(ErtsAlcType_t atype, void *seg, Uint size, void erts_mseg_dealloc(ErtsAlcType_t atype, void *seg, Uint size) { - erts_mseg_dealloc_opt(atype, seg, size, &default_opt); + erts_mseg_dealloc_opt(atype, seg, size, &erts_mseg_default_opt); } void * @@ -1405,7 +1406,7 @@ void * erts_mseg_realloc(ErtsAlcType_t atype, void *seg, Uint old_size, Uint *new_size_p) { - return erts_mseg_realloc_opt(atype, seg, old_size, new_size_p, &default_opt); + return erts_mseg_realloc_opt(atype, seg, old_size, new_size_p, &erts_mseg_default_opt); } void diff --git a/erts/emulator/sys/common/erl_mseg.h b/erts/emulator/sys/common/erl_mseg.h index d8053eb0d9..8f116030a8 100644 --- a/erts/emulator/sys/common/erl_mseg.h +++ b/erts/emulator/sys/common/erl_mseg.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2002-2010. All Rights Reserved. + * Copyright Ericsson AB 2002-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -60,15 +60,12 @@ typedef struct { int preserv; UWord abs_shrink_th; UWord rel_shrink_th; +#if HALFWORD_HEAP + int low_mem; +#endif } ErtsMsegOpt_t; -#define ERTS_MSEG_DEFAULT_OPT_INITIALIZER \ -{ \ - 1, /* Use cache */ \ - 1, /* Preserv data */ \ - 0, /* Absolute shrink threshold */ \ - 0 /* Relative shrink threshold */ \ -} +extern const ErtsMsegOpt_t erts_mseg_default_opt; void *erts_mseg_alloc(ErtsAlcType_t, Uint *); void *erts_mseg_alloc_opt(ErtsAlcType_t, Uint *, const ErtsMsegOpt_t *); diff --git a/erts/emulator/sys/common/erl_poll.c b/erts/emulator/sys/common/erl_poll.c index 3ae5b8d747..f5c785d683 100644 --- a/erts/emulator/sys/common/erl_poll.c +++ b/erts/emulator/sys/common/erl_poll.c @@ -766,7 +766,7 @@ write_batch_buf(ErtsPollSet ps, ErtsPollBatchBuf *bbp) short filter; int fd = (int) ebuf[i].ident; - switch ((int) ebuf[i].udata) { + switch ((int) (long) ebuf[i].udata) { /* * Since we use a lazy update approach EV_DELETE will @@ -805,7 +805,7 @@ write_batch_buf(ErtsPollSet ps, ErtsPollBatchBuf *bbp) if (fd == (int) ebuf[j].ident) { ebuf[j].udata = (void *) ERTS_POLL_KQ_OP_HANDLED; if (!(ebuf[j].flags & EV_ERROR)) { - switch ((int) ebuf[j].udata) { + switch ((int) (long) ebuf[j].udata) { case ERTS_POLL_KQ_OP_ADD2_W: filter = EVFILT_WRITE; goto rm_add_fb; @@ -823,7 +823,8 @@ write_batch_buf(ErtsPollSet ps, ErtsPollBatchBuf *bbp) } } /* The other add succeded... */ - filter = (((int) ebuf[i].udata == ERTS_POLL_KQ_OP_ADD2_W) + filter = ((((int) (long) ebuf[i].udata) + == ERTS_POLL_KQ_OP_ADD2_W) ? EVFILT_READ : EVFILT_WRITE); rm_add_fb: diff --git a/erts/emulator/sys/win32/erl_poll.c b/erts/emulator/sys/win32/erl_poll.c index 7662f190ef..074e2e247f 100644 --- a/erts/emulator/sys/win32/erl_poll.c +++ b/erts/emulator/sys/win32/erl_poll.c @@ -693,6 +693,7 @@ static void *break_waiter(void *param) ResetEvent(harr[0]); erts_mtx_lock(&break_waiter_lock); erts_atomic32_set(&break_waiter_state,BREAK_WAITER_GOT_BREAK); + ERTS_THR_MEMORY_BARRIER; SetEvent(break_happened_event); erts_mtx_unlock(&break_waiter_lock); break; @@ -700,6 +701,7 @@ static void *break_waiter(void *param) ResetEvent(harr[1]); erts_mtx_lock(&break_waiter_lock); erts_atomic32_set(&break_waiter_state,BREAK_WAITER_GOT_HALT); + ERTS_THR_MEMORY_BARRIER; SetEvent(break_happened_event); erts_mtx_unlock(&break_waiter_lock); break; diff --git a/erts/emulator/test/alloc_SUITE_data/allocator_test.h b/erts/emulator/test/alloc_SUITE_data/allocator_test.h index b869a4079c..8b34375980 100644 --- a/erts/emulator/test/alloc_SUITE_data/allocator_test.h +++ b/erts/emulator/test/alloc_SUITE_data/allocator_test.h @@ -82,15 +82,17 @@ typedef void* erts_cond; #define NO_OF_BKTS ((Ulong) ALC_TEST0(0x102)) #define FIND_BKT(A, I) ((int) ALC_TEST2(0x103, (A), (I))) -/* From erl_bestfit_alloc.c */ -#define IS_AOBF(A) ((Ulong) ALC_TEST1(0x200, (A))) -#define RBT_ROOT(A) ((RBT_t *) ALC_TEST1(0x201, (A))) -#define RBT_PARENT(T) ((RBT_t *) ALC_TEST1(0x202, (T))) -#define RBT_LEFT(T) ((RBT_t *) ALC_TEST1(0x203, (T))) -#define RBT_RIGHT(T) ((RBT_t *) ALC_TEST1(0x204, (T))) -#define RBT_NEXT(T) ((RBTL_t *) ALC_TEST1(0x205, (T))) -#define RBT_IS_BLACK(T) ((Ulong) ALC_TEST1(0x206, (T))) -#define RBT_IS_TREE(T) ((Ulong) ALC_TEST1(0x207, (T))) +/* From erl_bestfit_alloc.c and erl_ao_firstfit_alloc.c */ +#define IS_AOBF(A) ((Ulong) ALC_TEST1(RBT_OP(0), (A))) +#define RBT_ROOT(A) ((RBT_t *) ALC_TEST1(RBT_OP(1), (A))) +#define RBT_PARENT(T) ((RBT_t *) ALC_TEST1(RBT_OP(2), (T))) +#define RBT_LEFT(T) ((RBT_t *) ALC_TEST1(RBT_OP(3), (T))) +#define RBT_RIGHT(T) ((RBT_t *) ALC_TEST1(RBT_OP(4), (T))) +#define RBT_NEXT(T) ((RBTL_t *) ALC_TEST1(RBT_OP(5), (T))) +#define RBT_IS_BLACK(T) ((Ulong) ALC_TEST1(RBT_OP(6), (T))) +#define RBT_IS_TREE(T) ((Ulong) ALC_TEST1(RBT_OP(7), (T))) +#define IS_AOFF(A) ((Ulong) ALC_TEST1(RBT_OP(8), (A))) +#define RBT_MAX_SZ(T) ((Ulong) ALC_TEST1(RBT_OP(9), (T))) /* From erl_mseg.c */ #define HAVE_MSEG() ((int) ALC_TEST0(0x400)) diff --git a/erts/emulator/test/alloc_SUITE_data/coalesce.c b/erts/emulator/test/alloc_SUITE_data/coalesce.c index c84da97d35..6f35d3279b 100644 --- a/erts/emulator/test/alloc_SUITE_data/coalesce.c +++ b/erts/emulator/test/alloc_SUITE_data/coalesce.c @@ -267,7 +267,7 @@ void testcase_run(TestCaseState_t *tcs) { char *argv_org[] = {"-tmmbcs1024", "-tsbct2048", "-trmbcmt100", "-tas", NULL, NULL}; - char *alg[] = {"af", "gf", "bf", "aobf", NULL}; + char *alg[] = {"af", "gf", "bf", "aobf", "aoff", NULL}; int i; for (i = 0; alg[i]; i++) { diff --git a/erts/emulator/test/alloc_SUITE_data/rbtree.c b/erts/emulator/test/alloc_SUITE_data/rbtree.c index c97e0aac1a..4e7f821baf 100644 --- a/erts/emulator/test/alloc_SUITE_data/rbtree.c +++ b/erts/emulator/test/alloc_SUITE_data/rbtree.c @@ -34,6 +34,14 @@ typedef struct { #define PRINT_TREE #endif +/* Ugly hack to steer the test code towards the right allocator */ +#define RBT_OP(CMD) (current_rbt_type_op_base + (CMD)) +static enum { + BESTFIT_OP_BASE = 0x200, + AO_FIRSTFIT_OP_BASE = 0x500 +}current_rbt_type_op_base; + + #ifdef PRINT_TREE #define INDENT_STEP 5 @@ -65,12 +73,11 @@ print_tree_aux(TestCaseState_t *tcs, RBT_t *x, int indent) static void -print_tree(TestCaseState_t *tcs, RBT_t *root, int aobf) +print_tree(TestCaseState_t *tcs, RBT_t *root) { - char *type = aobf ? "Size-Adress" : "Size"; - testcase_printf(tcs, " --- %s tree begin ---\r\n", type); + testcase_printf(tcs, " --- Tree begin ---\r\n"); print_tree_aux(tcs, root, 0); - testcase_printf(tcs, " --- %s tree end ---\r\n", type); + testcase_printf(tcs, " --- Tree end ---\r\n"); } #endif @@ -78,7 +85,8 @@ print_tree(TestCaseState_t *tcs, RBT_t *root, int aobf) static RBT_t * check_tree(TestCaseState_t *tcs, Allctr_t *alc, Ulong size) { - int i, max_i, address_order; + enum { BF, AOBF, AOFF }type; + int i, max_i; char stk[128]; RBT_t *root, *x, *y, *res; Ulong x_sz, y_sz, is_x_black; @@ -86,11 +94,14 @@ check_tree(TestCaseState_t *tcs, Allctr_t *alc, Ulong size) res = NULL; - address_order = IS_AOBF(alc); + if (IS_AOBF(alc)) type = AOBF; + else if (IS_AOFF(alc)) type = AOFF; + else type = BF; + root = RBT_ROOT(alc); #ifdef PRINT_TREE - print_tree(tcs, root, address_order); + print_tree(tcs, root); #endif max_i = i = -1; @@ -165,12 +176,18 @@ check_tree(TestCaseState_t *tcs, Allctr_t *alc, Ulong size) if (y) { y_sz = BLK_SZ(y); ASSERT(tcs, RBT_PARENT(y) == x); - if (address_order) { + switch (type) { + case AOBF: ASSERT(tcs, y_sz < x_sz || (y_sz == x_sz && y < x)); - } - else { + break; + case BF: ASSERT(tcs, RBT_IS_TREE(y)); ASSERT(tcs, y_sz < x_sz); + break; + case AOFF: + ASSERT(tcs, y < x); + ASSERT(tcs, RBT_MAX_SZ(y) <= RBT_MAX_SZ(x)); + break; } } @@ -178,16 +195,22 @@ check_tree(TestCaseState_t *tcs, Allctr_t *alc, Ulong size) if (y) { y_sz = BLK_SZ(y); ASSERT(tcs, RBT_PARENT(y) == x); - if (address_order) { + switch (type) { + case AOBF: ASSERT(tcs, y_sz > x_sz || (y_sz == x_sz && y > x)); - } - else { + break; + case BF: ASSERT(tcs, RBT_IS_TREE(y)); ASSERT(tcs, y_sz > x_sz); + break; + case AOFF: + ASSERT(tcs, y > x); + ASSERT(tcs, RBT_MAX_SZ(y) <= RBT_MAX_SZ(x)); + break; } } - if (!address_order) { + if (type == BF) { Ulong l_sz; RBTL_t *l = RBT_NEXT(x); for (l = RBT_NEXT(x); l; l = RBT_NEXT(l)) { @@ -202,13 +225,20 @@ check_tree(TestCaseState_t *tcs, Allctr_t *alc, Ulong size) res = x; else { y_sz = BLK_SZ(res); - if (address_order) { + switch (type) { + case AOBF: if (x_sz < y_sz || (x_sz == y_sz && x < res)) res = x; - } - else { - if (!res || x_sz < y_sz) + break; + case BF: + if (x_sz < y_sz) res = x; + break; + case AOFF: + if (x < res) { + res = x; + } + break; } } } @@ -257,7 +287,7 @@ static void test_it(TestCaseState_t *tcs) { int i; - Allctr_t a = ((rbtree_test_data *) tcs->extra)->allocator; + Allctr_t* a = ((rbtree_test_data *) tcs->extra)->allocator; void **blk = ((rbtree_test_data *) tcs->extra)->blk; void **fence = ((rbtree_test_data *) tcs->extra)->fence; Ulong min_blk_sz; @@ -338,6 +368,7 @@ testcase_run(TestCaseState_t *tcs) { char *argv1[] = {"-tasbf", NULL}; char *argv2[] = {"-tasaobf", NULL}; + char *argv3[] = {"-tasaoff", NULL}; Allctr_t *a; rbtree_test_data *td; @@ -355,6 +386,7 @@ testcase_run(TestCaseState_t *tcs) testcase_printf(tcs, "Starting test of best fit...\n"); + current_rbt_type_op_base = BESTFIT_OP_BASE; td->allocator = a = START_ALC("rbtree_bf_", 0, argv1); ASSERT(tcs, a); @@ -371,6 +403,7 @@ testcase_run(TestCaseState_t *tcs) testcase_printf(tcs, "Starting test of address order best fit...\n"); + current_rbt_type_op_base = BESTFIT_OP_BASE; td->allocator = a = START_ALC("rbtree_aobf_", 0, argv2); ASSERT(tcs, a); @@ -383,4 +416,19 @@ testcase_run(TestCaseState_t *tcs) testcase_printf(tcs, "Address order best fit test succeeded!\n"); + /* Address order first fit... */ + + testcase_printf(tcs, "Starting test of address order first fit...\n"); + + current_rbt_type_op_base = AO_FIRSTFIT_OP_BASE; + td->allocator = a = START_ALC("rbtree_aoff_", 0, argv3); + + ASSERT(tcs, a); + + test_it(tcs); + + STOP_ALC(a); + td->allocator = NULL; + + testcase_printf(tcs, "Address order first fit test succeeded!\n"); } diff --git a/erts/emulator/test/binary_SUITE.erl b/erts/emulator/test/binary_SUITE.erl index 7e409f053e..fed5854112 100644 --- a/erts/emulator/test/binary_SUITE.erl +++ b/erts/emulator/test/binary_SUITE.erl @@ -23,12 +23,12 @@ %% Tests binaries and the BIFs: %% list_to_binary/1 %% iolist_to_binary/1 -%% bitstr_to_list/1 +%% list_to_bitstring/1 %% binary_to_list/1 %% binary_to_list/3 %% binary_to_term/1 %% binary_to_term/2 -%% bitstr_to_list/1 +%% bitstring_to_list/1 %% term_to_binary/1 %% erlang:external_size/1 %% size(Binary) @@ -275,12 +275,33 @@ bad_list_to_binary(Config) when is_list(Config) -> ?line test_bad_bin(fun(X, Y) -> X*Y end), ?line test_bad_bin([1,fun(X) -> X + 1 end,2|fun() -> 0 end]), ?line test_bad_bin([fun(X) -> X + 1 end]), + + %% Test iolists that do not fit in the address space. + %% Unfortunately, it would be too slow to test in a 64-bit emulator. + case erlang:system_info(wordsize) of + 4 -> huge_iolists(); + _ -> ok + end. + +huge_iolists() -> + FourGigs = 1 bsl 32, + ?line Sizes = [FourGigs+N || N <- lists:seq(0, 64)] ++ + [1 bsl N || N <- lists:seq(33, 37)], + ?line Base = <<0:(1 bsl 20)/unit:8>>, + [begin + L = build_iolist(Sz, Base), + ?line {'EXIT',{system_limit,_}} = (catch list_to_binary([L])), + ?line {'EXIT',{system_limit,_}} = (catch list_to_bitstring([L])), + ?line {'EXIT',{system_limit,_}} = (catch binary:list_to_bin([L])), + ?line {'EXIT',{system_limit,_}} = (catch iolist_to_binary(L)) + end || Sz <- Sizes], ok. test_bad_bin(List) -> {'EXIT',{badarg,_}} = (catch list_to_binary(List)), {'EXIT',{badarg,_}} = (catch iolist_to_binary(List)), - {'EXIT',{badarg,_}} = (catch list_to_bitstring(List)). + {'EXIT',{badarg,_}} = (catch list_to_bitstring(List)), + {'EXIT',{badarg,_}} = (catch iolist_size(List)). bad_binary_to_list(doc) -> "Tries binary_to_list/1,3 with bad arguments."; bad_binary_to_list(Config) when is_list(Config) -> @@ -457,6 +478,11 @@ terms(Config) when is_list(Config) -> Sz when is_integer(Sz), size(Bin) =< Sz -> ok end, + Bin1 = term_to_binary(Term, [{minor_version, 1}]), + case erlang:external_size(Bin1, [{minor_version, 1}]) of + Sz1 when is_integer(Sz1), size(Bin1) =< Sz1 -> + ok + end, Term = binary_to_term(Bin), Term = binary_to_term(Bin, [safe]), Unaligned = make_unaligned_sub_binary(Bin), @@ -489,7 +515,12 @@ terms_float(Config) when is_list(Config) -> Term = binary_to_term(Bin0), Bin1 = term_to_binary(Term, [{minor_version,1}]), Term = binary_to_term(Bin1), - true = size(Bin1) < size(Bin0) + true = size(Bin1) < size(Bin0), + Size0 = erlang:external_size(Term), + Size00 = erlang:external_size(Term, [{minor_version, 0}]), + Size1 = erlang:external_size(Term, [{minor_version, 1}]), + true = (Size0 =:= Size00), + true = Size1 < Size0 end). external_size(Config) when is_list(Config) -> @@ -505,7 +536,9 @@ external_size(Config) when is_list(Config) -> io:format(" Aligned size: ~p\n", [Sz1]), io:format("Unaligned size: ~p\n", [Sz2]), ?line ?t:fail() - end. + end, + ?line erlang:external_size(Bin) =:= erlang:external_size(Bin, [{minor_version, 1}]), + ?line erlang:external_size(Unaligned) =:= erlang:external_size(Unaligned, [{minor_version, 1}]). external_size_1(Term, Size0, Limit) when Size0 < Limit -> case erlang:external_size(Term) of @@ -516,18 +549,65 @@ external_size_1(Term, Size0, Limit) when Size0 < Limit -> external_size_1(_, _, _) -> ok. t_iolist_size(Config) when is_list(Config) -> - %% Build a term whose external size only fits in a big num (on 32-bit CPU). - Bin = iolist_to_binary(lists:seq(0, 254)), - ?line ok = t_iolist_size_1(Bin, 0, 16#7FFFFFFF), - ?line ok = t_iolist_size_1(make_unaligned_sub_binary(Bin), 0, 16#7FFFFFFF). + ?line Seed = now(), + ?line io:format("Seed: ~p", [Seed]), + ?line random:seed(Seed), + ?line Base = <<0:(1 bsl 20)/unit:8>>, + ?line Powers = [1 bsl N || N <- lists:seq(2, 37)], + ?line Sizes0 = [[N - random:uniform(N div 2), + lists:seq(N-2, N+2), + N+N div 2, + N + random:uniform(N div 2)] || + N <- Powers], + %% Test sizes around 1^32 more thoroughly. + FourGigs = 1 bsl 32, + ?line Sizes1 = [FourGigs+N || N <- lists:seq(-8, 40)] ++ Sizes0, + ?line Sizes2 = lists:flatten(Sizes1), + ?line Sizes = lists:usort(Sizes2), + io:format("~p sizes:", [length(Sizes)]), + io:format("~p\n", [Sizes]), + ?line [Sz = iolist_size(build_iolist(Sz, Base)) || Sz <- Sizes], + ok. -t_iolist_size_1(IOList, Size0, Limit) when Size0 < Limit -> - case iolist_size(IOList) of - Size when is_integer(Size), Size0 < Size -> - io:format("~p", [Size]), - t_iolist_size_1([IOList|IOList], Size, Limit) +build_iolist(N, Base) when N < 16 -> + case random:uniform(3) of + 1 -> + <<Bin:N/binary,_/binary>> = Base, + Bin; + _ -> + lists:seq(1, N) + end; +build_iolist(N, Base) when N =< byte_size(Base) -> + case random:uniform(3) of + 1 -> + <<Bin:N/binary,_/binary>> = Base, + Bin; + 2 -> + <<Bin:N/binary,_/binary>> = Base, + [Bin]; + 3 -> + case N rem 2 of + 0 -> + L = build_iolist(N div 2, Base), + [L,L]; + 1 -> + L = build_iolist(N div 2, Base), + [L,L,45] + end end; -t_iolist_size_1(_, _, _) -> ok. +build_iolist(N0, Base) -> + Small = random:uniform(15), + Seq = lists:seq(1, Small), + N = N0 - Small, + case N rem 2 of + 0 -> + L = build_iolist(N div 2, Base), + [L,L|Seq]; + 1 -> + L = build_iolist(N div 2, Base), + [47,L,L|Seq] + end. + bad_binary_to_term_2(doc) -> "OTP-4053."; bad_binary_to_term_2(suite) -> []; @@ -1183,34 +1263,7 @@ deep(Config) when is_list(Config) -> deep_roundtrip(T) -> B = term_to_binary(T), - true = deep_eq(T, binary_to_term(B)). - -%% -%% FIXME: =:= runs out of stack. -%% -deep_eq([H1|T1], [H2|T2]) -> - deep_eq(H1, H2) andalso deep_eq(T1, T2); -deep_eq(T1, T2) when tuple_size(T1) =:= tuple_size(T2) -> - deep_eq_tup(T1, T2, tuple_size(T1)); -deep_eq(T1, T2) when is_function(T1), is_function(T2) -> - {uniq,U1} = erlang:fun_info(T1, uniq), - {index,I1} = erlang:fun_info(T1, index), - {arity,A1} = erlang:fun_info(T1, arity), - {env,E1} = erlang:fun_info(T1, env), - {uniq,U2} = erlang:fun_info(T2, uniq), - {index,I2} = erlang:fun_info(T2, index), - {arity,A2} = erlang:fun_info(T2, arity), - {env,E2} = erlang:fun_info(T2, env), - U1 =:= U2 andalso I1 =:= I2 andalso A1 =:= A2 andalso - deep_eq(E1, E2); -deep_eq(T1, T2) -> - T1 =:= T2. - -deep_eq_tup(_T1, _T2, 0) -> - true; -deep_eq_tup(T1, T2, N) -> - deep_eq(element(N, T1), element(N, T2)) andalso - deep_eq_tup(T1, T2, N-1). + T = binary_to_term(B). obsolete_funs(Config) when is_list(Config) -> erts_debug:set_internal_state(available_internal_state, true), diff --git a/erts/emulator/test/bs_construct_SUITE.erl b/erts/emulator/test/bs_construct_SUITE.erl index 1959803385..7fdf36711b 100644 --- a/erts/emulator/test/bs_construct_SUITE.erl +++ b/erts/emulator/test/bs_construct_SUITE.erl @@ -553,6 +553,11 @@ huge_float_check({'EXIT',{badarg,_}}) -> ok. huge_binary(Config) when is_list(Config) -> ?line 16777216 = size(<<0:(id(1 bsl 26)),(-1):(id(1 bsl 26))>>), + ?line garbage_collect(), + ?line id(<<0:((1 bsl 32)-1)>>), + ?line garbage_collect(), + ?line id(<<0:(id((1 bsl 32)-1))>>), + ?line garbage_collect(), ok. system_limit(Config) when is_list(Config) -> @@ -565,6 +570,10 @@ system_limit(Config) when is_list(Config) -> ?line {'EXIT',{system_limit,_}} = (catch <<(id(<<>>))/binary,0:(id(1 bsl 100))>>), + %% Would fail to load. + ?line {'EXIT',{system_limit,_}} = (catch <<0:(1 bsl 67)>>), + ?line {'EXIT',{system_limit,_}} = (catch <<0:((1 bsl 64)+1)>>), + case WordSize of 4 -> system_limit_32(); @@ -581,6 +590,14 @@ system_limit_32() -> ?line {'EXIT',{system_limit,_}} = (catch <<0:(id(8)),42:536870912/unit:8>>), ?line {'EXIT',{system_limit,_}} = (catch <<0:(id(8)),42:(id(536870912))/unit:8>>), + + %% The size would be silently truncated, resulting in a crash. + ?line {'EXIT',{system_limit,_}} = (catch <<0:(1 bsl 35)>>), + ?line {'EXIT',{system_limit,_}} = (catch <<0:((1 bsl 32)+1)>>), + + %% Would fail to load. + ?line {'EXIT',{system_limit,_}} = (catch <<0:(1 bsl 43)>>), + ?line {'EXIT',{system_limit,_}} = (catch <<0:((1 bsl 40)+1)>>), ok. badarg(Config) when is_list(Config) -> diff --git a/erts/emulator/test/code_SUITE.erl b/erts/emulator/test/code_SUITE.erl index c1a048be75..29cbdedd17 100644 --- a/erts/emulator/test/code_SUITE.erl +++ b/erts/emulator/test/code_SUITE.erl @@ -20,7 +20,9 @@ -module(code_SUITE). -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2, - new_binary_types/1,t_check_process_code/1,t_check_process_code_ets/1, + new_binary_types/1, + t_check_process_code/1,t_check_old_code/1, + t_check_process_code_ets/1, external_fun/1,get_chunk/1,module_md5/1,make_stub/1, make_stub_many_funs/1,constant_pools/1, false_dependency/1,coverage/1]). @@ -31,7 +33,7 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> [new_binary_types, t_check_process_code, - t_check_process_code_ets, external_fun, get_chunk, + t_check_process_code_ets, t_check_old_code, external_fun, get_chunk, module_md5, make_stub, make_stub_many_funs, constant_pools, false_dependency, coverage]. @@ -248,6 +250,32 @@ fun_refc(F) -> Count. +%% Test the erlang:check_old_code/1 BIF. +t_check_old_code(Config) when is_list(Config) -> + ?line Data = ?config(data_dir, Config), + ?line File = filename:join(Data, "my_code_test"), + + ?line erlang:purge_module(my_code_test), + ?line erlang:delete_module(my_code_test), + ?line catch erlang:purge_module(my_code_test), + + ?line false = erlang:check_old_code(my_code_test), + + ?line {ok,my_code_test,Code} = compile:file(File, [binary]), + ?line {module,my_code_test} = code:load_binary(my_code_test, File, Code), + + ?line false = erlang:check_old_code(my_code_test), + ?line {module,my_code_test} = code:load_binary(my_code_test, File, Code), + ?line true = erlang:check_old_code(my_code_test), + + ?line true = erlang:purge_module(my_code_test), + ?line true = erlang:delete_module(my_code_test), + ?line true = erlang:purge_module(my_code_test), + + ?line {'EXIT',_} = (catch erlang:check_old_code([])), + + ok. + external_fun(Config) when is_list(Config) -> ?line false = erlang:function_exported(another_code_test, x, 1), ?line ExtFun = erlang:make_fun(id(another_code_test), x, 1), @@ -498,7 +526,9 @@ do_false_dependency(Init, Code) -> ?line unlink(Pid), exit(Pid, kill), ?line true = erlang:purge_module(cpbugx), ?line true = erlang:delete_module(cpbugx), + ?line code:is_module_native(cpbugx), % test is_module_native on deleted code ?line true = erlang:purge_module(cpbugx), + ?line code:is_module_native(cpbugx), % test is_module_native on purged code ok. false_dependency_loop(Parent, Init, SendInitAck) -> diff --git a/erts/emulator/test/driver_SUITE.erl b/erts/emulator/test/driver_SUITE.erl index 7600a44988..f6cf01ce16 100644 --- a/erts/emulator/test/driver_SUITE.erl +++ b/erts/emulator/test/driver_SUITE.erl @@ -38,7 +38,7 @@ timer_change/1, timer_delay/1, queue_echo/1, - fun_to_port/1, + outputv_errors/1, driver_unloaded/1, io_ready_exit/1, use_fallback_pollset/1, @@ -74,7 +74,8 @@ missing_callbacks/1, smp_select/1, driver_select_use/1, - thread_mseg_alloc_cache_clean/1]). + thread_mseg_alloc_cache_clean/1, + otp_9302/1]). -export([bin_prefix/2]). @@ -129,7 +130,7 @@ end_per_testcase(Case, Config) -> suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> - [fun_to_port, outputv_echo, queue_echo, {group, timer}, + [outputv_errors, outputv_echo, queue_echo, {group, timer}, driver_unloaded, io_ready_exit, use_fallback_pollset, bad_fd_in_pollset, driver_event, fd_change, steal_control, otp_6602, 'driver_system_info_ver1.0', @@ -141,7 +142,8 @@ all() -> smaller_minor_vsn_drv, peek_non_existing_queue, otp_6879, caller, many_events, missing_callbacks, smp_select, driver_select_use, - thread_mseg_alloc_cache_clean]. + thread_mseg_alloc_cache_clean, + otp_9302]. groups() -> [{timer, [], @@ -165,37 +167,89 @@ init_per_group(_GroupName, Config) -> end_per_group(_GroupName, Config) -> Config. - -fun_to_port(doc) -> "Test sending a fun to port with an outputv-capable driver."; -fun_to_port(Config) when is_list(Config) -> +outputv_errors(doc) -> "Test sending bad types to port with an outputv-capable driver."; +outputv_errors(Config) when is_list(Config) -> ?line Path = ?config(data_dir, Config), ?line erl_ddll:start(), ?line ok = load_driver(Path, outputv_drv), - ?line fun_to_port_1(fun() -> 33 end), - ?line fun_to_port_1([fun() -> 42 end]), - ?line fun_to_port_1([1|fun() -> 42 end]), - L = build_io_list(65536), - ?line fun_to_port_1([L,fun() -> 42 end]), - ?line fun_to_port_1([L|fun() -> 42 end]), + outputv_bad_types(fun(T) -> + ?line outputv_errors_1(T), + ?line outputv_errors_1([1|T]), + ?line L = [1,2,3], + ?line outputv_errors_1([L,T]), + ?line outputv_errors_1([L|T]) + end), + outputv_errors_1(42), + + %% Test iolists that do not fit in the address space. + %% Unfortunately, it would be too slow to test in a 64-bit emulator. + case erlang:system_info(wordsize) of + 4 -> outputv_huge_iolists(); + _ -> ok + end. + +outputv_bad_types(Test) -> + Types = [-1,256,atom,42.0,{a,b,c},make_ref(),fun() -> 42 end, + [1|2],<<1:1>>,<<1:9>>,<<1:15>>], + _ = [Test(Type) || Type <- Types], + ok. + +outputv_huge_iolists() -> + FourGigs = 1 bsl 32, + ?line Sizes = [FourGigs+N || N <- lists:seq(0, 64)] ++ + [1 bsl N || N <- lists:seq(33, 37)], + ?line Base = <<0:(1 bsl 20)/unit:8>>, + [begin + ?line L = build_iolist(Sz, Base), + ?line outputv_errors_1(L) + end || Sz <- Sizes], ok. -fun_to_port_1(Term) -> - Port = open_port({spawn,outputv_drv}, []), +outputv_errors_1(Term) -> + Port = open_port({spawn_driver,outputv_drv}, []), {'EXIT',{badarg,_}} = (catch port_command(Port, Term)), port_close(Port). -build_io_list(0) -> []; -build_io_list(1) -> [7]; -build_io_list(N) -> - L = build_io_list(N div 2), +build_iolist(N, Base) when N < 16 -> + case random:uniform(3) of + 1 -> + <<Bin:N/binary,_/binary>> = Base, + Bin; + _ -> + lists:seq(1, N) + end; +build_iolist(N, Base) when N =< byte_size(Base) -> + case random:uniform(3) of + 1 -> + <<Bin:N/binary,_/binary>> = Base, + Bin; + 2 -> + <<Bin:N/binary,_/binary>> = Base, + [Bin]; + 3 -> + case N rem 2 of + 0 -> + L = build_iolist(N div 2, Base), + [L,L]; + 1 -> + L = build_iolist(N div 2, Base), + [L,L,45] + end + end; +build_iolist(N0, Base) -> + Small = random:uniform(15), + Seq = lists:seq(1, Small), + N = N0 - Small, case N rem 2 of - 0 -> [L|L]; - 1 -> [7,L|L] + 0 -> + L = build_iolist(N div 2, Base), + [L,L|Seq]; + 1 -> + L = build_iolist(N div 2, Base), + [47,L,L|Seq] end. - - outputv_echo(doc) -> ["Test echoing data with a driver that supports outputv."]; outputv_echo(Config) when is_list(Config) -> ?line Dog = test_server:timetrap(test_server:minutes(10)), @@ -1838,13 +1892,39 @@ thread_mseg_alloc_cache_clean_test(Port, N, CCI, Size) -> ?line ?t:format("CCC = ~p~n", [CCC]), ?line true = CCC > OCCC, ?line thread_mseg_alloc_cache_clean_test(Port, N-1, CCI, Size). - - + +otp_9302(Config) when is_list(Config) -> + ?line Path = ?config(data_dir, Config), + ?line erl_ddll:start(), + ?line ok = load_driver(Path, otp_9302_drv), + ?line Port = open_port({spawn, otp_9302_drv}, []), + ?line true = is_port(Port), + ?line port_command(Port, ""), + ?line {msg, block} = get_port_msg(Port, infinity), + ?line {msg, job} = get_port_msg(Port, infinity), + ?line case erlang:system_info(thread_pool_size) of + 0 -> + {msg, cancel} = get_port_msg(Port, infinity); + _ -> + ok + end, + ?line {msg, job} = get_port_msg(Port, infinity), + ?line {msg, end_of_jobs} = get_port_msg(Port, infinity), + ?line no_msg = get_port_msg(Port, 2000), + ?line port_close(Port), + ?line ok. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Utilities %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - + +get_port_msg(Port, Timeout) -> + receive + {Port, What} -> + {msg, What} + after Timeout -> + no_msg + end. wait_until(Fun) -> case Fun() of diff --git a/erts/emulator/test/driver_SUITE_data/Makefile.src b/erts/emulator/test/driver_SUITE_data/Makefile.src index 4ac7987d2f..5b3ba1557e 100644 --- a/erts/emulator/test/driver_SUITE_data/Makefile.src +++ b/erts/emulator/test/driver_SUITE_data/Makefile.src @@ -11,7 +11,8 @@ MISC_DRVS = outputv_drv@dll@ \ caller_drv@dll@ \ many_events_drv@dll@ \ missing_callback_drv@dll@ \ - thr_alloc_drv@dll@ + thr_alloc_drv@dll@ \ + otp_9302_drv@dll@ SYS_INFO_DRVS = sys_info_1_0_drv@dll@ \ sys_info_1_1_drv@dll@ \ diff --git a/erts/emulator/test/driver_SUITE_data/otp_9302_drv.c b/erts/emulator/test/driver_SUITE_data/otp_9302_drv.c new file mode 100644 index 0000000000..beee1b735f --- /dev/null +++ b/erts/emulator/test/driver_SUITE_data/otp_9302_drv.c @@ -0,0 +1,232 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2011. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ +#ifdef __WIN32__ +#include <windows.h> +#endif +#ifdef HAVE_UNISTD_H +#include <unistd.h> +#endif +#include "erl_driver.h" + +static void stop(ErlDrvData drv_data); +static ErlDrvData start(ErlDrvPort port, + char *command); +static void output(ErlDrvData drv_data, + char *buf, int len); +static void ready_async(ErlDrvData drv_data, + ErlDrvThreadData thread_data); + +static ErlDrvEntry otp_9302_drv_entry = { + NULL /* init */, + start, + stop, + output, + NULL /* ready_input */, + NULL /* ready_output */, + "otp_9302_drv", + NULL /* finish */, + NULL /* handle */, + NULL /* control */, + NULL /* timeout */, + NULL /* outputv */, + ready_async, + NULL /* flush */, + NULL /* call */, + NULL /* event */, + ERL_DRV_EXTENDED_MARKER, + ERL_DRV_EXTENDED_MAJOR_VERSION, + ERL_DRV_EXTENDED_MINOR_VERSION, + ERL_DRV_FLAG_USE_PORT_LOCKING, + NULL /* handle2 */, + NULL /* handle_monitor */ +}; + +typedef struct Otp9302AsyncData_ Otp9302AsyncData; + +typedef struct { + ErlDrvMutex *mtx; + Otp9302AsyncData *start; + Otp9302AsyncData *end; +} Otp9302MsgQ; + +typedef struct { + ErlDrvPort port; + int smp; + Otp9302MsgQ msgq; +} Otp9302Data; + +struct Otp9302AsyncData_ { + Otp9302AsyncData *next; + ErlDrvPort port; + int smp; + int refc; + int block; + struct { + ErlDrvTermData port; + ErlDrvTermData receiver; + ErlDrvTermData msg; + } term_data; + Otp9302MsgQ *msgq; +}; + + +DRIVER_INIT(otp_9302_drv) +{ + return &otp_9302_drv_entry; +} + +static void stop(ErlDrvData drv_data) +{ + Otp9302Data *data = (Otp9302Data *) drv_data; + if (!data->smp) + erl_drv_mutex_destroy(data->msgq.mtx); + driver_free(data); +} + +static ErlDrvData start(ErlDrvPort port, + char *command) +{ + Otp9302Data *data; + ErlDrvSysInfo sys_info; + + data = driver_alloc(sizeof(Otp9302Data)); + if (!data) + return ERL_DRV_ERROR_GENERAL; + + data->port = port; + + driver_system_info(&sys_info, sizeof(ErlDrvSysInfo)); + data->smp = sys_info.smp_support; + + if (!data->smp) { + data->msgq.start = NULL; + data->msgq.end = NULL; + data->msgq.mtx = erl_drv_mutex_create(""); + if (!data->msgq.mtx) { + driver_free(data); + return ERL_DRV_ERROR_GENERAL; + } + } + + return (ErlDrvData) data; +} + +static void send_reply(Otp9302AsyncData *adata) +{ + ErlDrvTermData spec[] = { + ERL_DRV_PORT, adata->term_data.port, + ERL_DRV_ATOM, adata->term_data.msg, + ERL_DRV_TUPLE, 2 + }; + driver_send_term(adata->port, adata->term_data.receiver, + spec, sizeof(spec)/sizeof(spec[0])); +} + +static void enqueue_reply(Otp9302AsyncData *adata) +{ + Otp9302MsgQ *msgq = adata->msgq; + adata->next = NULL; + adata->refc++; + erl_drv_mutex_lock(msgq->mtx); + if (msgq->end) + msgq->end->next = adata; + else + msgq->end = msgq->start = adata; + msgq->end = adata; + erl_drv_mutex_unlock(msgq->mtx); +} + +static void dequeue_replies(Otp9302AsyncData *adata) +{ + Otp9302MsgQ *msgq = adata->msgq; + erl_drv_mutex_lock(msgq->mtx); + if (--adata->refc == 0) + driver_free(adata); + while (msgq->start) { + send_reply(msgq->start); + adata = msgq->start; + msgq->start = msgq->start->next; + if (--adata->refc == 0) + driver_free(adata); + } + msgq->start = msgq->end = NULL; + erl_drv_mutex_unlock(msgq->mtx); +} + +static void async_invoke(void *data) +{ + Otp9302AsyncData *adata = (Otp9302AsyncData *) data; + if (adata->block) { +#ifdef __WIN32__ + Sleep((DWORD) 2000); +#else + sleep(2); +#endif + } + if (adata->smp) + send_reply(adata); + else + enqueue_reply(adata); +} + +static void ready_async(ErlDrvData drv_data, + ErlDrvThreadData thread_data) +{ + Otp9302AsyncData *adata = (Otp9302AsyncData *) thread_data; + if (adata->smp) + driver_free(adata); + else + dequeue_replies(adata); +} + +static void output(ErlDrvData drv_data, + char *buf, int len) +{ + Otp9302Data *data = (Otp9302Data *) drv_data; + ErlDrvTermData td_port = driver_mk_port(data->port); + ErlDrvTermData td_receiver = driver_caller(data->port); + ErlDrvTermData td_job = driver_mk_atom("job"); + unsigned int key = (unsigned int) data->port; + long id[5]; + Otp9302AsyncData *ad[5]; + int i; + + for (i = 0; i < sizeof(ad)/sizeof(ad[0]); i++) { + ad[i] = driver_alloc(sizeof(Otp9302AsyncData)); + if (!ad[i]) + abort(); + + ad[i]->smp = data->smp; + ad[i]->port = data->port; + ad[i]->block = 0; + ad[i]->refc = 1; + ad[i]->term_data.port = td_port; + ad[i]->term_data.receiver = td_receiver; + ad[i]->term_data.msg = td_job; + ad[i]->msgq = &data->msgq; + } + ad[0]->block = 1; + ad[0]->term_data.msg = driver_mk_atom("block"); + ad[2]->term_data.msg = driver_mk_atom("cancel"); + ad[4]->term_data.msg = driver_mk_atom("end_of_jobs"); + for (i = 0; i < sizeof(id)/sizeof(id[0]); i++) + id[i] = driver_async(data->port, &key, async_invoke, ad[i], driver_free); + if (id[2] > 0) + driver_async_cancel(id[2]); +} diff --git a/erts/emulator/test/fun_SUITE.erl b/erts/emulator/test/fun_SUITE.erl index 7795efe57e..559e540016 100644 --- a/erts/emulator/test/fun_SUITE.erl +++ b/erts/emulator/test/fun_SUITE.erl @@ -647,17 +647,11 @@ refc_dist_1() -> %% Fun is passed in an exit signal. Wait until it is gone. ?line wait_until(fun () -> 4 =/= fun_refc(F2) end), ?line 3 = fun_refc(F2), - erts_debug:set_internal_state(available_internal_state, true), - ?line F_refc = case erts_debug:get_internal_state(force_heap_frags) of - false -> 3; - true -> 2 % GC after bif already decreased it - end, - ?line F_refc = fun_refc(F), - erts_debug:set_internal_state(available_internal_state, false), + ?line true = erlang:garbage_collect(), + ?line 2 = fun_refc(F), refc_dist_send(Node, F). refc_dist_send(Node, F) -> - ?line true = erlang:garbage_collect(), ?line Pid = spawn_link(Node, fun() -> receive {To,Fun} when is_function(Fun) -> diff --git a/erts/emulator/test/hibernate_SUITE.erl b/erts/emulator/test/hibernate_SUITE.erl index 203fa6b48e..82a0aad189 100644 --- a/erts/emulator/test/hibernate_SUITE.erl +++ b/erts/emulator/test/hibernate_SUITE.erl @@ -25,16 +25,16 @@ init_per_group/2,end_per_group/2, init_per_testcase/2,end_per_testcase/2, basic/1,dynamic_call/1,min_heap_size/1,bad_args/1, - messages_in_queue/1,undefined_mfa/1, no_heap/1]). + messages_in_queue/1,undefined_mfa/1,no_heap/1,wake_up_and_bif_trap/1]). %% Used by test cases. --export([basic_hibernator/1,dynamic_call_hibernator/2,messages_in_queue_restart/2, no_heap_loop/0]). +-export([basic_hibernator/1,dynamic_call_hibernator/2,messages_in_queue_restart/2, no_heap_loop/0,characters_to_list_trap/1]). suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> [basic, dynamic_call, min_heap_size, bad_args, messages_in_queue, - undefined_mfa, no_heap]. + undefined_mfa, no_heap, wake_up_and_bif_trap]. groups() -> []. @@ -384,6 +384,31 @@ clean_dict() -> lists:foreach(fun ({Key, _}) -> erase(Key) end, Dict). %% +%% Wake up and then immediatly bif trap with a lengthy computation. +%% + +wake_up_and_bif_trap(doc) -> []; +wake_up_and_bif_trap(suite) -> []; +wake_up_and_bif_trap(Config) when is_list(Config) -> + ?line Self = self(), + ?line Pid = spawn_link(fun() -> erlang:hibernate(?MODULE, characters_to_list_trap, [Self]) end), + ?line Pid ! wakeup, + ?line receive + {ok, Pid0} when Pid0 =:= Pid -> ok + after 5000 -> + ?line ?t:fail(process_blocked) + end, + ?line unlink(Pid), + ?line exit(Pid, bye). + +%% Lengthy computation that traps (in characters_to_list_trap_3). +characters_to_list_trap(Parent) -> + Bin0 = <<"abcdefghijklmnopqrstuvwxz0123456789">>, + Bin = binary:copy(Bin0, 1500), + unicode:characters_to_list(Bin), + Parent ! {ok, self()}. + +%% %% Misc %% diff --git a/erts/emulator/test/match_spec_SUITE.erl b/erts/emulator/test/match_spec_SUITE.erl index 2b21fa58f4..461773114e 100644 --- a/erts/emulator/test/match_spec_SUITE.erl +++ b/erts/emulator/test/match_spec_SUITE.erl @@ -27,8 +27,9 @@ destructive_in_test_bif/1, guard_exceptions/1, unary_plus/1, unary_minus/1, moving_labels/1]). -export([fpe/1]). +-export([otp_9422/1]). --export([runner/2]). +-export([runner/2, loop_runner/3]). -export([f1/1, f2/2, f3/2, fn/1, fn/2, fn/3]). -export([do_boxed_and_small/0]). @@ -57,7 +58,8 @@ all() -> trace_control_word, silent, silent_no_ms, ms_trace2, ms_trace3, boxed_and_small, destructive_in_test_bif, guard_exceptions, unary_plus, unary_minus, fpe, - moving_labels]; + moving_labels, + otp_9422]; true -> [not_run] end. @@ -208,6 +210,43 @@ test_3(Config) when is_list(Config) -> ?line collect(P1, [{trace, P1, call, {?MODULE, f2, [a, b]}, [true]}]), ?line ok. +otp_9422(doc) -> []; +otp_9422(Config) when is_list(Config) -> + Laps = 1000, + ?line Fun1 = fun() -> otp_9422_tracee() end, + ?line P1 = spawn_link(?MODULE, loop_runner, [self(), Fun1, Laps]), + io:format("spawned ~p as tracee\n", [P1]), + + ?line erlang:trace(P1, true, [call, silent]), + + ?line Fun2 = fun() -> otp_9422_trace_changer() end, + ?line P2 = spawn_link(?MODULE, loop_runner, [self(), Fun2, Laps]), + io:format("spawned ~p as trace_changer\n", [P2]), + + start_collect(P1), + start_collect(P2), + + %%receive after 10*1000 -> ok end, + + stop_collect(P1), + stop_collect(P2), + ok. + +otp_9422_tracee() -> + ?MODULE:f1(a), + ?MODULE:f1(b), + ?MODULE:f1(c). + +otp_9422_trace_changer() -> + Pat1 = [{[a], [], [{enable_trace, arity}]}], + ?line erlang:trace_pattern({?MODULE, f1, 1}, Pat1), + Pat2 = [{[b], [], [{disable_trace, arity}]}], + ?line erlang:trace_pattern({?MODULE, f1, 1}, Pat2). + + + + + bad_match_spec_bin(Config) when is_list(Config) -> {'EXIT',{badarg,_}} = (catch ets:match_spec_run([1], <<>>)), B0 = <<1,2>>, @@ -932,6 +971,24 @@ runner(Collector, Fun) -> Collector ! {gone, self()} end. +loop_runner(Collector, Fun, Laps) -> + receive + {go, Collector} -> + go + end, + loop_runner_cont(Collector, Fun, 0, Laps). + +loop_runner_cont(_Collector, _Fun, Laps, Laps) -> + receive + {done, Collector} -> + io:format("loop_runner ~p exit after ~p laps\n", [self(), Laps]), + Collector ! {gone, self()} + end; +loop_runner_cont(Collector, Fun, N, Laps) -> + Fun(), + loop_runner_cont(Collector, Fun, N+1, Laps). + + f1(X) -> {X}. diff --git a/erts/emulator/test/mtx_SUITE_data/mtx_SUITE.c b/erts/emulator/test/mtx_SUITE_data/mtx_SUITE.c index 818023211c..0e4065c26b 100644 --- a/erts/emulator/test/mtx_SUITE_data/mtx_SUITE.c +++ b/erts/emulator/test/mtx_SUITE_data/mtx_SUITE.c @@ -552,13 +552,19 @@ create_rwlock(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) static ERL_NIF_TERM rwlock_op(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { - rwlock_resource_t *rwlr; + /* + * Use a union for pointer type conversion to avoid compiler warnings + * about strict-aliasing violations with gcc-4.1. gcc >= 4.2 does not + * emit the warning. + * TODO: Reconsider use of union once gcc-4.1 is obsolete? + */ + union { void* vp; rwlock_resource_t *p; } rwlr; int blocking, write, wait_locked, wait_unlocked; if (argc != 5) goto badarg; - if (!enif_get_resource(env, argv[0], enif_priv_data(env), (void **) &rwlr)) + if (!enif_get_resource(env, argv[0], enif_priv_data(env), &rwlr.vp)) goto badarg; blocking = get_bool(env, argv[1]); @@ -581,22 +587,22 @@ rwlock_op(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) if (write) { if (blocking) - RWMUTEX_WLOCK(rwlr->rwlock); + RWMUTEX_WLOCK(rwlr.p->rwlock); else - while (EBUSY == RWMUTEX_TRYWLOCK(rwlr->rwlock)); - if (rwlr->lock_check) { - ASSERT(!ATOMIC_READ(&rwlr->is_locked)); - ATOMIC_SET(&rwlr->is_locked, -1); + while (EBUSY == RWMUTEX_TRYWLOCK(rwlr.p->rwlock)); + if (rwlr.p->lock_check) { + ASSERT(!ATOMIC_READ(&rwlr.p->is_locked)); + ATOMIC_SET(&rwlr.p->is_locked, -1); } } else { if (blocking) - RWMUTEX_RLOCK(rwlr->rwlock); + RWMUTEX_RLOCK(rwlr.p->rwlock); else - while (EBUSY == RWMUTEX_TRYRLOCK(rwlr->rwlock)); - if (rwlr->lock_check) { - ASSERT(ATOMIC_READ(&rwlr->is_locked) >= 0); - ATOMIC_INC(&rwlr->is_locked); + while (EBUSY == RWMUTEX_TRYRLOCK(rwlr.p->rwlock)); + if (rwlr.p->lock_check) { + ASSERT(ATOMIC_READ(&rwlr.p->is_locked) >= 0); + ATOMIC_INC(&rwlr.p->is_locked); } } @@ -604,18 +610,18 @@ rwlock_op(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) milli_sleep(wait_locked); if (write) { - if (rwlr->lock_check) { - ASSERT(ATOMIC_READ(&rwlr->is_locked) == -1); - ATOMIC_SET(&rwlr->is_locked, 0); + if (rwlr.p->lock_check) { + ASSERT(ATOMIC_READ(&rwlr.p->is_locked) == -1); + ATOMIC_SET(&rwlr.p->is_locked, 0); } - RWMUTEX_WUNLOCK(rwlr->rwlock); + RWMUTEX_WUNLOCK(rwlr.p->rwlock); } else { - if (rwlr->lock_check) { - ASSERT(ATOMIC_READ(&rwlr->is_locked) > 0); - ATOMIC_DEC(&rwlr->is_locked); + if (rwlr.p->lock_check) { + ASSERT(ATOMIC_READ(&rwlr.p->is_locked) > 0); + ATOMIC_DEC(&rwlr.p->is_locked); } - RWMUTEX_RUNLOCK(rwlr->rwlock); + RWMUTEX_RUNLOCK(rwlr.p->rwlock); } if (wait_unlocked) diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl index 91d695d979..f6344791f1 100644 --- a/erts/emulator/test/nif_SUITE.erl +++ b/erts/emulator/test/nif_SUITE.erl @@ -257,10 +257,54 @@ types(Config) when is_list(Config) -> end, [{},{ok},{{}},{[],{}},{1,2,3,4,5}]), Stuff = [[],{},0,0.0,(1 bsl 100),(fun()-> ok end),make_ref(),self()], - [eq_cmp(A,clone(B)) || A<-Stuff, B<-Stuff], + [eq_cmp(A,clone(B)) || A<-Stuff, B<-Stuff], + + {IntSz, LongSz} = type_sizes(), + UintMax = (1 bsl (IntSz*8)) - 1, + IntMax = UintMax bsr 1, + IntMin = -(IntMax+1), + UlongMax = (1 bsl (LongSz*8)) - 1, + LongMax = UlongMax bsr 1, + LongMin = -(LongMax+1), + Uint64Max = (1 bsl 64) - 1, + Int64Max = Uint64Max bsr 1, + Int64Min = -(Int64Max+1), + Limits = [{IntMin,IntMax},{0,UintMax},{LongMin,LongMax},{0,UlongMax},{Int64Min,Int64Max},{0,Uint64Max}], + io:format("Limits = ~p\n", [Limits]), + lists:foreach(fun(I) -> + R1 = echo_int(I), + %%io:format("echo_int(~p) -> ~p\n", [I, R1]), + R2 = my_echo_int(I, Limits), + ?line R1 = R2, + ?line true = (R1 =:= R2), + ?line true = (R1 == R2) + end, int_list()), + ?line verify_tmpmem(TmpMem), + ?line true = (compare(-1294536544000, -1178704800000) < 0), + ?line true = (compare(-1178704800000, -1294536544000) > 0), + ?line true = (compare(-295147905179352825856, -36893488147419103232) < 0), + ?line true = (compare(-36893488147419103232, -295147905179352825856) > 0), + ?line true = (compare(-29514790517935282585612345678, -36893488147419103232) < 0), + ?line true = (compare(-36893488147419103232, -29514790517935282585612345678) > 0), ok. +int_list() -> + Start = 1 bsl 200, + int_list([Start], -Start). +int_list([N | _]=List, End) when N<End -> + List; +int_list([N | _]=List, End) -> + int_list([N - (1 + (abs(N) div 3)) | List], End). + +my_echo_int(I, Limits) -> + lists:map(fun({Min,Max}) -> + if I < Min -> false; + I > Max -> false; + true -> I + end + end, Limits). + clone(X) -> binary_to_term(term_to_binary(X)). @@ -1170,9 +1214,10 @@ tmpmem() -> MemInfo -> MSBCS = lists:foldl( fun ({instance, _, L}, Acc) -> + {value,{_,SBMBCS}} = lists:keysearch(sbmbcs, 1, L), {value,{_,MBCS}} = lists:keysearch(mbcs, 1, L), {value,{_,SBCS}} = lists:keysearch(sbcs, 1, L), - [MBCS,SBCS | Acc] + [SBMBCS,MBCS,SBCS | Acc] end, [], MemInfo), @@ -1269,6 +1314,8 @@ send_blob_thread(_,_,_) -> ?nif_stub. join_send_thread(_) -> ?nif_stub. copy_blob(_) -> ?nif_stub. send_term(_,_) -> ?nif_stub. +echo_int(_) -> ?nif_stub. +type_sizes() -> ?nif_stub. nif_stub_error(Line) -> exit({nif_not_loaded,module,?MODULE,line,Line}). diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c index dc047394b5..92f1bab8dd 100644 --- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c +++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2009-2010. All Rights Reserved. + * Copyright Ericsson AB 2009-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -28,6 +28,7 @@ static int static_cntA; /* zero by default */ static int static_cntB = NIF_SUITE_LIB_VER * 100; +static ERL_NIF_TERM atom_false; static ERL_NIF_TERM atom_self; static ERL_NIF_TERM atom_ok; static ERL_NIF_TERM atom_join; @@ -40,7 +41,18 @@ typedef struct CallInfo* call_history; NifModPrivData* nif_mod; union { ErlNifResourceType* t; long l; } rt_arr[2]; -}PrivData; +} PrivData; + +/* + * Use a union for pointer type conversion to avoid compiler warnings + * about strict-aliasing violations with gcc-4.1. gcc >= 4.2 does not + * emit the warning. + * TODO: Reconsider use of union once gcc-4.1 is obsolete? + */ +typedef union { + void* vp; + struct make_term_info* p; +} mti_t; void add_call(ErlNifEnv* env, PrivData* data, const char* func_name) { @@ -103,7 +115,7 @@ static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) msgenv_resource_type = enif_open_resource_type(env,NULL,"nif_SUITE.msgenv", msgenv_dtor, ERL_NIF_RT_CREATE, NULL); - + atom_false = enif_make_atom(env,"false"); atom_self = enif_make_atom(env,"self"); atom_ok = enif_make_atom(env,"ok"); atom_join = enif_make_atom(env,"join"); @@ -481,6 +493,45 @@ error: return enif_make_atom(env,"error"); } +static ERL_NIF_TERM echo_int(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + int sint; + unsigned uint; + long slong; + unsigned long ulong; + ErlNifSInt64 sint64; + ErlNifUInt64 uint64; + ERL_NIF_TERM sint_term = atom_false, uint_term = atom_false; + ERL_NIF_TERM slong_term = atom_false, ulong_term = atom_false; + ERL_NIF_TERM sint64_term = atom_false, uint64_term = atom_false; + + if (enif_get_int(env, argv[0], &sint)) { + sint_term = enif_make_int(env, sint); + } + if (enif_get_uint(env, argv[0], &uint)) { + uint_term = enif_make_uint(env, uint); + } + if (enif_get_long(env, argv[0], &slong)) { + slong_term = enif_make_long(env, slong); + } + if (enif_get_ulong(env, argv[0], &ulong)) { + ulong_term = enif_make_ulong(env, ulong); + } + if (enif_get_int64(env, argv[0], &sint64)) { + sint64_term = enif_make_int64(env, sint64); + } + if (enif_get_uint64(env, argv[0], &uint64)) { + uint64_term = enif_make_uint64(env, uint64); + } + return enif_make_list6(env, sint_term, uint_term, slong_term, ulong_term, sint64_term, uint64_term); +} + +static ERL_NIF_TERM type_sizes(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + return enif_make_tuple2(env, enif_make_int(env, sizeof(int)), + enif_make_int(env, sizeof(long))); +} + static ERL_NIF_TERM tuple_2_list(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { int arity = -1; @@ -667,7 +718,7 @@ static ERL_NIF_TERM get_resource_type(ErlNifEnv* env, int argc, const ERL_NIF_TE static ERL_NIF_TERM alloc_resource(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { ErlNifBinary data_bin; - union { ErlNifResourceType* t; long l;} type; + union { ErlNifResourceType* t; long l; } type; union { void* p; long l;} data; if (!enif_get_long(env, argv[0], &type.l) || !enif_inspect_binary(env, argv[1], &data_bin) @@ -691,7 +742,7 @@ static ERL_NIF_TERM make_resource(ErlNifEnv* env, int argc, const ERL_NIF_TERM a static ERL_NIF_TERM make_new_resource(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { ErlNifBinary data_bin; - union { ErlNifResourceType* t; long l;} type; + union { ErlNifResourceType* t; long l; } type; void* data; ERL_NIF_TERM ret; if (!enif_get_long(env, argv[0], &type.l) @@ -709,7 +760,7 @@ static ERL_NIF_TERM make_new_resource(ErlNifEnv* env, int argc, const ERL_NIF_TE static ERL_NIF_TERM make_new_resource_binary(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { ErlNifBinary data_bin; - union { struct binary_resource* p; void* vp; long l;} br; + union { struct binary_resource* p; void* vp; long l; } br; void* buf; ERL_NIF_TERM ret; if (!enif_inspect_binary(env, argv[0], &data_bin) @@ -1229,10 +1280,7 @@ static void msgenv_dtor(ErlNifEnv* env, void* obj) static ERL_NIF_TERM clear_msgenv(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { - union { - void* vp; - struct make_term_info* p; - }mti; + mti_t mti; if (!enif_get_resource(env, argv[0], msgenv_resource_type, &mti.vp)) { return enif_make_badarg(env); } @@ -1245,7 +1293,7 @@ static ERL_NIF_TERM clear_msgenv(ErlNifEnv* env, int argc, const ERL_NIF_TERM ar static ERL_NIF_TERM grow_blob(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { - union { void* vp; struct make_term_info* p; }mti; + mti_t mti; ERL_NIF_TERM term; if (!enif_get_resource(env, argv[0], msgenv_resource_type, &mti.vp) || (argc>2 && !enif_get_uint(env,argv[2], &mti.p->n))) { @@ -1261,7 +1309,7 @@ static ERL_NIF_TERM grow_blob(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[ static ERL_NIF_TERM send_blob(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { - union { void* vp; struct make_term_info* p; }mti; + mti_t mti; ErlNifPid to; ERL_NIF_TERM copy; int res; @@ -1276,7 +1324,7 @@ static ERL_NIF_TERM send_blob(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[ static ERL_NIF_TERM send3_blob(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { - union { void* vp; struct make_term_info* p; }mti; + mti_t mti; ErlNifPid to; ERL_NIF_TERM copy; int res; @@ -1294,7 +1342,7 @@ static ERL_NIF_TERM send3_blob(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv void* threaded_sender(void *arg) { - union { void* vp; struct make_term_info* p; }mti; + mti_t mti; mti.vp = arg; enif_mutex_lock(mti.p->mtx); @@ -1309,7 +1357,7 @@ void* threaded_sender(void *arg) static ERL_NIF_TERM send_blob_thread(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { - union { void* vp; struct make_term_info* p; }mti; + mti_t mti; ERL_NIF_TERM copy; if (!enif_get_resource(env, argv[0], msgenv_resource_type, &mti.vp) || !enif_get_local_pid(env,argv[1], &mti.p->to_pid)) { @@ -1335,7 +1383,7 @@ static ERL_NIF_TERM send_blob_thread(ErlNifEnv* env, int argc, const ERL_NIF_TER static ERL_NIF_TERM join_send_thread(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { - union { void* vp; struct make_term_info* p; }mti; + mti_t mti; int err; if (!enif_get_resource(env, argv[0], msgenv_resource_type, &mti.vp)) { return enif_make_badarg(env); @@ -1352,7 +1400,7 @@ static ERL_NIF_TERM join_send_thread(ErlNifEnv* env, int argc, const ERL_NIF_TER static ERL_NIF_TERM copy_blob(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { - union { void* vp; struct make_term_info* p; }mti; + mti_t mti; if (!enif_get_resource(env, argv[0], msgenv_resource_type, &mti.vp)) { return enif_make_badarg(env); } @@ -1417,7 +1465,9 @@ static ErlNifFunc nif_funcs[] = {"send_blob_thread", 3, send_blob_thread}, {"join_send_thread", 1, join_send_thread}, {"copy_blob", 1, copy_blob}, - {"send_term", 2, send_term} + {"send_term", 2, send_term}, + {"echo_int", 1, echo_int}, + {"type_sizes", 0, type_sizes} }; ERL_NIF_INIT(nif_SUITE,nif_funcs,load,reload,upgrade,unload) diff --git a/erts/emulator/test/process_SUITE.erl b/erts/emulator/test/process_SUITE.erl index 36bae908aa..f68e712268 100644 --- a/erts/emulator/test/process_SUITE.erl +++ b/erts/emulator/test/process_SUITE.erl @@ -50,7 +50,8 @@ processes_last_call_trap/1, processes_gc_trap/1, processes_term_proc_list/1, otp_7738_waiting/1, otp_7738_suspended/1, - otp_7738_resume/1]). + otp_7738_resume/1, + garb_other_running/1]). -export([prio_server/2, prio_client/2]). -export([init_per_testcase/2, end_per_testcase/2]). @@ -72,7 +73,7 @@ all() -> bad_register, garbage_collect, process_info_messages, process_flag_badarg, process_flag_heap_size, spawn_opt_heap_size, otp_6237, {group, processes_bif}, - {group, otp_7738}]. + {group, otp_7738}, garb_other_running]. groups() -> [{t_exit_2, [], @@ -2116,6 +2117,41 @@ otp_7738_test(Type) -> end, ?line ok. +gor(Reds, Stop) -> + receive + {From, reds} -> + From ! {reds, Reds, self()}, + gor(Reds+1, Stop); + {From, Stop} -> + From ! {stopped, Stop, Reds, self()} + after 0 -> + gor(Reds+1, Stop) + end. + +garb_other_running(Config) when is_list(Config) -> + ?line Stop = make_ref(), + ?line {Pid, Mon} = spawn_monitor(fun () -> gor(0, Stop) end), + ?line Reds = lists:foldl(fun (_, OldReds) -> + ?line erlang:garbage_collect(Pid), + ?line receive after 1 -> ok end, + ?line Pid ! {self(), reds}, + ?line receive + {reds, NewReds, Pid} -> + ?line true = (NewReds > OldReds), + ?line NewReds + end + end, + 0, + lists:seq(1, 10000)), + ?line receive after 1 -> ok end, + ?line Pid ! {self(), Stop}, + ?line receive + {stopped, Stop, StopReds, Pid} -> + ?line true = (StopReds > Reds) + end, + ?line receive {'DOWN', Mon, process, Pid, normal} -> ok end, + ?line ok. + %% Internal functions wait_until(Fun) -> diff --git a/erts/emulator/test/send_term_SUITE.erl b/erts/emulator/test/send_term_SUITE.erl index 6615873392..ba0ba804ca 100644 --- a/erts/emulator/test/send_term_SUITE.erl +++ b/erts/emulator/test/send_term_SUITE.erl @@ -175,6 +175,10 @@ chk_temp_alloc() -> %% Verify that we havn't got anything allocated by temp_alloc lists:foreach( fun ({instance, _, TI}) -> + ?line {value, {sbmbcs, SBMBCInfo}} + = lists:keysearch(sbmbcs, 1, TI), + ?line {value, {blocks, 0, _, _}} + = lists:keysearch(blocks, 1, SBMBCInfo), ?line {value, {mbcs, MBCInfo}} = lists:keysearch(mbcs, 1, TI), ?line {value, {blocks, 0, _, _}} diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops index e7c57142c0..354439b5e3 100755 --- a/erts/emulator/utils/beam_makeops +++ b/erts/emulator/utils/beam_makeops @@ -67,6 +67,10 @@ my $max_gen_operands = 8; # Must be even. The beam_load.c file must be updated, too. my $max_spec_operands = 6; +# The maximum number of primitive genop_types. + +my $max_genop_types = 16; + my %gen_opnum; my %num_specific; my %gen_to_spec; @@ -106,7 +110,7 @@ my @pred_table; # Operand types for generic instructions. my $compiler_types = "uiaxyfhz"; -my $loader_types = "nprvlq"; +my $loader_types = "nprvlqo"; my $genop_types = $compiler_types . $loader_types; # @@ -142,34 +146,61 @@ my %arg_size = ('r' => 0, # x(0) - x register zero my %type_bit; my @tag_type; +sub define_type_bit { + my($tag,$val) = @_; + defined $type_bit{$tag} and + sanity("the tag '$tag' has already been defined with the value ", + $type_bit{$tag}); + $type_bit{$tag} = $val; +} + { my($bit) = 1; my(%bit); foreach (split('', $genop_types)) { push(@tag_type, $_); - $type_bit{$_} = $bit; + define_type_bit($_, $bit); $bit{$_} = $bit; $bit *= 2; } # Composed types. - $type_bit{'d'} = $type_bit{'x'} | $type_bit{'y'} | $type_bit{'r'}; - $type_bit{'c'} = $type_bit{'i'} | $type_bit{'a'} | $type_bit{'n'} | $type_bit{'q'}; - $type_bit{'s'} = $type_bit{'d'} | $type_bit{'i'} | $type_bit{'a'} | $type_bit{'n'}; - $type_bit{'j'} = $type_bit{'f'} | $type_bit{'p'}; + define_type_bit('d', $type_bit{'x'} | $type_bit{'y'} | $type_bit{'r'}); + define_type_bit('c', $type_bit{'i'} | $type_bit{'a'} | + $type_bit{'n'} | $type_bit{'q'}); + define_type_bit('s', $type_bit{'d'} | $type_bit{'i'} | + $type_bit{'a'} | $type_bit{'n'}); + define_type_bit('j', $type_bit{'f'} | $type_bit{'p'}); # Aliases (for matching purposes). - $type_bit{'I'} = $type_bit{'u'}; - $type_bit{'t'} = $type_bit{'u'}; - $type_bit{'A'} = $type_bit{'u'}; - $type_bit{'L'} = $type_bit{'u'}; - $type_bit{'b'} = $type_bit{'u'}; - $type_bit{'N'} = $type_bit{'u'}; - $type_bit{'U'} = $type_bit{'u'}; - $type_bit{'e'} = $type_bit{'u'}; - $type_bit{'P'} = $type_bit{'u'}; - $type_bit{'Q'} = $type_bit{'u'}; + define_type_bit('I', $type_bit{'u'}); + define_type_bit('t', $type_bit{'u'}); + define_type_bit('A', $type_bit{'u'}); + define_type_bit('L', $type_bit{'u'}); + define_type_bit('b', $type_bit{'u'}); + define_type_bit('N', $type_bit{'u'}); + define_type_bit('U', $type_bit{'u'}); + define_type_bit('e', $type_bit{'u'}); + define_type_bit('P', $type_bit{'u'}); + define_type_bit('Q', $type_bit{'u'}); +} + +# +# Sanity checks. +# + +{ + if (@tag_type > $max_genop_types) { + sanity("\$max_genop_types is $max_genop_types, ", + "but there are ", scalar(@tag_type), + " primitive tags defined\n"); + } + + foreach my $tag (@tag_type) { + sanity("tag '$tag': primitive tags must be named with lowercase letters") + unless $tag =~ /^[a-z]$/; + } } # @@ -436,12 +467,12 @@ sub emulator_output { # my(@bits) = (0) x ($max_spec_operands/2); - my($shift) = 16; my($i); for ($i = 0; $i < $max_spec_operands && defined $args[$i]; $i++) { my $t = $args[$i]; if (defined $type_bit{$t}) { - $bits[int($i/2)] |= $type_bit{$t} << (16*($i%2)); + my $shift = $max_genop_types * ($i % 2); + $bits[int($i/2)] |= $type_bit{$t} << $shift; } } @@ -753,6 +784,10 @@ sub error { die $where, @message, "\n"; } +sub sanity { + die "internal error: ", @_, "\n"; +} + sub comment { my($lang, @comments) = @_; my($prefix); |