diff options
Diffstat (limited to 'erts')
35 files changed, 773 insertions, 366 deletions
diff --git a/erts/aclocal.m4 b/erts/aclocal.m4 index a1211bbf0c..f7e5c31910 100644 --- a/erts/aclocal.m4 +++ b/erts/aclocal.m4 @@ -1153,7 +1153,7 @@ case "$THR_LIB_NAME" in AO_nop_full(); AO_store(&x, (AO_t) 0); z = AO_load(&x); - z = AO_compare_and_swap(&x, (AO_t) 0, (AO_t) 1); + z = AO_compare_and_swap_full(&x, (AO_t) 0, (AO_t) 1); ], [ethr_have_native_atomics=yes ethr_have_libatomic_ops=yes]) diff --git a/erts/doc/src/notes.xml b/erts/doc/src/notes.xml index f5607945a8..7383ea381d 100644 --- a/erts/doc/src/notes.xml +++ b/erts/doc/src/notes.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2004</year><year>2010</year> + <year>2004</year><year>2011</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index 327620772f..de76cb9680 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -76,6 +76,7 @@ atom allocator_sizes atom alloc_util_allocators atom allow_passive_connect atom already_loaded +atom amd64 atom anchored atom and atom andalso diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 1dbf6f9b92..d76a7d8e9f 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2010. All Rights Reserved. + * Copyright Ericsson AB 1999-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -142,7 +142,7 @@ BIF_RETTYPE code_is_module_native_1(BIF_ALIST_1) if ((modp = erts_get_module(BIF_ARG_1)) == NULL) { return am_undefined; } - return (is_native(modp->code) || + return ((modp->code && is_native(modp->code)) || (modp->old_code != 0 && is_native(modp->old_code))) ? am_true : am_false; } diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 32ea8588d2..fb90a7d4f7 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -1911,13 +1911,15 @@ void process_main(void) * Note that for the halfword emulator, the two first elements * of the array are used. */ - *((BeamInstr **) (UWord) c_p->def_arg_reg) = I+3; + BeamInstr** pi = (BeamInstr**) c_p->def_arg_reg; + *pi = I+3; set_timer(c_p, unsigned_val(timeout_value)); } else if (timeout_value == am_infinity) { c_p->flags |= F_TIMO; #if !defined(ARCH_64) || HALFWORD_HEAP } else if (term_to_Uint(timeout_value, &time_val)) { - *((BeamInstr **) (UWord) c_p->def_arg_reg) = I+3; + BeamInstr** pi = (BeamInstr**) c_p->def_arg_reg; + *pi = I+3; set_timer(c_p, time_val); #endif } else { /* Wrong time */ @@ -1974,7 +1976,8 @@ void process_main(void) * we must test the F_INSLPQUEUE flag as well as the F_TIMO flag. */ if ((c_p->flags & (F_INSLPQUEUE | F_TIMO)) == 0) { - *((BeamInstr **) (UWord) c_p->def_arg_reg) = I+3; + BeamInstr** p = (BeamInstr **) c_p->def_arg_reg; + *p = I+3; set_timer(c_p, Arg(1)); } goto wait2; @@ -5322,7 +5325,7 @@ void process_main(void) ep->code[3] = (BeamInstr) OpCode(apply_bif); ep->code[4] = (BeamInstr) bif_table[i].f; /* XXX: set func info for bifs */ - ((BeamInstr*)ep->code + 3)[-5] = (BeamInstr) BeamOp(op_i_func_info_IaaI); + ep->fake_op_func_info_for_hipe[0] = (BeamInstr) BeamOp(op_i_func_info_IaaI); } return; diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 8c35644125..68b3350d7f 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -2168,20 +2168,146 @@ BIF_RETTYPE tl_1(BIF_ALIST_1) /**********************************************************************/ /* return the size of an I/O list */ -BIF_RETTYPE iolist_size_1(BIF_ALIST_1) +static Eterm +accumulate(Eterm acc, Uint size) { - Sint size = io_list_len(BIF_ARG_1); + if (is_non_value(acc)) { + /* + * There is no pre-existing accumulator. Allocate a + * bignum buffer with one extra word to be used if + * the bignum grows in the future. + */ + Eterm* hp = (Eterm *) erts_alloc(ERTS_ALC_T_TEMP_TERM, + (BIG_UINT_HEAP_SIZE+1) * + sizeof(Eterm)); + return uint_to_big(size, hp); + } else { + Eterm* big; + int need_heap; - if (size == -1) { - BIF_ERROR(BIF_P, BADARG); - } else if (IS_USMALL(0, (Uint) size)) { - BIF_RET(make_small(size)); + /* + * Add 'size' to 'acc' in place. There is always one + * extra word allocated in case the bignum grows by one word. + */ + big = big_val(acc); + need_heap = BIG_NEED_SIZE(BIG_SIZE(big)); + acc = big_plus_small(acc, size, big); + if (BIG_NEED_SIZE(big_size(acc)) > need_heap) { + /* + * The extra word has been consumed. Grow the + * allocation by one word. + */ + big = (Eterm *) erts_realloc(ERTS_ALC_T_TEMP_TERM, + big_val(acc), + (need_heap+1) * sizeof(Eterm)); + acc = make_big(big); + } + return acc; + } +} + +static Eterm +consolidate(Process* p, Eterm acc, Uint size) +{ + Eterm* hp; + + if (is_non_value(acc)) { + return erts_make_integer(size, p); } else { - Eterm* hp = HAlloc(BIF_P, BIG_UINT_HEAP_SIZE); - BIF_RET(uint_to_big(size, hp)); + Eterm* big; + Uint sz; + Eterm res; + + acc = accumulate(acc, size); + big = big_val(acc); + sz = BIG_NEED_SIZE(BIG_SIZE(big)); + hp = HAlloc(p, sz); + res = make_big(hp); + while (sz--) { + *hp++ = *big++; + } + erts_free(ERTS_ALC_T_TEMP_TERM, (void *) big_val(acc)); + return res; } } +BIF_RETTYPE iolist_size_1(BIF_ALIST_1) +{ + Eterm obj, hd; + Eterm* objp; + Uint size = 0; + Uint cur_size; + Uint new_size; + Eterm acc = THE_NON_VALUE; + DECLARE_ESTACK(s); + + obj = BIF_ARG_1; + goto L_again; + + while (!ESTACK_ISEMPTY(s)) { + obj = ESTACK_POP(s); + L_again: + if (is_list(obj)) { + L_iter_list: + objp = list_val(obj); + hd = CAR(objp); + obj = CDR(objp); + /* Head */ + if (is_byte(hd)) { + size++; + if (size == 0) { + acc = accumulate(acc, (Uint) -1); + size = 1; + } + } else if (is_binary(hd) && binary_bitsize(hd) == 0) { + cur_size = binary_size(hd); + if ((new_size = size + cur_size) >= size) { + size = new_size; + } else { + acc = accumulate(acc, size); + size = cur_size; + } + } else if (is_list(hd)) { + ESTACK_PUSH(s, obj); + obj = hd; + goto L_iter_list; + } else if (is_not_nil(hd)) { + goto L_type_error; + } + /* Tail */ + if (is_list(obj)) { + goto L_iter_list; + } else if (is_binary(obj) && binary_bitsize(obj) == 0) { + cur_size = binary_size(obj); + if ((new_size = size + cur_size) >= size) { + size = new_size; + } else { + acc = accumulate(acc, size); + size = cur_size; + } + } else if (is_not_nil(obj)) { + goto L_type_error; + } + } else if (is_binary(obj) && binary_bitsize(obj) == 0) { + cur_size = binary_size(obj); + if ((new_size = size + cur_size) >= size) { + size = new_size; + } else { + acc = accumulate(acc, size); + size = cur_size; + } + } else if (is_not_nil(obj)) { + goto L_type_error; + } + } + + DESTROY_ESTACK(s); + BIF_RET(consolidate(BIF_P, acc, size)); + + L_type_error: + DESTROY_ESTACK(s); + BIF_ERROR(BIF_P, BADARG); +} /**********************************************************************/ @@ -3282,6 +3408,7 @@ BIF_RETTYPE ports_0(BIF_ALIST_0) Eterm* dead_ports; int alive, dead; Uint32 next_ss; + int i; /* To get a consistent snapshot... * We add alive ports from start of the buffer @@ -3293,21 +3420,18 @@ BIF_RETTYPE ports_0(BIF_ALIST_0) erts_smp_atomic_set(&erts_dead_ports_ptr, (erts_aint_t) (port_buf + erts_max_ports)); - next_ss = erts_smp_atomic_inctest(&erts_ports_snapshot); + next_ss = erts_smp_atomic32_inctest(&erts_ports_snapshot); - if (erts_smp_atomic_read(&erts_ports_alive) > 0) { - erts_aint_t i; - for (i = erts_max_ports-1; i >= 0; i--) { - Port* prt = &erts_port[i]; - erts_smp_port_state_lock(prt); - if (!(prt->status & ERTS_PORT_SFLGS_DEAD) - && prt->snapshot != next_ss) { - ASSERT(prt->snapshot == next_ss - 1); - *pp++ = prt->id; - prt->snapshot = next_ss; /* Consumed by this snapshot */ - } - erts_smp_port_state_unlock(prt); + for (i = erts_max_ports-1; i >= 0; i--) { + Port* prt = &erts_port[i]; + erts_smp_port_state_lock(prt); + if (!(prt->status & ERTS_PORT_SFLGS_DEAD) + && prt->snapshot != next_ss) { + ASSERT(prt->snapshot == next_ss - 1); + *pp++ = prt->id; + prt->snapshot = next_ss; /* Consumed by this snapshot */ } + erts_smp_port_state_unlock(prt); } dead_ports = (Eterm*)erts_smp_atomic_xchg(&erts_dead_ports_ptr, diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c index 9486602633..1fb39c6c67 100644 --- a/erts/emulator/beam/binary.c +++ b/erts/emulator/beam/binary.c @@ -32,11 +32,11 @@ #include "erl_bits.h" #ifdef DEBUG -static int list_to_bitstr_buf(Eterm obj, char* buf, int len); +static int list_to_bitstr_buf(Eterm obj, char* buf, Uint len); #else static int list_to_bitstr_buf(Eterm obj, char* buf); #endif -static Sint bitstr_list_len(Eterm obj); +static int bitstr_list_len(Eterm obj, Uint* num_bytes); void erts_init_binary(void) @@ -355,21 +355,24 @@ BIF_RETTYPE bitstring_to_list_1(BIF_ALIST_1) BIF_RETTYPE erts_list_to_binary_bif(Process *p, Eterm arg) { Eterm bin; - int i; + Uint size; int offset; byte* bytes; + if (is_nil(arg)) { BIF_RET(new_binary(p,(byte*)"",0)); } if (is_not_list(arg)) { goto error; } - if ((i = io_list_len(arg)) < 0) { - goto error; + switch (erts_iolist_size(arg, &size)) { + case ERTS_IOLIST_OVERFLOW: BIF_ERROR(p, SYSTEM_LIMIT); + case ERTS_IOLIST_TYPE: goto error; + default: ; } - bin = new_binary(p, (byte *)NULL, i); + bin = new_binary(p, (byte *)NULL, size); bytes = binary_bytes(bin); - offset = io_list_to_buf(arg, (char*) bytes, i); + offset = io_list_to_buf(arg, (char*) bytes, size); ASSERT(offset == 0); BIF_RET(bin); @@ -396,7 +399,8 @@ BIF_RETTYPE iolist_to_binary_1(BIF_ALIST_1) BIF_RETTYPE list_to_bitstring_1(BIF_ALIST_1) { Eterm bin; - int i,offset; + Uint sz; + int offset; byte* bytes; ErlSubBin* sb1; Eterm* hp; @@ -405,15 +409,19 @@ BIF_RETTYPE list_to_bitstring_1(BIF_ALIST_1) BIF_RET(new_binary(BIF_P,(byte*)"",0)); } if (is_not_list(BIF_ARG_1)) { - goto error; + error: + BIF_ERROR(BIF_P, BADARG); } - if ((i = bitstr_list_len(BIF_ARG_1)) < 0) { + switch (bitstr_list_len(BIF_ARG_1, &sz)) { + case ERTS_IOLIST_TYPE: goto error; + case ERTS_IOLIST_OVERFLOW: + BIF_ERROR(BIF_P, SYSTEM_LIMIT); } - bin = new_binary(BIF_P, (byte *)NULL, i); + bin = new_binary(BIF_P, (byte *)NULL, sz); bytes = binary_bytes(bin); #ifdef DEBUG - offset = list_to_bitstr_buf(BIF_ARG_1, (char*) bytes, i); + offset = list_to_bitstr_buf(BIF_ARG_1, (char*) bytes, sz); #else offset = list_to_bitstr_buf(BIF_ARG_1, (char*) bytes); #endif @@ -422,20 +430,16 @@ BIF_RETTYPE list_to_bitstring_1(BIF_ALIST_1) hp = HAlloc(BIF_P, ERL_SUB_BIN_SIZE); sb1 = (ErlSubBin *) hp; sb1->thing_word = HEADER_SUB_BIN; - sb1->size = i-1; + sb1->size = sz-1; sb1->offs = 0; sb1->orig = bin; sb1->bitoffs = 0; sb1->bitsize = offset; sb1->is_writable = 0; - hp += ERL_SUB_BIN_SIZE; bin = make_binary(sb1); } BIF_RET(bin); - - error: - BIF_ERROR(BIF_P, BADARG); } BIF_RETTYPE split_binary_2(BIF_ALIST_2) @@ -499,7 +503,7 @@ BIF_RETTYPE split_binary_2(BIF_ALIST_2) */ static int #ifdef DEBUG -list_to_bitstr_buf(Eterm obj, char* buf, int len) +list_to_bitstr_buf(Eterm obj, char* buf, Uint len) #else list_to_bitstr_buf(Eterm obj, char* buf) #endif @@ -602,8 +606,8 @@ list_to_bitstr_buf(Eterm obj, char* buf) return offset; } -static Sint -bitstr_list_len(Eterm obj) +static int +bitstr_list_len(Eterm obj, Uint* num_bytes) { Eterm* objp; Uint len = 0; @@ -611,6 +615,26 @@ bitstr_list_len(Eterm obj) DECLARE_ESTACK(s); goto L_again; +#define SAFE_ADD(Var, Val) \ + do { \ + Uint valvar = (Val); \ + Var += valvar; \ + if (Var < valvar) { \ + goto L_overflow_error; \ + } \ + } while (0) + +#define SAFE_ADD_BITSIZE(Var, Bin) \ + do { \ + if (*binary_val(Bin) == HEADER_SUB_BIN) { \ + Uint valvar = ((ErlSubBin *) binary_val(Bin))->bitsize; \ + Var += valvar; \ + if (Var < valvar) { \ + goto L_overflow_error; \ + } \ + } \ + } while (0) + while (!ESTACK_ISEMPTY(s)) { obj = ESTACK_POP(s); L_again: @@ -621,9 +645,12 @@ bitstr_list_len(Eterm obj) obj = CAR(objp); if (is_byte(obj)) { len++; + if (len == 0) { + goto L_overflow_error; + } } else if (is_binary(obj)) { - len += binary_size(obj); - offs += binary_bitsize(obj); + SAFE_ADD(len, binary_size(obj)); + SAFE_ADD_BITSIZE(offs, obj); } else if (is_list(obj)) { ESTACK_PUSH(s, CDR(objp)); goto L_iter_list; /* on head */ @@ -635,24 +662,44 @@ bitstr_list_len(Eterm obj) if (is_list(obj)) goto L_iter_list; /* on tail */ else if (is_binary(obj)) { - len += binary_size(obj); - offs += binary_bitsize(obj); + SAFE_ADD(len, binary_size(obj)); + SAFE_ADD_BITSIZE(offs, obj); } else if (is_not_nil(obj)) { goto L_type_error; } } else if (is_binary(obj)) { - len += binary_size(obj); - offs += binary_bitsize(obj); + SAFE_ADD(len, binary_size(obj)); + SAFE_ADD_BITSIZE(offs, obj); } else if (is_not_nil(obj)) { goto L_type_error; } } +#undef SAFE_ADD +#undef SAFE_ADD_BITSIZE DESTROY_ESTACK(s); - return (Sint) (len + (offs/8) + ((offs % 8) != 0)); + + /* + * Make sure that the number of bits in the bitstring will fit + * in an Uint to ensure that the binary can be matched using + * the binary syntax. + */ + if (len << 3 < len) { + goto L_overflow_error; + } + len += (offs >> 3) + ((offs & 7) != 0); + if (len << 3 < len) { + goto L_overflow_error; + } + *num_bytes = len; + return ERTS_IOLIST_OK; L_type_error: DESTROY_ESTACK(s); - return (Sint) -1; + return ERTS_IOLIST_TYPE; + + L_overflow_error: + DESTROY_ESTACK(s); + return ERTS_IOLIST_OVERFLOW; } diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c index c9cdcb87a6..9631fb50db 100644 --- a/erts/emulator/beam/erl_bif_ddll.c +++ b/erts/emulator/beam/erl_bif_ddll.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2006-2010. All Rights Reserved. + * Copyright Ericsson AB 2006-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -146,7 +146,7 @@ BIF_RETTYPE erl_ddll_try_load_3(Process *p, Eterm path_term, Eterm name_term, Eterm options) { char *path = NULL; - int path_len; + Uint path_len; char *name = NULL; DE_Handle *dh; erts_driver_t *drv; @@ -221,9 +221,7 @@ BIF_RETTYPE erl_ddll_try_load_3(Process *p, Eterm path_term, goto error; } - path_len = io_list_len(path_term); - - if (path_len <= 0) { + if (erts_iolist_size(path_term, &path_len)) { goto error; } path = erts_alloc(ERTS_ALC_T_DDLL_TMP_BUF, path_len + 1 /* might need path separator */ + sys_strlen(name) + 1); @@ -1878,7 +1876,7 @@ static Eterm mkatom(char *str) static char *pick_list_or_atom(Eterm name_term) { char *name = NULL; - int name_len; + Uint name_len; if (is_atom(name_term)) { Atom *ap = atom_tab(atom_val(name_term)); if (ap->len == 0) { @@ -1890,8 +1888,7 @@ static char *pick_list_or_atom(Eterm name_term) memcpy(name,ap->name,ap->len); name[ap->len] = '\0'; } else { - name_len = io_list_len(name_term); - if (name_len <= 0) { + if (erts_iolist_size(name_term, &name_len)) { goto error; } name = erts_alloc(ERTS_ALC_T_DDLL_TMP_BUF, name_len + 1); diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index e50fc18e64..f264bf44df 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -1732,14 +1732,14 @@ info_1_tuple(Process* BIF_P, /* Pointer to current process. */ # define ERTS_ERROR_CHECKER_PRINTF_XML VALGRIND_PRINTF_XML # endif #endif - int buf_size = 8*1024; /* Try with 8KB first */ + Uint buf_size = 8*1024; /* Try with 8KB first */ char *buf = erts_alloc(ERTS_ALC_T_TMP, buf_size); int r = io_list_to_buf(*tp, (char*) buf, buf_size - 1); if (r < 0) { erts_free(ERTS_ALC_T_TMP, (void *) buf); - buf_size = io_list_len(*tp); - if (buf_size < 0) + if (erts_iolist_size(*tp, &buf_size)) { goto badarg; + } buf_size++; buf = erts_alloc(ERTS_ALC_T_TMP, buf_size); r = io_list_to_buf(*tp, (char*) buf, buf_size - 1); diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c index fbc92b9730..3fd35dd963 100644 --- a/erts/emulator/beam/erl_bif_port.c +++ b/erts/emulator/beam/erl_bif_port.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2010. All Rights Reserved. + * Copyright Ericsson AB 2001-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -996,6 +996,7 @@ static byte* convert_environment(Process* p, Eterm env) Eterm* hp; Uint heap_size; int n; + Uint size; byte* bytes; if ((n = list_length(env)) < 0) { @@ -1039,15 +1040,15 @@ static byte* convert_environment(Process* p, Eterm env) if (is_not_nil(env)) { goto done; } - if ((n = io_list_len(all)) < 0) { + if (erts_iolist_size(all, &size)) { goto done; } /* * Put the result in a binary (no risk for a memory leak that way). */ - (void) erts_new_heap_binary(p, NULL, n, &bytes); - io_list_to_buf(all, (char*)bytes, n); + (void) erts_new_heap_binary(p, NULL, size, &bytes); + io_list_to_buf(all, (char*)bytes, size); done: erts_free(ERTS_ALC_T_TMP, temp_heap); diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c index d4a8a3aaa7..26891c4348 100644 --- a/erts/emulator/beam/erl_bif_re.c +++ b/erts/emulator/beam/erl_bif_re.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2008-2010. All Rights Reserved. + * Copyright Ericsson AB 2008-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -417,7 +417,7 @@ build_compile_result(Process *p, Eterm error_tag, pcre *result, int errcode, con BIF_RETTYPE re_compile_2(BIF_ALIST_2) { - int slen; + Uint slen; char *expr; pcre *result; int errcode = 0; @@ -444,7 +444,7 @@ re_compile_2(BIF_ALIST_2) BIF_TRAP2(ucompile_trap_exportp, BIF_P, BIF_ARG_1, BIF_ARG_2); } - if ((slen = io_list_len(BIF_ARG_1)) < 0) { + if (erts_iolist_size(BIF_ARG_1, &slen)) { BIF_ERROR(BIF_P,BADARG); } expr = erts_alloc(ERTS_ALC_T_RE_TMP_BUF, slen + 1); @@ -795,8 +795,8 @@ build_capture(Eterm capture_spec[CAPSPEC_SIZE], const pcre *code) memcpy(tmpb,ap->name,ap->len); tmpb[ap->len] = '\0'; } else { - int slen = io_list_len(val); - if (slen < 0) { + Uint slen; + if (erts_iolist_size(val, &slen)) { goto error; } if ((slen + 1) > tmpbsiz) { @@ -851,7 +851,7 @@ re_run_3(BIF_ALIST_3) const pcre *code_tmp; RestartContext restart; byte *temp_alloc = NULL; - int slength; + Uint slength; int startoffset = 0; int options = 0, comp_options = 0; int ovsize; @@ -875,7 +875,7 @@ re_run_3(BIF_ALIST_3) if (is_not_tuple(BIF_ARG_2) || (arityval(*tuple_val(BIF_ARG_2)) != 4)) { if (is_binary(BIF_ARG_2) || is_list(BIF_ARG_2) || is_nil(BIF_ARG_2)) { /* Compile from textual RE */ - int slen; + Uint slen; char *expr; pcre *result; int errcode = 0; @@ -889,7 +889,7 @@ re_run_3(BIF_ALIST_3) BIF_TRAP3(urun_trap_exportp, BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); } - if ((slen = io_list_len(BIF_ARG_2)) < 0) { + if (erts_iolist_size(BIF_ARG_2, &slen)) { BIF_ERROR(BIF_P,BADARG); } @@ -1027,7 +1027,7 @@ re_run_3(BIF_ALIST_3) restart.flags |= RESTART_FLAG_SUBJECT_IN_BINARY; } else { handle_iolist: - if ((slength = io_list_len(BIF_ARG_1)) < 0) { + if (erts_iolist_size(BIF_ARG_1, &slength)) { erts_free(ERTS_ALC_T_RE_SUBJECT, restart.ovector); erts_free(ERTS_ALC_T_RE_SUBJECT, restart.code); if (restart.ret_info != NULL) { diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 694348e31d..fdc82c8b88 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -105,7 +105,7 @@ #define NSEG_2 256 /* Size of second segment table */ #define NSEG_INC 128 /* Number of segments to grow after that */ -#define SEGTAB(tb) ((struct segment**)erts_smp_atomic_read(&(tb)->segtab)) +#define SEGTAB(tb) ((struct segment**)erts_smp_atomic_read_acqb(&(tb)->segtab)) #define NACTIVE(tb) ((int)erts_smp_atomic_read(&(tb)->nactive)) #define NITEMS(tb) ((int)erts_smp_atomic_read(&(tb)->common.nitems)) @@ -122,8 +122,8 @@ */ static ERTS_INLINE Uint hash_to_ix(DbTableHash* tb, HashValue hval) { - Uint mask = erts_smp_atomic_read(&tb->szm); - Uint ix = hval & mask; + Uint mask = erts_smp_atomic_read_acqb(&tb->szm); + Uint ix = hval & mask; if (ix >= erts_smp_atomic_read(&tb->nactive)) { ix &= mask>>1; ASSERT(ix < erts_smp_atomic_read(&tb->nactive)); @@ -668,6 +668,7 @@ int db_create_hash(Process *p, DbTable *tbl) else { /* coarse locking */ tb->locks = NULL; } + ERTS_THR_MEMORY_BARRIER; #endif /* ERST_SMP */ return DB_ERROR_NONE; } @@ -2349,7 +2350,7 @@ static int alloc_seg(DbTableHash *tb) struct ext_segment* eseg; eseg = (struct ext_segment*) SEGTAB(tb)[seg_ix-1]; MY_ASSERT(eseg!=NULL && eseg->s.is_ext_segment); - erts_smp_atomic_set(&tb->segtab, (erts_aint_t) eseg->segtab); + erts_smp_atomic_set_relb(&tb->segtab, (erts_aint_t) eseg->segtab); tb->nsegs = eseg->nsegs; } ASSERT(seg_ix < tb->nsegs); @@ -2421,7 +2422,7 @@ static int free_seg(DbTableHash *tb, int free_records) MY_ASSERT(newtop->s.is_ext_segment); if (newtop->prev_segtab != NULL) { /* Time to use a smaller segtab */ - erts_smp_atomic_set(&tb->segtab, (erts_aint_t)newtop->prev_segtab); + erts_smp_atomic_set_relb(&tb->segtab, (erts_aint_t)newtop->prev_segtab); tb->nsegs = seg_ix; ASSERT(tb->nsegs == EXTSEG(SEGTAB(tb))->nsegs); } @@ -2438,7 +2439,7 @@ static int free_seg(DbTableHash *tb, int free_records) if (seg_ix > 0) { if (seg_ix < tb->nsegs) SEGTAB(tb)[seg_ix] = NULL; } else { - erts_smp_atomic_set(&tb->segtab, (erts_aint_t)NULL); + erts_smp_atomic_set_relb(&tb->segtab, (erts_aint_t)NULL); } #endif tb->nslots -= SEGSZ; @@ -2533,9 +2534,9 @@ static void grow(DbTableHash* tb, int nactive) } erts_smp_atomic_inc(&tb->nactive); if (from_ix == 0) { - erts_smp_atomic_set(&tb->szm, szm); + erts_smp_atomic_set_relb(&tb->szm, szm); } - erts_smp_atomic_set(&tb->is_resizing, 0); + erts_smp_atomic_set_relb(&tb->is_resizing, 0); /* Finally, let's split the bucket. We try to do it in a smart way to keep link order and avoid unnecessary updates of next-pointers */ @@ -2567,7 +2568,7 @@ static void grow(DbTableHash* tb, int nactive) return; abort: - erts_smp_atomic_set(&tb->is_resizing, 0); + erts_smp_atomic_set_relb(&tb->is_resizing, 0); } @@ -2611,7 +2612,7 @@ static void shrink(DbTableHash* tb, int nactive) erts_smp_atomic_set(&tb->nactive, src_ix); if (dst_ix == 0) { - erts_smp_atomic_set(&tb->szm, low_szm); + erts_smp_atomic_set_relb(&tb->szm, low_szm); } WUNLOCK_HASH(lck); @@ -2625,7 +2626,7 @@ static void shrink(DbTableHash* tb, int nactive) } /*else already done */ - erts_smp_atomic_set(&tb->is_resizing, 0); + erts_smp_atomic_set_relb(&tb->is_resizing, 0); } diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c index eb77d1281a..9a0ba3a418 100644 --- a/erts/emulator/beam/erl_db_tree.c +++ b/erts/emulator/beam/erl_db_tree.c @@ -111,7 +111,7 @@ static void release_stack(DbTableTree* tb, DbTreeStack* stack) { if (stack == &tb->static_stack) { ASSERT(erts_smp_atomic_read(&tb->is_stack_busy) == 1); - erts_smp_atomic_set(&tb->is_stack_busy, 0); + erts_smp_atomic_set_relb(&tb->is_stack_busy, 0); } else { erts_db_free(ERTS_ALC_T_DB_STK, (DbTable *) tb, diff --git a/erts/emulator/beam/erl_drv_thread.c b/erts/emulator/beam/erl_drv_thread.c index 39bbe9633b..dc578f6d2a 100644 --- a/erts/emulator/beam/erl_drv_thread.c +++ b/erts/emulator/beam/erl_drv_thread.c @@ -700,6 +700,13 @@ erl_drv_thread_join(ErlDrvTid tid, void **respp) extern int erts_darwin_main_thread_pipe[2]; extern int erts_darwin_main_thread_result_pipe[2]; +int erl_drv_stolen_main_thread_join(ErlDrvTid tid, void **respp); +int erl_drv_steal_main_thread(char *name, + ErlDrvTid *dtid, + void* (*func)(void*), + void* arg, + ErlDrvThreadOpts *opts); + int erl_drv_stolen_main_thread_join(ErlDrvTid tid, void **respp) diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 8b48444904..68421b4387 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -472,7 +472,7 @@ static void tmp_alloc_dtor(struct enif_tmp_obj_t* obj) int enif_inspect_iolist_as_binary(ErlNifEnv* env, Eterm term, ErlNifBinary* bin) { struct enif_tmp_obj_t* tobj; - int sz; + Uint sz; if (is_binary(term)) { return enif_inspect_binary(env,term,bin); } @@ -483,7 +483,7 @@ int enif_inspect_iolist_as_binary(ErlNifEnv* env, Eterm term, ErlNifBinary* bin) bin->ref_bin = NULL; return 1; } - if ((sz = io_list_len(term)) < 0) { + if (erts_iolist_size(term, &sz)) { return 0; } diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c index 1b07024ca1..326021643f 100644 --- a/erts/emulator/beam/erl_port_task.c +++ b/erts/emulator/beam/erl_port_task.c @@ -658,8 +658,6 @@ erts_port_task_free_port(Port *pp) when scheduled out... */ ErtsPortTask *ptp = port_task_alloc(); erts_smp_port_state_lock(pp); - ASSERT(erts_smp_atomic_read(&erts_ports_alive) > 0); - erts_smp_atomic_dec(&erts_ports_alive); pp->status &= ~ERTS_PORT_SFLG_CLOSING; pp->status |= ERTS_PORT_SFLG_FREE_SCHEDULED; erts_may_save_closed_port(pp); @@ -681,7 +679,6 @@ erts_port_task_free_port(Port *pp) port_is_dequeued = 1; } erts_smp_port_state_lock(pp); - erts_smp_atomic_dec(&erts_ports_alive); pp->status &= ~ERTS_PORT_SFLG_CLOSING; pp->status |= ERTS_PORT_SFLG_FREE_SCHEDULED; erts_may_save_closed_port(pp); diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 8a56976905..2704359a8f 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -166,9 +166,8 @@ static struct { static struct { erts_smp_mtx_t update_mtx; - erts_smp_atomic32_t active_runqs; + erts_smp_atomic32_t no_runqs; int last_active_runqs; - erts_smp_atomic32_t used_runqs; int forced_check_balance; erts_smp_atomic32_t checking_balance; int halftime; @@ -965,7 +964,7 @@ sched_spin_wait(ErtsSchedulerSleepInfo *ssi, int spincount) erts_aint32_t flgs; do { - flgs = erts_smp_atomic32_read(&ssi->flags); + flgs = erts_smp_atomic32_read_acqb(&ssi->flags); if ((flgs & (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING)) != (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING)) { break; @@ -1114,7 +1113,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) else { erts_aint_t dt; - erts_smp_atomic32_set(&function_calls, 0); + erts_smp_atomic32_set_relb(&function_calls, 0); *fcalls = 0; sched_waiting_sys(esdp->no, rq); @@ -1147,7 +1146,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) nonblockable_aux_work(esdp, ssi, aux_work); #endif - flgs = erts_smp_atomic32_read(&ssi->flags); + flgs = erts_smp_atomic32_read_acqb(&ssi->flags); if (!(flgs & ERTS_SSI_FLG_WAITING)) { ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); goto sys_woken; @@ -1333,6 +1332,76 @@ wake_all_schedulers(void) } } +#define ERTS_NO_USED_RUNQS_SHIFT 16 +#define ERTS_NO_RUNQS_MASK 0xffff + +#if ERTS_MAX_NO_OF_SCHEDULERS > ERTS_NO_RUNQS_MASK +# error "Too large amount of schedulers allowed" +#endif + +static ERTS_INLINE void +init_no_runqs(int active, int used) +{ + erts_aint32_t no_runqs = (erts_aint32_t) (active & ERTS_NO_RUNQS_MASK); + no_runqs |= (erts_aint32_t) ((used & ERTS_NO_RUNQS_MASK) << ERTS_NO_USED_RUNQS_SHIFT); + erts_smp_atomic32_init(&balance_info.no_runqs, no_runqs); +} + +static ERTS_INLINE void +get_no_runqs(int *active, int *used) +{ + erts_aint32_t no_runqs = erts_smp_atomic32_read(&balance_info.no_runqs); + if (active) + *active = (int) (no_runqs & ERTS_NO_RUNQS_MASK); + if (used) + *used = (int) ((no_runqs >> ERTS_NO_USED_RUNQS_SHIFT) & ERTS_NO_RUNQS_MASK); +} + +static ERTS_INLINE void +set_no_used_runqs(int used) +{ + erts_aint32_t exp = erts_smp_atomic32_read(&balance_info.no_runqs); + while (1) { + erts_aint32_t act, new; + new = (used << ERTS_NO_USED_RUNQS_SHIFT) | (exp & ERTS_NO_RUNQS_MASK); + act = erts_smp_atomic32_cmpxchg(&balance_info.no_runqs, new, exp); + if (act == exp) + break; + exp = act; + } +} + +static ERTS_INLINE void +set_no_active_runqs(int active) +{ + erts_aint32_t exp = erts_smp_atomic32_read(&balance_info.no_runqs); + while (1) { + erts_aint32_t act, new; + new = (exp & (ERTS_NO_RUNQS_MASK << ERTS_NO_USED_RUNQS_SHIFT)) | active; + act = erts_smp_atomic32_cmpxchg(&balance_info.no_runqs, new, exp); + if (act == exp) + break; + exp = act; + } +} + +static ERTS_INLINE int +try_inc_no_active_runqs(int active) +{ + erts_aint32_t exp = erts_smp_atomic32_read(&balance_info.no_runqs); + if (((exp >> ERTS_NO_USED_RUNQS_SHIFT) & ERTS_NO_RUNQS_MASK) < active) + return 0; + if ((exp & ERTS_NO_RUNQS_MASK) + 1 == active) { + erts_aint32_t new, act; + new = (exp & ~ERTS_NO_RUNQS_MASK) | active; + act = erts_smp_atomic32_cmpxchg(&balance_info.no_runqs, new, exp); + if (act == exp) + return 1; + } + return 0; +} + + static ERTS_INLINE int chk_wake_sched(ErtsRunQueue *crq, int ix, int activate) { @@ -1344,9 +1413,7 @@ chk_wake_sched(ErtsRunQueue *crq, int ix, int activate) iflgs = erts_smp_atomic32_read(&wrq->info_flags); if (!(iflgs & (ERTS_RUNQ_IFLG_SUSPENDED|ERTS_RUNQ_IFLG_NONEMPTY))) { if (activate) { - if (ix == erts_smp_atomic32_cmpxchg(&balance_info.active_runqs, - ix+1, - ix)) { + if (try_inc_no_active_runqs(ix+1)) { erts_smp_xrunq_lock(crq, wrq); wrq->flags &= ~ERTS_RUNQ_FLG_INACTIVE; erts_smp_xrunq_unlock(crq, wrq); @@ -1363,8 +1430,9 @@ wake_scheduler_on_empty_runq(ErtsRunQueue *crq) { int ix = crq->ix; int stop_ix = ix; - int active_ix = erts_smp_atomic32_read(&balance_info.active_runqs); - int balance_ix = erts_smp_atomic32_read(&balance_info.used_runqs); + int active_ix, balance_ix; + + get_no_runqs(&active_ix, &balance_ix); if (active_ix > balance_ix) active_ix = balance_ix; @@ -1416,7 +1484,7 @@ erts_sched_notify_check_cpu_bind(void) int ix; if (erts_common_run_queue) { for (ix = 0; ix < erts_no_schedulers; ix++) - erts_smp_atomic32_set(&ERTS_SCHEDULER_IX(ix)->chk_cpu_bind, 1); + erts_smp_atomic32_set_relb(&ERTS_SCHEDULER_IX(ix)->chk_cpu_bind, 1); wake_all_schedulers(); } else { @@ -1871,8 +1939,7 @@ try_steal_task(ErtsRunQueue *rq) ERTS_SMP_LC_CHK_RUNQ_LOCK(rq, rq_locked); - active_rqs = erts_smp_atomic32_read(&balance_info.active_runqs); - blnc_rqs = erts_smp_atomic32_read(&balance_info.used_runqs); + get_no_runqs(&active_rqs, &blnc_rqs); if (active_rqs > blnc_rqs) active_rqs = blnc_rqs; @@ -1883,7 +1950,7 @@ try_steal_task(ErtsRunQueue *rq) if (active_rqs < blnc_rqs) { int no = blnc_rqs - active_rqs; int stop_ix = vix = active_rqs + rq->ix % no; - while (erts_smp_atomic32_read(&no_empty_run_queues) < blnc_rqs) { + while (erts_smp_atomic32_read_acqb(&no_empty_run_queues) < blnc_rqs) { res = check_possible_steal_victim(rq, &rq_locked, vix); if (res) goto done; @@ -1898,7 +1965,7 @@ try_steal_task(ErtsRunQueue *rq) vix = rq->ix; /* ... then try to steal a job from another active queue... */ - while (erts_smp_atomic32_read(&no_empty_run_queues) < blnc_rqs) { + while (erts_smp_atomic32_read_acqb(&no_empty_run_queues) < blnc_rqs) { vix++; if (vix >= active_rqs) vix = 0; @@ -1999,7 +2066,7 @@ check_balance(ErtsRunQueue *c_rq) return; } - blnc_no_rqs = (int) erts_smp_atomic32_read(&balance_info.used_runqs); + get_no_runqs(NULL, &blnc_no_rqs); if (blnc_no_rqs == 1) { c_rq->check_balance_reds = INT_MAX; erts_smp_atomic32_set(&balance_info.checking_balance, 0); @@ -2038,7 +2105,8 @@ check_balance(ErtsRunQueue *c_rq) forced = balance_info.forced_check_balance; balance_info.forced_check_balance = 0; - blnc_no_rqs = (int) erts_smp_atomic32_read(&balance_info.used_runqs); + get_no_runqs(¤t_active, &blnc_no_rqs); + if (blnc_no_rqs == 1) { erts_smp_mtx_unlock(&balance_info.update_mtx); erts_smp_runq_lock(c_rq); @@ -2052,8 +2120,6 @@ check_balance(ErtsRunQueue *c_rq) if (balance_info.full_reds_history_index >= ERTS_FULL_REDS_HISTORY_SIZE) balance_info.full_reds_history_index = 0; - current_active = erts_smp_atomic32_read(&balance_info.active_runqs); - /* Read balance information for all run queues */ for (qix = 0; qix < blnc_no_rqs; qix++) { ErtsRunQueue *rq = ERTS_RUNQ_IX(qix); @@ -2387,7 +2453,7 @@ erts_fprintf(stderr, "--------------------------------\n"); } balance_info.last_active_runqs = active; - erts_smp_atomic32_set(&balance_info.active_runqs, active); + set_no_active_runqs(active); balance_info.halftime = 1; erts_smp_atomic32_set(&balance_info.checking_balance, 0); @@ -2695,9 +2761,8 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) erts_smp_atomic32_init(&schdlr_sspnd.msb.ongoing, 0); erts_smp_atomic32_init(&schdlr_sspnd.active, no_schedulers); schdlr_sspnd.msb.procs = NULL; - erts_smp_atomic32_set(&balance_info.used_runqs, - erts_common_run_queue ? 1 : no_schedulers_online); - erts_smp_atomic32_init(&balance_info.active_runqs, no_schedulers); + init_no_runqs(no_schedulers, + erts_common_run_queue ? 1 : no_schedulers_online); balance_info.last_active_runqs = no_schedulers; erts_smp_mtx_init(&balance_info.update_mtx, "migration_info_update"); balance_info.forced_check_balance = 0; @@ -2939,7 +3004,7 @@ sched_spin_suspended(ErtsSchedulerSleepInfo *ssi, int spincount) erts_aint32_t flgs; do { - flgs = erts_smp_atomic32_read(&ssi->flags); + flgs = erts_smp_atomic32_read_acqb(&ssi->flags); if ((flgs & (ERTS_SSI_FLG_SLEEPING | ERTS_SSI_FLG_WAITING | ERTS_SSI_FLG_SUSPENDED)) @@ -3068,7 +3133,7 @@ suspend_scheduler(ErtsSchedulerData *esdp) wake = 0; } - flgs = erts_smp_atomic32_read(&ssi->flags); + flgs = erts_smp_atomic32_read_acqb(&ssi->flags); if (!(flgs & ERTS_SSI_FLG_SUSPENDED)) break; erts_smp_mtx_unlock(&schdlr_sspnd.mtx); @@ -3292,7 +3357,7 @@ erts_set_schedulers_online(Process *p, ErtsRunQueue *to_rq = ERTS_RUNQ_IX(ix % no); evacuate_run_queue(from_rq, to_rq); } - erts_smp_atomic32_set(&balance_info.used_runqs, no); + set_no_used_runqs(no); erts_smp_mtx_unlock(&balance_info.update_mtx); erts_smp_mtx_lock(&schdlr_sspnd.mtx); } @@ -3346,7 +3411,7 @@ erts_set_schedulers_online(Process *p, for (ix = erts_no_run_queues-1; ix >= no; ix--) evacuate_run_queue(ERTS_RUNQ_IX(ix), ERTS_RUNQ_IX(ix % no)); - erts_smp_atomic32_set(&balance_info.used_runqs, no); + set_no_used_runqs(no); erts_smp_mtx_unlock(&balance_info.update_mtx); erts_smp_mtx_lock(&schdlr_sspnd.mtx); for (ix = no; ix < online; ix++) { @@ -3443,7 +3508,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) else { erts_smp_mtx_unlock(&schdlr_sspnd.mtx); erts_smp_mtx_lock(&balance_info.update_mtx); - erts_smp_atomic32_set(&balance_info.used_runqs, 1); + set_no_used_runqs(1); for (ix = 0; ix < online; ix++) { ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); erts_smp_runq_lock(rq); @@ -3580,7 +3645,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) evacuate_run_queue(ERTS_RUNQ_IX(ix), ERTS_RUNQ_IX(ix % online)); - erts_smp_atomic32_set(&balance_info.used_runqs, online); + set_no_used_runqs(online); /* Make sure that we balance soon... */ balance_info.forced_check_balance = 1; erts_smp_runq_lock(ERTS_RUNQ_IX(0)); @@ -5131,7 +5196,7 @@ Process *schedule(Process *p, int calls) esdp = erts_get_scheduler_data(); rq = erts_get_runq_current(esdp); ASSERT(esdp); - fcalls = (int) erts_smp_atomic32_read(&function_calls); + fcalls = (int) erts_smp_atomic32_read_acqb(&function_calls); actual_reds = reds = 0; erts_smp_runq_lock(rq); } else { @@ -5282,14 +5347,14 @@ Process *schedule(Process *p, int calls) | ERTS_RUNQ_FLG_CHK_CPU_BIND | ERTS_RUNQ_FLG_SUSPENDED)) { if ((rq->flags & ERTS_RUNQ_FLG_SUSPENDED) - || (erts_smp_atomic32_read(&esdp->ssi->flags) + || (erts_smp_atomic32_read_acqb(&esdp->ssi->flags) & ERTS_SSI_FLG_SUSPENDED)) { ASSERT(erts_smp_atomic32_read(&esdp->ssi->flags) & ERTS_SSI_FLG_SUSPENDED); suspend_scheduler(esdp); } if ((rq->flags & ERTS_RUNQ_FLG_CHK_CPU_BIND) - || erts_smp_atomic32_read(&esdp->chk_cpu_bind)) { + || erts_smp_atomic32_read_acqb(&esdp->chk_cpu_bind)) { erts_sched_check_cpu_bind(esdp); } } @@ -5340,7 +5405,7 @@ Process *schedule(Process *p, int calls) if (rq->flags & (ERTS_RUNQ_FLG_SHARED_RUNQ | ERTS_RUNQ_FLG_SUSPENDED)) { if ((rq->flags & ERTS_RUNQ_FLG_SUSPENDED) - || (erts_smp_atomic32_read(&esdp->ssi->flags) + || (erts_smp_atomic32_read_acqb(&esdp->ssi->flags) & ERTS_SSI_FLG_SUSPENDED)) { ASSERT(erts_smp_atomic32_read(&esdp->ssi->flags) & ERTS_SSI_FLG_SUSPENDED); @@ -5384,7 +5449,7 @@ Process *schedule(Process *p, int calls) * Schedule system-level activities. */ - erts_smp_atomic32_set(&function_calls, 0); + erts_smp_atomic32_set_relb(&function_calls, 0); fcalls = 0; ASSERT(!erts_port_task_have_outstanding_io_tasks()); @@ -5426,7 +5491,7 @@ Process *schedule(Process *p, int calls) if (erts_common_run_queue->waiting) wake_scheduler(erts_common_run_queue, 0, 1); } - else if (erts_smp_atomic32_read(&no_empty_run_queues) != 0) { + else if (erts_smp_atomic32_read_acqb(&no_empty_run_queues) != 0) { wake_scheduler_on_empty_runq(rq); rq->wakeup_other = 0; } @@ -7637,7 +7702,8 @@ continue_exit_process(Process *p static void timeout_proc(Process* p) { - p->i = *((BeamInstr **) (UWord) p->def_arg_reg); + BeamInstr** pi = (BeamInstr **) p->def_arg_reg; + p->i = *pi; p->flags |= F_TIMO; p->flags &= ~F_INSLPQUEUE; diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index c0397ca6c3..8833137112 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -1538,8 +1538,7 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, Eterm tracee; #endif Eterm transformed_args[MAX_ARG]; - DeclareTmpHeap(sub_bin_heap_et,ERL_SUB_BIN_SIZE,p); - ErlSubBin *sub_bin_heap = (ErlSubBin *) sub_bin_heap_et; + DeclareTypedTmpHeap(ErlSubBin,sub_bin_heap,p); ASSERT(tracer_pid); if (*tracer_pid == am_true) { @@ -1600,21 +1599,20 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, if (is_boxed(arg) && header_is_bin_matchstate(*boxed_val(arg))) { ErlBinMatchState* ms = (ErlBinMatchState *) boxed_val(arg); ErlBinMatchBuffer* mb = &ms->mb; - ErlSubBin* sb = sub_bin_heap; Uint bit_size; ASSERT(sub_bin_heap->thing_word == 0); /* At most one of match context */ bit_size = mb->size - mb->offset; - sb->thing_word = HEADER_SUB_BIN; - sb->size = BYTE_OFFSET(bit_size); - sb->bitsize = BIT_OFFSET(bit_size); - sb->offs = BYTE_OFFSET(mb->offset); - sb->bitoffs = BIT_OFFSET(mb->offset); - sb->is_writable = 0; - sb->orig = mb->orig; - - arg = make_binary(sb); + sub_bin_heap->thing_word = HEADER_SUB_BIN; + sub_bin_heap->size = BYTE_OFFSET(bit_size); + sub_bin_heap->bitsize = BIT_OFFSET(bit_size); + sub_bin_heap->offs = BYTE_OFFSET(mb->offset); + sub_bin_heap->bitoffs = BIT_OFFSET(mb->offset); + sub_bin_heap->is_writable = 0; + sub_bin_heap->orig = mb->orig; + + arg = make_binary(sub_bin_heap); } transformed_args[i] = arg; } diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 96da894d90..499bdd77ba 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -183,7 +183,7 @@ struct port { process to get (line oriented I/O)*/ Uint32 status; /* Status and type flags */ int control_flags; /* Flags for port_control() */ - Uint32 snapshot; /* Next snapshot that port should be part of */ + erts_aint32_t snapshot; /* Next snapshot that port should be part of */ struct reg_proc *reg; ErlDrvPDL port_data_lock; @@ -527,11 +527,10 @@ union erl_off_heap_ptr { /* arrays that get malloced at startup */ extern Port* erts_port; -extern erts_smp_atomic_t erts_ports_alive; extern Uint erts_max_ports; extern Uint erts_port_tab_index_mask; -extern erts_smp_atomic_t erts_ports_snapshot; +extern erts_smp_atomic32_t erts_ports_snapshot; extern erts_smp_atomic_t erts_dead_ports_ptr; ERTS_GLB_INLINE void erts_may_save_closed_port(Port *prt); @@ -541,12 +540,12 @@ ERTS_GLB_INLINE void erts_may_save_closed_port(Port *prt); ERTS_GLB_INLINE void erts_may_save_closed_port(Port *prt) { ERTS_SMP_LC_ASSERT(erts_smp_lc_spinlock_is_locked(&prt->state_lck)); - if (prt->snapshot != erts_smp_atomic_read(&erts_ports_snapshot)) { + if (prt->snapshot != erts_smp_atomic32_read_acqb(&erts_ports_snapshot)) { /* Dead ports are added from the end of the snapshot buffer */ Eterm* tombstone = (Eterm*) erts_smp_atomic_addtest(&erts_dead_ports_ptr, -(erts_aint_t)sizeof(Eterm)); ASSERT(tombstone+1 != NULL); - ASSERT(prt->snapshot == (Uint32) erts_smp_atomic_read(&erts_ports_snapshot) - 1); + ASSERT(prt->snapshot == erts_smp_atomic32_read(&erts_ports_snapshot) - 1); *tombstone = prt->id; } /*else no ongoing snapshot or port was already included or created after snapshot */ @@ -1653,10 +1652,14 @@ struct Sint_buf { }; char* Sint_to_buf(Sint, struct Sint_buf*); +#define ERTS_IOLIST_OK 0 +#define ERTS_IOLIST_OVERFLOW 1 +#define ERTS_IOLIST_TYPE 2 + Eterm buf_to_intlist(Eterm**, char*, int, Eterm); /* most callers pass plain char*'s */ int io_list_to_buf(Eterm, char*, int); int io_list_to_buf2(Eterm, char*, int); -int io_list_len(Eterm); +int erts_iolist_size(Eterm, Uint *); int is_string(Eterm); void erl_at_exit(void (*) (void*), void*); Eterm collect_memory(Process *); @@ -1890,6 +1893,8 @@ erts_alloc_message_heap(Uint size, # if defined(DEBUG) # define DeclareTmpHeap(VariableName,Size,Process) \ Eterm *VariableName = erts_debug_allocate_tmp_heap(Size,Process) +# define DeclareTypedTmpHeap(Type,VariableName,Process) \ + Type *VariableName = (Type *) erts_debug_allocate_tmp_heap(sizeof(Type)/sizeof(Eterm),Process) # define DeclareTmpHeapNoproc(VariableName,Size) \ Eterm *VariableName = erts_debug_allocate_tmp_heap(Size,NULL) # define UseTmpHeap(Size,Proc) \ @@ -1911,6 +1916,8 @@ erts_alloc_message_heap(Uint size, # else # define DeclareTmpHeap(VariableName,Size,Process) \ Eterm *VariableName = (ERTS_PROC_GET_SCHDATA(Process)->tmp_heap)+(ERTS_PROC_GET_SCHDATA(Process)->num_tmp_heap_used) +# define DeclareTypedTmpHeap(Type,VariableName,Process) \ + Type *VariableName = (Type *) (ERTS_PROC_GET_SCHDATA(Process)->tmp_heap)+(ERTS_PROC_GET_SCHDATA(Process)->num_tmp_heap_used) # define DeclareTmpHeapNoproc(VariableName,Size) \ Eterm *VariableName = (erts_get_scheduler_data()->tmp_heap)+(erts_get_scheduler_data()->num_tmp_heap_used) # define UseTmpHeap(Size,Proc) \ @@ -1936,6 +1943,8 @@ erts_alloc_message_heap(Uint size, #else # define DeclareTmpHeap(VariableName,Size,Process) \ Eterm VariableName[Size] +# define DeclareTypedTmpHeap(Type,VariableName,Process) \ + Type VariableName[1] # define DeclareTmpHeapNoproc(VariableName,Size) \ Eterm VariableName[Size] # define UseTmpHeap(Size,Proc) /* Nothing */ diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index f619c6f88b..df5f8b22a3 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -56,7 +56,6 @@ static erts_smp_tsd_key_t driver_list_last_error_key; /* Save last DDLL error o per thread basis (for BC interfaces) */ Port* erts_port; /* The port table */ -erts_smp_atomic_t erts_ports_alive; erts_smp_atomic_t erts_bytes_out; /* No bytes sent out of the system */ erts_smp_atomic_t erts_bytes_in; /* No bytes gotten into the system */ @@ -82,6 +81,9 @@ static void driver_monitor_unlock_pdl(Port *p); #define DRV_MONITOR_UNLOCK_PDL(Port) /* nothing */ #endif +#define ERL_SMALL_IO_BIN_LIMIT (4*ERL_ONHEAP_BIN_LIMIT) +#define SMALL_WRITE_VEC 16 + static ERTS_INLINE ErlIOQueue* drvport2ioq(ErlDrvPort drvport) { @@ -190,7 +192,7 @@ typedef struct line_buf_context { static erts_smp_spinlock_t get_free_port_lck; static Uint last_port_num; static Uint port_num_mask; -erts_smp_atomic_t erts_ports_snapshot; /* Identifies the _next_ snapshot (not the ongoing) */ +erts_smp_atomic32_t erts_ports_snapshot; /* Identifies the _next_ snapshot (not the ongoing) */ static ERTS_INLINE void @@ -421,10 +423,9 @@ setup_port(Port* prt, Eterm pid, erts_driver_t *driver, new_name = (char*) erts_alloc(ERTS_ALC_T_PORT_NAME, sys_strlen(name)+1); sys_strcpy(new_name, name); erts_smp_runq_lock(runq); - erts_smp_atomic_inc(&erts_ports_alive); erts_smp_port_state_lock(prt); prt->status = ERTS_PORT_SFLG_CONNECTED | xstatus; - prt->snapshot = (Uint32) erts_smp_atomic_read(&erts_ports_snapshot); + prt->snapshot = erts_smp_atomic32_read(&erts_ports_snapshot); old_name = prt->name; prt->name = new_name; #ifdef ERTS_SMP @@ -954,13 +955,14 @@ do { \ int _bitoffs; \ int _bitsize; \ ERTS_GET_REAL_BIN(obj, _real, _offset, _bitoffs, _bitsize); \ - ASSERT(_bitsize == 0); \ + if (_bitsize != 0) goto L_type_error; \ if (thing_subtag(*binary_val(_real)) == REFC_BINARY_SUBTAG && \ _bitoffs == 0) { \ b_size += _size; \ + if (b_size < _size) goto L_overflow_error; \ in_clist = 0; \ v_size++; \ - if (_size >= bin_limit) { \ + if (_size >= ERL_SMALL_IO_BIN_LIMIT) { \ p_in_clist = 0; \ p_v_size++; \ } else { \ @@ -972,6 +974,7 @@ do { \ } \ } else { \ c_size += _size; \ + if (c_size < _size) goto L_overflow_error; \ if (!in_clist) { \ in_clist = 1; \ v_size++; \ @@ -986,29 +989,30 @@ do { \ /* -** Size of a io list in bytes -** return -1 if error -** returns: - Total size of io list -** vsize - SysIOVec size needed for a writev -** csize - Number of bytes not in binary (in the common binary) -** pvsize - SysIOVec size needed if packing small binaries -** pcsize - Number of bytes in the common binary if packing -*/ + * Returns 0 if successful and a non-zero value otherwise. + * + * Return values through pointers: + * *vsize - SysIOVec size needed for a writev + * *csize - Number of bytes not in binary (in the common binary) + * *pvsize - SysIOVec size needed if packing small binaries + * *pcsize - Number of bytes in the common binary if packing + * *total_size - Total size of iolist in bytes + */ static int -io_list_vec_len(Eterm obj, int* vsize, int* csize, - int bin_limit, /* small binaries limit */ - int * pvsize, int * pcsize) +io_list_vec_len(Eterm obj, Uint* vsize, Uint* csize, + Uint* pvsize, Uint* pcsize, Uint* total_size) { DECLARE_ESTACK(s); Eterm* objp; - int v_size = 0; - int c_size = 0; - int b_size = 0; - int in_clist = 0; - int p_v_size = 0; - int p_c_size = 0; - int p_in_clist = 0; + Uint v_size = 0; + Uint c_size = 0; + Uint b_size = 0; + Uint in_clist = 0; + Uint p_v_size = 0; + Uint p_c_size = 0; + Uint p_in_clist = 0; + Uint total; goto L_jump_start; /* avoid a push */ @@ -1022,6 +1026,9 @@ io_list_vec_len(Eterm obj, int* vsize, int* csize, if (is_byte(obj)) { c_size++; + if (c_size == 0) { + goto L_overflow_error; + } if (!in_clist) { in_clist = 1; v_size++; @@ -1061,32 +1068,31 @@ io_list_vec_len(Eterm obj, int* vsize, int* csize, } } + total = c_size + b_size; + if (total < c_size) { + goto L_overflow_error; + } + *total_size = total; + DESTROY_ESTACK(s); - if (vsize != NULL) - *vsize = v_size; - if (csize != NULL) - *csize = c_size; - if (pvsize != NULL) - *pvsize = p_v_size; - if (pcsize != NULL) - *pcsize = p_c_size; - return c_size + b_size; + *vsize = v_size; + *csize = c_size; + *pvsize = p_v_size; + *pcsize = p_c_size; + return 0; L_type_error: + L_overflow_error: DESTROY_ESTACK(s); - return -1; + return 1; } -#define ERL_SMALL_IO_BIN_LIMIT (4*ERL_ONHEAP_BIN_LIMIT) -#define SMALL_WRITE_VEC 16 - - /* write data to a port */ int erts_write_to_port(Eterm caller_id, Port *p, Eterm list) { char *buf; erts_driver_t *drv = p->drv_ptr; - int size; + Uint size; int fpe_was_unmasked; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(p)); @@ -1094,10 +1100,10 @@ int erts_write_to_port(Eterm caller_id, Port *p, Eterm list) p->caller = caller_id; if (drv->outputv != NULL) { - int vsize; - int csize; - int pvsize; - int pcsize; + Uint vsize; + Uint csize; + Uint pvsize; + Uint pcsize; int blimit; SysIOVec iv[SMALL_WRITE_VEC]; ErlDrvBinary* bv[SMALL_WRITE_VEC]; @@ -1106,9 +1112,8 @@ int erts_write_to_port(Eterm caller_id, Port *p, Eterm list) ErlDrvBinary* cbin; ErlIOVec ev; - if ((size = io_list_vec_len(list, &vsize, &csize, - ERL_SMALL_IO_BIN_LIMIT, - &pvsize, &pcsize)) < 0) { + if (io_list_vec_len(list, &vsize, &csize, + &pvsize, &pcsize, &size)) { goto bad_value; } /* To pack or not to pack (small binaries) ...? */ @@ -1183,7 +1188,7 @@ int erts_write_to_port(Eterm caller_id, Port *p, Eterm list) else { ASSERT(r == -1); /* Overflow */ erts_free(ERTS_ALC_T_TMP, buf); - if ((size = io_list_len(list)) < 0) { + if (erts_iolist_size(list, &size)) { goto bad_value; } @@ -1274,7 +1279,6 @@ void init_io(void) erts_smp_atomic_init(&erts_bytes_out, 0); erts_smp_atomic_init(&erts_bytes_in, 0); - erts_smp_atomic_init(&erts_ports_alive, 0); for (i = 0; i < erts_max_ports; i++) { erts_port_task_init_sched(&erts_port[i].sched); @@ -1296,7 +1300,7 @@ void init_io(void) erts_port[i].port_data_lock = NULL; } - erts_smp_atomic_init(&erts_ports_snapshot, (erts_aint_t) 0); + erts_smp_atomic32_init(&erts_ports_snapshot, (erts_aint32_t) 0); last_port_num = 0; erts_smp_spinlock_init(&get_free_port_lck, "get_free_port"); @@ -2147,7 +2151,7 @@ erts_port_control(Process* p, Port* prt, Uint command, Eterm iolist) byte* to_port = NULL; /* Buffer to write to port. */ /* Initialization is for shutting up warning about use before set. */ - int to_len = 0; /* Length of buffer. */ + Uint to_len = 0; /* Length of buffer. */ int must_free = 0; /* True if the buffer should be freed. */ char port_result[ERL_ONHEAP_BIN_LIMIT]; /* Default buffer for result from port. */ char* port_resp; /* Pointer to result buffer. */ @@ -2192,7 +2196,7 @@ erts_port_control(Process* p, Port* prt, Uint command, Eterm iolist) } else { ASSERT(r == -1); /* Overflow */ erts_free(ERTS_ALC_T_TMP, (void *) to_port); - if ((to_len = io_list_len(iolist)) < 0) { /* Type error */ + if (erts_iolist_size(iolist, &to_len)) { /* Type error */ return THE_NON_VALUE; } must_free = 1; diff --git a/erts/emulator/beam/safe_hash.c b/erts/emulator/beam/safe_hash.c index 21d6ce9304..3e9243c77d 100644 --- a/erts/emulator/beam/safe_hash.c +++ b/erts/emulator/beam/safe_hash.c @@ -99,7 +99,7 @@ static void rehash(SafeHash* h, int grow_limit) erts_free(h->type, (void *) old_tab); } /*else already done */ - erts_smp_atomic_set(&h->is_rehashing, 0); + erts_smp_atomic_set_relb(&h->is_rehashing, 0); } diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index 6b4f3b3b36..a17de717bc 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -3021,13 +3021,25 @@ int io_list_to_buf(Eterm obj, char* buf, int len) return -1; } -int io_list_len(Eterm obj) +/* + * Return 0 if successful, and non-zero if unsuccessful. + */ +int erts_iolist_size(Eterm obj, Uint* sizep) { Eterm* objp; - Sint len = 0; + Uint size = 0; DECLARE_ESTACK(s); goto L_again; +#define SAFE_ADD(Var, Val) \ + do { \ + Uint valvar = (Val); \ + Var += valvar; \ + if (Var < valvar) { \ + goto L_overflow_error; \ + } \ + } while (0) + while (!ESTACK_ISEMPTY(s)) { obj = ESTACK_POP(s); L_again: @@ -3037,9 +3049,12 @@ int io_list_len(Eterm obj) /* Head */ obj = CAR(objp); if (is_byte(obj)) { - len++; + size++; + if (size == 0) { + goto L_overflow_error; + } } else if (is_binary(obj) && binary_bitsize(obj) == 0) { - len += binary_size(obj); + SAFE_ADD(size, binary_size(obj)); } else if (is_list(obj)) { ESTACK_PUSH(s, CDR(objp)); goto L_iter_list; /* on head */ @@ -3051,23 +3066,29 @@ int io_list_len(Eterm obj) if (is_list(obj)) goto L_iter_list; /* on tail */ else if (is_binary(obj) && binary_bitsize(obj) == 0) { - len += binary_size(obj); + SAFE_ADD(size, binary_size(obj)); } else if (is_not_nil(obj)) { goto L_type_error; } } else if (is_binary(obj) && binary_bitsize(obj) == 0) { /* Tail was binary */ - len += binary_size(obj); + SAFE_ADD(size, binary_size(obj)); } else if (is_not_nil(obj)) { goto L_type_error; } } +#undef SAFE_ADD DESTROY_ESTACK(s); - return len; + *sizep = size; + return ERTS_IOLIST_OK; + + L_overflow_error: + DESTROY_ESTACK(s); + return ERTS_IOLIST_OVERFLOW; L_type_error: DESTROY_ESTACK(s); - return -1; + return ERTS_IOLIST_TYPE; } /* return 0 if item is not a non-empty flat list of bytes */ diff --git a/erts/emulator/hipe/hipe_bif2.c b/erts/emulator/hipe/hipe_bif2.c index 19cd32c68f..2660f74a82 100644 --- a/erts/emulator/hipe/hipe_bif2.c +++ b/erts/emulator/hipe/hipe_bif2.c @@ -1,9 +1,8 @@ /* * %CopyrightBegin% - - * + * * Copyright Ericsson AB 2001-2011. All Rights Reserved. - * + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the @@ -37,14 +36,25 @@ #include "hipe_arch.h" #include "hipe_stack.h" -BIF_RETTYPE hipe_bifs_show_estack_1(BIF_ALIST_1) +static void proc_unlock(Process* c_p, Process* rp) { + ErtsProcLocks locks = ERTS_PROC_LOCKS_ALL; + if (rp == c_p) { + locks &= ~ERTS_PROC_LOCK_MAIN; + } + if (rp && locks) { + erts_smp_proc_unlock(rp, locks); + } +} + +BIF_RETTYPE hipe_bifs_show_estack_1(BIF_ALIST_1) +{ Process *rp = erts_pid2proc(BIF_P, ERTS_PROC_LOCK_MAIN, BIF_ARG_1, ERTS_PROC_LOCKS_ALL); if (!rp) BIF_ERROR(BIF_P, BADARG); hipe_print_estack(rp); - erts_smp_proc_unlock(rp, ERTS_PROC_LOCKS_ALL); + proc_unlock(BIF_P, rp); BIF_RET(am_true); } @@ -55,7 +65,7 @@ BIF_RETTYPE hipe_bifs_show_heap_1(BIF_ALIST_1) if (!rp) BIF_ERROR(BIF_P, BADARG); hipe_print_heap(rp); - erts_smp_proc_unlock(rp, ERTS_PROC_LOCKS_ALL); + proc_unlock(BIF_P, rp); BIF_RET(am_true); } @@ -66,7 +76,7 @@ BIF_RETTYPE hipe_bifs_show_nstack_1(BIF_ALIST_1) if (!rp) BIF_ERROR(BIF_P, BADARG); hipe_print_nstack(rp); - erts_smp_proc_unlock(rp, ERTS_PROC_LOCKS_ALL); + proc_unlock(BIF_P, rp); BIF_RET(am_true); } @@ -82,7 +92,7 @@ BIF_RETTYPE hipe_bifs_show_pcb_1(BIF_ALIST_1) if (!rp) BIF_ERROR(BIF_P, BADARG); hipe_print_pcb(rp); - erts_smp_proc_unlock(rp, ERTS_PROC_LOCKS_ALL); + proc_unlock(BIF_P, rp); BIF_RET(am_true); } diff --git a/erts/emulator/hipe/hipe_mkliterals.c b/erts/emulator/hipe/hipe_mkliterals.c index 650861b54b..bced90785d 100644 --- a/erts/emulator/hipe/hipe_mkliterals.c +++ b/erts/emulator/hipe/hipe_mkliterals.c @@ -587,9 +587,9 @@ static void print_params(FILE *fp, void (*print_param)(FILE*,const struct rts_pa (*print_param)(fp, &rts_params[i]); } -static int do_c(FILE *fp) +static int do_c(FILE *fp, const char* this_exe) { - fprintf(fp, "/* File: hipe_literals.h, generated by hipe_mkliterals */\n"); + fprintf(fp, "/* File: hipe_literals.h, generated by %s */\n", this_exe); fprintf(fp, "#ifndef __HIPE_LITERALS_H__\n"); fprintf(fp, "#define __HIPE_LITERALS_H__\n\n"); print_literals(fp, c_define_literal); @@ -603,9 +603,9 @@ static int do_c(FILE *fp) return 0; } -static int do_e(FILE *fp) +static int do_e(FILE *fp, const char* this_exe) { - fprintf(fp, "%%%% File: hipe_literals.hrl, generated by hipe_mkliterals"); + fprintf(fp, "%%%% File: hipe_literals.hrl, generated by %s", this_exe); fprintf(fp, "\n\n"); print_literals(fp, e_define_literal); fprintf(fp, "\n"); @@ -622,9 +622,9 @@ int main(int argc, const char **argv) compute_crc(); if (argc == 2) { if (strcmp(argv[1], "-c") == 0) - return do_c(stdout); + return do_c(stdout, argv[0]); if (strcmp(argv[1], "-e") == 0) - return do_e(stdout); + return do_e(stdout, argv[0]); } fprintf(stderr, "usage: %s [-c | -e] > output-file\n", argv[0]); return 1; diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c index 218bd79584..71b374527e 100644 --- a/erts/emulator/sys/common/erl_check_io.c +++ b/erts/emulator/sys/common/erl_check_io.c @@ -1137,6 +1137,11 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait) restart: +#ifdef ERTS_BREAK_REQUESTED + if (ERTS_BREAK_REQUESTED) + erts_do_break_handling(); +#endif + /* Figure out timeout value */ if (do_wait) { erts_time_remaining(&wait_time); diff --git a/erts/emulator/sys/common/erl_poll.c b/erts/emulator/sys/common/erl_poll.c index 3ae5b8d747..f5c785d683 100644 --- a/erts/emulator/sys/common/erl_poll.c +++ b/erts/emulator/sys/common/erl_poll.c @@ -766,7 +766,7 @@ write_batch_buf(ErtsPollSet ps, ErtsPollBatchBuf *bbp) short filter; int fd = (int) ebuf[i].ident; - switch ((int) ebuf[i].udata) { + switch ((int) (long) ebuf[i].udata) { /* * Since we use a lazy update approach EV_DELETE will @@ -805,7 +805,7 @@ write_batch_buf(ErtsPollSet ps, ErtsPollBatchBuf *bbp) if (fd == (int) ebuf[j].ident) { ebuf[j].udata = (void *) ERTS_POLL_KQ_OP_HANDLED; if (!(ebuf[j].flags & EV_ERROR)) { - switch ((int) ebuf[j].udata) { + switch ((int) (long) ebuf[j].udata) { case ERTS_POLL_KQ_OP_ADD2_W: filter = EVFILT_WRITE; goto rm_add_fb; @@ -823,7 +823,8 @@ write_batch_buf(ErtsPollSet ps, ErtsPollBatchBuf *bbp) } } /* The other add succeded... */ - filter = (((int) ebuf[i].udata == ERTS_POLL_KQ_OP_ADD2_W) + filter = ((((int) (long) ebuf[i].udata) + == ERTS_POLL_KQ_OP_ADD2_W) ? EVFILT_READ : EVFILT_WRITE); rm_add_fb: diff --git a/erts/emulator/sys/win32/erl_poll.c b/erts/emulator/sys/win32/erl_poll.c index 7662f190ef..074e2e247f 100644 --- a/erts/emulator/sys/win32/erl_poll.c +++ b/erts/emulator/sys/win32/erl_poll.c @@ -693,6 +693,7 @@ static void *break_waiter(void *param) ResetEvent(harr[0]); erts_mtx_lock(&break_waiter_lock); erts_atomic32_set(&break_waiter_state,BREAK_WAITER_GOT_BREAK); + ERTS_THR_MEMORY_BARRIER; SetEvent(break_happened_event); erts_mtx_unlock(&break_waiter_lock); break; @@ -700,6 +701,7 @@ static void *break_waiter(void *param) ResetEvent(harr[1]); erts_mtx_lock(&break_waiter_lock); erts_atomic32_set(&break_waiter_state,BREAK_WAITER_GOT_HALT); + ERTS_THR_MEMORY_BARRIER; SetEvent(break_happened_event); erts_mtx_unlock(&break_waiter_lock); break; diff --git a/erts/emulator/test/binary_SUITE.erl b/erts/emulator/test/binary_SUITE.erl index 7e409f053e..4e82381fba 100644 --- a/erts/emulator/test/binary_SUITE.erl +++ b/erts/emulator/test/binary_SUITE.erl @@ -23,12 +23,12 @@ %% Tests binaries and the BIFs: %% list_to_binary/1 %% iolist_to_binary/1 -%% bitstr_to_list/1 +%% list_to_bitstring/1 %% binary_to_list/1 %% binary_to_list/3 %% binary_to_term/1 %% binary_to_term/2 -%% bitstr_to_list/1 +%% bitstring_to_list/1 %% term_to_binary/1 %% erlang:external_size/1 %% size(Binary) @@ -275,12 +275,33 @@ bad_list_to_binary(Config) when is_list(Config) -> ?line test_bad_bin(fun(X, Y) -> X*Y end), ?line test_bad_bin([1,fun(X) -> X + 1 end,2|fun() -> 0 end]), ?line test_bad_bin([fun(X) -> X + 1 end]), + + %% Test iolists that do not fit in the address space. + %% Unfortunately, it would be too slow to test in a 64-bit emulator. + case erlang:system_info(wordsize) of + 4 -> huge_iolists(); + _ -> ok + end. + +huge_iolists() -> + FourGigs = 1 bsl 32, + ?line Sizes = [FourGigs+N || N <- lists:seq(0, 64)] ++ + [1 bsl N || N <- lists:seq(33, 37)], + ?line Base = <<0:(1 bsl 20)/unit:8>>, + [begin + L = build_iolist(Sz, Base), + ?line {'EXIT',{system_limit,_}} = (catch list_to_binary([L])), + ?line {'EXIT',{system_limit,_}} = (catch list_to_bitstring([L])), + ?line {'EXIT',{system_limit,_}} = (catch binary:list_to_bin([L])), + ?line {'EXIT',{system_limit,_}} = (catch iolist_to_binary(L)) + end || Sz <- Sizes], ok. test_bad_bin(List) -> {'EXIT',{badarg,_}} = (catch list_to_binary(List)), {'EXIT',{badarg,_}} = (catch iolist_to_binary(List)), - {'EXIT',{badarg,_}} = (catch list_to_bitstring(List)). + {'EXIT',{badarg,_}} = (catch list_to_bitstring(List)), + {'EXIT',{badarg,_}} = (catch iolist_size(List)). bad_binary_to_list(doc) -> "Tries binary_to_list/1,3 with bad arguments."; bad_binary_to_list(Config) when is_list(Config) -> @@ -516,18 +537,65 @@ external_size_1(Term, Size0, Limit) when Size0 < Limit -> external_size_1(_, _, _) -> ok. t_iolist_size(Config) when is_list(Config) -> - %% Build a term whose external size only fits in a big num (on 32-bit CPU). - Bin = iolist_to_binary(lists:seq(0, 254)), - ?line ok = t_iolist_size_1(Bin, 0, 16#7FFFFFFF), - ?line ok = t_iolist_size_1(make_unaligned_sub_binary(Bin), 0, 16#7FFFFFFF). + ?line Seed = now(), + ?line io:format("Seed: ~p", [Seed]), + ?line random:seed(Seed), + ?line Base = <<0:(1 bsl 20)/unit:8>>, + ?line Powers = [1 bsl N || N <- lists:seq(2, 37)], + ?line Sizes0 = [[N - random:uniform(N div 2), + lists:seq(N-2, N+2), + N+N div 2, + N + random:uniform(N div 2)] || + N <- Powers], + %% Test sizes around 1^32 more thoroughly. + FourGigs = 1 bsl 32, + ?line Sizes1 = [FourGigs+N || N <- lists:seq(-8, 40)] ++ Sizes0, + ?line Sizes2 = lists:flatten(Sizes1), + ?line Sizes = lists:usort(Sizes2), + io:format("~p sizes:", [length(Sizes)]), + io:format("~p\n", [Sizes]), + ?line [Sz = iolist_size(build_iolist(Sz, Base)) || Sz <- Sizes], + ok. -t_iolist_size_1(IOList, Size0, Limit) when Size0 < Limit -> - case iolist_size(IOList) of - Size when is_integer(Size), Size0 < Size -> - io:format("~p", [Size]), - t_iolist_size_1([IOList|IOList], Size, Limit) +build_iolist(N, Base) when N < 16 -> + case random:uniform(3) of + 1 -> + <<Bin:N/binary,_/binary>> = Base, + Bin; + _ -> + lists:seq(1, N) + end; +build_iolist(N, Base) when N =< byte_size(Base) -> + case random:uniform(3) of + 1 -> + <<Bin:N/binary,_/binary>> = Base, + Bin; + 2 -> + <<Bin:N/binary,_/binary>> = Base, + [Bin]; + 3 -> + case N rem 2 of + 0 -> + L = build_iolist(N div 2, Base), + [L,L]; + 1 -> + L = build_iolist(N div 2, Base), + [L,L,45] + end end; -t_iolist_size_1(_, _, _) -> ok. +build_iolist(N0, Base) -> + Small = random:uniform(15), + Seq = lists:seq(1, Small), + N = N0 - Small, + case N rem 2 of + 0 -> + L = build_iolist(N div 2, Base), + [L,L|Seq]; + 1 -> + L = build_iolist(N div 2, Base), + [47,L,L|Seq] + end. + bad_binary_to_term_2(doc) -> "OTP-4053."; bad_binary_to_term_2(suite) -> []; @@ -1183,34 +1251,7 @@ deep(Config) when is_list(Config) -> deep_roundtrip(T) -> B = term_to_binary(T), - true = deep_eq(T, binary_to_term(B)). - -%% -%% FIXME: =:= runs out of stack. -%% -deep_eq([H1|T1], [H2|T2]) -> - deep_eq(H1, H2) andalso deep_eq(T1, T2); -deep_eq(T1, T2) when tuple_size(T1) =:= tuple_size(T2) -> - deep_eq_tup(T1, T2, tuple_size(T1)); -deep_eq(T1, T2) when is_function(T1), is_function(T2) -> - {uniq,U1} = erlang:fun_info(T1, uniq), - {index,I1} = erlang:fun_info(T1, index), - {arity,A1} = erlang:fun_info(T1, arity), - {env,E1} = erlang:fun_info(T1, env), - {uniq,U2} = erlang:fun_info(T2, uniq), - {index,I2} = erlang:fun_info(T2, index), - {arity,A2} = erlang:fun_info(T2, arity), - {env,E2} = erlang:fun_info(T2, env), - U1 =:= U2 andalso I1 =:= I2 andalso A1 =:= A2 andalso - deep_eq(E1, E2); -deep_eq(T1, T2) -> - T1 =:= T2. - -deep_eq_tup(_T1, _T2, 0) -> - true; -deep_eq_tup(T1, T2, N) -> - deep_eq(element(N, T1), element(N, T2)) andalso - deep_eq_tup(T1, T2, N-1). + T = binary_to_term(B). obsolete_funs(Config) when is_list(Config) -> erts_debug:set_internal_state(available_internal_state, true), diff --git a/erts/emulator/test/code_SUITE.erl b/erts/emulator/test/code_SUITE.erl index c1a048be75..a062cea117 100644 --- a/erts/emulator/test/code_SUITE.erl +++ b/erts/emulator/test/code_SUITE.erl @@ -498,7 +498,9 @@ do_false_dependency(Init, Code) -> ?line unlink(Pid), exit(Pid, kill), ?line true = erlang:purge_module(cpbugx), ?line true = erlang:delete_module(cpbugx), + ?line code:is_module_native(cpbugx), % test is_module_native on deleted code ?line true = erlang:purge_module(cpbugx), + ?line code:is_module_native(cpbugx), % test is_module_native on purged code ok. false_dependency_loop(Parent, Init, SendInitAck) -> diff --git a/erts/emulator/test/driver_SUITE.erl b/erts/emulator/test/driver_SUITE.erl index 7600a44988..520e3e8c76 100644 --- a/erts/emulator/test/driver_SUITE.erl +++ b/erts/emulator/test/driver_SUITE.erl @@ -38,7 +38,7 @@ timer_change/1, timer_delay/1, queue_echo/1, - fun_to_port/1, + outputv_errors/1, driver_unloaded/1, io_ready_exit/1, use_fallback_pollset/1, @@ -129,7 +129,7 @@ end_per_testcase(Case, Config) -> suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> - [fun_to_port, outputv_echo, queue_echo, {group, timer}, + [outputv_errors, outputv_echo, queue_echo, {group, timer}, driver_unloaded, io_ready_exit, use_fallback_pollset, bad_fd_in_pollset, driver_event, fd_change, steal_control, otp_6602, 'driver_system_info_ver1.0', @@ -165,37 +165,89 @@ init_per_group(_GroupName, Config) -> end_per_group(_GroupName, Config) -> Config. - -fun_to_port(doc) -> "Test sending a fun to port with an outputv-capable driver."; -fun_to_port(Config) when is_list(Config) -> +outputv_errors(doc) -> "Test sending bad types to port with an outputv-capable driver."; +outputv_errors(Config) when is_list(Config) -> ?line Path = ?config(data_dir, Config), ?line erl_ddll:start(), ?line ok = load_driver(Path, outputv_drv), - ?line fun_to_port_1(fun() -> 33 end), - ?line fun_to_port_1([fun() -> 42 end]), - ?line fun_to_port_1([1|fun() -> 42 end]), - L = build_io_list(65536), - ?line fun_to_port_1([L,fun() -> 42 end]), - ?line fun_to_port_1([L|fun() -> 42 end]), + outputv_bad_types(fun(T) -> + ?line outputv_errors_1(T), + ?line outputv_errors_1([1|T]), + ?line L = [1,2,3], + ?line outputv_errors_1([L,T]), + ?line outputv_errors_1([L|T]) + end), + outputv_errors_1(42), + + %% Test iolists that do not fit in the address space. + %% Unfortunately, it would be too slow to test in a 64-bit emulator. + case erlang:system_info(wordsize) of + 4 -> outputv_huge_iolists(); + _ -> ok + end. + +outputv_bad_types(Test) -> + Types = [-1,256,atom,42.0,{a,b,c},make_ref(),fun() -> 42 end, + [1|2],<<1:1>>,<<1:9>>,<<1:15>>], + _ = [Test(Type) || Type <- Types], ok. -fun_to_port_1(Term) -> - Port = open_port({spawn,outputv_drv}, []), +outputv_huge_iolists() -> + FourGigs = 1 bsl 32, + ?line Sizes = [FourGigs+N || N <- lists:seq(0, 64)] ++ + [1 bsl N || N <- lists:seq(33, 37)], + ?line Base = <<0:(1 bsl 20)/unit:8>>, + [begin + ?line L = build_iolist(Sz, Base), + ?line outputv_errors_1(L) + end || Sz <- Sizes], + ok. + +outputv_errors_1(Term) -> + Port = open_port({spawn_driver,outputv_drv}, []), {'EXIT',{badarg,_}} = (catch port_command(Port, Term)), port_close(Port). -build_io_list(0) -> []; -build_io_list(1) -> [7]; -build_io_list(N) -> - L = build_io_list(N div 2), +build_iolist(N, Base) when N < 16 -> + case random:uniform(3) of + 1 -> + <<Bin:N/binary,_/binary>> = Base, + Bin; + _ -> + lists:seq(1, N) + end; +build_iolist(N, Base) when N =< byte_size(Base) -> + case random:uniform(3) of + 1 -> + <<Bin:N/binary,_/binary>> = Base, + Bin; + 2 -> + <<Bin:N/binary,_/binary>> = Base, + [Bin]; + 3 -> + case N rem 2 of + 0 -> + L = build_iolist(N div 2, Base), + [L,L]; + 1 -> + L = build_iolist(N div 2, Base), + [L,L,45] + end + end; +build_iolist(N0, Base) -> + Small = random:uniform(15), + Seq = lists:seq(1, Small), + N = N0 - Small, case N rem 2 of - 0 -> [L|L]; - 1 -> [7,L|L] + 0 -> + L = build_iolist(N div 2, Base), + [L,L|Seq]; + 1 -> + L = build_iolist(N div 2, Base), + [47,L,L|Seq] end. - - outputv_echo(doc) -> ["Test echoing data with a driver that supports outputv."]; outputv_echo(Config) when is_list(Config) -> ?line Dog = test_server:timetrap(test_server:minutes(10)), diff --git a/erts/etc/common/erlexec.c b/erts/etc/common/erlexec.c index 60b3af7db7..90d3be9448 100644 --- a/erts/etc/common/erlexec.c +++ b/erts/etc/common/erlexec.c @@ -394,6 +394,7 @@ int main(int argc, char **argv) int print_args_exit = 0; int print_qouted_cmd_exit = 0; erts_cpu_info_t *cpuinfo = NULL; + char* emu_name; #ifdef __WIN32__ this_module_handle = module; @@ -566,6 +567,7 @@ int main(int argc, char **argv) usage("+MYm"); } emu = add_extra_suffixes(emu, emu_type); + emu_name = strsave(emu); erts_snprintf(tmpStr, sizeof(tmpStr), "%s" DIRSEP "%s" BINARY_EXT, bindir, emu); emu = strsave(tmpStr); @@ -682,6 +684,9 @@ int main(int argc, char **argv) verbose = 1; } else if (strcmp(argv[i], "-emu_args_exit") == 0) { print_args_exit = 1; + } else if (strcmp(argv[i], "-emu_name_exit") == 0) { + printf("%s\n", emu_name); + exit(0); } else if (strcmp(argv[i], "-emu_qouted_cmd_exit") == 0) { print_qouted_cmd_exit = 1; } else if (strcmp(argv[i], "-env") == 0) { /* -env VARNAME VARVALUE */ @@ -1970,6 +1975,11 @@ initial_argv_massage(int *argc, char ***argv) */ vix = 0; + + av = build_args_from_env("ERL_" OTP_SYSTEM_VERSION "_FLAGS"); + if (av) + avv[vix++].argv = av; + av = build_args_from_env("ERL_AFLAGS"); if (av) avv[vix++].argv = av; @@ -1984,10 +1994,6 @@ initial_argv_massage(int *argc, char ***argv) if (av) avv[vix++].argv = av; - av = build_args_from_env("ERL_" OTP_SYSTEM_VERSION "_FLAGS"); - if (av) - avv[vix++].argv = av; - av = build_args_from_env("ERL_ZFLAGS"); if (av) avv[vix++].argv = av; diff --git a/erts/etc/unix/cerl.src b/erts/etc/unix/cerl.src index 0355f2629f..0b2d6512ea 100644 --- a/erts/etc/unix/cerl.src +++ b/erts/etc/unix/cerl.src @@ -62,13 +62,21 @@ cxargs_add() { done } +eeargs= +eeargs_add() { + while [ $# -gt 0 ]; do + cargs="$cargs $1" + eeargs="$eeargs $1" + shift + done +} + core= GDB= GDBBP= GDBARGS= TYPE= -EMU_TYPE= debug= run_valgrind=no @@ -127,34 +135,37 @@ while [ $# -gt 0 ]; do ;; "-smp") shift - cargs="$cargs -smp" - EMU_TYPE=.smp + if [ $# -le 0 ]; then + eeargs_add -smp + else + case $1 in + disable) + shift + eeargs_add -smpdisable + ;; + enable) + shift + eeargs_add -smp + ;; + *) + eeargs_add -smp + esac + fi + ;; + "-smpdisable") + shift + eeargs_add -smpdisable ;; "-lcnt") shift cargs="$cargs -lcnt" TYPE=.lcnt ;; - "-frag") - shift - cargs="$cargs -frag" - EMU_TYPE=.frag - ;; - "-smp_frag") - shift - cargs="$cargs -smp_frag" - EMU_TYPE=.smp_frag - ;; "-gprof") shift cargs="$cargs -gprof" TYPE=.gprof ;; - "-hybrid") - shift - cargs="$cargs -hybrid" - EMU_TYPE=.hybrid - ;; "-debug") shift cargs="$cargs -debug" @@ -180,11 +191,6 @@ while [ $# -gt 0 ]; do # shift # GDB=xxgdb # ;; - "-shared") - shift - cargs="$cargs -shared" - TYPE=.shared - ;; "-purify") shift cargs="$cargs -purify" @@ -222,7 +228,9 @@ PATH=$BINDIR:$ROOTDIR/bin:$PATH EXEC=$BINDIR/erlexec PROGNAME="$PROGNAME $cargs" -EMU=$EMU$TYPE$EMU_TYPE +EMU="$EMU$TYPE" +EMU_NAME=`$EXEC -emu_name_exit $eeargs` + if [ $run_valgrind != yes ]; then xargs="$xargs -pz $PRELOADED --" fi @@ -248,9 +256,9 @@ if [ "x$GDB" = "x" ]; then valgrind_log= else if [ $valmajor -gt 2 -a $valminor -gt 4 ]; then - valgrind_log="$log_file_prefix$VALGRIND_LOG_DIR/$VALGRIND_LOGFILE_PREFIX$VALGRIND_LOGFILE_INFIX$EMU.log.$$" + valgrind_log="$log_file_prefix$VALGRIND_LOG_DIR/$VALGRIND_LOGFILE_PREFIX$VALGRIND_LOGFILE_INFIX$EMU_NAME.log.$$" else - valgrind_log="$log_file_prefix$VALGRIND_LOG_DIR/$VALGRIND_LOGFILE_PREFIX$VALGRIND_LOGFILE_INFIX$EMU.log" + valgrind_log="$log_file_prefix$VALGRIND_LOG_DIR/$VALGRIND_LOGFILE_PREFIX$VALGRIND_LOGFILE_INFIX$EMU_NAME.log" fi fi if [ "x$VALGRIND_MISC_FLAGS" = "x" ]; then @@ -263,9 +271,9 @@ if [ "x$GDB" = "x" ]; then early_beam_args=`echo $beam_args | sed "s|^\(.*-progname\).*$|\1|g"` late_beam_args=`echo $beam_args | sed "s|^$pre_beam_args.*\(-- -home.*\)$|\1|g"` - exec valgrind $valgrind_xml $valgrind_log $valgrind_misc_flags $BINDIR/$EMU $emu_xargs $early_beam_args "$PROGNAME" $late_beam_args -pz $PRELOADED + exec valgrind $valgrind_xml $valgrind_log $valgrind_misc_flags $BINDIR/$EMU_NAME $emu_xargs $early_beam_args "$PROGNAME" $late_beam_args -pz $PRELOADED else - exec $EXEC $xargs ${1+"$@"} + exec $EXEC $eeargs $xargs ${1+"$@"} fi else if [ "x$EMACS" = "x" ]; then @@ -300,5 +308,5 @@ else (insert-string \"source $ROOTDIR/erts/etc/unix/etp-commands\") \ (comint-send-input)" # Fire up gdb in emacs... - exec $EMACS --eval "(progn (gdb \"gdb $GDBARGS$EMU\") $gdbcmd)" + exec $EMACS --eval "(progn (gdb \"gdb $GDBARGS$EMU_NAME\") $gdbcmd)" fi diff --git a/erts/include/internal/libatomic_ops/ethr_atomic.h b/erts/include/internal/libatomic_ops/ethr_atomic.h index d56693dbf8..2fc82c99a8 100644 --- a/erts/include/internal/libatomic_ops/ethr_atomic.h +++ b/erts/include/internal/libatomic_ops/ethr_atomic.h @@ -146,13 +146,13 @@ ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var) static ETHR_INLINE ETHR_AINT_T__ ETHR_NATMC_FUNC__(add_return)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) { -#ifdef AO_HAVE_fetch_and_add - return ((ETHR_AINT_T__) AO_fetch_and_add(&var->counter, (AO_t) incr)) + incr; +#ifdef AO_HAVE_fetch_and_add_full + return ((ETHR_AINT_T__) AO_fetch_and_add_full(&var->counter, (AO_t) incr)) + incr; #else while (1) { AO_t exp = AO_load(&var->counter); AO_t new = exp + (AO_t) incr; - if (AO_compare_and_swap(&var->counter, exp, new)) + if (AO_compare_and_swap_full(&var->counter, exp, new)) return (ETHR_AINT_T__) new; } #endif @@ -167,8 +167,8 @@ ETHR_NATMC_FUNC__(add)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) static ETHR_INLINE ETHR_AINT_T__ ETHR_NATMC_FUNC__(inc_return)(ETHR_ATMC_T__ *var) { -#ifdef AO_HAVE_fetch_and_add1 - return ((ETHR_AINT_T__) AO_fetch_and_add1(&var->counter)) + 1; +#ifdef AO_HAVE_fetch_and_add1_full + return ((ETHR_AINT_T__) AO_fetch_and_add1_full(&var->counter)) + 1; #else return ETHR_NATMC_FUNC__(add_return)(var, 1); #endif @@ -183,8 +183,8 @@ ETHR_NATMC_FUNC__(inc)(ETHR_ATMC_T__ *var) static ETHR_INLINE ETHR_AINT_T__ ETHR_NATMC_FUNC__(dec_return)(ETHR_ATMC_T__ *var) { -#ifdef AO_HAVE_fetch_and_sub1 - return ((ETHR_AINT_T__) AO_fetch_and_sub1(&var->counter)) - 1; +#ifdef AO_HAVE_fetch_and_sub1_full + return ((ETHR_AINT_T__) AO_fetch_and_sub1_full(&var->counter)) - 1; #else return ETHR_NATMC_FUNC__(add_return)(var, -1); #endif @@ -202,7 +202,7 @@ ETHR_NATMC_FUNC__(and_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) while (1) { AO_t exp = AO_load(&var->counter); AO_t new = exp & ((AO_t) mask); - if (AO_compare_and_swap(&var->counter, exp, new)) + if (AO_compare_and_swap_full(&var->counter, exp, new)) return (ETHR_AINT_T__) exp; } } @@ -213,7 +213,7 @@ ETHR_NATMC_FUNC__(or_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) while (1) { AO_t exp = AO_load(&var->counter); AO_t new = exp | ((AO_t) mask); - if (AO_compare_and_swap(&var->counter, exp, new)) + if (AO_compare_and_swap_full(&var->counter, exp, new)) return (ETHR_AINT_T__) exp; } } @@ -225,7 +225,7 @@ ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var, { ETHR_AINT_T__ act; do { - if (AO_compare_and_swap(&var->counter, (AO_t) exp, (AO_t) new)) + if (AO_compare_and_swap_full(&var->counter, (AO_t) exp, (AO_t) new)) return exp; act = (ETHR_AINT_T__) AO_load(&var->counter); } while (act == exp); @@ -237,7 +237,7 @@ ETHR_NATMC_FUNC__(xchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new) { while (1) { AO_t exp = AO_load(&var->counter); - if (AO_compare_and_swap(&var->counter, exp, (AO_t) new)) + if (AO_compare_and_swap_full(&var->counter, exp, (AO_t) new)) return (ETHR_AINT_T__) exp; } } @@ -265,7 +265,6 @@ ETHR_NATMC_FUNC__(inc_return_acqb)(ETHR_ATMC_T__ *var) return ((ETHR_AINT_T__) AO_fetch_and_add1_acquire(&var->counter)) + 1; #else ETHR_AINT_T__ res = ETHR_NATMC_FUNC__(add_return)(var, 1); - ETHR_MEMORY_BARRIER; return res; #endif } @@ -287,7 +286,6 @@ ETHR_NATMC_FUNC__(dec_return_relb)(ETHR_ATMC_T__ *var) #ifdef AO_HAVE_fetch_and_sub1_release return ((ETHR_AINT_T__) AO_fetch_and_sub1_release(&var->counter)) - 1; #else - ETHR_MEMORY_BARRIER; return ETHR_NATMC_FUNC__(dec_return)(var); #endif } @@ -314,7 +312,6 @@ ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var, return act; #else ETHR_AINT_T__ act = ETHR_NATMC_FUNC__(cmpxchg)(var, new, exp); - ETHR_MEMORY_BARRIER; return act; #endif } @@ -333,7 +330,6 @@ ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var, } while (act == exp); return act; #else - ETHR_MEMORY_BARRIER; return ETHR_NATMC_FUNC__(cmpxchg)(var, new, exp); #endif } diff --git a/erts/include/internal/sparc32/atomic.h b/erts/include/internal/sparc32/atomic.h index 00380dbf07..16182f8b01 100644 --- a/erts/include/internal/sparc32/atomic.h +++ b/erts/include/internal/sparc32/atomic.h @@ -95,7 +95,7 @@ ETHR_NATMC_FUNC__(add_return)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) { ETHR_AINT_T__ old, tmp; - __asm__ __volatile__("membar #LoadLoad|#StoreLoad\n"); + __asm__ __volatile__("membar #LoadLoad|#StoreLoad\n" : : : "memory"); do { old = var->counter; tmp = old+incr; @@ -105,7 +105,7 @@ ETHR_NATMC_FUNC__(add_return)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) : "r"(old), "r"(&var->counter), "0"(tmp) : "memory"); } while (__builtin_expect(old != tmp, 0)); - __asm__ __volatile__("membar #StoreLoad|#StoreStore"); + __asm__ __volatile__("membar #StoreLoad|#StoreStore" : : : "memory"); return old+incr; } @@ -144,7 +144,7 @@ ETHR_NATMC_FUNC__(and_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) { ETHR_AINT_T__ old, tmp; - __asm__ __volatile__("membar #LoadLoad|#StoreLoad\n"); + __asm__ __volatile__("membar #LoadLoad|#StoreLoad\n" : : : "memory"); do { old = var->counter; tmp = old & mask; @@ -154,7 +154,7 @@ ETHR_NATMC_FUNC__(and_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) : "r"(old), "r"(&var->counter), "0"(tmp) : "memory"); } while (__builtin_expect(old != tmp, 0)); - __asm__ __volatile__("membar #StoreLoad|#StoreStore"); + __asm__ __volatile__("membar #StoreLoad|#StoreStore" : : : "memory"); return old; } @@ -163,7 +163,7 @@ ETHR_NATMC_FUNC__(or_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) { ETHR_AINT_T__ old, tmp; - __asm__ __volatile__("membar #LoadLoad|#StoreLoad\n"); + __asm__ __volatile__("membar #LoadLoad|#StoreLoad\n" : : : "memory"); do { old = var->counter; tmp = old | mask; @@ -173,7 +173,7 @@ ETHR_NATMC_FUNC__(or_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) : "r"(old), "r"(&var->counter), "0"(tmp) : "memory"); } while (__builtin_expect(old != tmp, 0)); - __asm__ __volatile__("membar #StoreLoad|#StoreStore"); + __asm__ __volatile__("membar #StoreLoad|#StoreStore" : : : "memory"); return old; } @@ -182,7 +182,7 @@ ETHR_NATMC_FUNC__(xchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ val) { ETHR_AINT_T__ old, new; - __asm__ __volatile__("membar #LoadLoad|#StoreLoad"); + __asm__ __volatile__("membar #LoadLoad|#StoreLoad" : : : "memory"); do { old = var->counter; new = val; @@ -192,20 +192,20 @@ ETHR_NATMC_FUNC__(xchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ val) : "r"(old), "r"(&var->counter), "0"(new) : "memory"); } while (__builtin_expect(old != new, 0)); - __asm__ __volatile__("membar #StoreLoad|#StoreStore"); + __asm__ __volatile__("membar #StoreLoad|#StoreStore" : : : "memory"); return old; } static ETHR_INLINE ETHR_AINT_T__ ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new, ETHR_AINT_T__ old) { - __asm__ __volatile__("membar #LoadLoad|#StoreLoad\n"); + __asm__ __volatile__("membar #LoadLoad|#StoreLoad\n" : : : "memory"); __asm__ __volatile__( ETHR_CAS__ " [%2], %1, %0" : "=&r"(new) : "r"(old), "r"(&var->counter), "0"(new) : "memory"); - __asm__ __volatile__("membar #StoreLoad|#StoreStore"); + __asm__ __volatile__("membar #StoreLoad|#StoreStore" : : : "memory"); return new; } @@ -213,13 +213,11 @@ ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new, ETHR_AINT_T__ * Atomic ops with at least specified barriers. */ -/* TODO: relax acquire barriers */ - static ETHR_INLINE ETHR_AINT_T__ ETHR_NATMC_FUNC__(read_acqb)(ETHR_ATMC_T__ *var) { ETHR_AINT_T__ res = ETHR_NATMC_FUNC__(read)(var); - __asm__ __volatile__("membar #LoadLoad|#LoadStore|#StoreLoad|#StoreStore" : : : "memory"); + __asm__ __volatile__("membar #LoadLoad|#LoadStore" : : : "memory"); return res; } @@ -234,21 +232,18 @@ static ETHR_INLINE ETHR_AINT_T__ ETHR_NATMC_FUNC__(inc_return_acqb)(ETHR_ATMC_T__ *var) { ETHR_AINT_T__ res = ETHR_NATMC_FUNC__(inc_return)(var); - __asm__ __volatile__("membar #LoadLoad|#LoadStore" : : : "memory"); return res; } static ETHR_INLINE void ETHR_NATMC_FUNC__(dec_relb)(ETHR_ATMC_T__ *var) { - __asm__ __volatile__("membar #LoadStore|#StoreStore" : : : "memory"); ETHR_NATMC_FUNC__(dec)(var); } static ETHR_INLINE ETHR_AINT_T__ ETHR_NATMC_FUNC__(dec_return_relb)(ETHR_ATMC_T__ *var) { - __asm__ __volatile__("membar #LoadStore|#StoreStore" : : : "memory"); return ETHR_NATMC_FUNC__(dec_return)(var); } @@ -256,14 +251,12 @@ static ETHR_INLINE ETHR_AINT_T__ ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new, ETHR_AINT_T__ old) { ETHR_AINT_T__ res = ETHR_NATMC_FUNC__(cmpxchg)(var, new, old); - __asm__ __volatile__("membar #LoadLoad|#LoadStore" : : : "memory"); return res; } static ETHR_INLINE ETHR_AINT_T__ ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new, ETHR_AINT_T__ old) { - __asm__ __volatile__("membar #LoadStore|#StoreStore" : : : "memory"); return ETHR_NATMC_FUNC__(cmpxchg)(var, new, old); } diff --git a/erts/include/internal/tile/atomic.h b/erts/include/internal/tile/atomic.h index 48e4c0c6c8..0c7b597a6b 100644 --- a/erts/include/internal/tile/atomic.h +++ b/erts/include/internal/tile/atomic.h @@ -65,25 +65,35 @@ ethr_native_atomic32_read(ethr_native_atomic32_t *var) static ETHR_INLINE void ethr_native_atomic32_add(ethr_native_atomic32_t *var, ethr_sint32_t incr) { + ETHR_MEMORY_BARRIER; atomic_add(&var->counter, incr); + ETHR_MEMORY_BARRIER; } static ETHR_INLINE void ethr_native_atomic32_inc(ethr_native_atomic32_t *var) { + ETHR_MEMORY_BARRIER; atomic_increment(&var->counter); + ETHR_MEMORY_BARRIER; } static ETHR_INLINE void ethr_native_atomic32_dec(ethr_native_atomic32_t *var) { + ETHR_MEMORY_BARRIER; atomic_decrement(&var->counter); + ETHR_MEMORY_BARRIER; } static ETHR_INLINE ethr_sint32_t ethr_native_atomic32_add_return(ethr_native_atomic32_t *var, ethr_sint32_t incr) { - return atomic_exchange_and_add(&var->counter, incr) + incr; + ethr_sint32_t res; + ETHR_MEMORY_BARRIER; + res = atomic_exchange_and_add(&var->counter, incr) + incr; + ETHR_MEMORY_BARRIER; + return res; } static ETHR_INLINE ethr_sint32_t @@ -101,18 +111,27 @@ ethr_native_atomic32_dec_return(ethr_native_atomic32_t *var) static ETHR_INLINE ethr_sint32_t ethr_native_atomic32_and_retold(ethr_native_atomic32_t *var, ethr_sint32_t mask) { - return atomic_and_val(&var->counter, mask); + ethr_sint32_t res; + ETHR_MEMORY_BARRIER; + res = atomic_and_val(&var->counter, mask); + ETHR_MEMORY_BARRIER; + return res; } static ETHR_INLINE ethr_sint32_t ethr_native_atomic32_or_retold(ethr_native_atomic32_t *var, ethr_sint32_t mask) { - return atomic_or_val(&var->counter, mask); + ethr_sint32_t res; + ETHR_MEMORY_BARRIER; + res = atomic_or_val(&var->counter, mask); + ETHR_MEMORY_BARRIER; + return res; } static ETHR_INLINE ethr_sint32_t ethr_native_atomic32_xchg(ethr_native_atomic32_t *var, ethr_sint32_t val) { + ETHR_MEMORY_BARRIER; return atomic_exchange_acq(&var->counter, val); } @@ -121,6 +140,7 @@ ethr_native_atomic32_cmpxchg(ethr_native_atomic32_t *var, ethr_sint32_t new, ethr_sint32_t expected) { + ETHR_MEMORY_BARRIER; return atomic_compare_and_exchange_val_acq(&var->counter, new, expected); } @@ -139,9 +159,7 @@ ethr_native_atomic32_read_acqb(ethr_native_atomic32_t *var) static ETHR_INLINE ethr_sint32_t ethr_native_atomic32_inc_return_acqb(ethr_native_atomic32_t *var) { - ethr_sint32_t res = ethr_native_atomic32_inc_return(var); - ETHR_MEMORY_BARRIER; - return res; + return ethr_native_atomic32_inc_return(var); } static ETHR_INLINE void @@ -154,14 +172,12 @@ ethr_native_atomic32_set_relb(ethr_native_atomic32_t *var, ethr_sint32_t val) static ETHR_INLINE void ethr_native_atomic32_dec_relb(ethr_native_atomic32_t *var) { - ETHR_MEMORY_BARRIER; ethr_native_atomic32_dec(var); } static ETHR_INLINE ethr_sint32_t ethr_native_atomic32_dec_return_relb(ethr_native_atomic32_t *var) { - ETHR_MEMORY_BARRIER; return ethr_native_atomic32_dec_return(var); } @@ -178,7 +194,6 @@ ethr_native_atomic32_cmpxchg_relb(ethr_native_atomic32_t *var, ethr_sint32_t new, ethr_sint32_t exp) { - ETHR_MEMORY_BARRIER; return ethr_native_atomic32_cmpxchg(var, new, exp); } |