diff options
Diffstat (limited to 'erts')
55 files changed, 1384 insertions, 574 deletions
diff --git a/erts/aclocal.m4 b/erts/aclocal.m4 index a1211bbf0c..f7e5c31910 100644 --- a/erts/aclocal.m4 +++ b/erts/aclocal.m4 @@ -1153,7 +1153,7 @@ case "$THR_LIB_NAME" in AO_nop_full(); AO_store(&x, (AO_t) 0); z = AO_load(&x); - z = AO_compare_and_swap(&x, (AO_t) 0, (AO_t) 1); + z = AO_compare_and_swap_full(&x, (AO_t) 0, (AO_t) 1); ], [ethr_have_native_atomics=yes ethr_have_libatomic_ops=yes]) diff --git a/erts/configure.in b/erts/configure.in index 31d1d55b8a..fac07f8b6a 100644 --- a/erts/configure.in +++ b/erts/configure.in @@ -769,6 +769,7 @@ if test "$enable_halfword_emulator" = "yes"; then if test "$ARCH" = "amd64"; then AC_DEFINE(HALFWORD_HEAP_EMULATOR, [1], [Define if building a halfword-heap 64bit emulator]) + ENABLE_ALLOC_TYPE_VARS="$ENABLE_ALLOC_TYPE_VARS halfword" AC_MSG_RESULT([yes]) else AC_MSG_ERROR(no; halfword emulator not supported on this architecture) @@ -3523,6 +3524,8 @@ dnl use "PATH/include" and "PATH/lib". AC_SUBST(SSL_INCLUDE) AC_SUBST(SSL_ROOT) AC_SUBST(SSL_LIBDIR) +AC_SUBST(SSL_CRYPTO_LIBNAME) +AC_SUBST(SSL_SSL_LIBNAME) AC_SUBST(SSL_CC_RUNTIME_LIBRARY_PATH) AC_SUBST(SSL_LD_RUNTIME_LIBRARY_PATH) AC_SUBST(SSL_DED_LD_RUNTIME_LIBRARY_PATH) @@ -3686,19 +3689,41 @@ case "$erl_xcomp_without_sysroot-$with_ssl" in CRYPTO_APP=crypto SSH_APP=ssh - AC_MSG_CHECKING(for OpenSSL >= 0.9.7 in standard locations) + SSL_CRYPTO_LIBNAME=crypto + SSL_SSL_LIBNAME=ssl + + AC_MSG_CHECKING(for OpenSSL >= 0.9.7 in standard locations) for rdir in $extra_dir /cygdrive/c/OpenSSL $std_ssl_locations; do dir="$erl_xcomp_sysroot$rdir" if test -f "$erl_xcomp_isysroot$rdir/include/openssl/opensslv.h"; then is_real_ssl=yes SSL_ROOT="$dir" if test "x$MIXED_CYGWIN" = "xyes" ; then - if test -f "$dir/lib/VC/ssleay32.lib" || \ - test -f "$dir/lib/VC/openssl.lib"; then + if test -f "$dir/lib/VC/libeay32.lib"; then + SSL_RUNTIME_LIBDIR="$rdir/lib/VC" + SSL_LIBDIR="$dir/lib/VC" + SSL_CRYPTO_LIBNAME=libeay32 + SSL_SSL_LIBNAME=ssleay32 + elif test -f "$dir/lib/VC/openssl.lib"; then SSL_RUNTIME_LIBDIR="$rdir/lib/VC" SSL_LIBDIR="$dir/lib/VC" - elif test -f "$dir/lib/ssleay32.lib" || \ - test -f "$dir/lib/openssl.lib"; then + elif test -f $dir/lib/VC/libeay32MD.lib; then + SSL_CRYPTO_LIBNAME=libeay32MD + SSL_SSL_LIBNAME=ssleay32MD + if test "x$enable_dynamic_ssl" = "xno" && \ + test -f $dir/lib/VC/static/libeay32MD.lib; then + SSL_RUNTIME_LIBDIR="$rdir/lib/VC/static" + SSL_LIBDIR="$dir/lib/VC/static" + else + SSL_RUNTIME_LIBDIR="$rdir/lib/VC" + SSL_LIBDIR="$dir/lib/VC" + fi + elif test -f "$dir/lib/libeay32.lib"; then + SSL_RUNTIME_LIBDIR="$rdir/lib" + SSL_LIBDIR="$dir/lib" + SSL_CRYPTO_LIBNAME=libeay32 + SSL_CRYPTO_LIBNAME=ssleay32 + elif test -f "$dir/lib/openssl.lib"; then SSL_RUNTIME_LIBDIR="$rdir/lib" SSL_LIBDIR="$dir/lib" else @@ -3858,8 +3883,32 @@ dnl so it is - be adoptable AC_MSG_ERROR(Invalid path to option --with-ssl=PATH) fi SSL_ROOT="$with_ssl" + SSL_CRYPTO_LIBNAME=crypto + SSL_SSL_LIBNAME=ssl if test "x$MIXED_CYGWIN" = "xyes" && test -d "$with_ssl/lib/VC"; then - SSL_LIBDIR="$with_ssl/lib/VC" + if test -f "$with_ssl/lib/VC/libeay32.lib"; then + SSL_LIBDIR="$with_ssl/lib/VC" + SSL_CRYPTO_LIBNAME=libeay32 + SSL_SSL_LIBNAME=ssleay32 + elif test -f "$with_ssl/lib/VC/openssl.lib"; then + SSL_LIBDIR="$with_ssl/lib/VC" + elif test -f $with_ssl/lib/VC/libeay32MD.lib; then + SSL_CRYPTO_LIBNAME=libeay32MD + SSL_SSL_LIBNAME=ssleay32MD + if test "x$enable_dynamic_ssl" = "xno" && \ + test -f $with_ssl/lib/VC/static/libeay32MD.lib; then + SSL_LIBDIR="$with_ssl/lib/VC/static" + else + SSL_LIBDIR="$with_ssl/lib/VC" + fi + elif test -f "$with_ssl/lib/libeay32.lib"; then + SSL_LIBDIR="$with_ssl/lib" + SSL_CRYPTO_LIBNAME=libeay32 + SSL_CRYPTO_LIBNAME=ssleay32 + else + # This probably wont work, but that's what the user said, so... + SSL_LIBDIR="$with_ssl/lib" + fi elif test "x$ac_cv_sizeof_void_p" = "x8"; then if test -f "$with_ssl/lib64/libcrypto.a"; then SSL_LIBDIR="$with_ssl/lib64" diff --git a/erts/doc/src/erlang.xml b/erts/doc/src/erlang.xml index 19f501391f..f98e15cb52 100644 --- a/erts/doc/src/erlang.xml +++ b/erts/doc/src/erlang.xml @@ -2356,6 +2356,14 @@ os_prompt%</pre> <seealso marker="tools:instrument">instrument(3)</seealso> and/or <seealso marker="erts:erl">erl(1)</seealso>.</p> </item> + <tag><c>low</c></tag> + <item> + <p>Only on 64-bit halfword emulator.</p> + <p>The total amount of memory allocated in low memory areas + that are restricted to less than 4 Gb even though + the system may have more physical memory.</p> + <p>May be removed in future releases of halfword emulator.</p> + </item> </taglist> <note> <p>The <c>system</c> value is not complete. Some allocated diff --git a/erts/doc/src/notes.xml b/erts/doc/src/notes.xml index f5607945a8..7383ea381d 100644 --- a/erts/doc/src/notes.xml +++ b/erts/doc/src/notes.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2004</year><year>2010</year> + <year>2004</year><year>2011</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index 327620772f..de76cb9680 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -76,6 +76,7 @@ atom allocator_sizes atom alloc_util_allocators atom allow_passive_connect atom already_loaded +atom amd64 atom anchored atom and atom andalso diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 1dbf6f9b92..d76a7d8e9f 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2010. All Rights Reserved. + * Copyright Ericsson AB 1999-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -142,7 +142,7 @@ BIF_RETTYPE code_is_module_native_1(BIF_ALIST_1) if ((modp = erts_get_module(BIF_ARG_1)) == NULL) { return am_undefined; } - return (is_native(modp->code) || + return ((modp->code && is_native(modp->code)) || (modp->old_code != 0 && is_native(modp->old_code))) ? am_true : am_false; } diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 32ea8588d2..fb90a7d4f7 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -1911,13 +1911,15 @@ void process_main(void) * Note that for the halfword emulator, the two first elements * of the array are used. */ - *((BeamInstr **) (UWord) c_p->def_arg_reg) = I+3; + BeamInstr** pi = (BeamInstr**) c_p->def_arg_reg; + *pi = I+3; set_timer(c_p, unsigned_val(timeout_value)); } else if (timeout_value == am_infinity) { c_p->flags |= F_TIMO; #if !defined(ARCH_64) || HALFWORD_HEAP } else if (term_to_Uint(timeout_value, &time_val)) { - *((BeamInstr **) (UWord) c_p->def_arg_reg) = I+3; + BeamInstr** pi = (BeamInstr**) c_p->def_arg_reg; + *pi = I+3; set_timer(c_p, time_val); #endif } else { /* Wrong time */ @@ -1974,7 +1976,8 @@ void process_main(void) * we must test the F_INSLPQUEUE flag as well as the F_TIMO flag. */ if ((c_p->flags & (F_INSLPQUEUE | F_TIMO)) == 0) { - *((BeamInstr **) (UWord) c_p->def_arg_reg) = I+3; + BeamInstr** p = (BeamInstr **) c_p->def_arg_reg; + *p = I+3; set_timer(c_p, Arg(1)); } goto wait2; @@ -5322,7 +5325,7 @@ void process_main(void) ep->code[3] = (BeamInstr) OpCode(apply_bif); ep->code[4] = (BeamInstr) bif_table[i].f; /* XXX: set func info for bifs */ - ((BeamInstr*)ep->code + 3)[-5] = (BeamInstr) BeamOp(op_i_func_info_IaaI); + ep->fake_op_func_info_for_hipe[0] = (BeamInstr) BeamOp(op_i_func_info_IaaI); } return; diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 8c35644125..68b3350d7f 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -2168,20 +2168,146 @@ BIF_RETTYPE tl_1(BIF_ALIST_1) /**********************************************************************/ /* return the size of an I/O list */ -BIF_RETTYPE iolist_size_1(BIF_ALIST_1) +static Eterm +accumulate(Eterm acc, Uint size) { - Sint size = io_list_len(BIF_ARG_1); + if (is_non_value(acc)) { + /* + * There is no pre-existing accumulator. Allocate a + * bignum buffer with one extra word to be used if + * the bignum grows in the future. + */ + Eterm* hp = (Eterm *) erts_alloc(ERTS_ALC_T_TEMP_TERM, + (BIG_UINT_HEAP_SIZE+1) * + sizeof(Eterm)); + return uint_to_big(size, hp); + } else { + Eterm* big; + int need_heap; - if (size == -1) { - BIF_ERROR(BIF_P, BADARG); - } else if (IS_USMALL(0, (Uint) size)) { - BIF_RET(make_small(size)); + /* + * Add 'size' to 'acc' in place. There is always one + * extra word allocated in case the bignum grows by one word. + */ + big = big_val(acc); + need_heap = BIG_NEED_SIZE(BIG_SIZE(big)); + acc = big_plus_small(acc, size, big); + if (BIG_NEED_SIZE(big_size(acc)) > need_heap) { + /* + * The extra word has been consumed. Grow the + * allocation by one word. + */ + big = (Eterm *) erts_realloc(ERTS_ALC_T_TEMP_TERM, + big_val(acc), + (need_heap+1) * sizeof(Eterm)); + acc = make_big(big); + } + return acc; + } +} + +static Eterm +consolidate(Process* p, Eterm acc, Uint size) +{ + Eterm* hp; + + if (is_non_value(acc)) { + return erts_make_integer(size, p); } else { - Eterm* hp = HAlloc(BIF_P, BIG_UINT_HEAP_SIZE); - BIF_RET(uint_to_big(size, hp)); + Eterm* big; + Uint sz; + Eterm res; + + acc = accumulate(acc, size); + big = big_val(acc); + sz = BIG_NEED_SIZE(BIG_SIZE(big)); + hp = HAlloc(p, sz); + res = make_big(hp); + while (sz--) { + *hp++ = *big++; + } + erts_free(ERTS_ALC_T_TEMP_TERM, (void *) big_val(acc)); + return res; } } +BIF_RETTYPE iolist_size_1(BIF_ALIST_1) +{ + Eterm obj, hd; + Eterm* objp; + Uint size = 0; + Uint cur_size; + Uint new_size; + Eterm acc = THE_NON_VALUE; + DECLARE_ESTACK(s); + + obj = BIF_ARG_1; + goto L_again; + + while (!ESTACK_ISEMPTY(s)) { + obj = ESTACK_POP(s); + L_again: + if (is_list(obj)) { + L_iter_list: + objp = list_val(obj); + hd = CAR(objp); + obj = CDR(objp); + /* Head */ + if (is_byte(hd)) { + size++; + if (size == 0) { + acc = accumulate(acc, (Uint) -1); + size = 1; + } + } else if (is_binary(hd) && binary_bitsize(hd) == 0) { + cur_size = binary_size(hd); + if ((new_size = size + cur_size) >= size) { + size = new_size; + } else { + acc = accumulate(acc, size); + size = cur_size; + } + } else if (is_list(hd)) { + ESTACK_PUSH(s, obj); + obj = hd; + goto L_iter_list; + } else if (is_not_nil(hd)) { + goto L_type_error; + } + /* Tail */ + if (is_list(obj)) { + goto L_iter_list; + } else if (is_binary(obj) && binary_bitsize(obj) == 0) { + cur_size = binary_size(obj); + if ((new_size = size + cur_size) >= size) { + size = new_size; + } else { + acc = accumulate(acc, size); + size = cur_size; + } + } else if (is_not_nil(obj)) { + goto L_type_error; + } + } else if (is_binary(obj) && binary_bitsize(obj) == 0) { + cur_size = binary_size(obj); + if ((new_size = size + cur_size) >= size) { + size = new_size; + } else { + acc = accumulate(acc, size); + size = cur_size; + } + } else if (is_not_nil(obj)) { + goto L_type_error; + } + } + + DESTROY_ESTACK(s); + BIF_RET(consolidate(BIF_P, acc, size)); + + L_type_error: + DESTROY_ESTACK(s); + BIF_ERROR(BIF_P, BADARG); +} /**********************************************************************/ @@ -3282,6 +3408,7 @@ BIF_RETTYPE ports_0(BIF_ALIST_0) Eterm* dead_ports; int alive, dead; Uint32 next_ss; + int i; /* To get a consistent snapshot... * We add alive ports from start of the buffer @@ -3293,21 +3420,18 @@ BIF_RETTYPE ports_0(BIF_ALIST_0) erts_smp_atomic_set(&erts_dead_ports_ptr, (erts_aint_t) (port_buf + erts_max_ports)); - next_ss = erts_smp_atomic_inctest(&erts_ports_snapshot); + next_ss = erts_smp_atomic32_inctest(&erts_ports_snapshot); - if (erts_smp_atomic_read(&erts_ports_alive) > 0) { - erts_aint_t i; - for (i = erts_max_ports-1; i >= 0; i--) { - Port* prt = &erts_port[i]; - erts_smp_port_state_lock(prt); - if (!(prt->status & ERTS_PORT_SFLGS_DEAD) - && prt->snapshot != next_ss) { - ASSERT(prt->snapshot == next_ss - 1); - *pp++ = prt->id; - prt->snapshot = next_ss; /* Consumed by this snapshot */ - } - erts_smp_port_state_unlock(prt); + for (i = erts_max_ports-1; i >= 0; i--) { + Port* prt = &erts_port[i]; + erts_smp_port_state_lock(prt); + if (!(prt->status & ERTS_PORT_SFLGS_DEAD) + && prt->snapshot != next_ss) { + ASSERT(prt->snapshot == next_ss - 1); + *pp++ = prt->id; + prt->snapshot = next_ss; /* Consumed by this snapshot */ } + erts_smp_port_state_unlock(prt); } dead_ports = (Eterm*)erts_smp_atomic_xchg(&erts_dead_ports_ptr, diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c index f47f5a9c0c..d18de9ae5d 100644 --- a/erts/emulator/beam/big.c +++ b/erts/emulator/beam/big.c @@ -1588,7 +1588,7 @@ big_to_double(Wterm x, double* resp) /* ** Estimate the number of decimal digits (include sign) */ -int big_decimal_estimate(Eterm x) +int big_decimal_estimate(Wterm x) { Eterm* xp = big_val(x); int lg = I_lg(BIG_V(xp), BIG_SIZE(xp)); @@ -1602,7 +1602,7 @@ int big_decimal_estimate(Eterm x) ** Convert a bignum into a string of decimal numbers */ -static void write_big(Eterm x, void (*write_func)(void *, char), void *arg) +static void write_big(Wterm x, void (*write_func)(void *, char), void *arg) { Eterm* xp = big_val(x); ErtsDigit* dx = BIG_V(xp); @@ -1681,7 +1681,7 @@ write_string(void *arg, char c) *(--(*((char **) arg))) = c; } -char *erts_big_to_string(Eterm x, char *buf, Uint buf_sz) +char *erts_big_to_string(Wterm x, char *buf, Uint buf_sz) { char *big_str = buf + buf_sz - 1; *big_str = '\0'; diff --git a/erts/emulator/beam/big.h b/erts/emulator/beam/big.h index f28a390aea..2afc37004f 100644 --- a/erts/emulator/beam/big.h +++ b/erts/emulator/beam/big.h @@ -114,9 +114,9 @@ typedef Uint dsize_t; /* Vector size type */ #endif -int big_decimal_estimate(Eterm); +int big_decimal_estimate(Wterm); Eterm erts_big_to_list(Eterm, Eterm**); -char *erts_big_to_string(Eterm x, char *buf, Uint buf_sz); +char *erts_big_to_string(Wterm x, char *buf, Uint buf_sz); Eterm small_times(Sint, Sint, Eterm*); diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c index 9486602633..1fb39c6c67 100644 --- a/erts/emulator/beam/binary.c +++ b/erts/emulator/beam/binary.c @@ -32,11 +32,11 @@ #include "erl_bits.h" #ifdef DEBUG -static int list_to_bitstr_buf(Eterm obj, char* buf, int len); +static int list_to_bitstr_buf(Eterm obj, char* buf, Uint len); #else static int list_to_bitstr_buf(Eterm obj, char* buf); #endif -static Sint bitstr_list_len(Eterm obj); +static int bitstr_list_len(Eterm obj, Uint* num_bytes); void erts_init_binary(void) @@ -355,21 +355,24 @@ BIF_RETTYPE bitstring_to_list_1(BIF_ALIST_1) BIF_RETTYPE erts_list_to_binary_bif(Process *p, Eterm arg) { Eterm bin; - int i; + Uint size; int offset; byte* bytes; + if (is_nil(arg)) { BIF_RET(new_binary(p,(byte*)"",0)); } if (is_not_list(arg)) { goto error; } - if ((i = io_list_len(arg)) < 0) { - goto error; + switch (erts_iolist_size(arg, &size)) { + case ERTS_IOLIST_OVERFLOW: BIF_ERROR(p, SYSTEM_LIMIT); + case ERTS_IOLIST_TYPE: goto error; + default: ; } - bin = new_binary(p, (byte *)NULL, i); + bin = new_binary(p, (byte *)NULL, size); bytes = binary_bytes(bin); - offset = io_list_to_buf(arg, (char*) bytes, i); + offset = io_list_to_buf(arg, (char*) bytes, size); ASSERT(offset == 0); BIF_RET(bin); @@ -396,7 +399,8 @@ BIF_RETTYPE iolist_to_binary_1(BIF_ALIST_1) BIF_RETTYPE list_to_bitstring_1(BIF_ALIST_1) { Eterm bin; - int i,offset; + Uint sz; + int offset; byte* bytes; ErlSubBin* sb1; Eterm* hp; @@ -405,15 +409,19 @@ BIF_RETTYPE list_to_bitstring_1(BIF_ALIST_1) BIF_RET(new_binary(BIF_P,(byte*)"",0)); } if (is_not_list(BIF_ARG_1)) { - goto error; + error: + BIF_ERROR(BIF_P, BADARG); } - if ((i = bitstr_list_len(BIF_ARG_1)) < 0) { + switch (bitstr_list_len(BIF_ARG_1, &sz)) { + case ERTS_IOLIST_TYPE: goto error; + case ERTS_IOLIST_OVERFLOW: + BIF_ERROR(BIF_P, SYSTEM_LIMIT); } - bin = new_binary(BIF_P, (byte *)NULL, i); + bin = new_binary(BIF_P, (byte *)NULL, sz); bytes = binary_bytes(bin); #ifdef DEBUG - offset = list_to_bitstr_buf(BIF_ARG_1, (char*) bytes, i); + offset = list_to_bitstr_buf(BIF_ARG_1, (char*) bytes, sz); #else offset = list_to_bitstr_buf(BIF_ARG_1, (char*) bytes); #endif @@ -422,20 +430,16 @@ BIF_RETTYPE list_to_bitstring_1(BIF_ALIST_1) hp = HAlloc(BIF_P, ERL_SUB_BIN_SIZE); sb1 = (ErlSubBin *) hp; sb1->thing_word = HEADER_SUB_BIN; - sb1->size = i-1; + sb1->size = sz-1; sb1->offs = 0; sb1->orig = bin; sb1->bitoffs = 0; sb1->bitsize = offset; sb1->is_writable = 0; - hp += ERL_SUB_BIN_SIZE; bin = make_binary(sb1); } BIF_RET(bin); - - error: - BIF_ERROR(BIF_P, BADARG); } BIF_RETTYPE split_binary_2(BIF_ALIST_2) @@ -499,7 +503,7 @@ BIF_RETTYPE split_binary_2(BIF_ALIST_2) */ static int #ifdef DEBUG -list_to_bitstr_buf(Eterm obj, char* buf, int len) +list_to_bitstr_buf(Eterm obj, char* buf, Uint len) #else list_to_bitstr_buf(Eterm obj, char* buf) #endif @@ -602,8 +606,8 @@ list_to_bitstr_buf(Eterm obj, char* buf) return offset; } -static Sint -bitstr_list_len(Eterm obj) +static int +bitstr_list_len(Eterm obj, Uint* num_bytes) { Eterm* objp; Uint len = 0; @@ -611,6 +615,26 @@ bitstr_list_len(Eterm obj) DECLARE_ESTACK(s); goto L_again; +#define SAFE_ADD(Var, Val) \ + do { \ + Uint valvar = (Val); \ + Var += valvar; \ + if (Var < valvar) { \ + goto L_overflow_error; \ + } \ + } while (0) + +#define SAFE_ADD_BITSIZE(Var, Bin) \ + do { \ + if (*binary_val(Bin) == HEADER_SUB_BIN) { \ + Uint valvar = ((ErlSubBin *) binary_val(Bin))->bitsize; \ + Var += valvar; \ + if (Var < valvar) { \ + goto L_overflow_error; \ + } \ + } \ + } while (0) + while (!ESTACK_ISEMPTY(s)) { obj = ESTACK_POP(s); L_again: @@ -621,9 +645,12 @@ bitstr_list_len(Eterm obj) obj = CAR(objp); if (is_byte(obj)) { len++; + if (len == 0) { + goto L_overflow_error; + } } else if (is_binary(obj)) { - len += binary_size(obj); - offs += binary_bitsize(obj); + SAFE_ADD(len, binary_size(obj)); + SAFE_ADD_BITSIZE(offs, obj); } else if (is_list(obj)) { ESTACK_PUSH(s, CDR(objp)); goto L_iter_list; /* on head */ @@ -635,24 +662,44 @@ bitstr_list_len(Eterm obj) if (is_list(obj)) goto L_iter_list; /* on tail */ else if (is_binary(obj)) { - len += binary_size(obj); - offs += binary_bitsize(obj); + SAFE_ADD(len, binary_size(obj)); + SAFE_ADD_BITSIZE(offs, obj); } else if (is_not_nil(obj)) { goto L_type_error; } } else if (is_binary(obj)) { - len += binary_size(obj); - offs += binary_bitsize(obj); + SAFE_ADD(len, binary_size(obj)); + SAFE_ADD_BITSIZE(offs, obj); } else if (is_not_nil(obj)) { goto L_type_error; } } +#undef SAFE_ADD +#undef SAFE_ADD_BITSIZE DESTROY_ESTACK(s); - return (Sint) (len + (offs/8) + ((offs % 8) != 0)); + + /* + * Make sure that the number of bits in the bitstring will fit + * in an Uint to ensure that the binary can be matched using + * the binary syntax. + */ + if (len << 3 < len) { + goto L_overflow_error; + } + len += (offs >> 3) + ((offs & 7) != 0); + if (len << 3 < len) { + goto L_overflow_error; + } + *num_bytes = len; + return ERTS_IOLIST_OK; L_type_error: DESTROY_ESTACK(s); - return (Sint) -1; + return ERTS_IOLIST_TYPE; + + L_overflow_error: + DESTROY_ESTACK(s); + return ERTS_IOLIST_OVERFLOW; } diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index 673eac7fea..cda404af5e 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -90,6 +90,10 @@ typedef union { static ErtsAllocatorState_t sl_alloc_state; static ErtsAllocatorState_t std_alloc_state; static ErtsAllocatorState_t ll_alloc_state; +#if HALFWORD_HEAP +static ErtsAllocatorState_t std_alloc_low_state; +static ErtsAllocatorState_t ll_alloc_low_state; +#endif static ErtsAllocatorState_t temp_alloc_state; static ErtsAllocatorState_t eheap_alloc_state; static ErtsAllocatorState_t binary_alloc_state; @@ -166,6 +170,10 @@ typedef struct { struct au_init binary_alloc; struct au_init ets_alloc; struct au_init driver_alloc; +#if HALFWORD_HEAP + struct au_init std_alloc_low; + struct au_init ll_alloc_low; +#endif } erts_alc_hndl_args_init_t; #define ERTS_AU_INIT__ {0, 0, GOODFIT, DEFAULT_ALLCTR_INIT, {1,1,1,1}} @@ -193,6 +201,10 @@ set_default_sl_alloc_opts(struct au_init *ip) #endif ip->init.util.ts = ERTS_ALC_MTA_SHORT_LIVED; ip->init.util.rsbcst = 80; +#if HALFWORD_HEAP + ip->init.util.low_mem = 1; +#endif + } static void @@ -256,6 +268,9 @@ set_default_temp_alloc_opts(struct au_init *ip) ip->init.util.ts = ERTS_ALC_MTA_TEMPORARY; ip->init.util.rsbcst = 90; ip->init.util.rmbcmt = 100; +#if HALFWORD_HEAP + ip->init.util.low_mem = 1; +#endif } static void @@ -275,6 +290,9 @@ set_default_eheap_alloc_opts(struct au_init *ip) #endif ip->init.util.ts = ERTS_ALC_MTA_EHEAP; ip->init.util.rsbcst = 50; +#if HALFWORD_HEAP + ip->init.util.low_mem = 1; +#endif } static void @@ -531,6 +549,20 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) erts_allctrs[ERTS_ALC_A_SYSTEM].free = erts_sys_free; erts_allctrs_info[ERTS_ALC_A_SYSTEM].enabled = 1; +#if HALFWORD_HEAP + /* Init low memory variants by cloning */ + init.std_alloc_low = init.std_alloc; + init.std_alloc_low.init.util.alloc_no = ERTS_ALC_A_STANDARD_LOW; + init.std_alloc_low.init.util.low_mem = 1; + + init.ll_alloc_low = init.ll_alloc; + init.ll_alloc_low.init.util.alloc_no = ERTS_ALC_A_LONG_LIVED_LOW; + init.ll_alloc_low.init.util.low_mem = 1; + + set_au_allocator(ERTS_ALC_A_STANDARD_LOW, &init.std_alloc_low); + set_au_allocator(ERTS_ALC_A_LONG_LIVED_LOW, &init.ll_alloc_low); +#endif /* HALFWORD */ + set_au_allocator(ERTS_ALC_A_TEMPORARY, &init.temp_alloc); set_au_allocator(ERTS_ALC_A_SHORT_LIVED, &init.sl_alloc); set_au_allocator(ERTS_ALC_A_STANDARD, &init.std_alloc); @@ -576,7 +608,14 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) start_au_allocator(ERTS_ALC_A_LONG_LIVED, &init.ll_alloc, &ll_alloc_state); - +#if HALFWORD_HEAP + start_au_allocator(ERTS_ALC_A_LONG_LIVED_LOW, + &init.ll_alloc_low, + &ll_alloc_low_state); + start_au_allocator(ERTS_ALC_A_STANDARD_LOW, + &init.std_alloc_low, + &std_alloc_low_state); +#endif start_au_allocator(ERTS_ALC_A_EHEAP, &init.eheap_alloc, &eheap_alloc_state); @@ -612,11 +651,9 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) erts_set_fix_size(ERTS_ALC_T_PROC, sizeof(Process)); erts_set_fix_size(ERTS_ALC_T_DB_TABLE, sizeof(DbTable)); erts_set_fix_size(ERTS_ALC_T_ATOM, sizeof(Atom)); - erts_set_fix_size(ERTS_ALC_T_EXPORT, sizeof(Export)); + erts_set_fix_size(ERTS_ALC_T_MODULE, sizeof(Module)); erts_set_fix_size(ERTS_ALC_T_REG_PROC, sizeof(RegProc)); - erts_set_fix_size(ERTS_ALC_T_MONITOR_SH, ERTS_MONITOR_SH_SIZE*sizeof(Uint)); - erts_set_fix_size(ERTS_ALC_T_NLINK_SH, ERTS_LINK_SH_SIZE*sizeof(Uint)); erts_set_fix_size(ERTS_ALC_T_FUN_ENTRY, sizeof(ErlFunEntry)); #ifdef ERTS_ALC_T_DRV_EV_D_STATE erts_set_fix_size(ERTS_ALC_T_DRV_EV_D_STATE, @@ -626,6 +663,11 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) erts_set_fix_size(ERTS_ALC_T_DRV_SEL_D_STATE, sizeof(ErtsDrvSelectDataState)); #endif +#if !HALFWORD_HEAP + erts_set_fix_size(ERTS_ALC_T_EXPORT, sizeof(Export)); + erts_set_fix_size(ERTS_ALC_T_MONITOR_SH, ERTS_MONITOR_SH_SIZE*sizeof(Uint)); + erts_set_fix_size(ERTS_ALC_T_NLINK_SH, ERTS_LINK_SH_SIZE*sizeof(Uint)); +#endif #endif #endif @@ -638,6 +680,15 @@ set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init) ErtsAllocatorInfo_t *ai = &erts_allctrs_info[alctr_n]; ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[alctr_n]; +#if HALFWORD_HEAP + /* If halfword heap, silently ignore any disabling of internal + * allocators for low memory + */ + if (init->init.util.low_mem) { + init->enable = 1; + } +#endif + if (!init->enable) { af->alloc = erts_sys_alloc; af->realloc = erts_sys_realloc; @@ -1348,14 +1399,6 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) argv[j++] = argv[i]; } *argc = j; -#if HALFWORD_HEAP - /* If halfword heap, silently ignore any disabling of internal - allocators */ - for (i = 0; i < aui_sz; ++i) - aui[i]->enable = 1; -#endif - - } static char *type_no_str(ErtsAlcType_t n) @@ -1528,10 +1571,10 @@ erts_realloc_n_enomem(ErtsAlcType_t n, void *ptr, Uint size) erts_alc_fatal_error(ERTS_ALC_E_NOMEM, ERTS_ALC_O_REALLOC, n, size); } -static ERTS_INLINE Uint +static ERTS_INLINE UWord alcu_size(ErtsAlcType_t ai) { - Uint res = 0; + UWord res = 0; ASSERT(erts_allctrs_info[ai].enabled); ASSERT(erts_allctrs_info[ai].alloc_util); @@ -1563,6 +1606,49 @@ alcu_size(ErtsAlcType_t ai) return res; } +#if HALFWORD_HEAP +static ERTS_INLINE int +alcu_is_low(ErtsAlcType_t ai) +{ + int is_low = 0; + ASSERT(erts_allctrs_info[ai].enabled); + ASSERT(erts_allctrs_info[ai].alloc_util); + + if (!erts_allctrs_info[ai].thr_spec) { + Allctr_t *allctr = erts_allctrs_info[ai].extra; + is_low = allctr->mseg_opt.low_mem; + } + else { + ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[ai]; + int i; +# ifdef DEBUG + int found_one = 0; +# endif + + ASSERT(tspec->all_thr_safe); + ASSERT(tspec->enabled); + + for (i = tspec->size - 1; i >= 0; i--) { + Allctr_t *allctr = tspec->allctr[i]; + if (allctr) { +# ifdef DEBUG + if (!found_one) { + is_low = allctr->mseg_opt.low_mem; + found_one = 1; + } + else ASSERT(is_low == allctr->mseg_opt.low_mem); +# else + is_low = allctr->mseg_opt.low_mem; + break; +# endif + } + } + ASSERT(found_one); + } + return is_low; +} +#endif /* HALFWORD */ + Eterm erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) { @@ -1579,22 +1665,28 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) int code; int ets; int maximum; +#if HALFWORD_HEAP + int low; +#endif } want = {0}; struct { - Uint total; - Uint processes; - Uint processes_used; - Uint system; - Uint atom; - Uint atom_used; - Uint binary; - Uint code; - Uint ets; - Uint maximum; + UWord total; + UWord processes; + UWord processes_used; + UWord system; + UWord atom; + UWord atom_used; + UWord binary; + UWord code; + UWord ets; + UWord maximum; +#if HALFWORD_HEAP + UWord low; +#endif } size = {0}; - Eterm atoms[sizeof(size)/sizeof(Uint)]; - Uint *uintps[sizeof(size)/sizeof(Uint)]; - Eterm euints[sizeof(size)/sizeof(Uint)]; + Eterm atoms[sizeof(size)/sizeof(UWord)]; + UWord *uintps[sizeof(size)/sizeof(UWord)]; + Eterm euints[sizeof(size)/sizeof(UWord)]; int want_tot_or_sys; int length; Eterm res = THE_NON_VALUE; @@ -1646,7 +1738,11 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) atoms[length] = am_maximum; uintps[length++] = &size.maximum; } - +#if HALFWORD_HEAP + want.low = 1; + atoms[length] = am_low; + uintps[length++] = &size.low; +#endif } else { DeclareTmpHeapNoproc(tmp_heap,2); @@ -1740,6 +1836,15 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) return am_badarg; } break; +#if HALFWORD_HEAP + case am_low: + if (!want.low) { + want.low = 1; + atoms[length] = am_low; + uintps[length++] = &size.low; + } + break; +#endif default: UnUseTmpHeapNoproc(2); return am_badarg; @@ -1769,7 +1874,7 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) ASSERT(length <= sizeof(atoms)/sizeof(Eterm)); ASSERT(length <= sizeof(euints)/sizeof(Eterm)); - ASSERT(length <= sizeof(uintps)/sizeof(Uint)); + ASSERT(length <= sizeof(uintps)/sizeof(UWord)); if (proc) { @@ -1788,8 +1893,8 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) for (ai = ERTS_ALC_A_MIN; ai <= ERTS_ALC_A_MAX; ai++) { if (erts_allctrs_info[ai].alloc_util) { - Uint *save; - Uint asz; + UWord *save; + UWord asz; switch (ai) { case ERTS_ALC_A_TEMPORARY: /* @@ -1814,6 +1919,11 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) if (save) *save = asz; size.total += asz; +#if HALFWORD_HEAP + if (alcu_is_low(ai)) { + size.low += asz; + } +#endif } } } @@ -1821,7 +1931,7 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) if (want_tot_or_sys || want.processes || want.processes_used) { - Uint tmp; + UWord tmp; if (ERTS_MEM_NEED_ALL_ALCU) tmp = size.processes; @@ -1836,6 +1946,9 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) size.processes = size.processes_used = tmp; +#if HALFWORD_HEAP + /* BUG: We ignore link and monitor memory */ +#else erts_fix_info(ERTS_ALC_T_NLINK_SH, &efi); size.processes += efi.total; size.processes_used += efi.used; @@ -1843,6 +1956,7 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) erts_fix_info(ERTS_ALC_T_MONITOR_SH, &efi); size.processes += efi.total; size.processes_used += efi.used; +#endif erts_fix_info(ERTS_ALC_T_PROC, &efi); size.processes += efi.total; @@ -1879,8 +1993,12 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) erts_fix_info(ERTS_ALC_T_MODULE, &efi); size.code += efi.used; size.code += export_table_sz(); +#if HALFWORD_HEAP + size.code += export_list_size() * sizeof(Export); +#else erts_fix_info(ERTS_ALC_T_EXPORT, &efi); size.code += efi.used; +#endif size.code += erts_fun_table_sz(); erts_fix_info(ERTS_ALC_T_FUN_ENTRY, &efi); size.code += efi.used; @@ -1913,7 +2031,7 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) /* Print result... */ erts_print(to, arg, "=memory\n"); for (i = 0; i < length; i++) - erts_print(to, arg, "%T: %beu\n", atoms[i], *uintps[i]); + erts_print(to, arg, "%T: %bpu\n", atoms[i], *uintps[i]); } if (proc) { @@ -1926,9 +2044,9 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) if (only_one_value) { ASSERT(length == 1); hsz = 0; - erts_bld_uint(NULL, &hsz, *uintps[0]); + erts_bld_uword(NULL, &hsz, *uintps[0]); hp = hsz ? HAlloc((Process *) proc, hsz) : NULL; - res = erts_bld_uint(&hp, NULL, *uintps[0]); + res = erts_bld_uword(&hp, NULL, *uintps[0]); } else { Uint **hpp = NULL; @@ -1938,7 +2056,7 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) while (1) { int i; for (i = 0; i < length; i++) - euints[i] = erts_bld_uint(hpp, hszp, *uintps[i]); + euints[i] = erts_bld_uword(hpp, hszp, *uintps[i]); res = erts_bld_2tup_list(hpp, hszp, length, atoms, euints); if (hpp) break; diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index ca71798917..c6cc0e1fac 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -75,6 +75,11 @@ allocator EHEAP true eheap_alloc allocator ETS true ets_alloc allocator FIXED_SIZE true fix_alloc ++if halfword +allocator LONG_LIVED_LOW true ll_alloc_low +allocator STANDARD_LOW true std_alloc_low ++endif + +else # Non smp build allocator TEMPORARY false temp_alloc @@ -85,12 +90,18 @@ allocator EHEAP false eheap_alloc allocator ETS false ets_alloc allocator FIXED_SIZE false fix_alloc ++if halfword +allocator LONG_LIVED_LOW false ll_alloc_low +allocator STANDARD_LOW false std_alloc_low ++endif + +endif allocator BINARY true binary_alloc allocator DRIVER true driver_alloc + # --- Class declarations ----------------------------------------------------- # # Syntax: class <CLASS> <DESCRIPTION> @@ -125,14 +136,9 @@ class SYSTEM system_data type PROC FIXED_SIZE PROCESSES proc type ATOM FIXED_SIZE ATOM atom_entry -type EXPORT FIXED_SIZE CODE export_entry type MODULE FIXED_SIZE CODE module_entry type REG_PROC FIXED_SIZE PROCESSES reg_proc type LINK_LH STANDARD PROCESSES link_lh -type MONITOR_SH FIXED_SIZE PROCESSES monitor_sh -type MONITOR_LH STANDARD PROCESSES monitor_lh -type NLINK_SH FIXED_SIZE PROCESSES nlink_sh -type NLINK_LH STANDARD PROCESSES nlink_lh type SUSPEND_MON STANDARD PROCESSES suspend_monitor type PEND_SUSPEND SHORT_LIVED PROCESSES pending_suspend type PROC_LIST SHORT_LIVED PROCESSES proc_list @@ -175,7 +181,6 @@ type DRIVER STANDARD SYSTEM driver type NIF DRIVER SYSTEM nif_internal type BINARY BINARY BINARIES binary type NBIF_TABLE SYSTEM SYSTEM nbif_tab -type CODE LONG_LIVED CODE code type ARG_REG STANDARD PROCESSES arg_reg type PROC_DICT STANDARD PROCESSES proc_dict type CALLS_BUF STANDARD PROCESSES calls_buf @@ -193,10 +198,8 @@ type DB_FIXATION SHORT_LIVED ETS db_fixation type DB_FIX_DEL SHORT_LIVED ETS fixed_del type DB_TABLES LONG_LIVED ETS db_tabs type DB_NTAB_ENT STANDARD ETS db_named_table_entry -type DB_HEIR_DATA STANDARD ETS db_heir_data type DB_TMP TEMPORARY ETS db_tmp type DB_MC_STK TEMPORARY ETS db_mc_stack -type DB_MS_PSDO_PROC LONG_LIVED ETS db_match_pseudo_proc type DB_MS_RUN_HEAP SHORT_LIVED ETS db_match_spec_run_heap type DB_MS_CMPL_HEAP TEMPORARY ETS db_match_spec_cmpl_heap type DB_SEG ETS ETS db_segment @@ -213,10 +216,8 @@ type LOGGER_DSBUF TEMPORARY SYSTEM logger_dsbuf type TMP_DSBUF TEMPORARY SYSTEM tmp_dsbuf type INFO_DSBUF SYSTEM SYSTEM info_dsbuf # INFO_DSBUF have to use the SYSTEM allocator; otherwise, a deadlock might occur -type SCHDLR_DATA LONG_LIVED SYSTEM scheduler_data type SCHDLR_SLP_INFO LONG_LIVED SYSTEM scheduler_sleep_info type RUNQS LONG_LIVED SYSTEM run_queues -type DDLL_PROCESS STANDARD SYSTEM ddll_processes type DDLL_HANDLE STANDARD SYSTEM ddll_handle type DDLL_ERRCODES LONG_LIVED SYSTEM ddll_errcodes type DDLL_TMP_BUF TEMPORARY SYSTEM ddll_tmp_buf @@ -327,13 +328,45 @@ type SSB SHORT_LIVED PROCESSES ssb +endif ++if halfword + +type DDLL_PROCESS STANDARD_LOW SYSTEM ddll_processes +type MONITOR_LH STANDARD_LOW PROCESSES monitor_lh +type NLINK_LH STANDARD_LOW PROCESSES nlink_lh +type CODE LONG_LIVED_LOW CODE code +type DB_HEIR_DATA STANDARD_LOW ETS db_heir_data +type DB_MS_PSDO_PROC LONG_LIVED_LOW ETS db_match_pseudo_proc +type SCHDLR_DATA LONG_LIVED_LOW SYSTEM scheduler_data +type LL_TEMP_TERM LONG_LIVED_LOW SYSTEM ll_temp_term + +# no FIXED_SIZE for low memory +type EXPORT STANDARD_LOW CODE export_entry +type MONITOR_SH STANDARD_LOW PROCESSES monitor_sh +type NLINK_SH STANDARD_LOW PROCESSES nlink_sh + ++else # "fullword" + +type DDLL_PROCESS STANDARD SYSTEM ddll_processes +type MONITOR_LH STANDARD PROCESSES monitor_lh +type NLINK_LH STANDARD PROCESSES nlink_lh +type CODE LONG_LIVED CODE code +type DB_HEIR_DATA STANDARD ETS db_heir_data +type DB_MS_PSDO_PROC LONG_LIVED ETS db_match_pseudo_proc +type SCHDLR_DATA LONG_LIVED SYSTEM scheduler_data +type LL_TEMP_TERM LONG_LIVED SYSTEM ll_temp_term + +type EXPORT FIXED_SIZE CODE export_entry +type MONITOR_SH FIXED_SIZE PROCESSES monitor_sh +type NLINK_SH FIXED_SIZE PROCESSES nlink_sh + ++endif + + # # Types used by system specific code # type TEMP_TERM TEMPORARY SYSTEM temp_term -type LL_TEMP_TERM LONG_LIVED SYSTEM ll_temp_term - type DRV_TAB LONG_LIVED SYSTEM drv_tab type DRV_EV_STATE LONG_LIVED SYSTEM driver_event_state type DRV_EV_D_STATE FIXED_SIZE SYSTEM driver_event_data_state diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index 84c72439a3..cc04ef65bf 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -1639,6 +1639,9 @@ static struct { Eterm e; Eterm t; Eterm ramv; +#if HALFWORD_HEAP + Eterm low; +#endif Eterm sbct; #if HAVE_ERTS_MSEG Eterm asbcst; @@ -1724,6 +1727,9 @@ init_atoms(Allctr_t *allctr) AM_INIT(e); AM_INIT(t); AM_INIT(ramv); +#if HALFWORD_HEAP + AM_INIT(low); +#endif AM_INIT(sbct); #if HAVE_ERTS_MSEG AM_INIT(asbcst); @@ -2168,6 +2174,9 @@ info_options(Allctr_t *allctr, "option e: true\n" "option t: %s\n" "option ramv: %s\n" +#if HALFWORD_HEAP + "option low: %s\n" +#endif "option sbct: %beu\n" #if HAVE_ERTS_MSEG "option asbcst: %bpu\n" @@ -2185,6 +2194,9 @@ info_options(Allctr_t *allctr, "option mbcgs: %beu\n", topt, allctr->ramv ? "true" : "false", +#if HALFWORD_HEAP + allctr->mseg_opt.low_mem ? "true" : "false", +#endif allctr->sbc_threshold, #if HAVE_ERTS_MSEG allctr->mseg_opt.abs_shrink_th, @@ -2243,6 +2255,9 @@ info_options(Allctr_t *allctr, add_2tup(hpp, szp, &res, am.sbct, bld_uint(hpp, szp, allctr->sbc_threshold)); +#if HALFWORD_HEAP + add_2tup(hpp, szp, &res, am.low, allctr->mseg_opt.low_mem ? am_true : am_false); +#endif add_2tup(hpp, szp, &res, am.ramv, allctr->ramv ? am_true : am_false); add_2tup(hpp, szp, &res, am.t, (allctr->t ? bld_uint(hpp, szp, (Uint) allctr->t) @@ -3105,13 +3120,12 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) goto error; #if HAVE_ERTS_MSEG - { - ErtsMsegOpt_t mseg_opt = ERTS_MSEG_DEFAULT_OPT_INITIALIZER; - - sys_memcpy((void *) &allctr->mseg_opt, - (void *) &mseg_opt, - sizeof(ErtsMsegOpt_t)); - } + sys_memcpy((void *) &allctr->mseg_opt, + (void *) &erts_mseg_default_opt, + sizeof(ErtsMsegOpt_t)); +# if HALFWORD_HEAP + allctr->mseg_opt.low_mem = init->low_mem; +# endif #endif allctr->name_prefix = init->name_prefix; diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index d296081714..ddf84c086c 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -38,6 +38,7 @@ typedef struct { int tspec; int tpref; int ramv; + int low_mem; /* HALFWORD only */ UWord sbct; UWord asbcst; UWord rsbcst; @@ -70,6 +71,7 @@ typedef struct { 0, /* (bool) tspec: thread specific */\ 0, /* (bool) tpref: thread preferred */\ 0, /* (bool) ramv: realloc always moves */\ + 0, /* (bool) low_mem: HALFWORD only */\ 512*1024, /* (bytes) sbct: sbc threshold */\ 2*1024*2024, /* (amount) asbcst: abs sbc shrink threshold */\ 20, /* (%) rsbcst: rel sbc shrink threshold */\ @@ -97,6 +99,7 @@ typedef struct { 0, /* (bool) tspec: thread specific */\ 0, /* (bool) tpref: thread preferred */\ 0, /* (bool) ramv: realloc always moves */\ + 0, /* (bool) low_mem: HALFWORD only */\ 64*1024, /* (bytes) sbct: sbc threshold */\ 2*1024*2024, /* (amount) asbcst: abs sbc shrink threshold */\ 20, /* (%) rsbcst: rel sbc shrink threshold */\ diff --git a/erts/emulator/beam/erl_async.c b/erts/emulator/beam/erl_async.c index 12c7631448..a920bd2c8c 100644 --- a/erts/emulator/beam/erl_async.c +++ b/erts/emulator/beam/erl_async.c @@ -251,6 +251,7 @@ static int async_del(long id) erts_free(ERTS_ALC_T_ASYNC, a); return 1; } + a = a->next; } erts_mtx_unlock(&async_q[i].mtx); } diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c index c9cdcb87a6..9631fb50db 100644 --- a/erts/emulator/beam/erl_bif_ddll.c +++ b/erts/emulator/beam/erl_bif_ddll.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2006-2010. All Rights Reserved. + * Copyright Ericsson AB 2006-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -146,7 +146,7 @@ BIF_RETTYPE erl_ddll_try_load_3(Process *p, Eterm path_term, Eterm name_term, Eterm options) { char *path = NULL; - int path_len; + Uint path_len; char *name = NULL; DE_Handle *dh; erts_driver_t *drv; @@ -221,9 +221,7 @@ BIF_RETTYPE erl_ddll_try_load_3(Process *p, Eterm path_term, goto error; } - path_len = io_list_len(path_term); - - if (path_len <= 0) { + if (erts_iolist_size(path_term, &path_len)) { goto error; } path = erts_alloc(ERTS_ALC_T_DDLL_TMP_BUF, path_len + 1 /* might need path separator */ + sys_strlen(name) + 1); @@ -1878,7 +1876,7 @@ static Eterm mkatom(char *str) static char *pick_list_or_atom(Eterm name_term) { char *name = NULL; - int name_len; + Uint name_len; if (is_atom(name_term)) { Atom *ap = atom_tab(atom_val(name_term)); if (ap->len == 0) { @@ -1890,8 +1888,7 @@ static char *pick_list_or_atom(Eterm name_term) memcpy(name,ap->name,ap->len); name[ap->len] = '\0'; } else { - name_len = io_list_len(name_term); - if (name_len <= 0) { + if (erts_iolist_size(name_term, &name_len)) { goto error; } name = erts_alloc(ERTS_ALC_T_DDLL_TMP_BUF, name_len + 1); diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index e50fc18e64..f264bf44df 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -1732,14 +1732,14 @@ info_1_tuple(Process* BIF_P, /* Pointer to current process. */ # define ERTS_ERROR_CHECKER_PRINTF_XML VALGRIND_PRINTF_XML # endif #endif - int buf_size = 8*1024; /* Try with 8KB first */ + Uint buf_size = 8*1024; /* Try with 8KB first */ char *buf = erts_alloc(ERTS_ALC_T_TMP, buf_size); int r = io_list_to_buf(*tp, (char*) buf, buf_size - 1); if (r < 0) { erts_free(ERTS_ALC_T_TMP, (void *) buf); - buf_size = io_list_len(*tp); - if (buf_size < 0) + if (erts_iolist_size(*tp, &buf_size)) { goto badarg; + } buf_size++; buf = erts_alloc(ERTS_ALC_T_TMP, buf_size); r = io_list_to_buf(*tp, (char*) buf, buf_size - 1); diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c index fbc92b9730..3fd35dd963 100644 --- a/erts/emulator/beam/erl_bif_port.c +++ b/erts/emulator/beam/erl_bif_port.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2010. All Rights Reserved. + * Copyright Ericsson AB 2001-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -996,6 +996,7 @@ static byte* convert_environment(Process* p, Eterm env) Eterm* hp; Uint heap_size; int n; + Uint size; byte* bytes; if ((n = list_length(env)) < 0) { @@ -1039,15 +1040,15 @@ static byte* convert_environment(Process* p, Eterm env) if (is_not_nil(env)) { goto done; } - if ((n = io_list_len(all)) < 0) { + if (erts_iolist_size(all, &size)) { goto done; } /* * Put the result in a binary (no risk for a memory leak that way). */ - (void) erts_new_heap_binary(p, NULL, n, &bytes); - io_list_to_buf(all, (char*)bytes, n); + (void) erts_new_heap_binary(p, NULL, size, &bytes); + io_list_to_buf(all, (char*)bytes, size); done: erts_free(ERTS_ALC_T_TMP, temp_heap); diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c index d4a8a3aaa7..26891c4348 100644 --- a/erts/emulator/beam/erl_bif_re.c +++ b/erts/emulator/beam/erl_bif_re.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2008-2010. All Rights Reserved. + * Copyright Ericsson AB 2008-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -417,7 +417,7 @@ build_compile_result(Process *p, Eterm error_tag, pcre *result, int errcode, con BIF_RETTYPE re_compile_2(BIF_ALIST_2) { - int slen; + Uint slen; char *expr; pcre *result; int errcode = 0; @@ -444,7 +444,7 @@ re_compile_2(BIF_ALIST_2) BIF_TRAP2(ucompile_trap_exportp, BIF_P, BIF_ARG_1, BIF_ARG_2); } - if ((slen = io_list_len(BIF_ARG_1)) < 0) { + if (erts_iolist_size(BIF_ARG_1, &slen)) { BIF_ERROR(BIF_P,BADARG); } expr = erts_alloc(ERTS_ALC_T_RE_TMP_BUF, slen + 1); @@ -795,8 +795,8 @@ build_capture(Eterm capture_spec[CAPSPEC_SIZE], const pcre *code) memcpy(tmpb,ap->name,ap->len); tmpb[ap->len] = '\0'; } else { - int slen = io_list_len(val); - if (slen < 0) { + Uint slen; + if (erts_iolist_size(val, &slen)) { goto error; } if ((slen + 1) > tmpbsiz) { @@ -851,7 +851,7 @@ re_run_3(BIF_ALIST_3) const pcre *code_tmp; RestartContext restart; byte *temp_alloc = NULL; - int slength; + Uint slength; int startoffset = 0; int options = 0, comp_options = 0; int ovsize; @@ -875,7 +875,7 @@ re_run_3(BIF_ALIST_3) if (is_not_tuple(BIF_ARG_2) || (arityval(*tuple_val(BIF_ARG_2)) != 4)) { if (is_binary(BIF_ARG_2) || is_list(BIF_ARG_2) || is_nil(BIF_ARG_2)) { /* Compile from textual RE */ - int slen; + Uint slen; char *expr; pcre *result; int errcode = 0; @@ -889,7 +889,7 @@ re_run_3(BIF_ALIST_3) BIF_TRAP3(urun_trap_exportp, BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); } - if ((slen = io_list_len(BIF_ARG_2)) < 0) { + if (erts_iolist_size(BIF_ARG_2, &slen)) { BIF_ERROR(BIF_P,BADARG); } @@ -1027,7 +1027,7 @@ re_run_3(BIF_ALIST_3) restart.flags |= RESTART_FLAG_SUBJECT_IN_BINARY; } else { handle_iolist: - if ((slength = io_list_len(BIF_ARG_1)) < 0) { + if (erts_iolist_size(BIF_ARG_1, &slength)) { erts_free(ERTS_ALC_T_RE_SUBJECT, restart.ovector); erts_free(ERTS_ALC_T_RE_SUBJECT, restart.code); if (restart.ret_info != NULL) { diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 9ef990cc4f..fdc82c8b88 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -105,7 +105,7 @@ #define NSEG_2 256 /* Size of second segment table */ #define NSEG_INC 128 /* Number of segments to grow after that */ -#define SEGTAB(tb) ((struct segment**)erts_smp_atomic_read(&(tb)->segtab)) +#define SEGTAB(tb) ((struct segment**)erts_smp_atomic_read_acqb(&(tb)->segtab)) #define NACTIVE(tb) ((int)erts_smp_atomic_read(&(tb)->nactive)) #define NITEMS(tb) ((int)erts_smp_atomic_read(&(tb)->common.nitems)) @@ -122,8 +122,8 @@ */ static ERTS_INLINE Uint hash_to_ix(DbTableHash* tb, HashValue hval) { - Uint mask = erts_smp_atomic_read(&tb->szm); - Uint ix = hval & mask; + Uint mask = erts_smp_atomic_read_acqb(&tb->szm); + Uint ix = hval & mask; if (ix >= erts_smp_atomic_read(&tb->nactive)) { ix &= mask>>1; ASSERT(ix < erts_smp_atomic_read(&tb->nactive)); @@ -668,6 +668,7 @@ int db_create_hash(Process *p, DbTable *tbl) else { /* coarse locking */ tb->locks = NULL; } + ERTS_THR_MEMORY_BARRIER; #endif /* ERST_SMP */ return DB_ERROR_NONE; } @@ -2085,7 +2086,14 @@ static void db_print_hash(int to, void *to_arg, int show, DbTable *tbl) while(list != 0) { if (list->hvalue == INVALID_HASH) erts_print(to, to_arg, "*"); - erts_print(to, to_arg, "%T", make_tuple(list->dbterm.tpl)); + if (tb->common.compress) { + Eterm key = GETKEY(tb, list->dbterm.tpl); + erts_print(to, to_arg, "key=%R", key, list->dbterm.tpl); + } + else { + Eterm obj = make_tuple_rel(list->dbterm.tpl,list->dbterm.tpl); + erts_print(to, to_arg, "%R", obj, list->dbterm.tpl); + } if (list->next != 0) erts_print(to, to_arg, ","); list = list->next; @@ -2342,7 +2350,7 @@ static int alloc_seg(DbTableHash *tb) struct ext_segment* eseg; eseg = (struct ext_segment*) SEGTAB(tb)[seg_ix-1]; MY_ASSERT(eseg!=NULL && eseg->s.is_ext_segment); - erts_smp_atomic_set(&tb->segtab, (erts_aint_t) eseg->segtab); + erts_smp_atomic_set_relb(&tb->segtab, (erts_aint_t) eseg->segtab); tb->nsegs = eseg->nsegs; } ASSERT(seg_ix < tb->nsegs); @@ -2414,7 +2422,7 @@ static int free_seg(DbTableHash *tb, int free_records) MY_ASSERT(newtop->s.is_ext_segment); if (newtop->prev_segtab != NULL) { /* Time to use a smaller segtab */ - erts_smp_atomic_set(&tb->segtab, (erts_aint_t)newtop->prev_segtab); + erts_smp_atomic_set_relb(&tb->segtab, (erts_aint_t)newtop->prev_segtab); tb->nsegs = seg_ix; ASSERT(tb->nsegs == EXTSEG(SEGTAB(tb))->nsegs); } @@ -2431,7 +2439,7 @@ static int free_seg(DbTableHash *tb, int free_records) if (seg_ix > 0) { if (seg_ix < tb->nsegs) SEGTAB(tb)[seg_ix] = NULL; } else { - erts_smp_atomic_set(&tb->segtab, (erts_aint_t)NULL); + erts_smp_atomic_set_relb(&tb->segtab, (erts_aint_t)NULL); } #endif tb->nslots -= SEGSZ; @@ -2526,9 +2534,9 @@ static void grow(DbTableHash* tb, int nactive) } erts_smp_atomic_inc(&tb->nactive); if (from_ix == 0) { - erts_smp_atomic_set(&tb->szm, szm); + erts_smp_atomic_set_relb(&tb->szm, szm); } - erts_smp_atomic_set(&tb->is_resizing, 0); + erts_smp_atomic_set_relb(&tb->is_resizing, 0); /* Finally, let's split the bucket. We try to do it in a smart way to keep link order and avoid unnecessary updates of next-pointers */ @@ -2560,7 +2568,7 @@ static void grow(DbTableHash* tb, int nactive) return; abort: - erts_smp_atomic_set(&tb->is_resizing, 0); + erts_smp_atomic_set_relb(&tb->is_resizing, 0); } @@ -2604,7 +2612,7 @@ static void shrink(DbTableHash* tb, int nactive) erts_smp_atomic_set(&tb->nactive, src_ix); if (dst_ix == 0) { - erts_smp_atomic_set(&tb->szm, low_szm); + erts_smp_atomic_set_relb(&tb->szm, low_szm); } WUNLOCK_HASH(lck); @@ -2618,7 +2626,7 @@ static void shrink(DbTableHash* tb, int nactive) } /*else already done */ - erts_smp_atomic_set(&tb->is_resizing, 0); + erts_smp_atomic_set_relb(&tb->is_resizing, 0); } diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c index a59c0c258d..9a0ba3a418 100644 --- a/erts/emulator/beam/erl_db_tree.c +++ b/erts/emulator/beam/erl_db_tree.c @@ -111,7 +111,7 @@ static void release_stack(DbTableTree* tb, DbTreeStack* stack) { if (stack == &tb->static_stack) { ASSERT(erts_smp_atomic_read(&tb->is_stack_busy) == 1); - erts_smp_atomic_set(&tb->is_stack_busy, 0); + erts_smp_atomic_set_relb(&tb->is_stack_busy, 0); } else { erts_db_free(ERTS_ALC_T_DB_STK, (DbTable *) tb, @@ -179,7 +179,7 @@ static ERTS_INLINE TreeDbTerm* replace_dbterm(DbTableTree *tb, TreeDbTerm* old, static TreeDbTerm *traverse_until(TreeDbTerm *t, int *current, int to); static void check_slot_pos(DbTableTree *tb); static void check_saved_stack(DbTableTree *tb); -static int check_table_tree(TreeDbTerm *t); +static int check_table_tree(DbTableTree* tb, TreeDbTerm *t); #define TREE_DEBUG #endif @@ -194,8 +194,8 @@ static int check_table_tree(TreeDbTerm *t); ** Debugging dump */ -static void do_dump_tree2(int to, void *to_arg, int show, TreeDbTerm *t, - int offset); +static void do_dump_tree2(DbTableTree*, int to, void *to_arg, int show, + TreeDbTerm *t, int offset); #else @@ -1730,6 +1730,7 @@ static int db_select_delete_tree(Process *p, DbTable *tbl, ** Other interface routines (not directly coupled to one bif) */ + /* Display tree contents (for dump) */ static void db_print_tree(int to, void *to_arg, int show, @@ -1740,7 +1741,7 @@ static void db_print_tree(int to, void *to_arg, if (show) erts_print(to, to_arg, "\nTree data dump:\n" "------------------------------------------------\n"); - do_dump_tree2(to, to_arg, show, tb->root, 0); + do_dump_tree2(&tbl->tree, to, to_arg, show, tb->root, 0); if (show) erts_print(to, to_arg, "\n" "------------------------------------------------\n"); @@ -2694,7 +2695,7 @@ static Sint do_cmp_partly_bound(Eterm a, Eterm b, Eterm* b_base, int *done) while (1) { if ((j = do_cmp_partly_bound(*aa++, *bb++, b_base, done)) != 0 || *done) return j; - if (*aa==*bb) + if (is_same(*aa, NULL, *bb, b_base)) return 0; if (is_not_list(*aa) || is_not_list(*bb)) return do_cmp_partly_bound(*aa, *bb, b_base, done); @@ -2742,7 +2743,7 @@ static Sint cmp_partly_bound(Eterm partly_bound_key, Eterm bound_key, Eterm* bk_ erts_fprintf(stderr," > "); else erts_fprintf(stderr," == "); - erts_fprintf(stderr,"%T\n",bound_key); // HALFWORD BUG: printing rterm + erts_fprintf(stderr,"%R\n", bound_key, bk_base); #endif return ret; } @@ -3084,19 +3085,28 @@ static int doit_select_delete(DbTableTree *tb, TreeDbTerm *this, void *ptr, } #ifdef TREE_DEBUG -static void do_dump_tree2(int to, void *to_arg, int show, TreeDbTerm *t, - int offset) +static void do_dump_tree2(DbTableTree* tb, int to, void *to_arg, int show, + TreeDbTerm *t, int offset) { if (t == NULL) - return 0; - do_dump_tree2(to, to_arg, show, t->right, offset + 4); + return; + do_dump_tree2(tb, to, to_arg, show, t->right, offset + 4); if (show) { - erts_print(to, to_arg, "%*s%T (addr = %p, bal = %d)\n" - offset, "", make_tuple(t->dbterm.tpl), + const char* prefix; + Eterm term; + if (tb->common.compress) { + prefix = "key="; + term = GETKEY(tb, t->dbterm.tpl); + } + else { + prefix = ""; + term = make_tuple_rel(t->dbterm.tpl,t->dbterm.tpl); + } + erts_print(to, to_arg, "%*s%s%R (addr = %p, bal = %d)\n", + offset, "", prefix, term, t->dbterm.tpl, t, t->balance); } - do_dump_tree2(to, to_arg, show, t->left, offset + 4); - return sum; + do_dump_tree2(tb, to, to_arg, show, t->left, offset + 4); } #endif @@ -3106,7 +3116,7 @@ static void do_dump_tree2(int to, void *to_arg, int show, TreeDbTerm *t, void db_check_table_tree(DbTable *tbl) { DbTableTree *tb = &tbl->tree; - check_table_tree(tb->root); + check_table_tree(tb, tb->root); check_saved_stack(tb); check_slot_pos(tb); } @@ -3137,7 +3147,7 @@ static void check_slot_pos(DbTableTree *tb) "element position %d is really 0x%08X, when stack says " "it's 0x%08X\n", tb->stack.slot, t, tb->stack.array[tb->stack.pos - 1]); - do_dump_tree2(ERTS_PRINT_STDERR, NULL, 1, tb->root, 0); + do_dump_tree2(tb, ERTS_PRINT_STDERR, NULL, 1, tb->root, 0); } } @@ -3152,14 +3162,14 @@ static void check_saved_stack(DbTableTree *tb) if (t != stack->array[0]) { erts_fprintf(stderr,"tb->stack[0] is 0x%08X, should be 0x%08X\n", stack->array[0], t); - do_dump_tree2(ERTS_PRINT_STDERR, NULL, 1, tb->root, 0); + do_dump_tree2(tb, ERTS_PRINT_STDERR, NULL, 1, tb->root, 0); return; } while (n < stack->pos) { if (t == NULL) { erts_fprintf(stderr, "NULL pointer in tree when stack not empty," " stack depth is %d\n", n); - do_dump_tree2(ERTS_PRINT_STDERR, NULL, 1, tb->root, 0); + do_dump_tree2(tb, ERTS_PRINT_STDERR, NULL, 1, tb->root, 0); return; } n++; @@ -3173,28 +3183,26 @@ static void check_saved_stack(DbTableTree *tb) "represent child pointer in tree!" "(left == 0x%08X, right == 0x%08X\n", n, tb->stack[n], t->left, t->right); - do_dump_tree2(ERTS_PRINT_STDERR, NULL, 1, tb->root, 0); + do_dump_tree2(tb, ERTS_PRINT_STDERR, NULL, 1, tb->root, 0); return; } } } } -static int check_table_tree(TreeDbTerm *t) +static int check_table_tree(DbTableTree* tb, TreeDbTerm *t) { int lh, rh; if (t == NULL) return 0; - lh = check_table_tree(t->left); - rh = check_table_tree(t->right); + lh = check_table_tree(tb, t->left); + rh = check_table_tree(tb, t->right); if ((rh - lh) != t->balance) { erts_fprintf(stderr, "Invalid tree balance for this node:\n"); - erts_fprintf(stderr,"balance = %d, left = 0x%08X, right = 0x%08X\n" - "data = %T", - t->balance, t->left, t->right, - make_tuple(t->dbterm.tpl)); + erts_fprintf(stderr,"balance = %d, left = 0x%08X, right = 0x%08X\n", + t->balance, t->left, t->right); erts_fprintf(stderr,"\nDump:\n---------------------------------\n"); - do_dump_tree2(ERTS_PRINT_STDERR, NULL, 1, t, 0); + do_dump_tree2(tb, ERTS_PRINT_STDERR, NULL, 1, t, 0); erts_fprintf(stderr,"\n---------------------------------\n"); } return ((rh > lh) ? rh : lh) + 1; diff --git a/erts/emulator/beam/erl_drv_thread.c b/erts/emulator/beam/erl_drv_thread.c index 39bbe9633b..dc578f6d2a 100644 --- a/erts/emulator/beam/erl_drv_thread.c +++ b/erts/emulator/beam/erl_drv_thread.c @@ -700,6 +700,13 @@ erl_drv_thread_join(ErlDrvTid tid, void **respp) extern int erts_darwin_main_thread_pipe[2]; extern int erts_darwin_main_thread_result_pipe[2]; +int erl_drv_stolen_main_thread_join(ErlDrvTid tid, void **respp); +int erl_drv_steal_main_thread(char *name, + ErlDrvTid *dtid, + void* (*func)(void*), + void* arg, + ErlDrvThreadOpts *opts); + int erl_drv_stolen_main_thread_join(ErlDrvTid tid, void **respp) diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c index 9e18997890..9180508a49 100644 --- a/erts/emulator/beam/erl_lock_check.c +++ b/erts/emulator/beam/erl_lock_check.c @@ -240,7 +240,7 @@ typedef struct erts_lc_locked_lock_t_ erts_lc_locked_lock_t; struct erts_lc_locked_lock_t_ { erts_lc_locked_lock_t *next; erts_lc_locked_lock_t *prev; - Eterm extra; + UWord extra; Sint16 id; Uint16 flags; }; @@ -441,12 +441,12 @@ new_locked_lock(erts_lc_lock_t *lck, Uint16 op_flags) } static void -print_lock2(char *prefix, Sint16 id, Eterm extra, Uint16 flags, char *suffix) +print_lock2(char *prefix, Sint16 id, Wterm extra, Uint16 flags, char *suffix) { char *lname = (0 <= id && id < ERTS_LOCK_ORDER_SIZE ? erts_lock_order[id].name : "unknown"); - if (is_boxed(extra)) + if (is_not_immed(extra)) erts_fprintf(stderr, "%s'%s:%p%s'%s%s", prefix, @@ -1260,7 +1260,8 @@ erts_lc_init_lock(erts_lc_lock_t *lck, char *name, Uint16 flags) { lck->id = erts_lc_get_lock_order_id(name); - lck->extra = make_boxed(&lck->extra); + lck->extra = &lck->extra; + ASSERT(is_not_immed(lck->extra)); lck->flags = flags; lck->inited = ERTS_LC_INITITALIZED; } @@ -1270,6 +1271,7 @@ erts_lc_init_lock_x(erts_lc_lock_t *lck, char *name, Uint16 flags, Eterm extra) { lck->id = erts_lc_get_lock_order_id(name); lck->extra = extra; + ASSERT(is_immed(lck->extra)); lck->flags = flags; lck->inited = ERTS_LC_INITITALIZED; } diff --git a/erts/emulator/beam/erl_lock_check.h b/erts/emulator/beam/erl_lock_check.h index cdb06d4458..b67f36fa06 100644 --- a/erts/emulator/beam/erl_lock_check.h +++ b/erts/emulator/beam/erl_lock_check.h @@ -39,7 +39,7 @@ typedef struct { int inited; Sint16 id; Uint16 flags; - Eterm extra; + UWord extra; } erts_lc_lock_t; #define ERTS_LC_INITITALIZED 0x7f7f7f7f diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 8b48444904..68421b4387 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -472,7 +472,7 @@ static void tmp_alloc_dtor(struct enif_tmp_obj_t* obj) int enif_inspect_iolist_as_binary(ErlNifEnv* env, Eterm term, ErlNifBinary* bin) { struct enif_tmp_obj_t* tobj; - int sz; + Uint sz; if (is_binary(term)) { return enif_inspect_binary(env,term,bin); } @@ -483,7 +483,7 @@ int enif_inspect_iolist_as_binary(ErlNifEnv* env, Eterm term, ErlNifBinary* bin) bin->ref_bin = NULL; return 1; } - if ((sz = io_list_len(term)) < 0) { + if (erts_iolist_size(term, &sz)) { return 0; } diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c index 1b07024ca1..326021643f 100644 --- a/erts/emulator/beam/erl_port_task.c +++ b/erts/emulator/beam/erl_port_task.c @@ -658,8 +658,6 @@ erts_port_task_free_port(Port *pp) when scheduled out... */ ErtsPortTask *ptp = port_task_alloc(); erts_smp_port_state_lock(pp); - ASSERT(erts_smp_atomic_read(&erts_ports_alive) > 0); - erts_smp_atomic_dec(&erts_ports_alive); pp->status &= ~ERTS_PORT_SFLG_CLOSING; pp->status |= ERTS_PORT_SFLG_FREE_SCHEDULED; erts_may_save_closed_port(pp); @@ -681,7 +679,6 @@ erts_port_task_free_port(Port *pp) port_is_dequeued = 1; } erts_smp_port_state_lock(pp); - erts_smp_atomic_dec(&erts_ports_alive); pp->status &= ~ERTS_PORT_SFLG_CLOSING; pp->status |= ERTS_PORT_SFLG_FREE_SCHEDULED; erts_may_save_closed_port(pp); diff --git a/erts/emulator/beam/erl_printf_term.c b/erts/emulator/beam/erl_printf_term.c index b71404fd27..34da9cab84 100644 --- a/erts/emulator/beam/erl_printf_term.c +++ b/erts/emulator/beam/erl_printf_term.c @@ -114,13 +114,13 @@ do { \ /* return 0 if list is not a non-empty flat list of printable characters */ static int -is_printable_string(Eterm list) +is_printable_string(Eterm list, Eterm* base) { int len = 0; int c; while(is_list(list)) { - Eterm* consp = list_val(list); + Eterm* consp = list_val_rel(list, base); Eterm hd = CAR(consp); if (!is_byte(hd)) @@ -226,17 +226,20 @@ static int print_atom_name(fmtfn_t fn, void* arg, Eterm atom, long *dcount) #define PRT_LAST_ARRAY_ELEMENT ((Eterm) 7) /* Note! Must be last... */ static int -print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount) +print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount, + Eterm* obj_base) /* ignored if !HALFWORD_HEAP */ { DECLARE_WSTACK(s); int res; int i; Eterm val; Uint32 *ref_num; + union { + UWord word; + Eterm* ptr; + }popped; Eterm* nobj; -#if HALFWORD_HEAP - UWord wobj; -#endif + Wterm wobj; res = 0; @@ -258,18 +261,17 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount) PRINT_CHAR(res, fn, arg, '}'); goto L_outer_loop; default: -#if HALFWORD_HEAP - obj = (Eterm) (wobj = WSTACK_POP(s)); -#else - obj = WSTACK_POP(s); -#endif + popped.word = WSTACK_POP(s); + switch (val) { case PRT_TERM: + obj = (Eterm) popped.word; break; case PRT_ONE_CONS: + obj = (Eterm) popped.word; L_print_one_cons: { - Eterm* cons = list_val(obj); + Eterm* cons = list_val_rel(obj, obj_base); Eterm tl; obj = CAR(cons); @@ -288,27 +290,13 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount) } break; case PRT_LAST_ARRAY_ELEMENT: - { -#if HALFWORD_HEAP - Eterm* ptr = (Eterm *) wobj; -#else - Eterm* ptr = (Eterm *) obj; -#endif - obj = *ptr; - } + obj = *popped.ptr; break; default: /* PRT_LAST_ARRAY_ELEMENT+1 and upwards */ - { -#if HALFWORD_HEAP - Eterm* ptr = (Eterm *) wobj; -#else - Eterm* ptr = (Eterm *) obj; -#endif - obj = *ptr++; - WSTACK_PUSH(s, (UWord) ptr); - WSTACK_PUSH(s, val-1); - WSTACK_PUSH(s, PRT_COMMA); - } + obj = *popped.ptr; + WSTACK_PUSH(s, (UWord) (popped.ptr + 1)); + WSTACK_PUSH(s, val-1); + WSTACK_PUSH(s, PRT_COMMA); break; } break; @@ -325,8 +313,12 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount) PRINT_CHAR(res, fn, arg, '>'); goto L_done; } - - switch (tag_val_def(obj)) { +#if HALFWORD_HEAP + wobj = is_immed(obj) ? (Wterm)obj : rterm2wterm(obj, obj_base); +#else + wobj = (Wterm)obj; +#endif + switch (tag_val_def(wobj)) { case NIL_DEF: PRINT_STRING(res, fn, arg, "[]"); break; @@ -348,13 +340,13 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount) int print_res; char def_buf[64]; char *buf, *big_str; - Uint sz = (Uint) big_decimal_estimate(obj); + Uint sz = (Uint) big_decimal_estimate(wobj); sz++; if (sz <= 64) buf = &def_buf[0]; else buf = erts_alloc(ERTS_ALC_T_TMP, sz); - big_str = erts_big_to_string(obj, buf, sz); + big_str = erts_big_to_string(wobj, buf, sz); print_res = erts_printf_string(fn, arg, big_str); if (buf != &def_buf[0]) erts_free(ERTS_ALC_T_TMP, (void *) buf); @@ -369,9 +361,9 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount) case EXTERNAL_REF_DEF: PRINT_STRING(res, fn, arg, "#Ref<"); PRINT_ULONG(res, fn, arg, 'u', 0, 1, - (unsigned long) ref_channel_no(obj)); - ref_num = ref_numbers(obj); - for (i = ref_no_of_numbers(obj)-1; i >= 0; i--) { + (unsigned long) ref_channel_no(wobj)); + ref_num = ref_numbers(wobj); + for (i = ref_no_of_numbers(wobj)-1; i >= 0; i--) { PRINT_CHAR(res, fn, arg, '.'); PRINT_ULONG(res, fn, arg, 'u', 0, 1, (unsigned long) ref_num[i]); } @@ -381,30 +373,30 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount) case EXTERNAL_PID_DEF: PRINT_CHAR(res, fn, arg, '<'); PRINT_ULONG(res, fn, arg, 'u', 0, 1, - (unsigned long) pid_channel_no(obj)); + (unsigned long) pid_channel_no(wobj)); PRINT_CHAR(res, fn, arg, '.'); PRINT_ULONG(res, fn, arg, 'u', 0, 1, - (unsigned long) pid_number(obj)); + (unsigned long) pid_number(wobj)); PRINT_CHAR(res, fn, arg, '.'); PRINT_ULONG(res, fn, arg, 'u', 0, 1, - (unsigned long) pid_serial(obj)); + (unsigned long) pid_serial(wobj)); PRINT_CHAR(res, fn, arg, '>'); break; case PORT_DEF: case EXTERNAL_PORT_DEF: PRINT_STRING(res, fn, arg, "#Port<"); PRINT_ULONG(res, fn, arg, 'u', 0, 1, - (unsigned long) port_channel_no(obj)); + (unsigned long) port_channel_no(wobj)); PRINT_CHAR(res, fn, arg, '.'); PRINT_ULONG(res, fn, arg, 'u', 0, 1, - (unsigned long) port_number(obj)); + (unsigned long) port_number(wobj)); PRINT_CHAR(res, fn, arg, '>'); break; case LIST_DEF: - if (is_printable_string(obj)) { + if (is_printable_string(obj, obj_base)) { int c; PRINT_CHAR(res, fn, arg, '"'); - nobj = list_val(obj); + nobj = list_val_rel(obj, obj_base); while (1) { if ((*dcount)-- <= 0) goto L_done; @@ -418,7 +410,7 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount) } if (is_not_list(*nobj)) break; - nobj = list_val(*nobj); + nobj = list_val_rel(*nobj, obj_base); } PRINT_CHAR(res, fn, arg, '"'); } else { @@ -428,7 +420,7 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount) } break; case TUPLE_DEF: - nobj = tuple_val(obj); /* pointer to arity */ + nobj = tuple_val(wobj); /* pointer to arity */ i = arityval(*nobj); /* arity */ PRINT_CHAR(res, fn, arg, '{'); WSTACK_PUSH(s,PRT_CLOSE_TUPLE); @@ -440,13 +432,13 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount) break; case FLOAT_DEF: { FloatDef ff; - GET_DOUBLE(obj, ff); + GET_DOUBLE(wobj, ff); PRINT_DOUBLE(res, fn, arg, 'e', 6, 0, ff.fd); } break; case BINARY_DEF: { - ProcBin* pb = (ProcBin *) binary_val(obj); + ProcBin* pb = (ProcBin *) binary_val(wobj); if (pb->size == 1) PRINT_STRING(res, fn, arg, "<<1 byte>>"); else { @@ -458,7 +450,7 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount) break; case EXPORT_DEF: { - Export* ep = *((Export **) (export_val(obj) + 1)); + Export* ep = *((Export **) (export_val(wobj) + 1)); Atom* module = atom_tab(atom_val(ep->code[0])); Atom* name = atom_tab(atom_val(ep->code[1])); @@ -474,7 +466,7 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount) break; case FUN_DEF: { - ErlFunThing *funp = (ErlFunThing *) fun_val(obj); + ErlFunThing *funp = (ErlFunThing *) fun_val(wobj); Atom *ap = atom_tab(atom_val(funp->fe->module)); PRINT_STRING(res, fn, arg, "#Fun<"); @@ -490,7 +482,7 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount) break; default: PRINT_STRING(res, fn, arg, "<unknown:"); - PRINT_POINTER(res, fn, arg, (UWord) obj); + PRINT_POINTER(res, fn, arg, wobj); PRINT_CHAR(res, fn, arg, '>'); break; } @@ -503,9 +495,10 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount) } int -erts_printf_term(fmtfn_t fn, void* arg, unsigned long term, long precision) +erts_printf_term(fmtfn_t fn, void* arg, unsigned long term, long precision, + unsigned long* term_base) { - int res = print_term(fn, arg, (Uint) term, &precision); + int res = print_term(fn, arg, (Eterm)term, &precision, (Eterm*)term_base); if (res < 0) return res; if (precision <= 0) diff --git a/erts/emulator/beam/erl_printf_term.h b/erts/emulator/beam/erl_printf_term.h index 4f76028396..4ba22f12de 100644 --- a/erts/emulator/beam/erl_printf_term.h +++ b/erts/emulator/beam/erl_printf_term.h @@ -21,6 +21,6 @@ #define ERL_PRINTF_TERM_H__ #include "erl_printf_format.h" -int erts_printf_term(fmtfn_t fn, void* arg, unsigned long term, long precision); - +int erts_printf_term(fmtfn_t fn, void* arg, unsigned long term, long precision, + unsigned long* term_base); #endif diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 8a56976905..2704359a8f 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -166,9 +166,8 @@ static struct { static struct { erts_smp_mtx_t update_mtx; - erts_smp_atomic32_t active_runqs; + erts_smp_atomic32_t no_runqs; int last_active_runqs; - erts_smp_atomic32_t used_runqs; int forced_check_balance; erts_smp_atomic32_t checking_balance; int halftime; @@ -965,7 +964,7 @@ sched_spin_wait(ErtsSchedulerSleepInfo *ssi, int spincount) erts_aint32_t flgs; do { - flgs = erts_smp_atomic32_read(&ssi->flags); + flgs = erts_smp_atomic32_read_acqb(&ssi->flags); if ((flgs & (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING)) != (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING)) { break; @@ -1114,7 +1113,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) else { erts_aint_t dt; - erts_smp_atomic32_set(&function_calls, 0); + erts_smp_atomic32_set_relb(&function_calls, 0); *fcalls = 0; sched_waiting_sys(esdp->no, rq); @@ -1147,7 +1146,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) nonblockable_aux_work(esdp, ssi, aux_work); #endif - flgs = erts_smp_atomic32_read(&ssi->flags); + flgs = erts_smp_atomic32_read_acqb(&ssi->flags); if (!(flgs & ERTS_SSI_FLG_WAITING)) { ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); goto sys_woken; @@ -1333,6 +1332,76 @@ wake_all_schedulers(void) } } +#define ERTS_NO_USED_RUNQS_SHIFT 16 +#define ERTS_NO_RUNQS_MASK 0xffff + +#if ERTS_MAX_NO_OF_SCHEDULERS > ERTS_NO_RUNQS_MASK +# error "Too large amount of schedulers allowed" +#endif + +static ERTS_INLINE void +init_no_runqs(int active, int used) +{ + erts_aint32_t no_runqs = (erts_aint32_t) (active & ERTS_NO_RUNQS_MASK); + no_runqs |= (erts_aint32_t) ((used & ERTS_NO_RUNQS_MASK) << ERTS_NO_USED_RUNQS_SHIFT); + erts_smp_atomic32_init(&balance_info.no_runqs, no_runqs); +} + +static ERTS_INLINE void +get_no_runqs(int *active, int *used) +{ + erts_aint32_t no_runqs = erts_smp_atomic32_read(&balance_info.no_runqs); + if (active) + *active = (int) (no_runqs & ERTS_NO_RUNQS_MASK); + if (used) + *used = (int) ((no_runqs >> ERTS_NO_USED_RUNQS_SHIFT) & ERTS_NO_RUNQS_MASK); +} + +static ERTS_INLINE void +set_no_used_runqs(int used) +{ + erts_aint32_t exp = erts_smp_atomic32_read(&balance_info.no_runqs); + while (1) { + erts_aint32_t act, new; + new = (used << ERTS_NO_USED_RUNQS_SHIFT) | (exp & ERTS_NO_RUNQS_MASK); + act = erts_smp_atomic32_cmpxchg(&balance_info.no_runqs, new, exp); + if (act == exp) + break; + exp = act; + } +} + +static ERTS_INLINE void +set_no_active_runqs(int active) +{ + erts_aint32_t exp = erts_smp_atomic32_read(&balance_info.no_runqs); + while (1) { + erts_aint32_t act, new; + new = (exp & (ERTS_NO_RUNQS_MASK << ERTS_NO_USED_RUNQS_SHIFT)) | active; + act = erts_smp_atomic32_cmpxchg(&balance_info.no_runqs, new, exp); + if (act == exp) + break; + exp = act; + } +} + +static ERTS_INLINE int +try_inc_no_active_runqs(int active) +{ + erts_aint32_t exp = erts_smp_atomic32_read(&balance_info.no_runqs); + if (((exp >> ERTS_NO_USED_RUNQS_SHIFT) & ERTS_NO_RUNQS_MASK) < active) + return 0; + if ((exp & ERTS_NO_RUNQS_MASK) + 1 == active) { + erts_aint32_t new, act; + new = (exp & ~ERTS_NO_RUNQS_MASK) | active; + act = erts_smp_atomic32_cmpxchg(&balance_info.no_runqs, new, exp); + if (act == exp) + return 1; + } + return 0; +} + + static ERTS_INLINE int chk_wake_sched(ErtsRunQueue *crq, int ix, int activate) { @@ -1344,9 +1413,7 @@ chk_wake_sched(ErtsRunQueue *crq, int ix, int activate) iflgs = erts_smp_atomic32_read(&wrq->info_flags); if (!(iflgs & (ERTS_RUNQ_IFLG_SUSPENDED|ERTS_RUNQ_IFLG_NONEMPTY))) { if (activate) { - if (ix == erts_smp_atomic32_cmpxchg(&balance_info.active_runqs, - ix+1, - ix)) { + if (try_inc_no_active_runqs(ix+1)) { erts_smp_xrunq_lock(crq, wrq); wrq->flags &= ~ERTS_RUNQ_FLG_INACTIVE; erts_smp_xrunq_unlock(crq, wrq); @@ -1363,8 +1430,9 @@ wake_scheduler_on_empty_runq(ErtsRunQueue *crq) { int ix = crq->ix; int stop_ix = ix; - int active_ix = erts_smp_atomic32_read(&balance_info.active_runqs); - int balance_ix = erts_smp_atomic32_read(&balance_info.used_runqs); + int active_ix, balance_ix; + + get_no_runqs(&active_ix, &balance_ix); if (active_ix > balance_ix) active_ix = balance_ix; @@ -1416,7 +1484,7 @@ erts_sched_notify_check_cpu_bind(void) int ix; if (erts_common_run_queue) { for (ix = 0; ix < erts_no_schedulers; ix++) - erts_smp_atomic32_set(&ERTS_SCHEDULER_IX(ix)->chk_cpu_bind, 1); + erts_smp_atomic32_set_relb(&ERTS_SCHEDULER_IX(ix)->chk_cpu_bind, 1); wake_all_schedulers(); } else { @@ -1871,8 +1939,7 @@ try_steal_task(ErtsRunQueue *rq) ERTS_SMP_LC_CHK_RUNQ_LOCK(rq, rq_locked); - active_rqs = erts_smp_atomic32_read(&balance_info.active_runqs); - blnc_rqs = erts_smp_atomic32_read(&balance_info.used_runqs); + get_no_runqs(&active_rqs, &blnc_rqs); if (active_rqs > blnc_rqs) active_rqs = blnc_rqs; @@ -1883,7 +1950,7 @@ try_steal_task(ErtsRunQueue *rq) if (active_rqs < blnc_rqs) { int no = blnc_rqs - active_rqs; int stop_ix = vix = active_rqs + rq->ix % no; - while (erts_smp_atomic32_read(&no_empty_run_queues) < blnc_rqs) { + while (erts_smp_atomic32_read_acqb(&no_empty_run_queues) < blnc_rqs) { res = check_possible_steal_victim(rq, &rq_locked, vix); if (res) goto done; @@ -1898,7 +1965,7 @@ try_steal_task(ErtsRunQueue *rq) vix = rq->ix; /* ... then try to steal a job from another active queue... */ - while (erts_smp_atomic32_read(&no_empty_run_queues) < blnc_rqs) { + while (erts_smp_atomic32_read_acqb(&no_empty_run_queues) < blnc_rqs) { vix++; if (vix >= active_rqs) vix = 0; @@ -1999,7 +2066,7 @@ check_balance(ErtsRunQueue *c_rq) return; } - blnc_no_rqs = (int) erts_smp_atomic32_read(&balance_info.used_runqs); + get_no_runqs(NULL, &blnc_no_rqs); if (blnc_no_rqs == 1) { c_rq->check_balance_reds = INT_MAX; erts_smp_atomic32_set(&balance_info.checking_balance, 0); @@ -2038,7 +2105,8 @@ check_balance(ErtsRunQueue *c_rq) forced = balance_info.forced_check_balance; balance_info.forced_check_balance = 0; - blnc_no_rqs = (int) erts_smp_atomic32_read(&balance_info.used_runqs); + get_no_runqs(¤t_active, &blnc_no_rqs); + if (blnc_no_rqs == 1) { erts_smp_mtx_unlock(&balance_info.update_mtx); erts_smp_runq_lock(c_rq); @@ -2052,8 +2120,6 @@ check_balance(ErtsRunQueue *c_rq) if (balance_info.full_reds_history_index >= ERTS_FULL_REDS_HISTORY_SIZE) balance_info.full_reds_history_index = 0; - current_active = erts_smp_atomic32_read(&balance_info.active_runqs); - /* Read balance information for all run queues */ for (qix = 0; qix < blnc_no_rqs; qix++) { ErtsRunQueue *rq = ERTS_RUNQ_IX(qix); @@ -2387,7 +2453,7 @@ erts_fprintf(stderr, "--------------------------------\n"); } balance_info.last_active_runqs = active; - erts_smp_atomic32_set(&balance_info.active_runqs, active); + set_no_active_runqs(active); balance_info.halftime = 1; erts_smp_atomic32_set(&balance_info.checking_balance, 0); @@ -2695,9 +2761,8 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) erts_smp_atomic32_init(&schdlr_sspnd.msb.ongoing, 0); erts_smp_atomic32_init(&schdlr_sspnd.active, no_schedulers); schdlr_sspnd.msb.procs = NULL; - erts_smp_atomic32_set(&balance_info.used_runqs, - erts_common_run_queue ? 1 : no_schedulers_online); - erts_smp_atomic32_init(&balance_info.active_runqs, no_schedulers); + init_no_runqs(no_schedulers, + erts_common_run_queue ? 1 : no_schedulers_online); balance_info.last_active_runqs = no_schedulers; erts_smp_mtx_init(&balance_info.update_mtx, "migration_info_update"); balance_info.forced_check_balance = 0; @@ -2939,7 +3004,7 @@ sched_spin_suspended(ErtsSchedulerSleepInfo *ssi, int spincount) erts_aint32_t flgs; do { - flgs = erts_smp_atomic32_read(&ssi->flags); + flgs = erts_smp_atomic32_read_acqb(&ssi->flags); if ((flgs & (ERTS_SSI_FLG_SLEEPING | ERTS_SSI_FLG_WAITING | ERTS_SSI_FLG_SUSPENDED)) @@ -3068,7 +3133,7 @@ suspend_scheduler(ErtsSchedulerData *esdp) wake = 0; } - flgs = erts_smp_atomic32_read(&ssi->flags); + flgs = erts_smp_atomic32_read_acqb(&ssi->flags); if (!(flgs & ERTS_SSI_FLG_SUSPENDED)) break; erts_smp_mtx_unlock(&schdlr_sspnd.mtx); @@ -3292,7 +3357,7 @@ erts_set_schedulers_online(Process *p, ErtsRunQueue *to_rq = ERTS_RUNQ_IX(ix % no); evacuate_run_queue(from_rq, to_rq); } - erts_smp_atomic32_set(&balance_info.used_runqs, no); + set_no_used_runqs(no); erts_smp_mtx_unlock(&balance_info.update_mtx); erts_smp_mtx_lock(&schdlr_sspnd.mtx); } @@ -3346,7 +3411,7 @@ erts_set_schedulers_online(Process *p, for (ix = erts_no_run_queues-1; ix >= no; ix--) evacuate_run_queue(ERTS_RUNQ_IX(ix), ERTS_RUNQ_IX(ix % no)); - erts_smp_atomic32_set(&balance_info.used_runqs, no); + set_no_used_runqs(no); erts_smp_mtx_unlock(&balance_info.update_mtx); erts_smp_mtx_lock(&schdlr_sspnd.mtx); for (ix = no; ix < online; ix++) { @@ -3443,7 +3508,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) else { erts_smp_mtx_unlock(&schdlr_sspnd.mtx); erts_smp_mtx_lock(&balance_info.update_mtx); - erts_smp_atomic32_set(&balance_info.used_runqs, 1); + set_no_used_runqs(1); for (ix = 0; ix < online; ix++) { ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); erts_smp_runq_lock(rq); @@ -3580,7 +3645,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) evacuate_run_queue(ERTS_RUNQ_IX(ix), ERTS_RUNQ_IX(ix % online)); - erts_smp_atomic32_set(&balance_info.used_runqs, online); + set_no_used_runqs(online); /* Make sure that we balance soon... */ balance_info.forced_check_balance = 1; erts_smp_runq_lock(ERTS_RUNQ_IX(0)); @@ -5131,7 +5196,7 @@ Process *schedule(Process *p, int calls) esdp = erts_get_scheduler_data(); rq = erts_get_runq_current(esdp); ASSERT(esdp); - fcalls = (int) erts_smp_atomic32_read(&function_calls); + fcalls = (int) erts_smp_atomic32_read_acqb(&function_calls); actual_reds = reds = 0; erts_smp_runq_lock(rq); } else { @@ -5282,14 +5347,14 @@ Process *schedule(Process *p, int calls) | ERTS_RUNQ_FLG_CHK_CPU_BIND | ERTS_RUNQ_FLG_SUSPENDED)) { if ((rq->flags & ERTS_RUNQ_FLG_SUSPENDED) - || (erts_smp_atomic32_read(&esdp->ssi->flags) + || (erts_smp_atomic32_read_acqb(&esdp->ssi->flags) & ERTS_SSI_FLG_SUSPENDED)) { ASSERT(erts_smp_atomic32_read(&esdp->ssi->flags) & ERTS_SSI_FLG_SUSPENDED); suspend_scheduler(esdp); } if ((rq->flags & ERTS_RUNQ_FLG_CHK_CPU_BIND) - || erts_smp_atomic32_read(&esdp->chk_cpu_bind)) { + || erts_smp_atomic32_read_acqb(&esdp->chk_cpu_bind)) { erts_sched_check_cpu_bind(esdp); } } @@ -5340,7 +5405,7 @@ Process *schedule(Process *p, int calls) if (rq->flags & (ERTS_RUNQ_FLG_SHARED_RUNQ | ERTS_RUNQ_FLG_SUSPENDED)) { if ((rq->flags & ERTS_RUNQ_FLG_SUSPENDED) - || (erts_smp_atomic32_read(&esdp->ssi->flags) + || (erts_smp_atomic32_read_acqb(&esdp->ssi->flags) & ERTS_SSI_FLG_SUSPENDED)) { ASSERT(erts_smp_atomic32_read(&esdp->ssi->flags) & ERTS_SSI_FLG_SUSPENDED); @@ -5384,7 +5449,7 @@ Process *schedule(Process *p, int calls) * Schedule system-level activities. */ - erts_smp_atomic32_set(&function_calls, 0); + erts_smp_atomic32_set_relb(&function_calls, 0); fcalls = 0; ASSERT(!erts_port_task_have_outstanding_io_tasks()); @@ -5426,7 +5491,7 @@ Process *schedule(Process *p, int calls) if (erts_common_run_queue->waiting) wake_scheduler(erts_common_run_queue, 0, 1); } - else if (erts_smp_atomic32_read(&no_empty_run_queues) != 0) { + else if (erts_smp_atomic32_read_acqb(&no_empty_run_queues) != 0) { wake_scheduler_on_empty_runq(rq); rq->wakeup_other = 0; } @@ -7637,7 +7702,8 @@ continue_exit_process(Process *p static void timeout_proc(Process* p) { - p->i = *((BeamInstr **) (UWord) p->def_arg_reg); + BeamInstr** pi = (BeamInstr **) p->def_arg_reg; + p->i = *pi; p->flags |= F_TIMO; p->flags &= ~F_INSLPQUEUE; diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index c0397ca6c3..8833137112 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -1538,8 +1538,7 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, Eterm tracee; #endif Eterm transformed_args[MAX_ARG]; - DeclareTmpHeap(sub_bin_heap_et,ERL_SUB_BIN_SIZE,p); - ErlSubBin *sub_bin_heap = (ErlSubBin *) sub_bin_heap_et; + DeclareTypedTmpHeap(ErlSubBin,sub_bin_heap,p); ASSERT(tracer_pid); if (*tracer_pid == am_true) { @@ -1600,21 +1599,20 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, if (is_boxed(arg) && header_is_bin_matchstate(*boxed_val(arg))) { ErlBinMatchState* ms = (ErlBinMatchState *) boxed_val(arg); ErlBinMatchBuffer* mb = &ms->mb; - ErlSubBin* sb = sub_bin_heap; Uint bit_size; ASSERT(sub_bin_heap->thing_word == 0); /* At most one of match context */ bit_size = mb->size - mb->offset; - sb->thing_word = HEADER_SUB_BIN; - sb->size = BYTE_OFFSET(bit_size); - sb->bitsize = BIT_OFFSET(bit_size); - sb->offs = BYTE_OFFSET(mb->offset); - sb->bitoffs = BIT_OFFSET(mb->offset); - sb->is_writable = 0; - sb->orig = mb->orig; - - arg = make_binary(sb); + sub_bin_heap->thing_word = HEADER_SUB_BIN; + sub_bin_heap->size = BYTE_OFFSET(bit_size); + sub_bin_heap->bitsize = BIT_OFFSET(bit_size); + sub_bin_heap->offs = BYTE_OFFSET(mb->offset); + sub_bin_heap->bitoffs = BIT_OFFSET(mb->offset); + sub_bin_heap->is_writable = 0; + sub_bin_heap->orig = mb->orig; + + arg = make_binary(sub_bin_heap); } transformed_args[i] = arg; } diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 96da894d90..499bdd77ba 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -183,7 +183,7 @@ struct port { process to get (line oriented I/O)*/ Uint32 status; /* Status and type flags */ int control_flags; /* Flags for port_control() */ - Uint32 snapshot; /* Next snapshot that port should be part of */ + erts_aint32_t snapshot; /* Next snapshot that port should be part of */ struct reg_proc *reg; ErlDrvPDL port_data_lock; @@ -527,11 +527,10 @@ union erl_off_heap_ptr { /* arrays that get malloced at startup */ extern Port* erts_port; -extern erts_smp_atomic_t erts_ports_alive; extern Uint erts_max_ports; extern Uint erts_port_tab_index_mask; -extern erts_smp_atomic_t erts_ports_snapshot; +extern erts_smp_atomic32_t erts_ports_snapshot; extern erts_smp_atomic_t erts_dead_ports_ptr; ERTS_GLB_INLINE void erts_may_save_closed_port(Port *prt); @@ -541,12 +540,12 @@ ERTS_GLB_INLINE void erts_may_save_closed_port(Port *prt); ERTS_GLB_INLINE void erts_may_save_closed_port(Port *prt) { ERTS_SMP_LC_ASSERT(erts_smp_lc_spinlock_is_locked(&prt->state_lck)); - if (prt->snapshot != erts_smp_atomic_read(&erts_ports_snapshot)) { + if (prt->snapshot != erts_smp_atomic32_read_acqb(&erts_ports_snapshot)) { /* Dead ports are added from the end of the snapshot buffer */ Eterm* tombstone = (Eterm*) erts_smp_atomic_addtest(&erts_dead_ports_ptr, -(erts_aint_t)sizeof(Eterm)); ASSERT(tombstone+1 != NULL); - ASSERT(prt->snapshot == (Uint32) erts_smp_atomic_read(&erts_ports_snapshot) - 1); + ASSERT(prt->snapshot == erts_smp_atomic32_read(&erts_ports_snapshot) - 1); *tombstone = prt->id; } /*else no ongoing snapshot or port was already included or created after snapshot */ @@ -1653,10 +1652,14 @@ struct Sint_buf { }; char* Sint_to_buf(Sint, struct Sint_buf*); +#define ERTS_IOLIST_OK 0 +#define ERTS_IOLIST_OVERFLOW 1 +#define ERTS_IOLIST_TYPE 2 + Eterm buf_to_intlist(Eterm**, char*, int, Eterm); /* most callers pass plain char*'s */ int io_list_to_buf(Eterm, char*, int); int io_list_to_buf2(Eterm, char*, int); -int io_list_len(Eterm); +int erts_iolist_size(Eterm, Uint *); int is_string(Eterm); void erl_at_exit(void (*) (void*), void*); Eterm collect_memory(Process *); @@ -1890,6 +1893,8 @@ erts_alloc_message_heap(Uint size, # if defined(DEBUG) # define DeclareTmpHeap(VariableName,Size,Process) \ Eterm *VariableName = erts_debug_allocate_tmp_heap(Size,Process) +# define DeclareTypedTmpHeap(Type,VariableName,Process) \ + Type *VariableName = (Type *) erts_debug_allocate_tmp_heap(sizeof(Type)/sizeof(Eterm),Process) # define DeclareTmpHeapNoproc(VariableName,Size) \ Eterm *VariableName = erts_debug_allocate_tmp_heap(Size,NULL) # define UseTmpHeap(Size,Proc) \ @@ -1911,6 +1916,8 @@ erts_alloc_message_heap(Uint size, # else # define DeclareTmpHeap(VariableName,Size,Process) \ Eterm *VariableName = (ERTS_PROC_GET_SCHDATA(Process)->tmp_heap)+(ERTS_PROC_GET_SCHDATA(Process)->num_tmp_heap_used) +# define DeclareTypedTmpHeap(Type,VariableName,Process) \ + Type *VariableName = (Type *) (ERTS_PROC_GET_SCHDATA(Process)->tmp_heap)+(ERTS_PROC_GET_SCHDATA(Process)->num_tmp_heap_used) # define DeclareTmpHeapNoproc(VariableName,Size) \ Eterm *VariableName = (erts_get_scheduler_data()->tmp_heap)+(erts_get_scheduler_data()->num_tmp_heap_used) # define UseTmpHeap(Size,Proc) \ @@ -1936,6 +1943,8 @@ erts_alloc_message_heap(Uint size, #else # define DeclareTmpHeap(VariableName,Size,Process) \ Eterm VariableName[Size] +# define DeclareTypedTmpHeap(Type,VariableName,Process) \ + Type VariableName[1] # define DeclareTmpHeapNoproc(VariableName,Size) \ Eterm VariableName[Size] # define UseTmpHeap(Size,Proc) /* Nothing */ diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index f619c6f88b..df5f8b22a3 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -56,7 +56,6 @@ static erts_smp_tsd_key_t driver_list_last_error_key; /* Save last DDLL error o per thread basis (for BC interfaces) */ Port* erts_port; /* The port table */ -erts_smp_atomic_t erts_ports_alive; erts_smp_atomic_t erts_bytes_out; /* No bytes sent out of the system */ erts_smp_atomic_t erts_bytes_in; /* No bytes gotten into the system */ @@ -82,6 +81,9 @@ static void driver_monitor_unlock_pdl(Port *p); #define DRV_MONITOR_UNLOCK_PDL(Port) /* nothing */ #endif +#define ERL_SMALL_IO_BIN_LIMIT (4*ERL_ONHEAP_BIN_LIMIT) +#define SMALL_WRITE_VEC 16 + static ERTS_INLINE ErlIOQueue* drvport2ioq(ErlDrvPort drvport) { @@ -190,7 +192,7 @@ typedef struct line_buf_context { static erts_smp_spinlock_t get_free_port_lck; static Uint last_port_num; static Uint port_num_mask; -erts_smp_atomic_t erts_ports_snapshot; /* Identifies the _next_ snapshot (not the ongoing) */ +erts_smp_atomic32_t erts_ports_snapshot; /* Identifies the _next_ snapshot (not the ongoing) */ static ERTS_INLINE void @@ -421,10 +423,9 @@ setup_port(Port* prt, Eterm pid, erts_driver_t *driver, new_name = (char*) erts_alloc(ERTS_ALC_T_PORT_NAME, sys_strlen(name)+1); sys_strcpy(new_name, name); erts_smp_runq_lock(runq); - erts_smp_atomic_inc(&erts_ports_alive); erts_smp_port_state_lock(prt); prt->status = ERTS_PORT_SFLG_CONNECTED | xstatus; - prt->snapshot = (Uint32) erts_smp_atomic_read(&erts_ports_snapshot); + prt->snapshot = erts_smp_atomic32_read(&erts_ports_snapshot); old_name = prt->name; prt->name = new_name; #ifdef ERTS_SMP @@ -954,13 +955,14 @@ do { \ int _bitoffs; \ int _bitsize; \ ERTS_GET_REAL_BIN(obj, _real, _offset, _bitoffs, _bitsize); \ - ASSERT(_bitsize == 0); \ + if (_bitsize != 0) goto L_type_error; \ if (thing_subtag(*binary_val(_real)) == REFC_BINARY_SUBTAG && \ _bitoffs == 0) { \ b_size += _size; \ + if (b_size < _size) goto L_overflow_error; \ in_clist = 0; \ v_size++; \ - if (_size >= bin_limit) { \ + if (_size >= ERL_SMALL_IO_BIN_LIMIT) { \ p_in_clist = 0; \ p_v_size++; \ } else { \ @@ -972,6 +974,7 @@ do { \ } \ } else { \ c_size += _size; \ + if (c_size < _size) goto L_overflow_error; \ if (!in_clist) { \ in_clist = 1; \ v_size++; \ @@ -986,29 +989,30 @@ do { \ /* -** Size of a io list in bytes -** return -1 if error -** returns: - Total size of io list -** vsize - SysIOVec size needed for a writev -** csize - Number of bytes not in binary (in the common binary) -** pvsize - SysIOVec size needed if packing small binaries -** pcsize - Number of bytes in the common binary if packing -*/ + * Returns 0 if successful and a non-zero value otherwise. + * + * Return values through pointers: + * *vsize - SysIOVec size needed for a writev + * *csize - Number of bytes not in binary (in the common binary) + * *pvsize - SysIOVec size needed if packing small binaries + * *pcsize - Number of bytes in the common binary if packing + * *total_size - Total size of iolist in bytes + */ static int -io_list_vec_len(Eterm obj, int* vsize, int* csize, - int bin_limit, /* small binaries limit */ - int * pvsize, int * pcsize) +io_list_vec_len(Eterm obj, Uint* vsize, Uint* csize, + Uint* pvsize, Uint* pcsize, Uint* total_size) { DECLARE_ESTACK(s); Eterm* objp; - int v_size = 0; - int c_size = 0; - int b_size = 0; - int in_clist = 0; - int p_v_size = 0; - int p_c_size = 0; - int p_in_clist = 0; + Uint v_size = 0; + Uint c_size = 0; + Uint b_size = 0; + Uint in_clist = 0; + Uint p_v_size = 0; + Uint p_c_size = 0; + Uint p_in_clist = 0; + Uint total; goto L_jump_start; /* avoid a push */ @@ -1022,6 +1026,9 @@ io_list_vec_len(Eterm obj, int* vsize, int* csize, if (is_byte(obj)) { c_size++; + if (c_size == 0) { + goto L_overflow_error; + } if (!in_clist) { in_clist = 1; v_size++; @@ -1061,32 +1068,31 @@ io_list_vec_len(Eterm obj, int* vsize, int* csize, } } + total = c_size + b_size; + if (total < c_size) { + goto L_overflow_error; + } + *total_size = total; + DESTROY_ESTACK(s); - if (vsize != NULL) - *vsize = v_size; - if (csize != NULL) - *csize = c_size; - if (pvsize != NULL) - *pvsize = p_v_size; - if (pcsize != NULL) - *pcsize = p_c_size; - return c_size + b_size; + *vsize = v_size; + *csize = c_size; + *pvsize = p_v_size; + *pcsize = p_c_size; + return 0; L_type_error: + L_overflow_error: DESTROY_ESTACK(s); - return -1; + return 1; } -#define ERL_SMALL_IO_BIN_LIMIT (4*ERL_ONHEAP_BIN_LIMIT) -#define SMALL_WRITE_VEC 16 - - /* write data to a port */ int erts_write_to_port(Eterm caller_id, Port *p, Eterm list) { char *buf; erts_driver_t *drv = p->drv_ptr; - int size; + Uint size; int fpe_was_unmasked; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(p)); @@ -1094,10 +1100,10 @@ int erts_write_to_port(Eterm caller_id, Port *p, Eterm list) p->caller = caller_id; if (drv->outputv != NULL) { - int vsize; - int csize; - int pvsize; - int pcsize; + Uint vsize; + Uint csize; + Uint pvsize; + Uint pcsize; int blimit; SysIOVec iv[SMALL_WRITE_VEC]; ErlDrvBinary* bv[SMALL_WRITE_VEC]; @@ -1106,9 +1112,8 @@ int erts_write_to_port(Eterm caller_id, Port *p, Eterm list) ErlDrvBinary* cbin; ErlIOVec ev; - if ((size = io_list_vec_len(list, &vsize, &csize, - ERL_SMALL_IO_BIN_LIMIT, - &pvsize, &pcsize)) < 0) { + if (io_list_vec_len(list, &vsize, &csize, + &pvsize, &pcsize, &size)) { goto bad_value; } /* To pack or not to pack (small binaries) ...? */ @@ -1183,7 +1188,7 @@ int erts_write_to_port(Eterm caller_id, Port *p, Eterm list) else { ASSERT(r == -1); /* Overflow */ erts_free(ERTS_ALC_T_TMP, buf); - if ((size = io_list_len(list)) < 0) { + if (erts_iolist_size(list, &size)) { goto bad_value; } @@ -1274,7 +1279,6 @@ void init_io(void) erts_smp_atomic_init(&erts_bytes_out, 0); erts_smp_atomic_init(&erts_bytes_in, 0); - erts_smp_atomic_init(&erts_ports_alive, 0); for (i = 0; i < erts_max_ports; i++) { erts_port_task_init_sched(&erts_port[i].sched); @@ -1296,7 +1300,7 @@ void init_io(void) erts_port[i].port_data_lock = NULL; } - erts_smp_atomic_init(&erts_ports_snapshot, (erts_aint_t) 0); + erts_smp_atomic32_init(&erts_ports_snapshot, (erts_aint32_t) 0); last_port_num = 0; erts_smp_spinlock_init(&get_free_port_lck, "get_free_port"); @@ -2147,7 +2151,7 @@ erts_port_control(Process* p, Port* prt, Uint command, Eterm iolist) byte* to_port = NULL; /* Buffer to write to port. */ /* Initialization is for shutting up warning about use before set. */ - int to_len = 0; /* Length of buffer. */ + Uint to_len = 0; /* Length of buffer. */ int must_free = 0; /* True if the buffer should be freed. */ char port_result[ERL_ONHEAP_BIN_LIMIT]; /* Default buffer for result from port. */ char* port_resp; /* Pointer to result buffer. */ @@ -2192,7 +2196,7 @@ erts_port_control(Process* p, Port* prt, Uint command, Eterm iolist) } else { ASSERT(r == -1); /* Overflow */ erts_free(ERTS_ALC_T_TMP, (void *) to_port); - if ((to_len = io_list_len(iolist)) < 0) { /* Type error */ + if (erts_iolist_size(iolist, &to_len)) { /* Type error */ return THE_NON_VALUE; } must_free = 1; diff --git a/erts/emulator/beam/safe_hash.c b/erts/emulator/beam/safe_hash.c index 21d6ce9304..3e9243c77d 100644 --- a/erts/emulator/beam/safe_hash.c +++ b/erts/emulator/beam/safe_hash.c @@ -99,7 +99,7 @@ static void rehash(SafeHash* h, int grow_limit) erts_free(h->type, (void *) old_tab); } /*else already done */ - erts_smp_atomic_set(&h->is_rehashing, 0); + erts_smp_atomic_set_relb(&h->is_rehashing, 0); } diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index 6b4f3b3b36..a17de717bc 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -3021,13 +3021,25 @@ int io_list_to_buf(Eterm obj, char* buf, int len) return -1; } -int io_list_len(Eterm obj) +/* + * Return 0 if successful, and non-zero if unsuccessful. + */ +int erts_iolist_size(Eterm obj, Uint* sizep) { Eterm* objp; - Sint len = 0; + Uint size = 0; DECLARE_ESTACK(s); goto L_again; +#define SAFE_ADD(Var, Val) \ + do { \ + Uint valvar = (Val); \ + Var += valvar; \ + if (Var < valvar) { \ + goto L_overflow_error; \ + } \ + } while (0) + while (!ESTACK_ISEMPTY(s)) { obj = ESTACK_POP(s); L_again: @@ -3037,9 +3049,12 @@ int io_list_len(Eterm obj) /* Head */ obj = CAR(objp); if (is_byte(obj)) { - len++; + size++; + if (size == 0) { + goto L_overflow_error; + } } else if (is_binary(obj) && binary_bitsize(obj) == 0) { - len += binary_size(obj); + SAFE_ADD(size, binary_size(obj)); } else if (is_list(obj)) { ESTACK_PUSH(s, CDR(objp)); goto L_iter_list; /* on head */ @@ -3051,23 +3066,29 @@ int io_list_len(Eterm obj) if (is_list(obj)) goto L_iter_list; /* on tail */ else if (is_binary(obj) && binary_bitsize(obj) == 0) { - len += binary_size(obj); + SAFE_ADD(size, binary_size(obj)); } else if (is_not_nil(obj)) { goto L_type_error; } } else if (is_binary(obj) && binary_bitsize(obj) == 0) { /* Tail was binary */ - len += binary_size(obj); + SAFE_ADD(size, binary_size(obj)); } else if (is_not_nil(obj)) { goto L_type_error; } } +#undef SAFE_ADD DESTROY_ESTACK(s); - return len; + *sizep = size; + return ERTS_IOLIST_OK; + + L_overflow_error: + DESTROY_ESTACK(s); + return ERTS_IOLIST_OVERFLOW; L_type_error: DESTROY_ESTACK(s); - return -1; + return ERTS_IOLIST_TYPE; } /* return 0 if item is not a non-empty flat list of bytes */ diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c index aec456151c..40c4a0df08 100644 --- a/erts/emulator/drivers/common/inet_drv.c +++ b/erts/emulator/drivers/common/inet_drv.c @@ -1948,7 +1948,7 @@ static int http_response_inetdrv(void *arg, int major, int minor, tcp_descriptor* desc = (tcp_descriptor*) arg; int i = 0; ErlDrvTermData spec[27]; - ErlDrvTermData caller; + ErlDrvTermData caller = ERL_DRV_NIL; if (desc->inet.active == INET_PASSIVE) { /* {inet_async,S,Ref,{ok,{http_response,Version,Status,Phrase}}} */ @@ -2041,7 +2041,7 @@ http_request_inetdrv(void* arg, const http_atom_t* meth, const char* meth_ptr, tcp_descriptor* desc = (tcp_descriptor*) arg; int i = 0; ErlDrvTermData spec[43]; - ErlDrvTermData caller; + ErlDrvTermData caller = ERL_DRV_NIL; if (desc->inet.active == INET_PASSIVE) { /* {inet_async, S, Ref, {ok,{http_request,Meth,Uri,Version}}} */ @@ -2092,7 +2092,7 @@ http_header_inetdrv(void* arg, const http_atom_t* name, const char* name_ptr, tcp_descriptor* desc = (tcp_descriptor*) arg; int i = 0; ErlDrvTermData spec[26]; - ErlDrvTermData caller; + ErlDrvTermData caller = ERL_DRV_NIL; if (desc->inet.active == INET_PASSIVE) { /* {inet_async,S,Ref,{ok,{http_header,Bit,Name,IValue,Value}} */ @@ -2221,7 +2221,7 @@ int ssl_tls_inetdrv(void* arg, unsigned type, unsigned major, unsigned minor, tcp_descriptor* desc = (tcp_descriptor*) arg; int i = 0; ErlDrvTermData spec[28]; - ErlDrvTermData caller; + ErlDrvTermData caller = ERL_DRV_NIL; ErlDrvBinary* bin; int ret; @@ -7980,11 +7980,11 @@ static int tcp_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf, int len, timeout = get_int32(buf); if (desc->inet.state == TCP_STATE_ACCEPTING) { - unsigned long time_left; - int oid; - ErlDrvTermData ocaller; - int oreq; - unsigned otimeout; + unsigned long time_left = 0; + int oid = 0; + ErlDrvTermData ocaller = ERL_DRV_NIL; + int oreq = 0; + unsigned otimeout = 0; ErlDrvTermData caller = driver_caller(desc->inet.port); MultiTimerData *mtd = NULL,*omtd = NULL; ErlDrvMonitor monitor, omonitor; diff --git a/erts/emulator/hipe/hipe_bif2.c b/erts/emulator/hipe/hipe_bif2.c index 19cd32c68f..2660f74a82 100644 --- a/erts/emulator/hipe/hipe_bif2.c +++ b/erts/emulator/hipe/hipe_bif2.c @@ -1,9 +1,8 @@ /* * %CopyrightBegin% - - * + * * Copyright Ericsson AB 2001-2011. All Rights Reserved. - * + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the @@ -37,14 +36,25 @@ #include "hipe_arch.h" #include "hipe_stack.h" -BIF_RETTYPE hipe_bifs_show_estack_1(BIF_ALIST_1) +static void proc_unlock(Process* c_p, Process* rp) { + ErtsProcLocks locks = ERTS_PROC_LOCKS_ALL; + if (rp == c_p) { + locks &= ~ERTS_PROC_LOCK_MAIN; + } + if (rp && locks) { + erts_smp_proc_unlock(rp, locks); + } +} + +BIF_RETTYPE hipe_bifs_show_estack_1(BIF_ALIST_1) +{ Process *rp = erts_pid2proc(BIF_P, ERTS_PROC_LOCK_MAIN, BIF_ARG_1, ERTS_PROC_LOCKS_ALL); if (!rp) BIF_ERROR(BIF_P, BADARG); hipe_print_estack(rp); - erts_smp_proc_unlock(rp, ERTS_PROC_LOCKS_ALL); + proc_unlock(BIF_P, rp); BIF_RET(am_true); } @@ -55,7 +65,7 @@ BIF_RETTYPE hipe_bifs_show_heap_1(BIF_ALIST_1) if (!rp) BIF_ERROR(BIF_P, BADARG); hipe_print_heap(rp); - erts_smp_proc_unlock(rp, ERTS_PROC_LOCKS_ALL); + proc_unlock(BIF_P, rp); BIF_RET(am_true); } @@ -66,7 +76,7 @@ BIF_RETTYPE hipe_bifs_show_nstack_1(BIF_ALIST_1) if (!rp) BIF_ERROR(BIF_P, BADARG); hipe_print_nstack(rp); - erts_smp_proc_unlock(rp, ERTS_PROC_LOCKS_ALL); + proc_unlock(BIF_P, rp); BIF_RET(am_true); } @@ -82,7 +92,7 @@ BIF_RETTYPE hipe_bifs_show_pcb_1(BIF_ALIST_1) if (!rp) BIF_ERROR(BIF_P, BADARG); hipe_print_pcb(rp); - erts_smp_proc_unlock(rp, ERTS_PROC_LOCKS_ALL); + proc_unlock(BIF_P, rp); BIF_RET(am_true); } diff --git a/erts/emulator/hipe/hipe_mkliterals.c b/erts/emulator/hipe/hipe_mkliterals.c index 650861b54b..bced90785d 100644 --- a/erts/emulator/hipe/hipe_mkliterals.c +++ b/erts/emulator/hipe/hipe_mkliterals.c @@ -587,9 +587,9 @@ static void print_params(FILE *fp, void (*print_param)(FILE*,const struct rts_pa (*print_param)(fp, &rts_params[i]); } -static int do_c(FILE *fp) +static int do_c(FILE *fp, const char* this_exe) { - fprintf(fp, "/* File: hipe_literals.h, generated by hipe_mkliterals */\n"); + fprintf(fp, "/* File: hipe_literals.h, generated by %s */\n", this_exe); fprintf(fp, "#ifndef __HIPE_LITERALS_H__\n"); fprintf(fp, "#define __HIPE_LITERALS_H__\n\n"); print_literals(fp, c_define_literal); @@ -603,9 +603,9 @@ static int do_c(FILE *fp) return 0; } -static int do_e(FILE *fp) +static int do_e(FILE *fp, const char* this_exe) { - fprintf(fp, "%%%% File: hipe_literals.hrl, generated by hipe_mkliterals"); + fprintf(fp, "%%%% File: hipe_literals.hrl, generated by %s", this_exe); fprintf(fp, "\n\n"); print_literals(fp, e_define_literal); fprintf(fp, "\n"); @@ -622,9 +622,9 @@ int main(int argc, const char **argv) compute_crc(); if (argc == 2) { if (strcmp(argv[1], "-c") == 0) - return do_c(stdout); + return do_c(stdout, argv[0]); if (strcmp(argv[1], "-e") == 0) - return do_e(stdout); + return do_e(stdout, argv[0]); } fprintf(stderr, "usage: %s [-c | -e] > output-file\n", argv[0]); return 1; diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c index 218bd79584..71b374527e 100644 --- a/erts/emulator/sys/common/erl_check_io.c +++ b/erts/emulator/sys/common/erl_check_io.c @@ -1137,6 +1137,11 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait) restart: +#ifdef ERTS_BREAK_REQUESTED + if (ERTS_BREAK_REQUESTED) + erts_do_break_handling(); +#endif + /* Figure out timeout value */ if (do_wait) { erts_time_remaining(&wait_time); diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c index ffa3a6328c..eaef6680dd 100644 --- a/erts/emulator/sys/common/erl_mseg.c +++ b/erts/emulator/sys/common/erl_mseg.c @@ -134,7 +134,16 @@ static int mmap_fd; #define CAN_PARTLY_DESTROY 0 #endif -static const ErtsMsegOpt_t default_opt = ERTS_MSEG_DEFAULT_OPT_INITIALIZER; +const ErtsMsegOpt_t erts_mseg_default_opt = { + 1, /* Use cache */ + 1, /* Preserv data */ + 0, /* Absolute shrink threshold */ + 0 /* Relative shrink threshold */ +#if HALFWORD_HEAP + ,0 /* need low memory */ +#endif +}; + typedef struct cache_desc_t_ { void *seg; @@ -605,18 +614,10 @@ mseg_clear_cache(MemKind* mk) INC_CC(clear_cache); } -static ERTS_INLINE MemKind* type2mk(ErtsAlcType_t atype) +static ERTS_INLINE MemKind* memkind(const ErtsMsegOpt_t *opt) { #if HALFWORD_HEAP - switch (atype) { - case ERTS_ALC_A_ETS: - case ERTS_ALC_A_BINARY: - case ERTS_ALC_A_FIXED_SIZE: - case ERTS_ALC_A_DRIVER: - return &hi_mem; - default: - return &low_mem; - } + return opt->low_mem ? &low_mem : &hi_mem; #else return &the_mem; #endif @@ -628,7 +629,7 @@ mseg_alloc(ErtsAlcType_t atype, Uint *size_p, const ErtsMsegOpt_t *opt) Uint max, min, diff_size, size; cache_desc_t *cd, *cand_cd; void *seg; - MemKind* mk = type2mk(atype); + MemKind* mk = memkind(opt); INC_CC(alloc); @@ -742,7 +743,7 @@ static void mseg_dealloc(ErtsAlcType_t atype, void *seg, Uint size, const ErtsMsegOpt_t *opt) { - MemKind* mk = type2mk(atype); + MemKind* mk = memkind(opt); cache_desc_t *cd; ERTS_MSEG_DEALLOC_STAT(mk,size); @@ -800,7 +801,7 @@ static void * mseg_realloc(ErtsAlcType_t atype, void *seg, Uint old_size, Uint *new_size_p, const ErtsMsegOpt_t *opt) { - MemKind* mk = type2mk(atype); + MemKind* mk = memkind(opt); void *new_seg; Uint new_size; @@ -1372,7 +1373,7 @@ erts_mseg_alloc_opt(ErtsAlcType_t atype, Uint *size_p, const ErtsMsegOpt_t *opt) void * erts_mseg_alloc(ErtsAlcType_t atype, Uint *size_p) { - return erts_mseg_alloc_opt(atype, size_p, &default_opt); + return erts_mseg_alloc_opt(atype, size_p, &erts_mseg_default_opt); } void @@ -1387,7 +1388,7 @@ erts_mseg_dealloc_opt(ErtsAlcType_t atype, void *seg, Uint size, void erts_mseg_dealloc(ErtsAlcType_t atype, void *seg, Uint size) { - erts_mseg_dealloc_opt(atype, seg, size, &default_opt); + erts_mseg_dealloc_opt(atype, seg, size, &erts_mseg_default_opt); } void * @@ -1405,7 +1406,7 @@ void * erts_mseg_realloc(ErtsAlcType_t atype, void *seg, Uint old_size, Uint *new_size_p) { - return erts_mseg_realloc_opt(atype, seg, old_size, new_size_p, &default_opt); + return erts_mseg_realloc_opt(atype, seg, old_size, new_size_p, &erts_mseg_default_opt); } void diff --git a/erts/emulator/sys/common/erl_mseg.h b/erts/emulator/sys/common/erl_mseg.h index d8053eb0d9..fbb66ee33b 100644 --- a/erts/emulator/sys/common/erl_mseg.h +++ b/erts/emulator/sys/common/erl_mseg.h @@ -60,15 +60,12 @@ typedef struct { int preserv; UWord abs_shrink_th; UWord rel_shrink_th; +#if HALFWORD_HEAP + int low_mem; +#endif } ErtsMsegOpt_t; -#define ERTS_MSEG_DEFAULT_OPT_INITIALIZER \ -{ \ - 1, /* Use cache */ \ - 1, /* Preserv data */ \ - 0, /* Absolute shrink threshold */ \ - 0 /* Relative shrink threshold */ \ -} +extern const ErtsMsegOpt_t erts_mseg_default_opt; void *erts_mseg_alloc(ErtsAlcType_t, Uint *); void *erts_mseg_alloc_opt(ErtsAlcType_t, Uint *, const ErtsMsegOpt_t *); diff --git a/erts/emulator/sys/common/erl_poll.c b/erts/emulator/sys/common/erl_poll.c index 3ae5b8d747..f5c785d683 100644 --- a/erts/emulator/sys/common/erl_poll.c +++ b/erts/emulator/sys/common/erl_poll.c @@ -766,7 +766,7 @@ write_batch_buf(ErtsPollSet ps, ErtsPollBatchBuf *bbp) short filter; int fd = (int) ebuf[i].ident; - switch ((int) ebuf[i].udata) { + switch ((int) (long) ebuf[i].udata) { /* * Since we use a lazy update approach EV_DELETE will @@ -805,7 +805,7 @@ write_batch_buf(ErtsPollSet ps, ErtsPollBatchBuf *bbp) if (fd == (int) ebuf[j].ident) { ebuf[j].udata = (void *) ERTS_POLL_KQ_OP_HANDLED; if (!(ebuf[j].flags & EV_ERROR)) { - switch ((int) ebuf[j].udata) { + switch ((int) (long) ebuf[j].udata) { case ERTS_POLL_KQ_OP_ADD2_W: filter = EVFILT_WRITE; goto rm_add_fb; @@ -823,7 +823,8 @@ write_batch_buf(ErtsPollSet ps, ErtsPollBatchBuf *bbp) } } /* The other add succeded... */ - filter = (((int) ebuf[i].udata == ERTS_POLL_KQ_OP_ADD2_W) + filter = ((((int) (long) ebuf[i].udata) + == ERTS_POLL_KQ_OP_ADD2_W) ? EVFILT_READ : EVFILT_WRITE); rm_add_fb: diff --git a/erts/emulator/sys/win32/erl_poll.c b/erts/emulator/sys/win32/erl_poll.c index 7662f190ef..074e2e247f 100644 --- a/erts/emulator/sys/win32/erl_poll.c +++ b/erts/emulator/sys/win32/erl_poll.c @@ -693,6 +693,7 @@ static void *break_waiter(void *param) ResetEvent(harr[0]); erts_mtx_lock(&break_waiter_lock); erts_atomic32_set(&break_waiter_state,BREAK_WAITER_GOT_BREAK); + ERTS_THR_MEMORY_BARRIER; SetEvent(break_happened_event); erts_mtx_unlock(&break_waiter_lock); break; @@ -700,6 +701,7 @@ static void *break_waiter(void *param) ResetEvent(harr[1]); erts_mtx_lock(&break_waiter_lock); erts_atomic32_set(&break_waiter_state,BREAK_WAITER_GOT_HALT); + ERTS_THR_MEMORY_BARRIER; SetEvent(break_happened_event); erts_mtx_unlock(&break_waiter_lock); break; diff --git a/erts/emulator/test/binary_SUITE.erl b/erts/emulator/test/binary_SUITE.erl index 7e409f053e..4e82381fba 100644 --- a/erts/emulator/test/binary_SUITE.erl +++ b/erts/emulator/test/binary_SUITE.erl @@ -23,12 +23,12 @@ %% Tests binaries and the BIFs: %% list_to_binary/1 %% iolist_to_binary/1 -%% bitstr_to_list/1 +%% list_to_bitstring/1 %% binary_to_list/1 %% binary_to_list/3 %% binary_to_term/1 %% binary_to_term/2 -%% bitstr_to_list/1 +%% bitstring_to_list/1 %% term_to_binary/1 %% erlang:external_size/1 %% size(Binary) @@ -275,12 +275,33 @@ bad_list_to_binary(Config) when is_list(Config) -> ?line test_bad_bin(fun(X, Y) -> X*Y end), ?line test_bad_bin([1,fun(X) -> X + 1 end,2|fun() -> 0 end]), ?line test_bad_bin([fun(X) -> X + 1 end]), + + %% Test iolists that do not fit in the address space. + %% Unfortunately, it would be too slow to test in a 64-bit emulator. + case erlang:system_info(wordsize) of + 4 -> huge_iolists(); + _ -> ok + end. + +huge_iolists() -> + FourGigs = 1 bsl 32, + ?line Sizes = [FourGigs+N || N <- lists:seq(0, 64)] ++ + [1 bsl N || N <- lists:seq(33, 37)], + ?line Base = <<0:(1 bsl 20)/unit:8>>, + [begin + L = build_iolist(Sz, Base), + ?line {'EXIT',{system_limit,_}} = (catch list_to_binary([L])), + ?line {'EXIT',{system_limit,_}} = (catch list_to_bitstring([L])), + ?line {'EXIT',{system_limit,_}} = (catch binary:list_to_bin([L])), + ?line {'EXIT',{system_limit,_}} = (catch iolist_to_binary(L)) + end || Sz <- Sizes], ok. test_bad_bin(List) -> {'EXIT',{badarg,_}} = (catch list_to_binary(List)), {'EXIT',{badarg,_}} = (catch iolist_to_binary(List)), - {'EXIT',{badarg,_}} = (catch list_to_bitstring(List)). + {'EXIT',{badarg,_}} = (catch list_to_bitstring(List)), + {'EXIT',{badarg,_}} = (catch iolist_size(List)). bad_binary_to_list(doc) -> "Tries binary_to_list/1,3 with bad arguments."; bad_binary_to_list(Config) when is_list(Config) -> @@ -516,18 +537,65 @@ external_size_1(Term, Size0, Limit) when Size0 < Limit -> external_size_1(_, _, _) -> ok. t_iolist_size(Config) when is_list(Config) -> - %% Build a term whose external size only fits in a big num (on 32-bit CPU). - Bin = iolist_to_binary(lists:seq(0, 254)), - ?line ok = t_iolist_size_1(Bin, 0, 16#7FFFFFFF), - ?line ok = t_iolist_size_1(make_unaligned_sub_binary(Bin), 0, 16#7FFFFFFF). + ?line Seed = now(), + ?line io:format("Seed: ~p", [Seed]), + ?line random:seed(Seed), + ?line Base = <<0:(1 bsl 20)/unit:8>>, + ?line Powers = [1 bsl N || N <- lists:seq(2, 37)], + ?line Sizes0 = [[N - random:uniform(N div 2), + lists:seq(N-2, N+2), + N+N div 2, + N + random:uniform(N div 2)] || + N <- Powers], + %% Test sizes around 1^32 more thoroughly. + FourGigs = 1 bsl 32, + ?line Sizes1 = [FourGigs+N || N <- lists:seq(-8, 40)] ++ Sizes0, + ?line Sizes2 = lists:flatten(Sizes1), + ?line Sizes = lists:usort(Sizes2), + io:format("~p sizes:", [length(Sizes)]), + io:format("~p\n", [Sizes]), + ?line [Sz = iolist_size(build_iolist(Sz, Base)) || Sz <- Sizes], + ok. -t_iolist_size_1(IOList, Size0, Limit) when Size0 < Limit -> - case iolist_size(IOList) of - Size when is_integer(Size), Size0 < Size -> - io:format("~p", [Size]), - t_iolist_size_1([IOList|IOList], Size, Limit) +build_iolist(N, Base) when N < 16 -> + case random:uniform(3) of + 1 -> + <<Bin:N/binary,_/binary>> = Base, + Bin; + _ -> + lists:seq(1, N) + end; +build_iolist(N, Base) when N =< byte_size(Base) -> + case random:uniform(3) of + 1 -> + <<Bin:N/binary,_/binary>> = Base, + Bin; + 2 -> + <<Bin:N/binary,_/binary>> = Base, + [Bin]; + 3 -> + case N rem 2 of + 0 -> + L = build_iolist(N div 2, Base), + [L,L]; + 1 -> + L = build_iolist(N div 2, Base), + [L,L,45] + end end; -t_iolist_size_1(_, _, _) -> ok. +build_iolist(N0, Base) -> + Small = random:uniform(15), + Seq = lists:seq(1, Small), + N = N0 - Small, + case N rem 2 of + 0 -> + L = build_iolist(N div 2, Base), + [L,L|Seq]; + 1 -> + L = build_iolist(N div 2, Base), + [47,L,L|Seq] + end. + bad_binary_to_term_2(doc) -> "OTP-4053."; bad_binary_to_term_2(suite) -> []; @@ -1183,34 +1251,7 @@ deep(Config) when is_list(Config) -> deep_roundtrip(T) -> B = term_to_binary(T), - true = deep_eq(T, binary_to_term(B)). - -%% -%% FIXME: =:= runs out of stack. -%% -deep_eq([H1|T1], [H2|T2]) -> - deep_eq(H1, H2) andalso deep_eq(T1, T2); -deep_eq(T1, T2) when tuple_size(T1) =:= tuple_size(T2) -> - deep_eq_tup(T1, T2, tuple_size(T1)); -deep_eq(T1, T2) when is_function(T1), is_function(T2) -> - {uniq,U1} = erlang:fun_info(T1, uniq), - {index,I1} = erlang:fun_info(T1, index), - {arity,A1} = erlang:fun_info(T1, arity), - {env,E1} = erlang:fun_info(T1, env), - {uniq,U2} = erlang:fun_info(T2, uniq), - {index,I2} = erlang:fun_info(T2, index), - {arity,A2} = erlang:fun_info(T2, arity), - {env,E2} = erlang:fun_info(T2, env), - U1 =:= U2 andalso I1 =:= I2 andalso A1 =:= A2 andalso - deep_eq(E1, E2); -deep_eq(T1, T2) -> - T1 =:= T2. - -deep_eq_tup(_T1, _T2, 0) -> - true; -deep_eq_tup(T1, T2, N) -> - deep_eq(element(N, T1), element(N, T2)) andalso - deep_eq_tup(T1, T2, N-1). + T = binary_to_term(B). obsolete_funs(Config) when is_list(Config) -> erts_debug:set_internal_state(available_internal_state, true), diff --git a/erts/emulator/test/code_SUITE.erl b/erts/emulator/test/code_SUITE.erl index c1a048be75..a062cea117 100644 --- a/erts/emulator/test/code_SUITE.erl +++ b/erts/emulator/test/code_SUITE.erl @@ -498,7 +498,9 @@ do_false_dependency(Init, Code) -> ?line unlink(Pid), exit(Pid, kill), ?line true = erlang:purge_module(cpbugx), ?line true = erlang:delete_module(cpbugx), + ?line code:is_module_native(cpbugx), % test is_module_native on deleted code ?line true = erlang:purge_module(cpbugx), + ?line code:is_module_native(cpbugx), % test is_module_native on purged code ok. false_dependency_loop(Parent, Init, SendInitAck) -> diff --git a/erts/emulator/test/driver_SUITE.erl b/erts/emulator/test/driver_SUITE.erl index 7600a44988..f6cf01ce16 100644 --- a/erts/emulator/test/driver_SUITE.erl +++ b/erts/emulator/test/driver_SUITE.erl @@ -38,7 +38,7 @@ timer_change/1, timer_delay/1, queue_echo/1, - fun_to_port/1, + outputv_errors/1, driver_unloaded/1, io_ready_exit/1, use_fallback_pollset/1, @@ -74,7 +74,8 @@ missing_callbacks/1, smp_select/1, driver_select_use/1, - thread_mseg_alloc_cache_clean/1]). + thread_mseg_alloc_cache_clean/1, + otp_9302/1]). -export([bin_prefix/2]). @@ -129,7 +130,7 @@ end_per_testcase(Case, Config) -> suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> - [fun_to_port, outputv_echo, queue_echo, {group, timer}, + [outputv_errors, outputv_echo, queue_echo, {group, timer}, driver_unloaded, io_ready_exit, use_fallback_pollset, bad_fd_in_pollset, driver_event, fd_change, steal_control, otp_6602, 'driver_system_info_ver1.0', @@ -141,7 +142,8 @@ all() -> smaller_minor_vsn_drv, peek_non_existing_queue, otp_6879, caller, many_events, missing_callbacks, smp_select, driver_select_use, - thread_mseg_alloc_cache_clean]. + thread_mseg_alloc_cache_clean, + otp_9302]. groups() -> [{timer, [], @@ -165,37 +167,89 @@ init_per_group(_GroupName, Config) -> end_per_group(_GroupName, Config) -> Config. - -fun_to_port(doc) -> "Test sending a fun to port with an outputv-capable driver."; -fun_to_port(Config) when is_list(Config) -> +outputv_errors(doc) -> "Test sending bad types to port with an outputv-capable driver."; +outputv_errors(Config) when is_list(Config) -> ?line Path = ?config(data_dir, Config), ?line erl_ddll:start(), ?line ok = load_driver(Path, outputv_drv), - ?line fun_to_port_1(fun() -> 33 end), - ?line fun_to_port_1([fun() -> 42 end]), - ?line fun_to_port_1([1|fun() -> 42 end]), - L = build_io_list(65536), - ?line fun_to_port_1([L,fun() -> 42 end]), - ?line fun_to_port_1([L|fun() -> 42 end]), + outputv_bad_types(fun(T) -> + ?line outputv_errors_1(T), + ?line outputv_errors_1([1|T]), + ?line L = [1,2,3], + ?line outputv_errors_1([L,T]), + ?line outputv_errors_1([L|T]) + end), + outputv_errors_1(42), + + %% Test iolists that do not fit in the address space. + %% Unfortunately, it would be too slow to test in a 64-bit emulator. + case erlang:system_info(wordsize) of + 4 -> outputv_huge_iolists(); + _ -> ok + end. + +outputv_bad_types(Test) -> + Types = [-1,256,atom,42.0,{a,b,c},make_ref(),fun() -> 42 end, + [1|2],<<1:1>>,<<1:9>>,<<1:15>>], + _ = [Test(Type) || Type <- Types], + ok. + +outputv_huge_iolists() -> + FourGigs = 1 bsl 32, + ?line Sizes = [FourGigs+N || N <- lists:seq(0, 64)] ++ + [1 bsl N || N <- lists:seq(33, 37)], + ?line Base = <<0:(1 bsl 20)/unit:8>>, + [begin + ?line L = build_iolist(Sz, Base), + ?line outputv_errors_1(L) + end || Sz <- Sizes], ok. -fun_to_port_1(Term) -> - Port = open_port({spawn,outputv_drv}, []), +outputv_errors_1(Term) -> + Port = open_port({spawn_driver,outputv_drv}, []), {'EXIT',{badarg,_}} = (catch port_command(Port, Term)), port_close(Port). -build_io_list(0) -> []; -build_io_list(1) -> [7]; -build_io_list(N) -> - L = build_io_list(N div 2), +build_iolist(N, Base) when N < 16 -> + case random:uniform(3) of + 1 -> + <<Bin:N/binary,_/binary>> = Base, + Bin; + _ -> + lists:seq(1, N) + end; +build_iolist(N, Base) when N =< byte_size(Base) -> + case random:uniform(3) of + 1 -> + <<Bin:N/binary,_/binary>> = Base, + Bin; + 2 -> + <<Bin:N/binary,_/binary>> = Base, + [Bin]; + 3 -> + case N rem 2 of + 0 -> + L = build_iolist(N div 2, Base), + [L,L]; + 1 -> + L = build_iolist(N div 2, Base), + [L,L,45] + end + end; +build_iolist(N0, Base) -> + Small = random:uniform(15), + Seq = lists:seq(1, Small), + N = N0 - Small, case N rem 2 of - 0 -> [L|L]; - 1 -> [7,L|L] + 0 -> + L = build_iolist(N div 2, Base), + [L,L|Seq]; + 1 -> + L = build_iolist(N div 2, Base), + [47,L,L|Seq] end. - - outputv_echo(doc) -> ["Test echoing data with a driver that supports outputv."]; outputv_echo(Config) when is_list(Config) -> ?line Dog = test_server:timetrap(test_server:minutes(10)), @@ -1838,13 +1892,39 @@ thread_mseg_alloc_cache_clean_test(Port, N, CCI, Size) -> ?line ?t:format("CCC = ~p~n", [CCC]), ?line true = CCC > OCCC, ?line thread_mseg_alloc_cache_clean_test(Port, N-1, CCI, Size). - - + +otp_9302(Config) when is_list(Config) -> + ?line Path = ?config(data_dir, Config), + ?line erl_ddll:start(), + ?line ok = load_driver(Path, otp_9302_drv), + ?line Port = open_port({spawn, otp_9302_drv}, []), + ?line true = is_port(Port), + ?line port_command(Port, ""), + ?line {msg, block} = get_port_msg(Port, infinity), + ?line {msg, job} = get_port_msg(Port, infinity), + ?line case erlang:system_info(thread_pool_size) of + 0 -> + {msg, cancel} = get_port_msg(Port, infinity); + _ -> + ok + end, + ?line {msg, job} = get_port_msg(Port, infinity), + ?line {msg, end_of_jobs} = get_port_msg(Port, infinity), + ?line no_msg = get_port_msg(Port, 2000), + ?line port_close(Port), + ?line ok. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Utilities %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - + +get_port_msg(Port, Timeout) -> + receive + {Port, What} -> + {msg, What} + after Timeout -> + no_msg + end. wait_until(Fun) -> case Fun() of diff --git a/erts/emulator/test/driver_SUITE_data/Makefile.src b/erts/emulator/test/driver_SUITE_data/Makefile.src index 4ac7987d2f..5b3ba1557e 100644 --- a/erts/emulator/test/driver_SUITE_data/Makefile.src +++ b/erts/emulator/test/driver_SUITE_data/Makefile.src @@ -11,7 +11,8 @@ MISC_DRVS = outputv_drv@dll@ \ caller_drv@dll@ \ many_events_drv@dll@ \ missing_callback_drv@dll@ \ - thr_alloc_drv@dll@ + thr_alloc_drv@dll@ \ + otp_9302_drv@dll@ SYS_INFO_DRVS = sys_info_1_0_drv@dll@ \ sys_info_1_1_drv@dll@ \ diff --git a/erts/emulator/test/driver_SUITE_data/otp_9302_drv.c b/erts/emulator/test/driver_SUITE_data/otp_9302_drv.c new file mode 100644 index 0000000000..1147549135 --- /dev/null +++ b/erts/emulator/test/driver_SUITE_data/otp_9302_drv.c @@ -0,0 +1,137 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2011. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ +#ifdef __WIN32__ +#include <windows.h> +#endif +#include "erl_driver.h" + +static ErlDrvData start(ErlDrvPort port, + char *command); +static void output(ErlDrvData drv_data, + char *buf, int len); +static void ready_async(ErlDrvData drv_data, + ErlDrvThreadData thread_data); + +static ErlDrvEntry otp_9302_drv_entry = { + NULL /* init */, + start, + NULL /* stop */, + output, + NULL /* ready_input */, + NULL /* ready_output */, + "otp_9302_drv", + NULL /* finish */, + NULL /* handle */, + NULL /* control */, + NULL /* timeout */, + NULL /* outputv */, + ready_async, + NULL /* flush */, + NULL /* call */, + NULL /* event */, + ERL_DRV_EXTENDED_MARKER, + ERL_DRV_EXTENDED_MAJOR_VERSION, + ERL_DRV_EXTENDED_MINOR_VERSION, + ERL_DRV_FLAG_USE_PORT_LOCKING, + NULL /* handle2 */, + NULL /* handle_monitor */ +}; + +DRIVER_INIT(otp_9302_drv) +{ + return &otp_9302_drv_entry; +} + +static ErlDrvData start(ErlDrvPort port, + char *command) +{ + return (ErlDrvData) port; +} + +typedef struct { + ErlDrvPort port; + ErlDrvTermData receiver; + int block; + int cancel; + int eoj; +} Otp9302AsyncData; + +static void async_invoke(void *data) +{ + Otp9302AsyncData *adata = (Otp9302AsyncData *) data; + char *what = (adata->block + ? "block" + : (adata->cancel + ? "cancel" + : (adata->eoj + ? "end_of_jobs" + : "job"))); + ErlDrvTermData spec[] = { + ERL_DRV_PORT, driver_mk_port(adata->port), + ERL_DRV_ATOM, driver_mk_atom(what), + ERL_DRV_TUPLE, 2 + }; + driver_send_term(adata->port, adata->receiver, + spec, sizeof(spec)/sizeof(spec[0])); + if (adata->block) { +#ifdef __WIN32__ + Sleep((DWORD) 2000); +#else + sleep(2); +#endif + } +} + +static void output(ErlDrvData drv_data, + char *buf, int len) +{ + ErlDrvPort port = (ErlDrvPort) drv_data; + ErlDrvTermData caller = driver_caller(port); + unsigned int key = (unsigned int) port; + long id[5]; + Otp9302AsyncData *ad[5]; + int i; + + for (i = 0; i < sizeof(ad)/sizeof(ad[0]); i++) { + ad[i] = driver_alloc(sizeof(Otp9302AsyncData)); + if (!ad[i]) + abort(); + + ad[i]->port = port; + ad[i]->receiver = caller; + ad[i]->block = 0; + ad[i]->eoj = 0; + ad[i]->cancel = 0; + } + ad[0]->block = 1; + ad[2]->cancel = 1; + ad[4]->eoj = 1; + for (i = 0; i < sizeof(id)/sizeof(id[0]); i++) + id[i] = driver_async(port, &key, async_invoke, ad[i], driver_free); + if (id[2] > 0) + driver_async_cancel(id[2]); +} + +static void ready_async(ErlDrvData drv_data, + ErlDrvThreadData thread_data) +{ + if ((void *) thread_data) + driver_free((void *) thread_data); +} + diff --git a/erts/etc/common/erlexec.c b/erts/etc/common/erlexec.c index 60b3af7db7..90d3be9448 100644 --- a/erts/etc/common/erlexec.c +++ b/erts/etc/common/erlexec.c @@ -394,6 +394,7 @@ int main(int argc, char **argv) int print_args_exit = 0; int print_qouted_cmd_exit = 0; erts_cpu_info_t *cpuinfo = NULL; + char* emu_name; #ifdef __WIN32__ this_module_handle = module; @@ -566,6 +567,7 @@ int main(int argc, char **argv) usage("+MYm"); } emu = add_extra_suffixes(emu, emu_type); + emu_name = strsave(emu); erts_snprintf(tmpStr, sizeof(tmpStr), "%s" DIRSEP "%s" BINARY_EXT, bindir, emu); emu = strsave(tmpStr); @@ -682,6 +684,9 @@ int main(int argc, char **argv) verbose = 1; } else if (strcmp(argv[i], "-emu_args_exit") == 0) { print_args_exit = 1; + } else if (strcmp(argv[i], "-emu_name_exit") == 0) { + printf("%s\n", emu_name); + exit(0); } else if (strcmp(argv[i], "-emu_qouted_cmd_exit") == 0) { print_qouted_cmd_exit = 1; } else if (strcmp(argv[i], "-env") == 0) { /* -env VARNAME VARVALUE */ @@ -1970,6 +1975,11 @@ initial_argv_massage(int *argc, char ***argv) */ vix = 0; + + av = build_args_from_env("ERL_" OTP_SYSTEM_VERSION "_FLAGS"); + if (av) + avv[vix++].argv = av; + av = build_args_from_env("ERL_AFLAGS"); if (av) avv[vix++].argv = av; @@ -1984,10 +1994,6 @@ initial_argv_massage(int *argc, char ***argv) if (av) avv[vix++].argv = av; - av = build_args_from_env("ERL_" OTP_SYSTEM_VERSION "_FLAGS"); - if (av) - avv[vix++].argv = av; - av = build_args_from_env("ERL_ZFLAGS"); if (av) avv[vix++].argv = av; diff --git a/erts/etc/unix/cerl.src b/erts/etc/unix/cerl.src index 0355f2629f..0b2d6512ea 100644 --- a/erts/etc/unix/cerl.src +++ b/erts/etc/unix/cerl.src @@ -62,13 +62,21 @@ cxargs_add() { done } +eeargs= +eeargs_add() { + while [ $# -gt 0 ]; do + cargs="$cargs $1" + eeargs="$eeargs $1" + shift + done +} + core= GDB= GDBBP= GDBARGS= TYPE= -EMU_TYPE= debug= run_valgrind=no @@ -127,34 +135,37 @@ while [ $# -gt 0 ]; do ;; "-smp") shift - cargs="$cargs -smp" - EMU_TYPE=.smp + if [ $# -le 0 ]; then + eeargs_add -smp + else + case $1 in + disable) + shift + eeargs_add -smpdisable + ;; + enable) + shift + eeargs_add -smp + ;; + *) + eeargs_add -smp + esac + fi + ;; + "-smpdisable") + shift + eeargs_add -smpdisable ;; "-lcnt") shift cargs="$cargs -lcnt" TYPE=.lcnt ;; - "-frag") - shift - cargs="$cargs -frag" - EMU_TYPE=.frag - ;; - "-smp_frag") - shift - cargs="$cargs -smp_frag" - EMU_TYPE=.smp_frag - ;; "-gprof") shift cargs="$cargs -gprof" TYPE=.gprof ;; - "-hybrid") - shift - cargs="$cargs -hybrid" - EMU_TYPE=.hybrid - ;; "-debug") shift cargs="$cargs -debug" @@ -180,11 +191,6 @@ while [ $# -gt 0 ]; do # shift # GDB=xxgdb # ;; - "-shared") - shift - cargs="$cargs -shared" - TYPE=.shared - ;; "-purify") shift cargs="$cargs -purify" @@ -222,7 +228,9 @@ PATH=$BINDIR:$ROOTDIR/bin:$PATH EXEC=$BINDIR/erlexec PROGNAME="$PROGNAME $cargs" -EMU=$EMU$TYPE$EMU_TYPE +EMU="$EMU$TYPE" +EMU_NAME=`$EXEC -emu_name_exit $eeargs` + if [ $run_valgrind != yes ]; then xargs="$xargs -pz $PRELOADED --" fi @@ -248,9 +256,9 @@ if [ "x$GDB" = "x" ]; then valgrind_log= else if [ $valmajor -gt 2 -a $valminor -gt 4 ]; then - valgrind_log="$log_file_prefix$VALGRIND_LOG_DIR/$VALGRIND_LOGFILE_PREFIX$VALGRIND_LOGFILE_INFIX$EMU.log.$$" + valgrind_log="$log_file_prefix$VALGRIND_LOG_DIR/$VALGRIND_LOGFILE_PREFIX$VALGRIND_LOGFILE_INFIX$EMU_NAME.log.$$" else - valgrind_log="$log_file_prefix$VALGRIND_LOG_DIR/$VALGRIND_LOGFILE_PREFIX$VALGRIND_LOGFILE_INFIX$EMU.log" + valgrind_log="$log_file_prefix$VALGRIND_LOG_DIR/$VALGRIND_LOGFILE_PREFIX$VALGRIND_LOGFILE_INFIX$EMU_NAME.log" fi fi if [ "x$VALGRIND_MISC_FLAGS" = "x" ]; then @@ -263,9 +271,9 @@ if [ "x$GDB" = "x" ]; then early_beam_args=`echo $beam_args | sed "s|^\(.*-progname\).*$|\1|g"` late_beam_args=`echo $beam_args | sed "s|^$pre_beam_args.*\(-- -home.*\)$|\1|g"` - exec valgrind $valgrind_xml $valgrind_log $valgrind_misc_flags $BINDIR/$EMU $emu_xargs $early_beam_args "$PROGNAME" $late_beam_args -pz $PRELOADED + exec valgrind $valgrind_xml $valgrind_log $valgrind_misc_flags $BINDIR/$EMU_NAME $emu_xargs $early_beam_args "$PROGNAME" $late_beam_args -pz $PRELOADED else - exec $EXEC $xargs ${1+"$@"} + exec $EXEC $eeargs $xargs ${1+"$@"} fi else if [ "x$EMACS" = "x" ]; then @@ -300,5 +308,5 @@ else (insert-string \"source $ROOTDIR/erts/etc/unix/etp-commands\") \ (comint-send-input)" # Fire up gdb in emacs... - exec $EMACS --eval "(progn (gdb \"gdb $GDBARGS$EMU\") $gdbcmd)" + exec $EMACS --eval "(progn (gdb \"gdb $GDBARGS$EMU_NAME\") $gdbcmd)" fi diff --git a/erts/include/internal/erl_printf_format.h b/erts/include/internal/erl_printf_format.h index 45818079ea..400cc7dafd 100644 --- a/erts/include/internal/erl_printf_format.h +++ b/erts/include/internal/erl_printf_format.h @@ -40,7 +40,7 @@ extern int erts_printf_ulong(fmtfn_t, void*, char, int, int, unsigned long); extern int erts_printf_slong(fmtfn_t, void*, char, int, int, signed long); extern int erts_printf_double(fmtfn_t, void *, char, int, int, double); -extern int (*erts_printf_eterm_func)(fmtfn_t, void*, unsigned long, long); +extern int (*erts_printf_eterm_func)(fmtfn_t, void*, unsigned long, long, unsigned long*); #endif diff --git a/erts/include/internal/libatomic_ops/ethr_atomic.h b/erts/include/internal/libatomic_ops/ethr_atomic.h index d56693dbf8..2fc82c99a8 100644 --- a/erts/include/internal/libatomic_ops/ethr_atomic.h +++ b/erts/include/internal/libatomic_ops/ethr_atomic.h @@ -146,13 +146,13 @@ ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var) static ETHR_INLINE ETHR_AINT_T__ ETHR_NATMC_FUNC__(add_return)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) { -#ifdef AO_HAVE_fetch_and_add - return ((ETHR_AINT_T__) AO_fetch_and_add(&var->counter, (AO_t) incr)) + incr; +#ifdef AO_HAVE_fetch_and_add_full + return ((ETHR_AINT_T__) AO_fetch_and_add_full(&var->counter, (AO_t) incr)) + incr; #else while (1) { AO_t exp = AO_load(&var->counter); AO_t new = exp + (AO_t) incr; - if (AO_compare_and_swap(&var->counter, exp, new)) + if (AO_compare_and_swap_full(&var->counter, exp, new)) return (ETHR_AINT_T__) new; } #endif @@ -167,8 +167,8 @@ ETHR_NATMC_FUNC__(add)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) static ETHR_INLINE ETHR_AINT_T__ ETHR_NATMC_FUNC__(inc_return)(ETHR_ATMC_T__ *var) { -#ifdef AO_HAVE_fetch_and_add1 - return ((ETHR_AINT_T__) AO_fetch_and_add1(&var->counter)) + 1; +#ifdef AO_HAVE_fetch_and_add1_full + return ((ETHR_AINT_T__) AO_fetch_and_add1_full(&var->counter)) + 1; #else return ETHR_NATMC_FUNC__(add_return)(var, 1); #endif @@ -183,8 +183,8 @@ ETHR_NATMC_FUNC__(inc)(ETHR_ATMC_T__ *var) static ETHR_INLINE ETHR_AINT_T__ ETHR_NATMC_FUNC__(dec_return)(ETHR_ATMC_T__ *var) { -#ifdef AO_HAVE_fetch_and_sub1 - return ((ETHR_AINT_T__) AO_fetch_and_sub1(&var->counter)) - 1; +#ifdef AO_HAVE_fetch_and_sub1_full + return ((ETHR_AINT_T__) AO_fetch_and_sub1_full(&var->counter)) - 1; #else return ETHR_NATMC_FUNC__(add_return)(var, -1); #endif @@ -202,7 +202,7 @@ ETHR_NATMC_FUNC__(and_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) while (1) { AO_t exp = AO_load(&var->counter); AO_t new = exp & ((AO_t) mask); - if (AO_compare_and_swap(&var->counter, exp, new)) + if (AO_compare_and_swap_full(&var->counter, exp, new)) return (ETHR_AINT_T__) exp; } } @@ -213,7 +213,7 @@ ETHR_NATMC_FUNC__(or_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) while (1) { AO_t exp = AO_load(&var->counter); AO_t new = exp | ((AO_t) mask); - if (AO_compare_and_swap(&var->counter, exp, new)) + if (AO_compare_and_swap_full(&var->counter, exp, new)) return (ETHR_AINT_T__) exp; } } @@ -225,7 +225,7 @@ ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var, { ETHR_AINT_T__ act; do { - if (AO_compare_and_swap(&var->counter, (AO_t) exp, (AO_t) new)) + if (AO_compare_and_swap_full(&var->counter, (AO_t) exp, (AO_t) new)) return exp; act = (ETHR_AINT_T__) AO_load(&var->counter); } while (act == exp); @@ -237,7 +237,7 @@ ETHR_NATMC_FUNC__(xchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new) { while (1) { AO_t exp = AO_load(&var->counter); - if (AO_compare_and_swap(&var->counter, exp, (AO_t) new)) + if (AO_compare_and_swap_full(&var->counter, exp, (AO_t) new)) return (ETHR_AINT_T__) exp; } } @@ -265,7 +265,6 @@ ETHR_NATMC_FUNC__(inc_return_acqb)(ETHR_ATMC_T__ *var) return ((ETHR_AINT_T__) AO_fetch_and_add1_acquire(&var->counter)) + 1; #else ETHR_AINT_T__ res = ETHR_NATMC_FUNC__(add_return)(var, 1); - ETHR_MEMORY_BARRIER; return res; #endif } @@ -287,7 +286,6 @@ ETHR_NATMC_FUNC__(dec_return_relb)(ETHR_ATMC_T__ *var) #ifdef AO_HAVE_fetch_and_sub1_release return ((ETHR_AINT_T__) AO_fetch_and_sub1_release(&var->counter)) - 1; #else - ETHR_MEMORY_BARRIER; return ETHR_NATMC_FUNC__(dec_return)(var); #endif } @@ -314,7 +312,6 @@ ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var, return act; #else ETHR_AINT_T__ act = ETHR_NATMC_FUNC__(cmpxchg)(var, new, exp); - ETHR_MEMORY_BARRIER; return act; #endif } @@ -333,7 +330,6 @@ ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var, } while (act == exp); return act; #else - ETHR_MEMORY_BARRIER; return ETHR_NATMC_FUNC__(cmpxchg)(var, new, exp); #endif } diff --git a/erts/include/internal/sparc32/atomic.h b/erts/include/internal/sparc32/atomic.h index 00380dbf07..16182f8b01 100644 --- a/erts/include/internal/sparc32/atomic.h +++ b/erts/include/internal/sparc32/atomic.h @@ -95,7 +95,7 @@ ETHR_NATMC_FUNC__(add_return)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) { ETHR_AINT_T__ old, tmp; - __asm__ __volatile__("membar #LoadLoad|#StoreLoad\n"); + __asm__ __volatile__("membar #LoadLoad|#StoreLoad\n" : : : "memory"); do { old = var->counter; tmp = old+incr; @@ -105,7 +105,7 @@ ETHR_NATMC_FUNC__(add_return)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) : "r"(old), "r"(&var->counter), "0"(tmp) : "memory"); } while (__builtin_expect(old != tmp, 0)); - __asm__ __volatile__("membar #StoreLoad|#StoreStore"); + __asm__ __volatile__("membar #StoreLoad|#StoreStore" : : : "memory"); return old+incr; } @@ -144,7 +144,7 @@ ETHR_NATMC_FUNC__(and_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) { ETHR_AINT_T__ old, tmp; - __asm__ __volatile__("membar #LoadLoad|#StoreLoad\n"); + __asm__ __volatile__("membar #LoadLoad|#StoreLoad\n" : : : "memory"); do { old = var->counter; tmp = old & mask; @@ -154,7 +154,7 @@ ETHR_NATMC_FUNC__(and_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) : "r"(old), "r"(&var->counter), "0"(tmp) : "memory"); } while (__builtin_expect(old != tmp, 0)); - __asm__ __volatile__("membar #StoreLoad|#StoreStore"); + __asm__ __volatile__("membar #StoreLoad|#StoreStore" : : : "memory"); return old; } @@ -163,7 +163,7 @@ ETHR_NATMC_FUNC__(or_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) { ETHR_AINT_T__ old, tmp; - __asm__ __volatile__("membar #LoadLoad|#StoreLoad\n"); + __asm__ __volatile__("membar #LoadLoad|#StoreLoad\n" : : : "memory"); do { old = var->counter; tmp = old | mask; @@ -173,7 +173,7 @@ ETHR_NATMC_FUNC__(or_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) : "r"(old), "r"(&var->counter), "0"(tmp) : "memory"); } while (__builtin_expect(old != tmp, 0)); - __asm__ __volatile__("membar #StoreLoad|#StoreStore"); + __asm__ __volatile__("membar #StoreLoad|#StoreStore" : : : "memory"); return old; } @@ -182,7 +182,7 @@ ETHR_NATMC_FUNC__(xchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ val) { ETHR_AINT_T__ old, new; - __asm__ __volatile__("membar #LoadLoad|#StoreLoad"); + __asm__ __volatile__("membar #LoadLoad|#StoreLoad" : : : "memory"); do { old = var->counter; new = val; @@ -192,20 +192,20 @@ ETHR_NATMC_FUNC__(xchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ val) : "r"(old), "r"(&var->counter), "0"(new) : "memory"); } while (__builtin_expect(old != new, 0)); - __asm__ __volatile__("membar #StoreLoad|#StoreStore"); + __asm__ __volatile__("membar #StoreLoad|#StoreStore" : : : "memory"); return old; } static ETHR_INLINE ETHR_AINT_T__ ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new, ETHR_AINT_T__ old) { - __asm__ __volatile__("membar #LoadLoad|#StoreLoad\n"); + __asm__ __volatile__("membar #LoadLoad|#StoreLoad\n" : : : "memory"); __asm__ __volatile__( ETHR_CAS__ " [%2], %1, %0" : "=&r"(new) : "r"(old), "r"(&var->counter), "0"(new) : "memory"); - __asm__ __volatile__("membar #StoreLoad|#StoreStore"); + __asm__ __volatile__("membar #StoreLoad|#StoreStore" : : : "memory"); return new; } @@ -213,13 +213,11 @@ ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new, ETHR_AINT_T__ * Atomic ops with at least specified barriers. */ -/* TODO: relax acquire barriers */ - static ETHR_INLINE ETHR_AINT_T__ ETHR_NATMC_FUNC__(read_acqb)(ETHR_ATMC_T__ *var) { ETHR_AINT_T__ res = ETHR_NATMC_FUNC__(read)(var); - __asm__ __volatile__("membar #LoadLoad|#LoadStore|#StoreLoad|#StoreStore" : : : "memory"); + __asm__ __volatile__("membar #LoadLoad|#LoadStore" : : : "memory"); return res; } @@ -234,21 +232,18 @@ static ETHR_INLINE ETHR_AINT_T__ ETHR_NATMC_FUNC__(inc_return_acqb)(ETHR_ATMC_T__ *var) { ETHR_AINT_T__ res = ETHR_NATMC_FUNC__(inc_return)(var); - __asm__ __volatile__("membar #LoadLoad|#LoadStore" : : : "memory"); return res; } static ETHR_INLINE void ETHR_NATMC_FUNC__(dec_relb)(ETHR_ATMC_T__ *var) { - __asm__ __volatile__("membar #LoadStore|#StoreStore" : : : "memory"); ETHR_NATMC_FUNC__(dec)(var); } static ETHR_INLINE ETHR_AINT_T__ ETHR_NATMC_FUNC__(dec_return_relb)(ETHR_ATMC_T__ *var) { - __asm__ __volatile__("membar #LoadStore|#StoreStore" : : : "memory"); return ETHR_NATMC_FUNC__(dec_return)(var); } @@ -256,14 +251,12 @@ static ETHR_INLINE ETHR_AINT_T__ ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new, ETHR_AINT_T__ old) { ETHR_AINT_T__ res = ETHR_NATMC_FUNC__(cmpxchg)(var, new, old); - __asm__ __volatile__("membar #LoadLoad|#LoadStore" : : : "memory"); return res; } static ETHR_INLINE ETHR_AINT_T__ ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new, ETHR_AINT_T__ old) { - __asm__ __volatile__("membar #LoadStore|#StoreStore" : : : "memory"); return ETHR_NATMC_FUNC__(cmpxchg)(var, new, old); } diff --git a/erts/include/internal/tile/atomic.h b/erts/include/internal/tile/atomic.h index 48e4c0c6c8..0c7b597a6b 100644 --- a/erts/include/internal/tile/atomic.h +++ b/erts/include/internal/tile/atomic.h @@ -65,25 +65,35 @@ ethr_native_atomic32_read(ethr_native_atomic32_t *var) static ETHR_INLINE void ethr_native_atomic32_add(ethr_native_atomic32_t *var, ethr_sint32_t incr) { + ETHR_MEMORY_BARRIER; atomic_add(&var->counter, incr); + ETHR_MEMORY_BARRIER; } static ETHR_INLINE void ethr_native_atomic32_inc(ethr_native_atomic32_t *var) { + ETHR_MEMORY_BARRIER; atomic_increment(&var->counter); + ETHR_MEMORY_BARRIER; } static ETHR_INLINE void ethr_native_atomic32_dec(ethr_native_atomic32_t *var) { + ETHR_MEMORY_BARRIER; atomic_decrement(&var->counter); + ETHR_MEMORY_BARRIER; } static ETHR_INLINE ethr_sint32_t ethr_native_atomic32_add_return(ethr_native_atomic32_t *var, ethr_sint32_t incr) { - return atomic_exchange_and_add(&var->counter, incr) + incr; + ethr_sint32_t res; + ETHR_MEMORY_BARRIER; + res = atomic_exchange_and_add(&var->counter, incr) + incr; + ETHR_MEMORY_BARRIER; + return res; } static ETHR_INLINE ethr_sint32_t @@ -101,18 +111,27 @@ ethr_native_atomic32_dec_return(ethr_native_atomic32_t *var) static ETHR_INLINE ethr_sint32_t ethr_native_atomic32_and_retold(ethr_native_atomic32_t *var, ethr_sint32_t mask) { - return atomic_and_val(&var->counter, mask); + ethr_sint32_t res; + ETHR_MEMORY_BARRIER; + res = atomic_and_val(&var->counter, mask); + ETHR_MEMORY_BARRIER; + return res; } static ETHR_INLINE ethr_sint32_t ethr_native_atomic32_or_retold(ethr_native_atomic32_t *var, ethr_sint32_t mask) { - return atomic_or_val(&var->counter, mask); + ethr_sint32_t res; + ETHR_MEMORY_BARRIER; + res = atomic_or_val(&var->counter, mask); + ETHR_MEMORY_BARRIER; + return res; } static ETHR_INLINE ethr_sint32_t ethr_native_atomic32_xchg(ethr_native_atomic32_t *var, ethr_sint32_t val) { + ETHR_MEMORY_BARRIER; return atomic_exchange_acq(&var->counter, val); } @@ -121,6 +140,7 @@ ethr_native_atomic32_cmpxchg(ethr_native_atomic32_t *var, ethr_sint32_t new, ethr_sint32_t expected) { + ETHR_MEMORY_BARRIER; return atomic_compare_and_exchange_val_acq(&var->counter, new, expected); } @@ -139,9 +159,7 @@ ethr_native_atomic32_read_acqb(ethr_native_atomic32_t *var) static ETHR_INLINE ethr_sint32_t ethr_native_atomic32_inc_return_acqb(ethr_native_atomic32_t *var) { - ethr_sint32_t res = ethr_native_atomic32_inc_return(var); - ETHR_MEMORY_BARRIER; - return res; + return ethr_native_atomic32_inc_return(var); } static ETHR_INLINE void @@ -154,14 +172,12 @@ ethr_native_atomic32_set_relb(ethr_native_atomic32_t *var, ethr_sint32_t val) static ETHR_INLINE void ethr_native_atomic32_dec_relb(ethr_native_atomic32_t *var) { - ETHR_MEMORY_BARRIER; ethr_native_atomic32_dec(var); } static ETHR_INLINE ethr_sint32_t ethr_native_atomic32_dec_return_relb(ethr_native_atomic32_t *var) { - ETHR_MEMORY_BARRIER; return ethr_native_atomic32_dec_return(var); } @@ -178,7 +194,6 @@ ethr_native_atomic32_cmpxchg_relb(ethr_native_atomic32_t *var, ethr_sint32_t new, ethr_sint32_t exp) { - ETHR_MEMORY_BARRIER; return ethr_native_atomic32_cmpxchg(var, new, exp); } diff --git a/erts/lib_src/common/erl_printf_format.c b/erts/lib_src/common/erl_printf_format.c index 968d563325..fba3fd723c 100644 --- a/erts/lib_src/common/erl_printf_format.c +++ b/erts/lib_src/common/erl_printf_format.c @@ -25,7 +25,7 @@ * width: [0-9]+ | '*' * precision: [0-9]+ | '*' * length: hh | h | l | ll | L | j | t | b<sz> - * conversion: d,i | o,u,x,X | e,E | f,F | g,G | a,A | c | s | T | + * conversion: d,i | o,u,x,X | e,E | f,F | g,G | a,A | c | s | T | R | * p | n | % * sz: 8 | 16 | 32 | 64 | p | e */ @@ -101,7 +101,7 @@ #endif #define FMTC_d 0x0000 -#define FMTC_i 0x0001 +#define FMTC_R 0x0001 #define FMTC_o 0x0002 #define FMTC_u 0x0003 #define FMTC_x 0x0004 @@ -165,7 +165,7 @@ static char heX[] = "0123456789ABCDEF"; #define SIGN(X) ((X) > 0 ? 1 : ((X) < 0 ? -1 : 0)) #define USIGN(X) ((X) == 0 ? 0 : 1) -int (*erts_printf_eterm_func)(fmtfn_t, void*, unsigned long, long) = NULL; +int (*erts_printf_eterm_func)(fmtfn_t, void*, unsigned long, long, unsigned long*) = NULL; static int noop_fn(void *vfp, char* buf, size_t len) @@ -183,8 +183,8 @@ static int fmt_fld(fmtfn_t fn,void* arg, int len; /* format the prefix */ - if ((sign || (fmt & (FMTF_sgn|FMTF_blk))) && - (((fmt & FMTC_MASK) == FMTC_d) || ((fmt & FMTC_MASK) == FMTC_i))) { + if ((sign || (fmt & (FMTF_sgn|FMTF_blk))) + && (fmt & FMTC_MASK) == FMTC_d) { if (sign < 0) *pp++ = '-'; else if ((fmt & FMTF_sgn)) @@ -245,7 +245,6 @@ static int fmt_long(fmtfn_t fn,void* arg,int sign,unsigned long uval, switch(fmt & FMTC_MASK) { case FMTC_d: - case FMTC_i: case FMTC_u: break; case FMTC_o: @@ -298,7 +297,6 @@ static int fmt_long_long(fmtfn_t fn,void* arg,int sign, switch(fmt & FMTC_MASK) { case FMTC_d: - case FMTC_i: case FMTC_u: break; case FMTC_o: @@ -622,7 +620,7 @@ int erts_printf_format(fmtfn_t fn, void* arg, char* fmt, va_list ap) /* specifier */ switch(*ptr) { case 'd': ptr++; fmt |= FMTC_d; break; - case 'i': ptr++; fmt |= FMTC_i; break; + case 'i': ptr++; fmt |= FMTC_d; break; case 'o': ptr++; fmt |= FMTC_o; break; case 'u': ptr++; fmt |= FMTC_u; break; case 'x': ptr++; fmt |= FMTC_x; break; @@ -637,6 +635,7 @@ int erts_printf_format(fmtfn_t fn, void* arg, char* fmt, va_list ap) case 'p': ptr++; fmt |= FMTC_p; break; case 'n': ptr++; fmt |= FMTC_n; break; case 'T': ptr++; fmt |= FMTC_T; break; + case 'R': ptr++; fmt |= FMTC_R; break; case '%': FMT(fn,arg,ptr,1,count); ptr++; @@ -650,7 +649,6 @@ int erts_printf_format(fmtfn_t fn, void* arg, char* fmt, va_list ap) switch(fmt & FMTC_MASK) { case FMTC_d: - case FMTC_i: switch(fmt & FMTL_MASK) { case FMTL_hh: { signed char tval = (signed char) va_arg(ap,int); @@ -814,9 +812,12 @@ int erts_printf_format(fmtfn_t fn, void* arg, char* fmt, va_list ap) default: *va_arg(ap,int*) = count; break; } break; - case FMTC_T: { + case FMTC_T: /* Eterm */ + case FMTC_R: { /* Eterm, Eterm* base (base ignored if !HALFWORD_HEAP) */ long prec; unsigned long eterm; + unsigned long* eterm_base; + if (!erts_printf_eterm_func) return -EINVAL; if (precision < 0) @@ -826,14 +827,16 @@ int erts_printf_format(fmtfn_t fn, void* arg, char* fmt, va_list ap) else prec = (long) precision; eterm = va_arg(ap, unsigned long); + eterm_base = ((fmt & FMTC_MASK) == FMTC_R) ? + va_arg(ap, unsigned long*) : NULL; if (width > 0 && !(fmt & FMTF_adj)) { - res = (*erts_printf_eterm_func)(noop_fn, NULL, eterm, prec); + res = (*erts_printf_eterm_func)(noop_fn, NULL, eterm, prec, eterm_base); if (res < 0) return res; if (width > res) BLANKS(fn, arg, width - res, count); } - res = (*erts_printf_eterm_func)(fn, arg, eterm, prec); + res = (*erts_printf_eterm_func)(fn, arg, eterm, prec, eterm_base); if (res < 0) return res; count += res; @@ -924,7 +927,7 @@ erts_printf_slong(fmtfn_t fn, void *arg, char conv, int pad, int width, unsigned long ul_val; switch (conv) { case 'd': fmt |= FMTC_d; break; - case 'i': fmt |= FMTC_i; break; + case 'i': fmt |= FMTC_d; break; case 'o': fmt |= FMTC_o; break; case 'x': fmt |= FMTC_x; break; case 'X': fmt |= FMTC_X; break; |