diff options
Diffstat (limited to 'erts')
226 files changed, 33218 insertions, 9569 deletions
diff --git a/erts/Makefile.in b/erts/Makefile.in index 2e63fc469e..8b86fbadf2 100644 --- a/erts/Makefile.in +++ b/erts/Makefile.in @@ -16,6 +16,9 @@ # # %CopyrightEnd% # + +.NOTPARALLEL: + include $(ERL_TOP)/make/target.mk include vsn.mk diff --git a/erts/aclocal.m4 b/erts/aclocal.m4 index 7d9d099bba..bd228a2d1f 100644 --- a/erts/aclocal.m4 +++ b/erts/aclocal.m4 @@ -682,6 +682,55 @@ AC_SUBST(ERTS_INTERNAL_X_LIBS) ]) +AC_DEFUN(ETHR_CHK_SYNC_OP, +[ + AC_MSG_CHECKING([for $3-bit $1()]) + case "$2" in + "1") sync_call="$1(&var);";; + "2") sync_call="$1(&var, ($4) 0);";; + "3") sync_call="$1(&var, ($4) 0, ($4) 0);";; + esac + have_sync_op=no + AC_TRY_LINK([], + [ + $4 res; + volatile $4 var; + res = $sync_call + ], + [have_sync_op=yes]) + test $have_sync_op = yes && $5 + AC_MSG_RESULT([$have_sync_op]) +]) + +AC_DEFUN(ETHR_CHK_INTERLOCKED, +[ + ilckd="$1" + AC_MSG_CHECKING([for ${ilckd}()]) + case "$2" in + "1") ilckd_call="${ilckd}(var);";; + "2") ilckd_call="${ilckd}(var, ($3) 0);";; + "3") ilckd_call="${ilckd}(var, ($3) 0, ($3) 0);";; + "4") ilckd_call="${ilckd}(var, ($3) 0, ($3) 0, arr);";; + esac + have_interlocked_op=no + AC_TRY_LINK( + [ + #define WIN32_LEAN_AND_MEAN + #include <windows.h> + #include <intrin.h> + ], + [ + volatile $3 *var; + volatile $3 arr[2]; + + $ilckd_call + return 0; + ], + [have_interlocked_op=yes]) + test $have_interlocked_op = yes && $4 + AC_MSG_RESULT([$have_interlocked_op]) +]) + dnl ---------------------------------------------------------------------- dnl dnl ERL_FIND_ETHR_LIB @@ -753,121 +802,41 @@ case "$THR_LIB_NAME" in AC_DEFINE(ETHR_WIN32_THREADS, 1, [Define if you have win32 threads]) - have_ilckd=no - AC_MSG_CHECKING([for _InterlockedCompareExchange64()]) - AC_TRY_LINK([ - #define WIN32_LEAN_AND_MEAN - #include <windows.h> - ], - [ - volatile __int64 *var; - _InterlockedCompareExchange64(var, (__int64) 1, (__int64) 0); - return 0; - ], - have_ilckd=yes) - AC_MSG_RESULT([$have_ilckd]) - test $have_ilckd = yes && AC_DEFINE(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64, 1, [Define if you have _InterlockedCompareExchange64()]) - - AC_CHECK_SIZEOF(void *) - case "$ac_cv_sizeof_void_p-$have_ilckd" in - 8-no) - ethr_have_native_atomics=no - ethr_have_native_spinlock=no;; - *) - ethr_have_native_atomics=yes - ethr_have_native_spinlock=yes;; - esac - - have_ilckd=no - AC_MSG_CHECKING([for _InterlockedDecrement64()]) - AC_TRY_LINK([ - #define WIN32_LEAN_AND_MEAN - #include <windows.h> - ], - [ - volatile __int64 *var; - _InterlockedDecrement64(var); - return 0; - ], - have_ilckd=yes) - AC_MSG_RESULT([$have_ilckd]) - test $have_ilckd = yes && AC_DEFINE(ETHR_HAVE__INTERLOCKEDDECREMENT64, 1, [Define if you have _InterlockedDecrement64()]) - - have_ilckd=no - AC_MSG_CHECKING([for _InterlockedIncrement64()]) - AC_TRY_LINK([ - #define WIN32_LEAN_AND_MEAN - #include <windows.h> - ], - [ - volatile __int64 *var; - _InterlockedIncrement64(var); - return 0; - ], - have_ilckd=yes) - AC_MSG_RESULT([$have_ilckd]) - test $have_ilckd = yes && AC_DEFINE(ETHR_HAVE__INTERLOCKEDINCREMENT64, 1, [Define if you have _InterlockedIncrement64()]) - - have_ilckd=no - AC_MSG_CHECKING([for _InterlockedExchangeAdd64()]) - AC_TRY_LINK([ - #define WIN32_LEAN_AND_MEAN - #include <windows.h> - ], - [ - volatile __int64 *var; - _InterlockedExchangeAdd64(var, (__int64) 1); - return 0; - ], - have_ilckd=yes) - AC_MSG_RESULT([$have_ilckd]) - test $have_ilckd = yes && AC_DEFINE(ETHR_HAVE__INTERLOCKEDEXCHANGEADD64, 1, [Define if you have _InterlockedExchangeAdd64()]) - - have_ilckd=no - AC_MSG_CHECKING([for _InterlockedExchange64()]) - AC_TRY_LINK([ - #define WIN32_LEAN_AND_MEAN - #include <windows.h> - ], - [ - volatile __int64 *var; - _InterlockedExchange64(var, (__int64) 1); - return 0; - ], - have_ilckd=yes) - AC_MSG_RESULT([$have_ilckd]) - test $have_ilckd = yes && AC_DEFINE(ETHR_HAVE__INTERLOCKEDEXCHANGE64, 1, [Define if you have _InterlockedExchange64()]) - - have_ilckd=no - AC_MSG_CHECKING([for _InterlockedAnd64()]) - AC_TRY_LINK([ - #define WIN32_LEAN_AND_MEAN - #include <windows.h> - ], - [ - volatile __int64 *var; - _InterlockedAnd64(var, (__int64) 1); - return 0; - ], - have_ilckd=yes) - AC_MSG_RESULT([$have_ilckd]) - test $have_ilckd = yes && AC_DEFINE(ETHR_HAVE__INTERLOCKEDAND64, 1, [Define if you have _InterlockedAnd64()]) - - have_ilckd=no - AC_MSG_CHECKING([for _InterlockedOr64()]) - AC_TRY_LINK([ - #define WIN32_LEAN_AND_MEAN - #include <windows.h> - ], - [ - volatile __int64 *var; - _InterlockedOr64(var, (__int64) 1); - return 0; - ], - have_ilckd=yes) - AC_MSG_RESULT([$have_ilckd]) - test $have_ilckd = yes && AC_DEFINE(ETHR_HAVE__INTERLOCKEDOR64, 1, [Define if you have _InterlockedOr64()]) - + ETHR_CHK_INTERLOCKED([_InterlockedDecrement], [1], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDDECREMENT, 1, [Define if you have _InterlockedDecrement()])) + ETHR_CHK_INTERLOCKED([_InterlockedDecrement_rel], [1], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDDECREMENT_REL, 1, [Define if you have _InterlockedDecrement_rel()])) + ETHR_CHK_INTERLOCKED([_InterlockedIncrement], [1], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDINCREMENT, 1, [Define if you have _InterlockedIncrement()])) + ETHR_CHK_INTERLOCKED([_InterlockedIncrement_acq], [1], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDINCREMENT_ACQ, 1, [Define if you have _InterlockedIncrement_acq()])) + ETHR_CHK_INTERLOCKED([_InterlockedExchangeAdd], [2], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDEXCHANGEADD, 1, [Define if you have _InterlockedExchangeAdd()])) + ETHR_CHK_INTERLOCKED([_InterlockedExchangeAdd_acq], [2], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDEXCHANGEADD_ACQ, 1, [Define if you have _InterlockedExchangeAdd_acq()])) + ETHR_CHK_INTERLOCKED([_InterlockedAnd], [2], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDAND, 1, [Define if you have _InterlockedAnd()])) + ETHR_CHK_INTERLOCKED([_InterlockedOr], [2], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDOR, 1, [Define if you have _InterlockedOr()])) + ETHR_CHK_INTERLOCKED([_InterlockedExchange], [2], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDEXCHANGE, 1, [Define if you have _InterlockedExchange()])) + ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange], [3], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE, 1, [Define if you have _InterlockedCompareExchange()])) + test "$have_interlocked_op" = "yes" && ethr_have_native_atomics=yes + ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange_acq], [3], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE_ACQ, 1, [Define if you have _InterlockedCompareExchange_acq()])) + test "$have_interlocked_op" = "yes" && ethr_have_native_atomics=yes + ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange_rel], [3], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE_REL, 1, [Define if you have _InterlockedCompareExchange_rel()])) + test "$have_interlocked_op" = "yes" && ethr_have_native_atomics=yes + + ETHR_CHK_INTERLOCKED([_InterlockedDecrement64], [1], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDDECREMENT64, 1, [Define if you have _InterlockedDecrement64()])) + ETHR_CHK_INTERLOCKED([_InterlockedDecrement64_rel], [1], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDDECREMENT64_REL, 1, [Define if you have _InterlockedDecrement64_rel()])) + ETHR_CHK_INTERLOCKED([_InterlockedIncrement64], [1], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDINCREMENT64, 1, [Define if you have _InterlockedIncrement64()])) + ETHR_CHK_INTERLOCKED([_InterlockedIncrement64_acq], [1], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDINCREMENT64_ACQ, 1, [Define if you have _InterlockedIncrement64_acq()])) + ETHR_CHK_INTERLOCKED([_InterlockedExchangeAdd64], [2], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDEXCHANGEADD64, 1, [Define if you have _InterlockedExchangeAdd64()])) + ETHR_CHK_INTERLOCKED([_InterlockedExchangeAdd64_acq], [2], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDEXCHANGEADD64_ACQ, 1, [Define if you have _InterlockedExchangeAdd64_acq()])) + ETHR_CHK_INTERLOCKED([_InterlockedAnd64], [2], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDAND64, 1, [Define if you have _InterlockedAnd64()])) + ETHR_CHK_INTERLOCKED([_InterlockedOr64], [2], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDOR64, 1, [Define if you have _InterlockedOr64()])) + ETHR_CHK_INTERLOCKED([_InterlockedExchange64], [2], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDEXCHANGE64, 1, [Define if you have _InterlockedExchange64()])) + ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange64], [3], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64, 1, [Define if you have _InterlockedCompareExchange64()])) + test "$have_interlocked_op" = "yes" && ethr_have_native_atomics=yes + ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange64_acq], [3], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_ACQ, 1, [Define if you have _InterlockedCompareExchange64_acq()])) + test "$have_interlocked_op" = "yes" && ethr_have_native_atomics=yes + ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange64_rel], [3], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_REL, 1, [Define if you have _InterlockedCompareExchange64_rel()])) + test "$have_interlocked_op" = "yes" && ethr_have_native_atomics=yes + + ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange128], [4], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE128, 1, [Define if you have _InterlockedCompareExchange128()])) + + test "$ethr_have_native_atomics" = "yes" && ethr_have_native_spinlock=yes ;; pthread) @@ -1103,35 +1072,51 @@ case "$THR_LIB_NAME" in AC_MSG_RESULT([$linux_futex]) test $linux_futex = yes && AC_DEFINE(ETHR_HAVE_LINUX_FUTEX, 1, [Define if you have a linux futex implementation.]) - AC_MSG_CHECKING([for GCC atomic operations]) - ethr_have_gcc_atomic_ops=no - AC_TRY_LINK([], - [ - long res; - volatile long val; - res = __sync_val_compare_and_swap(&val, (long) 1, (long) 0); - res = __sync_add_and_fetch(&val, (long) 1); - res = __sync_sub_and_fetch(&val, (long) 1); - res = __sync_fetch_and_and(&val, (long) 1); - res = __sync_fetch_and_or(&val, (long) 1); - ], - [ethr_have_native_atomics=yes - ethr_have_gcc_atomic_ops=yes]) - AC_MSG_RESULT([$ethr_have_gcc_atomic_ops]) - test $ethr_have_gcc_atomic_ops = yes && AC_DEFINE(ETHR_HAVE_GCC_ATOMIC_OPS, 1, [Define if you have gcc atomic operations]) + AC_CHECK_SIZEOF(int) + AC_CHECK_SIZEOF(long) + AC_CHECK_SIZEOF(long long) + AC_CHECK_SIZEOF(__int128_t) + + if test "$ac_cv_sizeof_int" = "4"; then + int32="int" + elif test "$ac_cv_sizeof_long" = "4"; then + int32="long" + elif test "$ac_cv_sizeof_long_long" = "4"; then + int32="long long" + else + AC_MSG_ERROR([No 32-bit type found]) + fi - case "$host_cpu" in - sun4u | sparc64 | sun4v) - ethr_have_native_atomics=yes;; - i86pc | i*86 | x86_64 | amd64) - ethr_have_native_atomics=yes;; - macppc | ppc | "Power Macintosh") - ethr_have_native_atomics=yes;; - tile) - ethr_have_native_atomics=yes;; - *) - ;; - esac + if test "$ac_cv_sizeof_int" = "8"; then + int64="int" + elif test "$ac_cv_sizeof_long" = "8"; then + int64="long" + elif test "$ac_cv_sizeof_long_long" = "8"; then + int64="long long" + else + AC_MSG_ERROR([No 64-bit type found]) + fi + + int128=no + if test "$ac_cv_sizeof___int128_t" = "16"; then + int128="__int128_t" + fi + + ETHR_CHK_SYNC_OP([__sync_val_compare_and_swap], [3], [32], [$int32], AC_DEFINE(ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP32, 1, [Define if you have __sync_val_compare_and_swap() for 32-bit integers])) + test "$have_sync_op" = "yes" && ethr_have_native_atomics=yes + ETHR_CHK_SYNC_OP([__sync_add_and_fetch], [2], [32], [$int32], AC_DEFINE(ETHR_HAVE___SYNC_ADD_AND_FETCH32, 1, [Define if you have __sync_add_and_fetch() for 32-bit integers])) + ETHR_CHK_SYNC_OP([__sync_fetch_and_and], [2], [32], [$int32], AC_DEFINE(ETHR_HAVE___SYNC_FETCH_AND_AND32, 1, [Define if you have __sync_fetch_and_and() for 32-bit integers])) + ETHR_CHK_SYNC_OP([__sync_fetch_and_or], [2], [32], [$int32], AC_DEFINE(ETHR_HAVE___SYNC_FETCH_AND_OR32, 1, [Define if you have __sync_fetch_and_or() for 32-bit integers])) + + ETHR_CHK_SYNC_OP([__sync_val_compare_and_swap], [3], [64], [$int64], AC_DEFINE(ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP64, 1, [Define if you have __sync_val_compare_and_swap() for 64-bit integers])) + test "$have_sync_op" = "yes" && ethr_have_native_atomics=yes + ETHR_CHK_SYNC_OP([__sync_add_and_fetch], [2], [64], [$int64], AC_DEFINE(ETHR_HAVE___SYNC_ADD_AND_FETCH64, 1, [Define if you have __sync_add_and_fetch() for 64-bit integers])) + ETHR_CHK_SYNC_OP([__sync_fetch_and_and], [2], [64], [$int64], AC_DEFINE(ETHR_HAVE___SYNC_FETCH_AND_AND64, 1, [Define if you have __sync_fetch_and_and() for 64-bit integers])) + ETHR_CHK_SYNC_OP([__sync_fetch_and_or], [2], [64], [$int64], AC_DEFINE(ETHR_HAVE___SYNC_FETCH_AND_OR64, 1, [Define if you have __sync_fetch_and_or() for 64-bit integers])) + + if test $int128 != no; then + ETHR_CHK_SYNC_OP([__sync_val_compare_and_swap], [3], [128], [$int128], AC_DEFINE(ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP128, 1, [Define if you have __sync_val_compare_and_swap() for 128-bit integers])) + fi AC_MSG_CHECKING([for a usable libatomic_ops implementation]) case "x$with_libatomic_ops" in @@ -1178,6 +1163,34 @@ case "$THR_LIB_NAME" in AC_MSG_ERROR([No usable libatomic_ops implementation found]) fi + case "$host_cpu" in + sparc | sun4u | sparc64 | sun4v) + case "$with_sparc_memory_order" in + "TSO") + AC_DEFINE(ETHR_SPARC_TSO, 1, [Define if only run in Sparc TSO mode]);; + "PSO") + AC_DEFINE(ETHR_SPARC_PSO, 1, [Define if only run in Sparc PSO, or TSO mode]);; + "RMO"|"") + AC_DEFINE(ETHR_SPARC_RMO, 1, [Define if run in Sparc RMO, PSO, or TSO mode]);; + *) + AC_MSG_ERROR([Unsupported Sparc memory order: $with_sparc_memory_order]);; + esac + ethr_have_native_atomics=yes;; + i86pc | i*86 | x86_64 | amd64) + if test "$enable_x86_out_of_order" = "yes"; then + AC_DEFINE(ETHR_X86_OUT_OF_ORDER, 1, [Define if x86/x86_64 out of order instructions should be synchronized]) + fi + ethr_have_native_atomics=yes;; + macppc | ppc | "Power Macintosh") + ethr_have_native_atomics=yes;; + tile) + ethr_have_native_atomics=yes;; + *) + ;; + esac + + test ethr_have_native_atomics = "yes" && ethr_have_native_spinlock=yes + dnl Restore LIBS LIBS=$saved_libs dnl restore CPPFLAGS @@ -1213,6 +1226,8 @@ AC_CHECK_SIZEOF(long long) AC_DEFINE_UNQUOTED(ETHR_SIZEOF_LONG_LONG, $ac_cv_sizeof_long_long, [Define to the size of long long]) AC_CHECK_SIZEOF(__int64) AC_DEFINE_UNQUOTED(ETHR_SIZEOF___INT64, $ac_cv_sizeof___int64, [Define to the size of __int64]) +AC_CHECK_SIZEOF(__int128_t) +AC_DEFINE_UNQUOTED(ETHR_SIZEOF___INT128_T, $ac_cv_sizeof___int128_t, [Define to the size of __int128_t]) case X$erl_xcomp_bigendian in @@ -1235,6 +1250,10 @@ AC_ARG_ENABLE(native-ethr-impls, *) disable_native_ethr_impls=no ;; esac ], disable_native_ethr_impls=no) +AC_ARG_ENABLE(x86-out-of-order, + AS_HELP_STRING([--enable-x86-out-of-order], + [enable x86/x84_64 out of order support (default disabled)])) + test "X$disable_native_ethr_impls" = "Xyes" && AC_DEFINE(ETHR_DISABLE_NATIVE_IMPLS, 1, [Define if you want to disable native ethread implementations]) @@ -1253,56 +1272,100 @@ AC_ARG_WITH(libatomic_ops, AS_HELP_STRING([--with-libatomic_ops=PATH], [specify and prefer usage of libatomic_ops in the ethread library])) -AC_ARG_ENABLE(ethread-pre-pentium4-compatibility, - AS_HELP_STRING([--enable-ethread-pre-pentium4-compatibility], - [enable compatibility with x86 processors before pentium 4 (back to 486) in the ethread library]), -[ - case "$enable_ethread_pre_pentium4_compatibility" in - yes|no) ;; - *) enable_ethread_pre_pentium4_compatibility=check;; - esac -], -[enable_ethread_pre_pentium4_compatibility=check]) - -test "$cross_compiling" != "yes" || enable_ethread_pre_pentium4_compatibility=no - -case "$enable_ethread_pre_pentium4_compatibility-$host_cpu" in - check-i86pc | check-i*86) - AC_MSG_CHECKING([whether pre pentium 4 compatibility should forced]) - AC_RUN_IFELSE([ -#if defined(__GNUC__) -# if defined(ETHR_PREFER_LIBATOMIC_OPS_NATIVE_IMPLS) -# define CHECK_LIBATOMIC_OPS__ -# else -# define CHECK_GCC_ASM__ -# endif -#elif defined(ETHR_HAVE_LIBATOMIC_OPS) -# define CHECK_LIBATOMIC_OPS__ +AC_ARG_WITH(with_sparc_memory_order, + AS_HELP_STRING([--with-sparc-memory-order=TSO|PSO|RMO], + [specify sparc memory order (defaults to RMO)])) + +ETHR_X86_SSE2_ASM=no +case "$GCC-$ac_cv_sizeof_void_p-$host_cpu" in + yes-4-i86pc | yes-4-i*86 | yes-4-x86_64 | yes-4-amd64) + AC_MSG_CHECKING([for gcc sse2 asm support]) + save_CFLAGS="$CFLAGS" + CFLAGS="$CFLAGS -msse2" + gcc_sse2_asm=no + AC_TRY_COMPILE([], + [ + long long x, *y; + __asm__ __volatile__("movq %1, %0\n\t" : "=x"(x) : "m"(*y) : "memory"); + ], + [gcc_sse2_asm=yes]) + CFLAGS="$save_CFLAGS" + AC_MSG_RESULT([$gcc_sse2_asm]) + if test "$gcc_sse2_asm" = "yes"; then + AC_DEFINE(ETHR_GCC_HAVE_SSE2_ASM_SUPPORT, 1, [Define if you use a gcc that supports -msse2 and understand sse2 specific asm statements]) + ETHR_X86_SSE2_ASM=yes + fi + ;; + *) + ;; +esac + +case "$GCC-$host_cpu" in + yes-i86pc | yes-i*86 | yes-x86_64 | yes-amd64) + gcc_dw_cmpxchg_asm=no + AC_MSG_CHECKING([for gcc double word cmpxchg asm support]) + AC_TRY_COMPILE([], + [ + char xchgd; + long new[2], xchg[2], *p; + __asm__ __volatile__( +#if ETHR_SIZEOF_PTR == 4 && defined(__PIC__) && __PIC__ + "pushl %%ebx\n\t" + "movl %8, %%ebx\n\t" #endif -#if defined(CHECK_LIBATOMIC_OPS__) -#include "atomic_ops.h" +#if ETHR_SIZEOF_PTR == 4 + "lock; cmpxchg8b %0\n\t" +#else + "lock; cmpxchg16b %0\n\t" #endif -int main(void) -{ -#if defined(CHECK_GCC_ASM__) - __asm__ __volatile__("mfence" : : : "memory"); -#elif defined(CHECK_LIBATOMIC_OPS__) - AO_nop_full(); + "setz %3\n\t" +#if ETHR_SIZEOF_PTR == 4 && defined(__PIC__) && __PIC__ + "popl %%ebx\n\t" #endif - return 0; -} + : "=m"(*p), "=d"(xchg[1]), "=a"(xchg[0]), "=c"(xchgd) + : "m"(*p), "1"(xchg[1]), "2"(xchg[0]), "3"(new[1]), +#if ETHR_SIZEOF_PTR == 4 && defined(__PIC__) && __PIC__ + "r"(new[0]) +#else + "b"(new[0]) +#endif + : "cc", "memory"); + + ], + [gcc_dw_cmpxchg_asm=yes]) + if test $gcc_dw_cmpxchg_asm = no && test $ac_cv_sizeof_void_p = 4; then + AC_TRY_COMPILE([], + [ + char xchgd; + long new[2], xchg[2], *p; +#if !defined(__PIC__) || !__PIC__ +# error nope +#endif + __asm__ __volatile__( + "pushl %%ebx\n\t" + "movl (%7), %%ebx\n\t" + "movl 4(%7), %%ecx\n\t" + "lock; cmpxchg8b %0\n\t" + "setz %3\n\t" + "popl %%ebx\n\t" + : "=m"(*p), "=d"(xchg[1]), "=a"(xchg[0]), "=c"(xchgd) + : "m"(*p), "1"(xchg[1]), "2"(xchg[0]), "3"(new) + : "cc", "memory"); + ], - [enable_ethread_pre_pentium4_compatibility=no], - [enable_ethread_pre_pentium4_compatibility=yes], - [enable_ethread_pre_pentium4_compatibility=no]) - AC_MSG_RESULT([$enable_ethread_pre_pentium4_compatibility]);; + [gcc_dw_cmpxchg_asm=yes]) + if test "$gcc_dw_cmpxchg_asm" = "yes"; then + AC_DEFINE(ETHR_CMPXCHG8B_REGISTER_SHORTAGE, 1, [Define if you get a register shortage with cmpxchg8b and position independent code]) + fi + fi + AC_MSG_RESULT([$gcc_dw_cmpxchg_asm]) + if test "$gcc_dw_cmpxchg_asm" = "yes"; then + AC_DEFINE(ETHR_GCC_HAVE_DW_CMPXCHG_ASM_SUPPORT, 1, [Define if you use a gcc that supports the double word cmpxchg instruction]) + fi;; *) ;; esac -test $enable_ethread_pre_pentium4_compatibility = yes && - AC_DEFINE(ETHR_PRE_PENTIUM4_COMPAT, 1, [Define if you want compatibility with x86 processors before pentium4.]) - AC_DEFINE(ETHR_HAVE_ETHREAD_DEFINES, 1, \ [Define if you have all ethread defines]) @@ -1312,6 +1375,7 @@ AC_SUBST(ETHR_LIB_NAME) AC_SUBST(ETHR_DEFS) AC_SUBST(ETHR_THR_LIB_BASE) AC_SUBST(ETHR_THR_LIB_BASE_DIR) +AC_SUBST(ETHR_X86_SSE2_ASM) ]) @@ -1582,11 +1646,11 @@ dnl Freely inspired by AC_TRY_LINK. (Maybe better to create a dnl AC_LANG_JAVA instead...) AC_DEFUN(ERL_TRY_LINK_JAVA, [java_link='$JAVAC conftest.java 1>&AC_FD_CC' -changequote(�, �)dnl +changequote(, )dnl cat > conftest.java <<EOF -�$1� +$1 class conftest { public static void main(String[] args) { - �$2� + $2 ; return; }} EOF changequote([, ])dnl diff --git a/erts/autoconf/win32.config.cache.static b/erts/autoconf/win32.config.cache.static index d25b1df9d9..b387db2b22 100755 --- a/erts/autoconf/win32.config.cache.static +++ b/erts/autoconf/win32.config.cache.static @@ -96,7 +96,6 @@ ac_cv_func_sbrk=${ac_cv_func_sbrk=no} ac_cv_func_select=${ac_cv_func_select=no} ac_cv_func_setlocale=${ac_cv_func_setlocale=yes} ac_cv_func_setsid=${ac_cv_func_setsid=no} -ac_cv_func_setvbuf_reversed=${ac_cv_func_setvbuf_reversed=yes} ac_cv_func_socket=${ac_cv_func_socket=no} ac_cv_func_strchr=${ac_cv_func_strchr=yes} ac_cv_func_strerror=${ac_cv_func_strerror=yes} @@ -124,7 +123,6 @@ ac_cv_header_ieeefp_h=${ac_cv_header_ieeefp_h=no} ac_cv_header_inttypes_h=${ac_cv_header_inttypes_h=no} ac_cv_header_langinfo_h=${ac_cv_header_langinfo_h=no} ac_cv_header_limits_h=${ac_cv_header_limits_h=yes} -ac_cv_header_mach_o_dyld_h=${ac_cv_header_mach_o_dyld_h=no} ac_cv_header_malloc_h=${ac_cv_header_malloc_h=yes} ac_cv_header_memory_h=${ac_cv_header_memory_h=yes} ac_cv_header_net_errno_h=${ac_cv_header_net_errno_h=no} diff --git a/erts/configure.in b/erts/configure.in index 3b9be41345..fafa1c7e92 100644 --- a/erts/configure.in +++ b/erts/configure.in @@ -259,13 +259,6 @@ AS_HELP_STRING([--enable-m32-build], esac ],enable_m32_build=no) -AC_ARG_ENABLE(fixalloc, -AS_HELP_STRING([--disable-fixalloc], [disable the use of fix_alloc])) -if test x${enable_fixalloc} = xno ; then - AC_DEFINE(NO_FIX_ALLOC,[], - [Define if you don't want the fix allocator in Erlang]) -fi - AC_SUBST(PERFCTR_PATH) AC_ARG_WITH(perfctr, AS_HELP_STRING([--with-perfctr=PATH], @@ -914,16 +907,6 @@ fi AC_SUBST(ERLANG_OSTYPE) -dnl Which sysv4 would this be, and what is it for??? -dnl XXX: replace with feature tests. -case $host_os in - sysv4*) - AC_DEFINE(SOCKOPT_CONNECT_STAT,[],[Obscure SYSV feature]) - AC_DEFINE(NO_PRAGMA_WEAK,[],[Obscure SYSV feature]) - LIBS="$LIBS -lgen -lc -L /usr/ucblib -lucb" - ;; -esac - # Check how to export functions from the emulator executable, needed # when dynamically loaded drivers are loaded (so that they can find # emulator functions). @@ -1484,7 +1467,7 @@ AC_CHECK_HEADERS(fcntl.h limits.h unistd.h syslog.h dlfcn.h ieeefp.h \ sys/types.h sys/stropts.h sys/sysctl.h \ sys/ioctl.h sys/time.h sys/uio.h \ sys/socket.h sys/sockio.h sys/socketio.h \ - net/errno.h malloc.h mach-o/dyld.h arpa/nameser.h \ + net/errno.h malloc.h arpa/nameser.h \ pty.h util.h utmp.h langinfo.h poll.h sdkddkver.h) AC_CHECK_HEADER(sys/resource.h, @@ -1809,11 +1792,6 @@ AC_CHECK_FUNCS([ieee_handler fpsetmask finite isnan isinf res_gethostbyname dlop AC_CHECK_DECLS([posix2time],,,[#include <time.h>]) -if test "X$host" = "Xwin32"; then - ac_cv_func_setvbuf_reversed=yes -fi -AC_FUNC_SETVBUF_REVERSED - disable_vfork=false if test "x$EMU_THR_LIB_NAME" != "x"; then AC_MSG_CHECKING([if vfork is known to hang multithreaded applications]) @@ -4312,7 +4290,6 @@ dnl Note that the output files are relative to $srcdir AC_OUTPUT( emulator/$host/Makefile:emulator/Makefile.in emulator/zlib/$host/Makefile:emulator/zlib/Makefile.in - emulator/pcre/$host/Makefile:emulator/pcre/Makefile.in epmd/src/$host/Makefile:epmd/src/Makefile.in etc/common/$host/Makefile:etc/common/Makefile.in include/internal/$host/ethread.mk:include/internal/ethread.mk.in @@ -4326,7 +4303,7 @@ dnl The ones below should be moved to their respective lib dnl ../lib/ic/c_src/$host/Makefile:../lib/ic/c_src/Makefile.in ../lib/os_mon/c_src/$host/Makefile:../lib/os_mon/c_src/Makefile.in - ../lib/ssl/c_src/$host/Makefile:../lib/ssl/c_src/Makefile.in +dnl ../lib/ssl/c_src/$host/Makefile:../lib/ssl/c_src/Makefile.in ../lib/crypto/c_src/$host/Makefile:../lib/crypto/c_src/Makefile.in ../lib/orber/c_src/$host/Makefile:../lib/orber/c_src/Makefile.in ../lib/runtime_tools/c_src/$host/Makefile:../lib/runtime_tools/c_src/Makefile.in diff --git a/erts/doc/src/absform.xml b/erts/doc/src/absform.xml index 4c84412dd6..88e8b284fb 100644 --- a/erts/doc/src/absform.xml +++ b/erts/doc/src/absform.xml @@ -285,7 +285,8 @@ <item>If E is <c><![CDATA[fun Name / Arity]]></c>, then Rep(E) = <c><![CDATA[{'fun',LINE,{function,Name,Arity}}]]></c>.</item> <item>If E is <c><![CDATA[fun Module:Name/Arity]]></c>, then - Rep(E) = <c><![CDATA[{'fun',LINE,{function,Module,Name,Arity}}]]></c>.</item> + Rep(E) = <c><![CDATA[{'fun',LINE,{function,Rep(Module),Rep(Name),Rep(Arity)}}]]></c>. + (Before the R15 release: Rep(E) = <c><![CDATA[{'fun',LINE,{function,Module,Name,Arity}}]]></c>.)</item> <item>If E is <c><![CDATA[fun Fc_1 ; ... ; Fc_k end]]></c> where each <c><![CDATA[Fc_i]]></c> is a function clause then Rep(E) = <c><![CDATA[{'fun',LINE,{clauses,[Rep(Fc_1), ..., Rep(Fc_k)]}}]]></c>.</item> diff --git a/erts/doc/src/erl.xml b/erts/doc/src/erl.xml index 02082e57c6..39c79a29df 100644 --- a/erts/doc/src/erl.xml +++ b/erts/doc/src/erl.xml @@ -587,6 +587,13 @@ <p>Enables auto load tracing, displaying info while loading code.</p> </item> + <tag><c><![CDATA[+L]]></c></tag> + <item> + <p>Don't load information about source filenames and line numbers. + This will save some memory, but exceptions will not contain + information about the filenames and line numbers. + </p> + </item> <tag><marker id="erts_alloc"><c><![CDATA[+MFlag Value]]></c></marker></tag> <item> <p>Memory allocator specific flags, see @@ -987,7 +994,7 @@ the <c><![CDATA[-extra]]></c> section, i.e. the end of the command line following after an <c><![CDATA[-extra]]></c> flag.</p> </item> - <tag><c><![CDATA[ERL_ZFLAGS]]></c>and <c><![CDATA[ERL_FLAGS]]></c></tag> + <tag><c><![CDATA[ERL_ZFLAGS]]></c> and <c><![CDATA[ERL_FLAGS]]></c></tag> <item> <p>The content of these environment variables will be added to the end of the command line for <c><![CDATA[erl]]></c>.</p> diff --git a/erts/doc/src/erl_driver.xml b/erts/doc/src/erl_driver.xml index 2fb03954b6..8e18dd6657 100644 --- a/erts/doc/src/erl_driver.xml +++ b/erts/doc/src/erl_driver.xml @@ -1638,12 +1638,19 @@ ERL_DRV_EXT2TERM char *buf, ErlDrvUInt len <fsummary>Cancel an asynchronous call</fsummary> <desc> <marker id="driver_async_cancel"></marker> - <p>This function cancels an asynchronous operation, by removing - it from the queue. Only functions in the queue can be - cancelled; if a function is executing, it's too late to - cancel it. The <c>async_free</c> function is also called.</p> - <p>The return value is 1 if the operation was removed from the - queue, otherwise 0.</p> + <p>This function used to cancel a scheduled asynchronous operation, + if it was still in the queue. It returned 1 if it succeeded, and + 0 if it failed.</p> + <p>Since it could not guarantee success, it was more or less useless. + The user had to implement synchronization of cancellation anyway. + It also unnecessarily complicated the implementation. Therefore, + as of OTP-R15B <c>driver_async_cancel()</c> is deprecated, and + scheduled for removal in OTP-R16. It will currently always fail, + and return 0.</p> + <warning><p><c>driver_async_cancel()</c> is deferred and will + be removed in the OTP-R16 release.</p> + </warning> + </desc> </func> <func> diff --git a/erts/doc/src/erl_ext_fig.gif b/erts/doc/src/erl_ext_fig.gif Binary files differindex 14d6bbc871..14d6bbc871 100755..100644 --- a/erts/doc/src/erl_ext_fig.gif +++ b/erts/doc/src/erl_ext_fig.gif diff --git a/erts/doc/src/erl_nif.xml b/erts/doc/src/erl_nif.xml index 48839e9081..8daa67aa87 100644 --- a/erts/doc/src/erl_nif.xml +++ b/erts/doc/src/erl_nif.xml @@ -692,6 +692,10 @@ typedef enum { <fsummary>Determine if a term is an exception</fsummary> <desc><p>Return true if <c>term</c> is an exception.</p></desc> </func> + <func><name><ret>int</ret><nametext>enif_is_number(ErlNifEnv* env, ERL_NIF_TERM term)</nametext></name> + <fsummary>Determine if a term is a number (integer or float)</fsummary> + <desc><p>Return true if <c>term</c> is a number.</p></desc> + </func> <func><name><ret>int</ret><nametext>enif_is_fun(ErlNifEnv* env, ERL_NIF_TERM term)</nametext></name> <fsummary>Determine if a term is a fun</fsummary> <desc><p>Return true if <c>term</c> is a fun.</p></desc> @@ -822,6 +826,13 @@ typedef enum { <desc><p>Create an ordinary list containing the elements of array <c>arr</c> of length <c>cnt</c>. An empty list is returned if <c>cnt</c> is 0.</p></desc> </func> + <func><name><ret>int</ret><nametext>enif_make_reverse_list(ErlNifEnv* env, ERL_NIF_TERM term, ERL_NIF_TERM *list)</nametext></name> + <fsummary>Create the reverse list of the list <c>term</c>.</fsummary> + <desc><p>Set <c>*list</c> to the reverse list of the list <c>term</c> and return true, + or return false if <c>term</c> is not a list. This function should only be used on + short lists as a copy will be created of the list which will not be released until after the + nif returns.</p></desc> + </func> <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_long(ErlNifEnv* env, long int i)</nametext></name> <fsummary>Create an integer term from a long int</fsummary> <desc><p>Create an integer term from a <c>long int</c>.</p></desc> diff --git a/erts/doc/src/erlang.xml b/erts/doc/src/erlang.xml index aef31f5b98..b923a5f1fc 100644 --- a/erts/doc/src/erlang.xml +++ b/erts/doc/src/erlang.xml @@ -550,15 +550,6 @@ false</pre> </desc> </func> <func> - <name name="concat_binary" arity="1"/> - <fsummary>Concatenate a list of binaries (deprecated)</fsummary> - <desc> - <p>Do not use; use - <seealso marker="#list_to_binary/1">list_to_binary/1</seealso> - instead.</p> - </desc> - </func> - <func> <name>erlang:crc32(Data) -> integer() >= 0</name> <fsummary>Compute crc32 (IEEE 802.3) checksum</fsummary> <type> @@ -811,8 +802,7 @@ false</pre> <type> <v>MonitorRef = reference()</v> <v>OptionList = [Option]</v> - <v>Option = flush</v> - <v>Option = info</v> + <v> Option = flush | info</v> </type> <desc> <p>The returned value is <c>true</c> unless <c>info</c> is part @@ -1205,11 +1195,16 @@ true </item> <tag><c>{new_uniq, Uniq}</c></tag> <item> - <p><c>Uniq</c> (a binary) is a unique value for this fun.</p> + <p><c>Uniq</c> (a binary) is a unique value for this fun. + It is calculated from the compiled code for the entire module.</p> </item> <tag><c>{uniq, Uniq}</c></tag> <item> - <p><c>Uniq</c> (an integer) is a unique value for this fun.</p> + <p><c>Uniq</c> (an integer) is a unique value for this fun. + Starting in the R15 release, this integer is calculated from + the compiled code for the entire module. Before R15, this + integer was based on only the body of the fun. + </p> </item> </taglist> </desc> @@ -1353,17 +1348,18 @@ true </desc> </func> <func> - <name>erlang:get_stacktrace() -> [{Module, Function, Arity | Args}]</name> + <name>erlang:get_stacktrace() -> [{Module, Function, Arity | Args, Location}]</name> <fsummary>Get the call stack back-trace of the last exception</fsummary> <type> <v>Module = Function = atom()</v> <v>Arity = arity()</v> <v>Args = [term()]</v> + <v>Location = [{atom(),term()}]</v> </type> <desc> <p>Get the call stack back-trace (<em>stacktrace</em>) of the last exception in the calling process as a list of - <c>{Module,Function,Arity}</c> tuples. + <c>{Module,Function,Arity,Location}</c> tuples. The <c>Arity</c> field in the first tuple may be the argument list of that function call instead of an arity integer, depending on the exception.</p> @@ -1373,6 +1369,25 @@ true <p>The stacktrace is the same data as the <c>catch</c> operator returns, for example:</p> <p><c>{'EXIT',{badarg,Stacktrace}} = catch abs(x)</c></p> + <p><c>Location</c> is a (possibly empty) list of two-tuples that + may indicate the location in the source code of the function. + The first element is an atom that describes the type of + information in the second element. Currently the following + items may occur:</p> + <taglist> + <tag><c>file</c></tag> + <item> + <p>The second element of the tuple is a string (list of + characters) representing the filename of the source file + of the function.</p> + </item> + <tag><c>line</c></tag> + <item> + <p>The second element of the tuple is the line number + (an integer greater than zero) in the source file + where the exception occurred or the function was called.</p> + </item> + </taglist> <p>See also <seealso marker="#error/1">erlang:error/1</seealso> and <seealso marker="#error/2">erlang:error/2</seealso>.</p> @@ -3880,11 +3895,26 @@ os_prompt%</pre> catches in this process. This <c>InfoTuple</c> may be changed or removed without prior notice.</p> </item> - <tag><c>{current_function, {Module, Function, Args}}</c></tag> + <tag><c>{current_function, {Module, Function, Arity}}</c></tag> <item> - <p><c>Module</c>, <c>Function</c>, <c>Args</c> is + <p><c>Module</c>, <c>Function</c>, <c>Arity</c> is the current function call of the process.</p> </item> + <tag><c>{current_location, {Module, Function, Arity, Location}}</c></tag> + <item> + <p><c>Module</c>, <c>Function</c>, <c>Arity</c> is + the current function call of the process. + <c>Location</c> is a list of two-tuples that describes the + location in the source code. + </p> + </item> + <tag><c>{current_stacktrace, Stack}</c></tag> + <item> + <p>Return the current call stack back-trace (<em>stacktrace</em>) + of the process. The stack has the same format as returned by + <seealso marker="#get_stacktrace/1">erlang:get_stacktrace/0</seealso>. + </p> + </item> <tag><c>{dictionary, Dictionary}</c></tag> <item> <p><c>Dictionary</c> is the dictionary of the process.</p> @@ -4151,11 +4181,14 @@ os_prompt%</pre> equivalent to <c>erlang:Class(Reason)</c>. <c>Reason</c> is any term and <c>Stacktrace</c> is a list as returned from <c>get_stacktrace()</c>, that is a list of - 3-tuples <c>{Module, Function, Arity | Args}</c> where - <c>Module</c> and <c>Function</c> are atoms and the third - element is an integer arity or an argument list. The - stacktrace may also contain <c>{Fun, Args}</c> tuples where + 4-tuples <c>{Module, Function, Arity | Args, + Location}</c> where <c>Module</c> and <c>Function</c> + are atoms and the third element is an integer arity or an + argument list. The stacktrace may also contain <c>{Fun, + Args, Location}</c> tuples where <c>Fun</c> is a local fun and <c>Args</c> is an argument list.</p> + <p>The <c>Location</c> element at the end is optional. + Omitting it is equivalent to specifying an empty list.</p> <p>The stacktrace is used as the exception stacktrace for the calling process; it will be truncated to the current maximum stacktrace depth.</p> diff --git a/erts/doc/src/erts_alloc.xml b/erts/doc/src/erts_alloc.xml index 86e1e5168a..3b5ee5391c 100644 --- a/erts/doc/src/erts_alloc.xml +++ b/erts/doc/src/erts_alloc.xml @@ -58,11 +58,8 @@ <item>Allocator used for memory blocks that are expected to be long-lived, for example Erlang code.</item> <tag><c>fix_alloc</c></tag> - <item>A very fast allocator used for some fix-sized - data. <c>fix_alloc</c> manages a set of memory pools from - which memory blocks are handed out. <c>fix_alloc</c> - allocates memory pools from <c>ll_alloc</c>. Memory pools - that have been allocated are never deallocated.</item> + <item>A fast allocator used for some frequently used + fixed size data types.</item> <tag><c>std_alloc</c></tag> <item>Allocator used for most memory blocks not allocated via any of the other allocators described above.</item> @@ -83,7 +80,7 @@ where only small blocks are placed. Currently this allocator is disabled by default.</item> </taglist> - <p><c>sys_alloc</c> and <c>fix_alloc</c> are always enabled and + <p><c>sys_alloc</c> is always enabled and cannot be disabled. <c>mseg_alloc</c> is always enabled if it is available and an allocator that uses it is enabled. All other allocators can be <seealso marker="#M_e">enabled or disabled</seealso>. @@ -104,7 +101,7 @@ <marker id="alloc_util"></marker> <title>The alloc_util framework</title> <p>Internally a framework called <c>alloc_util</c> is used for - implementing allocators. <c>sys_alloc</c>, <c>fix_alloc</c>, and + implementing allocators. <c>sys_alloc</c>, and <c>mseg_alloc</c> do not use this framework; hence, the following does <em>not</em> apply to them.</p> <p>An allocator manages multiple areas, called carriers, in which @@ -212,6 +209,14 @@ This since it will only cause problems for other allocators.</p> </item> </taglist> + <p>Apart from the ordinary allocators described above a number of + pre-allocators are used for some specific data types. These + pre-allocators pre-allocate a fixed amount of memory for certain data + types when the run-time system starts. As long as there are available + pre-allocated memory, it will be used. When no pre-allocated memory is + available, memory will be allocated in ordinary allocators. These + pre-allocators are typically much faster than the ordinary allocators, + but can only satisfy a limited amount of requests.</p> </section> <note><p> @@ -272,18 +277,6 @@ Max cached segments. The maximum number of memory segments stored in the memory segment cache. Valid range is 0-30. Default value is 5.</item> - <tag><marker id="MMcci"><c><![CDATA[+MMcci <time>]]></c></marker></tag> - <item> - Cache check interval (in milliseconds). The memory segment - cache is checked for segments to destroy at an interval - determined by this parameter. Default value is 1000.</item> - </taglist> - <p>The following flags are available for configuration of - <c>fix_alloc</c>:</p> - <taglist> - <tag><marker id="MFe"><c>+MFe true</c></marker></tag> - <item> - Enable <c>fix_alloc</c>. Note: <c>fix_alloc</c> cannot be disabled.</item> </taglist> <p>The following flags are available for configuration of <c>sys_alloc</c>:</p> @@ -322,7 +315,7 @@ based on <c>alloc_util</c>. If <c>u</c> is used as subsystem identifier (i.e., <c><![CDATA[<S> = u]]></c>) all allocators based on <c>alloc_util</c> will be effected. If <c>B</c>, <c>D</c>, <c>E</c>, - <c>H</c>, <c>L</c>, <c>R</c>, <c>S</c>, or <c>T</c> is used as + <c>F</c>, <c>H</c>, <c>L</c>, <c>R</c>, <c>S</c>, or <c>T</c> is used as subsystem identifier, only the specific allocator identified will be effected:</p> <taglist> @@ -441,26 +434,23 @@ kilobytes). See <seealso marker="#mseg_mbc_sizes">the description on how sizes for mseg_alloc multiblock carriers are decided</seealso> in "the <c>alloc_util</c> framework" section.</item> - <tag><marker id="M_t"><c><![CDATA[+M<S>t true|false|<amount>]]></c></marker></tag> + <tag><marker id="M_t"><c><![CDATA[+M<S>t true|false]]></c></marker></tag> <item> - <p>Multiple, thread specific instances of the allocator. - This option will only have any effect on the runtime system - with SMP support. Default behaviour on the runtime system with - SMP support (<c>N</c> equals the number of scheduler threads):</p> + Multiple, thread specific instances of the allocator. + This option will only have any effect on the runtime system + with SMP support. Default behaviour on the runtime system with + SMP support: <taglist> - <tag><c>temp_alloc</c></tag> - <item><c>N + 1</c> instances.</item> <tag><c>ll_alloc</c></tag> <item><c>1</c> instance.</item> <tag>Other allocators</tag> - <item><c>N</c> instances when <c>N</c> is less than or equal to - <c>16</c>. <c>16</c> instances when <c>N</c> is greater than - <c>16</c>.</item> + <item><c>NoSchedulers+1</c> instances. Each scheduler will use + a lock-free instance of its own and other threads will use + a common instance.</item> </taglist> - <p><c>temp_alloc</c> will always use <c>N + 1</c> instances when - this option has been enabled regardless of the amount passed. - Other allocators will use the same amount of instances as the - amount passed as long as it isn't greater than <c>N</c>.</p> + It was previously (before ERTS version 5.9) possible to configure + a smaller amount of thread specific instances than schedulers. + This is, however, not possible any more. </item> </taglist> <p>Currently the following flags are available for configuration of diff --git a/erts/doc/src/make.dep b/erts/doc/src/make.dep deleted file mode 100644 index 98bac78235..0000000000 --- a/erts/doc/src/make.dep +++ /dev/null @@ -1,32 +0,0 @@ -# ---------------------------------------------------- -# >>>> Do not edit this file <<<< -# This file was automaticly generated by -# /home/gandalf/otp/bin/docdepend -# ---------------------------------------------------- - - -# ---------------------------------------------------- -# TeX files that the DVI file depend on -# ---------------------------------------------------- - -book.dvi: absform.tex alt_dist.tex book.tex crash_dump.tex \ - driver.tex driver_entry.tex epmd.tex erl.tex \ - erl_dist_protocol.tex erl_driver.tex erl_ext_dist.tex \ - erl_prim_loader.tex erl_set_memory_block.tex \ - erlang.tex erlc.tex erlsrv.tex erts_alloc.tex \ - escript.tex inet_cfg.tex init.tex match_spec.tex \ - part.tex ref_man.tex run_erl.tex start.tex \ - start_erl.tex tty.tex werl.tex zlib.tex - -# ---------------------------------------------------- -# Source inlined when transforming from source to LaTeX -# ---------------------------------------------------- - -book.tex: ref_man.xml - -# ---------------------------------------------------- -# Pictures that the DVI file depend on -# ---------------------------------------------------- - -book.dvi: erl_ext_fig.ps - diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in index 620402fbfb..708d4ca0a3 100644 --- a/erts/emulator/Makefile.in +++ b/erts/emulator/Makefile.in @@ -291,7 +291,8 @@ else LIBS += $(ERL_TOP)/erts/emulator/pcre/obj/$(TARGET)/$(TYPE)/$(LIB_PREFIX)epcre$(LIB_SUFFIX) endif -DEPLIBS += $(ERL_TOP)/erts/emulator/pcre/obj/$(TARGET)/$(TYPE)/$(LIB_PREFIX)epcre$(LIB_SUFFIX) +EPCRE_LIB = $(ERL_TOP)/erts/emulator/pcre/obj/$(TARGET)/$(TYPE)/$(LIB_PREFIX)epcre$(LIB_SUFFIX) +DEPLIBS += $(EPCRE_LIB) PERFCTR_PATH=@PERFCTR_PATH@ USE_PERFCTR=@USE_PERFCTR@ @@ -382,7 +383,7 @@ ifeq ($(FLAVOR)-@ERTS_BUILD_SMP_EMU@,smp-no) all: @echo '*** Omitted build of emulator with smp support' else -all: generate erts_lib zlib pcre $(BINDIR)/$(EMULATOR_EXECUTABLE) $(UNIX_ONLY_BUILDS) +all: generate erts_lib zlib $(BINDIR)/$(EMULATOR_EXECUTABLE) $(UNIX_ONLY_BUILDS) ifeq ($(OMIT_OMIT_FP),yes) @echo '* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *' @echo '* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *' @@ -403,8 +404,8 @@ zlib: @set -e ; cd zlib && $(MAKE) TYPE=$(TYPE) $(TYPE) endif -pcre: - @set -e ; cd pcre && $(MAKE) TYPE=$(TYPE) $(TYPE) + +include pcre/pcre.mk erts_lib: cd $(ERL_TOP)/erts/lib_src && $(MAKE) $(TYPE) @@ -420,9 +421,9 @@ endif $(RM) -rf $(BINDIR)/child_setup $(BINDIR)/child_setup.* $(RM) -f $(BINDIR)/hipe_mkliterals $(BINDIR)/hipe_mkliterals.* @set -e ; cd zlib && $(MAKE) clean - @set -e ; cd pcre && $(MAKE) clean + rm -f $(OBJS) $(OBJDIR)/libepcre.a -.PHONY: all zlib pcre clean +.PHONY: all zlib clean docs: @@ -467,10 +468,11 @@ release_docs_spec: # Generated source code. Put in $(TARGET) directory # +_create_dirs := $(shell mkdir -p $(CREATE_DIRS)) + .PHONY : generate -GENERATE= $(CREATE_DIRS) \ - $(TTF_DIR)/beam_opcodes.h \ +GENERATE= $(TTF_DIR)/beam_opcodes.h \ $(TARGET)/erl_bif_table.c \ $(TARGET)/erl_version.h \ $(TTF_DIR)/driver_tab.c \ @@ -672,14 +674,8 @@ endif # rebuilding (is this a good idea?) add a dummy dependency to this target. # -ifeq ($(findstring clearmake,$(MAKE)),clearmake) -BEAMFILE_MAKEFLAG=-T -else -BEAMFILE_MAKEFLAG= -endif - $(ERL_TOP)/lib/%.beam: - cd $(@D)/../src && $(MAKE) $(BEAMFILE_MAKEFLAG) ../ebin/$(@F) + cd $(@D)/../src && $(MAKE) ../ebin/$(@F) # ---------------------------------------------------------------------- @@ -725,7 +721,7 @@ RUN_OBJS = \ $(OBJDIR)/external.o $(OBJDIR)/dist.o \ $(OBJDIR)/binary.o $(OBJDIR)/erl_db.o \ $(OBJDIR)/erl_db_util.o $(OBJDIR)/erl_db_hash.o \ - $(OBJDIR)/erl_db_tree.o $(OBJDIR)/fix_alloc.o \ + $(OBJDIR)/erl_db_tree.o $(OBJDIR)/erl_thr_progress.o \ $(OBJDIR)/big.o $(OBJDIR)/hash.o \ $(OBJDIR)/index.o $(OBJDIR)/atom.o \ $(OBJDIR)/module.o $(OBJDIR)/export.o \ @@ -742,7 +738,8 @@ RUN_OBJS = \ $(OBJDIR)/erl_bif_re.o $(OBJDIR)/erl_unicode.o \ $(OBJDIR)/packet_parser.o $(OBJDIR)/safe_hash.o \ $(OBJDIR)/erl_zlib.o $(OBJDIR)/erl_nif.o \ - $(OBJDIR)/erl_bif_binary.o $(OBJDIR)/erl_ao_firstfit_alloc.o + $(OBJDIR)/erl_bif_binary.o $(OBJDIR)/erl_ao_firstfit_alloc.o \ + $(OBJDIR)/erl_thr_queue.o $(OBJDIR)/erl_sched_spec_pre_alloc.o ifeq ($(TARGET),win32) DRV_OBJS = \ @@ -857,7 +854,7 @@ $(OBJDIR)/%.o: hipe/%.c $(BINDIR)/hipe_mkliterals$(TF_MARKER): $(OBJDIR)/hipe_mkliterals.o $(CC) $(CFLAGS) $(INCLUDES) -o $@ $< -$(OBJDIR)/hipe_mkliterals.o: $(TTF_DIR)/hipe_x86_asm.h $(TTF_DIR)/hipe_ppc_asm.h +$(OBJDIR)/hipe_mkliterals.o: $(TTF_DIR)/hipe_x86_asm.h $(TTF_DIR)/hipe_ppc_asm.h $(TTF_DIR)/beam_opcodes.h $(TTF_DIR)/hipe_literals.h: $(BINDIR)/hipe_mkliterals$(TF_MARKER) $(BINDIR)/hipe_mkliterals$(TF_MARKER) -c > $@ diff --git a/erts/emulator/beam/atom.c b/erts/emulator/beam/atom.c index b97705ed96..d7c7f117cf 100644 --- a/erts/emulator/beam/atom.c +++ b/erts/emulator/beam/atom.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * Copyright Ericsson AB 1996-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -75,7 +75,7 @@ void atom_info(int to, void *to_arg) index_info(to, to_arg, &erts_atom_table); #ifdef ERTS_ATOM_PUT_OPS_STAT erts_print(to, to_arg, "atom_put_ops: %ld\n", - erts_smp_atomic_read(&atom_put_ops)); + erts_smp_atomic_read_nob(&atom_put_ops)); #endif if (lock) @@ -213,7 +213,7 @@ am_atom_put(const char* name, int len) len = MAX_ATOM_LENGTH; } #ifdef ERTS_ATOM_PUT_OPS_STAT - erts_smp_atomic_inc(&atom_put_ops); + erts_smp_atomic_inc_nob(&atom_put_ops); #endif a.len = len; a.name = (byte*)name; @@ -309,7 +309,7 @@ init_atom_table(void) rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED; #ifdef ERTS_ATOM_PUT_OPS_STAT - erts_smp_atomic_init(&atom_put_ops, 0); + erts_smp_atomic_init_nob(&atom_put_ops, 0); #endif erts_smp_rwmtx_init_opt(&atom_table_lock, &rwmtx_opt, "atom_tab"); diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index 68d64fb7b0..71454b3e57 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -69,6 +69,8 @@ atom ac atom active atom all atom all_but_first +atom alloc_info +atom alloc_sizes atom allocated atom allocated_areas atom allocator @@ -156,6 +158,8 @@ atom cr atom crlf atom creation atom current_function +atom current_location +atom current_stacktrace atom data atom debug_flags atom delay_trap @@ -553,5 +557,6 @@ atom warning_msg atom wordsize atom write_concurrency atom xor +atom x86 atom yes atom yield diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 2561d7a630..294b1578be 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -33,6 +33,7 @@ #include "beam_catches.h" #include "erl_binary.h" #include "erl_nif.h" +#include "erl_thr_progress.h" static void set_default_trace_pattern(Eterm module); static Eterm check_process_code(Process* rp, Module* modp); @@ -64,9 +65,7 @@ load_module_2(BIF_ALIST_2) goto error; } erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(0); - - erts_export_consolidate(); + erts_smp_thr_progress_block(); hp = HAlloc(BIF_P, 3); sz = binary_size(BIF_ARG_2); @@ -103,7 +102,7 @@ load_module_2(BIF_ALIST_2) done: erts_free_aligned_binary_bytes(temp_alloc); - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); BIF_RET(res); @@ -118,12 +117,12 @@ BIF_RETTYPE purge_module_1(BIF_ALIST_1) } erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(0); + erts_smp_thr_progress_block(); erts_export_consolidate(); purge_res = purge_module(atom_val(BIF_ARG_1)); - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); if (purge_res < 0) { @@ -152,12 +151,12 @@ BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3) Eterm res; erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(0); + erts_smp_thr_progress_block(); erts_export_consolidate(); res = erts_make_stub_module(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); return res; } @@ -239,7 +238,7 @@ BIF_RETTYPE delete_module_1(BIF_ALIST_1) goto badarg; erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(0); + erts_smp_thr_progress_block(); { Module *modp = erts_get_module(BIF_ARG_1); @@ -260,7 +259,7 @@ BIF_RETTYPE delete_module_1(BIF_ALIST_1) } } - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); if (res == am_badarg) { @@ -352,7 +351,7 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) } erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(0); + erts_smp_thr_progress_block(); if (BIF_ARG_2 == am_true) { int i; @@ -391,7 +390,7 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) modp->catches = BEAM_CATCHES_NIL; remove_from_address_table(code); } - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); BIF_RET(am_true); } @@ -728,10 +727,10 @@ delete_code(Process *c_p, ErtsProcLocks c_p_locks, Module* modp) if (modp->code != NULL && modp->code[MI_NUM_BREAKPOINTS] > 0) { if (c_p && c_p_locks) erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(0); + erts_smp_thr_progress_block(); erts_clear_module_break(modp); modp->code[MI_NUM_BREAKPOINTS] = 0; - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); if (c_p && c_p_locks) erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); } diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c index 31910888d1..dd31376a2d 100644 --- a/erts/emulator/beam/beam_bp.c +++ b/erts/emulator/beam/beam_bp.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2000-2010. All Rights Reserved. + * Copyright Ericsson AB 2000-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -167,7 +167,7 @@ erts_bp_init(void) { int erts_set_trace_break(Eterm mfa[3], int specified, Binary *match_spec, Eterm tracer_pid) { - ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); return set_break(mfa, specified, match_spec, (BeamInstr) BeamOp(op_i_trace_breakpoint), 0, tracer_pid); } @@ -175,7 +175,7 @@ erts_set_trace_break(Eterm mfa[3], int specified, Binary *match_spec, int erts_set_mtrace_break(Eterm mfa[3], int specified, Binary *match_spec, Eterm tracer_pid) { - ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); return set_break(mfa, specified, match_spec, (BeamInstr) BeamOp(op_i_mtrace_breakpoint), 0, tracer_pid); } @@ -184,7 +184,7 @@ erts_set_mtrace_break(Eterm mfa[3], int specified, Binary *match_spec, void erts_set_mtrace_bif(BeamInstr *pc, Binary *match_spec, Eterm tracer_pid) { - ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); set_function_break(NULL, pc, BREAK_IS_BIF, match_spec, (BeamInstr) BeamOp(op_i_mtrace_breakpoint), 0, tracer_pid); } @@ -198,35 +198,35 @@ void erts_clear_time_trace_bif(BeamInstr *pc) { int erts_set_debug_break(Eterm mfa[3], int specified) { - ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); return set_break(mfa, specified, NULL, (BeamInstr) BeamOp(op_i_debug_breakpoint), 0, NIL); } int erts_set_count_break(Eterm mfa[3], int specified, enum erts_break_op count_op) { - ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); return set_break(mfa, specified, NULL, (BeamInstr) BeamOp(op_i_count_breakpoint), count_op, NIL); } int erts_set_time_break(Eterm mfa[3], int specified, enum erts_break_op count_op) { - ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); return set_break(mfa, specified, NULL, (BeamInstr) BeamOp(op_i_time_breakpoint), count_op, NIL); } int erts_clear_trace_break(Eterm mfa[3], int specified) { - ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); return clear_break(mfa, specified, (BeamInstr) BeamOp(op_i_trace_breakpoint)); } int erts_clear_mtrace_break(Eterm mfa[3], int specified) { - ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); return clear_break(mfa, specified, (BeamInstr) BeamOp(op_i_mtrace_breakpoint)); } @@ -238,41 +238,41 @@ erts_clear_mtrace_bif(BeamInstr *pc) { int erts_clear_debug_break(Eterm mfa[3], int specified) { - ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); return clear_break(mfa, specified, (BeamInstr) BeamOp(op_i_debug_breakpoint)); } int erts_clear_count_break(Eterm mfa[3], int specified) { - ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); return clear_break(mfa, specified, (BeamInstr) BeamOp(op_i_count_breakpoint)); } int erts_clear_time_break(Eterm mfa[3], int specified) { - ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); return clear_break(mfa, specified, (BeamInstr) BeamOp(op_i_time_breakpoint)); } int erts_clear_break(Eterm mfa[3], int specified) { - ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); return clear_break(mfa, specified, 0); } int erts_clear_module_break(Module *modp) { - ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); ASSERT(modp); return clear_module_break(modp, NULL, 0, 0); } int erts_clear_function_break(Module *modp, BeamInstr *pc) { - ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); ASSERT(modp); return clear_function_break(modp, pc, BREAK_IS_ERL, 0); } @@ -408,7 +408,7 @@ erts_is_count_break(BeamInstr *pc, Sint *count_ret) { if (bdc) { if (count_ret) { - *count_ret = (Sint) erts_smp_atomic_read(&bdc->acount); + *count_ret = (Sint) erts_smp_atomic_read_nob(&bdc->acount); } return !0; } @@ -612,9 +612,13 @@ static void bp_hash_delete(bp_time_hash_t *hash) { static void bp_time_diff(bp_data_time_item_t *item, /* out */ process_breakpoint_time_t *pbt, /* in */ Uint ms, Uint s, Uint us) { - int dms,ds,dus; + int ds,dus; +#ifdef DEBUG + int dms; + dms = ms - pbt->ms; +#endif ds = s - pbt->s; dus = us - pbt->us; @@ -622,7 +626,9 @@ static void bp_time_diff(bp_data_time_item_t *item, /* out */ * this is ok. */ +#ifdef DEBUG ASSERT(dms >= 0 || ds >= 0 || dus >= 0); +#endif if (dus < 0) { dus += 1000000; @@ -958,24 +964,24 @@ static int set_function_break(Module *modp, BeamInstr *pc, int bif, if (break_op == (BeamInstr) BeamOp(op_i_count_breakpoint)) { if (count_op == erts_break_stop) { - count = erts_smp_atomic_read(&bdc->acount); + count = erts_smp_atomic_read_nob(&bdc->acount); if (count >= 0) { while(1) { - res = erts_smp_atomic_cmpxchg(&bdc->acount, -count - 1, count); + res = erts_smp_atomic_cmpxchg_nob(&bdc->acount, -count - 1, count); if ((res == count) || count < 0) break; count = res; } } } else { /* Reset call counter */ - erts_smp_atomic_set(&bdc->acount, 0); + erts_smp_atomic_set_nob(&bdc->acount, 0); } } else if (break_op == (BeamInstr) BeamOp(op_i_time_breakpoint)) { BpDataTime *bdt = (BpDataTime *) bd; Uint i = 0; - ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); if (count_op == erts_break_stop) { bdt->pause = 1; @@ -1097,7 +1103,7 @@ static int set_function_break(Module *modp, BeamInstr *pc, int bif, } } else if (break_op == (BeamInstr) BeamOp(op_i_count_breakpoint)) { BpDataCount *bdc = (BpDataCount *) bd; - erts_smp_atomic_init(&bdc->acount, 0); + erts_smp_atomic_init_nob(&bdc->acount, 0); } if (bif == BREAK_IS_ERL) { diff --git a/erts/emulator/beam/beam_bp.h b/erts/emulator/beam/beam_bp.h index bd8a7249a7..2ec5818688 100644 --- a/erts/emulator/beam/beam_bp.h +++ b/erts/emulator/beam/beam_bp.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2000-2010. All Rights Reserved. + * Copyright Ericsson AB 2000-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -165,8 +165,8 @@ do { \ bdc = (BpDataCount *) bdc->next; \ ASSERT(bdc); \ bds[ix] = (BpData *) bdc; \ - count = erts_smp_atomic_read(&bdc->acount); \ - if (count >= 0) erts_smp_atomic_inc(&bdc->acount); \ + count = erts_smp_atomic_read_nob(&bdc->acount); \ + if (count >= 0) erts_smp_atomic_inc_nob(&bdc->acount); \ *(instr_result) = bdc->orig_instr; \ } while (0) diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c index fffb172c68..8041c92162 100644 --- a/erts/emulator/beam/beam_debug.c +++ b/erts/emulator/beam/beam_debug.c @@ -37,6 +37,7 @@ #include "beam_load.h" #include "beam_bp.h" #include "erl_binary.h" +#include "erl_thr_progress.h" #ifdef ARCH_64 # define HEXF "%016bpX" @@ -49,15 +50,18 @@ void dbg_bt(Process* p, Eterm* sp); void dbg_where(BeamInstr* addr, Eterm x0, Eterm* reg); static int print_op(int to, void *to_arg, int op, int size, BeamInstr* addr); -Eterm -erts_debug_same_2(Process* p, Eterm term1, Eterm term2) + +BIF_RETTYPE +erts_debug_same_2(BIF_ALIST_2) { - return (term1 == term2) ? am_true : am_false; + return (BIF_ARG_1 == BIF_ARG_2) ? am_true : am_false; } -Eterm -erts_debug_flat_size_1(Process* p, Eterm term) +BIF_RETTYPE +erts_debug_flat_size_1(BIF_ALIST_1) { + Process* p = BIF_P; + Eterm term = BIF_ARG_1; Uint size = size_object(term); if (IS_USMALL(0, size)) { @@ -68,9 +72,13 @@ erts_debug_flat_size_1(Process* p, Eterm term) } } -Eterm -erts_debug_breakpoint_2(Process* p, Eterm MFA, Eterm bool) + +BIF_RETTYPE +erts_debug_breakpoint_2(BIF_ALIST_2) { + Process* p = BIF_P; + Eterm MFA = BIF_ARG_1; + Eterm bool = BIF_ARG_2; Eterm* tp; Eterm mfa[3]; int i; @@ -107,7 +115,7 @@ erts_debug_breakpoint_2(Process* p, Eterm MFA, Eterm bool) } erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(0); + erts_smp_thr_progress_block(); if (bool == am_true) { res = make_small(erts_set_debug_break(mfa, specified)); @@ -115,7 +123,7 @@ erts_debug_breakpoint_2(Process* p, Eterm MFA, Eterm bool) res = make_small(erts_clear_debug_break(mfa, specified)); } - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); return res; @@ -175,9 +183,11 @@ erts_debug_instructions_0(BIF_ALIST_0) return res; } -Eterm -erts_debug_disassemble_1(Process* p, Eterm addr) +BIF_RETTYPE +erts_debug_disassemble_1(BIF_ALIST_1) { + Process* p = BIF_P; + Eterm addr = BIF_ARG_1; erts_dsprintf_buf_t *dsbufp; Eterm* hp; Eterm* tp; diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 937b3d9e53..9c5450bd48 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -36,6 +36,7 @@ #include "dist.h" #include "beam_bp.h" #include "beam_catches.h" +#include "erl_thr_progress.h" #ifdef HIPE #include "hipe_mode_switch.h" #include "hipe_bif1.h" @@ -70,7 +71,7 @@ do { \ } \ else \ erts_lc_check_exact(NULL, 0); \ - ERTS_SMP_LC_ASSERT(!ERTS_LC_IS_BLOCKING); \ + ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking()); \ } while (0) # define ERTS_SMP_REQ_PROC_MAIN_LOCK(P) \ if ((P)) erts_proc_lc_require_lock((P), ERTS_PROC_LOCK_MAIN) @@ -303,44 +304,6 @@ extern int count_instructions; PROCESS_MAIN_CHK_LOCKS((P)); \ ERTS_SMP_UNREQ_PROC_MAIN_LOCK((P)) -#if defined(HYBRID) -# define POST_BIF_GC_SWAPIN_0(_p, _res) \ - if (((_p)->mbuf) || (MSO(_p).overhead >= BIN_VHEAP_SZ(_p)) ) { \ - _res = erts_gc_after_bif_call((_p), (_res), NULL, 0); \ - } \ - SWAPIN - -# define POST_BIF_GC_SWAPIN(_p, _res, _regs, _arity) \ - if (((_p)->mbuf) || (MSO(_p).overhead >= BIN_VHEAP_SZ(_p)) ) { \ - _regs[0] = r(0); \ - _res = erts_gc_after_bif_call((_p), (_res), _regs, (_arity)); \ - r(0) = _regs[0]; \ - } \ - SWAPIN -#else -# define POST_BIF_GC_SWAPIN_0(_p, _res) \ - ERTS_SMP_REQ_PROC_MAIN_LOCK((_p)); \ - PROCESS_MAIN_CHK_LOCKS((_p)); \ - ERTS_VERIFY_UNUSED_TEMP_ALLOC((_p)); \ - if (((_p)->mbuf) || (MSO(_p).overhead >= BIN_VHEAP_SZ(_p)) ) { \ - _res = erts_gc_after_bif_call((_p), (_res), NULL, 0); \ - E = (_p)->stop; \ - } \ - HTOP = HEAP_TOP((_p)) - -# define POST_BIF_GC_SWAPIN(_p, _res, _regs, _arity) \ - ERTS_VERIFY_UNUSED_TEMP_ALLOC((_p)); \ - ERTS_SMP_REQ_PROC_MAIN_LOCK((_p)); \ - PROCESS_MAIN_CHK_LOCKS((_p)); \ - if (((_p)->mbuf) || (MSO(_p).overhead >= BIN_VHEAP_SZ(_p)) ) { \ - _regs[0] = r(0); \ - _res = erts_gc_after_bif_call((_p), (_res), _regs, (_arity)); \ - r(0) = _regs[0]; \ - E = (_p)->stop; \ - } \ - HTOP = HEAP_TOP((_p)) -#endif - #define db(N) (N) #define tb(N) (N) #define xb(N) (*(Eterm *) (((unsigned char *)reg) + (N))) @@ -794,11 +757,11 @@ extern int count_instructions; } \ } while (0) -#define IsFunction2(F, A, Action) \ - do { \ - if (is_function_2(c_p, F, A) != am_true ) {\ - Action; \ - } \ +#define IsFunction2(F, A, Action) \ + do { \ + if (erl_is_function(c_p, F, A) != am_true ) { \ + Action; \ + } \ } while (0) #define IsTupleOfArity(Src, Arity, Fail) \ @@ -1145,26 +1108,11 @@ void process_main(void) Eterm *tmp_big; /* Temporary buffer for small bignums if !HEAP_ON_C_STACK. */ #endif -#ifndef ERTS_SMP -#if !HALFWORD_HEAP - static Eterm save_reg[ERTS_X_REGS_ALLOCATED]; - /* X registers -- not used directly, but - * through 'reg', because using it directly - * needs two instructions on a SPARC, - * while using it through reg needs only - * one. - */ -#endif /* - * Floating point registers. - */ - static FloatDef freg[MAX_REG]; -#else - /* X regisers and floating point registers are located in + * X registers and floating point registers are located in * scheduler specific data. */ register FloatDef *freg; -#endif /* * For keeping the negative old value of 'reds' when call saving is active. @@ -1201,14 +1149,6 @@ void process_main(void) init_done = 1; goto init_emulator; } -#ifndef ERTS_SMP -#if !HALFWORD_HEAP - reg = save_reg; /* XXX: probably wastes a register on x86 */ -#else - /* Registers need to be heap allocated (correct memory range) for tracing to work */ - reg = erts_alloc(ERTS_ALC_T_BEAM_REGISTER, ERTS_X_REGS_ALLOCATED * sizeof(Eterm)); -#endif -#endif c_p = NULL; reds_used = 0; goto do_schedule1; @@ -1229,10 +1169,8 @@ void process_main(void) #endif ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); -#ifdef ERTS_SMP - reg = c_p->scheduler_data->save_reg; - freg = c_p->scheduler_data->freg; -#endif + reg = ERTS_PROC_GET_SCHDATA(c_p)->x_reg_array; + freg = ERTS_PROC_GET_SCHDATA(c_p)->f_reg_array; #if !HEAP_ON_C_STACK tmp_big = ERTS_PROC_GET_SCHDATA(c_p)->beam_emu_tmp_heap; #endif @@ -1566,9 +1504,17 @@ void process_main(void) PRE_BIF_SWAPOUT(c_p); c_p->fcalls = FCALLS - 1; - result = send_2(c_p, r(0), x(1)); + reg[0] = r(0); + result = erl_send(c_p, r(0), x(1)); PreFetch(0, next); - POST_BIF_GC_SWAPIN(c_p, result, reg, 2); + ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); + PROCESS_MAIN_CHK_LOCKS(c_p); + if (c_p->mbuf || MSO(c_p).overhead >= BIN_VHEAP_SZ(c_p)) { + result = erts_gc_after_bif_call(c_p, result, reg, 2); + r(0) = reg[0]; + E = c_p->stop; + } + HTOP = HEAP_TOP(c_p); FCALLS = c_p->fcalls; if (is_value(result)) { r(0) = result; @@ -1576,10 +1522,9 @@ void process_main(void) NextPF(0, next); } else if (c_p->freason == TRAP) { SET_CP(c_p, I+1); - SET_I(*((BeamInstr **) (BeamInstr) ((c_p)->def_arg_reg + 3))); + SET_I(c_p->i); SWAPIN; - r(0) = c_p->def_arg_reg[0]; - x(1) = c_p->def_arg_reg[1]; + r(0) = reg[0]; Dispatch(); } goto find_func_info; @@ -2234,16 +2179,16 @@ void process_main(void) OpCase(bif1_fbsd): { - Eterm (*bf)(Process*, Eterm); - Eterm arg; + Eterm (*bf)(Process*, Eterm*); + Eterm tmp_reg[1]; Eterm result; - GetArg1(2, arg); + GetArg1(2, tmp_reg[0]); bf = (BifFunction) Arg(1); c_p->fcalls = FCALLS; PROCESS_MAIN_CHK_LOCKS(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - result = (*bf)(c_p, arg); + result = (*bf)(c_p, tmp_reg); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); @@ -2262,17 +2207,17 @@ void process_main(void) OpCase(bif1_body_bsd): { - Eterm (*bf)(Process*, Eterm); + Eterm (*bf)(Process*, Eterm*); - Eterm arg; + Eterm tmp_reg[1]; Eterm result; - GetArg1(1, arg); + GetArg1(1, tmp_reg[0]); bf = (BifFunction) Arg(0); c_p->fcalls = FCALLS; PROCESS_MAIN_CHK_LOCKS(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - result = (*bf)(c_p, arg); + result = (*bf)(c_p, tmp_reg); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); @@ -2281,7 +2226,7 @@ void process_main(void) if (is_value(result)) { StoreBifResult(2, result); } - reg[0] = arg; + reg[0] = tmp_reg[0]; SWAPOUT; I = handle_error(c_p, I, reg, bf); goto post_error_handling; @@ -2405,14 +2350,15 @@ void process_main(void) */ OpCase(i_bif2_fbd): { - Eterm (*bf)(Process*, Eterm, Eterm); + Eterm tmp_reg[2] = {tmp_arg1, tmp_arg2}; + Eterm (*bf)(Process*, Eterm*); Eterm result; bf = (BifFunction) Arg(1); c_p->fcalls = FCALLS; PROCESS_MAIN_CHK_LOCKS(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - result = (*bf)(c_p, tmp_arg1, tmp_arg2); + result = (*bf)(c_p, tmp_reg); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); @@ -2430,13 +2376,14 @@ void process_main(void) */ OpCase(i_bif2_body_bd): { - Eterm (*bf)(Process*, Eterm, Eterm); + Eterm tmp_reg[2] = {tmp_arg1, tmp_arg2}; + Eterm (*bf)(Process*, Eterm*); Eterm result; bf = (BifFunction) Arg(0); PROCESS_MAIN_CHK_LOCKS(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - result = (*bf)(c_p, tmp_arg1, tmp_arg2); + result = (*bf)(c_p, tmp_reg); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); @@ -2456,77 +2403,9 @@ void process_main(void) * The most general BIF call. The BIF may build any amount of data * on the heap. The result is always returned in r(0). */ - OpCase(call_bif0_e): - { - Eterm (*bf)(Process*, BeamInstr*) = GET_BIF_ADDRESS(Arg(0)); - - PRE_BIF_SWAPOUT(c_p); - c_p->fcalls = FCALLS - 1; - if (FCALLS <= 0) { - save_calls(c_p, (Export *) Arg(0)); - } - - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - r(0) = (*bf)(c_p, I); - ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(r(0))); - ERTS_HOLE_CHECK(c_p); - POST_BIF_GC_SWAPIN_0(c_p, r(0)); - FCALLS = c_p->fcalls; - if (is_value(r(0))) { - CHECK_TERM(r(0)); - Next(1); - } - else if (c_p->freason == TRAP) { - goto call_bif_trap3; - } - - /* - * Error handling. SWAPOUT is not needed because it was done above. - */ - ASSERT(c_p->stop == E); - reg[0] = r(0); - I = handle_error(c_p, I, reg, bf); - goto post_error_handling; - } - - OpCase(call_bif1_e): - { - Eterm (*bf)(Process*, Eterm, BeamInstr*) = GET_BIF_ADDRESS(Arg(0)); - Eterm result; - BeamInstr *next; - - c_p->fcalls = FCALLS - 1; - if (FCALLS <= 0) { - save_calls(c_p, (Export *) Arg(0)); - } - PreFetch(1, next); - PRE_BIF_SWAPOUT(c_p); - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - result = (*bf)(c_p, r(0), I); - ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); - ERTS_HOLE_CHECK(c_p); - POST_BIF_GC_SWAPIN(c_p, result, reg, 1); - FCALLS = c_p->fcalls; - if (is_value(result)) { - r(0) = result; - CHECK_TERM(r(0)); - NextPF(1, next); - } else if (c_p->freason == TRAP) { - goto call_bif_trap3; - } - - /* - * Error handling. SWAPOUT is not needed because it was done above. - */ - ASSERT(c_p->stop == E); - reg[0] = r(0); - I = handle_error(c_p, I, reg, bf); - goto post_error_handling; - } - - OpCase(call_bif2_e): + OpCase(call_bif_e): { - Eterm (*bf)(Process*, Eterm, Eterm, BeamInstr*) = GET_BIF_ADDRESS(Arg(0)); + Eterm (*bf)(Process*, Eterm*, BeamInstr*) = GET_BIF_ADDRESS(Arg(0)); Eterm result; BeamInstr *next; @@ -2536,61 +2415,29 @@ void process_main(void) save_calls(c_p, (Export *) Arg(0)); } PreFetch(1, next); - CHECK_TERM(r(0)); - CHECK_TERM(x(1)); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - result = (*bf)(c_p, r(0), x(1), I); - ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); - ERTS_HOLE_CHECK(c_p); - POST_BIF_GC_SWAPIN(c_p, result, reg, 2); - FCALLS = c_p->fcalls; - if (is_value(result)) { - r(0) = result; - CHECK_TERM(r(0)); - NextPF(1, next); - } else if (c_p->freason == TRAP) { - goto call_bif_trap3; - } - - /* - * Error handling. SWAPOUT is not needed because it was done above. - */ - ASSERT(c_p->stop == E); reg[0] = r(0); - I = handle_error(c_p, I, reg, bf); - goto post_error_handling; - } - - OpCase(call_bif3_e): - { - Eterm (*bf)(Process*, Eterm, Eterm, Eterm, BeamInstr*) = GET_BIF_ADDRESS(Arg(0)); - Eterm result; - BeamInstr *next; - - PRE_BIF_SWAPOUT(c_p); - c_p->fcalls = FCALLS - 1; - if (FCALLS <= 0) { - save_calls(c_p, (Export *) Arg(0)); - } - PreFetch(1, next); - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - result = (*bf)(c_p, r(0), x(1), x(2), I); + result = (*bf)(c_p, reg, I); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); ERTS_HOLE_CHECK(c_p); - POST_BIF_GC_SWAPIN(c_p, result, reg, 3); + ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); + PROCESS_MAIN_CHK_LOCKS(c_p); + if (c_p->mbuf || MSO(c_p).overhead >= BIN_VHEAP_SZ(c_p)) { + Uint arity = ((Export *)Arg(0))->code[2]; + result = erts_gc_after_bif_call(c_p, result, reg, arity); + E = c_p->stop; + } + HTOP = HEAP_TOP(c_p); FCALLS = c_p->fcalls; if (is_value(result)) { r(0) = result; CHECK_TERM(r(0)); NextPF(1, next); } else if (c_p->freason == TRAP) { - call_bif_trap3: SET_CP(c_p, I+2); - SET_I(*((BeamInstr **) (UWord) ((c_p)->def_arg_reg + 3))); + SET_I(c_p->i); SWAPIN; - r(0) = c_p->def_arg_reg[0]; - x(1) = c_p->def_arg_reg[1]; - x(2) = c_p->def_arg_reg[2]; + r(0) = reg[0]; Dispatch(); } @@ -2598,7 +2445,6 @@ void process_main(void) * Error handling. SWAPOUT is not needed because it was done above. */ ASSERT(c_p->stop == E); - reg[0] = r(0); I = handle_error(c_p, I, reg, bf); goto post_error_handling; } @@ -3351,64 +3197,23 @@ void process_main(void) ASSERT(bif_nif_arity <= 3); ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - switch (bif_nif_arity) { - case 3: - { - Eterm (*bf)(Process*, Eterm, Eterm, Eterm, BeamInstr*) = vbf; - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - nif_bif_result = (*bf)(c_p, r(0), x(1), x(2), I); - ASSERT(!ERTS_PROC_IS_EXITING(c_p) || - is_non_value(nif_bif_result)); - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - PROCESS_MAIN_CHK_LOCKS(c_p); - } - break; - case 2: - { - Eterm (*bf)(Process*, Eterm, Eterm, BeamInstr*) = vbf; - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - nif_bif_result = (*bf)(c_p, r(0), x(1), I); - ASSERT(!ERTS_PROC_IS_EXITING(c_p) || - is_non_value(nif_bif_result)); - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - PROCESS_MAIN_CHK_LOCKS(c_p); - } - break; - case 1: - { - Eterm (*bf)(Process*, Eterm, BeamInstr*) = vbf; - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - nif_bif_result = (*bf)(c_p, r(0), I); - ASSERT(!ERTS_PROC_IS_EXITING(c_p) || - is_non_value(nif_bif_result)); - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - PROCESS_MAIN_CHK_LOCKS(c_p); - } - break; - case 0: - { - Eterm (*bf)(Process*, BeamInstr*) = vbf; - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - nif_bif_result = (*bf)(c_p, I); - ASSERT(!ERTS_PROC_IS_EXITING(c_p) || - is_non_value(nif_bif_result)); - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - PROCESS_MAIN_CHK_LOCKS(c_p); - break; - } - default: - erl_exit(1, "apply_bif: invalid arity: %u\n", - bif_nif_arity); + reg[0] = r(0); + { + Eterm (*bf)(Process*, Eterm*, BeamInstr*) = vbf; + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + nif_bif_result = (*bf)(c_p, reg, I); + ASSERT(!ERTS_PROC_IS_EXITING(c_p) || + is_non_value(nif_bif_result)); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + PROCESS_MAIN_CHK_LOCKS(c_p); } apply_bif_or_nif_epilogue: ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); ERTS_HOLE_CHECK(c_p); if (c_p->mbuf) { - reg[0] = r(0); nif_bif_result = erts_gc_after_bif_call(c_p, nif_bif_result, reg, bif_nif_arity); - r(0) = reg[0]; } SWAPIN; /* There might have been a garbage collection. */ FCALLS = c_p->fcalls; @@ -3419,17 +3224,14 @@ void process_main(void) c_p->cp = 0; Goto(*I); } else if (c_p->freason == TRAP) { - SET_I(*((BeamInstr **) (UWord) ((c_p)->def_arg_reg + 3))); - r(0) = c_p->def_arg_reg[0]; - x(1) = c_p->def_arg_reg[1]; - x(2) = c_p->def_arg_reg[2]; + SET_I(c_p->i); + r(0) = reg[0]; if (c_p->flags & F_HIBERNATE_SCHED) { c_p->flags &= ~F_HIBERNATE_SCHED; goto do_schedule; } Dispatch(); } - reg[0] = r(0); I = handle_error(c_p, c_p->cp, reg, vbf); goto post_error_handling; } @@ -3992,8 +3794,7 @@ void process_main(void) * too big numbers). */ if (is_not_small(val) || val > make_small(0x10FFFFUL) || - (make_small(0xD800UL) <= val && val <= make_small(0xDFFFUL)) || - val == make_small(0xFFFEUL) || val == make_small(0xFFFFUL)) { + (make_small(0xD800UL) <= val && val <= make_small(0xDFFFUL))) { goto badarg; } Next(2); @@ -4012,8 +3813,8 @@ void process_main(void) * the valid range). */ if (is_not_small(tmp_arg1) || tmp_arg1 > make_small(0x10FFFFUL) || - (make_small(0xD800UL) <= tmp_arg1 && tmp_arg1 <= make_small(0xDFFFUL)) || - tmp_arg1 == make_small(0xFFFEUL) || tmp_arg1 == make_small(0xFFFFUL)) { + (make_small(0xD800UL) <= tmp_arg1 && + tmp_arg1 <= make_small(0xDFFFUL))) { ErlBinMatchBuffer *mb = ms_matchbuffer(tmp_arg2); mb->offset -= 32; @@ -4888,92 +4689,6 @@ void process_main(void) } /* - * Instructions for allocating on the message area. - */ - - OpCase(i_global_cons): - { - BeamInstr *next; -#ifdef HYBRID - Eterm *hp; - - PreFetch(0,next); - TestGlobalHeap(2,2,hp); - hp[0] = r(0); - hp[1] = x(1); - r(0) = make_list(hp); -#ifndef INCREMENTAL - global_htop += 2; -#endif - NextPF(0,next); -#else - PreFetch(0,next); - c_p->freason = EXC_INTERNAL_ERROR; - goto find_func_info; -#endif - } - - OpCase(i_global_tuple): - { - BeamInstr *next; - int len; -#ifdef HYBRID - Eterm list; - Eterm *hp; -#endif - - if ((len = list_length(r(0))) < 0) { - goto badarg; - } - - PreFetch(0,next); -#ifdef HYBRID - TestGlobalHeap(len + 1,1,hp); - list = r(0); - r(0) = make_tuple(hp); - *hp++ = make_arityval(len); - while(is_list(list)) - { - Eterm* cons = list_val(list); - *hp++ = CAR(cons); - list = CDR(cons); - } -#ifndef INCREMENTAL - global_htop += len + 1; -#endif - NextPF(0,next); -#else - c_p->freason = EXC_INTERNAL_ERROR; - goto find_func_info; -#endif - } - - OpCase(i_global_copy): - { - BeamInstr *next; - PreFetch(0,next); -#ifdef HYBRID - if (!IS_CONST(r(0))) - { - BM_SWAP_TIMER(system,copy); - SWAPOUT; - reg[0] = r(0); - reg[1] = NIL; - r(0) = copy_struct_lazy(c_p,r(0),0); - ASSERT(ma_src_top == 0); - ASSERT(ma_dst_top == 0); - ASSERT(ma_offset_top == 0); - SWAPIN; - BM_SWAP_TIMER(copy,system); - } - NextPF(0,next); -#else - c_p->freason = EXC_INTERNAL_ERROR; - goto find_func_info; -#endif - } - - /* * New floating point instructions. */ @@ -5151,10 +4866,8 @@ void process_main(void) c_p->def_arg_reg[4] = -neg_o_reds; reg[0] = r(0); c_p = hipe_mode_switch(c_p, cmd, reg); -#ifdef ERTS_SMP - reg = c_p->scheduler_data->save_reg; - freg = c_p->scheduler_data->freg; -#endif + reg = ERTS_PROC_GET_SCHDATA(c_p)->x_reg_array; + freg = ERTS_PROC_GET_SCHDATA(c_p)->f_reg_array; ERL_BITS_RELOAD_STATEP(c_p); neg_o_reds = -c_p->def_arg_reg[4]; FCALLS = c_p->fcalls; @@ -5268,8 +4981,8 @@ void process_main(void) OpCase(int_code_end): OpCase(label_L): - OpCase(too_old_compiler): OpCase(on_load): + OpCase(line_I): erl_exit(1, "meta op\n"); /* @@ -5686,6 +5399,25 @@ expand_error_value(Process* c_p, Uint freason, Eterm Value) { * that c_p->ftrace will point to a cons cell which holds the given args * and the saved data (encoded as a bignum). * + * There is an issue with line number information. Line number + * information is associated with the address *before* an operation + * that may fail or be stored stored on the stack. But continuation + * pointers point after its call instruction, not before. To avoid + * finding the wrong line number, we'll need to adjust them so that + * they point at the beginning of the call instruction or inside the + * call instruction. Since its impractical to point at the beginning, + * we'll do the simplest thing and decrement the continuation pointers + * by one. + * + * Here is an example of what can go wrong. Without the adjustment + * of continuation pointers, the call at line 42 below would seem to + * be at line 43: + * + * line 42 + * call ... + * line 43 + * gc_bif ... + * * (It would be much better to put the arglist - when it exists - in the * error value instead of in the actual trace; e.g. '{badarg, Args}' * instead of using 'badarg' with Args in the trace. The arglist may @@ -5752,7 +5484,7 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf, } /* Save second stack entry if CP is valid and different from pc */ if (depth > 0 && c_p->cp != 0 && c_p->cp != pc) { - s->trace[s->depth++] = c_p->cp; + s->trace[s->depth++] = c_p->cp - 1; depth--; } s->pc = NULL; @@ -5772,13 +5504,13 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf, /* Save first stack entry */ ASSERT(c_p->cp); if (depth > 0) { - s->trace[s->depth++] = c_p->cp; + s->trace[s->depth++] = c_p->cp - 1; depth--; } s->pc = NULL; /* Ignore pc */ } else { if (depth > 0 && c_p->cp != 0 && c_p->cp != pc) { - s->trace[s->depth++] = c_p->cp; + s->trace[s->depth++] = c_p->cp - 1; depth--; } s->pc = pc; @@ -5793,24 +5525,31 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf, } /* Save the actual stack trace */ + erts_save_stacktrace(c_p, s, depth); +} + +void +erts_save_stacktrace(Process* p, struct StackTrace* s, int depth) +{ if (depth > 0) { Eterm *ptr; BeamInstr *prev = s->depth ? s->trace[s->depth-1] : NULL; BeamInstr i_return_trace = beam_return_trace[0]; BeamInstr i_return_to_trace = beam_return_to_trace[0]; + /* * Traverse the stack backwards and add all unique continuation * pointers to the buffer, up to the maximum stack trace size. * * Skip trace stack frames. */ - ptr = c_p->stop; - if (ptr < STACK_START(c_p) - && (is_not_CP(*ptr)|| (*cp_val(*ptr) != i_return_trace && - *cp_val(*ptr) != i_return_to_trace)) - && c_p->cp) { - /* Can not follow cp here - code may be unloaded */ - BeamInstr *cpp = c_p->cp; + ptr = p->stop; + if (ptr < STACK_START(p) && + (is_not_CP(*ptr)|| (*cp_val(*ptr) != i_return_trace && + *cp_val(*ptr) != i_return_to_trace)) && + p->cp) { + /* Cannot follow cp here - code may be unloaded */ + BeamInstr *cpp = p->cp; if (cpp == beam_exception_trace || cpp == beam_return_trace) { /* Skip return_trace parameters */ ptr += 2; @@ -5819,7 +5558,7 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf, ptr += 1; } } - while (ptr < STACK_START(c_p) && depth > 0) { + while (ptr < STACK_START(p) && depth > 0) { if (is_CP(*ptr)) { if (*cp_val(*ptr) == i_return_trace) { /* Skip stack frame variables */ @@ -5834,7 +5573,7 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf, if (cp != prev) { /* Record non-duplicates only */ prev = cp; - s->trace[s->depth++] = cp; + s->trace[s->depth++] = cp - 1; depth--; } ptr++; @@ -5902,9 +5641,14 @@ build_stacktrace(Process* c_p, Eterm exc) { struct StackTrace* s; Eterm args; int depth; - BeamInstr* current; - Eterm Where = NIL; - Eterm *next_p = &Where; + FunctionInfo fi; + FunctionInfo* stk; + FunctionInfo* stkp; + Eterm res = NIL; + Uint heap_size; + Eterm* hp; + Eterm mfa; + int i; if (! (s = get_trace_from_exc(exc))) { return NIL; @@ -5923,64 +5667,56 @@ build_stacktrace(Process* c_p, Eterm exc) { * saved s->current should already contain the proper value. */ if (s->pc != NULL) { - current = find_function_from_pc(s->pc); + erts_lookup_function_info(&fi, s->pc, 1); + } else if (GET_EXC_INDEX(s->freason) == + GET_EXC_INDEX(EXC_FUNCTION_CLAUSE)) { + erts_lookup_function_info(&fi, s->current, 1); } else { - current = s->current; + erts_set_current_function(&fi, s->current); } + /* - * If current is still NULL, default to the initial function + * If fi.current is still NULL, default to the initial function * (e.g. spawn_link(erlang, abs, [1])). */ - if (current == NULL) { - current = c_p->initial; + if (fi.current == NULL) { + erts_set_current_function(&fi, c_p->initial); args = am_true; /* Just in case */ } else { args = get_args_from_exc(exc); } - depth = s->depth; - /* - * Add the {M,F,A} for the current function - * (where A is arity or [Argument]). + * Look up all saved continuation pointers and calculate + * needed heap space. */ - { - int i; - Eterm mfa; - Uint heap_size = 6*(depth+1); - Eterm* hp = HAlloc(c_p, heap_size); - Eterm* hp_end = hp + heap_size; - - if (args != am_true) { - /* We have an arglist - use it */ - mfa = TUPLE3(hp, current[0], current[1], args); - } else { - Eterm arity = make_small(current[2]); - mfa = TUPLE3(hp, current[0], current[1], arity); + depth = s->depth; + stk = stkp = (FunctionInfo *) erts_alloc(ERTS_ALC_T_TMP, + depth*sizeof(FunctionInfo)); + heap_size = fi.needed + 2; + for (i = 0; i < depth; i++) { + erts_lookup_function_info(stkp, s->trace[i], 1); + if (stkp->current) { + heap_size += stkp->needed + 2; + stkp++; } - hp += 4; - ASSERT(*next_p == NIL); - *next_p = CONS(hp, mfa, NIL); - next_p = &CDR(list_val(*next_p)); - hp += 2; + } - /* - * Finally, we go through the saved continuation pointers. - */ - for (i = 0; i < depth; i++) { - BeamInstr *fi = find_function_from_pc((BeamInstr *) s->trace[i]); - if (fi == NULL) continue; - mfa = TUPLE3(hp, fi[0], fi[1], make_small(fi[2])); - hp += 4; - ASSERT(*next_p == NIL); - *next_p = CONS(hp, mfa, NIL); - next_p = &CDR(list_val(*next_p)); - hp += 2; - } - ASSERT(hp <= hp_end); - HRelease(c_p, hp_end, hp); + /* + * Allocate heap space and build the stacktrace. + */ + hp = HAlloc(c_p, heap_size); + while (stkp > stk) { + stkp--; + hp = erts_build_mfa_item(stkp, hp, am_true, &mfa); + res = CONS(hp, mfa, res); + hp += 2; } - return Where; + hp = erts_build_mfa_item(&fi, hp, args, &mfa); + res = CONS(hp, mfa, res); + + erts_free(ERTS_ALC_T_TMP, (void *) stk); + return res; } diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index fceb352cf3..3836f1ae96 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -158,6 +158,7 @@ typedef struct { #define LITERAL_CHUNK 6 #define ATTR_CHUNK 7 #define COMPILE_CHUNK 8 +#define LINE_CHUNK 9 #define NUM_CHUNK_TYPES (sizeof(chunk_types)/sizeof(chunk_types[0])) @@ -182,6 +183,7 @@ static Uint chunk_types[] = { MakeIffId('L', 'i', 't', 'T'), /* 6 */ MakeIffId('A', 't', 't', 'r'), /* 7 */ MakeIffId('C', 'I', 'n', 'f'), /* 8 */ + MakeIffId('L', 'i', 'n', 'e'), /* 9 */ }; /* @@ -231,6 +233,15 @@ struct string_patch { }; /* + * This structure associates a code offset with a source code location. + */ + +typedef struct { + int pos; /* Position in code */ + Uint32 loc; /* Location in source code */ +} LineInstr; + +/* * This structure contains all information about the module being loaded. */ @@ -325,6 +336,19 @@ typedef struct { Literal* literals; /* Array of literals. */ LiteralPatch* literal_patches; /* Operands that need to be patched. */ Uint total_literal_size; /* Total heap size for all literals. */ + + /* + * Line table. + */ + BeamInstr* line_item; /* Line items from the BEAM file. */ + int num_line_items; /* Number of line items. */ + LineInstr* line_instr; /* Line instructions */ + int num_line_instrs; /* Maximum number of line instructions */ + int current_li; /* Current line instruction */ + int* func_line; /* Mapping from function to first line instr */ + Eterm* fname; /* List of file names */ + int num_fnames; /* Number of filenames in fname table */ + int loc_size; /* Size of location info in bytes (2/4) */ } LoaderState; typedef struct { @@ -332,6 +356,27 @@ typedef struct { Eterm* func_tab[1]; /* Pointers to each function. */ } LoadedCode; +/* + * Layout of the line table. + */ + +#define MI_LINE_FNAME_PTR 0 +#define MI_LINE_LOC_TAB 1 +#define MI_LINE_LOC_SIZE 2 +#define MI_LINE_FUNC_TAB 3 + +#define LINE_INVALID_LOCATION (0) + +/* + * Macros for manipulating locations. + */ + +#define IS_VALID_LOCATION(File, Line) \ + ((unsigned) (File) < 255 && (unsigned) (Line) < ((1 << 24) - 1)) +#define MAKE_LOCATION(File, Line) (((File) << 24) | (Line)) +#define LOC_FILE(Loc) ((Loc) >> 24) +#define LOC_LINE(Loc) ((Loc) & ((1 << 24)-1)) + #define GetTagAndValue(Stp, Tag, Val) \ do { \ BeamInstr __w; \ @@ -468,6 +513,7 @@ static int load_import_table(LoaderState* stp); static int read_export_table(LoaderState* stp); static int read_lambda_table(LoaderState* stp); static int read_literal_table(LoaderState* stp); +static int read_line_table(LoaderState* stp); static int read_code_header(LoaderState* stp); static int load_code(LoaderState* stp); static GenOp* gen_element(LoaderState* stp, GenOpArg Fail, GenOpArg Index, @@ -506,6 +552,8 @@ static Eterm native_addresses(Process* p, Eterm mod); int patch_funentries(Eterm Patchlist); int patch(Eterm Addresses, Uint fe); static int safe_mul(UWord a, UWord b, UWord* resp); +static void lookup_loc(FunctionInfo* fi, BeamInstr* pc, + BeamInstr* modp, int idx); static int must_swap_floats; @@ -698,6 +746,26 @@ bin_load(Process *c_p, ErtsProcLocks c_p_locks, } /* + * Read the line table (if present). + */ + + CHKBLK(ERTS_ALC_T_CODE,state.code); + if (state.chunks[LINE_CHUNK].size > 0) { + define_file(&state, "line table", LINE_CHUNK); + if (!read_line_table(&state)) { + goto load_error; + } + } + + /* + * Since the literal table *may* have contained external + * funs (containing references to export entries), now is + * the time to consolidate the export tables. + */ + + erts_export_consolidate(); + + /* * Load the code chunk. */ @@ -805,6 +873,22 @@ bin_load(Process *c_p, ErtsProcLocks c_p_locks, state.genop_blocks = next; } + if (state.line_item != 0) { + erts_free(ERTS_ALC_T_LOADER_TMP, state.line_item); + } + + if (state.line_instr != 0) { + erts_free(ERTS_ALC_T_LOADER_TMP, state.line_instr); + } + + if (state.func_line != 0) { + erts_free(ERTS_ALC_T_LOADER_TMP, state.func_line); + } + + if (state.fname != 0) { + erts_free(ERTS_ALC_T_LOADER_TMP, state.fname); + } + return rval; } @@ -835,6 +919,10 @@ init_state(LoaderState* stp) stp->string_patches = 0; stp->may_load_nif = 0; stp->on_load = 0; + stp->line_item = 0; + stp->line_instr = 0; + stp->func_line = 0; + stp->fname = 0; } static int @@ -1324,6 +1412,138 @@ read_literal_table(LoaderState* stp) return 0; } +static int +read_line_table(LoaderState* stp) +{ + unsigned version; + unsigned flags; + int num_line_items; + BeamInstr* lp; + int i; + BeamInstr fname_index; + BeamInstr tag; + + /* + * If the emulator flag ignoring the line information was given, + * return immediately. + */ + + if (erts_no_line_info) { + return 1; + } + + /* + * Check version of line table. + */ + + GetInt(stp, 4, version); + if (version != 0) { + /* + * Wrong version. Silently ignore the line number chunk. + */ + return 1; + } + + /* + * Read the remaining header words. The flag word is reserved + * for possible future use; for the moment we ignore it. + */ + GetInt(stp, 4, flags); + GetInt(stp, 4, stp->num_line_instrs); + GetInt(stp, 4, num_line_items); + GetInt(stp, 4, stp->num_fnames); + + /* + * Calculate space and allocate memory for the line item table. + */ + + num_line_items++; + lp = (BeamInstr *) erts_alloc(ERTS_ALC_T_LOADER_TMP, + num_line_items * sizeof(BeamInstr)); + stp->line_item = lp; + stp->num_line_items = num_line_items; + + /* + * The zeroth entry in the line item table is special. + * It contains the undefined location. + */ + + *lp++ = LINE_INVALID_LOCATION; + num_line_items--; + + /* + * Read all the line items. + */ + + stp->loc_size = stp->num_fnames ? 4 : 2; + fname_index = 0; + while (num_line_items-- > 0) { + BeamInstr val; + BeamInstr loc; + + GetTagAndValue(stp, tag, val); + if (tag == TAG_i) { + if (IS_VALID_LOCATION(fname_index, val)) { + loc = MAKE_LOCATION(fname_index, val); + } else { + /* + * Too many files or huge line number. Silently invalidate + * the location. + */ + loc = LINE_INVALID_LOCATION; + } + *lp++ = loc; + if (val > 0xFFFF) { + stp->loc_size = 4; + } + } else if (tag == TAG_a) { + if (val > stp->num_fnames) { + LoadError2(stp, "file index overflow (%d/%d)", + val, stp->num_fnames); + } + fname_index = val; + num_line_items++; + } else { + LoadError1(stp, "bad tag '%c' (expected 'a' or 'i')", + tag_to_letter[tag]); + } + } + + /* + * Read all filenames. + */ + + if (stp->num_fnames != 0) { + stp->fname = (Eterm *) erts_alloc(ERTS_ALC_T_LOADER_TMP, + stp->num_fnames * + sizeof(Eterm)); + for (i = 0; i < stp->num_fnames; i++) { + byte* fname; + Uint n; + + GetInt(stp, 2, n); + GetString(stp, fname, n); + stp->fname[i] = am_atom_put((char*)fname, n); + } + } + + /* + * Allocate the arrays to be filled while code is being loaded. + */ + stp->line_instr = (LineInstr *) erts_alloc(ERTS_ALC_T_LOADER_TMP, + stp->num_line_instrs * + sizeof(LineInstr)); + stp->current_li = 0; + stp->func_line = (int *) erts_alloc(ERTS_ALC_T_LOADER_TMP, + stp->num_functions * + sizeof(int)); + + return 1; + + load_error: + return 0; +} + static int read_code_header(LoaderState* stp) @@ -1358,10 +1578,15 @@ read_code_header(LoaderState* stp) /* * Verify the number of the highest opcode used. */ - GetInt(stp, 4, opcode_max); if (opcode_max > MAX_GENERIC_OPCODE) { - LoadError2(stp, "use of opcode %d; this emulator supports only up to %d", + LoadError2(stp, + "This BEAM file was compiled for a later version" + " of the run-time system than " ERLANG_OTP_RELEASE ".\n" + " To fix this, please recompile this module with an " + ERLANG_OTP_RELEASE " compiler.\n" + " (Use of opcode %d; this emulator supports " + "only up to %d.)", opcode_max, MAX_GENERIC_OPCODE); } @@ -1415,7 +1640,7 @@ load_code(LoaderState* stp) { int i; int ci; - int last_func_start = 0; + int last_func_start = 0; /* Needed by nif loading and line instructions */ char* sign; int arg; /* Number of current argument. */ int num_specific; /* Number of specific ops for current. */ @@ -1428,6 +1653,14 @@ load_code(LoaderState* stp) GenOp** last_op_next = NULL; int arity; + /* + * The size of the loaded func_info instruction is needed + * by both the nif functionality and line instructions. + */ + enum { + FUNC_INFO_SZ = 5 + }; + code = stp->code; code_buffer_size = stp->code_buffer_size; ci = stp->ci; @@ -1667,14 +1900,6 @@ load_code(LoaderState* stp) } /* - * Special error message instruction. - */ - if (stp->genop->op == genop_too_old_compiler_0) { - LoadError0(stp, "please re-compile this module with an " - ERLANG_OTP_RELEASE " compiler"); - } - - /* * From the collected generic instruction, find the specific * instruction. */ @@ -1725,7 +1950,27 @@ load_code(LoaderState* stp) ERLANG_OTP_RELEASE " compiler "); } - LoadError0(stp, "no specific operation found"); + /* + * Some generic instructions should have a special + * error message. + */ + switch (stp->genop->op) { + case genop_too_old_compiler_0: + LoadError0(stp, "please re-compile this module with an " + ERLANG_OTP_RELEASE " compiler"); + case genop_unsupported_guard_bif_3: + { + Eterm Mod = (Eterm) stp->genop->a[0].val; + Eterm Name = (Eterm) stp->genop->a[1].val; + Uint arity = (Uint) stp->genop->a[2].val; + FREE_GENOP(stp, stp->genop); + stp->genop = 0; + LoadError3(stp, "unsupported guard BIF: %T:%T/%d\n", + Mod, Name, arity); + } + default: + LoadError0(stp, "no specific operation found"); + } } stp->specific_op = specific; @@ -2014,7 +2259,6 @@ load_code(LoaderState* stp) case op_i_func_info_IaaI: { Uint offset; - enum { FINFO_SZ = 5 }; if (function_number >= stp->num_functions) { LoadError1(stp, "too many functions in module (header said %d)", @@ -2022,27 +2266,37 @@ load_code(LoaderState* stp) } if (stp->may_load_nif) { - const int finfo_ix = ci - FINFO_SZ; + const int finfo_ix = ci - FUNC_INFO_SZ; enum { MIN_FUNC_SZ = 3 }; if (finfo_ix - last_func_start < MIN_FUNC_SZ && last_func_start) { /* Must make room for call_nif op */ int pad = MIN_FUNC_SZ - (finfo_ix - last_func_start); ASSERT(pad > 0 && pad < MIN_FUNC_SZ); CodeNeed(pad); - sys_memmove(&code[finfo_ix+pad], &code[finfo_ix], FINFO_SZ*sizeof(BeamInstr)); + sys_memmove(&code[finfo_ix+pad], &code[finfo_ix], + FUNC_INFO_SZ*sizeof(BeamInstr)); sys_memset(&code[finfo_ix], 0, pad*sizeof(BeamInstr)); ci += pad; stp->labels[last_label].value += pad; } } last_func_start = ci; + + /* + * Save current offset of into the line instruction array. + */ + + if (stp->func_line) { + stp->func_line[function_number] = stp->current_li; + } + /* * Save context for error messages. */ stp->function = code[ci-2]; stp->arity = code[ci-1]; - ASSERT(stp->labels[last_label].value == ci - FINFO_SZ); + ASSERT(stp->labels[last_label].value == ci - FUNC_INFO_SZ); offset = MI_FUNCTIONS + function_number; code[offset] = stp->labels[last_label].patches; stp->labels[last_label].patches = offset; @@ -2105,6 +2359,45 @@ load_code(LoaderState* stp) stp->catches = ci-3; break; + case op_line_I: + if (stp->line_item) { + BeamInstr item = code[ci-1]; + BeamInstr loc; + int li; + if (item >= stp->num_line_items) { + LoadError2(stp, "line instruction index overflow (%d/%d)", + item, stp->num_line_items); + } + li = stp->current_li; + if (li >= stp->num_line_instrs) { + LoadError2(stp, "line instruction table overflow (%d/%d)", + li, stp->num_line_instrs); + } + loc = stp->line_item[item]; + + if (ci - 2 == last_func_start) { + /* + * This line instruction directly follows the func_info + * instruction. Its address must be adjusted to point to + * func_info instruction. + */ + stp->line_instr[li].pos = last_func_start - FUNC_INFO_SZ; + stp->line_instr[li].loc = stp->line_item[item]; + stp->current_li++; + } else if (li <= stp->func_line[function_number-1] || + stp->line_instr[li-1].loc != loc) { + /* + * Only store the location if it is different + * from the previous location in the same function. + */ + stp->line_instr[li].pos = ci - 2; + stp->line_instr[li].loc = stp->line_item[item]; + stp->current_li++; + } + } + ci -= 2; /* Get rid of the instruction */ + break; + /* * End of code found. */ @@ -2141,6 +2434,8 @@ load_code(LoaderState* stp) #define no_fpe_signals(St) 0 #endif +#define never(St) 0 + /* * Predicate that tests whether a jump table can be used. */ @@ -3396,10 +3691,7 @@ gen_guard_bif1(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, BifFunction bf; NEW_GENOP(stp, op); - op->op = genop_i_gc_bif1_5; - op->arity = 5; - op->a[0] = Fail; - op->a[1].type = TAG_u; + op->next = NULL; bf = stp->import[Bif.val].bf; /* The translations here need to have a reverse counterpart in beam_emu.c:translate_gc_bif for error handling to work properly. */ @@ -3420,19 +3712,30 @@ gen_guard_bif1(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, } else if (bf == trunc_1) { op->a[1].val = (BeamInstr) (void *) erts_gc_trunc_1; } else { - abort(); + op->op = genop_unsupported_guard_bif_3; + op->arity = 3; + op->a[0].type = TAG_a; + op->a[0].val = stp->import[Bif.val].module; + op->a[1].type = TAG_a; + op->a[1].val = stp->import[Bif.val].function; + op->a[2].type = TAG_u; + op->a[2].val = stp->import[Bif.val].arity; + return op; } + op->op = genop_i_gc_bif1_5; + op->arity = 5; + op->a[0] = Fail; + op->a[1].type = TAG_u; op->a[2] = Src; op->a[3] = Live; op->a[4] = Dst; - op->next = NULL; return op; } /* - * This is used by the ops.tab rule that rewrites gc_bifs with two parameters + * This is used by the ops.tab rule that rewrites gc_bifs with two parameters. * The instruction returned is then again rewritten to an i_load instruction - * folowed by i_gc_bif2_jIId, to handle literals properly. + * followed by i_gc_bif2_jIId, to handle literals properly. * As opposed to the i_gc_bif1_jIsId, the instruction i_gc_bif2_jIId is * always rewritten, regardless of if there actually are any literals. */ @@ -3444,31 +3747,39 @@ gen_guard_bif2(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, BifFunction bf; NEW_GENOP(stp, op); - op->op = genop_ii_gc_bif2_6; - op->arity = 6; - op->a[0] = Fail; - op->a[1].type = TAG_u; + op->next = NULL; bf = stp->import[Bif.val].bf; /* The translations here need to have a reverse counterpart in beam_emu.c:translate_gc_bif for error handling to work properly. */ if (bf == binary_part_2) { op->a[1].val = (BeamInstr) (void *) erts_gc_binary_part_2; } else { - abort(); + op->op = genop_unsupported_guard_bif_3; + op->arity = 3; + op->a[0].type = TAG_a; + op->a[0].val = stp->import[Bif.val].module; + op->a[1].type = TAG_a; + op->a[1].val = stp->import[Bif.val].function; + op->a[2].type = TAG_u; + op->a[2].val = stp->import[Bif.val].arity; + return op; } + op->op = genop_ii_gc_bif2_6; + op->arity = 6; + op->a[0] = Fail; + op->a[1].type = TAG_u; op->a[2] = S1; op->a[3] = S2; op->a[4] = Live; op->a[5] = Dst; - op->next = NULL; return op; } /* - * This is used by the ops.tab rule that rewrites gc_bifs with three parameters + * This is used by the ops.tab rule that rewrites gc_bifs with three parameters. * The instruction returned is then again rewritten to a move instruction that * uses r[0] for temp storage, followed by an i_load instruction, - * folowed by i_gc_bif3_jIsId, to handle literals properly. Rewriting + * followed by i_gc_bif3_jIsId, to handle literals properly. Rewriting * always occur, as with the gc_bif2 counterpart. */ static GenOp* @@ -3479,18 +3790,27 @@ gen_guard_bif3(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, BifFunction bf; NEW_GENOP(stp, op); - op->op = genop_ii_gc_bif3_7; - op->arity = 7; - op->a[0] = Fail; - op->a[1].type = TAG_u; + op->next = NULL; bf = stp->import[Bif.val].bf; /* The translations here need to have a reverse counterpart in beam_emu.c:translate_gc_bif for error handling to work properly. */ if (bf == binary_part_3) { op->a[1].val = (BeamInstr) (void *) erts_gc_binary_part_3; } else { - abort(); + op->op = genop_unsupported_guard_bif_3; + op->arity = 3; + op->a[0].type = TAG_a; + op->a[0].val = stp->import[Bif.val].module; + op->a[1].type = TAG_a; + op->a[1].val = stp->import[Bif.val].function; + op->a[2].type = TAG_u; + op->a[2].val = stp->import[Bif.val].arity; + return op; } + op->op = genop_ii_gc_bif3_7; + op->arity = 7; + op->a[0] = Fail; + op->a[1].type = TAG_u; op->a[2] = S1; op->a[3] = S2; op->a[4] = S3; @@ -3570,6 +3890,7 @@ freeze_code(LoaderState* stp) Uint size; unsigned catches; Sint decoded_size; + Uint line_size; /* * Verify that there was a correct 'FunT' chunk if there were @@ -3580,13 +3901,19 @@ freeze_code(LoaderState* stp) LoadError0(stp, stp->lambda_error); } - /* * Calculate the final size of the code. */ - - size = (stp->ci * sizeof(BeamInstr)) + (stp->total_literal_size * sizeof(Eterm)) + - strtab_size + attr_size + compile_size; + if (stp->line_instr == 0) { + line_size = 0; + } else { + line_size = (MI_LINE_FUNC_TAB + (stp->num_functions + 1) + + (stp->current_li+1) + stp->num_fnames) * + sizeof(Eterm) + (stp->current_li+1) * stp->loc_size; + } + size = (stp->ci * sizeof(BeamInstr)) + + (stp->total_literal_size * sizeof(Eterm)) + + strtab_size + attr_size + compile_size + line_size; /* * Move the code to its final location. @@ -3674,15 +4001,66 @@ freeze_code(LoaderState* stp) } literal_end += stp->total_literal_size; } - + CHKBLK(ERTS_ALC_T_CODE,code); + /* - * Place the string table and, optionally, attributes, after the literal heap. + * If there is line information, place it here. */ - CHKBLK(ERTS_ALC_T_CODE,code); + if (stp->line_instr == 0) { + code[MI_LINE_TABLE] = (BeamInstr) 0; + str_table = (byte *) literal_end; + } else { + Eterm* line_tab = (Eterm *) literal_end; + Eterm* p; + int ftab_size = stp->num_functions; + int num_instrs = stp->current_li; + Eterm* first_line_item; + + code[MI_LINE_TABLE] = (BeamInstr) line_tab; + p = line_tab + MI_LINE_FUNC_TAB; + + first_line_item = (p + ftab_size + 1); + for (i = 0; i < ftab_size; i++) { + *p++ = (Eterm) (BeamInstr) (first_line_item + stp->func_line[i]); + } + *p++ = (Eterm) (BeamInstr) (first_line_item + num_instrs); + ASSERT(p == first_line_item); + for (i = 0; i < num_instrs; i++) { + *p++ = (Eterm) (BeamInstr) (code + stp->line_instr[i].pos); + } + *p++ = (Eterm) (BeamInstr) (code + stp->ci - 1); + + line_tab[MI_LINE_FNAME_PTR] = (Eterm) (BeamInstr) p; + memcpy(p, stp->fname, stp->num_fnames*sizeof(Eterm)); + p += stp->num_fnames; + + line_tab[MI_LINE_LOC_TAB] = (Eterm) (BeamInstr) p; + line_tab[MI_LINE_LOC_SIZE] = stp->loc_size; + if (stp->loc_size == 2) { + Uint16* locp = (Uint16 *) p; + for (i = 0; i < num_instrs; i++) { + *locp++ = (Uint16) stp->line_instr[i].loc; + } + *locp++ = LINE_INVALID_LOCATION; + str_table = (byte *) locp; + } else { + Uint32* locp = (Uint32 *) p; + ASSERT(stp->loc_size == 4); + for (i = 0; i < num_instrs; i++) { + *locp++ = stp->line_instr[i].loc; + } + *locp++ = LINE_INVALID_LOCATION; + str_table = (byte *) locp; + } - sys_memcpy(literal_end, stp->chunks[STR_CHUNK].start, strtab_size); + CHKBLK(ERTS_ALC_T_CODE,code); + } + + /* + * Place the string table and, optionally, attributes here. + */ + sys_memcpy(str_table, stp->chunks[STR_CHUNK].start, strtab_size); CHKBLK(ERTS_ALC_T_CODE,code); - str_table = (byte *) literal_end; if (attr_size) { byte* attr = str_table + strtab_size; sys_memcpy(attr, stp->chunks[ATTR_CHUNK].start, stp->chunks[ATTR_CHUNK].size); @@ -3899,6 +4277,7 @@ transform_engine(LoaderState* st) GenOp* instr; Uint* pc; int rval; + static Uint restart_fail[1] = {TOP_fail}; ASSERT(gen_opc[st->genop->op].transform != -1); pc = op_transform + gen_opc[st->genop->op].transform; @@ -3912,7 +4291,6 @@ transform_engine(LoaderState* st) ASSERT(restart != NULL); pc = restart; ASSERT(*pc < NUM_TOPS); /* Valid instruction? */ - ASSERT(*pc == TOP_try_me_else || *pc == TOP_fail); instr = st->genop; #define RETURN(r) rval = (r); goto do_return; @@ -3925,7 +4303,9 @@ transform_engine(LoaderState* st) op = *pc++; switch (op) { - case TOP_is_op: + case TOP_next_instr: + instr = instr->next; + ap = 0; if (instr == NULL) { /* * We'll need at least one more instruction to decide whether @@ -4112,10 +4492,6 @@ transform_engine(LoaderState* st) case TOP_next_arg: ap++; break; - case TOP_next_instr: - instr = instr->next; - ap = 0; - break; case TOP_commit: instr = instr->next; /* The next_instr was optimized away. */ @@ -4133,8 +4509,8 @@ transform_engine(LoaderState* st) #endif break; -#if defined(TOP_call) - case TOP_call: +#if defined(TOP_call_end) + case TOP_call_end: { GenOp** lastp; GenOp* new_instr; @@ -4171,7 +4547,7 @@ transform_engine(LoaderState* st) *lastp = st->genop; st->genop = new_instr; } - break; + RETURN(TE_OK); #endif case TOP_new_instr: /* @@ -4180,12 +4556,10 @@ transform_engine(LoaderState* st) NEW_GENOP(st, instr); instr->next = st->genop; st->genop = instr; + instr->op = op = *pc++; + instr->arity = gen_opc[op].arity; ap = 0; break; - case TOP_store_op: - instr->op = *pc++; - instr->arity = *pc++; - break; case TOP_store_type: i = *pc++; instr->a[ap].type = i; @@ -4195,21 +4569,25 @@ transform_engine(LoaderState* st) i = *pc++; instr->a[ap].val = i; break; - case TOP_store_var: + case TOP_store_var_next_arg: i = *pc++; ASSERT(i < TE_MAX_VARS); instr->a[ap].type = var[i].type; instr->a[ap].val = var[i].val; + ap++; break; case TOP_try_me_else: restart = pc + 1; restart += *pc++; ASSERT(*pc < NUM_TOPS); /* Valid instruction? */ break; + case TOP_try_me_else_fail: + restart = restart_fail; + break; case TOP_end: RETURN(TE_OK); case TOP_fail: - RETURN(TE_FAIL) + RETURN(TE_FAIL); default: ASSERT(0); } @@ -4810,17 +5188,24 @@ compilation_info_for_module(Process* p, /* Process whose heap to use. */ return result; } - /* - * Returns a pointer to {module, function, arity}, or NULL if not found. + * Find a function from the given pc and fill information in + * the FunctionInfo struct. If the full_info is non-zero, fill + * in all available information (including location in the + * source code). If no function is found, the 'current' field + * will be set to NULL. */ -BeamInstr * -find_function_from_pc(BeamInstr* pc) + +void +erts_lookup_function_info(FunctionInfo* fi, BeamInstr* pc, int full_info) { Range* low = modules; Range* high = low + num_loaded_modules; Range* mid = mid_module; + fi->current = NULL; + fi->needed = 5; + fi->loc = LINE_INVALID_LOCATION; while (low < high) { if (pc < mid->start) { high = mid; @@ -4837,25 +5222,159 @@ find_function_from_pc(BeamInstr* pc) high1 = mid1; } else if (pc < mid1[1]) { mid_module = mid; - return mid1[0]+2; + fi->current = mid1[0]+2; + if (full_info) { + BeamInstr** fp = (BeamInstr **) (mid->start + + MI_FUNCTIONS); + int idx = mid1 - fp; + lookup_loc(fi, pc, mid->start, idx); + } + return; } else { low1 = mid1 + 1; } } - return NULL; + return; } mid = low + (high-low) / 2; } - return NULL; +} + +static void +lookup_loc(FunctionInfo* fi, BeamInstr* orig_pc, BeamInstr* modp, int idx) +{ + Eterm* line = (Eterm *) modp[MI_LINE_TABLE]; + Eterm* low; + Eterm* high; + Eterm* mid; + Eterm pc; + + if (line == 0) { + return; + } + + pc = (Eterm) (BeamInstr) orig_pc; + fi->fname_ptr = (Eterm *) (BeamInstr) line[MI_LINE_FNAME_PTR]; + low = (Eterm *) (BeamInstr) line[MI_LINE_FUNC_TAB+idx]; + high = (Eterm *) (BeamInstr) line[MI_LINE_FUNC_TAB+idx+1]; + while (high > low) { + mid = low + (high-low) / 2; + if (pc < mid[0]) { + high = mid; + } else if (pc < mid[1]) { + int file; + int index = mid - (Eterm *) (BeamInstr) line[MI_LINE_FUNC_TAB]; + + if (line[MI_LINE_LOC_SIZE] == 2) { + Uint16* loc_table = + (Uint16 *) (BeamInstr) line[MI_LINE_LOC_TAB]; + fi->loc = loc_table[index]; + } else { + Uint32* loc_table = + (Uint32 *) (BeamInstr) line[MI_LINE_LOC_TAB]; + ASSERT(line[MI_LINE_LOC_SIZE] == 4); + fi->loc = loc_table[index]; + } + if (fi->loc == LINE_INVALID_LOCATION) { + return; + } + fi->needed += 3+2+3+2; + file = LOC_FILE(fi->loc); + if (file == 0) { + /* Special case: Module name with ".erl" appended */ + Atom* mod_atom = atom_tab(atom_val(fi->current[0])); + fi->needed += 2*(mod_atom->len+4); + } else { + Atom* ap = atom_tab(atom_val((fi->fname_ptr)[file-1])); + fi->needed += 2*ap->len; + } + return; + } else { + low = mid + 1; + } + } +} + +/* + * Build a single {M,F,A,Loction} item to be part of + * a stack trace. + */ +Eterm* +erts_build_mfa_item(FunctionInfo* fi, Eterm* hp, Eterm args, Eterm* mfa_p) +{ + BeamInstr* current = fi->current; + Eterm loc = NIL; + + if (fi->loc != LINE_INVALID_LOCATION) { + Eterm tuple; + int line = LOC_LINE(fi->loc); + int file = LOC_FILE(fi->loc); + Eterm file_term = NIL; + + if (file == 0) { + Atom* ap = atom_tab(atom_val(fi->current[0])); + file_term = buf_to_intlist(&hp, ".erl", 4, NIL); + file_term = buf_to_intlist(&hp, (char*)ap->name, ap->len, file_term); + } else { + Atom* ap = atom_tab(atom_val((fi->fname_ptr)[file-1])); + file_term = buf_to_intlist(&hp, (char*)ap->name, ap->len, NIL); + } + + tuple = TUPLE2(hp, am_line, make_small(line)); + hp += 3; + loc = CONS(hp, tuple, loc); + hp += 2; + tuple = TUPLE2(hp, am_file, file_term); + hp += 3; + loc = CONS(hp, tuple, loc); + hp += 2; + } + + if (is_list(args) || is_nil(args)) { + *mfa_p = TUPLE4(hp, current[0], current[1], args, loc); + } else { + Eterm arity = make_small(current[2]); + *mfa_p = TUPLE4(hp, current[0], current[1], arity, loc); + } + return hp + 5; +} + +/* + * Force setting of the current function in a FunctionInfo + * structure. No source code location will be associated with + * the function. + */ +void +erts_set_current_function(FunctionInfo* fi, BeamInstr* current) +{ + fi->current = current; + fi->needed = 5; + fi->loc = LINE_INVALID_LOCATION; +} + + +/* + * Returns a pointer to {module, function, arity}, or NULL if not found. + */ +BeamInstr* +find_function_from_pc(BeamInstr* pc) +{ + FunctionInfo fi; + + erts_lookup_function_info(&fi, pc, 0); + return fi.current; } /* * Read a specific chunk from a Beam binary. */ -Eterm -code_get_chunk_2(Process* p, Eterm Bin, Eterm Chunk) +BIF_RETTYPE +code_get_chunk_2(BIF_ALIST_2) { + Process* p = BIF_P; + Eterm Bin = BIF_ARG_1; + Eterm Chunk = BIF_ARG_2; LoaderState state; Uint chunk = 0; ErlSubBin* sb; @@ -4920,9 +5439,11 @@ code_get_chunk_2(Process* p, Eterm Bin, Eterm Chunk) * Calculate the MD5 for a module. */ -Eterm -code_module_md5_1(Process* p, Eterm Bin) +BIF_RETTYPE +code_module_md5_1(BIF_ALIST_1) { + Process* p = BIF_P; + Eterm Bin = BIF_ARG_1; LoaderState state; byte* temp_alloc = NULL; @@ -5179,7 +5700,17 @@ patch_funentries(Eterm Patchlist) fe = erts_get_fun_entry(Mod, uniq, index); fe->native_address = (Uint *)native_address; - erts_refc_dec(&fe->refc, 1); + + /* Deliberate MEMORY LEAK of native fun entries!!! + * + * Uncomment line below when hipe code upgrade and purging works correctly. + * Today we may get cases when old (leaked) native code of a purged module + * gets called and tries to create instances of a deleted fun entry. + * + * Reproduced on a debug emulator with stdlib_test/qlc_SUITE:join_merge + * + * erts_refc_dec(&fe->refc, 1); + */ if (!patch(Addresses, (Uint) fe)) return 0; diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h index 26e3054c4b..9d4a60fed1 100644 --- a/erts/emulator/beam/beam_load.h +++ b/erts/emulator/beam/beam_load.h @@ -108,6 +108,11 @@ extern Uint erts_total_code_size; #define MI_ON_LOAD_FUNCTION_PTR 10 /* + * Pointer to the line table (or NULL if none). + */ +#define MI_LINE_TABLE 11 + +/* * Start of function pointer table. This table contains pointers to * all functions in the module plus an additional pointer just beyond * the end of the last function. @@ -116,5 +121,5 @@ extern Uint erts_total_code_size; * this table. */ -#define MI_FUNCTIONS 11 +#define MI_FUNCTIONS 12 #endif /* _BEAM_LOAD_H */ diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 68b3350d7f..8ab363a1ec 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -36,6 +36,7 @@ #include "beam_bp.h" #include "erl_db_util.h" #include "register.h" +#include "erl_thr_progress.h" static Export* flush_monitor_message_trap = NULL; static Export* set_cpu_topology_trap = NULL; @@ -811,7 +812,7 @@ BIF_RETTYPE spawn_opt_1(BIF_ALIST_1) so.min_heap_size = H_MIN_SIZE; so.min_vheap_size = BIN_VH_MIN_SIZE; so.priority = PRIORITY_NORMAL; - so.max_gen_gcs = (Uint16) erts_smp_atomic32_read(&erts_max_gen_gcs); + so.max_gen_gcs = (Uint16) erts_smp_atomic32_read_nob(&erts_max_gen_gcs); so.scheduler = 0; /* @@ -1107,9 +1108,9 @@ BIF_RETTYPE hibernate_3(BIF_ALIST_3) /**********************************************************************/ -BIF_RETTYPE get_stacktrace_0(Process* p) +BIF_RETTYPE get_stacktrace_0(BIF_ALIST_0) { - Eterm t = build_stacktrace(p, p->ftrace); + Eterm t = build_stacktrace(BIF_P, BIF_P->ftrace); BIF_RET(t); } @@ -1119,10 +1120,10 @@ BIF_RETTYPE get_stacktrace_0(Process* p) * the process, and the final error value will be {Term,StackTrace}. */ -BIF_RETTYPE error_1(Process* p, Eterm term) +BIF_RETTYPE error_1(BIF_ALIST_1) { - p->fvalue = term; - BIF_ERROR(p, EXC_ERROR); + BIF_P->fvalue = BIF_ARG_1; + BIF_ERROR(BIF_P, EXC_ERROR); } /**********************************************************************/ @@ -1131,12 +1132,12 @@ BIF_RETTYPE error_1(Process* p, Eterm term) * in the stacktrace. */ -BIF_RETTYPE error_2(Process* p, Eterm value, Eterm args) +BIF_RETTYPE error_2(BIF_ALIST_2) { - Eterm* hp = HAlloc(p, 3); + Eterm* hp = HAlloc(BIF_P, 3); - p->fvalue = TUPLE2(hp, value, args); - BIF_ERROR(p, EXC_ERROR_2); + BIF_P->fvalue = TUPLE2(hp, BIF_ARG_1, BIF_ARG_2); + BIF_ERROR(BIF_P, EXC_ERROR_2); } /**********************************************************************/ @@ -1146,10 +1147,10 @@ BIF_RETTYPE error_2(Process* p, Eterm value, Eterm args) * It is useful in stub functions for NIFs. */ -BIF_RETTYPE nif_error_1(Process* p, Eterm term) +BIF_RETTYPE nif_error_1(BIF_ALIST_1) { - p->fvalue = term; - BIF_ERROR(p, EXC_ERROR); + BIF_P->fvalue = BIF_ARG_1; + BIF_ERROR(BIF_P, EXC_ERROR); } /**********************************************************************/ @@ -1159,12 +1160,12 @@ BIF_RETTYPE nif_error_1(Process* p, Eterm term) * It is useful in stub functions for NIFs. */ -BIF_RETTYPE nif_error_2(Process* p, Eterm value, Eterm args) +BIF_RETTYPE nif_error_2(BIF_ALIST_2) { - Eterm* hp = HAlloc(p, 3); + Eterm* hp = HAlloc(BIF_P, 3); - p->fvalue = TUPLE2(hp, value, args); - BIF_ERROR(p, EXC_ERROR_2); + BIF_P->fvalue = TUPLE2(hp, BIF_ARG_1, BIF_ARG_2); + BIF_ERROR(BIF_P, EXC_ERROR_2); } /**********************************************************************/ @@ -1183,14 +1184,19 @@ BIF_RETTYPE exit_1(BIF_ALIST_1) * If there is an error in the argument format, * return the atom 'badarg' instead. */ -Eterm -raise_3(Process *c_p, Eterm class, Eterm value, Eterm stacktrace) { +BIF_RETTYPE raise_3(BIF_ALIST_3) +{ + Process *c_p = BIF_P; + Eterm class = BIF_ARG_1; + Eterm value = BIF_ARG_2; + Eterm stacktrace = BIF_ARG_3; Eterm reason; Eterm l, *hp, *hp_end, *tp; int depth, cnt; size_t sz; + int must_copy = 0; struct StackTrace *s; - + if (class == am_error) { c_p->fvalue = value; reason = EXC_ERROR; @@ -1206,35 +1212,74 @@ raise_3(Process *c_p, Eterm class, Eterm value, Eterm stacktrace) { /* Check syntax of stacktrace, and count depth. * Accept anything that can be returned from erlang:get_stacktrace/0, * as well as a 2-tuple with a fun as first element that the - * error_handler may need to give us. + * error_handler may need to give us. Also allow old-style + * MFA three-tuples. */ for (l = stacktrace, depth = 0; is_list(l); l = CDR(list_val(l)), depth++) { Eterm t = CAR(list_val(l)); - int arity; + Eterm location = NIL; + if (is_not_tuple(t)) goto error; tp = tuple_val(t); - arity = arityval(tp[0]); - if ((arity == 3) && is_atom(tp[1]) && is_atom(tp[2])) continue; - if ((arity == 2) && is_fun(tp[1])) continue; - goto error; + switch (arityval(tp[0])) { + case 2: + /* {Fun,Args} */ + if (is_fun(tp[1])) { + must_copy = 1; + } else { + goto error; + } + break; + case 3: + /* + * One of: + * {Fun,Args,Location} + * {M,F,A} + */ + if (is_fun(tp[1])) { + location = tp[3]; + } else if (is_atom(tp[1]) && is_atom(tp[2])) { + must_copy = 1; + } else { + goto error; + } + break; + case 4: + if (!(is_atom(tp[1]) && is_atom(tp[2]))) { + goto error; + } + location = tp[4]; + break; + default: + goto error; + } + if (is_not_list(location) && is_not_nil(location)) { + goto error; + } } if (is_not_nil(l)) goto error; /* Create stacktrace and store */ - if (depth <= erts_backtrace_depth) { + if (erts_backtrace_depth < depth) { + depth = erts_backtrace_depth; + must_copy = 1; + } + if (must_copy) { + cnt = depth; + c_p->ftrace = NIL; + } else { + /* No need to copy the stacktrace */ cnt = 0; c_p->ftrace = stacktrace; - } else { - cnt = depth = erts_backtrace_depth; - c_p->ftrace = NIL; } + tp = &c_p->ftrace; sz = (offsetof(struct StackTrace, trace) + sizeof(Eterm) - 1) / sizeof(Eterm); - hp = HAlloc(c_p, sz + 2*(cnt + 1)); - hp_end = hp + sz + 2*(cnt + 1); + hp = HAlloc(c_p, sz + (2+6)*(cnt + 1)); + hp_end = hp + sz + (2+6)*(cnt + 1); s = (struct StackTrace *) hp; s->header = make_neg_bignum_header(sz - 1); s->freason = reason; @@ -1242,13 +1287,29 @@ raise_3(Process *c_p, Eterm class, Eterm value, Eterm stacktrace) { s->current = NULL; s->depth = 0; hp += sz; - if (cnt > 0) { + if (must_copy) { + int cnt; + /* Copy list up to depth */ for (cnt = 0, l = stacktrace; cnt < depth; cnt++, l = CDR(list_val(l))) { + Eterm t; + Eterm *tpp; + int arity; + ASSERT(*tp == NIL); - *tp = CONS(hp, CAR(list_val(l)), *tp); + t = CAR(list_val(l)); + tpp = tuple_val(t); + arity = arityval(tpp[0]); + if (arity == 2) { + t = TUPLE3(hp, tpp[1], tpp[2], NIL); + hp += 4; + } else if (arity == 3 && is_atom(tpp[1])) { + t = TUPLE4(hp, tpp[1], tpp[2], tpp[3], NIL); + hp += 5; + } + *tp = CONS(hp, t, *tp); tp = &CDR(list_val(*tp)); hp += 2; } @@ -1256,7 +1317,7 @@ raise_3(Process *c_p, Eterm class, Eterm value, Eterm stacktrace) { c_p->ftrace = CONS(hp, c_p->ftrace, make_big((Eterm *) s)); hp += 2; ASSERT(hp <= hp_end); - + HRelease(c_p, hp_end, hp); BIF_ERROR(c_p, reason); error: @@ -1674,10 +1735,10 @@ BIF_RETTYPE whereis_1(BIF_ALIST_1) * erlang:'!'/2 */ -Eterm -ebif_bang_2(Process* p, Eterm To, Eterm Message) +BIF_RETTYPE +ebif_bang_2(BIF_ALIST_2) { - return send_2(p, To, Message); + return erl_send(BIF_P, BIF_ARG_1, BIF_ARG_2); } @@ -2014,8 +2075,13 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend) { } -Eterm -send_3(Process *p, Eterm to, Eterm msg, Eterm opts) { +BIF_RETTYPE send_3(BIF_ALIST_3) +{ + Process *p = BIF_P; + Eterm to = BIF_ARG_1; + Eterm msg = BIF_ARG_2; + Eterm opts = BIF_ARG_3; + int connect = !0; int suspend = !0; Eterm l = opts; @@ -2079,8 +2145,13 @@ send_3(Process *p, Eterm to, Eterm msg, Eterm opts) { BIF_ERROR(p, BADARG); } -Eterm -send_2(Process *p, Eterm to, Eterm msg) { +BIF_RETTYPE send_2(BIF_ALIST_2) +{ + return erl_send(BIF_P, BIF_ARG_1, BIF_ARG_2); +} + +Eterm erl_send(Process *p, Eterm to, Eterm msg) +{ Sint result = do_send(p, to, msg, !0); if (result > 0) { @@ -3256,8 +3327,11 @@ time_to_parts(Eterm date, Sint* year, Sint* month, Sint* day, /* return the universal time */ BIF_RETTYPE -localtime_to_universaltime_2(Process *p, Eterm localtime, Eterm dst) +localtime_to_universaltime_2(BIF_ALIST_2) { + Process *p = BIF_P; + Eterm localtime = BIF_ARG_1; + Eterm dst = BIF_ARG_2; Sint year, month, day; Sint hour, minute, second; int isdst; @@ -3417,10 +3491,10 @@ BIF_RETTYPE ports_0(BIF_ALIST_0) erts_smp_mtx_lock(&ports_snapshot_mtx); /* One snapshot at a time */ - erts_smp_atomic_set(&erts_dead_ports_ptr, - (erts_aint_t) (port_buf + erts_max_ports)); + erts_smp_atomic_set_nob(&erts_dead_ports_ptr, + (erts_aint_t) (port_buf + erts_max_ports)); - next_ss = erts_smp_atomic32_inctest(&erts_ports_snapshot); + next_ss = erts_smp_atomic32_inc_read_relb(&erts_ports_snapshot); for (i = erts_max_ports-1; i >= 0; i--) { Port* prt = &erts_port[i]; @@ -3434,8 +3508,8 @@ BIF_RETTYPE ports_0(BIF_ALIST_0) erts_smp_port_state_unlock(prt); } - dead_ports = (Eterm*)erts_smp_atomic_xchg(&erts_dead_ports_ptr, - (erts_aint_t) NULL); + dead_ports = (Eterm*)erts_smp_atomic_xchg_nob(&erts_dead_ports_ptr, + (erts_aint_t) NULL); erts_smp_mtx_unlock(&ports_snapshot_mtx); ASSERT(pp <= dead_ports); @@ -3506,9 +3580,10 @@ BIF_RETTYPE erts_debug_display_1(BIF_ALIST_1) } -Eterm -display_string_1(Process* p, Eterm string) +BIF_RETTYPE display_string_1(BIF_ALIST_1) { + Process* p = BIF_P; + Eterm string = BIF_ARG_1; int len = is_string(string); char *str; @@ -3524,8 +3599,7 @@ display_string_1(Process* p, Eterm string) BIF_RET(am_true); } -Eterm -display_nl_0(Process* p) +BIF_RETTYPE display_nl_0(BIF_ALIST_0) { erts_fprintf(stderr, "\n"); BIF_RET(am_true); @@ -3589,8 +3663,13 @@ BIF_RETTYPE function_exported_3(BIF_ALIST_3) /**********************************************************************/ -BIF_RETTYPE is_builtin_3(Process* p, Eterm Mod, Eterm Name, Eterm Arity) +BIF_RETTYPE is_builtin_3(BIF_ALIST_3) { + Process* p = BIF_P; + Eterm Mod = BIF_ARG_1; + Eterm Name = BIF_ARG_2; + Eterm Arity = BIF_ARG_3; + if (is_not_atom(Mod) || is_not_atom(Name) || is_not_small(Arity)) { BIF_ERROR(p, BADARG); } @@ -3655,9 +3734,11 @@ BIF_RETTYPE make_fun_3(BIF_ALIST_3) BIF_RET(make_export(hp)); } -Eterm -fun_to_list_1(Process* p, Eterm fun) +BIF_RETTYPE fun_to_list_1(BIF_ALIST_1) { + Process* p = BIF_P; + Eterm fun = BIF_ARG_1; + if (is_not_any_fun(fun)) BIF_ERROR(p, BADARG); BIF_RET(term2list_dsprintf(p, fun)); @@ -3942,8 +4023,8 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) goto error; } nval = (n > (Sint) ((Uint16) -1)) ? ((Uint16) -1) : ((Uint16) n); - oval = (Uint) erts_smp_atomic32_xchg(&erts_max_gen_gcs, - (erts_aint32_t) nval); + oval = (Uint) erts_smp_atomic32_xchg_nob(&erts_max_gen_gcs, + (erts_aint32_t) nval); BIF_RET(make_small(oval)); } else if (BIF_ARG_1 == am_min_heap_size) { int oval = H_MIN_SIZE; @@ -3953,11 +4034,11 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) } erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(0); + erts_smp_thr_progress_block(); H_MIN_SIZE = erts_next_heap_size(n, 0); - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); BIF_RET(make_small(oval)); @@ -3969,11 +4050,11 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) } erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(0); + erts_smp_thr_progress_block(); BIN_VH_MIN_SIZE = erts_next_heap_size(n, 0); - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); BIF_RET(make_small(oval)); @@ -3995,7 +4076,7 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) erts_backtrace_depth = n; BIF_RET(make_small(oval)); } else if (BIF_ARG_1 == am_trace_control_word) { - BIF_RET(db_set_trace_control_word_1(BIF_P, BIF_ARG_2)); + BIF_RET(db_set_trace_control_word(BIF_P, BIF_ARG_2)); } else if (BIF_ARG_1 == am_sequential_tracer) { Eterm old_value = erts_set_system_seq_tracer(BIF_P, ERTS_PROC_LOCK_MAIN, @@ -4007,7 +4088,7 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) Uint i; ErlMessage* mp; erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(0); + erts_smp_thr_progress_block(); for (i = 0; i < erts_max_processes; i++) { if (process_tab[i] != (Process*) 0) { @@ -4024,7 +4105,7 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) } } - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); BIF_RET(am_true); @@ -4235,8 +4316,7 @@ void erts_bif_prep_await_proc_exit_data_trap(Process *c_p, Eterm pid, Eterm ret) { if (skip_current_msgq(c_p)) { - Eterm unused; - ERTS_BIF_PREP_TRAP3(unused, await_proc_exit_trap, c_p, pid, am_data, ret); + ERTS_BIF_PREP_TRAP3_NO_RET(await_proc_exit_trap, c_p, pid, am_data, ret); } } @@ -4244,8 +4324,7 @@ void erts_bif_prep_await_proc_exit_reason_trap(Process *c_p, Eterm pid) { if (skip_current_msgq(c_p)) { - Eterm unused; - ERTS_BIF_PREP_TRAP3(unused, await_proc_exit_trap, c_p, + ERTS_BIF_PREP_TRAP3_NO_RET(await_proc_exit_trap, c_p, pid, am_reason, am_undefined); } } @@ -4260,7 +4339,6 @@ erts_bif_prep_await_proc_exit_apply_trap(Process *c_p, { ASSERT(is_atom(module) && is_atom(function)); if (skip_current_msgq(c_p)) { - Eterm unused; Eterm term; Eterm *hp; int i; @@ -4272,7 +4350,7 @@ erts_bif_prep_await_proc_exit_apply_trap(Process *c_p, hp += 2; } term = TUPLE3(hp, module, function, term); - ERTS_BIF_PREP_TRAP3(unused, await_proc_exit_trap, c_p, pid, am_apply, term); + ERTS_BIF_PREP_TRAP3_NO_RET(await_proc_exit_trap, c_p, pid, am_apply, term); } } @@ -4286,7 +4364,7 @@ void erts_init_bif(void) erts_smp_spinlock_init(&make_ref_lock, "make_ref"); erts_smp_mtx_init(&ports_snapshot_mtx, "ports_snapshot"); - erts_smp_atomic_init(&erts_dead_ports_ptr, (erts_aint_t) NULL); + erts_smp_atomic_init_nob(&erts_dead_ports_ptr, (erts_aint_t) NULL); /* * bif_return_trap/1 is a hidden BIF that bifs that need to diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h index 8faa09feb8..d20089a9fb 100644 --- a/erts/emulator/beam/bif.h +++ b/erts/emulator/beam/bif.h @@ -26,14 +26,14 @@ extern Export* erts_format_cpu_topology_trap; #define BIF_P A__p -#define BIF_ALIST_0 Process* A__p -#define BIF_ALIST_1 Process* A__p, Eterm A_1 -#define BIF_ALIST_2 Process* A__p, Eterm A_1, Eterm A_2 -#define BIF_ALIST_3 Process* A__p, Eterm A_1, Eterm A_2, Eterm A_3 +#define BIF_ALIST_0 Process* A__p, Eterm* BIF__ARGS +#define BIF_ALIST_1 Process* A__p, Eterm* BIF__ARGS +#define BIF_ALIST_2 Process* A__p, Eterm* BIF__ARGS +#define BIF_ALIST_3 Process* A__p, Eterm* BIF__ARGS -#define BIF_ARG_1 A_1 -#define BIF_ARG_2 A_2 -#define BIF_ARG_3 A_3 +#define BIF_ARG_1 (BIF__ARGS[0]) +#define BIF_ARG_2 (BIF__ARGS[1]) +#define BIF_ARG_3 (BIF__ARGS[2]) #define BUMP_ALL_REDS(p) do { \ if (!ERTS_PROC_GET_SAVED_CALLS_BUF((p))) \ @@ -122,89 +122,106 @@ do { \ } while (0) -#define ERTS_BIF_PREP_TRAP0(Ret, Trap, Proc) \ -do { \ - (Proc)->arity = 0; \ - *((UWord *) (UWord) ((Proc)->def_arg_reg + 3)) = (UWord) ((Trap)->address); \ - (Proc)->freason = TRAP; \ - (Ret) = THE_NON_VALUE; \ +#define ERTS_BIF_PREP_TRAP0(Ret, Trap, Proc) \ +do { \ + (Proc)->arity = 0; \ + (Proc)->i = (BeamInstr*) ((Trap)->address); \ + (Proc)->freason = TRAP; \ + (Ret) = THE_NON_VALUE; \ } while (0) -#define ERTS_BIF_PREP_TRAP1(Ret, Trap, Proc, A0) \ -do { \ - (Proc)->arity = 1; \ - (Proc)->def_arg_reg[0] = (Eterm) (A0); \ - *((UWord *) (UWord) ((Proc)->def_arg_reg + 3)) = (UWord) ((Trap)->address); \ - (Proc)->freason = TRAP; \ - (Ret) = THE_NON_VALUE; \ +#define ERTS_BIF_PREP_TRAP1(Ret, Trap, Proc, A0) \ +do { \ + Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array; \ + (Proc)->arity = 1; \ + reg[0] = (Eterm) (A0); \ + (Proc)->i = (BeamInstr*) ((Trap)->address); \ + (Proc)->freason = TRAP; \ + (Ret) = THE_NON_VALUE; \ } while (0) -#define ERTS_BIF_PREP_TRAP2(Ret, Trap, Proc, A0, A1) \ -do { \ - (Proc)->arity = 2; \ - (Proc)->def_arg_reg[0] = (Eterm) (A0); \ - (Proc)->def_arg_reg[1] = (Eterm) (A1); \ - *((UWord *) (UWord) ((Proc)->def_arg_reg + 3)) = (UWord) ((Trap)->address); \ - (Proc)->freason = TRAP; \ - (Ret) = THE_NON_VALUE; \ +#define ERTS_BIF_PREP_TRAP2(Ret, Trap, Proc, A0, A1) \ +do { \ + Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array; \ + (Proc)->arity = 2; \ + reg[0] = (Eterm) (A0); \ + reg[1] = (Eterm) (A1); \ + (Proc)->i = (BeamInstr*) ((Trap)->address); \ + (Proc)->freason = TRAP; \ + (Ret) = THE_NON_VALUE; \ +} while (0) + +#define ERTS_BIF_PREP_TRAP3(Ret, Trap, Proc, A0, A1, A2) \ +do { \ + Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array; \ + (Proc)->arity = 3; \ + reg[0] = (Eterm) (A0); \ + reg[1] = (Eterm) (A1); \ + reg[2] = (Eterm) (A2); \ + (Proc)->i = (BeamInstr*) ((Trap)->address); \ + (Proc)->freason = TRAP; \ + (Ret) = THE_NON_VALUE; \ } while (0) -#define ERTS_BIF_PREP_TRAP3(Ret, Trap, Proc, A0, A1, A2)\ +#define ERTS_BIF_PREP_TRAP3_NO_RET(Trap, Proc, A0, A1, A2)\ do { \ + Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array; \ (Proc)->arity = 3; \ - (Proc)->def_arg_reg[0] = (Eterm) (A0); \ - (Proc)->def_arg_reg[1] = (Eterm) (A1); \ - (Proc)->def_arg_reg[2] = (Eterm) (A2); \ - *((UWord *) (UWord) ((Proc)->def_arg_reg + 3)) = (UWord) ((Trap)->address); \ + reg[0] = (Eterm) (A0); \ + reg[1] = (Eterm) (A1); \ + reg[2] = (Eterm) (A2); \ + (Proc)->i = (BeamInstr*) ((Trap)->address); \ (Proc)->freason = TRAP; \ - (Ret) = THE_NON_VALUE; \ } while (0) -#define BIF_TRAP0(p, Trap_) do { \ - (p)->arity = 0; \ - *((UWord *) (UWord) ((p)->def_arg_reg + 3)) = (UWord) ((Trap_)->address); \ - (p)->freason = TRAP; \ - return THE_NON_VALUE; \ +#define BIF_TRAP0(p, Trap_) do { \ + (p)->arity = 0; \ + (p)->i = (BeamInstr*) ((Trap_)->address); \ + (p)->freason = TRAP; \ + return THE_NON_VALUE; \ } while(0) -#define BIF_TRAP1(Trap_, p, A0) do { \ - (p)->arity = 1; \ - (p)->def_arg_reg[0] = (A0); \ - *((UWord *) (UWord) ((p)->def_arg_reg + 3)) = (UWord) ((Trap_)->address); \ - (p)->freason = TRAP; \ - return THE_NON_VALUE; \ +#define BIF_TRAP1(Trap_, p, A0) do { \ + Eterm* reg = ERTS_PROC_GET_SCHDATA((p))->x_reg_array; \ + (p)->arity = 1; \ + reg[0] = (A0); \ + (p)->i = (BeamInstr*) ((Trap_)->address); \ + (p)->freason = TRAP; \ + return THE_NON_VALUE; \ } while(0) -#define BIF_TRAP2(Trap_, p, A0, A1) do { \ - (p)->arity = 2; \ - (p)->def_arg_reg[0] = (A0); \ - (p)->def_arg_reg[1] = (A1); \ - *((UWord *) (UWord) ((p)->def_arg_reg + 3)) = (UWord) ((Trap_)->address); \ - (p)->freason = TRAP; \ - return THE_NON_VALUE; \ +#define BIF_TRAP2(Trap_, p, A0, A1) do { \ + Eterm* reg = ERTS_PROC_GET_SCHDATA((p))->x_reg_array; \ + (p)->arity = 2; \ + reg[0] = (A0); \ + reg[1] = (A1); \ + (p)->i = (BeamInstr*) ((Trap_)->address); \ + (p)->freason = TRAP; \ + return THE_NON_VALUE; \ } while(0) -#define BIF_TRAP3(Trap_, p, A0, A1, A2) do { \ - (p)->arity = 3; \ - (p)->def_arg_reg[0] = (A0); \ - (p)->def_arg_reg[1] = (A1); \ - (p)->def_arg_reg[2] = (A2); \ - *((UWord *) (UWord) ((p)->def_arg_reg + 3)) = (UWord) ((Trap_)->address); \ - (p)->freason = TRAP; \ - return THE_NON_VALUE; \ +#define BIF_TRAP3(Trap_, p, A0, A1, A2) do { \ + Eterm* reg = ERTS_PROC_GET_SCHDATA((p))->x_reg_array; \ + (p)->arity = 3; \ + reg[0] = (A0); \ + reg[1] = (A1); \ + reg[2] = (A2); \ + (p)->i = (BeamInstr*) ((Trap_)->address); \ + (p)->freason = TRAP; \ + return THE_NON_VALUE; \ } while(0) -#define BIF_TRAP_CODE_PTR_0(p, Code_) do { \ - (p)->arity = 0; \ - *((UWord *) (UWord) ((p)->def_arg_reg + 3)) = (UWord) (Code_); \ - (p)->freason = TRAP; \ - return THE_NON_VALUE; \ +#define BIF_TRAP_CODE_PTR_0(p, Code_) do { \ + (p)->arity = 0; \ + (p)->i = (BeamInstr*) (Code_); \ + (p)->freason = TRAP; \ + return THE_NON_VALUE; \ } while(0) -#define BIF_TRAP_CODE_PTR_(p, Code_) do { \ - *((UWord *) (UWord) ((p)->def_arg_reg + 3)) = (UWord) (Code_); \ - (p)->freason = TRAP; \ - return THE_NON_VALUE; \ +#define BIF_TRAP_CODE_PTR_(p, Code_) do { \ + (p)-> i = (BeamInstr*) (Code_); \ + (p)->freason = TRAP; \ + return THE_NON_VALUE; \ } while(0) extern Export bif_return_trap_export; diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index ba30fa85b8..987008c937 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -160,10 +160,6 @@ bif erlang:md5_update/2 bif 'erl.util.crypt.md5':update/2 ebif_md5_update_2 bif erlang:md5_final/1 bif 'erl.util.crypt.md5':final/1 ebif_md5_final_1 -bif erlang:memory/0 -bif 'erl.lang':memory/0 ebif_memory_0 -bif erlang:memory/1 -bif 'erl.lang':memory/1 ebif_memory_1 bif erlang:module_loaded/1 bif 'erl.system.code':is_loaded/1 ebif_is_loaded_1 module_loaded_1 bif erlang:function_exported/3 diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c index d18de9ae5d..b90ea6b478 100644 --- a/erts/emulator/beam/big.c +++ b/erts/emulator/beam/big.c @@ -1584,6 +1584,62 @@ big_to_double(Wterm x, double* resp) return 0; } +/* + * Logic has been copied from erl_bif_guard.c and slightly + * modified to use a static instead of dynamic heap + */ +Eterm +double_to_big(double x, Eterm *heap) +{ + int is_negative; + int ds; + ErtsDigit* xp; + Eterm res; + int i; + size_t sz; + Eterm* hp; + double dbase; + + if (x >= 0) { + is_negative = 0; + } else { + is_negative = 1; + x = -x; + } + + /* Unscale & (calculate exponent) */ + ds = 0; + dbase = ((double) (D_MASK) + 1); + while (x >= 1.0) { + x /= dbase; /* "shift" right */ + ds++; + } + sz = BIG_NEED_SIZE(ds); /* number of words including arity */ + + hp = heap; + res = make_big(hp); + xp = (ErtsDigit*) (hp + 1); + + for (i = ds - 1; i >= 0; i--) { + ErtsDigit d; + + x *= dbase; /* "shift" left */ + d = x; /* trunc */ + xp[i] = d; /* store digit */ + x -= d; /* remove integer part */ + } + while ((ds & (BIG_DIGITS_PER_WORD - 1)) != 0) { + xp[ds++] = 0; + } + + if (is_negative) { + *hp = make_neg_bignum_header(sz-1); + } else { + *hp = make_pos_bignum_header(sz-1); + } + return res; +} + /* ** Estimate the number of decimal digits (include sign) diff --git a/erts/emulator/beam/big.h b/erts/emulator/beam/big.h index 2afc37004f..256f1c2b45 100644 --- a/erts/emulator/beam/big.h +++ b/erts/emulator/beam/big.h @@ -140,6 +140,7 @@ Eterm big_lshift(Eterm, Sint, Eterm*); int big_comp (Wterm, Wterm); int big_ucomp (Eterm, Eterm); int big_to_double(Wterm x, double* resp); +Eterm double_to_big(double, Eterm*); Eterm small_to_big(Sint, Eterm*); Eterm uint_to_big(Uint, Eterm*); Eterm uword_to_big(UWord, Eterm*); diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c index 1fb39c6c67..29461877c5 100644 --- a/erts/emulator/beam/binary.c +++ b/erts/emulator/beam/binary.c @@ -356,8 +356,10 @@ BIF_RETTYPE erts_list_to_binary_bif(Process *p, Eterm arg) { Eterm bin; Uint size; - int offset; byte* bytes; +#ifdef DEBUG + int offset; +#endif if (is_nil(arg)) { BIF_RET(new_binary(p,(byte*)"",0)); @@ -372,7 +374,11 @@ BIF_RETTYPE erts_list_to_binary_bif(Process *p, Eterm arg) } bin = new_binary(p, (byte *)NULL, size); bytes = binary_bytes(bin); - offset = io_list_to_buf(arg, (char*) bytes, size); +#ifdef DEBUG + offset = +#endif + io_list_to_buf(arg, (char*) bytes, size); + ASSERT(offset == 0); BIF_RET(bin); diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index b8889e6206..784e55ecd2 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -37,6 +37,7 @@ #include "beam_load.h" #include "erl_instrument.h" #include "erl_bif_timer.h" +#include "erl_thr_progress.h" /* Forward declarations -- should really appear somewhere else */ static void process_killer(void); @@ -94,7 +95,7 @@ process_killer(void) erts_printf("(k)ill (n)ext (r)eturn:\n"); while(1) { if ((j = sys_get_key(0)) <= 0) - halt_0(0); + erl_exit(0, ""); switch(j) { case 'k': if (rp->status == P_WAITING) { @@ -626,7 +627,7 @@ bin_check(void) erts_printf("%p orig_size: %bpd, norefs = %bpd\n", bp->val, bp->val->orig_size, - erts_smp_atomic_read(&bp->val->refc)); + erts_smp_atomic_read_nob(&bp->val->refc)); } } if (printed) { @@ -650,24 +651,23 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) char dumpnamebuf[MAXPATHLEN]; char* dumpname; - if (ERTS_IS_CRASH_DUMPING) + if (ERTS_SOMEONE_IS_CRASH_DUMPING) return; - /* Wait for all threads to block. If all threads haven't blocked +#ifdef ERTS_SMP + /* + * Wait for all managed threads to block. If all threads haven't blocked * after a minute, we go anyway and hope for the best... * * We do not release system again. We expect an exit() or abort() after * dump has been written. - * - * NOTE: We allow gc therefore it is important not to lock *any* - * process locks. */ - erts_smp_emergency_block_system(60000, ERTS_BS_FLG_ALLOW_GC); + erts_thr_progress_fatal_error_block(60000); /* Either worked or not... */ /* Allow us to pass certain places without locking... */ -#ifdef ERTS_SMP - erts_smp_atomic_inc(&erts_writing_erl_crash_dump); + erts_smp_atomic32_set_mb(&erts_writing_erl_crash_dump, 1); + erts_smp_tsd_set(erts_is_crash_dumping_key, (void *) 1); #else erts_writing_erl_crash_dump = 1; #endif diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index b1cdd0660a..264374789c 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -41,6 +41,7 @@ #include "bif.h" #include "external.h" #include "erl_binary.h" +#include "erl_thr_progress.h" /* Turn this on to get printouts of all distribution messages * which go on the line @@ -128,8 +129,8 @@ delete_cache(ErtsAtomCache *cache) { if (cache) { erts_free(ERTS_ALC_T_DCACHE, (void *) cache); - ASSERT(erts_smp_atomic_read(&no_caches) > 0); - erts_smp_atomic_dec(&no_caches); + ASSERT(erts_smp_atomic_read_nob(&no_caches) > 0); + erts_smp_atomic_dec_nob(&no_caches); } } @@ -147,7 +148,7 @@ create_cache(DistEntry *dep) dep->cache = cp = (ErtsAtomCache*) erts_alloc(ERTS_ALC_T_DCACHE, sizeof(ErtsAtomCache)); - erts_smp_atomic_inc(&no_caches); + erts_smp_atomic_inc_nob(&no_caches); for (i = 0; i < sizeof(cp->in_arr)/sizeof(cp->in_arr[0]); i++) { cp->in_arr[i] = THE_NON_VALUE; cp->out_arr[i] = THE_NON_VALUE; @@ -156,7 +157,7 @@ create_cache(DistEntry *dep) Uint erts_dist_cache_size(void) { - return (Uint) erts_smp_atomic_read(&no_caches)*sizeof(ErtsAtomCache); + return (Uint) erts_smp_atomic_read_mb(&no_caches)*sizeof(ErtsAtomCache); } static ErtsProcList * @@ -430,11 +431,11 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason) erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx); nodename = erts_this_dist_entry->sysname; - erts_smp_block_system(ERTS_BS_FLG_ALLOW_GC); + erts_smp_thr_progress_block(); erts_set_this_node(am_Noname, 0); erts_is_alive = 0; send_nodes_mon_msgs(NULL, am_nodedown, nodename, am_visible, nd_reason); - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); } else { /* recursive call via erts_do_exit_port() will end up here */ @@ -444,7 +445,7 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason) ErtsMonitor *monitors; Uint32 flags; - erts_smp_atomic_set(&dep->dist_cmd_scheduled, 1); + erts_smp_atomic_set_mb(&dep->dist_cmd_scheduled, 1); erts_smp_de_rwlock(dep); ERTS_SMP_LC_ASSERT(is_internal_port(dep->cid) @@ -510,7 +511,7 @@ void init_dist(void) { init_nodes_monitors(); - erts_smp_atomic_init(&no_caches, 0); + erts_smp_atomic_init_nob(&no_caches, 0); /* Lookup/Install all references to trap functions */ dsend2_trap = trap_function(am_dsend,2); @@ -596,7 +597,7 @@ static void clear_dist_entry(DistEntry *dep) suspendees = get_suspended_on_de(dep, ERTS_DE_QFLGS_ALL); erts_smp_mtx_unlock(&dep->qlock); - erts_smp_atomic_set(&dep->dist_cmd_scheduled, 0); + erts_smp_atomic_set_nob(&dep->dist_cmd_scheduled, 0); dep->send = NULL; erts_smp_de_rwunlock(dep); @@ -1775,7 +1776,7 @@ erts_dist_command(Port *prt, int reds_limit) erts_refc_inc(&dep->refc, 1); /* Otherwise dist_entry might be removed if port command fails */ - erts_smp_atomic_xchg(&dep->dist_cmd_scheduled, 0); + erts_smp_atomic_set_mb(&dep->dist_cmd_scheduled, 0); erts_smp_de_rlock(dep); flags = dep->flags; @@ -2330,11 +2331,11 @@ BIF_RETTYPE setnode_2(BIF_ALIST_2) #endif erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(ERTS_BS_FLG_ALLOW_GC); + erts_smp_thr_progress_block(); erts_set_this_node(BIF_ARG_1, (Uint32) creation); erts_is_alive = 1; send_nodes_mon_msgs(NULL, am_nodeup, BIF_ARG_1, am_visible, NIL); - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); BIF_RET(am_true); @@ -2730,85 +2731,92 @@ BIF_RETTYPE is_alive_0(BIF_ALIST_0) /**********************************************************************/ /* erlang:monitor_node(Node, Bool, Options) -> Bool */ -BIF_RETTYPE monitor_node_3(BIF_ALIST_3) +static BIF_RETTYPE +monitor_node(Process* p, Eterm Node, Eterm Bool, Eterm Options) { DistEntry *dep; ErtsLink *lnk; Eterm l; - for (l = BIF_ARG_3; l != NIL && is_list(l); l = CDR(list_val(l))) { + for (l = Options; l != NIL && is_list(l); l = CDR(list_val(l))) { Eterm t = CAR(list_val(l)); /* allow_passive_connect the only available option right now */ if (t != am_allow_passive_connect) { - BIF_ERROR(BIF_P, BADARG); + BIF_ERROR(p, BADARG); } } if (l != NIL) { - BIF_ERROR(BIF_P, BADARG); + BIF_ERROR(p, BADARG); } - if (is_not_atom(BIF_ARG_1) || - ((BIF_ARG_2 != am_true) && (BIF_ARG_2 != am_false)) || + if (is_not_atom(Node) || + ((Bool != am_true) && (Bool != am_false)) || ((erts_this_node->sysname == am_Noname) - && (BIF_ARG_1 != erts_this_node->sysname))) { - BIF_ERROR(BIF_P, BADARG); + && (Node != erts_this_node->sysname))) { + BIF_ERROR(p, BADARG); } - dep = erts_sysname_to_connected_dist_entry(BIF_ARG_1); + dep = erts_sysname_to_connected_dist_entry(Node); if (!dep) { do_trap: - BIF_TRAP3(dmonitor_node_trap, BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); + BIF_TRAP3(dmonitor_node_trap, p, Node, Bool, Options); } if (dep == erts_this_dist_entry) goto done; - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK); + erts_smp_proc_lock(p, ERTS_PROC_LOCK_LINK); erts_smp_de_rlock(dep); if (ERTS_DE_IS_NOT_CONNECTED(dep)) { - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK); + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_LINK); erts_smp_de_runlock(dep); goto do_trap; } erts_smp_de_links_lock(dep); erts_smp_de_runlock(dep); - if (BIF_ARG_2 == am_true) { + if (Bool == am_true) { ASSERT(dep->cid != NIL); lnk = erts_add_or_lookup_link(&(dep->node_links), LINK_NODE, - BIF_P->id); + p->id); ++ERTS_LINK_REFC(lnk); - lnk = erts_add_or_lookup_link(&(BIF_P->nlinks), LINK_NODE, BIF_ARG_1); + lnk = erts_add_or_lookup_link(&(p->nlinks), LINK_NODE, Node); ++ERTS_LINK_REFC(lnk); } else { - lnk = erts_lookup_link(dep->node_links, BIF_P->id); + lnk = erts_lookup_link(dep->node_links, p->id); if (lnk != NULL) { if ((--ERTS_LINK_REFC(lnk)) == 0) { erts_destroy_link(erts_remove_link(&(dep->node_links), - BIF_P->id)); + p->id)); } } - lnk = erts_lookup_link(BIF_P->nlinks, BIF_ARG_1); + lnk = erts_lookup_link(p->nlinks, Node); if (lnk != NULL) { if ((--ERTS_LINK_REFC(lnk)) == 0) { - erts_destroy_link(erts_remove_link(&(BIF_P->nlinks), - BIF_ARG_1)); + erts_destroy_link(erts_remove_link(&(p->nlinks), + Node)); } } } erts_smp_de_links_unlock(dep); - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK); + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_LINK); done: erts_deref_dist_entry(dep); BIF_RET(am_true); } +BIF_RETTYPE monitor_node_3(BIF_ALIST_3) +{ + BIF_RET(monitor_node(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3)); +} + + /* monitor_node(Node, Bool) -> Bool */ BIF_RETTYPE monitor_node_2(BIF_ALIST_2) { - BIF_RET(monitor_node_3(BIF_P,BIF_ARG_1,BIF_ARG_2,NIL)); + BIF_RET(monitor_node(BIF_P, BIF_ARG_1, BIF_ARG_2, NIL)); } BIF_RETTYPE net_kernel_dflag_unicode_io_1(BIF_ALIST_1) diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h index 695a4fc3fe..845151c895 100644 --- a/erts/emulator/beam/dist.h +++ b/erts/emulator/beam/dist.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * Copyright Ericsson AB 1996-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -203,7 +203,7 @@ void erts_schedule_dist_command(Port *prt, DistEntry *dist_entry) id = dep->cid; } - if (!erts_smp_atomic_xchg(&dep->dist_cmd_scheduled, 1)) { + if (!erts_smp_atomic_xchg_mb(&dep->dist_cmd_scheduled, 1)) { (void) erts_port_task_schedule(id, &dep->dist_cmd, ERTS_PORT_TASK_DIST_CMD, diff --git a/erts/emulator/beam/erl_afit_alloc.c b/erts/emulator/beam/erl_afit_alloc.c index bcc7ea04ae..570cc59be2 100644 --- a/erts/emulator/beam/erl_afit_alloc.c +++ b/erts/emulator/beam/erl_afit_alloc.c @@ -65,16 +65,20 @@ erts_afalc_start(AFAllctr_t *afallctr, AFAllctrInit_t *afinit, AllctrInit_t *init) { - AFAllctr_t nulled_state = {{0}}; - /* {{0}} is used instead of {0}, in order to avoid (an incorrect) gcc - warning. gcc warns if {0} is used as initializer of a struct when - the first member is a struct (not if, for example, the third member - is a struct). */ + struct { + int dummy; + AFAllctr_t allctr; + } zero = {0}; + /* The struct with a dummy element first is used in order to avoid (an + incorrect) gcc warning. gcc warns if {0} is used as initializer of + a struct when the first member is a struct (not if, for example, + the third member is a struct). */ + Allctr_t *allctr = (Allctr_t *) afallctr; - init->sbmbct = 0; /* Small mbc not supported by afit */ + sys_memcpy((void *) afallctr, (void *) &zero.allctr, sizeof(AFAllctr_t)); - sys_memcpy((void *) afallctr, (void *) &nulled_state, sizeof(AFAllctr_t)); + init->sbmbct = 0; /* Small mbc not supported by afit */ allctr->mbc_header_size = sizeof(Carrier_t); allctr->min_mbc_size = MIN_MBC_SZ; diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index 9af80dd7a9..33d6cf5f2f 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -40,6 +40,8 @@ #include "erl_mseg.h" #include "erl_monitors.h" #include "erl_bif_timer.h" +#include "erl_cpu_topology.h" +#include "erl_thr_queue.h" #if defined(ERTS_ALC_T_DRV_SEL_D_STATE) || defined(ERTS_ALC_T_DRV_EV_D_STATE) #include "erl_check_io.h" #endif @@ -54,7 +56,14 @@ #include "erl_ao_firstfit_alloc.h" -#define ERTS_ALC_DEFAULT_MAX_THR_PREF 16 +#if ERTS_MAX_NO_OF_SCHEDULERS > ERTS_AU_MAX_PREF_ALLOC_INSTANCES +# error "Too many schedulers; cannot create that many pref alloc instances" +#endif + +#define ERTS_ALC_FIX_TYPE_IX(T) \ + (ERTS_ALC_T2N((T)) - ERTS_ALC_N_MIN_A_FIXED_SIZE) + +#define ERTS_ALC_DEFAULT_MAX_THR_PREF ERTS_MAX_NO_OF_SCHEDULERS #if defined(SMALL_MEMORY) || defined(PURIFY) || defined(VALGRIND) #define AU_ALLOC_DEFAULT_ENABLE(X) 0 @@ -106,24 +115,43 @@ static ErtsAllocatorState_t eheap_alloc_state; static ErtsAllocatorState_t binary_alloc_state; static ErtsAllocatorState_t ets_alloc_state; static ErtsAllocatorState_t driver_alloc_state; +static ErtsAllocatorState_t fix_alloc_state; -ErtsAlcType_t erts_fix_core_allocator_ix; -#ifdef ERTS_ALC_N_MIN_A_FIXED_SIZE -static void *(*fix_core_allocator)(ErtsAlcType_t, void *, Uint); -static void *fix_core_extra; -static void *fix_core_alloc(Uint size) +typedef struct { + erts_smp_atomic32_t refc; + int only_sz; + Uint req_sched; + Process *proc; + Eterm ref; + Eterm ref_heap[REF_THING_SIZE]; + int allocs[ERTS_ALC_A_MAX-ERTS_ALC_A_MIN+1+2]; +} ErtsAllocInfoReq; + +#define ERTS_ALC_INFO_A_ALLOC_UTIL (ERTS_ALC_A_MAX + 1) +#define ERTS_ALC_INFO_A_MSEG_ALLOC (ERTS_ALC_A_MAX + 2) +#define ERTS_ALC_INFO_A_MAX ERTS_ALC_INFO_A_MSEG_ALLOC + +#if !HALFWORD_HEAP +ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(aireq, + ErtsAllocInfoReq, + 5, + ERTS_ALC_T_AINFO_REQ) +#else +static ERTS_INLINE ErtsAllocInfoReq * +aireq_alloc(void) { - void *res; - res = (*fix_core_allocator)(ERTS_ALC_T_UNDEF, fix_core_extra, size); - if (erts_mtrace_enabled) - erts_mtrace_crr_alloc(res, - ERTS_ALC_A_FIXED_SIZE, - erts_fix_core_allocator_ix, - size); - return res; + return erts_alloc(ERTS_ALC_T_AINFO_REQ, sizeof(ErtsAllocInfoReq)); +} + +static ERTS_INLINE void +aireq_free(ErtsAllocInfoReq *ptr) +{ + erts_free(ERTS_ALC_T_AINFO_REQ, ptr); } #endif +ErtsAlcType_t erts_fix_core_allocator_ix; + enum allctr_type { GOODFIT, BESTFIT, @@ -181,6 +209,7 @@ typedef struct { struct au_init binary_alloc; struct au_init ets_alloc; struct au_init driver_alloc; + struct au_init fix_alloc; #if HALFWORD_HEAP struct au_init sbmbc_low_alloc; struct au_init std_low_alloc; @@ -393,46 +422,52 @@ set_default_driver_alloc_opts(struct au_init *ip) ip->init.util.ts = ERTS_ALC_MTA_DRIVER; } +static void +set_default_fix_alloc_opts(struct au_init *ip, + size_t *fix_type_sizes) +{ + SET_DEFAULT_ALLOC_OPTS(ip); + ip->enable = AU_ALLOC_DEFAULT_ENABLE(1); + ip->thr_spec = 1; + ip->atype = BESTFIT; + ip->init.bf.ao = 1; + ip->init.util.name_prefix = "fix_"; + ip->init.util.fix_type_size = fix_type_sizes; + ip->init.util.alloc_no = ERTS_ALC_A_FIXED_SIZE; +#ifndef SMALL_MEMORY + ip->init.util.mmbcs = 128*1024; /* Main carrier size */ +#else + ip->init.util.mmbcs = 128*1024; /* Main carrier size */ +#endif + ip->init.util.ts = ERTS_ALC_MTA_FIXED_SIZE; +} + #ifdef ERTS_SMP static void adjust_tpref(struct au_init *ip, int no_sched) { if (ip->thr_spec) { - Uint allocs; - if (ip->thr_spec < 0) {/* User specified amount */ - allocs = abs(ip->thr_spec); - if (allocs > no_sched) - allocs = no_sched; - } - else if (no_sched > ERTS_ALC_DEFAULT_MAX_THR_PREF) - allocs = ERTS_ALC_DEFAULT_MAX_THR_PREF; - else - allocs = no_sched; - if (allocs <= 1) - ip->thr_spec = 0; - else { - ip->thr_spec = (int) allocs; - ip->thr_spec *= -1; /* thread preferred */ - - /* If default ... */ - - /* ... shrink main multi-block carrier size */ - if (ip->default_.mmbcs) - ip->init.util.mmbcs /= ERTS_MIN(4, allocs); - /* ... shrink largest multi-block carrier size */ - if (ip->default_.lmbcs) - ip->init.util.lmbcs /= ERTS_MIN(2, allocs); - /* ... shrink smallest multi-block carrier size */ - if (ip->default_.smbcs) - ip->init.util.smbcs /= ERTS_MIN(4, allocs); - /* ... and more than three allocators shrink - max mseg multi-block carriers */ - if (ip->default_.mmmbc && allocs > 2) { - ip->init.util.mmmbc /= ERTS_MIN(4, allocs - 1); - if (ip->init.util.mmmbc < 3) - ip->init.util.mmmbc = 3; - } + ip->thr_spec = no_sched; + ip->thr_spec *= -1; /* thread preferred */ + + /* If default ... */ + + /* ... shrink main multi-block carrier size */ + if (ip->default_.mmbcs) + ip->init.util.mmbcs /= ERTS_MIN(4, no_sched); + /* ... shrink largest multi-block carrier size */ + if (ip->default_.lmbcs) + ip->init.util.lmbcs /= ERTS_MIN(2, no_sched); + /* ... shrink smallest multi-block carrier size */ + if (ip->default_.smbcs) + ip->init.util.smbcs /= ERTS_MIN(4, no_sched); + /* ... and more than three allocators shrink + max mseg multi-block carriers */ + if (ip->default_.mmmbc && no_sched > 2) { + ip->init.util.mmmbc /= ERTS_MIN(4, no_sched - 1); + if (ip->init.util.mmmbc < 3) + ip->init.util.mmmbc = 3; } } } @@ -442,7 +477,7 @@ adjust_tpref(struct au_init *ip, int no_sched) static void handle_args(int *, char **, erts_alc_hndl_args_init_t *); static void -set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init); +set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init, int ncpu); static void start_au_allocator(ErtsAlcType_t alctr_n, @@ -456,8 +491,6 @@ refuse_af_strategy(struct au_init *init) init->atype = GOODFIT; } -static void init_thr_ix(int static_ixs); - #ifdef HARD_DEBUG static void hdbg_init(void); #endif @@ -466,7 +499,7 @@ void erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) { UWord extra_block_size = 0; - int i; + int i, ncpu; erts_alc_hndl_args_init_t init = { 0, #if HAVE_ERTS_MSEG @@ -474,17 +507,38 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) #endif ERTS_DEFAULT_TRIM_THRESHOLD, ERTS_DEFAULT_TOP_PAD, - ERTS_DEFAULT_ALCU_INIT + ERTS_DEFAULT_ALCU_INIT, }; + size_t fix_type_sizes[ERTS_ALC_NO_FIXED_SIZES] = {0}; + fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_PROC)] + = sizeof(Process); +#if !HALFWORD_HEAP + fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MONITOR_SH)] + = ERTS_MONITOR_SH_SIZE; + fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_NLINK_SH)] + = ERTS_LINK_SH_SIZE; +#endif + fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_DRV_EV_D_STATE)] + = sizeof(ErtsDrvEventDataState); + fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_DRV_SEL_D_STATE)] + = sizeof(ErtsDrvSelectDataState); + fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MSG_REF)] + = sizeof(ErlMessage); +#ifdef ERTS_SMP + fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_THR_Q_EL_SL)] + = sizeof(ErtsThrQElement_t); +#endif #ifdef HARD_DEBUG hdbg_init(); #endif erts_have_sbmbc_alloc = 0; + ncpu = eaiop->ncpu; + if (ncpu < 1) + ncpu = 1; erts_sys_alloc_init(); - init_thr_ix(erts_no_schedulers); erts_init_utils_mem(); set_default_sbmbc_alloc_opts(&init.sbmbc_alloc); @@ -496,20 +550,23 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) set_default_binary_alloc_opts(&init.binary_alloc); set_default_ets_alloc_opts(&init.ets_alloc); set_default_driver_alloc_opts(&init.driver_alloc); + set_default_fix_alloc_opts(&init.fix_alloc, + fix_type_sizes); if (argc && argv) handle_args(argc, argv, &init); - if (erts_no_schedulers <= 1) { - init.sbmbc_alloc.thr_spec = 0; - init.sl_alloc.thr_spec = 0; - init.std_alloc.thr_spec = 0; - init.ll_alloc.thr_spec = 0; - init.eheap_alloc.thr_spec = 0; - init.binary_alloc.thr_spec = 0; - init.ets_alloc.thr_spec = 0; - init.driver_alloc.thr_spec = 0; - } +#ifndef ERTS_SMP + init.sbmbc_alloc.thr_spec = 0; + init.sl_alloc.thr_spec = 0; + init.std_alloc.thr_spec = 0; + init.ll_alloc.thr_spec = 0; + init.eheap_alloc.thr_spec = 0; + init.binary_alloc.thr_spec = 0; + init.ets_alloc.thr_spec = 0; + init.driver_alloc.thr_spec = 0; + init.fix_alloc.thr_spec = 0; +#endif if (init.erts_alloc_config) { /* Adjust flags that erts_alloc_config won't like */ @@ -522,6 +579,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) init.binary_alloc.thr_spec = 0; init.ets_alloc.thr_spec = 0; init.driver_alloc.thr_spec = 0; + init.fix_alloc.thr_spec = 0; } #ifdef ERTS_SMP @@ -538,6 +596,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) adjust_tpref(&init.binary_alloc, erts_no_schedulers); adjust_tpref(&init.ets_alloc, erts_no_schedulers); adjust_tpref(&init.driver_alloc, erts_no_schedulers); + adjust_tpref(&init.fix_alloc, erts_no_schedulers); #else /* No thread specific if not smp */ @@ -556,6 +615,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) refuse_af_strategy(&init.binary_alloc); refuse_af_strategy(&init.ets_alloc); refuse_af_strategy(&init.driver_alloc); + refuse_af_strategy(&init.fix_alloc); #ifdef ERTS_SMP if (!init.temp_alloc.thr_spec) @@ -564,6 +624,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) erts_mtrace_pre_init(); #if HAVE_ERTS_MSEG + init.mseg.nos = erts_no_schedulers; erts_mseg_init(&init.mseg); #endif erts_alcu_init(&init.alloc_util); @@ -583,20 +644,6 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) erts_allctrs_info[i].extra = NULL; } -#ifdef ERTS_ALC_N_MIN_A_FIXED_SIZE -#if !defined(PURIFY) && !defined(VALGRIND) - erts_allctrs[ERTS_ALC_A_FIXED_SIZE].alloc = erts_fix_alloc; - erts_allctrs[ERTS_ALC_A_FIXED_SIZE].realloc = erts_fix_realloc; - erts_allctrs[ERTS_ALC_A_FIXED_SIZE].free = erts_fix_free; - erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].enabled = 1; -#else - erts_allctrs[ERTS_ALC_A_FIXED_SIZE].alloc = erts_sys_alloc; - erts_allctrs[ERTS_ALC_A_FIXED_SIZE].realloc = erts_sys_realloc; - erts_allctrs[ERTS_ALC_A_FIXED_SIZE].free = erts_sys_free; - erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].enabled = 0; -#endif -#endif - erts_allctrs[ERTS_ALC_A_SYSTEM].alloc = erts_sys_alloc; erts_allctrs[ERTS_ALC_A_SYSTEM].realloc = erts_sys_realloc; erts_allctrs[ERTS_ALC_A_SYSTEM].free = erts_sys_free; @@ -621,20 +668,21 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) init.ll_low_alloc.init.util.force = 1; init.ll_low_alloc.init.util.low_mem = 1; - set_au_allocator(ERTS_ALC_A_SBMBC_LOW, &init.sbmbc_low_alloc); - set_au_allocator(ERTS_ALC_A_STANDARD_LOW, &init.std_low_alloc); - set_au_allocator(ERTS_ALC_A_LONG_LIVED_LOW, &init.ll_low_alloc); + set_au_allocator(ERTS_ALC_A_SBMBC_LOW, &init.sbmbc_low_alloc, ncpu); + set_au_allocator(ERTS_ALC_A_STANDARD_LOW, &init.std_low_alloc, ncpu); + set_au_allocator(ERTS_ALC_A_LONG_LIVED_LOW, &init.ll_low_alloc, ncpu); #endif /* HALFWORD */ - set_au_allocator(ERTS_ALC_A_TEMPORARY, &init.temp_alloc); - set_au_allocator(ERTS_ALC_A_SBMBC, &init.sbmbc_alloc); - set_au_allocator(ERTS_ALC_A_SHORT_LIVED, &init.sl_alloc); - set_au_allocator(ERTS_ALC_A_STANDARD, &init.std_alloc); - set_au_allocator(ERTS_ALC_A_LONG_LIVED, &init.ll_alloc); - set_au_allocator(ERTS_ALC_A_EHEAP, &init.eheap_alloc); - set_au_allocator(ERTS_ALC_A_BINARY, &init.binary_alloc); - set_au_allocator(ERTS_ALC_A_ETS, &init.ets_alloc); - set_au_allocator(ERTS_ALC_A_DRIVER, &init.driver_alloc); + set_au_allocator(ERTS_ALC_A_TEMPORARY, &init.temp_alloc, ncpu); + set_au_allocator(ERTS_ALC_A_SBMBC, &init.sbmbc_alloc, ncpu); + set_au_allocator(ERTS_ALC_A_SHORT_LIVED, &init.sl_alloc, ncpu); + set_au_allocator(ERTS_ALC_A_STANDARD, &init.std_alloc, ncpu); + set_au_allocator(ERTS_ALC_A_LONG_LIVED, &init.ll_alloc, ncpu); + set_au_allocator(ERTS_ALC_A_EHEAP, &init.eheap_alloc, ncpu); + set_au_allocator(ERTS_ALC_A_BINARY, &init.binary_alloc, ncpu); + set_au_allocator(ERTS_ALC_A_ETS, &init.ets_alloc, ncpu); + set_au_allocator(ERTS_ALC_A_DRIVER, &init.driver_alloc, ncpu); + set_au_allocator(ERTS_ALC_A_FIXED_SIZE, &init.fix_alloc, ncpu); for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) { if (!erts_allctrs[i].alloc) @@ -650,10 +698,6 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) sys_alloc_opt(SYS_ALLOC_OPT_TRIM_THRESHOLD, init.trim_threshold); sys_alloc_opt(SYS_ALLOC_OPT_TOP_PAD, init.top_pad); - if (erts_allctrs_info[ERTS_FIX_CORE_ALLOCATOR].enabled) - erts_fix_core_allocator_ix = ERTS_FIX_CORE_ALLOCATOR; - else - erts_fix_core_allocator_ix = ERTS_ALC_A_SYSTEM; erts_mtrace_init(init.instr.mtrace, init.instr.nodename); @@ -710,49 +754,40 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) &init.driver_alloc, &driver_alloc_state); - fix_core_allocator = erts_allctrs[erts_fix_core_allocator_ix].alloc; - fix_core_extra = erts_allctrs[erts_fix_core_allocator_ix].extra; + start_au_allocator(ERTS_ALC_A_FIXED_SIZE, + &init.fix_alloc, + &fix_alloc_state); erts_mtrace_install_wrapper_functions(); extra_block_size += erts_instr_init(init.instr.stat, init.instr.map); +#if !HALFWORD_HEAP + init_aireq_alloc(); +#endif + #ifdef DEBUG extra_block_size += install_debug_functions(); #endif -#ifdef ERTS_ALC_N_MIN_A_FIXED_SIZE - - erts_init_fix_alloc(extra_block_size, fix_core_alloc); - +} -#if !defined(PURIFY) && !defined(VALGRIND) - erts_set_fix_size(ERTS_ALC_T_PROC, sizeof(Process)); - erts_set_fix_size(ERTS_ALC_T_DB_TABLE, sizeof(DbTable)); - erts_set_fix_size(ERTS_ALC_T_ATOM, sizeof(Atom)); +void +erts_alloc_late_init(void) +{ - erts_set_fix_size(ERTS_ALC_T_MODULE, sizeof(Module)); - erts_set_fix_size(ERTS_ALC_T_REG_PROC, sizeof(RegProc)); - erts_set_fix_size(ERTS_ALC_T_FUN_ENTRY, sizeof(ErlFunEntry)); -#ifdef ERTS_ALC_T_DRV_EV_D_STATE - erts_set_fix_size(ERTS_ALC_T_DRV_EV_D_STATE, - sizeof(ErtsDrvEventDataState)); -#endif -#ifdef ERTS_ALC_T_DRV_SEL_D_STATE - erts_set_fix_size(ERTS_ALC_T_DRV_SEL_D_STATE, - sizeof(ErtsDrvSelectDataState)); -#endif -#if !HALFWORD_HEAP - erts_set_fix_size(ERTS_ALC_T_EXPORT, sizeof(Export)); - erts_set_fix_size(ERTS_ALC_T_MONITOR_SH, ERTS_MONITOR_SH_SIZE*sizeof(Uint)); - erts_set_fix_size(ERTS_ALC_T_NLINK_SH, ERTS_LINK_SH_SIZE*sizeof(Uint)); -#endif -#endif -#endif +} +static void * +erts_realloc_fixed_size(ErtsAlcType_t type, void *extra, void *p, Uint size) +{ + erl_exit(ERTS_ABORT_EXIT, + "Attempt to reallocate a block of the fixed size type %s\n", + ERTS_ALC_T2TD(type)); } + static void -set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init) +set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init, int ncpu) { ErtsAllocatorFunctions_t *af = &erts_allctrs[alctr_n]; ErtsAllocatorInfo_t *ai = &erts_allctrs_info[alctr_n]; @@ -764,6 +799,12 @@ set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init) if (init->init.util.force) init->enable = 1; + tspec->enabled = 0; + tspec->dd = 0; + tspec->aix = alctr_n; + tspec->size = 0; + ai->thr_spec = 0; + if (!init->enable) { af->alloc = erts_sys_alloc; af->realloc = erts_sys_realloc; @@ -775,14 +816,14 @@ set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init) return; } - tspec->enabled = 0; - tspec->all_thr_safe = 0; - ai->thr_spec = 0; #ifdef USE_THREADS +#ifdef ERTS_SMP if (init->thr_spec) { if (init->thr_spec > 0) { af->alloc = erts_alcu_alloc_thr_spec; - if (init->init.util.ramv) + if (init->init.util.fix_type_size) + af->realloc = erts_realloc_fixed_size; + else if (init->init.util.ramv) af->realloc = erts_alcu_realloc_mv_thr_spec; else af->realloc = erts_alcu_realloc_thr_spec; @@ -790,12 +831,14 @@ set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init) } else { af->alloc = erts_alcu_alloc_thr_pref; - if (init->init.util.ramv) + if (init->init.util.fix_type_size) + af->realloc = erts_realloc_fixed_size; + else if (init->init.util.ramv) af->realloc = erts_alcu_realloc_mv_thr_pref; else af->realloc = erts_alcu_realloc_thr_pref; af->free = erts_alcu_free_thr_pref; - tspec->all_thr_safe = 1; + tspec->dd = 1; } tspec->enabled = 1; @@ -803,9 +846,13 @@ set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init) ai->thr_spec = tspec->size; } - else if (init->init.util.ts) { + else +#endif + if (init->init.util.ts) { af->alloc = erts_alcu_alloc_ts; - if (init->init.util.ramv) + if (init->init.util.fix_type_size) + af->realloc = erts_realloc_fixed_size; + else if (init->init.util.ramv) af->realloc = erts_alcu_realloc_mv_ts; else af->realloc = erts_alcu_realloc_ts; @@ -815,7 +862,9 @@ set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init) #endif { af->alloc = erts_alcu_alloc; - if (init->init.util.ramv) + if (init->init.util.fix_type_size) + af->realloc = erts_realloc_fixed_size; + else if (init->init.util.ramv) af->realloc = erts_alcu_realloc_mv; else af->realloc = erts_alcu_realloc; @@ -838,12 +887,14 @@ start_au_allocator(ErtsAlcType_t alctr_n, ErtsAllocatorFunctions_t *af = &erts_allctrs[alctr_n]; ErtsAllocatorInfo_t *ai = &erts_allctrs_info[alctr_n]; ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[alctr_n]; + ErtsAlcFixList_t *fix_lists = NULL; + size_t fix_list_size = 0; if (!init->enable) return; if (init->thr_spec) { - void *states = erts_sys_alloc(0, + char *states = erts_sys_alloc(0, NULL, ((sizeof(Allctr_t *) * (tspec->size + 1)) @@ -855,18 +906,40 @@ start_au_allocator(ErtsAlcType_t alctr_n, "Failed to allocate allocator states for %salloc\n", init->init.util.name_prefix); tspec->allctr = (Allctr_t **) states; - states = ((char *) states) + sizeof(Allctr_t *) * (tspec->size + 1); + states += sizeof(Allctr_t *) * (tspec->size + 1); states = ((((UWord) states) & ERTS_CACHE_LINE_MASK) - ? (void *) ((((UWord) states) & ~ERTS_CACHE_LINE_MASK) + ? (char *) ((((UWord) states) & ~ERTS_CACHE_LINE_MASK) + ERTS_CACHE_LINE_SIZE) - : (void *) states); - tspec->allctr[0] = init->thr_spec > 0 ? (Allctr_t *) state : (Allctr_t *) NULL; + : (char *) states); + tspec->allctr[0] = (Allctr_t *) state; size = tspec->size; for (i = 1; i < size; i++) tspec->allctr[i] = (Allctr_t *) &((ErtsAllocatorState_t *) states)[i-1]; } + if (init->init.util.fix_type_size) { + size_t tot_fix_list_size; + fix_list_size = sizeof(ErtsAlcFixList_t)*ERTS_ALC_NO_FIXED_SIZES; + fix_list_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(fix_list_size); + tot_fix_list_size = fix_list_size; + if (init->thr_spec) + tot_fix_list_size *= tspec->size; + fix_lists = erts_sys_alloc(0, + NULL, + (tot_fix_list_size + + ERTS_CACHE_LINE_SIZE - 1)); + if (!fix_lists) + erl_exit(ERTS_ABORT_EXIT, + "Failed to allocate fix lists for %salloc\n", + init->init.util.name_prefix); + + if (((UWord) fix_lists) & ERTS_CACHE_LINE_MASK) + fix_lists = ((ErtsAlcFixList_t *) + ((((UWord) fix_lists) & ~ERTS_CACHE_LINE_MASK) + + ERTS_CACHE_LINE_SIZE)); + } + for (i = 0; i < size; i++) { void *as; atype = init->atype; @@ -877,25 +950,32 @@ start_au_allocator(ErtsAlcType_t alctr_n, as0 = (void *) tspec->allctr[i]; if (!as0) continue; - if (i == 0) { - if (atype == AFIT) - atype = GOODFIT; - init->init.util.ts = 1; + if (init->thr_spec < 0) { + init->init.util.ts = i == 0; + init->init.util.tspec = 0; + init->init.util.tpref = -1*init->thr_spec + 1; } else { - if (init->thr_spec < 0) { + if (i != 0) + init->init.util.ts = 0; + else { + if (atype == AFIT) + atype = GOODFIT; init->init.util.ts = 1; - init->init.util.tspec = 0; - init->init.util.tpref = -1*init->thr_spec; } - else { - init->init.util.ts = 0; - init->init.util.tspec = init->thr_spec + 1; - init->init.util.tpref = 0; - } - } + init->init.util.tspec = init->thr_spec + 1; + init->init.util.tpref = 0; + } + } + + if (fix_lists) { + init->init.util.fix = fix_lists; + fix_lists = ((ErtsAlcFixList_t *) + (((char *) fix_lists) + fix_list_size)); } + init->init.util.ix = i; + switch (atype) { case GOODFIT: as = (void *) erts_gfalc_start((GFAllctr_t *) as0, @@ -931,11 +1011,8 @@ start_au_allocator(ErtsAlcType_t alctr_n, af->extra = as; } - if (init->thr_spec) { + if (init->thr_spec) af->extra = tspec; - init->init.util.ts = 1; - } - ai->extra = af->extra; } @@ -1055,34 +1132,6 @@ get_amount_value(char *param_end, char** argv, int* ip) return (Uint) tmp; } -static int -get_bool_or_possitive_amount_value(int *bool, Uint *amount, - char *param_end, char** argv, int* ip) -{ - char *param = argv[*ip]+1; - char *value = get_value(param_end, argv, ip); - if (strcmp(value, "true") == 0) { - *bool = 1; - return 1; - } - else if (strcmp(value, "false") == 0) { - *bool = 0; - return 1; - } - else { - Sint tmp; - char *rest; - errno = 0; - tmp = (Sint) strtol(value, &rest, 10); - if (errno != 0 || rest == value || tmp <= 0) { - bad_value(param, param_end, value); - return -1; - } - *amount = (Uint) tmp; - return 0; - } -} - static void handle_au_arg(struct au_init *auip, char* sub_param, @@ -1197,25 +1246,16 @@ handle_au_arg(struct au_init *auip, goto bad_switch; break; case 't': { - Uint no; - int enable; - int res = get_bool_or_possitive_amount_value(&enable, - &no, - sub_param+1, - argv, - ip); - if (res > 0) - auip->thr_spec = enable ? 1 : 0; + int res = get_bool_value(sub_param+1, argv, ip); + if (res > 0) { + auip->thr_spec = 1; + break; + } else if (res == 0) { - int allocs = (int) no; - if (allocs < 0) - allocs = INT_MIN; - else { - allocs *= -1; - } - auip->thr_spec = allocs; + auip->thr_spec = 0; + break; } - break; + goto bad_switch; } default: bad_switch: @@ -1234,6 +1274,7 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) &init->eheap_alloc, &init->ll_alloc, &init->driver_alloc, + &init->fix_alloc, &init->sl_alloc, &init->temp_alloc }; @@ -1264,14 +1305,8 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) case 'E': handle_au_arg(&init->ets_alloc, &argv[i][3], argv, &i); break; - case 'F': /* fix_alloc */ - if (has_prefix("e", param+2)) { - arg = get_value(param+3, argv, &i); - if (strcmp("true", arg) != 0) - bad_value(param, param+3, arg); - } - else - bad_param(param, param+2); + case 'F': + handle_au_arg(&init->fix_alloc, &argv[i][3], argv, &i); break; case 'H': handle_au_arg(&init->eheap_alloc, &argv[i][3], argv, &i); @@ -1298,12 +1333,6 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) #endif get_amount_value(argv[i]+6, argv, &i); } - else if (has_prefix("cci", argv[i]+3)) { -#if HAVE_ERTS_MSEG - init->mseg.cci = -#endif - get_amount_value(argv[i]+6, argv, &i); - } else { bad_param(param, param+2); } @@ -1389,6 +1418,7 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) set_default_binary_alloc_opts(&init->binary_alloc); set_default_ets_alloc_opts(&init->ets_alloc); set_default_driver_alloc_opts(&init->driver_alloc); + set_default_driver_alloc_opts(&init->fix_alloc); init->driver_alloc.enable = 0; if (strcmp("r9c", arg) == 0) { @@ -1523,43 +1553,74 @@ static char *type_no_str(ErtsAlcType_t n) #define type_str(T) type_no_str(ERTS_ALC_T2N((T))) -erts_tsd_key_t thr_ix_key; -erts_spinlock_t alloc_thr_ix_lock; -int last_thr_ix; -int first_dyn_thr_ix; - -static void -init_thr_ix(int static_ixs) +void +erts_alloc_register_scheduler(void *vesdp) { - erts_tsd_key_create(&thr_ix_key); - erts_spinlock_init(&alloc_thr_ix_lock, "alloc_thr_ix_lock"); - last_thr_ix = -4711; - first_dyn_thr_ix = static_ixs+1; + ErtsSchedulerData *esdp = (ErtsSchedulerData *) vesdp; + int ix = (int) esdp->no; + int aix; + + for (aix = ERTS_ALC_A_MIN; aix <= ERTS_ALC_A_MAX; aix++) { + ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[aix]; + esdp->alloc_data.deallctr[aix] = NULL; + esdp->alloc_data.pref_ix[aix] = -1; + if (tspec->enabled) { + if (!tspec->dd) + esdp->alloc_data.pref_ix[aix] = ix; + else { + Allctr_t *allctr = tspec->allctr[ix]; + ASSERT(allctr); + esdp->alloc_data.deallctr[aix] = allctr; + esdp->alloc_data.pref_ix[aix] = ix; + } + } + } } -int -erts_alc_get_thr_ix(void) +void +erts_alloc_scheduler_handle_delayed_dealloc(void *vesdp, + int *need_thr_progress, + int *more_work) { - int ix = (int)(long) erts_tsd_get(thr_ix_key); - if (ix == 0) { - erts_spin_lock(&alloc_thr_ix_lock); - last_thr_ix++; - if (last_thr_ix < 0) - last_thr_ix = first_dyn_thr_ix; - ix = last_thr_ix; - erts_spin_unlock(&alloc_thr_ix_lock); - erts_tsd_set(thr_ix_key, (void *)(long) ix); + ErtsSchedulerData *esdp = (ErtsSchedulerData *) vesdp; + int aix; + for (aix = ERTS_ALC_A_MIN; aix <= ERTS_ALC_A_MAX; aix++) { + Allctr_t *allctr; + if (esdp) + allctr = esdp->alloc_data.deallctr[aix]; + else { + ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[aix]; + if (tspec->enabled && tspec->dd) + allctr = tspec->allctr[0]; + else + allctr = NULL; + } + if (allctr) { + erts_alcu_check_delayed_dealloc(allctr, + 1, + need_thr_progress, + more_work); + } } - ASSERT(ix > 0); - return ix; } -void erts_alloc_reg_scheduler_id(Uint id) +erts_aint32_t +erts_alloc_fix_alloc_shrink(int ix, erts_aint32_t flgs) { - int ix = (int) id; - ASSERT(0 < ix && ix <= first_dyn_thr_ix); - ASSERT(0 == (int) (long) erts_tsd_get(thr_ix_key)); - erts_tsd_set(thr_ix_key, (void *)(long) ix); +#ifdef ERTS_SMP + ErtsAllocatorThrSpec_t *tspec; + tspec = &erts_allctr_thr_spec[ERTS_ALC_A_FIXED_SIZE]; + if (erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].thr_spec && tspec->enabled) + return erts_alcu_fix_alloc_shrink(tspec->allctr[ix], flgs); + if (ix == 0 && erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].extra) + return erts_alcu_fix_alloc_shrink( + erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].extra, flgs); +#else + if (ix == 1 && erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].extra) + return erts_alcu_fix_alloc_shrink( + erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].extra, flgs); +#endif + return 0; } static void @@ -1574,14 +1635,12 @@ erts_alloc_get_verify_unused_temp_alloc(Allctr_t **allctr) if (erts_allctrs_info[ERTS_ALC_A_TEMPORARY].alloc_util && erts_allctrs_info[ERTS_ALC_A_TEMPORARY].thr_spec) { ErtsAllocatorThrSpec_t *tspec; + int ix = ERTS_ALC_GET_THR_IX(); tspec = &erts_allctr_thr_spec[ERTS_ALC_A_TEMPORARY]; - if (!tspec->all_thr_safe) { - int ix = erts_alc_get_thr_ix(); - if (ix < tspec->size) { - *allctr = tspec->allctr[ix]; - return erts_alcu_verify_unused; - } + if (ix < tspec->size) { + *allctr = tspec->allctr[ix]; + return erts_alcu_verify_unused; } } @@ -1680,7 +1739,7 @@ erts_realloc_n_enomem(ErtsAlcType_t n, void *ptr, Uint size) } static ERTS_INLINE UWord -alcu_size(ErtsAlcType_t ai) +alcu_size(ErtsAlcType_t ai, ErtsAlcUFixInfo_t *fi, int fisz) { UWord res = 0; @@ -1690,22 +1749,20 @@ alcu_size(ErtsAlcType_t ai) if (!erts_allctrs_info[ai].thr_spec) { Allctr_t *allctr = erts_allctrs_info[ai].extra; AllctrSize_t asize; - erts_alcu_current_size(allctr, &asize); + erts_alcu_current_size(allctr, &asize, fi, fisz); res += asize.blocks; } else { ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[ai]; int i; - ASSERT(tspec->all_thr_safe); - ASSERT(tspec->enabled); for (i = tspec->size - 1; i >= 0; i--) { Allctr_t *allctr = tspec->allctr[i]; AllctrSize_t asize; if (allctr) { - erts_alcu_current_size(allctr, &asize); + erts_alcu_current_size(allctr, &asize, fi, fisz); res += asize.blocks; } } @@ -1733,7 +1790,6 @@ alcu_is_low(ErtsAlcType_t ai) int found_one = 0; # endif - ASSERT(tspec->all_thr_safe); ASSERT(tspec->enabled); for (i = tspec->size - 1; i >= 0; i--) { @@ -1757,11 +1813,24 @@ alcu_is_low(ErtsAlcType_t ai) } #endif /* HALFWORD */ +static ERTS_INLINE void +add_fix_values(UWord *ap, UWord *up, ErtsAlcUFixInfo_t *fi, ErtsAlcType_t type) +{ + int ix = ERTS_ALC_T2N(type) - ERTS_ALC_N_MIN_A_FIXED_SIZE; + ASSERT(0 <= ix && ix < ERTS_ALC_NO_FIXED_SIZES); + + *ap += (UWord) fi[ix].allocated; + *up += (UWord) fi[ix].used; +} + Eterm erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) { +/* + * NOTE! When updating this function, make sure to also update + * erlang:memory/[0,1] in $ERL_TOP/erts/preloaded/src/erlang.erl + */ #define ERTS_MEM_NEED_ALL_ALCU (!erts_instr_stat && want_tot_or_sys) - ErtsFixInfo efi; struct { int total; int processes; @@ -1800,6 +1869,9 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) Eterm res = THE_NON_VALUE; ErtsAlcType_t ai; int only_one_value = 0; + ErtsAlcUFixInfo_t fi[ERTS_ALC_NO_FIXED_SIZES] = {{0,0}}; + + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); /* Figure out whats wanted... */ @@ -1969,7 +2041,6 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) for (ai = ERTS_ALC_A_MIN; ai <= ERTS_ALC_A_MAX; ai++) { switch (ai) { case ERTS_ALC_A_SYSTEM: - case ERTS_ALC_A_FIXED_SIZE: case ERTS_ALC_A_SBMBC: #if HALFWORD_HEAP case ERTS_ALC_A_SBMBC_LOW: @@ -2029,11 +2100,15 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) case ERTS_ALC_A_BINARY: save = &size.binary; break; + case ERTS_ALC_A_FIXED_SIZE: + asz = alcu_size(ai, fi, ERTS_ALC_NO_FIXED_SIZES); + size.total += asz; + continue; default: save = NULL; break; } - asz = alcu_size(ai); + asz = alcu_size(ai, NULL, 0); if (save) *save = asz; size.total += asz; @@ -2053,8 +2128,11 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) if (ERTS_MEM_NEED_ALL_ALCU) tmp = size.processes; - else - tmp = alcu_size(ERTS_ALC_A_EHEAP); + else { + alcu_size(ERTS_ALC_A_FIXED_SIZE, + fi, ERTS_ALC_NO_FIXED_SIZES); + tmp = alcu_size(ERTS_ALC_A_EHEAP, NULL, 0); + } tmp += erts_max_processes*sizeof(Process*); #ifdef HYBRID tmp += erts_max_processes*sizeof(Process*); @@ -2064,69 +2142,54 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) size.processes = size.processes_used = tmp; -#if HALFWORD_HEAP - /* BUG: We ignore link and monitor memory */ -#else - erts_fix_info(ERTS_ALC_T_NLINK_SH, &efi); - size.processes += efi.total; - size.processes_used += efi.used; + add_fix_values(&size.processes, + &size.processes_used, + fi, + ERTS_ALC_T_PROC); +#if !HALFWORD_HEAP + add_fix_values(&size.processes, + &size.processes_used, + fi, + ERTS_ALC_T_MONITOR_SH); - erts_fix_info(ERTS_ALC_T_MONITOR_SH, &efi); - size.processes += efi.total; - size.processes_used += efi.used; + add_fix_values(&size.processes, + &size.processes_used, + fi, + ERTS_ALC_T_NLINK_SH); #endif - - erts_fix_info(ERTS_ALC_T_PROC, &efi); - size.processes += efi.total; - size.processes_used += efi.used; - - erts_fix_info(ERTS_ALC_T_REG_PROC, &efi); - size.processes += efi.total; - size.processes_used += efi.used; - + add_fix_values(&size.processes, + &size.processes_used, + fi, + ERTS_ALC_T_MSG_REF); } if (want.atom || want.atom_used) { Uint reserved_atom_space, atom_space; erts_atom_get_text_space_sizes(&reserved_atom_space, &atom_space); size.atom = size.atom_used = atom_table_sz(); - erts_fix_info(ERTS_ALC_T_ATOM, &efi); - if (want.atom) { + if (want.atom) size.atom += reserved_atom_space; - size.atom += efi.total; - } - if (want.atom_used) { + if (want.atom_used) size.atom_used += atom_space; - size.atom_used += efi.used; - } } if (!ERTS_MEM_NEED_ALL_ALCU && want.binary) - size.binary = alcu_size(ERTS_ALC_A_BINARY); + size.binary = alcu_size(ERTS_ALC_A_BINARY, NULL, 0); if (want.code) { size.code = module_table_sz(); - erts_fix_info(ERTS_ALC_T_MODULE, &efi); - size.code += efi.used; size.code += export_table_sz(); -#if HALFWORD_HEAP size.code += export_list_size() * sizeof(Export); -#else - erts_fix_info(ERTS_ALC_T_EXPORT, &efi); - size.code += efi.used; -#endif size.code += erts_fun_table_sz(); - erts_fix_info(ERTS_ALC_T_FUN_ENTRY, &efi); - size.code += efi.used; size.code += allocated_modules*sizeof(Range); size.code += erts_total_code_size; } if (want.ets) { if (!ERTS_MEM_NEED_ALL_ALCU) - size.ets = alcu_size(ERTS_ALC_A_ETS); + size.ets = alcu_size(ERTS_ALC_A_ETS, NULL, 0); size.ets += erts_get_ets_misc_mem_size(); } @@ -2199,13 +2262,10 @@ struct aa_values { Eterm erts_allocated_areas(int *print_to_p, void *print_to_arg, void *proc) { -#define MAX_AA_VALUES \ - (20 + (ERTS_ALC_N_MAX_A_FIXED_SIZE - ERTS_ALC_N_MIN_A_FIXED_SIZE + 1)) - +#define MAX_AA_VALUES (23) struct aa_values values[MAX_AA_VALUES]; Eterm res = THE_NON_VALUE; int i, length; - ErtsFixInfo efi; Uint reserved_atom_space, atom_space; if (proc) { @@ -2270,6 +2330,11 @@ erts_allocated_areas(int *print_to_p, void *print_to_arg, void *proc) i++; values[i].arity = 2; + values[i].name = "export_list"; + values[i].ui[0] = export_list_size() * sizeof(Export); + i++; + + values[i].arity = 2; values[i].name = "register_table"; values[i].ui[0] = process_reg_sz(); i++; @@ -2314,22 +2379,15 @@ erts_allocated_areas(int *print_to_p, void *print_to_arg, void *proc) values[i].ui[0] = erts_tot_link_lh_size(); i++; - { - Uint n; - - for (n = ERTS_ALC_N_MIN_A_FIXED_SIZE; - n <= ERTS_ALC_N_MAX_A_FIXED_SIZE; - n++) { - erts_fix_info(ERTS_ALC_N2T(n), &efi); - - values[i].arity = 3; - values[i].name = ERTS_ALC_N2TD(n); - values[i].ui[0] = efi.total; - values[i].ui[1] = efi.used; - i++; - } + values[i].arity = 2; + values[i].name = "process_table"; + values[i].ui[0] = erts_max_processes*sizeof(Process*); + i++; - } + values[i].arity = 2; + values[i].name = "ets_misc"; + values[i].ui[0] = erts_get_ets_misc_mem_size(); + i++; length = i; ASSERT(length <= MAX_AA_VALUES); @@ -2423,17 +2481,16 @@ erts_alloc_util_allocators(void *proc) Uint sz; int i; /* - * Currently all allocators except sys_alloc and fix_alloc are + * Currently all allocators except sys_alloc are * alloc_util allocators. */ - sz = ((ERTS_ALC_A_MAX + 1 - ERTS_ALC_A_MIN) - 2)*2; + sz = ((ERTS_ALC_A_MAX + 1 - ERTS_ALC_A_MIN) - 1)*2; ASSERT(sz > 0); hp = HAlloc((Process *) proc, sz); res = NIL; for (i = ERTS_ALC_A_MAX; i >= ERTS_ALC_A_MIN; i--) { switch (i) { case ERTS_ALC_A_SYSTEM: - case ERTS_ALC_A_FIXED_SIZE: break; default: { char *alc_str = (char *) ERTS_ALC_A2AD(i); @@ -2447,267 +2504,12 @@ erts_alloc_util_allocators(void *proc) return res; } -Eterm -erts_allocator_info_term(void *proc, Eterm which_alloc, int only_sz) -{ -#define ERTS_AIT_RET(R) \ - do { res = (R); goto done; } while (0) -#define ERTS_AIT_HALLOC(P, S) \ - do { hp = HAlloc((P), (S)); hp_end = hp + (S); } while (0) - - ErtsAlcType_t i; - Uint sz = 0; - Uint *hp = NULL; - Uint *hp_end = NULL; - Eterm res = am_undefined; - - if (is_not_atom(which_alloc)) - goto done; - - for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) { - if (erts_is_atom_str((char *) ERTS_ALC_A2AD(i), which_alloc)) { - if (!erts_allctrs_info[i].enabled) - ERTS_AIT_RET(am_false); - else { - if (erts_allctrs_info[i].alloc_util) { - Eterm ires, tmp; - Eterm **hpp; - Uint *szp; - Eterm (*info_func)(Allctr_t *, - int, - int *, - void *, - Uint **, - Uint *); - - info_func = (only_sz - ? erts_alcu_sz_info - : erts_alcu_info); - - if (erts_allctrs_info[i].thr_spec) { - ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[i]; - int j; - int block_system = !tspec->all_thr_safe; - - if (block_system) { - erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(0); - } - ASSERT(tspec->enabled); - - szp = &sz; - hpp = NULL; - - while (1) { - ires = NIL; - for (j = tspec->size - 1; j >= 0; j--) { - Allctr_t *allctr = tspec->allctr[j]; - if (allctr) { - tmp = erts_bld_tuple(hpp, - szp, - 3, - erts_bld_atom(hpp, - szp, - "instance"), - make_small((Uint) j), - (*info_func)(allctr, - hpp != NULL, - NULL, - NULL, - hpp, - szp)); - ires = erts_bld_cons(hpp, szp, tmp, ires); - } - } - if (hpp) - break; - ERTS_AIT_HALLOC((Process *) proc, sz); - hpp = &hp; - szp = NULL; - } - - if (block_system) { - erts_smp_release_system(); - erts_smp_proc_lock(proc, ERTS_PROC_LOCK_MAIN); - } - } - else { - Allctr_t *allctr = erts_allctrs_info[i].extra; - szp = &sz; - hpp = NULL; - while (1) { - ires = NIL; - tmp = erts_bld_tuple(hpp, - szp, - 3, - erts_bld_atom(hpp, - szp, - "instance"), - make_small((Uint) 0), - (*info_func)(allctr, - hpp != NULL, - NULL, - NULL, - hpp, - szp)); - ires = erts_bld_cons(hpp, szp, tmp, ires); - if (hpp) - break; - ERTS_AIT_HALLOC((Process *) proc, sz); - hpp = &hp; - szp = NULL; - } - } - ERTS_AIT_RET(ires); - } - else { - Eterm *szp, **hpp; - - switch (i) { - case ERTS_ALC_A_SYSTEM: { - SysAllocStat sas; - Eterm opts_am; - Eterm opts; - Eterm as[4]; /* Ok even if !HEAP_ON_C_STACK, not really heap data on stack */ - Eterm ts[4]; /* Ok even if !HEAP_ON_C_STACK, not really heap data on stack */ - int l; - - if (only_sz) - ERTS_AIT_RET(NIL); - - sys_alloc_stat(&sas); - opts_am = am_atom_put("options", 7); - - szp = &sz; - hpp = NULL; - - restart_sys_alloc: - l = 0; - as[l] = am_atom_put("e", 1); - ts[l++] = am_true; - as[l] = am_atom_put("m", 1); - ts[l++] = am_atom_put("libc", 4); - if(sas.trim_threshold >= 0) { - as[l] = am_atom_put("tt", 2); - ts[l++] = erts_bld_uint(hpp, szp, - (Uint) sas.trim_threshold); - } - if(sas.top_pad >= 0) { - as[l] = am_atom_put("tp", 2); - ts[l++] = erts_bld_uint(hpp, szp, (Uint) sas.top_pad); - } - - opts = erts_bld_2tup_list(hpp, szp, l, as, ts); - res = erts_bld_2tup_list(hpp, szp, 1, &opts_am, &opts); - - if (szp) { - ERTS_AIT_HALLOC((Process *) proc, sz); - szp = NULL; - hpp = &hp; - goto restart_sys_alloc; - } - ERTS_AIT_RET(res); - } - case ERTS_ALC_A_FIXED_SIZE: { - ErtsAlcType_t n; - Eterm as[2], vs[2]; - - if (only_sz) - ERTS_AIT_RET(NIL); - - as[0] = am_atom_put("options", 7); - as[1] = am_atom_put("pools", 5); - - szp = &sz; - hpp = NULL; - - restart_fix_alloc: - - vs[0] = erts_bld_cons(hpp, szp, - erts_bld_tuple(hpp, szp, 2, - am_atom_put("e", - 1), - am_true), - NIL); - - vs[1] = NIL; - for (n = ERTS_ALC_N_MIN_A_FIXED_SIZE; - n <= ERTS_ALC_N_MAX_A_FIXED_SIZE; - n++) { - ErtsFixInfo efi; - erts_fix_info(ERTS_ALC_N2T(n), &efi); - - vs[1] = erts_bld_cons( - hpp, szp, - erts_bld_tuple( - hpp, szp, 3, - am_atom_put((char *) ERTS_ALC_N2TD(n), - strlen(ERTS_ALC_N2TD(n))), - erts_bld_uint(hpp, szp, efi.total), - erts_bld_uint(hpp, szp, efi.used)), - vs[1]); - - } - - res = erts_bld_2tup_list(hpp, szp, 2, as, vs); - if (szp) { - ERTS_AIT_HALLOC((Process *) proc, sz); - szp = NULL; - hpp = &hp; - goto restart_fix_alloc; - } - ERTS_AIT_RET(res); - } - default: - ASSERT(0); - goto done; - } - } - } - } - } - - if (ERTS_IS_ATOM_STR("mseg_alloc", which_alloc)) { -#if HAVE_ERTS_MSEG - if (only_sz) - ERTS_AIT_RET(NIL); - erts_mseg_info(NULL, NULL, 0, NULL, &sz); - if (sz) - ERTS_AIT_HALLOC((Process *) proc, sz); - ERTS_AIT_RET(erts_mseg_info(NULL, NULL, 1, &hp, NULL)); -#else - ERTS_AIT_RET(am_false); -#endif - - } - else if (ERTS_IS_ATOM_STR("alloc_util", which_alloc)) { - if (only_sz) - ERTS_AIT_RET(NIL); - erts_alcu_au_info_options(NULL, NULL, NULL, &sz); - if (sz) - ERTS_AIT_HALLOC((Process *) proc, sz); - ERTS_AIT_RET(erts_alcu_au_info_options(NULL, NULL, &hp, NULL)); - } - - done: - if (hp) { - ASSERT(hp_end >= hp); - HRelease((Process *) proc, hp_end, hp); - } - return res; - -#undef ERTS_AIT_RET -#undef ERTS_AIT_HALLOC -} - void erts_allocator_info(int to, void *arg) { ErtsAlcType_t a; - ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0) - || (ERTS_IS_CRASH_DUMPING - && erts_smp_is_system_blocked(ERTS_BS_FLG_ALLOW_GC))); + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); for (a = ERTS_ALC_A_MIN; a <= ERTS_ALC_A_MAX; a++) { int ai; @@ -2748,22 +2550,6 @@ erts_allocator_info(int to, void *arg) erts_print(to, arg, "option tp: %d\n", sas.top_pad); break; } - case ERTS_ALC_A_FIXED_SIZE: { - ErtsAlcType_t n; - erts_print(to, arg, "option e: true\n"); - - for (n = ERTS_ALC_N_MIN_A_FIXED_SIZE; - n <= ERTS_ALC_N_MAX_A_FIXED_SIZE; - n++) { - ErtsFixInfo efi; - erts_fix_info(ERTS_ALC_N2T(n), &efi); - erts_print(to, arg, "%s: %lu %lu\n", - ERTS_ALC_N2TD(n), - efi.total, - efi.used); - } - break; - } default: ASSERT(0); break; @@ -2774,8 +2560,18 @@ erts_allocator_info(int to, void *arg) } #if HAVE_ERTS_MSEG - erts_print(to, arg, "=allocator:mseg_alloc\n"); - erts_mseg_info(&to, arg, 0, NULL, NULL); + { +#ifdef ERTS_SMP + int max = (int) erts_no_schedulers; +#else + int max = 0; +#endif + int i; + for (i = 0; i <= max; i++) { + erts_print(to, arg, "=allocator:mseg_alloc[%d]\n", i); + erts_mseg_info(i, &to, arg, 0, NULL, NULL); + } + } #endif erts_print(to, arg, "=allocator:alloc_util\n"); @@ -2829,7 +2625,7 @@ erts_allocator_options(void *proc) use_mseg++; #endif if (erts_allctr_thr_spec[a].enabled) - allctr = erts_allctr_thr_spec[a].allctr[1]; + allctr = erts_allctr_thr_spec[a].allctr[0]; else allctr = erts_allctrs_info[a].extra; tmp = erts_alcu_info_options(allctr, NULL, NULL, hpp, szp); @@ -2878,7 +2674,7 @@ erts_allocator_options(void *proc) #if HAVE_ERTS_MSEG if (use_mseg) { atoms[length] = am_atom_put("mseg_alloc", 10); - terms[length++] = erts_mseg_info_options(NULL, NULL, hpp, szp); + terms[length++] = erts_mseg_info_options(0, NULL, NULL, hpp, szp); } #endif @@ -2982,6 +2778,313 @@ void *erts_alloc_permanent_cache_aligned(ErtsAlcType_t type, Uint size) return (void*)v; } +static void +reply_alloc_info(void *vair) +{ + ErtsAllocInfoReq *air = (ErtsAllocInfoReq *) vair; + Uint sched_id = erts_get_scheduler_id(); + int global_instances = air->req_sched == sched_id; + ErtsProcLocks rp_locks; + Process *rp = air->proc; + Eterm ref_copy = NIL, ai_list, msg; + Eterm *hp = NULL, *hp_end = NULL, *hp_start = NULL; + Eterm **hpp; + Uint sz, *szp; + ErlOffHeap *ohp = NULL; + ErlHeapFragment *bp = NULL; + int i; + Eterm (*info_func)(Allctr_t *, + int, + int *, + void *, + Uint **, + Uint *) = (air->only_sz + ? erts_alcu_sz_info + : erts_alcu_info); + + rp_locks = air->req_sched == sched_id ? ERTS_PROC_LOCK_MAIN : 0; + + sz = 0; + hpp = NULL; + szp = &sz; + + while (1) { + + if (hpp) + ref_copy = STORE_NC(hpp, ohp, air->ref); + else + *szp += REF_THING_SIZE; + + ai_list = NIL; + for (i = 0; air->allocs[i] != ERTS_ALC_A_INVALID; i++); + for (i--; i >= 0; i--) { + int ai = air->allocs[i]; + Allctr_t *allctr; + Eterm ainfo; + Eterm alloc_atom; + if (global_instances) { + switch (ai) { + case ERTS_ALC_A_SYSTEM: { + alloc_atom = erts_bld_atom(hpp, szp, "sys_alloc"); + ainfo = NIL; + if (!air->only_sz) { + SysAllocStat sas; + if (hpp) + sys_alloc_stat(&sas); + if (szp) { + /* ensure ehough heap */ + sas.top_pad = INT_MAX; + sas.trim_threshold = INT_MAX; + } + if (sas.top_pad >= 0) { + ainfo = erts_bld_cons( + hpp, szp, + erts_bld_tuple( + hpp, szp, 2, + erts_bld_atom(hpp, szp, "tp"), + erts_bld_uint( + hpp, szp, + (Uint) sas.top_pad)), + ainfo); + } + if (sas.trim_threshold >= 0) { + ainfo = erts_bld_cons( + hpp, szp, + erts_bld_tuple( + hpp, szp, 2, + erts_bld_atom(hpp, szp, "tt"), + erts_bld_uint( + hpp, szp, + (Uint) sas.trim_threshold)), + ainfo); + } + ainfo = erts_bld_cons(hpp, szp, + erts_bld_tuple( + hpp, szp, 2, + erts_bld_atom(hpp, szp, + "m"), + erts_bld_atom(hpp, szp, + "libc")), + ainfo); + ainfo = erts_bld_cons(hpp, szp, + erts_bld_tuple( + hpp, szp, 2, + erts_bld_atom(hpp, szp, + "e"), + am_true), + ainfo); + ainfo = erts_bld_tuple(hpp, szp, 2, + erts_bld_atom(hpp, szp, + "otps"), + ainfo); + } + ainfo = erts_bld_tuple(hpp, szp, 3, + alloc_atom, + make_small(0), + ainfo); + break; + } + case ERTS_ALC_INFO_A_ALLOC_UTIL: + alloc_atom = erts_bld_atom(hpp, szp, "alloc_util"); + ainfo = (air->only_sz + ? NIL + : erts_alcu_au_info_options(NULL, NULL, + hpp, szp)); + ainfo = erts_bld_tuple(hpp, szp, 3, + alloc_atom, + make_small(0), + ainfo); + break; + case ERTS_ALC_INFO_A_MSEG_ALLOC: + alloc_atom = erts_bld_atom(hpp, szp, "mseg_alloc"); +#if HAVE_ERTS_MSEG + ainfo = (air->only_sz + ? NIL + : erts_mseg_info(0, NULL, NULL, hpp != NULL, + hpp, szp)); + ainfo = erts_bld_tuple(hpp, szp, 3, + alloc_atom, + make_small(0), + ainfo); +#else + ainfo = erts_bld_tuple(hpp, szp, 2, alloc_atom, + am_false); +#endif + break; + default: + alloc_atom = erts_bld_atom(hpp, szp, + (char *) ERTS_ALC_A2AD(ai)); + if (!erts_allctrs_info[ai].enabled) + ainfo = erts_bld_tuple(hpp, szp, 2, alloc_atom, + am_false); + else if (erts_allctrs_info[ai].alloc_util) { + if (erts_allctrs_info[ai].thr_spec) + allctr = erts_allctr_thr_spec[ai].allctr[0]; + else + allctr = erts_allctrs_info[ai].extra; + ainfo = info_func(allctr, hpp != NULL, NULL, + NULL, hpp, szp); + ainfo = erts_bld_tuple(hpp, szp, 3, alloc_atom, + make_small(0), ainfo); + } + else { + erl_exit(ERTS_ABORT_EXIT, "%s:%d: internal error\n", + __FILE__, __LINE__); + } + } + ai_list = erts_bld_cons(hpp, szp, + ainfo, ai_list); + } + switch (ai) { + case ERTS_ALC_A_SYSTEM: + case ERTS_ALC_INFO_A_ALLOC_UTIL: + break; + case ERTS_ALC_INFO_A_MSEG_ALLOC: +#if HAVE_ERTS_MSEG && defined(ERTS_SMP) + alloc_atom = erts_bld_atom(hpp, szp, "mseg_alloc"); + ainfo = (air->only_sz + ? NIL + : erts_mseg_info(sched_id, NULL, NULL, + hpp != NULL, hpp, szp)); + ainfo = erts_bld_tuple(hpp, szp, 3, + alloc_atom, + make_small(sched_id), + ainfo); + ai_list = erts_bld_cons(hpp, szp, ainfo, ai_list); +#endif + break; + default: + if (erts_allctrs_info[ai].thr_spec) { + alloc_atom = erts_bld_atom(hpp, szp, + (char *) ERTS_ALC_A2AD(ai)); + allctr = erts_allctr_thr_spec[ai].allctr[sched_id]; + ainfo = info_func(allctr, hpp != NULL, NULL, + NULL, hpp, szp); + ai_list = erts_bld_cons(hpp, szp, + erts_bld_tuple( + hpp, szp, + 3, + alloc_atom, + make_small(sched_id), + ainfo), + ai_list); + } + break; + } + msg = erts_bld_tuple(hpp, szp, + 3, + ref_copy, + make_small(sched_id), + ai_list); + + } + if (hpp) + break; + + hp = erts_alloc_message_heap(sz, &bp, &ohp, rp, &rp_locks); + hp_start = hp; + hp_end = hp + sz; + szp = NULL; + hpp = &hp; + } + if (bp) + bp = erts_resize_message_buffer(bp, hp - hp_start, &msg, 1); + else { + ASSERT(hp); + HRelease(rp, hp_end, hp); + } + + erts_queue_message(rp, &rp_locks, bp, msg, NIL); + + if (air->req_sched == sched_id) + rp_locks &= ~ERTS_PROC_LOCK_MAIN; + + erts_smp_proc_unlock(rp, rp_locks); + erts_smp_proc_dec_refc(rp); + + if (erts_smp_atomic32_dec_read_nob(&air->refc) == 0) + aireq_free(air); +} + +int +erts_request_alloc_info(struct process *c_p, + Eterm ref, + Eterm allocs, + int only_sz) +{ + ErtsAllocInfoReq *air = aireq_alloc(); + Eterm req_ai[ERTS_ALC_A_MAX+1+2] = {0}; + Eterm alist; + Eterm *hp; + int airix = 0, ai; + + air->req_sched = erts_get_scheduler_id(); + + air->only_sz = only_sz; + + air->proc = c_p; + + if (is_not_internal_ref(ref)) + return 0; + + hp = &air->ref_heap[0]; + air->ref = STORE_NC(&hp, NULL, ref); + + if (is_not_list(allocs)) + return 0; + + alist = allocs; + + while (is_list(alist)) { + int saved = 0; + Eterm* consp = list_val(alist); + Eterm alloc = CAR(consp); + + for (ai = ERTS_ALC_A_MIN; ai <= ERTS_ALC_A_MAX; ai++) + if (erts_is_atom_str((char *) erts_alc_a2ad[ai], alloc)) + goto save_alloc; + if (erts_is_atom_str("mseg_alloc", alloc)) { + ai = ERTS_ALC_INFO_A_MSEG_ALLOC; + goto save_alloc; + } + if (erts_is_atom_str("alloc_util", alloc)) { + ai = ERTS_ALC_INFO_A_ALLOC_UTIL; + save_alloc: + if (req_ai[ai]) + return 0; + air->allocs[airix++] = ai; + req_ai[ai] = 1; + saved = 1; + } + + if (!saved) + return 0; + + alist = CDR(consp); + } + + if (is_not_nil(alist)) + return 0; + + air->allocs[airix] = ERTS_ALC_A_INVALID; + + erts_smp_atomic32_init_nob(&air->refc, + (erts_aint32_t) erts_no_schedulers); + + erts_smp_proc_add_refc(c_p, (Sint32) erts_no_schedulers); + +#ifdef ERTS_SMP + if (erts_no_schedulers > 1) + erts_schedule_multi_misc_aux_work(1, + erts_no_schedulers, + reply_alloc_info, + (void *) air); +#endif + + reply_alloc_info((void *) air); + + return 1; +} /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ * Deprecated functions * diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h index 80cb82c393..f4133cdb1a 100644 --- a/erts/emulator/beam/erl_alloc.h +++ b/erts/emulator/beam/erl_alloc.h @@ -43,43 +43,40 @@ # define ERTS_ALC_INLINE #endif -#define ERTS_FIX_CORE_ALLOCATOR ERTS_ALC_A_LONG_LIVED -extern ErtsAlcType_t erts_fix_core_allocator_ix; - -typedef struct { - Uint total; - Uint used; -} ErtsFixInfo; +#define ERTS_ALC_NO_FIXED_SIZES \ + (ERTS_ALC_N_MAX_A_FIXED_SIZE - ERTS_ALC_N_MIN_A_FIXED_SIZE + 1) void erts_sys_alloc_init(void); void *erts_sys_alloc(ErtsAlcType_t, void *, Uint); void *erts_sys_realloc(ErtsAlcType_t, void *, void *, Uint); void erts_sys_free(ErtsAlcType_t, void *, void *); - -void erts_init_fix_alloc(Uint, void *(*)(Uint)); -Uint erts_get_fix_size(ErtsAlcType_t); -void erts_set_fix_size(ErtsAlcType_t, Uint); -void erts_fix_info(ErtsAlcType_t, ErtsFixInfo *); -void *erts_fix_alloc(ErtsAlcType_t, void *, Uint); -void *erts_fix_realloc(ErtsAlcType_t, void *, void*, Uint); -void erts_fix_free(ErtsAlcType_t, void *, void*); - - Eterm erts_memory(int *, void *, void *, Eterm); Eterm erts_allocated_areas(int *, void *, void *); Eterm erts_alloc_util_allocators(void *proc); void erts_allocator_info(int, void *); -Eterm erts_allocator_info_term(void *proc, Eterm which_alloc, int only_sz); Eterm erts_allocator_options(void *proc); +struct process; + +int erts_request_alloc_info(struct process *c_p, Eterm ref, Eterm allocs, + int only_sz); + #define ERTS_ALLOC_INIT_DEF_OPTS_INITER {0} typedef struct { - int dummy; + int ncpu; } ErtsAllocInitOpts; +typedef struct { + Allctr_t *deallctr[ERTS_ALC_A_MAX+1]; + int pref_ix[ERTS_ALC_A_MAX+1]; + int flist_ix[ERTS_ALC_A_MAX+1]; + int pre_alc_ix; +} ErtsSchedAllocData; + void erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop); +void erts_alloc_late_init(void); #if defined(GET_ERTS_ALC_TEST) || defined(ERTS_ALC_INTERNAL__) /* Only for testing */ @@ -126,15 +123,19 @@ extern ErtsAllocatorInfo_t erts_allctrs_info[ERTS_ALC_A_MAX+1]; typedef struct { int enabled; - int all_thr_safe; + int dd; + int aix; int size; Allctr_t **allctr; } ErtsAllocatorThrSpec_t; extern ErtsAllocatorThrSpec_t erts_allctr_thr_spec[ERTS_ALC_A_MAX+1]; -int erts_alc_get_thr_ix(void); -void erts_alloc_reg_scheduler_id(Uint id); +void erts_alloc_register_scheduler(void *vesdp); +void erts_alloc_scheduler_handle_delayed_dealloc(void *vesdp, + int *need_thr_progress, + int *more_work); +erts_aint32_t erts_alloc_fix_alloc_shrink(int ix, erts_aint32_t flgs); __decl_noreturn void erts_alloc_enomem(ErtsAlcType_t,Uint) __noreturn; @@ -252,6 +253,8 @@ void *erts_realloc_fnf(ErtsAlcType_t type, void *ptr, Uint size) #endif /* #if ERTS_ALC_DO_INLINE || defined(ERTS_ALC_INTERNAL__) */ +#define ERTS_ALC_GET_THR_IX() ((int) erts_get_scheduler_id()) + typedef void (*erts_alloc_verify_func_t)(Allctr_t *); erts_alloc_verify_func_t @@ -436,136 +439,41 @@ NAME##_free(TYPE *p) \ } \ } -typedef struct { - void *start; - void *end; - int chunks_mem_size; -} erts_sched_pref_quick_alloc_data_t; - -#ifdef DEBUG -#define ERTS_SPPA_DBG_CHK_IN_CHNK(A, C, P) \ -do { \ - ASSERT((void *) (C) < (void *) (P)); \ - ASSERT((void *) (P) \ - < (void *) (((char *) (C)) + (A)->chunks_mem_size)); \ -} while (0) -#else -#define ERTS_SPPA_DBG_CHK_IN_CHNK(A, C, P) -#endif +#include "erl_sched_spec_pre_alloc.h" #define ERTS_SCHED_PREF_PRE_ALLOC_IMPL(NAME, TYPE, PASZ) \ -union erts_qa_##NAME##__ { \ +union erts_sspa_##NAME##__ { \ + erts_sspa_blk_t next; \ TYPE type; \ - union erts_qa_##NAME##__ *next; \ }; \ -typedef struct { \ - erts_smp_spinlock_t lock; \ - union erts_qa_##NAME##__ *freelist; \ - union erts_qa_##NAME##__ pre_alloced[1]; \ -} erts_qa_##NAME##_chunk__; \ -static erts_sched_pref_quick_alloc_data_t *qa_data_##NAME##__; \ -static ERTS_INLINE erts_qa_##NAME##_chunk__ * \ -get_##NAME##_chunk_ix(int cix) \ -{ \ - char *ptr = (char *) qa_data_##NAME##__->start; \ - ptr += cix*qa_data_##NAME##__->chunks_mem_size; \ - return (erts_qa_##NAME##_chunk__ *) ptr; \ -} \ -static ERTS_INLINE erts_qa_##NAME##_chunk__ * \ -get_##NAME##_chunk_ptr(void *ptr) \ -{ \ - int cix; \ - size_t diff; \ - if (ptr < qa_data_##NAME##__->start || qa_data_##NAME##__->end <= ptr)\ - return NULL; \ - diff = ((char *) ptr) - ((char *) qa_data_##NAME##__->start); \ - cix = diff / qa_data_##NAME##__->chunks_mem_size; \ - return get_##NAME##_chunk_ix(cix); \ -} \ + \ +static erts_sspa_data_t *sspa_data_##NAME##__; \ + \ static void \ init_##NAME##_alloc(void) \ { \ - size_t tot_size; \ - size_t chunk_mem_size; \ - char *chunk_start; \ - int cix; \ - int no_blocks = ERTS_PRE_ALLOC_SIZE((PASZ)); \ - int no_blocks_per_chunk = 2*((no_blocks-1)/erts_no_schedulers + 1); \ - no_blocks = no_blocks_per_chunk * erts_no_schedulers; \ - chunk_mem_size = sizeof(erts_qa_##NAME##_chunk__); \ - chunk_mem_size += (sizeof(union erts_qa_##NAME##__) \ - * (no_blocks_per_chunk - 1)); \ - chunk_mem_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(chunk_mem_size); \ - tot_size = sizeof(erts_sched_pref_quick_alloc_data_t); \ - tot_size += ERTS_CACHE_LINE_SIZE - 1; \ - tot_size += chunk_mem_size*erts_no_schedulers; \ - qa_data_##NAME##__ = erts_alloc(ERTS_ALC_T_PRE_ALLOC_DATA,tot_size);\ - chunk_start = (((char *) qa_data_##NAME##__) \ - + sizeof(erts_sched_pref_quick_alloc_data_t)); \ - if ((((UWord) chunk_start) & ERTS_CACHE_LINE_MASK) != ((UWord) 0)) \ - chunk_start = ((char *) \ - ((((UWord) chunk_start) & ~ERTS_CACHE_LINE_MASK) \ - + ERTS_CACHE_LINE_SIZE)); \ - qa_data_##NAME##__->chunks_mem_size = chunk_mem_size; \ - qa_data_##NAME##__->start = (void *) chunk_start; \ - qa_data_##NAME##__->end = (chunk_start \ - + chunk_mem_size*erts_no_schedulers); \ - for (cix = 0; cix < erts_no_schedulers; cix++) { \ - int i; \ - erts_qa_##NAME##_chunk__ *chunk = get_##NAME##_chunk_ix(cix); \ - erts_smp_spinlock_init(&chunk->lock, #NAME "_alloc_lock"); \ - chunk->freelist = &chunk->pre_alloced[0]; \ - for (i = 1; i < no_blocks_per_chunk; i++) { \ - ERTS_PRE_ALLOC_CLOBBER(&chunk->pre_alloced[i-1], \ - union erts_qa_##NAME##__); \ - chunk->pre_alloced[i-1].next = &chunk->pre_alloced[i]; \ - } \ - ERTS_PRE_ALLOC_CLOBBER(&chunk->pre_alloced[no_blocks_per_chunk-1],\ - union erts_qa_##NAME##__); \ - chunk->pre_alloced[no_blocks_per_chunk-1].next = NULL; \ - } \ + sspa_data_##NAME##__ = \ + erts_sspa_create(sizeof(union erts_sspa_##NAME##__), \ + ERTS_PRE_ALLOC_SIZE((PASZ))); \ } \ -static ERTS_INLINE TYPE * \ + \ +static TYPE * \ NAME##_alloc(void) \ { \ - int cix = ((int) erts_get_scheduler_id()) - 1; \ - TYPE *res; \ - if (cix < 0) \ - res = NULL; \ - else { \ - erts_qa_##NAME##_chunk__ *chunk = get_##NAME##_chunk_ix(cix); \ - erts_smp_spin_lock(&chunk->lock); \ - if (!chunk->freelist) \ - res = NULL; \ - else { \ - res = &chunk->freelist->type; \ - chunk->freelist = chunk->freelist->next; \ - ERTS_SPPA_DBG_CHK_IN_CHNK(qa_data_##NAME##__, chunk, res); \ - } \ - erts_smp_spin_unlock(&chunk->lock); \ - } \ - return res; \ + ErtsSchedulerData *esdp = erts_get_scheduler_data(); \ + if (!esdp) \ + return NULL; \ + return (TYPE *) erts_sspa_alloc(sspa_data_##NAME##__, \ + (int) esdp->no - 1); \ } \ -static ERTS_INLINE int \ + \ +static int \ NAME##_free(TYPE *p) \ { \ - erts_qa_##NAME##_chunk__ *chunk; \ - chunk = get_##NAME##_chunk_ptr((void *) p); \ - if (!chunk) \ - return 0; \ - else { \ - union erts_qa_##NAME##__ *up; \ - ERTS_SPPA_DBG_CHK_IN_CHNK(qa_data_##NAME##__, chunk, p); \ - up = ((union erts_qa_##NAME##__ *) \ - (((char *) p) \ - - ((char *) &((union erts_qa_##NAME##__ *) 0)->type))); \ - erts_smp_spin_lock(&chunk->lock); \ - ERTS_PRE_ALLOC_CLOBBER(up, union erts_qa_##NAME##__); \ - up->next = chunk->freelist; \ - chunk->freelist = up; \ - erts_smp_spin_unlock(&chunk->lock); \ - return 1; \ - } \ + ErtsSchedulerData *esdp = erts_get_scheduler_data(); \ + return erts_sspa_free(sspa_data_##NAME##__, \ + esdp ? (int) esdp->no - 1 : -1, \ + (char *) p); \ } #ifdef DEBUG diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index eda0831441..962db8b831 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -50,6 +50,15 @@ # command line argument to make_alloc_types. The variable X is false # after a "+disable X" statement or if it has never been mentioned. ++if smp ++disable threads_no_smp ++else ++if threads ++enable threads_no_smp ++else ++disable threads_no_smp ++endif ++endif # --- Allocator declarations ------------------------------------------------- # @@ -133,29 +142,25 @@ class SYSTEM system_data # should be deallocated before the emulator starts executing Erlang # code again. # -# NOTE: When adding or removing a type which uses the FIXED_SIZE allocator, -# also add or remove initialization of the type in erts_alloc_init() -# (erl_alloc.c). -# # <TYPE> <ALLOCATOR> <CLASS> <DESCRIPTION> type SBMBC SBMBC SYSTEM small_block_mbc type PROC FIXED_SIZE PROCESSES proc -type ATOM FIXED_SIZE ATOM atom_entry -type MODULE FIXED_SIZE CODE module_entry -type REG_PROC FIXED_SIZE PROCESSES reg_proc +type ATOM LONG_LIVED ATOM atom_entry +type MODULE LONG_LIVED CODE module_entry +type REG_PROC STANDARD PROCESSES reg_proc type LINK_LH STANDARD PROCESSES link_lh type SUSPEND_MON STANDARD PROCESSES suspend_monitor type PEND_SUSPEND SHORT_LIVED PROCESSES pending_suspend type PROC_LIST SHORT_LIVED PROCESSES proc_list -type FUN_ENTRY FIXED_SIZE CODE fun_entry +type FUN_ENTRY LONG_LIVED CODE fun_entry type ATOM_TXT LONG_LIVED ATOM atom_text type BEAM_REGISTER EHEAP PROCESSES beam_register type HEAP EHEAP PROCESSES heap type OLD_HEAP EHEAP PROCESSES old_heap type HEAP_FRAG EHEAP PROCESSES heap_frag type TMP_HEAP TEMPORARY PROCESSES tmp_heap -type MSG_REF SHORT_LIVED PROCESSES msg_ref +type MSG_REF FIXED_SIZE PROCESSES msg_ref type MSG_ROOTS TEMPORARY PROCESSES msg_roots type ROOTSET TEMPORARY PROCESSES root_set type LOADER_TMP TEMPORARY CODE loader_tmp @@ -196,10 +201,10 @@ type LINEBUF STANDARD SYSTEM line_buf type IOQ STANDARD SYSTEM io_queue type BITS_BUF STANDARD SYSTEM bits_buf type TMP_DIST_BUF TEMPORARY SYSTEM tmp_dist_buf -type ASYNC_Q LONG_LIVED SYSTEM async_queue +type ASYNC_DATA LONG_LIVED SYSTEM internal_async_data type ESTACK TEMPORARY SYSTEM estack type PORT_CALL_BUF TEMPORARY SYSTEM port_call_buf -type DB_TABLE FIXED_SIZE ETS db_tab +type DB_TABLE ETS ETS db_tab type DB_FIXATION SHORT_LIVED ETS db_fixation type DB_FIX_DEL SHORT_LIVED ETS fixed_del type DB_TABLES LONG_LIVED ETS db_tabs @@ -256,6 +261,23 @@ type TMP_CPU_IDS SHORT_LIVED SYSTEM tmp_cpu_ids type EXT_TERM_DATA SHORT_LIVED PROCESSES external_term_data type ZLIB STANDARD SYSTEM zlib type CPU_GRPS_MAP LONG_LIVED SYSTEM cpu_groups_map +type AUX_WORK_TMO LONG_LIVED SYSTEM aux_work_timeouts +type MISC_AUX_WORK_Q LONG_LIVED SYSTEM misc_aux_work_q + ++if threads_no_smp +# Need thread safe allocs, but std_alloc and fix_alloc are not; +# use driver_alloc which is... +type THR_Q_EL DRIVER SYSTEM thr_q_element +type THR_Q_EL_SL DRIVER SYSTEM sl_thr_q_element +type MISC_AUX_WORK DRIVER SYSTEM misc_aux_work ++else +type THR_Q_EL STANDARD SYSTEM thr_q_element +type THR_Q_EL_SL FIXED_SIZE SYSTEM sl_thr_q_element +type MISC_AUX_WORK SHORT_LIVED SYSTEM misc_aux_work ++endif +type THR_Q STANDARD SYSTEM thr_queue +type THR_Q_SL SHORT_LIVED SYSTEM short_lived_thr_queue +type THR_Q_LL LONG_LIVED SYSTEM long_lived_thr_queue +if smp type ASYNC SHORT_LIVED SYSTEM async @@ -271,8 +293,9 @@ type XPORTS_LIST SHORT_LIVED SYSTEM extra_port_list type PROC_LCK_WTR LONG_LIVED SYSTEM proc_lock_waiter type PROC_LCK_QS LONG_LIVED SYSTEM proc_lock_queues type RUNQ_BLNS LONG_LIVED SYSTEM run_queue_balancing -type MISC_AUX_WORK_Q LONG_LIVED SYSTEM misc_aux_work_q -type MISC_AUX_WORK SHORT_LIVED SYSTEM misc_aux_work +type THR_PRGR_IDATA LONG_LIVED SYSTEM thr_prgr_internal_data +type THR_PRGR_DATA LONG_LIVED SYSTEM thr_prgr_data +type T_THR_PRGR_DATA SHORT_LIVED SYSTEM temp_thr_prgr_data +endif # @@ -285,12 +308,6 @@ type ETHR_STD STANDARD SYSTEM ethread_standard type ETHR_SL SHORT_LIVED SYSTEM ethread_short_lived type ETHR_LL LONG_LIVED SYSTEM ethread_long_lived -+ifnot smp - -type ARCALLBACK LONG_LIVED SYSTEM async_ready_callback - -+endif - +endif +if shared_heap @@ -346,10 +363,10 @@ type DB_MS_PSDO_PROC LONG_LIVED_LOW ETS db_match_pseudo_proc type SCHDLR_DATA LONG_LIVED_LOW SYSTEM scheduler_data type LL_TEMP_TERM LONG_LIVED_LOW SYSTEM ll_temp_term -# no FIXED_SIZE for low memory -type EXPORT STANDARD_LOW CODE export_entry +type EXPORT LONG_LIVED_LOW CODE export_entry type MONITOR_SH STANDARD_LOW PROCESSES monitor_sh type NLINK_SH STANDARD_LOW PROCESSES nlink_sh +type AINFO_REQ STANDARD_LOW SYSTEM alloc_info_request +else # "fullword" @@ -362,9 +379,10 @@ type DB_MS_PSDO_PROC LONG_LIVED ETS db_match_pseudo_proc type SCHDLR_DATA LONG_LIVED SYSTEM scheduler_data type LL_TEMP_TERM LONG_LIVED SYSTEM ll_temp_term -type EXPORT FIXED_SIZE CODE export_entry +type EXPORT LONG_LIVED CODE export_entry type MONITOR_SH FIXED_SIZE PROCESSES monitor_sh type NLINK_SH FIXED_SIZE PROCESSES nlink_sh +type AINFO_REQ SHORT_LIVED SYSTEM alloc_info_request +endif diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index d51ed0c36d..af386c9197 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -46,6 +46,7 @@ #include "erl_alloc_util.h" #include "erl_mseg.h" #include "erl_threads.h" +#include "erl_thr_progress.h" #ifdef ERTS_ENABLE_LOCK_COUNT #include "erl_lock_count.h" @@ -61,6 +62,13 @@ #warning "* * * * * * * * * *" #endif +#define ERTS_ALCU_DD_OPS_LIM_HIGH 20 +#define ERTS_ALCU_DD_OPS_LIM_LOW 2 + +/* Fix alloc limit */ +#define ERTS_ALCU_FIX_MAX_LIST_SZ 1000 +#define ERTS_ALC_FIX_MAX_SHRINK_OPS 30 + #define ALLOC_ZERO_EQ_NULL 0 static int atoms_initialized = 0; @@ -269,7 +277,6 @@ static void check_blk_carrier(Allctr_t *, Block_t *); #define HARD_CHECK_BLK_CARRIER(A, B) #endif - /* Statistics updating ... */ #ifdef DEBUG @@ -465,26 +472,34 @@ do { \ #ifdef DEBUG #ifdef USE_THREADS -#define ERTS_ALCU_DBG_CHK_THR_SPEC(A) \ +#define ERTS_ALCU_DBG_CHK_THR_ACCESS(A) \ do { \ if (!(A)->thread_safe) { \ - if (!(A)->debug.saved_tid) \ + if (!(A)->debug.saved_tid) { \ (A)->debug.tid = erts_thr_self(); \ + (A)->debug.saved_tid = 1; \ + } \ else { \ - ASSERT(ethr_equal_tids((A)->debug.tid, erts_thr_self())); \ + ERTS_SMP_LC_ASSERT( \ + ethr_equal_tids((A)->debug.tid, erts_thr_self()) \ + || erts_thr_progress_is_blocking()); \ } \ } \ } while (0) #else -#define ERTS_ALCU_DBG_CHK_THR_SPEC(A) +#define ERTS_ALCU_DBG_CHK_THR_ACCESS(A) #endif #else -#define ERTS_ALCU_DBG_CHK_THR_SPEC(A) +#define ERTS_ALCU_DBG_CHK_THR_ACCESS(A) #endif static void make_name_atoms(Allctr_t *allctr); +static Block_t *create_carrier(Allctr_t *, Uint, UWord); +static void destroy_carrier(Allctr_t *, Block_t *); +static void mbc_free(Allctr_t *allctr, void *p); + /* mseg ... */ @@ -651,6 +666,446 @@ static void destroy_sbmbc(Allctr_t *allctr, Block_t *blk); static Block_t *create_carrier(Allctr_t *, Uint, UWord); static void destroy_carrier(Allctr_t *, Block_t *); +#if 0 +#define ERTS_DBG_CHK_FIX_LIST(A, FIX, IX, B) \ + do { if ((FIX)) chk_fix_list((A), (FIX), (IX), (B)); } while (0) +static void +chk_fix_list(Allctr_t *allctr, ErtsAlcFixList_t *fix, int ix, int before) +{ + void *p; + int n; + for (n = 0, p = fix[ix].list; p; p = *((void **) p)) + n++; + if (n != fix[ix].list_size) { + erts_fprintf(stderr, "FOUND IT ts=%d, sched=%d, ix=%d, n=%d, ls=%d %s!\n", + allctr->thread_safe, allctr->ix, ix, n, fix[ix].list_size, before ? "before" : "after"); + abort(); + } +} +#else +#define ERTS_DBG_CHK_FIX_LIST(A, FIX, IX, B) +#endif + +erts_aint32_t +erts_alcu_fix_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs) +{ + int all_empty = 1; + erts_aint32_t res = 0; + int ix, o; + ErtsAlcFixList_t *fix = allctr->fix; + int flush = flgs == 0; + +#ifdef USE_THREADS + if (allctr->thread_safe) + erts_mtx_lock(&allctr->mutex); +#endif + + for (ix = 0; ix < ERTS_ALC_NO_FIXED_SIZES; ix++) { + ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1); + if (flgs & ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM) { + fix[ix].limit = fix[ix].max_used; + if (fix[ix].limit < fix[ix].used) + fix[ix].limit = fix[ix].used; + fix[ix].max_used = fix[ix].used; + ASSERT(fix[ix].limit >= 0); + + } + if (flush) { + fix[ix].limit = 0; + fix[ix].max_used = fix[ix].used; + ASSERT(fix[ix].limit >= 0); + } + for (o = 0; o < ERTS_ALC_FIX_MAX_SHRINK_OPS || flush; o++) { + Block_t *blk; + void *ptr; + + if (!flush && fix[ix].limit >= fix[ix].allocated) + break; + if (fix[ix].list_size == 0) + break; + ptr = fix[ix].list; + fix[ix].list = *((void **) ptr); + fix[ix].list_size--; + + blk = UMEM2BLK(ptr); + + if (IS_SBC_BLK(blk)) + destroy_carrier(allctr, blk); + else + mbc_free(allctr, ptr); + + fix[ix].allocated--; + } + if (fix[ix].list_size != 0) { + if (fix[ix].limit < fix[ix].allocated) + res |= ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC; + all_empty = 0; + } + ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); + } + + if (all_empty && allctr->fix_shrink_scheduled) { + allctr->fix_shrink_scheduled = 0; + erts_set_aux_work_timeout(allctr->ix, + (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM + | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC), + 0); + } + +#ifdef USE_THREADS + if (allctr->thread_safe) + erts_mtx_unlock(&allctr->mutex); +#endif + + return res; +} + +#ifdef ERTS_SMP + +#define ERTS_ALCU_DD_FIX_TYPE_OFFS \ + ((sizeof(ErtsAllctrDDBlock_t)-1)/sizeof(UWord) + 1) + +#define ERTS_AU_PREF_ALLOC_IX_MASK \ + ((((UWord) 1) << ERTS_AU_PREF_ALLOC_BITS) - 1) +#define ERTS_AU_PREF_ALLOC_SIZE_MASK \ + ((((UWord) 1) << (sizeof(UWord)*8 - ERTS_AU_PREF_ALLOC_BITS)) - 1) + +static ERTS_INLINE int +get_pref_allctr(void *extra, Allctr_t **allctr) +{ + ErtsAllocatorThrSpec_t *tspec = (ErtsAllocatorThrSpec_t *) extra; + int pref_ix; + + pref_ix = ERTS_ALC_GET_THR_IX(); + + ASSERT(sizeof(UWord) == sizeof(Allctr_t *)); + ASSERT(0 <= pref_ix && pref_ix < tspec->size); + + *allctr = tspec->allctr[pref_ix]; + return pref_ix; +} + +static ERTS_INLINE void * +get_used_allctr(void *extra, void *p, Allctr_t **allctr, UWord *sizep) +{ + ErtsAllocatorThrSpec_t *tspec = (ErtsAllocatorThrSpec_t *) extra; + void *ptr = (void *) (((char *) p) - sizeof(UWord)); + UWord ainfo = *((UWord *) ptr); + int aix = (int) (ainfo & ERTS_AU_PREF_ALLOC_IX_MASK); + *allctr = tspec->allctr[aix]; + if (sizep) + *sizep = ((ainfo >> ERTS_AU_PREF_ALLOC_BITS) + & ERTS_AU_PREF_ALLOC_SIZE_MASK); + return ptr; +} + +static ERTS_INLINE void * +put_used_allctr(void *p, int ix, UWord size) +{ + UWord ainfo = (size >= ERTS_AU_PREF_ALLOC_SIZE_MASK + ? ERTS_AU_PREF_ALLOC_SIZE_MASK + : size); + ainfo <<= ERTS_AU_PREF_ALLOC_BITS; + ainfo |= (UWord) ix; + *((UWord *) p) = ainfo; + return (void *) (((char *) p) + sizeof(UWord)); +} + +static void +init_dd_queue(ErtsAllctrDDQueue_t *ddq) +{ + erts_atomic_init_nob(&ddq->tail.data.marker.atmc_next, ERTS_AINT_NULL); + erts_atomic_init_nob(&ddq->tail.data.last, + (erts_aint_t) &ddq->tail.data.marker); + erts_atomic_init_nob(&ddq->tail.data.um_refc[0], 0); + erts_atomic_init_nob(&ddq->tail.data.um_refc[1], 0); + erts_atomic32_init_nob(&ddq->tail.data.um_refc_ix, 0); + ddq->head.first = &ddq->tail.data.marker; + ddq->head.unref_end = &ddq->tail.data.marker; + ddq->head.next.thr_progress = erts_thr_progress_current(); + ddq->head.next.thr_progress_reached = 1; + ddq->head.next.um_refc_ix = 1; + ddq->head.next.unref_end = &ddq->tail.data.marker; + ddq->head.used_marker = 1; +} + +static ERTS_INLINE erts_aint_t +ddq_managed_thread_enqueue(ErtsAllctrDDQueue_t *ddq, void *ptr) +{ + erts_aint_t ilast, itmp; + ErtsAllctrDDBlock_t *this = ptr; + + erts_atomic_init_nob(&this->atmc_next, ERTS_AINT_NULL); + + /* Enqueue at end of list... */ + + ilast = erts_atomic_read_nob(&ddq->tail.data.last); + while (1) { + ErtsAllctrDDBlock_t *last = (ErtsAllctrDDBlock_t *) ilast; + itmp = erts_atomic_cmpxchg_mb(&last->atmc_next, + (erts_aint_t) this, + ERTS_AINT_NULL); + if (itmp == ERTS_AINT_NULL) + break; + ilast = itmp; + } + + /* Move last pointer forward... */ + while (1) { + if (erts_atomic_read_rb(&this->atmc_next) != ERTS_AINT_NULL) { + /* Someone else will move it forward */ + return erts_atomic_read_rb(&ddq->tail.data.last); + } + itmp = erts_atomic_cmpxchg_mb(&ddq->tail.data.last, + (erts_aint_t) this, + ilast); + if (ilast == itmp) + return (erts_aint_t) this; + ilast = itmp; + } +} + +static ERTS_INLINE int +ddq_enqueue(ErtsAlcType_t type, ErtsAllctrDDQueue_t *ddq, void *ptr) +{ + erts_aint_t ilast; + int um_refc_ix = 0; + int managed_thread = erts_thr_progress_is_managed_thread(); + if (!managed_thread) { + um_refc_ix = erts_atomic32_read_acqb(&ddq->tail.data.um_refc_ix); + while (1) { + int tmp_um_refc_ix; + erts_atomic_inc_acqb(&ddq->tail.data.um_refc[um_refc_ix]); + tmp_um_refc_ix = erts_atomic32_read_acqb(&ddq->tail.data.um_refc_ix); + if (tmp_um_refc_ix == um_refc_ix) + break; + erts_atomic_dec_relb(&ddq->tail.data.um_refc[um_refc_ix]); + um_refc_ix = tmp_um_refc_ix; + } + } + + ilast = ddq_managed_thread_enqueue(ddq, ptr); + + if (!managed_thread) + erts_atomic_dec_relb(&ddq->tail.data.um_refc[um_refc_ix]); + return ilast == (erts_aint_t) ptr; +} + +static ERTS_INLINE void * +ddq_dequeue(ErtsAllctrDDQueue_t *ddq) +{ + ErtsAllctrDDBlock_t *blk; + + if (ddq->head.first == ddq->head.unref_end) + return NULL; + + blk = ddq->head.first; + if (blk == &ddq->tail.data.marker) { + ASSERT(ddq->head.used_marker); + ddq->head.used_marker = 0; + blk = ((ErtsAllctrDDBlock_t *) + erts_atomic_read_nob(&blk->atmc_next)); + if (blk == ddq->head.unref_end) { + ddq->head.first = blk; + return NULL; + } + } + + ddq->head.first = ((ErtsAllctrDDBlock_t *) + erts_atomic_read_nob(&blk->atmc_next)); + + ASSERT(ddq->head.first); + + return (void *) blk; +} + +static int +ddq_check_incoming(ErtsAllctrDDQueue_t *ddq) +{ + erts_aint_t ilast = erts_atomic_read_nob(&ddq->tail.data.last); + if (((ErtsAllctrDDBlock_t *) ilast) == &ddq->tail.data.marker + && ddq->head.first == &ddq->tail.data.marker) { + /* Nothing more to do... */ + return 0; + } + + if (ddq->head.next.thr_progress_reached + || erts_thr_progress_has_reached(ddq->head.next.thr_progress)) { + int um_refc_ix; + ddq->head.next.thr_progress_reached = 1; + um_refc_ix = ddq->head.next.um_refc_ix; + if (erts_atomic_read_acqb(&ddq->tail.data.um_refc[um_refc_ix]) == 0) { + /* Move unreferenced end pointer forward... */ + + ddq->head.unref_end = ddq->head.next.unref_end; + + if (!ddq->head.used_marker + && ddq->head.unref_end == (ErtsAllctrDDBlock_t *) ilast) { + ddq->head.used_marker = 1; + ilast = ddq_managed_thread_enqueue(ddq, &ddq->tail.data.marker); + } + + if (ddq->head.unref_end == (ErtsAllctrDDBlock_t *) ilast) + ERTS_THR_MEMORY_BARRIER; + else { + ddq->head.next.unref_end = (ErtsAllctrDDBlock_t *) ilast; + ERTS_THR_MEMORY_BARRIER; + ddq->head.next.thr_progress = erts_thr_progress_later(); + erts_atomic32_set_relb(&ddq->tail.data.um_refc_ix, + um_refc_ix); + ddq->head.next.um_refc_ix = um_refc_ix == 0 ? 1 : 0; + ddq->head.next.thr_progress_reached = 0; + } + } + } + return 1; +} + +static ERTS_INLINE int +handle_delayed_dealloc(Allctr_t *allctr, + int allctr_locked, + int use_limit, + int ops_limit, + int *need_thr_progress, + int *need_more_work) +{ + int need_thr_prgr = 0; + int need_mr_wrk = 0; + int have_checked_incoming = 0; + int ops = 0; + ErtsAlcFixList_t *fix; + int res; + ErtsAllctrDDQueue_t *ddq; + + if (allctr->thread_safe && !allctr_locked) + erts_mtx_lock(&allctr->mutex); + + ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr); + + fix = allctr->fix; + + ddq = &allctr->dd.q; + + res = 0; + + while (1) { + Block_t *blk; + void *ptr; + int ix; + + if (use_limit && ++ops > ops_limit) { + if (ddq->head.first != ddq->head.unref_end) { + need_mr_wrk = 1; + if (need_more_work) + *need_more_work |= 1; + } + break; + } + + dequeue: + ptr = ddq_dequeue(ddq); + if (!ptr) { + if (have_checked_incoming) + break; + need_thr_prgr = ddq_check_incoming(ddq); + if (need_thr_progress) + *need_thr_progress |= need_thr_prgr; + have_checked_incoming = 1; + goto dequeue; + } + + res = 1; + + INC_CC(allctr->calls.this_free); + + if (fix) { + ErtsAlcType_t type; + + type = (ErtsAlcType_t) ((UWord *) ptr)[ERTS_ALCU_DD_FIX_TYPE_OFFS]; + ix = type - ERTS_ALC_N_MIN_A_FIXED_SIZE; + ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1); + fix[ix].used--; + if (fix[ix].allocated < fix[ix].limit + && fix[ix].list_size < ERTS_ALCU_FIX_MAX_LIST_SZ) { + *((void **) ptr) = fix[ix].list; + fix[ix].list = ptr; + fix[ix].list_size++; + if (!allctr->fix_shrink_scheduled) { + allctr->fix_shrink_scheduled = 1; + erts_set_aux_work_timeout( + allctr->ix, + (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM + | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC), + 1); + } + ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); + continue; + } + fix[ix].allocated--; + if (fix[ix].list && fix[ix].allocated > fix[ix].limit) { + blk = UMEM2BLK(ptr); + if (IS_SBC_BLK(blk)) + destroy_carrier(allctr, blk); + else + mbc_free(allctr, ptr); + ptr = fix[ix].list; + fix[ix].list = *((void **) ptr); + fix[ix].list_size--; + fix[ix].allocated--; + } + } + + blk = UMEM2BLK(ptr); + + if (IS_SBC_BLK(blk)) + destroy_carrier(allctr, blk); + else + mbc_free(allctr, ptr); + ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); + } + + if (need_thr_progress && !(need_thr_prgr | need_mr_wrk)) { + need_thr_prgr = ddq_check_incoming(ddq); + *need_thr_progress |= need_thr_prgr; + } + + if (allctr->thread_safe && !allctr_locked) + erts_mtx_unlock(&allctr->mutex); + return res; +} + +static ERTS_INLINE void +enqueue_dealloc_other_instance(ErtsAlcType_t type, Allctr_t *allctr, void *ptr) +{ + if (allctr->fix) + ((UWord *) ptr)[ERTS_ALCU_DD_FIX_TYPE_OFFS] = (UWord) type; + + if (ddq_enqueue(type, &allctr->dd.q, ptr)) + erts_alloc_notify_delayed_dealloc(allctr->ix); +} + +#endif + +void +erts_alcu_check_delayed_dealloc(Allctr_t *allctr, + int limit, + int *need_thr_progress, + int *more_work) +{ +#ifdef ERTS_SMP + handle_delayed_dealloc(allctr, + 0, + limit, + ERTS_ALCU_DD_OPS_LIM_HIGH, + need_thr_progress, + more_work); +#endif +} + +#define ERTS_ALCU_HANDLE_DD_IN_OP(Allctr, Locked) \ + handle_delayed_dealloc((Allctr), (Locked), 1, \ + ERTS_ALCU_DD_OPS_LIM_LOW, NULL, NULL) + /* Multi block carrier alloc/realloc/free ... */ /* NOTE! mbc_alloc() may in case of memory shortage place the requested @@ -680,8 +1135,21 @@ mbc_alloc_block(Allctr_t *allctr, Uint size, Uint *blk_szp, Uint32 *alcu_flgsp) } } +#ifdef ERTS_SMP + if (allctr->dd.use) + ERTS_ALCU_HANDLE_DD_IN_OP(allctr, 1); +#endif + blk = (*allctr->get_free_block)(allctr, get_blk_sz, NULL, 0, *alcu_flgsp); +#ifdef ERTS_SMP + if (!blk && allctr->dd.use) { + if (ERTS_ALCU_HANDLE_DD_IN_OP(allctr, 1)) + blk = (*allctr->get_free_block)(allctr, get_blk_sz, NULL, 0, + *alcu_flgsp); + } +#endif + if (!blk) { if ((*alcu_flgsp) & ERTS_ALCU_FLG_SBMBC) blk = create_sbmbc(allctr, get_blk_sz); @@ -939,6 +1407,11 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) Uint is_last_blk; #endif /* #ifndef MBC_REALLOC_ALWAYS_MOVES */ +#ifdef ERTS_SMP + if (allctr->dd.use) + ERTS_ALCU_HANDLE_DD_IN_OP(allctr, 1); +#endif + ASSERT(p); ASSERT(size); ASSERT(size < allctr->sbc_threshold); @@ -1005,7 +1478,6 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) cand_blk, cand_blk_sz, alcu_flgs); - if (new_blk || cand_blk != blk) goto move_into_new_blk; } @@ -1441,7 +1913,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) goto try_sys_alloc; if (flags & CFLG_FORCE_MSEG) goto try_mseg; - if (erts_mseg_no() >= max_mseg_carriers) + if (erts_mseg_no(&allctr->mseg_opt) >= max_mseg_carriers) goto try_sys_alloc; if (flags & CFLG_SBC) { if (allctr->sbcs.curr.norm.mseg.no >= allctr->max_mseg_sbcs) @@ -1840,8 +2312,12 @@ static struct { Eterm ycs; /* Eterm sbmbcs; */ + + Eterm fix_types; + Eterm mbcs; Eterm sbcs; + Eterm sys_alloc_carriers_size; #if HAVE_ERTS_MSEG Eterm mseg_alloc_carriers_size; @@ -1871,6 +2347,8 @@ static struct { #endif } am; +static Eterm fix_type_atoms[ERTS_ALC_NO_FIXED_SIZES]; + static ERTS_INLINE void atom_init(Eterm *atom, char *name) { *atom = am_atom_put(name, strlen(name)); @@ -1891,6 +2369,7 @@ init_atoms(Allctr_t *allctr) erts_mtx_lock(&init_atoms_mtx); if (!atoms_initialized) { + int ix; #ifdef DEBUG Eterm *atom; @@ -1933,8 +2412,12 @@ init_atoms(Allctr_t *allctr) AM_INIT(ycs); /*AM_INIT(sbmbcs);*/ + + AM_INIT(fix_types); + AM_INIT(mbcs); AM_INIT(sbcs); + AM_INIT(sys_alloc_carriers_size); #if HAVE_ERTS_MSEG AM_INIT(mseg_alloc_carriers_size); @@ -1965,6 +2448,13 @@ init_atoms(Allctr_t *allctr) ASSERT(*atom != THE_NON_VALUE); } #endif + + for (ix = 0; ix < ERTS_ALC_NO_FIXED_SIZES; ix++) { + ErtsAlcType_t n = ERTS_ALC_N_MIN_A_FIXED_SIZE + ix; + char *name = (char *) ERTS_ALC_N2TD(n); + size_t len = strlen(name); + fix_type_atoms[ix] = am_atom_put(name, len); + } } @@ -2043,6 +2533,48 @@ add_4tup(Uint **hpp, Uint *szp, Eterm *lp, } static Eterm +sz_info_fix(Allctr_t *allctr, + int *print_to_p, + void *print_to_arg, + Uint **hpp, + Uint *szp) +{ + Eterm res; + int ix; + ErtsAlcFixList_t *fix = allctr->fix; + + ASSERT(fix); + + res = NIL; + + for (ix = ERTS_ALC_NO_FIXED_SIZES-1; ix >= 0; ix--) { + ErtsAlcType_t n = ix + ERTS_ALC_N_MIN_A_FIXED_SIZE; + Uint alloced = (fix[ix].type_size * fix[ix].allocated); + Uint used = fix[ix].type_size*fix[ix].used; + + if (print_to_p) { + int to = *print_to_p; + void *arg = print_to_arg; + erts_print(to, + arg, + "fix type: %s %bpu %bpu\n", + (char *) ERTS_ALC_N2TD(n), + alloced, + used); + } + + if (hpp || szp) { + add_3tup(hpp, szp, &res, + fix_type_atoms[ix], + bld_unstable_uint(hpp, szp, alloced), + bld_unstable_uint(hpp, szp, used)); + } + } + + return res; +} + +static Eterm sz_info_carriers(Allctr_t *allctr, CarriersStats_t *cs, char *prefix, @@ -2590,7 +3122,7 @@ erts_alcu_sz_info(Allctr_t *allctr, Uint **hpp, Uint *szp) { - Eterm res, sbmbcs, mbcs, sbcs; + Eterm res, sbmbcs, mbcs, sbcs, fix = THE_NON_VALUE; res = THE_NON_VALUE; @@ -2607,6 +3139,8 @@ erts_alcu_sz_info(Allctr_t *allctr, erts_mtx_lock(&allctr->mutex); #endif + ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr); + if (hpp || szp) ensure_atoms_initialized(allctr); @@ -2619,6 +3153,8 @@ erts_alcu_sz_info(Allctr_t *allctr, update_max_ever_values(&allctr->mbcs); update_max_ever_values(&allctr->sbcs); + if (allctr->fix) + fix = sz_info_fix(allctr, print_to_p, print_to_arg, hpp, szp); sbmbcs = sz_info_carriers(allctr, &allctr->sbmbcs, "sbmbcs ", print_to_p, print_to_arg, hpp, szp); mbcs = sz_info_carriers(allctr, &allctr->mbcs, "mbcs ", print_to_p, @@ -2631,6 +3167,8 @@ erts_alcu_sz_info(Allctr_t *allctr, add_2tup(hpp, szp, &res, am.sbcs, sbcs); add_2tup(hpp, szp, &res, am.mbcs, mbcs); add_2tup(hpp, szp, &res, am.sbmbcs, sbmbcs); + if (allctr->fix) + add_2tup(hpp, szp, &res, am.fix_types, fix); } if (begin_max_period) { @@ -2656,7 +3194,7 @@ erts_alcu_info(Allctr_t *allctr, Uint **hpp, Uint *szp) { - Eterm res, sett, sbmbcs, mbcs, sbcs, calls; + Eterm res, sett, sbmbcs, mbcs, sbcs, calls, fix = THE_NON_VALUE; res = THE_NON_VALUE; @@ -2673,6 +3211,8 @@ erts_alcu_info(Allctr_t *allctr, erts_mtx_lock(&allctr->mutex); #endif + ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr); + if (hpp || szp) ensure_atoms_initialized(allctr); @@ -2694,6 +3234,8 @@ erts_alcu_info(Allctr_t *allctr, } sett = info_options(allctr, print_to_p, print_to_arg, hpp, szp); + if (allctr->fix) + fix = sz_info_fix(allctr, print_to_p, print_to_arg, hpp, szp); sbmbcs = info_carriers(allctr, &allctr->sbmbcs, "sbmbcs ", print_to_p, print_to_arg, hpp, szp); mbcs = info_carriers(allctr, &allctr->mbcs, "mbcs ", print_to_p, @@ -2709,6 +3251,8 @@ erts_alcu_info(Allctr_t *allctr, add_2tup(hpp, szp, &res, am.sbcs, sbcs); add_2tup(hpp, szp, &res, am.mbcs, mbcs); add_2tup(hpp, szp, &res, am.sbmbcs, sbmbcs); + if (allctr->fix) + add_2tup(hpp, szp, &res, am.fix_types, fix); add_2tup(hpp, szp, &res, am.options, sett); add_3tup(hpp, szp, &res, am.versions, @@ -2733,7 +3277,7 @@ erts_alcu_info(Allctr_t *allctr, void -erts_alcu_current_size(Allctr_t *allctr, AllctrSize_t *size) +erts_alcu_current_size(Allctr_t *allctr, AllctrSize_t *size, ErtsAlcUFixInfo_t *fi, int fisz) { #ifdef USE_THREADS @@ -2751,6 +3295,18 @@ erts_alcu_current_size(Allctr_t *allctr, AllctrSize_t *size) size->blocks += allctr->sbmbcs.blocks.curr.size; size->blocks += allctr->sbcs.blocks.curr.size; + if (fi) { + int ix; + for (ix = 0; ix < fisz; ix++) { + if (allctr->fix) { + fi[ix].allocated += (allctr->fix[ix].type_size + * allctr->fix[ix].allocated); + fi[ix].used += (allctr->fix[ix].type_size + * allctr->fix[ix].used); + } + } + } + #ifdef USE_THREADS if (allctr->thread_safe) erts_mtx_unlock(&allctr->mutex); @@ -2764,12 +3320,16 @@ do_erts_alcu_alloc(ErtsAlcType_t type, void *extra, Uint size) { Allctr_t *allctr = (Allctr_t *) extra; void *res; + ErtsAlcFixList_t *fix; ASSERT(initialized); ASSERT(allctr); - ERTS_ALCU_DBG_CHK_THR_SPEC(allctr); + ERTS_SMP_LC_ASSERT(!allctr->thread_safe + || erts_lc_mtx_is_locked(&allctr->mutex)); + + ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr); #if ALLOC_ZERO_EQ_NULL if (!size) @@ -2778,18 +3338,61 @@ do_erts_alcu_alloc(ErtsAlcType_t type, void *extra, Uint size) INC_CC(allctr->calls.this_alloc); + fix = allctr->fix; + if (fix) { + int ix = type - ERTS_ALC_N_MIN_A_FIXED_SIZE; + ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1); + fix[ix].used++; + res = fix[ix].list; + if (res) { + fix[ix].list_size--; + fix[ix].list = *((void **) res); + if (fix[ix].list && fix[ix].allocated > fix[ix].limit) { + void *p = fix[ix].list; + Block_t *blk; + fix[ix].list = *((void **) p); + fix[ix].list_size--; + blk = UMEM2BLK(p); + if (IS_SBC_BLK(blk)) + destroy_carrier(allctr, blk); + else + mbc_free(allctr, p); + fix[ix].allocated--; + } + ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); + return res; + } + if (size < 2*sizeof(UWord)) + size += sizeof(UWord); + if (fix[ix].limit < fix[ix].used) + fix[ix].limit = fix[ix].used; + if (fix[ix].max_used < fix[ix].used) + fix[ix].max_used = fix[ix].used; + fix[ix].allocated++; + } + if (size >= allctr->sbc_threshold) { + Block_t *blk; +#ifdef ERTS_SMP + if (allctr->dd.use) + ERTS_ALCU_HANDLE_DD_IN_OP(allctr, 1); +#endif #if HALFWORD_HEAP - Block_t *blk = create_carrier(allctr, size, - CFLG_SBC | CFLG_FORCE_MSEG); + blk = create_carrier(allctr, size, + CFLG_SBC | CFLG_FORCE_MSEG); #else - Block_t *blk = create_carrier(allctr, size, CFLG_SBC); + blk = create_carrier(allctr, size, CFLG_SBC); #endif res = blk ? BLK2UMEM(blk) : NULL; } else res = mbc_alloc(allctr, size); + if (!res && fix) { + int ix = type - ERTS_ALC_N_MIN_A_FIXED_SIZE; + fix[ix].allocated--; + fix[ix].used--; + } return res; } @@ -2818,29 +3421,28 @@ erts_alcu_alloc_ts(ErtsAlcType_t type, void *extra, Uint size) return res; } +#ifdef ERTS_SMP + void * erts_alcu_alloc_thr_spec(ErtsAlcType_t type, void *extra, Uint size) { ErtsAllocatorThrSpec_t *tspec = (ErtsAllocatorThrSpec_t *) extra; - int ix = erts_alc_get_thr_ix(); + int ix; Allctr_t *allctr; - int unlock; void *res; - ASSERT(ix > 0); - if (ix < tspec->size) { - allctr = tspec->allctr[ix]; - unlock = 0; - } - else { - allctr = tspec->allctr[0]; - unlock = 1; + ix = ERTS_ALC_GET_THR_IX(); + + ASSERT(0 <= ix && ix < tspec->size); + + allctr = tspec->allctr[ix]; + + if (allctr->thread_safe) erts_mtx_lock(&allctr->mutex); - } res = do_erts_alcu_alloc(type, allctr, size); - if (unlock) + if (allctr->thread_safe) erts_mtx_unlock(&allctr->mutex); DEBUG_CHECK_ALIGNMENT(res); @@ -2851,51 +3453,96 @@ erts_alcu_alloc_thr_spec(ErtsAlcType_t type, void *extra, Uint size) void * erts_alcu_alloc_thr_pref(ErtsAlcType_t type, void *extra, Uint size) { - ErtsAllocatorThrSpec_t *tspec = (ErtsAllocatorThrSpec_t *) extra; - int ix = erts_alc_get_thr_ix(); - Allctr_t *allctr; + int pref_ix; + Allctr_t *pref_allctr; void *res; - ASSERT(sizeof(UWord) == sizeof(Allctr_t *)); - ASSERT(ix > 0); - if (ix >= tspec->size) - ix = (ix % (tspec->size - 1)) + 1; - allctr = tspec->allctr[ix]; - erts_mtx_lock(&allctr->mutex); - res = do_erts_alcu_alloc(type, allctr, size + sizeof(UWord)); - if (res) { - *((Allctr_t **) res) = allctr; - res = (void *) (((char *) res) + sizeof(UWord)); - } - erts_mtx_unlock(&allctr->mutex); + pref_ix = get_pref_allctr(extra, &pref_allctr); + + if (pref_allctr->thread_safe) + erts_mtx_lock(&pref_allctr->mutex); + + ERTS_ALCU_DBG_CHK_THR_ACCESS(pref_allctr); + + res = do_erts_alcu_alloc(type, pref_allctr, size + sizeof(UWord)); + if (pref_allctr->thread_safe) + erts_mtx_unlock(&pref_allctr->mutex); + + if (res) + res = put_used_allctr(res, pref_ix, size); + DEBUG_CHECK_ALIGNMENT(res); + + return res; } #endif +#endif + /* ------------------------------------------------------------------------- */ static ERTS_INLINE void do_erts_alcu_free(ErtsAlcType_t type, void *extra, void *p) { + int ix; Allctr_t *allctr = (Allctr_t *) extra; ASSERT(initialized); ASSERT(allctr); - ERTS_ALCU_DBG_CHK_THR_SPEC(allctr); + ERTS_SMP_LC_ASSERT(!allctr->thread_safe + || erts_lc_mtx_is_locked(&allctr->mutex)); + + ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr); if (p) { + ErtsAlcFixList_t *fix = allctr->fix; Block_t *blk; INC_CC(allctr->calls.this_free); + if (fix) { + ix = type - ERTS_ALC_N_MIN_A_FIXED_SIZE; + ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1); + fix[ix].used--; + if (fix[ix].allocated < fix[ix].limit + && fix[ix].list_size < ERTS_ALCU_FIX_MAX_LIST_SZ) { + *((void **) p) = fix[ix].list; + fix[ix].list = p; + fix[ix].list_size++; + if (!allctr->fix_shrink_scheduled) { + allctr->fix_shrink_scheduled = 1; + erts_set_aux_work_timeout( + allctr->ix, + (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM + | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC), + 1); + } + ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); + return; + } + fix[ix].allocated--; + if (fix[ix].list && fix[ix].allocated > fix[ix].limit) { + blk = UMEM2BLK(p); + if (IS_SBC_BLK(blk)) + destroy_carrier(allctr, blk); + else + mbc_free(allctr, p); + p = fix[ix].list; + fix[ix].list = *((void **) p); + fix[ix].list_size--; + fix[ix].allocated--; + } + } + blk = UMEM2BLK(p); if (IS_SBC_BLK(blk)) destroy_carrier(allctr, blk); else mbc_free(allctr, p); + ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); } } @@ -2915,44 +3562,56 @@ erts_alcu_free_ts(ErtsAlcType_t type, void *extra, void *p) erts_mtx_unlock(&allctr->mutex); } +#ifdef ERTS_SMP + void erts_alcu_free_thr_spec(ErtsAlcType_t type, void *extra, void *p) { ErtsAllocatorThrSpec_t *tspec = (ErtsAllocatorThrSpec_t *) extra; - int ix = erts_alc_get_thr_ix(); - int unlock; + int ix; Allctr_t *allctr; - ASSERT(ix > 0); - if (ix < tspec->size) { - allctr = tspec->allctr[ix]; - unlock = 0; - } - else { - allctr = tspec->allctr[0]; - unlock = 1; + ix = ERTS_ALC_GET_THR_IX(); + + ASSERT(0 <= ix && ix < tspec->size); + + allctr = tspec->allctr[ix]; + + if (allctr->thread_safe) erts_mtx_lock(&allctr->mutex); - } do_erts_alcu_free(type, allctr, p); - if (unlock) + + if (allctr->thread_safe) erts_mtx_unlock(&allctr->mutex); } void -erts_alcu_free_thr_pref(ErtsAlcType_t type, void *unused, void *p) +erts_alcu_free_thr_pref(ErtsAlcType_t type, void *extra, void *p) { if (p) { - void *ptr = (void *) (((char *) p) - sizeof(UWord)); - Allctr_t *allctr = *((Allctr_t **) ptr); - erts_mtx_lock(&allctr->mutex); - do_erts_alcu_free(type, allctr, ptr); - erts_mtx_unlock(&allctr->mutex); + Allctr_t *pref_allctr, *used_allctr; + void *ptr; + + get_pref_allctr(extra, &pref_allctr); + ptr = get_used_allctr(extra, p, &used_allctr, NULL); + if (pref_allctr != used_allctr) + enqueue_dealloc_other_instance(type, used_allctr, ptr); + else { + if (used_allctr->thread_safe) + erts_mtx_lock(&used_allctr->mutex); + ERTS_ALCU_DBG_CHK_THR_ACCESS(used_allctr); + do_erts_alcu_free(type, used_allctr, ptr); + if (used_allctr->thread_safe) + erts_mtx_unlock(&used_allctr->mutex); + } } } #endif +#endif + /* ------------------------------------------------------------------------- */ static ERTS_INLINE void * @@ -2970,7 +3629,10 @@ do_erts_alcu_realloc(ErtsAlcType_t type, ASSERT(allctr); - ERTS_ALCU_DBG_CHK_THR_SPEC(allctr); + ERTS_SMP_LC_ASSERT(!allctr->thread_safe + || erts_lc_mtx_is_locked(&allctr->mutex)); + + ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr); if (!p) { res = do_erts_alcu_alloc(type, extra, size); @@ -3063,6 +3725,10 @@ do_erts_alcu_realloc(ErtsAlcType_t type, } else { Block_t *new_blk; +#ifdef ERTS_SMP + if (allctr->dd.use) + ERTS_ALCU_HANDLE_DD_IN_OP(allctr, 1); +#endif if(IS_SBC_BLK(blk)) { do_carrier_resize: #if HALFWORD_HEAP @@ -3166,30 +3832,29 @@ erts_alcu_realloc_mv_ts(ErtsAlcType_t type, void *extra, void *p, Uint size) return res; } +#ifdef ERTS_SMP + void * erts_alcu_realloc_thr_spec(ErtsAlcType_t type, void *extra, void *ptr, Uint size) { ErtsAllocatorThrSpec_t *tspec = (ErtsAllocatorThrSpec_t *) extra; - int ix = erts_alc_get_thr_ix(); + int ix; Allctr_t *allctr; - int unlock; void *res; - ASSERT(ix > 0); - if (ix < tspec->size) { - allctr = tspec->allctr[ix]; - unlock = 0; - } - else { - allctr = tspec->allctr[0]; - unlock = 1; + ix = ERTS_ALC_GET_THR_IX(); + + ASSERT(0 <= ix && ix < tspec->size); + + allctr = tspec->allctr[ix]; + + if (allctr->thread_safe) erts_mtx_lock(&allctr->mutex); - } res = do_erts_alcu_realloc(type, allctr, ptr, size, 0); - if (unlock) + if (allctr->thread_safe) erts_mtx_unlock(&allctr->mutex); DEBUG_CHECK_ALIGNMENT(res); @@ -3202,26 +3867,22 @@ erts_alcu_realloc_mv_thr_spec(ErtsAlcType_t type, void *extra, void *ptr, Uint size) { ErtsAllocatorThrSpec_t *tspec = (ErtsAllocatorThrSpec_t *) extra; - int ix = erts_alc_get_thr_ix(); + int ix; Allctr_t *allctr; - int unlock; void *res; - ASSERT(ix > 0); - if (ix < tspec->size) { - allctr = tspec->allctr[ix]; - unlock = 0; - } - else { - allctr = tspec->allctr[0]; - unlock = 1; - erts_mtx_lock(&allctr->mutex); - } + ix = ERTS_ALC_GET_THR_IX(); + ASSERT(0 <= ix && ix < tspec->size); + + allctr = tspec->allctr[ix]; + + if (allctr->thread_safe) + erts_mtx_lock(&allctr->mutex); res = do_erts_alcu_alloc(type, allctr, size); if (!res) { - if (unlock) + if (allctr->thread_safe) erts_mtx_unlock(&allctr->mutex); res = erts_alcu_realloc_thr_spec(type, allctr, ptr, size); } @@ -3235,7 +3896,7 @@ erts_alcu_realloc_mv_thr_spec(ErtsAlcType_t type, void *extra, cpy_size = size; sys_memcpy(res, ptr, cpy_size); do_erts_alcu_free(type, allctr, ptr); - if (unlock) + if (allctr->thread_safe) erts_mtx_unlock(&allctr->mutex); } @@ -3244,129 +3905,101 @@ erts_alcu_realloc_mv_thr_spec(ErtsAlcType_t type, void *extra, return res; } -void * -erts_alcu_realloc_thr_pref(ErtsAlcType_t type, void *extra, void *p, Uint size) +static ERTS_INLINE void * +realloc_thr_pref(ErtsAlcType_t type, void *extra, void *p, Uint size, + int force_move) { - ErtsAllocatorThrSpec_t *tspec = (ErtsAllocatorThrSpec_t *) extra; - int ix; + int pref_ix; void *ptr, *res; Allctr_t *pref_allctr, *used_allctr; + UWord old_user_size; if (!p) return erts_alcu_alloc_thr_pref(type, extra, size); - ptr = (void *) (((char *) p) - sizeof(UWord)); - used_allctr = *((Allctr_t **) ptr); + pref_ix = get_pref_allctr(extra, &pref_allctr); + ptr = get_used_allctr(extra, p, &used_allctr, &old_user_size); - ix = erts_alc_get_thr_ix(); - ASSERT(ix > 0); - if (ix >= tspec->size) - ix = (ix % (tspec->size - 1)) + 1; - pref_allctr = tspec->allctr[ix]; ASSERT(used_allctr && pref_allctr); - erts_mtx_lock(&used_allctr->mutex); - res = do_erts_alcu_realloc(type, - used_allctr, - ptr, - size + sizeof(UWord), - (pref_allctr != used_allctr - ? ERTS_ALCU_FLG_FAIL_REALLOC_MOVE - : 0)); - erts_mtx_unlock(&used_allctr->mutex); - if (res) { - ASSERT(used_allctr == *((Allctr_t **) res)); - res = (void *) (((char *) res) + sizeof(UWord)); - DEBUG_CHECK_ALIGNMENT(res); + if (!force_move && used_allctr == pref_allctr) { + if (used_allctr->thread_safe) + erts_mtx_lock(&used_allctr->mutex); + ERTS_ALCU_DBG_CHK_THR_ACCESS(used_allctr); + res = do_erts_alcu_realloc(type, + used_allctr, + ptr, + size + sizeof(UWord), + 0); + if (used_allctr->thread_safe) + erts_mtx_unlock(&used_allctr->mutex); + if (res) + res = put_used_allctr(res, pref_ix, size); } else { - erts_mtx_lock(&pref_allctr->mutex); + if (pref_allctr->thread_safe) + erts_mtx_lock(&pref_allctr->mutex); res = do_erts_alcu_alloc(type, pref_allctr, size + sizeof(UWord)); - erts_mtx_unlock(&pref_allctr->mutex); + if (pref_allctr->thread_safe && (!force_move + || used_allctr != pref_allctr)) + erts_mtx_unlock(&pref_allctr->mutex); if (res) { Block_t *blk; size_t cpy_size; - *((Allctr_t **) res) = pref_allctr; - res = (void *) (((char *) res) + sizeof(UWord)); + res = put_used_allctr(res, pref_ix, size); DEBUG_CHECK_ALIGNMENT(res); - erts_mtx_lock(&used_allctr->mutex); blk = UMEM2BLK(ptr); - cpy_size = BLK_SZ(blk) - ABLK_HDR_SZ - sizeof(UWord); + if (old_user_size != ERTS_AU_PREF_ALLOC_SIZE_MASK) + cpy_size = old_user_size; + else { + if (used_allctr->thread_safe && (!force_move + || used_allctr != pref_allctr)) + erts_mtx_lock(&used_allctr->mutex); + ERTS_SMP_LC_ASSERT(erts_lc_mtx_is_locked(&used_allctr->mutex)); + cpy_size = BLK_SZ(blk); + if (used_allctr->thread_safe && (!force_move + || used_allctr != pref_allctr)) + erts_mtx_unlock(&used_allctr->mutex); + cpy_size -= ABLK_HDR_SZ + sizeof(UWord); + } if (cpy_size > size) cpy_size = size; sys_memcpy(res, p, cpy_size); - do_erts_alcu_free(type, used_allctr, ptr); - erts_mtx_unlock(&used_allctr->mutex); + + if (!force_move || used_allctr != pref_allctr) + enqueue_dealloc_other_instance(type, used_allctr, ptr); + else { + do_erts_alcu_free(type, used_allctr, ptr); + ASSERT(pref_allctr == used_allctr); + if (pref_allctr->thread_safe) + erts_mtx_unlock(&pref_allctr->mutex); + } } } return res; } +void * +erts_alcu_realloc_thr_pref(ErtsAlcType_t type, void *extra, void *p, Uint size) +{ + return realloc_thr_pref(type, extra, p, size, 0); +} void * erts_alcu_realloc_mv_thr_pref(ErtsAlcType_t type, void *extra, void *p, Uint size) { - ErtsAllocatorThrSpec_t *tspec = (ErtsAllocatorThrSpec_t *) extra; - int ix; - void *ptr, *res; - Allctr_t *pref_allctr, *used_allctr; - - if (!p) - return erts_alcu_alloc_thr_pref(type, extra, size); - - ptr = (void *) (((char *) p) - sizeof(UWord)); - used_allctr = *((Allctr_t **) ptr); - - ix = erts_alc_get_thr_ix(); - ASSERT(ix > 0); - if (ix >= tspec->size) - ix = (ix % (tspec->size - 1)) + 1; - pref_allctr = tspec->allctr[ix]; - ASSERT(used_allctr && pref_allctr); - - erts_mtx_lock(&pref_allctr->mutex); - res = do_erts_alcu_alloc(type, pref_allctr, size + sizeof(UWord)); - if (!res) { - erts_mtx_unlock(&pref_allctr->mutex); - res = erts_alcu_realloc_thr_pref(type, extra, p, size); - } - else { - Block_t *blk; - size_t cpy_size; - Allctr_t *allctr; - - *((Allctr_t **) res) = pref_allctr; - res = (void *) (((char *) res) + sizeof(UWord)); - - DEBUG_CHECK_ALIGNMENT(res); - - if (used_allctr == pref_allctr) - allctr = pref_allctr; - else { - erts_mtx_unlock(&pref_allctr->mutex); - allctr = used_allctr; - erts_mtx_lock(&allctr->mutex); - } - - blk = UMEM2BLK(ptr); - cpy_size = BLK_SZ(blk) - ABLK_HDR_SZ - sizeof(UWord); - if (cpy_size > size) - cpy_size = size; - sys_memcpy(res, p, cpy_size); - do_erts_alcu_free(type, allctr, ptr); - erts_mtx_unlock(&allctr->mutex); - } - - return res; + return realloc_thr_pref(type, extra, p, size, 1); } #endif +#endif + /* ------------------------------------------------------------------------- */ int @@ -3381,6 +4014,10 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) sys_memcpy((void *) &allctr->mseg_opt, (void *) &erts_mseg_default_opt, sizeof(ErtsMsegOpt_t)); +#ifdef ERTS_SMP + if (init->tspec || init->tpref) + allctr->mseg_opt.sched_spec = 1; +#endif # if HALFWORD_HEAP allctr->mseg_opt.low_mem = init->low_mem; # endif @@ -3390,6 +4027,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) if (!allctr->name_prefix) goto error; + allctr->ix = init->ix; allctr->alloc_no = init->alloc_no; if (allctr->alloc_no < ERTS_ALC_A_MIN || ERTS_ALC_A_MAX < allctr->alloc_no) @@ -3431,6 +4069,18 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) goto error; allctr->min_block_size = UNIT_CEILING(allctr->min_block_size + sizeof(UWord)); +#if ERTS_SMP + if (init->tpref) { + Uint sz = sizeof(Block_t); + sz += ERTS_ALCU_DD_FIX_TYPE_OFFS*sizeof(UWord); + if (init->fix) + sz += sizeof(UWord); + sz = UNIT_CEILING(sz); + if (sz > allctr->min_block_size) + allctr->min_block_size = sz; + } +#endif + allctr->sbmbc_threshold = init->sbmbct; @@ -3493,7 +4143,8 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) if (allctr->mbc_header_size < sizeof(Carrier_t)) goto error; -#ifdef USE_THREADS +#ifdef ERTS_SMP + allctr->dd.use = 0; if (init->tpref) { allctr->mbc_header_size = (UNIT_CEILING(allctr->mbc_header_size + FBLK_FTR_SZ @@ -3507,6 +4158,9 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) + sizeof(UWord)) - ABLK_HDR_SZ - sizeof(UWord)); + + allctr->dd.use = 1; + init_dd_queue(&allctr->dd.q); } else #endif @@ -3548,6 +4202,21 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) } + if (init->fix) { + int i; + allctr->fix = init->fix; + allctr->fix_shrink_scheduled = 0; + for (i = 0; i < ERTS_ALC_NO_FIXED_SIZES; i++) { + allctr->fix[i].max_used = 0; + allctr->fix[i].limit = 0; + allctr->fix[i].type_size = init->fix_type_size[i]; + allctr->fix[i].list_size = 0; + allctr->fix[i].list = NULL; + allctr->fix[i].allocated = 0; + allctr->fix[i].used = 0; + } + } + return 1; error: diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index fed4d3dbe6..df560a0de2 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -20,10 +20,13 @@ #ifndef ERL_ALLOC_UTIL__ #define ERL_ALLOC_UTIL__ -#define ERTS_ALCU_VSN_STR "2.2" +#define ERTS_ALCU_VSN_STR "3.0" #include "erl_alloc_types.h" +#define ERTS_AU_PREF_ALLOC_BITS 11 +#define ERTS_AU_MAX_PREF_ALLOC_INSTANCES (1 << ERTS_AU_PREF_ALLOC_BITS) + typedef struct Allctr_t_ Allctr_t; typedef struct { @@ -35,6 +38,7 @@ typedef struct { char *name_prefix; ErtsAlcType_t alloc_no; int force; + int ix; int ts; int tspec; int tpref; @@ -53,6 +57,9 @@ typedef struct { UWord mbcgs; UWord sbmbct; UWord sbmbcs; + + void *fix; + size_t *fix_type_size; } AllctrInit_t; typedef struct { @@ -60,6 +67,11 @@ typedef struct { UWord carriers; } AllctrSize_t; +typedef struct { + UWord allocated; + UWord used; +} ErtsAlcUFixInfo_t; + #ifndef SMALL_MEMORY #define ERTS_DEFAULT_ALCU_INIT { \ @@ -71,6 +83,7 @@ typedef struct { NULL, \ ERTS_ALC_A_INVALID, /* (number) alloc_no: allocator number */\ 0, /* (bool) force: force enabled */\ + 0, /* (number) ix: instance index */\ 1, /* (bool) ts: thread safe */\ 0, /* (bool) tspec: thread specific */\ 0, /* (bool) tpref: thread preferred */\ @@ -88,7 +101,10 @@ typedef struct { 1024*1024, /* (bytes) smbcs: smallest mbc size */\ 10, /* (amount) mbcgs: mbc growth stages */\ 256, /* (bytes) sbmbct: small block mbc threshold */\ - 8*1024 /* (bytes) sbmbcs: small block mbc size */\ + 8*1024, /* (bytes) sbmbcs: small block mbc size */ \ + /* --- Data not options -------------------------------------------- */\ + NULL, /* (ptr) fix */\ + NULL /* (ptr) fix_type_size */\ } #else /* if SMALL_MEMORY */ @@ -102,6 +118,7 @@ typedef struct { NULL, \ ERTS_ALC_A_INVALID, /* (number) alloc_no: allocator number */\ 0, /* (bool) force: force enabled */\ + 0, /* (number) ix: instance index */\ 1, /* (bool) ts: thread safe */\ 0, /* (bool) tspec: thread specific */\ 0, /* (bool) tpref: thread preferred */\ @@ -118,7 +135,10 @@ typedef struct { 128*1024, /* (bytes) smbcs: smallest mbc size */\ 10, /* (amount) mbcgs: mbc growth stages */\ 256, /* (bytes) sbmbct: small block mbc threshold */\ - 8*1024 /* (bytes) sbmbcs: small block mbc size */\ + 8*1024, /* (bytes) sbmbcs: small block mbc size */ \ + /* --- Data not options -------------------------------------------- */\ + NULL, /* (ptr) fix */\ + NULL /* (ptr) fix_type_size */\ } #endif @@ -132,6 +152,7 @@ void * erts_alcu_alloc_ts(ErtsAlcType_t, void *, Uint); void * erts_alcu_realloc_ts(ErtsAlcType_t, void *, void *, Uint); void * erts_alcu_realloc_mv_ts(ErtsAlcType_t, void *, void *, Uint); void erts_alcu_free_ts(ErtsAlcType_t, void *, void *); +#ifdef ERTS_SMP void * erts_alcu_alloc_thr_spec(ErtsAlcType_t, void *, Uint); void * erts_alcu_realloc_thr_spec(ErtsAlcType_t, void *, void *, Uint); void * erts_alcu_realloc_mv_thr_spec(ErtsAlcType_t, void *, void *, Uint); @@ -141,12 +162,16 @@ void * erts_alcu_realloc_thr_pref(ErtsAlcType_t, void *, void *, Uint); void * erts_alcu_realloc_mv_thr_pref(ErtsAlcType_t, void *, void *, Uint); void erts_alcu_free_thr_pref(ErtsAlcType_t, void *, void *); #endif +#endif Eterm erts_alcu_au_info_options(int *, void *, Uint **, Uint *); Eterm erts_alcu_info_options(Allctr_t *, int *, void *, Uint **, Uint *); Eterm erts_alcu_sz_info(Allctr_t *, int, int *, void *, Uint **, Uint *); Eterm erts_alcu_info(Allctr_t *, int, int *, void *, Uint **, Uint *); void erts_alcu_init(AlcUInit_t *); -void erts_alcu_current_size(Allctr_t *, AllctrSize_t *); +void erts_alcu_current_size(Allctr_t *, AllctrSize_t *, + ErtsAlcUFixInfo_t *, int); +void erts_alcu_check_delayed_dealloc(Allctr_t *, int, int *, int *); +erts_aint32_t erts_alcu_fix_alloc_shrink(Allctr_t *, erts_aint32_t); #endif @@ -246,7 +271,74 @@ typedef struct { } blocks; } CarriersStats_t; +#ifdef ERTS_SMP + +typedef union ErtsAllctrDDBlock_t_ ErtsAllctrDDBlock_t; + +union ErtsAllctrDDBlock_t_ { + erts_atomic_t atmc_next; + ErtsAllctrDDBlock_t *ptr_next; +}; + +typedef struct { + ErtsAllctrDDBlock_t marker; + erts_atomic_t last; + erts_atomic_t um_refc[2]; + erts_atomic32_t um_refc_ix; +} ErtsDDTail_t; + +typedef struct { + /* + * This structure needs to be cache line aligned for best + * performance. + */ + union { + /* Modified by threads returning memory to this allocator */ + ErtsDDTail_t data; + char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsDDTail_t))]; + } tail; + /* + * Everything below this point is *only* accessed by the + * thread owning the allocator. + */ + struct { + ErtsAllctrDDBlock_t *first; + ErtsAllctrDDBlock_t *unref_end; + struct { + ErtsThrPrgrVal thr_progress; + int thr_progress_reached; + int um_refc_ix; + ErtsAllctrDDBlock_t *unref_end; + } next; + int used_marker; + } head; +} ErtsAllctrDDQueue_t; + +#endif + +typedef struct { + size_t type_size; + SWord list_size; + void *list; + SWord max_used; + SWord limit; + SWord allocated; + SWord used; +} ErtsAlcFixList_t; + struct Allctr_t_ { +#ifdef ERTS_SMP + struct { + /* + * We want the queue at the beginning of + * the Allctr_t struct, due to cache line + * alignment reasons. + */ + ErtsAllctrDDQueue_t q; + int use; + int ix; + } dd; +#endif /* Allocator name prefix */ char * name_prefix; @@ -254,6 +346,9 @@ struct Allctr_t_ { /* Allocator number */ ErtsAlcType_t alloc_no; + /* Instance index */ + int ix; + /* Alloc, realloc and free names as atoms */ struct { Eterm alloc; @@ -278,6 +373,7 @@ struct Allctr_t_ { Uint mbc_growth_stages; Uint sbmbc_threshold; Uint sbmbc_size; + #if HAVE_ERTS_MSEG ErtsMsegOpt_t mseg_opt; #endif @@ -315,6 +411,10 @@ struct Allctr_t_ { void (*check_mbc) (Allctr_t *, Carrier_t *); #endif + int fix_n_base; + int fix_shrink_scheduled; + ErtsAlcFixList_t *fix; + #ifdef USE_THREADS /* Mutex for this allocator */ erts_mtx_t mutex; @@ -323,6 +423,7 @@ struct Allctr_t_ { Allctr_t *prev; Allctr_t *next; } ts_list; + #endif int atoms_initialized; diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.c b/erts/emulator/beam/erl_ao_firstfit_alloc.c index 90d8ea7300..5bdb752d3a 100644 --- a/erts/emulator/beam/erl_ao_firstfit_alloc.c +++ b/erts/emulator/beam/erl_ao_firstfit_alloc.c @@ -170,14 +170,18 @@ erts_aoffalc_start(AOFFAllctr_t *alc, AOFFAllctrInit_t* aoffinit, AllctrInit_t *init) { - AOFFAllctr_t nulled_state = {{0}}; - /* {{0}} is used instead of {0}, in order to avoid (an incorrect) gcc - warning. gcc warns if {0} is used as initializer of a struct when - the first member is a struct (not if, for example, the third member - is a struct). */ + struct { + int dummy; + AOFFAllctr_t allctr; + } zero = {0}; + /* The struct with a dummy element first is used in order to avoid (an + incorrect) gcc warning. gcc warns if {0} is used as initializer of + a struct when the first member is a struct (not if, for example, + the third member is a struct). */ + Allctr_t *allctr = (Allctr_t *) alc; - sys_memcpy((void *) alc, (void *) &nulled_state, sizeof(AOFFAllctr_t)); + sys_memcpy((void *) alc, (void *) &zero.allctr, sizeof(AOFFAllctr_t)); allctr->mbc_header_size = sizeof(Carrier_t); allctr->min_mbc_size = MIN_MBC_SZ; diff --git a/erts/emulator/beam/erl_arith.c b/erts/emulator/beam/erl_arith.c index 64fad9fe0e..5150a8a507 100644 --- a/erts/emulator/beam/erl_arith.c +++ b/erts/emulator/beam/erl_arith.c @@ -164,14 +164,14 @@ BIF_RETTYPE bxor_2(BIF_ALIST_2) BIF_RET(erts_bxor(BIF_P, BIF_ARG_1, BIF_ARG_2)); } -BIF_RETTYPE bsl_2(Process* p, Eterm arg1, Eterm arg2) +BIF_RETTYPE bsl_2(BIF_ALIST_2) { - BIF_RET(shift(p, arg1, arg2, 0)); + BIF_RET(shift(BIF_P, BIF_ARG_1, BIF_ARG_2, 0)); } -BIF_RETTYPE bsr_2(Process* p, Eterm arg1, Eterm arg2) +BIF_RETTYPE bsr_2(BIF_ALIST_2) { - BIF_RET(shift(p, arg1, arg2, 1)); + BIF_RET(shift(BIF_P, BIF_ARG_1, BIF_ARG_2, 1)); } static Eterm diff --git a/erts/emulator/beam/erl_async.c b/erts/emulator/beam/erl_async.c index 91b64411d4..2dc7237f7c 100644 --- a/erts/emulator/beam/erl_async.c +++ b/erts/emulator/beam/erl_async.c @@ -24,10 +24,18 @@ #include "erl_sys_driver.h" #include "global.h" #include "erl_threads.h" +#include "erl_thr_queue.h" +#include "erl_async.h" + +#define ERTS_MAX_ASYNC_READY_CALLS_IN_SEQ 20 + +#define ERTS_ASYNC_PRINT_JOB 0 + +#if !defined(ERTS_SMP) && defined(USE_THREADS) && !ERTS_USE_ASYNC_READY_Q +# error "Need async ready queue in non-smp case" +#endif typedef struct _erl_async { - struct _erl_async* next; - struct _erl_async* prev; DE_Handle* hndl; /* The DE_Handle is needed when port is gone */ Eterm port; long async_id; @@ -35,345 +43,498 @@ typedef struct _erl_async { ErlDrvPDL pdl; void (*async_invoke)(void*); void (*async_free)(void*); -} ErlAsync; +#if ERTS_USE_ASYNC_READY_Q + Uint sched_id; + union { + ErtsThrQPrepEnQ_t *prep_enq; + ErtsThrQFinDeQ_t fin_deq; + } q; +#endif +} ErtsAsync; + +#if ERTS_USE_ASYNC_READY_Q + +/* + * We can do without the enqueue mutex since it isn't needed for + * thread safety. Its only purpose is to put async threads to sleep + * during a blast of ready async jobs. This in order to reduce + * contention on the enqueue end of the async ready queues. During + * such a blast without the enqueue mutex much cpu time is consumed + * by the async threads without them doing much progress which in turn + * slow down progress of scheduler threads. + */ +#define ERTS_USE_ASYNC_READY_ENQ_MTX 1 + +#if ERTS_USE_ASYNC_READY_ENQ_MTX typedef struct { - erts_mtx_t mtx; - erts_cnd_t cv; - erts_tid_t thr; - int len; -#ifndef ERTS_SMP - int hndl; + erts_mtx_t enq_mtx; +} ErtsAsyncReadyQXData; + #endif - ErlAsync* head; - ErlAsync* tail; -#ifdef ERTS_ENABLE_LOCK_CHECK - int no; + +typedef struct { +#if ERTS_USE_ASYNC_READY_ENQ_MTX + union { + ErtsAsyncReadyQXData data; + char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE( + sizeof(ErtsAsyncReadyQXData))]; + } x; #endif -} AsyncQueue; + ErtsThrQ_t thr_q; + ErtsThrQFinDeQ_t fin_deq; +} ErtsAsyncReadyQ; -static erts_smp_spinlock_t async_id_lock; -static long async_id = 0; +typedef union { + ErtsAsyncReadyQ arq; + char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsAsyncReadyQ))]; +} ErtsAlgndAsyncReadyQ; -#ifndef ERTS_SMP +#endif /* ERTS_USE_ASYNC_READY_Q */ -erts_mtx_t async_ready_mtx; -static ErlAsync* async_ready_list = NULL; +typedef struct { + ErtsThrQ_t thr_q; + erts_tid_t thr_id; +} ErtsAsyncQ; + +typedef union { + ErtsAsyncQ aq; + char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsAsyncQ))]; +} ErtsAlgndAsyncQ; +typedef struct { + int no_initialized; + erts_mtx_t mtx; + erts_cnd_t cnd; + erts_atomic_t id; +} ErtsAsyncInit; + +typedef struct { + union { + ErtsAsyncInit data; + char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsAsyncInit))]; + } init; + ErtsAlgndAsyncQ *queue; +#if ERTS_USE_ASYNC_READY_Q + ErtsAlgndAsyncReadyQ *ready_queue; #endif +} ErtsAsyncData; -/* -** Initialize worker threads (if supported) -*/ +int erts_async_max_threads; /* Initialized by erl_init.c */ +int erts_async_thread_suggested_stack_size; /* Initialized by erl_init.c */ -/* Detach from driver */ -static void async_detach(DE_Handle* dh) -{ - return; -} +static ErtsAsyncData *async; +#ifndef USE_THREADS -#ifdef USE_THREADS +void +erts_init_async(void) +{ -static AsyncQueue* async_q; +} -static void* async_main(void*); -static void async_add(ErlAsync*, AsyncQueue*); +#else -#ifndef ERTS_SMP -typedef struct ErtsAsyncReadyCallback_ ErtsAsyncReadyCallback; -struct ErtsAsyncReadyCallback_ { - struct ErtsAsyncReadyCallback_ *next; - void (*callback)(void); -}; +static void *async_main(void *); -static ErtsAsyncReadyCallback *callbacks; -static int async_handle; +static ERTS_INLINE ErtsAsyncQ * +async_q(int i) +{ + return &async->queue[i].aq; +} + +#if ERTS_USE_ASYNC_READY_Q -int erts_register_async_ready_callback(void (*funcp)(void)) +static ERTS_INLINE ErtsAsyncReadyQ * +async_ready_q(Uint sched_id) { - ErtsAsyncReadyCallback *cb = erts_alloc(ERTS_ALC_T_ARCALLBACK, - sizeof(ErtsAsyncReadyCallback)); - cb->next = callbacks; - cb->callback = funcp; - erts_mtx_lock(&async_ready_mtx); - callbacks = cb; - erts_mtx_unlock(&async_ready_mtx); - return async_handle; + return &async->ready_queue[((int)sched_id)-1].arq; } + #endif -int init_async(int hndl) +void +erts_init_async(void) { - erts_thr_opts_t thr_opts = ERTS_THR_OPTS_DEFAULT_INITER; - AsyncQueue* q; - int i; + async = NULL; + if (erts_async_max_threads > 0) { +#if ERTS_USE_ASYNC_READY_Q + ErtsThrQInit_t qinit = ERTS_THR_Q_INIT_DEFAULT; +#endif + erts_thr_opts_t thr_opts = ERTS_THR_OPTS_DEFAULT_INITER; + char *ptr; + size_t tot_size = 0; + int i; + + tot_size += ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsAsyncData)); + tot_size += sizeof(ErtsAlgndAsyncQ)*erts_async_max_threads; +#if ERTS_USE_ASYNC_READY_Q + tot_size += sizeof(ErtsAlgndAsyncReadyQ)*erts_no_schedulers; +#endif - thr_opts.detached = 0; - thr_opts.suggested_stack_size = erts_async_thread_suggested_stack_size; - -#ifndef ERTS_SMP - callbacks = NULL; - async_handle = hndl; - erts_mtx_init(&async_ready_mtx, "async_ready"); - async_ready_list = NULL; -#endif - - async_id = 0; - erts_smp_spinlock_init(&async_id_lock, "async_id"); - - async_q = q = (AsyncQueue*) - (erts_async_max_threads - ? erts_alloc(ERTS_ALC_T_ASYNC_Q, - erts_async_max_threads * sizeof(AsyncQueue)) - : NULL); - for (i = 0; i < erts_async_max_threads; i++) { - q->head = NULL; - q->tail = NULL; - q->len = 0; -#ifndef ERTS_SMP - q->hndl = hndl; -#endif -#ifdef ERTS_ENABLE_LOCK_CHECK - q->no = i; -#endif - erts_mtx_init(&q->mtx, "asyncq"); - erts_cnd_init(&q->cv); - erts_thr_create(&q->thr, async_main, (void*)q, &thr_opts); - q++; - } - return 0; -} + ptr = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_ASYNC_DATA, + tot_size); + async = (ErtsAsyncData *) ptr; + ptr += ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsAsyncData)); -int exit_async() -{ - int i; + async->init.data.no_initialized = 0; + erts_mtx_init(&async->init.data.mtx, "async_init_mtx"); + erts_cnd_init(&async->init.data.cnd); + erts_atomic_init_nob(&async->init.data.id, 0); - /* terminate threads */ - for (i = 0; i < erts_async_max_threads; i++) { - ErlAsync* a = (ErlAsync*) erts_alloc(ERTS_ALC_T_ASYNC, - sizeof(ErlAsync)); - a->port = NIL; - async_add(a, &async_q[i]); - } + async->queue = (ErtsAlgndAsyncQ *) ptr; + ptr += sizeof(ErtsAlgndAsyncQ)*erts_async_max_threads; - for (i = 0; i < erts_async_max_threads; i++) { - erts_thr_join(async_q[i].thr, NULL); - erts_mtx_destroy(&async_q[i].mtx); - erts_cnd_destroy(&async_q[i].cv); - } -#ifndef ERTS_SMP - erts_mtx_destroy(&async_ready_mtx); +#if ERTS_USE_ASYNC_READY_Q + + qinit.live.queue = ERTS_THR_Q_LIVE_LONG; + qinit.live.objects = ERTS_THR_Q_LIVE_SHORT; + qinit.notify = erts_notify_check_async_ready_queue; + + async->ready_queue = (ErtsAlgndAsyncReadyQ *) ptr; + ptr += sizeof(ErtsAlgndAsyncReadyQ)*erts_no_schedulers; + + for (i = 1; i <= erts_no_schedulers; i++) { + ErtsAsyncReadyQ *arq = async_ready_q(i); +#if ERTS_USE_ASYNC_READY_ENQ_MTX + erts_mtx_init(&arq->x.data.enq_mtx, "async_enq_mtx"); #endif - if (async_q) - erts_free(ERTS_ALC_T_ASYNC_Q, (void *) async_q); - return 0; + erts_thr_q_finalize_dequeue_state_init(&arq->fin_deq); + qinit.arg = (void *) (SWord) i; + erts_thr_q_initialize(&arq->thr_q, &qinit); + } + +#endif + + /* Create async threads... */ + + thr_opts.detached = 0; + thr_opts.suggested_stack_size + = erts_async_thread_suggested_stack_size; + + for (i = 0; i < erts_async_max_threads; i++) { + ErtsAsyncQ *aq = async_q(i); + erts_thr_create(&aq->thr_id, async_main, (void*) aq, &thr_opts); + } + + /* Wait for async threads to initialize... */ + + erts_mtx_lock(&async->init.data.mtx); + while (async->init.data.no_initialized != erts_async_max_threads) + erts_cnd_wait(&async->init.data.cnd, &async->init.data.mtx); + erts_mtx_unlock(&async->init.data.mtx); + + erts_mtx_destroy(&async->init.data.mtx); + erts_cnd_destroy(&async->init.data.cnd); + + } } +#if ERTS_USE_ASYNC_READY_Q -static void async_add(ErlAsync* a, AsyncQueue* q) +void * +erts_get_async_ready_queue(Uint sched_id) +{ + return (void *) async ? async_ready_q(sched_id) : NULL; +} + +#endif + +static ERTS_INLINE void async_add(ErtsAsync *a, ErtsAsyncQ* q) { if (is_internal_port(a->port)) { - ERTS_LC_ASSERT(erts_drvportid2port(a->port)); +#if ERTS_USE_ASYNC_READY_Q + ErtsAsyncReadyQ *arq = async_ready_q(a->sched_id); + a->q.prep_enq = erts_thr_q_prepare_enqueue(&arq->thr_q); +#endif /* make sure the driver will stay around */ - driver_lock_driver(internal_port_index(a->port)); + if (a->hndl) + erts_ddll_reference_referenced_driver(a->hndl); } - erts_mtx_lock(&q->mtx); +#if ERTS_ASYNC_PRINT_JOB + erts_fprintf(stderr, "-> %ld\n", a->async_id); +#endif - if (q->len == 0) { - q->head = a; - q->tail = a; - q->len = 1; - erts_cnd_signal(&q->cv); - } - else { /* no need to signal (since the worker is working) */ - a->next = q->head; - q->head->prev = a; - q->head = a; - q->len++; - } - erts_mtx_unlock(&q->mtx); + erts_thr_q_enqueue(&q->thr_q, a); } -static ErlAsync* async_get(AsyncQueue* q) +static ERTS_INLINE ErtsAsync *async_get(ErtsThrQ_t *q, + erts_tse_t *tse, + ErtsThrQPrepEnQ_t **prep_enq) { - ErlAsync* a; +#if ERTS_USE_ASYNC_READY_Q + int saved_fin_deq = 0; + ErtsThrQFinDeQ_t fin_deq; +#endif - erts_mtx_lock(&q->mtx); - while((a = q->tail) == NULL) { - erts_cnd_wait(&q->cv, &q->mtx); - } + while (1) { + ErtsAsync *a = (ErtsAsync *) erts_thr_q_dequeue(q); + if (a) { + +#if ERTS_USE_ASYNC_READY_Q + *prep_enq = a->q.prep_enq; + erts_thr_q_get_finalize_dequeue_data(q, &a->q.fin_deq); + if (saved_fin_deq) + erts_thr_q_append_finalize_dequeue_data(&a->q.fin_deq, &fin_deq); +#endif + + return a; + } + + if (ERTS_THR_Q_DIRTY != erts_thr_q_clean(q)) { + ErtsThrQFinDeQ_t tmp_fin_deq; + + erts_tse_reset(tse); + +#if ERTS_USE_ASYNC_READY_Q + chk_fin_deq: + if (erts_thr_q_get_finalize_dequeue_data(q, &tmp_fin_deq)) { + if (!saved_fin_deq) { + erts_thr_q_finalize_dequeue_state_init(&fin_deq); + saved_fin_deq = 1; + } + erts_thr_q_append_finalize_dequeue_data(&fin_deq, + &tmp_fin_deq); + } +#endif + + switch (erts_thr_q_inspect(q, 1)) { + case ERTS_THR_Q_DIRTY: + break; #ifdef ERTS_SMP - ASSERT(a && q->tail == a); + case ERTS_THR_Q_NEED_THR_PRGR: { + ErtsThrPrgrVal prgr = erts_thr_q_need_thr_progress(q); + erts_thr_progress_wakeup(NULL, prgr); + /* + * We do no dequeue finalizing in hope that a new async + * job will arrive before we are woken due to thread + * progress... + */ + erts_tse_wait(tse); + break; + } #endif - if (q->head == q->tail) { - q->head = q->tail = NULL; - q->len = 0; - } - else { - q->tail->prev->next = NULL; - q->tail = q->tail->prev; - q->len--; + case ERTS_THR_Q_CLEAN: + +#if ERTS_USE_ASYNC_READY_Q + if (saved_fin_deq) { + if (erts_thr_q_finalize_dequeue(&fin_deq)) + goto chk_fin_deq; + else + saved_fin_deq = 0; + } +#endif + + erts_tse_wait(tse); + break; + + default: + ASSERT(0); + break; + } + + } } - erts_mtx_unlock(&q->mtx); - return a; } - -static int async_del(long id) +static ERTS_INLINE void call_async_ready(ErtsAsync *a) { - int i; - /* scan all queue for an entry with async_id == 'id' */ - - for (i = 0; i < erts_async_max_threads; i++) { - ErlAsync* a; - erts_mtx_lock(&async_q[i].mtx); - - a = async_q[i].head; - while(a != NULL) { - if (a->async_id == id) { - if (a->prev != NULL) - a->prev->next = a->next; - else - async_q[i].head = a->next; - if (a->next != NULL) - a->next->prev = a->prev; - else - async_q[i].tail = a->prev; - async_q[i].len--; - erts_mtx_unlock(&async_q[i].mtx); - if (a->async_free != NULL) - a->async_free(a->async_data); - async_detach(a->hndl); - erts_free(ERTS_ALC_T_ASYNC, a); - return 1; - } - a = a->next; + Port *p = erts_id2port_sflgs(a->port, + NULL, + 0, + ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP); + if (!p) { + if (a->async_free) + a->async_free(a->async_data); + } + else { + if (async_ready(p, a->async_data)) { + if (a->async_free) + a->async_free(a->async_data); } - erts_mtx_unlock(&async_q[i].mtx); + erts_port_release(p); } - return 0; + if (a->hndl) + erts_ddll_dereference_driver(a->hndl); } -static void* async_main(void* arg) +static ERTS_INLINE void async_reply(ErtsAsync *a, ErtsThrQPrepEnQ_t *prep_enq) { - AsyncQueue* q = (AsyncQueue*) arg; +#if ERTS_USE_ASYNC_READY_Q + ErtsAsyncReadyQ *arq; -#ifdef ERTS_ENABLE_LOCK_CHECK - { - char buf[27]; - erts_snprintf(&buf[0], 27, "async %d", q->no); - erts_lc_set_thread_name(&buf[0]); - } + if (a->pdl) + driver_pdl_dec_refc(a->pdl); + +#if ERTS_ASYNC_PRINT_JOB + erts_fprintf(stderr, "=>> %ld\n", a->async_id); #endif - while(1) { - ErlAsync* a = async_get(q); + arq = async_ready_q(a->sched_id); - if (a->port == NIL) { /* TIME TO DIE SIGNAL */ - erts_free(ERTS_ALC_T_ASYNC, (void *) a); - break; - } - else { - (*a->async_invoke)(a->async_data); - /* Major problem if the code for async_invoke - or async_free is removed during a blocking operation */ +#if ERTS_USE_ASYNC_READY_ENQ_MTX + erts_mtx_lock(&arq->x.data.enq_mtx); +#endif + + erts_thr_q_enqueue_prepared(&arq->thr_q, (void *) a, prep_enq); + +#if ERTS_USE_ASYNC_READY_ENQ_MTX + erts_mtx_unlock(&arq->x.data.enq_mtx); +#endif + +#else /* ERTS_USE_ASYNC_READY_Q */ + + call_async_ready(a); + if (a->pdl) + driver_pdl_dec_refc(a->pdl); + erts_free(ERTS_ALC_T_ASYNC, (void *) a); + +#endif /* ERTS_USE_ASYNC_READY_Q */ +} + + +static void +async_wakeup(void *vtse) +{ + erts_tse_set((erts_tse_t *) vtse); +} + +static erts_tse_t *async_thread_init(ErtsAsyncQ *aq) +{ + ErtsThrQInit_t qinit = ERTS_THR_Q_INIT_DEFAULT; + erts_tse_t *tse = erts_tse_fetch(); #ifdef ERTS_SMP - { - Port *p; - p = erts_id2port_sflgs(a->port, - NULL, - 0, - ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP); - if (!p) { - if (a->async_free) - (*a->async_free)(a->async_data); - } - else { - if (async_ready(p, a->async_data)) { - if (a->async_free) - (*a->async_free)(a->async_data); - } - async_detach(a->hndl); - erts_port_release(p); - } - if (a->pdl) { - driver_pdl_dec_refc(a->pdl); - } - erts_free(ERTS_ALC_T_ASYNC, (void *) a); - } -#else - if (a->pdl) { - driver_pdl_dec_refc(a->pdl); - } - erts_mtx_lock(&async_ready_mtx); - a->next = async_ready_list; - async_ready_list = a; - erts_mtx_unlock(&async_ready_mtx); - sys_async_ready(q->hndl); + ErtsThrPrgrCallbacks callbacks; + + callbacks.arg = (void *) tse; + callbacks.wakeup = async_wakeup; + callbacks.prepare_wait = NULL; + callbacks.wait = NULL; + + erts_thr_progress_register_unmanaged_thread(&callbacks); #endif - } - } - return NULL; + qinit.live.queue = ERTS_THR_Q_LIVE_LONG; + qinit.live.objects = ERTS_THR_Q_LIVE_SHORT; + qinit.arg = (void *) tse; + qinit.notify = async_wakeup; +#if ERTS_USE_ASYNC_READY_Q + qinit.auto_finalize_dequeue = 0; +#endif + + erts_thr_q_initialize(&aq->thr_q, &qinit); + + /* Inform main thread that we are done initializing... */ + erts_mtx_lock(&async->init.data.mtx); + async->init.data.no_initialized++; + erts_cnd_signal(&async->init.data.cnd); + erts_mtx_unlock(&async->init.data.mtx); + + return tse; } +static void *async_main(void* arg) +{ + ErtsAsyncQ *aq = (ErtsAsyncQ *) arg; + erts_tse_t *tse = async_thread_init(aq); + + while (1) { + ErtsThrQPrepEnQ_t *prep_enq; + ErtsAsync *a = async_get(&aq->thr_q, tse, &prep_enq); + if (is_nil(a->port)) + break; /* Time to die */ +#if ERTS_ASYNC_PRINT_JOB + erts_fprintf(stderr, "<- %ld\n", a->async_id); #endif -#ifndef ERTS_SMP + a->async_invoke(a->async_data); + + async_reply(a, prep_enq); + } + + return NULL; +} + +#endif /* USE_THREADS */ -int check_async_ready(void) +void +erts_exit_flush_async(void) { #ifdef USE_THREADS - ErtsAsyncReadyCallback *cbs; + int i; + ErtsAsync a; + a.port = NIL; + /* + * Terminate threads in order to flush queues. We do not + * bother to clean everything up since we are about to + * terminate the runtime system and a cleanup would only + * delay the termination. + */ + for (i = 0; i < erts_async_max_threads; i++) + async_add(&a, async_q(i)); + for (i = 0; i < erts_async_max_threads; i++) + erts_thr_join(async->queue[i].aq.thr_id, NULL); #endif - ErlAsync* a; - int count = 0; +} - erts_mtx_lock(&async_ready_mtx); - a = async_ready_list; - async_ready_list = NULL; -#ifdef USE_THREADS - cbs = callbacks; -#endif - erts_mtx_unlock(&async_ready_mtx); - - while(a != NULL) { - ErlAsync* a_next = a->next; - /* Every port not dead */ - Port *p = erts_id2port_sflgs(a->port, - NULL, - 0, - ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP); - if (!p) { - if (a->async_free) - (*a->async_free)(a->async_data); - } - else { - count++; - if (async_ready(p, a->async_data)) { - if (a->async_free != NULL) - (*a->async_free)(a->async_data); - } - async_detach(a->hndl); - erts_port_release(p); +#if defined(USE_THREADS) && ERTS_USE_ASYNC_READY_Q + +int erts_check_async_ready(void *varq) +{ + ErtsAsyncReadyQ *arq = (ErtsAsyncReadyQ *) varq; + int res = 1; + int i; + + for (i = 0; i < ERTS_MAX_ASYNC_READY_CALLS_IN_SEQ; i++) { + ErtsAsync *a = (ErtsAsync *) erts_thr_q_dequeue(&arq->thr_q); + if (!a) { + res = 0; + break; } + +#if ERTS_ASYNC_PRINT_JOB + erts_fprintf(stderr, "<<= %ld\n", a->async_id); +#endif + erts_thr_q_append_finalize_dequeue_data(&arq->fin_deq, &a->q.fin_deq); + call_async_ready(a); erts_free(ERTS_ALC_T_ASYNC, (void *) a); - a = a_next; } -#ifdef USE_THREADS - for (; cbs; cbs = cbs->next) - (*cbs->callback)(); -#endif - return count; + + erts_thr_q_finalize_dequeue(&arq->fin_deq); + + return res; } +int erts_async_ready_clean(void *varq, void *val) +{ + ErtsAsyncReadyQ *arq = (ErtsAsyncReadyQ *) varq; + ErtsThrQCleanState_t cstate; + + cstate = erts_thr_q_clean(&arq->thr_q); + + if (erts_thr_q_finalize_dequeue(&arq->fin_deq)) + return ERTS_ASYNC_READY_DIRTY; + + switch (cstate) { + case ERTS_THR_Q_DIRTY: + return ERTS_ASYNC_READY_DIRTY; +#ifdef ERTS_SMP + case ERTS_THR_Q_NEED_THR_PRGR: + *((ErtsThrPrgrVal *) val) + = erts_thr_q_need_thr_progress(&arq->thr_q); + return ERTS_ASYNC_READY_NEED_THR_PRGR; #endif + case ERTS_THR_Q_CLEAN: + break; + } + return ERTS_ASYNC_READY_CLEAN; +} +#endif /* ** Schedule async_invoke on a worker thread @@ -393,19 +554,29 @@ long driver_async(ErlDrvPort ix, unsigned int* key, void (*async_invoke)(void*), void* async_data, void (*async_free)(void*)) { - ErlAsync* a = (ErlAsync*) erts_alloc(ERTS_ALC_T_ASYNC, sizeof(ErlAsync)); - Port* prt = erts_drvport2port(ix); + ErtsAsync* a; + Port* prt; long id; unsigned int qix; +#if ERTS_USE_ASYNC_READY_Q + Uint sched_id; + sched_id = erts_get_scheduler_id(); + if (!sched_id) + sched_id = 1; +#endif + prt = erts_drvport2port(ix); if (!prt) return -1; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); - a->next = NULL; - a->prev = NULL; + a = (ErtsAsync*) erts_alloc(ERTS_ALC_T_ASYNC, sizeof(ErtsAsync)); + +#if ERTS_USE_ASYNC_READY_Q + a->sched_id = sched_id; +#endif a->hndl = (DE_Handle*)prt->drv_ptr->handle; a->port = prt->id; a->pdl = NULL; @@ -413,12 +584,16 @@ long driver_async(ErlDrvPort ix, unsigned int* key, a->async_invoke = async_invoke; a->async_free = async_free; - erts_smp_spin_lock(&async_id_lock); - async_id = (async_id + 1) & 0x7fffffff; - if (async_id == 0) - async_id++; - id = async_id; - erts_smp_spin_unlock(&async_id_lock); + if (!async) + id = 0; + else { + do { + id = erts_atomic_inc_read_nob(&async->init.data.id); + } while (id == 0); + if (id < 0) + id *= -1; + ASSERT(id > 0); + } a->async_id = id; @@ -437,7 +612,7 @@ long driver_async(ErlDrvPort ix, unsigned int* key, driver_pdl_inc_refc(prt->port_data_lock); a->pdl = prt->port_data_lock; } - async_add(a, &async_q[qix]); + async_add(a, async_q(qix)); return id; } #endif @@ -455,10 +630,16 @@ long driver_async(ErlDrvPort ix, unsigned int* key, int driver_async_cancel(unsigned int id) { -#ifdef USE_THREADS - if (erts_async_max_threads > 0) - return async_del(id); -#endif + /* + * Not supported anymore. Always fail (which is backward + * compatible). + * + * This functionality could be implemented again. However, + * it is (and always has been) completely useless since + * it doesn't give you any guarantees whatsoever. The user + * needs to (and always have had to) synchronize in his/her + * own code in order to get any guarantees. + */ return 0; } diff --git a/erts/emulator/beam/erl_async.h b/erts/emulator/beam/erl_async.h new file mode 100644 index 0000000000..95374a8fc9 --- /dev/null +++ b/erts/emulator/beam/erl_async.h @@ -0,0 +1,66 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2011. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +#ifndef ERL_ASYNC_H__ +#define ERL_ASYNC_H__ + +#define ERTS_MAX_NO_OF_ASYNC_THREADS 1024 +extern int erts_async_max_threads; +#define ERTS_ASYNC_THREAD_MIN_STACK_SIZE 16 /* Kilo words */ +#define ERTS_ASYNC_THREAD_MAX_STACK_SIZE 8192 /* Kilo words */ +extern int erts_async_thread_suggested_stack_size; + +#ifdef USE_THREADS + +#ifdef ERTS_SMP +/* + * With smp support we can choose to have, or not to + * have an async ready queue. + */ +#define ERTS_USE_ASYNC_READY_Q 1 +#endif + +#ifndef ERTS_SMP +/* In non-smp case we *need* the async ready queue */ +# undef ERTS_USE_ASYNC_READY_Q +# define ERTS_USE_ASYNC_READY_Q 1 +#endif + +#ifndef ERTS_USE_ASYNC_READY_Q +# define ERTS_USE_ASYNC_READY_Q 0 +#endif + +#if ERTS_USE_ASYNC_READY_Q +int erts_check_async_ready(void *); +int erts_async_ready_clean(void *, void *); +void *erts_get_async_ready_queue(Uint sched_id); +#define ERTS_ASYNC_READY_CLEAN 0 +#define ERTS_ASYNC_READY_DIRTY 1 +#ifdef ERTS_SMP +#define ERTS_ASYNC_READY_NEED_THR_PRGR 2 +#endif +#endif /* ERTS_USE_ASYNC_READY_Q */ + +#endif /* USE_THREADS */ + +void erts_init_async(void); +void erts_exit_flush_async(void); + + +#endif /* ERL_ASYNC_H__ */ diff --git a/erts/emulator/beam/erl_bestfit_alloc.c b/erts/emulator/beam/erl_bestfit_alloc.c index f2199d41a1..c50fdeb4e8 100644 --- a/erts/emulator/beam/erl_bestfit_alloc.c +++ b/erts/emulator/beam/erl_bestfit_alloc.c @@ -161,14 +161,18 @@ erts_bfalc_start(BFAllctr_t *bfallctr, BFAllctrInit_t *bfinit, AllctrInit_t *init) { - BFAllctr_t nulled_state = {{0}}; - /* {{0}} is used instead of {0}, in order to avoid (an incorrect) gcc - warning. gcc warns if {0} is used as initializer of a struct when - the first member is a struct (not if, for example, the third member - is a struct). */ + struct { + int dummy; + BFAllctr_t allctr; + } zero = {0}; + /* The struct with a dummy element first is used in order to avoid (an + incorrect) gcc warning. gcc warns if {0} is used as initializer of + a struct when the first member is a struct (not if, for example, + the third member is a struct). */ + Allctr_t *allctr = (Allctr_t *) bfallctr; - sys_memcpy((void *) bfallctr, (void *) &nulled_state, sizeof(BFAllctr_t)); + sys_memcpy((void *) bfallctr, (void *) &zero.allctr, sizeof(BFAllctr_t)); bfallctr->address_order = bfinit->ao; diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c index 684fa5d12f..6d022e0d11 100644 --- a/erts/emulator/beam/erl_bif_binary.c +++ b/erts/emulator/beam/erl_bif_binary.c @@ -65,6 +65,10 @@ static Export binary_copy_trap_export; static BIF_RETTYPE binary_copy_trap(BIF_ALIST_2); static Uint max_loop_limit; +static BIF_RETTYPE +binary_match(Process *p, Eterm arg1, Eterm arg2, Eterm arg3); +static BIF_RETTYPE +binary_matches(Process *p, Eterm arg1, Eterm arg2, Eterm arg3); void erts_init_bif_binary(void) { @@ -1399,6 +1403,12 @@ static BIF_RETTYPE binary_matches_trap(BIF_ALIST_3) BIF_RETTYPE binary_match_3(BIF_ALIST_3) { + return binary_match(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); +} + +static BIF_RETTYPE +binary_match(Process *p, Eterm arg1, Eterm arg2, Eterm arg3) +{ Uint hsstart; Uint hsend; Eterm *tp; @@ -1408,17 +1418,17 @@ BIF_RETTYPE binary_match_3(BIF_ALIST_3) int runres; Eterm result; - if (is_not_binary(BIF_ARG_1)) { + if (is_not_binary(arg1)) { goto badarg; } - if (parse_match_opts_list(BIF_ARG_3,BIF_ARG_1,&hsstart,&hsend)) { + if (parse_match_opts_list(arg3,arg1,&hsstart,&hsend)) { goto badarg; } if (hsend == 0) { BIF_RET(am_nomatch); } - if (is_tuple(BIF_ARG_2)) { - tp = tuple_val(BIF_ARG_2); + if (is_tuple(arg2)) { + tp = tuple_val(arg2); if (arityval(*tp) != 2 || is_not_atom(tp[1])) { goto badarg; } @@ -1437,13 +1447,13 @@ BIF_RETTYPE binary_match_3(BIF_ALIST_3) goto badarg; } bin_term = tp[2]; - } else if (do_binary_match_compile(BIF_ARG_2,&type,&bin)) { + } else if (do_binary_match_compile(arg2,&type,&bin)) { goto badarg; } - runres = do_binary_match(BIF_P,BIF_ARG_1,hsstart,hsend,type,bin,NIL,&result); + runres = do_binary_match(p,arg1,hsstart,hsend,type,bin,NIL,&result); if (runres == DO_BIN_MATCH_RESTART && bin_term == NIL) { - Eterm *hp = HAlloc(BIF_P, PROC_BIN_SIZE); - bin_term = erts_mk_magic_binary_term(&hp, &MSO(BIF_P), bin); + Eterm *hp = HAlloc(p, PROC_BIN_SIZE); + bin_term = erts_mk_magic_binary_term(&hp, &MSO(p), bin); } else if (bin_term == NIL) { erts_bin_free(bin); } @@ -1451,17 +1461,23 @@ BIF_RETTYPE binary_match_3(BIF_ALIST_3) case DO_BIN_MATCH_OK: BIF_RET(result); case DO_BIN_MATCH_RESTART: - BUMP_ALL_REDS(BIF_P); - BIF_TRAP3(&binary_match_trap_export, BIF_P, BIF_ARG_1, result, bin_term); + BUMP_ALL_REDS(p); + BIF_TRAP3(&binary_match_trap_export, p, arg1, result, bin_term); default: goto badarg; } badarg: - BIF_ERROR(BIF_P,BADARG); + BIF_ERROR(p,BADARG); } BIF_RETTYPE binary_matches_3(BIF_ALIST_3) { + return binary_matches(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); +} + +static BIF_RETTYPE +binary_matches(Process *p, Eterm arg1, Eterm arg2, Eterm arg3) +{ Uint hsstart, hsend; Eterm *tp; Eterm type; @@ -1470,17 +1486,17 @@ BIF_RETTYPE binary_matches_3(BIF_ALIST_3) int runres; Eterm result; - if (is_not_binary(BIF_ARG_1)) { + if (is_not_binary(arg1)) { goto badarg; } - if (parse_match_opts_list(BIF_ARG_3,BIF_ARG_1,&hsstart,&hsend)) { + if (parse_match_opts_list(arg3,arg1,&hsstart,&hsend)) { goto badarg; } if (hsend == 0) { BIF_RET(NIL); } - if (is_tuple(BIF_ARG_2)) { - tp = tuple_val(BIF_ARG_2); + if (is_tuple(arg2)) { + tp = tuple_val(arg2); if (arityval(*tp) != 2 || is_not_atom(tp[1])) { goto badarg; } @@ -1499,14 +1515,14 @@ BIF_RETTYPE binary_matches_3(BIF_ALIST_3) goto badarg; } bin_term = tp[2]; - } else if (do_binary_match_compile(BIF_ARG_2,&type,&bin)) { + } else if (do_binary_match_compile(arg2,&type,&bin)) { goto badarg; } - runres = do_binary_matches(BIF_P,BIF_ARG_1,hsstart,hsend,type,bin, + runres = do_binary_matches(p,arg1,hsstart,hsend,type,bin, NIL,&result); if (runres == DO_BIN_MATCH_RESTART && bin_term == NIL) { - Eterm *hp = HAlloc(BIF_P, PROC_BIN_SIZE); - bin_term = erts_mk_magic_binary_term(&hp, &MSO(BIF_P), bin); + Eterm *hp = HAlloc(p, PROC_BIN_SIZE); + bin_term = erts_mk_magic_binary_term(&hp, &MSO(p), bin); } else if (bin_term == NIL) { erts_bin_free(bin); } @@ -1514,26 +1530,26 @@ BIF_RETTYPE binary_matches_3(BIF_ALIST_3) case DO_BIN_MATCH_OK: BIF_RET(result); case DO_BIN_MATCH_RESTART: - BUMP_ALL_REDS(BIF_P); - BIF_TRAP3(&binary_matches_trap_export, BIF_P, BIF_ARG_1, result, + BUMP_ALL_REDS(p); + BIF_TRAP3(&binary_matches_trap_export, p, arg1, result, bin_term); default: goto badarg; } badarg: - BIF_ERROR(BIF_P,BADARG); + BIF_ERROR(p,BADARG); } BIF_RETTYPE binary_match_2(BIF_ALIST_2) { - return binary_match_3(BIF_P,BIF_ARG_1,BIF_ARG_2,((Eterm) 0)); + return binary_match(BIF_P,BIF_ARG_1,BIF_ARG_2,((Eterm) 0)); } BIF_RETTYPE binary_matches_2(BIF_ALIST_2) { - return binary_matches_3(BIF_P,BIF_ARG_1,BIF_ARG_2,((Eterm) 0)); + return binary_matches(BIF_P,BIF_ARG_1,BIF_ARG_2,((Eterm) 0)); } diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c index 9631fb50db..a9fd28c66b 100644 --- a/erts/emulator/beam/erl_bif_ddll.c +++ b/erts/emulator/beam/erl_bif_ddll.c @@ -142,9 +142,11 @@ static void ddll_no_more_references(void *vdh); * really load and add as LOADED {ok,loaded} {ok,pending_driver} * {error, permanent} {error,load_error()} */ -BIF_RETTYPE erl_ddll_try_load_3(Process *p, Eterm path_term, - Eterm name_term, Eterm options) +BIF_RETTYPE erl_ddll_try_load_3(BIF_ALIST_3) { + Eterm path_term = BIF_ARG_1; + Eterm name_term = BIF_ARG_2; + Eterm options = BIF_ARG_3; char *path = NULL; Uint path_len; char *name = NULL; @@ -236,7 +238,7 @@ BIF_RETTYPE erl_ddll_try_load_3(Process *p, Eterm path_term, sys_strcpy(path+path_len,name); #if DDLL_SMP - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN); + erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); lock_drv_list(); #endif if ((drv = lookup_driver(name)) != NULL) { @@ -247,7 +249,7 @@ BIF_RETTYPE erl_ddll_try_load_3(Process *p, Eterm path_term, } else { dh = drv->handle; if (dh->status == ERL_DE_OK) { - int is_last = is_last_user(dh,p); + int is_last = is_last_user(dh, BIF_P); if (reload == 1 && !is_last) { /*Want reload if no other users, but there are others...*/ @@ -261,7 +263,8 @@ BIF_RETTYPE erl_ddll_try_load_3(Process *p, Eterm path_term, soft_error_term = am_inconsistent; goto soft_error; } - if ((old = find_proc_entry(dh, p, ERL_DE_PROC_LOADED)) == + if ((old = find_proc_entry(dh, BIF_P, + ERL_DE_PROC_LOADED)) == NULL) { soft_error_term = am_not_loaded_by_this_process; goto soft_error; @@ -272,7 +275,7 @@ BIF_RETTYPE erl_ddll_try_load_3(Process *p, Eterm path_term, } /* Reload requested and granted */ dereference_all_processes(dh); - set_driver_reloading(dh, p, path, name, flags); + set_driver_reloading(dh, BIF_P, path, name, flags); if (dh->flags & ERL_DE_FL_KILL_PORTS) { kill_ports = 1; } @@ -286,7 +289,7 @@ BIF_RETTYPE erl_ddll_try_load_3(Process *p, Eterm path_term, soft_error_term = am_inconsistent; goto soft_error; } - add_proc_loaded(dh,p); + add_proc_loaded(dh, BIF_P); erts_ddll_reference_driver(dh); monitor = 0; ok_term = mkatom("already_loaded"); @@ -308,7 +311,7 @@ BIF_RETTYPE erl_ddll_try_load_3(Process *p, Eterm path_term, notify_all(dh, drv->name, ERL_DE_PROC_AWAIT_UNLOAD, am_UP, am_unload_cancelled); - add_proc_loaded(dh,p); + add_proc_loaded(dh, BIF_P); erts_ddll_reference_driver(dh); monitor = 0; ok_term = mkatom("already_loaded"); @@ -325,7 +328,8 @@ BIF_RETTYPE erl_ddll_try_load_3(Process *p, Eterm path_term, goto soft_error; } /* Load of granted unload... */ - add_proc_loaded_deref(dh,p); /* Dont reference, will happen after reload */ + /* Don't reference, will happen after reload */ + add_proc_loaded_deref(dh, BIF_P); ++monitor; ok_term = am_pending_driver; } else { /* ERL_DE_PERMANENT */ @@ -345,7 +349,7 @@ BIF_RETTYPE erl_ddll_try_load_3(Process *p, Eterm path_term, goto soft_error; } else { dh->flags = flags; - add_proc_loaded(dh,p); + add_proc_loaded(dh, BIF_P); first_ddll_reference(dh); monitor = 0; ok_term = mkatom("loaded"); @@ -369,7 +373,7 @@ BIF_RETTYPE erl_ddll_try_load_3(Process *p, Eterm path_term, if (!(prt->status & FREE_PORT_FLAGS) && prt->drv_ptr->handle == dh) { #if DDLL_SMP - erts_smp_atomic_inc(&prt->refc); + erts_smp_atomic_inc_nob(&prt->refc); /* Extremely rare spinlock */ while(prt->status & ERTS_PORT_SFLG_INITIALIZING) { erts_smp_port_state_unlock(prt); @@ -397,18 +401,18 @@ BIF_RETTYPE erl_ddll_try_load_3(Process *p, Eterm path_term, #if DDLL_SMP erts_ddll_reference_driver(dh); unlock_drv_list(); - erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); + erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); lock_drv_list(); erts_ddll_dereference_driver(dh); #endif - p->flags |= F_USING_DDLL; + BIF_P->flags |= F_USING_DDLL; if (monitor) { - Eterm mref = add_monitor(p, dh, ERL_DE_PROC_AWAIT_LOAD); - hp = HAlloc(p,4); + Eterm mref = add_monitor(BIF_P, dh, ERL_DE_PROC_AWAIT_LOAD); + hp = HAlloc(BIF_P, 4); t = TUPLE3(hp, am_ok, ok_term, mref); } else { - hp = HAlloc(p,3); + hp = HAlloc(BIF_P, 3); t = TUPLE2(hp, am_ok, ok_term); } #if DDLL_SMP @@ -416,33 +420,33 @@ BIF_RETTYPE erl_ddll_try_load_3(Process *p, Eterm path_term, #endif erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) path); erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) name); - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p)); + ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(BIF_P)); BIF_RET(t); soft_error: #if DDLL_SMP unlock_drv_list(); - erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); + erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); #endif if (do_build_load_error) { - soft_error_term = build_load_error(p, build_this_load_error); + soft_error_term = build_load_error(BIF_P, build_this_load_error); } - hp = HAlloc(p,3); + hp = HAlloc(BIF_P, 3); t = TUPLE2(hp, am_error, soft_error_term); erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) path); erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) name); - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p)); + ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(BIF_P)); BIF_RET(t); error: assert_drv_list_not_locked(); - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p)); + ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(BIF_P)); if (path != NULL) { erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) path); } if (name != NULL) { erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) name); } - BIF_ERROR(p,BADARG); + BIF_ERROR(BIF_P, BADARG); } /* @@ -481,8 +485,10 @@ BIF_RETTYPE erl_ddll_try_load_3(Process *p, Eterm path_term, any AWAIT_LOAD-waiters with {'DOWN', ref(), driver, name(), load_cancelled} If the driver made itself permanent, {'UP', ref(), driver, name(), permanent} */ -Eterm erl_ddll_try_unload_2(Process *p, Eterm name_term, Eterm options) +Eterm erl_ddll_try_unload_2(BIF_ALIST_2) { + Eterm name_term = BIF_ARG_1; + Eterm options = BIF_ARG_2; char *name = NULL; Eterm ok_term = NIL; Eterm soft_error_term = NIL; @@ -495,7 +501,7 @@ Eterm erl_ddll_try_unload_2(Process *p, Eterm name_term, Eterm options) Eterm l; int kill_ports = 0; - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN); + erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); for(l = options; is_list(l); l = CDR(list_val(l))) { Eterm opt = CAR(list_val(l)); @@ -548,7 +554,7 @@ Eterm erl_ddll_try_unload_2(Process *p, Eterm name_term, Eterm options) if (dh->flags & ERL_DE_FL_KILL_PORTS) { kill_ports = 1; } - if ((pe = find_proc_entry(dh, p, ERL_DE_PROC_LOADED)) == NULL) { + if ((pe = find_proc_entry(dh, BIF_P, ERL_DE_PROC_LOADED)) == NULL) { if (num_procs(dh, ERL_DE_PROC_LOADED) > 0) { soft_error_term = am_not_loaded_by_this_process; goto soft_error; @@ -597,7 +603,7 @@ done: if (!(prt->status & FREE_PORT_FLAGS) && prt->drv_ptr->handle == dh) { #if DDLL_SMP - erts_smp_atomic_inc(&prt->refc); + erts_smp_atomic_inc_nob(&prt->refc); /* Extremely rare spinlock */ while(prt->status & ERTS_PORT_SFLG_INITIALIZING) { erts_smp_port_state_unlock(prt); @@ -624,22 +630,22 @@ done: #if DDLL_SMP erts_ddll_reference_driver(dh); unlock_drv_list(); - erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); + erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); lock_drv_list(); erts_ddll_dereference_driver(dh); #endif erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) name); - p->flags |= F_USING_DDLL; + BIF_P->flags |= F_USING_DDLL; if (monitor > 0) { - Eterm mref = add_monitor(p, dh, ERL_DE_PROC_AWAIT_UNLOAD); - hp = HAlloc(p,4); + Eterm mref = add_monitor(BIF_P, dh, ERL_DE_PROC_AWAIT_UNLOAD); + hp = HAlloc(BIF_P, 4); t = TUPLE3(hp, am_ok, ok_term, mref); } else { - hp = HAlloc(p,3); + hp = HAlloc(BIF_P, 3); t = TUPLE2(hp, am_ok, ok_term); } if (kill_ports > 1) { - ERTS_BIF_CHK_EXITED(p); /* May be exited by port killing */ + ERTS_BIF_CHK_EXITED(BIF_P); /* May be exited by port killing */ } #if DDLL_SMP unlock_drv_list(); @@ -651,8 +657,8 @@ soft_error: unlock_drv_list(); #endif erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) name); - erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); - hp = HAlloc(p,3); + erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + hp = HAlloc(BIF_P, 3); t = TUPLE2(hp, am_error, soft_error_term); BIF_RET(t); @@ -661,21 +667,21 @@ soft_error: if (name != NULL) { erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) name); } - erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); - BIF_ERROR(p,BADARG); + erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + BIF_ERROR(BIF_P, BADARG); } /* * A shadow of the "real" demonitor BIF */ -BIF_RETTYPE erl_ddll_demonitor_1(Process *p, Eterm ref) +BIF_RETTYPE erl_ddll_demonitor_1(BIF_ALIST_1) { - if (is_not_internal_ref(ref)) { - BIF_ERROR(p, BADARG); + if (is_not_internal_ref(BIF_ARG_1)) { + BIF_ERROR(BIF_P, BADARG); } - if (p->flags & F_USING_DDLL) { - erts_ddll_remove_monitor(p, ref, ERTS_PROC_LOCK_MAIN); + if (BIF_P->flags & F_USING_DDLL) { + erts_ddll_remove_monitor(BIF_P, BIF_ARG_1, ERTS_PROC_LOCK_MAIN); } BIF_RET(am_true); } @@ -683,18 +689,18 @@ BIF_RETTYPE erl_ddll_demonitor_1(Process *p, Eterm ref) /* * A shadow of the "real" monitor BIF */ -BIF_RETTYPE erl_ddll_monitor_2(Process *p, Eterm dr, Eterm what) +BIF_RETTYPE erl_ddll_monitor_2(BIF_ALIST_2) { - if (dr != am_driver) { - BIF_ERROR(p,BADARG); + if (BIF_ARG_1 != am_driver) { + BIF_ERROR(BIF_P, BADARG); } - return erts_ddll_monitor_driver(p, what, ERTS_PROC_LOCK_MAIN); + return erts_ddll_monitor_driver(BIF_P, BIF_ARG_2, ERTS_PROC_LOCK_MAIN); } /* * Return list of loaded drivers {ok,[string()]} */ -Eterm erl_ddll_loaded_drivers_0(Process *p) +BIF_RETTYPE erl_ddll_loaded_drivers_0(BIF_ALIST_0) { Eterm *hp; int need = 3; @@ -706,7 +712,7 @@ Eterm erl_ddll_loaded_drivers_0(Process *p) for (drv = driver_list; drv; drv = drv->next) { need += sys_strlen(drv->name)*2+2; } - hp = HAlloc(p,need); + hp = HAlloc(BIF_P, need); for (drv = driver_list; drv; drv = drv->next) { Eterm l; l = buf_to_intlist(&hp, drv->name, sys_strlen(drv->name), NIL); @@ -726,8 +732,11 @@ Eterm erl_ddll_loaded_drivers_0(Process *p) * item is processes, driver_options, port_count, linked_in_driver, * permanent, awaiting_load, awaiting_unload */ -Eterm erl_ddll_info_2(Process *p, Eterm name_term, Eterm item) +BIF_RETTYPE erl_ddll_info_2(BIF_ALIST_2) { + Process *p = BIF_P; + Eterm name_term = BIF_ARG_1; + Eterm item = BIF_ARG_2; char *name = NULL; Eterm res = NIL; erts_driver_t *drv; @@ -850,8 +859,10 @@ Eterm erl_ddll_info_2(Process *p, Eterm name_term, Eterm item) * Backend for erl_ddll:format_error, handles all "soft" errors returned by builtins, * possibly by calling the system specific error handler */ -Eterm erl_ddll_format_error_int_1(Process *p, Eterm code_term) +BIF_RETTYPE erl_ddll_format_error_int_1(BIF_ALIST_1) { + Process *p = BIF_P; + Eterm code_term = BIF_ARG_1; char *errstring = NULL; int errint; int len; @@ -1054,7 +1065,7 @@ void erts_ddll_proc_dead(Process *p, ErtsProcLocks plocks) if (!(prt->status & FREE_PORT_FLAGS) && prt->drv_ptr->handle == dh) { #if DDLL_SMP - erts_smp_atomic_inc(&prt->refc); + erts_smp_atomic_inc_nob(&prt->refc); while(prt->status & ERTS_PORT_SFLG_INITIALIZING) { erts_smp_port_state_unlock(prt); erts_smp_port_state_lock(prt); @@ -1602,7 +1613,7 @@ static int do_load_driver_entry(DE_Handle *dh, char *path, char *name) erts_sys_ddll_close(dh->handle); return ERL_DE_LOAD_ERROR_BAD_NAME; } - erts_smp_atomic_init(&(dh->refc), (erts_aint_t) 0); + erts_smp_atomic_init_nob(&(dh->refc), (erts_aint_t) 0); dh->port_count = 0; dh->full_path = erts_alloc(ERTS_ALC_T_DDLL_HANDLE, sys_strlen(path) + 1); sys_strcpy(dh->full_path, path); diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index f264bf44df..a79feaebdb 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -39,6 +39,8 @@ #include "dist.h" #include "erl_gc.h" #include "erl_cpu_topology.h" +#include "erl_async.h" +#include "erl_thr_progress.h" #ifdef HIPE #include "hipe_arch.h" #endif @@ -52,6 +54,9 @@ #include <valgrind/memcheck.h> #endif +static Export* alloc_info_trap = NULL; +static Export* alloc_sizes_trap = NULL; + #define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1) /* Keep erts_system_version as a global variable for easy access from a core */ @@ -119,6 +124,16 @@ static char erts_system_version[] = ("Erlang " ERLANG_OTP_RELEASE # define PERFMON_GETPCR _IOR('P', 2, unsigned long long) #endif +/* Cached, pre-built {OsType,OsFlavor} and {Major,Minor,Build} tuples */ +static Eterm os_type_tuple; +static Eterm os_version_tuple; + +static BIF_RETTYPE port_info(Process* p, Eterm portid, Eterm item); + +static Eterm +current_function(Process* p, Process* rp, Eterm** hpp, int full_info); +static Eterm current_stacktrace(Process* p, Process* rp, Eterm** hpp); + static Eterm bld_bin_list(Uint **hpp, Uint *szp, ErlOffHeap* oh) { @@ -135,7 +150,7 @@ bld_bin_list(Uint **hpp, Uint *szp, ErlOffHeap* oh) if (szp) *szp += 4+2; if (hpp) { - Uint refc = (Uint) erts_smp_atomic_read(&pb->val->refc); + Uint refc = (Uint) erts_smp_atomic_read_nob(&pb->val->refc); tuple = TUPLE3(*hpp, val, orig_size, make_small(refc)); res = CONS(*hpp + 4, tuple, res); *hpp += 4+2; @@ -554,6 +569,8 @@ static Eterm pi_args[] = { am_suspending, am_min_heap_size, am_min_bin_vheap_size, + am_current_location, + am_current_stacktrace, #ifdef HYBRID am_message_binary #endif @@ -602,8 +619,10 @@ pi_arg2ix(Eterm arg) case am_suspending: return 26; case am_min_heap_size: return 27; case am_min_bin_vheap_size: return 28; + case am_current_location: return 29; + case am_current_stacktrace: return 30; #ifdef HYBRID - case am_message_binary: return 29; + case am_message_binary: return 31; #endif default: return -1; } @@ -1006,35 +1025,15 @@ process_info_aux(Process *BIF_P, break; case am_current_function: - if (rp->current == NULL) { - rp->current = find_function_from_pc(rp->i); - } - if (rp->current == NULL) { - hp = HAlloc(BIF_P, 3); - res = am_undefined; - } else { - BeamInstr* current; - - if (rp->current[0] == am_erlang && - rp->current[1] == am_process_info && - (rp->current[2] == 1 || rp->current[2] == 2) && - (current = find_function_from_pc(rp->cp)) != NULL) { - - /* - * The current function is erlang:process_info/2, - * which is not the answer that the application want. - * We will use the function pointed into by rp->cp - * instead. - */ + res = current_function(BIF_P, rp, &hp, 0); + break; - rp->current = current; - } + case am_current_location: + res = current_function(BIF_P, rp, &hp, 1); + break; - hp = HAlloc(BIF_P, 3+4); - res = TUPLE3(hp, rp->current[0], - rp->current[1], make_small(rp->current[2])); - hp += 4; - } + case am_current_stacktrace: + res = current_stacktrace(BIF_P, rp, &hp); break; case am_initial_call: @@ -1608,6 +1607,113 @@ process_info_aux(Process *BIF_P, } #undef MI_INC +static Eterm +current_function(Process* BIF_P, Process* rp, Eterm** hpp, int full_info) +{ + Eterm* hp; + Eterm res; + FunctionInfo fi; + + if (rp->current == NULL) { + erts_lookup_function_info(&fi, rp->i, full_info); + rp->current = fi.current; + } else if (full_info) { + erts_lookup_function_info(&fi, rp->i, full_info); + if (fi.current == NULL) { + /* Use the current function without location info */ + erts_set_current_function(&fi, rp->current); + } + } + + if (BIF_P->id == rp->id) { + FunctionInfo fi2; + + /* + * The current function is erlang:process_info/{1,2}, + * which is not the answer that the application want. + * We will use the function pointed into by rp->cp + * instead if it can be looked up. + */ + erts_lookup_function_info(&fi2, rp->cp, full_info); + if (fi2.current) { + fi = fi2; + rp->current = fi2.current; + } + } + + /* + * Return the result. + */ + if (rp->current == NULL) { + hp = HAlloc(BIF_P, 3); + res = am_undefined; + } else if (full_info) { + hp = HAlloc(BIF_P, 3+fi.needed); + hp = erts_build_mfa_item(&fi, hp, am_true, &res); + } else { + hp = HAlloc(BIF_P, 3+4); + res = TUPLE3(hp, rp->current[0], + rp->current[1], make_small(rp->current[2])); + hp += 4; + } + *hpp = hp; + return res; +} + +static Eterm +current_stacktrace(Process* p, Process* rp, Eterm** hpp) +{ + Uint sz; + struct StackTrace* s; + int depth; + FunctionInfo* stk; + FunctionInfo* stkp; + Uint heap_size; + int i; + Eterm* hp = *hpp; + Eterm mfa; + Eterm res = NIL; + + depth = 8; + sz = offsetof(struct StackTrace, trace) + sizeof(BeamInstr *)*depth; + s = (struct StackTrace *) erts_alloc(ERTS_ALC_T_TMP, sz); + s->depth = 0; + if (rp->i) { + s->trace[s->depth++] = rp->i; + depth--; + } + if (depth > 0 && rp->cp != 0) { + s->trace[s->depth++] = rp->cp - 1; + depth--; + } + erts_save_stacktrace(rp, s, depth); + + depth = s->depth; + stk = stkp = (FunctionInfo *) erts_alloc(ERTS_ALC_T_TMP, + depth*sizeof(FunctionInfo)); + heap_size = 3; + for (i = 0; i < depth; i++) { + erts_lookup_function_info(stkp, s->trace[i], 1); + if (stkp->current) { + heap_size += stkp->needed + 2; + stkp++; + } + } + + hp = HAlloc(p, heap_size); + while (stkp > stk) { + stkp--; + hp = erts_build_mfa_item(stkp, hp, am_true, &mfa); + res = CONS(hp, mfa, res); + hp += 2; + } + + erts_free(ERTS_ALC_T_TMP, stk); + erts_free(ERTS_ALC_T_TMP, s); + *hpp = hp; + return res; +} + #if defined(VALGRIND) static int check_if_xml(void) { @@ -1633,9 +1739,19 @@ info_1_tuple(Process* BIF_P, /* Pointer to current process. */ sel = *tp++; - if (sel == am_allocator_sizes && arity == 2) { - return erts_allocator_info_term(BIF_P, *tp, 1); - } else if (sel == am_wordsize && arity == 2) { + if (sel == am_allocator_sizes) { + switch (arity) { + case 2: + ERTS_BIF_PREP_TRAP1(ret, alloc_sizes_trap, BIF_P, *tp); + return ret; + case 3: + if (erts_request_alloc_info(BIF_P, tp[0], tp[1], 1)) + return am_true; + default: + goto badarg; + } + } + else if (sel == am_wordsize && arity == 2) { if (tp[0] == am_internal) { return make_small(sizeof(Eterm)); } @@ -1682,8 +1798,17 @@ info_1_tuple(Process* BIF_P, /* Pointer to current process. */ } else goto badarg; - } else if (sel == am_allocator && arity == 2) { - return erts_allocator_info_term(BIF_P, *tp, 0); + } else if (sel == am_allocator) { + switch (arity) { + case 2: + ERTS_BIF_PREP_TRAP1(ret, alloc_info_trap, BIF_P, *tp); + return ret; + case 3: + if (erts_request_alloc_info(BIF_P, tp[0], tp[1], 0)) + return am_true; + default: + goto badarg; + } } else if (ERTS_IS_ATOM_STR("internal_cpu_topology", sel) && arity == 2) { return erts_get_cpu_topology_term(BIF_P, *tp); } else if (ERTS_IS_ATOM_STR("cpu_topology", sel) && arity == 2) { @@ -2005,7 +2130,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) BIF_RET(am_undefined); #endif } else if (BIF_ARG_1 == am_trace_control_word) { - BIF_RET(db_get_trace_control_word_0(BIF_P)); + BIF_RET(db_get_trace_control_word(BIF_P)); } else if (ERTS_IS_ATOM_STR("ets_realloc_moves", BIF_ARG_1)) { BIF_RET((erts_ets_realloc_always_moves) ? am_true : am_false); } else if (ERTS_IS_ATOM_STR("ets_always_compress", BIF_ARG_1)) { @@ -2026,7 +2151,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) res = TUPLE2(hp, am_sequential_tracer, val); BIF_RET(res); } else if (BIF_ARG_1 == am_garbage_collection){ - Uint val = (Uint) erts_smp_atomic32_read(&erts_max_gen_gcs); + Uint val = (Uint) erts_smp_atomic32_read_nob(&erts_max_gen_gcs); Eterm tup; hp = HAlloc(BIF_P, 3+2 + 3+2 + 3+2); @@ -2041,7 +2166,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) BIF_RET(res); } else if (BIF_ARG_1 == am_fullsweep_after){ - Uint val = (Uint) erts_smp_atomic32_read(&erts_max_gen_gcs); + Uint val = (Uint) erts_smp_atomic32_read_nob(&erts_max_gen_gcs); hp = HAlloc(BIF_P, 3); res = TUPLE2(hp, am_fullsweep_after, make_small(val)); BIF_RET(res); @@ -2065,7 +2190,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) /* Need to be the only thread running... */ erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(0); + erts_smp_thr_progress_block(); if (BIF_ARG_1 == am_info) info(ERTS_PRINT_DSBUF, (void *) dsbufp); @@ -2076,7 +2201,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) else distribution_info(ERTS_PRINT_DSBUF, (void *) dsbufp); - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); ASSERT(dsbufp && dsbufp->str); @@ -2088,7 +2213,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) i = 0; /* Need to be the only thread running... */ erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(0); + erts_smp_thr_progress_block(); for (dep = erts_visible_dist_entries; dep; dep = dep->next) ++i; for (dep = erts_hidden_dist_entries; dep; dep = dep->next) @@ -2111,7 +2236,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) res = CONS(hp, tpl, res); hp += 2; } - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); BIF_RET(res); } else if (BIF_ARG_1 == am_system_version) { @@ -2132,16 +2257,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) return erts_instr_get_type_info(BIF_P); } else if (BIF_ARG_1 == am_os_type) { - Eterm type = am_atom_put(os_type, strlen(os_type)); - Eterm flav, tup; - char *buf = erts_alloc(ERTS_ALC_T_TMP, 1024); /* More than enough */ - - os_flavor(buf, 1024); - flav = am_atom_put(buf, strlen(buf)); - hp = HAlloc(BIF_P, 3); - tup = TUPLE2(hp, type, flav); - erts_free(ERTS_ALC_T_TMP, (void *) buf); - BIF_RET(tup); + BIF_RET(os_type_tuple); } else if (BIF_ARG_1 == am_allocator) { BIF_RET(erts_allocator_options((void *) BIF_P)); @@ -2167,16 +2283,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) BIF_RET(am_false); } else if (BIF_ARG_1 == am_os_version) { - int major, minor, build; - Eterm tup; - - os_version(&major, &minor, &build); - hp = HAlloc(BIF_P, 4); - tup = TUPLE3(hp, - make_small(major), - make_small(minor), - make_small(build)); - BIF_RET(tup); + BIF_RET(os_version_tuple); } else if (BIF_ARG_1 == am_version) { int n = strlen(ERLANG_VERSION); @@ -2545,14 +2652,90 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) hp = hsz ? HAlloc(BIF_P, hsz) : NULL; res = erts_bld_uint(&hp, NULL, erts_dist_buf_busy_limit); BIF_RET(res); + } else if (ERTS_IS_ATOM_STR("print_ethread_info", BIF_ARG_1)) { +#if defined(ETHR_NATIVE_ATOMIC32_IMPL) \ + || defined(ETHR_NATIVE_ATOMIC64_IMPL) \ + || defined(ETHR_NATIVE_DW_ATOMIC_IMPL) + int i; + char **str; +#endif +#ifdef ETHR_NATIVE_ATOMIC32_IMPL + erts_printf("32-bit native atomics: %s\n", + ETHR_NATIVE_ATOMIC32_IMPL); + str = ethr_native_atomic32_ops(); + for (i = 0; str[i]; i++) + erts_printf("ethr_native_atomic32_%s()\n", str[i]); +#endif +#ifdef ETHR_NATIVE_ATOMIC64_IMPL + erts_printf("64-bit native atomics: %s\n", + ETHR_NATIVE_ATOMIC64_IMPL); + str = ethr_native_atomic64_ops(); + for (i = 0; str[i]; i++) + erts_printf("ethr_native_atomic64_%s()\n", str[i]); +#endif +#ifdef ETHR_NATIVE_DW_ATOMIC_IMPL + if (ethr_have_native_dw_atomic()) { + erts_printf("Double word native atomics: %s\n", + ETHR_NATIVE_DW_ATOMIC_IMPL); + str = ethr_native_dw_atomic_ops(); + for (i = 0; str[i]; i++) + erts_printf("ethr_native_dw_atomic_%s()\n", str[i]); + str = ethr_native_su_dw_atomic_ops(); + for (i = 0; str[i]; i++) + erts_printf("ethr_native_su_dw_atomic_%s()\n", str[i]); + } +#endif +#ifdef ETHR_NATIVE_SPINLOCK_IMPL + erts_printf("Native spin-locks: %s\n", ETHR_NATIVE_SPINLOCK_IMPL); +#endif +#ifdef ETHR_NATIVE_RWSPINLOCK_IMPL + erts_printf("Native rwspin-locks: %s\n", ETHR_NATIVE_RWSPINLOCK_IMPL); +#endif +#ifdef ETHR_X86_RUNTIME_CONF_HAVE_SSE2__ + erts_printf("SSE2 support: %s\n", (ETHR_X86_RUNTIME_CONF_HAVE_SSE2__ + ? "yes" : "no")); +#endif +#ifdef ETHR_X86_OUT_OF_ORDER + erts_printf("x86" +#ifdef ARCH_64 + "_64" +#endif + " out of order\n"); +#endif +#ifdef ETHR_SPARC_TSO + erts_printf("Sparc TSO\n"); +#endif +#ifdef ETHR_SPARC_PSO + erts_printf("Sparc PSO\n"); +#endif +#ifdef ETHR_SPARC_RMO + erts_printf("Sparc RMO\n"); +#endif +#if defined(ETHR_PPC_HAVE_LWSYNC) + erts_printf("Have lwsync instruction: yes\n"); +#elif defined(ETHR_PPC_HAVE_NO_LWSYNC) + erts_printf("Have lwsync instruction: no\n"); +#elif defined(ETHR_PPC_RUNTIME_CONF_HAVE_LWSYNC__) + erts_printf("Have lwsync instruction: %s (runtime test)\n", + ETHR_PPC_RUNTIME_CONF_HAVE_LWSYNC__ ? "yes" : "no"); +#endif + BIF_RET(am_true); } +#ifdef ERTS_SMP + else if (ERTS_IS_ATOM_STR("thread_progress", BIF_ARG_1)) { + erts_thr_progress_dbg_print_state(); + BIF_RET(am_true); + } +#endif BIF_ERROR(BIF_P, BADARG); } -Eterm -port_info_1(Process* p, Eterm pid) +BIF_RETTYPE +port_info_1(BIF_ALIST_1) { + Process* p = BIF_P; + Eterm pid = BIF_ARG_1; static Eterm keys[] = { am_name, am_links, @@ -2575,7 +2758,7 @@ port_info_1(Process* p, Eterm pid) for (i = 0; i < ASIZE(keys); i++) { Eterm item; - item = port_info_2(p, pid, keys[i]); + item = port_info(p, pid, keys[i]); if (is_non_value(item)) { return THE_NON_VALUE; } @@ -2584,7 +2767,7 @@ port_info_1(Process* p, Eterm pid) } items[i] = item; } - reg_name = port_info_2(p, pid, am_registered_name); + reg_name = port_info(p, pid, am_registered_name); /* * Build the resulting list. @@ -2620,24 +2803,27 @@ port_info_1(Process* p, Eterm pid) BIF_RETTYPE port_info_2(BIF_ALIST_2) { + return port_info(BIF_P, BIF_ARG_1, BIF_ARG_2); +} + +static BIF_RETTYPE port_info(Process* p, Eterm portid, Eterm item) +{ BIF_RETTYPE ret; - Eterm portid = BIF_ARG_1; Port *prt; - Eterm item = BIF_ARG_2; Eterm res; Eterm* hp; int count; if (is_internal_port(portid)) - prt = erts_id2port(portid, BIF_P, ERTS_PROC_LOCK_MAIN); + prt = erts_id2port(portid, p, ERTS_PROC_LOCK_MAIN); else if (is_atom(portid)) - erts_whereis_name(BIF_P, ERTS_PROC_LOCK_MAIN, + erts_whereis_name(p, ERTS_PROC_LOCK_MAIN, portid, NULL, 0, 0, &prt); else if (is_external_port(portid) && external_port_dist_entry(portid) == erts_this_dist_entry) BIF_RET(am_undefined); else { - BIF_ERROR(BIF_P, BADARG); + BIF_ERROR(p, BADARG); } if (!prt) { @@ -2645,7 +2831,7 @@ BIF_RETTYPE port_info_2(BIF_ALIST_2) } if (item == am_id) { - hp = HAlloc(BIF_P, 3); + hp = HAlloc(p, 3); res = make_small(internal_port_number(portid)); } else if (item == am_links) { @@ -2657,10 +2843,10 @@ BIF_RETTYPE port_info_2(BIF_ALIST_2) erts_doforall_links(prt->nlinks, &collect_one_link, &mic); - hp = HAlloc(BIF_P, 3 + mic.sz); + hp = HAlloc(p, 3 + mic.sz); res = NIL; for (i = 0; i < mic.mi_i; i++) { - item = STORE_NC(&hp, &MSO(BIF_P), mic.mi[i].entity); + item = STORE_NC(&hp, &MSO(p), mic.mi[i].entity); res = CONS(hp, item, res); hp += 2; } @@ -2676,11 +2862,11 @@ BIF_RETTYPE port_info_2(BIF_ALIST_2) erts_doforall_monitors(prt->monitors, &collect_one_origin_monitor, &mic); - hp = HAlloc(BIF_P, 3 + mic.sz); + hp = HAlloc(p, 3 + mic.sz); res = NIL; for (i = 0; i < mic.mi_i; i++) { Eterm t; - item = STORE_NC(&hp, &MSO(BIF_P), mic.mi[i].entity); + item = STORE_NC(&hp, &MSO(p), mic.mi[i].entity); t = TUPLE2(hp, am_process, item); hp += 3; res = CONS(hp, t, res); @@ -2692,25 +2878,25 @@ BIF_RETTYPE port_info_2(BIF_ALIST_2) else if (item == am_name) { count = sys_strlen(prt->name); - hp = HAlloc(BIF_P, 3 + 2*count); + hp = HAlloc(p, 3 + 2*count); res = buf_to_intlist(&hp, prt->name, count, NIL); } else if (item == am_connected) { - hp = HAlloc(BIF_P, 3); + hp = HAlloc(p, 3); res = prt->connected; /* internal pid */ } else if (item == am_input) { Uint hsz = 3; Uint n = prt->bytes_in; (void) erts_bld_uint(NULL, &hsz, n); - hp = HAlloc(BIF_P, hsz); + hp = HAlloc(p, hsz); res = erts_bld_uint(&hp, NULL, n); } else if (item == am_output) { Uint hsz = 3; Uint n = prt->bytes_out; (void) erts_bld_uint(NULL, &hsz, n); - hp = HAlloc(BIF_P, hsz); + hp = HAlloc(p, hsz); res = erts_bld_uint(&hp, NULL, n); } else if (item == am_registered_name) { @@ -2720,7 +2906,7 @@ BIF_RETTYPE port_info_2(BIF_ALIST_2) ERTS_BIF_PREP_RET(ret, NIL); goto done; } else { - hp = HAlloc(BIF_P, 3); + hp = HAlloc(p, 3); res = reg->name; } } @@ -2732,7 +2918,7 @@ BIF_RETTYPE port_info_2(BIF_ALIST_2) Uint size = 0; ErlHeapFragment* bp; - hp = HAlloc(BIF_P, 3); + hp = HAlloc(p, 3); erts_doforall_links(prt->nlinks, &one_link_size, &size); @@ -2749,18 +2935,18 @@ BIF_RETTYPE port_info_2(BIF_ALIST_2) hard to retrieve... */ (void) erts_bld_uint(NULL, &hsz, size); - hp = HAlloc(BIF_P, hsz); + hp = HAlloc(p, hsz); res = erts_bld_uint(&hp, NULL, size); } else if (item == am_queue_size) { Uint ioq_size = erts_port_ioq_size(prt); Uint hsz = 3; (void) erts_bld_uint(NULL, &hsz, ioq_size); - hp = HAlloc(BIF_P, hsz); + hp = HAlloc(p, hsz); res = erts_bld_uint(&hp, NULL, ioq_size); } else if (ERTS_IS_ATOM_STR("locking", item)) { - hp = HAlloc(BIF_P, 3); + hp = HAlloc(p, 3); #ifndef ERTS_SMP res = am_false; #else @@ -2779,7 +2965,7 @@ BIF_RETTYPE port_info_2(BIF_ALIST_2) #endif } else { - ERTS_BIF_PREP_ERROR(ret, BIF_P, BADARG); + ERTS_BIF_PREP_ERROR(ret, p, BADARG); goto done; } @@ -2793,9 +2979,12 @@ BIF_RETTYPE port_info_2(BIF_ALIST_2) } -Eterm -fun_info_2(Process* p, Eterm fun, Eterm what) +BIF_RETTYPE +fun_info_2(BIF_ALIST_2) { + Process* p = BIF_P; + Eterm fun = BIF_ARG_1; + Eterm what = BIF_ARG_2; Eterm* hp; Eterm val; @@ -2845,7 +3034,7 @@ fun_info_2(Process* p, Eterm fun, Eterm what) } break; case am_refc: - val = erts_make_integer(erts_smp_atomic_read(&funp->fe->refc), p); + val = erts_make_integer(erts_smp_atomic_read_nob(&funp->fe->refc), p); hp = HAlloc(p, 3); break; case am_arity: @@ -3065,8 +3254,8 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1) Eterm r1, r2; Eterm in, out; Uint hsz = 9; - Uint bytes_in = (Uint) erts_smp_atomic_read(&erts_bytes_in); - Uint bytes_out = (Uint) erts_smp_atomic_read(&erts_bytes_out); + Uint bytes_in = (Uint) erts_smp_atomic_read_nob(&erts_bytes_in); + Uint bytes_out = (Uint) erts_smp_atomic_read_nob(&erts_bytes_out); (void) erts_bld_uint(NULL, &hsz, bytes_in); (void) erts_bld_uint(NULL, &hsz, bytes_out); @@ -3106,26 +3295,6 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1) BIF_ERROR(BIF_P, BADARG); } -BIF_RETTYPE memory_0(BIF_ALIST_0) -{ - BIF_RETTYPE res = erts_memory(NULL, NULL, BIF_P, THE_NON_VALUE); - switch (res) { - case am_badarg: BIF_ERROR(BIF_P, EXC_INTERNAL_ERROR); /* never... */ - case am_notsup: BIF_ERROR(BIF_P, EXC_NOTSUP); - default: BIF_RET(res); - } -} - -BIF_RETTYPE memory_1(BIF_ALIST_1) -{ - BIF_RETTYPE res = erts_memory(NULL, NULL, BIF_P, BIF_ARG_1); - switch (res) { - case am_badarg: BIF_ERROR(BIF_P, BADARG); - case am_notsup: BIF_ERROR(BIF_P, EXC_NOTSUP); - default: BIF_RET(res); - } -} - BIF_RETTYPE error_logger_warning_map_0(BIF_ALIST_0) { BIF_RET(erts_error_logger_warnings); @@ -3139,7 +3308,7 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) * NOTE: Only supposed to be used for testing, and debugging. */ - if (!erts_smp_atomic_read(&available_internal_state)) { + if (!erts_smp_atomic_read_nob(&available_internal_state)) { BIF_ERROR(BIF_P, EXC_UNDEF); } @@ -3227,6 +3396,15 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) BIF_RET(am_false); #endif } + else if (ERTS_IS_ATOM_STR("memory", BIF_ARG_1)) { + Eterm res; + erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_smp_thr_progress_block(); + erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + res = erts_memory(NULL, NULL, BIF_P, THE_NON_VALUE); + erts_smp_thr_progress_unblock(); + BIF_RET(res); + } } else if (is_tuple(BIF_ARG_1)) { Eterm* tp = tuple_val(BIF_ARG_1); @@ -3429,6 +3607,7 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) static erts_smp_atomic_t hipe_test_reschedule_flag; + BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) { /* @@ -3437,7 +3616,7 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) if (ERTS_IS_ATOM_STR("available_internal_state", BIF_ARG_1) && (BIF_ARG_2 == am_true || BIF_ARG_2 == am_false)) { erts_aint_t on = (erts_aint_t) (BIF_ARG_2 == am_true); - erts_aint_t prev_on = erts_smp_atomic_xchg(&available_internal_state, on); + erts_aint_t prev_on = erts_smp_atomic_xchg_nob(&available_internal_state, on); if (on) { erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); erts_dsprintf(dsbufp, "Process %T ", BIF_P->id); @@ -3453,7 +3632,7 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) BIF_RET(prev_on ? am_true : am_false); } - if (!erts_smp_atomic_read(&available_internal_state)) { + if (!erts_smp_atomic_read_nob(&available_internal_state)) { BIF_ERROR(BIF_P, EXC_UNDEF); } @@ -3479,10 +3658,10 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) if (ms > 0) { erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); if (block) - erts_smp_block_system(0); + erts_smp_thr_progress_block(); while (erts_milli_sleep((long) ms) != 0); if (block) - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); } BIF_RET(am_true); @@ -3634,14 +3813,14 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) } else if (ERTS_IS_ATOM_STR("hipe_test_reschedule_suspend", BIF_ARG_1)) { /* Used by hipe test suites */ - erts_aint_t flag = erts_smp_atomic_read(&hipe_test_reschedule_flag); + erts_aint_t flag = erts_smp_atomic_read_nob(&hipe_test_reschedule_flag); if (!flag && BIF_ARG_2 != am_false) { - erts_smp_atomic_set(&hipe_test_reschedule_flag, 1); + erts_smp_atomic_set_nob(&hipe_test_reschedule_flag, 1); erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, NULL); ERTS_BIF_YIELD2(bif_export[BIF_erts_debug_set_internal_state_2], BIF_P, BIF_ARG_1, BIF_ARG_2); } - erts_smp_atomic_set(&hipe_test_reschedule_flag, !flag); + erts_smp_atomic_set_nob(&hipe_test_reschedule_flag, !flag); BIF_RET(NIL); } else if (ERTS_IS_ATOM_STR("hipe_test_reschedule_resume", BIF_ARG_1)) { @@ -3692,16 +3871,23 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) } erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(0); + erts_smp_thr_progress_block(); old_use_opt = !erts_disable_proc_not_running_opt; erts_disable_proc_not_running_opt = !use_opt; - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); BIF_RET(old_use_opt ? am_true : am_false); #else BIF_ERROR(BIF_P, EXC_NOTSUP); #endif } + else if (ERTS_IS_ATOM_STR("wait", BIF_ARG_1)) { + if (ERTS_IS_ATOM_STR("deallocations", BIF_ARG_2)) { + if (erts_debug_wait_deallocations(BIF_P)) { + ERTS_BIF_YIELD_RETURN(BIF_P, am_ok); + } + } + } } BIF_ERROR(BIF_P, BADARG); @@ -3860,7 +4046,7 @@ BIF_RETTYPE erts_debug_lock_counters_1(BIF_ALIST_1) Eterm* hp; erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(0); + erts_smp_thr_progress_block(); erts_lcnt_set_rt_opt(ERTS_LCNT_OPT_SUSPEND); data = erts_lcnt_get_data(); @@ -3878,17 +4064,17 @@ BIF_RETTYPE erts_debug_lock_counters_1(BIF_ALIST_1) erts_lcnt_clear_rt_opt(ERTS_LCNT_OPT_SUSPEND); - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); BIF_RET(res); } else if (BIF_ARG_1 == am_clear) { erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(0); + erts_smp_thr_progress_block(); erts_lcnt_clear_counters(); - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); BIF_RET(am_ok); @@ -3899,7 +4085,7 @@ BIF_RETTYPE erts_debug_lock_counters_1(BIF_ALIST_1) case 2: if (ERTS_IS_ATOM_STR("copy_save", tp[1])) { erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(0); + erts_smp_thr_progress_block(); if (tp[2] == am_true) { res = erts_lcnt_set_rt_opt(ERTS_LCNT_OPT_COPYSAVE) ? am_true : am_false; @@ -3909,17 +4095,17 @@ BIF_RETTYPE erts_debug_lock_counters_1(BIF_ALIST_1) res = erts_lcnt_clear_rt_opt(ERTS_LCNT_OPT_COPYSAVE) ? am_true : am_false; } else { - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); BIF_ERROR(BIF_P, BADARG); } - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); BIF_RET(res); } else if (ERTS_IS_ATOM_STR("process_locks", tp[1])) { erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(0); + erts_smp_thr_progress_block(); if (tp[2] == am_true) { res = erts_lcnt_set_rt_opt(ERTS_LCNT_OPT_PROCLOCK) ? am_true : am_false; @@ -3929,11 +4115,11 @@ BIF_RETTYPE erts_debug_lock_counters_1(BIF_ALIST_1) res = erts_lcnt_set_rt_opt(ERTS_LCNT_OPT_PROCLOCK) ? am_true : am_false; } else { - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); BIF_ERROR(BIF_P, BADARG); } - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); BIF_RET(res); } @@ -3948,11 +4134,35 @@ BIF_RETTYPE erts_debug_lock_counters_1(BIF_ALIST_1) BIF_ERROR(BIF_P, BADARG); } +static void os_info_init(void) +{ + Eterm type = am_atom_put(os_type, strlen(os_type)); + Eterm flav; + int major, minor, build; + char* buf = erts_alloc(ERTS_ALC_T_TMP, 1024); /* More than enough */ + Eterm* hp; + + os_flavor(buf, 1024); + flav = am_atom_put(buf, strlen(buf)); + erts_free(ERTS_ALC_T_TMP, (void *) buf); + hp = erts_alloc(ERTS_ALC_T_LL_TEMP_TERM, (3+4)*sizeof(Eterm)); + os_type_tuple = TUPLE2(hp, type, flav); + hp += 3; + os_version(&major, &minor, &build); + os_version_tuple = TUPLE3(hp, + make_small(major), + make_small(minor), + make_small(build)); +} + void erts_bif_info_init(void) { - erts_smp_atomic_init(&available_internal_state, 0); - erts_smp_atomic_init(&hipe_test_reschedule_flag, 0); + erts_smp_atomic_init_nob(&available_internal_state, 0); + erts_smp_atomic_init_nob(&hipe_test_reschedule_flag, 0); + alloc_info_trap = erts_export_put(am_erlang, am_alloc_info, 1); + alloc_sizes_trap = erts_export_put(am_erlang, am_alloc_sizes, 1); process_info_init(); + os_info_init(); } diff --git a/erts/emulator/beam/erl_bif_lists.c b/erts/emulator/beam/erl_bif_lists.c index 47c48e74d6..1805366cfe 100644 --- a/erts/emulator/beam/erl_bif_lists.c +++ b/erts/emulator/beam/erl_bif_lists.c @@ -34,27 +34,7 @@ static Eterm keyfind(int Bif, Process* p, Eterm Key, Eterm Pos, Eterm List); -/* - * erlang:'++'/2 - */ - -Eterm -ebif_plusplus_2(Process* p, Eterm A, Eterm B) -{ - return append_2(p, A, B); -} - -/* - * erlang:'--'/2 - */ - -Eterm -ebif_minusminus_2(Process* p, Eterm A, Eterm B) -{ - return subtract_2(p, A, B); -} - -BIF_RETTYPE append_2(BIF_ALIST_2) +static BIF_RETTYPE append(Process* p, Eterm A, Eterm B) { Eterm list; Eterm copy; @@ -63,18 +43,18 @@ BIF_RETTYPE append_2(BIF_ALIST_2) Eterm* hp; int i; - if ((i = list_length(BIF_ARG_1)) < 0) { - BIF_ERROR(BIF_P, BADARG); + if ((i = list_length(A)) < 0) { + BIF_ERROR(p, BADARG); } if (i == 0) { - BIF_RET(BIF_ARG_2); - } else if (is_nil(BIF_ARG_2)) { - BIF_RET(BIF_ARG_1); + BIF_RET(B); + } else if (is_nil(B)) { + BIF_RET(A); } need = 2*i; - hp = HAlloc(BIF_P, need); - list = BIF_ARG_1; + hp = HAlloc(p, need); + list = A; copy = last = CONS(hp, CAR(list_val(list)), make_list(hp+2)); list = CDR(list_val(list)); hp += 2; @@ -85,12 +65,31 @@ BIF_RETTYPE append_2(BIF_ALIST_2) list = CDR(listp); hp += 2; } - CDR(list_val(last)) = BIF_ARG_2; + CDR(list_val(last)) = B; BIF_RET(copy); } +/* + * erlang:'++'/2 + */ + +Eterm +ebif_plusplus_2(BIF_ALIST_2) +{ + return append(BIF_P, BIF_ARG_1, BIF_ARG_2); +} + +BIF_RETTYPE append_2(BIF_ALIST_2) +{ + return append(BIF_P, BIF_ARG_1, BIF_ARG_2); +} + +/* + * erlang:'--'/2 + */ + #define SMALL_VEC_SIZE 10 -BIF_RETTYPE subtract_2(BIF_ALIST_2) +static Eterm subtract(Process* p, Eterm A, Eterm B) { Eterm list; Eterm* hp; @@ -103,17 +102,17 @@ BIF_RETTYPE subtract_2(BIF_ALIST_2) int n; int m; - if ((n = list_length(BIF_ARG_1)) < 0) { - BIF_ERROR(BIF_P, BADARG); + if ((n = list_length(A)) < 0) { + BIF_ERROR(p, BADARG); } - if ((m = list_length(BIF_ARG_2)) < 0) { - BIF_ERROR(BIF_P, BADARG); + if ((m = list_length(B)) < 0) { + BIF_ERROR(p, BADARG); } if (n == 0) BIF_RET(NIL); if (m == 0) - BIF_RET(BIF_ARG_1); + BIF_RET(A); /* allocate element vector */ if (n <= SMALL_VEC_SIZE) @@ -123,7 +122,7 @@ BIF_RETTYPE subtract_2(BIF_ALIST_2) /* PUT ALL ELEMENTS IN VP */ vp = vec_p; - list = BIF_ARG_1; + list = A; i = n; while(i--) { Eterm* listp = list_val(list); @@ -132,7 +131,7 @@ BIF_RETTYPE subtract_2(BIF_ALIST_2) } /* UNMARK ALL DELETED CELLS */ - list = BIF_ARG_2; + list = B; m = 0; /* number of deleted elements */ while(is_list(list)) { Eterm* listp = list_val(list); @@ -153,11 +152,11 @@ BIF_RETTYPE subtract_2(BIF_ALIST_2) if (m == n) /* All deleted ? */ res = NIL; else if (m == 0) /* None deleted ? */ - res = BIF_ARG_1; + res = A; else { /* REBUILD LIST */ res = NIL; need = 2*(n - m); - hp = HAlloc(BIF_P, need); + hp = HAlloc(p, need); vp = vec_p + n - 1; while(vp >= vec_p) { if (is_value(*vp)) { @@ -172,6 +171,16 @@ BIF_RETTYPE subtract_2(BIF_ALIST_2) BIF_RET(res); } +BIF_RETTYPE ebif_minusminus_2(BIF_ALIST_2) +{ + return subtract(BIF_P, BIF_ARG_1, BIF_ARG_2); +} + +BIF_RETTYPE subtract_2(BIF_ALIST_2) +{ + return subtract(BIF_P, BIF_ARG_1, BIF_ARG_2); +} + BIF_RETTYPE lists_member_2(BIF_ALIST_2) { Eterm term; @@ -278,11 +287,12 @@ BIF_RETTYPE lists_reverse_2(BIF_ALIST_2) } BIF_RETTYPE -lists_keymember_3(Process* p, Eterm Key, Eterm Pos, Eterm List) +lists_keymember_3(BIF_ALIST_3) { Eterm res; - res = keyfind(BIF_lists_keymember_3, p, Key, Pos, List); + res = keyfind(BIF_lists_keymember_3, BIF_P, + BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); if (is_value(res) && is_tuple(res)) { return am_true; } else { @@ -291,23 +301,25 @@ lists_keymember_3(Process* p, Eterm Key, Eterm Pos, Eterm List) } BIF_RETTYPE -lists_keysearch_3(Process* p, Eterm Key, Eterm Pos, Eterm List) +lists_keysearch_3(BIF_ALIST_3) { Eterm res; - res = keyfind(BIF_lists_keysearch_3, p, Key, Pos, List); + res = keyfind(BIF_lists_keysearch_3, BIF_P, + BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); if (is_non_value(res) || is_not_tuple(res)) { return res; } else { /* Tuple */ - Eterm* hp = HAlloc(p, 3); + Eterm* hp = HAlloc(BIF_P, 3); return TUPLE2(hp, am_value, res); } } BIF_RETTYPE -lists_keyfind_3(Process* p, Eterm Key, Eterm Pos, Eterm List) +lists_keyfind_3(BIF_ALIST_3) { - return keyfind(BIF_lists_keyfind_3, p, Key, Pos, List); + return keyfind(BIF_lists_keyfind_3, BIF_P, + BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); } static Eterm diff --git a/erts/emulator/beam/erl_bif_op.c b/erts/emulator/beam/erl_bif_op.c index deda7adc1f..13f8b1f63c 100644 --- a/erts/emulator/beam/erl_bif_op.c +++ b/erts/emulator/beam/erl_bif_op.c @@ -225,18 +225,23 @@ BIF_RETTYPE is_function_1(BIF_ALIST_1) BIF_RETTYPE is_function_2(BIF_ALIST_2) { + BIF_RET(erl_is_function(BIF_P, BIF_ARG_1, BIF_ARG_2)); +} + +Eterm erl_is_function(Process* p, Eterm arg1, Eterm arg2) +{ Sint arity; /* * Verify argument 2 (arity); arity must be >= 0. */ - if (is_small(BIF_ARG_2)) { - arity = signed_val(BIF_ARG_2); + if (is_small(arg2)) { + arity = signed_val(arg2); if (arity < 0) { error: - BIF_ERROR(BIF_P, BADARG); + BIF_ERROR(p, BADARG); } - } else if (is_big(BIF_ARG_2) && !bignum_header_is_neg(*big_val(BIF_ARG_2))) { + } else if (is_big(arg2) && !bignum_header_is_neg(*big_val(arg2))) { /* A positive bignum is OK, but can't possibly match. */ arity = -1; } else { @@ -244,20 +249,20 @@ BIF_RETTYPE is_function_2(BIF_ALIST_2) goto error; } - if (is_fun(BIF_ARG_1)) { - ErlFunThing* funp = (ErlFunThing *) fun_val(BIF_ARG_1); + if (is_fun(arg1)) { + ErlFunThing* funp = (ErlFunThing *) fun_val(arg1); if (funp->arity == (Uint) arity) { BIF_RET(am_true); } - } else if (is_export(BIF_ARG_1)) { - Export* exp = (Export *) EXPAND_POINTER((export_val(BIF_ARG_1))[1]); + } else if (is_export(arg1)) { + Export* exp = (Export *) EXPAND_POINTER((export_val(arg1))[1]); if (exp->code[2] == (Uint) arity) { BIF_RET(am_true); } - } else if (is_tuple(BIF_ARG_1)) { - Eterm* tp = tuple_val(BIF_ARG_1); + } else if (is_tuple(arg1)) { + Eterm* tp = tuple_val(arg1); if (tp[0] == make_arityval(2) && is_atom(tp[1]) && is_atom(tp[2])) { BIF_RET(am_true); } diff --git a/erts/emulator/beam/erl_bif_os.c b/erts/emulator/beam/erl_bif_os.c index 954b1f9729..58d48199fa 100644 --- a/erts/emulator/beam/erl_bif_os.c +++ b/erts/emulator/beam/erl_bif_os.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2009. All Rights Reserved. + * Copyright Ericsson AB 1999-2010. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -53,20 +53,18 @@ BIF_RETTYPE os_timestamp_0(BIF_ALIST_0) } -Eterm -os_getpid_0(Process* p) +BIF_RETTYPE os_getpid_0(BIF_ALIST_0) { char pid_string[21]; /* enough for a 64 bit number */ int n; Eterm* hp; sys_get_pid(pid_string); /* In sys.c */ n = sys_strlen(pid_string); - hp = HAlloc(p, n*2); + hp = HAlloc(BIF_P, n*2); BIF_RET(buf_to_intlist(&hp, pid_string, n, NIL)); } -Eterm -os_getenv_0(Process* p) +BIF_RETTYPE os_getenv_0(BIF_ALIST_0) { GETENV_STATE state; char *cp; @@ -80,7 +78,7 @@ os_getenv_0(Process* p) ret = NIL; while ((cp = getenv_string(&state)) != NULL) { len = strlen(cp); - hp = HAlloc(p, len*2+2); + hp = HAlloc(BIF_P, len*2+2); str = buf_to_intlist(&hp, cp, len, NIL); ret = CONS(hp, str, ret); } @@ -90,9 +88,11 @@ os_getenv_0(Process* p) return ret; } -Eterm -os_getenv_1(Process* p, Eterm key) + +BIF_RETTYPE os_getenv_1(BIF_ALIST_1) { + Process* p = BIF_P; + Eterm key = BIF_ARG_1; Eterm str; int len, res; char *key_str, *val; @@ -145,9 +145,11 @@ os_getenv_1(Process* p, Eterm key) BIF_RET(str); } -Eterm -os_putenv_2(Process* p, Eterm key, Eterm value) +BIF_RETTYPE os_putenv_2(BIF_ALIST_2) { + Process* p = BIF_P; + Eterm key = BIF_ARG_1; + Eterm value = BIF_ARG_2; char def_buf[1024]; char *buf = NULL; int sep_ix, i, key_len, value_len, tot_len; diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c index 3fd35dd963..b21cda6347 100644 --- a/erts/emulator/beam/erl_bif_port.c +++ b/erts/emulator/beam/erl_bif_port.c @@ -48,6 +48,9 @@ static void free_args(char **); char *erts_default_arg0 = "default"; +static BIF_RETTYPE +port_call(Process* p, Eterm arg1, Eterm arg2, Eterm arg3); + BIF_RETTYPE open_port_2(BIF_ALIST_2) { int port_num; @@ -117,11 +120,9 @@ id_or_name2port(Process *c_p, Eterm id) #define ERTS_PORT_COMMAND_FLAG_FORCE (((Uint32) 1) << 0) #define ERTS_PORT_COMMAND_FLAG_NOSUSPEND (((Uint32) 1) << 1) -static BIF_RETTYPE do_port_command(Process *BIF_P, - Eterm BIF_ARG_1, - Eterm BIF_ARG_2, - Eterm BIF_ARG_3, - Uint32 flags) +static BIF_RETTYPE +do_port_command(Process *BIF_P, Eterm arg1, Eterm arg2, Eterm arg3, + Uint32 flags) { BIF_RETTYPE res; Port *p; @@ -135,7 +136,7 @@ static BIF_RETTYPE do_port_command(Process *BIF_P, profile_runnable_proc(BIF_P, am_inactive); } - p = id_or_name2port(BIF_P, BIF_ARG_1); + p = id_or_name2port(BIF_P, arg1); if (!p) { if (IS_TRACED_FL(BIF_P, F_TRACE_SCHED_PROCS)) { trace_virtual_sched(BIF_P, am_in); @@ -172,13 +173,13 @@ static BIF_RETTYPE do_port_command(Process *BIF_P, monitor_generic(BIF_P, am_busy_port, p->id); } ERTS_BIF_PREP_YIELD3(res, bif_export[BIF_port_command_3], BIF_P, - BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); + arg1, arg2, arg3); } } else { int wres; erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); ERTS_SMP_CHK_NO_PROC_LOCKS; - wres = erts_write_to_port(BIF_P->id, p, BIF_ARG_2); + wres = erts_write_to_port(BIF_P->id, p, arg2); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); if (wres != 0) { ERTS_BIF_PREP_ERROR(res, BIF_P, BADARG); @@ -237,11 +238,17 @@ BIF_RETTYPE port_command_3(BIF_ALIST_3) BIF_RETTYPE port_call_2(BIF_ALIST_2) { - return port_call_3(BIF_P,BIF_ARG_1,make_small(0),BIF_ARG_2); + return port_call(BIF_P,BIF_ARG_1, make_small(0), BIF_ARG_2); } BIF_RETTYPE port_call_3(BIF_ALIST_3) { + return port_call(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); +} + +static BIF_RETTYPE +port_call(Process* c_p, Eterm arg1, Eterm arg2, Eterm arg3) +{ Uint op; Port *p; Uint size; @@ -266,15 +273,15 @@ BIF_RETTYPE port_call_3(BIF_ALIST_3) /* trace of port scheduling with virtual process descheduling * lock wait */ - if (IS_TRACED_FL(BIF_P, F_TRACE_SCHED_PROCS)) { - trace_virtual_sched(BIF_P, am_out); + if (IS_TRACED_FL(c_p, F_TRACE_SCHED_PROCS)) { + trace_virtual_sched(c_p, am_out); } if (erts_system_profile_flags.runnable_procs && erts_system_profile_flags.exclusive) { - profile_runnable_proc(BIF_P, am_inactive); + profile_runnable_proc(c_p, am_inactive); } - p = id_or_name2port(BIF_P, BIF_ARG_1); + p = id_or_name2port(c_p, arg1); if (!p) { error: if (port_resp != port_result && @@ -286,22 +293,22 @@ BIF_RETTYPE port_call_3(BIF_ALIST_3) /* Need to virtual schedule in the process if there * was an error. */ - if (IS_TRACED_FL(BIF_P, F_TRACE_SCHED_PROCS)) { - trace_virtual_sched(BIF_P, am_in); + if (IS_TRACED_FL(c_p, F_TRACE_SCHED_PROCS)) { + trace_virtual_sched(c_p, am_in); } if (erts_system_profile_flags.runnable_procs && erts_system_profile_flags.exclusive) { - profile_runnable_proc(BIF_P, am_active); + profile_runnable_proc(c_p, am_active); } if (p) erts_port_release(p); #ifdef ERTS_SMP - ERTS_SMP_BIF_CHK_PENDING_EXIT(BIF_P, ERTS_PROC_LOCK_MAIN); + ERTS_SMP_BIF_CHK_PENDING_EXIT(c_p, ERTS_PROC_LOCK_MAIN); #else - ERTS_BIF_CHK_EXITED(BIF_P); + ERTS_BIF_CHK_EXITED(c_p); #endif - BIF_ERROR(BIF_P, BADARG); + BIF_ERROR(c_p, BADARG); } if ((drv = p->drv_ptr) == NULL) { @@ -310,10 +317,10 @@ BIF_RETTYPE port_call_3(BIF_ALIST_3) if (drv->call == NULL) { goto error; } - if (!term_to_Uint(BIF_ARG_2, &op)) { + if (!term_to_Uint(arg2, &op)) { goto error; } - p->caller = BIF_P->id; + p->caller = c_p->id; /* Lock taken, virtual schedule of port */ if (IS_TRACED_FL(p, F_TRACE_SCHED_PORTS)) { @@ -323,19 +330,19 @@ BIF_RETTYPE port_call_3(BIF_ALIST_3) if (erts_system_profile_flags.runnable_ports && !erts_port_is_scheduled(p)) { profile_runnable_port(p, am_active); } - size = erts_encode_ext_size(BIF_ARG_3); + size = erts_encode_ext_size(arg3); if (size > sizeof(port_input)) bytes = erts_alloc(ERTS_ALC_T_PORT_CALL_BUF, size); endp = bytes; - erts_encode_ext(BIF_ARG_3, &endp); + erts_encode_ext(arg3, &endp); real_size = endp - bytes; if (real_size > size) { erl_exit(1, "%s, line %d: buffer overflow: %d word(s)\n", __FILE__, __LINE__, endp - (bytes + size)); } - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); prc = (char *) port_resp; fpe_was_unmasked = erts_block_fpe(); ret = drv->call((ErlDrvData)p->drv_data, @@ -356,7 +363,7 @@ BIF_RETTYPE port_call_3(BIF_ALIST_3) port_resp = (byte *) prc; p->caller = NIL; - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); #ifdef HARDDEBUG { int z; @@ -382,14 +389,14 @@ BIF_RETTYPE port_call_3(BIF_ALIST_3) if (result_size < 0) { goto error; } - hp = HAlloc(BIF_P, result_size); + hp = HAlloc(c_p, result_size); hp_end = hp + result_size; endp = port_resp; - res = erts_decode_ext(&hp, &MSO(BIF_P), &endp); + res = erts_decode_ext(&hp, &MSO(c_p), &endp); if (res == THE_NON_VALUE) { goto error; } - HRelease(BIF_P, hp_end, hp); + HRelease(c_p, hp_end, hp); if (port_resp != port_result && !(ret_flags & DRIVER_CALL_KEEP_BUFFER)) { driver_free(port_resp); } @@ -398,16 +405,16 @@ BIF_RETTYPE port_call_3(BIF_ALIST_3) if (p) erts_port_release(p); #ifdef ERTS_SMP - ERTS_SMP_BIF_CHK_PENDING_EXIT(BIF_P, ERTS_PROC_LOCK_MAIN); + ERTS_SMP_BIF_CHK_PENDING_EXIT(c_p, ERTS_PROC_LOCK_MAIN); #else - ERTS_BIF_CHK_EXITED(BIF_P); + ERTS_BIF_CHK_EXITED(c_p); #endif - if (IS_TRACED_FL(BIF_P, F_TRACE_SCHED_PROCS)) { - trace_virtual_sched(BIF_P, am_in); + if (IS_TRACED_FL(c_p, F_TRACE_SCHED_PROCS)) { + trace_virtual_sched(c_p, am_in); } if (erts_system_profile_flags.runnable_procs && erts_system_profile_flags.exclusive) { - profile_runnable_proc(BIF_P, am_active); + profile_runnable_proc(c_p, am_active); } return res; diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c index 26891c4348..6b843d2e08 100644 --- a/erts/emulator/beam/erl_bif_re.c +++ b/erts/emulator/beam/erl_bif_re.c @@ -45,6 +45,7 @@ static Export *urun_trap_exportp = NULL; static Export *ucompile_trap_exportp = NULL; static BIF_RETTYPE re_exec_trap(BIF_ALIST_3); +static BIF_RETTYPE re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3); static void *erts_erts_pcre_malloc(size_t size) { return erts_alloc(ERTS_ALC_T_RE_HEAP,size); @@ -414,8 +415,8 @@ build_compile_result(Process *p, Eterm error_tag, pcre *result, int errcode, con * Compile BIFs */ -BIF_RETTYPE -re_compile_2(BIF_ALIST_2) +static BIF_RETTYPE +re_compile(Process* p, Eterm arg1, Eterm arg2) { Uint slen; char *expr; @@ -429,43 +430,49 @@ re_compile_2(BIF_ALIST_2) int unicode = 0; - if (parse_options(BIF_ARG_2,&options,NULL,&pflags,NULL,NULL) + if (parse_options(arg2,&options,NULL,&pflags,NULL,NULL) < 0) { - BIF_ERROR(BIF_P,BADARG); + BIF_ERROR(p,BADARG); } if (pflags & PARSE_FLAG_UNIQUE_EXEC_OPT) { - BIF_ERROR(BIF_P,BADARG); + BIF_ERROR(p,BADARG); } unicode = (pflags & PARSE_FLAG_UNICODE) ? 1 : 0; - if (pflags & PARSE_FLAG_UNICODE && !is_binary(BIF_ARG_1)) { - BIF_TRAP2(ucompile_trap_exportp, BIF_P, BIF_ARG_1, BIF_ARG_2); + if (pflags & PARSE_FLAG_UNICODE && !is_binary(arg1)) { + BIF_TRAP2(ucompile_trap_exportp, p, arg1, arg2); } - if (erts_iolist_size(BIF_ARG_1, &slen)) { - BIF_ERROR(BIF_P,BADARG); + if (erts_iolist_size(arg1, &slen)) { + BIF_ERROR(p,BADARG); } expr = erts_alloc(ERTS_ALC_T_RE_TMP_BUF, slen + 1); - if (io_list_to_buf(BIF_ARG_1, expr, slen) != 0) { + if (io_list_to_buf(arg1, expr, slen) != 0) { erts_free(ERTS_ALC_T_RE_TMP_BUF, expr); - BIF_ERROR(BIF_P,BADARG); + BIF_ERROR(p,BADARG); } expr[slen]='\0'; result = erts_pcre_compile2(expr, options, &errcode, &errstr, &errofset, default_table); - ret = build_compile_result(BIF_P, am_error, result, errcode, + ret = build_compile_result(p, am_error, result, errcode, errstr, errofset, unicode, 1); erts_free(ERTS_ALC_T_RE_TMP_BUF, expr); BIF_RET(ret); } BIF_RETTYPE +re_compile_2(BIF_ALIST_2) +{ + return re_compile(BIF_P, BIF_ARG_1, BIF_ARG_2); +} + +BIF_RETTYPE re_compile_1(BIF_ALIST_1) { - return re_compile_2(BIF_P,BIF_ARG_1,NIL); + return re_compile(BIF_P, BIF_ARG_1, NIL); } /* @@ -845,8 +852,8 @@ build_capture(Eterm capture_spec[CAPSPEC_SIZE], const pcre *code) /* * The actual re:run/2,3 BIFs */ -BIF_RETTYPE -re_run_3(BIF_ALIST_3) +static BIF_RETTYPE +re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3) { const pcre *code_tmp; RestartContext restart; @@ -865,15 +872,15 @@ re_run_3(BIF_ALIST_3) Eterm capture[CAPSPEC_SIZE]; int is_list_cap; - if (parse_options(BIF_ARG_3,&comp_options,&options,&pflags,&startoffset,capture) + if (parse_options(arg3,&comp_options,&options,&pflags,&startoffset,capture) < 0) { - BIF_ERROR(BIF_P,BADARG); + BIF_ERROR(p,BADARG); } is_list_cap = ((pflags & PARSE_FLAG_CAPTURE_OPT) && (capture[CAPSPEC_TYPE] == am_list)); - if (is_not_tuple(BIF_ARG_2) || (arityval(*tuple_val(BIF_ARG_2)) != 4)) { - if (is_binary(BIF_ARG_2) || is_list(BIF_ARG_2) || is_nil(BIF_ARG_2)) { + if (is_not_tuple(arg2) || (arityval(*tuple_val(arg2)) != 4)) { + if (is_binary(arg2) || is_list(arg2) || is_nil(arg2)) { /* Compile from textual RE */ Uint slen; char *expr; @@ -884,19 +891,19 @@ re_run_3(BIF_ALIST_3) int capture_count; if (pflags & PARSE_FLAG_UNICODE && - (!is_binary(BIF_ARG_2) || !is_binary(BIF_ARG_1) || + (!is_binary(arg2) || !is_binary(arg1) || (is_list_cap && !(pflags & PARSE_FLAG_GLOBAL)))) { - BIF_TRAP3(urun_trap_exportp, BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); + BIF_TRAP3(urun_trap_exportp, p, arg1, arg2, arg3); } - if (erts_iolist_size(BIF_ARG_2, &slen)) { - BIF_ERROR(BIF_P,BADARG); + if (erts_iolist_size(arg2, &slen)) { + BIF_ERROR(p,BADARG); } expr = erts_alloc(ERTS_ALC_T_RE_TMP_BUF, slen + 1); - if (io_list_to_buf(BIF_ARG_2, expr, slen) != 0) { + if (io_list_to_buf(arg2, expr, slen) != 0) { erts_free(ERTS_ALC_T_RE_TMP_BUF, expr); - BIF_ERROR(BIF_P,BADARG); + BIF_ERROR(p,BADARG); } expr[slen]='\0'; result = erts_pcre_compile2(expr, comp_options, &errcode, @@ -905,11 +912,11 @@ re_run_3(BIF_ALIST_3) erts_free(ERTS_ALC_T_RE_TMP_BUF, expr); /* Compilation error gives badarg except in the compile function */ - BIF_ERROR(BIF_P,BADARG); + BIF_ERROR(p,BADARG); } if (pflags & PARSE_FLAG_GLOBAL) { Eterm precompiled = - build_compile_result(BIF_P, am_error, + build_compile_result(p, am_error, result, errcode, errstr, errofset, (pflags & @@ -917,13 +924,13 @@ re_run_3(BIF_ALIST_3) 0); Eterm *hp,r; erts_free(ERTS_ALC_T_RE_TMP_BUF, expr); - hp = HAlloc(BIF_P,4); - /* BIF_ARG_2 is in the tuple just to make exceptions right */ - r = TUPLE3(hp,BIF_ARG_3, + hp = HAlloc(p,4); + /* arg2 is in the tuple just to make exceptions right */ + r = TUPLE3(hp,arg3, ((pflags & PARSE_FLAG_UNIQUE_COMPILE_OPT) ? am_true : - am_false), BIF_ARG_2); - BIF_TRAP3(grun_trap_exportp, BIF_P, BIF_ARG_1, precompiled, r); + am_false), arg2); + BIF_TRAP3(grun_trap_exportp, p, arg1, precompiled, r); } erts_pcre_fullinfo(result, NULL, PCRE_INFO_SIZE, &code_size); @@ -935,31 +942,31 @@ re_run_3(BIF_ALIST_3) erts_free(ERTS_ALC_T_RE_TMP_BUF, expr); /*unicode = (pflags & PARSE_FLAG_UNICODE) ? 1 : 0;*/ } else { - BIF_ERROR(BIF_P,BADARG); + BIF_ERROR(p,BADARG); } } else { if (pflags & PARSE_FLAG_UNIQUE_COMPILE_OPT) { - BIF_ERROR(BIF_P,BADARG); + BIF_ERROR(p,BADARG); } - tp = tuple_val(BIF_ARG_2); + tp = tuple_val(arg2); if (tp[1] != am_re_pattern || is_not_small(tp[2]) || is_not_small(tp[3]) || is_not_binary(tp[4])) { - BIF_ERROR(BIF_P,BADARG); + BIF_ERROR(p,BADARG); } if (unsigned_val(tp[3]) && - (!is_binary(BIF_ARG_1) || + (!is_binary(arg1) || (is_list_cap && !(pflags & PARSE_FLAG_GLOBAL)))) { /* unicode */ - BIF_TRAP3(urun_trap_exportp, BIF_P, BIF_ARG_1, BIF_ARG_2, - BIF_ARG_3); + BIF_TRAP3(urun_trap_exportp, p, arg1, arg2, + arg3); } if (pflags & PARSE_FLAG_GLOBAL) { Eterm *hp,r; - hp = HAlloc(BIF_P,3); - r = TUPLE2(hp,BIF_ARG_3,am_false); - BIF_TRAP3(grun_trap_exportp, BIF_P, BIF_ARG_1, BIF_ARG_2, + hp = HAlloc(p,3); + r = TUPLE2(hp,arg3,am_false); + BIF_TRAP3(grun_trap_exportp, p, arg1, arg2, r); } @@ -968,7 +975,7 @@ re_run_3(BIF_ALIST_3) if ((code_tmp = (const pcre *) erts_get_aligned_binary_bytes(tp[4], &temp_alloc)) == NULL) { erts_free_aligned_binary_bytes(temp_alloc); - BIF_ERROR(BIF_P, BADARG); + BIF_ERROR(p, BADARG); } restart.code = erts_alloc(ERTS_ALC_T_RE_SUBJECT, code_size); memcpy(restart.code, code_tmp, code_size); @@ -980,7 +987,7 @@ re_run_3(BIF_ALIST_3) restart.ovector = erts_alloc(ERTS_ALC_T_RE_SUBJECT, ovsize * sizeof(int)); restart.extra.flags = PCRE_EXTRA_TABLES | PCRE_EXTRA_LOOP_LIMIT; restart.extra.tables = default_table; - restart.extra.loop_limit = ERTS_BIF_REDS_LEFT(BIF_P) * LOOP_FACTOR; + restart.extra.loop_limit = ERTS_BIF_REDS_LEFT(p) * LOOP_FACTOR; loop_limit_tmp = max_loop_limit; /* To lesser probability of race in debug situation (erts_debug) */ if (restart.extra.loop_limit > loop_limit_tmp) { @@ -996,7 +1003,7 @@ re_run_3(BIF_ALIST_3) if ((restart.ret_info = build_capture(capture,restart.code)) == NULL) { erts_free(ERTS_ALC_T_RE_SUBJECT, restart.ovector); erts_free(ERTS_ALC_T_RE_SUBJECT, restart.code); - BIF_ERROR(BIF_P,BADARG); + BIF_ERROR(p,BADARG); } } @@ -1004,7 +1011,7 @@ re_run_3(BIF_ALIST_3) copying, also binary returns can be sub binaries in that case */ restart.flags = 0; - if (is_binary(BIF_ARG_1)) { + if (is_binary(arg1)) { Eterm real_bin; Uint offset; Eterm* bptr; @@ -1012,9 +1019,9 @@ re_run_3(BIF_ALIST_3) int bitsize; ProcBin* pb; - ERTS_GET_REAL_BIN(BIF_ARG_1, real_bin, offset, bitoffs, bitsize); + ERTS_GET_REAL_BIN(arg1, real_bin, offset, bitoffs, bitsize); - slength = binary_size(BIF_ARG_1); + slength = binary_size(arg1); bptr = binary_val(real_bin); if (bitsize != 0 || bitoffs != 0 || (*bptr != HEADER_PROC_BIN)) { goto handle_iolist; @@ -1027,24 +1034,24 @@ re_run_3(BIF_ALIST_3) restart.flags |= RESTART_FLAG_SUBJECT_IN_BINARY; } else { handle_iolist: - if (erts_iolist_size(BIF_ARG_1, &slength)) { + if (erts_iolist_size(arg1, &slength)) { erts_free(ERTS_ALC_T_RE_SUBJECT, restart.ovector); erts_free(ERTS_ALC_T_RE_SUBJECT, restart.code); if (restart.ret_info != NULL) { erts_free(ERTS_ALC_T_RE_SUBJECT, restart.ret_info); } - BIF_ERROR(BIF_P,BADARG); + BIF_ERROR(p,BADARG); } restart.subject = erts_alloc(ERTS_ALC_T_RE_SUBJECT, slength); - if (io_list_to_buf(BIF_ARG_1, restart.subject, slength) != 0) { + if (io_list_to_buf(arg1, restart.subject, slength) != 0) { erts_free(ERTS_ALC_T_RE_SUBJECT, restart.ovector); erts_free(ERTS_ALC_T_RE_SUBJECT, restart.code); erts_free(ERTS_ALC_T_RE_SUBJECT, restart.subject); if (restart.ret_info != NULL) { erts_free(ERTS_ALC_T_RE_SUBJECT, restart.ret_info); } - BIF_ERROR(BIF_P,BADARG); + BIF_ERROR(p,BADARG); } } @@ -1056,7 +1063,7 @@ handle_iolist: rc = erts_pcre_exec(restart.code, &(restart.extra), restart.subject, slength, startoffset, options, restart.ovector, ovsize); ASSERT(loop_count != 0xFFFFFFFF); - BUMP_REDS(BIF_P, loop_count / LOOP_FACTOR); + BUMP_REDS(p, loop_count / LOOP_FACTOR); if (rc == PCRE_ERROR_LOOP_LIMIT) { /* Trap */ Binary *mbp = erts_create_magic_binary(sizeof(RestartContext), @@ -1065,17 +1072,17 @@ handle_iolist: Eterm magic_bin; Eterm *hp; memcpy(restartp,&restart,sizeof(RestartContext)); - BUMP_ALL_REDS(BIF_P); - hp = HAlloc(BIF_P, PROC_BIN_SIZE); - magic_bin = erts_mk_magic_binary_term(&hp, &MSO(BIF_P), mbp); + BUMP_ALL_REDS(p); + hp = HAlloc(p, PROC_BIN_SIZE); + magic_bin = erts_mk_magic_binary_term(&hp, &MSO(p), mbp); BIF_TRAP3(&re_exec_trap_export, - BIF_P, - BIF_ARG_1, - BIF_ARG_2 /* To avoid GC of precompiled code, XXX: not utilized yet */, + p, + arg1, + arg2 /* To avoid GC of precompiled code, XXX: not utilized yet */, magic_bin); } - res = build_exec_return(BIF_P, rc, &restart, BIF_ARG_1); + res = build_exec_return(p, rc, &restart, arg1); cleanup_restart_context(&restart); @@ -1083,9 +1090,15 @@ handle_iolist: } BIF_RETTYPE +re_run_3(BIF_ALIST_3) +{ + return re_run(BIF_P,BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); +} + +BIF_RETTYPE re_run_2(BIF_ALIST_2) { - return re_run_3(BIF_P,BIF_ARG_1, BIF_ARG_2, NIL); + return re_run(BIF_P,BIF_ARG_1, BIF_ARG_2, NIL); } /* diff --git a/erts/emulator/beam/erl_bif_timer.c b/erts/emulator/beam/erl_bif_timer.c index db771bd216..a922a33da3 100644 --- a/erts/emulator/beam/erl_bif_timer.c +++ b/erts/emulator/beam/erl_bif_timer.c @@ -26,6 +26,7 @@ #include "bif.h" #include "error.h" #include "big.h" +#include "erl_thr_progress.h" /**************************************************************************** ** BIF Timer support @@ -686,7 +687,7 @@ erts_bif_timer_foreach(void (*func)(Eterm, Eterm, ErlHeapFragment *, void *), { int i; - ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); for (i = 0; i < TIMER_HASH_VEC_SZ; i++) { ErtsBifTimer *btm; diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c index 7a08182e18..b0a58c80ea 100644 --- a/erts/emulator/beam/erl_bif_trace.c +++ b/erts/emulator/beam/erl_bif_trace.c @@ -37,6 +37,7 @@ #include "erl_version.h" #include "beam_bp.h" #include "erl_binary.h" +#include "erl_thr_progress.h" #define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1) @@ -47,6 +48,11 @@ static Binary *erts_default_meta_match_spec; static struct trace_pattern_flags erts_default_trace_pattern_flags; static Eterm erts_default_meta_tracer_pid; +static Eterm +trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist); +static BIF_RETTYPE +system_monitor(Process *p, Eterm monitor_pid, Eterm list); + static void new_seq_trace_token(Process* p); /* help func for seq_trace_2*/ static int already_traced(Process *p, Process *tracee_p, Eterm tracer); static int port_already_traced(Process *p, Port *tracee_port, Eterm tracer); @@ -76,13 +82,19 @@ erts_bif_trace_init(void) */ Eterm -trace_pattern_2(Process* p, Eterm MFA, Eterm Pattern) +trace_pattern_2(BIF_ALIST_2) { - return trace_pattern_3(p,MFA,Pattern,NIL); + return trace_pattern(BIF_P, BIF_ARG_1, BIF_ARG_2, NIL); } Eterm -trace_pattern_3(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) +trace_pattern_3(BIF_ALIST_3) +{ + return trace_pattern(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); +} + +static Eterm +trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) { DeclareTmpHeap(mfa,3,p); /* Not really heap here, but might be when setting pattern */ int i; @@ -97,7 +109,7 @@ trace_pattern_3(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) Eterm meta_tracer_pid = p->id; erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(0); + erts_smp_thr_progress_block(); UseTmpHeap(3,p); /* @@ -326,7 +338,7 @@ trace_pattern_3(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) done: UnUseTmpHeap(3,p); - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); return make_small(matches); @@ -336,7 +348,7 @@ trace_pattern_3(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) MatchSetUnref(match_prog_set); UnUseTmpHeap(3,p); - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); BIF_ERROR(p, BADARG); } @@ -435,9 +447,12 @@ erts_trace_flags(Eterm List, return 0; } -Eterm -trace_3(Process* p, Eterm pid_spec, Eterm how, Eterm list) +Eterm trace_3(BIF_ALIST_3) { + Process* p = BIF_P; + Eterm pid_spec = BIF_ARG_1; + Eterm how = BIF_ARG_2; + Eterm list = BIF_ARG_3; int on; Eterm tracer = NIL; int matches = 0; @@ -630,7 +645,7 @@ trace_3(Process* p, Eterm pid_spec, Eterm how, Eterm list) #ifdef ERTS_SMP erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(0); + erts_smp_thr_progress_block(); system_blocked = 1; #endif @@ -711,7 +726,7 @@ trace_3(Process* p, Eterm pid_spec, Eterm how, Eterm list) #ifdef ERTS_SMP if (system_blocked) { - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); } #endif @@ -726,7 +741,7 @@ trace_3(Process* p, Eterm pid_spec, Eterm how, Eterm list) #ifdef ERTS_SMP if (system_blocked) { - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); } #endif @@ -820,9 +835,11 @@ static int already_traced(Process *c_p, Process *tracee_p, Eterm tracer) * Return information about a process or an external function being traced. */ -Eterm -trace_info_2(Process* p, Eterm What, Eterm Key) +Eterm trace_info_2(BIF_ALIST_2) { + Process* p = BIF_P; + Eterm What = BIF_ARG_1; + Eterm Key = BIF_ARG_2; Eterm res; if (What == am_on_load) { res = trace_info_on_load(p, Key); @@ -1060,7 +1077,7 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key) #ifdef ERTS_SMP if ( (key == am_call_time) || (key == am_all)) { erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(0); + erts_smp_thr_progress_block(); } #endif @@ -1068,7 +1085,7 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key) #ifdef ERTS_SMP if ( (key == am_call_time) || (key == am_all)) { - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); } #endif @@ -1752,23 +1769,20 @@ new_seq_trace_token(Process* p) } } -BIF_RETTYPE seq_trace_info_1(BIF_ALIST_1) +BIF_RETTYPE erl_seq_trace_info(Process *p, Eterm item) { - Eterm item; Eterm res; Eterm* hp; Uint current_flag; - if (is_not_atom(BIF_ARG_1)) { - BIF_ERROR(BIF_P, BADARG); + if (is_not_atom(item)) { + BIF_ERROR(p, BADARG); } - item = BIF_ARG_1; - - if (SEQ_TRACE_TOKEN(BIF_P) == NIL) { + if (SEQ_TRACE_TOKEN(p) == NIL) { if ((item == am_send) || (item == am_receive) || (item == am_print) || (item == am_timestamp)) { - hp = HAlloc(BIF_P,3); + hp = HAlloc(p,3); res = TUPLE2(hp, item, am_false); BIF_RET(res); } else if ((item == am_label) || (item == am_serial)) { @@ -1778,35 +1792,40 @@ BIF_RETTYPE seq_trace_info_1(BIF_ALIST_1) } } - if (BIF_ARG_1 == am_send) { + if (item == am_send) { current_flag = SEQ_TRACE_SEND; - } else if (BIF_ARG_1 == am_receive) { + } else if (item == am_receive) { current_flag = SEQ_TRACE_RECEIVE; - } else if (BIF_ARG_1 == am_print) { + } else if (item == am_print) { current_flag = SEQ_TRACE_PRINT; - } else if (BIF_ARG_1 == am_timestamp) { + } else if (item == am_timestamp) { current_flag = SEQ_TRACE_TIMESTAMP; } else { current_flag = 0; } if (current_flag) { - res = unsigned_val(SEQ_TRACE_TOKEN_FLAGS(BIF_P)) & current_flag ? + res = unsigned_val(SEQ_TRACE_TOKEN_FLAGS(p)) & current_flag ? am_true : am_false; } else if (item == am_label) { - res = SEQ_TRACE_TOKEN_LABEL(BIF_P); + res = SEQ_TRACE_TOKEN_LABEL(p); } else if (item == am_serial) { - hp = HAlloc(BIF_P, 3); - res = TUPLE2(hp, SEQ_TRACE_TOKEN_LASTCNT(BIF_P), SEQ_TRACE_TOKEN_SERIAL(BIF_P)); + hp = HAlloc(p, 3); + res = TUPLE2(hp, SEQ_TRACE_TOKEN_LASTCNT(p), SEQ_TRACE_TOKEN_SERIAL(p)); } else { error: - BIF_ERROR(BIF_P, BADARG); + BIF_ERROR(p, BADARG); } - hp = HAlloc(BIF_P, 3); + hp = HAlloc(p, 3); res = TUPLE2(hp, item, res); BIF_RET(res); } +BIF_RETTYPE seq_trace_info_1(BIF_ALIST_1) +{ + BIF_RET(erl_seq_trace_info(BIF_P, BIF_ARG_1)); +} + /* seq_trace_print(Message) -> true | false This function passes Message to the system_tracer @@ -1852,7 +1871,7 @@ void erts_system_monitor_clear(Process *c_p) { #ifdef ERTS_SMP if (c_p) { erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(0); + erts_smp_thr_progress_block(); } #endif erts_set_system_monitor(NIL); @@ -1862,7 +1881,7 @@ void erts_system_monitor_clear(Process *c_p) { erts_system_monitor_flags.busy_dist_port = 0; #ifdef ERTS_SMP if (c_p) { - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); } #endif @@ -1919,23 +1938,35 @@ static Eterm system_monitor_get(Process *p) } -BIF_RETTYPE system_monitor_0(Process *p) { - BIF_RET(system_monitor_get(p)); +BIF_RETTYPE system_monitor_0(BIF_ALIST_0) +{ + BIF_RET(system_monitor_get(BIF_P)); } -BIF_RETTYPE system_monitor_1(Process *p, Eterm spec) { +BIF_RETTYPE system_monitor_1(BIF_ALIST_1) +{ + Process* p = BIF_P; + Eterm spec = BIF_ARG_1; + if (spec == am_undefined) { - BIF_RET(system_monitor_2(p, spec, NIL)); + BIF_RET(system_monitor(p, spec, NIL)); } else if (is_tuple(spec)) { Eterm *tp = tuple_val(spec); if (tp[0] != make_arityval(2)) goto error; - BIF_RET(system_monitor_2(p, tp[1], tp[2])); + BIF_RET(system_monitor(p, tp[1], tp[2])); } error: BIF_ERROR(p, BADARG); } -BIF_RETTYPE system_monitor_2(Process *p, Eterm monitor_pid, Eterm list) { +BIF_RETTYPE system_monitor_2(BIF_ALIST_2) +{ + return system_monitor(BIF_P, BIF_ARG_1, BIF_ARG_2); +} + +static BIF_RETTYPE +system_monitor(Process *p, Eterm monitor_pid, Eterm list) +{ Eterm prev; int system_blocked = 0; @@ -1951,7 +1982,7 @@ BIF_RETTYPE system_monitor_2(Process *p, Eterm monitor_pid, Eterm list) { system_blocked = 1; erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(0); + erts_smp_thr_progress_block(); if (!erts_pid2proc(p, ERTS_PROC_LOCK_MAIN, monitor_pid, 0)) goto error; @@ -1985,7 +2016,7 @@ BIF_RETTYPE system_monitor_2(Process *p, Eterm monitor_pid, Eterm list) { erts_system_monitor_flags.busy_port = !!busy_port; erts_system_monitor_flags.busy_dist_port = !!busy_dist_port; - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); BIF_RET(prev); } @@ -1993,7 +2024,7 @@ BIF_RETTYPE system_monitor_2(Process *p, Eterm monitor_pid, Eterm list) { error: if (system_blocked) { - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); } @@ -2006,7 +2037,7 @@ void erts_system_profile_clear(Process *c_p) { #ifdef ERTS_SMP if (c_p) { erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(0); + erts_smp_thr_progress_block(); } #endif erts_set_system_profile(NIL); @@ -2016,7 +2047,7 @@ void erts_system_profile_clear(Process *c_p) { erts_system_profile_flags.exclusive = 0; #ifdef ERTS_SMP if (c_p) { - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); } #endif @@ -2053,11 +2084,16 @@ static Eterm system_profile_get(Process *p) { } } -BIF_RETTYPE system_profile_0(Process *p) { - BIF_RET(system_profile_get(p)); +BIF_RETTYPE system_profile_0(BIF_ALIST_0) +{ + BIF_RET(system_profile_get(BIF_P)); } -BIF_RETTYPE system_profile_2(Process *p, Eterm profiler, Eterm list) { +BIF_RETTYPE system_profile_2(BIF_ALIST_2) +{ + Process *p = BIF_P; + Eterm profiler = BIF_ARG_1; + Eterm list = BIF_ARG_2; Eterm prev; int system_blocked = 0; Process *profiler_p = NULL; @@ -2075,7 +2111,7 @@ BIF_RETTYPE system_profile_2(Process *p, Eterm profiler, Eterm list) { system_blocked = 1; erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(0); + erts_smp_thr_progress_block(); /* Check if valid process, no locks are taken */ @@ -2117,7 +2153,7 @@ BIF_RETTYPE system_profile_2(Process *p, Eterm profiler, Eterm list) { erts_system_profile_flags.runnable_procs = !!runnable_procs; erts_system_profile_flags.exclusive = !!exclusive; - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); BIF_RET(prev); @@ -2126,7 +2162,7 @@ BIF_RETTYPE system_profile_2(Process *p, Eterm profiler, Eterm list) { error: if (system_blocked) { - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); } diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c index e56084b9cb..6f7309f493 100644 --- a/erts/emulator/beam/erl_bits.c +++ b/erts/emulator/beam/erl_bits.c @@ -76,14 +76,12 @@ struct erl_bits_state ErlBitsState; #define byte_buf (ErlBitsState.byte_buf_) #define byte_buf_len (ErlBitsState.byte_buf_len_) -#ifdef ERTS_SMP static erts_smp_atomic_t bits_bufs_size; -#endif Uint erts_bits_bufs_size(void) { - return 0; + return (Uint) erts_smp_atomic_read_nob(&bits_bufs_size); } #if !defined(ERTS_SMP) @@ -109,8 +107,8 @@ erts_bits_destroy_state(ERL_BITS_PROTO_0) void erts_init_bits(void) { + erts_smp_atomic_init_nob(&bits_bufs_size, 0); #if defined(ERTS_SMP) - erts_smp_atomic_init(&bits_bufs_size, 0); /* erl_process.c calls erts_bits_init_state() on all state instances */ #else ERL_BITS_DECLARE_STATEP; @@ -713,9 +711,7 @@ static void ERTS_INLINE need_byte_buf(ERL_BITS_PROTO_1(int need)) { if (byte_buf_len < need) { -#ifdef ERTS_SMP - erts_smp_atomic_add(&bits_bufs_size, need - byte_buf_len); -#endif + erts_smp_atomic_add_nob(&bits_bufs_size, need - byte_buf_len); byte_buf_len = need; byte_buf = erts_realloc(ERTS_ALC_T_BITS_BUF, byte_buf, byte_buf_len); } @@ -849,8 +845,7 @@ erts_bs_put_utf8(ERL_BITS_PROTO_1(Eterm arg)) dst[1] = 0x80 | (val & 0x3F); num_bits = 16; } else if (val < 0x10000UL) { - if ((0xD800 <= val && val <= 0xDFFF) || - val == 0xFFFE || val == 0xFFFF) { + if (0xD800 <= val && val <= 0xDFFF) { return 0; } dst[0] = 0xE0 | (val >> 12); @@ -890,8 +885,7 @@ erts_bs_put_utf16(ERL_BITS_PROTO_2(Eterm arg, Uint flags)) return 0; } val = unsigned_val(arg); - if (val > 0x10FFFF || (0xD800 <= val && val <= 0xDFFF) || - val == 0xFFFE || val == 0xFFFF) { + if (val > 0x10FFFF || (0xD800 <= val && val <= 0xDFFF)) { return 0; } @@ -1656,8 +1650,7 @@ erts_bs_get_utf8(ErlBinMatchBuffer* mb) return THE_NON_VALUE; } result = (((result << 6) + a) << 6) + b - (Eterm) 0x000E2080UL; - if ((0xD800 <= result && result <= 0xDFFF) || - result == 0xFFFE || result == 0xFFFF) { + if (0xD800 <= result && result <= 0xDFFF) { return THE_NON_VALUE; } mb->offset += 24; @@ -1727,9 +1720,6 @@ erts_bs_get_utf16(ErlBinMatchBuffer* mb, Uint flags) w1 = (src[0] << 8) | src[1]; } if (w1 < 0xD800 || w1 > 0xDFFF) { - if (w1 == 0xFFFE || w1 == 0xFFFF) { - return THE_NON_VALUE; - } mb->offset += 16; return make_small(w1); } else if (w1 > 0xDBFF) { diff --git a/erts/emulator/beam/erl_cpu_topology.c b/erts/emulator/beam/erl_cpu_topology.c index bcf8bcf270..03c0ef904a 100644 --- a/erts/emulator/beam/erl_cpu_topology.c +++ b/erts/emulator/beam/erl_cpu_topology.c @@ -487,7 +487,7 @@ erts_sched_check_cpu_bind_post_suspend(ErtsSchedulerData *esdp) /* Make sure we check if we should bind to a cpu or not... */ if (esdp->run_queue->flags & ERTS_RUNQ_FLG_SHARED_RUNQ) - erts_smp_atomic32_set(&esdp->chk_cpu_bind, 1); + erts_smp_atomic32_set_nob(&esdp->chk_cpu_bind, 1); else esdp->run_queue->flags |= ERTS_RUNQ_FLG_CHK_CPU_BIND; } @@ -503,7 +503,7 @@ erts_sched_check_cpu_bind(ErtsSchedulerData *esdp) erts_cpu_groups_callback_call_t *cgcc; #ifdef ERTS_SMP if (erts_common_run_queue) - erts_smp_atomic32_set(&esdp->chk_cpu_bind, 0); + erts_smp_atomic32_set_nob(&esdp->chk_cpu_bind, 0); else { esdp->run_queue->flags &= ~ERTS_RUNQ_FLG_CHK_CPU_BIND; } diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index e9bdeb35ef..0079c13287 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -129,8 +129,6 @@ static Uint meta_main_tab_slot_mask; /* The slot index part of an unnamed tab static Uint meta_main_tab_seq_incr; static Uint meta_main_tab_seq_cnt = 0; /* To give unique(-ish) table identifiers */ - - /* ** The meta hash table of all NAMED ets tables */ @@ -202,12 +200,17 @@ static int free_table_cont(Process *p, int first, int clean_meta_tab); static void print_table(int to, void *to_arg, int show, DbTable* tb); -static BIF_RETTYPE ets_select_delete_1(Process *p, Eterm a1); -static BIF_RETTYPE ets_select_count_1(Process *p, Eterm a1); -static BIF_RETTYPE ets_select_trap_1(Process *p, Eterm a1); -static BIF_RETTYPE ets_delete_trap(Process *p, Eterm a1); +static BIF_RETTYPE ets_select_delete_1(BIF_ALIST_1); +static BIF_RETTYPE ets_select_count_1(BIF_ALIST_1); +static BIF_RETTYPE ets_select_trap_1(BIF_ALIST_1); +static BIF_RETTYPE ets_delete_trap(BIF_ALIST_1); static Eterm table_info(Process* p, DbTable* tb, Eterm What); +static BIF_RETTYPE ets_select1(Process* p, Eterm arg1); +static BIF_RETTYPE ets_select2(Process* p, Eterm arg1, Eterm arg2); +static BIF_RETTYPE ets_select3(Process* p, Eterm arg1, Eterm arg2, Eterm arg3); + + /* * Exported global */ @@ -224,21 +227,21 @@ static void free_dbtable(DbTable* tb) { #ifdef HARDDEBUG - if (erts_smp_atomic_read(&tb->common.memory_size) != sizeof(DbTable)) { + if (erts_smp_atomic_read_nob(&tb->common.memory_size) != sizeof(DbTable)) { erts_fprintf(stderr, "ets: free_dbtable memory remain=%ld fix=%x\n", - erts_smp_atomic_read(&tb->common.memory_size)-sizeof(DbTable), + erts_smp_atomic_read_nob(&tb->common.memory_size)-sizeof(DbTable), tb->common.fixations); } erts_fprintf(stderr, "ets: free_dbtable(%T) deleted!!!\r\n", tb->common.id); erts_fprintf(stderr, "ets: free_dbtable: meta_pid_to_tab common.memory_size = %ld\n", - erts_smp_atomic_read(&meta_pid_to_tab->common.memory_size)); + erts_smp_atomic_read_nob(&meta_pid_to_tab->common.memory_size)); print_table(ERTS_PRINT_STDOUT, NULL, 1, meta_pid_to_tab); erts_fprintf(stderr, "ets: free_dbtable: meta_pid_to_fixed_tab common.memory_size = %ld\n", - erts_smp_atomic_read(&meta_pid_to_fixed_tab->common.memory_size)); + erts_smp_atomic_read_nob(&meta_pid_to_fixed_tab->common.memory_size)); print_table(ERTS_PRINT_STDOUT, NULL, 1, meta_pid_to_fixed_tab); #endif #ifdef ERTS_SMP @@ -248,6 +251,7 @@ free_dbtable(DbTable* tb) ASSERT(is_immed(tb->common.heir_data)); erts_db_free(ERTS_ALC_T_DB_TABLE, tb, (void *) tb, sizeof(DbTable)); ERTS_ETS_MISC_MEM_ADD(-sizeof(DbTable)); + ERTS_THR_MEMORY_BARRIER; } #ifdef ERTS_SMP @@ -276,8 +280,7 @@ static void schedule_free_dbtable(DbTable* tb) ASSERT(scheds >= 1); ASSERT(erts_refc_read(&tb->common.ref, 0) == 0); erts_refc_init(&tb->common.ref, scheds); - ERTS_THR_MEMORY_BARRIER; - erts_smp_schedule_misc_aux_work(0, scheds, chk_free_dbtable, tb); + erts_schedule_multi_misc_aux_work(0, scheds, chk_free_dbtable, tb); #else free_dbtable(tb); #endif @@ -1296,7 +1299,9 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) Uint32 status; Sint keypos; int is_named, is_fine_locked, frequent_read, is_compressed; +#ifdef DEBUG int cret; +#endif DeclareTmpHeap(meta_tuple,3,BIF_P); DbTableMethod* meth; erts_smp_rwmtx_t *mmtl; @@ -1417,12 +1422,12 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) { DbTable init_tb; - erts_smp_atomic_init(&init_tb.common.memory_size, 0); + erts_smp_atomic_init_nob(&init_tb.common.memory_size, 0); tb = (DbTable*) erts_db_alloc(ERTS_ALC_T_DB_TABLE, &init_tb, sizeof(DbTable)); ERTS_ETS_MISC_MEM_ADD(sizeof(DbTable)); - erts_smp_atomic_init(&tb->common.memory_size, - erts_smp_atomic_read(&init_tb.common.memory_size)); + erts_smp_atomic_init_nob(&tb->common.memory_size, + erts_smp_atomic_read_nob(&init_tb.common.memory_size)); } tb->common.meth = meth; @@ -1439,12 +1444,15 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) tb->common.owner = BIF_P->id; set_heir(BIF_P, tb, heir, heir_data); - erts_smp_atomic_init(&tb->common.nitems, 0); + erts_smp_atomic_init_nob(&tb->common.nitems, 0); tb->common.fixations = NULL; tb->common.compress = is_compressed; - cret = meth->db_create(BIF_P, tb); +#ifdef DEBUG + cret = +#endif + meth->db_create(BIF_P, tb); ASSERT(cret == DB_ERROR_NONE); erts_smp_spin_lock(&meta_main_tab_main_lock); @@ -1505,9 +1513,9 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) BIF_ARG_1, BIF_ARG_2, ret, BIF_P->id, BIF_P->initial[0], BIF_P->initial[1], BIF_P->initial[2]); erts_fprintf(stderr, "ets: new: meta_pid_to_tab common.memory_size = %ld\n", - erts_smp_atomic_read(&meta_pid_to_tab->common.memory_size)); + erts_smp_atomic_read_nob(&meta_pid_to_tab->common.memory_size)); erts_fprintf(stderr, "ets: new: meta_pid_to_fixed_tab common.memory_size = %ld\n", - erts_smp_atomic_read(&meta_pid_to_fixed_tab->common.memory_size)); + erts_smp_atomic_read_nob(&meta_pid_to_fixed_tab->common.memory_size)); #endif UseTmpHeap(3,BIF_P); @@ -1941,8 +1949,10 @@ BIF_RETTYPE ets_delete_object_2(BIF_ALIST_2) /* ** This is for trapping, cannot be called directly. */ -static BIF_RETTYPE ets_select_delete_1(Process *p, Eterm a1) +static BIF_RETTYPE ets_select_delete_1(BIF_ALIST_1) { + Process *p = BIF_P; + Eterm a1 = BIF_ARG_1; BIF_RETTYPE result; DbTable* tb; int cret; @@ -1996,7 +2006,7 @@ BIF_RETTYPE ets_select_delete_2(BIF_ALIST_2) if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE)) == NULL) { BIF_ERROR(BIF_P, BADARG); } - nitems = erts_smp_atomic_read(&tb->common.nitems); + nitems = erts_smp_atomic_read_nob(&tb->common.nitems); tb->common.meth->db_delete_all_objects(BIF_P, tb); db_unlock(tb, LCK_WRITE); BIF_RET(erts_make_integer(nitems,BIF_P)); @@ -2108,7 +2118,7 @@ BIF_RETTYPE ets_slot_2(BIF_ALIST_2) BIF_RETTYPE ets_match_1(BIF_ALIST_1) { - return ets_select_1(BIF_P, BIF_ARG_1); + return ets_select1(BIF_P, BIF_ARG_1); } BIF_RETTYPE ets_match_2(BIF_ALIST_2) @@ -2124,7 +2134,7 @@ BIF_RETTYPE ets_match_2(BIF_ALIST_2) ms = TUPLE3(hp, BIF_ARG_2, NIL, ms); hp += 4; ms = CONS(hp, ms, NIL); - res = ets_select_2(BIF_P, BIF_ARG_1, ms); + res = ets_select2(BIF_P, BIF_ARG_1, ms); UnUseTmpHeap(8,BIF_P); return res; } @@ -2142,7 +2152,7 @@ BIF_RETTYPE ets_match_3(BIF_ALIST_3) ms = TUPLE3(hp, BIF_ARG_2, NIL, ms); hp += 4; ms = CONS(hp, ms, NIL); - res = ets_select_3(BIF_P, BIF_ARG_1, ms, BIF_ARG_3); + res = ets_select3(BIF_P, BIF_ARG_1, ms, BIF_ARG_3); UnUseTmpHeap(8,BIF_P); return res; } @@ -2150,6 +2160,12 @@ BIF_RETTYPE ets_match_3(BIF_ALIST_3) BIF_RETTYPE ets_select_3(BIF_ALIST_3) { + return ets_select3(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); +} + +static BIF_RETTYPE +ets_select3(Process* p, Eterm arg1, Eterm arg2, Eterm arg3) +{ BIF_RETTYPE result; DbTable* tb; int cret; @@ -2160,22 +2176,22 @@ BIF_RETTYPE ets_select_3(BIF_ALIST_3) CHECK_TABLES(); /* Chunk size strictly greater than 0 */ - if (is_not_small(BIF_ARG_3) || (chunk_size = signed_val(BIF_ARG_3)) <= 0) { - BIF_ERROR(BIF_P, BADARG); + if (is_not_small(arg3) || (chunk_size = signed_val(arg3)) <= 0) { + BIF_ERROR(p, BADARG); } - if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_READ, LCK_READ)) == NULL) { - BIF_ERROR(BIF_P, BADARG); + if ((tb = db_get_table(p, arg1, DB_READ, LCK_READ)) == NULL) { + BIF_ERROR(p, BADARG); } - safety = ITERATION_SAFETY(BIF_P,tb); + safety = ITERATION_SAFETY(p,tb); if (safety == ITER_UNSAFE) { local_fix_table(tb); } - cret = tb->common.meth->db_select_chunk(BIF_P, tb, - BIF_ARG_2, chunk_size, + cret = tb->common.meth->db_select_chunk(p, tb, + arg2, chunk_size, 0 /* not reversed */, &ret); - if (DID_TRAP(BIF_P,ret) && safety != ITER_SAFE) { - fix_table_locked(BIF_P, tb); + if (DID_TRAP(p,ret) && safety != ITER_SAFE) { + fix_table_locked(p, tb); } if (safety == ITER_UNSAFE) { local_unfix_table(tb); @@ -2187,22 +2203,24 @@ BIF_RETTYPE ets_select_3(BIF_ALIST_3) ERTS_BIF_PREP_RET(result, ret); break; case DB_ERROR_SYSRES: - ERTS_BIF_PREP_ERROR(result, BIF_P, SYSTEM_LIMIT); + ERTS_BIF_PREP_ERROR(result, p, SYSTEM_LIMIT); break; default: - ERTS_BIF_PREP_ERROR(result, BIF_P, BADARG); + ERTS_BIF_PREP_ERROR(result, p, BADARG); break; } - erts_match_set_release_result(BIF_P); + erts_match_set_release_result(p); return result; } /* We get here instead of in the real BIF when trapping */ -static BIF_RETTYPE ets_select_trap_1(Process *p, Eterm a1) +static BIF_RETTYPE ets_select_trap_1(BIF_ALIST_1) { + Process *p = BIF_P; + Eterm a1 = BIF_ARG_1; BIF_RETTYPE result; DbTable* tb; int cret; @@ -2247,6 +2265,11 @@ static BIF_RETTYPE ets_select_trap_1(Process *p, Eterm a1) BIF_RETTYPE ets_select_1(BIF_ALIST_1) { + return ets_select1(BIF_P, BIF_ARG_1); +} + +static BIF_RETTYPE ets_select1(Process *p, Eterm arg1) +{ BIF_RETTYPE result; DbTable* tb; int cret; @@ -2260,28 +2283,27 @@ BIF_RETTYPE ets_select_1(BIF_ALIST_1) * Make sure that the table exists. */ - if (!is_tuple(BIF_ARG_1)) { - if (BIF_ARG_1 == am_EOT) { + if (!is_tuple(arg1)) { + if (arg1 == am_EOT) { BIF_RET(am_EOT); } - BIF_ERROR(BIF_P, BADARG); + BIF_ERROR(p, BADARG); } - tptr = tuple_val(BIF_ARG_1); + tptr = tuple_val(arg1); if (arityval(*tptr) < 1 || - (tb = db_get_table(BIF_P, tptr[1], DB_READ, LCK_READ)) == NULL) { - BIF_ERROR(BIF_P, BADARG); + (tb = db_get_table(p, tptr[1], DB_READ, LCK_READ)) == NULL) { + BIF_ERROR(p, BADARG); } - safety = ITERATION_SAFETY(BIF_P,tb); + safety = ITERATION_SAFETY(p,tb); if (safety == ITER_UNSAFE) { local_fix_table(tb); } - cret = tb->common.meth->db_select_continue(BIF_P,tb, - BIF_ARG_1, &ret); + cret = tb->common.meth->db_select_continue(p,tb, arg1, &ret); - if (DID_TRAP(BIF_P,ret) && safety != ITER_SAFE) { - fix_table_locked(BIF_P, tb); + if (DID_TRAP(p,ret) && safety != ITER_SAFE) { + fix_table_locked(p, tb); } if (safety == ITER_UNSAFE) { local_unfix_table(tb); @@ -2293,20 +2315,26 @@ BIF_RETTYPE ets_select_1(BIF_ALIST_1) ERTS_BIF_PREP_RET(result, ret); break; case DB_ERROR_SYSRES: - ERTS_BIF_PREP_ERROR(result, BIF_P, SYSTEM_LIMIT); + ERTS_BIF_PREP_ERROR(result, p, SYSTEM_LIMIT); break; default: - ERTS_BIF_PREP_ERROR(result, BIF_P, BADARG); + ERTS_BIF_PREP_ERROR(result, p, BADARG); break; } - erts_match_set_release_result(BIF_P); + erts_match_set_release_result(p); return result; } BIF_RETTYPE ets_select_2(BIF_ALIST_2) { + return ets_select2(BIF_P, BIF_ARG_1, BIF_ARG_2); +} + +static BIF_RETTYPE +ets_select2(Process* p, Eterm arg1, Eterm arg2) +{ BIF_RETTYPE result; DbTable* tb; int cret; @@ -2319,19 +2347,19 @@ BIF_RETTYPE ets_select_2(BIF_ALIST_2) * Make sure that the table exists. */ - if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_READ, LCK_READ)) == NULL) { - BIF_ERROR(BIF_P, BADARG); + if ((tb = db_get_table(p, arg1, DB_READ, LCK_READ)) == NULL) { + BIF_ERROR(p, BADARG); } - safety = ITERATION_SAFETY(BIF_P,tb); + safety = ITERATION_SAFETY(p,tb); if (safety == ITER_UNSAFE) { local_fix_table(tb); } - cret = tb->common.meth->db_select(BIF_P, tb, BIF_ARG_2, + cret = tb->common.meth->db_select(p, tb, arg2, 0, &ret); - if (DID_TRAP(BIF_P,ret) && safety != ITER_SAFE) { - fix_table_locked(BIF_P, tb); + if (DID_TRAP(p,ret) && safety != ITER_SAFE) { + fix_table_locked(p, tb); } if (safety == ITER_UNSAFE) { local_unfix_table(tb); @@ -2343,21 +2371,23 @@ BIF_RETTYPE ets_select_2(BIF_ALIST_2) ERTS_BIF_PREP_RET(result, ret); break; case DB_ERROR_SYSRES: - ERTS_BIF_PREP_ERROR(result, BIF_P, SYSTEM_LIMIT); + ERTS_BIF_PREP_ERROR(result, p, SYSTEM_LIMIT); break; default: - ERTS_BIF_PREP_ERROR(result, BIF_P, BADARG); + ERTS_BIF_PREP_ERROR(result, p, BADARG); break; } - erts_match_set_release_result(BIF_P); + erts_match_set_release_result(p); return result; } /* We get here instead of in the real BIF when trapping */ -static BIF_RETTYPE ets_select_count_1(Process *p, Eterm a1) +static BIF_RETTYPE ets_select_count_1(BIF_ALIST_1) { + Process *p = BIF_P; + Eterm a1 = BIF_ARG_1; BIF_RETTYPE result; DbTable* tb; int cret; @@ -2498,7 +2528,7 @@ BIF_RETTYPE ets_select_reverse_3(BIF_ALIST_3) BIF_RETTYPE ets_select_reverse_1(BIF_ALIST_1) { - return ets_select_1(BIF_P, BIF_ARG_1); + return ets_select1(BIF_P, BIF_ARG_1); } BIF_RETTYPE ets_select_reverse_2(BIF_ALIST_2) @@ -2552,7 +2582,7 @@ BIF_RETTYPE ets_select_reverse_2(BIF_ALIST_2) */ BIF_RETTYPE ets_match_object_1(BIF_ALIST_1) { - return ets_select_1(BIF_P, BIF_ARG_1); + return ets_select1(BIF_P, BIF_ARG_1); } BIF_RETTYPE ets_match_object_2(BIF_ALIST_2) @@ -2568,7 +2598,7 @@ BIF_RETTYPE ets_match_object_2(BIF_ALIST_2) ms = TUPLE3(hp, BIF_ARG_2, NIL, ms); hp += 4; ms = CONS(hp, ms, NIL); - res = ets_select_2(BIF_P, BIF_ARG_1, ms); + res = ets_select2(BIF_P, BIF_ARG_1, ms); UnUseTmpHeap(8,BIF_P); return res; } @@ -2586,7 +2616,7 @@ BIF_RETTYPE ets_match_object_3(BIF_ALIST_3) ms = TUPLE3(hp, BIF_ARG_2, NIL, ms); hp += 4; ms = CONS(hp, ms, NIL); - res = ets_select_3(BIF_P, BIF_ARG_1, ms, BIF_ARG_3); + res = ets_select3(BIF_P, BIF_ARG_1, ms, BIF_ARG_3); UnUseTmpHeap(8,BIF_P); return res; } @@ -2605,7 +2635,9 @@ BIF_RETTYPE ets_info_1(BIF_ALIST_1) int i; Eterm* hp; /*Process* rp = NULL;*/ + /* If/when we implement lockless private tables: Eterm owner; + */ if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_INFO, LCK_READ)) == NULL) { if (is_atom(BIF_ARG_1) || is_small(BIF_ARG_1)) { @@ -2614,7 +2646,9 @@ BIF_RETTYPE ets_info_1(BIF_ALIST_1) BIF_ERROR(BIF_P, BADARG); } + /* If/when we implement lockless private tables: owner = tb->common.owner; + */ /* If/when we implement lockless private tables: if ((tb->common.status & DB_PRIVATE) && owner != BIF_P->id) { @@ -2790,7 +2824,7 @@ void init_db(void) } #endif - erts_smp_atomic_init(&erts_ets_misc_mem_size, 0); + erts_smp_atomic_init_nob(&erts_ets_misc_mem_size, 0); db_initialize_util(); if (user_requested_db_max_tabs < DB_DEF_MAX_TABS) @@ -2832,13 +2866,13 @@ void init_db(void) /*TT*/ /* Create meta table invertion. */ - erts_smp_atomic_init(&init_tb.common.memory_size, 0); + erts_smp_atomic_init_nob(&init_tb.common.memory_size, 0); meta_pid_to_tab = (DbTable*) erts_db_alloc(ERTS_ALC_T_DB_TABLE, &init_tb, sizeof(DbTable)); ERTS_ETS_MISC_MEM_ADD(sizeof(DbTable)); - erts_smp_atomic_init(&meta_pid_to_tab->common.memory_size, - erts_smp_atomic_read(&init_tb.common.memory_size)); + erts_smp_atomic_init_nob(&meta_pid_to_tab->common.memory_size, + erts_smp_atomic_read_nob(&init_tb.common.memory_size)); meta_pid_to_tab->common.id = NIL; meta_pid_to_tab->common.the_name = am_true; @@ -2851,7 +2885,7 @@ void init_db(void) #endif meta_pid_to_tab->common.keypos = 1; meta_pid_to_tab->common.owner = NIL; - erts_smp_atomic_init(&meta_pid_to_tab->common.nitems, 0); + erts_smp_atomic_init_nob(&meta_pid_to_tab->common.nitems, 0); meta_pid_to_tab->common.slot = -1; meta_pid_to_tab->common.meth = &db_hash; meta_pid_to_tab->common.compress = 0; @@ -2864,13 +2898,13 @@ void init_db(void) erl_exit(1,"Unable to create ets metadata tables."); } - erts_smp_atomic_set(&init_tb.common.memory_size, 0); + erts_smp_atomic_set_nob(&init_tb.common.memory_size, 0); meta_pid_to_fixed_tab = (DbTable*) erts_db_alloc(ERTS_ALC_T_DB_TABLE, &init_tb, sizeof(DbTable)); ERTS_ETS_MISC_MEM_ADD(sizeof(DbTable)); - erts_smp_atomic_init(&meta_pid_to_fixed_tab->common.memory_size, - erts_smp_atomic_read(&init_tb.common.memory_size)); + erts_smp_atomic_init_nob(&meta_pid_to_fixed_tab->common.memory_size, + erts_smp_atomic_read_nob(&init_tb.common.memory_size)); meta_pid_to_fixed_tab->common.id = NIL; meta_pid_to_fixed_tab->common.the_name = am_true; @@ -2883,7 +2917,7 @@ void init_db(void) #endif meta_pid_to_fixed_tab->common.keypos = 1; meta_pid_to_fixed_tab->common.owner = NIL; - erts_smp_atomic_init(&meta_pid_to_fixed_tab->common.nitems, 0); + erts_smp_atomic_init_nob(&meta_pid_to_fixed_tab->common.nitems, 0); meta_pid_to_fixed_tab->common.slot = -1; meta_pid_to_fixed_tab->common.meth = &db_hash; meta_pid_to_fixed_tab->common.compress = 0; @@ -3422,7 +3456,7 @@ static void unfix_table_locked(Process* p, DbTable* tb, unlocked: if (!IS_FIXED(tb) && IS_HASH_TABLE(tb->common.status) - && erts_smp_atomic_read(&tb->hash.fixdel) != (erts_aint_t)NULL) { + && erts_smp_atomic_read_nob(&tb->hash.fixdel) != (erts_aint_t)NULL) { #ifdef ERTS_SMP if (*kind_p == LCK_READ && tb->common.is_thread_safe) { /* Must have write lock while purging pseudo-deleted (OTP-8166) */ @@ -3520,8 +3554,10 @@ static void free_heir_data(DbTable* tb) #endif } -static BIF_RETTYPE ets_delete_trap(Process *p, Eterm cont) +static BIF_RETTYPE ets_delete_trap(BIF_ALIST_1) { + Process *p = BIF_P; + Eterm cont = BIF_ARG_1; int trap; Eterm* ptr = big_val(cont); DbTable *tb = *((DbTable **) (UWord) (ptr + 1)); @@ -3607,7 +3643,7 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What) Eterm ret = THE_NON_VALUE; if (What == am_size) { - ret = make_small(erts_smp_atomic_read(&tb->common.nitems)); + ret = make_small(erts_smp_atomic_read_nob(&tb->common.nitems)); } else if (What == am_type) { if (tb->common.status & DB_SET) { ret = am_set; @@ -3620,7 +3656,7 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What) ret = am_bag; } } else if (What == am_memory) { - Uint words = (Uint) ((erts_smp_atomic_read(&tb->common.memory_size) + Uint words = (Uint) ((erts_smp_atomic_read_nob(&tb->common.memory_size) + sizeof(Uint) - 1) / sizeof(Uint)); @@ -3714,7 +3750,7 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What) std_dev_exp = make_float(hp); PUT_DOUBLE(f, hp); hp += FLOAT_SIZE_OBJECT; - ret = TUPLE7(hp, make_small(erts_smp_atomic_read(&tb->hash.nactive)), + ret = TUPLE7(hp, make_small(erts_smp_atomic_read_nob(&tb->hash.nactive)), avg, std_dev_real, std_dev_exp, make_small(stats.min_chain_len), make_small(stats.max_chain_len), @@ -3734,9 +3770,9 @@ static void print_table(int to, void *to_arg, int show, DbTable* tb) tb->common.meth->db_print(to, to_arg, show, tb); - erts_print(to, to_arg, "Objects: %d\n", (int)erts_smp_atomic_read(&tb->common.nitems)); + erts_print(to, to_arg, "Objects: %d\n", (int)erts_smp_atomic_read_nob(&tb->common.nitems)); erts_print(to, to_arg, "Words: %bpu\n", - (UWord) ((erts_smp_atomic_read(&tb->common.memory_size) + (Uint) ((erts_smp_atomic_read_nob(&tb->common.memory_size) + sizeof(Uint) - 1) / sizeof(Uint))); @@ -3762,8 +3798,9 @@ void db_info(int to, void *to_arg, int show) /* Called by break handler */ Uint erts_get_ets_misc_mem_size(void) { + ERTS_THR_MEMORY_BARRIER; /* Memory not allocated in ets_alloc */ - return (Uint) erts_smp_atomic_read(&erts_ets_misc_mem_size); + return (Uint) erts_smp_atomic_read_nob(&erts_ets_misc_mem_size); } /* SMP Note: May only be used when system is locked */ diff --git a/erts/emulator/beam/erl_db.h b/erts/emulator/beam/erl_db.h index e0bdebcb01..2e5deaf338 100644 --- a/erts/emulator/beam/erl_db.h +++ b/erts/emulator/beam/erl_db.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * Copyright Ericsson AB 1996-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -86,11 +86,11 @@ do { \ erts_aint_t sz__ = (((erts_aint_t) (ALLOC_SZ)) \ - ((erts_aint_t) (FREE_SZ))); \ ASSERT((TAB)); \ - erts_smp_atomic_add(&(TAB)->common.memory_size, sz__); \ + erts_smp_atomic_add_nob(&(TAB)->common.memory_size, sz__); \ } while (0) #define ERTS_ETS_MISC_MEM_ADD(SZ) \ - erts_smp_atomic_add(&erts_ets_misc_mem_size, (SZ)); + erts_smp_atomic_add_nob(&erts_ets_misc_mem_size, (SZ)); ERTS_GLB_INLINE void *erts_db_alloc(ErtsAlcType_t type, DbTable *tab, @@ -227,7 +227,7 @@ erts_db_free(ErtsAlcType_t type, DbTable *tab, void *ptr, Uint size) ERTS_DB_ALC_MEM_UPDATE_(tab, size, 0); ASSERT(((void *) tab) != ptr - || erts_smp_atomic_read(&tab->common.memory_size) == 0); + || erts_smp_atomic_read_nob(&tab->common.memory_size) == 0); erts_free(type, ptr); } diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index fdc82c8b88..e3380a57b2 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -105,9 +105,13 @@ #define NSEG_2 256 /* Size of second segment table */ #define NSEG_INC 128 /* Number of segments to grow after that */ -#define SEGTAB(tb) ((struct segment**)erts_smp_atomic_read_acqb(&(tb)->segtab)) -#define NACTIVE(tb) ((int)erts_smp_atomic_read(&(tb)->nactive)) -#define NITEMS(tb) ((int)erts_smp_atomic_read(&(tb)->common.nitems)) +#ifdef ETHR_ORDERED_READ_DEPEND +#define SEGTAB(tb) ((struct segment**)erts_smp_atomic_read_nob(&(tb)->segtab)) +#else +#define SEGTAB(tb) ((struct segment**)erts_smp_atomic_read_rb(&(tb)->segtab)) +#endif +#define NACTIVE(tb) ((int)erts_smp_atomic_read_nob(&(tb)->nactive)) +#define NITEMS(tb) ((int)erts_smp_atomic_read_nob(&(tb)->common.nitems)) #define BUCKET(tb, i) SEGTAB(tb)[(i) >> SEGSZ_EXP]->buckets[(i) & SEGSZ_MASK] @@ -123,10 +127,10 @@ static ERTS_INLINE Uint hash_to_ix(DbTableHash* tb, HashValue hval) { Uint mask = erts_smp_atomic_read_acqb(&tb->szm); - Uint ix = hval & mask; - if (ix >= erts_smp_atomic_read(&tb->nactive)) { + Uint ix = hval & mask; + if (ix >= erts_smp_atomic_read_nob(&tb->nactive)) { ix &= mask>>1; - ASSERT(ix < erts_smp_atomic_read(&tb->nactive)); + ASSERT(ix < erts_smp_atomic_read_nob(&tb->nactive)); } return ix; } @@ -141,14 +145,14 @@ static ERTS_INLINE void add_fixed_deletion(DbTableHash* tb, int ix) (DbTable *) tb, sizeof(FixedDeletion)); ERTS_ETS_MISC_MEM_ADD(sizeof(FixedDeletion)); - fixd->slot = ix; - was_next = erts_smp_atomic_read(&tb->fixdel); + fixd->slot = ix; + was_next = erts_smp_atomic_read_acqb(&tb->fixdel); do { /* Lockless atomic insertion in linked list: */ exp_next = was_next; fixd->next = (FixedDeletion*) exp_next; - was_next = erts_smp_atomic_cmpxchg(&tb->fixdel, - (erts_aint_t) fixd, - exp_next); + was_next = erts_smp_atomic_cmpxchg_relb(&tb->fixdel, + (erts_aint_t) fixd, + exp_next); }while (was_next != exp_next); } @@ -540,22 +544,24 @@ static void restore_fixdel(DbTableHash* tb, FixedDeletion* fixdel) { /*int tries = 0;*/ DEBUG_WAIT(); - if (erts_smp_atomic_cmpxchg(&tb->fixdel, (erts_aint_t)fixdel, - (erts_aint_t)NULL) != (erts_aint_t)NULL) { + if (erts_smp_atomic_cmpxchg_relb(&tb->fixdel, + (erts_aint_t) fixdel, + (erts_aint_t) NULL) != (erts_aint_t) NULL) { /* Oboy, must join lists */ FixedDeletion* last = fixdel; erts_aint_t was_tail; erts_aint_t exp_tail; - while (last->next != NULL) last = last->next; - was_tail = erts_smp_atomic_read(&tb->fixdel); + while (last->next != NULL) last = last->next; + was_tail = erts_smp_atomic_read_acqb(&tb->fixdel); do { /* Lockless atomic list insertion */ exp_tail = was_tail; last->next = (FixedDeletion*) exp_tail; /*++tries;*/ DEBUG_WAIT(); - was_tail = erts_smp_atomic_cmpxchg(&tb->fixdel, (erts_aint_t)fixdel, - exp_tail); + was_tail = erts_smp_atomic_cmpxchg_relb(&tb->fixdel, + (erts_aint_t) fixdel, + exp_tail); }while (was_tail != exp_tail); } /*erts_fprintf(stderr,"erl_db_hash: restore_fixdel tries=%d\r\n", tries);*/ @@ -572,7 +578,8 @@ void db_unfix_table_hash(DbTableHash *tb) || (erts_smp_lc_rwmtx_is_rlocked(&tb->common.rwlock) && !tb->common.is_thread_safe)); restart: - fixdel = (FixedDeletion*) erts_smp_atomic_xchg(&tb->fixdel, (erts_aint_t)NULL); + fixdel = (FixedDeletion*) erts_smp_atomic_xchg_acqb(&tb->fixdel, + (erts_aint_t) NULL); while (fixdel != NULL) { FixedDeletion *fx = fixdel; int ix = fx->slot; @@ -639,14 +646,14 @@ int db_create_hash(Process *p, DbTable *tbl) { DbTableHash *tb = &tbl->hash; - erts_smp_atomic_init(&tb->szm, SEGSZ_MASK); - erts_smp_atomic_init(&tb->nactive, SEGSZ); - erts_smp_atomic_init(&tb->fixdel, (erts_aint_t)NULL); - erts_smp_atomic_init(&tb->segtab, (erts_aint_t) alloc_ext_seg(tb,0,NULL)->segtab); + erts_smp_atomic_init_nob(&tb->szm, SEGSZ_MASK); + erts_smp_atomic_init_nob(&tb->nactive, SEGSZ); + erts_smp_atomic_init_nob(&tb->fixdel, (erts_aint_t)NULL); + erts_smp_atomic_init_nob(&tb->segtab, (erts_aint_t) alloc_ext_seg(tb,0,NULL)->segtab); tb->nsegs = NSEG_1; tb->nslots = SEGSZ; - erts_smp_atomic_init(&tb->is_resizing, 0); + erts_smp_atomic_init_nob(&tb->is_resizing, 0); #ifdef ERTS_SMP if (tb->common.type & DB_FINE_LOCKED) { erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; @@ -663,7 +670,7 @@ int db_create_hash(Process *p, DbTable *tbl) /* This important property is needed to guarantee that the buckets * involved in a grow/shrink operation it protected by the same lock: */ - ASSERT(erts_smp_atomic_read(&tb->nactive) % DB_HASH_LOCK_CNT == 0); + ASSERT(erts_smp_atomic_read_nob(&tb->nactive) % DB_HASH_LOCK_CNT == 0); } else { /* coarse locking */ tb->locks = NULL; @@ -783,7 +790,7 @@ int db_put_hash(DbTable *tbl, Eterm obj, int key_clash_fail) if (tb->common.status & DB_SET) { HashDbTerm* bnext = b->next; if (b->hvalue == INVALID_HASH) { - erts_smp_atomic_inc(&tb->common.nitems); + erts_smp_atomic_inc_nob(&tb->common.nitems); } else if (key_clash_fail) { ret = DB_ERROR_BADKEY; @@ -811,7 +818,7 @@ int db_put_hash(DbTable *tbl, Eterm obj, int key_clash_fail) do { if (db_eq(&tb->common,obj,&q->dbterm)) { if (q->hvalue == INVALID_HASH) { - erts_smp_atomic_inc(&tb->common.nitems); + erts_smp_atomic_inc_nob(&tb->common.nitems); q->hvalue = hval; if (q != b) { /* must move to preserve key insertion order */ *qp = q->next; @@ -832,7 +839,7 @@ Lnew: q->hvalue = hval; q->next = b; *bp = q; - nitems = erts_smp_atomic_inctest(&tb->common.nitems); + nitems = erts_smp_atomic_inc_read_nob(&tb->common.nitems); WUNLOCK_HASH(lck); { int nactive = NACTIVE(tb); @@ -1069,7 +1076,7 @@ int db_erase_bag_exact2(DbTable *tbl, Eterm key, Eterm value) EQ(value, b->dbterm.tpl[2])) { *bp = b->next; free_term(tb, b); - erts_smp_atomic_dec(&tb->common.nitems); + erts_smp_atomic_dec_nob(&tb->common.nitems); b = *bp; break; } @@ -1128,7 +1135,7 @@ int db_erase_hash(DbTable *tbl, Eterm key, Eterm *ret) } WUNLOCK_HASH(lck); if (nitems_diff) { - erts_smp_atomic_add(&tb->common.nitems, nitems_diff); + erts_smp_atomic_add_nob(&tb->common.nitems, nitems_diff); try_shrink(tb); } *ret = am_true; @@ -1187,7 +1194,7 @@ static int db_erase_object_hash(DbTable *tbl, Eterm object, Eterm *ret) } WUNLOCK_HASH(lck); if (nitems_diff) { - erts_smp_atomic_add(&tb->common.nitems, nitems_diff); + erts_smp_atomic_add_nob(&tb->common.nitems, nitems_diff); try_shrink(tb); } *ret = am_true; @@ -1798,7 +1805,7 @@ static int db_select_delete_hash(Process *p, free_term(tb, del); did_erase = 1; } - erts_smp_atomic_dec(&tb->common.nitems); + erts_smp_atomic_dec_nob(&tb->common.nitems); ++got; } --num_left; @@ -1909,7 +1916,7 @@ static int db_select_delete_continue_hash(Process *p, free_term(tb, del); did_erase = 1; } - erts_smp_atomic_dec(&tb->common.nitems); + erts_smp_atomic_dec_nob(&tb->common.nitems); ++got; } @@ -2064,7 +2071,7 @@ int db_mark_all_deleted_hash(DbTable *tbl) }while(list != NULL); } } - erts_smp_atomic_set(&tb->common.nitems, 0); + erts_smp_atomic_set_nob(&tb->common.nitems, 0); return DB_ERROR_NONE; } @@ -2115,7 +2122,7 @@ static int db_free_table_continue_hash(DbTable *tbl) { DbTableHash *tb = &tbl->hash; int done; - FixedDeletion* fixdel = (FixedDeletion*) erts_smp_atomic_read(&tb->fixdel); + FixedDeletion* fixdel = (FixedDeletion*) erts_smp_atomic_read_acqb(&tb->fixdel); ERTS_SMP_LC_ASSERT(IS_TAB_WLOCKED(tb)); done = 0; @@ -2129,11 +2136,11 @@ static int db_free_table_continue_hash(DbTable *tbl) sizeof(FixedDeletion)); ERTS_ETS_MISC_MEM_ADD(-sizeof(FixedDeletion)); if (++done >= 2*DELETE_RECORD_LIMIT) { - erts_smp_atomic_set(&tb->fixdel, (erts_aint_t)fixdel); + erts_smp_atomic_set_relb(&tb->fixdel, (erts_aint_t)fixdel); return 0; /* Not done */ } } - erts_smp_atomic_set(&tb->fixdel, (erts_aint_t)NULL); + erts_smp_atomic_set_relb(&tb->fixdel, (erts_aint_t)NULL); done /= 2; while(tb->nslots != 0) { @@ -2157,7 +2164,7 @@ static int db_free_table_continue_hash(DbTable *tbl) tb->locks = NULL; } #endif - ASSERT(erts_smp_atomic_read(&tb->common.memory_size) == sizeof(DbTable)); + ASSERT(erts_smp_atomic_read_nob(&tb->common.memory_size) == sizeof(DbTable)); return 1; /* Done */ } @@ -2350,7 +2357,7 @@ static int alloc_seg(DbTableHash *tb) struct ext_segment* eseg; eseg = (struct ext_segment*) SEGTAB(tb)[seg_ix-1]; MY_ASSERT(eseg!=NULL && eseg->s.is_ext_segment); - erts_smp_atomic_set_relb(&tb->segtab, (erts_aint_t) eseg->segtab); + erts_smp_atomic_set_wb(&tb->segtab, (erts_aint_t) eseg->segtab); tb->nsegs = eseg->nsegs; } ASSERT(seg_ix < tb->nsegs); @@ -2422,7 +2429,7 @@ static int free_seg(DbTableHash *tb, int free_records) MY_ASSERT(newtop->s.is_ext_segment); if (newtop->prev_segtab != NULL) { /* Time to use a smaller segtab */ - erts_smp_atomic_set_relb(&tb->segtab, (erts_aint_t)newtop->prev_segtab); + erts_smp_atomic_set_wb(&tb->segtab, (erts_aint_t)newtop->prev_segtab); tb->nsegs = seg_ix; ASSERT(tb->nsegs == EXTSEG(SEGTAB(tb))->nsegs); } @@ -2439,7 +2446,7 @@ static int free_seg(DbTableHash *tb, int free_records) if (seg_ix > 0) { if (seg_ix < tb->nsegs) SEGTAB(tb)[seg_ix] = NULL; } else { - erts_smp_atomic_set_relb(&tb->segtab, (erts_aint_t)NULL); + erts_smp_atomic_set_wb(&tb->segtab, (erts_aint_t)NULL); } #endif tb->nslots -= SEGSZ; @@ -2500,7 +2507,7 @@ static void grow(DbTableHash* tb, int nactive) int from_ix; int szm; - if (erts_smp_atomic_xchg(&tb->is_resizing, 1)) { + if (erts_smp_atomic_xchg_acqb(&tb->is_resizing, 1)) { return; /* already in progress */ } if (NACTIVE(tb) != nactive) { @@ -2515,7 +2522,7 @@ static void grow(DbTableHash* tb, int nactive) } ASSERT(nactive < tb->nslots); - szm = erts_smp_atomic_read(&tb->szm); + szm = erts_smp_atomic_read_nob(&tb->szm); if (nactive <= szm) { from_ix = nactive & (szm >> 1); } else { @@ -2532,7 +2539,7 @@ static void grow(DbTableHash* tb, int nactive) WUNLOCK_HASH(lck); goto abort; } - erts_smp_atomic_inc(&tb->nactive); + erts_smp_atomic_inc_nob(&tb->nactive); if (from_ix == 0) { erts_smp_atomic_set_relb(&tb->szm, szm); } @@ -2577,13 +2584,13 @@ abort: */ static void shrink(DbTableHash* tb, int nactive) { - if (erts_smp_atomic_xchg(&tb->is_resizing, 1)) { + if (erts_smp_atomic_xchg_acqb(&tb->is_resizing, 1)) { return; /* already in progress */ } if (NACTIVE(tb) == nactive) { erts_smp_rwmtx_t* lck; int src_ix = nactive - 1; - int low_szm = erts_smp_atomic_read(&tb->szm) >> 1; + int low_szm = erts_smp_atomic_read_nob(&tb->szm) >> 1; int dst_ix = src_ix & low_szm; ASSERT(dst_ix < src_ix); @@ -2610,7 +2617,7 @@ static void shrink(DbTableHash* tb, int nactive) *dst_bp = *src_bp; *src_bp = NULL; - erts_smp_atomic_set(&tb->nactive, src_ix); + erts_smp_atomic_set_nob(&tb->nactive, src_ix); if (dst_ix == 0) { erts_smp_atomic_set_relb(&tb->szm, low_szm); } @@ -2746,7 +2753,7 @@ static int db_delete_all_objects_hash(Process* p, DbTable* tbl) } else { db_free_table_hash(tbl); db_create_hash(p, tbl); - erts_smp_atomic_set(&tbl->hash.common.nitems, 0); + erts_smp_atomic_set_nob(&tbl->hash.common.nitems, 0); } return 0; } diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c index 9a0ba3a418..312050b931 100644 --- a/erts/emulator/beam/erl_db_tree.c +++ b/erts/emulator/beam/erl_db_tree.c @@ -49,7 +49,7 @@ #include "erl_db_tree.h" #define GETKEY_WITH_POS(Keypos, Tplp) (*((Tplp) + Keypos)) -#define NITEMS(tb) ((int)erts_smp_atomic_read(&(tb)->common.nitems)) +#define NITEMS(tb) ((int)erts_smp_atomic_read_nob(&(tb)->common.nitems)) /* ** A stack of this size is enough for an AVL tree with more than @@ -84,7 +84,7 @@ */ static DbTreeStack* get_static_stack(DbTableTree* tb) { - if (!erts_smp_atomic_xchg(&tb->is_stack_busy, 1)) { + if (!erts_smp_atomic_xchg_acqb(&tb->is_stack_busy, 1)) { return &tb->static_stack; } return NULL; @@ -96,7 +96,7 @@ static DbTreeStack* get_static_stack(DbTableTree* tb) static DbTreeStack* get_any_stack(DbTableTree* tb) { DbTreeStack* stack; - if (!erts_smp_atomic_xchg(&tb->is_stack_busy, 1)) { + if (!erts_smp_atomic_xchg_acqb(&tb->is_stack_busy, 1)) { return &tb->static_stack; } stack = erts_db_alloc(ERTS_ALC_T_DB_STK, (DbTable *) tb, @@ -110,7 +110,7 @@ static DbTreeStack* get_any_stack(DbTableTree* tb) static void release_stack(DbTableTree* tb, DbTreeStack* stack) { if (stack == &tb->static_stack) { - ASSERT(erts_smp_atomic_read(&tb->is_stack_busy) == 1); + ASSERT(erts_smp_atomic_read_nob(&tb->is_stack_busy) == 1); erts_smp_atomic_set_relb(&tb->is_stack_busy, 0); } else { @@ -344,8 +344,8 @@ static int do_partly_bound_can_match_lesser(Eterm a, Eterm b, int *done); static int do_partly_bound_can_match_greater(Eterm a, Eterm b, int *done); -static BIF_RETTYPE ets_select_reverse(Process *p, Eterm a1, - Eterm a2, Eterm a3); +static BIF_RETTYPE ets_select_reverse(BIF_ALIST_3); + /* Method interface functions */ static int db_first_tree(Process *p, DbTable *tbl, @@ -478,7 +478,7 @@ int db_create_tree(Process *p, DbTable *tbl) sizeof(TreeDbTerm *) * STACK_NEED); tb->static_stack.pos = 0; tb->static_stack.slot = 0; - erts_smp_atomic_init(&tb->is_stack_busy, 0); + erts_smp_atomic_init_nob(&tb->is_stack_busy, 0); tb->deletion = 0; return DB_ERROR_NONE; } @@ -613,8 +613,8 @@ static int db_put_tree(DbTable *tbl, Eterm obj, int key_clash_fail) for (;;) if (!*this) { /* Found our place */ state = 1; - if (erts_smp_atomic_inctest(&tb->common.nitems) >= TREE_MAX_ELEMENTS) { - erts_smp_atomic_dec(&tb->common.nitems); + if (erts_smp_atomic_inc_read_nob(&tb->common.nitems) >= TREE_MAX_ELEMENTS) { + erts_smp_atomic_dec_nob(&tb->common.nitems); return DB_ERROR_SYSRES; } *this = new_dbterm(tb, obj); @@ -844,8 +844,12 @@ static int db_slot_tree(Process *p, DbTable *tbl, -static BIF_RETTYPE ets_select_reverse(Process *p, Eterm a1, Eterm a2, Eterm a3) +static BIF_RETTYPE ets_select_reverse(BIF_ALIST_3) { + Process *p = BIF_P; + Eterm a1 = BIF_ARG_1; + Eterm a2 = BIF_ARG_2; + Eterm a3 = BIF_ARG_3; Eterm list; Eterm result; Eterm* hp; @@ -1583,7 +1587,7 @@ static int db_select_delete_continue_tree(Process *p, sc.max = 1000; sc.keypos = tb->common.keypos; - ASSERT(!erts_smp_atomic_read(&tb->is_stack_busy)); + ASSERT(!erts_smp_atomic_read_nob(&tb->is_stack_busy)); traverse_backwards(tb, &tb->static_stack, lastkey, NULL, &doit_select_delete, &sc); BUMP_REDS(p, 1000 - sc.max); @@ -1774,7 +1778,7 @@ static int db_free_table_continue_tree(DbTable *tbl) (DbTable *) tb, (void *) tb->static_stack.array, sizeof(TreeDbTerm *) * STACK_NEED); - ASSERT(erts_smp_atomic_read(&tb->common.memory_size) + ASSERT(erts_smp_atomic_read_nob(&tb->common.memory_size) == sizeof(DbTable)); } return result; @@ -1784,7 +1788,7 @@ static int db_delete_all_objects_tree(Process* p, DbTable* tbl) { db_free_table_tree(tbl); db_create_tree(p, tbl); - erts_smp_atomic_set(&tbl->tree.common.nitems, 0); + erts_smp_atomic_set_nob(&tbl->tree.common.nitems, 0); return 0; } @@ -1866,7 +1870,7 @@ static TreeDbTerm *linkout_tree(DbTableTree *tb, tstack[tpos++] = this; state = delsub(this); } - erts_smp_atomic_dec(&tb->common.nitems); + erts_smp_atomic_dec_nob(&tb->common.nitems); break; } } @@ -1933,7 +1937,7 @@ static TreeDbTerm *linkout_object_tree(DbTableTree *tb, tstack[tpos++] = this; state = delsub(this); } - erts_smp_atomic_dec(&tb->common.nitems); + erts_smp_atomic_dec_nob(&tb->common.nitems); break; } } diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index e5be1f253a..4821a7d9fb 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -35,6 +35,7 @@ #include "bif.h" #include "big.h" #include "erl_binary.h" +#include "erl_thr_progress.h" #include "erl_db_util.h" @@ -491,11 +492,11 @@ erts_match_set_release_result(Process* c_p) /* The trace control word. */ -static erts_smp_atomic_t trace_control_word; +static erts_smp_atomic32_t trace_control_word; /* This needs to be here, before the bif table... */ -static Eterm db_set_trace_control_word_fake_1(Process *p, Eterm val); +static Eterm db_set_trace_control_word_fake_1(BIF_ALIST_1); /* ** The table of callable bif's, i e guard bif's and @@ -908,14 +909,18 @@ static void db_free_tmp_uncompressed(DbTerm* obj); /* ** Pseudo BIF:s to be callable from the PAM VM. */ - -BIF_RETTYPE db_get_trace_control_word_0(Process *p) +BIF_RETTYPE db_get_trace_control_word(Process *p) { - Uint32 tcw = (Uint32) erts_smp_atomic_read(&trace_control_word); + Uint32 tcw = (Uint32) erts_smp_atomic32_read_acqb(&trace_control_word); BIF_RET(erts_make_integer((Uint) tcw, p)); } -BIF_RETTYPE db_set_trace_control_word_1(Process *p, Eterm new) +BIF_RETTYPE db_get_trace_control_word_0(BIF_ALIST_0) +{ + BIF_RET(db_get_trace_control_word(BIF_P)); +} + +BIF_RETTYPE db_set_trace_control_word(Process *p, Eterm new) { Uint val; Uint32 old_tcw; @@ -923,19 +928,27 @@ BIF_RETTYPE db_set_trace_control_word_1(Process *p, Eterm new) BIF_ERROR(p, BADARG); if (val != ((Uint32)val)) BIF_ERROR(p, BADARG); - - old_tcw = (Uint32) erts_smp_atomic_xchg(&trace_control_word, (erts_aint_t) val); + + old_tcw = (Uint32) erts_smp_atomic32_xchg_relb(&trace_control_word, + (erts_aint32_t) val); BIF_RET(erts_make_integer((Uint) old_tcw, p)); } -static Eterm db_set_trace_control_word_fake_1(Process *p, Eterm new) +BIF_RETTYPE db_set_trace_control_word_1(BIF_ALIST_1) { + BIF_RET(db_set_trace_control_word(BIF_P, BIF_ARG_1)); +} + +static Eterm db_set_trace_control_word_fake_1(BIF_ALIST_1) +{ + Process *p = BIF_P; + Eterm new = BIF_ARG_1; Uint val; if (!term_to_Uint(new, &val)) BIF_ERROR(p, BADARG); if (val != ((Uint32)val)) BIF_ERROR(p, BADARG); - BIF_RET(db_get_trace_control_word_0(p)); + BIF_RET(db_get_trace_control_word(p)); } /* @@ -1249,7 +1262,7 @@ void db_initialize_util(void){ sizeof(DMCGuardBif), (int (*)(const void *, const void *)) &cmp_guard_bif); match_pseudo_process_init(); - erts_smp_atomic_init(&trace_control_word, 0); + erts_smp_atomic32_init_nob(&trace_control_word, 0); } @@ -1703,6 +1716,7 @@ Eterm db_prog_match(Process *c_p, Binary *bprog, Process *current_scheduled; ErtsSchedulerData *esdp; Eterm (*bif)(Process*, ...); + Eterm bif_args[3]; int fail_label; int atomic_trace; #if HALFWORD_HEAP @@ -1733,14 +1747,14 @@ Eterm db_prog_match(Process *c_p, Binary *bprog, if (! atomic_trace) { \ erts_refc_inc(&bprog->refc, 2); \ erts_smp_proc_unlock((p), ERTS_PROC_LOCK_MAIN); \ - erts_smp_block_system(0); \ + erts_smp_thr_progress_block(); \ atomic_trace = !0; \ } \ } while (0) #define END_ATOMIC_TRACE(p) \ do { \ if (atomic_trace) { \ - erts_smp_release_system(); \ + erts_smp_thr_progress_unblock(); \ erts_smp_proc_lock((p), ERTS_PROC_LOCK_MAIN); \ if (erts_refc_dectest(&bprog->refc, 0) == 0) {\ erts_bin_free(bprog); \ @@ -1956,7 +1970,7 @@ restart: break; case matchCall0: bif = (Eterm (*)(Process*, ...)) *pc++; - t = (*bif)(build_proc); + t = (*bif)(build_proc, bif_args); if (is_non_value(t)) { if (do_catch) t = FAIL_TERM; @@ -1967,7 +1981,7 @@ restart: break; case matchCall1: bif = (Eterm (*)(Process*, ...)) *pc++; - t = (*bif)(build_proc, esp[-1]); + t = (*bif)(build_proc, esp-1); if (is_non_value(t)) { if (do_catch) t = FAIL_TERM; @@ -1978,7 +1992,9 @@ restart: break; case matchCall2: bif = (Eterm (*)(Process*, ...)) *pc++; - t = (*bif)(build_proc, esp[-1], esp[-2]); + bif_args[0] = esp[-1]; + bif_args[1] = esp[-2]; + t = (*bif)(build_proc, bif_args); if (is_non_value(t)) { if (do_catch) t = FAIL_TERM; @@ -1990,7 +2006,10 @@ restart: break; case matchCall3: bif = (Eterm (*)(Process*, ...)) *pc++; - t = (*bif)(build_proc, esp[-1], esp[-2], esp[-3]); + bif_args[0] = esp[-1]; + bif_args[1] = esp[-2]; + bif_args[2] = esp[-3]; + t = (*bif)(build_proc, bif_args); if (is_non_value(t)) { if (do_catch) t = FAIL_TERM; @@ -2845,7 +2864,9 @@ void* db_store_term_comp(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj) Uint new_sz = offset + db_size_dbterm_comp(tb, obj); byte* basep; DbTerm* newp; +#ifdef DEBUG byte* top; +#endif ASSERT(tb->compress); if (old != 0) { @@ -2867,7 +2888,10 @@ void* db_store_term_comp(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj) } newp->size = size_object(obj); - top = copy_to_comp(tb, obj, newp, new_sz); +#ifdef DEBUG + top = +#endif + copy_to_comp(tb, obj, newp, new_sz); ASSERT(top <= basep + new_sz); /* ToDo: Maybe realloc if ((basep+new_sz) - top) > WASTED_SPACE_LIMIT */ @@ -4969,7 +4993,7 @@ static Eterm match_spec_test(Process *p, Eterm against, Eterm spec, int trace) static Eterm seq_trace_fake(Process *p, Eterm arg1) { - Eterm result = seq_trace_info_1(p,arg1); + Eterm result = erl_seq_trace_info(p, arg1); if (is_tuple(result) && *tuple_val(result) == 2) { return (tuple_val(result))[2]; } diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h index bb1751d309..6a96e174e1 100644 --- a/erts/emulator/beam/erl_db_util.h +++ b/erts/emulator/beam/erl_db_util.h @@ -326,8 +326,10 @@ ERTS_GLB_INLINE int db_eq(DbTableCommon* tb, Eterm a, DbTerm* b) (T)->common.owner == (P)->id) /* Function prototypes */ -Eterm db_get_trace_control_word_0(Process *p); -Eterm db_set_trace_control_word_1(Process *p, Eterm val); +BIF_RETTYPE db_get_trace_control_word(Process* p); +BIF_RETTYPE db_set_trace_control_word(Process* p, Eterm tcw); +BIF_RETTYPE db_get_trace_control_word_0(BIF_ALIST_0); +BIF_RETTYPE db_set_trace_control_word_1(BIF_ALIST_1); void db_initialize_util(void); Eterm db_getkey(int keypos, Eterm obj); diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h index 401967a8de..ae0c9def90 100644 --- a/erts/emulator/beam/erl_driver.h +++ b/erts/emulator/beam/erl_driver.h @@ -28,6 +28,14 @@ # include "config.h" #endif +#define ERL_DRV_DEPRECATED_FUNC +#ifdef __GNUC__ +# if __GNUC__ >= 3 +# undef ERL_DRV_DEPRECATED_FUNC +# define ERL_DRV_DEPRECATED_FUNC __attribute__((deprecated)) +# endif +#endif + #ifdef SIZEOF_CHAR # define SIZEOF_CHAR_SAVED__ SIZEOF_CHAR # undef SIZEOF_CHAR @@ -582,8 +590,11 @@ EXTERN long driver_async(ErlDrvPort ix, void* async_data, void (*async_free)(void*)); - -EXTERN int driver_async_cancel(unsigned int key); +/* + * driver_async_cancel() is deprecated. It is scheduled for removal + * in OTP-R16. For more information see the erl_driver(3) documentation. + */ +EXTERN int driver_async_cancel(unsigned int key) ERL_DRV_DEPRECATED_FUNC; /* Locks the driver in the machine "forever", there is no unlock function. Note that this is almost never useful, as an open diff --git a/erts/emulator/beam/erl_drv_thread.c b/erts/emulator/beam/erl_drv_thread.c index dc578f6d2a..a49a155701 100644 --- a/erts/emulator/beam/erl_drv_thread.c +++ b/erts/emulator/beam/erl_drv_thread.c @@ -158,7 +158,9 @@ erl_drv_mutex_create(char *name) (sizeof(ErlDrvMutex) + (name ? sys_strlen(name) + 1 : 0))); if (dmtx) { - if (ethr_mutex_init(&dmtx->mtx) != 0) { + ethr_mutex_opt opt = ETHR_MUTEX_OPT_DEFAULT_INITER; + opt.posix_compliant = 1; + if (ethr_mutex_init_opt(&dmtx->mtx, &opt) != 0) { erts_free(ERTS_ALC_T_DRV_MTX, (void *) dmtx); dmtx = NULL; } @@ -226,7 +228,9 @@ erl_drv_cond_create(char *name) (sizeof(ErlDrvCond) + (name ? sys_strlen(name) + 1 : 0))); if (dcnd) { - if (ethr_cond_init(&dcnd->cnd) != 0) { + ethr_cond_opt opt = ETHR_COND_OPT_DEFAULT_INITER; + opt.posix_compliant = 1; + if (ethr_cond_init_opt(&dcnd->cnd, &opt) != 0) { erts_free(ERTS_ALC_T_DRV_CND, (void *) dcnd); dcnd = NULL; } diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index e3445bcdc5..c29352a227 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -315,7 +315,12 @@ erts_gc_after_bif_call(Process* p, Eterm result, Eterm* regs, Uint arity) if (is_non_value(result)) { if (p->freason == TRAP) { - cost = erts_garbage_collect(p, 0, p->def_arg_reg, p->arity); + #if HIPE + if (regs == NULL) { + regs = ERTS_PROC_GET_SCHDATA(p)->x_reg_array; + } + #endif + cost = erts_garbage_collect(p, 0, regs, p->arity); } else { cost = erts_garbage_collect(p, 0, regs, arity); } @@ -357,8 +362,6 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj) } erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); - erts_smp_locked_activity_begin(ERTS_ACTIVITY_GC); - ERTS_CHK_OFFHEAP(p); ErtsGcQuickSanityCheck(p); @@ -392,8 +395,6 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj) trace_gc(p, am_gc_end); } - erts_smp_locked_activity_end(ERTS_ACTIVITY_GC); - if (erts_system_monitor_long_gc != 0) { Uint ms2, s2, us2; Sint t; @@ -477,7 +478,6 @@ erts_garbage_collect_hibernate(Process* p) p->gcstatus = p->status; p->status = P_GARBING; erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); - erts_smp_locked_activity_begin(ERTS_ACTIVITY_GC); ErtsGcQuickSanityCheck(p); ASSERT(p->mbuf_sz == 0); ASSERT(p->mbuf == 0); @@ -591,7 +591,6 @@ erts_garbage_collect_hibernate(Process* p) erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); p->status = p->gcstatus; erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); - erts_smp_locked_activity_end(ERTS_ACTIVITY_GC); } @@ -616,7 +615,6 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, Uint lit_size) p->gcstatus = p->status; p->status = P_GARBING; erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); - erts_smp_locked_activity_begin(ERTS_ACTIVITY_GC); /* * We assume that the caller has already done a major collection @@ -719,7 +717,6 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, Uint lit_size) erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); p->status = p->gcstatus; erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); - erts_smp_locked_activity_end(ERTS_ACTIVITY_GC); } static int diff --git a/erts/emulator/beam/erl_goodfit_alloc.c b/erts/emulator/beam/erl_goodfit_alloc.c index 8322b233ac..e7d4ac2b67 100644 --- a/erts/emulator/beam/erl_goodfit_alloc.c +++ b/erts/emulator/beam/erl_goodfit_alloc.c @@ -190,16 +190,20 @@ erts_gfalc_start(GFAllctr_t *gfallctr, GFAllctrInit_t *gfinit, AllctrInit_t *init) { - GFAllctr_t nulled_state = {{0}}; - /* {{0}} is used instead of {0}, in order to avoid (an incorrect) gcc - warning. gcc warns if {0} is used as initializer of a struct when - the first member is a struct (not if, for example, the third member - is a struct). */ + struct { + int dummy; + GFAllctr_t allctr; + } zero = {0}; + /* The struct with a dummy element first is used in order to avoid (an + incorrect) gcc warning. gcc warns if {0} is used as initializer of + a struct when the first member is a struct (not if, for example, + the third member is a struct). */ + Allctr_t *allctr = (Allctr_t *) gfallctr; - init->sbmbct = 0; /* Small mbc not yet supported by goodfit */ + sys_memcpy((void *) gfallctr, (void *) &zero.allctr, sizeof(GFAllctr_t)); - sys_memcpy((void *) gfallctr, (void *) &nulled_state, sizeof(GFAllctr_t)); + init->sbmbct = 0; /* Small mbc not yet supported by goodfit */ allctr->mbc_header_size = sizeof(Carrier_t); allctr->min_mbc_size = MIN_MBC_SZ; diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index e5fd2547a0..7c047891d9 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -42,6 +42,9 @@ #include "erl_misc_utils.h" #include "packet_parser.h" #include "erl_cpu_topology.h" +#include "erl_thr_progress.h" +#include "erl_thr_queue.h" +#include "erl_async.h" #ifdef HIPE #include "hipe_mode_switch.h" /* for hipe_mode_switch_init() */ @@ -68,8 +71,11 @@ static void erl_init(int ncpu); #define ERTS_MIN_COMPAT_REL 7 +static erts_atomic_t exiting; + #ifdef ERTS_SMP -erts_smp_atomic_t erts_writing_erl_crash_dump; +erts_smp_atomic32_t erts_writing_erl_crash_dump; +erts_tsd_key_t erts_is_crash_dumping_key; #else volatile int erts_writing_erl_crash_dump = 0; #endif @@ -98,8 +104,6 @@ int erts_backtrace_depth; /* How many functions to show in a backtrace * in error codes. */ -int erts_async_max_threads; /* number of threads for async support */ -int erts_async_thread_suggested_stack_size; erts_smp_atomic32_t erts_max_gen_gcs; Eterm erts_error_logger_warnings; /* What to map warning logs to, am_error, @@ -126,6 +130,8 @@ int erts_modified_timing_level; int erts_no_crash_dump = 0; /* Use -d to suppress crash dump. */ +int erts_no_line_info = 0; /* -L: Don't load line information */ + /* * Other global variables. */ @@ -244,10 +250,6 @@ erl_init(int ncpu) { init_benchmarking(); -#ifdef ERTS_SMP - erts_system_block_init(); -#endif - erts_init_monitors(); erts_init_gc(); erts_init_time(); @@ -257,6 +259,8 @@ erl_init(int ncpu) no_schedulers, no_schedulers_online); erts_init_cpu_topology(); /* Must be after init_scheduling */ + erts_alloc_late_init(); + H_MIN_SIZE = erts_next_heap_size(H_MIN_SIZE, 0); BIN_VH_MIN_SIZE = erts_next_heap_size(BIN_VH_MIN_SIZE, 0); @@ -278,6 +282,7 @@ erl_init(int ncpu) erts_init_node_tables(); init_dist(); erl_drv_thr_init(); + erts_init_async(); init_io(); init_copy(); init_load(); @@ -323,7 +328,7 @@ init_shared_memory(int argc, char **argv) #endif global_gen_gcs = 0; - global_max_gen_gcs = (Uint16) erts_smp_atomic32_read(&erts_max_gen_gcs); + global_max_gen_gcs = (Uint16) erts_smp_atomic32_read_nob(&erts_max_gen_gcs); global_gc_flags = erts_default_process_flags; erts_global_offheap.mso = NULL; @@ -604,6 +609,8 @@ early_init(int *argc, char **argv) /* int max_main_threads; int max_reader_groups; int reader_groups; + char envbuf[21]; /* enough for any 64-bit integer */ + size_t envbufsz; use_multi_run_queue = 1; erts_printf_eterm_func = erts_printf_term; @@ -641,17 +648,23 @@ early_init(int *argc, char **argv) /* erts_use_r9_pids_ports = 0; erts_sys_pre_init(); + erts_atomic_init_nob(&exiting, 0); +#ifdef ERTS_SMP + erts_thr_progress_pre_init(); +#endif #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_init(); #endif #ifdef ERTS_SMP - erts_smp_atomic_init(&erts_writing_erl_crash_dump, 0L); + erts_smp_atomic32_init_nob(&erts_writing_erl_crash_dump, 0L); + erts_tsd_key_create(&erts_is_crash_dumping_key); #else erts_writing_erl_crash_dump = 0; #endif - erts_smp_atomic32_init(&erts_max_gen_gcs, (erts_aint32_t) ((Uint16) -1)); + erts_smp_atomic32_init_nob(&erts_max_gen_gcs, + (erts_aint32_t) ((Uint16) -1)); erts_pre_init_process(); #if defined(USE_THREADS) && !defined(ERTS_SMP) @@ -670,6 +683,16 @@ early_init(int *argc, char **argv) /* schdlrs = no_schedulers; schdlrs_onln = no_schedulers_online; + envbufsz = sizeof(envbuf); + + /* erts_sys_getenv() not initialized yet; need erts_sys_getenv__() */ + if (erts_sys_getenv__("ERL_THREAD_POOL_SIZE", envbuf, &envbufsz) == 0) + erts_async_max_threads = atoi(envbuf); + else + erts_async_max_threads = 0; + if (erts_async_max_threads > ERTS_MAX_NO_OF_ASYNC_THREADS) + erts_async_max_threads = ERTS_MAX_NO_OF_ASYNC_THREADS; + if (argc && argv) { int i = 1; while (i < *argc) { @@ -697,6 +720,20 @@ early_init(int *argc, char **argv) /* } break; } + case 'A': { + /* set number of threads in thread pool */ + char *arg = get_arg(argv[i]+2, argv[i+1], &i); + if (((erts_async_max_threads = atoi(arg)) < 0) || + (erts_async_max_threads > ERTS_MAX_NO_OF_ASYNC_THREADS)) { + erts_fprintf(stderr, + "bad number of async threads %s\n", + arg); + erts_usage(); + VERBOSE(DEBUG_SYSTEM, ("using %d async-threads\n", + erts_async_max_threads)); + } + break; + } case 'S' : { int tot, onln; char *arg = get_arg(argv[i]+2, argv[i+1], &i); @@ -762,11 +799,29 @@ early_init(int *argc, char **argv) /* erts_no_schedulers = (Uint) no_schedulers; #endif + erts_early_init_scheduling(no_schedulers); + alloc_opts.ncpu = ncpu; erts_alloc_init(argc, argv, &alloc_opts); /* Handles (and removes) -M flags. */ /* Require allocators */ - erts_early_init_scheduling(); +#ifdef ERTS_SMP + /* + * Thread progress management: + * + * * Managed threads: + * ** Scheduler threads (see erl_process.c) + * ** Aux thread (see erl_process.c) + * ** Sys message dispatcher thread (see erl_trace.c) + * + * * Unmanaged threads that need to register: + * ** Async threads (see erl_async.c) + */ + erts_thr_progress_init(no_schedulers, + no_schedulers+2, + erts_async_max_threads); +#endif + erts_thr_q_init(); erts_init_utils(); erts_early_init_cpu_topology(no_schedulers, &max_main_threads, @@ -846,7 +901,6 @@ erl_start(int argc, char **argv) int have_break_handler = 1; char envbuf[21]; /* enough for any 64-bit integer */ size_t envbufsz; - int async_max_threads = erts_async_max_threads; int ncpu = early_init(&argc, argv); envbufsz = sizeof(envbuf); @@ -858,12 +912,8 @@ erl_start(int argc, char **argv) envbufsz = sizeof(envbuf); if (erts_sys_getenv("ERL_FULLSWEEP_AFTER", envbuf, &envbufsz) == 0) { Uint16 max_gen_gcs = atoi(envbuf); - erts_smp_atomic32_set(&erts_max_gen_gcs, (erts_aint32_t) max_gen_gcs); - } - - envbufsz = sizeof(envbuf); - if (erts_sys_getenv("ERL_THREAD_POOL_SIZE", envbuf, &envbufsz) == 0) { - async_max_threads = atoi(envbuf); + erts_smp_atomic32_set_nob(&erts_max_gen_gcs, + (erts_aint32_t) max_gen_gcs); } #if (defined(__APPLE__) && defined(__MACH__)) || defined(__DARWIN__) @@ -934,7 +984,9 @@ erl_start(int argc, char **argv) case 'l': display_loads++; break; - + case 'L': + erts_no_line_info = 1; + break; case 'v': #ifdef DEBUG if (argv[i][2] == '\0') { @@ -1293,17 +1345,8 @@ erl_start(int argc, char **argv) break; } - case 'A': - /* set number of threads in thread pool */ - arg = get_arg(argv[i]+2, argv[i+1], &i); - if (((async_max_threads = atoi(arg)) < 0) || - (async_max_threads > ERTS_MAX_NO_OF_ASYNC_THREADS)) { - erts_fprintf(stderr, "bad number of async threads %s\n", arg); - erts_usage(); - } - - VERBOSE(DEBUG_SYSTEM, ("using %d async-threads\n", - async_max_threads)); + case 'A': /* Was handled in early init just read past it */ + (void) get_arg(argv[i]+2, argv[i+1], &i); break; case 'a': @@ -1392,10 +1435,6 @@ erl_start(int argc, char **argv) i++; } -#ifdef USE_THREADS - erts_async_max_threads = async_max_threads; -#endif - /* Delayed check of +P flag */ if (erts_max_processes < ERTS_MIN_PROCESSES || erts_max_processes > ERTS_MAX_PROCESSES @@ -1441,6 +1480,10 @@ erl_start(int argc, char **argv) erts_sys_main_thread(); /* May or may not return! */ #else erts_thr_set_main_status(1, 1); +#if ERTS_USE_ASYNC_READY_Q + erts_get_scheduler_data()->aux_work_data.async_ready.queue + = erts_get_async_ready_queue(1); +#endif set_main_stack_size(); process_main(); #endif @@ -1466,6 +1509,29 @@ __decl_noreturn void erts_thr_fatal_error(int err, char *what) static void system_cleanup(int exit_code) { + /* + * Make sure only one thread exits the runtime system. + */ + if (erts_atomic_inc_read_nob(&exiting) != 1) { + /* + * Another thread is currently exiting the system; + * wait for it to do its job. + */ +#ifdef ERTS_SMP + if (erts_thr_progress_is_managed_thread()) { + /* + * The exiting thread might be waiting for + * us to block; need to update status... + */ + erts_thr_progress_active(NULL, 0); + erts_thr_progress_prepare_wait(NULL); + } +#endif + /* Wait forever... */ + while (1) + erts_milli_sleep(10000000); + } + /* No cleanup wanted if ... * 1. we are about to do an abnormal exit * 2. we haven't finished initializing, or @@ -1485,7 +1551,6 @@ system_cleanup(int exit_code) #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_check_exact(NULL, 0); #endif - erts_smp_block_system(ERTS_BS_FLG_ALLOW_GC); /* We never release it... */ #endif #ifdef HYBRID @@ -1514,17 +1579,7 @@ system_cleanup(int exit_code) erts_cleanup_incgc(); #endif -#if defined(USE_THREADS) - exit_async(); -#endif -#if HAVE_ERTS_MSEG - erts_mseg_exit(); -#endif - - /* - * A lot more cleaning could/should have been done... - */ - + erts_exit_flush_async(); } /* @@ -1541,10 +1596,10 @@ __decl_noreturn void erl_exit0(char *file, int line, int n, char *fmt,...) va_start(args, fmt); - save_statistics(); - system_cleanup(n); + save_statistics(); + an = abs(n); if (erts_mtrace_enabled) @@ -1581,10 +1636,10 @@ __decl_noreturn void erl_exit(int n, char *fmt,...) va_start(args, fmt); - save_statistics(); - system_cleanup(n); + save_statistics(); + an = abs(n); if (erts_mtrace_enabled) diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c index 587d82f2bb..44da6b6c51 100644 --- a/erts/emulator/beam/erl_lock_check.c +++ b/erts/emulator/beam/erl_lock_check.c @@ -110,10 +110,6 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "fun_tab", NULL }, { "environ", NULL }, #endif - { "asyncq", "address" }, -#ifndef ERTS_SMP - { "async_ready", NULL }, -#endif { "efile_drv", "address" }, #if defined(ENABLE_CHILD_WAITER_THREAD) || defined(ERTS_SMP) { "child_status", NULL }, @@ -125,7 +121,7 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "drv_ev_state", "address" }, { "safe_hash", "address" }, { "pollset_rm_list", NULL }, - { "removed_fd_pre_alloc_lock", NULL }, + { "removed_fd_pre_alloc_lock", "address" }, { "state_prealloc", NULL }, { "schdlr_sspnd", NULL }, { "run_queue", "address" }, @@ -138,6 +134,7 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "alcu_init_atoms", NULL }, { "mseg_init_atoms", NULL }, { "drv_tsd", NULL }, + { "async_enq_mtx", NULL }, #ifdef ERTS_SMP { "sys_msg_q", NULL }, { "atom_tab", NULL }, @@ -151,10 +148,8 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "mtrace_op", NULL }, { "instr_x", NULL }, { "instr", NULL }, - { "fix_alloc", "index" }, { "alcu_allocator", "index" }, { "sbmbc_alloc", "index" }, - { "alcu_delayed_free", "index" }, { "mseg", NULL }, #if HALFWORD_HEAP { "pmmap", NULL }, @@ -175,15 +170,12 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "timeofday", NULL }, { "breakpoints", NULL }, { "pollsets_lock", NULL }, - { "async_id", NULL }, { "pix_lock", "address" }, { "run_queues_lists", NULL }, - { "misc_aux_work_queue", "index" }, - { "misc_aux_work_pre_alloc_lock", "address" }, { "sched_stat", NULL }, { "run_queue_sleep_list", "address" }, #endif - { "alloc_thr_ix_lock", NULL }, + { "async_init_mtx", NULL }, #ifdef ERTS_SMP { "proc_lck_qs_alloc", NULL }, #endif diff --git a/erts/emulator/beam/erl_monitors.c b/erts/emulator/beam/erl_monitors.c index 9751b5d77c..1a84950120 100644 --- a/erts/emulator/beam/erl_monitors.c +++ b/erts/emulator/beam/erl_monitors.c @@ -125,7 +125,7 @@ static ErtsMonitor *create_monitor(Uint type, Eterm ref, Eterm pid, Eterm name) } else { n = (ErtsMonitor *) erts_alloc(ERTS_ALC_T_MONITOR_LH, mon_size*sizeof(Uint)); - erts_smp_atomic_add(&tot_link_lh_size, mon_size*sizeof(Uint)); + erts_smp_atomic_add_nob(&tot_link_lh_size, mon_size*sizeof(Uint)); } hp = n->heap; @@ -156,7 +156,7 @@ static ErtsLink *create_link(Uint type, Eterm pid) } else { n = (ErtsLink *) erts_alloc(ERTS_ALC_T_NLINK_LH, lnk_size*sizeof(Uint)); - erts_smp_atomic_add(&tot_link_lh_size, lnk_size*sizeof(Uint)); + erts_smp_atomic_add_nob(&tot_link_lh_size, lnk_size*sizeof(Uint)); } hp = n->heap; @@ -191,13 +191,13 @@ static ErtsSuspendMonitor *create_suspend_monitor(Eterm pid) void erts_init_monitors(void) { - erts_smp_atomic_init(&tot_link_lh_size, 0); + erts_smp_atomic_init_nob(&tot_link_lh_size, 0); } Uint erts_tot_link_lh_size(void) { - return (Uint) erts_smp_atomic_read(&tot_link_lh_size); + return (Uint) erts_smp_atomic_read_nob(&tot_link_lh_size); } void erts_destroy_monitor(ErtsMonitor *mon) @@ -222,7 +222,7 @@ void erts_destroy_monitor(ErtsMonitor *mon) erts_free(ERTS_ALC_T_MONITOR_SH, (void *) mon); } else { erts_free(ERTS_ALC_T_MONITOR_LH, (void *) mon); - erts_smp_atomic_add(&tot_link_lh_size, -1*mon_size*sizeof(Uint)); + erts_smp_atomic_add_nob(&tot_link_lh_size, -1*mon_size*sizeof(Uint)); } } @@ -244,7 +244,7 @@ void erts_destroy_link(ErtsLink *lnk) erts_free(ERTS_ALC_T_NLINK_SH, (void *) lnk); } else { erts_free(ERTS_ALC_T_NLINK_LH, (void *) lnk); - erts_smp_atomic_add(&tot_link_lh_size, -1*lnk_size*sizeof(Uint)); + erts_smp_atomic_add_nob(&tot_link_lh_size, -1*lnk_size*sizeof(Uint)); } } @@ -948,8 +948,10 @@ static void erts_dump_links(ErtsLink *root, int indent) erts_destroy_tmp_dsbuf(dsbufp); } -Eterm erts_debug_dump_monitors_1(Process *p, Eterm pid) +Eterm erts_debug_dump_monitors_1(BIF_ALIST_1) { + Process *p = BIF_P; + Eterm pid = BIF_ARG_1; Process *rp; DistEntry *dep; rp = erts_pid2proc(p, ERTS_PROC_LOCK_MAIN, pid, ERTS_PROC_LOCK_LINK); @@ -976,8 +978,10 @@ Eterm erts_debug_dump_monitors_1(Process *p, Eterm pid) } } -Eterm erts_debug_dump_links_1(Process *p, Eterm pid) +Eterm erts_debug_dump_links_1(BIF_ALIST_1) { + Process *p = BIF_P; + Eterm pid = BIF_ARG_1; Process *rp; DistEntry *dep; if (is_internal_port(pid)) { diff --git a/erts/emulator/beam/erl_mtrace.c b/erts/emulator/beam/erl_mtrace.c index b1478758a1..358c67bf20 100644 --- a/erts/emulator/beam/erl_mtrace.c +++ b/erts/emulator/beam/erl_mtrace.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2003-2010. All Rights Reserved. + * Copyright Ericsson AB 2003-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -503,12 +503,6 @@ write_trace_header(char *nodename, char *pid, char *hostname) case ERTS_ALC_A_SYSTEM: PUT_UI16(tracep, ERTS_MTRACE_SEGMENT_ID); break; - case ERTS_ALC_A_FIXED_SIZE: - if (erts_allctrs_info[ERTS_FIX_CORE_ALLOCATOR].enabled) - PUT_UI16(tracep, ERTS_FIX_CORE_ALLOCATOR); - else - PUT_UI16(tracep, ERTS_ALC_A_SYSTEM); - break; default: PUT_UI16(tracep, ERTS_MTRACE_SEGMENT_ID); break; diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 6e7ac43676..62798bb2c1 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -32,6 +32,7 @@ #include "error.h" #include "big.h" #include "beam_bp.h" +#include "erl_thr_progress.h" #include <limits.h> #include <stddef.h> /* offsetof */ @@ -130,10 +131,13 @@ static void pre_nif_noproc(ErlNifEnv* env, struct erl_module_nif* mod_nif) env->tmp_obj_list = NULL; } -/* Temporary object header, auto-deallocated when NIF returns. */ +/* Temporary object header, auto-deallocated when NIF returns + * or when independent environment is cleared. + */ struct enif_tmp_obj_t { struct enif_tmp_obj_t* next; void (*dtor)(struct enif_tmp_obj_t*); + ErtsAlcType_t allocator; /*char data[];*/ }; @@ -244,7 +248,7 @@ ErlNifEnv* enif_alloc_env(void) msg_env->env.hp_end = phony_heap; msg_env->env.heap_frag = NULL; msg_env->env.mod_nif = NULL; - msg_env->env.tmp_obj_list = (struct enif_tmp_obj_t*) 1; /* invalid non-NULL */ + msg_env->env.tmp_obj_list = NULL; msg_env->env.proc = &msg_env->phony_proc; memset(&msg_env->phony_proc, 0, sizeof(Process)); HEAP_START(&msg_env->phony_proc) = phony_heap; @@ -289,6 +293,7 @@ void enif_clear_env(ErlNifEnv* env) menv->env.hp = menv->env.hp_end = HEAP_TOP(p); ASSERT(!is_offheap(&MSO(p))); + free_tmp_objs(env); } int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, ErlNifEnv* msg_env, ERL_NIF_TERM msg) @@ -435,24 +440,36 @@ int enif_is_exception(ErlNifEnv* env, ERL_NIF_TERM term) return term == THE_NON_VALUE; } +int enif_is_number(ErlNifEnv* env, ERL_NIF_TERM term) +{ + return is_number(term); +} + +static ERTS_INLINE int is_proc_bound(ErlNifEnv* env) +{ + return env->mod_nif != NULL; +} + static void aligned_binary_dtor(struct enif_tmp_obj_t* obj) { - erts_free_aligned_binary_bytes_extra((byte*)obj,ERTS_ALC_T_TMP); + erts_free_aligned_binary_bytes_extra((byte*)obj, obj->allocator); } int enif_inspect_binary(ErlNifEnv* env, Eterm bin_term, ErlNifBinary* bin) { + ErtsAlcType_t allocator = is_proc_bound(env) ? ERTS_ALC_T_TMP : ERTS_ALC_T_NIF; union { struct enif_tmp_obj_t* tmp; byte* raw_ptr; }u; u.tmp = NULL; - bin->data = erts_get_aligned_binary_bytes_extra(bin_term, &u.raw_ptr, ERTS_ALC_T_TMP, + bin->data = erts_get_aligned_binary_bytes_extra(bin_term, &u.raw_ptr, allocator, sizeof(struct enif_tmp_obj_t)); if (bin->data == NULL) { return 0; } if (u.tmp != NULL) { + u.tmp->allocator = allocator; u.tmp->next = env->tmp_obj_list; u.tmp->dtor = &aligned_binary_dtor; env->tmp_obj_list = u.tmp; @@ -466,12 +483,13 @@ int enif_inspect_binary(ErlNifEnv* env, Eterm bin_term, ErlNifBinary* bin) static void tmp_alloc_dtor(struct enif_tmp_obj_t* obj) { - erts_free(ERTS_ALC_T_TMP, obj); + erts_free(obj->allocator, obj); } int enif_inspect_iolist_as_binary(ErlNifEnv* env, Eterm term, ErlNifBinary* bin) { struct enif_tmp_obj_t* tobj; + ErtsAlcType_t allocator; Uint sz; if (is_binary(term)) { return enif_inspect_binary(env,term,bin); @@ -486,8 +504,10 @@ int enif_inspect_iolist_as_binary(ErlNifEnv* env, Eterm term, ErlNifBinary* bin) if (erts_iolist_size(term, &sz)) { return 0; } - - tobj = erts_alloc(ERTS_ALC_T_TMP, sz + sizeof(struct enif_tmp_obj_t)); + + allocator = is_proc_bound(env) ? ERTS_ALC_T_TMP : ERTS_ALC_T_NIF; + tobj = erts_alloc(allocator, sz + sizeof(struct enif_tmp_obj_t)); + tobj->allocator = allocator; tobj->next = env->tmp_obj_list; tobj->dtor = &tmp_alloc_dtor; env->tmp_obj_list = tobj; @@ -676,6 +696,7 @@ Eterm enif_make_sub_binary(ErlNifEnv* env, ERL_NIF_TERM bin_term, ErlSubBin* sb; Eterm orig; Uint offset, bit_offset, bit_size; +#ifdef DEBUG unsigned src_size; ASSERT(is_binary(bin_term)); @@ -683,6 +704,7 @@ Eterm enif_make_sub_binary(ErlNifEnv* env, ERL_NIF_TERM bin_term, ASSERT(pos <= src_size); ASSERT(size <= src_size); ASSERT(pos + size <= src_size); +#endif sb = (ErlSubBin*) alloc_heap(env, ERL_SUB_BIN_SIZE); ERTS_GET_REAL_BIN(bin_term, orig, offset, bit_offset, bit_size); sb->thing_word = HEADER_SUB_BIN; @@ -1021,6 +1043,29 @@ void enif_system_info(ErlNifSysInfo *sip, size_t si_size) driver_system_info(sip, si_size); } +int enif_make_reverse_list(ErlNifEnv* env, ERL_NIF_TERM term, ERL_NIF_TERM *list) { + Eterm *listptr, ret = NIL, *hp; + + if (is_nil(term)) { + *list = term; + return 1; + } + + ret = NIL; + + while (is_not_nil(term)) { + if (is_not_list(term)) { + return 0; + } + hp = alloc_heap(env, 2); + listptr = list_val(term); + ret = CONS(hp, CAR(listptr), ret); + term = CDR(listptr); + } + *list = ret; + return 1; +} + ErlNifMutex* enif_mutex_create(char *name) { return erl_drv_mutex_create(name); } void enif_mutex_destroy(ErlNifMutex *mtx) { erl_drv_mutex_destroy(mtx); } @@ -1160,7 +1205,7 @@ enif_open_resource_type(ErlNifEnv* env, ErlNifResourceFlags op = flags; Eterm module_am, name_am; - ASSERT(erts_smp_is_system_blocked(0)); + ASSERT(erts_smp_thr_progress_is_blocking()); ASSERT(module_str == NULL); /* for now... */ module_am = make_atom(env->mod_nif->mod->module); name_am = enif_make_atom(env, name_str); @@ -1454,7 +1499,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) /* Block system (is this the right place to do it?) */ erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(0); + erts_smp_thr_progress_block(); /* Find calling module */ ASSERT(BIF_P->current != NULL); @@ -1643,7 +1688,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) erts_sys_ddll_free_error(&errdesc); } - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_free(ERTS_ALC_T_TMP, lib_name); BIF_RET(ret); @@ -1655,7 +1700,7 @@ erts_unload_nif(struct erl_module_nif* lib) { ErlNifResourceType* rt; ErlNifResourceType* next; - ASSERT(erts_smp_is_system_blocked(0)); + ASSERT(erts_smp_thr_progress_is_blocking()); ASSERT(lib != NULL); ASSERT(lib->mod != NULL); for (rt = resource_type_list.next; @@ -1715,8 +1760,10 @@ struct readonly_check_t }; static void add_readonly_check(ErlNifEnv* env, unsigned char* ptr, unsigned sz) { - struct readonly_check_t* obj = erts_alloc(ERTS_ALC_T_TMP, + ErtsAlcType_t allocator = is_proc_bound(env) ? ERTS_ALC_T_TMP : ERTS_ALC_T_NIF; + struct readonly_check_t* obj = erts_alloc(allocator, sizeof(struct readonly_check_t)); + obj->hdr.allocator = allocator; obj->hdr.next = env->tmp_obj_list; env->tmp_obj_list = &obj->hdr; obj->hdr.dtor = &readonly_check_dtor; @@ -1733,7 +1780,7 @@ static void readonly_check_dtor(struct enif_tmp_obj_t* o) " %x != %x\r\nABORTING\r\n", chksum, obj->checksum); abort(); } - erts_free(ERTS_ALC_T_TMP, obj); + erts_free(obj->hdr.allocator, obj); } static unsigned calc_checksum(unsigned char* ptr, unsigned size) { diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h index d028567faf..fea527f954 100644 --- a/erts/emulator/beam/erl_nif.h +++ b/erts/emulator/beam/erl_nif.h @@ -32,9 +32,10 @@ ** 2.0: R14A ** 2.1: R14B02 "vm_variant" ** 2.2: R14B03 enif_is_exception +** 2.3: R15 enif_make_reverse_list */ #define ERL_NIF_MAJOR_VERSION 2 -#define ERL_NIF_MINOR_VERSION 2 +#define ERL_NIF_MINOR_VERSION 3 #include <stdlib.h> diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h index c991b61abe..6396af09d0 100644 --- a/erts/emulator/beam/erl_nif_api_funcs.h +++ b/erts/emulator/beam/erl_nif_api_funcs.h @@ -136,6 +136,8 @@ ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_int64,(ErlNifEnv*, ErlNifSInt64)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_uint64,(ErlNifEnv*, ErlNifUInt64)); #endif ERL_NIF_API_FUNC_DECL(int,enif_is_exception,(ErlNifEnv*, ERL_NIF_TERM term)); +ERL_NIF_API_FUNC_DECL(int,enif_make_reverse_list,(ErlNifEnv*, ERL_NIF_TERM term, ERL_NIF_TERM *list)); +ERL_NIF_API_FUNC_DECL(int,enif_is_number,(ErlNifEnv*, ERL_NIF_TERM term)); /* ** Add new entries here to keep compatibility on Windows!!! @@ -256,12 +258,207 @@ ERL_NIF_API_FUNC_DECL(int,enif_is_exception,(ErlNifEnv*, ERL_NIF_TERM term)); #endif # define enif_is_exception ERL_NIF_API_FUNC_MACRO(enif_is_exception) +# define enif_make_reverse_list ERL_NIF_API_FUNC_MACRO(enif_make_reverse_list) +# define enif_is_number ERL_NIF_API_FUNC_MACRO(enif_is_number) /* ** Add new entries here */ #endif + +#if defined(__GNUC__) && !(defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_)) + +/* Inline functions for compile time type checking of arguments to + variadic functions. +*/ + +# define ERL_NIF_INLINE __inline__ + +static ERL_NIF_INLINE ERL_NIF_TERM enif_make_tuple1(ErlNifEnv* env, + ERL_NIF_TERM e1) +{ + return enif_make_tuple(env, 1, e1); +} + +static ERL_NIF_INLINE ERL_NIF_TERM enif_make_tuple2(ErlNifEnv* env, + ERL_NIF_TERM e1, + ERL_NIF_TERM e2) +{ + return enif_make_tuple(env, 2, e1, e2); +} + +static ERL_NIF_INLINE ERL_NIF_TERM enif_make_tuple3(ErlNifEnv* env, + ERL_NIF_TERM e1, + ERL_NIF_TERM e2, + ERL_NIF_TERM e3) +{ + return enif_make_tuple(env, 3, e1, e2, e3); +} + +static ERL_NIF_INLINE ERL_NIF_TERM enif_make_tuple4(ErlNifEnv* env, + ERL_NIF_TERM e1, + ERL_NIF_TERM e2, + ERL_NIF_TERM e3, + ERL_NIF_TERM e4) +{ + return enif_make_tuple(env, 4, e1, e2, e3, e4); +} + +static ERL_NIF_INLINE ERL_NIF_TERM enif_make_tuple5(ErlNifEnv* env, + ERL_NIF_TERM e1, + ERL_NIF_TERM e2, + ERL_NIF_TERM e3, + ERL_NIF_TERM e4, + ERL_NIF_TERM e5) +{ + return enif_make_tuple(env, 5, e1, e2, e3, e4, e5); +} + +static ERL_NIF_INLINE ERL_NIF_TERM enif_make_tuple6(ErlNifEnv* env, + ERL_NIF_TERM e1, + ERL_NIF_TERM e2, + ERL_NIF_TERM e3, + ERL_NIF_TERM e4, + ERL_NIF_TERM e5, + ERL_NIF_TERM e6) +{ + return enif_make_tuple(env, 6, e1, e2, e3, e4, e5, e6); +} + +static ERL_NIF_INLINE ERL_NIF_TERM enif_make_tuple7(ErlNifEnv* env, + ERL_NIF_TERM e1, + ERL_NIF_TERM e2, + ERL_NIF_TERM e3, + ERL_NIF_TERM e4, + ERL_NIF_TERM e5, + ERL_NIF_TERM e6, + ERL_NIF_TERM e7) +{ + return enif_make_tuple(env, 7, e1, e2, e3, e4, e5, e6, e7); +} + +static ERL_NIF_INLINE ERL_NIF_TERM enif_make_tuple8(ErlNifEnv* env, + ERL_NIF_TERM e1, + ERL_NIF_TERM e2, + ERL_NIF_TERM e3, + ERL_NIF_TERM e4, + ERL_NIF_TERM e5, + ERL_NIF_TERM e6, + ERL_NIF_TERM e7, + ERL_NIF_TERM e8) +{ + return enif_make_tuple(env, 8, e1, e2, e3, e4, e5, e6, e7, e8); +} + +static ERL_NIF_INLINE ERL_NIF_TERM enif_make_tuple9(ErlNifEnv* env, + ERL_NIF_TERM e1, + ERL_NIF_TERM e2, + ERL_NIF_TERM e3, + ERL_NIF_TERM e4, + ERL_NIF_TERM e5, + ERL_NIF_TERM e6, + ERL_NIF_TERM e7, + ERL_NIF_TERM e8, + ERL_NIF_TERM e9) +{ + return enif_make_tuple(env, 9, e1, e2, e3, e4, e5, e6, e7, e8, e9); +} + +static ERL_NIF_INLINE ERL_NIF_TERM enif_make_list1(ErlNifEnv* env, + ERL_NIF_TERM e1) +{ + return enif_make_list(env, 1, e1); +} + +static ERL_NIF_INLINE ERL_NIF_TERM enif_make_list2(ErlNifEnv* env, + ERL_NIF_TERM e1, + ERL_NIF_TERM e2) +{ + return enif_make_list(env, 2, e1, e2); +} + +static ERL_NIF_INLINE ERL_NIF_TERM enif_make_list3(ErlNifEnv* env, + ERL_NIF_TERM e1, + ERL_NIF_TERM e2, + ERL_NIF_TERM e3) +{ + return enif_make_list(env, 3, e1, e2, e3); +} + +static ERL_NIF_INLINE ERL_NIF_TERM enif_make_list4(ErlNifEnv* env, + ERL_NIF_TERM e1, + ERL_NIF_TERM e2, + ERL_NIF_TERM e3, + ERL_NIF_TERM e4) +{ + return enif_make_list(env, 4, e1, e2, e3, e4); +} + +static ERL_NIF_INLINE ERL_NIF_TERM enif_make_list5(ErlNifEnv* env, + ERL_NIF_TERM e1, + ERL_NIF_TERM e2, + ERL_NIF_TERM e3, + ERL_NIF_TERM e4, + ERL_NIF_TERM e5) +{ + return enif_make_list(env, 5, e1, e2, e3, e4, e5); +} + +static ERL_NIF_INLINE ERL_NIF_TERM enif_make_list6(ErlNifEnv* env, + ERL_NIF_TERM e1, + ERL_NIF_TERM e2, + ERL_NIF_TERM e3, + ERL_NIF_TERM e4, + ERL_NIF_TERM e5, + ERL_NIF_TERM e6) +{ + return enif_make_list(env, 6, e1, e2, e3, e4, e5, e6); +} + +static ERL_NIF_INLINE ERL_NIF_TERM enif_make_list7(ErlNifEnv* env, + ERL_NIF_TERM e1, + ERL_NIF_TERM e2, + ERL_NIF_TERM e3, + ERL_NIF_TERM e4, + ERL_NIF_TERM e5, + ERL_NIF_TERM e6, + ERL_NIF_TERM e7) +{ + return enif_make_list(env, 7, e1, e2, e3, e4, e5, e6, e7); +} + +static ERL_NIF_INLINE ERL_NIF_TERM enif_make_list8(ErlNifEnv* env, + ERL_NIF_TERM e1, + ERL_NIF_TERM e2, + ERL_NIF_TERM e3, + ERL_NIF_TERM e4, + ERL_NIF_TERM e5, + ERL_NIF_TERM e6, + ERL_NIF_TERM e7, + ERL_NIF_TERM e8) +{ + return enif_make_list(env, 8, e1, e2, e3, e4, e5, e6, e7, e8); +} + +static ERL_NIF_INLINE ERL_NIF_TERM enif_make_list9(ErlNifEnv* env, + ERL_NIF_TERM e1, + ERL_NIF_TERM e2, + ERL_NIF_TERM e3, + ERL_NIF_TERM e4, + ERL_NIF_TERM e5, + ERL_NIF_TERM e6, + ERL_NIF_TERM e7, + ERL_NIF_TERM e8, + ERL_NIF_TERM e9) +{ + return enif_make_list(env, 9, e1, e2, e3, e4, e5, e6, e7, e8, e9); +} + +# undef ERL_NIF_INLINE + +#else /* fallback with macros */ + #ifndef enif_make_list1 # define enif_make_list1(ENV,E1) enif_make_list(ENV,1,E1) # define enif_make_list2(ENV,E1,E2) enif_make_list(ENV,2,E1,E2) @@ -281,6 +478,11 @@ ERL_NIF_API_FUNC_DECL(int,enif_is_exception,(ErlNifEnv*, ERL_NIF_TERM term)); # define enif_make_tuple7(ENV,E1,E2,E3,E4,E5,E6,E7) enif_make_tuple(ENV,7,E1,E2,E3,E4,E5,E6,E7) # define enif_make_tuple8(ENV,E1,E2,E3,E4,E5,E6,E7,E8) enif_make_tuple(ENV,8,E1,E2,E3,E4,E5,E6,E7,E8) # define enif_make_tuple9(ENV,E1,E2,E3,E4,E5,E6,E7,E8,E9) enif_make_tuple(ENV,9,E1,E2,E3,E4,E5,E6,E7,E8,E9) +#endif + +#endif /* __GNUC__ && !WIN32 */ + +#ifndef enif_make_pid # define enif_make_pid(ENV, PID) ((const ERL_NIF_TERM)((PID)->pid)) diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c index 6daa127d23..908ba755ed 100644 --- a/erts/emulator/beam/erl_node_tables.c +++ b/erts/emulator/beam/erl_node_tables.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2010. All Rights Reserved. + * Copyright Ericsson AB 2001-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -26,6 +26,7 @@ #include "dist.h" #include "big.h" #include "error.h" +#include "erl_thr_progress.h" Hash erts_dist_table; Hash erts_node_table; @@ -118,7 +119,7 @@ dist_table_alloc(void *dep_tmpl) dep->finalized_out_queue.first = NULL; dep->finalized_out_queue.last = NULL; - erts_smp_atomic_init(&dep->dist_cmd_scheduled, 0); + erts_smp_atomic_init_nob(&dep->dist_cmd_scheduled, 0); erts_port_task_handle_init(&dep->dist_cmd); dep->send = NULL; dep->cache = NULL; @@ -767,7 +768,7 @@ void erts_init_node_tables(void) erts_this_dist_entry->finalized_out_queue.first = NULL; erts_this_dist_entry->finalized_out_queue.last = NULL; - erts_smp_atomic_init(&erts_this_dist_entry->dist_cmd_scheduled, 0); + erts_smp_atomic_init_nob(&erts_this_dist_entry->dist_cmd_scheduled, 0); erts_port_task_handle_init(&erts_this_dist_entry->dist_cmd); erts_this_dist_entry->send = NULL; erts_this_dist_entry->cache = NULL; @@ -907,7 +908,7 @@ erts_get_node_and_dist_references(struct process *proc) #endif erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(0); + erts_smp_thr_progress_block(); /* No need to lock any thing since we are alone... */ if (references_atoms_need_init) { @@ -951,7 +952,7 @@ erts_get_node_and_dist_references(struct process *proc) delete_reference_table(); - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(proc, ERTS_PROC_LOCK_MAIN); return res; } diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c index e6b55c45e4..87b8e5131b 100644 --- a/erts/emulator/beam/erl_port_task.c +++ b/erts/emulator/beam/erl_port_task.c @@ -121,7 +121,7 @@ ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(port_taskq, static ERTS_INLINE ErtsPortTask * handle2task(ErtsPortTaskHandle *pthp) { - return (ErtsPortTask *) erts_smp_atomic_read(pthp); + return (ErtsPortTask *) erts_smp_atomic_read_nob(pthp); } static ERTS_INLINE void @@ -129,7 +129,7 @@ reset_handle(ErtsPortTask *ptp) { if (ptp->handle) { ASSERT(ptp == handle2task(ptp->handle)); - erts_smp_atomic_set(ptp->handle, (erts_aint_t) NULL); + erts_smp_atomic_set_nob(ptp->handle, (erts_aint_t) NULL); } } @@ -138,7 +138,7 @@ set_handle(ErtsPortTask *ptp, ErtsPortTaskHandle *pthp) { ptp->handle = pthp; if (pthp) { - erts_smp_atomic_set(pthp, (erts_aint_t) ptp); + erts_smp_atomic_set_nob(pthp, (erts_aint_t) ptp); ASSERT(ptp == handle2task(ptp->handle)); } } @@ -479,8 +479,8 @@ erts_port_task_abort(Eterm id, ErtsPortTaskHandle *pthp) case ERTS_PORT_TASK_INPUT: case ERTS_PORT_TASK_OUTPUT: case ERTS_PORT_TASK_EVENT: - ASSERT(erts_smp_atomic_read(&erts_port_task_outstanding_io_tasks) > 0); - erts_smp_atomic_dec(&erts_port_task_outstanding_io_tasks); + ASSERT(erts_smp_atomic_read_nob(&erts_port_task_outstanding_io_tasks) > 0); + erts_smp_atomic_dec_relb(&erts_port_task_outstanding_io_tasks); break; default: break; @@ -568,7 +568,7 @@ erts_port_task_schedule(Eterm id, ErtsRunQueue *xrunq = erts_check_emigration_need(runq, ERTS_PORT_PRIO_LEVEL); if (xrunq) { /* Port emigrated ... */ - erts_smp_atomic_set(&pp->run_queue, (erts_aint_t) xrunq); + erts_smp_atomic_set_nob(&pp->run_queue, (erts_aint_t) xrunq); erts_smp_runq_unlock(runq); runq = xrunq; } @@ -594,7 +594,7 @@ erts_port_task_schedule(Eterm id, case ERTS_PORT_TASK_INPUT: case ERTS_PORT_TASK_OUTPUT: case ERTS_PORT_TASK_EVENT: - erts_smp_atomic_inc(&erts_port_task_outstanding_io_tasks); + erts_smp_atomic_inc_relb(&erts_port_task_outstanding_io_tasks); /* Fall through... */ default: enqueue_task(pp->sched.taskq, ptp); @@ -662,7 +662,7 @@ erts_port_task_free_port(Port *pp) pp->status |= ERTS_PORT_SFLG_FREE_SCHEDULED; erts_may_save_closed_port(pp); erts_smp_port_state_unlock(pp); - ERTS_SMP_LC_ASSERT(erts_smp_atomic_read(&pp->refc) > 1); + ERTS_SMP_LC_ASSERT(erts_smp_atomic_read_nob(&pp->refc) > 1); ptp->type = ERTS_PORT_TASK_FREE; ptp->event = (ErlDrvEvent) -1; ptp->event_data = NULL; @@ -684,9 +684,9 @@ erts_port_task_free_port(Port *pp) erts_may_save_closed_port(pp); erts_smp_port_state_unlock(pp); #ifdef ERTS_SMP - erts_smp_atomic_dec(&pp->refc); /* Not alive */ + erts_smp_atomic_dec_nob(&pp->refc); /* Not alive */ #endif - ERTS_SMP_LC_ASSERT(erts_smp_atomic_read(&pp->refc) > 0); /* Lock */ + ERTS_SMP_LC_ASSERT(erts_smp_atomic_read_nob(&pp->refc) > 0); /* Lock */ handle_remaining_tasks(runq, pp); /* May release runq lock */ ASSERT(!pp->sched.exe_taskq && (!ptqp || !ptqp->first)); pp->sched.taskq = NULL; @@ -711,23 +711,6 @@ typedef struct { int *resp; } ErtsPortTaskExeBlockData; -static void -prepare_for_block(void *vd) -{ - ErtsPortTaskExeBlockData *d = (ErtsPortTaskExeBlockData *) vd; - erts_smp_runq_unlock(d->runq); -} - -static void -resume_after_block(void *vd) -{ - ErtsPortTaskExeBlockData *d = (ErtsPortTaskExeBlockData *) vd; - erts_smp_runq_lock(d->runq); - if (d->resp) - *d->resp = (erts_smp_atomic_read(&erts_port_task_outstanding_io_tasks) - != (erts_aint_t) 0); -} - /* * Run all scheduled tasks for the first port in run queue. If * new tasks appear while running reschedule port (free task is @@ -752,11 +735,6 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq)); - erts_smp_activity_begin(ERTS_ACTIVITY_IO, - prepare_for_block, - resume_after_block, - (void *) &blk_data); - ERTS_PT_CHK_PORTQ(runq); pp = pop_port(runq); @@ -832,8 +810,8 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) ASSERT(!ptqp->first && (!pp->sched.taskq || !pp->sched.taskq->first)); #ifdef ERTS_SMP - erts_smp_atomic_dec(&pp->refc); /* Not alive */ - ERTS_SMP_LC_ASSERT(erts_smp_atomic_read(&pp->refc) > 0); /* Lock */ + erts_smp_atomic_dec_nob(&pp->refc); /* Not alive */ + ERTS_SMP_LC_ASSERT(erts_smp_atomic_read_nob(&pp->refc) > 0); /* Lock */ #else erts_port_status_bor_set(pp, ERTS_PORT_SFLG_FREE); #endif @@ -906,14 +884,16 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) erts_unblock_fpe(fpe_was_unmasked); if (io_tasks_executed) { - ASSERT(erts_smp_atomic_read(&erts_port_task_outstanding_io_tasks) >= io_tasks_executed); - erts_smp_atomic_add(&erts_port_task_outstanding_io_tasks, -1*io_tasks_executed); + ASSERT(erts_smp_atomic_read_nob(&erts_port_task_outstanding_io_tasks) + >= io_tasks_executed); + erts_smp_atomic_add_relb(&erts_port_task_outstanding_io_tasks, + -1*io_tasks_executed); } *curr_port_pp = NULL; #ifdef ERTS_SMP - ASSERT(runq == (ErtsRunQueue *) erts_smp_atomic_read(&pp->run_queue)); + ASSERT(runq == (ErtsRunQueue *) erts_smp_atomic_read_nob(&pp->run_queue)); #endif if (!pp->sched.taskq) { @@ -940,7 +920,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) } else { /* Port emigrated ... */ - erts_smp_atomic_set(&pp->run_queue, (erts_aint_t) xrunq); + erts_smp_atomic_set_nob(&pp->run_queue, (erts_aint_t) xrunq); enqueue_port(xrunq, pp); ASSERT(pp->sched.exe_taskq); pp->sched.exe_taskq = NULL; @@ -951,7 +931,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) port_was_enqueued = 1; } - res = (erts_smp_atomic_read(&erts_port_task_outstanding_io_tasks) + res = (erts_smp_atomic_read_nob(&erts_port_task_outstanding_io_tasks) != (erts_aint_t) 0); ERTS_PT_CHK_PRES_PORTQ(runq, pp); @@ -972,13 +952,13 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) { erts_aint_t refc; erts_smp_mtx_unlock(pp->lock); - refc = erts_smp_atomic_dectest(&pp->refc); + refc = erts_smp_atomic_dec_read_nob(&pp->refc); ASSERT(refc >= 0); if (refc == 0) { erts_smp_runq_unlock(runq); erts_port_cleanup(pp); /* Might aquire runq lock */ erts_smp_runq_lock(runq); - res = (erts_smp_atomic_read(&erts_port_task_outstanding_io_tasks) + res = (erts_smp_atomic_read_nob(&erts_port_task_outstanding_io_tasks) != (erts_aint_t) 0); } } @@ -986,10 +966,6 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) done: blk_data.resp = &res; - erts_smp_activity_end(ERTS_ACTIVITY_IO, - prepare_for_block, - resume_after_block, - (void *) &blk_data); ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq)); @@ -1107,12 +1083,12 @@ erts_port_migrate(Port *prt, int *prt_locked, /* Refuse to migrate to a suspended run queue */ if (to_rq->flags & ERTS_RUNQ_FLG_SUSPENDED) return ERTS_MIGRATE_FAILED_RUNQ_SUSPENDED; - if (from_rq != (ErtsRunQueue *) erts_smp_atomic_read(&prt->run_queue)) + if (from_rq != (ErtsRunQueue *) erts_smp_atomic_read_nob(&prt->run_queue)) return ERTS_MIGRATE_FAILED_RUNQ_CHANGED; if (!ERTS_PORT_IS_IN_RUNQ(from_rq, prt)) return ERTS_MIGRATE_FAILED_NOT_IN_RUNQ; dequeue_port(from_rq, prt); - erts_smp_atomic_set(&prt->run_queue, (erts_aint_t) to_rq); + erts_smp_atomic_set_nob(&prt->run_queue, (erts_aint_t) to_rq); enqueue_port(to_rq, prt); return ERTS_MIGRATE_SUCCESS; } @@ -1125,7 +1101,8 @@ erts_port_migrate(Port *prt, int *prt_locked, void erts_port_task_init(void) { - erts_smp_atomic_init(&erts_port_task_outstanding_io_tasks, (erts_aint_t) 0); + erts_smp_atomic_init_nob(&erts_port_task_outstanding_io_tasks, + (erts_aint_t) 0); init_port_task_alloc(); init_port_taskq_alloc(); } diff --git a/erts/emulator/beam/erl_port_task.h b/erts/emulator/beam/erl_port_task.h index 3e2c5f07ab..d7104e1143 100644 --- a/erts/emulator/beam/erl_port_task.h +++ b/erts/emulator/beam/erl_port_task.h @@ -79,13 +79,13 @@ ERTS_GLB_INLINE int erts_port_task_have_outstanding_io_tasks(void); ERTS_GLB_INLINE void erts_port_task_handle_init(ErtsPortTaskHandle *pthp) { - erts_smp_atomic_init(pthp, (erts_aint_t) NULL); + erts_smp_atomic_init_nob(pthp, (erts_aint_t) NULL); } ERTS_GLB_INLINE int erts_port_task_is_scheduled(ErtsPortTaskHandle *pthp) { - return ((void *) erts_smp_atomic_read(pthp)) != NULL; + return ((void *) erts_smp_atomic_read_nob(pthp)) != NULL; } ERTS_GLB_INLINE void @@ -102,8 +102,8 @@ erts_port_task_init_sched(ErtsPortTaskSched *ptsp) ERTS_GLB_INLINE int erts_port_task_have_outstanding_io_tasks(void) { - ERTS_THR_MEMORY_BARRIER; - return erts_smp_atomic_read(&erts_port_task_outstanding_io_tasks) != 0; + return (erts_smp_atomic_read_acqb(&erts_port_task_outstanding_io_tasks) + != 0); } #endif /* ERTS_INCLUDE_SCHEDULER_INTERNALS */ diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index ba3b32dd97..a3c1c9577b 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -39,6 +39,9 @@ #include "erl_binary.h" #include "beam_bp.h" #include "erl_cpu_topology.h" +#include "erl_thr_progress.h" +#include "erl_thr_queue.h" +#include "erl_async.h" #define ERTS_RUNQ_CHECK_BALANCE_REDS_PER_SCHED (2000*CONTEXT_REDS) #define ERTS_RUNQ_CALL_CHECK_BALANCE_REDS \ @@ -124,9 +127,10 @@ ErtsLcPSDLocks erts_psd_required_locks[ERTS_PSD_SIZE]; #endif #ifdef ERTS_SMP - int erts_disable_proc_not_running_opt; +static ErtsAuxWorkData *aux_thread_aux_work_data; + #define ERTS_SCHDLR_SSPND_CHNG_WAITER (((erts_aint32_t) 1) << 0) #define ERTS_SCHDLR_SSPND_CHNG_MSB (((erts_aint32_t) 1) << 1) #define ERTS_SCHDLR_SSPND_CHNG_ONLN (((erts_aint32_t) 1) << 2) @@ -134,15 +138,15 @@ int erts_disable_proc_not_running_opt; #ifndef DEBUG #define ERTS_SCHDLR_SSPND_CHNG_SET(VAL, OLD_VAL) \ - erts_smp_atomic32_set(&schdlr_sspnd.changing, (VAL)) + erts_smp_atomic32_set_nob(&schdlr_sspnd.changing, (VAL)) #else #define ERTS_SCHDLR_SSPND_CHNG_SET(VAL, OLD_VAL) \ do { \ erts_aint32_t old_val__; \ - old_val__ = erts_smp_atomic32_xchg(&schdlr_sspnd.changing, \ - (VAL)); \ + old_val__ = erts_smp_atomic32_xchg_nob(&schdlr_sspnd.changing, \ + (VAL)); \ ASSERT(old_val__ == (OLD_VAL)); \ } while (0) @@ -158,7 +162,7 @@ static struct { erts_smp_atomic32_t changing; erts_smp_atomic32_t active; struct { - erts_smp_atomic32_t ongoing; + int ongoing; long wait_active; ErtsProcList *procs; } msb; /* Multi Scheduling Block */ @@ -213,8 +217,6 @@ Uint erts_no_run_queues; ErtsAlignedSchedulerData *erts_aligned_scheduler_data; -#ifdef ERTS_SMP - typedef union { ErtsSchedulerSleepInfo ssi; char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsSchedulerSleepInfo))]; @@ -222,8 +224,6 @@ typedef union { static ErtsAlignedSchedulerSleepInfo *aligned_sched_sleep_info; -#endif - #ifndef BM_COUNTERS static int processes_busy; #endif @@ -285,8 +285,9 @@ ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(proclist, ERTS_ALC_T_PROC_LIST) #define ERTS_SCHED_SLEEP_INFO_IX(IX) \ - (ASSERT_EXPR(0 <= (IX) && (IX) < erts_no_schedulers), \ - &aligned_sched_sleep_info[(IX)].ssi) + (ASSERT_EXPR(-1 <= ((int) (IX)) \ + && ((int) (IX)) < ((int) erts_no_schedulers)), \ + &aligned_sched_sleep_info[(IX)].ssi) #define ERTS_FOREACH_RUNQ(RQVAR, DO) \ do { \ @@ -339,6 +340,66 @@ static void exec_misc_ops(ErtsRunQueue *); static void print_function_from_pc(int to, void *to_arg, BeamInstr* x); static int stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp, int yreg); + +static void aux_work_timeout(void *unused); +static void aux_work_timeout_early_init(int no_schedulers); +static void aux_work_timeout_late_init(void); +static void setup_aux_work_timer(void); + +#if defined(DEBUG) || 0 +#define ERTS_DBG_CHK_AUX_WORK_VAL(V) dbg_chk_aux_work_val((V)) +static void +dbg_chk_aux_work_val(erts_aint32_t value) +{ + erts_aint32_t valid = 0; + +#ifdef ERTS_SSI_AUX_WORK_SET_TMO + valid |= ERTS_SSI_AUX_WORK_SET_TMO; +#endif +#ifdef ERTS_SSI_AUX_WORK_CHECK_CHILDREN + valid |= ERTS_SSI_AUX_WORK_CHECK_CHILDREN; +#endif +#ifdef ERTS_SSI_AUX_WORK_MISC + valid |= ERTS_SSI_AUX_WORK_MISC; +#endif +#ifdef ERTS_SSI_AUX_WORK_MISC_THR_PRGR + valid |= ERTS_SSI_AUX_WORK_MISC_THR_PRGR; +#endif +#ifdef ERTS_SSI_AUX_WORK_ASYNC_READY + valid |= ERTS_SSI_AUX_WORK_ASYNC_READY; +#endif +#ifdef ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN + valid |= ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN; +#endif + +#ifdef ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM + valid |= ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM; +#endif +#ifdef ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC + valid |= ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC; +#endif +#ifdef ERTS_SSI_AUX_WORK_DD + valid |= ERTS_SSI_AUX_WORK_DD; +#endif +#ifdef ERTS_SSI_AUX_WORK_DD + valid |= ERTS_SSI_AUX_WORK_DD_THR_PRGR; +#endif +#ifdef ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK + valid |= ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK; +#endif + + if (~valid & value) + erl_exit(ERTS_ABORT_EXIT, + "Invalid aux_work value found: 0x%x\n", + ~valid & value); +} +#define ERTS_DBG_CHK_SSI_AUX_WORK(SSI) \ + ERTS_DBG_CHK_AUX_WORK_VAL(erts_atomic32_read_nob(&(SSI)->aux_work)) +#else +#define ERTS_DBG_CHK_AUX_WORK_VAL(V) +#define ERTS_DBG_CHK_SSI_AUX_WORK(SSI) +#endif + #ifdef ERTS_SMP static void handle_pending_exiters(ErtsProcList *); @@ -410,7 +471,7 @@ erts_init_process(int ncpu) init_proclist_alloc(); - erts_smp_atomic32_init(&process_count, 0); + erts_smp_atomic32_init_nob(&process_count, 0); if (erts_use_r9_pids_ports) { proc_bits = ERTS_R9_PROC_BITS; @@ -577,6 +638,13 @@ erts_sched_finish_poke(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flags) case ERTS_SSI_FLG_POLL_SLEEPING: erts_sys_schedule_interrupt(1); break; + case ERTS_SSI_FLG_POLL_SLEEPING|ERTS_SSI_FLG_TSE_SLEEPING: + /* + * Thread progress blocking while poll sleeping; need + * to signal on both... + */ + erts_sys_schedule_interrupt(1); + /* fall through */ case ERTS_SSI_FLG_TSE_SLEEPING: erts_tse_set(ssi->event); break; @@ -589,189 +657,712 @@ erts_sched_finish_poke(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flags) } } +#endif + +static ERTS_INLINE void +set_aux_work_flags_wakeup_nob(ErtsSchedulerSleepInfo *ssi, + erts_aint32_t flgs) +{ + erts_aint32_t old_flgs; + + ERTS_DBG_CHK_SSI_AUX_WORK(ssi); + + old_flgs = erts_atomic32_read_nob(&ssi->aux_work); + if ((old_flgs & flgs) == 0) { + + old_flgs = erts_atomic32_read_bor_nob(&ssi->aux_work, flgs); + + if ((old_flgs & flgs) == 0) { +#ifdef ERTS_SMP + erts_sched_poke(ssi); +#else + erts_sys_schedule_interrupt(1); +#endif + } + } +} + +#if 0 /* Currently not used */ + +static ERTS_INLINE void +set_aux_work_flags_wakeup_relb(ErtsSchedulerSleepInfo *ssi, + erts_aint32_t flgs) +{ + erts_aint32_t old_flgs; + + ERTS_DBG_CHK_SSI_AUX_WORK(ssi); + + old_flgs = erts_atomic32_read_bor_relb(&ssi->aux_work, flgs); + + if ((old_flgs & flgs) == 0) { +#ifdef ERTS_SMP + erts_sched_poke(ssi); +#else + erts_sys_schedule_interrupt(1); +#endif + } +} + +#endif + +static ERTS_INLINE erts_aint32_t +set_aux_work_flags(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flgs) +{ + return erts_atomic32_read_bor_nob(&ssi->aux_work, flgs); +} + +static ERTS_INLINE erts_aint32_t +unset_aux_work_flags(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flgs) +{ + return erts_atomic32_read_band_nob(&ssi->aux_work, ~flgs); +} + typedef struct erts_misc_aux_work_t_ erts_misc_aux_work_t; struct erts_misc_aux_work_t_ { - erts_misc_aux_work_t *next; void (*func)(void *); void *arg; }; -typedef struct { - erts_smp_mtx_t mtx; - erts_misc_aux_work_t *first; - erts_misc_aux_work_t *last; -} erts_misc_aux_work_q_t; +ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(misc_aux_work, + erts_misc_aux_work_t, + 200, + ERTS_ALC_T_MISC_AUX_WORK) typedef union { - erts_misc_aux_work_q_t data; - char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_misc_aux_work_q_t))]; + ErtsThrQ_t q; + char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsThrQ_t))]; } erts_algnd_misc_aux_work_q_t; static erts_algnd_misc_aux_work_q_t *misc_aux_work_queues; -ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(misc_aux_work, - erts_misc_aux_work_t, - 200, - ERTS_ALC_T_MISC_AUX_WORK) +static void +notify_aux_work(void *vssi) +{ + set_aux_work_flags_wakeup_nob((ErtsSchedulerSleepInfo *) vssi, + ERTS_SSI_AUX_WORK_MISC); +} static void init_misc_aux_work(void) { int ix; + ErtsThrQInit_t qinit = ERTS_THR_Q_INIT_DEFAULT; + qinit.notify = notify_aux_work; init_misc_aux_work_alloc(); misc_aux_work_queues = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_MISC_AUX_WORK_Q, - erts_no_schedulers * - sizeof(erts_algnd_misc_aux_work_q_t)); + sizeof(erts_algnd_misc_aux_work_q_t) + * (erts_no_schedulers+1)); + +#ifdef ERTS_SMP + ix = 0; /* aux_thread + schedulers */ +#else + ix = 1; /* scheduler only */ +#endif - for (ix = 0; ix < erts_no_schedulers; ix++) { - erts_smp_mtx_init_x(&misc_aux_work_queues[ix].data.mtx, - "misc_aux_work_queue", - make_small(ix + 1)); - misc_aux_work_queues[ix].data.first = NULL; - misc_aux_work_queues[ix].data.last = NULL; + for (; ix <= erts_no_schedulers; ix++) { + qinit.arg = (void *) ERTS_SCHED_SLEEP_INFO_IX(ix-1); + erts_thr_q_initialize(&misc_aux_work_queues[ix].q, &qinit); } } -static void -handle_misc_aux_work(ErtsSchedulerData *esdp) -{ - int ix = (int) esdp->no - 1; - erts_misc_aux_work_t *mawp; +static erts_aint32_t +misc_aux_work_clean(ErtsThrQ_t *q, + ErtsAuxWorkData *awdp, + erts_aint32_t aux_work) +{ + switch (erts_thr_q_clean(q)) { + case ERTS_THR_Q_DIRTY: + set_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_MISC); + return aux_work | ERTS_SSI_AUX_WORK_MISC; +#ifdef ERTS_SMP + case ERTS_THR_Q_NEED_THR_PRGR: + set_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_MISC_THR_PRGR); + erts_thr_progress_wakeup(awdp->esdp, + erts_thr_q_need_thr_progress(q)); +#endif + case ERTS_THR_Q_CLEAN: + break; + } + return aux_work; +} - erts_smp_mtx_lock(&misc_aux_work_queues[ix].data.mtx); - mawp = misc_aux_work_queues[ix].data.first; - misc_aux_work_queues[ix].data.first = NULL; - misc_aux_work_queues[ix].data.last = NULL; - erts_smp_mtx_unlock(&misc_aux_work_queues[ix].data.mtx); +static erts_aint32_t +handle_misc_aux_work(ErtsAuxWorkData *awdp, + erts_aint32_t aux_work) +{ + ErtsThrQ_t *q = &misc_aux_work_queues[awdp->sched_id].q; - while (mawp) { - erts_misc_aux_work_t *free_mawp; + unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_MISC); + while (1) { + erts_misc_aux_work_t *mawp = erts_thr_q_dequeue(q); + if (!mawp) + break; mawp->func(mawp->arg); - free_mawp = mawp; - mawp = mawp->next; - misc_aux_work_free(free_mawp); + misc_aux_work_free(mawp); } + + return misc_aux_work_clean(q, awdp, aux_work & ~ERTS_SSI_AUX_WORK_MISC); +} + +#ifdef ERTS_SMP + +static erts_aint32_t +handle_misc_aux_work_thr_prgr(ErtsAuxWorkData *awdp, + erts_aint32_t aux_work) +{ + if (!erts_thr_progress_has_reached(awdp->misc.thr_prgr)) + return aux_work; + + unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_MISC_THR_PRGR); + + return misc_aux_work_clean(&misc_aux_work_queues[awdp->sched_id].q, + awdp, + aux_work & ~ERTS_SSI_AUX_WORK_MISC_THR_PRGR); +} + +#endif + +static ERTS_INLINE void +schedule_misc_aux_work(int sched_id, + void (*func)(void *), + void *arg) +{ + ErtsThrQ_t *q; + erts_misc_aux_work_t *mawp; + +#ifdef ERTS_SMP + ASSERT(0 <= sched_id && sched_id <= erts_no_schedulers); +#else + ASSERT(sched_id == 1); +#endif + + q = &misc_aux_work_queues[sched_id].q; + mawp = misc_aux_work_alloc(); + mawp->func = func; + mawp->arg = arg; + erts_thr_q_enqueue(q, mawp); +} + +void +erts_schedule_misc_aux_work(int sched_id, + void (*func)(void *), + void *arg) +{ + schedule_misc_aux_work(sched_id, func, arg); } void -erts_smp_schedule_misc_aux_work(int ignore_self, - int max_sched, - void (*func)(void *), - void *arg) +erts_schedule_multi_misc_aux_work(int ignore_self, + int max_sched, + void (*func)(void *), + void *arg) { - int ix, ignore_ix = -1; + int id, self = 0; if (ignore_self) { ErtsSchedulerData *esdp = erts_get_scheduler_data(); if (esdp) - ignore_ix = (int) esdp->no - 1; + self = (int) esdp->no; } - ASSERT(0 <= max_sched && max_sched <= erts_no_schedulers); + ASSERT(0 < max_sched && max_sched <= erts_no_schedulers); - for (ix = 0; ix < max_sched; ix++) { - erts_aint32_t aux_work; - erts_misc_aux_work_t *mawp; - ErtsSchedulerSleepInfo *ssi; - if (ix == ignore_ix) + for (id = 1; id <= max_sched; id++) { + if (id == self) continue; + schedule_misc_aux_work(id, func, arg); + } +} - mawp = misc_aux_work_alloc(); +#if ERTS_USE_ASYNC_READY_Q - mawp->func = func; - mawp->arg = arg; - mawp->next = NULL; +void +erts_notify_check_async_ready_queue(void *vno) +{ + int ix = ((int) (SWord) vno) -1; + set_aux_work_flags_wakeup_nob(ERTS_SCHED_SLEEP_INFO_IX(ix), + ERTS_SSI_AUX_WORK_ASYNC_READY); +} - erts_smp_mtx_lock(&misc_aux_work_queues[ix].data.mtx); - if (!misc_aux_work_queues[ix].data.last) - misc_aux_work_queues[ix].data.first = mawp; - else - misc_aux_work_queues[ix].data.last->next = mawp; - misc_aux_work_queues[ix].data.last = mawp; - erts_smp_mtx_unlock(&misc_aux_work_queues[ix].data.mtx); - - ssi = ERTS_SCHED_SLEEP_INFO_IX(ix); - aux_work = erts_smp_atomic32_bor(&ssi->aux_work, - ERTS_SSI_AUX_WORK_MISC); - if ((aux_work & ERTS_SSI_AUX_WORK_MISC) == 0) - erts_sched_poke(ssi); - } +static erts_aint32_t +handle_async_ready(ErtsAuxWorkData *awdp, + erts_aint32_t aux_work) +{ + ErtsSchedulerSleepInfo *ssi = awdp->ssi; + unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_ASYNC_READY); + if (erts_check_async_ready(awdp->async_ready.queue)) { + if (set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_ASYNC_READY) + & ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN) { + unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN); + aux_work &= ~ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN; + } + return aux_work; + } +#ifdef ERTS_SMP + awdp->async_ready.need_thr_prgr = 0; +#endif + set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN); + return ((aux_work & ~ERTS_SSI_AUX_WORK_ASYNC_READY) + | ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN); +} + +static erts_aint32_t +handle_async_ready_clean(ErtsAuxWorkData *awdp, + erts_aint32_t aux_work) +{ + void *thr_prgr_p; + +#ifdef ERTS_SMP + if (awdp->async_ready.need_thr_prgr + && !erts_thr_progress_has_reached(awdp->misc.thr_prgr)) { + return aux_work & ~ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN; + } + + awdp->async_ready.need_thr_prgr = 0; + thr_prgr_p = (void *) &awdp->async_ready.thr_prgr; +#else + thr_prgr_p = NULL; +#endif + + switch (erts_async_ready_clean(awdp->async_ready.queue, thr_prgr_p)) { + case ERTS_ASYNC_READY_CLEAN: + unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN); + return aux_work & ~ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN; +#ifdef ERTS_SMP + case ERTS_ASYNC_READY_NEED_THR_PRGR: + erts_thr_progress_wakeup(awdp->esdp, + awdp->async_ready.thr_prgr); + awdp->async_ready.need_thr_prgr = 1; +#endif + default: + return aux_work; + } +} + +#endif + +static erts_aint32_t +handle_fix_alloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) +{ + ErtsSchedulerSleepInfo *ssi = awdp->ssi; + erts_aint32_t res; + + unset_aux_work_flags(ssi, (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM + | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC)); + aux_work &= ~(ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM + | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC); + res = erts_alloc_fix_alloc_shrink(awdp->sched_id, aux_work); + if (res) { + set_aux_work_flags(ssi, res); + aux_work |= res; + } + + return aux_work; +} + +#ifdef ERTS_SMP + +void +erts_alloc_notify_delayed_dealloc(int ix) +{ + set_aux_work_flags_wakeup_nob(ERTS_SCHED_SLEEP_INFO_IX(ix-1), + ERTS_SSI_AUX_WORK_DD); +} + +static erts_aint32_t +handle_delayed_dealloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) +{ + ErtsSchedulerSleepInfo *ssi = awdp->ssi; + int need_thr_progress = 0; + int more_work = 0; + + unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD); + erts_alloc_scheduler_handle_delayed_dealloc((void *) awdp->esdp, + &need_thr_progress, + &more_work); + if (more_work) { + if (set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD) + & ERTS_SSI_AUX_WORK_DD_THR_PRGR) { + unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD_THR_PRGR); + aux_work &= ~ERTS_SSI_AUX_WORK_DD_THR_PRGR; + } + return aux_work; + } + + if (need_thr_progress) { + set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD_THR_PRGR); + awdp->dd.thr_prgr = erts_thr_progress_later(); + erts_thr_progress_wakeup(awdp->esdp, awdp->dd.thr_prgr); + } + else if (awdp->dd.completed_callback) { + awdp->dd.completed_callback(awdp->dd.completed_arg); + awdp->dd.completed_callback = NULL; + awdp->dd.completed_arg = NULL; + } + return aux_work & ~ERTS_SSI_AUX_WORK_DD; } +static erts_aint32_t +handle_delayed_dealloc_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) +{ + ErtsSchedulerSleepInfo *ssi; + int need_thr_progress; + int more_work; + + if (!erts_thr_progress_has_reached(awdp->dd.thr_prgr)) + return aux_work & ~ERTS_SSI_AUX_WORK_DD_THR_PRGR; + + ssi = awdp->ssi; + need_thr_progress = 0; + more_work = 0; + + erts_alloc_scheduler_handle_delayed_dealloc((void *) awdp->esdp, + &need_thr_progress, + &more_work); + if (more_work) { + set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD); + unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD_THR_PRGR); + return ((aux_work & ~ERTS_SSI_AUX_WORK_DD_THR_PRGR) + | ERTS_SSI_AUX_WORK_DD); + } + + if (need_thr_progress) { + awdp->dd.thr_prgr = erts_thr_progress_later(); + erts_thr_progress_wakeup(awdp->esdp, awdp->dd.thr_prgr); + } + else { + unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD_THR_PRGR); + if (awdp->dd.completed_callback) { + awdp->dd.completed_callback(awdp->dd.completed_arg); + awdp->dd.completed_callback = NULL; + awdp->dd.completed_arg = NULL; + } + } + + return aux_work & ~ERTS_SSI_AUX_WORK_DD_THR_PRGR; +} + +static erts_atomic32_t completed_dealloc_count; + +static void +completed_dealloc(void *vproc) +{ + if (erts_atomic32_dec_read_mb(&completed_dealloc_count) == 0) { + erts_resume((Process *) vproc, (ErtsProcLocks) 0); + erts_smp_proc_dec_refc((Process *) vproc); + } +} + +static void +setup_completed_dealloc(void *vproc) +{ + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + ErtsAuxWorkData *awdp = (esdp + ? &esdp->aux_work_data + : aux_thread_aux_work_data); + erts_alloc_fix_alloc_shrink(awdp->sched_id, 0); + set_aux_work_flags_wakeup_nob(awdp->ssi, ERTS_SSI_AUX_WORK_DD); + awdp->dd.completed_callback = completed_dealloc; + awdp->dd.completed_arg = vproc; +} + +static void +prep_setup_completed_dealloc(void *vproc) +{ + erts_aint32_t count = (erts_aint32_t) (erts_no_schedulers+1); + if (erts_atomic32_dec_read_mb(&completed_dealloc_count) == count) { + /* scheduler threads */ + erts_schedule_multi_misc_aux_work(0, + erts_no_schedulers, + setup_completed_dealloc, + vproc); + /* aux_thread */ + erts_schedule_misc_aux_work(0, + setup_completed_dealloc, + vproc); + } +} + +#endif /* ERTS_SMP */ + +int +erts_debug_wait_deallocations(Process *c_p) +{ +#ifndef ERTS_SMP + erts_alloc_fix_alloc_shrink(1, 0); + return 1; +#else + /* Only one process at a time can do this */ + erts_aint32_t count = (erts_aint32_t) (2*(erts_no_schedulers+1)); + if (0 == erts_atomic32_cmpxchg_mb(&completed_dealloc_count, + count, + 0)) { + erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL); + erts_smp_proc_inc_refc(c_p); + /* scheduler threads */ + erts_schedule_multi_misc_aux_work(0, + erts_no_schedulers, + prep_setup_completed_dealloc, + (void *) c_p); + /* aux_thread */ + erts_schedule_misc_aux_work(0, + prep_setup_completed_dealloc, + (void *) c_p); + return 1; + } + return 0; +#endif +} + + #ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN void erts_smp_notify_check_children_needed(void) { int i; + for (i = 0; i < erts_no_schedulers; i++) + set_aux_work_flags_wakeup_nob(ERTS_SCHED_SLEEP_INFO_IX(i), + ERTS_SSI_AUX_WORK_CHECK_CHILDREN); +} - for (i = 0; i < erts_no_schedulers; i++) { - erts_aint32_t aux_work; - ErtsSchedulerSleepInfo *ssi; - ssi = ERTS_SCHED_SLEEP_INFO_IX(i); - aux_work = erts_smp_atomic32_bor(&ssi->aux_work, - ERTS_SSI_AUX_WORK_CHECK_CHILDREN); - if (!(aux_work & ERTS_SSI_AUX_WORK_CHECK_CHILDREN)) - erts_sched_poke(ssi); - } +static erts_aint32_t +handle_check_children(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) +{ + unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_CHECK_CHILDREN); + erts_check_children(); + return aux_work & ~ERTS_SSI_AUX_WORK_CHECK_CHILDREN; } + #endif -#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK +#ifdef ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK + +static erts_aint32_t +handle_mseg_cache_check(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) +{ + unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK); + erts_mseg_cache_check(); + return aux_work & ~ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK; +} + +#endif + +static erts_aint32_t +handle_setup_aux_work_timer(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) +{ + unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_SET_TMO); + setup_aux_work_timer(); + return aux_work & ~ERTS_SSI_AUX_WORK_SET_TMO; +} + static ERTS_INLINE erts_aint32_t -blockable_aux_work(ErtsSchedulerData *esdp, - ErtsSchedulerSleepInfo *ssi, - erts_aint32_t aux_work) +handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) { - if (aux_work & ERTS_SSI_BLOCKABLE_AUX_WORK_MASK) { - if (aux_work & ERTS_SSI_AUX_WORK_MISC) { - aux_work = erts_smp_atomic32_band(&ssi->aux_work, - ~ERTS_SSI_AUX_WORK_MISC); - aux_work &= ~ERTS_SSI_AUX_WORK_MISC; - handle_misc_aux_work(esdp); - } + /* + * Handlers are *only* allowed to modify flags in return value + * and ssi flags that are explicity handled by the handler. + * Handlers are, e.g., not allowed to read the ssi flag field and + * then unconditionally return that value. + */ + ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); + if (aux_work & ERTS_SSI_AUX_WORK_SET_TMO) { + aux_work = handle_setup_aux_work_timer(awdp, aux_work); + ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); + } +#ifdef ERTS_SMP + if (aux_work & ERTS_SSI_AUX_WORK_MISC_THR_PRGR) { + aux_work = handle_misc_aux_work_thr_prgr(awdp, aux_work); + ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); + } +#endif + if (aux_work & ERTS_SSI_AUX_WORK_MISC) { + aux_work = handle_misc_aux_work(awdp, aux_work); + ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); + } +#if ERTS_USE_ASYNC_READY_Q + if (aux_work & ERTS_SSI_AUX_WORK_ASYNC_READY) { + aux_work = handle_async_ready(awdp, aux_work); + ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); + } + if (aux_work & ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN) { + aux_work = handle_async_ready_clean(awdp, aux_work); + ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); + } +#endif #ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN - if (aux_work & ERTS_SSI_AUX_WORK_CHECK_CHILDREN) { - aux_work = erts_smp_atomic32_band(&ssi->aux_work, - ~ERTS_SSI_AUX_WORK_CHECK_CHILDREN); - aux_work &= ~ERTS_SSI_AUX_WORK_CHECK_CHILDREN; - erts_check_children(); - } + if (aux_work & ERTS_SSI_AUX_WORK_CHECK_CHILDREN) { + aux_work = handle_check_children(awdp, aux_work); + ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); + } +#endif + if (aux_work & (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM + | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC)) { + aux_work = handle_fix_alloc(awdp, aux_work); + ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); + } +#ifdef ERTS_SMP + if (aux_work & ERTS_SSI_AUX_WORK_DD) { + aux_work = handle_delayed_dealloc(awdp, aux_work); + ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); + } + if (aux_work & ERTS_SSI_AUX_WORK_DD_THR_PRGR) { + aux_work = handle_delayed_dealloc_thr_prgr(awdp, aux_work); + ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); + } #endif +#ifdef ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK + if (aux_work & ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK) { + aux_work = handle_mseg_cache_check(awdp, aux_work); + ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); } +#endif + ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); return aux_work; } -#endif +typedef struct { + union { + ErlTimer data; + char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErlTimer))]; + } timer; -#ifdef ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK -static ERTS_INLINE erts_aint32_t -nonblockable_aux_work(ErtsSchedulerData *esdp, - ErtsSchedulerSleepInfo *ssi, - erts_aint32_t aux_work) + int initialized; + erts_atomic32_t refc; + erts_atomic32_t type[1]; +} ErtsAuxWorkTmo; + +static ErtsAuxWorkTmo *aux_work_tmo; + +static void +aux_work_timeout_early_init(int no_schedulers) { - if (aux_work & ERTS_SSI_NONBLOCKABLE_AUX_WORK_MASK) { + int i; + UWord p; + + /* + * This is done really early. Our own allocators have + * not been started yet. + */ + + p = (UWord) malloc((sizeof(ErtsAuxWorkTmo) + + sizeof(erts_atomic32_t)*(no_schedulers+1)) + + ERTS_CACHE_LINE_SIZE-1); + if (p & ERTS_CACHE_LINE_MASK) + p = (p & ~ERTS_CACHE_LINE_MASK) + ERTS_CACHE_LINE_SIZE; + ASSERT((p & ERTS_CACHE_LINE_MASK) == 0); + aux_work_tmo = (ErtsAuxWorkTmo *) p; + aux_work_tmo->initialized = 0; + erts_atomic32_init_nob(&aux_work_tmo->refc, 0); + for (i = 0; i <= no_schedulers; i++) + erts_atomic32_init_nob(&aux_work_tmo->type[i], 0); +} + +void +aux_work_timeout_late_init(void) +{ + aux_work_tmo->initialized = 1; + if (erts_atomic32_read_nob(&aux_work_tmo->refc)) { + aux_work_tmo->timer.data.active = 0; + erts_set_timer(&aux_work_tmo->timer.data, + aux_work_timeout, + NULL, + NULL, + 1000); } } -#endif static void -prepare_for_block(void *vrq) +aux_work_timeout(void *unused) { - erts_smp_runq_unlock((ErtsRunQueue *) vrq); + erts_aint32_t refc; + int i; +#ifdef ERTS_SMP + i = 0; +#else + i = 1; +#endif + + for (; i <= erts_no_schedulers; i++) { + erts_aint32_t type; + type = erts_atomic32_read_acqb(&aux_work_tmo->type[i]); + if (type) + set_aux_work_flags_wakeup_nob(ERTS_SCHED_SLEEP_INFO_IX(i-1), + type); + } + + refc = erts_atomic32_read_nob(&aux_work_tmo->refc); + ASSERT(refc >= 1); + if (refc != 1 + || 1 != erts_atomic32_cmpxchg_relb(&aux_work_tmo->refc, 0, 1)) { + /* Setup next timeout... */ + aux_work_tmo->timer.data.active = 0; + erts_set_timer(&aux_work_tmo->timer.data, + aux_work_timeout, + NULL, + NULL, + 1000); + } } static void -resume_after_block(void *vrq) +setup_aux_work_timer(void) { - erts_smp_runq_lock((ErtsRunQueue *) vrq); +#ifndef ERTS_SMP + if (!erts_get_scheduler_data()) + set_aux_work_flags_wakeup_nob(ERTS_SCHED_SLEEP_INFO_IX(0), + ERTS_SSI_AUX_WORK_SET_TMO); + else +#endif + { + aux_work_tmo->timer.data.active = 0; + erts_set_timer(&aux_work_tmo->timer.data, + aux_work_timeout, + NULL, + NULL, + 1000); + } } +erts_aint32_t +erts_set_aux_work_timeout(int ix, erts_aint32_t type, int enable) +{ + erts_aint32_t old, refc; + +#ifndef ERTS_SMP + ix = 1; #endif + ERTS_DBG_CHK_AUX_WORK_VAL(type); + ERTS_DBG_CHK_AUX_WORK_VAL(erts_atomic32_read_nob(&aux_work_tmo->type[ix])); +// erts_fprintf(stderr, "t(%d, 0x%x, %d)\n", ix, type, enable); + + if (!enable) { + old = erts_atomic32_read_band_mb(&aux_work_tmo->type[ix], ~type); + ERTS_DBG_CHK_AUX_WORK_VAL(erts_atomic32_read_nob(&aux_work_tmo->type[ix])); + if (old != 0 && (old & ~type) == 0) + erts_atomic32_dec_relb(&aux_work_tmo->refc); + return old; + } + + old = erts_atomic32_read_bor_mb(&aux_work_tmo->type[ix], type); + ERTS_DBG_CHK_AUX_WORK_VAL(erts_atomic32_read_nob(&aux_work_tmo->type[ix])); + if (old == 0 && type != 0) { + refc = erts_atomic32_inc_read_acqb(&aux_work_tmo->refc); + if (refc == 1) { + erts_atomic32_inc_acqb(&aux_work_tmo->refc); + if (aux_work_tmo->initialized) + setup_aux_work_timer(); + } + } + return old; +} + + + static ERTS_INLINE void sched_waiting_sys(Uint no, ErtsRunQueue *rq) { @@ -800,8 +1391,6 @@ sched_active_sys(Uint no, ErtsRunQueue *rq) Uint erts_active_schedulers(void) { - /* RRRRRRRRR */ - Uint as = erts_no_schedulers; ERTS_ATOMIC_FOREACH_RUNQ(rq, as -= abs(rq->waiting)); @@ -815,7 +1404,7 @@ erts_active_schedulers(void) static ERTS_INLINE void clear_sys_scheduling(void) { - erts_smp_atomic32_set_relb(&doing_sys_schedule, 0); + erts_smp_atomic32_set_mb(&doing_sys_schedule, 0); } static ERTS_INLINE int @@ -882,42 +1471,43 @@ sched_active(Uint no, ErtsRunQueue *rq) static int ERTS_INLINE ongoing_multi_scheduling_block(void) { - return erts_smp_atomic32_read(&schdlr_sspnd.msb.ongoing) != 0; + ERTS_SMP_LC_ASSERT(erts_lc_mtx_is_locked(&schdlr_sspnd.mtx)); + return schdlr_sspnd.msb.ongoing; } static ERTS_INLINE void empty_runq(ErtsRunQueue *rq) { - erts_aint32_t oifls = erts_smp_atomic32_band(&rq->info_flags, - ~ERTS_RUNQ_IFLG_NONEMPTY); + erts_aint32_t oifls = erts_smp_atomic32_read_band_nob(&rq->info_flags, + ~ERTS_RUNQ_IFLG_NONEMPTY); if (oifls & ERTS_RUNQ_IFLG_NONEMPTY) { #ifdef DEBUG - erts_aint32_t empty = erts_smp_atomic32_read(&no_empty_run_queues); + erts_aint32_t empty = erts_smp_atomic32_read_nob(&no_empty_run_queues); /* * For a short period of time no_empty_run_queues may have * been increased twice for a specific run queue. */ ASSERT(0 <= empty && empty < 2*erts_no_run_queues); #endif - erts_smp_atomic32_inc(&no_empty_run_queues); + erts_smp_atomic32_inc_relb(&no_empty_run_queues); } } static ERTS_INLINE void non_empty_runq(ErtsRunQueue *rq) { - erts_aint32_t oifls = erts_smp_atomic32_bor(&rq->info_flags, - ERTS_RUNQ_IFLG_NONEMPTY); + erts_aint32_t oifls = erts_smp_atomic32_read_bor_nob(&rq->info_flags, + ERTS_RUNQ_IFLG_NONEMPTY); if (!(oifls & ERTS_RUNQ_IFLG_NONEMPTY)) { #ifdef DEBUG - erts_aint32_t empty = erts_smp_atomic32_read(&no_empty_run_queues); + erts_aint32_t empty = erts_smp_atomic32_read_nob(&no_empty_run_queues); /* * For a short period of time no_empty_run_queues may have * been increased twice for a specific run queue. */ ASSERT(0 < empty && empty <= 2*erts_no_run_queues); #endif - erts_smp_atomic32_dec(&no_empty_run_queues); + erts_smp_atomic32_dec_relb(&no_empty_run_queues); } } @@ -930,7 +1520,7 @@ sched_prep_spin_wait(ErtsSchedulerSleepInfo *ssi) erts_aint32_t xflgs = 0; do { - oflgs = erts_smp_atomic32_cmpxchg(&ssi->flags, nflgs, xflgs); + oflgs = erts_smp_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs); if (oflgs == xflgs) return nflgs; xflgs = oflgs; @@ -947,7 +1537,7 @@ sched_prep_cont_spin_wait(ErtsSchedulerSleepInfo *ssi) erts_aint32_t xflgs = ERTS_SSI_FLG_WAITING; do { - oflgs = erts_smp_atomic32_cmpxchg(&ssi->flags, nflgs, xflgs); + oflgs = erts_smp_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs); if (oflgs == xflgs) return nflgs; xflgs = oflgs; @@ -987,9 +1577,13 @@ sched_set_sleeptype(ErtsSchedulerSleepInfo *ssi, erts_aint32_t sleep_type) if (sleep_type == ERTS_SSI_FLG_TSE_SLEEPING) erts_tse_reset(ssi->event); + else { + ASSERT(sleep_type == ERTS_SSI_FLG_POLL_SLEEPING); + erts_sys_schedule_interrupt(0); + } while (1) { - oflgs = erts_smp_atomic32_cmpxchg(&ssi->flags, nflgs, xflgs); + oflgs = erts_smp_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs); if (oflgs == xflgs) return nflgs; if ((oflgs & (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING)) @@ -1005,16 +1599,127 @@ sched_set_sleeptype(ErtsSchedulerSleepInfo *ssi, erts_aint32_t sleep_type) (((FLGS) & (ERTS_SSI_FLG_WAITING|ERTS_SSI_FLG_SUSPENDED)) \ != ERTS_SSI_FLG_WAITING) + +static void +thr_prgr_wakeup(void *vssi) +{ + erts_sched_poke((ErtsSchedulerSleepInfo *) vssi); +} + +static void +thr_prgr_prep_wait(void *vssi) +{ + ErtsSchedulerSleepInfo *ssi = (ErtsSchedulerSleepInfo *) vssi; + erts_smp_atomic32_read_bor_acqb(&ssi->flags, + ERTS_SSI_FLG_SLEEPING); +} + +static void +thr_prgr_wait(void *vssi) +{ + ErtsSchedulerSleepInfo *ssi = (ErtsSchedulerSleepInfo *) vssi; + erts_aint32_t xflgs = ERTS_SSI_FLG_SLEEPING; + + erts_tse_reset(ssi->event); + + while (1) { + erts_aint32_t aflgs, nflgs; + nflgs = xflgs | ERTS_SSI_FLG_TSE_SLEEPING; + aflgs = erts_smp_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs); + if (aflgs == xflgs) { + erts_tse_wait(ssi->event); + break; + } + if ((aflgs & ERTS_SSI_FLG_SLEEPING) == 0) + break; + xflgs = aflgs; + } +} + +static void +thr_prgr_fin_wait(void *vssi) +{ + ErtsSchedulerSleepInfo *ssi = (ErtsSchedulerSleepInfo *) vssi; + erts_smp_atomic32_read_band_nob(&ssi->flags, + ~(ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_TSE_SLEEPING)); +} + +static void init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp); + +static void * +aux_thread(void *unused) +{ + ErtsAuxWorkData *awdp = aux_thread_aux_work_data; + ErtsSchedulerSleepInfo *ssi = ERTS_SCHED_SLEEP_INFO_IX(-1); + erts_aint32_t aux_work; + ErtsThrPrgrCallbacks callbacks; + int thr_prgr_active = 1; + + ssi->event = erts_tse_fetch(); + + callbacks.arg = (void *) ssi; + callbacks.wakeup = thr_prgr_wakeup; + callbacks.prepare_wait = thr_prgr_prep_wait; + callbacks.wait = thr_prgr_wait; + callbacks.finalize_wait = thr_prgr_fin_wait; + + erts_thr_progress_register_managed_thread(NULL, &callbacks, 1); + init_aux_work_data(awdp, NULL); + awdp->ssi = ssi; + + sched_prep_spin_wait(ssi); + + while (1) { + erts_aint32_t flgs; + + aux_work = erts_atomic32_read_acqb(&ssi->aux_work); + if (aux_work) { + if (!thr_prgr_active) + erts_thr_progress_active(NULL, thr_prgr_active = 1); + aux_work = handle_aux_work(awdp, aux_work); + if (aux_work && erts_thr_progress_update(NULL)) + erts_thr_progress_leader_update(NULL); + } + + if (!aux_work) { + if (thr_prgr_active) + erts_thr_progress_active(NULL, thr_prgr_active = 0); + erts_thr_progress_prepare_wait(NULL); + + flgs = sched_spin_wait(ssi, 0); + + if (flgs & ERTS_SSI_FLG_SLEEPING) { + ASSERT(flgs & ERTS_SSI_FLG_WAITING); + flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_TSE_SLEEPING); + if (flgs & ERTS_SSI_FLG_SLEEPING) { + int res; + ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING); + ASSERT(flgs & ERTS_SSI_FLG_WAITING); + do { + res = erts_tse_wait(ssi->event); + } while (res == EINTR); + } + } + erts_thr_progress_finalize_wait(NULL); + } + + flgs = sched_prep_spin_wait(ssi); + } + return NULL; +} + +#endif /* ERTS_SMP */ + static void scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) { ErtsSchedulerSleepInfo *ssi = esdp->ssi; int spincount; + erts_aint32_t aux_work = 0; +#ifdef ERTS_SMP + int thr_prgr_active = 1; erts_aint32_t flgs; -#if defined(ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK) \ - || defined(ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK) - erts_aint32_t aux_work; -#endif ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); @@ -1048,34 +1753,38 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) tse_wait: -#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK - aux_work = erts_smp_atomic32_read(&ssi->aux_work); - tse_blockable_aux_work: - aux_work = blockable_aux_work(esdp, ssi, aux_work); -#endif - erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); - while (1) { -#ifdef ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK -#ifndef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK - aux_work = erts_smp_atomic32_read(&ssi->aux_work); -#endif - nonblockable_aux_work(esdp, ssi, aux_work); -#endif + aux_work = erts_atomic32_read_acqb(&ssi->aux_work); + if (aux_work) { + if (!thr_prgr_active) + erts_thr_progress_active(esdp, thr_prgr_active = 1); + aux_work = handle_aux_work(&esdp->aux_work_data, aux_work); + if (aux_work && erts_thr_progress_update(esdp)) + erts_thr_progress_leader_update(esdp); + } - flgs = sched_spin_wait(ssi, spincount); - if (flgs & ERTS_SSI_FLG_SLEEPING) { - ASSERT(flgs & ERTS_SSI_FLG_WAITING); - flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_TSE_SLEEPING); + if (aux_work) + flgs = erts_smp_atomic32_read_acqb(&ssi->flags); + else { + if (thr_prgr_active) + erts_thr_progress_active(esdp, thr_prgr_active = 0); + erts_thr_progress_prepare_wait(esdp); + + flgs = sched_spin_wait(ssi, spincount); if (flgs & ERTS_SSI_FLG_SLEEPING) { - int res; - ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING); ASSERT(flgs & ERTS_SSI_FLG_WAITING); - do { - res = erts_tse_wait(ssi->event); - } while (res == EINTR); + flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_TSE_SLEEPING); + if (flgs & ERTS_SSI_FLG_SLEEPING) { + int res; + ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING); + ASSERT(flgs & ERTS_SSI_FLG_WAITING); + do { + res = erts_tse_wait(ssi->event); + } while (res == EINTR); + } } + erts_thr_progress_finalize_wait(esdp); } if (!(flgs & ERTS_SSI_FLG_WAITING)) { @@ -1091,26 +1800,21 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) break; } -#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK - aux_work = erts_smp_atomic32_read(&ssi->aux_work); - if (aux_work & ERTS_SSI_BLOCKABLE_AUX_WORK_MASK) { - erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); - goto tse_blockable_aux_work; - } -#endif - } - erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); - if (flgs & ~ERTS_SSI_FLG_SUSPENDED) - erts_smp_atomic32_band(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); + erts_smp_atomic32_read_band_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); + + if (!thr_prgr_active) + erts_thr_progress_active(esdp, thr_prgr_active = 1); erts_smp_runq_lock(rq); sched_active(esdp->no, rq); } - else { + else +#endif + { erts_aint_t dt; erts_smp_atomic32_set_relb(&function_calls, 0); @@ -1134,18 +1838,27 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) if (dt) erts_bump_timer(dt); sys_aux_work: - -#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK - aux_work = erts_smp_atomic32_read(&ssi->aux_work); - aux_work = blockable_aux_work(esdp, ssi, aux_work); +#ifndef ERTS_SMP + erts_sys_schedule_interrupt(0); #endif -#ifdef ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK -#ifndef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK - aux_work = erts_smp_atomic32_read(&ssi->aux_work); + + aux_work = erts_atomic32_read_acqb(&ssi->aux_work); + if (aux_work) { +#ifdef ERTS_SMP + if (!thr_prgr_active) + erts_thr_progress_active(esdp, thr_prgr_active = 1); #endif - nonblockable_aux_work(esdp, ssi, aux_work); + aux_work = handle_aux_work(&esdp->aux_work_data, aux_work); +#ifdef ERTS_SMP + if (aux_work && erts_thr_progress_update(esdp)) + erts_thr_progress_leader_update(esdp); #endif + } +#ifndef ERTS_SMP + if (rq->len != 0 || rq->misc.start) + goto sys_woken; +#else flgs = erts_smp_atomic32_read_acqb(&ssi->flags); if (!(flgs & ERTS_SSI_FLG_WAITING)) { ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); @@ -1167,10 +1880,12 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) goto tse_wait; } } +#endif } erts_smp_runq_lock(rq); +#ifdef ERTS_SMP /* * If we got new I/O tasks we aren't allowed to * sleep in erl_sys_schedule(). @@ -1182,64 +1897,88 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) * Got to check that we still got I/O tasks; otherwise * we have to wait in erl_sys_schedule() after all... */ - if (prepare_for_sys_schedule()) - goto do_sys_schedule; - - /* - * Not allowed to wait in erl_sys_schedule; - * do tse wait instead... - */ - sched_change_waiting_sys_to_waiting(esdp->no, rq); + if (!prepare_for_sys_schedule()) { + /* + * Not allowed to wait in erl_sys_schedule; + * do tse wait instead... + */ + sched_change_waiting_sys_to_waiting(esdp->no, rq); + erts_smp_runq_unlock(rq); + spincount = 0; + goto tse_wait; + } + } +#endif + if (aux_work) { erts_smp_runq_unlock(rq); - spincount = 0; - goto tse_wait; + goto sys_poll_aux_work; } - else { - do_sys_schedule: - erts_sys_schedule_interrupt(0); - flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_POLL_SLEEPING); - if (!(flgs & ERTS_SSI_FLG_SLEEPING)) { - if (!(flgs & ERTS_SSI_FLG_WAITING)) - goto sys_locked_woken; - erts_smp_runq_unlock(rq); - flgs = sched_prep_cont_spin_wait(ssi); - if (!(flgs & ERTS_SSI_FLG_WAITING)) { - ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); - goto sys_woken; - } - ASSERT(!erts_port_task_have_outstanding_io_tasks()); - goto sys_poll_aux_work; +#ifdef ERTS_SMP + flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_POLL_SLEEPING); + if (!(flgs & ERTS_SSI_FLG_SLEEPING)) { + if (!(flgs & ERTS_SSI_FLG_WAITING)) { + ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); + goto sys_locked_woken; + } + erts_smp_runq_unlock(rq); + flgs = sched_prep_cont_spin_wait(ssi); + if (!(flgs & ERTS_SSI_FLG_WAITING)) { + ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); + goto sys_woken; } + ASSERT(!erts_port_task_have_outstanding_io_tasks()); + goto sys_poll_aux_work; + } - ASSERT(flgs & ERTS_SSI_FLG_POLL_SLEEPING); - ASSERT(flgs & ERTS_SSI_FLG_WAITING); + ASSERT(flgs & ERTS_SSI_FLG_POLL_SLEEPING); + ASSERT(flgs & ERTS_SSI_FLG_WAITING); +#endif - erts_smp_runq_unlock(rq); + erts_smp_runq_unlock(rq); - ASSERT(!erts_port_task_have_outstanding_io_tasks()); +#ifdef ERTS_SMP + if (thr_prgr_active) + erts_thr_progress_active(esdp, thr_prgr_active = 0); +#endif - erl_sys_schedule(0); + ASSERT(!erts_port_task_have_outstanding_io_tasks()); - dt = erts_do_time_read_and_reset(); - if (dt) erts_bump_timer(dt); + erl_sys_schedule(0); - flgs = sched_prep_cont_spin_wait(ssi); - if (flgs & ERTS_SSI_FLG_WAITING) - goto sys_aux_work; + dt = erts_do_time_read_and_reset(); + if (dt) erts_bump_timer(dt); - sys_woken: +#ifndef ERTS_SMP + if (rq->len == 0 && !rq->misc.start) + goto sys_aux_work; + sys_woken: +#else + flgs = sched_prep_cont_spin_wait(ssi); + if (flgs & ERTS_SSI_FLG_WAITING) + goto sys_aux_work; + + sys_woken: + if (!thr_prgr_active) + erts_thr_progress_active(esdp, thr_prgr_active = 1); + erts_smp_runq_lock(rq); + sys_locked_woken: + if (!thr_prgr_active) { + erts_smp_runq_unlock(rq); + erts_thr_progress_active(esdp, thr_prgr_active = 1); erts_smp_runq_lock(rq); - sys_locked_woken: - clear_sys_scheduling(); - if (flgs & ~ERTS_SSI_FLG_SUSPENDED) - erts_smp_atomic32_band(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); - sched_active_sys(esdp->no, rq); } + clear_sys_scheduling(); + if (flgs & ~ERTS_SSI_FLG_SUSPENDED) + erts_smp_atomic32_read_band_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); +#endif + sched_active_sys(esdp->no, rq); } ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); } +#ifdef ERTS_SMP + static ERTS_INLINE erts_aint32_t ssi_flags_set_wake(ErtsSchedulerSleepInfo *ssi) { @@ -1248,7 +1987,7 @@ ssi_flags_set_wake(ErtsSchedulerSleepInfo *ssi) erts_aint32_t nflgs = 0; erts_aint32_t xflgs = ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING; while (1) { - oflgs = erts_smp_atomic32_cmpxchg(&ssi->flags, nflgs, xflgs); + oflgs = erts_smp_atomic32_cmpxchg_relb(&ssi->flags, nflgs, xflgs); if (oflgs == xflgs) return oflgs; nflgs = oflgs & ERTS_SSI_FLG_SUSPENDED; @@ -1291,7 +2030,6 @@ wake_scheduler(ErtsRunQueue *rq, int incq, int one) erts_smp_spin_unlock(&sl->lock); - ERTS_THR_MEMORY_BARRIER; flgs = ssi_flags_set_wake(ssi); erts_sched_finish_poke(ssi, flgs); @@ -1337,13 +2075,13 @@ init_no_runqs(int active, int used) { erts_aint32_t no_runqs = (erts_aint32_t) (active & ERTS_NO_RUNQS_MASK); no_runqs |= (erts_aint32_t) ((used & ERTS_NO_RUNQS_MASK) << ERTS_NO_USED_RUNQS_SHIFT); - erts_smp_atomic32_init(&balance_info.no_runqs, no_runqs); + erts_smp_atomic32_init_nob(&balance_info.no_runqs, no_runqs); } static ERTS_INLINE void get_no_runqs(int *active, int *used) { - erts_aint32_t no_runqs = erts_smp_atomic32_read(&balance_info.no_runqs); + erts_aint32_t no_runqs = erts_smp_atomic32_read_nob(&balance_info.no_runqs); if (active) *active = (int) (no_runqs & ERTS_NO_RUNQS_MASK); if (used) @@ -1353,11 +2091,12 @@ get_no_runqs(int *active, int *used) static ERTS_INLINE void set_no_used_runqs(int used) { - erts_aint32_t exp = erts_smp_atomic32_read(&balance_info.no_runqs); + erts_aint32_t exp = erts_smp_atomic32_read_nob(&balance_info.no_runqs); while (1) { erts_aint32_t act, new; - new = (used << ERTS_NO_USED_RUNQS_SHIFT) | (exp & ERTS_NO_RUNQS_MASK); - act = erts_smp_atomic32_cmpxchg(&balance_info.no_runqs, new, exp); + new = (used & ERTS_NO_RUNQS_MASK) << ERTS_NO_USED_RUNQS_SHIFT; + new |= exp & ERTS_NO_RUNQS_MASK; + act = erts_smp_atomic32_cmpxchg_nob(&balance_info.no_runqs, new, exp); if (act == exp) break; exp = act; @@ -1367,11 +2106,12 @@ set_no_used_runqs(int used) static ERTS_INLINE void set_no_active_runqs(int active) { - erts_aint32_t exp = erts_smp_atomic32_read(&balance_info.no_runqs); + erts_aint32_t exp = erts_smp_atomic32_read_nob(&balance_info.no_runqs); while (1) { erts_aint32_t act, new; - new = (exp & (ERTS_NO_RUNQS_MASK << ERTS_NO_USED_RUNQS_SHIFT)) | active; - act = erts_smp_atomic32_cmpxchg(&balance_info.no_runqs, new, exp); + new = exp & (ERTS_NO_RUNQS_MASK << ERTS_NO_USED_RUNQS_SHIFT); + new |= active & ERTS_NO_RUNQS_MASK; + act = erts_smp_atomic32_cmpxchg_nob(&balance_info.no_runqs, new, exp); if (act == exp) break; exp = act; @@ -1381,13 +2121,14 @@ set_no_active_runqs(int active) static ERTS_INLINE int try_inc_no_active_runqs(int active) { - erts_aint32_t exp = erts_smp_atomic32_read(&balance_info.no_runqs); + erts_aint32_t exp = erts_smp_atomic32_read_nob(&balance_info.no_runqs); if (((exp >> ERTS_NO_USED_RUNQS_SHIFT) & ERTS_NO_RUNQS_MASK) < active) return 0; if ((exp & ERTS_NO_RUNQS_MASK) + 1 == active) { erts_aint32_t new, act; - new = (exp & ~ERTS_NO_RUNQS_MASK) | active; - act = erts_smp_atomic32_cmpxchg(&balance_info.no_runqs, new, exp); + new = exp & (ERTS_NO_RUNQS_MASK << ERTS_NO_USED_RUNQS_SHIFT); + new |= active & ERTS_NO_RUNQS_MASK; + act = erts_smp_atomic32_cmpxchg_nob(&balance_info.no_runqs, new, exp); if (act == exp) return 1; } @@ -1403,7 +2144,7 @@ chk_wake_sched(ErtsRunQueue *crq, int ix, int activate) if (crq->ix == ix) return 0; wrq = ERTS_RUNQ_IX(ix); - iflgs = erts_smp_atomic32_read(&wrq->info_flags); + iflgs = erts_smp_atomic32_read_nob(&wrq->info_flags); if (!(iflgs & (ERTS_RUNQ_IFLG_SUSPENDED|ERTS_RUNQ_IFLG_NONEMPTY))) { if (activate) { if (try_inc_no_active_runqs(ix+1)) { @@ -1645,15 +2386,15 @@ evacuate_run_queue(ErtsRunQueue *evac_rq, ErtsRunQueue *rq) erts_smp_runq_lock(evac_rq); - erts_smp_atomic32_bor(&evac_rq->scheduler->ssi->flags, - ERTS_SSI_FLG_SUSPENDED); + erts_smp_atomic32_read_bor_nob(&evac_rq->scheduler->ssi->flags, + ERTS_SSI_FLG_SUSPENDED); evac_rq->flags &= ~ERTS_RUNQ_FLGS_IMMIGRATE_QMASK; evac_rq->flags |= (ERTS_RUNQ_FLGS_EMIGRATE_QMASK | ERTS_RUNQ_FLGS_EVACUATE_QMASK | ERTS_RUNQ_FLG_SUSPENDED); - erts_smp_atomic32_bor(&evac_rq->info_flags, ERTS_RUNQ_IFLG_SUSPENDED); + erts_smp_atomic32_read_bor_nob(&evac_rq->info_flags, ERTS_RUNQ_IFLG_SUSPENDED); /* * Need to set up evacuation paths first since we * may release the run queue lock on evac_rq @@ -1902,7 +2643,7 @@ static ERTS_INLINE int check_possible_steal_victim(ErtsRunQueue *rq, int *rq_lockedp, int vix) { ErtsRunQueue *vrq = ERTS_RUNQ_IX(vix); - erts_aint32_t iflgs = erts_smp_atomic32_read(&vrq->info_flags); + erts_aint32_t iflgs = erts_smp_atomic32_read_nob(&vrq->info_flags); if (iflgs & ERTS_RUNQ_IFLG_NONEMPTY) return try_steal_task_from_victim(rq, rq_lockedp, vrq); else @@ -2054,7 +2795,7 @@ check_balance(ErtsRunQueue *c_rq) int forced, active, current_active, oowc, half_full_scheds, full_scheds, mmax_len, blnc_no_rqs, qix, pix, freds_hist_ix; - if (erts_smp_atomic32_xchg(&balance_info.checking_balance, 1)) { + if (erts_smp_atomic32_xchg_nob(&balance_info.checking_balance, 1)) { c_rq->check_balance_reds = INT_MAX; return; } @@ -2062,7 +2803,7 @@ check_balance(ErtsRunQueue *c_rq) get_no_runqs(NULL, &blnc_no_rqs); if (blnc_no_rqs == 1) { c_rq->check_balance_reds = INT_MAX; - erts_smp_atomic32_set(&balance_info.checking_balance, 0); + erts_smp_atomic32_set_nob(&balance_info.checking_balance, 0); return; } @@ -2070,7 +2811,7 @@ check_balance(ErtsRunQueue *c_rq) if (balance_info.halftime) { balance_info.halftime = 0; - erts_smp_atomic32_set(&balance_info.checking_balance, 0); + erts_smp_atomic32_set_nob(&balance_info.checking_balance, 0); ERTS_FOREACH_RUNQ(rq, { if (rq->waiting) @@ -2104,7 +2845,7 @@ check_balance(ErtsRunQueue *c_rq) erts_smp_mtx_unlock(&balance_info.update_mtx); erts_smp_runq_lock(c_rq); c_rq->check_balance_reds = INT_MAX; - erts_smp_atomic32_set(&balance_info.checking_balance, 0); + erts_smp_atomic32_set_nob(&balance_info.checking_balance, 0); return; } @@ -2449,7 +3190,7 @@ erts_fprintf(stderr, "--------------------------------\n"); set_no_active_runqs(active); balance_info.halftime = 1; - erts_smp_atomic32_set(&balance_info.checking_balance, 0); + erts_smp_atomic32_set_nob(&balance_info.checking_balance, 0); /* Write migration paths and reset balance statistics in all queues */ for (qix = 0; qix < blnc_no_rqs; qix++) { @@ -2544,8 +3285,9 @@ erts_debug_nbalance(void) } void -erts_early_init_scheduling(void) +erts_early_init_scheduling(int no_schedulers) { + aux_work_timeout_early_init(no_schedulers); wakeup_other_limit = ERTS_WAKEUP_OTHER_LIMIT_MEDIUM; } @@ -2566,12 +3308,32 @@ erts_sched_set_wakeup_limit(char *str) return EINVAL; return 0; } - + +static void +init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp) +{ + awdp->sched_id = esdp ? (int) esdp->no : 0; + awdp->esdp = esdp; + awdp->ssi = esdp ? esdp->ssi : NULL; +#ifdef ERTS_SMP + awdp->misc.thr_prgr = ERTS_THR_PRGR_VAL_WAITING; + awdp->dd.thr_prgr = ERTS_THR_PRGR_VAL_WAITING; + awdp->dd.completed_callback = NULL; + awdp->dd.completed_arg = NULL; +#endif +#ifdef ERTS_USE_ASYNC_READY_Q +#ifdef ERTS_SMP + awdp->async_ready.need_thr_prgr = 0; + awdp->async_ready.thr_prgr = ERTS_THR_PRGR_VAL_WAITING; +#endif + awdp->async_ready.queue = NULL; +#endif +} void erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) { - int ix, n; + int ix, n, no_ssi; #ifndef ERTS_SMP mrq = 0; @@ -2591,7 +3353,7 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) erts_alloc_permanent_cache_aligned(ERTS_ALC_T_RUNQS, sizeof(ErtsAlignedRunQueue) * n); #ifdef ERTS_SMP - erts_smp_atomic32_init(&no_empty_run_queues, 0); + erts_smp_atomic32_init_nob(&no_empty_run_queues, 0); #endif erts_no_run_queues = n; @@ -2601,7 +3363,7 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); rq->ix = ix; - erts_smp_atomic32_init(&rq->info_flags, ERTS_RUNQ_IFLG_NONEMPTY); + erts_smp_atomic32_init_nob(&rq->info_flags, ERTS_RUNQ_IFLG_NONEMPTY); /* make sure that the "extra" id correponds to the schedulers * id if the esdp->no <-> ix+1 mapping change. @@ -2681,23 +3443,31 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) n = (int) no_schedulers; erts_no_schedulers = n; -#ifdef ERTS_SMP /* Create and initialize scheduler sleep info */ - +#ifdef ERTS_SMP + no_ssi = n+1; +#else + no_ssi = 1; +#endif aligned_sched_sleep_info = - erts_alloc_permanent_cache_aligned(ERTS_ALC_T_SCHDLR_SLP_INFO, - n * sizeof(ErtsAlignedSchedulerSleepInfo)); - - for (ix = 0; ix < n; ix++) { - ErtsSchedulerSleepInfo *ssi = ERTS_SCHED_SLEEP_INFO_IX(ix); + erts_alloc_permanent_cache_aligned( + ERTS_ALC_T_SCHDLR_SLP_INFO, + no_ssi*sizeof(ErtsAlignedSchedulerSleepInfo)); + for (ix = 0; ix < no_ssi; ix++) { + ErtsSchedulerSleepInfo *ssi = &aligned_sched_sleep_info[ix].ssi; +#ifdef ERTS_SMP #if 0 /* no need to initialize these... */ ssi->next = NULL; ssi->prev = NULL; #endif - erts_smp_atomic32_init(&ssi->flags, 0); + erts_smp_atomic32_init_nob(&ssi->flags, 0); ssi->event = NULL; /* initialized in sched_thread_func */ - erts_smp_atomic32_init(&ssi->aux_work, 0); +#endif + erts_atomic32_init_nob(&ssi->aux_work, 0); } + +#ifdef ERTS_SMP + aligned_sched_sleep_info++; #endif /* Create and initialize scheduler specific data */ @@ -2711,17 +3481,20 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) #ifdef ERTS_SMP erts_bits_init_state(&esdp->erl_bits_state); esdp->match_pseudo_process = NULL; - esdp->ssi = ERTS_SCHED_SLEEP_INFO_IX(ix); esdp->free_process = NULL; -#if HALFWORD_HEAP - /* Registers need to be heap allocated (correct memory range) for tracing to work */ - esdp->save_reg = erts_alloc(ERTS_ALC_T_BEAM_REGISTER, ERTS_X_REGS_ALLOCATED * sizeof(Eterm)); -#endif #endif + esdp->x_reg_array = + erts_alloc_permanent_cache_aligned(ERTS_ALC_T_BEAM_REGISTER, + ERTS_X_REGS_ALLOCATED * + sizeof(Eterm)); + esdp->f_reg_array = + erts_alloc_permanent_cache_aligned(ERTS_ALC_T_BEAM_REGISTER, + MAX_REG * sizeof(FloatDef)); #if !HEAP_ON_C_STACK esdp->num_tmp_heap_used = 0; #endif esdp->no = (Uint) ix+1; + esdp->ssi = ERTS_SCHED_SLEEP_INFO_IX(ix); esdp->current_process = NULL; esdp->current_port = NULL; @@ -2740,19 +3513,29 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) } #ifdef ERTS_SMP - erts_smp_atomic32_init(&esdp->chk_cpu_bind, 0); + erts_smp_atomic32_init_nob(&esdp->chk_cpu_bind, 0); #endif + init_aux_work_data(&esdp->aux_work_data, esdp); } + init_misc_aux_work(); + #ifdef ERTS_SMP + + erts_atomic32_init_nob(&completed_dealloc_count, 0); /* debug only */ + + aux_thread_aux_work_data = + erts_alloc_permanent_cache_aligned(ERTS_ALC_T_SCHDLR_DATA, + sizeof(ErtsAuxWorkData)); + erts_smp_mtx_init(&schdlr_sspnd.mtx, "schdlr_sspnd"); erts_smp_cnd_init(&schdlr_sspnd.cnd); - erts_smp_atomic32_init(&schdlr_sspnd.changing, 0); + erts_smp_atomic32_init_nob(&schdlr_sspnd.changing, 0); schdlr_sspnd.online = no_schedulers_online; schdlr_sspnd.curr_online = no_schedulers; - erts_smp_atomic32_init(&schdlr_sspnd.msb.ongoing, 0); - erts_smp_atomic32_init(&schdlr_sspnd.active, no_schedulers); + schdlr_sspnd.msb.ongoing = 0; + erts_smp_atomic32_init_nob(&schdlr_sspnd.active, no_schedulers); schdlr_sspnd.msb.procs = NULL; init_no_runqs(no_schedulers, erts_common_run_queue ? 1 : no_schedulers_online); @@ -2761,7 +3544,7 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) balance_info.forced_check_balance = 0; balance_info.halftime = 1; balance_info.full_reds_history_index = 0; - erts_smp_atomic32_init(&balance_info.checking_balance, 0); + erts_smp_atomic32_init_nob(&balance_info.checking_balance, 0); balance_info.prev_rise.active_runqs = 0; balance_info.prev_rise.max_len = 0; balance_info.prev_rise.reds = 0; @@ -2770,8 +3553,8 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) if (no_schedulers_online < no_schedulers) { if (erts_common_run_queue) { for (ix = no_schedulers_online; ix < no_schedulers; ix++) - erts_smp_atomic32_bor(&ERTS_SCHED_SLEEP_INFO_IX(ix)->flags, - ERTS_SSI_FLG_SUSPENDED); + erts_smp_atomic32_read_bor_nob(&ERTS_SCHED_SLEEP_INFO_IX(ix)->flags, + ERTS_SSI_FLG_SUSPENDED); } else { for (ix = no_schedulers_online; ix < erts_no_run_queues; ix++) @@ -2785,7 +3568,7 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) ERTS_SCHDLR_SSPND_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); - erts_smp_atomic32_init(&doing_sys_schedule, 0); + erts_smp_atomic32_init_nob(&doing_sys_schedule, 0); init_misc_aux_work(); @@ -2801,11 +3584,13 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) erts_no_schedulers = 1; #endif - erts_smp_atomic32_init(&function_calls, 0); + erts_smp_atomic32_init_nob(&function_calls, 0); /* init port tasks */ erts_port_task_init(); + aux_work_timeout_late_init(); + #ifndef ERTS_SMP #ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC erts_scheduler_data->verify_unused_temp_alloc @@ -2928,10 +3713,10 @@ int erts_get_max_no_executing_schedulers(void) { #ifdef ERTS_SMP - if (erts_smp_atomic32_read(&schdlr_sspnd.changing)) + if (erts_smp_atomic32_read_nob(&schdlr_sspnd.changing)) return (int) erts_no_schedulers; ERTS_THR_MEMORY_BARRIER; - return (int) erts_smp_atomic32_read(&schdlr_sspnd.active); + return (int) erts_smp_atomic32_read_nob(&schdlr_sspnd.active); #else return 1; #endif @@ -2940,18 +3725,6 @@ erts_get_max_no_executing_schedulers(void) #ifdef ERTS_SMP static void -susp_sched_prep_block(void *unused) -{ - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); -} - -static void -susp_sched_resume_block(void *unused) -{ - erts_smp_mtx_lock(&schdlr_sspnd.mtx); -} - -static void scheduler_ix_resume_wake(Uint ix) { ErtsSchedulerSleepInfo *ssi = ERTS_SCHED_SLEEP_INFO_IX(ix); @@ -2961,7 +3734,7 @@ scheduler_ix_resume_wake(Uint ix) | ERTS_SSI_FLG_SUSPENDED); erts_aint32_t oflgs; do { - oflgs = erts_smp_atomic32_cmpxchg(&ssi->flags, 0, xflgs); + oflgs = erts_smp_atomic32_cmpxchg_relb(&ssi->flags, 0, xflgs); if (oflgs == xflgs) { erts_sched_finish_poke(ssi, oflgs); break; @@ -2980,7 +3753,7 @@ sched_prep_spin_suspended(ErtsSchedulerSleepInfo *ssi, erts_aint32_t xpct) erts_aint32_t xflgs = xpct; do { - oflgs = erts_smp_atomic32_cmpxchg(&ssi->flags, nflgs, xflgs); + oflgs = erts_smp_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs); if (oflgs == xflgs) return nflgs; xflgs = oflgs; @@ -3030,7 +3803,7 @@ sched_set_suspended_sleeptype(ErtsSchedulerSleepInfo *ssi) erts_tse_reset(ssi->event); while (1) { - oflgs = erts_smp_atomic32_cmpxchg(&ssi->flags, nflgs, xflgs); + oflgs = erts_smp_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs); if (oflgs == xflgs) return nflgs; if ((oflgs & (ERTS_SSI_FLG_SLEEPING @@ -3055,10 +3828,8 @@ suspend_scheduler(ErtsSchedulerData *esdp) long active_schedulers; int curr_online = 1; int wake = 0; -#if defined(ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK) \ - || defined(ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK) erts_aint32_t aux_work; -#endif + int thr_prgr_active = 1; /* * Schedulers may be suspended in two different ways: @@ -3085,15 +3856,15 @@ suspend_scheduler(ErtsSchedulerData *esdp) flgs = sched_prep_spin_suspended(ssi, ERTS_SSI_FLG_SUSPENDED); if (flgs & ERTS_SSI_FLG_SUSPENDED) { - active_schedulers = erts_smp_atomic32_dectest(&schdlr_sspnd.active); + active_schedulers = erts_smp_atomic32_dec_read_nob(&schdlr_sspnd.active); ASSERT(active_schedulers >= 1); - changing = erts_smp_atomic32_read(&schdlr_sspnd.changing); + changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); if (changing & ERTS_SCHDLR_SSPND_CHNG_MSB) { if (active_schedulers == schdlr_sspnd.msb.wait_active) wake = 1; if (active_schedulers == 1) { - changing = erts_smp_atomic32_band(&schdlr_sspnd.changing, - ~ERTS_SCHDLR_SSPND_CHNG_MSB); + changing = erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, + ~ERTS_SCHDLR_SSPND_CHNG_MSB); changing &= ~ERTS_SCHDLR_SSPND_CHNG_MSB; } } @@ -3115,8 +3886,8 @@ suspend_scheduler(ErtsSchedulerData *esdp) && schdlr_sspnd.curr_online == schdlr_sspnd.wait_curr_online) wake = 1; if (schdlr_sspnd.online == schdlr_sspnd.curr_online) { - changing = erts_smp_atomic32_band(&schdlr_sspnd.changing, - ~ERTS_SCHDLR_SSPND_CHNG_ONLN); + changing = erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, + ~ERTS_SCHDLR_SSPND_CHNG_ONLN); changing &= ~ERTS_SCHDLR_SSPND_CHNG_ONLN; } } @@ -3131,75 +3902,65 @@ suspend_scheduler(ErtsSchedulerData *esdp) break; erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - -#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK - aux_work = erts_smp_atomic32_read(&ssi->aux_work); - blockable_aux_work: - blockable_aux_work(esdp, ssi, aux_work); -#endif - - erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); while (1) { erts_aint32_t flgs; -#ifdef ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK -#ifndef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK - aux_work = erts_smp_atomic32_read(&ssi->aux_work); -#endif - nonblockable_aux_work(esdp, ssi, aux_work); -#endif - flgs = sched_spin_suspended(ssi, - ERTS_SCHED_SUSPEND_SLEEP_SPINCOUNT); - if (flgs == (ERTS_SSI_FLG_SLEEPING - | ERTS_SSI_FLG_WAITING - | ERTS_SSI_FLG_SUSPENDED)) { - flgs = sched_set_suspended_sleeptype(ssi); + aux_work = erts_atomic32_read_acqb(&ssi->aux_work); + if (aux_work) { + if (!thr_prgr_active) + erts_thr_progress_active(esdp, thr_prgr_active = 1); + aux_work = handle_aux_work(&esdp->aux_work_data, aux_work); + if (aux_work && erts_thr_progress_update(esdp)) + erts_thr_progress_leader_update(esdp); + } + + if (!aux_work) { + if (thr_prgr_active) + erts_thr_progress_active(esdp, thr_prgr_active = 0); + erts_thr_progress_prepare_wait(esdp); + flgs = sched_spin_suspended(ssi, + ERTS_SCHED_SUSPEND_SLEEP_SPINCOUNT); if (flgs == (ERTS_SSI_FLG_SLEEPING - | ERTS_SSI_FLG_TSE_SLEEPING | ERTS_SSI_FLG_WAITING | ERTS_SSI_FLG_SUSPENDED)) { - int res; - do { - res = erts_tse_wait(ssi->event); - } while (res == EINTR); + flgs = sched_set_suspended_sleeptype(ssi); + if (flgs == (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_TSE_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED)) { + int res; + + do { + res = erts_tse_wait(ssi->event); + } while (res == EINTR); + } } + erts_thr_progress_finalize_wait(esdp); } flgs = sched_prep_spin_suspended(ssi, (ERTS_SSI_FLG_WAITING | ERTS_SSI_FLG_SUSPENDED)); if (!(flgs & ERTS_SSI_FLG_SUSPENDED)) break; - changing = erts_smp_atomic32_read(&schdlr_sspnd.changing); + changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); if (changing & ~ERTS_SCHDLR_SSPND_CHNG_WAITER) break; - - -#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK - aux_work = erts_smp_atomic32_read(&ssi->aux_work); - if (aux_work & ERTS_SSI_BLOCKABLE_AUX_WORK_MASK) { - erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); - goto blockable_aux_work; - } -#endif - } - erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); - erts_smp_mtx_lock(&schdlr_sspnd.mtx); - changing = erts_smp_atomic32_read(&schdlr_sspnd.changing); + changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); } - active_schedulers = erts_smp_atomic32_inctest(&schdlr_sspnd.active); - changing = erts_smp_atomic32_read(&schdlr_sspnd.changing); + active_schedulers = erts_smp_atomic32_inc_read_nob(&schdlr_sspnd.active); + changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); if ((changing & ERTS_SCHDLR_SSPND_CHNG_MSB) && schdlr_sspnd.online == active_schedulers) { - erts_smp_atomic32_band(&schdlr_sspnd.changing, - ~ERTS_SCHDLR_SSPND_CHNG_MSB); + erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, + ~ERTS_SCHDLR_SSPND_CHNG_MSB); } ASSERT(no <= schdlr_sspnd.online); - ASSERT(!erts_smp_atomic32_read(&schdlr_sspnd.msb.ongoing)); + ASSERT(!ongoing_multi_scheduling_block()); } @@ -3210,6 +3971,9 @@ suspend_scheduler(ErtsSchedulerData *esdp) if (erts_system_profile_flags.scheduler) profile_scheduler(make_small(esdp->no), am_active); + if (!thr_prgr_active) + erts_thr_progress_active(esdp, thr_prgr_active = 1); + erts_smp_runq_lock(esdp->run_queue); non_empty_runq(esdp->run_queue); @@ -3228,7 +3992,7 @@ do { \ (RQ)->flags |= (ERTS_RUNQ_FLG_OUT_OF_WORK \ | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK); \ (RQ)->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS; \ - erts_smp_atomic32_band(&(RQ)->info_flags, ~ERTS_RUNQ_IFLG_SUSPENDED);\ + erts_smp_atomic32_read_band_nob(&(RQ)->info_flags, ~ERTS_RUNQ_IFLG_SUSPENDED);\ for (pix__ = 0; pix__ < ERTS_NO_PROC_PRIO_LEVELS; pix__++) { \ (RQ)->procs.prio_info[pix__].max_len = 0; \ (RQ)->procs.prio_info[pix__].reds = 0; \ @@ -3272,7 +4036,7 @@ erts_schedulers_state(Uint *total, int res; erts_aint32_t changing; erts_smp_mtx_lock(&schdlr_sspnd.mtx); - changing = erts_smp_atomic32_read(&schdlr_sspnd.changing); + changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); if (yield_allowed && (changing & ~ERTS_SCHDLR_SSPND_CHNG_WAITER)) res = ERTS_SCHDLR_SSPND_YIELD_RESTART; else { @@ -3292,18 +4056,22 @@ erts_set_schedulers_online(Process *p, Sint new_no, Sint *old_no) { - int ix, res, no, have_unlocked_plocks; + ErtsSchedulerData *esdp; + int ix, res, no, have_unlocked_plocks, end_wait; erts_aint32_t changing; if (new_no < 1 || erts_no_schedulers < new_no) return ERTS_SCHDLR_SSPND_EINVAL; + esdp = ERTS_PROC_GET_SCHDATA(p); + end_wait = 0; + erts_smp_mtx_lock(&schdlr_sspnd.mtx); have_unlocked_plocks = 0; no = (int) new_no; - changing = erts_smp_atomic32_read(&schdlr_sspnd.changing); + changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); if (changing) { res = ERTS_SCHDLR_SSPND_YIELD_RESTART; } @@ -3378,8 +4146,8 @@ erts_set_schedulers_online(Process *p, for (ix = no; ix < online; ix++) { ErtsSchedulerSleepInfo *ssi; ssi = ERTS_SCHED_SLEEP_INFO_IX(ix); - erts_smp_atomic32_bor(&ssi->flags, - ERTS_SSI_FLG_SUSPENDED); + erts_smp_atomic32_read_bor_nob(&ssi->flags, + ERTS_SSI_FLG_SUSPENDED); } wake_all_schedulers(); } @@ -3414,27 +4182,37 @@ erts_set_schedulers_online(Process *p, } } - erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, - susp_sched_prep_block, - susp_sched_resume_block, - NULL); + if (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) { + erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + if (plocks && !have_unlocked_plocks) { + have_unlocked_plocks = 1; + erts_smp_proc_unlock(p, plocks); + } + erts_thr_progress_active(esdp, 0); + erts_thr_progress_prepare_wait(esdp); + end_wait = 1; + erts_smp_mtx_lock(&schdlr_sspnd.mtx); + } + while (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); - erts_smp_activity_end(ERTS_ACTIVITY_WAIT, - susp_sched_prep_block, - susp_sched_resume_block, - NULL); + ASSERT(res != ERTS_SCHDLR_SSPND_DONE ? (ERTS_SCHDLR_SSPND_CHNG_WAITER - & erts_smp_atomic32_read(&schdlr_sspnd.changing)) + & erts_smp_atomic32_read_nob(&schdlr_sspnd.changing)) : (ERTS_SCHDLR_SSPND_CHNG_WAITER - == erts_smp_atomic32_read(&schdlr_sspnd.changing))); - erts_smp_atomic32_band(&schdlr_sspnd.changing, - ~ERTS_SCHDLR_SSPND_CHNG_WAITER); + == erts_smp_atomic32_read_nob(&schdlr_sspnd.changing))); + erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, + ~ERTS_SCHDLR_SSPND_CHNG_WAITER); + } } erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + if (end_wait) { + erts_thr_progress_finalize_wait(esdp); + erts_thr_progress_active(esdp, 1); + } if (have_unlocked_plocks) erts_smp_proc_lock(p, plocks); @@ -3449,7 +4227,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) ErtsProcList *plp; erts_smp_mtx_lock(&schdlr_sspnd.mtx); - changing = erts_smp_atomic32_read(&schdlr_sspnd.changing); + changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); if (changing) { res = ERTS_SCHDLR_SSPND_YIELD_RESTART; /* Yield */ } @@ -3459,7 +4237,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) plp->next = schdlr_sspnd.msb.procs; schdlr_sspnd.msb.procs = plp; p->flags |= F_HAVE_BLCKD_MSCHED; - ASSERT(erts_smp_atomic32_read(&schdlr_sspnd.active) == 1); + ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.active) == 1); ASSERT(p->scheduler_data->no == 1); res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; } @@ -3470,11 +4248,11 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) have_unlocked_plocks = 1; erts_smp_proc_unlock(p, plocks); } - ASSERT(0 == erts_smp_atomic32_read(&schdlr_sspnd.msb.ongoing)); - erts_smp_atomic32_set(&schdlr_sspnd.msb.ongoing, 1); + ASSERT(!ongoing_multi_scheduling_block()); + schdlr_sspnd.msb.ongoing = 1; if (online == 1) { res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; - ASSERT(erts_smp_atomic32_read(&schdlr_sspnd.active) == 1); + ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.active) == 1); ASSERT(p->scheduler_data->no == 1); } else { @@ -3494,8 +4272,8 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) } if (erts_common_run_queue) { for (ix = 1; ix < online; ix++) - erts_smp_atomic32_bor(&ERTS_SCHED_SLEEP_INFO_IX(ix)->flags, - ERTS_SSI_FLG_SUSPENDED); + erts_smp_atomic32_read_bor_nob(&ERTS_SCHED_SLEEP_INFO_IX(ix)->flags, + ERTS_SSI_FLG_SUSPENDED); wake_all_schedulers(); } else { @@ -3519,24 +4297,45 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) erts_smp_mtx_unlock(&balance_info.update_mtx); erts_smp_mtx_lock(&schdlr_sspnd.mtx); } - erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, - susp_sched_prep_block, - susp_sched_resume_block, - NULL); - while (erts_smp_atomic32_read(&schdlr_sspnd.active) - != schdlr_sspnd.msb.wait_active) - erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); - erts_smp_activity_end(ERTS_ACTIVITY_WAIT, - susp_sched_prep_block, - susp_sched_resume_block, - NULL); + + if (erts_smp_atomic32_read_nob(&schdlr_sspnd.active) + != schdlr_sspnd.msb.wait_active) { + ErtsSchedulerData *esdp; + + erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + + if (plocks && !have_unlocked_plocks) { + have_unlocked_plocks = 1; + erts_smp_proc_unlock(p, plocks); + } + + esdp = ERTS_PROC_GET_SCHDATA(p); + + erts_thr_progress_active(esdp, 0); + erts_thr_progress_prepare_wait(esdp); + + erts_smp_mtx_lock(&schdlr_sspnd.mtx); + + while (erts_smp_atomic32_read_nob(&schdlr_sspnd.active) + != schdlr_sspnd.msb.wait_active) + erts_smp_cnd_wait(&schdlr_sspnd.cnd, + &schdlr_sspnd.mtx); + + erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + + erts_thr_progress_active(esdp, 1); + erts_thr_progress_finalize_wait(esdp); + + erts_smp_mtx_lock(&schdlr_sspnd.mtx); + + } ASSERT(res != ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED ? (ERTS_SCHDLR_SSPND_CHNG_WAITER - & erts_smp_atomic32_read(&schdlr_sspnd.changing)) + & erts_smp_atomic32_read_nob(&schdlr_sspnd.changing)) : (ERTS_SCHDLR_SSPND_CHNG_WAITER - == erts_smp_atomic32_read(&schdlr_sspnd.changing))); - erts_smp_atomic32_band(&schdlr_sspnd.changing, - ~ERTS_SCHDLR_SSPND_CHNG_WAITER); + == erts_smp_atomic32_read_nob(&schdlr_sspnd.changing))); + erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, + ~ERTS_SCHDLR_SSPND_CHNG_WAITER); } plp = proclist_create(p); plp->next = schdlr_sspnd.msb.procs; @@ -3603,16 +4402,16 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) }); #endif p->flags &= ~F_HAVE_BLCKD_MSCHED; - erts_smp_atomic32_set(&schdlr_sspnd.msb.ongoing, 0); + schdlr_sspnd.msb.ongoing = 0; if (schdlr_sspnd.online == 1) { /* No schedulers to resume */ - ASSERT(erts_smp_atomic32_read(&schdlr_sspnd.active) == 1); + ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.active) == 1); ERTS_SCHDLR_SSPND_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_MSB); } else if (erts_common_run_queue) { for (ix = 1; ix < schdlr_sspnd.online; ix++) - erts_smp_atomic32_band(&ERTS_SCHED_SLEEP_INFO_IX(ix)->flags, - ~ERTS_SSI_FLG_SUSPENDED); + erts_smp_atomic32_read_band_nob(&ERTS_SCHED_SLEEP_INFO_IX(ix)->flags, + ~ERTS_SSI_FLG_SUSPENDED); wake_all_schedulers(); } else { @@ -3662,7 +4461,7 @@ void erts_dbg_multi_scheduling_return_trap(Process *p, Eterm return_value) { if (return_value == am_blocked) { - erts_aint32_t active = erts_smp_atomic32_read(&schdlr_sspnd.active); + erts_aint32_t active = erts_smp_atomic32_read_nob(&schdlr_sspnd.active); ASSERT(1 <= active && active <= 2); ASSERT(ERTS_PROC_GET_SCHDATA(p)->no == 1); } @@ -3717,8 +4516,19 @@ erts_multi_scheduling_blockers(Process *p) static void * sched_thread_func(void *vesdp) { + ErtsThrPrgrCallbacks callbacks; + ErtsSchedulerData *esdp = vesdp; + Uint no = esdp->no; #ifdef ERTS_SMP - Uint no = ((ErtsSchedulerData *) vesdp)->no; + ERTS_SCHED_SLEEP_INFO_IX(no - 1)->event = erts_tse_fetch(); + callbacks.arg = (void *) esdp->ssi; + callbacks.wakeup = thr_prgr_wakeup; + callbacks.prepare_wait = thr_prgr_prep_wait; + callbacks.wait = thr_prgr_wait; + callbacks.finalize_wait = thr_prgr_fin_wait; + + erts_thr_progress_register_managed_thread(esdp, &callbacks, 0); + erts_alloc_register_scheduler(vesdp); #endif #ifdef ERTS_ENABLE_LOCK_CHECK { @@ -3727,65 +4537,71 @@ sched_thread_func(void *vesdp) erts_lc_set_thread_name(&buf[0]); } #endif - erts_alloc_reg_scheduler_id(no); erts_tsd_set(sched_data_key, vesdp); #ifdef ERTS_SMP +#if HAVE_ERTS_MSEG + erts_mseg_late_init(); +#endif +#if ERTS_USE_ASYNC_READY_Q + esdp->aux_work_data.async_ready.queue = erts_get_async_ready_queue(no); +#endif - erts_sched_init_check_cpu_bind((ErtsSchedulerData *) vesdp); + erts_sched_init_check_cpu_bind(esdp); erts_proc_lock_prepare_proc_lock_waiter(); - ERTS_SCHED_SLEEP_INFO_IX(no - 1)->event = erts_tse_fetch(); - - #endif - erts_register_blockable_thread(); + #ifdef HIPE hipe_thread_signal_init(); #endif erts_thread_init_float(); + + if (no == 1) { + erts_thr_progress_active(esdp, 0); + erts_thr_progress_prepare_wait(esdp); + } + erts_smp_mtx_lock(&schdlr_sspnd.mtx); - ASSERT(erts_smp_atomic32_read(&schdlr_sspnd.changing) + ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.changing) & ERTS_SCHDLR_SSPND_CHNG_ONLN); if (--schdlr_sspnd.curr_online == schdlr_sspnd.wait_curr_online) { - erts_smp_atomic32_band(&schdlr_sspnd.changing, - ~ERTS_SCHDLR_SSPND_CHNG_ONLN); - if (((ErtsSchedulerData *) vesdp)->no != 1) + erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, + ~ERTS_SCHDLR_SSPND_CHNG_ONLN); + if (no != 1) erts_smp_cnd_signal(&schdlr_sspnd.cnd); } - if (((ErtsSchedulerData *) vesdp)->no == 1) { - if (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) { - erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, - susp_sched_prep_block, - susp_sched_resume_block, - NULL); - while (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) - erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); - erts_smp_activity_end(ERTS_ACTIVITY_WAIT, - susp_sched_prep_block, - susp_sched_resume_block, - NULL); - } + if (no == 1) { + while (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) + erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); ERTS_SCHDLR_SSPND_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER); } erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + if (no == 1) { + erts_thr_progress_finalize_wait(esdp); + erts_thr_progress_active(esdp, 1); + } + #ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC - ((ErtsSchedulerData *) vesdp)->verify_unused_temp_alloc + esdp->verify_unused_temp_alloc = erts_alloc_get_verify_unused_temp_alloc( - &((ErtsSchedulerData *) vesdp)->verify_unused_temp_alloc_data); + &esdp->verify_unused_temp_alloc_data); ERTS_VERIFY_UNUSED_TEMP_ALLOC(NULL); #endif process_main(); /* No schedulers should *ever* terminate */ - erl_exit(ERTS_ABORT_EXIT, "Scheduler thread number %beu terminated\n", - ((ErtsSchedulerData *) vesdp)->no); + erl_exit(ERTS_ABORT_EXIT, + "Scheduler thread number %beu terminated\n", + no); return NULL; } +static ethr_tid aux_tid; + void erts_start_schedulers(void) { @@ -3805,8 +4621,6 @@ erts_start_schedulers(void) res = ENOTSUP; } - erts_block_system(0); - while (actual < wanted) { ErtsSchedulerData *esdp = ERTS_SCHEDULER_IX(actual); actual++; @@ -3819,7 +4633,12 @@ erts_start_schedulers(void) } erts_no_schedulers = actual; - erts_release_system(); + + ERTS_THR_MEMORY_BARRIER; + + res = ethr_thr_create(&aux_tid, aux_thread, NULL, &opts); + if (res != 0) + erl_exit(1, "Failed to create aux thread\n"); if (actual < 1) erl_exit(1, @@ -5180,7 +5999,7 @@ Process *schedule(Process *p, int calls) input_reductions = INPUT_REDUCTIONS; } - ERTS_SMP_LC_ASSERT(!ERTS_LC_IS_BLOCKING); + ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking()); /* * Clean up after the process being scheduled out. @@ -5207,7 +6026,7 @@ Process *schedule(Process *p, int calls) reds = ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST; esdp->virtual_reds = 0; - fcalls = (int) erts_smp_atomic32_addtest(&function_calls, reds); + fcalls = (int) erts_smp_atomic32_add_read_acqb(&function_calls, reds); ASSERT(esdp && esdp == erts_get_scheduler_data()); rq = erts_get_runq_current(esdp); @@ -5318,7 +6137,8 @@ Process *schedule(Process *p, int calls) } - ERTS_SMP_LC_ASSERT(!ERTS_LC_IS_BLOCKING); + ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking()); + check_activities_to_run: { #ifdef ERTS_SMP @@ -5328,7 +6148,7 @@ Process *schedule(Process *p, int calls) check_balance(rq); } - ERTS_SMP_LC_ASSERT(!ERTS_LC_IS_BLOCKING); + ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking()); ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); if (rq->flags & ERTS_RUNQ_FLGS_IMMIGRATE_QMASK) @@ -5342,7 +6162,7 @@ Process *schedule(Process *p, int calls) if ((rq->flags & ERTS_RUNQ_FLG_SUSPENDED) || (erts_smp_atomic32_read_acqb(&esdp->ssi->flags) & ERTS_SSI_FLG_SUSPENDED)) { - ASSERT(erts_smp_atomic32_read(&esdp->ssi->flags) + ASSERT(erts_smp_atomic32_read_nob(&esdp->ssi->flags) & ERTS_SSI_FLG_SUSPENDED); suspend_scheduler(esdp); } @@ -5352,42 +6172,38 @@ Process *schedule(Process *p, int calls) } } -#if defined(ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK) \ - || defined(ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK) { - ErtsSchedulerSleepInfo *ssi = esdp->ssi; - erts_aint32_t aux_work = erts_smp_atomic32_read(&ssi->aux_work); - if (aux_work) { + erts_aint32_t aux_work; + int leader_update = erts_thr_progress_update(esdp); + aux_work = erts_atomic32_read_acqb(&esdp->ssi->aux_work); + if (aux_work | leader_update) { erts_smp_runq_unlock(rq); -#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK - aux_work = blockable_aux_work(esdp, ssi, aux_work); -#endif -#ifdef ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK - nonblockable_aux_work(esdp, ssi, aux_work); -#endif + if (leader_update) + erts_thr_progress_leader_update(esdp); + if (aux_work) + handle_aux_work(&esdp->aux_work_data, aux_work); erts_smp_runq_lock(rq); } } -#endif - - erts_smp_chk_system_block(prepare_for_block, - resume_after_block, - (void *) rq); - ERTS_SMP_LC_ASSERT(!ERTS_LC_IS_BLOCKING); + ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking()); ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); -#endif +#else /* ERTS_SMP */ + { + erts_aint32_t aux_work; + aux_work = erts_atomic32_read_acqb(&esdp->ssi->aux_work); + if (aux_work) + handle_aux_work(&esdp->aux_work_data, aux_work); + } +#endif /* ERTS_SMP */ ASSERT(rq->len == rq->procs.len + rq->ports.info.len); -#ifndef ERTS_SMP + if (rq->len == 0 && !rq->misc.start) { - if (rq->len == 0 && !rq->misc.start) - goto do_sys_schedule; +#ifdef ERTS_SMP -#else /* ERTS_SMP */ - if (rq->len == 0 && !rq->misc.start) { ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); rq->wakeup_other = 0; @@ -5400,7 +6216,7 @@ Process *schedule(Process *p, int calls) if ((rq->flags & ERTS_RUNQ_FLG_SUSPENDED) || (erts_smp_atomic32_read_acqb(&esdp->ssi->flags) & ERTS_SSI_FLG_SUSPENDED)) { - ASSERT(erts_smp_atomic32_read(&esdp->ssi->flags) + ASSERT(erts_smp_atomic32_read_nob(&esdp->ssi->flags) & ERTS_SSI_FLG_SUSPENDED); non_empty_runq(rq); goto continue_check_activities_to_run; @@ -5418,26 +6234,17 @@ Process *schedule(Process *p, int calls) } } +#endif + scheduler_wait(&fcalls, esdp, rq); +#ifdef ERTS_SMP non_empty_runq(rq); +#endif goto check_activities_to_run; } - else -#endif /* ERTS_SMP */ - if (fcalls > input_reductions && prepare_for_sys_schedule()) { - int runnable; - -#ifdef ERTS_SMP - runnable = 1; -#else - do_sys_schedule: - runnable = rq->len != 0; - if (!runnable) - sched_waiting_sys(esdp->no, rq); -#endif - + else if (fcalls > input_reductions && prepare_for_sys_schedule()) { /* * Schedule system-level activities. */ @@ -5447,11 +6254,11 @@ Process *schedule(Process *p, int calls) ASSERT(!erts_port_task_have_outstanding_io_tasks()); -#ifdef ERTS_SMP - /* erts_sys_schedule_interrupt(0); */ +#if 0 /* Not needed since we wont wait in sys schedule */ + erts_sys_schedule_interrupt(0); #endif erts_smp_runq_unlock(rq); - erl_sys_schedule(runnable); + erl_sys_schedule(1); dt = erts_do_time_read_and_reset(); if (dt) erts_bump_timer(dt); #ifdef ERTS_SMP @@ -5459,8 +6266,6 @@ Process *schedule(Process *p, int calls) clear_sys_scheduling(); goto continue_check_activities_to_run; #else - if (!runnable) - sched_active_sys(esdp->no, rq); goto check_activities_to_run; #endif } @@ -5708,14 +6513,14 @@ erts_sched_stat_modify(int what) int ix; switch (what) { case ERTS_SCHED_STAT_MODIFY_ENABLE: - erts_smp_block_system(0); + erts_smp_thr_progress_block(); erts_sched_stat.enabled = 1; - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); break; case ERTS_SCHED_STAT_MODIFY_DISABLE: - erts_smp_block_system(0); + erts_smp_thr_progress_block(); erts_sched_stat.enabled = 1; - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); break; case ERTS_SCHED_STAT_MODIFY_CLEAR: erts_smp_spin_lock(&erts_sched_stat.lock); @@ -5775,20 +6580,9 @@ erts_sched_stat_term(Process *p, int total) void erts_schedule_misc_op(void (*func)(void *), void *arg) { - ErtsRunQueue *rq; - ErtsMiscOpList *molp = misc_op_list_alloc(); ErtsSchedulerData *esdp = erts_get_scheduler_data(); - - if (esdp) { - rq = esdp->run_queue; - } else { - /* - * This can only happen when the sys msg dispatcher - * thread schedules misc ops (this happens *very* - * seldom; only when trace drivers are unloaded). - */ - rq = ERTS_RUNQ_IX(0); - } + ErtsRunQueue *rq = esdp ? esdp->run_queue : ERTS_RUNQ_IX(0); + ErtsMiscOpList *molp = misc_op_list_alloc(); erts_smp_runq_lock(rq); @@ -5881,7 +6675,7 @@ erts_get_exact_total_reductions(Process *c_p, Uint *redsp, Uint *diffp) * Wait for other schedulers to schedule out their processes * and update 'reductions'. */ - erts_smp_block_system(0); + erts_smp_thr_progress_block(); for (reds = 0, ix = 0; ix < erts_no_run_queues; ix++) reds += ERTS_RUNQ_IX(ix)->procs.reductions; if (redsp) @@ -5889,7 +6683,7 @@ erts_get_exact_total_reductions(Process *c_p, Uint *redsp, Uint *diffp) if (diffp) *diffp = reds - last_exact_reductions; last_exact_reductions = reds; - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); } @@ -5944,7 +6738,7 @@ erts_test_next_pid(int set, Uint next) Uint erts_process_count(void) { - erts_aint32_t res = erts_smp_atomic32_read(&process_count); + erts_aint32_t res = erts_smp_atomic32_read_nob(&process_count); ASSERT(res >= 0); return (Uint) res; } @@ -5993,7 +6787,7 @@ alloc_process(void) ASSERT(!process_tab[p_next]); process_tab[p_next] = p; - erts_smp_atomic32_inc(&process_count); + erts_smp_atomic32_inc_nob(&process_count); p->id = make_internal_pid(p_serial << p_serial_shift | p_next); if (p->id == ERTS_INVALID_PID) { /* Do not use the invalid pid; change serial */ @@ -6119,7 +6913,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->min_heap_size = H_MIN_SIZE; p->min_vheap_size = BIN_VH_MIN_SIZE; p->prio = PRIORITY_NORMAL; - p->max_gen_gcs = (Uint16) erts_smp_atomic32_read(&erts_max_gen_gcs); + p->max_gen_gcs = (Uint16) erts_smp_atomic32_read_nob(&erts_max_gen_gcs); } p->skipped = 0; ASSERT(p->min_heap_size == erts_next_heap_size(p->min_heap_size, 0)); @@ -7576,8 +8370,8 @@ continue_exit_process(Process *p p->status_flags = 0; #endif process_tab[pix] = NULL; /* Time of death! */ - ASSERT(erts_smp_atomic32_read(&process_count) > 0); - erts_smp_atomic32_dec(&process_count); + ASSERT(erts_smp_atomic32_read_nob(&process_count) > 0); + erts_smp_atomic32_dec_nob(&process_count); #ifdef ERTS_SMP erts_pix_unlock(pix_lock); @@ -8694,6 +9488,22 @@ init_processes_bif(void) * Debug stuff */ +#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) +int +erts_dbg_check_halloc_lock(Process *p) +{ + if (ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p)) + return 1; + if (p->id == ERTS_INVALID_PID) + return 1; + if (p->scheduler_data && p == p->scheduler_data->match_pseudo_process) + return 1; + if (erts_thr_progress_is_blocking()) + return 1; + return 0; +} +#endif + Eterm erts_debug_processes(Process *c_p) { diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 296acc7367..4027fade35 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -53,11 +53,18 @@ typedef struct process Process; #include "erl_time.h" #include "erl_atom_table.h" #include "external.h" +#include "erl_mseg.h" +#include "erl_async.h" #ifdef HIPE #include "hipe_process.h" #endif +#undef ERL_THR_PROGRESS_TSD_TYPE_ONLY +#define ERL_THR_PROGRESS_TSD_TYPE_ONLY +#include "erl_thr_progress.h" +#undef ERL_THR_PROGRESS_TSD_TYPE_ONLY + struct ErtsNodesMonitor_; struct port; @@ -242,16 +249,25 @@ typedef enum { | ERTS_SSI_FLG_WAITING \ | ERTS_SSI_FLG_SUSPENDED) -#define ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK - -#define ERTS_SSI_AUX_WORK_CHECK_CHILDREN (((erts_aint32_t) 1) << 0) -#define ERTS_SSI_AUX_WORK_MISC (((erts_aint32_t) 1) << 1) +#define ERTS_SSI_AUX_WORK_SET_TMO (((erts_aint32_t) 1) << 0) +#define ERTS_SSI_AUX_WORK_CHECK_CHILDREN (((erts_aint32_t) 1) << 1) +#define ERTS_SSI_AUX_WORK_MISC (((erts_aint32_t) 1) << 2) +#ifdef ERTS_SMP +#define ERTS_SSI_AUX_WORK_MISC_THR_PRGR (((erts_aint32_t) 1) << 3) +#endif +#define ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM (((erts_aint32_t) 1) << 4) +#define ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC (((erts_aint32_t) 1) << 5) +#define ERTS_SSI_AUX_WORK_ASYNC_READY (((erts_aint32_t) 1) << 6) +#define ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN (((erts_aint32_t) 1) << 7) +#ifdef ERTS_SMP +#define ERTS_SSI_AUX_WORK_DD (((erts_aint32_t) 1) << 8) +#define ERTS_SSI_AUX_WORK_DD_THR_PRGR (((erts_aint32_t) 1) << 9) +#endif +#define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK (((erts_aint32_t) 1) << 10) -#define ERTS_SSI_BLOCKABLE_AUX_WORK_MASK \ - (ERTS_SSI_AUX_WORK_CHECK_CHILDREN \ - | ERTS_SSI_AUX_WORK_MISC) -#define ERTS_SSI_NONBLOCKABLE_AUX_WORK_MASK \ - (0) +#if !HAVE_ERTS_MSEG +# undef ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK +#endif typedef struct ErtsSchedulerSleepInfo_ ErtsSchedulerSleepInfo; @@ -261,11 +277,13 @@ typedef struct { } ErtsSchedulerSleepList; struct ErtsSchedulerSleepInfo_ { +#ifdef ERTS_SMP ErtsSchedulerSleepInfo *next; ErtsSchedulerSleepInfo *prev; erts_smp_atomic32_t flags; erts_tse_t *event; - erts_smp_atomic32_t aux_work; +#endif + erts_atomic32_t aux_work; }; /* times to reschedule low prio process before running */ @@ -386,25 +404,49 @@ do { \ (RQ)->wakeup_other_reds += (REDS); \ } while (0) -struct ErtsSchedulerData_ { - +typedef struct { + int sched_id; + ErtsSchedulerData *esdp; + ErtsSchedulerSleepInfo *ssi; + struct { + int ix; #ifdef ERTS_SMP + ErtsThrPrgrVal thr_prgr; +#endif + } misc; +#ifdef ERTS_SMP + struct { + ErtsThrPrgrVal thr_prgr; + void (*completed_callback)(void *); + void (*completed_arg)(void *); + } dd; +#endif +#ifdef ERTS_USE_ASYNC_READY_Q + struct { +#ifdef ERTS_SMP + int need_thr_prgr; + ErtsThrPrgrVal thr_prgr; +#endif + void *queue; + } async_ready; +#endif +} ErtsAuxWorkData; + +struct ErtsSchedulerData_ { /* * Keep X registers first (so we get as many low * numbered registers as possible in the same cache * line). */ -#if !HALFWORD_HEAP - Eterm save_reg[ERTS_X_REGS_ALLOCATED]; /* X registers */ -#else - Eterm *save_reg; -#endif - FloatDef freg[MAX_REG]; /* Floating point registers. */ + Eterm* x_reg_array; /* X registers */ + FloatDef* f_reg_array; /* Floating point registers. */ + +#ifdef ERTS_SMP ethr_tid tid; /* Thread id */ struct erl_bits_state erl_bits_state; /* erl_bits.c state */ void *match_pseudo_process; /* erl_db_util.c:db_prog_match() */ - ErtsSchedulerSleepInfo *ssi; Process *free_process; + ErtsThrPrgrData thr_progress_data; #endif #if !HEAP_ON_C_STACK Eterm tmp_heap[TMP_HEAP_SIZE]; @@ -413,16 +455,19 @@ struct ErtsSchedulerData_ { Eterm cmp_tmp_heap[CMP_TMP_HEAP_SIZE]; Eterm erl_arith_tmp_heap[ERL_ARITH_TMP_HEAP_SIZE]; #endif - + ErtsSchedulerSleepInfo *ssi; Process *current_process; Uint no; /* Scheduler number */ struct port *current_port; ErtsRunQueue *run_queue; int virtual_reds; int cpu_id; /* >= 0 when bound */ + ErtsAuxWorkData aux_work_data; ErtsAtomCacheMap atom_cache_map; + ErtsSchedAllocData alloc_data; + #ifdef ERTS_SMP /* NOTE: These fields are modified under held mutexes by other threads */ erts_smp_atomic32_t chk_cpu_bind; /* Only used when common run queue */ @@ -1032,7 +1077,7 @@ extern struct erts_system_profile_flags_t erts_system_profile_flags; void erts_pre_init_process(void); void erts_late_init_process(void); -void erts_early_init_scheduling(void); +void erts_early_init_scheduling(int); void erts_init_scheduling(int, int, int); ErtsProcList *erts_proclist_create(Process *); @@ -1041,6 +1086,9 @@ int erts_proclist_same(ErtsProcList *, Process *); int erts_sched_set_wakeup_limit(char *str); +#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) +int erts_dbg_check_halloc_lock(Process *p); +#endif #ifdef DEBUG void erts_dbg_multi_scheduling_return_trap(Process *, Eterm); #endif @@ -1058,13 +1106,20 @@ erts_block_multi_scheduling(Process *, ErtsProcLocks, int, int); int erts_is_multi_scheduling_blocked(void); Eterm erts_multi_scheduling_blockers(Process *); void erts_start_schedulers(void); +void erts_alloc_notify_delayed_dealloc(int); void erts_smp_notify_check_children_needed(void); -void -erts_smp_schedule_misc_aux_work(int ignore_self, - int max_sched, - void (*func)(void *), - void *arg); #endif +#if ERTS_USE_ASYNC_READY_Q +void erts_notify_check_async_ready_queue(void *); +#endif +void erts_schedule_misc_aux_work(int sched_id, + void (*func)(void *), + void *arg); +void erts_schedule_multi_misc_aux_work(int ignore_self, + int max_sched, + void (*func)(void *), + void *arg); +erts_aint32_t erts_set_aux_work_timeout(int, erts_aint32_t, int); void erts_sched_notify_check_cpu_bind(void); Uint erts_active_schedulers(void); void erts_init_process(int); @@ -1148,6 +1203,7 @@ Sint erts_test_next_pid(int, Uint); Eterm erts_debug_processes(Process *c_p); Eterm erts_debug_processes_bif_info(Process *c_p); Uint erts_debug_nbalance(void); +int erts_debug_wait_deallocations(Process *c_p); #ifdef ERTS_SMP # define ERTS_GET_SCHEDULER_DATA_FROM_PROC(PROC) ((PROC)->scheduler_data) @@ -1218,16 +1274,11 @@ erts_psd_get(Process *p, int ix) #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) ErtsProcLocks locks = erts_proc_lc_my_proc_locks(p); if (ERTS_LC_PSD_ANY_LOCK == erts_psd_required_locks[ix].get_locks) - ERTS_SMP_LC_ASSERT(locks - || erts_is_system_blocked(0) - || (ERTS_IS_CRASH_DUMPING - && erts_is_system_blocked(ERTS_BS_FLG_ALLOW_GC))); + ERTS_SMP_LC_ASSERT(locks || erts_thr_progress_is_blocking()); else { locks &= erts_psd_required_locks[ix].get_locks; ERTS_SMP_LC_ASSERT(erts_psd_required_locks[ix].get_locks == locks - || erts_is_system_blocked(0) - || (ERTS_IS_CRASH_DUMPING - && erts_is_system_blocked(ERTS_BS_FLG_ALLOW_GC))); + || erts_thr_progress_is_blocking()); } #endif ASSERT(0 <= ix && ix < ERTS_PSD_SIZE); @@ -1244,16 +1295,11 @@ erts_psd_set(Process *p, ErtsProcLocks plocks, int ix, void *data) #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) ErtsProcLocks locks = erts_proc_lc_my_proc_locks(p); if (ERTS_LC_PSD_ANY_LOCK == erts_psd_required_locks[ix].set_locks) - ERTS_SMP_LC_ASSERT(locks - || erts_is_system_blocked(0) - || (ERTS_IS_CRASH_DUMPING - && erts_is_system_blocked(ERTS_BS_FLG_ALLOW_GC))); + ERTS_SMP_LC_ASSERT(locks || erts_thr_progress_is_blocking()); else { locks &= erts_psd_required_locks[ix].set_locks; ERTS_SMP_LC_ASSERT(erts_psd_required_locks[ix].set_locks == locks - || erts_is_system_blocked(0) - || (ERTS_IS_CRASH_DUMPING - && erts_is_system_blocked(ERTS_BS_FLG_ALLOW_GC))); + || erts_thr_progress_is_blocking()); } #endif ASSERT(0 <= ix && ix < ERTS_PSD_SIZE); @@ -1599,11 +1645,9 @@ erts_sched_poke(ErtsSchedulerSleepInfo *ssi) { erts_aint32_t flags; ERTS_THR_MEMORY_BARRIER; - flags = erts_smp_atomic32_read(&ssi->flags); - ASSERT(!(flags & ERTS_SSI_FLG_SLEEPING) - || (flags & ERTS_SSI_FLG_WAITING)); + flags = erts_smp_atomic32_read_nob(&ssi->flags); if (flags & ERTS_SSI_FLG_SLEEPING) { - flags = erts_smp_atomic32_band(&ssi->flags, ~ERTS_SSI_FLGS_SLEEP); + flags = erts_smp_atomic32_read_band_nob(&ssi->flags, ~ERTS_SSI_FLGS_SLEEP); erts_sched_finish_poke(ssi, flags); } } diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c index 5410bcd495..3550f1396c 100644 --- a/erts/emulator/beam/erl_process_dump.c +++ b/erts/emulator/beam/erl_process_dump.c @@ -350,7 +350,7 @@ heap_dump(int to, void *to_arg, Eterm x) ProcBin* pb = (ProcBin *) binary_val(x); Binary* val = pb->val; - if (erts_smp_atomic_xchg(&val->refc, 0) != 0) { + if (erts_smp_atomic_xchg_nob(&val->refc, 0) != 0) { val->flags = (UWord) all_binaries; all_binaries = val; } diff --git a/erts/emulator/beam/erl_process_lock.c b/erts/emulator/beam/erl_process_lock.c index 72560aa124..b4d20480c5 100644 --- a/erts/emulator/beam/erl_process_lock.c +++ b/erts/emulator/beam/erl_process_lock.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2007-2010. All Rights Reserved. + * Copyright Ericsson AB 2007-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -316,7 +316,7 @@ try_aquire(erts_proc_lock_t *lck, erts_tse_t *wtr) break; } wflg = lock << ERTS_PROC_LOCK_WAITER_SHIFT; - old_lflgs = ERTS_PROC_LOCK_FLGS_BOR_(lck, wflg | lock); + old_lflgs = ERTS_PROC_LOCK_FLGS_BOR_ACQB_(lck, wflg | lock); if (old_lflgs & lock) { /* Didn't get the lock */ goto enqueue; @@ -413,7 +413,7 @@ transfer_locks(Process *p, do { erts_tse_t *tmp = wake; wake = wake->next; - erts_atomic32_set(&tmp->uaflgs, 0); + erts_atomic32_set_nob(&tmp->uaflgs, 0); erts_tse_set(tmp); } while (wake); @@ -509,14 +509,14 @@ wait_for_locks(Process *p, ASSERT((wtr->uflgs & ~ERTS_PROC_LOCKS_ALL) == 0); - erts_atomic32_set(&wtr->uaflgs, 1); + erts_atomic32_set_nob(&wtr->uaflgs, 1); erts_pix_unlock(pix_lock); while (1) { int res; erts_tse_reset(wtr); - if (erts_atomic32_read(&wtr->uaflgs) == 0) + if (erts_atomic32_read_nob(&wtr->uaflgs) == 0) break; /* @@ -669,7 +669,9 @@ proc_safelock(Process *a_proc, ErtsProcLocks b_need_locks) { Process *p1, *p2; +#ifdef ERTS_ENABLE_LOCK_CHECK Eterm pid1, pid2; +#endif erts_pix_lock_t *pix_lck1, *pix_lck2; ErtsProcLocks need_locks1, have_locks1, need_locks2, have_locks2; ErtsProcLocks unlock_mask; @@ -684,24 +686,32 @@ proc_safelock(Process *a_proc, if (a_proc) { if (a_proc->id < b_proc->id) { p1 = a_proc; +#ifdef ERTS_ENABLE_LOCK_CHECK pid1 = a_proc->id; +#endif pix_lck1 = a_pix_lck; need_locks1 = a_need_locks; have_locks1 = a_have_locks; p2 = b_proc; +#ifdef ERTS_ENABLE_LOCK_CHECK pid2 = b_proc->id; +#endif pix_lck2 = b_pix_lck; need_locks2 = b_need_locks; have_locks2 = b_have_locks; } else if (a_proc->id > b_proc->id) { p1 = b_proc; +#ifdef ERTS_ENABLE_LOCK_CHECK pid1 = b_proc->id; +#endif pix_lck1 = b_pix_lck; need_locks1 = b_need_locks; have_locks1 = b_have_locks; p2 = a_proc; +#ifdef ERTS_ENABLE_LOCK_CHECK pid2 = a_proc->id; +#endif pix_lck2 = a_pix_lck; need_locks2 = a_need_locks; have_locks2 = a_have_locks; @@ -710,12 +720,16 @@ proc_safelock(Process *a_proc, ERTS_LC_ASSERT(a_proc == b_proc); ERTS_LC_ASSERT(a_proc->id == b_proc->id); p1 = a_proc; +#ifdef ERTS_ENABLE_LOCK_CHECK pid1 = a_proc->id; +#endif pix_lck1 = a_pix_lck; need_locks1 = a_need_locks | b_need_locks; have_locks1 = a_have_locks | b_have_locks; p2 = NULL; +#ifdef ERTS_ENABLE_LOCK_CHECK pid2 = 0; +#endif pix_lck2 = NULL; need_locks2 = 0; have_locks2 = 0; @@ -723,12 +737,16 @@ proc_safelock(Process *a_proc, } else { p1 = b_proc; +#ifdef ERTS_ENABLE_LOCK_CHECK pid1 = b_proc->id; +#endif pix_lck1 = b_pix_lck; need_locks1 = b_need_locks; have_locks1 = b_have_locks; p2 = NULL; +#ifdef ERTS_ENABLE_LOCK_CHECK pid2 = 0; +#endif pix_lck2 = NULL; need_locks2 = 0; have_locks2 = 0; @@ -955,7 +973,8 @@ erts_proc_lock_init(Process *p) { /* We always start with all locks locked */ #if ERTS_PROC_LOCK_ATOMIC_IMPL - erts_smp_atomic32_init(&p->lock.flags, (erts_aint32_t) ERTS_PROC_LOCKS_ALL); + erts_smp_atomic32_init_nob(&p->lock.flags, + (erts_aint32_t) ERTS_PROC_LOCKS_ALL); #else p->lock.flags = ERTS_PROC_LOCKS_ALL; #endif @@ -974,7 +993,7 @@ erts_proc_lock_init(Process *p) { int i; for (i = 0; i <= ERTS_PROC_LOCK_MAX_BIT; i++) - erts_smp_atomic32_init(&p->lock.locked[i], (erts_aint32_t) 1); + erts_smp_atomic32_init_nob(&p->lock.locked[i], (erts_aint32_t) 1); } #endif } diff --git a/erts/emulator/beam/erl_process_lock.h b/erts/emulator/beam/erl_process_lock.h index 355179f084..97f250138e 100644 --- a/erts/emulator/beam/erl_process_lock.h +++ b/erts/emulator/beam/erl_process_lock.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2007-2010. All Rights Reserved. + * Copyright Ericsson AB 2007-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -41,10 +41,10 @@ #define ERTS_PROC_LOCK_SPINLOCK_IMPL 0 #define ERTS_PROC_LOCK_MUTEX_IMPL 0 -#if defined(ETHR_HAVE_OPTIMIZED_ATOMIC_OPS) +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) # undef ERTS_PROC_LOCK_ATOMIC_IMPL # define ERTS_PROC_LOCK_ATOMIC_IMPL 1 -#elif defined(ETHR_HAVE_OPTIMIZED_SPINLOCK) +#elif defined(ETHR_HAVE_NATIVE_SPINLOCKS) # undef ERTS_PROC_LOCK_SPINLOCK_IMPL # define ERTS_PROC_LOCK_SPINLOCK_IMPL 1 #else @@ -270,9 +270,11 @@ typedef struct { #if ERTS_PROC_LOCK_ATOMIC_IMPL #define ERTS_PROC_LOCK_FLGS_BAND_(L, MSK) \ - ((ErtsProcLocks) erts_smp_atomic32_band(&(L)->flags, (erts_aint32_t) (MSK))) -#define ERTS_PROC_LOCK_FLGS_BOR_(L, MSK) \ - ((ErtsProcLocks) erts_smp_atomic32_bor(&(L)->flags, (erts_aint32_t) (MSK))) + ((ErtsProcLocks) erts_smp_atomic32_read_band_nob(&(L)->flags, \ + (erts_aint32_t) (MSK))) +#define ERTS_PROC_LOCK_FLGS_BOR_ACQB_(L, MSK) \ + ((ErtsProcLocks) erts_smp_atomic32_read_bor_acqb(&(L)->flags, \ + (erts_aint32_t) (MSK))) #define ERTS_PROC_LOCK_FLGS_CMPXCHG_ACQB_(L, NEW, EXPECTED) \ ((ErtsProcLocks) erts_smp_atomic32_cmpxchg_acqb(&(L)->flags, \ (erts_aint32_t) (NEW), \ @@ -282,7 +284,7 @@ typedef struct { (erts_aint32_t) (NEW), \ (erts_aint32_t) (EXPECTED))) #define ERTS_PROC_LOCK_FLGS_READ_(L) \ - ((ErtsProcLocks) erts_smp_atomic32_read(&(L)->flags)) + ((ErtsProcLocks) erts_smp_atomic32_read_nob(&(L)->flags)) #else /* no opt atomic ops */ @@ -325,7 +327,7 @@ erts_proc_lock_flags_cmpxchg(erts_proc_lock_t *lck, ErtsProcLocks new, #endif #define ERTS_PROC_LOCK_FLGS_BAND_(L, MSK) erts_proc_lock_flags_band((L), (MSK)) -#define ERTS_PROC_LOCK_FLGS_BOR_(L, MSK) erts_proc_lock_flags_bor((L), (MSK)) +#define ERTS_PROC_LOCK_FLGS_BOR_ACQB_(L, MSK) erts_proc_lock_flags_bor((L), (MSK)) #define ERTS_PROC_LOCK_FLGS_CMPXCHG_ACQB_(L, NEW, EXPECTED) \ erts_proc_lock_flags_cmpxchg((L), (NEW), (EXPECTED)) #define ERTS_PROC_LOCK_FLGS_CMPXCHG_RELB_(L, NEW, EXPECTED) \ @@ -623,11 +625,11 @@ erts_proc_lock_op_debug(Process *p, ErtsProcLocks locks, int locked) if (locks & lock) { erts_aint32_t lock_count; if (locked) { - lock_count = erts_smp_atomic32_inctest(&p->lock.locked[i]); + lock_count = erts_smp_atomic32_inc_read_nob(&p->lock.locked[i]); ERTS_LC_ASSERT(lock_count == 1); } else { - lock_count = erts_smp_atomic32_dectest(&p->lock.locked[i]); + lock_count = erts_smp_atomic32_dec_read_nob(&p->lock.locked[i]); ERTS_LC_ASSERT(lock_count == 0); } } @@ -649,7 +651,7 @@ ERTS_GLB_INLINE int erts_smp_proc_trylock(Process *, ErtsProcLocks); ERTS_GLB_INLINE void erts_smp_proc_inc_refc(Process *); ERTS_GLB_INLINE void erts_smp_proc_dec_refc(Process *); - +ERTS_GLB_INLINE void erts_smp_proc_add_refc(Process *, Sint32); #if ERTS_GLB_INLINE_INCL_FUNC_DEF @@ -735,6 +737,21 @@ ERTS_GLB_INLINE void erts_smp_proc_dec_refc(Process *p) #endif } +ERTS_GLB_INLINE void erts_smp_proc_add_refc(Process *p, Sint32 refc) +{ +#ifdef ERTS_SMP + Process *fp; + erts_pix_lock_t *pixlck = ERTS_PID2PIXLOCK(p->id); + erts_pix_lock(pixlck); + ERTS_LC_ASSERT(p->lock.refc > 0); + p->lock.refc += refc; + fp = p->lock.refc == 0 ? p : NULL; + erts_pix_unlock(pixlck); + if (fp) + erts_free_proc(fp); +#endif +} + #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ #ifdef ERTS_SMP diff --git a/erts/emulator/beam/erl_sched_spec_pre_alloc.c b/erts/emulator/beam/erl_sched_spec_pre_alloc.c new file mode 100644 index 0000000000..a7ccea7403 --- /dev/null +++ b/erts/emulator/beam/erl_sched_spec_pre_alloc.c @@ -0,0 +1,305 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2011. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Scheduler specific pre-allocators. Each scheduler + * thread allocates memory in its own private chunk of + * memory. Memory blocks deallocated by remote + * schedulers (or other threads) are passed back to + * the chunk owner via a lock-free data structure. + * + * Author: Rickard Green + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#ifdef ERTS_SMP + +#include "erl_process.h" +#include "erl_thr_progress.h" + +erts_sspa_data_t * +erts_sspa_create(size_t blk_sz, int pa_size) +{ + erts_sspa_data_t *data; + size_t tot_size; + size_t chunk_mem_size; + char *p; + char *chunk_start; + int cix; + int no_blocks = pa_size; + int no_blocks_per_chunk; + + if (erts_no_schedulers == 1) + no_blocks_per_chunk = no_blocks; + else { + int extra = (no_blocks - 1)/4 + 1; + if (extra == 0) + extra = 1; + no_blocks_per_chunk = no_blocks; + no_blocks_per_chunk += extra*erts_no_schedulers; + no_blocks_per_chunk /= erts_no_schedulers; + } + no_blocks = no_blocks_per_chunk * erts_no_schedulers; + chunk_mem_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_sspa_chunk_header_t)); + chunk_mem_size += blk_sz * no_blocks_per_chunk; + chunk_mem_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(chunk_mem_size); + tot_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_sspa_data_t)); + tot_size += chunk_mem_size*erts_no_schedulers; + + p = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_PRE_ALLOC_DATA, tot_size); + data = (erts_sspa_data_t *) p; + p += ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_sspa_data_t)); + chunk_start = p; + + data->chunks_mem_size = chunk_mem_size; + data->start = chunk_start; + data->end = chunk_start + chunk_mem_size*erts_no_schedulers; + + /* Initialize all chunks */ + for (cix = 0; cix < erts_no_schedulers; cix++) { + erts_sspa_chunk_t *chnk = erts_sspa_cix2chunk(data, cix); + erts_sspa_chunk_header_t *chdr = &chnk->aligned.header; + erts_sspa_blk_t *blk; + int i; + + erts_atomic_init_nob(&chdr->tail.data.last, (erts_aint_t) &chdr->tail.data.marker); + erts_atomic_init_nob(&chdr->tail.data.marker.next_atmc, ERTS_AINT_NULL); + erts_atomic_init_nob(&chdr->tail.data.um_refc[0], 0); + erts_atomic_init_nob(&chdr->tail.data.um_refc[1], 0); + erts_atomic32_init_nob(&chdr->tail.data.um_refc_ix, 0); + + chdr->head.no_thr_progress_check = 0; + chdr->head.used_marker = 1; + chdr->head.first = &chdr->tail.data.marker; + chdr->head.unref_end = &chdr->tail.data.marker; + chdr->head.next.thr_progress = erts_thr_progress_current(); + chdr->head.next.thr_progress_reached = 1; + chdr->head.next.um_refc_ix = 1; + chdr->head.next.unref_end = &chdr->tail.data.marker; + + p = &chnk->data[0]; + chdr->local.first = (erts_sspa_blk_t *) p; + blk = (erts_sspa_blk_t *) p; + for (i = 0; i < no_blocks_per_chunk; i++) { + blk = (erts_sspa_blk_t *) p; + p += blk_sz; + blk->next_ptr = (erts_sspa_blk_t *) p; + } + + blk->next_ptr = NULL; + chdr->local.last = blk; + chdr->local.cnt = no_blocks_per_chunk; + chdr->local.lim = no_blocks_per_chunk / 3; + + ERTS_SSPA_DBG_CHK_LCL(chdr); + } + + return data; +} + +static ERTS_INLINE erts_aint_t +enqueue_remote_managed_thread(erts_sspa_chunk_header_t *chdr, + erts_sspa_blk_t *this, + int want_last) +{ + erts_aint_t ilast, itmp; + + erts_atomic_init_nob(&this->next_atmc, ERTS_AINT_NULL); + + /* Enqueue at end of list... */ + + ilast = erts_atomic_read_nob(&chdr->tail.data.last); + while (1) { + erts_sspa_blk_t *last = (erts_sspa_blk_t *) ilast; + itmp = erts_atomic_cmpxchg_mb(&last->next_atmc, + (erts_aint_t) this, + ERTS_AINT_NULL); + if (itmp == ERTS_AINT_NULL) + break; + ilast = itmp; + } + + /* Move last pointer forward... */ + while (1) { + erts_aint_t itmp; + if (want_last) { + if (erts_atomic_read_rb(&this->next_atmc) != ERTS_AINT_NULL) { + /* Someone else will move it forward */ + return erts_atomic_read_nob(&chdr->tail.data.last); + } + } + else { + if (erts_atomic_read_nob(&this->next_atmc) != ERTS_AINT_NULL) { + /* Someone else will move it forward */ + return ERTS_AINT_NULL; + } + } + itmp = erts_atomic_cmpxchg_mb(&chdr->tail.data.last, + (erts_aint_t) this, + ilast); + if (ilast == itmp) + return want_last ? (erts_aint_t) this : ERTS_AINT_NULL; + ilast = itmp; + } +} + +void +erts_sspa_remote_free(erts_sspa_chunk_header_t *chdr, erts_sspa_blk_t *blk) +{ + int um_refc_ix = 0; + int managed_thread = erts_thr_progress_is_managed_thread(); + if (!managed_thread) { + um_refc_ix = erts_atomic32_read_acqb(&chdr->tail.data.um_refc_ix); + while (1) { + int tmp_um_refc_ix; + erts_atomic_inc_acqb(&chdr->tail.data.um_refc[um_refc_ix]); + tmp_um_refc_ix = erts_atomic32_read_acqb(&chdr->tail.data.um_refc_ix); + if (tmp_um_refc_ix == um_refc_ix) + break; + erts_atomic_dec_relb(&chdr->tail.data.um_refc[um_refc_ix]); + um_refc_ix = tmp_um_refc_ix; + } + } + + (void) enqueue_remote_managed_thread(chdr, blk, 0); + + if (!managed_thread) + erts_atomic_dec_relb(&chdr->tail.data.um_refc[um_refc_ix]); +} + +static ERTS_INLINE void +fetch_remote(erts_sspa_chunk_header_t *chdr, int max) +{ + int new_local = 0; + + if (chdr->head.no_thr_progress_check < ERTS_SSPA_FORCE_THR_CHECK_PROGRESS) + chdr->head.no_thr_progress_check++; + else { + erts_aint_t ilast; + + chdr->head.no_thr_progress_check = 0; + + ilast = erts_atomic_read_nob(&chdr->tail.data.last); + if (((erts_sspa_blk_t *) ilast) == &chdr->tail.data.marker + && chdr->head.first == &chdr->tail.data.marker) + return; + + if (chdr->head.next.thr_progress_reached + || erts_thr_progress_has_reached(chdr->head.next.thr_progress)) { + int um_refc_ix; + chdr->head.next.thr_progress_reached = 1; + um_refc_ix = chdr->head.next.um_refc_ix; + if (erts_atomic_read_acqb(&chdr->tail.data.um_refc[um_refc_ix]) == 0) { + + /* Move unreferenced end pointer forward... */ + + chdr->head.unref_end = chdr->head.next.unref_end; + + if (!chdr->head.used_marker + && chdr->head.unref_end == (erts_sspa_blk_t *) ilast) { + /* Need to equeue marker */ + chdr->head.used_marker = 1; + ilast = enqueue_remote_managed_thread(chdr, + &chdr->tail.data.marker, + 1); + } + + if (chdr->head.unref_end == (erts_sspa_blk_t *) ilast) + ERTS_THR_MEMORY_BARRIER; + else { + chdr->head.next.unref_end = (erts_sspa_blk_t *) ilast; + ERTS_THR_MEMORY_BARRIER; + chdr->head.next.thr_progress = erts_thr_progress_later(); + erts_atomic32_set_relb(&chdr->tail.data.um_refc_ix, + um_refc_ix); + chdr->head.next.um_refc_ix = um_refc_ix == 0 ? 1 : 0; + chdr->head.next.thr_progress_reached = 0; + } + } + } + } + + if (new_local < max && chdr->head.first != chdr->head.unref_end) { + erts_sspa_blk_t *first, *this, *next, *last; + first = chdr->head.first; + if (first == &chdr->tail.data.marker) { + chdr->head.used_marker = 0; + first = ((erts_sspa_blk_t *) + erts_atomic_read_nob(&first->next_atmc)); + chdr->head.first = first; + } + if (first != chdr->head.unref_end) { + + ERTS_SSPA_DBG_CHK_LCL(chdr); + + this = last = first; + do { + next = (erts_sspa_blk_t *) erts_atomic_read_nob(&this->next_atmc); + if (this == &chdr->tail.data.marker) + chdr->head.used_marker = 0; + else { + last->next_ptr = this; + last = this; + new_local++; + } + this = next; + } while (new_local < max && this != chdr->head.unref_end); + chdr->head.first = this; + if (!chdr->local.last) + chdr->local.first = first; + else + chdr->local.last->next_ptr = first; + chdr->local.last = last; + last->next_ptr = NULL; + chdr->local.cnt += new_local; + + ERTS_SSPA_DBG_CHK_LCL(chdr); + } + } + +} + +erts_sspa_blk_t * +erts_sspa_process_remote_frees(erts_sspa_chunk_header_t *chdr, + erts_sspa_blk_t *old_res) +{ + erts_sspa_blk_t *res = old_res; + + fetch_remote(chdr, ERTS_SSPA_MAX_GET_NEW_LOCAL); + + if (!res && chdr->local.first) { + + ERTS_SSPA_DBG_CHK_LCL(chdr); + + res = chdr->local.first; + chdr->local.first = res->next_ptr; + chdr->local.cnt--; + if (!chdr->local.first) + chdr->local.last = NULL; + + ERTS_SSPA_DBG_CHK_LCL(chdr); + } + + return res; +} + +#endif /* ERTS_SMP */ diff --git a/erts/emulator/beam/erl_sched_spec_pre_alloc.h b/erts/emulator/beam/erl_sched_spec_pre_alloc.h new file mode 100644 index 0000000000..d36066c399 --- /dev/null +++ b/erts/emulator/beam/erl_sched_spec_pre_alloc.h @@ -0,0 +1,239 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2011. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Scheduler specific pre-allocators. Each scheduler + * thread allocates memory in its own private chunk of + * memory. Memory blocks deallocated by remote + * schedulers (or other threads) are passed back to + * the chunk owner via a lock-free data structure. + * + * Author: Rickard Green + */ + +#ifndef ERTS_SCHED_SPEC_PRE_ALLOC_H__ +#define ERTS_SCHED_SPEC_PRE_ALLOC_H__ + +#ifdef ERTS_SMP + +#undef ERL_THR_PROGRESS_TSD_TYPE_ONLY +#define ERL_THR_PROGRESS_TSD_TYPE_ONLY +#include "erl_thr_progress.h" +#undef ERL_THR_PROGRESS_TSD_TYPE_ONLY + +#ifdef DEBUG +#define ERTS_SPPA_DBG_CHK_IN_CHNK(A, C, P) \ +do { \ + ASSERT((void *) (C) < (void *) (P)); \ + ASSERT((void *) (P) \ + < (void *) (((char *) (C)) + (A)->chunks_mem_size)); \ +} while (0) +#else +#define ERTS_SPPA_DBG_CHK_IN_CHNK(A, C, P) +#endif + +#ifdef DEBUG +extern Uint erts_no_schedulers; +#endif + +#define ERTS_SSPA_FORCE_THR_CHECK_PROGRESS 10 +#define ERTS_SSPA_MAX_GET_NEW_LOCAL 5 + +typedef struct { + char *start; + char *end; + int chunks_mem_size; +} erts_sspa_data_t; + +typedef union erts_sspa_blk_t_ erts_sspa_blk_t; +union erts_sspa_blk_t_ { + erts_atomic_t next_atmc; + erts_sspa_blk_t *next_ptr; +}; + +typedef struct { + erts_sspa_blk_t *first; + erts_sspa_blk_t *last; + int cnt; + int lim; +} erts_sspa_local_freelist_t; + +typedef struct { + erts_sspa_blk_t marker; + erts_atomic_t last; + erts_atomic_t um_refc[2]; + erts_atomic32_t um_refc_ix; +} erts_sspa_tail_t; + +typedef struct { + /* + * This structure needs to be cache line aligned for best + * performance. + */ + union { + /* Modified by threads returning memory to this chunk */ + erts_sspa_tail_t data; + char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_sspa_tail_t))]; + } tail; + /* + * Everything below this point is *only* accessed by the + * thread owning this chunk. + */ + struct { + int no_thr_progress_check; + int used_marker; + erts_sspa_blk_t *first; + erts_sspa_blk_t *unref_end; + struct { + ErtsThrPrgrVal thr_progress; + int thr_progress_reached; + int um_refc_ix; + erts_sspa_blk_t *unref_end; + } next; + } head; + erts_sspa_local_freelist_t local; +} erts_sspa_chunk_header_t; + +typedef struct { + union { + erts_sspa_chunk_header_t header; + char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE( + sizeof(erts_sspa_chunk_header_t))]; + } aligned; + char data[1]; +} erts_sspa_chunk_t; + +#ifdef DEBUG +ERTS_GLB_INLINE void +check_local_list(erts_sspa_chunk_header_t *chdr); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE void +check_local_list(erts_sspa_chunk_header_t *chdr) +{ + erts_sspa_blk_t *blk; + int n = 0; + for (blk = chdr->local.first; blk; blk = blk->next_ptr) + n++; + ASSERT(n == chdr->local.cnt); +} +#endif +#define ERTS_SSPA_DBG_CHK_LCL(CHDR) check_local_list((CHDR)) +#else +#define ERTS_SSPA_DBG_CHK_LCL(CHDR) +#endif + +erts_sspa_data_t *erts_sspa_create(size_t blk_sz, + int pa_size); +void erts_sspa_remote_free(erts_sspa_chunk_header_t *chdr, + erts_sspa_blk_t *blk); +erts_sspa_blk_t *erts_sspa_process_remote_frees(erts_sspa_chunk_header_t *chdr, + erts_sspa_blk_t *old_res); + +ERTS_GLB_INLINE erts_sspa_chunk_t *erts_sspa_cix2chunk(erts_sspa_data_t *data, + int cix); +ERTS_GLB_INLINE int erts_sspa_ptr2cix(erts_sspa_data_t *data, void *ptr); +ERTS_GLB_INLINE char *erts_sspa_alloc(erts_sspa_data_t *data, int cix); +ERTS_GLB_INLINE int erts_sspa_free(erts_sspa_data_t *data, int cix, char *blk); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE erts_sspa_chunk_t * +erts_sspa_cix2chunk(erts_sspa_data_t *data, int cix) +{ + ASSERT(0 <= cix && cix < erts_no_schedulers); + return (erts_sspa_chunk_t *) (data->start + cix*data->chunks_mem_size); +} + +ERTS_GLB_INLINE int +erts_sspa_ptr2cix(erts_sspa_data_t *data, void *ptr) +{ + int cix; + size_t diff; + if ((char *) ptr < data->start || data->end <= (char *) ptr) + return -1; + diff = ((char *) ptr) - data->start; + cix = (int) diff / data->chunks_mem_size; + ASSERT(0 <= cix && cix < erts_no_schedulers); + return cix; +} + +ERTS_GLB_INLINE char * +erts_sspa_alloc(erts_sspa_data_t *data, int cix) +{ + erts_sspa_chunk_t *chnk; + erts_sspa_chunk_header_t *chdr; + erts_sspa_blk_t *res; + + chnk = erts_sspa_cix2chunk(data, cix); + chdr = &chnk->aligned.header; + res = chdr->local.first; + ERTS_SSPA_DBG_CHK_LCL(chdr); + if (res) { + ERTS_SSPA_DBG_CHK_LCL(chdr); + chdr->local.first = res->next_ptr; + chdr->local.cnt--; + if (!chdr->local.first) + chdr->local.last = NULL; + ERTS_SSPA_DBG_CHK_LCL(chdr); + } + if (chdr->local.cnt <= chdr->local.lim) + return (char *) erts_sspa_process_remote_frees(chdr, res); + else if (chdr->head.no_thr_progress_check < ERTS_SSPA_FORCE_THR_CHECK_PROGRESS) + chdr->head.no_thr_progress_check++; + ASSERT(res); + return (char *) res; +} + +ERTS_GLB_INLINE int +erts_sspa_free(erts_sspa_data_t *data, int cix, char *cblk) +{ + erts_sspa_chunk_t *chnk; + erts_sspa_chunk_header_t *chdr; + erts_sspa_blk_t *blk = (erts_sspa_blk_t *) cblk; + int chnk_cix = erts_sspa_ptr2cix(data, blk); + + if (chnk_cix < 0) + return 0; + + chnk = erts_sspa_cix2chunk(data, chnk_cix); + chdr = &chnk->aligned.header; + if (chnk_cix != cix) { + /* Remote chunk */ + erts_sspa_remote_free(chdr, blk); + } + else { + /* Local chunk */ + ERTS_SSPA_DBG_CHK_LCL(chdr); + blk->next_ptr = chdr->local.first; + chdr->local.first = blk; + if (!chdr->local.last) + chdr->local.last = blk; + chdr->local.cnt++; + ERTS_SSPA_DBG_CHK_LCL(chdr); + } + + return 1; +} + +#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ + +#endif /* ERTS_SMP */ + +#endif /* ERTS_SCHED_SPEC_PRE_ALLOC_H__ */ diff --git a/erts/emulator/beam/erl_smp.h b/erts/emulator/beam/erl_smp.h index 287327bfe1..63179dfad4 100644 --- a/erts/emulator/beam/erl_smp.h +++ b/erts/emulator/beam/erl_smp.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * Copyright Ericsson AB 2005-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -54,8 +54,9 @@ typedef erts_cnd_t erts_smp_cnd_t; typedef erts_rwmtx_opt_t erts_smp_rwmtx_opt_t; typedef erts_rwmtx_t erts_smp_rwmtx_t; typedef erts_tsd_key_t erts_smp_tsd_key_t; -typedef erts_atomic_t erts_smp_atomic_t; -typedef erts_atomic32_t erts_smp_atomic32_t; +#define erts_smp_dw_atomic_t erts_dw_atomic_t +#define erts_smp_atomic_t erts_atomic_t +#define erts_smp_atomic32_t erts_atomic32_t typedef erts_spinlock_t erts_smp_spinlock_t; typedef erts_rwlock_t erts_smp_rwlock_t; void erts_thr_fatal_error(int, char *); /* implemented in erl_init.c */ @@ -83,8 +84,9 @@ typedef struct { } erts_smp_rwmtx_opt_t; typedef int erts_smp_rwmtx_t; typedef int erts_smp_tsd_key_t; -typedef SWord erts_smp_atomic_t; -typedef Uint32 erts_smp_atomic32_t; +#define erts_smp_dw_atomic_t erts_no_dw_atomic_t +#define erts_smp_atomic_t erts_no_atomic_t +#define erts_smp_atomic32_t erts_no_atomic32_t #if __GNUC__ > 2 typedef struct { } erts_smp_spinlock_t; typedef struct { } erts_smp_rwlock_t; @@ -160,82 +162,6 @@ ERTS_GLB_INLINE int erts_smp_rwmtx_tryrwlock(erts_smp_rwmtx_t *rwmtx); ERTS_GLB_INLINE void erts_smp_rwmtx_rwunlock(erts_smp_rwmtx_t *rwmtx); ERTS_GLB_INLINE int erts_smp_lc_rwmtx_is_rlocked(erts_smp_rwmtx_t *mtx); ERTS_GLB_INLINE int erts_smp_lc_rwmtx_is_rwlocked(erts_smp_rwmtx_t *mtx); -ERTS_GLB_INLINE void erts_smp_atomic_init(erts_smp_atomic_t *var, - erts_aint_t i); -ERTS_GLB_INLINE void erts_smp_atomic_set(erts_smp_atomic_t *var, erts_aint_t i); -ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_read(erts_smp_atomic_t *var); -ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_inctest(erts_smp_atomic_t *incp); -ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_dectest(erts_smp_atomic_t *decp); -ERTS_GLB_INLINE void erts_smp_atomic_inc(erts_smp_atomic_t *incp); -ERTS_GLB_INLINE void erts_smp_atomic_dec(erts_smp_atomic_t *decp); -ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_addtest(erts_smp_atomic_t *addp, - erts_aint_t i); -ERTS_GLB_INLINE void erts_smp_atomic_add(erts_smp_atomic_t *addp, - erts_aint_t i); -ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_xchg(erts_smp_atomic_t *xchgp, - erts_aint_t new); -ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_cmpxchg(erts_smp_atomic_t *xchgp, - erts_aint_t new, - erts_aint_t expected); -ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_bor(erts_smp_atomic_t *var, - erts_aint_t mask); -ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_band(erts_smp_atomic_t *var, - erts_aint_t mask); -ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_read_acqb(erts_smp_atomic_t *var); -ERTS_GLB_INLINE void erts_smp_atomic_set_relb(erts_smp_atomic_t *var, - erts_aint_t i); -ERTS_GLB_INLINE void erts_smp_atomic_dec_relb(erts_smp_atomic_t *decp); -ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_dectest_relb(erts_smp_atomic_t *decp); -ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_cmpxchg_acqb(erts_smp_atomic_t *xchgp, - erts_aint_t new, - erts_aint_t exp); -ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_cmpxchg_relb(erts_smp_atomic_t *xchgp, - erts_aint_t new, - erts_aint_t exp); -ERTS_GLB_INLINE void -erts_smp_atomic32_init(erts_smp_atomic32_t *var, erts_aint32_t i); -ERTS_GLB_INLINE void -erts_smp_atomic32_set(erts_smp_atomic32_t *var, erts_aint32_t i); -ERTS_GLB_INLINE erts_aint32_t -erts_smp_atomic32_read(erts_smp_atomic32_t *var); -ERTS_GLB_INLINE erts_aint32_t -erts_smp_atomic32_inctest(erts_smp_atomic32_t *incp); -ERTS_GLB_INLINE erts_aint32_t -erts_smp_atomic32_dectest(erts_smp_atomic32_t *decp); -ERTS_GLB_INLINE void -erts_smp_atomic32_inc(erts_smp_atomic32_t *incp); -ERTS_GLB_INLINE void -erts_smp_atomic32_dec(erts_smp_atomic32_t *decp); -ERTS_GLB_INLINE erts_aint32_t -erts_smp_atomic32_addtest(erts_smp_atomic32_t *addp, erts_aint32_t i); -ERTS_GLB_INLINE void -erts_smp_atomic32_add(erts_smp_atomic32_t *addp, erts_aint32_t i); -ERTS_GLB_INLINE erts_aint32_t -erts_smp_atomic32_xchg(erts_smp_atomic32_t *xchgp, erts_aint32_t new); -ERTS_GLB_INLINE erts_aint32_t -erts_smp_atomic32_cmpxchg(erts_smp_atomic32_t *xchgp, - erts_aint32_t new, - erts_aint32_t expected); -ERTS_GLB_INLINE erts_aint32_t -erts_smp_atomic32_bor(erts_smp_atomic32_t *var, erts_aint32_t mask); -ERTS_GLB_INLINE erts_aint32_t -erts_smp_atomic32_band(erts_smp_atomic32_t *var, erts_aint32_t mask); -ERTS_GLB_INLINE erts_aint32_t -erts_smp_atomic32_read_acqb(erts_smp_atomic32_t *var); -ERTS_GLB_INLINE void -erts_smp_atomic32_set_relb(erts_smp_atomic32_t *var, erts_aint32_t i); -ERTS_GLB_INLINE void -erts_smp_atomic32_dec_relb(erts_smp_atomic32_t *decp); -ERTS_GLB_INLINE erts_aint32_t -erts_smp_atomic32_dectest_relb(erts_smp_atomic32_t *decp); -ERTS_GLB_INLINE erts_aint32_t -erts_smp_atomic32_cmpxchg_acqb(erts_smp_atomic32_t *xchgp, - erts_aint32_t new, - erts_aint32_t exp); -ERTS_GLB_INLINE erts_aint32_t -erts_smp_atomic32_cmpxchg_relb(erts_smp_atomic32_t *xchgp, - erts_aint32_t new, - erts_aint32_t exp); ERTS_GLB_INLINE void erts_smp_spinlock_init_x(erts_smp_spinlock_t *lock, char *name, Eterm extra); @@ -279,6 +205,429 @@ ERTS_GLB_INLINE void erts_smp_thr_sigmask(int how, ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #endif /* #ifdef ERTS_THR_HAVE_SIG_FUNCS */ +/* + * Functions implementing atomic operations with with no (nob), + * full (mb), acquire (acqb), release (relb), read (rb), and + * write (wb) memory barriers. + * + * If SMP support has been disabled, they are mapped to functions + * that performs the same operation, but aren't atomic and don't + * imply memory barriers. + */ + +#ifdef ERTS_SMP + +/* Double word size atomics */ + +#define erts_smp_dw_atomic_init_nob erts_dw_atomic_init_nob +#define erts_smp_dw_atomic_set_nob erts_dw_atomic_set_nob +#define erts_smp_dw_atomic_read_nob erts_dw_atomic_read_nob +#define erts_smp_dw_atomic_cmpxchg_nob erts_dw_atomic_cmpxchg_nob + +#define erts_smp_dw_atomic_init_mb erts_dw_atomic_init_mb +#define erts_smp_dw_atomic_set_mb erts_dw_atomic_set_mb +#define erts_smp_dw_atomic_read_mb erts_dw_atomic_read_mb +#define erts_smp_dw_atomic_cmpxchg_mb erts_dw_atomic_cmpxchg_mb + +#define erts_smp_dw_atomic_init_acqb erts_dw_atomic_init_acqb +#define erts_smp_dw_atomic_set_acqb erts_dw_atomic_set_acqb +#define erts_smp_dw_atomic_read_acqb erts_dw_atomic_read_acqb +#define erts_smp_dw_atomic_cmpxchg_acqb erts_dw_atomic_cmpxchg_acqb + +#define erts_smp_dw_atomic_init_relb erts_dw_atomic_init_relb +#define erts_smp_dw_atomic_set_relb erts_dw_atomic_set_relb +#define erts_smp_dw_atomic_read_relb erts_dw_atomic_read_relb +#define erts_smp_dw_atomic_cmpxchg_relb erts_dw_atomic_cmpxchg_relb + +#define erts_smp_dw_atomic_init_rb erts_dw_atomic_init_rb +#define erts_smp_dw_atomic_set_rb erts_dw_atomic_set_rb +#define erts_smp_dw_atomic_read_rb erts_dw_atomic_read_rb +#define erts_smp_dw_atomic_cmpxchg_rb erts_dw_atomic_cmpxchg_rb + +#define erts_smp_dw_atomic_init_wb erts_dw_atomic_init_wb +#define erts_smp_dw_atomic_set_wb erts_dw_atomic_set_wb +#define erts_smp_dw_atomic_read_wb erts_dw_atomic_read_wb +#define erts_smp_dw_atomic_cmpxchg_wb erts_dw_atomic_cmpxchg_wb + +/* Word size atomics */ + +#define erts_smp_atomic_init_nob erts_atomic_init_nob +#define erts_smp_atomic_set_nob erts_atomic_set_nob +#define erts_smp_atomic_read_nob erts_atomic_read_nob +#define erts_smp_atomic_inc_read_nob erts_atomic_inc_read_nob +#define erts_smp_atomic_dec_read_nob erts_atomic_dec_read_nob +#define erts_smp_atomic_inc_nob erts_atomic_inc_nob +#define erts_smp_atomic_dec_nob erts_atomic_dec_nob +#define erts_smp_atomic_add_read_nob erts_atomic_add_read_nob +#define erts_smp_atomic_add_nob erts_atomic_add_nob +#define erts_smp_atomic_read_bor_nob erts_atomic_read_bor_nob +#define erts_smp_atomic_read_band_nob erts_atomic_read_band_nob +#define erts_smp_atomic_xchg_nob erts_atomic_xchg_nob +#define erts_smp_atomic_cmpxchg_nob erts_atomic_cmpxchg_nob + +#define erts_smp_atomic_init_mb erts_atomic_init_mb +#define erts_smp_atomic_set_mb erts_atomic_set_mb +#define erts_smp_atomic_read_mb erts_atomic_read_mb +#define erts_smp_atomic_inc_read_mb erts_atomic_inc_read_mb +#define erts_smp_atomic_dec_read_mb erts_atomic_dec_read_mb +#define erts_smp_atomic_inc_mb erts_atomic_inc_mb +#define erts_smp_atomic_dec_mb erts_atomic_dec_mb +#define erts_smp_atomic_add_read_mb erts_atomic_add_read_mb +#define erts_smp_atomic_add_mb erts_atomic_add_mb +#define erts_smp_atomic_read_bor_mb erts_atomic_read_bor_mb +#define erts_smp_atomic_read_band_mb erts_atomic_read_band_mb +#define erts_smp_atomic_xchg_mb erts_atomic_xchg_mb +#define erts_smp_atomic_cmpxchg_mb erts_atomic_cmpxchg_mb + +#define erts_smp_atomic_init_acqb erts_atomic_init_acqb +#define erts_smp_atomic_set_acqb erts_atomic_set_acqb +#define erts_smp_atomic_read_acqb erts_atomic_read_acqb +#define erts_smp_atomic_inc_read_acqb erts_atomic_inc_read_acqb +#define erts_smp_atomic_dec_read_acqb erts_atomic_dec_read_acqb +#define erts_smp_atomic_inc_acqb erts_atomic_inc_acqb +#define erts_smp_atomic_dec_acqb erts_atomic_dec_acqb +#define erts_smp_atomic_add_read_acqb erts_atomic_add_read_acqb +#define erts_smp_atomic_add_acqb erts_atomic_add_acqb +#define erts_smp_atomic_read_bor_acqb erts_atomic_read_bor_acqb +#define erts_smp_atomic_read_band_acqb erts_atomic_read_band_acqb +#define erts_smp_atomic_xchg_acqb erts_atomic_xchg_acqb +#define erts_smp_atomic_cmpxchg_acqb erts_atomic_cmpxchg_acqb + +#define erts_smp_atomic_init_relb erts_atomic_init_relb +#define erts_smp_atomic_set_relb erts_atomic_set_relb +#define erts_smp_atomic_read_relb erts_atomic_read_relb +#define erts_smp_atomic_inc_read_relb erts_atomic_inc_read_relb +#define erts_smp_atomic_dec_read_relb erts_atomic_dec_read_relb +#define erts_smp_atomic_inc_relb erts_atomic_inc_relb +#define erts_smp_atomic_dec_relb erts_atomic_dec_relb +#define erts_smp_atomic_add_read_relb erts_atomic_add_read_relb +#define erts_smp_atomic_add_relb erts_atomic_add_relb +#define erts_smp_atomic_read_bor_relb erts_atomic_read_bor_relb +#define erts_smp_atomic_read_band_relb erts_atomic_read_band_relb +#define erts_smp_atomic_xchg_relb erts_atomic_xchg_relb +#define erts_smp_atomic_cmpxchg_relb erts_atomic_cmpxchg_relb + +#define erts_smp_atomic_init_rb erts_atomic_init_rb +#define erts_smp_atomic_set_rb erts_atomic_set_rb +#define erts_smp_atomic_read_rb erts_atomic_read_rb +#define erts_smp_atomic_inc_read_rb erts_atomic_inc_read_rb +#define erts_smp_atomic_dec_read_rb erts_atomic_dec_read_rb +#define erts_smp_atomic_inc_rb erts_atomic_inc_rb +#define erts_smp_atomic_dec_rb erts_atomic_dec_rb +#define erts_smp_atomic_add_read_rb erts_atomic_add_read_rb +#define erts_smp_atomic_add_rb erts_atomic_add_rb +#define erts_smp_atomic_read_bor_rb erts_atomic_read_bor_rb +#define erts_smp_atomic_read_band_rb erts_atomic_read_band_rb +#define erts_smp_atomic_xchg_rb erts_atomic_xchg_rb +#define erts_smp_atomic_cmpxchg_rb erts_atomic_cmpxchg_rb + +#define erts_smp_atomic_init_wb erts_atomic_init_wb +#define erts_smp_atomic_set_wb erts_atomic_set_wb +#define erts_smp_atomic_read_wb erts_atomic_read_wb +#define erts_smp_atomic_inc_read_wb erts_atomic_inc_read_wb +#define erts_smp_atomic_dec_read_wb erts_atomic_dec_read_wb +#define erts_smp_atomic_inc_wb erts_atomic_inc_wb +#define erts_smp_atomic_dec_wb erts_atomic_dec_wb +#define erts_smp_atomic_add_read_wb erts_atomic_add_read_wb +#define erts_smp_atomic_add_wb erts_atomic_add_wb +#define erts_smp_atomic_read_bor_wb erts_atomic_read_bor_wb +#define erts_smp_atomic_read_band_wb erts_atomic_read_band_wb +#define erts_smp_atomic_xchg_wb erts_atomic_xchg_wb +#define erts_smp_atomic_cmpxchg_wb erts_atomic_cmpxchg_wb + +/* 32-bit atomics */ + +#define erts_smp_atomic32_init_nob erts_atomic32_init_nob +#define erts_smp_atomic32_set_nob erts_atomic32_set_nob +#define erts_smp_atomic32_read_nob erts_atomic32_read_nob +#define erts_smp_atomic32_inc_read_nob erts_atomic32_inc_read_nob +#define erts_smp_atomic32_dec_read_nob erts_atomic32_dec_read_nob +#define erts_smp_atomic32_inc_nob erts_atomic32_inc_nob +#define erts_smp_atomic32_dec_nob erts_atomic32_dec_nob +#define erts_smp_atomic32_add_read_nob erts_atomic32_add_read_nob +#define erts_smp_atomic32_add_nob erts_atomic32_add_nob +#define erts_smp_atomic32_read_bor_nob erts_atomic32_read_bor_nob +#define erts_smp_atomic32_read_band_nob erts_atomic32_read_band_nob +#define erts_smp_atomic32_xchg_nob erts_atomic32_xchg_nob +#define erts_smp_atomic32_cmpxchg_nob erts_atomic32_cmpxchg_nob + +#define erts_smp_atomic32_init_mb erts_atomic32_init_mb +#define erts_smp_atomic32_set_mb erts_atomic32_set_mb +#define erts_smp_atomic32_read_mb erts_atomic32_read_mb +#define erts_smp_atomic32_inc_read_mb erts_atomic32_inc_read_mb +#define erts_smp_atomic32_dec_read_mb erts_atomic32_dec_read_mb +#define erts_smp_atomic32_inc_mb erts_atomic32_inc_mb +#define erts_smp_atomic32_dec_mb erts_atomic32_dec_mb +#define erts_smp_atomic32_add_read_mb erts_atomic32_add_read_mb +#define erts_smp_atomic32_add_mb erts_atomic32_add_mb +#define erts_smp_atomic32_read_bor_mb erts_atomic32_read_bor_mb +#define erts_smp_atomic32_read_band_mb erts_atomic32_read_band_mb +#define erts_smp_atomic32_xchg_mb erts_atomic32_xchg_mb +#define erts_smp_atomic32_cmpxchg_mb erts_atomic32_cmpxchg_mb + +#define erts_smp_atomic32_init_acqb erts_atomic32_init_acqb +#define erts_smp_atomic32_set_acqb erts_atomic32_set_acqb +#define erts_smp_atomic32_read_acqb erts_atomic32_read_acqb +#define erts_smp_atomic32_inc_read_acqb erts_atomic32_inc_read_acqb +#define erts_smp_atomic32_dec_read_acqb erts_atomic32_dec_read_acqb +#define erts_smp_atomic32_inc_acqb erts_atomic32_inc_acqb +#define erts_smp_atomic32_dec_acqb erts_atomic32_dec_acqb +#define erts_smp_atomic32_add_read_acqb erts_atomic32_add_read_acqb +#define erts_smp_atomic32_add_acqb erts_atomic32_add_acqb +#define erts_smp_atomic32_read_bor_acqb erts_atomic32_read_bor_acqb +#define erts_smp_atomic32_read_band_acqb erts_atomic32_read_band_acqb +#define erts_smp_atomic32_xchg_acqb erts_atomic32_xchg_acqb +#define erts_smp_atomic32_cmpxchg_acqb erts_atomic32_cmpxchg_acqb + +#define erts_smp_atomic32_init_relb erts_atomic32_init_relb +#define erts_smp_atomic32_set_relb erts_atomic32_set_relb +#define erts_smp_atomic32_read_relb erts_atomic32_read_relb +#define erts_smp_atomic32_inc_read_relb erts_atomic32_inc_read_relb +#define erts_smp_atomic32_dec_read_relb erts_atomic32_dec_read_relb +#define erts_smp_atomic32_inc_relb erts_atomic32_inc_relb +#define erts_smp_atomic32_dec_relb erts_atomic32_dec_relb +#define erts_smp_atomic32_add_read_relb erts_atomic32_add_read_relb +#define erts_smp_atomic32_add_relb erts_atomic32_add_relb +#define erts_smp_atomic32_read_bor_relb erts_atomic32_read_bor_relb +#define erts_smp_atomic32_read_band_relb erts_atomic32_read_band_relb +#define erts_smp_atomic32_xchg_relb erts_atomic32_xchg_relb +#define erts_smp_atomic32_cmpxchg_relb erts_atomic32_cmpxchg_relb + +#define erts_smp_atomic32_init_rb erts_atomic32_init_rb +#define erts_smp_atomic32_set_rb erts_atomic32_set_rb +#define erts_smp_atomic32_read_rb erts_atomic32_read_rb +#define erts_smp_atomic32_inc_read_rb erts_atomic32_inc_read_rb +#define erts_smp_atomic32_dec_read_rb erts_atomic32_dec_read_rb +#define erts_smp_atomic32_inc_rb erts_atomic32_inc_rb +#define erts_smp_atomic32_dec_rb erts_atomic32_dec_rb +#define erts_smp_atomic32_add_read_rb erts_atomic32_add_read_rb +#define erts_smp_atomic32_add_rb erts_atomic32_add_rb +#define erts_smp_atomic32_read_bor_rb erts_atomic32_read_bor_rb +#define erts_smp_atomic32_read_band_rb erts_atomic32_read_band_rb +#define erts_smp_atomic32_xchg_rb erts_atomic32_xchg_rb +#define erts_smp_atomic32_cmpxchg_rb erts_atomic32_cmpxchg_rb + +#define erts_smp_atomic32_init_wb erts_atomic32_init_wb +#define erts_smp_atomic32_set_wb erts_atomic32_set_wb +#define erts_smp_atomic32_read_wb erts_atomic32_read_wb +#define erts_smp_atomic32_inc_read_wb erts_atomic32_inc_read_wb +#define erts_smp_atomic32_dec_read_wb erts_atomic32_dec_read_wb +#define erts_smp_atomic32_inc_wb erts_atomic32_inc_wb +#define erts_smp_atomic32_dec_wb erts_atomic32_dec_wb +#define erts_smp_atomic32_add_read_wb erts_atomic32_add_read_wb +#define erts_smp_atomic32_add_wb erts_atomic32_add_wb +#define erts_smp_atomic32_read_bor_wb erts_atomic32_read_bor_wb +#define erts_smp_atomic32_read_band_wb erts_atomic32_read_band_wb +#define erts_smp_atomic32_xchg_wb erts_atomic32_xchg_wb +#define erts_smp_atomic32_cmpxchg_wb erts_atomic32_cmpxchg_wb + +#else /* !ERTS_SMP */ + +/* Double word size atomics */ + +#define erts_smp_dw_atomic_init_nob erts_no_dw_atomic_set +#define erts_smp_dw_atomic_set_nob erts_no_dw_atomic_set +#define erts_smp_dw_atomic_read_nob erts_no_dw_atomic_read +#define erts_smp_dw_atomic_cmpxchg_nob erts_no_dw_atomic_cmpxchg + +#define erts_smp_dw_atomic_init_mb erts_no_dw_atomic_init +#define erts_smp_dw_atomic_set_mb erts_no_dw_atomic_set +#define erts_smp_dw_atomic_read_mb erts_no_dw_atomic_read +#define erts_smp_dw_atomic_cmpxchg_mb erts_no_dw_atomic_cmpxchg + +#define erts_smp_dw_atomic_init_acqb erts_no_dw_atomic_init +#define erts_smp_dw_atomic_set_acqb erts_no_dw_atomic_set +#define erts_smp_dw_atomic_read_acqb erts_no_dw_atomic_read +#define erts_smp_dw_atomic_cmpxchg_acqb erts_no_dw_atomic_cmpxchg + +#define erts_smp_dw_atomic_init_relb erts_no_dw_atomic_init +#define erts_smp_dw_atomic_set_relb erts_no_dw_atomic_set +#define erts_smp_dw_atomic_read_relb erts_no_dw_atomic_read +#define erts_smp_dw_atomic_cmpxchg_relb erts_no_dw_atomic_cmpxchg + +#define erts_smp_dw_atomic_init_rb erts_no_dw_atomic_init +#define erts_smp_dw_atomic_set_rb erts_no_dw_atomic_set +#define erts_smp_dw_atomic_read_rb erts_no_dw_atomic_read +#define erts_smp_dw_atomic_cmpxchg_rb erts_no_dw_atomic_cmpxchg + +#define erts_smp_dw_atomic_init_wb erts_no_dw_atomic_init +#define erts_smp_dw_atomic_set_wb erts_no_dw_atomic_set +#define erts_smp_dw_atomic_read_wb erts_no_dw_atomic_read +#define erts_smp_dw_atomic_cmpxchg_wb erts_no_dw_atomic_cmpxchg + +/* Word size atomics */ + +#define erts_smp_atomic_init_nob erts_no_atomic_set +#define erts_smp_atomic_set_nob erts_no_atomic_set +#define erts_smp_atomic_read_nob erts_no_atomic_read +#define erts_smp_atomic_inc_read_nob erts_no_atomic_inc_read +#define erts_smp_atomic_dec_read_nob erts_no_atomic_dec_read +#define erts_smp_atomic_inc_nob erts_no_atomic_inc +#define erts_smp_atomic_dec_nob erts_no_atomic_dec +#define erts_smp_atomic_add_read_nob erts_no_atomic_add_read +#define erts_smp_atomic_add_nob erts_no_atomic_add +#define erts_smp_atomic_read_bor_nob erts_no_atomic_read_bor +#define erts_smp_atomic_read_band_nob erts_no_atomic_read_band +#define erts_smp_atomic_xchg_nob erts_no_atomic_xchg +#define erts_smp_atomic_cmpxchg_nob erts_no_atomic_cmpxchg + +#define erts_smp_atomic_init_mb erts_no_atomic_set +#define erts_smp_atomic_set_mb erts_no_atomic_set +#define erts_smp_atomic_read_mb erts_no_atomic_read +#define erts_smp_atomic_inc_read_mb erts_no_atomic_inc_read +#define erts_smp_atomic_dec_read_mb erts_no_atomic_dec_read +#define erts_smp_atomic_inc_mb erts_no_atomic_inc +#define erts_smp_atomic_dec_mb erts_no_atomic_dec +#define erts_smp_atomic_add_read_mb erts_no_atomic_add_read +#define erts_smp_atomic_add_mb erts_no_atomic_add +#define erts_smp_atomic_read_bor_mb erts_no_atomic_read_bor +#define erts_smp_atomic_read_band_mb erts_no_atomic_read_band +#define erts_smp_atomic_xchg_mb erts_no_atomic_xchg +#define erts_smp_atomic_cmpxchg_mb erts_no_atomic_cmpxchg + +#define erts_smp_atomic_init_acqb erts_no_atomic_set +#define erts_smp_atomic_set_acqb erts_no_atomic_set +#define erts_smp_atomic_read_acqb erts_no_atomic_read +#define erts_smp_atomic_inc_read_acqb erts_no_atomic_inc_read +#define erts_smp_atomic_dec_read_acqb erts_no_atomic_dec_read +#define erts_smp_atomic_inc_acqb erts_no_atomic_inc +#define erts_smp_atomic_dec_acqb erts_no_atomic_dec +#define erts_smp_atomic_add_read_acqb erts_no_atomic_add_read +#define erts_smp_atomic_add_acqb erts_no_atomic_add +#define erts_smp_atomic_read_bor_acqb erts_no_atomic_read_bor +#define erts_smp_atomic_read_band_acqb erts_no_atomic_read_band +#define erts_smp_atomic_xchg_acqb erts_no_atomic_xchg +#define erts_smp_atomic_cmpxchg_acqb erts_no_atomic_cmpxchg + +#define erts_smp_atomic_init_relb erts_no_atomic_set +#define erts_smp_atomic_set_relb erts_no_atomic_set +#define erts_smp_atomic_read_relb erts_no_atomic_read +#define erts_smp_atomic_inc_read_relb erts_no_atomic_inc_read +#define erts_smp_atomic_dec_read_relb erts_no_atomic_dec_read +#define erts_smp_atomic_inc_relb erts_no_atomic_inc +#define erts_smp_atomic_dec_relb erts_no_atomic_dec +#define erts_smp_atomic_add_read_relb erts_no_atomic_add_read +#define erts_smp_atomic_add_relb erts_no_atomic_add +#define erts_smp_atomic_read_bor_relb erts_no_atomic_read_bor +#define erts_smp_atomic_read_band_relb erts_no_atomic_read_band +#define erts_smp_atomic_xchg_relb erts_no_atomic_xchg +#define erts_smp_atomic_cmpxchg_relb erts_no_atomic_cmpxchg + +#define erts_smp_atomic_init_rb erts_no_atomic_set +#define erts_smp_atomic_set_rb erts_no_atomic_set +#define erts_smp_atomic_read_rb erts_no_atomic_read +#define erts_smp_atomic_inc_read_rb erts_no_atomic_inc_read +#define erts_smp_atomic_dec_read_rb erts_no_atomic_dec_read +#define erts_smp_atomic_inc_rb erts_no_atomic_inc +#define erts_smp_atomic_dec_rb erts_no_atomic_dec +#define erts_smp_atomic_add_read_rb erts_no_atomic_add_read +#define erts_smp_atomic_add_rb erts_no_atomic_add +#define erts_smp_atomic_read_bor_rb erts_no_atomic_read_bor +#define erts_smp_atomic_read_band_rb erts_no_atomic_read_band +#define erts_smp_atomic_xchg_rb erts_no_atomic_xchg +#define erts_smp_atomic_cmpxchg_rb erts_no_atomic_cmpxchg + +#define erts_smp_atomic_init_wb erts_no_atomic_set +#define erts_smp_atomic_set_wb erts_no_atomic_set +#define erts_smp_atomic_read_wb erts_no_atomic_read +#define erts_smp_atomic_inc_read_wb erts_no_atomic_inc_read +#define erts_smp_atomic_dec_read_wb erts_no_atomic_dec_read +#define erts_smp_atomic_inc_wb erts_no_atomic_inc +#define erts_smp_atomic_dec_wb erts_no_atomic_dec +#define erts_smp_atomic_add_read_wb erts_no_atomic_add_read +#define erts_smp_atomic_add_wb erts_no_atomic_add +#define erts_smp_atomic_read_bor_wb erts_no_atomic_read_bor +#define erts_smp_atomic_read_band_wb erts_no_atomic_read_band +#define erts_smp_atomic_xchg_wb erts_no_atomic_xchg +#define erts_smp_atomic_cmpxchg_wb erts_no_atomic_cmpxchg + +/* 32-bit atomics */ + +#define erts_smp_atomic32_init_nob erts_no_atomic32_set +#define erts_smp_atomic32_set_nob erts_no_atomic32_set +#define erts_smp_atomic32_read_nob erts_no_atomic32_read +#define erts_smp_atomic32_inc_read_nob erts_no_atomic32_inc_read +#define erts_smp_atomic32_dec_read_nob erts_no_atomic32_dec_read +#define erts_smp_atomic32_inc_nob erts_no_atomic32_inc +#define erts_smp_atomic32_dec_nob erts_no_atomic32_dec +#define erts_smp_atomic32_add_read_nob erts_no_atomic32_add_read +#define erts_smp_atomic32_add_nob erts_no_atomic32_add +#define erts_smp_atomic32_read_bor_nob erts_no_atomic32_read_bor +#define erts_smp_atomic32_read_band_nob erts_no_atomic32_read_band +#define erts_smp_atomic32_xchg_nob erts_no_atomic32_xchg +#define erts_smp_atomic32_cmpxchg_nob erts_no_atomic32_cmpxchg + +#define erts_smp_atomic32_init_mb erts_no_atomic32_set +#define erts_smp_atomic32_set_mb erts_no_atomic32_set +#define erts_smp_atomic32_read_mb erts_no_atomic32_read +#define erts_smp_atomic32_inc_read_mb erts_no_atomic32_inc_read +#define erts_smp_atomic32_dec_read_mb erts_no_atomic32_dec_read +#define erts_smp_atomic32_inc_mb erts_no_atomic32_inc +#define erts_smp_atomic32_dec_mb erts_no_atomic32_dec +#define erts_smp_atomic32_add_read_mb erts_no_atomic32_add_read +#define erts_smp_atomic32_add_mb erts_no_atomic32_add +#define erts_smp_atomic32_read_bor_mb erts_no_atomic32_read_bor +#define erts_smp_atomic32_read_band_mb erts_no_atomic32_read_band +#define erts_smp_atomic32_xchg_mb erts_no_atomic32_xchg +#define erts_smp_atomic32_cmpxchg_mb erts_no_atomic32_cmpxchg + +#define erts_smp_atomic32_init_acqb erts_no_atomic32_set +#define erts_smp_atomic32_set_acqb erts_no_atomic32_set +#define erts_smp_atomic32_read_acqb erts_no_atomic32_read +#define erts_smp_atomic32_inc_read_acqb erts_no_atomic32_inc_read +#define erts_smp_atomic32_dec_read_acqb erts_no_atomic32_dec_read +#define erts_smp_atomic32_inc_acqb erts_no_atomic32_inc +#define erts_smp_atomic32_dec_acqb erts_no_atomic32_dec +#define erts_smp_atomic32_add_read_acqb erts_no_atomic32_add_read +#define erts_smp_atomic32_add_acqb erts_no_atomic32_add +#define erts_smp_atomic32_read_bor_acqb erts_no_atomic32_read_bor +#define erts_smp_atomic32_read_band_acqb erts_no_atomic32_read_band +#define erts_smp_atomic32_xchg_acqb erts_no_atomic32_xchg +#define erts_smp_atomic32_cmpxchg_acqb erts_no_atomic32_cmpxchg + +#define erts_smp_atomic32_init_relb erts_no_atomic32_set +#define erts_smp_atomic32_set_relb erts_no_atomic32_set +#define erts_smp_atomic32_read_relb erts_no_atomic32_read +#define erts_smp_atomic32_inc_read_relb erts_no_atomic32_inc_read +#define erts_smp_atomic32_dec_read_relb erts_no_atomic32_dec_read +#define erts_smp_atomic32_inc_relb erts_no_atomic32_inc +#define erts_smp_atomic32_dec_relb erts_no_atomic32_dec +#define erts_smp_atomic32_add_read_relb erts_no_atomic32_add_read +#define erts_smp_atomic32_add_relb erts_no_atomic32_add +#define erts_smp_atomic32_read_bor_relb erts_no_atomic32_read_bor +#define erts_smp_atomic32_read_band_relb erts_no_atomic32_read_band +#define erts_smp_atomic32_xchg_relb erts_no_atomic32_xchg +#define erts_smp_atomic32_cmpxchg_relb erts_no_atomic32_cmpxchg + +#define erts_smp_atomic32_init_rb erts_no_atomic32_set +#define erts_smp_atomic32_set_rb erts_no_atomic32_set +#define erts_smp_atomic32_read_rb erts_no_atomic32_read +#define erts_smp_atomic32_inc_read_rb erts_no_atomic32_inc_read +#define erts_smp_atomic32_dec_read_rb erts_no_atomic32_dec_read +#define erts_smp_atomic32_inc_rb erts_no_atomic32_inc +#define erts_smp_atomic32_dec_rb erts_no_atomic32_dec +#define erts_smp_atomic32_add_read_rb erts_no_atomic32_add_read +#define erts_smp_atomic32_add_rb erts_no_atomic32_add +#define erts_smp_atomic32_read_bor_rb erts_no_atomic32_read_bor +#define erts_smp_atomic32_read_band_rb erts_no_atomic32_read_band +#define erts_smp_atomic32_xchg_rb erts_no_atomic32_xchg +#define erts_smp_atomic32_cmpxchg_rb erts_no_atomic32_cmpxchg + +#define erts_smp_atomic32_init_wb erts_no_atomic32_set +#define erts_smp_atomic32_set_wb erts_no_atomic32_set +#define erts_smp_atomic32_read_wb erts_no_atomic32_read +#define erts_smp_atomic32_inc_read_wb erts_no_atomic32_inc_read +#define erts_smp_atomic32_dec_read_wb erts_no_atomic32_dec_read +#define erts_smp_atomic32_inc_wb erts_no_atomic32_inc +#define erts_smp_atomic32_dec_wb erts_no_atomic32_dec +#define erts_smp_atomic32_add_read_wb erts_no_atomic32_add_read +#define erts_smp_atomic32_add_wb erts_no_atomic32_add +#define erts_smp_atomic32_read_bor_wb erts_no_atomic32_read_bor +#define erts_smp_atomic32_read_band_wb erts_no_atomic32_read_band +#define erts_smp_atomic32_xchg_wb erts_no_atomic32_xchg +#define erts_smp_atomic32_cmpxchg_wb erts_no_atomic32_cmpxchg + +#endif /* !ERTS_SMP */ #if ERTS_GLB_INLINE_INCL_FUNC_DEF @@ -473,6 +822,16 @@ erts_smp_cnd_wait(erts_smp_cnd_t *cnd, erts_smp_mtx_t *mtx) #endif } +/* + * IMPORTANT note about erts_smp_cnd_signal() and erts_smp_cnd_broadcast() + * + * POSIX allow a call to `pthread_cond_signal' or `pthread_cond_broadcast' + * even though the associated mutex/mutexes isn't/aren't locked by the + * caller. Our implementation do not allow that in order to avoid a + * performance penalty. That is, all associated mutexes *need* to be + * locked by the caller of erts_smp_cnd_signal()/erts_smp_cnd_broadcast()! + */ + ERTS_GLB_INLINE void erts_smp_cnd_signal(erts_smp_cnd_t *cnd) { @@ -655,434 +1014,6 @@ erts_smp_lc_rwmtx_is_rwlocked(erts_smp_rwmtx_t *mtx) } ERTS_GLB_INLINE void -erts_smp_atomic_init(erts_smp_atomic_t *var, erts_aint_t i) -{ -#ifdef ERTS_SMP - erts_atomic_init(var, i); -#else - *var = i; -#endif -} - -ERTS_GLB_INLINE void -erts_smp_atomic_set(erts_smp_atomic_t *var, erts_aint_t i) -{ -#ifdef ERTS_SMP - erts_atomic_set(var, i); -#else - *var = i; -#endif -} - -ERTS_GLB_INLINE erts_aint_t -erts_smp_atomic_read(erts_smp_atomic_t *var) -{ -#ifdef ERTS_SMP - return erts_atomic_read(var); -#else - return *var; -#endif -} - -ERTS_GLB_INLINE erts_aint_t -erts_smp_atomic_inctest(erts_smp_atomic_t *incp) -{ -#ifdef ERTS_SMP - return erts_atomic_inctest(incp); -#else - return ++(*incp); -#endif -} - -ERTS_GLB_INLINE erts_aint_t -erts_smp_atomic_dectest(erts_smp_atomic_t *decp) -{ -#ifdef ERTS_SMP - return erts_atomic_dectest(decp); -#else - return --(*decp); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_atomic_inc(erts_smp_atomic_t *incp) -{ -#ifdef ERTS_SMP - erts_atomic_inc(incp); -#else - ++(*incp); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_atomic_dec(erts_smp_atomic_t *decp) -{ -#ifdef ERTS_SMP - erts_atomic_dec(decp); -#else - --(*decp); -#endif -} - -ERTS_GLB_INLINE erts_aint_t -erts_smp_atomic_addtest(erts_smp_atomic_t *addp, erts_aint_t i) -{ -#ifdef ERTS_SMP - return erts_atomic_addtest(addp, i); -#else - return *addp += i; -#endif -} - -ERTS_GLB_INLINE void -erts_smp_atomic_add(erts_smp_atomic_t *addp, erts_aint_t i) -{ -#ifdef ERTS_SMP - erts_atomic_add(addp, i); -#else - *addp += i; -#endif -} - -ERTS_GLB_INLINE erts_aint_t -erts_smp_atomic_xchg(erts_smp_atomic_t *xchgp, erts_aint_t new) -{ -#ifdef ERTS_SMP - return erts_atomic_xchg(xchgp, new); -#else - erts_aint_t old; - old = *xchgp; - *xchgp = new; - return old; -#endif -} - -ERTS_GLB_INLINE erts_aint_t -erts_smp_atomic_cmpxchg(erts_smp_atomic_t *xchgp, - erts_aint_t new, - erts_aint_t expected) -{ -#ifdef ERTS_SMP - return erts_atomic_cmpxchg(xchgp, new, expected); -#else - erts_aint_t old = *xchgp; - if (old == expected) - *xchgp = new; - return old; -#endif -} - -ERTS_GLB_INLINE erts_aint_t -erts_smp_atomic_bor(erts_smp_atomic_t *var, erts_aint_t mask) -{ -#ifdef ERTS_SMP - return erts_atomic_bor(var, mask); -#else - erts_aint_t old; - old = *var; - *var |= mask; - return old; -#endif -} - -ERTS_GLB_INLINE erts_aint_t -erts_smp_atomic_band(erts_smp_atomic_t *var, erts_aint_t mask) -{ -#ifdef ERTS_SMP - return erts_atomic_band(var, mask); -#else - erts_aint_t old; - old = *var; - *var &= mask; - return old; -#endif -} - -ERTS_GLB_INLINE erts_aint_t -erts_smp_atomic_read_acqb(erts_smp_atomic_t *var) -{ -#ifdef ERTS_SMP - return erts_atomic_read_acqb(var); -#else - return *var; -#endif -} - -ERTS_GLB_INLINE void -erts_smp_atomic_set_relb(erts_smp_atomic_t *var, erts_aint_t i) -{ -#ifdef ERTS_SMP - erts_atomic_set_relb(var, i); -#else - *var = i; -#endif -} - -ERTS_GLB_INLINE void -erts_smp_atomic_dec_relb(erts_smp_atomic_t *decp) -{ -#ifdef ERTS_SMP - erts_atomic_dec_relb(decp); -#else - --(*decp); -#endif -} - -ERTS_GLB_INLINE erts_aint_t -erts_smp_atomic_dectest_relb(erts_smp_atomic_t *decp) -{ -#ifdef ERTS_SMP - return erts_atomic_dectest_relb(decp); -#else - return --(*decp); -#endif -} - -ERTS_GLB_INLINE erts_aint_t -erts_smp_atomic_cmpxchg_acqb(erts_smp_atomic_t *xchgp, - erts_aint_t new, - erts_aint_t exp) -{ -#ifdef ERTS_SMP - return erts_atomic_cmpxchg_acqb(xchgp, new, exp); -#else - erts_aint_t old = *xchgp; - if (old == exp) - *xchgp = new; - return old; -#endif -} - -ERTS_GLB_INLINE erts_aint_t -erts_smp_atomic_cmpxchg_relb(erts_smp_atomic_t *xchgp, - erts_aint_t new, - erts_aint_t exp) -{ -#ifdef ERTS_SMP - return erts_atomic_cmpxchg_relb(xchgp, new, exp); -#else - erts_aint_t old = *xchgp; - if (old == exp) - *xchgp = new; - return old; -#endif -} - -ERTS_GLB_INLINE void -erts_smp_atomic32_init(erts_smp_atomic32_t *var, erts_aint32_t i) -{ -#ifdef ERTS_SMP - erts_atomic32_init(var, i); -#else - *var = i; -#endif -} - -ERTS_GLB_INLINE void -erts_smp_atomic32_set(erts_smp_atomic32_t *var, erts_aint32_t i) -{ -#ifdef ERTS_SMP - erts_atomic32_set(var, i); -#else - *var = i; -#endif -} - -ERTS_GLB_INLINE erts_aint32_t -erts_smp_atomic32_read(erts_smp_atomic32_t *var) -{ -#ifdef ERTS_SMP - return erts_atomic32_read(var); -#else - return *var; -#endif -} - -ERTS_GLB_INLINE erts_aint32_t -erts_smp_atomic32_inctest(erts_smp_atomic32_t *incp) -{ -#ifdef ERTS_SMP - return erts_atomic32_inctest(incp); -#else - return ++(*incp); -#endif -} - -ERTS_GLB_INLINE erts_aint32_t -erts_smp_atomic32_dectest(erts_smp_atomic32_t *decp) -{ -#ifdef ERTS_SMP - return erts_atomic32_dectest(decp); -#else - return --(*decp); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_atomic32_inc(erts_smp_atomic32_t *incp) -{ -#ifdef ERTS_SMP - erts_atomic32_inc(incp); -#else - ++(*incp); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_atomic32_dec(erts_smp_atomic32_t *decp) -{ -#ifdef ERTS_SMP - erts_atomic32_dec(decp); -#else - --(*decp); -#endif -} - -ERTS_GLB_INLINE erts_aint32_t -erts_smp_atomic32_addtest(erts_smp_atomic32_t *addp, erts_aint32_t i) -{ -#ifdef ERTS_SMP - return erts_atomic32_addtest(addp, i); -#else - return *addp += i; -#endif -} - -ERTS_GLB_INLINE void -erts_smp_atomic32_add(erts_smp_atomic32_t *addp, erts_aint32_t i) -{ -#ifdef ERTS_SMP - erts_atomic32_add(addp, i); -#else - *addp += i; -#endif -} - -ERTS_GLB_INLINE erts_aint32_t -erts_smp_atomic32_xchg(erts_smp_atomic32_t *xchgp, erts_aint32_t new) -{ -#ifdef ERTS_SMP - return erts_atomic32_xchg(xchgp, new); -#else - erts_aint32_t old; - old = *xchgp; - *xchgp = new; - return old; -#endif -} - -ERTS_GLB_INLINE erts_aint32_t -erts_smp_atomic32_cmpxchg(erts_smp_atomic32_t *xchgp, - erts_aint32_t new, - erts_aint32_t expected) -{ -#ifdef ERTS_SMP - return erts_atomic32_cmpxchg(xchgp, new, expected); -#else - erts_aint32_t old = *xchgp; - if (old == expected) - *xchgp = new; - return old; -#endif -} - -ERTS_GLB_INLINE erts_aint32_t -erts_smp_atomic32_bor(erts_smp_atomic32_t *var, erts_aint32_t mask) -{ -#ifdef ERTS_SMP - return erts_atomic32_bor(var, mask); -#else - erts_aint32_t old; - old = *var; - *var |= mask; - return old; -#endif -} - -ERTS_GLB_INLINE erts_aint32_t -erts_smp_atomic32_band(erts_smp_atomic32_t *var, erts_aint32_t mask) -{ -#ifdef ERTS_SMP - return erts_atomic32_band(var, mask); -#else - erts_aint32_t old; - old = *var; - *var &= mask; - return old; -#endif -} - -ERTS_GLB_INLINE erts_aint32_t -erts_smp_atomic32_read_acqb(erts_smp_atomic32_t *var) -{ -#ifdef ERTS_SMP - return erts_atomic32_read_acqb(var); -#else - return *var; -#endif -} - -ERTS_GLB_INLINE void -erts_smp_atomic32_set_relb(erts_smp_atomic32_t *var, erts_aint32_t i) -{ -#ifdef ERTS_SMP - erts_atomic32_set_relb(var, i); -#else - *var = i; -#endif -} - -ERTS_GLB_INLINE void -erts_smp_atomic32_dec_relb(erts_smp_atomic32_t *decp) -{ -#ifdef ERTS_SMP - erts_atomic32_dec_relb(decp); -#else - --(*decp); -#endif -} - -ERTS_GLB_INLINE erts_aint32_t -erts_smp_atomic32_dectest_relb(erts_smp_atomic32_t *decp) -{ -#ifdef ERTS_SMP - return erts_atomic32_dectest_relb(decp); -#else - return --(*decp); -#endif -} - -ERTS_GLB_INLINE erts_aint32_t -erts_smp_atomic32_cmpxchg_acqb(erts_smp_atomic32_t *xchgp, - erts_aint32_t new, - erts_aint32_t exp) -{ -#ifdef ERTS_SMP - return erts_atomic32_cmpxchg_acqb(xchgp, new, exp); -#else - erts_aint32_t old = *xchgp; - if (old == exp) - *xchgp = new; - return old; -#endif -} - -ERTS_GLB_INLINE erts_aint32_t -erts_smp_atomic32_cmpxchg_relb(erts_smp_atomic32_t *xchgp, - erts_aint32_t new, - erts_aint32_t exp) -{ -#ifdef ERTS_SMP - return erts_atomic32_cmpxchg_relb(xchgp, new, exp); -#else - erts_aint32_t old = *xchgp; - if (old == exp) - *xchgp = new; - return old; -#endif -} - -ERTS_GLB_INLINE void erts_smp_spinlock_init_x(erts_smp_spinlock_t *lock, char *name, Eterm extra) { #ifdef ERTS_SMP @@ -1308,3 +1239,37 @@ erts_smp_thr_sigwait(const sigset_t *set, int *sig) #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ #endif /* ERL_SMP_H */ + +#ifdef ERTS_UNDEF_DEPRECATED_ATOMICS + +/* Deprecated functions to replace */ + +#undef erts_smp_atomic_init +#undef erts_smp_atomic_set +#undef erts_smp_atomic_read +#undef erts_smp_atomic_inctest +#undef erts_smp_atomic_dectest +#undef erts_smp_atomic_inc +#undef erts_smp_atomic_dec +#undef erts_smp_atomic_addtest +#undef erts_smp_atomic_add +#undef erts_smp_atomic_xchg +#undef erts_smp_atomic_cmpxchg +#undef erts_smp_atomic_bor +#undef erts_smp_atomic_band + +#undef erts_smp_atomic32_init +#undef erts_smp_atomic32_set +#undef erts_smp_atomic32_read +#undef erts_smp_atomic32_inctest +#undef erts_smp_atomic32_dectest +#undef erts_smp_atomic32_inc +#undef erts_smp_atomic32_dec +#undef erts_smp_atomic32_addtest +#undef erts_smp_atomic32_add +#undef erts_smp_atomic32_xchg +#undef erts_smp_atomic32_cmpxchg +#undef erts_smp_atomic32_bor +#undef erts_smp_atomic32_band + +#endif diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h index 1d75fa313c..bc20b2d798 100644 --- a/erts/emulator/beam/erl_term.h +++ b/erts/emulator/beam/erl_term.h @@ -331,7 +331,13 @@ _ET_DECLARE_CHECKED(Uint,thing_subtag,Eterm) * we now use a non-zero bit-pattern in debug mode. */ #if ET_DEBUG -#define THE_NON_VALUE _make_header(0,_TAG_HEADER_FLOAT) +# ifdef HIPE + /* A very large (or negative) value as work-around for ugly hipe-bifs + that return untagged integers (eg hipe_bs_put_utf8) */ +# define THE_NON_VALUE _make_header((Uint)~0,_TAG_HEADER_FLOAT) +# else +# define THE_NON_VALUE _make_header(0,_TAG_HEADER_FLOAT) +# endif #else #define THE_NON_VALUE (0) #endif diff --git a/erts/emulator/beam/erl_thr_progress.c b/erts/emulator/beam/erl_thr_progress.c new file mode 100644 index 0000000000..9324bcde51 --- /dev/null +++ b/erts/emulator/beam/erl_thr_progress.c @@ -0,0 +1,1373 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2011. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Thread progress information. Used by lock free algorithms + * to determine when all involved threads are guaranteed to + * have passed a specific point of execution. + * + * Usage instructions below. + * + * Author: Rickard Green + */ + +/* + * ------ Usage instructions ----------------------------------------------- + * + * This module keeps track of the progress of a set of managed threads. Only + * threads that behave well can be allowed to be managed. A managed thread + * should update its thread progress frequently. Currently only scheduler + * threads and the aux_thread are managed threads. We typically do not want + * any async threads as managed threads since they cannot guarantee a + * frequent update of thread progress, since they execute user implemented + * driver code. + * + * erts_thr_progress_current() returns the global current thread progress + * value of managed threads. I.e., the latest progress value that all + * managed threads have reached. Thread progress values are opaque. + * + * erts_thr_progress_has_reached(VAL) returns a value != 0 if current + * global thread progress has reached or passed VAL. + * + * erts_thr_progress_later() returns a thread progress value in the future + * which no managed thread have yet reached. + * + * All threads issue a full memory barrier when reaching a new thread + * progress value. They only reach new thread progress values in specific + * controlled states when calling erts_thr_progress_update(). Schedulers + * call erts_thr_progress_update() in between execution of processes, + * when going to sleep and when waking up. + * + * Sleeping managed threads are considered to have reached next thread + * progress value immediately. They are not woken and do therefore not + * issue any memory barriers when reaching a new thread progress value. + * A sleeping thread do however immediately issue a memory barrier upon + * wakeup. + * + * Both managed and registered unmanaged threads may request wakeup when + * the global thread progress reach a certain value using + * erts_thr_progress_wakeup(). + * + * Note that thread progress values are opaque, and that you are only + * allowed to use thread progress values retrieved from this API! + * + * ------------------------------------------------------------------------- + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include <stddef.h> /* offsetof() */ +#include "erl_thr_progress.h" +#include "global.h" + +#ifdef ERTS_SMP + +/* + * We use a 64-bit value for thread progress. By this wrapping of + * the thread progress will more or less never occur. + * + * On 32-bit systems we therefore need a double word atomic. + */ + +#define ERTS_THR_PRGR_PRINT_LEADER 0 +#define ERTS_THR_PRGR_PRINT_VAL 0 +#define ERTS_THR_PRGR_PRINT_BLOCKERS 0 + +#define ERTS_THR_PRGR_FTL_ERR_BLCK_POLL_INTERVAL 100 + +#define ERTS_THR_PRGR_LFLG_BLOCK (((erts_aint32_t) 1) << 31) +#define ERTS_THR_PRGR_LFLG_NO_LEADER (((erts_aint32_t) 1) << 30) +#define ERTS_THR_PRGR_LFLG_ACTIVE_MASK (~(ERTS_THR_PRGR_LFLG_NO_LEADER \ + | ERTS_THR_PRGR_LFLG_BLOCK)) + +#define ERTS_THR_PRGR_LFLGS_ACTIVE(LFLGS) \ + ((LFLGS) & ERTS_THR_PRGR_LFLG_ACTIVE_MASK) + +#define ERTS_THR_PRGR_LFLGS_ALL_WAITING(LFLGS) \ + (((LFLGS) & (ERTS_THR_PRGR_LFLG_NO_LEADER \ + |ERTS_THR_PRGR_LFLG_ACTIVE_MASK)) \ + == ERTS_THR_PRGR_LFLG_NO_LEADER) + +#define read_acqb erts_thr_prgr_read_acqb__ + +#ifdef ARCH_64 + +static ERTS_INLINE void +set_mb(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val) +{ + erts_atomic_set_mb(atmc, val); +} + +static ERTS_INLINE void +set_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val) +{ + erts_atomic_set_nob(atmc, val); +} + +static ERTS_INLINE ErtsThrPrgrVal +read_nob(ERTS_THR_PRGR_ATOMIC *atmc) +{ + return (ErtsThrPrgrVal) erts_atomic_read_nob(atmc); +} + +static ERTS_INLINE void +init_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val) +{ + erts_atomic_init_nob(atmc, val); +} + +#else + +#undef dw_sint_to_val +#define dw_sint_to_val erts_thr_prgr_dw_sint_to_val__ + +static void +val_to_dw_sint(ethr_dw_sint_t *dw_sint, ErtsThrPrgrVal val) +{ +#ifdef ETHR_SU_DW_NAINT_T__ + dw_sint->dw_sint = (ETHR_SU_DW_NAINT_T__) val; +#else + dw_sint->sint[ETHR_DW_SINT_LOW_WORD] + = (ethr_sint_t) (val & 0xffffffff); + dw_sint->sint[ETHR_DW_SINT_HIGH_WORD] + = (ethr_sint_t) ((val >> 32) & 0xffffffff); +#endif +} + +static ERTS_INLINE void +set_mb(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val) +{ + ethr_dw_sint_t dw_sint; + val_to_dw_sint(&dw_sint, val); + erts_dw_atomic_set_mb(atmc, &dw_sint); +} + +static ERTS_INLINE void +set_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val) +{ + ethr_dw_sint_t dw_sint; + val_to_dw_sint(&dw_sint, val); + erts_dw_atomic_set_nob(atmc, &dw_sint); +} + +static ERTS_INLINE ErtsThrPrgrVal +read_nob(ERTS_THR_PRGR_ATOMIC *atmc) +{ + ethr_dw_sint_t dw_sint; + erts_dw_atomic_read_nob(atmc, &dw_sint); + return erts_thr_prgr_dw_sint_to_val__(&dw_sint); +} + +static ERTS_INLINE void +init_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val) +{ + ethr_dw_sint_t dw_sint; + val_to_dw_sint(&dw_sint, val); + erts_dw_atomic_init_nob(atmc, &dw_sint); +} + +#endif + +/* #define ERTS_THR_PROGRESS_STATE_DEBUG */ + +#ifdef ERTS_THR_PROGRESS_STATE_DEBUG + +#ifdef __GNUC__ +#warning "Thread progress state debug is on" +#endif + +#define ERTS_THR_PROGRESS_STATE_DEBUG_LEADER (((erts_aint32_t) 1) << 0) +#define ERTS_THR_PROGRESS_STATE_DEBUG_ACTIVE (((erts_aint32_t) 1) << 1) + +#define ERTS_THR_PROGRESS_STATE_DEBUG_INIT(ID) \ + erts_atomic32_init_nob(&intrnl->thr[(ID)].data.state_debug, \ + ERTS_THR_PROGRESS_STATE_DEBUG_ACTIVE) + +#define ERTS_THR_PROGRESS_STATE_DEBUG_SET_ACTIVE(ID, ON) \ +do { \ + erts_aint32_t state_debug__; \ + state_debug__ = erts_atomic32_read_nob(&intrnl->thr[(ID)].data.state_debug); \ + if ((ON)) \ + state_debug__ |= ERTS_THR_PROGRESS_STATE_DEBUG_ACTIVE; \ + else \ + state_debug__ &= ~ERTS_THR_PROGRESS_STATE_DEBUG_ACTIVE; \ + erts_atomic32_set_nob(&intrnl->thr[(ID)].data.state_debug, state_debug__); \ +} while (0) + +#define ERTS_THR_PROGRESS_STATE_DEBUG_SET_LEADER(ID, ON) \ +do { \ + erts_aint32_t state_debug__; \ + state_debug__ = erts_atomic32_read_nob(&intrnl->thr[(ID)].data.state_debug); \ + if ((ON)) \ + state_debug__ |= ERTS_THR_PROGRESS_STATE_DEBUG_LEADER; \ + else \ + state_debug__ &= ~ERTS_THR_PROGRESS_STATE_DEBUG_LEADER; \ + erts_atomic32_set_nob(&intrnl->thr[(ID)].data.state_debug, state_debug__); \ +} while (0) + +#else + +#define ERTS_THR_PROGRESS_STATE_DEBUG_INIT(ID) +#define ERTS_THR_PROGRESS_STATE_DEBUG_SET_ACTIVE(ID, ON) +#define ERTS_THR_PROGRESS_STATE_DEBUG_SET_LEADER(ID, ON) + +#endif /* ERTS_THR_PROGRESS_STATE_DEBUG */ + +#define ERTS_THR_PRGR_BLCKR_INVALID (~((erts_aint32_t) 0)) +#define ERTS_THR_PRGR_BLCKR_UNMANAGED (((erts_aint32_t) 1) << 31) + +#define ERTS_THR_PRGR_BC_FLG_NOT_BLOCKING (((erts_aint32_t) 1) << 31) + +#define ERTS_THR_PRGR_BM_BITS 32 +#define ERTS_THR_PRGR_BM_SHIFT 5 +#define ERTS_THR_PRGR_BM_MASK 0x1f + +#define ERTS_THR_PRGR_WAKEUP_DATA_MASK (ERTS_THR_PRGR_WAKEUP_DATA_SIZE - 1) + +#define ERTS_THR_PRGR_WAKEUP_IX(V) \ + ((int) ((V) & ERTS_THR_PRGR_WAKEUP_DATA_MASK)) + +typedef struct { + erts_atomic32_t len; + int id[1]; +} ErtsThrPrgrManagedWakeupData; + +typedef struct { + erts_atomic32_t len; + int high_sz; + int low_sz; + erts_atomic32_t *high; + erts_atomic32_t *low; +} ErtsThrPrgrUnmanagedWakeupData; + +typedef struct { + erts_atomic32_t lflgs; + erts_atomic32_t block_count; + erts_atomic_t blocker_event; + erts_atomic32_t pref_wakeup_used; + erts_atomic32_t managed_count; + erts_atomic32_t managed_id; + erts_atomic32_t unmanaged_id; +} ErtsThrPrgrMiscData; + +typedef struct { + ERTS_THR_PRGR_ATOMIC current; +#ifdef ERTS_THR_PROGRESS_STATE_DEBUG + erts_atomic32_t state_debug; +#endif +} ErtsThrPrgrElement; + +typedef union { + ErtsThrPrgrElement data; + char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsThrPrgrElement))]; +} ErtsThrPrgrArray; + +typedef struct { + union { + ErtsThrPrgrMiscData data; + char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE( + sizeof(ErtsThrPrgrMiscData))]; + } misc; + ErtsThrPrgrArray *thr; + struct { + int no; + ErtsThrPrgrCallbacks *callbacks; + ErtsThrPrgrManagedWakeupData *data[ERTS_THR_PRGR_WAKEUP_DATA_SIZE]; + } managed; + struct { + int no; + ErtsThrPrgrCallbacks *callbacks; + ErtsThrPrgrUnmanagedWakeupData *data[ERTS_THR_PRGR_WAKEUP_DATA_SIZE]; + } unmanaged; +} ErtsThrPrgrInternalData; + +static ErtsThrPrgrInternalData *intrnl; + +ErtsThrPrgr erts_thr_prgr__; + +erts_tsd_key_t erts_thr_prgr_data_key__; + +static void handle_wakeup_requests(ErtsThrPrgrVal current); +static int got_sched_wakeups(void); +static erts_aint32_t block_thread(ErtsThrPrgrData *tpd); + +static ERTS_INLINE void +wakeup_managed(int id) +{ + ErtsThrPrgrCallbacks *cbp = &intrnl->managed.callbacks[id]; + ASSERT(0 <= id && id < intrnl->managed.no); + cbp->wakeup(cbp->arg); +} + + +static ERTS_INLINE void +wakeup_unmanaged(int id) +{ + ErtsThrPrgrCallbacks *cbp = &intrnl->unmanaged.callbacks[id]; + ASSERT(0 <= id && id < intrnl->unmanaged.no); + cbp->wakeup(cbp->arg); +} + +static ERTS_INLINE ErtsThrPrgrData * +perhaps_thr_prgr_data(ErtsSchedulerData *esdp) +{ + if (esdp) + return &esdp->thr_progress_data; + else + return erts_tsd_get(erts_thr_prgr_data_key__); +} + +static ERTS_INLINE ErtsThrPrgrData * +thr_prgr_data(ErtsSchedulerData *esdp) +{ + ErtsThrPrgrData *tpd = perhaps_thr_prgr_data(esdp); + ASSERT(tpd); + return tpd; +} + +static void +init_tmp_thr_prgr_data(ErtsThrPrgrData *tpd) +{ + tpd->id = -1; + tpd->is_managed = 0; + tpd->is_blocking = 0; + tpd->is_temporary = 1; + + erts_tsd_set(erts_thr_prgr_data_key__, (void *) tpd); +} + +static ERTS_INLINE ErtsThrPrgrData * +tmp_thr_prgr_data(ErtsSchedulerData *esdp) +{ + ErtsThrPrgrData *tpd = perhaps_thr_prgr_data(esdp); + + if (!tpd) { + /* + * We only allocate the part up to the wakeup_request field + * which is the first field only used by registered threads + */ + tpd = erts_alloc(ERTS_ALC_T_T_THR_PRGR_DATA, + offsetof(ErtsThrPrgrData, wakeup_request)); + init_tmp_thr_prgr_data(tpd); + } + + return tpd; +} + +static ERTS_INLINE void +return_tmp_thr_prgr_data(ErtsThrPrgrData *tpd) +{ + if (tpd->is_temporary) { + erts_tsd_set(erts_thr_prgr_data_key__, NULL); + erts_free(ERTS_ALC_T_T_THR_PRGR_DATA, tpd); + } +} + +static ERTS_INLINE int +block_count_dec(void) +{ + erts_aint32_t block_count; + block_count = erts_atomic32_dec_read_mb(&intrnl->misc.data.block_count); + if (block_count == 0) { + erts_tse_t *event; + event = ((erts_tse_t*) + erts_atomic_read_nob(&intrnl->misc.data.blocker_event)); + if (event) + erts_tse_set(event); + return 1; + } + + return (block_count & ERTS_THR_PRGR_BC_FLG_NOT_BLOCKING) == 0; +} + +static ERTS_INLINE int +block_count_inc(void) +{ + erts_aint32_t block_count; + block_count = erts_atomic32_inc_read_mb(&intrnl->misc.data.block_count); + return (block_count & ERTS_THR_PRGR_BC_FLG_NOT_BLOCKING) == 0; +} + + +void +erts_thr_progress_pre_init(void) +{ + intrnl = NULL; + erts_tsd_key_create(&erts_thr_prgr_data_key__); + init_nob(&erts_thr_prgr__.current, 0); +} + +void +erts_thr_progress_init(int no_schedulers, int managed, int unmanaged) +{ + int i, j, um_low, um_high; + char *ptr; + size_t cb_sz, intrnl_sz, thr_arr_sz, m_wakeup_size, um_wakeup_size, + tot_size; + + intrnl_sz = sizeof(ErtsThrPrgrInternalData); + intrnl_sz = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(intrnl_sz); + + cb_sz = sizeof(ErtsThrPrgrCallbacks)*(managed+unmanaged); + cb_sz = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(cb_sz); + + thr_arr_sz = sizeof(ErtsThrPrgrArray)*managed; + ASSERT(thr_arr_sz == ERTS_ALC_CACHE_LINE_ALIGN_SIZE(thr_arr_sz)); + + m_wakeup_size = sizeof(ErtsThrPrgrManagedWakeupData); + m_wakeup_size += (managed - 1)*sizeof(int); + m_wakeup_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(m_wakeup_size); + + um_low = (unmanaged - 1)/ERTS_THR_PRGR_BM_BITS + 1; + um_high = (um_low - 1)/ERTS_THR_PRGR_BM_BITS + 1; + + um_wakeup_size = sizeof(ErtsThrPrgrUnmanagedWakeupData); + um_wakeup_size += (um_high + um_low)*sizeof(erts_atomic32_t); + um_wakeup_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(um_wakeup_size); + + tot_size = intrnl_sz; + tot_size += cb_sz; + tot_size += thr_arr_sz; + tot_size += m_wakeup_size*ERTS_THR_PRGR_WAKEUP_DATA_SIZE; + tot_size += um_wakeup_size*ERTS_THR_PRGR_WAKEUP_DATA_SIZE; + + ptr = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_THR_PRGR_IDATA, + tot_size); + + intrnl = (ErtsThrPrgrInternalData *) ptr; + ptr += intrnl_sz; + + erts_atomic32_init_nob(&intrnl->misc.data.lflgs, + ERTS_THR_PRGR_LFLG_NO_LEADER); + erts_atomic32_init_nob(&intrnl->misc.data.block_count, + (ERTS_THR_PRGR_BC_FLG_NOT_BLOCKING + | (erts_aint32_t) managed)); + erts_atomic_init_nob(&intrnl->misc.data.blocker_event, ERTS_AINT_NULL); + erts_atomic32_init_nob(&intrnl->misc.data.pref_wakeup_used, 0); + erts_atomic32_init_nob(&intrnl->misc.data.managed_count, 0); + erts_atomic32_init_nob(&intrnl->misc.data.managed_id, no_schedulers); + erts_atomic32_init_nob(&intrnl->misc.data.unmanaged_id, -1); + + intrnl->thr = (ErtsThrPrgrArray *) ptr; + ptr += thr_arr_sz; + for (i = 0; i < managed; i++) + init_nob(&intrnl->thr[i].data.current, 0); + + intrnl->managed.callbacks = (ErtsThrPrgrCallbacks *) ptr; + intrnl->unmanaged.callbacks = &intrnl->managed.callbacks[managed]; + ptr += cb_sz; + + intrnl->managed.no = managed; + for (i = 0; i < managed; i++) { + intrnl->managed.callbacks[i].arg = NULL; + intrnl->managed.callbacks[i].wakeup = NULL; + } + + intrnl->unmanaged.no = unmanaged; + for (i = 0; i < unmanaged; i++) { + intrnl->unmanaged.callbacks[i].arg = NULL; + intrnl->unmanaged.callbacks[i].wakeup = NULL; + } + + for (i = 0; i < ERTS_THR_PRGR_WAKEUP_DATA_SIZE; i++) { + intrnl->managed.data[i] = (ErtsThrPrgrManagedWakeupData *) ptr; + erts_atomic32_init_nob(&intrnl->managed.data[i]->len, 0); + ptr += m_wakeup_size; + } + + for (i = 0; i < ERTS_THR_PRGR_WAKEUP_DATA_SIZE; i++) { + erts_atomic32_t *bm; + intrnl->unmanaged.data[i] = (ErtsThrPrgrUnmanagedWakeupData *) ptr; + erts_atomic32_init_nob(&intrnl->unmanaged.data[i]->len, 0); + bm = (erts_atomic32_t *) (ptr + sizeof(ErtsThrPrgrUnmanagedWakeupData)); + intrnl->unmanaged.data[i]->high = bm; + intrnl->unmanaged.data[i]->high_sz = um_high; + for (j = 0; j < um_high; j++) + erts_atomic32_init_nob(&intrnl->unmanaged.data[i]->high[j], 0); + intrnl->unmanaged.data[i]->low + = &intrnl->unmanaged.data[i]->high[um_high]; + intrnl->unmanaged.data[i]->low_sz = um_low; + for (j = 0; j < um_low; j++) + erts_atomic32_init_nob(&intrnl->unmanaged.data[i]->low[j], 0); + ptr += um_wakeup_size; + } + ERTS_THR_MEMORY_BARRIER; +} + +static void +init_wakeup_request_array(ErtsThrPrgrVal *w) +{ + int i; + ErtsThrPrgrVal current; + + current = read_acqb(&erts_thr_prgr__.current); + for (i = 0; i < ERTS_THR_PRGR_WAKEUP_DATA_SIZE; i++) { + w[i] = current - ((ErtsThrPrgrVal) (ERTS_THR_PRGR_WAKEUP_DATA_SIZE + i)); + if (w[i] > current) + w[i]--; + } +} + +void +erts_thr_progress_register_unmanaged_thread(ErtsThrPrgrCallbacks *callbacks) +{ + ErtsThrPrgrData *tpd = perhaps_thr_prgr_data(NULL); + int is_blocking = 0; + + if (tpd) { + if (!tpd->is_temporary) + erl_exit(ERTS_ABORT_EXIT, + "%s:%d:%s(): Double register of thread\n", + __FILE__, __LINE__, __func__); + is_blocking = tpd->is_blocking; + return_tmp_thr_prgr_data(tpd); + } + + /* + * We only allocate the part up to the leader field + * which is the first field only used by managed threads + */ + tpd = erts_alloc(ERTS_ALC_T_THR_PRGR_DATA, + offsetof(ErtsThrPrgrData, leader)); + tpd->id = (int) erts_atomic32_inc_read_nob(&intrnl->misc.data.unmanaged_id); + tpd->is_managed = 0; + tpd->is_blocking = is_blocking; + tpd->is_temporary = 0; + ASSERT(tpd->id >= 0); + if (tpd->id >= intrnl->unmanaged.no) + erl_exit(ERTS_ABORT_EXIT, + "%s:%d:%s(): Too many unmanaged registered threads\n", + __FILE__, __LINE__, __func__); + + init_wakeup_request_array(&tpd->wakeup_request[0]); + erts_tsd_set(erts_thr_prgr_data_key__, (void *) tpd); + + ASSERT(callbacks->wakeup); + + intrnl->unmanaged.callbacks[tpd->id] = *callbacks; +} + + +void +erts_thr_progress_register_managed_thread(ErtsSchedulerData *esdp, + ErtsThrPrgrCallbacks *callbacks, + int pref_wakeup) +{ + ErtsThrPrgrData *tpd = perhaps_thr_prgr_data(NULL); + int is_blocking = 0, managed; + + if (tpd) { + if (!tpd->is_temporary) + erl_exit(ERTS_ABORT_EXIT, + "%s:%d:%s(): Double register of thread\n", + __FILE__, __LINE__, __func__); + is_blocking = tpd->is_blocking; + return_tmp_thr_prgr_data(tpd); + } + + if (esdp) + tpd = &esdp->thr_progress_data; + else + tpd = erts_alloc(ERTS_ALC_T_THR_PRGR_DATA, sizeof(ErtsThrPrgrData)); + + if (pref_wakeup + && !erts_atomic32_xchg_nob(&intrnl->misc.data.pref_wakeup_used, 1)) + tpd->id = 0; + else if (esdp) + tpd->id = (int) esdp->no; + else + tpd->id = erts_atomic32_inc_read_nob(&intrnl->misc.data.managed_id); + ASSERT(tpd->id >= 0); + if (tpd->id >= intrnl->managed.no) + erl_exit(ERTS_ABORT_EXIT, + "%s:%d:%s(): Too many managed registered threads\n", + __FILE__, __LINE__, __func__); + + tpd->is_managed = 1; + tpd->is_blocking = is_blocking; + tpd->is_temporary = 0; + + init_wakeup_request_array(&tpd->wakeup_request[0]); + + ERTS_THR_PROGRESS_STATE_DEBUG_INIT(tpd->id); + + tpd->leader = 0; + tpd->active = 1; + tpd->previous.local = 0; + tpd->previous.current = ERTS_THR_PRGR_VAL_WAITING; + erts_tsd_set(erts_thr_prgr_data_key__, (void *) tpd); + + erts_atomic32_inc_nob(&intrnl->misc.data.lflgs); + + ASSERT(callbacks->wakeup); + ASSERT(callbacks->prepare_wait); + ASSERT(callbacks->wait); + ASSERT(callbacks->finalize_wait); + + intrnl->managed.callbacks[tpd->id] = *callbacks; + + callbacks->prepare_wait(callbacks->arg); + managed = erts_atomic32_inc_read_relb(&intrnl->misc.data.managed_count); + if (managed != intrnl->managed.no) { + /* Wait until all managed threads have registered... */ + do { + callbacks->wait(callbacks->arg); + callbacks->prepare_wait(callbacks->arg); + managed = erts_atomic32_read_acqb(&intrnl->misc.data.managed_count); + } while (managed != intrnl->managed.no); + } + else { + int id; + /* All managed threads have registered; lets go... */ + for (id = 0; id < managed; id++) + if (id != tpd->id) + wakeup_managed(id); + } + callbacks->finalize_wait(callbacks->arg); +} + +static ERTS_INLINE int +leader_update(ErtsThrPrgrData *tpd) +{ +#ifdef ERTS_ENABLE_LOCK_CHECK + erts_lc_check_exact(NULL, 0); +#endif + if (!tpd->leader) { + /* Probably need to block... */ + block_thread(tpd); + } + else { + erts_aint32_t lflgs; + ErtsThrPrgrVal next; + int ix, sz, make_progress; + + if (tpd->previous.current == ERTS_THR_PRGR_VAL_WAITING) { + /* Took over as leader from another thread */ + tpd->previous.current = read_acqb(&erts_thr_prgr__.current); + tpd->previous.next = tpd->previous.current; + tpd->previous.next++; + if (tpd->previous.next == ERTS_THR_PRGR_VAL_WAITING) + tpd->previous.next = 0; + } + + if (tpd->previous.local == tpd->previous.current) { + ErtsThrPrgrVal val = tpd->previous.current + 1; + if (val == ERTS_THR_PRGR_VAL_WAITING) + val = 0; + tpd->previous.local = val; + set_mb(&intrnl->thr[tpd->id].data.current, val); + } + + next = tpd->previous.next; + + make_progress = 1; + sz = intrnl->managed.no; + for (ix = 0; ix < sz; ix++) { + ErtsThrPrgrVal tmp; + tmp = read_nob(&intrnl->thr[ix].data.current); + if (tmp != next && tmp != ERTS_THR_PRGR_VAL_WAITING) { + make_progress = 0; + ASSERT(erts_thr_progress_has_passed__(next, tmp)); + break; + } + } + + if (make_progress) { + ErtsThrPrgrVal current = next; + + next++; + if (next == ERTS_THR_PRGR_VAL_WAITING) + next = 0; + + set_nob(&intrnl->thr[tpd->id].data.current, next); + set_mb(&erts_thr_prgr__.current, current); + tpd->previous.local = next; + tpd->previous.next = next; + tpd->previous.current = current; + +#if ERTS_THR_PRGR_PRINT_VAL + if (current % 1000 == 0) + erts_fprintf(stderr, "%b64u\n", current); +#endif + handle_wakeup_requests(current); + } + + if (tpd->active) { + lflgs = erts_atomic32_read_nob(&intrnl->misc.data.lflgs); + if (lflgs & ERTS_THR_PRGR_LFLG_BLOCK) + (void) block_thread(tpd); + } + else { + tpd->leader = 0; + tpd->previous.current = ERTS_THR_PRGR_VAL_WAITING; +#if ERTS_THR_PRGR_PRINT_LEADER + erts_fprintf(stderr, "L <- %d\n", tpd->id); +#endif + + ERTS_THR_PROGRESS_STATE_DEBUG_SET_LEADER(tpd->id, 0); + + lflgs = erts_atomic32_read_bor_relb(&intrnl->misc.data.lflgs, + ERTS_THR_PRGR_LFLG_NO_LEADER); + if (lflgs & ERTS_THR_PRGR_LFLG_BLOCK) + lflgs = block_thread(tpd); + if (ERTS_THR_PRGR_LFLGS_ACTIVE(lflgs) == 0 && got_sched_wakeups()) + wakeup_managed(0); + } + } + + return tpd->leader; +} + +static int +update(ErtsThrPrgrData *tpd) +{ + int res; + ErtsThrPrgrVal val; + + if (tpd->leader) + res = 1; + else { + erts_aint32_t lflgs; + res = 0; + val = read_acqb(&erts_thr_prgr__.current); + if (tpd->previous.local == val) { + val++; + if (val == ERTS_THR_PRGR_VAL_WAITING) + val = 0; + tpd->previous.local = val; + set_mb(&intrnl->thr[tpd->id].data.current, val); + } + + lflgs = erts_atomic32_read_nob(&intrnl->misc.data.lflgs); + if (lflgs & ERTS_THR_PRGR_LFLG_BLOCK) + res = 1; /* Need to block in leader_update() */ + + if ((lflgs & ERTS_THR_PRGR_LFLG_NO_LEADER) + && (tpd->active || ERTS_THR_PRGR_LFLGS_ACTIVE(lflgs) == 0)) { + /* Try to take over leadership... */ + erts_aint32_t olflgs; + olflgs = erts_atomic32_read_band_acqb( + &intrnl->misc.data.lflgs, + ~ERTS_THR_PRGR_LFLG_NO_LEADER); + if (olflgs & ERTS_THR_PRGR_LFLG_NO_LEADER) { + tpd->leader = 1; +#if ERTS_THR_PRGR_PRINT_LEADER + erts_fprintf(stderr, "L -> %d\n", tpd->id); +#endif + ERTS_THR_PROGRESS_STATE_DEBUG_SET_LEADER(tpd->id, 1); + } + } + res |= tpd->leader; + } + return res; +} + +int +erts_thr_progress_update(ErtsSchedulerData *esdp) +{ + return update(thr_prgr_data(esdp)); +} + + +int +erts_thr_progress_leader_update(ErtsSchedulerData *esdp) +{ + return leader_update(thr_prgr_data(esdp)); +} + +void +erts_thr_progress_prepare_wait(ErtsSchedulerData *esdp) +{ + erts_aint32_t lflgs; + ErtsThrPrgrData *tpd = thr_prgr_data(esdp); + +#ifdef ERTS_ENABLE_LOCK_CHECK + erts_lc_check_exact(NULL, 0); +#endif + + block_count_dec(); + + tpd->previous.local = ERTS_THR_PRGR_VAL_WAITING; + set_mb(&intrnl->thr[tpd->id].data.current, ERTS_THR_PRGR_VAL_WAITING); + + lflgs = erts_atomic32_read_nob(&intrnl->misc.data.lflgs); + if (ERTS_THR_PRGR_LFLGS_ALL_WAITING(lflgs) && got_sched_wakeups()) + wakeup_managed(0); /* Someone need to make progress */ +} + +void +erts_thr_progress_finalize_wait(ErtsSchedulerData *esdp) +{ + ErtsThrPrgrData *tpd = thr_prgr_data(esdp); + ErtsThrPrgrVal current, val; + +#ifdef ERTS_ENABLE_LOCK_CHECK + erts_lc_check_exact(NULL, 0); +#endif + + /* + * We aren't allowed to continue until our thread + * progress is past global current. + */ + val = current = read_acqb(&erts_thr_prgr__.current); + while (1) { + val++; + if (val == ERTS_THR_PRGR_VAL_WAITING) + val = 0; + tpd->previous.local = val; + set_mb(&intrnl->thr[tpd->id].data.current, val); + val = read_acqb(&erts_thr_prgr__.current); + if (current == val) + break; + current = val; + } + if (block_count_inc()) + block_thread(tpd); + if (update(tpd)) + leader_update(tpd); +} + +void +erts_thr_progress_active(ErtsSchedulerData *esdp, int on) +{ + ErtsThrPrgrData *tpd = thr_prgr_data(esdp); + +#ifdef ERTS_ENABLE_LOCK_CHECK + erts_lc_check_exact(NULL, 0); +#endif + + ERTS_THR_PROGRESS_STATE_DEBUG_SET_ACTIVE(tpd->id, on); + + if (on) { + ASSERT(!tpd->active); + tpd->active = 1; + erts_atomic32_inc_nob(&intrnl->misc.data.lflgs); + } + else { + ASSERT(tpd->active); + tpd->active = 0; + erts_atomic32_dec_nob(&intrnl->misc.data.lflgs); + if (update(tpd)) + leader_update(tpd); + } + +#ifdef DEBUG + { + erts_aint32_t n = erts_atomic32_read_nob(&intrnl->misc.data.lflgs); + n &= ERTS_THR_PRGR_LFLG_ACTIVE_MASK; + ASSERT(tpd->active <= n && n <= intrnl->managed.no); + } +#endif + +} + +static ERTS_INLINE int +has_reached_wakeup(ErtsThrPrgrVal wakeup) +{ + /* + * Exactly the same as erts_thr_progress_has_reached(), but + * also verify valid wakeup requests in debug mode. + */ + ErtsThrPrgrVal current; + + current = read_acqb(&erts_thr_prgr__.current); + +#if ERTS_THR_PRGR_DBG_CHK_WAKEUP_REQUEST_VALUE + { + ErtsThrPrgrVal limit; + /* + * erts_thr_progress_later() returns values which are + * equal to 'current + 2'. That is, users should never + * get a hold of values larger than that. + * + * That is, valid values are values less than 'current + 3'. + * + * Values larger than this won't work with the wakeup + * algorithm. + */ + + limit = current + 3; + if (limit == ERTS_THR_PRGR_VAL_WAITING) + limit = 0; + else if (limit < current) /* Wrapped */ + limit + 1; + + if (!erts_thr_progress_has_passed__(limit, wakeup)) + erl_exit(ERTS_ABORT_EXIT, + "Invalid wakeup request value found:" + " current=%b64u, wakeup=%b64u, limit=%b64u", + current, wakeup, limit); + } +#endif + + if (current == wakeup) + return 1; + return erts_thr_progress_has_passed__(current, wakeup); +} + +static void +request_wakeup_managed(ErtsThrPrgrData *tpd, ErtsThrPrgrVal value) +{ + ErtsThrPrgrManagedWakeupData *mwd; + int ix, wix; + + /* + * Only managed threads that aren't in waiting state + * are allowed to call this function. + */ + + ASSERT(tpd->is_managed); + ASSERT(tpd->previous.local != ERTS_THR_PRGR_VAL_WAITING); + + if (has_reached_wakeup(value)) + wakeup_managed(tpd->id); + + wix = ERTS_THR_PRGR_WAKEUP_IX(value); + if (tpd->wakeup_request[wix] == value) + return; /* Already got a request registered */ + + ASSERT(erts_thr_progress_has_passed__(value, + tpd->wakeup_request[wix])); + + + if (tpd->previous.local == value) { + /* + * We have already confirmed this value. We need to request + * wakeup for a value later than our latest confirmed value in + * order to prevent progress from reaching the requested value + * while we are writing the request. + * + * It is ok to move the wakeup request forward since the only + * guarantee we make (and can make) is that the thread will be + * woken some time *after* the requested value has been reached. + */ + value++; + if (value == ERTS_THR_PRGR_VAL_WAITING) + value = 0; + + wix = ERTS_THR_PRGR_WAKEUP_IX(value); + if (tpd->wakeup_request[wix] == value) + return; /* Already got a request registered */ + + ASSERT(erts_thr_progress_has_passed__(value, + tpd->wakeup_request[wix])); + } + + tpd->wakeup_request[wix] = value; + + mwd = intrnl->managed.data[wix]; + + ix = erts_atomic32_inc_read_nob(&mwd->len) - 1; + mwd->id[ix] = tpd->id; + + ASSERT(!erts_thr_progress_has_reached(value)); + + /* + * This thread is guarranteed to issue a full memory barrier: + * - after the request has been written, but + * - before the global thread progress reach the (possibly + * increased) requested wakeup value. + */ +} + +static void +request_wakeup_unmanaged(ErtsThrPrgrData *tpd, ErtsThrPrgrVal value) +{ + int wix, ix, id, bit; + ErtsThrPrgrUnmanagedWakeupData *umwd; + + ASSERT(!tpd->is_managed); + + /* + * Thread progress *can* reach and pass our requested value while + * we are writing the request. + */ + + if (has_reached_wakeup(value)) + wakeup_unmanaged(tpd->id); + + wix = ERTS_THR_PRGR_WAKEUP_IX(value); + + if (tpd->wakeup_request[wix] == value) + return; /* Already got a request registered */ + + ASSERT(erts_thr_progress_has_passed__(value, + tpd->wakeup_request[wix])); + + umwd = intrnl->unmanaged.data[wix]; + + id = tpd->id; + + bit = id & ERTS_THR_PRGR_BM_MASK; + ix = id >> ERTS_THR_PRGR_BM_SHIFT; + ASSERT(0 <= ix && ix < umwd->low_sz); + erts_atomic32_read_bor_nob(&umwd->low[ix], 1 << bit); + + bit = ix & ERTS_THR_PRGR_BM_MASK; + ix >>= ERTS_THR_PRGR_BM_SHIFT; + ASSERT(0 <= ix && ix < umwd->high_sz); + erts_atomic32_read_bor_nob(&umwd->high[ix], 1 << bit); + + erts_atomic32_inc_mb(&umwd->len); + + if (erts_thr_progress_has_reached(value)) + wakeup_unmanaged(tpd->id); + else + tpd->wakeup_request[wix] = value; +} + +void +erts_thr_progress_wakeup(ErtsSchedulerData *esdp, + ErtsThrPrgrVal value) +{ + ErtsThrPrgrData *tpd = thr_prgr_data(esdp); + ASSERT(!tpd->is_temporary); + if (tpd->is_managed) + request_wakeup_managed(tpd, value); + else + request_wakeup_unmanaged(tpd, value); +} + +static void +wakeup_unmanaged_threads(ErtsThrPrgrUnmanagedWakeupData *umwd) +{ + int hix; + for (hix = 0; hix < umwd->high_sz; hix++) { + erts_aint32_t hmask = erts_atomic32_read_nob(&umwd->high[hix]); + if (hmask) { + int hbase = hix << ERTS_THR_PRGR_BM_SHIFT; + int hbit; + for (hbit = 0; hbit < ERTS_THR_PRGR_BM_BITS; hbit++) { + if (hmask & (1 << hbit)) { + erts_aint_t lmask; + int lix = hbase + hbit; + ASSERT(0 <= lix && lix < umwd->low_sz); + lmask = erts_atomic32_read_nob(&umwd->low[lix]); + if (lmask) { + int lbase = lix << ERTS_THR_PRGR_BM_SHIFT; + int lbit; + for (lbit = 0; lbit < ERTS_THR_PRGR_BM_BITS; lbit++) { + if (lmask & (1 << lbit)) { + int id = lbase + lbit; + wakeup_unmanaged(id); + } + } + erts_atomic32_set_nob(&umwd->low[lix], 0); + } + } + } + erts_atomic32_set_nob(&umwd->high[hix], 0); + } + } +} + + +static void +handle_wakeup_requests(ErtsThrPrgrVal current) +{ + ErtsThrPrgrManagedWakeupData *mwd; + ErtsThrPrgrUnmanagedWakeupData *umwd; + int wix, len, i; + + wix = ERTS_THR_PRGR_WAKEUP_IX(current); + + mwd = intrnl->managed.data[wix]; + len = erts_atomic32_read_nob(&mwd->len); + ASSERT(len >= 0); + if (len) { + for (i = 0; i < len; i++) + wakeup_managed(mwd->id[i]); + erts_atomic32_set_nob(&mwd->len, 0); + } + + umwd = intrnl->unmanaged.data[wix]; + len = erts_atomic32_read_nob(&umwd->len); + ASSERT(len >= 0); + if (len) { + wakeup_unmanaged_threads(umwd); + erts_atomic32_set_nob(&umwd->len, 0); + } + +} + +static int +got_sched_wakeups(void) +{ + int wix; + + ERTS_THR_MEMORY_BARRIER; + + for (wix = 0; wix < ERTS_THR_PRGR_WAKEUP_DATA_SIZE; wix++) { + ErtsThrPrgrManagedWakeupData **mwd = intrnl->managed.data; + if (erts_atomic32_read_nob(&mwd[wix]->len)) + return 1; + } + for (wix = 0; wix < ERTS_THR_PRGR_WAKEUP_DATA_SIZE; wix++) { + ErtsThrPrgrUnmanagedWakeupData **umwd = intrnl->unmanaged.data; + if (erts_atomic32_read_nob(&umwd[wix]->len)) + return 1; + } + return 0; +} + +static erts_aint32_t +block_thread(ErtsThrPrgrData *tpd) +{ + erts_aint32_t lflgs; + ErtsThrPrgrCallbacks *cbp = &intrnl->managed.callbacks[tpd->id]; + + do { + block_count_dec(); + + while (1) { + cbp->prepare_wait(cbp->arg); + lflgs = erts_atomic32_read_nob(&intrnl->misc.data.lflgs); + if (lflgs & ERTS_THR_PRGR_LFLG_BLOCK) + cbp->wait(cbp->arg); + else + break; + } + + } while (block_count_inc()); + + cbp->finalize_wait(cbp->arg); + + return lflgs; +} + +static erts_aint32_t +thr_progress_block(ErtsThrPrgrData *tpd, int wait) +{ + erts_tse_t *event = NULL; /* Remove erroneous warning... sigh... */ + erts_aint32_t lflgs, bc; + + if (tpd->is_blocking++) + return (erts_aint32_t) 0; + + while (1) { + lflgs = erts_atomic32_read_bor_nob(&intrnl->misc.data.lflgs, + ERTS_THR_PRGR_LFLG_BLOCK); + if (lflgs & ERTS_THR_PRGR_LFLG_BLOCK) + block_thread(tpd); + else + break; + } + +#if ERTS_THR_PRGR_PRINT_BLOCKERS + erts_fprintf(stderr, "block(%d)\n", tpd->id); +#endif + + ASSERT(ERTS_AINT_NULL + == erts_atomic_read_nob(&intrnl->misc.data.blocker_event)); + + if (wait) { + event = erts_tse_fetch(); + erts_tse_reset(event); + erts_atomic_set_nob(&intrnl->misc.data.blocker_event, + (erts_aint_t) event); + } + if (tpd->is_managed) + erts_atomic32_dec_nob(&intrnl->misc.data.block_count); + bc = erts_atomic32_read_band_mb(&intrnl->misc.data.block_count, + ~ERTS_THR_PRGR_BC_FLG_NOT_BLOCKING); + bc &= ~ERTS_THR_PRGR_BC_FLG_NOT_BLOCKING; + if (wait) { + while (bc != 0) { + erts_tse_wait(event); + erts_tse_reset(event); + bc = erts_atomic32_read_acqb(&intrnl->misc.data.block_count); + } + } + return bc; + +} + +void +erts_thr_progress_block(void) +{ + thr_progress_block(tmp_thr_prgr_data(NULL), 1); +} + +void +erts_thr_progress_fatal_error_block(SWord timeout) +{ + ErtsThrPrgrData tpd_buf; + ErtsThrPrgrData *tpd = perhaps_thr_prgr_data(NULL); + erts_aint32_t bc; + SWord time_left = timeout; + SysTimeval to; + + /* + * Counting poll intervals may give us a too long timeout + * if cpu is busy. If we got tolerant time of day we use it + * to prevent this. + */ + if (!erts_disable_tolerant_timeofday) { + erts_get_timeval(&to); + to.tv_sec += timeout / 1000; + to.tv_sec += timeout % 1000; + } + + if (!tpd) { + /* + * We stack allocate since failure to allocate memory may + * have caused the problem in the first place. This is ok + * since we never complete an unblock after a fatal error + * block. + */ + tpd = &tpd_buf; + init_tmp_thr_prgr_data(tpd); + } + + bc = thr_progress_block(tpd, 0); + if (bc == 0) + return; /* Succefully blocked all managed threads */ + + while (1) { + if (erts_milli_sleep(ERTS_THR_PRGR_FTL_ERR_BLCK_POLL_INTERVAL) == 0) + time_left -= ERTS_THR_PRGR_FTL_ERR_BLCK_POLL_INTERVAL; + bc = erts_atomic32_read_acqb(&intrnl->misc.data.block_count); + if (bc == 0) + break; /* Succefully blocked all managed threads */ + if (time_left <= 0) + break; /* Timeout */ + if (!erts_disable_tolerant_timeofday) { + SysTimeval now; + erts_get_timeval(&now); + if (now.tv_sec > to.tv_sec) + break; /* Timeout */ + if (now.tv_sec == to.tv_sec && now.tv_usec >= to.tv_usec) + break; /* Timeout */ + } + } +} + +void +erts_thr_progress_unblock(void) +{ + erts_tse_t *event; + int id, break_id, sz, wakeup; + ErtsThrPrgrData *tpd = thr_prgr_data(NULL); + + ASSERT(tpd->is_blocking); + if (--tpd->is_blocking) + return; + + sz = intrnl->managed.no; + + wakeup = 1; + if (!tpd->is_managed) + id = break_id = tpd->id < 0 ? 0 : tpd->id % sz; + else { + break_id = tpd->id; + id = break_id + 1; + if (id >= sz) + id = 0; + if (id == break_id) + wakeup = 0; + erts_atomic32_inc_nob(&intrnl->misc.data.block_count); + } + + event = ((erts_tse_t *) + erts_atomic_read_nob(&intrnl->misc.data.blocker_event)); + ASSERT(event); + erts_atomic_set_nob(&intrnl->misc.data.blocker_event, ERTS_AINT_NULL); + + erts_atomic32_read_bor_relb(&intrnl->misc.data.block_count, + ERTS_THR_PRGR_BC_FLG_NOT_BLOCKING); +#if ERTS_THR_PRGR_PRINT_BLOCKERS + erts_fprintf(stderr, "unblock(%d)\n", tpd->id); +#endif + erts_atomic32_read_band_mb(&intrnl->misc.data.lflgs, + ~ERTS_THR_PRGR_LFLG_BLOCK); + + if (wakeup) { + do { + ErtsThrPrgrVal tmp; + tmp = read_nob(&intrnl->thr[id].data.current); + if (tmp != ERTS_THR_PRGR_VAL_WAITING) + wakeup_managed(id); + if (++id >= sz) + id = 0; + } while (id != break_id); + } + + return_tmp_thr_prgr_data(tpd); + erts_tse_return(event); +} + +int +erts_thr_progress_is_blocking(void) +{ + ErtsThrPrgrData *tpd = perhaps_thr_prgr_data(NULL); + return tpd && tpd->is_blocking; +} + +void erts_thr_progress_dbg_print_state(void) +{ + int id; + int sz = intrnl->managed.no; + + erts_fprintf(stderr, "--- thread progress ---\n"); + erts_fprintf(stderr,"current=%b64u\n", erts_thr_progress_current()); + for (id = 0; id < sz; id++) { + ErtsThrPrgrVal current = read_nob(&intrnl->thr[id].data.current); +#ifdef ERTS_THR_PROGRESS_STATE_DEBUG + erts_aint32_t state_debug; + char *active, *leader; + + state_debug = erts_atomic32_read_nob(&intrnl->thr[id].data.state_debug); + active = (state_debug & ERTS_THR_PROGRESS_STATE_DEBUG_ACTIVE + ? "true" + : "false"); + leader = (state_debug & ERTS_THR_PROGRESS_STATE_DEBUG_LEADER + ? "true" + : "false"); +#endif + if (current == ERTS_THR_PRGR_VAL_WAITING) + erts_fprintf(stderr, + " id=%d, current=WAITING" +#ifdef ERTS_THR_PROGRESS_STATE_DEBUG + ", active=%s, leader=%s" +#endif + "\n", id +#ifdef ERTS_THR_PROGRESS_STATE_DEBUG + , active, leader +#endif + ); + else + erts_fprintf(stderr, + " id=%d, current=%b64u" +#ifdef ERTS_THR_PROGRESS_STATE_DEBUG + ", active=%s, leader=%s" +#endif + "\n", id, current +#ifdef ERTS_THR_PROGRESS_STATE_DEBUG + , active, leader +#endif + ); + } + erts_fprintf(stderr, "-----------------------\n"); + + +} + +#endif diff --git a/erts/emulator/beam/erl_thr_progress.h b/erts/emulator/beam/erl_thr_progress.h new file mode 100644 index 0000000000..68d14174b9 --- /dev/null +++ b/erts/emulator/beam/erl_thr_progress.h @@ -0,0 +1,233 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2011. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Thread progress information. Used by lock free algorithms + * to determine when all involved threads are guaranteed to + * have passed a specific point of execution. + * + * Usage instructions can be found in ert_thr_progress.c + * + * Author: Rickard Green + */ + +#if !defined(ERL_THR_PROGRESS_H__TSD_TYPE__) +#define ERL_THR_PROGRESS_H__TSD_TYPE__ + +#include "sys.h" + +#ifndef ERTS_SMP + +#define erts_smp_thr_progress_block() ((void) 0) +#define erts_smp_thr_progress_unblock() ((void) 0) +#define erts_smp_thr_progress_is_blocking() 1 + +#else /* ERTS_SMP */ + +#define erts_smp_thr_progress_block erts_thr_progress_block +#define erts_smp_thr_progress_unblock erts_thr_progress_unblock +#define erts_smp_thr_progress_is_blocking erts_thr_progress_is_blocking + +void erts_thr_progress_fatal_error_block(SWord timeout); +void erts_thr_progress_block(void); +void erts_thr_progress_unblock(void); +int erts_thr_progress_is_blocking(void); + +typedef Uint64 ErtsThrPrgrVal; + +#define ERTS_THR_PRGR_WAKEUP_DATA_SIZE 4 /* Need to be an even power of 2. */ + +typedef struct { + int id; + int is_managed; + int is_blocking; + int is_temporary; + + /* --- Part below only for registered threads --- */ + + ErtsThrPrgrVal wakeup_request[ERTS_THR_PRGR_WAKEUP_DATA_SIZE]; + + /* --- Part below only for managed threads --- */ + + int leader; /* Needs to be first in the managed threads part */ + int active; + struct { + ErtsThrPrgrVal local; + ErtsThrPrgrVal next; + ErtsThrPrgrVal current; + } previous; +} ErtsThrPrgrData; +#endif /* ERTS_SMP */ + +#endif + +#if !defined(ERL_THR_PROGRESS_H__) && !defined(ERL_THR_PROGRESS_TSD_TYPE_ONLY) +#define ERL_THR_PROGRESS_H__ + +#include "erl_threads.h" +#include "erl_process.h" + +#ifdef ERTS_SMP + +#define ERTS_THR_PRGR_VAL_WAITING (~((ErtsThrPrgrVal) 0)) + +extern erts_tsd_key_t erts_thr_prgr_data_key__; + +#ifdef ARCH_64 +# define ERTS_THR_PRGR_ATOMIC erts_atomic_t +#else /* ARCH_32 */ +# define ERTS_THR_PRGR_ATOMIC erts_dw_atomic_t +#endif + +typedef struct { + void *arg; + void (*wakeup)(void *); + void (*prepare_wait)(void *); + void (*wait)(void *); + void (*finalize_wait)(void *); +} ErtsThrPrgrCallbacks; + +typedef struct { + ERTS_THR_PRGR_ATOMIC current; +} ErtsThrPrgr; + +extern ErtsThrPrgr erts_thr_prgr__; + +void erts_thr_progress_pre_init(void); +void erts_thr_progress_init(int no_schedulers, int managed, int unmanaged); +void erts_thr_progress_register_managed_thread(ErtsSchedulerData *esdp, + ErtsThrPrgrCallbacks *, + int); +void erts_thr_progress_register_unmanaged_thread(ErtsThrPrgrCallbacks *); +void erts_thr_progress_active(ErtsSchedulerData *esdp, int on); +void erts_thr_progress_wakeup(ErtsSchedulerData *esdp, + ErtsThrPrgrVal value); +int erts_thr_progress_update(ErtsSchedulerData *esdp); +int erts_thr_progress_leader_update(ErtsSchedulerData *esdp); +void erts_thr_progress_prepare_wait(ErtsSchedulerData *esdp); +void erts_thr_progress_finalize_wait(ErtsSchedulerData *esdp); + +void erts_thr_progress_dbg_print_state(void); + +#ifdef ARCH_32 +#define ERTS_THR_PRGR_ATOMIC erts_dw_atomic_t +ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_dw_sint_to_val__(ethr_dw_sint_t *dw_sint); +#endif +ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc); + +ERTS_GLB_INLINE int erts_thr_progress_is_managed_thread(void); +ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_progress_later(void); +ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_progress_current(void); +ERTS_GLB_INLINE int erts_thr_progress_has_passed__(ErtsThrPrgrVal val1, ErtsThrPrgrVal val2); +ERTS_GLB_INLINE int erts_thr_progress_has_reached(ErtsThrPrgrVal val); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +#ifdef ARCH_64 + +ERTS_GLB_INLINE ErtsThrPrgrVal +erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc) +{ + return (ErtsThrPrgrVal) erts_atomic_read_acqb(atmc); +} + +#else /* ARCH_32 */ + +ERTS_GLB_INLINE ErtsThrPrgrVal +erts_thr_prgr_dw_sint_to_val__(ethr_dw_sint_t *dw_sint) +{ +#ifdef ETHR_SU_DW_NAINT_T__ + return (ErtsThrPrgrVal) dw_sint->dw_sint; +#else + ErtsThrPrgrVal res; + res = (ErtsThrPrgrVal) ((Uint32) dw_sint->sint[ETHR_DW_SINT_HIGH_WORD]); + res <<= 32; + res |= (ErtsThrPrgrVal) ((Uint32) dw_sint->sint[ETHR_DW_SINT_LOW_WORD]); + return res; +#endif +} + +ERTS_GLB_INLINE ErtsThrPrgrVal +erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc) +{ + ethr_dw_sint_t dw_sint; + erts_dw_atomic_read_acqb(atmc, &dw_sint); + return erts_thr_prgr_dw_sint_to_val__(&dw_sint); +} + +#endif + +ERTS_GLB_INLINE int +erts_thr_progress_is_managed_thread(void) +{ + ErtsThrPrgrData *tpd = erts_tsd_get(erts_thr_prgr_data_key__); + return tpd && tpd->is_managed; +} + +ERTS_GLB_INLINE ErtsThrPrgrVal +erts_thr_progress_later(void) +{ + ErtsThrPrgrVal val = erts_thr_prgr_read_acqb__(&erts_thr_prgr__.current); + if (val == (ERTS_THR_PRGR_VAL_WAITING-((ErtsThrPrgrVal)2))) + return ((ErtsThrPrgrVal) 0); + else if (val == (ERTS_THR_PRGR_VAL_WAITING-((ErtsThrPrgrVal)1))) + return ((ErtsThrPrgrVal) 1); + else + return val + ((ErtsThrPrgrVal) 2); +} + +ERTS_GLB_INLINE ErtsThrPrgrVal +erts_thr_progress_current(void) +{ + return erts_thr_prgr_read_acqb__(&erts_thr_prgr__.current); +} + +ERTS_GLB_INLINE int +erts_thr_progress_has_passed__(ErtsThrPrgrVal val1, ErtsThrPrgrVal val0) +{ + if ((((((ErtsThrPrgrVal) 1) << 63) & val1) + ^ ((((ErtsThrPrgrVal) 1) << 63) & val0)) != 0) { + /* May have wrapped... */ + if (val1 < (((ErtsThrPrgrVal) 1) << 62) + && val0 > (((ErtsThrPrgrVal) 3) << 62)) { + /* + * 'val1' has wrapped but 'val0' has not yet wrapped. While in + * these ranges 'current' is considered later than 'val0'. + */ + return 1; + } + } + return val1 > val0; +} + +ERTS_GLB_INLINE int +erts_thr_progress_has_reached(ErtsThrPrgrVal val) +{ + ErtsThrPrgrVal current; + current = erts_thr_prgr_read_acqb__(&erts_thr_prgr__.current); + if (current == val) + return 1; + return erts_thr_progress_has_passed__(current, val); +} + +#endif + +#endif /* ERTS_SMP */ + +#endif diff --git a/erts/emulator/beam/erl_thr_queue.c b/erts/emulator/beam/erl_thr_queue.c new file mode 100644 index 0000000000..9ac4cd4b8e --- /dev/null +++ b/erts/emulator/beam/erl_thr_queue.c @@ -0,0 +1,745 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2011. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Lock-free queue for communication between threads. + * + * Currently only a many-to-one version has been, + * implemented, i.e., many threads can enqueue but + * only one thread can dequeue at a time. It doesn't + * have to be the same thread dequeuing every time, but + * synchronization so that only one thread dequeues + * at a time has to be provided by other means. + * + * When/If the need for a many-to-many queue arises, + * this implementation can relatively easy be extended + * to support that too. + * + * Usage instructions below. + * + * Author: Rickard Green + */ + +/* + * ------ Usage instructions ----------------------------------------------- + * + * Dequeuing generates garbage that needs to be cleaned up. + * erts_thr_q_dequeue() automatically cleans, but garbage may have to be + * cleaned up also when the queue is empty. This is done by calling + * erts_thr_q_clean(). In the SMP case thread progress may have to be made + * before cleaning can continue. If so, erts_thr_q_need_thr_progress() in + * combination with erts_thr_progress_wakeup() can be used in order to + * request a wakeup at appropriate time. + * + * Enqueuing implies memory allocation and dequeuing implies memory + * deallocation. Memory allocation can be moved to another more suitable + * thread using erts_thr_q_prepare_enqueue() together with + * erts_thr_q_enqueue_prepared() instead of using erts_thr_q_enqueue(). + * Memory deallocation can can be moved to another more suitable thread by + * disabling auto_finalize_dequeue when initializing the queue and then use + * erts_thr_q_get_finalize_dequeue_data() together + * erts_thr_q_finalize_dequeue() after dequeuing or cleaning. + * + * Ending the life of the queue using either erts_thr_q_destroy() + * or erts_thr_q_finalize() impies cleaning the queue. Both functions + * return the cleaning result and may have to be called multiple times + * until the queue is clean. Once one of these functions have been called + * enqueuing is not allowed. This has to be synchronized by the user. + * If auto_finalize_dequeue has been disabled, the finalize dequeue + * functionality has to be called after ending the life of the queue just + * as when dequeuing or cleaning on a queue that is alive. + * + * ------------------------------------------------------------------------- + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "erl_thr_queue.h" + +#if defined(DEBUG) +#define ERTS_THR_Q_DBG_CHK_DATA 1 +#else +#define ERTS_THR_Q_DBG_CHK_DATA 0 +#endif + +#define ERTS_THR_Q_MAX_CLEAN_REACHED_HEAD_COUNT 100 +#define ERTS_THR_Q_MAX_SCHED_CLEAN_OPS 50 +#define ERTS_THR_Q_MAX_DEQUEUE_CLEAN_OPS 3 + +#define ERTS_THR_Q_MAX_FINI_DEQ_OPS 50 + +#ifdef ERTS_SMP +ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(sl_element, + ErtsThrQElement_t, + 1000, + ERTS_ALC_T_THR_Q_EL_SL) +#else + +static void +init_sl_element_alloc(void) +{ +} + +static ErtsThrQElement_t * +sl_element_alloc(void) +{ + return erts_alloc(ERTS_ALC_T_THR_Q_EL_SL, + sizeof(ErtsThrQElement_t)); +} + +static void +sl_element_free(ErtsThrQElement_t *p) +{ + erts_free(ERTS_ALC_T_THR_Q_EL_SL, p); +} + +#endif + +typedef union { + ErtsThrQ_t q; + char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsThrQ_t))]; +} ErtsAlignedThrQ_t; + +void +erts_thr_q_init(void) +{ + init_sl_element_alloc(); +} + +static void noop_callback(void *arg) { } + +void +erts_thr_q_initialize(ErtsThrQ_t *q, ErtsThrQInit_t *qi) +{ +#ifndef USE_THREADS + q->init = *qi; + if (!q->init.notify) + q->init.notify = noop_callback; + q->first = NULL; + q->last = NULL; + q->q.blk = NULL; +#else + erts_atomic_init_nob(&q->tail.data.marker.next.atmc, ERTS_AINT_NULL); + q->tail.data.marker.data.ptr = NULL; + erts_atomic_init_nob(&q->tail.data.last, + (erts_aint_t) &q->tail.data.marker); + erts_atomic_init_nob(&q->tail.data.um_refc[0], 0); + erts_atomic_init_nob(&q->tail.data.um_refc[1], 0); + erts_atomic32_init_nob(&q->tail.data.um_refc_ix, 0); + q->tail.data.live = qi->live.objects; + q->tail.data.arg = qi->arg; + q->tail.data.notify = qi->notify; + if (!q->tail.data.notify) + q->tail.data.notify = noop_callback; + + q->head.head.ptr = &q->tail.data.marker; + q->head.live = qi->live.objects; + q->head.first = &q->tail.data.marker; + q->head.unref_end = &q->tail.data.marker; + q->head.clean_reached_head_count = 0; + q->head.deq_fini.automatic = qi->auto_finalize_dequeue; + q->head.deq_fini.start = NULL; + q->head.deq_fini.end = NULL; +#ifdef ERTS_SMP + q->head.next.thr_progress = erts_thr_progress_current(); + q->head.next.thr_progress_reached = 1; +#endif + q->head.next.um_refc_ix = 1; + q->head.next.unref_end = &q->tail.data.marker; + q->head.used_marker = 1; + q->head.arg = qi->arg; + q->head.notify = q->tail.data.notify; + q->q.finalizing = 0; + q->q.live = qi->live.queue; + q->q.blk = NULL; +#endif +} + +ErtsThrQCleanState_t +erts_thr_q_finalize(ErtsThrQ_t *q) +{ +#ifdef USE_THREADS + q->q.finalizing = 1; +#endif + while (erts_thr_q_dequeue(q)); + return erts_thr_q_clean(q); +} + +ErtsThrQ_t * +erts_thr_q_create(ErtsThrQInit_t *qi) +{ + ErtsAlcType_t atype; + ErtsThrQ_t *q, *qblk; + UWord qw; + + switch (qi->live.queue) { + case ERTS_THR_Q_LIVE_SHORT: + atype = ERTS_ALC_T_THR_Q_SL; + break; + case ERTS_THR_Q_LIVE_LONG: + atype = ERTS_ALC_T_THR_Q_LL; + break; + default: + atype = ERTS_ALC_T_THR_Q; + break; + } + + qw = (UWord) erts_alloc(atype, + sizeof(ErtsThrQ_t) + (ERTS_CACHE_LINE_SIZE-1)); + qblk = (ErtsThrQ_t *) qw; + if (qw & ERTS_CACHE_LINE_MASK) + qw = (qw & ~ERTS_CACHE_LINE_MASK) + ERTS_CACHE_LINE_SIZE; + ASSERT((qw & ERTS_CACHE_LINE_MASK) == 0); + q = (ErtsThrQ_t *) qw; + erts_thr_q_initialize(q, qi); + q->q.blk = qblk; + return q; +} + +ErtsThrQCleanState_t +erts_thr_q_destroy(ErtsThrQ_t *q) +{ + if (!q->q.blk) + erl_exit(ERTS_ABORT_EXIT, + "Trying to destroy not created thread queue\n"); + return erts_thr_q_finalize(q); +} + +#ifdef USE_THREADS + +static void +destroy(ErtsThrQ_t *q) +{ + ErtsAlcType_t atype; + switch (q->q.live) { + case ERTS_THR_Q_LIVE_SHORT: + atype = ERTS_ALC_T_THR_Q_SL; + break; + case ERTS_THR_Q_LIVE_LONG: + atype = ERTS_ALC_T_THR_Q_LL; + break; + default: + atype = ERTS_ALC_T_THR_Q; + break; + } + erts_free(atype, q->q.blk); +} + +#endif + +static ERTS_INLINE ErtsThrQElement_t * +element_live_alloc(ErtsThrQLive_t live) +{ + switch (live) { + case ERTS_THR_Q_LIVE_SHORT: + return sl_element_alloc(); + default: + return (ErtsThrQElement_t *) erts_alloc(ERTS_ALC_T_THR_Q_EL, + sizeof(ErtsThrQElement_t)); + } +} + +static ERTS_INLINE ErtsThrQElement_t * +element_alloc(ErtsThrQ_t *q) +{ + ErtsThrQLive_t live; +#ifdef USE_THREADS + live = q->tail.data.live; +#else + live = q->init.live.objects; +#endif + return element_live_alloc(live); +} + +static ERTS_INLINE void +element_live_free(ErtsThrQLive_t live, ErtsThrQElement_t *el) +{ + switch (live) { + case ERTS_THR_Q_LIVE_SHORT: + sl_element_free(el); + break; + default: + erts_free(ERTS_ALC_T_THR_Q_EL, el); + } +} + +static ERTS_INLINE void +element_free(ErtsThrQ_t *q, ErtsThrQElement_t *el) +{ + ErtsThrQLive_t live; +#ifdef USE_THREADS + live = q->head.live; +#else + live = q->init.live.objects; +#endif + element_live_free(live, el); +} + +#ifdef USE_THREADS + +static ERTS_INLINE ErtsThrQElement_t * +enqueue_managed(ErtsThrQ_t *q, ErtsThrQElement_t *this, int want_last) +{ + erts_aint_t ilast, itmp; + + erts_atomic_init_nob(&this->next.atmc, ERTS_AINT_NULL); + /* Enqueue at end of list... */ + + ilast = erts_atomic_read_nob(&q->tail.data.last); + while (1) { + ErtsThrQElement_t *last = (ErtsThrQElement_t *) ilast; + itmp = erts_atomic_cmpxchg_mb(&last->next.atmc, + (erts_aint_t) this, + ERTS_AINT_NULL); + if (itmp == ERTS_AINT_NULL) + break; + ilast = itmp; + } + + /* Move last pointer forward... */ + while (1) { + if (want_last) { + if (erts_atomic_read_rb(&this->next.atmc) != ERTS_AINT_NULL) { + /* Someone else will move it forward */ + ilast = erts_atomic_read_rb(&q->tail.data.last); + return (ErtsThrQElement_t *) ilast; + } + } + else { + if (erts_atomic_read_nob(&this->next.atmc) != ERTS_AINT_NULL) { + /* Someone else will move it forward */ + return NULL; + } + } + itmp = erts_atomic_cmpxchg_mb(&q->tail.data.last, + (erts_aint_t) this, + ilast); + if (ilast == itmp) + return want_last ? this : NULL; + ilast = itmp; + } +} + +static ErtsThrQCleanState_t +clean(ErtsThrQ_t *q, int max_ops, int do_notify) +{ + erts_aint_t ilast; + int um_refc_ix; + int ops; + + for (ops = 0; ops < max_ops; ops++) { + ErtsThrQElement_t *tmp; + restart: + ASSERT(q->head.first); + if (q->head.first == q->head.head.ptr) { + q->head.clean_reached_head_count++; + if (q->head.clean_reached_head_count + >= ERTS_THR_Q_MAX_CLEAN_REACHED_HEAD_COUNT) { + q->head.clean_reached_head_count = 0; + break; + } + goto inspect_head; + } + if (q->head.first == q->head.unref_end) + break; + if (q->head.first == &q->tail.data.marker) { + q->head.used_marker = 0; + q->head.first = q->head.first->next.ptr; + goto restart; + } + tmp = q->head.first; + q->head.first = q->head.first->next.ptr; + if (q->head.deq_fini.automatic) + element_free(q, tmp); + else { + tmp->data.ptr = (void *) (UWord) q->head.live; + if (!q->head.deq_fini.start) + q->head.deq_fini.start = tmp; + else if (q->head.deq_fini.end->next.ptr == &q->tail.data.marker) + q->head.deq_fini.end->next.ptr = tmp; + q->head.deq_fini.end = tmp; + } + } + + ilast = erts_atomic_read_nob(&q->tail.data.last); + if (q->head.first == ((ErtsThrQElement_t *) ilast) + && ((ErtsThrQElement_t *) ilast) == &q->tail.data.marker + && q->head.first == &q->tail.data.marker) { + /* Empty and clean queue */ + if (q->q.finalizing) + destroy(q); + return ERTS_THR_Q_CLEAN; + } + +#ifdef ERTS_SMP + if (q->head.next.thr_progress_reached + || erts_thr_progress_has_reached(q->head.next.thr_progress)) { + q->head.next.thr_progress_reached = 1; +#endif + um_refc_ix = q->head.next.um_refc_ix; + if (erts_atomic_read_acqb(&q->tail.data.um_refc[um_refc_ix]) == 0) { + /* Move unreferenced end pointer forward... */ + q->head.clean_reached_head_count = 0; + q->head.unref_end = q->head.next.unref_end; + + if (!q->head.used_marker + && q->head.unref_end == (ErtsThrQElement_t *) ilast) { + q->head.used_marker = 1; + ilast = (erts_aint_t) enqueue_managed(q, + &q->tail.data.marker, + 1); + if (q->head.head.ptr == q->head.unref_end) { + ErtsThrQElement_t *next; + next = ((ErtsThrQElement_t *) + erts_atomic_read_acqb(&q->head.head.ptr->next.atmc)); + if (next == &q->tail.data.marker) { + q->head.head.ptr->next.ptr = &q->tail.data.marker; + q->head.head.ptr = &q->tail.data.marker; + } + } + } + + if (q->head.unref_end == (ErtsThrQElement_t *) ilast) + ERTS_THR_MEMORY_BARRIER; + else { + q->head.next.unref_end = (ErtsThrQElement_t *) ilast; + ERTS_THR_MEMORY_BARRIER; +#ifdef ERTS_SMP + q->head.next.thr_progress = erts_thr_progress_later(); +#endif + erts_atomic32_set_relb(&q->tail.data.um_refc_ix, + um_refc_ix); + q->head.next.um_refc_ix = um_refc_ix == 0 ? 1 : 0; +#ifdef ERTS_SMP + q->head.next.thr_progress_reached = 0; +#endif + } + } +#ifdef ERTS_SMP + } +#endif + + if (q->head.first == q->head.head.ptr) { + inspect_head: + if (!q->head.used_marker) { + erts_aint_t inext; + inext = erts_atomic_read_acqb(&q->head.head.ptr->next.atmc); + if (inext == ERTS_AINT_NULL) { + q->head.used_marker = 1; + (void) enqueue_managed(q, &q->tail.data.marker, 0); + inext = erts_atomic_read_acqb(&q->head.head.ptr->next.atmc); + if (inext == (erts_aint_t) &q->tail.data.marker) { + q->head.head.ptr->next.ptr = &q->tail.data.marker; + q->head.head.ptr = &q->tail.data.marker; +#ifdef ERTS_SMP + if (!q->head.next.thr_progress_reached) + return ERTS_THR_Q_NEED_THR_PRGR; +#else + if (do_notify) + q->head.notify(q->head.arg); +#endif + return ERTS_THR_Q_DIRTY; + } + } + } + return ERTS_THR_Q_CLEAN; + } + + if (q->head.first != q->head.unref_end) { + if (do_notify) + q->head.notify(q->head.arg); + return ERTS_THR_Q_DIRTY; + } + +#ifdef ERTS_SMP + if (!q->head.next.thr_progress_reached) + return ERTS_THR_Q_NEED_THR_PRGR; +#endif + + return ERTS_THR_Q_CLEAN; /* Waiting for unmanaged threads to complete... */ +} + +#endif + +ErtsThrQCleanState_t +erts_thr_q_clean(ErtsThrQ_t *q) +{ +#ifdef USE_THREADS + return clean(q, ERTS_THR_Q_MAX_SCHED_CLEAN_OPS, 0); +#else + return ERTS_THR_Q_CLEAN; +#endif +} + +ErtsThrQCleanState_t +erts_thr_q_inspect(ErtsThrQ_t *q, int ensure_empty) +{ +#ifdef USE_THREADS + if (ensure_empty) { + erts_aint_t inext; + inext = erts_atomic_read_acqb(&q->head.head.ptr->next.atmc); + if (inext != ERTS_AINT_NULL) { + if (&q->tail.data.marker != (ErtsThrQElement_t *) inext) + return ERTS_THR_Q_DIRTY; + else { + q->head.head.ptr->next.ptr = (ErtsThrQElement_t *) inext; + q->head.head.ptr = (ErtsThrQElement_t *) inext; + inext = erts_atomic_read_acqb(&q->head.head.ptr->next.atmc); + if (inext != ERTS_AINT_NULL) + return ERTS_THR_Q_DIRTY; + } + } + } + + if (q->head.first == q->head.head.ptr) { + if (!q->head.used_marker) { + erts_aint_t inext; + inext = erts_atomic_read_acqb(&q->head.head.ptr->next.atmc); + if (inext == ERTS_AINT_NULL) + return ERTS_THR_Q_DIRTY; + } + return ERTS_THR_Q_CLEAN; + } + + if (q->head.first != q->head.unref_end) + return ERTS_THR_Q_DIRTY; + +#ifdef ERTS_SMP + if (!q->head.next.thr_progress_reached) + return ERTS_THR_Q_NEED_THR_PRGR; +#endif +#endif + return ERTS_THR_Q_CLEAN; +} + +static void +enqueue(ErtsThrQ_t *q, void *data, ErtsThrQElement_t *this) +{ +#ifndef USE_THREADS + ASSERT(data); + + this->next.ptr = NULL; + this->data.ptr = data; + + if (q->last) + q->last->next.ptr = this; + else { + q->first = q->last = this; + q->init.notify(q->init.arg); + } +#else + int notify; + int um_refc_ix = 0; +#ifdef ERTS_SMP + int unmanaged_thread; +#endif + +#if ERTS_THR_Q_DBG_CHK_DATA + if (!data) + erl_exit(ERTS_ABORT_EXIT, "Missing data in enqueue\n"); +#endif + + ASSERT(!q->q.finalizing); + + this->data.ptr = data; + +#ifdef ERTS_SMP + unmanaged_thread = !erts_thr_progress_is_managed_thread(); + if (unmanaged_thread) +#endif + { + um_refc_ix = erts_atomic32_read_acqb(&q->tail.data.um_refc_ix); + while (1) { + int tmp_um_refc_ix; + erts_atomic_inc_acqb(&q->tail.data.um_refc[um_refc_ix]); + tmp_um_refc_ix = erts_atomic32_read_acqb(&q->tail.data.um_refc_ix); + if (tmp_um_refc_ix == um_refc_ix) + break; + erts_atomic_dec_relb(&q->tail.data.um_refc[um_refc_ix]); + um_refc_ix = tmp_um_refc_ix; + } + } + + notify = this == enqueue_managed(q, this, 1); + + +#ifdef ERTS_SMP + if (unmanaged_thread) +#endif + { + if (notify) + erts_atomic_dec_relb(&q->tail.data.um_refc[um_refc_ix]); + else if (erts_atomic_dec_read_relb(&q->tail.data.um_refc[um_refc_ix]) == 0) + notify = 1; + } + if (notify) + q->tail.data.notify(q->tail.data.arg); +#endif +} + +void +erts_thr_q_enqueue(ErtsThrQ_t *q, void *data) +{ + enqueue(q, data, element_alloc(q)); +} + +ErtsThrQPrepEnQ_t * +erts_thr_q_prepare_enqueue(ErtsThrQ_t *q) +{ + return (ErtsThrQPrepEnQ_t *) element_alloc(q); +} + +int +erts_thr_q_get_finalize_dequeue_data(ErtsThrQ_t *q, ErtsThrQFinDeQ_t *fdp) +{ +#ifndef USE_THREADS + return 0; +#else +#ifdef DEBUG + if (!q->head.deq_fini.start) { + ASSERT(!q->head.deq_fini.end); + } + else { + ErtsThrQElement_t *e = q->head.deq_fini.start; + ErtsThrQElement_t *end = q->head.deq_fini.end; + while (e != end) { + ASSERT(q->head.head.ptr != e); + ASSERT(q->head.first != e); + ASSERT(q->head.unref_end != e); + e = e->next.ptr; + } + } +#endif + fdp->start = q->head.deq_fini.start; + fdp->end = q->head.deq_fini.end; + if (fdp->end) + fdp->end->next.ptr = NULL; + q->head.deq_fini.start = NULL; + q->head.deq_fini.end = NULL; + return fdp->start != NULL; +#endif +} + +void +erts_thr_q_append_finalize_dequeue_data(ErtsThrQFinDeQ_t *fdp0, + ErtsThrQFinDeQ_t *fdp1) +{ +#ifdef USE_THREADS + if (fdp1->start) { + if (fdp0->end) + fdp0->end->next.ptr = fdp1->start; + else + fdp0->start = fdp1->start; + fdp0->end = fdp1->end; + } +#endif +} + + +int erts_thr_q_finalize_dequeue(ErtsThrQFinDeQ_t *state) +{ +#ifdef USE_THREADS + ErtsThrQElement_t *start = state->start; + if (start) { + ErtsThrQLive_t live; + int i; + for (i = 0; i < ERTS_THR_Q_MAX_FINI_DEQ_OPS; i++) { + ErtsThrQElement_t *tmp; + if (!start) + break; + tmp = start; + start = start->next.ptr; + live = (ErtsThrQLive_t) (UWord) tmp->data.ptr; + element_live_free(live, tmp); + } + state->start = start; + if (start) + return 1; /* More to do */ + state->end = NULL; + } +#endif + return 0; +} + +void +erts_thr_q_finalize_dequeue_state_init(ErtsThrQFinDeQ_t *state) +{ +#ifdef USE_THREADS + state->start = NULL; + state->end = NULL; +#endif +} + + +void +erts_thr_q_enqueue_prepared(ErtsThrQ_t *q, void *data, ErtsThrQPrepEnQ_t *prep) +{ + ASSERT(prep); + enqueue(q, data, (ErtsThrQElement_t *) prep); +} + +void * +erts_thr_q_dequeue(ErtsThrQ_t *q) +{ +#ifndef USE_THREADS + void *res; + ErtsThrQElement_t *tmp; + + if (!q->first) + return NULL; + tmp = q->first; + res = tmp->data.ptr; + q->first = tmp->next.ptr; + if (!q->first) + q->last = NULL; + + element_free(q, tmp); + + return res; +#else + erts_aint_t inext; + void *res; + + inext = erts_atomic_read_acqb(&q->head.head.ptr->next.atmc); + if (inext == ERTS_AINT_NULL) + return NULL; + q->head.head.ptr->next.ptr = (ErtsThrQElement_t *) inext; + q->head.head.ptr = (ErtsThrQElement_t *) inext; + if (q->head.head.ptr == &q->tail.data.marker) { + inext = erts_atomic_read_acqb(&q->head.head.ptr->next.atmc); + if (inext == ERTS_AINT_NULL) + return NULL; + q->head.head.ptr->next.ptr = (ErtsThrQElement_t *) inext; + q->head.head.ptr = (ErtsThrQElement_t *) inext; + } + res = q->head.head.ptr->data.ptr; +#if ERTS_THR_Q_DBG_CHK_DATA + q->head.head.ptr->data.ptr = NULL; + if (!res) + erl_exit(ERTS_ABORT_EXIT, "Missing data in dequeue\n"); +#endif + clean(q, + (q->head.deq_fini.automatic + ? ERTS_THR_Q_MAX_DEQUEUE_CLEAN_OPS + : ERTS_THR_Q_MAX_SCHED_CLEAN_OPS), 1); + return res; +#endif +} diff --git a/erts/emulator/beam/erl_thr_queue.h b/erts/emulator/beam/erl_thr_queue.h new file mode 100644 index 0000000000..407c23f5eb --- /dev/null +++ b/erts/emulator/beam/erl_thr_queue.h @@ -0,0 +1,211 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2011. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Lock-free queue for communication between threads. + * + * Currently only a many-to-one version has been, + * implemented, i.e., many threads can enqueue but + * only one thread can dequeue at a time. It doesn't + * have to be the same thread dequeuing every time, but + * synchronization so that only one thread dequeues + * at a time has to be provided by other means. + * + * When/If the need for a many-to-many queue arises, + * this implementation can relatively easy be extended + * to support that too. + * + * Usage instructions can be found in erts_thr_queue.c + * + * Author: Rickard Green + */ + +#ifndef ERL_THR_QUEUE_H__ +#define ERL_THR_QUEUE_H__ + +#include "sys.h" +#include "erl_threads.h" +#include "erl_alloc.h" +#include "erl_thr_progress.h" + +typedef enum { + ERTS_THR_Q_LIVE_UNDEF, + ERTS_THR_Q_LIVE_SHORT, + ERTS_THR_Q_LIVE_LONG +} ErtsThrQLive_t; + +#define ERTS_THR_Q_INIT_DEFAULT \ +{ \ + { \ + ERTS_THR_Q_LIVE_UNDEF, \ + ERTS_THR_Q_LIVE_SHORT \ + }, \ + NULL, \ + NULL, \ + 1 \ +} + +typedef struct ErtsThrQ_t_ ErtsThrQ_t; + +typedef struct { + struct { + ErtsThrQLive_t queue; + ErtsThrQLive_t objects; + } live; + void *arg; + void (*notify)(void *); + int auto_finalize_dequeue; +} ErtsThrQInit_t; + +typedef struct ErtsThrQElement_t_ ErtsThrQElement_t; +typedef struct ErtsThrQElement_t ErtsThrQPrepEnQ_t; + +typedef union { + erts_atomic_t atmc; + ErtsThrQElement_t *ptr; +} ErtsThrQPtr_t; + +struct ErtsThrQElement_t_ { + ErtsThrQPtr_t next; + union { + erts_atomic_t atmc; + void *ptr; + } data; +}; + +typedef struct { + ErtsThrQElement_t *start; + ErtsThrQElement_t *end; +} ErtsThrQFinDeQ_t; + +typedef enum { + ERTS_THR_Q_CLEAN, +#ifdef ERTS_SMP + ERTS_THR_Q_NEED_THR_PRGR, +#endif + ERTS_THR_Q_DIRTY, +} ErtsThrQCleanState_t; + +#ifdef USE_THREADS + +typedef struct { + ErtsThrQElement_t marker; + erts_atomic_t last; + erts_atomic_t um_refc[2]; + erts_atomic32_t um_refc_ix; + ErtsThrQLive_t live; +#ifdef ERTS_SMP + erts_atomic32_t thr_prgr_clean_scheduled; +#endif + void *arg; + void (*notify)(void *); +} ErtsThrQTail_t; + +struct ErtsThrQ_t_ { + /* + * This structure needs to be cache line aligned for best + * performance. + */ + union { + /* Modified by threads enqueuing */ + ErtsThrQTail_t data; + char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsThrQTail_t))]; + } tail; + /* + * Everything below this point is *only* accessed by the + * thread dequeuing. + */ + struct { + ErtsThrQPtr_t head; + ErtsThrQLive_t live; + ErtsThrQElement_t *first; + ErtsThrQElement_t *unref_end; + int clean_reached_head_count; + struct { + int automatic; + ErtsThrQElement_t *start; + ErtsThrQElement_t *end; + } deq_fini; + struct { +#ifdef ERTS_SMP + ErtsThrPrgrVal thr_progress; + int thr_progress_reached; +#endif + int um_refc_ix; + ErtsThrQElement_t *unref_end; + } next; + int used_marker; + void *arg; + void (*notify)(void *); + } head; + struct { + int finalizing; + ErtsThrQLive_t live; + void *blk; + } q; +}; + +#else /* !USE_THREADS */ + +struct ErtsThrQ_t_ { + ErtsThrQInit_t init; + ErtsThrQElement_t *first; + ErtsThrQElement_t *last; + struct { + void *blk; + } q; +}; + +#endif + +void erts_thr_q_init(void); +void erts_thr_q_initialize(ErtsThrQ_t *, ErtsThrQInit_t *); +ErtsThrQCleanState_t erts_thr_q_finalize(ErtsThrQ_t *); +ErtsThrQ_t *erts_thr_q_create(ErtsThrQInit_t *); +ErtsThrQCleanState_t erts_thr_q_destroy(ErtsThrQ_t *); +ErtsThrQCleanState_t erts_thr_q_clean(ErtsThrQ_t *); +ErtsThrQCleanState_t erts_thr_q_inspect(ErtsThrQ_t *, int); +ErtsThrQPrepEnQ_t *erts_thr_q_prepare_enqueue(ErtsThrQ_t *); +void erts_thr_q_enqueue_prepared(ErtsThrQ_t *, void *, ErtsThrQPrepEnQ_t *); +void erts_thr_q_enqueue(ErtsThrQ_t *, void *); +void * erts_thr_q_dequeue(ErtsThrQ_t *); +int erts_thr_q_get_finalize_dequeue_data(ErtsThrQ_t *, + ErtsThrQFinDeQ_t *); +void erts_thr_q_append_finalize_dequeue_data(ErtsThrQFinDeQ_t *, + ErtsThrQFinDeQ_t *); +int erts_thr_q_finalize_dequeue(ErtsThrQFinDeQ_t *); +void erts_thr_q_finalize_dequeue_state_init(ErtsThrQFinDeQ_t *); + +#ifdef ERTS_SMP +ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_q_need_thr_progress(ErtsThrQ_t *q); +#endif + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +#ifdef ERTS_SMP +ERTS_GLB_INLINE ErtsThrPrgrVal +erts_thr_q_need_thr_progress(ErtsThrQ_t *q) +{ + return q->head.next.thr_progress; +} +#endif + +#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ + +#endif /* ERL_THR_QUEUE_H__ */ diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h index a0eda61ba5..065e7077c0 100644 --- a/erts/emulator/beam/erl_threads.h +++ b/erts/emulator/beam/erl_threads.h @@ -28,6 +28,11 @@ #define ERTS_SPIN_BODY ETHR_SPIN_BODY #include "sys.h" + +typedef struct { SWord sint[2]; } erts_no_dw_atomic_t; +typedef SWord erts_no_atomic_t; +typedef Sint32 erts_no_atomic32_t; + #ifdef USE_THREADS #define ETHR_TRY_INLINE_FUNCS @@ -87,6 +92,8 @@ typedef struct { #endif } erts_rwmtx_t; +#define ERTS_MTX_OPT_DEFAULT_INITER ETHR_MUTEX_OPT_DEFAULT_INITER +#define ERTS_CND_OPT_DEFAULT_INITER ETHR_COND_OPT_DEFAULT_INITER #define ERTS_RWMTX_OPT_DEFAULT_INITER ETHR_RWMUTEX_OPT_DEFAULT_INITER #define ERTS_RWMTX_TYPE_NORMAL ETHR_RWMUTEX_TYPE_NORMAL #define ERTS_RWMTX_TYPE_FREQUENT_READ ETHR_RWMUTEX_TYPE_FREQUENT_READ @@ -99,10 +106,12 @@ typedef ethr_rwmutex_opt erts_rwmtx_opt_t; typedef ethr_tsd_key erts_tsd_key_t; typedef ethr_ts_event erts_tse_t; -typedef ethr_sint_t erts_aint_t; -typedef ethr_atomic_t erts_atomic_t; -typedef ethr_sint32_t erts_aint32_t; -typedef ethr_atomic32_t erts_atomic32_t; +#define erts_dw_aint_t ethr_dw_sint_t +#define erts_dw_atomic_t ethr_dw_atomic_t +#define erts_aint_t ethr_sint_t +#define erts_atomic_t ethr_atomic_t +#define erts_aint32_t ethr_sint32_t +#define erts_atomic32_t ethr_atomic32_t /* spinlock */ typedef struct { @@ -164,10 +173,12 @@ typedef struct { typedef int erts_rwmtx_t; typedef int erts_tsd_key_t; typedef int erts_tse_t; -typedef SWord erts_aint_t; -typedef SWord erts_atomic_t; -typedef SWord erts_aint32_t; -typedef SWord erts_atomic32_t; +#define erts_dw_aint_t erts_no_dw_atomic_t +#define erts_dw_atomic_t erts_no_dw_atomic_t +#define erts_aint_t SWord +#define erts_atomic_t erts_no_atomic_t +#define erts_aint32_t Sint32 +#define erts_atomic32_t erts_no_atomic32_t #if __GNUC__ > 2 typedef struct { } erts_spinlock_t; typedef struct { } erts_rwlock_t; @@ -184,6 +195,8 @@ typedef struct { int gcc_is_buggy; } erts_rwlock_t; #endif /* #ifdef USE_THREADS */ +#define ERTS_AINT_NULL ((erts_aint_t) NULL) + #define ERTS_AINT_T_MAX (~(((erts_aint_t) 1) << (sizeof(erts_aint_t)*8-1))) #define ERTS_AINT_T_MIN ((((erts_aint_t) 1) << (sizeof(erts_aint_t)*8-1))) #define ERTS_AINT32_T_MAX (~(((erts_aint32_t) 1) << (sizeof(erts_aint32_t)*8-1))) @@ -247,65 +260,51 @@ ERTS_GLB_INLINE int erts_rwmtx_tryrwlock(erts_rwmtx_t *rwmtx); ERTS_GLB_INLINE void erts_rwmtx_rwunlock(erts_rwmtx_t *rwmtx); ERTS_GLB_INLINE int erts_lc_rwmtx_is_rlocked(erts_rwmtx_t *mtx); ERTS_GLB_INLINE int erts_lc_rwmtx_is_rwlocked(erts_rwmtx_t *mtx); -ERTS_GLB_INLINE void erts_atomic_init(erts_atomic_t *var, erts_aint_t i); -ERTS_GLB_INLINE void erts_atomic_set(erts_atomic_t *var, erts_aint_t i); -ERTS_GLB_INLINE erts_aint_t erts_atomic_read(erts_atomic_t *var); -ERTS_GLB_INLINE erts_aint_t erts_atomic_inctest(erts_atomic_t *incp); -ERTS_GLB_INLINE erts_aint_t erts_atomic_dectest(erts_atomic_t *decp); -ERTS_GLB_INLINE void erts_atomic_inc(erts_atomic_t *incp); -ERTS_GLB_INLINE void erts_atomic_dec(erts_atomic_t *decp); -ERTS_GLB_INLINE erts_aint_t erts_atomic_addtest(erts_atomic_t *addp, - erts_aint_t i); -ERTS_GLB_INLINE void erts_atomic_add(erts_atomic_t *addp, erts_aint_t i); -ERTS_GLB_INLINE erts_aint_t erts_atomic_xchg(erts_atomic_t *xchgp, - erts_aint_t new); -ERTS_GLB_INLINE erts_aint_t erts_atomic_cmpxchg(erts_atomic_t *xchgp, - erts_aint_t new, - erts_aint_t expected); -ERTS_GLB_INLINE erts_aint_t erts_atomic_bor(erts_atomic_t *var, - erts_aint_t mask); -ERTS_GLB_INLINE erts_aint_t erts_atomic_band(erts_atomic_t *var, - erts_aint_t mask); -ERTS_GLB_INLINE erts_aint_t erts_atomic_read_acqb(erts_atomic_t *var); -ERTS_GLB_INLINE void erts_atomic_set_relb(erts_atomic_t *var, erts_aint_t i); -ERTS_GLB_INLINE void erts_atomic_dec_relb(erts_atomic_t *decp); -ERTS_GLB_INLINE erts_aint_t erts_atomic_dectest_relb(erts_atomic_t *decp); -ERTS_GLB_INLINE erts_aint_t erts_atomic_cmpxchg_acqb(erts_atomic_t *xchgp, - erts_aint_t new, - erts_aint_t exp); -ERTS_GLB_INLINE erts_aint_t erts_atomic_cmpxchg_relb(erts_atomic_t *xchgp, - erts_aint_t new, - erts_aint_t exp); -ERTS_GLB_INLINE void erts_atomic32_init(erts_atomic32_t *var, erts_aint32_t i); -ERTS_GLB_INLINE void erts_atomic32_set(erts_atomic32_t *var, erts_aint32_t i); -ERTS_GLB_INLINE erts_aint32_t erts_atomic32_read(erts_atomic32_t *var); -ERTS_GLB_INLINE erts_aint32_t erts_atomic32_inctest(erts_atomic32_t *incp); -ERTS_GLB_INLINE erts_aint32_t erts_atomic32_dectest(erts_atomic32_t *decp); -ERTS_GLB_INLINE void erts_atomic32_inc(erts_atomic32_t *incp); -ERTS_GLB_INLINE void erts_atomic32_dec(erts_atomic32_t *decp); -ERTS_GLB_INLINE erts_aint32_t erts_atomic32_addtest(erts_atomic32_t *addp, - erts_aint32_t i); -ERTS_GLB_INLINE void erts_atomic32_add(erts_atomic32_t *addp, erts_aint32_t i); -ERTS_GLB_INLINE erts_aint32_t erts_atomic32_xchg(erts_atomic32_t *xchgp, - erts_aint32_t new); -ERTS_GLB_INLINE erts_aint32_t erts_atomic32_cmpxchg(erts_atomic32_t *xchgp, - erts_aint32_t new, - erts_aint32_t expected); -ERTS_GLB_INLINE erts_aint32_t erts_atomic32_bor(erts_atomic32_t *var, - erts_aint32_t mask); -ERTS_GLB_INLINE erts_aint32_t erts_atomic32_band(erts_atomic32_t *var, - erts_aint32_t mask); -ERTS_GLB_INLINE erts_aint32_t erts_atomic32_read_acqb(erts_atomic32_t *var); -ERTS_GLB_INLINE void erts_atomic32_set_relb(erts_atomic32_t *var, - erts_aint32_t i); -ERTS_GLB_INLINE void erts_atomic32_dec_relb(erts_atomic32_t *decp); -ERTS_GLB_INLINE erts_aint32_t erts_atomic32_dectest_relb(erts_atomic32_t *decp); -ERTS_GLB_INLINE erts_aint32_t erts_atomic32_cmpxchg_acqb(erts_atomic32_t *xchgp, - erts_aint32_t new, - erts_aint32_t exp); -ERTS_GLB_INLINE erts_aint32_t erts_atomic32_cmpxchg_relb(erts_atomic32_t *xchgp, - erts_aint32_t new, - erts_aint32_t exp); + +ERTS_GLB_INLINE void erts_no_dw_atomic_set(erts_no_dw_atomic_t *var, erts_no_dw_atomic_t *val); +ERTS_GLB_INLINE void erts_no_dw_atomic_read(erts_no_dw_atomic_t *var, erts_no_dw_atomic_t *val); +ERTS_GLB_INLINE int erts_no_dw_atomic_cmpxchg(erts_no_dw_atomic_t *var, + erts_no_dw_atomic_t *val, + erts_no_dw_atomic_t *old_val); +ERTS_GLB_INLINE void erts_no_atomic_set(erts_no_atomic_t *var, erts_aint_t i); +ERTS_GLB_INLINE erts_aint_t erts_no_atomic_read(erts_no_atomic_t *var); +ERTS_GLB_INLINE erts_aint_t erts_no_atomic_inc_read(erts_no_atomic_t *incp); +ERTS_GLB_INLINE erts_aint_t erts_no_atomic_dec_read(erts_no_atomic_t *decp); +ERTS_GLB_INLINE void erts_no_atomic_inc(erts_no_atomic_t *incp); +ERTS_GLB_INLINE void erts_no_atomic_dec(erts_no_atomic_t *decp); +ERTS_GLB_INLINE erts_aint_t erts_no_atomic_add_read(erts_no_atomic_t *addp, + erts_aint_t i); +ERTS_GLB_INLINE void erts_no_atomic_add(erts_no_atomic_t *addp, erts_aint_t i); +ERTS_GLB_INLINE erts_aint_t erts_no_atomic_read_bor(erts_no_atomic_t *var, + erts_aint_t mask); +ERTS_GLB_INLINE erts_aint_t erts_no_atomic_read_band(erts_no_atomic_t *var, + erts_aint_t mask); +ERTS_GLB_INLINE erts_aint_t erts_no_atomic_xchg(erts_no_atomic_t *xchgp, + erts_aint_t new); +ERTS_GLB_INLINE erts_aint_t erts_no_atomic_cmpxchg(erts_no_atomic_t *xchgp, + erts_aint_t new, + erts_aint_t expected); +ERTS_GLB_INLINE void erts_no_atomic32_set(erts_no_atomic32_t *var, + erts_aint32_t i); +ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_read(erts_no_atomic32_t *var); +ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_inc_read(erts_no_atomic32_t *incp); +ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_dec_read(erts_no_atomic32_t *decp); +ERTS_GLB_INLINE void erts_no_atomic32_inc(erts_no_atomic32_t *incp); +ERTS_GLB_INLINE void erts_no_atomic32_dec(erts_no_atomic32_t *decp); +ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_add_read(erts_no_atomic32_t *addp, + erts_aint32_t i); +ERTS_GLB_INLINE void erts_no_atomic32_add(erts_no_atomic32_t *addp, + erts_aint32_t i); +ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_read_bor(erts_no_atomic32_t *var, + erts_aint32_t mask); +ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_read_band(erts_no_atomic32_t *var, + erts_aint32_t mask); +ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_xchg(erts_no_atomic32_t *xchgp, + erts_aint32_t new); +ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_cmpxchg(erts_no_atomic32_t *xchgp, + erts_aint32_t new, + erts_aint32_t expected); + ERTS_GLB_INLINE void erts_spinlock_init_x_opt(erts_spinlock_t *lock, char *name, Eterm extra, @@ -362,6 +361,430 @@ ERTS_GLB_INLINE void erts_thr_sigmask(int how, const sigset_t *set, ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #endif /* #ifdef HAVE_ETHR_SIG_FUNCS */ +/* + * Functions implementing atomic operations with with no (nob), + * full (mb), acquire (acqb), release (relb), read (rb), and + * write (wb) memory barriers. + * + * If thread support has been disabled, they are mapped to + * functions that performs the same operation, but aren't atomic + * and don't imply memory barriers. + */ + +#ifdef USE_THREADS + +/* Double word size atomics */ + +#define erts_dw_atomic_init_nob ethr_dw_atomic_init +#define erts_dw_atomic_set_nob ethr_dw_atomic_set +#define erts_dw_atomic_read_nob ethr_dw_atomic_read +#define erts_dw_atomic_cmpxchg_nob ethr_dw_atomic_cmpxchg + +#define erts_dw_atomic_init_mb ethr_dw_atomic_init_mb +#define erts_dw_atomic_set_mb ethr_dw_atomic_set_mb +#define erts_dw_atomic_read_mb ethr_dw_atomic_read_mb +#define erts_dw_atomic_cmpxchg_mb ethr_dw_atomic_cmpxchg_mb + +#define erts_dw_atomic_init_acqb ethr_dw_atomic_init_acqb +#define erts_dw_atomic_set_acqb ethr_dw_atomic_set_acqb +#define erts_dw_atomic_read_acqb ethr_dw_atomic_read_acqb +#define erts_dw_atomic_cmpxchg_acqb ethr_dw_atomic_cmpxchg_acqb + +#define erts_dw_atomic_init_relb ethr_dw_atomic_init_relb +#define erts_dw_atomic_set_relb ethr_dw_atomic_set_relb +#define erts_dw_atomic_read_relb ethr_dw_atomic_read_relb +#define erts_dw_atomic_cmpxchg_relb ethr_dw_atomic_cmpxchg_relb + +#define erts_dw_atomic_init_rb ethr_dw_atomic_init_rb +#define erts_dw_atomic_set_rb ethr_dw_atomic_set_rb +#define erts_dw_atomic_read_rb ethr_dw_atomic_read_rb +#define erts_dw_atomic_cmpxchg_rb ethr_dw_atomic_cmpxchg_rb + +#define erts_dw_atomic_init_wb ethr_dw_atomic_init_wb +#define erts_dw_atomic_set_wb ethr_dw_atomic_set_wb +#define erts_dw_atomic_read_wb ethr_dw_atomic_read_wb +#define erts_dw_atomic_cmpxchg_wb ethr_dw_atomic_cmpxchg_wb + +/* Word size atomics */ + +#define erts_atomic_init_nob ethr_atomic_init +#define erts_atomic_set_nob ethr_atomic_set +#define erts_atomic_read_nob ethr_atomic_read +#define erts_atomic_inc_read_nob ethr_atomic_inc_read +#define erts_atomic_dec_read_nob ethr_atomic_dec_read +#define erts_atomic_inc_nob ethr_atomic_inc +#define erts_atomic_dec_nob ethr_atomic_dec +#define erts_atomic_add_read_nob ethr_atomic_add_read +#define erts_atomic_add_nob ethr_atomic_add +#define erts_atomic_read_bor_nob ethr_atomic_read_bor +#define erts_atomic_read_band_nob ethr_atomic_read_band +#define erts_atomic_xchg_nob ethr_atomic_xchg +#define erts_atomic_cmpxchg_nob ethr_atomic_cmpxchg + +#define erts_atomic_init_mb ethr_atomic_init_mb +#define erts_atomic_set_mb ethr_atomic_set_mb +#define erts_atomic_read_mb ethr_atomic_read_mb +#define erts_atomic_inc_read_mb ethr_atomic_inc_read_mb +#define erts_atomic_dec_read_mb ethr_atomic_dec_read_mb +#define erts_atomic_inc_mb ethr_atomic_inc_mb +#define erts_atomic_dec_mb ethr_atomic_dec_mb +#define erts_atomic_add_read_mb ethr_atomic_add_read_mb +#define erts_atomic_add_mb ethr_atomic_add_mb +#define erts_atomic_read_bor_mb ethr_atomic_read_bor_mb +#define erts_atomic_read_band_mb ethr_atomic_read_band_mb +#define erts_atomic_xchg_mb ethr_atomic_xchg_mb +#define erts_atomic_cmpxchg_mb ethr_atomic_cmpxchg_mb + +#define erts_atomic_init_acqb ethr_atomic_init_acqb +#define erts_atomic_set_acqb ethr_atomic_set_acqb +#define erts_atomic_read_acqb ethr_atomic_read_acqb +#define erts_atomic_inc_read_acqb ethr_atomic_inc_read_acqb +#define erts_atomic_dec_read_acqb ethr_atomic_dec_read_acqb +#define erts_atomic_inc_acqb ethr_atomic_inc_acqb +#define erts_atomic_dec_acqb ethr_atomic_dec_acqb +#define erts_atomic_add_read_acqb ethr_atomic_add_read_acqb +#define erts_atomic_add_acqb ethr_atomic_add_acqb +#define erts_atomic_read_bor_acqb ethr_atomic_read_bor_acqb +#define erts_atomic_read_band_acqb ethr_atomic_read_band_acqb +#define erts_atomic_xchg_acqb ethr_atomic_xchg_acqb +#define erts_atomic_cmpxchg_acqb ethr_atomic_cmpxchg_acqb + +#define erts_atomic_init_relb ethr_atomic_init_relb +#define erts_atomic_set_relb ethr_atomic_set_relb +#define erts_atomic_read_relb ethr_atomic_read_relb +#define erts_atomic_inc_read_relb ethr_atomic_inc_read_relb +#define erts_atomic_dec_read_relb ethr_atomic_dec_read_relb +#define erts_atomic_inc_relb ethr_atomic_inc_relb +#define erts_atomic_dec_relb ethr_atomic_dec_relb +#define erts_atomic_add_read_relb ethr_atomic_add_read_relb +#define erts_atomic_add_relb ethr_atomic_add_relb +#define erts_atomic_read_bor_relb ethr_atomic_read_bor_relb +#define erts_atomic_read_band_relb ethr_atomic_read_band_relb +#define erts_atomic_xchg_relb ethr_atomic_xchg_relb +#define erts_atomic_cmpxchg_relb ethr_atomic_cmpxchg_relb + +#define erts_atomic_init_rb ethr_atomic_init_rb +#define erts_atomic_set_rb ethr_atomic_set_rb +#define erts_atomic_read_rb ethr_atomic_read_rb +#define erts_atomic_inc_read_rb ethr_atomic_inc_read_rb +#define erts_atomic_dec_read_rb ethr_atomic_dec_read_rb +#define erts_atomic_inc_rb ethr_atomic_inc_rb +#define erts_atomic_dec_rb ethr_atomic_dec_rb +#define erts_atomic_add_read_rb ethr_atomic_add_read_rb +#define erts_atomic_add_rb ethr_atomic_add_rb +#define erts_atomic_read_bor_rb ethr_atomic_read_bor_rb +#define erts_atomic_read_band_rb ethr_atomic_read_band_rb +#define erts_atomic_xchg_rb ethr_atomic_xchg_rb +#define erts_atomic_cmpxchg_rb ethr_atomic_cmpxchg_rb + +#define erts_atomic_init_wb ethr_atomic_init_wb +#define erts_atomic_set_wb ethr_atomic_set_wb +#define erts_atomic_read_wb ethr_atomic_read_wb +#define erts_atomic_inc_read_wb ethr_atomic_inc_read_wb +#define erts_atomic_dec_read_wb ethr_atomic_dec_read_wb +#define erts_atomic_inc_wb ethr_atomic_inc_wb +#define erts_atomic_dec_wb ethr_atomic_dec_wb +#define erts_atomic_add_read_wb ethr_atomic_add_read_wb +#define erts_atomic_add_wb ethr_atomic_add_wb +#define erts_atomic_read_bor_wb ethr_atomic_read_bor_wb +#define erts_atomic_read_band_wb ethr_atomic_read_band_wb +#define erts_atomic_xchg_wb ethr_atomic_xchg_wb +#define erts_atomic_cmpxchg_wb ethr_atomic_cmpxchg_wb + +/* 32-bit atomics */ + +#define erts_atomic32_init_nob ethr_atomic32_init +#define erts_atomic32_set_nob ethr_atomic32_set +#define erts_atomic32_read_nob ethr_atomic32_read +#define erts_atomic32_inc_read_nob ethr_atomic32_inc_read +#define erts_atomic32_dec_read_nob ethr_atomic32_dec_read +#define erts_atomic32_inc_nob ethr_atomic32_inc +#define erts_atomic32_dec_nob ethr_atomic32_dec +#define erts_atomic32_add_read_nob ethr_atomic32_add_read +#define erts_atomic32_add_nob ethr_atomic32_add +#define erts_atomic32_read_bor_nob ethr_atomic32_read_bor +#define erts_atomic32_read_band_nob ethr_atomic32_read_band +#define erts_atomic32_xchg_nob ethr_atomic32_xchg +#define erts_atomic32_cmpxchg_nob ethr_atomic32_cmpxchg + +#define erts_atomic32_init_mb ethr_atomic32_init_mb +#define erts_atomic32_set_mb ethr_atomic32_set_mb +#define erts_atomic32_read_mb ethr_atomic32_read_mb +#define erts_atomic32_inc_read_mb ethr_atomic32_inc_read_mb +#define erts_atomic32_dec_read_mb ethr_atomic32_dec_read_mb +#define erts_atomic32_inc_mb ethr_atomic32_inc_mb +#define erts_atomic32_dec_mb ethr_atomic32_dec_mb +#define erts_atomic32_add_read_mb ethr_atomic32_add_read_mb +#define erts_atomic32_add_mb ethr_atomic32_add_mb +#define erts_atomic32_read_bor_mb ethr_atomic32_read_bor_mb +#define erts_atomic32_read_band_mb ethr_atomic32_read_band_mb +#define erts_atomic32_xchg_mb ethr_atomic32_xchg_mb +#define erts_atomic32_cmpxchg_mb ethr_atomic32_cmpxchg_mb + +#define erts_atomic32_init_acqb ethr_atomic32_init_acqb +#define erts_atomic32_set_acqb ethr_atomic32_set_acqb +#define erts_atomic32_read_acqb ethr_atomic32_read_acqb +#define erts_atomic32_inc_read_acqb ethr_atomic32_inc_read_acqb +#define erts_atomic32_dec_read_acqb ethr_atomic32_dec_read_acqb +#define erts_atomic32_inc_acqb ethr_atomic32_inc_acqb +#define erts_atomic32_dec_acqb ethr_atomic32_dec_acqb +#define erts_atomic32_add_read_acqb ethr_atomic32_add_read_acqb +#define erts_atomic32_add_acqb ethr_atomic32_add_acqb +#define erts_atomic32_read_bor_acqb ethr_atomic32_read_bor_acqb +#define erts_atomic32_read_band_acqb ethr_atomic32_read_band_acqb +#define erts_atomic32_xchg_acqb ethr_atomic32_xchg_acqb +#define erts_atomic32_cmpxchg_acqb ethr_atomic32_cmpxchg_acqb + +#define erts_atomic32_init_relb ethr_atomic32_init_relb +#define erts_atomic32_set_relb ethr_atomic32_set_relb +#define erts_atomic32_read_relb ethr_atomic32_read_relb +#define erts_atomic32_inc_read_relb ethr_atomic32_inc_read_relb +#define erts_atomic32_dec_read_relb ethr_atomic32_dec_read_relb +#define erts_atomic32_inc_relb ethr_atomic32_inc_relb +#define erts_atomic32_dec_relb ethr_atomic32_dec_relb +#define erts_atomic32_add_read_relb ethr_atomic32_add_read_relb +#define erts_atomic32_add_relb ethr_atomic32_add_relb +#define erts_atomic32_read_bor_relb ethr_atomic32_read_bor_relb +#define erts_atomic32_read_band_relb ethr_atomic32_read_band_relb +#define erts_atomic32_xchg_relb ethr_atomic32_xchg_relb +#define erts_atomic32_cmpxchg_relb ethr_atomic32_cmpxchg_relb + +#define erts_atomic32_init_rb ethr_atomic32_init_rb +#define erts_atomic32_set_rb ethr_atomic32_set_rb +#define erts_atomic32_read_rb ethr_atomic32_read_rb +#define erts_atomic32_inc_read_rb ethr_atomic32_inc_read_rb +#define erts_atomic32_dec_read_rb ethr_atomic32_dec_read_rb +#define erts_atomic32_inc_rb ethr_atomic32_inc_rb +#define erts_atomic32_dec_rb ethr_atomic32_dec_rb +#define erts_atomic32_add_read_rb ethr_atomic32_add_read_rb +#define erts_atomic32_add_rb ethr_atomic32_add_rb +#define erts_atomic32_read_bor_rb ethr_atomic32_read_bor_rb +#define erts_atomic32_read_band_rb ethr_atomic32_read_band_rb +#define erts_atomic32_xchg_rb ethr_atomic32_xchg_rb +#define erts_atomic32_cmpxchg_rb ethr_atomic32_cmpxchg_rb + +#define erts_atomic32_init_wb ethr_atomic32_init_wb +#define erts_atomic32_set_wb ethr_atomic32_set_wb +#define erts_atomic32_read_wb ethr_atomic32_read_wb +#define erts_atomic32_inc_read_wb ethr_atomic32_inc_read_wb +#define erts_atomic32_dec_read_wb ethr_atomic32_dec_read_wb +#define erts_atomic32_inc_wb ethr_atomic32_inc_wb +#define erts_atomic32_dec_wb ethr_atomic32_dec_wb +#define erts_atomic32_add_read_wb ethr_atomic32_add_read_wb +#define erts_atomic32_add_wb ethr_atomic32_add_wb +#define erts_atomic32_read_bor_wb ethr_atomic32_read_bor_wb +#define erts_atomic32_read_band_wb ethr_atomic32_read_band_wb +#define erts_atomic32_xchg_wb ethr_atomic32_xchg_wb +#define erts_atomic32_cmpxchg_wb ethr_atomic32_cmpxchg_wb + +#else /* !USE_THREADS */ + +/* Double word size atomics */ + +#define erts_dw_atomic_init_nob erts_no_dw_atomic_set +#define erts_dw_atomic_set_nob erts_no_dw_atomic_set +#define erts_dw_atomic_read_nob erts_no_dw_atomic_read +#define erts_dw_atomic_cmpxchg_nob erts_no_dw_atomic_cmpxchg + +#define erts_dw_atomic_init_mb erts_no_dw_atomic_init +#define erts_dw_atomic_set_mb erts_no_dw_atomic_set +#define erts_dw_atomic_read_mb erts_no_dw_atomic_read +#define erts_dw_atomic_cmpxchg_mb erts_no_dw_atomic_cmpxchg + +#define erts_dw_atomic_init_acqb erts_no_dw_atomic_init +#define erts_dw_atomic_set_acqb erts_no_dw_atomic_set +#define erts_dw_atomic_read_acqb erts_no_dw_atomic_read +#define erts_dw_atomic_cmpxchg_acqb erts_no_dw_atomic_cmpxchg + +#define erts_dw_atomic_init_relb erts_no_dw_atomic_init +#define erts_dw_atomic_set_relb erts_no_dw_atomic_set +#define erts_dw_atomic_read_relb erts_no_dw_atomic_read +#define erts_dw_atomic_cmpxchg_relb erts_no_dw_atomic_cmpxchg + +#define erts_dw_atomic_init_rb erts_no_dw_atomic_init +#define erts_dw_atomic_set_rb erts_no_dw_atomic_set +#define erts_dw_atomic_read_rb erts_no_dw_atomic_read +#define erts_dw_atomic_cmpxchg_rb erts_no_dw_atomic_cmpxchg + +#define erts_dw_atomic_init_wb erts_no_dw_atomic_init +#define erts_dw_atomic_set_wb erts_no_dw_atomic_set +#define erts_dw_atomic_read_wb erts_no_dw_atomic_read +#define erts_dw_atomic_cmpxchg_wb erts_no_dw_atomic_cmpxchg + +/* Word size atomics */ + +#define erts_atomic_init_nob erts_no_atomic_set +#define erts_atomic_set_nob erts_no_atomic_set +#define erts_atomic_read_nob erts_no_atomic_read +#define erts_atomic_inc_read_nob erts_no_atomic_inc_read +#define erts_atomic_dec_read_nob erts_no_atomic_dec_read +#define erts_atomic_inc_nob erts_no_atomic_inc +#define erts_atomic_dec_nob erts_no_atomic_dec +#define erts_atomic_add_read_nob erts_no_atomic_add_read +#define erts_atomic_add_nob erts_no_atomic_add +#define erts_atomic_read_bor_nob erts_no_atomic_read_bor +#define erts_atomic_read_band_nob erts_no_atomic_read_band +#define erts_atomic_xchg_nob erts_no_atomic_xchg +#define erts_atomic_cmpxchg_nob erts_no_atomic_cmpxchg + +#define erts_atomic_init_mb erts_no_atomic_set +#define erts_atomic_set_mb erts_no_atomic_set +#define erts_atomic_read_mb erts_no_atomic_read +#define erts_atomic_inc_read_mb erts_no_atomic_inc_read +#define erts_atomic_dec_read_mb erts_no_atomic_dec_read +#define erts_atomic_inc_mb erts_no_atomic_inc +#define erts_atomic_dec_mb erts_no_atomic_dec +#define erts_atomic_add_read_mb erts_no_atomic_add_read +#define erts_atomic_add_mb erts_no_atomic_add +#define erts_atomic_read_bor_mb erts_no_atomic_read_bor +#define erts_atomic_read_band_mb erts_no_atomic_read_band +#define erts_atomic_xchg_mb erts_no_atomic_xchg +#define erts_atomic_cmpxchg_mb erts_no_atomic_cmpxchg + +#define erts_atomic_init_acqb erts_no_atomic_set +#define erts_atomic_set_acqb erts_no_atomic_set +#define erts_atomic_read_acqb erts_no_atomic_read +#define erts_atomic_inc_read_acqb erts_no_atomic_inc_read +#define erts_atomic_dec_read_acqb erts_no_atomic_dec_read +#define erts_atomic_inc_acqb erts_no_atomic_inc +#define erts_atomic_dec_acqb erts_no_atomic_dec +#define erts_atomic_add_read_acqb erts_no_atomic_add_read +#define erts_atomic_add_acqb erts_no_atomic_add +#define erts_atomic_read_bor_acqb erts_no_atomic_read_bor +#define erts_atomic_read_band_acqb erts_no_atomic_read_band +#define erts_atomic_xchg_acqb erts_no_atomic_xchg +#define erts_atomic_cmpxchg_acqb erts_no_atomic_cmpxchg + +#define erts_atomic_init_relb erts_no_atomic_set +#define erts_atomic_set_relb erts_no_atomic_set +#define erts_atomic_read_relb erts_no_atomic_read +#define erts_atomic_inc_read_relb erts_no_atomic_inc_read +#define erts_atomic_dec_read_relb erts_no_atomic_dec_read +#define erts_atomic_inc_relb erts_no_atomic_inc +#define erts_atomic_dec_relb erts_no_atomic_dec +#define erts_atomic_add_read_relb erts_no_atomic_add_read +#define erts_atomic_add_relb erts_no_atomic_add +#define erts_atomic_read_bor_relb erts_no_atomic_read_bor +#define erts_atomic_read_band_relb erts_no_atomic_read_band +#define erts_atomic_xchg_relb erts_no_atomic_xchg +#define erts_atomic_cmpxchg_relb erts_no_atomic_cmpxchg + +#define erts_atomic_init_rb erts_no_atomic_set +#define erts_atomic_set_rb erts_no_atomic_set +#define erts_atomic_read_rb erts_no_atomic_read +#define erts_atomic_inc_read_rb erts_no_atomic_inc_read +#define erts_atomic_dec_read_rb erts_no_atomic_dec_read +#define erts_atomic_inc_rb erts_no_atomic_inc +#define erts_atomic_dec_rb erts_no_atomic_dec +#define erts_atomic_add_read_rb erts_no_atomic_add_read +#define erts_atomic_add_rb erts_no_atomic_add +#define erts_atomic_read_bor_rb erts_no_atomic_read_bor +#define erts_atomic_read_band_rb erts_no_atomic_read_band +#define erts_atomic_xchg_rb erts_no_atomic_xchg +#define erts_atomic_cmpxchg_rb erts_no_atomic_cmpxchg + +#define erts_atomic_init_wb erts_no_atomic_set +#define erts_atomic_set_wb erts_no_atomic_set +#define erts_atomic_read_wb erts_no_atomic_read +#define erts_atomic_inc_read_wb erts_no_atomic_inc_read +#define erts_atomic_dec_read_wb erts_no_atomic_dec_read +#define erts_atomic_inc_wb erts_no_atomic_inc +#define erts_atomic_dec_wb erts_no_atomic_dec +#define erts_atomic_add_read_wb erts_no_atomic_add_read +#define erts_atomic_add_wb erts_no_atomic_add +#define erts_atomic_read_bor_wb erts_no_atomic_read_bor +#define erts_atomic_read_band_wb erts_no_atomic_read_band +#define erts_atomic_xchg_wb erts_no_atomic_xchg +#define erts_atomic_cmpxchg_wb erts_no_atomic_cmpxchg + +/* 32-bit atomics */ + +#define erts_atomic32_init_nob erts_no_atomic32_set +#define erts_atomic32_set_nob erts_no_atomic32_set +#define erts_atomic32_read_nob erts_no_atomic32_read +#define erts_atomic32_inc_read_nob erts_no_atomic32_inc_read +#define erts_atomic32_dec_read_nob erts_no_atomic32_dec_read +#define erts_atomic32_inc_nob erts_no_atomic32_inc +#define erts_atomic32_dec_nob erts_no_atomic32_dec +#define erts_atomic32_add_read_nob erts_no_atomic32_add_read +#define erts_atomic32_add_nob erts_no_atomic32_add +#define erts_atomic32_read_bor_nob erts_no_atomic32_read_bor +#define erts_atomic32_read_band_nob erts_no_atomic32_read_band +#define erts_atomic32_xchg_nob erts_no_atomic32_xchg +#define erts_atomic32_cmpxchg_nob erts_no_atomic32_cmpxchg + +#define erts_atomic32_init_mb erts_no_atomic32_set +#define erts_atomic32_set_mb erts_no_atomic32_set +#define erts_atomic32_read_mb erts_no_atomic32_read +#define erts_atomic32_inc_read_mb erts_no_atomic32_inc_read +#define erts_atomic32_dec_read_mb erts_no_atomic32_dec_read +#define erts_atomic32_inc_mb erts_no_atomic32_inc +#define erts_atomic32_dec_mb erts_no_atomic32_dec +#define erts_atomic32_add_read_mb erts_no_atomic32_add_read +#define erts_atomic32_add_mb erts_no_atomic32_add +#define erts_atomic32_read_bor_mb erts_no_atomic32_read_bor +#define erts_atomic32_read_band_mb erts_no_atomic32_read_band +#define erts_atomic32_xchg_mb erts_no_atomic32_xchg +#define erts_atomic32_cmpxchg_mb erts_no_atomic32_cmpxchg + +#define erts_atomic32_init_acqb erts_no_atomic32_set +#define erts_atomic32_set_acqb erts_no_atomic32_set +#define erts_atomic32_read_acqb erts_no_atomic32_read +#define erts_atomic32_inc_read_acqb erts_no_atomic32_inc_read +#define erts_atomic32_dec_read_acqb erts_no_atomic32_dec_read +#define erts_atomic32_inc_acqb erts_no_atomic32_inc +#define erts_atomic32_dec_acqb erts_no_atomic32_dec +#define erts_atomic32_add_read_acqb erts_no_atomic32_add_read +#define erts_atomic32_add_acqb erts_no_atomic32_add +#define erts_atomic32_read_bor_acqb erts_no_atomic32_read_bor +#define erts_atomic32_read_band_acqb erts_no_atomic32_read_band +#define erts_atomic32_xchg_acqb erts_no_atomic32_xchg +#define erts_atomic32_cmpxchg_acqb erts_no_atomic32_cmpxchg + +#define erts_atomic32_init_relb erts_no_atomic32_set +#define erts_atomic32_set_relb erts_no_atomic32_set +#define erts_atomic32_read_relb erts_no_atomic32_read +#define erts_atomic32_inc_read_relb erts_no_atomic32_inc_read +#define erts_atomic32_dec_read_relb erts_no_atomic32_dec_read +#define erts_atomic32_inc_relb erts_no_atomic32_inc +#define erts_atomic32_dec_relb erts_no_atomic32_dec +#define erts_atomic32_add_read_relb erts_no_atomic32_add_read +#define erts_atomic32_add_relb erts_no_atomic32_add +#define erts_atomic32_read_bor_relb erts_no_atomic32_read_bor +#define erts_atomic32_read_band_relb erts_no_atomic32_read_band +#define erts_atomic32_xchg_relb erts_no_atomic32_xchg +#define erts_atomic32_cmpxchg_relb erts_no_atomic32_cmpxchg + +#define erts_atomic32_init_rb erts_no_atomic32_set +#define erts_atomic32_set_rb erts_no_atomic32_set +#define erts_atomic32_read_rb erts_no_atomic32_read +#define erts_atomic32_inc_read_rb erts_no_atomic32_inc_read +#define erts_atomic32_dec_read_rb erts_no_atomic32_dec_read +#define erts_atomic32_inc_rb erts_no_atomic32_inc +#define erts_atomic32_dec_rb erts_no_atomic32_dec +#define erts_atomic32_add_read_rb erts_no_atomic32_add_read +#define erts_atomic32_add_rb erts_no_atomic32_add +#define erts_atomic32_read_bor_rb erts_no_atomic32_read_bor +#define erts_atomic32_read_band_rb erts_no_atomic32_read_band +#define erts_atomic32_xchg_rb erts_no_atomic32_xchg +#define erts_atomic32_cmpxchg_rb erts_no_atomic32_cmpxchg + +#define erts_atomic32_init_wb erts_no_atomic32_set +#define erts_atomic32_set_wb erts_no_atomic32_set +#define erts_atomic32_read_wb erts_no_atomic32_read +#define erts_atomic32_inc_read_wb erts_no_atomic32_inc_read +#define erts_atomic32_dec_read_wb erts_no_atomic32_dec_read +#define erts_atomic32_inc_wb erts_no_atomic32_inc +#define erts_atomic32_dec_wb erts_no_atomic32_dec +#define erts_atomic32_add_read_wb erts_no_atomic32_add_read +#define erts_atomic32_add_wb erts_no_atomic32_add +#define erts_atomic32_read_bor_wb erts_no_atomic32_read_bor +#define erts_atomic32_read_band_wb erts_no_atomic32_read_band +#define erts_atomic32_xchg_wb erts_no_atomic32_xchg +#define erts_atomic32_cmpxchg_wb erts_no_atomic32_cmpxchg + +#endif /* !USE_THREADS */ + #if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_INLINE void @@ -709,6 +1132,16 @@ erts_cnd_wait(erts_cnd_t *cnd, erts_mtx_t *mtx) #endif } +/* + * IMPORTANT note about erts_cnd_signal() and erts_cnd_broadcast() + * + * POSIX allow a call to `pthread_cond_signal' or `pthread_cond_broadcast' + * even though the associated mutex/mutexes isn't/aren't locked by the + * caller. Our implementation do not allow that in order to avoid a + * performance penalty. That is, all associated mutexes *need* to be + * locked by the caller of erts_cnd_signal()/erts_cnd_broadcast()! + */ + ERTS_GLB_INLINE void erts_cnd_signal(erts_cnd_t *cnd) { @@ -998,428 +1431,206 @@ erts_lc_rwmtx_is_rwlocked(erts_rwmtx_t *mtx) #endif } +/* No atomic ops */ + ERTS_GLB_INLINE void -erts_atomic_init(erts_atomic_t *var, erts_aint_t i) +erts_no_dw_atomic_set(erts_no_dw_atomic_t *var, erts_no_dw_atomic_t *val) { -#ifdef USE_THREADS - ethr_atomic_init(var, i); -#else - *var = i; -#endif + var->sint[0] = val->sint[0]; + var->sint[1] = val->sint[1]; } ERTS_GLB_INLINE void -erts_atomic_set(erts_atomic_t *var, erts_aint_t i) +erts_no_dw_atomic_read(erts_no_dw_atomic_t *var, erts_no_dw_atomic_t *val) +{ + val->sint[0] = var->sint[0]; + val->sint[1] = var->sint[1]; +} + +ERTS_GLB_INLINE int erts_no_dw_atomic_cmpxchg(erts_no_dw_atomic_t *var, + erts_no_dw_atomic_t *new_val, + erts_no_dw_atomic_t *old_val) +{ + if (var->sint[0] != old_val->sint[0] || var->sint[1] != old_val->sint[1]) { + erts_no_dw_atomic_read(var, old_val); + return 0; + } + else { + erts_no_dw_atomic_set(var, new_val); + return !0; + } +} + +ERTS_GLB_INLINE void +erts_no_atomic_set(erts_no_atomic_t *var, erts_aint_t i) { -#ifdef USE_THREADS - ethr_atomic_set(var, i); -#else *var = i; -#endif } ERTS_GLB_INLINE erts_aint_t -erts_atomic_read(erts_atomic_t *var) +erts_no_atomic_read(erts_no_atomic_t *var) { -#ifdef USE_THREADS - return ethr_atomic_read(var); -#else return *var; -#endif } ERTS_GLB_INLINE erts_aint_t -erts_atomic_inctest(erts_atomic_t *incp) +erts_no_atomic_inc_read(erts_no_atomic_t *incp) { -#ifdef USE_THREADS - return ethr_atomic_inc_read(incp); -#else return ++(*incp); -#endif } ERTS_GLB_INLINE erts_aint_t -erts_atomic_dectest(erts_atomic_t *decp) +erts_no_atomic_dec_read(erts_no_atomic_t *decp) { -#ifdef USE_THREADS - return ethr_atomic_dec_read(decp); -#else return --(*decp); -#endif } ERTS_GLB_INLINE void -erts_atomic_inc(erts_atomic_t *incp) +erts_no_atomic_inc(erts_no_atomic_t *incp) { -#ifdef USE_THREADS - ethr_atomic_inc(incp); -#else ++(*incp); -#endif } ERTS_GLB_INLINE void -erts_atomic_dec(erts_atomic_t *decp) +erts_no_atomic_dec(erts_no_atomic_t *decp) { -#ifdef USE_THREADS - ethr_atomic_dec(decp); -#else --(*decp); -#endif } ERTS_GLB_INLINE erts_aint_t -erts_atomic_addtest(erts_atomic_t *addp, erts_aint_t i) +erts_no_atomic_add_read(erts_no_atomic_t *addp, erts_aint_t i) { -#ifdef USE_THREADS - return ethr_atomic_add_read(addp, i); -#else return *addp += i; -#endif } ERTS_GLB_INLINE void -erts_atomic_add(erts_atomic_t *addp, erts_aint_t i) +erts_no_atomic_add(erts_no_atomic_t *addp, erts_aint_t i) { -#ifdef USE_THREADS - ethr_atomic_add(addp, i); -#else *addp += i; -#endif } ERTS_GLB_INLINE erts_aint_t -erts_atomic_xchg(erts_atomic_t *xchgp, erts_aint_t new) +erts_no_atomic_read_bor(erts_no_atomic_t *var, erts_aint_t mask) { -#ifdef USE_THREADS - return ethr_atomic_xchg(xchgp, new); -#else - erts_aint_t old = *xchgp; - *xchgp = new; - return old; -#endif -} - -ERTS_GLB_INLINE erts_aint_t -erts_atomic_cmpxchg(erts_atomic_t *xchgp, erts_aint_t new, erts_aint_t expected) -{ -#ifdef USE_THREADS - return ethr_atomic_cmpxchg(xchgp, new, expected); -#else - erts_aint_t old = *xchgp; - if (old == expected) - *xchgp = new; - return old; -#endif -} - -ERTS_GLB_INLINE erts_aint_t -erts_atomic_bor(erts_atomic_t *var, erts_aint_t mask) -{ -#ifdef USE_THREADS - return ethr_atomic_read_bor(var, mask); -#else erts_aint_t old; old = *var; *var |= mask; return old; -#endif } ERTS_GLB_INLINE erts_aint_t -erts_atomic_band(erts_atomic_t *var, erts_aint_t mask) +erts_no_atomic_read_band(erts_no_atomic_t *var, erts_aint_t mask) { -#ifdef USE_THREADS - return ethr_atomic_read_band(var, mask); -#else erts_aint_t old; old = *var; *var &= mask; return old; -#endif -} - -ERTS_GLB_INLINE erts_aint_t -erts_atomic_read_acqb(erts_atomic_t *var) -{ -#ifdef USE_THREADS - return ethr_atomic_read_acqb(var); -#else - return *var; -#endif -} - -ERTS_GLB_INLINE void -erts_atomic_set_relb(erts_atomic_t *var, erts_aint_t i) -{ -#ifdef USE_THREADS - ethr_atomic_set_relb(var, i); -#else - *var = i; -#endif -} - -ERTS_GLB_INLINE void -erts_atomic_dec_relb(erts_atomic_t *decp) -{ -#ifdef USE_THREADS - ethr_atomic_dec_relb(decp); -#else - --(*decp); -#endif } ERTS_GLB_INLINE erts_aint_t -erts_atomic_dectest_relb(erts_atomic_t *decp) -{ -#ifdef USE_THREADS - return ethr_atomic_dec_read_relb(decp); -#else - return --(*decp); -#endif -} - -ERTS_GLB_INLINE erts_aint_t erts_atomic_cmpxchg_acqb(erts_atomic_t *xchgp, - erts_aint_t new, - erts_aint_t exp) +erts_no_atomic_xchg(erts_no_atomic_t *xchgp, erts_aint_t new) { -#ifdef USE_THREADS - return ethr_atomic_cmpxchg_acqb(xchgp, new, exp); -#else erts_aint_t old = *xchgp; - if (old == exp) - *xchgp = new; + *xchgp = new; return old; -#endif } -ERTS_GLB_INLINE erts_aint_t erts_atomic_cmpxchg_relb(erts_atomic_t *xchgp, - erts_aint_t new, - erts_aint_t exp) +ERTS_GLB_INLINE erts_aint_t +erts_no_atomic_cmpxchg(erts_no_atomic_t *xchgp, + erts_aint_t new, + erts_aint_t expected) { -#ifdef USE_THREADS - return ethr_atomic_cmpxchg_relb(xchgp, new, exp); -#else erts_aint_t old = *xchgp; - if (old == exp) + if (old == expected) *xchgp = new; return old; -#endif } /* atomic32 */ ERTS_GLB_INLINE void -erts_atomic32_init(erts_atomic32_t *var, erts_aint32_t i) -{ -#ifdef USE_THREADS - ethr_atomic32_init(var, i); -#else - *var = i; -#endif -} - -ERTS_GLB_INLINE void -erts_atomic32_set(erts_atomic32_t *var, erts_aint32_t i) +erts_no_atomic32_set(erts_no_atomic32_t *var, erts_aint32_t i) { -#ifdef USE_THREADS - ethr_atomic32_set(var, i); -#else *var = i; -#endif } ERTS_GLB_INLINE erts_aint32_t -erts_atomic32_read(erts_atomic32_t *var) +erts_no_atomic32_read(erts_no_atomic32_t *var) { -#ifdef USE_THREADS - return ethr_atomic32_read(var); -#else return *var; -#endif } ERTS_GLB_INLINE erts_aint32_t -erts_atomic32_inctest(erts_atomic32_t *incp) +erts_no_atomic32_inc_read(erts_no_atomic32_t *incp) { -#ifdef USE_THREADS - return ethr_atomic32_inc_read(incp); -#else return ++(*incp); -#endif } ERTS_GLB_INLINE erts_aint32_t -erts_atomic32_dectest(erts_atomic32_t *decp) +erts_no_atomic32_dec_read(erts_no_atomic32_t *decp) { -#ifdef USE_THREADS - return ethr_atomic32_dec_read(decp); -#else return --(*decp); -#endif } ERTS_GLB_INLINE void -erts_atomic32_inc(erts_atomic32_t *incp) +erts_no_atomic32_inc(erts_no_atomic32_t *incp) { -#ifdef USE_THREADS - ethr_atomic32_inc(incp); -#else ++(*incp); -#endif } ERTS_GLB_INLINE void -erts_atomic32_dec(erts_atomic32_t *decp) +erts_no_atomic32_dec(erts_no_atomic32_t *decp) { -#ifdef USE_THREADS - ethr_atomic32_dec(decp); -#else --(*decp); -#endif } ERTS_GLB_INLINE erts_aint32_t -erts_atomic32_addtest(erts_atomic32_t *addp, erts_aint32_t i) +erts_no_atomic32_add_read(erts_no_atomic32_t *addp, erts_aint32_t i) { -#ifdef USE_THREADS - return ethr_atomic32_add_read(addp, i); -#else return *addp += i; -#endif } ERTS_GLB_INLINE void -erts_atomic32_add(erts_atomic32_t *addp, erts_aint32_t i) +erts_no_atomic32_add(erts_no_atomic32_t *addp, erts_aint32_t i) { -#ifdef USE_THREADS - ethr_atomic32_add(addp, i); -#else *addp += i; -#endif -} - -ERTS_GLB_INLINE erts_aint32_t -erts_atomic32_xchg(erts_atomic32_t *xchgp, erts_aint32_t new) -{ -#ifdef USE_THREADS - return ethr_atomic32_xchg(xchgp, new); -#else - erts_aint32_t old = *xchgp; - *xchgp = new; - return old; -#endif -} - -ERTS_GLB_INLINE erts_aint32_t -erts_atomic32_cmpxchg(erts_atomic32_t *xchgp, - erts_aint32_t new, - erts_aint32_t expected) -{ -#ifdef USE_THREADS - return ethr_atomic32_cmpxchg(xchgp, new, expected); -#else - erts_aint32_t old = *xchgp; - if (old == expected) - *xchgp = new; - return old; -#endif } ERTS_GLB_INLINE erts_aint32_t -erts_atomic32_bor(erts_atomic32_t *var, erts_aint32_t mask) +erts_no_atomic32_read_bor(erts_no_atomic32_t *var, erts_aint32_t mask) { -#ifdef USE_THREADS - return ethr_atomic32_read_bor(var, mask); -#else erts_aint32_t old; old = *var; *var |= mask; return old; -#endif } ERTS_GLB_INLINE erts_aint32_t -erts_atomic32_band(erts_atomic32_t *var, erts_aint32_t mask) +erts_no_atomic32_read_band(erts_no_atomic32_t *var, erts_aint32_t mask) { -#ifdef USE_THREADS - return ethr_atomic32_read_band(var, mask); -#else erts_aint32_t old; old = *var; *var &= mask; return old; -#endif -} - -ERTS_GLB_INLINE erts_aint32_t -erts_atomic32_read_acqb(erts_atomic32_t *var) -{ -#ifdef USE_THREADS - return ethr_atomic32_read_acqb(var); -#else - return *var; -#endif -} - -ERTS_GLB_INLINE void -erts_atomic32_set_relb(erts_atomic32_t *var, erts_aint32_t i) -{ -#ifdef USE_THREADS - ethr_atomic32_set_relb(var, i); -#else - *var = i; -#endif -} - -ERTS_GLB_INLINE void -erts_atomic32_dec_relb(erts_atomic32_t *decp) -{ -#ifdef USE_THREADS - ethr_atomic32_dec_relb(decp); -#else - --(*decp); -#endif -} - -ERTS_GLB_INLINE erts_aint32_t -erts_atomic32_dectest_relb(erts_atomic32_t *decp) -{ -#ifdef USE_THREADS - return ethr_atomic32_dec_read_relb(decp); -#else - return --(*decp); -#endif } ERTS_GLB_INLINE erts_aint32_t -erts_atomic32_cmpxchg_acqb(erts_atomic32_t *xchgp, - erts_aint32_t new, - erts_aint32_t exp) +erts_no_atomic32_xchg(erts_no_atomic32_t *xchgp, erts_aint32_t new) { -#ifdef USE_THREADS - return ethr_atomic32_cmpxchg_acqb(xchgp, new, exp); -#else erts_aint32_t old = *xchgp; - if (old == exp) - *xchgp = new; + *xchgp = new; return old; -#endif } ERTS_GLB_INLINE erts_aint32_t -erts_atomic32_cmpxchg_relb(erts_atomic32_t *xchgp, - erts_aint32_t new, - erts_aint32_t exp) +erts_no_atomic32_cmpxchg(erts_no_atomic32_t *xchgp, + erts_aint32_t new, + erts_aint32_t expected) { -#ifdef USE_THREADS - return ethr_atomic32_cmpxchg_relb(xchgp, new, exp); -#else erts_aint32_t old = *xchgp; - if (old == exp) + if (old == expected) *xchgp = new; return old; -#endif } /* spinlock */ @@ -1892,3 +2103,37 @@ erts_thr_sigwait(const sigset_t *set, int *sig) #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ #endif /* #ifndef ERL_THREAD_H__ */ + +#ifdef ERTS_UNDEF_DEPRECATED_ATOMICS + +/* Deprecated functions to replace */ + +#undef erts_atomic_init +#undef erts_atomic_set +#undef erts_atomic_read +#undef erts_atomic_inctest +#undef erts_atomic_dectest +#undef erts_atomic_inc +#undef erts_atomic_dec +#undef erts_atomic_addtest +#undef erts_atomic_add +#undef erts_atomic_xchg +#undef erts_atomic_cmpxchg +#undef erts_atomic_bor +#undef erts_atomic_band + +#undef erts_atomic32_init +#undef erts_atomic32_set +#undef erts_atomic32_read +#undef erts_atomic32_inctest +#undef erts_atomic32_dectest +#undef erts_atomic32_inc +#undef erts_atomic32_dec +#undef erts_atomic32_addtest +#undef erts_atomic32_add +#undef erts_atomic32_xchg +#undef erts_atomic32_cmpxchg +#undef erts_atomic32_bor +#undef erts_atomic32_band + +#endif diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h index d0ad73cd81..7a09d30ff6 100644 --- a/erts/emulator/beam/erl_time.h +++ b/erts/emulator/beam/erl_time.h @@ -85,8 +85,8 @@ ERTS_GLB_INLINE void erts_do_time_add(long); #if ERTS_GLB_INLINE_INCL_FUNC_DEF -ERTS_GLB_INLINE erts_aint_t erts_do_time_read_and_reset(void) { return erts_smp_atomic_xchg(&do_time, 0L); } -ERTS_GLB_INLINE void erts_do_time_add(long elapsed) { erts_smp_atomic_add(&do_time, elapsed); } +ERTS_GLB_INLINE erts_aint_t erts_do_time_read_and_reset(void) { return erts_smp_atomic_xchg_acqb(&do_time, 0L); } +ERTS_GLB_INLINE void erts_do_time_add(long elapsed) { erts_smp_atomic_add_relb(&do_time, elapsed); } #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index 8833137112..b487dbf054 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -36,6 +36,7 @@ #include "error.h" #include "erl_binary.h" #include "erl_bits.h" +#include "erl_thr_progress.h" #if 0 #define DEBUG_PRINTOUTS @@ -159,7 +160,7 @@ static Uint active_sched; void erts_system_profile_setup_active_schedulers(void) { - ERTS_SMP_LC_ASSERT(erts_is_system_blocked(0)); + ERTS_SMP_LC_ASSERT(erts_thr_progress_is_blocking()); active_sched = erts_active_schedulers(); } @@ -1940,7 +1941,8 @@ trace_proc(Process *c_p, Process *t_p, Eterm what, Eterm data) Eterm* hp; int need; - ERTS_SMP_LC_ASSERT((erts_proc_lc_my_proc_locks(t_p) != 0) || erts_is_system_blocked(0)); + ERTS_SMP_LC_ASSERT((erts_proc_lc_my_proc_locks(t_p) != 0) + || erts_thr_progress_is_blocking()); if (is_internal_port(t_p->tracer_proc)) { #define LOCAL_HEAP_SIZE (5+5) DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); @@ -2092,8 +2094,7 @@ void save_calls(Process *p, Export *e) * entries instead of the original BIF functions. */ Eterm -erts_bif_trace(int bif_index, Process* p, - Eterm arg1, Eterm arg2, Eterm arg3, BeamInstr *I) +erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) { Eterm result; int meta = !!(erts_bif_trace_flags[bif_index] & BIF_TRACE_AS_META); @@ -2107,10 +2108,10 @@ erts_bif_trace(int bif_index, Process* p, * no tracing will occur. Doing the whole else branch will * also do nothing, only slower. */ - Eterm (*func)(Process*, Eterm, Eterm, Eterm, BeamInstr*) = bif_table[bif_index].f; - result = func(p, arg1, arg2, arg3, I); + Eterm (*func)(Process*, Eterm*, BeamInstr*) = bif_table[bif_index].f; + result = func(p, args, I); } else { - Eterm (*func)(Process*, Eterm, Eterm, Eterm, BeamInstr*); + Eterm (*func)(Process*, Eterm*, BeamInstr*); Export* ep = bif_export[bif_index]; Uint32 flags = 0, flags_meta = 0; int global = !!(erts_bif_trace_flags[bif_index] & BIF_TRACE_AS_GLOBAL); @@ -2122,8 +2123,6 @@ erts_bif_trace(int bif_index, Process* p, * export entry */ BeamInstr *cp = p->cp; - Eterm args[3] = {arg1, arg2, arg3}; - /* * Make continuation pointer OK, it is not during direct BIF calls, * but it is correct during apply of bif. @@ -2155,7 +2154,7 @@ erts_bif_trace(int bif_index, Process* p, func = bif_table[bif_index].f; - result = func(p, arg1, arg2, arg3, I); + result = func(p, args, I); if (applying && (flags & MATCH_SET_RETURN_TO_TRACE)) { BeamInstr i_return_trace = beam_return_trace[0]; @@ -2745,7 +2744,8 @@ trace_port(Port *t_p, Eterm what, Eterm data) { Eterm mess; Eterm* hp; - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(t_p) || erts_is_system_blocked(0)); + ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(t_p) + || erts_thr_progress_is_blocking()); if (is_internal_port(t_p->tracer_proc)) { #define LOCAL_HEAP_SIZE (5+5) @@ -3021,8 +3021,6 @@ static ErtsSysMsgQ *sys_message_queue_end; static erts_tid_t sys_msg_dispatcher_tid; static erts_cnd_t smq_cnd; -static int dispatcher_waiting; - ERTS_QUALLOC_IMPL(smq_element, ErtsSysMsgQ, 20, ERTS_ALC_T_SYS_MSG_Q) static void @@ -3066,18 +3064,6 @@ enqueue_sys_msg(enum ErtsSysMsgType type, erts_smp_mtx_unlock(&smq_mtx); } -static void -prepare_for_block(void *unused) -{ - erts_smp_mtx_unlock(&smq_mtx); -} - -static void -resume_after_block(void *unused) -{ - erts_smp_mtx_lock(&smq_mtx); -} - void erts_queue_error_logger_message(Eterm from, Eterm msg, ErlHeapFragment *bp) { @@ -3143,10 +3129,10 @@ sys_msg_disp_failure(ErtsSysMsgQ *smqp, Eterm receiver) && !erts_system_monitor_flags.busy_port && !erts_system_monitor_flags.busy_dist_port) break; /* Everything is disabled */ - erts_smp_block_system(ERTS_BS_FLG_ALLOW_GC); + erts_smp_thr_progress_block(); if (system_monitor == receiver || receiver == NIL) erts_system_monitor_clear(NULL); - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); break; case SYS_MSG_TYPE_SYSPROF: if (receiver == NIL @@ -3156,11 +3142,11 @@ sys_msg_disp_failure(ErtsSysMsgQ *smqp, Eterm receiver) && !erts_system_profile_flags.scheduler) break; /* Block system to clear flags */ - erts_smp_block_system(0); + erts_smp_thr_progress_block(); if (system_profile == receiver || receiver == NIL) { erts_system_profile_clear(NULL); } - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); break; case SYS_MSG_TYPE_ERRLGR: { char *no_elgger = "(no error logger present)"; @@ -3201,22 +3187,68 @@ sys_msg_disp_failure(ErtsSysMsgQ *smqp, Eterm receiver) } } +static void +sys_msg_dispatcher_wakeup(void *vwait_p) +{ + int *wait_p = (int *) vwait_p; + erts_smp_mtx_lock(&smq_mtx); + *wait_p = 0; + erts_smp_cnd_signal(&smq_cnd); + erts_smp_mtx_unlock(&smq_mtx); +} + +static void +sys_msg_dispatcher_prep_wait(void *vwait_p) +{ + int *wait_p = (int *) vwait_p; + erts_smp_mtx_lock(&smq_mtx); + *wait_p = 1; + erts_smp_mtx_unlock(&smq_mtx); +} + +static void +sys_msg_dispatcher_fin_wait(void *vwait_p) +{ + int *wait_p = (int *) vwait_p; + erts_smp_mtx_lock(&smq_mtx); + *wait_p = 0; + erts_smp_mtx_unlock(&smq_mtx); +} + +static void +sys_msg_dispatcher_wait(void *vwait_p) +{ + int *wait_p = (int *) vwait_p; + erts_smp_mtx_lock(&smq_mtx); + while (*wait_p) + erts_smp_cnd_wait(&smq_cnd, &smq_mtx); + erts_smp_mtx_unlock(&smq_mtx); +} + static void * sys_msg_dispatcher_func(void *unused) { + ErtsThrPrgrCallbacks callbacks; ErtsSysMsgQ *local_sys_message_queue = NULL; + int wait = 0; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_set_thread_name("system message dispatcher"); #endif - erts_register_blockable_thread(); - erts_smp_activity_begin(ERTS_ACTIVITY_IO, NULL, NULL, NULL); + callbacks.arg = (void *) &wait; + callbacks.wakeup = sys_msg_dispatcher_wakeup; + callbacks.prepare_wait = sys_msg_dispatcher_prep_wait; + callbacks.wait = sys_msg_dispatcher_wait; + callbacks.finalize_wait = sys_msg_dispatcher_fin_wait; + + erts_thr_progress_register_managed_thread(NULL, &callbacks, 0); while (1) { + int end_wait = 0; ErtsSysMsgQ *smqp; - ERTS_SMP_LC_ASSERT(!ERTS_LC_IS_BLOCKING); + ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking()); erts_smp_mtx_lock(&smq_mtx); @@ -3228,20 +3260,16 @@ sys_msg_dispatcher_func(void *unused) } /* Fetch current trace message queue ... */ - erts_smp_activity_change(ERTS_ACTIVITY_IO, - ERTS_ACTIVITY_WAIT, - prepare_for_block, - resume_after_block, - NULL); - dispatcher_waiting = 1; + if (!sys_message_queue) { + erts_smp_mtx_unlock(&smq_mtx); + end_wait = 1; + erts_thr_progress_active(NULL, 0); + erts_thr_progress_prepare_wait(NULL); + erts_smp_mtx_lock(&smq_mtx); + } + while (!sys_message_queue) erts_smp_cnd_wait(&smq_cnd, &smq_mtx); - dispatcher_waiting = 0; - erts_smp_activity_change(ERTS_ACTIVITY_WAIT, - ERTS_ACTIVITY_IO, - prepare_for_block, - resume_after_block, - NULL); local_sys_message_queue = sys_message_queue; sys_message_queue = NULL; @@ -3249,6 +3277,11 @@ sys_msg_dispatcher_func(void *unused) erts_smp_mtx_unlock(&smq_mtx); + if (end_wait) { + erts_thr_progress_finalize_wait(NULL); + erts_thr_progress_active(NULL, 1); + } + /* Send trace messages ... */ ASSERT(local_sys_message_queue); @@ -3259,6 +3292,9 @@ sys_msg_dispatcher_func(void *unused) Process *proc = NULL; Port *port = NULL; + if (erts_thr_progress_update(NULL)) + erts_thr_progress_leader_update(NULL); + #ifdef DEBUG_PRINTOUTS print_msg_type(smqp); #endif @@ -3372,7 +3408,6 @@ sys_msg_dispatcher_func(void *unused) } } - erts_smp_activity_end(ERTS_ACTIVITY_IO, NULL, NULL, NULL); return NULL; } @@ -3422,7 +3457,6 @@ init_sys_msg_dispatcher(void) sys_message_queue_end = NULL; erts_smp_cnd_init(&smq_cnd); erts_smp_mtx_init(&smq_mtx, "sys_msg_q"); - dispatcher_waiting = 0; erts_smp_thr_create(&sys_msg_dispatcher_tid, sys_msg_dispatcher_func, NULL, diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c index 158eb361a4..fca785a4de 100644 --- a/erts/emulator/beam/erl_unicode.c +++ b/erts/emulator/beam/erl_unicode.c @@ -47,7 +47,7 @@ typedef struct _restart_context { static Uint max_loop_limit; -static BIF_RETTYPE utf8_to_list(BIF_ALIST_1); +static BIF_RETTYPE utf8_to_list(Process *p, Eterm arg1); static BIF_RETTYPE finalize_list_to_list(Process *p, byte *bytes, Eterm rest, @@ -348,12 +348,6 @@ static int copy_utf8_bin(byte *target, byte *source, Uint size, return copied; } - if (((*source) == 0xEF) && (source[1] == 0xBF) && - ((source[2] == 0xBE) || (source[2] == 0xBF))) { - *err_pos = source; - return copied; - } - *(target++) = *(source++); *(target++) = *(source++); *(target++) = *(source++); @@ -714,9 +708,8 @@ L_Again: /* Restart with sublist, old listend was pushed on stack */ target[(*pos)++] = (((byte) (x & 0x3F)) | ((byte) 0x80)); } else if (x < 0x10000) { - if ((x >= 0xD800 && x <= 0xDFFF) || - (x == 0xFFFE) || - (x == 0xFFFF)) { /* Invalid unicode range */ + if (x >= 0xD800 && x <= 0xDFFF) { + /* Invalid unicode range */ *err = 1; goto done; } @@ -1230,10 +1223,6 @@ int erts_analyze_utf8(byte *source, Uint size, ((source[1] & 0x20) != 0)) { return ERTS_UTF8_ERROR; } - if (((*source) == 0xEF) && (source[1] == 0xBF) && - ((source[2] == 0xBE) || (source[2] == 0xBF))) { - return ERTS_UTF8_ERROR; - } source += 3; size -= 3; } else if (((*source) & ((byte) 0xF8)) == 0xF0) { @@ -1839,13 +1828,13 @@ static BIF_RETTYPE characters_to_list_trap_4(BIF_ALIST_1) * Instead of building an utf8 buffer, we analyze the binary given and use that. */ -static BIF_RETTYPE utf8_to_list(BIF_ALIST_1) +static BIF_RETTYPE utf8_to_list(Process* p, Eterm arg) { - if (!is_binary(BIF_ARG_1) || aligned_binary_size(BIF_ARG_1) < 0) { - BIF_ERROR(BIF_P,BADARG); + if (!is_binary(arg) || aligned_binary_size(arg) < 0) { + BIF_ERROR(p, BADARG); } - return do_bif_utf8_to_list(BIF_P, BIF_ARG_1, 0U, 0U, 0U, - ERTS_UTF8_ANALYZE_MORE,NIL); + return do_bif_utf8_to_list(p, arg, 0U, 0U, 0U, + ERTS_UTF8_ANALYZE_MORE, NIL); } @@ -2166,9 +2155,8 @@ L_Again: /* Restart with sublist, old listend was pushed on stack */ } else if (x < 0x800) { need += 2; } else if (x < 0x10000) { - if ((x >= 0xD800 && x <= 0xDFFF) || - (x == 0xFFFE) || - (x == 0xFFFF)) { /* Invalid unicode range */ + if (x >= 0xD800 && x <= 0xDFFF) { + /* Invalid unicode range */ DESTROY_ESTACK(stack); return ((Sint) -1); } @@ -2314,9 +2302,7 @@ L_Again: /* Restart with sublist, old listend was pushed on stack */ *p++ = (((byte) (x & 0x3F)) | ((byte) 0x80)); } else if (x < 0x10000) { - ASSERT(!((x >= 0xD800 && x <= 0xDFFF) || - (x == 0xFFFE) || - (x == 0xFFFF))); + ASSERT(!(x >= 0xD800 && x <= 0xDFFF)); *p++ = (((byte) (x >> 12)) | ((byte) 0xE0)); *p++ = ((((byte) (x >> 6)) & 0x3F) | diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h index e7fd144ec3..5dc307e383 100644 --- a/erts/emulator/beam/erl_vm.h +++ b/erts/emulator/beam/erl_vm.h @@ -55,7 +55,7 @@ heap data on the C stack or if we use the buffers in the scheduler data. */ #define TMP_HEAP_SIZE 128 /* Number of Eterm in the schedulers small heap for transient heap data */ -#define CMP_TMP_HEAP_SIZE 2 /* cmp wants its own tmp-heap... */ +#define CMP_TMP_HEAP_SIZE 32 /* cmp wants its own tmp-heap... */ #define ERL_ARITH_TMP_HEAP_SIZE 4 /* as does erl_arith... */ #define BEAM_EMU_TMP_HEAP_SIZE 2 /* and beam_emu... */ @@ -83,11 +83,7 @@ #define CP_SIZE 1 #define ErtsHAllocLockCheck(P) \ - ERTS_SMP_LC_ASSERT((ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks((P))) \ - || ((P)->id == ERTS_INVALID_PID) \ - || ((P)->scheduler_data \ - && (P) == (P)->scheduler_data->match_pseudo_process) \ - || erts_is_system_blocked(0)) + ERTS_SMP_LC_ASSERT(erts_dbg_check_halloc_lock((P))) #ifdef DEBUG diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c index 5bc402fe22..18d62dac1d 100644 --- a/erts/emulator/beam/export.c +++ b/erts/emulator/beam/export.c @@ -208,7 +208,8 @@ erts_export_put(Eterm mod, Eterm func, unsigned int arity) Export e; int ix; - ERTS_SMP_LC_ASSERT(erts_initialized == 0 || erts_smp_is_system_blocked(0)); + ERTS_SMP_LC_ASSERT(erts_initialized == 0 + || erts_smp_thr_progress_is_blocking()); ASSERT(is_atom(mod)); ASSERT(is_atom(func)); e.code[0] = mod; @@ -265,7 +266,8 @@ erts_export_consolidate(void) HashInfo hi; #endif - ERTS_SMP_LC_ASSERT(erts_initialized == 0 || erts_smp_is_system_blocked(0)); + ERTS_SMP_LC_ASSERT(erts_initialized == 0 + || erts_smp_thr_progress_is_blocking()); export_write_lock(); erts_index_merge(&secondary_export_table, &export_table); diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 6953e7fe7d..80ce4b969c 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -988,16 +988,16 @@ BIF_RETTYPE erts_debug_dist_ext_to_term_2(BIF_ALIST_2) } -Eterm -term_to_binary_1(Process* p, Eterm Term) +BIF_RETTYPE term_to_binary_1(BIF_ALIST_1) { - return erts_term_to_binary(p, Term, 0, TERM_TO_BINARY_DFLAGS); + return erts_term_to_binary(BIF_P, BIF_ARG_1, 0, TERM_TO_BINARY_DFLAGS); } - -Eterm -term_to_binary_2(Process* p, Eterm Term, Eterm Flags) +BIF_RETTYPE term_to_binary_2(BIF_ALIST_2) { + Process* p = BIF_P; + Eterm Term = BIF_ARG_1; + Eterm Flags = BIF_ARG_2; int level = 0; Uint flags = TERM_TO_BINARY_DFLAGS; @@ -1256,8 +1256,11 @@ BIF_RETTYPE binary_to_term_2(BIF_ALIST_2) } Eterm -external_size_1(Process* p, Eterm Term) +external_size_1(BIF_ALIST_1) { + Process* p = BIF_P; + Eterm Term = BIF_ARG_1; + Uint size = erts_encode_ext_size(Term); if (IS_USMALL(0, size)) { BIF_RET(make_small(size)); @@ -1268,13 +1271,13 @@ external_size_1(Process* p, Eterm Term) } Eterm -external_size_2(Process* p, Eterm Term, Eterm Flags) +external_size_2(BIF_ALIST_2) { Uint size; Uint flags = TERM_TO_BINARY_DFLAGS; - while (is_list(Flags)) { - Eterm arg = CAR(list_val(Flags)); + while (is_list(BIF_ARG_2)) { + Eterm arg = CAR(list_val(BIF_ARG_2)); Eterm* tp; if (is_tuple(arg) && *(tp = tuple_val(arg)) == make_arityval(2)) { @@ -1293,19 +1296,19 @@ external_size_2(Process* p, Eterm Term, Eterm Flags) } } else { error: - BIF_ERROR(p, BADARG); + BIF_ERROR(BIF_P, BADARG); } - Flags = CDR(list_val(Flags)); + BIF_ARG_2 = CDR(list_val(BIF_ARG_2)); } - if (is_not_nil(Flags)) { + if (is_not_nil(BIF_ARG_2)) { goto error; } - size = erts_encode_ext_size_2(Term, flags); + size = erts_encode_ext_size_2(BIF_ARG_1, flags); if (IS_USMALL(0, size)) { BIF_RET(make_small(size)); } else { - Eterm* hp = HAlloc(p, BIG_UINT_HEAP_SIZE); + Eterm* hp = HAlloc(BIF_P, BIG_UINT_HEAP_SIZE); BIF_RET(uint_to_big(size, hp)); } } diff --git a/erts/emulator/beam/fix_alloc.c b/erts/emulator/beam/fix_alloc.c deleted file mode 100644 index 5637281597..0000000000 --- a/erts/emulator/beam/fix_alloc.c +++ /dev/null @@ -1,287 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. - * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. - * - * %CopyrightEnd% - */ -/* General purpose Memory allocator for fixed block size objects */ -/* This allocater is at least an order of magnitude faster than malloc() */ - - -#define NOPERBLOCK 20 -#ifdef HAVE_CONFIG_H -# include "config.h" -#endif - -#include "sys.h" -#include "erl_vm.h" -#include "global.h" -#include "erl_db.h" - -#ifdef ERTS_ALC_N_MIN_A_FIXED_SIZE - -#if ERTS_ALC_MTA_FIXED_SIZE -#include "erl_threads.h" -#include "erl_smp.h" -# ifdef ERTS_SMP -# define FA_LOCK(FA) erts_smp_spin_lock(&(FA)->slck) -# define FA_UNLOCK(FA) erts_smp_spin_unlock(&(FA)->slck) -# else -# define FA_LOCK(FA) erts_mtx_lock(&(FA)->mtx) -# define FA_UNLOCK(FA) erts_mtx_unlock(&(FA)->mtx) -# endif -#else -# define FA_LOCK(FA) -# define FA_UNLOCK(FA) -#endif - -typedef union {double d; long l;} align_t; - -typedef struct fix_alloc_block { - struct fix_alloc_block *next; - align_t mem[1]; -} FixAllocBlock; - -typedef struct fix_alloc { - Uint item_size; - void *freelist; - Uint no_free; - Uint no_blocks; - FixAllocBlock *blocks; -#if ERTS_ALC_MTA_FIXED_SIZE -# ifdef ERTS_SMP - erts_smp_spinlock_t slck; -# else - erts_mtx_t mtx; -# endif -#endif -} FixAlloc; - -static void *(*core_alloc)(Uint); -static Uint xblk_sz; - -static FixAlloc **fa; -#define FA_SZ (1 + ERTS_ALC_N_MAX_A_FIXED_SIZE - ERTS_ALC_N_MIN_A_FIXED_SIZE) - -#define FIX_IX(N) ((N) - ERTS_ALC_N_MIN_A_FIXED_SIZE) - -#define FIX_POOL_SZ(I_SZ) \ - ((I_SZ)*NOPERBLOCK + sizeof(FixAllocBlock) - sizeof(align_t)) - -#if defined(DEBUG) && !ERTS_ALC_MTA_FIXED_SIZE -static int first_time; -#endif - -void erts_init_fix_alloc(Uint extra_block_size, - void *(*alloc)(Uint)) -{ - int i; - - xblk_sz = extra_block_size; - core_alloc = alloc; - - fa = (FixAlloc **) (*core_alloc)(FA_SZ * sizeof(FixAlloc *)); - if (!fa) - erts_alloc_enomem(ERTS_ALC_T_UNDEF, FA_SZ * sizeof(FixAlloc *)); - - for (i = 0; i < FA_SZ; i++) - fa[i] = NULL; -#if defined(DEBUG) && !ERTS_ALC_MTA_FIXED_SIZE - first_time = 1; -#endif -} - -Uint -erts_get_fix_size(ErtsAlcType_t type) -{ - Uint i = FIX_IX(ERTS_ALC_T2N(type)); - return i < FA_SZ && fa[i] ? fa[i]->item_size : 0; -} - -void -erts_set_fix_size(ErtsAlcType_t type, Uint size) -{ - Uint sz; - Uint i; - FixAlloc *fs; - ErtsAlcType_t t_no = ERTS_ALC_T2N(type); - sz = xblk_sz + size; - -#ifdef DEBUG - ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE <= t_no); - ASSERT(t_no <= ERTS_ALC_N_MAX_A_FIXED_SIZE); -#endif - - while (sz % sizeof(align_t) != 0) /* Alignment */ - sz++; - - i = FIX_IX(t_no); - fs = (FixAlloc *) (*core_alloc)(sizeof(FixAlloc)); - if (!fs) - erts_alloc_n_enomem(t_no, sizeof(FixAlloc)); - - fs->item_size = sz; - fs->no_blocks = 0; - fs->no_free = 0; - fs->blocks = NULL; - fs->freelist = NULL; - if (fa[i]) - erl_exit(-1, "Attempt to overwrite existing fix size (%d)", i); - fa[i] = fs; - -#if ERTS_ALC_MTA_FIXED_SIZE -#ifdef ERTS_SMP - erts_smp_spinlock_init_x(&fs->slck, "fix_alloc", make_small(i)); -#else - erts_mtx_init_x(&fs->mtx, "fix_alloc", make_small(i)); -#endif -#endif - -} - -void -erts_fix_info(ErtsAlcType_t type, ErtsFixInfo *efip) -{ - Uint i; - FixAlloc *f; -#ifdef DEBUG - FixAllocBlock *b; - void *fp; -#endif - Uint real_item_size; - ErtsAlcType_t t_no = ERTS_ALC_T2N(type); - - ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE <= t_no); - ASSERT(t_no <= ERTS_ALC_N_MAX_A_FIXED_SIZE); - - i = FIX_IX(t_no); - f = fa[i]; - - efip->total = sizeof(FixAlloc *); - efip->used = 0; - if (!f) - return; - - real_item_size = f->item_size - xblk_sz; - - FA_LOCK(f); - - efip->total += sizeof(FixAlloc); - efip->total += f->no_blocks*FIX_POOL_SZ(real_item_size); - efip->used = efip->total - f->no_free*real_item_size; - -#ifdef DEBUG - ASSERT(efip->total >= efip->used); - for(i = 0, b = f->blocks; b; i++, b = b->next); - ASSERT(f->no_blocks == i); - for (i = 0, fp = f->freelist; fp; i++, fp = *((void **) fp)); - ASSERT(f->no_free == i); -#endif - - FA_UNLOCK(f); - -} - -void -erts_fix_free(ErtsAlcType_t t_no, void *extra, void* ptr) -{ - Uint i; - FixAlloc *f; - - ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE <= t_no); - ASSERT(t_no <= ERTS_ALC_N_MAX_A_FIXED_SIZE); - - i = FIX_IX(t_no); - f = fa[i]; - - FA_LOCK(f); - *((void **) ptr) = f->freelist; - f->freelist = ptr; - f->no_free++; - FA_UNLOCK(f); -} - - -void *erts_fix_realloc(ErtsAlcType_t t_no, void *extra, void* ptr, Uint size) -{ - erts_alc_fatal_error(ERTS_ALC_E_NOTSUP, ERTS_ALC_O_REALLOC, t_no); - return NULL; -} - -void *erts_fix_alloc(ErtsAlcType_t t_no, void *extra, Uint size) -{ - void *ret; - int i; - FixAlloc *f; - -#if defined(DEBUG) && !ERTS_ALC_MTA_FIXED_SIZE - ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE <= t_no); - ASSERT(t_no <= ERTS_ALC_N_MAX_A_FIXED_SIZE); - if (first_time) { /* Check that all sizes have been initialized */ - int i; - for (i = 0; i < FA_SZ; i++) - ASSERT(fa[i]); - first_time = 0; - } -#endif - - - i = FIX_IX(t_no); - f = fa[i]; - - ASSERT(f); - ASSERT(f->item_size >= size); - - FA_LOCK(f); - if (f->freelist == NULL) { /* Gotta alloc some more mem */ - char *ptr; - FixAllocBlock *bl; - Uint n; - - - FA_UNLOCK(f); - bl = (*core_alloc)(FIX_POOL_SZ(f->item_size)); - if (!bl) - return NULL; - - FA_LOCK(f); - bl->next = f->blocks; /* link in first */ - f->blocks = bl; - - n = NOPERBLOCK; - ptr = (char *) &f->blocks->mem[0]; - while(n--) { - *((void **) ptr) = f->freelist; - f->freelist = (void *) ptr; - ptr += f->item_size; - } -#if !ERTS_ALC_MTA_FIXED_SIZE - ASSERT(f->no_free == 0); -#endif - f->no_free += NOPERBLOCK; - f->no_blocks++; - } - - ret = f->freelist; - f->freelist = *((void **) f->freelist); - ASSERT(f->no_free > 0); - f->no_free--; - - FA_UNLOCK(f); - - return ret; -} - -#endif /* #ifdef ERTS_ALC_N_MIN_A_FIXED_SIZE */ diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 499bdd77ba..4a4973baab 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -37,16 +37,11 @@ #include "erl_process.h" #include "erl_sys_driver.h" #include "erl_debug.h" +#include "error.h" typedef struct port Port; #include "erl_port_task.h" -#define ERTS_MAX_NO_OF_ASYNC_THREADS 1024 -extern int erts_async_max_threads; -#define ERTS_ASYNC_THREAD_MIN_STACK_SIZE 16 /* Kilo words */ -#define ERTS_ASYNC_THREAD_MAX_STACK_SIZE 8192 /* Kilo words */ -extern int erts_async_thread_suggested_stack_size; - typedef struct erts_driver_t_ erts_driver_t; #define SMALL_IO_QUEUE 5 /* Number of fixed elements */ @@ -200,10 +195,10 @@ erts_port_runq(Port *prt) { #ifdef ERTS_SMP ErtsRunQueue *rq1, *rq2; - rq1 = (ErtsRunQueue *) erts_smp_atomic_read(&prt->run_queue); + rq1 = (ErtsRunQueue *) erts_smp_atomic_read_nob(&prt->run_queue); while (1) { erts_smp_runq_lock(rq1); - rq2 = (ErtsRunQueue *) erts_smp_atomic_read(&prt->run_queue); + rq2 = (ErtsRunQueue *) erts_smp_atomic_read_nob(&prt->run_queue); if (rq1 == rq2) return rq1; erts_smp_runq_unlock(rq1); @@ -542,10 +537,11 @@ ERTS_GLB_INLINE void erts_may_save_closed_port(Port *prt) ERTS_SMP_LC_ASSERT(erts_smp_lc_spinlock_is_locked(&prt->state_lck)); if (prt->snapshot != erts_smp_atomic32_read_acqb(&erts_ports_snapshot)) { /* Dead ports are added from the end of the snapshot buffer */ - Eterm* tombstone = (Eterm*) erts_smp_atomic_addtest(&erts_dead_ports_ptr, - -(erts_aint_t)sizeof(Eterm)); + Eterm* tombstone; + tombstone = (Eterm*) erts_smp_atomic_add_read_nob(&erts_dead_ports_ptr, + -(erts_aint_t)sizeof(Eterm)); ASSERT(tombstone+1 != NULL); - ASSERT(prt->snapshot == erts_smp_atomic32_read(&erts_ports_snapshot) - 1); + ASSERT(prt->snapshot == erts_smp_atomic32_read_nob(&erts_ports_snapshot) - 1); *tombstone = prt->id; } /*else no ongoing snapshot or port was already included or created after snapshot */ @@ -850,18 +846,35 @@ void erts_queue_monitor_message(Process *, Eterm, Eterm); void erts_init_bif(void); +Eterm erl_send(Process *p, Eterm to, Eterm msg); + +/* erl_bif_op.c */ + +Eterm erl_is_function(Process* p, Eterm arg1, Eterm arg2); /* erl_bif_port.c */ /* erl_bif_trace.c */ +Eterm erl_seq_trace_info(Process *p, Eterm arg1); void erts_system_monitor_clear(Process *c_p); void erts_system_profile_clear(Process *c_p); /* beam_load.c */ +typedef struct { + BeamInstr* current; /* Pointer to: Mod, Name, Arity */ + Uint needed; /* Heap space needed for entire tuple */ + Uint32 loc; /* Location in source code */ + Eterm* fname_ptr; /* Pointer to fname table */ +} FunctionInfo; + int erts_load_module(Process *c_p, ErtsProcLocks c_p_locks, Eterm group_leader, Eterm* mod, byte* code, int size); void init_load(void); BeamInstr* find_function_from_pc(BeamInstr* pc); +Eterm* erts_build_mfa_item(FunctionInfo* fi, Eterm* hp, + Eterm args, Eterm* mfa_p); +void erts_lookup_function_info(FunctionInfo* fi, BeamInstr* pc, int full_info); +void erts_set_current_function(FunctionInfo* fi, BeamInstr* current); Eterm erts_module_info_0(Process* p, Eterm module); Eterm erts_module_info_1(Process* p, Eterm module, Eterm what); Eterm erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info); @@ -1052,6 +1065,7 @@ void init_emulator(void); void process_main(void); Eterm build_stacktrace(Process* c_p, Eterm exc); Eterm expand_error_value(Process* c_p, Uint freason, Eterm Value); +void erts_save_stacktrace(Process* p, struct StackTrace* s, int depth); /* erl_init.c */ @@ -1073,6 +1087,7 @@ extern ErtsModifiedTimings erts_modified_timings[]; #define ERTS_MODIFIED_TIMING_INPUT_REDS \ (erts_modified_timings[erts_modified_timing_level].input_reds) +extern int erts_no_line_info; extern Eterm erts_error_logger_warnings; extern int erts_initialized; extern int erts_compat_rel; @@ -1200,11 +1215,11 @@ erts_smp_port_trylock(Port *prt) #ifdef ERTS_SMP int res; - ASSERT(erts_smp_atomic_read(&prt->refc) > 0); - erts_smp_atomic_inc(&prt->refc); + ASSERT(erts_smp_atomic_read_nob(&prt->refc) > 0); + erts_smp_atomic_inc_nob(&prt->refc); res = erts_smp_mtx_trylock(prt->lock); if (res == EBUSY) { - erts_smp_atomic_dec(&prt->refc); + erts_smp_atomic_dec_nob(&prt->refc); } return res; @@ -1217,8 +1232,8 @@ ERTS_GLB_INLINE void erts_smp_port_lock(Port *prt) { #ifdef ERTS_SMP - ASSERT(erts_smp_atomic_read(&prt->refc) > 0); - erts_smp_atomic_inc(&prt->refc); + ASSERT(erts_smp_atomic_read_nob(&prt->refc) > 0); + erts_smp_atomic_inc_nob(&prt->refc); erts_smp_mtx_lock(prt->lock); #endif } @@ -1229,7 +1244,7 @@ erts_smp_port_unlock(Port *prt) #ifdef ERTS_SMP erts_aint_t refc; erts_smp_mtx_unlock(prt->lock); - refc = erts_smp_atomic_dectest(&prt->refc); + refc = erts_smp_atomic_dec_read_nob(&prt->refc); ASSERT(refc >= 0); if (refc == 0) erts_port_cleanup(prt); @@ -1298,7 +1313,7 @@ erts_id2port_sflgs(Eterm id, Process *c_p, ErtsProcLocks c_p_locks, Uint32 sflgs } #ifdef ERTS_SMP else { - erts_smp_atomic_inc(&prt->refc); + erts_smp_atomic_inc_nob(&prt->refc); erts_smp_port_state_unlock(prt); if (no_proc_locks) @@ -1626,8 +1641,7 @@ void monitor_generic(Process *p, Eterm type, Eterm spec); Uint erts_trace_flag2bit(Eterm flag); int erts_trace_flags(Eterm List, Uint *pMask, Eterm *pTracer, int *pCpuTimestamp); -Eterm erts_bif_trace(int bif_index, Process* p, - Eterm arg1, Eterm arg2, Eterm arg3, BeamInstr *I); +Eterm erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr *I); #ifdef ERTS_SMP void erts_send_pending_trace_msgs(ErtsSchedulerData *esdp); diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index df5f8b22a3..fff720634d 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -42,6 +42,7 @@ #include "erl_bits.h" #include "erl_version.h" #include "error.h" +#include "erl_async.h" extern ErlDrvEntry fd_driver_entry; extern ErlDrvEntry vanilla_driver_entry; @@ -244,8 +245,8 @@ get_free_port(void) } port->status = ERTS_PORT_SFLG_INITIALIZING; #ifdef ERTS_SMP - ERTS_SMP_LC_ASSERT(erts_smp_atomic_read(&port->refc) == 0); - erts_smp_atomic_set(&port->refc, 2); /* Port alive + lock */ + ERTS_SMP_LC_ASSERT(erts_smp_atomic_read_nob(&port->refc) == 0); + erts_smp_atomic_set_nob(&port->refc, 2); /* Port alive + lock */ #endif erts_smp_port_state_unlock(port); return num & port_num_mask; @@ -327,7 +328,7 @@ port_cleanup(Port *prt) #ifdef ERTS_SMP ASSERT(prt->status & ERTS_PORT_SFLG_FREE_SCHEDULED); - ERTS_SMP_LC_ASSERT(erts_smp_atomic_read(&prt->refc) == 0); + ERTS_SMP_LC_ASSERT(erts_smp_atomic_read_nob(&prt->refc) == 0); port_specific = (prt->status & ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK); @@ -425,11 +426,11 @@ setup_port(Port* prt, Eterm pid, erts_driver_t *driver, erts_smp_runq_lock(runq); erts_smp_port_state_lock(prt); prt->status = ERTS_PORT_SFLG_CONNECTED | xstatus; - prt->snapshot = erts_smp_atomic32_read(&erts_ports_snapshot); + prt->snapshot = erts_smp_atomic32_read_nob(&erts_ports_snapshot); old_name = prt->name; prt->name = new_name; #ifdef ERTS_SMP - erts_smp_atomic_set(&prt->run_queue, (erts_aint_t) runq); + erts_smp_atomic_set_nob(&prt->run_queue, (erts_aint_t) runq); #endif ASSERT(!prt->drv_ptr); prt->drv_ptr = driver; @@ -590,8 +591,8 @@ erts_open_driver(erts_driver_t* driver, /* Pointer to driver. */ erts_smp_port_state_lock(port); port->status = ERTS_PORT_SFLG_FREE; #ifdef ERTS_SMP - ERTS_SMP_LC_ASSERT(erts_smp_atomic_read(&port->refc) == 2); - erts_smp_atomic_set(&port->refc, 0); + ERTS_SMP_LC_ASSERT(erts_smp_atomic_read_nob(&port->refc) == 2); + erts_smp_atomic_set_nob(&port->refc, 0); #endif erts_smp_port_state_unlock(port); return -3; @@ -1206,7 +1207,7 @@ int erts_write_to_port(Eterm caller_id, Port *p, Eterm list) } } p->bytes_out += size; - erts_smp_atomic_add(&erts_bytes_out, size); + erts_smp_atomic_add_nob(&erts_bytes_out, size); #ifdef ERTS_SMP if (p->xports) @@ -1277,13 +1278,13 @@ void init_io(void) erts_port = (Port *) erts_alloc(ERTS_ALC_T_PORT_TABLE, erts_max_ports * sizeof(Port)); - erts_smp_atomic_init(&erts_bytes_out, 0); - erts_smp_atomic_init(&erts_bytes_in, 0); + erts_smp_atomic_init_nob(&erts_bytes_out, 0); + erts_smp_atomic_init_nob(&erts_bytes_in, 0); for (i = 0; i < erts_max_ports; i++) { erts_port_task_init_sched(&erts_port[i].sched); #ifdef ERTS_SMP - erts_smp_atomic_init(&erts_port[i].refc, 0); + erts_smp_atomic_init_nob(&erts_port[i].refc, 0); erts_port[i].lock = NULL; erts_port[i].xports = NULL; erts_smp_spinlock_init_x(&erts_port[i].state_lck, "port_state", make_small(i)); @@ -1300,7 +1301,7 @@ void init_io(void) erts_port[i].port_data_lock = NULL; } - erts_smp_atomic32_init(&erts_ports_snapshot, (erts_aint32_t) 0); + erts_smp_atomic32_init_nob(&erts_ports_snapshot, (erts_aint32_t) 0); last_port_num = 0; erts_smp_spinlock_init(&get_free_port_lck, "get_free_port"); @@ -3253,7 +3254,7 @@ int driver_output_binary(ErlDrvPort ix, char* hbuf, int hlen, return 0; prt->bytes_in += (hlen + len); - erts_smp_atomic_add(&erts_bytes_in, (erts_aint_t) (hlen + len)); + erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) (hlen + len)); if (prt->status & ERTS_PORT_SFLG_DISTRIBUTION) { return erts_net_message(prt, prt->dist_entry, @@ -3288,7 +3289,7 @@ int driver_output2(ErlDrvPort ix, char* hbuf, int hlen, char* buf, int len) return 0; prt->bytes_in += (hlen + len); - erts_smp_atomic_add(&erts_bytes_in, (erts_aint_t) (hlen + len)); + erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) (hlen + len)); if (prt->status & ERTS_PORT_SFLG_DISTRIBUTION) { if (len == 0) return erts_net_message(prt, @@ -3365,7 +3366,7 @@ int driver_outputv(ErlDrvPort ix, char* hbuf, int hlen, ErlIOVec* vec, int skip) /* XXX handle distribution !!! */ prt->bytes_in += (hlen + size); - erts_smp_atomic_add(&erts_bytes_in, (erts_aint_t) (hlen + size)); + erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) (hlen + size)); deliver_vec_message(prt, prt->connected, hbuf, hlen, binv, iov, n, size); return 0; } @@ -3539,13 +3540,13 @@ pdl_init(void) static ERTS_INLINE void pdl_init_refc(ErlDrvPDL pdl) { - erts_atomic_init(&pdl->refc, 1); + erts_atomic_init_nob(&pdl->refc, 1); } static ERTS_INLINE ErlDrvSInt pdl_read_refc(ErlDrvPDL pdl) { - erts_aint_t refc = erts_atomic_read(&pdl->refc); + erts_aint_t refc = erts_atomic_read_nob(&pdl->refc); ERTS_LC_ASSERT(refc >= 0); return (ErlDrvSInt) refc; } @@ -3553,14 +3554,14 @@ pdl_read_refc(ErlDrvPDL pdl) static ERTS_INLINE void pdl_inc_refc(ErlDrvPDL pdl) { - erts_atomic_inc(&pdl->refc); + erts_atomic_inc_nob(&pdl->refc); ERTS_LC_ASSERT(driver_pdl_get_refc(pdl) > 1); } static ERTS_INLINE ErlDrvSInt pdl_inctest_refc(ErlDrvPDL pdl) { - erts_aint_t refc = erts_atomic_inctest(&pdl->refc); + erts_aint_t refc = erts_atomic_inc_read_nob(&pdl->refc); ERTS_LC_ASSERT(refc > 1); return (ErlDrvSInt) refc; } @@ -3569,7 +3570,7 @@ pdl_inctest_refc(ErlDrvPDL pdl) static ERTS_INLINE void pdl_dec_refc(ErlDrvPDL pdl) { - erts_atomic_dec(&pdl->refc); + erts_atomic_dec_nob(&pdl->refc); ERTS_LC_ASSERT(driver_pdl_get_refc(pdl) > 0); } #endif @@ -3577,7 +3578,7 @@ pdl_dec_refc(ErlDrvPDL pdl) static ERTS_INLINE ErlDrvSInt pdl_dectest_refc(ErlDrvPDL pdl) { - erts_aint_t refc = erts_atomic_dectest(&pdl->refc); + erts_aint_t refc = erts_atomic_dec_read_nob(&pdl->refc); ERTS_LC_ASSERT(refc >= 0); return (ErlDrvSInt) refc; } @@ -4579,7 +4580,10 @@ int driver_lock_driver(ErlDrvPort ix) erts_smp_mtx_lock(&erts_driver_list_lock); - if (prt == NULL) return -1; + if (prt == NULL) { + erts_smp_mtx_unlock(&erts_driver_list_lock); + return -1; + } ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); if ((dh = (DE_Handle*)prt->drv_ptr->handle ) == NULL) { diff --git a/erts/emulator/beam/module.c b/erts/emulator/beam/module.c index 91e4ccce70..b93b1ad09a 100644 --- a/erts/emulator/beam/module.c +++ b/erts/emulator/beam/module.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * Copyright Ericsson AB 1996-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -108,7 +108,8 @@ erts_put_module(Eterm mod) int index; ASSERT(is_atom(mod)); - ERTS_SMP_LC_ASSERT(erts_initialized == 0 || erts_smp_is_system_blocked(0)); + ERTS_SMP_LC_ASSERT(erts_initialized == 0 + || erts_smp_thr_progress_is_blocking()); e.module = atom_val(mod); index = index_put(&module_table, (void*) &e); return (Module*) erts_index_lookup(&module_table, index); diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab index 304ce22ef2..fc53a88a3a 100644 --- a/erts/emulator/beam/ops.tab +++ b/erts/emulator/beam/ops.tab @@ -25,30 +25,12 @@ # instruction transformations; thus, they never occur in BEAM files. # -# Special instruction used to generate an error message when -# trying to load a module compiled by the V1 compiler (R5 & R6). -# (Specially treated in beam_load.c.) +# The too_old_compiler/0 instruction is specially handled in beam_load.c +# to produce a user-friendly message informing the user that the module +# needs to be re-compiled with a modern compiler. too_old_compiler/0 -too_old_compiler - -# -# Obsolete instruction usage follow. (Nowdays we use f with -# a zero label instead of p.) -# - -is_list p S => too_old_compiler -is_nonempty_list p R => too_old_compiler -is_nil p R => too_old_compiler - -is_tuple p S => too_old_compiler -test_arity p S Arity => too_old_compiler - -is_integer p R => too_old_compiler -is_float p R => too_old_compiler -is_atom p R => too_old_compiler - -is_eq_exact p S1 S2 => too_old_compiler +too_old_compiler | never() => # In R9C and earlier, the loader used to insert special instructions inside # the module_info/0,1 functions. (In R10B and later, the compiler inserts @@ -88,12 +70,42 @@ i_time_breakpoint i_return_time_trace i_return_to_trace i_yield -i_global_cons -i_global_tuple -i_global_copy return +# +# To ensure that a "move Src x(0)" instruction can be combined +# with the following call instruction, we need to make sure that +# there is no line/1 instruction between the move and the call. +# + +move S r | line Loc | call_ext Ar Func => \ + line Loc | move S r | call_ext Ar Func +move S r | line Loc | call_ext_last Ar Func=u$is_bif D => \ + line Loc | move S r | call_ext_last Ar Func D +move S r | line Loc | call_ext_only Ar Func=u$is_bif => \ + line Loc | move S r | call_ext_only Ar Func +move S r | line Loc | call Ar Func => \ + line Loc | move S r | call Ar Func + +# +# A tail-recursive call to an external function (non-BIF) will +# never be saved on the stack, so there is no reason to keep +# the line instruction. (The compiler did not remove the line +# instruction because it cannot tell the difference between +# BIFs and ordinary Erlang functions.) +# + +line Loc | call_ext_last Ar Func=u$is_not_bif D => \ + call_ext_last Ar Func D +line Loc | call_ext_only Ar Func=u$is_not_bif => \ + call_ext_only Ar Func + +line Loc | func_info M F A => func_info M F A | line Loc + +line I + + %macro: allocate Allocate -pack %macro: allocate_zero AllocateZero -pack %macro: allocate_heap AllocateHeap -pack @@ -277,8 +289,6 @@ raise s s badarg j system_limit j -move R R => - move C=cxy r | jump Lbl => move_jump Lbl C %macro: move_jump MoveJump -nonext @@ -585,8 +595,6 @@ get_tuple_element Reg P Dst => i_get_tuple_element Reg P Dst | original_reg Reg original_reg Reg Pos => -get_tuple_element Reg P Dst => i_get_tuple_element Reg P Dst - original_reg/2 extract_next_element D1=xy | original_reg Reg P1 | get_tuple_element Reg P2 D2=xy | \ @@ -837,11 +845,11 @@ call_ext_only u==3 u$func:erlang:apply/3 => i_apply_only # thus there is no need to generate any return instruction. # -call_ext_last u==1 Bif=u$bif:erlang:exit/1 D => call_bif1 Bif -call_ext_last u==1 Bif=u$bif:erlang:throw/1 D => call_bif1 Bif +call_ext_last u==1 Bif=u$bif:erlang:exit/1 D => call_bif Bif +call_ext_last u==1 Bif=u$bif:erlang:throw/1 D => call_bif Bif -call_ext_only u==1 Bif=u$bif:erlang:exit/1 => call_bif1 Bif -call_ext_only u==1 Bif=u$bif:erlang:throw/1 => call_bif1 Bif +call_ext_only u==1 Bif=u$bif:erlang:exit/1 => call_bif Bif +call_ext_only u==1 Bif=u$bif:erlang:throw/1 => call_bif Bif # # The error/1 and error/2 BIFs never execute the instruction following them; @@ -851,13 +859,13 @@ call_ext_only u==1 Bif=u$bif:erlang:throw/1 => call_bif1 Bif # the continuation pointer on the stack. # -call_ext_last u==1 Bif=u$bif:erlang:error/1 D => call_bif1 Bif -call_ext_last u==2 Bif=u$bif:erlang:error/2 D => call_bif2 Bif +call_ext_last u==1 Bif=u$bif:erlang:error/1 D => call_bif Bif +call_ext_last u==2 Bif=u$bif:erlang:error/2 D => call_bif Bif call_ext_only Ar=u==1 Bif=u$bif:erlang:error/1 => \ - allocate u Ar | call_bif1 Bif + allocate u Ar | call_bif Bif call_ext_only Ar=u==2 Bif=u$bif:erlang:error/2 => \ - allocate u Ar | call_bif2 Bif + allocate u Ar | call_bif Bif # # The yield/0 BIF is an instruction @@ -875,47 +883,18 @@ call_ext_last u==3 u$func:erlang:hibernate/3 D => i_hibernate call_ext_only u==3 u$func:erlang:hibernate/3 => i_hibernate # -# Hybrid memory architecture need special cons and tuple instructions -# that allocate on the message area. These looks like BIFs in the BEAM code. -# - -call_ext u==2 u$func:hybrid:cons/2 => i_global_cons -call_ext_last u==2 u$func:hybrid:cons/2 D => i_global_cons | deallocate_return D -call_ext_only Ar=u==2 u$func:hybrid:cons/2 => i_global_cons | return - -call_ext u==1 u$func:hybrid:tuple/1 => i_global_tuple -call_ext_last u==1 u$func:hybrid:tuple/1 D => i_global_tuple | deallocate_return D -call_ext_only Ar=u==1 u$func:hybrid:tuple/1 => i_global_tuple | return - -call_ext u==1 u$func:hybrid:copy/1 => i_global_copy -call_ext_last u==1 u$func:hybrid:copy/1 D => i_global_copy | deallocate_return D -call_ext_only u==1 Ar=u$func:hybrid:copy/1 => i_global_copy | return - -# # The general case for BIFs that have no special instructions. # A BIF used in the tail must be followed by a return instruction. # # To make trapping and stack backtraces work correctly, we make sure that # the continuation pointer is always stored on the stack. -call_ext u==0 Bif=u$is_bif => call_bif0 Bif -call_ext u==1 Bif=u$is_bif => call_bif1 Bif -call_ext u==2 Bif=u$is_bif => call_bif2 Bif -call_ext u==3 Bif=$is_bif => call_bif3 Bif +call_ext u Bif=u$is_bif => call_bif Bif -call_ext_last u==0 Bif=u$is_bif D => call_bif0 Bif | deallocate_return D -call_ext_last u==1 Bif=u$is_bif D => call_bif1 Bif | deallocate_return D -call_ext_last u==2 Bif=u$is_bif D => call_bif2 Bif | deallocate_return D -call_ext_last u==3 Bif=u$is_bif D => call_bif3 Bif | deallocate_return D +call_ext_last u Bif=u$is_bif D => call_bif Bif | deallocate_return D -call_ext_only Ar=u==0 Bif=u$is_bif => \ - allocate u Ar | call_bif0 Bif | deallocate_return u -call_ext_only Ar=u==1 Bif=u$is_bif => \ - allocate u Ar | call_bif1 Bif | deallocate_return u -call_ext_only Ar=u==2 Bif=u$is_bif => \ - allocate u Ar | call_bif2 Bif | deallocate_return u -call_ext_only Ar=u==3 Bif=u$is_bif => \ - allocate u Ar | call_bif3 Bif | deallocate_return u +call_ext_only Ar=u Bif=u$is_bif => \ + allocate u Ar | call_bif Bif | deallocate_return u # # Any remaining calls are calls to Erlang functions, not BIFs. @@ -928,9 +907,9 @@ move S=c r | call_ext Ar=u Func=u$is_not_bif => i_move_call_ext S r Func move S=c r | call_ext_last Ar=u Func=u$is_not_bif D => i_move_call_ext_last Func D S r move S=c r | call_ext_only Ar=u Func=u$is_not_bif => i_move_call_ext_only Func S r -call_ext Ar=u Func => i_call_ext Func -call_ext_last Ar=u Func D => i_call_ext_last Func D -call_ext_only Ar=u Func => i_call_ext_only Func +call_ext Ar Func => i_call_ext Func +call_ext_last Ar Func D => i_call_ext_last Func D +call_ext_only Ar Func => i_call_ext_only Func i_apply i_apply_last P @@ -942,10 +921,7 @@ i_apply_fun_only i_hibernate -call_bif0 e -call_bif1 e -call_bif2 e -call_bif3 e +call_bif e # # Calls to non-building and guard BIFs. @@ -964,7 +940,7 @@ bif1 p Bif S1 Dst => bif1_body Bif S1 Dst bif1_body Bif Literal=q Dst => move Literal x | bif1_body Bif x Dst bif2 p Bif S1 S2 Dst => i_fetch S1 S2 | i_bif2_body Bif Dst -bif2 Fail=f Bif S1 S2 Dst => i_fetch S1 S2 | i_bif2 Fail Bif Dst +bif2 Fail Bif S1 S2 Dst => i_fetch S1 S2 | i_bif2 Fail Bif Dst i_get s d @@ -1047,8 +1023,8 @@ i_move_call_ext_only e c r # Fun calls. -call_fun Arity=u | deallocate D | return => i_call_fun_last Arity D -call_fun Arity=u => i_call_fun Arity +call_fun Arity | deallocate D | return => i_call_fun_last Arity D +call_fun Arity => i_call_fun Arity i_call_fun I i_call_fun_last I P @@ -1304,13 +1280,13 @@ i_bs_utf16_size s d bs_put_utf8 Fail=j Flags=u Literal=q => \ move Literal x | bs_put_utf8 Fail Flags x -bs_put_utf8 Fail=j u Src=s => i_bs_put_utf8 Fail Src +bs_put_utf8 Fail u Src=s => i_bs_put_utf8 Fail Src i_bs_put_utf8 j s bs_put_utf16 Fail=j Flags=u Literal=q => \ move Literal x | bs_put_utf16 Fail Flags x -bs_put_utf16 Fail=j Flags=u Src=s => i_bs_put_utf16 Fail Flags Src +bs_put_utf16 Fail Flags=u Src=s => i_bs_put_utf16 Fail Flags Src i_bs_put_utf16 j I s @@ -1475,34 +1451,13 @@ bif1 Fail u$bif:erlang:trunc/1 s d => too_old_compiler # # Guard BIFs. # -gc_bif1 Fail I Bif=u$bif:erlang:length/1 Src Dst=d => \ - gen_guard_bif1(Fail, I, Bif, Src, Dst) - -gc_bif1 Fail I Bif=u$bif:erlang:size/1 Src Dst=d => \ - gen_guard_bif1(Fail, I, Bif, Src, Dst) - -gc_bif1 Fail I Bif=u$bif:erlang:bit_size/1 Src Dst=d => \ - gen_guard_bif1(Fail, I, Bif, Src, Dst) - -gc_bif1 Fail I Bif=u$bif:erlang:byte_size/1 Src Dst=d => \ - gen_guard_bif1(Fail, I, Bif, Src, Dst) - -gc_bif1 Fail I Bif=u$bif:erlang:abs/1 Src Dst=d => \ - gen_guard_bif1(Fail, I, Bif, Src, Dst) - -gc_bif1 Fail I Bif=u$bif:erlang:float/1 Src Dst=d => \ +gc_bif1 Fail I Bif Src Dst => \ gen_guard_bif1(Fail, I, Bif, Src, Dst) -gc_bif1 Fail I Bif=u$bif:erlang:round/1 Src Dst=d => \ - gen_guard_bif1(Fail, I, Bif, Src, Dst) - -gc_bif1 Fail I Bif=u$bif:erlang:trunc/1 Src Dst=d => \ - gen_guard_bif1(Fail, I, Bif, Src, Dst) - -gc_bif2 Fail I Bif=u$bif:erlang:binary_part/2 S1 S2 Dst=d => \ +gc_bif2 Fail I Bif S1 S2 Dst => \ gen_guard_bif2(Fail, I, Bif, S1, S2, Dst) -gc_bif3 Fail I Bif=u$bif:erlang:binary_part/3 S1 S2 S3 Dst=d => \ +gc_bif3 Fail I Bif S1 S2 S3 Dst => \ gen_guard_bif3(Fail, I, Bif, S1, S2, S3, Dst) i_gc_bif1 Fail Bif V=q Live D => move V x | i_gc_bif1 Fail Bif x Live D @@ -1520,6 +1475,15 @@ ii_gc_bif3/7 ii_gc_bif3 Fail Bif S1 S2 S3 Live D => move S1 x | i_fetch S2 S3 | i_gc_bif3 Fail Bif x Live D i_gc_bif3 j I s I d + +# +# The following instruction is specially handled in beam_load.c +# to produce a user-friendly message if an unsupported guard BIF is +# encountered. +# +unsupported_guard_bif/3 +unsupported_guard_bif A B C | never() => + # # R13B03 # diff --git a/erts/emulator/beam/safe_hash.c b/erts/emulator/beam/safe_hash.c index 4c54e19cdb..3326e5cc2a 100644 --- a/erts/emulator/beam/safe_hash.c +++ b/erts/emulator/beam/safe_hash.c @@ -61,7 +61,7 @@ static ERTS_INLINE int align_up_pow2(int val) */ static void rehash(SafeHash* h, int grow_limit) { - if (erts_smp_atomic_xchg(&h->is_rehashing, 1) != 0) { + if (erts_smp_atomic_xchg_acqb(&h->is_rehashing, 1) != 0) { return; /* already in progress */ } if (h->grow_limit == grow_limit) { @@ -166,8 +166,8 @@ SafeHash* safe_hash_init(ErtsAlcType_t type, SafeHash* h, char* name, int size, h->name = name; h->fun = fun; set_size(h,size); - erts_smp_atomic_init(&h->is_rehashing, 0); - erts_smp_atomic_init(&h->nitems, 0); + erts_smp_atomic_init_nob(&h->is_rehashing, 0); + erts_smp_atomic_init_nob(&h->nitems, 0); for (i=0; i<SAFE_HASH_LOCK_CNT; i++) { erts_smp_mtx_init(&h->lock_vec[i].mtx,"safe_hash"); } @@ -222,7 +222,7 @@ void* safe_hash_put(SafeHash* h, void* tmpl) *head = b; grow_limit = h->grow_limit; erts_smp_mtx_unlock(lock); - if (erts_smp_atomic_inctest(&h->nitems) > grow_limit) { + if (erts_smp_atomic_inc_read_nob(&h->nitems) > grow_limit) { rehash(h, grow_limit); } return (void*) b; @@ -245,7 +245,7 @@ void* safe_hash_erase(SafeHash* h, void* tmpl) if ((b->hvalue == hval) && (h->fun.cmp(tmpl, (void*)b) == 0)) { *prevp = b->next; erts_smp_mtx_unlock(lock); - erts_smp_atomic_dec(&h->nitems); + erts_smp_atomic_dec_nob(&h->nitems); h->fun.free((void*)b); return tmpl; } diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index e64c43de6e..f9cbcc5892 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -340,7 +340,8 @@ int erts_send_warning_to_logger_str_nogl(char *); #ifdef ERTS_WANT_BREAK_HANDLING # ifdef ERTS_SMP extern erts_smp_atomic32_t erts_break_requested; -# define ERTS_BREAK_REQUESTED ((int) erts_smp_atomic32_read(&erts_break_requested)) +# define ERTS_BREAK_REQUESTED \ + ((int) erts_smp_atomic32_read_nob(&erts_break_requested)) # else extern volatile int erts_break_requested; # define ERTS_BREAK_REQUESTED erts_break_requested @@ -354,7 +355,7 @@ void erts_do_break_handling(void); # else # ifdef ERTS_SMP extern erts_smp_atomic32_t erts_got_sigusr1; -# define ERTS_GOT_SIGUSR1 ((int) erts_smp_atomic32_read(&erts_got_sigusr1)) +# define ERTS_GOT_SIGUSR1 ((int) erts_smp_atomic32_read_mb(&erts_got_sigusr1)) # else extern volatile int erts_got_sigusr1; # define ERTS_GOT_SIGUSR1 erts_got_sigusr1 @@ -363,11 +364,15 @@ extern volatile int erts_got_sigusr1; #endif #ifdef ERTS_SMP -extern erts_smp_atomic_t erts_writing_erl_crash_dump; +extern erts_smp_atomic32_t erts_writing_erl_crash_dump; +extern erts_tsd_key_t erts_is_crash_dumping_key; +#define ERTS_SOMEONE_IS_CRASH_DUMPING \ + ((int) erts_smp_atomic32_read_mb(&erts_writing_erl_crash_dump)) #define ERTS_IS_CRASH_DUMPING \ - ((int) erts_smp_atomic_read(&erts_writing_erl_crash_dump)) + ((int) (SWord) erts_tsd_get(erts_is_crash_dumping_key)) #else extern volatile int erts_writing_erl_crash_dump; +#define ERTS_SOMEONE_IS_CRASH_DUMPING erts_writing_erl_crash_dump #define ERTS_IS_CRASH_DUMPING erts_writing_erl_crash_dump #endif @@ -470,15 +475,6 @@ __decl_noreturn void __noreturn erl_exit(int n, char*, ...); #define ERTS_ABORT_EXIT (INT_MIN + 1) /* no crash dump; only abort() */ #define ERTS_DUMP_EXIT (127) /* crash dump; then exit() */ - -#ifndef ERTS_SMP -int check_async_ready(void); -#ifdef USE_THREADS -void sys_async_ready(int hndl); -int erts_register_async_ready_callback(void (*funcp)(void)); -#endif -#endif - Eterm erts_check_io_info(void *p); /* Size of misc memory allocated from system dependent code */ @@ -611,13 +607,10 @@ extern char *erts_sys_ddll_error(int code); * System interfaces for startup. */ - -#ifdef ERTS_SMP void erts_sys_schedule_interrupt(int set); +#ifdef ERTS_SMP void erts_sys_schedule_interrupt_timed(int set, long msec); void erts_sys_main_thread(void); -#else -#define erts_sys_schedule_interrupt(Set) #endif extern void erts_sys_prepare_crash_dump(void); @@ -669,6 +662,8 @@ int erts_sys_putenv(char *key_value, int sep_ix); *size), a value > 0 if value buffer is too small (*size is set to needed size), and a value < 0 on failure. */ int erts_sys_getenv(char *key, char *value, size_t *size); +/* erts_sys_getenv__() is only allowed to be used in early init phase */ +int erts_sys_getenv__(char *key, char *value, size_t *size); /* Easier to use, but not as efficient, environment functions */ char *erts_read_env(char *key); @@ -692,291 +687,14 @@ int erts_write_env(char *key, char *value); int sys_alloc_opt(int, int); typedef struct { - Sint trim_threshold; - Sint top_pad; - Sint mmap_threshold; - Sint mmap_max; + int trim_threshold; + int top_pad; + int mmap_threshold; + int mmap_max; } SysAllocStat; void sys_alloc_stat(SysAllocStat *); -/* Block the whole system... */ - -#define ERTS_BS_FLG_ALLOW_GC (((Uint32) 1) << 0) -#define ERTS_BS_FLG_ALLOW_IO (((Uint32) 1) << 1) - -/* Activities... */ -typedef enum { - ERTS_ACTIVITY_UNDEFINED, /* Undefined activity */ - ERTS_ACTIVITY_WAIT, /* Waiting */ - ERTS_ACTIVITY_GC, /* Garbage collecting */ - ERTS_ACTIVITY_IO /* I/O including message passing to erl procs */ -} erts_activity_t; - -#ifdef ERTS_SMP - -typedef enum { - ERTS_ACT_ERR_LEAVE_WAIT_UNLOCKED, - ERTS_ACT_ERR_LEAVE_UNKNOWN_ACTIVITY, - ERTS_ACT_ERR_ENTER_UNKNOWN_ACTIVITY -} erts_activity_error_t; - -typedef struct { - erts_smp_atomic32_t do_block; - struct { - erts_smp_atomic32_t wait; - erts_smp_atomic32_t gc; - erts_smp_atomic32_t io; - } in_activity; -} erts_system_block_state_t; - -extern erts_system_block_state_t erts_system_block_state; - -int erts_is_system_blocked(erts_activity_t allowed_activities); -void erts_block_me(void (*prepare)(void *), void (*resume)(void *), void *arg); -void erts_register_blockable_thread(void); -void erts_unregister_blockable_thread(void); -void erts_note_activity_begin(erts_activity_t activity); -void -erts_check_block(erts_activity_t old_activity, - erts_activity_t new_activity, - int locked, - void (*prepare)(void *), - void (*resume)(void *), - void *arg); -void erts_block_system(Uint32 allowed_activities); -int erts_emergency_block_system(long timeout, Uint32 allowed_activities); -void erts_release_system(void); -void erts_system_block_init(void); -void erts_set_activity_error(erts_activity_error_t, char *, int); -#ifdef ERTS_ENABLE_LOCK_CHECK -void erts_lc_activity_change_begin(void); -void erts_lc_activity_change_end(void); -int erts_lc_is_blocking(void); -#define ERTS_LC_IS_BLOCKING \ - (erts_smp_pending_system_block() && erts_lc_is_blocking()) -#endif -#endif - -#define erts_smp_activity_begin(NACT, PRP, RSM, ARG) \ - erts_smp_set_activity(ERTS_ACTIVITY_UNDEFINED, \ - (NACT), \ - 0, \ - (PRP), \ - (RSM), \ - (ARG), \ - __FILE__, \ - __LINE__) -#define erts_smp_activity_change(OACT, NACT, PRP, RSM, ARG) \ - erts_smp_set_activity((OACT), \ - (NACT), \ - 0, \ - (PRP), \ - (RSM), \ - (ARG), \ - __FILE__, \ - __LINE__) -#define erts_smp_activity_end(OACT, PRP, RSM, ARG) \ - erts_smp_set_activity((OACT), \ - ERTS_ACTIVITY_UNDEFINED, \ - 0, \ - (PRP), \ - (RSM), \ - (ARG), \ - __FILE__, \ - __LINE__) - -#define erts_smp_locked_activity_begin(NACT) \ - erts_smp_set_activity(ERTS_ACTIVITY_UNDEFINED, \ - (NACT), \ - 1, \ - NULL, \ - NULL, \ - NULL, \ - __FILE__, \ - __LINE__) -#define erts_smp_locked_activity_change(OACT, NACT) \ - erts_smp_set_activity((OACT), \ - (NACT), \ - 1, \ - NULL, \ - NULL, \ - NULL, \ - __FILE__, \ - __LINE__) -#define erts_smp_locked_activity_end(OACT) \ - erts_smp_set_activity((OACT), \ - ERTS_ACTIVITY_UNDEFINED, \ - 1, \ - NULL, \ - NULL, \ - NULL, \ - __FILE__, \ - __LINE__) - - -ERTS_GLB_INLINE int erts_smp_is_system_blocked(erts_activity_t allowed_activities); -ERTS_GLB_INLINE void erts_smp_block_system(Uint32 allowed_activities); -ERTS_GLB_INLINE int erts_smp_emergency_block_system(long timeout, - Uint32 allowed_activities); -ERTS_GLB_INLINE void erts_smp_release_system(void); -ERTS_GLB_INLINE int erts_smp_pending_system_block(void); -ERTS_GLB_INLINE void erts_smp_chk_system_block(void (*prepare)(void *), - void (*resume)(void *), - void *arg); -ERTS_GLB_INLINE void -erts_smp_set_activity(erts_activity_t old_activity, - erts_activity_t new_activity, - int locked, - void (*prepare)(void *), - void (*resume)(void *), - void *arg, - char *file, - int line); - -#if ERTS_GLB_INLINE_INCL_FUNC_DEF - - -ERTS_GLB_INLINE int -erts_smp_is_system_blocked(erts_activity_t allowed_activities) -{ -#ifdef ERTS_SMP - return erts_is_system_blocked(allowed_activities); -#else - return 1; -#endif -} - -ERTS_GLB_INLINE void -erts_smp_block_system(Uint32 allowed_activities) -{ -#ifdef ERTS_SMP - erts_block_system(allowed_activities); -#endif -} - -ERTS_GLB_INLINE int -erts_smp_emergency_block_system(long timeout, Uint32 allowed_activities) -{ -#ifdef ERTS_SMP - return erts_emergency_block_system(timeout, allowed_activities); -#else - return 0; -#endif -} - -ERTS_GLB_INLINE void -erts_smp_release_system(void) -{ -#ifdef ERTS_SMP - erts_release_system(); -#endif -} - -ERTS_GLB_INLINE int -erts_smp_pending_system_block(void) -{ -#ifdef ERTS_SMP - return (int) erts_smp_atomic32_read(&erts_system_block_state.do_block); -#else - return 0; -#endif -} - - -ERTS_GLB_INLINE void -erts_smp_chk_system_block(void (*prepare)(void *), - void (*resume)(void *), - void *arg) -{ -#ifdef ERTS_SMP - if (erts_smp_pending_system_block()) - erts_block_me(prepare, resume, arg); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_set_activity(erts_activity_t old_activity, - erts_activity_t new_activity, - int locked, - void (*prepare)(void *), - void (*resume)(void *), - void *arg, - char *file, - int line) -{ -#ifdef ERTS_SMP -#ifdef ERTS_ENABLE_LOCK_CHECK - erts_lc_activity_change_begin(); -#endif - switch (old_activity) { - case ERTS_ACTIVITY_UNDEFINED: - break; - case ERTS_ACTIVITY_WAIT: - erts_smp_atomic32_dec(&erts_system_block_state.in_activity.wait); - if (locked) { - /* You are not allowed to leave activity waiting - * without supplying the possibility to block - * unlocked. - */ - erts_set_activity_error(ERTS_ACT_ERR_LEAVE_WAIT_UNLOCKED, - file, line); - } - break; - case ERTS_ACTIVITY_GC: - erts_smp_atomic32_dec(&erts_system_block_state.in_activity.gc); - break; - case ERTS_ACTIVITY_IO: - erts_smp_atomic32_dec(&erts_system_block_state.in_activity.io); - break; - default: - erts_set_activity_error(ERTS_ACT_ERR_LEAVE_UNKNOWN_ACTIVITY, - file, line); - break; - } - - /* We are not allowed to block when going to activity waiting... */ - if (new_activity != ERTS_ACTIVITY_WAIT && erts_smp_pending_system_block()) - erts_check_block(old_activity,new_activity,locked,prepare,resume,arg); - - switch (new_activity) { - case ERTS_ACTIVITY_UNDEFINED: - break; - case ERTS_ACTIVITY_WAIT: - erts_smp_atomic32_inc(&erts_system_block_state.in_activity.wait); - break; - case ERTS_ACTIVITY_GC: - erts_smp_atomic32_inc(&erts_system_block_state.in_activity.gc); - break; - case ERTS_ACTIVITY_IO: - erts_smp_atomic32_inc(&erts_system_block_state.in_activity.io); - break; - default: - erts_set_activity_error(ERTS_ACT_ERR_ENTER_UNKNOWN_ACTIVITY, - file, line); - break; - } - - switch (new_activity) { - case ERTS_ACTIVITY_WAIT: - case ERTS_ACTIVITY_GC: - case ERTS_ACTIVITY_IO: - if (erts_smp_pending_system_block()) - erts_note_activity_begin(new_activity); - break; - default: - break; - } - -#ifdef ERTS_ENABLE_LOCK_CHECK - erts_lc_activity_change_end(); -#endif - -#endif -} - -#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ - #if defined(DEBUG) || defined(ERTS_ENABLE_LOCK_CHECK) #undef ERTS_REFC_DEBUG #define ERTS_REFC_DEBUG @@ -1001,27 +719,27 @@ ERTS_GLB_INLINE erts_aint_t erts_refc_read(erts_refc_t *refcp, ERTS_GLB_INLINE void erts_refc_init(erts_refc_t *refcp, erts_aint_t val) { - erts_smp_atomic_init((erts_smp_atomic_t *) refcp, val); + erts_smp_atomic_init_nob((erts_smp_atomic_t *) refcp, val); } ERTS_GLB_INLINE void erts_refc_inc(erts_refc_t *refcp, erts_aint_t min_val) { #ifdef ERTS_REFC_DEBUG - erts_aint_t val = erts_smp_atomic_inctest((erts_smp_atomic_t *) refcp); + erts_aint_t val = erts_smp_atomic_inc_read_nob((erts_smp_atomic_t *) refcp); if (val < min_val) erl_exit(ERTS_ABORT_EXIT, "erts_refc_inc(): Bad refc found (refc=%ld < %ld)!\n", val, min_val); #else - erts_smp_atomic_inc((erts_smp_atomic_t *) refcp); + erts_smp_atomic_inc_nob((erts_smp_atomic_t *) refcp); #endif } ERTS_GLB_INLINE erts_aint_t erts_refc_inctest(erts_refc_t *refcp, erts_aint_t min_val) { - erts_aint_t val = erts_smp_atomic_inctest((erts_smp_atomic_t *) refcp); + erts_aint_t val = erts_smp_atomic_inc_read_nob((erts_smp_atomic_t *) refcp); #ifdef ERTS_REFC_DEBUG if (val < min_val) erl_exit(ERTS_ABORT_EXIT, @@ -1035,20 +753,20 @@ ERTS_GLB_INLINE void erts_refc_dec(erts_refc_t *refcp, erts_aint_t min_val) { #ifdef ERTS_REFC_DEBUG - erts_aint_t val = erts_smp_atomic_dectest((erts_smp_atomic_t *) refcp); + erts_aint_t val = erts_smp_atomic_dec_read_nob((erts_smp_atomic_t *) refcp); if (val < min_val) erl_exit(ERTS_ABORT_EXIT, "erts_refc_dec(): Bad refc found (refc=%ld < %ld)!\n", val, min_val); #else - erts_smp_atomic_dec((erts_smp_atomic_t *) refcp); + erts_smp_atomic_dec_nob((erts_smp_atomic_t *) refcp); #endif } ERTS_GLB_INLINE erts_aint_t erts_refc_dectest(erts_refc_t *refcp, erts_aint_t min_val) { - erts_aint_t val = erts_smp_atomic_dectest((erts_smp_atomic_t *) refcp); + erts_aint_t val = erts_smp_atomic_dec_read_nob((erts_smp_atomic_t *) refcp); #ifdef ERTS_REFC_DEBUG if (val < min_val) erl_exit(ERTS_ABORT_EXIT, @@ -1062,20 +780,20 @@ ERTS_GLB_INLINE void erts_refc_add(erts_refc_t *refcp, erts_aint_t diff, erts_aint_t min_val) { #ifdef ERTS_REFC_DEBUG - erts_aint_t val = erts_smp_atomic_addtest((erts_smp_atomic_t *) refcp, diff); + erts_aint_t val = erts_smp_atomic_add_read_nob((erts_smp_atomic_t *) refcp, diff); if (val < min_val) erl_exit(ERTS_ABORT_EXIT, "erts_refc_add(%ld): Bad refc found (refc=%ld < %ld)!\n", diff, val, min_val); #else - erts_smp_atomic_add((erts_smp_atomic_t *) refcp, diff); + erts_smp_atomic_add_nob((erts_smp_atomic_t *) refcp, diff); #endif } ERTS_GLB_INLINE erts_aint_t erts_refc_read(erts_refc_t *refcp, erts_aint_t min_val) { - erts_aint_t val = erts_smp_atomic_read((erts_smp_atomic_t *) refcp); + erts_aint_t val = erts_smp_atomic_read_nob((erts_smp_atomic_t *) refcp); #ifdef ERTS_REFC_DEBUG if (val < min_val) erl_exit(ERTS_ABORT_EXIT, diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c index a00faff912..db9a24e0a3 100644 --- a/erts/emulator/beam/time.c +++ b/erts/emulator/beam/time.c @@ -108,9 +108,9 @@ static ErlTimer *tiw_min_ptr; static int itime; /* Constant after init */ erts_smp_atomic_t do_time; /* set at clock interrupt */ -static ERTS_INLINE erts_aint_t do_time_read(void) { return erts_smp_atomic_read(&do_time); } +static ERTS_INLINE erts_aint_t do_time_read(void) { return erts_smp_atomic_read_acqb(&do_time); } static ERTS_INLINE erts_aint_t do_time_update(void) { return do_time_read(); } -static ERTS_INLINE void do_time_init(void) { erts_smp_atomic_init(&do_time, 0L); } +static ERTS_INLINE void do_time_init(void) { erts_smp_atomic_init_nob(&do_time, 0L); } /* get the time (in units of itime) to the next timeout, or -1 if there are no timeouts */ @@ -444,7 +444,7 @@ erts_time_left(ErlTimer *p) } #ifdef DEBUG -void erts_p_slpq() +void erts_p_slpq(void) { int i; ErlTimer* p; diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index a17de717bc..1bd178f280 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -42,6 +42,9 @@ #include "erl_threads.h" #include "erl_smp.h" #include "erl_time.h" +#include "erl_thr_progress.h" +#include "erl_thr_queue.h" +#include "erl_sched_spec_pre_alloc.h" #undef M_TRIM_THRESHOLD #undef M_TOP_PAD @@ -75,6 +78,7 @@ typedef struct { #ifdef ERTS_SMP +#if 0 /* Unused */ static void dispatch_profile_msg_q(profile_sched_msg_q *psmq) { @@ -86,6 +90,7 @@ dispatch_profile_msg_q(profile_sched_msg_q *psmq) profile_scheduler_q(make_small(msg->scheduler_id), msg->state, am_undefined, msg->Ms, msg->s, msg->us); } } +#endif #endif @@ -2642,7 +2647,7 @@ tailrecur_ne: FloatDef f1, f2; Eterm big; #if HEAP_ON_C_STACK - Eterm big_buf[2]; /* If HEAP_ON_C_STACK */ + Eterm big_buf[32]; /* If HEAP_ON_C_STACK */ #else Eterm *big_buf = erts_get_scheduler_data()->cmp_tmp_heap; #endif @@ -2653,41 +2658,108 @@ tailrecur_ne: Eterm aw = a; Eterm bw = b; #endif +#define MAX_LOSSLESS_FLOAT ((double)((1LL << 53) - 2)) +#define MIN_LOSSLESS_FLOAT ((double)(((1LL << 53) - 2)*-1)) b_tag = tag_val_def(bw); switch(_NUMBER_CODE(a_tag, b_tag)) { case SMALL_BIG: - big = small_to_big(signed_val(a), big_buf); - j = big_comp(big, bw); + j = big_sign(bw) ? 1 : -1; + break; + case BIG_SMALL: + j = big_sign(aw) ? -1 : 1; break; case SMALL_FLOAT: - f1.fd = signed_val(a); GET_DOUBLE(bw, f2); - j = float_comp(f1.fd, f2.fd); - break; - case BIG_SMALL: - big = small_to_big(signed_val(b), big_buf); - j = big_comp(aw, big); + if (f2.fd < MAX_LOSSLESS_FLOAT && f2.fd > MIN_LOSSLESS_FLOAT) { + // Float is within the no loss limit + f1.fd = signed_val(aw); + j = float_comp(f1.fd, f2.fd); +#if ERTS_SIZEOF_ETERM == 8 + } else if (f2.fd > (double) (MAX_SMALL + 1)) { + // Float is a positive bignum, i.e. bigger + j = -1; + } else if (f2.fd < (double) (MIN_SMALL - 1)) { + // Float is a negative bignum, i.e. smaller + j = 1; + } else { // Float is a Sint but less precise + j = signed_val(aw) - (Sint) f2.fd; + } +#else + } else { + // If float is positive it is bigger than small + j = (f2.fd > 0.0) ? -1 : 1; + } +#endif // ERTS_SIZEOF_ETERM == 8 break; case BIG_FLOAT: - if (big_to_double(aw, &f1.fd) < 0) { - j = big_sign(a) ? -1 : 1; + GET_DOUBLE(bw, f2); + if ((f2.fd < (double) (MAX_SMALL + 1)) + && (f2.fd > (double) (MIN_SMALL - 1))) { + // Float is a Sint + j = big_sign(aw) ? -1 : 1; + } else if ((pow(2.0,(big_arity(aw)-1.0)*D_EXP)-1.0) > fabs(f2.fd)) { + // If bignum size shows that it is bigger than the abs float + j = big_sign(aw) ? -1 : 1; + } else if ((pow(2.0,(big_arity(aw))*D_EXP)-1.0) < fabs(f2.fd)) { + // If bignum size shows that it is smaller than the abs float + j = f2.fd < 0 ? 1 : -1; + } else if (f2.fd < MAX_LOSSLESS_FLOAT && f2.fd > MIN_LOSSLESS_FLOAT) { + // Float is within the no loss limit + if (big_to_double(aw, &f1.fd) < 0) { + j = big_sign(aw) ? -1 : 1; + } else { + j = float_comp(f1.fd, f2.fd); + } } else { - GET_DOUBLE(bw, f2); - j = float_comp(f1.fd, f2.fd); + big = double_to_big(f2.fd, big_buf); + j = big_comp(aw, big); } break; case FLOAT_SMALL: GET_DOUBLE(aw, f1); - f2.fd = signed_val(b); - j = float_comp(f1.fd, f2.fd); + if (f1.fd < MAX_LOSSLESS_FLOAT && f1.fd > MIN_LOSSLESS_FLOAT) { + // Float is within the no loss limit + f2.fd = signed_val(bw); + j = float_comp(f1.fd, f2.fd); +#if ERTS_SIZEOF_ETERM == 8 + } else if (f1.fd > (double) (MAX_SMALL + 1)) { + // Float is a positive bignum, i.e. bigger + j = 1; + } else if (f1.fd < (double) (MIN_SMALL - 1)) { + // Float is a negative bignum, i.e. smaller + j = -1; + } else { // Float is a Sint but less precise it + j = (Sint) f1.fd - signed_val(bw); + } +#else + } else { + // If float is positive it is bigger than small + j = (f1.fd > 0.0) ? 1 : -1; + } +#endif // ERTS_SIZEOF_ETERM == 8 break; case FLOAT_BIG: - if (big_to_double(bw, &f2.fd) < 0) { - j = big_sign(b) ? 1 : -1; + GET_DOUBLE(aw, f1); + if ((f1.fd < (double) (MAX_SMALL + 1)) + && (f1.fd > (double) (MIN_SMALL - 1))) { // Float is a Sint + j = big_sign(bw) ? 1 : -1; + } else if ((pow(2.0, (big_arity(bw) - 1.0) * D_EXP) - 1.0) > fabs(f1.fd)) { + // If bignum size shows that it is bigger than the abs float + j = big_sign(bw) ? 1 : -1; + } else if ((pow(2.0,(big_arity(bw))*D_EXP)-1.0) < fabs(f1.fd)) { + // If bignum size shows that it is smaller than the abs float + j = f1.fd < 0 ? -1 : 1; + } else if (f1.fd < MAX_LOSSLESS_FLOAT && f1.fd > MIN_LOSSLESS_FLOAT) { + // Float is within the no loss limit + if (big_to_double(bw, &f2.fd) < 0) { + j = big_sign(bw) ? 1 : -1; + } else { + j = float_comp(f1.fd, f2.fd); + } } else { - GET_DOUBLE(aw, f1); - j = float_comp(f1.fd, f2.fd); + big = double_to_big(f1.fd, big_buf); + j = big_comp(big, bw); } break; default: @@ -3250,10 +3322,10 @@ erts_cancel_smp_ptimer(ErtsSmpPTimer *ptimer) #endif -static Sint trim_threshold; -static Sint top_pad; -static Sint mmap_threshold; -static Sint mmap_max; +static int trim_threshold; +static int top_pad; +static int mmap_threshold; +static int mmap_max; Uint tot_bin_allocated; @@ -3276,8 +3348,8 @@ int sys_alloc_opt(int opt, int value) { #if HAVE_MALLOPT - Sint m_opt; - Sint *curr_val; + int m_opt; + int *curr_val; switch(opt) { case SYS_ALLOC_OPT_TRIM_THRESHOLD: @@ -3317,7 +3389,7 @@ sys_alloc_opt(int opt, int value) } if(mallopt(m_opt, value)) { - *curr_val = (Sint) value; + *curr_val = value; return 1; } @@ -3336,686 +3408,6 @@ sys_alloc_stat(SysAllocStat *sasp) } -#ifdef ERTS_SMP - -/* Local system block state */ - -struct { - int emergency; - long emergency_timeout; - erts_smp_cnd_t watchdog_cnd; - erts_smp_tid_t watchdog_tid; - int threads_to_block; - int have_blocker; - erts_smp_tid_t blocker_tid; - int recursive_block; - Uint32 allowed_activities; - erts_smp_tsd_key_t blockable_key; - erts_smp_mtx_t mtx; - erts_smp_cnd_t cnd; -#ifdef ERTS_ENABLE_LOCK_CHECK - int activity_changing; - int checking; -#endif -} system_block_state; - -/* Global system block state */ -erts_system_block_state_t erts_system_block_state; - - -static ERTS_INLINE int -is_blockable_thread(void) -{ - return erts_smp_tsd_get(system_block_state.blockable_key) != NULL; -} - -static ERTS_INLINE int -is_blocker(void) -{ - return (system_block_state.have_blocker - && erts_smp_equal_tids(system_block_state.blocker_tid, - erts_smp_thr_self())); -} - -#ifdef ERTS_ENABLE_LOCK_CHECK -int -erts_lc_is_blocking(void) -{ - int res; - erts_smp_mtx_lock(&system_block_state.mtx); - res = erts_smp_pending_system_block() && is_blocker(); - erts_smp_mtx_unlock(&system_block_state.mtx); - return res; -} -#endif - -static ERTS_INLINE void -block_me(void (*prepare)(void *), - void (*resume)(void *), - void *arg, - int mtx_locked, - int want_to_block, - int update_act_changing, - profile_sched_msg_q *psmq) -{ - if (prepare) - (*prepare)(arg); - - /* Locks might be held... */ - - if (!mtx_locked) - erts_smp_mtx_lock(&system_block_state.mtx); - - if (erts_smp_pending_system_block() && !is_blocker()) { - int is_blockable = is_blockable_thread(); - ASSERT(is_blockable); - - if (is_blockable) - system_block_state.threads_to_block--; - - if (erts_system_profile_flags.scheduler && psmq) { - ErtsSchedulerData *esdp = erts_get_scheduler_data(); - if (esdp) { - profile_sched_msg *msg = NULL; - - ASSERT(psmq->n < 2); - msg = &((psmq->msg)[psmq->n]); - msg->scheduler_id = esdp->no; - get_now(&(msg->Ms), &(msg->s), &(msg->us)); - msg->no_schedulers = 0; - msg->state = am_inactive; - psmq->n++; - } - } - -#ifdef ERTS_ENABLE_LOCK_CHECK - if (update_act_changing) - system_block_state.activity_changing--; -#endif - - erts_smp_cnd_broadcast(&system_block_state.cnd); - - do { - erts_smp_cnd_wait(&system_block_state.cnd, &system_block_state.mtx); - } while (erts_smp_pending_system_block() - && !(want_to_block && !system_block_state.have_blocker)); - -#ifdef ERTS_ENABLE_LOCK_CHECK - if (update_act_changing) - system_block_state.activity_changing++; -#endif - if (erts_system_profile_flags.scheduler && psmq) { - ErtsSchedulerData *esdp = erts_get_scheduler_data(); - if (esdp) { - profile_sched_msg *msg = NULL; - - ASSERT(psmq->n < 2); - msg = &((psmq->msg)[psmq->n]); - msg->scheduler_id = esdp->no; - get_now(&(msg->Ms), &(msg->s), &(msg->us)); - msg->no_schedulers = 0; - msg->state = am_active; - psmq->n++; - } - } - - if (is_blockable) - system_block_state.threads_to_block++; - } - - if (!mtx_locked) - erts_smp_mtx_unlock(&system_block_state.mtx); - - if (resume) - (*resume)(arg); -} - -void -erts_block_me(void (*prepare)(void *), - void (*resume)(void *), - void *arg) -{ - profile_sched_msg_q psmq; - psmq.n = 0; - if (prepare) - (*prepare)(arg); - -#ifdef ERTS_ENABLE_LOCK_CHECK - erts_lc_check_exact(NULL, 0); /* No locks should be locked */ -#endif - - block_me(NULL, NULL, NULL, 0, 0, 0, &psmq); - - if (erts_system_profile_flags.scheduler && psmq.n > 0) - dispatch_profile_msg_q(&psmq); - - if (resume) - (*resume)(arg); -} - -void -erts_register_blockable_thread(void) -{ - profile_sched_msg_q psmq; - psmq.n = 0; - if (!is_blockable_thread()) { - erts_smp_mtx_lock(&system_block_state.mtx); - system_block_state.threads_to_block++; - erts_smp_tsd_set(system_block_state.blockable_key, - (void *) &erts_system_block_state); - - /* Someone might be waiting for us to block... */ - if (erts_smp_pending_system_block()) - block_me(NULL, NULL, NULL, 1, 0, 0, &psmq); - erts_smp_mtx_unlock(&system_block_state.mtx); - - if (erts_system_profile_flags.scheduler && psmq.n > 0) - dispatch_profile_msg_q(&psmq); - } -} - -void -erts_unregister_blockable_thread(void) -{ - if (is_blockable_thread()) { - erts_smp_mtx_lock(&system_block_state.mtx); - system_block_state.threads_to_block--; - ASSERT(system_block_state.threads_to_block >= 0); - erts_smp_tsd_set(system_block_state.blockable_key, NULL); - - /* Someone might be waiting for us to block... */ - if (erts_smp_pending_system_block()) - erts_smp_cnd_broadcast(&system_block_state.cnd); - erts_smp_mtx_unlock(&system_block_state.mtx); - } -} - -void -erts_note_activity_begin(erts_activity_t activity) -{ - erts_smp_mtx_lock(&system_block_state.mtx); - if (erts_smp_pending_system_block()) { - Uint32 broadcast = 0; - switch (activity) { - case ERTS_ACTIVITY_GC: - broadcast = (system_block_state.allowed_activities - & ERTS_BS_FLG_ALLOW_GC); - break; - case ERTS_ACTIVITY_IO: - broadcast = (system_block_state.allowed_activities - & ERTS_BS_FLG_ALLOW_IO); - break; - case ERTS_ACTIVITY_WAIT: - broadcast = 1; - break; - default: - abort(); - break; - } - if (broadcast) - erts_smp_cnd_broadcast(&system_block_state.cnd); - } - erts_smp_mtx_unlock(&system_block_state.mtx); -} - -void -erts_check_block(erts_activity_t old_activity, - erts_activity_t new_activity, - int locked, - void (*prepare)(void *), - void (*resume)(void *), - void *arg) -{ - int do_block; - profile_sched_msg_q psmq; - - psmq.n = 0; - if (!locked && prepare) - (*prepare)(arg); - - erts_smp_mtx_lock(&system_block_state.mtx); - - /* First check if it is ok to block... */ - if (!locked) - do_block = 1; - else { - switch (old_activity) { - case ERTS_ACTIVITY_UNDEFINED: - do_block = 0; - break; - case ERTS_ACTIVITY_GC: - do_block = (system_block_state.allowed_activities - & ERTS_BS_FLG_ALLOW_GC); - break; - case ERTS_ACTIVITY_IO: - do_block = (system_block_state.allowed_activities - & ERTS_BS_FLG_ALLOW_IO); - break; - case ERTS_ACTIVITY_WAIT: - /* You are not allowed to leave activity waiting - * without supplying the possibility to block - * unlocked. - */ - erts_set_activity_error(ERTS_ACT_ERR_LEAVE_WAIT_UNLOCKED, - __FILE__, __LINE__); - do_block = 0; - break; - default: - erts_set_activity_error(ERTS_ACT_ERR_LEAVE_UNKNOWN_ACTIVITY, - __FILE__, __LINE__); - do_block = 0; - break; - } - } - - if (do_block) { - /* ... then check if it is necessary to block... */ - - switch (new_activity) { - case ERTS_ACTIVITY_UNDEFINED: - do_block = 1; - break; - case ERTS_ACTIVITY_GC: - do_block = !(system_block_state.allowed_activities - & ERTS_BS_FLG_ALLOW_GC); - break; - case ERTS_ACTIVITY_IO: - do_block = !(system_block_state.allowed_activities - & ERTS_BS_FLG_ALLOW_IO); - break; - case ERTS_ACTIVITY_WAIT: - /* No need to block if we are going to wait */ - do_block = 0; - break; - default: - erts_set_activity_error(ERTS_ACT_ERR_ENTER_UNKNOWN_ACTIVITY, - __FILE__, __LINE__); - break; - } - } - - if (do_block) { - -#ifdef ERTS_ENABLE_LOCK_CHECK - if (!locked) { - /* Only system_block_state.mtx should be held */ - erts_lc_check_exact(&system_block_state.mtx.lc, 1); - } -#endif - - block_me(NULL, NULL, NULL, 1, 0, 1, &psmq); - - } - - erts_smp_mtx_unlock(&system_block_state.mtx); - - if (erts_system_profile_flags.scheduler && psmq.n > 0) - dispatch_profile_msg_q(&psmq); - - if (!locked && resume) - (*resume)(arg); -} - - - -void -erts_set_activity_error(erts_activity_error_t error, char *file, int line) -{ - switch (error) { - case ERTS_ACT_ERR_LEAVE_WAIT_UNLOCKED: - erl_exit(1, "%s:%d: Fatal error: Leaving activity waiting without " - "supplying the possibility to block unlocked.", - file, line); - break; - case ERTS_ACT_ERR_LEAVE_UNKNOWN_ACTIVITY: - erl_exit(1, "%s:%d: Fatal error: Leaving unknown activity.", - file, line); - break; - case ERTS_ACT_ERR_ENTER_UNKNOWN_ACTIVITY: - erl_exit(1, "%s:%d: Fatal error: Leaving unknown activity.", - file, line); - break; - default: - erl_exit(1, "%s:%d: Internal error in erts_smp_set_activity()", - file, line); - break; - } - -} - - -static ERTS_INLINE erts_aint32_t -threads_not_under_control(void) -{ - erts_aint32_t res = system_block_state.threads_to_block; - - /* Waiting is always an allowed activity... */ - res -= erts_smp_atomic32_read(&erts_system_block_state.in_activity.wait); - - if (system_block_state.allowed_activities & ERTS_BS_FLG_ALLOW_GC) - res -= erts_smp_atomic32_read(&erts_system_block_state.in_activity.gc); - - if (system_block_state.allowed_activities & ERTS_BS_FLG_ALLOW_IO) - res -= erts_smp_atomic32_read(&erts_system_block_state.in_activity.io); - - if (res < 0) { - ASSERT(0); - return 0; - } - return res; -} - -/* - * erts_block_system() blocks all threads registered as blockable. - * It doesn't return until either all threads have blocked (0 is returned) - * or it has timed out (ETIMEDOUT) is returned. - * - * If allowed activities == 0, blocked threads will release all locks - * before blocking. - * - * If allowed_activities is != 0, erts_block_system() will allow blockable - * threads to continue executing as long as they are doing an allowed - * activity. When they are done with the allowed activity they will block, - * *but* they will block holding locks. Therefore, the thread calling - * erts_block_system() must *not* try to aquire any locks that might be - * held by blocked threads holding locks from allowed activities. - * - * Currently allowed_activities are: - * * ERTS_BS_FLG_ALLOW_GC Thread continues with garbage - * collection and blocks with - * main process lock on current - * process locked. - * * ERTS_BS_FLG_ALLOW_IO Thread continues with I/O - */ - -void -erts_block_system(Uint32 allowed_activities) -{ - int do_block; - profile_sched_msg_q psmq; - - psmq.n = 0; -#ifdef ERTS_ENABLE_LOCK_CHECK - erts_lc_check_exact(NULL, 0); /* No locks should be locked */ -#endif - - erts_smp_mtx_lock(&system_block_state.mtx); - - do_block = erts_smp_pending_system_block(); - if (do_block - && system_block_state.have_blocker - && erts_smp_equal_tids(system_block_state.blocker_tid, - erts_smp_thr_self())) { - ASSERT(system_block_state.recursive_block >= 0); - system_block_state.recursive_block++; - - /* You are not allowed to restrict allowed activites - in a recursive block! */ - ERTS_SMP_LC_ASSERT((system_block_state.allowed_activities - & ~allowed_activities) == 0); - } - else { - - erts_smp_atomic32_inc(&erts_system_block_state.do_block); - - /* Someone else might be waiting for us to block... */ - if (do_block) { - do_block_me: - block_me(NULL, NULL, NULL, 1, 1, 0, &psmq); - } - - ASSERT(!system_block_state.have_blocker); - system_block_state.have_blocker = 1; - system_block_state.blocker_tid = erts_smp_thr_self(); - system_block_state.allowed_activities = allowed_activities; - - if (is_blockable_thread()) - system_block_state.threads_to_block--; - - while (threads_not_under_control() && !system_block_state.emergency) - erts_smp_cnd_wait(&system_block_state.cnd, &system_block_state.mtx); - - if (system_block_state.emergency) { - system_block_state.have_blocker = 0; - goto do_block_me; - } - } - - erts_smp_mtx_unlock(&system_block_state.mtx); - - if (erts_system_profile_flags.scheduler && psmq.n > 0 ) - dispatch_profile_msg_q(&psmq); -} - -/* - * erts_emergency_block_system() should only be called when we are - * about to write a crash dump... - */ - -int -erts_emergency_block_system(long timeout, Uint32 allowed_activities) -{ - int res = 0; - long another_blocker; - - erts_smp_mtx_lock(&system_block_state.mtx); - - if (system_block_state.emergency) { - /* Argh... */ - res = EINVAL; - goto done; - } - - another_blocker = erts_smp_pending_system_block(); - system_block_state.emergency = 1; - erts_smp_atomic32_inc(&erts_system_block_state.do_block); - - if (another_blocker) { - if (is_blocker()) { - erts_smp_atomic32_dec(&erts_system_block_state.do_block); - res = 0; - goto done; - } - /* kick the other blocker */ - erts_smp_cnd_broadcast(&system_block_state.cnd); - while (system_block_state.have_blocker) - erts_smp_cnd_wait(&system_block_state.cnd, &system_block_state.mtx); - } - - ASSERT(!system_block_state.have_blocker); - system_block_state.have_blocker = 1; - system_block_state.blocker_tid = erts_smp_thr_self(); - system_block_state.allowed_activities = allowed_activities; - - if (is_blockable_thread()) - system_block_state.threads_to_block--; - - if (timeout < 0) { - while (threads_not_under_control()) - erts_smp_cnd_wait(&system_block_state.cnd, &system_block_state.mtx); - } - else { - system_block_state.emergency_timeout = timeout; - erts_smp_cnd_signal(&system_block_state.watchdog_cnd); - - while (system_block_state.emergency_timeout >= 0 - && threads_not_under_control()) { - erts_smp_cnd_wait(&system_block_state.cnd, - &system_block_state.mtx); - } - } - done: - erts_smp_mtx_unlock(&system_block_state.mtx); - return res; -} - -void -erts_release_system(void) -{ - long do_block; - profile_sched_msg_q psmq; - - psmq.n = 0; - -#ifdef ERTS_ENABLE_LOCK_CHECK - erts_lc_check_exact(NULL, 0); /* No locks should be locked */ -#endif - - erts_smp_mtx_lock(&system_block_state.mtx); - ASSERT(is_blocker()); - - ASSERT(system_block_state.recursive_block >= 0); - - if (system_block_state.recursive_block) - system_block_state.recursive_block--; - else { - do_block = erts_smp_atomic32_dectest(&erts_system_block_state.do_block); - system_block_state.have_blocker = 0; - if (is_blockable_thread()) - system_block_state.threads_to_block++; - else - do_block = 0; - - /* Someone else might be waiting for us to block... */ - if (do_block) - block_me(NULL, NULL, NULL, 1, 0, 0, &psmq); - else - erts_smp_cnd_broadcast(&system_block_state.cnd); - } - - erts_smp_mtx_unlock(&system_block_state.mtx); - - if (erts_system_profile_flags.scheduler && psmq.n > 0) - dispatch_profile_msg_q(&psmq); -} - -#ifdef ERTS_ENABLE_LOCK_CHECK - -void -erts_lc_activity_change_begin(void) -{ - erts_smp_mtx_lock(&system_block_state.mtx); - system_block_state.activity_changing++; - erts_smp_mtx_unlock(&system_block_state.mtx); -} - -void -erts_lc_activity_change_end(void) -{ - erts_smp_mtx_lock(&system_block_state.mtx); - system_block_state.activity_changing--; - if (system_block_state.checking && !system_block_state.activity_changing) - erts_smp_cnd_broadcast(&system_block_state.cnd); - erts_smp_mtx_unlock(&system_block_state.mtx); -} - -#endif - -int -erts_is_system_blocked(erts_activity_t allowed_activities) -{ - int blkd; - - erts_smp_mtx_lock(&system_block_state.mtx); - blkd = (erts_smp_pending_system_block() - && system_block_state.have_blocker - && erts_smp_equal_tids(system_block_state.blocker_tid, - erts_smp_thr_self()) - && !(system_block_state.allowed_activities & ~allowed_activities)); -#ifdef ERTS_ENABLE_LOCK_CHECK - if (blkd) { - system_block_state.checking = 1; - while (system_block_state.activity_changing) - erts_smp_cnd_wait(&system_block_state.cnd, &system_block_state.mtx); - system_block_state.checking = 0; - blkd = !threads_not_under_control(); - } -#endif - erts_smp_mtx_unlock(&system_block_state.mtx); - return blkd; -} - -static void * -emergency_watchdog(void *unused) -{ - erts_smp_mtx_lock(&system_block_state.mtx); - while (1) { - long timeout; - while (system_block_state.emergency_timeout < 0) - erts_smp_cnd_wait(&system_block_state.watchdog_cnd, &system_block_state.mtx); - timeout = system_block_state.emergency_timeout; - erts_smp_mtx_unlock(&system_block_state.mtx); - - if (erts_disable_tolerant_timeofday) - erts_milli_sleep(timeout); - else { - SysTimeval to; - erts_get_timeval(&to); - to.tv_sec += timeout / 1000; - to.tv_usec += timeout % 1000; - - while (1) { - SysTimeval curr; - erts_milli_sleep(timeout); - erts_get_timeval(&curr); - if (curr.tv_sec > to.tv_sec - || (curr.tv_sec == to.tv_sec && curr.tv_usec >= to.tv_usec)) { - break; - } - timeout = (to.tv_sec - curr.tv_sec)*1000; - timeout += (to.tv_usec - curr.tv_usec)/1000; - } - } - - erts_smp_mtx_lock(&system_block_state.mtx); - system_block_state.emergency_timeout = -1; - erts_smp_cnd_broadcast(&system_block_state.cnd); - } - erts_smp_mtx_unlock(&system_block_state.mtx); - return NULL; -} - -void -erts_system_block_init(void) -{ - erts_smp_thr_opts_t thr_opts = ERTS_SMP_THR_OPTS_DEFAULT_INITER; - /* Local state... */ - system_block_state.emergency = 0; - system_block_state.emergency_timeout = -1; - erts_smp_cnd_init(&system_block_state.watchdog_cnd); - system_block_state.threads_to_block = 0; - system_block_state.have_blocker = 0; - /* system_block_state.block_tid */ - system_block_state.recursive_block = 0; - system_block_state.allowed_activities = 0; - erts_smp_tsd_key_create(&system_block_state.blockable_key); - erts_smp_mtx_init(&system_block_state.mtx, "system_block"); - erts_smp_cnd_init(&system_block_state.cnd); -#ifdef ERTS_ENABLE_LOCK_CHECK - system_block_state.activity_changing = 0; - system_block_state.checking = 0; -#endif - - thr_opts.suggested_stack_size = 8; - erts_smp_thr_create(&system_block_state.watchdog_tid, - emergency_watchdog, - NULL, - &thr_opts); - - /* Global state... */ - - erts_smp_atomic32_init(&erts_system_block_state.do_block, 0); - erts_smp_atomic32_init(&erts_system_block_state.in_activity.wait, 0); - erts_smp_atomic32_init(&erts_system_block_state.in_activity.gc, 0); - erts_smp_atomic32_init(&erts_system_block_state.in_activity.io, 0); - - /* Make sure blockable threads unregister when exiting... */ - erts_smp_install_exit_handler(erts_unregister_blockable_thread); -} - - -#endif /* #ifdef ERTS_SMP */ - char * erts_read_env(char *key) { diff --git a/erts/emulator/drivers/common/efile_drv.c b/erts/emulator/drivers/common/efile_drv.c index f0ff3f54c5..52f1b5312b 100644 --- a/erts/emulator/drivers/common/efile_drv.c +++ b/erts/emulator/drivers/common/efile_drv.c @@ -69,6 +69,7 @@ #define FILE_RESP_EOF 8 #define FILE_RESP_FNAME 9 #define FILE_RESP_ALL_DATA 10 +#define FILE_RESP_LFNAME 11 /* Options */ @@ -184,6 +185,7 @@ static ErlDrvSysInfo sys_info; # define RESBUFSIZE BUFSIZ #endif +#define READDIR_CHUNKS (5) @@ -317,15 +319,16 @@ struct t_preadv { Sint64 offsets[1]; }; -#define READDIR_BUFSIZE (8*1024) -#if READDIR_BUFSIZE < (FILENAME_CHARSIZE*2*(MAXPATHLEN+1)) +#define READDIR_BUFSIZE (8*1024)*READDIR_CHUNKS +#if READDIR_BUFSIZE < (1 + (2 + MAXPATHLEN)*FILENAME_CHARSIZE*READDIR_CHUNKS) # undef READDIR_BUFSIZE -# define READDIR_BUFSIZE (FILENAME_CHARSIZE*2*(MAXPATHLEN+1)) +# define READDIR_BUFSIZE (1 + (2 + MAXPATHLEN)*FILENAME_CHARSIZE*READDIR_CHUNKS) #endif struct t_readdir_buf { - struct t_readdir_buf *next; - char buf[READDIR_BUFSIZE]; + struct t_readdir_buf *next; + size_t n; + char buf[READDIR_BUFSIZE]; }; struct t_data @@ -1598,54 +1601,46 @@ static void invoke_lseek(void *data) static void invoke_readdir(void *data) { struct t_data *d = (struct t_data *) data; - int s; char *p = NULL; - int buf_sz = 0; - size_t tmp_bs; + size_t file_bs; + size_t n = 0, total = 0; + struct t_readdir_buf *b = NULL; + int res = 0; d->again = 0; d->errInfo.posix_errno = 0; - while (1) { - char *str; - if (buf_sz < (4 /* sz */ + 1 /* cmd */ + - FILENAME_CHARSIZE*(MAXPATHLEN + 1))) { - struct t_readdir_buf *b; - if (p) { - put_int32(0, p); /* EOB */ - } - b = EF_SAFE_ALLOC(sizeof(struct t_readdir_buf)); - b->next = NULL; - if (d->c.read_dir.last_buf) - d->c.read_dir.last_buf->next = b; - else - d->c.read_dir.first_buf = b; - d->c.read_dir.last_buf = b; - p = &b->buf[0]; - buf_sz = READDIR_BUFSIZE - 4/* EOB */; - } - - p[4] = FILE_RESP_FNAME; - buf_sz -= 4 + 1; - str = p + 4 + 1; - ASSERT(buf_sz >= MAXPATHLEN + 1); - tmp_bs = buf_sz; - s = efile_readdir(&d->errInfo, d->b, &d->dir_handle, str, &tmp_bs); - - if (s) { - put_int32(tmp_bs + 1 /* 1 byte for opcode */, p); - p += 4 + tmp_bs + 1; - ASSERT(p == (str + tmp_bs)); - buf_sz -= tmp_bs; - } - else { - put_int32(1, p); - p += 4 + 1; - put_int32(0, p); /* EOB */ - d->result_ok = (d->errInfo.posix_errno == 0); - break; + do { + total = READDIR_BUFSIZE; + n = 1; + b = EF_SAFE_ALLOC(sizeof(struct t_readdir_buf)); + b->next = NULL; + + if (d->c.read_dir.last_buf) { + d->c.read_dir.last_buf->next = b; + } else { + d->c.read_dir.first_buf = b; } - } + d->c.read_dir.last_buf = b; + + p = &b->buf[0]; + p[0] = FILE_RESP_LFNAME; + file_bs = READDIR_BUFSIZE - n; + + do { + res = efile_readdir(&d->errInfo, d->b, &d->dir_handle, p + n + 2, &file_bs); + + if (res) { + put_int16((Uint16)file_bs, p + n); + n += 2 + file_bs; + file_bs = READDIR_BUFSIZE - n; + } + } while( res && ((total - n - 2) >= MAXPATHLEN*FILENAME_CHARSIZE)); + + b->n = n; + } while(res); + + d->result_ok = (d->errInfo.posix_errno == 0); } static void invoke_open(void *data) @@ -2053,30 +2048,24 @@ file_async_ready(ErlDrvData e, ErlDrvThreadData data) free_data(data); break; case FILE_READDIR: - if (!d->result_ok) + if (!d->result_ok) { reply_error(desc, &d->errInfo); - else { + } else { struct t_readdir_buf *b1 = d->c.read_dir.first_buf; + char op = FILE_RESP_LFNAME; + TRACE_C('R'); ASSERT(b1); + while (b1) { struct t_readdir_buf *b2 = b1; char *p = &b1->buf[0]; - int sz = get_int32(p); - while (sz) { /* 0 == EOB */ - p += 4; - if (sz - 1 > 0) { - driver_output2(desc->port, p, 1, p+1, sz-1); - } else { - driver_output2(desc->port, p, 1, NULL, 0); - } - p += sz; - sz = get_int32(p); - } + driver_output2(desc->port, p, 1, p + 1, b1->n - 1); b1 = b1->next; EF_FREE(b2); } - + driver_output2(desc->port, &op, 1, NULL, 0); + d->c.read_dir.first_buf = NULL; d->c.read_dir.last_buf = NULL; } @@ -2126,6 +2115,7 @@ file_async_ready(ErlDrvData e, ErlDrvThreadData data) cq_execute(desc); } + /********************************************************************* * Driver entry point -> output */ @@ -2246,19 +2236,46 @@ file_output(ErlDrvData e, char* buf, int count) #endif { size_t resbufsize; - char resbuf[RESBUFSIZE+1]; + size_t n = 0, total = 0; + int res = 0; + char resbuf[READDIR_BUFSIZE]; + EFILE_DIR_HANDLE dir_handle; /* Handle to open directory. */ + total = READDIR_BUFSIZE; errInfo.posix_errno = 0; - dir_handle = NULL; - resbuf[0] = FILE_RESP_FNAME; - resbufsize = RESBUFSIZE; - - while (efile_readdir(&errInfo, name, &dir_handle, - resbuf+1, &resbufsize)) { - driver_output2(desc->port, resbuf, 1, resbuf+1, resbufsize); - resbufsize = RESBUFSIZE; - } + dir_handle = NULL; + resbuf[0] = FILE_RESP_LFNAME; + + /* Fill the buffer with multiple directory listings before sending it to the + * receiving process. READDIR_CHUNKS is minimum number of files sent to the + * receiver. + * Format for each driver_output2: + * ------------------------------------ + * | Type | Len | Filename | ... + * | 1 byte | 2 bytes | Len bytes | ... + * ------------------------------------ + */ + + do { + n = 1; + resbufsize = READDIR_BUFSIZE - n; + + do { + res = efile_readdir(&errInfo, name, &dir_handle, resbuf + n + 2, &resbufsize); + + if (res) { + put_int16((Uint16)resbufsize, resbuf + n); + n += 2 + resbufsize; + resbufsize = READDIR_BUFSIZE - n; + } + } while( res && ((total - n - 2) >= MAXPATHLEN*FILENAME_CHARSIZE)); + + if (n > 1) { + driver_output2(desc->port, resbuf, 1, resbuf + 1, n - 1); + } + } while(res); + if (errInfo.posix_errno != 0) { reply_error(desc, &errInfo); return; @@ -2476,13 +2493,20 @@ file_flush(ErlDrvData e) { static int file_control(ErlDrvData e, unsigned int command, char* buf, int len, char **rbuf, int rlen) { + /* + * warning: variable ‘desc’ set but not used + * [-Wunused-but-set-variable] + * ... no kidding ... + * + * file_descriptor *desc = (file_descriptor *)e; switch (command) { default: return 0; - } /* switch (command) */ + } ASSERT(0); - desc = NULL; /* XXX Avoid warning while empty switch */ + desc = NULL; + */ return 0; } diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c index 43114c6039..426917bd2c 100644 --- a/erts/emulator/drivers/common/inet_drv.c +++ b/erts/emulator/drivers/common/inet_drv.c @@ -9307,7 +9307,7 @@ static int tcp_inet_output(tcp_descriptor* desc, HANDLE event) goto done; } } -#endif /* SOCKOPT_CONNECT_STAT */ +#endif /* SO_ERROR */ #endif /* !__WIN32__ */ desc->inet.state = TCP_STATE_CONNECTED; @@ -10112,7 +10112,7 @@ static int packet_inet_output(udp_descriptor* udesc, HANDLE event) goto done; } } -#endif /* SOCKOPT_CONNECT_STAT */ +#endif /* SO_ERROR */ #endif /* !__WIN32__ */ desc->state = PACKET_STATE_CONNECTED; diff --git a/erts/emulator/drivers/unix/ttsl_drv.c b/erts/emulator/drivers/unix/ttsl_drv.c index d782b044a9..45d39a559f 100644 --- a/erts/emulator/drivers/unix/ttsl_drv.c +++ b/erts/emulator/drivers/unix/ttsl_drv.c @@ -242,7 +242,7 @@ static ErlDrvData ttysl_start(ErlDrvPort port, char* buf) #ifndef HAVE_TERMCAP return ERL_DRV_ERROR_GENERAL; #else - char *s, *t, c, *l; + char *s, *t, *l; int canon, echo, sig; /* Terminal characteristics */ int flag; extern int using_oldshell; /* set this to let the rest of erts know */ @@ -262,7 +262,6 @@ static ErlDrvData ttysl_start(ErlDrvPort port, char* buf) s++; /* Find end of this argument (start of next) and insert NUL. */ if ((t = strchr(s, ' '))) { - c = *t; *t = '\0'; } if ((flag = ((*s == '+') ? 1 : ((*s == '-') ? -1 : 0)))) { diff --git a/erts/emulator/drivers/win32/win_efile.c b/erts/emulator/drivers/win32/win_efile.c index 931bb196f1..931bb196f1 100755..100644 --- a/erts/emulator/drivers/win32/win_efile.c +++ b/erts/emulator/drivers/win32/win_efile.c diff --git a/erts/emulator/hipe/hipe_abi.txt b/erts/emulator/hipe/hipe_abi.txt index d0ec162342..9d4726de9d 100644 --- a/erts/emulator/hipe/hipe_abi.txt +++ b/erts/emulator/hipe/hipe_abi.txt @@ -62,7 +62,7 @@ exceptional condition, it puts an error code in p->freason and returns THE_NON_VALUE (zero, except in debug mode). If p->freason == TRAP, then the BIF redirects its call to some -other function, given by p->def_arg_reg[]. +other function, given by p->i The BIF and the new callee may have different arities. The "hipe_${ARCH}_bifs.m4" macro files take care of these issues diff --git a/erts/emulator/hipe/hipe_amd64_bifs.m4 b/erts/emulator/hipe/hipe_amd64_bifs.m4 index 0ba763cbea..97a8267647 100644 --- a/erts/emulator/hipe/hipe_amd64_bifs.m4 +++ b/erts/emulator/hipe/hipe_amd64_bifs.m4 @@ -20,24 +20,37 @@ changecom(`/*', `*/')dnl include(`hipe/hipe_amd64_asm.m4') +#`include' "config.h" #`include' "hipe_literals.h" + `#if THE_NON_VALUE == 0 #define TEST_GOT_EXN testq %rax, %rax #else #define TEST_GOT_EXN cmpq $THE_NON_VALUE, %rax #endif' -`#define TEST_GOT_MBUF movq P_MBUF(P), %rdx; testq %rdx, %rdx; jnz 3f; 2: -#define JOIN3(A,B,C) A##B##C -#define HANDLE_GOT_MBUF(ARITY) 3: call JOIN3(nbif_,ARITY,_gc_after_bif); jmp 2b' +define(TEST_GOT_MBUF,`movq P_MBUF(P), %rdx # `TEST_GOT_MBUF' + testq %rdx, %rdx + jnz 3f +2:') +define(HANDLE_GOT_MBUF,` +3: call nbif_$1_gc_after_bif # `HANDLE_GOT_MBUF' + jmp 2b') + +`#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP) +# define CALL_BIF(F) movq $CSYM(F), P_BIF_CALLEE(P); call CSYM(hipe_debug_bif_wrapper) +#else +# define CALL_BIF(F) call CSYM(F) +#endif' /* * standard_bif_interface_1(nbif_name, cbif_name) * standard_bif_interface_2(nbif_name, cbif_name) * standard_bif_interface_3(nbif_name, cbif_name) + * standard_bif_interface_0(nbif_name, cbif_name) * - * Generate native interface for a BIF with 1-3 parameters and + * Generate native interface for a BIF with 0-3 parameters and * standard failure mode. */ define(standard_bif_interface_1, @@ -54,7 +67,11 @@ ASYM($1): /* make the call on the C stack */ SWITCH_ERLANG_TO_C - call CSYM($2) + pushq %rsi + movq %rsp, %rsi /* Eterm* BIF__ARGS */ + sub $(8), %rsp /* stack frame 16-byte alignment */ + CALL_BIF($2) + add $(1*8 + 8), %rsp TEST_GOT_MBUF SWITCH_C_TO_ERLANG @@ -82,7 +99,11 @@ ASYM($1): /* make the call on the C stack */ SWITCH_ERLANG_TO_C - call CSYM($2) + pushq %rdx + pushq %rsi + movq %rsp, %rsi /* Eterm* BIF__ARGS */ + CALL_BIF($2) + add $(2*8), %rsp TEST_GOT_MBUF SWITCH_C_TO_ERLANG @@ -111,7 +132,13 @@ ASYM($1): /* make the call on the C stack */ SWITCH_ERLANG_TO_C - call CSYM($2) + pushq %rcx + pushq %rdx + pushq %rsi + movq %rsp, %rsi /* Eterm* BIF__ARGS */ + sub $(8), %rsp /* stack frame 16-byte alignment */ + CALL_BIF($2) + add $(3*8 + 8), %rsp TEST_GOT_MBUF SWITCH_C_TO_ERLANG @@ -124,13 +151,7 @@ ASYM($1): TYPE_FUNCTION(ASYM($1)) #endif') -/* - * fail_bif_interface_0(nbif_name, cbif_name) - * - * Generate native interface for a BIF with 0 parameters and - * standard failure mode. - */ -define(fail_bif_interface_0, +define(standard_bif_interface_0, ` #ifndef HAVE_$1 #`define' HAVE_$1 @@ -143,7 +164,7 @@ ASYM($1): /* make the call on the C stack */ SWITCH_ERLANG_TO_C - call CSYM($2) + CALL_BIF($2) TEST_GOT_MBUF SWITCH_C_TO_ERLANG diff --git a/erts/emulator/hipe/hipe_arm_bifs.m4 b/erts/emulator/hipe/hipe_arm_bifs.m4 index 3664fb6502..e0c6f09796 100644 --- a/erts/emulator/hipe/hipe_arm_bifs.m4 +++ b/erts/emulator/hipe/hipe_arm_bifs.m4 @@ -20,18 +20,27 @@ changecom(`/*', `*/')dnl include(`hipe/hipe_arm_asm.m4') +#`include' "config.h" #`include' "hipe_literals.h" .text .p2align 2 -`#define JOIN3(A,B,C) A##B##C -#define TEST_GOT_MBUF(ARITY) ldr r1, [P, #P_MBUF]; cmp r1, #0; blne JOIN3(nbif_,ARITY,_gc_after_bif)' +`#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP) +# define CALL_BIF(F) mov r14, #F; str r14, [r0, #P_BIF_CALLEE]; bl hipe_debug_bif_wrapper +#else +# define CALL_BIF(F) bl F +#endif' + +define(TEST_GOT_MBUF,`ldr r1, [P, #P_MBUF] /* `TEST_GOT_MBUF' */ + cmp r1, #0 + blne nbif_$1_gc_after_bif') /* * standard_bif_interface_1(nbif_name, cbif_name) * standard_bif_interface_2(nbif_name, cbif_name) * standard_bif_interface_3(nbif_name, cbif_name) + * standard_bif_interface_0(nbif_name, cbif_name) * * Generate native interface for a BIF with 1-3 parameters and * standard failure mode. @@ -48,7 +57,9 @@ $1: /* Save caller-save registers and call the C function. */ SAVE_CONTEXT_BIF - bl $2 + str r1, [r0, #P_ARG0] /* Store BIF__ARGS in def_arg_reg[] */ + add r1, r0, #P_ARG0 + CALL_BIF($2) TEST_GOT_MBUF(1) /* Restore registers. Check for exception. */ @@ -73,7 +84,10 @@ $1: /* Save caller-save registers and call the C function. */ SAVE_CONTEXT_BIF - bl $2 + str r1, [r0, #P_ARG0] /* Store BIF__ARGS in def_arg_reg[] */ + str r2, [r0, #P_ARG1] + add r1, r0, #P_ARG0 + CALL_BIF($2) TEST_GOT_MBUF(2) /* Restore registers. Check for exception. */ @@ -99,7 +113,11 @@ $1: /* Save caller-save registers and call the C function. */ SAVE_CONTEXT_BIF - bl $2 + str r1, [r0, #P_ARG0] /* Store BIF__ARGS in def_arg_reg[] */ + str r2, [r0, #P_ARG1] + str r3, [r0, #P_ARG2] + add r1, r0, #P_ARG0 + CALL_BIF($2) TEST_GOT_MBUF(3) /* Restore registers. Check for exception. */ @@ -111,13 +129,7 @@ $1: .type $1, %function #endif') -/* - * fail_bif_interface_0(nbif_name, cbif_name) - * - * Generate native interface for a BIF with 0 parameters and - * standard failure mode. - */ -define(fail_bif_interface_0, +define(standard_bif_interface_0, ` #ifndef HAVE_$1 #`define' HAVE_$1 @@ -128,7 +140,8 @@ $1: /* Save caller-save registers and call the C function. */ SAVE_CONTEXT_BIF - bl $2 + /* ignore empty BIF__ARGS */ + CALL_BIF($2) TEST_GOT_MBUF(0) /* Restore registers. Check for exception. */ diff --git a/erts/emulator/hipe/hipe_bif0.h b/erts/emulator/hipe/hipe_bif0.h index f02e8862dc..c512d66f9d 100644 --- a/erts/emulator/hipe/hipe_bif0.h +++ b/erts/emulator/hipe/hipe_bif0.h @@ -29,7 +29,7 @@ extern Uint *hipe_bifs_find_pc_from_mfa(Eterm mfa); extern void hipe_mfa_info_table_init(void); extern void *hipe_get_remote_na(Eterm m, Eterm f, unsigned int a); -extern Eterm hipe_find_na_or_make_stub(Process*, Eterm, Eterm, Eterm); +extern BIF_RETTYPE hipe_find_na_or_make_stub(BIF_ALIST_3); extern int hipe_find_mfa_from_ra(const void *ra, Eterm *m, Eterm *f, unsigned int *a); #if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__arm__) extern void *hipe_mfa_get_trampoline(Eterm m, Eterm f, unsigned int a); diff --git a/erts/emulator/hipe/hipe_bif2.c b/erts/emulator/hipe/hipe_bif2.c index 2660f74a82..ee97541e15 100644 --- a/erts/emulator/hipe/hipe_bif2.c +++ b/erts/emulator/hipe/hipe_bif2.c @@ -166,3 +166,26 @@ BIF_RETTYPE hipe_bifs_show_message_area_0(BIF_ALIST_0) BIF_RET(am_false); #endif } + +#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP) + +BIF_RETTYPE hipe_debug_bif_wrapper(BIF_ALIST_1); + +# define ERTS_SMP_REQ_PROC_MAIN_LOCK(P) \ + if ((P)) erts_proc_lc_require_lock((P), ERTS_PROC_LOCK_MAIN) +# define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P) \ + if ((P)) erts_proc_lc_unrequire_lock((P), ERTS_PROC_LOCK_MAIN) + +BIF_RETTYPE hipe_debug_bif_wrapper(BIF_ALIST_1) +{ + typedef BIF_RETTYPE Bif(BIF_ALIST_1); + Bif* fp = (Bif*) (BIF_P->hipe.bif_callee); + BIF_RETTYPE res; + ERTS_SMP_UNREQ_PROC_MAIN_LOCK(BIF_P); + res = (*fp)(BIF_P, BIF__ARGS); + ERTS_SMP_REQ_PROC_MAIN_LOCK(BIF_P); + return res; +} + +#endif /* ERTS_ENABLE_LOCK_CHECK && ERTS_SMP */ + diff --git a/erts/emulator/hipe/hipe_bif_list.m4 b/erts/emulator/hipe/hipe_bif_list.m4 index 083788997b..48c7c1bc9b 100644 --- a/erts/emulator/hipe/hipe_bif_list.m4 +++ b/erts/emulator/hipe/hipe_bif_list.m4 @@ -70,24 +70,18 @@ ****************************************************************/ /* + * standard_bif_interface_0(nbif_name, cbif_name) * standard_bif_interface_1(nbif_name, cbif_name) * standard_bif_interface_2(nbif_name, cbif_name) * standard_bif_interface_3(nbif_name, cbif_name) * - * A BIF with implicit P parameter, 1-3 ordinary parameters, + * A BIF with implicit P parameter, 0-3 ordinary parameters, * which may fail. * HP and FCALLS may be read and updated. * HP_LIMIT, NSP, NSP_LIMIT, and NRA may not be accessed. */ /* - * fail_bif_interface_0(nbif_name, cbif_name) - * - * A zero-arity BIF which may fail, otherwise - * identical to standard_bif_interface_N. - */ - -/* * nofail_primop_interface_0(nbif_name, cbif_name) * nofail_primop_interface_1(nbif_name, cbif_name) * nofail_primop_interface_2(nbif_name, cbif_name) @@ -150,8 +144,7 @@ /* * Zero-arity BIFs that can fail. */ -fail_bif_interface_0(nbif_memory_0, memory_0) -fail_bif_interface_0(nbif_processes_0, processes_0) +standard_bif_interface_0(nbif_processes_0, processes_0) /* * BIFs and primops that may do a GC (change heap limit and walk the native stack). @@ -176,10 +169,10 @@ gc_bif_interface_0(nbif_hipe_bifs_nstack_used_size_0, hipe_bifs_nstack_used_size /* * Arithmetic operators called indirectly by the HiPE compiler. */ -standard_bif_interface_2(nbif_add_2, erts_mixed_plus) -standard_bif_interface_2(nbif_sub_2, erts_mixed_minus) -standard_bif_interface_2(nbif_mul_2, erts_mixed_times) -standard_bif_interface_2(nbif_div_2, erts_mixed_div) +standard_bif_interface_2(nbif_add_2, splus_2) +standard_bif_interface_2(nbif_sub_2, sminus_2) +standard_bif_interface_2(nbif_mul_2, stimes_2) +standard_bif_interface_2(nbif_div_2, div_2) standard_bif_interface_2(nbif_intdiv_2, intdiv_2) standard_bif_interface_2(nbif_rem_2, rem_2) standard_bif_interface_2(nbif_bsl_2, bsl_2) @@ -261,11 +254,6 @@ noproc_primop_interface_1(nbif_atomic_inc, hipe_atomic_inc) ',)dnl /* - * Implement standard_bif_interface_0 as nofail_primop_interface_0. - */ -define(standard_bif_interface_0,`nofail_primop_interface_0($1, $2)') - -/* * Standard BIFs. * BIF_LIST(ModuleAtom,FunctionAtom,Arity,CFun,Index) */ diff --git a/erts/emulator/hipe/hipe_mkliterals.c b/erts/emulator/hipe/hipe_mkliterals.c index bced90785d..61e15f1d58 100644 --- a/erts/emulator/hipe/hipe_mkliterals.c +++ b/erts/emulator/hipe/hipe_mkliterals.c @@ -1,9 +1,8 @@ /* * %CopyrightBegin% - - * + * * Copyright Ericsson AB 2001-2011. All Rights Reserved. - * + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the @@ -212,6 +211,11 @@ static const unsigned int CRCTABLE[256] = { 0xB40BBE37, 0xC30C8EA1, 0x5A05DF1B, 0x2D02EF8D, }; +/* For hipe cross compiler. Hard code all values. + No calls by hipe compiler to query the running emulator. +*/ +static int is_xcomp = 0; + /* * The algorithm for calculating the 32 bit CRC checksum is based upon * documentation and algorithms provided by Dr. Ross N. Williams in the @@ -243,7 +247,7 @@ crc_update_buf(unsigned int crc_value, } static unsigned int -crc_update_int(unsigned int crc_value, const unsigned int *p) +crc_update_int(unsigned int crc_value, const int *p) { return crc_update_buf(crc_value, p, sizeof *p); } @@ -256,7 +260,7 @@ crc_update_int(unsigned int crc_value, const unsigned int *p) */ static const struct literal { const char *name; - unsigned int value; + int value; } literals[] = { /* Field offsets in a process struct */ { "P_HP", offsetof(struct process, htop) }, @@ -289,6 +293,9 @@ static const struct literal { { "P_NRA", offsetof(struct process, hipe.nra) }, #endif { "P_NARITY", offsetof(struct process, hipe.narity) }, +# if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP) + { "P_BIF_CALLEE", offsetof(struct process, hipe.bif_callee) }, +# endif #endif /* HIPE */ /* process flags bits */ @@ -298,7 +305,7 @@ static const struct literal { { "FREASON_TRAP", TRAP }, /* special Erlang constants */ - { "THE_NON_VALUE", THE_NON_VALUE }, + { "THE_NON_VALUE", (int)THE_NON_VALUE }, /* funs */ #ifdef HIPE @@ -452,7 +459,7 @@ static const struct rts_param { unsigned int nr; const char *name; unsigned int is_defined; - unsigned int value; + int value; } rts_params[] = { { 1, "P_OFF_HEAP_FUNS", #if !defined(HYBRID) @@ -528,12 +535,12 @@ static void compute_crc(void) static void c_define_literal(FILE *fp, const struct literal *literal) { - fprintf(fp, "#define %s %u\n", literal->name, literal->value); + fprintf(fp, "#define %s %d\n", literal->name, literal->value); } static void e_define_literal(FILE *fp, const struct literal *literal) { - fprintf(fp, "-define(%s, %u).\n", literal->name, literal->value); + fprintf(fp, "-define(%s, %d).\n", literal->name, literal->value); } static void print_literals(FILE *fp, void (*print_literal)(FILE*, const struct literal*)) @@ -560,7 +567,7 @@ static void print_atom_literals(FILE *fp, void (*print_atom_literal)(FILE*, cons static void c_define_param(FILE *fp, const struct rts_param *param) { if (param->is_defined) - fprintf(fp, "#define %s %u\n", param->name, param->value); + fprintf(fp, "#define %s %d\n", param->name, param->value); } static void c_case_param(FILE *fp, const struct rts_param *param) @@ -568,7 +575,7 @@ static void c_case_param(FILE *fp, const struct rts_param *param) fprintf(fp, " \\\n"); fprintf(fp, "\tcase %u: ", param->nr); if (param->is_defined) - fprintf(fp, "value = %u", param->value); + fprintf(fp, "value = %d", param->value); else fprintf(fp, "is_defined = 0"); fprintf(fp, "; break;"); @@ -576,7 +583,15 @@ static void c_case_param(FILE *fp, const struct rts_param *param) static void e_define_param(FILE *fp, const struct rts_param *param) { - fprintf(fp, "-define(%s, hipe_bifs:get_rts_param(%u)).\n", param->name, param->nr); + if (is_xcomp) { + if (param->is_defined) + fprintf(fp, "-define(%s, %d).\n", param->name, param->value); + else + fprintf(fp, "-define(%s, []).\n", param->name); + } + else { + fprintf(fp, "-define(%s, hipe_bifs:get_rts_param(%u)).\n", param->name, param->nr); + } } static void print_params(FILE *fp, void (*print_param)(FILE*,const struct rts_param*)) @@ -613,19 +628,40 @@ static int do_e(FILE *fp, const char* this_exe) fprintf(fp, "\n"); print_params(fp, e_define_param); fprintf(fp, "\n"); - fprintf(fp, "-define(HIPE_SYSTEM_CRC, hipe_bifs:system_crc(%u)).\n", literals_crc); + if (is_xcomp) { + fprintf(fp, "-define(HIPE_SYSTEM_CRC, %u).\n", system_crc); + } + else { + fprintf(fp, "-define(HIPE_SYSTEM_CRC, hipe_bifs:system_crc(%u)).\n", + literals_crc); + } return 0; } int main(int argc, const char **argv) { + int i; + int (*do_func_ptr)(FILE *, const char*) = NULL; + compute_crc(); - if (argc == 2) { - if (strcmp(argv[1], "-c") == 0) - return do_c(stdout, argv[0]); - if (strcmp(argv[1], "-e") == 0) - return do_e(stdout, argv[0]); + for (i = 1; i < argc; i++) { + if (strcmp(argv[i], "-c") == 0) + do_func_ptr = &do_c; + else if (strcmp(argv[i], "-e") == 0) + do_func_ptr = &do_e; + else if (strcmp(argv[i], "-x") == 0) + is_xcomp = 1; + else + goto error; + } + if (do_func_ptr) { + return do_func_ptr(stdout, argv[0]); } - fprintf(stderr, "usage: %s [-c | -e] > output-file\n", argv[0]); +error: + fprintf(stderr, "usage: %s [-x] [-c | -e] > output-file\n" + "\t-c\tC header file\n" + "\t-e\tErlang header file\n" + "\t-x\tCross compile. No dependencies to compiling emulator\n", + argv[0]); return 1; } diff --git a/erts/emulator/hipe/hipe_mode_switch.c b/erts/emulator/hipe/hipe_mode_switch.c index e3e8367b62..4d75883fc5 100644 --- a/erts/emulator/hipe/hipe_mode_switch.c +++ b/erts/emulator/hipe/hipe_mode_switch.c @@ -35,6 +35,17 @@ #include "hipe_stack.h" #include "hipe_bif0.h" /* hipe_mfa_info_table_init() */ +#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP) +# define ERTS_SMP_REQ_PROC_MAIN_LOCK(P) \ + if ((P)) erts_proc_lc_require_lock((P), ERTS_PROC_LOCK_MAIN) +# define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P) \ + if ((P)) erts_proc_lc_unrequire_lock((P), ERTS_PROC_LOCK_MAIN) +#else +# define ERTS_SMP_REQ_PROC_MAIN_LOCK(P) +# define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P) +#endif + + /* * Internal debug support. * #define HIPE_DEBUG to the desired debug level: @@ -318,8 +329,8 @@ Process *hipe_mode_switch(Process *p, unsigned cmd, Eterm reg[]) * Native code called a BIF, which "failed" with a TRAP to BEAM. * Prior to returning, the BIF stored (see BIF_TRAP<N>): - * the callee's address in p->def_arg_reg[3] - * the callee's parameters in p->def_arg_reg[0..2] + * the callee's address in p->i + * the callee's parameters in reg[0..2] * the callee's arity in p->arity (for BEAM gc purposes) * * We need to remove the BIF's parameters from the native @@ -331,19 +342,8 @@ Process *hipe_mode_switch(Process *p, unsigned cmd, Eterm reg[]) */ unsigned int i, is_recursive = 0; - /* Save p->arity, then update it with the original BIF's arity. - Get rid of any stacked parameters in that call. */ - /* XXX: hipe_call_from_native_is_recursive() copies data to - reg[], which is useless in the TRAP case. Maybe write a - specialised hipe_trap_from_native_is_recursive() later. */ if (p->hipe.nsp != NULL) { - unsigned int callee_arity; - callee_arity = p->arity; - p->arity = p->hipe.narity; /* caller's arity */ - is_recursive = hipe_call_from_native_is_recursive(p, reg); - - p->i = (Eterm *)(p->def_arg_reg[3]); - p->arity = callee_arity; + is_recursive = hipe_trap_from_native_is_recursive(p); } /* Schedule next process if current process was hibernated or is waiting @@ -353,15 +353,14 @@ Process *hipe_mode_switch(Process *p, unsigned cmd, Eterm reg[]) goto do_schedule; } if (p->status == P_WAITING) { + for (i = 0; i < p->arity; ++i) + p->arg_reg[i] = reg[i]; goto do_schedule; } - for (i = 0; i < p->arity; ++i) - reg[i] = p->def_arg_reg[i]; - if (is_recursive) hipe_push_beam_trap_frame(p, reg, p->arity); - + result = HIPE_MODE_SWITCH_RES_CALL; break; } @@ -470,10 +469,12 @@ Process *hipe_mode_switch(Process *p, unsigned cmd, Eterm reg[]) #if !(NR_ARG_REGS > 5) int reds_in = p->def_arg_reg[5]; #endif + ERTS_SMP_UNREQ_PROC_MAIN_LOCK(p); p = schedule(p, reds_in - p->fcalls); + ERTS_SMP_REQ_PROC_MAIN_LOCK(p); #ifdef ERTS_SMP p->hipe_smp.have_receive_locks = 0; - reg = p->scheduler_data->save_reg; + reg = p->scheduler_data->x_reg_array; #endif } { @@ -648,7 +649,7 @@ Eterm hipe_build_stacktrace(Process *p, struct StackTrace *s) if (depth < 1) return NIL; - heap_size = 6 * depth; /* each [{M,F,A}|_] is 2+4 == 6 words */ + heap_size = 7 * depth; /* each [{M,F,A,[]}|_] is 2+5 == 7 words */ hp = HAlloc(p, heap_size); hp_end = hp + heap_size; @@ -659,8 +660,8 @@ Eterm hipe_build_stacktrace(Process *p, struct StackTrace *s) ra = (const void*)s->trace[i]; if (!hipe_find_mfa_from_ra(ra, &m, &f, &a)) continue; - mfa = TUPLE3(hp, m, f, make_small(a)); - hp += 4; + mfa = TUPLE4(hp, m, f, make_small(a), NIL); + hp += 5; next = CONS(hp, mfa, NIL); *next_p = next; next_p = &CDR(list_val(next)); diff --git a/erts/emulator/hipe/hipe_native_bif.c b/erts/emulator/hipe/hipe_native_bif.c index 8d31348496..77dee6f9e9 100644 --- a/erts/emulator/hipe/hipe_native_bif.c +++ b/erts/emulator/hipe/hipe_native_bif.c @@ -41,9 +41,9 @@ */ /* for -Wmissing-prototypes :-( */ -extern Eterm hipe_check_process_code_2(Process*, Eterm, Eterm); -extern Eterm hipe_garbage_collect_1(Process*, Eterm); -extern Eterm hipe_show_nstack_1(Process*, Eterm); +extern Eterm hipe_check_process_code_2(BIF_ALIST_2); +extern Eterm hipe_garbage_collect_1(BIF_ALIST_1); +extern Eterm hipe_show_nstack_1(BIF_ALIST_1); /* Used when a BIF can trigger a stack walk. */ static __inline__ void hipe_set_narity(Process *p, unsigned int arity) @@ -56,7 +56,7 @@ Eterm hipe_check_process_code_2(BIF_ALIST_2) Eterm ret; hipe_set_narity(BIF_P, 2); - ret = check_process_code_2(BIF_P, BIF_ARG_1, BIF_ARG_2); + ret = check_process_code_2(BIF_P, BIF__ARGS); hipe_set_narity(BIF_P, 0); return ret; } @@ -66,7 +66,7 @@ Eterm hipe_garbage_collect_1(BIF_ALIST_1) Eterm ret; hipe_set_narity(BIF_P, 1); - ret = garbage_collect_1(BIF_P, BIF_ARG_1); + ret = garbage_collect_1(BIF_P, BIF__ARGS); hipe_set_narity(BIF_P, 0); return ret; } @@ -76,7 +76,7 @@ Eterm hipe_show_nstack_1(BIF_ALIST_1) Eterm ret; hipe_set_narity(BIF_P, 1); - ret = hipe_bifs_show_nstack_1(BIF_P, BIF_ARG_1); + ret = hipe_bifs_show_nstack_1(BIF_P, BIF__ARGS); hipe_set_narity(BIF_P, 0); return ret; } @@ -99,8 +99,10 @@ void hipe_gc(Process *p, Eterm need) * has begun. * XXX: BUG: native code should check return status */ -Eterm hipe_set_timeout(Process *p, Eterm timeout_value) +BIF_RETTYPE hipe_set_timeout(BIF_ALIST_1) { + Process* p = BIF_P; + Eterm timeout_value = BIF_ARG_1; #if !defined(ARCH_64) Uint time_val; #endif @@ -286,8 +288,13 @@ static struct StackTrace *get_trace_from_exc(Eterm exc) * This does what the (misnamed) Beam instruction 'raise_ss' does, * namely, a proper re-throw of an exception that was caught by 'try'. */ -Eterm hipe_rethrow(Process *c_p, Eterm exc, Eterm value) + +BIF_RETTYPE hipe_rethrow(BIF_ALIST_2) { + Process* c_p = BIF_P; + Eterm exc = BIF_ARG_1; + Eterm value = BIF_ARG_2; + c_p->fvalue = value; if (c_p->freason == EXC_NULL) { /* a safety check for the R10-0 case; should not happen */ @@ -334,7 +341,7 @@ char *hipe_bs_allocate(int len) bptr = erts_bin_nrml_alloc(len); bptr->flags = 0; bptr->orig_size = len; - erts_smp_atomic_init(&bptr->refc, 1); + erts_smp_atomic_init_nob(&bptr->refc, 1); return bptr->orig_bytes; } @@ -411,8 +418,12 @@ Eterm hipe_bs_utf8_size(Eterm arg) return make_small(4); } -Eterm hipe_bs_put_utf8(Process *p, Eterm arg, byte *base, unsigned int offset) +BIF_RETTYPE hipe_bs_put_utf8(BIF_ALIST_3) { + Process* p = BIF_P; + Eterm arg = BIF_ARG_1; + byte* base = (byte*) BIF_ARG_2; + Uint offset = (Uint) BIF_ARG_3; byte *save_bin_buf; Uint save_bin_offset; int res; @@ -468,13 +479,21 @@ Eterm hipe_bs_put_utf16(Process *p, Eterm arg, byte *base, unsigned int offset, return new_offset; } -Eterm hipe_bs_put_utf16be(Process *p, Eterm arg, byte *base, unsigned int offset) +BIF_RETTYPE hipe_bs_put_utf16be(BIF_ALIST_3) { + Process *p = BIF_P; + Eterm arg = BIF_ARG_1; + byte *base = (byte*) BIF_ARG_2; + Uint offset = (Uint) BIF_ARG_3; return hipe_bs_put_utf16(p, arg, base, offset, 0); } -Eterm hipe_bs_put_utf16le(Process *p, Eterm arg, byte *base, unsigned int offset) +BIF_RETTYPE hipe_bs_put_utf16le(BIF_ALIST_3) { + Process *p = BIF_P; + Eterm arg = BIF_ARG_1; + byte *base = (byte*) BIF_ARG_2; + Uint offset = (Uint) BIF_ARG_3; return hipe_bs_put_utf16(p, arg, base, offset, BSF_LITTLE); } @@ -489,8 +508,10 @@ static int validate_unicode(Eterm arg) return 1; } -Eterm hipe_bs_validate_unicode(Process *p, Eterm arg) +BIF_RETTYPE hipe_bs_validate_unicode(BIF_ALIST_1) { + Process *p = BIF_P; + Eterm arg = BIF_ARG_1; if (!validate_unicode(arg)) BIF_ERROR(p, BADARG); return NIL; @@ -584,7 +605,7 @@ void hipe_clear_timeout(Process *c_p) void hipe_atomic_inc(int *counter) { - erts_smp_atomic_inc((erts_smp_atomic_t*)counter); + erts_smp_atomic_inc_nob((erts_smp_atomic_t*)counter); } #endif diff --git a/erts/emulator/hipe/hipe_native_bif.h b/erts/emulator/hipe/hipe_native_bif.h index 13a02b84a2..8c9dec180e 100644 --- a/erts/emulator/hipe/hipe_native_bif.h +++ b/erts/emulator/hipe/hipe_native_bif.h @@ -23,6 +23,7 @@ #ifndef HIPE_NATIVE_BIF_H #define HIPE_NATIVE_BIF_H +#include "bif.h" #include "hipe_arch.h" /* @@ -71,24 +72,24 @@ AEXTERN(void,nbif_select_msg,(Process*)); AEXTERN(Eterm,nbif_cmp_2,(void)); AEXTERN(Eterm,nbif_eq_2,(void)); -Eterm hipe_nonclosure_address(Process*, Eterm, Uint); -Eterm hipe_conv_big_to_float(Process*, Eterm); +BIF_RETTYPE hipe_nonclosure_address(BIF_ALIST_2); +BIF_RETTYPE hipe_conv_big_to_float(BIF_ALIST_1); void hipe_fclearerror_error(Process*); void hipe_select_msg(Process*); void hipe_gc(Process*, Eterm); -Eterm hipe_set_timeout(Process*, Eterm); +BIF_RETTYPE hipe_set_timeout(BIF_ALIST_1); void hipe_handle_exception(Process*); -Eterm hipe_rethrow(Process *c_p, Eterm exc, Eterm value); +BIF_RETTYPE hipe_rethrow(BIF_ALIST_2); char *hipe_bs_allocate(int); Binary *hipe_bs_reallocate(Binary*, int); int hipe_bs_put_small_float(Process*, Eterm, Uint, byte*, unsigned, unsigned); void hipe_bs_put_bits(Eterm, Uint, byte*, unsigned, unsigned); Eterm hipe_bs_utf8_size(Eterm); -Eterm hipe_bs_put_utf8(Process*, Eterm, byte*, unsigned int); +BIF_RETTYPE hipe_bs_put_utf8(BIF_ALIST_3); Eterm hipe_bs_utf16_size(Eterm); -Eterm hipe_bs_put_utf16be(Process*, Eterm, byte*, unsigned int); -Eterm hipe_bs_put_utf16le(Process*, Eterm, byte*, unsigned int); -Eterm hipe_bs_validate_unicode(Process*, Eterm); +BIF_RETTYPE hipe_bs_put_utf16be(BIF_ALIST_3); +BIF_RETTYPE hipe_bs_put_utf16le(BIF_ALIST_3); +BIF_RETTYPE hipe_bs_validate_unicode(BIF_ALIST_1); struct erl_bin_match_buffer; int hipe_bs_validate_unicode_retract(struct erl_bin_match_buffer*, Eterm); diff --git a/erts/emulator/hipe/hipe_ppc_asm.m4 b/erts/emulator/hipe/hipe_ppc_asm.m4 index 0eb5c441e6..343402f9f0 100644 --- a/erts/emulator/hipe/hipe_ppc_asm.m4 +++ b/erts/emulator/hipe/hipe_ppc_asm.m4 @@ -31,12 +31,23 @@ define(LOAD,ld)dnl define(STORE,std)dnl define(CMPI,cmpdi)dnl define(WSIZE,8)dnl +`#define STORE_IA(ADDR, DST, TMP) \ + addis TMP, 0, ADDR@highest SEMI\ + ori TMP, TMP, ADDR@higher SEMI\ + rldicr TMP, TMP, 32, 31 SEMI\ + oris TMP, TMP, ADDR@h SEMI\ + ori TMP, TMP, ADDR@l SEMI\ + std TMP, DST' ',` /* 32-bit PowerPC */ define(LOAD,lwz)dnl define(STORE,stw)dnl define(CMPI,cmpwi)dnl define(WSIZE,4)dnl +`#define STORE_IA(ADDR, DST, TMP) \ + lis TMP, ADDR@ha SEMI\ + addi TMP, TMP, ADDR@l SEMI\ + stw TMP, DST' ')dnl `#define LOAD 'LOAD `#define STORE 'STORE diff --git a/erts/emulator/hipe/hipe_ppc_bifs.m4 b/erts/emulator/hipe/hipe_ppc_bifs.m4 index 203fefe1a1..d09551d10d 100644 --- a/erts/emulator/hipe/hipe_ppc_bifs.m4 +++ b/erts/emulator/hipe/hipe_ppc_bifs.m4 @@ -20,21 +20,34 @@ changecom(`/*', `*/')dnl include(`hipe/hipe_ppc_asm.m4') +#`include' "config.h" #`include' "hipe_literals.h" +`#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP) +# define CALL_BIF(F) STORE_IA(CSYM(F), P_BIF_CALLEE(P), r29); bl CSYM(hipe_debug_bif_wrapper) +#else +# define CALL_BIF(F) bl CSYM(F) +#endif' + .text .p2align 2 -`#define TEST_GOT_MBUF LOAD r4, P_MBUF(P) SEMI CMPI r4, 0 SEMI bne- 3f SEMI 2: -#define JOIN3(A,B,C) A##B##C -#define HANDLE_GOT_MBUF(ARITY) 3: bl CSYM(JOIN3(nbif_,ARITY,_gc_after_bif)) SEMI b 2b' +define(TEST_GOT_MBUF,`LOAD r4, P_MBUF(P) # `TEST_GOT_MBUF' + CMPI r4, 0 + bne- 3f +2:') +define(HANDLE_GOT_MBUF,` +3: bl CSYM(nbif_$1_gc_after_bif) # `HANDLE_GOT_MBUF' + b 2b') + /* * standard_bif_interface_1(nbif_name, cbif_name) * standard_bif_interface_2(nbif_name, cbif_name) * standard_bif_interface_3(nbif_name, cbif_name) + * standard_bif_interface_0(nbif_name, cbif_name) * - * Generate native interface for a BIF with 1-3 parameters and + * Generate native interface for a BIF with 0-3 parameters and * standard failure mode. */ define(standard_bif_interface_1, @@ -49,7 +62,9 @@ ASYM($1): /* Save caller-save registers and call the C function. */ SAVE_CONTEXT_BIF - bl CSYM($2) + STORE r4, P_ARG0(r3) # Store BIF__ARGS in def_arg_reg[] + addi r4, r3, P_ARG0 + CALL_BIF($2) TEST_GOT_MBUF /* Restore registers. Check for exception. */ @@ -77,7 +92,10 @@ ASYM($1): /* Save caller-save registers and call the C function. */ SAVE_CONTEXT_BIF - bl CSYM($2) + STORE r4, P_ARG0(r3) # Store BIF__ARGS in def_arg_reg[] + STORE r5, P_ARG1(r3) + addi r4, r3, P_ARG0 + CALL_BIF($2) TEST_GOT_MBUF /* Restore registers. Check for exception. */ @@ -106,7 +124,11 @@ ASYM($1): /* Save caller-save registers and call the C function. */ SAVE_CONTEXT_BIF - bl CSYM($2) + STORE r4, P_ARG0(r3) # Store BIF__ARGS in def_arg_reg[] + STORE r5, P_ARG1(r3) + STORE r6, P_ARG2(r3) + addi r4, r3, P_ARG0 + CALL_BIF($2) TEST_GOT_MBUF /* Restore registers. Check for exception. */ @@ -121,13 +143,7 @@ ASYM($1): TYPE_FUNCTION(ASYM($1)) #endif') -/* - * fail_bif_interface_0(nbif_name, cbif_name) - * - * Generate native interface for a BIF with 0 parameters and - * standard failure mode. - */ -define(fail_bif_interface_0, +define(standard_bif_interface_0, ` #ifndef HAVE_$1 #`define' HAVE_$1 @@ -138,7 +154,8 @@ ASYM($1): /* Save caller-save registers and call the C function. */ SAVE_CONTEXT_BIF - bl CSYM($2) + /* ignore empty BIF__ARGS */ + CALL_BIF($2) TEST_GOT_MBUF /* Restore registers. Check for exception. */ @@ -173,7 +190,8 @@ ASYM($1): /* Save caller-save registers and call the C function. */ SAVE_CONTEXT_GC - bl CSYM($2) + /* ignore empty BIF__ARGS */ + CALL_BIF($2) TEST_GOT_MBUF /* Restore registers. */ @@ -196,7 +214,9 @@ ASYM($1): /* Save caller-save registers and call the C function. */ SAVE_CONTEXT_GC - bl CSYM($2) + STORE r4, P_ARG0(r3) # Store BIF__ARGS in def_arg_reg[] + addi r4, r3, P_ARG0 + CALL_BIF($2) TEST_GOT_MBUF /* Restore registers. Check for exception. */ @@ -224,7 +244,10 @@ ASYM($1): /* Save caller-save registers and call the C function. */ SAVE_CONTEXT_GC - bl CSYM($2) + STORE r4, P_ARG0(r3) # Store BIF__ARGS in def_arg_reg[] + STORE r5, P_ARG1(r3) + addi r4, r3, P_ARG0 + CALL_BIF($2) TEST_GOT_MBUF /* Restore registers. Check for exception. */ diff --git a/erts/emulator/hipe/hipe_process.h b/erts/emulator/hipe/hipe_process.h index 5effacb398..43f47d1a28 100644 --- a/erts/emulator/hipe/hipe_process.h +++ b/erts/emulator/hipe/hipe_process.h @@ -42,6 +42,9 @@ struct hipe_process_state { void (*nra)(void); /* Native code return address. */ #endif unsigned int narity; /* Arity of BIF call, for stack walks. */ +#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP) + void (*bif_callee)(void); /* When calling BIF's via debug wrapper */ +#endif }; extern void hipe_arch_print_pcb(struct hipe_process_state *p); diff --git a/erts/emulator/hipe/hipe_risc_glue.h b/erts/emulator/hipe/hipe_risc_glue.h index e74023e3e9..cc2671c016 100644 --- a/erts/emulator/hipe/hipe_risc_glue.h +++ b/erts/emulator/hipe/hipe_risc_glue.h @@ -199,6 +199,22 @@ hipe_call_from_native_is_recursive(Process *p, Eterm reg[]) return 0; } +/* BEAM called native, which called BIF that returned trap + * Discard bif parameters. + * If tailcall, also clean up native stub continuation. */ +static __inline__ int +hipe_trap_from_native_is_recursive(Process *p) +{ + if (p->hipe.narity > NR_ARG_REGS) { + p->hipe.nsp += (p->hipe.narity - NR_ARG_REGS); + } + if (p->hipe.nra != (void(*)(void))&nbif_return) + return 1; + hipe_pop_risc_nra_frame(p); + return 0; +} + + /* Native makes a call which needs to unload the parameters. This differs from hipe_call_from_native_is_recursive() in that it doesn't check for or pop the BEAM-calls-native frame. diff --git a/erts/emulator/hipe/hipe_sparc_bifs.m4 b/erts/emulator/hipe/hipe_sparc_bifs.m4 index 03db7f3413..ca5af45d58 100644 --- a/erts/emulator/hipe/hipe_sparc_bifs.m4 +++ b/erts/emulator/hipe/hipe_sparc_bifs.m4 @@ -20,27 +20,42 @@ changecom(`/*', `*/')dnl include(`hipe/hipe_sparc_asm.m4') +#`include' "config.h" #`include' "hipe_literals.h" .section ".text" .align 4 +`#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP) +# define CALL_BIF(F) set F, %o7; st %o7, [%o0+P_BIF_CALLEE]; call hipe_debug_bif_wrapper +#else +# define CALL_BIF(F) call F +#endif' + /* * Test for exception. This macro executes its delay slot. */ -`#define __TEST_GOT_EXN(LABEL) cmp %o0, THE_NON_VALUE; bz,pn %icc, LABEL -#define TEST_GOT_EXN(ARITY) __TEST_GOT_EXN(JOIN3(nbif_,ARITY,_simple_exception))' +define(TEST_GOT_EXN,`cmp %o0, THE_NON_VALUE ! `TEST_GOT_EXN' + bz,pn %icc, nbif_$1_simple_exception') -`#define TEST_GOT_MBUF ld [P+P_MBUF], %o1; cmp %o1, 0; bne 3f; nop; 2: -#define JOIN3(A,B,C) A##B##C -#define HANDLE_GOT_MBUF(ARITY) 3: call JOIN3(nbif_,ARITY,_gc_after_bif); nop; b 2b; nop' +define(TEST_GOT_MBUF,`ld [P+P_MBUF], %o1 ! `TEST_GOT_MBUF' + cmp %o1, 0 + bne 3f + nop +2:') +define(HANDLE_GOT_MBUF,` +3: call nbif_$1_gc_after_bif ! `HANDLE_GOT_MBUF' + nop + b 2b + nop') /* * standard_bif_interface_1(nbif_name, cbif_name) * standard_bif_interface_2(nbif_name, cbif_name) * standard_bif_interface_3(nbif_name, cbif_name) + * standard_bif_interface_0(nbif_name, cbif_name) * - * Generate native interface for a BIF with 1-3 parameters and + * Generate native interface for a BIF with 0-3 parameters and * standard failure mode. */ define(standard_bif_interface_1, @@ -55,7 +70,9 @@ $1: /* Save caller-save registers and call the C function. */ SAVE_CONTEXT_BIF - call $2 + st %o1, [%o0+P_ARG0] ! Store BIF__ARGS in def_arg_reg + add %o0, P_ARG0, %o1 + CALL_BIF($2) nop TEST_GOT_MBUF @@ -81,7 +98,10 @@ $1: /* Save caller-save registers and call the C function. */ SAVE_CONTEXT_BIF - call $2 + st %o1, [%o0+P_ARG0] ! Store BIF__ARGS in def_arg_reg + st %o2, [%o0+P_ARG1] + add %o0, P_ARG0, %o1 + CALL_BIF($2) nop TEST_GOT_MBUF @@ -108,7 +128,11 @@ $1: /* Save caller-save registers and call the C function. */ SAVE_CONTEXT_BIF - call $2 + st %o1, [%o0+P_ARG0] ! Store BIF__ARGS in def_arg_reg + st %o2, [%o0+P_ARG1] + st %o3, [%o0+P_ARG2] + add %o0, P_ARG0, %o1 + CALL_BIF($2) nop TEST_GOT_MBUF @@ -121,13 +145,7 @@ $1: .type $1, #function #endif') -/* - * fail_bif_interface_0(nbif_name, cbif_name) - * - * Generate native interface for a BIF with 0 parameters and - * standard failure mode. - */ -define(fail_bif_interface_0, +define(standard_bif_interface_0, ` #ifndef HAVE_$1 #`define' HAVE_$1 @@ -138,7 +156,8 @@ $1: /* Save caller-save registers and call the C function. */ SAVE_CONTEXT_BIF - call $2 + /* ignore empty BIF__ARGS */ + CALL_BIF($2) nop TEST_GOT_MBUF @@ -171,7 +190,8 @@ $1: /* Save caller-save registers and call the C function. */ SAVE_CONTEXT_GC - call $2 + /* ignore empty BIF__ARGS */ + CALL_BIF($2) nop TEST_GOT_MBUF @@ -195,7 +215,9 @@ $1: /* Save caller-save registers and call the C function. */ SAVE_CONTEXT_GC - call $2 + st %o1, [%o0+P_ARG0] ! Store BIF__ARGS in def_arg_reg + add %o0, P_ARG0, %o1 + CALL_BIF($2) nop TEST_GOT_MBUF @@ -221,7 +243,10 @@ $1: /* Save caller-save registers and call the C function. */ SAVE_CONTEXT_GC - call $2 + st %o1, [%o0+P_ARG0] ! Store BIF__ARGS in def_arg_reg + st %o2, [%o0+P_ARG1] + add %o0, P_ARG0, %o1 + CALL_BIF($2) nop TEST_GOT_MBUF diff --git a/erts/emulator/hipe/hipe_x86_bifs.m4 b/erts/emulator/hipe/hipe_x86_bifs.m4 index 1bb6488b00..2ea69bde3c 100644 --- a/erts/emulator/hipe/hipe_x86_bifs.m4 +++ b/erts/emulator/hipe/hipe_x86_bifs.m4 @@ -20,6 +20,7 @@ changecom(`/*', `*/')dnl include(`hipe/hipe_x86_asm.m4') +#`include' "config.h" #`include' "hipe_literals.h" `#if THE_NON_VALUE == 0 @@ -28,16 +29,27 @@ include(`hipe/hipe_x86_asm.m4') #define TEST_GOT_EXN cmpl $THE_NON_VALUE,%eax #endif' -`#define TEST_GOT_MBUF movl P_MBUF(P), %edx; testl %edx, %edx; jnz 3f; 2: -#define JOIN3(A,B,C) A##B##C -#define HANDLE_GOT_MBUF(ARITY) 3: call JOIN3(nbif_,ARITY,_gc_after_bif); jmp 2b' +`#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP) +# define CALL_BIF(F) movl $CSYM(F), P_BIF_CALLEE(P); call CSYM(hipe_debug_bif_wrapper) +#else +# define CALL_BIF(F) call CSYM(F) +#endif' + +define(TEST_GOT_MBUF,`movl P_MBUF(P), %edx # `TEST_GOT_MBUF' + testl %edx, %edx + jnz 3f +2:') +define(HANDLE_GOT_MBUF,` +3: call nbif_$1_gc_after_bif # `HANDLE_GOT_MBUF' + jmp 2b') /* * standard_bif_interface_1(nbif_name, cbif_name) * standard_bif_interface_2(nbif_name, cbif_name) * standard_bif_interface_3(nbif_name, cbif_name) + * standard_bif_interface_0(nbif_name, cbif_name) * - * Generate native interface for a BIF with 1-3 parameters and + * Generate native interface for a BIF with 0-3 parameters and * standard failure mode. */ define(standard_bif_interface_1, @@ -56,8 +68,10 @@ ASYM($1): /* make the call on the C stack */ NBIF_ARG_REG(0,P) - NBIF_ARG(1,1,0) - call CSYM($2) + NBIF_ARG(2,1,0) + lea 8(%esp), %eax + NBIF_ARG_REG(1,%eax) # BIF__ARGS + CALL_BIF($2) TEST_GOT_MBUF /* switch to native stack */ @@ -88,9 +102,11 @@ ASYM($1): /* make the call on the C stack */ NBIF_ARG_REG(0,P) - NBIF_ARG(1,2,0) - NBIF_ARG(2,2,1) - call CSYM($2) + NBIF_ARG(2,2,0) + NBIF_ARG(3,2,1) + lea 8(%esp), %eax + NBIF_ARG_REG(1,%eax) # BIF__ARGS + CALL_BIF($2) TEST_GOT_MBUF /* switch to native stack */ @@ -121,10 +137,12 @@ ASYM($1): /* make the call on the C stack */ NBIF_ARG_REG(0,P) - NBIF_ARG(1,3,0) - NBIF_ARG(2,3,1) - NBIF_ARG(3,3,2) - call CSYM($2) + NBIF_ARG(2,3,0) + NBIF_ARG(3,3,1) + NBIF_ARG(4,3,2) + lea 8(%esp), %eax + NBIF_ARG_REG(1,%eax) # BIF__ARGS + CALL_BIF($2) TEST_GOT_MBUF /* switch to native stack */ @@ -139,13 +157,7 @@ ASYM($1): TYPE_FUNCTION(ASYM($1)) #endif') -/* - * fail_bif_interface_0(nbif_name, cbif_name) - * - * Generate native interface for a BIF with 0 parameters and - * standard failure mode. - */ -define(fail_bif_interface_0, +define(standard_bif_interface_0, ` #ifndef HAVE_$1 #`define' HAVE_$1 @@ -158,7 +170,8 @@ ASYM($1): /* make the call on the C stack */ NBIF_ARG_REG(0,P) - call CSYM($2) + /* skip BIF__ARGS */ + CALL_BIF($2) TEST_GOT_MBUF /* switch to native stack */ diff --git a/erts/emulator/hipe/hipe_x86_glue.h b/erts/emulator/hipe/hipe_x86_glue.h index a7b0f164be..b0db93267c 100644 --- a/erts/emulator/hipe/hipe_x86_glue.h +++ b/erts/emulator/hipe/hipe_x86_glue.h @@ -186,6 +186,25 @@ hipe_call_from_native_is_recursive(Process *p, Eterm reg[]) return 0; } +/* BEAM called native, which called BIF that returned trap + * Discard bif parameters. + * If tailcall, also clean up native stub continuation. */ +static __inline__ int +hipe_trap_from_native_is_recursive(Process *p) +{ + Eterm nra = *(p->hipe.nsp++); + + if (p->hipe.narity > NR_ARG_REGS) { + p->hipe.nsp += (p->hipe.narity - NR_ARG_REGS); + } + if (nra != (Eterm)nbif_return) { + *--(p->hipe.nsp) = nra; + return 1; + } + return 0; +} + + /* Native makes a call which needs to unload the parameters. This differs from hipe_call_from_native_is_recursive() in that it doesn't check for or pop the BEAM-calls-native frame. diff --git a/erts/emulator/pcre/Makefile b/erts/emulator/pcre/Makefile deleted file mode 100644 index 72eea01130..0000000000 --- a/erts/emulator/pcre/Makefile +++ /dev/null @@ -1,26 +0,0 @@ -# -# %CopyrightBegin% -# -# Copyright Ericsson AB 2008-2009. All Rights Reserved. -# -# The contents of this file are subject to the Erlang Public License, -# Version 1.1, (the "License"); you may not use this file except in -# compliance with the License. You should have received a copy of the -# Erlang Public License along with this software. If not, it can be -# retrieved online at http://www.erlang.org/. -# -# Software distributed under the License is distributed on an "AS IS" -# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -# the License for the specific language governing rights and limitations -# under the License. -# -# %CopyrightEnd% -# -# -# Invoke with GNU make or clearmake -C gnu. -# - -include $(ERL_TOP)/make/run_make.mk - -table: - $(MAKE) -f $(TARGET)/Makefile $@
\ No newline at end of file diff --git a/erts/emulator/pcre/Makefile.in b/erts/emulator/pcre/Makefile.in deleted file mode 100644 index f62700ec4e..0000000000 --- a/erts/emulator/pcre/Makefile.in +++ /dev/null @@ -1,165 +0,0 @@ -# Makefile for zlib -# Copyright (C) 1995-1996 Jean-loup Gailly. -# For conditions of distribution and use, see copyright notice in zlib.h - -# To compile and test, type: -# ./configure; make test -# The call of configure is optional if you don't have special requirements - -# To install /usr/local/lib/libz.* and /usr/local/include/zlib.h, type: -# make install -# To install in $HOME instead of /usr/local, use: -# make install prefix=$HOME - -# %ExternalCopyright% - -ARFLAGS = rc - -O = \ -pcre_latin_1_table.o \ -pcre_compile.o \ -pcre_config.o \ -pcre_dfa_exec.o \ -pcre_exec.o \ -pcre_fullinfo.o \ -pcre_get.o \ -pcre_globals.o \ -pcre_info.o \ -pcre_maketables.o \ -pcre_newline.o \ -pcre_ord2utf8.o \ -pcre_refcount.o \ -pcre_study.o \ -pcre_tables.o \ -pcre_try_flipped.o \ -pcre_ucp_searchfuncs.o \ -pcre_valid_utf8.o \ -pcre_version.o \ -pcre_xclass.o - -OBJS = $(O:%=$(OBJDIR)/%) - -GENINC = pcre_exec_loop_break_cases.inc - -#### Begin OTP targets - -include $(ERL_TOP)/make/target.mk - -# On windows we need a separate zlib during debug build -ifeq ($(TARGET),win32) - -ifeq ($(TYPE),debug) -CFLAGS = $(subst -O2, -g, @CFLAGS@ @DEFS@ @DEBUG_FLAGS@ @EMU_THR_DEFS@ -DERLANG_INTEGRATION) -else # debug -CFLAGS = @CFLAGS@ @DEFS@ @EMU_THR_DEFS@ -DERLANG_INTEGRATION -endif # debug - -else # win32 - -ifeq ($(TYPE),debug) -TYPE_FLAGS = @DEBUG_CFLAGS@ -else # debug -ifeq ($(TYPE),gcov) -TYPE_FLAGS = -O0 -fprofile-arcs -ftest-coverage -else # gcov -TYPE_FLAGS = -O3 -endif # gcov -endif # debug - -CFLAGS = $(TYPE_FLAGS) $(subst -O2,, @CFLAGS@) @DEFS@ @EMU_THR_DEFS@ -DERLANG_INTEGRATION - -endif # win32 - -OBJDIR = $(ERL_TOP)/erts/emulator/pcre/obj/$(TARGET)/$(TYPE) - -include $(ERL_TOP)/make/$(TARGET)/otp.mk - -ifeq ($(TARGET), win32) -LIBRARY=$(OBJDIR)/epcre.lib -else -LIBRARY=$(OBJDIR)/libepcre.a -endif - -all: $(LIBRARY) - -# ---------------------------------------------------- -# Release Target -# ---------------------------------------------------- -include $(ERL_TOP)/make/otp_release_targets.mk - -release_spec: opt - -tests release_tests: - -docs release_docs release_docs_spec: - -clean: - rm -f $(OBJS) $(OBJDIR)/libepcre.a - -#### end OTP targets - -ifeq ($(TARGET), win32) -$(LIBRARY): $(OBJS) - $(AR) -out:$@ $(OBJS) -else -$(LIBRARY): $(OBJS) - $(AR) $(ARFLAGS) $@ $(OBJS) - -@ ($(RANLIB) $@ || true) 2>/dev/null -endif - -$(OBJDIR)/%.o: %.c - $(CC) -c $(CFLAGS) -o $@ $< - -$(GENINC): pcre_exec.c - for x in `grep -n COST_CHK pcre_exec.c | grep -v 'COST_CHK(N)' | awk -F: '{print $$1}'`; \ - do \ - N=`expr $$x + 100`; \ - echo "case $$N: goto L_LOOP_COUNT_$${x};"; \ - done > $(GENINC) - -table: ./gen_table - ./gen_table pcre_latin_1_table.c - -./gen_table: pcre_make_latin1_default.c make_latin1_table.c - $(CC) $(CFLAGS) -o gen_table pcre_make_latin1_default.c make_latin1_table.c - -# DO NOT DELETE THIS LINE -- make depend depends on it. - -$(OBJDIR)/pcre_chartables.o: pcre_chartables.c pcre_internal.h local_config.h \ - pcre.h ucp.h -$(OBJDIR)/pcre_compile.o: pcre_compile.c pcre_internal.h local_config.h \ - pcre.h ucp.h -$(OBJDIR)/pcre_config.o: pcre_config.c pcre_internal.h local_config.h pcre.h \ - ucp.h -$(OBJDIR)/pcre_dfa_exec.o: pcre_dfa_exec.c pcre_internal.h local_config.h \ - pcre.h ucp.h -$(OBJDIR)/pcre_exec.o: pcre_exec.c pcre_internal.h local_config.h pcre.h ucp.h \ - $(GENINC) -$(OBJDIR)/pcre_fullinfo.o: pcre_fullinfo.c pcre_internal.h local_config.h \ - pcre.h ucp.h -$(OBJDIR)/pcre_get.o: pcre_get.c pcre_internal.h local_config.h pcre.h ucp.h -$(OBJDIR)/pcre_globals.o: pcre_globals.c pcre_internal.h local_config.h \ - pcre.h ucp.h -$(OBJDIR)/pcre_info.o: pcre_info.c pcre_internal.h local_config.h pcre.h ucp.h -$(OBJDIR)/pcre_maketables.o: pcre_maketables.c pcre_internal.h local_config.h \ - pcre.h ucp.h -$(OBJDIR)/pcre_newline.o: pcre_newline.c pcre_internal.h local_config.h \ - pcre.h ucp.h -$(OBJDIR)/pcre_ord2utf8.o: pcre_ord2utf8.c pcre_internal.h local_config.h \ - pcre.h ucp.h -$(OBJDIR)/pcre_refcount.o: pcre_refcount.c pcre_internal.h local_config.h \ - pcre.h ucp.h -$(OBJDIR)/pcre_study.o: pcre_study.c pcre_internal.h local_config.h pcre.h \ - ucp.h -$(OBJDIR)/pcre_tables.o: pcre_tables.c pcre_internal.h local_config.h pcre.h \ - ucp.h -$(OBJDIR)/pcre_try_flipped.o: pcre_try_flipped.c pcre_internal.h \ - local_config.h pcre.h ucp.h -$(OBJDIR)/pcre_ucp_searchfuncs.o: pcre_ucp_searchfuncs.c pcre_internal.h \ - local_config.h pcre.h ucp.h ucpinternal.h ucptable.h -$(OBJDIR)/pcre_valid_utf8.o: pcre_valid_utf8.c pcre_internal.h local_config.h \ - pcre.h ucp.h -pcre_version.o: pcre_version.c pcre_internal.h local_config.h pcre.h \ - ucp.h -$(OBJDIR)/pcre_xclass.o: pcre_xclass.c pcre_internal.h local_config.h pcre.h \ - ucp.h diff --git a/erts/emulator/pcre/pcre.mk b/erts/emulator/pcre/pcre.mk new file mode 100644 index 0000000000..b752c11459 --- /dev/null +++ b/erts/emulator/pcre/pcre.mk @@ -0,0 +1,113 @@ +# +# %CopyrightBegin% +# +# Copyright Ericsson AB 2011. All Rights Reserved. +# +# The contents of this file are subject to the Erlang Public License, +# Version 1.1, (the "License"); you may not use this file except in +# compliance with the License. You should have received a copy of the +# Erlang Public License along with this software. If not, it can be +# retrieved online at http://www.erlang.org/. +# +# Software distributed under the License is distributed on an "AS IS" +# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +# the License for the specific language governing rights and limitations +# under the License. +# +# %CopyrightEnd% +# + +ARFLAGS = rc + +PCRE_O = \ +pcre_latin_1_table.o \ +pcre_compile.o \ +pcre_config.o \ +pcre_dfa_exec.o \ +pcre_exec.o \ +pcre_fullinfo.o \ +pcre_get.o \ +pcre_globals.o \ +pcre_info.o \ +pcre_maketables.o \ +pcre_newline.o \ +pcre_ord2utf8.o \ +pcre_refcount.o \ +pcre_study.o \ +pcre_tables.o \ +pcre_try_flipped.o \ +pcre_ucp_searchfuncs.o \ +pcre_valid_utf8.o \ +pcre_version.o \ +pcre_xclass.o + +PCRE_OBJS = $(PCRE_O:%=$(PCRE_OBJDIR)/%) + +GENINC = pcre/pcre_exec_loop_break_cases.inc + +PCRE_OBJDIR = $(ERL_TOP)/erts/emulator/pcre/obj/$(TARGET)/$(TYPE) + +PCRE_CFLAGS = $(filter-out -DDEBUG,$(CFLAGS)) -DERLANG_INTEGRATION + +ifeq ($(TARGET), win32) +$(EPCRE_LIB): $(PCRE_OBJS) + $(AR) -out:$@ $(PCRE_OBJS) +else +$(EPCRE_LIB): $(PCRE_OBJS) + $(AR) $(ARFLAGS) $@ $(PCRE_OBJS) + -@ ($(RANLIB) $@ || true) 2>/dev/null +endif + +$(PCRE_OBJDIR)/%.o: pcre/%.c + $(CC) -c $(PCRE_CFLAGS) -o $@ $< + +$(GENINC): pcre/pcre_exec.c + for x in `grep -n COST_CHK pcre/pcre_exec.c | grep -v 'COST_CHK(N)' | awk -F: '{print $$1}'`; \ + do \ + N=`expr $$x + 100`; \ + echo "case $$N: goto L_LOOP_COUNT_$${x};"; \ + done > $(GENINC) + +# Dependencies. + +$(PCRE_OBJDIR)/pcre_chartables.o: pcre/pcre_chartables.c pcre/pcre_internal.h \ + pcre/local_config.h pcre/pcre.h pcre/ucp.h +$(PCRE_OBJDIR)/pcre_compile.o: pcre/pcre_compile.c pcre/pcre_internal.h \ + pcre/local_config.h pcre/pcre.h pcre/ucp.h +$(PCRE_OBJDIR)/pcre_config.o: pcre/pcre_config.c pcre/pcre_internal.h \ + pcre/local_config.h pcre/pcre.h pcre/ucp.h +$(PCRE_OBJDIR)/pcre_dfa_exec.o: pcre/pcre_dfa_exec.c pcre/pcre_internal.h \ + pcre/local_config.h pcre/pcre.h pcre/ucp.h +$(PCRE_OBJDIR)/pcre_exec.o: pcre/pcre_exec.c pcre/pcre_internal.h \ + pcre/local_config.h pcre/pcre.h pcre/ucp.h $(GENINC) +$(PCRE_OBJDIR)/pcre_fullinfo.o: pcre/pcre_fullinfo.c pcre/pcre_internal.h \ + pcre/local_config.h pcre/pcre.h pcre/ucp.h +$(PCRE_OBJDIR)/pcre_get.o: pcre/pcre_get.c pcre/pcre_internal.h \ + pcre/local_config.h pcre/pcre.h pcre/ucp.h +$(PCRE_OBJDIR)/pcre_globals.o: pcre/pcre_globals.c pcre/pcre_internal.h \ + pcre/local_config.h pcre/pcre.h pcre/ucp.h +$(PCRE_OBJDIR)/pcre_info.o: pcre/pcre_info.c pcre/pcre_internal.h \ + pcre/local_config.h pcre/pcre.h pcre/ucp.h +$(PCRE_OBJDIR)/pcre_maketables.o: pcre/pcre_maketables.c pcre/pcre_internal.h \ + pcre/local_config.h pcre/pcre.h pcre/ucp.h +$(PCRE_OBJDIR)/pcre_newline.o: pcre/pcre_newline.c pcre/pcre_internal.h \ + pcre/local_config.h pcre/pcre.h pcre/ucp.h +$(PCRE_OBJDIR)/pcre_ord2utf8.o: pcre/pcre_ord2utf8.c pcre/pcre_internal.h \ + pcre/local_config.h pcre/pcre.h pcre/ucp.h +$(PCRE_OBJDIR)/pcre/pcre_refcount.o: pcre/pcre_refcount.c pcre/pcre_internal.h \ + pcre/local_config.h pcre/pcre.h pcre/ucp.h +$(PCRE_OBJDIR)/pcre_study.o: pcre/pcre_study.c pcre/pcre_internal.h \ + pcre/local_config.h pcre/pcre.h pcre/ucp.h +$(PCRE_OBJDIR)/pcre_tables.o: pcre/pcre_tables.c pcre/pcre_internal.h \ + pcre/local_config.h pcre/pcre.h pcre/ucp.h +$(PCRE_OBJDIR)/pcre_try_flipped.o: pcre/pcre_try_flipped.c pcre/pcre_internal.h \ + pcre/local_config.h pcre/pcre.h pcre/ucp.h +$(PCRE_OBJDIR)/pcre_ucp_searchfuncs.o: pcre/pcre_ucp_searchfuncs.c \ + pcre/pcre_internal.h pcre/local_config.h pcre/pcre.h pcre/ucp.h \ + pcre/ucpinternal.h pcre/ucptable.h +$(PCRE_OBJDIR)/pcre_valid_utf8.o: pcre/pcre_valid_utf8.c pcre/pcre_internal.h \ + pcre/local_config.h pcre/pcre.h pcre/ucp.h +pcre_version.o: pcre/pcre_version.c pcre/pcre_internal.h pcre/local_config.h \ + pcre/pcre.h pcre/ucp.h +$(PCRE_OBJDIR)/pcre/pcre_xclass.o: pcre/pcre_xclass.c pcre/pcre_internal.h \ + pcre/local_config.h pcre/pcre.h pcre/ucp.h diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c index cd4de21d65..ba88fd1d39 100644 --- a/erts/emulator/sys/common/erl_check_io.c +++ b/erts/emulator/sys/common/erl_check_io.c @@ -35,6 +35,7 @@ #include "sys.h" #include "global.h" #include "erl_check_io.h" +#include "erl_thr_progress.h" #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS # define ERTS_DRV_EV_STATE_EXTRA_SIZE 128 @@ -66,6 +67,9 @@ typedef char EventStateFlags; #define ERTS_CIO_POLL_CTL ERTS_POLL_EXPORT(erts_poll_control) #define ERTS_CIO_POLL_WAIT ERTS_POLL_EXPORT(erts_poll_wait) +#ifdef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT +#define ERTS_CIO_POLL_AS_INTR ERTS_POLL_EXPORT(erts_poll_async_sig_interrupt) +#endif #define ERTS_CIO_POLL_INTR ERTS_POLL_EXPORT(erts_poll_interrupt) #define ERTS_CIO_POLL_INTR_TMD ERTS_POLL_EXPORT(erts_poll_interrupt_timed) #define ERTS_CIO_NEW_POLLSET ERTS_POLL_EXPORT(erts_poll_create_pollset) @@ -218,7 +222,7 @@ remember_removed(ErtsDrvEventState *state, struct pollset_info* psi) #ifdef ERTS_SMP struct removed_fd *fdlp; ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(fd_mtx(state->fd))); - if (erts_smp_atomic_read(&psi->in_poll_wait)) { + if (erts_smp_atomic_read_nob(&psi->in_poll_wait)) { state->remove_cnt++; ASSERT(state->remove_cnt > 0); fdlp = removed_fd_alloc(); @@ -333,7 +337,7 @@ grow_drv_ev_state(int min_ix) new_len = max_fds; erts_smp_mtx_lock(&drv_ev_state_grow_lock); - if (erts_smp_atomic_read(&drv_ev_state_len) <= min_ix) { + if (erts_smp_atomic_read_nob(&drv_ev_state_len) <= min_ix) { for (i=0; i<DRV_EV_STATE_LOCK_CNT; i++) { /* lock all fd's */ erts_smp_mtx_lock(&drv_ev_state_locks[i].lck); } @@ -343,7 +347,7 @@ grow_drv_ev_state(int min_ix) sizeof(ErtsDrvEventState)*new_len) : erts_alloc(ERTS_ALC_T_DRV_EV_STATE, sizeof(ErtsDrvEventState)*new_len)); - for (i = erts_smp_atomic_read(&drv_ev_state_len); i < new_len; i++) { + for (i = erts_smp_atomic_read_nob(&drv_ev_state_len); i < new_len; i++) { drv_ev_state[i].fd = (ErtsSysFdType) i; drv_ev_state[i].driver.select = NULL; drv_ev_state[i].events = 0; @@ -351,7 +355,7 @@ grow_drv_ev_state(int min_ix) drv_ev_state[i].type = ERTS_EV_TYPE_NONE; drv_ev_state[i].flags = 0; } - erts_smp_atomic_set(&drv_ev_state_len, new_len); + erts_smp_atomic_set_nob(&drv_ev_state_len, new_len); for (i=0; i<DRV_EV_STATE_LOCK_CNT; i++) { erts_smp_mtx_unlock(&drv_ev_state_locks[i].lck); } @@ -497,7 +501,7 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix, && erts_lc_is_port_locked(erts_drvport2port(ix))); #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - if ((unsigned)fd >= (unsigned)erts_smp_atomic_read(&drv_ev_state_len)) { + if ((unsigned)fd >= (unsigned)erts_smp_atomic_read_nob(&drv_ev_state_len)) { if (fd < 0) { return -1; } @@ -709,7 +713,7 @@ ERTS_CIO_EXPORT(driver_event)(ErlDrvPort ix, && erts_lc_is_port_locked(erts_drvport2port(ix))); #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - if ((unsigned)fd >= (unsigned)erts_smp_atomic_read(&drv_ev_state_len)) { + if ((unsigned)fd >= (unsigned)erts_smp_atomic_read_nob(&drv_ev_state_len)) { if (fd < 0) return -1; if (fd >= max_fds) { @@ -1115,6 +1119,14 @@ eready(Eterm id, ErtsDrvEventState *state, ErlDrvEventData event_data) static void bad_fd_in_pollset( ErtsDrvEventState *, Eterm, Eterm, ErtsPollEvents); +#ifdef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT +void +ERTS_CIO_EXPORT(erts_check_io_async_sig_interrupt)(void) +{ + ERTS_CIO_POLL_AS_INTR(pollset.ps); +} +#endif + void ERTS_CIO_EXPORT(erts_check_io_interrupt)(int set) { @@ -1153,17 +1165,15 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait) #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_check_exact(NULL, 0); /* No locks should be locked */ #endif - erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); pollres_len = sizeof(pollres)/sizeof(ErtsPollResFd); - erts_smp_atomic_set(&pollset.in_poll_wait, 1); + erts_smp_atomic_set_nob(&pollset.in_poll_wait, 1); poll_ret = ERTS_CIO_POLL_WAIT(pollset.ps, pollres, &pollres_len, &wait_time); #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_check_exact(NULL, 0); /* No locks should be locked */ #endif - erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); erts_deliver_time(); /* sync the machine's idea of time */ @@ -1173,7 +1183,7 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait) #endif if (poll_ret != 0) { - erts_smp_atomic_set(&pollset.in_poll_wait, 0); + erts_smp_atomic_set_nob(&pollset.in_poll_wait, 0); forget_removed(&pollset); if (poll_ret == EAGAIN) { goto restart; @@ -1304,7 +1314,7 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait) #endif } - erts_smp_atomic_set(&pollset.in_poll_wait, 0); + erts_smp_atomic_set_nob(&pollset.in_poll_wait, 0); forget_removed(&pollset); } @@ -1419,7 +1429,7 @@ static void drv_ev_state_free(void *des) void ERTS_CIO_EXPORT(erts_init_check_io)(void) { - erts_smp_atomic_init(&pollset.in_poll_wait, 0); + erts_smp_atomic_init_nob(&pollset.in_poll_wait, 0); ERTS_CIO_POLL_INIT(); pollset.ps = ERTS_CIO_NEW_POLLSET(); @@ -1441,7 +1451,7 @@ ERTS_CIO_EXPORT(erts_init_check_io)(void) #endif #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS max_fds = ERTS_CIO_POLL_MAX_FDS(); - erts_smp_atomic_init(&drv_ev_state_len, 0); + erts_smp_atomic_init_nob(&drv_ev_state_len, 0); drv_ev_state = NULL; erts_smp_mtx_init(&drv_ev_state_grow_lock, "drv_ev_state_grow"); #else @@ -1479,7 +1489,7 @@ ERTS_CIO_EXPORT(erts_check_io_size)(void) ERTS_CIO_POLL_INFO(pollset.ps, &pi); res = pi.memory_size; #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - res += sizeof(ErtsDrvEventState) * erts_smp_atomic_read(&drv_ev_state_len); + res += sizeof(ErtsDrvEventState) * erts_smp_atomic_read_nob(&drv_ev_state_len); #else res += safe_hash_table_sz(&drv_ev_state_tab); { @@ -1506,7 +1516,7 @@ ERTS_CIO_EXPORT(erts_check_io_info)(void *proc) ERTS_CIO_POLL_INFO(pollset.ps, &pi); memory_size = pi.memory_size; #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - memory_size += sizeof(ErtsDrvEventState) * erts_smp_atomic_read(&drv_ev_state_len); + memory_size += sizeof(ErtsDrvEventState) * erts_smp_atomic_read_nob(&drv_ev_state_len); #else memory_size += safe_hash_table_sz(&drv_ev_state_tab); { @@ -1870,13 +1880,12 @@ ERTS_CIO_EXPORT(erts_check_io_debug)(void) erts_printf("--- fds in pollset --------------------------------------\n"); -#ifdef ERTS_SMP -# ifdef ERTS_ENABLE_LOCK_CHECK +#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) erts_lc_check_exact(NULL, 0); /* No locks should be locked */ -# endif - erts_block_system(0); /* stop the world to avoid messy locking */ #endif + erts_smp_thr_progress_block(); /* stop the world to avoid messy locking */ + #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS counters.epep = erts_alloc(ERTS_ALC_T_TMP, sizeof(ErtsPollEvents)*max_fds); ERTS_POLL_EXPORT(erts_poll_get_selected_events)(pollset.ps, counters.epep, max_fds); @@ -1886,7 +1895,7 @@ ERTS_CIO_EXPORT(erts_check_io_debug)(void) counters.num_errors = 0; #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - len = erts_smp_atomic_read(&drv_ev_state_len); + len = erts_smp_atomic_read_nob(&drv_ev_state_len); for (fd = 0; fd < len; fd++) { doit_erts_check_io_debug((void *) &drv_ev_state[fd], (void *) &counters); } @@ -1898,9 +1907,7 @@ ERTS_CIO_EXPORT(erts_check_io_debug)(void) safe_hash_for_each(&drv_ev_state_tab, &doit_erts_check_io_debug, (void *) &counters); #endif -#ifdef ERTS_SMP - erts_release_system(); -#endif + erts_smp_thr_progress_unblock(); erts_printf("\n"); erts_printf("used fds=%d\n", counters.used_fds); diff --git a/erts/emulator/sys/common/erl_check_io.h b/erts/emulator/sys/common/erl_check_io.h index 9b45a63913..7cc1658062 100644 --- a/erts/emulator/sys/common/erl_check_io.h +++ b/erts/emulator/sys/common/erl_check_io.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2006-2009. All Rights Reserved. + * Copyright Ericsson AB 2006-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -40,6 +40,10 @@ Eterm erts_check_io_info_kp(void *); Eterm erts_check_io_info_nkp(void *); int erts_check_io_max_files_kp(void); int erts_check_io_max_files_nkp(void); +#ifdef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT +void erts_check_io_async_sig_interrupt_kp(void); +void erts_check_io_async_sig_interrupt_nkp(void); +#endif void erts_check_io_interrupt_kp(int); void erts_check_io_interrupt_nkp(int); void erts_check_io_interrupt_timed_kp(int, long); @@ -56,6 +60,9 @@ int erts_check_io_debug_nkp(void); Uint erts_check_io_size(void); Eterm erts_check_io_info(void *); int erts_check_io_max_files(void); +#ifdef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT +void erts_check_io_async_sig_interrupt(void); +#endif void erts_check_io_interrupt(int); void erts_check_io_interrupt_timed(int, long); void erts_check_io(int); diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c index eaef6680dd..49750ff6ce 100644 --- a/erts/emulator/sys/common/erl_mseg.c +++ b/erts/emulator/sys/common/erl_mseg.c @@ -36,14 +36,12 @@ #include "erl_threads.h" #include "erl_mtrace.h" #include "erl_time.h" +#include "erl_alloc.h" #include "big.h" +#include "erl_thr_progress.h" #if HAVE_ERTS_MSEG -#if defined(USE_THREADS) && !defined(ERTS_SMP) -# define ERTS_THREADS_NO_SMP -#endif - #define SEGTYPE ERTS_MTRACE_SEGMENT_ID #ifndef HAVE_GETPAGESIZE @@ -75,16 +73,9 @@ static int atoms_initialized; -static Uint cache_check_interval; - typedef struct mem_kind_t MemKind; -static void check_cache(void *unused); static void mseg_clear_cache(MemKind*); -static int is_cache_check_scheduled; -#ifdef ERTS_THREADS_NO_SMP -static int is_cache_check_requested; -#endif #if HALFWORD_HEAP static int initialize_pmmap(void); @@ -138,7 +129,8 @@ const ErtsMsegOpt_t erts_mseg_default_opt = { 1, /* Use cache */ 1, /* Preserv data */ 0, /* Absolute shrink threshold */ - 0 /* Relative shrink threshold */ + 0, /* Relative shrink threshold */ + 0 /* Scheduler specific */ #if HALFWORD_HEAP ,0 /* need low memory */ #endif @@ -157,11 +149,10 @@ typedef struct { Uint32 no; } CallCounter; -static int is_init_done; static Uint page_size; static Uint page_shift; -static struct { +typedef struct { CallCounter alloc; CallCounter dealloc; CallCounter realloc; @@ -172,7 +163,9 @@ static struct { #endif CallCounter clear_cache; CallCounter check_cache; -} calls; +} ErtsMsegCalls; + +typedef struct ErtsMsegAllctr_t_ ErtsMsegAllctr_t; struct mem_kind_t { cache_desc_t cache_descs[MAX_CACHE_SIZE]; @@ -201,25 +194,84 @@ struct mem_kind_t { } max_ever; } segments; + ErtsMsegAllctr_t *ma; const char* name; MemKind* next; };/*MemKind*/ +struct ErtsMsegAllctr_t_ { + int ix; + + int is_init_done; + int is_thread_safe; + erts_mtx_t mtx; + + int is_cache_check_scheduled; + + MemKind* mk_list; + #if HALFWORD_HEAP -static MemKind low_mem, hi_mem; + MemKind low_mem; + MemKind hi_mem; #else -static MemKind the_mem; + MemKind the_mem; #endif -static MemKind* mk_list = NULL; -static Uint max_cache_size; -static Uint abs_max_cache_bad_fit; -static Uint rel_max_cache_bad_fit; + Uint max_cache_size; + Uint abs_max_cache_bad_fit; + Uint rel_max_cache_bad_fit; + + ErtsMsegCalls calls; #if CAN_PARTLY_DESTROY -static Uint min_seg_size; + Uint min_seg_size; +#endif + +}; + +typedef union { + ErtsMsegAllctr_t mseg_alloc; + char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsMsegAllctr_t))]; +} ErtsAlgndMsegAllctr_t; + +static int no_mseg_allocators; +static ErtsAlgndMsegAllctr_t *aligned_mseg_allctr; + +#ifdef ERTS_SMP + +#define ERTS_MSEG_ALLCTR_IX(IX) \ + (&aligned_mseg_allctr[(IX)].mseg_alloc) + +#define ERTS_MSEG_ALLCTR_SS() \ + ERTS_MSEG_ALLCTR_IX((int) erts_get_scheduler_id()) + +#define ERTS_MSEG_ALLCTR_OPT(OPT) \ + ((OPT)->sched_spec ? ERTS_MSEG_ALLCTR_SS() : ERTS_MSEG_ALLCTR_IX(0)) + +#else + +#define ERTS_MSEG_ALLCTR_IX(IX) \ + (&aligned_mseg_allctr[0].mseg_alloc) + +#define ERTS_MSEG_ALLCTR_SS() \ + (&aligned_mseg_allctr[0].mseg_alloc) + +#define ERTS_MSEG_ALLCTR_OPT(OPT) \ + (&aligned_mseg_allctr[0].mseg_alloc) + #endif +#define ERTS_MSEG_LOCK(MA) \ +do { \ + if ((MA)->is_thread_safe) \ + erts_mtx_lock(&(MA)->mtx); \ +} while (0) + +#define ERTS_MSEG_UNLOCK(MA) \ +do { \ + if ((MA)->is_thread_safe) \ + erts_mtx_unlock(&(MA)->mtx); \ +} while (0) #define ERTS_MSEG_ALLOC_STAT(C,SZ) \ do { \ @@ -250,104 +302,44 @@ do { \ #define ONE_GIGA (1000000000) -#define ZERO_CC(CC) (calls.CC.no = 0, calls.CC.giga_no = 0) +#define ZERO_CC(MA, CC) ((MA)->calls.CC.no = 0, \ + (MA)->calls.CC.giga_no = 0) -#define INC_CC(CC) (calls.CC.no == ONE_GIGA - 1 \ - ? (calls.CC.giga_no++, calls.CC.no = 0) \ - : calls.CC.no++) +#define INC_CC(MA, CC) ((MA)->calls.CC.no == ONE_GIGA - 1 \ + ? ((MA)->calls.CC.giga_no++, \ + (MA)->calls.CC.no = 0) \ + : (MA)->calls.CC.no++) -#define DEC_CC(CC) (calls.CC.no == 0 \ - ? (calls.CC.giga_no--, \ - calls.CC.no = ONE_GIGA - 1) \ - : calls.CC.no--) +#define DEC_CC(MA, CC) ((MA)->calls.CC.no == 0 \ + ? ((MA)->calls.CC.giga_no--, \ + (MA)->calls.CC.no = ONE_GIGA - 1) \ + : (MA)->calls.CC.no--) -static erts_mtx_t mseg_mutex; /* Also needed when !USE_THREADS */ static erts_mtx_t init_atoms_mutex; /* Also needed when !USE_THREADS */ -#ifdef USE_THREADS -#ifdef ERTS_THREADS_NO_SMP -static erts_tid_t main_tid; -static int async_handle = -1; -#endif - -static void thread_safe_init(void) -{ - erts_mtx_init(&init_atoms_mutex, "mseg_init_atoms"); - erts_mtx_init(&mseg_mutex, "mseg"); - -#ifdef ERTS_THREADS_NO_SMP - main_tid = erts_thr_self(); -#endif -} - -#endif - -static ErlTimer cache_check_timer; static ERTS_INLINE void -schedule_cache_check(void) -{ - if (!is_cache_check_scheduled && is_init_done) { -#ifdef ERTS_THREADS_NO_SMP - if (!erts_equal_tids(erts_thr_self(), main_tid)) { - if (!is_cache_check_requested) { - is_cache_check_requested = 1; - sys_async_ready(async_handle); - } - } - else -#endif - { - cache_check_timer.active = 0; - erts_set_timer(&cache_check_timer, - check_cache, - NULL, - NULL, - cache_check_interval); - is_cache_check_scheduled = 1; -#ifdef ERTS_THREADS_NO_SMP - is_cache_check_requested = 0; -#endif - } - } -} - -#ifdef ERTS_THREADS_NO_SMP - -static void -check_schedule_cache_check(void) +schedule_cache_check(ErtsMsegAllctr_t *ma) { - erts_mtx_lock(&mseg_mutex); - if (is_cache_check_requested - && !is_cache_check_scheduled) { - schedule_cache_check(); - } - erts_mtx_unlock(&mseg_mutex); -} - -#endif -static void -mseg_shutdown(void) -{ - MemKind* mk; - erts_mtx_lock(&mseg_mutex); - for (mk=mk_list; mk; mk=mk->next) { - mseg_clear_cache(mk); + if (!ma->is_cache_check_scheduled && ma->is_init_done) { + erts_set_aux_work_timeout(ma->ix, + ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK, + 1); + ma->is_cache_check_scheduled = 1; } - erts_mtx_unlock(&mseg_mutex); } static ERTS_INLINE void * -mseg_create(MemKind* mk, Uint size) +mseg_create(ErtsMsegAllctr_t *ma, MemKind* mk, Uint size) { void *seg; ASSERT(size % page_size == 0); #if HALFWORD_HEAP - if (mk == &low_mem) { + if (mk == &ma->low_mem) { seg = pmmap(size); if ((unsigned long) seg & CHECK_POINTER_MASK) { erts_fprintf(stderr,"Pointer mask failure (0x%08lx)\n",(unsigned long) seg); @@ -371,18 +363,18 @@ mseg_create(MemKind* mk, Uint size) #endif } - INC_CC(create); + INC_CC(ma, create); return seg; } static ERTS_INLINE void -mseg_destroy(MemKind* mk, void *seg, Uint size) +mseg_destroy(ErtsMsegAllctr_t *ma, MemKind* mk, void *seg, Uint size) { int res; #if HALFWORD_HEAP - if (mk == &low_mem) { + if (mk == &ma->low_mem) { res = pmunmap((void *) seg, size); } else @@ -401,14 +393,14 @@ mseg_destroy(MemKind* mk, void *seg, Uint size) ASSERT(size % page_size == 0); ASSERT(res == 0); - INC_CC(destroy); + INC_CC(ma, destroy); } #if HAVE_MSEG_RECREATE static ERTS_INLINE void * -mseg_recreate(MemKind* mk, void *old_seg, Uint old_size, Uint new_size) +mseg_recreate(ErtsMsegAllctr_t *ma, MemKind* mk, void *old_seg, Uint old_size, Uint new_size) { void *new_seg; @@ -416,7 +408,7 @@ mseg_recreate(MemKind* mk, void *old_seg, Uint old_size, Uint new_size) ASSERT(new_size % page_size == 0); #if HALFWORD_HEAP - if (mk == &low_mem) { + if (mk == &ma->low_mem) { new_seg = (void *) pmremap((void *) old_seg, (size_t) old_size, (size_t) new_size); @@ -447,19 +439,37 @@ mseg_recreate(MemKind* mk, void *old_seg, Uint old_size, Uint new_size) #endif } - INC_CC(recreate); + INC_CC(ma, recreate); return new_seg; } #endif /* #if HAVE_MSEG_RECREATE */ +#ifdef DEBUG +#define ERTS_DBG_MA_CHK_THR_ACCESS(MA) \ +do { \ + if ((MA)->is_thread_safe) \ + ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&(MA)->mtx) \ + || erts_smp_thr_progress_is_blocking() \ + || ERTS_IS_CRASH_DUMPING); \ + else \ + ERTS_LC_ASSERT((MA)->ix == (int) erts_get_scheduler_id() \ + || erts_smp_thr_progress_is_blocking() \ + || ERTS_IS_CRASH_DUMPING); \ +} while (0) +#define ERTS_DBG_MK_CHK_THR_ACCESS(MK) \ + ERTS_DBG_MA_CHK_THR_ACCESS((MK)->ma) +#else +#define ERTS_DBG_MA_CHK_THR_ACCESS(MA) +#define ERTS_DBG_MK_CHK_THR_ACCESS(MK) +#endif static ERTS_INLINE cache_desc_t * alloc_cd(MemKind* mk) { cache_desc_t *cd = mk->free_cache_descs; - ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&mseg_mutex)); + ERTS_DBG_MK_CHK_THR_ACCESS(mk); if (cd) mk->free_cache_descs = cd->next; return cd; @@ -468,7 +478,7 @@ alloc_cd(MemKind* mk) static ERTS_INLINE void free_cd(MemKind* mk, cache_desc_t *cd) { - ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&mseg_mutex)); + ERTS_DBG_MK_CHK_THR_ACCESS(mk); cd->next = mk->free_cache_descs; mk->free_cache_descs = cd; } @@ -477,7 +487,7 @@ free_cd(MemKind* mk, cache_desc_t *cd) static ERTS_INLINE void link_cd(MemKind* mk, cache_desc_t *cd) { - ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&mseg_mutex)); + ERTS_DBG_MK_CHK_THR_ACCESS(mk); if (mk->cache) mk->cache->prev = cd; cd->next = mk->cache; @@ -496,7 +506,7 @@ link_cd(MemKind* mk, cache_desc_t *cd) static ERTS_INLINE void end_link_cd(MemKind* mk, cache_desc_t *cd) { - ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&mseg_mutex)); + ERTS_DBG_MK_CHK_THR_ACCESS(mk); if (mk->cache_end) mk->cache_end->next = cd; cd->next = NULL; @@ -515,7 +525,7 @@ end_link_cd(MemKind* mk, cache_desc_t *cd) static ERTS_INLINE void unlink_cd(MemKind* mk, cache_desc_t *cd) { - ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&mseg_mutex)); + ERTS_DBG_MK_CHK_THR_ACCESS(mk); if (cd->next) cd->next->prev = cd->prev; else @@ -533,7 +543,7 @@ static ERTS_INLINE void check_cache_limits(MemKind* mk) { cache_desc_t *cd; - ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&mseg_mutex)); + ERTS_DBG_MK_CHK_THR_ACCESS(mk); mk->max_cached_seg_size = 0; mk->min_cached_seg_size = ~((Uint) 0); for (cd = mk->cache; cd; cd = cd->next) { @@ -551,7 +561,7 @@ adjust_cache_size(MemKind* mk, int force_check_limits) int check_limits = force_check_limits; Sint max_cached = ((Sint) mk->segments.current.watermark - (Sint) mk->segments.current.no); - ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&mseg_mutex)); + ERTS_DBG_MK_CHK_THR_ACCESS(mk); while (((Sint) mk->cache_size) > max_cached && ((Sint) mk->cache_size) > 0) { ASSERT(mk->cache_end); cd = mk->cache_end; @@ -562,7 +572,7 @@ adjust_cache_size(MemKind* mk, int force_check_limits) } if (erts_mtrace_enabled) erts_mtrace_crr_free(SEGTYPE, SEGTYPE, cd->seg); - mseg_destroy(mk, cd->seg, cd->size); + mseg_destroy(mk->ma, mk, cd->seg, cd->size); unlink_cd(mk,cd); free_cd(mk,cd); } @@ -571,7 +581,7 @@ adjust_cache_size(MemKind* mk, int force_check_limits) check_cache_limits(mk); } -static void +static Uint check_one_cache(MemKind* mk) { if (mk->segments.current.watermark > mk->segments.current.no) @@ -579,23 +589,37 @@ check_one_cache(MemKind* mk) adjust_cache_size(mk, 0); if (mk->cache_size) - schedule_cache_check(); + schedule_cache_check(mk->ma); + return mk->cache_size; } -static void check_cache(void* unused) +static void do_cache_check(ErtsMsegAllctr_t *ma) { + int empty_cache = 1; MemKind* mk; - erts_mtx_lock(&mseg_mutex); - is_cache_check_scheduled = 0; + ERTS_MSEG_LOCK(ma); - for (mk=mk_list; mk; mk=mk->next) { - check_one_cache(mk); + for (mk=ma->mk_list; mk; mk=mk->next) { + if (check_one_cache(mk)) + empty_cache = 0; + } + + if (empty_cache) { + ma->is_cache_check_scheduled = 0; + erts_set_aux_work_timeout(ma->ix, + ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK, + 0); } - INC_CC(check_cache); + INC_CC(ma, check_cache); - erts_mtx_unlock(&mseg_mutex); + ERTS_MSEG_UNLOCK(ma); +} + +void erts_mseg_cache_check(void) +{ + do_cache_check(ERTS_MSEG_ALLCTR_SS()); } static void @@ -611,42 +635,44 @@ mseg_clear_cache(MemKind* mk) mk->segments.current.watermark = mk->segments.current.no; - INC_CC(clear_cache); + INC_CC(mk->ma, clear_cache); } -static ERTS_INLINE MemKind* memkind(const ErtsMsegOpt_t *opt) +static ERTS_INLINE MemKind* memkind(ErtsMsegAllctr_t *ma, + const ErtsMsegOpt_t *opt) { #if HALFWORD_HEAP - return opt->low_mem ? &low_mem : &hi_mem; + return opt->low_mem ? &ma->low_mem : &ma->hi_mem; #else - return &the_mem; + return &ma->the_mem; #endif } static void * -mseg_alloc(ErtsAlcType_t atype, Uint *size_p, const ErtsMsegOpt_t *opt) +mseg_alloc(ErtsMsegAllctr_t *ma, ErtsAlcType_t atype, Uint *size_p, + const ErtsMsegOpt_t *opt) { Uint max, min, diff_size, size; cache_desc_t *cd, *cand_cd; void *seg; - MemKind* mk = memkind(opt); + MemKind* mk = memkind(ma, opt); - INC_CC(alloc); + INC_CC(ma, alloc); size = PAGE_CEILING(*size_p); #if CAN_PARTLY_DESTROY - if (size < min_seg_size) - min_seg_size = size; + if (size < ma->min_seg_size) + ma->min_seg_size = size; #endif if (!opt->cache) { create_seg: adjust_cache_size(mk,0); - seg = mseg_create(mk, size); + seg = mseg_create(ma, mk, size); if (!seg) { mseg_clear_cache(mk); - seg = mseg_create(mk, size); + seg = mseg_create(ma, mk, size); if (!seg) size = 0; } @@ -667,10 +693,10 @@ mseg_alloc(ErtsAlcType_t atype, Uint *size_p, const ErtsMsegOpt_t *opt) diff_size = mk->min_cached_seg_size - size; - if (diff_size > abs_max_cache_bad_fit) + if (diff_size > ma->abs_max_cache_bad_fit) goto create_seg; - if (100*PAGES(diff_size) > rel_max_cache_bad_fit*PAGES(size)) + if (100*PAGES(diff_size) > ma->rel_max_cache_bad_fit*PAGES(size)) goto create_seg; } @@ -708,8 +734,8 @@ mseg_alloc(ErtsAlcType_t atype, Uint *size_p, const ErtsMsegOpt_t *opt) diff_size = cand_cd->size - size; - if (diff_size > abs_max_cache_bad_fit - || 100*PAGES(diff_size) > rel_max_cache_bad_fit*PAGES(size)) { + if (diff_size > ma->abs_max_cache_bad_fit + || 100*PAGES(diff_size) > ma->rel_max_cache_bad_fit*PAGES(size)) { if (mk->max_cached_seg_size < cand_cd->size) mk->max_cached_seg_size = cand_cd->size; if (mk->min_cached_seg_size > cand_cd->size) @@ -740,18 +766,18 @@ mseg_alloc(ErtsAlcType_t atype, Uint *size_p, const ErtsMsegOpt_t *opt) static void -mseg_dealloc(ErtsAlcType_t atype, void *seg, Uint size, +mseg_dealloc(ErtsMsegAllctr_t *ma, ErtsAlcType_t atype, void *seg, Uint size, const ErtsMsegOpt_t *opt) { - MemKind* mk = memkind(opt); + MemKind* mk = memkind(ma, opt); cache_desc_t *cd; ERTS_MSEG_DEALLOC_STAT(mk,size); - if (!opt->cache || max_cache_size == 0) { + if (!opt->cache || ma->max_cache_size == 0) { if (erts_mtrace_enabled) erts_mtrace_crr_free(atype, SEGTYPE, seg); - mseg_destroy(mk, seg, size); + mseg_destroy(ma, mk, seg, size); } else { int check_limits = 0; @@ -769,7 +795,7 @@ mseg_dealloc(ErtsAlcType_t atype, void *seg, Uint size, } if (erts_mtrace_enabled) erts_mtrace_crr_free(SEGTYPE, SEGTYPE, cd->seg); - mseg_destroy(mk, cd->seg, cd->size); + mseg_destroy(ma, mk, cd->seg, cd->size); unlink_cd(mk,cd); free_cd(mk,cd); } @@ -790,33 +816,34 @@ mseg_dealloc(ErtsAlcType_t atype, void *seg, Uint size, if (check_limits) check_cache_limits(mk); - schedule_cache_check(); + schedule_cache_check(ma); } - INC_CC(dealloc); + INC_CC(ma, dealloc); } static void * -mseg_realloc(ErtsAlcType_t atype, void *seg, Uint old_size, Uint *new_size_p, - const ErtsMsegOpt_t *opt) +mseg_realloc(ErtsMsegAllctr_t *ma, ErtsAlcType_t atype, void *seg, + Uint old_size, Uint *new_size_p, const ErtsMsegOpt_t *opt) { - MemKind* mk = memkind(opt); + MemKind* mk; void *new_seg; Uint new_size; if (!seg || !old_size) { - new_seg = mseg_alloc(atype, new_size_p, opt); - DEC_CC(alloc); + new_seg = mseg_alloc(ma, atype, new_size_p, opt); + DEC_CC(ma, alloc); return new_seg; } if (!(*new_size_p)) { - mseg_dealloc(atype, seg, old_size, opt); - DEC_CC(dealloc); + mseg_dealloc(ma, atype, seg, old_size, opt); + DEC_CC(ma, dealloc); return NULL; } + mk = memkind(ma, opt); new_seg = seg; new_size = PAGE_CEILING(*new_size_p); @@ -826,8 +853,8 @@ mseg_realloc(ErtsAlcType_t atype, void *seg, Uint old_size, Uint *new_size_p, Uint shrink_sz = old_size - new_size; #if CAN_PARTLY_DESTROY - if (new_size < min_seg_size) - min_seg_size = new_size; + if (new_size < ma->min_seg_size) + ma->min_seg_size = new_size; #endif if (shrink_sz < opt->abs_shrink_th @@ -838,7 +865,7 @@ mseg_realloc(ErtsAlcType_t atype, void *seg, Uint old_size, Uint *new_size_p, #if CAN_PARTLY_DESTROY - if (shrink_sz > min_seg_size + if (shrink_sz > ma->min_seg_size && mk->free_cache_descs && opt->cache) { cache_desc_t *cd; @@ -857,7 +884,7 @@ mseg_realloc(ErtsAlcType_t atype, void *seg, Uint old_size, Uint *new_size_p, new_size); erts_mtrace_crr_alloc(cd->seg, SEGTYPE, SEGTYPE, cd->size); } - schedule_cache_check(); + schedule_cache_check(ma); } else { if (erts_mtrace_enabled) @@ -866,7 +893,7 @@ mseg_realloc(ErtsAlcType_t atype, void *seg, Uint old_size, Uint *new_size_p, SEGTYPE, seg, new_size); - mseg_destroy(mk, ((char *) seg) + new_size, shrink_sz); + mseg_destroy(ma, mk, ((char *) seg) + new_size, shrink_sz); } #elif HAVE_MSEG_RECREATE @@ -875,14 +902,14 @@ mseg_realloc(ErtsAlcType_t atype, void *seg, Uint old_size, Uint *new_size_p, #else - new_seg = mseg_alloc(atype, &new_size, opt); + new_seg = mseg_alloc(ma, atype, &new_size, opt); if (!new_seg) new_size = old_size; else { sys_memcpy(((char *) new_seg), ((char *) seg), MIN(new_size, old_size)); - mseg_dealloc(atype, seg, old_size, opt); + mseg_dealloc(ma, atype, seg, old_size, opt); } #endif @@ -892,34 +919,34 @@ mseg_realloc(ErtsAlcType_t atype, void *seg, Uint old_size, Uint *new_size_p, else { if (!opt->preserv) { - mseg_dealloc(atype, seg, old_size, opt); - new_seg = mseg_alloc(atype, &new_size, opt); + mseg_dealloc(ma, atype, seg, old_size, opt); + new_seg = mseg_alloc(ma, atype, &new_size, opt); } else { #if HAVE_MSEG_RECREATE #if !CAN_PARTLY_DESTROY do_recreate: #endif - new_seg = mseg_recreate(mk, (void *) seg, old_size, new_size); + new_seg = mseg_recreate(ma, mk, (void *) seg, old_size, new_size); if (erts_mtrace_enabled) erts_mtrace_crr_realloc(new_seg, atype, SEGTYPE, seg, new_size); if (!new_seg) new_size = old_size; #else - new_seg = mseg_alloc(atype, &new_size, opt); + new_seg = mseg_alloc(ma, atype, &new_size, opt); if (!new_seg) new_size = old_size; else { sys_memcpy(((char *) new_seg), ((char *) seg), MIN(new_size, old_size)); - mseg_dealloc(atype, seg, old_size, opt); + mseg_dealloc(ma, atype, seg, old_size, opt); } #endif } } - INC_CC(realloc); + INC_CC(ma, realloc); *new_size_p = new_size; @@ -937,7 +964,6 @@ static struct { Eterm amcbf; Eterm rmcbf; Eterm mcs; - Eterm cci; Eterm memkind; Eterm name; @@ -973,13 +999,13 @@ static void ERTS_INLINE atom_init(Eterm *atom, char *name) #define AM_INIT(AM) atom_init(&am.AM, #AM) static void -init_atoms(void) +init_atoms(ErtsMsegAllctr_t *ma) { #ifdef DEBUG Eterm *atom; #endif - erts_mtx_unlock(&mseg_mutex); + ERTS_MSEG_UNLOCK(ma); erts_mtx_lock(&init_atoms_mutex); if (!atoms_initialized) { @@ -997,7 +1023,6 @@ init_atoms(void) AM_INIT(amcbf); AM_INIT(rmcbf); AM_INIT(mcs); - AM_INIT(cci); AM_INIT(status); AM_INIT(cached_segments); @@ -1025,7 +1050,7 @@ init_atoms(void) #endif } - erts_mtx_lock(&mseg_mutex); + ERTS_MSEG_LOCK(ma); atoms_initialized = 1; erts_mtx_unlock(&init_atoms_mutex); } @@ -1082,7 +1107,8 @@ add_4tup(Uint **hpp, Uint *szp, Eterm *lp, } static Eterm -info_options(char *prefix, +info_options(ErtsMsegAllctr_t *ma, + char *prefix, int *print_to_p, void *print_to_arg, Uint **hpp, @@ -1093,30 +1119,26 @@ info_options(char *prefix, if (print_to_p) { int to = *print_to_p; void *arg = print_to_arg; - erts_print(to, arg, "%samcbf: %beu\n", prefix, abs_max_cache_bad_fit); - erts_print(to, arg, "%srmcbf: %beu\n", prefix, rel_max_cache_bad_fit); - erts_print(to, arg, "%smcs: %beu\n", prefix, max_cache_size); - erts_print(to, arg, "%scci: %beu\n", prefix, cache_check_interval); + erts_print(to, arg, "%samcbf: %beu\n", prefix, ma->abs_max_cache_bad_fit); + erts_print(to, arg, "%srmcbf: %beu\n", prefix, ma->rel_max_cache_bad_fit); + erts_print(to, arg, "%smcs: %beu\n", prefix, ma->max_cache_size); } if (hpp || szp) { if (!atoms_initialized) - init_atoms(); + init_atoms(ma); res = NIL; add_2tup(hpp, szp, &res, - am.cci, - bld_uint(hpp, szp, cache_check_interval)); - add_2tup(hpp, szp, &res, am.mcs, - bld_uint(hpp, szp, max_cache_size)); + bld_uint(hpp, szp, ma->max_cache_size)); add_2tup(hpp, szp, &res, am.rmcbf, - bld_uint(hpp, szp, rel_max_cache_bad_fit)); + bld_uint(hpp, szp, ma->rel_max_cache_bad_fit)); add_2tup(hpp, szp, &res, am.amcbf, - bld_uint(hpp, szp, abs_max_cache_bad_fit)); + bld_uint(hpp, szp, ma->abs_max_cache_bad_fit)); } @@ -1124,18 +1146,18 @@ info_options(char *prefix, } static Eterm -info_calls(int *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp) +info_calls(ErtsMsegAllctr_t *ma, int *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp) { Eterm res = THE_NON_VALUE; if (print_to_p) { -#define PRINT_CC(TO, TOA, CC) \ - if (calls.CC.giga_no == 0) \ - erts_print(TO, TOA, "mseg_%s calls: %b32u\n", #CC, calls.CC.no); \ - else \ +#define PRINT_CC(TO, TOA, CC) \ + if (ma->calls.CC.giga_no == 0) \ + erts_print(TO, TOA, "mseg_%s calls: %b32u\n", #CC, ma->calls.CC.no); \ + else \ erts_print(TO, TOA, "mseg_%s calls: %b32u%09b32u\n", #CC, \ - calls.CC.giga_no, calls.CC.no) + ma->calls.CC.giga_no, ma->calls.CC.no) int to = *print_to_p; void *arg = print_to_arg; @@ -1161,48 +1183,48 @@ info_calls(int *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp) add_3tup(hpp, szp, &res, am.mseg_check_cache, - bld_unstable_uint(hpp, szp, calls.check_cache.giga_no), - bld_unstable_uint(hpp, szp, calls.check_cache.no)); + bld_unstable_uint(hpp, szp, ma->calls.check_cache.giga_no), + bld_unstable_uint(hpp, szp, ma->calls.check_cache.no)); add_3tup(hpp, szp, &res, am.mseg_clear_cache, - bld_unstable_uint(hpp, szp, calls.clear_cache.giga_no), - bld_unstable_uint(hpp, szp, calls.clear_cache.no)); + bld_unstable_uint(hpp, szp, ma->calls.clear_cache.giga_no), + bld_unstable_uint(hpp, szp, ma->calls.clear_cache.no)); #if HAVE_MSEG_RECREATE add_3tup(hpp, szp, &res, am.mseg_recreate, - bld_unstable_uint(hpp, szp, calls.recreate.giga_no), - bld_unstable_uint(hpp, szp, calls.recreate.no)); + bld_unstable_uint(hpp, szp, ma->calls.recreate.giga_no), + bld_unstable_uint(hpp, szp, ma->calls.recreate.no)); #endif add_3tup(hpp, szp, &res, am.mseg_destroy, - bld_unstable_uint(hpp, szp, calls.destroy.giga_no), - bld_unstable_uint(hpp, szp, calls.destroy.no)); + bld_unstable_uint(hpp, szp, ma->calls.destroy.giga_no), + bld_unstable_uint(hpp, szp, ma->calls.destroy.no)); add_3tup(hpp, szp, &res, am.mseg_create, - bld_unstable_uint(hpp, szp, calls.create.giga_no), - bld_unstable_uint(hpp, szp, calls.create.no)); + bld_unstable_uint(hpp, szp, ma->calls.create.giga_no), + bld_unstable_uint(hpp, szp, ma->calls.create.no)); add_3tup(hpp, szp, &res, am.mseg_realloc, - bld_unstable_uint(hpp, szp, calls.realloc.giga_no), - bld_unstable_uint(hpp, szp, calls.realloc.no)); + bld_unstable_uint(hpp, szp, ma->calls.realloc.giga_no), + bld_unstable_uint(hpp, szp, ma->calls.realloc.no)); add_3tup(hpp, szp, &res, am.mseg_dealloc, - bld_unstable_uint(hpp, szp, calls.dealloc.giga_no), - bld_unstable_uint(hpp, szp, calls.dealloc.no)); + bld_unstable_uint(hpp, szp, ma->calls.dealloc.giga_no), + bld_unstable_uint(hpp, szp, ma->calls.dealloc.no)); add_3tup(hpp, szp, &res, am.mseg_alloc, - bld_unstable_uint(hpp, szp, calls.alloc.giga_no), - bld_unstable_uint(hpp, szp, calls.alloc.no)); + bld_unstable_uint(hpp, szp, ma->calls.alloc.giga_no), + bld_unstable_uint(hpp, szp, ma->calls.alloc.no)); } return res; } static Eterm -info_status(MemKind* mk, int *print_to_p, void *print_to_arg, +info_status(ErtsMsegAllctr_t *ma, MemKind* mk, int *print_to_p, void *print_to_arg, int begin_new_max_period, Uint **hpp, Uint *szp) { Eterm res = THE_NON_VALUE; @@ -1258,7 +1280,7 @@ info_status(MemKind* mk, int *print_to_p, void *print_to_arg, return res; } -static Eterm info_memkind(MemKind* mk, int *print_to_p, void *print_to_arg, +static Eterm info_memkind(ErtsMsegAllctr_t *ma, MemKind* mk, int *print_to_p, void *print_to_arg, int begin_max_per, Uint **hpp, Uint *szp) { Eterm res = THE_NON_VALUE; @@ -1274,8 +1296,8 @@ static Eterm info_memkind(MemKind* mk, int *print_to_p, void *print_to_arg, atoms[2] = am.calls; values[0] = erts_bld_string(hpp, szp, mk->name); } - values[1] = info_status(mk, print_to_p, print_to_arg, begin_max_per, hpp, szp); - values[2] = info_calls(print_to_p, print_to_arg, hpp, szp); + values[1] = info_status(ma, mk, print_to_p, print_to_arg, begin_max_per, hpp, szp); + values[2] = info_calls(ma, print_to_p, print_to_arg, hpp, szp); if (hpp || szp) res = bld_2tup_list(hpp, szp, 3, atoms, values); @@ -1285,7 +1307,7 @@ static Eterm info_memkind(MemKind* mk, int *print_to_p, void *print_to_arg, static Eterm -info_version(int *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp) +info_version(ErtsMsegAllctr_t *ma, int *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp) { Eterm res = THE_NON_VALUE; @@ -1306,56 +1328,64 @@ info_version(int *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp) \* */ Eterm -erts_mseg_info_options(int *print_to_p, void *print_to_arg, +erts_mseg_info_options(int ix, + int *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp) { + ErtsMsegAllctr_t *ma = ERTS_MSEG_ALLCTR_IX(ix); Eterm res; - erts_mtx_lock(&mseg_mutex); + ERTS_MSEG_LOCK(ma); - res = info_options("option ", print_to_p, print_to_arg, hpp, szp); + ERTS_DBG_MA_CHK_THR_ACCESS(ma); - erts_mtx_unlock(&mseg_mutex); + res = info_options(ma, "option ", print_to_p, print_to_arg, hpp, szp); + + ERTS_MSEG_UNLOCK(ma); return res; } Eterm -erts_mseg_info(int *print_to_p, +erts_mseg_info(int ix, + int *print_to_p, void *print_to_arg, int begin_max_per, Uint **hpp, Uint *szp) { + ErtsMsegAllctr_t *ma = ERTS_MSEG_ALLCTR_IX(ix); Eterm res = THE_NON_VALUE; Eterm atoms[4]; Eterm values[4]; Uint n = 0; - erts_mtx_lock(&mseg_mutex); + ERTS_MSEG_LOCK(ma); + + ERTS_DBG_MA_CHK_THR_ACCESS(ma); if (hpp || szp) { if (!atoms_initialized) - init_atoms(); + init_atoms(ma); atoms[0] = am.version; atoms[1] = am.options; atoms[2] = am.memkind; atoms[3] = am.memkind; } - values[n++] = info_version(print_to_p, print_to_arg, hpp, szp); - values[n++] = info_options("option ", print_to_p, print_to_arg, hpp, szp); + values[n++] = info_version(ma, print_to_p, print_to_arg, hpp, szp); + values[n++] = info_options(ma, "option ", print_to_p, print_to_arg, hpp, szp); #if HALFWORD_HEAP - values[n++] = info_memkind(&low_mem, print_to_p, print_to_arg, begin_max_per, hpp, szp); - values[n++] = info_memkind(&hi_mem, print_to_p, print_to_arg, begin_max_per, hpp, szp); + values[n++] = info_memkind(ma, &ma->low_mem, print_to_p, print_to_arg, begin_max_per, hpp, szp); + values[n++] = info_memkind(ma, &ma->hi_mem, print_to_p, print_to_arg, begin_max_per, hpp, szp); #else - values[n++] = info_memkind(&the_mem, print_to_p, print_to_arg, begin_max_per, hpp, szp); + values[n++] = info_memkind(ma, &ma->the_mem, print_to_p, print_to_arg, begin_max_per, hpp, szp); #endif if (hpp || szp) res = bld_2tup_list(hpp, szp, n, atoms, values); - erts_mtx_unlock(&mseg_mutex); + ERTS_MSEG_UNLOCK(ma); return res; } @@ -1363,10 +1393,12 @@ erts_mseg_info(int *print_to_p, void * erts_mseg_alloc_opt(ErtsAlcType_t atype, Uint *size_p, const ErtsMsegOpt_t *opt) { + ErtsMsegAllctr_t *ma = ERTS_MSEG_ALLCTR_OPT(opt); void *seg; - erts_mtx_lock(&mseg_mutex); - seg = mseg_alloc(atype, size_p, opt); - erts_mtx_unlock(&mseg_mutex); + ERTS_MSEG_LOCK(ma); + ERTS_DBG_MA_CHK_THR_ACCESS(ma); + seg = mseg_alloc(ma, atype, size_p, opt); + ERTS_MSEG_UNLOCK(ma); return seg; } @@ -1377,12 +1409,14 @@ erts_mseg_alloc(ErtsAlcType_t atype, Uint *size_p) } void -erts_mseg_dealloc_opt(ErtsAlcType_t atype, void *seg, Uint size, - const ErtsMsegOpt_t *opt) +erts_mseg_dealloc_opt(ErtsAlcType_t atype, void *seg, + Uint size, const ErtsMsegOpt_t *opt) { - erts_mtx_lock(&mseg_mutex); - mseg_dealloc(atype, seg, size, opt); - erts_mtx_unlock(&mseg_mutex); + ErtsMsegAllctr_t *ma = ERTS_MSEG_ALLCTR_OPT(opt); + ERTS_MSEG_LOCK(ma); + ERTS_DBG_MA_CHK_THR_ACCESS(ma); + mseg_dealloc(ma, atype, seg, size, opt); + ERTS_MSEG_UNLOCK(ma); } void @@ -1392,44 +1426,60 @@ erts_mseg_dealloc(ErtsAlcType_t atype, void *seg, Uint size) } void * -erts_mseg_realloc_opt(ErtsAlcType_t atype, void *seg, Uint old_size, - Uint *new_size_p, const ErtsMsegOpt_t *opt) +erts_mseg_realloc_opt(ErtsAlcType_t atype, void *seg, + Uint old_size, Uint *new_size_p, + const ErtsMsegOpt_t *opt) { + ErtsMsegAllctr_t *ma = ERTS_MSEG_ALLCTR_OPT(opt); void *new_seg; - erts_mtx_lock(&mseg_mutex); - new_seg = mseg_realloc(atype, seg, old_size, new_size_p, opt); - erts_mtx_unlock(&mseg_mutex); + ERTS_MSEG_LOCK(ma); + ERTS_DBG_MA_CHK_THR_ACCESS(ma); + new_seg = mseg_realloc(ma, atype, seg, old_size, new_size_p, opt); + ERTS_MSEG_UNLOCK(ma); return new_seg; } void * -erts_mseg_realloc(ErtsAlcType_t atype, void *seg, Uint old_size, - Uint *new_size_p) +erts_mseg_realloc(ErtsAlcType_t atype, void *seg, + Uint old_size, Uint *new_size_p) { - return erts_mseg_realloc_opt(atype, seg, old_size, new_size_p, &erts_mseg_default_opt); + return erts_mseg_realloc_opt(atype, seg, old_size, new_size_p, + &erts_mseg_default_opt); } void erts_mseg_clear_cache(void) { + ErtsMsegAllctr_t *ma = ERTS_MSEG_ALLCTR_SS(); MemKind* mk; - erts_mtx_lock(&mseg_mutex); - for (mk=mk_list; mk; mk=mk->next) { + +start: + + ERTS_MSEG_LOCK(ma); + ERTS_DBG_MA_CHK_THR_ACCESS(ma); + for (mk=ma->mk_list; mk; mk=mk->next) { mseg_clear_cache(mk); } - erts_mtx_unlock(&mseg_mutex); + ERTS_MSEG_UNLOCK(ma); + + if (ma->ix != 0) { + ma = ERTS_MSEG_ALLCTR_IX(0); + goto start; + } } Uint -erts_mseg_no(void) +erts_mseg_no(const ErtsMsegOpt_t *opt) { + ErtsMsegAllctr_t *ma = ERTS_MSEG_ALLCTR_OPT(opt); MemKind* mk; Uint n = 0; - erts_mtx_lock(&mseg_mutex); - for (mk=mk_list; mk; mk=mk->next) { + ERTS_MSEG_LOCK(ma); + ERTS_DBG_MA_CHK_THR_ACCESS(ma); + for (mk=ma->mk_list; mk; mk=mk->next) { n += mk->segments.current.no; } - erts_mtx_unlock(&mseg_mutex); + ERTS_MSEG_UNLOCK(ma); return n; } @@ -1439,7 +1489,7 @@ erts_mseg_unit_size(void) return page_size; } -static void mem_kind_init(MemKind* mk, const char* name) +static void mem_kind_init(ErtsMsegAllctr_t *ma, MemKind* mk, const char* name) { unsigned i; @@ -1450,10 +1500,10 @@ static void mem_kind_init(MemKind* mk, const char* name) mk->cache_size = 0; mk->cache_hits = 0; - if (max_cache_size > 0) { - for (i = 0; i < max_cache_size - 1; i++) + if (ma->max_cache_size > 0) { + for (i = 0; i < ma->max_cache_size - 1; i++) mk->cache_descs[i].next = &mk->cache_descs[i + 1]; - mk->cache_descs[max_cache_size - 1].next = NULL; + mk->cache_descs[ma->max_cache_size - 1].next = NULL; mk->free_cache_descs = &mk->cache_descs[0]; } else @@ -1467,30 +1517,38 @@ static void mem_kind_init(MemKind* mk, const char* name) mk->segments.max_ever.no = 0; mk->segments.max_ever.sz = 0; + mk->ma = ma; mk->name = name; - mk->next = mk_list; - mk_list = mk; + mk->next = ma->mk_list; + ma->mk_list = mk; } + + void erts_mseg_init(ErtsMsegInit_t *init) { - atoms_initialized = 0; - is_init_done = 0; + int i; + UWord x; - /* Options ... */ +#ifdef ERTS_SMP + no_mseg_allocators = init->nos + 1; +#else + no_mseg_allocators = 1; +#endif - abs_max_cache_bad_fit = init->amcbf; - rel_max_cache_bad_fit = init->rmcbf; - max_cache_size = init->mcs; - cache_check_interval = init->cci; + x = (UWord) malloc(sizeof(ErtsAlgndMsegAllctr_t) + *no_mseg_allocators + + (ERTS_CACHE_LINE_SIZE-1)); + if (x & ERTS_CACHE_LINE_MASK) + x = (x & ~ERTS_CACHE_LINE_MASK) + ERTS_CACHE_LINE_SIZE; + ASSERT((x & ERTS_CACHE_LINE_MASK) == 0); + aligned_mseg_allctr = (ErtsAlgndMsegAllctr_t *) x; - /* */ + atoms_initialized = 0; -#ifdef USE_THREADS - thread_safe_init(); -#endif + erts_mtx_init(&init_atoms_mutex, "mseg_init_atoms"); #if HAVE_MMAP && !defined(MAP_ANON) mmap_fd = open("/dev/zero", O_RDWR); @@ -1512,34 +1570,55 @@ erts_mseg_init(ErtsMsegInit_t *init) page_shift++; } - sys_memzero((void *) &calls, sizeof(calls)); + for (i = 0; i < no_mseg_allocators; i++) { + ErtsMsegAllctr_t *ma = ERTS_MSEG_ALLCTR_IX(i); -#if CAN_PARTLY_DESTROY - min_seg_size = ~((Uint) 0); -#endif + ma->ix = i; + + ma->is_init_done = 0; + + if (i != 0) + ma->is_thread_safe = 0; + else { + ma->is_thread_safe = 1; + erts_mtx_init(&ma->mtx, "mseg"); + } + + ma->is_cache_check_scheduled = 0; + + /* Options ... */ + + ma->abs_max_cache_bad_fit = init->amcbf; + ma->rel_max_cache_bad_fit = init->rmcbf; + ma->max_cache_size = init->mcs; - if (max_cache_size > MAX_CACHE_SIZE) - max_cache_size = MAX_CACHE_SIZE; + if (ma->max_cache_size > MAX_CACHE_SIZE) + ma->max_cache_size = MAX_CACHE_SIZE; + + ma->mk_list = NULL; #if HALFWORD_HEAP - mem_kind_init(&low_mem, "low memory"); - mem_kind_init(&hi_mem, "high memory"); + mem_kind_init(ma, &ma->low_mem, "low memory"); + mem_kind_init(ma, &ma->hi_mem, "high memory"); #else - mem_kind_init(&the_mem, "all memory"); + mem_kind_init(ma, &ma->the_mem, "all memory"); #endif - is_cache_check_scheduled = 0; -#ifdef ERTS_THREADS_NO_SMP - is_cache_check_requested = 0; + sys_memzero((void *) &ma->calls, sizeof(ErtsMsegCalls)); + +#if CAN_PARTLY_DESTROY + ma->min_seg_size = ~((Uint) 0); #endif + } } -static ERTS_INLINE Uint tot_cache_size(void) +static ERTS_INLINE Uint tot_cache_size(ErtsMsegAllctr_t *ma) { MemKind* mk; Uint sz = 0; - for (mk=mk_list; mk; mk=mk->next) { + ERTS_DBG_MA_CHK_THR_ACCESS(ma); + for (mk=ma->mk_list; mk; mk=mk->next) { sz += mk->cache_size; } return sz; @@ -1552,25 +1631,13 @@ static ERTS_INLINE Uint tot_cache_size(void) void erts_mseg_late_init(void) { -#ifdef ERTS_THREADS_NO_SMP - int handle = - erts_register_async_ready_callback( - check_schedule_cache_check); -#endif - erts_mtx_lock(&mseg_mutex); - is_init_done = 1; -#ifdef ERTS_THREADS_NO_SMP - async_handle = handle; -#endif - if (tot_cache_size()) - schedule_cache_check(); - erts_mtx_unlock(&mseg_mutex); -} - -void -erts_mseg_exit(void) -{ - mseg_shutdown(); + ErtsMsegAllctr_t *ma = ERTS_MSEG_ALLCTR_SS(); + ERTS_MSEG_LOCK(ma); + ERTS_DBG_MA_CHK_THR_ACCESS(ma); + ma->is_init_done = 1; + if (tot_cache_size(ma)) + schedule_cache_check(ma); + ERTS_MSEG_UNLOCK(ma); } #endif /* #if HAVE_ERTS_MSEG */ @@ -1599,12 +1666,13 @@ erts_mseg_test(unsigned long op, erts_mseg_clear_cache(); return (unsigned long) 0; case 0x405: - return (unsigned long) erts_mseg_no(); + return (unsigned long) erts_mseg_no(&erts_mseg_default_opt); case 0x406: { + ErtsMsegAllctr_t *ma = ERTS_MSEG_ALLCTR_IX(0); unsigned long res; - erts_mtx_lock(&mseg_mutex); - res = (unsigned long) tot_cache_size(); - erts_mtx_unlock(&mseg_mutex); + ERTS_MSEG_LOCK(ma); + res = (unsigned long) tot_cache_size(ma); + ERTS_MSEG_UNLOCK(ma); return res; } #else /* #if HAVE_ERTS_MSEG */ diff --git a/erts/emulator/sys/common/erl_mseg.h b/erts/emulator/sys/common/erl_mseg.h index 8f116030a8..741080fb78 100644 --- a/erts/emulator/sys/common/erl_mseg.h +++ b/erts/emulator/sys/common/erl_mseg.h @@ -44,7 +44,7 @@ typedef struct { Uint amcbf; Uint rmcbf; Uint mcs; - Uint cci; + Uint nos; } ErtsMsegInit_t; #define ERTS_MSEG_INIT_DEFAULT_INITIALIZER \ @@ -60,6 +60,7 @@ typedef struct { int preserv; UWord abs_shrink_th; UWord rel_shrink_th; + int sched_spec; #if HALFWORD_HEAP int low_mem; #endif @@ -75,14 +76,14 @@ void *erts_mseg_realloc(ErtsAlcType_t, void *, Uint, Uint *); void *erts_mseg_realloc_opt(ErtsAlcType_t, void *, Uint, Uint *, const ErtsMsegOpt_t *); void erts_mseg_clear_cache(void); -Uint erts_mseg_no(void); +void erts_mseg_cache_check(void); +Uint erts_mseg_no( const ErtsMsegOpt_t *); Uint erts_mseg_unit_size(void); void erts_mseg_init(ErtsMsegInit_t *init); void erts_mseg_late_init(void); /* Have to be called after all allocators, threads and timers have been initialized. */ -void erts_mseg_exit(void); -Eterm erts_mseg_info_options(int *, void*, Uint **, Uint *); -Eterm erts_mseg_info(int *, void*, int, Uint **, Uint *); +Eterm erts_mseg_info_options(int, int *, void*, Uint **, Uint *); +Eterm erts_mseg_info(int, int *, void*, int, Uint **, Uint *); #endif /* #if HAVE_ERTS_MSEG */ diff --git a/erts/emulator/sys/common/erl_poll.c b/erts/emulator/sys/common/erl_poll.c index 7278075f54..80db2055a2 100644 --- a/erts/emulator/sys/common/erl_poll.c +++ b/erts/emulator/sys/common/erl_poll.c @@ -68,6 +68,7 @@ # endif # endif #endif +#include "erl_thr_progress.h" #include "erl_driver.h" #include "erl_alloc.h" @@ -114,7 +115,7 @@ #endif #define ERTS_POLL_USE_WAKEUP_PIPE \ - (ERTS_POLL_ASYNC_INTERRUPT_SUPPORT || defined(ERTS_SMP)) + (ERTS_POLL_ASYNC_INTERRUPT_SUPPORT || defined(USE_THREADS)) #ifdef ERTS_SMP @@ -124,11 +125,11 @@ erts_smp_mtx_unlock(&(PS)->mtx) #define ERTS_POLLSET_SET_POLLED_CHK(PS) \ - ((int) erts_atomic32_xchg(&(PS)->polled, (erts_aint32_t) 1)) + ((int) erts_atomic32_xchg_nob(&(PS)->polled, (erts_aint32_t) 1)) #define ERTS_POLLSET_UNSET_POLLED(PS) \ - erts_atomic32_set(&(PS)->polled, (erts_aint32_t) 0) + erts_atomic32_set_nob(&(PS)->polled, (erts_aint32_t) 0) #define ERTS_POLLSET_IS_POLLED(PS) \ - ((int) erts_atomic32_read(&(PS)->polled)) + ((int) erts_atomic32_read_nob(&(PS)->polled)) #else @@ -142,11 +143,11 @@ #if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE #define ERTS_POLLSET_SET_HAVE_UPDATE_REQUESTS(PS) \ - erts_smp_atomic32_set(&(PS)->have_update_requests, (erts_aint32_t) 1) + erts_smp_atomic32_set_nob(&(PS)->have_update_requests, (erts_aint32_t) 1) #define ERTS_POLLSET_UNSET_HAVE_UPDATE_REQUESTS(PS) \ - erts_smp_atomic32_set(&(PS)->have_update_requests, (erts_aint32_t) 0) + erts_smp_atomic32_set_nob(&(PS)->have_update_requests, (erts_aint32_t) 0) #define ERTS_POLLSET_HAVE_UPDATE_REQUESTS(PS) \ - ((int) erts_smp_atomic32_read(&(PS)->have_update_requests)) + ((int) erts_smp_atomic32_read_nob(&(PS)->have_update_requests)) #else #define ERTS_POLLSET_SET_HAVE_UPDATE_REQUESTS(PS) #define ERTS_POLLSET_UNSET_HAVE_UPDATE_REQUESTS(PS) @@ -261,7 +262,6 @@ struct ErtsPollSet_ { #ifdef ERTS_SMP erts_atomic32_t polled; erts_smp_mtx_t mtx; -#elif ERTS_POLL_ASYNC_INTERRUPT_SUPPORT #endif #if ERTS_POLL_USE_WAKEUP_PIPE int wake_fds[2]; @@ -269,10 +269,8 @@ struct ErtsPollSet_ { #if ERTS_POLL_USE_FALLBACK int fallback_used; #endif -#ifdef ERTS_SMP +#if defined(USE_THREADS) || ERTS_POLL_ASYNC_INTERRUPT_SUPPORT erts_atomic32_t wakeup_state; -#elif ERTS_POLL_ASYNC_INTERRUPT_SUPPORT - volatile int wakeup_state; #endif erts_smp_atomic32_t timeout; #ifdef ERTS_POLL_COUNT_AVOIDED_WAKEUPS @@ -345,21 +343,16 @@ static void print_misc_debug_info(void); static ERTS_INLINE void reset_wakeup_state(ErtsPollSet ps) { -#ifdef ERTS_SMP - erts_atomic32_set(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN); - ERTS_THR_MEMORY_BARRIER; -#elif ERTS_POLL_ASYNC_INTERRUPT_SUPPORT - ps->wakeup_state = 0; +#if defined(USE_THREADS) || ERTS_POLL_ASYNC_INTERRUPT_SUPPORT + erts_atomic32_set_mb(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN); #endif } static ERTS_INLINE int is_woken(ErtsPollSet ps) { -#ifdef ERTS_SMP +#if defined(USE_THREADS) || ERTS_POLL_ASYNC_INTERRUPT_SUPPORT return erts_atomic32_read_acqb(&ps->wakeup_state) != ERTS_POLL_NOT_WOKEN; -#elif ERTS_POLL_ASYNC_INTERRUPT_SUPPORT - return ps->wakeup_state != ERTS_POLL_NOT_WOKEN; #else return 0; #endif @@ -368,13 +361,9 @@ is_woken(ErtsPollSet ps) static ERTS_INLINE int is_interrupted_reset(ErtsPollSet ps) { -#ifdef ERTS_SMP - return (erts_atomic32_xchg(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN) +#if defined(USE_THREADS) || ERTS_POLL_ASYNC_INTERRUPT_SUPPORT + return (erts_atomic32_xchg_nob(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN) == ERTS_POLL_WOKEN_INTR); -#elif ERTS_POLL_ASYNC_INTERRUPT_SUPPORT - int res = ps->wakeup_state == ERTS_POLL_WOKEN_INTR; - ps->wakeup_state = ERTS_POLL_NOT_WOKEN; - return res; #else return 0; #endif @@ -383,16 +372,13 @@ is_interrupted_reset(ErtsPollSet ps) static ERTS_INLINE void woke_up(ErtsPollSet ps) { -#ifdef ERTS_SMP - erts_aint32_t wakeup_state = erts_atomic32_read(&ps->wakeup_state); +#if defined(USE_THREADS) || ERTS_POLL_ASYNC_INTERRUPT_SUPPORT + erts_aint32_t wakeup_state = erts_atomic32_read_nob(&ps->wakeup_state); if (wakeup_state == ERTS_POLL_NOT_WOKEN) - (void) erts_atomic32_cmpxchg(&ps->wakeup_state, - ERTS_POLL_WOKEN, - ERTS_POLL_NOT_WOKEN); - ASSERT(erts_atomic32_read(&ps->wakeup_state) != ERTS_POLL_NOT_WOKEN); -#elif ERTS_POLL_ASYNC_INTERRUPT_SUPPORT - if (ps->wakeup_state == ERTS_POLL_NOT_WOKEN) - ps->wakeup_state = ERTS_POLL_WOKEN; + (void) erts_atomic32_cmpxchg_nob(&ps->wakeup_state, + ERTS_POLL_WOKEN, + ERTS_POLL_NOT_WOKEN); + ASSERT(erts_atomic32_read_nob(&ps->wakeup_state) != ERTS_POLL_NOT_WOKEN); #endif } @@ -403,28 +389,27 @@ woke_up(ErtsPollSet ps) #if ERTS_POLL_USE_WAKEUP_PIPE static ERTS_INLINE void -wake_poller(ErtsPollSet ps, int interrupted) +wake_poller(ErtsPollSet ps, int interrupted, int async_signal_safe) { - int wake = 0; -#ifdef ERTS_SMP - erts_aint32_t wakeup_state; - if (!interrupted) - wakeup_state = erts_atomic32_cmpxchg_relb(&ps->wakeup_state, - ERTS_POLL_WOKEN, - ERTS_POLL_NOT_WOKEN); + int wake; + if (async_signal_safe) + wake = 1; else { - /* - * We might unnecessarily write to the pipe, however, - * that isn't problematic. - */ - wakeup_state = erts_atomic32_read(&ps->wakeup_state); - erts_atomic32_set_relb(&ps->wakeup_state, ERTS_POLL_WOKEN_INTR); + erts_aint32_t wakeup_state; + if (!interrupted) + wakeup_state = erts_atomic32_cmpxchg_relb(&ps->wakeup_state, + ERTS_POLL_WOKEN, + ERTS_POLL_NOT_WOKEN); + else { + /* + * We might unnecessarily write to the pipe, however, + * that isn't problematic. + */ + wakeup_state = erts_atomic32_read_nob(&ps->wakeup_state); + erts_atomic32_set_relb(&ps->wakeup_state, ERTS_POLL_WOKEN_INTR); + } + wake = wakeup_state == ERTS_POLL_NOT_WOKEN; } - wake = wakeup_state == ERTS_POLL_NOT_WOKEN; -#elif ERTS_POLL_ASYNC_INTERRUPT_SUPPORT - wake = ps->wakeup_state == ERTS_POLL_NOT_WOKEN; - ps->wakeup_state = interrupted ? ERTS_POLL_WOKEN_INTR : ERTS_POLL_NOT_WOKEN; -#endif /* * NOTE: This function might be called from signal handlers in the * non-smp case; therefore, it has to be async-signal safe in @@ -439,9 +424,17 @@ wake_poller(ErtsPollSet ps, int interrupted) res = write(ps->wake_fds[1], "!", 1); } while (res < 0 && errno == EINTR); if (res <= 0 && errno != ERRNO_BLOCK) { - fatal_error_async_signal_safe(__FILE__ - ":XXX:wake_poller(): " - "Failed to write on wakeup pipe\n"); + if (async_signal_safe) + fatal_error_async_signal_safe(__FILE__ + ":XXX:wake_poller(): " + "Failed to write on wakeup pipe\n"); + else + fatal_error("%s:%d:wake_poller(): " + "Failed to write to wakeup pipe fd=%d: " + "%s (%d)\n", + __FILE__, __LINE__, + ps->wake_fds[1], + erl_errno_id(errno), errno); } } } @@ -449,11 +442,18 @@ wake_poller(ErtsPollSet ps, int interrupted) static ERTS_INLINE void cleanup_wakeup_pipe(ErtsPollSet ps) { +#if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT + int intr = 0; +#endif int fd = ps->wake_fds[0]; int res; do { char buf[32]; res = read(fd, buf, sizeof(buf)); +#if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT + if (res > 0) + intr = 1; +#endif } while (res > 0 || (res < 0 && errno == EINTR)); if (res < 0 && errno != ERRNO_BLOCK) { fatal_error("%s:%d:cleanup_wakeup_pipe(): " @@ -463,6 +463,10 @@ cleanup_wakeup_pipe(ErtsPollSet ps) fd, erl_errno_id(errno), errno); } +#if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT + if (intr) + erts_atomic32_set_nob(&ps->wakeup_state, ERTS_POLL_WOKEN_INTR); +#endif } static void @@ -839,7 +843,7 @@ write_batch_buf(ErtsPollSet ps, ErtsPollBatchBuf *bbp) ps->fds_status[fd].flags |= ERTS_POLL_FD_FLG_USEFLBCK; ASSERT(ps->fds_status[fd].used_events); ps->fds_status[fd].used_events = 0; - erts_smp_atomic_dec(&ps->no_of_user_fds); + erts_smp_atomic_dec_nob(&ps->no_of_user_fds); update_fallback_pollset(ps, fd); ASSERT(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK); break; @@ -889,11 +893,11 @@ batch_update_pollset(ErtsPollSet ps, int fd, ErtsPollBatchBuf *bbp) events = ERTS_POLL_EV_E2N(ps->fds_status[fd].events); if (!events) { buf[buf_len].events = POLLREMOVE; - erts_smp_atomic_dec(&ps->no_of_user_fds); + erts_smp_atomic_dec_nob(&ps->no_of_user_fds); } else if (!ps->fds_status[fd].used_events) { buf[buf_len].events = events; - erts_smp_atomic_inc(&ps->no_of_user_fds); + erts_smp_atomic_inc_nob(&ps->no_of_user_fds); } else { if ((ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_RST) @@ -983,12 +987,12 @@ batch_update_pollset(ErtsPollSet ps, int fd, ErtsPollBatchBuf *bbp) } if (used_events) { if (!events) { - erts_smp_atomic_dec(&ps->no_of_user_fds); + erts_smp_atomic_dec_nob(&ps->no_of_user_fds); } } else { if (events) - erts_smp_atomic_inc(&ps->no_of_user_fds); + erts_smp_atomic_inc_nob(&ps->no_of_user_fds); } ASSERT((events & ~(ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT)) == 0); ASSERT((used_events & ~(ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT)) == 0); @@ -1062,7 +1066,7 @@ update_pollset(ErtsPollSet ps, int fd) epe.data.fd = epe_templ.data.fd; res = epoll_ctl(ps->kp_fd, EPOLL_CTL_DEL, fd, &epe); } while (res != 0 && errno == EINTR); - erts_smp_atomic_dec(&ps->no_of_user_fds); + erts_smp_atomic_dec_nob(&ps->no_of_user_fds); ps->fds_status[fd].used_events = 0; } @@ -1070,11 +1074,11 @@ update_pollset(ErtsPollSet ps, int fd) /* A note on EPOLL_CTL_DEL: linux kernel versions before 2.6.9 need a non-NULL event pointer even though it is ignored... */ op = EPOLL_CTL_DEL; - erts_smp_atomic_dec(&ps->no_of_user_fds); + erts_smp_atomic_dec_nob(&ps->no_of_user_fds); } else if (!ps->fds_status[fd].used_events) { op = EPOLL_CTL_ADD; - erts_smp_atomic_inc(&ps->no_of_user_fds); + erts_smp_atomic_inc_nob(&ps->no_of_user_fds); } else { op = EPOLL_CTL_MOD; @@ -1124,7 +1128,7 @@ update_pollset(ErtsPollSet ps, int fd) /* Fall through ... */ case EPOLL_CTL_ADD: { ps->fds_status[fd].flags |= ERTS_POLL_FD_FLG_USEFLBCK; - erts_smp_atomic_dec(&ps->no_of_user_fds); + erts_smp_atomic_dec_nob(&ps->no_of_user_fds); #if ERTS_POLL_USE_CONCURRENT_UPDATE if (!*update_fallback) { *update_fallback = 1; @@ -1212,7 +1216,7 @@ static int update_pollset(ErtsPollSet ps, int fd) #if ERTS_POLL_USE_FALLBACK ASSERT(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK); #endif - erts_smp_atomic_dec(&ps->no_of_user_fds); + erts_smp_atomic_dec_nob(&ps->no_of_user_fds); last_pix = --ps->no_poll_fds; if (pix != last_pix) { /* Move last pix to this pix */ @@ -1239,7 +1243,7 @@ static int update_pollset(ErtsPollSet ps, int fd) ASSERT(!(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK) || fd == ps->kp_fd); #endif - erts_smp_atomic_inc(&ps->no_of_user_fds); + erts_smp_atomic_inc_nob(&ps->no_of_user_fds); ps->fds_status[fd].pix = pix = ps->no_poll_fds++; if (pix >= ps->poll_fds_len) grow_poll_fds(ps, pix); @@ -1290,7 +1294,7 @@ static int update_pollset(ErtsPollSet ps, int fd) if (!ps->fds_status[fd].used_events) { ASSERT(events); - erts_smp_atomic_inc(&ps->no_of_user_fds); + erts_smp_atomic_inc_nob(&ps->no_of_user_fds); #if ERTS_POLL_USE_FALLBACK ps->no_select_fds++; ps->fds_status[fd].flags |= ERTS_POLL_FD_FLG_INFLBCK; @@ -1298,7 +1302,7 @@ static int update_pollset(ErtsPollSet ps, int fd) } else if (!events) { ASSERT(ps->fds_status[fd].used_events); - erts_smp_atomic_dec(&ps->no_of_user_fds); + erts_smp_atomic_dec_nob(&ps->no_of_user_fds); ps->fds_status[fd].events = events; #if ERTS_POLL_USE_FALLBACK ps->no_select_fds--; @@ -1497,7 +1501,7 @@ ERTS_POLL_EXPORT(erts_poll_controlv)(ErtsPollSet ps, #ifdef ERTS_SMP if (final_do_wake) - wake_poller(ps, 0); + wake_poller(ps, 0, 0); #endif /* ERTS_SMP */ } @@ -1520,7 +1524,7 @@ ERTS_POLL_EXPORT(erts_poll_control)(ErtsPollSet ps, #ifdef ERTS_SMP if (*do_wake) { - wake_poller(ps, 0); + wake_poller(ps, 0, 0); } #endif /* ERTS_SMP */ @@ -1893,10 +1897,10 @@ save_poll_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res, } static ERTS_INLINE int -check_fd_events(ErtsPollSet ps, SysTimeval *tv, int max_res, int *ps_locked) +check_fd_events(ErtsPollSet ps, SysTimeval *tv, int max_res) { - ASSERT(!*ps_locked); - if (erts_smp_atomic_read(&ps->no_of_user_fds) == 0 + int res; + if (erts_smp_atomic_read_nob(&ps->no_of_user_fds) == 0 && tv->tv_usec == 0 && tv->tv_sec == 0) { /* Nothing to poll and zero timeout; done... */ return 0; @@ -1915,16 +1919,23 @@ check_fd_events(ErtsPollSet ps, SysTimeval *tv, int max_res, int *ps_locked) timeout = INT_MAX; if (max_res > ps->res_events_len) grow_res_events(ps, max_res); - return epoll_wait(ps->kp_fd, ps->res_events, max_res, (int)timeout); +#ifdef ERTS_SMP + if (timeout) + erts_thr_progress_prepare_wait(NULL); +#endif + res = epoll_wait(ps->kp_fd, ps->res_events, max_res, (int)timeout); #elif ERTS_POLL_USE_KQUEUE /* --- kqueue ------------------------------ */ struct timespec ts; - ts.tv_sec = tv->tv_sec; - ts.tv_nsec = tv->tv_usec*1000; if (max_res > ps->res_events_len) grow_res_events(ps, max_res); - return kevent(ps->kp_fd, NULL, 0, ps->res_events, max_res, &ts); +#ifdef ERTS_SMP + if (timeout) + erts_thr_progress_prepare_wait(NULL); +#endif + ts.tv_sec = tv->tv_sec; + ts.tv_nsec = tv->tv_usec*1000; + res = kevent(ps->kp_fd, NULL, 0, ps->res_events, max_res, &ts); #endif /* ----------------------------------------- */ - } else /* use fallback (i.e. poll() or select()) */ #endif /* ERTS_POLL_USE_FALLBACK */ @@ -1937,7 +1948,7 @@ check_fd_events(ErtsPollSet ps, SysTimeval *tv, int max_res, int *ps_locked) * the maximum number of file descriptors in the poll set. */ struct dvpoll poll_res; - int nfds = (int) erts_smp_atomic_read(&ps->no_of_user_fds); + int nfds = (int) erts_smp_atomic_read_nob(&ps->no_of_user_fds); #ifdef ERTS_SMP nfds++; /* Wakeup pipe */ #endif @@ -1947,22 +1958,38 @@ check_fd_events(ErtsPollSet ps, SysTimeval *tv, int max_res, int *ps_locked) if (poll_res.dp_nfds > ps->res_events_len) grow_res_events(ps, poll_res.dp_nfds); poll_res.dp_fds = ps->res_events; +#ifdef ERTS_SMP + if (timeout) + erts_thr_progress_prepare_wait(NULL); +#endif poll_res.dp_timeout = (int) timeout; - return ioctl(ps->kp_fd, DP_POLL, &poll_res); + res = ioctl(ps->kp_fd, DP_POLL, &poll_res); #elif ERTS_POLL_USE_POLL /* --- poll -------------------------------- */ if (timeout > INT_MAX) timeout = INT_MAX; - return poll(ps->poll_fds, ps->no_poll_fds, (int) timeout); +#ifdef ERTS_SMP + if (timeout) + erts_thr_progress_prepare_wait(NULL); +#endif + res = poll(ps->poll_fds, ps->no_poll_fds, (int) timeout); #elif ERTS_POLL_USE_SELECT /* --- select ------------------------------ */ - int res; + SysTimeval to = *tv; + ps->res_input_fds = ps->input_fds; ps->res_output_fds = ps->output_fds; + +#ifdef ERTS_SMP + if (to.tv_sec || to.tv_usec) + erts_thr_progress_prepare_wait(NULL); +#endif res = select(ps->max_fd + 1, &ps->res_input_fds, &ps->res_output_fds, NULL, - tv); + &to); #ifdef ERTS_SMP + if (to.tv_sec || to.tv_usec) + erts_thr_progress_finalize_wait(NULL); if (res < 0 && errno == EBADF && ERTS_POLLSET_HAVE_UPDATE_REQUESTS(ps)) { @@ -1978,15 +2005,16 @@ check_fd_events(ErtsPollSet ps, SysTimeval *tv, int max_res, int *ps_locked) * have triggered, we fake an EAGAIN error and let the caller * restart us. */ - SysTimeval zero_tv = {0, 0}; - *ps_locked = 1; + to.tv_sec = 0; + to.tv_usec = 0; ERTS_POLLSET_LOCK(ps); handle_update_requests(ps); + ERTS_POLLSET_UNLOCK(ps); res = select(ps->max_fd + 1, &ps->res_input_fds, &ps->res_output_fds, NULL, - &zero_tv); + &to); if (res == 0) { errno = EAGAIN; res = -1; @@ -1996,6 +2024,11 @@ check_fd_events(ErtsPollSet ps, SysTimeval *tv, int max_res, int *ps_locked) return res; #endif /* ----------------------------------------- */ } +#ifdef ERTS_SMP + if (timeout) + erts_thr_progress_finalize_wait(NULL); +#endif + return res; } } @@ -2007,7 +2040,9 @@ ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet ps, { int res, no_fds; int ebadf = 0; - int ps_locked; +#ifdef ERTS_SMP + int ps_locked = 0; +#endif SysTimeval *tvp; SysTimeval itv; @@ -2049,8 +2084,7 @@ ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet ps, } #endif - ps_locked = 0; - res = check_fd_events(ps, tvp, no_fds, &ps_locked); + res = check_fd_events(ps, tvp, no_fds); woke_up(ps); @@ -2072,10 +2106,8 @@ ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet ps, #endif #ifdef ERTS_SMP - if (!ps_locked) { - ps_locked = 1; - ERTS_POLLSET_LOCK(ps); - } + ps_locked = 1; + ERTS_POLLSET_LOCK(ps); #endif no_fds = save_poll_result(ps, pr, no_fds, res, ebadf); @@ -2111,19 +2143,26 @@ ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet ps, void ERTS_POLL_EXPORT(erts_poll_interrupt)(ErtsPollSet ps, int set) { -#if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT || defined(ERTS_SMP) - /* - * NOTE: This function might be called from signal handlers in the - * non-smp case; therefore, it has to be async-signal safe in - * the non-smp case. - */ +#if defined(USE_THREADS) if (!set) reset_wakeup_state(ps); else - wake_poller(ps, 1); + wake_poller(ps, 1, 0); #endif } +#if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT +void +ERTS_POLL_EXPORT(erts_poll_async_sig_interrupt)(ErtsPollSet ps) +{ + /* + * NOTE: This function is called from signal handlers, it, + * therefore, it has to be async-signal safe. + */ + wake_poller(ps, 1, 1); +} +#endif + /* * erts_poll_interrupt_timed(): * If 'set' != 0, interrupt thread blocked in erts_poll_wait() if it @@ -2139,14 +2178,14 @@ ERTS_POLL_EXPORT(erts_poll_interrupt_timed)(ErtsPollSet ps, reset_wakeup_state(ps); else { if (erts_smp_atomic32_read_acqb(&ps->timeout) > (erts_aint32_t) msec) - wake_poller(ps, 1); + wake_poller(ps, 1, 0); #ifdef ERTS_POLL_COUNT_AVOIDED_WAKEUPS else { if (ERTS_POLLSET_IS_POLLED(ps)) - erts_smp_atomic_inc(&ps->no_avoided_wakeups); - erts_smp_atomic_inc(&ps->no_avoided_interrupts); + erts_smp_atomic_inc_nob(&ps->no_avoided_wakeups); + erts_smp_atomic_inc_nob(&ps->no_avoided_interrupts); } - erts_smp_atomic_inc(&ps->no_interrupt_timed); + erts_smp_atomic_inc_nob(&ps->no_interrupt_timed); #endif } #endif @@ -2208,7 +2247,7 @@ ERTS_POLL_EXPORT(erts_poll_create_pollset)(void) ps->internal_fd_limit = 0; ps->fds_status = NULL; ps->fds_status_len = 0; - erts_smp_atomic_init(&ps->no_of_user_fds, 0); + erts_smp_atomic_init_nob(&ps->no_of_user_fds, 0); #if ERTS_POLL_USE_KERNEL_POLL ps->kp_fd = -1; #if ERTS_POLL_USE_EPOLL @@ -2260,16 +2299,14 @@ ERTS_POLL_EXPORT(erts_poll_create_pollset)(void) ps->update_requests.next = NULL; ps->update_requests.len = 0; ps->curr_upd_req_block = &ps->update_requests; - erts_smp_atomic32_init(&ps->have_update_requests, 0); + erts_smp_atomic32_init_nob(&ps->have_update_requests, 0); #endif #ifdef ERTS_SMP - erts_atomic32_init(&ps->polled, 0); + erts_atomic32_init_nob(&ps->polled, 0); erts_smp_mtx_init(&ps->mtx, "pollset"); #endif -#ifdef ERTS_SMP - erts_atomic32_init(&ps->wakeup_state, (erts_aint32_t) 0); -#elif ERTS_POLL_ASYNC_INTERRUPT_SUPPORT - ps->wakeup_state = 0; +#if defined(USE_THREADS) || ERTS_POLL_ASYNC_INTERRUPT_SUPPORT + erts_atomic32_init_nob(&ps->wakeup_state, (erts_aint32_t) 0); #endif #if ERTS_POLL_USE_WAKEUP_PIPE create_wakeup_pipe(ps); @@ -2291,11 +2328,11 @@ ERTS_POLL_EXPORT(erts_poll_create_pollset)(void) ps->internal_fd_limit = kp_fd + 1; ps->kp_fd = kp_fd; #endif - erts_smp_atomic32_init(&ps->timeout, ERTS_AINT32_T_MAX); + erts_smp_atomic32_init_nob(&ps->timeout, ERTS_AINT32_T_MAX); #ifdef ERTS_POLL_COUNT_AVOIDED_WAKEUPS - erts_smp_atomic_init(&ps->no_avoided_wakeups, 0); - erts_smp_atomic_init(&ps->no_avoided_interrupts, 0); - erts_smp_atomic_init(&ps->no_interrupt_timed, 0); + erts_smp_atomic_init_nob(&ps->no_avoided_wakeups, 0); + erts_smp_atomic_init_nob(&ps->no_avoided_interrupts, 0); + erts_smp_atomic_init_nob(&ps->no_interrupt_timed, 0); #endif #if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE handle_update_requests(ps); @@ -2303,7 +2340,7 @@ ERTS_POLL_EXPORT(erts_poll_create_pollset)(void) #if ERTS_POLL_USE_FALLBACK ps->fallback_used = 0; #endif - erts_smp_atomic_set(&ps->no_of_user_fds, 0); /* Don't count wakeup pipe and fallback fd */ + erts_smp_atomic_set_nob(&ps->no_of_user_fds, 0); /* Don't count wakeup pipe and fallback fd */ erts_smp_spin_lock(&pollsets_lock); ps->next = pollsets; @@ -2449,7 +2486,7 @@ ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet ps, ErtsPollInfo *pip) pip->memory_size = size; - pip->poll_set_size = (int) erts_smp_atomic_read(&ps->no_of_user_fds); + pip->poll_set_size = (int) erts_smp_atomic_read_nob(&ps->no_of_user_fds); #ifdef ERTS_SMP pip->poll_set_size++; /* Wakeup pipe */ #endif @@ -2507,9 +2544,9 @@ ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet ps, ErtsPollInfo *pip) pip->max_fds = max_fds; #ifdef ERTS_POLL_COUNT_AVOIDED_WAKEUPS - pip->no_avoided_wakeups = erts_smp_atomic_read(&ps->no_avoided_wakeups); - pip->no_avoided_interrupts = erts_smp_atomic_read(&ps->no_avoided_interrupts); - pip->no_interrupt_timed = erts_smp_atomic_read(&ps->no_interrupt_timed); + pip->no_avoided_wakeups = erts_smp_atomic_read_nob(&ps->no_avoided_wakeups); + pip->no_avoided_interrupts = erts_smp_atomic_read_nob(&ps->no_avoided_interrupts); + pip->no_interrupt_timed = erts_smp_atomic_read_nob(&ps->no_interrupt_timed); #endif ERTS_POLLSET_UNLOCK(ps); @@ -2529,7 +2566,7 @@ fatal_error(char *format, ...) { va_list ap; - if (ERTS_IS_CRASH_DUMPING || ERTS_GOT_SIGUSR1) { + if (ERTS_SOMEONE_IS_CRASH_DUMPING || ERTS_GOT_SIGUSR1) { /* * Crash dump writing and reception of sigusr1 (which will * result in a crash dump) closes all file descriptors. This @@ -2549,7 +2586,7 @@ fatal_error(char *format, ...) static void fatal_error_async_signal_safe(char *error_str) { - if (ERTS_IS_CRASH_DUMPING || ERTS_GOT_SIGUSR1) { + if (ERTS_SOMEONE_IS_CRASH_DUMPING || ERTS_GOT_SIGUSR1) { /* See comment above in fatal_error() */ return; } diff --git a/erts/emulator/sys/common/erl_poll.h b/erts/emulator/sys/common/erl_poll.h index 725a77a152..e0296c6a33 100644 --- a/erts/emulator/sys/common/erl_poll.h +++ b/erts/emulator/sys/common/erl_poll.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2006-2009. All Rights Reserved. + * Copyright Ericsson AB 2006-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -216,6 +216,9 @@ typedef struct { #endif } ErtsPollInfo; +#ifdef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT +void ERTS_POLL_EXPORT(erts_poll_async_sig_interrupt)(ErtsPollSet); +#endif void ERTS_POLL_EXPORT(erts_poll_interrupt)(ErtsPollSet, int); void ERTS_POLL_EXPORT(erts_poll_interrupt_timed)(ErtsPollSet, diff --git a/erts/emulator/sys/unix/erl_unix_sys.h b/erts/emulator/sys/unix/erl_unix_sys.h index d8d51b192c..9a5ed9f5bc 100644 --- a/erts/emulator/sys/unix/erl_unix_sys.h +++ b/erts/emulator/sys/unix/erl_unix_sys.h @@ -129,10 +129,12 @@ #define HAVE_ERTS_CHECK_IO_DEBUG int erts_check_io_debug(void); - -#ifndef ENABLE_CHILD_WAITER_THREAD +#ifndef ERTS_SMP # undef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT # define ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT +#endif + +#ifndef ENABLE_CHILD_WAITER_THREAD # ifdef ERTS_SMP # define ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN void erts_check_children(void); diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c index e5ee0df7fa..c6b63350e5 100644 --- a/erts/emulator/sys/unix/sys.c +++ b/erts/emulator/sys/unix/sys.c @@ -52,6 +52,7 @@ #define ERTS_WANT_GOT_SIGUSR1 #define WANT_NONBLOCKING /* must define this to pull in defs from sys.h */ #include "sys.h" +#include "erl_thr_progress.h" #if defined(__APPLE__) && defined(__MACH__) && !defined(__DARWIN__) #define __DARWIN__ 1 @@ -127,7 +128,6 @@ static ErtsSysReportExit *report_exit_list; static ErtsSysReportExit *report_exit_transit_list; #endif -extern int check_async_ready(void); extern int driver_interrupt(int, int); extern void do_break(void); @@ -167,12 +167,12 @@ static int debug_log = 0; #ifdef ERTS_SMP erts_smp_atomic32_t erts_got_sigusr1; #define ERTS_SET_GOT_SIGUSR1 \ - erts_smp_atomic32_set(&erts_got_sigusr1, 1) + erts_smp_atomic32_set_mb(&erts_got_sigusr1, 1) #define ERTS_UNSET_GOT_SIGUSR1 \ - erts_smp_atomic32_set(&erts_got_sigusr1, 0) + erts_smp_atomic32_set_mb(&erts_got_sigusr1, 0) static erts_smp_atomic32_t have_prepared_crash_dump; #define ERTS_PREPARED_CRASH_DUMP \ - ((int) erts_smp_atomic32_xchg(&have_prepared_crash_dump, 1)) + ((int) erts_smp_atomic32_xchg_nob(&have_prepared_crash_dump, 1)) #else volatile int erts_got_sigusr1; #define ERTS_SET_GOT_SIGUSR1 (erts_got_sigusr1 = 1) @@ -242,9 +242,9 @@ static int max_files = -1; #ifdef ERTS_SMP erts_smp_atomic32_t erts_break_requested; #define ERTS_SET_BREAK_REQUESTED \ - erts_smp_atomic32_set(&erts_break_requested, (erts_aint32_t) 1) + erts_smp_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 1) #define ERTS_UNSET_BREAK_REQUESTED \ - erts_smp_atomic32_set(&erts_break_requested, (erts_aint32_t) 0) + erts_smp_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 0) #else volatile int erts_break_requested = 0; #define ERTS_SET_BREAK_REQUESTED (erts_break_requested = 1) @@ -263,6 +263,7 @@ int erts_use_kernel_poll = 0; struct { int (*select)(ErlDrvPort, ErlDrvEvent, int, int); int (*event)(ErlDrvPort, ErlDrvEvent, ErlDrvEventData); + void (*check_io_as_interrupt)(void); void (*check_io_interrupt)(int); void (*check_io_interrupt_tmd)(int, long); void (*check_io)(int); @@ -302,6 +303,9 @@ init_check_io(void) if (erts_use_kernel_poll) { io_func.select = driver_select_kp; io_func.event = driver_event_kp; +#ifdef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT + io_func.check_io_as_interrupt = erts_check_io_async_sig_interrupt_kp; +#endif io_func.check_io_interrupt = erts_check_io_interrupt_kp; io_func.check_io_interrupt_tmd = erts_check_io_interrupt_timed_kp; io_func.check_io = erts_check_io_kp; @@ -314,6 +318,9 @@ init_check_io(void) else { io_func.select = driver_select_nkp; io_func.event = driver_event_nkp; +#ifdef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT + io_func.check_io_as_interrupt = erts_check_io_async_sig_interrupt_nkp; +#endif io_func.check_io_interrupt = erts_check_io_interrupt_nkp; io_func.check_io_interrupt_tmd = erts_check_io_interrupt_timed_nkp; io_func.check_io = erts_check_io_nkp; @@ -325,6 +332,11 @@ init_check_io(void) } } +#ifdef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT +#define ERTS_CHK_IO_AS_INTR() (*io_func.check_io_as_interrupt)() +#else +#define ERTS_CHK_IO_AS_INTR() (*io_func.check_io_interrupt)(1) +#endif #define ERTS_CHK_IO_INTR (*io_func.check_io_interrupt) #define ERTS_CHK_IO_INTR_TMD (*io_func.check_io_interrupt_tmd) #define ERTS_CHK_IO (*io_func.check_io) @@ -339,6 +351,11 @@ init_check_io(void) max_files = erts_check_io_max_files(); } +#ifdef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT +#define ERTS_CHK_IO_AS_INTR() erts_check_io_async_sig_interrupt() +#else +#define ERTS_CHK_IO_AS_INTR() erts_check_io_interrupt(1) +#endif #define ERTS_CHK_IO_INTR erts_check_io_interrupt #define ERTS_CHK_IO_INTR_TMD erts_check_io_interrupt_timed #define ERTS_CHK_IO erts_check_io @@ -346,13 +363,13 @@ init_check_io(void) #endif -#ifdef ERTS_SMP void erts_sys_schedule_interrupt(int set) { ERTS_CHK_IO_INTR(set); } +#ifdef ERTS_SMP void erts_sys_schedule_interrupt_timed(int set, long msec) { @@ -364,7 +381,7 @@ Uint erts_sys_misc_mem_sz(void) { Uint res = ERTS_CHK_IO_SZ(); - res += erts_smp_atomic_read(&sys_misc_mem_sz); + res += erts_smp_atomic_read_mb(&sys_misc_mem_sz); return res; } @@ -509,9 +526,9 @@ erts_sys_pre_init(void) #endif } #ifdef ERTS_SMP - erts_smp_atomic32_init(&erts_break_requested, 0); - erts_smp_atomic32_init(&erts_got_sigusr1, 0); - erts_smp_atomic32_init(&have_prepared_crash_dump, 0); + erts_smp_atomic32_init_nob(&erts_break_requested, 0); + erts_smp_atomic32_init_nob(&erts_got_sigusr1, 0); + erts_smp_atomic32_init_nob(&have_prepared_crash_dump, 0); #else erts_break_requested = 0; erts_got_sigusr1 = 0; @@ -521,7 +538,7 @@ erts_sys_pre_init(void) children_died = 0; #endif #endif /* USE_THREADS */ - erts_smp_atomic_init(&sys_misc_mem_sz, 0); + erts_smp_atomic_init_nob(&sys_misc_mem_sz, 0); } void @@ -552,7 +569,7 @@ erl_sys_init(void) + sizeof(CHILD_SETUP_PROG_NAME) + 1); child_setup_prog = erts_alloc(ERTS_ALC_T_CS_PROG_PATH, csp_path_sz); - erts_smp_atomic_add(&sys_misc_mem_sz, csp_path_sz); + erts_smp_atomic_add_nob(&sys_misc_mem_sz, csp_path_sz); sprintf(child_setup_prog, "%s%c%s", bindir, @@ -731,7 +748,7 @@ break_requested(void) erl_exit(ERTS_INTR_EXIT, ""); ERTS_SET_BREAK_REQUESTED; - ERTS_CHK_IO_INTR(1); /* Make sure we don't sleep in poll */ + ERTS_CHK_IO_AS_INTR(); /* Make sure we don't sleep in poll */ } /* set up signal handlers for break and quit */ @@ -931,18 +948,13 @@ void os_flavor(char* namebuf, /* Where to return the name. */ unsigned size) /* Size of name buffer. */ { - static int called = 0; - static struct utsname uts; /* Information about the system. */ - - if (!called) { - char* s; + struct utsname uts; /* Information about the system. */ + char* s; - (void) uname(&uts); - called = 1; - for (s = uts.sysname; *s; s++) { - if (isupper((int) *s)) { - *s = tolower((int) *s); - } + (void) uname(&uts); + for (s = uts.sysname; *s; s++) { + if (isupper((int) *s)) { + *s = tolower((int) *s); } } strcpy(namebuf, uts.sysname); @@ -1107,31 +1119,6 @@ struct erl_drv_entry vanilla_driver_entry = { stop_select }; -#if defined(USE_THREADS) && !defined(ERTS_SMP) -static int async_drv_init(void); -static ErlDrvData async_drv_start(ErlDrvPort, char*, SysDriverOpts*); -static void async_drv_stop(ErlDrvData); -static void async_drv_input(ErlDrvData, ErlDrvEvent); - -/* INTERNAL use only */ - -struct erl_drv_entry async_driver_entry = { - async_drv_init, - async_drv_start, - async_drv_stop, - NULL, - async_drv_input, - NULL, - "async", - NULL, - NULL, - NULL, - NULL, - NULL, - NULL -}; -#endif - /* Handle SIGCHLD signals. */ #if (defined(SIG_SIGSET) || defined(SIG_SIGNAL)) static RETSIGTYPE onchld(void) @@ -1145,7 +1132,7 @@ static RETSIGTYPE onchld(int signum) smp_sig_notify('C'); #else children_died = 1; - ERTS_CHK_IO_INTR(1); /* Make sure we don't sleep in poll */ + ERTS_CHK_IO_AS_INTR(); /* Make sure we don't sleep in poll */ #endif } @@ -1215,8 +1202,8 @@ static int spawn_init() sys_sigset(SIGPIPE, SIG_IGN); /* Ignore - we'll handle the write failure */ driver_data = (struct driver_data *) erts_alloc(ERTS_ALC_T_DRV_TAB, max_files * sizeof(struct driver_data)); - erts_smp_atomic_add(&sys_misc_mem_sz, - max_files * sizeof(struct driver_data)); + erts_smp_atomic_add_nob(&sys_misc_mem_sz, + max_files * sizeof(struct driver_data)); for (i = 0; i < max_files; i++) driver_data[i].pid = -1; @@ -1924,8 +1911,8 @@ static void clear_fd_data(int fd) { if (fd_data[fd].sz > 0) { erts_free(ERTS_ALC_T_FD_ENTRY_BUF, (void *) fd_data[fd].buf); - ASSERT(erts_smp_atomic_read(&sys_misc_mem_sz) >= fd_data[fd].sz); - erts_smp_atomic_add(&sys_misc_mem_sz, -1*fd_data[fd].sz); + ASSERT(erts_smp_atomic_read_nob(&sys_misc_mem_sz) >= fd_data[fd].sz); + erts_smp_atomic_add_nob(&sys_misc_mem_sz, -1*fd_data[fd].sz); } fd_data[fd].buf = NULL; fd_data[fd].sz = 0; @@ -2260,7 +2247,7 @@ static void ready_input(ErlDrvData e, ErlDrvEvent ready_fd) port_inp_failure(port_num, ready_fd, -1); } else { - erts_smp_atomic_add(&sys_misc_mem_sz, h); + erts_smp_atomic_add_nob(&sys_misc_mem_sz, h); sys_memcpy(buf, cpos, bytes_left); fd_data[ready_fd].buf = buf; fd_data[ready_fd].sz = h; @@ -2316,87 +2303,6 @@ static void stop_select(ErlDrvEvent fd, void* _) close((int)fd); } -/* -** Async opertation support -*/ -#if defined(USE_THREADS) && !defined(ERTS_SMP) -static void -sys_async_ready_failed(int fd, int r, int err) -{ - char buf[120]; - sprintf(buf, "sys_async_ready(): Fatal error: fd=%d, r=%d, errno=%d\n", - fd, r, err); - erts_silence_warn_unused_result(write(2, buf, strlen(buf))); - abort(); -} - -/* called from threads !! */ -void sys_async_ready(int fd) -{ - int r; - while (1) { - r = write(fd, "0", 1); /* signal main thread fd MUST be async_fd[1] */ - if (r == 1) { - DEBUGF(("sys_async_ready(): r = 1\r\n")); - break; - } - if (r < 0 && errno == EINTR) { - DEBUGF(("sys_async_ready(): r = %d\r\n", r)); - continue; - } - sys_async_ready_failed(fd, r, errno); - } -} - -static int async_drv_init(void) -{ - async_fd[0] = -1; - async_fd[1] = -1; - return 0; -} - -static ErlDrvData async_drv_start(ErlDrvPort port_num, - char* name, SysDriverOpts* opts) -{ - if (async_fd[0] != -1) - return ERL_DRV_ERROR_GENERAL; - if (pipe(async_fd) < 0) - return ERL_DRV_ERROR_GENERAL; - - DEBUGF(("async_drv_start: %d\r\n", port_num)); - - SET_NONBLOCKING(async_fd[0]); - driver_select(port_num, async_fd[0], ERL_DRV_READ, 1); - - if (init_async(async_fd[1]) < 0) - return ERL_DRV_ERROR_GENERAL; - return (ErlDrvData)port_num; -} - -static void async_drv_stop(ErlDrvData e) -{ - int port_num = (int)(long)e; - - DEBUGF(("async_drv_stop: %d\r\n", port_num)); - - exit_async(); - - driver_select(port_num, async_fd[0], ERL_DRV_READ, 0); - - close(async_fd[0]); - close(async_fd[1]); - async_fd[0] = async_fd[1] = -1; -} - - -static void async_drv_input(ErlDrvData e, ErlDrvEvent fd) -{ - char *buf[32]; - DEBUGF(("async_drv_input\r\n")); - while (read((int) fd, (void *) buf, 32) > 0); /* fd MUST be async_fd[0] */ - check_async_ready(); /* invoke all async_ready */ -} -#endif void erts_do_break_handling(void) { @@ -2408,11 +2314,7 @@ void erts_do_break_handling(void) * therefore, make sure that all threads but this one are blocked before * proceeding! */ - erts_smp_block_system(0); - /* - * NOTE: since we allow gc we are not allowed to lock - * (any) process main locks while blocking system... - */ + erts_smp_thr_progress_block(); /* during break we revert to initial settings */ /* this is done differently for oldshell */ @@ -2440,7 +2342,7 @@ void erts_do_break_handling(void) tcsetattr(0,TCSANOW,&temp_mode); } - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); } /* Fills in the systems representation of the jam/beam process identifier. @@ -2464,7 +2366,7 @@ erts_sys_putenv(char *buffer, int sep_ix) #else Uint sz = strlen(buffer)+1; env = erts_alloc(ERTS_ALC_T_PUTENV_STR, sz); - erts_smp_atomic_add(&sys_misc_mem_sz, sz); + erts_smp_atomic_add_nob(&sys_misc_mem_sz, sz); strcpy(env,buffer); #endif erts_smp_rwmtx_rwlock(&environ_rwmtx); @@ -2474,12 +2376,10 @@ erts_sys_putenv(char *buffer, int sep_ix) } int -erts_sys_getenv(char *key, char *value, size_t *size) +erts_sys_getenv__(char *key, char *value, size_t *size) { - char *orig_value; int res; - erts_smp_rwmtx_rlock(&environ_rwmtx); - orig_value = getenv(key); + char *orig_value = getenv(key); if (!orig_value) res = -1; else { @@ -2494,6 +2394,15 @@ erts_sys_getenv(char *key, char *value, size_t *size) res = 0; } } + return res; +} + +int +erts_sys_getenv(char *key, char *value, size_t *size) +{ + int res; + erts_smp_rwmtx_rlock(&environ_rwmtx); + res = erts_sys_getenv__(key, value, size); erts_smp_rwmtx_runlock(&environ_rwmtx); return res; } @@ -2503,33 +2412,8 @@ sys_init_io(void) { fd_data = (struct fd_data *) erts_alloc(ERTS_ALC_T_FD_TAB, max_files * sizeof(struct fd_data)); - erts_smp_atomic_add(&sys_misc_mem_sz, - max_files * sizeof(struct fd_data)); - -#ifdef USE_THREADS -#ifdef ERTS_SMP - if (init_async(-1) < 0) - erl_exit(1, "Failed to initialize async-threads\n"); -#else - { - /* This is speical stuff, starting a driver from the - * system routines, but is a nice way of handling stuff - * the erlang way - */ - SysDriverOpts dopts; - int ret; - - sys_memset((void*)&dopts, 0, sizeof(SysDriverOpts)); - add_driver_entry(&async_driver_entry); - ret = erts_open_driver(NULL, NIL, "async", &dopts, NULL); - DEBUGF(("open_driver = %d\n", ret)); - if (ret < 0) - erl_exit(1, "Failed to open async driver\n"); - erts_port[ret].status |= ERTS_PORT_SFLG_IMMORTAL; - } -#endif -#endif - + erts_smp_atomic_add_nob(&sys_misc_mem_sz, + max_files * sizeof(struct fd_data)); } #if (0) /* unused? */ @@ -2756,15 +2640,7 @@ initiate_report_exit_status(ErtsSysReportExit *rep, int status) rep->next = report_exit_transit_list; rep->status = status; report_exit_transit_list = rep; - /* - * We need the scheduler thread to call check_children(). - * If the scheduler thread is sleeping in a poll with a - * timeout, we need to wake the scheduler thread. We use the - * functionality of the async driver to do this, instead of - * implementing yet another driver doing the same thing. A - * little bit ugly, but it works... - */ - sys_async_ready(async_fd[1]); + erts_sys_schedule_interrupt(1); } static int check_children(void) @@ -2851,20 +2727,11 @@ erl_sys_schedule(int runnable) { #ifdef ERTS_SMP ERTS_CHK_IO(!runnable); - ERTS_SMP_LC_ASSERT(!ERTS_LC_IS_BLOCKING); #else - ERTS_CHK_IO_INTR(0); - if (runnable) { - ERTS_CHK_IO(0); /* Poll for I/O */ - check_async_ready(); /* Check async completions */ - } else { - int wait_for_io = !check_async_ready(); - if (wait_for_io) - wait_for_io = !check_children(); - ERTS_CHK_IO(wait_for_io); - } - (void) check_children(); + ERTS_CHK_IO(runnable ? 0 : !check_children()); #endif + ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking()); + (void) check_children(); } @@ -2892,8 +2759,8 @@ smp_sig_notify(char c) static void * signal_dispatcher_thread_func(void *unused) { - int initialized = 0; #if !CHLDWTHR + int initialized = 0; int notify_check_children = 0; #endif #ifdef ERTS_ENABLE_LOCK_CHECK @@ -2921,20 +2788,20 @@ signal_dispatcher_thread_func(void *unused) * to other threads. * * NOTE 2: The signal dispatcher thread is not a blockable - * thread (i.e., it hasn't called - * erts_register_blockable_thread()). This is - * intentional. We want to be able to interrupt - * writing of a crash dump by hitting C-c twice. - * Since it isn't a blockable thread it is important - * that it doesn't change the state of any data that - * a blocking thread expects to have exclusive access - * to (unless the signal dispatcher itself explicitly - * is blocking all blockable threads). + * thread (i.e., not a thread managed by the + * erl_thr_progress module). This is intentional. + * We want to be able to interrupt writing of a crash + * dump by hitting C-c twice. Since it isn't a + * blockable thread it is important that it doesn't + * change the state of any data that a blocking thread + * expects to have exclusive access to (unless the + * signal dispatcher itself explicitly is blocking all + * blockable threads). */ switch (buf[i]) { case 0: /* Emulator initialized */ - initialized = 1; #if !CHLDWTHR + initialized = 1; if (!notify_check_children) #endif break; @@ -2969,7 +2836,7 @@ signal_dispatcher_thread_func(void *unused) buf[i]); } } - ERTS_SMP_LC_ASSERT(!ERTS_LC_IS_BLOCKING); + ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking()); } return NULL; } diff --git a/erts/emulator/sys/vxworks/sys.c b/erts/emulator/sys/vxworks/sys.c index c6e7b65f32..d6d1fe64e0 100644 --- a/erts/emulator/sys/vxworks/sys.c +++ b/erts/emulator/sys/vxworks/sys.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1997-2010. All Rights Reserved. + * Copyright Ericsson AB 1997-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -238,6 +238,12 @@ erl_sys_args(int* argc, char** argv) ASSERT(max_files <= erts_vxworks_max_files); } +void +erts_sys_schedule_interrupt(int set) +{ + erts_check_io_interrupt(set); +} + /* * Called from schedule() when it runs out of runnable processes, * or when Erlang code has performed INPUT_REDUCTIONS reduction @@ -246,7 +252,6 @@ erl_sys_args(int* argc, char** argv) void erl_sys_schedule(int runnable) { - erts_check_io_interrupt(0); erts_check_io(!runnable); } @@ -309,7 +314,7 @@ static void request_break(void) fprintf(stderr,"break!\n"); #endif erts_break_requested = 1; - erts_check_io_interrupt(1); /* Make sure we don't sleep in erts_poll_wait */ + erts_check_io_async_sig_interrupt(1); /* Make sure we don't sleep in erts_poll_wait */ } static void do_quit(void) @@ -1515,6 +1520,12 @@ erts_sys_getenv(char *key, char *value, size_t *size) return res; } +int +erts_sys_getenv__(char *key, char *value, size_t *size) +{ + return erts_sys_getenv(key, value, size); +} + void sys_init_io(void) { @@ -2025,9 +2036,6 @@ int erl_memory_show(int p0, int p1, int p2, int p3, int p4, int p5, erts_printf("The memory block used by elib is save_malloc'ed " "at 0x%08x.\n", (unsigned int) alloc_pool_ptr); } -#ifdef NO_FIX_ALLOC - erts_printf("Fix_alloc is disabled in this build\n"); -#endif erts_printf("Statistics from elib_malloc:\n"); ELIB_LOCK; diff --git a/erts/emulator/sys/win32/erl_poll.c b/erts/emulator/sys/win32/erl_poll.c index 074e2e247f..ab4ef05118 100644 --- a/erts/emulator/sys/win32/erl_poll.c +++ b/erts/emulator/sys/win32/erl_poll.c @@ -309,9 +309,9 @@ struct ErtsPollSet_ { #ifdef ERTS_SMP extern erts_smp_atomic32_t erts_break_requested; #define ERTS_SET_BREAK_REQUESTED \ - erts_smp_atomic32_set(&erts_break_requested, (erts_aint32_t) 1) + erts_smp_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 1) #define ERTS_UNSET_BREAK_REQUESTED \ - erts_smp_atomic32_set(&erts_break_requested, (erts_aint32_t) 0) + erts_smp_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 0) #else extern volatile int erts_break_requested; #define ERTS_SET_BREAK_REQUESTED (erts_break_requested = 1) @@ -371,19 +371,19 @@ do { \ static ERTS_INLINE int is_io_ready(ErtsPollSet ps) { - return erts_atomic32_read(&ps->wakeup_state) == ERTS_POLL_WOKEN_IO_READY; + return erts_atomic32_read_nob(&ps->wakeup_state) == ERTS_POLL_WOKEN_IO_READY; } static ERTS_INLINE void woke_up(ErtsPollSet ps) { - if (erts_atomic32_read(&ps->wakeup_state) == ERTS_POLL_NOT_WOKEN) - erts_atomic32_cmpxchg(&ps->wakeup_state, - ERTS_POLL_WOKEN_TIMEDOUT, - ERTS_POLL_NOT_WOKEN); + if (erts_atomic32_read_nob(&ps->wakeup_state) == ERTS_POLL_NOT_WOKEN) + erts_atomic32_cmpxchg_nob(&ps->wakeup_state, + ERTS_POLL_WOKEN_TIMEDOUT, + ERTS_POLL_NOT_WOKEN); #ifdef DEBUG { - erts_aint32_t wakeup_state = erts_atomic32_read(&ps->wakeup_state); + erts_aint32_t wakeup_state = erts_atomic32_read_nob(&ps->wakeup_state); switch (wakeup_state) { case ERTS_POLL_WOKEN_IO_READY: case ERTS_POLL_WOKEN_INTR: @@ -401,7 +401,7 @@ static ERTS_INLINE int wakeup_cause(ErtsPollSet ps) { int res; - erts_aint32_t wakeup_state = erts_atomic32_read(&ps->wakeup_state); + erts_aint32_t wakeup_state = erts_atomic32_read_nob(&ps->wakeup_state); switch (wakeup_state) { case ERTS_POLL_WOKEN_IO_READY: res = 0; @@ -439,7 +439,7 @@ poll_wait_timeout(ErtsPollSet ps, SysTimeval *tvp) * by ResetEvent(). */ ERTS_THR_MEMORY_BARRIER; - if (erts_atomic32_read(&ps->wakeup_state) != ERTS_POLL_NOT_WOKEN) + if (erts_atomic32_read_nob(&ps->wakeup_state) != ERTS_POLL_NOT_WOKEN) return (DWORD) 0; if (timeout > ERTS_AINT32_T_MAX) /* Also prevents DWORD overflow */ @@ -455,17 +455,17 @@ wake_poller(ErtsPollSet ps, int io_ready) erts_aint32_t wakeup_state; if (io_ready) { /* We may set the event multiple times. This is, however, harmless. */ - wakeup_state = erts_atomic32_read(&ps->wakeup_state); + wakeup_state = erts_atomic32_read_nob(&ps->wakeup_state); erts_atomic32_set_relb(&ps->wakeup_state, ERTS_POLL_WOKEN_IO_READY); } else { ERTS_THR_MEMORY_BARRIER; - wakeup_state = erts_atomic32_read(&ps->wakeup_state); + wakeup_state = erts_atomic32_read_nob(&ps->wakeup_state); while (wakeup_state != ERTS_POLL_WOKEN_IO_READY && wakeup_state != ERTS_POLL_WOKEN_INTR) { - erts_aint32_t act = erts_atomic32_cmpxchg(&ps->wakeup_state, - ERTS_POLL_WOKEN_INTR, - wakeup_state); + erts_aint32_t act = erts_atomic32_cmpxchg_nob(&ps->wakeup_state, + ERTS_POLL_WOKEN_INTR, + wakeup_state); if (act == wakeup_state) { wakeup_state = act; break; @@ -488,13 +488,13 @@ wake_poller(ErtsPollSet ps, int io_ready) static ERTS_INLINE void reset_io_ready(ErtsPollSet ps) { - erts_atomic32_set(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN); + erts_atomic32_set_nob(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN); } static ERTS_INLINE void restore_io_ready(ErtsPollSet ps) { - erts_atomic32_set(&ps->wakeup_state, ERTS_POLL_WOKEN_IO_READY); + erts_atomic32_set_nob(&ps->wakeup_state, ERTS_POLL_WOKEN_IO_READY); } /* @@ -511,12 +511,12 @@ static ERTS_INLINE void reset_interrupt(ErtsPollSet ps) { /* We need to keep io-ready if set */ - erts_aint32_t wakeup_state = erts_atomic32_read(&ps->wakeup_state); + erts_aint32_t wakeup_state = erts_atomic32_read_nob(&ps->wakeup_state); while (wakeup_state != ERTS_POLL_WOKEN_IO_READY && wakeup_state != ERTS_POLL_NOT_WOKEN) { - erts_aint32_t act = erts_atomic32_cmpxchg(&ps->wakeup_state, - ERTS_POLL_NOT_WOKEN, - wakeup_state); + erts_aint32_t act = erts_atomic32_cmpxchg_nob(&ps->wakeup_state, + ERTS_POLL_NOT_WOKEN, + wakeup_state); if (wakeup_state == act) break; wakeup_state = act; @@ -692,7 +692,7 @@ static void *break_waiter(void *param) case WAIT_OBJECT_0: ResetEvent(harr[0]); erts_mtx_lock(&break_waiter_lock); - erts_atomic32_set(&break_waiter_state,BREAK_WAITER_GOT_BREAK); + erts_atomic32_set_nob(&break_waiter_state,BREAK_WAITER_GOT_BREAK); ERTS_THR_MEMORY_BARRIER; SetEvent(break_happened_event); erts_mtx_unlock(&break_waiter_lock); @@ -700,7 +700,7 @@ static void *break_waiter(void *param) case (WAIT_OBJECT_0+1): ResetEvent(harr[1]); erts_mtx_lock(&break_waiter_lock); - erts_atomic32_set(&break_waiter_state,BREAK_WAITER_GOT_HALT); + erts_atomic32_set_nob(&break_waiter_state,BREAK_WAITER_GOT_HALT); ERTS_THR_MEMORY_BARRIER; SetEvent(break_happened_event); erts_mtx_unlock(&break_waiter_lock); @@ -1153,23 +1153,29 @@ int erts_poll_wait(ErtsPollSet ps, /*HARDDEBUGF(("timeout = %ld",(long) timeout));*/ - if (timeout > 0 && !erts_atomic32_read(&break_waiter_state)) { + if (timeout > 0 && !erts_atomic32_read_nob(&break_waiter_state)) { HANDLE harr[2] = {ps->event_io_ready, break_happened_event}; int num_h = 2; HARDDEBUGF(("Start waiting %d [%d]",num_h, (int) timeout)); ERTS_POLLSET_UNLOCK(ps); +#ifdef ERTS_SMP + erts_thr_progress_prepare_wait(NULL); +#endif WaitForMultipleObjects(num_h, harr, FALSE, timeout); +#ifdef ERTS_SMP + erts_thr_progress_finalize_wait(NULL); +#endif ERTS_POLLSET_LOCK(ps); HARDDEBUGF(("Stop waiting %d [%d]",num_h, (int) timeout)); woke_up(ps); } ERTS_UNSET_BREAK_REQUESTED; - if(erts_atomic32_read(&break_waiter_state)) { + if(erts_atomic32_read_nob(&break_waiter_state)) { erts_mtx_lock(&break_waiter_lock); - break_state = erts_atomic32_read(&break_waiter_state); - erts_atomic32_set(&break_waiter_state,0); + break_state = erts_atomic32_read_nob(&break_waiter_state); + erts_atomic32_set_nob(&break_waiter_state,0); ResetEvent(break_happened_event); erts_mtx_unlock(&break_waiter_lock); switch (break_state) { @@ -1236,7 +1242,7 @@ int erts_poll_wait(ErtsPollSet ps, erts_mtx_unlock(&w->mtx); } done: - erts_smp_atomic32_set(&ps->timeout, ERTS_AINT32_T_MAX); + erts_smp_atomic32_set_nob(&ps->timeout, ERTS_AINT32_T_MAX); *len = num; ERTS_POLLSET_UNLOCK(ps); HARDTRACEF(("Out erts_poll_wait")); @@ -1316,11 +1322,11 @@ ErtsPollSet erts_poll_create_pollset(void) ps->standby_wait_event = CreateManualEvent(FALSE); ps->restore_events = 0; - erts_atomic32_init(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN); + erts_atomic32_init_nob(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN); #ifdef ERTS_SMP erts_smp_mtx_init(&ps->mtx, "pollset"); #endif - erts_smp_atomic32_init(&ps->timeout, ERTS_AINT32_T_MAX); + erts_smp_atomic32_init_nob(&ps->timeout, ERTS_AINT32_T_MAX); HARDTRACEF(("Out erts_poll_create_pollset")); return ps; @@ -1372,7 +1378,7 @@ void erts_poll_init(void) erts_mtx_init(&break_waiter_lock,"break_waiter_lock"); break_happened_event = CreateManualEvent(FALSE); - erts_atomic32_init(&break_waiter_state, 0); + erts_atomic32_init_nob(&break_waiter_state, 0); erts_thr_create(&thread, &break_waiter, NULL, NULL); ERTS_UNSET_BREAK_REQUESTED; diff --git a/erts/emulator/sys/win32/sys.c b/erts/emulator/sys/win32/sys.c index c8075c1916..6f33ef7ad6 100644 --- a/erts/emulator/sys/win32/sys.c +++ b/erts/emulator/sys/win32/sys.c @@ -198,7 +198,7 @@ Uint erts_sys_misc_mem_sz(void) { Uint res = (Uint) erts_check_io_size(); - res += (Uint) erts_smp_atomic_read(&sys_misc_mem_sz); + res += (Uint) erts_smp_atomic_read_mb(&sys_misc_mem_sz); return res; } @@ -216,6 +216,9 @@ void sys_tty_reset(int exit_code) void erl_sys_args(int* argc, char** argv) { char *event_name; + + erts_sys_env_init(); + nohup = get_and_remove_option(argc, argv, "-nohup"); #ifdef DEBUG @@ -566,51 +569,6 @@ struct erl_drv_entry vanilla_driver_entry = { stop_select }; -#if defined(USE_THREADS) && !defined(ERTS_SMP) - -static int async_drv_init(void); -static ErlDrvData async_drv_start(ErlDrvPort, char*, SysDriverOpts*); -static void async_drv_stop(ErlDrvData); -static void async_drv_input(ErlDrvData, ErlDrvEvent); - -/* INTERNAL use only */ - -void null_output(ErlDrvData drv_data, char* buf, int len) -{ -} - -void null_ready_output(ErlDrvData drv_data, ErlDrvEvent event) -{ -} - -struct erl_drv_entry async_driver_entry = { - async_drv_init, - async_drv_start, - async_drv_stop, - null_output, - async_drv_input, - null_ready_output, - "async", - NULL, /* finish */ - NULL, /* handle */ - NULL, /* control */ - NULL, /* timeout */ - NULL, /* outputv */ - NULL, /* ready_async */ - NULL, /* flush */ - NULL, /* call */ - NULL, /* event */ - ERL_DRV_EXTENDED_MARKER, - ERL_DRV_EXTENDED_MAJOR_VERSION, - ERL_DRV_EXTENDED_MINOR_VERSION, - 0, /* ERL_DRV_FLAGs */ - NULL, - NULL, /* process_exit */ - stop_select -}; - -#endif - /* * Initialises a DriverData structure. * @@ -648,7 +606,7 @@ new_driver_data(int port_num, int packet_bytes, int wait_objs_required, int use_ erts_smp_mtx_unlock(&sys_driver_data_lock); return NULL; } - erts_smp_atomic_add(&sys_misc_mem_sz, dp->inBufSize); + erts_smp_atomic_add_nob(&sys_misc_mem_sz, dp->inBufSize); dp->outBufSize = 0; dp->outbuf = NULL; dp->port_num = port_num; @@ -733,8 +691,8 @@ release_driver_data(DriverData* dp) #endif if (dp->inbuf != NULL) { - ASSERT(erts_smp_atomic_read(&sys_misc_mem_sz) >= dp->inBufSize); - erts_smp_atomic_add(&sys_misc_mem_sz, -1*dp->inBufSize); + ASSERT(erts_smp_atomic_read_nob(&sys_misc_mem_sz) >= dp->inBufSize); + erts_smp_atomic_add_nob(&sys_misc_mem_sz, -1*dp->inBufSize); DRV_BUF_FREE(dp->inbuf); dp->inBufSize = 0; dp->inbuf = NULL; @@ -742,8 +700,8 @@ release_driver_data(DriverData* dp) ASSERT(dp->inBufSize == 0); if (dp->outbuf != NULL) { - ASSERT(erts_smp_atomic_read(&sys_misc_mem_sz) >= dp->outBufSize); - erts_smp_atomic_add(&sys_misc_mem_sz, -1*dp->outBufSize); + ASSERT(erts_smp_atomic_read_nob(&sys_misc_mem_sz) >= dp->outBufSize); + erts_smp_atomic_add_nob(&sys_misc_mem_sz, -1*dp->outBufSize); DRV_BUF_FREE(dp->outbuf); dp->outBufSize = 0; dp->outbuf = NULL; @@ -1162,7 +1120,8 @@ spawn_init(void) #endif driver_data = (struct driver_data *) erts_alloc(ERTS_ALC_T_DRV_TAB, max_files * sizeof(struct driver_data)); - erts_smp_atomic_add(&sys_misc_mem_sz, max_files*sizeof(struct driver_data)); + erts_smp_atomic_add_nob(&sys_misc_mem_sz, + max_files*sizeof(struct driver_data)); for (i = 0; i < max_files; i++) driver_data[i].port_num = PORT_FREE; @@ -1698,7 +1657,7 @@ create_child_process static int create_pipe(HANDLE *phRead, HANDLE *phWrite, BOOL inheritRead, BOOL overlapped_io) { SECURITY_ATTRIBUTES sa = {sizeof(SECURITY_ATTRIBUTES), NULL, TRUE}; - char pipe_name[128]; /* Name of pipe. */ + char pipe_name[256]; /* Name of pipe. */ Uint calls; /* @@ -1735,9 +1694,9 @@ static int create_pipe(HANDLE *phRead, HANDLE *phWrite, BOOL inheritRead, BOOL o * Otherwise, create named pipes. */ - calls = (Uint) erts_smp_atomic_inctest(&pipe_creation_counter); - sprintf(pipe_name, "\\\\.\\pipe\\erlang44_%d_%d", - getpid(), calls); + calls = (UWord) erts_smp_atomic_inc_read_nob(&pipe_creation_counter); + erts_snprintf(pipe_name, sizeof(pipe_name), + "\\\\.\\pipe\\erlang44_%d_%bpu", getpid(), calls); DEBUGF(("Creating pipe %s\n", pipe_name)); sa.bInheritHandle = inheritRead; @@ -2529,7 +2488,7 @@ output(ErlDrvData drv_data, char* buf, int len) } dp->outBufSize = pb+len; - erts_smp_atomic_add(&sys_misc_mem_sz, dp->outBufSize); + erts_smp_atomic_add_nob(&sys_misc_mem_sz, dp->outBufSize); /* * Store header bytes (if any). @@ -2558,8 +2517,8 @@ output(ErlDrvData drv_data, char* buf, int len) } else { dp->out.ov.Offset += pb+len; /* For vanilla driver. */ /* XXX OffsetHigh should be changed too. */ - ASSERT(erts_smp_atomic_read(&sys_misc_mem_sz) >= dp->outBufSize); - erts_smp_atomic_add(&sys_misc_mem_sz, -1*dp->outBufSize); + ASSERT(erts_smp_atomic_read_nob(&sys_misc_mem_sz) >= dp->outBufSize); + erts_smp_atomic_add_nob(&sys_misc_mem_sz, -1*dp->outBufSize); DRV_BUF_FREE(dp->outbuf); dp->outBufSize = 0; dp->outbuf = NULL; @@ -2673,9 +2632,9 @@ ready_input(ErlDrvData drv_data, ErlDrvEvent ready_event) error = ERROR_NOT_ENOUGH_MEMORY; break; /* Break out of loop into error handler. */ } - ASSERT(erts_smp_atomic_read(&sys_misc_mem_sz) >= dp->inBufSize); - erts_smp_atomic_add(&sys_misc_mem_sz, - dp->totalNeeded - dp->inBufSize); + ASSERT(erts_smp_atomic_read_nob(&sys_misc_mem_sz) >= dp->inBufSize); + erts_smp_atomic_add_nob(&sys_misc_mem_sz, + dp->totalNeeded - dp->inBufSize); dp->inBufSize = dp->totalNeeded; dp->inbuf = new_buf; } @@ -2775,8 +2734,8 @@ ready_output(ErlDrvData drv_data, ErlDrvEvent ready_event) write... */ return; } - ASSERT(erts_smp_atomic_read(&sys_misc_mem_sz) >= dp->outBufSize); - erts_smp_atomic_add(&sys_misc_mem_sz, -1*dp->outBufSize); + ASSERT(erts_smp_atomic_read_nob(&sys_misc_mem_sz) >= dp->outBufSize); + erts_smp_atomic_add_nob(&sys_misc_mem_sz, -1*dp->outBufSize); DRV_BUF_FREE(dp->outbuf); dp->outBufSize = 0; dp->outbuf = NULL; @@ -2824,30 +2783,6 @@ sys_init_io(void) We estimate the number to twice the amount of ports. We really dont know on windows, do we? */ max_files = 2*erts_max_ports; - -#ifdef USE_THREADS -#ifdef ERTS_SMP - if (init_async(-1) < 0) - erl_exit(1, "Failed to initialize async-threads\n"); -#else - { - /* This is special stuff, starting a driver from the - * system routines, but is a nice way of handling stuff - * the erlang way - */ - SysDriverOpts dopts; - int ret; - - sys_memset((void*)&dopts, 0, sizeof(SysDriverOpts)); - add_driver_entry(&async_driver_entry); - ret = erts_open_driver(NULL, NIL, "async", &dopts, NULL); - DEBUGF(("open_driver = %d\n", ret)); - if (ret < 0) - erl_exit(1, "Failed to open async driver\n"); - erts_port[ret].status |= ERTS_PORT_SFLG_IMMORTAL; - } -#endif -#endif } #ifdef ERTS_SMP @@ -2926,8 +2861,8 @@ Preload* sys_preloaded(void) (num_preloaded+1)*sizeof(Preload)); res_name = erts_alloc(ERTS_ALC_T_PRELOADED, (num_preloaded+1)*sizeof(unsigned)); - erts_smp_atomic_add(&sys_misc_mem_sz, - (num_preloaded+1)*sizeof(Preload) + erts_smp_atomic_add_nob(&sys_misc_mem_sz, + (num_preloaded+1)*sizeof(Preload) + (num_preloaded+1)*sizeof(unsigned)); for (i = 0; i < num_preloaded; i++) { int n; @@ -2939,7 +2874,7 @@ Preload* sys_preloaded(void) n = GETWORD(data); data += 2; preloaded[i].name = erts_alloc(ERTS_ALC_T_PRELOADED, n+1); - erts_smp_atomic_add(&sys_misc_mem_sz, n+1); + erts_smp_atomic_add_nob(&sys_misc_mem_sz, n+1); sys_memcpy(preloaded[i].name, data, n); preloaded[i].name[n] = '\0'; data += n; @@ -3281,8 +3216,7 @@ erts_sys_pre_init(void) #endif } #endif - erts_smp_atomic_init(&sys_misc_mem_sz, 0); - erts_sys_env_init(); + erts_smp_atomic_init_nob(&sys_misc_mem_sz, 0); } void noinherit_std_handle(DWORD type) @@ -3309,7 +3243,7 @@ void erl_sys_init(void) erts_smp_tsd_key_create(&win32_errstr_key); InitializeCriticalSection(&htbc_lock); #endif - erts_smp_atomic_init(&pipe_creation_counter,0); + erts_smp_atomic_init_nob(&pipe_creation_counter,0); /* * Test if we have named pipes or not. */ @@ -3359,13 +3293,13 @@ void erl_sys_init(void) SEM_FAILCRITICALERRORS | SEM_NOOPENFILEERRORBOX); } -#ifdef ERTS_SMP void erts_sys_schedule_interrupt(int set) { erts_check_io_interrupt(set); } +#ifdef ERTS_SMP void erts_sys_schedule_interrupt_timed(int set, long msec) { @@ -3381,76 +3315,7 @@ erts_sys_schedule_interrupt_timed(int set, long msec) void erl_sys_schedule(int runnable) { -#ifdef ERTS_SMP erts_check_io(!runnable); - ERTS_SMP_LC_ASSERT(!ERTS_LC_IS_BLOCKING); -#else - erts_check_io_interrupt(0); - if (runnable) { - erts_check_io(0); /* Poll for I/O */ - check_async_ready(); /* Check async completions */ - } else { - erts_check_io(check_async_ready() ? 0 : 1); - } -#endif -} - -#if defined(USE_THREADS) && !defined(ERTS_SMP) -/* - * Async operation support. - */ - -static ErlDrvEvent async_drv_event; - -void -sys_async_ready(int fd) -{ - SetEvent((HANDLE)async_drv_event); -} - -static int -async_drv_init(void) -{ - async_drv_event = (ErlDrvEvent) NULL; - return 0; -} - -static ErlDrvData -async_drv_start(ErlDrvPort port_num, char* name, SysDriverOpts* opts) -{ - if (async_drv_event != (ErlDrvEvent) NULL) { - return ERL_DRV_ERROR_GENERAL; - } - if ((async_drv_event = (ErlDrvEvent)CreateAutoEvent(FALSE)) == (ErlDrvEvent) NULL) { - return ERL_DRV_ERROR_GENERAL; - } - - driver_select(port_num, async_drv_event, ERL_DRV_READ|ERL_DRV_USE, 1); - if (init_async(async_drv_event) < 0) { - return ERL_DRV_ERROR_GENERAL; - } - return (ErlDrvData)port_num; -} - -static void -async_drv_stop(ErlDrvData port_num) -{ - exit_async(); - driver_select((ErlDrvPort)port_num, async_drv_event, ERL_DRV_READ|ERL_DRV_USE, 0); - /*CloseHandle((HANDLE)async_drv_event);*/ - async_drv_event = (ErlDrvEvent) NULL; + ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking()); } - -static void -async_drv_input(ErlDrvData port_num, ErlDrvEvent e) -{ - check_async_ready(); - - /* - * Our event is auto-resetting. - */ -} - -#endif - diff --git a/erts/emulator/sys/win32/sys_env.c b/erts/emulator/sys/win32/sys_env.c index 02c8433a10..7acc7f07ee 100644 --- a/erts/emulator/sys/win32/sys_env.c +++ b/erts/emulator/sys/win32/sys_env.c @@ -55,19 +55,17 @@ erts_sys_putenv(char *key_value, int sep_ix) } int -erts_sys_getenv(char *key, char *value, size_t *size) +erts_sys_getenv__(char *key, char *value, size_t *size) { size_t req_size = 0; int res = 0; DWORD new_size; - erts_smp_rwmtx_rlock(&environ_rwmtx); SetLastError(0); new_size = GetEnvironmentVariable((LPCTSTR) key, (LPTSTR) value, (DWORD) *size); res = !new_size && GetLastError() == ERROR_ENVVAR_NOT_FOUND ? -1 : 0; - erts_smp_rwmtx_runlock(&environ_rwmtx); if (res < 0) return res; res = new_size > *size ? 1 : 0; @@ -75,6 +73,16 @@ erts_sys_getenv(char *key, char *value, size_t *size) return res; } +int +erts_sys_getenv(char *key, char *value, size_t *size) +{ + int res; + erts_smp_rwmtx_rlock(&environ_rwmtx); + res = erts_sys_getenv__(key, value, size); + erts_smp_rwmtx_runlock(&environ_rwmtx); + return res; +} + struct win32_getenv_state { char *env; char *next; diff --git a/erts/emulator/sys/win32/sys_interrupt.c b/erts/emulator/sys/win32/sys_interrupt.c index 943c338794..93aaa23f97 100644 --- a/erts/emulator/sys/win32/sys_interrupt.c +++ b/erts/emulator/sys/win32/sys_interrupt.c @@ -21,6 +21,7 @@ */ #include "sys.h" #include "erl_alloc.h" +#include "erl_thr_progress.h" #include "erl_driver.h" #include "../../drivers/win32/win_con.h" @@ -33,9 +34,9 @@ #ifdef ERTS_SMP erts_smp_atomic32_t erts_break_requested; #define ERTS_SET_BREAK_REQUESTED \ - erts_smp_atomic32_set(&erts_break_requested, (erts_aint32_t) 1) + erts_smp_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 1) #define ERTS_UNSET_BREAK_REQUESTED \ - erts_smp_atomic32_set(&erts_break_requested, (erts_aint32_t) 0) + erts_smp_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 0) #else volatile int erts_break_requested = 0; #define ERTS_SET_BREAK_REQUESTED (erts_break_requested = 1) @@ -52,14 +53,14 @@ void erts_do_break_handling(void) * therefore, make sure that all threads but this one are blocked before * proceeding! */ - erts_smp_block_system(0); + erts_smp_thr_progress_block(); /* call the break handling function, reset the flag */ do_break(); ResetEvent(erts_sys_break_event); ERTS_UNSET_BREAK_REQUESTED; - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); } diff --git a/erts/emulator/test/binary_SUITE.erl b/erts/emulator/test/binary_SUITE.erl index fed5854112..d9fc876482 100644 --- a/erts/emulator/test/binary_SUITE.erl +++ b/erts/emulator/test/binary_SUITE.erl @@ -33,7 +33,6 @@ %% erlang:external_size/1 %% size(Binary) %% iolist_size/1 -%% concat_binary/1 %% split_binary/2 %% hash(Binary, N) %% phash(Binary, N) @@ -47,7 +46,7 @@ init_per_testcase/2, end_per_testcase/2, copy_terms/1, conversions/1, deep_lists/1, deep_bitstr_lists/1, bad_list_to_binary/1, bad_binary_to_list/1, - t_split_binary/1, bad_split/1, t_concat_binary/1, + t_split_binary/1, bad_split/1, terms/1, terms_float/1, external_size/1, t_iolist_size/1, t_hash/1, bad_size/1, @@ -68,7 +67,7 @@ suite() -> [{ct_hooks,[ts_install_cth]}, all() -> [copy_terms, conversions, deep_lists, deep_bitstr_lists, - t_split_binary, bad_split, t_concat_binary, + t_split_binary, bad_split, bad_list_to_binary, bad_binary_to_list, terms, terms_float, external_size, t_iolist_size, bad_binary_to_term_2, safe_binary_to_term2, @@ -93,7 +92,6 @@ init_per_group(_GroupName, Config) -> end_per_group(_GroupName, Config) -> Config. - init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) -> Config. @@ -381,41 +379,6 @@ bad_split(Config) when is_list(Config) -> bad_split(Bin, Pos) -> {'EXIT',{badarg,_}} = (catch split_binary(Bin, Pos)). -%% Tests concat_binary/2 and size/1. - -t_concat_binary(suite) -> []; -t_concat_binary(Config) when is_list(Config) -> - test_concat([]), - - test_concat([[]]), - test_concat([[], []]), - test_concat([[], [], []]), - - test_concat([[1], []]), - test_concat([[], [2]]), - test_concat([[], [3], []]), - - test_concat([[1, 2, 3], [4, 5, 6, 7]]), - test_concat([[1, 2, 3], [4, 5, 6, 7], [9, 10]]), - - test_concat([lists:seq(0, 255), lists:duplicate(1024, $@), - lists:duplicate(2048, $a), - lists:duplicate(4000, $b)]), - ok. - -test_concat(Lists) -> - test_concat(Lists, 0, [], []). - -test_concat([List|Rest], Size, Combined, Binaries) -> - ?line Bin = list_to_binary(List), - ?line test_concat(Rest, Size+length(List), Combined++List, [Bin|Binaries]); -test_concat([], Size, Combined, Binaries0) -> - ?line Binaries = lists:reverse(Binaries0), - ?line Bin = concat_binary(Binaries), - ?line Size = size(Bin), - ?line Size = iolist_size(Bin), - ?line Combined = binary_to_list(Bin). - t_hash(doc) -> "Test hash/2 with different type of binaries."; t_hash(Config) when is_list(Config) -> test_hash([]), diff --git a/erts/emulator/test/bs_match_misc_SUITE.erl b/erts/emulator/test/bs_match_misc_SUITE.erl index b022f96740..15427661f3 100644 --- a/erts/emulator/test/bs_match_misc_SUITE.erl +++ b/erts/emulator/test/bs_match_misc_SUITE.erl @@ -23,7 +23,7 @@ bound_var/1,bound_tail/1,t_float/1,little_float/1,sean/1, kenneth/1,encode_binary/1,native/1,happi/1, size_var/1,wiger/1,x0_context/1,huge_float_field/1, - writable_binary_matched/1,otp_7198/1]). + writable_binary_matched/1,otp_7198/1,unordered_bindings/1]). -include_lib("test_server/include/test_server.hrl"). @@ -33,7 +33,7 @@ all() -> [bound_var, bound_tail, t_float, little_float, sean, kenneth, encode_binary, native, happi, size_var, wiger, x0_context, huge_float_field, writable_binary_matched, - otp_7198]. + otp_7198, unordered_bindings]. groups() -> []. @@ -553,5 +553,15 @@ otp_7198_scan(<<C, Rest/binary>>, TokAcc) when otp_7198_scan(Rest, [{'KEYWORD', C} | TokAcc]) end. +unordered_bindings(Config) when is_list(Config) -> + {<<1,2,3,4>>,<<42,42>>,<<3,3,3>>} = + unordered_bindings(4, 2, 3, <<1,2,3,4, 42,42, 3,3,3, 3>>), + ok. + +unordered_bindings(CompressedLength, HashSize, PadLength, T) -> + <<Content:CompressedLength/binary,Mac:HashSize/binary, + Padding:PadLength/binary,PadLength>> = T, + {Content,Mac,Padding}. + id(I) -> I. diff --git a/erts/emulator/test/bs_utf_SUITE.erl b/erts/emulator/test/bs_utf_SUITE.erl index 72c656c400..4ab7d674a6 100644 --- a/erts/emulator/test/bs_utf_SUITE.erl +++ b/erts/emulator/test/bs_utf_SUITE.erl @@ -64,8 +64,7 @@ end_per_group(_GroupName, Config) -> utf8_roundtrip(Config) when is_list(Config) -> ?line utf8_roundtrip(0, 16#D7FF), - ?line utf8_roundtrip(16#E000, 16#FFFD), - ?line utf8_roundtrip(16#10000, 16#10FFFF), + ?line utf8_roundtrip(16#E000, 16#10FFFF), ok. utf8_roundtrip(First, Last) when First =< Last -> @@ -91,8 +90,7 @@ utf16_roundtrip(Config) when is_list(Config) -> do_utf16_roundtrip(Fun) -> do_utf16_roundtrip(0, 16#D7FF, Fun), - do_utf16_roundtrip(16#E000, 16#FFFD, Fun), - do_utf16_roundtrip(16#10000, 16#10FFFF, Fun). + do_utf16_roundtrip(16#E000, 16#10FFFF, Fun). do_utf16_roundtrip(First, Last, Fun) when First =< Last -> Fun(First), @@ -129,8 +127,7 @@ utf32_roundtrip(Config) when is_list(Config) -> do_utf32_roundtrip(Fun) -> do_utf32_roundtrip(0, 16#D7FF, Fun), - do_utf32_roundtrip(16#E000, 16#FFFD, Fun), - do_utf32_roundtrip(16#10000, 16#10FFFF, Fun). + do_utf32_roundtrip(16#E000, 16#10FFFF, Fun). do_utf32_roundtrip(First, Last, Fun) when First =< Last -> Fun(First), @@ -158,7 +155,6 @@ utf32_little_roundtrip(Char) -> utf8_illegal_sequences(Config) when is_list(Config) -> ?line fail_range(16#10FFFF+1, 16#10FFFF+512), %Too large. ?line fail_range(16#D800, 16#DFFF), %Reserved for UTF-16. - ?line fail_range(16#FFFE, 16#FFFF), %Non-characters. %% Illegal first character. ?line [fail(<<I,16#8F,16#8F,16#8F>>) || I <- lists:seq(16#80, 16#BF)], @@ -251,7 +247,6 @@ fail_1(_) -> ok. utf16_illegal_sequences(Config) when is_list(Config) -> ?line utf16_fail_range(16#10FFFF+1, 16#10FFFF+512), %Too large. ?line utf16_fail_range(16#D800, 16#DFFF), %Reserved for UTF-16. - ?line utf16_fail_range(16#FFFE, 16#FFFF), %Non-characters. ?line lonely_hi_surrogate(16#D800, 16#DFFF), ?line leading_lo_surrogate(16#DC00, 16#DFFF), @@ -300,7 +295,6 @@ leading_lo_surrogate(_, _, _) -> ok. utf32_illegal_sequences(Config) when is_list(Config) -> ?line utf32_fail_range(16#10FFFF+1, 16#10FFFF+512), %Too large. ?line utf32_fail_range(16#D800, 16#DFFF), %Reserved for UTF-16. - ?line utf32_fail_range(16#FFFE, 16#FFFF), %Non-characters. ?line utf32_fail_range(-100, -1), ok. diff --git a/erts/emulator/test/call_trace_SUITE.erl b/erts/emulator/test/call_trace_SUITE.erl index 93fdc157f7..3e2bee06d1 100644 --- a/erts/emulator/test/call_trace_SUITE.erl +++ b/erts/emulator/test/call_trace_SUITE.erl @@ -934,6 +934,10 @@ exception_nocatch(Config) when is_list(Config) -> exception_nocatch(). exception_nocatch() -> + Deep4LocThrow = get_deep_4_loc({throw,[42]}), + Deep4LocError = get_deep_4_loc({error,[42]}), + Deep4LocBadmatch = get_deep_4_loc({'=',[a,b]}), + Prog = [{'_',[],[{exception_trace}]}], ?line 1 = erlang:trace_pattern({?MODULE,deep_1,'_'}, Prog), ?line 1 = erlang:trace_pattern({?MODULE,deep_2,'_'}, Prog), @@ -959,8 +963,9 @@ exception_nocatch() -> {trace,t2,exception_from,{erlang,throw,1}, {error,{nocatch,Q2}}}], exception_from, {error,{nocatch,Q2}}), - ?line expect({trace,T2,exit,{{nocatch,Q2},[{erlang,throw,[Q2]}, - {?MODULE,deep_4,1}]}}), + ?line expect({trace,T2,exit,{{nocatch,Q2},[{erlang,throw,[Q2],[]}, + {?MODULE,deep_4,1, + Deep4LocThrow}]}}), ?line Q3 = {dump,[dump,{dump}]}, ?line T3 = exception_nocatch(?LINE, error, [Q3], 4, @@ -968,18 +973,29 @@ exception_nocatch() -> {trace,t3,exception_from,{erlang,error,1}, {error,Q3}}], exception_from, {error,Q3}), - ?line expect({trace,T3,exit,{Q3,[{erlang,error,[Q3]}, - {?MODULE,deep_4,1}]}}), + ?line expect({trace,T3,exit,{Q3,[{erlang,error,[Q3],[]}, + {?MODULE,deep_4,1,Deep4LocError}]}}), ?line T4 = exception_nocatch(?LINE, '=', [17,4711], 5, [], exception_from, {error,{badmatch,4711}}), - ?line expect({trace,T4,exit,{{badmatch,4711},[{?MODULE,deep_4,1}]}}), + ?line expect({trace,T4,exit,{{badmatch,4711}, + [{?MODULE,deep_4,1,Deep4LocBadmatch}]}}), %% ?line erlang:trace_pattern({?MODULE,'_','_'}, false), ?line erlang:trace_pattern({erlang,'_','_'}, false), ?line expect(), ?line ok. +get_deep_4_loc(Arg) -> + try + deep_4(Arg), + ?t:fail(should_not_return_to_here) + catch + _:_ -> + [{?MODULE,deep_4,1,Loc0}|_] = erlang:get_stacktrace(), + Loc0 + end. + exception_nocatch(Line, B, Q, N, Extra, Tag, R) -> ?line io:format("== Subtest: ~w", [Line]), ?line Go = make_ref(), diff --git a/erts/emulator/test/code_SUITE.erl b/erts/emulator/test/code_SUITE.erl index 29cbdedd17..61eeec5ffd 100644 --- a/erts/emulator/test/code_SUITE.erl +++ b/erts/emulator/test/code_SUITE.erl @@ -25,7 +25,7 @@ t_check_process_code_ets/1, external_fun/1,get_chunk/1,module_md5/1,make_stub/1, make_stub_many_funs/1,constant_pools/1, - false_dependency/1,coverage/1]). + false_dependency/1,coverage/1,fun_confusion/1]). -include_lib("test_server/include/test_server.hrl"). @@ -35,7 +35,7 @@ all() -> [new_binary_types, t_check_process_code, t_check_process_code_ets, t_check_old_code, external_fun, get_chunk, module_md5, make_stub, make_stub_many_funs, - constant_pools, false_dependency, coverage]. + constant_pools, false_dependency, coverage, fun_confusion]. groups() -> []. @@ -278,7 +278,8 @@ t_check_old_code(Config) when is_list(Config) -> external_fun(Config) when is_list(Config) -> ?line false = erlang:function_exported(another_code_test, x, 1), - ?line ExtFun = erlang:make_fun(id(another_code_test), x, 1), + AnotherCodeTest = id(another_code_test), + ExtFun = fun AnotherCodeTest:x/1, ?line {'EXIT',{undef,_}} = (catch ExtFun(answer)), ?line false = erlang:function_exported(another_code_test, x, 1), ?line false = lists:member(another_code_test, erlang:loaded()), @@ -555,6 +556,30 @@ coverage(Config) when is_list(Config) -> ?line {'EXIT',{badarg,_}} = (catch erlang:module_loaded(42)), ok. +fun_confusion(Config) when is_list(Config) -> + Data = ?config(data_dir, Config), + Src = filename:join(Data, "fun_confusion"), + Mod = fun_confusion, + + %% Load first version of module. + compile_load(Mod, Src, 1), + F1 = Mod:f(), + 1 = F1(), + + %% Load second version of module. + compile_load(Mod, Src, 2), + F2 = Mod:f(), + + %% F1 should refer to the old code, not the newly loaded code. + 1 = F1(), + 2 = F2(), + ok. + +compile_load(Mod, Src, Ver) -> + {ok,Mod,Code1} = compile:file(Src, [binary,{d,version,Ver}]), + {module,Mod} = code:load_binary(Mod, "fun_confusion.beam", Code1), + ok. + %% Utilities. make_sub_binary(Bin) when is_binary(Bin) -> diff --git a/erts/emulator/test/code_SUITE_data/fun_confusion.erl b/erts/emulator/test/code_SUITE_data/fun_confusion.erl new file mode 100644 index 0000000000..16000861df --- /dev/null +++ b/erts/emulator/test/code_SUITE_data/fun_confusion.erl @@ -0,0 +1,31 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2011. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +-module(fun_confusion). + +-export([f/0]). + +f() -> + fun() -> version() end. + +version() -> + %% Changing the value returned here should change + %% the identity of the fun in f/0. + ?version. + diff --git a/erts/emulator/test/driver_SUITE.erl b/erts/emulator/test/driver_SUITE.erl index a77ea4f3be..c07dbc5871 100644 --- a/erts/emulator/test/driver_SUITE.erl +++ b/erts/emulator/test/driver_SUITE.erl @@ -75,7 +75,9 @@ smp_select/1, driver_select_use/1, thread_mseg_alloc_cache_clean/1, - otp_9302/1]). + otp_9302/1, + thr_free_drv/1, + async_blast/1]). -export([bin_prefix/2]). @@ -143,7 +145,9 @@ all() -> otp_6879, caller, many_events, missing_callbacks, smp_select, driver_select_use, thread_mseg_alloc_cache_clean, - otp_9302]. + otp_9302, + thr_free_drv, + async_blast]. groups() -> [{timer, [], @@ -1792,7 +1796,7 @@ driver_select_use0(Config) -> thread_mseg_alloc_cache_clean(Config) when is_list(Config) -> case {erlang:system_info(threads), - erlang:system_info({allocator,mseg_alloc}), + mseg_inst_info(0), driver_alloc_sbct()} of {_, false, _} -> ?line {skipped, "No mseg_alloc"}; @@ -1804,13 +1808,13 @@ thread_mseg_alloc_cache_clean(Config) when is_list(Config) -> ?line {skipped, "driver_alloc() using too large single block threshold"}; {_, _, 0} -> ?line {skipped, "driver_alloc() using too low single block threshold"}; - {true, MsegAllocInfo, SBCT} -> + {true, _MsegAllocInfo, SBCT} -> ?line DrvName = 'thr_alloc_drv', ?line Path = ?config(data_dir, Config), ?line erl_ddll:start(), ?line ok = load_driver(Path, DrvName), ?line Port = open_port({spawn, DrvName}, []), - ?line CCI = mseg_alloc_cci(MsegAllocInfo), + ?line CCI = 1000, ?line ?t:format("CCI = ~p~n", [CCI]), ?line CCC = mseg_alloc_ccc(), ?line ?t:format("CCC = ~p~n", [CCC]), @@ -1831,7 +1835,7 @@ mseg_alloc_cci(MsegAllocInfo) -> ?line CCI. mseg_alloc_ccc() -> - mseg_alloc_ccc(erlang:system_info({allocator,mseg_alloc})). + mseg_alloc_ccc(mseg_inst_info(0)). mseg_alloc_ccc(MsegAllocInfo) -> ?line {value,{memkind, MKL}} = lists:keysearch(memkind,1,MsegAllocInfo), @@ -1841,7 +1845,7 @@ mseg_alloc_ccc(MsegAllocInfo) -> ?line GigaCCC*1000000000 + CCC. mseg_alloc_cached_segments() -> - mseg_alloc_cached_segments(erlang:system_info({allocator,mseg_alloc})). + mseg_alloc_cached_segments(mseg_inst_info(0)). mseg_alloc_cached_segments(MsegAllocInfo) -> MemName = case is_halfword_vm() of @@ -1859,6 +1863,13 @@ mseg_alloc_cached_segments(MsegAllocInfo) -> = lists:keysearch(cached_segments, 1, SL), ?line CS. +mseg_inst_info(I) -> + {value, {instance, I, Value}} + = lists:keysearch(I, + 2, + erlang:system_info({allocator,mseg_alloc})), + Value. + is_halfword_vm() -> case {erlang:system_info({wordsize, internal}), erlang:system_info({wordsize, external})} of @@ -1902,18 +1913,105 @@ otp_9302(Config) when is_list(Config) -> ?line port_command(Port, ""), ?line {msg, block} = get_port_msg(Port, infinity), ?line {msg, job} = get_port_msg(Port, infinity), - ?line case erlang:system_info(thread_pool_size) of - 0 -> - {msg, cancel} = get_port_msg(Port, infinity); - _ -> - ok - end, - ?line {msg, job} = get_port_msg(Port, infinity), + ?line C = case erlang:system_info(thread_pool_size) of + 0 -> + ?line {msg, cancel} = get_port_msg(Port, infinity), + ?line {msg, job} = get_port_msg(Port, infinity), + ?line false; + _ -> + case get_port_msg(Port, infinity) of + {msg, cancel} -> %% Cancel always fail in Rel >= 15 + ?line {msg, job} = get_port_msg(Port, infinity), + ?line false; + {msg, job} -> + ?line ok, + ?line true + end + end, ?line {msg, end_of_jobs} = get_port_msg(Port, infinity), ?line no_msg = get_port_msg(Port, 2000), ?line port_close(Port), + ?line case C of + true -> + ?line {comment, "Async job cancelled"}; + false -> + ?line {comment, "Async job not cancelled"} + end. + +thr_free_drv(Config) when is_list(Config) -> + ?line Path = ?config(data_dir, Config), + ?line erl_ddll:start(), + ?line ok = load_driver(Path, thr_free_drv), + ?line MemBefore = driver_alloc_size(), +% io:format("SID=~p", [erlang:system_info(scheduler_id)]), + ?line Port = open_port({spawn, thr_free_drv}, []), + ?line MemPeek = driver_alloc_size(), + ?line true = is_port(Port), + ?line ok = thr_free_drv_control(Port, 0), + ?line port_close(Port), + ?line MemAfter = driver_alloc_size(), + ?line io:format("MemPeek=~p~n", [MemPeek]), + ?line io:format("MemBefore=~p, MemAfter=~p~n", [MemBefore, MemAfter]), + ?line MemBefore = MemAfter, + ?line case MemPeek of + undefined -> ok; + _ -> + ?line true = MemPeek > MemBefore + end, ?line ok. +thr_free_drv_control(Port, N) -> + case erlang:port_control(Port, 0, "") of + "done" -> + ok; + "more" -> + erlang:yield(), +% io:format("N=~p, SID=~p", [N, erlang:system_info(scheduler_id)]), + thr_free_drv_control(Port, N+1) + end. + +async_blast(Config) when is_list(Config) -> + ?line Path = ?config(data_dir, Config), + ?line erl_ddll:start(), + ?line ok = load_driver(Path, async_blast_drv), + ?line SchedOnln = erlang:system_info(schedulers_online), + ?line MemBefore = driver_alloc_size(), + ?line Start = os:timestamp(), + ?line Blast = fun () -> + Port = open_port({spawn, async_blast_drv}, []), + true = is_port(Port), + port_command(Port, ""), + receive + {Port, done} -> + ok + end, + port_close(Port) + end, + ?line Ps = lists:map(fun (N) -> + spawn_opt(Blast, + [{scheduler, + (N rem SchedOnln)+ 1}, + monitor]) + end, + lists:seq(1, 100)), + ?line MemMid = driver_alloc_size(), + ?line lists:foreach(fun ({Pid, Mon}) -> + receive + {'DOWN',Mon,process,Pid,_} -> ok + end + end, Ps), + ?line End = os:timestamp(), + ?line MemAfter = driver_alloc_size(), + ?line io:format("MemBefore=~p, MemMid=~p, MemAfter=~p~n", + [MemBefore, MemMid, MemAfter]), + ?line AsyncBlastTime = timer:now_diff(End,Start)/1000000, + ?line io:format("AsyncBlastTime=~p~n", [AsyncBlastTime]), + ?line MemBefore = MemAfter, + ?line erlang:display({async_blast_time, AsyncBlastTime}), + ?line ok. + + + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Utilities %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -2077,3 +2175,33 @@ start_node(Config) when is_list(Config) -> stop_node(Node) -> ?t:stop_node(Node). + +wait_deallocations() -> + try + erts_debug:set_internal_state(wait, deallocations) + catch error:undef -> + erts_debug:set_internal_state(available_internal_state, true), + wait_deallocations() + end. + +driver_alloc_size() -> + wait_deallocations(), + case erlang:system_info({allocator_sizes, driver_alloc}) of + false -> + undefined; + MemInfo -> + CS = lists:foldl( + fun ({instance, _, L}, Acc) -> + {value,{_,SBMBCS}} = lists:keysearch(sbmbcs, 1, L), + {value,{_,MBCS}} = lists:keysearch(mbcs, 1, L), + {value,{_,SBCS}} = lists:keysearch(sbcs, 1, L), + [SBMBCS,MBCS,SBCS | Acc] + end, + [], + MemInfo), + lists:foldl( + fun(L, Sz0) -> + {value,{_,Sz,_,_}} = lists:keysearch(blocks_size, 1, L), + Sz0+Sz + end, 0, CS) + end. diff --git a/erts/emulator/test/driver_SUITE_data/Makefile.src b/erts/emulator/test/driver_SUITE_data/Makefile.src index 5b3ba1557e..dd48f6a0f7 100644 --- a/erts/emulator/test/driver_SUITE_data/Makefile.src +++ b/erts/emulator/test/driver_SUITE_data/Makefile.src @@ -12,7 +12,9 @@ MISC_DRVS = outputv_drv@dll@ \ many_events_drv@dll@ \ missing_callback_drv@dll@ \ thr_alloc_drv@dll@ \ - otp_9302_drv@dll@ + otp_9302_drv@dll@ \ + thr_free_drv@dll@ \ + async_blast_drv@dll@ SYS_INFO_DRVS = sys_info_1_0_drv@dll@ \ sys_info_1_1_drv@dll@ \ diff --git a/erts/emulator/test/driver_SUITE_data/async_blast_drv.c b/erts/emulator/test/driver_SUITE_data/async_blast_drv.c new file mode 100644 index 0000000000..3821f7e3dc --- /dev/null +++ b/erts/emulator/test/driver_SUITE_data/async_blast_drv.c @@ -0,0 +1,124 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2011. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +#include "erl_driver.h" + +#define NO_ASYNC_JOBS 10000 + +static void stop(ErlDrvData drv_data); +static ErlDrvData start(ErlDrvPort port, + char *command); +static void output(ErlDrvData drv_data, + char *buf, int len); +static void ready_async(ErlDrvData drv_data, + ErlDrvThreadData thread_data); + +static ErlDrvEntry async_blast_drv_entry = { + NULL /* init */, + start, + stop, + output, + NULL /* ready_input */, + NULL /* ready_output */, + "async_blast_drv", + NULL /* finish */, + NULL /* handle */, + NULL /* control */, + NULL /* timeout */, + NULL /* outputv */, + ready_async, + NULL /* flush */, + NULL /* call */, + NULL /* event */, + ERL_DRV_EXTENDED_MARKER, + ERL_DRV_EXTENDED_MAJOR_VERSION, + ERL_DRV_EXTENDED_MINOR_VERSION, + ERL_DRV_FLAG_USE_PORT_LOCKING, + NULL /* handle2 */, + NULL /* handle_monitor */ +}; + +typedef struct { + ErlDrvPort port; + ErlDrvTermData caller; + int counter; +} async_blast_data_t; + + +DRIVER_INIT(async_blast_drv) +{ + return &async_blast_drv_entry; +} + +static void stop(ErlDrvData drv_data) +{ + driver_free((void *) drv_data); +} + +static ErlDrvData start(ErlDrvPort port, + char *command) +{ + async_blast_data_t *abd; + + abd = driver_alloc(sizeof(async_blast_data_t)); + if (!abd) + return ERL_DRV_ERROR_GENERAL; + + abd->port = port; + abd->counter = 0; + return (ErlDrvData) abd; +} + +static void async_invoke(void *data) +{ + +} +#include <stdio.h> + +static void ready_async(ErlDrvData drv_data, + ErlDrvThreadData thread_data) +{ + async_blast_data_t *abd = (async_blast_data_t *) drv_data; + if (--abd->counter == 0) { + ErlDrvTermData spec[] = { + ERL_DRV_PORT, driver_mk_port(abd->port), + ERL_DRV_ATOM, driver_mk_atom("done"), + ERL_DRV_TUPLE, 2 + }; + driver_send_term(abd->port, abd->caller, + spec, sizeof(spec)/sizeof(spec[0])); + } +} + +static void output(ErlDrvData drv_data, + char *buf, int len) +{ + async_blast_data_t *abd = (async_blast_data_t *) drv_data; + if (abd->counter == 0) { + int i; + abd->caller = driver_caller(abd->port); + abd->counter = NO_ASYNC_JOBS; + for (i = 0; i < NO_ASYNC_JOBS; i++) { + if (0 > driver_async(abd->port, NULL, async_invoke, NULL, NULL)) { + driver_failure_atom(abd->port, "driver_async_failed"); + break; + } + } + } +} diff --git a/erts/emulator/test/driver_SUITE_data/thr_free_drv.c b/erts/emulator/test/driver_SUITE_data/thr_free_drv.c new file mode 100644 index 0000000000..622a62ebea --- /dev/null +++ b/erts/emulator/test/driver_SUITE_data/thr_free_drv.c @@ -0,0 +1,241 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2011. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ +#include <stdlib.h> +#include <errno.h> +#include <string.h> +#include "erl_driver.h" + +#define BLOCKS_PER_THREAD 100000 +#define NO_THREADS 10 +#define BLOCKS_PER_CTRL 1000 + +typedef struct { + ErlDrvMutex *mtx; + ErlDrvCond *cnd; + int b; + int *go; + int *skip; + void *blocks[BLOCKS_PER_THREAD]; +} test_thread_data; + +typedef struct { + ErlDrvPort port; + int b; + int go; + int skip; + test_thread_data ttd[NO_THREADS+1]; + ErlDrvTid tids[NO_THREADS+1]; +} test_data; + +static ErlDrvData start(ErlDrvPort port, char *command); +static void stop(ErlDrvData data); +static int control(ErlDrvData drv_data, unsigned int command, char *buf, + int len, char **rbuf, int rlen); + +static ErlDrvEntry thr_free_drv_entry = { + NULL /* init */, + start, + stop, + NULL /* output */, + NULL /* ready_input */, + NULL /* ready_output */, + "thr_free_drv", + NULL /* finish */, + NULL /* handle */, + control, + NULL /* timeout */, + NULL /* outputv */, + NULL /* ready_async */, + NULL /* flush */, + NULL /* call */, + NULL /* event */, + ERL_DRV_EXTENDED_MARKER, + ERL_DRV_EXTENDED_MAJOR_VERSION, + ERL_DRV_EXTENDED_MINOR_VERSION, + ERL_DRV_FLAG_USE_PORT_LOCKING, + NULL /* handle2 */, + NULL /* handle_monitor */ +}; + +DRIVER_INIT(thr_free_drv) +{ + return &thr_free_drv_entry; +} + +void * +test_thread(void *vttd) +{ + test_thread_data *ttd = (test_thread_data *) vttd; + int i, skip; + + erl_drv_mutex_lock(ttd->mtx); + + while (!*ttd->go) + erl_drv_cond_wait(ttd->cnd, ttd->mtx); + skip = *ttd->skip; + erl_drv_mutex_unlock(ttd->mtx); + + if (!skip) { + for (i = 0; i < BLOCKS_PER_THREAD; i++) + driver_free(ttd->blocks[i]); + } + return NULL; +} + +ErlDrvData start(ErlDrvPort port, char *command) +{ + int join = 0, t, b, res; + test_thread_data *ttd; + test_data *td = driver_alloc(sizeof(test_data)); + if (!td) + return ERL_DRV_ERROR_GENERAL; + ttd = td->ttd; + for (b = 0; b < BLOCKS_PER_THREAD; b++) + for (t = 0; t <= NO_THREADS; t++) + ttd[t].blocks[b] = NULL; + ttd[0].mtx = NULL; + ttd[0].cnd = NULL; + + for (b = 0; b < BLOCKS_PER_THREAD; b++) { + for (t = 0; t <= NO_THREADS; t++) { + ttd[t].blocks[b] = driver_alloc(1); + if (ttd[t].blocks[b] == NULL) + goto fail; + } + } + + td->b = -1; + td->go = 0; + td->skip = 0; + + ttd[0].mtx = erl_drv_mutex_create("test_mutex"); + if (!ttd[0].mtx) + goto fail; + ttd[0].cnd = erl_drv_cond_create("test_cnd"); + if (!ttd[0].cnd) + goto fail; + ttd[0].go = &td->go; + ttd[0].skip = &td->skip; + + for (t = 1; t <= NO_THREADS; t++) { + ttd[t].mtx = ttd[0].mtx; + ttd[t].cnd = ttd[0].cnd; + ttd[t].go = ttd[0].go; + ttd[t].skip = ttd[0].skip; + res = erl_drv_thread_create("test_thread", + &td->tids[t], + test_thread, + &ttd[t], + NULL); + if (res != 0) + goto fail; + join = t; + } + + td->port = port; + + return (ErlDrvData) td; + +fail: + + if (join) { + erl_drv_mutex_lock(ttd[0].mtx); + td->go = 1; + td->skip = 1; + erl_drv_cond_broadcast(ttd[0].cnd); + erl_drv_mutex_unlock(ttd[0].mtx); + for (t = 1; t <= join; t++) + erl_drv_thread_join(td->tids[t], NULL); + } + + if (ttd[0].mtx) + erl_drv_mutex_destroy(ttd[0].mtx); + if (ttd[0].cnd) + erl_drv_cond_destroy(ttd[0].cnd); + + for (b = 0; b < BLOCKS_PER_THREAD; b++) { + for (t = 0; t <= NO_THREADS; t++) { + if (ttd[t].blocks[b] != NULL) + driver_free(ttd[t].blocks[b]); + } + } + + return ERL_DRV_ERROR_GENERAL; +} + +static void stop(ErlDrvData drv_data) +{ + test_data *td = (test_data *) drv_data; + int t, b; + for (t = 1; t <= NO_THREADS; t++) + erl_drv_thread_join(td->tids[t], NULL); + for (b = 0; b < BLOCKS_PER_THREAD; b++) { + if (td->ttd[0].blocks[b]) + driver_free(td->ttd[0].blocks[b]); + } + erl_drv_mutex_destroy(td->ttd[0].mtx); + erl_drv_cond_destroy(td->ttd[0].cnd); + driver_free(td); +} + +static int control(ErlDrvData drv_data, unsigned int command, char *buf, + int len, char **rbuf, int rlen) +{ + test_data *td = (test_data *) drv_data; + char *result = "failure"; + int i, b; + int res; + int result_len; + + if (td->b == -1) { + erl_drv_mutex_lock(td->ttd[0].mtx); + td->go = 1; + erl_drv_cond_broadcast(td->ttd[0].cnd); + erl_drv_mutex_unlock(td->ttd[0].mtx); + td->b = 0; + } + + for (i = 0, b = td->b; i < BLOCKS_PER_CTRL && b < BLOCKS_PER_THREAD; i++, b++) { + driver_free(td->ttd[0].blocks[b]); + td->ttd[0].blocks[b] = NULL; + } + + td->b = b; + if (b >= BLOCKS_PER_THREAD) + result = "done"; + else + result = "more"; + + result_len = strlen(result); + if (result_len <= rlen) { + memcpy(*rbuf, result, result_len); + return result_len; + } + else { + *rbuf = driver_alloc(result_len); + if (!*rbuf) { + driver_failure_posix(td->port, ENOMEM); + return 0; + } + else { + memcpy(*rbuf, result, result_len); + return result_len; + } + } +} diff --git a/erts/emulator/test/exception_SUITE.erl b/erts/emulator/test/exception_SUITE.erl index 9d6fc9521d..109cec25cb 100644 --- a/erts/emulator/test/exception_SUITE.erl +++ b/erts/emulator/test/exception_SUITE.erl @@ -23,9 +23,10 @@ init_per_group/2,end_per_group/2, badmatch/1, pending_errors/1, nil_arith/1, stacktrace/1, nested_stacktrace/1, raise/1, gunilla/1, per/1, - exception_with_heap_frag/1]). + exception_with_heap_frag/1, line_numbers/1]). -export([bad_guy/2]). +-export([crash/1]). -include_lib("test_server/include/test_server.hrl"). -import(lists, [foreach/2]). @@ -35,7 +36,7 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> [badmatch, pending_errors, nil_arith, stacktrace, nested_stacktrace, raise, gunilla, per, - exception_with_heap_frag]. + exception_with_heap_frag, line_numbers]. groups() -> []. @@ -141,14 +142,20 @@ pending_exit_message(Args, Expected) -> end, process_flag(trap_exit, false). -pending({badarg, [{erlang,Bif,BifArgs},{?MODULE,Func,Arity}|_]}, Func, Args, _Code) - when is_atom(Bif), is_list(BifArgs), length(Args) == Arity -> +pending({badarg,[{erlang,Bif,BifArgs,Loc1}, + {?MODULE,Func,Arity,Loc2}|_]}, + Func, Args, _Code) + when is_atom(Bif), is_list(BifArgs), length(Args) =:= Arity, + is_list(Loc1), is_list(Loc2) -> ok; -pending({undef,[{non_existing_module,foo,[]}|_]}, _, _, _) -> +pending({undef,[{non_existing_module,foo,[],Loc}|_]}, _, _, _) + when is_list(Loc) -> ok; -pending({function_clause,[{?MODULE,Func,Args}|_]}, Func, Args, _Code) -> +pending({function_clause,[{?MODULE,Func,Args,Loc}|_]}, Func, Args, _Code) + when is_list(Loc) -> ok; -pending({Code,[{?MODULE,Func,Arity}|_]}, Func, Args, Code) when length(Args) == Arity -> +pending({Code,[{?MODULE,Func,Arity,Loc}|_]}, Func, Args, Code) + when length(Args) =:= Arity, is_list(Loc) -> ok; pending(Reason, _Function, _Args, _Code) -> test_server:fail({bad_exit_reason,Reason}). @@ -255,24 +262,24 @@ stacktrace(Conf) when is_list(Conf) -> ?line {_,Mref} = spawn_monitor(fun() -> exit({Tag,erlang:get_stacktrace()}) end), ?line {Tag,[]} = receive {'DOWN',Mref,_,_,Info} -> Info end, V = [make_ref()|self()], - ?line {value2,{caught1,badarg,[{erlang,abs,[V]}|_]=St1}} = + ?line {value2,{caught1,badarg,[{erlang,abs,[V],_}|_]=St1}} = stacktrace_1({'abs',V}, error, {value,V}), ?line St1 = erase(stacktrace1), ?line St1 = erase(stacktrace2), ?line St1 = erlang:get_stacktrace(), - ?line {caught2,{error,badarith},[{?MODULE,my_add,2}|_]=St2} = + ?line {caught2,{error,badarith},[{?MODULE,my_add,2,_}|_]=St2} = stacktrace_1({'div',{1,0}}, error, {'add',{0,a}}), - ?line [{?MODULE,my_div,2}|_] = erase(stacktrace1), + ?line [{?MODULE,my_div,2,_}|_] = erase(stacktrace1), ?line St2 = erase(stacktrace2), ?line St2 = erlang:get_stacktrace(), - ?line {caught2,{error,{try_clause,V}},[{?MODULE,stacktrace_1,3}|_]=St3} = + ?line {caught2,{error,{try_clause,V}},[{?MODULE,stacktrace_1,3,_}|_]=St3} = stacktrace_1({value,V}, error, {value,V}), ?line St3 = erase(stacktrace1), ?line St3 = erase(stacktrace2), ?line St3 = erlang:get_stacktrace(), - ?line {caught2,{throw,V},[{?MODULE,foo,1}|_]=St4} = + ?line {caught2,{throw,V},[{?MODULE,foo,1,_}|_]=St4} = stacktrace_1({value,V}, error, {throw,V}), - ?line [{?MODULE,stacktrace_1,3}|_] = erase(stacktrace1), + ?line [{?MODULE,stacktrace_1,3,_}|_] = erase(stacktrace1), ?line St4 = erase(stacktrace2), ?line St4 = erlang:get_stacktrace(), @@ -280,8 +287,8 @@ stacktrace(Conf) when is_list(Conf) -> ?line stacktrace_2() catch error:{badmatch,_} -> - [{?MODULE,stacktrace_2,0}, - {?MODULE,stacktrace,1}|_] = + [{?MODULE,stacktrace_2,0,_}, + {?MODULE,stacktrace,1,_}|_] = erlang:get_stacktrace(), ok end. @@ -315,15 +322,15 @@ nested_stacktrace(Conf) when is_list(Conf) -> nested_stacktrace_1({{value,{V,x1}},void,{V,x1}}, {void,void,void}), ?line {caught1, - [{?MODULE,my_add,2}|_], + [{?MODULE,my_add,2,_}|_], value2, - [{?MODULE,my_add,2}|_]} = + [{?MODULE,my_add,2,_}|_]} = nested_stacktrace_1({{'add',{V,x1}},error,badarith}, {{value,{V,x2}},void,{V,x2}}), ?line {caught1, - [{?MODULE,my_add,2}|_], - {caught2,[{erlang,abs,[V]}|_]}, - [{erlang,abs,[V]}|_]} = + [{?MODULE,my_add,2,_}|_], + {caught2,[{erlang,abs,[V],_}|_]}, + [{erlang,abs,[V],_}|_]} = nested_stacktrace_1({{'add',{V,x1}},error,badarith}, {{'abs',V},error,badarg}), ok. @@ -362,14 +369,14 @@ raise(Conf) when is_list(Conf) -> end, ?line A = erlang:get_stacktrace(), ?line A = get(raise), - ?line [{?MODULE,my_div,2}|_] = A, + ?line [{?MODULE,my_div,2,_}|_] = A, %% N = 8, % Must be even ?line N = erlang:system_flag(backtrace_depth, N), + ?line B = odd_even(N, []), ?line try even(N) catch error:function_clause -> ok end, - ?line B = odd_even(N, []), ?line B = erlang:get_stacktrace(), %% ?line C0 = odd_even(N+1, []), @@ -387,19 +394,12 @@ raise(Conf) when is_list(Conf) -> odd_even(N, R) when is_integer(N), N > 1 -> odd_even(N-1, [if (N rem 2) == 0 -> - {?MODULE,even,1}; + {?MODULE,even,1,[{file,"odd_even.erl"},{line,3}]}; true -> - {?MODULE,odd,1} + {?MODULE,odd,1,[{file,"odd_even.erl"},{line,6}]} end|R]); odd_even(1, R) -> - [{?MODULE,odd,[1]}|R]. - -even(N) when is_integer(N), N > 1, (N rem 2) == 0 -> - odd(N-1)++[N]. - -odd(N) when is_integer(N), N > 1, (N rem 2) == 1 -> - even(N-1)++[N]. - + [{?MODULE,odd,[1],[{file,"odd_even.erl"},{line,5}]}|R]. foo({value,Value}) -> Value; foo({'div',{A,B}}) -> @@ -526,4 +526,186 @@ do_exception_with_heap_frag(Bin, [Sz|Sizes]) -> do_exception_with_heap_frag(Bin, Sizes); do_exception_with_heap_frag(_, []) -> ok. +line_numbers(Config) when is_list(Config) -> + {'EXIT',{{case_clause,bad_tag}, + [{?MODULE,line1,2, + [{file,"fake_file.erl"},{line,3}]}, + {?MODULE,line_numbers,1,_}|_]}} = + (catch line1(bad_tag, 0)), + {'EXIT',{badarith, + [{?MODULE,line1,2, + [{file,"fake_file.erl"},{line,5}]}, + {?MODULE,line_numbers,1,_}|_]}} = + (catch line1(a, not_an_integer)), + {'EXIT',{{badmatch,{ok,1}}, + [{?MODULE,line1,2, + [{file,"fake_file.erl"},{line,7}]}, + {?MODULE,line_numbers,1,_}|_]}} = + (catch line1(a, 0)), + {'EXIT',{crash, + [{?MODULE,crash,1, + [{file,"fake_file.erl"},{line,14}]}, + {?MODULE,line_numbers,1,_}|_]}} = + (catch line1(a, 41)), + + ModFile = ?MODULE_STRING++".erl", + [{?MODULE,maybe_crash,1,[{file,"call.erl"},{line,28}]}, + {?MODULE,call1,0,[{file,"call.erl"},{line,14}]}, + {?MODULE,close_calls,1,[{file,"call.erl"},{line,5}]}, + {?MODULE,line_numbers,1,[{file,ModFile},{line,_}]}|_] = + close_calls(call1), + [{?MODULE,maybe_crash,1,[{file,"call.erl"},{line,28}]}, + {?MODULE,call2,0,[{file,"call.erl"},{line,18}]}, + {?MODULE,close_calls,1,[{file,"call.erl"},{line,6}]}, + {?MODULE,line_numbers,1,[{file,ModFile},{line,_}]}|_] = + close_calls(call2), + [{?MODULE,maybe_crash,1,[{file,"call.erl"},{line,28}]}, + {?MODULE,call3,0,[{file,"call.erl"},{line,22}]}, + {?MODULE,close_calls,1,[{file,"call.erl"},{line,7}]}, + {?MODULE,line_numbers,1,[{file,ModFile},{line,_}]}|_] = + close_calls(call3), + no_crash = close_calls(other), + + <<0,0>> = build_binary1(16), + {'EXIT',{badarg, + [{?MODULE,build_binary1,1, + [{file,"bit_syntax.erl"},{line,72503}]}, + {?MODULE,line_numbers,1, + [{file,ModFile},{line,_}]}|_]}} = + (catch build_binary1(bad_size)), + + <<7,1,2,3>> = build_binary2(8, <<1,2,3>>), + {'EXIT',{badarg, + [{?MODULE,build_binary2,2, + [{file,"bit_syntax.erl"},{line,72507}]}, + {?MODULE,line_numbers,1, + [{file,ModFile},{line,_}]}|_]}} = + (catch build_binary2(bad_size, <<>>)), + {'EXIT',{badarg, + [{erlang,bit_size,[bad_binary],[]}, + {?MODULE,build_binary2,2, + [{file,"bit_syntax.erl"},{line,72507}]}, + {?MODULE,line_numbers,1, + [{file,ModFile},{line,_}]}|_]}} = + (catch build_binary2(8, bad_binary)), + + {'EXIT',{function_clause, + [{?MODULE,do_call_abs,[y,y], + [{file,"gc_bif.erl"},{line,18}]}, + {?MODULE,line_numbers,1,_}|_]}} = + (catch do_call_abs(y, y)), + {'EXIT',{badarg, + [{erlang,abs,[[]],[]}, + {?MODULE,do_call_abs,2, + [{file,"gc_bif.erl"},{line,19}]}, + {?MODULE,line_numbers,1,_}|_]}} = + (catch do_call_abs(x, [])), + + {'EXIT',{{badmatch,"42"}, + [{MODULE,applied_bif_1,1,[{file,"applied_bif.erl"},{line,5}]}, + {?MODULE,line_numbers,1,_}|_]}} = + (catch applied_bif_1(42)), + + {'EXIT',{{badmatch,{current_location, + {?MODULE,applied_bif_2,0, + [{file,"applied_bif.erl"},{line,9}]}}}, + [{MODULE,applied_bif_2,0,[{file,"applied_bif.erl"},{line,10}]}, + {?MODULE,line_numbers,1,_}|_]}} = + (catch applied_bif_2()), + + ok. + id(I) -> I. + +-file("odd_even.erl", 1). %Line 1 +even(N) when is_integer(N), N > 1, (N rem 2) == 0 -> + odd(N-1)++[N]. %Line 3 + +odd(N) when is_integer(N), N > 1, (N rem 2) == 1 -> + even(N-1)++[N]. %Line 6 + +%% +%% If the compiler removes redundant line instructions (any +%% line instruction with the same location as the previous), +%% and the loader also removes line instructions before +%% tail-recursive calls to external functions, then the +%% badmatch exception in line 7 below will be reported as +%% occurring in line 6. +%% +%% That means that any removal of redundant line instructions +%% must all be done in the compiler OR in the loader. +%% +-file("fake_file.erl", 1). %Line 1 +line1(Tag, X) -> %Line 2 + case Tag of %Line 3 + a -> + Y = X + 1, %Line 5 + Res = id({ok,Y}), %Line 6 + ?MODULE:crash({ok,42} = Res); %Line 7 + b -> + x = id(x), %Line 9 + ok %Line 10 + end. %Line 11 + +crash(_) -> %Line 13 + erlang:error(crash). %Line 14 + +-file("call.erl", 1). %Line 1 +close_calls(Where) -> %Line 2 + put(where_to_crash, Where), %Line 3 + try + call1(), %Line 5 + call2(), %Line 6 + call3(), %Line 7 + no_crash %Line 8 + catch error:crash -> + erlang:get_stacktrace() %Line 10 + end. %Line 11 + +call1() -> %Line 13 + maybe_crash(call1), %Line 14 + ok. %Line 15 + +call2() -> %Line 17 + maybe_crash(call2), %Line 18 + ok. %Line 19 + +call3() -> %Line 21 + maybe_crash(call3), %Line 22 + ok. %Line 23 + +maybe_crash(Name) -> %Line 25 + case get(where_to_crash) of %Line 26 + Name -> + erlang:error(crash); %Line 28 + _ -> + ok %Line 30 + end. + +-file("bit_syntax.erl", 72500). %Line 72500 +build_binary1(Size) -> %Line 72501 + id(42), %Line 72502 + <<0:Size>>. %Line 72503 + +build_binary2(Size, Bin) -> %Line 72505 + id(0), %Line 72506 + <<7:Size,Bin/binary>>. %Line 72507 + +-file("gc_bif.erl", 17). +do_call_abs(x, Arg) -> %Line 18 + abs(Arg). %Line 19 + +%% Make sure a BIF that is applied does not leave the p->cp +%% set (and thus generating an extra entry on the stack). + +-file("applied_bif.erl", 1). +%% Explicit apply. +applied_bif_1(I) -> %Line 3 + L = apply(erlang, integer_to_list, [I]), %Line 4 + fail = L, %Line 5 + ok. %Line 6 +%% Implicit apply. +applied_bif_2() -> %Line 8 + R = process_info(self(), current_location), %Line 9 + fail = R, %Line 10 + ok. %Line 11 diff --git a/erts/emulator/test/float_SUITE.erl b/erts/emulator/test/float_SUITE.erl index 736510339f..46466427c5 100644 --- a/erts/emulator/test/float_SUITE.erl +++ b/erts/emulator/test/float_SUITE.erl @@ -25,7 +25,7 @@ init_per_group/2,end_per_group/2, init_per_testcase/2,end_per_testcase/2, fpe/1,fp_drv/1,fp_drv_thread/1,denormalized/1,match/1, - bad_float_unpack/1]). + bad_float_unpack/1,cmp_zero/1, cmp_integer/1, cmp_bignum/1]). -export([otp_7178/1]). @@ -41,10 +41,10 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> [fpe, fp_drv, fp_drv_thread, otp_7178, denormalized, - match, bad_float_unpack]. + match, bad_float_unpack, {group, comparison}]. groups() -> - []. + [{comparison, [parallel], [cmp_zero, cmp_integer, cmp_bignum]}]. init_per_suite(Config) -> Config. @@ -187,6 +187,101 @@ bad_float_unpack(Config) when is_list(Config) -> bad_float_unpack_match(<<F:64/float>>) -> F; bad_float_unpack_match(<<I:64/integer-signed>>) -> I. +cmp_zero(_Config) -> + cmp(0.5e-323,0). + +cmp_integer(_Config) -> + Axis = (1 bsl 53)-2.0, %% The point where floating points become unprecise + span_cmp(Axis,2,200), + cmp(Axis*Axis,round(Axis)). + +cmp_bignum(_Config) -> + span_cmp((1 bsl 58) - 1.0),%% Smallest bignum float + + %% Test when the big num goes from I to I+1 in size + [span_cmp((1 bsl (32*I)) - 1.0) || I <- lists:seq(2,30)], + + %% Test bignum greater then largest float + cmp((1 bsl (64*16)) - 1, (1 bsl (64*15)) * 1.0), + %% Test when num is much larger then float + [cmp((1 bsl (32*I)) - 1, (1 bsl (32*(I-2))) * 1.0) || I <- lists:seq(3,30)], + %% Test when float is much larger than num + [cmp((1 bsl (64*15)) * 1.0, (1 bsl (32*(I)))) || I <- lists:seq(1,29)], + + %% Test that all int == float works as they should + [true = 1 bsl N == (1 bsl N)*1.0 || N <- lists:seq(0, 1023)], + [true = (1 bsl N)*-1 == (1 bsl N)*-1.0 || N <- lists:seq(0, 1023)]. + +span_cmp(Axis) -> + span_cmp(Axis, 25). +span_cmp(Axis, Length) -> + span_cmp(Axis, round(Axis) bsr 52, Length). +span_cmp(Axis, Incr, Length) -> + [span_cmp(Axis, Incr, Length, 1 bsl (1 bsl I)) || I <- lists:seq(0,6)]. +%% This function creates tests around number axis. Both <, > and == is tested +%% for both negative and positive numbers. +%% +%% Axis: The number around which to do the tests eg. (1 bsl 58) - 1.0 +%% Incr: How much to increment the test numbers inbetween each test. +%% Length: Length/2 is the number of Incr away from Axis to test on the +%% negative and positive plane. +%% Diff: How much the float and int should differ when comparing +span_cmp(Axis, Incr, Length, Diff) -> + [begin + cmp(round(Axis*-1.0)+Diff+I*Incr,Axis*-1.0+I*Incr), + cmp(Axis*-1.0+I*Incr,round(Axis*-1.0)-Diff+I*Incr) + end || I <- lists:seq((Length div 2)*-1,(Length div 2))], + [begin + cmp(round(Axis)+Diff+I*Incr,Axis+I*Incr), + cmp(Axis+I*Incr,round(Axis)-Diff+I*Incr) + end || I <- lists:seq((Length div 2)*-1,(Length div 2))]. + +cmp(Big,Small) when is_float(Big) -> + BigGtSmall = lists:flatten( + io_lib:format("~f > ~p",[Big,Small])), + BigLtSmall = lists:flatten( + io_lib:format("~f < ~p",[Big,Small])), + BigEqSmall = lists:flatten( + io_lib:format("~f == ~p",[Big,Small])), + SmallGtBig = lists:flatten( + io_lib:format("~p > ~f",[Small,Big])), + SmallLtBig = lists:flatten( + io_lib:format("~p < ~f",[Small,Big])), + SmallEqBig = lists:flatten( + io_lib:format("~p == ~f",[Small,Big])), + cmp(Big,Small,BigGtSmall,BigLtSmall,SmallGtBig,SmallLtBig, + SmallEqBig,BigEqSmall); +cmp(Big,Small) when is_float(Small) -> + BigGtSmall = lists:flatten( + io_lib:format("~p > ~f",[Big,Small])), + BigLtSmall = lists:flatten( + io_lib:format("~p < ~f",[Big,Small])), + BigEqSmall = lists:flatten( + io_lib:format("~p == ~f",[Big,Small])), + SmallGtBig = lists:flatten( + io_lib:format("~f > ~p",[Small,Big])), + SmallLtBig = lists:flatten( + io_lib:format("~f < ~p",[Small,Big])), + SmallEqBig = lists:flatten( + io_lib:format("~f == ~p",[Small,Big])), + cmp(Big,Small,BigGtSmall,BigLtSmall,SmallGtBig,SmallLtBig, + SmallEqBig,BigEqSmall). + +cmp(Big,Small,BigGtSmall,BigLtSmall,SmallGtBig,SmallLtBig, + SmallEqBig,BigEqSmall) -> + {_,_,_,true} = {Big,Small,BigGtSmall, + Big > Small}, + {_,_,_,false} = {Big,Small,BigLtSmall, + Big < Small}, + {_,_,_,false} = {Big,Small,SmallGtBig, + Small > Big}, + {_,_,_,true} = {Big,Small,SmallLtBig, + Small < Big}, + {_,_,_,false} = {Big,Small,SmallEqBig, + Small == Big}, + {_,_,_,false} = {Big,Small,BigEqSmall, + Big == Small}. + id(I) -> I. start_node(Config) when is_list(Config) -> diff --git a/erts/emulator/test/guard_SUITE.erl b/erts/emulator/test/guard_SUITE.erl index f41324c2cc..a5df9b59a0 100644 --- a/erts/emulator/test/guard_SUITE.erl +++ b/erts/emulator/test/guard_SUITE.erl @@ -421,7 +421,7 @@ try_gbif(Id, X, Y) -> try_fail_gbif(Id, X, Y) -> case catch guard_bif(Id, X, Y) of - {'EXIT', {function_clause,[{?MODULE,guard_bif,[Id,X,Y]}|_]}} -> + {'EXIT',{function_clause,[{?MODULE,guard_bif,[Id,X,Y],_}|_]}} -> io:format("guard_bif(~p, ~p, ~p) -- ok", [Id,X,Y]); Other -> ?line ok = io:format("guard_bif(~p, ~p, ~p) -- bad result: ~p\n", @@ -493,9 +493,9 @@ type_tests(Test, [Type|T], Allowed) -> end; false -> case catch type_test(Test, Value) of - {'EXIT', {function_clause, {?MODULE,type_test,[Test,Value]}}} -> - ok; - {'EXIT', {function_clause,[{?MODULE,type_test,[Test,Value]}|_]}} -> + {'EXIT',{function_clause, + [{?MODULE,type_test,[Test,Value],Loc}|_]}} + when is_list(Loc) -> ok; {'EXIT',Other} -> ?line test_server:fail({unexpected_error_reason,Other}); diff --git a/erts/emulator/test/mtx_SUITE.erl b/erts/emulator/test/mtx_SUITE.erl index e0a7878bd8..879d2f61dd 100644 --- a/erts/emulator/test/mtx_SUITE.erl +++ b/erts/emulator/test/mtx_SUITE.erl @@ -62,16 +62,29 @@ init_per_suite(Config) when is_list(Config) -> Config. end_per_suite(Config) when is_list(Config) -> + catch erts_debug:set_internal_state(available_internal_state, false), Config. init_per_testcase(_Case, Config) -> Dog = ?t:timetrap(?t:minutes(15)), + %% Wait for deallocations to complete since we measure + %% runtime in test cases. + wait_deallocations(), [{watchdog, Dog}|Config]. end_per_testcase(_Func, Config) -> Dog = ?config(watchdog, Config), ?t:timetrap_cancel(Dog). +wait_deallocations() -> + try + erts_debug:set_internal_state(wait, deallocations) + catch + error:undef -> + erts_debug:set_internal_state(available_internal_state, true), + wait_deallocations() + end. + suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl index f6344791f1..370363bf9e 100644 --- a/erts/emulator/test/nif_SUITE.erl +++ b/erts/emulator/test/nif_SUITE.erl @@ -35,7 +35,9 @@ resource_takeover/1, threading/1, send/1, send2/1, send3/1, send_threaded/1, neg/1, is_checks/1, - get_length/1, make_atom/1, make_string/1]). + get_length/1, make_atom/1, make_string/1, reverse_list_test/1, + otp_9668/1 + ]). -export([many_args_100/100]). @@ -60,7 +62,9 @@ all() -> iolist_as_binary, resource, resource_binary, resource_takeover, threading, send, send2, send3, send_threaded, neg, is_checks, get_length, make_atom, - make_string]. + make_string,reverse_list_test, + otp_9668 + ]. groups() -> []. @@ -1165,7 +1169,28 @@ is_checks(Config) when is_list(Config) -> ?line ensure_lib_loaded(Config, 1), ?line ok = check_is(hejsan, <<19,98>>, make_ref(), ok, fun() -> ok end, self(), hd(erlang:ports()), [], [1,9,9,8], - {hejsan, "hejsan", [$h,"ejs",<<"an">>]}), + {hejsan, "hejsan", [$h,"ejs",<<"an">>]}, 12), + ?line ok = check_is(hejsan, <<19,98>>, make_ref(), ok, fun() -> ok end, + self(), hd(erlang:ports()), [], [1,9,9,8], + {hejsan, "hejsan", [$h,"ejs",<<"an">>]}, -12), + ?line ok = check_is(hejsan, <<19,98>>, make_ref(), ok, fun() -> ok end, + self(), hd(erlang:ports()), [], [1,9,9,8], + {hejsan, "hejsan", [$h,"ejs",<<"an">>]}, 18446744073709551617), + ?line ok = check_is(hejsan, <<19,98>>, make_ref(), ok, fun() -> ok end, + self(), hd(erlang:ports()), [], [1,9,9,8], + {hejsan, "hejsan", [$h,"ejs",<<"an">>]}, -18446744073709551617), + ?line ok = check_is(hejsan, <<19,98>>, make_ref(), ok, fun() -> ok end, + self(), hd(erlang:ports()), [], [1,9,9,8], + {hejsan, "hejsan", [$h,"ejs",<<"an">>]}, 99.146), + ?line ok = check_is(hejsan, <<19,98>>, make_ref(), ok, fun() -> ok end, + self(), hd(erlang:ports()), [], [1,9,9,8], + {hejsan, "hejsan", [$h,"ejs",<<"an">>]}, -99.146), + ?line ok = check_is(hejsan, <<19,98>>, make_ref(), ok, fun() -> ok end, + self(), hd(erlang:ports()), [], [1,9,9,8], + {hejsan, "hejsan", [$h,"ejs",<<"an">>]}, 18446744073709551616.2e2), + ?line ok = check_is(hejsan, <<19,98>>, make_ref(), ok, fun() -> ok end, + self(), hd(erlang:ports()), [], [1,9,9,8], + {hejsan, "hejsan", [$h,"ejs",<<"an">>]}, -18446744073709551616.2e2), try ?line error = check_is_exception(), ?line throw(expected_badarg) @@ -1208,6 +1233,27 @@ make_string(Config) when is_list(Config) -> AStringWithAccents = [$E,$r,$l,$a,$n,$g,$ ,16#e4,$r,$ ,$e,$t,$t,$ ,$g,$e,$n,$e,$r,$e,$l,$l,$t,$ ,$p,$r,$o,$g,$r,$a,$m,$s,$p,$r,16#e5,$k], ?line Strings = {A0String,A0String,A0String,A0String0, AStringWithAccents}. +reverse_list_test(Config) -> + ?line ensure_lib_loaded(Config, 1), + List = lists:seq(1,100), + RevList = lists:reverse(List), + ?line RevList = reverse_list(List), + ?line badarg = reverse_list(foo). + +otp_9668(doc) -> ["Memory leak of tmp-buffer when inspecting iolist or unaligned binary in unbound environment"]; +otp_9668(Config) -> + ensure_lib_loaded(Config, 1), + TmpMem = tmpmem(), + IOList = ["This",' ',<<"is">>,' ',[<<"an iolist">>,'.']], + otp_9668_nif(IOList), + + <<_:5/bitstring,UnalignedBin:10/binary,_/bitstring>> = <<"Abuse me as unaligned">>, + otp_9668_nif(UnalignedBin), + + ?line verify_tmpmem(TmpMem), + ok. + + tmpmem() -> case erlang:system_info({allocator,temp_alloc}) of false -> undefined; @@ -1296,7 +1342,7 @@ get_resource(_,_) -> ?nif_stub. release_resource(_) -> ?nif_stub. last_resource_dtor_call() -> ?nif_stub. make_new_resource(_,_) -> ?nif_stub. -check_is(_,_,_,_,_,_,_,_,_,_) -> ?nif_stub. +check_is(_,_,_,_,_,_,_,_,_,_,_) -> ?nif_stub. check_is_exception() -> ?nif_stub. length_test(_,_,_,_,_) -> ?nif_stub. make_atoms() -> ?nif_stub. @@ -1314,8 +1360,10 @@ send_blob_thread(_,_,_) -> ?nif_stub. join_send_thread(_) -> ?nif_stub. copy_blob(_) -> ?nif_stub. send_term(_,_) -> ?nif_stub. +reverse_list(_) -> ?nif_stub. echo_int(_) -> ?nif_stub. type_sizes() -> ?nif_stub. +otp_9668_nif(_) -> ?nif_stub. nif_stub_error(Line) -> exit({nif_not_loaded,module,?MODULE,line,Line}). diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c index 92f1bab8dd..7d7903af25 100644 --- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c +++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c @@ -832,6 +832,7 @@ static ERL_NIF_TERM release_resource(ErlNifEnv* env, int argc, const ERL_NIF_TER * argv[7] an empty list * argv[8] a non-empty list * argv[9] a tuple + * argv[10] a number (small, big integer or float) */ static ERL_NIF_TERM check_is(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { @@ -848,6 +849,7 @@ static ERL_NIF_TERM check_is(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[] if (!enif_is_list(env, argv[7])) return enif_make_badarg(env); if (!enif_is_list(env, argv[8])) return enif_make_badarg(env); if (!enif_is_tuple(env, argv[9])) return enif_make_badarg(env); + if (!enif_is_number(env, argv[10])) return enif_make_badarg(env); return ok_atom; } @@ -1421,6 +1423,34 @@ static ERL_NIF_TERM send_term(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[ return enif_make_int(env, ret); } +static ERL_NIF_TERM reverse_list(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { + ERL_NIF_TERM rev_list; + + if(!enif_make_reverse_list(env, argv[0], &rev_list)) + return enif_make_atom(env, "badarg"); + return rev_list; +} + +static ERL_NIF_TERM otp_9668_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + /* Inspect in process independent env */ + ErlNifEnv* myenv = enif_alloc_env(); + ERL_NIF_TERM mycopy = enif_make_copy(myenv, argv[0]); + ErlNifBinary obin, cbin; + + if ((enif_inspect_binary(env, argv[0], &obin) + && enif_inspect_binary(myenv, mycopy, &cbin)) + || + (enif_inspect_iolist_as_binary(env, argv[0], &obin) + && enif_inspect_iolist_as_binary(myenv, mycopy, &cbin))) + { + assert(obin.size == cbin.size); + assert(memcmp(obin.data, cbin.data, obin.size) == 0); + } + enif_free_env(myenv); + return atom_ok; +} + static ErlNifFunc nif_funcs[] = { {"lib_version", 0, lib_version}, @@ -1447,7 +1477,7 @@ static ErlNifFunc nif_funcs[] = {"release_resource", 1, release_resource}, {"last_resource_dtor_call", 0, last_resource_dtor_call}, {"make_new_resource", 2, make_new_resource}, - {"check_is", 10, check_is}, + {"check_is", 11, check_is}, {"check_is_exception", 0, check_is_exception}, {"length_test", 5, length_test}, {"make_atoms", 0, make_atoms}, @@ -1466,8 +1496,10 @@ static ErlNifFunc nif_funcs[] = {"join_send_thread", 1, join_send_thread}, {"copy_blob", 1, copy_blob}, {"send_term", 2, send_term}, + {"reverse_list",1, reverse_list}, {"echo_int", 1, echo_int}, - {"type_sizes", 0, type_sizes} + {"type_sizes", 0, type_sizes}, + {"otp_9668_nif", 1, otp_9668_nif} }; ERL_NIF_INIT(nif_SUITE,nif_funcs,load,reload,upgrade,unload) diff --git a/erts/emulator/test/process_SUITE.erl b/erts/emulator/test/process_SUITE.erl index f68e712268..fdc55a4cc5 100644 --- a/erts/emulator/test/process_SUITE.erl +++ b/erts/emulator/test/process_SUITE.erl @@ -35,7 +35,7 @@ self_exit/1, normal_suicide_exit/1, abnormal_suicide_exit/1, t_exit_2_catch/1, trap_exit_badarg/1, trap_exit_badarg_in_bif/1, exit_and_timeout/1, exit_twice/1, - t_process_info/1, process_info_other_msg/1, + t_process_info/1, process_info_other/1, process_info_other_msg/1, process_info_other_dist_msg/1, process_info_2_list/1, process_info_lock_reschedule/1, process_info_lock_reschedule2/1, @@ -64,7 +64,7 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> [spawn_with_binaries, t_exit_1, {group, t_exit_2}, trap_exit_badarg, trap_exit_badarg_in_bif, - t_process_info, process_info_other_msg, + t_process_info, process_info_other, process_info_other_msg, process_info_other_dist_msg, process_info_2_list, process_info_lock_reschedule, process_info_lock_reschedule2, @@ -258,7 +258,9 @@ trap_exit_badarg() -> ?line Pid = fun_spawn(fun() -> bad_guy(kb_128()) end), ?line Garbage = kb_128(), ?line receive - {'EXIT', Pid, {badarg,[{erlang,abs,[Garbage]},{?MODULE,bad_guy,1}|_]}} -> + {'EXIT',Pid,{badarg,[{erlang,abs,[Garbage],Loc1}, + {?MODULE,bad_guy,1,Loc2}|_]}} + when is_list(Loc1), is_list(Loc2) -> ok; Other -> ?line ok = io:format("Bad EXIT message: ~P", [Other, 30]), @@ -410,7 +412,7 @@ etwice_high(Low) -> exit(Low, first), exit(Low, second). -%% Tests the process_info/1 BIF. +%% Tests the process_info/2 BIF. t_process_info(Config) when is_list(Config) -> ?line [] = process_info(self(), registered_name), ?line register(my_name, self()), @@ -418,13 +420,100 @@ t_process_info(Config) when is_list(Config) -> ?line {status, running} = process_info(self(), status), ?line {min_heap_size, 233} = process_info(self(), min_heap_size), ?line {min_bin_vheap_size, 46368} = process_info(self(), min_bin_vheap_size), - ?line {current_function, {?MODULE, t_process_info, 1}} = + ?line {current_function,{?MODULE,t_process_info,1}} = process_info(self(), current_function), + ?line {current_function,{?MODULE,t_process_info,1}} = + apply(erlang, process_info, [self(),current_function]), + + %% current_location and current_stacktrace + {Line1,Res1} = {?LINE,process_info(self(), current_location)}, + verify_loc(Line1, Res1), + {Line2,Res2} = {?LINE,apply(erlang, process_info, + [self(),current_location])}, + verify_loc(Line2, Res2), + pi_stacktrace([{?MODULE,t_process_info,1,?LINE}]), + ?line Gleader = group_leader(), ?line {group_leader, Gleader} = process_info(self(), group_leader), ?line {'EXIT',{badarg,_Info}} = (catch process_info('not_a_pid')), ok. +pi_stacktrace(Expected0) -> + {Line,Res} = {?LINE,erlang:process_info(self(), current_stacktrace)}, + {current_stacktrace,Stack} = Res, + Expected = [{?MODULE,pi_stacktrace,1,Line}|Expected0], + pi_stacktrace_1(Stack, Expected). + +pi_stacktrace_1([{M,F,A,Loc}|Stk], [{M,F,A,Line}|Exp]) -> + case Loc of + [] -> + %% No location info for some reason (+L, native code). + io:format("Missing location information for ~w:~w/~w", + [M,F,A]), + ok; + [_|_] -> + Line = proplists:get_value(line, Loc), + File = proplists:get_value(file, Loc), + File = ?MODULE_STRING ++ ".erl" + end, + pi_stacktrace_1(Stk, Exp); +pi_stacktrace_1([_|_], []) -> ok. + +verify_loc(Line, {current_location,{?MODULE,t_process_info=F,1=A,Loc}}) -> + case Loc of + [] -> + %% No location info for some reason (+L, native code). + io:format("Missing location information for ~w:~w/~w", + [?MODULE,F,A]), + ok; + [_|_] -> + Line = proplists:get_value(line, Loc), + File = proplists:get_value(file, Loc), + File = ?MODULE_STRING ++ ".erl" + end. + +process_info_other(Config) when is_list(Config) -> + Self = self(), + Pid = spawn_link(fun() -> process_info_looper(Self) end), + receive after 1 -> ok end, + pio_current_location(10000, Pid, 0, 0), + pio_current_stacktrace(). + +pio_current_location(0, _, Pi, Looper) -> + io:format("~w call(s) to erlang:process_info/2", [Pi]), + io:format("~w call(s) to ~w:process_info_looper/1", [Looper,?MODULE]); +pio_current_location(N, Pid, Pi, Looper) -> + erlang:yield(), + {current_location,Where} = process_info(Pid, current_location), + case Where of + {erlang,process_info,2,[]} -> + pio_current_location(N-1, Pid, Pi+1, Looper); + {?MODULE,process_info_looper,1,Loc} when is_list(Loc) -> + pio_current_location(N-1, Pid, Pi, Looper+1) + end. + +pio_current_stacktrace() -> + L = [begin + {current_stacktrace,Stk} = process_info(P, current_stacktrace), + {P,Stk} + end || P <- processes()], + [erlang:garbage_collect(P) || {P,_} <- L], + erlang:garbage_collect(), + [verify_stacktrace(Stk) || {_,Stk} <- L], + ok. + +verify_stacktrace([{M,F,A,Loc}|T]) + when is_atom(M), + is_atom(F), + is_integer(A), + is_list(Loc) -> + verify_stacktrace(T); +verify_stacktrace([]) -> ok. + +process_info_looper(Parent) -> + process_info(Parent, current_location), + process_info_looper(Parent). + %% Tests the process_info/1 BIF on another process with messages. process_info_other_msg(Config) when is_list(Config) -> Self = self(), diff --git a/erts/emulator/test/system_info_SUITE.erl b/erts/emulator/test/system_info_SUITE.erl index 9b782b35a2..0350eb671d 100644 --- a/erts/emulator/test/system_info_SUITE.erl +++ b/erts/emulator/test/system_info_SUITE.erl @@ -37,7 +37,7 @@ init_per_group/2,end_per_group/2, init_per_testcase/2, end_per_testcase/2]). --export([process_count/1, system_version/1, misc_smoke_tests/1, heap_size/1, wordsize/1]). +-export([process_count/1, system_version/1, misc_smoke_tests/1, heap_size/1, wordsize/1, memory/1]). -define(DEFAULT_TIMEOUT, ?t:minutes(2)). @@ -45,7 +45,7 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> [process_count, system_version, misc_smoke_tests, - heap_size, wordsize]. + heap_size, wordsize, memory]. groups() -> []. @@ -187,3 +187,312 @@ wordsize(Config) when is_list(Config) -> Other -> exit({unexpected_wordsizes,Other}) end. + +memory(doc) -> ["Verify that erlang:memory/0 and memory results in crashdump produce are similar"]; +memory(Config) when is_list(Config) -> + %% + %% Verify that erlang:memory/0 and memory results in + %% crashdump produce are similar. + %% + %% erlang:memory/0 requests information from each scheduler + %% thread and puts the information together in erlang code + %% (erlang.erl). + %% + %% When a crash dump is written we cannot use the + %% erlang:memory/0 implementation. The crashdump implementation + %% is a pure C implementation inspecting all allocator instances + %% after the system has been blocked (erts_memory() in erl_alloc.c). + %% + %% Since we got two implementations, modifications can easily + %% cause them to produce different results. + %% + %% erts_debug:get_internal_state(memory) blocks the system and + %% execute the same code as the crash dump writing uses. + %% + + erts_debug:set_internal_state(available_internal_state, true), + %% Use a large heap size on the controling process in + %% order to avoid changes in its heap size during + %% comparisons. + MinHeapSize = process_flag(min_heap_size, 1024*1024), + Prio = process_flag(priority, max), + try + erlang:memory(), %% first call will init stat atoms + garbage_collect(), %% blow up heap + memory_test(Config) + catch + error:notsup -> {skipped, "erlang:memory() not supported"} + after + process_flag(min_heap_size, MinHeapSize), + process_flag(priority, Prio), + catch erts_debug:set_internal_state(available_internal_state, false) + end. + +memory_test(_Config) -> + + MWs = spawn_mem_workers(), + + DPs = mem_workers_call(MWs, + fun () -> + mapn(fun (_) -> + spawn(fun () -> + receive + after infinity -> + ok + end + end) + end, + 1000 div erlang:system_info(schedulers_online)) + end, + []), + cmp_memory(MWs, "spawn procs"), + + Ps = lists:flatten(DPs), + + mem_workers_call(MWs, + fun () -> + lists:foreach(fun (P) -> link(P) end, Ps) + end, + []), + cmp_memory(MWs, "link procs"), + mem_workers_call(MWs, + fun () -> + lists:foreach(fun (P) -> unlink(P) end, Ps) + end, + []), + cmp_memory(MWs, "unlink procs"), + + DMs = mem_workers_call(MWs, + fun () -> + lists:map(fun (P) -> + monitor(process, P) + end, Ps) + end, + []), + cmp_memory(MWs, "monitor procs"), + Ms = lists:flatten(DMs), + mem_workers_call(MWs, + fun () -> + lists:foreach(fun (M) -> + demonitor(M) + end, Ms) + end, + []), + cmp_memory(MWs, "demonitor procs"), + + mem_workers_call(MWs, + fun () -> + lists:foreach(fun (P) -> + P ! {a, "message", make_ref()} + end, Ps) + end, + []), + cmp_memory(MWs, "message procs"), + + mem_workers_call(MWs, + fun () -> + Mons = lists:map(fun (P) -> + exit(P, kill), + monitor(process, P) + end, + Ps), + lists:foreach(fun (Mon) -> + receive + {'DOWN', Mon, _, _, _} -> ok + end + end, + Mons) + end, []), + cmp_memory(MWs, "kill procs"), + + mem_workers_call(MWs, + fun () -> + put(binary_data, + mapn(fun (_) -> list_to_binary(lists:duplicate(256,$?)) end, 100)) + end, + []), + + cmp_memory(MWs, "store binary data"), + + mem_workers_call(MWs, + fun () -> + put(binary_data, false), + garbage_collect() + end, + []), + cmp_memory(MWs, "release binary data"), + + mem_workers_call(MWs, + fun () -> + list_to_atom("an ugly atom "++integer_to_list(erlang:system_info(scheduler_id))), + list_to_atom("another ugly atom "++integer_to_list(erlang:system_info(scheduler_id))), + list_to_atom("yet another ugly atom "++integer_to_list(erlang:system_info(scheduler_id))) + end, + []), + cmp_memory(MWs, "new atoms"), + + + mem_workers_call(MWs, + fun () -> + T = ets:new(?MODULE, []), + ets:insert(T, {gurka, lists:seq(1,10000)}), + ets:insert(T, {banan, lists:seq(1,1024)}), + ets:insert(T, {appelsin, make_ref()}), + put(ets_id, T) + end, + []), + cmp_memory(MWs, "store ets data"), + + mem_workers_call(MWs, + fun () -> + ets:delete(get(ets_id)), + put(ets_id, false) + end, + []), + cmp_memory(MWs, "remove ets data"), + + lists:foreach(fun (MW) -> + unlink(MW), + Mon = monitor(process, MW), + exit(MW, kill), + receive + {'DOWN', Mon, _, _, _} -> ok + end + end, + MWs), + ok. + +mem_worker() -> + receive + {call, From, Fun, Args} -> + From ! {reply, self(), apply(Fun, Args)}, + mem_worker(); + {cast, _From, Fun, Args} -> + apply(Fun, Args), + mem_worker() + end. + +mem_workers_call(MWs, Fun, Args) -> + lists:foreach(fun (MW) -> + MW ! {call, self(), Fun, Args} + end, + MWs), + lists:map(fun (MW) -> + receive + {reply, MW, Res} -> + Res + end + end, + MWs). + +mem_workers_cast(MWs, Fun, Args) -> + lists:foreach(fun (MW) -> + MW ! {cast, self(), Fun, Args} + end, + MWs). + +spawn_mem_workers() -> + spawn_mem_workers(erlang:system_info(schedulers_online)). + +spawn_mem_workers(0) -> + []; +spawn_mem_workers(N) -> + [spawn_opt(fun () -> mem_worker() end, + [{scheduler, N rem erlang:system_info(schedulers_online) + 1}, + link]) | spawn_mem_workers(N-1)]. + + + +mem_get(X, Mem) -> + case lists:keyfind(X, 1, Mem) of + {X, Val} -> Val; + false -> false + end. + +cmp_memory(What, Mem1, Mem2, 1) -> + R1 = mem_get(What, Mem1), + R2 = mem_get(What, Mem2), + true = R1 == R2; +cmp_memory(What, Mem1, Mem2, RelDiff) -> + %% We allow RealDiff diff + R1 = mem_get(What, Mem1), + R2 = mem_get(What, Mem2), + case R1 == R2 of + true -> + ok; + false -> + case R1 > R2 of + true -> + true = R2*RelDiff > R1; + false -> + true = R1*RelDiff > R2 + end + end. + +pos_int(Val) when Val >= 0 -> + Val; +pos_int(Val) -> + exit({not_pos_int, Val}). + +check_sane_memory(Mem) -> + Tot = pos_int(mem_get(total, Mem)), + Proc = pos_int(mem_get(processes, Mem)), + ProcUsed = pos_int(mem_get(processes_used, Mem)), + Sys = pos_int(mem_get(system, Mem)), + Atom = pos_int(mem_get(atom, Mem)), + AtomUsed = pos_int(mem_get(atom_used, Mem)), + Bin = pos_int(mem_get(binary, Mem)), + Code = pos_int(mem_get(code, Mem)), + Ets = pos_int(mem_get(ets, Mem)), + + Tot = Proc + Sys, + true = Sys > Atom + Bin + Code + Ets, + true = Proc >= ProcUsed, + true = Atom >= AtomUsed, + + case mem_get(maximum, Mem) of + false -> ok; + Max -> true = pos_int(Max) >= Tot + end, + ok. + +cmp_memory(MWs, Str) -> + erlang:display(Str), + lists:foreach(fun (MW) -> garbage_collect(MW) end, MWs), + garbage_collect(), + erts_debug:set_internal_state(wait, deallocations), + + EDM = erts_debug:get_internal_state(memory), + EM = erlang:memory(), + + io:format("~s:~n" + "erlang:memory() = ~p~n" + "crash dump memory = ~p~n", + [Str, EM, EDM]), + + ?line check_sane_memory(EM), + ?line check_sane_memory(EDM), + + %% We expect these to always give us exactly the same result + + ?line cmp_memory(atom, EM, EDM, 1), + ?line cmp_memory(atom_used, EM, EDM, 1), + ?line cmp_memory(binary, EM, EDM, 1), + ?line cmp_memory(code, EM, EDM, 1), + ?line cmp_memory(ets, EM, EDM, 1), + + %% Total, processes, processes_used, and system will seldom + %% give us exactly the same result since the two readings + %% aren't taken atomically. + + ?line cmp_memory(total, EM, EDM, 1.05), + ?line cmp_memory(processes, EM, EDM, 1.05), + ?line cmp_memory(processes_used, EM, EDM, 1.05), + ?line cmp_memory(system, EM, EDM, 1.05), + + ok. + +mapn(_Fun, 0) -> + []; +mapn(Fun, N) -> + [Fun(N) | mapn(Fun, N-1)]. diff --git a/erts/emulator/test/trace_local_SUITE.erl b/erts/emulator/test/trace_local_SUITE.erl index 091e960610..32e2a98e3c 100644 --- a/erts/emulator/test/trace_local_SUITE.erl +++ b/erts/emulator/test/trace_local_SUITE.erl @@ -767,8 +767,8 @@ exception_test(Opts, Func0, Args0) -> end, ?line R1 = exc_slave(ExcOpts, Func, Args), - ?line Stack2 = [{?MODULE,exc_top,3},{?MODULE,slave,2}], - ?line Stack3 = [{?MODULE,exc,2}|Stack2], + ?line Stack2 = [{?MODULE,exc_top,3,[]},{?MODULE,slave,2,[]}], + ?line Stack3 = [{?MODULE,exc,2,[]}|Stack2], ?line Rs = case x_exc_top(ExcOpts, Func, Args) of % Emulation {crash,{Reason,Stack}}=R when is_list(Stack) -> @@ -789,21 +789,29 @@ exception_test(Opts, Func0, Args0) -> end, ?line expect({nm}). -exception_validate(R1, [R2|Rs]) -> +exception_validate(R0, Rs0) -> + R = clean_location(R0), + Rs = [clean_location(E) || E <- Rs0], + exception_validate_1(R, Rs). + +exception_validate_1(R1, [R2|Rs]) -> case [R1|R2] of [R|R] -> ok; - [{crash,{badarg,[{lists,reverse,[L1a,L1b]}|T]}}| - {crash,{badarg,[{lists,reverse,[L2a,L2b]}|T]}}] -> + [{crash,{badarg,[{lists,reverse,[L1a,L1b],_}|T]}}| + {crash,{badarg,[{lists,reverse,[L2a,L2b],_}|T]}}] -> same({crash,{badarg,[{lists,reverse, - [lists:reverse(L1b, L1a),[]]}|T]}}, + [lists:reverse(L1b, L1a),[]],[]}|T]}}, {crash,{badarg,[{lists,reverse, - [lists:reverse(L2b, L2a),[]]}|T]}}); + [lists:reverse(L2b, L2a),[]],[]}|T]}}); _ when is_list(Rs), Rs =/= [] -> exception_validate(R1, Rs) end. - +clean_location({crash,{Reason,Stk0}}) -> + Stk = [{M,F,A,[]} || {M,F,A,_} <- Stk0], + {crash,{Reason,Stk}}; +clean_location(Term) -> Term. %%% Tracee target functions %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%% @@ -1057,10 +1065,10 @@ x_exc_exception(_Rtt, M, F, _, Arity, CR) -> x_exc_stacktrace() -> x_exc_stacktrace(erlang:get_stacktrace()). %% Truncate stacktrace to below exc/2 -x_exc_stacktrace([{?MODULE,x_exc,4}|_]) -> []; -x_exc_stacktrace([{?MODULE,x_exc_func,4}|_]) -> []; -x_exc_stacktrace([{?MODULE,x_exc_body,4}|_]) -> []; -x_exc_stacktrace([{?MODULE,exc,2}|_]) -> []; +x_exc_stacktrace([{?MODULE,x_exc,4,_}|_]) -> []; +x_exc_stacktrace([{?MODULE,x_exc_func,4,_}|_]) -> []; +x_exc_stacktrace([{?MODULE,x_exc_body,4,_}|_]) -> []; +x_exc_stacktrace([{?MODULE,exc,2,_}|_]) -> []; x_exc_stacktrace([H|T]) -> [H|x_exc_stacktrace(T)]. diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops index ebf7db3277..58c36c3bdc 100755 --- a/erts/emulator/utils/beam_makeops +++ b/erts/emulator/utils/beam_makeops @@ -105,7 +105,9 @@ my %match_engine_ops; # All opcodes for the match engine. my %gen_transform_offset; my @transformations; my @call_table; +my %call_table; my @pred_table; +my %pred_table; # Operand types for generic instructions. @@ -187,6 +189,12 @@ sub define_type_bit { } # +# Pre-define the 'fail' instruction. It is used internally +# by the 'try_me_else_fail' instruction. +# +$match_engine_ops{'TOP_fail'} = 1; + +# # Sanity checks. # @@ -1304,7 +1312,8 @@ sub tr_gen { foreach $ref (@g) { my($line, $orig_transform, $from_ref, $to_ref) = @$ref; - my $so_far = tr_gen_from($line, @$from_ref); + my $used_ref = used_vars($from_ref, $to_ref); + my $so_far = tr_gen_from($line, $used_ref, @$from_ref); tr_gen_to($line, $orig_transform, $so_far, @$to_ref); } @@ -1313,9 +1322,22 @@ sub tr_gen { # my($offset) = 0; print "Uint op_transform[] = {\n"; - foreach $key (keys %gen_transform) { + foreach $key (sort keys %gen_transform) { $gen_transform_offset{$key} = $offset; - foreach $instr (@{$gen_transform{$key}}) { + my @instr = @{$gen_transform{$key}}; + + # + # If the last instruction is 'fail', remove it and + # convert the previous 'try_me_else' to 'try_me_else_fail'. + # + if (is_instr($instr[$#instr], 'fail')) { + pop(@instr); + my $i = $#instr; + $i-- while !is_instr($instr[$i], 'try_me_else'); + $instr[$i] = make_op('', 'try_me_else_fail'); + } + + foreach $instr (@instr) { my($size, $instr_ref, $comment) = @$instr; my($op, @args) = @$instr_ref; print " "; @@ -1342,8 +1364,48 @@ sub tr_gen { print "};\n\n"; } +sub used_vars { + my($from_ref,$to_ref) = @_; + my %used; + my %seen; + + foreach my $ref (@$from_ref) { + my($name,$arity,@ops) = @$ref; + if ($name =~ /^[.]/) { + foreach my $var (@ops) { + $used{$var} = 1; + } + } else { + # Any variable that is used at least twice on the + # left-hand side is used. (E.g. "move R R".) + foreach my $op (@ops) { + my($var, $type, $type_val) = @$op; + next if $var eq ''; + $used{$var} = 1 if $seen{$var}; + $seen{$var} = 1; + } + } + } + + foreach my $ref (@$to_ref) { + my($name, $arity, @ops) = @$ref; + if ($name =~ /^[.]/) { + foreach my $var (@ops) { + $used{$var} = 1; + } + } else { + foreach my $op (@ops) { + my($var, $type, $type_val) = @$op; + next if $var eq ''; + $used{$var} = 1; + } + } + } + \%used; +} + sub tr_gen_from { - my($line, @tr) = @_; + my($line,$used_ref,@tr) = @_; my(%var) = (); my(%var_type); my($var_num) = 0; @@ -1353,25 +1415,30 @@ sub tr_gen_from { my(@fix_pred_funcs); my($op, $ref); # Loop variables. my $where = "left side of transformation in line $line: "; + my %var_used = %$used_ref; + my $may_fail = 0; + my $is_first = 1; foreach $ref (@tr) { my($name, $arity, @ops) = @$ref; my($key) = "$name/$arity"; my($opnum); + $may_fail = 1 unless $is_first; + $is_first = 0; + # # A name starting with a period is a C pred function to be called. # if ($name =~ /^\.(\w+)/) { $name = $1; + $may_fail = 1; my $var; my(@args); - my $next_instr = pop(@code); # Get rid of 'next_instr' push(@fix_pred_funcs, scalar(@code)); push(@code, [$name, @ops]); - push(@code, $next_instr); next; } @@ -1383,17 +1450,21 @@ sub tr_gen_from { unless defined $gen_opnum{$name,$arity}; $opnum = $gen_opnum{$name,$arity}; - push(@code, &make_op("$name/$arity", 'is_op', $opnum)); + push(@code, make_op("$name/$arity", 'next_instr', $opnum)); $min_window++; foreach $op (@ops) { my($var, $type, $type_val, $cond, $val) = @$op; + my $ignored_var = "$var (ignored)"; if ($type ne '' && $type ne '*') { + $may_fail = 1; + # # The is_bif, is_not_bif, and is_func instructions have # their own built-in type test and don't need to # be guarded with a type test instruction. # + $ignored_var = ''; unless ($cond eq 'is_bif' or $cond eq 'is_not_bif' or $cond eq 'is_func') { @@ -1407,7 +1478,7 @@ sub tr_gen_from { push(@code, &make_op($types, 'is_type', $type_mask)); } else { $cond = ''; - push(@code, &make_op($types, 'is_type_eq', + push(@code, &make_op("$types== $val", 'is_type_eq', $type_mask, $val)); } } @@ -1415,46 +1486,55 @@ sub tr_gen_from { if ($cond eq 'is_func') { my($m, $f, $a) = split(/:/, $val); + $ignored_var = ''; + $may_fail = 1; push(@code, &make_op('', "$cond", "am_$m", "am_$f", $a)); } elsif ($cond ne '') { + $ignored_var = ''; + $may_fail = 1; push(@code, &make_op('', "$cond", $val)); } if ($var ne '') { if (defined $var{$var}) { + $ignored_var = ''; + $may_fail = 1; push(@code, &make_op($var, 'is_same_var', $var{$var})); } elsif ($type eq '*') { # # Reserve a hole for a 'rest_args' instruction. # + $ignored_var = ''; push(@fix_rest_args, scalar(@code)); push(@code, $var); - } else { + } elsif ($var_used{$var}) { + $ignored_var = ''; $var_type{$var} = 'scalar'; $var{$var} = $var_num; $var_num++; push(@code, &make_op($var, 'set_var', $var{$var})); } } - if (is_set_var_instr($code[$#code])) { + if (is_instr($code[$#code], 'set_var')) { my $ref = pop @code; my $comment = $ref->[2]; my $var = $ref->[1][1]; push(@code, make_op($comment, 'set_var_next_arg', $var)); } else { - push(@code, &make_op('', 'next_arg')); + push(@code, &make_op($ignored_var, 'next_arg')); } } - push(@code, &make_op('', 'next_instr')); - pop(@code) if $code[$#code]->[1][0] eq 'next_arg'; + + # Remove redundant 'next_arg' instructions before the end + # of the instruction. + pop(@code) while is_instr($code[$#code], 'next_arg'); } # # Insert the commit operation. # - pop(@code); # Get rid of 'next_instr' - push(@code, &make_op('', 'commit')); + push(@code, make_op($may_fail ? '' : 'always reached', 'commit')); # # If there is an rest_args instruction, we must insert its correct @@ -1484,9 +1564,8 @@ sub tr_gen_from { push(@args, "var+$var{$var}"); } } - splice(@code, $index, 1, &make_op("$name()", - 'pred', scalar(@pred_table))); - push(@pred_table, [$name, @args]); + my $pi = tr_next_index(\@pred_table, \%pred_table, $name, @args); + splice(@code, $index, 1, make_op("$name()", 'pred', $pi)); } $te_max_vars = $var_num @@ -1503,6 +1582,10 @@ sub tr_gen_to { my($op, $ref); # Loop variables. my($where) = "right side of transformation in line $line: "; + my $last_instr = $code[$#code]; + my $cannot_fail = is_instr($last_instr, 'commit') && + (get_comment($last_instr) =~ /^always/); + foreach $ref (@tr) { my($name, $arity, @ops) = @$ref; @@ -1524,9 +1607,10 @@ sub tr_gen_to { push(@args, "var+$var{$var}"); } } - pop(@code); # Get rid of 'next_instr' - push(@code, &make_op("$name()", 'call', scalar(@call_table))); - push(@call_table, [$name, @args]); + pop(@code); # Get rid of 'commit' instruction + my $index = tr_next_index(\@call_table, \%call_table, + $name, @args); + push(@code, make_op("$name()", 'call_end', $index)); last; } @@ -1543,27 +1627,27 @@ sub tr_gen_to { # Create code to build the generic instruction. # - push(@code, &make_op('', 'new_instr')); - push(@code, &make_op("$name/$arity", 'store_op', $opnum, $arity)); + push(@code, make_op("$name/$arity", 'new_instr', $opnum)); foreach $op (@ops) { my($var, $type, $type_val) = @$op; if ($var ne '') { &error($where, "variable '$var' unbound") unless defined $var{$var}; - push(@code, &make_op($var, 'store_var', $var{$var})); + push(@code, &make_op($var, 'store_var_next_arg', $var{$var})); } elsif ($type ne '') { push(@code, &make_op('', 'store_type', "TAG_$type")); if ($type_val) { push(@code, &make_op('', 'store_val', $type_val)); } + push(@code, make_op('', 'next_arg')); } - push(@code, &make_op('', 'next_arg')); } - pop(@code) if $code[$#code]->[1][0] eq 'next_arg'; + pop(@code) if is_instr($code[$#code], 'next_arg'); } - push(@code, &make_op('', 'end')); + push(@code, make_op('', 'end')) + unless is_instr($code[$#code], 'call_end'); # # Chain together all codes segments having the same first operation. @@ -1575,11 +1659,20 @@ sub tr_gen_to { $min_window{$key} = $min_window if $min_window{$key} > $min_window; - pop(@{$gen_transform{$key}}) + my $prev_last; + $prev_last = pop(@{$gen_transform{$key}}) if defined @{$gen_transform{$key}}; # Fail - my(@prefix) = (&make_op($comment), &make_op('', 'try_me_else', &tr_code_len(@code))); - unshift(@code, @prefix); - push(@{$gen_transform{$key}}, @code, &make_op('', 'fail')); + + if ($prev_last && !is_instr($prev_last, 'fail')) { + error("Line $line: A previous transformation shadows '$orig_transform'"); + } + unless ($cannot_fail) { + unshift(@code, make_op('', 'try_me_else', + tr_code_len(@code))); + push(@code, make_op(""), make_op("$key", 'fail')); + } + unshift(@code, make_op($comment)); + push(@{$gen_transform{$key}}, @code), } sub tr_code_len { @@ -1597,21 +1690,38 @@ sub make_op { [scalar(@op), [@op], $comment]; } -sub is_set_var_instr { - my($ref) = @_; +sub is_instr { + my($ref,$op) = @_; return 0 unless ref($ref) eq 'ARRAY'; - $ref->[1][0] eq 'set_var'; + $ref->[1][0] eq $op; +} + +sub get_comment { + my($ref,$op) = @_; + return '' unless ref($ref) eq 'ARRAY'; + $ref->[2]; +} + +sub tr_next_index { + my($lref,$href,$name,@args) = @_; + my $code = "RVAL = $name(" . join(', ', 'st', @args) . "); break;\n"; + my $index; + + if (defined $$href{$code}) { + $index = $$href{$code}; + } else { + $index = scalar(@$lref); + push(@$lref, $code); + $$href{$code} = $index; + } + $index; } sub tr_gen_call { my(@call_table) = @_; my($i); - print "\n"; for ($i = 0; $i < @call_table; $i++) { - my $ref = $call_table[$i]; - my($name, @args) = @$ref; - print "case $i: RVAL = $name(", join(', ', 'st', @args), "); break;\n"; + print "case $i: $call_table[$i]"; } - print "\n"; } diff --git a/erts/emulator/utils/make_preload b/erts/emulator/utils/make_preload index d0671e998d..d22f08f993 100755 --- a/erts/emulator/utils/make_preload +++ b/erts/emulator/utils/make_preload @@ -88,6 +88,7 @@ foreach $file (@ARGV) { print "unsigned char preloaded_$module", "[] = {\n"; for ($i = 0; $i < length($_); $i++) { if ($i % 8 == 0 && $comment ne '') { + $comment =~ s@/\*@..@g; # Comment start -- avoid warning. $comment =~ s@\*/@..@g; # Comment terminator. print " /* $comment */\n "; $comment = ''; diff --git a/erts/emulator/utils/make_tables b/erts/emulator/utils/make_tables index 918ef62094..91efb4c023 100755 --- a/erts/emulator/utils/make_tables +++ b/erts/emulator/utils/make_tables @@ -181,8 +181,7 @@ for ($i = 0; $i < @bif; $i++) { print "\n"; for ($i = 0; $i < @bif; $i++) { - my $arity = $bif[$i]->[2]; - my $args = join(', ', 'Process*', ('Eterm') x $arity); + my $args = join(', ', 'Process*', 'Eterm*'); print "Eterm $bif[$i]->[3]($args);\n"; print "Eterm wrap_$bif[$i]->[3]($args, UWord *I);\n"; } @@ -219,28 +218,10 @@ for ($i = 0; $i < @bif; $i++) { next if $bif[$i]->[3] eq $bif[$i]->[4]; # Skip unwrapped bifs my $arity = $bif[$i]->[2]; my $func = $bif[$i]->[3]; - my $arg; print "Eterm\n"; - print "wrap_$func(Process* p"; - for ($arg = 1; $arg <= $arity; $arg++) { - print ", Eterm arg$arg"; - } - print ", UWord *I)\n"; + print "wrap_$func(Process* p, Eterm* args, UWord* I)\n"; print "{\n"; - print " return erts_bif_trace($i, p"; - for ($arg = 1; $arg <= 3; $arg++) { - if ($arg <= $arity) { - print ", arg$arg"; - #} elsif ($arg == ($arity + 1)) { - # # Place I in correct position - # print ", (Eterm) I"; - } else { - print ", 0"; - } - } - # I is always last, as well as in the correct position - # Note that "last" and "correct position" may be the same... - print ", I);\n"; + print " return erts_bif_trace($i, p, args, I);\n"; print "}\n\n"; } @@ -261,19 +242,9 @@ for ($i = 0; $i < @bif; $i++) { my $orig_func = $1; $orig_func = $implementation[$i] if $implementation[$i]; print "Eterm\n"; - print "$func(Process* p"; - for ($arg = 1; $arg <= $arity; $arg++) { - print ", Eterm arg$arg"; - } - print ")\n"; + print "$func(Process* p, Eterm* BIF__ARGS)\n"; print "{\n"; - print " return $orig_func(p"; - for ($arg = 1; $arg <= 3; $arg++) { - if ($arg <= $arity) { - print ", arg$arg"; - } - } - print ");\n"; + print " return $orig_func(p, BIF__ARGS);\n"; print "}\n\n"; } diff --git a/erts/etc/common/erlexec.c b/erts/etc/common/erlexec.c index 2bd576d8e8..8d119984df 100644 --- a/erts/etc/common/erlexec.c +++ b/erts/etc/common/erlexec.c @@ -67,6 +67,7 @@ static const char plusM_au_allocs[]= { 'C', /* sbmbc_alloc */ 'D', /* std_alloc */ 'E', /* ets_alloc */ + 'F', /* fix_alloc */ 'H', /* eheap_alloc */ 'L', /* ll_alloc */ 'R', /* driver_alloc */ @@ -110,8 +111,6 @@ static char *plusM_other_switches[] = { "Mamcbf", "Mrmcbf", "Mmcs", - "Mcci", - "Fe", "Ye", "Ym", "Ytp", diff --git a/erts/etc/win32/nsis/erlang_uninst.ico b/erts/etc/win32/nsis/erlang_uninst.ico Binary files differindex edbd8a6f2c..edbd8a6f2c 100755..100644 --- a/erts/etc/win32/nsis/erlang_uninst.ico +++ b/erts/etc/win32/nsis/erlang_uninst.ico diff --git a/erts/include/internal/ethr_atomics.h b/erts/include/internal/ethr_atomics.h index 1caf4d0567..0f3c26f1df 100644 --- a/erts/include/internal/ethr_atomics.h +++ b/erts/include/internal/ethr_atomics.h @@ -1,7 +1,16 @@ /* + * --------------- DO NOT EDIT THIS FILE! --------------- + * This file was automatically generated by the + * $ERL_TOP/erts/lib_src/utils/make_atomics_api script. + * If you need to make changes, edit the script and + * regenerate this file. + * --------------- DO NOT EDIT THIS FILE! --------------- + */ + +/* * %CopyrightBegin% * - * Copyright Ericsson AB 2010. All Rights Reserved. + * Copyright Ericsson AB 2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -18,709 +27,8818 @@ */ /* - * Description: The ethread atomic API + * Description: The ethread atomics API * Author: Rickard Green */ -#ifndef ETHR_ATOMIC_H__ -#define ETHR_ATOMIC_H__ +/* + * This file maps native atomic implementations to ethread + * API atomics. If no native atomic implementation + * is available, a less efficient fallback is used instead. + * The API consists of 32-bit size, word size (pointer size), + * and double word size atomics. + * + * The following atomic operations are implemented for + * 32-bit size, and word size atomics: + * - cmpxchg + * - xchg + * - set + * - init + * - add_read + * - read + * - inc_read + * - dec_read + * - add + * - inc + * - dec + * - read_band + * - read_bor + * + * The following atomic operations are implemented for + * double word size atomics: + * - cmpxchg + * - set + * - read + * - init + * + * Appart from a function implementing the atomic operation + * with unspecified memory barrier semantics, there are + * functions implementing each operation with the following + * memory barrier semantics: + * - rb (read barrier) + * - wb (write barrier) + * - acqb (acquire barrier) + * - relb (release barrier) + * - mb (full memory barrier) + * + * We implement all of these operation/barrier + * combinations, regardless of whether they are useful + * or not (some of them are useless). + * + * Double word size atomic functions are on the followning + * form: + * ethr_dw_atomic_<OP>[_<BARRIER>] + * + * Word size atomic functions are on the followning + * form: + * ethr_atomic_<OP>[_<BARRIER>] + * + * 32-bit size atomic functions are on the followning + * form: + * ethr_atomic32_<OP>[_<BARRIER>] + * + * Apart from the operation/barrier functions + * described above also 'addr' functions are implemented + * which return the actual memory address used of the + * atomic variable. The 'addr' functions have no barrier + * versions. + * + * The native atomic implementation does not need to + * implement all operation/barrier combinations. + * Functions that have no native implementation will be + * constructed from existing native functionality. These + * functions will perform the wanted operation and will + * produce sufficient memory barriers, but may + * in some cases be less efficient than pure native + * versions. + * + * When we create ethread API operation/barrier functions by + * adding barriers before and after native operations it is + * assumed that: + * - A native read operation begins, and ends with a load. + * - A native set operation begins, and ends with a store. + * - An init operation begins with either a load, or a store, + * and ends with either a load, or a store. + * - All other operations begins with a load, and ends with + * either a load, or a store. + * + * This is the minimum functionality that a native + * implementation needs to provide: + * + * - Functions that need to be implemented: + * + * - ethr_native_[dw_|su_dw_]atomic[BITS]_addr + * - ethr_native_[dw_|su_dw_]atomic[BITS]_cmpxchg[_<BARRIER>] + * (at least one cmpxchg of optional barrier) + * + * - Macros that needs to be defined: + * + * A macro informing about the presence of the native + * implementation: + * + * - ETHR_HAVE_NATIVE_[DW_|SU_DW_]ATOMIC[BITS] + * + * A macro naming (a string constant) the implementation: + * + * - ETHR_NATIVE_[DW_]ATOMIC[BITS]_IMPL + * + * Each implemented native atomic function has to + * be accompanied by a defined macro on the following + * form informing about its presence: + * + * - ETHR_HAVE_ETHR_NATIVE_[DW_|SU_DW_]ATOMIC[BITS]_<OP>[_<BARRIER>] + * + * A (sparc-v9 style) membar macro: + * + * - ETHR_MEMBAR(B) + * + * Which takes a combination of the following macros + * or:ed (using |) together: + * + * - ETHR_LoadLoad + * - ETHR_LoadStore + * - ETHR_StoreLoad + * - ETHR_StoreStore + * + */ -#if !defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) -# define ETHR_NEED_ATOMIC_PROTOTYPES__ -#endif +#ifndef ETHR_ATOMICS_H__ +#define ETHR_ATOMICS_H__ -#ifndef ETHR_HAVE_NATIVE_ATOMICS +#undef ETHR_AMC_FALLBACK__ +#undef ETHR_AMC_NO_ATMCS__ +#undef ETHR_AMC_ATMC_T__ +#undef ETHR_AMC_ATMC_FUNC__ + +/* -- 32-bit atomics -- */ + +#undef ETHR_NAINT32_T__ +#undef ETHR_NATMC32_FUNC__ +#undef ETHR_NATMC32_ADDR_FUNC__ +#undef ETHR_NATMC32_BITS__ +#if defined(ETHR_HAVE_NATIVE_ATOMIC32) +# define ETHR_NEED_NATMC32_ADDR +# define ETHR_NATMC32_ADDR_FUNC__ ethr_native_atomic32_addr +typedef ethr_native_atomic32_t ethr_atomic32_t; +# define ETHR_NAINT32_T__ ethr_sint32_t +# define ETHR_NATMC32_FUNC__(X) ethr_native_atomic32_ ## X +# define ETHR_NATMC32_BITS__ 32 +#elif defined(ETHR_HAVE_NATIVE_ATOMIC64) +# define ETHR_NEED_NATMC64_ADDR +#ifdef ETHR_BIGENDIAN +# define ETHR_NATMC32_ADDR_FUNC__(VAR) \ + (((ethr_sint32_t *) ethr_native_atomic64_addr((VAR))) + 1) +#else +# define ETHR_NATMC32_ADDR_FUNC__(VAR) \ + ((ethr_sint32_t *) ethr_native_atomic64_addr((VAR))) +#endif +typedef ethr_native_atomic64_t ethr_atomic32_t; +# define ETHR_NAINT32_T__ ethr_sint64_t +# define ETHR_NATMC32_FUNC__(X) ethr_native_atomic64_ ## X +# define ETHR_NATMC32_BITS__ 64 +#else /* - * No native atomic implementation available. :( + * No native atomics usable for 32-bits atomics :( * Use fallback... */ typedef ethr_sint32_t ethr_atomic32_t; -typedef ethr_sint_t ethr_atomic_t; -#else -/* - * Map ethread native atomics to ethread API atomics. - * - * We do at least have a native atomic implementation that - * can handle integers of a size larger than or equal to - * the size of pointers. - */ +#endif -/* -- Pointer size atomics -- */ +#undef ETHR_ATMC32_INLINE__ +#ifdef ETHR_NATMC32_BITS__ +# ifdef ETHR_TRY_INLINE_FUNCS +# define ETHR_ATMC32_INLINE__ +# endif +# define ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS +#endif + +#if !defined(ETHR_ATMC32_INLINE__) || defined(ETHR_ATOMIC_IMPL__) +# define ETHR_NEED_ATMC32_PROTOTYPES__ +#endif + +#ifndef ETHR_INLINE_ATMC32_FUNC_NAME_ +# define ETHR_INLINE_ATMC32_FUNC_NAME_(X) X +#endif + +#undef ETHR_ATMC32_FUNC__ +#define ETHR_ATMC32_FUNC__(X) ETHR_INLINE_ATMC32_FUNC_NAME_(ethr_atomic32_ ## X) + + +/* -- Word size atomics -- */ + +#undef ETHR_NEED_NATMC32_ADDR +#undef ETHR_NEED_NATMC64_ADDR #undef ETHR_NAINT_T__ #undef ETHR_NATMC_FUNC__ #undef ETHR_NATMC_ADDR_FUNC__ -#if ETHR_SIZEOF_PTR == 8 -# if defined(ETHR_HAVE_NATIVE_ATOMIC64) -# define ETHR_NATMC_ADDR_FUNC__ ethr_native_atomic64_addr +#undef ETHR_NATMC_BITS__ +#if ETHR_SIZEOF_PTR == 8 && defined(ETHR_HAVE_NATIVE_ATOMIC64) +# ifndef ETHR_NEED_NATMC64_ADDR +# define ETHR_NEED_NATMC64_ADDR +# endif +# define ETHR_NATMC_ADDR_FUNC__ ethr_native_atomic64_addr typedef ethr_native_atomic64_t ethr_atomic_t; -# define ETHR_NAINT_T__ ethr_sint64_t -# define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X -# else -# error "Missing native atomic implementation" +# define ETHR_NAINT_T__ ethr_sint64_t +# define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X +# define ETHR_NATMC_BITS__ 64 +#elif ETHR_SIZEOF_PTR == 4 && defined(ETHR_HAVE_NATIVE_ATOMIC32) +# ifndef ETHR_NEED_NATMC64_ADDR +# define ETHR_NEED_NATMC32_ADDR # endif -#elif ETHR_SIZEOF_PTR == 4 # define ETHR_NATMC_ADDR_FUNC__ ethr_native_atomic32_addr -# ifdef ETHR_HAVE_NATIVE_ATOMIC32 typedef ethr_native_atomic32_t ethr_atomic_t; -# define ETHR_NAINT_T__ ethr_sint32_t -# define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X -# elif defined(ETHR_HAVE_NATIVE_ATOMIC64) +# define ETHR_NAINT_T__ ethr_sint32_t +# define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X +# define ETHR_NATMC_BITS__ 32 +#elif ETHR_SIZEOF_PTR == 4 && defined(ETHR_HAVE_NATIVE_ATOMIC64) +# ifndef ETHR_NEED_NATMC64_ADDR +# define ETHR_NEED_NATMC64_ADDR +# endif +#ifdef ETHR_BIGENDIAN +# define ETHR_NATMC_ADDR_FUNC__(VAR) \ + (((ethr_sint32_t *) ethr_native_atomic64_addr((VAR))) + 1) +#else +# define ETHR_NATMC_ADDR_FUNC__(VAR) \ + ((ethr_sint32_t *) ethr_native_atomic64_addr((VAR))) +#endif typedef ethr_native_atomic64_t ethr_atomic_t; -# define ETHR_NATMC_T__ ethr_native_atomic64_t -# define ETHR_NAINT_T__ ethr_sint64_t -# define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X +# define ETHR_NATMC_T__ ethr_native_atomic64_t +# define ETHR_NAINT_T__ ethr_sint64_t +# define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X +# define ETHR_NATMC_BITS__ 64 +#else +/* + * No native atomics usable for pointer size atomics :( + * Use fallback... + */ + +# if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) +# define ETHR_AMC_FALLBACK__ +# define ETHR_AMC_NO_ATMCS__ 2 +# define ETHR_AMC_SINT_T__ ethr_sint32_t +# define ETHR_AMC_ATMC_T__ ethr_atomic32_t +# define ETHR_AMC_ATMC_FUNC__(X) ETHR_INLINE_ATMC32_FUNC_NAME_(ethr_atomic32_ ## X) +typedef struct { + ETHR_AMC_ATMC_T__ atomic[ETHR_AMC_NO_ATMCS__]; +} ethr_amc_t; +typedef struct { + ethr_amc_t amc; + ethr_sint_t sint; +} ethr_atomic_t; +# else /* locked fallback */ +typedef ethr_sint_t ethr_atomic_t; +# endif +#endif + +#undef ETHR_ATMC_INLINE__ +#ifdef ETHR_NATMC_BITS__ +# ifdef ETHR_TRY_INLINE_FUNCS +# define ETHR_ATMC_INLINE__ +# endif +# define ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS +#endif + +#if !defined(ETHR_ATMC_INLINE__) || defined(ETHR_ATOMIC_IMPL__) +# define ETHR_NEED_ATMC_PROTOTYPES__ +#endif + +#ifndef ETHR_INLINE_ATMC_FUNC_NAME_ +# define ETHR_INLINE_ATMC_FUNC_NAME_(X) X +#endif + +#undef ETHR_ATMC_FUNC__ +#define ETHR_ATMC_FUNC__(X) ETHR_INLINE_ATMC_FUNC_NAME_(ethr_atomic_ ## X) + +/* -- Double word atomics -- */ + +#undef ETHR_SU_DW_NAINT_T__ +#undef ETHR_SU_DW_NATMC_FUNC__ +#undef ETHR_SU_DW_NATMC_ADDR_FUNC__ +#undef ETHR_DW_NATMC_FUNC__ +#undef ETHR_DW_NATMC_ADDR_FUNC__ +#undef ETHR_DW_NATMC_BITS__ +#if defined(ETHR_HAVE_NATIVE_DW_ATOMIC) || defined(ETHR_HAVE_NATIVE_SU_DW_ATOMIC) +# define ETHR_NEED_DW_NATMC_ADDR +# define ETHR_DW_NATMC_ADDR_FUNC__ ethr_native_dw_atomic_addr +# define ETHR_NATIVE_DW_ATOMIC_T__ ethr_native_dw_atomic_t +# define ETHR_DW_NATMC_FUNC__(X) ethr_native_dw_atomic_ ## X +# define ETHR_SU_DW_NATMC_FUNC__(X) ethr_native_su_dw_atomic_ ## X +# if ETHR_SIZEOF_PTR == 8 +# define ETHR_DW_NATMC_BITS__ 128 +# elif ETHR_SIZEOF_PTR == 4 +# define ETHR_DW_NATMC_BITS__ 64 # else -# error "Missing native atomic implementation" +# error "Word size not supported" +# endif +# ifdef ETHR_NATIVE_SU_DW_SINT_T +# define ETHR_SU_DW_NAINT_T__ ETHR_NATIVE_SU_DW_SINT_T # endif +#elif ETHR_SIZEOF_PTR == 4 && defined(ETHR_HAVE_NATIVE_ATOMIC64) +# define ETHR_HAVE_NATIVE_SU_DW_ATOMIC +# ifndef ETHR_NEED_NATMC64_ADDR +# define ETHR_NEED_NATMC64_ADDR +# endif +# define ETHR_DW_NATMC_ADDR_FUNC__(VAR) \ + ((ethr_dw_sint_t *) ethr_native_atomic64_addr((VAR))) +# define ETHR_NATIVE_DW_ATOMIC_T__ ethr_native_atomic64_t +# define ETHR_SU_DW_NAINT_T__ ethr_sint64_t +# define ETHR_SU_DW_NATMC_FUNC__(X) ethr_native_atomic64_ ## X +# define ETHR_DW_NATMC_BITS__ 64 #endif -/* -- 32-bit atomics -- */ +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) +#define ETHR_DW_ATOMIC_FUNC__(X) ethr_dw_atomic_ ## X ## _fallback__ +#else +#define ETHR_DW_ATOMIC_FUNC__(X) ethr_dw_atomic_ ## X +#endif -#undef ETHR_NAINT32_T__ -#undef ETHR_NATMC32_FUNC__ -#if defined(ETHR_HAVE_NATIVE_ATOMIC32) -typedef ethr_native_atomic32_t ethr_atomic32_t; -# define ETHR_NAINT32_T__ ethr_sint32_t -# define ETHR_NATMC32_FUNC__(X) ethr_native_atomic32_ ## X -#elif defined(ETHR_HAVE_NATIVE_ATOMIC64) -typedef ethr_native_atomic64_t ethr_atomic32_t; -# define ETHR_NAINT32_T__ ethr_sint64_t -# define ETHR_NATMC32_FUNC__(X) ethr_native_atomic64_ ## X +#if !defined(ETHR_DW_NATMC_BITS__) || defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) +# define ETHR_NEED_DW_FALLBACK__ +#endif + +#if defined(ETHR_NEED_DW_FALLBACK__) +/* + * No native atomics usable for double word atomics :( + * Use fallback... + */ + +# ifndef ETHR_AMC_FALLBACK__ +# if ETHR_SIZEOF_PTR == 8 && defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) +# define ETHR_AMC_FALLBACK__ +# define ETHR_AMC_NO_ATMCS__ 1 +# define ETHR_AMC_SINT_T__ ethr_sint_t +# define ETHR_AMC_ATMC_T__ ethr_atomic_t +# define ETHR_AMC_ATMC_FUNC__(X) ETHR_INLINE_ATMC_FUNC_NAME_(ethr_atomic_ ## X) +# elif defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) +# define ETHR_AMC_FALLBACK__ +# define ETHR_AMC_NO_ATMCS__ 2 +# define ETHR_AMC_SINT_T__ ethr_sint32_t +# define ETHR_AMC_ATMC_T__ ethr_atomic32_t +# define ETHR_AMC_ATMC_FUNC__(X) ETHR_INLINE_ATMC32_FUNC_NAME_(ethr_atomic32_ ## X) +# endif +# ifdef ETHR_AMC_FALLBACK__ +typedef struct { + ETHR_AMC_ATMC_T__ atomic[ETHR_AMC_NO_ATMCS__]; +} ethr_amc_t; +# endif +# endif + +typedef struct { +#ifdef ETHR_AMC_FALLBACK__ + ethr_amc_t amc; +#endif + ethr_sint_t sint[2]; +} ethr_dw_atomic_fallback_t; + +#endif + +typedef union { +#ifdef ETHR_NATIVE_DW_ATOMIC_T__ + ETHR_NATIVE_DW_ATOMIC_T__ native; +#endif +#ifdef ETHR_NEED_DW_FALLBACK__ + ethr_dw_atomic_fallback_t fallback; +#endif + ethr_sint_t sint[2]; +} ethr_dw_atomic_t; + +typedef union { +#ifdef ETHR_SU_DW_NAINT_T__ + ETHR_SU_DW_NAINT_T__ dw_sint; +#endif + ethr_sint_t sint[2]; +} ethr_dw_sint_t; + +#ifdef ETHR_BIGENDIAN +# define ETHR_DW_SINT_LOW_WORD 1 +# define ETHR_DW_SINT_HIGH_WORD 0 #else -# error "Missing native atomic implementation" +# define ETHR_DW_SINT_LOW_WORD 0 +# define ETHR_DW_SINT_HIGH_WORD 1 #endif +#undef ETHR_DW_ATMC_INLINE__ +#ifdef ETHR_DW_NATMC_BITS__ +# ifdef ETHR_TRY_INLINE_FUNCS +# define ETHR_ATMC32_INLINE__ +# endif +# define ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS #endif -#ifdef ETHR_NEED_ATOMIC_PROTOTYPES__ -ethr_sint_t *ethr_atomic_addr(ethr_atomic_t *); -void ethr_atomic_init(ethr_atomic_t *, ethr_sint_t); -void ethr_atomic_set(ethr_atomic_t *, ethr_sint_t); -ethr_sint_t ethr_atomic_read(ethr_atomic_t *); -ethr_sint_t ethr_atomic_inc_read(ethr_atomic_t *); -ethr_sint_t ethr_atomic_dec_read(ethr_atomic_t *); -void ethr_atomic_inc(ethr_atomic_t *); -void ethr_atomic_dec(ethr_atomic_t *); -ethr_sint_t ethr_atomic_add_read(ethr_atomic_t *, ethr_sint_t); -void ethr_atomic_add(ethr_atomic_t *, ethr_sint_t); -ethr_sint_t ethr_atomic_read_band(ethr_atomic_t *, ethr_sint_t); -ethr_sint_t ethr_atomic_read_bor(ethr_atomic_t *, ethr_sint_t); -ethr_sint_t ethr_atomic_xchg(ethr_atomic_t *, ethr_sint_t); -ethr_sint_t ethr_atomic_cmpxchg(ethr_atomic_t *, ethr_sint_t, ethr_sint_t); -ethr_sint_t ethr_atomic_read_acqb(ethr_atomic_t *); -ethr_sint_t ethr_atomic_inc_read_acqb(ethr_atomic_t *); -void ethr_atomic_set_relb(ethr_atomic_t *, ethr_sint_t); -void ethr_atomic_dec_relb(ethr_atomic_t *); -ethr_sint_t ethr_atomic_dec_read_relb(ethr_atomic_t *); -ethr_sint_t ethr_atomic_cmpxchg_acqb(ethr_atomic_t *, ethr_sint_t, ethr_sint_t); -ethr_sint_t ethr_atomic_cmpxchg_relb(ethr_atomic_t *, ethr_sint_t, ethr_sint_t); - -ethr_sint32_t *ethr_atomic32_addr(ethr_atomic32_t *); -void ethr_atomic32_init(ethr_atomic32_t *, ethr_sint32_t); -void ethr_atomic32_set(ethr_atomic32_t *, ethr_sint32_t); -ethr_sint32_t ethr_atomic32_read(ethr_atomic32_t *); -ethr_sint32_t ethr_atomic32_inc_read(ethr_atomic32_t *); -ethr_sint32_t ethr_atomic32_dec_read(ethr_atomic32_t *); -void ethr_atomic32_inc(ethr_atomic32_t *); -void ethr_atomic32_dec(ethr_atomic32_t *); -ethr_sint32_t ethr_atomic32_add_read(ethr_atomic32_t *, ethr_sint32_t); -void ethr_atomic32_add(ethr_atomic32_t *, ethr_sint32_t); -ethr_sint32_t ethr_atomic32_read_band(ethr_atomic32_t *, ethr_sint32_t); -ethr_sint32_t ethr_atomic32_read_bor(ethr_atomic32_t *, ethr_sint32_t); -ethr_sint32_t ethr_atomic32_xchg(ethr_atomic32_t *, ethr_sint32_t); -ethr_sint32_t ethr_atomic32_cmpxchg(ethr_atomic32_t *, - ethr_sint32_t, - ethr_sint32_t); -ethr_sint32_t ethr_atomic32_read_acqb(ethr_atomic32_t *); -ethr_sint32_t ethr_atomic32_inc_read_acqb(ethr_atomic32_t *); -void ethr_atomic32_set_relb(ethr_atomic32_t *, ethr_sint32_t); -void ethr_atomic32_dec_relb(ethr_atomic32_t *); -ethr_sint32_t ethr_atomic32_dec_read_relb(ethr_atomic32_t *); -ethr_sint32_t ethr_atomic32_cmpxchg_acqb(ethr_atomic32_t *, - ethr_sint32_t, - ethr_sint32_t); -ethr_sint32_t ethr_atomic32_cmpxchg_relb(ethr_atomic32_t *, - ethr_sint32_t, - ethr_sint32_t); +#if !defined(ETHR_DW_ATMC_INLINE__) || defined(ETHR_ATOMIC_IMPL__) +# define ETHR_NEED_DW_ATMC_PROTOTYPES__ #endif -int ethr_init_atomics(void); +#ifndef ETHR_INLINE_DW_ATMC_FUNC_NAME_ +# define ETHR_INLINE_DW_ATMC_FUNC_NAME_(X) X +#endif + +#undef ETHR_DW_ATMC_FUNC__ +#define ETHR_DW_ATMC_FUNC__(X) ETHR_INLINE_DW_ATMC_FUNC_NAME_(ethr_dw_atomic_ ## X) + +#if defined(ETHR_NEED_DW_ATMC_PROTOTYPES__) +int ethr_have_native_dw_atomic(void); +#endif +#if defined(ETHR_DW_ATMC_INLINE__) || defined(ETHR_ATOMIC_IMPL__) +static ETHR_INLINE int +ETHR_INLINE_DW_ATMC_FUNC_NAME_(ethr_have_native_dw_atomic)(void) +{ +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + return ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__; +#elif defined(ETHR_DW_NATMC_BITS__) + return 1; +#else + return 0; +#endif +} +#endif + +/* -- Misc -- */ #if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) +/* + * Unusual values are used by read() fallbacks implemented via cmpxchg(). + * We want to use an unusual value in hope that it is more efficient + * not to match the value in memory. + * + * - Negative integer values are probably more unusual. + * - Very large absolute integer values are probably more unusual. + * - Odd pointers are probably more unusual (only char pointers can be odd). + */ +# define ETHR_UNUSUAL_SINT32_VAL__ ((ethr_sint32_t) 0x81818181) +# if ETHR_SIZEOF_PTR == 4 +# define ETHR_UNUSUAL_SINT_VAL__ ((ethr_sint_t) ETHR_UNUSUAL_SINT32_VAL__) +# elif ETHR_SIZEOF_PTR == 8 +# define ETHR_UNUSUAL_SINT_VAL__ ((ethr_sint_t) 0x8181818181818181L) +# else +# error "Word size not supported" +# endif +# if defined(ETHR_NEED_DW_NATMC_ADDR) && !defined(ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_ADDR) +# error "No ethr_native_dw_atomic_addr() available" +# endif +# if defined(ETHR_NEED_NATMC32_ADDR) && !defined(ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADDR) +# error "No ethr_native_atomic32_addr() available" +# endif +# if defined(ETHR_NEED_NATMC64_ADDR) && !defined(ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADDR) +# error "No ethr_native_atomic64_addr() available" +# endif +#endif + +#if defined(__GNUC__) +# ifndef ETHR_COMPILER_BARRIER +# define ETHR_COMPILER_BARRIER __asm__ __volatile__("" : : : "memory") +# endif +#elif defined(ETHR_WIN32_THREADS) +# ifndef ETHR_COMPILER_BARRIER +# include <intrin.h> +# pragma intrinsic(_ReadWriteBarrier) +# define ETHR_COMPILER_BARRIER _ReadWriteBarrier() +# endif +#endif -#ifndef ETHR_HAVE_NATIVE_ATOMICS +void ethr_compiler_barrier_fallback(void); +#ifndef ETHR_COMPILER_BARRIER +# define ETHR_COMPILER_BARRIER ethr_compiler_barrier_fallback() +#endif + +int ethr_init_atomics(void); + +/* info */ +char **ethr_native_atomic32_ops(void); +char **ethr_native_atomic64_ops(void); +char **ethr_native_dw_atomic_ops(void); +char **ethr_native_su_dw_atomic_ops(void); + +#if !defined(ETHR_DW_NATMC_BITS__) && !defined(ETHR_NATMC_BITS__) && !defined(ETHR_NATMC32_BITS__) /* - * Fallbacks for atomics used in absence of a native implementation. + * ETHR_*MEMORY_BARRIER orders between locked and atomic accesses only, + * i.e. when no native atomic implementation exist and only our lock + * based atomic fallback is used, a noop is sufficient. */ +# undef ETHR_MEMORY_BARRIER +# undef ETHR_WRITE_MEMORY_BARRIER +# undef ETHR_READ_MEMORY_BARRIER +# undef ETHR_READ_DEPEND_MEMORY_BARRIER +# undef ETHR_MEMBAR +# define ETHR_MEMBAR(B) do { } while (0) +#endif + +#ifndef ETHR_MEMBAR +# error "No ETHR_MEMBAR defined" +#endif -#define ETHR_ATOMIC_ADDR_BITS 10 -#define ETHR_ATOMIC_ADDR_SHIFT 6 +#define ETHR_MEMORY_BARRIER ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore) +#define ETHR_WRITE_MEMORY_BARRIER ETHR_MEMBAR(ETHR_StoreStore) +#define ETHR_READ_MEMORY_BARRIER ETHR_MEMBAR(ETHR_LoadLoad) +#ifdef ETHR_READ_DEPEND_MEMORY_BARRIER +# undef ETHR_ORDERED_READ_DEPEND +#else +# define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_COMPILER_BARRIER +# define ETHR_ORDERED_READ_DEPEND +#endif -typedef struct { - union { - ethr_spinlock_t lck; - char buf[ETHR_CACHE_LINE_SIZE]; - } u; -} ethr_atomic_protection_t; -extern ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATOMIC_ADDR_BITS]; +/* ---------- Double word size atomic implementation ---------- */ + + +#ifdef ETHR_NEED_DW_ATMC_PROTOTYPES__ +ethr_sint_t *ethr_dw_atomic_addr(ethr_dw_atomic_t *var); +int ethr_dw_atomic_cmpxchg(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val); +int ethr_dw_atomic_cmpxchg_rb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val); +int ethr_dw_atomic_cmpxchg_wb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val); +int ethr_dw_atomic_cmpxchg_acqb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val); +int ethr_dw_atomic_cmpxchg_relb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val); +int ethr_dw_atomic_cmpxchg_mb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val); +void ethr_dw_atomic_set(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ethr_dw_atomic_set_rb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ethr_dw_atomic_set_wb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ethr_dw_atomic_set_acqb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ethr_dw_atomic_set_relb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ethr_dw_atomic_set_mb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ethr_dw_atomic_read(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ethr_dw_atomic_read_rb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ethr_dw_atomic_read_wb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ethr_dw_atomic_read_acqb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ethr_dw_atomic_read_relb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ethr_dw_atomic_read_mb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ethr_dw_atomic_init(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ethr_dw_atomic_init_rb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ethr_dw_atomic_init_wb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ethr_dw_atomic_init_acqb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ethr_dw_atomic_init_relb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ethr_dw_atomic_init_mb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) +ethr_sint_t *ETHR_DW_ATOMIC_FUNC__(addr)(ethr_dw_atomic_t *var); +int ETHR_DW_ATOMIC_FUNC__(cmpxchg)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val); +int ETHR_DW_ATOMIC_FUNC__(cmpxchg_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val); +int ETHR_DW_ATOMIC_FUNC__(cmpxchg_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val); +int ETHR_DW_ATOMIC_FUNC__(cmpxchg_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val); +int ETHR_DW_ATOMIC_FUNC__(cmpxchg_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val); +int ETHR_DW_ATOMIC_FUNC__(cmpxchg_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val); +void ETHR_DW_ATOMIC_FUNC__(set)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ETHR_DW_ATOMIC_FUNC__(set_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ETHR_DW_ATOMIC_FUNC__(set_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ETHR_DW_ATOMIC_FUNC__(set_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ETHR_DW_ATOMIC_FUNC__(set_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ETHR_DW_ATOMIC_FUNC__(set_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ETHR_DW_ATOMIC_FUNC__(read)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ETHR_DW_ATOMIC_FUNC__(read_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ETHR_DW_ATOMIC_FUNC__(read_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ETHR_DW_ATOMIC_FUNC__(read_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ETHR_DW_ATOMIC_FUNC__(read_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ETHR_DW_ATOMIC_FUNC__(read_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ETHR_DW_ATOMIC_FUNC__(init)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ETHR_DW_ATOMIC_FUNC__(init_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ETHR_DW_ATOMIC_FUNC__(init_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ETHR_DW_ATOMIC_FUNC__(init_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ETHR_DW_ATOMIC_FUNC__(init_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ETHR_DW_ATOMIC_FUNC__(init_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +#endif +#endif /* ETHR_NEED_DW_ATMC_PROTOTYPES__ */ + +#if (defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) \ + && (defined(ETHR_DW_ATMC_INLINE__) || defined(ETHR_ATOMIC_IMPL__))) + +#if !defined(ETHR_DW_NATMC_BITS__) +# error "Missing native atomic implementation" +#elif defined(ETHR_HAVE_NATIVE_DW_ATOMIC) || defined(ETHR_HAVE_NATIVE_SU_DW_ATOMIC) +# undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG +# undef ETHR_HAVE_DW_NATMC_CMPXCHG +# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG +# define ETHR_HAVE_DW_NATMC_CMPXCHG 1 +# endif +# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG +# define ETHR_HAVE_SU_DW_NATMC_CMPXCHG 1 +# endif +# undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB +# undef ETHR_HAVE_DW_NATMC_CMPXCHG_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_RB +# define ETHR_HAVE_DW_NATMC_CMPXCHG_RB 1 +# endif +# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_RB +# define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB 1 +# endif +# undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB +# undef ETHR_HAVE_DW_NATMC_CMPXCHG_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_WB +# define ETHR_HAVE_DW_NATMC_CMPXCHG_WB 1 +# endif +# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_WB +# define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB 1 +# endif +# undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB +# undef ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_ACQB +# define ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB 1 +# endif +# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_ACQB +# define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB 1 +# endif +# undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB +# undef ETHR_HAVE_DW_NATMC_CMPXCHG_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_RELB +# define ETHR_HAVE_DW_NATMC_CMPXCHG_RELB 1 +# endif +# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_RELB +# define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB 1 +# endif +# undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB +# undef ETHR_HAVE_DW_NATMC_CMPXCHG_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_MB +# define ETHR_HAVE_DW_NATMC_CMPXCHG_MB 1 +# endif +# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_MB +# define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB 1 +# endif +# undef ETHR_HAVE_SU_DW_NATMC_SET +# undef ETHR_HAVE_DW_NATMC_SET +# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET +# define ETHR_HAVE_DW_NATMC_SET 1 +# endif +# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET +# define ETHR_HAVE_SU_DW_NATMC_SET 1 +# endif +# undef ETHR_HAVE_SU_DW_NATMC_SET_RB +# undef ETHR_HAVE_DW_NATMC_SET_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_RB +# define ETHR_HAVE_DW_NATMC_SET_RB 1 +# endif +# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_RB +# define ETHR_HAVE_SU_DW_NATMC_SET_RB 1 +# endif +# undef ETHR_HAVE_SU_DW_NATMC_SET_WB +# undef ETHR_HAVE_DW_NATMC_SET_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_WB +# define ETHR_HAVE_DW_NATMC_SET_WB 1 +# endif +# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_WB +# define ETHR_HAVE_SU_DW_NATMC_SET_WB 1 +# endif +# undef ETHR_HAVE_SU_DW_NATMC_SET_ACQB +# undef ETHR_HAVE_DW_NATMC_SET_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_ACQB +# define ETHR_HAVE_DW_NATMC_SET_ACQB 1 +# endif +# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_ACQB +# define ETHR_HAVE_SU_DW_NATMC_SET_ACQB 1 +# endif +# undef ETHR_HAVE_SU_DW_NATMC_SET_RELB +# undef ETHR_HAVE_DW_NATMC_SET_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_RELB +# define ETHR_HAVE_DW_NATMC_SET_RELB 1 +# endif +# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_RELB +# define ETHR_HAVE_SU_DW_NATMC_SET_RELB 1 +# endif +# undef ETHR_HAVE_SU_DW_NATMC_SET_MB +# undef ETHR_HAVE_DW_NATMC_SET_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_MB +# define ETHR_HAVE_DW_NATMC_SET_MB 1 +# endif +# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_MB +# define ETHR_HAVE_SU_DW_NATMC_SET_MB 1 +# endif +# undef ETHR_HAVE_SU_DW_NATMC_READ +# undef ETHR_HAVE_DW_NATMC_READ +# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ +# define ETHR_HAVE_DW_NATMC_READ 1 +# endif +# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ +# define ETHR_HAVE_SU_DW_NATMC_READ 1 +# endif +# undef ETHR_HAVE_SU_DW_NATMC_READ_RB +# undef ETHR_HAVE_DW_NATMC_READ_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_RB +# define ETHR_HAVE_DW_NATMC_READ_RB 1 +# endif +# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_RB +# define ETHR_HAVE_SU_DW_NATMC_READ_RB 1 +# endif +# undef ETHR_HAVE_SU_DW_NATMC_READ_WB +# undef ETHR_HAVE_DW_NATMC_READ_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_WB +# define ETHR_HAVE_DW_NATMC_READ_WB 1 +# endif +# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_WB +# define ETHR_HAVE_SU_DW_NATMC_READ_WB 1 +# endif +# undef ETHR_HAVE_SU_DW_NATMC_READ_ACQB +# undef ETHR_HAVE_DW_NATMC_READ_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_ACQB +# define ETHR_HAVE_DW_NATMC_READ_ACQB 1 +# endif +# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_ACQB +# define ETHR_HAVE_SU_DW_NATMC_READ_ACQB 1 +# endif +# undef ETHR_HAVE_SU_DW_NATMC_READ_RELB +# undef ETHR_HAVE_DW_NATMC_READ_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_RELB +# define ETHR_HAVE_DW_NATMC_READ_RELB 1 +# endif +# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_RELB +# define ETHR_HAVE_SU_DW_NATMC_READ_RELB 1 +# endif +# undef ETHR_HAVE_SU_DW_NATMC_READ_MB +# undef ETHR_HAVE_DW_NATMC_READ_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_MB +# define ETHR_HAVE_DW_NATMC_READ_MB 1 +# endif +# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_MB +# define ETHR_HAVE_SU_DW_NATMC_READ_MB 1 +# endif +# undef ETHR_HAVE_SU_DW_NATMC_INIT +# undef ETHR_HAVE_DW_NATMC_INIT +# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT +# define ETHR_HAVE_DW_NATMC_INIT 1 +# endif +# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT +# define ETHR_HAVE_SU_DW_NATMC_INIT 1 +# endif +# undef ETHR_HAVE_SU_DW_NATMC_INIT_RB +# undef ETHR_HAVE_DW_NATMC_INIT_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_RB +# define ETHR_HAVE_DW_NATMC_INIT_RB 1 +# endif +# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_RB +# define ETHR_HAVE_SU_DW_NATMC_INIT_RB 1 +# endif +# undef ETHR_HAVE_SU_DW_NATMC_INIT_WB +# undef ETHR_HAVE_DW_NATMC_INIT_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_WB +# define ETHR_HAVE_DW_NATMC_INIT_WB 1 +# endif +# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_WB +# define ETHR_HAVE_SU_DW_NATMC_INIT_WB 1 +# endif +# undef ETHR_HAVE_SU_DW_NATMC_INIT_ACQB +# undef ETHR_HAVE_DW_NATMC_INIT_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_ACQB +# define ETHR_HAVE_DW_NATMC_INIT_ACQB 1 +# endif +# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_ACQB +# define ETHR_HAVE_SU_DW_NATMC_INIT_ACQB 1 +# endif +# undef ETHR_HAVE_SU_DW_NATMC_INIT_RELB +# undef ETHR_HAVE_DW_NATMC_INIT_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_RELB +# define ETHR_HAVE_DW_NATMC_INIT_RELB 1 +# endif +# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_RELB +# define ETHR_HAVE_SU_DW_NATMC_INIT_RELB 1 +# endif +# undef ETHR_HAVE_SU_DW_NATMC_INIT_MB +# undef ETHR_HAVE_DW_NATMC_INIT_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_MB +# define ETHR_HAVE_DW_NATMC_INIT_MB 1 +# endif +# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_MB +# define ETHR_HAVE_SU_DW_NATMC_INIT_MB 1 +# endif +#elif ETHR_DW_NATMC_BITS__ == 64 +# undef ETHR_HAVE_DW_NATMC_CMPXCHG +# undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG +# define ETHR_HAVE_SU_DW_NATMC_CMPXCHG 1 +# endif +# undef ETHR_HAVE_DW_NATMC_CMPXCHG_RB +# undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RB +# define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB 1 +# endif +# undef ETHR_HAVE_DW_NATMC_CMPXCHG_WB +# undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_WB +# define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB 1 +# endif +# undef ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB +# undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_ACQB +# define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB 1 +# endif +# undef ETHR_HAVE_DW_NATMC_CMPXCHG_RELB +# undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RELB +# define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB 1 +# endif +# undef ETHR_HAVE_DW_NATMC_CMPXCHG_MB +# undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_MB +# define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB 1 +# endif +# undef ETHR_HAVE_DW_NATMC_SET +# undef ETHR_HAVE_SU_DW_NATMC_SET +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET +# define ETHR_HAVE_SU_DW_NATMC_SET 1 +# endif +# undef ETHR_HAVE_DW_NATMC_SET_RB +# undef ETHR_HAVE_SU_DW_NATMC_SET_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RB +# define ETHR_HAVE_SU_DW_NATMC_SET_RB 1 +# endif +# undef ETHR_HAVE_DW_NATMC_SET_WB +# undef ETHR_HAVE_SU_DW_NATMC_SET_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_WB +# define ETHR_HAVE_SU_DW_NATMC_SET_WB 1 +# endif +# undef ETHR_HAVE_DW_NATMC_SET_ACQB +# undef ETHR_HAVE_SU_DW_NATMC_SET_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_ACQB +# define ETHR_HAVE_SU_DW_NATMC_SET_ACQB 1 +# endif +# undef ETHR_HAVE_DW_NATMC_SET_RELB +# undef ETHR_HAVE_SU_DW_NATMC_SET_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RELB +# define ETHR_HAVE_SU_DW_NATMC_SET_RELB 1 +# endif +# undef ETHR_HAVE_DW_NATMC_SET_MB +# undef ETHR_HAVE_SU_DW_NATMC_SET_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_MB +# define ETHR_HAVE_SU_DW_NATMC_SET_MB 1 +# endif +# undef ETHR_HAVE_DW_NATMC_READ +# undef ETHR_HAVE_SU_DW_NATMC_READ +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ +# define ETHR_HAVE_SU_DW_NATMC_READ 1 +# endif +# undef ETHR_HAVE_DW_NATMC_READ_RB +# undef ETHR_HAVE_SU_DW_NATMC_READ_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_RB +# define ETHR_HAVE_SU_DW_NATMC_READ_RB 1 +# endif +# undef ETHR_HAVE_DW_NATMC_READ_WB +# undef ETHR_HAVE_SU_DW_NATMC_READ_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_WB +# define ETHR_HAVE_SU_DW_NATMC_READ_WB 1 +# endif +# undef ETHR_HAVE_DW_NATMC_READ_ACQB +# undef ETHR_HAVE_SU_DW_NATMC_READ_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_ACQB +# define ETHR_HAVE_SU_DW_NATMC_READ_ACQB 1 +# endif +# undef ETHR_HAVE_DW_NATMC_READ_RELB +# undef ETHR_HAVE_SU_DW_NATMC_READ_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_RELB +# define ETHR_HAVE_SU_DW_NATMC_READ_RELB 1 +# endif +# undef ETHR_HAVE_DW_NATMC_READ_MB +# undef ETHR_HAVE_SU_DW_NATMC_READ_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_MB +# define ETHR_HAVE_SU_DW_NATMC_READ_MB 1 +# endif +# undef ETHR_HAVE_DW_NATMC_INIT +# undef ETHR_HAVE_SU_DW_NATMC_INIT +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT +# define ETHR_HAVE_SU_DW_NATMC_INIT 1 +# endif +# undef ETHR_HAVE_DW_NATMC_INIT_RB +# undef ETHR_HAVE_SU_DW_NATMC_INIT_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_RB +# define ETHR_HAVE_SU_DW_NATMC_INIT_RB 1 +# endif +# undef ETHR_HAVE_DW_NATMC_INIT_WB +# undef ETHR_HAVE_SU_DW_NATMC_INIT_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_WB +# define ETHR_HAVE_SU_DW_NATMC_INIT_WB 1 +# endif +# undef ETHR_HAVE_DW_NATMC_INIT_ACQB +# undef ETHR_HAVE_SU_DW_NATMC_INIT_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_ACQB +# define ETHR_HAVE_SU_DW_NATMC_INIT_ACQB 1 +# endif +# undef ETHR_HAVE_DW_NATMC_INIT_RELB +# undef ETHR_HAVE_SU_DW_NATMC_INIT_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_RELB +# define ETHR_HAVE_SU_DW_NATMC_INIT_RELB 1 +# endif +# undef ETHR_HAVE_DW_NATMC_INIT_MB +# undef ETHR_HAVE_SU_DW_NATMC_INIT_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_MB +# define ETHR_HAVE_SU_DW_NATMC_INIT_MB 1 +# endif +#else +# error "Invalid native atomic size" +#endif + -#define ETHR_ATOMIC_PTR2LCK__(PTR) \ -(ðr_atomic_protection__[((((ethr_uint_t) (PTR)) >> ETHR_ATOMIC_ADDR_SHIFT) \ - & ((1 << ETHR_ATOMIC_ADDR_BITS) - 1))].u.lck) +#if defined(ETHR_HAVE_NATIVE_DW_ATOMIC) +#if (!defined(ETHR_HAVE_DW_NATMC_CMPXCHG) \ + && !defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB) \ + && !defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB) \ + && !defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB) \ + && !defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB) \ + && !defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB)) +# error "No native cmpxchg() op available" +#endif + + +/* + * Read op used together with cmpxchg() fallback when no native op present. + */ +#if defined(ETHR_HAVE_DW_NATMC_READ) +#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \ + ETHR_DW_NATMC_FUNC__(read)(VAR, VAL) +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ) +#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \ + VAL.dw_sint = ETHR_SU_DW_NATMC_FUNC__(read)(VAR) +#elif defined(ETHR_HAVE_DW_NATMC_READ_RB) +#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \ + ETHR_DW_NATMC_FUNC__(read_rb)(VAR, VAL) +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RB) +#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \ + VAL.dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_rb)(VAR) +#elif defined(ETHR_HAVE_DW_NATMC_READ_WB) +#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \ + ETHR_DW_NATMC_FUNC__(read_wb)(VAR, VAL) +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_WB) +#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \ + VAL.dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_wb)(VAR) +#elif defined(ETHR_HAVE_DW_NATMC_READ_ACQB) +#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \ + ETHR_DW_NATMC_FUNC__(read_acqb)(VAR, VAL) +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_ACQB) +#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \ + VAL.dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_acqb)(VAR) +#elif defined(ETHR_HAVE_DW_NATMC_READ_RELB) +#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \ + ETHR_DW_NATMC_FUNC__(read_relb)(VAR, VAL) +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RELB) +#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \ + VAL.dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_relb)(VAR) +#elif defined(ETHR_HAVE_DW_NATMC_READ_MB) +#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \ + ETHR_DW_NATMC_FUNC__(read_mb)(VAR, VAL) +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_MB) +#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \ + VAL.dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_mb)(VAR) +#else +/* + * We have no native read() op; guess zero and then use the + * the atomics actual value returned from cmpxchg(). + */ +#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \ +do { \ + VAL.sint[0] = (ethr_sint_t) 0; \ + VAL.sint[1] = (ethr_sint_t) 0; \ +} while (0) +#endif -#define ETHR_ATOMIC_OP_FALLBACK_IMPL__(AP, EXPS) \ -do { \ - ethr_spinlock_t *slp__ = ETHR_ATOMIC_PTR2LCK__((AP)); \ - ethr_spin_lock(slp__); \ - { EXPS; } \ - ethr_spin_unlock(slp__); \ +/* + * Native cmpxchg() fallback used when no native op present. + */ +#define ETHR_DW_NATMC_CMPXCHG_FALLBACK__(CMPXCHG, VAR, AVAL, OPS) \ +do { \ + int res__; \ + ethr_dw_sint_t AVAL, exp_act__; \ + ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, exp_act__); \ + do { \ + AVAL.sint[0] = exp_act__.sint[0]; \ + AVAL.sint[1] = exp_act__.sint[1]; \ + { OPS; } \ + res__ = CMPXCHG(VAR, AVAL.sint, exp_act__.sint); \ + } while (__builtin_expect(res__ == 0, 0)); \ } while (0) + +#elif defined(ETHR_HAVE_NATIVE_SU_DW_ATOMIC) + +#if (!defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG) \ + && !defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB) \ + && !defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB) \ + && !defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB) \ + && !defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB) \ + && !defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB)) +# error "No native cmpxchg() op available" +#endif + + +/* + * Read op used together with cmpxchg() fallback when no native op present. + */ +#if defined(ETHR_HAVE_SU_DW_NATMC_READ) +#define ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \ + ETHR_SU_DW_NATMC_FUNC__(read)(VAR) +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RB) +#define ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \ + ETHR_SU_DW_NATMC_FUNC__(read_rb)(VAR) +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_WB) +#define ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \ + ETHR_SU_DW_NATMC_FUNC__(read_wb)(VAR) +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_ACQB) +#define ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \ + ETHR_SU_DW_NATMC_FUNC__(read_acqb)(VAR) +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RELB) +#define ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \ + ETHR_SU_DW_NATMC_FUNC__(read_relb)(VAR) +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_MB) +#define ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \ + ETHR_SU_DW_NATMC_FUNC__(read_mb)(VAR) +#else +/* + * We have no native read() op; guess zero and then use the + * the atomics actual value returned from cmpxchg(). + */ +#define ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \ + ((ETHR_SU_DW_NAINT_T__) 0) +#endif + +/* + * Native cmpxchg() fallback used when no native op present. + */ +#define ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(CMPXCHG, VAR, AVAL, OPS) \ +do { \ + ETHR_SU_DW_NAINT_T__ AVAL; \ + ETHR_SU_DW_NAINT_T__ new__, act__, exp__; \ + act__ = ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR); \ + do { \ + exp__ = act__; \ + AVAL = (ETHR_SU_DW_NAINT_T__) act__; \ + { OPS; } \ + new__ = (ETHR_SU_DW_NAINT_T__) AVAL; \ + act__ = CMPXCHG(VAR, new__, exp__); \ + } while (__builtin_expect(act__ != exp__, 0)); \ +} while (0) + + +#else +# error "?!?" +#endif + + + +/* --- addr() --- */ + +static ETHR_INLINE ethr_sint_t *ETHR_DW_ATMC_FUNC__(addr)(ethr_dw_atomic_t *var) +{ +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) { +#endif + + return (ethr_sint_t *) ETHR_DW_NATMC_ADDR_FUNC__((&var->native)); + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + } else { return ETHR_DW_ATOMIC_FUNC__(addr)(var); } +#endif + +} + + +/* --- cmpxchg() --- */ + + +static ETHR_INLINE int ETHR_DW_ATMC_FUNC__(cmpxchg)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val) +{ + int res; + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) { +#endif + +#if defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG) + ETHR_SU_DW_NAINT_T__ act; + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->dw_sint, old_val->dw_sint); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB) + ETHR_SU_DW_NAINT_T__ act; + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->dw_sint, old_val->dw_sint); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB) + ETHR_SU_DW_NAINT_T__ act; + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->dw_sint, old_val->dw_sint); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB) + ETHR_SU_DW_NAINT_T__ act; + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->dw_sint, old_val->dw_sint); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB) + ETHR_SU_DW_NAINT_T__ act; + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->dw_sint, old_val->dw_sint); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB) + ETHR_SU_DW_NAINT_T__ act; + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->dw_sint, old_val->dw_sint); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG) + res = ETHR_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->sint, old_val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB) + res = ETHR_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->sint, old_val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB) + res = ETHR_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->sint, old_val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB) + res = ETHR_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->sint, old_val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB) + res = ETHR_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->sint, old_val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB) + res = ETHR_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->sint, old_val->sint); +#else +#error "Missing implementation of ethr_dw_atomic_cmpxchg()!" +#endif + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + } else { res = ETHR_DW_ATOMIC_FUNC__(cmpxchg)(var, val, old_val); } +#endif + + return res; +} + +static ETHR_INLINE int ETHR_DW_ATMC_FUNC__(cmpxchg_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val) +{ + int res; + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) { +#endif + +#if defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB) + ETHR_SU_DW_NAINT_T__ act; + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->dw_sint, old_val->dw_sint); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG) + ETHR_SU_DW_NAINT_T__ act; + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->dw_sint, old_val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB) + ETHR_SU_DW_NAINT_T__ act; + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->dw_sint, old_val->dw_sint); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB) + ETHR_SU_DW_NAINT_T__ act; + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->dw_sint, old_val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB) + ETHR_SU_DW_NAINT_T__ act; + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->dw_sint, old_val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB) + ETHR_SU_DW_NAINT_T__ act; + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->dw_sint, old_val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB) + res = ETHR_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->sint, old_val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG) + res = ETHR_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->sint, old_val->sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB) + res = ETHR_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->sint, old_val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB) + res = ETHR_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->sint, old_val->sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB) + res = ETHR_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->sint, old_val->sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB) + res = ETHR_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->sint, old_val->sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#else +#error "Missing implementation of ethr_dw_atomic_cmpxchg_rb()!" +#endif + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + } else { res = ETHR_DW_ATOMIC_FUNC__(cmpxchg_rb)(var, val, old_val); } +#endif + + return res; +} + +static ETHR_INLINE int ETHR_DW_ATMC_FUNC__(cmpxchg_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val) +{ + int res; + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) { +#endif + +#if defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB) + ETHR_SU_DW_NAINT_T__ act; + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->dw_sint, old_val->dw_sint); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG) + ETHR_SU_DW_NAINT_T__ act; + ETHR_MEMBAR(ETHR_StoreStore); + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->dw_sint, old_val->dw_sint); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB) + ETHR_SU_DW_NAINT_T__ act; + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->dw_sint, old_val->dw_sint); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB) + ETHR_SU_DW_NAINT_T__ act; + ETHR_MEMBAR(ETHR_StoreStore); + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->dw_sint, old_val->dw_sint); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB) + ETHR_SU_DW_NAINT_T__ act; + ETHR_MEMBAR(ETHR_StoreStore); + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->dw_sint, old_val->dw_sint); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB) + ETHR_SU_DW_NAINT_T__ act; + ETHR_MEMBAR(ETHR_StoreStore); + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->dw_sint, old_val->dw_sint); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB) + res = ETHR_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->sint, old_val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG) + ETHR_MEMBAR(ETHR_StoreStore); + res = ETHR_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->sint, old_val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB) + res = ETHR_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->sint, old_val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_StoreStore); + res = ETHR_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->sint, old_val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + res = ETHR_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->sint, old_val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + res = ETHR_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->sint, old_val->sint); +#else +#error "Missing implementation of ethr_dw_atomic_cmpxchg_wb()!" +#endif + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + } else { res = ETHR_DW_ATOMIC_FUNC__(cmpxchg_wb)(var, val, old_val); } +#endif + + return res; +} + +static ETHR_INLINE int ETHR_DW_ATMC_FUNC__(cmpxchg_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val) +{ + int res; + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) { +#endif + +#if defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB) + ETHR_SU_DW_NAINT_T__ act; + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->dw_sint, old_val->dw_sint); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB) + ETHR_SU_DW_NAINT_T__ act; + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->dw_sint, old_val->dw_sint); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG) + ETHR_SU_DW_NAINT_T__ act; + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->dw_sint, old_val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB) + ETHR_SU_DW_NAINT_T__ act; + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->dw_sint, old_val->dw_sint); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB) + ETHR_SU_DW_NAINT_T__ act; + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->dw_sint, old_val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB) + ETHR_SU_DW_NAINT_T__ act; + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->dw_sint, old_val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB) + res = ETHR_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->sint, old_val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB) + res = ETHR_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->sint, old_val->sint); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG) + res = ETHR_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->sint, old_val->sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB) + res = ETHR_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->sint, old_val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB) + res = ETHR_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->sint, old_val->sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB) + res = ETHR_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->sint, old_val->sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else +#error "Missing implementation of ethr_dw_atomic_cmpxchg_acqb()!" +#endif + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + } else { res = ETHR_DW_ATOMIC_FUNC__(cmpxchg_acqb)(var, val, old_val); } +#endif + + return res; +} + +static ETHR_INLINE int ETHR_DW_ATMC_FUNC__(cmpxchg_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val) +{ + int res; + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) { +#endif + +#if defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB) + ETHR_SU_DW_NAINT_T__ act; + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->dw_sint, old_val->dw_sint); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB) + ETHR_SU_DW_NAINT_T__ act; + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->dw_sint, old_val->dw_sint); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG) + ETHR_SU_DW_NAINT_T__ act; + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->dw_sint, old_val->dw_sint); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB) + ETHR_SU_DW_NAINT_T__ act; + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->dw_sint, old_val->dw_sint); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB) + ETHR_SU_DW_NAINT_T__ act; + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->dw_sint, old_val->dw_sint); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB) + ETHR_SU_DW_NAINT_T__ act; + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->dw_sint, old_val->dw_sint); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB) + res = ETHR_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->sint, old_val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = ETHR_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->sint, old_val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = ETHR_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->sint, old_val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB) + res = ETHR_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->sint, old_val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = ETHR_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->sint, old_val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = ETHR_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->sint, old_val->sint); +#else +#error "Missing implementation of ethr_dw_atomic_cmpxchg_relb()!" +#endif + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + } else { res = ETHR_DW_ATOMIC_FUNC__(cmpxchg_relb)(var, val, old_val); } +#endif + + return res; +} + +static ETHR_INLINE int ETHR_DW_ATMC_FUNC__(cmpxchg_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val) +{ + int res; + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) { +#endif + +#if defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB) + ETHR_SU_DW_NAINT_T__ act; + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->dw_sint, old_val->dw_sint); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB) + ETHR_SU_DW_NAINT_T__ act; + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->dw_sint, old_val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB) + ETHR_SU_DW_NAINT_T__ act; + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->dw_sint, old_val->dw_sint); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB) + ETHR_SU_DW_NAINT_T__ act; + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->dw_sint, old_val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB) + ETHR_SU_DW_NAINT_T__ act; + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->dw_sint, old_val->dw_sint); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG) + ETHR_SU_DW_NAINT_T__ act; + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->dw_sint, old_val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB) + res = ETHR_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->sint, old_val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB) + res = ETHR_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->sint, old_val->sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = ETHR_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->sint, old_val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = ETHR_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->sint, old_val->sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = ETHR_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->sint, old_val->sint); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = ETHR_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->sint, old_val->sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else +#error "Missing implementation of ethr_dw_atomic_cmpxchg_mb()!" +#endif + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + } else { res = ETHR_DW_ATOMIC_FUNC__(cmpxchg_mb)(var, val, old_val); } +#endif + + return res; +} + + +/* --- set() --- */ + + +static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(set)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) { +#endif + +#if defined(ETHR_HAVE_SU_DW_NATMC_SET) + ETHR_SU_DW_NATMC_FUNC__(set)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RB) + ETHR_SU_DW_NATMC_FUNC__(set_rb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_WB) + ETHR_SU_DW_NATMC_FUNC__(set_wb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_ACQB) + ETHR_SU_DW_NATMC_FUNC__(set_acqb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RELB) + ETHR_SU_DW_NATMC_FUNC__(set_relb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_MB) + ETHR_SU_DW_NATMC_FUNC__(set_mb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_DW_NATMC_SET) + ETHR_DW_NATMC_FUNC__(set)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_SET_RB) + ETHR_DW_NATMC_FUNC__(set_rb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_SET_WB) + ETHR_DW_NATMC_FUNC__(set_wb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_SET_ACQB) + ETHR_DW_NATMC_FUNC__(set_acqb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_SET_RELB) + ETHR_DW_NATMC_FUNC__(set_relb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_SET_MB) + ETHR_DW_NATMC_FUNC__(set_mb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG) + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval = val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB) + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval = val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB) + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval = val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB) + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval = val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB) + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval = val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB) + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval = val->dw_sint); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG) + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB) + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB) + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB) + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB) + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB) + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); +#else +#error "Missing implementation of ethr_dw_atomic_set()!" +#endif + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + } else { ETHR_DW_ATOMIC_FUNC__(set)(var, val); } +#endif + +} + +static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(set_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) { +#endif + +#if defined(ETHR_HAVE_SU_DW_NATMC_SET_RB) + ETHR_SU_DW_NATMC_FUNC__(set_rb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET) + ETHR_SU_DW_NATMC_FUNC__(set)(&var->native, val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_MB) + ETHR_SU_DW_NATMC_FUNC__(set_mb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_WB) + ETHR_SU_DW_NATMC_FUNC__(set_wb)(&var->native, val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_ACQB) + ETHR_SU_DW_NATMC_FUNC__(set_acqb)(&var->native, val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RELB) + ETHR_SU_DW_NATMC_FUNC__(set_relb)(&var->native, val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_DW_NATMC_SET_RB) + ETHR_DW_NATMC_FUNC__(set_rb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_SET) + ETHR_DW_NATMC_FUNC__(set)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_DW_NATMC_SET_MB) + ETHR_DW_NATMC_FUNC__(set_mb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_SET_WB) + ETHR_DW_NATMC_FUNC__(set_wb)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_DW_NATMC_SET_ACQB) + ETHR_DW_NATMC_FUNC__(set_acqb)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_DW_NATMC_SET_RELB) + ETHR_DW_NATMC_FUNC__(set_relb)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB) + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval = val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG) + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval = val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB) + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval = val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB) + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval = val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB) + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval = val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB) + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval = val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB) + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG) + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB) + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB) + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB) + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB) + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); + ETHR_MEMBAR(ETHR_LoadLoad); +#else +#error "Missing implementation of ethr_dw_atomic_set_rb()!" +#endif + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + } else { ETHR_DW_ATOMIC_FUNC__(set_rb)(var, val); } +#endif + +} + +static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(set_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) { +#endif + +#if defined(ETHR_HAVE_SU_DW_NATMC_SET_WB) + ETHR_SU_DW_NATMC_FUNC__(set_wb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_SU_DW_NATMC_FUNC__(set)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_MB) + ETHR_SU_DW_NATMC_FUNC__(set_mb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_SU_DW_NATMC_FUNC__(set_rb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_SU_DW_NATMC_FUNC__(set_acqb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_SU_DW_NATMC_FUNC__(set_relb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_DW_NATMC_SET_WB) + ETHR_DW_NATMC_FUNC__(set_wb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_SET) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_DW_NATMC_FUNC__(set)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_SET_MB) + ETHR_DW_NATMC_FUNC__(set_mb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_SET_RB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_DW_NATMC_FUNC__(set_rb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_SET_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_DW_NATMC_FUNC__(set_acqb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_SET_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_DW_NATMC_FUNC__(set_relb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB) + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval = val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval = val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB) + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval = val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval = val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval = val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval = val->dw_sint); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB) + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB) + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); +#else +#error "Missing implementation of ethr_dw_atomic_set_wb()!" +#endif + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + } else { ETHR_DW_ATOMIC_FUNC__(set_wb)(var, val); } +#endif + +} + +static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(set_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) { +#endif + +#if defined(ETHR_HAVE_SU_DW_NATMC_SET_ACQB) + ETHR_SU_DW_NATMC_FUNC__(set_acqb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RB) + ETHR_SU_DW_NATMC_FUNC__(set_rb)(&var->native, val->dw_sint); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET) + ETHR_SU_DW_NATMC_FUNC__(set)(&var->native, val->dw_sint); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_MB) + ETHR_SU_DW_NATMC_FUNC__(set_mb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_WB) + ETHR_SU_DW_NATMC_FUNC__(set_wb)(&var->native, val->dw_sint); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RELB) + ETHR_SU_DW_NATMC_FUNC__(set_relb)(&var->native, val->dw_sint); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_SET_ACQB) + ETHR_DW_NATMC_FUNC__(set_acqb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_SET_RB) + ETHR_DW_NATMC_FUNC__(set_rb)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_SET) + ETHR_DW_NATMC_FUNC__(set)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_SET_MB) + ETHR_DW_NATMC_FUNC__(set_mb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_SET_WB) + ETHR_DW_NATMC_FUNC__(set_wb)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_SET_RELB) + ETHR_DW_NATMC_FUNC__(set_relb)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB) + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval = val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB) + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval = val->dw_sint); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG) + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval = val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB) + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval = val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB) + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval = val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB) + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval = val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB) + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB) + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG) + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB) + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB) + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB) + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else +#error "Missing implementation of ethr_dw_atomic_set_acqb()!" +#endif + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + } else { ETHR_DW_ATOMIC_FUNC__(set_acqb)(var, val); } +#endif + +} + +static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(set_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) { +#endif + +#if defined(ETHR_HAVE_SU_DW_NATMC_SET_RELB) + ETHR_SU_DW_NATMC_FUNC__(set_relb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_WB) + ETHR_MEMBAR(ETHR_LoadStore); + ETHR_SU_DW_NATMC_FUNC__(set_wb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET) + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_SU_DW_NATMC_FUNC__(set)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_MB) + ETHR_SU_DW_NATMC_FUNC__(set_mb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RB) + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_SU_DW_NATMC_FUNC__(set_rb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_ACQB) + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_SU_DW_NATMC_FUNC__(set_acqb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_DW_NATMC_SET_RELB) + ETHR_DW_NATMC_FUNC__(set_relb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_SET_WB) + ETHR_MEMBAR(ETHR_LoadStore); + ETHR_DW_NATMC_FUNC__(set_wb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_SET) + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_DW_NATMC_FUNC__(set)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_SET_MB) + ETHR_DW_NATMC_FUNC__(set_mb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_SET_RB) + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_DW_NATMC_FUNC__(set_rb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_SET_ACQB) + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_DW_NATMC_FUNC__(set_acqb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB) + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval = val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval = val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval = val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB) + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval = val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval = val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval = val->dw_sint); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB) + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB) + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); +#else +#error "Missing implementation of ethr_dw_atomic_set_relb()!" +#endif + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + } else { ETHR_DW_ATOMIC_FUNC__(set_relb)(var, val); } +#endif + +} + +static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(set_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) { +#endif + +#if defined(ETHR_HAVE_SU_DW_NATMC_SET_MB) + ETHR_SU_DW_NATMC_FUNC__(set_mb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RELB) + ETHR_SU_DW_NATMC_FUNC__(set_relb)(&var->native, val->dw_sint); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_ACQB) + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_SU_DW_NATMC_FUNC__(set_acqb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_WB) + ETHR_MEMBAR(ETHR_LoadStore); + ETHR_SU_DW_NATMC_FUNC__(set_wb)(&var->native, val->dw_sint); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RB) + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_SU_DW_NATMC_FUNC__(set_rb)(&var->native, val->dw_sint); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET) + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_SU_DW_NATMC_FUNC__(set)(&var->native, val->dw_sint); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_SET_MB) + ETHR_DW_NATMC_FUNC__(set_mb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_SET_RELB) + ETHR_DW_NATMC_FUNC__(set_relb)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_SET_ACQB) + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_DW_NATMC_FUNC__(set_acqb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_SET_WB) + ETHR_MEMBAR(ETHR_LoadStore); + ETHR_DW_NATMC_FUNC__(set_wb)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_SET_RB) + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_DW_NATMC_FUNC__(set_rb)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_SET) + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_DW_NATMC_FUNC__(set)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB) + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval = val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB) + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval = val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval = val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval = val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval = val->dw_sint); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval = val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB) + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB) + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else +#error "Missing implementation of ethr_dw_atomic_set_mb()!" +#endif + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + } else { ETHR_DW_ATOMIC_FUNC__(set_mb)(var, val); } +#endif + +} + + +/* --- read() --- */ + + +static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(read)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) { +#endif + +#if defined(ETHR_HAVE_SU_DW_NATMC_READ) + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read)(&var->native); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RB) + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_rb)(&var->native); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_WB) + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_wb)(&var->native); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_ACQB) + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_acqb)(&var->native); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RELB) + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_relb)(&var->native); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_MB) + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_mb)(&var->native); +#elif defined(ETHR_HAVE_DW_NATMC_READ) + ETHR_DW_NATMC_FUNC__(read)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_READ_RB) + ETHR_DW_NATMC_FUNC__(read_rb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_READ_WB) + ETHR_DW_NATMC_FUNC__(read_wb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_READ_ACQB) + ETHR_DW_NATMC_FUNC__(read_acqb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_READ_RELB) + ETHR_DW_NATMC_FUNC__(read_relb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_READ_MB) + ETHR_DW_NATMC_FUNC__(read_mb)(&var->native, val->sint); +#else + ethr_dw_sint_t tmp; + tmp.sint[0] = ETHR_UNUSUAL_SINT_VAL__; + tmp.sint[1] = ETHR_UNUSUAL_SINT_VAL__; + val->sint[0] = ETHR_UNUSUAL_SINT_VAL__; + val->sint[1] = ETHR_UNUSUAL_SINT_VAL__; + (void) ETHR_DW_ATMC_FUNC__(cmpxchg)(var, &tmp, val); +#endif + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + } else { ETHR_DW_ATOMIC_FUNC__(read)(var, val); } +#endif + +} + +static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(read_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) { +#endif + +#if defined(ETHR_HAVE_SU_DW_NATMC_READ_RB) + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_rb)(&var->native); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ) + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read)(&var->native); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_MB) + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_mb)(&var->native); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_WB) + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_wb)(&var->native); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_ACQB) + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_acqb)(&var->native); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RELB) + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_relb)(&var->native); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_DW_NATMC_READ_RB) + ETHR_DW_NATMC_FUNC__(read_rb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_READ) + ETHR_DW_NATMC_FUNC__(read)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_DW_NATMC_READ_MB) + ETHR_DW_NATMC_FUNC__(read_mb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_READ_WB) + ETHR_DW_NATMC_FUNC__(read_wb)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_DW_NATMC_READ_ACQB) + ETHR_DW_NATMC_FUNC__(read_acqb)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_DW_NATMC_READ_RELB) + ETHR_DW_NATMC_FUNC__(read_relb)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + ethr_dw_sint_t tmp; + tmp.sint[0] = ETHR_UNUSUAL_SINT_VAL__; + tmp.sint[1] = ETHR_UNUSUAL_SINT_VAL__; + val->sint[0] = ETHR_UNUSUAL_SINT_VAL__; + val->sint[1] = ETHR_UNUSUAL_SINT_VAL__; + (void) ETHR_DW_ATMC_FUNC__(cmpxchg_rb)(var, &tmp, val); +#endif + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + } else { ETHR_DW_ATOMIC_FUNC__(read_rb)(var, val); } +#endif + +} + +static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(read_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) { +#endif + +#if defined(ETHR_HAVE_SU_DW_NATMC_READ_WB) + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_wb)(&var->native); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ) + ETHR_MEMBAR(ETHR_StoreStore); + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read)(&var->native); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_MB) + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_mb)(&var->native); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RB) + ETHR_MEMBAR(ETHR_StoreStore); + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_rb)(&var->native); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_acqb)(&var->native); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_relb)(&var->native); +#elif defined(ETHR_HAVE_DW_NATMC_READ_WB) + ETHR_DW_NATMC_FUNC__(read_wb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_READ) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_DW_NATMC_FUNC__(read)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_READ_MB) + ETHR_DW_NATMC_FUNC__(read_mb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_READ_RB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_DW_NATMC_FUNC__(read_rb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_READ_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_DW_NATMC_FUNC__(read_acqb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_READ_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_DW_NATMC_FUNC__(read_relb)(&var->native, val->sint); +#else + ethr_dw_sint_t tmp; + tmp.sint[0] = ETHR_UNUSUAL_SINT_VAL__; + tmp.sint[1] = ETHR_UNUSUAL_SINT_VAL__; + val->sint[0] = ETHR_UNUSUAL_SINT_VAL__; + val->sint[1] = ETHR_UNUSUAL_SINT_VAL__; + (void) ETHR_DW_ATMC_FUNC__(cmpxchg_wb)(var, &tmp, val); +#endif + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + } else { ETHR_DW_ATOMIC_FUNC__(read_wb)(var, val); } +#endif + +} + +static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(read_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) { +#endif + +#if defined(ETHR_HAVE_SU_DW_NATMC_READ_ACQB) + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_acqb)(&var->native); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RB) + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_rb)(&var->native); + ETHR_MEMBAR(ETHR_LoadStore); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ) + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read)(&var->native); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_MB) + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_mb)(&var->native); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_WB) + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_wb)(&var->native); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RELB) + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_relb)(&var->native); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#elif defined(ETHR_HAVE_DW_NATMC_READ_ACQB) + ETHR_DW_NATMC_FUNC__(read_acqb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_READ_RB) + ETHR_DW_NATMC_FUNC__(read_rb)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_LoadStore); +#elif defined(ETHR_HAVE_DW_NATMC_READ) + ETHR_DW_NATMC_FUNC__(read)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#elif defined(ETHR_HAVE_DW_NATMC_READ_MB) + ETHR_DW_NATMC_FUNC__(read_mb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_READ_WB) + ETHR_DW_NATMC_FUNC__(read_wb)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#elif defined(ETHR_HAVE_DW_NATMC_READ_RELB) + ETHR_DW_NATMC_FUNC__(read_relb)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#else + ethr_dw_sint_t tmp; + tmp.sint[0] = ETHR_UNUSUAL_SINT_VAL__; + tmp.sint[1] = ETHR_UNUSUAL_SINT_VAL__; + val->sint[0] = ETHR_UNUSUAL_SINT_VAL__; + val->sint[1] = ETHR_UNUSUAL_SINT_VAL__; + (void) ETHR_DW_ATMC_FUNC__(cmpxchg_acqb)(var, &tmp, val); +#endif + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + } else { ETHR_DW_ATOMIC_FUNC__(read_acqb)(var, val); } +#endif + +} + +static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(read_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) { +#endif + +#if defined(ETHR_HAVE_SU_DW_NATMC_READ_RELB) + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_relb)(&var->native); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_wb)(&var->native); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read)(&var->native); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_MB) + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_mb)(&var->native); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_rb)(&var->native); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_acqb)(&var->native); +#elif defined(ETHR_HAVE_DW_NATMC_READ_RELB) + ETHR_DW_NATMC_FUNC__(read_relb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_READ_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_DW_NATMC_FUNC__(read_wb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_READ) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_DW_NATMC_FUNC__(read)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_READ_MB) + ETHR_DW_NATMC_FUNC__(read_mb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_READ_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_DW_NATMC_FUNC__(read_rb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_READ_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_DW_NATMC_FUNC__(read_acqb)(&var->native, val->sint); +#else + ethr_dw_sint_t tmp; + tmp.sint[0] = ETHR_UNUSUAL_SINT_VAL__; + tmp.sint[1] = ETHR_UNUSUAL_SINT_VAL__; + val->sint[0] = ETHR_UNUSUAL_SINT_VAL__; + val->sint[1] = ETHR_UNUSUAL_SINT_VAL__; + (void) ETHR_DW_ATMC_FUNC__(cmpxchg_relb)(var, &tmp, val); +#endif + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + } else { ETHR_DW_ATOMIC_FUNC__(read_relb)(var, val); } +#endif + +} + +static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(read_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) { +#endif + +#if defined(ETHR_HAVE_SU_DW_NATMC_READ_MB) + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_mb)(&var->native); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RELB) + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_relb)(&var->native); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_acqb)(&var->native); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_wb)(&var->native); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_rb)(&var->native); + ETHR_MEMBAR(ETHR_LoadStore); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read)(&var->native); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#elif defined(ETHR_HAVE_DW_NATMC_READ_MB) + ETHR_DW_NATMC_FUNC__(read_mb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_READ_RELB) + ETHR_DW_NATMC_FUNC__(read_relb)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#elif defined(ETHR_HAVE_DW_NATMC_READ_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_DW_NATMC_FUNC__(read_acqb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_READ_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_DW_NATMC_FUNC__(read_wb)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#elif defined(ETHR_HAVE_DW_NATMC_READ_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_DW_NATMC_FUNC__(read_rb)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_LoadStore); +#elif defined(ETHR_HAVE_DW_NATMC_READ) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_DW_NATMC_FUNC__(read)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#else + ethr_dw_sint_t tmp; + tmp.sint[0] = ETHR_UNUSUAL_SINT_VAL__; + tmp.sint[1] = ETHR_UNUSUAL_SINT_VAL__; + val->sint[0] = ETHR_UNUSUAL_SINT_VAL__; + val->sint[1] = ETHR_UNUSUAL_SINT_VAL__; + (void) ETHR_DW_ATMC_FUNC__(cmpxchg_mb)(var, &tmp, val); +#endif + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + } else { ETHR_DW_ATOMIC_FUNC__(read_mb)(var, val); } +#endif + +} + + +/* --- init() --- */ + + +static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(init)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) { +#endif + +#if defined(ETHR_HAVE_SU_DW_NATMC_INIT) + ETHR_SU_DW_NATMC_FUNC__(init)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RB) + ETHR_SU_DW_NATMC_FUNC__(init_rb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_WB) + ETHR_SU_DW_NATMC_FUNC__(init_wb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_ACQB) + ETHR_SU_DW_NATMC_FUNC__(init_acqb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RELB) + ETHR_SU_DW_NATMC_FUNC__(init_relb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_MB) + ETHR_SU_DW_NATMC_FUNC__(init_mb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_DW_NATMC_INIT) + ETHR_DW_NATMC_FUNC__(init)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_RB) + ETHR_DW_NATMC_FUNC__(init_rb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_WB) + ETHR_DW_NATMC_FUNC__(init_wb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_ACQB) + ETHR_DW_NATMC_FUNC__(init_acqb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_RELB) + ETHR_DW_NATMC_FUNC__(init_relb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_MB) + ETHR_DW_NATMC_FUNC__(init_mb)(&var->native, val->sint); +#else + ETHR_DW_ATMC_FUNC__(set)(var, val); +#endif + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + } else { ETHR_DW_ATOMIC_FUNC__(init)(var, val); } +#endif + +} + +static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(init_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) { +#endif + +#if defined(ETHR_HAVE_SU_DW_NATMC_INIT_RB) + ETHR_SU_DW_NATMC_FUNC__(init_rb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT) + ETHR_SU_DW_NATMC_FUNC__(init)(&var->native, val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_MB) + ETHR_SU_DW_NATMC_FUNC__(init_mb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_WB) + ETHR_SU_DW_NATMC_FUNC__(init_wb)(&var->native, val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_ACQB) + ETHR_SU_DW_NATMC_FUNC__(init_acqb)(&var->native, val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RELB) + ETHR_SU_DW_NATMC_FUNC__(init_relb)(&var->native, val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_RB) + ETHR_DW_NATMC_FUNC__(init_rb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_INIT) + ETHR_DW_NATMC_FUNC__(init)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_MB) + ETHR_DW_NATMC_FUNC__(init_mb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_WB) + ETHR_DW_NATMC_FUNC__(init_wb)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_ACQB) + ETHR_DW_NATMC_FUNC__(init_acqb)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_RELB) + ETHR_DW_NATMC_FUNC__(init_relb)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + ETHR_DW_ATMC_FUNC__(set_rb)(var, val); +#endif + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + } else { ETHR_DW_ATOMIC_FUNC__(init_rb)(var, val); } +#endif + +} + +static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(init_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) { +#endif + +#if defined(ETHR_HAVE_SU_DW_NATMC_INIT_WB) + ETHR_SU_DW_NATMC_FUNC__(init_wb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_SU_DW_NATMC_FUNC__(init)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_MB) + ETHR_SU_DW_NATMC_FUNC__(init_mb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_SU_DW_NATMC_FUNC__(init_rb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_SU_DW_NATMC_FUNC__(init_acqb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_SU_DW_NATMC_FUNC__(init_relb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_WB) + ETHR_DW_NATMC_FUNC__(init_wb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_INIT) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_DW_NATMC_FUNC__(init)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_MB) + ETHR_DW_NATMC_FUNC__(init_mb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_RB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_DW_NATMC_FUNC__(init_rb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_DW_NATMC_FUNC__(init_acqb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_DW_NATMC_FUNC__(init_relb)(&var->native, val->sint); +#else + ETHR_DW_ATMC_FUNC__(set_wb)(var, val); +#endif + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + } else { ETHR_DW_ATOMIC_FUNC__(init_wb)(var, val); } +#endif + +} + +static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(init_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) { +#endif + +#if defined(ETHR_HAVE_SU_DW_NATMC_INIT_ACQB) + ETHR_SU_DW_NATMC_FUNC__(init_acqb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RB) + ETHR_SU_DW_NATMC_FUNC__(init_rb)(&var->native, val->dw_sint); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT) + ETHR_SU_DW_NATMC_FUNC__(init)(&var->native, val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_MB) + ETHR_SU_DW_NATMC_FUNC__(init_mb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_WB) + ETHR_SU_DW_NATMC_FUNC__(init_wb)(&var->native, val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RELB) + ETHR_SU_DW_NATMC_FUNC__(init_relb)(&var->native, val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_ACQB) + ETHR_DW_NATMC_FUNC__(init_acqb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_RB) + ETHR_DW_NATMC_FUNC__(init_rb)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_INIT) + ETHR_DW_NATMC_FUNC__(init)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_MB) + ETHR_DW_NATMC_FUNC__(init_mb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_WB) + ETHR_DW_NATMC_FUNC__(init_wb)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_RELB) + ETHR_DW_NATMC_FUNC__(init_relb)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_DW_ATMC_FUNC__(set_acqb)(var, val); +#endif + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + } else { ETHR_DW_ATOMIC_FUNC__(init_acqb)(var, val); } +#endif + +} + +static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(init_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) { +#endif + +#if defined(ETHR_HAVE_SU_DW_NATMC_INIT_RELB) + ETHR_SU_DW_NATMC_FUNC__(init_relb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad); + ETHR_SU_DW_NATMC_FUNC__(init_wb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_SU_DW_NATMC_FUNC__(init)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_MB) + ETHR_SU_DW_NATMC_FUNC__(init_mb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_SU_DW_NATMC_FUNC__(init_rb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_SU_DW_NATMC_FUNC__(init_acqb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_RELB) + ETHR_DW_NATMC_FUNC__(init_relb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad); + ETHR_DW_NATMC_FUNC__(init_wb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_INIT) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_DW_NATMC_FUNC__(init)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_MB) + ETHR_DW_NATMC_FUNC__(init_mb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_DW_NATMC_FUNC__(init_rb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_DW_NATMC_FUNC__(init_acqb)(&var->native, val->sint); +#else + ETHR_DW_ATMC_FUNC__(set_relb)(var, val); +#endif + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + } else { ETHR_DW_ATOMIC_FUNC__(init_relb)(var, val); } +#endif + +} + +static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(init_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) { +#endif + +#if defined(ETHR_HAVE_SU_DW_NATMC_INIT_MB) + ETHR_SU_DW_NATMC_FUNC__(init_mb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RELB) + ETHR_SU_DW_NATMC_FUNC__(init_relb)(&var->native, val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_SU_DW_NATMC_FUNC__(init_acqb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad); + ETHR_SU_DW_NATMC_FUNC__(init_wb)(&var->native, val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_SU_DW_NATMC_FUNC__(init_rb)(&var->native, val->dw_sint); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_SU_DW_NATMC_FUNC__(init)(&var->native, val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_MB) + ETHR_DW_NATMC_FUNC__(init_mb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_RELB) + ETHR_DW_NATMC_FUNC__(init_relb)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_DW_NATMC_FUNC__(init_acqb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad); + ETHR_DW_NATMC_FUNC__(init_wb)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_DW_NATMC_FUNC__(init_rb)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_INIT) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_DW_NATMC_FUNC__(init)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_DW_ATMC_FUNC__(set_mb)(var, val); +#endif + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + } else { ETHR_DW_ATOMIC_FUNC__(init_mb)(var, val); } +#endif + +} + +#endif /* ETHR_DW_ATMC_INLINE__ */ + + +/* ---------- Word size atomic implementation ---------- */ + + +#ifdef ETHR_NEED_ATMC_PROTOTYPES__ +ethr_sint_t *ethr_atomic_addr(ethr_atomic_t *var); +ethr_sint_t ethr_atomic_cmpxchg(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val); +ethr_sint_t ethr_atomic_cmpxchg_rb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val); +ethr_sint_t ethr_atomic_cmpxchg_wb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val); +ethr_sint_t ethr_atomic_cmpxchg_acqb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val); +ethr_sint_t ethr_atomic_cmpxchg_relb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val); +ethr_sint_t ethr_atomic_cmpxchg_mb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val); +ethr_sint_t ethr_atomic_xchg(ethr_atomic_t *var, ethr_sint_t val); +ethr_sint_t ethr_atomic_xchg_rb(ethr_atomic_t *var, ethr_sint_t val); +ethr_sint_t ethr_atomic_xchg_wb(ethr_atomic_t *var, ethr_sint_t val); +ethr_sint_t ethr_atomic_xchg_acqb(ethr_atomic_t *var, ethr_sint_t val); +ethr_sint_t ethr_atomic_xchg_relb(ethr_atomic_t *var, ethr_sint_t val); +ethr_sint_t ethr_atomic_xchg_mb(ethr_atomic_t *var, ethr_sint_t val); +void ethr_atomic_set(ethr_atomic_t *var, ethr_sint_t val); +void ethr_atomic_set_rb(ethr_atomic_t *var, ethr_sint_t val); +void ethr_atomic_set_wb(ethr_atomic_t *var, ethr_sint_t val); +void ethr_atomic_set_acqb(ethr_atomic_t *var, ethr_sint_t val); +void ethr_atomic_set_relb(ethr_atomic_t *var, ethr_sint_t val); +void ethr_atomic_set_mb(ethr_atomic_t *var, ethr_sint_t val); +void ethr_atomic_init(ethr_atomic_t *var, ethr_sint_t val); +void ethr_atomic_init_rb(ethr_atomic_t *var, ethr_sint_t val); +void ethr_atomic_init_wb(ethr_atomic_t *var, ethr_sint_t val); +void ethr_atomic_init_acqb(ethr_atomic_t *var, ethr_sint_t val); +void ethr_atomic_init_relb(ethr_atomic_t *var, ethr_sint_t val); +void ethr_atomic_init_mb(ethr_atomic_t *var, ethr_sint_t val); +ethr_sint_t ethr_atomic_add_read(ethr_atomic_t *var, ethr_sint_t val); +ethr_sint_t ethr_atomic_add_read_rb(ethr_atomic_t *var, ethr_sint_t val); +ethr_sint_t ethr_atomic_add_read_wb(ethr_atomic_t *var, ethr_sint_t val); +ethr_sint_t ethr_atomic_add_read_acqb(ethr_atomic_t *var, ethr_sint_t val); +ethr_sint_t ethr_atomic_add_read_relb(ethr_atomic_t *var, ethr_sint_t val); +ethr_sint_t ethr_atomic_add_read_mb(ethr_atomic_t *var, ethr_sint_t val); +ethr_sint_t ethr_atomic_read(ethr_atomic_t *var); +ethr_sint_t ethr_atomic_read_rb(ethr_atomic_t *var); +ethr_sint_t ethr_atomic_read_wb(ethr_atomic_t *var); +ethr_sint_t ethr_atomic_read_acqb(ethr_atomic_t *var); +ethr_sint_t ethr_atomic_read_relb(ethr_atomic_t *var); +ethr_sint_t ethr_atomic_read_mb(ethr_atomic_t *var); +ethr_sint_t ethr_atomic_inc_read(ethr_atomic_t *var); +ethr_sint_t ethr_atomic_inc_read_rb(ethr_atomic_t *var); +ethr_sint_t ethr_atomic_inc_read_wb(ethr_atomic_t *var); +ethr_sint_t ethr_atomic_inc_read_acqb(ethr_atomic_t *var); +ethr_sint_t ethr_atomic_inc_read_relb(ethr_atomic_t *var); +ethr_sint_t ethr_atomic_inc_read_mb(ethr_atomic_t *var); +ethr_sint_t ethr_atomic_dec_read(ethr_atomic_t *var); +ethr_sint_t ethr_atomic_dec_read_rb(ethr_atomic_t *var); +ethr_sint_t ethr_atomic_dec_read_wb(ethr_atomic_t *var); +ethr_sint_t ethr_atomic_dec_read_acqb(ethr_atomic_t *var); +ethr_sint_t ethr_atomic_dec_read_relb(ethr_atomic_t *var); +ethr_sint_t ethr_atomic_dec_read_mb(ethr_atomic_t *var); +void ethr_atomic_add(ethr_atomic_t *var, ethr_sint_t val); +void ethr_atomic_add_rb(ethr_atomic_t *var, ethr_sint_t val); +void ethr_atomic_add_wb(ethr_atomic_t *var, ethr_sint_t val); +void ethr_atomic_add_acqb(ethr_atomic_t *var, ethr_sint_t val); +void ethr_atomic_add_relb(ethr_atomic_t *var, ethr_sint_t val); +void ethr_atomic_add_mb(ethr_atomic_t *var, ethr_sint_t val); +void ethr_atomic_inc(ethr_atomic_t *var); +void ethr_atomic_inc_rb(ethr_atomic_t *var); +void ethr_atomic_inc_wb(ethr_atomic_t *var); +void ethr_atomic_inc_acqb(ethr_atomic_t *var); +void ethr_atomic_inc_relb(ethr_atomic_t *var); +void ethr_atomic_inc_mb(ethr_atomic_t *var); +void ethr_atomic_dec(ethr_atomic_t *var); +void ethr_atomic_dec_rb(ethr_atomic_t *var); +void ethr_atomic_dec_wb(ethr_atomic_t *var); +void ethr_atomic_dec_acqb(ethr_atomic_t *var); +void ethr_atomic_dec_relb(ethr_atomic_t *var); +void ethr_atomic_dec_mb(ethr_atomic_t *var); +ethr_sint_t ethr_atomic_read_band(ethr_atomic_t *var, ethr_sint_t val); +ethr_sint_t ethr_atomic_read_band_rb(ethr_atomic_t *var, ethr_sint_t val); +ethr_sint_t ethr_atomic_read_band_wb(ethr_atomic_t *var, ethr_sint_t val); +ethr_sint_t ethr_atomic_read_band_acqb(ethr_atomic_t *var, ethr_sint_t val); +ethr_sint_t ethr_atomic_read_band_relb(ethr_atomic_t *var, ethr_sint_t val); +ethr_sint_t ethr_atomic_read_band_mb(ethr_atomic_t *var, ethr_sint_t val); +ethr_sint_t ethr_atomic_read_bor(ethr_atomic_t *var, ethr_sint_t val); +ethr_sint_t ethr_atomic_read_bor_rb(ethr_atomic_t *var, ethr_sint_t val); +ethr_sint_t ethr_atomic_read_bor_wb(ethr_atomic_t *var, ethr_sint_t val); +ethr_sint_t ethr_atomic_read_bor_acqb(ethr_atomic_t *var, ethr_sint_t val); +ethr_sint_t ethr_atomic_read_bor_relb(ethr_atomic_t *var, ethr_sint_t val); +ethr_sint_t ethr_atomic_read_bor_mb(ethr_atomic_t *var, ethr_sint_t val); +#endif /* ETHR_NEED_ATMC_PROTOTYPES__ */ + +#if (defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) \ + && (defined(ETHR_ATMC_INLINE__) || defined(ETHR_ATOMIC_IMPL__))) + +#if !defined(ETHR_NATMC_BITS__) +# error "Missing native atomic implementation" +#elif ETHR_NATMC_BITS__ == 64 +# undef ETHR_HAVE_NATMC_CMPXCHG +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG +# define ETHR_HAVE_NATMC_CMPXCHG 1 +# endif +# undef ETHR_HAVE_NATMC_CMPXCHG_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RB +# define ETHR_HAVE_NATMC_CMPXCHG_RB 1 +# endif +# undef ETHR_HAVE_NATMC_CMPXCHG_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_WB +# define ETHR_HAVE_NATMC_CMPXCHG_WB 1 +# endif +# undef ETHR_HAVE_NATMC_CMPXCHG_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_ACQB +# define ETHR_HAVE_NATMC_CMPXCHG_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC_CMPXCHG_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RELB +# define ETHR_HAVE_NATMC_CMPXCHG_RELB 1 +# endif +# undef ETHR_HAVE_NATMC_CMPXCHG_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_MB +# define ETHR_HAVE_NATMC_CMPXCHG_MB 1 +# endif +# undef ETHR_HAVE_NATMC_XCHG +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG +# define ETHR_HAVE_NATMC_XCHG 1 +# endif +# undef ETHR_HAVE_NATMC_XCHG_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_RB +# define ETHR_HAVE_NATMC_XCHG_RB 1 +# endif +# undef ETHR_HAVE_NATMC_XCHG_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_WB +# define ETHR_HAVE_NATMC_XCHG_WB 1 +# endif +# undef ETHR_HAVE_NATMC_XCHG_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_ACQB +# define ETHR_HAVE_NATMC_XCHG_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC_XCHG_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_RELB +# define ETHR_HAVE_NATMC_XCHG_RELB 1 +# endif +# undef ETHR_HAVE_NATMC_XCHG_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_MB +# define ETHR_HAVE_NATMC_XCHG_MB 1 +# endif +# undef ETHR_HAVE_NATMC_SET +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET +# define ETHR_HAVE_NATMC_SET 1 +# endif +# undef ETHR_HAVE_NATMC_SET_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RB +# define ETHR_HAVE_NATMC_SET_RB 1 +# endif +# undef ETHR_HAVE_NATMC_SET_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_WB +# define ETHR_HAVE_NATMC_SET_WB 1 +# endif +# undef ETHR_HAVE_NATMC_SET_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_ACQB +# define ETHR_HAVE_NATMC_SET_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC_SET_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RELB +# define ETHR_HAVE_NATMC_SET_RELB 1 +# endif +# undef ETHR_HAVE_NATMC_SET_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_MB +# define ETHR_HAVE_NATMC_SET_MB 1 +# endif +# undef ETHR_HAVE_NATMC_INIT +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT +# define ETHR_HAVE_NATMC_INIT 1 +# endif +# undef ETHR_HAVE_NATMC_INIT_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_RB +# define ETHR_HAVE_NATMC_INIT_RB 1 +# endif +# undef ETHR_HAVE_NATMC_INIT_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_WB +# define ETHR_HAVE_NATMC_INIT_WB 1 +# endif +# undef ETHR_HAVE_NATMC_INIT_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_ACQB +# define ETHR_HAVE_NATMC_INIT_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC_INIT_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_RELB +# define ETHR_HAVE_NATMC_INIT_RELB 1 +# endif +# undef ETHR_HAVE_NATMC_INIT_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_MB +# define ETHR_HAVE_NATMC_INIT_MB 1 +# endif +# undef ETHR_HAVE_NATMC_ADD_RETURN +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN +# define ETHR_HAVE_NATMC_ADD_RETURN 1 +# endif +# undef ETHR_HAVE_NATMC_ADD_RETURN_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_RB +# define ETHR_HAVE_NATMC_ADD_RETURN_RB 1 +# endif +# undef ETHR_HAVE_NATMC_ADD_RETURN_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_WB +# define ETHR_HAVE_NATMC_ADD_RETURN_WB 1 +# endif +# undef ETHR_HAVE_NATMC_ADD_RETURN_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_ACQB +# define ETHR_HAVE_NATMC_ADD_RETURN_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC_ADD_RETURN_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_RELB +# define ETHR_HAVE_NATMC_ADD_RETURN_RELB 1 +# endif +# undef ETHR_HAVE_NATMC_ADD_RETURN_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_MB +# define ETHR_HAVE_NATMC_ADD_RETURN_MB 1 +# endif +# undef ETHR_HAVE_NATMC_READ +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ +# define ETHR_HAVE_NATMC_READ 1 +# endif +# undef ETHR_HAVE_NATMC_READ_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_RB +# define ETHR_HAVE_NATMC_READ_RB 1 +# endif +# undef ETHR_HAVE_NATMC_READ_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_WB +# define ETHR_HAVE_NATMC_READ_WB 1 +# endif +# undef ETHR_HAVE_NATMC_READ_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_ACQB +# define ETHR_HAVE_NATMC_READ_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC_READ_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_RELB +# define ETHR_HAVE_NATMC_READ_RELB 1 +# endif +# undef ETHR_HAVE_NATMC_READ_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_MB +# define ETHR_HAVE_NATMC_READ_MB 1 +# endif +# undef ETHR_HAVE_NATMC_INC_RETURN +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN +# define ETHR_HAVE_NATMC_INC_RETURN 1 +# endif +# undef ETHR_HAVE_NATMC_INC_RETURN_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_RB +# define ETHR_HAVE_NATMC_INC_RETURN_RB 1 +# endif +# undef ETHR_HAVE_NATMC_INC_RETURN_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_WB +# define ETHR_HAVE_NATMC_INC_RETURN_WB 1 +# endif +# undef ETHR_HAVE_NATMC_INC_RETURN_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_ACQB +# define ETHR_HAVE_NATMC_INC_RETURN_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC_INC_RETURN_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_RELB +# define ETHR_HAVE_NATMC_INC_RETURN_RELB 1 +# endif +# undef ETHR_HAVE_NATMC_INC_RETURN_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_MB +# define ETHR_HAVE_NATMC_INC_RETURN_MB 1 +# endif +# undef ETHR_HAVE_NATMC_DEC_RETURN +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN +# define ETHR_HAVE_NATMC_DEC_RETURN 1 +# endif +# undef ETHR_HAVE_NATMC_DEC_RETURN_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_RB +# define ETHR_HAVE_NATMC_DEC_RETURN_RB 1 +# endif +# undef ETHR_HAVE_NATMC_DEC_RETURN_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_WB +# define ETHR_HAVE_NATMC_DEC_RETURN_WB 1 +# endif +# undef ETHR_HAVE_NATMC_DEC_RETURN_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_ACQB +# define ETHR_HAVE_NATMC_DEC_RETURN_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC_DEC_RETURN_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_RELB +# define ETHR_HAVE_NATMC_DEC_RETURN_RELB 1 +# endif +# undef ETHR_HAVE_NATMC_DEC_RETURN_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_MB +# define ETHR_HAVE_NATMC_DEC_RETURN_MB 1 +# endif +# undef ETHR_HAVE_NATMC_ADD +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD +# define ETHR_HAVE_NATMC_ADD 1 +# endif +# undef ETHR_HAVE_NATMC_ADD_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RB +# define ETHR_HAVE_NATMC_ADD_RB 1 +# endif +# undef ETHR_HAVE_NATMC_ADD_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_WB +# define ETHR_HAVE_NATMC_ADD_WB 1 +# endif +# undef ETHR_HAVE_NATMC_ADD_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_ACQB +# define ETHR_HAVE_NATMC_ADD_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC_ADD_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RELB +# define ETHR_HAVE_NATMC_ADD_RELB 1 +# endif +# undef ETHR_HAVE_NATMC_ADD_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_MB +# define ETHR_HAVE_NATMC_ADD_MB 1 +# endif +# undef ETHR_HAVE_NATMC_INC +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC +# define ETHR_HAVE_NATMC_INC 1 +# endif +# undef ETHR_HAVE_NATMC_INC_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RB +# define ETHR_HAVE_NATMC_INC_RB 1 +# endif +# undef ETHR_HAVE_NATMC_INC_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_WB +# define ETHR_HAVE_NATMC_INC_WB 1 +# endif +# undef ETHR_HAVE_NATMC_INC_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_ACQB +# define ETHR_HAVE_NATMC_INC_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC_INC_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RELB +# define ETHR_HAVE_NATMC_INC_RELB 1 +# endif +# undef ETHR_HAVE_NATMC_INC_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_MB +# define ETHR_HAVE_NATMC_INC_MB 1 +# endif +# undef ETHR_HAVE_NATMC_DEC +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC +# define ETHR_HAVE_NATMC_DEC 1 +# endif +# undef ETHR_HAVE_NATMC_DEC_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RB +# define ETHR_HAVE_NATMC_DEC_RB 1 +# endif +# undef ETHR_HAVE_NATMC_DEC_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_WB +# define ETHR_HAVE_NATMC_DEC_WB 1 +# endif +# undef ETHR_HAVE_NATMC_DEC_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_ACQB +# define ETHR_HAVE_NATMC_DEC_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC_DEC_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RELB +# define ETHR_HAVE_NATMC_DEC_RELB 1 +# endif +# undef ETHR_HAVE_NATMC_DEC_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_MB +# define ETHR_HAVE_NATMC_DEC_MB 1 +# endif +# undef ETHR_HAVE_NATMC_AND_RETOLD +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD +# define ETHR_HAVE_NATMC_AND_RETOLD 1 +# endif +# undef ETHR_HAVE_NATMC_AND_RETOLD_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_RB +# define ETHR_HAVE_NATMC_AND_RETOLD_RB 1 +# endif +# undef ETHR_HAVE_NATMC_AND_RETOLD_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_WB +# define ETHR_HAVE_NATMC_AND_RETOLD_WB 1 +# endif +# undef ETHR_HAVE_NATMC_AND_RETOLD_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_ACQB +# define ETHR_HAVE_NATMC_AND_RETOLD_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC_AND_RETOLD_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_RELB +# define ETHR_HAVE_NATMC_AND_RETOLD_RELB 1 +# endif +# undef ETHR_HAVE_NATMC_AND_RETOLD_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_MB +# define ETHR_HAVE_NATMC_AND_RETOLD_MB 1 +# endif +# undef ETHR_HAVE_NATMC_OR_RETOLD +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD +# define ETHR_HAVE_NATMC_OR_RETOLD 1 +# endif +# undef ETHR_HAVE_NATMC_OR_RETOLD_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_RB +# define ETHR_HAVE_NATMC_OR_RETOLD_RB 1 +# endif +# undef ETHR_HAVE_NATMC_OR_RETOLD_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_WB +# define ETHR_HAVE_NATMC_OR_RETOLD_WB 1 +# endif +# undef ETHR_HAVE_NATMC_OR_RETOLD_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_ACQB +# define ETHR_HAVE_NATMC_OR_RETOLD_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC_OR_RETOLD_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_RELB +# define ETHR_HAVE_NATMC_OR_RETOLD_RELB 1 +# endif +# undef ETHR_HAVE_NATMC_OR_RETOLD_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_MB +# define ETHR_HAVE_NATMC_OR_RETOLD_MB 1 +# endif +#elif ETHR_NATMC_BITS__ == 32 +# undef ETHR_HAVE_NATMC_CMPXCHG +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG +# define ETHR_HAVE_NATMC_CMPXCHG 1 +# endif +# undef ETHR_HAVE_NATMC_CMPXCHG_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_RB +# define ETHR_HAVE_NATMC_CMPXCHG_RB 1 +# endif +# undef ETHR_HAVE_NATMC_CMPXCHG_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_WB +# define ETHR_HAVE_NATMC_CMPXCHG_WB 1 +# endif +# undef ETHR_HAVE_NATMC_CMPXCHG_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_ACQB +# define ETHR_HAVE_NATMC_CMPXCHG_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC_CMPXCHG_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_RELB +# define ETHR_HAVE_NATMC_CMPXCHG_RELB 1 +# endif +# undef ETHR_HAVE_NATMC_CMPXCHG_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_MB +# define ETHR_HAVE_NATMC_CMPXCHG_MB 1 +# endif +# undef ETHR_HAVE_NATMC_XCHG +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG +# define ETHR_HAVE_NATMC_XCHG 1 +# endif +# undef ETHR_HAVE_NATMC_XCHG_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_RB +# define ETHR_HAVE_NATMC_XCHG_RB 1 +# endif +# undef ETHR_HAVE_NATMC_XCHG_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_WB +# define ETHR_HAVE_NATMC_XCHG_WB 1 +# endif +# undef ETHR_HAVE_NATMC_XCHG_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_ACQB +# define ETHR_HAVE_NATMC_XCHG_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC_XCHG_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_RELB +# define ETHR_HAVE_NATMC_XCHG_RELB 1 +# endif +# undef ETHR_HAVE_NATMC_XCHG_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_MB +# define ETHR_HAVE_NATMC_XCHG_MB 1 +# endif +# undef ETHR_HAVE_NATMC_SET +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET +# define ETHR_HAVE_NATMC_SET 1 +# endif +# undef ETHR_HAVE_NATMC_SET_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_RB +# define ETHR_HAVE_NATMC_SET_RB 1 +# endif +# undef ETHR_HAVE_NATMC_SET_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_WB +# define ETHR_HAVE_NATMC_SET_WB 1 +# endif +# undef ETHR_HAVE_NATMC_SET_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_ACQB +# define ETHR_HAVE_NATMC_SET_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC_SET_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_RELB +# define ETHR_HAVE_NATMC_SET_RELB 1 +# endif +# undef ETHR_HAVE_NATMC_SET_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_MB +# define ETHR_HAVE_NATMC_SET_MB 1 +# endif +# undef ETHR_HAVE_NATMC_INIT +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT +# define ETHR_HAVE_NATMC_INIT 1 +# endif +# undef ETHR_HAVE_NATMC_INIT_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_RB +# define ETHR_HAVE_NATMC_INIT_RB 1 +# endif +# undef ETHR_HAVE_NATMC_INIT_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_WB +# define ETHR_HAVE_NATMC_INIT_WB 1 +# endif +# undef ETHR_HAVE_NATMC_INIT_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_ACQB +# define ETHR_HAVE_NATMC_INIT_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC_INIT_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_RELB +# define ETHR_HAVE_NATMC_INIT_RELB 1 +# endif +# undef ETHR_HAVE_NATMC_INIT_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_MB +# define ETHR_HAVE_NATMC_INIT_MB 1 +# endif +# undef ETHR_HAVE_NATMC_ADD_RETURN +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN +# define ETHR_HAVE_NATMC_ADD_RETURN 1 +# endif +# undef ETHR_HAVE_NATMC_ADD_RETURN_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_RB +# define ETHR_HAVE_NATMC_ADD_RETURN_RB 1 +# endif +# undef ETHR_HAVE_NATMC_ADD_RETURN_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_WB +# define ETHR_HAVE_NATMC_ADD_RETURN_WB 1 +# endif +# undef ETHR_HAVE_NATMC_ADD_RETURN_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_ACQB +# define ETHR_HAVE_NATMC_ADD_RETURN_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC_ADD_RETURN_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_RELB +# define ETHR_HAVE_NATMC_ADD_RETURN_RELB 1 +# endif +# undef ETHR_HAVE_NATMC_ADD_RETURN_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_MB +# define ETHR_HAVE_NATMC_ADD_RETURN_MB 1 +# endif +# undef ETHR_HAVE_NATMC_READ +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ +# define ETHR_HAVE_NATMC_READ 1 +# endif +# undef ETHR_HAVE_NATMC_READ_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_RB +# define ETHR_HAVE_NATMC_READ_RB 1 +# endif +# undef ETHR_HAVE_NATMC_READ_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_WB +# define ETHR_HAVE_NATMC_READ_WB 1 +# endif +# undef ETHR_HAVE_NATMC_READ_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_ACQB +# define ETHR_HAVE_NATMC_READ_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC_READ_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_RELB +# define ETHR_HAVE_NATMC_READ_RELB 1 +# endif +# undef ETHR_HAVE_NATMC_READ_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_MB +# define ETHR_HAVE_NATMC_READ_MB 1 +# endif +# undef ETHR_HAVE_NATMC_INC_RETURN +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN +# define ETHR_HAVE_NATMC_INC_RETURN 1 +# endif +# undef ETHR_HAVE_NATMC_INC_RETURN_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_RB +# define ETHR_HAVE_NATMC_INC_RETURN_RB 1 +# endif +# undef ETHR_HAVE_NATMC_INC_RETURN_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_WB +# define ETHR_HAVE_NATMC_INC_RETURN_WB 1 +# endif +# undef ETHR_HAVE_NATMC_INC_RETURN_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_ACQB +# define ETHR_HAVE_NATMC_INC_RETURN_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC_INC_RETURN_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_RELB +# define ETHR_HAVE_NATMC_INC_RETURN_RELB 1 +# endif +# undef ETHR_HAVE_NATMC_INC_RETURN_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_MB +# define ETHR_HAVE_NATMC_INC_RETURN_MB 1 +# endif +# undef ETHR_HAVE_NATMC_DEC_RETURN +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN +# define ETHR_HAVE_NATMC_DEC_RETURN 1 +# endif +# undef ETHR_HAVE_NATMC_DEC_RETURN_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_RB +# define ETHR_HAVE_NATMC_DEC_RETURN_RB 1 +# endif +# undef ETHR_HAVE_NATMC_DEC_RETURN_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_WB +# define ETHR_HAVE_NATMC_DEC_RETURN_WB 1 +# endif +# undef ETHR_HAVE_NATMC_DEC_RETURN_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_ACQB +# define ETHR_HAVE_NATMC_DEC_RETURN_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC_DEC_RETURN_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_RELB +# define ETHR_HAVE_NATMC_DEC_RETURN_RELB 1 +# endif +# undef ETHR_HAVE_NATMC_DEC_RETURN_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_MB +# define ETHR_HAVE_NATMC_DEC_RETURN_MB 1 +# endif +# undef ETHR_HAVE_NATMC_ADD +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD +# define ETHR_HAVE_NATMC_ADD 1 +# endif +# undef ETHR_HAVE_NATMC_ADD_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RB +# define ETHR_HAVE_NATMC_ADD_RB 1 +# endif +# undef ETHR_HAVE_NATMC_ADD_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_WB +# define ETHR_HAVE_NATMC_ADD_WB 1 +# endif +# undef ETHR_HAVE_NATMC_ADD_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_ACQB +# define ETHR_HAVE_NATMC_ADD_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC_ADD_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RELB +# define ETHR_HAVE_NATMC_ADD_RELB 1 +# endif +# undef ETHR_HAVE_NATMC_ADD_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_MB +# define ETHR_HAVE_NATMC_ADD_MB 1 +# endif +# undef ETHR_HAVE_NATMC_INC +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC +# define ETHR_HAVE_NATMC_INC 1 +# endif +# undef ETHR_HAVE_NATMC_INC_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RB +# define ETHR_HAVE_NATMC_INC_RB 1 +# endif +# undef ETHR_HAVE_NATMC_INC_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_WB +# define ETHR_HAVE_NATMC_INC_WB 1 +# endif +# undef ETHR_HAVE_NATMC_INC_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_ACQB +# define ETHR_HAVE_NATMC_INC_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC_INC_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RELB +# define ETHR_HAVE_NATMC_INC_RELB 1 +# endif +# undef ETHR_HAVE_NATMC_INC_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_MB +# define ETHR_HAVE_NATMC_INC_MB 1 +# endif +# undef ETHR_HAVE_NATMC_DEC +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC +# define ETHR_HAVE_NATMC_DEC 1 +# endif +# undef ETHR_HAVE_NATMC_DEC_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RB +# define ETHR_HAVE_NATMC_DEC_RB 1 +# endif +# undef ETHR_HAVE_NATMC_DEC_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_WB +# define ETHR_HAVE_NATMC_DEC_WB 1 +# endif +# undef ETHR_HAVE_NATMC_DEC_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_ACQB +# define ETHR_HAVE_NATMC_DEC_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC_DEC_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RELB +# define ETHR_HAVE_NATMC_DEC_RELB 1 +# endif +# undef ETHR_HAVE_NATMC_DEC_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_MB +# define ETHR_HAVE_NATMC_DEC_MB 1 +# endif +# undef ETHR_HAVE_NATMC_AND_RETOLD +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD +# define ETHR_HAVE_NATMC_AND_RETOLD 1 +# endif +# undef ETHR_HAVE_NATMC_AND_RETOLD_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_RB +# define ETHR_HAVE_NATMC_AND_RETOLD_RB 1 +# endif +# undef ETHR_HAVE_NATMC_AND_RETOLD_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_WB +# define ETHR_HAVE_NATMC_AND_RETOLD_WB 1 +# endif +# undef ETHR_HAVE_NATMC_AND_RETOLD_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_ACQB +# define ETHR_HAVE_NATMC_AND_RETOLD_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC_AND_RETOLD_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_RELB +# define ETHR_HAVE_NATMC_AND_RETOLD_RELB 1 +# endif +# undef ETHR_HAVE_NATMC_AND_RETOLD_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_MB +# define ETHR_HAVE_NATMC_AND_RETOLD_MB 1 +# endif +# undef ETHR_HAVE_NATMC_OR_RETOLD +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD +# define ETHR_HAVE_NATMC_OR_RETOLD 1 +# endif +# undef ETHR_HAVE_NATMC_OR_RETOLD_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_RB +# define ETHR_HAVE_NATMC_OR_RETOLD_RB 1 +# endif +# undef ETHR_HAVE_NATMC_OR_RETOLD_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_WB +# define ETHR_HAVE_NATMC_OR_RETOLD_WB 1 +# endif +# undef ETHR_HAVE_NATMC_OR_RETOLD_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_ACQB +# define ETHR_HAVE_NATMC_OR_RETOLD_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC_OR_RETOLD_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_RELB +# define ETHR_HAVE_NATMC_OR_RETOLD_RELB 1 +# endif +# undef ETHR_HAVE_NATMC_OR_RETOLD_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_MB +# define ETHR_HAVE_NATMC_OR_RETOLD_MB 1 +# endif +#else +# error "Invalid native atomic size" #endif +#if (!defined(ETHR_HAVE_NATMC_CMPXCHG) \ + && !defined(ETHR_HAVE_NATMC_CMPXCHG_RB) \ + && !defined(ETHR_HAVE_NATMC_CMPXCHG_WB) \ + && !defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) \ + && !defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) \ + && !defined(ETHR_HAVE_NATMC_CMPXCHG_MB)) +# error "No native cmpxchg() op available" +#endif + + /* - * --- Pointer size atomics --------------------------------------------------- + * Read op used together with cmpxchg() fallback when no native op present. */ +#if defined(ETHR_HAVE_NATMC_READ) +#define ETHR_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \ + ETHR_NATMC_FUNC__(read)(VAR) +#elif defined(ETHR_HAVE_NATMC_READ_RB) +#define ETHR_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \ + ETHR_NATMC_FUNC__(read_rb)(VAR) +#elif defined(ETHR_HAVE_NATMC_READ_WB) +#define ETHR_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \ + ETHR_NATMC_FUNC__(read_wb)(VAR) +#elif defined(ETHR_HAVE_NATMC_READ_ACQB) +#define ETHR_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \ + ETHR_NATMC_FUNC__(read_acqb)(VAR) +#elif defined(ETHR_HAVE_NATMC_READ_RELB) +#define ETHR_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \ + ETHR_NATMC_FUNC__(read_relb)(VAR) +#elif defined(ETHR_HAVE_NATMC_READ_MB) +#define ETHR_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \ + ETHR_NATMC_FUNC__(read_mb)(VAR) +#else +/* + * We have no native read() op; guess zero and then use the + * the atomics actual value returned from cmpxchg(). + */ +#define ETHR_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \ + ((ETHR_NAINT_T__) 0) +#endif + +/* + * Native cmpxchg() fallback used when no native op present. + */ +#define ETHR_NATMC_CMPXCHG_FALLBACK__(CMPXCHG, VAR, AVAL, OPS) \ +do { \ + ethr_sint_t AVAL; \ + ETHR_NAINT_T__ new__, act__, exp__; \ + act__ = ETHR_NATMC_CMPXCHG_FALLBACK_READ__(VAR); \ + do { \ + exp__ = act__; \ + AVAL = (ethr_sint_t) act__; \ + { OPS; } \ + new__ = (ETHR_NAINT_T__) AVAL; \ + act__ = CMPXCHG(VAR, new__, exp__); \ + } while (__builtin_expect(act__ != exp__, 0)); \ +} while (0) + + + +/* --- addr() --- */ -static ETHR_INLINE ethr_sint_t * -ETHR_INLINE_FUNC_NAME_(ethr_atomic_addr)(ethr_atomic_t *var) +static ETHR_INLINE ethr_sint_t *ETHR_ATMC_FUNC__(addr)(ethr_atomic_t *var) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS return (ethr_sint_t *) ETHR_NATMC_ADDR_FUNC__(var); + +} + + +/* --- cmpxchg() --- */ + + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(cmpxchg)(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val) +{ + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_CMPXCHG) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); #else - return (ethr_sint_t *) var; +#error "Missing implementation of ethr_atomic_cmpxchg()!" #endif + return res; } -static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_atomic_init)(ethr_atomic_t *var, ethr_sint_t i) +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(cmpxchg_rb)(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - ETHR_NATMC_FUNC__(init)(var, (ETHR_NAINT_T__) i); + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); + ETHR_MEMBAR(ETHR_LoadLoad); #else - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = i); +#error "Missing implementation of ethr_atomic_cmpxchg_rb()!" #endif + return res; } -static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_atomic_set)(ethr_atomic_t *var, ethr_sint_t i) +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(cmpxchg_wb)(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - ETHR_NATMC_FUNC__(set)(var, (ETHR_NAINT_T__) i); + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); #else - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = i); +#error "Missing implementation of ethr_atomic_cmpxchg_wb()!" #endif + return res; } -static ETHR_INLINE ethr_sint_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic_read)(ethr_atomic_t *var) +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(cmpxchg_acqb)(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return (ethr_sint_t) ETHR_NATMC_FUNC__(read)(var); + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); #else +#error "Missing implementation of ethr_atomic_cmpxchg_acqb()!" +#endif + return res; +} + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(cmpxchg_relb)(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val) +{ ethr_sint_t res; - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var); +#if defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); +#else +#error "Missing implementation of ethr_atomic_cmpxchg_relb()!" +#endif return res; +} + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(cmpxchg_mb)(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val) +{ + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else +#error "Missing implementation of ethr_atomic_cmpxchg_mb()!" #endif + return res; } -static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_atomic_add)(ethr_atomic_t *var, ethr_sint_t incr) + +/* --- xchg() --- */ + + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(xchg)(ethr_atomic_t *var, ethr_sint_t val) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - ETHR_NATMC_FUNC__(add)(var, (ETHR_NAINT_T__) incr); + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_XCHG) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_XCHG_RB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_rb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_XCHG_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_wb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_XCHG_ACQB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_XCHG_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_relb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_XCHG_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val); #else - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += incr); +#error "Missing implementation of ethr_atomic_xchg()!" #endif -} - -static ETHR_INLINE ethr_sint_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic_add_read)(ethr_atomic_t *var, ethr_sint_t i) + return res; +} + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(xchg_rb)(ethr_atomic_t *var, ethr_sint_t val) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return (ethr_sint_t) ETHR_NATMC_FUNC__(add_return)(var, (ETHR_NAINT_T__) i); + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_XCHG_RB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_rb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_XCHG) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_XCHG_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_XCHG_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_wb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_XCHG_ACQB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_acqb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_XCHG_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_relb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval = val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val); + ETHR_MEMBAR(ETHR_LoadLoad); #else +#error "Missing implementation of ethr_atomic_xchg_rb()!" +#endif + return res; +} + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(xchg_wb)(ethr_atomic_t *var, ethr_sint_t val) +{ ethr_sint_t res; - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += i; res = *var); +#if defined(ETHR_HAVE_NATMC_XCHG_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_wb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_XCHG) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_XCHG_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_XCHG_RB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_rb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_XCHG_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_XCHG_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_relb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val); +#else +#error "Missing implementation of ethr_atomic_xchg_wb()!" +#endif return res; +} + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(xchg_acqb)(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_XCHG_ACQB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_XCHG_RB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_rb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_XCHG) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_XCHG_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_XCHG_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_wb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_XCHG_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_relb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval = val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else +#error "Missing implementation of ethr_atomic_xchg_acqb()!" #endif + return res; } -static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc)(ethr_atomic_t *var) +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(xchg_relb)(ethr_atomic_t *var, ethr_sint_t val) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - ETHR_NATMC_FUNC__(inc)(var); + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_XCHG_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_relb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_XCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_wb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_XCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_XCHG_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_XCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_rb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_XCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val); #else - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var)); +#error "Missing implementation of ethr_atomic_xchg_relb()!" #endif + return res; } -static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec)(ethr_atomic_t *var) +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(xchg_mb)(ethr_atomic_t *var, ethr_sint_t val) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - ETHR_NATMC_FUNC__(dec)(var); + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_XCHG_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_XCHG_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_relb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_XCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_XCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_wb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_XCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_rb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_XCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval = val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else +#error "Missing implementation of ethr_atomic_xchg_mb()!" +#endif + return res; +} + + +/* --- set() --- */ + + +static ETHR_INLINE void ETHR_ATMC_FUNC__(set)(ethr_atomic_t *var, ethr_sint_t val) +{ +#if defined(ETHR_HAVE_NATMC_SET) + ETHR_NATMC_FUNC__(set)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_SET_RB) + ETHR_NATMC_FUNC__(set_rb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_SET_WB) + ETHR_NATMC_FUNC__(set_wb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_SET_ACQB) + ETHR_NATMC_FUNC__(set_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_SET_RELB) + ETHR_NATMC_FUNC__(set_relb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_SET_MB) + ETHR_NATMC_FUNC__(set_mb)(var, (ETHR_NAINT_T__) val); +#else + (void) ETHR_ATMC_FUNC__(xchg)(var, val); +#endif +} + +static ETHR_INLINE void ETHR_ATMC_FUNC__(set_rb)(ethr_atomic_t *var, ethr_sint_t val) +{ +#if defined(ETHR_HAVE_NATMC_SET_RB) + ETHR_NATMC_FUNC__(set_rb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_SET) + ETHR_NATMC_FUNC__(set)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_SET_MB) + ETHR_NATMC_FUNC__(set_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_SET_WB) + ETHR_NATMC_FUNC__(set_wb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_SET_ACQB) + ETHR_NATMC_FUNC__(set_acqb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_SET_RELB) + ETHR_NATMC_FUNC__(set_relb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + (void) ETHR_ATMC_FUNC__(xchg_rb)(var, val); +#endif +} + +static ETHR_INLINE void ETHR_ATMC_FUNC__(set_wb)(ethr_atomic_t *var, ethr_sint_t val) +{ +#if defined(ETHR_HAVE_NATMC_SET_WB) + ETHR_NATMC_FUNC__(set_wb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_SET) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_FUNC__(set)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_SET_MB) + ETHR_NATMC_FUNC__(set_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_SET_RB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_FUNC__(set_rb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_SET_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_FUNC__(set_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_SET_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_FUNC__(set_relb)(var, (ETHR_NAINT_T__) val); +#else + (void) ETHR_ATMC_FUNC__(xchg_wb)(var, val); +#endif +} + +static ETHR_INLINE void ETHR_ATMC_FUNC__(set_acqb)(ethr_atomic_t *var, ethr_sint_t val) +{ +#if defined(ETHR_HAVE_NATMC_SET_ACQB) + ETHR_NATMC_FUNC__(set_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_SET_RB) + ETHR_NATMC_FUNC__(set_rb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_SET) + ETHR_NATMC_FUNC__(set)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_SET_MB) + ETHR_NATMC_FUNC__(set_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_SET_WB) + ETHR_NATMC_FUNC__(set_wb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_SET_RELB) + ETHR_NATMC_FUNC__(set_relb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#else + (void) ETHR_ATMC_FUNC__(xchg_acqb)(var, val); +#endif +} + +static ETHR_INLINE void ETHR_ATMC_FUNC__(set_relb)(ethr_atomic_t *var, ethr_sint_t val) +{ +#if defined(ETHR_HAVE_NATMC_SET_RELB) + ETHR_NATMC_FUNC__(set_relb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_SET_WB) + ETHR_MEMBAR(ETHR_LoadStore); + ETHR_NATMC_FUNC__(set_wb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_SET) + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_NATMC_FUNC__(set)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_SET_MB) + ETHR_NATMC_FUNC__(set_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_SET_RB) + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_NATMC_FUNC__(set_rb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_SET_ACQB) + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_NATMC_FUNC__(set_acqb)(var, (ETHR_NAINT_T__) val); +#else + (void) ETHR_ATMC_FUNC__(xchg_relb)(var, val); +#endif +} + +static ETHR_INLINE void ETHR_ATMC_FUNC__(set_mb)(ethr_atomic_t *var, ethr_sint_t val) +{ +#if defined(ETHR_HAVE_NATMC_SET_MB) + ETHR_NATMC_FUNC__(set_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_SET_RELB) + ETHR_NATMC_FUNC__(set_relb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_SET_ACQB) + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_NATMC_FUNC__(set_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_SET_WB) + ETHR_MEMBAR(ETHR_LoadStore); + ETHR_NATMC_FUNC__(set_wb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_SET_RB) + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_NATMC_FUNC__(set_rb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_SET) + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_NATMC_FUNC__(set)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#else + (void) ETHR_ATMC_FUNC__(xchg_mb)(var, val); +#endif +} + + +/* --- init() --- */ + + +static ETHR_INLINE void ETHR_ATMC_FUNC__(init)(ethr_atomic_t *var, ethr_sint_t val) +{ +#if defined(ETHR_HAVE_NATMC_INIT) + ETHR_NATMC_FUNC__(init)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_INIT_RB) + ETHR_NATMC_FUNC__(init_rb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_INIT_WB) + ETHR_NATMC_FUNC__(init_wb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_INIT_ACQB) + ETHR_NATMC_FUNC__(init_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_INIT_RELB) + ETHR_NATMC_FUNC__(init_relb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_INIT_MB) + ETHR_NATMC_FUNC__(init_mb)(var, (ETHR_NAINT_T__) val); +#else + ETHR_ATMC_FUNC__(set)(var, val); +#endif +} + +static ETHR_INLINE void ETHR_ATMC_FUNC__(init_rb)(ethr_atomic_t *var, ethr_sint_t val) +{ +#if defined(ETHR_HAVE_NATMC_INIT_RB) + ETHR_NATMC_FUNC__(init_rb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_INIT) + ETHR_NATMC_FUNC__(init)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_INIT_MB) + ETHR_NATMC_FUNC__(init_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_INIT_WB) + ETHR_NATMC_FUNC__(init_wb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_INIT_ACQB) + ETHR_NATMC_FUNC__(init_acqb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_INIT_RELB) + ETHR_NATMC_FUNC__(init_relb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + ETHR_ATMC_FUNC__(set_rb)(var, val); +#endif +} + +static ETHR_INLINE void ETHR_ATMC_FUNC__(init_wb)(ethr_atomic_t *var, ethr_sint_t val) +{ +#if defined(ETHR_HAVE_NATMC_INIT_WB) + ETHR_NATMC_FUNC__(init_wb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_INIT) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_FUNC__(init)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_INIT_MB) + ETHR_NATMC_FUNC__(init_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_INIT_RB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_FUNC__(init_rb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_INIT_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_FUNC__(init_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_INIT_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_FUNC__(init_relb)(var, (ETHR_NAINT_T__) val); +#else + ETHR_ATMC_FUNC__(set_wb)(var, val); +#endif +} + +static ETHR_INLINE void ETHR_ATMC_FUNC__(init_acqb)(ethr_atomic_t *var, ethr_sint_t val) +{ +#if defined(ETHR_HAVE_NATMC_INIT_ACQB) + ETHR_NATMC_FUNC__(init_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_INIT_RB) + ETHR_NATMC_FUNC__(init_rb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_INIT) + ETHR_NATMC_FUNC__(init)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_INIT_MB) + ETHR_NATMC_FUNC__(init_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_INIT_WB) + ETHR_NATMC_FUNC__(init_wb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_INIT_RELB) + ETHR_NATMC_FUNC__(init_relb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_ATMC_FUNC__(set_acqb)(var, val); +#endif +} + +static ETHR_INLINE void ETHR_ATMC_FUNC__(init_relb)(ethr_atomic_t *var, ethr_sint_t val) +{ +#if defined(ETHR_HAVE_NATMC_INIT_RELB) + ETHR_NATMC_FUNC__(init_relb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_INIT_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad); + ETHR_NATMC_FUNC__(init_wb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_INIT) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_NATMC_FUNC__(init)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_INIT_MB) + ETHR_NATMC_FUNC__(init_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_INIT_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_NATMC_FUNC__(init_rb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_INIT_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_NATMC_FUNC__(init_acqb)(var, (ETHR_NAINT_T__) val); +#else + ETHR_ATMC_FUNC__(set_relb)(var, val); +#endif +} + +static ETHR_INLINE void ETHR_ATMC_FUNC__(init_mb)(ethr_atomic_t *var, ethr_sint_t val) +{ +#if defined(ETHR_HAVE_NATMC_INIT_MB) + ETHR_NATMC_FUNC__(init_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_INIT_RELB) + ETHR_NATMC_FUNC__(init_relb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_INIT_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_NATMC_FUNC__(init_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_INIT_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad); + ETHR_NATMC_FUNC__(init_wb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_INIT_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_NATMC_FUNC__(init_rb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_INIT) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_NATMC_FUNC__(init)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_ATMC_FUNC__(set_mb)(var, val); +#endif +} + + +/* --- add_read() --- */ + + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(add_read)(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_ADD_RETURN) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_rb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_wb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_ACQB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_relb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval); +#else +#error "Missing implementation of ethr_atomic_add_read()!" +#endif + return res; +} + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(add_read_rb)(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_ADD_RETURN_RB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_rb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_wb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_ACQB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_acqb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_relb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, aval += val; res = aval); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval); + ETHR_MEMBAR(ETHR_LoadLoad); +#else +#error "Missing implementation of ethr_atomic_add_read_rb()!" +#endif + return res; +} + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(add_read_wb)(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_ADD_RETURN_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_wb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_rb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_relb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval); +#else +#error "Missing implementation of ethr_atomic_add_read_wb()!" +#endif + return res; +} + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(add_read_acqb)(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_ADD_RETURN_ACQB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_rb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_wb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_relb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, aval += val; res = aval); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else +#error "Missing implementation of ethr_atomic_add_read_acqb()!" +#endif + return res; +} + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(add_read_relb)(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_ADD_RETURN_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_relb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_wb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_rb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval); #else - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var)); +#error "Missing implementation of ethr_atomic_add_read_relb()!" #endif + return res; } -static ETHR_INLINE ethr_sint_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc_read)(ethr_atomic_t *var) +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(add_read_mb)(ethr_atomic_t *var, ethr_sint_t val) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return)(var); + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_ADD_RETURN_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_relb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_wb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_rb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, aval += val; res = aval); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); #else +#error "Missing implementation of ethr_atomic_add_read_mb()!" +#endif + return res; +} + + +/* --- read() --- */ + + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read)(ethr_atomic_t *var) +{ ethr_sint_t res; - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var)); +#if defined(ETHR_HAVE_NATMC_READ) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read)(var); +#elif defined(ETHR_HAVE_NATMC_READ_RB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_rb)(var); +#elif defined(ETHR_HAVE_NATMC_READ_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_wb)(var); +#elif defined(ETHR_HAVE_NATMC_READ_ACQB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_acqb)(var); +#elif defined(ETHR_HAVE_NATMC_READ_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_relb)(var); +#elif defined(ETHR_HAVE_NATMC_READ_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_mb)(var); +#else + res = ETHR_ATMC_FUNC__(cmpxchg)(var, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__); +#endif return res; +} + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_rb)(ethr_atomic_t *var) +{ + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_READ_RB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_rb)(var); +#elif defined(ETHR_HAVE_NATMC_READ) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_READ_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_mb)(var); +#elif defined(ETHR_HAVE_NATMC_READ_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_READ_ACQB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_acqb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_READ_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + res = ETHR_ATMC_FUNC__(cmpxchg_rb)(var, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__); #endif + return res; } -static ETHR_INLINE ethr_sint_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_read)(ethr_atomic_t *var) +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_wb)(ethr_atomic_t *var) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return)(var); + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_READ_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_wb)(var); +#elif defined(ETHR_HAVE_NATMC_READ) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read)(var); +#elif defined(ETHR_HAVE_NATMC_READ_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_mb)(var); +#elif defined(ETHR_HAVE_NATMC_READ_RB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_rb)(var); +#elif defined(ETHR_HAVE_NATMC_READ_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_acqb)(var); +#elif defined(ETHR_HAVE_NATMC_READ_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_relb)(var); #else + res = ETHR_ATMC_FUNC__(cmpxchg_wb)(var, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__); +#endif + return res; +} + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_acqb)(ethr_atomic_t *var) +{ ethr_sint_t res; - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var)); +#if defined(ETHR_HAVE_NATMC_READ_ACQB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_acqb)(var); +#elif defined(ETHR_HAVE_NATMC_READ_RB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_rb)(var); + ETHR_MEMBAR(ETHR_LoadStore); +#elif defined(ETHR_HAVE_NATMC_READ) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#elif defined(ETHR_HAVE_NATMC_READ_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_mb)(var); +#elif defined(ETHR_HAVE_NATMC_READ_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#elif defined(ETHR_HAVE_NATMC_READ_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#else + res = ETHR_ATMC_FUNC__(cmpxchg_acqb)(var, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__); +#endif return res; +} + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_relb)(ethr_atomic_t *var) +{ + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_READ_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_relb)(var); +#elif defined(ETHR_HAVE_NATMC_READ_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_wb)(var); +#elif defined(ETHR_HAVE_NATMC_READ) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read)(var); +#elif defined(ETHR_HAVE_NATMC_READ_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_mb)(var); +#elif defined(ETHR_HAVE_NATMC_READ_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_rb)(var); +#elif defined(ETHR_HAVE_NATMC_READ_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_acqb)(var); +#else + res = ETHR_ATMC_FUNC__(cmpxchg_relb)(var, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__); #endif + return res; } -static ETHR_INLINE ethr_sint_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic_read_band)(ethr_atomic_t *var, - ethr_sint_t mask) +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_mb)(ethr_atomic_t *var) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold)(var, - (ETHR_NAINT_T__) mask); + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_READ_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_mb)(var); +#elif defined(ETHR_HAVE_NATMC_READ_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#elif defined(ETHR_HAVE_NATMC_READ_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_acqb)(var); +#elif defined(ETHR_HAVE_NATMC_READ_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#elif defined(ETHR_HAVE_NATMC_READ_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_rb)(var); + ETHR_MEMBAR(ETHR_LoadStore); +#elif defined(ETHR_HAVE_NATMC_READ) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); #else + res = ETHR_ATMC_FUNC__(cmpxchg_mb)(var, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__); +#endif + return res; +} + + +/* --- inc_read() --- */ + + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(inc_read)(ethr_atomic_t *var) +{ ethr_sint_t res; - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= mask); +#if defined(ETHR_HAVE_NATMC_INC_RETURN) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return)(var); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN_RB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_rb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_wb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN_ACQB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_acqb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_relb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_mb)(var); +#else + res = ETHR_ATMC_FUNC__(add_read)(var, (ethr_sint_t) 1); +#endif return res; +} + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(inc_read_rb)(ethr_atomic_t *var) +{ + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_INC_RETURN_RB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_rb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_mb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN_ACQB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_acqb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + res = ETHR_ATMC_FUNC__(add_read_rb)(var, (ethr_sint_t) 1); #endif + return res; } -static ETHR_INLINE ethr_sint_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic_read_bor)(ethr_atomic_t *var, - ethr_sint_t mask) +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(inc_read_wb)(ethr_atomic_t *var) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold)(var, - (ETHR_NAINT_T__) mask); + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_INC_RETURN_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_wb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return)(var); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_mb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN_RB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_rb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_acqb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_relb)(var); #else + res = ETHR_ATMC_FUNC__(add_read_wb)(var, (ethr_sint_t) 1); +#endif + return res; +} + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(inc_read_acqb)(ethr_atomic_t *var) +{ + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_INC_RETURN_ACQB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_acqb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN_RB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_rb)(var); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_mb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + res = ETHR_ATMC_FUNC__(add_read_acqb)(var, (ethr_sint_t) 1); +#endif + return res; +} + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(inc_read_relb)(ethr_atomic_t *var) +{ ethr_sint_t res; - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= mask); +#if defined(ETHR_HAVE_NATMC_INC_RETURN_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_relb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_wb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return)(var); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_mb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_rb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_acqb)(var); +#else + res = ETHR_ATMC_FUNC__(add_read_relb)(var, (ethr_sint_t) 1); +#endif return res; +} + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(inc_read_mb)(ethr_atomic_t *var) +{ + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_INC_RETURN_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_mb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_acqb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_rb)(var); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + res = ETHR_ATMC_FUNC__(add_read_mb)(var, (ethr_sint_t) 1); #endif + return res; } -static ETHR_INLINE ethr_sint_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic_xchg)(ethr_atomic_t *var, ethr_sint_t new) + +/* --- dec_read() --- */ + + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(dec_read)(ethr_atomic_t *var) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return (ethr_sint_t) ETHR_NATMC_FUNC__(xchg)(var, - (ETHR_NAINT_T__) new); + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_DEC_RETURN) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_rb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_wb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_ACQB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_acqb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_relb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_mb)(var); #else + res = ETHR_ATMC_FUNC__(add_read)(var, (ethr_sint_t) -1); +#endif + return res; +} + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(dec_read_rb)(ethr_atomic_t *var) +{ ethr_sint_t res; - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = new); +#if defined(ETHR_HAVE_NATMC_DEC_RETURN_RB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_rb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_mb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_ACQB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_acqb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + res = ETHR_ATMC_FUNC__(add_read_rb)(var, (ethr_sint_t) -1); +#endif return res; +} + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(dec_read_wb)(ethr_atomic_t *var) +{ + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_DEC_RETURN_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_wb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_mb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_rb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_acqb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_relb)(var); +#else + res = ETHR_ATMC_FUNC__(add_read_wb)(var, (ethr_sint_t) -1); #endif + return res; } -static ETHR_INLINE ethr_sint_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(ethr_atomic_t *var, - ethr_sint_t new, - ethr_sint_t exp) +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(dec_read_acqb)(ethr_atomic_t *var) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg)(var, - (ETHR_NAINT_T__) new, - (ETHR_NAINT_T__) exp); + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_DEC_RETURN_ACQB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_acqb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_rb)(var); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_mb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); #else + res = ETHR_ATMC_FUNC__(add_read_acqb)(var, (ethr_sint_t) -1); +#endif + return res; +} + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(dec_read_relb)(ethr_atomic_t *var) +{ ethr_sint_t res; - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, - { - res = *var; - if (__builtin_expect(res == exp, 1)) - *var = new; - }); +#if defined(ETHR_HAVE_NATMC_DEC_RETURN_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_relb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_wb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_mb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_rb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_acqb)(var); +#else + res = ETHR_ATMC_FUNC__(add_read_relb)(var, (ethr_sint_t) -1); +#endif return res; +} + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(dec_read_mb)(ethr_atomic_t *var) +{ + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_DEC_RETURN_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_mb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_acqb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_rb)(var); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + res = ETHR_ATMC_FUNC__(add_read_mb)(var, (ethr_sint_t) -1); #endif + return res; } -/* - * Important memory barrier requirements. - * - * The following atomic operations *must* supply a memory barrier of - * at least the type specified by its suffix: - * _acqb = acquire barrier - * _relb = release barrier - */ -static ETHR_INLINE ethr_sint_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic_read_acqb)(ethr_atomic_t *var) +/* --- add() --- */ + + +static ETHR_INLINE void ETHR_ATMC_FUNC__(add)(ethr_atomic_t *var, ethr_sint_t val) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return (ethr_sint_t) ETHR_NATMC_FUNC__(read_acqb)(var); +#if defined(ETHR_HAVE_NATMC_ADD) + ETHR_NATMC_FUNC__(add)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_RB) + ETHR_NATMC_FUNC__(add_rb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_WB) + ETHR_NATMC_FUNC__(add_wb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_ACQB) + ETHR_NATMC_FUNC__(add_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_RELB) + ETHR_NATMC_FUNC__(add_relb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_MB) + ETHR_NATMC_FUNC__(add_mb)(var, (ETHR_NAINT_T__) val); #else - return ETHR_INLINE_FUNC_NAME_(ethr_atomic_read)(var); + (void) ETHR_ATMC_FUNC__(add_read)(var, val); #endif } -static ETHR_INLINE ethr_sint_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc_read_acqb)(ethr_atomic_t *var) +static ETHR_INLINE void ETHR_ATMC_FUNC__(add_rb)(ethr_atomic_t *var, ethr_sint_t val) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_acqb)(var); +#if defined(ETHR_HAVE_NATMC_ADD_RB) + ETHR_NATMC_FUNC__(add_rb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD) + ETHR_NATMC_FUNC__(add)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_ADD_MB) + ETHR_NATMC_FUNC__(add_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_WB) + ETHR_NATMC_FUNC__(add_wb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_ADD_ACQB) + ETHR_NATMC_FUNC__(add_acqb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_ADD_RELB) + ETHR_NATMC_FUNC__(add_relb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); #else - return ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc_read)(var); + (void) ETHR_ATMC_FUNC__(add_read_rb)(var, val); #endif } -static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_atomic_set_relb)(ethr_atomic_t *var, - ethr_sint_t val) +static ETHR_INLINE void ETHR_ATMC_FUNC__(add_wb)(ethr_atomic_t *var, ethr_sint_t val) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - ETHR_NATMC_FUNC__(set_relb)(var, (ETHR_NAINT_T__) val); +#if defined(ETHR_HAVE_NATMC_ADD_WB) + ETHR_NATMC_FUNC__(add_wb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_FUNC__(add)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_MB) + ETHR_NATMC_FUNC__(add_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_RB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_FUNC__(add_rb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_FUNC__(add_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_FUNC__(add_relb)(var, (ETHR_NAINT_T__) val); #else - ETHR_INLINE_FUNC_NAME_(ethr_atomic_set)(var, val); + (void) ETHR_ATMC_FUNC__(add_read_wb)(var, val); #endif } -static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_relb)(ethr_atomic_t *var) +static ETHR_INLINE void ETHR_ATMC_FUNC__(add_acqb)(ethr_atomic_t *var, ethr_sint_t val) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS +#if defined(ETHR_HAVE_NATMC_ADD_ACQB) + ETHR_NATMC_FUNC__(add_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_RB) + ETHR_NATMC_FUNC__(add_rb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_ADD) + ETHR_NATMC_FUNC__(add)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_ADD_MB) + ETHR_NATMC_FUNC__(add_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_WB) + ETHR_NATMC_FUNC__(add_wb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_ADD_RELB) + ETHR_NATMC_FUNC__(add_relb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + (void) ETHR_ATMC_FUNC__(add_read_acqb)(var, val); +#endif +} + +static ETHR_INLINE void ETHR_ATMC_FUNC__(add_relb)(ethr_atomic_t *var, ethr_sint_t val) +{ +#if defined(ETHR_HAVE_NATMC_ADD_RELB) + ETHR_NATMC_FUNC__(add_relb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_FUNC__(add_wb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_FUNC__(add)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_MB) + ETHR_NATMC_FUNC__(add_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_FUNC__(add_rb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_FUNC__(add_acqb)(var, (ETHR_NAINT_T__) val); +#else + (void) ETHR_ATMC_FUNC__(add_read_relb)(var, val); +#endif +} + +static ETHR_INLINE void ETHR_ATMC_FUNC__(add_mb)(ethr_atomic_t *var, ethr_sint_t val) +{ +#if defined(ETHR_HAVE_NATMC_ADD_MB) + ETHR_NATMC_FUNC__(add_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_RELB) + ETHR_NATMC_FUNC__(add_relb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_ADD_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_FUNC__(add_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_FUNC__(add_wb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_ADD_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_FUNC__(add_rb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_ADD) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_FUNC__(add)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + (void) ETHR_ATMC_FUNC__(add_read_mb)(var, val); +#endif +} + + +/* --- inc() --- */ + + +static ETHR_INLINE void ETHR_ATMC_FUNC__(inc)(ethr_atomic_t *var) +{ +#if defined(ETHR_HAVE_NATMC_INC) + ETHR_NATMC_FUNC__(inc)(var); +#elif defined(ETHR_HAVE_NATMC_INC_RB) + ETHR_NATMC_FUNC__(inc_rb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_WB) + ETHR_NATMC_FUNC__(inc_wb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_ACQB) + ETHR_NATMC_FUNC__(inc_acqb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_RELB) + ETHR_NATMC_FUNC__(inc_relb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_MB) + ETHR_NATMC_FUNC__(inc_mb)(var); +#else + (void) ETHR_ATMC_FUNC__(inc_read)(var); +#endif +} + +static ETHR_INLINE void ETHR_ATMC_FUNC__(inc_rb)(ethr_atomic_t *var) +{ +#if defined(ETHR_HAVE_NATMC_INC_RB) + ETHR_NATMC_FUNC__(inc_rb)(var); +#elif defined(ETHR_HAVE_NATMC_INC) + ETHR_NATMC_FUNC__(inc)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_INC_MB) + ETHR_NATMC_FUNC__(inc_mb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_WB) + ETHR_NATMC_FUNC__(inc_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_INC_ACQB) + ETHR_NATMC_FUNC__(inc_acqb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_INC_RELB) + ETHR_NATMC_FUNC__(inc_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + (void) ETHR_ATMC_FUNC__(inc_read_rb)(var); +#endif +} + +static ETHR_INLINE void ETHR_ATMC_FUNC__(inc_wb)(ethr_atomic_t *var) +{ +#if defined(ETHR_HAVE_NATMC_INC_WB) + ETHR_NATMC_FUNC__(inc_wb)(var); +#elif defined(ETHR_HAVE_NATMC_INC) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_FUNC__(inc)(var); +#elif defined(ETHR_HAVE_NATMC_INC_MB) + ETHR_NATMC_FUNC__(inc_mb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_RB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_FUNC__(inc_rb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_FUNC__(inc_acqb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_FUNC__(inc_relb)(var); +#else + (void) ETHR_ATMC_FUNC__(inc_read_wb)(var); +#endif +} + +static ETHR_INLINE void ETHR_ATMC_FUNC__(inc_acqb)(ethr_atomic_t *var) +{ +#if defined(ETHR_HAVE_NATMC_INC_ACQB) + ETHR_NATMC_FUNC__(inc_acqb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_RB) + ETHR_NATMC_FUNC__(inc_rb)(var); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_INC) + ETHR_NATMC_FUNC__(inc)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_INC_MB) + ETHR_NATMC_FUNC__(inc_mb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_WB) + ETHR_NATMC_FUNC__(inc_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_INC_RELB) + ETHR_NATMC_FUNC__(inc_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + (void) ETHR_ATMC_FUNC__(inc_read_acqb)(var); +#endif +} + +static ETHR_INLINE void ETHR_ATMC_FUNC__(inc_relb)(ethr_atomic_t *var) +{ +#if defined(ETHR_HAVE_NATMC_INC_RELB) + ETHR_NATMC_FUNC__(inc_relb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_FUNC__(inc_wb)(var); +#elif defined(ETHR_HAVE_NATMC_INC) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_FUNC__(inc)(var); +#elif defined(ETHR_HAVE_NATMC_INC_MB) + ETHR_NATMC_FUNC__(inc_mb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_FUNC__(inc_rb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_FUNC__(inc_acqb)(var); +#else + (void) ETHR_ATMC_FUNC__(inc_read_relb)(var); +#endif +} + +static ETHR_INLINE void ETHR_ATMC_FUNC__(inc_mb)(ethr_atomic_t *var) +{ +#if defined(ETHR_HAVE_NATMC_INC_MB) + ETHR_NATMC_FUNC__(inc_mb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_RELB) + ETHR_NATMC_FUNC__(inc_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_INC_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_FUNC__(inc_acqb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_FUNC__(inc_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_INC_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_FUNC__(inc_rb)(var); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_INC) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_FUNC__(inc)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + (void) ETHR_ATMC_FUNC__(inc_read_mb)(var); +#endif +} + + +/* --- dec() --- */ + + +static ETHR_INLINE void ETHR_ATMC_FUNC__(dec)(ethr_atomic_t *var) +{ +#if defined(ETHR_HAVE_NATMC_DEC) + ETHR_NATMC_FUNC__(dec)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_RB) + ETHR_NATMC_FUNC__(dec_rb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_WB) + ETHR_NATMC_FUNC__(dec_wb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_ACQB) + ETHR_NATMC_FUNC__(dec_acqb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_RELB) + ETHR_NATMC_FUNC__(dec_relb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_MB) + ETHR_NATMC_FUNC__(dec_mb)(var); +#else + (void) ETHR_ATMC_FUNC__(dec_read)(var); +#endif +} + +static ETHR_INLINE void ETHR_ATMC_FUNC__(dec_rb)(ethr_atomic_t *var) +{ +#if defined(ETHR_HAVE_NATMC_DEC_RB) + ETHR_NATMC_FUNC__(dec_rb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC) + ETHR_NATMC_FUNC__(dec)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_DEC_MB) + ETHR_NATMC_FUNC__(dec_mb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_WB) + ETHR_NATMC_FUNC__(dec_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_DEC_ACQB) + ETHR_NATMC_FUNC__(dec_acqb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_DEC_RELB) + ETHR_NATMC_FUNC__(dec_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + (void) ETHR_ATMC_FUNC__(dec_read_rb)(var); +#endif +} + +static ETHR_INLINE void ETHR_ATMC_FUNC__(dec_wb)(ethr_atomic_t *var) +{ +#if defined(ETHR_HAVE_NATMC_DEC_WB) + ETHR_NATMC_FUNC__(dec_wb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_FUNC__(dec)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_MB) + ETHR_NATMC_FUNC__(dec_mb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_RB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_FUNC__(dec_rb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_FUNC__(dec_acqb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_FUNC__(dec_relb)(var); +#else + (void) ETHR_ATMC_FUNC__(dec_read_wb)(var); +#endif +} + +static ETHR_INLINE void ETHR_ATMC_FUNC__(dec_acqb)(ethr_atomic_t *var) +{ +#if defined(ETHR_HAVE_NATMC_DEC_ACQB) + ETHR_NATMC_FUNC__(dec_acqb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_RB) + ETHR_NATMC_FUNC__(dec_rb)(var); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_DEC) + ETHR_NATMC_FUNC__(dec)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_DEC_MB) + ETHR_NATMC_FUNC__(dec_mb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_WB) + ETHR_NATMC_FUNC__(dec_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_DEC_RELB) + ETHR_NATMC_FUNC__(dec_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + (void) ETHR_ATMC_FUNC__(dec_read_acqb)(var); +#endif +} + +static ETHR_INLINE void ETHR_ATMC_FUNC__(dec_relb)(ethr_atomic_t *var) +{ +#if defined(ETHR_HAVE_NATMC_DEC_RELB) + ETHR_NATMC_FUNC__(dec_relb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_FUNC__(dec_wb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_FUNC__(dec)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_MB) + ETHR_NATMC_FUNC__(dec_mb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_FUNC__(dec_rb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_FUNC__(dec_acqb)(var); +#else + (void) ETHR_ATMC_FUNC__(dec_read_relb)(var); +#endif +} + +static ETHR_INLINE void ETHR_ATMC_FUNC__(dec_mb)(ethr_atomic_t *var) +{ +#if defined(ETHR_HAVE_NATMC_DEC_MB) + ETHR_NATMC_FUNC__(dec_mb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_RELB) ETHR_NATMC_FUNC__(dec_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_DEC_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_FUNC__(dec_acqb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_FUNC__(dec_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_DEC_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_FUNC__(dec_rb)(var); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_DEC) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_FUNC__(dec)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + (void) ETHR_ATMC_FUNC__(dec_read_mb)(var); +#endif +} + + +/* --- read_band() --- */ + + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_band)(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_AND_RETOLD) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_rb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_wb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_ACQB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_relb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val); +#else +#error "Missing implementation of ethr_atomic_read_band()!" +#endif + return res; +} + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_band_rb)(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_AND_RETOLD_RB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_rb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_wb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_ACQB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_acqb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_relb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval &= val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val); + ETHR_MEMBAR(ETHR_LoadLoad); +#else +#error "Missing implementation of ethr_atomic_read_band_rb()!" +#endif + return res; +} + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_band_wb)(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_AND_RETOLD_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_wb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_rb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_relb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val); #else - ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec)(var); +#error "Missing implementation of ethr_atomic_read_band_wb()!" #endif + return res; } -static ETHR_INLINE ethr_sint_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_read_relb)(ethr_atomic_t *var) +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_band_acqb)(ethr_atomic_t *var, ethr_sint_t val) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_relb)(var); + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_AND_RETOLD_ACQB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_rb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_wb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_relb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval &= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); #else - return ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_read)(var); +#error "Missing implementation of ethr_atomic_read_band_acqb()!" #endif + return res; } -static ETHR_INLINE ethr_sint_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg_acqb)(ethr_atomic_t *var, - ethr_sint_t new, - ethr_sint_t exp) +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_band_relb)(ethr_atomic_t *var, ethr_sint_t val) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_acqb)(var, - (ETHR_NAINT_T__) new, - (ETHR_NAINT_T__) exp); + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_AND_RETOLD_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_relb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_wb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_rb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val); #else - return ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(var, new, exp); +#error "Missing implementation of ethr_atomic_read_band_relb()!" #endif + return res; } -static ETHR_INLINE ethr_sint_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg_relb)(ethr_atomic_t *var, - ethr_sint_t new, - ethr_sint_t exp) +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_band_mb)(ethr_atomic_t *var, ethr_sint_t val) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_relb)(var, - (ETHR_NAINT_T__) new, - (ETHR_NAINT_T__) exp); + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_AND_RETOLD_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_relb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_wb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_rb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval &= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); #else - return ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(var, new, exp); +#error "Missing implementation of ethr_atomic_read_band_mb()!" #endif + return res; } + +/* --- read_bor() --- */ + + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_bor)(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_OR_RETOLD) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_rb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_wb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_ACQB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_relb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val); +#else +#error "Missing implementation of ethr_atomic_read_bor()!" +#endif + return res; +} + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_bor_rb)(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_OR_RETOLD_RB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_rb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_wb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_ACQB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_acqb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_relb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval |= val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val); + ETHR_MEMBAR(ETHR_LoadLoad); +#else +#error "Missing implementation of ethr_atomic_read_bor_rb()!" +#endif + return res; +} + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_bor_wb)(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_OR_RETOLD_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_wb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_rb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_relb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val); +#else +#error "Missing implementation of ethr_atomic_read_bor_wb()!" +#endif + return res; +} + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_bor_acqb)(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_OR_RETOLD_ACQB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_rb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_wb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_relb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval |= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else +#error "Missing implementation of ethr_atomic_read_bor_acqb()!" +#endif + return res; +} + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_bor_relb)(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_OR_RETOLD_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_relb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_wb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_rb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val); +#else +#error "Missing implementation of ethr_atomic_read_bor_relb()!" +#endif + return res; +} + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_bor_mb)(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_OR_RETOLD_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_relb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_wb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_rb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval |= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else +#error "Missing implementation of ethr_atomic_read_bor_mb()!" +#endif + return res; +} + +#endif /* ETHR_ATMC_INLINE__ */ + + +/* ---------- 32-bit atomic implementation ---------- */ + + +#ifdef ETHR_NEED_ATMC32_PROTOTYPES__ +ethr_sint32_t *ethr_atomic32_addr(ethr_atomic32_t *var); +ethr_sint32_t ethr_atomic32_cmpxchg(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val); +ethr_sint32_t ethr_atomic32_cmpxchg_rb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val); +ethr_sint32_t ethr_atomic32_cmpxchg_wb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val); +ethr_sint32_t ethr_atomic32_cmpxchg_acqb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val); +ethr_sint32_t ethr_atomic32_cmpxchg_relb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val); +ethr_sint32_t ethr_atomic32_cmpxchg_mb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val); +ethr_sint32_t ethr_atomic32_xchg(ethr_atomic32_t *var, ethr_sint32_t val); +ethr_sint32_t ethr_atomic32_xchg_rb(ethr_atomic32_t *var, ethr_sint32_t val); +ethr_sint32_t ethr_atomic32_xchg_wb(ethr_atomic32_t *var, ethr_sint32_t val); +ethr_sint32_t ethr_atomic32_xchg_acqb(ethr_atomic32_t *var, ethr_sint32_t val); +ethr_sint32_t ethr_atomic32_xchg_relb(ethr_atomic32_t *var, ethr_sint32_t val); +ethr_sint32_t ethr_atomic32_xchg_mb(ethr_atomic32_t *var, ethr_sint32_t val); +void ethr_atomic32_set(ethr_atomic32_t *var, ethr_sint32_t val); +void ethr_atomic32_set_rb(ethr_atomic32_t *var, ethr_sint32_t val); +void ethr_atomic32_set_wb(ethr_atomic32_t *var, ethr_sint32_t val); +void ethr_atomic32_set_acqb(ethr_atomic32_t *var, ethr_sint32_t val); +void ethr_atomic32_set_relb(ethr_atomic32_t *var, ethr_sint32_t val); +void ethr_atomic32_set_mb(ethr_atomic32_t *var, ethr_sint32_t val); +void ethr_atomic32_init(ethr_atomic32_t *var, ethr_sint32_t val); +void ethr_atomic32_init_rb(ethr_atomic32_t *var, ethr_sint32_t val); +void ethr_atomic32_init_wb(ethr_atomic32_t *var, ethr_sint32_t val); +void ethr_atomic32_init_acqb(ethr_atomic32_t *var, ethr_sint32_t val); +void ethr_atomic32_init_relb(ethr_atomic32_t *var, ethr_sint32_t val); +void ethr_atomic32_init_mb(ethr_atomic32_t *var, ethr_sint32_t val); +ethr_sint32_t ethr_atomic32_add_read(ethr_atomic32_t *var, ethr_sint32_t val); +ethr_sint32_t ethr_atomic32_add_read_rb(ethr_atomic32_t *var, ethr_sint32_t val); +ethr_sint32_t ethr_atomic32_add_read_wb(ethr_atomic32_t *var, ethr_sint32_t val); +ethr_sint32_t ethr_atomic32_add_read_acqb(ethr_atomic32_t *var, ethr_sint32_t val); +ethr_sint32_t ethr_atomic32_add_read_relb(ethr_atomic32_t *var, ethr_sint32_t val); +ethr_sint32_t ethr_atomic32_add_read_mb(ethr_atomic32_t *var, ethr_sint32_t val); +ethr_sint32_t ethr_atomic32_read(ethr_atomic32_t *var); +ethr_sint32_t ethr_atomic32_read_rb(ethr_atomic32_t *var); +ethr_sint32_t ethr_atomic32_read_wb(ethr_atomic32_t *var); +ethr_sint32_t ethr_atomic32_read_acqb(ethr_atomic32_t *var); +ethr_sint32_t ethr_atomic32_read_relb(ethr_atomic32_t *var); +ethr_sint32_t ethr_atomic32_read_mb(ethr_atomic32_t *var); +ethr_sint32_t ethr_atomic32_inc_read(ethr_atomic32_t *var); +ethr_sint32_t ethr_atomic32_inc_read_rb(ethr_atomic32_t *var); +ethr_sint32_t ethr_atomic32_inc_read_wb(ethr_atomic32_t *var); +ethr_sint32_t ethr_atomic32_inc_read_acqb(ethr_atomic32_t *var); +ethr_sint32_t ethr_atomic32_inc_read_relb(ethr_atomic32_t *var); +ethr_sint32_t ethr_atomic32_inc_read_mb(ethr_atomic32_t *var); +ethr_sint32_t ethr_atomic32_dec_read(ethr_atomic32_t *var); +ethr_sint32_t ethr_atomic32_dec_read_rb(ethr_atomic32_t *var); +ethr_sint32_t ethr_atomic32_dec_read_wb(ethr_atomic32_t *var); +ethr_sint32_t ethr_atomic32_dec_read_acqb(ethr_atomic32_t *var); +ethr_sint32_t ethr_atomic32_dec_read_relb(ethr_atomic32_t *var); +ethr_sint32_t ethr_atomic32_dec_read_mb(ethr_atomic32_t *var); +void ethr_atomic32_add(ethr_atomic32_t *var, ethr_sint32_t val); +void ethr_atomic32_add_rb(ethr_atomic32_t *var, ethr_sint32_t val); +void ethr_atomic32_add_wb(ethr_atomic32_t *var, ethr_sint32_t val); +void ethr_atomic32_add_acqb(ethr_atomic32_t *var, ethr_sint32_t val); +void ethr_atomic32_add_relb(ethr_atomic32_t *var, ethr_sint32_t val); +void ethr_atomic32_add_mb(ethr_atomic32_t *var, ethr_sint32_t val); +void ethr_atomic32_inc(ethr_atomic32_t *var); +void ethr_atomic32_inc_rb(ethr_atomic32_t *var); +void ethr_atomic32_inc_wb(ethr_atomic32_t *var); +void ethr_atomic32_inc_acqb(ethr_atomic32_t *var); +void ethr_atomic32_inc_relb(ethr_atomic32_t *var); +void ethr_atomic32_inc_mb(ethr_atomic32_t *var); +void ethr_atomic32_dec(ethr_atomic32_t *var); +void ethr_atomic32_dec_rb(ethr_atomic32_t *var); +void ethr_atomic32_dec_wb(ethr_atomic32_t *var); +void ethr_atomic32_dec_acqb(ethr_atomic32_t *var); +void ethr_atomic32_dec_relb(ethr_atomic32_t *var); +void ethr_atomic32_dec_mb(ethr_atomic32_t *var); +ethr_sint32_t ethr_atomic32_read_band(ethr_atomic32_t *var, ethr_sint32_t val); +ethr_sint32_t ethr_atomic32_read_band_rb(ethr_atomic32_t *var, ethr_sint32_t val); +ethr_sint32_t ethr_atomic32_read_band_wb(ethr_atomic32_t *var, ethr_sint32_t val); +ethr_sint32_t ethr_atomic32_read_band_acqb(ethr_atomic32_t *var, ethr_sint32_t val); +ethr_sint32_t ethr_atomic32_read_band_relb(ethr_atomic32_t *var, ethr_sint32_t val); +ethr_sint32_t ethr_atomic32_read_band_mb(ethr_atomic32_t *var, ethr_sint32_t val); +ethr_sint32_t ethr_atomic32_read_bor(ethr_atomic32_t *var, ethr_sint32_t val); +ethr_sint32_t ethr_atomic32_read_bor_rb(ethr_atomic32_t *var, ethr_sint32_t val); +ethr_sint32_t ethr_atomic32_read_bor_wb(ethr_atomic32_t *var, ethr_sint32_t val); +ethr_sint32_t ethr_atomic32_read_bor_acqb(ethr_atomic32_t *var, ethr_sint32_t val); +ethr_sint32_t ethr_atomic32_read_bor_relb(ethr_atomic32_t *var, ethr_sint32_t val); +ethr_sint32_t ethr_atomic32_read_bor_mb(ethr_atomic32_t *var, ethr_sint32_t val); +#endif /* ETHR_NEED_ATMC32_PROTOTYPES__ */ + +#if (defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) \ + && (defined(ETHR_ATMC32_INLINE__) || defined(ETHR_ATOMIC_IMPL__))) + +#if !defined(ETHR_NATMC32_BITS__) +# error "Missing native atomic implementation" +#elif ETHR_NATMC32_BITS__ == 64 +# undef ETHR_HAVE_NATMC32_CMPXCHG +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG +# define ETHR_HAVE_NATMC32_CMPXCHG 1 +# endif +# undef ETHR_HAVE_NATMC32_CMPXCHG_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RB +# define ETHR_HAVE_NATMC32_CMPXCHG_RB 1 +# endif +# undef ETHR_HAVE_NATMC32_CMPXCHG_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_WB +# define ETHR_HAVE_NATMC32_CMPXCHG_WB 1 +# endif +# undef ETHR_HAVE_NATMC32_CMPXCHG_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_ACQB +# define ETHR_HAVE_NATMC32_CMPXCHG_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC32_CMPXCHG_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RELB +# define ETHR_HAVE_NATMC32_CMPXCHG_RELB 1 +# endif +# undef ETHR_HAVE_NATMC32_CMPXCHG_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_MB +# define ETHR_HAVE_NATMC32_CMPXCHG_MB 1 +# endif +# undef ETHR_HAVE_NATMC32_XCHG +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG +# define ETHR_HAVE_NATMC32_XCHG 1 +# endif +# undef ETHR_HAVE_NATMC32_XCHG_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_RB +# define ETHR_HAVE_NATMC32_XCHG_RB 1 +# endif +# undef ETHR_HAVE_NATMC32_XCHG_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_WB +# define ETHR_HAVE_NATMC32_XCHG_WB 1 +# endif +# undef ETHR_HAVE_NATMC32_XCHG_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_ACQB +# define ETHR_HAVE_NATMC32_XCHG_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC32_XCHG_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_RELB +# define ETHR_HAVE_NATMC32_XCHG_RELB 1 +# endif +# undef ETHR_HAVE_NATMC32_XCHG_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_MB +# define ETHR_HAVE_NATMC32_XCHG_MB 1 +# endif +# undef ETHR_HAVE_NATMC32_SET +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET +# define ETHR_HAVE_NATMC32_SET 1 +# endif +# undef ETHR_HAVE_NATMC32_SET_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RB +# define ETHR_HAVE_NATMC32_SET_RB 1 +# endif +# undef ETHR_HAVE_NATMC32_SET_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_WB +# define ETHR_HAVE_NATMC32_SET_WB 1 +# endif +# undef ETHR_HAVE_NATMC32_SET_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_ACQB +# define ETHR_HAVE_NATMC32_SET_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC32_SET_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RELB +# define ETHR_HAVE_NATMC32_SET_RELB 1 +# endif +# undef ETHR_HAVE_NATMC32_SET_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_MB +# define ETHR_HAVE_NATMC32_SET_MB 1 +# endif +# undef ETHR_HAVE_NATMC32_INIT +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT +# define ETHR_HAVE_NATMC32_INIT 1 +# endif +# undef ETHR_HAVE_NATMC32_INIT_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_RB +# define ETHR_HAVE_NATMC32_INIT_RB 1 +# endif +# undef ETHR_HAVE_NATMC32_INIT_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_WB +# define ETHR_HAVE_NATMC32_INIT_WB 1 +# endif +# undef ETHR_HAVE_NATMC32_INIT_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_ACQB +# define ETHR_HAVE_NATMC32_INIT_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC32_INIT_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_RELB +# define ETHR_HAVE_NATMC32_INIT_RELB 1 +# endif +# undef ETHR_HAVE_NATMC32_INIT_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_MB +# define ETHR_HAVE_NATMC32_INIT_MB 1 +# endif +# undef ETHR_HAVE_NATMC32_ADD_RETURN +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN +# define ETHR_HAVE_NATMC32_ADD_RETURN 1 +# endif +# undef ETHR_HAVE_NATMC32_ADD_RETURN_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_RB +# define ETHR_HAVE_NATMC32_ADD_RETURN_RB 1 +# endif +# undef ETHR_HAVE_NATMC32_ADD_RETURN_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_WB +# define ETHR_HAVE_NATMC32_ADD_RETURN_WB 1 +# endif +# undef ETHR_HAVE_NATMC32_ADD_RETURN_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_ACQB +# define ETHR_HAVE_NATMC32_ADD_RETURN_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC32_ADD_RETURN_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_RELB +# define ETHR_HAVE_NATMC32_ADD_RETURN_RELB 1 +# endif +# undef ETHR_HAVE_NATMC32_ADD_RETURN_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_MB +# define ETHR_HAVE_NATMC32_ADD_RETURN_MB 1 +# endif +# undef ETHR_HAVE_NATMC32_READ +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ +# define ETHR_HAVE_NATMC32_READ 1 +# endif +# undef ETHR_HAVE_NATMC32_READ_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_RB +# define ETHR_HAVE_NATMC32_READ_RB 1 +# endif +# undef ETHR_HAVE_NATMC32_READ_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_WB +# define ETHR_HAVE_NATMC32_READ_WB 1 +# endif +# undef ETHR_HAVE_NATMC32_READ_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_ACQB +# define ETHR_HAVE_NATMC32_READ_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC32_READ_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_RELB +# define ETHR_HAVE_NATMC32_READ_RELB 1 +# endif +# undef ETHR_HAVE_NATMC32_READ_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_MB +# define ETHR_HAVE_NATMC32_READ_MB 1 +# endif +# undef ETHR_HAVE_NATMC32_INC_RETURN +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN +# define ETHR_HAVE_NATMC32_INC_RETURN 1 +# endif +# undef ETHR_HAVE_NATMC32_INC_RETURN_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_RB +# define ETHR_HAVE_NATMC32_INC_RETURN_RB 1 +# endif +# undef ETHR_HAVE_NATMC32_INC_RETURN_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_WB +# define ETHR_HAVE_NATMC32_INC_RETURN_WB 1 +# endif +# undef ETHR_HAVE_NATMC32_INC_RETURN_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_ACQB +# define ETHR_HAVE_NATMC32_INC_RETURN_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC32_INC_RETURN_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_RELB +# define ETHR_HAVE_NATMC32_INC_RETURN_RELB 1 +# endif +# undef ETHR_HAVE_NATMC32_INC_RETURN_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_MB +# define ETHR_HAVE_NATMC32_INC_RETURN_MB 1 +# endif +# undef ETHR_HAVE_NATMC32_DEC_RETURN +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN +# define ETHR_HAVE_NATMC32_DEC_RETURN 1 +# endif +# undef ETHR_HAVE_NATMC32_DEC_RETURN_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_RB +# define ETHR_HAVE_NATMC32_DEC_RETURN_RB 1 +# endif +# undef ETHR_HAVE_NATMC32_DEC_RETURN_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_WB +# define ETHR_HAVE_NATMC32_DEC_RETURN_WB 1 +# endif +# undef ETHR_HAVE_NATMC32_DEC_RETURN_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_ACQB +# define ETHR_HAVE_NATMC32_DEC_RETURN_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC32_DEC_RETURN_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_RELB +# define ETHR_HAVE_NATMC32_DEC_RETURN_RELB 1 +# endif +# undef ETHR_HAVE_NATMC32_DEC_RETURN_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_MB +# define ETHR_HAVE_NATMC32_DEC_RETURN_MB 1 +# endif +# undef ETHR_HAVE_NATMC32_ADD +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD +# define ETHR_HAVE_NATMC32_ADD 1 +# endif +# undef ETHR_HAVE_NATMC32_ADD_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RB +# define ETHR_HAVE_NATMC32_ADD_RB 1 +# endif +# undef ETHR_HAVE_NATMC32_ADD_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_WB +# define ETHR_HAVE_NATMC32_ADD_WB 1 +# endif +# undef ETHR_HAVE_NATMC32_ADD_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_ACQB +# define ETHR_HAVE_NATMC32_ADD_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC32_ADD_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RELB +# define ETHR_HAVE_NATMC32_ADD_RELB 1 +# endif +# undef ETHR_HAVE_NATMC32_ADD_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_MB +# define ETHR_HAVE_NATMC32_ADD_MB 1 +# endif +# undef ETHR_HAVE_NATMC32_INC +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC +# define ETHR_HAVE_NATMC32_INC 1 +# endif +# undef ETHR_HAVE_NATMC32_INC_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RB +# define ETHR_HAVE_NATMC32_INC_RB 1 +# endif +# undef ETHR_HAVE_NATMC32_INC_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_WB +# define ETHR_HAVE_NATMC32_INC_WB 1 +# endif +# undef ETHR_HAVE_NATMC32_INC_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_ACQB +# define ETHR_HAVE_NATMC32_INC_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC32_INC_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RELB +# define ETHR_HAVE_NATMC32_INC_RELB 1 +# endif +# undef ETHR_HAVE_NATMC32_INC_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_MB +# define ETHR_HAVE_NATMC32_INC_MB 1 +# endif +# undef ETHR_HAVE_NATMC32_DEC +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC +# define ETHR_HAVE_NATMC32_DEC 1 +# endif +# undef ETHR_HAVE_NATMC32_DEC_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RB +# define ETHR_HAVE_NATMC32_DEC_RB 1 +# endif +# undef ETHR_HAVE_NATMC32_DEC_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_WB +# define ETHR_HAVE_NATMC32_DEC_WB 1 +# endif +# undef ETHR_HAVE_NATMC32_DEC_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_ACQB +# define ETHR_HAVE_NATMC32_DEC_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC32_DEC_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RELB +# define ETHR_HAVE_NATMC32_DEC_RELB 1 +# endif +# undef ETHR_HAVE_NATMC32_DEC_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_MB +# define ETHR_HAVE_NATMC32_DEC_MB 1 +# endif +# undef ETHR_HAVE_NATMC32_AND_RETOLD +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD +# define ETHR_HAVE_NATMC32_AND_RETOLD 1 +# endif +# undef ETHR_HAVE_NATMC32_AND_RETOLD_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_RB +# define ETHR_HAVE_NATMC32_AND_RETOLD_RB 1 +# endif +# undef ETHR_HAVE_NATMC32_AND_RETOLD_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_WB +# define ETHR_HAVE_NATMC32_AND_RETOLD_WB 1 +# endif +# undef ETHR_HAVE_NATMC32_AND_RETOLD_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_ACQB +# define ETHR_HAVE_NATMC32_AND_RETOLD_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC32_AND_RETOLD_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_RELB +# define ETHR_HAVE_NATMC32_AND_RETOLD_RELB 1 +# endif +# undef ETHR_HAVE_NATMC32_AND_RETOLD_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_MB +# define ETHR_HAVE_NATMC32_AND_RETOLD_MB 1 +# endif +# undef ETHR_HAVE_NATMC32_OR_RETOLD +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD +# define ETHR_HAVE_NATMC32_OR_RETOLD 1 +# endif +# undef ETHR_HAVE_NATMC32_OR_RETOLD_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_RB +# define ETHR_HAVE_NATMC32_OR_RETOLD_RB 1 +# endif +# undef ETHR_HAVE_NATMC32_OR_RETOLD_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_WB +# define ETHR_HAVE_NATMC32_OR_RETOLD_WB 1 +# endif +# undef ETHR_HAVE_NATMC32_OR_RETOLD_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_ACQB +# define ETHR_HAVE_NATMC32_OR_RETOLD_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC32_OR_RETOLD_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_RELB +# define ETHR_HAVE_NATMC32_OR_RETOLD_RELB 1 +# endif +# undef ETHR_HAVE_NATMC32_OR_RETOLD_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_MB +# define ETHR_HAVE_NATMC32_OR_RETOLD_MB 1 +# endif +#elif ETHR_NATMC32_BITS__ == 32 +# undef ETHR_HAVE_NATMC32_CMPXCHG +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG +# define ETHR_HAVE_NATMC32_CMPXCHG 1 +# endif +# undef ETHR_HAVE_NATMC32_CMPXCHG_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_RB +# define ETHR_HAVE_NATMC32_CMPXCHG_RB 1 +# endif +# undef ETHR_HAVE_NATMC32_CMPXCHG_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_WB +# define ETHR_HAVE_NATMC32_CMPXCHG_WB 1 +# endif +# undef ETHR_HAVE_NATMC32_CMPXCHG_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_ACQB +# define ETHR_HAVE_NATMC32_CMPXCHG_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC32_CMPXCHG_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_RELB +# define ETHR_HAVE_NATMC32_CMPXCHG_RELB 1 +# endif +# undef ETHR_HAVE_NATMC32_CMPXCHG_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_MB +# define ETHR_HAVE_NATMC32_CMPXCHG_MB 1 +# endif +# undef ETHR_HAVE_NATMC32_XCHG +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG +# define ETHR_HAVE_NATMC32_XCHG 1 +# endif +# undef ETHR_HAVE_NATMC32_XCHG_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_RB +# define ETHR_HAVE_NATMC32_XCHG_RB 1 +# endif +# undef ETHR_HAVE_NATMC32_XCHG_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_WB +# define ETHR_HAVE_NATMC32_XCHG_WB 1 +# endif +# undef ETHR_HAVE_NATMC32_XCHG_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_ACQB +# define ETHR_HAVE_NATMC32_XCHG_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC32_XCHG_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_RELB +# define ETHR_HAVE_NATMC32_XCHG_RELB 1 +# endif +# undef ETHR_HAVE_NATMC32_XCHG_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_MB +# define ETHR_HAVE_NATMC32_XCHG_MB 1 +# endif +# undef ETHR_HAVE_NATMC32_SET +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET +# define ETHR_HAVE_NATMC32_SET 1 +# endif +# undef ETHR_HAVE_NATMC32_SET_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_RB +# define ETHR_HAVE_NATMC32_SET_RB 1 +# endif +# undef ETHR_HAVE_NATMC32_SET_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_WB +# define ETHR_HAVE_NATMC32_SET_WB 1 +# endif +# undef ETHR_HAVE_NATMC32_SET_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_ACQB +# define ETHR_HAVE_NATMC32_SET_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC32_SET_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_RELB +# define ETHR_HAVE_NATMC32_SET_RELB 1 +# endif +# undef ETHR_HAVE_NATMC32_SET_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_MB +# define ETHR_HAVE_NATMC32_SET_MB 1 +# endif +# undef ETHR_HAVE_NATMC32_INIT +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT +# define ETHR_HAVE_NATMC32_INIT 1 +# endif +# undef ETHR_HAVE_NATMC32_INIT_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_RB +# define ETHR_HAVE_NATMC32_INIT_RB 1 +# endif +# undef ETHR_HAVE_NATMC32_INIT_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_WB +# define ETHR_HAVE_NATMC32_INIT_WB 1 +# endif +# undef ETHR_HAVE_NATMC32_INIT_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_ACQB +# define ETHR_HAVE_NATMC32_INIT_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC32_INIT_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_RELB +# define ETHR_HAVE_NATMC32_INIT_RELB 1 +# endif +# undef ETHR_HAVE_NATMC32_INIT_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_MB +# define ETHR_HAVE_NATMC32_INIT_MB 1 +# endif +# undef ETHR_HAVE_NATMC32_ADD_RETURN +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN +# define ETHR_HAVE_NATMC32_ADD_RETURN 1 +# endif +# undef ETHR_HAVE_NATMC32_ADD_RETURN_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_RB +# define ETHR_HAVE_NATMC32_ADD_RETURN_RB 1 +# endif +# undef ETHR_HAVE_NATMC32_ADD_RETURN_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_WB +# define ETHR_HAVE_NATMC32_ADD_RETURN_WB 1 +# endif +# undef ETHR_HAVE_NATMC32_ADD_RETURN_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_ACQB +# define ETHR_HAVE_NATMC32_ADD_RETURN_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC32_ADD_RETURN_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_RELB +# define ETHR_HAVE_NATMC32_ADD_RETURN_RELB 1 +# endif +# undef ETHR_HAVE_NATMC32_ADD_RETURN_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_MB +# define ETHR_HAVE_NATMC32_ADD_RETURN_MB 1 +# endif +# undef ETHR_HAVE_NATMC32_READ +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ +# define ETHR_HAVE_NATMC32_READ 1 +# endif +# undef ETHR_HAVE_NATMC32_READ_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_RB +# define ETHR_HAVE_NATMC32_READ_RB 1 +# endif +# undef ETHR_HAVE_NATMC32_READ_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_WB +# define ETHR_HAVE_NATMC32_READ_WB 1 +# endif +# undef ETHR_HAVE_NATMC32_READ_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_ACQB +# define ETHR_HAVE_NATMC32_READ_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC32_READ_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_RELB +# define ETHR_HAVE_NATMC32_READ_RELB 1 +# endif +# undef ETHR_HAVE_NATMC32_READ_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_MB +# define ETHR_HAVE_NATMC32_READ_MB 1 +# endif +# undef ETHR_HAVE_NATMC32_INC_RETURN +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN +# define ETHR_HAVE_NATMC32_INC_RETURN 1 +# endif +# undef ETHR_HAVE_NATMC32_INC_RETURN_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_RB +# define ETHR_HAVE_NATMC32_INC_RETURN_RB 1 +# endif +# undef ETHR_HAVE_NATMC32_INC_RETURN_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_WB +# define ETHR_HAVE_NATMC32_INC_RETURN_WB 1 +# endif +# undef ETHR_HAVE_NATMC32_INC_RETURN_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_ACQB +# define ETHR_HAVE_NATMC32_INC_RETURN_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC32_INC_RETURN_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_RELB +# define ETHR_HAVE_NATMC32_INC_RETURN_RELB 1 +# endif +# undef ETHR_HAVE_NATMC32_INC_RETURN_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_MB +# define ETHR_HAVE_NATMC32_INC_RETURN_MB 1 +# endif +# undef ETHR_HAVE_NATMC32_DEC_RETURN +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN +# define ETHR_HAVE_NATMC32_DEC_RETURN 1 +# endif +# undef ETHR_HAVE_NATMC32_DEC_RETURN_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_RB +# define ETHR_HAVE_NATMC32_DEC_RETURN_RB 1 +# endif +# undef ETHR_HAVE_NATMC32_DEC_RETURN_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_WB +# define ETHR_HAVE_NATMC32_DEC_RETURN_WB 1 +# endif +# undef ETHR_HAVE_NATMC32_DEC_RETURN_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_ACQB +# define ETHR_HAVE_NATMC32_DEC_RETURN_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC32_DEC_RETURN_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_RELB +# define ETHR_HAVE_NATMC32_DEC_RETURN_RELB 1 +# endif +# undef ETHR_HAVE_NATMC32_DEC_RETURN_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_MB +# define ETHR_HAVE_NATMC32_DEC_RETURN_MB 1 +# endif +# undef ETHR_HAVE_NATMC32_ADD +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD +# define ETHR_HAVE_NATMC32_ADD 1 +# endif +# undef ETHR_HAVE_NATMC32_ADD_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RB +# define ETHR_HAVE_NATMC32_ADD_RB 1 +# endif +# undef ETHR_HAVE_NATMC32_ADD_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_WB +# define ETHR_HAVE_NATMC32_ADD_WB 1 +# endif +# undef ETHR_HAVE_NATMC32_ADD_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_ACQB +# define ETHR_HAVE_NATMC32_ADD_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC32_ADD_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RELB +# define ETHR_HAVE_NATMC32_ADD_RELB 1 +# endif +# undef ETHR_HAVE_NATMC32_ADD_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_MB +# define ETHR_HAVE_NATMC32_ADD_MB 1 +# endif +# undef ETHR_HAVE_NATMC32_INC +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC +# define ETHR_HAVE_NATMC32_INC 1 +# endif +# undef ETHR_HAVE_NATMC32_INC_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RB +# define ETHR_HAVE_NATMC32_INC_RB 1 +# endif +# undef ETHR_HAVE_NATMC32_INC_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_WB +# define ETHR_HAVE_NATMC32_INC_WB 1 +# endif +# undef ETHR_HAVE_NATMC32_INC_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_ACQB +# define ETHR_HAVE_NATMC32_INC_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC32_INC_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RELB +# define ETHR_HAVE_NATMC32_INC_RELB 1 +# endif +# undef ETHR_HAVE_NATMC32_INC_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_MB +# define ETHR_HAVE_NATMC32_INC_MB 1 +# endif +# undef ETHR_HAVE_NATMC32_DEC +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC +# define ETHR_HAVE_NATMC32_DEC 1 +# endif +# undef ETHR_HAVE_NATMC32_DEC_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RB +# define ETHR_HAVE_NATMC32_DEC_RB 1 +# endif +# undef ETHR_HAVE_NATMC32_DEC_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_WB +# define ETHR_HAVE_NATMC32_DEC_WB 1 +# endif +# undef ETHR_HAVE_NATMC32_DEC_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_ACQB +# define ETHR_HAVE_NATMC32_DEC_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC32_DEC_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RELB +# define ETHR_HAVE_NATMC32_DEC_RELB 1 +# endif +# undef ETHR_HAVE_NATMC32_DEC_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_MB +# define ETHR_HAVE_NATMC32_DEC_MB 1 +# endif +# undef ETHR_HAVE_NATMC32_AND_RETOLD +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD +# define ETHR_HAVE_NATMC32_AND_RETOLD 1 +# endif +# undef ETHR_HAVE_NATMC32_AND_RETOLD_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_RB +# define ETHR_HAVE_NATMC32_AND_RETOLD_RB 1 +# endif +# undef ETHR_HAVE_NATMC32_AND_RETOLD_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_WB +# define ETHR_HAVE_NATMC32_AND_RETOLD_WB 1 +# endif +# undef ETHR_HAVE_NATMC32_AND_RETOLD_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_ACQB +# define ETHR_HAVE_NATMC32_AND_RETOLD_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC32_AND_RETOLD_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_RELB +# define ETHR_HAVE_NATMC32_AND_RETOLD_RELB 1 +# endif +# undef ETHR_HAVE_NATMC32_AND_RETOLD_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_MB +# define ETHR_HAVE_NATMC32_AND_RETOLD_MB 1 +# endif +# undef ETHR_HAVE_NATMC32_OR_RETOLD +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD +# define ETHR_HAVE_NATMC32_OR_RETOLD 1 +# endif +# undef ETHR_HAVE_NATMC32_OR_RETOLD_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_RB +# define ETHR_HAVE_NATMC32_OR_RETOLD_RB 1 +# endif +# undef ETHR_HAVE_NATMC32_OR_RETOLD_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_WB +# define ETHR_HAVE_NATMC32_OR_RETOLD_WB 1 +# endif +# undef ETHR_HAVE_NATMC32_OR_RETOLD_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_ACQB +# define ETHR_HAVE_NATMC32_OR_RETOLD_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC32_OR_RETOLD_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_RELB +# define ETHR_HAVE_NATMC32_OR_RETOLD_RELB 1 +# endif +# undef ETHR_HAVE_NATMC32_OR_RETOLD_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_MB +# define ETHR_HAVE_NATMC32_OR_RETOLD_MB 1 +# endif +#else +# error "Invalid native atomic size" +#endif + +#if (!defined(ETHR_HAVE_NATMC32_CMPXCHG) \ + && !defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) \ + && !defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) \ + && !defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) \ + && !defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) \ + && !defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)) +# error "No native cmpxchg() op available" +#endif + + /* - * --- 32-bit atomics --------------------------------------------------------- + * Read op used together with cmpxchg() fallback when no native op present. */ +#if defined(ETHR_HAVE_NATMC32_READ) +#define ETHR_NATMC32_CMPXCHG_FALLBACK_READ__(VAR) \ + ETHR_NATMC32_FUNC__(read)(VAR) +#elif defined(ETHR_HAVE_NATMC32_READ_RB) +#define ETHR_NATMC32_CMPXCHG_FALLBACK_READ__(VAR) \ + ETHR_NATMC32_FUNC__(read_rb)(VAR) +#elif defined(ETHR_HAVE_NATMC32_READ_WB) +#define ETHR_NATMC32_CMPXCHG_FALLBACK_READ__(VAR) \ + ETHR_NATMC32_FUNC__(read_wb)(VAR) +#elif defined(ETHR_HAVE_NATMC32_READ_ACQB) +#define ETHR_NATMC32_CMPXCHG_FALLBACK_READ__(VAR) \ + ETHR_NATMC32_FUNC__(read_acqb)(VAR) +#elif defined(ETHR_HAVE_NATMC32_READ_RELB) +#define ETHR_NATMC32_CMPXCHG_FALLBACK_READ__(VAR) \ + ETHR_NATMC32_FUNC__(read_relb)(VAR) +#elif defined(ETHR_HAVE_NATMC32_READ_MB) +#define ETHR_NATMC32_CMPXCHG_FALLBACK_READ__(VAR) \ + ETHR_NATMC32_FUNC__(read_mb)(VAR) +#else +/* + * We have no native read() op; guess zero and then use the + * the atomics actual value returned from cmpxchg(). + */ +#define ETHR_NATMC32_CMPXCHG_FALLBACK_READ__(VAR) \ + ((ETHR_NAINT32_T__) 0) +#endif + +/* + * Native cmpxchg() fallback used when no native op present. + */ +#define ETHR_NATMC32_CMPXCHG_FALLBACK__(CMPXCHG, VAR, AVAL, OPS) \ +do { \ + ethr_sint32_t AVAL; \ + ETHR_NAINT32_T__ new__, act__, exp__; \ + act__ = ETHR_NATMC32_CMPXCHG_FALLBACK_READ__(VAR); \ + do { \ + exp__ = act__; \ + AVAL = (ethr_sint32_t) act__; \ + { OPS; } \ + new__ = (ETHR_NAINT32_T__) AVAL; \ + act__ = CMPXCHG(VAR, new__, exp__); \ + } while (__builtin_expect(act__ != exp__, 0)); \ +} while (0) -static ETHR_INLINE ethr_sint32_t * -ETHR_INLINE_FUNC_NAME_(ethr_atomic32_addr)(ethr_atomic32_t *var) + + +/* --- addr() --- */ + +static ETHR_INLINE ethr_sint32_t *ETHR_ATMC32_FUNC__(addr)(ethr_atomic32_t *var) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return ethr_native_atomic32_addr(var); + return (ethr_sint32_t *) ETHR_NATMC32_ADDR_FUNC__(var); + +} + + +/* --- cmpxchg() --- */ + + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(cmpxchg)(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val) +{ + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_CMPXCHG) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); #else - return (ethr_sint32_t *) var; +#error "Missing implementation of ethr_atomic32_cmpxchg()!" #endif + return res; } -static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_atomic32_init)(ethr_atomic32_t *var, - ethr_sint32_t i) +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(cmpxchg_rb)(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - ETHR_NATMC32_FUNC__(init)(var, (ETHR_NAINT32_T__) i); + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); + ETHR_MEMBAR(ETHR_LoadLoad); #else - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = i); +#error "Missing implementation of ethr_atomic32_cmpxchg_rb()!" #endif + return res; } -static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_atomic32_set)(ethr_atomic32_t *var, ethr_sint32_t i) +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(cmpxchg_wb)(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - ETHR_NATMC32_FUNC__(set)(var, (ETHR_NAINT32_T__) i); + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); #else - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = i); +#error "Missing implementation of ethr_atomic32_cmpxchg_wb()!" #endif + return res; } -static ETHR_INLINE ethr_sint32_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic32_read)(ethr_atomic32_t *var) +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(cmpxchg_acqb)(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return (ethr_sint32_t) ETHR_NATMC32_FUNC__(read)(var); + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); #else +#error "Missing implementation of ethr_atomic32_cmpxchg_acqb()!" +#endif + return res; +} + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(cmpxchg_relb)(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val) +{ ethr_sint32_t res; - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var); +#if defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); +#else +#error "Missing implementation of ethr_atomic32_cmpxchg_relb()!" +#endif return res; +} + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(cmpxchg_mb)(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val) +{ + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else +#error "Missing implementation of ethr_atomic32_cmpxchg_mb()!" #endif + return res; } -static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_atomic32_add)(ethr_atomic32_t *var, - ethr_sint32_t incr) + +/* --- xchg() --- */ + + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(xchg)(ethr_atomic32_t *var, ethr_sint32_t val) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - ETHR_NATMC32_FUNC__(add)(var, (ETHR_NAINT32_T__) incr); + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_XCHG) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_XCHG_RB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_rb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_XCHG_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_wb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_XCHG_ACQB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_XCHG_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_relb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_XCHG_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val); #else - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += incr); +#error "Missing implementation of ethr_atomic32_xchg()!" #endif -} - -static ETHR_INLINE ethr_sint32_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic32_add_read)(ethr_atomic32_t *var, - ethr_sint32_t i) + return res; +} + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(xchg_rb)(ethr_atomic32_t *var, ethr_sint32_t val) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return (ethr_sint32_t) - ETHR_NATMC32_FUNC__(add_return)(var, (ETHR_NAINT32_T__) i); + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_XCHG_RB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_rb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_XCHG) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_XCHG_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_XCHG_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_wb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_XCHG_ACQB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_acqb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_XCHG_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_relb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval = val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val); + ETHR_MEMBAR(ETHR_LoadLoad); #else +#error "Missing implementation of ethr_atomic32_xchg_rb()!" +#endif + return res; +} + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(xchg_wb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ ethr_sint32_t res; - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += i; res = *var); +#if defined(ETHR_HAVE_NATMC32_XCHG_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_wb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_XCHG) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_XCHG_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_XCHG_RB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_rb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_XCHG_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_XCHG_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_relb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val); +#else +#error "Missing implementation of ethr_atomic32_xchg_wb()!" +#endif return res; +} + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(xchg_acqb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_XCHG_ACQB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_XCHG_RB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_rb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_XCHG) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_XCHG_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_XCHG_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_wb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_XCHG_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_relb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval = val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else +#error "Missing implementation of ethr_atomic32_xchg_acqb()!" #endif + return res; } -static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_atomic32_inc)(ethr_atomic32_t *var) +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(xchg_relb)(ethr_atomic32_t *var, ethr_sint32_t val) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - ETHR_NATMC32_FUNC__(inc)(var); + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_XCHG_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_relb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_XCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_wb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_XCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_XCHG_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_XCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_rb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_XCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val); #else - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var)); +#error "Missing implementation of ethr_atomic32_xchg_relb()!" #endif + return res; } -static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_atomic32_dec)(ethr_atomic32_t *var) +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(xchg_mb)(ethr_atomic32_t *var, ethr_sint32_t val) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - ETHR_NATMC32_FUNC__(dec)(var); + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_XCHG_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_XCHG_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_relb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_XCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_XCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_wb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_XCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_rb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_XCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval = val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else +#error "Missing implementation of ethr_atomic32_xchg_mb()!" +#endif + return res; +} + + +/* --- set() --- */ + + +static ETHR_INLINE void ETHR_ATMC32_FUNC__(set)(ethr_atomic32_t *var, ethr_sint32_t val) +{ +#if defined(ETHR_HAVE_NATMC32_SET) + ETHR_NATMC32_FUNC__(set)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_SET_RB) + ETHR_NATMC32_FUNC__(set_rb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_SET_WB) + ETHR_NATMC32_FUNC__(set_wb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_SET_ACQB) + ETHR_NATMC32_FUNC__(set_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_SET_RELB) + ETHR_NATMC32_FUNC__(set_relb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_SET_MB) + ETHR_NATMC32_FUNC__(set_mb)(var, (ETHR_NAINT32_T__) val); +#else + (void) ETHR_ATMC32_FUNC__(xchg)(var, val); +#endif +} + +static ETHR_INLINE void ETHR_ATMC32_FUNC__(set_rb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ +#if defined(ETHR_HAVE_NATMC32_SET_RB) + ETHR_NATMC32_FUNC__(set_rb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_SET) + ETHR_NATMC32_FUNC__(set)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_SET_MB) + ETHR_NATMC32_FUNC__(set_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_SET_WB) + ETHR_NATMC32_FUNC__(set_wb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_SET_ACQB) + ETHR_NATMC32_FUNC__(set_acqb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_SET_RELB) + ETHR_NATMC32_FUNC__(set_relb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + (void) ETHR_ATMC32_FUNC__(xchg_rb)(var, val); +#endif +} + +static ETHR_INLINE void ETHR_ATMC32_FUNC__(set_wb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ +#if defined(ETHR_HAVE_NATMC32_SET_WB) + ETHR_NATMC32_FUNC__(set_wb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_SET) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_FUNC__(set)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_SET_MB) + ETHR_NATMC32_FUNC__(set_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_SET_RB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_FUNC__(set_rb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_SET_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_FUNC__(set_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_SET_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_FUNC__(set_relb)(var, (ETHR_NAINT32_T__) val); +#else + (void) ETHR_ATMC32_FUNC__(xchg_wb)(var, val); +#endif +} + +static ETHR_INLINE void ETHR_ATMC32_FUNC__(set_acqb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ +#if defined(ETHR_HAVE_NATMC32_SET_ACQB) + ETHR_NATMC32_FUNC__(set_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_SET_RB) + ETHR_NATMC32_FUNC__(set_rb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_SET) + ETHR_NATMC32_FUNC__(set)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_SET_MB) + ETHR_NATMC32_FUNC__(set_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_SET_WB) + ETHR_NATMC32_FUNC__(set_wb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_SET_RELB) + ETHR_NATMC32_FUNC__(set_relb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#else + (void) ETHR_ATMC32_FUNC__(xchg_acqb)(var, val); +#endif +} + +static ETHR_INLINE void ETHR_ATMC32_FUNC__(set_relb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ +#if defined(ETHR_HAVE_NATMC32_SET_RELB) + ETHR_NATMC32_FUNC__(set_relb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_SET_WB) + ETHR_MEMBAR(ETHR_LoadStore); + ETHR_NATMC32_FUNC__(set_wb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_SET) + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_NATMC32_FUNC__(set)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_SET_MB) + ETHR_NATMC32_FUNC__(set_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_SET_RB) + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_NATMC32_FUNC__(set_rb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_SET_ACQB) + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_NATMC32_FUNC__(set_acqb)(var, (ETHR_NAINT32_T__) val); +#else + (void) ETHR_ATMC32_FUNC__(xchg_relb)(var, val); +#endif +} + +static ETHR_INLINE void ETHR_ATMC32_FUNC__(set_mb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ +#if defined(ETHR_HAVE_NATMC32_SET_MB) + ETHR_NATMC32_FUNC__(set_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_SET_RELB) + ETHR_NATMC32_FUNC__(set_relb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_SET_ACQB) + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_NATMC32_FUNC__(set_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_SET_WB) + ETHR_MEMBAR(ETHR_LoadStore); + ETHR_NATMC32_FUNC__(set_wb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_SET_RB) + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_NATMC32_FUNC__(set_rb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_SET) + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_NATMC32_FUNC__(set)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#else + (void) ETHR_ATMC32_FUNC__(xchg_mb)(var, val); +#endif +} + + +/* --- init() --- */ + + +static ETHR_INLINE void ETHR_ATMC32_FUNC__(init)(ethr_atomic32_t *var, ethr_sint32_t val) +{ +#if defined(ETHR_HAVE_NATMC32_INIT) + ETHR_NATMC32_FUNC__(init)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_INIT_RB) + ETHR_NATMC32_FUNC__(init_rb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_INIT_WB) + ETHR_NATMC32_FUNC__(init_wb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_INIT_ACQB) + ETHR_NATMC32_FUNC__(init_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_INIT_RELB) + ETHR_NATMC32_FUNC__(init_relb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_INIT_MB) + ETHR_NATMC32_FUNC__(init_mb)(var, (ETHR_NAINT32_T__) val); #else - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var)); + ETHR_ATMC32_FUNC__(set)(var, val); #endif } -static ETHR_INLINE ethr_sint32_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic32_inc_read)(ethr_atomic32_t *var) +static ETHR_INLINE void ETHR_ATMC32_FUNC__(init_rb)(ethr_atomic32_t *var, ethr_sint32_t val) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return)(var); +#if defined(ETHR_HAVE_NATMC32_INIT_RB) + ETHR_NATMC32_FUNC__(init_rb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_INIT) + ETHR_NATMC32_FUNC__(init)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_INIT_MB) + ETHR_NATMC32_FUNC__(init_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_INIT_WB) + ETHR_NATMC32_FUNC__(init_wb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_INIT_ACQB) + ETHR_NATMC32_FUNC__(init_acqb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_INIT_RELB) + ETHR_NATMC32_FUNC__(init_relb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); #else + ETHR_ATMC32_FUNC__(set_rb)(var, val); +#endif +} + +static ETHR_INLINE void ETHR_ATMC32_FUNC__(init_wb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ +#if defined(ETHR_HAVE_NATMC32_INIT_WB) + ETHR_NATMC32_FUNC__(init_wb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_INIT) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_FUNC__(init)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_INIT_MB) + ETHR_NATMC32_FUNC__(init_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_INIT_RB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_FUNC__(init_rb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_INIT_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_FUNC__(init_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_INIT_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_FUNC__(init_relb)(var, (ETHR_NAINT32_T__) val); +#else + ETHR_ATMC32_FUNC__(set_wb)(var, val); +#endif +} + +static ETHR_INLINE void ETHR_ATMC32_FUNC__(init_acqb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ +#if defined(ETHR_HAVE_NATMC32_INIT_ACQB) + ETHR_NATMC32_FUNC__(init_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_INIT_RB) + ETHR_NATMC32_FUNC__(init_rb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_INIT) + ETHR_NATMC32_FUNC__(init)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_INIT_MB) + ETHR_NATMC32_FUNC__(init_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_INIT_WB) + ETHR_NATMC32_FUNC__(init_wb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_INIT_RELB) + ETHR_NATMC32_FUNC__(init_relb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_ATMC32_FUNC__(set_acqb)(var, val); +#endif +} + +static ETHR_INLINE void ETHR_ATMC32_FUNC__(init_relb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ +#if defined(ETHR_HAVE_NATMC32_INIT_RELB) + ETHR_NATMC32_FUNC__(init_relb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_INIT_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad); + ETHR_NATMC32_FUNC__(init_wb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_INIT) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_NATMC32_FUNC__(init)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_INIT_MB) + ETHR_NATMC32_FUNC__(init_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_INIT_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_NATMC32_FUNC__(init_rb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_INIT_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_NATMC32_FUNC__(init_acqb)(var, (ETHR_NAINT32_T__) val); +#else + ETHR_ATMC32_FUNC__(set_relb)(var, val); +#endif +} + +static ETHR_INLINE void ETHR_ATMC32_FUNC__(init_mb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ +#if defined(ETHR_HAVE_NATMC32_INIT_MB) + ETHR_NATMC32_FUNC__(init_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_INIT_RELB) + ETHR_NATMC32_FUNC__(init_relb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_INIT_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_NATMC32_FUNC__(init_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_INIT_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad); + ETHR_NATMC32_FUNC__(init_wb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_INIT_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_NATMC32_FUNC__(init_rb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_INIT) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_NATMC32_FUNC__(init)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_ATMC32_FUNC__(set_mb)(var, val); +#endif +} + + +/* --- add_read() --- */ + + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(add_read)(ethr_atomic32_t *var, ethr_sint32_t val) +{ ethr_sint32_t res; - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var)); +#if defined(ETHR_HAVE_NATMC32_ADD_RETURN) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_rb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_wb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_ACQB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_relb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval); +#else +#error "Missing implementation of ethr_atomic32_add_read()!" +#endif return res; +} + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(add_read_rb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_ADD_RETURN_RB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_rb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_wb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_ACQB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_acqb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_relb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, aval += val; res = aval); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval); + ETHR_MEMBAR(ETHR_LoadLoad); +#else +#error "Missing implementation of ethr_atomic32_add_read_rb()!" #endif + return res; } -static ETHR_INLINE ethr_sint32_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic32_dec_read)(ethr_atomic32_t *var) +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(add_read_wb)(ethr_atomic32_t *var, ethr_sint32_t val) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return)(var); + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_ADD_RETURN_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_wb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_rb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_relb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval); #else +#error "Missing implementation of ethr_atomic32_add_read_wb()!" +#endif + return res; +} + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(add_read_acqb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ ethr_sint32_t res; - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var)); +#if defined(ETHR_HAVE_NATMC32_ADD_RETURN_ACQB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_rb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_wb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_relb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, aval += val; res = aval); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else +#error "Missing implementation of ethr_atomic32_add_read_acqb()!" +#endif return res; +} + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(add_read_relb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_ADD_RETURN_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_relb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_wb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_rb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval); +#else +#error "Missing implementation of ethr_atomic32_add_read_relb()!" #endif + return res; } -static ETHR_INLINE ethr_sint32_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic32_read_band)(ethr_atomic32_t *var, - ethr_sint32_t mask) +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(add_read_mb)(ethr_atomic32_t *var, ethr_sint32_t val) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return (ethr_sint32_t) - ETHR_NATMC32_FUNC__(and_retold)(var, (ETHR_NAINT32_T__) mask); + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_ADD_RETURN_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_relb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_wb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_rb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, aval += val; res = aval); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); #else +#error "Missing implementation of ethr_atomic32_add_read_mb()!" +#endif + return res; +} + + +/* --- read() --- */ + + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read)(ethr_atomic32_t *var) +{ ethr_sint32_t res; - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= mask); +#if defined(ETHR_HAVE_NATMC32_READ) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read)(var); +#elif defined(ETHR_HAVE_NATMC32_READ_RB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_rb)(var); +#elif defined(ETHR_HAVE_NATMC32_READ_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_wb)(var); +#elif defined(ETHR_HAVE_NATMC32_READ_ACQB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_acqb)(var); +#elif defined(ETHR_HAVE_NATMC32_READ_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_relb)(var); +#elif defined(ETHR_HAVE_NATMC32_READ_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_mb)(var); +#else + res = ETHR_ATMC32_FUNC__(cmpxchg)(var, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__); +#endif return res; +} + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_rb)(ethr_atomic32_t *var) +{ + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_READ_RB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_rb)(var); +#elif defined(ETHR_HAVE_NATMC32_READ) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_READ_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_mb)(var); +#elif defined(ETHR_HAVE_NATMC32_READ_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_READ_ACQB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_acqb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_READ_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + res = ETHR_ATMC32_FUNC__(cmpxchg_rb)(var, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__); #endif + return res; } -static ETHR_INLINE ethr_sint32_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic32_read_bor)(ethr_atomic32_t *var, - ethr_sint32_t mask) +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_wb)(ethr_atomic32_t *var) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return - (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold)(var, - (ETHR_NAINT32_T__) mask); + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_READ_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_wb)(var); +#elif defined(ETHR_HAVE_NATMC32_READ) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read)(var); +#elif defined(ETHR_HAVE_NATMC32_READ_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_mb)(var); +#elif defined(ETHR_HAVE_NATMC32_READ_RB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_rb)(var); +#elif defined(ETHR_HAVE_NATMC32_READ_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_acqb)(var); +#elif defined(ETHR_HAVE_NATMC32_READ_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_relb)(var); #else + res = ETHR_ATMC32_FUNC__(cmpxchg_wb)(var, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__); +#endif + return res; +} + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_acqb)(ethr_atomic32_t *var) +{ ethr_sint32_t res; - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= mask); +#if defined(ETHR_HAVE_NATMC32_READ_ACQB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_acqb)(var); +#elif defined(ETHR_HAVE_NATMC32_READ_RB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_rb)(var); + ETHR_MEMBAR(ETHR_LoadStore); +#elif defined(ETHR_HAVE_NATMC32_READ) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#elif defined(ETHR_HAVE_NATMC32_READ_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_mb)(var); +#elif defined(ETHR_HAVE_NATMC32_READ_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#elif defined(ETHR_HAVE_NATMC32_READ_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#else + res = ETHR_ATMC32_FUNC__(cmpxchg_acqb)(var, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__); +#endif return res; +} + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_relb)(ethr_atomic32_t *var) +{ + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_READ_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_relb)(var); +#elif defined(ETHR_HAVE_NATMC32_READ_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_wb)(var); +#elif defined(ETHR_HAVE_NATMC32_READ) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read)(var); +#elif defined(ETHR_HAVE_NATMC32_READ_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_mb)(var); +#elif defined(ETHR_HAVE_NATMC32_READ_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_rb)(var); +#elif defined(ETHR_HAVE_NATMC32_READ_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_acqb)(var); +#else + res = ETHR_ATMC32_FUNC__(cmpxchg_relb)(var, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__); #endif + return res; } -static ETHR_INLINE ethr_sint32_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic32_xchg)(ethr_atomic32_t *var, - ethr_sint32_t new) +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_mb)(ethr_atomic32_t *var) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg)(var, - (ETHR_NAINT32_T__) new); + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_READ_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_mb)(var); +#elif defined(ETHR_HAVE_NATMC32_READ_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#elif defined(ETHR_HAVE_NATMC32_READ_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_acqb)(var); +#elif defined(ETHR_HAVE_NATMC32_READ_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#elif defined(ETHR_HAVE_NATMC32_READ_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_rb)(var); + ETHR_MEMBAR(ETHR_LoadStore); +#elif defined(ETHR_HAVE_NATMC32_READ) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); #else + res = ETHR_ATMC32_FUNC__(cmpxchg_mb)(var, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__); +#endif + return res; +} + + +/* --- inc_read() --- */ + + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(inc_read)(ethr_atomic32_t *var) +{ ethr_sint32_t res; - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = new); +#if defined(ETHR_HAVE_NATMC32_INC_RETURN) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_rb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_wb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_ACQB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_acqb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_relb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_mb)(var); +#else + res = ETHR_ATMC32_FUNC__(add_read)(var, (ethr_sint32_t) 1); +#endif return res; +} + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(inc_read_rb)(ethr_atomic32_t *var) +{ + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_INC_RETURN_RB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_rb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_mb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_ACQB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_acqb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + res = ETHR_ATMC32_FUNC__(add_read_rb)(var, (ethr_sint32_t) 1); #endif + return res; } -static ETHR_INLINE ethr_sint32_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic32_cmpxchg)(ethr_atomic32_t *var, - ethr_sint32_t new, - ethr_sint32_t exp) +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(inc_read_wb)(ethr_atomic32_t *var) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg)(var, - (ETHR_NAINT32_T__) new, - (ETHR_NAINT32_T__) exp); + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_INC_RETURN_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_wb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_mb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_rb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_acqb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_relb)(var); #else + res = ETHR_ATMC32_FUNC__(add_read_wb)(var, (ethr_sint32_t) 1); +#endif + return res; +} + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(inc_read_acqb)(ethr_atomic32_t *var) +{ ethr_sint32_t res; - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, - { - res = *var; - if (__builtin_expect(res == exp, 1)) - *var = new; - }); +#if defined(ETHR_HAVE_NATMC32_INC_RETURN_ACQB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_acqb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_rb)(var); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_mb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + res = ETHR_ATMC32_FUNC__(add_read_acqb)(var, (ethr_sint32_t) 1); +#endif return res; +} + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(inc_read_relb)(ethr_atomic32_t *var) +{ + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_INC_RETURN_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_relb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_wb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_mb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_rb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_acqb)(var); +#else + res = ETHR_ATMC32_FUNC__(add_read_relb)(var, (ethr_sint32_t) 1); #endif + return res; +} + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(inc_read_mb)(ethr_atomic32_t *var) +{ + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_INC_RETURN_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_mb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_acqb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_rb)(var); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + res = ETHR_ATMC32_FUNC__(add_read_mb)(var, (ethr_sint32_t) 1); +#endif + return res; } -/* - * Important memory barrier requirements. - * - * The following atomic operations *must* supply a memory barrier of - * at least the type specified by its suffix: - * _acqb = acquire barrier - * _relb = release barrier - */ -static ETHR_INLINE ethr_sint32_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic32_read_acqb)(ethr_atomic32_t *var) +/* --- dec_read() --- */ + + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(dec_read)(ethr_atomic32_t *var) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_acqb)(var); + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_DEC_RETURN) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_rb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_wb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_ACQB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_acqb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_relb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_mb)(var); #else - return ETHR_INLINE_FUNC_NAME_(ethr_atomic32_read)(var); + res = ETHR_ATMC32_FUNC__(add_read)(var, (ethr_sint32_t) -1); #endif + return res; } -static ETHR_INLINE ethr_sint32_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic32_inc_read_acqb)(ethr_atomic32_t *var) +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(dec_read_rb)(ethr_atomic32_t *var) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_acqb)(var); + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_DEC_RETURN_RB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_rb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_mb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_ACQB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_acqb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); #else - return ETHR_INLINE_FUNC_NAME_(ethr_atomic32_inc_read)(var); + res = ETHR_ATMC32_FUNC__(add_read_rb)(var, (ethr_sint32_t) -1); #endif + return res; } -static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_atomic32_set_relb)(ethr_atomic32_t *var, - ethr_sint32_t val) +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(dec_read_wb)(ethr_atomic32_t *var) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - ETHR_NATMC32_FUNC__(set_relb)(var, (ETHR_NAINT32_T__) val); + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_DEC_RETURN_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_wb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_mb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_rb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_acqb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_relb)(var); +#else + res = ETHR_ATMC32_FUNC__(add_read_wb)(var, (ethr_sint32_t) -1); +#endif + return res; +} + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(dec_read_acqb)(ethr_atomic32_t *var) +{ + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_DEC_RETURN_ACQB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_acqb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_rb)(var); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_mb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); #else - ETHR_INLINE_FUNC_NAME_(ethr_atomic32_set)(var, val); + res = ETHR_ATMC32_FUNC__(add_read_acqb)(var, (ethr_sint32_t) -1); #endif + return res; } -static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_atomic32_dec_relb)(ethr_atomic32_t *var) +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(dec_read_relb)(ethr_atomic32_t *var) +{ + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_DEC_RETURN_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_relb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_wb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_mb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_rb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_acqb)(var); +#else + res = ETHR_ATMC32_FUNC__(add_read_relb)(var, (ethr_sint32_t) -1); +#endif + return res; +} + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(dec_read_mb)(ethr_atomic32_t *var) +{ + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_DEC_RETURN_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_mb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_acqb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_rb)(var); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + res = ETHR_ATMC32_FUNC__(add_read_mb)(var, (ethr_sint32_t) -1); +#endif + return res; +} + + +/* --- add() --- */ + + +static ETHR_INLINE void ETHR_ATMC32_FUNC__(add)(ethr_atomic32_t *var, ethr_sint32_t val) +{ +#if defined(ETHR_HAVE_NATMC32_ADD) + ETHR_NATMC32_FUNC__(add)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_RB) + ETHR_NATMC32_FUNC__(add_rb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_WB) + ETHR_NATMC32_FUNC__(add_wb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_ACQB) + ETHR_NATMC32_FUNC__(add_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_RELB) + ETHR_NATMC32_FUNC__(add_relb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_MB) + ETHR_NATMC32_FUNC__(add_mb)(var, (ETHR_NAINT32_T__) val); +#else + (void) ETHR_ATMC32_FUNC__(add_read)(var, val); +#endif +} + +static ETHR_INLINE void ETHR_ATMC32_FUNC__(add_rb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ +#if defined(ETHR_HAVE_NATMC32_ADD_RB) + ETHR_NATMC32_FUNC__(add_rb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD) + ETHR_NATMC32_FUNC__(add)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_ADD_MB) + ETHR_NATMC32_FUNC__(add_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_WB) + ETHR_NATMC32_FUNC__(add_wb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_ADD_ACQB) + ETHR_NATMC32_FUNC__(add_acqb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_ADD_RELB) + ETHR_NATMC32_FUNC__(add_relb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + (void) ETHR_ATMC32_FUNC__(add_read_rb)(var, val); +#endif +} + +static ETHR_INLINE void ETHR_ATMC32_FUNC__(add_wb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ +#if defined(ETHR_HAVE_NATMC32_ADD_WB) + ETHR_NATMC32_FUNC__(add_wb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_FUNC__(add)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_MB) + ETHR_NATMC32_FUNC__(add_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_RB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_FUNC__(add_rb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_FUNC__(add_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_FUNC__(add_relb)(var, (ETHR_NAINT32_T__) val); +#else + (void) ETHR_ATMC32_FUNC__(add_read_wb)(var, val); +#endif +} + +static ETHR_INLINE void ETHR_ATMC32_FUNC__(add_acqb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ +#if defined(ETHR_HAVE_NATMC32_ADD_ACQB) + ETHR_NATMC32_FUNC__(add_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_RB) + ETHR_NATMC32_FUNC__(add_rb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_ADD) + ETHR_NATMC32_FUNC__(add)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_ADD_MB) + ETHR_NATMC32_FUNC__(add_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_WB) + ETHR_NATMC32_FUNC__(add_wb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_ADD_RELB) + ETHR_NATMC32_FUNC__(add_relb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + (void) ETHR_ATMC32_FUNC__(add_read_acqb)(var, val); +#endif +} + +static ETHR_INLINE void ETHR_ATMC32_FUNC__(add_relb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ +#if defined(ETHR_HAVE_NATMC32_ADD_RELB) + ETHR_NATMC32_FUNC__(add_relb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_FUNC__(add_wb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_FUNC__(add)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_MB) + ETHR_NATMC32_FUNC__(add_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_FUNC__(add_rb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_FUNC__(add_acqb)(var, (ETHR_NAINT32_T__) val); +#else + (void) ETHR_ATMC32_FUNC__(add_read_relb)(var, val); +#endif +} + +static ETHR_INLINE void ETHR_ATMC32_FUNC__(add_mb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ +#if defined(ETHR_HAVE_NATMC32_ADD_MB) + ETHR_NATMC32_FUNC__(add_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_RELB) + ETHR_NATMC32_FUNC__(add_relb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_ADD_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_FUNC__(add_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_FUNC__(add_wb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_ADD_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_FUNC__(add_rb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_ADD) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_FUNC__(add)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + (void) ETHR_ATMC32_FUNC__(add_read_mb)(var, val); +#endif +} + + +/* --- inc() --- */ + + +static ETHR_INLINE void ETHR_ATMC32_FUNC__(inc)(ethr_atomic32_t *var) +{ +#if defined(ETHR_HAVE_NATMC32_INC) + ETHR_NATMC32_FUNC__(inc)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_RB) + ETHR_NATMC32_FUNC__(inc_rb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_WB) + ETHR_NATMC32_FUNC__(inc_wb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_ACQB) + ETHR_NATMC32_FUNC__(inc_acqb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_RELB) + ETHR_NATMC32_FUNC__(inc_relb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_MB) + ETHR_NATMC32_FUNC__(inc_mb)(var); +#else + (void) ETHR_ATMC32_FUNC__(inc_read)(var); +#endif +} + +static ETHR_INLINE void ETHR_ATMC32_FUNC__(inc_rb)(ethr_atomic32_t *var) +{ +#if defined(ETHR_HAVE_NATMC32_INC_RB) + ETHR_NATMC32_FUNC__(inc_rb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC) + ETHR_NATMC32_FUNC__(inc)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_INC_MB) + ETHR_NATMC32_FUNC__(inc_mb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_WB) + ETHR_NATMC32_FUNC__(inc_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_INC_ACQB) + ETHR_NATMC32_FUNC__(inc_acqb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_INC_RELB) + ETHR_NATMC32_FUNC__(inc_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + (void) ETHR_ATMC32_FUNC__(inc_read_rb)(var); +#endif +} + +static ETHR_INLINE void ETHR_ATMC32_FUNC__(inc_wb)(ethr_atomic32_t *var) +{ +#if defined(ETHR_HAVE_NATMC32_INC_WB) + ETHR_NATMC32_FUNC__(inc_wb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_FUNC__(inc)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_MB) + ETHR_NATMC32_FUNC__(inc_mb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_RB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_FUNC__(inc_rb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_FUNC__(inc_acqb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_FUNC__(inc_relb)(var); +#else + (void) ETHR_ATMC32_FUNC__(inc_read_wb)(var); +#endif +} + +static ETHR_INLINE void ETHR_ATMC32_FUNC__(inc_acqb)(ethr_atomic32_t *var) +{ +#if defined(ETHR_HAVE_NATMC32_INC_ACQB) + ETHR_NATMC32_FUNC__(inc_acqb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_RB) + ETHR_NATMC32_FUNC__(inc_rb)(var); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_INC) + ETHR_NATMC32_FUNC__(inc)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_INC_MB) + ETHR_NATMC32_FUNC__(inc_mb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_WB) + ETHR_NATMC32_FUNC__(inc_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_INC_RELB) + ETHR_NATMC32_FUNC__(inc_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + (void) ETHR_ATMC32_FUNC__(inc_read_acqb)(var); +#endif +} + +static ETHR_INLINE void ETHR_ATMC32_FUNC__(inc_relb)(ethr_atomic32_t *var) +{ +#if defined(ETHR_HAVE_NATMC32_INC_RELB) + ETHR_NATMC32_FUNC__(inc_relb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_FUNC__(inc_wb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_FUNC__(inc)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_MB) + ETHR_NATMC32_FUNC__(inc_mb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_FUNC__(inc_rb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_FUNC__(inc_acqb)(var); +#else + (void) ETHR_ATMC32_FUNC__(inc_read_relb)(var); +#endif +} + +static ETHR_INLINE void ETHR_ATMC32_FUNC__(inc_mb)(ethr_atomic32_t *var) +{ +#if defined(ETHR_HAVE_NATMC32_INC_MB) + ETHR_NATMC32_FUNC__(inc_mb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_RELB) + ETHR_NATMC32_FUNC__(inc_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_INC_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_FUNC__(inc_acqb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_FUNC__(inc_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_INC_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_FUNC__(inc_rb)(var); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_INC) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_FUNC__(inc)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + (void) ETHR_ATMC32_FUNC__(inc_read_mb)(var); +#endif +} + + +/* --- dec() --- */ + + +static ETHR_INLINE void ETHR_ATMC32_FUNC__(dec)(ethr_atomic32_t *var) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS +#if defined(ETHR_HAVE_NATMC32_DEC) + ETHR_NATMC32_FUNC__(dec)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_RB) + ETHR_NATMC32_FUNC__(dec_rb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_WB) + ETHR_NATMC32_FUNC__(dec_wb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_ACQB) + ETHR_NATMC32_FUNC__(dec_acqb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_RELB) + ETHR_NATMC32_FUNC__(dec_relb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_MB) + ETHR_NATMC32_FUNC__(dec_mb)(var); +#else + (void) ETHR_ATMC32_FUNC__(dec_read)(var); +#endif +} + +static ETHR_INLINE void ETHR_ATMC32_FUNC__(dec_rb)(ethr_atomic32_t *var) +{ +#if defined(ETHR_HAVE_NATMC32_DEC_RB) + ETHR_NATMC32_FUNC__(dec_rb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC) + ETHR_NATMC32_FUNC__(dec)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_DEC_MB) + ETHR_NATMC32_FUNC__(dec_mb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_WB) + ETHR_NATMC32_FUNC__(dec_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_DEC_ACQB) + ETHR_NATMC32_FUNC__(dec_acqb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_DEC_RELB) + ETHR_NATMC32_FUNC__(dec_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + (void) ETHR_ATMC32_FUNC__(dec_read_rb)(var); +#endif +} + +static ETHR_INLINE void ETHR_ATMC32_FUNC__(dec_wb)(ethr_atomic32_t *var) +{ +#if defined(ETHR_HAVE_NATMC32_DEC_WB) + ETHR_NATMC32_FUNC__(dec_wb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_FUNC__(dec)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_MB) + ETHR_NATMC32_FUNC__(dec_mb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_RB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_FUNC__(dec_rb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_FUNC__(dec_acqb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_RELB) + ETHR_MEMBAR(ETHR_StoreStore); ETHR_NATMC32_FUNC__(dec_relb)(var); #else - ETHR_INLINE_FUNC_NAME_(ethr_atomic32_dec)(var); + (void) ETHR_ATMC32_FUNC__(dec_read_wb)(var); #endif } -static ETHR_INLINE ethr_sint32_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic32_dec_read_relb)(ethr_atomic32_t *var) +static ETHR_INLINE void ETHR_ATMC32_FUNC__(dec_acqb)(ethr_atomic32_t *var) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_relb)(var); +#if defined(ETHR_HAVE_NATMC32_DEC_ACQB) + ETHR_NATMC32_FUNC__(dec_acqb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_RB) + ETHR_NATMC32_FUNC__(dec_rb)(var); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_DEC) + ETHR_NATMC32_FUNC__(dec)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_DEC_MB) + ETHR_NATMC32_FUNC__(dec_mb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_WB) + ETHR_NATMC32_FUNC__(dec_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_DEC_RELB) + ETHR_NATMC32_FUNC__(dec_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); #else - return ETHR_INLINE_FUNC_NAME_(ethr_atomic32_dec_read)(var); + (void) ETHR_ATMC32_FUNC__(dec_read_acqb)(var); #endif } -static ETHR_INLINE ethr_sint32_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic32_cmpxchg_acqb)(ethr_atomic32_t *var, - ethr_sint32_t new, - ethr_sint32_t exp) +static ETHR_INLINE void ETHR_ATMC32_FUNC__(dec_relb)(ethr_atomic32_t *var) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return (ethr_sint32_t) - ETHR_NATMC32_FUNC__(cmpxchg_acqb)(var, - (ETHR_NAINT32_T__) new, - (ETHR_NAINT32_T__) exp); +#if defined(ETHR_HAVE_NATMC32_DEC_RELB) + ETHR_NATMC32_FUNC__(dec_relb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_FUNC__(dec_wb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_FUNC__(dec)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_MB) + ETHR_NATMC32_FUNC__(dec_mb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_FUNC__(dec_rb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_FUNC__(dec_acqb)(var); #else - return ETHR_INLINE_FUNC_NAME_(ethr_atomic32_cmpxchg)(var, new, exp); + (void) ETHR_ATMC32_FUNC__(dec_read_relb)(var); #endif } -static ETHR_INLINE ethr_sint32_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic32_cmpxchg_relb)(ethr_atomic32_t *var, - ethr_sint32_t new, - ethr_sint32_t exp) +static ETHR_INLINE void ETHR_ATMC32_FUNC__(dec_mb)(ethr_atomic32_t *var) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return (ethr_sint32_t) - ETHR_NATMC32_FUNC__(cmpxchg_relb)(var, - (ETHR_NAINT32_T__) new, - (ETHR_NAINT32_T__) exp); +#if defined(ETHR_HAVE_NATMC32_DEC_MB) + ETHR_NATMC32_FUNC__(dec_mb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_RELB) + ETHR_NATMC32_FUNC__(dec_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_DEC_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_FUNC__(dec_acqb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_FUNC__(dec_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_DEC_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_FUNC__(dec_rb)(var); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_DEC) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_FUNC__(dec)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); #else - return ETHR_INLINE_FUNC_NAME_(ethr_atomic32_cmpxchg)(var, new, exp); + (void) ETHR_ATMC32_FUNC__(dec_read_mb)(var); #endif } -#endif /* ETHR_TRY_INLINE_FUNCS */ +/* --- read_band() --- */ -#undef ETHR_NAINT_T__ -#undef ETHR_NATMC_FUNC__ -#undef ETHR_NATMC_ADDR_FUNC__ -#undef ETHR_NAINT32_T__ -#undef ETHR_NATMC32_FUNC__ +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_band)(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_AND_RETOLD) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_rb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_wb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_ACQB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_relb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val); +#else +#error "Missing implementation of ethr_atomic32_read_band()!" +#endif + return res; +} + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_band_rb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_AND_RETOLD_RB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_rb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_wb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_ACQB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_acqb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_relb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval &= val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val); + ETHR_MEMBAR(ETHR_LoadLoad); +#else +#error "Missing implementation of ethr_atomic32_read_band_rb()!" +#endif + return res; +} + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_band_wb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_AND_RETOLD_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_wb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_rb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_relb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val); +#else +#error "Missing implementation of ethr_atomic32_read_band_wb()!" +#endif + return res; +} + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_band_acqb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_AND_RETOLD_ACQB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_rb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_wb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_relb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval &= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else +#error "Missing implementation of ethr_atomic32_read_band_acqb()!" +#endif + return res; +} + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_band_relb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_AND_RETOLD_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_relb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_wb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_rb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val); +#else +#error "Missing implementation of ethr_atomic32_read_band_relb()!" +#endif + return res; +} + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_band_mb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_AND_RETOLD_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_relb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_wb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_rb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval &= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else +#error "Missing implementation of ethr_atomic32_read_band_mb()!" +#endif + return res; +} + + +/* --- read_bor() --- */ + + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_bor)(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_OR_RETOLD) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_rb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_wb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_ACQB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_relb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val); +#else +#error "Missing implementation of ethr_atomic32_read_bor()!" +#endif + return res; +} + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_bor_rb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_OR_RETOLD_RB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_rb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_wb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_ACQB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_acqb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_relb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval |= val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val); + ETHR_MEMBAR(ETHR_LoadLoad); +#else +#error "Missing implementation of ethr_atomic32_read_bor_rb()!" +#endif + return res; +} +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_bor_wb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_OR_RETOLD_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_wb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_rb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_relb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val); +#else +#error "Missing implementation of ethr_atomic32_read_bor_wb()!" +#endif + return res; +} + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_bor_acqb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_OR_RETOLD_ACQB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_rb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_wb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_relb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval |= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else +#error "Missing implementation of ethr_atomic32_read_bor_acqb()!" +#endif + return res; +} + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_bor_relb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_OR_RETOLD_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_relb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_wb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_rb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val); +#else +#error "Missing implementation of ethr_atomic32_read_bor_relb()!" +#endif + return res; +} + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_bor_mb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_OR_RETOLD_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_relb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_wb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_rb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval |= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else +#error "Missing implementation of ethr_atomic32_read_bor_mb()!" #endif + return res; +} + +#endif /* ETHR_ATMC32_INLINE__ */ + +#endif /* ETHR_ATOMICS_H__ */ diff --git a/erts/include/internal/ethr_internal.h b/erts/include/internal/ethr_internal.h index e9c3daf783..c9b1db5b46 100644 --- a/erts/include/internal/ethr_internal.h +++ b/erts/include/internal/ethr_internal.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2010. All Rights Reserved. + * Copyright Ericsson AB 2010-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -63,5 +63,9 @@ int ethr_late_init_common__(ethr_late_init_data *lid); void ethr_run_exit_handlers__(void); void ethr_ts_event_destructor__(void *vtsep); +#if defined(ETHR_X86_RUNTIME_CONF__) +int ethr_x86_have_cpuid__(void); +void ethr_x86_cpuid__(int *eax, int *ebx, int *ecx, int *edx); +#endif #endif diff --git a/erts/include/internal/ethr_mutex.h b/erts/include/internal/ethr_mutex.h index fadaf1e2a4..86a1e9fbdf 100644 --- a/erts/include/internal/ethr_mutex.h +++ b/erts/include/internal/ethr_mutex.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2010. All Rights Reserved. + * Copyright Ericsson AB 2010-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -22,6 +22,23 @@ * Author: Rickard Green */ +/* + * IMPORTANT note about ethr_cond_signal() and ethr_cond_broadcast() + * + * POSIX allow a call to `pthread_cond_signal' or `pthread_cond_broadcast' + * even though the associated mutex/mutexes isn't/aren't locked by the + * caller. We do not allow that by default in order to avoid a performance + * penalty on some platforms. + * + * Mutexes and condition variables can, however, be initialized as POSIX + * compliant. When initialized as such ethr_cond_signal(), and + * ethr_cond_broadcast() are allowed to be called even though the associated + * mutexes aren't locked. This will, however, incur a performance penalty on + * some platforms. + * + * POSIX compliant mutexes and condition variables *need* to be used together. + */ + #ifndef ETHR_MUTEX_H__ #define ETHR_MUTEX_H__ @@ -40,6 +57,14 @@ #endif #endif +/* #define ETHR_DBG_WIN_MTX_WITH_PTHREADS */ +#ifdef ETHR_DBG_WIN_MTX_WITH_PTHREADS +typedef pthread_mutex_t CRITICAL_SECTION; +int TryEnterCriticalSection(CRITICAL_SECTION *); +void EnterCriticalSection(CRITICAL_SECTION *); +void LeaveCriticalSection(CRITICAL_SECTION *); +#endif + #ifdef ETHR_MTX_HARD_DEBUG # ifdef __GNUC__ # warning ETHR_MTX_HARD_DEBUG @@ -54,6 +79,10 @@ # endif #endif +#ifndef ETHR_INLINE_MTX_FUNC_NAME_ +# define ETHR_INLINE_MTX_FUNC_NAME_(X) X +#endif + #if defined(ETHR_USE_OWN_RWMTX_IMPL__) || defined(ETHR_USE_OWN_MTX_IMPL__) #ifdef ETHR_DEBUG @@ -136,13 +165,19 @@ struct ethr_mutex_base_ { typedef struct { int main_spincount; int aux_spincount; + int posix_compliant; } ethr_mutex_opt; +#define ETHR_MUTEX_OPT_DEFAULT_INITER {-1, -1, 0} + typedef struct { int main_spincount; int aux_spincount; + int posix_compliant; } ethr_cond_opt; +#define ETHR_COND_OPT_DEFAULT_INITER {-1, -1, 0} + #ifdef ETHR_USE_OWN_MTX_IMPL__ typedef struct ethr_mutex_ ethr_mutex; @@ -175,7 +210,7 @@ struct ethr_cond_ { #endif }; -#else /* pthread */ +#elif defined(ETHR_PTHREADS) && !defined(ETHR_DBG_WIN_MTX_WITH_PTHREADS) typedef struct ethr_mutex_ ethr_mutex; struct ethr_mutex_ { @@ -193,7 +228,36 @@ struct ethr_cond_ { #endif }; -#endif /* pthread */ +#elif defined(ETHR_WIN32_THREADS) || defined(ETHR_DBG_WIN_MTX_WITH_PTHREADS) +# define ETHR_WIN_MUTEX__ + +typedef struct ethr_mutex_ ethr_mutex; +struct ethr_mutex_ { + int posix_compliant; + CRITICAL_SECTION cs; + ethr_ts_event *wakeups; + ethr_atomic32_t have_wakeups; /* only when posix compliant */ + ethr_atomic32_t locked; /* only when posix compliant */ + ethr_spinlock_t lock; /* only when posix compliant */ +#if ETHR_XCHK + int initialized; +#endif +}; + +typedef struct ethr_cond_ ethr_cond; +struct ethr_cond_ { + int posix_compliant; + CRITICAL_SECTION cs; + ethr_ts_event *waiters; + int spincount; +#if ETHR_XCHK + int initialized; +#endif +}; + +#else +# error "no mutex implementation" +#endif int ethr_mutex_init_opt(ethr_mutex *, ethr_mutex_opt *); int ethr_mutex_init(ethr_mutex *); @@ -505,7 +569,7 @@ void ethr_mutex_lock_wait__(ethr_mutex *, ethr_sint32_t); void ethr_mutex_unlock_wake__(ethr_mutex *, ethr_sint32_t); static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_mutex_trylock)(ethr_mutex *mtx) +ETHR_INLINE_MTX_FUNC_NAME_(ethr_mutex_trylock)(ethr_mutex *mtx) { ethr_sint32_t act; int res; @@ -529,7 +593,7 @@ ETHR_INLINE_FUNC_NAME_(ethr_mutex_trylock)(ethr_mutex *mtx) } static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_mutex_lock)(ethr_mutex *mtx) +ETHR_INLINE_MTX_FUNC_NAME_(ethr_mutex_lock)(ethr_mutex *mtx) { ethr_sint32_t act; ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx); @@ -549,7 +613,7 @@ ETHR_INLINE_FUNC_NAME_(ethr_mutex_lock)(ethr_mutex *mtx) } static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_mutex_unlock)(ethr_mutex *mtx) +ETHR_INLINE_MTX_FUNC_NAME_(ethr_mutex_unlock)(ethr_mutex *mtx) { ethr_sint32_t act; ETHR_COMPILER_BARRIER; @@ -569,12 +633,12 @@ ETHR_INLINE_FUNC_NAME_(ethr_mutex_unlock)(ethr_mutex *mtx) #endif /* ETHR_TRY_INLINE_FUNCS */ -#else /* pthread_mutex */ +#elif defined(ETHR_PTHREADS) && !defined(ETHR_DBG_WIN_MTX_WITH_PTHREADS) #if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_MUTEX_IMPL__) static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_mutex_trylock)(ethr_mutex *mtx) +ETHR_INLINE_MTX_FUNC_NAME_(ethr_mutex_trylock)(ethr_mutex *mtx) { int res; res = pthread_mutex_trylock(&mtx->pt_mtx); @@ -584,7 +648,7 @@ ETHR_INLINE_FUNC_NAME_(ethr_mutex_trylock)(ethr_mutex *mtx) } static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_mutex_lock)(ethr_mutex *mtx) +ETHR_INLINE_MTX_FUNC_NAME_(ethr_mutex_lock)(ethr_mutex *mtx) { int res = pthread_mutex_lock(&mtx->pt_mtx); if (res != 0) @@ -592,7 +656,7 @@ ETHR_INLINE_FUNC_NAME_(ethr_mutex_lock)(ethr_mutex *mtx) } static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_mutex_unlock)(ethr_mutex *mtx) +ETHR_INLINE_MTX_FUNC_NAME_(ethr_mutex_unlock)(ethr_mutex *mtx) { int res = pthread_mutex_unlock(&mtx->pt_mtx); if (res != 0) @@ -601,7 +665,54 @@ ETHR_INLINE_FUNC_NAME_(ethr_mutex_unlock)(ethr_mutex *mtx) #endif /* ETHR_TRY_INLINE_FUNCS */ -#endif /* pthread_mutex */ +#elif defined(ETHR_WIN32_THREADS) || defined(ETHR_DBG_WIN_MTX_WITH_PTHREADS) + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_MUTEX_IMPL__) + +static ETHR_INLINE int +ETHR_INLINE_MTX_FUNC_NAME_(ethr_mutex_trylock)(ethr_mutex *mtx) +{ + if (!TryEnterCriticalSection(&mtx->cs)) + return EBUSY; + if (mtx->posix_compliant) + ethr_atomic32_set(&mtx->locked, 1); + return 0; +} + +static ETHR_INLINE void +ETHR_INLINE_MTX_FUNC_NAME_(ethr_mutex_lock)(ethr_mutex *mtx) +{ + EnterCriticalSection(&mtx->cs); + if (mtx->posix_compliant) + ethr_atomic32_set(&mtx->locked, 1); +} + +void ethr_mutex_cond_wakeup__(ethr_mutex *mtx); + +static ETHR_INLINE void +ETHR_INLINE_MTX_FUNC_NAME_(ethr_mutex_unlock)(ethr_mutex *mtx) +{ + if (mtx->posix_compliant) { + ethr_atomic32_set_mb(&mtx->locked, 0); + if (ethr_atomic32_read_acqb(&mtx->have_wakeups)) + goto cond_wakeup; + else + goto leave_cs; + } + + if (mtx->wakeups) { + cond_wakeup: + ethr_mutex_cond_wakeup__(mtx); + } + else { + leave_cs: + LeaveCriticalSection(&mtx->cs); + } +} + +#endif /* ETHR_TRY_INLINE_FUNCS */ + +#endif #ifdef ETHR_USE_OWN_RWMTX_IMPL__ @@ -615,7 +726,7 @@ ETHR_INLINE_FUNC_NAME_(ethr_mutex_unlock)(ethr_mutex *mtx) #if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_MUTEX_IMPL__) static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_tryrlock)(ethr_rwmutex *rwmtx) +ETHR_INLINE_MTX_FUNC_NAME_(ethr_rwmutex_tryrlock)(ethr_rwmutex *rwmtx) { int res = pthread_rwlock_tryrdlock(&rwmtx->pt_rwlock); if (res != 0 && res != EBUSY) @@ -624,7 +735,7 @@ ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_tryrlock)(ethr_rwmutex *rwmtx) } static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_rlock)(ethr_rwmutex *rwmtx) +ETHR_INLINE_MTX_FUNC_NAME_(ethr_rwmutex_rlock)(ethr_rwmutex *rwmtx) { int res = pthread_rwlock_rdlock(&rwmtx->pt_rwlock); if (res != 0) @@ -632,7 +743,7 @@ ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_rlock)(ethr_rwmutex *rwmtx) } static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_runlock)(ethr_rwmutex *rwmtx) +ETHR_INLINE_MTX_FUNC_NAME_(ethr_rwmutex_runlock)(ethr_rwmutex *rwmtx) { int res = pthread_rwlock_unlock(&rwmtx->pt_rwlock); if (res != 0) @@ -640,7 +751,7 @@ ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_runlock)(ethr_rwmutex *rwmtx) } static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_tryrwlock)(ethr_rwmutex *rwmtx) +ETHR_INLINE_MTX_FUNC_NAME_(ethr_rwmutex_tryrwlock)(ethr_rwmutex *rwmtx) { int res = pthread_rwlock_trywrlock(&rwmtx->pt_rwlock); if (res != 0 && res != EBUSY) @@ -649,7 +760,7 @@ ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_tryrwlock)(ethr_rwmutex *rwmtx) } static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_rwlock)(ethr_rwmutex *rwmtx) +ETHR_INLINE_MTX_FUNC_NAME_(ethr_rwmutex_rwlock)(ethr_rwmutex *rwmtx) { int res = pthread_rwlock_wrlock(&rwmtx->pt_rwlock); if (res != 0) @@ -657,7 +768,7 @@ ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_rwlock)(ethr_rwmutex *rwmtx) } static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_rwunlock)(ethr_rwmutex *rwmtx) +ETHR_INLINE_MTX_FUNC_NAME_(ethr_rwmutex_rwunlock)(ethr_rwmutex *rwmtx) { int res = pthread_rwlock_unlock(&rwmtx->pt_rwlock); if (res != 0) diff --git a/erts/include/internal/ethr_optimized_fallbacks.h b/erts/include/internal/ethr_optimized_fallbacks.h index 8e04692856..45399d18e6 100644 --- a/erts/include/internal/ethr_optimized_fallbacks.h +++ b/erts/include/internal/ethr_optimized_fallbacks.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2010. All Rights Reserved. + * Copyright Ericsson AB 2010-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -18,152 +18,172 @@ */ /* - * Description: "Optimized" fallbacks used when native ops are missing + * Description: Optimized fallbacks used when native ops are missing * Author: Rickard Green */ #ifndef ETHR_OPTIMIZED_FALLBACKS_H__ #define ETHR_OPTIMIZED_FALLBACKS_H__ -#ifdef ETHR_HAVE_NATIVE_ATOMICS -#define ETHR_HAVE_OPTIMIZED_ATOMIC_OPS 1 +#if defined(ETHR_HAVE_NATIVE_SPINLOCKS) + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) + +static ETHR_INLINE int +ethr_native_spinlock_destroy(ethr_native_spinlock_t *lock) +{ + return 0; +} + #endif -#ifdef ETHR_HAVE_NATIVE_SPINLOCKS -#define ETHR_HAVE_OPTIMIZED_SPINLOCKS 1 #elif defined(ETHR_HAVE_PTHREAD_SPIN_LOCK) -/* --- Optimized spinlocks using pthread spinlocks -------------------------- */ -#define ETHR_HAVE_OPTIMIZED_SPINLOCKS 1 +/* --- Native spinlocks using pthread spinlocks -------------------------- */ +#define ETHR_HAVE_NATIVE_SPINLOCKS 1 + +#define ETHR_NATIVE_SPINLOCK_IMPL "pthread" -typedef pthread_spinlock_t ethr_opt_spinlock_t; +typedef pthread_spinlock_t ethr_native_spinlock_t; #if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) -static ETHR_INLINE int -ethr_opt_spinlock_init(ethr_opt_spinlock_t *lock) +static ETHR_INLINE void +ethr_native_spinlock_init(ethr_native_spinlock_t *lock) { - return pthread_spin_init((pthread_spinlock_t *) lock, 0); + int err = pthread_spin_init((pthread_spinlock_t *) lock, 0); + if (err) + ETHR_FATAL_ERROR__(err); } static ETHR_INLINE int -ethr_opt_spinlock_destroy(ethr_opt_spinlock_t *lock) +ethr_native_spinlock_destroy(ethr_native_spinlock_t *lock) { return pthread_spin_destroy((pthread_spinlock_t *) lock); } - -static ETHR_INLINE int -ethr_opt_spin_unlock(ethr_opt_spinlock_t *lock) +static ETHR_INLINE void +ethr_native_spin_unlock(ethr_native_spinlock_t *lock) { - return pthread_spin_unlock((pthread_spinlock_t *) lock); + int err = pthread_spin_unlock((pthread_spinlock_t *) lock); + if (err) + ETHR_FATAL_ERROR__(err); } -static ETHR_INLINE int -ethr_opt_spin_lock(ethr_opt_spinlock_t *lock) +static ETHR_INLINE void +ethr_native_spin_lock(ethr_native_spinlock_t *lock) { - return pthread_spin_lock((pthread_spinlock_t *) lock); + int err = pthread_spin_lock((pthread_spinlock_t *) lock); + if (err) + ETHR_FATAL_ERROR__(err); } #endif -#elif defined(ETHR_HAVE_NATIVE_ATOMICS) +#elif defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) /* --- Native spinlocks using native atomics -------------------------------- */ #define ETHR_HAVE_NATIVE_SPINLOCKS 1 -#define ETHR_HAVE_OPTIMIZED_SPINLOCKS 1 - -#if defined(ETHR_HAVE_NATIVE_ATOMIC32) -typedef ethr_native_atomic32_t ethr_native_spinlock_t; -# define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X -#elif defined(ETHR_HAVE_NATIVE_ATOMIC64) -typedef ethr_native_atomic64_t ethr_native_spinlock_t; -# define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X -#else -# error "Missing native atomic implementation" -#endif + +#define ETHR_NATIVE_SPINLOCK_IMPL "native-atomics" + +typedef ethr_atomic32_t ethr_native_spinlock_t; #if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) +#undef ETHR_NSPN_AOP__ +#define ETHR_NSPN_AOP__(X) ETHR_INLINE_ATMC32_FUNC_NAME_(ethr_atomic32_ ## X) + static ETHR_INLINE void ethr_native_spinlock_init(ethr_native_spinlock_t *lock) { - ETHR_NATMC_FUNC__(init)(lock, 0); + ETHR_NSPN_AOP__(init)(lock, 0); +} + +static ETHR_INLINE int +ethr_native_spinlock_destroy(ethr_native_spinlock_t *lock) +{ + return ETHR_NSPN_AOP__(read)(lock) == 0 ? 0 : EBUSY; } static ETHR_INLINE void ethr_native_spin_unlock(ethr_native_spinlock_t *lock) { - ETHR_COMPILER_BARRIER; - ETHR_ASSERT(ETHR_NATMC_FUNC__(read)(lock) == 1); - ETHR_NATMC_FUNC__(set_relb)(lock, 0); + ETHR_ASSERT(ETHR_NSPN_AOP__(read)(lock) == 1); + ETHR_NSPN_AOP__(set_relb)(lock, 0); } static ETHR_INLINE void ethr_native_spin_lock(ethr_native_spinlock_t *lock) { - while (ETHR_NATMC_FUNC__(cmpxchg_acqb)(lock, 1, 0) != 0) { - while (ETHR_NATMC_FUNC__(read)(lock) != 0) + while (ETHR_NSPN_AOP__(cmpxchg_acqb)(lock, 1, 0) != 0) { + while (ETHR_NSPN_AOP__(read)(lock) != 0) ETHR_SPIN_BODY; } ETHR_COMPILER_BARRIER; } -#endif +#undef ETHR_NSPN_AOP__ -#undef ETHR_NATMC_FUNC__ +#endif #endif -#ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS -#define ETHR_HAVE_OPTIMIZED_RWSPINLOCKS 1 -#elif defined(ETHR_HAVE_NATIVE_ATOMICS) +#if defined(ETHR_HAVE_NATIVE_RWSPINLOCKS) + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) + +static ETHR_INLINE int +ethr_native_rwlock_destroy(ethr_native_rwlock_t *lock) +{ + return 0; +} + +#endif + +#elif defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) /* --- Native rwspinlocks using native atomics ------------------------------ */ #define ETHR_HAVE_NATIVE_RWSPINLOCKS 1 -#define ETHR_HAVE_OPTIMIZED_RWSPINLOCKS 1 +#define ETHR_NATIVE_RWSPINLOCK_IMPL "native-atomics" -#if defined(ETHR_HAVE_NATIVE_ATOMIC32) -typedef ethr_native_atomic32_t ethr_native_rwlock_t; -# define ETHR_NAINT_T__ ethr_sint32_t +typedef ethr_atomic32_t ethr_native_rwlock_t; # define ETHR_WLOCK_FLAG__ (((ethr_sint32_t) 1) << 30) -# define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X -#elif defined(ETHR_HAVE_NATIVE_ATOMIC64) -typedef ethr_native_atomic64_t ethr_native_rwlock_t; -# define ETHR_NAINT_T__ ethr_sint64_t -# define ETHR_WLOCK_FLAG__ (((ethr_sint64_t) 1) << 62) -# define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X -#else -# error "Missing native atomic implementation" -#endif #if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) +#undef ETHR_NRWSPN_AOP__ +#define ETHR_NRWSPN_AOP__(X) ETHR_INLINE_ATMC32_FUNC_NAME_(ethr_atomic32_ ## X) + static ETHR_INLINE void ethr_native_rwlock_init(ethr_native_rwlock_t *lock) { - ETHR_NATMC_FUNC__(init)(lock, 0); + ETHR_NRWSPN_AOP__(init)(lock, 0); +} + +static ETHR_INLINE int +ethr_native_rwlock_destroy(ethr_native_rwlock_t *lock) +{ + return ETHR_NRWSPN_AOP__(read)(lock) == 0 ? 0 : EBUSY; } static ETHR_INLINE void ethr_native_read_unlock(ethr_native_rwlock_t *lock) { - ETHR_COMPILER_BARRIER; -#ifdef DEBUG - ETHR_ASSERT(ETHR_NATMC_FUNC__(read)(lock) >= 0); -#endif - ETHR_NATMC_FUNC__(dec_relb)(lock); + ETHR_ASSERT(ETHR_NRWSPN_AOP__(read)(lock) >= 0); + ETHR_NRWSPN_AOP__(dec_relb)(lock); } static ETHR_INLINE void ethr_native_read_lock(ethr_native_rwlock_t *lock) { - ETHR_NAINT_T__ act, exp = 0; + ethr_sint32_t act, exp = 0; while (1) { - act = ETHR_NATMC_FUNC__(cmpxchg_acqb)(lock, exp+1, exp); + act = ETHR_NRWSPN_AOP__(cmpxchg_acqb)(lock, exp+1, exp); if (act == exp) break; + /* Wait for writer to leave */ while (act & ETHR_WLOCK_FLAG__) { ETHR_SPIN_BODY; - act = ETHR_NATMC_FUNC__(read)(lock); + act = ETHR_NRWSPN_AOP__(read)(lock); } exp = act; } @@ -173,36 +193,37 @@ ethr_native_read_lock(ethr_native_rwlock_t *lock) static ETHR_INLINE void ethr_native_write_unlock(ethr_native_rwlock_t *lock) { - ETHR_COMPILER_BARRIER; - ETHR_ASSERT(ETHR_NATMC_FUNC__(read)(lock) == ETHR_WLOCK_FLAG__); - ETHR_NATMC_FUNC__(set_relb)(lock, 0); + ETHR_ASSERT(ETHR_NRWSPN_AOP__(read)(lock) == ETHR_WLOCK_FLAG__); + ETHR_NRWSPN_AOP__(set_relb)(lock, 0); } static ETHR_INLINE void ethr_native_write_lock(ethr_native_rwlock_t *lock) { - ETHR_NAINT_T__ act, exp = 0; + ethr_sint32_t act, exp = 0; while (1) { - act = ETHR_NATMC_FUNC__(cmpxchg_acqb)(lock, exp|ETHR_WLOCK_FLAG__, exp); + act = ETHR_NRWSPN_AOP__(cmpxchg_acqb)(lock, exp|ETHR_WLOCK_FLAG__, exp); if (act == exp) break; - ETHR_SPIN_BODY; - exp = act & ~ETHR_WLOCK_FLAG__; + /* Wait for writer to leave */ + while (act & ETHR_WLOCK_FLAG__) { + ETHR_SPIN_BODY; + act = ETHR_NRWSPN_AOP__(read)(lock); + } + exp = act; } act |= ETHR_WLOCK_FLAG__; /* Wait for readers to leave */ while (act != ETHR_WLOCK_FLAG__) { ETHR_SPIN_BODY; - act = ETHR_NATMC_FUNC__(read_acqb)(lock); + act = ETHR_NRWSPN_AOP__(read_acqb)(lock); } ETHR_COMPILER_BARRIER; } -#endif +#undef ETHR_NRWSPN_AOP__ -#undef ETHR_NAINT_T__ -#undef ETHR_NATMC_FUNC__ -#undef ETHR_WLOCK_FLAG__ +#endif #endif diff --git a/erts/include/internal/ethread.h b/erts/include/internal/ethread.h index 4cd95faf6a..142c26c0ca 100644 --- a/erts/include/internal/ethread.h +++ b/erts/include/internal/ethread.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2004-2010. All Rights Reserved. + * Copyright Ericsson AB 2004-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -33,10 +33,6 @@ #include <stdlib.h> #include "erl_errno.h" -#undef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS -#undef ETHR_HAVE_OPTIMIZED_SPINLOCK -#undef ETHR_HAVE_OPTIMIZED_RWSPINLOCK - #if defined(DEBUG) # define ETHR_DEBUG #endif @@ -68,7 +64,7 @@ #endif /* Assume 64-byte cache line size */ -#define ETHR_CACHE_LINE_SIZE ((ethr_uint_t) 64) +#define ETHR_CACHE_LINE_SIZE 64 #define ETHR_CACHE_LINE_MASK (ETHR_CACHE_LINE_SIZE - 1) #define ETHR_CACHE_LINE_ALIGN_SIZE(SZ) \ @@ -195,7 +191,6 @@ typedef DWORD ethr_tsd_key; #undef ETHR_HAVE_ETHR_SIG_FUNCS #define ETHR_USE_OWN_RWMTX_IMPL__ -#define ETHR_USE_OWN_MTX_IMPL__ #define ETHR_YIELD() (Sleep(0), 0) @@ -251,13 +246,90 @@ typedef ethr_sint64_t ethr_sint_t; typedef ethr_uint64_t ethr_uint_t; #endif -/* __builtin_expect() is needed by both native atomics code - * and the fallback code */ -#if !defined(__GNUC__) || (__GNUC__ < 2) || (__GNUC__ == 2 && __GNUC_MINOR__ < 96) +#if defined(ETHR_SIZEOF___INT128_T) && ETHR_SIZEOF___INT128_T == 16 +#define ETHR_HAVE_INT128_T +typedef __int128_t ethr_sint128_t; +typedef __uint128_t ethr_uint128_t; +#endif + +#define ETHR_FATAL_ERROR__(ERR) \ + ethr_fatal_error__(__FILE__, __LINE__, __func__, (ERR)) + +ETHR_PROTO_NORETURN__ ethr_fatal_error__(const char *file, + int line, + const char *func, + int err); + +#if !defined(__GNUC__) +# define ETHR_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) 0 +#elif !defined(__GNUC_MINOR__) +# define ETHR_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \ + ((__GNUC__ << 24) >= (((MAJ) << 24) | ((MIN) << 12) | (PL))) +#elif !defined(__GNUC_PATCHLEVEL__) +# define ETHR_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \ + (((__GNUC__ << 24) | (__GNUC_MINOR__ << 12)) >= (((MAJ) << 24) | ((MIN) << 12) | (PL))) +#else +# define ETHR_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \ + (((__GNUC__ << 24) | (__GNUC_MINOR__ << 12) | __GNUC_PATCHLEVEL__) >= (((MAJ) << 24) | ((MIN) << 12) | (PL))) +#endif + +#if !ETHR_AT_LEAST_GCC_VSN__(2, 96, 0) #define __builtin_expect(X, Y) (X) #endif -/* For CPU-optimised atomics, spinlocks, and rwlocks. */ +#if ETHR_AT_LEAST_GCC_VSN__(3, 1, 1) +# define ETHR_CHOOSE_EXPR __builtin_choose_expr +#else +# define ETHR_CHOOSE_EXPR(B, E1, E2) ((B) ? (E1) : (E2)) +#endif + +#if ((defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__))) \ + || (defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_AMD64)))) +# define ETHR_X86_RUNTIME_CONF__ + +# define ETHR_X86_RUNTIME_CONF_HAVE_DW_CMPXCHG__ \ + (__builtin_expect(ethr_runtime__.conf.have_dw_cmpxchg != 0, 1)) +# define ETHR_X86_RUNTIME_CONF_HAVE_NO_DW_CMPXCHG__ \ + (__builtin_expect(ethr_runtime__.conf.have_dw_cmpxchg == 0, 0)) +# define ETHR_X86_RUNTIME_CONF_HAVE_SSE2__ \ + (__builtin_expect(ethr_runtime__.conf.have_sse2 != 0, 1)) +# define ETHR_X86_RUNTIME_CONF_HAVE_NO_SSE2__ \ + (__builtin_expect(ethr_runtime__.conf.have_sse2 == 0, 0)) +#endif + +#if (defined(__GNUC__) \ + && !defined(ETHR_PPC_HAVE_LWSYNC) \ + && !defined(ETHR_PPC_HAVE_NO_LWSYNC) \ + && (defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__))) +# define ETHR_PPC_RUNTIME_CONF__ + +# define ETHR_PPC_RUNTIME_CONF_HAVE_LWSYNC__ \ + (__builtin_expect(ethr_runtime__.conf.have_lwsync != 0, 1)) +# define ETHR_PPC_RUNTIME_CONF_HAVE_NO_LWSYNC__ \ + (__builtin_expect(ethr_runtime__.conf.have_lwsync == 0, 0)) +#endif + +typedef struct { +#if defined(ETHR_X86_RUNTIME_CONF__) + int have_dw_cmpxchg; + int have_sse2; +#endif +#if defined(ETHR_PPC_RUNTIME_CONF__) + int have_lwsync; +#endif + int dummy; +} ethr_runtime_conf_t; + + +typedef union { + ethr_runtime_conf_t conf; + char pad__[ETHR_CACHE_LINE_ALIGN_SIZE(sizeof(ethr_runtime_conf_t))+ETHR_CACHE_LINE_SIZE]; +} ethr_runtime_t; + + +extern ethr_runtime_t ethr_runtime__; + +/* For native CPU-optimised atomics, spinlocks, and rwlocks. */ #if !defined(ETHR_DISABLE_NATIVE_IMPLS) # if defined(__GNUC__) # if defined(ETHR_PREFER_GCC_NATIVE_IMPLS) @@ -265,7 +337,7 @@ typedef ethr_uint64_t ethr_uint_t; # elif defined(ETHR_PREFER_LIBATOMIC_OPS_NATIVE_IMPLS) # include "libatomic_ops/ethread.h" # endif -# ifndef ETHR_HAVE_NATIVE_ATOMICS +# if !defined(ETHR_HAVE_NATIVE_ATOMIC32) && !defined(ETHR_HAVE_NATIVE_ATOMIC64) # if ETHR_SIZEOF_PTR == 4 # if defined(__i386__) # include "i386/ethread.h" @@ -283,8 +355,10 @@ typedef ethr_uint64_t ethr_uint_t; # include "sparc64/ethread.h" # endif # endif +#if 0 # include "gcc/ethread.h" # include "libatomic_ops/ethread.h" +#endif # endif # elif defined(ETHR_HAVE_LIBATOMIC_OPS) # include "libatomic_ops/ethread.h" @@ -293,10 +367,9 @@ typedef ethr_uint64_t ethr_uint_t; # endif #endif /* !ETHR_DISABLE_NATIVE_IMPLS */ +#include "ethr_atomics.h" /* The atomics API */ + #if defined(__GNUC__) -# ifndef ETHR_COMPILER_BARRIER -# define ETHR_COMPILER_BARRIER __asm__ __volatile__("" : : : "memory") -# endif # ifndef ETHR_SPIN_BODY # if defined(__i386__) || defined(__x86_64__) # define ETHR_SPIN_BODY __asm__ __volatile__("rep;nop" : : : "memory") @@ -309,11 +382,6 @@ typedef ethr_uint64_t ethr_uint_t; # endif # endif #elif defined(ETHR_WIN32_THREADS) -# ifndef ETHR_COMPILER_BARRIER -# include <intrin.h> -# pragma intrinsic(_ReadWriteBarrier) -# define ETHR_COMPILER_BARRIER _ReadWriteBarrier() -# endif # ifndef ETHR_SPIN_BODY # define ETHR_SPIN_BODY do {YieldProcessor();ETHR_COMPILER_BARRIER;} while(0) # endif @@ -321,43 +389,6 @@ typedef ethr_uint64_t ethr_uint_t; #define ETHR_YIELD_AFTER_BUSY_LOOPS 50 -#ifndef ETHR_HAVE_NATIVE_ATOMICS -/* - * ETHR_*MEMORY_BARRIER orders between locked and atomic accesses only, - * i.e. when our lock based atomic fallback is used, a noop is sufficient. - */ -#define ETHR_MEMORY_BARRIER do { } while (0) -#define ETHR_WRITE_MEMORY_BARRIER do { } while (0) -#define ETHR_READ_MEMORY_BARRIER do { } while (0) -#define ETHR_READ_DEPEND_MEMORY_BARRIER do { } while (0) -#endif - -#ifndef ETHR_WRITE_MEMORY_BARRIER -# define ETHR_WRITE_MEMORY_BARRIER ETHR_MEMORY_BARRIER -# define ETHR_WRITE_MEMORY_BARRIER_IS_FULL -#endif -#ifndef ETHR_READ_MEMORY_BARRIER -# define ETHR_READ_MEMORY_BARRIER ETHR_MEMORY_BARRIER -# define ETHR_READ_MEMORY_BARRIER_IS_FULL -#endif -#ifndef ETHR_READ_DEPEND_MEMORY_BARRIER -# define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_COMPILER_BARRIER -# define ETHR_READ_DEPEND_MEMORY_BARRIER_IS_COMPILER_BARRIER -#endif - -#define ETHR_FATAL_ERROR__(ERR) \ - ethr_fatal_error__(__FILE__, __LINE__, __func__, (ERR)) - -ETHR_PROTO_NORETURN__ ethr_fatal_error__(const char *file, - int line, - const char *func, - int err); - -void ethr_compiler_barrier_fallback(void); -#ifndef ETHR_COMPILER_BARRIER -# define ETHR_COMPILER_BARRIER ethr_compiler_barrier_fallback() -#endif - #ifndef ETHR_SPIN_BODY # define ETHR_SPIN_BODY ETHR_COMPILER_BARRIER #endif @@ -460,8 +491,6 @@ void ethr_compiler_barrier(void); #if defined(ETHR_HAVE_NATIVE_SPINLOCKS) typedef ethr_native_spinlock_t ethr_spinlock_t; -#elif defined(ETHR_HAVE_OPTIMIZED_SPINLOCKS) -typedef ethr_opt_spinlock_t ethr_spinlock_t; #elif defined(__WIN32__) typedef CRITICAL_SECTION ethr_spinlock_t; #else @@ -483,8 +512,6 @@ ETHR_INLINE_FUNC_NAME_(ethr_spinlock_init)(ethr_spinlock_t *lock) #ifdef ETHR_HAVE_NATIVE_SPINLOCKS ethr_native_spinlock_init(lock); return 0; -#elif defined(ETHR_HAVE_OPTIMIZED_SPINLOCKS) - return ethr_opt_spinlock_init((ethr_opt_spinlock_t *) lock); #elif defined(__WIN32__) if (!InitializeCriticalSectionAndSpinCount((CRITICAL_SECTION *) lock, INT_MAX)) return ethr_win_get_errno__(); @@ -498,9 +525,7 @@ static ETHR_INLINE int ETHR_INLINE_FUNC_NAME_(ethr_spinlock_destroy)(ethr_spinlock_t *lock) { #ifdef ETHR_HAVE_NATIVE_SPINLOCKS - return 0; -#elif defined(ETHR_HAVE_OPTIMIZED_SPINLOCKS) - return ethr_opt_spinlock_destroy((ethr_opt_spinlock_t *) lock); + return ethr_native_spinlock_destroy(lock); #elif defined(__WIN32__) DeleteCriticalSection((CRITICAL_SECTION *) lock); return 0; @@ -514,10 +539,6 @@ ETHR_INLINE_FUNC_NAME_(ethr_spin_unlock)(ethr_spinlock_t *lock) { #ifdef ETHR_HAVE_NATIVE_SPINLOCKS ethr_native_spin_unlock(lock); -#elif defined(ETHR_HAVE_OPTIMIZED_SPINLOCKS) - int err = ethr_opt_spin_unlock((ethr_opt_spinlock_t *) lock); - if (err) - ETHR_FATAL_ERROR__(err); #elif defined(__WIN32__) LeaveCriticalSection((CRITICAL_SECTION *) lock); #else @@ -532,10 +553,6 @@ ETHR_INLINE_FUNC_NAME_(ethr_spin_lock)(ethr_spinlock_t *lock) { #ifdef ETHR_HAVE_NATIVE_SPINLOCKS ethr_native_spin_lock(lock); -#elif defined(ETHR_HAVE_OPTIMIZED_SPINLOCKS) - int err = ethr_opt_spin_lock((ethr_opt_spinlock_t *) lock); - if (err) - ETHR_FATAL_ERROR__(err); #elif defined(__WIN32__) EnterCriticalSection((CRITICAL_SECTION *) lock); #else @@ -547,8 +564,6 @@ ETHR_INLINE_FUNC_NAME_(ethr_spin_lock)(ethr_spinlock_t *lock) #endif /* ETHR_TRY_INLINE_FUNCS */ -#include "ethr_atomics.h" - typedef struct ethr_ts_event_ ethr_ts_event; /* Needed by ethr_mutex.h */ #if defined(ETHR_WIN32_THREADS) diff --git a/erts/include/internal/ethread_header_config.h.in b/erts/include/internal/ethread_header_config.h.in index 1f82db8693..dd3599f86d 100644 --- a/erts/include/internal/ethread_header_config.h.in +++ b/erts/include/internal/ethread_header_config.h.in @@ -32,6 +32,9 @@ /* Define to the size of __int64 */ #undef ETHR_SIZEOF___INT64 +/* Define to the size of __int128_t */ +#undef ETHR_SIZEOF___INT128_T + /* Define if bigendian */ #undef ETHR_BIGENDIAN @@ -69,8 +72,44 @@ /* Define if you have a linux futex implementation. */ #undef ETHR_HAVE_LINUX_FUTEX -/* Define if you have gcc atomic operations */ -#undef ETHR_HAVE_GCC_ATOMIC_OPS +/* Define if x86/x86_64 out of order instructions should be synchronized */ +#undef ETHR_X86_OUT_OF_ORDER + +/* Define if only run in Sparc TSO mode */ +#undef ETHR_SPARC_TSO + +/* Define if only run in Sparc PSO, or TSO mode */ +#undef ETHR_SPARC_PSO + +/* Define if run in Sparc RMO, PSO, or TSO mode */ +#undef ETHR_SPARC_RMO + +/* Define if you have __sync_add_and_fetch() for 32-bit integers */ +#undef ETHR_HAVE___SYNC_ADD_AND_FETCH32 + +/* Define if you have __sync_add_and_fetch() for 64-bit integers */ +#undef ETHR_HAVE___SYNC_ADD_AND_FETCH64 + +/* Define if you have __sync_fetch_and_and() for 32-bit integers */ +#undef ETHR_HAVE___SYNC_FETCH_AND_AND32 + +/* Define if you have __sync_fetch_and_and() for 64-bit integers */ +#undef ETHR_HAVE___SYNC_FETCH_AND_AND64 + +/* Define if you have __sync_fetch_and_or() for 32-bit integers */ +#undef ETHR_HAVE___SYNC_FETCH_AND_OR32 + +/* Define if you have __sync_fetch_and_or() for 64-bit integers */ +#undef ETHR_HAVE___SYNC_FETCH_AND_OR64 + +/* Define if you have __sync_val_compare_and_swap() for 32-bit integers */ +#undef ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP32 + +/* Define if you have __sync_val_compare_and_swap() for 64-bit integers */ +#undef ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP64 + +/* Define if you have __sync_val_compare_and_swap() for 128-bit integers */ +#undef ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP128 /* Define if you prefer gcc native ethread implementations */ #undef ETHR_PREFER_GCC_NATIVE_IMPLS @@ -90,8 +129,14 @@ /* Define if sched_yield() returns an int. */ #undef ETHR_SCHED_YIELD_RET_INT -/* Define if you want compatibility with x86 processors before pentium4. */ -#undef ETHR_PRE_PENTIUM4_COMPAT +/* Define if you use a gcc that supports -msse2 and understand sse2 specific asm statements */ +#undef ETHR_GCC_HAVE_SSE2_ASM_SUPPORT + +/* Define if you use a gcc that supports the double word cmpxchg instruction */ +#undef ETHR_GCC_HAVE_DW_CMPXCHG_ASM_SUPPORT + +/* Define if you get a register shortage with cmpxchg8b and position independent code */ +#undef ETHR_CMPXCHG8B_REGISTER_SHORTAGE /* Define if you have the pthread_rwlockattr_setkind_np() function. */ #undef ETHR_HAVE_PTHREAD_RWLOCKATTR_SETKIND_NP @@ -115,23 +160,74 @@ /* Define to the size of AO_t if libatomic_ops is used */ #undef ETHR_SIZEOF_AO_T +/* Define if you have _InterlockedAnd() */ +#undef ETHR_HAVE__INTERLOCKEDAND + +/* Define if you have _InterlockedAnd64() */ +#undef ETHR_HAVE__INTERLOCKEDAND64 + +/* Define if you have _InterlockedCompareExchange() */ +#undef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE + /* Define if you have _InterlockedCompareExchange64() */ #undef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64 +/* Define if you have _InterlockedCompareExchange64_acq() */ +#undef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_ACQ + +/* Define if you have _InterlockedCompareExchange64_rel() */ +#undef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_REL + +/* Define if you have _InterlockedCompareExchange_acq() */ +#undef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE_ACQ + +/* Define if you have _InterlockedCompareExchange_rel() */ +#undef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE_REL + +/* Define if you have _InterlockedDecrement() */ +#undef ETHR_HAVE__INTERLOCKEDDECREMENT + /* Define if you have _InterlockedDecrement64() */ #undef ETHR_HAVE__INTERLOCKEDDECREMENT64 -/* Define if you have _InterlockedIncrement64() */ -#undef ETHR_HAVE__INTERLOCKEDINCREMENT64 +/* Define if you have _InterlockedDecrement64_rel() */ +#undef ETHR_HAVE__INTERLOCKEDDECREMENT64_REL -/* Define if you have _InterlockedExchangeAdd64() */ -#undef ETHR_HAVE__INTERLOCKEDEXCHANGEADD64 +/* Define if you have _InterlockedDecrement_rel() */ +#undef ETHR_HAVE__INTERLOCKEDDECREMENT_REL + +/* Define if you have _InterlockedExchange() */ +#undef ETHR_HAVE__INTERLOCKEDEXCHANGE /* Define if you have _InterlockedExchange64() */ #undef ETHR_HAVE__INTERLOCKEDEXCHANGE64 -/* Define if you have _InterlockedAnd64() */ -#undef ETHR_HAVE__INTERLOCKEDAND64 +/* Define if you have _InterlockedExchangeAdd() */ +#undef ETHR_HAVE__INTERLOCKEDEXCHANGEADD + +/* Define if you have _InterlockedExchangeAdd64() */ +#undef ETHR_HAVE__INTERLOCKEDEXCHANGEADD64 + +/* Define if you have _InterlockedExchangeAdd64_acq() */ +#undef ETHR_HAVE__INTERLOCKEDEXCHANGEADD64_ACQ + +/* Define if you have _InterlockedExchangeAdd_acq() */ +#undef ETHR_HAVE__INTERLOCKEDEXCHANGEADD_ACQ + +/* Define if you have _InterlockedIncrement() */ +#undef ETHR_HAVE__INTERLOCKEDINCREMENT + +/* Define if you have _InterlockedIncrement64() */ +#undef ETHR_HAVE__INTERLOCKEDINCREMENT64 + +/* Define if you have _InterlockedIncrement64_acq() */ +#undef ETHR_HAVE__INTERLOCKEDINCREMENT64_ACQ + +/* Define if you have _InterlockedIncrement_acq() */ +#undef ETHR_HAVE__INTERLOCKEDINCREMENT_ACQ + +/* Define if you have _InterlockedOr() */ +#undef ETHR_HAVE__INTERLOCKEDOR /* Define if you have _InterlockedOr64() */ #undef ETHR_HAVE__INTERLOCKEDOR64 diff --git a/erts/include/internal/gcc/ethr_atomic.h b/erts/include/internal/gcc/ethr_atomic.h index 16935084b1..f598f8537b 100644 --- a/erts/include/internal/gcc/ethr_atomic.h +++ b/erts/include/internal/gcc/ethr_atomic.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2010. All Rights Reserved. + * Copyright Ericsson AB 2010-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -25,11 +25,15 @@ #undef ETHR_INCLUDE_ATOMIC_IMPL__ #if !defined(ETHR_GCC_ATOMIC32_H__) && defined(ETHR_ATOMIC_WANT_32BIT_IMPL__) #define ETHR_GCC_ATOMIC32_H__ -#define ETHR_INCLUDE_ATOMIC_IMPL__ 4 +#if defined(ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP32) +# define ETHR_INCLUDE_ATOMIC_IMPL__ 4 +#endif #undef ETHR_ATOMIC_WANT_32BIT_IMPL__ #elif !defined(ETHR_GCC_ATOMIC64_H__) && defined(ETHR_ATOMIC_WANT_64BIT_IMPL__) #define ETHR_GCC_ATOMIC64_H__ -#define ETHR_INCLUDE_ATOMIC_IMPL__ 8 +#if defined(ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP64) +# define ETHR_INCLUDE_ATOMIC_IMPL__ 8 +#endif #undef ETHR_ATOMIC_WANT_64BIT_IMPL__ #endif @@ -45,58 +49,38 @@ # define ETHR_READ_AND_SET_WITHOUT_SYNC_OP__ 1 #endif -#if defined(__x86_64__) || (defined(__i386__) \ - && !defined(ETHR_PRE_PENTIUM4_COMPAT)) -# define ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ 1 -#else -# define ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ 0 -#endif - -/* - * According to the documentation this is what we want: - * #define ETHR_MEMORY_BARRIER __sync_synchronize() - * However, __sync_synchronize() is known to erroneously be - * a noop on at least some platforms with some gcc versions. - * This has suposedly been fixed in some gcc version, but we - * don't know from which version. Therefore, we only use - * it when it has been verified to work. Otherwise - * we use a workaround. - */ -#if defined(__mips__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 2)) -/* __sync_synchronize() has been verified to work here */ -#define ETHR_MEMORY_BARRIER __sync_synchronize() -#define ETHR_READ_DEPEND_MEMORY_BARRIER __sync_synchronize() -#elif defined(__x86_64__) || (defined(__i386__) \ - && !defined(ETHR_PRE_PENTIUM4_COMPAT)) -/* Use fence instructions directly instead of workaround */ -#define ETHR_MEMORY_BARRIER __asm__ __volatile__("mfence" : : : "memory") -#define ETHR_WRITE_MEMORY_BARRIER __asm__ __volatile__("sfence" : : : "memory") -#define ETHR_READ_MEMORY_BARRIER __asm__ __volatile__("lfence" : : : "memory") -#define ETHR_READ_DEPEND_MEMORY_BARRIER __asm__ __volatile__("" : : : "memory") -#else -/* Workaround */ -#define ETHR_MEMORY_BARRIER \ -do { \ - volatile ethr_sint32_t x___ = 0; \ - (void) __sync_val_compare_and_swap(&x___, (ethr_sint32_t) 0, (ethr_sint32_t) 1); \ -} while (0) -#define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_MEMORY_BARRIER -#endif - -#define ETHR_COMPILER_BARRIER __asm__ __volatile__("" : : : "memory") - #endif /* ETHR_GCC_ATOMIC_COMMON__ */ #if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 #define ETHR_HAVE_NATIVE_ATOMIC32 1 +#define ETHR_NATIVE_ATOMIC32_IMPL "gcc" #define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X #define ETHR_ATMC_T__ ethr_native_atomic32_t #define ETHR_AINT_T__ ethr_sint32_t +#if defined(ETHR_HAVE___SYNC_ADD_AND_FETCH32) +# define ETHR_HAVE___SYNC_ADD_AND_FETCH +#endif +#if defined(ETHR_HAVE___SYNC_FETCH_AND_AND32) +# define ETHR_HAVE___SYNC_FETCH_AND_AND +#endif +#if defined(ETHR_HAVE___SYNC_FETCH_AND_OR32) +# define ETHR_HAVE___SYNC_FETCH_AND_OR +#endif #elif ETHR_INCLUDE_ATOMIC_IMPL__ == 8 #define ETHR_HAVE_NATIVE_ATOMIC64 1 +#define ETHR_NATIVE_ATOMIC64_IMPL "gcc" #define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X #define ETHR_ATMC_T__ ethr_native_atomic64_t #define ETHR_AINT_T__ ethr_sint64_t +#if defined(ETHR_HAVE___SYNC_ADD_AND_FETCH64) +# define ETHR_HAVE___SYNC_ADD_AND_FETCH +#endif +#if defined(ETHR_HAVE___SYNC_FETCH_AND_AND64) +# define ETHR_HAVE___SYNC_FETCH_AND_AND +#endif +#if defined(ETHR_HAVE___SYNC_FETCH_AND_OR64) +# define ETHR_HAVE___SYNC_FETCH_AND_OR +#endif #else #error "Unsupported integer size" #endif @@ -108,183 +92,120 @@ typedef struct { #if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADDR 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADDR 1 +#endif + static ETHR_INLINE ETHR_AINT_T__ * ETHR_NATMC_FUNC__(addr)(ETHR_ATMC_T__ *var) { return (ETHR_AINT_T__ *) &var->counter; } -static ETHR_INLINE void -ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value) -{ #if ETHR_READ_AND_SET_WITHOUT_SYNC_OP__ - var->counter = value; + +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET 1 #else - /* - * Unfortunately no __sync_store() or similar exist in the gcc atomic - * op interface. We therefore have to simulate it this way... - */ - ETHR_AINT_T__ act = 0, exp; - do { - exp = act; - act = __sync_val_compare_and_swap(&var->counter, exp, value); - } while (act != exp); +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET 1 #endif -} static ETHR_INLINE void -ETHR_NATMC_FUNC__(init)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value) +ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value) { - ETHR_NATMC_FUNC__(set)(var, value); + var->counter = value; } -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var) -{ +#endif /* ETHR_READ_AND_SET_WITHOUT_SYNC_OP__ */ + #if ETHR_READ_AND_SET_WITHOUT_SYNC_OP__ - return var->counter; + +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ 1 #else - /* - * Unfortunately no __sync_fetch() or similar exist in the gcc atomic - * op interface. We therefore have to simulate it this way... - */ - return __sync_add_and_fetch(&var->counter, (ETHR_AINT_T__) 0); +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ 1 #endif -} - -static ETHR_INLINE void -ETHR_NATMC_FUNC__(add)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) -{ - (void) __sync_add_and_fetch(&var->counter, incr); -} static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(add_return)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) +ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var) { - return __sync_add_and_fetch(&var->counter, incr); + return var->counter; } -static ETHR_INLINE void -ETHR_NATMC_FUNC__(inc)(ETHR_ATMC_T__ *var) -{ - (void) __sync_add_and_fetch(&var->counter, (ETHR_AINT_T__) 1); -} +#endif /* ETHR_READ_AND_SET_WITHOUT_SYNC_OP__ */ -static ETHR_INLINE void -ETHR_NATMC_FUNC__(dec)(ETHR_ATMC_T__ *var) -{ - (void) __sync_sub_and_fetch(&var->counter, (ETHR_AINT_T__) 1); -} +#if defined(ETHR_HAVE___SYNC_ADD_AND_FETCH) -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(inc_return)(ETHR_ATMC_T__ *var) -{ - return __sync_add_and_fetch(&var->counter, (ETHR_AINT_T__) 1); -} +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_MB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_MB 1 +#endif static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(dec_return)(ETHR_ATMC_T__ *var) +ETHR_NATMC_FUNC__(add_return_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) { - return __sync_sub_and_fetch(&var->counter, (ETHR_AINT_T__) 1); + return __sync_add_and_fetch(&var->counter, incr); } -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(and_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) -{ - return __sync_fetch_and_and(&var->counter, mask); -} +#endif -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(or_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) -{ - return (ETHR_AINT_T__) __sync_fetch_and_or(&var->counter, mask); -} +#if defined(ETHR_HAVE___SYNC_FETCH_AND_AND) -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var, - ETHR_AINT_T__ new, - ETHR_AINT_T__ old) -{ - return __sync_val_compare_and_swap(&var->counter, old, new); -} +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_MB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_MB 1 +#endif static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(xchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new) +ETHR_NATMC_FUNC__(and_retold_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) { - ETHR_AINT_T__ exp, act = 0; - do { - exp = act; - act = __sync_val_compare_and_swap(&var->counter, exp, new); - } while (act != exp); - return act; + return __sync_fetch_and_and(&var->counter, mask); } -/* - * Atomic ops with at least specified barriers. - */ - -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(read_acqb)(ETHR_ATMC_T__ *var) -{ -#if ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ - ETHR_AINT_T__ val = var->counter; - ETHR_COMPILER_BARRIER; - return val; -#else - return __sync_add_and_fetch(&var->counter, (ETHR_AINT_T__) 0); #endif -} -static ETHR_INLINE void -ETHR_NATMC_FUNC__(set_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) -{ -#if ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ - ETHR_COMPILER_BARRIER; - var->counter = i; +#if defined(ETHR_HAVE___SYNC_FETCH_AND_OR) + +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_MB 1 #else - (void) ETHR_NATMC_FUNC__(xchg)(var, i); +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_MB 1 #endif -} static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(inc_return_acqb)(ETHR_ATMC_T__ *var) -{ - return ETHR_NATMC_FUNC__(inc_return)(var); -} - -static ETHR_INLINE void -ETHR_NATMC_FUNC__(dec_relb)(ETHR_ATMC_T__ *var) +ETHR_NATMC_FUNC__(or_retold_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) { - ETHR_NATMC_FUNC__(dec)(var); + return (ETHR_AINT_T__) __sync_fetch_and_or(&var->counter, mask); } -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(dec_return_relb)(ETHR_ATMC_T__ *var) -{ - return ETHR_NATMC_FUNC__(dec_return)(var); -} +#endif -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var, - ETHR_AINT_T__ new, - ETHR_AINT_T__ old) -{ - return ETHR_NATMC_FUNC__(cmpxchg)(var, new, old); -} +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_MB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_MB 1 +#endif static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var, - ETHR_AINT_T__ new, - ETHR_AINT_T__ old) +ETHR_NATMC_FUNC__(cmpxchg_mb)(ETHR_ATMC_T__ *var, + ETHR_AINT_T__ new, + ETHR_AINT_T__ old) { - return ETHR_NATMC_FUNC__(cmpxchg)(var, new, old); + return __sync_val_compare_and_swap(&var->counter, old, new); } -#endif +#endif /* ETHR_TRY_INLINE_FUNCS */ #undef ETHR_NATMC_FUNC__ #undef ETHR_ATMC_T__ #undef ETHR_AINT_T__ #undef ETHR_AINT_SUFFIX__ +#undef ETHR_HAVE___SYNC_ADD_AND_FETCH +#undef ETHR_HAVE___SYNC_FETCH_AND_AND +#undef ETHR_HAVE___SYNC_FETCH_AND_OR #endif diff --git a/erts/include/internal/gcc/ethr_dw_atomic.h b/erts/include/internal/gcc/ethr_dw_atomic.h new file mode 100644 index 0000000000..6736f9c547 --- /dev/null +++ b/erts/include/internal/gcc/ethr_dw_atomic.h @@ -0,0 +1,115 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2011. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Native double word atomics using gcc's builtins + * Author: Rickard Green + */ + +#undef ETHR_INCLUDE_DW_ATOMIC_IMPL__ +#ifndef ETHR_GCC_DW_ATOMIC_H__ +# define ETHR_GCC_DW_ATOMIC_H__ +# if ((ETHR_SIZEOF_PTR == 4 \ + && defined(ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP64)) \ + || (ETHR_SIZEOF_PTR == 8 \ + && defined(ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP128) \ + && defined(ETHR_HAVE_INT128_T))) +# define ETHR_INCLUDE_DW_ATOMIC_IMPL__ +# endif +#endif + +#ifdef ETHR_INCLUDE_DW_ATOMIC_IMPL__ +# define ETHR_HAVE_NATIVE_SU_DW_ATOMIC +# define ETHR_NATIVE_DW_ATOMIC_IMPL "gcc" + +# if defined(__i386__) || defined(__x86_64__) +/* + * If ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ is defined, it will be used + * at runtime in order to determine if native or fallback implementation + * should be used. + */ +# define ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ \ + ETHR_X86_RUNTIME_CONF_HAVE_DW_CMPXCHG__ +# endif + +# if ETHR_SIZEOF_PTR == 4 +# define ETHR_DW_NATMC_ALIGN_MASK__ 0x7 +# define ETHR_NATIVE_SU_DW_SINT_T ethr_sint64_t +# elif ETHR_SIZEOF_PTR == 8 +# define ETHR_DW_NATMC_ALIGN_MASK__ 0xf +# define ETHR_NATIVE_SU_DW_SINT_T ethr_sint128_t +# endif + +typedef volatile ETHR_NATIVE_SU_DW_SINT_T * ethr_native_dw_ptr_t; + +/* + * We need 16 byte aligned memory in 64-bit mode, and 8 byte aligned + * memory in 32-bit mode. 16 byte aligned malloc in 64-bit mode is + * not common, and at least some glibc malloc implementations + * only 4 byte align in 32-bit mode. + * + * This code assumes 8 byte aligned memory in 64-bit mode, and 4 byte + * aligned memory in 32-bit mode. A malloc implementation that does + * not adhere to these alignment requirements is seriously broken, + * and we wont bother trying to work around it. + * + * Since memory alignment may be off by one word we need to align at + * runtime. We, therefore, need an extra word allocated. + */ +#define ETHR_DW_NATMC_MEM__(VAR) \ + (&var->c[(int) ((ethr_uint_t) &(VAR)->c[0]) & ETHR_DW_NATMC_ALIGN_MASK__]) +typedef union { + volatile ETHR_NATIVE_SU_DW_SINT_T dw_sint; + volatile ethr_sint_t sint[3]; + volatile char c[ETHR_SIZEOF_PTR*3]; +} ethr_native_dw_atomic_t; + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) + +# ifdef ETHR_DEBUG +# define ETHR_DW_DBG_ALIGNED__(PTR) \ + ETHR_ASSERT((((ethr_uint_t) (PTR)) & ETHR_DW_NATMC_ALIGN_MASK__) == 0); +# else +# define ETHR_DW_DBG_ALIGNED__(PTR) +# endif + +#define ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_ADDR +static ETHR_INLINE ethr_sint_t * +ethr_native_dw_atomic_addr(ethr_native_dw_atomic_t *var) +{ + return (ethr_sint_t *) ETHR_DW_NATMC_MEM__(var); +} + + +#define ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_MB + +static ETHR_INLINE ETHR_NATIVE_SU_DW_SINT_T +ethr_native_su_dw_atomic_cmpxchg_mb(ethr_native_dw_atomic_t *var, + ETHR_NATIVE_SU_DW_SINT_T new, + ETHR_NATIVE_SU_DW_SINT_T old) +{ + ethr_native_dw_ptr_t p = (ethr_native_dw_ptr_t) ETHR_DW_NATMC_MEM__(var); + ETHR_DW_DBG_ALIGNED__(p); + return __sync_val_compare_and_swap(p, old, new); +} + +#endif /* ETHR_TRY_INLINE_FUNCS */ + +#endif /* ETHR_GCC_DW_ATOMIC_H__ */ + diff --git a/erts/include/internal/gcc/ethr_membar.h b/erts/include/internal/gcc/ethr_membar.h new file mode 100644 index 0000000000..7d428fc68e --- /dev/null +++ b/erts/include/internal/gcc/ethr_membar.h @@ -0,0 +1,73 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2011. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Memory barriers when using gcc's builtins + * Author: Rickard Green + */ + +#ifndef ETHR_GCC_MEMBAR_H__ +#define ETHR_GCC_MEMBAR_H__ + +#define ETHR_LoadLoad (1 << 0) +#define ETHR_LoadStore (1 << 1) +#define ETHR_StoreLoad (1 << 2) +#define ETHR_StoreStore (1 << 3) + +/* + * According to the documentation __sync_synchronize() will + * issue a full memory barrier. However, __sync_synchronize() + * is known to erroneously be a noop on at least some + * platforms with some gcc versions. This has suposedly been + * fixed in some gcc version, but we don't know from which + * version. Therefore, we only use it when it has been + * verified to work. Otherwise we use the workaround + * below. + */ + +#if defined(ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP32) +# define ETHR_MB_T__ ethr_sint32_t +#elif defined(ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP64) +# define ETHR_MB_T__ ethr_sint64_t +#else +# error "No __sync_val_compare_and_swap" +#endif +#define ETHR_SYNC_SYNCHRONIZE_WORKAROUND__ \ +do { \ + volatile ETHR_MB_T__ x___ = 0; \ + (void) __sync_val_compare_and_swap(&x___, (ETHR_MB_T__) 0, (ETHR_MB_T__) 1); \ +} while (0) + +#define ETHR_COMPILER_BARRIER __asm__ __volatile__("" : : : "memory") + +#if defined(__mips__) && ETHR_AT_LEAST_GCC_VSN__(4, 2, 0) +# define ETHR_MEMBAR(B) __sync_synchronize() +# define ETHR_READ_DEPEND_MEMORY_BARRIER __sync_synchronize() +#elif ((defined(__powerpc__) || defined(__ppc__)) \ + && ETHR_AT_LEAST_GCC_VSN__(4, 1, 2)) +# define ETHR_MEMBAR(B) __sync_synchronize() +#else /* Use workaround */ +# define ETHR_MEMBAR(B) \ + ETHR_SYNC_SYNCHRONIZE_WORKAROUND__ +# define ETHR_READ_DEPEND_MEMORY_BARRIER \ + ETHR_SYNC_SYNCHRONIZE_WORKAROUND__ +#endif + + +#endif /* ETHR_GCC_MEMBAR_H__ */ diff --git a/erts/include/internal/gcc/ethread.h b/erts/include/internal/gcc/ethread.h index 392a1aa2b2..fcfdc39441 100644 --- a/erts/include/internal/gcc/ethread.h +++ b/erts/include/internal/gcc/ethread.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2010. All Rights Reserved. + * Copyright Ericsson AB 2010-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -25,16 +25,24 @@ #ifndef ETHREAD_GCC_H__ #define ETHREAD_GCC_H__ -#if !defined(ETHR_HAVE_NATIVE_ATOMICS) && defined(ETHR_HAVE_GCC_ATOMIC_OPS) -#define ETHR_HAVE_NATIVE_ATOMICS 1 +#ifndef ETHR_MEMBAR +# include "ethr_membar.h" +#endif + +#if !defined(ETHR_HAVE_NATIVE_ATOMIC32) +# define ETHR_ATOMIC_WANT_32BIT_IMPL__ +# include "ethr_atomic.h" +#endif -#define ETHR_ATOMIC_WANT_32BIT_IMPL__ -#include "ethr_atomic.h" -#if ETHR_SIZEOF_PTR == 8 +#if ETHR_SIZEOF_PTR == 8 && !defined(ETHR_HAVE_NATIVE_ATOMIC64) # define ETHR_ATOMIC_WANT_64BIT_IMPL__ # include "ethr_atomic.h" #endif +#if (!defined(ETHR_HAVE_NATIVE_DW_ATOMIC) \ + && !(ETHR_SIZEOF_PTR == 4 && defined(ETHR_HAVE_NATIVE_ATOMIC64)) \ + && !(ETHR_SIZEOF_PTR == 8 && defined(ETHR_HAVE_NATIVE_ATOMIC128))) +# include "ethr_dw_atomic.h" #endif #endif diff --git a/erts/include/internal/i386/atomic.h b/erts/include/internal/i386/atomic.h index 4e402f261a..fc1b619935 100644 --- a/erts/include/internal/i386/atomic.h +++ b/erts/include/internal/i386/atomic.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * Copyright Ericsson AB 2005-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -25,53 +25,42 @@ */ #undef ETHR_INCLUDE_ATOMIC_IMPL__ -#if !defined(ETHR_X86_ATOMIC32_H__) && defined(ETHR_ATOMIC_WANT_32BIT_IMPL__) -#define ETHR_X86_ATOMIC32_H__ -#define ETHR_INCLUDE_ATOMIC_IMPL__ 4 -#undef ETHR_ATOMIC_WANT_32BIT_IMPL__ -#elif !defined(ETHR_X86_ATOMIC64_H__) && defined(ETHR_ATOMIC_WANT_64BIT_IMPL__) -#define ETHR_X86_ATOMIC64_H__ -#define ETHR_INCLUDE_ATOMIC_IMPL__ 8 -#undef ETHR_ATOMIC_WANT_64BIT_IMPL__ +#if !defined(ETHR_X86_ATOMIC32_H__) \ + && defined(ETHR_ATOMIC_WANT_32BIT_IMPL__) +# define ETHR_X86_ATOMIC32_H__ +# define ETHR_INCLUDE_ATOMIC_IMPL__ 4 +# undef ETHR_ATOMIC_WANT_32BIT_IMPL__ +#elif !defined(ETHR_X86_ATOMIC64_H__) \ + && defined(ETHR_ATOMIC_WANT_64BIT_IMPL__) +# define ETHR_X86_ATOMIC64_H__ +# define ETHR_INCLUDE_ATOMIC_IMPL__ 8 +# undef ETHR_ATOMIC_WANT_64BIT_IMPL__ #endif #ifdef ETHR_INCLUDE_ATOMIC_IMPL__ -#ifndef ETHR_X86_ATOMIC_COMMON__ -#define ETHR_X86_ATOMIC_COMMON__ - -#define ETHR_ATOMIC_HAVE_INC_DEC_INSTRUCTIONS 1 - -#if defined(__x86_64__) || !defined(ETHR_PRE_PENTIUM4_COMPAT) -#define ETHR_MEMORY_BARRIER __asm__ __volatile__("mfence" : : : "memory") -#define ETHR_WRITE_MEMORY_BARRIER __asm__ __volatile__("sfence" : : : "memory") -#define ETHR_READ_MEMORY_BARRIER __asm__ __volatile__("lfence" : : : "memory") -#define ETHR_READ_DEPEND_MEMORY_BARRIER __asm__ __volatile__("" : : : "memory") -#else -#define ETHR_MEMORY_BARRIER \ -do { \ - volatile ethr_sint32_t x___ = 0; \ - __asm__ __volatile__("lock; incl %0" : "=m"(x___) : "m"(x___) : "memory"); \ -} while (0) -#endif - -#endif /* ETHR_X86_ATOMIC_COMMON__ */ - -#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 -#define ETHR_HAVE_NATIVE_ATOMIC32 1 -#define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X -#define ETHR_ATMC_T__ ethr_native_atomic32_t -#define ETHR_AINT_T__ ethr_sint32_t -#define ETHR_AINT_SUFFIX__ "l" -#elif ETHR_INCLUDE_ATOMIC_IMPL__ == 8 -#define ETHR_HAVE_NATIVE_ATOMIC64 1 -#define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X -#define ETHR_ATMC_T__ ethr_native_atomic64_t -#define ETHR_AINT_T__ ethr_sint64_t -#define ETHR_AINT_SUFFIX__ "q" -#else -#error "Unsupported integer size" -#endif +# ifndef ETHR_X86_ATOMIC_COMMON__ +# define ETHR_X86_ATOMIC_COMMON__ +# define ETHR_ATOMIC_HAVE_INC_DEC_INSTRUCTIONS 1 +# endif /* ETHR_X86_ATOMIC_COMMON__ */ + +# if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_NATIVE_ATOMIC32 1 +# define ETHR_NATIVE_ATOMIC32_IMPL "ethread" +# define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X +# define ETHR_ATMC_T__ ethr_native_atomic32_t +# define ETHR_AINT_T__ ethr_sint32_t +# define ETHR_AINT_SUFFIX__ "l" +# elif ETHR_INCLUDE_ATOMIC_IMPL__ == 8 +# define ETHR_HAVE_NATIVE_ATOMIC64 1 +# define ETHR_NATIVE_ATOMIC64_IMPL "ethread" +# define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X +# define ETHR_ATMC_T__ ethr_native_atomic64_t +# define ETHR_AINT_T__ ethr_sint64_t +# define ETHR_AINT_SUFFIX__ "q" +# else +# error "Unsupported integer size" +# endif /* An atomic is an aligned ETHR_AINT_T__ accessed via locked operations. */ @@ -81,87 +70,28 @@ typedef struct { #if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADDR 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADDR 1 +#endif + static ETHR_INLINE ETHR_AINT_T__ * ETHR_NATMC_FUNC__(addr)(ETHR_ATMC_T__ *var) { return (ETHR_AINT_T__ *) &var->counter; } -static ETHR_INLINE void -ETHR_NATMC_FUNC__(init)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) -{ - var->counter = i; -} - -static ETHR_INLINE void -ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) -{ - var->counter = i; -} - -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var) -{ - return var->counter; -} - -static ETHR_INLINE void -ETHR_NATMC_FUNC__(add)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) -{ - __asm__ __volatile__( - "lock; add" ETHR_AINT_SUFFIX__ " %1, %0" - : "=m"(var->counter) - : "ir"(incr), "m"(var->counter)); -} - -static ETHR_INLINE void -ETHR_NATMC_FUNC__(inc)(ETHR_ATMC_T__ *var) -{ - __asm__ __volatile__( - "lock; inc" ETHR_AINT_SUFFIX__ " %0" - : "=m"(var->counter) - : "m"(var->counter)); -} - -static ETHR_INLINE void -ETHR_NATMC_FUNC__(dec)(ETHR_ATMC_T__ *var) -{ - __asm__ __volatile__( - "lock; dec" ETHR_AINT_SUFFIX__ " %0" - : "=m"(var->counter) - : "m"(var->counter)); -} - -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(add_return)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) -{ - ETHR_AINT_T__ tmp; - - tmp = incr; - __asm__ __volatile__( - "lock; xadd" ETHR_AINT_SUFFIX__ " %0, %1" /* xadd didn't exist prior to the 486 */ - : "=r"(tmp) - : "m"(var->counter), "0"(tmp)); - /* now tmp is the atomic's previous value */ - return tmp + incr; -} - -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(inc_return)(ETHR_ATMC_T__ *var) -{ - return ETHR_NATMC_FUNC__(add_return)(var, (ETHR_AINT_T__) 1); -} - -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(dec_return)(ETHR_ATMC_T__ *var) -{ - return ETHR_NATMC_FUNC__(add_return)(var, (ETHR_AINT_T__) -1); -} +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_MB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_MB 1 +#endif static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var, - ETHR_AINT_T__ new, - ETHR_AINT_T__ old) +ETHR_NATMC_FUNC__(cmpxchg_mb)(ETHR_ATMC_T__ *var, + ETHR_AINT_T__ new, + ETHR_AINT_T__ old) { __asm__ __volatile__( "lock; cmpxchg" ETHR_AINT_SUFFIX__ " %2, %3" @@ -171,110 +101,148 @@ ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var, return old; } -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(and_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) -{ - ETHR_AINT_T__ tmp, old; - - tmp = var->counter; - do { - old = tmp; - tmp = ETHR_NATMC_FUNC__(cmpxchg)(var, tmp & mask, tmp); - } while (__builtin_expect(tmp != old, 0)); - /* now tmp is the atomic's previous value */ - return tmp; -} - -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(or_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) -{ - ETHR_AINT_T__ tmp, old; - - tmp = var->counter; - do { - old = tmp; - tmp = ETHR_NATMC_FUNC__(cmpxchg)(var, tmp | mask, tmp); - } while (__builtin_expect(tmp != old, 0)); - /* now tmp is the atomic's previous value */ - return tmp; -} +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_MB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_MB 1 +#endif static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(xchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ val) +ETHR_NATMC_FUNC__(xchg_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ val) { ETHR_AINT_T__ tmp = val; __asm__ __volatile__( "xchg" ETHR_AINT_SUFFIX__ " %0, %1" : "=r"(tmp) - : "m"(var->counter), "0"(tmp)); + : "m"(var->counter), "0"(tmp) + : "memory"); /* now tmp is the atomic's previous value */ return tmp; } -/* - * Atomic ops with at least specified barriers. - */ +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET 1 +#endif -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(read_acqb)(ETHR_ATMC_T__ *var) +static ETHR_INLINE void +ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) { - ETHR_AINT_T__ val; -#if defined(__x86_64__) || !defined(ETHR_PRE_PENTIUM4_COMPAT) - val = var->counter; + var->counter = i; +} + +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_RELB 1 #else - val = ETHR_NATMC_FUNC__(add_return)(var, 0); +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RELB 1 #endif - __asm__ __volatile__("" : : : "memory"); - return val; -} static ETHR_INLINE void ETHR_NATMC_FUNC__(set_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) { - __asm__ __volatile__("" : : : "memory"); -#if defined(__x86_64__) || !defined(ETHR_PRE_PENTIUM4_COMPAT) - var->counter = i; +#if defined(_M_IX86) + if (ETHR_X86_RUNTIME_CONF_HAVE_NO_SSE2__) + (void) ETHR_NATMC_FUNC__(xchg_mb)(var, i); + else +#endif /* _M_IX86 */ + { + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + var->counter = i; + } +} + +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_MB 1 #else - (void) ETHR_NATMC_FUNC__(xchg)(var, i); +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_MB 1 #endif + +static ETHR_INLINE void +ETHR_NATMC_FUNC__(set_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) +{ + (void) ETHR_NATMC_FUNC__(xchg_mb)(var, i); } +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ 1 +#endif + static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(inc_return_acqb)(ETHR_ATMC_T__ *var) +ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var) { - ETHR_AINT_T__ res = ETHR_NATMC_FUNC__(inc_return)(var); - __asm__ __volatile__("" : : : "memory"); - return res; + return var->counter; } +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_MB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_MB 1 +#endif + static ETHR_INLINE void -ETHR_NATMC_FUNC__(dec_relb)(ETHR_ATMC_T__ *var) +ETHR_NATMC_FUNC__(add_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) { - __asm__ __volatile__("" : : : "memory"); - ETHR_NATMC_FUNC__(dec)(var); -} + __asm__ __volatile__( + "lock; add" ETHR_AINT_SUFFIX__ " %1, %0" + : "=m"(var->counter) + : "ir"(incr), "m"(var->counter) + : "memory"); +} -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(dec_return_relb)(ETHR_ATMC_T__ *var) +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_MB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_MB 1 +#endif + +static ETHR_INLINE void +ETHR_NATMC_FUNC__(inc_mb)(ETHR_ATMC_T__ *var) { - __asm__ __volatile__("" : : : "memory"); - return ETHR_NATMC_FUNC__(dec_return)(var); + __asm__ __volatile__( + "lock; inc" ETHR_AINT_SUFFIX__ " %0" + : "=m"(var->counter) + : "m"(var->counter) + : "memory"); } -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var, - ETHR_AINT_T__ new, - ETHR_AINT_T__ old) +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_MB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_MB 1 +#endif + +static ETHR_INLINE void +ETHR_NATMC_FUNC__(dec_mb)(ETHR_ATMC_T__ *var) { - return ETHR_NATMC_FUNC__(cmpxchg)(var, new, old); + __asm__ __volatile__( + "lock; dec" ETHR_AINT_SUFFIX__ " %0" + : "=m"(var->counter) + : "m"(var->counter) + : "memory"); } +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_MB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_MB 1 +#endif + static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var, - ETHR_AINT_T__ new, - ETHR_AINT_T__ old) +ETHR_NATMC_FUNC__(add_return_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) { - return ETHR_NATMC_FUNC__(cmpxchg)(var, new, old); + ETHR_AINT_T__ tmp; + + tmp = incr; + __asm__ __volatile__( + "lock; xadd" ETHR_AINT_SUFFIX__ " %0, %1" /* xadd didn't exist prior to the 486 */ + : "=r"(tmp) + : "m"(var->counter), "0"(tmp) + : "memory"); + /* now tmp is the atomic's previous value */ + return tmp + incr; } #endif /* ETHR_TRY_INLINE_FUNCS */ diff --git a/erts/include/internal/i386/ethr_dw_atomic.h b/erts/include/internal/i386/ethr_dw_atomic.h new file mode 100644 index 0000000000..9fb89bbe43 --- /dev/null +++ b/erts/include/internal/i386/ethr_dw_atomic.h @@ -0,0 +1,278 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2011. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Native double word atomics for x86/x86_64 + * Author: Rickard Green + */ + +#ifndef ETHR_X86_DW_ATOMIC_H__ +#define ETHR_X86_DW_ATOMIC_H__ + +#ifdef ETHR_GCC_HAVE_DW_CMPXCHG_ASM_SUPPORT + +#define ETHR_HAVE_NATIVE_DW_ATOMIC +#define ETHR_NATIVE_DW_ATOMIC_IMPL "ethread" + +/* + * If ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ is defined, it will be used + * at runtime in order to determine if native or fallback implementation + * should be used. + */ +#define ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ \ + ETHR_X86_RUNTIME_CONF_HAVE_DW_CMPXCHG__ + +#if ETHR_SIZEOF_PTR == 4 +typedef volatile ethr_sint64_t * ethr_native_dw_ptr_t; +# define ETHR_DW_NATMC_ALIGN_MASK__ 0x7 +# define ETHR_DW_CMPXCHG_SFX__ "8b" +# define ETHR_NATIVE_SU_DW_SINT_T ethr_sint64_t +#else +#ifdef ETHR_HAVE_INT128_T +# define ETHR_NATIVE_SU_DW_SINT_T ethr_sint128_t +typedef volatile ethr_sint128_t * ethr_native_dw_ptr_t; +#else +typedef struct { + ethr_sint64_t sint64[2]; +} ethr_native_sint128_t__; +typedef volatile ethr_native_sint128_t__ * ethr_native_dw_ptr_t; +#endif +# define ETHR_DW_NATMC_ALIGN_MASK__ 0xf +# define ETHR_DW_CMPXCHG_SFX__ "16b" +#endif + +/* + * We need 16 byte aligned memory in 64-bit mode, and 8 byte aligned + * memory in 32-bit mode. 16 byte aligned malloc in 64-bit mode is + * not common, and at least some glibc malloc implementations + * only 4 byte align in 32-bit mode. + * + * This code assumes 8 byte aligned memory in 64-bit mode, and 4 byte + * aligned memory in 32-bit mode. A malloc implementation that does + * not adhere to these alignment requirements is seriously broken, + * and we wont bother trying to work around it. + * + * Since memory alignment may be off by one word we need to align at + * runtime. We, therefore, need an extra word allocated. + */ +#define ETHR_DW_NATMC_MEM__(VAR) \ + (&var->c[(int) ((ethr_uint_t) &(VAR)->c[0]) & ETHR_DW_NATMC_ALIGN_MASK__]) +typedef union { +#ifdef ETHR_NATIVE_SU_DW_SINT_T + volatile ETHR_NATIVE_SU_DW_SINT_T dw_sint; +#endif + volatile ethr_sint_t sint[3]; + volatile char c[ETHR_SIZEOF_PTR*3]; +} ethr_native_dw_atomic_t; + + +#if (defined(ETHR_TRY_INLINE_FUNCS) \ + || defined(ETHR_ATOMIC_IMPL__) \ + || defined(ETHR_X86_SSE2_ASM_C__)) \ + && ETHR_SIZEOF_PTR == 4 \ + && defined(ETHR_GCC_HAVE_SSE2_ASM_SUPPORT) +ethr_sint64_t +ethr_sse2_native_su_dw_atomic_read(ethr_native_dw_atomic_t *var); +void +ethr_sse2_native_su_dw_atomic_set(ethr_native_dw_atomic_t *var, + ethr_sint64_t val); +#endif + +#if (defined(ETHR_TRY_INLINE_FUNCS) \ + || defined(ETHR_ATOMIC_IMPL__) \ + || defined(ETHR_X86_SSE2_ASM_C__)) +# ifdef ETHR_DEBUG +# define ETHR_DW_DBG_ALIGNED__(PTR) \ + ETHR_ASSERT((((ethr_uint_t) (PTR)) & ETHR_DW_NATMC_ALIGN_MASK__) == 0); +# else +# define ETHR_DW_DBG_ALIGNED__(PTR) +# endif +#endif + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) + +#define ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_ADDR +static ETHR_INLINE ethr_sint_t * +ethr_native_dw_atomic_addr(ethr_native_dw_atomic_t *var) +{ + return (ethr_sint_t *) ETHR_DW_NATMC_MEM__(var); +} + +#if ETHR_SIZEOF_PTR == 4 && defined(__PIC__) && __PIC__ +/* + * When position independent code is used in 32-bit mode, the EBX register + * is used for storage of global offset table address, and we may not + * use it as input or output in an asm. We need to save and restore the + * EBX register explicitly (for some reason gcc doesn't provide this + * service to us). + */ +# define ETHR_NO_CLOBBER_EBX__ 1 +#else +# define ETHR_NO_CLOBBER_EBX__ 0 +#endif + +#if ETHR_NO_CLOBBER_EBX__ && !defined(ETHR_CMPXCHG8B_REGISTER_SHORTAGE) +/* When no optimization is on, we'll run into a register shortage */ +# if defined(ETHR_DEBUG) || defined(DEBUG) || defined(VALGRIND) \ + || defined(GCOV) || defined(PURIFY) || defined(PURECOV) +# define ETHR_CMPXCHG8B_REGISTER_SHORTAGE 1 +# else +# define ETHR_CMPXCHG8B_REGISTER_SHORTAGE 0 +# endif +#endif + + +#define ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_MB + +static ETHR_INLINE int +ethr_native_dw_atomic_cmpxchg_mb(ethr_native_dw_atomic_t *var, + ethr_sint_t *new, + ethr_sint_t *xchg) +{ + ethr_native_dw_ptr_t p = (ethr_native_dw_ptr_t) ETHR_DW_NATMC_MEM__(var); + char xchgd; + + ETHR_DW_DBG_ALIGNED__(p); + + __asm__ __volatile__( +#if ETHR_NO_CLOBBER_EBX__ + "pushl %%ebx\n\t" +# if ETHR_CMPXCHG8B_REGISTER_SHORTAGE + "movl (%7), %%ebx\n\t" + "movl 4(%7), %%ecx\n\t" +# else + "movl %8, %%ebx\n\t" +# endif +#endif + "lock; cmpxchg" ETHR_DW_CMPXCHG_SFX__ " %0\n\t" + "setz %3\n\t" +#if ETHR_NO_CLOBBER_EBX__ + "popl %%ebx\n\t" +#endif + : "=m"(*p), "=d"(xchg[1]), "=a"(xchg[0]), "=c"(xchgd) + : "m"(*p), "1"(xchg[1]), "2"(xchg[0]), +#if ETHR_NO_CLOBBER_EBX__ +# if ETHR_CMPXCHG8B_REGISTER_SHORTAGE + "3"(new) +# else + "3"(new[1]), + "r"(new[0]) +# endif +#else + "3"(new[1]), + "b"(new[0]) +#endif + : "cc", "memory"); + + return (int) xchgd; +} + +#undef ETHR_NO_CLOBBER_EBX__ + +#if ETHR_SIZEOF_PTR == 4 && defined(ETHR_GCC_HAVE_SSE2_ASM_SUPPORT) + +typedef union { + ethr_sint64_t sint64; + ethr_sint_t sint[2]; +} ethr_dw_atomic_no_sse2_convert_t; + +#define ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ + +static ETHR_INLINE ethr_sint64_t +ethr_native_su_dw_atomic_read(ethr_native_dw_atomic_t *var) +{ + if (ETHR_X86_RUNTIME_CONF_HAVE_SSE2__) + return ethr_sse2_native_su_dw_atomic_read(var); + else { + ethr_sint_t new[2]; + ethr_dw_atomic_no_sse2_convert_t xchg; + new[0] = new[1] = xchg.sint[0] = xchg.sint[1] = 0x83838383; + (void) ethr_native_dw_atomic_cmpxchg_mb(var, new, xchg.sint); + return xchg.sint64; + } +} + +#define ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET + +static ETHR_INLINE void +ethr_native_su_dw_atomic_set(ethr_native_dw_atomic_t *var, + ethr_sint64_t val) +{ + if (ETHR_X86_RUNTIME_CONF_HAVE_SSE2__) + ethr_sse2_native_su_dw_atomic_set(var, val); + else { + ethr_sint_t xchg[2] = {0, 0}; + ethr_dw_atomic_no_sse2_convert_t new; + new.sint64 = val; + while (!ethr_native_dw_atomic_cmpxchg_mb(var, new.sint, xchg)); + } +} + +#endif /* ETHR_SIZEOF_PTR == 4 */ + +#endif /* ETHR_TRY_INLINE_FUNCS */ + +#if defined(ETHR_X86_SSE2_ASM_C__) \ + && ETHR_SIZEOF_PTR == 4 \ + && defined(ETHR_GCC_HAVE_SSE2_ASM_SUPPORT) + +/* + * 8-byte aligned loads and stores of 64-bit values are atomic from + * pentium and forward. An ordinary volatile load or store in 32-bit + * mode generates two 32-bit operations (at least with gcc-4.1.2 using + * -msse2). In order to guarantee one 64-bit load/store operation + * from/to memory we load/store via an xmm register using movq. + * + * Load/store can be achieved using cmpxchg8b, however, using movq is + * much faster. Unfortunately we cannot do the same thing in 64-bit + * mode; instead, we have to do loads and stores via cmpxchg16b. + * + * We do not inline these, but instead compile these into a separate + * object file using -msse2. This since we don't want to use -msse2 for + * the whole system. If we detect sse2 support (pentium4 and forward) + * at runtime, we use them; otherwise, we fall back to using cmpxchg8b + * for loads and stores. This way the binary can be moved between + * processors with and without sse2 support. + */ + +ethr_sint64_t +ethr_sse2_native_su_dw_atomic_read(ethr_native_dw_atomic_t *var) +{ + ethr_native_dw_ptr_t p = (ethr_native_dw_ptr_t) ETHR_DW_NATMC_MEM__(var); + ethr_sint64_t val; + ETHR_DW_DBG_ALIGNED__(p); + __asm__ __volatile__("movq %1, %0\n\t" : "=x"(val) : "m"(*p) : "memory"); + return val; +} + +void +ethr_sse2_native_su_dw_atomic_set(ethr_native_dw_atomic_t *var, + ethr_sint64_t val) +{ + ethr_native_dw_ptr_t p = (ethr_native_dw_ptr_t) ETHR_DW_NATMC_MEM__(var); + ETHR_DW_DBG_ALIGNED__(p); + __asm__ __volatile__("movq %1, %0\n\t" : "=m"(*p) : "x"(val) : "memory"); +} + +#endif /* ETHR_X86_SSE2_ASM_C__ */ + +#endif /* ETHR_GCC_HAVE_DW_CMPXCHG_ASM_SUPPORT */ + +#endif /* ETHR_X86_DW_ATOMIC_H__ */ + diff --git a/erts/include/internal/i386/ethr_membar.h b/erts/include/internal/i386/ethr_membar.h new file mode 100644 index 0000000000..92d9de7f3f --- /dev/null +++ b/erts/include/internal/i386/ethr_membar.h @@ -0,0 +1,114 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2011. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Memory barriers for x86/x86-64 + * Author: Rickard Green + */ + +#ifndef ETHR_X86_MEMBAR_H__ +#define ETHR_X86_MEMBAR_H__ + +#define ETHR_LoadLoad (1 << 0) +#define ETHR_LoadStore (1 << 1) +#define ETHR_StoreLoad (1 << 2) +#define ETHR_StoreStore (1 << 3) + +#define ETHR_NO_SSE2_MEMORY_BARRIER__ \ +do { \ + volatile ethr_sint32_t x__ = 0; \ + __asm__ __volatile__ ("lock; orl $0x0, %0\n\t" \ + : "=m"(x__) \ + : "m"(x__) \ + : "memory"); \ +} while (0) + +static __inline__ void +ethr_cfence__(void) +{ + __asm__ __volatile__ ("" : : : "memory"); +} + +static __inline__ void +ethr_mfence__(void) +{ +#if ETHR_SIZEOF_PTR == 4 + if (ETHR_X86_RUNTIME_CONF_HAVE_NO_SSE2__) + ETHR_NO_SSE2_MEMORY_BARRIER__; + else +#endif + __asm__ __volatile__ ("mfence\n\t" : : : "memory"); +} + +static __inline__ void +ethr_sfence__(void) +{ +#if ETHR_SIZEOF_PTR == 4 + if (ETHR_X86_RUNTIME_CONF_HAVE_NO_SSE2__) + ETHR_NO_SSE2_MEMORY_BARRIER__; + else +#endif + __asm__ __volatile__ ("sfence\n\t" : : : "memory"); +} + +static __inline__ void +ethr_lfence__(void) +{ +#if ETHR_SIZEOF_PTR == 4 + if (ETHR_X86_RUNTIME_CONF_HAVE_NO_SSE2__) + ETHR_NO_SSE2_MEMORY_BARRIER__; + else +#endif + __asm__ __volatile__ ("lfence\n\t" : : : "memory"); +} + +#define ETHR_X86_OUT_OF_ORDER_MEMBAR(B) \ + ETHR_CHOOSE_EXPR((B) == ETHR_StoreStore, \ + ethr_sfence__(), \ + ETHR_CHOOSE_EXPR((B) == ETHR_LoadLoad, \ + ethr_lfence__(), \ + ethr_mfence__())) + +#ifdef ETHR_X86_OUT_OF_ORDER + +#define ETHR_MEMBAR(B) \ + ETHR_X86_OUT_OF_ORDER_MEMBAR((B)) + +#else /* !ETHR_X86_OUT_OF_ORDER (the default) */ + +/* + * We assume that only stores before loads may be reordered. That is, + * we assume that *no* instructions like these are used: + * - CLFLUSH, + * - streaming stores executed with non-temporal move, + * - string operations, or + * - other instructions which aren't LoadLoad, LoadStore, and StoreStore + * ordered by themselves + * If such instructions are used, either insert memory barriers + * using ETHR_X86_OUT_OF_ORDER_MEMBAR() at appropriate places, or + * define ETHR_X86_OUT_OF_ORDER. For more info see Intel 64 and IA-32 + * Architectures Software Developer's Manual; Vol 3A; Chapter 8.2.2. + */ + +#define ETHR_MEMBAR(B) \ + ETHR_CHOOSE_EXPR((B) & ETHR_StoreLoad, ethr_mfence__(), ethr_cfence__()) + +#endif /* !ETHR_X86_OUT_OF_ORDER */ + +#endif /* ETHR_X86_MEMBAR_H__ */ diff --git a/erts/include/internal/i386/ethread.h b/erts/include/internal/i386/ethread.h index b5a17caefb..80e4dc7b99 100644 --- a/erts/include/internal/i386/ethread.h +++ b/erts/include/internal/i386/ethread.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * Copyright Ericsson AB 2005-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -24,17 +24,15 @@ #ifndef ETHREAD_I386_ETHREAD_H #define ETHREAD_I386_ETHREAD_H +#include "ethr_membar.h" #define ETHR_ATOMIC_WANT_32BIT_IMPL__ #include "atomic.h" #if ETHR_SIZEOF_PTR == 8 # define ETHR_ATOMIC_WANT_64BIT_IMPL__ # include "atomic.h" #endif +#include "ethr_dw_atomic.h" #include "spinlock.h" #include "rwlock.h" -#define ETHR_HAVE_NATIVE_ATOMICS 1 -#define ETHR_HAVE_NATIVE_SPINLOCKS 1 -#define ETHR_HAVE_NATIVE_RWSPINLOCKS 1 - #endif /* ETHREAD_I386_ETHREAD_H */ diff --git a/erts/include/internal/i386/rwlock.h b/erts/include/internal/i386/rwlock.h index be47f459ce..1a8cd7da0c 100644 --- a/erts/include/internal/i386/rwlock.h +++ b/erts/include/internal/i386/rwlock.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * Copyright Ericsson AB 2005-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -26,6 +26,9 @@ #ifndef ETHREAD_I386_RWLOCK_H #define ETHREAD_I386_RWLOCK_H +#define ETHR_HAVE_NATIVE_RWSPINLOCKS 1 +#define ETHR_NATIVE_RWSPINLOCK_IMPL "ethread" + /* XXX: describe the algorithm */ typedef struct { volatile int lock; diff --git a/erts/include/internal/i386/spinlock.h b/erts/include/internal/i386/spinlock.h index 0325324895..a84fba91b1 100644 --- a/erts/include/internal/i386/spinlock.h +++ b/erts/include/internal/i386/spinlock.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * Copyright Ericsson AB 2005-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -24,6 +24,9 @@ #ifndef ETHREAD_I386_SPINLOCK_H #define ETHREAD_I386_SPINLOCK_H +#define ETHR_HAVE_NATIVE_SPINLOCKS 1 +#define ETHR_NATIVE_SPINLOCK_IMPL "ethread" + /* A spinlock is the low byte of an aligned 32-bit integer. * A non-zero value means that the lock is locked. */ @@ -46,16 +49,20 @@ ethr_native_spin_unlock(ethr_native_spinlock_t *lock) * On i386 this needs to be a locked operation * to avoid Pentium Pro errata 66 and 92. */ -#if defined(__x86_64__) || !defined(ETHR_PRE_PENTIUM4_COMPAT) - __asm__ __volatile__("" : : : "memory"); - *(unsigned char*)&lock->lock = 0; -#else - char tmp = 0; - __asm__ __volatile__( - "xchgb %b0, %1" - : "=q"(tmp), "=m"(lock->lock) - : "0"(tmp) : "memory"); +#if !defined(__x86_64__) + if (ETHR_X86_RUNTIME_CONF_HAVE_NO_SSE2__) { + char tmp = 0; + __asm__ __volatile__( + "xchgb %b0, %1" + : "=q"(tmp), "=m"(lock->lock) + : "0"(tmp) : "memory"); + } + else #endif + { + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + *(unsigned char*)&lock->lock = 0; + } } static ETHR_INLINE int diff --git a/erts/include/internal/libatomic_ops/ethr_atomic.h b/erts/include/internal/libatomic_ops/ethr_atomic.h index 93bc06036f..fb1288c330 100644 --- a/erts/include/internal/libatomic_ops/ethr_atomic.h +++ b/erts/include/internal/libatomic_ops/ethr_atomic.h @@ -25,16 +25,6 @@ #ifndef ETHR_LIBATOMIC_OPS_ATOMIC_H__ #define ETHR_LIBATOMIC_OPS_ATOMIC_H__ -#if !defined(ETHR_HAVE_NATIVE_ATOMICS) && defined(ETHR_HAVE_LIBATOMIC_OPS) -#define ETHR_HAVE_NATIVE_ATOMICS 1 - -#if (defined(__i386__) && !defined(ETHR_PRE_PENTIUM4_COMPAT)) \ - || defined(__x86_64__) -#define AO_USE_PENTIUM4_INSTRS -#endif - -#include "atomic_ops.h" - /* * libatomic_ops can be downloaded from: * http://www.hpl.hp.com/research/linux/atomic_ops/ @@ -46,21 +36,18 @@ * - AO_store() * - AO_compare_and_swap() * - * The `AO_t' type also have to be at least as large as the `void *' type. */ -#if ETHR_SIZEOF_AO_T < ETHR_SIZEOF_PTR -#error The AO_t type is too small -#endif - #if ETHR_SIZEOF_AO_T == 4 #define ETHR_HAVE_NATIVE_ATOMIC32 1 +#define ETHR_NATIVE_ATOMIC32_IMPL "libatomic_ops" #define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X #define ETHR_ATMC_T__ ethr_native_atomic32_t #define ETHR_AINT_T__ ethr_sint32_t #define ETHR_AINT_SUFFIX__ "l" #elif ETHR_SIZEOF_AO_T == 8 #define ETHR_HAVE_NATIVE_ATOMIC64 1 +#define ETHR_NATIVE_ATOMIC64_IMPL "libatomic_ops" #define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X #define ETHR_ATMC_T__ ethr_native_atomic64_t #define ETHR_AINT_T__ ethr_sint64_t @@ -69,61 +56,29 @@ #error "Unsupported integer size" #endif -#if ETHR_SIZEOF_AO_T == 8 -typedef union { - volatile AO_t counter; - ethr_sint32_t sint32[2]; -} ETHR_ATMC_T__; -#else typedef struct { volatile AO_t counter; } ETHR_ATMC_T__; -#endif -#define ETHR_MEMORY_BARRIER AO_nop_full() -#ifdef AO_HAVE_nop_write -# define ETHR_WRITE_MEMORY_BARRIER AO_nop_write() -#else -# define ETHR_WRITE_MEMORY_BARRIER ETHR_MEMORY_BARRIER -#endif -#ifdef AO_HAVE_nop_read -# define ETHR_READ_MEMORY_BARRIER AO_nop_read() -#else -# define ETHR_READ_MEMORY_BARRIER ETHR_MEMORY_BARRIER -#endif -#ifdef AO_NO_DD_ORDERING -# define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_READ_MEMORY_BARRIER +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) + +#if ETHR_SIZEOF_AO_T == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADDR 1 #else -# define ETHR_READ_DEPEND_MEMORY_BARRIER AO_compiler_barrier() +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADDR 1 #endif -#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) - static ETHR_INLINE ETHR_AINT_T__ * ETHR_NATMC_FUNC__(addr)(ETHR_ATMC_T__ *var) { return (ETHR_AINT_T__ *) &var->counter; } -#if ETHR_SIZEOF_AO_T == 8 -/* - * We also need to provide an ethr_native_atomic32_addr(), since - * this 64-bit implementation will be used implementing 32-bit - * native atomics. - */ - -static ETHR_INLINE ethr_sint32_t * -ethr_native_atomic32_addr(ETHR_ATMC_T__ *var) -{ - ETHR_ASSERT(((void *) &var->sint32[0]) == ((void *) &var->counter)); -#ifdef ETHR_BIGENDIAN - return &var->sint32[1]; +#if ETHR_SIZEOF_AO_T == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET 1 #else - return &var->sint32[0]; +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET 1 #endif -} - -#endif /* ETHR_SIZEOF_AO_T == 8 */ static ETHR_INLINE void ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value) @@ -131,197 +86,198 @@ ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value) AO_store(&var->counter, (AO_t) value); } +#ifdef AO_HAVE_store_release + +#if ETHR_SIZEOF_AO_T == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_RELB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RELB 1 +#endif + static ETHR_INLINE void -ETHR_NATMC_FUNC__(init)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value) +ETHR_NATMC_FUNC__(set_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value) { - ETHR_NATMC_FUNC__(set)(var, value); + AO_store_release(&var->counter, (AO_t) value); } +#endif + +#if ETHR_SIZEOF_AO_T == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ 1 +#endif + static ETHR_INLINE ETHR_AINT_T__ ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var) { return (ETHR_AINT_T__) AO_load(&var->counter); } -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(add_return)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) -{ -#ifdef AO_HAVE_fetch_and_add_full - return ((ETHR_AINT_T__) AO_fetch_and_add_full(&var->counter, (AO_t) incr)) + incr; +#ifdef AO_HAVE_load_acquire + +#if ETHR_SIZEOF_AO_T == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_ACQB 1 #else - while (1) { - AO_t exp = AO_load(&var->counter); - AO_t new = exp + (AO_t) incr; - if (AO_compare_and_swap_full(&var->counter, exp, new)) - return (ETHR_AINT_T__) new; - } +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_ACQB 1 #endif -} -static ETHR_INLINE void -ETHR_NATMC_FUNC__(add)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(read_acqb)(ETHR_ATMC_T__ *var) { - (void) ETHR_NATMC_FUNC__(add_return)(var, incr); + return (ETHR_AINT_T__) AO_load_acquire(&var->counter); } -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(inc_return)(ETHR_ATMC_T__ *var) -{ -#ifdef AO_HAVE_fetch_and_add1_full - return ((ETHR_AINT_T__) AO_fetch_and_add1_full(&var->counter)) + 1; -#else - return ETHR_NATMC_FUNC__(add_return)(var, 1); #endif -} -static ETHR_INLINE void -ETHR_NATMC_FUNC__(inc)(ETHR_ATMC_T__ *var) -{ - (void) ETHR_NATMC_FUNC__(inc_return)(var); -} +#ifdef AO_HAVE_fetch_and_add -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(dec_return)(ETHR_ATMC_T__ *var) -{ -#ifdef AO_HAVE_fetch_and_sub1_full - return ((ETHR_AINT_T__) AO_fetch_and_sub1_full(&var->counter)) - 1; +#if ETHR_SIZEOF_AO_T == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN 1 #else - return ETHR_NATMC_FUNC__(add_return)(var, -1); +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN 1 #endif -} - -static ETHR_INLINE void -ETHR_NATMC_FUNC__(dec)(ETHR_ATMC_T__ *var) -{ - (void) ETHR_NATMC_FUNC__(dec_return)(var); -} static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(and_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) +ETHR_NATMC_FUNC__(add_return)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) { - while (1) { - AO_t exp = AO_load(&var->counter); - AO_t new = exp & ((AO_t) mask); - if (AO_compare_and_swap_full(&var->counter, exp, new)) - return (ETHR_AINT_T__) exp; - } + return ((ETHR_AINT_T__) AO_fetch_and_add(&var->counter, (AO_t) incr)) + incr; } -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(or_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) -{ - while (1) { - AO_t exp = AO_load(&var->counter); - AO_t new = exp | ((AO_t) mask); - if (AO_compare_and_swap_full(&var->counter, exp, new)) - return (ETHR_AINT_T__) exp; - } -} +#endif -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var, - ETHR_AINT_T__ new, - ETHR_AINT_T__ exp) -{ - ETHR_AINT_T__ act; - do { - if (AO_compare_and_swap_full(&var->counter, (AO_t) exp, (AO_t) new)) - return exp; - act = (ETHR_AINT_T__) AO_load(&var->counter); - } while (act == exp); - return act; -} +#ifdef AO_HAVE_fetch_and_add1 + +#if ETHR_SIZEOF_AO_T == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN 1 +#endif static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(xchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new) +ETHR_NATMC_FUNC__(inc_return)(ETHR_ATMC_T__ *var) { - while (1) { - AO_t exp = AO_load(&var->counter); - if (AO_compare_and_swap_full(&var->counter, exp, (AO_t) new)) - return (ETHR_AINT_T__) exp; - } + return ((ETHR_AINT_T__) AO_fetch_and_add1(&var->counter)) + 1; } -/* - * Atomic ops with at least specified barriers. - */ +#endif -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(read_acqb)(ETHR_ATMC_T__ *var) -{ -#ifdef AO_HAVE_load_acquire - return (ETHR_AINT_T__) AO_load_acquire(&var->counter); +#ifdef AO_HAVE_fetch_and_add1_acquire + +#if ETHR_SIZEOF_AO_T == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_ACQB 1 #else - ETHR_AINT_T__ res = ETHR_NATMC_FUNC__(read)(var); - ETHR_MEMORY_BARRIER; - return res; +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_ACQB 1 #endif -} static ETHR_INLINE ETHR_AINT_T__ ETHR_NATMC_FUNC__(inc_return_acqb)(ETHR_ATMC_T__ *var) { -#ifdef AO_HAVE_fetch_and_add1_acquire return ((ETHR_AINT_T__) AO_fetch_and_add1_acquire(&var->counter)) + 1; +} + +#endif + +#ifdef AO_HAVE_fetch_and_sub1 + +#if ETHR_SIZEOF_AO_T == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN 1 #else - ETHR_AINT_T__ res = ETHR_NATMC_FUNC__(add_return)(var, 1); - return res; +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN 1 #endif -} -static ETHR_INLINE void -ETHR_NATMC_FUNC__(set_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(dec_return)(ETHR_ATMC_T__ *var) { -#ifdef AO_HAVE_store_release - AO_store_release(&var->counter, (AO_t) value); + return ((ETHR_AINT_T__) AO_fetch_and_sub1(&var->counter)) - 1; +} + +#endif + +#ifdef AO_HAVE_fetch_and_sub1_release + +#if ETHR_SIZEOF_AO_T == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_RELB 1 #else - ETHR_MEMORY_BARRIER; - ETHR_NATMC_FUNC__(set)(var, value); +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_RELB 1 #endif -} static ETHR_INLINE ETHR_AINT_T__ ETHR_NATMC_FUNC__(dec_return_relb)(ETHR_ATMC_T__ *var) { -#ifdef AO_HAVE_fetch_and_sub1_release return ((ETHR_AINT_T__) AO_fetch_and_sub1_release(&var->counter)) - 1; +} + +#endif + +#ifdef AO_HAVE_compare_and_swap + +#if ETHR_SIZEOF_AO_T == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG 1 #else - return ETHR_NATMC_FUNC__(dec_return)(var); +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG 1 #endif -} -static ETHR_INLINE void -ETHR_NATMC_FUNC__(dec_relb)(ETHR_ATMC_T__ *var) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var, + ETHR_AINT_T__ new, + ETHR_AINT_T__ exp) { - (void) ETHR_NATMC_FUNC__(dec_return_relb)(var); + ETHR_AINT_T__ act; + do { + if (AO_compare_and_swap(&var->counter, (AO_t) exp, (AO_t) new)) + return exp; + act = (ETHR_AINT_T__) AO_load(&var->counter); + } while (act == exp); + return act; } +#endif + +#ifdef AO_HAVE_compare_and_swap_acquire + +#if ETHR_SIZEOF_AO_T == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_ACQB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_ACQB 1 +#endif + static ETHR_INLINE ETHR_AINT_T__ ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new, ETHR_AINT_T__ exp) { -#ifdef AO_HAVE_compare_and_swap_acquire ETHR_AINT_T__ act; do { if (AO_compare_and_swap_acquire(&var->counter, (AO_t) exp, (AO_t) new)) return exp; +#ifdef AO_HAVE_load_acquire + act = (ETHR_AINT_T__) AO_load_acquire(&var->counter); +#else act = (ETHR_AINT_T__) AO_load(&var->counter); +#endif } while (act == exp); +#ifndef AO_HAVE_load_acquire AO_nop_full(); +#endif return act; +} + +#endif + +#ifdef AO_HAVE_compare_and_swap_release + +#if ETHR_SIZEOF_AO_T == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_RELB 1 #else - ETHR_AINT_T__ act = ETHR_NATMC_FUNC__(cmpxchg)(var, new, exp); - return act; +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RELB 1 #endif -} static ETHR_INLINE ETHR_AINT_T__ ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new, ETHR_AINT_T__ exp) { -#ifdef AO_HAVE_compare_and_swap_release ETHR_AINT_T__ act; do { if (AO_compare_and_swap_release(&var->counter, (AO_t) exp, (AO_t) new)) @@ -329,11 +285,9 @@ ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var, act = (ETHR_AINT_T__) AO_load(&var->counter); } while (act == exp); return act; -#else - return ETHR_NATMC_FUNC__(cmpxchg)(var, new, exp); -#endif } +#endif #endif /* ETHR_TRY_INLINE_FUNCS */ @@ -341,6 +295,4 @@ ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var, #undef ETHR_ATMC_T__ #undef ETHR_AINT_T__ -#endif /* !defined(ETHR_HAVE_NATIVE_ATOMICS) && defined(ETHR_HAVE_LIBATOMIC_OPS) */ - #endif /* ETHR_LIBATOMIC_OPS_ATOMIC_H__ */ diff --git a/erts/include/internal/libatomic_ops/ethr_membar.h b/erts/include/internal/libatomic_ops/ethr_membar.h new file mode 100644 index 0000000000..b8530a0094 --- /dev/null +++ b/erts/include/internal/libatomic_ops/ethr_membar.h @@ -0,0 +1,75 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2011. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Memory barriers when using libatomic_ops + * Author: Rickard Green + */ + +#ifndef ETHR_LIBATOMIC_OPS_MEMBAR_H__ +#define ETHR_LIBATOMIC_OPS_MEMBAR_H__ + +#define ETHR_LoadLoad (1 << 0) +#define ETHR_LoadStore (1 << 1) +#define ETHR_StoreLoad (1 << 2) +#define ETHR_StoreStore (1 << 3) + +#ifndef AO_HAVE_nop_full +# error "No AO_nop_full()" +#endif + +static __inline__ void +ethr_mb__(void) +{ + AO_nop_full(); +} + +static __inline__ void +ethr_rb__(void) +{ +#ifdef AO_HAVE_nop_read + AO_nop_read(); +#else + AO_nop_full(); +#endif +} + +static __inline__ void +ethr_wb__(void) +{ +#ifdef AO_HAVE_nop_write + AO_nop_write(); +#else + AO_nop_full(); +#endif +} + +#define ETHR_MEMBAR(B) \ + ETHR_CHOOSE_EXPR((B) == ETHR_StoreStore, \ + ethr_wb__(), \ + ETHR_CHOOSE_EXPR((B) == ETHR_LoadLoad, \ + ethr_rb__(), \ + ethr_mb__())) + +#define ETHR_COMPILER_BARRIER AO_compiler_barrier() +#ifdef AO_NO_DD_ORDERING +# define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_READ_MEMORY_BARRIER +#endif + +#endif /* ETHR_LIBATOMIC_OPS_MEMBAR_H__ */ diff --git a/erts/include/internal/libatomic_ops/ethread.h b/erts/include/internal/libatomic_ops/ethread.h index ee73ba73bc..e1fdd588bb 100644 --- a/erts/include/internal/libatomic_ops/ethread.h +++ b/erts/include/internal/libatomic_ops/ethread.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2010. All Rights Reserved. + * Copyright Ericsson AB 2010-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -25,6 +25,18 @@ #ifndef ETHREAD_LIBATOMIC_OPS_H__ #define ETHREAD_LIBATOMIC_OPS_H__ +#if (defined(ETHR_HAVE_LIBATOMIC_OPS) \ + && ((ETHR_SIZEOF_AO_T == 4 && !defined(ETHR_HAVE_NATIVE_ATOMIC32)) \ + || (ETHR_SIZEOF_AO_T == 8 && !defined(ETHR_HAVE_NATIVE_ATOMIC64)))) + +#if defined(__x86_64__) +#define AO_USE_PENTIUM4_INSTRS +#endif + +#include "atomic_ops.h" +#include "ethr_membar.h" #include "ethr_atomic.h" #endif + +#endif diff --git a/erts/include/internal/ppc32/atomic.h b/erts/include/internal/ppc32/atomic.h index 522f433649..6001620677 100644 --- a/erts/include/internal/ppc32/atomic.h +++ b/erts/include/internal/ppc32/atomic.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * Copyright Ericsson AB 2005-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -29,27 +29,31 @@ #define ETHREAD_PPC_ATOMIC_H #define ETHR_HAVE_NATIVE_ATOMIC32 1 +#define ETHR_NATIVE_ATOMIC32_IMPL "ethread" typedef struct { volatile ethr_sint32_t counter; } ethr_native_atomic32_t; -#define ETHR_MEMORY_BARRIER __asm__ __volatile__("sync" : : : "memory") - #if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADDR 1 + static ETHR_INLINE ethr_sint32_t * ethr_native_atomic32_addr(ethr_native_atomic32_t *var) { return (ethr_sint32_t *) &var->counter; } +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET 1 + static ETHR_INLINE void -ethr_native_atomic32_init(ethr_native_atomic32_t *var, ethr_sint32_t i) +ethr_native_atomic32_set(ethr_native_atomic32_t *var, ethr_sint32_t i) { var->counter = i; } -#define ethr_native_atomic32_set(v, i) ethr_native_atomic32_init((v), (i)) + +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ 1 static ETHR_INLINE ethr_sint32_t ethr_native_atomic32_read(ethr_native_atomic32_t *var) @@ -57,57 +61,68 @@ ethr_native_atomic32_read(ethr_native_atomic32_t *var) return var->counter; } +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN 1 + static ETHR_INLINE ethr_sint32_t ethr_native_atomic32_add_return(ethr_native_atomic32_t *var, ethr_sint32_t incr) { ethr_sint32_t tmp; __asm__ __volatile__( - "eieio\n\t" "1:\t" "lwarx %0,0,%1\n\t" "add %0,%2,%0\n\t" "stwcx. %0,0,%1\n\t" "bne- 1b\n\t" - "isync" : "=&r"(tmp) : "r"(&var->counter), "r"(incr) : "cc", "memory"); return tmp; } -static ETHR_INLINE void -ethr_native_atomic32_add(ethr_native_atomic32_t *var, ethr_sint32_t incr) +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_ACQB 1 + +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_add_return_acqb(ethr_native_atomic32_t *var, ethr_sint32_t incr) { - /* XXX: could use weaker version here w/o eieio+isync */ - (void)ethr_native_atomic32_add_return(var, incr); + ethr_sint32_t res; + res = ethr_native_atomic32_add_return(var, incr); + __asm__ __volatile("isync\n\t" : : : "memory"); + return res; } +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN 1 + static ETHR_INLINE ethr_sint32_t ethr_native_atomic32_inc_return(ethr_native_atomic32_t *var) { ethr_sint32_t tmp; __asm__ __volatile__( - "eieio\n\t" "1:\t" "lwarx %0,0,%1\n\t" "addic %0,%0,1\n\t" /* due to addi's (rA|0) behaviour */ "stwcx. %0,0,%1\n\t" "bne- 1b\n\t" - "isync" : "=&r"(tmp) : "r"(&var->counter) : "cc", "memory"); return tmp; } -static ETHR_INLINE void -ethr_native_atomic32_inc(ethr_native_atomic32_t *var) +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_ACQB 1 + +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_inc_return_acqb(ethr_native_atomic32_t *var) { - /* XXX: could use weaker version here w/o eieio+isync */ - (void)ethr_native_atomic32_inc_return(var); + ethr_sint32_t res; + res = ethr_native_atomic32_inc_return(var); + __asm__ __volatile("isync\n\t" : : : "memory"); + return res; } + + +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN 1 static ETHR_INLINE ethr_sint32_t ethr_native_atomic32_dec_return(ethr_native_atomic32_t *var) @@ -115,82 +130,120 @@ ethr_native_atomic32_dec_return(ethr_native_atomic32_t *var) ethr_sint32_t tmp; __asm__ __volatile__( - "eieio\n\t" "1:\t" "lwarx %0,0,%1\n\t" "addic %0,%0,-1\n\t" "stwcx. %0,0,%1\n\t" "bne- 1b\n\t" - "isync" : "=&r"(tmp) : "r"(&var->counter) : "cc", "memory"); return tmp; } -static ETHR_INLINE void -ethr_native_atomic32_dec(ethr_native_atomic32_t *var) +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_ACQB 1 + +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_dec_return_acqb(ethr_native_atomic32_t *var) { - /* XXX: could use weaker version here w/o eieio+isync */ - (void)ethr_native_atomic32_dec_return(var); + ethr_sint32_t res; + res = ethr_native_atomic32_dec_return(var); + __asm__ __volatile("isync\n\t" : : : "memory"); + return res; } +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD 1 + static ETHR_INLINE ethr_sint32_t ethr_native_atomic32_and_retold(ethr_native_atomic32_t *var, ethr_sint32_t mask) { ethr_sint32_t old, new; __asm__ __volatile__( - "eieio\n\t" "1:\t" "lwarx %0,0,%2\n\t" "and %1,%0,%3\n\t" "stwcx. %1,0,%2\n\t" "bne- 1b\n\t" - "isync" : "=&r"(old), "=&r"(new) : "r"(&var->counter), "r"(mask) : "cc", "memory"); return old; } +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_ACQB 1 + +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_and_retold_acqb(ethr_native_atomic32_t *var, ethr_sint32_t mask) +{ + ethr_sint32_t res; + res = ethr_native_atomic32_and_retold(var, mask); + __asm__ __volatile("isync\n\t" : : : "memory"); + return res; +} + +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD 1 + static ETHR_INLINE ethr_sint32_t ethr_native_atomic32_or_retold(ethr_native_atomic32_t *var, ethr_sint32_t mask) { ethr_sint32_t old, new; __asm__ __volatile__( - "eieio\n\t" "1:\t" "lwarx %0,0,%2\n\t" "or %1,%0,%3\n\t" "stwcx. %1,0,%2\n\t" "bne- 1b\n\t" - "isync" : "=&r"(old), "=&r"(new) : "r"(&var->counter), "r"(mask) : "cc", "memory"); return old; } +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_ACQB 1 + +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_or_retold_acqb(ethr_native_atomic32_t *var, ethr_sint32_t mask) +{ + ethr_sint32_t res; + res = ethr_native_atomic32_or_retold(var, mask); + __asm__ __volatile("isync\n\t" : : : "memory"); + return res; +} + + +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG 1 + static ETHR_INLINE ethr_sint32_t ethr_native_atomic32_xchg(ethr_native_atomic32_t *var, ethr_sint32_t val) { ethr_sint32_t tmp; __asm__ __volatile__( - "eieio\n\t" "1:\t" "lwarx %0,0,%1\n\t" "stwcx. %2,0,%1\n\t" "bne- 1b\n\t" - "isync" : "=&r"(tmp) : "r"(&var->counter), "r"(val) : "cc", "memory"); return tmp; } +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_ACQB 1 + +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_xchg_acqb(ethr_native_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; + res = ethr_native_atomic32_xchg(var, val); + __asm__ __volatile("isync\n\t" : : : "memory"); + return res; +} + +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG 1 + static ETHR_INLINE ethr_sint32_t ethr_native_atomic32_cmpxchg(ethr_native_atomic32_t *var, ethr_sint32_t new, @@ -199,14 +252,12 @@ ethr_native_atomic32_cmpxchg(ethr_native_atomic32_t *var, ethr_sint32_t old; __asm__ __volatile__( - "eieio\n\t" "1:\t" "lwarx %0,0,%2\n\t" "cmpw 0,%0,%3\n\t" "bne 2f\n\t" "stwcx. %1,0,%2\n\t" "bne- 1b\n\t" - "isync\n" "2:" : "=&r"(old) : "r"(new), "r"(&var->counter), "r"(expected) @@ -215,25 +266,30 @@ ethr_native_atomic32_cmpxchg(ethr_native_atomic32_t *var, return old; } -/* - * Atomic ops with at least specified barriers. - */ +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_ACQB 1 -static ETHR_INLINE long -ethr_native_atomic32_read_acqb(ethr_native_atomic32_t *var) +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_cmpxchg_acqb(ethr_native_atomic32_t *var, + ethr_sint32_t new, + ethr_sint32_t expected) { - long res = ethr_native_atomic32_read(var); - ETHR_MEMORY_BARRIER; - return res; -} + ethr_sint32_t old; -#define ethr_native_atomic32_set_relb ethr_native_atomic32_xchg -#define ethr_native_atomic32_inc_return_acqb ethr_native_atomic32_inc_return -#define ethr_native_atomic32_dec_relb ethr_native_atomic32_dec_return -#define ethr_native_atomic32_dec_return_relb ethr_native_atomic32_dec_return + __asm__ __volatile__( + "1:\t" + "lwarx %0,0,%2\n\t" + "cmpw 0,%0,%3\n\t" + "bne 2f\n\t" + "stwcx. %1,0,%2\n\t" + "bne- 1b\n\t" + "isync\n" + "2:" + : "=&r"(old) + : "r"(new), "r"(&var->counter), "r"(expected) + : "cc", "memory"); -#define ethr_native_atomic32_cmpxchg_acqb ethr_native_atomic32_cmpxchg -#define ethr_native_atomic32_cmpxchg_relb ethr_native_atomic32_cmpxchg + return old; +} #endif /* ETHR_TRY_INLINE_FUNCS */ diff --git a/erts/include/internal/ppc32/ethr_membar.h b/erts/include/internal/ppc32/ethr_membar.h new file mode 100644 index 0000000000..ff5cc86bfb --- /dev/null +++ b/erts/include/internal/ppc32/ethr_membar.h @@ -0,0 +1,63 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2011. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Memory barriers for PowerPC + * Author: Rickard Green + */ + +#ifndef ETHR_PPC_MEMBAR_H__ +#define ETHR_PPC_MEMBAR_H__ + +#define ETHR_LoadLoad (1 << 0) +#define ETHR_LoadStore (1 << 1) +#define ETHR_StoreLoad (1 << 2) +#define ETHR_StoreStore (1 << 3) + +static __inline__ void +ethr_lwsync__(void) +{ +#ifdef ETHR_PPC_HAVE_NO_LWSYNC + __asm__ __volatile__ ("sync\n\t" : : : "memory"); +#else +#ifndef ETHR_PPC_HAVE_LWSYNC + if (ETHR_PPC_RUNTIME_CONF_HAVE_NO_LWSYNC__) + __asm__ __volatile__ ("sync\n\t" : : : "memory"); + else +#endif + __asm__ __volatile__ ("lwsync\n\t" : : : "memory"); +#endif +} + +static __inline__ void +ethr_sync__(void) +{ + __asm__ __volatile__ ("sync\n\t" : : : "memory"); +} + +/* + * According to the "memory barrier intstructions" section of + * http://www.ibm.com/developerworks/systems/articles/powerpc.html + * we want to use sync when a StoreLoad is needed and lwsync for + * everything else. + */ +#define ETHR_MEMBAR(B) \ + ETHR_CHOOSE_EXPR((B) & ETHR_StoreLoad, ethr_sync__(), ethr_lwsync__()) + +#endif diff --git a/erts/include/internal/ppc32/ethread.h b/erts/include/internal/ppc32/ethread.h index 3b619e9d01..e41c83c5da 100644 --- a/erts/include/internal/ppc32/ethread.h +++ b/erts/include/internal/ppc32/ethread.h @@ -24,12 +24,9 @@ #ifndef ETHREAD_PPC32_ETHREAD_H #define ETHREAD_PPC32_ETHREAD_H +#include "ethr_membar.h" #include "atomic.h" #include "spinlock.h" #include "rwlock.h" -#define ETHR_HAVE_NATIVE_ATOMICS 1 -#define ETHR_HAVE_NATIVE_SPINLOCKS 1 -#define ETHR_HAVE_NATIVE_RWSPINLOCKS 1 - #endif /* ETHREAD_PPC32_ETHREAD_H */ diff --git a/erts/include/internal/ppc32/rwlock.h b/erts/include/internal/ppc32/rwlock.h index 19ec26ab68..311f000b69 100644 --- a/erts/include/internal/ppc32/rwlock.h +++ b/erts/include/internal/ppc32/rwlock.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * Copyright Ericsson AB 2005-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -23,12 +23,14 @@ * * Based on the examples in Appendix E of Motorola's * "Programming Environments Manual For 32-Bit Implementations - * of the PowerPC Architecture". Uses eieio instead of sync - * in the unlock sequence, as suggested in the manual. + * of the PowerPC Architecture". */ #ifndef ETHREAD_PPC_RWLOCK_H #define ETHREAD_PPC_RWLOCK_H +#define ETHR_HAVE_NATIVE_RWSPINLOCKS 1 +#define ETHR_NATIVE_RWSPINLOCK_IMPL "ethread" + /* Unlocked if zero, read-locked if negative, write-locked if +1. */ typedef struct { volatile int lock; @@ -47,9 +49,10 @@ ethr_native_read_unlock(ethr_native_rwlock_t *lock) { int tmp; - /* this is eieio + ethr_native_atomic_inc() - isync */ + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + + /* this is ethr_native_atomic_inc() - isync */ __asm__ __volatile__( - "eieio\n\t" "1:\t" "lwarx %0,0,%1\n\t" "addic %0,%0,1\n\t" @@ -105,7 +108,7 @@ ethr_native_read_lock(ethr_native_rwlock_t *lock) static ETHR_INLINE void ethr_native_write_unlock(ethr_native_rwlock_t *lock) { - __asm__ __volatile__("eieio" : : : "memory"); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); lock->lock = 0; } diff --git a/erts/include/internal/ppc32/spinlock.h b/erts/include/internal/ppc32/spinlock.h index c8460a3e8a..4c95ec9efb 100644 --- a/erts/include/internal/ppc32/spinlock.h +++ b/erts/include/internal/ppc32/spinlock.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * Copyright Ericsson AB 2005-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -23,12 +23,14 @@ * * Based on the examples in Appendix E of Motorola's * "Programming Environments Manual For 32-Bit Implementations - * of the PowerPC Architecture". Uses eieio instead of sync - * in the unlock sequence, as suggested in the manual. + * of the PowerPC Architecture". */ #ifndef ETHREAD_PPC_SPINLOCK_H #define ETHREAD_PPC_SPINLOCK_H +#define ETHR_HAVE_NATIVE_SPINLOCKS 1 +#define ETHR_NATIVE_SPINLOCK_IMPL "ethread" + /* Unlocked if zero, locked if non-zero. */ typedef struct { volatile unsigned int lock; @@ -45,7 +47,7 @@ ethr_native_spinlock_init(ethr_native_spinlock_t *lock) static ETHR_INLINE void ethr_native_spin_unlock(ethr_native_spinlock_t *lock) { - __asm__ __volatile__("eieio" : : : "memory"); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); lock->lock = 0; } diff --git a/erts/include/internal/pthread/ethr_event.h b/erts/include/internal/pthread/ethr_event.h index 4c29b28536..d0a77990cc 100644 --- a/erts/include/internal/pthread/ethr_event.h +++ b/erts/include/internal/pthread/ethr_event.h @@ -21,7 +21,7 @@ * Author: Rickard Green */ -#if defined(ETHR_HAVE_LINUX_FUTEX) && defined(ETHR_HAVE_NATIVE_ATOMICS) +#if defined(ETHR_HAVE_LINUX_FUTEX) && defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) /* --- Linux futex implementation of ethread events ------------------------- */ #define ETHR_LINUX_FUTEX_IMPL__ @@ -62,8 +62,7 @@ static void ETHR_INLINE ETHR_INLINE_FUNC_NAME_(ethr_event_set)(ethr_event *e) { ethr_sint32_t val; - ETHR_MEMORY_BARRIER; - val = ethr_atomic32_xchg(&e->futex, ETHR_EVENT_ON__); + val = ethr_atomic32_xchg_mb(&e->futex, ETHR_EVENT_ON__); if (val == ETHR_EVENT_OFF_WAITER__) { int res = ETHR_FUTEX__(&e->futex, ETHR_FUTEX_WAKE__, 1); if (res != 0) @@ -99,8 +98,7 @@ static void ETHR_INLINE ETHR_INLINE_FUNC_NAME_(ethr_event_set)(ethr_event *e) { ethr_sint32_t val; - ETHR_MEMORY_BARRIER; - val = ethr_atomic32_xchg(&e->state, ETHR_EVENT_ON__); + val = ethr_atomic32_xchg_mb(&e->state, ETHR_EVENT_ON__); if (val == ETHR_EVENT_OFF_WAITER__) { int res = pthread_mutex_lock(&e->mtx); if (res != 0) diff --git a/erts/include/internal/sparc32/atomic.h b/erts/include/internal/sparc32/atomic.h index c297522ab1..fe1daaa9cf 100644 --- a/erts/include/internal/sparc32/atomic.h +++ b/erts/include/internal/sparc32/atomic.h @@ -35,23 +35,16 @@ #ifdef ETHR_INCLUDE_ATOMIC_IMPL__ -#ifndef ETHR_SPARC_V9_ATOMIC_COMMON__ -#define ETHR_SPARC_V9_ATOMIC_COMMON__ - -#define ETHR_MEMORY_BARRIER \ - __asm__ __volatile__("membar #LoadLoad|#LoadStore|#StoreLoad|#StoreStore\n" \ - : : : "memory") - -#endif /* ETHR_SPARC_V9_ATOMIC_COMMON__ */ - #if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 #define ETHR_HAVE_NATIVE_ATOMIC32 1 +#define ETHR_NATIVE_ATOMIC32_IMPL "ethread" #define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X #define ETHR_ATMC_T__ ethr_native_atomic32_t #define ETHR_AINT_T__ ethr_sint32_t #define ETHR_CAS__ "cas" #elif ETHR_INCLUDE_ATOMIC_IMPL__ == 8 #define ETHR_HAVE_NATIVE_ATOMIC64 1 +#define ETHR_NATIVE_ATOMIC64_IMPL "ethread" #define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X #define ETHR_ATMC_T__ ethr_native_atomic64_t #define ETHR_AINT_T__ ethr_sint64_t @@ -66,17 +59,23 @@ typedef struct { #if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADDR 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADDR 1 +#endif + static ETHR_INLINE ETHR_AINT_T__ * ETHR_NATMC_FUNC__(addr)(ETHR_ATMC_T__ *var) { return (ETHR_AINT_T__ *) &var->counter; } -static ETHR_INLINE void -ETHR_NATMC_FUNC__(init)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) -{ - var->counter = i; -} +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET 1 +#endif static ETHR_INLINE void ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) @@ -84,182 +83,49 @@ ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) var->counter = i; } +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ 1 +#endif + static ETHR_INLINE ETHR_AINT_T__ ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var) { return var->counter; } -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(add_return)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) -{ - ETHR_AINT_T__ old, tmp; - - __asm__ __volatile__("membar #LoadLoad|#StoreLoad\n" : : : "memory"); - do { - old = var->counter; - tmp = old+incr; - __asm__ __volatile__( - ETHR_CAS__ " [%2], %1, %0" - : "=&r"(tmp) - : "r"(old), "r"(&var->counter), "0"(tmp) - : "memory"); - } while (__builtin_expect(old != tmp, 0)); - __asm__ __volatile__("membar #StoreLoad|#StoreStore" : : : "memory"); - return old+incr; -} - -static ETHR_INLINE void -ETHR_NATMC_FUNC__(add)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) -{ - (void)ETHR_NATMC_FUNC__(add_return)(var, incr); -} - -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(inc_return)(ETHR_ATMC_T__ *var) -{ - return ETHR_NATMC_FUNC__(add_return)(var, 1); -} - -static ETHR_INLINE void -ETHR_NATMC_FUNC__(inc)(ETHR_ATMC_T__ *var) -{ - (void)ETHR_NATMC_FUNC__(add_return)(var, 1); -} - -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(dec_return)(ETHR_ATMC_T__ *var) -{ - return ETHR_NATMC_FUNC__(add_return)(var, -1); -} - -static ETHR_INLINE void -ETHR_NATMC_FUNC__(dec)(ETHR_ATMC_T__ *var) -{ - (void)ETHR_NATMC_FUNC__(add_return)(var, -1); -} - -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(and_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) -{ - ETHR_AINT_T__ old, tmp; - - __asm__ __volatile__("membar #LoadLoad|#StoreLoad\n" : : : "memory"); - do { - old = var->counter; - tmp = old & mask; - __asm__ __volatile__( - ETHR_CAS__ " [%2], %1, %0" - : "=&r"(tmp) - : "r"(old), "r"(&var->counter), "0"(tmp) - : "memory"); - } while (__builtin_expect(old != tmp, 0)); - __asm__ __volatile__("membar #StoreLoad|#StoreStore" : : : "memory"); - return old; -} - -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(or_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) -{ - ETHR_AINT_T__ old, tmp; - - __asm__ __volatile__("membar #LoadLoad|#StoreLoad\n" : : : "memory"); - do { - old = var->counter; - tmp = old | mask; - __asm__ __volatile__( - ETHR_CAS__ " [%2], %1, %0" - : "=&r"(tmp) - : "r"(old), "r"(&var->counter), "0"(tmp) - : "memory"); - } while (__builtin_expect(old != tmp, 0)); - __asm__ __volatile__("membar #StoreLoad|#StoreStore" : : : "memory"); - return old; -} - -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(xchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ val) -{ - ETHR_AINT_T__ old, new; - - __asm__ __volatile__("membar #LoadLoad|#StoreLoad" : : : "memory"); - do { - old = var->counter; - new = val; - __asm__ __volatile__( - ETHR_CAS__ " [%2], %1, %0" - : "=&r"(new) - : "r"(old), "r"(&var->counter), "0"(new) - : "memory"); - } while (__builtin_expect(old != new, 0)); - __asm__ __volatile__("membar #StoreLoad|#StoreStore" : : : "memory"); - return old; -} +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG 1 +#endif static ETHR_INLINE ETHR_AINT_T__ ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new, ETHR_AINT_T__ old) { - __asm__ __volatile__("membar #LoadLoad|#StoreLoad\n" : : : "memory"); __asm__ __volatile__( ETHR_CAS__ " [%2], %1, %0" : "=&r"(new) : "r"(old), "r"(&var->counter), "0"(new) : "memory"); - __asm__ __volatile__("membar #StoreLoad|#StoreStore" : : : "memory"); return new; } -/* - * Atomic ops with at least specified barriers. - */ - -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(read_acqb)(ETHR_ATMC_T__ *var) -{ - ETHR_AINT_T__ res = ETHR_NATMC_FUNC__(read)(var); - __asm__ __volatile__("membar #LoadLoad|#LoadStore" : : : "memory"); - return res; -} - -static ETHR_INLINE void -ETHR_NATMC_FUNC__(set_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) -{ - __asm__ __volatile__("membar #LoadStore|#StoreStore" : : : "memory"); - ETHR_NATMC_FUNC__(set)(var, i); -} - -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(inc_return_acqb)(ETHR_ATMC_T__ *var) -{ - ETHR_AINT_T__ res = ETHR_NATMC_FUNC__(inc_return)(var); - return res; -} - -static ETHR_INLINE void -ETHR_NATMC_FUNC__(dec_relb)(ETHR_ATMC_T__ *var) -{ - ETHR_NATMC_FUNC__(dec)(var); -} - -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(dec_return_relb)(ETHR_ATMC_T__ *var) -{ - return ETHR_NATMC_FUNC__(dec_return)(var); -} +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_ACQB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_ACQB 1 +#endif static ETHR_INLINE ETHR_AINT_T__ ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new, ETHR_AINT_T__ old) { ETHR_AINT_T__ res = ETHR_NATMC_FUNC__(cmpxchg)(var, new, old); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); return res; } -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new, ETHR_AINT_T__ old) -{ - return ETHR_NATMC_FUNC__(cmpxchg)(var, new, old); -} - #endif /* ETHR_TRY_INLINE_FUNCS */ #undef ETHR_NATMC_FUNC__ diff --git a/erts/include/internal/sparc32/ethr_membar.h b/erts/include/internal/sparc32/ethr_membar.h new file mode 100644 index 0000000000..6eb0c5a1d6 --- /dev/null +++ b/erts/include/internal/sparc32/ethr_membar.h @@ -0,0 +1,115 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2011. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Memory barriers for sparc-v9 + * Author: Rickard Green + */ + +#ifndef ETHR_SPARC_V9_MEMBAR_H__ +#define ETHR_SPARC_V9_MEMBAR_H__ + +#if defined(ETHR_SPARC_TSO) +/* --- Total Store Order ------------------------------------------------ */ + +#define ETHR_LoadLoad 0 +#define ETHR_LoadStore 0 +#define ETHR_StoreLoad 1 +#define ETHR_StoreStore 0 + +static __inline__ void +ethr_cb__(void) +{ + __asm__ __volatile__ ("" : : : "memory"); +} + +static __inline__ void +ethr_StoreLoad__(void) +{ + __asm__ __volatile__ ("membar #StoreLoad\n\t" : : : "memory"); +} + + +#define ETHR_MEMBAR(B) \ + ETHR_CHOOSE_EXPR((B), ethr_StoreLoad__(), ethr_cb__()) + +#elif defined(ETHR_SPARC_PSO) +/* --- Partial Store Order ---------------------------------------------- */ + +#define ETHR_LoadLoad 0 +#define ETHR_LoadStore 0 +#define ETHR_StoreLoad (1 << 0) +#define ETHR_StoreStore (1 << 1) + +static __inline__ void +ethr_cb__(void) +{ + __asm__ __volatile__ ("" : : : "memory"); +} + +static __inline__ void +ethr_StoreLoad__(void) +{ + __asm__ __volatile__ ("membar #StoreLoad\n\t" : : : "memory"); +} + +static __inline__ void +ethr_StoreStore__(void) +{ + __asm__ __volatile__ ("membar #StoreStore\n\t" : : : "memory"); +} + +static __inline__ void +ethr_StoreLoad_StoreStore__(void) +{ + __asm__ __volatile__ ("membar #StoreLoad|StoreStore\n\t" : : : "memory"); +} + +#define ETHR_MEMBAR(B) \ + ETHR_CHOOSE_EXPR( \ + (B) == ETHR_StoreLoad, \ + ethr_StoreLoad__(), \ + ETHR_CHOOSE_EXPR( \ + (B) == ETHR_StoreStore, \ + ethr_StoreStore__(), \ + ETHR_CHOOSE_EXPR( \ + (B) == (ETHR_StoreLoad|ETHR_StoreStore), \ + ethr_StoreLoad_StoreStore__(), \ + ethr_cb__()))) + +#elif defined(ETHR_SPARC_RMO) +/* --- Relaxed Memory Order --------------------------------------------- */ + +# define ETHR_LoadLoad #LoadLoad +# define ETHR_LoadStore #LoadStore +# define ETHR_StoreLoad #StoreLoad +# define ETHR_StoreStore #StoreStore + +# define ETHR_MEMBAR_AUX__(B) \ + __asm__ __volatile__("membar " #B "\n\t" : : : "memory") + +# define ETHR_MEMBAR(B) ETHR_MEMBAR_AUX__(B) + +#else + +# error "No memory order defined" + +#endif + +#endif diff --git a/erts/include/internal/sparc32/ethread.h b/erts/include/internal/sparc32/ethread.h index aea9794390..5ad92d3da7 100644 --- a/erts/include/internal/sparc32/ethread.h +++ b/erts/include/internal/sparc32/ethread.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * Copyright Ericsson AB 2005-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -24,6 +24,7 @@ #ifndef ETHREAD_SPARC32_ETHREAD_H #define ETHREAD_SPARC32_ETHREAD_H +#include "ethr_membar.h" #define ETHR_ATOMIC_WANT_32BIT_IMPL__ #include "atomic.h" #if ETHR_SIZEOF_PTR == 8 @@ -33,8 +34,4 @@ #include "spinlock.h" #include "rwlock.h" -#define ETHR_HAVE_NATIVE_ATOMICS 1 -#define ETHR_HAVE_NATIVE_SPINLOCKS 1 -#define ETHR_HAVE_NATIVE_RWSPINLOCKS 1 - #endif /* ETHREAD_SPARC32_ETHREAD_H */ diff --git a/erts/include/internal/sparc32/rwlock.h b/erts/include/internal/sparc32/rwlock.h index 465ec96866..8b6f2e9c57 100644 --- a/erts/include/internal/sparc32/rwlock.h +++ b/erts/include/internal/sparc32/rwlock.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * Copyright Ericsson AB 2005-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -24,6 +24,9 @@ #ifndef ETHREAD_SPARC32_RWLOCK_H #define ETHREAD_SPARC32_RWLOCK_H +#define ETHR_HAVE_NATIVE_RWSPINLOCKS 1 +#define ETHR_NATIVE_RWSPINLOCK_IMPL "ethread" + /* Unlocked if zero, read-locked if positive, write-locked if -1. */ typedef struct { volatile int lock; @@ -42,7 +45,7 @@ ethr_native_read_unlock(ethr_native_rwlock_t *lock) { unsigned int old, new; - __asm__ __volatile__("membar #LoadLoad|#StoreLoad"); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); do { old = lock->lock; new = old-1; @@ -70,7 +73,7 @@ ethr_native_read_trylock(ethr_native_rwlock_t *lock) : "r"(old), "r"(&lock->lock), "0"(new) : "memory"); } while (__builtin_expect(old != new, 0)); - __asm__ __volatile__("membar #StoreLoad|#StoreStore"); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); return 1; } @@ -87,7 +90,7 @@ ethr_native_read_lock(ethr_native_rwlock_t *lock) if (__builtin_expect(ethr_native_read_trylock(lock) != 0, 1)) break; do { - __asm__ __volatile__("membar #LoadLoad"); + ETHR_MEMBAR(ETHR_LoadLoad); } while (ethr_native_read_is_locked(lock)); } } @@ -95,7 +98,7 @@ ethr_native_read_lock(ethr_native_rwlock_t *lock) static ETHR_INLINE void ethr_native_write_unlock(ethr_native_rwlock_t *lock) { - __asm__ __volatile__("membar #LoadStore|#StoreStore"); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); lock->lock = 0; } @@ -115,7 +118,7 @@ ethr_native_write_trylock(ethr_native_rwlock_t *lock) : "r"(old), "r"(&lock->lock), "0"(new) : "memory"); } while (__builtin_expect(old != new, 0)); - __asm__ __volatile__("membar #StoreLoad|#StoreStore"); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); return 1; } @@ -132,7 +135,7 @@ ethr_native_write_lock(ethr_native_rwlock_t *lock) if (__builtin_expect(ethr_native_write_trylock(lock) != 0, 1)) break; do { - __asm__ __volatile__("membar #LoadLoad"); + ETHR_MEMBAR(ETHR_LoadLoad); } while (ethr_native_write_is_locked(lock)); } } diff --git a/erts/include/internal/sparc32/spinlock.h b/erts/include/internal/sparc32/spinlock.h index 493d514210..d4e36e09cf 100644 --- a/erts/include/internal/sparc32/spinlock.h +++ b/erts/include/internal/sparc32/spinlock.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * Copyright Ericsson AB 2005-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -24,6 +24,9 @@ #ifndef ETHR_SPARC32_SPINLOCK_H #define ETHR_SPARC32_SPINLOCK_H +#define ETHR_HAVE_NATIVE_SPINLOCKS 1 +#define ETHR_NATIVE_SPINLOCK_IMPL "ethread" + /* Locked with ldstub, so unlocked when 0 and locked when non-zero. */ typedef struct { volatile unsigned char lock; @@ -40,7 +43,7 @@ ethr_native_spinlock_init(ethr_native_spinlock_t *lock) static ETHR_INLINE void ethr_native_spin_unlock(ethr_native_spinlock_t *lock) { - __asm__ __volatile__("membar #LoadStore|#StoreStore"); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); lock->lock = 0; } @@ -51,10 +54,10 @@ ethr_native_spin_trylock(ethr_native_spinlock_t *lock) __asm__ __volatile__( "ldstub [%1], %0\n\t" - "membar #StoreLoad|#StoreStore" : "=r"(prev) : "r"(&lock->lock) : "memory"); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); return prev == 0; } @@ -71,7 +74,7 @@ ethr_native_spin_lock(ethr_native_spinlock_t *lock) if (__builtin_expect(ethr_native_spin_trylock(lock) != 0, 1)) break; do { - __asm__ __volatile__("membar #LoadLoad"); + ETHR_MEMBAR(ETHR_LoadLoad); } while (ethr_native_spin_is_locked(lock)); } } diff --git a/erts/include/internal/tile/atomic.h b/erts/include/internal/tile/atomic.h index 5697afda25..1f1553c346 100644 --- a/erts/include/internal/tile/atomic.h +++ b/erts/include/internal/tile/atomic.h @@ -25,6 +25,7 @@ #define ETHREAD_TILE_ATOMIC_H #define ETHR_HAVE_NATIVE_ATOMIC32 1 +#define ETHR_NATIVE_ATOMIC32_IMPL "tilera" #include <atomic.h> @@ -34,27 +35,25 @@ typedef struct { volatile ethr_sint32_t counter; } ethr_native_atomic32_t; -#define ETHR_MEMORY_BARRIER __insn_mf() - #if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADDR 1 + static ETHR_INLINE ethr_sint32_t * ethr_native_atomic32_addr(ethr_native_atomic32_t *var) { return (ethr_sint32_t *) &var->counter; } +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT 1 + static ETHR_INLINE void ethr_native_atomic32_init(ethr_native_atomic32_t *var, ethr_sint32_t i) { var->counter = i; } -static ETHR_INLINE void -ethr_native_atomic32_set(ethr_native_atomic32_t *var, ethr_sint32_t i) -{ - atomic_exchange_acq(&var->counter, i); -} +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ 1 static ETHR_INLINE ethr_sint32_t ethr_native_atomic32_read(ethr_native_atomic32_t *var) @@ -62,139 +61,80 @@ ethr_native_atomic32_read(ethr_native_atomic32_t *var) return var->counter; } +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_ACQB 1 + +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_read_acqb(ethr_native_atomic32_t *var) +{ + return atomic_compare_and_exchange_val_acq(&var->counter, + 0x81818181, + 0x81818181); +} + +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD 1 + static ETHR_INLINE void ethr_native_atomic32_add(ethr_native_atomic32_t *var, ethr_sint32_t incr) { - ETHR_MEMORY_BARRIER; atomic_add(&var->counter, incr); - ETHR_MEMORY_BARRIER; -} - +} + +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC 1 + static ETHR_INLINE void ethr_native_atomic32_inc(ethr_native_atomic32_t *var) { - ETHR_MEMORY_BARRIER; atomic_increment(&var->counter); - ETHR_MEMORY_BARRIER; } +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC 1 + static ETHR_INLINE void ethr_native_atomic32_dec(ethr_native_atomic32_t *var) { - ETHR_MEMORY_BARRIER; atomic_decrement(&var->counter); - ETHR_MEMORY_BARRIER; } +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN 1 + static ETHR_INLINE ethr_sint32_t ethr_native_atomic32_add_return(ethr_native_atomic32_t *var, ethr_sint32_t incr) { - ethr_sint32_t res; - ETHR_MEMORY_BARRIER; - res = atomic_exchange_and_add(&var->counter, incr) + incr; - ETHR_MEMORY_BARRIER; - return res; + return atomic_exchange_and_add(&var->counter, incr) + incr; } -static ETHR_INLINE ethr_sint32_t -ethr_native_atomic32_inc_return(ethr_native_atomic32_t *var) -{ - return ethr_native_atomic32_add_return(var, 1); -} - -static ETHR_INLINE ethr_sint32_t -ethr_native_atomic32_dec_return(ethr_native_atomic32_t *var) -{ - return ethr_native_atomic32_add_return(var, -1); -} +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD 1 static ETHR_INLINE ethr_sint32_t ethr_native_atomic32_and_retold(ethr_native_atomic32_t *var, ethr_sint32_t mask) { - ethr_sint32_t res; - ETHR_MEMORY_BARRIER; - res = atomic_and_val(&var->counter, mask); - ETHR_MEMORY_BARRIER; - return res; + return atomic_and_val(&var->counter, mask); } +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD 1 + static ETHR_INLINE ethr_sint32_t ethr_native_atomic32_or_retold(ethr_native_atomic32_t *var, ethr_sint32_t mask) { - ethr_sint32_t res; - ETHR_MEMORY_BARRIER; - res = atomic_or_val(&var->counter, mask); - ETHR_MEMORY_BARRIER; - return res; + return atomic_or_val(&var->counter, mask); } +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_ACQB 1 + static ETHR_INLINE ethr_sint32_t -ethr_native_atomic32_xchg(ethr_native_atomic32_t *var, ethr_sint32_t val) +ethr_native_atomic32_xchg_acqb(ethr_native_atomic32_t *var, ethr_sint32_t val) { - ETHR_MEMORY_BARRIER; return atomic_exchange_acq(&var->counter, val); } -static ETHR_INLINE ethr_sint32_t -ethr_native_atomic32_cmpxchg(ethr_native_atomic32_t *var, - ethr_sint32_t new, - ethr_sint32_t expected) -{ - ETHR_MEMORY_BARRIER; - return atomic_compare_and_exchange_val_acq(&var->counter, new, expected); -} - -/* - * Atomic ops with at least specified barriers. - */ - -static ETHR_INLINE ethr_sint32_t -ethr_native_atomic32_read_acqb(ethr_native_atomic32_t *var) -{ - ethr_sint32_t res = ethr_native_atomic32_read(var); - ETHR_MEMORY_BARRIER; - return res; -} - -static ETHR_INLINE ethr_sint32_t -ethr_native_atomic32_inc_return_acqb(ethr_native_atomic32_t *var) -{ - return ethr_native_atomic32_inc_return(var); -} - -static ETHR_INLINE void -ethr_native_atomic32_set_relb(ethr_native_atomic32_t *var, ethr_sint32_t val) -{ - ETHR_MEMORY_BARRIER; - ethr_native_atomic32_set(var, val); -} - -static ETHR_INLINE void -ethr_native_atomic32_dec_relb(ethr_native_atomic32_t *var) -{ - ethr_native_atomic32_dec(var); -} - -static ETHR_INLINE ethr_sint32_t -ethr_native_atomic32_dec_return_relb(ethr_native_atomic32_t *var) -{ - return ethr_native_atomic32_dec_return(var); -} +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_ACQB 1 static ETHR_INLINE ethr_sint32_t ethr_native_atomic32_cmpxchg_acqb(ethr_native_atomic32_t *var, ethr_sint32_t new, - ethr_sint32_t exp) + ethr_sint32_t expected) { - return ethr_native_atomic32_cmpxchg(var, new, exp); -} - -static ETHR_INLINE ethr_sint32_t -ethr_native_atomic32_cmpxchg_relb(ethr_native_atomic32_t *var, - ethr_sint32_t new, - ethr_sint32_t exp) -{ - return ethr_native_atomic32_cmpxchg(var, new, exp); + return atomic_compare_and_exchange_val_acq(&var->counter, new, expected); } #endif /* ETHR_TRY_INLINE_FUNCS */ diff --git a/erts/include/internal/tile/ethr_membar.h b/erts/include/internal/tile/ethr_membar.h new file mode 100644 index 0000000000..7cb4f3cf9a --- /dev/null +++ b/erts/include/internal/tile/ethr_membar.h @@ -0,0 +1,35 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2011. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Memory barriers for TILE64/TILEPro + * Author: Rickard Green + */ + +#ifndef ETHR_TILE_MEMBAR_H__ +#define ETHR_TILE_MEMBAR_H__ + +#define ETHR_LoadLoad (1 << 0) +#define ETHR_LoadStore (1 << 1) +#define ETHR_StoreLoad (1 << 2) +#define ETHR_StoreStore (1 << 3) + +#define ETHR_MEMBAR(B) __insn_mf() + +#endif diff --git a/erts/include/internal/tile/ethread.h b/erts/include/internal/tile/ethread.h index 2de4d42bc6..7f579b50e7 100644 --- a/erts/include/internal/tile/ethread.h +++ b/erts/include/internal/tile/ethread.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2008-2009. All Rights Reserved. + * Copyright Ericsson AB 2008-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -23,8 +23,7 @@ #ifndef ETHREAD_TILE_ETHREAD_H #define ETHREAD_TILE_ETHREAD_H +#include "ethr_membar.h" #include "atomic.h" -#define ETHR_HAVE_NATIVE_ATOMICS 1 - #endif /* ETHREAD_TILE_ETHREAD_H */ diff --git a/erts/include/internal/win/ethr_atomic.h b/erts/include/internal/win/ethr_atomic.h index 60def01a7e..e11f1abf47 100644 --- a/erts/include/internal/win/ethr_atomic.h +++ b/erts/include/internal/win/ethr_atomic.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2010. All Rights Reserved. + * Copyright Ericsson AB 2010-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -24,364 +24,408 @@ #undef ETHR_INCLUDE_ATOMIC_IMPL__ #if !defined(ETHR_WIN_ATOMIC32_H__) && defined(ETHR_ATOMIC_WANT_32BIT_IMPL__) -#define ETHR_WIN_ATOMIC32_H__ -#define ETHR_INCLUDE_ATOMIC_IMPL__ 4 -#undef ETHR_ATOMIC_WANT_32BIT_IMPL__ +# define ETHR_WIN_ATOMIC32_H__ +# if (defined(ETHR_MEMBAR) \ + && defined(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE) \ + && defined(ETHR_HAVE__INTERLOCKEDEXCHANGE)) +# define ETHR_INCLUDE_ATOMIC_IMPL__ 4 +# endif +# undef ETHR_ATOMIC_WANT_32BIT_IMPL__ #elif !defined(ETHR_WIN_ATOMIC64_H__) && defined(ETHR_ATOMIC_WANT_64BIT_IMPL__) -#define ETHR_WIN_ATOMIC64_H__ -#ifdef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64 -/* _InterlockedCompareExchange64() required... */ -#define ETHR_INCLUDE_ATOMIC_IMPL__ 8 +# define ETHR_WIN_ATOMIC64_H__ +# if (defined(ETHR_MEMBAR) \ + && (defined(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64) \ + || defined(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_ACQ) \ + || defined(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_REL))) +# define ETHR_INCLUDE_ATOMIC_IMPL__ 8 +# endif +# undef ETHR_ATOMIC_WANT_64BIT_IMPL__ #endif -#undef ETHR_ATOMIC_WANT_64BIT_IMPL__ + +#if !defined(_MSC_VER) || _MSC_VER < 1400 +# undef ETHR_INCLUDE_ATOMIC_IMPL__ #endif #ifdef ETHR_INCLUDE_ATOMIC_IMPL__ -#if defined(_MSC_VER) && _MSC_VER >= 1400 +# ifndef ETHR_WIN_ATOMIC_COMMON__ +# define ETHR_WIN_ATOMIC_COMMON__ + +# if defined(_M_IX86) || defined(_M_AMD64) || defined(_M_IA64) +# define ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ 1 +# else +# define ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ 0 +# endif + +# endif /* ETHR_WIN_ATOMIC_COMMON__ */ + +# if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 + +# define ETHR_HAVE_NATIVE_ATOMIC32 1 +# define ETHR_NATIVE_ATOMIC32_IMPL "windows-interlocked" + +# ifdef ETHR_HAVE__INTERLOCKEDDECREMENT +# define ETHR_WIN_HAVE_DEC +# pragma intrinsic(_InterlockedDecrement) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDDECREMENT_REL +# define ETHR_WIN_HAVE_DEC_REL +# pragma intrinsic(_InterlockedDecrement_rel) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDINCREMENT +# define ETHR_WIN_HAVE_INC +# pragma intrinsic(_InterlockedIncrement) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDINCREMENT_ACQ +# define ETHR_WIN_HAVE_INC_ACQ +# pragma intrinsic(_InterlockedIncrement_acq) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDEXCHANGEADD +# define ETHR_WIN_HAVE_XCHG_ADD +# pragma intrinsic(_InterlockedExchangeAdd) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDEXCHANGEADD_ACQ +# define ETHR_WIN_HAVE_XCHG_ADD_ACQ +# pragma intrinsic(_InterlockedExchangeAdd_acq) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDEXCHANGE +# define ETHR_WIN_HAVE_XCHG +# pragma intrinsic(_InterlockedExchange) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDAND +# define ETHR_WIN_HAVE_AND +# pragma intrinsic(_InterlockedAnd) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDOR +# define ETHR_WIN_HAVE_OR +# pragma intrinsic(_InterlockedOr) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE +# define ETHR_WIN_HAVE_CMPXCHG +# pragma intrinsic(_InterlockedCompareExchange) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE_ACQ +# define ETHR_WIN_HAVE_CMPXCHG_ACQ +# pragma intrinsic(_InterlockedCompareExchange_acq) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE_REL +# define ETHR_WIN_HAVE_CMPXCHG_REL +# pragma intrinsic(_InterlockedCompareExchange_rel) +# endif + +# define ETHR_ILCKD__(X) _Interlocked ## X +# define ETHR_ILCKD_ACQ__(X) _Interlocked ## X ## _acq +# define ETHR_ILCKD_REL__(X) _Interlocked ## X ## _rel + +# define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X +# define ETHR_ATMC_T__ ethr_native_atomic32_t +# define ETHR_AINT_T__ ethr_sint32_t + +# elif ETHR_INCLUDE_ATOMIC_IMPL__ == 8 + +# define ETHR_HAVE_NATIVE_ATOMIC64 1 +# define ETHR_NATIVE_ATOMIC64_IMPL "windows-interlocked" + +# ifdef ETHR_HAVE__INTERLOCKEDDECREMENT64 +# define ETHR_WIN_HAVE_DEC +# pragma intrinsic(_InterlockedDecrement64) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDDECREMENT64_REL +# define ETHR_WIN_HAVE_DEC_REL +# pragma intrinsic(_InterlockedDecrement64_rel) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDINCREMENT64_ACQ +# define ETHR_WIN_HAVE_INC_ACQ +# pragma intrinsic(_InterlockedIncrement64_acq) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDINCREMENT64 +# define ETHR_WIN_HAVE_INC +# pragma intrinsic(_InterlockedIncrement64) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDEXCHANGEADD64 +# define ETHR_WIN_HAVE_XCHG_ADD +# pragma intrinsic(_InterlockedExchangeAdd64) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDEXCHANGEADD64_ACQ +# define ETHR_WIN_HAVE_XCHG_ADD_ACQ +# pragma intrinsic(_InterlockedExchangeAdd64_acq) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDEXCHANGE64 +# define ETHR_WIN_HAVE_XCHG +# pragma intrinsic(_InterlockedExchange64) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDAND64 +# define ETHR_WIN_HAVE_AND +# pragma intrinsic(_InterlockedAnd64) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDOR64 +# define ETHR_WIN_HAVE_OR +# pragma intrinsic(_InterlockedOr64) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64 +# define ETHR_WIN_HAVE_CMPXCHG +# pragma intrinsic(_InterlockedCompareExchange64) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_ACQ +# define ETHR_WIN_HAVE_CMPXCHG_ACQ +# pragma intrinsic(_InterlockedCompareExchange64_acq) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_REL +# define ETHR_WIN_HAVE_CMPXCHG_REL +# pragma intrinsic(_InterlockedCompareExchange64_rel) +# endif + +# define ETHR_ILCKD__(X) _Interlocked ## X ## 64 +# define ETHR_ILCKD_ACQ__(X) _Interlocked ## X ## 64_acq +# define ETHR_ILCKD_REL__(X) _Interlocked ## X ## 64_rel + +# define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X +# define ETHR_ATMC_T__ ethr_native_atomic64_t +# define ETHR_AINT_T__ ethr_sint64_t + +# else +# error "Unsupported integer size" +# endif -#ifndef ETHR_WIN_ATOMIC_COMMON__ -#define ETHR_WIN_ATOMIC_COMMON__ +typedef struct { + volatile ETHR_AINT_T__ value; +} ETHR_ATMC_T__; -#define ETHR_HAVE_NATIVE_ATOMICS 1 +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) -#if defined(_M_IX86) || defined(_M_AMD64) || defined(_M_IA64) -# define ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ 1 +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADDR 1 #else -# define ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ 0 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADDR 1 #endif -#if defined(_M_AMD64) || (defined(_M_IX86) \ - && !defined(ETHR_PRE_PENTIUM4_COMPAT)) -# define ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ 1 -#else -# define ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ 0 -#endif -/* - * No configure test checking for interlocked acquire/release - * versions have been written, yet. It should define - * ETHR_HAVE_INTERLOCKED_ACQUIRE_RELEASE_BARRIERS if, and - * only if, all used interlocked operations with barriers - * exists. - * - * Note, that these are pure optimizations for the itanium - * processor. - */ +static ETHR_INLINE ETHR_AINT_T__ * +ETHR_NATMC_FUNC__(addr)(ETHR_ATMC_T__ *var) +{ + return (ETHR_AINT_T__ *) &var->value; +} -#include <intrin.h> -#undef ETHR_COMPILER_BARRIER -#define ETHR_COMPILER_BARRIER _ReadWriteBarrier() -#pragma intrinsic(_ReadWriteBarrier) -#pragma intrinsic(_InterlockedCompareExchange) - -#if defined(_M_AMD64) || (defined(_M_IX86) \ - && !defined(ETHR_PRE_PENTIUM4_COMPAT)) -#include <emmintrin.h> -#include <mmintrin.h> -#pragma intrinsic(_mm_mfence) -#define ETHR_MEMORY_BARRIER _mm_mfence() -#pragma intrinsic(_mm_sfence) -#define ETHR_WRITE_MEMORY_BARRIER _mm_sfence() -#pragma intrinsic(_mm_lfence) -#define ETHR_READ_MEMORY_BARRIER _mm_lfence() -#define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_COMPILER_BARRIER +#if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET 1 #else - -#define ETHR_MEMORY_BARRIER \ -do { \ - volatile long x___ = 0; \ - _InterlockedCompareExchange(&x___, (long) 1, (long) 0); \ -} while (0) - +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET 1 #endif -#endif /* ETHR_WIN_ATOMIC_COMMON__ */ +static ETHR_INLINE void +ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) +{ + var->value = i; +} #if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_RELB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RELB 1 +#endif -#define ETHR_HAVE_NATIVE_ATOMIC32 1 +static ETHR_INLINE void +ETHR_NATMC_FUNC__(set_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) +{ +#if defined(_M_IX86) + if (ETHR_X86_RUNTIME_CONF_HAVE_NO_SSE2__) + (void) ETHR_ILCKD__(Exchange)(&var->value, i); + else +#endif /* _M_IX86 */ + { + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + var->value = i; + } +} -/* - * All used operations available as 32-bit intrinsics - */ +#endif /* ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ */ -#pragma intrinsic(_InterlockedDecrement) -#pragma intrinsic(_InterlockedIncrement) -#pragma intrinsic(_InterlockedExchangeAdd) -#pragma intrinsic(_InterlockedExchange) -#pragma intrinsic(_InterlockedAnd) -#pragma intrinsic(_InterlockedOr) -#ifdef ETHR_HAVE_INTERLOCKED_ACQUIRE_RELEASE_BARRIERS -#pragma intrinsic(_InterlockedExchangeAdd_acq) -#pragma intrinsic(_InterlockedIncrement_acq) -#pragma intrinsic(_InterlockedDecrement_rel) -#pragma intrinsic(_InterlockedCompareExchange_acq) -#pragma intrinsic(_InterlockedCompareExchange_rel) -#endif +#if defined(ETHR_WIN_HAVE_XCHG) -#define ETHR_ILCKD__(X) _Interlocked ## X -#ifdef ETHR_HAVE_INTERLOCKED_ACQUIRE_RELEASE_BARRIERS -#define ETHR_ILCKD_ACQ__(X) _Interlocked ## X ## _acq -#define ETHR_ILCKD_REL__(X) _Interlocked ## X ## _rel +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_MB 1 #else -#define ETHR_ILCKD_ACQ__(X) _Interlocked ## X -#define ETHR_ILCKD_REL__(X) _Interlocked ## X +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_MB 1 #endif -#define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X -#define ETHR_ATMC_T__ ethr_native_atomic32_t -#define ETHR_AINT_T__ ethr_sint32_t - -#elif ETHR_INCLUDE_ATOMIC_IMPL__ == 8 +static ETHR_INLINE void +ETHR_NATMC_FUNC__(set_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) +{ + (void) ETHR_ILCKD__(Exchange)(&var->value, i); +} -#define ETHR_HAVE_NATIVE_ATOMIC64 1 +#endif -/* - * _InterlockedCompareExchange64() is required. The other may not - * be available, but if so, we can generate them. - */ -#pragma intrinsic(_InterlockedCompareExchange64) +#if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ -#if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ -#define ETHR_OWN_ILCKD_INIT_VAL__(PTR) *(PTR) +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ 1 #else -#define ETHR_OWN_ILCKD_INIT_VAL__(PTR) (__int64) 0 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ 1 #endif -#define ETHR_OWN_ILCKD_BODY_IMPL__(FUNC, PTR, NEW, ACT, EXP, OPS, RET) \ -{ \ - __int64 NEW, ACT, EXP; \ - ACT = ETHR_OWN_ILCKD_INIT_VAL__(PTR); \ - do { \ - EXP = ACT; \ - { OPS; } \ - ACT = _InterlockedCompareExchange64(PTR, NEW, EXP); \ - } while (ACT != EXP); \ - return RET; \ +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var) +{ + return var->value; } -#define ETHR_OWN_ILCKD_1_IMPL__(FUNC, NEW, ACT, EXP, OPS, RET) \ -static __forceinline __int64 \ -FUNC(__int64 volatile *ptr) \ -ETHR_OWN_ILCKD_BODY_IMPL__(FUNC, ptr, NEW, ACT, EXP, OPS, RET) - -#define ETHR_OWN_ILCKD_2_IMPL__(FUNC, NEW, ACT, EXP, OPS, ARG, RET) \ -static __forceinline __int64 \ -FUNC(__int64 volatile *ptr, __int64 ARG) \ -ETHR_OWN_ILCKD_BODY_IMPL__(FUNC, ptr, NEW, ACT, EXP, OPS, RET) +#endif /* ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ */ +#if defined(ETHR_WIN_HAVE_XCHG_ADD) -#ifdef ETHR_HAVE__INTERLOCKEDDECREMENT64 -#pragma intrinsic(_InterlockedDecrement64) -#else -ETHR_OWN_ILCKD_1_IMPL__(_InterlockedDecrement64, new, act, exp, - new = act - 1, new) -#endif -#ifdef ETHR_HAVE__INTERLOCKEDINCREMENT64 -#pragma intrinsic(_InterlockedIncrement64) -#else -ETHR_OWN_ILCKD_1_IMPL__(_InterlockedIncrement64, new, act, exp, - new = act + 1, new) -#endif -#ifdef ETHR_HAVE__INTERLOCKEDEXCHANGEADD64 -#pragma intrinsic(_InterlockedExchangeAdd64) -#else -ETHR_OWN_ILCKD_2_IMPL__(_InterlockedExchangeAdd64, new, act, exp, - new = act + arg, arg, act) -#endif -#ifdef ETHR_HAVE__INTERLOCKEDEXCHANGE64 -#pragma intrinsic(_InterlockedExchange64) -#else -ETHR_OWN_ILCKD_2_IMPL__(_InterlockedExchange64, new, act, exp, - new = arg, arg, act) -#endif -#ifdef ETHR_HAVE__INTERLOCKEDAND64 -#pragma intrinsic(_InterlockedAnd64) -#else -ETHR_OWN_ILCKD_2_IMPL__(_InterlockedAnd64, new, act, exp, - new = act & arg, arg, act) -#endif -#ifdef ETHR_HAVE__INTERLOCKEDOR64 -#pragma intrinsic(_InterlockedOr64) +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_MB 1 #else -ETHR_OWN_ILCKD_2_IMPL__(_InterlockedOr64, new, act, exp, - new = act | arg, arg, act) -#endif -#ifdef ETHR_HAVE_INTERLOCKED_ACQUIRE_RELEASE_BARRIERS -#pragma intrinsic(_InterlockedExchangeAdd64_acq) -#pragma intrinsic(_InterlockedIncrement64_acq) -#pragma intrinsic(_InterlockedDecrement64_rel) -#pragma intrinsic(_InterlockedCompareExchange64_acq) -#pragma intrinsic(_InterlockedCompareExchange64_rel) +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_MB 1 #endif -#define ETHR_ILCKD__(X) _Interlocked ## X ## 64 -#ifdef ETHR_HAVE_INTERLOCKED_ACQUIRE_RELEASE_BARRIERS -#define ETHR_ILCKD_ACQ__(X) _Interlocked ## X ## 64_acq -#define ETHR_ILCKD_REL__(X) _Interlocked ## X ## 64_rel -#else -#define ETHR_ILCKD_ACQ__(X) _Interlocked ## X ## 64 -#define ETHR_ILCKD_REL__(X) _Interlocked ## X ## 64 +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(add_return_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) +{ + return ETHR_ILCKD__(ExchangeAdd)(&var->value, i) + i; +} + #endif -#define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X -#define ETHR_ATMC_T__ ethr_native_atomic64_t -#define ETHR_AINT_T__ ethr_sint64_t +#if defined(ETHR_WIN_HAVE_INC) +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_MB 1 #else -#error "Unsupported integer size" +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_MB 1 #endif -typedef struct { - volatile ETHR_AINT_T__ value; -} ETHR_ATMC_T__; - -#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) - -static ETHR_INLINE ETHR_AINT_T__ * -ETHR_NATMC_FUNC__(addr)(ETHR_ATMC_T__ *var) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(inc_return_mb)(ETHR_ATMC_T__ *var) { - return (ETHR_AINT_T__ *) &var->value; + return ETHR_ILCKD__(Increment)(&var->value); } -static ETHR_INLINE void -ETHR_NATMC_FUNC__(init)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) -{ -#if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ - var->value = i; -#else - (void) ETHR_ILCKD__(Exchange)(&var->value, i); #endif -} -static ETHR_INLINE void -ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) -{ -#if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ - var->value = i; +#if defined(ETHR_WIN_HAVE_INC_ACQ) + +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_ACQB 1 #else - (void) ETHR_ILCKD__(Exchange)(&var->value, i); +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_ACQB 1 #endif -} static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var) +ETHR_NATMC_FUNC__(inc_return_acqb)(ETHR_ATMC_T__ *var) { -#if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ - return var->value; -#else - return ETHR_ILCKD__(ExchangeAdd)(&var->value, (ETHR_AINT_T__) 0); -#endif + return ETHR_ILCKD_ACQ__(Increment)(&var->value); } -static ETHR_INLINE void -ETHR_NATMC_FUNC__(add)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) -{ - (void) ETHR_ILCKD__(ExchangeAdd)(&var->value, incr); -} +#endif + +#if defined(ETHR_WIN_HAVE_DEC) + +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_MB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_MB 1 +#endif static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(add_return)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) +ETHR_NATMC_FUNC__(dec_return_mb)(ETHR_ATMC_T__ *var) { - return ETHR_ILCKD__(ExchangeAdd)(&var->value, i) + i; + return ETHR_ILCKD__(Decrement)(&var->value); } -static ETHR_INLINE void -ETHR_NATMC_FUNC__(inc)(ETHR_ATMC_T__ *var) -{ - (void) ETHR_ILCKD__(Increment)(&var->value); -} +#endif -static ETHR_INLINE void -ETHR_NATMC_FUNC__(dec)(ETHR_ATMC_T__ *var) -{ - (void) ETHR_ILCKD__(Decrement)(&var->value); -} +#if defined(ETHR_WIN_HAVE_DEC_REL) -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(inc_return)(ETHR_ATMC_T__ *var) -{ - return ETHR_ILCKD__(Increment)(&var->value); -} +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_RELB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_RELB 1 +#endif static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(dec_return)(ETHR_ATMC_T__ *var) +ETHR_NATMC_FUNC__(dec_return_relb)(ETHR_ATMC_T__ *var) { - return ETHR_ILCKD__(Decrement)(&var->value); + return ETHR_ILCKD_REL__(Decrement)(&var->value); } +#endif + +#if defined(ETHR_WIN_HAVE_AND) + +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_MB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_MB 1 +#endif + static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(and_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) +ETHR_NATMC_FUNC__(and_retold_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) { return ETHR_ILCKD__(And)(&var->value, mask); } +#endif + +#if defined(ETHR_WIN_HAVE_OR) + +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_MB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_MB 1 +#endif + static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(or_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) +ETHR_NATMC_FUNC__(or_retold_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) { return ETHR_ILCKD__(Or)(&var->value, mask); } -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var, - ETHR_AINT_T__ new, - ETHR_AINT_T__ old) -{ - return ETHR_ILCKD__(CompareExchange)(&var->value, new, old); -} +#endif +#if defined(ETHR_WIN_HAVE_XCHG) + +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_MB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_MB 1 +#endif static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(xchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new) +ETHR_NATMC_FUNC__(xchg_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new) { return ETHR_ILCKD__(Exchange)(&var->value, new); } -/* - * Atomic ops with at least specified barriers. - */ +#endif -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(read_acqb)(ETHR_ATMC_T__ *var) -{ -#if ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ - ETHR_AINT_T__ val = var->value; - ETHR_COMPILER_BARRIER; - return val; +#if defined(ETHR_WIN_HAVE_CMPXCHG) + +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_MB 1 #else - return ETHR_ILCKD_ACQ__(ExchangeAdd)(&var->value, (ETHR_AINT_T__) 0); +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_MB 1 #endif -} static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(inc_return_acqb)(ETHR_ATMC_T__ *var) +ETHR_NATMC_FUNC__(cmpxchg_mb)(ETHR_ATMC_T__ *var, + ETHR_AINT_T__ new, + ETHR_AINT_T__ old) { - return ETHR_ILCKD_ACQ__(Increment)(&var->value); + return ETHR_ILCKD__(CompareExchange)(&var->value, new, old); } -static ETHR_INLINE void -ETHR_NATMC_FUNC__(set_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) -{ -#if ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ - ETHR_COMPILER_BARRIER; - var->value = i; -#else - (void) ETHR_ILCKD_REL__(Exchange)(&var->value, i); #endif -} -static ETHR_INLINE void -ETHR_NATMC_FUNC__(dec_relb)(ETHR_ATMC_T__ *var) -{ - (void) ETHR_ILCKD_REL__(Decrement)(&var->value); -} +#if defined(ETHR_WIN_HAVE_CMPXCHG_ACQ) -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(dec_return_relb)(ETHR_ATMC_T__ *var) -{ - return ETHR_ILCKD_REL__(Decrement)(&var->value); -} +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_ACQB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_ACQB 1 +#endif static ETHR_INLINE ETHR_AINT_T__ ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var, @@ -391,6 +435,16 @@ ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var, return ETHR_ILCKD_ACQ__(CompareExchange)(&var->value, new, old); } +#endif + +#if defined(ETHR_WIN_HAVE_CMPXCHG_REL) + +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_RELB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RELB 1 +#endif + static ETHR_INLINE ETHR_AINT_T__ ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new, @@ -399,6 +453,8 @@ ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var, return ETHR_ILCKD_REL__(CompareExchange)(&var->value, new, old); } +#endif + #endif /* ETHR_TRY_INLINE_FUNCS */ #undef ETHR_ILCKD__ @@ -408,8 +464,17 @@ ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var, #undef ETHR_ATMC_T__ #undef ETHR_AINT_T__ #undef ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ -#undef ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ - -#endif /* _MSC_VER */ +#undef ETHR_WIN_HAVE_CMPXCHG +#undef ETHR_WIN_HAVE_DEC +#undef ETHR_WIN_HAVE_INC +#undef ETHR_WIN_HAVE_XCHG_ADD +#undef ETHR_WIN_HAVE_XCHG +#undef ETHR_WIN_HAVE_AND +#undef ETHR_WIN_HAVE_OR +#undef ETHR_WIN_HAVE_XCHG_ADD_ACQ +#undef ETHR_WIN_HAVE_INC_ACQ +#undef ETHR_WIN_HAVE_DEC_REL +#undef ETHR_WIN_HAVE_CMPXCHG_ACQ +#undef ETHR_WIN_HAVE_CMPXCHG_REL #endif /* ETHR_INCLUDE_ATOMIC_IMPL__ */ diff --git a/erts/include/internal/win/ethr_dw_atomic.h b/erts/include/internal/win/ethr_dw_atomic.h new file mode 100644 index 0000000000..a3e7ffc3aa --- /dev/null +++ b/erts/include/internal/win/ethr_dw_atomic.h @@ -0,0 +1,154 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2011. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Native double word atomics for windows + * Author: Rickard Green + */ + +#undef ETHR_INCLUDE_DW_ATOMIC_IMPL__ +#ifndef ETHR_X86_DW_ATOMIC_H__ +# define ETHR_X86_DW_ATOMIC_H__ +# if ((ETHR_SIZEOF_PTR == 4 \ + && defined(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64)) \ + || (ETHR_SIZEOF_PTR == 8 \ + && defined(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE128))) +# define ETHR_INCLUDE_DW_ATOMIC_IMPL__ +# endif +#endif + +#ifdef ETHR_INCLUDE_DW_ATOMIC_IMPL__ + +# if ETHR_SIZEOF_PTR == 4 +# define ETHR_HAVE_NATIVE_SU_DW_ATOMIC +# else +# define ETHR_HAVE_NATIVE_DW_ATOMIC +# endif +# define ETHR_NATIVE_DW_ATOMIC_IMPL "windows-interlocked" + +# if defined(_M_IX86) || defined(_M_AMD64) +/* + * If ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ is defined, it will be used + * at runtime in order to determine if native or fallback implementation + * should be used. + */ +# define ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ \ + ETHR_X86_RUNTIME_CONF_HAVE_DW_CMPXCHG__ +# endif + +# include <intrin.h> +# if ETHR_SIZEOF_PTR == 4 +# pragma intrinsic(_InterlockedCompareExchange64) +# define ETHR_DW_NATMC_ALIGN_MASK__ 0x7 +# define ETHR_NATIVE_SU_DW_SINT_T ethr_sint64_t +# else +# pragma intrinsic(_InterlockedCompareExchange128) +# define ETHR_DW_NATMC_ALIGN_MASK__ 0xf +# endif + +typedef volatile __int64 * ethr_native_dw_ptr_t; + +/* + * We need 16 byte aligned memory in 64-bit mode, and 8 byte aligned + * memory in 32-bit mode. 16 byte aligned malloc in 64-bit mode is + * not common, and at least some glibc malloc implementations + * only 4 byte align in 32-bit mode. + * + * This code assumes 8 byte aligned memory in 64-bit mode, and 4 byte + * aligned memory in 32-bit mode. A malloc implementation that does + * not adhere to these alignment requirements is seriously broken, + * and we wont bother trying to work around it. + * + * Since memory alignment may be off by one word we need to align at + * runtime. We, therefore, need an extra word allocated. + */ +#define ETHR_DW_NATMC_MEM__(VAR) \ + (&var->c[(int) ((ethr_uint_t) &(VAR)->c[0]) & ETHR_DW_NATMC_ALIGN_MASK__]) +typedef union { +#ifdef ETHR_NATIVE_SU_DW_SINT_T + volatile ETHR_NATIVE_SU_DW_SINT_T dw_sint; +#endif + volatile ethr_sint_t sint[3]; + volatile char c[ETHR_SIZEOF_PTR*3]; +} ethr_native_dw_atomic_t; + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) + +#ifdef ETHR_DEBUG +# define ETHR_DW_DBG_ALIGNED__(PTR) \ + ETHR_ASSERT((((ethr_uint_t) (PTR)) & ETHR_DW_NATMC_ALIGN_MASK__) == 0); +#else +# define ETHR_DW_DBG_ALIGNED__(PTR) +#endif + +#define ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_ADDR + +static ETHR_INLINE ethr_sint_t * +ethr_native_dw_atomic_addr(ethr_native_dw_atomic_t *var) +{ + ethr_sint_t *p = (ethr_sint_t *) ETHR_DW_NATMC_MEM__(var); + ETHR_DW_DBG_ALIGNED__(p); + return p; +} + + +#if ETHR_SIZEOF_PTR == 4 + +#define ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_MB + +static ETHR_INLINE ethr_sint64_t +ethr_native_su_dw_atomic_cmpxchg_mb(ethr_native_dw_atomic_t *var, + ethr_sint64_t new, + ethr_sint64_t exp) +{ + ethr_native_dw_ptr_t p = (ethr_native_dw_ptr_t) ETHR_DW_NATMC_MEM__(var); + ETHR_DW_DBG_ALIGNED__(p); + return (ethr_sint64_t) _InterlockedCompareExchange64(p, new, exp); +} + +#elif ETHR_SIZEOF_PTR == 8 + +#define ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_MB + +#ifdef ETHR_BIGENDIAN +# define ETHR_WIN_LOW_WORD__ 1 +# define ETHR_WIN_HIGH_WORD__ 0 +#else +# define ETHR_WIN_LOW_WORD__ 0 +# define ETHR_WIN_HIGH_WORD__ 1 +#endif + +static ETHR_INLINE int +ethr_native_dw_atomic_cmpxchg_mb(ethr_native_dw_atomic_t *var, + ethr_sint_t *new, + ethr_sint_t *xchg) +{ + ethr_native_dw_ptr_t p = (ethr_native_dw_ptr_t) ETHR_DW_NATMC_MEM__(var); + ETHR_DW_DBG_ALIGNED__(p); + return (int) _InterlockedCompareExchange128(p, + new[ETHR_WIN_HIGH_WORD__], + new[ETHR_WIN_LOW_WORD__], + xchg); +} + +#endif + +#endif /* ETHR_TRY_INLINE_FUNCS */ + +#endif /* ETHR_INCLUDE_DW_ATOMIC_IMPL__ */ diff --git a/erts/include/internal/win/ethr_event.h b/erts/include/internal/win/ethr_event.h index 598816b2c6..6363174a74 100644 --- a/erts/include/internal/win/ethr_event.h +++ b/erts/include/internal/win/ethr_event.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2009-2010. All Rights Reserved. + * Copyright Ericsson AB 2009-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -21,12 +21,12 @@ * Author: Rickard Green */ -#define ETHR_EVENT_OFF_WAITER__ ((long) -1) -#define ETHR_EVENT_OFF__ ((long) 1) -#define ETHR_EVENT_ON__ ((long) 0) +#define ETHR_EVENT_OFF_WAITER__ ((ethr_sint32_t) -1) +#define ETHR_EVENT_OFF__ ((ethr_sint32_t) 1) +#define ETHR_EVENT_ON__ ((ethr_sint32_t) 0) typedef struct { - volatile long state; + ethr_atomic32_t state; HANDLE handle; } ethr_event; @@ -38,7 +38,7 @@ static ETHR_INLINE void ETHR_INLINE_FUNC_NAME_(ethr_event_set)(ethr_event *e) { /* _InterlockedExchange() imply a full memory barrier which is important */ - long state = _InterlockedExchange(&e->state, ETHR_EVENT_ON__); + ethr_sint32_t state = ethr_atomic32_xchg_wb(&e->state, ETHR_EVENT_ON__); if (state == ETHR_EVENT_OFF_WAITER__) { if (!SetEvent(e->handle)) ETHR_FATAL_ERROR__(ethr_win_get_errno__()); @@ -48,8 +48,8 @@ ETHR_INLINE_FUNC_NAME_(ethr_event_set)(ethr_event *e) static ETHR_INLINE void ETHR_INLINE_FUNC_NAME_(ethr_event_reset)(ethr_event *e) { - /* _InterlockedExchange() imply a full memory barrier which is important */ - InterlockedExchange(&e->state, ETHR_EVENT_OFF__); + ethr_atomic32_set(&e->state, ETHR_EVENT_OFF__); + ETHR_MEMORY_BARRIER; } #endif diff --git a/erts/include/internal/win/ethr_membar.h b/erts/include/internal/win/ethr_membar.h new file mode 100644 index 0000000000..8237660b2c --- /dev/null +++ b/erts/include/internal/win/ethr_membar.h @@ -0,0 +1,145 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2011. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Memory barriers for Windows + * Author: Rickard Green + */ + +#if (!defined(ETHR_WIN_MEMBAR_H__) \ + && (defined(_MSC_VER) && _MSC_VER >= 1400) \ + && (defined(_M_AMD64) \ + || defined(_M_IA64) \ + || defined(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE))) +#define ETHR_WIN_MEMBAR_H__ + +#define ETHR_LoadLoad (1 << 0) +#define ETHR_LoadStore (1 << 1) +#define ETHR_StoreLoad (1 << 2) +#define ETHR_StoreStore (1 << 3) + +#include <intrin.h> +#undef ETHR_COMPILER_BARRIER +#define ETHR_COMPILER_BARRIER _ReadWriteBarrier() +#pragma intrinsic(_ReadWriteBarrier) + +#pragma intrinsic(_InterlockedCompareExchange) + +#define ETHR_MB_USING_INTERLOCKED__ \ +do { \ + volatile long x___ = 0; \ + (void) _InterlockedCompareExchange(&x___, 2, 1); \ +} while (0) + +#if defined(_M_IA64) + +#define ETHR_MEMBAR(B) __mf() + +#elif defined(_M_AMD64) || defined(_M_IX86) + +#include <emmintrin.h> +#include <mmintrin.h> + +#if ETHR_SIZEOF_PTR == 4 +# define ETHR_NO_SSE2_MB__ ETHR_MB_USING_INTERLOCKED__ +#endif +#pragma intrinsic(_mm_mfence) +#pragma intrinsic(_mm_sfence) +#pragma intrinsic(_mm_lfence) + +static __forceinline void +ethr_cfence__(void) +{ + _ReadWriteBarrier(); +} + +static __forceinline void +ethr_mfence__(void) +{ +#if ETHR_SIZEOF_PTR == 4 + if (ETHR_X86_RUNTIME_CONF_HAVE_NO_SSE2__) + ETHR_NO_SSE2_MB__; + else +#endif + _mm_mfence(); +} + +static __forceinline void +ethr_sfence__(void) +{ +#if ETHR_SIZEOF_PTR == 4 + if (ETHR_X86_RUNTIME_CONF_HAVE_NO_SSE2__) + ETHR_NO_SSE2_MB__; + else +#endif + _mm_sfence(); +} + +static __forceinline void +ethr_lfence__(void) +{ +#if ETHR_SIZEOF_PTR == 4 + if (ETHR_X86_RUNTIME_CONF_HAVE_NO_SSE2__) + ETHR_NO_SSE2_MB__; + else +#endif + _mm_lfence(); +} + +#define ETHR_X86_OUT_OF_ORDER_MEMBAR(B) \ + ETHR_CHOOSE_EXPR((B) == ETHR_StoreStore, \ + ethr_sfence__(), \ + ETHR_CHOOSE_EXPR((B) == ETHR_LoadLoad, \ + ethr_lfence__(), \ + ethr_mfence__())) + +#ifdef ETHR_X86_OUT_OF_ORDER + +#define ETHR_MEMBAR(B) \ + ETHR_X86_OUT_OF_ORDER_MEMBAR((B)) + +#else /* !ETHR_X86_OUT_OF_ORDER (the default) */ + +/* + * We assume that only stores before loads may be reordered. That is, + * we assume that *no* instructions like these are used: + * - CLFLUSH, + * - streaming stores executed with non-temporal move, + * - string operations, or + * - other instructions which aren't LoadLoad, LoadStore, and StoreStore + * ordered by themselves + * If such instructions are used, either insert memory barriers + * using ETHR_X86_OUT_OF_ORDER_MEMBAR() at appropriate places, or + * define ETHR_X86_OUT_OF_ORDER. For more info see Intel 64 and IA-32 + * Architectures Software Developer's Manual; Vol 3A; Chapter 8.2.2. + */ + +#define ETHR_MEMBAR(B) \ + ETHR_CHOOSE_EXPR((B) & ETHR_StoreLoad, ethr_mfence__(), ethr_cfence__()) + +#endif + +#else /* No knowledge about platform; use interlocked fallback */ + +#define ETHR_MEMBAR(B) ETHR_MB_USING_INTERLOCKED__ +#define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_MB_USING_INTERLOCKED__ + +#endif + +#endif /* ETHR_WIN_MEMBAR_H__ */ diff --git a/erts/include/internal/win/ethread.h b/erts/include/internal/win/ethread.h index c01b17cf14..8be35e810e 100644 --- a/erts/include/internal/win/ethread.h +++ b/erts/include/internal/win/ethread.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2010. All Rights Reserved. + * Copyright Ericsson AB 2010-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -25,11 +25,13 @@ #ifndef ETHREAD_WIN_H__ #define ETHREAD_WIN_H__ +#include "ethr_membar.h" #define ETHR_ATOMIC_WANT_32BIT_IMPL__ #include "ethr_atomic.h" #if ETHR_SIZEOF_PTR == 8 # define ETHR_ATOMIC_WANT_64BIT_IMPL__ # include "ethr_atomic.h" #endif +#include "ethr_dw_atomic.h" #endif diff --git a/erts/lib_src/Makefile.in b/erts/lib_src/Makefile.in index 757b3b24e2..12b8732735 100644 --- a/erts/lib_src/Makefile.in +++ b/erts/lib_src/Makefile.in @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 2004-2010. All Rights Reserved. +# Copyright Ericsson AB 2004-2011. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in @@ -65,7 +65,7 @@ TYPE_SUFFIX=.purecov PRE_LD=purecov $(PURECOV_BUILD_OPTIONS) else ifeq ($(TYPE),gcov) -CFLAGS=@DEBUG_CFLAGS@ -fprofile-arcs -ftest-coverage -O0 +CFLAGS=@DEBUG_CFLAGS@ -DGCOV -fprofile-arcs -ftest-coverage -O0 TYPE_SUFFIX=.gcov PRE_LD= else @@ -288,6 +288,10 @@ ETHREAD_LIB_SRC=common/ethr_aux.c \ common/ethr_cbf.c \ $(ETHR_THR_LIB_BASE_DIR)/ethread.c \ $(ETHR_THR_LIB_BASE_DIR)/ethr_event.c +ETHR_X86_SSE2_ASM=@ETHR_X86_SSE2_ASM@ +ifeq ($(ETHR_X86_SSE2_ASM),yes) +ETHREAD_LIB_SRC += pthread/ethr_x86_sse2_asm.c +endif ETHREAD_LIB_NAME=ethread$(TYPE_SUFFIX) ifeq ($(USING_VC),yes) @@ -382,10 +386,8 @@ $(ERTS_LIB): $(ERTS_LIB_OBJS) # Object files # -ifeq ($(TYPE)-@GCC@,debug-yes) -$(r_OBJ_DIR)/ethr_aux.o: common/ethr_aux.c - $(CC) $(THR_DEFS) $(CFLAGS) -Wno-unused-function $(INCLUDES) -c $< -o $@ -endif +$(r_OBJ_DIR)/ethr_x86_sse2_asm.o: pthread/ethr_x86_sse2_asm.c + $(CC) -msse2 $(THR_DEFS) $(CFLAGS) $(INCLUDES) -c $< -o $@ $(r_OBJ_DIR)/%.o: common/%.c $(CC) $(THR_DEFS) $(CFLAGS) $(INCLUDES) -c $< -o $@ diff --git a/erts/lib_src/common/ethr_atomics.c b/erts/lib_src/common/ethr_atomics.c index 94557d904a..5796bdc22e 100644 --- a/erts/lib_src/common/ethr_atomics.c +++ b/erts/lib_src/common/ethr_atomics.c @@ -1,7 +1,16 @@ /* + * --------------- DO NOT EDIT THIS FILE! --------------- + * This file was automatically generated by the + * $ERL_TOP/erts/lib_src/utils/make_atomics_api script. + * If you need to make changes, edit the script and + * regenerate this file. + * --------------- DO NOT EDIT THIS FILE! --------------- + */ + +/* * %CopyrightBegin% * - * Copyright Ericsson AB 2010. All Rights Reserved. + * Copyright Ericsson AB 2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -18,385 +27,4329 @@ */ /* - * Description: The ethread atomic API + * Description: The ethread atomics API * Author: Rickard Green */ +/* + * This file maps native atomic implementations to ethread + * API atomics. If no native atomic implementation + * is available, a less efficient fallback is used instead. + * The API consists of 32-bit size, word size (pointer size), + * and double word size atomics. + * + * The following atomic operations are implemented for + * 32-bit size, and word size atomics: + * - cmpxchg + * - xchg + * - set + * - init + * - add_read + * - read + * - inc_read + * - dec_read + * - add + * - inc + * - dec + * - read_band + * - read_bor + * + * The following atomic operations are implemented for + * double word size atomics: + * - cmpxchg + * - set + * - read + * - init + * + * Appart from a function implementing the atomic operation + * with unspecified memory barrier semantics, there are + * functions implementing each operation with the following + * memory barrier semantics: + * - rb (read barrier) + * - wb (write barrier) + * - acqb (acquire barrier) + * - relb (release barrier) + * - mb (full memory barrier) + * + * We implement all of these operation/barrier + * combinations, regardless of whether they are useful + * or not (some of them are useless). + * + * Double word size atomic functions are on the followning + * form: + * ethr_dw_atomic_<OP>[_<BARRIER>] + * + * Word size atomic functions are on the followning + * form: + * ethr_atomic_<OP>[_<BARRIER>] + * + * 32-bit size atomic functions are on the followning + * form: + * ethr_atomic32_<OP>[_<BARRIER>] + * + * Apart from the operation/barrier functions + * described above also 'addr' functions are implemented + * which return the actual memory address used of the + * atomic variable. The 'addr' functions have no barrier + * versions. + * + * The native atomic implementation does not need to + * implement all operation/barrier combinations. + * Functions that have no native implementation will be + * constructed from existing native functionality. These + * functions will perform the wanted operation and will + * produce sufficient memory barriers, but may + * in some cases be less efficient than pure native + * versions. + * + * When we create ethread API operation/barrier functions by + * adding barriers before and after native operations it is + * assumed that: + * - A native read operation begins, and ends with a load. + * - A native set operation begins, and ends with a store. + * - An init operation begins with either a load, or a store, + * and ends with either a load, or a store. + * - All other operations begins with a load, and ends with + * either a load, or a store. + * + * This is the minimum functionality that a native + * implementation needs to provide: + * + * - Functions that need to be implemented: + * + * - ethr_native_[dw_|su_dw_]atomic[BITS]_addr + * - ethr_native_[dw_|su_dw_]atomic[BITS]_cmpxchg[_<BARRIER>] + * (at least one cmpxchg of optional barrier) + * + * - Macros that needs to be defined: + * + * A macro informing about the presence of the native + * implementation: + * + * - ETHR_HAVE_NATIVE_[DW_|SU_DW_]ATOMIC[BITS] + * + * A macro naming (a string constant) the implementation: + * + * - ETHR_NATIVE_[DW_]ATOMIC[BITS]_IMPL + * + * Each implemented native atomic function has to + * be accompanied by a defined macro on the following + * form informing about its presence: + * + * - ETHR_HAVE_ETHR_NATIVE_[DW_|SU_DW_]ATOMIC[BITS]_<OP>[_<BARRIER>] + * + * A (sparc-v9 style) membar macro: + * + * - ETHR_MEMBAR(B) + * + * Which takes a combination of the following macros + * or:ed (using |) together: + * + * - ETHR_LoadLoad + * - ETHR_LoadStore + * - ETHR_StoreLoad + * - ETHR_StoreStore + * + */ + + #ifdef HAVE_CONFIG_H #include "config.h" #endif -#define ETHR_INLINE_FUNC_NAME_(X) X ## __ +#define ETHR_TRY_INLINE_FUNCS +#define ETHR_INLINE_DW_ATMC_FUNC_NAME_(X) X ## __ +#define ETHR_INLINE_ATMC_FUNC_NAME_(X) X ## __ +#define ETHR_INLINE_ATMC32_FUNC_NAME_(X) X ## __ #define ETHR_ATOMIC_IMPL__ #include "ethread.h" #include "ethr_internal.h" -#ifndef ETHR_HAVE_NATIVE_ATOMICS -ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATOMIC_ADDR_BITS]; +#if (!defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) \ + || !defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)) +/* + * Spinlock based fallback for atomics used in absence of a native + * implementation. + */ + +#define ETHR_ATMC_FLLBK_ADDR_BITS 10 +#define ETHR_ATMC_FLLBK_ADDR_SHIFT 6 + +typedef struct { + union { + ethr_spinlock_t lck; + char buf[ETHR_CACHE_LINE_ALIGN_SIZE(sizeof(ethr_spinlock_t))]; + } u; +} ethr_atomic_protection_t; + +extern ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATMC_FLLBK_ADDR_BITS]; + +#define ETHR_ATOMIC_PTR2LCK__(PTR) \ +(ðr_atomic_protection__[((((ethr_uint_t) (PTR)) >> ETHR_ATMC_FLLBK_ADDR_SHIFT) \ + & ((1 << ETHR_ATMC_FLLBK_ADDR_BITS) - 1))].u.lck) + + +#define ETHR_ATOMIC_OP_FALLBACK_IMPL__(AP, EXPS) \ +do { \ + ethr_spinlock_t *slp__ = ETHR_ATOMIC_PTR2LCK__((AP)); \ + ethr_spin_lock(slp__); \ + { EXPS; } \ + ethr_spin_unlock(slp__); \ +} while (0) + +ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATMC_FLLBK_ADDR_BITS]; + +#endif + + +#if defined(ETHR_AMC_FALLBACK__) + +/* + * Fallback for large sized (word and/or double word size) atomics using + * an "Atomic Modification Counter" based on smaller sized native atomics. + * + * We use a 63-bit modification counter and a one bit exclusive flag. + * If 32-bit native atomics are used, we need two 32-bit native atomics. + * The exclusive flag is the least significant bit, or if multiple atomics + * are used, the least significant bit of the least significant atomic. + * + * When using the AMC fallback the following is true: + * - Reads of the same atomic variable can be done in parallel. + * - Uncontended reads doesn't cause any cache line invalidations, + * since no modifications are done. + * - Assuming that the AMC atomic(s) and the integer(s) containing the + * value of the implemented atomic resides in the same cache line, + * modifications will only cause invalidations of one cache line. + * + * When using the spinlock based fallback none of the above is true, + * however, the spinlock based fallback consumes less memory. + */ + +# if ETHR_AMC_NO_ATMCS__ != 1 && ETHR_AMC_NO_ATMCS__ != 2 +# error "Not supported" +# endif +# define ETHR_AMC_MAX_TRY_READ__ 10 +# ifdef ETHR_DEBUG +# define ETHR_DBG_CHK_EXCL_STATE(ASP, S) \ +do { \ + ETHR_AMC_SINT_T__ act = ETHR_AMC_ATMC_FUNC__(read)(&(ASP)->atomic[0]); \ + ETHR_ASSERT(act == (S) + 1); \ + ETHR_ASSERT(act & 1); \ +} while (0) +# else +# define ETHR_DBG_CHK_EXCL_STATE(ASP, S) +# endif + +static ETHR_INLINE void +amc_init(ethr_amc_t *amc, int dw, ethr_sint_t *avar, ethr_sint_t *val) +{ + avar[0] = val[0]; + if (dw) + avar[1] = val[1]; +#if ETHR_AMC_NO_ATMCS__ == 2 + ETHR_AMC_ATMC_FUNC__(init)(&amc->atomic[1], 0); #endif + ETHR_AMC_ATMC_FUNC__(init_wb)(&amc->atomic[0], 0); +} + +static ETHR_INLINE ETHR_AMC_SINT_T__ +amc_set_excl(ethr_amc_t *amc, ETHR_AMC_SINT_T__ prev_state0) +{ + ETHR_AMC_SINT_T__ state0 = prev_state0; + /* Set exclusive flag. */ + while (1) { + ETHR_AMC_SINT_T__ act_state0, new_state0; + while (state0 & 1) { /* Wait until exclusive bit has been cleared */ + ETHR_SPIN_BODY; + state0 = ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[0]); + } + /* Try to set exclusive bit */ + new_state0 = state0 + 1; + act_state0 = ETHR_AMC_ATMC_FUNC__(cmpxchg_acqb)(&amc->atomic[0], + new_state0, + state0); + if (state0 == act_state0) + return state0; /* old state0 */ + state0 = act_state0; + } +} + +static ETHR_INLINE void +amc_inc_mc_unset_excl(ethr_amc_t *amc, ETHR_AMC_SINT_T__ old_state0) +{ + ETHR_AMC_SINT_T__ state0 = old_state0; + + /* Increment modification counter and reset exclusive flag. */ + + ETHR_DBG_CHK_EXCL_STATE(amc, state0); + + state0 += 2; + + ETHR_ASSERT((state0 & 1) == 0); + +#if ETHR_AMC_NO_ATMCS__ == 2 + if (state0 == 0) { + /* + * state0 wrapped, so we need to increment state1. There is no need + * for atomic inc op, since this is always done while having exclusive + * flag. + */ + ETHR_AMC_SINT_T__ state1 = ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[1]); + state1++; + ETHR_AMC_ATMC_FUNC__(set)(&amc->atomic[1], state1); + } +#endif + ETHR_AMC_ATMC_FUNC__(set_relb)(&amc->atomic[0], state0); +} + +static ETHR_INLINE void +amc_unset_excl(ethr_amc_t *amc, ETHR_AMC_SINT_T__ old_state0) +{ + ETHR_DBG_CHK_EXCL_STATE(amc, old_state0); + /* + * Reset exclusive flag, but leave modification counter unchanged, + * i.e., restore state to what it was before setting exclusive + * flag. + */ + ETHR_AMC_ATMC_FUNC__(set_relb)(&amc->atomic[0], old_state0); +} + +static ETHR_INLINE void +amc_set(ethr_amc_t *amc, int dw, ethr_sint_t *avar, ethr_sint_t *val) +{ + ETHR_AMC_SINT_T__ state0 = ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[0]); + + state0 = amc_set_excl(amc, state0); + + avar[0] = val[0]; + if (dw) + avar[1] = val[1]; + + amc_inc_mc_unset_excl(amc, state0); +} + +static ETHR_INLINE int +amc_try_read(ethr_amc_t *amc, int dw, ethr_sint_t *avar, + ethr_sint_t *val, ETHR_AMC_SINT_T__ *state0p) +{ + /* *state0p should contain last read value if aborting */ + ETHR_AMC_SINT_T__ old_state0; +#if ETHR_AMC_NO_ATMCS__ == 2 + ETHR_AMC_SINT_T__ state1; + int abrt; +#endif + + *state0p = ETHR_AMC_ATMC_FUNC__(read_rb)(&amc->atomic[0]); + if ((*state0p) & 1) + return 0; /* exclusive flag set; abort */ +#if ETHR_AMC_NO_ATMCS__ == 2 + state1 = ETHR_AMC_ATMC_FUNC__(read_rb)(&amc->atomic[1]); +#else + ETHR_COMPILER_BARRIER; +#endif + + val[0] = avar[0]; + if (dw) + val[1] = avar[1]; + + ETHR_READ_MEMORY_BARRIER; + + /* + * Abort if state has changed (i.e, either the exclusive + * flag is set, or modification counter changed). + */ + old_state0 = *state0p; +#if ETHR_AMC_NO_ATMCS__ == 2 + *state0p = ETHR_AMC_ATMC_FUNC__(read_rb)(&amc->atomic[0]); + abrt = (old_state0 != *state0p); + abrt |= (state1 != ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[1])); + return abrt == 0; +#else + *state0p = ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[0]); + return old_state0 == *state0p; +#endif +} + +static ETHR_INLINE void +amc_read(ethr_amc_t *amc, int dw, ethr_sint_t *avar, ethr_sint_t *val) +{ + ETHR_AMC_SINT_T__ state0; + int i; + +#if ETHR_AMC_MAX_TRY_READ__ == 0 + state0 = ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[0]); +#else + for (i = 0; i < ETHR_AMC_MAX_TRY_READ__; i++) { + if (amc_try_read(amc, dw, avar, val, &state0)) + return; /* read success */ + ETHR_SPIN_BODY; + } +#endif + + state0 = amc_set_excl(amc, state0); + + val[0] = avar[0]; + if (dw) + val[1] = avar[1]; + + amc_unset_excl(amc, state0); +} + +static ETHR_INLINE int +amc_cmpxchg(ethr_amc_t *amc, int dw, ethr_sint_t *avar, + ethr_sint_t *new, ethr_sint_t *xchg) +{ + ethr_sint_t val[2]; + ETHR_AMC_SINT_T__ state0; + + if (amc_try_read(amc, dw, avar, val, &state0)) { + if (val[0] != xchg[0] || (dw && val[1] != xchg[1])) { + xchg[0] = val[0]; + if (dw) + xchg[1] = val[1]; + return 0; /* failed */ + } + /* Operation will succeed if not interrupted */ + } + + state0 = amc_set_excl(amc, state0); + + if (xchg[0] != avar[0] || (dw && xchg[1] != avar[1])) { + xchg[0] = avar[0]; + if (dw) + xchg[1] = avar[1]; + + ETHR_DBG_CHK_EXCL_STATE(amc, state0); + + amc_unset_excl(amc, state0); + return 0; /* failed */ + } + + avar[0] = new[0]; + if (dw) + avar[1] = new[1]; + + amc_inc_mc_unset_excl(amc, state0); + return 1; +} + + +#define ETHR_AMC_MODIFICATION_OPS__(AMC, OPS) \ +do { \ + ETHR_AMC_SINT_T__ state0__; \ + state0__ = ETHR_AMC_ATMC_FUNC__(read)(&(AMC)->atomic[0]); \ + state0__ = amc_set_excl((AMC), state0__); \ + { OPS; } \ + amc_inc_mc_unset_excl((AMC), state0__); \ +} while (0) + +#endif /* amc fallback */ + int ethr_init_atomics(void) { -#ifndef ETHR_HAVE_NATIVE_ATOMICS - { - int i; - for (i = 0; i < (1 << ETHR_ATOMIC_ADDR_BITS); i++) { - int res = ethr_spinlock_init(ðr_atomic_protection__[i].u.lck); - if (res != 0) - return res; - } +#if (!defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) \ + || !defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)) + int i; + for (i = 0; i < (1 << ETHR_ATMC_FLLBK_ADDR_BITS); i++) { + int res = ethr_spinlock_init(ðr_atomic_protection__[i].u.lck); + if (res != 0) + return res; } #endif return 0; } + +/* ---------- Double word size atomic implementation ---------- */ + + + /* - * --- Pointer size atomics --------------------------------------------------- + * Double word atomics need runtime test. */ -ethr_sint_t * -ethr_atomic_addr(ethr_atomic_t *var) +int ethr_have_native_dw_atomic(void) +{ + return ethr_have_native_dw_atomic__(); +} + + +/* --- addr() --- */ + +ethr_sint_t *ETHR_DW_ATOMIC_FUNC__(addr)(ethr_dw_atomic_t *var) +{ + ethr_sint_t *res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + res = ethr_dw_atomic_addr__(var); +#elif defined(ETHR_AMC_FALLBACK__) + res = (ethr_sint_t *) ((&var->fallback))->sint; +#else + res = (ethr_sint_t *) (&var->fallback); +#endif + return res; +} + +#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ +ethr_sint_t *ethr_dw_atomic_addr(ethr_dw_atomic_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_dw_atomic_addr__(var); +} +#endif + + +/* -- cmpxchg() -- */ + + +int ETHR_DW_ATOMIC_FUNC__(cmpxchg)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val) +{ + int res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + res = ethr_dw_atomic_cmpxchg__(var, val, old_val); +#elif defined(ETHR_AMC_FALLBACK__) + res = amc_cmpxchg(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint, old_val->sint); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), + { + res = ((&var->fallback)->sint[0] == old_val->sint[0] && (&var->fallback)->sint[1] == old_val->sint[1]); + if (res) { + (&var->fallback)->sint[0] = val->sint[0]; + (&var->fallback)->sint[1] = val->sint[1]; + } + else { + old_val->sint[0] = (&var->fallback)->sint[0]; + old_val->sint[1] = (&var->fallback)->sint[1]; + } + }); +#endif + return res; +} + +#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ +int ethr_dw_atomic_cmpxchg(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_dw_atomic_cmpxchg__(var, val, old_val); +} +#endif + +int ETHR_DW_ATOMIC_FUNC__(cmpxchg_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val) +{ + int res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + res = ethr_dw_atomic_cmpxchg_rb__(var, val, old_val); +#elif defined(ETHR_AMC_FALLBACK__) + res = amc_cmpxchg(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint, old_val->sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), + { + res = ((&var->fallback)->sint[0] == old_val->sint[0] && (&var->fallback)->sint[1] == old_val->sint[1]); + if (res) { + (&var->fallback)->sint[0] = val->sint[0]; + (&var->fallback)->sint[1] = val->sint[1]; + } + else { + old_val->sint[0] = (&var->fallback)->sint[0]; + old_val->sint[1] = (&var->fallback)->sint[1]; + } + }); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + return res; +} + +#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ +int ethr_dw_atomic_cmpxchg_rb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_dw_atomic_cmpxchg_rb__(var, val, old_val); +} +#endif + +int ETHR_DW_ATOMIC_FUNC__(cmpxchg_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val) +{ + int res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + res = ethr_dw_atomic_cmpxchg_wb__(var, val, old_val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_StoreStore); + res = amc_cmpxchg(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint, old_val->sint); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), + { + res = ((&var->fallback)->sint[0] == old_val->sint[0] && (&var->fallback)->sint[1] == old_val->sint[1]); + if (res) { + (&var->fallback)->sint[0] = val->sint[0]; + (&var->fallback)->sint[1] = val->sint[1]; + } + else { + old_val->sint[0] = (&var->fallback)->sint[0]; + old_val->sint[1] = (&var->fallback)->sint[1]; + } + }); +#endif + return res; +} + +#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ +int ethr_dw_atomic_cmpxchg_wb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_dw_atomic_cmpxchg_wb__(var, val, old_val); +} +#endif + +int ETHR_DW_ATOMIC_FUNC__(cmpxchg_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val) +{ + int res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + res = ethr_dw_atomic_cmpxchg_acqb__(var, val, old_val); +#elif defined(ETHR_AMC_FALLBACK__) + res = amc_cmpxchg(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint, old_val->sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), + { + res = ((&var->fallback)->sint[0] == old_val->sint[0] && (&var->fallback)->sint[1] == old_val->sint[1]); + if (res) { + (&var->fallback)->sint[0] = val->sint[0]; + (&var->fallback)->sint[1] = val->sint[1]; + } + else { + old_val->sint[0] = (&var->fallback)->sint[0]; + old_val->sint[1] = (&var->fallback)->sint[1]; + } + }); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; +} + +#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ +int ethr_dw_atomic_cmpxchg_acqb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_dw_atomic_cmpxchg_acqb__(var, val, old_val); +} +#endif + +int ETHR_DW_ATOMIC_FUNC__(cmpxchg_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val) +{ + int res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + res = ethr_dw_atomic_cmpxchg_relb__(var, val, old_val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = amc_cmpxchg(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint, old_val->sint); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), + { + res = ((&var->fallback)->sint[0] == old_val->sint[0] && (&var->fallback)->sint[1] == old_val->sint[1]); + if (res) { + (&var->fallback)->sint[0] = val->sint[0]; + (&var->fallback)->sint[1] = val->sint[1]; + } + else { + old_val->sint[0] = (&var->fallback)->sint[0]; + old_val->sint[1] = (&var->fallback)->sint[1]; + } + }); +#endif + return res; +} + +#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ +int ethr_dw_atomic_cmpxchg_relb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_dw_atomic_cmpxchg_relb__(var, val, old_val); +} +#endif + +int ETHR_DW_ATOMIC_FUNC__(cmpxchg_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val) +{ + int res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + res = ethr_dw_atomic_cmpxchg_mb__(var, val, old_val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = amc_cmpxchg(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint, old_val->sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), + { + res = ((&var->fallback)->sint[0] == old_val->sint[0] && (&var->fallback)->sint[1] == old_val->sint[1]); + if (res) { + (&var->fallback)->sint[0] = val->sint[0]; + (&var->fallback)->sint[1] = val->sint[1]; + } + else { + old_val->sint[0] = (&var->fallback)->sint[0]; + old_val->sint[1] = (&var->fallback)->sint[1]; + } + }); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; +} + +#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ +int ethr_dw_atomic_cmpxchg_mb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_dw_atomic_cmpxchg_mb__(var, val, old_val); +} +#endif + + +/* -- set() -- */ + + +void ETHR_DW_ATOMIC_FUNC__(set)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + ethr_dw_atomic_set__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + amc_set(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]); +#endif + +} + +#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ +void ethr_dw_atomic_set(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_dw_atomic_set__(var, val); +} +#endif + +void ETHR_DW_ATOMIC_FUNC__(set_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + ethr_dw_atomic_set_rb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + amc_set(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + +} + +#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ +void ethr_dw_atomic_set_rb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_dw_atomic_set_rb__(var, val); +} +#endif + +void ETHR_DW_ATOMIC_FUNC__(set_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + ethr_dw_atomic_set_wb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_StoreStore); + amc_set(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]); +#endif + +} + +#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ +void ethr_dw_atomic_set_wb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_dw_atomic_set_wb__(var, val); +} +#endif + +void ETHR_DW_ATOMIC_FUNC__(set_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + ethr_dw_atomic_set_acqb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + amc_set(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#endif + +} + +#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ +void ethr_dw_atomic_set_acqb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_dw_atomic_set_acqb__(var, val); +} +#endif + +void ETHR_DW_ATOMIC_FUNC__(set_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + ethr_dw_atomic_set_relb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + amc_set(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint); +#else + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]); +#endif + +} + +#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ +void ethr_dw_atomic_set_relb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_dw_atomic_set_relb__(var, val); +} +#endif + +void ETHR_DW_ATOMIC_FUNC__(set_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + ethr_dw_atomic_set_mb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + amc_set(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#endif + +} + +#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ +void ethr_dw_atomic_set_mb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_dw_atomic_set_mb__(var, val); +} +#endif + + +/* -- read() -- */ + + +void ETHR_DW_ATOMIC_FUNC__(read)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + ethr_dw_atomic_read__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + amc_read(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), val->sint[0] = (&var->fallback)->sint[0]; val->sint[1] = (&var->fallback)->sint[1]); +#endif + +} + +#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ +void ethr_dw_atomic_read(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_dw_atomic_read__(var, val); +} +#endif + +void ETHR_DW_ATOMIC_FUNC__(read_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + ethr_dw_atomic_read_rb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + amc_read(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), val->sint[0] = (&var->fallback)->sint[0]; val->sint[1] = (&var->fallback)->sint[1]); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + +} + +#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ +void ethr_dw_atomic_read_rb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_dw_atomic_read_rb__(var, val); +} +#endif + +void ETHR_DW_ATOMIC_FUNC__(read_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + ethr_dw_atomic_read_wb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_StoreStore); + amc_read(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), val->sint[0] = (&var->fallback)->sint[0]; val->sint[1] = (&var->fallback)->sint[1]); +#endif + +} + +#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ +void ethr_dw_atomic_read_wb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_dw_atomic_read_wb__(var, val); +} +#endif + +void ETHR_DW_ATOMIC_FUNC__(read_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + ethr_dw_atomic_read_acqb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + amc_read(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint); + ETHR_MEMBAR(ETHR_LoadStore); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), val->sint[0] = (&var->fallback)->sint[0]; val->sint[1] = (&var->fallback)->sint[1]); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#endif + +} + +#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ +void ethr_dw_atomic_read_acqb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_dw_atomic_read_acqb__(var, val); +} +#endif + +void ETHR_DW_ATOMIC_FUNC__(read_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + ethr_dw_atomic_read_relb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + amc_read(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), val->sint[0] = (&var->fallback)->sint[0]; val->sint[1] = (&var->fallback)->sint[1]); +#endif + +} + +#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ +void ethr_dw_atomic_read_relb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_dw_atomic_read_relb__(var, val); +} +#endif + +void ETHR_DW_ATOMIC_FUNC__(read_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + ethr_dw_atomic_read_mb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + amc_read(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint); + ETHR_MEMBAR(ETHR_LoadStore); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), val->sint[0] = (&var->fallback)->sint[0]; val->sint[1] = (&var->fallback)->sint[1]); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#endif + +} + +#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ +void ethr_dw_atomic_read_mb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_dw_atomic_read_mb__(var, val); +} +#endif + + +/* -- init() -- */ + + +void ETHR_DW_ATOMIC_FUNC__(init)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + ethr_dw_atomic_init__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + amc_init(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]); +#endif + +} + +#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ +void ethr_dw_atomic_init(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(var); + ethr_dw_atomic_init__(var, val); +} +#endif + +void ETHR_DW_ATOMIC_FUNC__(init_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + ethr_dw_atomic_init_rb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + amc_init(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + +} + +#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ +void ethr_dw_atomic_init_rb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(var); + ethr_dw_atomic_init_rb__(var, val); +} +#endif + +void ETHR_DW_ATOMIC_FUNC__(init_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + ethr_dw_atomic_init_wb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_StoreStore); + amc_init(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]); +#endif + +} + +#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ +void ethr_dw_atomic_init_wb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(var); + ethr_dw_atomic_init_wb__(var, val); +} +#endif + +void ETHR_DW_ATOMIC_FUNC__(init_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + ethr_dw_atomic_init_acqb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + amc_init(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + +} + +#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ +void ethr_dw_atomic_init_acqb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(var); + ethr_dw_atomic_init_acqb__(var, val); +} +#endif + +void ETHR_DW_ATOMIC_FUNC__(init_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + ethr_dw_atomic_init_relb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + amc_init(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]); +#endif + +} + +#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ +void ethr_dw_atomic_init_relb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(var); + ethr_dw_atomic_init_relb__(var, val); +} +#endif + +void ETHR_DW_ATOMIC_FUNC__(init_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + ethr_dw_atomic_init_mb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + amc_init(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + +} + +#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ +void ethr_dw_atomic_init_mb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(var); + ethr_dw_atomic_init_mb__(var, val); +} +#endif + + +/* ---------- Word size atomic implementation ---------- */ + + + + +/* --- addr() --- */ + +ethr_sint_t *ethr_atomic_addr(ethr_atomic_t *var) +{ + ethr_sint_t *res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_addr__(var); +#elif defined(ETHR_AMC_FALLBACK__) + res = (ethr_sint_t *) (var)->sint; +#else + res = (ethr_sint_t *) var; +#endif + return res; +} + + +/* -- cmpxchg() -- */ + + +ethr_sint_t ethr_atomic_cmpxchg(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_cmpxchg__(var, val, old_val); +#elif defined(ETHR_AMC_FALLBACK__) + res = old_val; + (void) amc_cmpxchg(&var->amc, 0, &var->sint, &val, &res); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var)); +#endif + return res; +} + +ethr_sint_t ethr_atomic_cmpxchg_rb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_cmpxchg_rb__(var, val, old_val); +#elif defined(ETHR_AMC_FALLBACK__) + res = old_val; + (void) amc_cmpxchg(&var->amc, 0, &var->sint, &val, &res); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var)); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + return res; +} + +ethr_sint_t ethr_atomic_cmpxchg_wb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_cmpxchg_wb__(var, val, old_val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_StoreStore); + res = old_val; + (void) amc_cmpxchg(&var->amc, 0, &var->sint, &val, &res); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var)); +#endif + return res; +} + +ethr_sint_t ethr_atomic_cmpxchg_acqb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_cmpxchg_acqb__(var, val, old_val); +#elif defined(ETHR_AMC_FALLBACK__) + res = old_val; + (void) amc_cmpxchg(&var->amc, 0, &var->sint, &val, &res); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var)); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; +} + +ethr_sint_t ethr_atomic_cmpxchg_relb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_cmpxchg_relb__(var, val, old_val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = old_val; + (void) amc_cmpxchg(&var->amc, 0, &var->sint, &val, &res); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var)); +#endif + return res; +} + +ethr_sint_t ethr_atomic_cmpxchg_mb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_cmpxchg_mb__(var, val, old_val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = old_val; + (void) amc_cmpxchg(&var->amc, 0, &var->sint, &val, &res); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var)); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; +} + + +/* -- xchg() -- */ + + +ethr_sint_t ethr_atomic_xchg(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_xchg__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint = val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val); +#endif + return res; +} + +ethr_sint_t ethr_atomic_xchg_rb(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_xchg_rb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint = val); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + return res; +} + +ethr_sint_t ethr_atomic_xchg_wb(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_xchg_wb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint = val); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val); +#endif + return res; +} + +ethr_sint_t ethr_atomic_xchg_acqb(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_xchg_acqb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint = val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; +} + +ethr_sint_t ethr_atomic_xchg_relb(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_xchg_relb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint = val); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val); +#endif + return res; +} + +ethr_sint_t ethr_atomic_xchg_mb(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_xchg_mb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint = val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; +} + + +/* -- set() -- */ + + +void ethr_atomic_set(ethr_atomic_t *var, ethr_sint_t val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + ethr_atomic_set__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + amc_set(&var->amc, 0, &var->sint, &val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val); +#endif + +} + +void ethr_atomic_set_rb(ethr_atomic_t *var, ethr_sint_t val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + ethr_atomic_set_rb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + amc_set(&var->amc, 0, &var->sint, &val); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + +} + +void ethr_atomic_set_wb(ethr_atomic_t *var, ethr_sint_t val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + ethr_atomic_set_wb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_StoreStore); + amc_set(&var->amc, 0, &var->sint, &val); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val); +#endif + +} + +void ethr_atomic_set_acqb(ethr_atomic_t *var, ethr_sint_t val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + ethr_atomic_set_acqb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + amc_set(&var->amc, 0, &var->sint, &val); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#endif + +} + +void ethr_atomic_set_relb(ethr_atomic_t *var, ethr_sint_t val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + ethr_atomic_set_relb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + amc_set(&var->amc, 0, &var->sint, &val); +#else + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val); +#endif + +} + +void ethr_atomic_set_mb(ethr_atomic_t *var, ethr_sint_t val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + ethr_atomic_set_mb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + amc_set(&var->amc, 0, &var->sint, &val); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#endif + +} + + +/* -- init() -- */ + + +void ethr_atomic_init(ethr_atomic_t *var, ethr_sint_t val) +{ + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + ethr_atomic_init__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + amc_init(&var->amc, 0, &var->sint, &val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val); +#endif + +} + +void ethr_atomic_init_rb(ethr_atomic_t *var, ethr_sint_t val) +{ + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + ethr_atomic_init_rb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + amc_init(&var->amc, 0, &var->sint, &val); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + +} + +void ethr_atomic_init_wb(ethr_atomic_t *var, ethr_sint_t val) +{ + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + ethr_atomic_init_wb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_StoreStore); + amc_init(&var->amc, 0, &var->sint, &val); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val); +#endif + +} + +void ethr_atomic_init_acqb(ethr_atomic_t *var, ethr_sint_t val) +{ + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + ethr_atomic_init_acqb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + amc_init(&var->amc, 0, &var->sint, &val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + +} + +void ethr_atomic_init_relb(ethr_atomic_t *var, ethr_sint_t val) +{ + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + ethr_atomic_init_relb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + amc_init(&var->amc, 0, &var->sint, &val); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val); +#endif + +} + +void ethr_atomic_init_mb(ethr_atomic_t *var, ethr_sint_t val) +{ + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + ethr_atomic_init_mb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + amc_init(&var->amc, 0, &var->sint, &val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + +} + + +/* -- add_read() -- */ + + +ethr_sint_t ethr_atomic_add_read(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_add_read__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val; res = var->sint); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var); +#endif + return res; +} + +ethr_sint_t ethr_atomic_add_read_rb(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_add_read_rb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val; res = var->sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + return res; +} + +ethr_sint_t ethr_atomic_add_read_wb(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_add_read_wb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val; res = var->sint); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var); +#endif + return res; +} + +ethr_sint_t ethr_atomic_add_read_acqb(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_add_read_acqb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val; res = var->sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; +} + +ethr_sint_t ethr_atomic_add_read_relb(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_add_read_relb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val; res = var->sint); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var); +#endif + return res; +} + +ethr_sint_t ethr_atomic_add_read_mb(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_add_read_mb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val; res = var->sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; +} + + +/* -- read() -- */ + + +ethr_sint_t ethr_atomic_read(ethr_atomic_t *var) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_read__(var); +#elif defined(ETHR_AMC_FALLBACK__) + amc_read(&var->amc, 0, &var->sint, &res); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var); +#endif + return res; +} + +ethr_sint_t ethr_atomic_read_rb(ethr_atomic_t *var) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_read_rb__(var); +#elif defined(ETHR_AMC_FALLBACK__) + amc_read(&var->amc, 0, &var->sint, &res); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + return res; +} + +ethr_sint_t ethr_atomic_read_wb(ethr_atomic_t *var) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_read_wb__(var); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_StoreStore); + amc_read(&var->amc, 0, &var->sint, &res); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var); +#endif + return res; +} + +ethr_sint_t ethr_atomic_read_acqb(ethr_atomic_t *var) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_read_acqb__(var); +#elif defined(ETHR_AMC_FALLBACK__) + amc_read(&var->amc, 0, &var->sint, &res); + ETHR_MEMBAR(ETHR_LoadStore); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#endif + return res; +} + +ethr_sint_t ethr_atomic_read_relb(ethr_atomic_t *var) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_read_relb__(var); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + amc_read(&var->amc, 0, &var->sint, &res); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var); +#endif + return res; +} + +ethr_sint_t ethr_atomic_read_mb(ethr_atomic_t *var) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_read_mb__(var); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + amc_read(&var->amc, 0, &var->sint, &res); + ETHR_MEMBAR(ETHR_LoadStore); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#endif + return res; +} + + +/* -- inc_read() -- */ + + +ethr_sint_t ethr_atomic_inc_read(ethr_atomic_t *var) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_inc_read__(var); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = ++(var->sint)); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var)); +#endif + return res; +} + +ethr_sint_t ethr_atomic_inc_read_rb(ethr_atomic_t *var) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_inc_read_rb__(var); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = ++(var->sint)); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var)); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + return res; +} + +ethr_sint_t ethr_atomic_inc_read_wb(ethr_atomic_t *var) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_inc_read_wb__(var); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = ++(var->sint)); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var)); +#endif + return res; +} + +ethr_sint_t ethr_atomic_inc_read_acqb(ethr_atomic_t *var) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_inc_read_acqb__(var); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = ++(var->sint)); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var)); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; +} + +ethr_sint_t ethr_atomic_inc_read_relb(ethr_atomic_t *var) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_inc_read_relb__(var); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = ++(var->sint)); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var)); +#endif + return res; +} + +ethr_sint_t ethr_atomic_inc_read_mb(ethr_atomic_t *var) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_inc_read_mb__(var); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = ++(var->sint)); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var)); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; +} + + +/* -- dec_read() -- */ + + +ethr_sint_t ethr_atomic_dec_read(ethr_atomic_t *var) { + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - return ethr_atomic_addr__(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_dec_read__(var); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = --(var->sint)); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var)); +#endif + return res; } -void -ethr_atomic_init(ethr_atomic_t *var, ethr_sint_t i) +ethr_sint_t ethr_atomic_dec_read_rb(ethr_atomic_t *var) { + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - ethr_atomic_init__(var, i); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_dec_read_rb__(var); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = --(var->sint)); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var)); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + return res; } -void -ethr_atomic_set(ethr_atomic_t *var, ethr_sint_t i) +ethr_sint_t ethr_atomic_dec_read_wb(ethr_atomic_t *var) { + ethr_sint_t res; ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - ethr_atomic_set__(var, i); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_dec_read_wb__(var); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = --(var->sint)); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var)); +#endif + return res; } -ethr_sint_t -ethr_atomic_read(ethr_atomic_t *var) +ethr_sint_t ethr_atomic_dec_read_acqb(ethr_atomic_t *var) { + ethr_sint_t res; ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - return ethr_atomic_read__(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_dec_read_acqb__(var); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = --(var->sint)); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var)); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; } -ethr_sint_t -ethr_atomic_add_read(ethr_atomic_t *var, ethr_sint_t incr) +ethr_sint_t ethr_atomic_dec_read_relb(ethr_atomic_t *var) { + ethr_sint_t res; ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - return ethr_atomic_add_read__(var, incr); -} +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_dec_read_relb__(var); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = --(var->sint)); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var)); +#endif + return res; +} -ethr_sint_t -ethr_atomic_inc_read(ethr_atomic_t *var) +ethr_sint_t ethr_atomic_dec_read_mb(ethr_atomic_t *var) { + ethr_sint_t res; ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - return ethr_atomic_inc_read__(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_dec_read_mb__(var); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = --(var->sint)); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var)); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; } -ethr_sint_t -ethr_atomic_dec_read(ethr_atomic_t *var) + +/* -- add() -- */ + + +void ethr_atomic_add(ethr_atomic_t *var, ethr_sint_t val) { ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - return ethr_atomic_dec_read__(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + ethr_atomic_add__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val); +#endif + +} + +void ethr_atomic_add_rb(ethr_atomic_t *var, ethr_sint_t val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + ethr_atomic_add_rb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + } -void -ethr_atomic_add(ethr_atomic_t *var, ethr_sint_t incr) +void ethr_atomic_add_wb(ethr_atomic_t *var, ethr_sint_t val) { ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - ethr_atomic_add__(var, incr); -} - -void -ethr_atomic_inc(ethr_atomic_t *var) +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + ethr_atomic_add_wb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val); +#endif + +} + +void ethr_atomic_add_acqb(ethr_atomic_t *var, ethr_sint_t val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + ethr_atomic_add_acqb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + +} + +void ethr_atomic_add_relb(ethr_atomic_t *var, ethr_sint_t val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + ethr_atomic_add_relb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val); +#endif + +} + +void ethr_atomic_add_mb(ethr_atomic_t *var, ethr_sint_t val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + ethr_atomic_add_mb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + +} + + +/* -- inc() -- */ + + +void ethr_atomic_inc(ethr_atomic_t *var) { ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) ethr_atomic_inc__(var); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_AMC_MODIFICATION_OPS__(&var->amc, ++(var->sint)); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var)); +#endif + } -void -ethr_atomic_dec(ethr_atomic_t *var) +void ethr_atomic_inc_rb(ethr_atomic_t *var) { ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - ethr_atomic_dec__(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + ethr_atomic_inc_rb__(var); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_AMC_MODIFICATION_OPS__(&var->amc, ++(var->sint)); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var)); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + } -ethr_sint_t -ethr_atomic_read_band(ethr_atomic_t *var, ethr_sint_t mask) +void ethr_atomic_inc_wb(ethr_atomic_t *var) { ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - return ethr_atomic_read_band__(var, mask); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + ethr_atomic_inc_wb__(var); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_AMC_MODIFICATION_OPS__(&var->amc, ++(var->sint)); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var)); +#endif + } -ethr_sint_t -ethr_atomic_read_bor(ethr_atomic_t *var, ethr_sint_t mask) +void ethr_atomic_inc_acqb(ethr_atomic_t *var) { ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - return ethr_atomic_read_bor__(var, mask); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + ethr_atomic_inc_acqb__(var); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_AMC_MODIFICATION_OPS__(&var->amc, ++(var->sint)); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var)); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + } -ethr_sint_t -ethr_atomic_xchg(ethr_atomic_t *var, ethr_sint_t new) +void ethr_atomic_inc_relb(ethr_atomic_t *var) { ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - return ethr_atomic_xchg__(var, new); -} +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + ethr_atomic_inc_relb__(var); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_AMC_MODIFICATION_OPS__(&var->amc, ++(var->sint)); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var)); +#endif + +} -ethr_sint_t -ethr_atomic_cmpxchg(ethr_atomic_t *var, ethr_sint_t new, ethr_sint_t expected) +void ethr_atomic_inc_mb(ethr_atomic_t *var) { ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - return ethr_atomic_cmpxchg__(var, new, expected); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + ethr_atomic_inc_mb__(var); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_AMC_MODIFICATION_OPS__(&var->amc, ++(var->sint)); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var)); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + } -ethr_sint_t -ethr_atomic_read_acqb(ethr_atomic_t *var) + +/* -- dec() -- */ + + +void ethr_atomic_dec(ethr_atomic_t *var) { ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - return ethr_atomic_read_acqb__(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + ethr_atomic_dec__(var); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_AMC_MODIFICATION_OPS__(&var->amc, --(var->sint)); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var)); +#endif + } -ethr_sint_t -ethr_atomic_inc_read_acqb(ethr_atomic_t *var) +void ethr_atomic_dec_rb(ethr_atomic_t *var) { ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - return ethr_atomic_inc_read_acqb__(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + ethr_atomic_dec_rb__(var); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_AMC_MODIFICATION_OPS__(&var->amc, --(var->sint)); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var)); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + } -void -ethr_atomic_set_relb(ethr_atomic_t *var, ethr_sint_t i) +void ethr_atomic_dec_wb(ethr_atomic_t *var) { ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - ethr_atomic_set_relb__(var, i); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + ethr_atomic_dec_wb__(var); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_AMC_MODIFICATION_OPS__(&var->amc, --(var->sint)); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var)); +#endif + } -void -ethr_atomic_dec_relb(ethr_atomic_t *var) +void ethr_atomic_dec_acqb(ethr_atomic_t *var) { ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + ethr_atomic_dec_acqb__(var); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_AMC_MODIFICATION_OPS__(&var->amc, --(var->sint)); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var)); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + +} + +void ethr_atomic_dec_relb(ethr_atomic_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) ethr_atomic_dec_relb__(var); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_AMC_MODIFICATION_OPS__(&var->amc, --(var->sint)); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var)); +#endif + } -ethr_sint_t -ethr_atomic_dec_read_relb(ethr_atomic_t *var) +void ethr_atomic_dec_mb(ethr_atomic_t *var) { ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - return ethr_atomic_dec_read_relb__(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + ethr_atomic_dec_mb__(var); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_AMC_MODIFICATION_OPS__(&var->amc, --(var->sint)); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var)); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + } -ethr_sint_t -ethr_atomic_cmpxchg_acqb(ethr_atomic_t *var, ethr_sint_t new, ethr_sint_t exp) + +/* -- read_band() -- */ + + +ethr_sint_t ethr_atomic_read_band(ethr_atomic_t *var, ethr_sint_t val) { + ethr_sint_t res; ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - return ethr_atomic_cmpxchg_acqb__(var, new, exp); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_read_band__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint &= val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val); +#endif + return res; } -ethr_sint_t -ethr_atomic_cmpxchg_relb(ethr_atomic_t *var, ethr_sint_t new, ethr_sint_t exp) +ethr_sint_t ethr_atomic_read_band_rb(ethr_atomic_t *var, ethr_sint_t val) { + ethr_sint_t res; ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - return ethr_atomic_cmpxchg_relb__(var, new, exp); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_read_band_rb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint &= val); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + return res; } +ethr_sint_t ethr_atomic_read_band_wb(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_read_band_wb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint &= val); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val); +#endif + return res; +} + +ethr_sint_t ethr_atomic_read_band_acqb(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_read_band_acqb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint &= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; +} + +ethr_sint_t ethr_atomic_read_band_relb(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_read_band_relb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint &= val); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val); +#endif + return res; +} + +ethr_sint_t ethr_atomic_read_band_mb(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_read_band_mb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint &= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; +} + + +/* -- read_bor() -- */ + + +ethr_sint_t ethr_atomic_read_bor(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_read_bor__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint |= val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val); +#endif + return res; +} + +ethr_sint_t ethr_atomic_read_bor_rb(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_read_bor_rb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint |= val); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + return res; +} + +ethr_sint_t ethr_atomic_read_bor_wb(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_read_bor_wb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint |= val); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val); +#endif + return res; +} + +ethr_sint_t ethr_atomic_read_bor_acqb(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_read_bor_acqb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint |= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; +} + +ethr_sint_t ethr_atomic_read_bor_relb(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_read_bor_relb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint |= val); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val); +#endif + return res; +} + +ethr_sint_t ethr_atomic_read_bor_mb(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_read_bor_mb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint |= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; +} + + +/* ---------- 32-bit atomic implementation ---------- */ + + + + +/* --- addr() --- */ + +ethr_sint32_t *ethr_atomic32_addr(ethr_atomic32_t *var) +{ + ethr_sint32_t *res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_addr__(var); + +#else + res = (ethr_sint32_t *) var; +#endif + return res; +} + + +/* -- cmpxchg() -- */ + + +ethr_sint32_t ethr_atomic32_cmpxchg(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_cmpxchg__(var, val, old_val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var)); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_cmpxchg_rb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_cmpxchg_rb__(var, val, old_val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var)); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_cmpxchg_wb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_cmpxchg_wb__(var, val, old_val); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var)); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_cmpxchg_acqb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_cmpxchg_acqb__(var, val, old_val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var)); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_cmpxchg_relb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_cmpxchg_relb__(var, val, old_val); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var)); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_cmpxchg_mb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_cmpxchg_mb__(var, val, old_val); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var)); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; +} + + +/* -- xchg() -- */ + + +ethr_sint32_t ethr_atomic32_xchg(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_xchg__(var, val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_xchg_rb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_xchg_rb__(var, val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_xchg_wb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_xchg_wb__(var, val); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_xchg_acqb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_xchg_acqb__(var, val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_xchg_relb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_xchg_relb__(var, val); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_xchg_mb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_xchg_mb__(var, val); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; +} + + +/* -- set() -- */ + + +void ethr_atomic32_set(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + ethr_atomic32_set__(var, val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val); +#endif + +} + +void ethr_atomic32_set_rb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + ethr_atomic32_set_rb__(var, val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + +} + +void ethr_atomic32_set_wb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + ethr_atomic32_set_wb__(var, val); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val); +#endif + +} + +void ethr_atomic32_set_acqb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + ethr_atomic32_set_acqb__(var, val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#endif + +} + +void ethr_atomic32_set_relb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + ethr_atomic32_set_relb__(var, val); +#else + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val); +#endif + +} + +void ethr_atomic32_set_mb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + ethr_atomic32_set_mb__(var, val); +#else + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#endif + +} + + +/* -- init() -- */ + + +void ethr_atomic32_init(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + ethr_atomic32_init__(var, val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val); +#endif + +} + +void ethr_atomic32_init_rb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + ethr_atomic32_init_rb__(var, val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + +} + +void ethr_atomic32_init_wb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + ethr_atomic32_init_wb__(var, val); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val); +#endif + +} + +void ethr_atomic32_init_acqb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + ethr_atomic32_init_acqb__(var, val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + +} + +void ethr_atomic32_init_relb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + ethr_atomic32_init_relb__(var, val); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val); +#endif + +} + +void ethr_atomic32_init_mb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + ethr_atomic32_init_mb__(var, val); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + +} + + +/* -- add_read() -- */ + + +ethr_sint32_t ethr_atomic32_add_read(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_add_read__(var, val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_add_read_rb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_add_read_rb__(var, val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_add_read_wb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_add_read_wb__(var, val); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_add_read_acqb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_add_read_acqb__(var, val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_add_read_relb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_add_read_relb__(var, val); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_add_read_mb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_add_read_mb__(var, val); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; +} + + +/* -- read() -- */ + + +ethr_sint32_t ethr_atomic32_read(ethr_atomic32_t *var) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_read__(var); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_read_rb(ethr_atomic32_t *var) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_read_rb__(var); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_read_wb(ethr_atomic32_t *var) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_read_wb__(var); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_read_acqb(ethr_atomic32_t *var) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_read_acqb__(var); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_read_relb(ethr_atomic32_t *var) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_read_relb__(var); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_read_mb(ethr_atomic32_t *var) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_read_mb__(var); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#endif + return res; +} + + +/* -- inc_read() -- */ + + +ethr_sint32_t ethr_atomic32_inc_read(ethr_atomic32_t *var) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_inc_read__(var); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var)); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_inc_read_rb(ethr_atomic32_t *var) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_inc_read_rb__(var); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var)); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_inc_read_wb(ethr_atomic32_t *var) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_inc_read_wb__(var); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var)); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_inc_read_acqb(ethr_atomic32_t *var) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_inc_read_acqb__(var); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var)); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_inc_read_relb(ethr_atomic32_t *var) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_inc_read_relb__(var); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var)); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_inc_read_mb(ethr_atomic32_t *var) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_inc_read_mb__(var); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var)); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; +} + + +/* -- dec_read() -- */ -/* - * --- 32-bit atomics --------------------------------------------------------- - */ -ethr_sint32_t * -ethr_atomic32_addr(ethr_atomic32_t *var) +ethr_sint32_t ethr_atomic32_dec_read(ethr_atomic32_t *var) { + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - return ethr_atomic32_addr__(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_dec_read__(var); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var)); +#endif + return res; } -void -ethr_atomic32_init(ethr_atomic32_t *var, ethr_sint32_t i) +ethr_sint32_t ethr_atomic32_dec_read_rb(ethr_atomic32_t *var) { + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - ethr_atomic32_init__(var, i); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_dec_read_rb__(var); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var)); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + return res; } -void -ethr_atomic32_set(ethr_atomic32_t *var, ethr_sint32_t i) +ethr_sint32_t ethr_atomic32_dec_read_wb(ethr_atomic32_t *var) { + ethr_sint32_t res; ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - ethr_atomic32_set__(var, i); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_dec_read_wb__(var); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var)); +#endif + return res; } -ethr_sint32_t -ethr_atomic32_read(ethr_atomic32_t *var) +ethr_sint32_t ethr_atomic32_dec_read_acqb(ethr_atomic32_t *var) { + ethr_sint32_t res; ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - return ethr_atomic32_read__(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_dec_read_acqb__(var); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var)); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; } +ethr_sint32_t ethr_atomic32_dec_read_relb(ethr_atomic32_t *var) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_dec_read_relb__(var); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var)); +#endif + return res; +} -ethr_sint32_t -ethr_atomic32_add_read(ethr_atomic32_t *var, ethr_sint32_t incr) +ethr_sint32_t ethr_atomic32_dec_read_mb(ethr_atomic32_t *var) { + ethr_sint32_t res; ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - return ethr_atomic32_add_read__(var, incr); -} +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_dec_read_mb__(var); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var)); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; +} + + +/* -- add() -- */ -ethr_sint32_t -ethr_atomic32_inc_read(ethr_atomic32_t *var) + +void ethr_atomic32_add(ethr_atomic32_t *var, ethr_sint32_t val) { ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - return ethr_atomic32_inc_read__(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + ethr_atomic32_add__(var, val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val); +#endif + } -ethr_sint32_t -ethr_atomic32_dec_read(ethr_atomic32_t *var) +void ethr_atomic32_add_rb(ethr_atomic32_t *var, ethr_sint32_t val) { ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - return ethr_atomic32_dec_read__(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + ethr_atomic32_add_rb__(var, val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + +} + +void ethr_atomic32_add_wb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + ethr_atomic32_add_wb__(var, val); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val); +#endif + +} + +void ethr_atomic32_add_acqb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + ethr_atomic32_add_acqb__(var, val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + +} + +void ethr_atomic32_add_relb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + ethr_atomic32_add_relb__(var, val); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val); +#endif + } -void -ethr_atomic32_add(ethr_atomic32_t *var, ethr_sint32_t incr) +void ethr_atomic32_add_mb(ethr_atomic32_t *var, ethr_sint32_t val) { ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - ethr_atomic32_add__(var, incr); -} - -void -ethr_atomic32_inc(ethr_atomic32_t *var) +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + ethr_atomic32_add_mb__(var, val); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + +} + + +/* -- inc() -- */ + + +void ethr_atomic32_inc(ethr_atomic32_t *var) { ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) ethr_atomic32_inc__(var); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var)); +#endif + } -void -ethr_atomic32_dec(ethr_atomic32_t *var) +void ethr_atomic32_inc_rb(ethr_atomic32_t *var) { ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - ethr_atomic32_dec__(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + ethr_atomic32_inc_rb__(var); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var)); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + } -ethr_sint32_t -ethr_atomic32_read_band(ethr_atomic32_t *var, ethr_sint32_t mask) +void ethr_atomic32_inc_wb(ethr_atomic32_t *var) { ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - return ethr_atomic32_read_band__(var, mask); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + ethr_atomic32_inc_wb__(var); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var)); +#endif + } -ethr_sint32_t -ethr_atomic32_read_bor(ethr_atomic32_t *var, ethr_sint32_t mask) +void ethr_atomic32_inc_acqb(ethr_atomic32_t *var) { ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - return ethr_atomic32_read_bor__(var, mask); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + ethr_atomic32_inc_acqb__(var); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var)); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + } -ethr_sint32_t -ethr_atomic32_xchg(ethr_atomic32_t *var, ethr_sint32_t new) +void ethr_atomic32_inc_relb(ethr_atomic32_t *var) { ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - return ethr_atomic32_xchg__(var, new); -} +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + ethr_atomic32_inc_relb__(var); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var)); +#endif -ethr_sint32_t -ethr_atomic32_cmpxchg(ethr_atomic32_t *var, - ethr_sint32_t new, - ethr_sint32_t expected) +} + +void ethr_atomic32_inc_mb(ethr_atomic32_t *var) { ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - return ethr_atomic32_cmpxchg__(var, new, expected); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + ethr_atomic32_inc_mb__(var); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var)); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + } -ethr_sint32_t -ethr_atomic32_read_acqb(ethr_atomic32_t *var) + +/* -- dec() -- */ + + +void ethr_atomic32_dec(ethr_atomic32_t *var) { ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - return ethr_atomic32_read_acqb__(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + ethr_atomic32_dec__(var); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var)); +#endif + +} + +void ethr_atomic32_dec_rb(ethr_atomic32_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + ethr_atomic32_dec_rb__(var); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var)); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + } -ethr_sint32_t -ethr_atomic32_inc_read_acqb(ethr_atomic32_t *var) +void ethr_atomic32_dec_wb(ethr_atomic32_t *var) { ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - return ethr_atomic32_inc_read_acqb__(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + ethr_atomic32_dec_wb__(var); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var)); +#endif + } -void -ethr_atomic32_set_relb(ethr_atomic32_t *var, ethr_sint32_t i) +void ethr_atomic32_dec_acqb(ethr_atomic32_t *var) { ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - ethr_atomic32_set_relb__(var, i); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + ethr_atomic32_dec_acqb__(var); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var)); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + } -void -ethr_atomic32_dec_relb(ethr_atomic32_t *var) +void ethr_atomic32_dec_relb(ethr_atomic32_t *var) { ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) ethr_atomic32_dec_relb__(var); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var)); +#endif + +} + +void ethr_atomic32_dec_mb(ethr_atomic32_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + ethr_atomic32_dec_mb__(var); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var)); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + +} + + +/* -- read_band() -- */ + + +ethr_sint32_t ethr_atomic32_read_band(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_read_band__(var, val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_read_band_rb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_read_band_rb__(var, val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_read_band_wb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_read_band_wb__(var, val); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_read_band_acqb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_read_band_acqb__(var, val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_read_band_relb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_read_band_relb__(var, val); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_read_band_mb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_read_band_mb__(var, val); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; +} + + +/* -- read_bor() -- */ + + +ethr_sint32_t ethr_atomic32_read_bor(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_read_bor__(var, val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_read_bor_rb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_read_bor_rb__(var, val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + return res; } -ethr_sint32_t -ethr_atomic32_dec_read_relb(ethr_atomic32_t *var) +ethr_sint32_t ethr_atomic32_read_bor_wb(ethr_atomic32_t *var, ethr_sint32_t val) { + ethr_sint32_t res; ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - return ethr_atomic32_dec_read_relb__(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_read_bor_wb__(var, val); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val); +#endif + return res; } -ethr_sint32_t -ethr_atomic32_cmpxchg_acqb(ethr_atomic32_t *var, - ethr_sint32_t new, - ethr_sint32_t exp) +ethr_sint32_t ethr_atomic32_read_bor_acqb(ethr_atomic32_t *var, ethr_sint32_t val) { + ethr_sint32_t res; ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - return ethr_atomic32_cmpxchg_acqb__(var, new, exp); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_read_bor_acqb__(var, val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; } -ethr_sint32_t -ethr_atomic32_cmpxchg_relb(ethr_atomic32_t *var, - ethr_sint32_t new, - ethr_sint32_t exp) +ethr_sint32_t ethr_atomic32_read_bor_relb(ethr_atomic32_t *var, ethr_sint32_t val) { + ethr_sint32_t res; ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - return ethr_atomic32_cmpxchg_relb__(var, new, exp); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_read_bor_relb__(var, val); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_read_bor_mb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_read_bor_mb__(var, val); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; +} + + + +/* --------- Info functions --------- */ + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) +char *zero_ops[] = {NULL}; +#endif + + +static char *native_su_dw_atomic_ops[] = { +#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG + "cmpxchg", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_RB + "cmpxchg_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_WB + "cmpxchg_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_ACQB + "cmpxchg_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_RELB + "cmpxchg_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_MB + "cmpxchg_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET + "set", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_RB + "set_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_WB + "set_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_ACQB + "set_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_RELB + "set_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_MB + "set_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ + "read", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_RB + "read_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_WB + "read_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_ACQB + "read_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_RELB + "read_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_MB + "read_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT + "init", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_RB + "init_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_WB + "init_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_ACQB + "init_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_RELB + "init_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_MB + "init_mb", +#endif + NULL +}; + +char ** +ethr_native_su_dw_atomic_ops(void) +{ + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + if (!ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + return &zero_ops[0]; +#endif + return &native_su_dw_atomic_ops[0]; +} + + +static char *native_dw_atomic_ops[] = { +#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG + "cmpxchg", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_RB + "cmpxchg_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_WB + "cmpxchg_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_ACQB + "cmpxchg_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_RELB + "cmpxchg_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_MB + "cmpxchg_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET + "set", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_RB + "set_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_WB + "set_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_ACQB + "set_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_RELB + "set_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_MB + "set_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ + "read", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_RB + "read_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_WB + "read_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_ACQB + "read_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_RELB + "read_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_MB + "read_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT + "init", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_RB + "init_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_WB + "init_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_ACQB + "init_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_RELB + "init_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_MB + "init_mb", +#endif + NULL +}; + +char ** +ethr_native_dw_atomic_ops(void) +{ + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + if (!ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + return &zero_ops[0]; +#endif + return &native_dw_atomic_ops[0]; +} + + +static char *native_atomic64_ops[] = { +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG + "cmpxchg", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RB + "cmpxchg_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_WB + "cmpxchg_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_ACQB + "cmpxchg_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RELB + "cmpxchg_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_MB + "cmpxchg_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG + "xchg", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_RB + "xchg_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_WB + "xchg_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_ACQB + "xchg_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_RELB + "xchg_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_MB + "xchg_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET + "set", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RB + "set_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_WB + "set_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_ACQB + "set_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RELB + "set_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_MB + "set_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT + "init", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_RB + "init_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_WB + "init_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_ACQB + "init_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_RELB + "init_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_MB + "init_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN + "add_return", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_RB + "add_return_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_WB + "add_return_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_ACQB + "add_return_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_RELB + "add_return_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_MB + "add_return_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ + "read", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_RB + "read_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_WB + "read_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_ACQB + "read_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_RELB + "read_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_MB + "read_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN + "inc_return", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_RB + "inc_return_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_WB + "inc_return_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_ACQB + "inc_return_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_RELB + "inc_return_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_MB + "inc_return_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN + "dec_return", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_RB + "dec_return_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_WB + "dec_return_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_ACQB + "dec_return_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_RELB + "dec_return_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_MB + "dec_return_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD + "add", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RB + "add_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_WB + "add_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_ACQB + "add_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RELB + "add_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_MB + "add_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC + "inc", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RB + "inc_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_WB + "inc_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_ACQB + "inc_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RELB + "inc_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_MB + "inc_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC + "dec", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RB + "dec_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_WB + "dec_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_ACQB + "dec_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RELB + "dec_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_MB + "dec_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD + "and_retold", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_RB + "and_retold_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_WB + "and_retold_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_ACQB + "and_retold_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_RELB + "and_retold_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_MB + "and_retold_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD + "or_retold", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_RB + "or_retold_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_WB + "or_retold_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_ACQB + "or_retold_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_RELB + "or_retold_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_MB + "or_retold_mb", +#endif + NULL +}; + +char ** +ethr_native_atomic64_ops(void) +{ + + return &native_atomic64_ops[0]; } + +static char *native_atomic32_ops[] = { +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG + "cmpxchg", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_RB + "cmpxchg_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_WB + "cmpxchg_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_ACQB + "cmpxchg_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_RELB + "cmpxchg_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_MB + "cmpxchg_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG + "xchg", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_RB + "xchg_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_WB + "xchg_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_ACQB + "xchg_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_RELB + "xchg_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_MB + "xchg_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET + "set", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_RB + "set_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_WB + "set_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_ACQB + "set_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_RELB + "set_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_MB + "set_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT + "init", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_RB + "init_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_WB + "init_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_ACQB + "init_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_RELB + "init_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_MB + "init_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN + "add_return", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_RB + "add_return_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_WB + "add_return_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_ACQB + "add_return_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_RELB + "add_return_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_MB + "add_return_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ + "read", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_RB + "read_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_WB + "read_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_ACQB + "read_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_RELB + "read_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_MB + "read_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN + "inc_return", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_RB + "inc_return_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_WB + "inc_return_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_ACQB + "inc_return_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_RELB + "inc_return_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_MB + "inc_return_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN + "dec_return", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_RB + "dec_return_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_WB + "dec_return_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_ACQB + "dec_return_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_RELB + "dec_return_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_MB + "dec_return_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD + "add", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RB + "add_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_WB + "add_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_ACQB + "add_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RELB + "add_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_MB + "add_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC + "inc", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RB + "inc_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_WB + "inc_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_ACQB + "inc_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RELB + "inc_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_MB + "inc_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC + "dec", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RB + "dec_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_WB + "dec_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_ACQB + "dec_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RELB + "dec_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_MB + "dec_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD + "and_retold", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_RB + "and_retold_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_WB + "and_retold_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_ACQB + "and_retold_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_RELB + "and_retold_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_MB + "and_retold_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD + "or_retold", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_RB + "or_retold_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_WB + "or_retold_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_ACQB + "or_retold_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_RELB + "or_retold_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_MB + "or_retold_mb", +#endif + NULL +}; + +char ** +ethr_native_atomic32_ops(void) +{ + + return &native_atomic32_ops[0]; +} diff --git a/erts/lib_src/common/ethr_aux.c b/erts/lib_src/common/ethr_aux.c index 2c3e25a805..521640317e 100644 --- a/erts/lib_src/common/ethr_aux.c +++ b/erts/lib_src/common/ethr_aux.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2010. All Rights Reserved. + * Copyright Ericsson AB 2010-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -31,10 +31,6 @@ #define ETHR_INLINE_FUNC_NAME_(X) X ## __ #define ETHR_AUX_IMPL__ -#define ETHR_ATOMIC_IMPL__ /* Needed in order to pull in - native atomic implementations - for optimized fallbacks of - spinlocks and rwspinlocks */ #include "ethread.h" #include "ethr_internal.h" #include <string.h> @@ -75,10 +71,87 @@ static int main_threads; static int init_ts_event_alloc(void); +ethr_runtime_t ethr_runtime__ +#ifdef __GNUC__ +__attribute__ ((aligned (ETHR_CACHE_LINE_SIZE))) +#endif + ; + +#if defined(ETHR_X86_RUNTIME_CONF__) + +/* + * x86/x86_64 specifics shared between windows and + * pthread implementations. + */ + +#define ETHR_IS_X86_VENDOR(V, B, C, D) \ + (sizeof(V) == 13 && is_x86_vendor((V), (B), (C), (D))) + +static ETHR_INLINE int +is_x86_vendor(char *str, int ebx, int ecx, int edx) +{ + return (*((int *) &str[0]) == ebx + && *((int *) &str[sizeof(int)]) == edx + && *((int *) &str[sizeof(int)*2]) == ecx); +} + +static void +x86_init(void) +{ + int eax, ebx, ecx, edx; + + eax = ebx = ecx = edx = 0; + + ethr_x86_cpuid__(&eax, &ebx, &ecx, &edx); + + if (eax > 0 + && (ETHR_IS_X86_VENDOR("GenuineIntel", ebx, ecx, edx) + || ETHR_IS_X86_VENDOR("AuthenticAMD", ebx, ecx, edx))) { + eax = 1; + ethr_x86_cpuid__(&eax, &ebx, &ecx, &edx); + } + else { + /* + * The meaning of the feature flags for this + * vendor have not been verified. + */ + eax = ebx = ecx = edx = 0; + } + + /* + * The feature flags tested below have only been verified + * for vendors checked above. Also note that only these + * feature flags have been verified to have these specific + * meanings. If another feature flag test is introduced, + * it has to be verified to have the same meaning for all + * vendors above. + */ + +#if ETHR_SIZEOF_PTR == 8 + /* bit 13 of ecx is set if we have cmpxchg16b */ + ethr_runtime__.conf.have_dw_cmpxchg = (ecx & (1 << 13)); +#elif ETHR_SIZEOF_PTR == 4 + /* bit 8 of edx is set if we have cmpxchg8b */ + ethr_runtime__.conf.have_dw_cmpxchg = (edx & (1 << 8)); +#else +# error "Not supported" +#endif + /* bit 26 of edx is set if we have sse2 */ + ethr_runtime__.conf.have_sse2 = (edx & (1 << 26)); +} + +#endif /* ETHR_X86_RUNTIME_CONF__ */ + + int ethr_init_common__(ethr_init_data *id) { int res; + +#if defined(ETHR_X86_RUNTIME_CONF__) + x86_init(); +#endif + if (id) { ethr_thr_prepare_func__ = id->thread_create_prepare_func; ethr_thr_parent_func__ = id->thread_create_parent_func; diff --git a/erts/lib_src/common/ethr_mutex.c b/erts/lib_src/common/ethr_mutex.c index d1b9c96083..e363279f2e 100644 --- a/erts/lib_src/common/ethr_mutex.c +++ b/erts/lib_src/common/ethr_mutex.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2010. All Rights Reserved. + * Copyright Ericsson AB 2010-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -26,8 +26,9 @@ #include "config.h" #endif -#define ETHR_INLINE_FUNC_NAME_(X) X ## __ +#define ETHR_INLINE_MTX_FUNC_NAME_(X) X ## __ #define ETHR_MUTEX_IMPL__ +#define ETHR_TRY_INLINE_FUNCS #include <limits.h> #include "ethread.h" @@ -222,9 +223,59 @@ rwmutex_try_complete_runlock(ethr_rwmutex *rwmtx, int try_write_lock); #endif +/* -- Utilities used by multiple implementations -- */ + +#if defined(ETHR_USE_OWN_RWMTX_IMPL__) || defined(ETHR_USE_OWN_MTX_IMPL__) \ + || defined(ETHR_WIN32_THREADS) + +static ETHR_INLINE void +enqueue(ethr_ts_event **queue, + ethr_ts_event *tse_start, + ethr_ts_event *tse_end) +{ + if (!*queue) { + *queue = tse_start; + tse_start->prev = tse_end; + tse_end->next = tse_start; + } + else { + tse_end->next = *queue; + tse_start->prev = (*queue)->prev; + (*queue)->prev->next = tse_start; + (*queue)->prev = tse_end; + } +} + + +static ETHR_INLINE void +dequeue(ethr_ts_event **queue, + ethr_ts_event *tse_start, + ethr_ts_event *tse_end) +{ + if (tse_start->prev == tse_end) { + ETHR_ASSERT(*queue == tse_start && tse_end->next == tse_start); + *queue = NULL; + } + else { + if (*queue == tse_start) + *queue = tse_end->next; + tse_end->next->prev = tse_start->prev; + tse_start->prev->next = tse_end->next; + } +} + +#endif + #if defined(ETHR_USE_OWN_RWMTX_IMPL__) || defined(ETHR_USE_OWN_MTX_IMPL__) -/* -- Utilities operating both on ordinary mutexes and read write mutexes -- */ +static ETHR_INLINE void +insert(ethr_ts_event *tse_pred, ethr_ts_event *tse) +{ + tse->next = tse_pred->next; + tse->prev = tse_pred; + tse_pred->next->prev = tse; + tse_pred->next = tse; +} static ETHR_INLINE void rwmutex_freqread_wtng_rdrs_inc(ethr_rwmutex *rwmtx, ethr_ts_event *tse) @@ -354,51 +405,6 @@ rwmutex_freqread_rdrs_read(ethr_rwmutex *rwmtx, int ix) return res; } - -static ETHR_INLINE void -enqueue(ethr_ts_event **queue, - ethr_ts_event *tse_start, - ethr_ts_event *tse_end) -{ - if (!*queue) { - *queue = tse_start; - tse_start->prev = tse_end; - tse_end->next = tse_start; - } - else { - tse_end->next = *queue; - tse_start->prev = (*queue)->prev; - (*queue)->prev->next = tse_start; - (*queue)->prev = tse_end; - } -} - -static ETHR_INLINE void -insert(ethr_ts_event *tse_pred, ethr_ts_event *tse) -{ - tse->next = tse_pred->next; - tse->prev = tse_pred; - tse_pred->next->prev = tse; - tse_pred->next = tse; -} - -static ETHR_INLINE void -dequeue(ethr_ts_event **queue, - ethr_ts_event *tse_start, - ethr_ts_event *tse_end) -{ - if (tse_start->prev == tse_end) { - ETHR_ASSERT(*queue == tse_start && tse_end->next == tse_start); - *queue = NULL; - } - else { - if (*queue == tse_start) - *queue = tse_end->next; - tse_end->next->prev = tse_start->prev; - tse_start->prev->next = tse_end->next; - } -} - static void event_wait(struct ethr_mutex_base_ *mtxb, ethr_ts_event *tse, @@ -1243,7 +1249,7 @@ ethr_cond_wait(ethr_cond *cnd, ethr_mutex *mtx) return 0; } -#else +#elif defined(ETHR_PTHREADS) && !defined(ETHR_DBG_WIN_MTX_WITH_PTHREADS) /* -- pthread mutex and condition variables -------------------------------- */ int @@ -1260,6 +1266,12 @@ ethr_mutex_init(ethr_mutex *mtx) } int +ethr_mutex_init_opt(ethr_mutex *mtx, ethr_mutex_opt *opt) +{ + return ethr_mutex_init(mtx); +} + +int ethr_mutex_destroy(ethr_mutex *mtx) { #if ETHR_XCHK @@ -1292,6 +1304,12 @@ ethr_cond_init(ethr_cond *cnd) } int +ethr_cond_init_opt(ethr_cond *cnd, ethr_cond_opt *opt) +{ + return ethr_cond_init(cnd); +} + +int ethr_cond_destroy(ethr_cond *cnd) { #if ETHR_XCHK @@ -1353,7 +1371,388 @@ ethr_cond_wait(ethr_cond *cnd, ethr_mutex *mtx) return res; } -#endif /* pthread_mutex */ +#elif defined(ETHR_WIN32_THREADS) || defined(ETHR_DBG_WIN_MTX_WITH_PTHREADS) + +/* + * As of Vista/Server, 2008 Windows has condition variables that can be + * used with critical sections. However, we need to be able to run on + * older Windows versions too, so we need to implement condition variables + * ourselves. + */ + +#ifdef ETHR_DBG_WIN_MTX_WITH_PTHREADS +/* + * For debugging of this implementation on POSIX platforms... + */ + +#define ethr_win_get_errno__() EINVAL +#if defined(__GNUC__) +#define __forceinline __inline__ +#else +#define __forceinline +#endif + +static int +InitializeCriticalSectionAndSpinCount(CRITICAL_SECTION *cs, int sc) +{ + return 0 == pthread_mutex_init((pthread_mutex_t *) cs, NULL); +} + +static void DeleteCriticalSection(CRITICAL_SECTION *cs) +{ + int res = pthread_mutex_destroy((pthread_mutex_t *) cs); + if (res != 0) + ETHR_FATAL_ERROR__(res); +} + +int TryEnterCriticalSection(CRITICAL_SECTION *cs) +{ + int res; + res = pthread_mutex_trylock((pthread_mutex_t *) cs); + if (res != 0 && res != EBUSY) + ETHR_FATAL_ERROR__(res); + return res == 0; +} + +void EnterCriticalSection(CRITICAL_SECTION *cs) +{ + int res = pthread_mutex_lock((pthread_mutex_t *) cs); + if (res != 0) + ETHR_FATAL_ERROR__(res); +} + +void LeaveCriticalSection(CRITICAL_SECTION *cs) +{ + int res = pthread_mutex_unlock((pthread_mutex_t *) cs); + if (res != 0) + ETHR_FATAL_ERROR__(res); +} + +#endif + +#define ETHR_CND_WAIT__ ((ethr_sint32_t) 0x11dead11) +#define ETHR_CND_WAKEUP__ ((ethr_sint32_t) 0x11beef11) + +static __forceinline void +cond_wakeup(ethr_ts_event *tse) +{ + ETHR_ASSERT(ethr_atomic32_read(&tse->uaflgs) == ETHR_CND_WAIT__); + + ethr_atomic32_set_relb(&tse->uaflgs, ETHR_CND_WAKEUP__); + ethr_event_set(&tse->event); +} + +void +ethr_mutex_cond_wakeup__(ethr_mutex *mtx) +{ + /* + * Called by ethr_mutex_unlock() when we have + * cond signal/broadcast wakeups waiting to + * be completed. + */ + ethr_ts_event *tse; + + if (!mtx->posix_compliant) { + tse = mtx->wakeups; + dequeue(&mtx->wakeups, tse, tse); + } + else { + ethr_spin_lock(&mtx->lock); + tse = mtx->wakeups; + if (tse) + dequeue(&mtx->wakeups, tse, tse); + if (!mtx->wakeups) + ethr_atomic32_set_relb(&mtx->have_wakeups, 0); + ethr_spin_unlock(&mtx->lock); + } + + LeaveCriticalSection(&mtx->cs); + + ETHR_ASSERT(tse || mtx->posix_compliant); + + /* + * We delay actual condition variable wakeup until + * this point when we have left the critical section. + * This in order to avoid that the other thread is + * woken and then right away have to go to sleep + * waiting for the critical section that we are in. + * + * We also only wake one thread at a time even if + * there are multiple threads waiting to be woken. + * Otherwise all but one will be woken and then right + * away have to go to sleep on the critical section. + * Since each wakeup is guaranteed to generate at + * least one lock/unlock sequence on this mutex, all + * threads will eventually be woken. + */ + + if (tse) + cond_wakeup(tse); +} + +int +ethr_mutex_init_opt(ethr_mutex *mtx, ethr_mutex_opt *opt) +{ + int spincount; +#if ETHR_XCHK + if (!mtx) { + ETHR_ASSERT(0); + return EINVAL; + } + mtx->initialized = ETHR_MUTEX_INITIALIZED; +#endif + + spincount = opt ? opt->aux_spincount : 0; + if (spincount < 0) + spincount = 0; + + if (!InitializeCriticalSectionAndSpinCount(&mtx->cs, spincount)) { +#if ETHR_XCHK + mtx->initialized = 0; +#endif + return ethr_win_get_errno__(); + } + + mtx->posix_compliant = opt ? opt->posix_compliant : 0; + mtx->wakeups = NULL; + if (mtx->posix_compliant) { + ethr_atomic32_init(&mtx->locked, 0); + ethr_atomic32_init(&mtx->have_wakeups, 0); + ethr_spinlock_init(&mtx->lock); + } + return 0; +} + +int +ethr_mutex_init(ethr_mutex *mtx) +{ + return ethr_mutex_init_opt(mtx, NULL); +} + +int +ethr_mutex_destroy(ethr_mutex *mtx) +{ + DeleteCriticalSection(&mtx->cs); + if (mtx->posix_compliant) + return ethr_spinlock_destroy(&mtx->lock); + else + return 0; +} + +int +ethr_cond_wait(ethr_cond *cnd, ethr_mutex *mtx) +{ + void *udata; + ethr_ts_event *tse = ethr_get_ts_event(); + int spincount; + + udata = tse->udata; + tse->udata = (void *) mtx; + ethr_atomic32_set_relb(&tse->uaflgs, ETHR_CND_WAIT__); + + EnterCriticalSection(&cnd->cs); + enqueue(&cnd->waiters, tse, tse); + LeaveCriticalSection(&cnd->cs); + + ethr_mutex_unlock(mtx); + + spincount = cnd->spincount; + + while (ethr_atomic32_read_acqb(&tse->uaflgs) != ETHR_CND_WAKEUP__) { + ethr_event_reset(&tse->event); + if (ethr_atomic32_read_acqb(&tse->uaflgs) == ETHR_CND_WAKEUP__) + break; + ethr_event_swait(&tse->event, spincount); + spincount = 0; + } + + tse->udata = udata; + ethr_leave_ts_event(tse); + + ethr_mutex_lock(mtx); + + return 0; +} + +static __forceinline void +posix_compliant_mtx_enqueue(ethr_mutex *mtx, + ethr_ts_event *tse_start, + ethr_ts_event *tse_end) +{ + ethr_ts_event *tse_wakeup = NULL; /* Avoid erroneous compiler warning... */ + /* + * The associated mutex might not be locked, so we need to + * check if it is. If locked, enqueue for wakeup at unlock; + * otherwise, wakeup the first one now and enqueue the rest. + */ + if (tse_start == tse_end && !ethr_atomic32_read(&mtx->locked)) { + tse_wakeup = tse_start; + wakeup: + cond_wakeup(tse_wakeup); + } + else { + int need_wakeup; + ethr_spin_lock(&mtx->lock); + if (!mtx->wakeups) + ethr_atomic32_set_mb(&mtx->have_wakeups, 1); + need_wakeup = !ethr_atomic32_read(&mtx->locked); + if (need_wakeup) { + if (tse_start == tse_end) { + if (!mtx->wakeups) + ethr_atomic32_set_relb(&mtx->have_wakeups, 0); + ethr_spin_unlock(&mtx->lock); + tse_wakeup = tse_start; + goto wakeup; + } + tse_wakeup = tse_start; + tse_start = tse_start->next; + } + enqueue(&mtx->wakeups, tse_start, tse_end); + ethr_spin_unlock(&mtx->lock); + if (need_wakeup) + goto wakeup; + } +} + +static __forceinline void +enqueue_cond_wakeups(ethr_ts_event *queue, int posix_compliant) +{ + if (queue) { + int more; + ethr_ts_event *q = queue; + + /* + * Waiters may be using different mutexes... + */ + + do { + ethr_mutex *mtx; + ethr_ts_event *tse, *tse_start, *tse_end; + + more = 0; + tse_start = q; + mtx = (ethr_mutex *) tse_start->udata; + + ETHR_ASSERT(posix_compliant + ? mtx->posix_compliant + : !mtx->posix_compliant); + + ETHR_ASSERT(ethr_atomic32_read(&tse_start->uaflgs) + == ETHR_CND_WAIT__); + ETHR_ASSERT(mtx->initialized == ETHR_MUTEX_INITIALIZED); + + tse_end = tse_start->prev; + + for (tse = tse_start->next; tse != tse_start; tse = tse->next) { + + ETHR_ASSERT(ethr_atomic32_read(&tse->uaflgs) + == ETHR_CND_WAIT__); + + if (mtx != (ethr_mutex *) tse->udata) { + tse_end = tse->prev; + dequeue(&q, tse_start, tse_end); + more = 1; + break; + } + } + + if (posix_compliant) + posix_compliant_mtx_enqueue(mtx, tse_start, tse_end); + else + enqueue(&mtx->wakeups, tse_start, tse_end); + + } while (more); + } +} + +void +ethr_cond_broadcast(ethr_cond *cnd) +{ + ethr_ts_event *waiters; + + EnterCriticalSection(&cnd->cs); + waiters = cnd->waiters; + cnd->waiters = NULL; + LeaveCriticalSection(&cnd->cs); + + if (cnd->posix_compliant) + enqueue_cond_wakeups(waiters, 1); + else + enqueue_cond_wakeups(waiters, 0); +} + +void +ethr_cond_signal(ethr_cond *cnd) +{ + ethr_mutex *mtx; + ethr_ts_event *tse; + + EnterCriticalSection(&cnd->cs); + tse = cnd->waiters; + if (tse) + dequeue(&cnd->waiters, tse, tse); + LeaveCriticalSection(&cnd->cs); + + if (tse) { + mtx = (ethr_mutex *) tse->udata; + + ETHR_ASSERT(ethr_atomic32_read(&tse->uaflgs) == ETHR_CND_WAIT__); + ETHR_ASSERT(mtx->initialized == ETHR_MUTEX_INITIALIZED); + ETHR_ASSERT(cnd->posix_compliant + ? mtx->posix_compliant + : !mtx->posix_compliant); + + if (cnd->posix_compliant) + posix_compliant_mtx_enqueue(mtx, tse, tse); + else + enqueue(&mtx->wakeups, tse, tse); + } +} + +int +ethr_cond_init_opt(ethr_cond *cnd, ethr_cond_opt *opt) +{ + int spincount; + +#if ETHR_XCHK + if (!cnd) { + ETHR_ASSERT(0); + return EINVAL; + } + cnd->initialized = ETHR_COND_INITIALIZED; +#endif + + spincount = opt ? opt->aux_spincount : 0; + if (spincount < 0) + spincount = 0; + + if (!InitializeCriticalSectionAndSpinCount(&cnd->cs, spincount)) { +#if ETHR_XCHK + cnd->initialized = 0; +#endif + return ethr_win_get_errno__(); + } + + cnd->posix_compliant = opt ? opt->posix_compliant : 0; + cnd->waiters = NULL; + cnd->spincount = spincount; + return 0; +} + +int +ethr_cond_init(ethr_cond *cnd) +{ + return ethr_cond_init_opt(cnd, NULL); +} + +int +ethr_cond_destroy(ethr_cond *cnd) +{ + DeleteCriticalSection(&cnd->cs); + return 0; +} + +#endif /* -- Exported symbols of inline functions --------------------------------- */ diff --git a/erts/lib_src/pthread/ethr_x86_sse2_asm.c b/erts/lib_src/pthread/ethr_x86_sse2_asm.c new file mode 100644 index 0000000000..6cbe73cf16 --- /dev/null +++ b/erts/lib_src/pthread/ethr_x86_sse2_asm.c @@ -0,0 +1,31 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2011. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: sse2 asm:s + * Author: Rickard Green + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +/* ETHR_X86_SSE2_ASM_C__ will trigger asm:s to compile to be included */ +#define ETHR_X86_SSE2_ASM_C__ +#include "ethread.h" diff --git a/erts/lib_src/pthread/ethread.c b/erts/lib_src/pthread/ethread.c index f047104103..ad29249bac 100644 --- a/erts/lib_src/pthread/ethread.c +++ b/erts/lib_src/pthread/ethread.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2010. All Rights Reserved. + * Copyright Ericsson AB 2010-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -121,6 +121,98 @@ ethr_ts_event *ethr_get_tse__(void) return pthread_getspecific(ethr_ts_event_key__); } +#if defined(ETHR_PPC_RUNTIME_CONF__) + +static volatile int lwsync_caused_sigill; + +static void +handle_lwsync_sigill(int signum) +{ + lwsync_caused_sigill = 1; +} + +static int +ppc_init__(void) +{ + struct sigaction act, oact; + lwsync_caused_sigill = 0; + + sigemptyset(&act.sa_mask); + act.sa_flags = 0; + act.sa_handler = handle_lwsync_sigill; + if (sigaction(SIGILL, &act, &oact) != 0) + return errno; + + __asm__ __volatile__ ("lwsync\n\t" : : : "memory"); + + act.sa_flags = 0; + act.sa_handler = SIG_DFL; + if (sigaction(SIGILL, &act, &oact) != 0) + return errno; + + ethr_runtime__.conf.have_lwsync = (int) !lwsync_caused_sigill; + return 0; +} + +#endif + +#if defined(ETHR_X86_RUNTIME_CONF__) + +void +ethr_x86_cpuid__(int *eax, int *ebx, int *ecx, int *edx) +{ +#if ETHR_SIZEOF_PTR == 4 + int have_cpuid; + /* + * If it is possible to toggle eflags bit 21, + * we have the cpuid instruction. + */ + __asm__ ("pushf\n\t" + "popl %%eax\n\t" + "movl %%eax, %%ecx\n\t" + "xorl $0x200000, %%eax\n\t" + "pushl %%eax\n\t" + "popf\n\t" + "pushf\n\t" + "popl %%eax\n\t" + "movl $0x0, %0\n\t" + "xorl %%ecx, %%eax\n\t" + "jz no_cpuid\n\t" + "movl $0x1, %0\n\t" + "no_cpuid:\n\t" + : "=r"(have_cpuid) + : + : "%eax", "%ecx", "cc"); + if (!have_cpuid) { + *eax = *ebx = *ecx = *edx = 0; + return; + } +#endif +#if ETHR_SIZEOF_PTR == 4 && defined(__PIC__) && __PIC__ + /* + * When position independet code is used in 32-bit mode, the B register + * is used for storage of global offset table address, and we may not + * use it as input or output in an asm. We need to save and restore the + * B register explicitly (for some reason gcc doesn't provide this + * service to us). + */ + __asm__ ("pushl %%ebx\n\t" + "cpuid\n\t" + "movl %%ebx, %1\n\t" + "popl %%ebx\n\t" + : "=a"(*eax), "=r"(*ebx), "=c"(*ecx), "=d"(*edx) + : "0"(*eax) + : "cc"); +#else + __asm__ ("cpuid\n\t" + : "=a"(*eax), "=b"(*ebx), "=c"(*ecx), "=d"(*edx) + : "0"(*eax) + : "cc"); +#endif +} + +#endif /* ETHR_X86_RUNTIME_CONF__ */ + /* * -------------------------------------------------------------------------- * Exported functions @@ -137,6 +229,12 @@ ethr_init(ethr_init_data *id) ethr_not_inited__ = 0; +#if defined(ETHR_PPC_RUNTIME_CONF__) + res = ppc_init__(); + if (res != 0) + goto error; +#endif + res = ethr_init_common__(id); if (res != 0) goto error; @@ -146,6 +244,8 @@ ethr_init(ethr_init_data *id) child_wait_spin_count = 0; res = pthread_key_create(ðr_ts_event_key__, ethr_ts_event_destructor__); + if (res != 0) + goto error; return 0; error: diff --git a/erts/lib_src/utils/make_atomics_api b/erts/lib_src/utils/make_atomics_api new file mode 100755 index 0000000000..f4e71c7618 --- /dev/null +++ b/erts/lib_src/utils/make_atomics_api @@ -0,0 +1,2186 @@ +#!/usr/bin/env escript +%% -*- erlang -*- + +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2011. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +-mode(compile). + +%%%------------------------------------------------------------------- +%%% @author Rickard Green <[email protected]> +%%% @copyright (C) 2011, Rickard Green +%%% @doc +%%% Generation of the ethread atomic API +%%% @end +%%% Created : 17 Jan 2011 by Rickard Green <[email protected]> +%%%------------------------------------------------------------------- + +-define(H_FILE, "erts/include/internal/ethr_atomics.h"). +-define(C_FILE, "erts/lib_src/common/ethr_atomics.c"). + +%% These order constraints are important: +%% - 'cmpxchg' needs to appear before 'read' +%% - 'xchg' needs to apper before 'set' +%% - 'set' needs to apper before 'init' +%% - 'add_read' needs to apper before 'add', 'inc_read', and 'dec_read' +%% - 'inc_read' needs to apper before and 'inc' +%% - 'dec_read' needs to apper before and 'dec' +-define(ATOMIC_OPS, [cmpxchg, xchg, set, init, add_read, + read, inc_read, dec_read, add, inc, + dec, read_band, read_bor]). + +-define(DW_ATOMIC_OPS, [cmpxchg, set, read, init]). +-define(DW_FUNC_MACRO, "ETHR_DW_ATOMIC_FUNC__"). +-define(DW_RTCHK_MACRO, "ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__"). + +%% Barrier versions we implement +-define(BARRIERS, [none, rb, wb, acqb, relb, mb]). + +-define(ATOMIC_SIZES, ["dword", "word", "32"]). + +-define(HAVE_NATIVE_ATOMIC, "ETHR_HAVE_ETHR_NATIVE_ATOMIC"). + +-define(SU_DW_SINT_FIELD, "dw_sint"). +-define(DW_SINT_FIELD, "sint"). + +%% Fallback +-define(ETHR_ATMC_FLLBK_ADDR_BITS, "10"). +-define(ETHR_ATMC_FLLBK_ADDR_SHIFT, "6"). + +-record(atomic_context, {dw, + amc_fallback, + ret_type, + ret_var, + arg1, + arg2, + arg3, + have_native_atomic_ops, + atomic, + atomic_t, + addr_aint_t, + aint_t, + naint_t, + 'NATMC', + 'ATMC', + unusual_val}). + +atomic_context("dword") -> + #atomic_context{dw = true, + amc_fallback = true, + ret_type = "int", + ret_var = "res", + arg1 = "var", + arg2 = "val", + arg3 = "old_val", + have_native_atomic_ops = "ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS", + atomic = "ethr_dw_atomic", + atomic_t = "ethr_dw_atomic_t", + addr_aint_t = "ethr_sint_t", + aint_t = "ethr_dw_sint_t", + naint_t = "ETHR_SU_DW_NAINT_T__", + 'NATMC' = "DW_NATMC", + 'ATMC' = "DW_ATMC", + unusual_val = "ETHR_UNUSUAL_SINT_VAL__"}; +atomic_context(Size) -> + {SizeSuffix, HaveSize, AMC} = case Size of + "word" -> {"", "WORD_SZ", true}; + _ -> {Size, Size++"BIT", false} + end, + AintT = ["ethr_sint", SizeSuffix, "_t"], + #atomic_context{dw = false, + amc_fallback = AMC, + ret_type = AintT, + ret_var = "res", + arg1 = "var", + arg2 = "val", + arg3 = "old_val", + have_native_atomic_ops = ["ETHR_HAVE_", HaveSize, "_NATIVE_ATOMIC_OPS"], + atomic = ["ethr_atomic", SizeSuffix], + atomic_t = ["ethr_atomic", SizeSuffix, "_t"], + addr_aint_t = AintT, + aint_t = AintT, + naint_t = ["ETHR_NAINT", SizeSuffix, "_T__"], + 'NATMC' = ["NATMC", SizeSuffix], + 'ATMC' = ["ATMC", SizeSuffix], + unusual_val = ["ETHR_UNUSUAL_SINT", SizeSuffix, "_VAL__"]}. + +-record(op_context, {ret, var, val1, val2}). + +-define(POTENTIAL_NBITS, ["64", "32"]). + +is_return_op(#atomic_context{dw = false}, add) -> false; +is_return_op(#atomic_context{dw = false}, inc) -> false; +is_return_op(#atomic_context{dw = false}, dec) -> false; +is_return_op(#atomic_context{dw = true}, read) -> false; +is_return_op(_AC, init) -> false; +is_return_op(_AC, set) -> false; +is_return_op(_AC, _OP) -> true. + +native(add_read) -> add_return; +native(inc_read) -> inc_return; +native(dec_read) -> dec_return; +native(read_band) -> and_retold; +native(read_bor) -> or_retold; +native(Op) -> Op. + +op(Op, #op_context{var = Var, val1 = Val1}) when Op == init; Op == set -> + [Var, " = ", Val1]; +op(read, #op_context{ret = Ret, var = Var}) -> + [Ret, " = ", Var]; +op(add_read, OpC) -> + [op(add, OpC), "; ", op(read, OpC)]; +op(add, #op_context{var = Var, val1 = Val1}) -> + [Var, " += ", Val1]; +op(inc, #op_context{var = Var}) -> + ["++(", Var, ")"]; +op(dec, #op_context{var = Var}) -> + ["--(", Var, ")"]; +op(inc_read, #op_context{ret = Ret, var = Var}) -> + [Ret, " = ++(", Var, ")"]; +op(dec_read, #op_context{ret = Ret, var = Var}) -> + [Ret, " = --(", Var, ")"]; +op(read_band, #op_context{var = Var, val1 = Val1} = OpC) -> + [op(read, OpC), "; ", Var, " &= ", Val1]; +op(read_bor, #op_context{var = Var, val1 = Val1} = OpC) -> + [op(read, OpC), "; ", Var, " |= ", Val1]; +op(xchg, OpC) -> + [op(read, OpC), "; ", op(set, OpC)]; +op(cmpxchg, #op_context{ret = Ret, var = Var, val1 = Val1, val2 = Val2}) -> + [Ret, " = (", Var, " == ", Val2, " ? (", Var, " = ", Val1, ", ", Val2, ") : ", Var, ")"]. + +dw_op(Op, #op_context{var = Var, val1 = Val1}) when Op == init; Op == set -> + [Var, "[0] = ", Val1, "[0]; ", Var, "[1] = ", Val1, "[1]"]; +dw_op(read, #op_context{var = Var, val1 = Val1}) -> + [Val1, "[0] = ", Var, "[0]; ", Val1, "[1] = ", Var, "[1]"]; +dw_op(cmpxchg, #op_context{ret = Ret, var = Var, val1 = Val1, val2 = Val2}) -> + [" + { + ", Ret, " = (", Var, "[0] == ", Val2, "[0] && ", Var, "[1] == ", Val2, "[1]); + if (", Ret, ") { + ", Var, "[0] = ", Val1, "[0]; + ", Var, "[1] = ", Val1, "[1]; + } + else { + ", Val2, "[0] = ", Var, "[0]; + ", Val2, "[1] = ", Var, "[1]; + } + }"]. + +op_head_tail(init) -> {undef, undef}; +op_head_tail(set) -> {store, store}; +op_head_tail(read) -> {load, load}; +op_head_tail(_) -> {load, undef}. + +op_barrier_ext(none) -> ""; +op_barrier_ext(Barrier) -> [$_, a2l(Barrier)]. + +op_call(addr, _DW, Ret, Func, Arg1, _Arg2, _Arg3, _TypeCast) -> + [Ret, " ", Func, "(", Arg1, ");"]; +op_call(Op, false, Ret, Func, Arg1, _Arg2, _Arg3, _TypeCast) when Op == read; + Op == inc_read; + Op == inc_return; + Op == dec_read; + Op == dec_return -> + [Ret, " ", Func, "(", Arg1, ");"]; +op_call(Op, false, _Ret, Func, Arg1, _Arg2, _Arg3, _TypeCast) when Op == inc; + Op == dec -> + [Func, "(", Arg1, ");"]; +op_call(Op, false, Ret, Func, Arg1, Arg2, _Arg3, TypeCast) when Op == add_return; + Op == add_read; + Op == read_band; + Op == and_retold; + Op == read_bor; + Op == or_retold; + Op == xchg -> + [Ret, " ", Func, "(", Arg1, ",", TypeCast, " ", Arg2, ");"]; +op_call(cmpxchg, _DW, Ret, Func, Arg1, Arg2, Arg3, TypeCast) -> + [Ret, " ", Func, "(", Arg1, ",", TypeCast, " ", Arg2, ",", TypeCast, " ", Arg3, ");"]; +op_call(_Op, _DW, _Ret, Func, Arg1, Arg2, _Arg3, TypeCast) -> + [Func, "(", Arg1, ",", TypeCast, " ", Arg2, ");"]. % set, init, add (!= dw), read (== dw) + +native_op_call(#atomic_context{dw = DW, + ret_var = RetVar, + arg1 = Arg1, + arg2 = Arg2, + arg3 = Arg3, + aint_t = AintT, + 'NATMC' = NATMC, + naint_t = NAintT}, + Op, B, TypeCasts) -> + op_call(Op, + DW, + [RetVar, " =", + case TypeCasts of + true -> [" (", AintT, ")"]; + false -> "" + end], + ["ETHR_", NATMC, "_FUNC__(", opstr(native(Op)), op_barrier_ext(B), ")"], + Arg1, + Arg2, + Arg3, + case TypeCasts of + true -> [" (", NAintT, ")"]; + false -> "" + end). + +simple_fallback(#atomic_context{arg1 = Arg1, + arg2 = Arg2, + 'ATMC' = ATMC}, + init, B) -> %% Also double word + [" ETHR_", ATMC, "_FUNC__(set", op_barrier_ext(B),")(", Arg1, ", ", Arg2, ");\n"]; +simple_fallback(#atomic_context{dw = false, + arg1 = Arg1, + arg2 = Arg2, + 'ATMC' = ATMC}, + set, B) -> + [" (void) ETHR_", ATMC, "_FUNC__(xchg", op_barrier_ext(B),")(", Arg1, ", ", Arg2, ");\n"]; +simple_fallback(#atomic_context{dw = false, + arg1 = Arg1, + arg2 = Arg2, + 'ATMC' = ATMC}, + add, B) -> + [" (void) ETHR_", ATMC, "_FUNC__(add_read", op_barrier_ext(B), ")(", Arg1, ", ", Arg2, ");\n"]; +simple_fallback(#atomic_context{dw = false, + ret_var = RetVar, + arg1 = Arg1, + aint_t = AintT, + 'ATMC' = ATMC}, + inc_read, B) -> + [" ", RetVar, " = ETHR_", ATMC, "_FUNC__(add_read", op_barrier_ext(B), ")(", Arg1, ", (", AintT,") 1);\n"]; +simple_fallback(#atomic_context{dw = false, + ret_var = RetVar, + arg1 = Arg1, + aint_t = AintT, + 'ATMC' = ATMC}, + dec_read, B) -> + [" ", RetVar, " = ETHR_", ATMC, "_FUNC__(add_read", op_barrier_ext(B), ")(", Arg1, ", (", AintT,") -1);\n"]; +simple_fallback(#atomic_context{dw = false, + arg1 = Arg1, + 'ATMC' = ATMC}, + inc, B) -> + [" (void) ETHR_", ATMC, "_FUNC__(inc_read", op_barrier_ext(B), ")(", Arg1, ");\n"]; +simple_fallback(#atomic_context{dw = false, + arg1 = Arg1, + 'ATMC' = ATMC}, + dec, B) -> + [" (void) ETHR_", ATMC, "_FUNC__(dec_read", op_barrier_ext(B), ")(", Arg1, ");\n"]; +simple_fallback(#atomic_context{dw = false, + unusual_val = UnusualVal, + ret_var = RetVar, + arg1 = Arg1, + aint_t = AintT, + 'ATMC' = ATMC}, + read, B) -> + [" ", RetVar, " = ETHR_", ATMC, "_FUNC__(cmpxchg", op_barrier_ext(B), ")(", Arg1, ", (", AintT, ") ", UnusualVal, ", (", AintT,") ", UnusualVal, ");\n"]; +simple_fallback(#atomic_context{dw = true, + unusual_val = UnusualVal, + arg1 = Arg1, + arg2 = Arg2, + aint_t = AintT, + 'ATMC' = ATMC}, + read, B) -> + [" ", AintT, " tmp; + tmp.", ?DW_SINT_FIELD, "[0] = ", UnusualVal, "; + tmp.", ?DW_SINT_FIELD, "[1] = ", UnusualVal, "; + ", Arg2, "->", ?DW_SINT_FIELD, "[0] = ", UnusualVal, "; + ", Arg2, "->", ?DW_SINT_FIELD, "[1] = ", UnusualVal, "; + (void) ETHR_", ATMC, "_FUNC__(cmpxchg", op_barrier_ext(B), ")(", Arg1, ", &tmp, ", Arg2, "); +" + ]; +simple_fallback(_AC, _Op, _B) -> + []. + +func_header(AC, prototype, MacroName, Op, B) -> + [func_header(AC, implementation, MacroName, Op, B), ";"]; +func_header(#atomic_context{'ATMC' = ATMC} = AC, inline_implementation, _MacroName, Op, B) -> + do_func_header(AC, Op, "static ETHR_INLINE ", + ["ETHR_", ATMC, "_FUNC__(", opstr(Op), op_barrier_ext(B), ")"]); +func_header(#atomic_context{atomic = Atomic} = AC, implementation, false, Op, B) -> + do_func_header(AC, Op, "", [Atomic, "_", opstr(Op), op_barrier_ext(B)]); +func_header(AC, implementation, MacroName, Op, B) -> + do_func_header(AC, Op, "", [MacroName, "(", opstr(Op), op_barrier_ext(B), ")"]). + + +do_func_header(#atomic_context{atomic_t = AtomicT, + addr_aint_t = AddrAintT, + arg1 = Arg1}, + addr, Inline, Func) -> + [Inline, AddrAintT, " *", Func, "(", AtomicT, " *", Arg1, ")"]; +do_func_header(#atomic_context{dw = false, + atomic_t = AtomicT, + aint_t = AintT, + arg1 = Arg1, + arg2 = Arg2}, + Op, Inline, Func) when Op == init; + Op == set; + Op == add -> + [Inline, "void ", Func, "(", AtomicT, " *", Arg1, ", ", AintT, " ", Arg2, ")"]; +do_func_header(#atomic_context{dw = false, + atomic_t = AtomicT, + arg1 = Arg1}, + Op, Inline, Func) when Op == inc; + Op == dec -> + [Inline, "void ", Func, "(", AtomicT, " *", Arg1, ")"]; +do_func_header(#atomic_context{dw = false, + atomic_t = AtomicT, + aint_t = AintT, + arg1 = Arg1}, + Op, Inline, Func) when Op == read; + Op == inc_read; + Op == dec_read -> + [Inline, AintT, " ", Func, "(", AtomicT, " *", Arg1, ")"]; +do_func_header(#atomic_context{dw = false, + atomic_t = AtomicT, + aint_t = AintT, + arg1 = Arg1, + arg2 = Arg2}, + Op, Inline, Func) when Op == add_read; + Op == read_band; + Op == read_bor; + Op == xchg -> + [Inline, AintT, " ", Func, "(", AtomicT, " *", Arg1, ", ", AintT, " ", Arg2, ")"]; +do_func_header(#atomic_context{dw = false, + atomic_t = AtomicT, + aint_t = AintT, + arg1 = Arg1, + arg2 = Arg2, + arg3 = Arg3}, + cmpxchg, Inline, Func) -> + [Inline, AintT, " ", Func, "(", AtomicT, " *", Arg1, ", ", AintT, " ", Arg2, ", ", AintT, " ", Arg3, ")"]; +do_func_header(#atomic_context{dw = true, + atomic_t = AtomicT, + aint_t = AintT, + arg1 = Arg1, + arg2 = Arg2}, + Op, Inline, Func) when Op == init; + Op == set; + Op == read -> + [Inline, "void ", Func, "(", AtomicT, " *", Arg1, ", ", AintT, " *", Arg2, ")"]; +do_func_header(#atomic_context{dw = true, + atomic_t = AtomicT, + aint_t = AintT, + arg1 = Arg1, + arg2 = Arg2, + arg3 = Arg3}, + cmpxchg, Inline, Func) -> + [Inline, "int ", Func, "(", AtomicT, " *", Arg1, ", ", AintT, " *", Arg2, ", ", AintT, " *", Arg3, ")"]. + + +xbarriers(_Op, none, _NB) -> + {"", ""}; + +xbarriers(_Op, acqb, NB) when NB == acqb; NB == mb -> + {"", ""}; +xbarriers(Op, acqb, NB) -> + case {op_head_tail(Op), NB} of + {{_, load}, rb} -> {"", "ETHR_MEMBAR(ETHR_LoadStore);"}; + {{_, load}, _} -> {"", "ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);"}; + {{_, store}, _} -> {"", "ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);"}; + {_, rb} -> {"", "ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);"}; + _ -> {"", "ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);"} + end; + +xbarriers(_Op, relb, NB) when NB == relb; NB == mb -> + {"", ""}; +xbarriers(Op, relb, NB) -> + case {op_head_tail(Op), NB} of + {{store, _}, wb} -> {"ETHR_MEMBAR(ETHR_LoadStore);", ""}; + {{store, _}, _} -> {"ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);", ""}; + {{load, _}, _} -> {"ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);", ""}; + {_, wb} -> {"ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad);", ""}; + _ -> {"ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);", ""} + end; + +xbarriers(_Op, wb, NB) when NB == wb; NB == mb -> + {"", ""}; +xbarriers(_Op, wb, _NB) -> + {"ETHR_MEMBAR(ETHR_StoreStore);", ""}; + +xbarriers(_Op, rb, NB) when NB == rb; NB == mb -> + {"", ""}; +xbarriers(_Op, rb, _NB) -> + {"", "ETHR_MEMBAR(ETHR_LoadLoad);"}; + +xbarriers(_Op, mb, mb) -> + {"", ""}; +xbarriers(Op, mb, NB) -> + MB = "ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);", + {Head, Tail} = op_head_tail(Op), + PreOp = case {Head, NB} of + {_, relb} -> ""; + {store, wb} -> "ETHR_MEMBAR(ETHR_LoadStore);"; + {store, _} -> "ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);"; + {load, _} -> "ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);"; + {_, wb} -> "ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad);"; + _ -> MB + end, + PostOp = case {Tail, NB} of + {_, acqb} -> ""; + {load, rb} -> "ETHR_MEMBAR(ETHR_LoadStore);"; + {load, _} -> "ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);"; + {store, _} -> "ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);"; + {_, rb} -> "ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);"; + _ -> MB + end, + {PreOp, PostOp}. + +try_barrier_order_first(none) -> + [none, rb, wb, acqb, relb]; +try_barrier_order_first(acqb) -> + [acqb, rb, none, mb]; +try_barrier_order_first(relb) -> + [relb, wb, none, mb]; +try_barrier_order_first(rb) -> + [rb, none, mb]; +try_barrier_order_first(wb) -> + [wb, none, mb]; +try_barrier_order_first(mb) -> + [mb, relb, acqb, wb, rb, none]. + +try_barrier_order(B) -> + First = try_barrier_order_first(B), + First ++ (?BARRIERS -- First). + +native_barrier_op(#atomic_context{'NATMC' = NATMC} = AC, If, ExtraDecl, Op, B, NB, TypeCasts) -> + NOpStr = opstr(native(Op)), + CapNOpStr = to_upper(NOpStr), + NBExt = op_barrier_ext(NB), + CapNBExt = to_upper(NBExt), + {PreB, PostB} = xbarriers(Op, B, NB), + [If, " defined(ETHR_HAVE_", NATMC, "_", CapNOpStr, CapNBExt, ")\n", + ExtraDecl, + case PreB of + "" -> ""; + _ -> [" ", PreB, "\n"] + end, + " ", native_op_call(AC, Op, NB, TypeCasts), "\n", + case PostB of + "" -> ""; + _ -> [" ", PostB, "\n"] + end]. + +dw_native_barrier_op(#atomic_context{arg1 = Arg1, arg2 = Arg2, arg3 = Arg3} = AC, If, ExtraDecl, Op, B, NB) -> + native_barrier_op(AC#atomic_context{arg1 = ["&", Arg1, "->native"], + arg2 = [Arg2, "->", ?DW_SINT_FIELD], + arg3 = [Arg3, "->", ?DW_SINT_FIELD]}, + If, ExtraDecl, Op, B, NB, false). + +su_dw_native_barrier_op(#atomic_context{dw = true, + naint_t = NAintT, + ret_var = RetVar, + arg1 = Arg1, + arg2 = Arg2, + arg3 = Arg3, + 'NATMC' = NATMC} = AC, If, cmpxchg, B, NB) -> + SU = ["->", ?SU_DW_SINT_FIELD], + TmpVar = "act", + SUArg1 = ["&", Arg1, "->native"], + SUArg2 = [Arg2, SU], + SUArg3 = [Arg3, SU], + ExtraDecl = [" ", NAintT, " ", TmpVar, ";\n"], + [native_barrier_op(AC#atomic_context{dw = false, + ret_var = TmpVar, + arg1 = SUArg1, + arg2 = SUArg2, + arg3 = SUArg3, + 'NATMC' = ["SU_", NATMC]}, + If, ExtraDecl, cmpxchg, B, NB, false), + " ", RetVar, " = (", TmpVar, " == ", SUArg3, "); + ", SUArg3, " = ", TmpVar, "; +" + ]; +su_dw_native_barrier_op(#atomic_context{dw = true, + arg1 = Arg1, + arg2 = Arg2, + 'NATMC' = NATMC} = AC, If, Op, B, NB) -> + SUArg1 = ["&", Arg1, "->native"], + SUArg2 = [Arg2, "->", ?SU_DW_SINT_FIELD], + native_barrier_op(AC#atomic_context{dw = false, + ret_var = SUArg2, + arg1 = SUArg1, + arg2 = SUArg2, + arg3 = not_used, + 'NATMC' = ["SU_", NATMC]}, If, "", Op, B, NB, false). + +cmpxchg_fallback_define(#atomic_context{dw = false, aint_t = AintT} = AC) -> + do_cmpxchg_fallback_define(AC, true, AintT); +cmpxchg_fallback_define(#atomic_context{dw = true, + 'NATMC' = NATMC, + naint_t = NAintT} = AC) -> + ["\n\n#if defined(ETHR_HAVE_NATIVE_DW_ATOMIC)\n", + do_cmpxchg_fallback_define(AC, false, not_used), + "\n\n#elif defined(ETHR_HAVE_NATIVE_SU_DW_ATOMIC)\n", + do_cmpxchg_fallback_define(AC#atomic_context{'NATMC' = ["SU_", NATMC], + naint_t = NAintT}, + true, + NAintT), + " + +#else +# error \"?!?\" +#endif +"]. + +do_cmpxchg_fallback_define(#atomic_context{'NATMC' = NATMC, + aint_t = AintT, + naint_t = NAintT}, + SU, SUType) -> + + ReadFunc = fun (IF) -> + fun (B) -> + BExt = op_barrier_ext(B), + CapBExt = to_upper(BExt), + [IF, " defined(ETHR_HAVE_", NATMC, "_READ", CapBExt, ")", + case SU of + true -> [" +#define ETHR_", NATMC, "_CMPXCHG_FALLBACK_READ__(VAR) \\ + ETHR_", NATMC, "_FUNC__(read", BExt, ")(VAR) +" + ]; + false -> [" +#define ETHR_", NATMC, "_CMPXCHG_FALLBACK_READ__(VAR, VAL) \\ + ETHR_", NATMC, "_FUNC__(read", BExt, ")(VAR, VAL) +#elif defined(ETHR_HAVE_SU_", NATMC, "_READ", CapBExt, ") +#define ETHR_", NATMC, "_CMPXCHG_FALLBACK_READ__(VAR, VAL) \\ + VAL.", ?SU_DW_SINT_FIELD, " = ETHR_SU_", NATMC, "_FUNC__(read", BExt, ")(VAR) +" + ] + end] + end + end, + NotDefCMPXCHG = fun (B) -> + CapBExt = to_upper(op_barrier_ext(B)), + ["!defined(ETHR_HAVE_", NATMC, "_CMPXCHG", CapBExt, ")"] + end, + NoneTryBarrierOrder = try_barrier_order(none), + %% First a sanity check + [" +#if (", NotDefCMPXCHG(hd(?BARRIERS)) , + lists:map(fun (B) -> + [" \\ + && ", NotDefCMPXCHG(B)] + end, + tl(?BARRIERS)), ") +# error \"No native cmpxchg() op available\" +#endif + + +/* + * Read op used together with cmpxchg() fallback when no native op present. + */ +", + + %% Read op to use with cmpxchg fallback + (ReadFunc("#if"))(hd(NoneTryBarrierOrder)), + lists:map(ReadFunc("#elif"), tl(NoneTryBarrierOrder)), +"#else +/* + * We have no native read() op; guess zero and then use the + * the atomics actual value returned from cmpxchg(). + */", + case SU of + true -> [" +#define ETHR_", NATMC, "_CMPXCHG_FALLBACK_READ__(VAR) \\ + ((", NAintT, ") 0)"]; + false -> [" +#define ETHR_", NATMC, "_CMPXCHG_FALLBACK_READ__(VAR, VAL) \\ +do { \\ + VAL.", ?DW_SINT_FIELD, "[0] = (ethr_sint_t) 0; \\ + VAL.", ?DW_SINT_FIELD, "[1] = (ethr_sint_t) 0; \\ +} while (0)"] + end, " +#endif +", + + %% The fallback + " +/* + * Native cmpxchg() fallback used when no native op present. + */ +#define ETHR_", NATMC, "_CMPXCHG_FALLBACK__(CMPXCHG, VAR, AVAL, OPS) \\ +do { \\", + case SU of + true -> [" + ", SUType, " AVAL; \\ + ", NAintT, " new__, act__, exp__; \\ + act__ = ETHR_", NATMC, "_CMPXCHG_FALLBACK_READ__(VAR); \\ + do { \\ + exp__ = act__; \\ + AVAL = (", SUType, ") act__; \\ + { OPS; } \\ + new__ = (", NAintT, ") AVAL; \\ + act__ = CMPXCHG(VAR, new__, exp__); \\ + } while (__builtin_expect(act__ != exp__, 0)); \\"]; + false -> [" + int res__; \\ + ", AintT, " AVAL, exp_act__; \\ + ETHR_", NATMC, "_CMPXCHG_FALLBACK_READ__(VAR, exp_act__); \\ + do { \\ + AVAL.", ?DW_SINT_FIELD, "[0] = exp_act__.", ?DW_SINT_FIELD, "[0]; \\ + AVAL.", ?DW_SINT_FIELD, "[1] = exp_act__.", ?DW_SINT_FIELD, "[1]; \\ + { OPS; } \\ + res__ = CMPXCHG(VAR, AVAL.", ?DW_SINT_FIELD, ", exp_act__.", ?DW_SINT_FIELD, "); \\ + } while (__builtin_expect(res__ == 0, 0)); \\"] + end, " +} while (0) +" + ]. + +cmpxchg_fallbacks(#atomic_context{}, _SUDW, cmpxchg, _B) -> + ""; %% No need for a fallback +cmpxchg_fallbacks(#atomic_context{dw = DW, + ret_var = RetVar, + arg1 = Arg1, + arg2 = Arg2, + arg3 = Arg3, + 'NATMC' = NATMC}, + SUDW, Op, B) -> + Operation = case DW of + false -> + op(Op, #op_context{ret = RetVar, + var = "aval", + val1 = Arg2, + val2 = Arg3}); + true -> + case SUDW of + true -> + op(Op, #op_context{ret = [Arg2, "->", ?SU_DW_SINT_FIELD], + var = "aval", + val1 = [Arg2, "->", ?SU_DW_SINT_FIELD]}); + false -> + dw_op(Op, #op_context{ret = RetVar, + var = ["aval.", ?DW_SINT_FIELD], + val1 = [Arg2, "->", ?DW_SINT_FIELD]}) + end + end, + [lists:map(fun (NB) -> + NativeVar = case DW of + true -> ["&", Arg1, "->native"]; + false -> Arg1 + end, + NBExt = op_barrier_ext(NB), + CapNBExt = to_upper(NBExt), + {PreB, PostB} = xbarriers(cmpxchg, B, NB), + ["#elif defined(ETHR_HAVE_", NATMC, "_CMPXCHG", CapNBExt, ")\n", + case PreB of + "" -> ""; + _ -> [" ", PreB, "\n"] + end, + " ETHR_", NATMC, "_CMPXCHG_FALLBACK__(ETHR_", NATMC, "_FUNC__(cmpxchg", NBExt, "), ", NativeVar, ", aval, ", Operation, ");\n", + case PostB of + "" -> ""; + _ -> [" ", PostB, "\n"] + end] + end, + try_barrier_order(B))]. + +translate_have_defs(#atomic_context{dw = DW, 'NATMC' = NATMC}) -> + [" +#if !defined(ETHR_", NATMC, "_BITS__) +# error \"Missing native atomic implementation\"", + lists:map(fun (NBits) -> + {HaveInPrefix, + HaveOutPrefix, + HaveInPrefixExtra, + HaveOutPrefixExtra, + NativeTypeCheck} = case NBits of + "dw" -> + {"ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC", + ["ETHR_HAVE_", NATMC], + "ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC", + ["ETHR_HAVE_SU_", NATMC], + "\n#elif defined(ETHR_HAVE_NATIVE_DW_ATOMIC) || defined(ETHR_HAVE_NATIVE_SU_DW_ATOMIC)"}; + _ -> + {[?HAVE_NATIVE_ATOMIC, NBits], + case DW of + true -> ["ETHR_HAVE_SU_", NATMC]; + false -> ["ETHR_HAVE_", NATMC] + end, + false, + ["ETHR_HAVE_", NATMC], + ["\n#elif ETHR_", NATMC, "_BITS__ == ", NBits]} + end, + [NativeTypeCheck, + lists:map(fun (Op) -> + NOpStr = opstr(native(Op)), + CapNOpStr = to_upper(NOpStr), + lists:map(fun (B) -> + NBExt = op_barrier_ext(B), + CapNBExt = to_upper(NBExt), + HaveOutDef = [HaveOutPrefix, "_", CapNOpStr, CapNBExt], + HaveOutDefExtra = [HaveOutPrefixExtra, "_", CapNOpStr, CapNBExt], + [case DW of + true -> + ["\n# undef ", HaveOutDefExtra]; + false -> + "" + end, " +# undef ", HaveOutDef," +# ifdef ", HaveInPrefix, "_", CapNOpStr, CapNBExt, " +# define ", HaveOutDef, " 1 +# endif", + case HaveInPrefixExtra of + false -> ""; + _ -> [" +# ifdef ", HaveInPrefixExtra, "_", CapNOpStr, CapNBExt, " +# define ", HaveOutDefExtra, " 1 +# endif" + ] + end] + end, + ?BARRIERS) + end, + case DW of + true -> ?DW_ATOMIC_OPS; + false -> ?ATOMIC_OPS + end)] + end, + case DW of + true -> ["dw", "64"]; + false -> ?POTENTIAL_NBITS + end), + " +#else +# error \"Invalid native atomic size\" +#endif +"]. + + + +make_prototypes(#atomic_context{dw = DW, 'ATMC' = ATMC} = AC) -> + MkProt = fun (MacroName) -> + %% addr() is special + [func_header(AC, prototype, MacroName, addr, none), "\n", + lists:map(fun (Op) -> + lists:map(fun (B) -> + [func_header(AC, prototype, MacroName, Op, B), "\n"] + end, + ?BARRIERS) + end, + case DW of + true -> ?DW_ATOMIC_OPS; + false -> ?ATOMIC_OPS + end)] + end, + [" +#ifdef ETHR_NEED_", ATMC, "_PROTOTYPES__ +", + MkProt(false), + case DW of + true -> ["#if defined(", ?DW_RTCHK_MACRO, ")\n", + MkProt(?DW_FUNC_MACRO), + "#endif\n"]; + false -> "" + end, + "#endif /* ETHR_NEED_", ATMC, "_PROTOTYPES__ */\n"]. + +rtchk_fallback_call(Return, #atomic_context{dw = DW, + ret_var = RetVar, + arg1 = Arg1, + arg2 = Arg2, + arg3 = Arg3}, + Op, B) -> + op_call(Op, DW, case Return of + true -> "return"; + false -> [RetVar, " ="] + end, [?DW_FUNC_MACRO, "(", opstr(Op), op_barrier_ext(B), ")"], Arg1, Arg2, Arg3, ""). + +make_implementations(#atomic_context{dw = DW, + ret_type = RetType, + ret_var = RetVar, + arg1 = Arg1, + addr_aint_t = AddrAintT, + atomic = Atomic, + have_native_atomic_ops = HaveNativeAtomicOps, + 'ATMC' = ATMC, + 'NATMC' = NATMC} = AC) -> + NativeVar = case DW of + true -> ["(&", Arg1, "->native)"]; + false -> Arg1 + end, + RtchkBegin = [" +#if defined(", ?DW_RTCHK_MACRO, ") + if (", ?DW_RTCHK_MACRO, ") { +#endif +"], + RtchkEnd = fun (Return, Operation, Barrier) -> + [" +#if defined(", ?DW_RTCHK_MACRO, ") + } else { ", rtchk_fallback_call(Return, AC, Operation, Barrier), " } +#endif\n" + ] + end, + [" +#if (defined(", HaveNativeAtomicOps, ") \\ + && (defined(ETHR_", ATMC, "_INLINE__) || defined(ETHR_ATOMIC_IMPL__))) +", + translate_have_defs(AC), + cmpxchg_fallback_define(AC), + %% addr() is special + " + + +/* --- addr() --- */ + +", func_header(AC, inline_implementation, false, addr, none), " +{", case DW of + true -> RtchkBegin; + false -> "" + end, " + return (", AddrAintT, " *) ETHR_", NATMC, "_ADDR_FUNC__(", NativeVar, "); +",case DW of + true -> RtchkEnd(true, addr, none); + false -> "" + end, " +} +", + lists:map(fun (Op) -> + OpStr = opstr(Op), + [" + +/* --- ", OpStr, "() --- */ + +", + lists:map(fun (B) -> + TryBarriers = try_barrier_order(B), + [" +", func_header(AC, inline_implementation, false, Op, B), " +{ +", + case is_return_op(AC, Op) of + true -> + [" ", RetType, " ", RetVar, ";\n"]; + _ -> "" + end, + case DW of + true -> + [RtchkBegin, + "\n", + su_dw_native_barrier_op(AC, "#if", Op, B, hd(TryBarriers)), + lists:map(fun (NB) -> + su_dw_native_barrier_op(AC, "#elif", Op, B, NB) + end, + tl(TryBarriers)), + lists:map(fun (NB) -> + dw_native_barrier_op(AC, "#elif", "", Op, B, NB) + end, + TryBarriers), + case simple_fallback(AC, Op, B) of + "" -> + %% No simple fallback available; + %% use cmpxchg() fallbacks... + [cmpxchg_fallbacks(AC#atomic_context{'NATMC' = ["SU_", NATMC]}, true, Op, B), + cmpxchg_fallbacks(AC, false, Op, B), + "#else +#error \"Missing implementation of ", Atomic, "_", opstr(Op), op_barrier_ext(B), "()!\" +#endif +" + ]; + SimpleFallback -> + ["#else\n", SimpleFallback, "#endif\n"] + end, + RtchkEnd(false, Op, B), "\n"]; + false -> + [native_barrier_op(AC, "#if", "", Op, B, hd(TryBarriers), true), + lists:map(fun (NB) -> + native_barrier_op(AC, "#elif", "", Op, B, NB, true) + end, + tl(TryBarriers)), + case simple_fallback(AC, Op, B) of + "" -> + %% No simple fallback available; + %% use cmpxchg() fallbacks... + [cmpxchg_fallbacks(AC, false, Op, B), + "#else +#error \"Missing implementation of ", Atomic, "_", opstr(Op), op_barrier_ext(B), "()!\" +#endif +" + ]; + SimpleFallback -> + ["#else\n", SimpleFallback, "#endif\n"] + end] + end, + case is_return_op(AC, Op) of + true -> + [" return ", RetVar, ";\n"]; + false -> + "" + end, + "}\n"] + end, + ?BARRIERS)] + end, + case DW of + true -> ?DW_ATOMIC_OPS; + false -> ?ATOMIC_OPS + end), + " +#endif /* ETHR_", ATMC, "_INLINE__ */ +" + ]. + +atomic_implementation_comment(AtomicSize) -> + CSz = case AtomicSize of + "dword" -> "Double word size"; + "word" -> "Word size"; + _ -> AtomicSize ++ "-bit" + end, + [" + +/* ---------- ", CSz, " atomic implementation ---------- */ + +" + ]. + +write_h_file(FileName) -> + {ok, FD} = file:open(FileName, [write, latin1]), + ok = file:write(FD, comments()), + ok = file:write(FD, " +#ifndef ETHR_ATOMICS_H__ +#define ETHR_ATOMICS_H__ +" + ), + ok = file:write(FD, h_top()), + ok = lists:foreach(fun (AtomicSize) -> + AC = atomic_context(AtomicSize), + ok = file:write(FD, + [atomic_implementation_comment(AtomicSize), + make_prototypes(AC), + make_implementations(AC)]) + end, + ?ATOMIC_SIZES), + ok = file:write(FD, " +#endif /* ETHR_ATOMICS_H__ */ +" + ), + ok = file:close(FD). + + +make_native_impl_op(#atomic_context{dw = DW, + atomic = Atomic, + have_native_atomic_ops = HaveNativeAtomicOps, + ret_var = RetVar, + arg1 = Arg1, + arg2 = Arg2, + arg3 = Arg3}, Op, B) -> + ["#if defined(", HaveNativeAtomicOps, ")", + case DW of + true -> [" && !defined(", ?DW_RTCHK_MACRO, ")"]; + false -> "" + end, + "\n", + " ", op_call(Op, DW, [RetVar, " = "], [Atomic, "_", opstr(Op), op_barrier_ext(B), "__"], Arg1, Arg2, Arg3, ""), + "\n"]. + +amc_op_dw_arg(#atomic_context{dw = false}) -> + "0"; +amc_op_dw_arg(#atomic_context{dw = true}) -> + "1". + +amc_op_arg_prefix(#atomic_context{dw = false}) -> + "&"; +amc_op_arg_prefix(#atomic_context{dw = true}) -> + "". + +amc_sint_arg(#atomic_context{dw = DW, arg2 = Arg}, arg2) -> + amc_sint_arg(DW, Arg); +amc_sint_arg(#atomic_context{dw = DW, arg3 = Arg}, arg3) -> + amc_sint_arg(DW, Arg); +amc_sint_arg(#atomic_context{dw = DW, ret_var = Arg}, ret_var) -> + amc_sint_arg(DW, Arg); +amc_sint_arg(true, Arg) -> + [Arg, "->" ?DW_SINT_FIELD]; +amc_sint_arg(false, Arg) -> + ["&", Arg]. + +amc_op_call(#atomic_context{arg1 = Arg1} = AC, init) -> + [" amc_init(&", Arg1, "->amc, ", amc_op_dw_arg(AC), ", ", amc_op_arg_prefix(AC), Arg1, "->sint, ", amc_sint_arg(AC, arg2), ");\n"]; +amc_op_call(#atomic_context{arg1 = Arg1} = AC, set) -> + [" amc_set(&", Arg1, "->amc, ", amc_op_dw_arg(AC), ", ", amc_op_arg_prefix(AC), Arg1, "->sint, ", amc_sint_arg(AC, arg2), ");\n"]; +amc_op_call(#atomic_context{dw = false, arg1 = Arg1} = AC, read) -> + [" amc_read(&", Arg1, "->amc, ", amc_op_dw_arg(AC), ", ", amc_op_arg_prefix(AC), Arg1, "->sint, ", amc_sint_arg(AC, ret_var), ");\n"]; +amc_op_call(#atomic_context{dw = true, arg1 = Arg1} = AC, read) -> + [" amc_read(&", Arg1, "->amc, ", amc_op_dw_arg(AC), ", ", amc_op_arg_prefix(AC), Arg1, "->sint, ", amc_sint_arg(AC, arg2), ");\n"]; +amc_op_call(#atomic_context{dw = false, arg1 = Arg1, arg3 = Arg3, ret_var = RetVar} = AC, cmpxchg) -> + [" ", RetVar, " = ", Arg3, "; + (void) amc_cmpxchg(&", Arg1, "->amc, ", amc_op_dw_arg(AC), ", ", amc_op_arg_prefix(AC), Arg1, "->sint, ", amc_sint_arg(AC, arg2), ", ", amc_sint_arg(AC, ret_var), ");\n"]; +amc_op_call(#atomic_context{dw = true, arg1 = Arg1, ret_var = RetVar} = AC, cmpxchg) -> + [" ", RetVar, " = amc_cmpxchg(&", Arg1, "->amc, ", amc_op_dw_arg(AC), ", ", amc_op_arg_prefix(AC), Arg1, "->sint, ", amc_sint_arg(AC, arg2), ", ", amc_sint_arg(AC, arg3), ");\n"]; +amc_op_call(#atomic_context{dw = DW, arg1 = Arg1, arg2 = Arg2, arg3 = Arg3, ret_var = RetVar}, Op) -> + OpCtxt = #op_context{ret = RetVar, var = [Arg1,"->sint"], val1 = Arg2, val2 = Arg3}, + OpStr = case DW of + true -> dw_op(Op, OpCtxt); + false -> op(Op, OpCtxt) + end, + [" ETHR_AMC_MODIFICATION_OPS__(&", Arg1, "->amc, ", OpStr, ");\n"]. + +make_amc_fallback_op(#atomic_context{amc_fallback = false}, _Op, _B) -> + ""; +make_amc_fallback_op(#atomic_context{amc_fallback = true} = AC, Op, B) -> + NB = case Op of + read -> rb; + _ -> none + end, + {PreB, PostB} = xbarriers(Op, B, NB), + ["#elif defined(ETHR_AMC_FALLBACK__)\n", + case PreB of + "" -> ""; + _ -> [" ", PreB, "\n"] + end, + amc_op_call(AC, Op), + case PostB of + "" -> ""; + _ -> [" ", PostB, "\n"] + end]. + +make_locked_fallback_op(#atomic_context{dw = DW, + ret_var = RetVar, + arg1 = Arg1, + arg2 = Arg2, + arg3 = Arg3}, Op, B) -> + OpStr = case DW of + true -> + dw_op(Op, #op_context{ret = RetVar, + var = [Arg1, "->" ?DW_SINT_FIELD], + val1 = [Arg2, "->" ?DW_SINT_FIELD], + val2 = [Arg3, "->" ?DW_SINT_FIELD]}); + false -> + op(Op, #op_context{ret = RetVar, + var = ["*", Arg1], + val1 = Arg2, + val2 = Arg3}) + end, + {PreB, PostB} = xbarriers(Op, B, none), + ["#else\n", + case PreB of + "" -> ""; + _ -> [" ", PreB, "\n"] + end, + [" ETHR_ATOMIC_OP_FALLBACK_IMPL__(", Arg1, ", ", OpStr, ");\n"], + case PostB of + "" -> ""; + _ -> [" ", PostB, "\n"] + end, + "#endif\n"]. + +make_symbol_to_fallback_impl(#atomic_context{dw = true, + atomic = Atomic, + arg1 = Arg1, + arg2 = Arg2, + arg3 = Arg3} = AC, + Op, B) -> + [" +#ifdef ", ?DW_RTCHK_MACRO, " +", func_header(AC, implementation, false, Op, B), " +{", + case Op of + init -> ""; + _ -> ["\n ETHR_ASSERT(!ethr_not_inited__);"] + end, " + ETHR_ASSERT(", Arg1, "); + ", op_call(Op, true, "return", [Atomic, "_", opstr(Op), op_barrier_ext(B), "__"], Arg1, Arg2, Arg3, ""), " +} +#endif +" + ]; +make_symbol_to_fallback_impl(_, _, _) -> + "". + +make_symbol_implementations(#atomic_context{dw = DW, + amc_fallback = AMC, + ret_type = RetType, + addr_aint_t = AddrAintT, + ret_var = RetVar, + arg1 = Arg1} = AC) -> + FallbackVar = case DW of + true -> ["(&", Arg1, "->fallback)"]; + false -> Arg1 + end, + [" +", + case DW of + true -> [" +/* + * Double word atomics need runtime test. + */ + +int ethr_have_native_dw_atomic(void) +{ + return ethr_have_native_dw_atomic__(); +} + "]; + false -> "" + end, " + +/* --- addr() --- */ + +", func_header(AC, implementation, + case DW of + true -> ?DW_FUNC_MACRO; + false -> false + end, addr, none), " +{ + ", AddrAintT, " *", RetVar, "; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(", Arg1, "); +", make_native_impl_op(AC, addr, none), + case AMC of + true -> ["#elif defined(ETHR_AMC_FALLBACK__) + ", RetVar ," = (", AddrAintT, " *) (", FallbackVar, ")->sint;"]; + false -> "" + end, " +#else + ", RetVar, " = (", AddrAintT, " *) ", FallbackVar, "; +#endif + return ", RetVar, "; +} +", + make_symbol_to_fallback_impl(AC, addr, none), + lists:map(fun (Op) -> + [" + +/* -- ", opstr(Op), "() -- */ + +", + lists:map(fun (B) -> + ["\n", + func_header(AC, implementation, + case DW of + true -> ?DW_FUNC_MACRO; + false -> false + end, Op, B), + "\n{\n", + case is_return_op(AC, Op) of + true -> [" ", RetType, " ", RetVar, ";\n"]; + false -> "" + end, + case Op of + init -> ""; + _ -> [" ETHR_ASSERT(!ethr_not_inited__);\n"] + end, + [" ETHR_ASSERT(", Arg1, ");\n"], + make_native_impl_op(AC, Op, B), + make_amc_fallback_op(AC#atomic_context{arg1 = FallbackVar}, Op, B), + make_locked_fallback_op(AC#atomic_context{arg1 = FallbackVar}, Op, B), + case is_return_op(AC, Op) of + true -> [" return ", RetVar, ";" + ]; + false -> + "" + end, + "\n}\n", + make_symbol_to_fallback_impl(AC, Op, B)] + end, + ?BARRIERS)] + end, + case DW of + true -> ?DW_ATOMIC_OPS; + false -> ?ATOMIC_OPS + end)]. + +make_info_functions() -> + [" + + +/* --------- Info functions --------- */ + +#if defined(", ?DW_RTCHK_MACRO, ") +char *zero_ops[] = {NULL}; +#endif +", + [lists:map(fun (NBits) -> + {DW, Bits} = case NBits of + "su_dw" -> {"su_dw_", ""}; + "dw" -> {"dw_", ""}; + _ -> {"", NBits} + end, + [" + +static char *native_", DW, "atomic", Bits, "_ops[] = {", + lists:map(fun (Op) -> + NOpStr = opstr(native(Op)), + CapNOpStr = to_upper(NOpStr), + lists:map(fun (B) -> + HaveNative = case NBits of + "dw" -> + "ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC"; + "su_dw" -> + "ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC"; + _ -> + [?HAVE_NATIVE_ATOMIC, NBits] + end, + NBExt = op_barrier_ext(B), + CapNBExt = to_upper(NBExt), + [" +#ifdef ", HaveNative, "_", CapNOpStr, CapNBExt, " + \"", NOpStr, NBExt, "\", +#endif" + ] + end, + ?BARRIERS) + end, + case NBits of + "dw" -> ?DW_ATOMIC_OPS; + "su_dw" -> ?DW_ATOMIC_OPS; + _ -> ?ATOMIC_OPS + end), " + NULL +}; + +char ** +ethr_native_", DW, "atomic", Bits, "_ops(void) +{ +", + case DW of + "" -> ""; + _ -> [" +#if defined(", ?DW_RTCHK_MACRO, ") + if (!", ?DW_RTCHK_MACRO, ") + return &zero_ops[0]; +#endif" + ] + end, " + return &native_", DW, "atomic", Bits, "_ops[0]; +} +" + ] + end, ["su_dw", "dw" | ?POTENTIAL_NBITS])]]. + +write_c_file(FileName) -> + {ok, FD} = file:open(FileName, [write, latin1]), + ok = file:write(FD, comments()), + ok = file:write(FD, c_top()), + lists:foreach(fun (AtomicSize) -> + ok = file:write(FD, + [atomic_implementation_comment(AtomicSize), + make_symbol_implementations(atomic_context(AtomicSize))]) + end, + ?ATOMIC_SIZES), + ok = file:write(FD, make_info_functions()). + + +main([]) -> + case os:getenv("ERL_TOP") of + false -> + io:format("$ERL_TOP not set!~n", []), + halt(1); + ErlTop -> + HFile = filename:join(ErlTop, ?H_FILE), + WHFile = fun () -> + write_h_file(HFile) + end, + CFile = filename:join(ErlTop, ?C_FILE), + WCFile = fun () -> + write_c_file(CFile) + end, + case erlang:system_info(schedulers_online) of + 1 -> + WHFile(), + WCFile(); + _ -> + {HPid, HMon} = spawn_monitor(WHFile), + {CPid, CMon} = spawn_monitor(WCFile), + receive + {'DOWN', HMon, process, HPid, HReason} -> + normal = HReason + end, + receive + {'DOWN', CMon, process, CPid, CReason} -> + normal = CReason + end + end, + io:format("Wrote: ~s~n", [HFile]), + io:format("Wrote: ~s~n", [CFile]), + init:stop() + end. + +a2l(A) -> + atom_to_list(A). + +opstr(A) -> + a2l(A). + +to_upper([]) -> + []; +to_upper([C|Cs]) when is_list(C) -> + [to_upper(C)|to_upper(Cs)]; +to_upper([C|Cs]) when is_integer(C), 97 =< C, C =< 122 -> + [C-32|to_upper(Cs)]; +to_upper([C|Cs]) -> + [C|to_upper(Cs)]. + + +comments() -> + Years = case erlang:date() of + {2011, _, _} -> "2011"; + {Y, _, _} -> "2011-"++integer_to_list(Y) + end, + ["/* + * --------------- DO NOT EDIT THIS FILE! --------------- + * This file was automatically generated by the + * \$ERL_TOP/erts/lib_src/utils/make_atomics_api script. + * If you need to make changes, edit the script and + * regenerate this file. + * --------------- DO NOT EDIT THIS FILE! --------------- + */ + +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB ", Years, ". All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the \"License\"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an \"AS IS\" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: The ethread atomics API + * Author: Rickard Green + */ + +/* + * This file maps native atomic implementations to ethread + * API atomics. If no native atomic implementation + * is available, a less efficient fallback is used instead. + * The API consists of 32-bit size, word size (pointer size), + * and double word size atomics. + * + * The following atomic operations are implemented for + * 32-bit size, and word size atomics: +", + lists:map(fun (Op) -> + [" * - ", opstr(Op), "\n"] + end, + ?ATOMIC_OPS), + " * + * The following atomic operations are implemented for + * double word size atomics: +", + lists:map(fun (Op) -> + [" * - ", opstr(Op), "\n"] + end, + ?DW_ATOMIC_OPS), + " * + * Appart from a function implementing the atomic operation + * with unspecified memory barrier semantics, there are + * functions implementing each operation with the following + * memory barrier semantics: +", + lists:map(fun (none) -> + ""; + (rb) -> + [" * - rb (read barrier)\n"]; + (wb) -> + [" * - wb (write barrier)\n"]; + (acqb) -> + [" * - acqb (acquire barrier)\n"]; + (relb) -> + [" * - relb (release barrier)\n"]; + (mb) -> + [" * - mb (full memory barrier)\n"]; + (B) -> + [" * - ", a2l(B), "\n"] + end, + ?BARRIERS), + " * + * We implement all of these operation/barrier + * combinations, regardless of whether they are useful + * or not (some of them are useless). + * + * Double word size atomic functions are on the followning + * form: + * ethr_dw_atomic_<OP>[_<BARRIER>] + * + * Word size atomic functions are on the followning + * form: + * ethr_atomic_<OP>[_<BARRIER>] + * + * 32-bit size atomic functions are on the followning + * form: + * ethr_atomic32_<OP>[_<BARRIER>] + * + * Apart from the operation/barrier functions + * described above also 'addr' functions are implemented + * which return the actual memory address used of the + * atomic variable. The 'addr' functions have no barrier + * versions. + * + * The native atomic implementation does not need to + * implement all operation/barrier combinations. + * Functions that have no native implementation will be + * constructed from existing native functionality. These + * functions will perform the wanted operation and will + * produce sufficient memory barriers, but may + * in some cases be less efficient than pure native + * versions. + * + * When we create ethread API operation/barrier functions by + * adding barriers before and after native operations it is + * assumed that: + * - A native read operation begins, and ends with a load. + * - A native set operation begins, and ends with a store. + * - An init operation begins with either a load, or a store, + * and ends with either a load, or a store. + * - All other operations begins with a load, and ends with + * either a load, or a store. + * + * This is the minimum functionality that a native + * implementation needs to provide: + * + * - Functions that need to be implemented: + * + * - ethr_native_[dw_|su_dw_]atomic[BITS]_addr + * - ethr_native_[dw_|su_dw_]atomic[BITS]_cmpxchg[_<BARRIER>] + * (at least one cmpxchg of optional barrier) + * + * - Macros that needs to be defined: + * + * A macro informing about the presence of the native + * implementation: + * + * - ETHR_HAVE_NATIVE_[DW_|SU_DW_]ATOMIC[BITS] + * + * A macro naming (a string constant) the implementation: + * + * - ETHR_NATIVE_[DW_]ATOMIC[BITS]_IMPL + * + * Each implemented native atomic function has to + * be accompanied by a defined macro on the following + * form informing about its presence: + * + * - ETHR_HAVE_ETHR_NATIVE_[DW_|SU_DW_]ATOMIC[BITS]_<OP>[_<BARRIER>] + * + * A (sparc-v9 style) membar macro: + * + * - ETHR_MEMBAR(B) + * + * Which takes a combination of the following macros + * or:ed (using |) together: + * + * - ETHR_LoadLoad + * - ETHR_LoadStore + * - ETHR_StoreLoad + * - ETHR_StoreStore + * + */ +" + ]. + +h_top() -> + [" +#undef ETHR_AMC_FALLBACK__ +#undef ETHR_AMC_NO_ATMCS__ +#undef ETHR_AMC_ATMC_T__ +#undef ETHR_AMC_ATMC_FUNC__ + +/* -- 32-bit atomics -- */ + +#undef ETHR_NAINT32_T__ +#undef ETHR_NATMC32_FUNC__ +#undef ETHR_NATMC32_ADDR_FUNC__ +#undef ETHR_NATMC32_BITS__ +#if defined(ETHR_HAVE_NATIVE_ATOMIC32) +# define ETHR_NEED_NATMC32_ADDR +# define ETHR_NATMC32_ADDR_FUNC__ ethr_native_atomic32_addr +typedef ethr_native_atomic32_t ethr_atomic32_t; +# define ETHR_NAINT32_T__ ethr_sint32_t +# define ETHR_NATMC32_FUNC__(X) ethr_native_atomic32_ ## X +# define ETHR_NATMC32_BITS__ 32 +#elif defined(ETHR_HAVE_NATIVE_ATOMIC64) +# define ETHR_NEED_NATMC64_ADDR +#ifdef ETHR_BIGENDIAN +# define ETHR_NATMC32_ADDR_FUNC__(VAR) \\ + (((ethr_sint32_t *) ethr_native_atomic64_addr((VAR))) + 1) +#else +# define ETHR_NATMC32_ADDR_FUNC__(VAR) \\ + ((ethr_sint32_t *) ethr_native_atomic64_addr((VAR))) +#endif +typedef ethr_native_atomic64_t ethr_atomic32_t; +# define ETHR_NAINT32_T__ ethr_sint64_t +# define ETHR_NATMC32_FUNC__(X) ethr_native_atomic64_ ## X +# define ETHR_NATMC32_BITS__ 64 +#else +/* + * No native atomics usable for 32-bits atomics :( + * Use fallback... + */ +typedef ethr_sint32_t ethr_atomic32_t; +#endif + +#undef ETHR_ATMC32_INLINE__ +#ifdef ETHR_NATMC32_BITS__ +# ifdef ETHR_TRY_INLINE_FUNCS +# define ETHR_ATMC32_INLINE__ +# endif +# define ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS +#endif + +#if !defined(ETHR_ATMC32_INLINE__) || defined(ETHR_ATOMIC_IMPL__) +# define ETHR_NEED_ATMC32_PROTOTYPES__ +#endif + +#ifndef ETHR_INLINE_ATMC32_FUNC_NAME_ +# define ETHR_INLINE_ATMC32_FUNC_NAME_(X) X +#endif + +#undef ETHR_ATMC32_FUNC__ +#define ETHR_ATMC32_FUNC__(X) ETHR_INLINE_ATMC32_FUNC_NAME_(ethr_atomic32_ ## X) + + +/* -- Word size atomics -- */ + +#undef ETHR_NEED_NATMC32_ADDR +#undef ETHR_NEED_NATMC64_ADDR + +#undef ETHR_NAINT_T__ +#undef ETHR_NATMC_FUNC__ +#undef ETHR_NATMC_ADDR_FUNC__ +#undef ETHR_NATMC_BITS__ +#if ETHR_SIZEOF_PTR == 8 && defined(ETHR_HAVE_NATIVE_ATOMIC64) +# ifndef ETHR_NEED_NATMC64_ADDR +# define ETHR_NEED_NATMC64_ADDR +# endif +# define ETHR_NATMC_ADDR_FUNC__ ethr_native_atomic64_addr +typedef ethr_native_atomic64_t ethr_atomic_t; +# define ETHR_NAINT_T__ ethr_sint64_t +# define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X +# define ETHR_NATMC_BITS__ 64 +#elif ETHR_SIZEOF_PTR == 4 && defined(ETHR_HAVE_NATIVE_ATOMIC32) +# ifndef ETHR_NEED_NATMC64_ADDR +# define ETHR_NEED_NATMC32_ADDR +# endif +# define ETHR_NATMC_ADDR_FUNC__ ethr_native_atomic32_addr +typedef ethr_native_atomic32_t ethr_atomic_t; +# define ETHR_NAINT_T__ ethr_sint32_t +# define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X +# define ETHR_NATMC_BITS__ 32 +#elif ETHR_SIZEOF_PTR == 4 && defined(ETHR_HAVE_NATIVE_ATOMIC64) +# ifndef ETHR_NEED_NATMC64_ADDR +# define ETHR_NEED_NATMC64_ADDR +# endif +#ifdef ETHR_BIGENDIAN +# define ETHR_NATMC_ADDR_FUNC__(VAR) \\ + (((ethr_sint32_t *) ethr_native_atomic64_addr((VAR))) + 1) +#else +# define ETHR_NATMC_ADDR_FUNC__(VAR) \\ + ((ethr_sint32_t *) ethr_native_atomic64_addr((VAR))) +#endif +typedef ethr_native_atomic64_t ethr_atomic_t; +# define ETHR_NATMC_T__ ethr_native_atomic64_t +# define ETHR_NAINT_T__ ethr_sint64_t +# define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X +# define ETHR_NATMC_BITS__ 64 +#else +/* + * No native atomics usable for pointer size atomics :( + * Use fallback... + */ + +# if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) +# define ETHR_AMC_FALLBACK__ +# define ETHR_AMC_NO_ATMCS__ 2 +# define ETHR_AMC_SINT_T__ ethr_sint32_t +# define ETHR_AMC_ATMC_T__ ethr_atomic32_t +# define ETHR_AMC_ATMC_FUNC__(X) ETHR_INLINE_ATMC32_FUNC_NAME_(ethr_atomic32_ ## X) +typedef struct { + ETHR_AMC_ATMC_T__ atomic[ETHR_AMC_NO_ATMCS__]; +} ethr_amc_t; +typedef struct { + ethr_amc_t amc; + ethr_sint_t sint; +} ethr_atomic_t; +# else /* locked fallback */ +typedef ethr_sint_t ethr_atomic_t; +# endif +#endif + +#undef ETHR_ATMC_INLINE__ +#ifdef ETHR_NATMC_BITS__ +# ifdef ETHR_TRY_INLINE_FUNCS +# define ETHR_ATMC_INLINE__ +# endif +# define ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS +#endif + +#if !defined(ETHR_ATMC_INLINE__) || defined(ETHR_ATOMIC_IMPL__) +# define ETHR_NEED_ATMC_PROTOTYPES__ +#endif + +#ifndef ETHR_INLINE_ATMC_FUNC_NAME_ +# define ETHR_INLINE_ATMC_FUNC_NAME_(X) X +#endif + +#undef ETHR_ATMC_FUNC__ +#define ETHR_ATMC_FUNC__(X) ETHR_INLINE_ATMC_FUNC_NAME_(ethr_atomic_ ## X) + +/* -- Double word atomics -- */ + +#undef ETHR_SU_DW_NAINT_T__ +#undef ETHR_SU_DW_NATMC_FUNC__ +#undef ETHR_SU_DW_NATMC_ADDR_FUNC__ +#undef ETHR_DW_NATMC_FUNC__ +#undef ETHR_DW_NATMC_ADDR_FUNC__ +#undef ETHR_DW_NATMC_BITS__ +#if defined(ETHR_HAVE_NATIVE_DW_ATOMIC) || defined(ETHR_HAVE_NATIVE_SU_DW_ATOMIC) +# define ETHR_NEED_DW_NATMC_ADDR +# define ETHR_DW_NATMC_ADDR_FUNC__ ethr_native_dw_atomic_addr +# define ETHR_NATIVE_DW_ATOMIC_T__ ethr_native_dw_atomic_t +# define ETHR_DW_NATMC_FUNC__(X) ethr_native_dw_atomic_ ## X +# define ETHR_SU_DW_NATMC_FUNC__(X) ethr_native_su_dw_atomic_ ## X +# if ETHR_SIZEOF_PTR == 8 +# define ETHR_DW_NATMC_BITS__ 128 +# elif ETHR_SIZEOF_PTR == 4 +# define ETHR_DW_NATMC_BITS__ 64 +# else +# error \"Word size not supported\" +# endif +# ifdef ETHR_NATIVE_SU_DW_SINT_T +# define ETHR_SU_DW_NAINT_T__ ETHR_NATIVE_SU_DW_SINT_T +# endif +#elif ETHR_SIZEOF_PTR == 4 && defined(ETHR_HAVE_NATIVE_ATOMIC64) +# define ETHR_HAVE_NATIVE_SU_DW_ATOMIC +# ifndef ETHR_NEED_NATMC64_ADDR +# define ETHR_NEED_NATMC64_ADDR +# endif +# define ETHR_DW_NATMC_ADDR_FUNC__(VAR) \\ + ((ethr_dw_sint_t *) ethr_native_atomic64_addr((VAR))) +# define ETHR_NATIVE_DW_ATOMIC_T__ ethr_native_atomic64_t +# define ETHR_SU_DW_NAINT_T__ ethr_sint64_t +# define ETHR_SU_DW_NATMC_FUNC__(X) ethr_native_atomic64_ ## X +# define ETHR_DW_NATMC_BITS__ 64 +#endif + +#if defined(", ?DW_RTCHK_MACRO, ") +#define ", ?DW_FUNC_MACRO, "(X) ethr_dw_atomic_ ## X ## _fallback__ +#else +#define ", ?DW_FUNC_MACRO, "(X) ethr_dw_atomic_ ## X +#endif + +#if !defined(ETHR_DW_NATMC_BITS__) || defined(", ?DW_RTCHK_MACRO, ") +# define ETHR_NEED_DW_FALLBACK__ +#endif + +#if defined(ETHR_NEED_DW_FALLBACK__) +/* + * No native atomics usable for double word atomics :( + * Use fallback... + */ + +# ifndef ETHR_AMC_FALLBACK__ +# if ETHR_SIZEOF_PTR == 8 && defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) +# define ETHR_AMC_FALLBACK__ +# define ETHR_AMC_NO_ATMCS__ 1 +# define ETHR_AMC_SINT_T__ ethr_sint_t +# define ETHR_AMC_ATMC_T__ ethr_atomic_t +# define ETHR_AMC_ATMC_FUNC__(X) ETHR_INLINE_ATMC_FUNC_NAME_(ethr_atomic_ ## X) +# elif defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) +# define ETHR_AMC_FALLBACK__ +# define ETHR_AMC_NO_ATMCS__ 2 +# define ETHR_AMC_SINT_T__ ethr_sint32_t +# define ETHR_AMC_ATMC_T__ ethr_atomic32_t +# define ETHR_AMC_ATMC_FUNC__(X) ETHR_INLINE_ATMC32_FUNC_NAME_(ethr_atomic32_ ## X) +# endif +# ifdef ETHR_AMC_FALLBACK__ +typedef struct { + ETHR_AMC_ATMC_T__ atomic[ETHR_AMC_NO_ATMCS__]; +} ethr_amc_t; +# endif +# endif + +typedef struct { +#ifdef ETHR_AMC_FALLBACK__ + ethr_amc_t amc; +#endif + ethr_sint_t sint[2]; +} ethr_dw_atomic_fallback_t; + +#endif + +typedef union { +#ifdef ETHR_NATIVE_DW_ATOMIC_T__ + ETHR_NATIVE_DW_ATOMIC_T__ native; +#endif +#ifdef ETHR_NEED_DW_FALLBACK__ + ethr_dw_atomic_fallback_t fallback; +#endif + ethr_sint_t sint[2]; +} ethr_dw_atomic_t; + +typedef union { +#ifdef ETHR_SU_DW_NAINT_T__ + ETHR_SU_DW_NAINT_T__ ", ?SU_DW_SINT_FIELD, "; +#endif + ethr_sint_t ", ?DW_SINT_FIELD, "[2]; +} ethr_dw_sint_t; + +#ifdef ETHR_BIGENDIAN +# define ETHR_DW_SINT_LOW_WORD 1 +# define ETHR_DW_SINT_HIGH_WORD 0 +#else +# define ETHR_DW_SINT_LOW_WORD 0 +# define ETHR_DW_SINT_HIGH_WORD 1 +#endif + +#undef ETHR_DW_ATMC_INLINE__ +#ifdef ETHR_DW_NATMC_BITS__ +# ifdef ETHR_TRY_INLINE_FUNCS +# define ETHR_ATMC32_INLINE__ +# endif +# define ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS +#endif + +#if !defined(ETHR_DW_ATMC_INLINE__) || defined(ETHR_ATOMIC_IMPL__) +# define ETHR_NEED_DW_ATMC_PROTOTYPES__ +#endif + +#ifndef ETHR_INLINE_DW_ATMC_FUNC_NAME_ +# define ETHR_INLINE_DW_ATMC_FUNC_NAME_(X) X +#endif + +#undef ETHR_DW_ATMC_FUNC__ +#define ETHR_DW_ATMC_FUNC__(X) ETHR_INLINE_DW_ATMC_FUNC_NAME_(ethr_dw_atomic_ ## X) + +#if defined(ETHR_NEED_DW_ATMC_PROTOTYPES__) +int ethr_have_native_dw_atomic(void); +#endif +#if defined(ETHR_DW_ATMC_INLINE__) || defined(ETHR_ATOMIC_IMPL__) +static ETHR_INLINE int +ETHR_INLINE_DW_ATMC_FUNC_NAME_(ethr_have_native_dw_atomic)(void) +{ +#if defined(", ?DW_RTCHK_MACRO, ") + return ", ?DW_RTCHK_MACRO, "; +#elif defined(ETHR_DW_NATMC_BITS__) + return 1; +#else + return 0; +#endif +} +#endif + +/* -- Misc -- */ + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) +/* + * Unusual values are used by read() fallbacks implemented via cmpxchg(). + * We want to use an unusual value in hope that it is more efficient + * not to match the value in memory. + * + * - Negative integer values are probably more unusual. + * - Very large absolute integer values are probably more unusual. + * - Odd pointers are probably more unusual (only char pointers can be odd). + */ +# define ETHR_UNUSUAL_SINT32_VAL__ ((ethr_sint32_t) 0x81818181) +# if ETHR_SIZEOF_PTR == 4 +# define ETHR_UNUSUAL_SINT_VAL__ ((ethr_sint_t) ETHR_UNUSUAL_SINT32_VAL__) +# elif ETHR_SIZEOF_PTR == 8 +# define ETHR_UNUSUAL_SINT_VAL__ ((ethr_sint_t) 0x8181818181818181L) +# else +# error \"Word size not supported\" +# endif +# if defined(ETHR_NEED_DW_NATMC_ADDR) && !defined(ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_ADDR) +# error \"No ethr_native_dw_atomic_addr() available\" +# endif +# if defined(ETHR_NEED_NATMC32_ADDR) && !defined(ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADDR) +# error \"No ethr_native_atomic32_addr() available\" +# endif +# if defined(ETHR_NEED_NATMC64_ADDR) && !defined(ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADDR) +# error \"No ethr_native_atomic64_addr() available\" +# endif +#endif + +#if defined(__GNUC__) +# ifndef ETHR_COMPILER_BARRIER +# define ETHR_COMPILER_BARRIER __asm__ __volatile__(\"\" : : : \"memory\") +# endif +#elif defined(ETHR_WIN32_THREADS) +# ifndef ETHR_COMPILER_BARRIER +# include <intrin.h> +# pragma intrinsic(_ReadWriteBarrier) +# define ETHR_COMPILER_BARRIER _ReadWriteBarrier() +# endif +#endif + +void ethr_compiler_barrier_fallback(void); +#ifndef ETHR_COMPILER_BARRIER +# define ETHR_COMPILER_BARRIER ethr_compiler_barrier_fallback() +#endif + +int ethr_init_atomics(void); + +/* info */ +char **ethr_native_atomic32_ops(void); +char **ethr_native_atomic64_ops(void); +char **ethr_native_dw_atomic_ops(void); +char **ethr_native_su_dw_atomic_ops(void); + +#if !defined(ETHR_DW_NATMC_BITS__) && !defined(ETHR_NATMC_BITS__) && !defined(ETHR_NATMC32_BITS__) +/* + * ETHR_*MEMORY_BARRIER orders between locked and atomic accesses only, + * i.e. when no native atomic implementation exist and only our lock + * based atomic fallback is used, a noop is sufficient. + */ +# undef ETHR_MEMORY_BARRIER +# undef ETHR_WRITE_MEMORY_BARRIER +# undef ETHR_READ_MEMORY_BARRIER +# undef ETHR_READ_DEPEND_MEMORY_BARRIER +# undef ETHR_MEMBAR +# define ETHR_MEMBAR(B) do { } while (0) +#endif + +#ifndef ETHR_MEMBAR +# error \"No ETHR_MEMBAR defined\" +#endif + +#define ETHR_MEMORY_BARRIER ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore) +#define ETHR_WRITE_MEMORY_BARRIER ETHR_MEMBAR(ETHR_StoreStore) +#define ETHR_READ_MEMORY_BARRIER ETHR_MEMBAR(ETHR_LoadLoad) +#ifdef ETHR_READ_DEPEND_MEMORY_BARRIER +# undef ETHR_ORDERED_READ_DEPEND +#else +# define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_COMPILER_BARRIER +# define ETHR_ORDERED_READ_DEPEND +#endif +"]. + +c_top() -> + [" + +#ifdef HAVE_CONFIG_H +#include \"config.h\" +#endif + +#define ETHR_TRY_INLINE_FUNCS +#define ETHR_INLINE_DW_ATMC_FUNC_NAME_(X) X ## __ +#define ETHR_INLINE_ATMC_FUNC_NAME_(X) X ## __ +#define ETHR_INLINE_ATMC32_FUNC_NAME_(X) X ## __ +#define ETHR_ATOMIC_IMPL__ + +#include \"ethread.h\" +#include \"ethr_internal.h\" + +#if (!defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) \\ + || !defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)) +/* + * Spinlock based fallback for atomics used in absence of a native + * implementation. + */ + +#define ETHR_ATMC_FLLBK_ADDR_BITS ", ?ETHR_ATMC_FLLBK_ADDR_BITS, " +#define ETHR_ATMC_FLLBK_ADDR_SHIFT ", ?ETHR_ATMC_FLLBK_ADDR_SHIFT, " + +typedef struct { + union { + ethr_spinlock_t lck; + char buf[ETHR_CACHE_LINE_ALIGN_SIZE(sizeof(ethr_spinlock_t))]; + } u; +} ethr_atomic_protection_t; + +extern ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATMC_FLLBK_ADDR_BITS]; + +#define ETHR_ATOMIC_PTR2LCK__(PTR) \\ +(ðr_atomic_protection__[((((ethr_uint_t) (PTR)) >> ETHR_ATMC_FLLBK_ADDR_SHIFT) \\ + & ((1 << ETHR_ATMC_FLLBK_ADDR_BITS) - 1))].u.lck) + + +#define ETHR_ATOMIC_OP_FALLBACK_IMPL__(AP, EXPS) \\ +do { \\ + ethr_spinlock_t *slp__ = ETHR_ATOMIC_PTR2LCK__((AP)); \\ + ethr_spin_lock(slp__); \\ + { EXPS; } \\ + ethr_spin_unlock(slp__); \\ +} while (0) + +ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATMC_FLLBK_ADDR_BITS]; + +#endif + +", make_amc_fallback(), " + +int +ethr_init_atomics(void) +{ +#if (!defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) \\ + || !defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)) + int i; + for (i = 0; i < (1 << ETHR_ATMC_FLLBK_ADDR_BITS); i++) { + int res = ethr_spinlock_init(ðr_atomic_protection__[i].u.lck); + if (res != 0) + return res; + } +#endif + return 0; +} +"]. + +make_amc_fallback() -> + [" +#if defined(ETHR_AMC_FALLBACK__) + +/* + * Fallback for large sized (word and/or double word size) atomics using + * an \"Atomic Modification Counter\" based on smaller sized native atomics. + * + * We use a 63-bit modification counter and a one bit exclusive flag. + * If 32-bit native atomics are used, we need two 32-bit native atomics. + * The exclusive flag is the least significant bit, or if multiple atomics + * are used, the least significant bit of the least significant atomic. + * + * When using the AMC fallback the following is true: + * - Reads of the same atomic variable can be done in parallel. + * - Uncontended reads doesn't cause any cache line invalidations, + * since no modifications are done. + * - Assuming that the AMC atomic(s) and the integer(s) containing the + * value of the implemented atomic resides in the same cache line, + * modifications will only cause invalidations of one cache line. + * + * When using the spinlock based fallback none of the above is true, + * however, the spinlock based fallback consumes less memory. + */ + +# if ETHR_AMC_NO_ATMCS__ != 1 && ETHR_AMC_NO_ATMCS__ != 2 +# error \"Not supported\" +# endif +# define ETHR_AMC_MAX_TRY_READ__ 10 +# ifdef ETHR_DEBUG +# define ETHR_DBG_CHK_EXCL_STATE(ASP, S) \\ +do { \\ + ETHR_AMC_SINT_T__ act = ETHR_AMC_ATMC_FUNC__(read)(&(ASP)->atomic[0]); \\ + ETHR_ASSERT(act == (S) + 1); \\ + ETHR_ASSERT(act & 1); \\ +} while (0) +# else +# define ETHR_DBG_CHK_EXCL_STATE(ASP, S) +# endif + +static ETHR_INLINE void +amc_init(ethr_amc_t *amc, int dw, ethr_sint_t *avar, ethr_sint_t *val) +{ + avar[0] = val[0]; + if (dw) + avar[1] = val[1]; +#if ETHR_AMC_NO_ATMCS__ == 2 + ETHR_AMC_ATMC_FUNC__(init)(&amc->atomic[1], 0); +#endif + ETHR_AMC_ATMC_FUNC__(init_wb)(&amc->atomic[0], 0); +} + +static ETHR_INLINE ETHR_AMC_SINT_T__ +amc_set_excl(ethr_amc_t *amc, ETHR_AMC_SINT_T__ prev_state0) +{ + ETHR_AMC_SINT_T__ state0 = prev_state0; + /* Set exclusive flag. */ + while (1) { + ETHR_AMC_SINT_T__ act_state0, new_state0; + while (state0 & 1) { /* Wait until exclusive bit has been cleared */ + ETHR_SPIN_BODY; + state0 = ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[0]); + } + /* Try to set exclusive bit */ + new_state0 = state0 + 1; + act_state0 = ETHR_AMC_ATMC_FUNC__(cmpxchg_acqb)(&amc->atomic[0], + new_state0, + state0); + if (state0 == act_state0) + return state0; /* old state0 */ + state0 = act_state0; + } +} + +static ETHR_INLINE void +amc_inc_mc_unset_excl(ethr_amc_t *amc, ETHR_AMC_SINT_T__ old_state0) +{ + ETHR_AMC_SINT_T__ state0 = old_state0; + + /* Increment modification counter and reset exclusive flag. */ + + ETHR_DBG_CHK_EXCL_STATE(amc, state0); + + state0 += 2; + + ETHR_ASSERT((state0 & 1) == 0); + +#if ETHR_AMC_NO_ATMCS__ == 2 + if (state0 == 0) { + /* + * state0 wrapped, so we need to increment state1. There is no need + * for atomic inc op, since this is always done while having exclusive + * flag. + */ + ETHR_AMC_SINT_T__ state1 = ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[1]); + state1++; + ETHR_AMC_ATMC_FUNC__(set)(&amc->atomic[1], state1); + } +#endif + ETHR_AMC_ATMC_FUNC__(set_relb)(&amc->atomic[0], state0); +} + +static ETHR_INLINE void +amc_unset_excl(ethr_amc_t *amc, ETHR_AMC_SINT_T__ old_state0) +{ + ETHR_DBG_CHK_EXCL_STATE(amc, old_state0); + /* + * Reset exclusive flag, but leave modification counter unchanged, + * i.e., restore state to what it was before setting exclusive + * flag. + */ + ETHR_AMC_ATMC_FUNC__(set_relb)(&amc->atomic[0], old_state0); +} + +static ETHR_INLINE void +amc_set(ethr_amc_t *amc, int dw, ethr_sint_t *avar, ethr_sint_t *val) +{ + ETHR_AMC_SINT_T__ state0 = ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[0]); + + state0 = amc_set_excl(amc, state0); + + avar[0] = val[0]; + if (dw) + avar[1] = val[1]; + + amc_inc_mc_unset_excl(amc, state0); +} + +static ETHR_INLINE int +amc_try_read(ethr_amc_t *amc, int dw, ethr_sint_t *avar, + ethr_sint_t *val, ETHR_AMC_SINT_T__ *state0p) +{ + /* *state0p should contain last read value if aborting */ + ETHR_AMC_SINT_T__ old_state0; +#if ETHR_AMC_NO_ATMCS__ == 2 + ETHR_AMC_SINT_T__ state1; + int abrt; +#endif + + *state0p = ETHR_AMC_ATMC_FUNC__(read_rb)(&amc->atomic[0]); + if ((*state0p) & 1) + return 0; /* exclusive flag set; abort */ +#if ETHR_AMC_NO_ATMCS__ == 2 + state1 = ETHR_AMC_ATMC_FUNC__(read_rb)(&amc->atomic[1]); +#else + ETHR_COMPILER_BARRIER; +#endif + + val[0] = avar[0]; + if (dw) + val[1] = avar[1]; + + ETHR_READ_MEMORY_BARRIER; + + /* + * Abort if state has changed (i.e, either the exclusive + * flag is set, or modification counter changed). + */ + old_state0 = *state0p; +#if ETHR_AMC_NO_ATMCS__ == 2 + *state0p = ETHR_AMC_ATMC_FUNC__(read_rb)(&amc->atomic[0]); + abrt = (old_state0 != *state0p); + abrt |= (state1 != ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[1])); + return abrt == 0; +#else + *state0p = ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[0]); + return old_state0 == *state0p; +#endif +} + +static ETHR_INLINE void +amc_read(ethr_amc_t *amc, int dw, ethr_sint_t *avar, ethr_sint_t *val) +{ + ETHR_AMC_SINT_T__ state0; + int i; + +#if ETHR_AMC_MAX_TRY_READ__ == 0 + state0 = ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[0]); +#else + for (i = 0; i < ETHR_AMC_MAX_TRY_READ__; i++) { + if (amc_try_read(amc, dw, avar, val, &state0)) + return; /* read success */ + ETHR_SPIN_BODY; + } +#endif + + state0 = amc_set_excl(amc, state0); + + val[0] = avar[0]; + if (dw) + val[1] = avar[1]; + + amc_unset_excl(amc, state0); +} + +static ETHR_INLINE int +amc_cmpxchg(ethr_amc_t *amc, int dw, ethr_sint_t *avar, + ethr_sint_t *new, ethr_sint_t *xchg) +{ + ethr_sint_t val[2]; + ETHR_AMC_SINT_T__ state0; + + if (amc_try_read(amc, dw, avar, val, &state0)) { + if (val[0] != xchg[0] || (dw && val[1] != xchg[1])) { + xchg[0] = val[0]; + if (dw) + xchg[1] = val[1]; + return 0; /* failed */ + } + /* Operation will succeed if not interrupted */ + } + + state0 = amc_set_excl(amc, state0); + + if (xchg[0] != avar[0] || (dw && xchg[1] != avar[1])) { + xchg[0] = avar[0]; + if (dw) + xchg[1] = avar[1]; + + ETHR_DBG_CHK_EXCL_STATE(amc, state0); + + amc_unset_excl(amc, state0); + return 0; /* failed */ + } + + avar[0] = new[0]; + if (dw) + avar[1] = new[1]; + + amc_inc_mc_unset_excl(amc, state0); + return 1; +} + + +#define ETHR_AMC_MODIFICATION_OPS__(AMC, OPS) \\ +do { \\ + ETHR_AMC_SINT_T__ state0__; \\ + state0__ = ETHR_AMC_ATMC_FUNC__(read)(&(AMC)->atomic[0]); \\ + state0__ = amc_set_excl((AMC), state0__); \\ + { OPS; } \\ + amc_inc_mc_unset_excl((AMC), state0__); \\ +} while (0) + +#endif /* amc fallback */ +"]. + diff --git a/erts/lib_src/win/ethr_event.c b/erts/lib_src/win/ethr_event.c index 68f093f49c..bc2f635c26 100644 --- a/erts/lib_src/win/ethr_event.c +++ b/erts/lib_src/win/ethr_event.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2009-2010. All Rights Reserved. + * Copyright Ericsson AB 2009-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -28,13 +28,10 @@ /* --- Windows implementation of thread events ------------------------------ */ -#pragma intrinsic(_InterlockedExchangeAdd) -#pragma intrinsic(_InterlockedCompareExchange) - int ethr_event_init(ethr_event *e) { - e->state = ETHR_EVENT_OFF__; + ethr_atomic32_init(&e->state, ETHR_EVENT_OFF__); e->handle = CreateEvent(NULL, FALSE, FALSE, NULL); if (e->handle == INVALID_HANDLE_VALUE) return ethr_win_get_errno__(); @@ -63,7 +60,6 @@ ethr_event_reset(ethr_event *e) static ETHR_INLINE int wait(ethr_event *e, int spincount) { - LONG state; DWORD code; int sc, res, until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS; @@ -73,13 +69,9 @@ wait(ethr_event *e, int spincount) sc = spincount; while (1) { - long on; + ethr_sint32_t state; while (1) { -#if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ - state = e->state; -#else - state = _InterlockedExchangeAdd(&e->state, (LONG) 0); -#endif + state = ethr_atomic32_read(&e->state); if (state == ETHR_EVENT_ON__) return 0; if (sc == 0) @@ -95,9 +87,9 @@ wait(ethr_event *e, int spincount) } if (state != ETHR_EVENT_OFF_WAITER__) { - state = _InterlockedCompareExchange(&e->state, - ETHR_EVENT_OFF_WAITER__, - ETHR_EVENT_OFF__); + state = ethr_atomic32_cmpxchg(&e->state, + ETHR_EVENT_OFF_WAITER__, + ETHR_EVENT_OFF__); if (state == ETHR_EVENT_ON__) return 0; ETHR_ASSERT(state == ETHR_EVENT_OFF__); diff --git a/erts/lib_src/win/ethread.c b/erts/lib_src/win/ethread.c index 789a360b11..3abda6de4c 100644 --- a/erts/lib_src/win/ethread.c +++ b/erts/lib_src/win/ethread.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2010. All Rights Reserved. + * Copyright Ericsson AB 2010-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -35,6 +35,7 @@ #include <winerror.h> #include <stdio.h> #include <limits.h> +#include <intrin.h> #define ETHR_INLINE_FUNC_NAME_(X) X ## __ #define ETHREAD_IMPL__ @@ -158,6 +159,25 @@ ethr_abort__(void) #endif } +#if defined(ETHR_X86_RUNTIME_CONF__) + +#pragma intrinsic(__cpuid) + +void +ethr_x86_cpuid__(int *eax, int *ebx, int *ecx, int *edx) +{ + int CPUInfo[4]; + + __cpuid(CPUInfo, *eax); + + *eax = CPUInfo[0]; + *ebx = CPUInfo[1]; + *ecx = CPUInfo[2]; + *edx = CPUInfo[3]; +} + +#endif /* ETHR_X86_RUNTIME_CONF__ */ + /* * ---------------------------------------------------------------------------- * Exported functions diff --git a/erts/preloaded/ebin/erl_prim_loader.beam b/erts/preloaded/ebin/erl_prim_loader.beam Binary files differindex a28c24ab0c..ffee1f489f 100644 --- a/erts/preloaded/ebin/erl_prim_loader.beam +++ b/erts/preloaded/ebin/erl_prim_loader.beam diff --git a/erts/preloaded/ebin/erlang.beam b/erts/preloaded/ebin/erlang.beam Binary files differindex 9202b5be4f..dda24d4405 100644 --- a/erts/preloaded/ebin/erlang.beam +++ b/erts/preloaded/ebin/erlang.beam diff --git a/erts/preloaded/ebin/init.beam b/erts/preloaded/ebin/init.beam Binary files differindex faa2cf573c..37fd8bb832 100644 --- a/erts/preloaded/ebin/init.beam +++ b/erts/preloaded/ebin/init.beam diff --git a/erts/preloaded/ebin/otp_ring0.beam b/erts/preloaded/ebin/otp_ring0.beam Binary files differindex 62ebbb9ffe..e255cc803f 100644 --- a/erts/preloaded/ebin/otp_ring0.beam +++ b/erts/preloaded/ebin/otp_ring0.beam diff --git a/erts/preloaded/ebin/prim_file.beam b/erts/preloaded/ebin/prim_file.beam Binary files differindex 00a1cf065a..6227b562a1 100644 --- a/erts/preloaded/ebin/prim_file.beam +++ b/erts/preloaded/ebin/prim_file.beam diff --git a/erts/preloaded/ebin/prim_inet.beam b/erts/preloaded/ebin/prim_inet.beam Binary files differindex 9894050cba..22c51c79c0 100644 --- a/erts/preloaded/ebin/prim_inet.beam +++ b/erts/preloaded/ebin/prim_inet.beam diff --git a/erts/preloaded/ebin/prim_zip.beam b/erts/preloaded/ebin/prim_zip.beam Binary files differindex 9ee70d59ec..7e1a5d1fdb 100644 --- a/erts/preloaded/ebin/prim_zip.beam +++ b/erts/preloaded/ebin/prim_zip.beam diff --git a/erts/preloaded/ebin/zlib.beam b/erts/preloaded/ebin/zlib.beam Binary files differindex cda53f7692..ebf9f8e6d6 100644 --- a/erts/preloaded/ebin/zlib.beam +++ b/erts/preloaded/ebin/zlib.beam diff --git a/erts/preloaded/src/erlang.erl b/erts/preloaded/src/erlang.erl index 5deb69edab..e9a59a7aaf 100644 --- a/erts/preloaded/src/erlang.erl +++ b/erts/preloaded/src/erlang.erl @@ -35,14 +35,15 @@ -export([delay_trap/2]). -export([set_cookie/2, get_cookie/0]). -export([nodes/0]). --export([concat_binary/1]). + -export([list_to_integer/2,integer_to_list/2]). -export([flush_monitor_message/2]). -export([set_cpu_topology/1, format_cpu_topology/1]). -export([await_proc_exit/3]). +-export([memory/0, memory/1]). +-export([alloc_info/1, alloc_sizes/1]). -deprecated([hash/2]). --deprecated([concat_binary/1]). % Get rid of autoimports of spawn to avoid clashes with ourselves. -compile({no_auto_import,[spawn/1]}). @@ -535,11 +536,6 @@ set_cookie(Node, C) when Node =/= nonode@nohost, is_atom(Node) -> get_cookie() -> auth:get_cookie(). --spec concat_binary(ListOfBinaries) -> binary() when - ListOfBinaries :: iolist(). -concat_binary(List) -> - list_to_binary(List). - -spec integer_to_list(Integer, Base) -> string() when Integer :: integer(), Base :: 2..36. @@ -806,3 +802,405 @@ min(A, _) -> A. Maximum :: term(). max(A, B) when A < B -> B; max(A, _) -> A. + + +%% +%% erlang:memory/[0,1] +%% +%% NOTE! When updating these functions, make sure to also update +%% erts_memory() in $ERL_TOP/erts/emulator/beam/erl_alloc.c +%% + +-type memory_type() :: 'total' | 'processes' | 'processes_used' | 'system' | 'atom' | 'atom_used' | 'binary' | 'code' | 'ets' | 'low' | 'maximum'. + +-define(CARRIER_ALLOCS, [mseg_alloc, sbmbc_alloc, sbmbc_low_alloc]). +-define(LOW_ALLOCS, [sbmbc_low_alloc, ll_low_alloc, std_low_alloc]). +-define(ALL_NEEDED_ALLOCS, (erlang:system_info(alloc_util_allocators) + -- ?CARRIER_ALLOCS)). + +-record(memory, {total = 0, + processes = 0, + processes_used = 0, + system = 0, + atom = 0, + atom_used = 0, + binary = 0, + code = 0, + ets = 0, + low = 0, + maximum = 0}). + +-spec memory() -> [{memory_type(), non_neg_integer()}]. +memory() -> + case aa_mem_data(au_mem_data(?ALL_NEEDED_ALLOCS)) of + notsup -> + erlang:error(notsup); + Mem -> + InstrTail = case Mem#memory.maximum of + 0 -> []; + _ -> [{maximum, Mem#memory.maximum}] + end, + Tail = case Mem#memory.low of + 0 -> InstrTail; + _ -> [{low, Mem#memory.low} | InstrTail] + end, + [{total, Mem#memory.total}, + {processes, Mem#memory.processes}, + {processes_used, Mem#memory.processes_used}, + {system, Mem#memory.system}, + {atom, Mem#memory.atom}, + {atom_used, Mem#memory.atom_used}, + {binary, Mem#memory.binary}, + {code, Mem#memory.code}, + {ets, Mem#memory.ets} | Tail] + end. + +-spec memory(memory_type()|[memory_type()]) -> non_neg_integer() | [{memory_type(), non_neg_integer()}]. +memory(Type) when is_atom(Type) -> + {AA, ALCU, ChkSup, BadArgZero} = need_mem_info(Type), + case get_mem_data(ChkSup, ALCU, AA) of + notsup -> + erlang:error(notsup, [Type]); + Mem -> + Value = get_memval(Type, Mem), + case {BadArgZero, Value} of + {true, 0} -> erlang:error(badarg, [Type]); + _ -> Value + end + end; +memory(Types) when is_list(Types) -> + {AA, ALCU, ChkSup, BadArgZeroList} = need_mem_info_list(Types), + case get_mem_data(ChkSup, ALCU, AA) of + notsup -> + erlang:error(notsup, [Types]); + Mem -> + case memory_result_list(Types, BadArgZeroList, Mem) of + badarg -> erlang:error(badarg, [Types]); + Result -> Result + end + end. + +memory_result_list([], [], _Mem) -> + []; +memory_result_list([T|Ts], [BAZ|BAZs], Mem) -> + case memory_result_list(Ts, BAZs, Mem) of + badarg -> badarg; + TVs -> + V = get_memval(T, Mem), + case {BAZ, V} of + {true, 0} -> badarg; + _ -> [{T, V}| TVs] + end + end. + +get_mem_data(true, AlcUAllocs, NeedAllocatedAreas) -> + case memory_is_supported() of + false -> notsup; + true -> get_mem_data(false, AlcUAllocs, NeedAllocatedAreas) + end; +get_mem_data(false, AlcUAllocs, NeedAllocatedAreas) -> + AlcUMem = case AlcUAllocs of + [] -> #memory{}; + _ -> + au_mem_data(AlcUAllocs) + end, + case NeedAllocatedAreas of + true -> aa_mem_data(AlcUMem); + false -> AlcUMem + end. + +need_mem_info_list([]) -> + {false, [], false, []}; +need_mem_info_list([T|Ts]) -> + {MAA, MALCU, MChkSup, MBadArgZero} = need_mem_info_list(Ts), + {AA, ALCU, ChkSup, BadArgZero} = need_mem_info(T), + {case AA of + true -> true; + _ -> MAA + end, + ALCU ++ (MALCU -- ALCU), + case ChkSup of + true -> true; + _ -> MChkSup + end, + [BadArgZero|MBadArgZero]}. + +need_mem_info(Type) when Type == total; + Type == system -> + {true, ?ALL_NEEDED_ALLOCS, false, false}; +need_mem_info(Type) when Type == processes; + Type == processes_used -> + {true, [eheap_alloc, fix_alloc], true, false}; +need_mem_info(Type) when Type == atom; + Type == atom_used; + Type == code -> + {true, [], true, false}; +need_mem_info(binary) -> + {false, [binary_alloc], true, false}; +need_mem_info(ets) -> + {true, [ets_alloc], true, false}; +need_mem_info(low) -> + LowAllocs = ?LOW_ALLOCS -- ?CARRIER_ALLOCS, + {_, _, FeatureList, _} = erlang:system_info(allocator), + AlcUAllocs = case LowAllocs -- FeatureList of + [] -> LowAllocs; + _ -> [] + end, + {false, AlcUAllocs, true, true}; +need_mem_info(maximum) -> + {true, [], true, true}; +need_mem_info(_) -> + {false, [], false, true}. + +get_memval(total, #memory{total = V}) -> V; +get_memval(processes, #memory{processes = V}) -> V; +get_memval(processes_used, #memory{processes_used = V}) -> V; +get_memval(system, #memory{system = V}) -> V; +get_memval(atom, #memory{atom = V}) -> V; +get_memval(atom_used, #memory{atom_used = V}) -> V; +get_memval(binary, #memory{binary = V}) -> V; +get_memval(code, #memory{code = V}) -> V; +get_memval(ets, #memory{ets = V}) -> V; +get_memval(low, #memory{low = V}) -> V; +get_memval(maximum, #memory{maximum = V}) -> V; +get_memval(_, #memory{}) -> 0. + +memory_is_supported() -> + {_, _, FeatureList, _} = erlang:system_info(allocator), + case ((erlang:system_info(alloc_util_allocators) + -- ?CARRIER_ALLOCS) + -- FeatureList) of + [] -> true; + _ -> false + end. + +get_blocks_size([{blocks_size, Sz, _, _} | Rest], Acc) -> + get_blocks_size(Rest, Acc+Sz); +get_blocks_size([{_, _, _, _} | Rest], Acc) -> + get_blocks_size(Rest, Acc); +get_blocks_size([], Acc) -> + Acc. + +blocks_size([{Carriers, SizeList} | Rest], Acc) when Carriers == mbcs; + Carriers == sbcs; + Carriers == sbmbcs -> + blocks_size(Rest, get_blocks_size(SizeList, Acc)); +blocks_size([_ | Rest], Acc) -> + blocks_size(Rest, Acc); +blocks_size([], Acc) -> + Acc. + +get_fix_proc([{ProcType, A1, U1}| Rest], {A0, U0}) when ProcType == proc; + ProcType == monitor_sh; + ProcType == nlink_sh; + ProcType == msg_ref -> + get_fix_proc(Rest, {A0+A1, U0+U1}); +get_fix_proc([_|Rest], Acc) -> + get_fix_proc(Rest, Acc); +get_fix_proc([], Acc) -> + Acc. + +fix_proc([{fix_types, SizeList} | _Rest], Acc) -> + get_fix_proc(SizeList, Acc); +fix_proc([_ | Rest], Acc) -> + fix_proc(Rest, Acc); +fix_proc([], Acc) -> + Acc. + +is_low_alloc(_A, []) -> + false; +is_low_alloc(A, [A|_As]) -> + true; +is_low_alloc(A, [_A|As]) -> + is_low_alloc(A, As). + +is_low_alloc(A) -> + is_low_alloc(A, ?LOW_ALLOCS). + +au_mem_data(notsup, _) -> + notsup; +au_mem_data(_, [{_, false} | _]) -> + notsup; +au_mem_data(#memory{total = Tot, + processes = Proc, + processes_used = ProcU} = Mem, + [{eheap_alloc, _, Data} | Rest]) -> + Sz = blocks_size(Data, 0), + au_mem_data(Mem#memory{total = Tot+Sz, + processes = Proc+Sz, + processes_used = ProcU+Sz}, + Rest); +au_mem_data(#memory{total = Tot, + system = Sys, + ets = Ets} = Mem, + [{ets_alloc, _, Data} | Rest]) -> + Sz = blocks_size(Data, 0), + au_mem_data(Mem#memory{total = Tot+Sz, + system = Sys+Sz, + ets = Ets+Sz}, + Rest); +au_mem_data(#memory{total = Tot, + system = Sys, + binary = Bin} = Mem, + [{binary_alloc, _, Data} | Rest]) -> + Sz = blocks_size(Data, 0), + au_mem_data(Mem#memory{total = Tot+Sz, + system = Sys+Sz, + binary = Bin+Sz}, + Rest); +au_mem_data(#memory{total = Tot, + processes = Proc, + processes_used = ProcU, + system = Sys} = Mem, + [{fix_alloc, _, Data} | Rest]) -> + {A, U} = fix_proc(Data, {0, 0}), + Sz = blocks_size(Data, 0), + au_mem_data(Mem#memory{total = Tot+Sz, + processes = Proc+A, + processes_used = ProcU+U, + system = Sys+Sz-A}, + Rest); +au_mem_data(#memory{total = Tot, + system = Sys, + low = Low} = Mem, + [{A, _, Data} | Rest]) -> + Sz = blocks_size(Data, 0), + au_mem_data(Mem#memory{total = Tot+Sz, + system = Sys+Sz, + low = case is_low_alloc(A) of + true -> Low+Sz; + false -> Low + end}, + Rest); +au_mem_data(EMD, []) -> + EMD. + +au_mem_data(Allocs) -> + Ref = make_ref(), + erlang:system_info({allocator_sizes, Ref, Allocs}), + receive_emd(Ref). + +receive_emd(_Ref, EMD, 0) -> + EMD; +receive_emd(Ref, EMD, N) -> + receive + {Ref, _, Data} -> + receive_emd(Ref, au_mem_data(EMD, Data), N-1) + end. + +receive_emd(Ref) -> + receive_emd(Ref, #memory{}, erlang:system_info(schedulers)). + +aa_mem_data(notsup, _) -> + notsup; +aa_mem_data(#memory{} = Mem, + [{maximum, Max} | Rest]) -> + aa_mem_data(Mem#memory{maximum = Max}, + Rest); +aa_mem_data(#memory{} = Mem, + [{total, Tot} | Rest]) -> + aa_mem_data(Mem#memory{total = Tot, + system = 0}, % system will be adjusted later + Rest); +aa_mem_data(#memory{atom = Atom, + atom_used = AtomU} = Mem, + [{atom_space, Alloced, Used} | Rest]) -> + aa_mem_data(Mem#memory{atom = Atom+Alloced, + atom_used = AtomU+Used}, + Rest); +aa_mem_data(#memory{atom = Atom, + atom_used = AtomU} = Mem, + [{atom_table, Sz} | Rest]) -> + aa_mem_data(Mem#memory{atom = Atom+Sz, + atom_used = AtomU+Sz}, + Rest); +aa_mem_data(#memory{ets = Ets} = Mem, + [{ets_misc, Sz} | Rest]) -> + aa_mem_data(Mem#memory{ets = Ets+Sz}, + Rest); +aa_mem_data(#memory{processes = Proc, + processes_used = ProcU, + system = Sys} = Mem, + [{ProcData, Sz} | Rest]) when ProcData == bif_timer; + ProcData == link_lh; + ProcData == process_table -> + aa_mem_data(Mem#memory{processes = Proc+Sz, + processes_used = ProcU+Sz, + system = Sys-Sz}, + Rest); +aa_mem_data(#memory{code = Code} = Mem, + [{CodeData, Sz} | Rest]) when CodeData == module_table; + CodeData == export_table; + CodeData == export_list; + CodeData == fun_table; + CodeData == module_refs; + CodeData == loaded_code -> + aa_mem_data(Mem#memory{code = Code+Sz}, + Rest); +aa_mem_data(EMD, [{_, _} | Rest]) -> + aa_mem_data(EMD, Rest); +aa_mem_data(#memory{total = Tot, + processes = Proc, + system = Sys} = Mem, + []) when Sys =< 0 -> + %% Instrumented runtime system -> Sys = Tot - Proc + Mem#memory{system = Tot - Proc}; +aa_mem_data(EMD, []) -> + EMD. + +aa_mem_data(notsup) -> + notsup; +aa_mem_data(EMD) -> + aa_mem_data(EMD, erlang:system_info(allocated_areas)). + +%% +%% alloc_info/1 and alloc_sizes/1 are for internal use only (used by +%% erlang:system_info({allocator|allocator_sizes, _})). +%% + +alloc_info(Allocs) -> + get_alloc_info(allocator, Allocs). + +alloc_sizes(Allocs) -> + get_alloc_info(allocator_sizes, Allocs). + +get_alloc_info(Type, AAtom) when is_atom(AAtom) -> + [{AAtom, Result}] = get_alloc_info(Type, [AAtom]), + Result; +get_alloc_info(Type, AList) when is_list(AList) -> + Ref = make_ref(), + erlang:system_info({Type, Ref, AList}), + receive_allocator(Ref, + erlang:system_info(schedulers), + mk_res_list(AList)). + +mk_res_list([]) -> + []; +mk_res_list([Alloc | Rest]) -> + [{Alloc, []} | mk_res_list(Rest)]. + +insert_instance(I, N, []) -> + [{instance, N, I}]; +insert_instance(I, N, [{instance, M, _}|_] = Rest) when N < M -> + [{instance, N, I} | Rest]; +insert_instance(I, N, [Prev|Rest]) -> + [Prev | insert_instance(I, N, Rest)]. + +insert_info([], Ys) -> + Ys; +insert_info([{A, false}|Xs], [{A, _IList}|Ys]) -> + insert_info(Xs, [{A, false}|Ys]); +insert_info([{A, N, I}|Xs], [{A, IList}|Ys]) -> + insert_info(Xs, [{A, insert_instance(I, N, IList)}|Ys]); +insert_info([{A1, _}|_] = Xs, [{A2, _} = Y | Ys]) when A1 /= A2 -> + [Y | insert_info(Xs, Ys)]; +insert_info([{A1, _, _}|_] = Xs, [{A2, _} = Y | Ys]) when A1 /= A2 -> + [Y | insert_info(Xs, Ys)]. + +receive_allocator(_Ref, 0, Acc) -> + Acc; +receive_allocator(Ref, N, Acc) -> + receive + {Ref, _, InfoList} -> + receive_allocator(Ref, N-1, insert_info(InfoList, Acc)) + end. diff --git a/erts/preloaded/src/prim_file.erl b/erts/preloaded/src/prim_file.erl index fd2d4a1530..30b7a5246a 100644 --- a/erts/preloaded/src/prim_file.erl +++ b/erts/preloaded/src/prim_file.erl @@ -111,6 +111,7 @@ -define(FILE_RESP_EOF, 8). -define(FILE_RESP_FNAME, 9). -define(FILE_RESP_ALL_DATA, 10). +-define(FILE_RESP_LFNAME, 11). %% Open modes for the driver's open function. -define(EFILE_MODE_READ, 1). @@ -914,6 +915,8 @@ drv_get_response(Port, R) when is_list(R) -> {ok, R}; {ok, Name} -> drv_get_response(Port, [Name|R]); + {append, Names} -> + drv_get_response(Port, append(Names, R)); Error -> Error end; @@ -938,6 +941,8 @@ drv_get_response(Port) -> %%%----------------------------------------------------------------- %%% Utility functions. +append([I | Is], R) when is_list(R) -> append(Is, [I | R]); +append([], R) -> R. %% Converts a list of mode atoms into a mode word for the driver. @@ -1067,7 +1072,7 @@ translate_response(?FILE_RESP_N2DATA, {ok, {Size, Offset, D}}; translate_response(?FILE_RESP_N2DATA = X, L0) when is_list(L0) -> {Offset, L1} = get_uint64(L0), - {ReadSize, L2} = get_uint64(L1), + {ReadSize, L2} = get_uint64(L1), {Size, L3} = get_uint64(L2), case {ReadSize, L3} of {0, []} -> @@ -1087,6 +1092,12 @@ translate_response(?FILE_RESP_FNAME, Data) when is_binary(Data) -> {ok, prim_file:internal_native2name(Data)}; translate_response(?FILE_RESP_FNAME, Data) -> {ok, Data}; +translate_response(?FILE_RESP_LFNAME, []) -> + ok; +translate_response(?FILE_RESP_LFNAME, Data) when is_binary(Data) -> + {append, transform_lfname(Data)}; +translate_response(?FILE_RESP_LFNAME, Data) -> + {append, transform_lfname(Data)}; translate_response(?FILE_RESP_ALL_DATA, Data) -> {ok, Data}; translate_response(X, Data) -> @@ -1199,6 +1210,14 @@ transform_ldata(0, List, [Size | Sizes], R) -> {Front, Rear} = lists_split(List, Size), transform_ldata(0, Rear, Sizes, [Front | R]). +transform_lfname(<<>>) -> []; +transform_lfname(<<L:16, Name:L/binary, Names/binary>>) -> + [ prim_file:internal_native2name(Name) | transform_lfname(Names)]; +transform_lfname([]) -> []; +transform_lfname([L1,L2|Names]) -> + L = (L1 bsl 8) bor L2, + {Name, Rest} = lists_split(Names, L), + [Name | transform_lfname(Rest)]. lists_split(List, 0) when is_list(List) -> diff --git a/erts/test/ethread_SUITE.erl b/erts/test/ethread_SUITE.erl index 71d8c1c679..4206bebfe7 100644 --- a/erts/test/ethread_SUITE.erl +++ b/erts/test/ethread_SUITE.erl @@ -47,14 +47,29 @@ spinlock/1, rwspinlock/1, rwmutex/1, - atomic/1]). + atomic/1, + dw_atomic_massage/1]). -include_lib("test_server/include/test_server.hrl"). -tests() -> - [create_join_thread, equal_tids, mutex, try_lock_mutex, - cond_wait, broadcast, detached_thread, - max_threads, tsd, spinlock, rwspinlock, rwmutex, atomic]. +tests() -> + [create_join_thread, + equal_tids, + mutex, + try_lock_mutex, + cond_wait, + broadcast, + detached_thread, + max_threads, + tsd, + spinlock, + rwspinlock, + rwmutex, + atomic, + dw_atomic_massage]. + +all(doc) -> []; +all(suite) -> tests(). suite() -> [{ct_hooks,[ts_install_cth]}]. @@ -211,6 +226,13 @@ atomic(suite) -> atomic(Config) -> run_case(Config, "atomic", ""). +dw_atomic_massage(doc) -> + ["Massage double word atomics"]; +dw_atomic_massage(suite) -> + []; +dw_atomic_massage(Config) -> + run_case(Config, "dw_atomic_massage", ""). + %% %% %% Auxiliary functions diff --git a/erts/test/ethread_SUITE_data/ethread_tests.c b/erts/test/ethread_SUITE_data/ethread_tests.c index 0b59ff5aa6..7e7e133d6c 100644 --- a/erts/test/ethread_SUITE_data/ethread_tests.c +++ b/erts/test/ethread_SUITE_data/ethread_tests.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2004-2010. All Rights Reserved. + * Copyright Ericsson AB 2004-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -1310,6 +1310,9 @@ rwmutex_test(void) * Tests atomics. */ +#define AT_AINT32_MAX 0x7fffffff +#define AT_AINT32_MIN 0x80000000 + #define AT_THREADS 4 #define AT_ITER 10000 @@ -1320,12 +1323,428 @@ static ethr_atomic_t at_go; static ethr_atomic_t at_done; static ethr_atomic_t at_data; +#define AT_TEST_INIT(T, A, B) \ +do { \ + ethr_ ## A ## _init ## B(&A, 17); \ + ASSERT(ethr_ ## A ## _read ## B(&A) == 17); \ +} while (0) + +#define AT_TEST_SET(T, A, B) \ +do { \ + ethr_ ## A ## _set ## B(&A, 4711); \ + ASSERT(ethr_ ## A ## _read ## B(&A) == 4711); \ +} while (0) + +#define AT_TEST_XCHG(T, A, B) \ +do { \ + ethr_ ## A ## _set ## B(&A, 4711); \ + ASSERT(ethr_ ## A ## _xchg ## B(&A, 17) == 4711); \ + ASSERT(ethr_ ## A ## _read ## B(&A) == 17); \ +} while (0) + +#define AT_TEST_CMPXCHG(T, A, B) \ +do { \ + ethr_ ## A ## _set ## B(&A, 4711); \ + ASSERT(ethr_ ## A ## _cmpxchg ## B(&A, 17, 33) == 4711); \ + ASSERT(ethr_ ## A ## _read ## B(&A) == 4711); \ + ASSERT(ethr_ ## A ## _cmpxchg ## B(&A, 17, 4711) == 4711); \ + ASSERT(ethr_ ## A ## _read ## B(&A) == 17); \ +} while (0) + +#define AT_TEST_ADD_READ(T, A, B) \ +do { \ + T var_ = AT_AINT32_MAX; \ + var_ += 4711; \ + ethr_ ## A ## _set ## B(&A, AT_AINT32_MAX); \ + ASSERT(ethr_ ## A ## _add_read ## B(&A, 4711) == var_); \ + ASSERT(ethr_ ## A ## _read ## B(&A) == var_); \ + var_ = AT_AINT32_MIN; \ + var_ -= 4711; \ + ethr_ ## A ## _set ## B(&A, AT_AINT32_MIN); \ + ASSERT(ethr_ ## A ## _add_read ## B(&A, -4711) == var_); \ + ASSERT(ethr_ ## A ## _read ## B(&A) == var_); \ + ethr_ ## A ## _set ## B(&A, 4711); \ + ASSERT(ethr_ ## A ## _add_read ## B(&A, 10) == 4721); \ + ASSERT(ethr_ ## A ## _read ## B(&A) == 4721); \ +} while (0) + +#define AT_TEST_ADD(T, A, B) \ +do { \ + T var_ = AT_AINT32_MAX; \ + var_ += 4711; \ + ethr_ ## A ## _set ## B(&A, AT_AINT32_MAX); \ + ethr_ ## A ## _add ## B(&A, 4711); \ + ASSERT(ethr_ ## A ## _read ## B(&A) == var_); \ + var_ = AT_AINT32_MIN; \ + var_ -= 4711; \ + ethr_ ## A ## _set ## B(&A, AT_AINT32_MIN); \ + ethr_ ## A ## _add ## B(&A, -4711); \ + ASSERT(ethr_ ## A ## _read ## B(&A) == var_); \ + ethr_ ## A ## _set ## B(&A, 11); \ + ethr_ ## A ## _add ## B(&A, 4700); \ + ASSERT(ethr_ ## A ## _read ## B(&A) == 4711); \ +} while (0) + +#define AT_TEST_INC_READ(T, A, B) \ +do { \ + T var_ = AT_AINT32_MAX; \ + var_++; \ + ethr_ ## A ## _set ## B(&A, AT_AINT32_MAX); \ + ASSERT(ethr_ ## A ## _inc_read ## B(&A) == var_); \ + ASSERT(ethr_ ## A ## _read ## B(&A) == var_); \ + ethr_ ## A ## _set ## B(&A, 4710); \ + ASSERT(ethr_ ## A ## _inc_read ## B(&A) == 4711); \ + ASSERT(ethr_ ## A ## _read ## B(&A) == 4711); \ +} while (0) + +#define AT_TEST_DEC_READ(T, A, B) \ +do { \ + T var_ = AT_AINT32_MIN; \ + var_--; \ + ethr_ ## A ## _set ## B(&A, AT_AINT32_MIN); \ + ASSERT(ethr_ ## A ## _dec_read ## B(&A) == var_); \ + ASSERT(ethr_ ## A ## _read ## B(&A) == var_); \ + ethr_ ## A ## _set ## B(&A, 17); \ + ASSERT(ethr_ ## A ## _dec_read ## B(&A) == 16); \ + ASSERT(ethr_ ## A ## _read ## B(&A) == 16); \ +} while (0) + + +#define AT_TEST_INC(T, A, B) \ +do { \ + T var_ = AT_AINT32_MAX; \ + var_++; \ + ethr_ ## A ## _set ## B(&A, AT_AINT32_MAX); \ + ethr_ ## A ## _inc ## B(&A); \ + ASSERT(ethr_ ## A ## _read ## B(&A) == var_); \ + ethr_ ## A ## _set ## B(&A, 4710); \ + ethr_ ## A ## _inc ## B(&A); \ + ASSERT(ethr_ ## A ## _read ## B(&A) == 4711); \ +} while (0) + +#define AT_TEST_DEC(T, A, B) \ +do { \ + T var_ = AT_AINT32_MIN; \ + var_--; \ + ethr_ ## A ## _set ## B(&A, AT_AINT32_MIN); \ + ethr_ ## A ## _dec ## B(&A); \ + ASSERT(ethr_ ## A ## _read ## B(&A) == var_); \ + ethr_ ## A ## _set ## B(&A, 17); \ + ethr_ ## A ## _dec ## B(&A); \ + ASSERT(ethr_ ## A ## _read ## B(&A) == 16); \ +} while (0) + +#define AT_TEST_READ_BAND(T, A, B) \ +do { \ + ethr_ ## A ## _set ## B(&A, 0x13131313); \ + ASSERT(ethr_ ## A ## _read_band ## B(&A, 0x31313131) == 0x13131313); \ + ASSERT(ethr_ ## A ## _read ## B(&A) == 0x11111111); \ +} while (0) + +#define AT_TEST_READ_BOR(T, A, B) \ +do { \ + ethr_ ## A ## _set ## B(&A, 0x11111111); \ + ASSERT(ethr_ ## A ## _read_bor ## B(&A, 0x23232323) == 0x11111111); \ + ASSERT(ethr_ ## A ## _read ## B(&A) == 0x33333333); \ +} while (0) + + +static void +atomic_basic_test(void) +{ + /* + * Verify that each op does what it is expected + * to do for at least one input. + */ + ethr_atomic32_t atomic32; + ethr_atomic_t atomic; + + print_line("AT_AINT32_MAX=%d",AT_AINT32_MAX); + print_line("AT_AINT32_MIN=%d",AT_AINT32_MIN); + + AT_TEST_INIT(ethr_sint32_t, atomic32, ); + AT_TEST_SET(ethr_sint32_t, atomic32, ); + AT_TEST_XCHG(ethr_sint32_t, atomic32, ); + AT_TEST_CMPXCHG(ethr_sint32_t, atomic32, ); + AT_TEST_ADD_READ(ethr_sint32_t, atomic32, ); + AT_TEST_ADD(ethr_sint32_t, atomic32, ); + AT_TEST_INC_READ(ethr_sint32_t, atomic32, ); + AT_TEST_DEC_READ(ethr_sint32_t, atomic32, ); + AT_TEST_INC(ethr_sint32_t, atomic32, ); + AT_TEST_DEC(ethr_sint32_t, atomic32, ); + AT_TEST_READ_BAND(ethr_sint32_t, atomic32, ); + AT_TEST_READ_BOR(ethr_sint32_t, atomic32, ); + + AT_TEST_INIT(ethr_sint32_t, atomic32, _acqb); + AT_TEST_SET(ethr_sint32_t, atomic32, _acqb); + AT_TEST_XCHG(ethr_sint32_t, atomic32, _acqb); + AT_TEST_CMPXCHG(ethr_sint32_t, atomic32, _acqb); + AT_TEST_ADD_READ(ethr_sint32_t, atomic32, _acqb); + AT_TEST_ADD(ethr_sint32_t, atomic32, _acqb); + AT_TEST_INC_READ(ethr_sint32_t, atomic32, _acqb); + AT_TEST_DEC_READ(ethr_sint32_t, atomic32, _acqb); + AT_TEST_INC(ethr_sint32_t, atomic32, _acqb); + AT_TEST_DEC(ethr_sint32_t, atomic32, _acqb); + AT_TEST_READ_BAND(ethr_sint32_t, atomic32, _acqb); + AT_TEST_READ_BOR(ethr_sint32_t, atomic32, _acqb); + + AT_TEST_INIT(ethr_sint32_t, atomic32, _relb); + AT_TEST_SET(ethr_sint32_t, atomic32, _relb); + AT_TEST_XCHG(ethr_sint32_t, atomic32, _relb); + AT_TEST_CMPXCHG(ethr_sint32_t, atomic32, _relb); + AT_TEST_ADD_READ(ethr_sint32_t, atomic32, _relb); + AT_TEST_ADD(ethr_sint32_t, atomic32, _relb); + AT_TEST_INC_READ(ethr_sint32_t, atomic32, _relb); + AT_TEST_DEC_READ(ethr_sint32_t, atomic32, _relb); + AT_TEST_INC(ethr_sint32_t, atomic32, _relb); + AT_TEST_DEC(ethr_sint32_t, atomic32, _relb); + AT_TEST_READ_BAND(ethr_sint32_t, atomic32, _relb); + AT_TEST_READ_BOR(ethr_sint32_t, atomic32, _relb); + + AT_TEST_INIT(ethr_sint32_t, atomic32, _rb); + AT_TEST_SET(ethr_sint32_t, atomic32, _rb); + AT_TEST_XCHG(ethr_sint32_t, atomic32, _rb); + AT_TEST_CMPXCHG(ethr_sint32_t, atomic32, _rb); + AT_TEST_ADD_READ(ethr_sint32_t, atomic32, _rb); + AT_TEST_ADD(ethr_sint32_t, atomic32, _rb); + AT_TEST_INC_READ(ethr_sint32_t, atomic32, _rb); + AT_TEST_DEC_READ(ethr_sint32_t, atomic32, _rb); + AT_TEST_INC(ethr_sint32_t, atomic32, _rb); + AT_TEST_DEC(ethr_sint32_t, atomic32, _rb); + AT_TEST_READ_BAND(ethr_sint32_t, atomic32, _rb); + AT_TEST_READ_BOR(ethr_sint32_t, atomic32, _rb); + + AT_TEST_INIT(ethr_sint32_t, atomic32, _wb); + AT_TEST_SET(ethr_sint32_t, atomic32, _wb); + AT_TEST_XCHG(ethr_sint32_t, atomic32, _wb); + AT_TEST_CMPXCHG(ethr_sint32_t, atomic32, _wb); + AT_TEST_ADD_READ(ethr_sint32_t, atomic32, _wb); + AT_TEST_ADD(ethr_sint32_t, atomic32, _wb); + AT_TEST_INC_READ(ethr_sint32_t, atomic32, _wb); + AT_TEST_DEC_READ(ethr_sint32_t, atomic32, _wb); + AT_TEST_INC(ethr_sint32_t, atomic32, _wb); + AT_TEST_DEC(ethr_sint32_t, atomic32, _wb); + AT_TEST_READ_BAND(ethr_sint32_t, atomic32, _wb); + AT_TEST_READ_BOR(ethr_sint32_t, atomic32, _wb); + + AT_TEST_INIT(ethr_sint32_t, atomic32, _mb); + AT_TEST_SET(ethr_sint32_t, atomic32, _mb); + AT_TEST_XCHG(ethr_sint32_t, atomic32, _mb); + AT_TEST_CMPXCHG(ethr_sint32_t, atomic32, _mb); + AT_TEST_ADD_READ(ethr_sint32_t, atomic32, _mb); + AT_TEST_ADD(ethr_sint32_t, atomic32, _mb); + AT_TEST_INC_READ(ethr_sint32_t, atomic32, _mb); + AT_TEST_DEC_READ(ethr_sint32_t, atomic32, _mb); + AT_TEST_INC(ethr_sint32_t, atomic32, _mb); + AT_TEST_DEC(ethr_sint32_t, atomic32, _mb); + AT_TEST_READ_BAND(ethr_sint32_t, atomic32, _mb); + AT_TEST_READ_BOR(ethr_sint32_t, atomic32, _mb); + + AT_TEST_INIT(ethr_sint_t, atomic, ); + AT_TEST_SET(ethr_sint_t, atomic, ); + AT_TEST_XCHG(ethr_sint_t, atomic, ); + AT_TEST_CMPXCHG(ethr_sint_t, atomic, ); + AT_TEST_ADD_READ(ethr_sint_t, atomic, ); + AT_TEST_ADD(ethr_sint_t, atomic, ); + AT_TEST_INC_READ(ethr_sint_t, atomic, ); + AT_TEST_DEC_READ(ethr_sint_t, atomic, ); + AT_TEST_INC(ethr_sint_t, atomic, ); + AT_TEST_DEC(ethr_sint_t, atomic, ); + AT_TEST_READ_BAND(ethr_sint_t, atomic, ); + AT_TEST_READ_BOR(ethr_sint_t, atomic, ); + + AT_TEST_INIT(ethr_sint_t, atomic, _acqb); + AT_TEST_SET(ethr_sint_t, atomic, _acqb); + AT_TEST_XCHG(ethr_sint_t, atomic, _acqb); + AT_TEST_CMPXCHG(ethr_sint_t, atomic, _acqb); + AT_TEST_ADD_READ(ethr_sint_t, atomic, _acqb); + AT_TEST_ADD(ethr_sint_t, atomic, _acqb); + AT_TEST_INC_READ(ethr_sint_t, atomic, _acqb); + AT_TEST_DEC_READ(ethr_sint_t, atomic, _acqb); + AT_TEST_INC(ethr_sint_t, atomic, _acqb); + AT_TEST_DEC(ethr_sint_t, atomic, _acqb); + AT_TEST_READ_BAND(ethr_sint_t, atomic, _acqb); + AT_TEST_READ_BOR(ethr_sint_t, atomic, _acqb); + + AT_TEST_INIT(ethr_sint_t, atomic, _relb); + AT_TEST_SET(ethr_sint_t, atomic, _relb); + AT_TEST_XCHG(ethr_sint_t, atomic, _relb); + AT_TEST_CMPXCHG(ethr_sint_t, atomic, _relb); + AT_TEST_ADD_READ(ethr_sint_t, atomic, _relb); + AT_TEST_ADD(ethr_sint_t, atomic, _relb); + AT_TEST_INC_READ(ethr_sint_t, atomic, _relb); + AT_TEST_DEC_READ(ethr_sint_t, atomic, _relb); + AT_TEST_INC(ethr_sint_t, atomic, _relb); + AT_TEST_DEC(ethr_sint_t, atomic, _relb); + AT_TEST_READ_BAND(ethr_sint_t, atomic, _relb); + AT_TEST_READ_BOR(ethr_sint_t, atomic, _relb); + + AT_TEST_INIT(ethr_sint_t, atomic, _rb); + AT_TEST_SET(ethr_sint_t, atomic, _rb); + AT_TEST_XCHG(ethr_sint_t, atomic, _rb); + AT_TEST_CMPXCHG(ethr_sint_t, atomic, _rb); + AT_TEST_ADD_READ(ethr_sint_t, atomic, _rb); + AT_TEST_ADD(ethr_sint_t, atomic, _rb); + AT_TEST_INC_READ(ethr_sint_t, atomic, _rb); + AT_TEST_DEC_READ(ethr_sint_t, atomic, _rb); + AT_TEST_INC(ethr_sint_t, atomic, _rb); + AT_TEST_DEC(ethr_sint_t, atomic, _rb); + AT_TEST_READ_BAND(ethr_sint_t, atomic, _rb); + AT_TEST_READ_BOR(ethr_sint_t, atomic, _rb); + + AT_TEST_INIT(ethr_sint_t, atomic, _wb); + AT_TEST_SET(ethr_sint_t, atomic, _wb); + AT_TEST_XCHG(ethr_sint_t, atomic, _wb); + AT_TEST_CMPXCHG(ethr_sint_t, atomic, _wb); + AT_TEST_ADD_READ(ethr_sint_t, atomic, _wb); + AT_TEST_ADD(ethr_sint_t, atomic, _wb); + AT_TEST_INC_READ(ethr_sint_t, atomic, _wb); + AT_TEST_DEC_READ(ethr_sint_t, atomic, _wb); + AT_TEST_INC(ethr_sint_t, atomic, _wb); + AT_TEST_DEC(ethr_sint_t, atomic, _wb); + AT_TEST_READ_BAND(ethr_sint_t, atomic, _wb); + AT_TEST_READ_BOR(ethr_sint_t, atomic, _wb); + + AT_TEST_INIT(ethr_sint_t, atomic, _mb); + AT_TEST_SET(ethr_sint_t, atomic, _mb); + AT_TEST_XCHG(ethr_sint_t, atomic, _mb); + AT_TEST_CMPXCHG(ethr_sint_t, atomic, _mb); + AT_TEST_ADD_READ(ethr_sint_t, atomic, _mb); + AT_TEST_ADD(ethr_sint_t, atomic, _mb); + AT_TEST_INC_READ(ethr_sint_t, atomic, _mb); + AT_TEST_DEC_READ(ethr_sint_t, atomic, _mb); + AT_TEST_INC(ethr_sint_t, atomic, _mb); + AT_TEST_DEC(ethr_sint_t, atomic, _mb); + AT_TEST_READ_BAND(ethr_sint_t, atomic, _mb); + AT_TEST_READ_BOR(ethr_sint_t, atomic, _mb); + + /* Double word */ + { + ethr_dw_atomic_t dw_atomic; + ethr_dw_sint_t dw0, dw1; + dw0.sint[0] = 4711; + dw0.sint[1] = 4712; + + /* init */ + ethr_dw_atomic_init(&dw_atomic, &dw0); + ethr_dw_atomic_read(&dw_atomic, &dw1); + ETHR_ASSERT(dw1.sint[0] == 4711); + ETHR_ASSERT(dw1.sint[1] == 4712); + + /* set */ + dw0.sint[0] = 42; + dw0.sint[1] = ~((ethr_sint_t) 0); + ethr_dw_atomic_set(&dw_atomic, &dw0); + ethr_dw_atomic_read(&dw_atomic, &dw1); + ASSERT(dw1.sint[0] == 42); + ASSERT(dw1.sint[1] == ~((ethr_sint_t) 0)); + + /* cmpxchg */ + dw0.sint[0] = 17; + dw0.sint[1] = 18; + dw1.sint[0] = 19; + dw1.sint[1] = 20; + ASSERT(!ethr_dw_atomic_cmpxchg(&dw_atomic, &dw1, &dw0)); + ethr_dw_atomic_read(&dw_atomic, &dw0); + ASSERT(dw0.sint[0] == 42); + ASSERT(dw0.sint[1] == ~((ethr_sint_t) 0)); + + ASSERT(ethr_dw_atomic_cmpxchg(&dw_atomic, &dw1, &dw0)); + + ethr_dw_atomic_read(&dw_atomic, &dw0); + ASSERT(dw0.sint[0] == 19); + ASSERT(dw0.sint[1] == 20); + } +} + + +#define AT_DW_MIN 12 +#define AT_DW_MAX 42 +#define AT_DW_THREADS (AT_DW_MAX - AT_DW_MIN + 1) + +#define AT_DW_LOOPS 200000 +#define AT_DW_R_LOOPS 10 + +ethr_dw_atomic_t at_dw_atomic; + +void +at_dw_valid(ethr_dw_sint_t *dw) +{ + int i; + char c; + char *cp; + + ASSERT(dw->sint[0] == dw->sint[1]); + + cp = (char *) &dw->sint[0]; + c = cp[0]; + + ASSERT(AT_DW_MIN <= c && c <= AT_DW_MAX); + + for (i = 0; i < sizeof(ethr_sint_t); i++) + ASSERT(c == cp[i]); +} + +void * +at_dw_thr(void *vval) +{ + int l, r; + ethr_sint_t val = (ethr_sint_t) vval; + ethr_dw_sint_t dw; + ethr_dw_sint_t my_dw; + + my_dw.sint[0] = val; + my_dw.sint[1] = val; + + ethr_dw_atomic_set(&at_dw_atomic, &my_dw); + for (l = 0; l < AT_DW_LOOPS; l++) { + for (r = 0; r < AT_DW_R_LOOPS; r++) { + ethr_dw_atomic_read(&at_dw_atomic, &dw); + at_dw_valid(&dw); + } + ethr_dw_atomic_set(&at_dw_atomic, &my_dw); + for (r = 0; r < AT_DW_R_LOOPS; r++) { + ethr_dw_atomic_read(&at_dw_atomic, &dw); + at_dw_valid(&dw); + } + dw.sint[0] = 0; + dw.sint[1] = 0; + while (1) { + if (ethr_dw_atomic_cmpxchg(&at_dw_atomic, &my_dw, &dw)) + break; + } + } +} + +static void +dw_atomic_massage_test(void) +{ + int i, res; + ethr_tid tid[AT_DW_THREADS]; + ethr_thr_opts thr_opts = ETHR_THR_OPTS_DEFAULT_INITER; + ethr_dw_sint_t dw; + + dw.sint[0] = dw.sint[1] = 0; + + ethr_dw_atomic_init(&at_dw_atomic, &dw); + + for (i = AT_DW_MIN; i <= AT_DW_MAX; i++) { + ethr_sint_t val; + memset(&val, i, sizeof(ethr_sint_t)); + res = ethr_thr_create(&tid[i-AT_DW_MIN], at_dw_thr, (void *) val, &thr_opts); + ASSERT(res == 0); + } + for (i = AT_DW_MIN; i <= AT_DW_MAX; i++) { + res = ethr_thr_join(tid[i-AT_DW_MIN], NULL); + ASSERT(res == 0); + } +} + void * at_thread(void *unused) { int i; long val, go; - val = ethr_atomic_inc_read(&at_ready); ASSERT(val > 0); @@ -1373,7 +1792,6 @@ at_thread(void *unused) return NULL; } - static void atomic_test(void) { @@ -1382,6 +1800,8 @@ atomic_test(void) ethr_tid tid[AT_THREADS]; ethr_thr_opts thr_opts = ETHR_THR_OPTS_DEFAULT_INITER; + atomic_basic_test(); + #if ETHR_SIZEOF_PTR > 4 at_rm_val = ((long) 1) << 57; at_set_val = ((long) 1) << 60; @@ -1493,6 +1913,8 @@ main(int argc, char *argv[]) rwmutex_test(); else if (strcmp(testcase, "atomic") == 0) atomic_test(); + else if (strcmp(testcase, "dw_atomic_massage") == 0) + dw_atomic_massage_test(); else skip("Test case \"%s\" not implemented yet", testcase); diff --git a/erts/test/nt_SUITE.erl b/erts/test/nt_SUITE.erl index 7d6da28ad6..f9bd15a0ce 100644 --- a/erts/test/nt_SUITE.erl +++ b/erts/test/nt_SUITE.erl @@ -490,12 +490,12 @@ middleman(Waitfor) -> match_event(_X, []) -> nomatch; match_event({Time,Cat,Fac,Sev,Mes},[{Pid,Ref,{Cat,Fac,Sev,MesRE}} | Tail]) -> - case regexp:match(Mes,MesRE) of - {match,_,_} -> + case re:run(Mes,MesRE,[{capture,none}]) of + match -> %%io:format("Match!~n"), {ok,{Pid,Ref,Time,Mes},Tail}; - _Z -> - %%io:format("No match (~p)~n",[_Z]), + nomatch -> + %%io:format("No match~n"), case match_event({Time,Cat,Fac,Sev,Mes},Tail) of {ok,X,Rest} -> {ok,X,[{Pid,Ref,{Cat,Fac,Sev,MesRE}} | Rest]}; diff --git a/erts/test/z_SUITE.erl b/erts/test/z_SUITE.erl index 8fceab32a6..482ecb8fba 100644 --- a/erts/test/z_SUITE.erl +++ b/erts/test/z_SUITE.erl @@ -166,9 +166,12 @@ core_search_conf(RunByTS, DBTop, XDir) -> file_inspect(#core_search_conf{file = File}, Core) -> FRes0 = os:cmd(File ++ " " ++ Core), - FRes = case regexp:match(FRes0, Core) of - {match, S, E} -> + FRes = case string:str(FRes0, Core) of + 0 -> + FRes0; + S -> L = length(FRes0), + E = length(Core), case S of 1 -> lists:sublist(FRes0, E+1, L+1); @@ -178,19 +181,13 @@ file_inspect(#core_search_conf{file = File}, Core) -> " " ++ lists:sublist(FRes0, E+1, L+1) - end; - _ -> FRes0 + end end, - case regexp:match(FRes, "[Tt][Ee][Xx][Tt]") of + case re:run(FRes, "text|ascii", [caseless,{capture,none}]) of + match -> + not_a_core; nomatch -> - case regexp:match(FRes, "[Aa][Ss][Cc][Ii][Ii]") of - nomatch -> - probably_a_core; - _ -> - not_a_core - end; - _ -> - not_a_core + probably_a_core end. mk_readable(F) -> diff --git a/erts/vsn.mk b/erts/vsn.mk index 18799d2fba..d56d03146d 100644 --- a/erts/vsn.mk +++ b/erts/vsn.mk @@ -17,8 +17,8 @@ # %CopyrightEnd% # -VSN = 5.8.5 -SYSTEM_VSN = R14B04 +VSN = 5.9 +SYSTEM_VSN = R15A # Port number 4365 in 4.2 # Port number 4366 in 4.3 |
