diff options
216 files changed, 25569 insertions, 7135 deletions
diff --git a/INSTALL.md b/INSTALL.md index 1061c5187a..a19e35b01f 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -273,12 +273,6 @@ Some of the available `configure` options are: * `--with-ssl=PATH` - Specify location of OpenSSL include and lib * `--{with,without}-ssl` - OpenSSL (without implies that the `crypto`, `ssh`, and `ssl` won't be built) -* `--enable-ethread-pre-pentium4-compatibility` - Enable compatibility with - x86 processors before pentium 4 (back to 486) in the ethread library. If - not passed the ethread library (part of the runtime system) will use - instructions that first appeared on the pentium 4 processor when building - for x86. This option will be automatically enabled if required on the - build machine. * `--with-libatomic_ops=PATH` - Use the `libatomic_ops` library for atomic memory accesses. If `configure` should inform you about no native atomic implementation available, you typically want to try using the @@ -726,7 +720,7 @@ Copyright and License %CopyrightBegin% -Copyright Ericsson AB 1998-2010. All Rights Reserved. +Copyright Ericsson AB 1998-2011. All Rights Reserved. The contents of this file are subject to the Erlang Public License, Version 1.1, (the "License"); you may not use this file except in diff --git a/configure.in b/configure.in index 36b33ec399..c8d4561a44 100644 --- a/configure.in +++ b/configure.in @@ -2,7 +2,7 @@ dnl Process this file with autoconf to produce a configure script. dnl %CopyrightBegin% dnl -dnl Copyright Ericsson AB 1998-2010. All Rights Reserved. +dnl Copyright Ericsson AB 1998-2011. All Rights Reserved. dnl dnl The contents of this file are subject to the Erlang Public License, dnl Version 1.1, (the "License"); you may not use this file except in @@ -306,10 +306,6 @@ AS_HELP_STRING([--enable-m32-build], esac ],enable_m32_build=no) -AC_ARG_ENABLE(ethread-pre-pentium4-compatibility, - AS_HELP_STRING([--enable-ethread-pre-pentium4-compatibility], - [enable compatibility with x86 processors before pentium 4 (back to 486) in the ethread library])) - AC_ARG_WITH(libatomic_ops, AS_HELP_STRING([--with-libatomic_ops=PATH], [specify and prefer usage of libatomic_ops in the ethread library])) diff --git a/erts/aclocal.m4 b/erts/aclocal.m4 index b64380e817..3ebea66d30 100644 --- a/erts/aclocal.m4 +++ b/erts/aclocal.m4 @@ -679,6 +679,55 @@ AC_SUBST(ERTS_INTERNAL_X_LIBS) ]) +AC_DEFUN(ETHR_CHK_SYNC_OP, +[ + AC_MSG_CHECKING([for $3-bit $1()]) + case "$2" in + "1") sync_call="$1(&var);";; + "2") sync_call="$1(&var, ($4) 0);";; + "3") sync_call="$1(&var, ($4) 0, ($4) 0);";; + esac + have_sync_op=no + AC_TRY_LINK([], + [ + $4 res; + volatile $4 var; + res = $sync_call + ], + [have_sync_op=yes]) + test $have_sync_op = yes && $5 + AC_MSG_RESULT([$have_sync_op]) +]) + +AC_DEFUN(ETHR_CHK_INTERLOCKED, +[ + ilckd="$1" + AC_MSG_CHECKING([for ${ilckd}()]) + case "$2" in + "1") ilckd_call="${ilckd}(var);";; + "2") ilckd_call="${ilckd}(var, ($3) 0);";; + "3") ilckd_call="${ilckd}(var, ($3) 0, ($3) 0);";; + "4") ilckd_call="${ilckd}(var, ($3) 0, ($3) 0, arr);";; + esac + have_interlocked_op=no + AC_TRY_LINK( + [ + #define WIN32_LEAN_AND_MEAN + #include <windows.h> + #include <intrin.h> + ], + [ + volatile $3 *var; + volatile $3 arr[2]; + + $ilckd_call + return 0; + ], + [have_interlocked_op=yes]) + test $have_interlocked_op = yes && $4 + AC_MSG_RESULT([$have_interlocked_op]) +]) + dnl ---------------------------------------------------------------------- dnl dnl ERL_FIND_ETHR_LIB @@ -750,121 +799,41 @@ case "$THR_LIB_NAME" in AC_DEFINE(ETHR_WIN32_THREADS, 1, [Define if you have win32 threads]) - have_ilckd=no - AC_MSG_CHECKING([for _InterlockedCompareExchange64()]) - AC_TRY_LINK([ - #define WIN32_LEAN_AND_MEAN - #include <windows.h> - ], - [ - volatile __int64 *var; - _InterlockedCompareExchange64(var, (__int64) 1, (__int64) 0); - return 0; - ], - have_ilckd=yes) - AC_MSG_RESULT([$have_ilckd]) - test $have_ilckd = yes && AC_DEFINE(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64, 1, [Define if you have _InterlockedCompareExchange64()]) - - AC_CHECK_SIZEOF(void *) - case "$ac_cv_sizeof_void_p-$have_ilckd" in - 8-no) - ethr_have_native_atomics=no - ethr_have_native_spinlock=no;; - *) - ethr_have_native_atomics=yes - ethr_have_native_spinlock=yes;; - esac - - have_ilckd=no - AC_MSG_CHECKING([for _InterlockedDecrement64()]) - AC_TRY_LINK([ - #define WIN32_LEAN_AND_MEAN - #include <windows.h> - ], - [ - volatile __int64 *var; - _InterlockedDecrement64(var); - return 0; - ], - have_ilckd=yes) - AC_MSG_RESULT([$have_ilckd]) - test $have_ilckd = yes && AC_DEFINE(ETHR_HAVE__INTERLOCKEDDECREMENT64, 1, [Define if you have _InterlockedDecrement64()]) - - have_ilckd=no - AC_MSG_CHECKING([for _InterlockedIncrement64()]) - AC_TRY_LINK([ - #define WIN32_LEAN_AND_MEAN - #include <windows.h> - ], - [ - volatile __int64 *var; - _InterlockedIncrement64(var); - return 0; - ], - have_ilckd=yes) - AC_MSG_RESULT([$have_ilckd]) - test $have_ilckd = yes && AC_DEFINE(ETHR_HAVE__INTERLOCKEDINCREMENT64, 1, [Define if you have _InterlockedIncrement64()]) - - have_ilckd=no - AC_MSG_CHECKING([for _InterlockedExchangeAdd64()]) - AC_TRY_LINK([ - #define WIN32_LEAN_AND_MEAN - #include <windows.h> - ], - [ - volatile __int64 *var; - _InterlockedExchangeAdd64(var, (__int64) 1); - return 0; - ], - have_ilckd=yes) - AC_MSG_RESULT([$have_ilckd]) - test $have_ilckd = yes && AC_DEFINE(ETHR_HAVE__INTERLOCKEDEXCHANGEADD64, 1, [Define if you have _InterlockedExchangeAdd64()]) - - have_ilckd=no - AC_MSG_CHECKING([for _InterlockedExchange64()]) - AC_TRY_LINK([ - #define WIN32_LEAN_AND_MEAN - #include <windows.h> - ], - [ - volatile __int64 *var; - _InterlockedExchange64(var, (__int64) 1); - return 0; - ], - have_ilckd=yes) - AC_MSG_RESULT([$have_ilckd]) - test $have_ilckd = yes && AC_DEFINE(ETHR_HAVE__INTERLOCKEDEXCHANGE64, 1, [Define if you have _InterlockedExchange64()]) - - have_ilckd=no - AC_MSG_CHECKING([for _InterlockedAnd64()]) - AC_TRY_LINK([ - #define WIN32_LEAN_AND_MEAN - #include <windows.h> - ], - [ - volatile __int64 *var; - _InterlockedAnd64(var, (__int64) 1); - return 0; - ], - have_ilckd=yes) - AC_MSG_RESULT([$have_ilckd]) - test $have_ilckd = yes && AC_DEFINE(ETHR_HAVE__INTERLOCKEDAND64, 1, [Define if you have _InterlockedAnd64()]) - - have_ilckd=no - AC_MSG_CHECKING([for _InterlockedOr64()]) - AC_TRY_LINK([ - #define WIN32_LEAN_AND_MEAN - #include <windows.h> - ], - [ - volatile __int64 *var; - _InterlockedOr64(var, (__int64) 1); - return 0; - ], - have_ilckd=yes) - AC_MSG_RESULT([$have_ilckd]) - test $have_ilckd = yes && AC_DEFINE(ETHR_HAVE__INTERLOCKEDOR64, 1, [Define if you have _InterlockedOr64()]) - + ETHR_CHK_INTERLOCKED([_InterlockedDecrement], [1], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDDECREMENT, 1, [Define if you have _InterlockedDecrement()])) + ETHR_CHK_INTERLOCKED([_InterlockedDecrement_rel], [1], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDDECREMENT_REL, 1, [Define if you have _InterlockedDecrement_rel()])) + ETHR_CHK_INTERLOCKED([_InterlockedIncrement], [1], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDINCREMENT, 1, [Define if you have _InterlockedIncrement()])) + ETHR_CHK_INTERLOCKED([_InterlockedIncrement_acq], [1], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDINCREMENT_ACQ, 1, [Define if you have _InterlockedIncrement_acq()])) + ETHR_CHK_INTERLOCKED([_InterlockedExchangeAdd], [2], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDEXCHANGEADD, 1, [Define if you have _InterlockedExchangeAdd()])) + ETHR_CHK_INTERLOCKED([_InterlockedExchangeAdd_acq], [2], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDEXCHANGEADD_ACQ, 1, [Define if you have _InterlockedExchangeAdd_acq()])) + ETHR_CHK_INTERLOCKED([_InterlockedAnd], [2], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDAND, 1, [Define if you have _InterlockedAnd()])) + ETHR_CHK_INTERLOCKED([_InterlockedOr], [2], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDOR, 1, [Define if you have _InterlockedOr()])) + ETHR_CHK_INTERLOCKED([_InterlockedExchange], [2], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDEXCHANGE, 1, [Define if you have _InterlockedExchange()])) + ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange], [3], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE, 1, [Define if you have _InterlockedCompareExchange()])) + test "$have_interlocked_op" = "yes" && ethr_have_native_atomics=yes + ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange_acq], [3], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE_ACQ, 1, [Define if you have _InterlockedCompareExchange_acq()])) + test "$have_interlocked_op" = "yes" && ethr_have_native_atomics=yes + ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange_rel], [3], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE_REL, 1, [Define if you have _InterlockedCompareExchange_rel()])) + test "$have_interlocked_op" = "yes" && ethr_have_native_atomics=yes + + ETHR_CHK_INTERLOCKED([_InterlockedDecrement64], [1], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDDECREMENT64, 1, [Define if you have _InterlockedDecrement64()])) + ETHR_CHK_INTERLOCKED([_InterlockedDecrement64_rel], [1], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDDECREMENT64_REL, 1, [Define if you have _InterlockedDecrement64_rel()])) + ETHR_CHK_INTERLOCKED([_InterlockedIncrement64], [1], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDINCREMENT64, 1, [Define if you have _InterlockedIncrement64()])) + ETHR_CHK_INTERLOCKED([_InterlockedIncrement64_acq], [1], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDINCREMENT64_ACQ, 1, [Define if you have _InterlockedIncrement64_acq()])) + ETHR_CHK_INTERLOCKED([_InterlockedExchangeAdd64], [2], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDEXCHANGEADD64, 1, [Define if you have _InterlockedExchangeAdd64()])) + ETHR_CHK_INTERLOCKED([_InterlockedExchangeAdd64_acq], [2], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDEXCHANGEADD64_ACQ, 1, [Define if you have _InterlockedExchangeAdd64_acq()])) + ETHR_CHK_INTERLOCKED([_InterlockedAnd64], [2], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDAND64, 1, [Define if you have _InterlockedAnd64()])) + ETHR_CHK_INTERLOCKED([_InterlockedOr64], [2], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDOR64, 1, [Define if you have _InterlockedOr64()])) + ETHR_CHK_INTERLOCKED([_InterlockedExchange64], [2], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDEXCHANGE64, 1, [Define if you have _InterlockedExchange64()])) + ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange64], [3], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64, 1, [Define if you have _InterlockedCompareExchange64()])) + test "$have_interlocked_op" = "yes" && ethr_have_native_atomics=yes + ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange64_acq], [3], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_ACQ, 1, [Define if you have _InterlockedCompareExchange64_acq()])) + test "$have_interlocked_op" = "yes" && ethr_have_native_atomics=yes + ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange64_rel], [3], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_REL, 1, [Define if you have _InterlockedCompareExchange64_rel()])) + test "$have_interlocked_op" = "yes" && ethr_have_native_atomics=yes + + ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange128], [4], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE128, 1, [Define if you have _InterlockedCompareExchange128()])) + + test "$ethr_have_native_atomics" = "yes" && ethr_have_native_spinlock=yes ;; pthread) @@ -1100,35 +1069,51 @@ case "$THR_LIB_NAME" in AC_MSG_RESULT([$linux_futex]) test $linux_futex = yes && AC_DEFINE(ETHR_HAVE_LINUX_FUTEX, 1, [Define if you have a linux futex implementation.]) - AC_MSG_CHECKING([for GCC atomic operations]) - ethr_have_gcc_atomic_ops=no - AC_TRY_LINK([], - [ - long res; - volatile long val; - res = __sync_val_compare_and_swap(&val, (long) 1, (long) 0); - res = __sync_add_and_fetch(&val, (long) 1); - res = __sync_sub_and_fetch(&val, (long) 1); - res = __sync_fetch_and_and(&val, (long) 1); - res = __sync_fetch_and_or(&val, (long) 1); - ], - [ethr_have_native_atomics=yes - ethr_have_gcc_atomic_ops=yes]) - AC_MSG_RESULT([$ethr_have_gcc_atomic_ops]) - test $ethr_have_gcc_atomic_ops = yes && AC_DEFINE(ETHR_HAVE_GCC_ATOMIC_OPS, 1, [Define if you have gcc atomic operations]) + AC_CHECK_SIZEOF(int) + AC_CHECK_SIZEOF(long) + AC_CHECK_SIZEOF(long long) + AC_CHECK_SIZEOF(__int128_t) + + if test "$ac_cv_sizeof_int" = "4"; then + int32="int" + elif test "$ac_cv_sizeof_long" = "4"; then + int32="long" + elif test "$ac_cv_sizeof_long_long" = "4"; then + int32="long long" + else + AC_MSG_ERROR([No 32-bit type found]) + fi - case "$host_cpu" in - sun4u | sparc64 | sun4v) - ethr_have_native_atomics=yes;; - i86pc | i*86 | x86_64 | amd64) - ethr_have_native_atomics=yes;; - macppc | ppc | "Power Macintosh") - ethr_have_native_atomics=yes;; - tile) - ethr_have_native_atomics=yes;; - *) - ;; - esac + if test "$ac_cv_sizeof_int" = "8"; then + int64="int" + elif test "$ac_cv_sizeof_long" = "8"; then + int64="long" + elif test "$ac_cv_sizeof_long_long" = "8"; then + int64="long long" + else + AC_MSG_ERROR([No 64-bit type found]) + fi + + int128=no + if test "$ac_cv_sizeof___int128_t" = "16"; then + int128="__int128_t" + fi + + ETHR_CHK_SYNC_OP([__sync_val_compare_and_swap], [3], [32], [$int32], AC_DEFINE(ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP32, 1, [Define if you have __sync_val_compare_and_swap() for 32-bit integers])) + test "$have_sync_op" = "yes" && ethr_have_native_atomics=yes + ETHR_CHK_SYNC_OP([__sync_add_and_fetch], [2], [32], [$int32], AC_DEFINE(ETHR_HAVE___SYNC_ADD_AND_FETCH32, 1, [Define if you have __sync_add_and_fetch() for 32-bit integers])) + ETHR_CHK_SYNC_OP([__sync_fetch_and_and], [2], [32], [$int32], AC_DEFINE(ETHR_HAVE___SYNC_FETCH_AND_AND32, 1, [Define if you have __sync_fetch_and_and() for 32-bit integers])) + ETHR_CHK_SYNC_OP([__sync_fetch_and_or], [2], [32], [$int32], AC_DEFINE(ETHR_HAVE___SYNC_FETCH_AND_OR32, 1, [Define if you have __sync_fetch_and_or() for 32-bit integers])) + + ETHR_CHK_SYNC_OP([__sync_val_compare_and_swap], [3], [64], [$int64], AC_DEFINE(ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP64, 1, [Define if you have __sync_val_compare_and_swap() for 64-bit integers])) + test "$have_sync_op" = "yes" && ethr_have_native_atomics=yes + ETHR_CHK_SYNC_OP([__sync_add_and_fetch], [2], [64], [$int64], AC_DEFINE(ETHR_HAVE___SYNC_ADD_AND_FETCH64, 1, [Define if you have __sync_add_and_fetch() for 64-bit integers])) + ETHR_CHK_SYNC_OP([__sync_fetch_and_and], [2], [64], [$int64], AC_DEFINE(ETHR_HAVE___SYNC_FETCH_AND_AND64, 1, [Define if you have __sync_fetch_and_and() for 64-bit integers])) + ETHR_CHK_SYNC_OP([__sync_fetch_and_or], [2], [64], [$int64], AC_DEFINE(ETHR_HAVE___SYNC_FETCH_AND_OR64, 1, [Define if you have __sync_fetch_and_or() for 64-bit integers])) + + if test $int128 != no; then + ETHR_CHK_SYNC_OP([__sync_val_compare_and_swap], [3], [128], [$int128], AC_DEFINE(ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP128, 1, [Define if you have __sync_val_compare_and_swap() for 128-bit integers])) + fi AC_MSG_CHECKING([for a usable libatomic_ops implementation]) case "x$with_libatomic_ops" in @@ -1175,6 +1160,34 @@ case "$THR_LIB_NAME" in AC_MSG_ERROR([No usable libatomic_ops implementation found]) fi + case "$host_cpu" in + sparc | sun4u | sparc64 | sun4v) + case "$with_sparc_memory_order" in + "TSO") + AC_DEFINE(ETHR_SPARC_TSO, 1, [Define if only run in Sparc TSO mode]);; + "PSO") + AC_DEFINE(ETHR_SPARC_PSO, 1, [Define if only run in Sparc PSO, or TSO mode]);; + "RMO"|"") + AC_DEFINE(ETHR_SPARC_RMO, 1, [Define if run in Sparc RMO, PSO, or TSO mode]);; + *) + AC_MSG_ERROR([Unsupported Sparc memory order: $with_sparc_memory_order]);; + esac + ethr_have_native_atomics=yes;; + i86pc | i*86 | x86_64 | amd64) + if test "$enable_x86_out_of_order" = "yes"; then + AC_DEFINE(ETHR_X86_OUT_OF_ORDER, 1, [Define if x86/x86_64 out of order instructions should be synchronized]) + fi + ethr_have_native_atomics=yes;; + macppc | ppc | "Power Macintosh") + ethr_have_native_atomics=yes;; + tile) + ethr_have_native_atomics=yes;; + *) + ;; + esac + + test ethr_have_native_atomics = "yes" && ethr_have_native_spinlock=yes + dnl Restore LIBS LIBS=$saved_libs dnl restore CPPFLAGS @@ -1210,6 +1223,8 @@ AC_CHECK_SIZEOF(long long) AC_DEFINE_UNQUOTED(ETHR_SIZEOF_LONG_LONG, $ac_cv_sizeof_long_long, [Define to the size of long long]) AC_CHECK_SIZEOF(__int64) AC_DEFINE_UNQUOTED(ETHR_SIZEOF___INT64, $ac_cv_sizeof___int64, [Define to the size of __int64]) +AC_CHECK_SIZEOF(__int128_t) +AC_DEFINE_UNQUOTED(ETHR_SIZEOF___INT128_T, $ac_cv_sizeof___int128_t, [Define to the size of __int128_t]) case X$erl_xcomp_bigendian in @@ -1232,6 +1247,10 @@ AC_ARG_ENABLE(native-ethr-impls, *) disable_native_ethr_impls=no ;; esac ], disable_native_ethr_impls=no) +AC_ARG_ENABLE(x86-out-of-order, + AS_HELP_STRING([--enable-x86-out-of-order], + [enable x86/x84_64 out of order support (default disabled)])) + test "X$disable_native_ethr_impls" = "Xyes" && AC_DEFINE(ETHR_DISABLE_NATIVE_IMPLS, 1, [Define if you want to disable native ethread implementations]) @@ -1250,55 +1269,101 @@ AC_ARG_WITH(libatomic_ops, AS_HELP_STRING([--with-libatomic_ops=PATH], [specify and prefer usage of libatomic_ops in the ethread library])) -AC_ARG_ENABLE(ethread-pre-pentium4-compatibility, - AS_HELP_STRING([--enable-ethread-pre-pentium4-compatibility], - [enable compatibility with x86 processors before pentium 4 (back to 486) in the ethread library]), -[ - case "$enable_ethread_pre_pentium4_compatibility" in - yes|no) ;; - *) enable_ethread_pre_pentium4_compatibility=check;; - esac -], -[enable_ethread_pre_pentium4_compatibility=check]) - -test "$cross_compiling" != "yes" || enable_ethread_pre_pentium4_compatibility=no - -case "$enable_ethread_pre_pentium4_compatibility-$host_cpu" in - check-i86pc | check-i*86) - AC_MSG_CHECKING([whether pre pentium 4 compatibility should forced]) - AC_RUN_IFELSE([ -#if defined(__GNUC__) -# if defined(ETHR_PREFER_LIBATOMIC_OPS_NATIVE_IMPLS) -# define CHECK_LIBATOMIC_OPS__ -# else -# define CHECK_GCC_ASM__ -# endif -#elif defined(ETHR_HAVE_LIBATOMIC_OPS) -# define CHECK_LIBATOMIC_OPS__ +AC_ARG_WITH(with_sparc_memory_order, + AS_HELP_STRING([--with-sparc-memory-order=TSO|PSO|RMO], + [specify sparc memory order (defaults to RMO)])) + +ETHR_X86_SSE2_ASM=no +case "$GCC-$ac_cv_sizeof_void_p-$host_cpu" in + yes-4-i86pc | yes-4-i*86 | yes-4-x86_64 | yes-4-amd64) + AC_MSG_CHECKING([for gcc sse2 asm support]) + save_CFLAGS="$CFLAGS" + CFLAGS="$CFLAGS -msse2" + gcc_sse2_asm=no + AC_TRY_COMPILE([], + [ + long long x, *y; + __asm__ __volatile__("movq %1, %0\n\t" : "=x"(x) : "m"(*y) : "memory"); + ], + [gcc_sse2_asm=yes]) + CFLAGS="$save_CFLAGS" + AC_MSG_RESULT([$gcc_sse2_asm]) + if test "$gcc_sse2_asm" = "yes"; then + AC_DEFINE(ETHR_GCC_HAVE_SSE2_ASM_SUPPORT, 1, [Define if you use a gcc that supports -msse2 and understand sse2 specific asm statements]) + ETHR_X86_SSE2_ASM=yes + fi + ;; + *) + ;; +esac + +case "$GCC-$host_cpu" in + yes-i86pc | yes-i*86 | yes-x86_64 | yes-amd64) + gcc_dw_cmpxchg_asm=no + AC_MSG_CHECKING([for gcc double word cmpxchg asm support]) + AC_TRY_COMPILE([], + [ + char xchgd; + long new[2], xchg[2], *p; + __asm__ __volatile__( +#if ETHR_SIZEOF_PTR == 4 && defined(__PIC__) && __PIC__ + "pushl %%ebx\n\t" + "movl %8, %%ebx\n\t" #endif -#if defined(CHECK_LIBATOMIC_OPS__) -#include "atomic_ops.h" +#if ETHR_SIZEOF_PTR == 4 + "lock; cmpxchg8b %0\n\t" +#else + "lock; cmpxchg16b %0\n\t" #endif -int main(void) -{ -#if defined(CHECK_GCC_ASM__) - __asm__ __volatile__("mfence" : : : "memory"); -#elif defined(CHECK_LIBATOMIC_OPS__) - AO_nop_full(); + "setz %3\n\t" +#if ETHR_SIZEOF_PTR == 4 && defined(__PIC__) && __PIC__ + "popl %%ebx\n\t" #endif - return 0; -} + : "=m"(*p), "=d"(xchg[1]), "=a"(xchg[0]), "=c"(xchgd) + : "m"(*p), "1"(xchg[1]), "2"(xchg[0]), "3"(new[1]), +#if ETHR_SIZEOF_PTR == 4 && defined(__PIC__) && __PIC__ + "r"(new[0]) +#else + "b"(new[0]) +#endif + : "cc", "memory"); + ], - [enable_ethread_pre_pentium4_compatibility=no], - [enable_ethread_pre_pentium4_compatibility=yes], - [enable_ethread_pre_pentium4_compatibility=no]) - AC_MSG_RESULT([$enable_ethread_pre_pentium4_compatibility]);; + [gcc_dw_cmpxchg_asm=yes]) + if test $gcc_dw_cmpxchg_asm = no && test $ac_cv_sizeof_void_p = 4; then + AC_TRY_COMPILE([], + [ + char xchgd; + long new[2], xchg[2], *p; +#if !defined(__PIC__) || !__PIC__ +# error nope +#endif + __asm__ __volatile__( + "pushl %%ebx\n\t" + "movl (%7), %%ebx\n\t" + "movl 4(%7), %%ecx\n\t" + "lock; cmpxchg8b %0\n\t" + "setz %3\n\t" + "popl %%ebx\n\t" + : "=m"(*p), "=d"(xchg[1]), "=a"(xchg[0]), "=c"(xchgd) + : "m"(*p), "1"(xchg[1]), "2"(xchg[0]), "3"(new) + : "cc", "memory"); + + ], + [gcc_dw_cmpxchg_asm=yes]) + if test "$gcc_dw_cmpxchg_asm" = "yes"; then + AC_DEFINE(ETHR_CMPXCHG8B_REGISTER_SHORTAGE, 1, [Define if you get a register shortage with cmpxchg8b and position independent code]) + fi + fi + AC_MSG_RESULT([$gcc_dw_cmpxchg_asm]) + if test "$gcc_dw_cmpxchg_asm" = "yes"; then + AC_DEFINE(ETHR_GCC_HAVE_DW_CMPXCHG_ASM_SUPPORT, 1, [Define if you use a gcc that supports the double word cmpxchg instruction]) + fi;; *) ;; esac -test $enable_ethread_pre_pentium4_compatibility = yes && - AC_DEFINE(ETHR_PRE_PENTIUM4_COMPAT, 1, [Define if you want compatibilty with x86 processors before pentium4.]) + AC_DEFINE(ETHR_HAVE_ETHREAD_DEFINES, 1, \ [Define if you have all ethread defines]) @@ -1309,6 +1374,7 @@ AC_SUBST(ETHR_LIB_NAME) AC_SUBST(ETHR_DEFS) AC_SUBST(ETHR_THR_LIB_BASE) AC_SUBST(ETHR_THR_LIB_BASE_DIR) +AC_SUBST(ETHR_X86_SSE2_ASM) ]) diff --git a/erts/doc/src/erl_nif.xml b/erts/doc/src/erl_nif.xml index cdce4ec0b8..382e446dce 100644 --- a/erts/doc/src/erl_nif.xml +++ b/erts/doc/src/erl_nif.xml @@ -822,6 +822,13 @@ typedef enum { <desc><p>Create an ordinary list containing the elements of array <c>arr</c> of length <c>cnt</c>. An empty list is returned if <c>cnt</c> is 0.</p></desc> </func> + <func><name><ret>int</ret><nametext>enif_make_reverse_list(ErlNifEnv* env, ERL_NIF_TERM term, ERL_NIF_TERM *list)</nametext></name> + <fsummary>Create the reverse list of the list <c>term</c>.</fsummary> + <desc><p>Set <c>*list</c> to the reverse list of the list <c>term</c> and return true, + or return false if <c>term</c> is not a list. This function should only be used on + short lists as a copy will be created of the list which will not be released until after the + nif returns.</p></desc> + </func> <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_long(ErlNifEnv* env, long int i)</nametext></name> <fsummary>Create an integer term from a long int</fsummary> <desc><p>Create an integer term from a <c>long int</c>.</p></desc> diff --git a/erts/doc/src/erlang.xml b/erts/doc/src/erlang.xml index 7cfab0785d..b7d775541f 100644 --- a/erts/doc/src/erlang.xml +++ b/erts/doc/src/erlang.xml @@ -538,15 +538,6 @@ false</pre> </desc> </func> <func> - <name name="concat_binary" arity="1"/> - <fsummary>Concatenate a list of binaries (deprecated)</fsummary> - <desc> - <p>Do not use; use - <seealso marker="#list_to_binary/1">list_to_binary/1</seealso> - instead.</p> - </desc> - </func> - <func> <name>erlang:crc32(Data) -> integer() >= 0</name> <fsummary>Compute crc32 (IEEE 802.3) checksum</fsummary> <type> diff --git a/erts/doc/src/erts_alloc.xml b/erts/doc/src/erts_alloc.xml index 51a4a2bca0..90347824d5 100644 --- a/erts/doc/src/erts_alloc.xml +++ b/erts/doc/src/erts_alloc.xml @@ -78,14 +78,20 @@ segments are allocated, cached segments are used if possible instead of creating new segments. This in order to reduce the number of system calls made.</item> + <tag><c>sbmbc_alloc</c></tag> + <item>Allocator used by other allocators for allocation of carriers + where only small blocks are placed. Currently this allocator is + disabled by default.</item> </taglist> <p><c>sys_alloc</c> and <c>fix_alloc</c> are always enabled and cannot be disabled. <c>mseg_alloc</c> is always enabled if it is available and an allocator that uses it is enabled. All other allocators can be <seealso marker="#M_e">enabled or disabled</seealso>. By default all allocators are enabled. - When an allocator is disabled, <c>sys_alloc</c> - is used instead of the disabled allocator.</p> + When an allocator is disabled, <c>sys_alloc</c> is used instead of + the disabled allocator. <c>sbmbc_alloc</c> is an exception. If + <c>sbmbc_alloc</c> is disabled, other allocators will not handle + small blocks in separate carriers.</p> <p>The main idea with the <c>erts_alloc</c> library is to separate memory blocks that are used differently into different memory areas, and by this achieving less memory fragmentation. By @@ -103,15 +109,20 @@ following does <em>not</em> apply to them.</p> <p>An allocator manages multiple areas, called carriers, in which memory blocks are placed. A carrier is either placed in a - separate memory segment (allocated via <c>mseg_alloc</c>) or in - the heap segment (allocated via <c>sys_alloc</c>). Multiblock + separate memory segment (allocated via <c>mseg_alloc</c>), in + the heap segment (allocated via <c>sys_alloc</c>), or inside + another carrier (in case it is a carrier created by + <c>sbmbc_alloc</c>). Multiblock carriers are used for storage of several blocks. Singleblock carriers are used for storage of one block. Blocks that are larger than the value of the singleblock carrier threshold (<seealso marker="#M_sbct">sbct</seealso>) parameter are placed - in singleblock carriers. Blocks smaller than the value of the - <c>sbct</c> parameter are placed in multiblock - carriers. Normally an allocator creates a "main multiblock + in singleblock carriers. Blocks that are smaller than the value + of the <c>sbct</c> parameter are placed in multiblock + carriers. Blocks that are smaller than the small block multiblock + carrier threshold (<seealso marker="#M_sbmbct">sbmbct</seealso>) + will be placed in multiblock carriers only used for small blocks. + Normally an allocator creates a "main multiblock carrier". Main multiblock carriers are never deallocated. The size of the main multiblock carrier is determined by the value of the <seealso marker="#M_mmbcs">mmbcs</seealso> parameter.</p> @@ -133,8 +144,11 @@ <c>sbct</c> parameter should be larger than the value of the <c>lmbcs</c> parameter, the allocator may have to create multiblock carriers that are larger than the value of the - <c>lmbcs</c> parameter, though. Singleblock carriers allocated - via <c>mseg_alloc</c> are sized to whole pages.</p> + <c>lmbcs</c> parameter, though. The size of multiblock carriers + for small blocks is determined by the small block multiblock + carrier size (<seealso marker="#M_sbmbcs">sbmbcs</seealso>). + Singleblock carriers allocated via <c>mseg_alloc</c> are sized + to whole pages.</p> <p>Sizes of carriers allocated via <c>sys_alloc</c> are decided based on the value of the <c>sys_alloc</c> carrier size (<seealso marker="#Muycs">ycs</seealso>) parameter. The size of @@ -166,6 +180,14 @@ used. The time complexity is proportional to log N, where N is the number of free blocks.</p> </item> + <tag>Address order first fit</tag> + <item> + <p>Strategy: Find the block with the lowest address that satisfies the + requested block size.</p> + <p>Implementation: A balanced binary search tree is + used. The time complexity is proportional to log N, where + N is the number of free blocks.</p> + </item> <tag>Good fit</tag> <item> <p>Strategy: Try to find the best fit, but settle for the best fit @@ -194,6 +216,11 @@ </taglist> </section> + <note><p> + Currently only allocators using the best fit and the address order + best fit strategies are able to use "small block multi block carriers". + </p></note> + <section> <marker id="flags"></marker> <title>System Flags Effecting erts_alloc</title> @@ -215,6 +242,7 @@ the currently present allocators:</p> <list type="bulleted"> <item><c>B: binary_alloc</c></item> + <item><c>C: sbmbc_alloc</c></item> <item><c>D: std_alloc</c></item> <item><c>E: ets_alloc</c></item> <item><c>F: fix_alloc</c></item> @@ -300,11 +328,11 @@ subsystem identifier, only the specific allocator identified will be effected:</p> <taglist> - <tag><marker id="M_as"><c><![CDATA[+M<S>as bf|aobf|gf|af]]></c></marker></tag> + <tag><marker id="M_as"><c><![CDATA[+M<S>as bf|aobf|aoff|gf|af]]></c></marker></tag> <item> Allocation strategy. Valid strategies are <c>bf</c> (best fit), - <c>aobf</c> (address order best fit), <c>gf</c> (good fit), - and <c>af</c> (a fit). See + <c>aobf</c> (address order best fit), <c>aoff</c> (address order first fit), + <c>gf</c> (good fit), and <c>af</c> (a fit). See <seealso marker="#strategy">the description of allocation strategies</seealso> in "the <c>alloc_util</c> framework" section.</item> <tag><marker id="M_asbcst"><c><![CDATA[+M<S>asbcst <size>]]></c></marker></tag> <item> @@ -395,6 +423,20 @@ threshold will be placed in singleblock carriers. Blocks smaller than this threshold will be placed in multiblock carriers.</item> + <tag><marker id="M_sbmbcs"><c><![CDATA[+M<S>sbmbcs <size>]]></c></marker></tag> + <item> + Small block multiblock carrier size (in bytes). Memory blocks smaller + than the small block multiblock carrier threshold + (<seealso marker="#M_sbmbct">sbmbct</seealso>) will be placed in + multiblock carriers used for small blocks only. This parameter + determines the size of such carriers. + </item> + <tag><marker id="M_sbmbct"><c><![CDATA[+M<S>sbmbct <size>]]></c></marker></tag> + <item> + Small block multiblock carrier threshold (in bytes). Memory blocks + smaller than this threshold will be placed in multiblock carriers + used for small blocks only. + </item> <tag><marker id="M_smbcs"><c><![CDATA[+M<S>smbcs <size>]]></c></marker></tag> <item> Smallest (<c>mseg_alloc</c>) multiblock carrier size (in diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in index d9362a2a8f..b658e79378 100644 --- a/erts/emulator/Makefile.in +++ b/erts/emulator/Makefile.in @@ -742,7 +742,7 @@ RUN_OBJS = \ $(OBJDIR)/erl_bif_re.o $(OBJDIR)/erl_unicode.o \ $(OBJDIR)/packet_parser.o $(OBJDIR)/safe_hash.o \ $(OBJDIR)/erl_zlib.o $(OBJDIR)/erl_nif.o \ - $(OBJDIR)/erl_bif_binary.o + $(OBJDIR)/erl_bif_binary.o $(OBJDIR)/erl_ao_firstfit_alloc.o ifeq ($(TARGET),win32) DRV_OBJS = \ diff --git a/erts/emulator/beam/atom.c b/erts/emulator/beam/atom.c index b97705ed96..d7c7f117cf 100644 --- a/erts/emulator/beam/atom.c +++ b/erts/emulator/beam/atom.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * Copyright Ericsson AB 1996-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -75,7 +75,7 @@ void atom_info(int to, void *to_arg) index_info(to, to_arg, &erts_atom_table); #ifdef ERTS_ATOM_PUT_OPS_STAT erts_print(to, to_arg, "atom_put_ops: %ld\n", - erts_smp_atomic_read(&atom_put_ops)); + erts_smp_atomic_read_nob(&atom_put_ops)); #endif if (lock) @@ -213,7 +213,7 @@ am_atom_put(const char* name, int len) len = MAX_ATOM_LENGTH; } #ifdef ERTS_ATOM_PUT_OPS_STAT - erts_smp_atomic_inc(&atom_put_ops); + erts_smp_atomic_inc_nob(&atom_put_ops); #endif a.len = len; a.name = (byte*)name; @@ -309,7 +309,7 @@ init_atom_table(void) rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED; #ifdef ERTS_ATOM_PUT_OPS_STAT - erts_smp_atomic_init(&atom_put_ops, 0); + erts_smp_atomic_init_nob(&atom_put_ops, 0); #endif erts_smp_rwmtx_init_opt(&atom_table_lock, &rwmtx_opt, "atom_tab"); diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c index 31910888d1..773baad01f 100644 --- a/erts/emulator/beam/beam_bp.c +++ b/erts/emulator/beam/beam_bp.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2000-2010. All Rights Reserved. + * Copyright Ericsson AB 2000-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -408,7 +408,7 @@ erts_is_count_break(BeamInstr *pc, Sint *count_ret) { if (bdc) { if (count_ret) { - *count_ret = (Sint) erts_smp_atomic_read(&bdc->acount); + *count_ret = (Sint) erts_smp_atomic_read_nob(&bdc->acount); } return !0; } @@ -958,17 +958,17 @@ static int set_function_break(Module *modp, BeamInstr *pc, int bif, if (break_op == (BeamInstr) BeamOp(op_i_count_breakpoint)) { if (count_op == erts_break_stop) { - count = erts_smp_atomic_read(&bdc->acount); + count = erts_smp_atomic_read_nob(&bdc->acount); if (count >= 0) { while(1) { - res = erts_smp_atomic_cmpxchg(&bdc->acount, -count - 1, count); + res = erts_smp_atomic_cmpxchg_nob(&bdc->acount, -count - 1, count); if ((res == count) || count < 0) break; count = res; } } } else { /* Reset call counter */ - erts_smp_atomic_set(&bdc->acount, 0); + erts_smp_atomic_set_nob(&bdc->acount, 0); } } else if (break_op == (BeamInstr) BeamOp(op_i_time_breakpoint)) { @@ -1097,7 +1097,7 @@ static int set_function_break(Module *modp, BeamInstr *pc, int bif, } } else if (break_op == (BeamInstr) BeamOp(op_i_count_breakpoint)) { BpDataCount *bdc = (BpDataCount *) bd; - erts_smp_atomic_init(&bdc->acount, 0); + erts_smp_atomic_init_nob(&bdc->acount, 0); } if (bif == BREAK_IS_ERL) { diff --git a/erts/emulator/beam/beam_bp.h b/erts/emulator/beam/beam_bp.h index bd8a7249a7..2ec5818688 100644 --- a/erts/emulator/beam/beam_bp.h +++ b/erts/emulator/beam/beam_bp.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2000-2010. All Rights Reserved. + * Copyright Ericsson AB 2000-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -165,8 +165,8 @@ do { \ bdc = (BpDataCount *) bdc->next; \ ASSERT(bdc); \ bds[ix] = (BpData *) bdc; \ - count = erts_smp_atomic_read(&bdc->acount); \ - if (count >= 0) erts_smp_atomic_inc(&bdc->acount); \ + count = erts_smp_atomic_read_nob(&bdc->acount); \ + if (count >= 0) erts_smp_atomic_inc_nob(&bdc->acount); \ *(instr_result) = bdc->orig_instr; \ } while (0) diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 68b3350d7f..98dde066fc 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -811,7 +811,7 @@ BIF_RETTYPE spawn_opt_1(BIF_ALIST_1) so.min_heap_size = H_MIN_SIZE; so.min_vheap_size = BIN_VH_MIN_SIZE; so.priority = PRIORITY_NORMAL; - so.max_gen_gcs = (Uint16) erts_smp_atomic32_read(&erts_max_gen_gcs); + so.max_gen_gcs = (Uint16) erts_smp_atomic32_read_nob(&erts_max_gen_gcs); so.scheduler = 0; /* @@ -3417,10 +3417,10 @@ BIF_RETTYPE ports_0(BIF_ALIST_0) erts_smp_mtx_lock(&ports_snapshot_mtx); /* One snapshot at a time */ - erts_smp_atomic_set(&erts_dead_ports_ptr, - (erts_aint_t) (port_buf + erts_max_ports)); + erts_smp_atomic_set_nob(&erts_dead_ports_ptr, + (erts_aint_t) (port_buf + erts_max_ports)); - next_ss = erts_smp_atomic32_inctest(&erts_ports_snapshot); + next_ss = erts_smp_atomic32_inc_read_relb(&erts_ports_snapshot); for (i = erts_max_ports-1; i >= 0; i--) { Port* prt = &erts_port[i]; @@ -3434,8 +3434,8 @@ BIF_RETTYPE ports_0(BIF_ALIST_0) erts_smp_port_state_unlock(prt); } - dead_ports = (Eterm*)erts_smp_atomic_xchg(&erts_dead_ports_ptr, - (erts_aint_t) NULL); + dead_ports = (Eterm*)erts_smp_atomic_xchg_nob(&erts_dead_ports_ptr, + (erts_aint_t) NULL); erts_smp_mtx_unlock(&ports_snapshot_mtx); ASSERT(pp <= dead_ports); @@ -3942,8 +3942,8 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) goto error; } nval = (n > (Sint) ((Uint16) -1)) ? ((Uint16) -1) : ((Uint16) n); - oval = (Uint) erts_smp_atomic32_xchg(&erts_max_gen_gcs, - (erts_aint32_t) nval); + oval = (Uint) erts_smp_atomic32_xchg_nob(&erts_max_gen_gcs, + (erts_aint32_t) nval); BIF_RET(make_small(oval)); } else if (BIF_ARG_1 == am_min_heap_size) { int oval = H_MIN_SIZE; @@ -4286,7 +4286,7 @@ void erts_init_bif(void) erts_smp_spinlock_init(&make_ref_lock, "make_ref"); erts_smp_mtx_init(&ports_snapshot_mtx, "ports_snapshot"); - erts_smp_atomic_init(&erts_dead_ports_ptr, (erts_aint_t) NULL); + erts_smp_atomic_init_nob(&erts_dead_ports_ptr, (erts_aint_t) NULL); /* * bif_return_trap/1 is a hidden BIF that bifs that need to diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index b8889e6206..432b3d0780 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -626,7 +626,7 @@ bin_check(void) erts_printf("%p orig_size: %bpd, norefs = %bpd\n", bp->val, bp->val->orig_size, - erts_smp_atomic_read(&bp->val->refc)); + erts_smp_atomic_read_nob(&bp->val->refc)); } } if (printed) { @@ -650,7 +650,7 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) char dumpnamebuf[MAXPATHLEN]; char* dumpname; - if (ERTS_IS_CRASH_DUMPING) + if (ERTS_SOMEONE_IS_CRASH_DUMPING) return; /* Wait for all threads to block. If all threads haven't blocked @@ -667,7 +667,8 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) /* Allow us to pass certain places without locking... */ #ifdef ERTS_SMP - erts_smp_atomic_inc(&erts_writing_erl_crash_dump); + erts_smp_atomic32_set_mb(&erts_writing_erl_crash_dump, 1); + erts_smp_tsd_set(erts_is_crash_dumping_key, (void *) 1); #else erts_writing_erl_crash_dump = 1; #endif diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index b1cdd0660a..ad042ec088 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -128,8 +128,8 @@ delete_cache(ErtsAtomCache *cache) { if (cache) { erts_free(ERTS_ALC_T_DCACHE, (void *) cache); - ASSERT(erts_smp_atomic_read(&no_caches) > 0); - erts_smp_atomic_dec(&no_caches); + ASSERT(erts_smp_atomic_read_nob(&no_caches) > 0); + erts_smp_atomic_dec_nob(&no_caches); } } @@ -147,7 +147,7 @@ create_cache(DistEntry *dep) dep->cache = cp = (ErtsAtomCache*) erts_alloc(ERTS_ALC_T_DCACHE, sizeof(ErtsAtomCache)); - erts_smp_atomic_inc(&no_caches); + erts_smp_atomic_inc_nob(&no_caches); for (i = 0; i < sizeof(cp->in_arr)/sizeof(cp->in_arr[0]); i++) { cp->in_arr[i] = THE_NON_VALUE; cp->out_arr[i] = THE_NON_VALUE; @@ -156,7 +156,7 @@ create_cache(DistEntry *dep) Uint erts_dist_cache_size(void) { - return (Uint) erts_smp_atomic_read(&no_caches)*sizeof(ErtsAtomCache); + return (Uint) erts_smp_atomic_read_mb(&no_caches)*sizeof(ErtsAtomCache); } static ErtsProcList * @@ -444,7 +444,7 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason) ErtsMonitor *monitors; Uint32 flags; - erts_smp_atomic_set(&dep->dist_cmd_scheduled, 1); + erts_smp_atomic_set_mb(&dep->dist_cmd_scheduled, 1); erts_smp_de_rwlock(dep); ERTS_SMP_LC_ASSERT(is_internal_port(dep->cid) @@ -510,7 +510,7 @@ void init_dist(void) { init_nodes_monitors(); - erts_smp_atomic_init(&no_caches, 0); + erts_smp_atomic_init_nob(&no_caches, 0); /* Lookup/Install all references to trap functions */ dsend2_trap = trap_function(am_dsend,2); @@ -596,7 +596,7 @@ static void clear_dist_entry(DistEntry *dep) suspendees = get_suspended_on_de(dep, ERTS_DE_QFLGS_ALL); erts_smp_mtx_unlock(&dep->qlock); - erts_smp_atomic_set(&dep->dist_cmd_scheduled, 0); + erts_smp_atomic_set_nob(&dep->dist_cmd_scheduled, 0); dep->send = NULL; erts_smp_de_rwunlock(dep); @@ -1775,7 +1775,7 @@ erts_dist_command(Port *prt, int reds_limit) erts_refc_inc(&dep->refc, 1); /* Otherwise dist_entry might be removed if port command fails */ - erts_smp_atomic_xchg(&dep->dist_cmd_scheduled, 0); + erts_smp_atomic_set_mb(&dep->dist_cmd_scheduled, 0); erts_smp_de_rlock(dep); flags = dep->flags; diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h index 695a4fc3fe..845151c895 100644 --- a/erts/emulator/beam/dist.h +++ b/erts/emulator/beam/dist.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * Copyright Ericsson AB 1996-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -203,7 +203,7 @@ void erts_schedule_dist_command(Port *prt, DistEntry *dist_entry) id = dep->cid; } - if (!erts_smp_atomic_xchg(&dep->dist_cmd_scheduled, 1)) { + if (!erts_smp_atomic_xchg_mb(&dep->dist_cmd_scheduled, 1)) { (void) erts_port_task_schedule(id, &dep->dist_cmd, ERTS_PORT_TASK_DIST_CMD, diff --git a/erts/emulator/beam/erl_afit_alloc.c b/erts/emulator/beam/erl_afit_alloc.c index e8b594bb47..d397cd8848 100644 --- a/erts/emulator/beam/erl_afit_alloc.c +++ b/erts/emulator/beam/erl_afit_alloc.c @@ -43,9 +43,9 @@ /* Prototypes of callback functions */ static Block_t * get_free_block (Allctr_t *, Uint, - Block_t *, Uint); -static void link_free_block (Allctr_t *, Block_t *); -static void unlink_free_block (Allctr_t *, Block_t *); + Block_t *, Uint, Uint32); +static void link_free_block (Allctr_t *, Block_t *, Uint32); +static void unlink_free_block (Allctr_t *, Block_t *, Uint32); static Eterm info_options (Allctr_t *, char *, int *, @@ -72,6 +72,8 @@ erts_afalc_start(AFAllctr_t *afallctr, is a struct). */ Allctr_t *allctr = (Allctr_t *) afallctr; + init->sbmbct = 0; /* Small mbc not supported by afit */ + sys_memcpy((void *) afallctr, (void *) &nulled_state, sizeof(AFAllctr_t)); allctr->mbc_header_size = sizeof(Carrier_t); @@ -105,7 +107,8 @@ erts_afalc_start(AFAllctr_t *afallctr, } static Block_t * -get_free_block(Allctr_t *allctr, Uint size, Block_t *cand_blk, Uint cand_size) +get_free_block(Allctr_t *allctr, Uint size, Block_t *cand_blk, Uint cand_size, + Uint32 flags) { AFAllctr_t *afallctr = (AFAllctr_t *) allctr; @@ -123,7 +126,7 @@ get_free_block(Allctr_t *allctr, Uint size, Block_t *cand_blk, Uint cand_size) } static void -link_free_block(Allctr_t *allctr, Block_t *block) +link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) { AFFreeBlock_t *blk = (AFFreeBlock_t *) block; AFAllctr_t *afallctr = (AFAllctr_t *) allctr; @@ -144,7 +147,7 @@ link_free_block(Allctr_t *allctr, Block_t *block) } static void -unlink_free_block(Allctr_t *allctr, Block_t *block) +unlink_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) { AFFreeBlock_t *blk = (AFFreeBlock_t *) block; AFAllctr_t *afallctr = (AFAllctr_t *) allctr; diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index cda404af5e..bbc8a445a7 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -50,6 +50,9 @@ #include "erl_bestfit_alloc.h" #define GET_ERL_AF_ALLOC_IMPL #include "erl_afit_alloc.h" +#define GET_ERL_AOFF_ALLOC_IMPL +#include "erl_ao_firstfit_alloc.h" + #define ERTS_ALC_DEFAULT_MAX_THR_PREF 16 @@ -85,15 +88,19 @@ typedef union { char align_bfa[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(BFAllctr_t))]; AFAllctr_t afa; char align_afa[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(AFAllctr_t))]; + AOFFAllctr_t aoffa; + char align_aoffa[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(AOFFAllctr_t))]; } ErtsAllocatorState_t; -static ErtsAllocatorState_t sl_alloc_state; +static ErtsAllocatorState_t sbmbc_alloc_state; static ErtsAllocatorState_t std_alloc_state; static ErtsAllocatorState_t ll_alloc_state; #if HALFWORD_HEAP -static ErtsAllocatorState_t std_alloc_low_state; -static ErtsAllocatorState_t ll_alloc_low_state; +static ErtsAllocatorState_t sbmbc_low_alloc_state; +static ErtsAllocatorState_t std_low_alloc_state; +static ErtsAllocatorState_t ll_low_alloc_state; #endif +static ErtsAllocatorState_t sl_alloc_state; static ErtsAllocatorState_t temp_alloc_state; static ErtsAllocatorState_t eheap_alloc_state; static ErtsAllocatorState_t binary_alloc_state; @@ -120,7 +127,8 @@ static void *fix_core_alloc(Uint size) enum allctr_type { GOODFIT, BESTFIT, - AFIT + AFIT, + AOFIRSTFIT }; struct au_init { @@ -132,6 +140,7 @@ struct au_init { GFAllctrInit_t gf; BFAllctrInit_t bf; AFAllctrInit_t af; + AOFFAllctrInit_t aoff; } init; struct { int mmbcs; @@ -145,7 +154,8 @@ struct au_init { ERTS_DEFAULT_ALLCTR_INIT, \ ERTS_DEFAULT_GF_ALLCTR_INIT, \ ERTS_DEFAULT_BF_ALLCTR_INIT, \ - ERTS_DEFAULT_AF_ALLCTR_INIT \ + ERTS_DEFAULT_AF_ALLCTR_INIT, \ + ERTS_DEFAULT_AOFF_ALLCTR_INIT \ } typedef struct { @@ -162,6 +172,7 @@ typedef struct { char *mtrace; char *nodename; } instr; + struct au_init sbmbc_alloc; struct au_init sl_alloc; struct au_init std_alloc; struct au_init ll_alloc; @@ -171,8 +182,9 @@ typedef struct { struct au_init ets_alloc; struct au_init driver_alloc; #if HALFWORD_HEAP - struct au_init std_alloc_low; - struct au_init ll_alloc_low; + struct au_init sbmbc_low_alloc; + struct au_init std_low_alloc; + struct au_init ll_low_alloc; #endif } erts_alc_hndl_args_init_t; @@ -185,6 +197,34 @@ do { \ } while (0) static void +set_default_sbmbc_alloc_opts(struct au_init *ip) +{ + SET_DEFAULT_ALLOC_OPTS(ip); + ip->enable = 0; + ip->thr_spec = 0; + ip->atype = BESTFIT; + ip->init.bf.ao = 1; + ip->init.util.ramv = 0; + ip->init.util.mmsbc = 0; + ip->init.util.mmmbc = 500; + ip->init.util.sbct = ~((UWord) 0); + ip->init.util.name_prefix = "sbmbc_"; + ip->init.util.alloc_no = ERTS_ALC_A_SBMBC; +#ifndef SMALL_MEMORY + ip->init.util.mmbcs = 2*1024*1024; /* Main carrier size */ +#else + ip->init.util.mmbcs = 1*1024*1024; /* Main carrier size */ +#endif + ip->init.util.ts = ERTS_ALC_MTA_SBMBC; + ip->init.util.asbcst = 0; + ip->init.util.rsbcst = 0; + ip->init.util.rsbcmt = 0; + ip->init.util.rmbcmt = 0; + ip->init.util.sbmbct = 0; + ip->init.util.sbmbcs = 0; +} + +static void set_default_sl_alloc_opts(struct au_init *ip) { SET_DEFAULT_ALLOC_OPTS(ip); @@ -202,6 +242,7 @@ set_default_sl_alloc_opts(struct au_init *ip) ip->init.util.ts = ERTS_ALC_MTA_SHORT_LIVED; ip->init.util.rsbcst = 80; #if HALFWORD_HEAP + ip->init.util.force = 1; ip->init.util.low_mem = 1; #endif @@ -249,6 +290,8 @@ set_default_ll_alloc_opts(struct au_init *ip) ip->init.util.rsbcst = 0; ip->init.util.rsbcmt = 0; ip->init.util.rmbcmt = 0; + ip->init.util.sbmbct = 0; + ip->init.util.sbmbcs = 0; } static void @@ -269,6 +312,7 @@ set_default_temp_alloc_opts(struct au_init *ip) ip->init.util.rsbcst = 90; ip->init.util.rmbcmt = 100; #if HALFWORD_HEAP + ip->init.util.force = 1; ip->init.util.low_mem = 1; #endif } @@ -291,6 +335,7 @@ set_default_eheap_alloc_opts(struct au_init *ip) ip->init.util.ts = ERTS_ALC_MTA_EHEAP; ip->init.util.rsbcst = 50; #if HALFWORD_HEAP + ip->init.util.force = 1; ip->init.util.low_mem = 1; #endif } @@ -436,10 +481,13 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) hdbg_init(); #endif + erts_have_sbmbc_alloc = 0; + erts_sys_alloc_init(); init_thr_ix(erts_no_schedulers); erts_init_utils_mem(); + set_default_sbmbc_alloc_opts(&init.sbmbc_alloc); set_default_sl_alloc_opts(&init.sl_alloc); set_default_std_alloc_opts(&init.std_alloc); set_default_ll_alloc_opts(&init.ll_alloc); @@ -453,6 +501,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) handle_args(argc, argv, &init); if (erts_no_schedulers <= 1) { + init.sbmbc_alloc.thr_spec = 0; init.sl_alloc.thr_spec = 0; init.std_alloc.thr_spec = 0; init.ll_alloc.thr_spec = 0; @@ -464,6 +513,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) if (init.erts_alloc_config) { /* Adjust flags that erts_alloc_config won't like */ + init.sbmbc_alloc.thr_spec = 0; init.temp_alloc.thr_spec = 0; init.sl_alloc.thr_spec = 0; init.std_alloc.thr_spec = 0; @@ -480,6 +530,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) init.temp_alloc.thr_spec = erts_no_schedulers; /* Others must use thread preferred interface */ + adjust_tpref(&init.sbmbc_alloc, erts_no_schedulers); adjust_tpref(&init.sl_alloc, erts_no_schedulers); adjust_tpref(&init.std_alloc, erts_no_schedulers); adjust_tpref(&init.ll_alloc, erts_no_schedulers); @@ -497,6 +548,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) * The following allocators cannot be run with afit strategy. * Make sure they don't... */ + refuse_af_strategy(&init.sbmbc_alloc); refuse_af_strategy(&init.sl_alloc); refuse_af_strategy(&init.std_alloc); refuse_af_strategy(&init.ll_alloc); @@ -518,6 +570,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) erts_afalc_init(); erts_bfalc_init(); erts_gfalc_init(); + erts_aoffalc_init(); for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) { erts_allctrs[i].alloc = NULL; @@ -551,19 +604,30 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) #if HALFWORD_HEAP /* Init low memory variants by cloning */ - init.std_alloc_low = init.std_alloc; - init.std_alloc_low.init.util.alloc_no = ERTS_ALC_A_STANDARD_LOW; - init.std_alloc_low.init.util.low_mem = 1; - - init.ll_alloc_low = init.ll_alloc; - init.ll_alloc_low.init.util.alloc_no = ERTS_ALC_A_LONG_LIVED_LOW; - init.ll_alloc_low.init.util.low_mem = 1; - - set_au_allocator(ERTS_ALC_A_STANDARD_LOW, &init.std_alloc_low); - set_au_allocator(ERTS_ALC_A_LONG_LIVED_LOW, &init.ll_alloc_low); + init.sbmbc_low_alloc = init.sbmbc_alloc; + init.sbmbc_low_alloc.init.util.name_prefix = "sbmbc_low_"; + init.sbmbc_low_alloc.init.util.alloc_no = ERTS_ALC_A_SBMBC_LOW; + init.sbmbc_low_alloc.init.util.low_mem = 1; + + init.std_low_alloc = init.std_alloc; + init.std_low_alloc.init.util.name_prefix = "std_low_"; + init.std_low_alloc.init.util.alloc_no = ERTS_ALC_A_STANDARD_LOW; + init.std_low_alloc.init.util.force = 1; + init.std_low_alloc.init.util.low_mem = 1; + + init.ll_low_alloc = init.ll_alloc; + init.ll_low_alloc.init.util.name_prefix = "ll_low_"; + init.ll_low_alloc.init.util.alloc_no = ERTS_ALC_A_LONG_LIVED_LOW; + init.ll_low_alloc.init.util.force = 1; + init.ll_low_alloc.init.util.low_mem = 1; + + set_au_allocator(ERTS_ALC_A_SBMBC_LOW, &init.sbmbc_low_alloc); + set_au_allocator(ERTS_ALC_A_STANDARD_LOW, &init.std_low_alloc); + set_au_allocator(ERTS_ALC_A_LONG_LIVED_LOW, &init.ll_low_alloc); #endif /* HALFWORD */ set_au_allocator(ERTS_ALC_A_TEMPORARY, &init.temp_alloc); + set_au_allocator(ERTS_ALC_A_SBMBC, &init.sbmbc_alloc); set_au_allocator(ERTS_ALC_A_SHORT_LIVED, &init.sl_alloc); set_au_allocator(ERTS_ALC_A_STANDARD, &init.std_alloc); set_au_allocator(ERTS_ALC_A_LONG_LIVED, &init.ll_alloc); @@ -593,6 +657,20 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) erts_mtrace_init(init.instr.mtrace, init.instr.nodename); + /* sbmbc_alloc() needs to be started first */ + start_au_allocator(ERTS_ALC_A_SBMBC, + &init.sbmbc_alloc, + &sbmbc_alloc_state); +#if HALFWORD_HEAP + start_au_allocator(ERTS_ALC_A_SBMBC_LOW, + &init.sbmbc_low_alloc, + &sbmbc_low_alloc_state); + erts_have_sbmbc_alloc = (init.sbmbc_alloc.enable + && init.sbmbc_low_alloc.enable); +#else + erts_have_sbmbc_alloc = init.sbmbc_alloc.enable; +#endif + start_au_allocator(ERTS_ALC_A_TEMPORARY, &init.temp_alloc, &temp_alloc_state); @@ -610,11 +688,11 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) &ll_alloc_state); #if HALFWORD_HEAP start_au_allocator(ERTS_ALC_A_LONG_LIVED_LOW, - &init.ll_alloc_low, - &ll_alloc_low_state); + &init.ll_low_alloc, + &ll_low_alloc_state); start_au_allocator(ERTS_ALC_A_STANDARD_LOW, - &init.std_alloc_low, - &std_alloc_low_state); + &init.std_low_alloc, + &std_low_alloc_state); #endif start_au_allocator(ERTS_ALC_A_EHEAP, &init.eheap_alloc, @@ -680,14 +758,11 @@ set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init) ErtsAllocatorInfo_t *ai = &erts_allctrs_info[alctr_n]; ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[alctr_n]; -#if HALFWORD_HEAP - /* If halfword heap, silently ignore any disabling of internal - * allocators for low memory + /* + * Some allocators are forced on if halfword heap is used. */ - if (init->init.util.low_mem) { + if (init->init.util.force) init->enable = 1; - } -#endif if (!init->enable) { af->alloc = erts_sys_alloc; @@ -837,6 +912,12 @@ start_au_allocator(ErtsAlcType_t alctr_n, &init->init.af, &init->init.util); break; + case AOFIRSTFIT: + as = (void *) erts_aoffalc_start((AOFFAllctr_t *) as0, + &init->init.aoff, + &init->init.util); + break; + default: as = NULL; ASSERT(0); @@ -947,6 +1028,20 @@ get_kb_value(char *param_end, char** argv, int* ip) } static Uint +get_byte_value(char *param_end, char** argv, int* ip) +{ + Sint tmp; + char *rest; + char *param = argv[*ip]+1; + char *value = get_value(param_end, argv, ip); + errno = 0; + tmp = (Sint) strtol(value, &rest, 10); + if (errno != 0 || rest == value || tmp < 0) + bad_value(param, param_end, value); + return (Uint) tmp; +} + +static Uint get_amount_value(char *param_end, char** argv, int* ip) { Sint tmp; @@ -1017,6 +1112,9 @@ handle_au_arg(struct au_init *auip, else if (strcmp("af", alg) == 0) { auip->atype = AFIT; } + else if (strcmp("aoff", alg) == 0) { + auip->atype = AOFIRSTFIT; + } else { bad_value(param, sub_param + 1, alg); } @@ -1085,6 +1183,12 @@ handle_au_arg(struct au_init *auip, if(has_prefix("sbct", sub_param)) { auip->init.util.sbct = get_kb_value(sub_param + 4, argv, ip); } + else if (has_prefix("sbmbcs", sub_param)) { + auip->init.util.sbmbcs = get_byte_value(sub_param + 6, argv, ip); + } + else if (has_prefix("sbmbct", sub_param)) { + auip->init.util.sbmbct = get_byte_value(sub_param + 6, argv, ip); + } else if (has_prefix("smbcs", sub_param)) { auip->default_.smbcs = 0; auip->init.util.smbcs = get_kb_value(sub_param + 5, argv, ip); @@ -1123,6 +1227,7 @@ static void handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) { struct au_init *aui[] = { + &init->sbmbc_alloc, &init->binary_alloc, &init->std_alloc, &init->ets_alloc, @@ -1150,6 +1255,9 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) case 'B': handle_au_arg(&init->binary_alloc, &argv[i][3], argv, &i); break; + case 'C': + handle_au_arg(&init->sbmbc_alloc, &argv[i][3], argv, &i); + break; case 'D': handle_au_arg(&init->std_alloc, &argv[i][3], argv, &i); break; @@ -1856,12 +1964,16 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) return am_badarg; } - /* All alloc_util allocators *have* to be enabled */ + /* All alloc_util allocators except sbmbc_alloc *have* to be enabled */ for (ai = ERTS_ALC_A_MIN; ai <= ERTS_ALC_A_MAX; ai++) { switch (ai) { case ERTS_ALC_A_SYSTEM: case ERTS_ALC_A_FIXED_SIZE: + case ERTS_ALC_A_SBMBC: +#if HALFWORD_HEAP + case ERTS_ALC_A_SBMBC_LOW: +#endif break; default: if (!erts_allctrs_info[ai].enabled @@ -1901,6 +2013,12 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) * Often not thread safe and usually never * contain any allocated memory. */ + case ERTS_ALC_A_SBMBC: + /* Included in other allocators */ +#if HALFWORD_HEAP + case ERTS_ALC_A_SBMBC_LOW: + /* Included in other allocators */ +#endif continue; case ERTS_ALC_A_EHEAP: save = &size.processes; @@ -2882,6 +3000,7 @@ unsigned long erts_alc_test(unsigned long op, case 0x2: return erts_bfalc_test(op, a1, a2); case 0x3: return erts_afalc_test(op, a1, a2); case 0x4: return erts_mseg_test(op, a1, a2, a3); + case 0x5: return erts_aoffalc_test(op, a1, a2); case 0xf: switch (op) { case 0xf00: @@ -2925,6 +3044,7 @@ unsigned long erts_alc_test(unsigned long op, init.atype = GOODFIT; init.init.util.name_prefix = (char *) a1; init.init.util.ts = a2 ? 1 : 0; + init.init.util.sbmbct = 0; if ((char **) a3) { char **argv = (char **) a3; @@ -2960,6 +3080,14 @@ unsigned long erts_alc_test(unsigned long op, &init.init.af, &init.init.util); break; + case AOFIRSTFIT: + allctr = erts_aoffalc_start((AOFFAllctr_t *) + erts_alloc(ERTS_ALC_T_UNDEF, + sizeof(AOFFAllctr_t)), + &init.init.aoff, + &init.init.util); + break; + default: ASSERT(0); allctr = NULL; diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h index ce792d4d17..c35a60da22 100644 --- a/erts/emulator/beam/erl_alloc.h +++ b/erts/emulator/beam/erl_alloc.h @@ -99,6 +99,14 @@ unsigned long erts_alc_test(unsigned long, #define ERTS_ALC_MIN_LONG_LIVED_TIME (10*60*1000) +#if HALFWORD_HEAP +#define ERTS_IS_SBMBC_ALLOCATOR_NO__(NO) \ + ((NO) == ERTS_ALC_A_SBMBC || (NO) == ERTS_ALC_A_SBMBC_LOW) +#else +#define ERTS_IS_SBMBC_ALLOCATOR_NO__(NO) \ + ((NO) == ERTS_ALC_A_SBMBC) +#endif + typedef struct { int alloc_util; int enabled; diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index c6cc0e1fac..eda0831441 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -65,6 +65,11 @@ allocator SYSTEM true sys_alloc +allocator SBMBC true sbmbc_alloc ++if halfword +allocator SBMBC_LOW true sbmbc_low_alloc ++endif + +if smp allocator TEMPORARY true temp_alloc @@ -76,8 +81,8 @@ allocator ETS true ets_alloc allocator FIXED_SIZE true fix_alloc +if halfword -allocator LONG_LIVED_LOW true ll_alloc_low -allocator STANDARD_LOW true std_alloc_low +allocator LONG_LIVED_LOW true ll_low_alloc +allocator STANDARD_LOW true std_low_alloc +endif +else # Non smp build @@ -91,8 +96,8 @@ allocator ETS false ets_alloc allocator FIXED_SIZE false fix_alloc +if halfword -allocator LONG_LIVED_LOW false ll_alloc_low -allocator STANDARD_LOW false std_alloc_low +allocator LONG_LIVED_LOW false ll_low_alloc +allocator STANDARD_LOW false std_low_alloc +endif +endif @@ -134,6 +139,7 @@ class SYSTEM system_data # # <TYPE> <ALLOCATOR> <CLASS> <DESCRIPTION> +type SBMBC SBMBC SYSTEM small_block_mbc type PROC FIXED_SIZE PROCESSES proc type ATOM FIXED_SIZE ATOM atom_entry type MODULE FIXED_SIZE CODE module_entry @@ -330,6 +336,7 @@ type SSB SHORT_LIVED PROCESSES ssb +if halfword +type SBMBC_LOW SBMBC_LOW SYSTEM small_block_mbc_low type DDLL_PROCESS STANDARD_LOW SYSTEM ddll_processes type MONITOR_LH STANDARD_LOW PROCESSES monitor_lh type NLINK_LH STANDARD_LOW PROCESSES nlink_lh diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index cc04ef65bf..d51ed0c36d 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -66,6 +66,7 @@ static int atoms_initialized = 0; static int initialized = 0; +int erts_have_sbmbc_alloc; #if HAVE_ERTS_MSEG @@ -85,8 +86,6 @@ static int initialized = 0; #undef ASSERT #define ASSERT ASSERT_EXPR -#define ERTS_ALCU_FLG_FAIL_REALLOC_MOVE ((UWord) 1) - #if 0 /* Can be useful for debugging */ #define MBC_REALLOC_ALWAYS_MOVES @@ -275,14 +274,26 @@ static void check_blk_carrier(Allctr_t *, Block_t *); #ifdef DEBUG #define DEBUG_CHECK_CARRIER_NO_SZ(AP) \ - ASSERT(((AP)->sbcs.curr_mseg.no && (AP)->sbcs.curr_mseg.size) \ - || (!(AP)->sbcs.curr_mseg.no && !(AP)->sbcs.curr_mseg.size));\ - ASSERT(((AP)->sbcs.curr_sys_alloc.no && (AP)->sbcs.curr_sys_alloc.size)\ - || (!(AP)->sbcs.curr_sys_alloc.no && !(AP)->sbcs.curr_sys_alloc.size));\ - ASSERT(((AP)->mbcs.curr_mseg.no && (AP)->mbcs.curr_mseg.size) \ - || (!(AP)->mbcs.curr_mseg.no && !(AP)->mbcs.curr_mseg.size));\ - ASSERT(((AP)->mbcs.curr_sys_alloc.no && (AP)->mbcs.curr_sys_alloc.size)\ - || (!(AP)->mbcs.curr_sys_alloc.no && !(AP)->mbcs.curr_sys_alloc.size)) + ASSERT(((AP)->sbcs.curr.norm.mseg.no \ + && (AP)->sbcs.curr.norm.mseg.size) \ + || (!(AP)->sbcs.curr.norm.mseg.no \ + && !(AP)->sbcs.curr.norm.mseg.size)); \ + ASSERT(((AP)->sbcs.curr.norm.sys_alloc.no \ + && (AP)->sbcs.curr.norm.sys_alloc.size) \ + || (!(AP)->sbcs.curr.norm.sys_alloc.no \ + && !(AP)->sbcs.curr.norm.sys_alloc.size)); \ + ASSERT(((AP)->mbcs.curr.norm.mseg.no \ + && (AP)->mbcs.curr.norm.mseg.size) \ + || (!(AP)->mbcs.curr.norm.mseg.no \ + && !(AP)->mbcs.curr.norm.mseg.size)); \ + ASSERT(((AP)->mbcs.curr.norm.sys_alloc.no \ + && (AP)->mbcs.curr.norm.sys_alloc.size) \ + || (!(AP)->mbcs.curr.norm.sys_alloc.no \ + && !(AP)->mbcs.curr.norm.sys_alloc.size)); \ + ASSERT(((AP)->sbmbcs.curr.small_block.no \ + && (AP)->sbmbcs.curr.small_block.size) \ + || (!(AP)->sbmbcs.curr.small_block.no \ + && !(AP)->sbmbcs.curr.small_block.size)) #else #define DEBUG_CHECK_CARRIER_NO_SZ(AP) @@ -292,27 +303,27 @@ static void check_blk_carrier(Allctr_t *, Block_t *); (AP)->sbcs.blocks.curr.size += (BSZ); \ if ((AP)->sbcs.blocks.max.size < (AP)->sbcs.blocks.curr.size) \ (AP)->sbcs.blocks.max.size = (AP)->sbcs.blocks.curr.size; \ - if ((AP)->sbcs.max.no < ((AP)->sbcs.curr_mseg.no \ - + (AP)->sbcs.curr_sys_alloc.no)) \ - (AP)->sbcs.max.no = ((AP)->sbcs.curr_mseg.no \ - + (AP)->sbcs.curr_sys_alloc.no); \ - if ((AP)->sbcs.max.size < ((AP)->sbcs.curr_mseg.size \ - + (AP)->sbcs.curr_sys_alloc.size)) \ - (AP)->sbcs.max.size = ((AP)->sbcs.curr_mseg.size \ - + (AP)->sbcs.curr_sys_alloc.size) + if ((AP)->sbcs.max.no < ((AP)->sbcs.curr.norm.mseg.no \ + + (AP)->sbcs.curr.norm.sys_alloc.no)) \ + (AP)->sbcs.max.no = ((AP)->sbcs.curr.norm.mseg.no \ + + (AP)->sbcs.curr.norm.sys_alloc.no); \ + if ((AP)->sbcs.max.size < ((AP)->sbcs.curr.norm.mseg.size \ + + (AP)->sbcs.curr.norm.sys_alloc.size)) \ + (AP)->sbcs.max.size = ((AP)->sbcs.curr.norm.mseg.size \ + + (AP)->sbcs.curr.norm.sys_alloc.size) #define STAT_MSEG_SBC_ALLOC(AP, CSZ, BSZ) \ do { \ - (AP)->sbcs.curr_mseg.no++; \ - (AP)->sbcs.curr_mseg.size += (CSZ); \ + (AP)->sbcs.curr.norm.mseg.no++; \ + (AP)->sbcs.curr.norm.mseg.size += (CSZ); \ STAT_SBC_ALLOC((AP), (BSZ)); \ DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ } while (0) #define STAT_SYS_ALLOC_SBC_ALLOC(AP, CSZ, BSZ) \ do { \ - (AP)->sbcs.curr_sys_alloc.no++; \ - (AP)->sbcs.curr_sys_alloc.size += (CSZ); \ + (AP)->sbcs.curr.norm.sys_alloc.no++; \ + (AP)->sbcs.curr.norm.sys_alloc.size += (CSZ); \ STAT_SBC_ALLOC((AP), (BSZ)); \ DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ } while (0) @@ -324,85 +335,111 @@ do { \ #define STAT_MSEG_SBC_FREE(AP, CSZ, BSZ) \ do { \ - ASSERT((AP)->sbcs.curr_mseg.no > 0); \ - (AP)->sbcs.curr_mseg.no--; \ - ASSERT((AP)->sbcs.curr_mseg.size >= (CSZ)); \ - (AP)->sbcs.curr_mseg.size -= (CSZ); \ + ASSERT((AP)->sbcs.curr.norm.mseg.no > 0); \ + (AP)->sbcs.curr.norm.mseg.no--; \ + ASSERT((AP)->sbcs.curr.norm.mseg.size >= (CSZ)); \ + (AP)->sbcs.curr.norm.mseg.size -= (CSZ); \ STAT_SBC_FREE((AP), (BSZ)); \ DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ } while (0) #define STAT_SYS_ALLOC_SBC_FREE(AP, CSZ, BSZ) \ do { \ - ASSERT((AP)->sbcs.curr_sys_alloc.no > 0); \ - (AP)->sbcs.curr_sys_alloc.no--; \ - ASSERT((AP)->sbcs.curr_sys_alloc.size >= (CSZ)); \ - (AP)->sbcs.curr_sys_alloc.size -= (CSZ); \ + ASSERT((AP)->sbcs.curr.norm.sys_alloc.no > 0); \ + (AP)->sbcs.curr.norm.sys_alloc.no--; \ + ASSERT((AP)->sbcs.curr.norm.sys_alloc.size >= (CSZ)); \ + (AP)->sbcs.curr.norm.sys_alloc.size -= (CSZ); \ STAT_SBC_FREE((AP), (BSZ)); \ DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ } while (0) #define STAT_MBC_ALLOC(AP) \ - if ((AP)->mbcs.max.no < ((AP)->mbcs.curr_mseg.no \ - + (AP)->mbcs.curr_sys_alloc.no)) \ - (AP)->mbcs.max.no = ((AP)->mbcs.curr_mseg.no \ - + (AP)->mbcs.curr_sys_alloc.no); \ - if ((AP)->mbcs.max.size < ((AP)->mbcs.curr_mseg.size \ - + (AP)->mbcs.curr_sys_alloc.size)) \ - (AP)->mbcs.max.size = ((AP)->mbcs.curr_mseg.size \ - + (AP)->mbcs.curr_sys_alloc.size) + if ((AP)->mbcs.max.no < ((AP)->mbcs.curr.norm.mseg.no \ + + (AP)->mbcs.curr.norm.sys_alloc.no)) \ + (AP)->mbcs.max.no = ((AP)->mbcs.curr.norm.mseg.no \ + + (AP)->mbcs.curr.norm.sys_alloc.no); \ + if ((AP)->mbcs.max.size < ((AP)->mbcs.curr.norm.mseg.size \ + + (AP)->mbcs.curr.norm.sys_alloc.size)) \ + (AP)->mbcs.max.size = ((AP)->mbcs.curr.norm.mseg.size \ + + (AP)->mbcs.curr.norm.sys_alloc.size) + +#define STAT_SBMBC_ALLOC(AP, CSZ) \ +do { \ + (AP)->sbmbcs.curr.small_block.no++; \ + (AP)->sbmbcs.curr.small_block.size += (CSZ); \ + if ((AP)->sbmbcs.max.no < (AP)->sbmbcs.curr.small_block.no) \ + (AP)->sbmbcs.max.no = (AP)->sbmbcs.curr.small_block.no; \ + if ((AP)->sbmbcs.max.size < (AP)->sbmbcs.curr.small_block.size) \ + (AP)->sbmbcs.max.size = (AP)->sbmbcs.curr.small_block.size; \ + DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ +} while (0) #define STAT_MSEG_MBC_ALLOC(AP, CSZ) \ do { \ - (AP)->mbcs.curr_mseg.no++; \ - (AP)->mbcs.curr_mseg.size += (CSZ); \ + (AP)->mbcs.curr.norm.mseg.no++; \ + (AP)->mbcs.curr.norm.mseg.size += (CSZ); \ STAT_MBC_ALLOC((AP)); \ DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ } while (0) #define STAT_SYS_ALLOC_MBC_ALLOC(AP, CSZ) \ do { \ - (AP)->mbcs.curr_sys_alloc.no++; \ - (AP)->mbcs.curr_sys_alloc.size += (CSZ); \ + (AP)->mbcs.curr.norm.sys_alloc.no++; \ + (AP)->mbcs.curr.norm.sys_alloc.size += (CSZ); \ STAT_MBC_ALLOC((AP)); \ DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ } while (0) +#define STAT_SBMBC_FREE(AP, CSZ) \ +do { \ + ASSERT((AP)->sbmbcs.curr.small_block.no > 0); \ + (AP)->sbmbcs.curr.small_block.no--; \ + ASSERT((AP)->sbmbcs.curr.small_block.size >= (CSZ)); \ + (AP)->sbmbcs.curr.small_block.size -= (CSZ); \ + DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ +} while (0) + #define STAT_MSEG_MBC_FREE(AP, CSZ) \ do { \ - ASSERT((AP)->mbcs.curr_mseg.no > 0); \ - (AP)->mbcs.curr_mseg.no--; \ - ASSERT((AP)->mbcs.curr_mseg.size >= (CSZ)); \ - (AP)->mbcs.curr_mseg.size -= (CSZ); \ + ASSERT((AP)->mbcs.curr.norm.mseg.no > 0); \ + (AP)->mbcs.curr.norm.mseg.no--; \ + ASSERT((AP)->mbcs.curr.norm.mseg.size >= (CSZ)); \ + (AP)->mbcs.curr.norm.mseg.size -= (CSZ); \ DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ } while (0) #define STAT_SYS_ALLOC_MBC_FREE(AP, CSZ) \ do { \ - ASSERT((AP)->mbcs.curr_sys_alloc.no > 0); \ - (AP)->mbcs.curr_sys_alloc.no--; \ - ASSERT((AP)->mbcs.curr_sys_alloc.size >= (CSZ)); \ - (AP)->mbcs.curr_sys_alloc.size -= (CSZ); \ + ASSERT((AP)->mbcs.curr.norm.sys_alloc.no > 0); \ + (AP)->mbcs.curr.norm.sys_alloc.no--; \ + ASSERT((AP)->mbcs.curr.norm.sys_alloc.size >= (CSZ)); \ + (AP)->mbcs.curr.norm.sys_alloc.size -= (CSZ); \ DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ } while (0) -#define STAT_MBC_BLK_ALLOC(AP, BSZ) \ +#define STAT_MBC_BLK_ALLOC(AP, BSZ, FLGS) \ do { \ - (AP)->mbcs.blocks.curr.no++; \ - if ((AP)->mbcs.blocks.max.no < (AP)->mbcs.blocks.curr.no) \ - (AP)->mbcs.blocks.max.no = (AP)->mbcs.blocks.curr.no; \ - (AP)->mbcs.blocks.curr.size += (BSZ); \ - if ((AP)->mbcs.blocks.max.size < (AP)->mbcs.blocks.curr.size) \ - (AP)->mbcs.blocks.max.size = (AP)->mbcs.blocks.curr.size; \ + CarriersStats_t *cstats__ = (((FLGS) & ERTS_ALCU_FLG_SBMBC) \ + ? &(AP)->sbmbcs \ + : &(AP)->mbcs); \ + cstats__->blocks.curr.no++; \ + if (cstats__->blocks.max.no < cstats__->blocks.curr.no) \ + cstats__->blocks.max.no = cstats__->blocks.curr.no; \ + cstats__->blocks.curr.size += (BSZ); \ + if (cstats__->blocks.max.size < cstats__->blocks.curr.size) \ + cstats__->blocks.max.size = cstats__->blocks.curr.size; \ } while (0) -#define STAT_MBC_BLK_FREE(AP, BSZ) \ +#define STAT_MBC_BLK_FREE(AP, BSZ, FLGS) \ do { \ - ASSERT((AP)->mbcs.blocks.curr.no > 0); \ - (AP)->mbcs.blocks.curr.no--; \ - ASSERT((AP)->mbcs.blocks.curr.size >= (BSZ)); \ - (AP)->mbcs.blocks.curr.size -= (BSZ); \ + CarriersStats_t *cstats__ = (((FLGS) & ERTS_ALCU_FLG_SBMBC) \ + ? &(AP)->sbmbcs \ + : &(AP)->mbcs); \ + ASSERT(cstats__->blocks.curr.no > 0); \ + cstats__->blocks.curr.no--; \ + ASSERT(cstats__->blocks.curr.size >= (BSZ)); \ + cstats__->blocks.curr.size -= (BSZ); \ } while (0) /* Debug stuff... */ @@ -410,7 +447,7 @@ do { \ static UWord carrier_alignment; #define DEBUG_SAVE_ALIGNMENT(C) \ do { \ - UWord algnmnt__ = sizeof(Unit_t) - (((UWord) (C)) % sizeof(Unit_t)); \ + UWord algnmnt__ = sizeof(Unit_t) - (((UWord) (C)) % sizeof(Unit_t));\ carrier_alignment = MIN(carrier_alignment, algnmnt__); \ ASSERT(((UWord) (C)) % sizeof(UWord) == 0); \ } while (0) @@ -524,8 +561,8 @@ static Uint get_next_mbc_size(Allctr_t *allctr) { Uint size; - int cs = (allctr->mbcs.curr_mseg.no - + allctr->mbcs.curr_sys_alloc.no + int cs = (allctr->mbcs.curr.norm.mseg.no + + allctr->mbcs.curr.norm.sys_alloc.no - (allctr->main_carrier ? 1 : 0)); ASSERT(cs >= 0); @@ -609,7 +646,8 @@ unlink_carrier(CarrierList_t *cl, Carrier_t *crr) } } - +static Block_t *create_sbmbc(Allctr_t *allctr, Uint umem_sz); +static void destroy_sbmbc(Allctr_t *allctr, Block_t *blk); static Block_t *create_carrier(Allctr_t *, Uint, UWord); static void destroy_carrier(Allctr_t *, Block_t *); @@ -619,39 +657,57 @@ static void destroy_carrier(Allctr_t *, Block_t *); * block in a sbc. */ static ERTS_INLINE void * -mbc_alloc_block(Allctr_t *allctr, Uint size, Uint *blk_szp) +mbc_alloc_block(Allctr_t *allctr, Uint size, Uint *blk_szp, Uint32 *alcu_flgsp) { Block_t *blk; + Uint get_blk_sz; + Uint sbmbct; ASSERT(size); ASSERT(size < allctr->sbc_threshold); - *blk_szp = UMEMSZ2BLKSZ(allctr, size); + *blk_szp = get_blk_sz = UMEMSZ2BLKSZ(allctr, size); + + sbmbct = allctr->sbmbc_threshold; + if (sbmbct) { + if (get_blk_sz < sbmbct) { + *alcu_flgsp |= ERTS_ALCU_FLG_SBMBC; + if (get_blk_sz + allctr->min_block_size > sbmbct) { + /* Since we use block size to determine if blocks are + located in sbmbc or not... */ + get_blk_sz += allctr->min_block_size; + } + } + } - blk = (*allctr->get_free_block)(allctr, *blk_szp, NULL, 0); + blk = (*allctr->get_free_block)(allctr, get_blk_sz, NULL, 0, *alcu_flgsp); -#if HALFWORD_HEAP if (!blk) { - blk = create_carrier(allctr, *blk_szp, CFLG_MBC|CFLG_FORCE_MSEG); - } + if ((*alcu_flgsp) & ERTS_ALCU_FLG_SBMBC) + blk = create_sbmbc(allctr, get_blk_sz); + else { +#if HALFWORD_HEAP + blk = create_carrier(allctr, get_blk_sz, CFLG_MBC|CFLG_FORCE_MSEG); #else - if (!blk) { - blk = create_carrier(allctr, *blk_szp, CFLG_MBC); - if (!blk) { - /* Emergency! We couldn't create the carrier as we wanted. - Try to place it in a sys_alloced sbc. */ - blk = create_carrier(allctr, - size, - CFLG_SBC|CFLG_FORCE_SIZE|CFLG_FORCE_SYS_ALLOC); + blk = create_carrier(allctr, get_blk_sz, CFLG_MBC); + if (!blk) { + /* Emergency! We couldn't create the carrier as we wanted. + Try to place it in a sys_alloced sbc. */ + blk = create_carrier(allctr, + size, + (CFLG_SBC + | CFLG_FORCE_SIZE + | CFLG_FORCE_SYS_ALLOC)); + } +#endif } } -#endif #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG if (IS_MBC_BLK(blk)) { - (*allctr->link_free_block)(allctr, blk); + (*allctr->link_free_block)(allctr, blk, *alcu_flgsp); HARD_CHECK_BLK_CARRIER(allctr, blk); - (*allctr->unlink_free_block)(allctr, blk); + (*allctr->unlink_free_block)(allctr, blk, *alcu_flgsp); } #endif @@ -664,7 +720,8 @@ mbc_alloc_finalize(Allctr_t *allctr, Uint org_blk_sz, UWord flags, Uint want_blk_sz, - int valid_blk_info) + int valid_blk_info, + Uint32 alcu_flgs) { Uint blk_sz; Uint nxt_blk_sz; @@ -700,7 +757,7 @@ mbc_alloc_finalize(Allctr_t *allctr, SET_PREV_BLK_FREE(nxt_nxt_blk); } } - (*allctr->link_free_block)(allctr, nxt_blk); + (*allctr->link_free_block)(allctr, nxt_blk, alcu_flgs); ASSERT(IS_NOT_LAST_BLK(blk)); ASSERT(IS_FREE_BLK(nxt_blk)); @@ -741,7 +798,7 @@ mbc_alloc_finalize(Allctr_t *allctr, : IS_NOT_LAST_BLK(blk)); } - STAT_MBC_BLK_ALLOC(allctr, blk_sz); + STAT_MBC_BLK_ALLOC(allctr, blk_sz, alcu_flgs); ASSERT(IS_ALLOCED_BLK(blk)); ASSERT(blk_sz == BLK_SZ(blk)); @@ -761,7 +818,8 @@ mbc_alloc(Allctr_t *allctr, Uint size) { Block_t *blk; Uint blk_sz; - blk = mbc_alloc_block(allctr, size, &blk_sz); + Uint32 alcu_flgs = 0; + blk = mbc_alloc_block(allctr, size, &blk_sz, &alcu_flgs); if (!blk) return NULL; if (IS_MBC_BLK(blk)) @@ -770,7 +828,8 @@ mbc_alloc(Allctr_t *allctr, Uint size) BLK_SZ(blk), GET_BLK_HDR_FLGS(blk), blk_sz, - 1); + 1, + alcu_flgs); return BLK2UMEM(blk); } @@ -779,6 +838,7 @@ mbc_free(Allctr_t *allctr, void *p) { Uint is_first_blk; Uint is_last_blk; + Uint32 alcu_flgs = 0; Uint blk_sz; Block_t *blk; Block_t *nxt_blk; @@ -788,13 +848,15 @@ mbc_free(Allctr_t *allctr, void *p) blk = UMEM2BLK(p); blk_sz = BLK_SZ(blk); + if (blk_sz < allctr->sbmbc_threshold) + alcu_flgs |= ERTS_ALCU_FLG_SBMBC; ASSERT(IS_MBC_BLK(blk)); ASSERT(blk_sz >= allctr->min_block_size); HARD_CHECK_BLK_CARRIER(allctr, blk); - STAT_MBC_BLK_FREE(allctr, blk_sz); + STAT_MBC_BLK_FREE(allctr, blk_sz, alcu_flgs); is_first_blk = IS_FIRST_BLK(blk); is_last_blk = IS_LAST_BLK(blk); @@ -802,7 +864,7 @@ mbc_free(Allctr_t *allctr, void *p) if (!is_first_blk && IS_PREV_BLK_FREE(blk)) { /* Coalesce with previous block... */ blk = PREV_BLK(blk); - (*allctr->unlink_free_block)(allctr, blk); + (*allctr->unlink_free_block)(allctr, blk, alcu_flgs); blk_sz += BLK_SZ(blk); is_first_blk = IS_FIRST_BLK(blk); @@ -818,7 +880,7 @@ mbc_free(Allctr_t *allctr, void *p) nxt_blk = NXT_BLK(blk); if (IS_FREE_BLK(nxt_blk)) { /* Coalesce with next block... */ - (*allctr->unlink_free_block)(allctr, nxt_blk); + (*allctr->unlink_free_block)(allctr, nxt_blk, alcu_flgs); blk_sz += BLK_SZ(nxt_blk); SET_BLK_SZ(blk, blk_sz); @@ -850,16 +912,20 @@ mbc_free(Allctr_t *allctr, void *p) if (is_first_blk && is_last_blk - && allctr->main_carrier != FBLK2MBC(allctr, blk)) - destroy_carrier(allctr, blk); + && allctr->main_carrier != FBLK2MBC(allctr, blk)) { + if (alcu_flgs & ERTS_ALCU_FLG_SBMBC) + destroy_sbmbc(allctr, blk); + else + destroy_carrier(allctr, blk); + } else { - (*allctr->link_free_block)(allctr, blk); + (*allctr->link_free_block)(allctr, blk, alcu_flgs); HARD_CHECK_BLK_CARRIER(allctr, blk); } } static void * -mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs) +mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) { void *new_p; Uint old_blk_sz; @@ -867,7 +933,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs) #ifndef MBC_REALLOC_ALWAYS_MOVES Block_t *new_blk, *cand_blk; Uint cand_blk_sz; - Uint blk_sz; + Uint blk_sz, get_blk_sz; Block_t *nxt_blk; Uint nxt_blk_sz; Uint is_last_blk; @@ -883,10 +949,16 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs) ASSERT(old_blk_sz >= allctr->min_block_size); #ifdef MBC_REALLOC_ALWAYS_MOVES - if (flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE) + if (alcu_flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE) return NULL; #else /* !MBC_REALLOC_ALWAYS_MOVES */ - blk_sz = UMEMSZ2BLKSZ(allctr, size); + get_blk_sz = blk_sz = UMEMSZ2BLKSZ(allctr, size); + if ((alcu_flgs & ERTS_ALCU_FLG_SBMBC) + && (blk_sz + allctr->min_block_size > allctr->sbmbc_threshold)) { + /* Since we use block size to determine if blocks are + located in sbmbc or not... */ + get_blk_sz = blk_sz + allctr->min_block_size; + } ASSERT(IS_ALLOCED_BLK(blk)); ASSERT(IS_MBC_BLK(blk)); @@ -901,6 +973,9 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs) Uint diff_sz_val = old_blk_sz - blk_sz; Uint old_blk_sz_val = old_blk_sz; + if (get_blk_sz >= old_blk_sz) + return p; + if (diff_sz_val >= (~((Uint) 0) / 100)) { /* div both by 128 */ old_blk_sz_val >>= 7; @@ -909,7 +984,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs) /* Avoid fragmentation by moving the block if it is shrunk much */ if (100*diff_sz_val > allctr->mbc_move_threshold*old_blk_sz_val) { - if (flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE) + if (alcu_flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE) return NULL; cand_blk_sz = old_blk_sz; @@ -926,9 +1001,10 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs) } new_blk = (*allctr->get_free_block)(allctr, - blk_sz, + get_blk_sz, cand_blk, - cand_blk_sz); + cand_blk_sz, + alcu_flgs); if (new_blk || cand_blk != blk) goto move_into_new_blk; @@ -952,8 +1028,8 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs) nxt_blk_sz, SBH_THIS_FREE|SBH_PREV_ALLOCED|SBH_NOT_LAST_BLK); - STAT_MBC_BLK_FREE(allctr, old_blk_sz); - STAT_MBC_BLK_ALLOC(allctr, blk_sz); + STAT_MBC_BLK_FREE(allctr, old_blk_sz, alcu_flgs); + STAT_MBC_BLK_ALLOC(allctr, blk_sz, alcu_flgs); ASSERT(BLK_SZ(blk) >= allctr->min_block_size); @@ -964,7 +1040,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs) if (IS_FREE_BLK(nxt_nxt_blk)) { /* Coalesce with next free block... */ nxt_blk_sz += BLK_SZ(nxt_nxt_blk); - (*allctr->unlink_free_block)(allctr, nxt_nxt_blk); + (*allctr->unlink_free_block)(allctr, nxt_nxt_blk, alcu_flgs); SET_BLK_SZ(nxt_blk, nxt_blk_sz); is_last_blk = IS_LAST_BLK(nxt_nxt_blk); @@ -979,7 +1055,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs) } } - (*allctr->link_free_block)(allctr, nxt_blk); + (*allctr->link_free_block)(allctr, nxt_blk, alcu_flgs); ASSERT(IS_ALLOCED_BLK(blk)); @@ -1009,12 +1085,12 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs) if (!is_last_blk) { nxt_blk = NXT_BLK(blk); nxt_blk_sz = BLK_SZ(nxt_blk); - if (IS_FREE_BLK(nxt_blk) && blk_sz <= old_blk_sz + nxt_blk_sz) { + if (IS_FREE_BLK(nxt_blk) && get_blk_sz <= old_blk_sz + nxt_blk_sz) { /* Grow into next block... */ HARD_CHECK_BLK_CARRIER(allctr, blk); - (*allctr->unlink_free_block)(allctr, nxt_blk); + (*allctr->unlink_free_block)(allctr, nxt_blk, alcu_flgs); nxt_blk_sz -= blk_sz - old_blk_sz; is_last_blk = IS_LAST_BLK(nxt_blk); @@ -1051,13 +1127,13 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs) else SET_BLK_SZ_FTR(nxt_blk, nxt_blk_sz); - (*allctr->link_free_block)(allctr, nxt_blk); + (*allctr->link_free_block)(allctr, nxt_blk, alcu_flgs); ASSERT(IS_FREE_BLK(nxt_blk)); } - STAT_MBC_BLK_FREE(allctr, old_blk_sz); - STAT_MBC_BLK_ALLOC(allctr, blk_sz); + STAT_MBC_BLK_FREE(allctr, old_blk_sz, alcu_flgs); + STAT_MBC_BLK_ALLOC(allctr, blk_sz, alcu_flgs); ASSERT(IS_ALLOCED_BLK(blk)); @@ -1088,7 +1164,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs) } } - if (flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE) + if (alcu_flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE) return NULL; /* Need to grow in another block */ @@ -1108,7 +1184,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs) } } - if (cand_blk_sz < blk_sz) { + if (cand_blk_sz < get_blk_sz) { /* We wont fit in cand_blk get a new one */ #endif /* !MBC_REALLOC_ALWAYS_MOVES */ @@ -1127,9 +1203,10 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs) /* We will at least fit in cand_blk */ new_blk = (*allctr->get_free_block)(allctr, - blk_sz, + get_blk_sz, cand_blk, - cand_blk_sz); + cand_blk_sz, + alcu_flgs); move_into_new_blk: /* * new_blk, and cand_blk have to be correctly set @@ -1142,7 +1219,8 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs) BLK_SZ(new_blk), GET_BLK_HDR_FLGS(new_blk), blk_sz, - 1); + 1, + alcu_flgs); new_p = BLK2UMEM(new_blk); sys_memcpy(new_p, p, MIN(size, old_blk_sz - ABLK_HDR_SZ)); mbc_free(allctr, p); @@ -1164,7 +1242,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs) HARD_CHECK_BLK_CARRIER(allctr, blk); - (*allctr->unlink_free_block)(allctr, new_blk); /* prev */ + (*allctr->unlink_free_block)(allctr, new_blk, alcu_flgs); /* prev */ if (is_last_blk) new_blk_flgs |= LAST_BLK_HDR_FLG; @@ -1173,7 +1251,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs) if (IS_FREE_BLK(nxt_blk)) { new_blk_flgs |= GET_LAST_BLK_HDR_FLG(nxt_blk); new_blk_sz += BLK_SZ(nxt_blk); - (*allctr->unlink_free_block)(allctr, nxt_blk); + (*allctr->unlink_free_block)(allctr, nxt_blk, alcu_flgs); } } @@ -1196,9 +1274,10 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs) new_blk_sz, new_blk_flgs, blk_sz, - 0); + 0, + alcu_flgs); - STAT_MBC_BLK_FREE(allctr, old_blk_sz); + STAT_MBC_BLK_FREE(allctr, old_blk_sz, alcu_flgs); return new_p; } @@ -1243,6 +1322,100 @@ do { \ #define CHECK_1BLK_CARRIER(A, SBC, MSEGED, C, CSZ, B, BSZ) #endif +static Block_t * +create_sbmbc(Allctr_t *allctr, Uint umem_sz) +{ + Block_t *blk; + Uint blk_sz; + Uint crr_sz = allctr->sbmbc_size; + Carrier_t *crr; + +#if HALFWORD_HEAP + if (allctr->mseg_opt.low_mem) + crr = erts_alloc(ERTS_ALC_T_SBMBC_LOW, crr_sz); + else +#endif + crr = erts_alloc(ERTS_ALC_T_SBMBC, crr_sz); + + INC_CC(allctr->calls.sbmbc_alloc); + SET_CARRIER_HDR(crr, crr_sz, SCH_SYS_ALLOC|SCH_MBC); + + blk = MBC2FBLK(allctr, crr); + +#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG + if (allctr->mbc_header_size % sizeof(Unit_t) == 0) + crr_sz -= sizeof(UWord); +#endif + + blk_sz = UNIT_FLOOR(crr_sz - allctr->mbc_header_size); + + SET_MBC_BLK_FTR(((UWord *) blk)[-1]); + SET_BLK_HDR(blk, blk_sz, SBH_THIS_FREE|SBH_PREV_FREE|SBH_LAST_BLK); + +#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG + *((Carrier_t **) NXT_BLK(blk)) = crr; +#endif + + link_carrier(&allctr->sbmbc_list, crr); + +#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG + if (allctr->mbc_header_size % sizeof(Unit_t) == 0) + crr_sz += sizeof(UWord); +#endif + + STAT_SBMBC_ALLOC(allctr, crr_sz); + CHECK_1BLK_CARRIER(allctr, 0, 0, crr, crr_sz, blk, blk_sz); +#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG + if (allctr->mbc_header_size % sizeof(Unit_t) == 0) + crr_sz -= sizeof(UWord); +#endif + if (allctr->creating_mbc) + (*allctr->creating_mbc)(allctr, crr, ERTS_ALCU_FLG_SBMBC); + + DEBUG_SAVE_ALIGNMENT(crr); + return blk; +} + +static void +destroy_sbmbc(Allctr_t *allctr, Block_t *blk) +{ + Uint crr_sz; + Carrier_t *crr; + + ASSERT(IS_FIRST_BLK(blk)); + + ASSERT(IS_MBC_BLK(blk)); + + crr = FBLK2MBC(allctr, blk); + crr_sz = CARRIER_SZ(crr); + +#ifdef DEBUG + if (!allctr->stopped) { + ASSERT(IS_LAST_BLK(blk)); + +#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG + (*allctr->link_free_block)(allctr, blk, ERTS_ALCU_FLG_SBMBC); + HARD_CHECK_BLK_CARRIER(allctr, blk); + (*allctr->unlink_free_block)(allctr, blk, ERTS_ALCU_FLG_SBMBC); +#endif + } +#endif + + STAT_SBMBC_FREE(allctr, crr_sz); + + unlink_carrier(&allctr->sbmbc_list, crr); + if (allctr->destroying_mbc) + (*allctr->destroying_mbc)(allctr, crr, ERTS_ALCU_FLG_SBMBC); + + INC_CC(allctr->calls.sbmbc_free); + +#if HALFWORD_HEAP + if (allctr->mseg_opt.low_mem) + erts_free(ERTS_ALC_T_SBMBC_LOW, crr); + else +#endif + erts_free(ERTS_ALC_T_SBMBC, crr); +} static Block_t * create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) @@ -1271,11 +1444,11 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) if (erts_mseg_no() >= max_mseg_carriers) goto try_sys_alloc; if (flags & CFLG_SBC) { - if (allctr->sbcs.curr_mseg.no >= allctr->max_mseg_sbcs) + if (allctr->sbcs.curr.norm.mseg.no >= allctr->max_mseg_sbcs) goto try_sys_alloc; } else { - if (allctr->mbcs.curr_mseg.no >= allctr->max_mseg_mbcs) + if (allctr->mbcs.curr.norm.mseg.no >= allctr->max_mseg_mbcs) goto try_sys_alloc; } @@ -1289,7 +1462,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) if (crr_sz < allctr->mbc_header_size + blk_sz) crr_sz = allctr->mbc_header_size + blk_sz; #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG - if (sizeof(Unit_t) == sizeof(UWord)) + if (allctr->mbc_header_size % sizeof(Unit_t) == 0) crr_sz += sizeof(UWord); #endif } @@ -1330,7 +1503,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) && bcrr_sz < allctr->smallest_mbc_size) bcrr_sz = allctr->smallest_mbc_size; #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG - if (sizeof(Unit_t) == sizeof(UWord)) + if (allctr->mbc_header_size % sizeof(Unit_t) == 0) bcrr_sz += sizeof(UWord); #endif @@ -1385,7 +1558,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) blk = MBC2FBLK(allctr, crr); #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG - if (sizeof(Unit_t) == sizeof(UWord)) + if (allctr->mbc_header_size % sizeof(Unit_t) == 0) crr_sz -= sizeof(UWord); #endif @@ -1406,16 +1579,16 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) link_carrier(&allctr->mbc_list, crr); #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG - if (sizeof(Unit_t) == sizeof(UWord)) + if (allctr->mbc_header_size % sizeof(Unit_t) == 0) crr_sz += sizeof(UWord); #endif CHECK_1BLK_CARRIER(allctr, 0, is_mseg, crr, crr_sz, blk, blk_sz); #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG - if (sizeof(Unit_t) == sizeof(UWord)) + if (allctr->mbc_header_size % sizeof(Unit_t) == 0) crr_sz -= sizeof(UWord); #endif if (allctr->creating_mbc) - (*allctr->creating_mbc)(allctr, crr); + (*allctr->creating_mbc)(allctr, crr, 0); } @@ -1595,9 +1768,9 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk) ASSERT(IS_LAST_BLK(blk)); #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG - (*allctr->link_free_block)(allctr, blk); + (*allctr->link_free_block)(allctr, blk, 0); HARD_CHECK_BLK_CARRIER(allctr, blk); - (*allctr->unlink_free_block)(allctr, blk); + (*allctr->unlink_free_block)(allctr, blk, 0); #endif } #endif @@ -1614,7 +1787,7 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk) unlink_carrier(&allctr->mbc_list, crr); if (allctr->destroying_mbc) - (*allctr->destroying_mbc)(allctr, crr); + (*allctr->destroying_mbc)(allctr, crr, 0); } @@ -1658,12 +1831,15 @@ static struct { Eterm lmbcs; Eterm smbcs; Eterm mbcgs; + Eterm sbmbcs; + Eterm sbmbct; #if HAVE_ERTS_MSEG Eterm mmc; #endif Eterm ycs; + /* Eterm sbmbcs; */ Eterm mbcs; Eterm sbcs; Eterm sys_alloc_carriers_size; @@ -1688,6 +1864,8 @@ static struct { Eterm mseg_dealloc; Eterm mseg_realloc; #endif + Eterm sbmbc_alloc; + Eterm sbmbc_free; #ifdef DEBUG Eterm end_of_atoms; #endif @@ -1746,12 +1924,15 @@ init_atoms(Allctr_t *allctr) AM_INIT(lmbcs); AM_INIT(smbcs); AM_INIT(mbcgs); + AM_INIT(sbmbcs); + AM_INIT(sbmbct); #if HAVE_ERTS_MSEG AM_INIT(mmc); #endif AM_INIT(ycs); + /*AM_INIT(sbmbcs);*/ AM_INIT(mbcs); AM_INIT(sbcs); AM_INIT(sys_alloc_carriers_size); @@ -1776,6 +1957,8 @@ init_atoms(Allctr_t *allctr) AM_INIT(mseg_dealloc); AM_INIT(mseg_realloc); #endif + AM_INIT(sbmbc_free); + AM_INIT(sbmbc_alloc); #ifdef DEBUG for (atom = (Eterm *) &am; atom < &am.end_of_atoms; atom++) { @@ -1869,7 +2052,9 @@ sz_info_carriers(Allctr_t *allctr, Uint *szp) { Eterm res = THE_NON_VALUE; - Uint curr_size = cs->curr_mseg.size + cs->curr_sys_alloc.size; + Uint curr_size = (cs == &allctr->sbmbcs + ? cs->curr.small_block.size + : cs->curr.norm.mseg.size + cs->curr.norm.sys_alloc.size); if (print_to_p) { int to = *print_to_p; @@ -1917,8 +2102,17 @@ info_carriers(Allctr_t *allctr, Uint *szp) { Eterm res = THE_NON_VALUE; - Uint curr_no = cs->curr_mseg.no + cs->curr_sys_alloc.no; - Uint curr_size = cs->curr_mseg.size + cs->curr_sys_alloc.size; + Uint curr_no, curr_size; + int small_block = cs == &allctr->sbmbcs; + + if (small_block) { + curr_no = cs->curr.small_block.no; + curr_size = cs->curr.small_block.size; + } + else { + curr_no = cs->curr.norm.mseg.no + cs->curr.norm.sys_alloc.no; + curr_size = cs->curr.norm.mseg.size + cs->curr.norm.sys_alloc.size; + } if (print_to_p) { int to = *print_to_p; @@ -1944,18 +2138,20 @@ info_carriers(Allctr_t *allctr, curr_no, cs->max.no, cs->max_ever.no); + if (!small_block) { #if HAVE_ERTS_MSEG - erts_print(to, - arg, - "%smseg carriers: %bpu\n", - prefix, - cs->curr_mseg.no); -#endif - erts_print(to, - arg, - "%ssys_alloc carriers: %bpu\n", - prefix, - cs->curr_sys_alloc.no); + erts_print(to, + arg, + "%smseg carriers: %bpu\n", + prefix, + cs->curr.norm.mseg.no); +#endif + erts_print(to, + arg, + "%ssys_alloc carriers: %bpu\n", + prefix, + cs->curr.norm.sys_alloc.no); + } erts_print(to, arg, "%scarriers size: %beu %bpu %bpu\n", @@ -1963,43 +2159,49 @@ info_carriers(Allctr_t *allctr, curr_size, cs->max.size, cs->max_ever.size); + if (!small_block) { #if HAVE_ERTS_MSEG - erts_print(to, - arg, - "%smseg carriers size: %bpu\n", - prefix, - cs->curr_mseg.size); -#endif - erts_print(to, - arg, - "%ssys_alloc carriers size: %bpu\n", - prefix, - cs->curr_sys_alloc.size); + erts_print(to, + arg, + "%smseg carriers size: %bpu\n", + prefix, + cs->curr.norm.mseg.size); +#endif + erts_print(to, + arg, + "%ssys_alloc carriers size: %bpu\n", + prefix, + cs->curr.norm.sys_alloc.size); + } } if (hpp || szp) { res = NIL; - add_2tup(hpp, szp, &res, - am.sys_alloc_carriers_size, - bld_unstable_uint(hpp, szp, cs->curr_sys_alloc.size)); + if (!small_block) { + add_2tup(hpp, szp, &res, + am.sys_alloc_carriers_size, + bld_unstable_uint(hpp, szp, cs->curr.norm.sys_alloc.size)); #if HAVE_ERTS_MSEG - add_2tup(hpp, szp, &res, - am.mseg_alloc_carriers_size, - bld_unstable_uint(hpp, szp, cs->curr_mseg.size)); + add_2tup(hpp, szp, &res, + am.mseg_alloc_carriers_size, + bld_unstable_uint(hpp, szp, cs->curr.norm.mseg.size)); #endif + } add_4tup(hpp, szp, &res, am.carriers_size, bld_unstable_uint(hpp, szp, curr_size), bld_unstable_uint(hpp, szp, cs->max.size), bld_unstable_uint(hpp, szp, cs->max_ever.size)); - add_2tup(hpp, szp, &res, - am.sys_alloc_carriers, - bld_unstable_uint(hpp, szp, cs->curr_sys_alloc.no)); + if (!small_block) { + add_2tup(hpp, szp, &res, + am.sys_alloc_carriers, + bld_unstable_uint(hpp, szp, cs->curr.norm.sys_alloc.no)); #if HAVE_ERTS_MSEG - add_2tup(hpp, szp, &res, - am.mseg_alloc_carriers, - bld_unstable_uint(hpp, szp, cs->curr_mseg.no)); + add_2tup(hpp, szp, &res, + am.mseg_alloc_carriers, + bld_unstable_uint(hpp, szp, cs->curr.norm.mseg.no)); #endif + } add_4tup(hpp, szp, &res, am.carriers, bld_unstable_uint(hpp, szp, curr_no), @@ -2077,6 +2279,9 @@ info_calls(Allctr_t *allctr, PRINT_CC_5(to, arg, prefix, "free", allctr->calls.this_free); PRINT_CC_5(to, arg, prefix, "realloc", allctr->calls.this_realloc); + PRINT_CC_4(to, arg, "sbmbc_alloc", allctr->calls.sbmbc_alloc); + PRINT_CC_4(to, arg, "sbmbc_free", allctr->calls.sbmbc_free); + #if HAVE_ERTS_MSEG PRINT_CC_4(to, arg, "mseg_alloc", allctr->calls.mseg_alloc); PRINT_CC_4(to, arg, "mseg_dealloc", allctr->calls.mseg_dealloc); @@ -2128,6 +2333,14 @@ info_calls(Allctr_t *allctr, bld_unstable_uint(hpp, szp, allctr->calls.mseg_alloc.no)); #endif add_3tup(hpp, szp, &res, + am.sbmbc_free, + bld_unstable_uint(hpp, szp, allctr->calls.sbmbc_free.giga_no), + bld_unstable_uint(hpp, szp, allctr->calls.sbmbc_free.no)); + add_3tup(hpp, szp, &res, + am.sbmbc_alloc, + bld_unstable_uint(hpp, szp, allctr->calls.sbmbc_alloc.giga_no), + bld_unstable_uint(hpp, szp, allctr->calls.sbmbc_alloc.no)); + add_3tup(hpp, szp, &res, allctr->name.realloc, bld_unstable_uint(hpp, szp, allctr->calls.this_realloc.giga_no), bld_unstable_uint(hpp, szp, allctr->calls.this_realloc.no)); @@ -2191,7 +2404,9 @@ info_options(Allctr_t *allctr, #endif "option lmbcs: %beu\n" "option smbcs: %beu\n" - "option mbcgs: %beu\n", + "option mbcgs: %beu\n" + "option sbmbcs: %beu\n" + "option sbmbct: %beu\n", topt, allctr->ramv ? "true" : "false", #if HALFWORD_HEAP @@ -2211,7 +2426,9 @@ info_options(Allctr_t *allctr, #endif allctr->largest_mbc_size, allctr->smallest_mbc_size, - allctr->mbc_growth_stages); + allctr->mbc_growth_stages, + allctr->sbmbc_size, + allctr->sbmbc_threshold); } res = (*allctr->info_options)(allctr, "option ", print_to_p, print_to_arg, @@ -2219,6 +2436,12 @@ info_options(Allctr_t *allctr, if (hpp || szp) { add_2tup(hpp, szp, &res, + am.sbmbct, + bld_uint(hpp, szp, allctr->sbmbc_threshold)); + add_2tup(hpp, szp, &res, + am.sbmbcs, + bld_uint(hpp, szp, allctr->sbmbc_size)); + add_2tup(hpp, szp, &res, am.mbcgs, bld_uint(hpp, szp, allctr->mbc_growth_stages)); add_2tup(hpp, szp, &res, @@ -2285,10 +2508,10 @@ update_max_ever_values(CarriersStats_t *cs) static ERTS_INLINE void reset_max_values(CarriersStats_t *cs) { - cs->max.no = cs->curr_mseg.no + cs->curr_sys_alloc.no; - cs->max.size = cs->curr_mseg.size + cs->curr_sys_alloc.size; - cs->blocks.max.no = cs->blocks.curr.no; - cs->blocks.max.size = cs->blocks.curr.size; + cs->max.no = cs->curr.norm.mseg.no + cs->curr.norm.sys_alloc.no; + cs->max.size = cs->curr.norm.mseg.size + cs->curr.norm.sys_alloc.size; + cs->blocks.max.no = cs->blocks.curr.no; + cs->blocks.max.size = cs->blocks.curr.size; } @@ -2367,7 +2590,7 @@ erts_alcu_sz_info(Allctr_t *allctr, Uint **hpp, Uint *szp) { - Eterm res, mbcs, sbcs; + Eterm res, sbmbcs, mbcs, sbcs; res = THE_NON_VALUE; @@ -2389,24 +2612,29 @@ erts_alcu_sz_info(Allctr_t *allctr, /* Update sbc values not continously updated */ allctr->sbcs.blocks.curr.no - = allctr->sbcs.curr_mseg.no + allctr->sbcs.curr_sys_alloc.no; + = allctr->sbcs.curr.norm.mseg.no + allctr->sbcs.curr.norm.sys_alloc.no; allctr->sbcs.blocks.max.no = allctr->sbcs.max.no; + update_max_ever_values(&allctr->sbmbcs); update_max_ever_values(&allctr->mbcs); update_max_ever_values(&allctr->sbcs); - mbcs = sz_info_carriers(allctr, &allctr->mbcs, "mbcs ", print_to_p, - print_to_arg, hpp, szp); - sbcs = sz_info_carriers(allctr, &allctr->sbcs, "sbcs ", print_to_p, - print_to_arg, hpp, szp); + sbmbcs = sz_info_carriers(allctr, &allctr->sbmbcs, "sbmbcs ", print_to_p, + print_to_arg, hpp, szp); + mbcs = sz_info_carriers(allctr, &allctr->mbcs, "mbcs ", print_to_p, + print_to_arg, hpp, szp); + sbcs = sz_info_carriers(allctr, &allctr->sbcs, "sbcs ", print_to_p, + print_to_arg, hpp, szp); if (hpp || szp) { res = NIL; add_2tup(hpp, szp, &res, am.sbcs, sbcs); add_2tup(hpp, szp, &res, am.mbcs, mbcs); + add_2tup(hpp, szp, &res, am.sbmbcs, sbmbcs); } if (begin_max_period) { + reset_max_values(&allctr->sbmbcs); reset_max_values(&allctr->mbcs); reset_max_values(&allctr->sbcs); } @@ -2428,7 +2656,7 @@ erts_alcu_info(Allctr_t *allctr, Uint **hpp, Uint *szp) { - Eterm res, sett, mbcs, sbcs, calls; + Eterm res, sett, sbmbcs, mbcs, sbcs, calls; res = THE_NON_VALUE; @@ -2450,9 +2678,10 @@ erts_alcu_info(Allctr_t *allctr, /* Update sbc values not continously updated */ allctr->sbcs.blocks.curr.no - = allctr->sbcs.curr_mseg.no + allctr->sbcs.curr_sys_alloc.no; + = allctr->sbcs.curr.norm.mseg.no + allctr->sbcs.curr.norm.sys_alloc.no; allctr->sbcs.blocks.max.no = allctr->sbcs.max.no; + update_max_ever_values(&allctr->sbmbcs); update_max_ever_values(&allctr->mbcs); update_max_ever_values(&allctr->sbcs); @@ -2464,11 +2693,13 @@ erts_alcu_info(Allctr_t *allctr, ERTS_ALCU_VSN_STR); } - sett = info_options(allctr, print_to_p, print_to_arg, hpp, szp); - mbcs = info_carriers(allctr, &allctr->mbcs, "mbcs ", print_to_p, - print_to_arg, hpp, szp); - sbcs = info_carriers(allctr, &allctr->sbcs, "sbcs ", print_to_p, - print_to_arg, hpp, szp); + sett = info_options(allctr, print_to_p, print_to_arg, hpp, szp); + sbmbcs = info_carriers(allctr, &allctr->sbmbcs, "sbmbcs ", print_to_p, + print_to_arg, hpp, szp); + mbcs = info_carriers(allctr, &allctr->mbcs, "mbcs ", print_to_p, + print_to_arg, hpp, szp); + sbcs = info_carriers(allctr, &allctr->sbcs, "sbcs ", print_to_p, + print_to_arg, hpp, szp); calls = info_calls(allctr, print_to_p, print_to_arg, hpp, szp); if (hpp || szp) { @@ -2477,6 +2708,7 @@ erts_alcu_info(Allctr_t *allctr, add_2tup(hpp, szp, &res, am.calls, calls); add_2tup(hpp, szp, &res, am.sbcs, sbcs); add_2tup(hpp, szp, &res, am.mbcs, mbcs); + add_2tup(hpp, szp, &res, am.sbmbcs, sbmbcs); add_2tup(hpp, szp, &res, am.options, sett); add_3tup(hpp, szp, &res, am.versions, @@ -2485,6 +2717,7 @@ erts_alcu_info(Allctr_t *allctr, } if (begin_max_period) { + reset_max_values(&allctr->sbmbcs); reset_max_values(&allctr->mbcs); reset_max_values(&allctr->sbcs); } @@ -2508,12 +2741,14 @@ erts_alcu_current_size(Allctr_t *allctr, AllctrSize_t *size) erts_mtx_lock(&allctr->mutex); #endif - size->carriers = allctr->mbcs.curr_mseg.size; - size->carriers += allctr->mbcs.curr_sys_alloc.size; - size->carriers += allctr->sbcs.curr_mseg.size; - size->carriers += allctr->sbcs.curr_sys_alloc.size; + size->carriers = allctr->mbcs.curr.norm.mseg.size; + size->carriers += allctr->mbcs.curr.norm.sys_alloc.size; + size->carriers += allctr->sbmbcs.curr.small_block.size; + size->carriers += allctr->sbcs.curr.norm.mseg.size; + size->carriers += allctr->sbcs.curr.norm.sys_alloc.size; size->blocks = allctr->mbcs.blocks.curr.size; + size->blocks += allctr->sbmbcs.blocks.curr.size; size->blocks += allctr->sbcs.blocks.curr.size; #ifdef USE_THREADS @@ -2725,7 +2960,7 @@ do_erts_alcu_realloc(ErtsAlcType_t type, void *extra, void *p, Uint size, - UWord flgs) + Uint32 alcu_flgs) { Allctr_t *allctr = (Allctr_t *) extra; Block_t *blk; @@ -2758,9 +2993,32 @@ do_erts_alcu_realloc(ErtsAlcType_t type, blk = UMEM2BLK(p); + if (allctr->sbmbc_threshold > 0) { + Uint old_sz, new_sz, lim; + lim = allctr->sbmbc_threshold; + old_sz = BLK_SZ(blk); + new_sz = UMEMSZ2BLKSZ(allctr, size); + if ((old_sz < lim && lim <= new_sz) + || (new_sz < lim && lim <= old_sz)) { + /* *Need* to move it... */ + + INC_CC(allctr->calls.this_realloc); + res = do_erts_alcu_alloc(type, extra, size); + DEC_CC(allctr->calls.this_alloc); + + sys_memcpy(res, p, MIN(size, old_sz - ABLK_HDR_SZ)); + + do_erts_alcu_free(type, extra, p); + DEC_CC(allctr->calls.this_free); + return res; + } + if (old_sz < lim) + alcu_flgs |= ERTS_ALCU_FLG_SBMBC; + } + if (size < allctr->sbc_threshold) { if (IS_MBC_BLK(blk)) - res = mbc_realloc(allctr, p, size, flgs); + res = mbc_realloc(allctr, p, size, alcu_flgs); else { Uint used_sz = allctr->sbc_header_size + ABLK_HDR_SZ + size; Uint crr_sz; @@ -2791,7 +3049,7 @@ do_erts_alcu_realloc(ErtsAlcType_t type, if (100*diff_sz_val < allctr->sbc_move_threshold*crr_sz_val) /* Data won't be copied into a new carrier... */ goto do_carrier_resize; - else if (flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE) + else if (alcu_flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE) return NULL; res = mbc_alloc(allctr, size); @@ -2814,7 +3072,7 @@ do_erts_alcu_realloc(ErtsAlcType_t type, #endif res = new_blk ? BLK2UMEM(new_blk) : NULL; } - else if (flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE) + else if (alcu_flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE) return NULL; else { #if HALFWORD_HEAP @@ -3174,6 +3432,26 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->min_block_size = UNIT_CEILING(allctr->min_block_size + sizeof(UWord)); + + allctr->sbmbc_threshold = init->sbmbct; + + if (!erts_have_sbmbc_alloc + || ERTS_IS_SBMBC_ALLOCATOR_NO__(allctr->alloc_no)) + allctr->sbmbc_threshold = 0; + + if (!allctr->sbmbc_threshold) + allctr->sbmbc_size = 0; + else { + Uint min_size; + allctr->sbmbc_size = init->sbmbcs; + min_size = allctr->sbmbc_threshold; + min_size += allctr->min_block_size; + min_size += allctr->mbc_header_size; + if (allctr->sbmbc_size < min_size) + allctr->sbmbc_size = min_size; + } + + #if HAVE_ERTS_MSEG if (allctr->mseg_opt.abs_shrink_th > ~((UWord) 0) / 100) allctr->mseg_opt.abs_shrink_th = ~((UWord) 0) / 100; @@ -3185,12 +3463,16 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) #ifdef ERTS_ENABLE_LOCK_COUNT erts_mtx_init_x_opt(&allctr->mutex, - "alcu_allocator", - make_small(allctr->alloc_no), - ERTS_LCNT_LT_ALLOC); + ERTS_IS_SBMBC_ALLOCATOR_NO__(allctr->alloc_no) + ? "sbmbc_alloc" + : "alcu_allocator", + make_small(allctr->alloc_no), + ERTS_LCNT_LT_ALLOC); #else erts_mtx_init_x(&allctr->mutex, - "alcu_allocator", + ERTS_IS_SBMBC_ALLOCATOR_NO__(allctr->alloc_no) + ? "sbmbc_alloc" + : "alcu_allocator", make_small(allctr->alloc_no)); #endif /*ERTS_ENABLE_LOCK_COUNT*/ @@ -3260,7 +3542,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) if (!blk) goto error; - (*allctr->link_free_block)(allctr, blk); + (*allctr->link_free_block)(allctr, blk, 0); HARD_CHECK_BLK_CARRIER(allctr, blk); @@ -3290,6 +3572,8 @@ erts_alcu_stop(Allctr_t *allctr) destroy_carrier(allctr, SBC2BLK(allctr, allctr->sbc_list.first)); while (allctr->mbc_list.first) destroy_carrier(allctr, MBC2FBLK(allctr, allctr->mbc_list.first)); + while (allctr->sbmbc_list.first) + destroy_sbmbc(allctr, MBC2FBLK(allctr, allctr->sbmbc_list.first)); #ifdef USE_THREADS if (allctr->thread_safe) @@ -3387,13 +3671,15 @@ erts_alcu_verify_unused(Allctr_t *allctr) { UWord no; - no = allctr->sbcs.curr_mseg.no; - no += allctr->sbcs.curr_sys_alloc.no; + no = allctr->sbcs.curr.norm.mseg.no; + no += allctr->sbcs.curr.norm.sys_alloc.no; no += allctr->mbcs.blocks.curr.no; + no += allctr->sbmbcs.blocks.curr.no; if (no) { UWord sz = allctr->sbcs.blocks.curr.size; sz += allctr->mbcs.blocks.curr.size; + sz += allctr->sbmbcs.blocks.curr.size; erl_exit(ERTS_ABORT_EXIT, "%salloc() used when expected to be unused!\n" "Total amount of blocks allocated: %bpu\n" @@ -3492,7 +3778,7 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk) (*allctr->check_block)(allctr, blk, (int) is_free_blk); if (IS_LAST_BLK(blk)) { - carrier_end = ((char *) NXT_BLK(blk)) + sizeof(UWord); + carrier_end = ((char *) NXT_BLK(blk)); mbc = *((Carrier_t **) NXT_BLK(blk)); prev_blk = NULL; blk = MBC2FBLK(allctr, mbc); @@ -3507,9 +3793,9 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk) ASSERT(IS_MB_CARRIER(mbc)); ASSERT((((char *) mbc) + allctr->mbc_header_size - + tot_blk_sz - + sizeof(UWord)) == carrier_end); - ASSERT(((char *) mbc) + CARRIER_SZ(mbc) == carrier_end); + + tot_blk_sz) == carrier_end); + ASSERT(((char *) mbc) + CARRIER_SZ(mbc) - sizeof(Unit_t) <= carrier_end + && carrier_end <= ((char *) mbc) + CARRIER_SZ(mbc)); if (allctr->check_mbc) (*allctr->check_mbc)(allctr, mbc); @@ -3523,6 +3809,7 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk) cl = &allctr->mbc_list; } +#if 0 /* FIXIT sbmbc */ if (cl->first == crr) { ASSERT(!crr->prev); } @@ -3537,6 +3824,7 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk) ASSERT(crr->next); ASSERT(crr->next->prev == crr); } +#endif } #endif diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index ddf84c086c..fed4d3dbe6 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -34,6 +34,7 @@ typedef struct { typedef struct { char *name_prefix; ErtsAlcType_t alloc_no; + int force; int ts; int tspec; int tpref; @@ -50,6 +51,8 @@ typedef struct { UWord lmbcs; UWord smbcs; UWord mbcgs; + UWord sbmbct; + UWord sbmbcs; } AllctrInit_t; typedef struct { @@ -67,6 +70,7 @@ typedef struct { #define ERTS_DEFAULT_ALLCTR_INIT { \ NULL, \ ERTS_ALC_A_INVALID, /* (number) alloc_no: allocator number */\ + 0, /* (bool) force: force enabled */\ 1, /* (bool) ts: thread safe */\ 0, /* (bool) tspec: thread specific */\ 0, /* (bool) tpref: thread preferred */\ @@ -82,7 +86,9 @@ typedef struct { 10, /* (amount) mmmbc: max mseg mbcs */\ 10*1024*1024, /* (bytes) lmbcs: largest mbc size */\ 1024*1024, /* (bytes) smbcs: smallest mbc size */\ - 10 /* (amount) mbcgs: mbc growth stages */\ + 10, /* (amount) mbcgs: mbc growth stages */\ + 256, /* (bytes) sbmbct: small block mbc threshold */\ + 8*1024 /* (bytes) sbmbcs: small block mbc size */\ } #else /* if SMALL_MEMORY */ @@ -95,6 +101,7 @@ typedef struct { #define ERTS_DEFAULT_ALLCTR_INIT { \ NULL, \ ERTS_ALC_A_INVALID, /* (number) alloc_no: allocator number */\ + 0, /* (bool) force: force enabled */\ 1, /* (bool) ts: thread safe */\ 0, /* (bool) tspec: thread specific */\ 0, /* (bool) tpref: thread preferred */\ @@ -109,7 +116,9 @@ typedef struct { 10, /* (amount) mmmbc: max mseg mbcs */\ 1024*1024, /* (bytes) lmbcs: largest mbc size */\ 128*1024, /* (bytes) smbcs: smallest mbc size */\ - 10 /* (amount) mbcgs: mbc growth stages */\ + 10, /* (amount) mbcgs: mbc growth stages */\ + 256, /* (bytes) sbmbct: small block mbc threshold */\ + 8*1024 /* (bytes) sbmbcs: small block mbc size */\ } #endif @@ -144,6 +153,9 @@ void erts_alcu_current_size(Allctr_t *, AllctrSize_t *); #if defined(GET_ERL_ALLOC_UTIL_IMPL) && !defined(ERL_ALLOC_UTIL_IMPL__) #define ERL_ALLOC_UTIL_IMPL__ +#define ERTS_ALCU_FLG_FAIL_REALLOC_MOVE (((Uint32) 1) << 0) +#define ERTS_ALCU_FLG_SBMBC (((Uint32) 1) << 1) + #ifdef USE_THREADS #define ERL_THREADS_EMU_INTERNAL__ #include "erl_threads.h" @@ -188,6 +200,8 @@ void erts_alcu_current_size(Allctr_t *, AllctrSize_t *); #define CARRIER_SZ(C) \ ((C)->chdr & SZ_MASK) +extern int erts_have_sbmbc_alloc; + typedef union {char c[8]; long l; double d;} Unit_t; typedef struct Carrier_t_ Carrier_t; @@ -216,8 +230,13 @@ typedef struct { } StatValues_t; typedef struct { - StatValues_t curr_mseg; - StatValues_t curr_sys_alloc; + union { + struct { + StatValues_t mseg; + StatValues_t sys_alloc; + } norm; + StatValues_t small_block; + } curr; StatValues_t max; StatValues_t max_ever; struct { @@ -257,6 +276,8 @@ struct Allctr_t_ { Uint largest_mbc_size; Uint smallest_mbc_size; Uint mbc_growth_stages; + Uint sbmbc_threshold; + Uint sbmbc_size; #if HAVE_ERTS_MSEG ErtsMsegOpt_t mseg_opt; #endif @@ -269,6 +290,7 @@ struct Allctr_t_ { Uint min_block_size; /* Carriers */ + CarrierList_t sbmbc_list; CarrierList_t mbc_list; CarrierList_t sbc_list; @@ -277,15 +299,15 @@ struct Allctr_t_ { /* Callback functions (first 4 are mandatory) */ Block_t * (*get_free_block) (Allctr_t *, Uint, - Block_t *, Uint); - void (*link_free_block) (Allctr_t *, Block_t *); - void (*unlink_free_block) (Allctr_t *, Block_t *); + Block_t *, Uint, Uint32); + void (*link_free_block) (Allctr_t *, Block_t *, Uint32); + void (*unlink_free_block) (Allctr_t *, Block_t *, Uint32); Eterm (*info_options) (Allctr_t *, char *, int *, void *, Uint **, Uint *); Uint (*get_next_mbc_size) (Allctr_t *); - void (*creating_mbc) (Allctr_t *, Carrier_t *); - void (*destroying_mbc) (Allctr_t *, Carrier_t *); + void (*creating_mbc) (Allctr_t *, Carrier_t *, Uint32); + void (*destroying_mbc) (Allctr_t *, Carrier_t *, Uint32); void (*init_atoms) (void); #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG @@ -312,6 +334,8 @@ struct Allctr_t_ { CallCounter_t this_alloc; CallCounter_t this_free; CallCounter_t this_realloc; + CallCounter_t sbmbc_alloc; + CallCounter_t sbmbc_free; CallCounter_t mseg_alloc; CallCounter_t mseg_dealloc; CallCounter_t mseg_realloc; @@ -322,6 +346,7 @@ struct Allctr_t_ { CarriersStats_t sbcs; CarriersStats_t mbcs; + CarriersStats_t sbmbcs; #ifdef DEBUG #ifdef USE_THREADS diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.c b/erts/emulator/beam/erl_ao_firstfit_alloc.c new file mode 100644 index 0000000000..002852cdad --- /dev/null +++ b/erts/emulator/beam/erl_ao_firstfit_alloc.c @@ -0,0 +1,972 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2003-2009. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + + +/* + * Description: An "address order first fit" allocator + * based on a Red-Black (binary search) Tree. The search, + * insert, and delete operations are all O(log n) operations + * on a Red-Black Tree. + * Red-Black Trees are described in "Introduction to Algorithms", + * by Thomas H. Cormen, Charles E. Leiserson, and Ronald L. Riverest. + * + * This module is a callback-module for erl_alloc_util.c + * + * Algorithm: The tree nodes are free-blocks ordered in address order. + * Every node also keeps the size of the largest block in its + * sub-tree ('max_size'). By that we can start from root and keep + * left (for low addresses) while dismissing entire sub-trees with + * too small blocks. + * + * Authors: Rickard Green/Sverker Eriksson + */ + + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif +#include "global.h" +#define GET_ERL_AOFF_ALLOC_IMPL +#include "erl_ao_firstfit_alloc.h" + +#ifdef DEBUG +#if 0 +#define HARD_DEBUG +#endif +#else +#undef HARD_DEBUG +#endif + +#define MIN_MBC_SZ (16*1024) +#define MIN_MBC_FIRST_FREE_SZ (4*1024) + +#define TREE_NODE_FLG (((Uint) 1) << 0) +#define RED_FLG (((Uint) 1) << 1) +#ifdef HARD_DEBUG +# define LEFT_VISITED_FLG (((Uint) 1) << 2) +# define RIGHT_VISITED_FLG (((Uint) 1) << 3) +#endif + +#define IS_RED(N) (((AOFF_RBTree_t *) (N)) \ + && ((AOFF_RBTree_t *) (N))->flags & RED_FLG) +#define IS_BLACK(N) (!IS_RED(((AOFF_RBTree_t *) (N)))) + +#define SET_RED(N) (((AOFF_RBTree_t *) (N))->flags |= RED_FLG) +#define SET_BLACK(N) (((AOFF_RBTree_t *) (N))->flags &= ~RED_FLG) + +#undef ASSERT +#define ASSERT ASSERT_EXPR + +#if 1 +#define RBT_ASSERT ASSERT +#else +#define RBT_ASSERT(x) +#endif + + +/* Types... */ +typedef struct AOFF_RBTree_t_ AOFF_RBTree_t; + +struct AOFF_RBTree_t_ { + Block_t hdr; + Uint flags; + AOFF_RBTree_t *parent; + AOFF_RBTree_t *left; + AOFF_RBTree_t *right; + Uint max_sz; /* of all blocks in this sub-tree */ +}; + +#ifdef HARD_DEBUG +static AOFF_RBTree_t * check_tree(AOFF_RBTree_t* root, Uint); +#endif + + +/* Calculate 'max_size' of tree node x by only looking at the direct children + * of x and x itself. + */ +static ERTS_INLINE Uint node_max_size(AOFF_RBTree_t *x) +{ + Uint sz = BLK_SZ(x); + if (x->left && x->left->max_sz > sz) { + sz = x->left->max_sz; + } + if (x->right && x->right->max_sz > sz) { + sz = x->right->max_sz; + } + return sz; +} + +/* Set new possibly lower 'max_size' of node and propagate change toward root +*/ +static ERTS_INLINE void lower_max_size(AOFF_RBTree_t *node, + AOFF_RBTree_t* stop_at) +{ + AOFF_RBTree_t* x = node; + Uint old_max = x->max_sz; + Uint new_max = node_max_size(x); + + if (new_max < old_max) { + x->max_sz = new_max; + while ((x=x->parent) != stop_at && x->max_sz == old_max) { + x->max_sz = node_max_size(x); + } + ASSERT(x == stop_at || x->max_sz > old_max); + } + else ASSERT(new_max == old_max); +} + + +/* Prototypes of callback functions */ +static Block_t* aoff_get_free_block(Allctr_t *, Uint, Block_t *, Uint, Uint32 flags); +static void aoff_link_free_block(Allctr_t *, Block_t*, Uint32 flags); +static void aoff_unlink_free_block(Allctr_t *allctr, Block_t *del, Uint32 flags); + +static Eterm info_options(Allctr_t *, char *, int *, void *, Uint **, Uint *); +static void init_atoms(void); + + + +#ifdef DEBUG + +/* Destroy all tree fields */ +#define DESTROY_TREE_NODE(N) \ + sys_memset((void *) (((Block_t *) (N)) + 1), \ + 0xff, \ + (sizeof(AOFF_RBTree_t) - sizeof(Block_t))) + +#else + +#define DESTROY_TREE_NODE(N) + +#endif + + +static int atoms_initialized = 0; + +void +erts_aoffalc_init(void) +{ + atoms_initialized = 0; +} + +Allctr_t * +erts_aoffalc_start(AOFFAllctr_t *alc, + AOFFAllctrInit_t* aoffinit, + AllctrInit_t *init) +{ + AOFFAllctr_t nulled_state = {{0}}; + /* {{0}} is used instead of {0}, in order to avoid (an incorrect) gcc + warning. gcc warns if {0} is used as initializer of a struct when + the first member is a struct (not if, for example, the third member + is a struct). */ + Allctr_t *allctr = (Allctr_t *) alc; + + sys_memcpy((void *) alc, (void *) &nulled_state, sizeof(AOFFAllctr_t)); + + allctr->mbc_header_size = sizeof(Carrier_t); + allctr->min_mbc_size = MIN_MBC_SZ; + allctr->min_mbc_first_free_size = MIN_MBC_FIRST_FREE_SZ; + allctr->min_block_size = sizeof(AOFF_RBTree_t); + + allctr->vsn_str = ERTS_ALC_AOFF_ALLOC_VSN_STR; + + + /* Callback functions */ + + allctr->get_free_block = aoff_get_free_block; + allctr->link_free_block = aoff_link_free_block; + allctr->unlink_free_block = aoff_unlink_free_block; + allctr->info_options = info_options; + + allctr->get_next_mbc_size = NULL; + allctr->creating_mbc = NULL; + allctr->destroying_mbc = NULL; + allctr->init_atoms = init_atoms; + +#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG + allctr->check_block = NULL; + allctr->check_mbc = NULL; +#endif + + allctr->atoms_initialized = 0; + + if (!erts_alcu_start(allctr, init)) + return NULL; + + return allctr; +} + +/* + * Red-Black Tree operations needed + */ + +static ERTS_INLINE void +left_rotate(AOFF_RBTree_t **root, AOFF_RBTree_t *x) +{ + AOFF_RBTree_t *y = x->right; + x->right = y->left; + if (y->left) + y->left->parent = x; + y->parent = x->parent; + if (!y->parent) { + RBT_ASSERT(*root == x); + *root = y; + } + else if (x == x->parent->left) + x->parent->left = y; + else { + RBT_ASSERT(x == x->parent->right); + x->parent->right = y; + } + y->left = x; + x->parent = y; + + y->max_sz = x->max_sz; + x->max_sz = node_max_size(x); + ASSERT(y->max_sz >= x->max_sz); +} + +static ERTS_INLINE void +right_rotate(AOFF_RBTree_t **root, AOFF_RBTree_t *x) +{ + AOFF_RBTree_t *y = x->left; + x->left = y->right; + if (y->right) + y->right->parent = x; + y->parent = x->parent; + if (!y->parent) { + RBT_ASSERT(*root == x); + *root = y; + } + else if (x == x->parent->right) + x->parent->right = y; + else { + RBT_ASSERT(x == x->parent->left); + x->parent->left = y; + } + y->right = x; + x->parent = y; + y->max_sz = x->max_sz; + x->max_sz = node_max_size(x); + ASSERT(y->max_sz >= x->max_sz); +} + + +/* + * Replace node x with node y + * NOTE: block header of y is not changed + */ +static ERTS_INLINE void +replace(AOFF_RBTree_t **root, AOFF_RBTree_t *x, AOFF_RBTree_t *y) +{ + + if (!x->parent) { + RBT_ASSERT(*root == x); + *root = y; + } + else if (x == x->parent->left) + x->parent->left = y; + else { + RBT_ASSERT(x == x->parent->right); + x->parent->right = y; + } + if (x->left) { + RBT_ASSERT(x->left->parent == x); + x->left->parent = y; + } + if (x->right) { + RBT_ASSERT(x->right->parent == x); + x->right->parent = y; + } + + y->flags = x->flags; + y->parent = x->parent; + y->right = x->right; + y->left = x->left; + + y->max_sz = x->max_sz; + lower_max_size(y, NULL); + DESTROY_TREE_NODE(x); +} + +static void +tree_insert_fixup(AOFF_RBTree_t** root, AOFF_RBTree_t *blk) +{ + AOFF_RBTree_t *x = blk, *y; + + /* + * Rearrange the tree so that it satisfies the Red-Black Tree properties + */ + + RBT_ASSERT(x != *root && IS_RED(x->parent)); + do { + + /* + * x and its parent are both red. Move the red pair up the tree + * until we get to the root or until we can separate them. + */ + + RBT_ASSERT(IS_RED(x)); + RBT_ASSERT(IS_BLACK(x->parent->parent)); + RBT_ASSERT(x->parent->parent); + + if (x->parent == x->parent->parent->left) { + y = x->parent->parent->right; + if (IS_RED(y)) { + SET_BLACK(y); + x = x->parent; + SET_BLACK(x); + x = x->parent; + SET_RED(x); + } + else { + + if (x == x->parent->right) { + x = x->parent; + left_rotate(root, x); + } + + RBT_ASSERT(x == x->parent->parent->left->left); + RBT_ASSERT(IS_RED(x)); + RBT_ASSERT(IS_RED(x->parent)); + RBT_ASSERT(IS_BLACK(x->parent->parent)); + RBT_ASSERT(IS_BLACK(y)); + + SET_BLACK(x->parent); + SET_RED(x->parent->parent); + right_rotate(root, x->parent->parent); + + RBT_ASSERT(x == x->parent->left); + RBT_ASSERT(IS_RED(x)); + RBT_ASSERT(IS_RED(x->parent->right)); + RBT_ASSERT(IS_BLACK(x->parent)); + break; + } + } + else { + RBT_ASSERT(x->parent == x->parent->parent->right); + y = x->parent->parent->left; + if (IS_RED(y)) { + SET_BLACK(y); + x = x->parent; + SET_BLACK(x); + x = x->parent; + SET_RED(x); + } + else { + + if (x == x->parent->left) { + x = x->parent; + right_rotate(root, x); + } + + RBT_ASSERT(x == x->parent->parent->right->right); + RBT_ASSERT(IS_RED(x)); + RBT_ASSERT(IS_RED(x->parent)); + RBT_ASSERT(IS_BLACK(x->parent->parent)); + RBT_ASSERT(IS_BLACK(y)); + + SET_BLACK(x->parent); + SET_RED(x->parent->parent); + left_rotate(root, x->parent->parent); + + RBT_ASSERT(x == x->parent->right); + RBT_ASSERT(IS_RED(x)); + RBT_ASSERT(IS_RED(x->parent->left)); + RBT_ASSERT(IS_BLACK(x->parent)); + break; + } + } + } while (x != *root && IS_RED(x->parent)); + + SET_BLACK(*root); +} + +static void +aoff_unlink_free_block(Allctr_t *allctr, Block_t *del, Uint32 flags) +{ + AOFFAllctr_t *alc = (AOFFAllctr_t *) allctr; + AOFF_RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC) + ? &alc->sbmbc_root : &alc->mbc_root); + Uint spliced_is_black; + AOFF_RBTree_t *x, *y, *z = (AOFF_RBTree_t *) del; + AOFF_RBTree_t null_x; /* null_x is used to get the fixup started when we + splice out a node without children. */ + + null_x.parent = NULL; + +#ifdef HARD_DEBUG + check_tree(*root, 0); +#endif + + /* Remove node from tree... */ + + /* Find node to splice out */ + if (!z->left || !z->right) + y = z; + else + /* Set y to z:s successor */ + for(y = z->right; y->left; y = y->left); + /* splice out y */ + x = y->left ? y->left : y->right; + spliced_is_black = IS_BLACK(y); + if (x) { + x->parent = y->parent; + } + else if (spliced_is_black) { + x = &null_x; + x->flags = 0; + SET_BLACK(x); + x->right = x->left = NULL; + x->max_sz = 0; + x->parent = y->parent; + y->left = x; + } + + if (!y->parent) { + RBT_ASSERT(*root == y); + *root = x; + } + else { + if (y == y->parent->left) { + y->parent->left = x; + } + else { + RBT_ASSERT(y == y->parent->right); + y->parent->right = x; + } + if (y->parent != z) { + lower_max_size(y->parent, (y==z ? NULL : z)); + } + } + if (y != z) { + /* We spliced out the successor of z; replace z by the successor */ + replace(root, z, y); + } + + if (spliced_is_black) { + /* We removed a black node which makes the resulting tree + violate the Red-Black Tree properties. Fixup tree... */ + + while (IS_BLACK(x) && x->parent) { + + /* + * x has an "extra black" which we move up the tree + * until we reach the root or until we can get rid of it. + * + * y is the sibbling of x + */ + + if (x == x->parent->left) { + y = x->parent->right; + RBT_ASSERT(y); + if (IS_RED(y)) { + RBT_ASSERT(y->right); + RBT_ASSERT(y->left); + SET_BLACK(y); + RBT_ASSERT(IS_BLACK(x->parent)); + SET_RED(x->parent); + left_rotate(root, x->parent); + y = x->parent->right; + } + RBT_ASSERT(y); + RBT_ASSERT(IS_BLACK(y)); + if (IS_BLACK(y->left) && IS_BLACK(y->right)) { + SET_RED(y); + x = x->parent; + } + else { + if (IS_BLACK(y->right)) { + SET_BLACK(y->left); + SET_RED(y); + right_rotate(root, y); + y = x->parent->right; + } + RBT_ASSERT(y); + if (IS_RED(x->parent)) { + + SET_BLACK(x->parent); + SET_RED(y); + } + RBT_ASSERT(y->right); + SET_BLACK(y->right); + left_rotate(root, x->parent); + x = *root; + break; + } + } + else { + RBT_ASSERT(x == x->parent->right); + y = x->parent->left; + RBT_ASSERT(y); + if (IS_RED(y)) { + RBT_ASSERT(y->right); + RBT_ASSERT(y->left); + SET_BLACK(y); + RBT_ASSERT(IS_BLACK(x->parent)); + SET_RED(x->parent); + right_rotate(root, x->parent); + y = x->parent->left; + } + RBT_ASSERT(y); + RBT_ASSERT(IS_BLACK(y)); + if (IS_BLACK(y->right) && IS_BLACK(y->left)) { + SET_RED(y); + x = x->parent; + } + else { + if (IS_BLACK(y->left)) { + SET_BLACK(y->right); + SET_RED(y); + left_rotate(root, y); + y = x->parent->left; + } + RBT_ASSERT(y); + if (IS_RED(x->parent)) { + SET_BLACK(x->parent); + SET_RED(y); + } + RBT_ASSERT(y->left); + SET_BLACK(y->left); + right_rotate(root, x->parent); + x = *root; + break; + } + } + } + SET_BLACK(x); + + if (null_x.parent) { + if (null_x.parent->left == &null_x) + null_x.parent->left = NULL; + else { + RBT_ASSERT(null_x.parent->right == &null_x); + null_x.parent->right = NULL; + } + RBT_ASSERT(!null_x.left); + RBT_ASSERT(!null_x.right); + } + else if (*root == &null_x) { + *root = NULL; + RBT_ASSERT(!null_x.left); + RBT_ASSERT(!null_x.right); + } + } + + DESTROY_TREE_NODE(del); + +#ifdef HARD_DEBUG + check_tree(*root, 0); +#endif +} + +static void +aoff_link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) +{ + AOFFAllctr_t *alc = (AOFFAllctr_t *) allctr; + AOFF_RBTree_t *blk = (AOFF_RBTree_t *) block; + AOFF_RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC) + ? &alc->sbmbc_root : &alc->mbc_root); + Uint blk_sz = BLK_SZ(blk); + +#ifdef HARD_DEBUG + check_tree(*root, 0); +#endif + + blk->flags = 0; + blk->left = NULL; + blk->right = NULL; + blk->max_sz = blk_sz; + + if (!*root) { + blk->parent = NULL; + SET_BLACK(blk); + *root = blk; + } + else { + AOFF_RBTree_t *x = *root; + while (1) { + if (x->max_sz < blk_sz) { + x->max_sz = blk_sz; + } + if (blk < x) { + if (!x->left) { + blk->parent = x; + x->left = blk; + break; + } + x = x->left; + } + else { + if (!x->right) { + blk->parent = x; + x->right = blk; + break; + } + x = x->right; + } + + } + + /* Insert block into size tree */ + RBT_ASSERT(blk->parent); + + SET_RED(blk); + if (IS_RED(blk->parent)) + tree_insert_fixup(root, blk); + } + +#ifdef HARD_DEBUG + check_tree(*root, 0); +#endif +} + +static Block_t * +aoff_get_free_block(Allctr_t *allctr, Uint size, + Block_t *cand_blk, Uint cand_size, Uint32 flags) +{ + AOFFAllctr_t *alc = (AOFFAllctr_t *) allctr; + AOFF_RBTree_t *x = ((flags & ERTS_ALCU_FLG_SBMBC) + ? alc->sbmbc_root : alc->mbc_root); + AOFF_RBTree_t *blk = NULL; +#ifdef HARD_DEBUG + AOFF_RBTree_t* dbg_blk = check_tree(x, size); +#endif + + ASSERT(!cand_blk || cand_size >= size); + + while (x) { + if (x->left && x->left->max_sz >= size) { + x = x->left; + } + else if (BLK_SZ(x) >= size) { + blk = x; + break; + } + else { + x = x->right; + } + } + +#ifdef HARD_DEBUG + ASSERT(blk == dbg_blk); +#endif + + if (!blk) + return NULL; + + if (cand_blk && cand_blk < &blk->hdr) { + return NULL; /* cand_blk was better */ + } + + aoff_unlink_free_block(allctr, (Block_t *) blk, flags); + + return (Block_t *) blk; +} + + +/* + * info_options() + */ + +static struct { + Eterm as; + Eterm aoff; +#ifdef DEBUG + Eterm end_of_atoms; +#endif +} am; + +static void ERTS_INLINE atom_init(Eterm *atom, char *name) +{ + *atom = am_atom_put(name, strlen(name)); +} +#define AM_INIT(AM) atom_init(&am.AM, #AM) + +static void +init_atoms(void) +{ +#ifdef DEBUG + Eterm *atom; +#endif + + if (atoms_initialized) + return; + +#ifdef DEBUG + for (atom = (Eterm *) &am; atom <= &am.end_of_atoms; atom++) { + *atom = THE_NON_VALUE; + } +#endif + AM_INIT(as); + AM_INIT(aoff); + +#ifdef DEBUG + for (atom = (Eterm *) &am; atom < &am.end_of_atoms; atom++) { + ASSERT(*atom != THE_NON_VALUE); + } +#endif + + atoms_initialized = 1; +} + + +#define bld_uint erts_bld_uint +#define bld_cons erts_bld_cons +#define bld_tuple erts_bld_tuple + +static ERTS_INLINE void +add_2tup(Uint **hpp, Uint *szp, Eterm *lp, Eterm el1, Eterm el2) +{ + *lp = bld_cons(hpp, szp, bld_tuple(hpp, szp, 2, el1, el2), *lp); +} + +static Eterm +info_options(Allctr_t *allctr, + char *prefix, + int *print_to_p, + void *print_to_arg, + Uint **hpp, + Uint *szp) +{ + Eterm res = THE_NON_VALUE; + + if (print_to_p) { + erts_print(*print_to_p, + print_to_arg, + "%sas: %s\n", + prefix, + "aoff"); + } + + if (hpp || szp) { + + if (!atoms_initialized) + erl_exit(1, "%s:%d: Internal error: Atoms not initialized", + __FILE__, __LINE__);; + + res = NIL; + add_2tup(hpp, szp, &res, am.as, am.aoff); + } + + return res; +} + + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ + * NOTE: erts_aoffalc_test() is only supposed to be used for testing. * + * * + * Keep alloc_SUITE_data/allocator_test.h updated if changes are made * + * to erts_aoffalc_test() * +\* */ + +unsigned long +erts_aoffalc_test(unsigned long op, unsigned long a1, unsigned long a2) +{ + switch (op) { + case 0x500: return (unsigned long) 0; /* IS_AOBF */ + case 0x501: return (unsigned long) ((AOFFAllctr_t *) a1)->mbc_root; + case 0x502: return (unsigned long) ((AOFF_RBTree_t *) a1)->parent; + case 0x503: return (unsigned long) ((AOFF_RBTree_t *) a1)->left; + case 0x504: return (unsigned long) ((AOFF_RBTree_t *) a1)->right; + case 0x506: return (unsigned long) IS_BLACK((AOFF_RBTree_t *) a1); + case 0x508: return (unsigned long) 1; /* IS_AOFF */ + case 0x509: return (unsigned long) ((AOFF_RBTree_t *) a1)->max_sz; + default: ASSERT(0); return ~((unsigned long) 0); + } +} + + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ + * Debug functions * +\* */ + + +#ifdef HARD_DEBUG + +#define IS_LEFT_VISITED(FB) ((FB)->flags & LEFT_VISITED_FLG) +#define IS_RIGHT_VISITED(FB) ((FB)->flags & RIGHT_VISITED_FLG) + +#define SET_LEFT_VISITED(FB) ((FB)->flags |= LEFT_VISITED_FLG) +#define SET_RIGHT_VISITED(FB) ((FB)->flags |= RIGHT_VISITED_FLG) + +#define UNSET_LEFT_VISITED(FB) ((FB)->flags &= ~LEFT_VISITED_FLG) +#define UNSET_RIGHT_VISITED(FB) ((FB)->flags &= ~RIGHT_VISITED_FLG) + + +#if 0 +# define PRINT_TREE +#else +# undef PRINT_TREE +#endif + +#ifdef PRINT_TREE +static void print_tree(AOFF_RBTree_t*); +#endif + +/* + * Checks that the order between parent and children are correct, + * and that the Red-Black Tree properies are satisfied. if size > 0, + * check_tree() returns the node that satisfies "address order first fit" + * + * The Red-Black Tree properies are: + * 1. Every node is either red or black. + * 2. Every leaf (NIL) is black. + * 3. If a node is red, then both its children are black. + * 4. Every simple path from a node to a descendant leaf + * contains the same number of black nodes. + * + * + own.max_size == MAX(own.size, left.max_size, right.max_size) + */ + +static AOFF_RBTree_t * +check_tree(AOFF_RBTree_t* root, Uint size) +{ + AOFF_RBTree_t *res = NULL; + Sint blacks; + Sint curr_blacks; + AOFF_RBTree_t *x; + +#ifdef PRINT_TREE + print_tree(root); +#endif + + if (!root) + return res; + + x = root; + ASSERT(IS_BLACK(x)); + ASSERT(!x->parent); + curr_blacks = 1; + blacks = -1; + + while (x) { + if (!IS_LEFT_VISITED(x)) { + SET_LEFT_VISITED(x); + if (x->left) { + x = x->left; + if (IS_BLACK(x)) + curr_blacks++; + continue; + } + else { + if (blacks < 0) + blacks = curr_blacks; + ASSERT(blacks == curr_blacks); + } + } + + if (!IS_RIGHT_VISITED(x)) { + SET_RIGHT_VISITED(x); + if (x->right) { + x = x->right; + if (IS_BLACK(x)) + curr_blacks++; + continue; + } + else { + if (blacks < 0) + blacks = curr_blacks; + ASSERT(blacks == curr_blacks); + } + } + + + if (IS_RED(x)) { + ASSERT(IS_BLACK(x->right)); + ASSERT(IS_BLACK(x->left)); + } + + ASSERT(x->parent || x == root); + + if (x->left) { + ASSERT(x->left->parent == x); + ASSERT(x->left < x); + ASSERT(x->left->max_sz <= x->max_sz); + } + + if (x->right) { + ASSERT(x->right->parent == x); + ASSERT(x->right > x); + ASSERT(x->right->max_sz <= x->max_sz); + } + ASSERT(x->max_sz >= BLK_SZ(x)); + ASSERT(x->max_sz == BLK_SZ(x) + || x->max_sz == (x->left ? x->left->max_sz : 0) + || x->max_sz == (x->right ? x->right->max_sz : 0)); + + if (size && BLK_SZ(x) >= size) { + if (!res || x < res) { + res = x; + } + } + + UNSET_LEFT_VISITED(x); + UNSET_RIGHT_VISITED(x); + if (IS_BLACK(x)) + curr_blacks--; + x = x->parent; + + } + + ASSERT(curr_blacks == 0); + + UNSET_LEFT_VISITED(root); + UNSET_RIGHT_VISITED(root); + + return res; + +} + + +#ifdef PRINT_TREE +#define INDENT_STEP 2 + +#include <stdio.h> + +static void +print_tree_aux(AOFF_RBTree_t *x, int indent) +{ + int i; + + if (x) { + print_tree_aux(x->right, indent + INDENT_STEP); + for (i = 0; i < indent; i++) { + putc(' ', stderr); + } + fprintf(stderr, "%s: sz=%lu addr=0x%lx max_size=%lu\r\n", + IS_BLACK(x) ? "BLACK" : "RED", + BLK_SZ(x), (Uint)x, x->max_sz); + print_tree_aux(x->left, indent + INDENT_STEP); + } +} + + +static void +print_tree(AOFF_RBTree_t* root) +{ + fprintf(stderr, " --- AOFF tree begin ---\r\n"); + print_tree_aux(root, 0); + fprintf(stderr, " --- AOFF tree end ---\r\n"); +} + +#endif /* PRINT_TREE */ + +#endif /* HARD_DEBUG */ + diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.h b/erts/emulator/beam/erl_ao_firstfit_alloc.h new file mode 100644 index 0000000000..0bf0ec8cee --- /dev/null +++ b/erts/emulator/beam/erl_ao_firstfit_alloc.h @@ -0,0 +1,60 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2003-2009. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + + +#ifndef ERL_AO_FIRSTFIT_ALLOC__ +#define ERL_AO_FIRSTFIT_ALLOC__ + +#include "erl_alloc_util.h" + +#define ERTS_ALC_AOFF_ALLOC_VSN_STR "0.9" + +typedef struct AOFFAllctr_t_ AOFFAllctr_t; + +typedef struct { + int dummy; +} AOFFAllctrInit_t; + +#define ERTS_DEFAULT_AOFF_ALLCTR_INIT {0/*dummy*/} + +void erts_aoffalc_init(void); +Allctr_t *erts_aoffalc_start(AOFFAllctr_t *, AOFFAllctrInit_t*, AllctrInit_t *); + +#endif /* #ifndef ERL_AO_FIRSTFIT_ALLOC__ */ + + + +#if defined(GET_ERL_AOFF_ALLOC_IMPL) && !defined(ERL_AOFF_ALLOC_IMPL__) +#define ERL_AOFF_ALLOC_IMPL__ + +#define GET_ERL_ALLOC_UTIL_IMPL +#include "erl_alloc_util.h" + + +struct AOFFAllctr_t_ { + Allctr_t allctr; /* Has to be first! */ + + struct AOFF_RBTree_t_* mbc_root; + struct AOFF_RBTree_t_* sbmbc_root; +}; + +unsigned long erts_aoffalc_test(unsigned long, unsigned long, unsigned long); + +#endif /* #if defined(GET_ERL_AOFF_ALLOC_IMPL) + && !defined(ERL_AOFF_ALLOC_IMPL__) */ diff --git a/erts/emulator/beam/erl_bestfit_alloc.c b/erts/emulator/beam/erl_bestfit_alloc.c index 3035e5df16..5e3032ddaa 100644 --- a/erts/emulator/beam/erl_bestfit_alloc.c +++ b/erts/emulator/beam/erl_bestfit_alloc.c @@ -84,24 +84,24 @@ #ifdef HARD_DEBUG -static RBTree_t * check_tree(BFAllctr_t *, Uint); +static RBTree_t * check_tree(RBTree_t, int, Uint); #endif -static void tree_delete(Allctr_t *allctr, Block_t *del); +static void tree_delete(Allctr_t *allctr, Block_t *del, Uint32 flags); /* Prototypes of callback functions */ /* "address order best fit" specific callback functions */ static Block_t * aobf_get_free_block (Allctr_t *, Uint, - Block_t *, Uint); -static void aobf_link_free_block (Allctr_t *, Block_t *); + Block_t *, Uint, Uint32); +static void aobf_link_free_block (Allctr_t *, Block_t *, Uint32); #define aobf_unlink_free_block tree_delete /* "best fit" specific callback functions */ static Block_t * bf_get_free_block (Allctr_t *, Uint, - Block_t *, Uint); -static void bf_link_free_block (Allctr_t *, Block_t *); -static ERTS_INLINE void bf_unlink_free_block (Allctr_t *, Block_t *); + Block_t *, Uint, Uint32); +static void bf_link_free_block (Allctr_t *, Block_t *, Uint32); +static ERTS_INLINE void bf_unlink_free_block (Allctr_t *, Block_t *, Uint32); static Eterm info_options (Allctr_t *, char *, int *, @@ -303,7 +303,7 @@ replace(RBTree_t **root, RBTree_t *x, RBTree_t *y) } static void -tree_insert_fixup(BFAllctr_t *bfallctr, RBTree_t *blk) +tree_insert_fixup(RBTree_t **root, RBTree_t *blk) { RBTree_t *x = blk, *y; @@ -311,7 +311,7 @@ tree_insert_fixup(BFAllctr_t *bfallctr, RBTree_t *blk) * Rearrange the tree so that it satisfies the Red-Black Tree properties */ - RBT_ASSERT(x != bfallctr->root && IS_RED(x->parent)); + RBT_ASSERT(x != *root && IS_RED(x->parent)); do { /* @@ -336,7 +336,7 @@ tree_insert_fixup(BFAllctr_t *bfallctr, RBTree_t *blk) if (x == x->parent->right) { x = x->parent; - left_rotate(&bfallctr->root, x); + left_rotate(root, x); } RBT_ASSERT(x == x->parent->parent->left->left); @@ -347,7 +347,7 @@ tree_insert_fixup(BFAllctr_t *bfallctr, RBTree_t *blk) SET_BLACK(x->parent); SET_RED(x->parent->parent); - right_rotate(&bfallctr->root, x->parent->parent); + right_rotate(root, x->parent->parent); RBT_ASSERT(x == x->parent->left); RBT_ASSERT(IS_RED(x)); @@ -370,7 +370,7 @@ tree_insert_fixup(BFAllctr_t *bfallctr, RBTree_t *blk) if (x == x->parent->left) { x = x->parent; - right_rotate(&bfallctr->root, x); + right_rotate(root, x); } RBT_ASSERT(x == x->parent->parent->right->right); @@ -381,7 +381,7 @@ tree_insert_fixup(BFAllctr_t *bfallctr, RBTree_t *blk) SET_BLACK(x->parent); SET_RED(x->parent->parent); - left_rotate(&bfallctr->root, x->parent->parent); + left_rotate(root, x->parent->parent); RBT_ASSERT(x == x->parent->right); RBT_ASSERT(IS_RED(x)); @@ -390,9 +390,9 @@ tree_insert_fixup(BFAllctr_t *bfallctr, RBTree_t *blk) break; } } - } while (x != bfallctr->root && IS_RED(x->parent)); + } while (x != *root && IS_RED(x->parent)); - SET_BLACK(bfallctr->root); + SET_BLACK(*root); } @@ -402,18 +402,22 @@ tree_insert_fixup(BFAllctr_t *bfallctr, RBTree_t *blk) * callback function in the address order case. */ static void -tree_delete(Allctr_t *allctr, Block_t *del) +tree_delete(Allctr_t *allctr, Block_t *del, Uint32 flags) { BFAllctr_t *bfallctr = (BFAllctr_t *) allctr; Uint spliced_is_black; + RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC) + ? &bfallctr->sbmbc_root + : &bfallctr->mbc_root); RBTree_t *x, *y, *z = (RBTree_t *) del; RBTree_t null_x; /* null_x is used to get the fixup started when we splice out a node without children. */ null_x.parent = NULL; + #ifdef HARD_DEBUG - check_tree(bfallctr, 0); + check_tree(*root, bfallctr->address_order, 0); #endif /* Remove node from tree... */ @@ -440,8 +444,8 @@ tree_delete(Allctr_t *allctr, Block_t *del) } if (!y->parent) { - RBT_ASSERT(bfallctr->root == y); - bfallctr->root = x; + RBT_ASSERT(*root == y); + *root = x; } else if (y == y->parent->left) y->parent->left = x; @@ -451,7 +455,7 @@ tree_delete(Allctr_t *allctr, Block_t *del) } if (y != z) { /* We spliced out the successor of z; replace z by the successor */ - replace(&bfallctr->root, z, y); + replace(root, z, y); } if (spliced_is_black) { @@ -476,7 +480,7 @@ tree_delete(Allctr_t *allctr, Block_t *del) SET_BLACK(y); RBT_ASSERT(IS_BLACK(x->parent)); SET_RED(x->parent); - left_rotate(&bfallctr->root, x->parent); + left_rotate(root, x->parent); y = x->parent->right; } RBT_ASSERT(y); @@ -489,7 +493,7 @@ tree_delete(Allctr_t *allctr, Block_t *del) if (IS_BLACK(y->right)) { SET_BLACK(y->left); SET_RED(y); - right_rotate(&bfallctr->root, y); + right_rotate(root, y); y = x->parent->right; } RBT_ASSERT(y); @@ -500,8 +504,8 @@ tree_delete(Allctr_t *allctr, Block_t *del) } RBT_ASSERT(y->right); SET_BLACK(y->right); - left_rotate(&bfallctr->root, x->parent); - x = bfallctr->root; + left_rotate(root, x->parent); + x = *root; break; } } @@ -515,7 +519,7 @@ tree_delete(Allctr_t *allctr, Block_t *del) SET_BLACK(y); RBT_ASSERT(IS_BLACK(x->parent)); SET_RED(x->parent); - right_rotate(&bfallctr->root, x->parent); + right_rotate(root, x->parent); y = x->parent->left; } RBT_ASSERT(y); @@ -528,7 +532,7 @@ tree_delete(Allctr_t *allctr, Block_t *del) if (IS_BLACK(y->left)) { SET_BLACK(y->right); SET_RED(y); - left_rotate(&bfallctr->root, y); + left_rotate(root, y); y = x->parent->left; } RBT_ASSERT(y); @@ -538,8 +542,8 @@ tree_delete(Allctr_t *allctr, Block_t *del) } RBT_ASSERT(y->left); SET_BLACK(y->left); - right_rotate(&bfallctr->root, x->parent); - x = bfallctr->root; + right_rotate(root, x->parent); + x = *root; break; } } @@ -556,8 +560,8 @@ tree_delete(Allctr_t *allctr, Block_t *del) RBT_ASSERT(!null_x.left); RBT_ASSERT(!null_x.right); } - else if (bfallctr->root == &null_x) { - bfallctr->root = NULL; + else if (*root == &null_x) { + *root = NULL; RBT_ASSERT(!null_x.left); RBT_ASSERT(!null_x.right); } @@ -567,7 +571,7 @@ tree_delete(Allctr_t *allctr, Block_t *del) DESTROY_TREE_NODE(del); #ifdef HARD_DEBUG - check_tree(bfallctr, 0); + check_tree(root, bfallctr->address_order, 0); #endif } @@ -577,23 +581,28 @@ tree_delete(Allctr_t *allctr, Block_t *del) \* */ static void -aobf_link_free_block(Allctr_t *allctr, Block_t *block) +aobf_link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) { BFAllctr_t *bfallctr = (BFAllctr_t *) allctr; + RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC) + ? &bfallctr->sbmbc_root + : &bfallctr->mbc_root); RBTree_t *blk = (RBTree_t *) block; Uint blk_sz = BLK_SZ(blk); + + blk->flags = 0; blk->left = NULL; blk->right = NULL; - if (!bfallctr->root) { + if (!*root) { blk->parent = NULL; SET_BLACK(blk); - bfallctr->root = blk; + *root = blk; } else { - RBTree_t *x = bfallctr->root; + RBTree_t *x = *root; while (1) { Uint size; @@ -623,28 +632,32 @@ aobf_link_free_block(Allctr_t *allctr, Block_t *block) SET_RED(blk); if (IS_RED(blk->parent)) - tree_insert_fixup(bfallctr, blk); + tree_insert_fixup(root, blk); } #ifdef HARD_DEBUG - check_tree(bfallctr, 0); + check_tree(root, 1, 0); #endif } #if 0 /* tree_delete() is directly used instead */ static void -aobf_unlink_free_block(Allctr_t *allctr, Block_t *block) +aobf_unlink_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) { - tree_delete(allctr, block); + tree_delete(allctr, block, flags); } #endif static Block_t * aobf_get_free_block(Allctr_t *allctr, Uint size, - Block_t *cand_blk, Uint cand_size) + Block_t *cand_blk, Uint cand_size, + Uint32 flags) { BFAllctr_t *bfallctr = (BFAllctr_t *) allctr; - RBTree_t *x = bfallctr->root; + RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC) + ? &bfallctr->sbmbc_root + : &bfallctr->mbc_root); + RBTree_t *x = *root; RBTree_t *blk = NULL; Uint blk_sz; @@ -665,7 +678,7 @@ aobf_get_free_block(Allctr_t *allctr, Uint size, return NULL; #ifdef HARD_DEBUG - ASSERT(blk == check_tree(bfallctr, size)); + ASSERT(blk == check_tree(root, 1, size)); #endif if (cand_blk) { @@ -676,7 +689,7 @@ aobf_get_free_block(Allctr_t *allctr, Uint size, return NULL; /* cand_blk was better */ } - aobf_unlink_free_block(allctr, (Block_t *) blk); + aobf_unlink_free_block(allctr, (Block_t *) blk, flags); return (Block_t *) blk; } @@ -687,9 +700,12 @@ aobf_get_free_block(Allctr_t *allctr, Uint size, \* */ static void -bf_link_free_block(Allctr_t *allctr, Block_t *block) +bf_link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) { BFAllctr_t *bfallctr = (BFAllctr_t *) allctr; + RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC) + ? &bfallctr->sbmbc_root + : &bfallctr->mbc_root); RBTree_t *blk = (RBTree_t *) block; Uint blk_sz = BLK_SZ(blk); @@ -700,13 +716,13 @@ bf_link_free_block(Allctr_t *allctr, Block_t *block) blk->left = NULL; blk->right = NULL; - if (!bfallctr->root) { + if (!*root) { blk->parent = NULL; SET_BLACK(blk); - bfallctr->root = blk; + *root = blk; } else { - RBTree_t *x = bfallctr->root; + RBTree_t *x = *root; while (1) { Uint size; @@ -745,7 +761,7 @@ bf_link_free_block(Allctr_t *allctr, Block_t *block) SET_RED(blk); if (IS_RED(blk->parent)) - tree_insert_fixup(bfallctr, blk); + tree_insert_fixup(root, blk); } @@ -753,14 +769,17 @@ bf_link_free_block(Allctr_t *allctr, Block_t *block) LIST_NEXT(blk) = NULL; #ifdef HARD_DEBUG - check_tree(bfallctr, 0); + check_tree(root, 0, 0); #endif } static ERTS_INLINE void -bf_unlink_free_block(Allctr_t *allctr, Block_t *block) +bf_unlink_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) { BFAllctr_t *bfallctr = (BFAllctr_t *) allctr; + RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC) + ? &bfallctr->sbmbc_root + : &bfallctr->mbc_root); RBTree_t *x = (RBTree_t *) block; if (IS_LIST_ELEM(x)) { @@ -778,9 +797,9 @@ bf_unlink_free_block(Allctr_t *allctr, Block_t *block) ASSERT(IS_LIST_ELEM(LIST_NEXT(x))); #ifdef HARD_DEBUG - check_tree(bfallctr, 0); + check_tree(root, 0, 0); #endif - replace(&bfallctr->root, x, LIST_NEXT(x)); + replace(root, x, LIST_NEXT(x)); #ifdef HARD_DEBUG check_tree(bfallctr, 0); @@ -788,7 +807,7 @@ bf_unlink_free_block(Allctr_t *allctr, Block_t *block) } else { /* Remove from tree */ - tree_delete(allctr, block); + tree_delete(allctr, block, flags); } DESTROY_LIST_ELEM(x); @@ -797,10 +816,14 @@ bf_unlink_free_block(Allctr_t *allctr, Block_t *block) static Block_t * bf_get_free_block(Allctr_t *allctr, Uint size, - Block_t *cand_blk, Uint cand_size) + Block_t *cand_blk, Uint cand_size, + Uint32 flags) { BFAllctr_t *bfallctr = (BFAllctr_t *) allctr; - RBTree_t *x = bfallctr->root; + RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC) + ? &bfallctr->sbmbc_root + : &bfallctr->mbc_root); + RBTree_t *x = *root; RBTree_t *blk = NULL; Uint blk_sz; @@ -827,7 +850,7 @@ bf_get_free_block(Allctr_t *allctr, Uint size, #ifdef HARD_DEBUG { - RBTree_t *ct_blk = check_tree(bfallctr, size); + RBTree_t *ct_blk = check_tree(root, 0, size); ASSERT(BLK_SZ(ct_blk) == BLK_SZ(blk)); } #endif @@ -839,7 +862,7 @@ bf_get_free_block(Allctr_t *allctr, Uint size, the tree node */ blk = LIST_NEXT(blk) ? LIST_NEXT(blk) : blk; - bf_unlink_free_block(allctr, (Block_t *) blk); + bf_unlink_free_block(allctr, (Block_t *) blk, flags); return (Block_t *) blk; } @@ -949,13 +972,14 @@ erts_bfalc_test(unsigned long op, unsigned long a1, unsigned long a2) { switch (op) { case 0x200: return (unsigned long) ((BFAllctr_t *) a1)->address_order; - case 0x201: return (unsigned long) ((BFAllctr_t *) a1)->root; + case 0x201: return (unsigned long) ((BFAllctr_t *) a1)->mbc_root; case 0x202: return (unsigned long) ((RBTree_t *) a1)->parent; case 0x203: return (unsigned long) ((RBTree_t *) a1)->left; case 0x204: return (unsigned long) ((RBTree_t *) a1)->right; case 0x205: return (unsigned long) ((RBTreeList_t *) a1)->next; case 0x206: return (unsigned long) IS_BLACK((RBTree_t *) a1); case 0x207: return (unsigned long) IS_TREE_NODE((RBTree_t *) a1); + case 0x208: return (unsigned long) 0; /* IS_AOFF */ default: ASSERT(0); return ~((unsigned long) 0); } } @@ -985,7 +1009,7 @@ erts_bfalc_test(unsigned long op, unsigned long a1, unsigned long a2) #endif #ifdef PRINT_TREE -static void print_tree(BFAllctr_t *); +static void print_tree(RBTree_t *, int); #endif /* @@ -1003,7 +1027,7 @@ static void print_tree(BFAllctr_t *); */ static RBTree_t * -check_tree(BFAllctr_t *bfallctr, Uint size) +check_tree(RBTree_t *root, int ao, Uint size) { RBTree_t *res = NULL; Sint blacks; @@ -1011,13 +1035,13 @@ check_tree(BFAllctr_t *bfallctr, Uint size) RBTree_t *x; #ifdef PRINT_TREE - print_tree(bfallctr); + print_tree(root, ao); #endif - if (!bfallctr->root) + if (!root) return res; - x = bfallctr->root; + x = root; ASSERT(IS_BLACK(x)); ASSERT(!x->parent); curr_blacks = 1; @@ -1060,11 +1084,11 @@ check_tree(BFAllctr_t *bfallctr, Uint size) ASSERT(IS_BLACK(x->left)); } - ASSERT(x->parent || x == bfallctr->root); + ASSERT(x->parent || x == root); if (x->left) { ASSERT(x->left->parent == x); - if (bfallctr->address_order) { + if (ao) { ASSERT(BLK_SZ(x->left) < BLK_SZ(x) || (BLK_SZ(x->left) == BLK_SZ(x) && x->left < x)); } @@ -1076,7 +1100,7 @@ check_tree(BFAllctr_t *bfallctr, Uint size) if (x->right) { ASSERT(x->right->parent == x); - if (bfallctr->address_order) { + if (ao) { ASSERT(BLK_SZ(x->right) > BLK_SZ(x) || (BLK_SZ(x->right) == BLK_SZ(x) && x->right > x)); } @@ -1087,7 +1111,7 @@ check_tree(BFAllctr_t *bfallctr, Uint size) } if (size && BLK_SZ(x) >= size) { - if (bfallctr->address_order) { + if (ao) { if (!res || BLK_SZ(x) < BLK_SZ(res) || (BLK_SZ(x) == BLK_SZ(res) && x < res)) @@ -1109,8 +1133,8 @@ check_tree(BFAllctr_t *bfallctr, Uint size) ASSERT(curr_blacks == 0); - UNSET_LEFT_VISITED(bfallctr->root); - UNSET_RIGHT_VISITED(bfallctr->root); + UNSET_LEFT_VISITED(root); + UNSET_RIGHT_VISITED(root); return res; @@ -1148,11 +1172,11 @@ print_tree_aux(RBTree_t *x, int indent) static void -print_tree(BFAllctr_t *bfallctr) +print_tree(RBTree_t *root, int ao) { - char *type = bfallctr->address_order ? "Size-Adress" : "Size"; + char *type = ao ? "Size-Adress" : "Size"; fprintf(stderr, " --- %s tree begin ---\r\n", type); - print_tree_aux(bfallctr->root, 0); + print_tree_aux(root, 0); fprintf(stderr, " --- %s tree end ---\r\n", type); } diff --git a/erts/emulator/beam/erl_bestfit_alloc.h b/erts/emulator/beam/erl_bestfit_alloc.h index cb35e21e57..faa2d9742e 100644 --- a/erts/emulator/beam/erl_bestfit_alloc.h +++ b/erts/emulator/beam/erl_bestfit_alloc.h @@ -54,7 +54,8 @@ typedef struct RBTree_t_ RBTree_t; struct BFAllctr_t_ { Allctr_t allctr; /* Has to be first! */ - RBTree_t * root; + RBTree_t * mbc_root; + RBTree_t * sbmbc_root; int address_order; }; diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c index 9631fb50db..d714eacd06 100644 --- a/erts/emulator/beam/erl_bif_ddll.c +++ b/erts/emulator/beam/erl_bif_ddll.c @@ -369,7 +369,7 @@ BIF_RETTYPE erl_ddll_try_load_3(Process *p, Eterm path_term, if (!(prt->status & FREE_PORT_FLAGS) && prt->drv_ptr->handle == dh) { #if DDLL_SMP - erts_smp_atomic_inc(&prt->refc); + erts_smp_atomic_inc_nob(&prt->refc); /* Extremely rare spinlock */ while(prt->status & ERTS_PORT_SFLG_INITIALIZING) { erts_smp_port_state_unlock(prt); @@ -597,7 +597,7 @@ done: if (!(prt->status & FREE_PORT_FLAGS) && prt->drv_ptr->handle == dh) { #if DDLL_SMP - erts_smp_atomic_inc(&prt->refc); + erts_smp_atomic_inc_nob(&prt->refc); /* Extremely rare spinlock */ while(prt->status & ERTS_PORT_SFLG_INITIALIZING) { erts_smp_port_state_unlock(prt); @@ -1054,7 +1054,7 @@ void erts_ddll_proc_dead(Process *p, ErtsProcLocks plocks) if (!(prt->status & FREE_PORT_FLAGS) && prt->drv_ptr->handle == dh) { #if DDLL_SMP - erts_smp_atomic_inc(&prt->refc); + erts_smp_atomic_inc_nob(&prt->refc); while(prt->status & ERTS_PORT_SFLG_INITIALIZING) { erts_smp_port_state_unlock(prt); erts_smp_port_state_lock(prt); @@ -1602,7 +1602,7 @@ static int do_load_driver_entry(DE_Handle *dh, char *path, char *name) erts_sys_ddll_close(dh->handle); return ERL_DE_LOAD_ERROR_BAD_NAME; } - erts_smp_atomic_init(&(dh->refc), (erts_aint_t) 0); + erts_smp_atomic_init_nob(&(dh->refc), (erts_aint_t) 0); dh->port_count = 0; dh->full_path = erts_alloc(ERTS_ALC_T_DDLL_HANDLE, sys_strlen(path) + 1); sys_strcpy(dh->full_path, path); diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index f264bf44df..6a74596f76 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -135,7 +135,7 @@ bld_bin_list(Uint **hpp, Uint *szp, ErlOffHeap* oh) if (szp) *szp += 4+2; if (hpp) { - Uint refc = (Uint) erts_smp_atomic_read(&pb->val->refc); + Uint refc = (Uint) erts_smp_atomic_read_nob(&pb->val->refc); tuple = TUPLE3(*hpp, val, orig_size, make_small(refc)); res = CONS(*hpp + 4, tuple, res); *hpp += 4+2; @@ -2026,7 +2026,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) res = TUPLE2(hp, am_sequential_tracer, val); BIF_RET(res); } else if (BIF_ARG_1 == am_garbage_collection){ - Uint val = (Uint) erts_smp_atomic32_read(&erts_max_gen_gcs); + Uint val = (Uint) erts_smp_atomic32_read_nob(&erts_max_gen_gcs); Eterm tup; hp = HAlloc(BIF_P, 3+2 + 3+2 + 3+2); @@ -2041,7 +2041,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) BIF_RET(res); } else if (BIF_ARG_1 == am_fullsweep_after){ - Uint val = (Uint) erts_smp_atomic32_read(&erts_max_gen_gcs); + Uint val = (Uint) erts_smp_atomic32_read_nob(&erts_max_gen_gcs); hp = HAlloc(BIF_P, 3); res = TUPLE2(hp, am_fullsweep_after, make_small(val)); BIF_RET(res); @@ -2545,6 +2545,70 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) hp = hsz ? HAlloc(BIF_P, hsz) : NULL; res = erts_bld_uint(&hp, NULL, erts_dist_buf_busy_limit); BIF_RET(res); + } else if (ERTS_IS_ATOM_STR("print_ethread_info", BIF_ARG_1)) { + int i; + char **str; +#ifdef ETHR_NATIVE_ATOMIC32_IMPL + erts_printf("32-bit native atomics: %s\n", + ETHR_NATIVE_ATOMIC32_IMPL); + str = ethr_native_atomic32_ops(); + for (i = 0; str[i]; i++) + erts_printf("ethr_native_atomic32_%s()\n", str[i]); +#endif +#ifdef ETHR_NATIVE_ATOMIC64_IMPL + erts_printf("64-bit native atomics: %s\n", + ETHR_NATIVE_ATOMIC64_IMPL); + str = ethr_native_atomic64_ops(); + for (i = 0; str[i]; i++) + erts_printf("ethr_native_atomic64_%s()\n", str[i]); +#endif +#ifdef ETHR_NATIVE_DW_ATOMIC_IMPL + if (ethr_have_native_dw_atomic()) { + erts_printf("Double word native atomics: %s\n", + ETHR_NATIVE_DW_ATOMIC_IMPL); + str = ethr_native_dw_atomic_ops(); + for (i = 0; str[i]; i++) + erts_printf("ethr_native_dw_atomic_%s()\n", str[i]); + str = ethr_native_su_dw_atomic_ops(); + for (i = 0; str[i]; i++) + erts_printf("ethr_native_su_dw_atomic_%s()\n", str[i]); + } +#endif +#ifdef ETHR_NATIVE_SPINLOCK_IMPL + erts_printf("Native spin-locks: %s\n", ETHR_NATIVE_SPINLOCK_IMPL); +#endif +#ifdef ETHR_NATIVE_RWSPINLOCK_IMPL + erts_printf("Native rwspin-locks: %s\n", ETHR_NATIVE_RWSPINLOCK_IMPL); +#endif +#ifdef ETHR_X86_RUNTIME_CONF_HAVE_SSE2__ + erts_printf("SSE2 support: %s\n", (ETHR_X86_RUNTIME_CONF_HAVE_SSE2__ + ? "yes" : "no")); +#endif +#ifdef ETHR_X86_OUT_OF_ORDER + erts_printf("x86" +#ifdef ARCH_64 + "_64" +#endif + " out of order\n"); +#endif +#ifdef ETHR_SPARC_TSO + erts_printf("Sparc TSO\n"); +#endif +#ifdef ETHR_SPARC_PSO + erts_printf("Sparc PSO\n"); +#endif +#ifdef ETHR_SPARC_RMO + erts_printf("Sparc RMO\n"); +#endif +#if defined(ETHR_PPC_HAVE_LWSYNC) + erts_printf("Have lwsync instruction: yes\n"); +#elif defined(ETHR_PPC_HAVE_NO_LWSYNC) + erts_printf("Have lwsync instruction: no\n"); +#elif defined(ETHR_PPC_RUNTIME_CONF_HAVE_LWSYNC__) + erts_printf("Have lwsync instruction: %s (runtime test)\n", + ETHR_PPC_RUNTIME_CONF_HAVE_LWSYNC__ ? "yes" : "no"); +#endif + BIF_RET(am_true); } BIF_ERROR(BIF_P, BADARG); @@ -2845,7 +2909,7 @@ fun_info_2(Process* p, Eterm fun, Eterm what) } break; case am_refc: - val = erts_make_integer(erts_smp_atomic_read(&funp->fe->refc), p); + val = erts_make_integer(erts_smp_atomic_read_nob(&funp->fe->refc), p); hp = HAlloc(p, 3); break; case am_arity: @@ -3065,8 +3129,8 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1) Eterm r1, r2; Eterm in, out; Uint hsz = 9; - Uint bytes_in = (Uint) erts_smp_atomic_read(&erts_bytes_in); - Uint bytes_out = (Uint) erts_smp_atomic_read(&erts_bytes_out); + Uint bytes_in = (Uint) erts_smp_atomic_read_nob(&erts_bytes_in); + Uint bytes_out = (Uint) erts_smp_atomic_read_nob(&erts_bytes_out); (void) erts_bld_uint(NULL, &hsz, bytes_in); (void) erts_bld_uint(NULL, &hsz, bytes_out); @@ -3139,7 +3203,7 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) * NOTE: Only supposed to be used for testing, and debugging. */ - if (!erts_smp_atomic_read(&available_internal_state)) { + if (!erts_smp_atomic_read_nob(&available_internal_state)) { BIF_ERROR(BIF_P, EXC_UNDEF); } @@ -3437,7 +3501,7 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) if (ERTS_IS_ATOM_STR("available_internal_state", BIF_ARG_1) && (BIF_ARG_2 == am_true || BIF_ARG_2 == am_false)) { erts_aint_t on = (erts_aint_t) (BIF_ARG_2 == am_true); - erts_aint_t prev_on = erts_smp_atomic_xchg(&available_internal_state, on); + erts_aint_t prev_on = erts_smp_atomic_xchg_nob(&available_internal_state, on); if (on) { erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); erts_dsprintf(dsbufp, "Process %T ", BIF_P->id); @@ -3453,7 +3517,7 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) BIF_RET(prev_on ? am_true : am_false); } - if (!erts_smp_atomic_read(&available_internal_state)) { + if (!erts_smp_atomic_read_nob(&available_internal_state)) { BIF_ERROR(BIF_P, EXC_UNDEF); } @@ -3634,14 +3698,14 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) } else if (ERTS_IS_ATOM_STR("hipe_test_reschedule_suspend", BIF_ARG_1)) { /* Used by hipe test suites */ - erts_aint_t flag = erts_smp_atomic_read(&hipe_test_reschedule_flag); + erts_aint_t flag = erts_smp_atomic_read_nob(&hipe_test_reschedule_flag); if (!flag && BIF_ARG_2 != am_false) { - erts_smp_atomic_set(&hipe_test_reschedule_flag, 1); + erts_smp_atomic_set_nob(&hipe_test_reschedule_flag, 1); erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, NULL); ERTS_BIF_YIELD2(bif_export[BIF_erts_debug_set_internal_state_2], BIF_P, BIF_ARG_1, BIF_ARG_2); } - erts_smp_atomic_set(&hipe_test_reschedule_flag, !flag); + erts_smp_atomic_set_nob(&hipe_test_reschedule_flag, !flag); BIF_RET(NIL); } else if (ERTS_IS_ATOM_STR("hipe_test_reschedule_resume", BIF_ARG_1)) { @@ -3951,8 +4015,8 @@ BIF_RETTYPE erts_debug_lock_counters_1(BIF_ALIST_1) void erts_bif_info_init(void) { - erts_smp_atomic_init(&available_internal_state, 0); - erts_smp_atomic_init(&hipe_test_reschedule_flag, 0); + erts_smp_atomic_init_nob(&available_internal_state, 0); + erts_smp_atomic_init_nob(&hipe_test_reschedule_flag, 0); process_info_init(); } diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c index e56084b9cb..326a5c136b 100644 --- a/erts/emulator/beam/erl_bits.c +++ b/erts/emulator/beam/erl_bits.c @@ -76,14 +76,12 @@ struct erl_bits_state ErlBitsState; #define byte_buf (ErlBitsState.byte_buf_) #define byte_buf_len (ErlBitsState.byte_buf_len_) -#ifdef ERTS_SMP static erts_smp_atomic_t bits_bufs_size; -#endif Uint erts_bits_bufs_size(void) { - return 0; + return (Uint) erts_smp_atomic_read_nob(&bits_bufs_size); } #if !defined(ERTS_SMP) @@ -109,8 +107,8 @@ erts_bits_destroy_state(ERL_BITS_PROTO_0) void erts_init_bits(void) { + erts_smp_atomic_init_nob(&bits_bufs_size, 0); #if defined(ERTS_SMP) - erts_smp_atomic_init(&bits_bufs_size, 0); /* erl_process.c calls erts_bits_init_state() on all state instances */ #else ERL_BITS_DECLARE_STATEP; @@ -713,9 +711,7 @@ static void ERTS_INLINE need_byte_buf(ERL_BITS_PROTO_1(int need)) { if (byte_buf_len < need) { -#ifdef ERTS_SMP - erts_smp_atomic_add(&bits_bufs_size, need - byte_buf_len); -#endif + erts_smp_atomic_add_nob(&bits_bufs_size, need - byte_buf_len); byte_buf_len = need; byte_buf = erts_realloc(ERTS_ALC_T_BITS_BUF, byte_buf, byte_buf_len); } diff --git a/erts/emulator/beam/erl_cpu_topology.c b/erts/emulator/beam/erl_cpu_topology.c index bcf8bcf270..03c0ef904a 100644 --- a/erts/emulator/beam/erl_cpu_topology.c +++ b/erts/emulator/beam/erl_cpu_topology.c @@ -487,7 +487,7 @@ erts_sched_check_cpu_bind_post_suspend(ErtsSchedulerData *esdp) /* Make sure we check if we should bind to a cpu or not... */ if (esdp->run_queue->flags & ERTS_RUNQ_FLG_SHARED_RUNQ) - erts_smp_atomic32_set(&esdp->chk_cpu_bind, 1); + erts_smp_atomic32_set_nob(&esdp->chk_cpu_bind, 1); else esdp->run_queue->flags |= ERTS_RUNQ_FLG_CHK_CPU_BIND; } @@ -503,7 +503,7 @@ erts_sched_check_cpu_bind(ErtsSchedulerData *esdp) erts_cpu_groups_callback_call_t *cgcc; #ifdef ERTS_SMP if (erts_common_run_queue) - erts_smp_atomic32_set(&esdp->chk_cpu_bind, 0); + erts_smp_atomic32_set_nob(&esdp->chk_cpu_bind, 0); else { esdp->run_queue->flags &= ~ERTS_RUNQ_FLG_CHK_CPU_BIND; } diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index e0a6aa05c6..59ef8fc3ea 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -224,21 +224,21 @@ static void free_dbtable(DbTable* tb) { #ifdef HARDDEBUG - if (erts_smp_atomic_read(&tb->common.memory_size) != sizeof(DbTable)) { + if (erts_smp_atomic_read_nob(&tb->common.memory_size) != sizeof(DbTable)) { erts_fprintf(stderr, "ets: free_dbtable memory remain=%ld fix=%x\n", - erts_smp_atomic_read(&tb->common.memory_size)-sizeof(DbTable), + erts_smp_atomic_read_nob(&tb->common.memory_size)-sizeof(DbTable), tb->common.fixations); } erts_fprintf(stderr, "ets: free_dbtable(%T) deleted!!!\r\n", tb->common.id); erts_fprintf(stderr, "ets: free_dbtable: meta_pid_to_tab common.memory_size = %ld\n", - erts_smp_atomic_read(&meta_pid_to_tab->common.memory_size)); + erts_smp_atomic_read_nob(&meta_pid_to_tab->common.memory_size)); print_table(ERTS_PRINT_STDOUT, NULL, 1, meta_pid_to_tab); erts_fprintf(stderr, "ets: free_dbtable: meta_pid_to_fixed_tab common.memory_size = %ld\n", - erts_smp_atomic_read(&meta_pid_to_fixed_tab->common.memory_size)); + erts_smp_atomic_read_nob(&meta_pid_to_fixed_tab->common.memory_size)); print_table(ERTS_PRINT_STDOUT, NULL, 1, meta_pid_to_fixed_tab); #endif #ifdef ERTS_SMP @@ -248,6 +248,7 @@ free_dbtable(DbTable* tb) ASSERT(is_immed(tb->common.heir_data)); erts_db_free(ERTS_ALC_T_DB_TABLE, tb, (void *) tb, sizeof(DbTable)); ERTS_ETS_MISC_MEM_ADD(-sizeof(DbTable)); + ERTS_THR_MEMORY_BARRIER; } #ifdef ERTS_SMP @@ -338,13 +339,13 @@ static ERTS_INLINE void db_unlock(DbTable* tb, db_lock_kind_t kind) ASSERT(tb != meta_pid_to_tab && tb != meta_pid_to_fixed_tab); if (tb->common.type & DB_FINE_LOCKED) { - if (tb->common.is_thread_safe) { - ASSERT(kind == LCK_WRITE); + if (kind == LCK_WRITE) { + ASSERT(tb->common.is_thread_safe); tb->common.is_thread_safe = 0; erts_smp_rwmtx_rwunlock(&tb->common.rwlock); } else { - ASSERT(kind != LCK_WRITE); + ASSERT(!tb->common.is_thread_safe); erts_smp_rwmtx_runlock(&tb->common.rwlock); } } @@ -543,9 +544,9 @@ static int remove_named_tab(DbTable *tb, int have_lock) * We keep our increased refc over this op in order to * prevent the table from disapearing. */ - erts_smp_rwmtx_rwunlock(&tb->common.rwlock); + db_unlock(tb, LCK_WRITE); erts_smp_rwmtx_rwlock(rwlock); - erts_smp_rwmtx_rwlock(&tb->common.rwlock); + db_lock(tb, LCK_WRITE); } #endif @@ -1417,12 +1418,12 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) { DbTable init_tb; - erts_smp_atomic_init(&init_tb.common.memory_size, 0); + erts_smp_atomic_init_nob(&init_tb.common.memory_size, 0); tb = (DbTable*) erts_db_alloc(ERTS_ALC_T_DB_TABLE, &init_tb, sizeof(DbTable)); ERTS_ETS_MISC_MEM_ADD(sizeof(DbTable)); - erts_smp_atomic_init(&tb->common.memory_size, - erts_smp_atomic_read(&init_tb.common.memory_size)); + erts_smp_atomic_init_nob(&tb->common.memory_size, + erts_smp_atomic_read_nob(&init_tb.common.memory_size)); } tb->common.meth = meth; @@ -1439,7 +1440,7 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) tb->common.owner = BIF_P->id; set_heir(BIF_P, tb, heir, heir_data); - erts_smp_atomic_init(&tb->common.nitems, 0); + erts_smp_atomic_init_nob(&tb->common.nitems, 0); tb->common.fixations = NULL; tb->common.compress = is_compressed; @@ -1505,9 +1506,9 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) BIF_ARG_1, BIF_ARG_2, ret, BIF_P->id, BIF_P->initial[0], BIF_P->initial[1], BIF_P->initial[2]); erts_fprintf(stderr, "ets: new: meta_pid_to_tab common.memory_size = %ld\n", - erts_smp_atomic_read(&meta_pid_to_tab->common.memory_size)); + erts_smp_atomic_read_nob(&meta_pid_to_tab->common.memory_size)); erts_fprintf(stderr, "ets: new: meta_pid_to_fixed_tab common.memory_size = %ld\n", - erts_smp_atomic_read(&meta_pid_to_fixed_tab->common.memory_size)); + erts_smp_atomic_read_nob(&meta_pid_to_fixed_tab->common.memory_size)); #endif UseTmpHeap(3,BIF_P); @@ -1650,24 +1651,6 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1) tb->common.status &= ~(DB_PROTECTED|DB_PUBLIC|DB_PRIVATE); tb->common.status |= DB_DELETE; - mmtl = get_meta_main_tab_lock(tb->common.slot); -#ifdef ERTS_SMP - if (erts_smp_rwmtx_tryrwlock(mmtl) == EBUSY) { - /* - * We keep our increased refc over this op in order to - * prevent the table from disapearing. - */ - erts_smp_rwmtx_rwunlock(&tb->common.rwlock); - erts_smp_rwmtx_rwlock(mmtl); - erts_smp_rwmtx_rwlock(&tb->common.rwlock); - } -#endif - /* We must keep the slot, to be found by db_proc_dead() if process dies */ - MARK_SLOT_DEAD(tb->common.slot); - erts_smp_rwmtx_rwunlock(mmtl); - if (is_atom(tb->common.id)) - remove_named_tab(tb, 0); - if (tb->common.owner != BIF_P->id) { DeclareTmpHeap(meta_tuple,3,BIF_P); @@ -1691,6 +1674,25 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1) db_meta_unlock(meta_pid_to_tab, LCK_WRITE_REC); UnUseTmpHeap(3,BIF_P); } + + mmtl = get_meta_main_tab_lock(tb->common.slot); +#ifdef ERTS_SMP + if (erts_smp_rwmtx_tryrwlock(mmtl) == EBUSY) { + /* + * We keep our increased refc over this op in order to + * prevent the table from disapearing. + */ + db_unlock(tb, LCK_WRITE); + erts_smp_rwmtx_rwlock(mmtl); + db_lock(tb, LCK_WRITE); + } +#endif + /* We must keep the slot, to be found by db_proc_dead() if process dies */ + MARK_SLOT_DEAD(tb->common.slot); + erts_smp_rwmtx_rwunlock(mmtl); + if (is_atom(tb->common.id)) + remove_named_tab(tb, 0); + /* disable inheritance */ free_heir_data(tb); tb->common.heir = am_none; @@ -1995,7 +1997,7 @@ BIF_RETTYPE ets_select_delete_2(BIF_ALIST_2) if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE)) == NULL) { BIF_ERROR(BIF_P, BADARG); } - nitems = erts_smp_atomic_read(&tb->common.nitems); + nitems = erts_smp_atomic_read_nob(&tb->common.nitems); tb->common.meth->db_delete_all_objects(BIF_P, tb); db_unlock(tb, LCK_WRITE); BIF_RET(erts_make_integer(nitems,BIF_P)); @@ -2789,7 +2791,7 @@ void init_db(void) } #endif - erts_smp_atomic_init(&erts_ets_misc_mem_size, 0); + erts_smp_atomic_init_nob(&erts_ets_misc_mem_size, 0); db_initialize_util(); if (user_requested_db_max_tabs < DB_DEF_MAX_TABS) @@ -2831,13 +2833,13 @@ void init_db(void) /*TT*/ /* Create meta table invertion. */ - erts_smp_atomic_init(&init_tb.common.memory_size, 0); + erts_smp_atomic_init_nob(&init_tb.common.memory_size, 0); meta_pid_to_tab = (DbTable*) erts_db_alloc(ERTS_ALC_T_DB_TABLE, &init_tb, sizeof(DbTable)); ERTS_ETS_MISC_MEM_ADD(sizeof(DbTable)); - erts_smp_atomic_init(&meta_pid_to_tab->common.memory_size, - erts_smp_atomic_read(&init_tb.common.memory_size)); + erts_smp_atomic_init_nob(&meta_pid_to_tab->common.memory_size, + erts_smp_atomic_read_nob(&init_tb.common.memory_size)); meta_pid_to_tab->common.id = NIL; meta_pid_to_tab->common.the_name = am_true; @@ -2850,7 +2852,7 @@ void init_db(void) #endif meta_pid_to_tab->common.keypos = 1; meta_pid_to_tab->common.owner = NIL; - erts_smp_atomic_init(&meta_pid_to_tab->common.nitems, 0); + erts_smp_atomic_init_nob(&meta_pid_to_tab->common.nitems, 0); meta_pid_to_tab->common.slot = -1; meta_pid_to_tab->common.meth = &db_hash; meta_pid_to_tab->common.compress = 0; @@ -2863,13 +2865,13 @@ void init_db(void) erl_exit(1,"Unable to create ets metadata tables."); } - erts_smp_atomic_set(&init_tb.common.memory_size, 0); + erts_smp_atomic_set_nob(&init_tb.common.memory_size, 0); meta_pid_to_fixed_tab = (DbTable*) erts_db_alloc(ERTS_ALC_T_DB_TABLE, &init_tb, sizeof(DbTable)); ERTS_ETS_MISC_MEM_ADD(sizeof(DbTable)); - erts_smp_atomic_init(&meta_pid_to_fixed_tab->common.memory_size, - erts_smp_atomic_read(&init_tb.common.memory_size)); + erts_smp_atomic_init_nob(&meta_pid_to_fixed_tab->common.memory_size, + erts_smp_atomic_read_nob(&init_tb.common.memory_size)); meta_pid_to_fixed_tab->common.id = NIL; meta_pid_to_fixed_tab->common.the_name = am_true; @@ -2882,7 +2884,7 @@ void init_db(void) #endif meta_pid_to_fixed_tab->common.keypos = 1; meta_pid_to_fixed_tab->common.owner = NIL; - erts_smp_atomic_init(&meta_pid_to_fixed_tab->common.nitems, 0); + erts_smp_atomic_init_nob(&meta_pid_to_fixed_tab->common.nitems, 0); meta_pid_to_fixed_tab->common.slot = -1; meta_pid_to_fixed_tab->common.meth = &db_hash; meta_pid_to_fixed_tab->common.compress = 0; @@ -3421,7 +3423,7 @@ static void unfix_table_locked(Process* p, DbTable* tb, unlocked: if (!IS_FIXED(tb) && IS_HASH_TABLE(tb->common.status) - && erts_smp_atomic_read(&tb->hash.fixdel) != (erts_aint_t)NULL) { + && erts_smp_atomic_read_nob(&tb->hash.fixdel) != (erts_aint_t)NULL) { #ifdef ERTS_SMP if (*kind_p == LCK_READ && tb->common.is_thread_safe) { /* Must have write lock while purging pseudo-deleted (OTP-8166) */ @@ -3606,7 +3608,7 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What) Eterm ret = THE_NON_VALUE; if (What == am_size) { - ret = make_small(erts_smp_atomic_read(&tb->common.nitems)); + ret = make_small(erts_smp_atomic_read_nob(&tb->common.nitems)); } else if (What == am_type) { if (tb->common.status & DB_SET) { ret = am_set; @@ -3619,7 +3621,7 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What) ret = am_bag; } } else if (What == am_memory) { - Uint words = (Uint) ((erts_smp_atomic_read(&tb->common.memory_size) + Uint words = (Uint) ((erts_smp_atomic_read_nob(&tb->common.memory_size) + sizeof(Uint) - 1) / sizeof(Uint)); @@ -3716,7 +3718,7 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What) std_dev_exp = make_float(hp); PUT_DOUBLE(f, hp); hp += FLOAT_SIZE_OBJECT; - ret = TUPLE6(hp, make_small(erts_smp_atomic_read(&tb->hash.nactive)), + ret = TUPLE6(hp, make_small(erts_smp_atomic_read_nob(&tb->hash.nactive)), avg, std_dev_real, std_dev_exp, make_small(stats.min_chain_len), make_small(stats.max_chain_len)); @@ -3735,9 +3737,9 @@ static void print_table(int to, void *to_arg, int show, DbTable* tb) tb->common.meth->db_print(to, to_arg, show, tb); - erts_print(to, to_arg, "Objects: %d\n", (int)erts_smp_atomic_read(&tb->common.nitems)); + erts_print(to, to_arg, "Objects: %d\n", (int)erts_smp_atomic_read_nob(&tb->common.nitems)); erts_print(to, to_arg, "Words: %bpu\n", - (UWord) ((erts_smp_atomic_read(&tb->common.memory_size) + (Uint) ((erts_smp_atomic_read_nob(&tb->common.memory_size) + sizeof(Uint) - 1) / sizeof(Uint))); @@ -3763,8 +3765,9 @@ void db_info(int to, void *to_arg, int show) /* Called by break handler */ Uint erts_get_ets_misc_mem_size(void) { + ERTS_THR_MEMORY_BARRIER; /* Memory not allocated in ets_alloc */ - return (Uint) erts_smp_atomic_read(&erts_ets_misc_mem_size); + return (Uint) erts_smp_atomic_read_nob(&erts_ets_misc_mem_size); } /* SMP Note: May only be used when system is locked */ diff --git a/erts/emulator/beam/erl_db.h b/erts/emulator/beam/erl_db.h index e0bdebcb01..2e5deaf338 100644 --- a/erts/emulator/beam/erl_db.h +++ b/erts/emulator/beam/erl_db.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * Copyright Ericsson AB 1996-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -86,11 +86,11 @@ do { \ erts_aint_t sz__ = (((erts_aint_t) (ALLOC_SZ)) \ - ((erts_aint_t) (FREE_SZ))); \ ASSERT((TAB)); \ - erts_smp_atomic_add(&(TAB)->common.memory_size, sz__); \ + erts_smp_atomic_add_nob(&(TAB)->common.memory_size, sz__); \ } while (0) #define ERTS_ETS_MISC_MEM_ADD(SZ) \ - erts_smp_atomic_add(&erts_ets_misc_mem_size, (SZ)); + erts_smp_atomic_add_nob(&erts_ets_misc_mem_size, (SZ)); ERTS_GLB_INLINE void *erts_db_alloc(ErtsAlcType_t type, DbTable *tab, @@ -227,7 +227,7 @@ erts_db_free(ErtsAlcType_t type, DbTable *tab, void *ptr, Uint size) ERTS_DB_ALC_MEM_UPDATE_(tab, size, 0); ASSERT(((void *) tab) != ptr - || erts_smp_atomic_read(&tab->common.memory_size) == 0); + || erts_smp_atomic_read_nob(&tab->common.memory_size) == 0); erts_free(type, ptr); } diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index fdc82c8b88..e3380a57b2 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -105,9 +105,13 @@ #define NSEG_2 256 /* Size of second segment table */ #define NSEG_INC 128 /* Number of segments to grow after that */ -#define SEGTAB(tb) ((struct segment**)erts_smp_atomic_read_acqb(&(tb)->segtab)) -#define NACTIVE(tb) ((int)erts_smp_atomic_read(&(tb)->nactive)) -#define NITEMS(tb) ((int)erts_smp_atomic_read(&(tb)->common.nitems)) +#ifdef ETHR_ORDERED_READ_DEPEND +#define SEGTAB(tb) ((struct segment**)erts_smp_atomic_read_nob(&(tb)->segtab)) +#else +#define SEGTAB(tb) ((struct segment**)erts_smp_atomic_read_rb(&(tb)->segtab)) +#endif +#define NACTIVE(tb) ((int)erts_smp_atomic_read_nob(&(tb)->nactive)) +#define NITEMS(tb) ((int)erts_smp_atomic_read_nob(&(tb)->common.nitems)) #define BUCKET(tb, i) SEGTAB(tb)[(i) >> SEGSZ_EXP]->buckets[(i) & SEGSZ_MASK] @@ -123,10 +127,10 @@ static ERTS_INLINE Uint hash_to_ix(DbTableHash* tb, HashValue hval) { Uint mask = erts_smp_atomic_read_acqb(&tb->szm); - Uint ix = hval & mask; - if (ix >= erts_smp_atomic_read(&tb->nactive)) { + Uint ix = hval & mask; + if (ix >= erts_smp_atomic_read_nob(&tb->nactive)) { ix &= mask>>1; - ASSERT(ix < erts_smp_atomic_read(&tb->nactive)); + ASSERT(ix < erts_smp_atomic_read_nob(&tb->nactive)); } return ix; } @@ -141,14 +145,14 @@ static ERTS_INLINE void add_fixed_deletion(DbTableHash* tb, int ix) (DbTable *) tb, sizeof(FixedDeletion)); ERTS_ETS_MISC_MEM_ADD(sizeof(FixedDeletion)); - fixd->slot = ix; - was_next = erts_smp_atomic_read(&tb->fixdel); + fixd->slot = ix; + was_next = erts_smp_atomic_read_acqb(&tb->fixdel); do { /* Lockless atomic insertion in linked list: */ exp_next = was_next; fixd->next = (FixedDeletion*) exp_next; - was_next = erts_smp_atomic_cmpxchg(&tb->fixdel, - (erts_aint_t) fixd, - exp_next); + was_next = erts_smp_atomic_cmpxchg_relb(&tb->fixdel, + (erts_aint_t) fixd, + exp_next); }while (was_next != exp_next); } @@ -540,22 +544,24 @@ static void restore_fixdel(DbTableHash* tb, FixedDeletion* fixdel) { /*int tries = 0;*/ DEBUG_WAIT(); - if (erts_smp_atomic_cmpxchg(&tb->fixdel, (erts_aint_t)fixdel, - (erts_aint_t)NULL) != (erts_aint_t)NULL) { + if (erts_smp_atomic_cmpxchg_relb(&tb->fixdel, + (erts_aint_t) fixdel, + (erts_aint_t) NULL) != (erts_aint_t) NULL) { /* Oboy, must join lists */ FixedDeletion* last = fixdel; erts_aint_t was_tail; erts_aint_t exp_tail; - while (last->next != NULL) last = last->next; - was_tail = erts_smp_atomic_read(&tb->fixdel); + while (last->next != NULL) last = last->next; + was_tail = erts_smp_atomic_read_acqb(&tb->fixdel); do { /* Lockless atomic list insertion */ exp_tail = was_tail; last->next = (FixedDeletion*) exp_tail; /*++tries;*/ DEBUG_WAIT(); - was_tail = erts_smp_atomic_cmpxchg(&tb->fixdel, (erts_aint_t)fixdel, - exp_tail); + was_tail = erts_smp_atomic_cmpxchg_relb(&tb->fixdel, + (erts_aint_t) fixdel, + exp_tail); }while (was_tail != exp_tail); } /*erts_fprintf(stderr,"erl_db_hash: restore_fixdel tries=%d\r\n", tries);*/ @@ -572,7 +578,8 @@ void db_unfix_table_hash(DbTableHash *tb) || (erts_smp_lc_rwmtx_is_rlocked(&tb->common.rwlock) && !tb->common.is_thread_safe)); restart: - fixdel = (FixedDeletion*) erts_smp_atomic_xchg(&tb->fixdel, (erts_aint_t)NULL); + fixdel = (FixedDeletion*) erts_smp_atomic_xchg_acqb(&tb->fixdel, + (erts_aint_t) NULL); while (fixdel != NULL) { FixedDeletion *fx = fixdel; int ix = fx->slot; @@ -639,14 +646,14 @@ int db_create_hash(Process *p, DbTable *tbl) { DbTableHash *tb = &tbl->hash; - erts_smp_atomic_init(&tb->szm, SEGSZ_MASK); - erts_smp_atomic_init(&tb->nactive, SEGSZ); - erts_smp_atomic_init(&tb->fixdel, (erts_aint_t)NULL); - erts_smp_atomic_init(&tb->segtab, (erts_aint_t) alloc_ext_seg(tb,0,NULL)->segtab); + erts_smp_atomic_init_nob(&tb->szm, SEGSZ_MASK); + erts_smp_atomic_init_nob(&tb->nactive, SEGSZ); + erts_smp_atomic_init_nob(&tb->fixdel, (erts_aint_t)NULL); + erts_smp_atomic_init_nob(&tb->segtab, (erts_aint_t) alloc_ext_seg(tb,0,NULL)->segtab); tb->nsegs = NSEG_1; tb->nslots = SEGSZ; - erts_smp_atomic_init(&tb->is_resizing, 0); + erts_smp_atomic_init_nob(&tb->is_resizing, 0); #ifdef ERTS_SMP if (tb->common.type & DB_FINE_LOCKED) { erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; @@ -663,7 +670,7 @@ int db_create_hash(Process *p, DbTable *tbl) /* This important property is needed to guarantee that the buckets * involved in a grow/shrink operation it protected by the same lock: */ - ASSERT(erts_smp_atomic_read(&tb->nactive) % DB_HASH_LOCK_CNT == 0); + ASSERT(erts_smp_atomic_read_nob(&tb->nactive) % DB_HASH_LOCK_CNT == 0); } else { /* coarse locking */ tb->locks = NULL; @@ -783,7 +790,7 @@ int db_put_hash(DbTable *tbl, Eterm obj, int key_clash_fail) if (tb->common.status & DB_SET) { HashDbTerm* bnext = b->next; if (b->hvalue == INVALID_HASH) { - erts_smp_atomic_inc(&tb->common.nitems); + erts_smp_atomic_inc_nob(&tb->common.nitems); } else if (key_clash_fail) { ret = DB_ERROR_BADKEY; @@ -811,7 +818,7 @@ int db_put_hash(DbTable *tbl, Eterm obj, int key_clash_fail) do { if (db_eq(&tb->common,obj,&q->dbterm)) { if (q->hvalue == INVALID_HASH) { - erts_smp_atomic_inc(&tb->common.nitems); + erts_smp_atomic_inc_nob(&tb->common.nitems); q->hvalue = hval; if (q != b) { /* must move to preserve key insertion order */ *qp = q->next; @@ -832,7 +839,7 @@ Lnew: q->hvalue = hval; q->next = b; *bp = q; - nitems = erts_smp_atomic_inctest(&tb->common.nitems); + nitems = erts_smp_atomic_inc_read_nob(&tb->common.nitems); WUNLOCK_HASH(lck); { int nactive = NACTIVE(tb); @@ -1069,7 +1076,7 @@ int db_erase_bag_exact2(DbTable *tbl, Eterm key, Eterm value) EQ(value, b->dbterm.tpl[2])) { *bp = b->next; free_term(tb, b); - erts_smp_atomic_dec(&tb->common.nitems); + erts_smp_atomic_dec_nob(&tb->common.nitems); b = *bp; break; } @@ -1128,7 +1135,7 @@ int db_erase_hash(DbTable *tbl, Eterm key, Eterm *ret) } WUNLOCK_HASH(lck); if (nitems_diff) { - erts_smp_atomic_add(&tb->common.nitems, nitems_diff); + erts_smp_atomic_add_nob(&tb->common.nitems, nitems_diff); try_shrink(tb); } *ret = am_true; @@ -1187,7 +1194,7 @@ static int db_erase_object_hash(DbTable *tbl, Eterm object, Eterm *ret) } WUNLOCK_HASH(lck); if (nitems_diff) { - erts_smp_atomic_add(&tb->common.nitems, nitems_diff); + erts_smp_atomic_add_nob(&tb->common.nitems, nitems_diff); try_shrink(tb); } *ret = am_true; @@ -1798,7 +1805,7 @@ static int db_select_delete_hash(Process *p, free_term(tb, del); did_erase = 1; } - erts_smp_atomic_dec(&tb->common.nitems); + erts_smp_atomic_dec_nob(&tb->common.nitems); ++got; } --num_left; @@ -1909,7 +1916,7 @@ static int db_select_delete_continue_hash(Process *p, free_term(tb, del); did_erase = 1; } - erts_smp_atomic_dec(&tb->common.nitems); + erts_smp_atomic_dec_nob(&tb->common.nitems); ++got; } @@ -2064,7 +2071,7 @@ int db_mark_all_deleted_hash(DbTable *tbl) }while(list != NULL); } } - erts_smp_atomic_set(&tb->common.nitems, 0); + erts_smp_atomic_set_nob(&tb->common.nitems, 0); return DB_ERROR_NONE; } @@ -2115,7 +2122,7 @@ static int db_free_table_continue_hash(DbTable *tbl) { DbTableHash *tb = &tbl->hash; int done; - FixedDeletion* fixdel = (FixedDeletion*) erts_smp_atomic_read(&tb->fixdel); + FixedDeletion* fixdel = (FixedDeletion*) erts_smp_atomic_read_acqb(&tb->fixdel); ERTS_SMP_LC_ASSERT(IS_TAB_WLOCKED(tb)); done = 0; @@ -2129,11 +2136,11 @@ static int db_free_table_continue_hash(DbTable *tbl) sizeof(FixedDeletion)); ERTS_ETS_MISC_MEM_ADD(-sizeof(FixedDeletion)); if (++done >= 2*DELETE_RECORD_LIMIT) { - erts_smp_atomic_set(&tb->fixdel, (erts_aint_t)fixdel); + erts_smp_atomic_set_relb(&tb->fixdel, (erts_aint_t)fixdel); return 0; /* Not done */ } } - erts_smp_atomic_set(&tb->fixdel, (erts_aint_t)NULL); + erts_smp_atomic_set_relb(&tb->fixdel, (erts_aint_t)NULL); done /= 2; while(tb->nslots != 0) { @@ -2157,7 +2164,7 @@ static int db_free_table_continue_hash(DbTable *tbl) tb->locks = NULL; } #endif - ASSERT(erts_smp_atomic_read(&tb->common.memory_size) == sizeof(DbTable)); + ASSERT(erts_smp_atomic_read_nob(&tb->common.memory_size) == sizeof(DbTable)); return 1; /* Done */ } @@ -2350,7 +2357,7 @@ static int alloc_seg(DbTableHash *tb) struct ext_segment* eseg; eseg = (struct ext_segment*) SEGTAB(tb)[seg_ix-1]; MY_ASSERT(eseg!=NULL && eseg->s.is_ext_segment); - erts_smp_atomic_set_relb(&tb->segtab, (erts_aint_t) eseg->segtab); + erts_smp_atomic_set_wb(&tb->segtab, (erts_aint_t) eseg->segtab); tb->nsegs = eseg->nsegs; } ASSERT(seg_ix < tb->nsegs); @@ -2422,7 +2429,7 @@ static int free_seg(DbTableHash *tb, int free_records) MY_ASSERT(newtop->s.is_ext_segment); if (newtop->prev_segtab != NULL) { /* Time to use a smaller segtab */ - erts_smp_atomic_set_relb(&tb->segtab, (erts_aint_t)newtop->prev_segtab); + erts_smp_atomic_set_wb(&tb->segtab, (erts_aint_t)newtop->prev_segtab); tb->nsegs = seg_ix; ASSERT(tb->nsegs == EXTSEG(SEGTAB(tb))->nsegs); } @@ -2439,7 +2446,7 @@ static int free_seg(DbTableHash *tb, int free_records) if (seg_ix > 0) { if (seg_ix < tb->nsegs) SEGTAB(tb)[seg_ix] = NULL; } else { - erts_smp_atomic_set_relb(&tb->segtab, (erts_aint_t)NULL); + erts_smp_atomic_set_wb(&tb->segtab, (erts_aint_t)NULL); } #endif tb->nslots -= SEGSZ; @@ -2500,7 +2507,7 @@ static void grow(DbTableHash* tb, int nactive) int from_ix; int szm; - if (erts_smp_atomic_xchg(&tb->is_resizing, 1)) { + if (erts_smp_atomic_xchg_acqb(&tb->is_resizing, 1)) { return; /* already in progress */ } if (NACTIVE(tb) != nactive) { @@ -2515,7 +2522,7 @@ static void grow(DbTableHash* tb, int nactive) } ASSERT(nactive < tb->nslots); - szm = erts_smp_atomic_read(&tb->szm); + szm = erts_smp_atomic_read_nob(&tb->szm); if (nactive <= szm) { from_ix = nactive & (szm >> 1); } else { @@ -2532,7 +2539,7 @@ static void grow(DbTableHash* tb, int nactive) WUNLOCK_HASH(lck); goto abort; } - erts_smp_atomic_inc(&tb->nactive); + erts_smp_atomic_inc_nob(&tb->nactive); if (from_ix == 0) { erts_smp_atomic_set_relb(&tb->szm, szm); } @@ -2577,13 +2584,13 @@ abort: */ static void shrink(DbTableHash* tb, int nactive) { - if (erts_smp_atomic_xchg(&tb->is_resizing, 1)) { + if (erts_smp_atomic_xchg_acqb(&tb->is_resizing, 1)) { return; /* already in progress */ } if (NACTIVE(tb) == nactive) { erts_smp_rwmtx_t* lck; int src_ix = nactive - 1; - int low_szm = erts_smp_atomic_read(&tb->szm) >> 1; + int low_szm = erts_smp_atomic_read_nob(&tb->szm) >> 1; int dst_ix = src_ix & low_szm; ASSERT(dst_ix < src_ix); @@ -2610,7 +2617,7 @@ static void shrink(DbTableHash* tb, int nactive) *dst_bp = *src_bp; *src_bp = NULL; - erts_smp_atomic_set(&tb->nactive, src_ix); + erts_smp_atomic_set_nob(&tb->nactive, src_ix); if (dst_ix == 0) { erts_smp_atomic_set_relb(&tb->szm, low_szm); } @@ -2746,7 +2753,7 @@ static int db_delete_all_objects_hash(Process* p, DbTable* tbl) } else { db_free_table_hash(tbl); db_create_hash(p, tbl); - erts_smp_atomic_set(&tbl->hash.common.nitems, 0); + erts_smp_atomic_set_nob(&tbl->hash.common.nitems, 0); } return 0; } diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c index 9a0ba3a418..c6f0d80e32 100644 --- a/erts/emulator/beam/erl_db_tree.c +++ b/erts/emulator/beam/erl_db_tree.c @@ -49,7 +49,7 @@ #include "erl_db_tree.h" #define GETKEY_WITH_POS(Keypos, Tplp) (*((Tplp) + Keypos)) -#define NITEMS(tb) ((int)erts_smp_atomic_read(&(tb)->common.nitems)) +#define NITEMS(tb) ((int)erts_smp_atomic_read_nob(&(tb)->common.nitems)) /* ** A stack of this size is enough for an AVL tree with more than @@ -84,7 +84,7 @@ */ static DbTreeStack* get_static_stack(DbTableTree* tb) { - if (!erts_smp_atomic_xchg(&tb->is_stack_busy, 1)) { + if (!erts_smp_atomic_xchg_acqb(&tb->is_stack_busy, 1)) { return &tb->static_stack; } return NULL; @@ -96,7 +96,7 @@ static DbTreeStack* get_static_stack(DbTableTree* tb) static DbTreeStack* get_any_stack(DbTableTree* tb) { DbTreeStack* stack; - if (!erts_smp_atomic_xchg(&tb->is_stack_busy, 1)) { + if (!erts_smp_atomic_xchg_acqb(&tb->is_stack_busy, 1)) { return &tb->static_stack; } stack = erts_db_alloc(ERTS_ALC_T_DB_STK, (DbTable *) tb, @@ -110,7 +110,7 @@ static DbTreeStack* get_any_stack(DbTableTree* tb) static void release_stack(DbTableTree* tb, DbTreeStack* stack) { if (stack == &tb->static_stack) { - ASSERT(erts_smp_atomic_read(&tb->is_stack_busy) == 1); + ASSERT(erts_smp_atomic_read_nob(&tb->is_stack_busy) == 1); erts_smp_atomic_set_relb(&tb->is_stack_busy, 0); } else { @@ -478,7 +478,7 @@ int db_create_tree(Process *p, DbTable *tbl) sizeof(TreeDbTerm *) * STACK_NEED); tb->static_stack.pos = 0; tb->static_stack.slot = 0; - erts_smp_atomic_init(&tb->is_stack_busy, 0); + erts_smp_atomic_init_nob(&tb->is_stack_busy, 0); tb->deletion = 0; return DB_ERROR_NONE; } @@ -613,8 +613,8 @@ static int db_put_tree(DbTable *tbl, Eterm obj, int key_clash_fail) for (;;) if (!*this) { /* Found our place */ state = 1; - if (erts_smp_atomic_inctest(&tb->common.nitems) >= TREE_MAX_ELEMENTS) { - erts_smp_atomic_dec(&tb->common.nitems); + if (erts_smp_atomic_inc_read_nob(&tb->common.nitems) >= TREE_MAX_ELEMENTS) { + erts_smp_atomic_dec_nob(&tb->common.nitems); return DB_ERROR_SYSRES; } *this = new_dbterm(tb, obj); @@ -1583,7 +1583,7 @@ static int db_select_delete_continue_tree(Process *p, sc.max = 1000; sc.keypos = tb->common.keypos; - ASSERT(!erts_smp_atomic_read(&tb->is_stack_busy)); + ASSERT(!erts_smp_atomic_read_nob(&tb->is_stack_busy)); traverse_backwards(tb, &tb->static_stack, lastkey, NULL, &doit_select_delete, &sc); BUMP_REDS(p, 1000 - sc.max); @@ -1774,7 +1774,7 @@ static int db_free_table_continue_tree(DbTable *tbl) (DbTable *) tb, (void *) tb->static_stack.array, sizeof(TreeDbTerm *) * STACK_NEED); - ASSERT(erts_smp_atomic_read(&tb->common.memory_size) + ASSERT(erts_smp_atomic_read_nob(&tb->common.memory_size) == sizeof(DbTable)); } return result; @@ -1784,7 +1784,7 @@ static int db_delete_all_objects_tree(Process* p, DbTable* tbl) { db_free_table_tree(tbl); db_create_tree(p, tbl); - erts_smp_atomic_set(&tbl->tree.common.nitems, 0); + erts_smp_atomic_set_nob(&tbl->tree.common.nitems, 0); return 0; } @@ -1866,7 +1866,7 @@ static TreeDbTerm *linkout_tree(DbTableTree *tb, tstack[tpos++] = this; state = delsub(this); } - erts_smp_atomic_dec(&tb->common.nitems); + erts_smp_atomic_dec_nob(&tb->common.nitems); break; } } @@ -1933,7 +1933,7 @@ static TreeDbTerm *linkout_object_tree(DbTableTree *tb, tstack[tpos++] = this; state = delsub(this); } - erts_smp_atomic_dec(&tb->common.nitems); + erts_smp_atomic_dec_nob(&tb->common.nitems); break; } } diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index c3b074f782..7dfbb2ed02 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -491,7 +491,7 @@ erts_match_set_release_result(Process* c_p) /* The trace control word. */ -static erts_smp_atomic_t trace_control_word; +static erts_smp_atomic32_t trace_control_word; /* This needs to be here, before the bif table... */ @@ -911,7 +911,7 @@ static void db_free_tmp_uncompressed(DbTerm* obj); BIF_RETTYPE db_get_trace_control_word_0(Process *p) { - Uint32 tcw = (Uint32) erts_smp_atomic_read(&trace_control_word); + Uint32 tcw = (Uint32) erts_smp_atomic32_read_acqb(&trace_control_word); BIF_RET(erts_make_integer((Uint) tcw, p)); } @@ -924,7 +924,8 @@ BIF_RETTYPE db_set_trace_control_word_1(Process *p, Eterm new) if (val != ((Uint32)val)) BIF_ERROR(p, BADARG); - old_tcw = (Uint32) erts_smp_atomic_xchg(&trace_control_word, (erts_aint_t) val); + old_tcw = (Uint32) erts_smp_atomic32_xchg_relb(&trace_control_word, + (erts_aint32_t) val); BIF_RET(erts_make_integer((Uint) old_tcw, p)); } @@ -1249,7 +1250,7 @@ void db_initialize_util(void){ sizeof(DMCGuardBif), (int (*)(const void *, const void *)) &cmp_guard_bif); match_pseudo_process_init(); - erts_smp_atomic_init(&trace_control_word, 0); + erts_smp_atomic32_init_nob(&trace_control_word, 0); } @@ -1731,6 +1732,7 @@ Eterm db_prog_match(Process *c_p, Binary *bprog, #define BEGIN_ATOMIC_TRACE(p) \ do { \ if (! atomic_trace) { \ + erts_refc_inc(&bprog->refc, 2); \ erts_smp_proc_unlock((p), ERTS_PROC_LOCK_MAIN); \ erts_smp_block_system(0); \ atomic_trace = !0; \ @@ -1741,6 +1743,9 @@ Eterm db_prog_match(Process *c_p, Binary *bprog, if (atomic_trace) { \ erts_smp_release_system(); \ erts_smp_proc_lock((p), ERTS_PROC_LOCK_MAIN); \ + if (erts_refc_dectest(&bprog->refc, 0) == 0) {\ + erts_bin_free(bprog); \ + } \ atomic_trace = 0; \ } \ } while (0) diff --git a/erts/emulator/beam/erl_goodfit_alloc.c b/erts/emulator/beam/erl_goodfit_alloc.c index 76b206d76f..1cc508ac5a 100644 --- a/erts/emulator/beam/erl_goodfit_alloc.c +++ b/erts/emulator/beam/erl_goodfit_alloc.c @@ -163,10 +163,10 @@ BKT_MIN_SZ(GFAllctr_t *gfallctr, int ix) /* Prototypes of callback functions */ static Block_t * get_free_block (Allctr_t *, Uint, - Block_t *, Uint); -static void link_free_block (Allctr_t *, Block_t *); -static void unlink_free_block (Allctr_t *, Block_t *); -static void update_last_aux_mbc (Allctr_t *, Carrier_t *); + Block_t *, Uint, Uint32); +static void link_free_block (Allctr_t *, Block_t *, Uint32); +static void unlink_free_block (Allctr_t *, Block_t *, Uint32); +static void update_last_aux_mbc (Allctr_t *, Carrier_t *, Uint32); static Eterm info_options (Allctr_t *, char *, int *, void *, Uint **, Uint *); static void init_atoms (void); @@ -197,6 +197,8 @@ erts_gfalc_start(GFAllctr_t *gfallctr, is a struct). */ Allctr_t *allctr = (Allctr_t *) gfallctr; + init->sbmbct = 0; /* Small mbc not yet supported by goodfit */ + sys_memcpy((void *) gfallctr, (void *) &nulled_state, sizeof(GFAllctr_t)); allctr->mbc_header_size = sizeof(Carrier_t); @@ -379,7 +381,7 @@ search_bucket(Allctr_t *allctr, int ix, Uint size) static Block_t * get_free_block(Allctr_t *allctr, Uint size, - Block_t *cand_blk, Uint cand_size) + Block_t *cand_blk, Uint cand_size, Uint32 flags) { GFAllctr_t *gfallctr = (GFAllctr_t *) allctr; int unsafe_bi, min_bi; @@ -398,7 +400,7 @@ get_free_block(Allctr_t *allctr, Uint size, if (blk) { if (cand_blk && cand_size <= BLK_SZ(blk)) return NULL; /* cand_blk was better */ - unlink_free_block(allctr, blk); + unlink_free_block(allctr, blk, flags); return blk; } if (min_bi < NO_OF_BKTS - 1) { @@ -418,14 +420,14 @@ get_free_block(Allctr_t *allctr, Uint size, ASSERT(blk); if (cand_blk && cand_size <= BLK_SZ(blk)) return NULL; /* cand_blk was better */ - unlink_free_block(allctr, blk); + unlink_free_block(allctr, blk, flags); return blk; } static void -link_free_block(Allctr_t *allctr, Block_t *block) +link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) { GFAllctr_t *gfallctr = (GFAllctr_t *) allctr; GFFreeBlock_t *blk = (GFFreeBlock_t *) block; @@ -446,7 +448,7 @@ link_free_block(Allctr_t *allctr, Block_t *block) } static void -unlink_free_block(Allctr_t *allctr, Block_t *block) +unlink_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) { GFAllctr_t *gfallctr = (GFAllctr_t *) allctr; GFFreeBlock_t *blk = (GFFreeBlock_t *) block; @@ -467,7 +469,7 @@ unlink_free_block(Allctr_t *allctr, Block_t *block) } static void -update_last_aux_mbc(Allctr_t *allctr, Carrier_t *mbc) +update_last_aux_mbc(Allctr_t *allctr, Carrier_t *mbc, Uint32 flags) { GFAllctr_t *gfallctr = (GFAllctr_t *) allctr; diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index 0a57eb6d88..5f3f653e99 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1997-2010. All Rights Reserved. + * Copyright Ericsson AB 1997-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -69,7 +69,8 @@ static void erl_init(int ncpu); #define ERTS_MIN_COMPAT_REL 7 #ifdef ERTS_SMP -erts_smp_atomic_t erts_writing_erl_crash_dump; +erts_smp_atomic32_t erts_writing_erl_crash_dump; +erts_tsd_key_t erts_is_crash_dumping_key; #else volatile int erts_writing_erl_crash_dump = 0; #endif @@ -323,7 +324,7 @@ init_shared_memory(int argc, char **argv) #endif global_gen_gcs = 0; - global_max_gen_gcs = (Uint16) erts_smp_atomic32_read(&erts_max_gen_gcs); + global_max_gen_gcs = (Uint16) erts_smp_atomic32_read_nob(&erts_max_gen_gcs); global_gc_flags = erts_default_process_flags; erts_global_offheap.mso = NULL; @@ -646,12 +647,14 @@ early_init(int *argc, char **argv) /* erts_lc_init(); #endif #ifdef ERTS_SMP - erts_smp_atomic_init(&erts_writing_erl_crash_dump, 0L); + erts_smp_atomic32_init_nob(&erts_writing_erl_crash_dump, 0L); + erts_tsd_key_create(&erts_is_crash_dumping_key); #else erts_writing_erl_crash_dump = 0; #endif - erts_smp_atomic32_init(&erts_max_gen_gcs, (erts_aint32_t) ((Uint16) -1)); + erts_smp_atomic32_init_nob(&erts_max_gen_gcs, + (erts_aint32_t) ((Uint16) -1)); erts_pre_init_process(); #if defined(USE_THREADS) && !defined(ERTS_SMP) @@ -856,7 +859,8 @@ erl_start(int argc, char **argv) envbufsz = sizeof(envbuf); if (erts_sys_getenv("ERL_FULLSWEEP_AFTER", envbuf, &envbufsz) == 0) { Uint16 max_gen_gcs = atoi(envbuf); - erts_smp_atomic32_set(&erts_max_gen_gcs, (erts_aint32_t) max_gen_gcs); + erts_smp_atomic32_set_nob(&erts_max_gen_gcs, + (erts_aint32_t) max_gen_gcs); } envbufsz = sizeof(envbuf); diff --git a/erts/emulator/beam/erl_instrument.c b/erts/emulator/beam/erl_instrument.c index f3f3c22933..04ea004ef7 100644 --- a/erts/emulator/beam/erl_instrument.c +++ b/erts/emulator/beam/erl_instrument.c @@ -1186,6 +1186,8 @@ erts_instr_init(int stat, int map_stat) sys_memzero((void *) stats->n, sizeof(Stat_t)*(ERTS_ALC_N_MAX+1)); for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) { + if (ERTS_IS_SBMBC_ALLOCATOR_NO__(i)) + continue; if (erts_allctrs_info[i].enabled) stats->ap[i] = &stats->a[i]; else @@ -1199,6 +1201,8 @@ erts_instr_init(int stat, int map_stat) erts_instr_memory_map = 1; erts_instr_stat = 1; for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) { + if (ERTS_IS_SBMBC_ALLOCATOR_NO__(i)) + continue; erts_allctrs[i].alloc = map_stat_alloc; erts_allctrs[i].realloc = map_stat_realloc; erts_allctrs[i].free = map_stat_free; @@ -1209,6 +1213,8 @@ erts_instr_init(int stat, int map_stat) else { erts_instr_stat = 1; for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) { + if (ERTS_IS_SBMBC_ALLOCATOR_NO__(i)) + continue; erts_allctrs[i].alloc = stat_alloc; erts_allctrs[i].realloc = stat_realloc; erts_allctrs[i].free = stat_free; diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c index 9180508a49..587d82f2bb 100644 --- a/erts/emulator/beam/erl_lock_check.c +++ b/erts/emulator/beam/erl_lock_check.c @@ -153,6 +153,7 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "instr", NULL }, { "fix_alloc", "index" }, { "alcu_allocator", "index" }, + { "sbmbc_alloc", "index" }, { "alcu_delayed_free", "index" }, { "mseg", NULL }, #if HALFWORD_HEAP diff --git a/erts/emulator/beam/erl_monitors.c b/erts/emulator/beam/erl_monitors.c index 9751b5d77c..47597f302b 100644 --- a/erts/emulator/beam/erl_monitors.c +++ b/erts/emulator/beam/erl_monitors.c @@ -125,7 +125,7 @@ static ErtsMonitor *create_monitor(Uint type, Eterm ref, Eterm pid, Eterm name) } else { n = (ErtsMonitor *) erts_alloc(ERTS_ALC_T_MONITOR_LH, mon_size*sizeof(Uint)); - erts_smp_atomic_add(&tot_link_lh_size, mon_size*sizeof(Uint)); + erts_smp_atomic_add_nob(&tot_link_lh_size, mon_size*sizeof(Uint)); } hp = n->heap; @@ -156,7 +156,7 @@ static ErtsLink *create_link(Uint type, Eterm pid) } else { n = (ErtsLink *) erts_alloc(ERTS_ALC_T_NLINK_LH, lnk_size*sizeof(Uint)); - erts_smp_atomic_add(&tot_link_lh_size, lnk_size*sizeof(Uint)); + erts_smp_atomic_add_nob(&tot_link_lh_size, lnk_size*sizeof(Uint)); } hp = n->heap; @@ -191,13 +191,13 @@ static ErtsSuspendMonitor *create_suspend_monitor(Eterm pid) void erts_init_monitors(void) { - erts_smp_atomic_init(&tot_link_lh_size, 0); + erts_smp_atomic_init_nob(&tot_link_lh_size, 0); } Uint erts_tot_link_lh_size(void) { - return (Uint) erts_smp_atomic_read(&tot_link_lh_size); + return (Uint) erts_smp_atomic_read_nob(&tot_link_lh_size); } void erts_destroy_monitor(ErtsMonitor *mon) @@ -222,7 +222,7 @@ void erts_destroy_monitor(ErtsMonitor *mon) erts_free(ERTS_ALC_T_MONITOR_SH, (void *) mon); } else { erts_free(ERTS_ALC_T_MONITOR_LH, (void *) mon); - erts_smp_atomic_add(&tot_link_lh_size, -1*mon_size*sizeof(Uint)); + erts_smp_atomic_add_nob(&tot_link_lh_size, -1*mon_size*sizeof(Uint)); } } @@ -244,7 +244,7 @@ void erts_destroy_link(ErtsLink *lnk) erts_free(ERTS_ALC_T_NLINK_SH, (void *) lnk); } else { erts_free(ERTS_ALC_T_NLINK_LH, (void *) lnk); - erts_smp_atomic_add(&tot_link_lh_size, -1*lnk_size*sizeof(Uint)); + erts_smp_atomic_add_nob(&tot_link_lh_size, -1*lnk_size*sizeof(Uint)); } } diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 68421b4387..ea781a6cd0 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -833,8 +833,11 @@ ERL_NIF_TERM enif_make_uint(ErlNifEnv* env, unsigned i) ERL_NIF_TERM enif_make_long(ErlNifEnv* env, long i) { + if (IS_SSMALL(i)) { + return make_small(i); + } #if SIZEOF_LONG == ERTS_SIZEOF_ETERM - return IS_SSMALL(i) ? make_small(i) : small_to_big(i, alloc_heap(env,2)); + return small_to_big(i, alloc_heap(env,2)); #elif SIZEOF_LONG == 8 ensure_heap(env,3); return erts_sint64_to_big(i, &env->hp); @@ -843,8 +846,11 @@ ERL_NIF_TERM enif_make_long(ErlNifEnv* env, long i) ERL_NIF_TERM enif_make_ulong(ErlNifEnv* env, unsigned long i) { + if (IS_USMALL(0,i)) { + return make_small(i); + } #if SIZEOF_LONG == ERTS_SIZEOF_ETERM - return IS_USMALL(0,i) ? make_small(i) : uint_to_big(i,alloc_heap(env,2)); + return uint_to_big(i,alloc_heap(env,2)); #elif SIZEOF_LONG == 8 ensure_heap(env,3); return erts_uint64_to_big(i, &env->hp); @@ -1007,6 +1013,29 @@ void enif_system_info(ErlNifSysInfo *sip, size_t si_size) driver_system_info(sip, si_size); } +int enif_make_reverse_list(ErlNifEnv* env, ERL_NIF_TERM term, ERL_NIF_TERM *list) { + Eterm *listptr, ret = NIL, *hp; + + if (is_nil(term)) { + *list = term; + return 1; + } + + ret = NIL; + + while (is_not_nil(term)) { + if (is_not_list(term)) { + return 0; + } + hp = alloc_heap(env, 2); + listptr = list_val(term); + ret = CONS(hp, CAR(listptr), ret); + term = CDR(listptr); + } + *list = ret; + return 1; +} + ErlNifMutex* enif_mutex_create(char *name) { return erl_drv_mutex_create(name); } void enif_mutex_destroy(ErlNifMutex *mtx) { erl_drv_mutex_destroy(mtx); } diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h index d028567faf..fea527f954 100644 --- a/erts/emulator/beam/erl_nif.h +++ b/erts/emulator/beam/erl_nif.h @@ -32,9 +32,10 @@ ** 2.0: R14A ** 2.1: R14B02 "vm_variant" ** 2.2: R14B03 enif_is_exception +** 2.3: R15 enif_make_reverse_list */ #define ERL_NIF_MAJOR_VERSION 2 -#define ERL_NIF_MINOR_VERSION 2 +#define ERL_NIF_MINOR_VERSION 3 #include <stdlib.h> diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h index c991b61abe..4af9f61000 100644 --- a/erts/emulator/beam/erl_nif_api_funcs.h +++ b/erts/emulator/beam/erl_nif_api_funcs.h @@ -136,6 +136,7 @@ ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_int64,(ErlNifEnv*, ErlNifSInt64)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_uint64,(ErlNifEnv*, ErlNifUInt64)); #endif ERL_NIF_API_FUNC_DECL(int,enif_is_exception,(ErlNifEnv*, ERL_NIF_TERM term)); +ERL_NIF_API_FUNC_DECL(int,enif_make_reverse_list,(ErlNifEnv*, ERL_NIF_TERM term, ERL_NIF_TERM *list)); /* ** Add new entries here to keep compatibility on Windows!!! @@ -256,6 +257,7 @@ ERL_NIF_API_FUNC_DECL(int,enif_is_exception,(ErlNifEnv*, ERL_NIF_TERM term)); #endif # define enif_is_exception ERL_NIF_API_FUNC_MACRO(enif_is_exception) +# define enif_make_reverse_list ERL_NIF_API_FUNC_MACRO(enif_make_reverse_list) /* ** Add new entries here diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c index 6daa127d23..af3873995e 100644 --- a/erts/emulator/beam/erl_node_tables.c +++ b/erts/emulator/beam/erl_node_tables.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2010. All Rights Reserved. + * Copyright Ericsson AB 2001-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -118,7 +118,7 @@ dist_table_alloc(void *dep_tmpl) dep->finalized_out_queue.first = NULL; dep->finalized_out_queue.last = NULL; - erts_smp_atomic_init(&dep->dist_cmd_scheduled, 0); + erts_smp_atomic_init_nob(&dep->dist_cmd_scheduled, 0); erts_port_task_handle_init(&dep->dist_cmd); dep->send = NULL; dep->cache = NULL; @@ -767,7 +767,7 @@ void erts_init_node_tables(void) erts_this_dist_entry->finalized_out_queue.first = NULL; erts_this_dist_entry->finalized_out_queue.last = NULL; - erts_smp_atomic_init(&erts_this_dist_entry->dist_cmd_scheduled, 0); + erts_smp_atomic_init_nob(&erts_this_dist_entry->dist_cmd_scheduled, 0); erts_port_task_handle_init(&erts_this_dist_entry->dist_cmd); erts_this_dist_entry->send = NULL; erts_this_dist_entry->cache = NULL; diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c index e6b55c45e4..6aa5161b08 100644 --- a/erts/emulator/beam/erl_port_task.c +++ b/erts/emulator/beam/erl_port_task.c @@ -121,7 +121,7 @@ ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(port_taskq, static ERTS_INLINE ErtsPortTask * handle2task(ErtsPortTaskHandle *pthp) { - return (ErtsPortTask *) erts_smp_atomic_read(pthp); + return (ErtsPortTask *) erts_smp_atomic_read_nob(pthp); } static ERTS_INLINE void @@ -129,7 +129,7 @@ reset_handle(ErtsPortTask *ptp) { if (ptp->handle) { ASSERT(ptp == handle2task(ptp->handle)); - erts_smp_atomic_set(ptp->handle, (erts_aint_t) NULL); + erts_smp_atomic_set_nob(ptp->handle, (erts_aint_t) NULL); } } @@ -138,7 +138,7 @@ set_handle(ErtsPortTask *ptp, ErtsPortTaskHandle *pthp) { ptp->handle = pthp; if (pthp) { - erts_smp_atomic_set(pthp, (erts_aint_t) ptp); + erts_smp_atomic_set_nob(pthp, (erts_aint_t) ptp); ASSERT(ptp == handle2task(ptp->handle)); } } @@ -479,8 +479,8 @@ erts_port_task_abort(Eterm id, ErtsPortTaskHandle *pthp) case ERTS_PORT_TASK_INPUT: case ERTS_PORT_TASK_OUTPUT: case ERTS_PORT_TASK_EVENT: - ASSERT(erts_smp_atomic_read(&erts_port_task_outstanding_io_tasks) > 0); - erts_smp_atomic_dec(&erts_port_task_outstanding_io_tasks); + ASSERT(erts_smp_atomic_read_nob(&erts_port_task_outstanding_io_tasks) > 0); + erts_smp_atomic_dec_relb(&erts_port_task_outstanding_io_tasks); break; default: break; @@ -568,7 +568,7 @@ erts_port_task_schedule(Eterm id, ErtsRunQueue *xrunq = erts_check_emigration_need(runq, ERTS_PORT_PRIO_LEVEL); if (xrunq) { /* Port emigrated ... */ - erts_smp_atomic_set(&pp->run_queue, (erts_aint_t) xrunq); + erts_smp_atomic_set_nob(&pp->run_queue, (erts_aint_t) xrunq); erts_smp_runq_unlock(runq); runq = xrunq; } @@ -594,7 +594,7 @@ erts_port_task_schedule(Eterm id, case ERTS_PORT_TASK_INPUT: case ERTS_PORT_TASK_OUTPUT: case ERTS_PORT_TASK_EVENT: - erts_smp_atomic_inc(&erts_port_task_outstanding_io_tasks); + erts_smp_atomic_inc_relb(&erts_port_task_outstanding_io_tasks); /* Fall through... */ default: enqueue_task(pp->sched.taskq, ptp); @@ -662,7 +662,7 @@ erts_port_task_free_port(Port *pp) pp->status |= ERTS_PORT_SFLG_FREE_SCHEDULED; erts_may_save_closed_port(pp); erts_smp_port_state_unlock(pp); - ERTS_SMP_LC_ASSERT(erts_smp_atomic_read(&pp->refc) > 1); + ERTS_SMP_LC_ASSERT(erts_smp_atomic_read_nob(&pp->refc) > 1); ptp->type = ERTS_PORT_TASK_FREE; ptp->event = (ErlDrvEvent) -1; ptp->event_data = NULL; @@ -684,9 +684,9 @@ erts_port_task_free_port(Port *pp) erts_may_save_closed_port(pp); erts_smp_port_state_unlock(pp); #ifdef ERTS_SMP - erts_smp_atomic_dec(&pp->refc); /* Not alive */ + erts_smp_atomic_dec_nob(&pp->refc); /* Not alive */ #endif - ERTS_SMP_LC_ASSERT(erts_smp_atomic_read(&pp->refc) > 0); /* Lock */ + ERTS_SMP_LC_ASSERT(erts_smp_atomic_read_nob(&pp->refc) > 0); /* Lock */ handle_remaining_tasks(runq, pp); /* May release runq lock */ ASSERT(!pp->sched.exe_taskq && (!ptqp || !ptqp->first)); pp->sched.taskq = NULL; @@ -724,7 +724,7 @@ resume_after_block(void *vd) ErtsPortTaskExeBlockData *d = (ErtsPortTaskExeBlockData *) vd; erts_smp_runq_lock(d->runq); if (d->resp) - *d->resp = (erts_smp_atomic_read(&erts_port_task_outstanding_io_tasks) + *d->resp = (erts_smp_atomic_read_nob(&erts_port_task_outstanding_io_tasks) != (erts_aint_t) 0); } @@ -832,8 +832,8 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) ASSERT(!ptqp->first && (!pp->sched.taskq || !pp->sched.taskq->first)); #ifdef ERTS_SMP - erts_smp_atomic_dec(&pp->refc); /* Not alive */ - ERTS_SMP_LC_ASSERT(erts_smp_atomic_read(&pp->refc) > 0); /* Lock */ + erts_smp_atomic_dec_nob(&pp->refc); /* Not alive */ + ERTS_SMP_LC_ASSERT(erts_smp_atomic_read_nob(&pp->refc) > 0); /* Lock */ #else erts_port_status_bor_set(pp, ERTS_PORT_SFLG_FREE); #endif @@ -906,14 +906,16 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) erts_unblock_fpe(fpe_was_unmasked); if (io_tasks_executed) { - ASSERT(erts_smp_atomic_read(&erts_port_task_outstanding_io_tasks) >= io_tasks_executed); - erts_smp_atomic_add(&erts_port_task_outstanding_io_tasks, -1*io_tasks_executed); + ASSERT(erts_smp_atomic_read_nob(&erts_port_task_outstanding_io_tasks) + >= io_tasks_executed); + erts_smp_atomic_add_relb(&erts_port_task_outstanding_io_tasks, + -1*io_tasks_executed); } *curr_port_pp = NULL; #ifdef ERTS_SMP - ASSERT(runq == (ErtsRunQueue *) erts_smp_atomic_read(&pp->run_queue)); + ASSERT(runq == (ErtsRunQueue *) erts_smp_atomic_read_nob(&pp->run_queue)); #endif if (!pp->sched.taskq) { @@ -940,7 +942,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) } else { /* Port emigrated ... */ - erts_smp_atomic_set(&pp->run_queue, (erts_aint_t) xrunq); + erts_smp_atomic_set_nob(&pp->run_queue, (erts_aint_t) xrunq); enqueue_port(xrunq, pp); ASSERT(pp->sched.exe_taskq); pp->sched.exe_taskq = NULL; @@ -951,7 +953,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) port_was_enqueued = 1; } - res = (erts_smp_atomic_read(&erts_port_task_outstanding_io_tasks) + res = (erts_smp_atomic_read_nob(&erts_port_task_outstanding_io_tasks) != (erts_aint_t) 0); ERTS_PT_CHK_PRES_PORTQ(runq, pp); @@ -972,13 +974,13 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) { erts_aint_t refc; erts_smp_mtx_unlock(pp->lock); - refc = erts_smp_atomic_dectest(&pp->refc); + refc = erts_smp_atomic_dec_read_nob(&pp->refc); ASSERT(refc >= 0); if (refc == 0) { erts_smp_runq_unlock(runq); erts_port_cleanup(pp); /* Might aquire runq lock */ erts_smp_runq_lock(runq); - res = (erts_smp_atomic_read(&erts_port_task_outstanding_io_tasks) + res = (erts_smp_atomic_read_nob(&erts_port_task_outstanding_io_tasks) != (erts_aint_t) 0); } } @@ -1107,12 +1109,12 @@ erts_port_migrate(Port *prt, int *prt_locked, /* Refuse to migrate to a suspended run queue */ if (to_rq->flags & ERTS_RUNQ_FLG_SUSPENDED) return ERTS_MIGRATE_FAILED_RUNQ_SUSPENDED; - if (from_rq != (ErtsRunQueue *) erts_smp_atomic_read(&prt->run_queue)) + if (from_rq != (ErtsRunQueue *) erts_smp_atomic_read_nob(&prt->run_queue)) return ERTS_MIGRATE_FAILED_RUNQ_CHANGED; if (!ERTS_PORT_IS_IN_RUNQ(from_rq, prt)) return ERTS_MIGRATE_FAILED_NOT_IN_RUNQ; dequeue_port(from_rq, prt); - erts_smp_atomic_set(&prt->run_queue, (erts_aint_t) to_rq); + erts_smp_atomic_set_nob(&prt->run_queue, (erts_aint_t) to_rq); enqueue_port(to_rq, prt); return ERTS_MIGRATE_SUCCESS; } @@ -1125,7 +1127,8 @@ erts_port_migrate(Port *prt, int *prt_locked, void erts_port_task_init(void) { - erts_smp_atomic_init(&erts_port_task_outstanding_io_tasks, (erts_aint_t) 0); + erts_smp_atomic_init_nob(&erts_port_task_outstanding_io_tasks, + (erts_aint_t) 0); init_port_task_alloc(); init_port_taskq_alloc(); } diff --git a/erts/emulator/beam/erl_port_task.h b/erts/emulator/beam/erl_port_task.h index 3e2c5f07ab..d7104e1143 100644 --- a/erts/emulator/beam/erl_port_task.h +++ b/erts/emulator/beam/erl_port_task.h @@ -79,13 +79,13 @@ ERTS_GLB_INLINE int erts_port_task_have_outstanding_io_tasks(void); ERTS_GLB_INLINE void erts_port_task_handle_init(ErtsPortTaskHandle *pthp) { - erts_smp_atomic_init(pthp, (erts_aint_t) NULL); + erts_smp_atomic_init_nob(pthp, (erts_aint_t) NULL); } ERTS_GLB_INLINE int erts_port_task_is_scheduled(ErtsPortTaskHandle *pthp) { - return ((void *) erts_smp_atomic_read(pthp)) != NULL; + return ((void *) erts_smp_atomic_read_nob(pthp)) != NULL; } ERTS_GLB_INLINE void @@ -102,8 +102,8 @@ erts_port_task_init_sched(ErtsPortTaskSched *ptsp) ERTS_GLB_INLINE int erts_port_task_have_outstanding_io_tasks(void) { - ERTS_THR_MEMORY_BARRIER; - return erts_smp_atomic_read(&erts_port_task_outstanding_io_tasks) != 0; + return (erts_smp_atomic_read_acqb(&erts_port_task_outstanding_io_tasks) + != 0); } #endif /* ERTS_INCLUDE_SCHEDULER_INTERNALS */ diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 2704359a8f..bbdcf79d00 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -134,15 +134,15 @@ int erts_disable_proc_not_running_opt; #ifndef DEBUG #define ERTS_SCHDLR_SSPND_CHNG_SET(VAL, OLD_VAL) \ - erts_smp_atomic32_set(&schdlr_sspnd.changing, (VAL)) + erts_smp_atomic32_set_nob(&schdlr_sspnd.changing, (VAL)) #else #define ERTS_SCHDLR_SSPND_CHNG_SET(VAL, OLD_VAL) \ do { \ erts_aint32_t old_val__; \ - old_val__ = erts_smp_atomic32_xchg(&schdlr_sspnd.changing, \ - (VAL)); \ + old_val__ = erts_smp_atomic32_xchg_nob(&schdlr_sspnd.changing, \ + (VAL)); \ ASSERT(old_val__ == (OLD_VAL)); \ } while (0) @@ -158,7 +158,7 @@ static struct { erts_smp_atomic32_t changing; erts_smp_atomic32_t active; struct { - erts_smp_atomic32_t ongoing; + int ongoing; long wait_active; ErtsProcList *procs; } msb; /* Multi Scheduling Block */ @@ -410,7 +410,7 @@ erts_init_process(int ncpu) init_proclist_alloc(); - erts_smp_atomic32_init(&process_count, 0); + erts_smp_atomic32_init_nob(&process_count, 0); if (erts_use_r9_pids_ports) { proc_bits = ERTS_R9_PROC_BITS; @@ -694,8 +694,8 @@ erts_smp_schedule_misc_aux_work(int ignore_self, erts_smp_mtx_unlock(&misc_aux_work_queues[ix].data.mtx); ssi = ERTS_SCHED_SLEEP_INFO_IX(ix); - aux_work = erts_smp_atomic32_bor(&ssi->aux_work, - ERTS_SSI_AUX_WORK_MISC); + aux_work = erts_smp_atomic32_read_bor_nob(&ssi->aux_work, + ERTS_SSI_AUX_WORK_MISC); if ((aux_work & ERTS_SSI_AUX_WORK_MISC) == 0) erts_sched_poke(ssi); } @@ -711,8 +711,8 @@ erts_smp_notify_check_children_needed(void) erts_aint32_t aux_work; ErtsSchedulerSleepInfo *ssi; ssi = ERTS_SCHED_SLEEP_INFO_IX(i); - aux_work = erts_smp_atomic32_bor(&ssi->aux_work, - ERTS_SSI_AUX_WORK_CHECK_CHILDREN); + aux_work = erts_smp_atomic32_read_bor_nob(&ssi->aux_work, + ERTS_SSI_AUX_WORK_CHECK_CHILDREN); if (!(aux_work & ERTS_SSI_AUX_WORK_CHECK_CHILDREN)) erts_sched_poke(ssi); } @@ -727,15 +727,15 @@ blockable_aux_work(ErtsSchedulerData *esdp, { if (aux_work & ERTS_SSI_BLOCKABLE_AUX_WORK_MASK) { if (aux_work & ERTS_SSI_AUX_WORK_MISC) { - aux_work = erts_smp_atomic32_band(&ssi->aux_work, - ~ERTS_SSI_AUX_WORK_MISC); + aux_work = erts_smp_atomic32_read_band_nob(&ssi->aux_work, + ~ERTS_SSI_AUX_WORK_MISC); aux_work &= ~ERTS_SSI_AUX_WORK_MISC; handle_misc_aux_work(esdp); } #ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN if (aux_work & ERTS_SSI_AUX_WORK_CHECK_CHILDREN) { - aux_work = erts_smp_atomic32_band(&ssi->aux_work, - ~ERTS_SSI_AUX_WORK_CHECK_CHILDREN); + aux_work = erts_smp_atomic32_band_nob(&ssi->aux_work, + ~ERTS_SSI_AUX_WORK_CHECK_CHILDREN); aux_work &= ~ERTS_SSI_AUX_WORK_CHECK_CHILDREN; erts_check_children(); } @@ -815,7 +815,7 @@ erts_active_schedulers(void) static ERTS_INLINE void clear_sys_scheduling(void) { - erts_smp_atomic32_set_relb(&doing_sys_schedule, 0); + erts_smp_atomic32_set_mb(&doing_sys_schedule, 0); } static ERTS_INLINE int @@ -882,42 +882,43 @@ sched_active(Uint no, ErtsRunQueue *rq) static int ERTS_INLINE ongoing_multi_scheduling_block(void) { - return erts_smp_atomic32_read(&schdlr_sspnd.msb.ongoing) != 0; + ERTS_SMP_LC_ASSERT(erts_lc_mtx_is_locked(&schdlr_sspnd.mtx)); + return schdlr_sspnd.msb.ongoing; } static ERTS_INLINE void empty_runq(ErtsRunQueue *rq) { - erts_aint32_t oifls = erts_smp_atomic32_band(&rq->info_flags, - ~ERTS_RUNQ_IFLG_NONEMPTY); + erts_aint32_t oifls = erts_smp_atomic32_read_band_nob(&rq->info_flags, + ~ERTS_RUNQ_IFLG_NONEMPTY); if (oifls & ERTS_RUNQ_IFLG_NONEMPTY) { #ifdef DEBUG - erts_aint32_t empty = erts_smp_atomic32_read(&no_empty_run_queues); + erts_aint32_t empty = erts_smp_atomic32_read_nob(&no_empty_run_queues); /* * For a short period of time no_empty_run_queues may have * been increased twice for a specific run queue. */ ASSERT(0 <= empty && empty < 2*erts_no_run_queues); #endif - erts_smp_atomic32_inc(&no_empty_run_queues); + erts_smp_atomic32_inc_relb(&no_empty_run_queues); } } static ERTS_INLINE void non_empty_runq(ErtsRunQueue *rq) { - erts_aint32_t oifls = erts_smp_atomic32_bor(&rq->info_flags, - ERTS_RUNQ_IFLG_NONEMPTY); + erts_aint32_t oifls = erts_smp_atomic32_read_bor_nob(&rq->info_flags, + ERTS_RUNQ_IFLG_NONEMPTY); if (!(oifls & ERTS_RUNQ_IFLG_NONEMPTY)) { #ifdef DEBUG - erts_aint32_t empty = erts_smp_atomic32_read(&no_empty_run_queues); + erts_aint32_t empty = erts_smp_atomic32_read_nob(&no_empty_run_queues); /* * For a short period of time no_empty_run_queues may have * been increased twice for a specific run queue. */ ASSERT(0 < empty && empty <= 2*erts_no_run_queues); #endif - erts_smp_atomic32_dec(&no_empty_run_queues); + erts_smp_atomic32_dec_relb(&no_empty_run_queues); } } @@ -930,7 +931,7 @@ sched_prep_spin_wait(ErtsSchedulerSleepInfo *ssi) erts_aint32_t xflgs = 0; do { - oflgs = erts_smp_atomic32_cmpxchg(&ssi->flags, nflgs, xflgs); + oflgs = erts_smp_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs); if (oflgs == xflgs) return nflgs; xflgs = oflgs; @@ -947,7 +948,7 @@ sched_prep_cont_spin_wait(ErtsSchedulerSleepInfo *ssi) erts_aint32_t xflgs = ERTS_SSI_FLG_WAITING; do { - oflgs = erts_smp_atomic32_cmpxchg(&ssi->flags, nflgs, xflgs); + oflgs = erts_smp_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs); if (oflgs == xflgs) return nflgs; xflgs = oflgs; @@ -989,7 +990,7 @@ sched_set_sleeptype(ErtsSchedulerSleepInfo *ssi, erts_aint32_t sleep_type) erts_tse_reset(ssi->event); while (1) { - oflgs = erts_smp_atomic32_cmpxchg(&ssi->flags, nflgs, xflgs); + oflgs = erts_smp_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs); if (oflgs == xflgs) return nflgs; if ((oflgs & (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING)) @@ -1049,7 +1050,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) tse_wait: #ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK - aux_work = erts_smp_atomic32_read(&ssi->aux_work); + aux_work = erts_smp_atomic32_read_nob(&ssi->aux_work); tse_blockable_aux_work: aux_work = blockable_aux_work(esdp, ssi, aux_work); #endif @@ -1059,7 +1060,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) #ifdef ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK #ifndef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK - aux_work = erts_smp_atomic32_read(&ssi->aux_work); + aux_work = erts_smp_atomic32_read_nob(&ssi->aux_work); #endif nonblockable_aux_work(esdp, ssi, aux_work); #endif @@ -1092,7 +1093,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) } #ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK - aux_work = erts_smp_atomic32_read(&ssi->aux_work); + aux_work = erts_smp_atomic32_read_nob(&ssi->aux_work); if (aux_work & ERTS_SSI_BLOCKABLE_AUX_WORK_MASK) { erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); goto tse_blockable_aux_work; @@ -1104,7 +1105,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); if (flgs & ~ERTS_SSI_FLG_SUSPENDED) - erts_smp_atomic32_band(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); + erts_smp_atomic32_read_band_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); erts_smp_runq_lock(rq); sched_active(esdp->no, rq); @@ -1136,12 +1137,12 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) sys_aux_work: #ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK - aux_work = erts_smp_atomic32_read(&ssi->aux_work); + aux_work = erts_smp_atomic32_read_nob(&ssi->aux_work); aux_work = blockable_aux_work(esdp, ssi, aux_work); #endif #ifdef ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK #ifndef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK - aux_work = erts_smp_atomic32_read(&ssi->aux_work); + aux_work = erts_smp_atomic32_read_nob(&ssi->aux_work); #endif nonblockable_aux_work(esdp, ssi, aux_work); #endif @@ -1239,7 +1240,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) sys_locked_woken: clear_sys_scheduling(); if (flgs & ~ERTS_SSI_FLG_SUSPENDED) - erts_smp_atomic32_band(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); + erts_smp_atomic32_read_band_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); sched_active_sys(esdp->no, rq); } } @@ -1255,7 +1256,7 @@ ssi_flags_set_wake(ErtsSchedulerSleepInfo *ssi) erts_aint32_t nflgs = 0; erts_aint32_t xflgs = ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING; while (1) { - oflgs = erts_smp_atomic32_cmpxchg(&ssi->flags, nflgs, xflgs); + oflgs = erts_smp_atomic32_cmpxchg_relb(&ssi->flags, nflgs, xflgs); if (oflgs == xflgs) return oflgs; nflgs = oflgs & ERTS_SSI_FLG_SUSPENDED; @@ -1298,7 +1299,6 @@ wake_scheduler(ErtsRunQueue *rq, int incq, int one) erts_smp_spin_unlock(&sl->lock); - ERTS_THR_MEMORY_BARRIER; flgs = ssi_flags_set_wake(ssi); erts_sched_finish_poke(ssi, flgs); @@ -1344,13 +1344,13 @@ init_no_runqs(int active, int used) { erts_aint32_t no_runqs = (erts_aint32_t) (active & ERTS_NO_RUNQS_MASK); no_runqs |= (erts_aint32_t) ((used & ERTS_NO_RUNQS_MASK) << ERTS_NO_USED_RUNQS_SHIFT); - erts_smp_atomic32_init(&balance_info.no_runqs, no_runqs); + erts_smp_atomic32_init_nob(&balance_info.no_runqs, no_runqs); } static ERTS_INLINE void get_no_runqs(int *active, int *used) { - erts_aint32_t no_runqs = erts_smp_atomic32_read(&balance_info.no_runqs); + erts_aint32_t no_runqs = erts_smp_atomic32_read_nob(&balance_info.no_runqs); if (active) *active = (int) (no_runqs & ERTS_NO_RUNQS_MASK); if (used) @@ -1360,11 +1360,12 @@ get_no_runqs(int *active, int *used) static ERTS_INLINE void set_no_used_runqs(int used) { - erts_aint32_t exp = erts_smp_atomic32_read(&balance_info.no_runqs); + erts_aint32_t exp = erts_smp_atomic32_read_nob(&balance_info.no_runqs); while (1) { erts_aint32_t act, new; - new = (used << ERTS_NO_USED_RUNQS_SHIFT) | (exp & ERTS_NO_RUNQS_MASK); - act = erts_smp_atomic32_cmpxchg(&balance_info.no_runqs, new, exp); + new = (used & ERTS_NO_RUNQS_MASK) << ERTS_NO_USED_RUNQS_SHIFT; + new |= exp & ERTS_NO_RUNQS_MASK; + act = erts_smp_atomic32_cmpxchg_nob(&balance_info.no_runqs, new, exp); if (act == exp) break; exp = act; @@ -1374,11 +1375,12 @@ set_no_used_runqs(int used) static ERTS_INLINE void set_no_active_runqs(int active) { - erts_aint32_t exp = erts_smp_atomic32_read(&balance_info.no_runqs); + erts_aint32_t exp = erts_smp_atomic32_read_nob(&balance_info.no_runqs); while (1) { erts_aint32_t act, new; - new = (exp & (ERTS_NO_RUNQS_MASK << ERTS_NO_USED_RUNQS_SHIFT)) | active; - act = erts_smp_atomic32_cmpxchg(&balance_info.no_runqs, new, exp); + new = exp & (ERTS_NO_RUNQS_MASK << ERTS_NO_USED_RUNQS_SHIFT); + new |= active & ERTS_NO_RUNQS_MASK; + act = erts_smp_atomic32_cmpxchg_nob(&balance_info.no_runqs, new, exp); if (act == exp) break; exp = act; @@ -1388,13 +1390,14 @@ set_no_active_runqs(int active) static ERTS_INLINE int try_inc_no_active_runqs(int active) { - erts_aint32_t exp = erts_smp_atomic32_read(&balance_info.no_runqs); + erts_aint32_t exp = erts_smp_atomic32_read_nob(&balance_info.no_runqs); if (((exp >> ERTS_NO_USED_RUNQS_SHIFT) & ERTS_NO_RUNQS_MASK) < active) return 0; if ((exp & ERTS_NO_RUNQS_MASK) + 1 == active) { erts_aint32_t new, act; - new = (exp & ~ERTS_NO_RUNQS_MASK) | active; - act = erts_smp_atomic32_cmpxchg(&balance_info.no_runqs, new, exp); + new = exp & (ERTS_NO_RUNQS_MASK << ERTS_NO_USED_RUNQS_SHIFT); + new |= active & ERTS_NO_RUNQS_MASK; + act = erts_smp_atomic32_cmpxchg_nob(&balance_info.no_runqs, new, exp); if (act == exp) return 1; } @@ -1410,7 +1413,7 @@ chk_wake_sched(ErtsRunQueue *crq, int ix, int activate) if (crq->ix == ix) return 0; wrq = ERTS_RUNQ_IX(ix); - iflgs = erts_smp_atomic32_read(&wrq->info_flags); + iflgs = erts_smp_atomic32_read_nob(&wrq->info_flags); if (!(iflgs & (ERTS_RUNQ_IFLG_SUSPENDED|ERTS_RUNQ_IFLG_NONEMPTY))) { if (activate) { if (try_inc_no_active_runqs(ix+1)) { @@ -1652,15 +1655,15 @@ evacuate_run_queue(ErtsRunQueue *evac_rq, ErtsRunQueue *rq) erts_smp_runq_lock(evac_rq); - erts_smp_atomic32_bor(&evac_rq->scheduler->ssi->flags, - ERTS_SSI_FLG_SUSPENDED); + erts_smp_atomic32_read_bor_nob(&evac_rq->scheduler->ssi->flags, + ERTS_SSI_FLG_SUSPENDED); evac_rq->flags &= ~ERTS_RUNQ_FLGS_IMMIGRATE_QMASK; evac_rq->flags |= (ERTS_RUNQ_FLGS_EMIGRATE_QMASK | ERTS_RUNQ_FLGS_EVACUATE_QMASK | ERTS_RUNQ_FLG_SUSPENDED); - erts_smp_atomic32_bor(&evac_rq->info_flags, ERTS_RUNQ_IFLG_SUSPENDED); + erts_smp_atomic32_read_bor_nob(&evac_rq->info_flags, ERTS_RUNQ_IFLG_SUSPENDED); /* * Need to set up evacuation paths first since we * may release the run queue lock on evac_rq @@ -1909,7 +1912,7 @@ static ERTS_INLINE int check_possible_steal_victim(ErtsRunQueue *rq, int *rq_lockedp, int vix) { ErtsRunQueue *vrq = ERTS_RUNQ_IX(vix); - erts_aint32_t iflgs = erts_smp_atomic32_read(&vrq->info_flags); + erts_aint32_t iflgs = erts_smp_atomic32_read_nob(&vrq->info_flags); if (iflgs & ERTS_RUNQ_IFLG_NONEMPTY) return try_steal_task_from_victim(rq, rq_lockedp, vrq); else @@ -2061,7 +2064,7 @@ check_balance(ErtsRunQueue *c_rq) int forced, active, current_active, oowc, half_full_scheds, full_scheds, mmax_len, blnc_no_rqs, qix, pix, freds_hist_ix; - if (erts_smp_atomic32_xchg(&balance_info.checking_balance, 1)) { + if (erts_smp_atomic32_xchg_nob(&balance_info.checking_balance, 1)) { c_rq->check_balance_reds = INT_MAX; return; } @@ -2069,7 +2072,7 @@ check_balance(ErtsRunQueue *c_rq) get_no_runqs(NULL, &blnc_no_rqs); if (blnc_no_rqs == 1) { c_rq->check_balance_reds = INT_MAX; - erts_smp_atomic32_set(&balance_info.checking_balance, 0); + erts_smp_atomic32_set_nob(&balance_info.checking_balance, 0); return; } @@ -2077,7 +2080,7 @@ check_balance(ErtsRunQueue *c_rq) if (balance_info.halftime) { balance_info.halftime = 0; - erts_smp_atomic32_set(&balance_info.checking_balance, 0); + erts_smp_atomic32_set_nob(&balance_info.checking_balance, 0); ERTS_FOREACH_RUNQ(rq, { if (rq->waiting) @@ -2111,7 +2114,7 @@ check_balance(ErtsRunQueue *c_rq) erts_smp_mtx_unlock(&balance_info.update_mtx); erts_smp_runq_lock(c_rq); c_rq->check_balance_reds = INT_MAX; - erts_smp_atomic32_set(&balance_info.checking_balance, 0); + erts_smp_atomic32_set_nob(&balance_info.checking_balance, 0); return; } @@ -2456,7 +2459,7 @@ erts_fprintf(stderr, "--------------------------------\n"); set_no_active_runqs(active); balance_info.halftime = 1; - erts_smp_atomic32_set(&balance_info.checking_balance, 0); + erts_smp_atomic32_set_nob(&balance_info.checking_balance, 0); /* Write migration paths and reset balance statistics in all queues */ for (qix = 0; qix < blnc_no_rqs; qix++) { @@ -2598,7 +2601,7 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) erts_alloc_permanent_cache_aligned(ERTS_ALC_T_RUNQS, sizeof(ErtsAlignedRunQueue) * n); #ifdef ERTS_SMP - erts_smp_atomic32_init(&no_empty_run_queues, 0); + erts_smp_atomic32_init_nob(&no_empty_run_queues, 0); #endif erts_no_run_queues = n; @@ -2608,7 +2611,7 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); rq->ix = ix; - erts_smp_atomic32_init(&rq->info_flags, ERTS_RUNQ_IFLG_NONEMPTY); + erts_smp_atomic32_init_nob(&rq->info_flags, ERTS_RUNQ_IFLG_NONEMPTY); /* make sure that the "extra" id correponds to the schedulers * id if the esdp->no <-> ix+1 mapping change. @@ -2701,9 +2704,9 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) ssi->next = NULL; ssi->prev = NULL; #endif - erts_smp_atomic32_init(&ssi->flags, 0); + erts_smp_atomic32_init_nob(&ssi->flags, 0); ssi->event = NULL; /* initialized in sched_thread_func */ - erts_smp_atomic32_init(&ssi->aux_work, 0); + erts_smp_atomic32_init_nob(&ssi->aux_work, 0); } #endif @@ -2747,7 +2750,7 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) } #ifdef ERTS_SMP - erts_smp_atomic32_init(&esdp->chk_cpu_bind, 0); + erts_smp_atomic32_init_nob(&esdp->chk_cpu_bind, 0); #endif } @@ -2755,11 +2758,11 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) erts_smp_mtx_init(&schdlr_sspnd.mtx, "schdlr_sspnd"); erts_smp_cnd_init(&schdlr_sspnd.cnd); - erts_smp_atomic32_init(&schdlr_sspnd.changing, 0); + erts_smp_atomic32_init_nob(&schdlr_sspnd.changing, 0); schdlr_sspnd.online = no_schedulers_online; schdlr_sspnd.curr_online = no_schedulers; - erts_smp_atomic32_init(&schdlr_sspnd.msb.ongoing, 0); - erts_smp_atomic32_init(&schdlr_sspnd.active, no_schedulers); + schdlr_sspnd.msb.ongoing = 0; + erts_smp_atomic32_init_nob(&schdlr_sspnd.active, no_schedulers); schdlr_sspnd.msb.procs = NULL; init_no_runqs(no_schedulers, erts_common_run_queue ? 1 : no_schedulers_online); @@ -2768,7 +2771,7 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) balance_info.forced_check_balance = 0; balance_info.halftime = 1; balance_info.full_reds_history_index = 0; - erts_smp_atomic32_init(&balance_info.checking_balance, 0); + erts_smp_atomic32_init_nob(&balance_info.checking_balance, 0); balance_info.prev_rise.active_runqs = 0; balance_info.prev_rise.max_len = 0; balance_info.prev_rise.reds = 0; @@ -2777,8 +2780,8 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) if (no_schedulers_online < no_schedulers) { if (erts_common_run_queue) { for (ix = no_schedulers_online; ix < no_schedulers; ix++) - erts_smp_atomic32_bor(&ERTS_SCHED_SLEEP_INFO_IX(ix)->flags, - ERTS_SSI_FLG_SUSPENDED); + erts_smp_atomic32_read_bor_nob(&ERTS_SCHED_SLEEP_INFO_IX(ix)->flags, + ERTS_SSI_FLG_SUSPENDED); } else { for (ix = no_schedulers_online; ix < erts_no_run_queues; ix++) @@ -2792,7 +2795,7 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) ERTS_SCHDLR_SSPND_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); - erts_smp_atomic32_init(&doing_sys_schedule, 0); + erts_smp_atomic32_init_nob(&doing_sys_schedule, 0); init_misc_aux_work(); @@ -2808,7 +2811,7 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) erts_no_schedulers = 1; #endif - erts_smp_atomic32_init(&function_calls, 0); + erts_smp_atomic32_init_nob(&function_calls, 0); /* init port tasks */ erts_port_task_init(); @@ -2935,10 +2938,10 @@ int erts_get_max_no_executing_schedulers(void) { #ifdef ERTS_SMP - if (erts_smp_atomic32_read(&schdlr_sspnd.changing)) + if (erts_smp_atomic32_read_nob(&schdlr_sspnd.changing)) return (int) erts_no_schedulers; ERTS_THR_MEMORY_BARRIER; - return (int) erts_smp_atomic32_read(&schdlr_sspnd.active); + return (int) erts_smp_atomic32_read_nob(&schdlr_sspnd.active); #else return 1; #endif @@ -2968,7 +2971,7 @@ scheduler_ix_resume_wake(Uint ix) | ERTS_SSI_FLG_SUSPENDED); erts_aint32_t oflgs; do { - oflgs = erts_smp_atomic32_cmpxchg(&ssi->flags, 0, xflgs); + oflgs = erts_smp_atomic32_cmpxchg_relb(&ssi->flags, 0, xflgs); if (oflgs == xflgs) { erts_sched_finish_poke(ssi, oflgs); break; @@ -2987,7 +2990,7 @@ sched_prep_spin_suspended(ErtsSchedulerSleepInfo *ssi, erts_aint32_t xpct) erts_aint32_t xflgs = xpct; do { - oflgs = erts_smp_atomic32_cmpxchg(&ssi->flags, nflgs, xflgs); + oflgs = erts_smp_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs); if (oflgs == xflgs) return nflgs; xflgs = oflgs; @@ -3037,7 +3040,7 @@ sched_set_suspended_sleeptype(ErtsSchedulerSleepInfo *ssi) erts_tse_reset(ssi->event); while (1) { - oflgs = erts_smp_atomic32_cmpxchg(&ssi->flags, nflgs, xflgs); + oflgs = erts_smp_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs); if (oflgs == xflgs) return nflgs; if ((oflgs & (ERTS_SSI_FLG_SLEEPING @@ -3092,15 +3095,15 @@ suspend_scheduler(ErtsSchedulerData *esdp) flgs = sched_prep_spin_suspended(ssi, ERTS_SSI_FLG_SUSPENDED); if (flgs & ERTS_SSI_FLG_SUSPENDED) { - active_schedulers = erts_smp_atomic32_dectest(&schdlr_sspnd.active); + active_schedulers = erts_smp_atomic32_dec_read_nob(&schdlr_sspnd.active); ASSERT(active_schedulers >= 1); - changing = erts_smp_atomic32_read(&schdlr_sspnd.changing); + changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); if (changing & ERTS_SCHDLR_SSPND_CHNG_MSB) { if (active_schedulers == schdlr_sspnd.msb.wait_active) wake = 1; if (active_schedulers == 1) { - changing = erts_smp_atomic32_band(&schdlr_sspnd.changing, - ~ERTS_SCHDLR_SSPND_CHNG_MSB); + changing = erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, + ~ERTS_SCHDLR_SSPND_CHNG_MSB); changing &= ~ERTS_SCHDLR_SSPND_CHNG_MSB; } } @@ -3122,8 +3125,8 @@ suspend_scheduler(ErtsSchedulerData *esdp) && schdlr_sspnd.curr_online == schdlr_sspnd.wait_curr_online) wake = 1; if (schdlr_sspnd.online == schdlr_sspnd.curr_online) { - changing = erts_smp_atomic32_band(&schdlr_sspnd.changing, - ~ERTS_SCHDLR_SSPND_CHNG_ONLN); + changing = erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, + ~ERTS_SCHDLR_SSPND_CHNG_ONLN); changing &= ~ERTS_SCHDLR_SSPND_CHNG_ONLN; } } @@ -3140,7 +3143,7 @@ suspend_scheduler(ErtsSchedulerData *esdp) #ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK - aux_work = erts_smp_atomic32_read(&ssi->aux_work); + aux_work = erts_smp_atomic32_read_nob(&ssi->aux_work); blockable_aux_work: blockable_aux_work(esdp, ssi, aux_work); #endif @@ -3176,13 +3179,13 @@ suspend_scheduler(ErtsSchedulerData *esdp) | ERTS_SSI_FLG_SUSPENDED)); if (!(flgs & ERTS_SSI_FLG_SUSPENDED)) break; - changing = erts_smp_atomic32_read(&schdlr_sspnd.changing); + changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); if (changing & ~ERTS_SCHDLR_SSPND_CHNG_WAITER) break; #ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK - aux_work = erts_smp_atomic32_read(&ssi->aux_work); + aux_work = erts_smp_atomic32_read_nob(&ssi->aux_work); if (aux_work & ERTS_SSI_BLOCKABLE_AUX_WORK_MASK) { erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); goto blockable_aux_work; @@ -3194,19 +3197,19 @@ suspend_scheduler(ErtsSchedulerData *esdp) erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); erts_smp_mtx_lock(&schdlr_sspnd.mtx); - changing = erts_smp_atomic32_read(&schdlr_sspnd.changing); + changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); } - active_schedulers = erts_smp_atomic32_inctest(&schdlr_sspnd.active); - changing = erts_smp_atomic32_read(&schdlr_sspnd.changing); + active_schedulers = erts_smp_atomic32_inc_read_nob(&schdlr_sspnd.active); + changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); if ((changing & ERTS_SCHDLR_SSPND_CHNG_MSB) && schdlr_sspnd.online == active_schedulers) { - erts_smp_atomic32_band(&schdlr_sspnd.changing, - ~ERTS_SCHDLR_SSPND_CHNG_MSB); + erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, + ~ERTS_SCHDLR_SSPND_CHNG_MSB); } ASSERT(no <= schdlr_sspnd.online); - ASSERT(!erts_smp_atomic32_read(&schdlr_sspnd.msb.ongoing)); + ASSERT(!ongoing_multi_scheduling_block()); } @@ -3235,7 +3238,7 @@ do { \ (RQ)->flags |= (ERTS_RUNQ_FLG_OUT_OF_WORK \ | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK); \ (RQ)->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS; \ - erts_smp_atomic32_band(&(RQ)->info_flags, ~ERTS_RUNQ_IFLG_SUSPENDED);\ + erts_smp_atomic32_read_band_nob(&(RQ)->info_flags, ~ERTS_RUNQ_IFLG_SUSPENDED);\ for (pix__ = 0; pix__ < ERTS_NO_PROC_PRIO_LEVELS; pix__++) { \ (RQ)->procs.prio_info[pix__].max_len = 0; \ (RQ)->procs.prio_info[pix__].reds = 0; \ @@ -3279,7 +3282,7 @@ erts_schedulers_state(Uint *total, int res; erts_aint32_t changing; erts_smp_mtx_lock(&schdlr_sspnd.mtx); - changing = erts_smp_atomic32_read(&schdlr_sspnd.changing); + changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); if (yield_allowed && (changing & ~ERTS_SCHDLR_SSPND_CHNG_WAITER)) res = ERTS_SCHDLR_SSPND_YIELD_RESTART; else { @@ -3310,7 +3313,7 @@ erts_set_schedulers_online(Process *p, have_unlocked_plocks = 0; no = (int) new_no; - changing = erts_smp_atomic32_read(&schdlr_sspnd.changing); + changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); if (changing) { res = ERTS_SCHDLR_SSPND_YIELD_RESTART; } @@ -3385,8 +3388,8 @@ erts_set_schedulers_online(Process *p, for (ix = no; ix < online; ix++) { ErtsSchedulerSleepInfo *ssi; ssi = ERTS_SCHED_SLEEP_INFO_IX(ix); - erts_smp_atomic32_bor(&ssi->flags, - ERTS_SSI_FLG_SUSPENDED); + erts_smp_atomic32_read_bor_nob(&ssi->flags, + ERTS_SSI_FLG_SUSPENDED); } wake_all_schedulers(); } @@ -3433,11 +3436,11 @@ erts_set_schedulers_online(Process *p, NULL); ASSERT(res != ERTS_SCHDLR_SSPND_DONE ? (ERTS_SCHDLR_SSPND_CHNG_WAITER - & erts_smp_atomic32_read(&schdlr_sspnd.changing)) + & erts_smp_atomic32_read_nob(&schdlr_sspnd.changing)) : (ERTS_SCHDLR_SSPND_CHNG_WAITER - == erts_smp_atomic32_read(&schdlr_sspnd.changing))); - erts_smp_atomic32_band(&schdlr_sspnd.changing, - ~ERTS_SCHDLR_SSPND_CHNG_WAITER); + == erts_smp_atomic32_read_nob(&schdlr_sspnd.changing))); + erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, + ~ERTS_SCHDLR_SSPND_CHNG_WAITER); } } @@ -3456,7 +3459,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) ErtsProcList *plp; erts_smp_mtx_lock(&schdlr_sspnd.mtx); - changing = erts_smp_atomic32_read(&schdlr_sspnd.changing); + changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); if (changing) { res = ERTS_SCHDLR_SSPND_YIELD_RESTART; /* Yield */ } @@ -3466,7 +3469,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) plp->next = schdlr_sspnd.msb.procs; schdlr_sspnd.msb.procs = plp; p->flags |= F_HAVE_BLCKD_MSCHED; - ASSERT(erts_smp_atomic32_read(&schdlr_sspnd.active) == 1); + ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.active) == 1); ASSERT(p->scheduler_data->no == 1); res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; } @@ -3477,11 +3480,11 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) have_unlocked_plocks = 1; erts_smp_proc_unlock(p, plocks); } - ASSERT(0 == erts_smp_atomic32_read(&schdlr_sspnd.msb.ongoing)); - erts_smp_atomic32_set(&schdlr_sspnd.msb.ongoing, 1); + ASSERT(!ongoing_multi_scheduling_block()); + schdlr_sspnd.msb.ongoing = 1; if (online == 1) { res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; - ASSERT(erts_smp_atomic32_read(&schdlr_sspnd.active) == 1); + ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.active) == 1); ASSERT(p->scheduler_data->no == 1); } else { @@ -3501,8 +3504,8 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) } if (erts_common_run_queue) { for (ix = 1; ix < online; ix++) - erts_smp_atomic32_bor(&ERTS_SCHED_SLEEP_INFO_IX(ix)->flags, - ERTS_SSI_FLG_SUSPENDED); + erts_smp_atomic32_read_bor_nob(&ERTS_SCHED_SLEEP_INFO_IX(ix)->flags, + ERTS_SSI_FLG_SUSPENDED); wake_all_schedulers(); } else { @@ -3530,7 +3533,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) susp_sched_prep_block, susp_sched_resume_block, NULL); - while (erts_smp_atomic32_read(&schdlr_sspnd.active) + while (erts_smp_atomic32_read_nob(&schdlr_sspnd.active) != schdlr_sspnd.msb.wait_active) erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); erts_smp_activity_end(ERTS_ACTIVITY_WAIT, @@ -3539,11 +3542,11 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) NULL); ASSERT(res != ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED ? (ERTS_SCHDLR_SSPND_CHNG_WAITER - & erts_smp_atomic32_read(&schdlr_sspnd.changing)) + & erts_smp_atomic32_read_nob(&schdlr_sspnd.changing)) : (ERTS_SCHDLR_SSPND_CHNG_WAITER - == erts_smp_atomic32_read(&schdlr_sspnd.changing))); - erts_smp_atomic32_band(&schdlr_sspnd.changing, - ~ERTS_SCHDLR_SSPND_CHNG_WAITER); + == erts_smp_atomic32_read_nob(&schdlr_sspnd.changing))); + erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, + ~ERTS_SCHDLR_SSPND_CHNG_WAITER); } plp = proclist_create(p); plp->next = schdlr_sspnd.msb.procs; @@ -3610,16 +3613,16 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) }); #endif p->flags &= ~F_HAVE_BLCKD_MSCHED; - erts_smp_atomic32_set(&schdlr_sspnd.msb.ongoing, 0); + schdlr_sspnd.msb.ongoing = 0; if (schdlr_sspnd.online == 1) { /* No schedulers to resume */ - ASSERT(erts_smp_atomic32_read(&schdlr_sspnd.active) == 1); + ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.active) == 1); ERTS_SCHDLR_SSPND_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_MSB); } else if (erts_common_run_queue) { for (ix = 1; ix < schdlr_sspnd.online; ix++) - erts_smp_atomic32_band(&ERTS_SCHED_SLEEP_INFO_IX(ix)->flags, - ~ERTS_SSI_FLG_SUSPENDED); + erts_smp_atomic32_read_band_nob(&ERTS_SCHED_SLEEP_INFO_IX(ix)->flags, + ~ERTS_SSI_FLG_SUSPENDED); wake_all_schedulers(); } else { @@ -3669,7 +3672,7 @@ void erts_dbg_multi_scheduling_return_trap(Process *p, Eterm return_value) { if (return_value == am_blocked) { - erts_aint32_t active = erts_smp_atomic32_read(&schdlr_sspnd.active); + erts_aint32_t active = erts_smp_atomic32_read_nob(&schdlr_sspnd.active); ASSERT(1 <= active && active <= 2); ASSERT(ERTS_PROC_GET_SCHDATA(p)->no == 1); } @@ -3752,12 +3755,12 @@ sched_thread_func(void *vesdp) erts_thread_init_float(); erts_smp_mtx_lock(&schdlr_sspnd.mtx); - ASSERT(erts_smp_atomic32_read(&schdlr_sspnd.changing) + ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.changing) & ERTS_SCHDLR_SSPND_CHNG_ONLN); if (--schdlr_sspnd.curr_online == schdlr_sspnd.wait_curr_online) { - erts_smp_atomic32_band(&schdlr_sspnd.changing, - ~ERTS_SCHDLR_SSPND_CHNG_ONLN); + erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, + ~ERTS_SCHDLR_SSPND_CHNG_ONLN); if (((ErtsSchedulerData *) vesdp)->no != 1) erts_smp_cnd_signal(&schdlr_sspnd.cnd); } @@ -5214,7 +5217,7 @@ Process *schedule(Process *p, int calls) reds = ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST; esdp->virtual_reds = 0; - fcalls = (int) erts_smp_atomic32_addtest(&function_calls, reds); + fcalls = (int) erts_smp_atomic32_add_read_acqb(&function_calls, reds); ASSERT(esdp && esdp == erts_get_scheduler_data()); rq = erts_get_runq_current(esdp); @@ -5349,7 +5352,7 @@ Process *schedule(Process *p, int calls) if ((rq->flags & ERTS_RUNQ_FLG_SUSPENDED) || (erts_smp_atomic32_read_acqb(&esdp->ssi->flags) & ERTS_SSI_FLG_SUSPENDED)) { - ASSERT(erts_smp_atomic32_read(&esdp->ssi->flags) + ASSERT(erts_smp_atomic32_read_nob(&esdp->ssi->flags) & ERTS_SSI_FLG_SUSPENDED); suspend_scheduler(esdp); } @@ -5363,7 +5366,7 @@ Process *schedule(Process *p, int calls) || defined(ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK) { ErtsSchedulerSleepInfo *ssi = esdp->ssi; - erts_aint32_t aux_work = erts_smp_atomic32_read(&ssi->aux_work); + erts_aint32_t aux_work = erts_smp_atomic32_read_nob(&ssi->aux_work); if (aux_work) { erts_smp_runq_unlock(rq); #ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK @@ -5407,7 +5410,7 @@ Process *schedule(Process *p, int calls) if ((rq->flags & ERTS_RUNQ_FLG_SUSPENDED) || (erts_smp_atomic32_read_acqb(&esdp->ssi->flags) & ERTS_SSI_FLG_SUSPENDED)) { - ASSERT(erts_smp_atomic32_read(&esdp->ssi->flags) + ASSERT(erts_smp_atomic32_read_nob(&esdp->ssi->flags) & ERTS_SSI_FLG_SUSPENDED); non_empty_runq(rq); goto continue_check_activities_to_run; @@ -5948,7 +5951,7 @@ erts_test_next_pid(int set, Uint next) Uint erts_process_count(void) { - erts_aint32_t res = erts_smp_atomic32_read(&process_count); + erts_aint32_t res = erts_smp_atomic32_read_nob(&process_count); ASSERT(res >= 0); return (Uint) res; } @@ -5997,7 +6000,7 @@ alloc_process(void) ASSERT(!process_tab[p_next]); process_tab[p_next] = p; - erts_smp_atomic32_inc(&process_count); + erts_smp_atomic32_inc_nob(&process_count); p->id = make_internal_pid(p_serial << p_serial_shift | p_next); if (p->id == ERTS_INVALID_PID) { /* Do not use the invalid pid; change serial */ @@ -6123,7 +6126,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->min_heap_size = H_MIN_SIZE; p->min_vheap_size = BIN_VH_MIN_SIZE; p->prio = PRIORITY_NORMAL; - p->max_gen_gcs = (Uint16) erts_smp_atomic32_read(&erts_max_gen_gcs); + p->max_gen_gcs = (Uint16) erts_smp_atomic32_read_nob(&erts_max_gen_gcs); } p->skipped = 0; ASSERT(p->min_heap_size == erts_next_heap_size(p->min_heap_size, 0)); @@ -7580,8 +7583,8 @@ continue_exit_process(Process *p p->status_flags = 0; #endif process_tab[pix] = NULL; /* Time of death! */ - ASSERT(erts_smp_atomic32_read(&process_count) > 0); - erts_smp_atomic32_dec(&process_count); + ASSERT(erts_smp_atomic32_read_nob(&process_count) > 0); + erts_smp_atomic32_dec_nob(&process_count); #ifdef ERTS_SMP erts_pix_unlock(pix_lock); diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 296acc7367..739aef3130 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -1599,11 +1599,11 @@ erts_sched_poke(ErtsSchedulerSleepInfo *ssi) { erts_aint32_t flags; ERTS_THR_MEMORY_BARRIER; - flags = erts_smp_atomic32_read(&ssi->flags); + flags = erts_smp_atomic32_read_nob(&ssi->flags); ASSERT(!(flags & ERTS_SSI_FLG_SLEEPING) || (flags & ERTS_SSI_FLG_WAITING)); if (flags & ERTS_SSI_FLG_SLEEPING) { - flags = erts_smp_atomic32_band(&ssi->flags, ~ERTS_SSI_FLGS_SLEEP); + flags = erts_smp_atomic32_read_band_nob(&ssi->flags, ~ERTS_SSI_FLGS_SLEEP); erts_sched_finish_poke(ssi, flags); } } diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c index 5410bcd495..3550f1396c 100644 --- a/erts/emulator/beam/erl_process_dump.c +++ b/erts/emulator/beam/erl_process_dump.c @@ -350,7 +350,7 @@ heap_dump(int to, void *to_arg, Eterm x) ProcBin* pb = (ProcBin *) binary_val(x); Binary* val = pb->val; - if (erts_smp_atomic_xchg(&val->refc, 0) != 0) { + if (erts_smp_atomic_xchg_nob(&val->refc, 0) != 0) { val->flags = (UWord) all_binaries; all_binaries = val; } diff --git a/erts/emulator/beam/erl_process_lock.c b/erts/emulator/beam/erl_process_lock.c index 72560aa124..83379d7352 100644 --- a/erts/emulator/beam/erl_process_lock.c +++ b/erts/emulator/beam/erl_process_lock.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2007-2010. All Rights Reserved. + * Copyright Ericsson AB 2007-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -316,7 +316,7 @@ try_aquire(erts_proc_lock_t *lck, erts_tse_t *wtr) break; } wflg = lock << ERTS_PROC_LOCK_WAITER_SHIFT; - old_lflgs = ERTS_PROC_LOCK_FLGS_BOR_(lck, wflg | lock); + old_lflgs = ERTS_PROC_LOCK_FLGS_BOR_ACQB_(lck, wflg | lock); if (old_lflgs & lock) { /* Didn't get the lock */ goto enqueue; @@ -413,7 +413,7 @@ transfer_locks(Process *p, do { erts_tse_t *tmp = wake; wake = wake->next; - erts_atomic32_set(&tmp->uaflgs, 0); + erts_atomic32_set_nob(&tmp->uaflgs, 0); erts_tse_set(tmp); } while (wake); @@ -509,14 +509,14 @@ wait_for_locks(Process *p, ASSERT((wtr->uflgs & ~ERTS_PROC_LOCKS_ALL) == 0); - erts_atomic32_set(&wtr->uaflgs, 1); + erts_atomic32_set_nob(&wtr->uaflgs, 1); erts_pix_unlock(pix_lock); while (1) { int res; erts_tse_reset(wtr); - if (erts_atomic32_read(&wtr->uaflgs) == 0) + if (erts_atomic32_read_nob(&wtr->uaflgs) == 0) break; /* @@ -955,7 +955,8 @@ erts_proc_lock_init(Process *p) { /* We always start with all locks locked */ #if ERTS_PROC_LOCK_ATOMIC_IMPL - erts_smp_atomic32_init(&p->lock.flags, (erts_aint32_t) ERTS_PROC_LOCKS_ALL); + erts_smp_atomic32_init_nob(&p->lock.flags, + (erts_aint32_t) ERTS_PROC_LOCKS_ALL); #else p->lock.flags = ERTS_PROC_LOCKS_ALL; #endif @@ -974,7 +975,7 @@ erts_proc_lock_init(Process *p) { int i; for (i = 0; i <= ERTS_PROC_LOCK_MAX_BIT; i++) - erts_smp_atomic32_init(&p->lock.locked[i], (erts_aint32_t) 1); + erts_smp_atomic32_init_nob(&p->lock.locked[i], (erts_aint32_t) 1); } #endif } diff --git a/erts/emulator/beam/erl_process_lock.h b/erts/emulator/beam/erl_process_lock.h index 355179f084..cd3b2182fd 100644 --- a/erts/emulator/beam/erl_process_lock.h +++ b/erts/emulator/beam/erl_process_lock.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2007-2010. All Rights Reserved. + * Copyright Ericsson AB 2007-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -41,10 +41,10 @@ #define ERTS_PROC_LOCK_SPINLOCK_IMPL 0 #define ERTS_PROC_LOCK_MUTEX_IMPL 0 -#if defined(ETHR_HAVE_OPTIMIZED_ATOMIC_OPS) +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) # undef ERTS_PROC_LOCK_ATOMIC_IMPL # define ERTS_PROC_LOCK_ATOMIC_IMPL 1 -#elif defined(ETHR_HAVE_OPTIMIZED_SPINLOCK) +#elif defined(ETHR_HAVE_NATIVE_SPINLOCKS) # undef ERTS_PROC_LOCK_SPINLOCK_IMPL # define ERTS_PROC_LOCK_SPINLOCK_IMPL 1 #else @@ -270,9 +270,11 @@ typedef struct { #if ERTS_PROC_LOCK_ATOMIC_IMPL #define ERTS_PROC_LOCK_FLGS_BAND_(L, MSK) \ - ((ErtsProcLocks) erts_smp_atomic32_band(&(L)->flags, (erts_aint32_t) (MSK))) -#define ERTS_PROC_LOCK_FLGS_BOR_(L, MSK) \ - ((ErtsProcLocks) erts_smp_atomic32_bor(&(L)->flags, (erts_aint32_t) (MSK))) + ((ErtsProcLocks) erts_smp_atomic32_read_band_nob(&(L)->flags, \ + (erts_aint32_t) (MSK))) +#define ERTS_PROC_LOCK_FLGS_BOR_ACQB_(L, MSK) \ + ((ErtsProcLocks) erts_smp_atomic32_read_bor_acqb(&(L)->flags, \ + (erts_aint32_t) (MSK))) #define ERTS_PROC_LOCK_FLGS_CMPXCHG_ACQB_(L, NEW, EXPECTED) \ ((ErtsProcLocks) erts_smp_atomic32_cmpxchg_acqb(&(L)->flags, \ (erts_aint32_t) (NEW), \ @@ -282,7 +284,7 @@ typedef struct { (erts_aint32_t) (NEW), \ (erts_aint32_t) (EXPECTED))) #define ERTS_PROC_LOCK_FLGS_READ_(L) \ - ((ErtsProcLocks) erts_smp_atomic32_read(&(L)->flags)) + ((ErtsProcLocks) erts_smp_atomic32_read_nob(&(L)->flags)) #else /* no opt atomic ops */ @@ -325,7 +327,7 @@ erts_proc_lock_flags_cmpxchg(erts_proc_lock_t *lck, ErtsProcLocks new, #endif #define ERTS_PROC_LOCK_FLGS_BAND_(L, MSK) erts_proc_lock_flags_band((L), (MSK)) -#define ERTS_PROC_LOCK_FLGS_BOR_(L, MSK) erts_proc_lock_flags_bor((L), (MSK)) +#define ERTS_PROC_LOCK_FLGS_BOR_ACQB_(L, MSK) erts_proc_lock_flags_bor((L), (MSK)) #define ERTS_PROC_LOCK_FLGS_CMPXCHG_ACQB_(L, NEW, EXPECTED) \ erts_proc_lock_flags_cmpxchg((L), (NEW), (EXPECTED)) #define ERTS_PROC_LOCK_FLGS_CMPXCHG_RELB_(L, NEW, EXPECTED) \ @@ -623,11 +625,11 @@ erts_proc_lock_op_debug(Process *p, ErtsProcLocks locks, int locked) if (locks & lock) { erts_aint32_t lock_count; if (locked) { - lock_count = erts_smp_atomic32_inctest(&p->lock.locked[i]); + lock_count = erts_smp_atomic32_inc_read_nob(&p->lock.locked[i]); ERTS_LC_ASSERT(lock_count == 1); } else { - lock_count = erts_smp_atomic32_dectest(&p->lock.locked[i]); + lock_count = erts_smp_atomic32_dec_read_nob(&p->lock.locked[i]); ERTS_LC_ASSERT(lock_count == 0); } } diff --git a/erts/emulator/beam/erl_smp.h b/erts/emulator/beam/erl_smp.h index 287327bfe1..a89ddfbcc1 100644 --- a/erts/emulator/beam/erl_smp.h +++ b/erts/emulator/beam/erl_smp.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * Copyright Ericsson AB 2005-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -54,8 +54,9 @@ typedef erts_cnd_t erts_smp_cnd_t; typedef erts_rwmtx_opt_t erts_smp_rwmtx_opt_t; typedef erts_rwmtx_t erts_smp_rwmtx_t; typedef erts_tsd_key_t erts_smp_tsd_key_t; -typedef erts_atomic_t erts_smp_atomic_t; -typedef erts_atomic32_t erts_smp_atomic32_t; +#define erts_smp_dw_atomic_t erts_dw_atomic_t +#define erts_smp_atomic_t erts_atomic_t +#define erts_smp_atomic32_t erts_atomic32_t typedef erts_spinlock_t erts_smp_spinlock_t; typedef erts_rwlock_t erts_smp_rwlock_t; void erts_thr_fatal_error(int, char *); /* implemented in erl_init.c */ @@ -83,8 +84,9 @@ typedef struct { } erts_smp_rwmtx_opt_t; typedef int erts_smp_rwmtx_t; typedef int erts_smp_tsd_key_t; -typedef SWord erts_smp_atomic_t; -typedef Uint32 erts_smp_atomic32_t; +#define erts_smp_dw_atomic_t erts_no_dw_atomic_t +#define erts_smp_atomic_t erts_no_atomic_t +#define erts_smp_atomic32_t erts_no_atomic32_t #if __GNUC__ > 2 typedef struct { } erts_smp_spinlock_t; typedef struct { } erts_smp_rwlock_t; @@ -160,82 +162,6 @@ ERTS_GLB_INLINE int erts_smp_rwmtx_tryrwlock(erts_smp_rwmtx_t *rwmtx); ERTS_GLB_INLINE void erts_smp_rwmtx_rwunlock(erts_smp_rwmtx_t *rwmtx); ERTS_GLB_INLINE int erts_smp_lc_rwmtx_is_rlocked(erts_smp_rwmtx_t *mtx); ERTS_GLB_INLINE int erts_smp_lc_rwmtx_is_rwlocked(erts_smp_rwmtx_t *mtx); -ERTS_GLB_INLINE void erts_smp_atomic_init(erts_smp_atomic_t *var, - erts_aint_t i); -ERTS_GLB_INLINE void erts_smp_atomic_set(erts_smp_atomic_t *var, erts_aint_t i); -ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_read(erts_smp_atomic_t *var); -ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_inctest(erts_smp_atomic_t *incp); -ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_dectest(erts_smp_atomic_t *decp); -ERTS_GLB_INLINE void erts_smp_atomic_inc(erts_smp_atomic_t *incp); -ERTS_GLB_INLINE void erts_smp_atomic_dec(erts_smp_atomic_t *decp); -ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_addtest(erts_smp_atomic_t *addp, - erts_aint_t i); -ERTS_GLB_INLINE void erts_smp_atomic_add(erts_smp_atomic_t *addp, - erts_aint_t i); -ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_xchg(erts_smp_atomic_t *xchgp, - erts_aint_t new); -ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_cmpxchg(erts_smp_atomic_t *xchgp, - erts_aint_t new, - erts_aint_t expected); -ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_bor(erts_smp_atomic_t *var, - erts_aint_t mask); -ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_band(erts_smp_atomic_t *var, - erts_aint_t mask); -ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_read_acqb(erts_smp_atomic_t *var); -ERTS_GLB_INLINE void erts_smp_atomic_set_relb(erts_smp_atomic_t *var, - erts_aint_t i); -ERTS_GLB_INLINE void erts_smp_atomic_dec_relb(erts_smp_atomic_t *decp); -ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_dectest_relb(erts_smp_atomic_t *decp); -ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_cmpxchg_acqb(erts_smp_atomic_t *xchgp, - erts_aint_t new, - erts_aint_t exp); -ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_cmpxchg_relb(erts_smp_atomic_t *xchgp, - erts_aint_t new, - erts_aint_t exp); -ERTS_GLB_INLINE void -erts_smp_atomic32_init(erts_smp_atomic32_t *var, erts_aint32_t i); -ERTS_GLB_INLINE void -erts_smp_atomic32_set(erts_smp_atomic32_t *var, erts_aint32_t i); -ERTS_GLB_INLINE erts_aint32_t -erts_smp_atomic32_read(erts_smp_atomic32_t *var); -ERTS_GLB_INLINE erts_aint32_t -erts_smp_atomic32_inctest(erts_smp_atomic32_t *incp); -ERTS_GLB_INLINE erts_aint32_t -erts_smp_atomic32_dectest(erts_smp_atomic32_t *decp); -ERTS_GLB_INLINE void -erts_smp_atomic32_inc(erts_smp_atomic32_t *incp); -ERTS_GLB_INLINE void -erts_smp_atomic32_dec(erts_smp_atomic32_t *decp); -ERTS_GLB_INLINE erts_aint32_t -erts_smp_atomic32_addtest(erts_smp_atomic32_t *addp, erts_aint32_t i); -ERTS_GLB_INLINE void -erts_smp_atomic32_add(erts_smp_atomic32_t *addp, erts_aint32_t i); -ERTS_GLB_INLINE erts_aint32_t -erts_smp_atomic32_xchg(erts_smp_atomic32_t *xchgp, erts_aint32_t new); -ERTS_GLB_INLINE erts_aint32_t -erts_smp_atomic32_cmpxchg(erts_smp_atomic32_t *xchgp, - erts_aint32_t new, - erts_aint32_t expected); -ERTS_GLB_INLINE erts_aint32_t -erts_smp_atomic32_bor(erts_smp_atomic32_t *var, erts_aint32_t mask); -ERTS_GLB_INLINE erts_aint32_t -erts_smp_atomic32_band(erts_smp_atomic32_t *var, erts_aint32_t mask); -ERTS_GLB_INLINE erts_aint32_t -erts_smp_atomic32_read_acqb(erts_smp_atomic32_t *var); -ERTS_GLB_INLINE void -erts_smp_atomic32_set_relb(erts_smp_atomic32_t *var, erts_aint32_t i); -ERTS_GLB_INLINE void -erts_smp_atomic32_dec_relb(erts_smp_atomic32_t *decp); -ERTS_GLB_INLINE erts_aint32_t -erts_smp_atomic32_dectest_relb(erts_smp_atomic32_t *decp); -ERTS_GLB_INLINE erts_aint32_t -erts_smp_atomic32_cmpxchg_acqb(erts_smp_atomic32_t *xchgp, - erts_aint32_t new, - erts_aint32_t exp); -ERTS_GLB_INLINE erts_aint32_t -erts_smp_atomic32_cmpxchg_relb(erts_smp_atomic32_t *xchgp, - erts_aint32_t new, - erts_aint32_t exp); ERTS_GLB_INLINE void erts_smp_spinlock_init_x(erts_smp_spinlock_t *lock, char *name, Eterm extra); @@ -279,6 +205,429 @@ ERTS_GLB_INLINE void erts_smp_thr_sigmask(int how, ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #endif /* #ifdef ERTS_THR_HAVE_SIG_FUNCS */ +/* + * Functions implementing atomic operations with with no (nob), + * full (mb), acquire (acqb), release (relb), read (rb), and + * write (wb) memory barriers. + * + * If SMP support has been disabled, they are mapped to functions + * that performs the same operation, but aren't atomic and don't + * imply memory barriers. + */ + +#ifdef ERTS_SMP + +/* Double word size atomics */ + +#define erts_smp_dw_atomic_init_nob erts_dw_atomic_init_nob +#define erts_smp_dw_atomic_set_nob erts_dw_atomic_set_nob +#define erts_smp_dw_atomic_read_nob erts_dw_atomic_read_nob +#define erts_smp_dw_atomic_cmpxchg_nob erts_dw_atomic_cmpxchg_nob + +#define erts_smp_dw_atomic_init_mb erts_dw_atomic_init_mb +#define erts_smp_dw_atomic_set_mb erts_dw_atomic_set_mb +#define erts_smp_dw_atomic_read_mb erts_dw_atomic_read_mb +#define erts_smp_dw_atomic_cmpxchg_mb erts_dw_atomic_cmpxchg_mb + +#define erts_smp_dw_atomic_init_acqb erts_dw_atomic_init_acqb +#define erts_smp_dw_atomic_set_acqb erts_dw_atomic_set_acqb +#define erts_smp_dw_atomic_read_acqb erts_dw_atomic_read_acqb +#define erts_smp_dw_atomic_cmpxchg_acqb erts_dw_atomic_cmpxchg_acqb + +#define erts_smp_dw_atomic_init_relb erts_dw_atomic_init_relb +#define erts_smp_dw_atomic_set_relb erts_dw_atomic_set_relb +#define erts_smp_dw_atomic_read_relb erts_dw_atomic_read_relb +#define erts_smp_dw_atomic_cmpxchg_relb erts_dw_atomic_cmpxchg_relb + +#define erts_smp_dw_atomic_init_rb erts_dw_atomic_init_rb +#define erts_smp_dw_atomic_set_rb erts_dw_atomic_set_rb +#define erts_smp_dw_atomic_read_rb erts_dw_atomic_read_rb +#define erts_smp_dw_atomic_cmpxchg_rb erts_dw_atomic_cmpxchg_rb + +#define erts_smp_dw_atomic_init_wb erts_dw_atomic_init_wb +#define erts_smp_dw_atomic_set_wb erts_dw_atomic_set_wb +#define erts_smp_dw_atomic_read_wb erts_dw_atomic_read_wb +#define erts_smp_dw_atomic_cmpxchg_wb erts_dw_atomic_cmpxchg_wb + +/* Word size atomics */ + +#define erts_smp_atomic_init_nob erts_atomic_init_nob +#define erts_smp_atomic_set_nob erts_atomic_set_nob +#define erts_smp_atomic_read_nob erts_atomic_read_nob +#define erts_smp_atomic_inc_read_nob erts_atomic_inc_read_nob +#define erts_smp_atomic_dec_read_nob erts_atomic_dec_read_nob +#define erts_smp_atomic_inc_nob erts_atomic_inc_nob +#define erts_smp_atomic_dec_nob erts_atomic_dec_nob +#define erts_smp_atomic_add_read_nob erts_atomic_add_read_nob +#define erts_smp_atomic_add_nob erts_atomic_add_nob +#define erts_smp_atomic_read_bor_nob erts_atomic_read_bor_nob +#define erts_smp_atomic_read_band_nob erts_atomic_read_band_nob +#define erts_smp_atomic_xchg_nob erts_atomic_xchg_nob +#define erts_smp_atomic_cmpxchg_nob erts_atomic_cmpxchg_nob + +#define erts_smp_atomic_init_mb erts_atomic_init_mb +#define erts_smp_atomic_set_mb erts_atomic_set_mb +#define erts_smp_atomic_read_mb erts_atomic_read_mb +#define erts_smp_atomic_inc_read_mb erts_atomic_inc_read_mb +#define erts_smp_atomic_dec_read_mb erts_atomic_dec_read_mb +#define erts_smp_atomic_inc_mb erts_atomic_inc_mb +#define erts_smp_atomic_dec_mb erts_atomic_dec_mb +#define erts_smp_atomic_add_read_mb erts_atomic_add_read_mb +#define erts_smp_atomic_add_mb erts_atomic_add_mb +#define erts_smp_atomic_read_bor_mb erts_atomic_read_bor_mb +#define erts_smp_atomic_read_band_mb erts_atomic_read_band_mb +#define erts_smp_atomic_xchg_mb erts_atomic_xchg_mb +#define erts_smp_atomic_cmpxchg_mb erts_atomic_cmpxchg_mb + +#define erts_smp_atomic_init_acqb erts_atomic_init_acqb +#define erts_smp_atomic_set_acqb erts_atomic_set_acqb +#define erts_smp_atomic_read_acqb erts_atomic_read_acqb +#define erts_smp_atomic_inc_read_acqb erts_atomic_inc_read_acqb +#define erts_smp_atomic_dec_read_acqb erts_atomic_dec_read_acqb +#define erts_smp_atomic_inc_acqb erts_atomic_inc_acqb +#define erts_smp_atomic_dec_acqb erts_atomic_dec_acqb +#define erts_smp_atomic_add_read_acqb erts_atomic_add_read_acqb +#define erts_smp_atomic_add_acqb erts_atomic_add_acqb +#define erts_smp_atomic_read_bor_acqb erts_atomic_read_bor_acqb +#define erts_smp_atomic_read_band_acqb erts_atomic_read_band_acqb +#define erts_smp_atomic_xchg_acqb erts_atomic_xchg_acqb +#define erts_smp_atomic_cmpxchg_acqb erts_atomic_cmpxchg_acqb + +#define erts_smp_atomic_init_relb erts_atomic_init_relb +#define erts_smp_atomic_set_relb erts_atomic_set_relb +#define erts_smp_atomic_read_relb erts_atomic_read_relb +#define erts_smp_atomic_inc_read_relb erts_atomic_inc_read_relb +#define erts_smp_atomic_dec_read_relb erts_atomic_dec_read_relb +#define erts_smp_atomic_inc_relb erts_atomic_inc_relb +#define erts_smp_atomic_dec_relb erts_atomic_dec_relb +#define erts_smp_atomic_add_read_relb erts_atomic_add_read_relb +#define erts_smp_atomic_add_relb erts_atomic_add_relb +#define erts_smp_atomic_read_bor_relb erts_atomic_read_bor_relb +#define erts_smp_atomic_read_band_relb erts_atomic_read_band_relb +#define erts_smp_atomic_xchg_relb erts_atomic_xchg_relb +#define erts_smp_atomic_cmpxchg_relb erts_atomic_cmpxchg_relb + +#define erts_smp_atomic_init_rb erts_atomic_init_rb +#define erts_smp_atomic_set_rb erts_atomic_set_rb +#define erts_smp_atomic_read_rb erts_atomic_read_rb +#define erts_smp_atomic_inc_read_rb erts_atomic_inc_read_rb +#define erts_smp_atomic_dec_read_rb erts_atomic_dec_read_rb +#define erts_smp_atomic_inc_rb erts_atomic_inc_rb +#define erts_smp_atomic_dec_rb erts_atomic_dec_rb +#define erts_smp_atomic_add_read_rb erts_atomic_add_read_rb +#define erts_smp_atomic_add_rb erts_atomic_add_rb +#define erts_smp_atomic_read_bor_rb erts_atomic_read_bor_rb +#define erts_smp_atomic_read_band_rb erts_atomic_read_band_rb +#define erts_smp_atomic_xchg_rb erts_atomic_xchg_rb +#define erts_smp_atomic_cmpxchg_rb erts_atomic_cmpxchg_rb + +#define erts_smp_atomic_init_wb erts_atomic_init_wb +#define erts_smp_atomic_set_wb erts_atomic_set_wb +#define erts_smp_atomic_read_wb erts_atomic_read_wb +#define erts_smp_atomic_inc_read_wb erts_atomic_inc_read_wb +#define erts_smp_atomic_dec_read_wb erts_atomic_dec_read_wb +#define erts_smp_atomic_inc_wb erts_atomic_inc_wb +#define erts_smp_atomic_dec_wb erts_atomic_dec_wb +#define erts_smp_atomic_add_read_wb erts_atomic_add_read_wb +#define erts_smp_atomic_add_wb erts_atomic_add_wb +#define erts_smp_atomic_read_bor_wb erts_atomic_read_bor_wb +#define erts_smp_atomic_read_band_wb erts_atomic_read_band_wb +#define erts_smp_atomic_xchg_wb erts_atomic_xchg_wb +#define erts_smp_atomic_cmpxchg_wb erts_atomic_cmpxchg_wb + +/* 32-bit atomics */ + +#define erts_smp_atomic32_init_nob erts_atomic32_init_nob +#define erts_smp_atomic32_set_nob erts_atomic32_set_nob +#define erts_smp_atomic32_read_nob erts_atomic32_read_nob +#define erts_smp_atomic32_inc_read_nob erts_atomic32_inc_read_nob +#define erts_smp_atomic32_dec_read_nob erts_atomic32_dec_read_nob +#define erts_smp_atomic32_inc_nob erts_atomic32_inc_nob +#define erts_smp_atomic32_dec_nob erts_atomic32_dec_nob +#define erts_smp_atomic32_add_read_nob erts_atomic32_add_read_nob +#define erts_smp_atomic32_add_nob erts_atomic32_add_nob +#define erts_smp_atomic32_read_bor_nob erts_atomic32_read_bor_nob +#define erts_smp_atomic32_read_band_nob erts_atomic32_read_band_nob +#define erts_smp_atomic32_xchg_nob erts_atomic32_xchg_nob +#define erts_smp_atomic32_cmpxchg_nob erts_atomic32_cmpxchg_nob + +#define erts_smp_atomic32_init_mb erts_atomic32_init_mb +#define erts_smp_atomic32_set_mb erts_atomic32_set_mb +#define erts_smp_atomic32_read_mb erts_atomic32_read_mb +#define erts_smp_atomic32_inc_read_mb erts_atomic32_inc_read_mb +#define erts_smp_atomic32_dec_read_mb erts_atomic32_dec_read_mb +#define erts_smp_atomic32_inc_mb erts_atomic32_inc_mb +#define erts_smp_atomic32_dec_mb erts_atomic32_dec_mb +#define erts_smp_atomic32_add_read_mb erts_atomic32_add_read_mb +#define erts_smp_atomic32_add_mb erts_atomic32_add_mb +#define erts_smp_atomic32_read_bor_mb erts_atomic32_read_bor_mb +#define erts_smp_atomic32_read_band_mb erts_atomic32_read_band_mb +#define erts_smp_atomic32_xchg_mb erts_atomic32_xchg_mb +#define erts_smp_atomic32_cmpxchg_mb erts_atomic32_cmpxchg_mb + +#define erts_smp_atomic32_init_acqb erts_atomic32_init_acqb +#define erts_smp_atomic32_set_acqb erts_atomic32_set_acqb +#define erts_smp_atomic32_read_acqb erts_atomic32_read_acqb +#define erts_smp_atomic32_inc_read_acqb erts_atomic32_inc_read_acqb +#define erts_smp_atomic32_dec_read_acqb erts_atomic32_dec_read_acqb +#define erts_smp_atomic32_inc_acqb erts_atomic32_inc_acqb +#define erts_smp_atomic32_dec_acqb erts_atomic32_dec_acqb +#define erts_smp_atomic32_add_read_acqb erts_atomic32_add_read_acqb +#define erts_smp_atomic32_add_acqb erts_atomic32_add_acqb +#define erts_smp_atomic32_read_bor_acqb erts_atomic32_read_bor_acqb +#define erts_smp_atomic32_read_band_acqb erts_atomic32_read_band_acqb +#define erts_smp_atomic32_xchg_acqb erts_atomic32_xchg_acqb +#define erts_smp_atomic32_cmpxchg_acqb erts_atomic32_cmpxchg_acqb + +#define erts_smp_atomic32_init_relb erts_atomic32_init_relb +#define erts_smp_atomic32_set_relb erts_atomic32_set_relb +#define erts_smp_atomic32_read_relb erts_atomic32_read_relb +#define erts_smp_atomic32_inc_read_relb erts_atomic32_inc_read_relb +#define erts_smp_atomic32_dec_read_relb erts_atomic32_dec_read_relb +#define erts_smp_atomic32_inc_relb erts_atomic32_inc_relb +#define erts_smp_atomic32_dec_relb erts_atomic32_dec_relb +#define erts_smp_atomic32_add_read_relb erts_atomic32_add_read_relb +#define erts_smp_atomic32_add_relb erts_atomic32_add_relb +#define erts_smp_atomic32_read_bor_relb erts_atomic32_read_bor_relb +#define erts_smp_atomic32_read_band_relb erts_atomic32_read_band_relb +#define erts_smp_atomic32_xchg_relb erts_atomic32_xchg_relb +#define erts_smp_atomic32_cmpxchg_relb erts_atomic32_cmpxchg_relb + +#define erts_smp_atomic32_init_rb erts_atomic32_init_rb +#define erts_smp_atomic32_set_rb erts_atomic32_set_rb +#define erts_smp_atomic32_read_rb erts_atomic32_read_rb +#define erts_smp_atomic32_inc_read_rb erts_atomic32_inc_read_rb +#define erts_smp_atomic32_dec_read_rb erts_atomic32_dec_read_rb +#define erts_smp_atomic32_inc_rb erts_atomic32_inc_rb +#define erts_smp_atomic32_dec_rb erts_atomic32_dec_rb +#define erts_smp_atomic32_add_read_rb erts_atomic32_add_read_rb +#define erts_smp_atomic32_add_rb erts_atomic32_add_rb +#define erts_smp_atomic32_read_bor_rb erts_atomic32_read_bor_rb +#define erts_smp_atomic32_read_band_rb erts_atomic32_read_band_rb +#define erts_smp_atomic32_xchg_rb erts_atomic32_xchg_rb +#define erts_smp_atomic32_cmpxchg_rb erts_atomic32_cmpxchg_rb + +#define erts_smp_atomic32_init_wb erts_atomic32_init_wb +#define erts_smp_atomic32_set_wb erts_atomic32_set_wb +#define erts_smp_atomic32_read_wb erts_atomic32_read_wb +#define erts_smp_atomic32_inc_read_wb erts_atomic32_inc_read_wb +#define erts_smp_atomic32_dec_read_wb erts_atomic32_dec_read_wb +#define erts_smp_atomic32_inc_wb erts_atomic32_inc_wb +#define erts_smp_atomic32_dec_wb erts_atomic32_dec_wb +#define erts_smp_atomic32_add_read_wb erts_atomic32_add_read_wb +#define erts_smp_atomic32_add_wb erts_atomic32_add_wb +#define erts_smp_atomic32_read_bor_wb erts_atomic32_read_bor_wb +#define erts_smp_atomic32_read_band_wb erts_atomic32_read_band_wb +#define erts_smp_atomic32_xchg_wb erts_atomic32_xchg_wb +#define erts_smp_atomic32_cmpxchg_wb erts_atomic32_cmpxchg_wb + +#else /* !ERTS_SMP */ + +/* Double word size atomics */ + +#define erts_smp_dw_atomic_init_nob erts_no_dw_atomic_set +#define erts_smp_dw_atomic_set_nob erts_no_dw_atomic_set +#define erts_smp_dw_atomic_read_nob erts_no_dw_atomic_read +#define erts_smp_dw_atomic_cmpxchg_nob erts_no_dw_atomic_cmpxchg + +#define erts_smp_dw_atomic_init_mb erts_no_dw_atomic_init +#define erts_smp_dw_atomic_set_mb erts_no_dw_atomic_set +#define erts_smp_dw_atomic_read_mb erts_no_dw_atomic_read +#define erts_smp_dw_atomic_cmpxchg_mb erts_no_dw_atomic_cmpxchg + +#define erts_smp_dw_atomic_init_acqb erts_no_dw_atomic_init +#define erts_smp_dw_atomic_set_acqb erts_no_dw_atomic_set +#define erts_smp_dw_atomic_read_acqb erts_no_dw_atomic_read +#define erts_smp_dw_atomic_cmpxchg_acqb erts_no_dw_atomic_cmpxchg + +#define erts_smp_dw_atomic_init_relb erts_no_dw_atomic_init +#define erts_smp_dw_atomic_set_relb erts_no_dw_atomic_set +#define erts_smp_dw_atomic_read_relb erts_no_dw_atomic_read +#define erts_smp_dw_atomic_cmpxchg_relb erts_no_dw_atomic_cmpxchg + +#define erts_smp_dw_atomic_init_rb erts_no_dw_atomic_init +#define erts_smp_dw_atomic_set_rb erts_no_dw_atomic_set +#define erts_smp_dw_atomic_read_rb erts_no_dw_atomic_read +#define erts_smp_dw_atomic_cmpxchg_rb erts_no_dw_atomic_cmpxchg + +#define erts_smp_dw_atomic_init_wb erts_no_dw_atomic_init +#define erts_smp_dw_atomic_set_wb erts_no_dw_atomic_set +#define erts_smp_dw_atomic_read_wb erts_no_dw_atomic_read +#define erts_smp_dw_atomic_cmpxchg_wb erts_no_dw_atomic_cmpxchg + +/* Word size atomics */ + +#define erts_smp_atomic_init_nob erts_no_atomic_set +#define erts_smp_atomic_set_nob erts_no_atomic_set +#define erts_smp_atomic_read_nob erts_no_atomic_read +#define erts_smp_atomic_inc_read_nob erts_no_atomic_inc_read +#define erts_smp_atomic_dec_read_nob erts_no_atomic_dec_read +#define erts_smp_atomic_inc_nob erts_no_atomic_inc +#define erts_smp_atomic_dec_nob erts_no_atomic_dec +#define erts_smp_atomic_add_read_nob erts_no_atomic_add_read +#define erts_smp_atomic_add_nob erts_no_atomic_add +#define erts_smp_atomic_read_bor_nob erts_no_atomic_read_bor +#define erts_smp_atomic_read_band_nob erts_no_atomic_read_band +#define erts_smp_atomic_xchg_nob erts_no_atomic_xchg +#define erts_smp_atomic_cmpxchg_nob erts_no_atomic_cmpxchg + +#define erts_smp_atomic_init_mb erts_no_atomic_set +#define erts_smp_atomic_set_mb erts_no_atomic_set +#define erts_smp_atomic_read_mb erts_no_atomic_read +#define erts_smp_atomic_inc_read_mb erts_no_atomic_inc_read +#define erts_smp_atomic_dec_read_mb erts_no_atomic_dec_read +#define erts_smp_atomic_inc_mb erts_no_atomic_inc +#define erts_smp_atomic_dec_mb erts_no_atomic_dec +#define erts_smp_atomic_add_read_mb erts_no_atomic_add_read +#define erts_smp_atomic_add_mb erts_no_atomic_add +#define erts_smp_atomic_read_bor_mb erts_no_atomic_read_bor +#define erts_smp_atomic_read_band_mb erts_no_atomic_read_band +#define erts_smp_atomic_xchg_mb erts_no_atomic_xchg +#define erts_smp_atomic_cmpxchg_mb erts_no_atomic_cmpxchg + +#define erts_smp_atomic_init_acqb erts_no_atomic_set +#define erts_smp_atomic_set_acqb erts_no_atomic_set +#define erts_smp_atomic_read_acqb erts_no_atomic_read +#define erts_smp_atomic_inc_read_acqb erts_no_atomic_inc_read +#define erts_smp_atomic_dec_read_acqb erts_no_atomic_dec_read +#define erts_smp_atomic_inc_acqb erts_no_atomic_inc +#define erts_smp_atomic_dec_acqb erts_no_atomic_dec +#define erts_smp_atomic_add_read_acqb erts_no_atomic_add_read +#define erts_smp_atomic_add_acqb erts_no_atomic_add +#define erts_smp_atomic_read_bor_acqb erts_no_atomic_read_bor +#define erts_smp_atomic_read_band_acqb erts_no_atomic_read_band +#define erts_smp_atomic_xchg_acqb erts_no_atomic_xchg +#define erts_smp_atomic_cmpxchg_acqb erts_no_atomic_cmpxchg + +#define erts_smp_atomic_init_relb erts_no_atomic_set +#define erts_smp_atomic_set_relb erts_no_atomic_set +#define erts_smp_atomic_read_relb erts_no_atomic_read +#define erts_smp_atomic_inc_read_relb erts_no_atomic_inc_read +#define erts_smp_atomic_dec_read_relb erts_no_atomic_dec_read +#define erts_smp_atomic_inc_relb erts_no_atomic_inc +#define erts_smp_atomic_dec_relb erts_no_atomic_dec +#define erts_smp_atomic_add_read_relb erts_no_atomic_add_read +#define erts_smp_atomic_add_relb erts_no_atomic_add +#define erts_smp_atomic_read_bor_relb erts_no_atomic_read_bor +#define erts_smp_atomic_read_band_relb erts_no_atomic_read_band +#define erts_smp_atomic_xchg_relb erts_no_atomic_xchg +#define erts_smp_atomic_cmpxchg_relb erts_no_atomic_cmpxchg + +#define erts_smp_atomic_init_rb erts_no_atomic_set +#define erts_smp_atomic_set_rb erts_no_atomic_set +#define erts_smp_atomic_read_rb erts_no_atomic_read +#define erts_smp_atomic_inc_read_rb erts_no_atomic_inc_read +#define erts_smp_atomic_dec_read_rb erts_no_atomic_dec_read +#define erts_smp_atomic_inc_rb erts_no_atomic_inc +#define erts_smp_atomic_dec_rb erts_no_atomic_dec +#define erts_smp_atomic_add_read_rb erts_no_atomic_add_read +#define erts_smp_atomic_add_rb erts_no_atomic_add +#define erts_smp_atomic_read_bor_rb erts_no_atomic_read_bor +#define erts_smp_atomic_read_band_rb erts_no_atomic_read_band +#define erts_smp_atomic_xchg_rb erts_no_atomic_xchg +#define erts_smp_atomic_cmpxchg_rb erts_no_atomic_cmpxchg + +#define erts_smp_atomic_init_wb erts_no_atomic_set +#define erts_smp_atomic_set_wb erts_no_atomic_set +#define erts_smp_atomic_read_wb erts_no_atomic_read +#define erts_smp_atomic_inc_read_wb erts_no_atomic_inc_read +#define erts_smp_atomic_dec_read_wb erts_no_atomic_dec_read +#define erts_smp_atomic_inc_wb erts_no_atomic_inc +#define erts_smp_atomic_dec_wb erts_no_atomic_dec +#define erts_smp_atomic_add_read_wb erts_no_atomic_add_read +#define erts_smp_atomic_add_wb erts_no_atomic_add +#define erts_smp_atomic_read_bor_wb erts_no_atomic_read_bor +#define erts_smp_atomic_read_band_wb erts_no_atomic_read_band +#define erts_smp_atomic_xchg_wb erts_no_atomic_xchg +#define erts_smp_atomic_cmpxchg_wb erts_no_atomic_cmpxchg + +/* 32-bit atomics */ + +#define erts_smp_atomic32_init_nob erts_no_atomic32_set +#define erts_smp_atomic32_set_nob erts_no_atomic32_set +#define erts_smp_atomic32_read_nob erts_no_atomic32_read +#define erts_smp_atomic32_inc_read_nob erts_no_atomic32_inc_read +#define erts_smp_atomic32_dec_read_nob erts_no_atomic32_dec_read +#define erts_smp_atomic32_inc_nob erts_no_atomic32_inc +#define erts_smp_atomic32_dec_nob erts_no_atomic32_dec +#define erts_smp_atomic32_add_read_nob erts_no_atomic32_add_read +#define erts_smp_atomic32_add_nob erts_no_atomic32_add +#define erts_smp_atomic32_read_bor_nob erts_no_atomic32_read_bor +#define erts_smp_atomic32_read_band_nob erts_no_atomic32_read_band +#define erts_smp_atomic32_xchg_nob erts_no_atomic32_xchg +#define erts_smp_atomic32_cmpxchg_nob erts_no_atomic32_cmpxchg + +#define erts_smp_atomic32_init_mb erts_no_atomic32_set +#define erts_smp_atomic32_set_mb erts_no_atomic32_set +#define erts_smp_atomic32_read_mb erts_no_atomic32_read +#define erts_smp_atomic32_inc_read_mb erts_no_atomic32_inc_read +#define erts_smp_atomic32_dec_read_mb erts_no_atomic32_dec_read +#define erts_smp_atomic32_inc_mb erts_no_atomic32_inc +#define erts_smp_atomic32_dec_mb erts_no_atomic32_dec +#define erts_smp_atomic32_add_read_mb erts_no_atomic32_add_read +#define erts_smp_atomic32_add_mb erts_no_atomic32_add +#define erts_smp_atomic32_read_bor_mb erts_no_atomic32_read_bor +#define erts_smp_atomic32_read_band_mb erts_no_atomic32_read_band +#define erts_smp_atomic32_xchg_mb erts_no_atomic32_xchg +#define erts_smp_atomic32_cmpxchg_mb erts_no_atomic32_cmpxchg + +#define erts_smp_atomic32_init_acqb erts_no_atomic32_set +#define erts_smp_atomic32_set_acqb erts_no_atomic32_set +#define erts_smp_atomic32_read_acqb erts_no_atomic32_read +#define erts_smp_atomic32_inc_read_acqb erts_no_atomic32_inc_read +#define erts_smp_atomic32_dec_read_acqb erts_no_atomic32_dec_read +#define erts_smp_atomic32_inc_acqb erts_no_atomic32_inc +#define erts_smp_atomic32_dec_acqb erts_no_atomic32_dec +#define erts_smp_atomic32_add_read_acqb erts_no_atomic32_add_read +#define erts_smp_atomic32_add_acqb erts_no_atomic32_add +#define erts_smp_atomic32_read_bor_acqb erts_no_atomic32_read_bor +#define erts_smp_atomic32_read_band_acqb erts_no_atomic32_read_band +#define erts_smp_atomic32_xchg_acqb erts_no_atomic32_xchg +#define erts_smp_atomic32_cmpxchg_acqb erts_no_atomic32_cmpxchg + +#define erts_smp_atomic32_init_relb erts_no_atomic32_set +#define erts_smp_atomic32_set_relb erts_no_atomic32_set +#define erts_smp_atomic32_read_relb erts_no_atomic32_read +#define erts_smp_atomic32_inc_read_relb erts_no_atomic32_inc_read +#define erts_smp_atomic32_dec_read_relb erts_no_atomic32_dec_read +#define erts_smp_atomic32_inc_relb erts_no_atomic32_inc +#define erts_smp_atomic32_dec_relb erts_no_atomic32_dec +#define erts_smp_atomic32_add_read_relb erts_no_atomic32_add_read +#define erts_smp_atomic32_add_relb erts_no_atomic32_add +#define erts_smp_atomic32_read_bor_relb erts_no_atomic32_read_bor +#define erts_smp_atomic32_read_band_relb erts_no_atomic32_read_band +#define erts_smp_atomic32_xchg_relb erts_no_atomic32_xchg +#define erts_smp_atomic32_cmpxchg_relb erts_no_atomic32_cmpxchg + +#define erts_smp_atomic32_init_rb erts_no_atomic32_set +#define erts_smp_atomic32_set_rb erts_no_atomic32_set +#define erts_smp_atomic32_read_rb erts_no_atomic32_read +#define erts_smp_atomic32_inc_read_rb erts_no_atomic32_inc_read +#define erts_smp_atomic32_dec_read_rb erts_no_atomic32_dec_read +#define erts_smp_atomic32_inc_rb erts_no_atomic32_inc +#define erts_smp_atomic32_dec_rb erts_no_atomic32_dec +#define erts_smp_atomic32_add_read_rb erts_no_atomic32_add_read +#define erts_smp_atomic32_add_rb erts_no_atomic32_add +#define erts_smp_atomic32_read_bor_rb erts_no_atomic32_read_bor +#define erts_smp_atomic32_read_band_rb erts_no_atomic32_read_band +#define erts_smp_atomic32_xchg_rb erts_no_atomic32_xchg +#define erts_smp_atomic32_cmpxchg_rb erts_no_atomic32_cmpxchg + +#define erts_smp_atomic32_init_wb erts_no_atomic32_set +#define erts_smp_atomic32_set_wb erts_no_atomic32_set +#define erts_smp_atomic32_read_wb erts_no_atomic32_read +#define erts_smp_atomic32_inc_read_wb erts_no_atomic32_inc_read +#define erts_smp_atomic32_dec_read_wb erts_no_atomic32_dec_read +#define erts_smp_atomic32_inc_wb erts_no_atomic32_inc +#define erts_smp_atomic32_dec_wb erts_no_atomic32_dec +#define erts_smp_atomic32_add_read_wb erts_no_atomic32_add_read +#define erts_smp_atomic32_add_wb erts_no_atomic32_add +#define erts_smp_atomic32_read_bor_wb erts_no_atomic32_read_bor +#define erts_smp_atomic32_read_band_wb erts_no_atomic32_read_band +#define erts_smp_atomic32_xchg_wb erts_no_atomic32_xchg +#define erts_smp_atomic32_cmpxchg_wb erts_no_atomic32_cmpxchg + +#endif /* !ERTS_SMP */ #if ERTS_GLB_INLINE_INCL_FUNC_DEF @@ -655,434 +1004,6 @@ erts_smp_lc_rwmtx_is_rwlocked(erts_smp_rwmtx_t *mtx) } ERTS_GLB_INLINE void -erts_smp_atomic_init(erts_smp_atomic_t *var, erts_aint_t i) -{ -#ifdef ERTS_SMP - erts_atomic_init(var, i); -#else - *var = i; -#endif -} - -ERTS_GLB_INLINE void -erts_smp_atomic_set(erts_smp_atomic_t *var, erts_aint_t i) -{ -#ifdef ERTS_SMP - erts_atomic_set(var, i); -#else - *var = i; -#endif -} - -ERTS_GLB_INLINE erts_aint_t -erts_smp_atomic_read(erts_smp_atomic_t *var) -{ -#ifdef ERTS_SMP - return erts_atomic_read(var); -#else - return *var; -#endif -} - -ERTS_GLB_INLINE erts_aint_t -erts_smp_atomic_inctest(erts_smp_atomic_t *incp) -{ -#ifdef ERTS_SMP - return erts_atomic_inctest(incp); -#else - return ++(*incp); -#endif -} - -ERTS_GLB_INLINE erts_aint_t -erts_smp_atomic_dectest(erts_smp_atomic_t *decp) -{ -#ifdef ERTS_SMP - return erts_atomic_dectest(decp); -#else - return --(*decp); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_atomic_inc(erts_smp_atomic_t *incp) -{ -#ifdef ERTS_SMP - erts_atomic_inc(incp); -#else - ++(*incp); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_atomic_dec(erts_smp_atomic_t *decp) -{ -#ifdef ERTS_SMP - erts_atomic_dec(decp); -#else - --(*decp); -#endif -} - -ERTS_GLB_INLINE erts_aint_t -erts_smp_atomic_addtest(erts_smp_atomic_t *addp, erts_aint_t i) -{ -#ifdef ERTS_SMP - return erts_atomic_addtest(addp, i); -#else - return *addp += i; -#endif -} - -ERTS_GLB_INLINE void -erts_smp_atomic_add(erts_smp_atomic_t *addp, erts_aint_t i) -{ -#ifdef ERTS_SMP - erts_atomic_add(addp, i); -#else - *addp += i; -#endif -} - -ERTS_GLB_INLINE erts_aint_t -erts_smp_atomic_xchg(erts_smp_atomic_t *xchgp, erts_aint_t new) -{ -#ifdef ERTS_SMP - return erts_atomic_xchg(xchgp, new); -#else - erts_aint_t old; - old = *xchgp; - *xchgp = new; - return old; -#endif -} - -ERTS_GLB_INLINE erts_aint_t -erts_smp_atomic_cmpxchg(erts_smp_atomic_t *xchgp, - erts_aint_t new, - erts_aint_t expected) -{ -#ifdef ERTS_SMP - return erts_atomic_cmpxchg(xchgp, new, expected); -#else - erts_aint_t old = *xchgp; - if (old == expected) - *xchgp = new; - return old; -#endif -} - -ERTS_GLB_INLINE erts_aint_t -erts_smp_atomic_bor(erts_smp_atomic_t *var, erts_aint_t mask) -{ -#ifdef ERTS_SMP - return erts_atomic_bor(var, mask); -#else - erts_aint_t old; - old = *var; - *var |= mask; - return old; -#endif -} - -ERTS_GLB_INLINE erts_aint_t -erts_smp_atomic_band(erts_smp_atomic_t *var, erts_aint_t mask) -{ -#ifdef ERTS_SMP - return erts_atomic_band(var, mask); -#else - erts_aint_t old; - old = *var; - *var &= mask; - return old; -#endif -} - -ERTS_GLB_INLINE erts_aint_t -erts_smp_atomic_read_acqb(erts_smp_atomic_t *var) -{ -#ifdef ERTS_SMP - return erts_atomic_read_acqb(var); -#else - return *var; -#endif -} - -ERTS_GLB_INLINE void -erts_smp_atomic_set_relb(erts_smp_atomic_t *var, erts_aint_t i) -{ -#ifdef ERTS_SMP - erts_atomic_set_relb(var, i); -#else - *var = i; -#endif -} - -ERTS_GLB_INLINE void -erts_smp_atomic_dec_relb(erts_smp_atomic_t *decp) -{ -#ifdef ERTS_SMP - erts_atomic_dec_relb(decp); -#else - --(*decp); -#endif -} - -ERTS_GLB_INLINE erts_aint_t -erts_smp_atomic_dectest_relb(erts_smp_atomic_t *decp) -{ -#ifdef ERTS_SMP - return erts_atomic_dectest_relb(decp); -#else - return --(*decp); -#endif -} - -ERTS_GLB_INLINE erts_aint_t -erts_smp_atomic_cmpxchg_acqb(erts_smp_atomic_t *xchgp, - erts_aint_t new, - erts_aint_t exp) -{ -#ifdef ERTS_SMP - return erts_atomic_cmpxchg_acqb(xchgp, new, exp); -#else - erts_aint_t old = *xchgp; - if (old == exp) - *xchgp = new; - return old; -#endif -} - -ERTS_GLB_INLINE erts_aint_t -erts_smp_atomic_cmpxchg_relb(erts_smp_atomic_t *xchgp, - erts_aint_t new, - erts_aint_t exp) -{ -#ifdef ERTS_SMP - return erts_atomic_cmpxchg_relb(xchgp, new, exp); -#else - erts_aint_t old = *xchgp; - if (old == exp) - *xchgp = new; - return old; -#endif -} - -ERTS_GLB_INLINE void -erts_smp_atomic32_init(erts_smp_atomic32_t *var, erts_aint32_t i) -{ -#ifdef ERTS_SMP - erts_atomic32_init(var, i); -#else - *var = i; -#endif -} - -ERTS_GLB_INLINE void -erts_smp_atomic32_set(erts_smp_atomic32_t *var, erts_aint32_t i) -{ -#ifdef ERTS_SMP - erts_atomic32_set(var, i); -#else - *var = i; -#endif -} - -ERTS_GLB_INLINE erts_aint32_t -erts_smp_atomic32_read(erts_smp_atomic32_t *var) -{ -#ifdef ERTS_SMP - return erts_atomic32_read(var); -#else - return *var; -#endif -} - -ERTS_GLB_INLINE erts_aint32_t -erts_smp_atomic32_inctest(erts_smp_atomic32_t *incp) -{ -#ifdef ERTS_SMP - return erts_atomic32_inctest(incp); -#else - return ++(*incp); -#endif -} - -ERTS_GLB_INLINE erts_aint32_t -erts_smp_atomic32_dectest(erts_smp_atomic32_t *decp) -{ -#ifdef ERTS_SMP - return erts_atomic32_dectest(decp); -#else - return --(*decp); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_atomic32_inc(erts_smp_atomic32_t *incp) -{ -#ifdef ERTS_SMP - erts_atomic32_inc(incp); -#else - ++(*incp); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_atomic32_dec(erts_smp_atomic32_t *decp) -{ -#ifdef ERTS_SMP - erts_atomic32_dec(decp); -#else - --(*decp); -#endif -} - -ERTS_GLB_INLINE erts_aint32_t -erts_smp_atomic32_addtest(erts_smp_atomic32_t *addp, erts_aint32_t i) -{ -#ifdef ERTS_SMP - return erts_atomic32_addtest(addp, i); -#else - return *addp += i; -#endif -} - -ERTS_GLB_INLINE void -erts_smp_atomic32_add(erts_smp_atomic32_t *addp, erts_aint32_t i) -{ -#ifdef ERTS_SMP - erts_atomic32_add(addp, i); -#else - *addp += i; -#endif -} - -ERTS_GLB_INLINE erts_aint32_t -erts_smp_atomic32_xchg(erts_smp_atomic32_t *xchgp, erts_aint32_t new) -{ -#ifdef ERTS_SMP - return erts_atomic32_xchg(xchgp, new); -#else - erts_aint32_t old; - old = *xchgp; - *xchgp = new; - return old; -#endif -} - -ERTS_GLB_INLINE erts_aint32_t -erts_smp_atomic32_cmpxchg(erts_smp_atomic32_t *xchgp, - erts_aint32_t new, - erts_aint32_t expected) -{ -#ifdef ERTS_SMP - return erts_atomic32_cmpxchg(xchgp, new, expected); -#else - erts_aint32_t old = *xchgp; - if (old == expected) - *xchgp = new; - return old; -#endif -} - -ERTS_GLB_INLINE erts_aint32_t -erts_smp_atomic32_bor(erts_smp_atomic32_t *var, erts_aint32_t mask) -{ -#ifdef ERTS_SMP - return erts_atomic32_bor(var, mask); -#else - erts_aint32_t old; - old = *var; - *var |= mask; - return old; -#endif -} - -ERTS_GLB_INLINE erts_aint32_t -erts_smp_atomic32_band(erts_smp_atomic32_t *var, erts_aint32_t mask) -{ -#ifdef ERTS_SMP - return erts_atomic32_band(var, mask); -#else - erts_aint32_t old; - old = *var; - *var &= mask; - return old; -#endif -} - -ERTS_GLB_INLINE erts_aint32_t -erts_smp_atomic32_read_acqb(erts_smp_atomic32_t *var) -{ -#ifdef ERTS_SMP - return erts_atomic32_read_acqb(var); -#else - return *var; -#endif -} - -ERTS_GLB_INLINE void -erts_smp_atomic32_set_relb(erts_smp_atomic32_t *var, erts_aint32_t i) -{ -#ifdef ERTS_SMP - erts_atomic32_set_relb(var, i); -#else - *var = i; -#endif -} - -ERTS_GLB_INLINE void -erts_smp_atomic32_dec_relb(erts_smp_atomic32_t *decp) -{ -#ifdef ERTS_SMP - erts_atomic32_dec_relb(decp); -#else - --(*decp); -#endif -} - -ERTS_GLB_INLINE erts_aint32_t -erts_smp_atomic32_dectest_relb(erts_smp_atomic32_t *decp) -{ -#ifdef ERTS_SMP - return erts_atomic32_dectest_relb(decp); -#else - return --(*decp); -#endif -} - -ERTS_GLB_INLINE erts_aint32_t -erts_smp_atomic32_cmpxchg_acqb(erts_smp_atomic32_t *xchgp, - erts_aint32_t new, - erts_aint32_t exp) -{ -#ifdef ERTS_SMP - return erts_atomic32_cmpxchg_acqb(xchgp, new, exp); -#else - erts_aint32_t old = *xchgp; - if (old == exp) - *xchgp = new; - return old; -#endif -} - -ERTS_GLB_INLINE erts_aint32_t -erts_smp_atomic32_cmpxchg_relb(erts_smp_atomic32_t *xchgp, - erts_aint32_t new, - erts_aint32_t exp) -{ -#ifdef ERTS_SMP - return erts_atomic32_cmpxchg_relb(xchgp, new, exp); -#else - erts_aint32_t old = *xchgp; - if (old == exp) - *xchgp = new; - return old; -#endif -} - -ERTS_GLB_INLINE void erts_smp_spinlock_init_x(erts_smp_spinlock_t *lock, char *name, Eterm extra) { #ifdef ERTS_SMP @@ -1308,3 +1229,37 @@ erts_smp_thr_sigwait(const sigset_t *set, int *sig) #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ #endif /* ERL_SMP_H */ + +#ifdef ERTS_UNDEF_DEPRECATED_ATOMICS + +/* Deprecated functions to replace */ + +#undef erts_smp_atomic_init +#undef erts_smp_atomic_set +#undef erts_smp_atomic_read +#undef erts_smp_atomic_inctest +#undef erts_smp_atomic_dectest +#undef erts_smp_atomic_inc +#undef erts_smp_atomic_dec +#undef erts_smp_atomic_addtest +#undef erts_smp_atomic_add +#undef erts_smp_atomic_xchg +#undef erts_smp_atomic_cmpxchg +#undef erts_smp_atomic_bor +#undef erts_smp_atomic_band + +#undef erts_smp_atomic32_init +#undef erts_smp_atomic32_set +#undef erts_smp_atomic32_read +#undef erts_smp_atomic32_inctest +#undef erts_smp_atomic32_dectest +#undef erts_smp_atomic32_inc +#undef erts_smp_atomic32_dec +#undef erts_smp_atomic32_addtest +#undef erts_smp_atomic32_add +#undef erts_smp_atomic32_xchg +#undef erts_smp_atomic32_cmpxchg +#undef erts_smp_atomic32_bor +#undef erts_smp_atomic32_band + +#endif diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h index 8c9cace0c5..9b897ffd24 100644 --- a/erts/emulator/beam/erl_threads.h +++ b/erts/emulator/beam/erl_threads.h @@ -28,6 +28,11 @@ #define ERTS_SPIN_BODY ETHR_SPIN_BODY #include "sys.h" + +typedef struct { SWord sint[2]; } erts_no_dw_atomic_t; +typedef SWord erts_no_atomic_t; +typedef Sint32 erts_no_atomic32_t; + #ifdef USE_THREADS #define ETHR_TRY_INLINE_FUNCS @@ -99,10 +104,12 @@ typedef ethr_rwmutex_opt erts_rwmtx_opt_t; typedef ethr_tsd_key erts_tsd_key_t; typedef ethr_ts_event erts_tse_t; -typedef ethr_sint_t erts_aint_t; -typedef ethr_atomic_t erts_atomic_t; -typedef ethr_sint32_t erts_aint32_t; -typedef ethr_atomic32_t erts_atomic32_t; +#define erts_dw_aint_t ethr_dw_sint_t +#define erts_dw_atomic_t ethr_dw_atomic_t +#define erts_aint_t ethr_sint_t +#define erts_atomic_t ethr_atomic_t +#define erts_aint32_t ethr_sint32_t +#define erts_atomic32_t ethr_atomic32_t /* spinlock */ typedef struct { @@ -164,10 +171,12 @@ typedef struct { typedef int erts_rwmtx_t; typedef int erts_tsd_key_t; typedef int erts_tse_t; -typedef SWord erts_aint_t; -typedef SWord erts_atomic_t; -typedef SWord erts_aint32_t; -typedef SWord erts_atomic32_t; +#define erts_dw_aint_t erts_no_dw_atomic_t +#define erts_dw_atomic_t erts_no_dw_atomic_t +#define erts_aint_t SWord +#define erts_atomic_t erts_no_atomic_t +#define erts_aint32_t Sint32 +#define erts_atomic32_t erts_no_atomic32_t #if __GNUC__ > 2 typedef struct { } erts_spinlock_t; typedef struct { } erts_rwlock_t; @@ -247,65 +256,51 @@ ERTS_GLB_INLINE int erts_rwmtx_tryrwlock(erts_rwmtx_t *rwmtx); ERTS_GLB_INLINE void erts_rwmtx_rwunlock(erts_rwmtx_t *rwmtx); ERTS_GLB_INLINE int erts_lc_rwmtx_is_rlocked(erts_rwmtx_t *mtx); ERTS_GLB_INLINE int erts_lc_rwmtx_is_rwlocked(erts_rwmtx_t *mtx); -ERTS_GLB_INLINE void erts_atomic_init(erts_atomic_t *var, erts_aint_t i); -ERTS_GLB_INLINE void erts_atomic_set(erts_atomic_t *var, erts_aint_t i); -ERTS_GLB_INLINE erts_aint_t erts_atomic_read(erts_atomic_t *var); -ERTS_GLB_INLINE erts_aint_t erts_atomic_inctest(erts_atomic_t *incp); -ERTS_GLB_INLINE erts_aint_t erts_atomic_dectest(erts_atomic_t *decp); -ERTS_GLB_INLINE void erts_atomic_inc(erts_atomic_t *incp); -ERTS_GLB_INLINE void erts_atomic_dec(erts_atomic_t *decp); -ERTS_GLB_INLINE erts_aint_t erts_atomic_addtest(erts_atomic_t *addp, - erts_aint_t i); -ERTS_GLB_INLINE void erts_atomic_add(erts_atomic_t *addp, erts_aint_t i); -ERTS_GLB_INLINE erts_aint_t erts_atomic_xchg(erts_atomic_t *xchgp, - erts_aint_t new); -ERTS_GLB_INLINE erts_aint_t erts_atomic_cmpxchg(erts_atomic_t *xchgp, - erts_aint_t new, - erts_aint_t expected); -ERTS_GLB_INLINE erts_aint_t erts_atomic_bor(erts_atomic_t *var, - erts_aint_t mask); -ERTS_GLB_INLINE erts_aint_t erts_atomic_band(erts_atomic_t *var, - erts_aint_t mask); -ERTS_GLB_INLINE erts_aint_t erts_atomic_read_acqb(erts_atomic_t *var); -ERTS_GLB_INLINE void erts_atomic_set_relb(erts_atomic_t *var, erts_aint_t i); -ERTS_GLB_INLINE void erts_atomic_dec_relb(erts_atomic_t *decp); -ERTS_GLB_INLINE erts_aint_t erts_atomic_dectest_relb(erts_atomic_t *decp); -ERTS_GLB_INLINE erts_aint_t erts_atomic_cmpxchg_acqb(erts_atomic_t *xchgp, - erts_aint_t new, - erts_aint_t exp); -ERTS_GLB_INLINE erts_aint_t erts_atomic_cmpxchg_relb(erts_atomic_t *xchgp, - erts_aint_t new, - erts_aint_t exp); -ERTS_GLB_INLINE void erts_atomic32_init(erts_atomic32_t *var, erts_aint32_t i); -ERTS_GLB_INLINE void erts_atomic32_set(erts_atomic32_t *var, erts_aint32_t i); -ERTS_GLB_INLINE erts_aint32_t erts_atomic32_read(erts_atomic32_t *var); -ERTS_GLB_INLINE erts_aint32_t erts_atomic32_inctest(erts_atomic32_t *incp); -ERTS_GLB_INLINE erts_aint32_t erts_atomic32_dectest(erts_atomic32_t *decp); -ERTS_GLB_INLINE void erts_atomic32_inc(erts_atomic32_t *incp); -ERTS_GLB_INLINE void erts_atomic32_dec(erts_atomic32_t *decp); -ERTS_GLB_INLINE erts_aint32_t erts_atomic32_addtest(erts_atomic32_t *addp, - erts_aint32_t i); -ERTS_GLB_INLINE void erts_atomic32_add(erts_atomic32_t *addp, erts_aint32_t i); -ERTS_GLB_INLINE erts_aint32_t erts_atomic32_xchg(erts_atomic32_t *xchgp, - erts_aint32_t new); -ERTS_GLB_INLINE erts_aint32_t erts_atomic32_cmpxchg(erts_atomic32_t *xchgp, - erts_aint32_t new, - erts_aint32_t expected); -ERTS_GLB_INLINE erts_aint32_t erts_atomic32_bor(erts_atomic32_t *var, - erts_aint32_t mask); -ERTS_GLB_INLINE erts_aint32_t erts_atomic32_band(erts_atomic32_t *var, - erts_aint32_t mask); -ERTS_GLB_INLINE erts_aint32_t erts_atomic32_read_acqb(erts_atomic32_t *var); -ERTS_GLB_INLINE void erts_atomic32_set_relb(erts_atomic32_t *var, - erts_aint32_t i); -ERTS_GLB_INLINE void erts_atomic32_dec_relb(erts_atomic32_t *decp); -ERTS_GLB_INLINE erts_aint32_t erts_atomic32_dectest_relb(erts_atomic32_t *decp); -ERTS_GLB_INLINE erts_aint32_t erts_atomic32_cmpxchg_acqb(erts_atomic32_t *xchgp, - erts_aint32_t new, - erts_aint32_t exp); -ERTS_GLB_INLINE erts_aint32_t erts_atomic32_cmpxchg_relb(erts_atomic32_t *xchgp, - erts_aint32_t new, - erts_aint32_t exp); + +ERTS_GLB_INLINE void erts_no_dw_atomic_set(erts_no_dw_atomic_t *var, erts_no_dw_atomic_t *val); +ERTS_GLB_INLINE void erts_no_dw_atomic_read(erts_no_dw_atomic_t *var, erts_no_dw_atomic_t *val); +ERTS_GLB_INLINE int erts_no_dw_atomic_cmpxchg(erts_no_dw_atomic_t *var, + erts_no_dw_atomic_t *val, + erts_no_dw_atomic_t *old_val); +ERTS_GLB_INLINE void erts_no_atomic_set(erts_no_atomic_t *var, erts_aint_t i); +ERTS_GLB_INLINE erts_aint_t erts_no_atomic_read(erts_no_atomic_t *var); +ERTS_GLB_INLINE erts_aint_t erts_no_atomic_inc_read(erts_no_atomic_t *incp); +ERTS_GLB_INLINE erts_aint_t erts_no_atomic_dec_read(erts_no_atomic_t *decp); +ERTS_GLB_INLINE void erts_no_atomic_inc(erts_no_atomic_t *incp); +ERTS_GLB_INLINE void erts_no_atomic_dec(erts_no_atomic_t *decp); +ERTS_GLB_INLINE erts_aint_t erts_no_atomic_add_read(erts_no_atomic_t *addp, + erts_aint_t i); +ERTS_GLB_INLINE void erts_no_atomic_add(erts_no_atomic_t *addp, erts_aint_t i); +ERTS_GLB_INLINE erts_aint_t erts_no_atomic_read_bor(erts_no_atomic_t *var, + erts_aint_t mask); +ERTS_GLB_INLINE erts_aint_t erts_no_atomic_read_band(erts_no_atomic_t *var, + erts_aint_t mask); +ERTS_GLB_INLINE erts_aint_t erts_no_atomic_xchg(erts_no_atomic_t *xchgp, + erts_aint_t new); +ERTS_GLB_INLINE erts_aint_t erts_no_atomic_cmpxchg(erts_no_atomic_t *xchgp, + erts_aint_t new, + erts_aint_t expected); +ERTS_GLB_INLINE void erts_no_atomic32_set(erts_no_atomic32_t *var, + erts_aint32_t i); +ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_read(erts_no_atomic32_t *var); +ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_inc_read(erts_no_atomic32_t *incp); +ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_dec_read(erts_no_atomic32_t *decp); +ERTS_GLB_INLINE void erts_no_atomic32_inc(erts_no_atomic32_t *incp); +ERTS_GLB_INLINE void erts_no_atomic32_dec(erts_no_atomic32_t *decp); +ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_add_read(erts_no_atomic32_t *addp, + erts_aint32_t i); +ERTS_GLB_INLINE void erts_no_atomic32_add(erts_no_atomic32_t *addp, + erts_aint32_t i); +ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_read_bor(erts_no_atomic32_t *var, + erts_aint32_t mask); +ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_read_band(erts_no_atomic32_t *var, + erts_aint32_t mask); +ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_xchg(erts_no_atomic32_t *xchgp, + erts_aint32_t new); +ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_cmpxchg(erts_no_atomic32_t *xchgp, + erts_aint32_t new, + erts_aint32_t expected); + ERTS_GLB_INLINE void erts_spinlock_init_x_opt(erts_spinlock_t *lock, char *name, Eterm extra, @@ -362,6 +357,430 @@ ERTS_GLB_INLINE void erts_thr_sigmask(int how, const sigset_t *set, ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #endif /* #ifdef HAVE_ETHR_SIG_FUNCS */ +/* + * Functions implementing atomic operations with with no (nob), + * full (mb), acquire (acqb), release (relb), read (rb), and + * write (wb) memory barriers. + * + * If thread support has been disabled, they are mapped to + * functions that performs the same operation, but aren't atomic + * and don't imply memory barriers. + */ + +#ifdef USE_THREADS + +/* Double word size atomics */ + +#define erts_dw_atomic_init_nob ethr_dw_atomic_init +#define erts_dw_atomic_set_nob ethr_dw_atomic_set +#define erts_dw_atomic_read_nob ethr_dw_atomic_read +#define erts_dw_atomic_cmpxchg_nob ethr_dw_atomic_cmpxchg + +#define erts_dw_atomic_init_mb ethr_dw_atomic_init_mb +#define erts_dw_atomic_set_mb ethr_dw_atomic_set_mb +#define erts_dw_atomic_read_mb ethr_dw_atomic_read_mb +#define erts_dw_atomic_cmpxchg_mb ethr_dw_atomic_cmpxchg_mb + +#define erts_dw_atomic_init_acqb ethr_dw_atomic_init_acqb +#define erts_dw_atomic_set_acqb ethr_dw_atomic_set_acqb +#define erts_dw_atomic_read_acqb ethr_dw_atomic_read_acqb +#define erts_dw_atomic_cmpxchg_acqb ethr_dw_atomic_cmpxchg_acqb + +#define erts_dw_atomic_init_relb ethr_dw_atomic_init_relb +#define erts_dw_atomic_set_relb ethr_dw_atomic_set_relb +#define erts_dw_atomic_read_relb ethr_dw_atomic_read_relb +#define erts_dw_atomic_cmpxchg_relb ethr_dw_atomic_cmpxchg_relb + +#define erts_dw_atomic_init_rb ethr_dw_atomic_init_rb +#define erts_dw_atomic_set_rb ethr_dw_atomic_set_rb +#define erts_dw_atomic_read_rb ethr_dw_atomic_read_rb +#define erts_dw_atomic_cmpxchg_rb ethr_dw_atomic_cmpxchg_rb + +#define erts_dw_atomic_init_wb ethr_dw_atomic_init_wb +#define erts_dw_atomic_set_wb ethr_dw_atomic_set_wb +#define erts_dw_atomic_read_wb ethr_dw_atomic_read_wb +#define erts_dw_atomic_cmpxchg_wb ethr_dw_atomic_cmpxchg_wb + +/* Word size atomics */ + +#define erts_atomic_init_nob ethr_atomic_init +#define erts_atomic_set_nob ethr_atomic_set +#define erts_atomic_read_nob ethr_atomic_read +#define erts_atomic_inc_read_nob ethr_atomic_inc_read +#define erts_atomic_dec_read_nob ethr_atomic_dec_read +#define erts_atomic_inc_nob ethr_atomic_inc +#define erts_atomic_dec_nob ethr_atomic_dec +#define erts_atomic_add_read_nob ethr_atomic_add_read +#define erts_atomic_add_nob ethr_atomic_add +#define erts_atomic_read_bor_nob ethr_atomic_read_bor +#define erts_atomic_read_band_nob ethr_atomic_read_band +#define erts_atomic_xchg_nob ethr_atomic_xchg +#define erts_atomic_cmpxchg_nob ethr_atomic_cmpxchg + +#define erts_atomic_init_mb ethr_atomic_init_mb +#define erts_atomic_set_mb ethr_atomic_set_mb +#define erts_atomic_read_mb ethr_atomic_read_mb +#define erts_atomic_inc_read_mb ethr_atomic_inc_read_mb +#define erts_atomic_dec_read_mb ethr_atomic_dec_read_mb +#define erts_atomic_inc_mb ethr_atomic_inc_mb +#define erts_atomic_dec_mb ethr_atomic_dec_mb +#define erts_atomic_add_read_mb ethr_atomic_add_read_mb +#define erts_atomic_add_mb ethr_atomic_add_mb +#define erts_atomic_read_bor_mb ethr_atomic_read_bor_mb +#define erts_atomic_read_band_mb ethr_atomic_read_band_mb +#define erts_atomic_xchg_mb ethr_atomic_xchg_mb +#define erts_atomic_cmpxchg_mb ethr_atomic_cmpxchg_mb + +#define erts_atomic_init_acqb ethr_atomic_init_acqb +#define erts_atomic_set_acqb ethr_atomic_set_acqb +#define erts_atomic_read_acqb ethr_atomic_read_acqb +#define erts_atomic_inc_read_acqb ethr_atomic_inc_read_acqb +#define erts_atomic_dec_read_acqb ethr_atomic_dec_read_acqb +#define erts_atomic_inc_acqb ethr_atomic_inc_acqb +#define erts_atomic_dec_acqb ethr_atomic_dec_acqb +#define erts_atomic_add_read_acqb ethr_atomic_add_read_acqb +#define erts_atomic_add_acqb ethr_atomic_add_acqb +#define erts_atomic_read_bor_acqb ethr_atomic_read_bor_acqb +#define erts_atomic_read_band_acqb ethr_atomic_read_band_acqb +#define erts_atomic_xchg_acqb ethr_atomic_xchg_acqb +#define erts_atomic_cmpxchg_acqb ethr_atomic_cmpxchg_acqb + +#define erts_atomic_init_relb ethr_atomic_init_relb +#define erts_atomic_set_relb ethr_atomic_set_relb +#define erts_atomic_read_relb ethr_atomic_read_relb +#define erts_atomic_inc_read_relb ethr_atomic_inc_read_relb +#define erts_atomic_dec_read_relb ethr_atomic_dec_read_relb +#define erts_atomic_inc_relb ethr_atomic_inc_relb +#define erts_atomic_dec_relb ethr_atomic_dec_relb +#define erts_atomic_add_read_relb ethr_atomic_add_read_relb +#define erts_atomic_add_relb ethr_atomic_add_relb +#define erts_atomic_read_bor_relb ethr_atomic_read_bor_relb +#define erts_atomic_read_band_relb ethr_atomic_read_band_relb +#define erts_atomic_xchg_relb ethr_atomic_xchg_relb +#define erts_atomic_cmpxchg_relb ethr_atomic_cmpxchg_relb + +#define erts_atomic_init_rb ethr_atomic_init_rb +#define erts_atomic_set_rb ethr_atomic_set_rb +#define erts_atomic_read_rb ethr_atomic_read_rb +#define erts_atomic_inc_read_rb ethr_atomic_inc_read_rb +#define erts_atomic_dec_read_rb ethr_atomic_dec_read_rb +#define erts_atomic_inc_rb ethr_atomic_inc_rb +#define erts_atomic_dec_rb ethr_atomic_dec_rb +#define erts_atomic_add_read_rb ethr_atomic_add_read_rb +#define erts_atomic_add_rb ethr_atomic_add_rb +#define erts_atomic_read_bor_rb ethr_atomic_read_bor_rb +#define erts_atomic_read_band_rb ethr_atomic_read_band_rb +#define erts_atomic_xchg_rb ethr_atomic_xchg_rb +#define erts_atomic_cmpxchg_rb ethr_atomic_cmpxchg_rb + +#define erts_atomic_init_wb ethr_atomic_init_wb +#define erts_atomic_set_wb ethr_atomic_set_wb +#define erts_atomic_read_wb ethr_atomic_read_wb +#define erts_atomic_inc_read_wb ethr_atomic_inc_read_wb +#define erts_atomic_dec_read_wb ethr_atomic_dec_read_wb +#define erts_atomic_inc_wb ethr_atomic_inc_wb +#define erts_atomic_dec_wb ethr_atomic_dec_wb +#define erts_atomic_add_read_wb ethr_atomic_add_read_wb +#define erts_atomic_add_wb ethr_atomic_add_wb +#define erts_atomic_read_bor_wb ethr_atomic_read_bor_wb +#define erts_atomic_read_band_wb ethr_atomic_read_band_wb +#define erts_atomic_xchg_wb ethr_atomic_xchg_wb +#define erts_atomic_cmpxchg_wb ethr_atomic_cmpxchg_wb + +/* 32-bit atomics */ + +#define erts_atomic32_init_nob ethr_atomic32_init +#define erts_atomic32_set_nob ethr_atomic32_set +#define erts_atomic32_read_nob ethr_atomic32_read +#define erts_atomic32_inc_read_nob ethr_atomic32_inc_read +#define erts_atomic32_dec_read_nob ethr_atomic32_dec_read +#define erts_atomic32_inc_nob ethr_atomic32_inc +#define erts_atomic32_dec_nob ethr_atomic32_dec +#define erts_atomic32_add_read_nob ethr_atomic32_add_read +#define erts_atomic32_add_nob ethr_atomic32_add +#define erts_atomic32_read_bor_nob ethr_atomic32_read_bor +#define erts_atomic32_read_band_nob ethr_atomic32_read_band +#define erts_atomic32_xchg_nob ethr_atomic32_xchg +#define erts_atomic32_cmpxchg_nob ethr_atomic32_cmpxchg + +#define erts_atomic32_init_mb ethr_atomic32_init_mb +#define erts_atomic32_set_mb ethr_atomic32_set_mb +#define erts_atomic32_read_mb ethr_atomic32_read_mb +#define erts_atomic32_inc_read_mb ethr_atomic32_inc_read_mb +#define erts_atomic32_dec_read_mb ethr_atomic32_dec_read_mb +#define erts_atomic32_inc_mb ethr_atomic32_inc_mb +#define erts_atomic32_dec_mb ethr_atomic32_dec_mb +#define erts_atomic32_add_read_mb ethr_atomic32_add_read_mb +#define erts_atomic32_add_mb ethr_atomic32_add_mb +#define erts_atomic32_read_bor_mb ethr_atomic32_read_bor_mb +#define erts_atomic32_read_band_mb ethr_atomic32_read_band_mb +#define erts_atomic32_xchg_mb ethr_atomic32_xchg_mb +#define erts_atomic32_cmpxchg_mb ethr_atomic32_cmpxchg_mb + +#define erts_atomic32_init_acqb ethr_atomic32_init_acqb +#define erts_atomic32_set_acqb ethr_atomic32_set_acqb +#define erts_atomic32_read_acqb ethr_atomic32_read_acqb +#define erts_atomic32_inc_read_acqb ethr_atomic32_inc_read_acqb +#define erts_atomic32_dec_read_acqb ethr_atomic32_dec_read_acqb +#define erts_atomic32_inc_acqb ethr_atomic32_inc_acqb +#define erts_atomic32_dec_acqb ethr_atomic32_dec_acqb +#define erts_atomic32_add_read_acqb ethr_atomic32_add_read_acqb +#define erts_atomic32_add_acqb ethr_atomic32_add_acqb +#define erts_atomic32_read_bor_acqb ethr_atomic32_read_bor_acqb +#define erts_atomic32_read_band_acqb ethr_atomic32_read_band_acqb +#define erts_atomic32_xchg_acqb ethr_atomic32_xchg_acqb +#define erts_atomic32_cmpxchg_acqb ethr_atomic32_cmpxchg_acqb + +#define erts_atomic32_init_relb ethr_atomic32_init_relb +#define erts_atomic32_set_relb ethr_atomic32_set_relb +#define erts_atomic32_read_relb ethr_atomic32_read_relb +#define erts_atomic32_inc_read_relb ethr_atomic32_inc_read_relb +#define erts_atomic32_dec_read_relb ethr_atomic32_dec_read_relb +#define erts_atomic32_inc_relb ethr_atomic32_inc_relb +#define erts_atomic32_dec_relb ethr_atomic32_dec_relb +#define erts_atomic32_add_read_relb ethr_atomic32_add_read_relb +#define erts_atomic32_add_relb ethr_atomic32_add_relb +#define erts_atomic32_read_bor_relb ethr_atomic32_read_bor_relb +#define erts_atomic32_read_band_relb ethr_atomic32_read_band_relb +#define erts_atomic32_xchg_relb ethr_atomic32_xchg_relb +#define erts_atomic32_cmpxchg_relb ethr_atomic32_cmpxchg_relb + +#define erts_atomic32_init_rb ethr_atomic32_init_rb +#define erts_atomic32_set_rb ethr_atomic32_set_rb +#define erts_atomic32_read_rb ethr_atomic32_read_rb +#define erts_atomic32_inc_read_rb ethr_atomic32_inc_read_rb +#define erts_atomic32_dec_read_rb ethr_atomic32_dec_read_rb +#define erts_atomic32_inc_rb ethr_atomic32_inc_rb +#define erts_atomic32_dec_rb ethr_atomic32_dec_rb +#define erts_atomic32_add_read_rb ethr_atomic32_add_read_rb +#define erts_atomic32_add_rb ethr_atomic32_add_rb +#define erts_atomic32_read_bor_rb ethr_atomic32_read_bor_rb +#define erts_atomic32_read_band_rb ethr_atomic32_read_band_rb +#define erts_atomic32_xchg_rb ethr_atomic32_xchg_rb +#define erts_atomic32_cmpxchg_rb ethr_atomic32_cmpxchg_rb + +#define erts_atomic32_init_wb ethr_atomic32_init_wb +#define erts_atomic32_set_wb ethr_atomic32_set_wb +#define erts_atomic32_read_wb ethr_atomic32_read_wb +#define erts_atomic32_inc_read_wb ethr_atomic32_inc_read_wb +#define erts_atomic32_dec_read_wb ethr_atomic32_dec_read_wb +#define erts_atomic32_inc_wb ethr_atomic32_inc_wb +#define erts_atomic32_dec_wb ethr_atomic32_dec_wb +#define erts_atomic32_add_read_wb ethr_atomic32_add_read_wb +#define erts_atomic32_add_wb ethr_atomic32_add_wb +#define erts_atomic32_read_bor_wb ethr_atomic32_read_bor_wb +#define erts_atomic32_read_band_wb ethr_atomic32_read_band_wb +#define erts_atomic32_xchg_wb ethr_atomic32_xchg_wb +#define erts_atomic32_cmpxchg_wb ethr_atomic32_cmpxchg_wb + +#else /* !USE_THREADS */ + +/* Double word size atomics */ + +#define erts_dw_atomic_init_nob erts_no_dw_atomic_set +#define erts_dw_atomic_set_nob erts_no_dw_atomic_set +#define erts_dw_atomic_read_nob erts_no_dw_atomic_read +#define erts_dw_atomic_cmpxchg_nob erts_no_dw_atomic_cmpxchg + +#define erts_dw_atomic_init_mb erts_no_dw_atomic_init +#define erts_dw_atomic_set_mb erts_no_dw_atomic_set +#define erts_dw_atomic_read_mb erts_no_dw_atomic_read +#define erts_dw_atomic_cmpxchg_mb erts_no_dw_atomic_cmpxchg + +#define erts_dw_atomic_init_acqb erts_no_dw_atomic_init +#define erts_dw_atomic_set_acqb erts_no_dw_atomic_set +#define erts_dw_atomic_read_acqb erts_no_dw_atomic_read +#define erts_dw_atomic_cmpxchg_acqb erts_no_dw_atomic_cmpxchg + +#define erts_dw_atomic_init_relb erts_no_dw_atomic_init +#define erts_dw_atomic_set_relb erts_no_dw_atomic_set +#define erts_dw_atomic_read_relb erts_no_dw_atomic_read +#define erts_dw_atomic_cmpxchg_relb erts_no_dw_atomic_cmpxchg + +#define erts_dw_atomic_init_rb erts_no_dw_atomic_init +#define erts_dw_atomic_set_rb erts_no_dw_atomic_set +#define erts_dw_atomic_read_rb erts_no_dw_atomic_read +#define erts_dw_atomic_cmpxchg_rb erts_no_dw_atomic_cmpxchg + +#define erts_dw_atomic_init_wb erts_no_dw_atomic_init +#define erts_dw_atomic_set_wb erts_no_dw_atomic_set +#define erts_dw_atomic_read_wb erts_no_dw_atomic_read +#define erts_dw_atomic_cmpxchg_wb erts_no_dw_atomic_cmpxchg + +/* Word size atomics */ + +#define erts_atomic_init_nob erts_no_atomic_set +#define erts_atomic_set_nob erts_no_atomic_set +#define erts_atomic_read_nob erts_no_atomic_read +#define erts_atomic_inc_read_nob erts_no_atomic_inc_read +#define erts_atomic_dec_read_nob erts_no_atomic_dec_read +#define erts_atomic_inc_nob erts_no_atomic_inc +#define erts_atomic_dec_nob erts_no_atomic_dec +#define erts_atomic_add_read_nob erts_no_atomic_add_read +#define erts_atomic_add_nob erts_no_atomic_add +#define erts_atomic_read_bor_nob erts_no_atomic_read_bor +#define erts_atomic_read_band_nob erts_no_atomic_read_band +#define erts_atomic_xchg_nob erts_no_atomic_xchg +#define erts_atomic_cmpxchg_nob erts_no_atomic_cmpxchg + +#define erts_atomic_init_mb erts_no_atomic_set +#define erts_atomic_set_mb erts_no_atomic_set +#define erts_atomic_read_mb erts_no_atomic_read +#define erts_atomic_inc_read_mb erts_no_atomic_inc_read +#define erts_atomic_dec_read_mb erts_no_atomic_dec_read +#define erts_atomic_inc_mb erts_no_atomic_inc +#define erts_atomic_dec_mb erts_no_atomic_dec +#define erts_atomic_add_read_mb erts_no_atomic_add_read +#define erts_atomic_add_mb erts_no_atomic_add +#define erts_atomic_read_bor_mb erts_no_atomic_read_bor +#define erts_atomic_read_band_mb erts_no_atomic_read_band +#define erts_atomic_xchg_mb erts_no_atomic_xchg +#define erts_atomic_cmpxchg_mb erts_no_atomic_cmpxchg + +#define erts_atomic_init_acqb erts_no_atomic_set +#define erts_atomic_set_acqb erts_no_atomic_set +#define erts_atomic_read_acqb erts_no_atomic_read +#define erts_atomic_inc_read_acqb erts_no_atomic_inc_read +#define erts_atomic_dec_read_acqb erts_no_atomic_dec_read +#define erts_atomic_inc_acqb erts_no_atomic_inc +#define erts_atomic_dec_acqb erts_no_atomic_dec +#define erts_atomic_add_read_acqb erts_no_atomic_add_read +#define erts_atomic_add_acqb erts_no_atomic_add +#define erts_atomic_read_bor_acqb erts_no_atomic_read_bor +#define erts_atomic_read_band_acqb erts_no_atomic_read_band +#define erts_atomic_xchg_acqb erts_no_atomic_xchg +#define erts_atomic_cmpxchg_acqb erts_no_atomic_cmpxchg + +#define erts_atomic_init_relb erts_no_atomic_set +#define erts_atomic_set_relb erts_no_atomic_set +#define erts_atomic_read_relb erts_no_atomic_read +#define erts_atomic_inc_read_relb erts_no_atomic_inc_read +#define erts_atomic_dec_read_relb erts_no_atomic_dec_read +#define erts_atomic_inc_relb erts_no_atomic_inc +#define erts_atomic_dec_relb erts_no_atomic_dec +#define erts_atomic_add_read_relb erts_no_atomic_add_read +#define erts_atomic_add_relb erts_no_atomic_add +#define erts_atomic_read_bor_relb erts_no_atomic_read_bor +#define erts_atomic_read_band_relb erts_no_atomic_read_band +#define erts_atomic_xchg_relb erts_no_atomic_xchg +#define erts_atomic_cmpxchg_relb erts_no_atomic_cmpxchg + +#define erts_atomic_init_rb erts_no_atomic_set +#define erts_atomic_set_rb erts_no_atomic_set +#define erts_atomic_read_rb erts_no_atomic_read +#define erts_atomic_inc_read_rb erts_no_atomic_inc_read +#define erts_atomic_dec_read_rb erts_no_atomic_dec_read +#define erts_atomic_inc_rb erts_no_atomic_inc +#define erts_atomic_dec_rb erts_no_atomic_dec +#define erts_atomic_add_read_rb erts_no_atomic_add_read +#define erts_atomic_add_rb erts_no_atomic_add +#define erts_atomic_read_bor_rb erts_no_atomic_read_bor +#define erts_atomic_read_band_rb erts_no_atomic_read_band +#define erts_atomic_xchg_rb erts_no_atomic_xchg +#define erts_atomic_cmpxchg_rb erts_no_atomic_cmpxchg + +#define erts_atomic_init_wb erts_no_atomic_set +#define erts_atomic_set_wb erts_no_atomic_set +#define erts_atomic_read_wb erts_no_atomic_read +#define erts_atomic_inc_read_wb erts_no_atomic_inc_read +#define erts_atomic_dec_read_wb erts_no_atomic_dec_read +#define erts_atomic_inc_wb erts_no_atomic_inc +#define erts_atomic_dec_wb erts_no_atomic_dec +#define erts_atomic_add_read_wb erts_no_atomic_add_read +#define erts_atomic_add_wb erts_no_atomic_add +#define erts_atomic_read_bor_wb erts_no_atomic_read_bor +#define erts_atomic_read_band_wb erts_no_atomic_read_band +#define erts_atomic_xchg_wb erts_no_atomic_xchg +#define erts_atomic_cmpxchg_wb erts_no_atomic_cmpxchg + +/* 32-bit atomics */ + +#define erts_atomic32_init_nob erts_no_atomic32_set +#define erts_atomic32_set_nob erts_no_atomic32_set +#define erts_atomic32_read_nob erts_no_atomic32_read +#define erts_atomic32_inc_read_nob erts_no_atomic32_inc_read +#define erts_atomic32_dec_read_nob erts_no_atomic32_dec_read +#define erts_atomic32_inc_nob erts_no_atomic32_inc +#define erts_atomic32_dec_nob erts_no_atomic32_dec +#define erts_atomic32_add_read_nob erts_no_atomic32_add_read +#define erts_atomic32_add_nob erts_no_atomic32_add +#define erts_atomic32_read_bor_nob erts_no_atomic32_read_bor +#define erts_atomic32_read_band_nob erts_no_atomic32_read_band +#define erts_atomic32_xchg_nob erts_no_atomic32_xchg +#define erts_atomic32_cmpxchg_nob erts_no_atomic32_cmpxchg + +#define erts_atomic32_init_mb erts_no_atomic32_set +#define erts_atomic32_set_mb erts_no_atomic32_set +#define erts_atomic32_read_mb erts_no_atomic32_read +#define erts_atomic32_inc_read_mb erts_no_atomic32_inc_read +#define erts_atomic32_dec_read_mb erts_no_atomic32_dec_read +#define erts_atomic32_inc_mb erts_no_atomic32_inc +#define erts_atomic32_dec_mb erts_no_atomic32_dec +#define erts_atomic32_add_read_mb erts_no_atomic32_add_read +#define erts_atomic32_add_mb erts_no_atomic32_add +#define erts_atomic32_read_bor_mb erts_no_atomic32_read_bor +#define erts_atomic32_read_band_mb erts_no_atomic32_read_band +#define erts_atomic32_xchg_mb erts_no_atomic32_xchg +#define erts_atomic32_cmpxchg_mb erts_no_atomic32_cmpxchg + +#define erts_atomic32_init_acqb erts_no_atomic32_set +#define erts_atomic32_set_acqb erts_no_atomic32_set +#define erts_atomic32_read_acqb erts_no_atomic32_read +#define erts_atomic32_inc_read_acqb erts_no_atomic32_inc_read +#define erts_atomic32_dec_read_acqb erts_no_atomic32_dec_read +#define erts_atomic32_inc_acqb erts_no_atomic32_inc +#define erts_atomic32_dec_acqb erts_no_atomic32_dec +#define erts_atomic32_add_read_acqb erts_no_atomic32_add_read +#define erts_atomic32_add_acqb erts_no_atomic32_add +#define erts_atomic32_read_bor_acqb erts_no_atomic32_read_bor +#define erts_atomic32_read_band_acqb erts_no_atomic32_read_band +#define erts_atomic32_xchg_acqb erts_no_atomic32_xchg +#define erts_atomic32_cmpxchg_acqb erts_no_atomic32_cmpxchg + +#define erts_atomic32_init_relb erts_no_atomic32_set +#define erts_atomic32_set_relb erts_no_atomic32_set +#define erts_atomic32_read_relb erts_no_atomic32_read +#define erts_atomic32_inc_read_relb erts_no_atomic32_inc_read +#define erts_atomic32_dec_read_relb erts_no_atomic32_dec_read +#define erts_atomic32_inc_relb erts_no_atomic32_inc +#define erts_atomic32_dec_relb erts_no_atomic32_dec +#define erts_atomic32_add_read_relb erts_no_atomic32_add_read +#define erts_atomic32_add_relb erts_no_atomic32_add +#define erts_atomic32_read_bor_relb erts_no_atomic32_read_bor +#define erts_atomic32_read_band_relb erts_no_atomic32_read_band +#define erts_atomic32_xchg_relb erts_no_atomic32_xchg +#define erts_atomic32_cmpxchg_relb erts_no_atomic32_cmpxchg + +#define erts_atomic32_init_rb erts_no_atomic32_set +#define erts_atomic32_set_rb erts_no_atomic32_set +#define erts_atomic32_read_rb erts_no_atomic32_read +#define erts_atomic32_inc_read_rb erts_no_atomic32_inc_read +#define erts_atomic32_dec_read_rb erts_no_atomic32_dec_read +#define erts_atomic32_inc_rb erts_no_atomic32_inc +#define erts_atomic32_dec_rb erts_no_atomic32_dec +#define erts_atomic32_add_read_rb erts_no_atomic32_add_read +#define erts_atomic32_add_rb erts_no_atomic32_add +#define erts_atomic32_read_bor_rb erts_no_atomic32_read_bor +#define erts_atomic32_read_band_rb erts_no_atomic32_read_band +#define erts_atomic32_xchg_rb erts_no_atomic32_xchg +#define erts_atomic32_cmpxchg_rb erts_no_atomic32_cmpxchg + +#define erts_atomic32_init_wb erts_no_atomic32_set +#define erts_atomic32_set_wb erts_no_atomic32_set +#define erts_atomic32_read_wb erts_no_atomic32_read +#define erts_atomic32_inc_read_wb erts_no_atomic32_inc_read +#define erts_atomic32_dec_read_wb erts_no_atomic32_dec_read +#define erts_atomic32_inc_wb erts_no_atomic32_inc +#define erts_atomic32_dec_wb erts_no_atomic32_dec +#define erts_atomic32_add_read_wb erts_no_atomic32_add_read +#define erts_atomic32_add_wb erts_no_atomic32_add +#define erts_atomic32_read_bor_wb erts_no_atomic32_read_bor +#define erts_atomic32_read_band_wb erts_no_atomic32_read_band +#define erts_atomic32_xchg_wb erts_no_atomic32_xchg +#define erts_atomic32_cmpxchg_wb erts_no_atomic32_cmpxchg + +#endif /* !USE_THREADS */ + #if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_INLINE void @@ -995,428 +1414,206 @@ erts_lc_rwmtx_is_rwlocked(erts_rwmtx_t *mtx) #endif } +/* No atomic ops */ + ERTS_GLB_INLINE void -erts_atomic_init(erts_atomic_t *var, erts_aint_t i) +erts_no_dw_atomic_set(erts_no_dw_atomic_t *var, erts_no_dw_atomic_t *val) { -#ifdef USE_THREADS - ethr_atomic_init(var, i); -#else - *var = i; -#endif + var->sint[0] = val->sint[0]; + var->sint[1] = val->sint[1]; } ERTS_GLB_INLINE void -erts_atomic_set(erts_atomic_t *var, erts_aint_t i) +erts_no_dw_atomic_read(erts_no_dw_atomic_t *var, erts_no_dw_atomic_t *val) +{ + val->sint[0] = var->sint[0]; + val->sint[1] = var->sint[1]; +} + +ERTS_GLB_INLINE int erts_no_dw_atomic_cmpxchg(erts_no_dw_atomic_t *var, + erts_no_dw_atomic_t *new_val, + erts_no_dw_atomic_t *old_val) +{ + if (var->sint[0] != old_val->sint[0] || var->sint[1] != old_val->sint[1]) { + erts_no_dw_atomic_read(var, old_val); + return 0; + } + else { + erts_no_dw_atomic_set(var, new_val); + return !0; + } +} + +ERTS_GLB_INLINE void +erts_no_atomic_set(erts_no_atomic_t *var, erts_aint_t i) { -#ifdef USE_THREADS - ethr_atomic_set(var, i); -#else *var = i; -#endif } ERTS_GLB_INLINE erts_aint_t -erts_atomic_read(erts_atomic_t *var) +erts_no_atomic_read(erts_no_atomic_t *var) { -#ifdef USE_THREADS - return ethr_atomic_read(var); -#else return *var; -#endif } ERTS_GLB_INLINE erts_aint_t -erts_atomic_inctest(erts_atomic_t *incp) +erts_no_atomic_inc_read(erts_no_atomic_t *incp) { -#ifdef USE_THREADS - return ethr_atomic_inc_read(incp); -#else return ++(*incp); -#endif } ERTS_GLB_INLINE erts_aint_t -erts_atomic_dectest(erts_atomic_t *decp) +erts_no_atomic_dec_read(erts_no_atomic_t *decp) { -#ifdef USE_THREADS - return ethr_atomic_dec_read(decp); -#else return --(*decp); -#endif } ERTS_GLB_INLINE void -erts_atomic_inc(erts_atomic_t *incp) +erts_no_atomic_inc(erts_no_atomic_t *incp) { -#ifdef USE_THREADS - ethr_atomic_inc(incp); -#else ++(*incp); -#endif } ERTS_GLB_INLINE void -erts_atomic_dec(erts_atomic_t *decp) +erts_no_atomic_dec(erts_no_atomic_t *decp) { -#ifdef USE_THREADS - ethr_atomic_dec(decp); -#else --(*decp); -#endif } ERTS_GLB_INLINE erts_aint_t -erts_atomic_addtest(erts_atomic_t *addp, erts_aint_t i) +erts_no_atomic_add_read(erts_no_atomic_t *addp, erts_aint_t i) { -#ifdef USE_THREADS - return ethr_atomic_add_read(addp, i); -#else return *addp += i; -#endif } ERTS_GLB_INLINE void -erts_atomic_add(erts_atomic_t *addp, erts_aint_t i) +erts_no_atomic_add(erts_no_atomic_t *addp, erts_aint_t i) { -#ifdef USE_THREADS - ethr_atomic_add(addp, i); -#else *addp += i; -#endif -} - -ERTS_GLB_INLINE erts_aint_t -erts_atomic_xchg(erts_atomic_t *xchgp, erts_aint_t new) -{ -#ifdef USE_THREADS - return ethr_atomic_xchg(xchgp, new); -#else - erts_aint_t old = *xchgp; - *xchgp = new; - return old; -#endif -} - -ERTS_GLB_INLINE erts_aint_t -erts_atomic_cmpxchg(erts_atomic_t *xchgp, erts_aint_t new, erts_aint_t expected) -{ -#ifdef USE_THREADS - return ethr_atomic_cmpxchg(xchgp, new, expected); -#else - erts_aint_t old = *xchgp; - if (old == expected) - *xchgp = new; - return old; -#endif } ERTS_GLB_INLINE erts_aint_t -erts_atomic_bor(erts_atomic_t *var, erts_aint_t mask) +erts_no_atomic_read_bor(erts_no_atomic_t *var, erts_aint_t mask) { -#ifdef USE_THREADS - return ethr_atomic_read_bor(var, mask); -#else erts_aint_t old; old = *var; *var |= mask; return old; -#endif } ERTS_GLB_INLINE erts_aint_t -erts_atomic_band(erts_atomic_t *var, erts_aint_t mask) +erts_no_atomic_read_band(erts_no_atomic_t *var, erts_aint_t mask) { -#ifdef USE_THREADS - return ethr_atomic_read_band(var, mask); -#else erts_aint_t old; old = *var; *var &= mask; return old; -#endif } ERTS_GLB_INLINE erts_aint_t -erts_atomic_read_acqb(erts_atomic_t *var) +erts_no_atomic_xchg(erts_no_atomic_t *xchgp, erts_aint_t new) { -#ifdef USE_THREADS - return ethr_atomic_read_acqb(var); -#else - return *var; -#endif -} - -ERTS_GLB_INLINE void -erts_atomic_set_relb(erts_atomic_t *var, erts_aint_t i) -{ -#ifdef USE_THREADS - ethr_atomic_set_relb(var, i); -#else - *var = i; -#endif -} - -ERTS_GLB_INLINE void -erts_atomic_dec_relb(erts_atomic_t *decp) -{ -#ifdef USE_THREADS - ethr_atomic_dec_relb(decp); -#else - --(*decp); -#endif -} - -ERTS_GLB_INLINE erts_aint_t -erts_atomic_dectest_relb(erts_atomic_t *decp) -{ -#ifdef USE_THREADS - return ethr_atomic_dec_read_relb(decp); -#else - return --(*decp); -#endif -} - -ERTS_GLB_INLINE erts_aint_t erts_atomic_cmpxchg_acqb(erts_atomic_t *xchgp, - erts_aint_t new, - erts_aint_t exp) -{ -#ifdef USE_THREADS - return ethr_atomic_cmpxchg_acqb(xchgp, new, exp); -#else erts_aint_t old = *xchgp; - if (old == exp) - *xchgp = new; + *xchgp = new; return old; -#endif } -ERTS_GLB_INLINE erts_aint_t erts_atomic_cmpxchg_relb(erts_atomic_t *xchgp, - erts_aint_t new, - erts_aint_t exp) +ERTS_GLB_INLINE erts_aint_t +erts_no_atomic_cmpxchg(erts_no_atomic_t *xchgp, + erts_aint_t new, + erts_aint_t expected) { -#ifdef USE_THREADS - return ethr_atomic_cmpxchg_relb(xchgp, new, exp); -#else erts_aint_t old = *xchgp; - if (old == exp) + if (old == expected) *xchgp = new; return old; -#endif } /* atomic32 */ ERTS_GLB_INLINE void -erts_atomic32_init(erts_atomic32_t *var, erts_aint32_t i) -{ -#ifdef USE_THREADS - ethr_atomic32_init(var, i); -#else - *var = i; -#endif -} - -ERTS_GLB_INLINE void -erts_atomic32_set(erts_atomic32_t *var, erts_aint32_t i) +erts_no_atomic32_set(erts_no_atomic32_t *var, erts_aint32_t i) { -#ifdef USE_THREADS - ethr_atomic32_set(var, i); -#else *var = i; -#endif } ERTS_GLB_INLINE erts_aint32_t -erts_atomic32_read(erts_atomic32_t *var) +erts_no_atomic32_read(erts_no_atomic32_t *var) { -#ifdef USE_THREADS - return ethr_atomic32_read(var); -#else return *var; -#endif } ERTS_GLB_INLINE erts_aint32_t -erts_atomic32_inctest(erts_atomic32_t *incp) +erts_no_atomic32_inc_read(erts_no_atomic32_t *incp) { -#ifdef USE_THREADS - return ethr_atomic32_inc_read(incp); -#else return ++(*incp); -#endif } ERTS_GLB_INLINE erts_aint32_t -erts_atomic32_dectest(erts_atomic32_t *decp) +erts_no_atomic32_dec_read(erts_no_atomic32_t *decp) { -#ifdef USE_THREADS - return ethr_atomic32_dec_read(decp); -#else return --(*decp); -#endif } ERTS_GLB_INLINE void -erts_atomic32_inc(erts_atomic32_t *incp) +erts_no_atomic32_inc(erts_no_atomic32_t *incp) { -#ifdef USE_THREADS - ethr_atomic32_inc(incp); -#else ++(*incp); -#endif } ERTS_GLB_INLINE void -erts_atomic32_dec(erts_atomic32_t *decp) +erts_no_atomic32_dec(erts_no_atomic32_t *decp) { -#ifdef USE_THREADS - ethr_atomic32_dec(decp); -#else --(*decp); -#endif } ERTS_GLB_INLINE erts_aint32_t -erts_atomic32_addtest(erts_atomic32_t *addp, erts_aint32_t i) +erts_no_atomic32_add_read(erts_no_atomic32_t *addp, erts_aint32_t i) { -#ifdef USE_THREADS - return ethr_atomic32_add_read(addp, i); -#else return *addp += i; -#endif } ERTS_GLB_INLINE void -erts_atomic32_add(erts_atomic32_t *addp, erts_aint32_t i) +erts_no_atomic32_add(erts_no_atomic32_t *addp, erts_aint32_t i) { -#ifdef USE_THREADS - ethr_atomic32_add(addp, i); -#else *addp += i; -#endif -} - -ERTS_GLB_INLINE erts_aint32_t -erts_atomic32_xchg(erts_atomic32_t *xchgp, erts_aint32_t new) -{ -#ifdef USE_THREADS - return ethr_atomic32_xchg(xchgp, new); -#else - erts_aint32_t old = *xchgp; - *xchgp = new; - return old; -#endif -} - -ERTS_GLB_INLINE erts_aint32_t -erts_atomic32_cmpxchg(erts_atomic32_t *xchgp, - erts_aint32_t new, - erts_aint32_t expected) -{ -#ifdef USE_THREADS - return ethr_atomic32_cmpxchg(xchgp, new, expected); -#else - erts_aint32_t old = *xchgp; - if (old == expected) - *xchgp = new; - return old; -#endif } ERTS_GLB_INLINE erts_aint32_t -erts_atomic32_bor(erts_atomic32_t *var, erts_aint32_t mask) +erts_no_atomic32_read_bor(erts_no_atomic32_t *var, erts_aint32_t mask) { -#ifdef USE_THREADS - return ethr_atomic32_read_bor(var, mask); -#else erts_aint32_t old; old = *var; *var |= mask; return old; -#endif } ERTS_GLB_INLINE erts_aint32_t -erts_atomic32_band(erts_atomic32_t *var, erts_aint32_t mask) +erts_no_atomic32_read_band(erts_no_atomic32_t *var, erts_aint32_t mask) { -#ifdef USE_THREADS - return ethr_atomic32_read_band(var, mask); -#else erts_aint32_t old; old = *var; *var &= mask; return old; -#endif } ERTS_GLB_INLINE erts_aint32_t -erts_atomic32_read_acqb(erts_atomic32_t *var) -{ -#ifdef USE_THREADS - return ethr_atomic32_read_acqb(var); -#else - return *var; -#endif -} - -ERTS_GLB_INLINE void -erts_atomic32_set_relb(erts_atomic32_t *var, erts_aint32_t i) -{ -#ifdef USE_THREADS - ethr_atomic32_set_relb(var, i); -#else - *var = i; -#endif -} - -ERTS_GLB_INLINE void -erts_atomic32_dec_relb(erts_atomic32_t *decp) +erts_no_atomic32_xchg(erts_no_atomic32_t *xchgp, erts_aint32_t new) { -#ifdef USE_THREADS - ethr_atomic32_dec_relb(decp); -#else - --(*decp); -#endif -} - -ERTS_GLB_INLINE erts_aint32_t -erts_atomic32_dectest_relb(erts_atomic32_t *decp) -{ -#ifdef USE_THREADS - return ethr_atomic32_dec_read_relb(decp); -#else - return --(*decp); -#endif -} - -ERTS_GLB_INLINE erts_aint32_t -erts_atomic32_cmpxchg_acqb(erts_atomic32_t *xchgp, - erts_aint32_t new, - erts_aint32_t exp) -{ -#ifdef USE_THREADS - return ethr_atomic32_cmpxchg_acqb(xchgp, new, exp); -#else erts_aint32_t old = *xchgp; - if (old == exp) - *xchgp = new; + *xchgp = new; return old; -#endif } ERTS_GLB_INLINE erts_aint32_t -erts_atomic32_cmpxchg_relb(erts_atomic32_t *xchgp, - erts_aint32_t new, - erts_aint32_t exp) +erts_no_atomic32_cmpxchg(erts_no_atomic32_t *xchgp, + erts_aint32_t new, + erts_aint32_t expected) { -#ifdef USE_THREADS - return ethr_atomic32_cmpxchg_relb(xchgp, new, exp); -#else erts_aint32_t old = *xchgp; - if (old == exp) + if (old == expected) *xchgp = new; return old; -#endif } /* spinlock */ @@ -1887,3 +2084,37 @@ erts_thr_sigwait(const sigset_t *set, int *sig) #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ #endif /* #ifndef ERL_THREAD_H__ */ + +#ifdef ERTS_UNDEF_DEPRECATED_ATOMICS + +/* Deprecated functions to replace */ + +#undef erts_atomic_init +#undef erts_atomic_set +#undef erts_atomic_read +#undef erts_atomic_inctest +#undef erts_atomic_dectest +#undef erts_atomic_inc +#undef erts_atomic_dec +#undef erts_atomic_addtest +#undef erts_atomic_add +#undef erts_atomic_xchg +#undef erts_atomic_cmpxchg +#undef erts_atomic_bor +#undef erts_atomic_band + +#undef erts_atomic32_init +#undef erts_atomic32_set +#undef erts_atomic32_read +#undef erts_atomic32_inctest +#undef erts_atomic32_dectest +#undef erts_atomic32_inc +#undef erts_atomic32_dec +#undef erts_atomic32_addtest +#undef erts_atomic32_add +#undef erts_atomic32_xchg +#undef erts_atomic32_cmpxchg +#undef erts_atomic32_bor +#undef erts_atomic32_band + +#endif diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h index d0ad73cd81..7a09d30ff6 100644 --- a/erts/emulator/beam/erl_time.h +++ b/erts/emulator/beam/erl_time.h @@ -85,8 +85,8 @@ ERTS_GLB_INLINE void erts_do_time_add(long); #if ERTS_GLB_INLINE_INCL_FUNC_DEF -ERTS_GLB_INLINE erts_aint_t erts_do_time_read_and_reset(void) { return erts_smp_atomic_xchg(&do_time, 0L); } -ERTS_GLB_INLINE void erts_do_time_add(long elapsed) { erts_smp_atomic_add(&do_time, elapsed); } +ERTS_GLB_INLINE erts_aint_t erts_do_time_read_and_reset(void) { return erts_smp_atomic_xchg_acqb(&do_time, 0L); } +ERTS_GLB_INLINE void erts_do_time_add(long elapsed) { erts_smp_atomic_add_relb(&do_time, elapsed); } #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 499bdd77ba..249df54015 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -200,10 +200,10 @@ erts_port_runq(Port *prt) { #ifdef ERTS_SMP ErtsRunQueue *rq1, *rq2; - rq1 = (ErtsRunQueue *) erts_smp_atomic_read(&prt->run_queue); + rq1 = (ErtsRunQueue *) erts_smp_atomic_read_nob(&prt->run_queue); while (1) { erts_smp_runq_lock(rq1); - rq2 = (ErtsRunQueue *) erts_smp_atomic_read(&prt->run_queue); + rq2 = (ErtsRunQueue *) erts_smp_atomic_read_nob(&prt->run_queue); if (rq1 == rq2) return rq1; erts_smp_runq_unlock(rq1); @@ -542,10 +542,11 @@ ERTS_GLB_INLINE void erts_may_save_closed_port(Port *prt) ERTS_SMP_LC_ASSERT(erts_smp_lc_spinlock_is_locked(&prt->state_lck)); if (prt->snapshot != erts_smp_atomic32_read_acqb(&erts_ports_snapshot)) { /* Dead ports are added from the end of the snapshot buffer */ - Eterm* tombstone = (Eterm*) erts_smp_atomic_addtest(&erts_dead_ports_ptr, - -(erts_aint_t)sizeof(Eterm)); + Eterm* tombstone; + tombstone = (Eterm*) erts_smp_atomic_add_read_nob(&erts_dead_ports_ptr, + -(erts_aint_t)sizeof(Eterm)); ASSERT(tombstone+1 != NULL); - ASSERT(prt->snapshot == erts_smp_atomic32_read(&erts_ports_snapshot) - 1); + ASSERT(prt->snapshot == erts_smp_atomic_read_nob(&erts_ports_snapshot) - 1); *tombstone = prt->id; } /*else no ongoing snapshot or port was already included or created after snapshot */ @@ -1200,11 +1201,11 @@ erts_smp_port_trylock(Port *prt) #ifdef ERTS_SMP int res; - ASSERT(erts_smp_atomic_read(&prt->refc) > 0); - erts_smp_atomic_inc(&prt->refc); + ASSERT(erts_smp_atomic_read_nob(&prt->refc) > 0); + erts_smp_atomic_inc_nob(&prt->refc); res = erts_smp_mtx_trylock(prt->lock); if (res == EBUSY) { - erts_smp_atomic_dec(&prt->refc); + erts_smp_atomic_dec_nob(&prt->refc); } return res; @@ -1217,8 +1218,8 @@ ERTS_GLB_INLINE void erts_smp_port_lock(Port *prt) { #ifdef ERTS_SMP - ASSERT(erts_smp_atomic_read(&prt->refc) > 0); - erts_smp_atomic_inc(&prt->refc); + ASSERT(erts_smp_atomic_read_nob(&prt->refc) > 0); + erts_smp_atomic_inc_nob(&prt->refc); erts_smp_mtx_lock(prt->lock); #endif } @@ -1229,7 +1230,7 @@ erts_smp_port_unlock(Port *prt) #ifdef ERTS_SMP erts_aint_t refc; erts_smp_mtx_unlock(prt->lock); - refc = erts_smp_atomic_dectest(&prt->refc); + refc = erts_smp_atomic_dec_read_nob(&prt->refc); ASSERT(refc >= 0); if (refc == 0) erts_port_cleanup(prt); @@ -1298,7 +1299,7 @@ erts_id2port_sflgs(Eterm id, Process *c_p, ErtsProcLocks c_p_locks, Uint32 sflgs } #ifdef ERTS_SMP else { - erts_smp_atomic_inc(&prt->refc); + erts_smp_atomic_inc_nob(&prt->refc); erts_smp_port_state_unlock(prt); if (no_proc_locks) diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index df5f8b22a3..151c776a3d 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -244,8 +244,8 @@ get_free_port(void) } port->status = ERTS_PORT_SFLG_INITIALIZING; #ifdef ERTS_SMP - ERTS_SMP_LC_ASSERT(erts_smp_atomic_read(&port->refc) == 0); - erts_smp_atomic_set(&port->refc, 2); /* Port alive + lock */ + ERTS_SMP_LC_ASSERT(erts_smp_atomic_read_nob(&port->refc) == 0); + erts_smp_atomic_set_nob(&port->refc, 2); /* Port alive + lock */ #endif erts_smp_port_state_unlock(port); return num & port_num_mask; @@ -327,7 +327,7 @@ port_cleanup(Port *prt) #ifdef ERTS_SMP ASSERT(prt->status & ERTS_PORT_SFLG_FREE_SCHEDULED); - ERTS_SMP_LC_ASSERT(erts_smp_atomic_read(&prt->refc) == 0); + ERTS_SMP_LC_ASSERT(erts_smp_atomic_read_nob(&prt->refc) == 0); port_specific = (prt->status & ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK); @@ -425,11 +425,11 @@ setup_port(Port* prt, Eterm pid, erts_driver_t *driver, erts_smp_runq_lock(runq); erts_smp_port_state_lock(prt); prt->status = ERTS_PORT_SFLG_CONNECTED | xstatus; - prt->snapshot = erts_smp_atomic32_read(&erts_ports_snapshot); + prt->snapshot = erts_smp_atomic32_read_nob(&erts_ports_snapshot); old_name = prt->name; prt->name = new_name; #ifdef ERTS_SMP - erts_smp_atomic_set(&prt->run_queue, (erts_aint_t) runq); + erts_smp_atomic_set_nob(&prt->run_queue, (erts_aint_t) runq); #endif ASSERT(!prt->drv_ptr); prt->drv_ptr = driver; @@ -590,8 +590,8 @@ erts_open_driver(erts_driver_t* driver, /* Pointer to driver. */ erts_smp_port_state_lock(port); port->status = ERTS_PORT_SFLG_FREE; #ifdef ERTS_SMP - ERTS_SMP_LC_ASSERT(erts_smp_atomic_read(&port->refc) == 2); - erts_smp_atomic_set(&port->refc, 0); + ERTS_SMP_LC_ASSERT(erts_smp_atomic_read_nob(&port->refc) == 2); + erts_smp_atomic_set_nob(&port->refc, 0); #endif erts_smp_port_state_unlock(port); return -3; @@ -1206,7 +1206,7 @@ int erts_write_to_port(Eterm caller_id, Port *p, Eterm list) } } p->bytes_out += size; - erts_smp_atomic_add(&erts_bytes_out, size); + erts_smp_atomic_add_nob(&erts_bytes_out, size); #ifdef ERTS_SMP if (p->xports) @@ -1277,13 +1277,13 @@ void init_io(void) erts_port = (Port *) erts_alloc(ERTS_ALC_T_PORT_TABLE, erts_max_ports * sizeof(Port)); - erts_smp_atomic_init(&erts_bytes_out, 0); - erts_smp_atomic_init(&erts_bytes_in, 0); + erts_smp_atomic_init_nob(&erts_bytes_out, 0); + erts_smp_atomic_init_nob(&erts_bytes_in, 0); for (i = 0; i < erts_max_ports; i++) { erts_port_task_init_sched(&erts_port[i].sched); #ifdef ERTS_SMP - erts_smp_atomic_init(&erts_port[i].refc, 0); + erts_smp_atomic_init_nob(&erts_port[i].refc, 0); erts_port[i].lock = NULL; erts_port[i].xports = NULL; erts_smp_spinlock_init_x(&erts_port[i].state_lck, "port_state", make_small(i)); @@ -1300,7 +1300,7 @@ void init_io(void) erts_port[i].port_data_lock = NULL; } - erts_smp_atomic32_init(&erts_ports_snapshot, (erts_aint32_t) 0); + erts_smp_atomic32_init_nob(&erts_ports_snapshot, (erts_aint32_t) 0); last_port_num = 0; erts_smp_spinlock_init(&get_free_port_lck, "get_free_port"); @@ -3253,7 +3253,7 @@ int driver_output_binary(ErlDrvPort ix, char* hbuf, int hlen, return 0; prt->bytes_in += (hlen + len); - erts_smp_atomic_add(&erts_bytes_in, (erts_aint_t) (hlen + len)); + erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) (hlen + len)); if (prt->status & ERTS_PORT_SFLG_DISTRIBUTION) { return erts_net_message(prt, prt->dist_entry, @@ -3288,7 +3288,7 @@ int driver_output2(ErlDrvPort ix, char* hbuf, int hlen, char* buf, int len) return 0; prt->bytes_in += (hlen + len); - erts_smp_atomic_add(&erts_bytes_in, (erts_aint_t) (hlen + len)); + erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) (hlen + len)); if (prt->status & ERTS_PORT_SFLG_DISTRIBUTION) { if (len == 0) return erts_net_message(prt, @@ -3365,7 +3365,7 @@ int driver_outputv(ErlDrvPort ix, char* hbuf, int hlen, ErlIOVec* vec, int skip) /* XXX handle distribution !!! */ prt->bytes_in += (hlen + size); - erts_smp_atomic_add(&erts_bytes_in, (erts_aint_t) (hlen + size)); + erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) (hlen + size)); deliver_vec_message(prt, prt->connected, hbuf, hlen, binv, iov, n, size); return 0; } @@ -3539,13 +3539,13 @@ pdl_init(void) static ERTS_INLINE void pdl_init_refc(ErlDrvPDL pdl) { - erts_atomic_init(&pdl->refc, 1); + erts_atomic_init_nob(&pdl->refc, 1); } static ERTS_INLINE ErlDrvSInt pdl_read_refc(ErlDrvPDL pdl) { - erts_aint_t refc = erts_atomic_read(&pdl->refc); + erts_aint_t refc = erts_atomic_read_nob(&pdl->refc); ERTS_LC_ASSERT(refc >= 0); return (ErlDrvSInt) refc; } @@ -3553,14 +3553,14 @@ pdl_read_refc(ErlDrvPDL pdl) static ERTS_INLINE void pdl_inc_refc(ErlDrvPDL pdl) { - erts_atomic_inc(&pdl->refc); + erts_atomic_inc_nob(&pdl->refc); ERTS_LC_ASSERT(driver_pdl_get_refc(pdl) > 1); } static ERTS_INLINE ErlDrvSInt pdl_inctest_refc(ErlDrvPDL pdl) { - erts_aint_t refc = erts_atomic_inctest(&pdl->refc); + erts_aint_t refc = erts_atomic_inc_read_nob(&pdl->refc); ERTS_LC_ASSERT(refc > 1); return (ErlDrvSInt) refc; } @@ -3569,7 +3569,7 @@ pdl_inctest_refc(ErlDrvPDL pdl) static ERTS_INLINE void pdl_dec_refc(ErlDrvPDL pdl) { - erts_atomic_dec(&pdl->refc); + erts_atomic_dec_nob(&pdl->refc); ERTS_LC_ASSERT(driver_pdl_get_refc(pdl) > 0); } #endif @@ -3577,7 +3577,7 @@ pdl_dec_refc(ErlDrvPDL pdl) static ERTS_INLINE ErlDrvSInt pdl_dectest_refc(ErlDrvPDL pdl) { - erts_aint_t refc = erts_atomic_dectest(&pdl->refc); + erts_aint_t refc = erts_atomic_dec_read_nob(&pdl->refc); ERTS_LC_ASSERT(refc >= 0); return (ErlDrvSInt) refc; } diff --git a/erts/emulator/beam/safe_hash.c b/erts/emulator/beam/safe_hash.c index 4c54e19cdb..3326e5cc2a 100644 --- a/erts/emulator/beam/safe_hash.c +++ b/erts/emulator/beam/safe_hash.c @@ -61,7 +61,7 @@ static ERTS_INLINE int align_up_pow2(int val) */ static void rehash(SafeHash* h, int grow_limit) { - if (erts_smp_atomic_xchg(&h->is_rehashing, 1) != 0) { + if (erts_smp_atomic_xchg_acqb(&h->is_rehashing, 1) != 0) { return; /* already in progress */ } if (h->grow_limit == grow_limit) { @@ -166,8 +166,8 @@ SafeHash* safe_hash_init(ErtsAlcType_t type, SafeHash* h, char* name, int size, h->name = name; h->fun = fun; set_size(h,size); - erts_smp_atomic_init(&h->is_rehashing, 0); - erts_smp_atomic_init(&h->nitems, 0); + erts_smp_atomic_init_nob(&h->is_rehashing, 0); + erts_smp_atomic_init_nob(&h->nitems, 0); for (i=0; i<SAFE_HASH_LOCK_CNT; i++) { erts_smp_mtx_init(&h->lock_vec[i].mtx,"safe_hash"); } @@ -222,7 +222,7 @@ void* safe_hash_put(SafeHash* h, void* tmpl) *head = b; grow_limit = h->grow_limit; erts_smp_mtx_unlock(lock); - if (erts_smp_atomic_inctest(&h->nitems) > grow_limit) { + if (erts_smp_atomic_inc_read_nob(&h->nitems) > grow_limit) { rehash(h, grow_limit); } return (void*) b; @@ -245,7 +245,7 @@ void* safe_hash_erase(SafeHash* h, void* tmpl) if ((b->hvalue == hval) && (h->fun.cmp(tmpl, (void*)b) == 0)) { *prevp = b->next; erts_smp_mtx_unlock(lock); - erts_smp_atomic_dec(&h->nitems); + erts_smp_atomic_dec_nob(&h->nitems); h->fun.free((void*)b); return tmpl; } diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index e64c43de6e..669a601b35 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -340,7 +340,8 @@ int erts_send_warning_to_logger_str_nogl(char *); #ifdef ERTS_WANT_BREAK_HANDLING # ifdef ERTS_SMP extern erts_smp_atomic32_t erts_break_requested; -# define ERTS_BREAK_REQUESTED ((int) erts_smp_atomic32_read(&erts_break_requested)) +# define ERTS_BREAK_REQUESTED \ + ((int) erts_smp_atomic32_read_nob(&erts_break_requested)) # else extern volatile int erts_break_requested; # define ERTS_BREAK_REQUESTED erts_break_requested @@ -354,7 +355,7 @@ void erts_do_break_handling(void); # else # ifdef ERTS_SMP extern erts_smp_atomic32_t erts_got_sigusr1; -# define ERTS_GOT_SIGUSR1 ((int) erts_smp_atomic32_read(&erts_got_sigusr1)) +# define ERTS_GOT_SIGUSR1 ((int) erts_smp_atomic32_read_mb(&erts_got_sigusr1)) # else extern volatile int erts_got_sigusr1; # define ERTS_GOT_SIGUSR1 erts_got_sigusr1 @@ -363,11 +364,15 @@ extern volatile int erts_got_sigusr1; #endif #ifdef ERTS_SMP -extern erts_smp_atomic_t erts_writing_erl_crash_dump; +extern erts_smp_atomic32_t erts_writing_erl_crash_dump; +extern erts_tsd_key_t erts_is_crash_dumping_key; +#define ERTS_SOMEONE_IS_CRASH_DUMPING \ + ((int) erts_smp_atomic32_read_mb(&erts_writing_erl_crash_dump)) #define ERTS_IS_CRASH_DUMPING \ - ((int) erts_smp_atomic_read(&erts_writing_erl_crash_dump)) + ((int) (SWord) erts_tsd_get(erts_is_crash_dumping_key)) #else extern volatile int erts_writing_erl_crash_dump; +#define ERTS_SOMEONE_IS_CRASH_DUMPING erts_writing_erl_crash_dump #define ERTS_IS_CRASH_DUMPING erts_writing_erl_crash_dump #endif @@ -877,7 +882,7 @@ ERTS_GLB_INLINE int erts_smp_pending_system_block(void) { #ifdef ERTS_SMP - return (int) erts_smp_atomic32_read(&erts_system_block_state.do_block); + return (int) erts_smp_atomic32_read_nob(&erts_system_block_state.do_block); #else return 0; #endif @@ -913,7 +918,7 @@ erts_smp_set_activity(erts_activity_t old_activity, case ERTS_ACTIVITY_UNDEFINED: break; case ERTS_ACTIVITY_WAIT: - erts_smp_atomic32_dec(&erts_system_block_state.in_activity.wait); + erts_smp_atomic32_dec_acqb(&erts_system_block_state.in_activity.wait); if (locked) { /* You are not allowed to leave activity waiting * without supplying the possibility to block @@ -924,10 +929,10 @@ erts_smp_set_activity(erts_activity_t old_activity, } break; case ERTS_ACTIVITY_GC: - erts_smp_atomic32_dec(&erts_system_block_state.in_activity.gc); + erts_smp_atomic32_dec_acqb(&erts_system_block_state.in_activity.gc); break; case ERTS_ACTIVITY_IO: - erts_smp_atomic32_dec(&erts_system_block_state.in_activity.io); + erts_smp_atomic32_dec_acqb(&erts_system_block_state.in_activity.io); break; default: erts_set_activity_error(ERTS_ACT_ERR_LEAVE_UNKNOWN_ACTIVITY, @@ -943,13 +948,13 @@ erts_smp_set_activity(erts_activity_t old_activity, case ERTS_ACTIVITY_UNDEFINED: break; case ERTS_ACTIVITY_WAIT: - erts_smp_atomic32_inc(&erts_system_block_state.in_activity.wait); + erts_smp_atomic32_inc_mb(&erts_system_block_state.in_activity.wait); break; case ERTS_ACTIVITY_GC: - erts_smp_atomic32_inc(&erts_system_block_state.in_activity.gc); + erts_smp_atomic32_inc_mb(&erts_system_block_state.in_activity.gc); break; case ERTS_ACTIVITY_IO: - erts_smp_atomic32_inc(&erts_system_block_state.in_activity.io); + erts_smp_atomic32_inc_mb(&erts_system_block_state.in_activity.io); break; default: erts_set_activity_error(ERTS_ACT_ERR_ENTER_UNKNOWN_ACTIVITY, @@ -1001,27 +1006,27 @@ ERTS_GLB_INLINE erts_aint_t erts_refc_read(erts_refc_t *refcp, ERTS_GLB_INLINE void erts_refc_init(erts_refc_t *refcp, erts_aint_t val) { - erts_smp_atomic_init((erts_smp_atomic_t *) refcp, val); + erts_smp_atomic_init_nob((erts_smp_atomic_t *) refcp, val); } ERTS_GLB_INLINE void erts_refc_inc(erts_refc_t *refcp, erts_aint_t min_val) { #ifdef ERTS_REFC_DEBUG - erts_aint_t val = erts_smp_atomic_inctest((erts_smp_atomic_t *) refcp); + erts_aint_t val = erts_smp_atomic_inc_read_nob((erts_smp_atomic_t *) refcp); if (val < min_val) erl_exit(ERTS_ABORT_EXIT, "erts_refc_inc(): Bad refc found (refc=%ld < %ld)!\n", val, min_val); #else - erts_smp_atomic_inc((erts_smp_atomic_t *) refcp); + erts_smp_atomic_inc_nob((erts_smp_atomic_t *) refcp); #endif } ERTS_GLB_INLINE erts_aint_t erts_refc_inctest(erts_refc_t *refcp, erts_aint_t min_val) { - erts_aint_t val = erts_smp_atomic_inctest((erts_smp_atomic_t *) refcp); + erts_aint_t val = erts_smp_atomic_inc_read_nob((erts_smp_atomic_t *) refcp); #ifdef ERTS_REFC_DEBUG if (val < min_val) erl_exit(ERTS_ABORT_EXIT, @@ -1035,20 +1040,20 @@ ERTS_GLB_INLINE void erts_refc_dec(erts_refc_t *refcp, erts_aint_t min_val) { #ifdef ERTS_REFC_DEBUG - erts_aint_t val = erts_smp_atomic_dectest((erts_smp_atomic_t *) refcp); + erts_aint_t val = erts_smp_atomic_dec_read_nob((erts_smp_atomic_t *) refcp); if (val < min_val) erl_exit(ERTS_ABORT_EXIT, "erts_refc_dec(): Bad refc found (refc=%ld < %ld)!\n", val, min_val); #else - erts_smp_atomic_dec((erts_smp_atomic_t *) refcp); + erts_smp_atomic_dec_nob((erts_smp_atomic_t *) refcp); #endif } ERTS_GLB_INLINE erts_aint_t erts_refc_dectest(erts_refc_t *refcp, erts_aint_t min_val) { - erts_aint_t val = erts_smp_atomic_dectest((erts_smp_atomic_t *) refcp); + erts_aint_t val = erts_smp_atomic_dec_read_nob((erts_smp_atomic_t *) refcp); #ifdef ERTS_REFC_DEBUG if (val < min_val) erl_exit(ERTS_ABORT_EXIT, @@ -1062,20 +1067,20 @@ ERTS_GLB_INLINE void erts_refc_add(erts_refc_t *refcp, erts_aint_t diff, erts_aint_t min_val) { #ifdef ERTS_REFC_DEBUG - erts_aint_t val = erts_smp_atomic_addtest((erts_smp_atomic_t *) refcp, diff); + erts_aint_t val = erts_smp_atomic_add_read_nob((erts_smp_atomic_t *) refcp, diff); if (val < min_val) erl_exit(ERTS_ABORT_EXIT, "erts_refc_add(%ld): Bad refc found (refc=%ld < %ld)!\n", diff, val, min_val); #else - erts_smp_atomic_add((erts_smp_atomic_t *) refcp, diff); + erts_smp_atomic_add_nob((erts_smp_atomic_t *) refcp, diff); #endif } ERTS_GLB_INLINE erts_aint_t erts_refc_read(erts_refc_t *refcp, erts_aint_t min_val) { - erts_aint_t val = erts_smp_atomic_read((erts_smp_atomic_t *) refcp); + erts_aint_t val = erts_smp_atomic_read_nob((erts_smp_atomic_t *) refcp); #ifdef ERTS_REFC_DEBUG if (val < min_val) erl_exit(ERTS_ABORT_EXIT, diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c index a00faff912..8fa8c1cfe0 100644 --- a/erts/emulator/beam/time.c +++ b/erts/emulator/beam/time.c @@ -108,9 +108,9 @@ static ErlTimer *tiw_min_ptr; static int itime; /* Constant after init */ erts_smp_atomic_t do_time; /* set at clock interrupt */ -static ERTS_INLINE erts_aint_t do_time_read(void) { return erts_smp_atomic_read(&do_time); } +static ERTS_INLINE erts_aint_t do_time_read(void) { return erts_smp_atomic_read_acqb(&do_time); } static ERTS_INLINE erts_aint_t do_time_update(void) { return do_time_read(); } -static ERTS_INLINE void do_time_init(void) { erts_smp_atomic_init(&do_time, 0L); } +static ERTS_INLINE void do_time_init(void) { erts_smp_atomic_init_nob(&do_time, 0L); } /* get the time (in units of itime) to the next timeout, or -1 if there are no timeouts */ diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index a17de717bc..3f6accba2d 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -3689,14 +3689,16 @@ threads_not_under_control(void) { erts_aint32_t res = system_block_state.threads_to_block; + ERTS_THR_MEMORY_BARRIER; + /* Waiting is always an allowed activity... */ - res -= erts_smp_atomic32_read(&erts_system_block_state.in_activity.wait); + res -= erts_smp_atomic32_read_nob(&erts_system_block_state.in_activity.wait); if (system_block_state.allowed_activities & ERTS_BS_FLG_ALLOW_GC) - res -= erts_smp_atomic32_read(&erts_system_block_state.in_activity.gc); + res -= erts_smp_atomic32_read_nob(&erts_system_block_state.in_activity.gc); if (system_block_state.allowed_activities & ERTS_BS_FLG_ALLOW_IO) - res -= erts_smp_atomic32_read(&erts_system_block_state.in_activity.io); + res -= erts_smp_atomic32_read_nob(&erts_system_block_state.in_activity.io); if (res < 0) { ASSERT(0); @@ -3756,7 +3758,7 @@ erts_block_system(Uint32 allowed_activities) } else { - erts_smp_atomic32_inc(&erts_system_block_state.do_block); + erts_smp_atomic32_inc_nob(&erts_system_block_state.do_block); /* Someone else might be waiting for us to block... */ if (do_block) { @@ -3808,11 +3810,11 @@ erts_emergency_block_system(long timeout, Uint32 allowed_activities) another_blocker = erts_smp_pending_system_block(); system_block_state.emergency = 1; - erts_smp_atomic32_inc(&erts_system_block_state.do_block); + erts_smp_atomic32_inc_nob(&erts_system_block_state.do_block); if (another_blocker) { if (is_blocker()) { - erts_smp_atomic32_dec(&erts_system_block_state.do_block); + erts_smp_atomic32_dec_nob(&erts_system_block_state.do_block); res = 0; goto done; } @@ -3869,7 +3871,7 @@ erts_release_system(void) if (system_block_state.recursive_block) system_block_state.recursive_block--; else { - do_block = erts_smp_atomic32_dectest(&erts_system_block_state.do_block); + do_block = erts_smp_atomic32_dec_read_nob(&erts_system_block_state.do_block); system_block_state.have_blocker = 0; if (is_blockable_thread()) system_block_state.threads_to_block++; @@ -4004,10 +4006,10 @@ erts_system_block_init(void) /* Global state... */ - erts_smp_atomic32_init(&erts_system_block_state.do_block, 0); - erts_smp_atomic32_init(&erts_system_block_state.in_activity.wait, 0); - erts_smp_atomic32_init(&erts_system_block_state.in_activity.gc, 0); - erts_smp_atomic32_init(&erts_system_block_state.in_activity.io, 0); + erts_smp_atomic32_init_nob(&erts_system_block_state.do_block, 0); + erts_smp_atomic32_init_nob(&erts_system_block_state.in_activity.wait, 0); + erts_smp_atomic32_init_nob(&erts_system_block_state.in_activity.gc, 0); + erts_smp_atomic32_init_nob(&erts_system_block_state.in_activity.io, 0); /* Make sure blockable threads unregister when exiting... */ erts_smp_install_exit_handler(erts_unregister_blockable_thread); diff --git a/erts/emulator/hipe/hipe_native_bif.c b/erts/emulator/hipe/hipe_native_bif.c index 8d31348496..dfb4ca794a 100644 --- a/erts/emulator/hipe/hipe_native_bif.c +++ b/erts/emulator/hipe/hipe_native_bif.c @@ -334,7 +334,7 @@ char *hipe_bs_allocate(int len) bptr = erts_bin_nrml_alloc(len); bptr->flags = 0; bptr->orig_size = len; - erts_smp_atomic_init(&bptr->refc, 1); + erts_smp_atomic_init_nob(&bptr->refc, 1); return bptr->orig_bytes; } @@ -584,7 +584,7 @@ void hipe_clear_timeout(Process *c_p) void hipe_atomic_inc(int *counter) { - erts_smp_atomic_inc((erts_smp_atomic_t*)counter); + erts_smp_atomic_inc_nob((erts_smp_atomic_t*)counter); } #endif diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c index cd4de21d65..57321259f9 100644 --- a/erts/emulator/sys/common/erl_check_io.c +++ b/erts/emulator/sys/common/erl_check_io.c @@ -218,7 +218,7 @@ remember_removed(ErtsDrvEventState *state, struct pollset_info* psi) #ifdef ERTS_SMP struct removed_fd *fdlp; ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(fd_mtx(state->fd))); - if (erts_smp_atomic_read(&psi->in_poll_wait)) { + if (erts_smp_atomic_read_nob(&psi->in_poll_wait)) { state->remove_cnt++; ASSERT(state->remove_cnt > 0); fdlp = removed_fd_alloc(); @@ -333,7 +333,7 @@ grow_drv_ev_state(int min_ix) new_len = max_fds; erts_smp_mtx_lock(&drv_ev_state_grow_lock); - if (erts_smp_atomic_read(&drv_ev_state_len) <= min_ix) { + if (erts_smp_atomic_read_nob(&drv_ev_state_len) <= min_ix) { for (i=0; i<DRV_EV_STATE_LOCK_CNT; i++) { /* lock all fd's */ erts_smp_mtx_lock(&drv_ev_state_locks[i].lck); } @@ -343,7 +343,7 @@ grow_drv_ev_state(int min_ix) sizeof(ErtsDrvEventState)*new_len) : erts_alloc(ERTS_ALC_T_DRV_EV_STATE, sizeof(ErtsDrvEventState)*new_len)); - for (i = erts_smp_atomic_read(&drv_ev_state_len); i < new_len; i++) { + for (i = erts_smp_atomic_read_nob(&drv_ev_state_len); i < new_len; i++) { drv_ev_state[i].fd = (ErtsSysFdType) i; drv_ev_state[i].driver.select = NULL; drv_ev_state[i].events = 0; @@ -351,7 +351,7 @@ grow_drv_ev_state(int min_ix) drv_ev_state[i].type = ERTS_EV_TYPE_NONE; drv_ev_state[i].flags = 0; } - erts_smp_atomic_set(&drv_ev_state_len, new_len); + erts_smp_atomic_set_nob(&drv_ev_state_len, new_len); for (i=0; i<DRV_EV_STATE_LOCK_CNT; i++) { erts_smp_mtx_unlock(&drv_ev_state_locks[i].lck); } @@ -497,7 +497,7 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix, && erts_lc_is_port_locked(erts_drvport2port(ix))); #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - if ((unsigned)fd >= (unsigned)erts_smp_atomic_read(&drv_ev_state_len)) { + if ((unsigned)fd >= (unsigned)erts_smp_atomic_read_nob(&drv_ev_state_len)) { if (fd < 0) { return -1; } @@ -709,7 +709,7 @@ ERTS_CIO_EXPORT(driver_event)(ErlDrvPort ix, && erts_lc_is_port_locked(erts_drvport2port(ix))); #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - if ((unsigned)fd >= (unsigned)erts_smp_atomic_read(&drv_ev_state_len)) { + if ((unsigned)fd >= (unsigned)erts_smp_atomic_read_nob(&drv_ev_state_len)) { if (fd < 0) return -1; if (fd >= max_fds) { @@ -1156,7 +1156,7 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait) erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); pollres_len = sizeof(pollres)/sizeof(ErtsPollResFd); - erts_smp_atomic_set(&pollset.in_poll_wait, 1); + erts_smp_atomic_set_nob(&pollset.in_poll_wait, 1); poll_ret = ERTS_CIO_POLL_WAIT(pollset.ps, pollres, &pollres_len, &wait_time); @@ -1173,7 +1173,7 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait) #endif if (poll_ret != 0) { - erts_smp_atomic_set(&pollset.in_poll_wait, 0); + erts_smp_atomic_set_nob(&pollset.in_poll_wait, 0); forget_removed(&pollset); if (poll_ret == EAGAIN) { goto restart; @@ -1304,7 +1304,7 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait) #endif } - erts_smp_atomic_set(&pollset.in_poll_wait, 0); + erts_smp_atomic_set_nob(&pollset.in_poll_wait, 0); forget_removed(&pollset); } @@ -1419,7 +1419,7 @@ static void drv_ev_state_free(void *des) void ERTS_CIO_EXPORT(erts_init_check_io)(void) { - erts_smp_atomic_init(&pollset.in_poll_wait, 0); + erts_smp_atomic_init_nob(&pollset.in_poll_wait, 0); ERTS_CIO_POLL_INIT(); pollset.ps = ERTS_CIO_NEW_POLLSET(); @@ -1441,7 +1441,7 @@ ERTS_CIO_EXPORT(erts_init_check_io)(void) #endif #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS max_fds = ERTS_CIO_POLL_MAX_FDS(); - erts_smp_atomic_init(&drv_ev_state_len, 0); + erts_smp_atomic_init_nob(&drv_ev_state_len, 0); drv_ev_state = NULL; erts_smp_mtx_init(&drv_ev_state_grow_lock, "drv_ev_state_grow"); #else @@ -1479,7 +1479,7 @@ ERTS_CIO_EXPORT(erts_check_io_size)(void) ERTS_CIO_POLL_INFO(pollset.ps, &pi); res = pi.memory_size; #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - res += sizeof(ErtsDrvEventState) * erts_smp_atomic_read(&drv_ev_state_len); + res += sizeof(ErtsDrvEventState) * erts_smp_atomic_read_nob(&drv_ev_state_len); #else res += safe_hash_table_sz(&drv_ev_state_tab); { @@ -1506,7 +1506,7 @@ ERTS_CIO_EXPORT(erts_check_io_info)(void *proc) ERTS_CIO_POLL_INFO(pollset.ps, &pi); memory_size = pi.memory_size; #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - memory_size += sizeof(ErtsDrvEventState) * erts_smp_atomic_read(&drv_ev_state_len); + memory_size += sizeof(ErtsDrvEventState) * erts_smp_atomic_read_nob(&drv_ev_state_len); #else memory_size += safe_hash_table_sz(&drv_ev_state_tab); { @@ -1886,7 +1886,7 @@ ERTS_CIO_EXPORT(erts_check_io_debug)(void) counters.num_errors = 0; #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - len = erts_smp_atomic_read(&drv_ev_state_len); + len = erts_smp_atomic_read_nob(&drv_ev_state_len); for (fd = 0; fd < len; fd++) { doit_erts_check_io_debug((void *) &drv_ev_state[fd], (void *) &counters); } diff --git a/erts/emulator/sys/common/erl_poll.c b/erts/emulator/sys/common/erl_poll.c index f5c785d683..81f1c95020 100644 --- a/erts/emulator/sys/common/erl_poll.c +++ b/erts/emulator/sys/common/erl_poll.c @@ -124,11 +124,11 @@ erts_smp_mtx_unlock(&(PS)->mtx) #define ERTS_POLLSET_SET_POLLED_CHK(PS) \ - ((int) erts_atomic32_xchg(&(PS)->polled, (erts_aint32_t) 1)) + ((int) erts_atomic32_xchg_nob(&(PS)->polled, (erts_aint32_t) 1)) #define ERTS_POLLSET_UNSET_POLLED(PS) \ - erts_atomic32_set(&(PS)->polled, (erts_aint32_t) 0) + erts_atomic32_set_nob(&(PS)->polled, (erts_aint32_t) 0) #define ERTS_POLLSET_IS_POLLED(PS) \ - ((int) erts_atomic32_read(&(PS)->polled)) + ((int) erts_atomic32_read_nob(&(PS)->polled)) #else @@ -142,11 +142,11 @@ #if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE #define ERTS_POLLSET_SET_HAVE_UPDATE_REQUESTS(PS) \ - erts_smp_atomic32_set(&(PS)->have_update_requests, (erts_aint32_t) 1) + erts_smp_atomic32_set_nob(&(PS)->have_update_requests, (erts_aint32_t) 1) #define ERTS_POLLSET_UNSET_HAVE_UPDATE_REQUESTS(PS) \ - erts_smp_atomic32_set(&(PS)->have_update_requests, (erts_aint32_t) 0) + erts_smp_atomic32_set_nob(&(PS)->have_update_requests, (erts_aint32_t) 0) #define ERTS_POLLSET_HAVE_UPDATE_REQUESTS(PS) \ - ((int) erts_smp_atomic32_read(&(PS)->have_update_requests)) + ((int) erts_smp_atomic32_read_nob(&(PS)->have_update_requests)) #else #define ERTS_POLLSET_SET_HAVE_UPDATE_REQUESTS(PS) #define ERTS_POLLSET_UNSET_HAVE_UPDATE_REQUESTS(PS) @@ -346,7 +346,7 @@ static ERTS_INLINE void reset_wakeup_state(ErtsPollSet ps) { #ifdef ERTS_SMP - erts_atomic32_set(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN); + erts_atomic32_set_nob(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN); ERTS_THR_MEMORY_BARRIER; #elif ERTS_POLL_ASYNC_INTERRUPT_SUPPORT ps->wakeup_state = 0; @@ -369,7 +369,7 @@ static ERTS_INLINE int is_interrupted_reset(ErtsPollSet ps) { #ifdef ERTS_SMP - return (erts_atomic32_xchg(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN) + return (erts_atomic32_xchg_nob(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN) == ERTS_POLL_WOKEN_INTR); #elif ERTS_POLL_ASYNC_INTERRUPT_SUPPORT int res = ps->wakeup_state == ERTS_POLL_WOKEN_INTR; @@ -384,12 +384,12 @@ static ERTS_INLINE void woke_up(ErtsPollSet ps) { #ifdef ERTS_SMP - erts_aint32_t wakeup_state = erts_atomic32_read(&ps->wakeup_state); + erts_aint32_t wakeup_state = erts_atomic32_read_nob(&ps->wakeup_state); if (wakeup_state == ERTS_POLL_NOT_WOKEN) - (void) erts_atomic32_cmpxchg(&ps->wakeup_state, - ERTS_POLL_WOKEN, - ERTS_POLL_NOT_WOKEN); - ASSERT(erts_atomic32_read(&ps->wakeup_state) != ERTS_POLL_NOT_WOKEN); + (void) erts_atomic32_cmpxchg_nob(&ps->wakeup_state, + ERTS_POLL_WOKEN, + ERTS_POLL_NOT_WOKEN); + ASSERT(erts_atomic32_read_nob(&ps->wakeup_state) != ERTS_POLL_NOT_WOKEN); #elif ERTS_POLL_ASYNC_INTERRUPT_SUPPORT if (ps->wakeup_state == ERTS_POLL_NOT_WOKEN) ps->wakeup_state = ERTS_POLL_WOKEN; @@ -417,7 +417,7 @@ wake_poller(ErtsPollSet ps, int interrupted) * We might unnecessarily write to the pipe, however, * that isn't problematic. */ - wakeup_state = erts_atomic32_read(&ps->wakeup_state); + wakeup_state = erts_atomic32_read_nob(&ps->wakeup_state); erts_atomic32_set_relb(&ps->wakeup_state, ERTS_POLL_WOKEN_INTR); } wake = wakeup_state == ERTS_POLL_NOT_WOKEN; @@ -839,7 +839,7 @@ write_batch_buf(ErtsPollSet ps, ErtsPollBatchBuf *bbp) ps->fds_status[fd].flags |= ERTS_POLL_FD_FLG_USEFLBCK; ASSERT(ps->fds_status[fd].used_events); ps->fds_status[fd].used_events = 0; - erts_smp_atomic_dec(&ps->no_of_user_fds); + erts_smp_atomic_dec_nob(&ps->no_of_user_fds); update_fallback_pollset(ps, fd); ASSERT(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK); break; @@ -889,11 +889,11 @@ batch_update_pollset(ErtsPollSet ps, int fd, ErtsPollBatchBuf *bbp) events = ERTS_POLL_EV_E2N(ps->fds_status[fd].events); if (!events) { buf[buf_len].events = POLLREMOVE; - erts_smp_atomic_dec(&ps->no_of_user_fds); + erts_smp_atomic_dec_nob(&ps->no_of_user_fds); } else if (!ps->fds_status[fd].used_events) { buf[buf_len].events = events; - erts_smp_atomic_inc(&ps->no_of_user_fds); + erts_smp_atomic_inc_nob(&ps->no_of_user_fds); } else { if ((ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_RST) @@ -983,12 +983,12 @@ batch_update_pollset(ErtsPollSet ps, int fd, ErtsPollBatchBuf *bbp) } if (used_events) { if (!events) { - erts_smp_atomic_dec(&ps->no_of_user_fds); + erts_smp_atomic_dec_nob(&ps->no_of_user_fds); } } else { if (events) - erts_smp_atomic_inc(&ps->no_of_user_fds); + erts_smp_atomic_inc_nob(&ps->no_of_user_fds); } ASSERT((events & ~(ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT)) == 0); ASSERT((used_events & ~(ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT)) == 0); @@ -1062,7 +1062,7 @@ update_pollset(ErtsPollSet ps, int fd) epe.data.fd = epe_templ.data.fd; res = epoll_ctl(ps->kp_fd, EPOLL_CTL_DEL, fd, &epe); } while (res != 0 && errno == EINTR); - erts_smp_atomic_dec(&ps->no_of_user_fds); + erts_smp_atomic_dec_nob(&ps->no_of_user_fds); ps->fds_status[fd].used_events = 0; } @@ -1070,11 +1070,11 @@ update_pollset(ErtsPollSet ps, int fd) /* A note on EPOLL_CTL_DEL: linux kernel versions before 2.6.9 need a non-NULL event pointer even though it is ignored... */ op = EPOLL_CTL_DEL; - erts_smp_atomic_dec(&ps->no_of_user_fds); + erts_smp_atomic_dec_nob(&ps->no_of_user_fds); } else if (!ps->fds_status[fd].used_events) { op = EPOLL_CTL_ADD; - erts_smp_atomic_inc(&ps->no_of_user_fds); + erts_smp_atomic_inc_nob(&ps->no_of_user_fds); } else { op = EPOLL_CTL_MOD; @@ -1124,7 +1124,7 @@ update_pollset(ErtsPollSet ps, int fd) /* Fall through ... */ case EPOLL_CTL_ADD: { ps->fds_status[fd].flags |= ERTS_POLL_FD_FLG_USEFLBCK; - erts_smp_atomic_dec(&ps->no_of_user_fds); + erts_smp_atomic_dec_nob(&ps->no_of_user_fds); #if ERTS_POLL_USE_CONCURRENT_UPDATE if (!*update_fallback) { *update_fallback = 1; @@ -1212,7 +1212,7 @@ static int update_pollset(ErtsPollSet ps, int fd) #if ERTS_POLL_USE_FALLBACK ASSERT(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK); #endif - erts_smp_atomic_dec(&ps->no_of_user_fds); + erts_smp_atomic_dec_nob(&ps->no_of_user_fds); last_pix = --ps->no_poll_fds; if (pix != last_pix) { /* Move last pix to this pix */ @@ -1239,7 +1239,7 @@ static int update_pollset(ErtsPollSet ps, int fd) ASSERT(!(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK) || fd == ps->kp_fd); #endif - erts_smp_atomic_inc(&ps->no_of_user_fds); + erts_smp_atomic_inc_nob(&ps->no_of_user_fds); ps->fds_status[fd].pix = pix = ps->no_poll_fds++; if (pix >= ps->poll_fds_len) grow_poll_fds(ps, pix); @@ -1290,7 +1290,7 @@ static int update_pollset(ErtsPollSet ps, int fd) if (!ps->fds_status[fd].used_events) { ASSERT(events); - erts_smp_atomic_inc(&ps->no_of_user_fds); + erts_smp_atomic_inc_nob(&ps->no_of_user_fds); #if ERTS_POLL_USE_FALLBACK ps->no_select_fds++; ps->fds_status[fd].flags |= ERTS_POLL_FD_FLG_INFLBCK; @@ -1298,7 +1298,7 @@ static int update_pollset(ErtsPollSet ps, int fd) } else if (!events) { ASSERT(ps->fds_status[fd].used_events); - erts_smp_atomic_dec(&ps->no_of_user_fds); + erts_smp_atomic_dec_nob(&ps->no_of_user_fds); ps->fds_status[fd].events = events; #if ERTS_POLL_USE_FALLBACK ps->no_select_fds--; @@ -1896,7 +1896,7 @@ static ERTS_INLINE int check_fd_events(ErtsPollSet ps, SysTimeval *tv, int max_res, int *ps_locked) { ASSERT(!*ps_locked); - if (erts_smp_atomic_read(&ps->no_of_user_fds) == 0 + if (erts_smp_atomic_read_nob(&ps->no_of_user_fds) == 0 && tv->tv_usec == 0 && tv->tv_sec == 0) { /* Nothing to poll and zero timeout; done... */ return 0; @@ -1937,7 +1937,7 @@ check_fd_events(ErtsPollSet ps, SysTimeval *tv, int max_res, int *ps_locked) * the maximum number of file descriptors in the poll set. */ struct dvpoll poll_res; - int nfds = (int) erts_smp_atomic_read(&ps->no_of_user_fds); + int nfds = (int) erts_smp_atomic_read_nob(&ps->no_of_user_fds); #ifdef ERTS_SMP nfds++; /* Wakeup pipe */ #endif @@ -2143,10 +2143,10 @@ ERTS_POLL_EXPORT(erts_poll_interrupt_timed)(ErtsPollSet ps, #ifdef ERTS_POLL_COUNT_AVOIDED_WAKEUPS else { if (ERTS_POLLSET_IS_POLLED(ps)) - erts_smp_atomic_inc(&ps->no_avoided_wakeups); - erts_smp_atomic_inc(&ps->no_avoided_interrupts); + erts_smp_atomic_inc_nob(&ps->no_avoided_wakeups); + erts_smp_atomic_inc_nob(&ps->no_avoided_interrupts); } - erts_smp_atomic_inc(&ps->no_interrupt_timed); + erts_smp_atomic_inc_nob(&ps->no_interrupt_timed); #endif } #endif @@ -2208,7 +2208,7 @@ ERTS_POLL_EXPORT(erts_poll_create_pollset)(void) ps->internal_fd_limit = 0; ps->fds_status = NULL; ps->fds_status_len = 0; - erts_smp_atomic_init(&ps->no_of_user_fds, 0); + erts_smp_atomic_init_nob(&ps->no_of_user_fds, 0); #if ERTS_POLL_USE_KERNEL_POLL ps->kp_fd = -1; #if ERTS_POLL_USE_EPOLL @@ -2260,14 +2260,14 @@ ERTS_POLL_EXPORT(erts_poll_create_pollset)(void) ps->update_requests.next = NULL; ps->update_requests.len = 0; ps->curr_upd_req_block = &ps->update_requests; - erts_smp_atomic32_init(&ps->have_update_requests, 0); + erts_smp_atomic32_init_nob(&ps->have_update_requests, 0); #endif #ifdef ERTS_SMP - erts_atomic32_init(&ps->polled, 0); + erts_atomic32_init_nob(&ps->polled, 0); erts_smp_mtx_init(&ps->mtx, "pollset"); #endif #ifdef ERTS_SMP - erts_atomic32_init(&ps->wakeup_state, (erts_aint32_t) 0); + erts_atomic32_init_nob(&ps->wakeup_state, (erts_aint32_t) 0); #elif ERTS_POLL_ASYNC_INTERRUPT_SUPPORT ps->wakeup_state = 0; #endif @@ -2291,11 +2291,11 @@ ERTS_POLL_EXPORT(erts_poll_create_pollset)(void) ps->internal_fd_limit = kp_fd + 1; ps->kp_fd = kp_fd; #endif - erts_smp_atomic32_init(&ps->timeout, ERTS_AINT32_T_MAX); + erts_smp_atomic32_init_nob(&ps->timeout, ERTS_AINT32_T_MAX); #ifdef ERTS_POLL_COUNT_AVOIDED_WAKEUPS - erts_smp_atomic_init(&ps->no_avoided_wakeups, 0); - erts_smp_atomic_init(&ps->no_avoided_interrupts, 0); - erts_smp_atomic_init(&ps->no_interrupt_timed, 0); + erts_smp_atomic_init_nob(&ps->no_avoided_wakeups, 0); + erts_smp_atomic_init_nob(&ps->no_avoided_interrupts, 0); + erts_smp_atomic_init_nob(&ps->no_interrupt_timed, 0); #endif #if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE handle_update_requests(ps); @@ -2303,7 +2303,7 @@ ERTS_POLL_EXPORT(erts_poll_create_pollset)(void) #if ERTS_POLL_USE_FALLBACK ps->fallback_used = 0; #endif - erts_smp_atomic_set(&ps->no_of_user_fds, 0); /* Don't count wakeup pipe and fallback fd */ + erts_smp_atomic_set_nob(&ps->no_of_user_fds, 0); /* Don't count wakeup pipe and fallback fd */ erts_smp_spin_lock(&pollsets_lock); ps->next = pollsets; @@ -2449,7 +2449,7 @@ ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet ps, ErtsPollInfo *pip) pip->memory_size = size; - pip->poll_set_size = (int) erts_smp_atomic_read(&ps->no_of_user_fds); + pip->poll_set_size = (int) erts_smp_atomic_read_nob(&ps->no_of_user_fds); #ifdef ERTS_SMP pip->poll_set_size++; /* Wakeup pipe */ #endif @@ -2507,9 +2507,9 @@ ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet ps, ErtsPollInfo *pip) pip->max_fds = max_fds; #ifdef ERTS_POLL_COUNT_AVOIDED_WAKEUPS - pip->no_avoided_wakeups = erts_smp_atomic_read(&ps->no_avoided_wakeups); - pip->no_avoided_interrupts = erts_smp_atomic_read(&ps->no_avoided_interrupts); - pip->no_interrupt_timed = erts_smp_atomic_read(&ps->no_interrupt_timed); + pip->no_avoided_wakeups = erts_smp_atomic_read_nob(&ps->no_avoided_wakeups); + pip->no_avoided_interrupts = erts_smp_atomic_read_nob(&ps->no_avoided_interrupts); + pip->no_interrupt_timed = erts_smp_atomic_read_nob(&ps->no_interrupt_timed); #endif ERTS_POLLSET_UNLOCK(ps); @@ -2529,7 +2529,7 @@ fatal_error(char *format, ...) { va_list ap; - if (ERTS_IS_CRASH_DUMPING || ERTS_GOT_SIGUSR1) { + if (ERTS_SOMEONE_IS_CRASH_DUMPING || ERTS_GOT_SIGUSR1) { /* * Crash dump writing and reception of sigusr1 (which will * result in a crash dump) closes all file descriptors. This @@ -2549,7 +2549,7 @@ fatal_error(char *format, ...) static void fatal_error_async_signal_safe(char *error_str) { - if (ERTS_IS_CRASH_DUMPING || ERTS_GOT_SIGUSR1) { + if (ERTS_SOMEONE_IS_CRASH_DUMPING || ERTS_GOT_SIGUSR1) { /* See comment above in fatal_error() */ return; } diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c index bafbbb0f6c..fd15635168 100644 --- a/erts/emulator/sys/unix/sys.c +++ b/erts/emulator/sys/unix/sys.c @@ -167,12 +167,12 @@ static int debug_log = 0; #ifdef ERTS_SMP erts_smp_atomic32_t erts_got_sigusr1; #define ERTS_SET_GOT_SIGUSR1 \ - erts_smp_atomic32_set(&erts_got_sigusr1, 1) + erts_smp_atomic32_set_mb(&erts_got_sigusr1, 1) #define ERTS_UNSET_GOT_SIGUSR1 \ - erts_smp_atomic32_set(&erts_got_sigusr1, 0) + erts_smp_atomic32_set_mb(&erts_got_sigusr1, 0) static erts_smp_atomic32_t have_prepared_crash_dump; #define ERTS_PREPARED_CRASH_DUMP \ - ((int) erts_smp_atomic32_xchg(&have_prepared_crash_dump, 1)) + ((int) erts_smp_atomic32_xchg_nob(&have_prepared_crash_dump, 1)) #else volatile int erts_got_sigusr1; #define ERTS_SET_GOT_SIGUSR1 (erts_got_sigusr1 = 1) @@ -242,9 +242,9 @@ static int max_files = -1; #ifdef ERTS_SMP erts_smp_atomic32_t erts_break_requested; #define ERTS_SET_BREAK_REQUESTED \ - erts_smp_atomic32_set(&erts_break_requested, (erts_aint32_t) 1) + erts_smp_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 1) #define ERTS_UNSET_BREAK_REQUESTED \ - erts_smp_atomic32_set(&erts_break_requested, (erts_aint32_t) 0) + erts_smp_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 0) #else volatile int erts_break_requested = 0; #define ERTS_SET_BREAK_REQUESTED (erts_break_requested = 1) @@ -364,7 +364,7 @@ Uint erts_sys_misc_mem_sz(void) { Uint res = ERTS_CHK_IO_SZ(); - res += erts_smp_atomic_read(&sys_misc_mem_sz); + res += erts_smp_atomic_read_mb(&sys_misc_mem_sz); return res; } @@ -509,9 +509,9 @@ erts_sys_pre_init(void) #endif } #ifdef ERTS_SMP - erts_smp_atomic32_init(&erts_break_requested, 0); - erts_smp_atomic32_init(&erts_got_sigusr1, 0); - erts_smp_atomic32_init(&have_prepared_crash_dump, 0); + erts_smp_atomic32_init_nob(&erts_break_requested, 0); + erts_smp_atomic32_init_nob(&erts_got_sigusr1, 0); + erts_smp_atomic32_init_nob(&have_prepared_crash_dump, 0); #else erts_break_requested = 0; erts_got_sigusr1 = 0; @@ -521,7 +521,7 @@ erts_sys_pre_init(void) children_died = 0; #endif #endif /* USE_THREADS */ - erts_smp_atomic_init(&sys_misc_mem_sz, 0); + erts_smp_atomic_init_nob(&sys_misc_mem_sz, 0); } void @@ -553,7 +553,7 @@ erl_sys_init(void) + sizeof(CHILD_SETUP_PROG_NAME) + 1); child_setup_prog = erts_alloc(ERTS_ALC_T_CS_PROG_PATH, csp_path_sz); - erts_smp_atomic_add(&sys_misc_mem_sz, csp_path_sz); + erts_smp_atomic_add_nob(&sys_misc_mem_sz, csp_path_sz); sprintf(child_setup_prog, "%s%c%s", bindir, @@ -1216,8 +1216,8 @@ static int spawn_init() sys_sigset(SIGPIPE, SIG_IGN); /* Ignore - we'll handle the write failure */ driver_data = (struct driver_data *) erts_alloc(ERTS_ALC_T_DRV_TAB, max_files * sizeof(struct driver_data)); - erts_smp_atomic_add(&sys_misc_mem_sz, - max_files * sizeof(struct driver_data)); + erts_smp_atomic_add_nob(&sys_misc_mem_sz, + max_files * sizeof(struct driver_data)); for (i = 0; i < max_files; i++) driver_data[i].pid = -1; @@ -1925,8 +1925,8 @@ static void clear_fd_data(int fd) { if (fd_data[fd].sz > 0) { erts_free(ERTS_ALC_T_FD_ENTRY_BUF, (void *) fd_data[fd].buf); - ASSERT(erts_smp_atomic_read(&sys_misc_mem_sz) >= fd_data[fd].sz); - erts_smp_atomic_add(&sys_misc_mem_sz, -1*fd_data[fd].sz); + ASSERT(erts_smp_atomic_read_nob(&sys_misc_mem_sz) >= fd_data[fd].sz); + erts_smp_atomic_add_nob(&sys_misc_mem_sz, -1*fd_data[fd].sz); } fd_data[fd].buf = NULL; fd_data[fd].sz = 0; @@ -2261,7 +2261,7 @@ static void ready_input(ErlDrvData e, ErlDrvEvent ready_fd) port_inp_failure(port_num, ready_fd, -1); } else { - erts_smp_atomic_add(&sys_misc_mem_sz, h); + erts_smp_atomic_add_nob(&sys_misc_mem_sz, h); sys_memcpy(buf, cpos, bytes_left); fd_data[ready_fd].buf = buf; fd_data[ready_fd].sz = h; @@ -2465,7 +2465,7 @@ erts_sys_putenv(char *buffer, int sep_ix) #else Uint sz = strlen(buffer)+1; env = erts_alloc(ERTS_ALC_T_PUTENV_STR, sz); - erts_smp_atomic_add(&sys_misc_mem_sz, sz); + erts_smp_atomic_add_nob(&sys_misc_mem_sz, sz); strcpy(env,buffer); #endif erts_smp_rwmtx_rwlock(&environ_rwmtx); @@ -2504,8 +2504,8 @@ sys_init_io(void) { fd_data = (struct fd_data *) erts_alloc(ERTS_ALC_T_FD_TAB, max_files * sizeof(struct fd_data)); - erts_smp_atomic_add(&sys_misc_mem_sz, - max_files * sizeof(struct fd_data)); + erts_smp_atomic_add_nob(&sys_misc_mem_sz, + max_files * sizeof(struct fd_data)); #ifdef USE_THREADS #ifdef ERTS_SMP diff --git a/erts/emulator/sys/win32/erl_poll.c b/erts/emulator/sys/win32/erl_poll.c index 074e2e247f..735c420d8e 100644 --- a/erts/emulator/sys/win32/erl_poll.c +++ b/erts/emulator/sys/win32/erl_poll.c @@ -309,9 +309,9 @@ struct ErtsPollSet_ { #ifdef ERTS_SMP extern erts_smp_atomic32_t erts_break_requested; #define ERTS_SET_BREAK_REQUESTED \ - erts_smp_atomic32_set(&erts_break_requested, (erts_aint32_t) 1) + erts_smp_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 1) #define ERTS_UNSET_BREAK_REQUESTED \ - erts_smp_atomic32_set(&erts_break_requested, (erts_aint32_t) 0) + erts_smp_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 0) #else extern volatile int erts_break_requested; #define ERTS_SET_BREAK_REQUESTED (erts_break_requested = 1) @@ -371,19 +371,19 @@ do { \ static ERTS_INLINE int is_io_ready(ErtsPollSet ps) { - return erts_atomic32_read(&ps->wakeup_state) == ERTS_POLL_WOKEN_IO_READY; + return erts_atomic32_read_nob(&ps->wakeup_state) == ERTS_POLL_WOKEN_IO_READY; } static ERTS_INLINE void woke_up(ErtsPollSet ps) { - if (erts_atomic32_read(&ps->wakeup_state) == ERTS_POLL_NOT_WOKEN) - erts_atomic32_cmpxchg(&ps->wakeup_state, - ERTS_POLL_WOKEN_TIMEDOUT, - ERTS_POLL_NOT_WOKEN); + if (erts_atomic32_read_nob(&ps->wakeup_state) == ERTS_POLL_NOT_WOKEN) + erts_atomic32_cmpxchg_nob(&ps->wakeup_state, + ERTS_POLL_WOKEN_TIMEDOUT, + ERTS_POLL_NOT_WOKEN); #ifdef DEBUG { - erts_aint32_t wakeup_state = erts_atomic32_read(&ps->wakeup_state); + erts_aint32_t wakeup_state = erts_atomic32_read_nob(&ps->wakeup_state); switch (wakeup_state) { case ERTS_POLL_WOKEN_IO_READY: case ERTS_POLL_WOKEN_INTR: @@ -401,7 +401,7 @@ static ERTS_INLINE int wakeup_cause(ErtsPollSet ps) { int res; - erts_aint32_t wakeup_state = erts_atomic32_read(&ps->wakeup_state); + erts_aint32_t wakeup_state = erts_atomic32_read_nob(&ps->wakeup_state); switch (wakeup_state) { case ERTS_POLL_WOKEN_IO_READY: res = 0; @@ -439,7 +439,7 @@ poll_wait_timeout(ErtsPollSet ps, SysTimeval *tvp) * by ResetEvent(). */ ERTS_THR_MEMORY_BARRIER; - if (erts_atomic32_read(&ps->wakeup_state) != ERTS_POLL_NOT_WOKEN) + if (erts_atomic32_read_nob(&ps->wakeup_state) != ERTS_POLL_NOT_WOKEN) return (DWORD) 0; if (timeout > ERTS_AINT32_T_MAX) /* Also prevents DWORD overflow */ @@ -455,17 +455,17 @@ wake_poller(ErtsPollSet ps, int io_ready) erts_aint32_t wakeup_state; if (io_ready) { /* We may set the event multiple times. This is, however, harmless. */ - wakeup_state = erts_atomic32_read(&ps->wakeup_state); + wakeup_state = erts_atomic32_read_nob(&ps->wakeup_state); erts_atomic32_set_relb(&ps->wakeup_state, ERTS_POLL_WOKEN_IO_READY); } else { ERTS_THR_MEMORY_BARRIER; - wakeup_state = erts_atomic32_read(&ps->wakeup_state); + wakeup_state = erts_atomic32_read_nob(&ps->wakeup_state); while (wakeup_state != ERTS_POLL_WOKEN_IO_READY && wakeup_state != ERTS_POLL_WOKEN_INTR) { - erts_aint32_t act = erts_atomic32_cmpxchg(&ps->wakeup_state, - ERTS_POLL_WOKEN_INTR, - wakeup_state); + erts_aint32_t act = erts_atomic32_cmpxchg_nob(&ps->wakeup_state, + ERTS_POLL_WOKEN_INTR, + wakeup_state); if (act == wakeup_state) { wakeup_state = act; break; @@ -488,13 +488,13 @@ wake_poller(ErtsPollSet ps, int io_ready) static ERTS_INLINE void reset_io_ready(ErtsPollSet ps) { - erts_atomic32_set(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN); + erts_atomic32_set_nob(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN); } static ERTS_INLINE void restore_io_ready(ErtsPollSet ps) { - erts_atomic32_set(&ps->wakeup_state, ERTS_POLL_WOKEN_IO_READY); + erts_atomic32_set_nob(&ps->wakeup_state, ERTS_POLL_WOKEN_IO_READY); } /* @@ -511,12 +511,12 @@ static ERTS_INLINE void reset_interrupt(ErtsPollSet ps) { /* We need to keep io-ready if set */ - erts_aint32_t wakeup_state = erts_atomic32_read(&ps->wakeup_state); + erts_aint32_t wakeup_state = erts_atomic32_read_nob(&ps->wakeup_state); while (wakeup_state != ERTS_POLL_WOKEN_IO_READY && wakeup_state != ERTS_POLL_NOT_WOKEN) { - erts_aint32_t act = erts_atomic32_cmpxchg(&ps->wakeup_state, - ERTS_POLL_NOT_WOKEN, - wakeup_state); + erts_aint32_t act = erts_atomic32_cmpxchg_nob(&ps->wakeup_state, + ERTS_POLL_NOT_WOKEN, + wakeup_state); if (wakeup_state == act) break; wakeup_state = act; @@ -692,7 +692,7 @@ static void *break_waiter(void *param) case WAIT_OBJECT_0: ResetEvent(harr[0]); erts_mtx_lock(&break_waiter_lock); - erts_atomic32_set(&break_waiter_state,BREAK_WAITER_GOT_BREAK); + erts_atomic32_set_nob(&break_waiter_state,BREAK_WAITER_GOT_BREAK); ERTS_THR_MEMORY_BARRIER; SetEvent(break_happened_event); erts_mtx_unlock(&break_waiter_lock); @@ -700,7 +700,7 @@ static void *break_waiter(void *param) case (WAIT_OBJECT_0+1): ResetEvent(harr[1]); erts_mtx_lock(&break_waiter_lock); - erts_atomic32_set(&break_waiter_state,BREAK_WAITER_GOT_HALT); + erts_atomic32_set_nob(&break_waiter_state,BREAK_WAITER_GOT_HALT); ERTS_THR_MEMORY_BARRIER; SetEvent(break_happened_event); erts_mtx_unlock(&break_waiter_lock); @@ -1153,7 +1153,7 @@ int erts_poll_wait(ErtsPollSet ps, /*HARDDEBUGF(("timeout = %ld",(long) timeout));*/ - if (timeout > 0 && !erts_atomic32_read(&break_waiter_state)) { + if (timeout > 0 && !erts_atomic32_read_nob(&break_waiter_state)) { HANDLE harr[2] = {ps->event_io_ready, break_happened_event}; int num_h = 2; @@ -1166,10 +1166,10 @@ int erts_poll_wait(ErtsPollSet ps, } ERTS_UNSET_BREAK_REQUESTED; - if(erts_atomic32_read(&break_waiter_state)) { + if(erts_atomic32_read_nob(&break_waiter_state)) { erts_mtx_lock(&break_waiter_lock); - break_state = erts_atomic32_read(&break_waiter_state); - erts_atomic32_set(&break_waiter_state,0); + break_state = erts_atomic32_read_nob(&break_waiter_state); + erts_atomic32_set_nob(&break_waiter_state,0); ResetEvent(break_happened_event); erts_mtx_unlock(&break_waiter_lock); switch (break_state) { @@ -1236,7 +1236,7 @@ int erts_poll_wait(ErtsPollSet ps, erts_mtx_unlock(&w->mtx); } done: - erts_smp_atomic32_set(&ps->timeout, ERTS_AINT32_T_MAX); + erts_smp_atomic32_set_nob(&ps->timeout, ERTS_AINT32_T_MAX); *len = num; ERTS_POLLSET_UNLOCK(ps); HARDTRACEF(("Out erts_poll_wait")); @@ -1316,11 +1316,11 @@ ErtsPollSet erts_poll_create_pollset(void) ps->standby_wait_event = CreateManualEvent(FALSE); ps->restore_events = 0; - erts_atomic32_init(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN); + erts_atomic32_init_nob(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN); #ifdef ERTS_SMP erts_smp_mtx_init(&ps->mtx, "pollset"); #endif - erts_smp_atomic32_init(&ps->timeout, ERTS_AINT32_T_MAX); + erts_smp_atomic32_init_nob(&ps->timeout, ERTS_AINT32_T_MAX); HARDTRACEF(("Out erts_poll_create_pollset")); return ps; @@ -1372,7 +1372,7 @@ void erts_poll_init(void) erts_mtx_init(&break_waiter_lock,"break_waiter_lock"); break_happened_event = CreateManualEvent(FALSE); - erts_atomic32_init(&break_waiter_state, 0); + erts_atomic32_init_nob(&break_waiter_state, 0); erts_thr_create(&thread, &break_waiter, NULL, NULL); ERTS_UNSET_BREAK_REQUESTED; diff --git a/erts/emulator/sys/win32/sys.c b/erts/emulator/sys/win32/sys.c index a2159d063c..ce1d376a54 100644 --- a/erts/emulator/sys/win32/sys.c +++ b/erts/emulator/sys/win32/sys.c @@ -198,7 +198,7 @@ Uint erts_sys_misc_mem_sz(void) { Uint res = (Uint) erts_check_io_size(); - res += (Uint) erts_smp_atomic_read(&sys_misc_mem_sz); + res += (Uint) erts_smp_atomic_read_mb(&sys_misc_mem_sz); return res; } @@ -648,7 +648,7 @@ new_driver_data(int port_num, int packet_bytes, int wait_objs_required, int use_ erts_smp_mtx_unlock(&sys_driver_data_lock); return NULL; } - erts_smp_atomic_add(&sys_misc_mem_sz, dp->inBufSize); + erts_smp_atomic_add_nob(&sys_misc_mem_sz, dp->inBufSize); dp->outBufSize = 0; dp->outbuf = NULL; dp->port_num = port_num; @@ -733,8 +733,8 @@ release_driver_data(DriverData* dp) #endif if (dp->inbuf != NULL) { - ASSERT(erts_smp_atomic_read(&sys_misc_mem_sz) >= dp->inBufSize); - erts_smp_atomic_add(&sys_misc_mem_sz, -1*dp->inBufSize); + ASSERT(erts_smp_atomic_read_nob(&sys_misc_mem_sz) >= dp->inBufSize); + erts_smp_atomic_add_nob(&sys_misc_mem_sz, -1*dp->inBufSize); DRV_BUF_FREE(dp->inbuf); dp->inBufSize = 0; dp->inbuf = NULL; @@ -742,8 +742,8 @@ release_driver_data(DriverData* dp) ASSERT(dp->inBufSize == 0); if (dp->outbuf != NULL) { - ASSERT(erts_smp_atomic_read(&sys_misc_mem_sz) >= dp->outBufSize); - erts_smp_atomic_add(&sys_misc_mem_sz, -1*dp->outBufSize); + ASSERT(erts_smp_atomic_read_nob(&sys_misc_mem_sz) >= dp->outBufSize); + erts_smp_atomic_add_nob(&sys_misc_mem_sz, -1*dp->outBufSize); DRV_BUF_FREE(dp->outbuf); dp->outBufSize = 0; dp->outbuf = NULL; @@ -1162,7 +1162,8 @@ spawn_init(void) #endif driver_data = (struct driver_data *) erts_alloc(ERTS_ALC_T_DRV_TAB, max_files * sizeof(struct driver_data)); - erts_smp_atomic_add(&sys_misc_mem_sz, max_files*sizeof(struct driver_data)); + erts_smp_atomic_add_nob(&sys_misc_mem_sz, + max_files*sizeof(struct driver_data)); for (i = 0; i < max_files; i++) driver_data[i].port_num = PORT_FREE; @@ -1698,7 +1699,7 @@ create_child_process static int create_pipe(HANDLE *phRead, HANDLE *phWrite, BOOL inheritRead, BOOL overlapped_io) { SECURITY_ATTRIBUTES sa = {sizeof(SECURITY_ATTRIBUTES), NULL, TRUE}; - char pipe_name[128]; /* Name of pipe. */ + char pipe_name[256]; /* Name of pipe. */ Uint calls; /* @@ -1735,9 +1736,9 @@ static int create_pipe(HANDLE *phRead, HANDLE *phWrite, BOOL inheritRead, BOOL o * Otherwise, create named pipes. */ - calls = (Uint) erts_smp_atomic_inctest(&pipe_creation_counter); - sprintf(pipe_name, "\\\\.\\pipe\\erlang44_%d_%d", - getpid(), calls); + calls = (UWord) erts_smp_atomic_inc_read_nob(&pipe_creation_counter); + erts_snprintf(pipe_name, sizeof(pipe_name), + "\\\\.\\pipe\\erlang44_%d_%bpu", getpid(), calls); DEBUGF(("Creating pipe %s\n", pipe_name)); sa.bInheritHandle = inheritRead; @@ -2529,7 +2530,7 @@ output(ErlDrvData drv_data, char* buf, int len) } dp->outBufSize = pb+len; - erts_smp_atomic_add(&sys_misc_mem_sz, dp->outBufSize); + erts_smp_atomic_add_nob(&sys_misc_mem_sz, dp->outBufSize); /* * Store header bytes (if any). @@ -2558,8 +2559,8 @@ output(ErlDrvData drv_data, char* buf, int len) } else { dp->out.ov.Offset += pb+len; /* For vanilla driver. */ /* XXX OffsetHigh should be changed too. */ - ASSERT(erts_smp_atomic_read(&sys_misc_mem_sz) >= dp->outBufSize); - erts_smp_atomic_add(&sys_misc_mem_sz, -1*dp->outBufSize); + ASSERT(erts_smp_atomic_read_nob(&sys_misc_mem_sz) >= dp->outBufSize); + erts_smp_atomic_add_nob(&sys_misc_mem_sz, -1*dp->outBufSize); DRV_BUF_FREE(dp->outbuf); dp->outBufSize = 0; dp->outbuf = NULL; @@ -2673,9 +2674,9 @@ ready_input(ErlDrvData drv_data, ErlDrvEvent ready_event) error = ERROR_NOT_ENOUGH_MEMORY; break; /* Break out of loop into error handler. */ } - ASSERT(erts_smp_atomic_read(&sys_misc_mem_sz) >= dp->inBufSize); - erts_smp_atomic_add(&sys_misc_mem_sz, - dp->totalNeeded - dp->inBufSize); + ASSERT(erts_smp_atomic_read_nob(&sys_misc_mem_sz) >= dp->inBufSize); + erts_smp_atomic_add_nob(&sys_misc_mem_sz, + dp->totalNeeded - dp->inBufSize); dp->inBufSize = dp->totalNeeded; dp->inbuf = new_buf; } @@ -2775,8 +2776,8 @@ ready_output(ErlDrvData drv_data, ErlDrvEvent ready_event) write... */ return; } - ASSERT(erts_smp_atomic_read(&sys_misc_mem_sz) >= dp->outBufSize); - erts_smp_atomic_add(&sys_misc_mem_sz, -1*dp->outBufSize); + ASSERT(erts_smp_atomic_read_nob(&sys_misc_mem_sz) >= dp->outBufSize); + erts_smp_atomic_add_nob(&sys_misc_mem_sz, -1*dp->outBufSize); DRV_BUF_FREE(dp->outbuf); dp->outBufSize = 0; dp->outbuf = NULL; @@ -2926,8 +2927,8 @@ Preload* sys_preloaded(void) (num_preloaded+1)*sizeof(Preload)); res_name = erts_alloc(ERTS_ALC_T_PRELOADED, (num_preloaded+1)*sizeof(unsigned)); - erts_smp_atomic_add(&sys_misc_mem_sz, - (num_preloaded+1)*sizeof(Preload) + erts_smp_atomic_add_nob(&sys_misc_mem_sz, + (num_preloaded+1)*sizeof(Preload) + (num_preloaded+1)*sizeof(unsigned)); for (i = 0; i < num_preloaded; i++) { int n; @@ -2939,7 +2940,7 @@ Preload* sys_preloaded(void) n = GETWORD(data); data += 2; preloaded[i].name = erts_alloc(ERTS_ALC_T_PRELOADED, n+1); - erts_smp_atomic_add(&sys_misc_mem_sz, n+1); + erts_smp_atomic_add_nob(&sys_misc_mem_sz, n+1); sys_memcpy(preloaded[i].name, data, n); preloaded[i].name[n] = '\0'; data += n; @@ -3281,7 +3282,7 @@ erts_sys_pre_init(void) #endif } #endif - erts_smp_atomic_init(&sys_misc_mem_sz, 0); + erts_smp_atomic_init_nob(&sys_misc_mem_sz, 0); } void noinherit_std_handle(DWORD type) @@ -3310,7 +3311,7 @@ void erl_sys_init(void) erts_smp_tsd_key_create(&win32_errstr_key); InitializeCriticalSection(&htbc_lock); #endif - erts_smp_atomic_init(&pipe_creation_counter,0); + erts_smp_atomic_init_nob(&pipe_creation_counter,0); /* * Test if we have named pipes or not. */ diff --git a/erts/emulator/sys/win32/sys_interrupt.c b/erts/emulator/sys/win32/sys_interrupt.c index 943c338794..1d73edd30b 100644 --- a/erts/emulator/sys/win32/sys_interrupt.c +++ b/erts/emulator/sys/win32/sys_interrupt.c @@ -33,9 +33,9 @@ #ifdef ERTS_SMP erts_smp_atomic32_t erts_break_requested; #define ERTS_SET_BREAK_REQUESTED \ - erts_smp_atomic32_set(&erts_break_requested, (erts_aint32_t) 1) + erts_smp_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 1) #define ERTS_UNSET_BREAK_REQUESTED \ - erts_smp_atomic32_set(&erts_break_requested, (erts_aint32_t) 0) + erts_smp_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 0) #else volatile int erts_break_requested = 0; #define ERTS_SET_BREAK_REQUESTED (erts_break_requested = 1) diff --git a/erts/emulator/test/alloc_SUITE_data/allocator_test.h b/erts/emulator/test/alloc_SUITE_data/allocator_test.h index b869a4079c..8b34375980 100644 --- a/erts/emulator/test/alloc_SUITE_data/allocator_test.h +++ b/erts/emulator/test/alloc_SUITE_data/allocator_test.h @@ -82,15 +82,17 @@ typedef void* erts_cond; #define NO_OF_BKTS ((Ulong) ALC_TEST0(0x102)) #define FIND_BKT(A, I) ((int) ALC_TEST2(0x103, (A), (I))) -/* From erl_bestfit_alloc.c */ -#define IS_AOBF(A) ((Ulong) ALC_TEST1(0x200, (A))) -#define RBT_ROOT(A) ((RBT_t *) ALC_TEST1(0x201, (A))) -#define RBT_PARENT(T) ((RBT_t *) ALC_TEST1(0x202, (T))) -#define RBT_LEFT(T) ((RBT_t *) ALC_TEST1(0x203, (T))) -#define RBT_RIGHT(T) ((RBT_t *) ALC_TEST1(0x204, (T))) -#define RBT_NEXT(T) ((RBTL_t *) ALC_TEST1(0x205, (T))) -#define RBT_IS_BLACK(T) ((Ulong) ALC_TEST1(0x206, (T))) -#define RBT_IS_TREE(T) ((Ulong) ALC_TEST1(0x207, (T))) +/* From erl_bestfit_alloc.c and erl_ao_firstfit_alloc.c */ +#define IS_AOBF(A) ((Ulong) ALC_TEST1(RBT_OP(0), (A))) +#define RBT_ROOT(A) ((RBT_t *) ALC_TEST1(RBT_OP(1), (A))) +#define RBT_PARENT(T) ((RBT_t *) ALC_TEST1(RBT_OP(2), (T))) +#define RBT_LEFT(T) ((RBT_t *) ALC_TEST1(RBT_OP(3), (T))) +#define RBT_RIGHT(T) ((RBT_t *) ALC_TEST1(RBT_OP(4), (T))) +#define RBT_NEXT(T) ((RBTL_t *) ALC_TEST1(RBT_OP(5), (T))) +#define RBT_IS_BLACK(T) ((Ulong) ALC_TEST1(RBT_OP(6), (T))) +#define RBT_IS_TREE(T) ((Ulong) ALC_TEST1(RBT_OP(7), (T))) +#define IS_AOFF(A) ((Ulong) ALC_TEST1(RBT_OP(8), (A))) +#define RBT_MAX_SZ(T) ((Ulong) ALC_TEST1(RBT_OP(9), (T))) /* From erl_mseg.c */ #define HAVE_MSEG() ((int) ALC_TEST0(0x400)) diff --git a/erts/emulator/test/alloc_SUITE_data/coalesce.c b/erts/emulator/test/alloc_SUITE_data/coalesce.c index c84da97d35..6f35d3279b 100644 --- a/erts/emulator/test/alloc_SUITE_data/coalesce.c +++ b/erts/emulator/test/alloc_SUITE_data/coalesce.c @@ -267,7 +267,7 @@ void testcase_run(TestCaseState_t *tcs) { char *argv_org[] = {"-tmmbcs1024", "-tsbct2048", "-trmbcmt100", "-tas", NULL, NULL}; - char *alg[] = {"af", "gf", "bf", "aobf", NULL}; + char *alg[] = {"af", "gf", "bf", "aobf", "aoff", NULL}; int i; for (i = 0; alg[i]; i++) { diff --git a/erts/emulator/test/alloc_SUITE_data/rbtree.c b/erts/emulator/test/alloc_SUITE_data/rbtree.c index c97e0aac1a..4e7f821baf 100644 --- a/erts/emulator/test/alloc_SUITE_data/rbtree.c +++ b/erts/emulator/test/alloc_SUITE_data/rbtree.c @@ -34,6 +34,14 @@ typedef struct { #define PRINT_TREE #endif +/* Ugly hack to steer the test code towards the right allocator */ +#define RBT_OP(CMD) (current_rbt_type_op_base + (CMD)) +static enum { + BESTFIT_OP_BASE = 0x200, + AO_FIRSTFIT_OP_BASE = 0x500 +}current_rbt_type_op_base; + + #ifdef PRINT_TREE #define INDENT_STEP 5 @@ -65,12 +73,11 @@ print_tree_aux(TestCaseState_t *tcs, RBT_t *x, int indent) static void -print_tree(TestCaseState_t *tcs, RBT_t *root, int aobf) +print_tree(TestCaseState_t *tcs, RBT_t *root) { - char *type = aobf ? "Size-Adress" : "Size"; - testcase_printf(tcs, " --- %s tree begin ---\r\n", type); + testcase_printf(tcs, " --- Tree begin ---\r\n"); print_tree_aux(tcs, root, 0); - testcase_printf(tcs, " --- %s tree end ---\r\n", type); + testcase_printf(tcs, " --- Tree end ---\r\n"); } #endif @@ -78,7 +85,8 @@ print_tree(TestCaseState_t *tcs, RBT_t *root, int aobf) static RBT_t * check_tree(TestCaseState_t *tcs, Allctr_t *alc, Ulong size) { - int i, max_i, address_order; + enum { BF, AOBF, AOFF }type; + int i, max_i; char stk[128]; RBT_t *root, *x, *y, *res; Ulong x_sz, y_sz, is_x_black; @@ -86,11 +94,14 @@ check_tree(TestCaseState_t *tcs, Allctr_t *alc, Ulong size) res = NULL; - address_order = IS_AOBF(alc); + if (IS_AOBF(alc)) type = AOBF; + else if (IS_AOFF(alc)) type = AOFF; + else type = BF; + root = RBT_ROOT(alc); #ifdef PRINT_TREE - print_tree(tcs, root, address_order); + print_tree(tcs, root); #endif max_i = i = -1; @@ -165,12 +176,18 @@ check_tree(TestCaseState_t *tcs, Allctr_t *alc, Ulong size) if (y) { y_sz = BLK_SZ(y); ASSERT(tcs, RBT_PARENT(y) == x); - if (address_order) { + switch (type) { + case AOBF: ASSERT(tcs, y_sz < x_sz || (y_sz == x_sz && y < x)); - } - else { + break; + case BF: ASSERT(tcs, RBT_IS_TREE(y)); ASSERT(tcs, y_sz < x_sz); + break; + case AOFF: + ASSERT(tcs, y < x); + ASSERT(tcs, RBT_MAX_SZ(y) <= RBT_MAX_SZ(x)); + break; } } @@ -178,16 +195,22 @@ check_tree(TestCaseState_t *tcs, Allctr_t *alc, Ulong size) if (y) { y_sz = BLK_SZ(y); ASSERT(tcs, RBT_PARENT(y) == x); - if (address_order) { + switch (type) { + case AOBF: ASSERT(tcs, y_sz > x_sz || (y_sz == x_sz && y > x)); - } - else { + break; + case BF: ASSERT(tcs, RBT_IS_TREE(y)); ASSERT(tcs, y_sz > x_sz); + break; + case AOFF: + ASSERT(tcs, y > x); + ASSERT(tcs, RBT_MAX_SZ(y) <= RBT_MAX_SZ(x)); + break; } } - if (!address_order) { + if (type == BF) { Ulong l_sz; RBTL_t *l = RBT_NEXT(x); for (l = RBT_NEXT(x); l; l = RBT_NEXT(l)) { @@ -202,13 +225,20 @@ check_tree(TestCaseState_t *tcs, Allctr_t *alc, Ulong size) res = x; else { y_sz = BLK_SZ(res); - if (address_order) { + switch (type) { + case AOBF: if (x_sz < y_sz || (x_sz == y_sz && x < res)) res = x; - } - else { - if (!res || x_sz < y_sz) + break; + case BF: + if (x_sz < y_sz) res = x; + break; + case AOFF: + if (x < res) { + res = x; + } + break; } } } @@ -257,7 +287,7 @@ static void test_it(TestCaseState_t *tcs) { int i; - Allctr_t a = ((rbtree_test_data *) tcs->extra)->allocator; + Allctr_t* a = ((rbtree_test_data *) tcs->extra)->allocator; void **blk = ((rbtree_test_data *) tcs->extra)->blk; void **fence = ((rbtree_test_data *) tcs->extra)->fence; Ulong min_blk_sz; @@ -338,6 +368,7 @@ testcase_run(TestCaseState_t *tcs) { char *argv1[] = {"-tasbf", NULL}; char *argv2[] = {"-tasaobf", NULL}; + char *argv3[] = {"-tasaoff", NULL}; Allctr_t *a; rbtree_test_data *td; @@ -355,6 +386,7 @@ testcase_run(TestCaseState_t *tcs) testcase_printf(tcs, "Starting test of best fit...\n"); + current_rbt_type_op_base = BESTFIT_OP_BASE; td->allocator = a = START_ALC("rbtree_bf_", 0, argv1); ASSERT(tcs, a); @@ -371,6 +403,7 @@ testcase_run(TestCaseState_t *tcs) testcase_printf(tcs, "Starting test of address order best fit...\n"); + current_rbt_type_op_base = BESTFIT_OP_BASE; td->allocator = a = START_ALC("rbtree_aobf_", 0, argv2); ASSERT(tcs, a); @@ -383,4 +416,19 @@ testcase_run(TestCaseState_t *tcs) testcase_printf(tcs, "Address order best fit test succeeded!\n"); + /* Address order first fit... */ + + testcase_printf(tcs, "Starting test of address order first fit...\n"); + + current_rbt_type_op_base = AO_FIRSTFIT_OP_BASE; + td->allocator = a = START_ALC("rbtree_aoff_", 0, argv3); + + ASSERT(tcs, a); + + test_it(tcs); + + STOP_ALC(a); + td->allocator = NULL; + + testcase_printf(tcs, "Address order first fit test succeeded!\n"); } diff --git a/erts/emulator/test/binary_SUITE.erl b/erts/emulator/test/binary_SUITE.erl index 4e82381fba..459dc84565 100644 --- a/erts/emulator/test/binary_SUITE.erl +++ b/erts/emulator/test/binary_SUITE.erl @@ -33,7 +33,6 @@ %% erlang:external_size/1 %% size(Binary) %% iolist_size/1 -%% concat_binary/1 %% split_binary/2 %% hash(Binary, N) %% phash(Binary, N) @@ -47,7 +46,7 @@ init_per_testcase/2, end_per_testcase/2, copy_terms/1, conversions/1, deep_lists/1, deep_bitstr_lists/1, bad_list_to_binary/1, bad_binary_to_list/1, - t_split_binary/1, bad_split/1, t_concat_binary/1, + t_split_binary/1, bad_split/1, terms/1, terms_float/1, external_size/1, t_iolist_size/1, t_hash/1, bad_size/1, @@ -68,7 +67,7 @@ suite() -> [{ct_hooks,[ts_install_cth]}, all() -> [copy_terms, conversions, deep_lists, deep_bitstr_lists, - t_split_binary, bad_split, t_concat_binary, + t_split_binary, bad_split, bad_list_to_binary, bad_binary_to_list, terms, terms_float, external_size, t_iolist_size, bad_binary_to_term_2, safe_binary_to_term2, @@ -93,7 +92,6 @@ init_per_group(_GroupName, Config) -> end_per_group(_GroupName, Config) -> Config. - init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) -> Config. @@ -381,41 +379,6 @@ bad_split(Config) when is_list(Config) -> bad_split(Bin, Pos) -> {'EXIT',{badarg,_}} = (catch split_binary(Bin, Pos)). -%% Tests concat_binary/2 and size/1. - -t_concat_binary(suite) -> []; -t_concat_binary(Config) when is_list(Config) -> - test_concat([]), - - test_concat([[]]), - test_concat([[], []]), - test_concat([[], [], []]), - - test_concat([[1], []]), - test_concat([[], [2]]), - test_concat([[], [3], []]), - - test_concat([[1, 2, 3], [4, 5, 6, 7]]), - test_concat([[1, 2, 3], [4, 5, 6, 7], [9, 10]]), - - test_concat([lists:seq(0, 255), lists:duplicate(1024, $@), - lists:duplicate(2048, $a), - lists:duplicate(4000, $b)]), - ok. - -test_concat(Lists) -> - test_concat(Lists, 0, [], []). - -test_concat([List|Rest], Size, Combined, Binaries) -> - ?line Bin = list_to_binary(List), - ?line test_concat(Rest, Size+length(List), Combined++List, [Bin|Binaries]); -test_concat([], Size, Combined, Binaries0) -> - ?line Binaries = lists:reverse(Binaries0), - ?line Bin = concat_binary(Binaries), - ?line Size = size(Bin), - ?line Size = iolist_size(Bin), - ?line Combined = binary_to_list(Bin). - t_hash(doc) -> "Test hash/2 with different type of binaries."; t_hash(Config) when is_list(Config) -> test_hash([]), diff --git a/erts/emulator/test/fun_SUITE.erl b/erts/emulator/test/fun_SUITE.erl index 7795efe57e..559e540016 100644 --- a/erts/emulator/test/fun_SUITE.erl +++ b/erts/emulator/test/fun_SUITE.erl @@ -647,17 +647,11 @@ refc_dist_1() -> %% Fun is passed in an exit signal. Wait until it is gone. ?line wait_until(fun () -> 4 =/= fun_refc(F2) end), ?line 3 = fun_refc(F2), - erts_debug:set_internal_state(available_internal_state, true), - ?line F_refc = case erts_debug:get_internal_state(force_heap_frags) of - false -> 3; - true -> 2 % GC after bif already decreased it - end, - ?line F_refc = fun_refc(F), - erts_debug:set_internal_state(available_internal_state, false), + ?line true = erlang:garbage_collect(), + ?line 2 = fun_refc(F), refc_dist_send(Node, F). refc_dist_send(Node, F) -> - ?line true = erlang:garbage_collect(), ?line Pid = spawn_link(Node, fun() -> receive {To,Fun} when is_function(Fun) -> diff --git a/erts/emulator/test/match_spec_SUITE.erl b/erts/emulator/test/match_spec_SUITE.erl index 2b21fa58f4..461773114e 100644 --- a/erts/emulator/test/match_spec_SUITE.erl +++ b/erts/emulator/test/match_spec_SUITE.erl @@ -27,8 +27,9 @@ destructive_in_test_bif/1, guard_exceptions/1, unary_plus/1, unary_minus/1, moving_labels/1]). -export([fpe/1]). +-export([otp_9422/1]). --export([runner/2]). +-export([runner/2, loop_runner/3]). -export([f1/1, f2/2, f3/2, fn/1, fn/2, fn/3]). -export([do_boxed_and_small/0]). @@ -57,7 +58,8 @@ all() -> trace_control_word, silent, silent_no_ms, ms_trace2, ms_trace3, boxed_and_small, destructive_in_test_bif, guard_exceptions, unary_plus, unary_minus, fpe, - moving_labels]; + moving_labels, + otp_9422]; true -> [not_run] end. @@ -208,6 +210,43 @@ test_3(Config) when is_list(Config) -> ?line collect(P1, [{trace, P1, call, {?MODULE, f2, [a, b]}, [true]}]), ?line ok. +otp_9422(doc) -> []; +otp_9422(Config) when is_list(Config) -> + Laps = 1000, + ?line Fun1 = fun() -> otp_9422_tracee() end, + ?line P1 = spawn_link(?MODULE, loop_runner, [self(), Fun1, Laps]), + io:format("spawned ~p as tracee\n", [P1]), + + ?line erlang:trace(P1, true, [call, silent]), + + ?line Fun2 = fun() -> otp_9422_trace_changer() end, + ?line P2 = spawn_link(?MODULE, loop_runner, [self(), Fun2, Laps]), + io:format("spawned ~p as trace_changer\n", [P2]), + + start_collect(P1), + start_collect(P2), + + %%receive after 10*1000 -> ok end, + + stop_collect(P1), + stop_collect(P2), + ok. + +otp_9422_tracee() -> + ?MODULE:f1(a), + ?MODULE:f1(b), + ?MODULE:f1(c). + +otp_9422_trace_changer() -> + Pat1 = [{[a], [], [{enable_trace, arity}]}], + ?line erlang:trace_pattern({?MODULE, f1, 1}, Pat1), + Pat2 = [{[b], [], [{disable_trace, arity}]}], + ?line erlang:trace_pattern({?MODULE, f1, 1}, Pat2). + + + + + bad_match_spec_bin(Config) when is_list(Config) -> {'EXIT',{badarg,_}} = (catch ets:match_spec_run([1], <<>>)), B0 = <<1,2>>, @@ -932,6 +971,24 @@ runner(Collector, Fun) -> Collector ! {gone, self()} end. +loop_runner(Collector, Fun, Laps) -> + receive + {go, Collector} -> + go + end, + loop_runner_cont(Collector, Fun, 0, Laps). + +loop_runner_cont(_Collector, _Fun, Laps, Laps) -> + receive + {done, Collector} -> + io:format("loop_runner ~p exit after ~p laps\n", [self(), Laps]), + Collector ! {gone, self()} + end; +loop_runner_cont(Collector, Fun, N, Laps) -> + Fun(), + loop_runner_cont(Collector, Fun, N+1, Laps). + + f1(X) -> {X}. diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl index 91d695d979..9c31b7f78d 100644 --- a/erts/emulator/test/nif_SUITE.erl +++ b/erts/emulator/test/nif_SUITE.erl @@ -35,7 +35,7 @@ resource_takeover/1, threading/1, send/1, send2/1, send3/1, send_threaded/1, neg/1, is_checks/1, - get_length/1, make_atom/1, make_string/1]). + get_length/1, make_atom/1, make_string/1, reverse_list_test/1]). -export([many_args_100/100]). @@ -60,7 +60,7 @@ all() -> iolist_as_binary, resource, resource_binary, resource_takeover, threading, send, send2, send3, send_threaded, neg, is_checks, get_length, make_atom, - make_string]. + make_string,reverse_list_test]. groups() -> []. @@ -257,10 +257,48 @@ types(Config) when is_list(Config) -> end, [{},{ok},{{}},{[],{}},{1,2,3,4,5}]), Stuff = [[],{},0,0.0,(1 bsl 100),(fun()-> ok end),make_ref(),self()], - [eq_cmp(A,clone(B)) || A<-Stuff, B<-Stuff], + [eq_cmp(A,clone(B)) || A<-Stuff, B<-Stuff], + + {IntSz, LongSz} = type_sizes(), + UintMax = (1 bsl (IntSz*8)) - 1, + IntMax = UintMax bsr 1, + IntMin = -(IntMax+1), + UlongMax = (1 bsl (LongSz*8)) - 1, + LongMax = UlongMax bsr 1, + LongMin = -(LongMax+1), + Uint64Max = (1 bsl 64) - 1, + Int64Max = Uint64Max bsr 1, + Int64Min = -(Int64Max+1), + Limits = [{IntMin,IntMax},{0,UintMax},{LongMin,LongMax},{0,UlongMax},{Int64Min,Int64Max},{0,Uint64Max}], + io:format("Limits = ~p\n", [Limits]), + lists:foreach(fun(I) -> + R1 = echo_int(I), + %%io:format("echo_int(~p) -> ~p\n", [I, R1]), + R2 = my_echo_int(I, Limits), + ?line R1 = R2, + ?line true = (R1 =:= R2), + ?line true = (R1 == R2) + end, int_list()), + ?line verify_tmpmem(TmpMem), ok. +int_list() -> + Start = 1 bsl 200, + int_list([Start], -Start). +int_list([N | _]=List, End) when N<End -> + List; +int_list([N | _]=List, End) -> + int_list([N - (1 + (abs(N) div 3)) | List], End). + +my_echo_int(I, Limits) -> + lists:map(fun({Min,Max}) -> + if I < Min -> false; + I > Max -> false; + true -> I + end + end, Limits). + clone(X) -> binary_to_term(term_to_binary(X)). @@ -1164,15 +1202,23 @@ make_string(Config) when is_list(Config) -> AStringWithAccents = [$E,$r,$l,$a,$n,$g,$ ,16#e4,$r,$ ,$e,$t,$t,$ ,$g,$e,$n,$e,$r,$e,$l,$l,$t,$ ,$p,$r,$o,$g,$r,$a,$m,$s,$p,$r,16#e5,$k], ?line Strings = {A0String,A0String,A0String,A0String0, AStringWithAccents}. +reverse_list_test(Config) -> + ?line ensure_lib_loaded(Config, 1), + List = lists:seq(1,100), + RevList = lists:reverse(List), + ?line RevList = reverse_list(List), + ?line badarg = reverse_list(foo). + tmpmem() -> case erlang:system_info({allocator,temp_alloc}) of false -> undefined; MemInfo -> MSBCS = lists:foldl( fun ({instance, _, L}, Acc) -> + {value,{_,SBMBCS}} = lists:keysearch(sbmbcs, 1, L), {value,{_,MBCS}} = lists:keysearch(mbcs, 1, L), {value,{_,SBCS}} = lists:keysearch(sbcs, 1, L), - [MBCS,SBCS | Acc] + [SBMBCS,MBCS,SBCS | Acc] end, [], MemInfo), @@ -1269,6 +1315,9 @@ send_blob_thread(_,_,_) -> ?nif_stub. join_send_thread(_) -> ?nif_stub. copy_blob(_) -> ?nif_stub. send_term(_,_) -> ?nif_stub. +reverse_list(_) -> ?nif_stub. +echo_int(_) -> ?nif_stub. +type_sizes() -> ?nif_stub. nif_stub_error(Line) -> exit({nif_not_loaded,module,?MODULE,line,Line}). diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c index 0bb93daa33..bdf1549862 100644 --- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c +++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c @@ -28,6 +28,7 @@ static int static_cntA; /* zero by default */ static int static_cntB = NIF_SUITE_LIB_VER * 100; +static ERL_NIF_TERM atom_false; static ERL_NIF_TERM atom_self; static ERL_NIF_TERM atom_ok; static ERL_NIF_TERM atom_join; @@ -103,7 +104,7 @@ static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) msgenv_resource_type = enif_open_resource_type(env,NULL,"nif_SUITE.msgenv", msgenv_dtor, ERL_NIF_RT_CREATE, NULL); - + atom_false = enif_make_atom(env,"false"); atom_self = enif_make_atom(env,"self"); atom_ok = enif_make_atom(env,"ok"); atom_join = enif_make_atom(env,"join"); @@ -481,6 +482,45 @@ error: return enif_make_atom(env,"error"); } +static ERL_NIF_TERM echo_int(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + int sint; + unsigned uint; + long slong; + unsigned long ulong; + ErlNifSInt64 sint64; + ErlNifUInt64 uint64; + ERL_NIF_TERM sint_term = atom_false, uint_term = atom_false; + ERL_NIF_TERM slong_term = atom_false, ulong_term = atom_false; + ERL_NIF_TERM sint64_term = atom_false, uint64_term = atom_false; + + if (enif_get_int(env, argv[0], &sint)) { + sint_term = enif_make_int(env, sint); + } + if (enif_get_uint(env, argv[0], &uint)) { + uint_term = enif_make_uint(env, uint); + } + if (enif_get_long(env, argv[0], &slong)) { + slong_term = enif_make_long(env, slong); + } + if (enif_get_ulong(env, argv[0], &ulong)) { + ulong_term = enif_make_ulong(env, ulong); + } + if (enif_get_int64(env, argv[0], &sint64)) { + sint64_term = enif_make_int64(env, sint64); + } + if (enif_get_uint64(env, argv[0], &uint64)) { + uint64_term = enif_make_uint64(env, uint64); + } + return enif_make_list6(env, sint_term, uint_term, slong_term, ulong_term, sint64_term, uint64_term); +} + +static ERL_NIF_TERM type_sizes(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + return enif_make_tuple2(env, enif_make_int(env, sizeof(int)), + enif_make_int(env, sizeof(long))); +} + static ERL_NIF_TERM tuple_2_list(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { int arity = -1; @@ -1373,6 +1413,14 @@ static ERL_NIF_TERM send_term(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[ return enif_make_int(env, ret); } +static ERL_NIF_TERM reverse_list(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { + ERL_NIF_TERM rev_list; + + if(!enif_make_reverse_list(env, argv[0], &rev_list)) + return enif_make_atom(env, "badarg"); + return rev_list; +} + static ErlNifFunc nif_funcs[] = { {"lib_version", 0, lib_version}, @@ -1417,7 +1465,10 @@ static ErlNifFunc nif_funcs[] = {"send_blob_thread", 3, send_blob_thread}, {"join_send_thread", 1, join_send_thread}, {"copy_blob", 1, copy_blob}, - {"send_term", 2, send_term} + {"send_term", 2, send_term}, + {"reverse_list",1, reverse_list}, + {"echo_int", 1, echo_int}, + {"type_sizes", 0, type_sizes} }; ERL_NIF_INIT(nif_SUITE,nif_funcs,load,reload,upgrade,unload) diff --git a/erts/emulator/test/send_term_SUITE.erl b/erts/emulator/test/send_term_SUITE.erl index 6615873392..ba0ba804ca 100644 --- a/erts/emulator/test/send_term_SUITE.erl +++ b/erts/emulator/test/send_term_SUITE.erl @@ -175,6 +175,10 @@ chk_temp_alloc() -> %% Verify that we havn't got anything allocated by temp_alloc lists:foreach( fun ({instance, _, TI}) -> + ?line {value, {sbmbcs, SBMBCInfo}} + = lists:keysearch(sbmbcs, 1, TI), + ?line {value, {blocks, 0, _, _}} + = lists:keysearch(blocks, 1, SBMBCInfo), ?line {value, {mbcs, MBCInfo}} = lists:keysearch(mbcs, 1, TI), ?line {value, {blocks, 0, _, _}} diff --git a/erts/epmd/src/epmd_int.h b/erts/epmd/src/epmd_int.h index 2a0de4df9c..a2d7559f9d 100644 --- a/erts/epmd/src/epmd_int.h +++ b/erts/epmd/src/epmd_int.h @@ -240,6 +240,14 @@ #define put_int16(i, s) {((unsigned char*)(s))[0] = ((i) >> 8) & 0xff; \ ((unsigned char*)(s))[1] = (i) & 0xff;} +#if defined(__GNUC__) +# define EPMD_INLINE __inline__ +#elif defined(__WIN32__) +# define EPMD_INLINE __inline +#else +# define EPMD_INLINE +#endif + /* ************************************************************************ */ /* Stuctures used by server */ @@ -295,6 +303,7 @@ typedef struct { unsigned delay_write; int max_conn; int active_conn; + int select_fd_top; char *progname; Connection *conn; Nodes nodes; diff --git a/erts/epmd/src/epmd_srv.c b/erts/epmd/src/epmd_srv.c index 4d9b454f97..5debae26b6 100644 --- a/erts/epmd/src/epmd_srv.c +++ b/erts/epmd/src/epmd_srv.c @@ -80,6 +80,13 @@ static int reply(EpmdVars*,int,char *,int); static void dbg_print_buf(EpmdVars*,char *,int); static void print_names(EpmdVars*); +static EPMD_INLINE void select_fd_set(EpmdVars* g, int fd) +{ + FD_SET(fd, &g->orig_read_mask); + if (fd >= g->select_fd_top) { + g->select_fd_top = fd + 1; + } +} void run(EpmdVars *g) { @@ -171,6 +178,7 @@ void run(EpmdVars *g) g->max_conn -= num_sockets; FD_ZERO(&g->orig_read_mask); + g->select_fd_top = 0; for (i = 0; i < num_sockets; i++) { @@ -232,14 +240,14 @@ void run(EpmdVars *g) dbg_perror(g,"failed to listen on socket"); epmd_cleanup_exit(g,1); } - FD_SET(listensock[i],&g->orig_read_mask); + select_fd_set(g, listensock[i]); } dbg_tty_printf(g,2,"entering the main select() loop"); select_again: while(1) - { + { fd_set read_mask = g->orig_read_mask; struct timeval timeout; int ret; @@ -251,7 +259,8 @@ void run(EpmdVars *g) timeout.tv_sec = (g->packet_timeout < IDLE_TIMEOUT) ? 1 : IDLE_TIMEOUT; timeout.tv_usec = 0; - if ((ret = select(g->max_conn,&read_mask,(fd_set *)0,(fd_set *)0,&timeout)) < 0) { + if ((ret = select(g->select_fd_top, + &read_mask, (fd_set *)0,(fd_set *)0,&timeout)) < 0) { dbg_perror(g,"error in select "); switch (errno) { case EAGAIN: @@ -821,7 +830,7 @@ static int conn_open(EpmdVars *g,int fd) s = &g->conn[i]; /* From now on we want to know if there are data to be read */ - FD_SET(fd, &g->orig_read_mask); + select_fd_set(g, fd); s->fd = fd; s->open = EPMD_TRUE; @@ -886,6 +895,7 @@ int epmd_conn_close(EpmdVars *g,Connection *s) dbg_tty_printf(g,2,"closing connection on file descriptor %d",s->fd); FD_CLR(s->fd,&g->orig_read_mask); + /* we don't bother lowering g->select_fd_top */ close(s->fd); /* Sometimes already closed but close anyway */ s->open = EPMD_FALSE; if (s->buf != NULL) { /* Should never be NULL but test anyway */ @@ -1115,7 +1125,7 @@ static Node *node_reg2(EpmdVars *g, node->extralen = extralen; memcpy(node->extra,extra,extralen); strcpy(node->symname,name); - FD_SET(fd,&g->orig_read_mask); + select_fd_set(g, fd); if (highvsn == 0) { dbg_tty_printf(g,1,"registering '%s:%d', port %d", diff --git a/erts/etc/common/erlexec.c b/erts/etc/common/erlexec.c index 7e04724f5a..2bd576d8e8 100644 --- a/erts/etc/common/erlexec.c +++ b/erts/etc/common/erlexec.c @@ -64,6 +64,7 @@ static const char plusM_au_allocs[]= { 'u', /* all alloc_util allocators */ 'B', /* binary_alloc */ + 'C', /* sbmbc_alloc */ 'D', /* std_alloc */ 'E', /* ets_alloc */ 'H', /* eheap_alloc */ @@ -93,6 +94,8 @@ static char *plusM_au_alloc_switches[] = { "rsbcst", "sbct", "smbcs", + "sbmbcs", + "sbmbct", NULL }; diff --git a/erts/include/internal/ethr_atomics.h b/erts/include/internal/ethr_atomics.h index 1caf4d0567..0f3c26f1df 100644 --- a/erts/include/internal/ethr_atomics.h +++ b/erts/include/internal/ethr_atomics.h @@ -1,7 +1,16 @@ /* + * --------------- DO NOT EDIT THIS FILE! --------------- + * This file was automatically generated by the + * $ERL_TOP/erts/lib_src/utils/make_atomics_api script. + * If you need to make changes, edit the script and + * regenerate this file. + * --------------- DO NOT EDIT THIS FILE! --------------- + */ + +/* * %CopyrightBegin% * - * Copyright Ericsson AB 2010. All Rights Reserved. + * Copyright Ericsson AB 2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -18,709 +27,8818 @@ */ /* - * Description: The ethread atomic API + * Description: The ethread atomics API * Author: Rickard Green */ -#ifndef ETHR_ATOMIC_H__ -#define ETHR_ATOMIC_H__ +/* + * This file maps native atomic implementations to ethread + * API atomics. If no native atomic implementation + * is available, a less efficient fallback is used instead. + * The API consists of 32-bit size, word size (pointer size), + * and double word size atomics. + * + * The following atomic operations are implemented for + * 32-bit size, and word size atomics: + * - cmpxchg + * - xchg + * - set + * - init + * - add_read + * - read + * - inc_read + * - dec_read + * - add + * - inc + * - dec + * - read_band + * - read_bor + * + * The following atomic operations are implemented for + * double word size atomics: + * - cmpxchg + * - set + * - read + * - init + * + * Appart from a function implementing the atomic operation + * with unspecified memory barrier semantics, there are + * functions implementing each operation with the following + * memory barrier semantics: + * - rb (read barrier) + * - wb (write barrier) + * - acqb (acquire barrier) + * - relb (release barrier) + * - mb (full memory barrier) + * + * We implement all of these operation/barrier + * combinations, regardless of whether they are useful + * or not (some of them are useless). + * + * Double word size atomic functions are on the followning + * form: + * ethr_dw_atomic_<OP>[_<BARRIER>] + * + * Word size atomic functions are on the followning + * form: + * ethr_atomic_<OP>[_<BARRIER>] + * + * 32-bit size atomic functions are on the followning + * form: + * ethr_atomic32_<OP>[_<BARRIER>] + * + * Apart from the operation/barrier functions + * described above also 'addr' functions are implemented + * which return the actual memory address used of the + * atomic variable. The 'addr' functions have no barrier + * versions. + * + * The native atomic implementation does not need to + * implement all operation/barrier combinations. + * Functions that have no native implementation will be + * constructed from existing native functionality. These + * functions will perform the wanted operation and will + * produce sufficient memory barriers, but may + * in some cases be less efficient than pure native + * versions. + * + * When we create ethread API operation/barrier functions by + * adding barriers before and after native operations it is + * assumed that: + * - A native read operation begins, and ends with a load. + * - A native set operation begins, and ends with a store. + * - An init operation begins with either a load, or a store, + * and ends with either a load, or a store. + * - All other operations begins with a load, and ends with + * either a load, or a store. + * + * This is the minimum functionality that a native + * implementation needs to provide: + * + * - Functions that need to be implemented: + * + * - ethr_native_[dw_|su_dw_]atomic[BITS]_addr + * - ethr_native_[dw_|su_dw_]atomic[BITS]_cmpxchg[_<BARRIER>] + * (at least one cmpxchg of optional barrier) + * + * - Macros that needs to be defined: + * + * A macro informing about the presence of the native + * implementation: + * + * - ETHR_HAVE_NATIVE_[DW_|SU_DW_]ATOMIC[BITS] + * + * A macro naming (a string constant) the implementation: + * + * - ETHR_NATIVE_[DW_]ATOMIC[BITS]_IMPL + * + * Each implemented native atomic function has to + * be accompanied by a defined macro on the following + * form informing about its presence: + * + * - ETHR_HAVE_ETHR_NATIVE_[DW_|SU_DW_]ATOMIC[BITS]_<OP>[_<BARRIER>] + * + * A (sparc-v9 style) membar macro: + * + * - ETHR_MEMBAR(B) + * + * Which takes a combination of the following macros + * or:ed (using |) together: + * + * - ETHR_LoadLoad + * - ETHR_LoadStore + * - ETHR_StoreLoad + * - ETHR_StoreStore + * + */ -#if !defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) -# define ETHR_NEED_ATOMIC_PROTOTYPES__ -#endif +#ifndef ETHR_ATOMICS_H__ +#define ETHR_ATOMICS_H__ -#ifndef ETHR_HAVE_NATIVE_ATOMICS +#undef ETHR_AMC_FALLBACK__ +#undef ETHR_AMC_NO_ATMCS__ +#undef ETHR_AMC_ATMC_T__ +#undef ETHR_AMC_ATMC_FUNC__ + +/* -- 32-bit atomics -- */ + +#undef ETHR_NAINT32_T__ +#undef ETHR_NATMC32_FUNC__ +#undef ETHR_NATMC32_ADDR_FUNC__ +#undef ETHR_NATMC32_BITS__ +#if defined(ETHR_HAVE_NATIVE_ATOMIC32) +# define ETHR_NEED_NATMC32_ADDR +# define ETHR_NATMC32_ADDR_FUNC__ ethr_native_atomic32_addr +typedef ethr_native_atomic32_t ethr_atomic32_t; +# define ETHR_NAINT32_T__ ethr_sint32_t +# define ETHR_NATMC32_FUNC__(X) ethr_native_atomic32_ ## X +# define ETHR_NATMC32_BITS__ 32 +#elif defined(ETHR_HAVE_NATIVE_ATOMIC64) +# define ETHR_NEED_NATMC64_ADDR +#ifdef ETHR_BIGENDIAN +# define ETHR_NATMC32_ADDR_FUNC__(VAR) \ + (((ethr_sint32_t *) ethr_native_atomic64_addr((VAR))) + 1) +#else +# define ETHR_NATMC32_ADDR_FUNC__(VAR) \ + ((ethr_sint32_t *) ethr_native_atomic64_addr((VAR))) +#endif +typedef ethr_native_atomic64_t ethr_atomic32_t; +# define ETHR_NAINT32_T__ ethr_sint64_t +# define ETHR_NATMC32_FUNC__(X) ethr_native_atomic64_ ## X +# define ETHR_NATMC32_BITS__ 64 +#else /* - * No native atomic implementation available. :( + * No native atomics usable for 32-bits atomics :( * Use fallback... */ typedef ethr_sint32_t ethr_atomic32_t; -typedef ethr_sint_t ethr_atomic_t; -#else -/* - * Map ethread native atomics to ethread API atomics. - * - * We do at least have a native atomic implementation that - * can handle integers of a size larger than or equal to - * the size of pointers. - */ +#endif -/* -- Pointer size atomics -- */ +#undef ETHR_ATMC32_INLINE__ +#ifdef ETHR_NATMC32_BITS__ +# ifdef ETHR_TRY_INLINE_FUNCS +# define ETHR_ATMC32_INLINE__ +# endif +# define ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS +#endif + +#if !defined(ETHR_ATMC32_INLINE__) || defined(ETHR_ATOMIC_IMPL__) +# define ETHR_NEED_ATMC32_PROTOTYPES__ +#endif + +#ifndef ETHR_INLINE_ATMC32_FUNC_NAME_ +# define ETHR_INLINE_ATMC32_FUNC_NAME_(X) X +#endif + +#undef ETHR_ATMC32_FUNC__ +#define ETHR_ATMC32_FUNC__(X) ETHR_INLINE_ATMC32_FUNC_NAME_(ethr_atomic32_ ## X) + + +/* -- Word size atomics -- */ + +#undef ETHR_NEED_NATMC32_ADDR +#undef ETHR_NEED_NATMC64_ADDR #undef ETHR_NAINT_T__ #undef ETHR_NATMC_FUNC__ #undef ETHR_NATMC_ADDR_FUNC__ -#if ETHR_SIZEOF_PTR == 8 -# if defined(ETHR_HAVE_NATIVE_ATOMIC64) -# define ETHR_NATMC_ADDR_FUNC__ ethr_native_atomic64_addr +#undef ETHR_NATMC_BITS__ +#if ETHR_SIZEOF_PTR == 8 && defined(ETHR_HAVE_NATIVE_ATOMIC64) +# ifndef ETHR_NEED_NATMC64_ADDR +# define ETHR_NEED_NATMC64_ADDR +# endif +# define ETHR_NATMC_ADDR_FUNC__ ethr_native_atomic64_addr typedef ethr_native_atomic64_t ethr_atomic_t; -# define ETHR_NAINT_T__ ethr_sint64_t -# define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X -# else -# error "Missing native atomic implementation" +# define ETHR_NAINT_T__ ethr_sint64_t +# define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X +# define ETHR_NATMC_BITS__ 64 +#elif ETHR_SIZEOF_PTR == 4 && defined(ETHR_HAVE_NATIVE_ATOMIC32) +# ifndef ETHR_NEED_NATMC64_ADDR +# define ETHR_NEED_NATMC32_ADDR # endif -#elif ETHR_SIZEOF_PTR == 4 # define ETHR_NATMC_ADDR_FUNC__ ethr_native_atomic32_addr -# ifdef ETHR_HAVE_NATIVE_ATOMIC32 typedef ethr_native_atomic32_t ethr_atomic_t; -# define ETHR_NAINT_T__ ethr_sint32_t -# define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X -# elif defined(ETHR_HAVE_NATIVE_ATOMIC64) +# define ETHR_NAINT_T__ ethr_sint32_t +# define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X +# define ETHR_NATMC_BITS__ 32 +#elif ETHR_SIZEOF_PTR == 4 && defined(ETHR_HAVE_NATIVE_ATOMIC64) +# ifndef ETHR_NEED_NATMC64_ADDR +# define ETHR_NEED_NATMC64_ADDR +# endif +#ifdef ETHR_BIGENDIAN +# define ETHR_NATMC_ADDR_FUNC__(VAR) \ + (((ethr_sint32_t *) ethr_native_atomic64_addr((VAR))) + 1) +#else +# define ETHR_NATMC_ADDR_FUNC__(VAR) \ + ((ethr_sint32_t *) ethr_native_atomic64_addr((VAR))) +#endif typedef ethr_native_atomic64_t ethr_atomic_t; -# define ETHR_NATMC_T__ ethr_native_atomic64_t -# define ETHR_NAINT_T__ ethr_sint64_t -# define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X +# define ETHR_NATMC_T__ ethr_native_atomic64_t +# define ETHR_NAINT_T__ ethr_sint64_t +# define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X +# define ETHR_NATMC_BITS__ 64 +#else +/* + * No native atomics usable for pointer size atomics :( + * Use fallback... + */ + +# if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) +# define ETHR_AMC_FALLBACK__ +# define ETHR_AMC_NO_ATMCS__ 2 +# define ETHR_AMC_SINT_T__ ethr_sint32_t +# define ETHR_AMC_ATMC_T__ ethr_atomic32_t +# define ETHR_AMC_ATMC_FUNC__(X) ETHR_INLINE_ATMC32_FUNC_NAME_(ethr_atomic32_ ## X) +typedef struct { + ETHR_AMC_ATMC_T__ atomic[ETHR_AMC_NO_ATMCS__]; +} ethr_amc_t; +typedef struct { + ethr_amc_t amc; + ethr_sint_t sint; +} ethr_atomic_t; +# else /* locked fallback */ +typedef ethr_sint_t ethr_atomic_t; +# endif +#endif + +#undef ETHR_ATMC_INLINE__ +#ifdef ETHR_NATMC_BITS__ +# ifdef ETHR_TRY_INLINE_FUNCS +# define ETHR_ATMC_INLINE__ +# endif +# define ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS +#endif + +#if !defined(ETHR_ATMC_INLINE__) || defined(ETHR_ATOMIC_IMPL__) +# define ETHR_NEED_ATMC_PROTOTYPES__ +#endif + +#ifndef ETHR_INLINE_ATMC_FUNC_NAME_ +# define ETHR_INLINE_ATMC_FUNC_NAME_(X) X +#endif + +#undef ETHR_ATMC_FUNC__ +#define ETHR_ATMC_FUNC__(X) ETHR_INLINE_ATMC_FUNC_NAME_(ethr_atomic_ ## X) + +/* -- Double word atomics -- */ + +#undef ETHR_SU_DW_NAINT_T__ +#undef ETHR_SU_DW_NATMC_FUNC__ +#undef ETHR_SU_DW_NATMC_ADDR_FUNC__ +#undef ETHR_DW_NATMC_FUNC__ +#undef ETHR_DW_NATMC_ADDR_FUNC__ +#undef ETHR_DW_NATMC_BITS__ +#if defined(ETHR_HAVE_NATIVE_DW_ATOMIC) || defined(ETHR_HAVE_NATIVE_SU_DW_ATOMIC) +# define ETHR_NEED_DW_NATMC_ADDR +# define ETHR_DW_NATMC_ADDR_FUNC__ ethr_native_dw_atomic_addr +# define ETHR_NATIVE_DW_ATOMIC_T__ ethr_native_dw_atomic_t +# define ETHR_DW_NATMC_FUNC__(X) ethr_native_dw_atomic_ ## X +# define ETHR_SU_DW_NATMC_FUNC__(X) ethr_native_su_dw_atomic_ ## X +# if ETHR_SIZEOF_PTR == 8 +# define ETHR_DW_NATMC_BITS__ 128 +# elif ETHR_SIZEOF_PTR == 4 +# define ETHR_DW_NATMC_BITS__ 64 # else -# error "Missing native atomic implementation" +# error "Word size not supported" +# endif +# ifdef ETHR_NATIVE_SU_DW_SINT_T +# define ETHR_SU_DW_NAINT_T__ ETHR_NATIVE_SU_DW_SINT_T # endif +#elif ETHR_SIZEOF_PTR == 4 && defined(ETHR_HAVE_NATIVE_ATOMIC64) +# define ETHR_HAVE_NATIVE_SU_DW_ATOMIC +# ifndef ETHR_NEED_NATMC64_ADDR +# define ETHR_NEED_NATMC64_ADDR +# endif +# define ETHR_DW_NATMC_ADDR_FUNC__(VAR) \ + ((ethr_dw_sint_t *) ethr_native_atomic64_addr((VAR))) +# define ETHR_NATIVE_DW_ATOMIC_T__ ethr_native_atomic64_t +# define ETHR_SU_DW_NAINT_T__ ethr_sint64_t +# define ETHR_SU_DW_NATMC_FUNC__(X) ethr_native_atomic64_ ## X +# define ETHR_DW_NATMC_BITS__ 64 #endif -/* -- 32-bit atomics -- */ +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) +#define ETHR_DW_ATOMIC_FUNC__(X) ethr_dw_atomic_ ## X ## _fallback__ +#else +#define ETHR_DW_ATOMIC_FUNC__(X) ethr_dw_atomic_ ## X +#endif -#undef ETHR_NAINT32_T__ -#undef ETHR_NATMC32_FUNC__ -#if defined(ETHR_HAVE_NATIVE_ATOMIC32) -typedef ethr_native_atomic32_t ethr_atomic32_t; -# define ETHR_NAINT32_T__ ethr_sint32_t -# define ETHR_NATMC32_FUNC__(X) ethr_native_atomic32_ ## X -#elif defined(ETHR_HAVE_NATIVE_ATOMIC64) -typedef ethr_native_atomic64_t ethr_atomic32_t; -# define ETHR_NAINT32_T__ ethr_sint64_t -# define ETHR_NATMC32_FUNC__(X) ethr_native_atomic64_ ## X +#if !defined(ETHR_DW_NATMC_BITS__) || defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) +# define ETHR_NEED_DW_FALLBACK__ +#endif + +#if defined(ETHR_NEED_DW_FALLBACK__) +/* + * No native atomics usable for double word atomics :( + * Use fallback... + */ + +# ifndef ETHR_AMC_FALLBACK__ +# if ETHR_SIZEOF_PTR == 8 && defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) +# define ETHR_AMC_FALLBACK__ +# define ETHR_AMC_NO_ATMCS__ 1 +# define ETHR_AMC_SINT_T__ ethr_sint_t +# define ETHR_AMC_ATMC_T__ ethr_atomic_t +# define ETHR_AMC_ATMC_FUNC__(X) ETHR_INLINE_ATMC_FUNC_NAME_(ethr_atomic_ ## X) +# elif defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) +# define ETHR_AMC_FALLBACK__ +# define ETHR_AMC_NO_ATMCS__ 2 +# define ETHR_AMC_SINT_T__ ethr_sint32_t +# define ETHR_AMC_ATMC_T__ ethr_atomic32_t +# define ETHR_AMC_ATMC_FUNC__(X) ETHR_INLINE_ATMC32_FUNC_NAME_(ethr_atomic32_ ## X) +# endif +# ifdef ETHR_AMC_FALLBACK__ +typedef struct { + ETHR_AMC_ATMC_T__ atomic[ETHR_AMC_NO_ATMCS__]; +} ethr_amc_t; +# endif +# endif + +typedef struct { +#ifdef ETHR_AMC_FALLBACK__ + ethr_amc_t amc; +#endif + ethr_sint_t sint[2]; +} ethr_dw_atomic_fallback_t; + +#endif + +typedef union { +#ifdef ETHR_NATIVE_DW_ATOMIC_T__ + ETHR_NATIVE_DW_ATOMIC_T__ native; +#endif +#ifdef ETHR_NEED_DW_FALLBACK__ + ethr_dw_atomic_fallback_t fallback; +#endif + ethr_sint_t sint[2]; +} ethr_dw_atomic_t; + +typedef union { +#ifdef ETHR_SU_DW_NAINT_T__ + ETHR_SU_DW_NAINT_T__ dw_sint; +#endif + ethr_sint_t sint[2]; +} ethr_dw_sint_t; + +#ifdef ETHR_BIGENDIAN +# define ETHR_DW_SINT_LOW_WORD 1 +# define ETHR_DW_SINT_HIGH_WORD 0 #else -# error "Missing native atomic implementation" +# define ETHR_DW_SINT_LOW_WORD 0 +# define ETHR_DW_SINT_HIGH_WORD 1 #endif +#undef ETHR_DW_ATMC_INLINE__ +#ifdef ETHR_DW_NATMC_BITS__ +# ifdef ETHR_TRY_INLINE_FUNCS +# define ETHR_ATMC32_INLINE__ +# endif +# define ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS #endif -#ifdef ETHR_NEED_ATOMIC_PROTOTYPES__ -ethr_sint_t *ethr_atomic_addr(ethr_atomic_t *); -void ethr_atomic_init(ethr_atomic_t *, ethr_sint_t); -void ethr_atomic_set(ethr_atomic_t *, ethr_sint_t); -ethr_sint_t ethr_atomic_read(ethr_atomic_t *); -ethr_sint_t ethr_atomic_inc_read(ethr_atomic_t *); -ethr_sint_t ethr_atomic_dec_read(ethr_atomic_t *); -void ethr_atomic_inc(ethr_atomic_t *); -void ethr_atomic_dec(ethr_atomic_t *); -ethr_sint_t ethr_atomic_add_read(ethr_atomic_t *, ethr_sint_t); -void ethr_atomic_add(ethr_atomic_t *, ethr_sint_t); -ethr_sint_t ethr_atomic_read_band(ethr_atomic_t *, ethr_sint_t); -ethr_sint_t ethr_atomic_read_bor(ethr_atomic_t *, ethr_sint_t); -ethr_sint_t ethr_atomic_xchg(ethr_atomic_t *, ethr_sint_t); -ethr_sint_t ethr_atomic_cmpxchg(ethr_atomic_t *, ethr_sint_t, ethr_sint_t); -ethr_sint_t ethr_atomic_read_acqb(ethr_atomic_t *); -ethr_sint_t ethr_atomic_inc_read_acqb(ethr_atomic_t *); -void ethr_atomic_set_relb(ethr_atomic_t *, ethr_sint_t); -void ethr_atomic_dec_relb(ethr_atomic_t *); -ethr_sint_t ethr_atomic_dec_read_relb(ethr_atomic_t *); -ethr_sint_t ethr_atomic_cmpxchg_acqb(ethr_atomic_t *, ethr_sint_t, ethr_sint_t); -ethr_sint_t ethr_atomic_cmpxchg_relb(ethr_atomic_t *, ethr_sint_t, ethr_sint_t); - -ethr_sint32_t *ethr_atomic32_addr(ethr_atomic32_t *); -void ethr_atomic32_init(ethr_atomic32_t *, ethr_sint32_t); -void ethr_atomic32_set(ethr_atomic32_t *, ethr_sint32_t); -ethr_sint32_t ethr_atomic32_read(ethr_atomic32_t *); -ethr_sint32_t ethr_atomic32_inc_read(ethr_atomic32_t *); -ethr_sint32_t ethr_atomic32_dec_read(ethr_atomic32_t *); -void ethr_atomic32_inc(ethr_atomic32_t *); -void ethr_atomic32_dec(ethr_atomic32_t *); -ethr_sint32_t ethr_atomic32_add_read(ethr_atomic32_t *, ethr_sint32_t); -void ethr_atomic32_add(ethr_atomic32_t *, ethr_sint32_t); -ethr_sint32_t ethr_atomic32_read_band(ethr_atomic32_t *, ethr_sint32_t); -ethr_sint32_t ethr_atomic32_read_bor(ethr_atomic32_t *, ethr_sint32_t); -ethr_sint32_t ethr_atomic32_xchg(ethr_atomic32_t *, ethr_sint32_t); -ethr_sint32_t ethr_atomic32_cmpxchg(ethr_atomic32_t *, - ethr_sint32_t, - ethr_sint32_t); -ethr_sint32_t ethr_atomic32_read_acqb(ethr_atomic32_t *); -ethr_sint32_t ethr_atomic32_inc_read_acqb(ethr_atomic32_t *); -void ethr_atomic32_set_relb(ethr_atomic32_t *, ethr_sint32_t); -void ethr_atomic32_dec_relb(ethr_atomic32_t *); -ethr_sint32_t ethr_atomic32_dec_read_relb(ethr_atomic32_t *); -ethr_sint32_t ethr_atomic32_cmpxchg_acqb(ethr_atomic32_t *, - ethr_sint32_t, - ethr_sint32_t); -ethr_sint32_t ethr_atomic32_cmpxchg_relb(ethr_atomic32_t *, - ethr_sint32_t, - ethr_sint32_t); +#if !defined(ETHR_DW_ATMC_INLINE__) || defined(ETHR_ATOMIC_IMPL__) +# define ETHR_NEED_DW_ATMC_PROTOTYPES__ #endif -int ethr_init_atomics(void); +#ifndef ETHR_INLINE_DW_ATMC_FUNC_NAME_ +# define ETHR_INLINE_DW_ATMC_FUNC_NAME_(X) X +#endif + +#undef ETHR_DW_ATMC_FUNC__ +#define ETHR_DW_ATMC_FUNC__(X) ETHR_INLINE_DW_ATMC_FUNC_NAME_(ethr_dw_atomic_ ## X) + +#if defined(ETHR_NEED_DW_ATMC_PROTOTYPES__) +int ethr_have_native_dw_atomic(void); +#endif +#if defined(ETHR_DW_ATMC_INLINE__) || defined(ETHR_ATOMIC_IMPL__) +static ETHR_INLINE int +ETHR_INLINE_DW_ATMC_FUNC_NAME_(ethr_have_native_dw_atomic)(void) +{ +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + return ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__; +#elif defined(ETHR_DW_NATMC_BITS__) + return 1; +#else + return 0; +#endif +} +#endif + +/* -- Misc -- */ #if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) +/* + * Unusual values are used by read() fallbacks implemented via cmpxchg(). + * We want to use an unusual value in hope that it is more efficient + * not to match the value in memory. + * + * - Negative integer values are probably more unusual. + * - Very large absolute integer values are probably more unusual. + * - Odd pointers are probably more unusual (only char pointers can be odd). + */ +# define ETHR_UNUSUAL_SINT32_VAL__ ((ethr_sint32_t) 0x81818181) +# if ETHR_SIZEOF_PTR == 4 +# define ETHR_UNUSUAL_SINT_VAL__ ((ethr_sint_t) ETHR_UNUSUAL_SINT32_VAL__) +# elif ETHR_SIZEOF_PTR == 8 +# define ETHR_UNUSUAL_SINT_VAL__ ((ethr_sint_t) 0x8181818181818181L) +# else +# error "Word size not supported" +# endif +# if defined(ETHR_NEED_DW_NATMC_ADDR) && !defined(ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_ADDR) +# error "No ethr_native_dw_atomic_addr() available" +# endif +# if defined(ETHR_NEED_NATMC32_ADDR) && !defined(ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADDR) +# error "No ethr_native_atomic32_addr() available" +# endif +# if defined(ETHR_NEED_NATMC64_ADDR) && !defined(ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADDR) +# error "No ethr_native_atomic64_addr() available" +# endif +#endif + +#if defined(__GNUC__) +# ifndef ETHR_COMPILER_BARRIER +# define ETHR_COMPILER_BARRIER __asm__ __volatile__("" : : : "memory") +# endif +#elif defined(ETHR_WIN32_THREADS) +# ifndef ETHR_COMPILER_BARRIER +# include <intrin.h> +# pragma intrinsic(_ReadWriteBarrier) +# define ETHR_COMPILER_BARRIER _ReadWriteBarrier() +# endif +#endif -#ifndef ETHR_HAVE_NATIVE_ATOMICS +void ethr_compiler_barrier_fallback(void); +#ifndef ETHR_COMPILER_BARRIER +# define ETHR_COMPILER_BARRIER ethr_compiler_barrier_fallback() +#endif + +int ethr_init_atomics(void); + +/* info */ +char **ethr_native_atomic32_ops(void); +char **ethr_native_atomic64_ops(void); +char **ethr_native_dw_atomic_ops(void); +char **ethr_native_su_dw_atomic_ops(void); + +#if !defined(ETHR_DW_NATMC_BITS__) && !defined(ETHR_NATMC_BITS__) && !defined(ETHR_NATMC32_BITS__) /* - * Fallbacks for atomics used in absence of a native implementation. + * ETHR_*MEMORY_BARRIER orders between locked and atomic accesses only, + * i.e. when no native atomic implementation exist and only our lock + * based atomic fallback is used, a noop is sufficient. */ +# undef ETHR_MEMORY_BARRIER +# undef ETHR_WRITE_MEMORY_BARRIER +# undef ETHR_READ_MEMORY_BARRIER +# undef ETHR_READ_DEPEND_MEMORY_BARRIER +# undef ETHR_MEMBAR +# define ETHR_MEMBAR(B) do { } while (0) +#endif + +#ifndef ETHR_MEMBAR +# error "No ETHR_MEMBAR defined" +#endif -#define ETHR_ATOMIC_ADDR_BITS 10 -#define ETHR_ATOMIC_ADDR_SHIFT 6 +#define ETHR_MEMORY_BARRIER ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore) +#define ETHR_WRITE_MEMORY_BARRIER ETHR_MEMBAR(ETHR_StoreStore) +#define ETHR_READ_MEMORY_BARRIER ETHR_MEMBAR(ETHR_LoadLoad) +#ifdef ETHR_READ_DEPEND_MEMORY_BARRIER +# undef ETHR_ORDERED_READ_DEPEND +#else +# define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_COMPILER_BARRIER +# define ETHR_ORDERED_READ_DEPEND +#endif -typedef struct { - union { - ethr_spinlock_t lck; - char buf[ETHR_CACHE_LINE_SIZE]; - } u; -} ethr_atomic_protection_t; -extern ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATOMIC_ADDR_BITS]; +/* ---------- Double word size atomic implementation ---------- */ + + +#ifdef ETHR_NEED_DW_ATMC_PROTOTYPES__ +ethr_sint_t *ethr_dw_atomic_addr(ethr_dw_atomic_t *var); +int ethr_dw_atomic_cmpxchg(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val); +int ethr_dw_atomic_cmpxchg_rb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val); +int ethr_dw_atomic_cmpxchg_wb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val); +int ethr_dw_atomic_cmpxchg_acqb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val); +int ethr_dw_atomic_cmpxchg_relb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val); +int ethr_dw_atomic_cmpxchg_mb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val); +void ethr_dw_atomic_set(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ethr_dw_atomic_set_rb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ethr_dw_atomic_set_wb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ethr_dw_atomic_set_acqb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ethr_dw_atomic_set_relb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ethr_dw_atomic_set_mb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ethr_dw_atomic_read(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ethr_dw_atomic_read_rb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ethr_dw_atomic_read_wb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ethr_dw_atomic_read_acqb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ethr_dw_atomic_read_relb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ethr_dw_atomic_read_mb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ethr_dw_atomic_init(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ethr_dw_atomic_init_rb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ethr_dw_atomic_init_wb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ethr_dw_atomic_init_acqb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ethr_dw_atomic_init_relb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ethr_dw_atomic_init_mb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) +ethr_sint_t *ETHR_DW_ATOMIC_FUNC__(addr)(ethr_dw_atomic_t *var); +int ETHR_DW_ATOMIC_FUNC__(cmpxchg)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val); +int ETHR_DW_ATOMIC_FUNC__(cmpxchg_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val); +int ETHR_DW_ATOMIC_FUNC__(cmpxchg_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val); +int ETHR_DW_ATOMIC_FUNC__(cmpxchg_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val); +int ETHR_DW_ATOMIC_FUNC__(cmpxchg_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val); +int ETHR_DW_ATOMIC_FUNC__(cmpxchg_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val); +void ETHR_DW_ATOMIC_FUNC__(set)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ETHR_DW_ATOMIC_FUNC__(set_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ETHR_DW_ATOMIC_FUNC__(set_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ETHR_DW_ATOMIC_FUNC__(set_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ETHR_DW_ATOMIC_FUNC__(set_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ETHR_DW_ATOMIC_FUNC__(set_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ETHR_DW_ATOMIC_FUNC__(read)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ETHR_DW_ATOMIC_FUNC__(read_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ETHR_DW_ATOMIC_FUNC__(read_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ETHR_DW_ATOMIC_FUNC__(read_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ETHR_DW_ATOMIC_FUNC__(read_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ETHR_DW_ATOMIC_FUNC__(read_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ETHR_DW_ATOMIC_FUNC__(init)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ETHR_DW_ATOMIC_FUNC__(init_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ETHR_DW_ATOMIC_FUNC__(init_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ETHR_DW_ATOMIC_FUNC__(init_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ETHR_DW_ATOMIC_FUNC__(init_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +void ETHR_DW_ATOMIC_FUNC__(init_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val); +#endif +#endif /* ETHR_NEED_DW_ATMC_PROTOTYPES__ */ + +#if (defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) \ + && (defined(ETHR_DW_ATMC_INLINE__) || defined(ETHR_ATOMIC_IMPL__))) + +#if !defined(ETHR_DW_NATMC_BITS__) +# error "Missing native atomic implementation" +#elif defined(ETHR_HAVE_NATIVE_DW_ATOMIC) || defined(ETHR_HAVE_NATIVE_SU_DW_ATOMIC) +# undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG +# undef ETHR_HAVE_DW_NATMC_CMPXCHG +# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG +# define ETHR_HAVE_DW_NATMC_CMPXCHG 1 +# endif +# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG +# define ETHR_HAVE_SU_DW_NATMC_CMPXCHG 1 +# endif +# undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB +# undef ETHR_HAVE_DW_NATMC_CMPXCHG_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_RB +# define ETHR_HAVE_DW_NATMC_CMPXCHG_RB 1 +# endif +# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_RB +# define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB 1 +# endif +# undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB +# undef ETHR_HAVE_DW_NATMC_CMPXCHG_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_WB +# define ETHR_HAVE_DW_NATMC_CMPXCHG_WB 1 +# endif +# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_WB +# define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB 1 +# endif +# undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB +# undef ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_ACQB +# define ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB 1 +# endif +# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_ACQB +# define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB 1 +# endif +# undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB +# undef ETHR_HAVE_DW_NATMC_CMPXCHG_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_RELB +# define ETHR_HAVE_DW_NATMC_CMPXCHG_RELB 1 +# endif +# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_RELB +# define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB 1 +# endif +# undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB +# undef ETHR_HAVE_DW_NATMC_CMPXCHG_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_MB +# define ETHR_HAVE_DW_NATMC_CMPXCHG_MB 1 +# endif +# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_MB +# define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB 1 +# endif +# undef ETHR_HAVE_SU_DW_NATMC_SET +# undef ETHR_HAVE_DW_NATMC_SET +# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET +# define ETHR_HAVE_DW_NATMC_SET 1 +# endif +# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET +# define ETHR_HAVE_SU_DW_NATMC_SET 1 +# endif +# undef ETHR_HAVE_SU_DW_NATMC_SET_RB +# undef ETHR_HAVE_DW_NATMC_SET_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_RB +# define ETHR_HAVE_DW_NATMC_SET_RB 1 +# endif +# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_RB +# define ETHR_HAVE_SU_DW_NATMC_SET_RB 1 +# endif +# undef ETHR_HAVE_SU_DW_NATMC_SET_WB +# undef ETHR_HAVE_DW_NATMC_SET_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_WB +# define ETHR_HAVE_DW_NATMC_SET_WB 1 +# endif +# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_WB +# define ETHR_HAVE_SU_DW_NATMC_SET_WB 1 +# endif +# undef ETHR_HAVE_SU_DW_NATMC_SET_ACQB +# undef ETHR_HAVE_DW_NATMC_SET_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_ACQB +# define ETHR_HAVE_DW_NATMC_SET_ACQB 1 +# endif +# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_ACQB +# define ETHR_HAVE_SU_DW_NATMC_SET_ACQB 1 +# endif +# undef ETHR_HAVE_SU_DW_NATMC_SET_RELB +# undef ETHR_HAVE_DW_NATMC_SET_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_RELB +# define ETHR_HAVE_DW_NATMC_SET_RELB 1 +# endif +# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_RELB +# define ETHR_HAVE_SU_DW_NATMC_SET_RELB 1 +# endif +# undef ETHR_HAVE_SU_DW_NATMC_SET_MB +# undef ETHR_HAVE_DW_NATMC_SET_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_MB +# define ETHR_HAVE_DW_NATMC_SET_MB 1 +# endif +# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_MB +# define ETHR_HAVE_SU_DW_NATMC_SET_MB 1 +# endif +# undef ETHR_HAVE_SU_DW_NATMC_READ +# undef ETHR_HAVE_DW_NATMC_READ +# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ +# define ETHR_HAVE_DW_NATMC_READ 1 +# endif +# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ +# define ETHR_HAVE_SU_DW_NATMC_READ 1 +# endif +# undef ETHR_HAVE_SU_DW_NATMC_READ_RB +# undef ETHR_HAVE_DW_NATMC_READ_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_RB +# define ETHR_HAVE_DW_NATMC_READ_RB 1 +# endif +# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_RB +# define ETHR_HAVE_SU_DW_NATMC_READ_RB 1 +# endif +# undef ETHR_HAVE_SU_DW_NATMC_READ_WB +# undef ETHR_HAVE_DW_NATMC_READ_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_WB +# define ETHR_HAVE_DW_NATMC_READ_WB 1 +# endif +# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_WB +# define ETHR_HAVE_SU_DW_NATMC_READ_WB 1 +# endif +# undef ETHR_HAVE_SU_DW_NATMC_READ_ACQB +# undef ETHR_HAVE_DW_NATMC_READ_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_ACQB +# define ETHR_HAVE_DW_NATMC_READ_ACQB 1 +# endif +# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_ACQB +# define ETHR_HAVE_SU_DW_NATMC_READ_ACQB 1 +# endif +# undef ETHR_HAVE_SU_DW_NATMC_READ_RELB +# undef ETHR_HAVE_DW_NATMC_READ_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_RELB +# define ETHR_HAVE_DW_NATMC_READ_RELB 1 +# endif +# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_RELB +# define ETHR_HAVE_SU_DW_NATMC_READ_RELB 1 +# endif +# undef ETHR_HAVE_SU_DW_NATMC_READ_MB +# undef ETHR_HAVE_DW_NATMC_READ_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_MB +# define ETHR_HAVE_DW_NATMC_READ_MB 1 +# endif +# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_MB +# define ETHR_HAVE_SU_DW_NATMC_READ_MB 1 +# endif +# undef ETHR_HAVE_SU_DW_NATMC_INIT +# undef ETHR_HAVE_DW_NATMC_INIT +# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT +# define ETHR_HAVE_DW_NATMC_INIT 1 +# endif +# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT +# define ETHR_HAVE_SU_DW_NATMC_INIT 1 +# endif +# undef ETHR_HAVE_SU_DW_NATMC_INIT_RB +# undef ETHR_HAVE_DW_NATMC_INIT_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_RB +# define ETHR_HAVE_DW_NATMC_INIT_RB 1 +# endif +# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_RB +# define ETHR_HAVE_SU_DW_NATMC_INIT_RB 1 +# endif +# undef ETHR_HAVE_SU_DW_NATMC_INIT_WB +# undef ETHR_HAVE_DW_NATMC_INIT_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_WB +# define ETHR_HAVE_DW_NATMC_INIT_WB 1 +# endif +# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_WB +# define ETHR_HAVE_SU_DW_NATMC_INIT_WB 1 +# endif +# undef ETHR_HAVE_SU_DW_NATMC_INIT_ACQB +# undef ETHR_HAVE_DW_NATMC_INIT_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_ACQB +# define ETHR_HAVE_DW_NATMC_INIT_ACQB 1 +# endif +# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_ACQB +# define ETHR_HAVE_SU_DW_NATMC_INIT_ACQB 1 +# endif +# undef ETHR_HAVE_SU_DW_NATMC_INIT_RELB +# undef ETHR_HAVE_DW_NATMC_INIT_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_RELB +# define ETHR_HAVE_DW_NATMC_INIT_RELB 1 +# endif +# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_RELB +# define ETHR_HAVE_SU_DW_NATMC_INIT_RELB 1 +# endif +# undef ETHR_HAVE_SU_DW_NATMC_INIT_MB +# undef ETHR_HAVE_DW_NATMC_INIT_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_MB +# define ETHR_HAVE_DW_NATMC_INIT_MB 1 +# endif +# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_MB +# define ETHR_HAVE_SU_DW_NATMC_INIT_MB 1 +# endif +#elif ETHR_DW_NATMC_BITS__ == 64 +# undef ETHR_HAVE_DW_NATMC_CMPXCHG +# undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG +# define ETHR_HAVE_SU_DW_NATMC_CMPXCHG 1 +# endif +# undef ETHR_HAVE_DW_NATMC_CMPXCHG_RB +# undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RB +# define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB 1 +# endif +# undef ETHR_HAVE_DW_NATMC_CMPXCHG_WB +# undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_WB +# define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB 1 +# endif +# undef ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB +# undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_ACQB +# define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB 1 +# endif +# undef ETHR_HAVE_DW_NATMC_CMPXCHG_RELB +# undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RELB +# define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB 1 +# endif +# undef ETHR_HAVE_DW_NATMC_CMPXCHG_MB +# undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_MB +# define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB 1 +# endif +# undef ETHR_HAVE_DW_NATMC_SET +# undef ETHR_HAVE_SU_DW_NATMC_SET +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET +# define ETHR_HAVE_SU_DW_NATMC_SET 1 +# endif +# undef ETHR_HAVE_DW_NATMC_SET_RB +# undef ETHR_HAVE_SU_DW_NATMC_SET_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RB +# define ETHR_HAVE_SU_DW_NATMC_SET_RB 1 +# endif +# undef ETHR_HAVE_DW_NATMC_SET_WB +# undef ETHR_HAVE_SU_DW_NATMC_SET_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_WB +# define ETHR_HAVE_SU_DW_NATMC_SET_WB 1 +# endif +# undef ETHR_HAVE_DW_NATMC_SET_ACQB +# undef ETHR_HAVE_SU_DW_NATMC_SET_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_ACQB +# define ETHR_HAVE_SU_DW_NATMC_SET_ACQB 1 +# endif +# undef ETHR_HAVE_DW_NATMC_SET_RELB +# undef ETHR_HAVE_SU_DW_NATMC_SET_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RELB +# define ETHR_HAVE_SU_DW_NATMC_SET_RELB 1 +# endif +# undef ETHR_HAVE_DW_NATMC_SET_MB +# undef ETHR_HAVE_SU_DW_NATMC_SET_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_MB +# define ETHR_HAVE_SU_DW_NATMC_SET_MB 1 +# endif +# undef ETHR_HAVE_DW_NATMC_READ +# undef ETHR_HAVE_SU_DW_NATMC_READ +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ +# define ETHR_HAVE_SU_DW_NATMC_READ 1 +# endif +# undef ETHR_HAVE_DW_NATMC_READ_RB +# undef ETHR_HAVE_SU_DW_NATMC_READ_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_RB +# define ETHR_HAVE_SU_DW_NATMC_READ_RB 1 +# endif +# undef ETHR_HAVE_DW_NATMC_READ_WB +# undef ETHR_HAVE_SU_DW_NATMC_READ_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_WB +# define ETHR_HAVE_SU_DW_NATMC_READ_WB 1 +# endif +# undef ETHR_HAVE_DW_NATMC_READ_ACQB +# undef ETHR_HAVE_SU_DW_NATMC_READ_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_ACQB +# define ETHR_HAVE_SU_DW_NATMC_READ_ACQB 1 +# endif +# undef ETHR_HAVE_DW_NATMC_READ_RELB +# undef ETHR_HAVE_SU_DW_NATMC_READ_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_RELB +# define ETHR_HAVE_SU_DW_NATMC_READ_RELB 1 +# endif +# undef ETHR_HAVE_DW_NATMC_READ_MB +# undef ETHR_HAVE_SU_DW_NATMC_READ_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_MB +# define ETHR_HAVE_SU_DW_NATMC_READ_MB 1 +# endif +# undef ETHR_HAVE_DW_NATMC_INIT +# undef ETHR_HAVE_SU_DW_NATMC_INIT +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT +# define ETHR_HAVE_SU_DW_NATMC_INIT 1 +# endif +# undef ETHR_HAVE_DW_NATMC_INIT_RB +# undef ETHR_HAVE_SU_DW_NATMC_INIT_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_RB +# define ETHR_HAVE_SU_DW_NATMC_INIT_RB 1 +# endif +# undef ETHR_HAVE_DW_NATMC_INIT_WB +# undef ETHR_HAVE_SU_DW_NATMC_INIT_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_WB +# define ETHR_HAVE_SU_DW_NATMC_INIT_WB 1 +# endif +# undef ETHR_HAVE_DW_NATMC_INIT_ACQB +# undef ETHR_HAVE_SU_DW_NATMC_INIT_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_ACQB +# define ETHR_HAVE_SU_DW_NATMC_INIT_ACQB 1 +# endif +# undef ETHR_HAVE_DW_NATMC_INIT_RELB +# undef ETHR_HAVE_SU_DW_NATMC_INIT_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_RELB +# define ETHR_HAVE_SU_DW_NATMC_INIT_RELB 1 +# endif +# undef ETHR_HAVE_DW_NATMC_INIT_MB +# undef ETHR_HAVE_SU_DW_NATMC_INIT_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_MB +# define ETHR_HAVE_SU_DW_NATMC_INIT_MB 1 +# endif +#else +# error "Invalid native atomic size" +#endif + -#define ETHR_ATOMIC_PTR2LCK__(PTR) \ -(ðr_atomic_protection__[((((ethr_uint_t) (PTR)) >> ETHR_ATOMIC_ADDR_SHIFT) \ - & ((1 << ETHR_ATOMIC_ADDR_BITS) - 1))].u.lck) +#if defined(ETHR_HAVE_NATIVE_DW_ATOMIC) +#if (!defined(ETHR_HAVE_DW_NATMC_CMPXCHG) \ + && !defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB) \ + && !defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB) \ + && !defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB) \ + && !defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB) \ + && !defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB)) +# error "No native cmpxchg() op available" +#endif + + +/* + * Read op used together with cmpxchg() fallback when no native op present. + */ +#if defined(ETHR_HAVE_DW_NATMC_READ) +#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \ + ETHR_DW_NATMC_FUNC__(read)(VAR, VAL) +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ) +#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \ + VAL.dw_sint = ETHR_SU_DW_NATMC_FUNC__(read)(VAR) +#elif defined(ETHR_HAVE_DW_NATMC_READ_RB) +#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \ + ETHR_DW_NATMC_FUNC__(read_rb)(VAR, VAL) +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RB) +#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \ + VAL.dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_rb)(VAR) +#elif defined(ETHR_HAVE_DW_NATMC_READ_WB) +#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \ + ETHR_DW_NATMC_FUNC__(read_wb)(VAR, VAL) +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_WB) +#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \ + VAL.dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_wb)(VAR) +#elif defined(ETHR_HAVE_DW_NATMC_READ_ACQB) +#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \ + ETHR_DW_NATMC_FUNC__(read_acqb)(VAR, VAL) +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_ACQB) +#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \ + VAL.dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_acqb)(VAR) +#elif defined(ETHR_HAVE_DW_NATMC_READ_RELB) +#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \ + ETHR_DW_NATMC_FUNC__(read_relb)(VAR, VAL) +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RELB) +#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \ + VAL.dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_relb)(VAR) +#elif defined(ETHR_HAVE_DW_NATMC_READ_MB) +#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \ + ETHR_DW_NATMC_FUNC__(read_mb)(VAR, VAL) +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_MB) +#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \ + VAL.dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_mb)(VAR) +#else +/* + * We have no native read() op; guess zero and then use the + * the atomics actual value returned from cmpxchg(). + */ +#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \ +do { \ + VAL.sint[0] = (ethr_sint_t) 0; \ + VAL.sint[1] = (ethr_sint_t) 0; \ +} while (0) +#endif -#define ETHR_ATOMIC_OP_FALLBACK_IMPL__(AP, EXPS) \ -do { \ - ethr_spinlock_t *slp__ = ETHR_ATOMIC_PTR2LCK__((AP)); \ - ethr_spin_lock(slp__); \ - { EXPS; } \ - ethr_spin_unlock(slp__); \ +/* + * Native cmpxchg() fallback used when no native op present. + */ +#define ETHR_DW_NATMC_CMPXCHG_FALLBACK__(CMPXCHG, VAR, AVAL, OPS) \ +do { \ + int res__; \ + ethr_dw_sint_t AVAL, exp_act__; \ + ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, exp_act__); \ + do { \ + AVAL.sint[0] = exp_act__.sint[0]; \ + AVAL.sint[1] = exp_act__.sint[1]; \ + { OPS; } \ + res__ = CMPXCHG(VAR, AVAL.sint, exp_act__.sint); \ + } while (__builtin_expect(res__ == 0, 0)); \ } while (0) + +#elif defined(ETHR_HAVE_NATIVE_SU_DW_ATOMIC) + +#if (!defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG) \ + && !defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB) \ + && !defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB) \ + && !defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB) \ + && !defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB) \ + && !defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB)) +# error "No native cmpxchg() op available" +#endif + + +/* + * Read op used together with cmpxchg() fallback when no native op present. + */ +#if defined(ETHR_HAVE_SU_DW_NATMC_READ) +#define ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \ + ETHR_SU_DW_NATMC_FUNC__(read)(VAR) +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RB) +#define ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \ + ETHR_SU_DW_NATMC_FUNC__(read_rb)(VAR) +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_WB) +#define ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \ + ETHR_SU_DW_NATMC_FUNC__(read_wb)(VAR) +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_ACQB) +#define ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \ + ETHR_SU_DW_NATMC_FUNC__(read_acqb)(VAR) +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RELB) +#define ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \ + ETHR_SU_DW_NATMC_FUNC__(read_relb)(VAR) +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_MB) +#define ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \ + ETHR_SU_DW_NATMC_FUNC__(read_mb)(VAR) +#else +/* + * We have no native read() op; guess zero and then use the + * the atomics actual value returned from cmpxchg(). + */ +#define ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \ + ((ETHR_SU_DW_NAINT_T__) 0) +#endif + +/* + * Native cmpxchg() fallback used when no native op present. + */ +#define ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(CMPXCHG, VAR, AVAL, OPS) \ +do { \ + ETHR_SU_DW_NAINT_T__ AVAL; \ + ETHR_SU_DW_NAINT_T__ new__, act__, exp__; \ + act__ = ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR); \ + do { \ + exp__ = act__; \ + AVAL = (ETHR_SU_DW_NAINT_T__) act__; \ + { OPS; } \ + new__ = (ETHR_SU_DW_NAINT_T__) AVAL; \ + act__ = CMPXCHG(VAR, new__, exp__); \ + } while (__builtin_expect(act__ != exp__, 0)); \ +} while (0) + + +#else +# error "?!?" +#endif + + + +/* --- addr() --- */ + +static ETHR_INLINE ethr_sint_t *ETHR_DW_ATMC_FUNC__(addr)(ethr_dw_atomic_t *var) +{ +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) { +#endif + + return (ethr_sint_t *) ETHR_DW_NATMC_ADDR_FUNC__((&var->native)); + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + } else { return ETHR_DW_ATOMIC_FUNC__(addr)(var); } +#endif + +} + + +/* --- cmpxchg() --- */ + + +static ETHR_INLINE int ETHR_DW_ATMC_FUNC__(cmpxchg)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val) +{ + int res; + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) { +#endif + +#if defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG) + ETHR_SU_DW_NAINT_T__ act; + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->dw_sint, old_val->dw_sint); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB) + ETHR_SU_DW_NAINT_T__ act; + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->dw_sint, old_val->dw_sint); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB) + ETHR_SU_DW_NAINT_T__ act; + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->dw_sint, old_val->dw_sint); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB) + ETHR_SU_DW_NAINT_T__ act; + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->dw_sint, old_val->dw_sint); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB) + ETHR_SU_DW_NAINT_T__ act; + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->dw_sint, old_val->dw_sint); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB) + ETHR_SU_DW_NAINT_T__ act; + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->dw_sint, old_val->dw_sint); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG) + res = ETHR_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->sint, old_val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB) + res = ETHR_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->sint, old_val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB) + res = ETHR_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->sint, old_val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB) + res = ETHR_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->sint, old_val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB) + res = ETHR_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->sint, old_val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB) + res = ETHR_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->sint, old_val->sint); +#else +#error "Missing implementation of ethr_dw_atomic_cmpxchg()!" +#endif + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + } else { res = ETHR_DW_ATOMIC_FUNC__(cmpxchg)(var, val, old_val); } +#endif + + return res; +} + +static ETHR_INLINE int ETHR_DW_ATMC_FUNC__(cmpxchg_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val) +{ + int res; + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) { +#endif + +#if defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB) + ETHR_SU_DW_NAINT_T__ act; + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->dw_sint, old_val->dw_sint); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG) + ETHR_SU_DW_NAINT_T__ act; + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->dw_sint, old_val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB) + ETHR_SU_DW_NAINT_T__ act; + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->dw_sint, old_val->dw_sint); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB) + ETHR_SU_DW_NAINT_T__ act; + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->dw_sint, old_val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB) + ETHR_SU_DW_NAINT_T__ act; + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->dw_sint, old_val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB) + ETHR_SU_DW_NAINT_T__ act; + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->dw_sint, old_val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB) + res = ETHR_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->sint, old_val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG) + res = ETHR_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->sint, old_val->sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB) + res = ETHR_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->sint, old_val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB) + res = ETHR_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->sint, old_val->sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB) + res = ETHR_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->sint, old_val->sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB) + res = ETHR_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->sint, old_val->sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#else +#error "Missing implementation of ethr_dw_atomic_cmpxchg_rb()!" +#endif + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + } else { res = ETHR_DW_ATOMIC_FUNC__(cmpxchg_rb)(var, val, old_val); } +#endif + + return res; +} + +static ETHR_INLINE int ETHR_DW_ATMC_FUNC__(cmpxchg_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val) +{ + int res; + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) { +#endif + +#if defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB) + ETHR_SU_DW_NAINT_T__ act; + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->dw_sint, old_val->dw_sint); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG) + ETHR_SU_DW_NAINT_T__ act; + ETHR_MEMBAR(ETHR_StoreStore); + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->dw_sint, old_val->dw_sint); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB) + ETHR_SU_DW_NAINT_T__ act; + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->dw_sint, old_val->dw_sint); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB) + ETHR_SU_DW_NAINT_T__ act; + ETHR_MEMBAR(ETHR_StoreStore); + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->dw_sint, old_val->dw_sint); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB) + ETHR_SU_DW_NAINT_T__ act; + ETHR_MEMBAR(ETHR_StoreStore); + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->dw_sint, old_val->dw_sint); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB) + ETHR_SU_DW_NAINT_T__ act; + ETHR_MEMBAR(ETHR_StoreStore); + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->dw_sint, old_val->dw_sint); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB) + res = ETHR_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->sint, old_val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG) + ETHR_MEMBAR(ETHR_StoreStore); + res = ETHR_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->sint, old_val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB) + res = ETHR_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->sint, old_val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_StoreStore); + res = ETHR_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->sint, old_val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + res = ETHR_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->sint, old_val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + res = ETHR_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->sint, old_val->sint); +#else +#error "Missing implementation of ethr_dw_atomic_cmpxchg_wb()!" +#endif + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + } else { res = ETHR_DW_ATOMIC_FUNC__(cmpxchg_wb)(var, val, old_val); } +#endif + + return res; +} + +static ETHR_INLINE int ETHR_DW_ATMC_FUNC__(cmpxchg_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val) +{ + int res; + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) { +#endif + +#if defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB) + ETHR_SU_DW_NAINT_T__ act; + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->dw_sint, old_val->dw_sint); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB) + ETHR_SU_DW_NAINT_T__ act; + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->dw_sint, old_val->dw_sint); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG) + ETHR_SU_DW_NAINT_T__ act; + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->dw_sint, old_val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB) + ETHR_SU_DW_NAINT_T__ act; + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->dw_sint, old_val->dw_sint); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB) + ETHR_SU_DW_NAINT_T__ act; + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->dw_sint, old_val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB) + ETHR_SU_DW_NAINT_T__ act; + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->dw_sint, old_val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB) + res = ETHR_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->sint, old_val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB) + res = ETHR_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->sint, old_val->sint); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG) + res = ETHR_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->sint, old_val->sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB) + res = ETHR_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->sint, old_val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB) + res = ETHR_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->sint, old_val->sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB) + res = ETHR_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->sint, old_val->sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else +#error "Missing implementation of ethr_dw_atomic_cmpxchg_acqb()!" +#endif + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + } else { res = ETHR_DW_ATOMIC_FUNC__(cmpxchg_acqb)(var, val, old_val); } +#endif + + return res; +} + +static ETHR_INLINE int ETHR_DW_ATMC_FUNC__(cmpxchg_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val) +{ + int res; + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) { +#endif + +#if defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB) + ETHR_SU_DW_NAINT_T__ act; + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->dw_sint, old_val->dw_sint); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB) + ETHR_SU_DW_NAINT_T__ act; + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->dw_sint, old_val->dw_sint); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG) + ETHR_SU_DW_NAINT_T__ act; + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->dw_sint, old_val->dw_sint); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB) + ETHR_SU_DW_NAINT_T__ act; + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->dw_sint, old_val->dw_sint); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB) + ETHR_SU_DW_NAINT_T__ act; + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->dw_sint, old_val->dw_sint); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB) + ETHR_SU_DW_NAINT_T__ act; + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->dw_sint, old_val->dw_sint); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB) + res = ETHR_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->sint, old_val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = ETHR_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->sint, old_val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = ETHR_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->sint, old_val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB) + res = ETHR_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->sint, old_val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = ETHR_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->sint, old_val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = ETHR_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->sint, old_val->sint); +#else +#error "Missing implementation of ethr_dw_atomic_cmpxchg_relb()!" +#endif + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + } else { res = ETHR_DW_ATOMIC_FUNC__(cmpxchg_relb)(var, val, old_val); } +#endif + + return res; +} + +static ETHR_INLINE int ETHR_DW_ATMC_FUNC__(cmpxchg_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val) +{ + int res; + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) { +#endif + +#if defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB) + ETHR_SU_DW_NAINT_T__ act; + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->dw_sint, old_val->dw_sint); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB) + ETHR_SU_DW_NAINT_T__ act; + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->dw_sint, old_val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB) + ETHR_SU_DW_NAINT_T__ act; + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->dw_sint, old_val->dw_sint); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB) + ETHR_SU_DW_NAINT_T__ act; + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->dw_sint, old_val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB) + ETHR_SU_DW_NAINT_T__ act; + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->dw_sint, old_val->dw_sint); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG) + ETHR_SU_DW_NAINT_T__ act; + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->dw_sint, old_val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + res = (act == old_val->dw_sint); + old_val->dw_sint = act; +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB) + res = ETHR_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->sint, old_val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB) + res = ETHR_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->sint, old_val->sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = ETHR_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->sint, old_val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = ETHR_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->sint, old_val->sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = ETHR_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->sint, old_val->sint); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = ETHR_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->sint, old_val->sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else +#error "Missing implementation of ethr_dw_atomic_cmpxchg_mb()!" +#endif + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + } else { res = ETHR_DW_ATOMIC_FUNC__(cmpxchg_mb)(var, val, old_val); } +#endif + + return res; +} + + +/* --- set() --- */ + + +static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(set)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) { +#endif + +#if defined(ETHR_HAVE_SU_DW_NATMC_SET) + ETHR_SU_DW_NATMC_FUNC__(set)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RB) + ETHR_SU_DW_NATMC_FUNC__(set_rb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_WB) + ETHR_SU_DW_NATMC_FUNC__(set_wb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_ACQB) + ETHR_SU_DW_NATMC_FUNC__(set_acqb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RELB) + ETHR_SU_DW_NATMC_FUNC__(set_relb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_MB) + ETHR_SU_DW_NATMC_FUNC__(set_mb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_DW_NATMC_SET) + ETHR_DW_NATMC_FUNC__(set)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_SET_RB) + ETHR_DW_NATMC_FUNC__(set_rb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_SET_WB) + ETHR_DW_NATMC_FUNC__(set_wb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_SET_ACQB) + ETHR_DW_NATMC_FUNC__(set_acqb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_SET_RELB) + ETHR_DW_NATMC_FUNC__(set_relb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_SET_MB) + ETHR_DW_NATMC_FUNC__(set_mb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG) + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval = val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB) + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval = val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB) + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval = val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB) + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval = val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB) + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval = val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB) + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval = val->dw_sint); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG) + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB) + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB) + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB) + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB) + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB) + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); +#else +#error "Missing implementation of ethr_dw_atomic_set()!" +#endif + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + } else { ETHR_DW_ATOMIC_FUNC__(set)(var, val); } +#endif + +} + +static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(set_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) { +#endif + +#if defined(ETHR_HAVE_SU_DW_NATMC_SET_RB) + ETHR_SU_DW_NATMC_FUNC__(set_rb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET) + ETHR_SU_DW_NATMC_FUNC__(set)(&var->native, val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_MB) + ETHR_SU_DW_NATMC_FUNC__(set_mb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_WB) + ETHR_SU_DW_NATMC_FUNC__(set_wb)(&var->native, val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_ACQB) + ETHR_SU_DW_NATMC_FUNC__(set_acqb)(&var->native, val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RELB) + ETHR_SU_DW_NATMC_FUNC__(set_relb)(&var->native, val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_DW_NATMC_SET_RB) + ETHR_DW_NATMC_FUNC__(set_rb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_SET) + ETHR_DW_NATMC_FUNC__(set)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_DW_NATMC_SET_MB) + ETHR_DW_NATMC_FUNC__(set_mb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_SET_WB) + ETHR_DW_NATMC_FUNC__(set_wb)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_DW_NATMC_SET_ACQB) + ETHR_DW_NATMC_FUNC__(set_acqb)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_DW_NATMC_SET_RELB) + ETHR_DW_NATMC_FUNC__(set_relb)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB) + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval = val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG) + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval = val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB) + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval = val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB) + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval = val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB) + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval = val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB) + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval = val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB) + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG) + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB) + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB) + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB) + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB) + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); + ETHR_MEMBAR(ETHR_LoadLoad); +#else +#error "Missing implementation of ethr_dw_atomic_set_rb()!" +#endif + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + } else { ETHR_DW_ATOMIC_FUNC__(set_rb)(var, val); } +#endif + +} + +static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(set_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) { +#endif + +#if defined(ETHR_HAVE_SU_DW_NATMC_SET_WB) + ETHR_SU_DW_NATMC_FUNC__(set_wb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_SU_DW_NATMC_FUNC__(set)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_MB) + ETHR_SU_DW_NATMC_FUNC__(set_mb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_SU_DW_NATMC_FUNC__(set_rb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_SU_DW_NATMC_FUNC__(set_acqb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_SU_DW_NATMC_FUNC__(set_relb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_DW_NATMC_SET_WB) + ETHR_DW_NATMC_FUNC__(set_wb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_SET) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_DW_NATMC_FUNC__(set)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_SET_MB) + ETHR_DW_NATMC_FUNC__(set_mb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_SET_RB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_DW_NATMC_FUNC__(set_rb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_SET_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_DW_NATMC_FUNC__(set_acqb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_SET_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_DW_NATMC_FUNC__(set_relb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB) + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval = val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval = val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB) + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval = val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval = val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval = val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval = val->dw_sint); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB) + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB) + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); +#else +#error "Missing implementation of ethr_dw_atomic_set_wb()!" +#endif + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + } else { ETHR_DW_ATOMIC_FUNC__(set_wb)(var, val); } +#endif + +} + +static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(set_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) { +#endif + +#if defined(ETHR_HAVE_SU_DW_NATMC_SET_ACQB) + ETHR_SU_DW_NATMC_FUNC__(set_acqb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RB) + ETHR_SU_DW_NATMC_FUNC__(set_rb)(&var->native, val->dw_sint); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET) + ETHR_SU_DW_NATMC_FUNC__(set)(&var->native, val->dw_sint); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_MB) + ETHR_SU_DW_NATMC_FUNC__(set_mb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_WB) + ETHR_SU_DW_NATMC_FUNC__(set_wb)(&var->native, val->dw_sint); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RELB) + ETHR_SU_DW_NATMC_FUNC__(set_relb)(&var->native, val->dw_sint); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_SET_ACQB) + ETHR_DW_NATMC_FUNC__(set_acqb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_SET_RB) + ETHR_DW_NATMC_FUNC__(set_rb)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_SET) + ETHR_DW_NATMC_FUNC__(set)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_SET_MB) + ETHR_DW_NATMC_FUNC__(set_mb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_SET_WB) + ETHR_DW_NATMC_FUNC__(set_wb)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_SET_RELB) + ETHR_DW_NATMC_FUNC__(set_relb)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB) + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval = val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB) + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval = val->dw_sint); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG) + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval = val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB) + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval = val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB) + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval = val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB) + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval = val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB) + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB) + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG) + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB) + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB) + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB) + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else +#error "Missing implementation of ethr_dw_atomic_set_acqb()!" +#endif + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + } else { ETHR_DW_ATOMIC_FUNC__(set_acqb)(var, val); } +#endif + +} + +static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(set_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) { +#endif + +#if defined(ETHR_HAVE_SU_DW_NATMC_SET_RELB) + ETHR_SU_DW_NATMC_FUNC__(set_relb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_WB) + ETHR_MEMBAR(ETHR_LoadStore); + ETHR_SU_DW_NATMC_FUNC__(set_wb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET) + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_SU_DW_NATMC_FUNC__(set)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_MB) + ETHR_SU_DW_NATMC_FUNC__(set_mb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RB) + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_SU_DW_NATMC_FUNC__(set_rb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_ACQB) + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_SU_DW_NATMC_FUNC__(set_acqb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_DW_NATMC_SET_RELB) + ETHR_DW_NATMC_FUNC__(set_relb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_SET_WB) + ETHR_MEMBAR(ETHR_LoadStore); + ETHR_DW_NATMC_FUNC__(set_wb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_SET) + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_DW_NATMC_FUNC__(set)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_SET_MB) + ETHR_DW_NATMC_FUNC__(set_mb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_SET_RB) + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_DW_NATMC_FUNC__(set_rb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_SET_ACQB) + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_DW_NATMC_FUNC__(set_acqb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB) + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval = val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval = val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval = val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB) + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval = val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval = val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval = val->dw_sint); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB) + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB) + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); +#else +#error "Missing implementation of ethr_dw_atomic_set_relb()!" +#endif + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + } else { ETHR_DW_ATOMIC_FUNC__(set_relb)(var, val); } +#endif + +} + +static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(set_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) { +#endif + +#if defined(ETHR_HAVE_SU_DW_NATMC_SET_MB) + ETHR_SU_DW_NATMC_FUNC__(set_mb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RELB) + ETHR_SU_DW_NATMC_FUNC__(set_relb)(&var->native, val->dw_sint); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_ACQB) + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_SU_DW_NATMC_FUNC__(set_acqb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_WB) + ETHR_MEMBAR(ETHR_LoadStore); + ETHR_SU_DW_NATMC_FUNC__(set_wb)(&var->native, val->dw_sint); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RB) + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_SU_DW_NATMC_FUNC__(set_rb)(&var->native, val->dw_sint); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_SU_DW_NATMC_SET) + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_SU_DW_NATMC_FUNC__(set)(&var->native, val->dw_sint); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_SET_MB) + ETHR_DW_NATMC_FUNC__(set_mb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_SET_RELB) + ETHR_DW_NATMC_FUNC__(set_relb)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_SET_ACQB) + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_DW_NATMC_FUNC__(set_acqb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_SET_WB) + ETHR_MEMBAR(ETHR_LoadStore); + ETHR_DW_NATMC_FUNC__(set_wb)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_SET_RB) + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_DW_NATMC_FUNC__(set_rb)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_SET) + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_DW_NATMC_FUNC__(set)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB) + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval = val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB) + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval = val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval = val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval = val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval = val->dw_sint); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval = val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB) + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB) + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else +#error "Missing implementation of ethr_dw_atomic_set_mb()!" +#endif + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + } else { ETHR_DW_ATOMIC_FUNC__(set_mb)(var, val); } +#endif + +} + + +/* --- read() --- */ + + +static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(read)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) { +#endif + +#if defined(ETHR_HAVE_SU_DW_NATMC_READ) + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read)(&var->native); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RB) + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_rb)(&var->native); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_WB) + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_wb)(&var->native); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_ACQB) + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_acqb)(&var->native); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RELB) + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_relb)(&var->native); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_MB) + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_mb)(&var->native); +#elif defined(ETHR_HAVE_DW_NATMC_READ) + ETHR_DW_NATMC_FUNC__(read)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_READ_RB) + ETHR_DW_NATMC_FUNC__(read_rb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_READ_WB) + ETHR_DW_NATMC_FUNC__(read_wb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_READ_ACQB) + ETHR_DW_NATMC_FUNC__(read_acqb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_READ_RELB) + ETHR_DW_NATMC_FUNC__(read_relb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_READ_MB) + ETHR_DW_NATMC_FUNC__(read_mb)(&var->native, val->sint); +#else + ethr_dw_sint_t tmp; + tmp.sint[0] = ETHR_UNUSUAL_SINT_VAL__; + tmp.sint[1] = ETHR_UNUSUAL_SINT_VAL__; + val->sint[0] = ETHR_UNUSUAL_SINT_VAL__; + val->sint[1] = ETHR_UNUSUAL_SINT_VAL__; + (void) ETHR_DW_ATMC_FUNC__(cmpxchg)(var, &tmp, val); +#endif + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + } else { ETHR_DW_ATOMIC_FUNC__(read)(var, val); } +#endif + +} + +static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(read_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) { +#endif + +#if defined(ETHR_HAVE_SU_DW_NATMC_READ_RB) + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_rb)(&var->native); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ) + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read)(&var->native); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_MB) + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_mb)(&var->native); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_WB) + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_wb)(&var->native); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_ACQB) + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_acqb)(&var->native); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RELB) + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_relb)(&var->native); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_DW_NATMC_READ_RB) + ETHR_DW_NATMC_FUNC__(read_rb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_READ) + ETHR_DW_NATMC_FUNC__(read)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_DW_NATMC_READ_MB) + ETHR_DW_NATMC_FUNC__(read_mb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_READ_WB) + ETHR_DW_NATMC_FUNC__(read_wb)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_DW_NATMC_READ_ACQB) + ETHR_DW_NATMC_FUNC__(read_acqb)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_DW_NATMC_READ_RELB) + ETHR_DW_NATMC_FUNC__(read_relb)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + ethr_dw_sint_t tmp; + tmp.sint[0] = ETHR_UNUSUAL_SINT_VAL__; + tmp.sint[1] = ETHR_UNUSUAL_SINT_VAL__; + val->sint[0] = ETHR_UNUSUAL_SINT_VAL__; + val->sint[1] = ETHR_UNUSUAL_SINT_VAL__; + (void) ETHR_DW_ATMC_FUNC__(cmpxchg_rb)(var, &tmp, val); +#endif + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + } else { ETHR_DW_ATOMIC_FUNC__(read_rb)(var, val); } +#endif + +} + +static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(read_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) { +#endif + +#if defined(ETHR_HAVE_SU_DW_NATMC_READ_WB) + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_wb)(&var->native); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ) + ETHR_MEMBAR(ETHR_StoreStore); + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read)(&var->native); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_MB) + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_mb)(&var->native); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RB) + ETHR_MEMBAR(ETHR_StoreStore); + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_rb)(&var->native); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_acqb)(&var->native); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_relb)(&var->native); +#elif defined(ETHR_HAVE_DW_NATMC_READ_WB) + ETHR_DW_NATMC_FUNC__(read_wb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_READ) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_DW_NATMC_FUNC__(read)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_READ_MB) + ETHR_DW_NATMC_FUNC__(read_mb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_READ_RB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_DW_NATMC_FUNC__(read_rb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_READ_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_DW_NATMC_FUNC__(read_acqb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_READ_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_DW_NATMC_FUNC__(read_relb)(&var->native, val->sint); +#else + ethr_dw_sint_t tmp; + tmp.sint[0] = ETHR_UNUSUAL_SINT_VAL__; + tmp.sint[1] = ETHR_UNUSUAL_SINT_VAL__; + val->sint[0] = ETHR_UNUSUAL_SINT_VAL__; + val->sint[1] = ETHR_UNUSUAL_SINT_VAL__; + (void) ETHR_DW_ATMC_FUNC__(cmpxchg_wb)(var, &tmp, val); +#endif + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + } else { ETHR_DW_ATOMIC_FUNC__(read_wb)(var, val); } +#endif + +} + +static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(read_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) { +#endif + +#if defined(ETHR_HAVE_SU_DW_NATMC_READ_ACQB) + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_acqb)(&var->native); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RB) + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_rb)(&var->native); + ETHR_MEMBAR(ETHR_LoadStore); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ) + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read)(&var->native); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_MB) + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_mb)(&var->native); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_WB) + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_wb)(&var->native); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RELB) + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_relb)(&var->native); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#elif defined(ETHR_HAVE_DW_NATMC_READ_ACQB) + ETHR_DW_NATMC_FUNC__(read_acqb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_READ_RB) + ETHR_DW_NATMC_FUNC__(read_rb)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_LoadStore); +#elif defined(ETHR_HAVE_DW_NATMC_READ) + ETHR_DW_NATMC_FUNC__(read)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#elif defined(ETHR_HAVE_DW_NATMC_READ_MB) + ETHR_DW_NATMC_FUNC__(read_mb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_READ_WB) + ETHR_DW_NATMC_FUNC__(read_wb)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#elif defined(ETHR_HAVE_DW_NATMC_READ_RELB) + ETHR_DW_NATMC_FUNC__(read_relb)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#else + ethr_dw_sint_t tmp; + tmp.sint[0] = ETHR_UNUSUAL_SINT_VAL__; + tmp.sint[1] = ETHR_UNUSUAL_SINT_VAL__; + val->sint[0] = ETHR_UNUSUAL_SINT_VAL__; + val->sint[1] = ETHR_UNUSUAL_SINT_VAL__; + (void) ETHR_DW_ATMC_FUNC__(cmpxchg_acqb)(var, &tmp, val); +#endif + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + } else { ETHR_DW_ATOMIC_FUNC__(read_acqb)(var, val); } +#endif + +} + +static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(read_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) { +#endif + +#if defined(ETHR_HAVE_SU_DW_NATMC_READ_RELB) + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_relb)(&var->native); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_wb)(&var->native); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read)(&var->native); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_MB) + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_mb)(&var->native); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_rb)(&var->native); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_acqb)(&var->native); +#elif defined(ETHR_HAVE_DW_NATMC_READ_RELB) + ETHR_DW_NATMC_FUNC__(read_relb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_READ_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_DW_NATMC_FUNC__(read_wb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_READ) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_DW_NATMC_FUNC__(read)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_READ_MB) + ETHR_DW_NATMC_FUNC__(read_mb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_READ_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_DW_NATMC_FUNC__(read_rb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_READ_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_DW_NATMC_FUNC__(read_acqb)(&var->native, val->sint); +#else + ethr_dw_sint_t tmp; + tmp.sint[0] = ETHR_UNUSUAL_SINT_VAL__; + tmp.sint[1] = ETHR_UNUSUAL_SINT_VAL__; + val->sint[0] = ETHR_UNUSUAL_SINT_VAL__; + val->sint[1] = ETHR_UNUSUAL_SINT_VAL__; + (void) ETHR_DW_ATMC_FUNC__(cmpxchg_relb)(var, &tmp, val); +#endif + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + } else { ETHR_DW_ATOMIC_FUNC__(read_relb)(var, val); } +#endif + +} + +static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(read_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) { +#endif + +#if defined(ETHR_HAVE_SU_DW_NATMC_READ_MB) + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_mb)(&var->native); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RELB) + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_relb)(&var->native); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_acqb)(&var->native); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_wb)(&var->native); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_rb)(&var->native); + ETHR_MEMBAR(ETHR_LoadStore); +#elif defined(ETHR_HAVE_SU_DW_NATMC_READ) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read)(&var->native); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#elif defined(ETHR_HAVE_DW_NATMC_READ_MB) + ETHR_DW_NATMC_FUNC__(read_mb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_READ_RELB) + ETHR_DW_NATMC_FUNC__(read_relb)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#elif defined(ETHR_HAVE_DW_NATMC_READ_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_DW_NATMC_FUNC__(read_acqb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_READ_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_DW_NATMC_FUNC__(read_wb)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#elif defined(ETHR_HAVE_DW_NATMC_READ_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_DW_NATMC_FUNC__(read_rb)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_LoadStore); +#elif defined(ETHR_HAVE_DW_NATMC_READ) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_DW_NATMC_FUNC__(read)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#else + ethr_dw_sint_t tmp; + tmp.sint[0] = ETHR_UNUSUAL_SINT_VAL__; + tmp.sint[1] = ETHR_UNUSUAL_SINT_VAL__; + val->sint[0] = ETHR_UNUSUAL_SINT_VAL__; + val->sint[1] = ETHR_UNUSUAL_SINT_VAL__; + (void) ETHR_DW_ATMC_FUNC__(cmpxchg_mb)(var, &tmp, val); +#endif + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + } else { ETHR_DW_ATOMIC_FUNC__(read_mb)(var, val); } +#endif + +} + + +/* --- init() --- */ + + +static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(init)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) { +#endif + +#if defined(ETHR_HAVE_SU_DW_NATMC_INIT) + ETHR_SU_DW_NATMC_FUNC__(init)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RB) + ETHR_SU_DW_NATMC_FUNC__(init_rb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_WB) + ETHR_SU_DW_NATMC_FUNC__(init_wb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_ACQB) + ETHR_SU_DW_NATMC_FUNC__(init_acqb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RELB) + ETHR_SU_DW_NATMC_FUNC__(init_relb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_MB) + ETHR_SU_DW_NATMC_FUNC__(init_mb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_DW_NATMC_INIT) + ETHR_DW_NATMC_FUNC__(init)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_RB) + ETHR_DW_NATMC_FUNC__(init_rb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_WB) + ETHR_DW_NATMC_FUNC__(init_wb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_ACQB) + ETHR_DW_NATMC_FUNC__(init_acqb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_RELB) + ETHR_DW_NATMC_FUNC__(init_relb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_MB) + ETHR_DW_NATMC_FUNC__(init_mb)(&var->native, val->sint); +#else + ETHR_DW_ATMC_FUNC__(set)(var, val); +#endif + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + } else { ETHR_DW_ATOMIC_FUNC__(init)(var, val); } +#endif + +} + +static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(init_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) { +#endif + +#if defined(ETHR_HAVE_SU_DW_NATMC_INIT_RB) + ETHR_SU_DW_NATMC_FUNC__(init_rb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT) + ETHR_SU_DW_NATMC_FUNC__(init)(&var->native, val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_MB) + ETHR_SU_DW_NATMC_FUNC__(init_mb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_WB) + ETHR_SU_DW_NATMC_FUNC__(init_wb)(&var->native, val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_ACQB) + ETHR_SU_DW_NATMC_FUNC__(init_acqb)(&var->native, val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RELB) + ETHR_SU_DW_NATMC_FUNC__(init_relb)(&var->native, val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_RB) + ETHR_DW_NATMC_FUNC__(init_rb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_INIT) + ETHR_DW_NATMC_FUNC__(init)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_MB) + ETHR_DW_NATMC_FUNC__(init_mb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_WB) + ETHR_DW_NATMC_FUNC__(init_wb)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_ACQB) + ETHR_DW_NATMC_FUNC__(init_acqb)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_RELB) + ETHR_DW_NATMC_FUNC__(init_relb)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + ETHR_DW_ATMC_FUNC__(set_rb)(var, val); +#endif + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + } else { ETHR_DW_ATOMIC_FUNC__(init_rb)(var, val); } +#endif + +} + +static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(init_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) { +#endif + +#if defined(ETHR_HAVE_SU_DW_NATMC_INIT_WB) + ETHR_SU_DW_NATMC_FUNC__(init_wb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_SU_DW_NATMC_FUNC__(init)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_MB) + ETHR_SU_DW_NATMC_FUNC__(init_mb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_SU_DW_NATMC_FUNC__(init_rb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_SU_DW_NATMC_FUNC__(init_acqb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_SU_DW_NATMC_FUNC__(init_relb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_WB) + ETHR_DW_NATMC_FUNC__(init_wb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_INIT) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_DW_NATMC_FUNC__(init)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_MB) + ETHR_DW_NATMC_FUNC__(init_mb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_RB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_DW_NATMC_FUNC__(init_rb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_DW_NATMC_FUNC__(init_acqb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_DW_NATMC_FUNC__(init_relb)(&var->native, val->sint); +#else + ETHR_DW_ATMC_FUNC__(set_wb)(var, val); +#endif + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + } else { ETHR_DW_ATOMIC_FUNC__(init_wb)(var, val); } +#endif + +} + +static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(init_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) { +#endif + +#if defined(ETHR_HAVE_SU_DW_NATMC_INIT_ACQB) + ETHR_SU_DW_NATMC_FUNC__(init_acqb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RB) + ETHR_SU_DW_NATMC_FUNC__(init_rb)(&var->native, val->dw_sint); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT) + ETHR_SU_DW_NATMC_FUNC__(init)(&var->native, val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_MB) + ETHR_SU_DW_NATMC_FUNC__(init_mb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_WB) + ETHR_SU_DW_NATMC_FUNC__(init_wb)(&var->native, val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RELB) + ETHR_SU_DW_NATMC_FUNC__(init_relb)(&var->native, val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_ACQB) + ETHR_DW_NATMC_FUNC__(init_acqb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_RB) + ETHR_DW_NATMC_FUNC__(init_rb)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_INIT) + ETHR_DW_NATMC_FUNC__(init)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_MB) + ETHR_DW_NATMC_FUNC__(init_mb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_WB) + ETHR_DW_NATMC_FUNC__(init_wb)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_RELB) + ETHR_DW_NATMC_FUNC__(init_relb)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_DW_ATMC_FUNC__(set_acqb)(var, val); +#endif + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + } else { ETHR_DW_ATOMIC_FUNC__(init_acqb)(var, val); } +#endif + +} + +static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(init_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) { +#endif + +#if defined(ETHR_HAVE_SU_DW_NATMC_INIT_RELB) + ETHR_SU_DW_NATMC_FUNC__(init_relb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad); + ETHR_SU_DW_NATMC_FUNC__(init_wb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_SU_DW_NATMC_FUNC__(init)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_MB) + ETHR_SU_DW_NATMC_FUNC__(init_mb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_SU_DW_NATMC_FUNC__(init_rb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_SU_DW_NATMC_FUNC__(init_acqb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_RELB) + ETHR_DW_NATMC_FUNC__(init_relb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad); + ETHR_DW_NATMC_FUNC__(init_wb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_INIT) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_DW_NATMC_FUNC__(init)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_MB) + ETHR_DW_NATMC_FUNC__(init_mb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_DW_NATMC_FUNC__(init_rb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_DW_NATMC_FUNC__(init_acqb)(&var->native, val->sint); +#else + ETHR_DW_ATMC_FUNC__(set_relb)(var, val); +#endif + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + } else { ETHR_DW_ATOMIC_FUNC__(init_relb)(var, val); } +#endif + +} + +static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(init_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) { +#endif + +#if defined(ETHR_HAVE_SU_DW_NATMC_INIT_MB) + ETHR_SU_DW_NATMC_FUNC__(init_mb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RELB) + ETHR_SU_DW_NATMC_FUNC__(init_relb)(&var->native, val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_SU_DW_NATMC_FUNC__(init_acqb)(&var->native, val->dw_sint); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad); + ETHR_SU_DW_NATMC_FUNC__(init_wb)(&var->native, val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_SU_DW_NATMC_FUNC__(init_rb)(&var->native, val->dw_sint); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_SU_DW_NATMC_FUNC__(init)(&var->native, val->dw_sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_MB) + ETHR_DW_NATMC_FUNC__(init_mb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_RELB) + ETHR_DW_NATMC_FUNC__(init_relb)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_DW_NATMC_FUNC__(init_acqb)(&var->native, val->sint); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad); + ETHR_DW_NATMC_FUNC__(init_wb)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_INIT_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_DW_NATMC_FUNC__(init_rb)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_DW_NATMC_INIT) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_DW_NATMC_FUNC__(init)(&var->native, val->sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_DW_ATMC_FUNC__(set_mb)(var, val); +#endif + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + } else { ETHR_DW_ATOMIC_FUNC__(init_mb)(var, val); } +#endif + +} + +#endif /* ETHR_DW_ATMC_INLINE__ */ + + +/* ---------- Word size atomic implementation ---------- */ + + +#ifdef ETHR_NEED_ATMC_PROTOTYPES__ +ethr_sint_t *ethr_atomic_addr(ethr_atomic_t *var); +ethr_sint_t ethr_atomic_cmpxchg(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val); +ethr_sint_t ethr_atomic_cmpxchg_rb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val); +ethr_sint_t ethr_atomic_cmpxchg_wb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val); +ethr_sint_t ethr_atomic_cmpxchg_acqb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val); +ethr_sint_t ethr_atomic_cmpxchg_relb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val); +ethr_sint_t ethr_atomic_cmpxchg_mb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val); +ethr_sint_t ethr_atomic_xchg(ethr_atomic_t *var, ethr_sint_t val); +ethr_sint_t ethr_atomic_xchg_rb(ethr_atomic_t *var, ethr_sint_t val); +ethr_sint_t ethr_atomic_xchg_wb(ethr_atomic_t *var, ethr_sint_t val); +ethr_sint_t ethr_atomic_xchg_acqb(ethr_atomic_t *var, ethr_sint_t val); +ethr_sint_t ethr_atomic_xchg_relb(ethr_atomic_t *var, ethr_sint_t val); +ethr_sint_t ethr_atomic_xchg_mb(ethr_atomic_t *var, ethr_sint_t val); +void ethr_atomic_set(ethr_atomic_t *var, ethr_sint_t val); +void ethr_atomic_set_rb(ethr_atomic_t *var, ethr_sint_t val); +void ethr_atomic_set_wb(ethr_atomic_t *var, ethr_sint_t val); +void ethr_atomic_set_acqb(ethr_atomic_t *var, ethr_sint_t val); +void ethr_atomic_set_relb(ethr_atomic_t *var, ethr_sint_t val); +void ethr_atomic_set_mb(ethr_atomic_t *var, ethr_sint_t val); +void ethr_atomic_init(ethr_atomic_t *var, ethr_sint_t val); +void ethr_atomic_init_rb(ethr_atomic_t *var, ethr_sint_t val); +void ethr_atomic_init_wb(ethr_atomic_t *var, ethr_sint_t val); +void ethr_atomic_init_acqb(ethr_atomic_t *var, ethr_sint_t val); +void ethr_atomic_init_relb(ethr_atomic_t *var, ethr_sint_t val); +void ethr_atomic_init_mb(ethr_atomic_t *var, ethr_sint_t val); +ethr_sint_t ethr_atomic_add_read(ethr_atomic_t *var, ethr_sint_t val); +ethr_sint_t ethr_atomic_add_read_rb(ethr_atomic_t *var, ethr_sint_t val); +ethr_sint_t ethr_atomic_add_read_wb(ethr_atomic_t *var, ethr_sint_t val); +ethr_sint_t ethr_atomic_add_read_acqb(ethr_atomic_t *var, ethr_sint_t val); +ethr_sint_t ethr_atomic_add_read_relb(ethr_atomic_t *var, ethr_sint_t val); +ethr_sint_t ethr_atomic_add_read_mb(ethr_atomic_t *var, ethr_sint_t val); +ethr_sint_t ethr_atomic_read(ethr_atomic_t *var); +ethr_sint_t ethr_atomic_read_rb(ethr_atomic_t *var); +ethr_sint_t ethr_atomic_read_wb(ethr_atomic_t *var); +ethr_sint_t ethr_atomic_read_acqb(ethr_atomic_t *var); +ethr_sint_t ethr_atomic_read_relb(ethr_atomic_t *var); +ethr_sint_t ethr_atomic_read_mb(ethr_atomic_t *var); +ethr_sint_t ethr_atomic_inc_read(ethr_atomic_t *var); +ethr_sint_t ethr_atomic_inc_read_rb(ethr_atomic_t *var); +ethr_sint_t ethr_atomic_inc_read_wb(ethr_atomic_t *var); +ethr_sint_t ethr_atomic_inc_read_acqb(ethr_atomic_t *var); +ethr_sint_t ethr_atomic_inc_read_relb(ethr_atomic_t *var); +ethr_sint_t ethr_atomic_inc_read_mb(ethr_atomic_t *var); +ethr_sint_t ethr_atomic_dec_read(ethr_atomic_t *var); +ethr_sint_t ethr_atomic_dec_read_rb(ethr_atomic_t *var); +ethr_sint_t ethr_atomic_dec_read_wb(ethr_atomic_t *var); +ethr_sint_t ethr_atomic_dec_read_acqb(ethr_atomic_t *var); +ethr_sint_t ethr_atomic_dec_read_relb(ethr_atomic_t *var); +ethr_sint_t ethr_atomic_dec_read_mb(ethr_atomic_t *var); +void ethr_atomic_add(ethr_atomic_t *var, ethr_sint_t val); +void ethr_atomic_add_rb(ethr_atomic_t *var, ethr_sint_t val); +void ethr_atomic_add_wb(ethr_atomic_t *var, ethr_sint_t val); +void ethr_atomic_add_acqb(ethr_atomic_t *var, ethr_sint_t val); +void ethr_atomic_add_relb(ethr_atomic_t *var, ethr_sint_t val); +void ethr_atomic_add_mb(ethr_atomic_t *var, ethr_sint_t val); +void ethr_atomic_inc(ethr_atomic_t *var); +void ethr_atomic_inc_rb(ethr_atomic_t *var); +void ethr_atomic_inc_wb(ethr_atomic_t *var); +void ethr_atomic_inc_acqb(ethr_atomic_t *var); +void ethr_atomic_inc_relb(ethr_atomic_t *var); +void ethr_atomic_inc_mb(ethr_atomic_t *var); +void ethr_atomic_dec(ethr_atomic_t *var); +void ethr_atomic_dec_rb(ethr_atomic_t *var); +void ethr_atomic_dec_wb(ethr_atomic_t *var); +void ethr_atomic_dec_acqb(ethr_atomic_t *var); +void ethr_atomic_dec_relb(ethr_atomic_t *var); +void ethr_atomic_dec_mb(ethr_atomic_t *var); +ethr_sint_t ethr_atomic_read_band(ethr_atomic_t *var, ethr_sint_t val); +ethr_sint_t ethr_atomic_read_band_rb(ethr_atomic_t *var, ethr_sint_t val); +ethr_sint_t ethr_atomic_read_band_wb(ethr_atomic_t *var, ethr_sint_t val); +ethr_sint_t ethr_atomic_read_band_acqb(ethr_atomic_t *var, ethr_sint_t val); +ethr_sint_t ethr_atomic_read_band_relb(ethr_atomic_t *var, ethr_sint_t val); +ethr_sint_t ethr_atomic_read_band_mb(ethr_atomic_t *var, ethr_sint_t val); +ethr_sint_t ethr_atomic_read_bor(ethr_atomic_t *var, ethr_sint_t val); +ethr_sint_t ethr_atomic_read_bor_rb(ethr_atomic_t *var, ethr_sint_t val); +ethr_sint_t ethr_atomic_read_bor_wb(ethr_atomic_t *var, ethr_sint_t val); +ethr_sint_t ethr_atomic_read_bor_acqb(ethr_atomic_t *var, ethr_sint_t val); +ethr_sint_t ethr_atomic_read_bor_relb(ethr_atomic_t *var, ethr_sint_t val); +ethr_sint_t ethr_atomic_read_bor_mb(ethr_atomic_t *var, ethr_sint_t val); +#endif /* ETHR_NEED_ATMC_PROTOTYPES__ */ + +#if (defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) \ + && (defined(ETHR_ATMC_INLINE__) || defined(ETHR_ATOMIC_IMPL__))) + +#if !defined(ETHR_NATMC_BITS__) +# error "Missing native atomic implementation" +#elif ETHR_NATMC_BITS__ == 64 +# undef ETHR_HAVE_NATMC_CMPXCHG +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG +# define ETHR_HAVE_NATMC_CMPXCHG 1 +# endif +# undef ETHR_HAVE_NATMC_CMPXCHG_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RB +# define ETHR_HAVE_NATMC_CMPXCHG_RB 1 +# endif +# undef ETHR_HAVE_NATMC_CMPXCHG_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_WB +# define ETHR_HAVE_NATMC_CMPXCHG_WB 1 +# endif +# undef ETHR_HAVE_NATMC_CMPXCHG_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_ACQB +# define ETHR_HAVE_NATMC_CMPXCHG_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC_CMPXCHG_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RELB +# define ETHR_HAVE_NATMC_CMPXCHG_RELB 1 +# endif +# undef ETHR_HAVE_NATMC_CMPXCHG_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_MB +# define ETHR_HAVE_NATMC_CMPXCHG_MB 1 +# endif +# undef ETHR_HAVE_NATMC_XCHG +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG +# define ETHR_HAVE_NATMC_XCHG 1 +# endif +# undef ETHR_HAVE_NATMC_XCHG_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_RB +# define ETHR_HAVE_NATMC_XCHG_RB 1 +# endif +# undef ETHR_HAVE_NATMC_XCHG_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_WB +# define ETHR_HAVE_NATMC_XCHG_WB 1 +# endif +# undef ETHR_HAVE_NATMC_XCHG_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_ACQB +# define ETHR_HAVE_NATMC_XCHG_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC_XCHG_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_RELB +# define ETHR_HAVE_NATMC_XCHG_RELB 1 +# endif +# undef ETHR_HAVE_NATMC_XCHG_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_MB +# define ETHR_HAVE_NATMC_XCHG_MB 1 +# endif +# undef ETHR_HAVE_NATMC_SET +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET +# define ETHR_HAVE_NATMC_SET 1 +# endif +# undef ETHR_HAVE_NATMC_SET_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RB +# define ETHR_HAVE_NATMC_SET_RB 1 +# endif +# undef ETHR_HAVE_NATMC_SET_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_WB +# define ETHR_HAVE_NATMC_SET_WB 1 +# endif +# undef ETHR_HAVE_NATMC_SET_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_ACQB +# define ETHR_HAVE_NATMC_SET_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC_SET_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RELB +# define ETHR_HAVE_NATMC_SET_RELB 1 +# endif +# undef ETHR_HAVE_NATMC_SET_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_MB +# define ETHR_HAVE_NATMC_SET_MB 1 +# endif +# undef ETHR_HAVE_NATMC_INIT +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT +# define ETHR_HAVE_NATMC_INIT 1 +# endif +# undef ETHR_HAVE_NATMC_INIT_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_RB +# define ETHR_HAVE_NATMC_INIT_RB 1 +# endif +# undef ETHR_HAVE_NATMC_INIT_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_WB +# define ETHR_HAVE_NATMC_INIT_WB 1 +# endif +# undef ETHR_HAVE_NATMC_INIT_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_ACQB +# define ETHR_HAVE_NATMC_INIT_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC_INIT_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_RELB +# define ETHR_HAVE_NATMC_INIT_RELB 1 +# endif +# undef ETHR_HAVE_NATMC_INIT_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_MB +# define ETHR_HAVE_NATMC_INIT_MB 1 +# endif +# undef ETHR_HAVE_NATMC_ADD_RETURN +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN +# define ETHR_HAVE_NATMC_ADD_RETURN 1 +# endif +# undef ETHR_HAVE_NATMC_ADD_RETURN_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_RB +# define ETHR_HAVE_NATMC_ADD_RETURN_RB 1 +# endif +# undef ETHR_HAVE_NATMC_ADD_RETURN_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_WB +# define ETHR_HAVE_NATMC_ADD_RETURN_WB 1 +# endif +# undef ETHR_HAVE_NATMC_ADD_RETURN_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_ACQB +# define ETHR_HAVE_NATMC_ADD_RETURN_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC_ADD_RETURN_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_RELB +# define ETHR_HAVE_NATMC_ADD_RETURN_RELB 1 +# endif +# undef ETHR_HAVE_NATMC_ADD_RETURN_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_MB +# define ETHR_HAVE_NATMC_ADD_RETURN_MB 1 +# endif +# undef ETHR_HAVE_NATMC_READ +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ +# define ETHR_HAVE_NATMC_READ 1 +# endif +# undef ETHR_HAVE_NATMC_READ_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_RB +# define ETHR_HAVE_NATMC_READ_RB 1 +# endif +# undef ETHR_HAVE_NATMC_READ_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_WB +# define ETHR_HAVE_NATMC_READ_WB 1 +# endif +# undef ETHR_HAVE_NATMC_READ_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_ACQB +# define ETHR_HAVE_NATMC_READ_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC_READ_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_RELB +# define ETHR_HAVE_NATMC_READ_RELB 1 +# endif +# undef ETHR_HAVE_NATMC_READ_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_MB +# define ETHR_HAVE_NATMC_READ_MB 1 +# endif +# undef ETHR_HAVE_NATMC_INC_RETURN +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN +# define ETHR_HAVE_NATMC_INC_RETURN 1 +# endif +# undef ETHR_HAVE_NATMC_INC_RETURN_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_RB +# define ETHR_HAVE_NATMC_INC_RETURN_RB 1 +# endif +# undef ETHR_HAVE_NATMC_INC_RETURN_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_WB +# define ETHR_HAVE_NATMC_INC_RETURN_WB 1 +# endif +# undef ETHR_HAVE_NATMC_INC_RETURN_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_ACQB +# define ETHR_HAVE_NATMC_INC_RETURN_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC_INC_RETURN_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_RELB +# define ETHR_HAVE_NATMC_INC_RETURN_RELB 1 +# endif +# undef ETHR_HAVE_NATMC_INC_RETURN_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_MB +# define ETHR_HAVE_NATMC_INC_RETURN_MB 1 +# endif +# undef ETHR_HAVE_NATMC_DEC_RETURN +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN +# define ETHR_HAVE_NATMC_DEC_RETURN 1 +# endif +# undef ETHR_HAVE_NATMC_DEC_RETURN_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_RB +# define ETHR_HAVE_NATMC_DEC_RETURN_RB 1 +# endif +# undef ETHR_HAVE_NATMC_DEC_RETURN_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_WB +# define ETHR_HAVE_NATMC_DEC_RETURN_WB 1 +# endif +# undef ETHR_HAVE_NATMC_DEC_RETURN_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_ACQB +# define ETHR_HAVE_NATMC_DEC_RETURN_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC_DEC_RETURN_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_RELB +# define ETHR_HAVE_NATMC_DEC_RETURN_RELB 1 +# endif +# undef ETHR_HAVE_NATMC_DEC_RETURN_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_MB +# define ETHR_HAVE_NATMC_DEC_RETURN_MB 1 +# endif +# undef ETHR_HAVE_NATMC_ADD +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD +# define ETHR_HAVE_NATMC_ADD 1 +# endif +# undef ETHR_HAVE_NATMC_ADD_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RB +# define ETHR_HAVE_NATMC_ADD_RB 1 +# endif +# undef ETHR_HAVE_NATMC_ADD_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_WB +# define ETHR_HAVE_NATMC_ADD_WB 1 +# endif +# undef ETHR_HAVE_NATMC_ADD_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_ACQB +# define ETHR_HAVE_NATMC_ADD_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC_ADD_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RELB +# define ETHR_HAVE_NATMC_ADD_RELB 1 +# endif +# undef ETHR_HAVE_NATMC_ADD_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_MB +# define ETHR_HAVE_NATMC_ADD_MB 1 +# endif +# undef ETHR_HAVE_NATMC_INC +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC +# define ETHR_HAVE_NATMC_INC 1 +# endif +# undef ETHR_HAVE_NATMC_INC_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RB +# define ETHR_HAVE_NATMC_INC_RB 1 +# endif +# undef ETHR_HAVE_NATMC_INC_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_WB +# define ETHR_HAVE_NATMC_INC_WB 1 +# endif +# undef ETHR_HAVE_NATMC_INC_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_ACQB +# define ETHR_HAVE_NATMC_INC_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC_INC_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RELB +# define ETHR_HAVE_NATMC_INC_RELB 1 +# endif +# undef ETHR_HAVE_NATMC_INC_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_MB +# define ETHR_HAVE_NATMC_INC_MB 1 +# endif +# undef ETHR_HAVE_NATMC_DEC +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC +# define ETHR_HAVE_NATMC_DEC 1 +# endif +# undef ETHR_HAVE_NATMC_DEC_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RB +# define ETHR_HAVE_NATMC_DEC_RB 1 +# endif +# undef ETHR_HAVE_NATMC_DEC_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_WB +# define ETHR_HAVE_NATMC_DEC_WB 1 +# endif +# undef ETHR_HAVE_NATMC_DEC_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_ACQB +# define ETHR_HAVE_NATMC_DEC_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC_DEC_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RELB +# define ETHR_HAVE_NATMC_DEC_RELB 1 +# endif +# undef ETHR_HAVE_NATMC_DEC_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_MB +# define ETHR_HAVE_NATMC_DEC_MB 1 +# endif +# undef ETHR_HAVE_NATMC_AND_RETOLD +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD +# define ETHR_HAVE_NATMC_AND_RETOLD 1 +# endif +# undef ETHR_HAVE_NATMC_AND_RETOLD_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_RB +# define ETHR_HAVE_NATMC_AND_RETOLD_RB 1 +# endif +# undef ETHR_HAVE_NATMC_AND_RETOLD_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_WB +# define ETHR_HAVE_NATMC_AND_RETOLD_WB 1 +# endif +# undef ETHR_HAVE_NATMC_AND_RETOLD_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_ACQB +# define ETHR_HAVE_NATMC_AND_RETOLD_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC_AND_RETOLD_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_RELB +# define ETHR_HAVE_NATMC_AND_RETOLD_RELB 1 +# endif +# undef ETHR_HAVE_NATMC_AND_RETOLD_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_MB +# define ETHR_HAVE_NATMC_AND_RETOLD_MB 1 +# endif +# undef ETHR_HAVE_NATMC_OR_RETOLD +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD +# define ETHR_HAVE_NATMC_OR_RETOLD 1 +# endif +# undef ETHR_HAVE_NATMC_OR_RETOLD_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_RB +# define ETHR_HAVE_NATMC_OR_RETOLD_RB 1 +# endif +# undef ETHR_HAVE_NATMC_OR_RETOLD_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_WB +# define ETHR_HAVE_NATMC_OR_RETOLD_WB 1 +# endif +# undef ETHR_HAVE_NATMC_OR_RETOLD_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_ACQB +# define ETHR_HAVE_NATMC_OR_RETOLD_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC_OR_RETOLD_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_RELB +# define ETHR_HAVE_NATMC_OR_RETOLD_RELB 1 +# endif +# undef ETHR_HAVE_NATMC_OR_RETOLD_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_MB +# define ETHR_HAVE_NATMC_OR_RETOLD_MB 1 +# endif +#elif ETHR_NATMC_BITS__ == 32 +# undef ETHR_HAVE_NATMC_CMPXCHG +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG +# define ETHR_HAVE_NATMC_CMPXCHG 1 +# endif +# undef ETHR_HAVE_NATMC_CMPXCHG_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_RB +# define ETHR_HAVE_NATMC_CMPXCHG_RB 1 +# endif +# undef ETHR_HAVE_NATMC_CMPXCHG_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_WB +# define ETHR_HAVE_NATMC_CMPXCHG_WB 1 +# endif +# undef ETHR_HAVE_NATMC_CMPXCHG_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_ACQB +# define ETHR_HAVE_NATMC_CMPXCHG_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC_CMPXCHG_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_RELB +# define ETHR_HAVE_NATMC_CMPXCHG_RELB 1 +# endif +# undef ETHR_HAVE_NATMC_CMPXCHG_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_MB +# define ETHR_HAVE_NATMC_CMPXCHG_MB 1 +# endif +# undef ETHR_HAVE_NATMC_XCHG +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG +# define ETHR_HAVE_NATMC_XCHG 1 +# endif +# undef ETHR_HAVE_NATMC_XCHG_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_RB +# define ETHR_HAVE_NATMC_XCHG_RB 1 +# endif +# undef ETHR_HAVE_NATMC_XCHG_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_WB +# define ETHR_HAVE_NATMC_XCHG_WB 1 +# endif +# undef ETHR_HAVE_NATMC_XCHG_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_ACQB +# define ETHR_HAVE_NATMC_XCHG_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC_XCHG_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_RELB +# define ETHR_HAVE_NATMC_XCHG_RELB 1 +# endif +# undef ETHR_HAVE_NATMC_XCHG_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_MB +# define ETHR_HAVE_NATMC_XCHG_MB 1 +# endif +# undef ETHR_HAVE_NATMC_SET +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET +# define ETHR_HAVE_NATMC_SET 1 +# endif +# undef ETHR_HAVE_NATMC_SET_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_RB +# define ETHR_HAVE_NATMC_SET_RB 1 +# endif +# undef ETHR_HAVE_NATMC_SET_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_WB +# define ETHR_HAVE_NATMC_SET_WB 1 +# endif +# undef ETHR_HAVE_NATMC_SET_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_ACQB +# define ETHR_HAVE_NATMC_SET_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC_SET_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_RELB +# define ETHR_HAVE_NATMC_SET_RELB 1 +# endif +# undef ETHR_HAVE_NATMC_SET_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_MB +# define ETHR_HAVE_NATMC_SET_MB 1 +# endif +# undef ETHR_HAVE_NATMC_INIT +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT +# define ETHR_HAVE_NATMC_INIT 1 +# endif +# undef ETHR_HAVE_NATMC_INIT_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_RB +# define ETHR_HAVE_NATMC_INIT_RB 1 +# endif +# undef ETHR_HAVE_NATMC_INIT_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_WB +# define ETHR_HAVE_NATMC_INIT_WB 1 +# endif +# undef ETHR_HAVE_NATMC_INIT_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_ACQB +# define ETHR_HAVE_NATMC_INIT_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC_INIT_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_RELB +# define ETHR_HAVE_NATMC_INIT_RELB 1 +# endif +# undef ETHR_HAVE_NATMC_INIT_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_MB +# define ETHR_HAVE_NATMC_INIT_MB 1 +# endif +# undef ETHR_HAVE_NATMC_ADD_RETURN +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN +# define ETHR_HAVE_NATMC_ADD_RETURN 1 +# endif +# undef ETHR_HAVE_NATMC_ADD_RETURN_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_RB +# define ETHR_HAVE_NATMC_ADD_RETURN_RB 1 +# endif +# undef ETHR_HAVE_NATMC_ADD_RETURN_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_WB +# define ETHR_HAVE_NATMC_ADD_RETURN_WB 1 +# endif +# undef ETHR_HAVE_NATMC_ADD_RETURN_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_ACQB +# define ETHR_HAVE_NATMC_ADD_RETURN_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC_ADD_RETURN_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_RELB +# define ETHR_HAVE_NATMC_ADD_RETURN_RELB 1 +# endif +# undef ETHR_HAVE_NATMC_ADD_RETURN_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_MB +# define ETHR_HAVE_NATMC_ADD_RETURN_MB 1 +# endif +# undef ETHR_HAVE_NATMC_READ +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ +# define ETHR_HAVE_NATMC_READ 1 +# endif +# undef ETHR_HAVE_NATMC_READ_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_RB +# define ETHR_HAVE_NATMC_READ_RB 1 +# endif +# undef ETHR_HAVE_NATMC_READ_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_WB +# define ETHR_HAVE_NATMC_READ_WB 1 +# endif +# undef ETHR_HAVE_NATMC_READ_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_ACQB +# define ETHR_HAVE_NATMC_READ_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC_READ_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_RELB +# define ETHR_HAVE_NATMC_READ_RELB 1 +# endif +# undef ETHR_HAVE_NATMC_READ_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_MB +# define ETHR_HAVE_NATMC_READ_MB 1 +# endif +# undef ETHR_HAVE_NATMC_INC_RETURN +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN +# define ETHR_HAVE_NATMC_INC_RETURN 1 +# endif +# undef ETHR_HAVE_NATMC_INC_RETURN_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_RB +# define ETHR_HAVE_NATMC_INC_RETURN_RB 1 +# endif +# undef ETHR_HAVE_NATMC_INC_RETURN_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_WB +# define ETHR_HAVE_NATMC_INC_RETURN_WB 1 +# endif +# undef ETHR_HAVE_NATMC_INC_RETURN_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_ACQB +# define ETHR_HAVE_NATMC_INC_RETURN_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC_INC_RETURN_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_RELB +# define ETHR_HAVE_NATMC_INC_RETURN_RELB 1 +# endif +# undef ETHR_HAVE_NATMC_INC_RETURN_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_MB +# define ETHR_HAVE_NATMC_INC_RETURN_MB 1 +# endif +# undef ETHR_HAVE_NATMC_DEC_RETURN +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN +# define ETHR_HAVE_NATMC_DEC_RETURN 1 +# endif +# undef ETHR_HAVE_NATMC_DEC_RETURN_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_RB +# define ETHR_HAVE_NATMC_DEC_RETURN_RB 1 +# endif +# undef ETHR_HAVE_NATMC_DEC_RETURN_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_WB +# define ETHR_HAVE_NATMC_DEC_RETURN_WB 1 +# endif +# undef ETHR_HAVE_NATMC_DEC_RETURN_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_ACQB +# define ETHR_HAVE_NATMC_DEC_RETURN_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC_DEC_RETURN_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_RELB +# define ETHR_HAVE_NATMC_DEC_RETURN_RELB 1 +# endif +# undef ETHR_HAVE_NATMC_DEC_RETURN_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_MB +# define ETHR_HAVE_NATMC_DEC_RETURN_MB 1 +# endif +# undef ETHR_HAVE_NATMC_ADD +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD +# define ETHR_HAVE_NATMC_ADD 1 +# endif +# undef ETHR_HAVE_NATMC_ADD_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RB +# define ETHR_HAVE_NATMC_ADD_RB 1 +# endif +# undef ETHR_HAVE_NATMC_ADD_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_WB +# define ETHR_HAVE_NATMC_ADD_WB 1 +# endif +# undef ETHR_HAVE_NATMC_ADD_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_ACQB +# define ETHR_HAVE_NATMC_ADD_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC_ADD_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RELB +# define ETHR_HAVE_NATMC_ADD_RELB 1 +# endif +# undef ETHR_HAVE_NATMC_ADD_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_MB +# define ETHR_HAVE_NATMC_ADD_MB 1 +# endif +# undef ETHR_HAVE_NATMC_INC +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC +# define ETHR_HAVE_NATMC_INC 1 +# endif +# undef ETHR_HAVE_NATMC_INC_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RB +# define ETHR_HAVE_NATMC_INC_RB 1 +# endif +# undef ETHR_HAVE_NATMC_INC_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_WB +# define ETHR_HAVE_NATMC_INC_WB 1 +# endif +# undef ETHR_HAVE_NATMC_INC_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_ACQB +# define ETHR_HAVE_NATMC_INC_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC_INC_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RELB +# define ETHR_HAVE_NATMC_INC_RELB 1 +# endif +# undef ETHR_HAVE_NATMC_INC_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_MB +# define ETHR_HAVE_NATMC_INC_MB 1 +# endif +# undef ETHR_HAVE_NATMC_DEC +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC +# define ETHR_HAVE_NATMC_DEC 1 +# endif +# undef ETHR_HAVE_NATMC_DEC_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RB +# define ETHR_HAVE_NATMC_DEC_RB 1 +# endif +# undef ETHR_HAVE_NATMC_DEC_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_WB +# define ETHR_HAVE_NATMC_DEC_WB 1 +# endif +# undef ETHR_HAVE_NATMC_DEC_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_ACQB +# define ETHR_HAVE_NATMC_DEC_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC_DEC_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RELB +# define ETHR_HAVE_NATMC_DEC_RELB 1 +# endif +# undef ETHR_HAVE_NATMC_DEC_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_MB +# define ETHR_HAVE_NATMC_DEC_MB 1 +# endif +# undef ETHR_HAVE_NATMC_AND_RETOLD +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD +# define ETHR_HAVE_NATMC_AND_RETOLD 1 +# endif +# undef ETHR_HAVE_NATMC_AND_RETOLD_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_RB +# define ETHR_HAVE_NATMC_AND_RETOLD_RB 1 +# endif +# undef ETHR_HAVE_NATMC_AND_RETOLD_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_WB +# define ETHR_HAVE_NATMC_AND_RETOLD_WB 1 +# endif +# undef ETHR_HAVE_NATMC_AND_RETOLD_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_ACQB +# define ETHR_HAVE_NATMC_AND_RETOLD_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC_AND_RETOLD_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_RELB +# define ETHR_HAVE_NATMC_AND_RETOLD_RELB 1 +# endif +# undef ETHR_HAVE_NATMC_AND_RETOLD_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_MB +# define ETHR_HAVE_NATMC_AND_RETOLD_MB 1 +# endif +# undef ETHR_HAVE_NATMC_OR_RETOLD +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD +# define ETHR_HAVE_NATMC_OR_RETOLD 1 +# endif +# undef ETHR_HAVE_NATMC_OR_RETOLD_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_RB +# define ETHR_HAVE_NATMC_OR_RETOLD_RB 1 +# endif +# undef ETHR_HAVE_NATMC_OR_RETOLD_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_WB +# define ETHR_HAVE_NATMC_OR_RETOLD_WB 1 +# endif +# undef ETHR_HAVE_NATMC_OR_RETOLD_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_ACQB +# define ETHR_HAVE_NATMC_OR_RETOLD_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC_OR_RETOLD_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_RELB +# define ETHR_HAVE_NATMC_OR_RETOLD_RELB 1 +# endif +# undef ETHR_HAVE_NATMC_OR_RETOLD_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_MB +# define ETHR_HAVE_NATMC_OR_RETOLD_MB 1 +# endif +#else +# error "Invalid native atomic size" #endif +#if (!defined(ETHR_HAVE_NATMC_CMPXCHG) \ + && !defined(ETHR_HAVE_NATMC_CMPXCHG_RB) \ + && !defined(ETHR_HAVE_NATMC_CMPXCHG_WB) \ + && !defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) \ + && !defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) \ + && !defined(ETHR_HAVE_NATMC_CMPXCHG_MB)) +# error "No native cmpxchg() op available" +#endif + + /* - * --- Pointer size atomics --------------------------------------------------- + * Read op used together with cmpxchg() fallback when no native op present. */ +#if defined(ETHR_HAVE_NATMC_READ) +#define ETHR_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \ + ETHR_NATMC_FUNC__(read)(VAR) +#elif defined(ETHR_HAVE_NATMC_READ_RB) +#define ETHR_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \ + ETHR_NATMC_FUNC__(read_rb)(VAR) +#elif defined(ETHR_HAVE_NATMC_READ_WB) +#define ETHR_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \ + ETHR_NATMC_FUNC__(read_wb)(VAR) +#elif defined(ETHR_HAVE_NATMC_READ_ACQB) +#define ETHR_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \ + ETHR_NATMC_FUNC__(read_acqb)(VAR) +#elif defined(ETHR_HAVE_NATMC_READ_RELB) +#define ETHR_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \ + ETHR_NATMC_FUNC__(read_relb)(VAR) +#elif defined(ETHR_HAVE_NATMC_READ_MB) +#define ETHR_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \ + ETHR_NATMC_FUNC__(read_mb)(VAR) +#else +/* + * We have no native read() op; guess zero and then use the + * the atomics actual value returned from cmpxchg(). + */ +#define ETHR_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \ + ((ETHR_NAINT_T__) 0) +#endif + +/* + * Native cmpxchg() fallback used when no native op present. + */ +#define ETHR_NATMC_CMPXCHG_FALLBACK__(CMPXCHG, VAR, AVAL, OPS) \ +do { \ + ethr_sint_t AVAL; \ + ETHR_NAINT_T__ new__, act__, exp__; \ + act__ = ETHR_NATMC_CMPXCHG_FALLBACK_READ__(VAR); \ + do { \ + exp__ = act__; \ + AVAL = (ethr_sint_t) act__; \ + { OPS; } \ + new__ = (ETHR_NAINT_T__) AVAL; \ + act__ = CMPXCHG(VAR, new__, exp__); \ + } while (__builtin_expect(act__ != exp__, 0)); \ +} while (0) + + + +/* --- addr() --- */ -static ETHR_INLINE ethr_sint_t * -ETHR_INLINE_FUNC_NAME_(ethr_atomic_addr)(ethr_atomic_t *var) +static ETHR_INLINE ethr_sint_t *ETHR_ATMC_FUNC__(addr)(ethr_atomic_t *var) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS return (ethr_sint_t *) ETHR_NATMC_ADDR_FUNC__(var); + +} + + +/* --- cmpxchg() --- */ + + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(cmpxchg)(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val) +{ + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_CMPXCHG) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); #else - return (ethr_sint_t *) var; +#error "Missing implementation of ethr_atomic_cmpxchg()!" #endif + return res; } -static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_atomic_init)(ethr_atomic_t *var, ethr_sint_t i) +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(cmpxchg_rb)(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - ETHR_NATMC_FUNC__(init)(var, (ETHR_NAINT_T__) i); + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); + ETHR_MEMBAR(ETHR_LoadLoad); #else - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = i); +#error "Missing implementation of ethr_atomic_cmpxchg_rb()!" #endif + return res; } -static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_atomic_set)(ethr_atomic_t *var, ethr_sint_t i) +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(cmpxchg_wb)(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - ETHR_NATMC_FUNC__(set)(var, (ETHR_NAINT_T__) i); + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); #else - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = i); +#error "Missing implementation of ethr_atomic_cmpxchg_wb()!" #endif + return res; } -static ETHR_INLINE ethr_sint_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic_read)(ethr_atomic_t *var) +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(cmpxchg_acqb)(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return (ethr_sint_t) ETHR_NATMC_FUNC__(read)(var); + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); #else +#error "Missing implementation of ethr_atomic_cmpxchg_acqb()!" +#endif + return res; +} + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(cmpxchg_relb)(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val) +{ ethr_sint_t res; - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var); +#if defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); +#else +#error "Missing implementation of ethr_atomic_cmpxchg_relb()!" +#endif return res; +} + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(cmpxchg_mb)(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val) +{ + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else +#error "Missing implementation of ethr_atomic_cmpxchg_mb()!" #endif + return res; } -static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_atomic_add)(ethr_atomic_t *var, ethr_sint_t incr) + +/* --- xchg() --- */ + + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(xchg)(ethr_atomic_t *var, ethr_sint_t val) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - ETHR_NATMC_FUNC__(add)(var, (ETHR_NAINT_T__) incr); + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_XCHG) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_XCHG_RB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_rb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_XCHG_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_wb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_XCHG_ACQB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_XCHG_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_relb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_XCHG_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val); #else - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += incr); +#error "Missing implementation of ethr_atomic_xchg()!" #endif -} - -static ETHR_INLINE ethr_sint_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic_add_read)(ethr_atomic_t *var, ethr_sint_t i) + return res; +} + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(xchg_rb)(ethr_atomic_t *var, ethr_sint_t val) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return (ethr_sint_t) ETHR_NATMC_FUNC__(add_return)(var, (ETHR_NAINT_T__) i); + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_XCHG_RB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_rb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_XCHG) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_XCHG_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_XCHG_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_wb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_XCHG_ACQB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_acqb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_XCHG_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_relb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval = val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val); + ETHR_MEMBAR(ETHR_LoadLoad); #else +#error "Missing implementation of ethr_atomic_xchg_rb()!" +#endif + return res; +} + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(xchg_wb)(ethr_atomic_t *var, ethr_sint_t val) +{ ethr_sint_t res; - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += i; res = *var); +#if defined(ETHR_HAVE_NATMC_XCHG_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_wb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_XCHG) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_XCHG_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_XCHG_RB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_rb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_XCHG_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_XCHG_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_relb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val); +#else +#error "Missing implementation of ethr_atomic_xchg_wb()!" +#endif return res; +} + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(xchg_acqb)(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_XCHG_ACQB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_XCHG_RB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_rb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_XCHG) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_XCHG_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_XCHG_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_wb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_XCHG_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_relb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval = val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else +#error "Missing implementation of ethr_atomic_xchg_acqb()!" #endif + return res; } -static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc)(ethr_atomic_t *var) +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(xchg_relb)(ethr_atomic_t *var, ethr_sint_t val) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - ETHR_NATMC_FUNC__(inc)(var); + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_XCHG_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_relb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_XCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_wb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_XCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_XCHG_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_XCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_rb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_XCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val); #else - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var)); +#error "Missing implementation of ethr_atomic_xchg_relb()!" #endif + return res; } -static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec)(ethr_atomic_t *var) +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(xchg_mb)(ethr_atomic_t *var, ethr_sint_t val) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - ETHR_NATMC_FUNC__(dec)(var); + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_XCHG_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_XCHG_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_relb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_XCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_XCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_wb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_XCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_rb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_XCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval = val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else +#error "Missing implementation of ethr_atomic_xchg_mb()!" +#endif + return res; +} + + +/* --- set() --- */ + + +static ETHR_INLINE void ETHR_ATMC_FUNC__(set)(ethr_atomic_t *var, ethr_sint_t val) +{ +#if defined(ETHR_HAVE_NATMC_SET) + ETHR_NATMC_FUNC__(set)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_SET_RB) + ETHR_NATMC_FUNC__(set_rb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_SET_WB) + ETHR_NATMC_FUNC__(set_wb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_SET_ACQB) + ETHR_NATMC_FUNC__(set_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_SET_RELB) + ETHR_NATMC_FUNC__(set_relb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_SET_MB) + ETHR_NATMC_FUNC__(set_mb)(var, (ETHR_NAINT_T__) val); +#else + (void) ETHR_ATMC_FUNC__(xchg)(var, val); +#endif +} + +static ETHR_INLINE void ETHR_ATMC_FUNC__(set_rb)(ethr_atomic_t *var, ethr_sint_t val) +{ +#if defined(ETHR_HAVE_NATMC_SET_RB) + ETHR_NATMC_FUNC__(set_rb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_SET) + ETHR_NATMC_FUNC__(set)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_SET_MB) + ETHR_NATMC_FUNC__(set_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_SET_WB) + ETHR_NATMC_FUNC__(set_wb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_SET_ACQB) + ETHR_NATMC_FUNC__(set_acqb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_SET_RELB) + ETHR_NATMC_FUNC__(set_relb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + (void) ETHR_ATMC_FUNC__(xchg_rb)(var, val); +#endif +} + +static ETHR_INLINE void ETHR_ATMC_FUNC__(set_wb)(ethr_atomic_t *var, ethr_sint_t val) +{ +#if defined(ETHR_HAVE_NATMC_SET_WB) + ETHR_NATMC_FUNC__(set_wb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_SET) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_FUNC__(set)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_SET_MB) + ETHR_NATMC_FUNC__(set_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_SET_RB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_FUNC__(set_rb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_SET_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_FUNC__(set_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_SET_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_FUNC__(set_relb)(var, (ETHR_NAINT_T__) val); +#else + (void) ETHR_ATMC_FUNC__(xchg_wb)(var, val); +#endif +} + +static ETHR_INLINE void ETHR_ATMC_FUNC__(set_acqb)(ethr_atomic_t *var, ethr_sint_t val) +{ +#if defined(ETHR_HAVE_NATMC_SET_ACQB) + ETHR_NATMC_FUNC__(set_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_SET_RB) + ETHR_NATMC_FUNC__(set_rb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_SET) + ETHR_NATMC_FUNC__(set)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_SET_MB) + ETHR_NATMC_FUNC__(set_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_SET_WB) + ETHR_NATMC_FUNC__(set_wb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_SET_RELB) + ETHR_NATMC_FUNC__(set_relb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#else + (void) ETHR_ATMC_FUNC__(xchg_acqb)(var, val); +#endif +} + +static ETHR_INLINE void ETHR_ATMC_FUNC__(set_relb)(ethr_atomic_t *var, ethr_sint_t val) +{ +#if defined(ETHR_HAVE_NATMC_SET_RELB) + ETHR_NATMC_FUNC__(set_relb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_SET_WB) + ETHR_MEMBAR(ETHR_LoadStore); + ETHR_NATMC_FUNC__(set_wb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_SET) + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_NATMC_FUNC__(set)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_SET_MB) + ETHR_NATMC_FUNC__(set_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_SET_RB) + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_NATMC_FUNC__(set_rb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_SET_ACQB) + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_NATMC_FUNC__(set_acqb)(var, (ETHR_NAINT_T__) val); +#else + (void) ETHR_ATMC_FUNC__(xchg_relb)(var, val); +#endif +} + +static ETHR_INLINE void ETHR_ATMC_FUNC__(set_mb)(ethr_atomic_t *var, ethr_sint_t val) +{ +#if defined(ETHR_HAVE_NATMC_SET_MB) + ETHR_NATMC_FUNC__(set_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_SET_RELB) + ETHR_NATMC_FUNC__(set_relb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_SET_ACQB) + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_NATMC_FUNC__(set_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_SET_WB) + ETHR_MEMBAR(ETHR_LoadStore); + ETHR_NATMC_FUNC__(set_wb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_SET_RB) + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_NATMC_FUNC__(set_rb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_SET) + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_NATMC_FUNC__(set)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#else + (void) ETHR_ATMC_FUNC__(xchg_mb)(var, val); +#endif +} + + +/* --- init() --- */ + + +static ETHR_INLINE void ETHR_ATMC_FUNC__(init)(ethr_atomic_t *var, ethr_sint_t val) +{ +#if defined(ETHR_HAVE_NATMC_INIT) + ETHR_NATMC_FUNC__(init)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_INIT_RB) + ETHR_NATMC_FUNC__(init_rb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_INIT_WB) + ETHR_NATMC_FUNC__(init_wb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_INIT_ACQB) + ETHR_NATMC_FUNC__(init_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_INIT_RELB) + ETHR_NATMC_FUNC__(init_relb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_INIT_MB) + ETHR_NATMC_FUNC__(init_mb)(var, (ETHR_NAINT_T__) val); +#else + ETHR_ATMC_FUNC__(set)(var, val); +#endif +} + +static ETHR_INLINE void ETHR_ATMC_FUNC__(init_rb)(ethr_atomic_t *var, ethr_sint_t val) +{ +#if defined(ETHR_HAVE_NATMC_INIT_RB) + ETHR_NATMC_FUNC__(init_rb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_INIT) + ETHR_NATMC_FUNC__(init)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_INIT_MB) + ETHR_NATMC_FUNC__(init_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_INIT_WB) + ETHR_NATMC_FUNC__(init_wb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_INIT_ACQB) + ETHR_NATMC_FUNC__(init_acqb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_INIT_RELB) + ETHR_NATMC_FUNC__(init_relb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + ETHR_ATMC_FUNC__(set_rb)(var, val); +#endif +} + +static ETHR_INLINE void ETHR_ATMC_FUNC__(init_wb)(ethr_atomic_t *var, ethr_sint_t val) +{ +#if defined(ETHR_HAVE_NATMC_INIT_WB) + ETHR_NATMC_FUNC__(init_wb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_INIT) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_FUNC__(init)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_INIT_MB) + ETHR_NATMC_FUNC__(init_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_INIT_RB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_FUNC__(init_rb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_INIT_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_FUNC__(init_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_INIT_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_FUNC__(init_relb)(var, (ETHR_NAINT_T__) val); +#else + ETHR_ATMC_FUNC__(set_wb)(var, val); +#endif +} + +static ETHR_INLINE void ETHR_ATMC_FUNC__(init_acqb)(ethr_atomic_t *var, ethr_sint_t val) +{ +#if defined(ETHR_HAVE_NATMC_INIT_ACQB) + ETHR_NATMC_FUNC__(init_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_INIT_RB) + ETHR_NATMC_FUNC__(init_rb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_INIT) + ETHR_NATMC_FUNC__(init)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_INIT_MB) + ETHR_NATMC_FUNC__(init_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_INIT_WB) + ETHR_NATMC_FUNC__(init_wb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_INIT_RELB) + ETHR_NATMC_FUNC__(init_relb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_ATMC_FUNC__(set_acqb)(var, val); +#endif +} + +static ETHR_INLINE void ETHR_ATMC_FUNC__(init_relb)(ethr_atomic_t *var, ethr_sint_t val) +{ +#if defined(ETHR_HAVE_NATMC_INIT_RELB) + ETHR_NATMC_FUNC__(init_relb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_INIT_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad); + ETHR_NATMC_FUNC__(init_wb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_INIT) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_NATMC_FUNC__(init)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_INIT_MB) + ETHR_NATMC_FUNC__(init_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_INIT_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_NATMC_FUNC__(init_rb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_INIT_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_NATMC_FUNC__(init_acqb)(var, (ETHR_NAINT_T__) val); +#else + ETHR_ATMC_FUNC__(set_relb)(var, val); +#endif +} + +static ETHR_INLINE void ETHR_ATMC_FUNC__(init_mb)(ethr_atomic_t *var, ethr_sint_t val) +{ +#if defined(ETHR_HAVE_NATMC_INIT_MB) + ETHR_NATMC_FUNC__(init_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_INIT_RELB) + ETHR_NATMC_FUNC__(init_relb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_INIT_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_NATMC_FUNC__(init_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_INIT_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad); + ETHR_NATMC_FUNC__(init_wb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_INIT_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_NATMC_FUNC__(init_rb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_INIT) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_NATMC_FUNC__(init)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_ATMC_FUNC__(set_mb)(var, val); +#endif +} + + +/* --- add_read() --- */ + + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(add_read)(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_ADD_RETURN) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_rb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_wb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_ACQB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_relb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval); +#else +#error "Missing implementation of ethr_atomic_add_read()!" +#endif + return res; +} + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(add_read_rb)(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_ADD_RETURN_RB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_rb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_wb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_ACQB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_acqb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_relb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, aval += val; res = aval); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval); + ETHR_MEMBAR(ETHR_LoadLoad); +#else +#error "Missing implementation of ethr_atomic_add_read_rb()!" +#endif + return res; +} + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(add_read_wb)(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_ADD_RETURN_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_wb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_rb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_relb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval); +#else +#error "Missing implementation of ethr_atomic_add_read_wb()!" +#endif + return res; +} + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(add_read_acqb)(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_ADD_RETURN_ACQB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_rb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_wb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_relb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, aval += val; res = aval); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else +#error "Missing implementation of ethr_atomic_add_read_acqb()!" +#endif + return res; +} + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(add_read_relb)(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_ADD_RETURN_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_relb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_wb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_rb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval); #else - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var)); +#error "Missing implementation of ethr_atomic_add_read_relb()!" #endif + return res; } -static ETHR_INLINE ethr_sint_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc_read)(ethr_atomic_t *var) +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(add_read_mb)(ethr_atomic_t *var, ethr_sint_t val) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return)(var); + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_ADD_RETURN_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_relb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_wb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_rb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_ADD_RETURN) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, aval += val; res = aval); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); #else +#error "Missing implementation of ethr_atomic_add_read_mb()!" +#endif + return res; +} + + +/* --- read() --- */ + + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read)(ethr_atomic_t *var) +{ ethr_sint_t res; - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var)); +#if defined(ETHR_HAVE_NATMC_READ) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read)(var); +#elif defined(ETHR_HAVE_NATMC_READ_RB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_rb)(var); +#elif defined(ETHR_HAVE_NATMC_READ_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_wb)(var); +#elif defined(ETHR_HAVE_NATMC_READ_ACQB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_acqb)(var); +#elif defined(ETHR_HAVE_NATMC_READ_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_relb)(var); +#elif defined(ETHR_HAVE_NATMC_READ_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_mb)(var); +#else + res = ETHR_ATMC_FUNC__(cmpxchg)(var, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__); +#endif return res; +} + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_rb)(ethr_atomic_t *var) +{ + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_READ_RB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_rb)(var); +#elif defined(ETHR_HAVE_NATMC_READ) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_READ_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_mb)(var); +#elif defined(ETHR_HAVE_NATMC_READ_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_READ_ACQB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_acqb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_READ_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + res = ETHR_ATMC_FUNC__(cmpxchg_rb)(var, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__); #endif + return res; } -static ETHR_INLINE ethr_sint_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_read)(ethr_atomic_t *var) +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_wb)(ethr_atomic_t *var) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return)(var); + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_READ_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_wb)(var); +#elif defined(ETHR_HAVE_NATMC_READ) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read)(var); +#elif defined(ETHR_HAVE_NATMC_READ_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_mb)(var); +#elif defined(ETHR_HAVE_NATMC_READ_RB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_rb)(var); +#elif defined(ETHR_HAVE_NATMC_READ_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_acqb)(var); +#elif defined(ETHR_HAVE_NATMC_READ_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_relb)(var); #else + res = ETHR_ATMC_FUNC__(cmpxchg_wb)(var, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__); +#endif + return res; +} + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_acqb)(ethr_atomic_t *var) +{ ethr_sint_t res; - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var)); +#if defined(ETHR_HAVE_NATMC_READ_ACQB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_acqb)(var); +#elif defined(ETHR_HAVE_NATMC_READ_RB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_rb)(var); + ETHR_MEMBAR(ETHR_LoadStore); +#elif defined(ETHR_HAVE_NATMC_READ) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#elif defined(ETHR_HAVE_NATMC_READ_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_mb)(var); +#elif defined(ETHR_HAVE_NATMC_READ_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#elif defined(ETHR_HAVE_NATMC_READ_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#else + res = ETHR_ATMC_FUNC__(cmpxchg_acqb)(var, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__); +#endif return res; +} + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_relb)(ethr_atomic_t *var) +{ + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_READ_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_relb)(var); +#elif defined(ETHR_HAVE_NATMC_READ_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_wb)(var); +#elif defined(ETHR_HAVE_NATMC_READ) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read)(var); +#elif defined(ETHR_HAVE_NATMC_READ_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_mb)(var); +#elif defined(ETHR_HAVE_NATMC_READ_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_rb)(var); +#elif defined(ETHR_HAVE_NATMC_READ_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_acqb)(var); +#else + res = ETHR_ATMC_FUNC__(cmpxchg_relb)(var, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__); #endif + return res; } -static ETHR_INLINE ethr_sint_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic_read_band)(ethr_atomic_t *var, - ethr_sint_t mask) +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_mb)(ethr_atomic_t *var) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold)(var, - (ETHR_NAINT_T__) mask); + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_READ_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_mb)(var); +#elif defined(ETHR_HAVE_NATMC_READ_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#elif defined(ETHR_HAVE_NATMC_READ_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_acqb)(var); +#elif defined(ETHR_HAVE_NATMC_READ_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#elif defined(ETHR_HAVE_NATMC_READ_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_rb)(var); + ETHR_MEMBAR(ETHR_LoadStore); +#elif defined(ETHR_HAVE_NATMC_READ) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(read)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); #else + res = ETHR_ATMC_FUNC__(cmpxchg_mb)(var, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__); +#endif + return res; +} + + +/* --- inc_read() --- */ + + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(inc_read)(ethr_atomic_t *var) +{ ethr_sint_t res; - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= mask); +#if defined(ETHR_HAVE_NATMC_INC_RETURN) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return)(var); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN_RB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_rb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_wb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN_ACQB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_acqb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_relb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_mb)(var); +#else + res = ETHR_ATMC_FUNC__(add_read)(var, (ethr_sint_t) 1); +#endif return res; +} + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(inc_read_rb)(ethr_atomic_t *var) +{ + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_INC_RETURN_RB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_rb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_mb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN_ACQB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_acqb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + res = ETHR_ATMC_FUNC__(add_read_rb)(var, (ethr_sint_t) 1); #endif + return res; } -static ETHR_INLINE ethr_sint_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic_read_bor)(ethr_atomic_t *var, - ethr_sint_t mask) +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(inc_read_wb)(ethr_atomic_t *var) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold)(var, - (ETHR_NAINT_T__) mask); + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_INC_RETURN_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_wb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return)(var); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_mb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN_RB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_rb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_acqb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_relb)(var); #else + res = ETHR_ATMC_FUNC__(add_read_wb)(var, (ethr_sint_t) 1); +#endif + return res; +} + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(inc_read_acqb)(ethr_atomic_t *var) +{ + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_INC_RETURN_ACQB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_acqb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN_RB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_rb)(var); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_mb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + res = ETHR_ATMC_FUNC__(add_read_acqb)(var, (ethr_sint_t) 1); +#endif + return res; +} + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(inc_read_relb)(ethr_atomic_t *var) +{ ethr_sint_t res; - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= mask); +#if defined(ETHR_HAVE_NATMC_INC_RETURN_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_relb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_wb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return)(var); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_mb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_rb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_acqb)(var); +#else + res = ETHR_ATMC_FUNC__(add_read_relb)(var, (ethr_sint_t) 1); +#endif return res; +} + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(inc_read_mb)(ethr_atomic_t *var) +{ + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_INC_RETURN_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_mb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_acqb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_rb)(var); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_INC_RETURN) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + res = ETHR_ATMC_FUNC__(add_read_mb)(var, (ethr_sint_t) 1); #endif + return res; } -static ETHR_INLINE ethr_sint_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic_xchg)(ethr_atomic_t *var, ethr_sint_t new) + +/* --- dec_read() --- */ + + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(dec_read)(ethr_atomic_t *var) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return (ethr_sint_t) ETHR_NATMC_FUNC__(xchg)(var, - (ETHR_NAINT_T__) new); + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_DEC_RETURN) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_rb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_wb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_ACQB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_acqb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_relb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_mb)(var); #else + res = ETHR_ATMC_FUNC__(add_read)(var, (ethr_sint_t) -1); +#endif + return res; +} + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(dec_read_rb)(ethr_atomic_t *var) +{ ethr_sint_t res; - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = new); +#if defined(ETHR_HAVE_NATMC_DEC_RETURN_RB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_rb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_mb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_ACQB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_acqb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + res = ETHR_ATMC_FUNC__(add_read_rb)(var, (ethr_sint_t) -1); +#endif return res; +} + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(dec_read_wb)(ethr_atomic_t *var) +{ + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_DEC_RETURN_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_wb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_mb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_rb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_acqb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_relb)(var); +#else + res = ETHR_ATMC_FUNC__(add_read_wb)(var, (ethr_sint_t) -1); #endif + return res; } -static ETHR_INLINE ethr_sint_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(ethr_atomic_t *var, - ethr_sint_t new, - ethr_sint_t exp) +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(dec_read_acqb)(ethr_atomic_t *var) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg)(var, - (ETHR_NAINT_T__) new, - (ETHR_NAINT_T__) exp); + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_DEC_RETURN_ACQB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_acqb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_rb)(var); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_mb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); #else + res = ETHR_ATMC_FUNC__(add_read_acqb)(var, (ethr_sint_t) -1); +#endif + return res; +} + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(dec_read_relb)(ethr_atomic_t *var) +{ ethr_sint_t res; - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, - { - res = *var; - if (__builtin_expect(res == exp, 1)) - *var = new; - }); +#if defined(ETHR_HAVE_NATMC_DEC_RETURN_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_relb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_wb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_mb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_rb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_acqb)(var); +#else + res = ETHR_ATMC_FUNC__(add_read_relb)(var, (ethr_sint_t) -1); +#endif return res; +} + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(dec_read_mb)(ethr_atomic_t *var) +{ + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_DEC_RETURN_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_mb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_acqb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_rb)(var); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_DEC_RETURN) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + res = ETHR_ATMC_FUNC__(add_read_mb)(var, (ethr_sint_t) -1); #endif + return res; } -/* - * Important memory barrier requirements. - * - * The following atomic operations *must* supply a memory barrier of - * at least the type specified by its suffix: - * _acqb = acquire barrier - * _relb = release barrier - */ -static ETHR_INLINE ethr_sint_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic_read_acqb)(ethr_atomic_t *var) +/* --- add() --- */ + + +static ETHR_INLINE void ETHR_ATMC_FUNC__(add)(ethr_atomic_t *var, ethr_sint_t val) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return (ethr_sint_t) ETHR_NATMC_FUNC__(read_acqb)(var); +#if defined(ETHR_HAVE_NATMC_ADD) + ETHR_NATMC_FUNC__(add)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_RB) + ETHR_NATMC_FUNC__(add_rb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_WB) + ETHR_NATMC_FUNC__(add_wb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_ACQB) + ETHR_NATMC_FUNC__(add_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_RELB) + ETHR_NATMC_FUNC__(add_relb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_MB) + ETHR_NATMC_FUNC__(add_mb)(var, (ETHR_NAINT_T__) val); #else - return ETHR_INLINE_FUNC_NAME_(ethr_atomic_read)(var); + (void) ETHR_ATMC_FUNC__(add_read)(var, val); #endif } -static ETHR_INLINE ethr_sint_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc_read_acqb)(ethr_atomic_t *var) +static ETHR_INLINE void ETHR_ATMC_FUNC__(add_rb)(ethr_atomic_t *var, ethr_sint_t val) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_acqb)(var); +#if defined(ETHR_HAVE_NATMC_ADD_RB) + ETHR_NATMC_FUNC__(add_rb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD) + ETHR_NATMC_FUNC__(add)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_ADD_MB) + ETHR_NATMC_FUNC__(add_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_WB) + ETHR_NATMC_FUNC__(add_wb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_ADD_ACQB) + ETHR_NATMC_FUNC__(add_acqb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_ADD_RELB) + ETHR_NATMC_FUNC__(add_relb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); #else - return ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc_read)(var); + (void) ETHR_ATMC_FUNC__(add_read_rb)(var, val); #endif } -static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_atomic_set_relb)(ethr_atomic_t *var, - ethr_sint_t val) +static ETHR_INLINE void ETHR_ATMC_FUNC__(add_wb)(ethr_atomic_t *var, ethr_sint_t val) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - ETHR_NATMC_FUNC__(set_relb)(var, (ETHR_NAINT_T__) val); +#if defined(ETHR_HAVE_NATMC_ADD_WB) + ETHR_NATMC_FUNC__(add_wb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_FUNC__(add)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_MB) + ETHR_NATMC_FUNC__(add_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_RB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_FUNC__(add_rb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_FUNC__(add_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_FUNC__(add_relb)(var, (ETHR_NAINT_T__) val); #else - ETHR_INLINE_FUNC_NAME_(ethr_atomic_set)(var, val); + (void) ETHR_ATMC_FUNC__(add_read_wb)(var, val); #endif } -static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_relb)(ethr_atomic_t *var) +static ETHR_INLINE void ETHR_ATMC_FUNC__(add_acqb)(ethr_atomic_t *var, ethr_sint_t val) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS +#if defined(ETHR_HAVE_NATMC_ADD_ACQB) + ETHR_NATMC_FUNC__(add_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_RB) + ETHR_NATMC_FUNC__(add_rb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_ADD) + ETHR_NATMC_FUNC__(add)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_ADD_MB) + ETHR_NATMC_FUNC__(add_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_WB) + ETHR_NATMC_FUNC__(add_wb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_ADD_RELB) + ETHR_NATMC_FUNC__(add_relb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + (void) ETHR_ATMC_FUNC__(add_read_acqb)(var, val); +#endif +} + +static ETHR_INLINE void ETHR_ATMC_FUNC__(add_relb)(ethr_atomic_t *var, ethr_sint_t val) +{ +#if defined(ETHR_HAVE_NATMC_ADD_RELB) + ETHR_NATMC_FUNC__(add_relb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_FUNC__(add_wb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_FUNC__(add)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_MB) + ETHR_NATMC_FUNC__(add_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_FUNC__(add_rb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_FUNC__(add_acqb)(var, (ETHR_NAINT_T__) val); +#else + (void) ETHR_ATMC_FUNC__(add_read_relb)(var, val); +#endif +} + +static ETHR_INLINE void ETHR_ATMC_FUNC__(add_mb)(ethr_atomic_t *var, ethr_sint_t val) +{ +#if defined(ETHR_HAVE_NATMC_ADD_MB) + ETHR_NATMC_FUNC__(add_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_RELB) + ETHR_NATMC_FUNC__(add_relb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_ADD_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_FUNC__(add_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_ADD_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_FUNC__(add_wb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_ADD_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_FUNC__(add_rb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_ADD) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_FUNC__(add)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + (void) ETHR_ATMC_FUNC__(add_read_mb)(var, val); +#endif +} + + +/* --- inc() --- */ + + +static ETHR_INLINE void ETHR_ATMC_FUNC__(inc)(ethr_atomic_t *var) +{ +#if defined(ETHR_HAVE_NATMC_INC) + ETHR_NATMC_FUNC__(inc)(var); +#elif defined(ETHR_HAVE_NATMC_INC_RB) + ETHR_NATMC_FUNC__(inc_rb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_WB) + ETHR_NATMC_FUNC__(inc_wb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_ACQB) + ETHR_NATMC_FUNC__(inc_acqb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_RELB) + ETHR_NATMC_FUNC__(inc_relb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_MB) + ETHR_NATMC_FUNC__(inc_mb)(var); +#else + (void) ETHR_ATMC_FUNC__(inc_read)(var); +#endif +} + +static ETHR_INLINE void ETHR_ATMC_FUNC__(inc_rb)(ethr_atomic_t *var) +{ +#if defined(ETHR_HAVE_NATMC_INC_RB) + ETHR_NATMC_FUNC__(inc_rb)(var); +#elif defined(ETHR_HAVE_NATMC_INC) + ETHR_NATMC_FUNC__(inc)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_INC_MB) + ETHR_NATMC_FUNC__(inc_mb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_WB) + ETHR_NATMC_FUNC__(inc_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_INC_ACQB) + ETHR_NATMC_FUNC__(inc_acqb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_INC_RELB) + ETHR_NATMC_FUNC__(inc_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + (void) ETHR_ATMC_FUNC__(inc_read_rb)(var); +#endif +} + +static ETHR_INLINE void ETHR_ATMC_FUNC__(inc_wb)(ethr_atomic_t *var) +{ +#if defined(ETHR_HAVE_NATMC_INC_WB) + ETHR_NATMC_FUNC__(inc_wb)(var); +#elif defined(ETHR_HAVE_NATMC_INC) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_FUNC__(inc)(var); +#elif defined(ETHR_HAVE_NATMC_INC_MB) + ETHR_NATMC_FUNC__(inc_mb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_RB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_FUNC__(inc_rb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_FUNC__(inc_acqb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_FUNC__(inc_relb)(var); +#else + (void) ETHR_ATMC_FUNC__(inc_read_wb)(var); +#endif +} + +static ETHR_INLINE void ETHR_ATMC_FUNC__(inc_acqb)(ethr_atomic_t *var) +{ +#if defined(ETHR_HAVE_NATMC_INC_ACQB) + ETHR_NATMC_FUNC__(inc_acqb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_RB) + ETHR_NATMC_FUNC__(inc_rb)(var); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_INC) + ETHR_NATMC_FUNC__(inc)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_INC_MB) + ETHR_NATMC_FUNC__(inc_mb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_WB) + ETHR_NATMC_FUNC__(inc_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_INC_RELB) + ETHR_NATMC_FUNC__(inc_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + (void) ETHR_ATMC_FUNC__(inc_read_acqb)(var); +#endif +} + +static ETHR_INLINE void ETHR_ATMC_FUNC__(inc_relb)(ethr_atomic_t *var) +{ +#if defined(ETHR_HAVE_NATMC_INC_RELB) + ETHR_NATMC_FUNC__(inc_relb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_FUNC__(inc_wb)(var); +#elif defined(ETHR_HAVE_NATMC_INC) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_FUNC__(inc)(var); +#elif defined(ETHR_HAVE_NATMC_INC_MB) + ETHR_NATMC_FUNC__(inc_mb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_FUNC__(inc_rb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_FUNC__(inc_acqb)(var); +#else + (void) ETHR_ATMC_FUNC__(inc_read_relb)(var); +#endif +} + +static ETHR_INLINE void ETHR_ATMC_FUNC__(inc_mb)(ethr_atomic_t *var) +{ +#if defined(ETHR_HAVE_NATMC_INC_MB) + ETHR_NATMC_FUNC__(inc_mb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_RELB) + ETHR_NATMC_FUNC__(inc_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_INC_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_FUNC__(inc_acqb)(var); +#elif defined(ETHR_HAVE_NATMC_INC_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_FUNC__(inc_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_INC_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_FUNC__(inc_rb)(var); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_INC) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_FUNC__(inc)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + (void) ETHR_ATMC_FUNC__(inc_read_mb)(var); +#endif +} + + +/* --- dec() --- */ + + +static ETHR_INLINE void ETHR_ATMC_FUNC__(dec)(ethr_atomic_t *var) +{ +#if defined(ETHR_HAVE_NATMC_DEC) + ETHR_NATMC_FUNC__(dec)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_RB) + ETHR_NATMC_FUNC__(dec_rb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_WB) + ETHR_NATMC_FUNC__(dec_wb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_ACQB) + ETHR_NATMC_FUNC__(dec_acqb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_RELB) + ETHR_NATMC_FUNC__(dec_relb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_MB) + ETHR_NATMC_FUNC__(dec_mb)(var); +#else + (void) ETHR_ATMC_FUNC__(dec_read)(var); +#endif +} + +static ETHR_INLINE void ETHR_ATMC_FUNC__(dec_rb)(ethr_atomic_t *var) +{ +#if defined(ETHR_HAVE_NATMC_DEC_RB) + ETHR_NATMC_FUNC__(dec_rb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC) + ETHR_NATMC_FUNC__(dec)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_DEC_MB) + ETHR_NATMC_FUNC__(dec_mb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_WB) + ETHR_NATMC_FUNC__(dec_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_DEC_ACQB) + ETHR_NATMC_FUNC__(dec_acqb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_DEC_RELB) + ETHR_NATMC_FUNC__(dec_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + (void) ETHR_ATMC_FUNC__(dec_read_rb)(var); +#endif +} + +static ETHR_INLINE void ETHR_ATMC_FUNC__(dec_wb)(ethr_atomic_t *var) +{ +#if defined(ETHR_HAVE_NATMC_DEC_WB) + ETHR_NATMC_FUNC__(dec_wb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_FUNC__(dec)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_MB) + ETHR_NATMC_FUNC__(dec_mb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_RB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_FUNC__(dec_rb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_FUNC__(dec_acqb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_FUNC__(dec_relb)(var); +#else + (void) ETHR_ATMC_FUNC__(dec_read_wb)(var); +#endif +} + +static ETHR_INLINE void ETHR_ATMC_FUNC__(dec_acqb)(ethr_atomic_t *var) +{ +#if defined(ETHR_HAVE_NATMC_DEC_ACQB) + ETHR_NATMC_FUNC__(dec_acqb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_RB) + ETHR_NATMC_FUNC__(dec_rb)(var); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_DEC) + ETHR_NATMC_FUNC__(dec)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_DEC_MB) + ETHR_NATMC_FUNC__(dec_mb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_WB) + ETHR_NATMC_FUNC__(dec_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_DEC_RELB) + ETHR_NATMC_FUNC__(dec_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + (void) ETHR_ATMC_FUNC__(dec_read_acqb)(var); +#endif +} + +static ETHR_INLINE void ETHR_ATMC_FUNC__(dec_relb)(ethr_atomic_t *var) +{ +#if defined(ETHR_HAVE_NATMC_DEC_RELB) + ETHR_NATMC_FUNC__(dec_relb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_FUNC__(dec_wb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_FUNC__(dec)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_MB) + ETHR_NATMC_FUNC__(dec_mb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_FUNC__(dec_rb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_FUNC__(dec_acqb)(var); +#else + (void) ETHR_ATMC_FUNC__(dec_read_relb)(var); +#endif +} + +static ETHR_INLINE void ETHR_ATMC_FUNC__(dec_mb)(ethr_atomic_t *var) +{ +#if defined(ETHR_HAVE_NATMC_DEC_MB) + ETHR_NATMC_FUNC__(dec_mb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_RELB) ETHR_NATMC_FUNC__(dec_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_DEC_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_FUNC__(dec_acqb)(var); +#elif defined(ETHR_HAVE_NATMC_DEC_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_FUNC__(dec_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_DEC_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_FUNC__(dec_rb)(var); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_DEC) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_FUNC__(dec)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + (void) ETHR_ATMC_FUNC__(dec_read_mb)(var); +#endif +} + + +/* --- read_band() --- */ + + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_band)(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_AND_RETOLD) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_rb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_wb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_ACQB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_relb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val); +#else +#error "Missing implementation of ethr_atomic_read_band()!" +#endif + return res; +} + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_band_rb)(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_AND_RETOLD_RB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_rb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_wb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_ACQB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_acqb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_relb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval &= val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val); + ETHR_MEMBAR(ETHR_LoadLoad); +#else +#error "Missing implementation of ethr_atomic_read_band_rb()!" +#endif + return res; +} + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_band_wb)(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_AND_RETOLD_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_wb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_rb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_relb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val); #else - ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec)(var); +#error "Missing implementation of ethr_atomic_read_band_wb()!" #endif + return res; } -static ETHR_INLINE ethr_sint_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_read_relb)(ethr_atomic_t *var) +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_band_acqb)(ethr_atomic_t *var, ethr_sint_t val) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_relb)(var); + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_AND_RETOLD_ACQB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_rb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_wb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_relb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval &= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); #else - return ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_read)(var); +#error "Missing implementation of ethr_atomic_read_band_acqb()!" #endif + return res; } -static ETHR_INLINE ethr_sint_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg_acqb)(ethr_atomic_t *var, - ethr_sint_t new, - ethr_sint_t exp) +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_band_relb)(ethr_atomic_t *var, ethr_sint_t val) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_acqb)(var, - (ETHR_NAINT_T__) new, - (ETHR_NAINT_T__) exp); + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_AND_RETOLD_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_relb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_wb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_rb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val); #else - return ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(var, new, exp); +#error "Missing implementation of ethr_atomic_read_band_relb()!" #endif + return res; } -static ETHR_INLINE ethr_sint_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg_relb)(ethr_atomic_t *var, - ethr_sint_t new, - ethr_sint_t exp) +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_band_mb)(ethr_atomic_t *var, ethr_sint_t val) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_relb)(var, - (ETHR_NAINT_T__) new, - (ETHR_NAINT_T__) exp); + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_AND_RETOLD_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_relb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_wb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_rb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_AND_RETOLD) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval &= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); #else - return ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(var, new, exp); +#error "Missing implementation of ethr_atomic_read_band_mb()!" #endif + return res; } + +/* --- read_bor() --- */ + + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_bor)(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_OR_RETOLD) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_rb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_wb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_ACQB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_relb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val); +#else +#error "Missing implementation of ethr_atomic_read_bor()!" +#endif + return res; +} + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_bor_rb)(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_OR_RETOLD_RB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_rb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_wb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_ACQB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_acqb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_relb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval |= val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val); + ETHR_MEMBAR(ETHR_LoadLoad); +#else +#error "Missing implementation of ethr_atomic_read_bor_rb()!" +#endif + return res; +} + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_bor_wb)(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_OR_RETOLD_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_wb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_rb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_relb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val); +#else +#error "Missing implementation of ethr_atomic_read_bor_wb()!" +#endif + return res; +} + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_bor_acqb)(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_OR_RETOLD_ACQB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_rb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_WB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_wb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_relb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval |= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else +#error "Missing implementation of ethr_atomic_read_bor_acqb()!" +#endif + return res; +} + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_bor_relb)(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_OR_RETOLD_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_relb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_wb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_rb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val); +#else +#error "Missing implementation of ethr_atomic_read_bor_relb()!" +#endif + return res; +} + +static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_bor_mb)(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; +#if defined(ETHR_HAVE_NATMC_OR_RETOLD_MB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_mb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RELB) + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_relb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_acqb)(var, (ETHR_NAINT_T__) val); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_wb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_rb)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_OR_RETOLD) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold)(var, (ETHR_NAINT_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC_CMPXCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval |= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else +#error "Missing implementation of ethr_atomic_read_bor_mb()!" +#endif + return res; +} + +#endif /* ETHR_ATMC_INLINE__ */ + + +/* ---------- 32-bit atomic implementation ---------- */ + + +#ifdef ETHR_NEED_ATMC32_PROTOTYPES__ +ethr_sint32_t *ethr_atomic32_addr(ethr_atomic32_t *var); +ethr_sint32_t ethr_atomic32_cmpxchg(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val); +ethr_sint32_t ethr_atomic32_cmpxchg_rb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val); +ethr_sint32_t ethr_atomic32_cmpxchg_wb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val); +ethr_sint32_t ethr_atomic32_cmpxchg_acqb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val); +ethr_sint32_t ethr_atomic32_cmpxchg_relb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val); +ethr_sint32_t ethr_atomic32_cmpxchg_mb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val); +ethr_sint32_t ethr_atomic32_xchg(ethr_atomic32_t *var, ethr_sint32_t val); +ethr_sint32_t ethr_atomic32_xchg_rb(ethr_atomic32_t *var, ethr_sint32_t val); +ethr_sint32_t ethr_atomic32_xchg_wb(ethr_atomic32_t *var, ethr_sint32_t val); +ethr_sint32_t ethr_atomic32_xchg_acqb(ethr_atomic32_t *var, ethr_sint32_t val); +ethr_sint32_t ethr_atomic32_xchg_relb(ethr_atomic32_t *var, ethr_sint32_t val); +ethr_sint32_t ethr_atomic32_xchg_mb(ethr_atomic32_t *var, ethr_sint32_t val); +void ethr_atomic32_set(ethr_atomic32_t *var, ethr_sint32_t val); +void ethr_atomic32_set_rb(ethr_atomic32_t *var, ethr_sint32_t val); +void ethr_atomic32_set_wb(ethr_atomic32_t *var, ethr_sint32_t val); +void ethr_atomic32_set_acqb(ethr_atomic32_t *var, ethr_sint32_t val); +void ethr_atomic32_set_relb(ethr_atomic32_t *var, ethr_sint32_t val); +void ethr_atomic32_set_mb(ethr_atomic32_t *var, ethr_sint32_t val); +void ethr_atomic32_init(ethr_atomic32_t *var, ethr_sint32_t val); +void ethr_atomic32_init_rb(ethr_atomic32_t *var, ethr_sint32_t val); +void ethr_atomic32_init_wb(ethr_atomic32_t *var, ethr_sint32_t val); +void ethr_atomic32_init_acqb(ethr_atomic32_t *var, ethr_sint32_t val); +void ethr_atomic32_init_relb(ethr_atomic32_t *var, ethr_sint32_t val); +void ethr_atomic32_init_mb(ethr_atomic32_t *var, ethr_sint32_t val); +ethr_sint32_t ethr_atomic32_add_read(ethr_atomic32_t *var, ethr_sint32_t val); +ethr_sint32_t ethr_atomic32_add_read_rb(ethr_atomic32_t *var, ethr_sint32_t val); +ethr_sint32_t ethr_atomic32_add_read_wb(ethr_atomic32_t *var, ethr_sint32_t val); +ethr_sint32_t ethr_atomic32_add_read_acqb(ethr_atomic32_t *var, ethr_sint32_t val); +ethr_sint32_t ethr_atomic32_add_read_relb(ethr_atomic32_t *var, ethr_sint32_t val); +ethr_sint32_t ethr_atomic32_add_read_mb(ethr_atomic32_t *var, ethr_sint32_t val); +ethr_sint32_t ethr_atomic32_read(ethr_atomic32_t *var); +ethr_sint32_t ethr_atomic32_read_rb(ethr_atomic32_t *var); +ethr_sint32_t ethr_atomic32_read_wb(ethr_atomic32_t *var); +ethr_sint32_t ethr_atomic32_read_acqb(ethr_atomic32_t *var); +ethr_sint32_t ethr_atomic32_read_relb(ethr_atomic32_t *var); +ethr_sint32_t ethr_atomic32_read_mb(ethr_atomic32_t *var); +ethr_sint32_t ethr_atomic32_inc_read(ethr_atomic32_t *var); +ethr_sint32_t ethr_atomic32_inc_read_rb(ethr_atomic32_t *var); +ethr_sint32_t ethr_atomic32_inc_read_wb(ethr_atomic32_t *var); +ethr_sint32_t ethr_atomic32_inc_read_acqb(ethr_atomic32_t *var); +ethr_sint32_t ethr_atomic32_inc_read_relb(ethr_atomic32_t *var); +ethr_sint32_t ethr_atomic32_inc_read_mb(ethr_atomic32_t *var); +ethr_sint32_t ethr_atomic32_dec_read(ethr_atomic32_t *var); +ethr_sint32_t ethr_atomic32_dec_read_rb(ethr_atomic32_t *var); +ethr_sint32_t ethr_atomic32_dec_read_wb(ethr_atomic32_t *var); +ethr_sint32_t ethr_atomic32_dec_read_acqb(ethr_atomic32_t *var); +ethr_sint32_t ethr_atomic32_dec_read_relb(ethr_atomic32_t *var); +ethr_sint32_t ethr_atomic32_dec_read_mb(ethr_atomic32_t *var); +void ethr_atomic32_add(ethr_atomic32_t *var, ethr_sint32_t val); +void ethr_atomic32_add_rb(ethr_atomic32_t *var, ethr_sint32_t val); +void ethr_atomic32_add_wb(ethr_atomic32_t *var, ethr_sint32_t val); +void ethr_atomic32_add_acqb(ethr_atomic32_t *var, ethr_sint32_t val); +void ethr_atomic32_add_relb(ethr_atomic32_t *var, ethr_sint32_t val); +void ethr_atomic32_add_mb(ethr_atomic32_t *var, ethr_sint32_t val); +void ethr_atomic32_inc(ethr_atomic32_t *var); +void ethr_atomic32_inc_rb(ethr_atomic32_t *var); +void ethr_atomic32_inc_wb(ethr_atomic32_t *var); +void ethr_atomic32_inc_acqb(ethr_atomic32_t *var); +void ethr_atomic32_inc_relb(ethr_atomic32_t *var); +void ethr_atomic32_inc_mb(ethr_atomic32_t *var); +void ethr_atomic32_dec(ethr_atomic32_t *var); +void ethr_atomic32_dec_rb(ethr_atomic32_t *var); +void ethr_atomic32_dec_wb(ethr_atomic32_t *var); +void ethr_atomic32_dec_acqb(ethr_atomic32_t *var); +void ethr_atomic32_dec_relb(ethr_atomic32_t *var); +void ethr_atomic32_dec_mb(ethr_atomic32_t *var); +ethr_sint32_t ethr_atomic32_read_band(ethr_atomic32_t *var, ethr_sint32_t val); +ethr_sint32_t ethr_atomic32_read_band_rb(ethr_atomic32_t *var, ethr_sint32_t val); +ethr_sint32_t ethr_atomic32_read_band_wb(ethr_atomic32_t *var, ethr_sint32_t val); +ethr_sint32_t ethr_atomic32_read_band_acqb(ethr_atomic32_t *var, ethr_sint32_t val); +ethr_sint32_t ethr_atomic32_read_band_relb(ethr_atomic32_t *var, ethr_sint32_t val); +ethr_sint32_t ethr_atomic32_read_band_mb(ethr_atomic32_t *var, ethr_sint32_t val); +ethr_sint32_t ethr_atomic32_read_bor(ethr_atomic32_t *var, ethr_sint32_t val); +ethr_sint32_t ethr_atomic32_read_bor_rb(ethr_atomic32_t *var, ethr_sint32_t val); +ethr_sint32_t ethr_atomic32_read_bor_wb(ethr_atomic32_t *var, ethr_sint32_t val); +ethr_sint32_t ethr_atomic32_read_bor_acqb(ethr_atomic32_t *var, ethr_sint32_t val); +ethr_sint32_t ethr_atomic32_read_bor_relb(ethr_atomic32_t *var, ethr_sint32_t val); +ethr_sint32_t ethr_atomic32_read_bor_mb(ethr_atomic32_t *var, ethr_sint32_t val); +#endif /* ETHR_NEED_ATMC32_PROTOTYPES__ */ + +#if (defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) \ + && (defined(ETHR_ATMC32_INLINE__) || defined(ETHR_ATOMIC_IMPL__))) + +#if !defined(ETHR_NATMC32_BITS__) +# error "Missing native atomic implementation" +#elif ETHR_NATMC32_BITS__ == 64 +# undef ETHR_HAVE_NATMC32_CMPXCHG +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG +# define ETHR_HAVE_NATMC32_CMPXCHG 1 +# endif +# undef ETHR_HAVE_NATMC32_CMPXCHG_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RB +# define ETHR_HAVE_NATMC32_CMPXCHG_RB 1 +# endif +# undef ETHR_HAVE_NATMC32_CMPXCHG_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_WB +# define ETHR_HAVE_NATMC32_CMPXCHG_WB 1 +# endif +# undef ETHR_HAVE_NATMC32_CMPXCHG_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_ACQB +# define ETHR_HAVE_NATMC32_CMPXCHG_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC32_CMPXCHG_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RELB +# define ETHR_HAVE_NATMC32_CMPXCHG_RELB 1 +# endif +# undef ETHR_HAVE_NATMC32_CMPXCHG_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_MB +# define ETHR_HAVE_NATMC32_CMPXCHG_MB 1 +# endif +# undef ETHR_HAVE_NATMC32_XCHG +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG +# define ETHR_HAVE_NATMC32_XCHG 1 +# endif +# undef ETHR_HAVE_NATMC32_XCHG_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_RB +# define ETHR_HAVE_NATMC32_XCHG_RB 1 +# endif +# undef ETHR_HAVE_NATMC32_XCHG_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_WB +# define ETHR_HAVE_NATMC32_XCHG_WB 1 +# endif +# undef ETHR_HAVE_NATMC32_XCHG_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_ACQB +# define ETHR_HAVE_NATMC32_XCHG_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC32_XCHG_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_RELB +# define ETHR_HAVE_NATMC32_XCHG_RELB 1 +# endif +# undef ETHR_HAVE_NATMC32_XCHG_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_MB +# define ETHR_HAVE_NATMC32_XCHG_MB 1 +# endif +# undef ETHR_HAVE_NATMC32_SET +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET +# define ETHR_HAVE_NATMC32_SET 1 +# endif +# undef ETHR_HAVE_NATMC32_SET_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RB +# define ETHR_HAVE_NATMC32_SET_RB 1 +# endif +# undef ETHR_HAVE_NATMC32_SET_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_WB +# define ETHR_HAVE_NATMC32_SET_WB 1 +# endif +# undef ETHR_HAVE_NATMC32_SET_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_ACQB +# define ETHR_HAVE_NATMC32_SET_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC32_SET_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RELB +# define ETHR_HAVE_NATMC32_SET_RELB 1 +# endif +# undef ETHR_HAVE_NATMC32_SET_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_MB +# define ETHR_HAVE_NATMC32_SET_MB 1 +# endif +# undef ETHR_HAVE_NATMC32_INIT +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT +# define ETHR_HAVE_NATMC32_INIT 1 +# endif +# undef ETHR_HAVE_NATMC32_INIT_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_RB +# define ETHR_HAVE_NATMC32_INIT_RB 1 +# endif +# undef ETHR_HAVE_NATMC32_INIT_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_WB +# define ETHR_HAVE_NATMC32_INIT_WB 1 +# endif +# undef ETHR_HAVE_NATMC32_INIT_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_ACQB +# define ETHR_HAVE_NATMC32_INIT_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC32_INIT_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_RELB +# define ETHR_HAVE_NATMC32_INIT_RELB 1 +# endif +# undef ETHR_HAVE_NATMC32_INIT_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_MB +# define ETHR_HAVE_NATMC32_INIT_MB 1 +# endif +# undef ETHR_HAVE_NATMC32_ADD_RETURN +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN +# define ETHR_HAVE_NATMC32_ADD_RETURN 1 +# endif +# undef ETHR_HAVE_NATMC32_ADD_RETURN_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_RB +# define ETHR_HAVE_NATMC32_ADD_RETURN_RB 1 +# endif +# undef ETHR_HAVE_NATMC32_ADD_RETURN_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_WB +# define ETHR_HAVE_NATMC32_ADD_RETURN_WB 1 +# endif +# undef ETHR_HAVE_NATMC32_ADD_RETURN_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_ACQB +# define ETHR_HAVE_NATMC32_ADD_RETURN_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC32_ADD_RETURN_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_RELB +# define ETHR_HAVE_NATMC32_ADD_RETURN_RELB 1 +# endif +# undef ETHR_HAVE_NATMC32_ADD_RETURN_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_MB +# define ETHR_HAVE_NATMC32_ADD_RETURN_MB 1 +# endif +# undef ETHR_HAVE_NATMC32_READ +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ +# define ETHR_HAVE_NATMC32_READ 1 +# endif +# undef ETHR_HAVE_NATMC32_READ_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_RB +# define ETHR_HAVE_NATMC32_READ_RB 1 +# endif +# undef ETHR_HAVE_NATMC32_READ_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_WB +# define ETHR_HAVE_NATMC32_READ_WB 1 +# endif +# undef ETHR_HAVE_NATMC32_READ_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_ACQB +# define ETHR_HAVE_NATMC32_READ_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC32_READ_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_RELB +# define ETHR_HAVE_NATMC32_READ_RELB 1 +# endif +# undef ETHR_HAVE_NATMC32_READ_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_MB +# define ETHR_HAVE_NATMC32_READ_MB 1 +# endif +# undef ETHR_HAVE_NATMC32_INC_RETURN +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN +# define ETHR_HAVE_NATMC32_INC_RETURN 1 +# endif +# undef ETHR_HAVE_NATMC32_INC_RETURN_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_RB +# define ETHR_HAVE_NATMC32_INC_RETURN_RB 1 +# endif +# undef ETHR_HAVE_NATMC32_INC_RETURN_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_WB +# define ETHR_HAVE_NATMC32_INC_RETURN_WB 1 +# endif +# undef ETHR_HAVE_NATMC32_INC_RETURN_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_ACQB +# define ETHR_HAVE_NATMC32_INC_RETURN_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC32_INC_RETURN_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_RELB +# define ETHR_HAVE_NATMC32_INC_RETURN_RELB 1 +# endif +# undef ETHR_HAVE_NATMC32_INC_RETURN_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_MB +# define ETHR_HAVE_NATMC32_INC_RETURN_MB 1 +# endif +# undef ETHR_HAVE_NATMC32_DEC_RETURN +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN +# define ETHR_HAVE_NATMC32_DEC_RETURN 1 +# endif +# undef ETHR_HAVE_NATMC32_DEC_RETURN_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_RB +# define ETHR_HAVE_NATMC32_DEC_RETURN_RB 1 +# endif +# undef ETHR_HAVE_NATMC32_DEC_RETURN_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_WB +# define ETHR_HAVE_NATMC32_DEC_RETURN_WB 1 +# endif +# undef ETHR_HAVE_NATMC32_DEC_RETURN_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_ACQB +# define ETHR_HAVE_NATMC32_DEC_RETURN_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC32_DEC_RETURN_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_RELB +# define ETHR_HAVE_NATMC32_DEC_RETURN_RELB 1 +# endif +# undef ETHR_HAVE_NATMC32_DEC_RETURN_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_MB +# define ETHR_HAVE_NATMC32_DEC_RETURN_MB 1 +# endif +# undef ETHR_HAVE_NATMC32_ADD +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD +# define ETHR_HAVE_NATMC32_ADD 1 +# endif +# undef ETHR_HAVE_NATMC32_ADD_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RB +# define ETHR_HAVE_NATMC32_ADD_RB 1 +# endif +# undef ETHR_HAVE_NATMC32_ADD_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_WB +# define ETHR_HAVE_NATMC32_ADD_WB 1 +# endif +# undef ETHR_HAVE_NATMC32_ADD_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_ACQB +# define ETHR_HAVE_NATMC32_ADD_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC32_ADD_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RELB +# define ETHR_HAVE_NATMC32_ADD_RELB 1 +# endif +# undef ETHR_HAVE_NATMC32_ADD_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_MB +# define ETHR_HAVE_NATMC32_ADD_MB 1 +# endif +# undef ETHR_HAVE_NATMC32_INC +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC +# define ETHR_HAVE_NATMC32_INC 1 +# endif +# undef ETHR_HAVE_NATMC32_INC_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RB +# define ETHR_HAVE_NATMC32_INC_RB 1 +# endif +# undef ETHR_HAVE_NATMC32_INC_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_WB +# define ETHR_HAVE_NATMC32_INC_WB 1 +# endif +# undef ETHR_HAVE_NATMC32_INC_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_ACQB +# define ETHR_HAVE_NATMC32_INC_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC32_INC_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RELB +# define ETHR_HAVE_NATMC32_INC_RELB 1 +# endif +# undef ETHR_HAVE_NATMC32_INC_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_MB +# define ETHR_HAVE_NATMC32_INC_MB 1 +# endif +# undef ETHR_HAVE_NATMC32_DEC +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC +# define ETHR_HAVE_NATMC32_DEC 1 +# endif +# undef ETHR_HAVE_NATMC32_DEC_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RB +# define ETHR_HAVE_NATMC32_DEC_RB 1 +# endif +# undef ETHR_HAVE_NATMC32_DEC_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_WB +# define ETHR_HAVE_NATMC32_DEC_WB 1 +# endif +# undef ETHR_HAVE_NATMC32_DEC_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_ACQB +# define ETHR_HAVE_NATMC32_DEC_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC32_DEC_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RELB +# define ETHR_HAVE_NATMC32_DEC_RELB 1 +# endif +# undef ETHR_HAVE_NATMC32_DEC_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_MB +# define ETHR_HAVE_NATMC32_DEC_MB 1 +# endif +# undef ETHR_HAVE_NATMC32_AND_RETOLD +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD +# define ETHR_HAVE_NATMC32_AND_RETOLD 1 +# endif +# undef ETHR_HAVE_NATMC32_AND_RETOLD_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_RB +# define ETHR_HAVE_NATMC32_AND_RETOLD_RB 1 +# endif +# undef ETHR_HAVE_NATMC32_AND_RETOLD_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_WB +# define ETHR_HAVE_NATMC32_AND_RETOLD_WB 1 +# endif +# undef ETHR_HAVE_NATMC32_AND_RETOLD_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_ACQB +# define ETHR_HAVE_NATMC32_AND_RETOLD_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC32_AND_RETOLD_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_RELB +# define ETHR_HAVE_NATMC32_AND_RETOLD_RELB 1 +# endif +# undef ETHR_HAVE_NATMC32_AND_RETOLD_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_MB +# define ETHR_HAVE_NATMC32_AND_RETOLD_MB 1 +# endif +# undef ETHR_HAVE_NATMC32_OR_RETOLD +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD +# define ETHR_HAVE_NATMC32_OR_RETOLD 1 +# endif +# undef ETHR_HAVE_NATMC32_OR_RETOLD_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_RB +# define ETHR_HAVE_NATMC32_OR_RETOLD_RB 1 +# endif +# undef ETHR_HAVE_NATMC32_OR_RETOLD_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_WB +# define ETHR_HAVE_NATMC32_OR_RETOLD_WB 1 +# endif +# undef ETHR_HAVE_NATMC32_OR_RETOLD_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_ACQB +# define ETHR_HAVE_NATMC32_OR_RETOLD_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC32_OR_RETOLD_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_RELB +# define ETHR_HAVE_NATMC32_OR_RETOLD_RELB 1 +# endif +# undef ETHR_HAVE_NATMC32_OR_RETOLD_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_MB +# define ETHR_HAVE_NATMC32_OR_RETOLD_MB 1 +# endif +#elif ETHR_NATMC32_BITS__ == 32 +# undef ETHR_HAVE_NATMC32_CMPXCHG +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG +# define ETHR_HAVE_NATMC32_CMPXCHG 1 +# endif +# undef ETHR_HAVE_NATMC32_CMPXCHG_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_RB +# define ETHR_HAVE_NATMC32_CMPXCHG_RB 1 +# endif +# undef ETHR_HAVE_NATMC32_CMPXCHG_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_WB +# define ETHR_HAVE_NATMC32_CMPXCHG_WB 1 +# endif +# undef ETHR_HAVE_NATMC32_CMPXCHG_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_ACQB +# define ETHR_HAVE_NATMC32_CMPXCHG_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC32_CMPXCHG_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_RELB +# define ETHR_HAVE_NATMC32_CMPXCHG_RELB 1 +# endif +# undef ETHR_HAVE_NATMC32_CMPXCHG_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_MB +# define ETHR_HAVE_NATMC32_CMPXCHG_MB 1 +# endif +# undef ETHR_HAVE_NATMC32_XCHG +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG +# define ETHR_HAVE_NATMC32_XCHG 1 +# endif +# undef ETHR_HAVE_NATMC32_XCHG_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_RB +# define ETHR_HAVE_NATMC32_XCHG_RB 1 +# endif +# undef ETHR_HAVE_NATMC32_XCHG_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_WB +# define ETHR_HAVE_NATMC32_XCHG_WB 1 +# endif +# undef ETHR_HAVE_NATMC32_XCHG_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_ACQB +# define ETHR_HAVE_NATMC32_XCHG_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC32_XCHG_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_RELB +# define ETHR_HAVE_NATMC32_XCHG_RELB 1 +# endif +# undef ETHR_HAVE_NATMC32_XCHG_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_MB +# define ETHR_HAVE_NATMC32_XCHG_MB 1 +# endif +# undef ETHR_HAVE_NATMC32_SET +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET +# define ETHR_HAVE_NATMC32_SET 1 +# endif +# undef ETHR_HAVE_NATMC32_SET_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_RB +# define ETHR_HAVE_NATMC32_SET_RB 1 +# endif +# undef ETHR_HAVE_NATMC32_SET_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_WB +# define ETHR_HAVE_NATMC32_SET_WB 1 +# endif +# undef ETHR_HAVE_NATMC32_SET_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_ACQB +# define ETHR_HAVE_NATMC32_SET_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC32_SET_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_RELB +# define ETHR_HAVE_NATMC32_SET_RELB 1 +# endif +# undef ETHR_HAVE_NATMC32_SET_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_MB +# define ETHR_HAVE_NATMC32_SET_MB 1 +# endif +# undef ETHR_HAVE_NATMC32_INIT +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT +# define ETHR_HAVE_NATMC32_INIT 1 +# endif +# undef ETHR_HAVE_NATMC32_INIT_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_RB +# define ETHR_HAVE_NATMC32_INIT_RB 1 +# endif +# undef ETHR_HAVE_NATMC32_INIT_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_WB +# define ETHR_HAVE_NATMC32_INIT_WB 1 +# endif +# undef ETHR_HAVE_NATMC32_INIT_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_ACQB +# define ETHR_HAVE_NATMC32_INIT_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC32_INIT_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_RELB +# define ETHR_HAVE_NATMC32_INIT_RELB 1 +# endif +# undef ETHR_HAVE_NATMC32_INIT_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_MB +# define ETHR_HAVE_NATMC32_INIT_MB 1 +# endif +# undef ETHR_HAVE_NATMC32_ADD_RETURN +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN +# define ETHR_HAVE_NATMC32_ADD_RETURN 1 +# endif +# undef ETHR_HAVE_NATMC32_ADD_RETURN_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_RB +# define ETHR_HAVE_NATMC32_ADD_RETURN_RB 1 +# endif +# undef ETHR_HAVE_NATMC32_ADD_RETURN_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_WB +# define ETHR_HAVE_NATMC32_ADD_RETURN_WB 1 +# endif +# undef ETHR_HAVE_NATMC32_ADD_RETURN_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_ACQB +# define ETHR_HAVE_NATMC32_ADD_RETURN_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC32_ADD_RETURN_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_RELB +# define ETHR_HAVE_NATMC32_ADD_RETURN_RELB 1 +# endif +# undef ETHR_HAVE_NATMC32_ADD_RETURN_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_MB +# define ETHR_HAVE_NATMC32_ADD_RETURN_MB 1 +# endif +# undef ETHR_HAVE_NATMC32_READ +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ +# define ETHR_HAVE_NATMC32_READ 1 +# endif +# undef ETHR_HAVE_NATMC32_READ_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_RB +# define ETHR_HAVE_NATMC32_READ_RB 1 +# endif +# undef ETHR_HAVE_NATMC32_READ_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_WB +# define ETHR_HAVE_NATMC32_READ_WB 1 +# endif +# undef ETHR_HAVE_NATMC32_READ_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_ACQB +# define ETHR_HAVE_NATMC32_READ_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC32_READ_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_RELB +# define ETHR_HAVE_NATMC32_READ_RELB 1 +# endif +# undef ETHR_HAVE_NATMC32_READ_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_MB +# define ETHR_HAVE_NATMC32_READ_MB 1 +# endif +# undef ETHR_HAVE_NATMC32_INC_RETURN +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN +# define ETHR_HAVE_NATMC32_INC_RETURN 1 +# endif +# undef ETHR_HAVE_NATMC32_INC_RETURN_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_RB +# define ETHR_HAVE_NATMC32_INC_RETURN_RB 1 +# endif +# undef ETHR_HAVE_NATMC32_INC_RETURN_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_WB +# define ETHR_HAVE_NATMC32_INC_RETURN_WB 1 +# endif +# undef ETHR_HAVE_NATMC32_INC_RETURN_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_ACQB +# define ETHR_HAVE_NATMC32_INC_RETURN_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC32_INC_RETURN_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_RELB +# define ETHR_HAVE_NATMC32_INC_RETURN_RELB 1 +# endif +# undef ETHR_HAVE_NATMC32_INC_RETURN_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_MB +# define ETHR_HAVE_NATMC32_INC_RETURN_MB 1 +# endif +# undef ETHR_HAVE_NATMC32_DEC_RETURN +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN +# define ETHR_HAVE_NATMC32_DEC_RETURN 1 +# endif +# undef ETHR_HAVE_NATMC32_DEC_RETURN_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_RB +# define ETHR_HAVE_NATMC32_DEC_RETURN_RB 1 +# endif +# undef ETHR_HAVE_NATMC32_DEC_RETURN_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_WB +# define ETHR_HAVE_NATMC32_DEC_RETURN_WB 1 +# endif +# undef ETHR_HAVE_NATMC32_DEC_RETURN_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_ACQB +# define ETHR_HAVE_NATMC32_DEC_RETURN_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC32_DEC_RETURN_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_RELB +# define ETHR_HAVE_NATMC32_DEC_RETURN_RELB 1 +# endif +# undef ETHR_HAVE_NATMC32_DEC_RETURN_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_MB +# define ETHR_HAVE_NATMC32_DEC_RETURN_MB 1 +# endif +# undef ETHR_HAVE_NATMC32_ADD +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD +# define ETHR_HAVE_NATMC32_ADD 1 +# endif +# undef ETHR_HAVE_NATMC32_ADD_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RB +# define ETHR_HAVE_NATMC32_ADD_RB 1 +# endif +# undef ETHR_HAVE_NATMC32_ADD_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_WB +# define ETHR_HAVE_NATMC32_ADD_WB 1 +# endif +# undef ETHR_HAVE_NATMC32_ADD_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_ACQB +# define ETHR_HAVE_NATMC32_ADD_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC32_ADD_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RELB +# define ETHR_HAVE_NATMC32_ADD_RELB 1 +# endif +# undef ETHR_HAVE_NATMC32_ADD_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_MB +# define ETHR_HAVE_NATMC32_ADD_MB 1 +# endif +# undef ETHR_HAVE_NATMC32_INC +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC +# define ETHR_HAVE_NATMC32_INC 1 +# endif +# undef ETHR_HAVE_NATMC32_INC_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RB +# define ETHR_HAVE_NATMC32_INC_RB 1 +# endif +# undef ETHR_HAVE_NATMC32_INC_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_WB +# define ETHR_HAVE_NATMC32_INC_WB 1 +# endif +# undef ETHR_HAVE_NATMC32_INC_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_ACQB +# define ETHR_HAVE_NATMC32_INC_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC32_INC_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RELB +# define ETHR_HAVE_NATMC32_INC_RELB 1 +# endif +# undef ETHR_HAVE_NATMC32_INC_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_MB +# define ETHR_HAVE_NATMC32_INC_MB 1 +# endif +# undef ETHR_HAVE_NATMC32_DEC +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC +# define ETHR_HAVE_NATMC32_DEC 1 +# endif +# undef ETHR_HAVE_NATMC32_DEC_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RB +# define ETHR_HAVE_NATMC32_DEC_RB 1 +# endif +# undef ETHR_HAVE_NATMC32_DEC_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_WB +# define ETHR_HAVE_NATMC32_DEC_WB 1 +# endif +# undef ETHR_HAVE_NATMC32_DEC_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_ACQB +# define ETHR_HAVE_NATMC32_DEC_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC32_DEC_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RELB +# define ETHR_HAVE_NATMC32_DEC_RELB 1 +# endif +# undef ETHR_HAVE_NATMC32_DEC_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_MB +# define ETHR_HAVE_NATMC32_DEC_MB 1 +# endif +# undef ETHR_HAVE_NATMC32_AND_RETOLD +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD +# define ETHR_HAVE_NATMC32_AND_RETOLD 1 +# endif +# undef ETHR_HAVE_NATMC32_AND_RETOLD_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_RB +# define ETHR_HAVE_NATMC32_AND_RETOLD_RB 1 +# endif +# undef ETHR_HAVE_NATMC32_AND_RETOLD_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_WB +# define ETHR_HAVE_NATMC32_AND_RETOLD_WB 1 +# endif +# undef ETHR_HAVE_NATMC32_AND_RETOLD_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_ACQB +# define ETHR_HAVE_NATMC32_AND_RETOLD_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC32_AND_RETOLD_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_RELB +# define ETHR_HAVE_NATMC32_AND_RETOLD_RELB 1 +# endif +# undef ETHR_HAVE_NATMC32_AND_RETOLD_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_MB +# define ETHR_HAVE_NATMC32_AND_RETOLD_MB 1 +# endif +# undef ETHR_HAVE_NATMC32_OR_RETOLD +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD +# define ETHR_HAVE_NATMC32_OR_RETOLD 1 +# endif +# undef ETHR_HAVE_NATMC32_OR_RETOLD_RB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_RB +# define ETHR_HAVE_NATMC32_OR_RETOLD_RB 1 +# endif +# undef ETHR_HAVE_NATMC32_OR_RETOLD_WB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_WB +# define ETHR_HAVE_NATMC32_OR_RETOLD_WB 1 +# endif +# undef ETHR_HAVE_NATMC32_OR_RETOLD_ACQB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_ACQB +# define ETHR_HAVE_NATMC32_OR_RETOLD_ACQB 1 +# endif +# undef ETHR_HAVE_NATMC32_OR_RETOLD_RELB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_RELB +# define ETHR_HAVE_NATMC32_OR_RETOLD_RELB 1 +# endif +# undef ETHR_HAVE_NATMC32_OR_RETOLD_MB +# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_MB +# define ETHR_HAVE_NATMC32_OR_RETOLD_MB 1 +# endif +#else +# error "Invalid native atomic size" +#endif + +#if (!defined(ETHR_HAVE_NATMC32_CMPXCHG) \ + && !defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) \ + && !defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) \ + && !defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) \ + && !defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) \ + && !defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)) +# error "No native cmpxchg() op available" +#endif + + /* - * --- 32-bit atomics --------------------------------------------------------- + * Read op used together with cmpxchg() fallback when no native op present. */ +#if defined(ETHR_HAVE_NATMC32_READ) +#define ETHR_NATMC32_CMPXCHG_FALLBACK_READ__(VAR) \ + ETHR_NATMC32_FUNC__(read)(VAR) +#elif defined(ETHR_HAVE_NATMC32_READ_RB) +#define ETHR_NATMC32_CMPXCHG_FALLBACK_READ__(VAR) \ + ETHR_NATMC32_FUNC__(read_rb)(VAR) +#elif defined(ETHR_HAVE_NATMC32_READ_WB) +#define ETHR_NATMC32_CMPXCHG_FALLBACK_READ__(VAR) \ + ETHR_NATMC32_FUNC__(read_wb)(VAR) +#elif defined(ETHR_HAVE_NATMC32_READ_ACQB) +#define ETHR_NATMC32_CMPXCHG_FALLBACK_READ__(VAR) \ + ETHR_NATMC32_FUNC__(read_acqb)(VAR) +#elif defined(ETHR_HAVE_NATMC32_READ_RELB) +#define ETHR_NATMC32_CMPXCHG_FALLBACK_READ__(VAR) \ + ETHR_NATMC32_FUNC__(read_relb)(VAR) +#elif defined(ETHR_HAVE_NATMC32_READ_MB) +#define ETHR_NATMC32_CMPXCHG_FALLBACK_READ__(VAR) \ + ETHR_NATMC32_FUNC__(read_mb)(VAR) +#else +/* + * We have no native read() op; guess zero and then use the + * the atomics actual value returned from cmpxchg(). + */ +#define ETHR_NATMC32_CMPXCHG_FALLBACK_READ__(VAR) \ + ((ETHR_NAINT32_T__) 0) +#endif + +/* + * Native cmpxchg() fallback used when no native op present. + */ +#define ETHR_NATMC32_CMPXCHG_FALLBACK__(CMPXCHG, VAR, AVAL, OPS) \ +do { \ + ethr_sint32_t AVAL; \ + ETHR_NAINT32_T__ new__, act__, exp__; \ + act__ = ETHR_NATMC32_CMPXCHG_FALLBACK_READ__(VAR); \ + do { \ + exp__ = act__; \ + AVAL = (ethr_sint32_t) act__; \ + { OPS; } \ + new__ = (ETHR_NAINT32_T__) AVAL; \ + act__ = CMPXCHG(VAR, new__, exp__); \ + } while (__builtin_expect(act__ != exp__, 0)); \ +} while (0) -static ETHR_INLINE ethr_sint32_t * -ETHR_INLINE_FUNC_NAME_(ethr_atomic32_addr)(ethr_atomic32_t *var) + + +/* --- addr() --- */ + +static ETHR_INLINE ethr_sint32_t *ETHR_ATMC32_FUNC__(addr)(ethr_atomic32_t *var) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return ethr_native_atomic32_addr(var); + return (ethr_sint32_t *) ETHR_NATMC32_ADDR_FUNC__(var); + +} + + +/* --- cmpxchg() --- */ + + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(cmpxchg)(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val) +{ + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_CMPXCHG) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); #else - return (ethr_sint32_t *) var; +#error "Missing implementation of ethr_atomic32_cmpxchg()!" #endif + return res; } -static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_atomic32_init)(ethr_atomic32_t *var, - ethr_sint32_t i) +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(cmpxchg_rb)(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - ETHR_NATMC32_FUNC__(init)(var, (ETHR_NAINT32_T__) i); + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); + ETHR_MEMBAR(ETHR_LoadLoad); #else - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = i); +#error "Missing implementation of ethr_atomic32_cmpxchg_rb()!" #endif + return res; } -static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_atomic32_set)(ethr_atomic32_t *var, ethr_sint32_t i) +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(cmpxchg_wb)(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - ETHR_NATMC32_FUNC__(set)(var, (ETHR_NAINT32_T__) i); + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); #else - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = i); +#error "Missing implementation of ethr_atomic32_cmpxchg_wb()!" #endif + return res; } -static ETHR_INLINE ethr_sint32_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic32_read)(ethr_atomic32_t *var) +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(cmpxchg_acqb)(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return (ethr_sint32_t) ETHR_NATMC32_FUNC__(read)(var); + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); #else +#error "Missing implementation of ethr_atomic32_cmpxchg_acqb()!" +#endif + return res; +} + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(cmpxchg_relb)(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val) +{ ethr_sint32_t res; - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var); +#if defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); +#else +#error "Missing implementation of ethr_atomic32_cmpxchg_relb()!" +#endif return res; +} + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(cmpxchg_mb)(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val) +{ + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else +#error "Missing implementation of ethr_atomic32_cmpxchg_mb()!" #endif + return res; } -static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_atomic32_add)(ethr_atomic32_t *var, - ethr_sint32_t incr) + +/* --- xchg() --- */ + + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(xchg)(ethr_atomic32_t *var, ethr_sint32_t val) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - ETHR_NATMC32_FUNC__(add)(var, (ETHR_NAINT32_T__) incr); + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_XCHG) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_XCHG_RB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_rb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_XCHG_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_wb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_XCHG_ACQB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_XCHG_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_relb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_XCHG_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val); #else - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += incr); +#error "Missing implementation of ethr_atomic32_xchg()!" #endif -} - -static ETHR_INLINE ethr_sint32_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic32_add_read)(ethr_atomic32_t *var, - ethr_sint32_t i) + return res; +} + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(xchg_rb)(ethr_atomic32_t *var, ethr_sint32_t val) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return (ethr_sint32_t) - ETHR_NATMC32_FUNC__(add_return)(var, (ETHR_NAINT32_T__) i); + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_XCHG_RB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_rb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_XCHG) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_XCHG_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_XCHG_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_wb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_XCHG_ACQB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_acqb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_XCHG_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_relb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval = val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val); + ETHR_MEMBAR(ETHR_LoadLoad); #else +#error "Missing implementation of ethr_atomic32_xchg_rb()!" +#endif + return res; +} + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(xchg_wb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ ethr_sint32_t res; - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += i; res = *var); +#if defined(ETHR_HAVE_NATMC32_XCHG_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_wb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_XCHG) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_XCHG_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_XCHG_RB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_rb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_XCHG_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_XCHG_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_relb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val); +#else +#error "Missing implementation of ethr_atomic32_xchg_wb()!" +#endif return res; +} + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(xchg_acqb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_XCHG_ACQB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_XCHG_RB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_rb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_XCHG) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_XCHG_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_XCHG_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_wb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_XCHG_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_relb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval = val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else +#error "Missing implementation of ethr_atomic32_xchg_acqb()!" #endif + return res; } -static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_atomic32_inc)(ethr_atomic32_t *var) +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(xchg_relb)(ethr_atomic32_t *var, ethr_sint32_t val) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - ETHR_NATMC32_FUNC__(inc)(var); + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_XCHG_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_relb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_XCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_wb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_XCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_XCHG_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_XCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_rb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_XCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val); #else - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var)); +#error "Missing implementation of ethr_atomic32_xchg_relb()!" #endif + return res; } -static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_atomic32_dec)(ethr_atomic32_t *var) +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(xchg_mb)(ethr_atomic32_t *var, ethr_sint32_t val) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - ETHR_NATMC32_FUNC__(dec)(var); + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_XCHG_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_XCHG_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_relb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_XCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_XCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_wb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_XCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_rb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_XCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval = val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else +#error "Missing implementation of ethr_atomic32_xchg_mb()!" +#endif + return res; +} + + +/* --- set() --- */ + + +static ETHR_INLINE void ETHR_ATMC32_FUNC__(set)(ethr_atomic32_t *var, ethr_sint32_t val) +{ +#if defined(ETHR_HAVE_NATMC32_SET) + ETHR_NATMC32_FUNC__(set)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_SET_RB) + ETHR_NATMC32_FUNC__(set_rb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_SET_WB) + ETHR_NATMC32_FUNC__(set_wb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_SET_ACQB) + ETHR_NATMC32_FUNC__(set_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_SET_RELB) + ETHR_NATMC32_FUNC__(set_relb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_SET_MB) + ETHR_NATMC32_FUNC__(set_mb)(var, (ETHR_NAINT32_T__) val); +#else + (void) ETHR_ATMC32_FUNC__(xchg)(var, val); +#endif +} + +static ETHR_INLINE void ETHR_ATMC32_FUNC__(set_rb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ +#if defined(ETHR_HAVE_NATMC32_SET_RB) + ETHR_NATMC32_FUNC__(set_rb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_SET) + ETHR_NATMC32_FUNC__(set)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_SET_MB) + ETHR_NATMC32_FUNC__(set_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_SET_WB) + ETHR_NATMC32_FUNC__(set_wb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_SET_ACQB) + ETHR_NATMC32_FUNC__(set_acqb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_SET_RELB) + ETHR_NATMC32_FUNC__(set_relb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + (void) ETHR_ATMC32_FUNC__(xchg_rb)(var, val); +#endif +} + +static ETHR_INLINE void ETHR_ATMC32_FUNC__(set_wb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ +#if defined(ETHR_HAVE_NATMC32_SET_WB) + ETHR_NATMC32_FUNC__(set_wb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_SET) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_FUNC__(set)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_SET_MB) + ETHR_NATMC32_FUNC__(set_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_SET_RB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_FUNC__(set_rb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_SET_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_FUNC__(set_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_SET_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_FUNC__(set_relb)(var, (ETHR_NAINT32_T__) val); +#else + (void) ETHR_ATMC32_FUNC__(xchg_wb)(var, val); +#endif +} + +static ETHR_INLINE void ETHR_ATMC32_FUNC__(set_acqb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ +#if defined(ETHR_HAVE_NATMC32_SET_ACQB) + ETHR_NATMC32_FUNC__(set_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_SET_RB) + ETHR_NATMC32_FUNC__(set_rb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_SET) + ETHR_NATMC32_FUNC__(set)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_SET_MB) + ETHR_NATMC32_FUNC__(set_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_SET_WB) + ETHR_NATMC32_FUNC__(set_wb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_SET_RELB) + ETHR_NATMC32_FUNC__(set_relb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#else + (void) ETHR_ATMC32_FUNC__(xchg_acqb)(var, val); +#endif +} + +static ETHR_INLINE void ETHR_ATMC32_FUNC__(set_relb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ +#if defined(ETHR_HAVE_NATMC32_SET_RELB) + ETHR_NATMC32_FUNC__(set_relb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_SET_WB) + ETHR_MEMBAR(ETHR_LoadStore); + ETHR_NATMC32_FUNC__(set_wb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_SET) + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_NATMC32_FUNC__(set)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_SET_MB) + ETHR_NATMC32_FUNC__(set_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_SET_RB) + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_NATMC32_FUNC__(set_rb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_SET_ACQB) + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_NATMC32_FUNC__(set_acqb)(var, (ETHR_NAINT32_T__) val); +#else + (void) ETHR_ATMC32_FUNC__(xchg_relb)(var, val); +#endif +} + +static ETHR_INLINE void ETHR_ATMC32_FUNC__(set_mb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ +#if defined(ETHR_HAVE_NATMC32_SET_MB) + ETHR_NATMC32_FUNC__(set_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_SET_RELB) + ETHR_NATMC32_FUNC__(set_relb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_SET_ACQB) + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_NATMC32_FUNC__(set_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_SET_WB) + ETHR_MEMBAR(ETHR_LoadStore); + ETHR_NATMC32_FUNC__(set_wb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_SET_RB) + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_NATMC32_FUNC__(set_rb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_SET) + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_NATMC32_FUNC__(set)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#else + (void) ETHR_ATMC32_FUNC__(xchg_mb)(var, val); +#endif +} + + +/* --- init() --- */ + + +static ETHR_INLINE void ETHR_ATMC32_FUNC__(init)(ethr_atomic32_t *var, ethr_sint32_t val) +{ +#if defined(ETHR_HAVE_NATMC32_INIT) + ETHR_NATMC32_FUNC__(init)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_INIT_RB) + ETHR_NATMC32_FUNC__(init_rb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_INIT_WB) + ETHR_NATMC32_FUNC__(init_wb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_INIT_ACQB) + ETHR_NATMC32_FUNC__(init_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_INIT_RELB) + ETHR_NATMC32_FUNC__(init_relb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_INIT_MB) + ETHR_NATMC32_FUNC__(init_mb)(var, (ETHR_NAINT32_T__) val); #else - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var)); + ETHR_ATMC32_FUNC__(set)(var, val); #endif } -static ETHR_INLINE ethr_sint32_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic32_inc_read)(ethr_atomic32_t *var) +static ETHR_INLINE void ETHR_ATMC32_FUNC__(init_rb)(ethr_atomic32_t *var, ethr_sint32_t val) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return)(var); +#if defined(ETHR_HAVE_NATMC32_INIT_RB) + ETHR_NATMC32_FUNC__(init_rb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_INIT) + ETHR_NATMC32_FUNC__(init)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_INIT_MB) + ETHR_NATMC32_FUNC__(init_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_INIT_WB) + ETHR_NATMC32_FUNC__(init_wb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_INIT_ACQB) + ETHR_NATMC32_FUNC__(init_acqb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_INIT_RELB) + ETHR_NATMC32_FUNC__(init_relb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); #else + ETHR_ATMC32_FUNC__(set_rb)(var, val); +#endif +} + +static ETHR_INLINE void ETHR_ATMC32_FUNC__(init_wb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ +#if defined(ETHR_HAVE_NATMC32_INIT_WB) + ETHR_NATMC32_FUNC__(init_wb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_INIT) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_FUNC__(init)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_INIT_MB) + ETHR_NATMC32_FUNC__(init_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_INIT_RB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_FUNC__(init_rb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_INIT_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_FUNC__(init_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_INIT_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_FUNC__(init_relb)(var, (ETHR_NAINT32_T__) val); +#else + ETHR_ATMC32_FUNC__(set_wb)(var, val); +#endif +} + +static ETHR_INLINE void ETHR_ATMC32_FUNC__(init_acqb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ +#if defined(ETHR_HAVE_NATMC32_INIT_ACQB) + ETHR_NATMC32_FUNC__(init_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_INIT_RB) + ETHR_NATMC32_FUNC__(init_rb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_INIT) + ETHR_NATMC32_FUNC__(init)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_INIT_MB) + ETHR_NATMC32_FUNC__(init_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_INIT_WB) + ETHR_NATMC32_FUNC__(init_wb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_INIT_RELB) + ETHR_NATMC32_FUNC__(init_relb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_ATMC32_FUNC__(set_acqb)(var, val); +#endif +} + +static ETHR_INLINE void ETHR_ATMC32_FUNC__(init_relb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ +#if defined(ETHR_HAVE_NATMC32_INIT_RELB) + ETHR_NATMC32_FUNC__(init_relb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_INIT_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad); + ETHR_NATMC32_FUNC__(init_wb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_INIT) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_NATMC32_FUNC__(init)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_INIT_MB) + ETHR_NATMC32_FUNC__(init_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_INIT_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_NATMC32_FUNC__(init_rb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_INIT_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_NATMC32_FUNC__(init_acqb)(var, (ETHR_NAINT32_T__) val); +#else + ETHR_ATMC32_FUNC__(set_relb)(var, val); +#endif +} + +static ETHR_INLINE void ETHR_ATMC32_FUNC__(init_mb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ +#if defined(ETHR_HAVE_NATMC32_INIT_MB) + ETHR_NATMC32_FUNC__(init_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_INIT_RELB) + ETHR_NATMC32_FUNC__(init_relb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_INIT_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_NATMC32_FUNC__(init_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_INIT_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad); + ETHR_NATMC32_FUNC__(init_wb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_INIT_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_NATMC32_FUNC__(init_rb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_INIT) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_NATMC32_FUNC__(init)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_ATMC32_FUNC__(set_mb)(var, val); +#endif +} + + +/* --- add_read() --- */ + + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(add_read)(ethr_atomic32_t *var, ethr_sint32_t val) +{ ethr_sint32_t res; - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var)); +#if defined(ETHR_HAVE_NATMC32_ADD_RETURN) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_rb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_wb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_ACQB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_relb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval); +#else +#error "Missing implementation of ethr_atomic32_add_read()!" +#endif return res; +} + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(add_read_rb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_ADD_RETURN_RB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_rb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_wb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_ACQB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_acqb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_relb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, aval += val; res = aval); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval); + ETHR_MEMBAR(ETHR_LoadLoad); +#else +#error "Missing implementation of ethr_atomic32_add_read_rb()!" #endif + return res; } -static ETHR_INLINE ethr_sint32_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic32_dec_read)(ethr_atomic32_t *var) +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(add_read_wb)(ethr_atomic32_t *var, ethr_sint32_t val) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return)(var); + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_ADD_RETURN_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_wb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_rb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_relb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval); #else +#error "Missing implementation of ethr_atomic32_add_read_wb()!" +#endif + return res; +} + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(add_read_acqb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ ethr_sint32_t res; - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var)); +#if defined(ETHR_HAVE_NATMC32_ADD_RETURN_ACQB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_rb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_wb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_relb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, aval += val; res = aval); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else +#error "Missing implementation of ethr_atomic32_add_read_acqb()!" +#endif return res; +} + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(add_read_relb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_ADD_RETURN_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_relb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_wb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_rb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval); +#else +#error "Missing implementation of ethr_atomic32_add_read_relb()!" #endif + return res; } -static ETHR_INLINE ethr_sint32_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic32_read_band)(ethr_atomic32_t *var, - ethr_sint32_t mask) +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(add_read_mb)(ethr_atomic32_t *var, ethr_sint32_t val) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return (ethr_sint32_t) - ETHR_NATMC32_FUNC__(and_retold)(var, (ETHR_NAINT32_T__) mask); + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_ADD_RETURN_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_relb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_wb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_rb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, aval += val; res = aval); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); #else +#error "Missing implementation of ethr_atomic32_add_read_mb()!" +#endif + return res; +} + + +/* --- read() --- */ + + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read)(ethr_atomic32_t *var) +{ ethr_sint32_t res; - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= mask); +#if defined(ETHR_HAVE_NATMC32_READ) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read)(var); +#elif defined(ETHR_HAVE_NATMC32_READ_RB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_rb)(var); +#elif defined(ETHR_HAVE_NATMC32_READ_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_wb)(var); +#elif defined(ETHR_HAVE_NATMC32_READ_ACQB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_acqb)(var); +#elif defined(ETHR_HAVE_NATMC32_READ_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_relb)(var); +#elif defined(ETHR_HAVE_NATMC32_READ_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_mb)(var); +#else + res = ETHR_ATMC32_FUNC__(cmpxchg)(var, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__); +#endif return res; +} + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_rb)(ethr_atomic32_t *var) +{ + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_READ_RB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_rb)(var); +#elif defined(ETHR_HAVE_NATMC32_READ) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_READ_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_mb)(var); +#elif defined(ETHR_HAVE_NATMC32_READ_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_READ_ACQB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_acqb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_READ_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + res = ETHR_ATMC32_FUNC__(cmpxchg_rb)(var, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__); #endif + return res; } -static ETHR_INLINE ethr_sint32_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic32_read_bor)(ethr_atomic32_t *var, - ethr_sint32_t mask) +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_wb)(ethr_atomic32_t *var) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return - (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold)(var, - (ETHR_NAINT32_T__) mask); + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_READ_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_wb)(var); +#elif defined(ETHR_HAVE_NATMC32_READ) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read)(var); +#elif defined(ETHR_HAVE_NATMC32_READ_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_mb)(var); +#elif defined(ETHR_HAVE_NATMC32_READ_RB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_rb)(var); +#elif defined(ETHR_HAVE_NATMC32_READ_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_acqb)(var); +#elif defined(ETHR_HAVE_NATMC32_READ_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_relb)(var); #else + res = ETHR_ATMC32_FUNC__(cmpxchg_wb)(var, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__); +#endif + return res; +} + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_acqb)(ethr_atomic32_t *var) +{ ethr_sint32_t res; - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= mask); +#if defined(ETHR_HAVE_NATMC32_READ_ACQB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_acqb)(var); +#elif defined(ETHR_HAVE_NATMC32_READ_RB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_rb)(var); + ETHR_MEMBAR(ETHR_LoadStore); +#elif defined(ETHR_HAVE_NATMC32_READ) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#elif defined(ETHR_HAVE_NATMC32_READ_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_mb)(var); +#elif defined(ETHR_HAVE_NATMC32_READ_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#elif defined(ETHR_HAVE_NATMC32_READ_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#else + res = ETHR_ATMC32_FUNC__(cmpxchg_acqb)(var, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__); +#endif return res; +} + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_relb)(ethr_atomic32_t *var) +{ + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_READ_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_relb)(var); +#elif defined(ETHR_HAVE_NATMC32_READ_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_wb)(var); +#elif defined(ETHR_HAVE_NATMC32_READ) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read)(var); +#elif defined(ETHR_HAVE_NATMC32_READ_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_mb)(var); +#elif defined(ETHR_HAVE_NATMC32_READ_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_rb)(var); +#elif defined(ETHR_HAVE_NATMC32_READ_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_acqb)(var); +#else + res = ETHR_ATMC32_FUNC__(cmpxchg_relb)(var, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__); #endif + return res; } -static ETHR_INLINE ethr_sint32_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic32_xchg)(ethr_atomic32_t *var, - ethr_sint32_t new) +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_mb)(ethr_atomic32_t *var) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg)(var, - (ETHR_NAINT32_T__) new); + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_READ_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_mb)(var); +#elif defined(ETHR_HAVE_NATMC32_READ_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#elif defined(ETHR_HAVE_NATMC32_READ_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_acqb)(var); +#elif defined(ETHR_HAVE_NATMC32_READ_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#elif defined(ETHR_HAVE_NATMC32_READ_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_rb)(var); + ETHR_MEMBAR(ETHR_LoadStore); +#elif defined(ETHR_HAVE_NATMC32_READ) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); #else + res = ETHR_ATMC32_FUNC__(cmpxchg_mb)(var, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__); +#endif + return res; +} + + +/* --- inc_read() --- */ + + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(inc_read)(ethr_atomic32_t *var) +{ ethr_sint32_t res; - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = new); +#if defined(ETHR_HAVE_NATMC32_INC_RETURN) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_rb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_wb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_ACQB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_acqb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_relb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_mb)(var); +#else + res = ETHR_ATMC32_FUNC__(add_read)(var, (ethr_sint32_t) 1); +#endif return res; +} + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(inc_read_rb)(ethr_atomic32_t *var) +{ + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_INC_RETURN_RB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_rb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_mb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_ACQB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_acqb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + res = ETHR_ATMC32_FUNC__(add_read_rb)(var, (ethr_sint32_t) 1); #endif + return res; } -static ETHR_INLINE ethr_sint32_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic32_cmpxchg)(ethr_atomic32_t *var, - ethr_sint32_t new, - ethr_sint32_t exp) +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(inc_read_wb)(ethr_atomic32_t *var) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg)(var, - (ETHR_NAINT32_T__) new, - (ETHR_NAINT32_T__) exp); + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_INC_RETURN_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_wb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_mb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_rb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_acqb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_relb)(var); #else + res = ETHR_ATMC32_FUNC__(add_read_wb)(var, (ethr_sint32_t) 1); +#endif + return res; +} + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(inc_read_acqb)(ethr_atomic32_t *var) +{ ethr_sint32_t res; - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, - { - res = *var; - if (__builtin_expect(res == exp, 1)) - *var = new; - }); +#if defined(ETHR_HAVE_NATMC32_INC_RETURN_ACQB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_acqb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_rb)(var); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_mb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + res = ETHR_ATMC32_FUNC__(add_read_acqb)(var, (ethr_sint32_t) 1); +#endif return res; +} + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(inc_read_relb)(ethr_atomic32_t *var) +{ + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_INC_RETURN_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_relb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_wb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_mb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_rb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_acqb)(var); +#else + res = ETHR_ATMC32_FUNC__(add_read_relb)(var, (ethr_sint32_t) 1); #endif + return res; +} + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(inc_read_mb)(ethr_atomic32_t *var) +{ + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_INC_RETURN_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_mb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_acqb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_rb)(var); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_INC_RETURN) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + res = ETHR_ATMC32_FUNC__(add_read_mb)(var, (ethr_sint32_t) 1); +#endif + return res; } -/* - * Important memory barrier requirements. - * - * The following atomic operations *must* supply a memory barrier of - * at least the type specified by its suffix: - * _acqb = acquire barrier - * _relb = release barrier - */ -static ETHR_INLINE ethr_sint32_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic32_read_acqb)(ethr_atomic32_t *var) +/* --- dec_read() --- */ + + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(dec_read)(ethr_atomic32_t *var) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_acqb)(var); + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_DEC_RETURN) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_rb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_wb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_ACQB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_acqb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_relb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_mb)(var); #else - return ETHR_INLINE_FUNC_NAME_(ethr_atomic32_read)(var); + res = ETHR_ATMC32_FUNC__(add_read)(var, (ethr_sint32_t) -1); #endif + return res; } -static ETHR_INLINE ethr_sint32_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic32_inc_read_acqb)(ethr_atomic32_t *var) +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(dec_read_rb)(ethr_atomic32_t *var) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_acqb)(var); + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_DEC_RETURN_RB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_rb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_mb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_ACQB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_acqb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); #else - return ETHR_INLINE_FUNC_NAME_(ethr_atomic32_inc_read)(var); + res = ETHR_ATMC32_FUNC__(add_read_rb)(var, (ethr_sint32_t) -1); #endif + return res; } -static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_atomic32_set_relb)(ethr_atomic32_t *var, - ethr_sint32_t val) +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(dec_read_wb)(ethr_atomic32_t *var) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - ETHR_NATMC32_FUNC__(set_relb)(var, (ETHR_NAINT32_T__) val); + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_DEC_RETURN_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_wb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_mb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_rb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_acqb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_relb)(var); +#else + res = ETHR_ATMC32_FUNC__(add_read_wb)(var, (ethr_sint32_t) -1); +#endif + return res; +} + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(dec_read_acqb)(ethr_atomic32_t *var) +{ + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_DEC_RETURN_ACQB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_acqb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_rb)(var); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_mb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); #else - ETHR_INLINE_FUNC_NAME_(ethr_atomic32_set)(var, val); + res = ETHR_ATMC32_FUNC__(add_read_acqb)(var, (ethr_sint32_t) -1); #endif + return res; } -static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_atomic32_dec_relb)(ethr_atomic32_t *var) +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(dec_read_relb)(ethr_atomic32_t *var) +{ + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_DEC_RETURN_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_relb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_wb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_mb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_rb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_acqb)(var); +#else + res = ETHR_ATMC32_FUNC__(add_read_relb)(var, (ethr_sint32_t) -1); +#endif + return res; +} + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(dec_read_mb)(ethr_atomic32_t *var) +{ + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_DEC_RETURN_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_mb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_acqb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_rb)(var); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + res = ETHR_ATMC32_FUNC__(add_read_mb)(var, (ethr_sint32_t) -1); +#endif + return res; +} + + +/* --- add() --- */ + + +static ETHR_INLINE void ETHR_ATMC32_FUNC__(add)(ethr_atomic32_t *var, ethr_sint32_t val) +{ +#if defined(ETHR_HAVE_NATMC32_ADD) + ETHR_NATMC32_FUNC__(add)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_RB) + ETHR_NATMC32_FUNC__(add_rb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_WB) + ETHR_NATMC32_FUNC__(add_wb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_ACQB) + ETHR_NATMC32_FUNC__(add_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_RELB) + ETHR_NATMC32_FUNC__(add_relb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_MB) + ETHR_NATMC32_FUNC__(add_mb)(var, (ETHR_NAINT32_T__) val); +#else + (void) ETHR_ATMC32_FUNC__(add_read)(var, val); +#endif +} + +static ETHR_INLINE void ETHR_ATMC32_FUNC__(add_rb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ +#if defined(ETHR_HAVE_NATMC32_ADD_RB) + ETHR_NATMC32_FUNC__(add_rb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD) + ETHR_NATMC32_FUNC__(add)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_ADD_MB) + ETHR_NATMC32_FUNC__(add_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_WB) + ETHR_NATMC32_FUNC__(add_wb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_ADD_ACQB) + ETHR_NATMC32_FUNC__(add_acqb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_ADD_RELB) + ETHR_NATMC32_FUNC__(add_relb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + (void) ETHR_ATMC32_FUNC__(add_read_rb)(var, val); +#endif +} + +static ETHR_INLINE void ETHR_ATMC32_FUNC__(add_wb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ +#if defined(ETHR_HAVE_NATMC32_ADD_WB) + ETHR_NATMC32_FUNC__(add_wb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_FUNC__(add)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_MB) + ETHR_NATMC32_FUNC__(add_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_RB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_FUNC__(add_rb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_FUNC__(add_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_FUNC__(add_relb)(var, (ETHR_NAINT32_T__) val); +#else + (void) ETHR_ATMC32_FUNC__(add_read_wb)(var, val); +#endif +} + +static ETHR_INLINE void ETHR_ATMC32_FUNC__(add_acqb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ +#if defined(ETHR_HAVE_NATMC32_ADD_ACQB) + ETHR_NATMC32_FUNC__(add_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_RB) + ETHR_NATMC32_FUNC__(add_rb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_ADD) + ETHR_NATMC32_FUNC__(add)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_ADD_MB) + ETHR_NATMC32_FUNC__(add_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_WB) + ETHR_NATMC32_FUNC__(add_wb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_ADD_RELB) + ETHR_NATMC32_FUNC__(add_relb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + (void) ETHR_ATMC32_FUNC__(add_read_acqb)(var, val); +#endif +} + +static ETHR_INLINE void ETHR_ATMC32_FUNC__(add_relb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ +#if defined(ETHR_HAVE_NATMC32_ADD_RELB) + ETHR_NATMC32_FUNC__(add_relb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_FUNC__(add_wb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_FUNC__(add)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_MB) + ETHR_NATMC32_FUNC__(add_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_FUNC__(add_rb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_FUNC__(add_acqb)(var, (ETHR_NAINT32_T__) val); +#else + (void) ETHR_ATMC32_FUNC__(add_read_relb)(var, val); +#endif +} + +static ETHR_INLINE void ETHR_ATMC32_FUNC__(add_mb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ +#if defined(ETHR_HAVE_NATMC32_ADD_MB) + ETHR_NATMC32_FUNC__(add_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_RELB) + ETHR_NATMC32_FUNC__(add_relb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_ADD_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_FUNC__(add_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_ADD_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_FUNC__(add_wb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_ADD_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_FUNC__(add_rb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_ADD) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_FUNC__(add)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + (void) ETHR_ATMC32_FUNC__(add_read_mb)(var, val); +#endif +} + + +/* --- inc() --- */ + + +static ETHR_INLINE void ETHR_ATMC32_FUNC__(inc)(ethr_atomic32_t *var) +{ +#if defined(ETHR_HAVE_NATMC32_INC) + ETHR_NATMC32_FUNC__(inc)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_RB) + ETHR_NATMC32_FUNC__(inc_rb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_WB) + ETHR_NATMC32_FUNC__(inc_wb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_ACQB) + ETHR_NATMC32_FUNC__(inc_acqb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_RELB) + ETHR_NATMC32_FUNC__(inc_relb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_MB) + ETHR_NATMC32_FUNC__(inc_mb)(var); +#else + (void) ETHR_ATMC32_FUNC__(inc_read)(var); +#endif +} + +static ETHR_INLINE void ETHR_ATMC32_FUNC__(inc_rb)(ethr_atomic32_t *var) +{ +#if defined(ETHR_HAVE_NATMC32_INC_RB) + ETHR_NATMC32_FUNC__(inc_rb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC) + ETHR_NATMC32_FUNC__(inc)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_INC_MB) + ETHR_NATMC32_FUNC__(inc_mb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_WB) + ETHR_NATMC32_FUNC__(inc_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_INC_ACQB) + ETHR_NATMC32_FUNC__(inc_acqb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_INC_RELB) + ETHR_NATMC32_FUNC__(inc_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + (void) ETHR_ATMC32_FUNC__(inc_read_rb)(var); +#endif +} + +static ETHR_INLINE void ETHR_ATMC32_FUNC__(inc_wb)(ethr_atomic32_t *var) +{ +#if defined(ETHR_HAVE_NATMC32_INC_WB) + ETHR_NATMC32_FUNC__(inc_wb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_FUNC__(inc)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_MB) + ETHR_NATMC32_FUNC__(inc_mb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_RB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_FUNC__(inc_rb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_FUNC__(inc_acqb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_FUNC__(inc_relb)(var); +#else + (void) ETHR_ATMC32_FUNC__(inc_read_wb)(var); +#endif +} + +static ETHR_INLINE void ETHR_ATMC32_FUNC__(inc_acqb)(ethr_atomic32_t *var) +{ +#if defined(ETHR_HAVE_NATMC32_INC_ACQB) + ETHR_NATMC32_FUNC__(inc_acqb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_RB) + ETHR_NATMC32_FUNC__(inc_rb)(var); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_INC) + ETHR_NATMC32_FUNC__(inc)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_INC_MB) + ETHR_NATMC32_FUNC__(inc_mb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_WB) + ETHR_NATMC32_FUNC__(inc_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_INC_RELB) + ETHR_NATMC32_FUNC__(inc_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + (void) ETHR_ATMC32_FUNC__(inc_read_acqb)(var); +#endif +} + +static ETHR_INLINE void ETHR_ATMC32_FUNC__(inc_relb)(ethr_atomic32_t *var) +{ +#if defined(ETHR_HAVE_NATMC32_INC_RELB) + ETHR_NATMC32_FUNC__(inc_relb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_FUNC__(inc_wb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_FUNC__(inc)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_MB) + ETHR_NATMC32_FUNC__(inc_mb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_FUNC__(inc_rb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_FUNC__(inc_acqb)(var); +#else + (void) ETHR_ATMC32_FUNC__(inc_read_relb)(var); +#endif +} + +static ETHR_INLINE void ETHR_ATMC32_FUNC__(inc_mb)(ethr_atomic32_t *var) +{ +#if defined(ETHR_HAVE_NATMC32_INC_MB) + ETHR_NATMC32_FUNC__(inc_mb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_RELB) + ETHR_NATMC32_FUNC__(inc_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_INC_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_FUNC__(inc_acqb)(var); +#elif defined(ETHR_HAVE_NATMC32_INC_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_FUNC__(inc_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_INC_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_FUNC__(inc_rb)(var); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_INC) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_FUNC__(inc)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + (void) ETHR_ATMC32_FUNC__(inc_read_mb)(var); +#endif +} + + +/* --- dec() --- */ + + +static ETHR_INLINE void ETHR_ATMC32_FUNC__(dec)(ethr_atomic32_t *var) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS +#if defined(ETHR_HAVE_NATMC32_DEC) + ETHR_NATMC32_FUNC__(dec)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_RB) + ETHR_NATMC32_FUNC__(dec_rb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_WB) + ETHR_NATMC32_FUNC__(dec_wb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_ACQB) + ETHR_NATMC32_FUNC__(dec_acqb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_RELB) + ETHR_NATMC32_FUNC__(dec_relb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_MB) + ETHR_NATMC32_FUNC__(dec_mb)(var); +#else + (void) ETHR_ATMC32_FUNC__(dec_read)(var); +#endif +} + +static ETHR_INLINE void ETHR_ATMC32_FUNC__(dec_rb)(ethr_atomic32_t *var) +{ +#if defined(ETHR_HAVE_NATMC32_DEC_RB) + ETHR_NATMC32_FUNC__(dec_rb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC) + ETHR_NATMC32_FUNC__(dec)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_DEC_MB) + ETHR_NATMC32_FUNC__(dec_mb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_WB) + ETHR_NATMC32_FUNC__(dec_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_DEC_ACQB) + ETHR_NATMC32_FUNC__(dec_acqb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_DEC_RELB) + ETHR_NATMC32_FUNC__(dec_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + (void) ETHR_ATMC32_FUNC__(dec_read_rb)(var); +#endif +} + +static ETHR_INLINE void ETHR_ATMC32_FUNC__(dec_wb)(ethr_atomic32_t *var) +{ +#if defined(ETHR_HAVE_NATMC32_DEC_WB) + ETHR_NATMC32_FUNC__(dec_wb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_FUNC__(dec)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_MB) + ETHR_NATMC32_FUNC__(dec_mb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_RB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_FUNC__(dec_rb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_FUNC__(dec_acqb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_RELB) + ETHR_MEMBAR(ETHR_StoreStore); ETHR_NATMC32_FUNC__(dec_relb)(var); #else - ETHR_INLINE_FUNC_NAME_(ethr_atomic32_dec)(var); + (void) ETHR_ATMC32_FUNC__(dec_read_wb)(var); #endif } -static ETHR_INLINE ethr_sint32_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic32_dec_read_relb)(ethr_atomic32_t *var) +static ETHR_INLINE void ETHR_ATMC32_FUNC__(dec_acqb)(ethr_atomic32_t *var) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_relb)(var); +#if defined(ETHR_HAVE_NATMC32_DEC_ACQB) + ETHR_NATMC32_FUNC__(dec_acqb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_RB) + ETHR_NATMC32_FUNC__(dec_rb)(var); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_DEC) + ETHR_NATMC32_FUNC__(dec)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_DEC_MB) + ETHR_NATMC32_FUNC__(dec_mb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_WB) + ETHR_NATMC32_FUNC__(dec_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_DEC_RELB) + ETHR_NATMC32_FUNC__(dec_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); #else - return ETHR_INLINE_FUNC_NAME_(ethr_atomic32_dec_read)(var); + (void) ETHR_ATMC32_FUNC__(dec_read_acqb)(var); #endif } -static ETHR_INLINE ethr_sint32_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic32_cmpxchg_acqb)(ethr_atomic32_t *var, - ethr_sint32_t new, - ethr_sint32_t exp) +static ETHR_INLINE void ETHR_ATMC32_FUNC__(dec_relb)(ethr_atomic32_t *var) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return (ethr_sint32_t) - ETHR_NATMC32_FUNC__(cmpxchg_acqb)(var, - (ETHR_NAINT32_T__) new, - (ETHR_NAINT32_T__) exp); +#if defined(ETHR_HAVE_NATMC32_DEC_RELB) + ETHR_NATMC32_FUNC__(dec_relb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_FUNC__(dec_wb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_FUNC__(dec)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_MB) + ETHR_NATMC32_FUNC__(dec_mb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_FUNC__(dec_rb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_FUNC__(dec_acqb)(var); #else - return ETHR_INLINE_FUNC_NAME_(ethr_atomic32_cmpxchg)(var, new, exp); + (void) ETHR_ATMC32_FUNC__(dec_read_relb)(var); #endif } -static ETHR_INLINE ethr_sint32_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic32_cmpxchg_relb)(ethr_atomic32_t *var, - ethr_sint32_t new, - ethr_sint32_t exp) +static ETHR_INLINE void ETHR_ATMC32_FUNC__(dec_mb)(ethr_atomic32_t *var) { -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return (ethr_sint32_t) - ETHR_NATMC32_FUNC__(cmpxchg_relb)(var, - (ETHR_NAINT32_T__) new, - (ETHR_NAINT32_T__) exp); +#if defined(ETHR_HAVE_NATMC32_DEC_MB) + ETHR_NATMC32_FUNC__(dec_mb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_RELB) + ETHR_NATMC32_FUNC__(dec_relb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_DEC_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_FUNC__(dec_acqb)(var); +#elif defined(ETHR_HAVE_NATMC32_DEC_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_FUNC__(dec_wb)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_DEC_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_FUNC__(dec_rb)(var); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_DEC) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_FUNC__(dec)(var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); #else - return ETHR_INLINE_FUNC_NAME_(ethr_atomic32_cmpxchg)(var, new, exp); + (void) ETHR_ATMC32_FUNC__(dec_read_mb)(var); #endif } -#endif /* ETHR_TRY_INLINE_FUNCS */ +/* --- read_band() --- */ -#undef ETHR_NAINT_T__ -#undef ETHR_NATMC_FUNC__ -#undef ETHR_NATMC_ADDR_FUNC__ -#undef ETHR_NAINT32_T__ -#undef ETHR_NATMC32_FUNC__ +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_band)(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_AND_RETOLD) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_rb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_wb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_ACQB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_relb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val); +#else +#error "Missing implementation of ethr_atomic32_read_band()!" +#endif + return res; +} + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_band_rb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_AND_RETOLD_RB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_rb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_wb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_ACQB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_acqb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_relb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval &= val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val); + ETHR_MEMBAR(ETHR_LoadLoad); +#else +#error "Missing implementation of ethr_atomic32_read_band_rb()!" +#endif + return res; +} + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_band_wb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_AND_RETOLD_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_wb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_rb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_relb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val); +#else +#error "Missing implementation of ethr_atomic32_read_band_wb()!" +#endif + return res; +} + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_band_acqb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_AND_RETOLD_ACQB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_rb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_wb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_relb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval &= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else +#error "Missing implementation of ethr_atomic32_read_band_acqb()!" +#endif + return res; +} + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_band_relb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_AND_RETOLD_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_relb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_wb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_rb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val); +#else +#error "Missing implementation of ethr_atomic32_read_band_relb()!" +#endif + return res; +} + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_band_mb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_AND_RETOLD_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_relb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_wb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_rb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval &= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else +#error "Missing implementation of ethr_atomic32_read_band_mb()!" +#endif + return res; +} + + +/* --- read_bor() --- */ + + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_bor)(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_OR_RETOLD) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_rb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_wb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_ACQB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_relb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val); +#else +#error "Missing implementation of ethr_atomic32_read_bor()!" +#endif + return res; +} + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_bor_rb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_OR_RETOLD_RB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_rb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_wb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_ACQB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_acqb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_relb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval |= val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val); + ETHR_MEMBAR(ETHR_LoadLoad); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val); + ETHR_MEMBAR(ETHR_LoadLoad); +#else +#error "Missing implementation of ethr_atomic32_read_bor_rb()!" +#endif + return res; +} +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_bor_wb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_OR_RETOLD_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_wb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_rb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_relb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val); +#else +#error "Missing implementation of ethr_atomic32_read_bor_wb()!" +#endif + return res; +} + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_bor_acqb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_OR_RETOLD_ACQB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_rb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_WB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_wb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_relb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval |= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else +#error "Missing implementation of ethr_atomic32_read_bor_acqb()!" +#endif + return res; +} + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_bor_relb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_OR_RETOLD_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_relb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_wb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_rb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val); +#else +#error "Missing implementation of ethr_atomic32_read_bor_relb()!" +#endif + return res; +} + +static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_bor_mb)(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; +#if defined(ETHR_HAVE_NATMC32_OR_RETOLD_MB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_mb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RELB) + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_relb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_acqb)(var, (ETHR_NAINT32_T__) val); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_wb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_rb)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold)(var, (ETHR_NAINT32_T__) val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#elif defined(ETHR_HAVE_NATMC32_CMPXCHG) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval |= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else +#error "Missing implementation of ethr_atomic32_read_bor_mb()!" #endif + return res; +} + +#endif /* ETHR_ATMC32_INLINE__ */ + +#endif /* ETHR_ATOMICS_H__ */ diff --git a/erts/include/internal/ethr_internal.h b/erts/include/internal/ethr_internal.h index e9c3daf783..c9b1db5b46 100644 --- a/erts/include/internal/ethr_internal.h +++ b/erts/include/internal/ethr_internal.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2010. All Rights Reserved. + * Copyright Ericsson AB 2010-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -63,5 +63,9 @@ int ethr_late_init_common__(ethr_late_init_data *lid); void ethr_run_exit_handlers__(void); void ethr_ts_event_destructor__(void *vtsep); +#if defined(ETHR_X86_RUNTIME_CONF__) +int ethr_x86_have_cpuid__(void); +void ethr_x86_cpuid__(int *eax, int *ebx, int *ecx, int *edx); +#endif #endif diff --git a/erts/include/internal/ethr_mutex.h b/erts/include/internal/ethr_mutex.h index fadaf1e2a4..a0685ea3c0 100644 --- a/erts/include/internal/ethr_mutex.h +++ b/erts/include/internal/ethr_mutex.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2010. All Rights Reserved. + * Copyright Ericsson AB 2010-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -54,6 +54,10 @@ # endif #endif +#ifndef ETHR_INLINE_MTX_FUNC_NAME_ +# define ETHR_INLINE_MTX_FUNC_NAME_(X) X +#endif + #if defined(ETHR_USE_OWN_RWMTX_IMPL__) || defined(ETHR_USE_OWN_MTX_IMPL__) #ifdef ETHR_DEBUG @@ -505,7 +509,7 @@ void ethr_mutex_lock_wait__(ethr_mutex *, ethr_sint32_t); void ethr_mutex_unlock_wake__(ethr_mutex *, ethr_sint32_t); static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_mutex_trylock)(ethr_mutex *mtx) +ETHR_INLINE_MTX_FUNC_NAME_(ethr_mutex_trylock)(ethr_mutex *mtx) { ethr_sint32_t act; int res; @@ -529,7 +533,7 @@ ETHR_INLINE_FUNC_NAME_(ethr_mutex_trylock)(ethr_mutex *mtx) } static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_mutex_lock)(ethr_mutex *mtx) +ETHR_INLINE_MTX_FUNC_NAME_(ethr_mutex_lock)(ethr_mutex *mtx) { ethr_sint32_t act; ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx); @@ -549,7 +553,7 @@ ETHR_INLINE_FUNC_NAME_(ethr_mutex_lock)(ethr_mutex *mtx) } static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_mutex_unlock)(ethr_mutex *mtx) +ETHR_INLINE_MTX_FUNC_NAME_(ethr_mutex_unlock)(ethr_mutex *mtx) { ethr_sint32_t act; ETHR_COMPILER_BARRIER; @@ -574,7 +578,7 @@ ETHR_INLINE_FUNC_NAME_(ethr_mutex_unlock)(ethr_mutex *mtx) #if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_MUTEX_IMPL__) static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_mutex_trylock)(ethr_mutex *mtx) +ETHR_INLINE_MTX_FUNC_NAME_(ethr_mutex_trylock)(ethr_mutex *mtx) { int res; res = pthread_mutex_trylock(&mtx->pt_mtx); @@ -584,7 +588,7 @@ ETHR_INLINE_FUNC_NAME_(ethr_mutex_trylock)(ethr_mutex *mtx) } static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_mutex_lock)(ethr_mutex *mtx) +ETHR_INLINE_MTX_FUNC_NAME_(ethr_mutex_lock)(ethr_mutex *mtx) { int res = pthread_mutex_lock(&mtx->pt_mtx); if (res != 0) @@ -592,7 +596,7 @@ ETHR_INLINE_FUNC_NAME_(ethr_mutex_lock)(ethr_mutex *mtx) } static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_mutex_unlock)(ethr_mutex *mtx) +ETHR_INLINE_MTX_FUNC_NAME_(ethr_mutex_unlock)(ethr_mutex *mtx) { int res = pthread_mutex_unlock(&mtx->pt_mtx); if (res != 0) @@ -615,7 +619,7 @@ ETHR_INLINE_FUNC_NAME_(ethr_mutex_unlock)(ethr_mutex *mtx) #if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_MUTEX_IMPL__) static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_tryrlock)(ethr_rwmutex *rwmtx) +ETHR_INLINE_MTX_FUNC_NAME_(ethr_rwmutex_tryrlock)(ethr_rwmutex *rwmtx) { int res = pthread_rwlock_tryrdlock(&rwmtx->pt_rwlock); if (res != 0 && res != EBUSY) @@ -624,7 +628,7 @@ ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_tryrlock)(ethr_rwmutex *rwmtx) } static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_rlock)(ethr_rwmutex *rwmtx) +ETHR_INLINE_MTX_FUNC_NAME_(ethr_rwmutex_rlock)(ethr_rwmutex *rwmtx) { int res = pthread_rwlock_rdlock(&rwmtx->pt_rwlock); if (res != 0) @@ -632,7 +636,7 @@ ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_rlock)(ethr_rwmutex *rwmtx) } static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_runlock)(ethr_rwmutex *rwmtx) +ETHR_INLINE_MTX_FUNC_NAME_(ethr_rwmutex_runlock)(ethr_rwmutex *rwmtx) { int res = pthread_rwlock_unlock(&rwmtx->pt_rwlock); if (res != 0) @@ -640,7 +644,7 @@ ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_runlock)(ethr_rwmutex *rwmtx) } static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_tryrwlock)(ethr_rwmutex *rwmtx) +ETHR_INLINE_MTX_FUNC_NAME_(ethr_rwmutex_tryrwlock)(ethr_rwmutex *rwmtx) { int res = pthread_rwlock_trywrlock(&rwmtx->pt_rwlock); if (res != 0 && res != EBUSY) @@ -649,7 +653,7 @@ ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_tryrwlock)(ethr_rwmutex *rwmtx) } static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_rwlock)(ethr_rwmutex *rwmtx) +ETHR_INLINE_MTX_FUNC_NAME_(ethr_rwmutex_rwlock)(ethr_rwmutex *rwmtx) { int res = pthread_rwlock_wrlock(&rwmtx->pt_rwlock); if (res != 0) @@ -657,7 +661,7 @@ ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_rwlock)(ethr_rwmutex *rwmtx) } static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_rwunlock)(ethr_rwmutex *rwmtx) +ETHR_INLINE_MTX_FUNC_NAME_(ethr_rwmutex_rwunlock)(ethr_rwmutex *rwmtx) { int res = pthread_rwlock_unlock(&rwmtx->pt_rwlock); if (res != 0) diff --git a/erts/include/internal/ethr_optimized_fallbacks.h b/erts/include/internal/ethr_optimized_fallbacks.h index 8e04692856..45399d18e6 100644 --- a/erts/include/internal/ethr_optimized_fallbacks.h +++ b/erts/include/internal/ethr_optimized_fallbacks.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2010. All Rights Reserved. + * Copyright Ericsson AB 2010-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -18,152 +18,172 @@ */ /* - * Description: "Optimized" fallbacks used when native ops are missing + * Description: Optimized fallbacks used when native ops are missing * Author: Rickard Green */ #ifndef ETHR_OPTIMIZED_FALLBACKS_H__ #define ETHR_OPTIMIZED_FALLBACKS_H__ -#ifdef ETHR_HAVE_NATIVE_ATOMICS -#define ETHR_HAVE_OPTIMIZED_ATOMIC_OPS 1 +#if defined(ETHR_HAVE_NATIVE_SPINLOCKS) + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) + +static ETHR_INLINE int +ethr_native_spinlock_destroy(ethr_native_spinlock_t *lock) +{ + return 0; +} + #endif -#ifdef ETHR_HAVE_NATIVE_SPINLOCKS -#define ETHR_HAVE_OPTIMIZED_SPINLOCKS 1 #elif defined(ETHR_HAVE_PTHREAD_SPIN_LOCK) -/* --- Optimized spinlocks using pthread spinlocks -------------------------- */ -#define ETHR_HAVE_OPTIMIZED_SPINLOCKS 1 +/* --- Native spinlocks using pthread spinlocks -------------------------- */ +#define ETHR_HAVE_NATIVE_SPINLOCKS 1 + +#define ETHR_NATIVE_SPINLOCK_IMPL "pthread" -typedef pthread_spinlock_t ethr_opt_spinlock_t; +typedef pthread_spinlock_t ethr_native_spinlock_t; #if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) -static ETHR_INLINE int -ethr_opt_spinlock_init(ethr_opt_spinlock_t *lock) +static ETHR_INLINE void +ethr_native_spinlock_init(ethr_native_spinlock_t *lock) { - return pthread_spin_init((pthread_spinlock_t *) lock, 0); + int err = pthread_spin_init((pthread_spinlock_t *) lock, 0); + if (err) + ETHR_FATAL_ERROR__(err); } static ETHR_INLINE int -ethr_opt_spinlock_destroy(ethr_opt_spinlock_t *lock) +ethr_native_spinlock_destroy(ethr_native_spinlock_t *lock) { return pthread_spin_destroy((pthread_spinlock_t *) lock); } - -static ETHR_INLINE int -ethr_opt_spin_unlock(ethr_opt_spinlock_t *lock) +static ETHR_INLINE void +ethr_native_spin_unlock(ethr_native_spinlock_t *lock) { - return pthread_spin_unlock((pthread_spinlock_t *) lock); + int err = pthread_spin_unlock((pthread_spinlock_t *) lock); + if (err) + ETHR_FATAL_ERROR__(err); } -static ETHR_INLINE int -ethr_opt_spin_lock(ethr_opt_spinlock_t *lock) +static ETHR_INLINE void +ethr_native_spin_lock(ethr_native_spinlock_t *lock) { - return pthread_spin_lock((pthread_spinlock_t *) lock); + int err = pthread_spin_lock((pthread_spinlock_t *) lock); + if (err) + ETHR_FATAL_ERROR__(err); } #endif -#elif defined(ETHR_HAVE_NATIVE_ATOMICS) +#elif defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) /* --- Native spinlocks using native atomics -------------------------------- */ #define ETHR_HAVE_NATIVE_SPINLOCKS 1 -#define ETHR_HAVE_OPTIMIZED_SPINLOCKS 1 - -#if defined(ETHR_HAVE_NATIVE_ATOMIC32) -typedef ethr_native_atomic32_t ethr_native_spinlock_t; -# define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X -#elif defined(ETHR_HAVE_NATIVE_ATOMIC64) -typedef ethr_native_atomic64_t ethr_native_spinlock_t; -# define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X -#else -# error "Missing native atomic implementation" -#endif + +#define ETHR_NATIVE_SPINLOCK_IMPL "native-atomics" + +typedef ethr_atomic32_t ethr_native_spinlock_t; #if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) +#undef ETHR_NSPN_AOP__ +#define ETHR_NSPN_AOP__(X) ETHR_INLINE_ATMC32_FUNC_NAME_(ethr_atomic32_ ## X) + static ETHR_INLINE void ethr_native_spinlock_init(ethr_native_spinlock_t *lock) { - ETHR_NATMC_FUNC__(init)(lock, 0); + ETHR_NSPN_AOP__(init)(lock, 0); +} + +static ETHR_INLINE int +ethr_native_spinlock_destroy(ethr_native_spinlock_t *lock) +{ + return ETHR_NSPN_AOP__(read)(lock) == 0 ? 0 : EBUSY; } static ETHR_INLINE void ethr_native_spin_unlock(ethr_native_spinlock_t *lock) { - ETHR_COMPILER_BARRIER; - ETHR_ASSERT(ETHR_NATMC_FUNC__(read)(lock) == 1); - ETHR_NATMC_FUNC__(set_relb)(lock, 0); + ETHR_ASSERT(ETHR_NSPN_AOP__(read)(lock) == 1); + ETHR_NSPN_AOP__(set_relb)(lock, 0); } static ETHR_INLINE void ethr_native_spin_lock(ethr_native_spinlock_t *lock) { - while (ETHR_NATMC_FUNC__(cmpxchg_acqb)(lock, 1, 0) != 0) { - while (ETHR_NATMC_FUNC__(read)(lock) != 0) + while (ETHR_NSPN_AOP__(cmpxchg_acqb)(lock, 1, 0) != 0) { + while (ETHR_NSPN_AOP__(read)(lock) != 0) ETHR_SPIN_BODY; } ETHR_COMPILER_BARRIER; } -#endif +#undef ETHR_NSPN_AOP__ -#undef ETHR_NATMC_FUNC__ +#endif #endif -#ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS -#define ETHR_HAVE_OPTIMIZED_RWSPINLOCKS 1 -#elif defined(ETHR_HAVE_NATIVE_ATOMICS) +#if defined(ETHR_HAVE_NATIVE_RWSPINLOCKS) + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) + +static ETHR_INLINE int +ethr_native_rwlock_destroy(ethr_native_rwlock_t *lock) +{ + return 0; +} + +#endif + +#elif defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) /* --- Native rwspinlocks using native atomics ------------------------------ */ #define ETHR_HAVE_NATIVE_RWSPINLOCKS 1 -#define ETHR_HAVE_OPTIMIZED_RWSPINLOCKS 1 +#define ETHR_NATIVE_RWSPINLOCK_IMPL "native-atomics" -#if defined(ETHR_HAVE_NATIVE_ATOMIC32) -typedef ethr_native_atomic32_t ethr_native_rwlock_t; -# define ETHR_NAINT_T__ ethr_sint32_t +typedef ethr_atomic32_t ethr_native_rwlock_t; # define ETHR_WLOCK_FLAG__ (((ethr_sint32_t) 1) << 30) -# define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X -#elif defined(ETHR_HAVE_NATIVE_ATOMIC64) -typedef ethr_native_atomic64_t ethr_native_rwlock_t; -# define ETHR_NAINT_T__ ethr_sint64_t -# define ETHR_WLOCK_FLAG__ (((ethr_sint64_t) 1) << 62) -# define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X -#else -# error "Missing native atomic implementation" -#endif #if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) +#undef ETHR_NRWSPN_AOP__ +#define ETHR_NRWSPN_AOP__(X) ETHR_INLINE_ATMC32_FUNC_NAME_(ethr_atomic32_ ## X) + static ETHR_INLINE void ethr_native_rwlock_init(ethr_native_rwlock_t *lock) { - ETHR_NATMC_FUNC__(init)(lock, 0); + ETHR_NRWSPN_AOP__(init)(lock, 0); +} + +static ETHR_INLINE int +ethr_native_rwlock_destroy(ethr_native_rwlock_t *lock) +{ + return ETHR_NRWSPN_AOP__(read)(lock) == 0 ? 0 : EBUSY; } static ETHR_INLINE void ethr_native_read_unlock(ethr_native_rwlock_t *lock) { - ETHR_COMPILER_BARRIER; -#ifdef DEBUG - ETHR_ASSERT(ETHR_NATMC_FUNC__(read)(lock) >= 0); -#endif - ETHR_NATMC_FUNC__(dec_relb)(lock); + ETHR_ASSERT(ETHR_NRWSPN_AOP__(read)(lock) >= 0); + ETHR_NRWSPN_AOP__(dec_relb)(lock); } static ETHR_INLINE void ethr_native_read_lock(ethr_native_rwlock_t *lock) { - ETHR_NAINT_T__ act, exp = 0; + ethr_sint32_t act, exp = 0; while (1) { - act = ETHR_NATMC_FUNC__(cmpxchg_acqb)(lock, exp+1, exp); + act = ETHR_NRWSPN_AOP__(cmpxchg_acqb)(lock, exp+1, exp); if (act == exp) break; + /* Wait for writer to leave */ while (act & ETHR_WLOCK_FLAG__) { ETHR_SPIN_BODY; - act = ETHR_NATMC_FUNC__(read)(lock); + act = ETHR_NRWSPN_AOP__(read)(lock); } exp = act; } @@ -173,36 +193,37 @@ ethr_native_read_lock(ethr_native_rwlock_t *lock) static ETHR_INLINE void ethr_native_write_unlock(ethr_native_rwlock_t *lock) { - ETHR_COMPILER_BARRIER; - ETHR_ASSERT(ETHR_NATMC_FUNC__(read)(lock) == ETHR_WLOCK_FLAG__); - ETHR_NATMC_FUNC__(set_relb)(lock, 0); + ETHR_ASSERT(ETHR_NRWSPN_AOP__(read)(lock) == ETHR_WLOCK_FLAG__); + ETHR_NRWSPN_AOP__(set_relb)(lock, 0); } static ETHR_INLINE void ethr_native_write_lock(ethr_native_rwlock_t *lock) { - ETHR_NAINT_T__ act, exp = 0; + ethr_sint32_t act, exp = 0; while (1) { - act = ETHR_NATMC_FUNC__(cmpxchg_acqb)(lock, exp|ETHR_WLOCK_FLAG__, exp); + act = ETHR_NRWSPN_AOP__(cmpxchg_acqb)(lock, exp|ETHR_WLOCK_FLAG__, exp); if (act == exp) break; - ETHR_SPIN_BODY; - exp = act & ~ETHR_WLOCK_FLAG__; + /* Wait for writer to leave */ + while (act & ETHR_WLOCK_FLAG__) { + ETHR_SPIN_BODY; + act = ETHR_NRWSPN_AOP__(read)(lock); + } + exp = act; } act |= ETHR_WLOCK_FLAG__; /* Wait for readers to leave */ while (act != ETHR_WLOCK_FLAG__) { ETHR_SPIN_BODY; - act = ETHR_NATMC_FUNC__(read_acqb)(lock); + act = ETHR_NRWSPN_AOP__(read_acqb)(lock); } ETHR_COMPILER_BARRIER; } -#endif +#undef ETHR_NRWSPN_AOP__ -#undef ETHR_NAINT_T__ -#undef ETHR_NATMC_FUNC__ -#undef ETHR_WLOCK_FLAG__ +#endif #endif diff --git a/erts/include/internal/ethread.h b/erts/include/internal/ethread.h index 4cd95faf6a..8ad0ded144 100644 --- a/erts/include/internal/ethread.h +++ b/erts/include/internal/ethread.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2004-2010. All Rights Reserved. + * Copyright Ericsson AB 2004-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -33,10 +33,6 @@ #include <stdlib.h> #include "erl_errno.h" -#undef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS -#undef ETHR_HAVE_OPTIMIZED_SPINLOCK -#undef ETHR_HAVE_OPTIMIZED_RWSPINLOCK - #if defined(DEBUG) # define ETHR_DEBUG #endif @@ -68,7 +64,7 @@ #endif /* Assume 64-byte cache line size */ -#define ETHR_CACHE_LINE_SIZE ((ethr_uint_t) 64) +#define ETHR_CACHE_LINE_SIZE 64 #define ETHR_CACHE_LINE_MASK (ETHR_CACHE_LINE_SIZE - 1) #define ETHR_CACHE_LINE_ALIGN_SIZE(SZ) \ @@ -251,13 +247,90 @@ typedef ethr_sint64_t ethr_sint_t; typedef ethr_uint64_t ethr_uint_t; #endif -/* __builtin_expect() is needed by both native atomics code - * and the fallback code */ -#if !defined(__GNUC__) || (__GNUC__ < 2) || (__GNUC__ == 2 && __GNUC_MINOR__ < 96) +#if defined(ETHR_SIZEOF___INT128_T) && ETHR_SIZEOF___INT128_T == 16 +#define ETHR_HAVE_INT128_T +typedef __int128_t ethr_sint128_t; +typedef __uint128_t ethr_uint128_t; +#endif + +#define ETHR_FATAL_ERROR__(ERR) \ + ethr_fatal_error__(__FILE__, __LINE__, __func__, (ERR)) + +ETHR_PROTO_NORETURN__ ethr_fatal_error__(const char *file, + int line, + const char *func, + int err); + +#if !defined(__GNUC__) +# define ETHR_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) 0 +#elif !defined(__GNUC_MINOR__) +# define ETHR_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \ + ((__GNUC__ << 24) >= (((MAJ) << 24) | ((MIN) << 12) | (PL))) +#elif !defined(__GNUC_PATCHLEVEL__) +# define ETHR_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \ + (((__GNUC__ << 24) | (__GNUC_MINOR__ << 12)) >= (((MAJ) << 24) | ((MIN) << 12) | (PL))) +#else +# define ETHR_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \ + (((__GNUC__ << 24) | (__GNUC_MINOR__ << 12) | __GNUC_PATCHLEVEL__) >= (((MAJ) << 24) | ((MIN) << 12) | (PL))) +#endif + +#if !ETHR_AT_LEAST_GCC_VSN__(2, 96, 0) #define __builtin_expect(X, Y) (X) #endif -/* For CPU-optimised atomics, spinlocks, and rwlocks. */ +#if ETHR_AT_LEAST_GCC_VSN__(3, 1, 1) +# define ETHR_CHOOSE_EXPR __builtin_choose_expr +#else +# define ETHR_CHOOSE_EXPR(B, E1, E2) ((B) ? (E1) : (E2)) +#endif + +#if ((defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__))) \ + || (defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_AMD64)))) +# define ETHR_X86_RUNTIME_CONF__ + +# define ETHR_X86_RUNTIME_CONF_HAVE_DW_CMPXCHG__ \ + (__builtin_expect(ethr_runtime__.conf.have_dw_cmpxchg != 0, 1)) +# define ETHR_X86_RUNTIME_CONF_HAVE_NO_DW_CMPXCHG__ \ + (__builtin_expect(ethr_runtime__.conf.have_dw_cmpxchg == 0, 0)) +# define ETHR_X86_RUNTIME_CONF_HAVE_SSE2__ \ + (__builtin_expect(ethr_runtime__.conf.have_sse2 != 0, 1)) +# define ETHR_X86_RUNTIME_CONF_HAVE_NO_SSE2__ \ + (__builtin_expect(ethr_runtime__.conf.have_sse2 == 0, 0)) +#endif + +#if (defined(__GNUC__) \ + && !defined(ETHR_PPC_HAVE_LWSYNC) \ + && !defined(ETHR_PPC_HAVE_NO_LWSYNC) \ + && (defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__))) +# define ETHR_PPC_RUNTIME_CONF__ + +# define ETHR_PPC_RUNTIME_CONF_HAVE_LWSYNC__ \ + (__builtin_expect(ethr_runtime__.conf.have_lwsync != 0, 1)) +# define ETHR_PPC_RUNTIME_CONF_HAVE_NO_LWSYNC__ \ + (__builtin_expect(ethr_runtime__.conf.have_lwsync == 0, 0)) +#endif + +typedef struct { +#if defined(ETHR_X86_RUNTIME_CONF__) + int have_dw_cmpxchg; + int have_sse2; +#endif +#if defined(ETHR_PPC_RUNTIME_CONF__) + int have_lwsync; +#endif + int dummy; +} ethr_runtime_conf_t; + + +typedef union { + ethr_runtime_conf_t conf; + char pad__[ETHR_CACHE_LINE_ALIGN_SIZE(sizeof(ethr_runtime_conf_t))+ETHR_CACHE_LINE_SIZE]; +} ethr_runtime_t; + + +extern ethr_runtime_t ethr_runtime__; + +/* For native CPU-optimised atomics, spinlocks, and rwlocks. */ #if !defined(ETHR_DISABLE_NATIVE_IMPLS) # if defined(__GNUC__) # if defined(ETHR_PREFER_GCC_NATIVE_IMPLS) @@ -265,7 +338,7 @@ typedef ethr_uint64_t ethr_uint_t; # elif defined(ETHR_PREFER_LIBATOMIC_OPS_NATIVE_IMPLS) # include "libatomic_ops/ethread.h" # endif -# ifndef ETHR_HAVE_NATIVE_ATOMICS +# if !defined(ETHR_HAVE_NATIVE_ATOMIC32) && !defined(ETHR_HAVE_NATIVE_ATOMIC64) # if ETHR_SIZEOF_PTR == 4 # if defined(__i386__) # include "i386/ethread.h" @@ -283,8 +356,10 @@ typedef ethr_uint64_t ethr_uint_t; # include "sparc64/ethread.h" # endif # endif +#if 0 # include "gcc/ethread.h" # include "libatomic_ops/ethread.h" +#endif # endif # elif defined(ETHR_HAVE_LIBATOMIC_OPS) # include "libatomic_ops/ethread.h" @@ -293,10 +368,9 @@ typedef ethr_uint64_t ethr_uint_t; # endif #endif /* !ETHR_DISABLE_NATIVE_IMPLS */ +#include "ethr_atomics.h" /* The atomics API */ + #if defined(__GNUC__) -# ifndef ETHR_COMPILER_BARRIER -# define ETHR_COMPILER_BARRIER __asm__ __volatile__("" : : : "memory") -# endif # ifndef ETHR_SPIN_BODY # if defined(__i386__) || defined(__x86_64__) # define ETHR_SPIN_BODY __asm__ __volatile__("rep;nop" : : : "memory") @@ -309,11 +383,6 @@ typedef ethr_uint64_t ethr_uint_t; # endif # endif #elif defined(ETHR_WIN32_THREADS) -# ifndef ETHR_COMPILER_BARRIER -# include <intrin.h> -# pragma intrinsic(_ReadWriteBarrier) -# define ETHR_COMPILER_BARRIER _ReadWriteBarrier() -# endif # ifndef ETHR_SPIN_BODY # define ETHR_SPIN_BODY do {YieldProcessor();ETHR_COMPILER_BARRIER;} while(0) # endif @@ -321,43 +390,6 @@ typedef ethr_uint64_t ethr_uint_t; #define ETHR_YIELD_AFTER_BUSY_LOOPS 50 -#ifndef ETHR_HAVE_NATIVE_ATOMICS -/* - * ETHR_*MEMORY_BARRIER orders between locked and atomic accesses only, - * i.e. when our lock based atomic fallback is used, a noop is sufficient. - */ -#define ETHR_MEMORY_BARRIER do { } while (0) -#define ETHR_WRITE_MEMORY_BARRIER do { } while (0) -#define ETHR_READ_MEMORY_BARRIER do { } while (0) -#define ETHR_READ_DEPEND_MEMORY_BARRIER do { } while (0) -#endif - -#ifndef ETHR_WRITE_MEMORY_BARRIER -# define ETHR_WRITE_MEMORY_BARRIER ETHR_MEMORY_BARRIER -# define ETHR_WRITE_MEMORY_BARRIER_IS_FULL -#endif -#ifndef ETHR_READ_MEMORY_BARRIER -# define ETHR_READ_MEMORY_BARRIER ETHR_MEMORY_BARRIER -# define ETHR_READ_MEMORY_BARRIER_IS_FULL -#endif -#ifndef ETHR_READ_DEPEND_MEMORY_BARRIER -# define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_COMPILER_BARRIER -# define ETHR_READ_DEPEND_MEMORY_BARRIER_IS_COMPILER_BARRIER -#endif - -#define ETHR_FATAL_ERROR__(ERR) \ - ethr_fatal_error__(__FILE__, __LINE__, __func__, (ERR)) - -ETHR_PROTO_NORETURN__ ethr_fatal_error__(const char *file, - int line, - const char *func, - int err); - -void ethr_compiler_barrier_fallback(void); -#ifndef ETHR_COMPILER_BARRIER -# define ETHR_COMPILER_BARRIER ethr_compiler_barrier_fallback() -#endif - #ifndef ETHR_SPIN_BODY # define ETHR_SPIN_BODY ETHR_COMPILER_BARRIER #endif @@ -460,8 +492,6 @@ void ethr_compiler_barrier(void); #if defined(ETHR_HAVE_NATIVE_SPINLOCKS) typedef ethr_native_spinlock_t ethr_spinlock_t; -#elif defined(ETHR_HAVE_OPTIMIZED_SPINLOCKS) -typedef ethr_opt_spinlock_t ethr_spinlock_t; #elif defined(__WIN32__) typedef CRITICAL_SECTION ethr_spinlock_t; #else @@ -483,8 +513,6 @@ ETHR_INLINE_FUNC_NAME_(ethr_spinlock_init)(ethr_spinlock_t *lock) #ifdef ETHR_HAVE_NATIVE_SPINLOCKS ethr_native_spinlock_init(lock); return 0; -#elif defined(ETHR_HAVE_OPTIMIZED_SPINLOCKS) - return ethr_opt_spinlock_init((ethr_opt_spinlock_t *) lock); #elif defined(__WIN32__) if (!InitializeCriticalSectionAndSpinCount((CRITICAL_SECTION *) lock, INT_MAX)) return ethr_win_get_errno__(); @@ -498,9 +526,7 @@ static ETHR_INLINE int ETHR_INLINE_FUNC_NAME_(ethr_spinlock_destroy)(ethr_spinlock_t *lock) { #ifdef ETHR_HAVE_NATIVE_SPINLOCKS - return 0; -#elif defined(ETHR_HAVE_OPTIMIZED_SPINLOCKS) - return ethr_opt_spinlock_destroy((ethr_opt_spinlock_t *) lock); + return ethr_native_spinlock_destroy(lock); #elif defined(__WIN32__) DeleteCriticalSection((CRITICAL_SECTION *) lock); return 0; @@ -514,10 +540,6 @@ ETHR_INLINE_FUNC_NAME_(ethr_spin_unlock)(ethr_spinlock_t *lock) { #ifdef ETHR_HAVE_NATIVE_SPINLOCKS ethr_native_spin_unlock(lock); -#elif defined(ETHR_HAVE_OPTIMIZED_SPINLOCKS) - int err = ethr_opt_spin_unlock((ethr_opt_spinlock_t *) lock); - if (err) - ETHR_FATAL_ERROR__(err); #elif defined(__WIN32__) LeaveCriticalSection((CRITICAL_SECTION *) lock); #else @@ -532,10 +554,6 @@ ETHR_INLINE_FUNC_NAME_(ethr_spin_lock)(ethr_spinlock_t *lock) { #ifdef ETHR_HAVE_NATIVE_SPINLOCKS ethr_native_spin_lock(lock); -#elif defined(ETHR_HAVE_OPTIMIZED_SPINLOCKS) - int err = ethr_opt_spin_lock((ethr_opt_spinlock_t *) lock); - if (err) - ETHR_FATAL_ERROR__(err); #elif defined(__WIN32__) EnterCriticalSection((CRITICAL_SECTION *) lock); #else @@ -547,8 +565,6 @@ ETHR_INLINE_FUNC_NAME_(ethr_spin_lock)(ethr_spinlock_t *lock) #endif /* ETHR_TRY_INLINE_FUNCS */ -#include "ethr_atomics.h" - typedef struct ethr_ts_event_ ethr_ts_event; /* Needed by ethr_mutex.h */ #if defined(ETHR_WIN32_THREADS) diff --git a/erts/include/internal/ethread_header_config.h.in b/erts/include/internal/ethread_header_config.h.in index f394d790d2..dd3599f86d 100644 --- a/erts/include/internal/ethread_header_config.h.in +++ b/erts/include/internal/ethread_header_config.h.in @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2004-2010. All Rights Reserved. + * Copyright Ericsson AB 2004-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -32,6 +32,9 @@ /* Define to the size of __int64 */ #undef ETHR_SIZEOF___INT64 +/* Define to the size of __int128_t */ +#undef ETHR_SIZEOF___INT128_T + /* Define if bigendian */ #undef ETHR_BIGENDIAN @@ -69,8 +72,44 @@ /* Define if you have a linux futex implementation. */ #undef ETHR_HAVE_LINUX_FUTEX -/* Define if you have gcc atomic operations */ -#undef ETHR_HAVE_GCC_ATOMIC_OPS +/* Define if x86/x86_64 out of order instructions should be synchronized */ +#undef ETHR_X86_OUT_OF_ORDER + +/* Define if only run in Sparc TSO mode */ +#undef ETHR_SPARC_TSO + +/* Define if only run in Sparc PSO, or TSO mode */ +#undef ETHR_SPARC_PSO + +/* Define if run in Sparc RMO, PSO, or TSO mode */ +#undef ETHR_SPARC_RMO + +/* Define if you have __sync_add_and_fetch() for 32-bit integers */ +#undef ETHR_HAVE___SYNC_ADD_AND_FETCH32 + +/* Define if you have __sync_add_and_fetch() for 64-bit integers */ +#undef ETHR_HAVE___SYNC_ADD_AND_FETCH64 + +/* Define if you have __sync_fetch_and_and() for 32-bit integers */ +#undef ETHR_HAVE___SYNC_FETCH_AND_AND32 + +/* Define if you have __sync_fetch_and_and() for 64-bit integers */ +#undef ETHR_HAVE___SYNC_FETCH_AND_AND64 + +/* Define if you have __sync_fetch_and_or() for 32-bit integers */ +#undef ETHR_HAVE___SYNC_FETCH_AND_OR32 + +/* Define if you have __sync_fetch_and_or() for 64-bit integers */ +#undef ETHR_HAVE___SYNC_FETCH_AND_OR64 + +/* Define if you have __sync_val_compare_and_swap() for 32-bit integers */ +#undef ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP32 + +/* Define if you have __sync_val_compare_and_swap() for 64-bit integers */ +#undef ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP64 + +/* Define if you have __sync_val_compare_and_swap() for 128-bit integers */ +#undef ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP128 /* Define if you prefer gcc native ethread implementations */ #undef ETHR_PREFER_GCC_NATIVE_IMPLS @@ -90,8 +129,14 @@ /* Define if sched_yield() returns an int. */ #undef ETHR_SCHED_YIELD_RET_INT -/* Define if you want compatibilty with x86 processors before pentium4. */ -#undef ETHR_PRE_PENTIUM4_COMPAT +/* Define if you use a gcc that supports -msse2 and understand sse2 specific asm statements */ +#undef ETHR_GCC_HAVE_SSE2_ASM_SUPPORT + +/* Define if you use a gcc that supports the double word cmpxchg instruction */ +#undef ETHR_GCC_HAVE_DW_CMPXCHG_ASM_SUPPORT + +/* Define if you get a register shortage with cmpxchg8b and position independent code */ +#undef ETHR_CMPXCHG8B_REGISTER_SHORTAGE /* Define if you have the pthread_rwlockattr_setkind_np() function. */ #undef ETHR_HAVE_PTHREAD_RWLOCKATTR_SETKIND_NP @@ -115,23 +160,74 @@ /* Define to the size of AO_t if libatomic_ops is used */ #undef ETHR_SIZEOF_AO_T +/* Define if you have _InterlockedAnd() */ +#undef ETHR_HAVE__INTERLOCKEDAND + +/* Define if you have _InterlockedAnd64() */ +#undef ETHR_HAVE__INTERLOCKEDAND64 + +/* Define if you have _InterlockedCompareExchange() */ +#undef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE + /* Define if you have _InterlockedCompareExchange64() */ #undef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64 +/* Define if you have _InterlockedCompareExchange64_acq() */ +#undef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_ACQ + +/* Define if you have _InterlockedCompareExchange64_rel() */ +#undef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_REL + +/* Define if you have _InterlockedCompareExchange_acq() */ +#undef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE_ACQ + +/* Define if you have _InterlockedCompareExchange_rel() */ +#undef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE_REL + +/* Define if you have _InterlockedDecrement() */ +#undef ETHR_HAVE__INTERLOCKEDDECREMENT + /* Define if you have _InterlockedDecrement64() */ #undef ETHR_HAVE__INTERLOCKEDDECREMENT64 -/* Define if you have _InterlockedIncrement64() */ -#undef ETHR_HAVE__INTERLOCKEDINCREMENT64 +/* Define if you have _InterlockedDecrement64_rel() */ +#undef ETHR_HAVE__INTERLOCKEDDECREMENT64_REL -/* Define if you have _InterlockedExchangeAdd64() */ -#undef ETHR_HAVE__INTERLOCKEDEXCHANGEADD64 +/* Define if you have _InterlockedDecrement_rel() */ +#undef ETHR_HAVE__INTERLOCKEDDECREMENT_REL + +/* Define if you have _InterlockedExchange() */ +#undef ETHR_HAVE__INTERLOCKEDEXCHANGE /* Define if you have _InterlockedExchange64() */ #undef ETHR_HAVE__INTERLOCKEDEXCHANGE64 -/* Define if you have _InterlockedAnd64() */ -#undef ETHR_HAVE__INTERLOCKEDAND64 +/* Define if you have _InterlockedExchangeAdd() */ +#undef ETHR_HAVE__INTERLOCKEDEXCHANGEADD + +/* Define if you have _InterlockedExchangeAdd64() */ +#undef ETHR_HAVE__INTERLOCKEDEXCHANGEADD64 + +/* Define if you have _InterlockedExchangeAdd64_acq() */ +#undef ETHR_HAVE__INTERLOCKEDEXCHANGEADD64_ACQ + +/* Define if you have _InterlockedExchangeAdd_acq() */ +#undef ETHR_HAVE__INTERLOCKEDEXCHANGEADD_ACQ + +/* Define if you have _InterlockedIncrement() */ +#undef ETHR_HAVE__INTERLOCKEDINCREMENT + +/* Define if you have _InterlockedIncrement64() */ +#undef ETHR_HAVE__INTERLOCKEDINCREMENT64 + +/* Define if you have _InterlockedIncrement64_acq() */ +#undef ETHR_HAVE__INTERLOCKEDINCREMENT64_ACQ + +/* Define if you have _InterlockedIncrement_acq() */ +#undef ETHR_HAVE__INTERLOCKEDINCREMENT_ACQ + +/* Define if you have _InterlockedOr() */ +#undef ETHR_HAVE__INTERLOCKEDOR /* Define if you have _InterlockedOr64() */ #undef ETHR_HAVE__INTERLOCKEDOR64 diff --git a/erts/include/internal/gcc/ethr_atomic.h b/erts/include/internal/gcc/ethr_atomic.h index 16935084b1..f598f8537b 100644 --- a/erts/include/internal/gcc/ethr_atomic.h +++ b/erts/include/internal/gcc/ethr_atomic.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2010. All Rights Reserved. + * Copyright Ericsson AB 2010-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -25,11 +25,15 @@ #undef ETHR_INCLUDE_ATOMIC_IMPL__ #if !defined(ETHR_GCC_ATOMIC32_H__) && defined(ETHR_ATOMIC_WANT_32BIT_IMPL__) #define ETHR_GCC_ATOMIC32_H__ -#define ETHR_INCLUDE_ATOMIC_IMPL__ 4 +#if defined(ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP32) +# define ETHR_INCLUDE_ATOMIC_IMPL__ 4 +#endif #undef ETHR_ATOMIC_WANT_32BIT_IMPL__ #elif !defined(ETHR_GCC_ATOMIC64_H__) && defined(ETHR_ATOMIC_WANT_64BIT_IMPL__) #define ETHR_GCC_ATOMIC64_H__ -#define ETHR_INCLUDE_ATOMIC_IMPL__ 8 +#if defined(ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP64) +# define ETHR_INCLUDE_ATOMIC_IMPL__ 8 +#endif #undef ETHR_ATOMIC_WANT_64BIT_IMPL__ #endif @@ -45,58 +49,38 @@ # define ETHR_READ_AND_SET_WITHOUT_SYNC_OP__ 1 #endif -#if defined(__x86_64__) || (defined(__i386__) \ - && !defined(ETHR_PRE_PENTIUM4_COMPAT)) -# define ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ 1 -#else -# define ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ 0 -#endif - -/* - * According to the documentation this is what we want: - * #define ETHR_MEMORY_BARRIER __sync_synchronize() - * However, __sync_synchronize() is known to erroneously be - * a noop on at least some platforms with some gcc versions. - * This has suposedly been fixed in some gcc version, but we - * don't know from which version. Therefore, we only use - * it when it has been verified to work. Otherwise - * we use a workaround. - */ -#if defined(__mips__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 2)) -/* __sync_synchronize() has been verified to work here */ -#define ETHR_MEMORY_BARRIER __sync_synchronize() -#define ETHR_READ_DEPEND_MEMORY_BARRIER __sync_synchronize() -#elif defined(__x86_64__) || (defined(__i386__) \ - && !defined(ETHR_PRE_PENTIUM4_COMPAT)) -/* Use fence instructions directly instead of workaround */ -#define ETHR_MEMORY_BARRIER __asm__ __volatile__("mfence" : : : "memory") -#define ETHR_WRITE_MEMORY_BARRIER __asm__ __volatile__("sfence" : : : "memory") -#define ETHR_READ_MEMORY_BARRIER __asm__ __volatile__("lfence" : : : "memory") -#define ETHR_READ_DEPEND_MEMORY_BARRIER __asm__ __volatile__("" : : : "memory") -#else -/* Workaround */ -#define ETHR_MEMORY_BARRIER \ -do { \ - volatile ethr_sint32_t x___ = 0; \ - (void) __sync_val_compare_and_swap(&x___, (ethr_sint32_t) 0, (ethr_sint32_t) 1); \ -} while (0) -#define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_MEMORY_BARRIER -#endif - -#define ETHR_COMPILER_BARRIER __asm__ __volatile__("" : : : "memory") - #endif /* ETHR_GCC_ATOMIC_COMMON__ */ #if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 #define ETHR_HAVE_NATIVE_ATOMIC32 1 +#define ETHR_NATIVE_ATOMIC32_IMPL "gcc" #define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X #define ETHR_ATMC_T__ ethr_native_atomic32_t #define ETHR_AINT_T__ ethr_sint32_t +#if defined(ETHR_HAVE___SYNC_ADD_AND_FETCH32) +# define ETHR_HAVE___SYNC_ADD_AND_FETCH +#endif +#if defined(ETHR_HAVE___SYNC_FETCH_AND_AND32) +# define ETHR_HAVE___SYNC_FETCH_AND_AND +#endif +#if defined(ETHR_HAVE___SYNC_FETCH_AND_OR32) +# define ETHR_HAVE___SYNC_FETCH_AND_OR +#endif #elif ETHR_INCLUDE_ATOMIC_IMPL__ == 8 #define ETHR_HAVE_NATIVE_ATOMIC64 1 +#define ETHR_NATIVE_ATOMIC64_IMPL "gcc" #define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X #define ETHR_ATMC_T__ ethr_native_atomic64_t #define ETHR_AINT_T__ ethr_sint64_t +#if defined(ETHR_HAVE___SYNC_ADD_AND_FETCH64) +# define ETHR_HAVE___SYNC_ADD_AND_FETCH +#endif +#if defined(ETHR_HAVE___SYNC_FETCH_AND_AND64) +# define ETHR_HAVE___SYNC_FETCH_AND_AND +#endif +#if defined(ETHR_HAVE___SYNC_FETCH_AND_OR64) +# define ETHR_HAVE___SYNC_FETCH_AND_OR +#endif #else #error "Unsupported integer size" #endif @@ -108,183 +92,120 @@ typedef struct { #if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADDR 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADDR 1 +#endif + static ETHR_INLINE ETHR_AINT_T__ * ETHR_NATMC_FUNC__(addr)(ETHR_ATMC_T__ *var) { return (ETHR_AINT_T__ *) &var->counter; } -static ETHR_INLINE void -ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value) -{ #if ETHR_READ_AND_SET_WITHOUT_SYNC_OP__ - var->counter = value; + +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET 1 #else - /* - * Unfortunately no __sync_store() or similar exist in the gcc atomic - * op interface. We therefore have to simulate it this way... - */ - ETHR_AINT_T__ act = 0, exp; - do { - exp = act; - act = __sync_val_compare_and_swap(&var->counter, exp, value); - } while (act != exp); +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET 1 #endif -} static ETHR_INLINE void -ETHR_NATMC_FUNC__(init)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value) +ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value) { - ETHR_NATMC_FUNC__(set)(var, value); + var->counter = value; } -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var) -{ +#endif /* ETHR_READ_AND_SET_WITHOUT_SYNC_OP__ */ + #if ETHR_READ_AND_SET_WITHOUT_SYNC_OP__ - return var->counter; + +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ 1 #else - /* - * Unfortunately no __sync_fetch() or similar exist in the gcc atomic - * op interface. We therefore have to simulate it this way... - */ - return __sync_add_and_fetch(&var->counter, (ETHR_AINT_T__) 0); +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ 1 #endif -} - -static ETHR_INLINE void -ETHR_NATMC_FUNC__(add)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) -{ - (void) __sync_add_and_fetch(&var->counter, incr); -} static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(add_return)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) +ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var) { - return __sync_add_and_fetch(&var->counter, incr); + return var->counter; } -static ETHR_INLINE void -ETHR_NATMC_FUNC__(inc)(ETHR_ATMC_T__ *var) -{ - (void) __sync_add_and_fetch(&var->counter, (ETHR_AINT_T__) 1); -} +#endif /* ETHR_READ_AND_SET_WITHOUT_SYNC_OP__ */ -static ETHR_INLINE void -ETHR_NATMC_FUNC__(dec)(ETHR_ATMC_T__ *var) -{ - (void) __sync_sub_and_fetch(&var->counter, (ETHR_AINT_T__) 1); -} +#if defined(ETHR_HAVE___SYNC_ADD_AND_FETCH) -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(inc_return)(ETHR_ATMC_T__ *var) -{ - return __sync_add_and_fetch(&var->counter, (ETHR_AINT_T__) 1); -} +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_MB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_MB 1 +#endif static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(dec_return)(ETHR_ATMC_T__ *var) +ETHR_NATMC_FUNC__(add_return_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) { - return __sync_sub_and_fetch(&var->counter, (ETHR_AINT_T__) 1); + return __sync_add_and_fetch(&var->counter, incr); } -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(and_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) -{ - return __sync_fetch_and_and(&var->counter, mask); -} +#endif -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(or_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) -{ - return (ETHR_AINT_T__) __sync_fetch_and_or(&var->counter, mask); -} +#if defined(ETHR_HAVE___SYNC_FETCH_AND_AND) -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var, - ETHR_AINT_T__ new, - ETHR_AINT_T__ old) -{ - return __sync_val_compare_and_swap(&var->counter, old, new); -} +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_MB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_MB 1 +#endif static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(xchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new) +ETHR_NATMC_FUNC__(and_retold_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) { - ETHR_AINT_T__ exp, act = 0; - do { - exp = act; - act = __sync_val_compare_and_swap(&var->counter, exp, new); - } while (act != exp); - return act; + return __sync_fetch_and_and(&var->counter, mask); } -/* - * Atomic ops with at least specified barriers. - */ - -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(read_acqb)(ETHR_ATMC_T__ *var) -{ -#if ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ - ETHR_AINT_T__ val = var->counter; - ETHR_COMPILER_BARRIER; - return val; -#else - return __sync_add_and_fetch(&var->counter, (ETHR_AINT_T__) 0); #endif -} -static ETHR_INLINE void -ETHR_NATMC_FUNC__(set_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) -{ -#if ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ - ETHR_COMPILER_BARRIER; - var->counter = i; +#if defined(ETHR_HAVE___SYNC_FETCH_AND_OR) + +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_MB 1 #else - (void) ETHR_NATMC_FUNC__(xchg)(var, i); +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_MB 1 #endif -} static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(inc_return_acqb)(ETHR_ATMC_T__ *var) -{ - return ETHR_NATMC_FUNC__(inc_return)(var); -} - -static ETHR_INLINE void -ETHR_NATMC_FUNC__(dec_relb)(ETHR_ATMC_T__ *var) +ETHR_NATMC_FUNC__(or_retold_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) { - ETHR_NATMC_FUNC__(dec)(var); + return (ETHR_AINT_T__) __sync_fetch_and_or(&var->counter, mask); } -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(dec_return_relb)(ETHR_ATMC_T__ *var) -{ - return ETHR_NATMC_FUNC__(dec_return)(var); -} +#endif -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var, - ETHR_AINT_T__ new, - ETHR_AINT_T__ old) -{ - return ETHR_NATMC_FUNC__(cmpxchg)(var, new, old); -} +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_MB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_MB 1 +#endif static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var, - ETHR_AINT_T__ new, - ETHR_AINT_T__ old) +ETHR_NATMC_FUNC__(cmpxchg_mb)(ETHR_ATMC_T__ *var, + ETHR_AINT_T__ new, + ETHR_AINT_T__ old) { - return ETHR_NATMC_FUNC__(cmpxchg)(var, new, old); + return __sync_val_compare_and_swap(&var->counter, old, new); } -#endif +#endif /* ETHR_TRY_INLINE_FUNCS */ #undef ETHR_NATMC_FUNC__ #undef ETHR_ATMC_T__ #undef ETHR_AINT_T__ #undef ETHR_AINT_SUFFIX__ +#undef ETHR_HAVE___SYNC_ADD_AND_FETCH +#undef ETHR_HAVE___SYNC_FETCH_AND_AND +#undef ETHR_HAVE___SYNC_FETCH_AND_OR #endif diff --git a/erts/include/internal/gcc/ethr_dw_atomic.h b/erts/include/internal/gcc/ethr_dw_atomic.h new file mode 100644 index 0000000000..6736f9c547 --- /dev/null +++ b/erts/include/internal/gcc/ethr_dw_atomic.h @@ -0,0 +1,115 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2011. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Native double word atomics using gcc's builtins + * Author: Rickard Green + */ + +#undef ETHR_INCLUDE_DW_ATOMIC_IMPL__ +#ifndef ETHR_GCC_DW_ATOMIC_H__ +# define ETHR_GCC_DW_ATOMIC_H__ +# if ((ETHR_SIZEOF_PTR == 4 \ + && defined(ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP64)) \ + || (ETHR_SIZEOF_PTR == 8 \ + && defined(ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP128) \ + && defined(ETHR_HAVE_INT128_T))) +# define ETHR_INCLUDE_DW_ATOMIC_IMPL__ +# endif +#endif + +#ifdef ETHR_INCLUDE_DW_ATOMIC_IMPL__ +# define ETHR_HAVE_NATIVE_SU_DW_ATOMIC +# define ETHR_NATIVE_DW_ATOMIC_IMPL "gcc" + +# if defined(__i386__) || defined(__x86_64__) +/* + * If ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ is defined, it will be used + * at runtime in order to determine if native or fallback implementation + * should be used. + */ +# define ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ \ + ETHR_X86_RUNTIME_CONF_HAVE_DW_CMPXCHG__ +# endif + +# if ETHR_SIZEOF_PTR == 4 +# define ETHR_DW_NATMC_ALIGN_MASK__ 0x7 +# define ETHR_NATIVE_SU_DW_SINT_T ethr_sint64_t +# elif ETHR_SIZEOF_PTR == 8 +# define ETHR_DW_NATMC_ALIGN_MASK__ 0xf +# define ETHR_NATIVE_SU_DW_SINT_T ethr_sint128_t +# endif + +typedef volatile ETHR_NATIVE_SU_DW_SINT_T * ethr_native_dw_ptr_t; + +/* + * We need 16 byte aligned memory in 64-bit mode, and 8 byte aligned + * memory in 32-bit mode. 16 byte aligned malloc in 64-bit mode is + * not common, and at least some glibc malloc implementations + * only 4 byte align in 32-bit mode. + * + * This code assumes 8 byte aligned memory in 64-bit mode, and 4 byte + * aligned memory in 32-bit mode. A malloc implementation that does + * not adhere to these alignment requirements is seriously broken, + * and we wont bother trying to work around it. + * + * Since memory alignment may be off by one word we need to align at + * runtime. We, therefore, need an extra word allocated. + */ +#define ETHR_DW_NATMC_MEM__(VAR) \ + (&var->c[(int) ((ethr_uint_t) &(VAR)->c[0]) & ETHR_DW_NATMC_ALIGN_MASK__]) +typedef union { + volatile ETHR_NATIVE_SU_DW_SINT_T dw_sint; + volatile ethr_sint_t sint[3]; + volatile char c[ETHR_SIZEOF_PTR*3]; +} ethr_native_dw_atomic_t; + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) + +# ifdef ETHR_DEBUG +# define ETHR_DW_DBG_ALIGNED__(PTR) \ + ETHR_ASSERT((((ethr_uint_t) (PTR)) & ETHR_DW_NATMC_ALIGN_MASK__) == 0); +# else +# define ETHR_DW_DBG_ALIGNED__(PTR) +# endif + +#define ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_ADDR +static ETHR_INLINE ethr_sint_t * +ethr_native_dw_atomic_addr(ethr_native_dw_atomic_t *var) +{ + return (ethr_sint_t *) ETHR_DW_NATMC_MEM__(var); +} + + +#define ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_MB + +static ETHR_INLINE ETHR_NATIVE_SU_DW_SINT_T +ethr_native_su_dw_atomic_cmpxchg_mb(ethr_native_dw_atomic_t *var, + ETHR_NATIVE_SU_DW_SINT_T new, + ETHR_NATIVE_SU_DW_SINT_T old) +{ + ethr_native_dw_ptr_t p = (ethr_native_dw_ptr_t) ETHR_DW_NATMC_MEM__(var); + ETHR_DW_DBG_ALIGNED__(p); + return __sync_val_compare_and_swap(p, old, new); +} + +#endif /* ETHR_TRY_INLINE_FUNCS */ + +#endif /* ETHR_GCC_DW_ATOMIC_H__ */ + diff --git a/erts/include/internal/gcc/ethr_membar.h b/erts/include/internal/gcc/ethr_membar.h new file mode 100644 index 0000000000..7d428fc68e --- /dev/null +++ b/erts/include/internal/gcc/ethr_membar.h @@ -0,0 +1,73 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2011. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Memory barriers when using gcc's builtins + * Author: Rickard Green + */ + +#ifndef ETHR_GCC_MEMBAR_H__ +#define ETHR_GCC_MEMBAR_H__ + +#define ETHR_LoadLoad (1 << 0) +#define ETHR_LoadStore (1 << 1) +#define ETHR_StoreLoad (1 << 2) +#define ETHR_StoreStore (1 << 3) + +/* + * According to the documentation __sync_synchronize() will + * issue a full memory barrier. However, __sync_synchronize() + * is known to erroneously be a noop on at least some + * platforms with some gcc versions. This has suposedly been + * fixed in some gcc version, but we don't know from which + * version. Therefore, we only use it when it has been + * verified to work. Otherwise we use the workaround + * below. + */ + +#if defined(ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP32) +# define ETHR_MB_T__ ethr_sint32_t +#elif defined(ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP64) +# define ETHR_MB_T__ ethr_sint64_t +#else +# error "No __sync_val_compare_and_swap" +#endif +#define ETHR_SYNC_SYNCHRONIZE_WORKAROUND__ \ +do { \ + volatile ETHR_MB_T__ x___ = 0; \ + (void) __sync_val_compare_and_swap(&x___, (ETHR_MB_T__) 0, (ETHR_MB_T__) 1); \ +} while (0) + +#define ETHR_COMPILER_BARRIER __asm__ __volatile__("" : : : "memory") + +#if defined(__mips__) && ETHR_AT_LEAST_GCC_VSN__(4, 2, 0) +# define ETHR_MEMBAR(B) __sync_synchronize() +# define ETHR_READ_DEPEND_MEMORY_BARRIER __sync_synchronize() +#elif ((defined(__powerpc__) || defined(__ppc__)) \ + && ETHR_AT_LEAST_GCC_VSN__(4, 1, 2)) +# define ETHR_MEMBAR(B) __sync_synchronize() +#else /* Use workaround */ +# define ETHR_MEMBAR(B) \ + ETHR_SYNC_SYNCHRONIZE_WORKAROUND__ +# define ETHR_READ_DEPEND_MEMORY_BARRIER \ + ETHR_SYNC_SYNCHRONIZE_WORKAROUND__ +#endif + + +#endif /* ETHR_GCC_MEMBAR_H__ */ diff --git a/erts/include/internal/gcc/ethread.h b/erts/include/internal/gcc/ethread.h index 392a1aa2b2..fcfdc39441 100644 --- a/erts/include/internal/gcc/ethread.h +++ b/erts/include/internal/gcc/ethread.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2010. All Rights Reserved. + * Copyright Ericsson AB 2010-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -25,16 +25,24 @@ #ifndef ETHREAD_GCC_H__ #define ETHREAD_GCC_H__ -#if !defined(ETHR_HAVE_NATIVE_ATOMICS) && defined(ETHR_HAVE_GCC_ATOMIC_OPS) -#define ETHR_HAVE_NATIVE_ATOMICS 1 +#ifndef ETHR_MEMBAR +# include "ethr_membar.h" +#endif + +#if !defined(ETHR_HAVE_NATIVE_ATOMIC32) +# define ETHR_ATOMIC_WANT_32BIT_IMPL__ +# include "ethr_atomic.h" +#endif -#define ETHR_ATOMIC_WANT_32BIT_IMPL__ -#include "ethr_atomic.h" -#if ETHR_SIZEOF_PTR == 8 +#if ETHR_SIZEOF_PTR == 8 && !defined(ETHR_HAVE_NATIVE_ATOMIC64) # define ETHR_ATOMIC_WANT_64BIT_IMPL__ # include "ethr_atomic.h" #endif +#if (!defined(ETHR_HAVE_NATIVE_DW_ATOMIC) \ + && !(ETHR_SIZEOF_PTR == 4 && defined(ETHR_HAVE_NATIVE_ATOMIC64)) \ + && !(ETHR_SIZEOF_PTR == 8 && defined(ETHR_HAVE_NATIVE_ATOMIC128))) +# include "ethr_dw_atomic.h" #endif #endif diff --git a/erts/include/internal/i386/atomic.h b/erts/include/internal/i386/atomic.h index 4e402f261a..fc1b619935 100644 --- a/erts/include/internal/i386/atomic.h +++ b/erts/include/internal/i386/atomic.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * Copyright Ericsson AB 2005-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -25,53 +25,42 @@ */ #undef ETHR_INCLUDE_ATOMIC_IMPL__ -#if !defined(ETHR_X86_ATOMIC32_H__) && defined(ETHR_ATOMIC_WANT_32BIT_IMPL__) -#define ETHR_X86_ATOMIC32_H__ -#define ETHR_INCLUDE_ATOMIC_IMPL__ 4 -#undef ETHR_ATOMIC_WANT_32BIT_IMPL__ -#elif !defined(ETHR_X86_ATOMIC64_H__) && defined(ETHR_ATOMIC_WANT_64BIT_IMPL__) -#define ETHR_X86_ATOMIC64_H__ -#define ETHR_INCLUDE_ATOMIC_IMPL__ 8 -#undef ETHR_ATOMIC_WANT_64BIT_IMPL__ +#if !defined(ETHR_X86_ATOMIC32_H__) \ + && defined(ETHR_ATOMIC_WANT_32BIT_IMPL__) +# define ETHR_X86_ATOMIC32_H__ +# define ETHR_INCLUDE_ATOMIC_IMPL__ 4 +# undef ETHR_ATOMIC_WANT_32BIT_IMPL__ +#elif !defined(ETHR_X86_ATOMIC64_H__) \ + && defined(ETHR_ATOMIC_WANT_64BIT_IMPL__) +# define ETHR_X86_ATOMIC64_H__ +# define ETHR_INCLUDE_ATOMIC_IMPL__ 8 +# undef ETHR_ATOMIC_WANT_64BIT_IMPL__ #endif #ifdef ETHR_INCLUDE_ATOMIC_IMPL__ -#ifndef ETHR_X86_ATOMIC_COMMON__ -#define ETHR_X86_ATOMIC_COMMON__ - -#define ETHR_ATOMIC_HAVE_INC_DEC_INSTRUCTIONS 1 - -#if defined(__x86_64__) || !defined(ETHR_PRE_PENTIUM4_COMPAT) -#define ETHR_MEMORY_BARRIER __asm__ __volatile__("mfence" : : : "memory") -#define ETHR_WRITE_MEMORY_BARRIER __asm__ __volatile__("sfence" : : : "memory") -#define ETHR_READ_MEMORY_BARRIER __asm__ __volatile__("lfence" : : : "memory") -#define ETHR_READ_DEPEND_MEMORY_BARRIER __asm__ __volatile__("" : : : "memory") -#else -#define ETHR_MEMORY_BARRIER \ -do { \ - volatile ethr_sint32_t x___ = 0; \ - __asm__ __volatile__("lock; incl %0" : "=m"(x___) : "m"(x___) : "memory"); \ -} while (0) -#endif - -#endif /* ETHR_X86_ATOMIC_COMMON__ */ - -#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 -#define ETHR_HAVE_NATIVE_ATOMIC32 1 -#define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X -#define ETHR_ATMC_T__ ethr_native_atomic32_t -#define ETHR_AINT_T__ ethr_sint32_t -#define ETHR_AINT_SUFFIX__ "l" -#elif ETHR_INCLUDE_ATOMIC_IMPL__ == 8 -#define ETHR_HAVE_NATIVE_ATOMIC64 1 -#define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X -#define ETHR_ATMC_T__ ethr_native_atomic64_t -#define ETHR_AINT_T__ ethr_sint64_t -#define ETHR_AINT_SUFFIX__ "q" -#else -#error "Unsupported integer size" -#endif +# ifndef ETHR_X86_ATOMIC_COMMON__ +# define ETHR_X86_ATOMIC_COMMON__ +# define ETHR_ATOMIC_HAVE_INC_DEC_INSTRUCTIONS 1 +# endif /* ETHR_X86_ATOMIC_COMMON__ */ + +# if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_NATIVE_ATOMIC32 1 +# define ETHR_NATIVE_ATOMIC32_IMPL "ethread" +# define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X +# define ETHR_ATMC_T__ ethr_native_atomic32_t +# define ETHR_AINT_T__ ethr_sint32_t +# define ETHR_AINT_SUFFIX__ "l" +# elif ETHR_INCLUDE_ATOMIC_IMPL__ == 8 +# define ETHR_HAVE_NATIVE_ATOMIC64 1 +# define ETHR_NATIVE_ATOMIC64_IMPL "ethread" +# define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X +# define ETHR_ATMC_T__ ethr_native_atomic64_t +# define ETHR_AINT_T__ ethr_sint64_t +# define ETHR_AINT_SUFFIX__ "q" +# else +# error "Unsupported integer size" +# endif /* An atomic is an aligned ETHR_AINT_T__ accessed via locked operations. */ @@ -81,87 +70,28 @@ typedef struct { #if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADDR 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADDR 1 +#endif + static ETHR_INLINE ETHR_AINT_T__ * ETHR_NATMC_FUNC__(addr)(ETHR_ATMC_T__ *var) { return (ETHR_AINT_T__ *) &var->counter; } -static ETHR_INLINE void -ETHR_NATMC_FUNC__(init)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) -{ - var->counter = i; -} - -static ETHR_INLINE void -ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) -{ - var->counter = i; -} - -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var) -{ - return var->counter; -} - -static ETHR_INLINE void -ETHR_NATMC_FUNC__(add)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) -{ - __asm__ __volatile__( - "lock; add" ETHR_AINT_SUFFIX__ " %1, %0" - : "=m"(var->counter) - : "ir"(incr), "m"(var->counter)); -} - -static ETHR_INLINE void -ETHR_NATMC_FUNC__(inc)(ETHR_ATMC_T__ *var) -{ - __asm__ __volatile__( - "lock; inc" ETHR_AINT_SUFFIX__ " %0" - : "=m"(var->counter) - : "m"(var->counter)); -} - -static ETHR_INLINE void -ETHR_NATMC_FUNC__(dec)(ETHR_ATMC_T__ *var) -{ - __asm__ __volatile__( - "lock; dec" ETHR_AINT_SUFFIX__ " %0" - : "=m"(var->counter) - : "m"(var->counter)); -} - -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(add_return)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) -{ - ETHR_AINT_T__ tmp; - - tmp = incr; - __asm__ __volatile__( - "lock; xadd" ETHR_AINT_SUFFIX__ " %0, %1" /* xadd didn't exist prior to the 486 */ - : "=r"(tmp) - : "m"(var->counter), "0"(tmp)); - /* now tmp is the atomic's previous value */ - return tmp + incr; -} - -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(inc_return)(ETHR_ATMC_T__ *var) -{ - return ETHR_NATMC_FUNC__(add_return)(var, (ETHR_AINT_T__) 1); -} - -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(dec_return)(ETHR_ATMC_T__ *var) -{ - return ETHR_NATMC_FUNC__(add_return)(var, (ETHR_AINT_T__) -1); -} +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_MB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_MB 1 +#endif static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var, - ETHR_AINT_T__ new, - ETHR_AINT_T__ old) +ETHR_NATMC_FUNC__(cmpxchg_mb)(ETHR_ATMC_T__ *var, + ETHR_AINT_T__ new, + ETHR_AINT_T__ old) { __asm__ __volatile__( "lock; cmpxchg" ETHR_AINT_SUFFIX__ " %2, %3" @@ -171,110 +101,148 @@ ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var, return old; } -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(and_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) -{ - ETHR_AINT_T__ tmp, old; - - tmp = var->counter; - do { - old = tmp; - tmp = ETHR_NATMC_FUNC__(cmpxchg)(var, tmp & mask, tmp); - } while (__builtin_expect(tmp != old, 0)); - /* now tmp is the atomic's previous value */ - return tmp; -} - -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(or_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) -{ - ETHR_AINT_T__ tmp, old; - - tmp = var->counter; - do { - old = tmp; - tmp = ETHR_NATMC_FUNC__(cmpxchg)(var, tmp | mask, tmp); - } while (__builtin_expect(tmp != old, 0)); - /* now tmp is the atomic's previous value */ - return tmp; -} +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_MB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_MB 1 +#endif static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(xchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ val) +ETHR_NATMC_FUNC__(xchg_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ val) { ETHR_AINT_T__ tmp = val; __asm__ __volatile__( "xchg" ETHR_AINT_SUFFIX__ " %0, %1" : "=r"(tmp) - : "m"(var->counter), "0"(tmp)); + : "m"(var->counter), "0"(tmp) + : "memory"); /* now tmp is the atomic's previous value */ return tmp; } -/* - * Atomic ops with at least specified barriers. - */ +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET 1 +#endif -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(read_acqb)(ETHR_ATMC_T__ *var) +static ETHR_INLINE void +ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) { - ETHR_AINT_T__ val; -#if defined(__x86_64__) || !defined(ETHR_PRE_PENTIUM4_COMPAT) - val = var->counter; + var->counter = i; +} + +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_RELB 1 #else - val = ETHR_NATMC_FUNC__(add_return)(var, 0); +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RELB 1 #endif - __asm__ __volatile__("" : : : "memory"); - return val; -} static ETHR_INLINE void ETHR_NATMC_FUNC__(set_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) { - __asm__ __volatile__("" : : : "memory"); -#if defined(__x86_64__) || !defined(ETHR_PRE_PENTIUM4_COMPAT) - var->counter = i; +#if defined(_M_IX86) + if (ETHR_X86_RUNTIME_CONF_HAVE_NO_SSE2__) + (void) ETHR_NATMC_FUNC__(xchg_mb)(var, i); + else +#endif /* _M_IX86 */ + { + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + var->counter = i; + } +} + +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_MB 1 #else - (void) ETHR_NATMC_FUNC__(xchg)(var, i); +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_MB 1 #endif + +static ETHR_INLINE void +ETHR_NATMC_FUNC__(set_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) +{ + (void) ETHR_NATMC_FUNC__(xchg_mb)(var, i); } +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ 1 +#endif + static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(inc_return_acqb)(ETHR_ATMC_T__ *var) +ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var) { - ETHR_AINT_T__ res = ETHR_NATMC_FUNC__(inc_return)(var); - __asm__ __volatile__("" : : : "memory"); - return res; + return var->counter; } +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_MB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_MB 1 +#endif + static ETHR_INLINE void -ETHR_NATMC_FUNC__(dec_relb)(ETHR_ATMC_T__ *var) +ETHR_NATMC_FUNC__(add_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) { - __asm__ __volatile__("" : : : "memory"); - ETHR_NATMC_FUNC__(dec)(var); -} + __asm__ __volatile__( + "lock; add" ETHR_AINT_SUFFIX__ " %1, %0" + : "=m"(var->counter) + : "ir"(incr), "m"(var->counter) + : "memory"); +} -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(dec_return_relb)(ETHR_ATMC_T__ *var) +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_MB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_MB 1 +#endif + +static ETHR_INLINE void +ETHR_NATMC_FUNC__(inc_mb)(ETHR_ATMC_T__ *var) { - __asm__ __volatile__("" : : : "memory"); - return ETHR_NATMC_FUNC__(dec_return)(var); + __asm__ __volatile__( + "lock; inc" ETHR_AINT_SUFFIX__ " %0" + : "=m"(var->counter) + : "m"(var->counter) + : "memory"); } -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var, - ETHR_AINT_T__ new, - ETHR_AINT_T__ old) +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_MB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_MB 1 +#endif + +static ETHR_INLINE void +ETHR_NATMC_FUNC__(dec_mb)(ETHR_ATMC_T__ *var) { - return ETHR_NATMC_FUNC__(cmpxchg)(var, new, old); + __asm__ __volatile__( + "lock; dec" ETHR_AINT_SUFFIX__ " %0" + : "=m"(var->counter) + : "m"(var->counter) + : "memory"); } +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_MB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_MB 1 +#endif + static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var, - ETHR_AINT_T__ new, - ETHR_AINT_T__ old) +ETHR_NATMC_FUNC__(add_return_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) { - return ETHR_NATMC_FUNC__(cmpxchg)(var, new, old); + ETHR_AINT_T__ tmp; + + tmp = incr; + __asm__ __volatile__( + "lock; xadd" ETHR_AINT_SUFFIX__ " %0, %1" /* xadd didn't exist prior to the 486 */ + : "=r"(tmp) + : "m"(var->counter), "0"(tmp) + : "memory"); + /* now tmp is the atomic's previous value */ + return tmp + incr; } #endif /* ETHR_TRY_INLINE_FUNCS */ diff --git a/erts/include/internal/i386/ethr_dw_atomic.h b/erts/include/internal/i386/ethr_dw_atomic.h new file mode 100644 index 0000000000..9fb89bbe43 --- /dev/null +++ b/erts/include/internal/i386/ethr_dw_atomic.h @@ -0,0 +1,278 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2011. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Native double word atomics for x86/x86_64 + * Author: Rickard Green + */ + +#ifndef ETHR_X86_DW_ATOMIC_H__ +#define ETHR_X86_DW_ATOMIC_H__ + +#ifdef ETHR_GCC_HAVE_DW_CMPXCHG_ASM_SUPPORT + +#define ETHR_HAVE_NATIVE_DW_ATOMIC +#define ETHR_NATIVE_DW_ATOMIC_IMPL "ethread" + +/* + * If ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ is defined, it will be used + * at runtime in order to determine if native or fallback implementation + * should be used. + */ +#define ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ \ + ETHR_X86_RUNTIME_CONF_HAVE_DW_CMPXCHG__ + +#if ETHR_SIZEOF_PTR == 4 +typedef volatile ethr_sint64_t * ethr_native_dw_ptr_t; +# define ETHR_DW_NATMC_ALIGN_MASK__ 0x7 +# define ETHR_DW_CMPXCHG_SFX__ "8b" +# define ETHR_NATIVE_SU_DW_SINT_T ethr_sint64_t +#else +#ifdef ETHR_HAVE_INT128_T +# define ETHR_NATIVE_SU_DW_SINT_T ethr_sint128_t +typedef volatile ethr_sint128_t * ethr_native_dw_ptr_t; +#else +typedef struct { + ethr_sint64_t sint64[2]; +} ethr_native_sint128_t__; +typedef volatile ethr_native_sint128_t__ * ethr_native_dw_ptr_t; +#endif +# define ETHR_DW_NATMC_ALIGN_MASK__ 0xf +# define ETHR_DW_CMPXCHG_SFX__ "16b" +#endif + +/* + * We need 16 byte aligned memory in 64-bit mode, and 8 byte aligned + * memory in 32-bit mode. 16 byte aligned malloc in 64-bit mode is + * not common, and at least some glibc malloc implementations + * only 4 byte align in 32-bit mode. + * + * This code assumes 8 byte aligned memory in 64-bit mode, and 4 byte + * aligned memory in 32-bit mode. A malloc implementation that does + * not adhere to these alignment requirements is seriously broken, + * and we wont bother trying to work around it. + * + * Since memory alignment may be off by one word we need to align at + * runtime. We, therefore, need an extra word allocated. + */ +#define ETHR_DW_NATMC_MEM__(VAR) \ + (&var->c[(int) ((ethr_uint_t) &(VAR)->c[0]) & ETHR_DW_NATMC_ALIGN_MASK__]) +typedef union { +#ifdef ETHR_NATIVE_SU_DW_SINT_T + volatile ETHR_NATIVE_SU_DW_SINT_T dw_sint; +#endif + volatile ethr_sint_t sint[3]; + volatile char c[ETHR_SIZEOF_PTR*3]; +} ethr_native_dw_atomic_t; + + +#if (defined(ETHR_TRY_INLINE_FUNCS) \ + || defined(ETHR_ATOMIC_IMPL__) \ + || defined(ETHR_X86_SSE2_ASM_C__)) \ + && ETHR_SIZEOF_PTR == 4 \ + && defined(ETHR_GCC_HAVE_SSE2_ASM_SUPPORT) +ethr_sint64_t +ethr_sse2_native_su_dw_atomic_read(ethr_native_dw_atomic_t *var); +void +ethr_sse2_native_su_dw_atomic_set(ethr_native_dw_atomic_t *var, + ethr_sint64_t val); +#endif + +#if (defined(ETHR_TRY_INLINE_FUNCS) \ + || defined(ETHR_ATOMIC_IMPL__) \ + || defined(ETHR_X86_SSE2_ASM_C__)) +# ifdef ETHR_DEBUG +# define ETHR_DW_DBG_ALIGNED__(PTR) \ + ETHR_ASSERT((((ethr_uint_t) (PTR)) & ETHR_DW_NATMC_ALIGN_MASK__) == 0); +# else +# define ETHR_DW_DBG_ALIGNED__(PTR) +# endif +#endif + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) + +#define ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_ADDR +static ETHR_INLINE ethr_sint_t * +ethr_native_dw_atomic_addr(ethr_native_dw_atomic_t *var) +{ + return (ethr_sint_t *) ETHR_DW_NATMC_MEM__(var); +} + +#if ETHR_SIZEOF_PTR == 4 && defined(__PIC__) && __PIC__ +/* + * When position independent code is used in 32-bit mode, the EBX register + * is used for storage of global offset table address, and we may not + * use it as input or output in an asm. We need to save and restore the + * EBX register explicitly (for some reason gcc doesn't provide this + * service to us). + */ +# define ETHR_NO_CLOBBER_EBX__ 1 +#else +# define ETHR_NO_CLOBBER_EBX__ 0 +#endif + +#if ETHR_NO_CLOBBER_EBX__ && !defined(ETHR_CMPXCHG8B_REGISTER_SHORTAGE) +/* When no optimization is on, we'll run into a register shortage */ +# if defined(ETHR_DEBUG) || defined(DEBUG) || defined(VALGRIND) \ + || defined(GCOV) || defined(PURIFY) || defined(PURECOV) +# define ETHR_CMPXCHG8B_REGISTER_SHORTAGE 1 +# else +# define ETHR_CMPXCHG8B_REGISTER_SHORTAGE 0 +# endif +#endif + + +#define ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_MB + +static ETHR_INLINE int +ethr_native_dw_atomic_cmpxchg_mb(ethr_native_dw_atomic_t *var, + ethr_sint_t *new, + ethr_sint_t *xchg) +{ + ethr_native_dw_ptr_t p = (ethr_native_dw_ptr_t) ETHR_DW_NATMC_MEM__(var); + char xchgd; + + ETHR_DW_DBG_ALIGNED__(p); + + __asm__ __volatile__( +#if ETHR_NO_CLOBBER_EBX__ + "pushl %%ebx\n\t" +# if ETHR_CMPXCHG8B_REGISTER_SHORTAGE + "movl (%7), %%ebx\n\t" + "movl 4(%7), %%ecx\n\t" +# else + "movl %8, %%ebx\n\t" +# endif +#endif + "lock; cmpxchg" ETHR_DW_CMPXCHG_SFX__ " %0\n\t" + "setz %3\n\t" +#if ETHR_NO_CLOBBER_EBX__ + "popl %%ebx\n\t" +#endif + : "=m"(*p), "=d"(xchg[1]), "=a"(xchg[0]), "=c"(xchgd) + : "m"(*p), "1"(xchg[1]), "2"(xchg[0]), +#if ETHR_NO_CLOBBER_EBX__ +# if ETHR_CMPXCHG8B_REGISTER_SHORTAGE + "3"(new) +# else + "3"(new[1]), + "r"(new[0]) +# endif +#else + "3"(new[1]), + "b"(new[0]) +#endif + : "cc", "memory"); + + return (int) xchgd; +} + +#undef ETHR_NO_CLOBBER_EBX__ + +#if ETHR_SIZEOF_PTR == 4 && defined(ETHR_GCC_HAVE_SSE2_ASM_SUPPORT) + +typedef union { + ethr_sint64_t sint64; + ethr_sint_t sint[2]; +} ethr_dw_atomic_no_sse2_convert_t; + +#define ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ + +static ETHR_INLINE ethr_sint64_t +ethr_native_su_dw_atomic_read(ethr_native_dw_atomic_t *var) +{ + if (ETHR_X86_RUNTIME_CONF_HAVE_SSE2__) + return ethr_sse2_native_su_dw_atomic_read(var); + else { + ethr_sint_t new[2]; + ethr_dw_atomic_no_sse2_convert_t xchg; + new[0] = new[1] = xchg.sint[0] = xchg.sint[1] = 0x83838383; + (void) ethr_native_dw_atomic_cmpxchg_mb(var, new, xchg.sint); + return xchg.sint64; + } +} + +#define ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET + +static ETHR_INLINE void +ethr_native_su_dw_atomic_set(ethr_native_dw_atomic_t *var, + ethr_sint64_t val) +{ + if (ETHR_X86_RUNTIME_CONF_HAVE_SSE2__) + ethr_sse2_native_su_dw_atomic_set(var, val); + else { + ethr_sint_t xchg[2] = {0, 0}; + ethr_dw_atomic_no_sse2_convert_t new; + new.sint64 = val; + while (!ethr_native_dw_atomic_cmpxchg_mb(var, new.sint, xchg)); + } +} + +#endif /* ETHR_SIZEOF_PTR == 4 */ + +#endif /* ETHR_TRY_INLINE_FUNCS */ + +#if defined(ETHR_X86_SSE2_ASM_C__) \ + && ETHR_SIZEOF_PTR == 4 \ + && defined(ETHR_GCC_HAVE_SSE2_ASM_SUPPORT) + +/* + * 8-byte aligned loads and stores of 64-bit values are atomic from + * pentium and forward. An ordinary volatile load or store in 32-bit + * mode generates two 32-bit operations (at least with gcc-4.1.2 using + * -msse2). In order to guarantee one 64-bit load/store operation + * from/to memory we load/store via an xmm register using movq. + * + * Load/store can be achieved using cmpxchg8b, however, using movq is + * much faster. Unfortunately we cannot do the same thing in 64-bit + * mode; instead, we have to do loads and stores via cmpxchg16b. + * + * We do not inline these, but instead compile these into a separate + * object file using -msse2. This since we don't want to use -msse2 for + * the whole system. If we detect sse2 support (pentium4 and forward) + * at runtime, we use them; otherwise, we fall back to using cmpxchg8b + * for loads and stores. This way the binary can be moved between + * processors with and without sse2 support. + */ + +ethr_sint64_t +ethr_sse2_native_su_dw_atomic_read(ethr_native_dw_atomic_t *var) +{ + ethr_native_dw_ptr_t p = (ethr_native_dw_ptr_t) ETHR_DW_NATMC_MEM__(var); + ethr_sint64_t val; + ETHR_DW_DBG_ALIGNED__(p); + __asm__ __volatile__("movq %1, %0\n\t" : "=x"(val) : "m"(*p) : "memory"); + return val; +} + +void +ethr_sse2_native_su_dw_atomic_set(ethr_native_dw_atomic_t *var, + ethr_sint64_t val) +{ + ethr_native_dw_ptr_t p = (ethr_native_dw_ptr_t) ETHR_DW_NATMC_MEM__(var); + ETHR_DW_DBG_ALIGNED__(p); + __asm__ __volatile__("movq %1, %0\n\t" : "=m"(*p) : "x"(val) : "memory"); +} + +#endif /* ETHR_X86_SSE2_ASM_C__ */ + +#endif /* ETHR_GCC_HAVE_DW_CMPXCHG_ASM_SUPPORT */ + +#endif /* ETHR_X86_DW_ATOMIC_H__ */ + diff --git a/erts/include/internal/i386/ethr_membar.h b/erts/include/internal/i386/ethr_membar.h new file mode 100644 index 0000000000..92d9de7f3f --- /dev/null +++ b/erts/include/internal/i386/ethr_membar.h @@ -0,0 +1,114 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2011. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Memory barriers for x86/x86-64 + * Author: Rickard Green + */ + +#ifndef ETHR_X86_MEMBAR_H__ +#define ETHR_X86_MEMBAR_H__ + +#define ETHR_LoadLoad (1 << 0) +#define ETHR_LoadStore (1 << 1) +#define ETHR_StoreLoad (1 << 2) +#define ETHR_StoreStore (1 << 3) + +#define ETHR_NO_SSE2_MEMORY_BARRIER__ \ +do { \ + volatile ethr_sint32_t x__ = 0; \ + __asm__ __volatile__ ("lock; orl $0x0, %0\n\t" \ + : "=m"(x__) \ + : "m"(x__) \ + : "memory"); \ +} while (0) + +static __inline__ void +ethr_cfence__(void) +{ + __asm__ __volatile__ ("" : : : "memory"); +} + +static __inline__ void +ethr_mfence__(void) +{ +#if ETHR_SIZEOF_PTR == 4 + if (ETHR_X86_RUNTIME_CONF_HAVE_NO_SSE2__) + ETHR_NO_SSE2_MEMORY_BARRIER__; + else +#endif + __asm__ __volatile__ ("mfence\n\t" : : : "memory"); +} + +static __inline__ void +ethr_sfence__(void) +{ +#if ETHR_SIZEOF_PTR == 4 + if (ETHR_X86_RUNTIME_CONF_HAVE_NO_SSE2__) + ETHR_NO_SSE2_MEMORY_BARRIER__; + else +#endif + __asm__ __volatile__ ("sfence\n\t" : : : "memory"); +} + +static __inline__ void +ethr_lfence__(void) +{ +#if ETHR_SIZEOF_PTR == 4 + if (ETHR_X86_RUNTIME_CONF_HAVE_NO_SSE2__) + ETHR_NO_SSE2_MEMORY_BARRIER__; + else +#endif + __asm__ __volatile__ ("lfence\n\t" : : : "memory"); +} + +#define ETHR_X86_OUT_OF_ORDER_MEMBAR(B) \ + ETHR_CHOOSE_EXPR((B) == ETHR_StoreStore, \ + ethr_sfence__(), \ + ETHR_CHOOSE_EXPR((B) == ETHR_LoadLoad, \ + ethr_lfence__(), \ + ethr_mfence__())) + +#ifdef ETHR_X86_OUT_OF_ORDER + +#define ETHR_MEMBAR(B) \ + ETHR_X86_OUT_OF_ORDER_MEMBAR((B)) + +#else /* !ETHR_X86_OUT_OF_ORDER (the default) */ + +/* + * We assume that only stores before loads may be reordered. That is, + * we assume that *no* instructions like these are used: + * - CLFLUSH, + * - streaming stores executed with non-temporal move, + * - string operations, or + * - other instructions which aren't LoadLoad, LoadStore, and StoreStore + * ordered by themselves + * If such instructions are used, either insert memory barriers + * using ETHR_X86_OUT_OF_ORDER_MEMBAR() at appropriate places, or + * define ETHR_X86_OUT_OF_ORDER. For more info see Intel 64 and IA-32 + * Architectures Software Developer's Manual; Vol 3A; Chapter 8.2.2. + */ + +#define ETHR_MEMBAR(B) \ + ETHR_CHOOSE_EXPR((B) & ETHR_StoreLoad, ethr_mfence__(), ethr_cfence__()) + +#endif /* !ETHR_X86_OUT_OF_ORDER */ + +#endif /* ETHR_X86_MEMBAR_H__ */ diff --git a/erts/include/internal/i386/ethread.h b/erts/include/internal/i386/ethread.h index b5a17caefb..80e4dc7b99 100644 --- a/erts/include/internal/i386/ethread.h +++ b/erts/include/internal/i386/ethread.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * Copyright Ericsson AB 2005-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -24,17 +24,15 @@ #ifndef ETHREAD_I386_ETHREAD_H #define ETHREAD_I386_ETHREAD_H +#include "ethr_membar.h" #define ETHR_ATOMIC_WANT_32BIT_IMPL__ #include "atomic.h" #if ETHR_SIZEOF_PTR == 8 # define ETHR_ATOMIC_WANT_64BIT_IMPL__ # include "atomic.h" #endif +#include "ethr_dw_atomic.h" #include "spinlock.h" #include "rwlock.h" -#define ETHR_HAVE_NATIVE_ATOMICS 1 -#define ETHR_HAVE_NATIVE_SPINLOCKS 1 -#define ETHR_HAVE_NATIVE_RWSPINLOCKS 1 - #endif /* ETHREAD_I386_ETHREAD_H */ diff --git a/erts/include/internal/i386/rwlock.h b/erts/include/internal/i386/rwlock.h index be47f459ce..1a8cd7da0c 100644 --- a/erts/include/internal/i386/rwlock.h +++ b/erts/include/internal/i386/rwlock.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * Copyright Ericsson AB 2005-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -26,6 +26,9 @@ #ifndef ETHREAD_I386_RWLOCK_H #define ETHREAD_I386_RWLOCK_H +#define ETHR_HAVE_NATIVE_RWSPINLOCKS 1 +#define ETHR_NATIVE_RWSPINLOCK_IMPL "ethread" + /* XXX: describe the algorithm */ typedef struct { volatile int lock; diff --git a/erts/include/internal/i386/spinlock.h b/erts/include/internal/i386/spinlock.h index 0325324895..a84fba91b1 100644 --- a/erts/include/internal/i386/spinlock.h +++ b/erts/include/internal/i386/spinlock.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * Copyright Ericsson AB 2005-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -24,6 +24,9 @@ #ifndef ETHREAD_I386_SPINLOCK_H #define ETHREAD_I386_SPINLOCK_H +#define ETHR_HAVE_NATIVE_SPINLOCKS 1 +#define ETHR_NATIVE_SPINLOCK_IMPL "ethread" + /* A spinlock is the low byte of an aligned 32-bit integer. * A non-zero value means that the lock is locked. */ @@ -46,16 +49,20 @@ ethr_native_spin_unlock(ethr_native_spinlock_t *lock) * On i386 this needs to be a locked operation * to avoid Pentium Pro errata 66 and 92. */ -#if defined(__x86_64__) || !defined(ETHR_PRE_PENTIUM4_COMPAT) - __asm__ __volatile__("" : : : "memory"); - *(unsigned char*)&lock->lock = 0; -#else - char tmp = 0; - __asm__ __volatile__( - "xchgb %b0, %1" - : "=q"(tmp), "=m"(lock->lock) - : "0"(tmp) : "memory"); +#if !defined(__x86_64__) + if (ETHR_X86_RUNTIME_CONF_HAVE_NO_SSE2__) { + char tmp = 0; + __asm__ __volatile__( + "xchgb %b0, %1" + : "=q"(tmp), "=m"(lock->lock) + : "0"(tmp) : "memory"); + } + else #endif + { + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + *(unsigned char*)&lock->lock = 0; + } } static ETHR_INLINE int diff --git a/erts/include/internal/libatomic_ops/ethr_atomic.h b/erts/include/internal/libatomic_ops/ethr_atomic.h index 93bc06036f..fb1288c330 100644 --- a/erts/include/internal/libatomic_ops/ethr_atomic.h +++ b/erts/include/internal/libatomic_ops/ethr_atomic.h @@ -25,16 +25,6 @@ #ifndef ETHR_LIBATOMIC_OPS_ATOMIC_H__ #define ETHR_LIBATOMIC_OPS_ATOMIC_H__ -#if !defined(ETHR_HAVE_NATIVE_ATOMICS) && defined(ETHR_HAVE_LIBATOMIC_OPS) -#define ETHR_HAVE_NATIVE_ATOMICS 1 - -#if (defined(__i386__) && !defined(ETHR_PRE_PENTIUM4_COMPAT)) \ - || defined(__x86_64__) -#define AO_USE_PENTIUM4_INSTRS -#endif - -#include "atomic_ops.h" - /* * libatomic_ops can be downloaded from: * http://www.hpl.hp.com/research/linux/atomic_ops/ @@ -46,21 +36,18 @@ * - AO_store() * - AO_compare_and_swap() * - * The `AO_t' type also have to be at least as large as the `void *' type. */ -#if ETHR_SIZEOF_AO_T < ETHR_SIZEOF_PTR -#error The AO_t type is too small -#endif - #if ETHR_SIZEOF_AO_T == 4 #define ETHR_HAVE_NATIVE_ATOMIC32 1 +#define ETHR_NATIVE_ATOMIC32_IMPL "libatomic_ops" #define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X #define ETHR_ATMC_T__ ethr_native_atomic32_t #define ETHR_AINT_T__ ethr_sint32_t #define ETHR_AINT_SUFFIX__ "l" #elif ETHR_SIZEOF_AO_T == 8 #define ETHR_HAVE_NATIVE_ATOMIC64 1 +#define ETHR_NATIVE_ATOMIC64_IMPL "libatomic_ops" #define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X #define ETHR_ATMC_T__ ethr_native_atomic64_t #define ETHR_AINT_T__ ethr_sint64_t @@ -69,61 +56,29 @@ #error "Unsupported integer size" #endif -#if ETHR_SIZEOF_AO_T == 8 -typedef union { - volatile AO_t counter; - ethr_sint32_t sint32[2]; -} ETHR_ATMC_T__; -#else typedef struct { volatile AO_t counter; } ETHR_ATMC_T__; -#endif -#define ETHR_MEMORY_BARRIER AO_nop_full() -#ifdef AO_HAVE_nop_write -# define ETHR_WRITE_MEMORY_BARRIER AO_nop_write() -#else -# define ETHR_WRITE_MEMORY_BARRIER ETHR_MEMORY_BARRIER -#endif -#ifdef AO_HAVE_nop_read -# define ETHR_READ_MEMORY_BARRIER AO_nop_read() -#else -# define ETHR_READ_MEMORY_BARRIER ETHR_MEMORY_BARRIER -#endif -#ifdef AO_NO_DD_ORDERING -# define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_READ_MEMORY_BARRIER +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) + +#if ETHR_SIZEOF_AO_T == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADDR 1 #else -# define ETHR_READ_DEPEND_MEMORY_BARRIER AO_compiler_barrier() +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADDR 1 #endif -#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) - static ETHR_INLINE ETHR_AINT_T__ * ETHR_NATMC_FUNC__(addr)(ETHR_ATMC_T__ *var) { return (ETHR_AINT_T__ *) &var->counter; } -#if ETHR_SIZEOF_AO_T == 8 -/* - * We also need to provide an ethr_native_atomic32_addr(), since - * this 64-bit implementation will be used implementing 32-bit - * native atomics. - */ - -static ETHR_INLINE ethr_sint32_t * -ethr_native_atomic32_addr(ETHR_ATMC_T__ *var) -{ - ETHR_ASSERT(((void *) &var->sint32[0]) == ((void *) &var->counter)); -#ifdef ETHR_BIGENDIAN - return &var->sint32[1]; +#if ETHR_SIZEOF_AO_T == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET 1 #else - return &var->sint32[0]; +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET 1 #endif -} - -#endif /* ETHR_SIZEOF_AO_T == 8 */ static ETHR_INLINE void ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value) @@ -131,197 +86,198 @@ ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value) AO_store(&var->counter, (AO_t) value); } +#ifdef AO_HAVE_store_release + +#if ETHR_SIZEOF_AO_T == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_RELB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RELB 1 +#endif + static ETHR_INLINE void -ETHR_NATMC_FUNC__(init)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value) +ETHR_NATMC_FUNC__(set_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value) { - ETHR_NATMC_FUNC__(set)(var, value); + AO_store_release(&var->counter, (AO_t) value); } +#endif + +#if ETHR_SIZEOF_AO_T == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ 1 +#endif + static ETHR_INLINE ETHR_AINT_T__ ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var) { return (ETHR_AINT_T__) AO_load(&var->counter); } -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(add_return)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) -{ -#ifdef AO_HAVE_fetch_and_add_full - return ((ETHR_AINT_T__) AO_fetch_and_add_full(&var->counter, (AO_t) incr)) + incr; +#ifdef AO_HAVE_load_acquire + +#if ETHR_SIZEOF_AO_T == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_ACQB 1 #else - while (1) { - AO_t exp = AO_load(&var->counter); - AO_t new = exp + (AO_t) incr; - if (AO_compare_and_swap_full(&var->counter, exp, new)) - return (ETHR_AINT_T__) new; - } +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_ACQB 1 #endif -} -static ETHR_INLINE void -ETHR_NATMC_FUNC__(add)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(read_acqb)(ETHR_ATMC_T__ *var) { - (void) ETHR_NATMC_FUNC__(add_return)(var, incr); + return (ETHR_AINT_T__) AO_load_acquire(&var->counter); } -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(inc_return)(ETHR_ATMC_T__ *var) -{ -#ifdef AO_HAVE_fetch_and_add1_full - return ((ETHR_AINT_T__) AO_fetch_and_add1_full(&var->counter)) + 1; -#else - return ETHR_NATMC_FUNC__(add_return)(var, 1); #endif -} -static ETHR_INLINE void -ETHR_NATMC_FUNC__(inc)(ETHR_ATMC_T__ *var) -{ - (void) ETHR_NATMC_FUNC__(inc_return)(var); -} +#ifdef AO_HAVE_fetch_and_add -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(dec_return)(ETHR_ATMC_T__ *var) -{ -#ifdef AO_HAVE_fetch_and_sub1_full - return ((ETHR_AINT_T__) AO_fetch_and_sub1_full(&var->counter)) - 1; +#if ETHR_SIZEOF_AO_T == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN 1 #else - return ETHR_NATMC_FUNC__(add_return)(var, -1); +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN 1 #endif -} - -static ETHR_INLINE void -ETHR_NATMC_FUNC__(dec)(ETHR_ATMC_T__ *var) -{ - (void) ETHR_NATMC_FUNC__(dec_return)(var); -} static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(and_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) +ETHR_NATMC_FUNC__(add_return)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) { - while (1) { - AO_t exp = AO_load(&var->counter); - AO_t new = exp & ((AO_t) mask); - if (AO_compare_and_swap_full(&var->counter, exp, new)) - return (ETHR_AINT_T__) exp; - } + return ((ETHR_AINT_T__) AO_fetch_and_add(&var->counter, (AO_t) incr)) + incr; } -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(or_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) -{ - while (1) { - AO_t exp = AO_load(&var->counter); - AO_t new = exp | ((AO_t) mask); - if (AO_compare_and_swap_full(&var->counter, exp, new)) - return (ETHR_AINT_T__) exp; - } -} +#endif -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var, - ETHR_AINT_T__ new, - ETHR_AINT_T__ exp) -{ - ETHR_AINT_T__ act; - do { - if (AO_compare_and_swap_full(&var->counter, (AO_t) exp, (AO_t) new)) - return exp; - act = (ETHR_AINT_T__) AO_load(&var->counter); - } while (act == exp); - return act; -} +#ifdef AO_HAVE_fetch_and_add1 + +#if ETHR_SIZEOF_AO_T == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN 1 +#endif static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(xchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new) +ETHR_NATMC_FUNC__(inc_return)(ETHR_ATMC_T__ *var) { - while (1) { - AO_t exp = AO_load(&var->counter); - if (AO_compare_and_swap_full(&var->counter, exp, (AO_t) new)) - return (ETHR_AINT_T__) exp; - } + return ((ETHR_AINT_T__) AO_fetch_and_add1(&var->counter)) + 1; } -/* - * Atomic ops with at least specified barriers. - */ +#endif -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(read_acqb)(ETHR_ATMC_T__ *var) -{ -#ifdef AO_HAVE_load_acquire - return (ETHR_AINT_T__) AO_load_acquire(&var->counter); +#ifdef AO_HAVE_fetch_and_add1_acquire + +#if ETHR_SIZEOF_AO_T == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_ACQB 1 #else - ETHR_AINT_T__ res = ETHR_NATMC_FUNC__(read)(var); - ETHR_MEMORY_BARRIER; - return res; +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_ACQB 1 #endif -} static ETHR_INLINE ETHR_AINT_T__ ETHR_NATMC_FUNC__(inc_return_acqb)(ETHR_ATMC_T__ *var) { -#ifdef AO_HAVE_fetch_and_add1_acquire return ((ETHR_AINT_T__) AO_fetch_and_add1_acquire(&var->counter)) + 1; +} + +#endif + +#ifdef AO_HAVE_fetch_and_sub1 + +#if ETHR_SIZEOF_AO_T == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN 1 #else - ETHR_AINT_T__ res = ETHR_NATMC_FUNC__(add_return)(var, 1); - return res; +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN 1 #endif -} -static ETHR_INLINE void -ETHR_NATMC_FUNC__(set_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(dec_return)(ETHR_ATMC_T__ *var) { -#ifdef AO_HAVE_store_release - AO_store_release(&var->counter, (AO_t) value); + return ((ETHR_AINT_T__) AO_fetch_and_sub1(&var->counter)) - 1; +} + +#endif + +#ifdef AO_HAVE_fetch_and_sub1_release + +#if ETHR_SIZEOF_AO_T == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_RELB 1 #else - ETHR_MEMORY_BARRIER; - ETHR_NATMC_FUNC__(set)(var, value); +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_RELB 1 #endif -} static ETHR_INLINE ETHR_AINT_T__ ETHR_NATMC_FUNC__(dec_return_relb)(ETHR_ATMC_T__ *var) { -#ifdef AO_HAVE_fetch_and_sub1_release return ((ETHR_AINT_T__) AO_fetch_and_sub1_release(&var->counter)) - 1; +} + +#endif + +#ifdef AO_HAVE_compare_and_swap + +#if ETHR_SIZEOF_AO_T == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG 1 #else - return ETHR_NATMC_FUNC__(dec_return)(var); +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG 1 #endif -} -static ETHR_INLINE void -ETHR_NATMC_FUNC__(dec_relb)(ETHR_ATMC_T__ *var) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var, + ETHR_AINT_T__ new, + ETHR_AINT_T__ exp) { - (void) ETHR_NATMC_FUNC__(dec_return_relb)(var); + ETHR_AINT_T__ act; + do { + if (AO_compare_and_swap(&var->counter, (AO_t) exp, (AO_t) new)) + return exp; + act = (ETHR_AINT_T__) AO_load(&var->counter); + } while (act == exp); + return act; } +#endif + +#ifdef AO_HAVE_compare_and_swap_acquire + +#if ETHR_SIZEOF_AO_T == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_ACQB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_ACQB 1 +#endif + static ETHR_INLINE ETHR_AINT_T__ ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new, ETHR_AINT_T__ exp) { -#ifdef AO_HAVE_compare_and_swap_acquire ETHR_AINT_T__ act; do { if (AO_compare_and_swap_acquire(&var->counter, (AO_t) exp, (AO_t) new)) return exp; +#ifdef AO_HAVE_load_acquire + act = (ETHR_AINT_T__) AO_load_acquire(&var->counter); +#else act = (ETHR_AINT_T__) AO_load(&var->counter); +#endif } while (act == exp); +#ifndef AO_HAVE_load_acquire AO_nop_full(); +#endif return act; +} + +#endif + +#ifdef AO_HAVE_compare_and_swap_release + +#if ETHR_SIZEOF_AO_T == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_RELB 1 #else - ETHR_AINT_T__ act = ETHR_NATMC_FUNC__(cmpxchg)(var, new, exp); - return act; +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RELB 1 #endif -} static ETHR_INLINE ETHR_AINT_T__ ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new, ETHR_AINT_T__ exp) { -#ifdef AO_HAVE_compare_and_swap_release ETHR_AINT_T__ act; do { if (AO_compare_and_swap_release(&var->counter, (AO_t) exp, (AO_t) new)) @@ -329,11 +285,9 @@ ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var, act = (ETHR_AINT_T__) AO_load(&var->counter); } while (act == exp); return act; -#else - return ETHR_NATMC_FUNC__(cmpxchg)(var, new, exp); -#endif } +#endif #endif /* ETHR_TRY_INLINE_FUNCS */ @@ -341,6 +295,4 @@ ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var, #undef ETHR_ATMC_T__ #undef ETHR_AINT_T__ -#endif /* !defined(ETHR_HAVE_NATIVE_ATOMICS) && defined(ETHR_HAVE_LIBATOMIC_OPS) */ - #endif /* ETHR_LIBATOMIC_OPS_ATOMIC_H__ */ diff --git a/erts/include/internal/libatomic_ops/ethr_membar.h b/erts/include/internal/libatomic_ops/ethr_membar.h new file mode 100644 index 0000000000..b8530a0094 --- /dev/null +++ b/erts/include/internal/libatomic_ops/ethr_membar.h @@ -0,0 +1,75 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2011. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Memory barriers when using libatomic_ops + * Author: Rickard Green + */ + +#ifndef ETHR_LIBATOMIC_OPS_MEMBAR_H__ +#define ETHR_LIBATOMIC_OPS_MEMBAR_H__ + +#define ETHR_LoadLoad (1 << 0) +#define ETHR_LoadStore (1 << 1) +#define ETHR_StoreLoad (1 << 2) +#define ETHR_StoreStore (1 << 3) + +#ifndef AO_HAVE_nop_full +# error "No AO_nop_full()" +#endif + +static __inline__ void +ethr_mb__(void) +{ + AO_nop_full(); +} + +static __inline__ void +ethr_rb__(void) +{ +#ifdef AO_HAVE_nop_read + AO_nop_read(); +#else + AO_nop_full(); +#endif +} + +static __inline__ void +ethr_wb__(void) +{ +#ifdef AO_HAVE_nop_write + AO_nop_write(); +#else + AO_nop_full(); +#endif +} + +#define ETHR_MEMBAR(B) \ + ETHR_CHOOSE_EXPR((B) == ETHR_StoreStore, \ + ethr_wb__(), \ + ETHR_CHOOSE_EXPR((B) == ETHR_LoadLoad, \ + ethr_rb__(), \ + ethr_mb__())) + +#define ETHR_COMPILER_BARRIER AO_compiler_barrier() +#ifdef AO_NO_DD_ORDERING +# define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_READ_MEMORY_BARRIER +#endif + +#endif /* ETHR_LIBATOMIC_OPS_MEMBAR_H__ */ diff --git a/erts/include/internal/libatomic_ops/ethread.h b/erts/include/internal/libatomic_ops/ethread.h index ee73ba73bc..e1fdd588bb 100644 --- a/erts/include/internal/libatomic_ops/ethread.h +++ b/erts/include/internal/libatomic_ops/ethread.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2010. All Rights Reserved. + * Copyright Ericsson AB 2010-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -25,6 +25,18 @@ #ifndef ETHREAD_LIBATOMIC_OPS_H__ #define ETHREAD_LIBATOMIC_OPS_H__ +#if (defined(ETHR_HAVE_LIBATOMIC_OPS) \ + && ((ETHR_SIZEOF_AO_T == 4 && !defined(ETHR_HAVE_NATIVE_ATOMIC32)) \ + || (ETHR_SIZEOF_AO_T == 8 && !defined(ETHR_HAVE_NATIVE_ATOMIC64)))) + +#if defined(__x86_64__) +#define AO_USE_PENTIUM4_INSTRS +#endif + +#include "atomic_ops.h" +#include "ethr_membar.h" #include "ethr_atomic.h" #endif + +#endif diff --git a/erts/include/internal/ppc32/atomic.h b/erts/include/internal/ppc32/atomic.h index 522f433649..6001620677 100644 --- a/erts/include/internal/ppc32/atomic.h +++ b/erts/include/internal/ppc32/atomic.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * Copyright Ericsson AB 2005-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -29,27 +29,31 @@ #define ETHREAD_PPC_ATOMIC_H #define ETHR_HAVE_NATIVE_ATOMIC32 1 +#define ETHR_NATIVE_ATOMIC32_IMPL "ethread" typedef struct { volatile ethr_sint32_t counter; } ethr_native_atomic32_t; -#define ETHR_MEMORY_BARRIER __asm__ __volatile__("sync" : : : "memory") - #if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADDR 1 + static ETHR_INLINE ethr_sint32_t * ethr_native_atomic32_addr(ethr_native_atomic32_t *var) { return (ethr_sint32_t *) &var->counter; } +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET 1 + static ETHR_INLINE void -ethr_native_atomic32_init(ethr_native_atomic32_t *var, ethr_sint32_t i) +ethr_native_atomic32_set(ethr_native_atomic32_t *var, ethr_sint32_t i) { var->counter = i; } -#define ethr_native_atomic32_set(v, i) ethr_native_atomic32_init((v), (i)) + +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ 1 static ETHR_INLINE ethr_sint32_t ethr_native_atomic32_read(ethr_native_atomic32_t *var) @@ -57,57 +61,68 @@ ethr_native_atomic32_read(ethr_native_atomic32_t *var) return var->counter; } +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN 1 + static ETHR_INLINE ethr_sint32_t ethr_native_atomic32_add_return(ethr_native_atomic32_t *var, ethr_sint32_t incr) { ethr_sint32_t tmp; __asm__ __volatile__( - "eieio\n\t" "1:\t" "lwarx %0,0,%1\n\t" "add %0,%2,%0\n\t" "stwcx. %0,0,%1\n\t" "bne- 1b\n\t" - "isync" : "=&r"(tmp) : "r"(&var->counter), "r"(incr) : "cc", "memory"); return tmp; } -static ETHR_INLINE void -ethr_native_atomic32_add(ethr_native_atomic32_t *var, ethr_sint32_t incr) +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_ACQB 1 + +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_add_return_acqb(ethr_native_atomic32_t *var, ethr_sint32_t incr) { - /* XXX: could use weaker version here w/o eieio+isync */ - (void)ethr_native_atomic32_add_return(var, incr); + ethr_sint32_t res; + res = ethr_native_atomic32_add_return(var, incr); + __asm__ __volatile("isync\n\t" : : : "memory"); + return res; } +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN 1 + static ETHR_INLINE ethr_sint32_t ethr_native_atomic32_inc_return(ethr_native_atomic32_t *var) { ethr_sint32_t tmp; __asm__ __volatile__( - "eieio\n\t" "1:\t" "lwarx %0,0,%1\n\t" "addic %0,%0,1\n\t" /* due to addi's (rA|0) behaviour */ "stwcx. %0,0,%1\n\t" "bne- 1b\n\t" - "isync" : "=&r"(tmp) : "r"(&var->counter) : "cc", "memory"); return tmp; } -static ETHR_INLINE void -ethr_native_atomic32_inc(ethr_native_atomic32_t *var) +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_ACQB 1 + +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_inc_return_acqb(ethr_native_atomic32_t *var) { - /* XXX: could use weaker version here w/o eieio+isync */ - (void)ethr_native_atomic32_inc_return(var); + ethr_sint32_t res; + res = ethr_native_atomic32_inc_return(var); + __asm__ __volatile("isync\n\t" : : : "memory"); + return res; } + + +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN 1 static ETHR_INLINE ethr_sint32_t ethr_native_atomic32_dec_return(ethr_native_atomic32_t *var) @@ -115,82 +130,120 @@ ethr_native_atomic32_dec_return(ethr_native_atomic32_t *var) ethr_sint32_t tmp; __asm__ __volatile__( - "eieio\n\t" "1:\t" "lwarx %0,0,%1\n\t" "addic %0,%0,-1\n\t" "stwcx. %0,0,%1\n\t" "bne- 1b\n\t" - "isync" : "=&r"(tmp) : "r"(&var->counter) : "cc", "memory"); return tmp; } -static ETHR_INLINE void -ethr_native_atomic32_dec(ethr_native_atomic32_t *var) +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_ACQB 1 + +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_dec_return_acqb(ethr_native_atomic32_t *var) { - /* XXX: could use weaker version here w/o eieio+isync */ - (void)ethr_native_atomic32_dec_return(var); + ethr_sint32_t res; + res = ethr_native_atomic32_dec_return(var); + __asm__ __volatile("isync\n\t" : : : "memory"); + return res; } +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD 1 + static ETHR_INLINE ethr_sint32_t ethr_native_atomic32_and_retold(ethr_native_atomic32_t *var, ethr_sint32_t mask) { ethr_sint32_t old, new; __asm__ __volatile__( - "eieio\n\t" "1:\t" "lwarx %0,0,%2\n\t" "and %1,%0,%3\n\t" "stwcx. %1,0,%2\n\t" "bne- 1b\n\t" - "isync" : "=&r"(old), "=&r"(new) : "r"(&var->counter), "r"(mask) : "cc", "memory"); return old; } +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_ACQB 1 + +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_and_retold_acqb(ethr_native_atomic32_t *var, ethr_sint32_t mask) +{ + ethr_sint32_t res; + res = ethr_native_atomic32_and_retold(var, mask); + __asm__ __volatile("isync\n\t" : : : "memory"); + return res; +} + +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD 1 + static ETHR_INLINE ethr_sint32_t ethr_native_atomic32_or_retold(ethr_native_atomic32_t *var, ethr_sint32_t mask) { ethr_sint32_t old, new; __asm__ __volatile__( - "eieio\n\t" "1:\t" "lwarx %0,0,%2\n\t" "or %1,%0,%3\n\t" "stwcx. %1,0,%2\n\t" "bne- 1b\n\t" - "isync" : "=&r"(old), "=&r"(new) : "r"(&var->counter), "r"(mask) : "cc", "memory"); return old; } +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_ACQB 1 + +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_or_retold_acqb(ethr_native_atomic32_t *var, ethr_sint32_t mask) +{ + ethr_sint32_t res; + res = ethr_native_atomic32_or_retold(var, mask); + __asm__ __volatile("isync\n\t" : : : "memory"); + return res; +} + + +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG 1 + static ETHR_INLINE ethr_sint32_t ethr_native_atomic32_xchg(ethr_native_atomic32_t *var, ethr_sint32_t val) { ethr_sint32_t tmp; __asm__ __volatile__( - "eieio\n\t" "1:\t" "lwarx %0,0,%1\n\t" "stwcx. %2,0,%1\n\t" "bne- 1b\n\t" - "isync" : "=&r"(tmp) : "r"(&var->counter), "r"(val) : "cc", "memory"); return tmp; } +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_ACQB 1 + +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_xchg_acqb(ethr_native_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; + res = ethr_native_atomic32_xchg(var, val); + __asm__ __volatile("isync\n\t" : : : "memory"); + return res; +} + +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG 1 + static ETHR_INLINE ethr_sint32_t ethr_native_atomic32_cmpxchg(ethr_native_atomic32_t *var, ethr_sint32_t new, @@ -199,14 +252,12 @@ ethr_native_atomic32_cmpxchg(ethr_native_atomic32_t *var, ethr_sint32_t old; __asm__ __volatile__( - "eieio\n\t" "1:\t" "lwarx %0,0,%2\n\t" "cmpw 0,%0,%3\n\t" "bne 2f\n\t" "stwcx. %1,0,%2\n\t" "bne- 1b\n\t" - "isync\n" "2:" : "=&r"(old) : "r"(new), "r"(&var->counter), "r"(expected) @@ -215,25 +266,30 @@ ethr_native_atomic32_cmpxchg(ethr_native_atomic32_t *var, return old; } -/* - * Atomic ops with at least specified barriers. - */ +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_ACQB 1 -static ETHR_INLINE long -ethr_native_atomic32_read_acqb(ethr_native_atomic32_t *var) +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_cmpxchg_acqb(ethr_native_atomic32_t *var, + ethr_sint32_t new, + ethr_sint32_t expected) { - long res = ethr_native_atomic32_read(var); - ETHR_MEMORY_BARRIER; - return res; -} + ethr_sint32_t old; -#define ethr_native_atomic32_set_relb ethr_native_atomic32_xchg -#define ethr_native_atomic32_inc_return_acqb ethr_native_atomic32_inc_return -#define ethr_native_atomic32_dec_relb ethr_native_atomic32_dec_return -#define ethr_native_atomic32_dec_return_relb ethr_native_atomic32_dec_return + __asm__ __volatile__( + "1:\t" + "lwarx %0,0,%2\n\t" + "cmpw 0,%0,%3\n\t" + "bne 2f\n\t" + "stwcx. %1,0,%2\n\t" + "bne- 1b\n\t" + "isync\n" + "2:" + : "=&r"(old) + : "r"(new), "r"(&var->counter), "r"(expected) + : "cc", "memory"); -#define ethr_native_atomic32_cmpxchg_acqb ethr_native_atomic32_cmpxchg -#define ethr_native_atomic32_cmpxchg_relb ethr_native_atomic32_cmpxchg + return old; +} #endif /* ETHR_TRY_INLINE_FUNCS */ diff --git a/erts/include/internal/ppc32/ethr_membar.h b/erts/include/internal/ppc32/ethr_membar.h new file mode 100644 index 0000000000..ff5cc86bfb --- /dev/null +++ b/erts/include/internal/ppc32/ethr_membar.h @@ -0,0 +1,63 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2011. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Memory barriers for PowerPC + * Author: Rickard Green + */ + +#ifndef ETHR_PPC_MEMBAR_H__ +#define ETHR_PPC_MEMBAR_H__ + +#define ETHR_LoadLoad (1 << 0) +#define ETHR_LoadStore (1 << 1) +#define ETHR_StoreLoad (1 << 2) +#define ETHR_StoreStore (1 << 3) + +static __inline__ void +ethr_lwsync__(void) +{ +#ifdef ETHR_PPC_HAVE_NO_LWSYNC + __asm__ __volatile__ ("sync\n\t" : : : "memory"); +#else +#ifndef ETHR_PPC_HAVE_LWSYNC + if (ETHR_PPC_RUNTIME_CONF_HAVE_NO_LWSYNC__) + __asm__ __volatile__ ("sync\n\t" : : : "memory"); + else +#endif + __asm__ __volatile__ ("lwsync\n\t" : : : "memory"); +#endif +} + +static __inline__ void +ethr_sync__(void) +{ + __asm__ __volatile__ ("sync\n\t" : : : "memory"); +} + +/* + * According to the "memory barrier intstructions" section of + * http://www.ibm.com/developerworks/systems/articles/powerpc.html + * we want to use sync when a StoreLoad is needed and lwsync for + * everything else. + */ +#define ETHR_MEMBAR(B) \ + ETHR_CHOOSE_EXPR((B) & ETHR_StoreLoad, ethr_sync__(), ethr_lwsync__()) + +#endif diff --git a/erts/include/internal/ppc32/ethread.h b/erts/include/internal/ppc32/ethread.h index 3b619e9d01..e41c83c5da 100644 --- a/erts/include/internal/ppc32/ethread.h +++ b/erts/include/internal/ppc32/ethread.h @@ -24,12 +24,9 @@ #ifndef ETHREAD_PPC32_ETHREAD_H #define ETHREAD_PPC32_ETHREAD_H +#include "ethr_membar.h" #include "atomic.h" #include "spinlock.h" #include "rwlock.h" -#define ETHR_HAVE_NATIVE_ATOMICS 1 -#define ETHR_HAVE_NATIVE_SPINLOCKS 1 -#define ETHR_HAVE_NATIVE_RWSPINLOCKS 1 - #endif /* ETHREAD_PPC32_ETHREAD_H */ diff --git a/erts/include/internal/ppc32/rwlock.h b/erts/include/internal/ppc32/rwlock.h index 19ec26ab68..311f000b69 100644 --- a/erts/include/internal/ppc32/rwlock.h +++ b/erts/include/internal/ppc32/rwlock.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * Copyright Ericsson AB 2005-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -23,12 +23,14 @@ * * Based on the examples in Appendix E of Motorola's * "Programming Environments Manual For 32-Bit Implementations - * of the PowerPC Architecture". Uses eieio instead of sync - * in the unlock sequence, as suggested in the manual. + * of the PowerPC Architecture". */ #ifndef ETHREAD_PPC_RWLOCK_H #define ETHREAD_PPC_RWLOCK_H +#define ETHR_HAVE_NATIVE_RWSPINLOCKS 1 +#define ETHR_NATIVE_RWSPINLOCK_IMPL "ethread" + /* Unlocked if zero, read-locked if negative, write-locked if +1. */ typedef struct { volatile int lock; @@ -47,9 +49,10 @@ ethr_native_read_unlock(ethr_native_rwlock_t *lock) { int tmp; - /* this is eieio + ethr_native_atomic_inc() - isync */ + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + + /* this is ethr_native_atomic_inc() - isync */ __asm__ __volatile__( - "eieio\n\t" "1:\t" "lwarx %0,0,%1\n\t" "addic %0,%0,1\n\t" @@ -105,7 +108,7 @@ ethr_native_read_lock(ethr_native_rwlock_t *lock) static ETHR_INLINE void ethr_native_write_unlock(ethr_native_rwlock_t *lock) { - __asm__ __volatile__("eieio" : : : "memory"); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); lock->lock = 0; } diff --git a/erts/include/internal/ppc32/spinlock.h b/erts/include/internal/ppc32/spinlock.h index c8460a3e8a..4c95ec9efb 100644 --- a/erts/include/internal/ppc32/spinlock.h +++ b/erts/include/internal/ppc32/spinlock.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * Copyright Ericsson AB 2005-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -23,12 +23,14 @@ * * Based on the examples in Appendix E of Motorola's * "Programming Environments Manual For 32-Bit Implementations - * of the PowerPC Architecture". Uses eieio instead of sync - * in the unlock sequence, as suggested in the manual. + * of the PowerPC Architecture". */ #ifndef ETHREAD_PPC_SPINLOCK_H #define ETHREAD_PPC_SPINLOCK_H +#define ETHR_HAVE_NATIVE_SPINLOCKS 1 +#define ETHR_NATIVE_SPINLOCK_IMPL "ethread" + /* Unlocked if zero, locked if non-zero. */ typedef struct { volatile unsigned int lock; @@ -45,7 +47,7 @@ ethr_native_spinlock_init(ethr_native_spinlock_t *lock) static ETHR_INLINE void ethr_native_spin_unlock(ethr_native_spinlock_t *lock) { - __asm__ __volatile__("eieio" : : : "memory"); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); lock->lock = 0; } diff --git a/erts/include/internal/pthread/ethr_event.h b/erts/include/internal/pthread/ethr_event.h index 4c29b28536..d0a77990cc 100644 --- a/erts/include/internal/pthread/ethr_event.h +++ b/erts/include/internal/pthread/ethr_event.h @@ -21,7 +21,7 @@ * Author: Rickard Green */ -#if defined(ETHR_HAVE_LINUX_FUTEX) && defined(ETHR_HAVE_NATIVE_ATOMICS) +#if defined(ETHR_HAVE_LINUX_FUTEX) && defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) /* --- Linux futex implementation of ethread events ------------------------- */ #define ETHR_LINUX_FUTEX_IMPL__ @@ -62,8 +62,7 @@ static void ETHR_INLINE ETHR_INLINE_FUNC_NAME_(ethr_event_set)(ethr_event *e) { ethr_sint32_t val; - ETHR_MEMORY_BARRIER; - val = ethr_atomic32_xchg(&e->futex, ETHR_EVENT_ON__); + val = ethr_atomic32_xchg_mb(&e->futex, ETHR_EVENT_ON__); if (val == ETHR_EVENT_OFF_WAITER__) { int res = ETHR_FUTEX__(&e->futex, ETHR_FUTEX_WAKE__, 1); if (res != 0) @@ -99,8 +98,7 @@ static void ETHR_INLINE ETHR_INLINE_FUNC_NAME_(ethr_event_set)(ethr_event *e) { ethr_sint32_t val; - ETHR_MEMORY_BARRIER; - val = ethr_atomic32_xchg(&e->state, ETHR_EVENT_ON__); + val = ethr_atomic32_xchg_mb(&e->state, ETHR_EVENT_ON__); if (val == ETHR_EVENT_OFF_WAITER__) { int res = pthread_mutex_lock(&e->mtx); if (res != 0) diff --git a/erts/include/internal/sparc32/atomic.h b/erts/include/internal/sparc32/atomic.h index c297522ab1..fe1daaa9cf 100644 --- a/erts/include/internal/sparc32/atomic.h +++ b/erts/include/internal/sparc32/atomic.h @@ -35,23 +35,16 @@ #ifdef ETHR_INCLUDE_ATOMIC_IMPL__ -#ifndef ETHR_SPARC_V9_ATOMIC_COMMON__ -#define ETHR_SPARC_V9_ATOMIC_COMMON__ - -#define ETHR_MEMORY_BARRIER \ - __asm__ __volatile__("membar #LoadLoad|#LoadStore|#StoreLoad|#StoreStore\n" \ - : : : "memory") - -#endif /* ETHR_SPARC_V9_ATOMIC_COMMON__ */ - #if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 #define ETHR_HAVE_NATIVE_ATOMIC32 1 +#define ETHR_NATIVE_ATOMIC32_IMPL "ethread" #define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X #define ETHR_ATMC_T__ ethr_native_atomic32_t #define ETHR_AINT_T__ ethr_sint32_t #define ETHR_CAS__ "cas" #elif ETHR_INCLUDE_ATOMIC_IMPL__ == 8 #define ETHR_HAVE_NATIVE_ATOMIC64 1 +#define ETHR_NATIVE_ATOMIC64_IMPL "ethread" #define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X #define ETHR_ATMC_T__ ethr_native_atomic64_t #define ETHR_AINT_T__ ethr_sint64_t @@ -66,17 +59,23 @@ typedef struct { #if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADDR 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADDR 1 +#endif + static ETHR_INLINE ETHR_AINT_T__ * ETHR_NATMC_FUNC__(addr)(ETHR_ATMC_T__ *var) { return (ETHR_AINT_T__ *) &var->counter; } -static ETHR_INLINE void -ETHR_NATMC_FUNC__(init)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) -{ - var->counter = i; -} +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET 1 +#endif static ETHR_INLINE void ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) @@ -84,182 +83,49 @@ ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) var->counter = i; } +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ 1 +#endif + static ETHR_INLINE ETHR_AINT_T__ ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var) { return var->counter; } -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(add_return)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) -{ - ETHR_AINT_T__ old, tmp; - - __asm__ __volatile__("membar #LoadLoad|#StoreLoad\n" : : : "memory"); - do { - old = var->counter; - tmp = old+incr; - __asm__ __volatile__( - ETHR_CAS__ " [%2], %1, %0" - : "=&r"(tmp) - : "r"(old), "r"(&var->counter), "0"(tmp) - : "memory"); - } while (__builtin_expect(old != tmp, 0)); - __asm__ __volatile__("membar #StoreLoad|#StoreStore" : : : "memory"); - return old+incr; -} - -static ETHR_INLINE void -ETHR_NATMC_FUNC__(add)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) -{ - (void)ETHR_NATMC_FUNC__(add_return)(var, incr); -} - -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(inc_return)(ETHR_ATMC_T__ *var) -{ - return ETHR_NATMC_FUNC__(add_return)(var, 1); -} - -static ETHR_INLINE void -ETHR_NATMC_FUNC__(inc)(ETHR_ATMC_T__ *var) -{ - (void)ETHR_NATMC_FUNC__(add_return)(var, 1); -} - -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(dec_return)(ETHR_ATMC_T__ *var) -{ - return ETHR_NATMC_FUNC__(add_return)(var, -1); -} - -static ETHR_INLINE void -ETHR_NATMC_FUNC__(dec)(ETHR_ATMC_T__ *var) -{ - (void)ETHR_NATMC_FUNC__(add_return)(var, -1); -} - -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(and_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) -{ - ETHR_AINT_T__ old, tmp; - - __asm__ __volatile__("membar #LoadLoad|#StoreLoad\n" : : : "memory"); - do { - old = var->counter; - tmp = old & mask; - __asm__ __volatile__( - ETHR_CAS__ " [%2], %1, %0" - : "=&r"(tmp) - : "r"(old), "r"(&var->counter), "0"(tmp) - : "memory"); - } while (__builtin_expect(old != tmp, 0)); - __asm__ __volatile__("membar #StoreLoad|#StoreStore" : : : "memory"); - return old; -} - -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(or_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) -{ - ETHR_AINT_T__ old, tmp; - - __asm__ __volatile__("membar #LoadLoad|#StoreLoad\n" : : : "memory"); - do { - old = var->counter; - tmp = old | mask; - __asm__ __volatile__( - ETHR_CAS__ " [%2], %1, %0" - : "=&r"(tmp) - : "r"(old), "r"(&var->counter), "0"(tmp) - : "memory"); - } while (__builtin_expect(old != tmp, 0)); - __asm__ __volatile__("membar #StoreLoad|#StoreStore" : : : "memory"); - return old; -} - -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(xchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ val) -{ - ETHR_AINT_T__ old, new; - - __asm__ __volatile__("membar #LoadLoad|#StoreLoad" : : : "memory"); - do { - old = var->counter; - new = val; - __asm__ __volatile__( - ETHR_CAS__ " [%2], %1, %0" - : "=&r"(new) - : "r"(old), "r"(&var->counter), "0"(new) - : "memory"); - } while (__builtin_expect(old != new, 0)); - __asm__ __volatile__("membar #StoreLoad|#StoreStore" : : : "memory"); - return old; -} +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG 1 +#endif static ETHR_INLINE ETHR_AINT_T__ ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new, ETHR_AINT_T__ old) { - __asm__ __volatile__("membar #LoadLoad|#StoreLoad\n" : : : "memory"); __asm__ __volatile__( ETHR_CAS__ " [%2], %1, %0" : "=&r"(new) : "r"(old), "r"(&var->counter), "0"(new) : "memory"); - __asm__ __volatile__("membar #StoreLoad|#StoreStore" : : : "memory"); return new; } -/* - * Atomic ops with at least specified barriers. - */ - -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(read_acqb)(ETHR_ATMC_T__ *var) -{ - ETHR_AINT_T__ res = ETHR_NATMC_FUNC__(read)(var); - __asm__ __volatile__("membar #LoadLoad|#LoadStore" : : : "memory"); - return res; -} - -static ETHR_INLINE void -ETHR_NATMC_FUNC__(set_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) -{ - __asm__ __volatile__("membar #LoadStore|#StoreStore" : : : "memory"); - ETHR_NATMC_FUNC__(set)(var, i); -} - -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(inc_return_acqb)(ETHR_ATMC_T__ *var) -{ - ETHR_AINT_T__ res = ETHR_NATMC_FUNC__(inc_return)(var); - return res; -} - -static ETHR_INLINE void -ETHR_NATMC_FUNC__(dec_relb)(ETHR_ATMC_T__ *var) -{ - ETHR_NATMC_FUNC__(dec)(var); -} - -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(dec_return_relb)(ETHR_ATMC_T__ *var) -{ - return ETHR_NATMC_FUNC__(dec_return)(var); -} +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_ACQB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_ACQB 1 +#endif static ETHR_INLINE ETHR_AINT_T__ ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new, ETHR_AINT_T__ old) { ETHR_AINT_T__ res = ETHR_NATMC_FUNC__(cmpxchg)(var, new, old); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); return res; } -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new, ETHR_AINT_T__ old) -{ - return ETHR_NATMC_FUNC__(cmpxchg)(var, new, old); -} - #endif /* ETHR_TRY_INLINE_FUNCS */ #undef ETHR_NATMC_FUNC__ diff --git a/erts/include/internal/sparc32/ethr_membar.h b/erts/include/internal/sparc32/ethr_membar.h new file mode 100644 index 0000000000..6eb0c5a1d6 --- /dev/null +++ b/erts/include/internal/sparc32/ethr_membar.h @@ -0,0 +1,115 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2011. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Memory barriers for sparc-v9 + * Author: Rickard Green + */ + +#ifndef ETHR_SPARC_V9_MEMBAR_H__ +#define ETHR_SPARC_V9_MEMBAR_H__ + +#if defined(ETHR_SPARC_TSO) +/* --- Total Store Order ------------------------------------------------ */ + +#define ETHR_LoadLoad 0 +#define ETHR_LoadStore 0 +#define ETHR_StoreLoad 1 +#define ETHR_StoreStore 0 + +static __inline__ void +ethr_cb__(void) +{ + __asm__ __volatile__ ("" : : : "memory"); +} + +static __inline__ void +ethr_StoreLoad__(void) +{ + __asm__ __volatile__ ("membar #StoreLoad\n\t" : : : "memory"); +} + + +#define ETHR_MEMBAR(B) \ + ETHR_CHOOSE_EXPR((B), ethr_StoreLoad__(), ethr_cb__()) + +#elif defined(ETHR_SPARC_PSO) +/* --- Partial Store Order ---------------------------------------------- */ + +#define ETHR_LoadLoad 0 +#define ETHR_LoadStore 0 +#define ETHR_StoreLoad (1 << 0) +#define ETHR_StoreStore (1 << 1) + +static __inline__ void +ethr_cb__(void) +{ + __asm__ __volatile__ ("" : : : "memory"); +} + +static __inline__ void +ethr_StoreLoad__(void) +{ + __asm__ __volatile__ ("membar #StoreLoad\n\t" : : : "memory"); +} + +static __inline__ void +ethr_StoreStore__(void) +{ + __asm__ __volatile__ ("membar #StoreStore\n\t" : : : "memory"); +} + +static __inline__ void +ethr_StoreLoad_StoreStore__(void) +{ + __asm__ __volatile__ ("membar #StoreLoad|StoreStore\n\t" : : : "memory"); +} + +#define ETHR_MEMBAR(B) \ + ETHR_CHOOSE_EXPR( \ + (B) == ETHR_StoreLoad, \ + ethr_StoreLoad__(), \ + ETHR_CHOOSE_EXPR( \ + (B) == ETHR_StoreStore, \ + ethr_StoreStore__(), \ + ETHR_CHOOSE_EXPR( \ + (B) == (ETHR_StoreLoad|ETHR_StoreStore), \ + ethr_StoreLoad_StoreStore__(), \ + ethr_cb__()))) + +#elif defined(ETHR_SPARC_RMO) +/* --- Relaxed Memory Order --------------------------------------------- */ + +# define ETHR_LoadLoad #LoadLoad +# define ETHR_LoadStore #LoadStore +# define ETHR_StoreLoad #StoreLoad +# define ETHR_StoreStore #StoreStore + +# define ETHR_MEMBAR_AUX__(B) \ + __asm__ __volatile__("membar " #B "\n\t" : : : "memory") + +# define ETHR_MEMBAR(B) ETHR_MEMBAR_AUX__(B) + +#else + +# error "No memory order defined" + +#endif + +#endif diff --git a/erts/include/internal/sparc32/ethread.h b/erts/include/internal/sparc32/ethread.h index aea9794390..5ad92d3da7 100644 --- a/erts/include/internal/sparc32/ethread.h +++ b/erts/include/internal/sparc32/ethread.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * Copyright Ericsson AB 2005-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -24,6 +24,7 @@ #ifndef ETHREAD_SPARC32_ETHREAD_H #define ETHREAD_SPARC32_ETHREAD_H +#include "ethr_membar.h" #define ETHR_ATOMIC_WANT_32BIT_IMPL__ #include "atomic.h" #if ETHR_SIZEOF_PTR == 8 @@ -33,8 +34,4 @@ #include "spinlock.h" #include "rwlock.h" -#define ETHR_HAVE_NATIVE_ATOMICS 1 -#define ETHR_HAVE_NATIVE_SPINLOCKS 1 -#define ETHR_HAVE_NATIVE_RWSPINLOCKS 1 - #endif /* ETHREAD_SPARC32_ETHREAD_H */ diff --git a/erts/include/internal/sparc32/rwlock.h b/erts/include/internal/sparc32/rwlock.h index 465ec96866..8b6f2e9c57 100644 --- a/erts/include/internal/sparc32/rwlock.h +++ b/erts/include/internal/sparc32/rwlock.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * Copyright Ericsson AB 2005-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -24,6 +24,9 @@ #ifndef ETHREAD_SPARC32_RWLOCK_H #define ETHREAD_SPARC32_RWLOCK_H +#define ETHR_HAVE_NATIVE_RWSPINLOCKS 1 +#define ETHR_NATIVE_RWSPINLOCK_IMPL "ethread" + /* Unlocked if zero, read-locked if positive, write-locked if -1. */ typedef struct { volatile int lock; @@ -42,7 +45,7 @@ ethr_native_read_unlock(ethr_native_rwlock_t *lock) { unsigned int old, new; - __asm__ __volatile__("membar #LoadLoad|#StoreLoad"); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); do { old = lock->lock; new = old-1; @@ -70,7 +73,7 @@ ethr_native_read_trylock(ethr_native_rwlock_t *lock) : "r"(old), "r"(&lock->lock), "0"(new) : "memory"); } while (__builtin_expect(old != new, 0)); - __asm__ __volatile__("membar #StoreLoad|#StoreStore"); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); return 1; } @@ -87,7 +90,7 @@ ethr_native_read_lock(ethr_native_rwlock_t *lock) if (__builtin_expect(ethr_native_read_trylock(lock) != 0, 1)) break; do { - __asm__ __volatile__("membar #LoadLoad"); + ETHR_MEMBAR(ETHR_LoadLoad); } while (ethr_native_read_is_locked(lock)); } } @@ -95,7 +98,7 @@ ethr_native_read_lock(ethr_native_rwlock_t *lock) static ETHR_INLINE void ethr_native_write_unlock(ethr_native_rwlock_t *lock) { - __asm__ __volatile__("membar #LoadStore|#StoreStore"); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); lock->lock = 0; } @@ -115,7 +118,7 @@ ethr_native_write_trylock(ethr_native_rwlock_t *lock) : "r"(old), "r"(&lock->lock), "0"(new) : "memory"); } while (__builtin_expect(old != new, 0)); - __asm__ __volatile__("membar #StoreLoad|#StoreStore"); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); return 1; } @@ -132,7 +135,7 @@ ethr_native_write_lock(ethr_native_rwlock_t *lock) if (__builtin_expect(ethr_native_write_trylock(lock) != 0, 1)) break; do { - __asm__ __volatile__("membar #LoadLoad"); + ETHR_MEMBAR(ETHR_LoadLoad); } while (ethr_native_write_is_locked(lock)); } } diff --git a/erts/include/internal/sparc32/spinlock.h b/erts/include/internal/sparc32/spinlock.h index 493d514210..d4e36e09cf 100644 --- a/erts/include/internal/sparc32/spinlock.h +++ b/erts/include/internal/sparc32/spinlock.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * Copyright Ericsson AB 2005-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -24,6 +24,9 @@ #ifndef ETHR_SPARC32_SPINLOCK_H #define ETHR_SPARC32_SPINLOCK_H +#define ETHR_HAVE_NATIVE_SPINLOCKS 1 +#define ETHR_NATIVE_SPINLOCK_IMPL "ethread" + /* Locked with ldstub, so unlocked when 0 and locked when non-zero. */ typedef struct { volatile unsigned char lock; @@ -40,7 +43,7 @@ ethr_native_spinlock_init(ethr_native_spinlock_t *lock) static ETHR_INLINE void ethr_native_spin_unlock(ethr_native_spinlock_t *lock) { - __asm__ __volatile__("membar #LoadStore|#StoreStore"); + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); lock->lock = 0; } @@ -51,10 +54,10 @@ ethr_native_spin_trylock(ethr_native_spinlock_t *lock) __asm__ __volatile__( "ldstub [%1], %0\n\t" - "membar #StoreLoad|#StoreStore" : "=r"(prev) : "r"(&lock->lock) : "memory"); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); return prev == 0; } @@ -71,7 +74,7 @@ ethr_native_spin_lock(ethr_native_spinlock_t *lock) if (__builtin_expect(ethr_native_spin_trylock(lock) != 0, 1)) break; do { - __asm__ __volatile__("membar #LoadLoad"); + ETHR_MEMBAR(ETHR_LoadLoad); } while (ethr_native_spin_is_locked(lock)); } } diff --git a/erts/include/internal/tile/atomic.h b/erts/include/internal/tile/atomic.h index 5697afda25..1f1553c346 100644 --- a/erts/include/internal/tile/atomic.h +++ b/erts/include/internal/tile/atomic.h @@ -25,6 +25,7 @@ #define ETHREAD_TILE_ATOMIC_H #define ETHR_HAVE_NATIVE_ATOMIC32 1 +#define ETHR_NATIVE_ATOMIC32_IMPL "tilera" #include <atomic.h> @@ -34,27 +35,25 @@ typedef struct { volatile ethr_sint32_t counter; } ethr_native_atomic32_t; -#define ETHR_MEMORY_BARRIER __insn_mf() - #if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADDR 1 + static ETHR_INLINE ethr_sint32_t * ethr_native_atomic32_addr(ethr_native_atomic32_t *var) { return (ethr_sint32_t *) &var->counter; } +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT 1 + static ETHR_INLINE void ethr_native_atomic32_init(ethr_native_atomic32_t *var, ethr_sint32_t i) { var->counter = i; } -static ETHR_INLINE void -ethr_native_atomic32_set(ethr_native_atomic32_t *var, ethr_sint32_t i) -{ - atomic_exchange_acq(&var->counter, i); -} +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ 1 static ETHR_INLINE ethr_sint32_t ethr_native_atomic32_read(ethr_native_atomic32_t *var) @@ -62,139 +61,80 @@ ethr_native_atomic32_read(ethr_native_atomic32_t *var) return var->counter; } +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_ACQB 1 + +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_read_acqb(ethr_native_atomic32_t *var) +{ + return atomic_compare_and_exchange_val_acq(&var->counter, + 0x81818181, + 0x81818181); +} + +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD 1 + static ETHR_INLINE void ethr_native_atomic32_add(ethr_native_atomic32_t *var, ethr_sint32_t incr) { - ETHR_MEMORY_BARRIER; atomic_add(&var->counter, incr); - ETHR_MEMORY_BARRIER; -} - +} + +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC 1 + static ETHR_INLINE void ethr_native_atomic32_inc(ethr_native_atomic32_t *var) { - ETHR_MEMORY_BARRIER; atomic_increment(&var->counter); - ETHR_MEMORY_BARRIER; } +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC 1 + static ETHR_INLINE void ethr_native_atomic32_dec(ethr_native_atomic32_t *var) { - ETHR_MEMORY_BARRIER; atomic_decrement(&var->counter); - ETHR_MEMORY_BARRIER; } +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN 1 + static ETHR_INLINE ethr_sint32_t ethr_native_atomic32_add_return(ethr_native_atomic32_t *var, ethr_sint32_t incr) { - ethr_sint32_t res; - ETHR_MEMORY_BARRIER; - res = atomic_exchange_and_add(&var->counter, incr) + incr; - ETHR_MEMORY_BARRIER; - return res; + return atomic_exchange_and_add(&var->counter, incr) + incr; } -static ETHR_INLINE ethr_sint32_t -ethr_native_atomic32_inc_return(ethr_native_atomic32_t *var) -{ - return ethr_native_atomic32_add_return(var, 1); -} - -static ETHR_INLINE ethr_sint32_t -ethr_native_atomic32_dec_return(ethr_native_atomic32_t *var) -{ - return ethr_native_atomic32_add_return(var, -1); -} +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD 1 static ETHR_INLINE ethr_sint32_t ethr_native_atomic32_and_retold(ethr_native_atomic32_t *var, ethr_sint32_t mask) { - ethr_sint32_t res; - ETHR_MEMORY_BARRIER; - res = atomic_and_val(&var->counter, mask); - ETHR_MEMORY_BARRIER; - return res; + return atomic_and_val(&var->counter, mask); } +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD 1 + static ETHR_INLINE ethr_sint32_t ethr_native_atomic32_or_retold(ethr_native_atomic32_t *var, ethr_sint32_t mask) { - ethr_sint32_t res; - ETHR_MEMORY_BARRIER; - res = atomic_or_val(&var->counter, mask); - ETHR_MEMORY_BARRIER; - return res; + return atomic_or_val(&var->counter, mask); } +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_ACQB 1 + static ETHR_INLINE ethr_sint32_t -ethr_native_atomic32_xchg(ethr_native_atomic32_t *var, ethr_sint32_t val) +ethr_native_atomic32_xchg_acqb(ethr_native_atomic32_t *var, ethr_sint32_t val) { - ETHR_MEMORY_BARRIER; return atomic_exchange_acq(&var->counter, val); } -static ETHR_INLINE ethr_sint32_t -ethr_native_atomic32_cmpxchg(ethr_native_atomic32_t *var, - ethr_sint32_t new, - ethr_sint32_t expected) -{ - ETHR_MEMORY_BARRIER; - return atomic_compare_and_exchange_val_acq(&var->counter, new, expected); -} - -/* - * Atomic ops with at least specified barriers. - */ - -static ETHR_INLINE ethr_sint32_t -ethr_native_atomic32_read_acqb(ethr_native_atomic32_t *var) -{ - ethr_sint32_t res = ethr_native_atomic32_read(var); - ETHR_MEMORY_BARRIER; - return res; -} - -static ETHR_INLINE ethr_sint32_t -ethr_native_atomic32_inc_return_acqb(ethr_native_atomic32_t *var) -{ - return ethr_native_atomic32_inc_return(var); -} - -static ETHR_INLINE void -ethr_native_atomic32_set_relb(ethr_native_atomic32_t *var, ethr_sint32_t val) -{ - ETHR_MEMORY_BARRIER; - ethr_native_atomic32_set(var, val); -} - -static ETHR_INLINE void -ethr_native_atomic32_dec_relb(ethr_native_atomic32_t *var) -{ - ethr_native_atomic32_dec(var); -} - -static ETHR_INLINE ethr_sint32_t -ethr_native_atomic32_dec_return_relb(ethr_native_atomic32_t *var) -{ - return ethr_native_atomic32_dec_return(var); -} +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_ACQB 1 static ETHR_INLINE ethr_sint32_t ethr_native_atomic32_cmpxchg_acqb(ethr_native_atomic32_t *var, ethr_sint32_t new, - ethr_sint32_t exp) + ethr_sint32_t expected) { - return ethr_native_atomic32_cmpxchg(var, new, exp); -} - -static ETHR_INLINE ethr_sint32_t -ethr_native_atomic32_cmpxchg_relb(ethr_native_atomic32_t *var, - ethr_sint32_t new, - ethr_sint32_t exp) -{ - return ethr_native_atomic32_cmpxchg(var, new, exp); + return atomic_compare_and_exchange_val_acq(&var->counter, new, expected); } #endif /* ETHR_TRY_INLINE_FUNCS */ diff --git a/erts/include/internal/tile/ethr_membar.h b/erts/include/internal/tile/ethr_membar.h new file mode 100644 index 0000000000..7cb4f3cf9a --- /dev/null +++ b/erts/include/internal/tile/ethr_membar.h @@ -0,0 +1,35 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2011. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Memory barriers for TILE64/TILEPro + * Author: Rickard Green + */ + +#ifndef ETHR_TILE_MEMBAR_H__ +#define ETHR_TILE_MEMBAR_H__ + +#define ETHR_LoadLoad (1 << 0) +#define ETHR_LoadStore (1 << 1) +#define ETHR_StoreLoad (1 << 2) +#define ETHR_StoreStore (1 << 3) + +#define ETHR_MEMBAR(B) __insn_mf() + +#endif diff --git a/erts/include/internal/tile/ethread.h b/erts/include/internal/tile/ethread.h index 2de4d42bc6..7f579b50e7 100644 --- a/erts/include/internal/tile/ethread.h +++ b/erts/include/internal/tile/ethread.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2008-2009. All Rights Reserved. + * Copyright Ericsson AB 2008-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -23,8 +23,7 @@ #ifndef ETHREAD_TILE_ETHREAD_H #define ETHREAD_TILE_ETHREAD_H +#include "ethr_membar.h" #include "atomic.h" -#define ETHR_HAVE_NATIVE_ATOMICS 1 - #endif /* ETHREAD_TILE_ETHREAD_H */ diff --git a/erts/include/internal/win/ethr_atomic.h b/erts/include/internal/win/ethr_atomic.h index 60def01a7e..e11f1abf47 100644 --- a/erts/include/internal/win/ethr_atomic.h +++ b/erts/include/internal/win/ethr_atomic.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2010. All Rights Reserved. + * Copyright Ericsson AB 2010-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -24,364 +24,408 @@ #undef ETHR_INCLUDE_ATOMIC_IMPL__ #if !defined(ETHR_WIN_ATOMIC32_H__) && defined(ETHR_ATOMIC_WANT_32BIT_IMPL__) -#define ETHR_WIN_ATOMIC32_H__ -#define ETHR_INCLUDE_ATOMIC_IMPL__ 4 -#undef ETHR_ATOMIC_WANT_32BIT_IMPL__ +# define ETHR_WIN_ATOMIC32_H__ +# if (defined(ETHR_MEMBAR) \ + && defined(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE) \ + && defined(ETHR_HAVE__INTERLOCKEDEXCHANGE)) +# define ETHR_INCLUDE_ATOMIC_IMPL__ 4 +# endif +# undef ETHR_ATOMIC_WANT_32BIT_IMPL__ #elif !defined(ETHR_WIN_ATOMIC64_H__) && defined(ETHR_ATOMIC_WANT_64BIT_IMPL__) -#define ETHR_WIN_ATOMIC64_H__ -#ifdef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64 -/* _InterlockedCompareExchange64() required... */ -#define ETHR_INCLUDE_ATOMIC_IMPL__ 8 +# define ETHR_WIN_ATOMIC64_H__ +# if (defined(ETHR_MEMBAR) \ + && (defined(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64) \ + || defined(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_ACQ) \ + || defined(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_REL))) +# define ETHR_INCLUDE_ATOMIC_IMPL__ 8 +# endif +# undef ETHR_ATOMIC_WANT_64BIT_IMPL__ #endif -#undef ETHR_ATOMIC_WANT_64BIT_IMPL__ + +#if !defined(_MSC_VER) || _MSC_VER < 1400 +# undef ETHR_INCLUDE_ATOMIC_IMPL__ #endif #ifdef ETHR_INCLUDE_ATOMIC_IMPL__ -#if defined(_MSC_VER) && _MSC_VER >= 1400 +# ifndef ETHR_WIN_ATOMIC_COMMON__ +# define ETHR_WIN_ATOMIC_COMMON__ + +# if defined(_M_IX86) || defined(_M_AMD64) || defined(_M_IA64) +# define ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ 1 +# else +# define ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ 0 +# endif + +# endif /* ETHR_WIN_ATOMIC_COMMON__ */ + +# if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 + +# define ETHR_HAVE_NATIVE_ATOMIC32 1 +# define ETHR_NATIVE_ATOMIC32_IMPL "windows-interlocked" + +# ifdef ETHR_HAVE__INTERLOCKEDDECREMENT +# define ETHR_WIN_HAVE_DEC +# pragma intrinsic(_InterlockedDecrement) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDDECREMENT_REL +# define ETHR_WIN_HAVE_DEC_REL +# pragma intrinsic(_InterlockedDecrement_rel) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDINCREMENT +# define ETHR_WIN_HAVE_INC +# pragma intrinsic(_InterlockedIncrement) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDINCREMENT_ACQ +# define ETHR_WIN_HAVE_INC_ACQ +# pragma intrinsic(_InterlockedIncrement_acq) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDEXCHANGEADD +# define ETHR_WIN_HAVE_XCHG_ADD +# pragma intrinsic(_InterlockedExchangeAdd) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDEXCHANGEADD_ACQ +# define ETHR_WIN_HAVE_XCHG_ADD_ACQ +# pragma intrinsic(_InterlockedExchangeAdd_acq) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDEXCHANGE +# define ETHR_WIN_HAVE_XCHG +# pragma intrinsic(_InterlockedExchange) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDAND +# define ETHR_WIN_HAVE_AND +# pragma intrinsic(_InterlockedAnd) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDOR +# define ETHR_WIN_HAVE_OR +# pragma intrinsic(_InterlockedOr) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE +# define ETHR_WIN_HAVE_CMPXCHG +# pragma intrinsic(_InterlockedCompareExchange) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE_ACQ +# define ETHR_WIN_HAVE_CMPXCHG_ACQ +# pragma intrinsic(_InterlockedCompareExchange_acq) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE_REL +# define ETHR_WIN_HAVE_CMPXCHG_REL +# pragma intrinsic(_InterlockedCompareExchange_rel) +# endif + +# define ETHR_ILCKD__(X) _Interlocked ## X +# define ETHR_ILCKD_ACQ__(X) _Interlocked ## X ## _acq +# define ETHR_ILCKD_REL__(X) _Interlocked ## X ## _rel + +# define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X +# define ETHR_ATMC_T__ ethr_native_atomic32_t +# define ETHR_AINT_T__ ethr_sint32_t + +# elif ETHR_INCLUDE_ATOMIC_IMPL__ == 8 + +# define ETHR_HAVE_NATIVE_ATOMIC64 1 +# define ETHR_NATIVE_ATOMIC64_IMPL "windows-interlocked" + +# ifdef ETHR_HAVE__INTERLOCKEDDECREMENT64 +# define ETHR_WIN_HAVE_DEC +# pragma intrinsic(_InterlockedDecrement64) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDDECREMENT64_REL +# define ETHR_WIN_HAVE_DEC_REL +# pragma intrinsic(_InterlockedDecrement64_rel) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDINCREMENT64_ACQ +# define ETHR_WIN_HAVE_INC_ACQ +# pragma intrinsic(_InterlockedIncrement64_acq) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDINCREMENT64 +# define ETHR_WIN_HAVE_INC +# pragma intrinsic(_InterlockedIncrement64) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDEXCHANGEADD64 +# define ETHR_WIN_HAVE_XCHG_ADD +# pragma intrinsic(_InterlockedExchangeAdd64) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDEXCHANGEADD64_ACQ +# define ETHR_WIN_HAVE_XCHG_ADD_ACQ +# pragma intrinsic(_InterlockedExchangeAdd64_acq) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDEXCHANGE64 +# define ETHR_WIN_HAVE_XCHG +# pragma intrinsic(_InterlockedExchange64) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDAND64 +# define ETHR_WIN_HAVE_AND +# pragma intrinsic(_InterlockedAnd64) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDOR64 +# define ETHR_WIN_HAVE_OR +# pragma intrinsic(_InterlockedOr64) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64 +# define ETHR_WIN_HAVE_CMPXCHG +# pragma intrinsic(_InterlockedCompareExchange64) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_ACQ +# define ETHR_WIN_HAVE_CMPXCHG_ACQ +# pragma intrinsic(_InterlockedCompareExchange64_acq) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_REL +# define ETHR_WIN_HAVE_CMPXCHG_REL +# pragma intrinsic(_InterlockedCompareExchange64_rel) +# endif + +# define ETHR_ILCKD__(X) _Interlocked ## X ## 64 +# define ETHR_ILCKD_ACQ__(X) _Interlocked ## X ## 64_acq +# define ETHR_ILCKD_REL__(X) _Interlocked ## X ## 64_rel + +# define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X +# define ETHR_ATMC_T__ ethr_native_atomic64_t +# define ETHR_AINT_T__ ethr_sint64_t + +# else +# error "Unsupported integer size" +# endif -#ifndef ETHR_WIN_ATOMIC_COMMON__ -#define ETHR_WIN_ATOMIC_COMMON__ +typedef struct { + volatile ETHR_AINT_T__ value; +} ETHR_ATMC_T__; -#define ETHR_HAVE_NATIVE_ATOMICS 1 +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) -#if defined(_M_IX86) || defined(_M_AMD64) || defined(_M_IA64) -# define ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ 1 +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADDR 1 #else -# define ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ 0 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADDR 1 #endif -#if defined(_M_AMD64) || (defined(_M_IX86) \ - && !defined(ETHR_PRE_PENTIUM4_COMPAT)) -# define ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ 1 -#else -# define ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ 0 -#endif -/* - * No configure test checking for interlocked acquire/release - * versions have been written, yet. It should define - * ETHR_HAVE_INTERLOCKED_ACQUIRE_RELEASE_BARRIERS if, and - * only if, all used interlocked operations with barriers - * exists. - * - * Note, that these are pure optimizations for the itanium - * processor. - */ +static ETHR_INLINE ETHR_AINT_T__ * +ETHR_NATMC_FUNC__(addr)(ETHR_ATMC_T__ *var) +{ + return (ETHR_AINT_T__ *) &var->value; +} -#include <intrin.h> -#undef ETHR_COMPILER_BARRIER -#define ETHR_COMPILER_BARRIER _ReadWriteBarrier() -#pragma intrinsic(_ReadWriteBarrier) -#pragma intrinsic(_InterlockedCompareExchange) - -#if defined(_M_AMD64) || (defined(_M_IX86) \ - && !defined(ETHR_PRE_PENTIUM4_COMPAT)) -#include <emmintrin.h> -#include <mmintrin.h> -#pragma intrinsic(_mm_mfence) -#define ETHR_MEMORY_BARRIER _mm_mfence() -#pragma intrinsic(_mm_sfence) -#define ETHR_WRITE_MEMORY_BARRIER _mm_sfence() -#pragma intrinsic(_mm_lfence) -#define ETHR_READ_MEMORY_BARRIER _mm_lfence() -#define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_COMPILER_BARRIER +#if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET 1 #else - -#define ETHR_MEMORY_BARRIER \ -do { \ - volatile long x___ = 0; \ - _InterlockedCompareExchange(&x___, (long) 1, (long) 0); \ -} while (0) - +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET 1 #endif -#endif /* ETHR_WIN_ATOMIC_COMMON__ */ +static ETHR_INLINE void +ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) +{ + var->value = i; +} #if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_RELB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RELB 1 +#endif -#define ETHR_HAVE_NATIVE_ATOMIC32 1 +static ETHR_INLINE void +ETHR_NATMC_FUNC__(set_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) +{ +#if defined(_M_IX86) + if (ETHR_X86_RUNTIME_CONF_HAVE_NO_SSE2__) + (void) ETHR_ILCKD__(Exchange)(&var->value, i); + else +#endif /* _M_IX86 */ + { + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + var->value = i; + } +} -/* - * All used operations available as 32-bit intrinsics - */ +#endif /* ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ */ -#pragma intrinsic(_InterlockedDecrement) -#pragma intrinsic(_InterlockedIncrement) -#pragma intrinsic(_InterlockedExchangeAdd) -#pragma intrinsic(_InterlockedExchange) -#pragma intrinsic(_InterlockedAnd) -#pragma intrinsic(_InterlockedOr) -#ifdef ETHR_HAVE_INTERLOCKED_ACQUIRE_RELEASE_BARRIERS -#pragma intrinsic(_InterlockedExchangeAdd_acq) -#pragma intrinsic(_InterlockedIncrement_acq) -#pragma intrinsic(_InterlockedDecrement_rel) -#pragma intrinsic(_InterlockedCompareExchange_acq) -#pragma intrinsic(_InterlockedCompareExchange_rel) -#endif +#if defined(ETHR_WIN_HAVE_XCHG) -#define ETHR_ILCKD__(X) _Interlocked ## X -#ifdef ETHR_HAVE_INTERLOCKED_ACQUIRE_RELEASE_BARRIERS -#define ETHR_ILCKD_ACQ__(X) _Interlocked ## X ## _acq -#define ETHR_ILCKD_REL__(X) _Interlocked ## X ## _rel +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_MB 1 #else -#define ETHR_ILCKD_ACQ__(X) _Interlocked ## X -#define ETHR_ILCKD_REL__(X) _Interlocked ## X +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_MB 1 #endif -#define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X -#define ETHR_ATMC_T__ ethr_native_atomic32_t -#define ETHR_AINT_T__ ethr_sint32_t - -#elif ETHR_INCLUDE_ATOMIC_IMPL__ == 8 +static ETHR_INLINE void +ETHR_NATMC_FUNC__(set_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) +{ + (void) ETHR_ILCKD__(Exchange)(&var->value, i); +} -#define ETHR_HAVE_NATIVE_ATOMIC64 1 +#endif -/* - * _InterlockedCompareExchange64() is required. The other may not - * be available, but if so, we can generate them. - */ -#pragma intrinsic(_InterlockedCompareExchange64) +#if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ -#if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ -#define ETHR_OWN_ILCKD_INIT_VAL__(PTR) *(PTR) +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ 1 #else -#define ETHR_OWN_ILCKD_INIT_VAL__(PTR) (__int64) 0 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ 1 #endif -#define ETHR_OWN_ILCKD_BODY_IMPL__(FUNC, PTR, NEW, ACT, EXP, OPS, RET) \ -{ \ - __int64 NEW, ACT, EXP; \ - ACT = ETHR_OWN_ILCKD_INIT_VAL__(PTR); \ - do { \ - EXP = ACT; \ - { OPS; } \ - ACT = _InterlockedCompareExchange64(PTR, NEW, EXP); \ - } while (ACT != EXP); \ - return RET; \ +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var) +{ + return var->value; } -#define ETHR_OWN_ILCKD_1_IMPL__(FUNC, NEW, ACT, EXP, OPS, RET) \ -static __forceinline __int64 \ -FUNC(__int64 volatile *ptr) \ -ETHR_OWN_ILCKD_BODY_IMPL__(FUNC, ptr, NEW, ACT, EXP, OPS, RET) - -#define ETHR_OWN_ILCKD_2_IMPL__(FUNC, NEW, ACT, EXP, OPS, ARG, RET) \ -static __forceinline __int64 \ -FUNC(__int64 volatile *ptr, __int64 ARG) \ -ETHR_OWN_ILCKD_BODY_IMPL__(FUNC, ptr, NEW, ACT, EXP, OPS, RET) +#endif /* ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ */ +#if defined(ETHR_WIN_HAVE_XCHG_ADD) -#ifdef ETHR_HAVE__INTERLOCKEDDECREMENT64 -#pragma intrinsic(_InterlockedDecrement64) -#else -ETHR_OWN_ILCKD_1_IMPL__(_InterlockedDecrement64, new, act, exp, - new = act - 1, new) -#endif -#ifdef ETHR_HAVE__INTERLOCKEDINCREMENT64 -#pragma intrinsic(_InterlockedIncrement64) -#else -ETHR_OWN_ILCKD_1_IMPL__(_InterlockedIncrement64, new, act, exp, - new = act + 1, new) -#endif -#ifdef ETHR_HAVE__INTERLOCKEDEXCHANGEADD64 -#pragma intrinsic(_InterlockedExchangeAdd64) -#else -ETHR_OWN_ILCKD_2_IMPL__(_InterlockedExchangeAdd64, new, act, exp, - new = act + arg, arg, act) -#endif -#ifdef ETHR_HAVE__INTERLOCKEDEXCHANGE64 -#pragma intrinsic(_InterlockedExchange64) -#else -ETHR_OWN_ILCKD_2_IMPL__(_InterlockedExchange64, new, act, exp, - new = arg, arg, act) -#endif -#ifdef ETHR_HAVE__INTERLOCKEDAND64 -#pragma intrinsic(_InterlockedAnd64) -#else -ETHR_OWN_ILCKD_2_IMPL__(_InterlockedAnd64, new, act, exp, - new = act & arg, arg, act) -#endif -#ifdef ETHR_HAVE__INTERLOCKEDOR64 -#pragma intrinsic(_InterlockedOr64) +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_MB 1 #else -ETHR_OWN_ILCKD_2_IMPL__(_InterlockedOr64, new, act, exp, - new = act | arg, arg, act) -#endif -#ifdef ETHR_HAVE_INTERLOCKED_ACQUIRE_RELEASE_BARRIERS -#pragma intrinsic(_InterlockedExchangeAdd64_acq) -#pragma intrinsic(_InterlockedIncrement64_acq) -#pragma intrinsic(_InterlockedDecrement64_rel) -#pragma intrinsic(_InterlockedCompareExchange64_acq) -#pragma intrinsic(_InterlockedCompareExchange64_rel) +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_MB 1 #endif -#define ETHR_ILCKD__(X) _Interlocked ## X ## 64 -#ifdef ETHR_HAVE_INTERLOCKED_ACQUIRE_RELEASE_BARRIERS -#define ETHR_ILCKD_ACQ__(X) _Interlocked ## X ## 64_acq -#define ETHR_ILCKD_REL__(X) _Interlocked ## X ## 64_rel -#else -#define ETHR_ILCKD_ACQ__(X) _Interlocked ## X ## 64 -#define ETHR_ILCKD_REL__(X) _Interlocked ## X ## 64 +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(add_return_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) +{ + return ETHR_ILCKD__(ExchangeAdd)(&var->value, i) + i; +} + #endif -#define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X -#define ETHR_ATMC_T__ ethr_native_atomic64_t -#define ETHR_AINT_T__ ethr_sint64_t +#if defined(ETHR_WIN_HAVE_INC) +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_MB 1 #else -#error "Unsupported integer size" +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_MB 1 #endif -typedef struct { - volatile ETHR_AINT_T__ value; -} ETHR_ATMC_T__; - -#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) - -static ETHR_INLINE ETHR_AINT_T__ * -ETHR_NATMC_FUNC__(addr)(ETHR_ATMC_T__ *var) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(inc_return_mb)(ETHR_ATMC_T__ *var) { - return (ETHR_AINT_T__ *) &var->value; + return ETHR_ILCKD__(Increment)(&var->value); } -static ETHR_INLINE void -ETHR_NATMC_FUNC__(init)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) -{ -#if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ - var->value = i; -#else - (void) ETHR_ILCKD__(Exchange)(&var->value, i); #endif -} -static ETHR_INLINE void -ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) -{ -#if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ - var->value = i; +#if defined(ETHR_WIN_HAVE_INC_ACQ) + +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_ACQB 1 #else - (void) ETHR_ILCKD__(Exchange)(&var->value, i); +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_ACQB 1 #endif -} static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var) +ETHR_NATMC_FUNC__(inc_return_acqb)(ETHR_ATMC_T__ *var) { -#if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ - return var->value; -#else - return ETHR_ILCKD__(ExchangeAdd)(&var->value, (ETHR_AINT_T__) 0); -#endif + return ETHR_ILCKD_ACQ__(Increment)(&var->value); } -static ETHR_INLINE void -ETHR_NATMC_FUNC__(add)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) -{ - (void) ETHR_ILCKD__(ExchangeAdd)(&var->value, incr); -} +#endif + +#if defined(ETHR_WIN_HAVE_DEC) + +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_MB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_MB 1 +#endif static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(add_return)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) +ETHR_NATMC_FUNC__(dec_return_mb)(ETHR_ATMC_T__ *var) { - return ETHR_ILCKD__(ExchangeAdd)(&var->value, i) + i; + return ETHR_ILCKD__(Decrement)(&var->value); } -static ETHR_INLINE void -ETHR_NATMC_FUNC__(inc)(ETHR_ATMC_T__ *var) -{ - (void) ETHR_ILCKD__(Increment)(&var->value); -} +#endif -static ETHR_INLINE void -ETHR_NATMC_FUNC__(dec)(ETHR_ATMC_T__ *var) -{ - (void) ETHR_ILCKD__(Decrement)(&var->value); -} +#if defined(ETHR_WIN_HAVE_DEC_REL) -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(inc_return)(ETHR_ATMC_T__ *var) -{ - return ETHR_ILCKD__(Increment)(&var->value); -} +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_RELB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_RELB 1 +#endif static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(dec_return)(ETHR_ATMC_T__ *var) +ETHR_NATMC_FUNC__(dec_return_relb)(ETHR_ATMC_T__ *var) { - return ETHR_ILCKD__(Decrement)(&var->value); + return ETHR_ILCKD_REL__(Decrement)(&var->value); } +#endif + +#if defined(ETHR_WIN_HAVE_AND) + +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_MB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_MB 1 +#endif + static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(and_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) +ETHR_NATMC_FUNC__(and_retold_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) { return ETHR_ILCKD__(And)(&var->value, mask); } +#endif + +#if defined(ETHR_WIN_HAVE_OR) + +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_MB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_MB 1 +#endif + static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(or_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) +ETHR_NATMC_FUNC__(or_retold_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) { return ETHR_ILCKD__(Or)(&var->value, mask); } -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var, - ETHR_AINT_T__ new, - ETHR_AINT_T__ old) -{ - return ETHR_ILCKD__(CompareExchange)(&var->value, new, old); -} +#endif +#if defined(ETHR_WIN_HAVE_XCHG) + +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_MB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_MB 1 +#endif static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(xchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new) +ETHR_NATMC_FUNC__(xchg_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new) { return ETHR_ILCKD__(Exchange)(&var->value, new); } -/* - * Atomic ops with at least specified barriers. - */ +#endif -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(read_acqb)(ETHR_ATMC_T__ *var) -{ -#if ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ - ETHR_AINT_T__ val = var->value; - ETHR_COMPILER_BARRIER; - return val; +#if defined(ETHR_WIN_HAVE_CMPXCHG) + +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_MB 1 #else - return ETHR_ILCKD_ACQ__(ExchangeAdd)(&var->value, (ETHR_AINT_T__) 0); +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_MB 1 #endif -} static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(inc_return_acqb)(ETHR_ATMC_T__ *var) +ETHR_NATMC_FUNC__(cmpxchg_mb)(ETHR_ATMC_T__ *var, + ETHR_AINT_T__ new, + ETHR_AINT_T__ old) { - return ETHR_ILCKD_ACQ__(Increment)(&var->value); + return ETHR_ILCKD__(CompareExchange)(&var->value, new, old); } -static ETHR_INLINE void -ETHR_NATMC_FUNC__(set_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) -{ -#if ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ - ETHR_COMPILER_BARRIER; - var->value = i; -#else - (void) ETHR_ILCKD_REL__(Exchange)(&var->value, i); #endif -} -static ETHR_INLINE void -ETHR_NATMC_FUNC__(dec_relb)(ETHR_ATMC_T__ *var) -{ - (void) ETHR_ILCKD_REL__(Decrement)(&var->value); -} +#if defined(ETHR_WIN_HAVE_CMPXCHG_ACQ) -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(dec_return_relb)(ETHR_ATMC_T__ *var) -{ - return ETHR_ILCKD_REL__(Decrement)(&var->value); -} +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_ACQB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_ACQB 1 +#endif static ETHR_INLINE ETHR_AINT_T__ ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var, @@ -391,6 +435,16 @@ ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var, return ETHR_ILCKD_ACQ__(CompareExchange)(&var->value, new, old); } +#endif + +#if defined(ETHR_WIN_HAVE_CMPXCHG_REL) + +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_RELB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RELB 1 +#endif + static ETHR_INLINE ETHR_AINT_T__ ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new, @@ -399,6 +453,8 @@ ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var, return ETHR_ILCKD_REL__(CompareExchange)(&var->value, new, old); } +#endif + #endif /* ETHR_TRY_INLINE_FUNCS */ #undef ETHR_ILCKD__ @@ -408,8 +464,17 @@ ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var, #undef ETHR_ATMC_T__ #undef ETHR_AINT_T__ #undef ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ -#undef ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ - -#endif /* _MSC_VER */ +#undef ETHR_WIN_HAVE_CMPXCHG +#undef ETHR_WIN_HAVE_DEC +#undef ETHR_WIN_HAVE_INC +#undef ETHR_WIN_HAVE_XCHG_ADD +#undef ETHR_WIN_HAVE_XCHG +#undef ETHR_WIN_HAVE_AND +#undef ETHR_WIN_HAVE_OR +#undef ETHR_WIN_HAVE_XCHG_ADD_ACQ +#undef ETHR_WIN_HAVE_INC_ACQ +#undef ETHR_WIN_HAVE_DEC_REL +#undef ETHR_WIN_HAVE_CMPXCHG_ACQ +#undef ETHR_WIN_HAVE_CMPXCHG_REL #endif /* ETHR_INCLUDE_ATOMIC_IMPL__ */ diff --git a/erts/include/internal/win/ethr_dw_atomic.h b/erts/include/internal/win/ethr_dw_atomic.h new file mode 100644 index 0000000000..a3e7ffc3aa --- /dev/null +++ b/erts/include/internal/win/ethr_dw_atomic.h @@ -0,0 +1,154 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2011. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Native double word atomics for windows + * Author: Rickard Green + */ + +#undef ETHR_INCLUDE_DW_ATOMIC_IMPL__ +#ifndef ETHR_X86_DW_ATOMIC_H__ +# define ETHR_X86_DW_ATOMIC_H__ +# if ((ETHR_SIZEOF_PTR == 4 \ + && defined(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64)) \ + || (ETHR_SIZEOF_PTR == 8 \ + && defined(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE128))) +# define ETHR_INCLUDE_DW_ATOMIC_IMPL__ +# endif +#endif + +#ifdef ETHR_INCLUDE_DW_ATOMIC_IMPL__ + +# if ETHR_SIZEOF_PTR == 4 +# define ETHR_HAVE_NATIVE_SU_DW_ATOMIC +# else +# define ETHR_HAVE_NATIVE_DW_ATOMIC +# endif +# define ETHR_NATIVE_DW_ATOMIC_IMPL "windows-interlocked" + +# if defined(_M_IX86) || defined(_M_AMD64) +/* + * If ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ is defined, it will be used + * at runtime in order to determine if native or fallback implementation + * should be used. + */ +# define ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ \ + ETHR_X86_RUNTIME_CONF_HAVE_DW_CMPXCHG__ +# endif + +# include <intrin.h> +# if ETHR_SIZEOF_PTR == 4 +# pragma intrinsic(_InterlockedCompareExchange64) +# define ETHR_DW_NATMC_ALIGN_MASK__ 0x7 +# define ETHR_NATIVE_SU_DW_SINT_T ethr_sint64_t +# else +# pragma intrinsic(_InterlockedCompareExchange128) +# define ETHR_DW_NATMC_ALIGN_MASK__ 0xf +# endif + +typedef volatile __int64 * ethr_native_dw_ptr_t; + +/* + * We need 16 byte aligned memory in 64-bit mode, and 8 byte aligned + * memory in 32-bit mode. 16 byte aligned malloc in 64-bit mode is + * not common, and at least some glibc malloc implementations + * only 4 byte align in 32-bit mode. + * + * This code assumes 8 byte aligned memory in 64-bit mode, and 4 byte + * aligned memory in 32-bit mode. A malloc implementation that does + * not adhere to these alignment requirements is seriously broken, + * and we wont bother trying to work around it. + * + * Since memory alignment may be off by one word we need to align at + * runtime. We, therefore, need an extra word allocated. + */ +#define ETHR_DW_NATMC_MEM__(VAR) \ + (&var->c[(int) ((ethr_uint_t) &(VAR)->c[0]) & ETHR_DW_NATMC_ALIGN_MASK__]) +typedef union { +#ifdef ETHR_NATIVE_SU_DW_SINT_T + volatile ETHR_NATIVE_SU_DW_SINT_T dw_sint; +#endif + volatile ethr_sint_t sint[3]; + volatile char c[ETHR_SIZEOF_PTR*3]; +} ethr_native_dw_atomic_t; + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) + +#ifdef ETHR_DEBUG +# define ETHR_DW_DBG_ALIGNED__(PTR) \ + ETHR_ASSERT((((ethr_uint_t) (PTR)) & ETHR_DW_NATMC_ALIGN_MASK__) == 0); +#else +# define ETHR_DW_DBG_ALIGNED__(PTR) +#endif + +#define ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_ADDR + +static ETHR_INLINE ethr_sint_t * +ethr_native_dw_atomic_addr(ethr_native_dw_atomic_t *var) +{ + ethr_sint_t *p = (ethr_sint_t *) ETHR_DW_NATMC_MEM__(var); + ETHR_DW_DBG_ALIGNED__(p); + return p; +} + + +#if ETHR_SIZEOF_PTR == 4 + +#define ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_MB + +static ETHR_INLINE ethr_sint64_t +ethr_native_su_dw_atomic_cmpxchg_mb(ethr_native_dw_atomic_t *var, + ethr_sint64_t new, + ethr_sint64_t exp) +{ + ethr_native_dw_ptr_t p = (ethr_native_dw_ptr_t) ETHR_DW_NATMC_MEM__(var); + ETHR_DW_DBG_ALIGNED__(p); + return (ethr_sint64_t) _InterlockedCompareExchange64(p, new, exp); +} + +#elif ETHR_SIZEOF_PTR == 8 + +#define ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_MB + +#ifdef ETHR_BIGENDIAN +# define ETHR_WIN_LOW_WORD__ 1 +# define ETHR_WIN_HIGH_WORD__ 0 +#else +# define ETHR_WIN_LOW_WORD__ 0 +# define ETHR_WIN_HIGH_WORD__ 1 +#endif + +static ETHR_INLINE int +ethr_native_dw_atomic_cmpxchg_mb(ethr_native_dw_atomic_t *var, + ethr_sint_t *new, + ethr_sint_t *xchg) +{ + ethr_native_dw_ptr_t p = (ethr_native_dw_ptr_t) ETHR_DW_NATMC_MEM__(var); + ETHR_DW_DBG_ALIGNED__(p); + return (int) _InterlockedCompareExchange128(p, + new[ETHR_WIN_HIGH_WORD__], + new[ETHR_WIN_LOW_WORD__], + xchg); +} + +#endif + +#endif /* ETHR_TRY_INLINE_FUNCS */ + +#endif /* ETHR_INCLUDE_DW_ATOMIC_IMPL__ */ diff --git a/erts/include/internal/win/ethr_event.h b/erts/include/internal/win/ethr_event.h index 598816b2c6..6363174a74 100644 --- a/erts/include/internal/win/ethr_event.h +++ b/erts/include/internal/win/ethr_event.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2009-2010. All Rights Reserved. + * Copyright Ericsson AB 2009-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -21,12 +21,12 @@ * Author: Rickard Green */ -#define ETHR_EVENT_OFF_WAITER__ ((long) -1) -#define ETHR_EVENT_OFF__ ((long) 1) -#define ETHR_EVENT_ON__ ((long) 0) +#define ETHR_EVENT_OFF_WAITER__ ((ethr_sint32_t) -1) +#define ETHR_EVENT_OFF__ ((ethr_sint32_t) 1) +#define ETHR_EVENT_ON__ ((ethr_sint32_t) 0) typedef struct { - volatile long state; + ethr_atomic32_t state; HANDLE handle; } ethr_event; @@ -38,7 +38,7 @@ static ETHR_INLINE void ETHR_INLINE_FUNC_NAME_(ethr_event_set)(ethr_event *e) { /* _InterlockedExchange() imply a full memory barrier which is important */ - long state = _InterlockedExchange(&e->state, ETHR_EVENT_ON__); + ethr_sint32_t state = ethr_atomic32_xchg_wb(&e->state, ETHR_EVENT_ON__); if (state == ETHR_EVENT_OFF_WAITER__) { if (!SetEvent(e->handle)) ETHR_FATAL_ERROR__(ethr_win_get_errno__()); @@ -48,8 +48,8 @@ ETHR_INLINE_FUNC_NAME_(ethr_event_set)(ethr_event *e) static ETHR_INLINE void ETHR_INLINE_FUNC_NAME_(ethr_event_reset)(ethr_event *e) { - /* _InterlockedExchange() imply a full memory barrier which is important */ - InterlockedExchange(&e->state, ETHR_EVENT_OFF__); + ethr_atomic32_set(&e->state, ETHR_EVENT_OFF__); + ETHR_MEMORY_BARRIER; } #endif diff --git a/erts/include/internal/win/ethr_membar.h b/erts/include/internal/win/ethr_membar.h new file mode 100644 index 0000000000..8237660b2c --- /dev/null +++ b/erts/include/internal/win/ethr_membar.h @@ -0,0 +1,145 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2011. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Memory barriers for Windows + * Author: Rickard Green + */ + +#if (!defined(ETHR_WIN_MEMBAR_H__) \ + && (defined(_MSC_VER) && _MSC_VER >= 1400) \ + && (defined(_M_AMD64) \ + || defined(_M_IA64) \ + || defined(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE))) +#define ETHR_WIN_MEMBAR_H__ + +#define ETHR_LoadLoad (1 << 0) +#define ETHR_LoadStore (1 << 1) +#define ETHR_StoreLoad (1 << 2) +#define ETHR_StoreStore (1 << 3) + +#include <intrin.h> +#undef ETHR_COMPILER_BARRIER +#define ETHR_COMPILER_BARRIER _ReadWriteBarrier() +#pragma intrinsic(_ReadWriteBarrier) + +#pragma intrinsic(_InterlockedCompareExchange) + +#define ETHR_MB_USING_INTERLOCKED__ \ +do { \ + volatile long x___ = 0; \ + (void) _InterlockedCompareExchange(&x___, 2, 1); \ +} while (0) + +#if defined(_M_IA64) + +#define ETHR_MEMBAR(B) __mf() + +#elif defined(_M_AMD64) || defined(_M_IX86) + +#include <emmintrin.h> +#include <mmintrin.h> + +#if ETHR_SIZEOF_PTR == 4 +# define ETHR_NO_SSE2_MB__ ETHR_MB_USING_INTERLOCKED__ +#endif +#pragma intrinsic(_mm_mfence) +#pragma intrinsic(_mm_sfence) +#pragma intrinsic(_mm_lfence) + +static __forceinline void +ethr_cfence__(void) +{ + _ReadWriteBarrier(); +} + +static __forceinline void +ethr_mfence__(void) +{ +#if ETHR_SIZEOF_PTR == 4 + if (ETHR_X86_RUNTIME_CONF_HAVE_NO_SSE2__) + ETHR_NO_SSE2_MB__; + else +#endif + _mm_mfence(); +} + +static __forceinline void +ethr_sfence__(void) +{ +#if ETHR_SIZEOF_PTR == 4 + if (ETHR_X86_RUNTIME_CONF_HAVE_NO_SSE2__) + ETHR_NO_SSE2_MB__; + else +#endif + _mm_sfence(); +} + +static __forceinline void +ethr_lfence__(void) +{ +#if ETHR_SIZEOF_PTR == 4 + if (ETHR_X86_RUNTIME_CONF_HAVE_NO_SSE2__) + ETHR_NO_SSE2_MB__; + else +#endif + _mm_lfence(); +} + +#define ETHR_X86_OUT_OF_ORDER_MEMBAR(B) \ + ETHR_CHOOSE_EXPR((B) == ETHR_StoreStore, \ + ethr_sfence__(), \ + ETHR_CHOOSE_EXPR((B) == ETHR_LoadLoad, \ + ethr_lfence__(), \ + ethr_mfence__())) + +#ifdef ETHR_X86_OUT_OF_ORDER + +#define ETHR_MEMBAR(B) \ + ETHR_X86_OUT_OF_ORDER_MEMBAR((B)) + +#else /* !ETHR_X86_OUT_OF_ORDER (the default) */ + +/* + * We assume that only stores before loads may be reordered. That is, + * we assume that *no* instructions like these are used: + * - CLFLUSH, + * - streaming stores executed with non-temporal move, + * - string operations, or + * - other instructions which aren't LoadLoad, LoadStore, and StoreStore + * ordered by themselves + * If such instructions are used, either insert memory barriers + * using ETHR_X86_OUT_OF_ORDER_MEMBAR() at appropriate places, or + * define ETHR_X86_OUT_OF_ORDER. For more info see Intel 64 and IA-32 + * Architectures Software Developer's Manual; Vol 3A; Chapter 8.2.2. + */ + +#define ETHR_MEMBAR(B) \ + ETHR_CHOOSE_EXPR((B) & ETHR_StoreLoad, ethr_mfence__(), ethr_cfence__()) + +#endif + +#else /* No knowledge about platform; use interlocked fallback */ + +#define ETHR_MEMBAR(B) ETHR_MB_USING_INTERLOCKED__ +#define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_MB_USING_INTERLOCKED__ + +#endif + +#endif /* ETHR_WIN_MEMBAR_H__ */ diff --git a/erts/include/internal/win/ethread.h b/erts/include/internal/win/ethread.h index c01b17cf14..8be35e810e 100644 --- a/erts/include/internal/win/ethread.h +++ b/erts/include/internal/win/ethread.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2010. All Rights Reserved. + * Copyright Ericsson AB 2010-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -25,11 +25,13 @@ #ifndef ETHREAD_WIN_H__ #define ETHREAD_WIN_H__ +#include "ethr_membar.h" #define ETHR_ATOMIC_WANT_32BIT_IMPL__ #include "ethr_atomic.h" #if ETHR_SIZEOF_PTR == 8 # define ETHR_ATOMIC_WANT_64BIT_IMPL__ # include "ethr_atomic.h" #endif +#include "ethr_dw_atomic.h" #endif diff --git a/erts/lib_src/Makefile.in b/erts/lib_src/Makefile.in index 757b3b24e2..12b8732735 100644 --- a/erts/lib_src/Makefile.in +++ b/erts/lib_src/Makefile.in @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 2004-2010. All Rights Reserved. +# Copyright Ericsson AB 2004-2011. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in @@ -65,7 +65,7 @@ TYPE_SUFFIX=.purecov PRE_LD=purecov $(PURECOV_BUILD_OPTIONS) else ifeq ($(TYPE),gcov) -CFLAGS=@DEBUG_CFLAGS@ -fprofile-arcs -ftest-coverage -O0 +CFLAGS=@DEBUG_CFLAGS@ -DGCOV -fprofile-arcs -ftest-coverage -O0 TYPE_SUFFIX=.gcov PRE_LD= else @@ -288,6 +288,10 @@ ETHREAD_LIB_SRC=common/ethr_aux.c \ common/ethr_cbf.c \ $(ETHR_THR_LIB_BASE_DIR)/ethread.c \ $(ETHR_THR_LIB_BASE_DIR)/ethr_event.c +ETHR_X86_SSE2_ASM=@ETHR_X86_SSE2_ASM@ +ifeq ($(ETHR_X86_SSE2_ASM),yes) +ETHREAD_LIB_SRC += pthread/ethr_x86_sse2_asm.c +endif ETHREAD_LIB_NAME=ethread$(TYPE_SUFFIX) ifeq ($(USING_VC),yes) @@ -382,10 +386,8 @@ $(ERTS_LIB): $(ERTS_LIB_OBJS) # Object files # -ifeq ($(TYPE)-@GCC@,debug-yes) -$(r_OBJ_DIR)/ethr_aux.o: common/ethr_aux.c - $(CC) $(THR_DEFS) $(CFLAGS) -Wno-unused-function $(INCLUDES) -c $< -o $@ -endif +$(r_OBJ_DIR)/ethr_x86_sse2_asm.o: pthread/ethr_x86_sse2_asm.c + $(CC) -msse2 $(THR_DEFS) $(CFLAGS) $(INCLUDES) -c $< -o $@ $(r_OBJ_DIR)/%.o: common/%.c $(CC) $(THR_DEFS) $(CFLAGS) $(INCLUDES) -c $< -o $@ diff --git a/erts/lib_src/common/ethr_atomics.c b/erts/lib_src/common/ethr_atomics.c index 94557d904a..5796bdc22e 100644 --- a/erts/lib_src/common/ethr_atomics.c +++ b/erts/lib_src/common/ethr_atomics.c @@ -1,7 +1,16 @@ /* + * --------------- DO NOT EDIT THIS FILE! --------------- + * This file was automatically generated by the + * $ERL_TOP/erts/lib_src/utils/make_atomics_api script. + * If you need to make changes, edit the script and + * regenerate this file. + * --------------- DO NOT EDIT THIS FILE! --------------- + */ + +/* * %CopyrightBegin% * - * Copyright Ericsson AB 2010. All Rights Reserved. + * Copyright Ericsson AB 2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -18,385 +27,4329 @@ */ /* - * Description: The ethread atomic API + * Description: The ethread atomics API * Author: Rickard Green */ +/* + * This file maps native atomic implementations to ethread + * API atomics. If no native atomic implementation + * is available, a less efficient fallback is used instead. + * The API consists of 32-bit size, word size (pointer size), + * and double word size atomics. + * + * The following atomic operations are implemented for + * 32-bit size, and word size atomics: + * - cmpxchg + * - xchg + * - set + * - init + * - add_read + * - read + * - inc_read + * - dec_read + * - add + * - inc + * - dec + * - read_band + * - read_bor + * + * The following atomic operations are implemented for + * double word size atomics: + * - cmpxchg + * - set + * - read + * - init + * + * Appart from a function implementing the atomic operation + * with unspecified memory barrier semantics, there are + * functions implementing each operation with the following + * memory barrier semantics: + * - rb (read barrier) + * - wb (write barrier) + * - acqb (acquire barrier) + * - relb (release barrier) + * - mb (full memory barrier) + * + * We implement all of these operation/barrier + * combinations, regardless of whether they are useful + * or not (some of them are useless). + * + * Double word size atomic functions are on the followning + * form: + * ethr_dw_atomic_<OP>[_<BARRIER>] + * + * Word size atomic functions are on the followning + * form: + * ethr_atomic_<OP>[_<BARRIER>] + * + * 32-bit size atomic functions are on the followning + * form: + * ethr_atomic32_<OP>[_<BARRIER>] + * + * Apart from the operation/barrier functions + * described above also 'addr' functions are implemented + * which return the actual memory address used of the + * atomic variable. The 'addr' functions have no barrier + * versions. + * + * The native atomic implementation does not need to + * implement all operation/barrier combinations. + * Functions that have no native implementation will be + * constructed from existing native functionality. These + * functions will perform the wanted operation and will + * produce sufficient memory barriers, but may + * in some cases be less efficient than pure native + * versions. + * + * When we create ethread API operation/barrier functions by + * adding barriers before and after native operations it is + * assumed that: + * - A native read operation begins, and ends with a load. + * - A native set operation begins, and ends with a store. + * - An init operation begins with either a load, or a store, + * and ends with either a load, or a store. + * - All other operations begins with a load, and ends with + * either a load, or a store. + * + * This is the minimum functionality that a native + * implementation needs to provide: + * + * - Functions that need to be implemented: + * + * - ethr_native_[dw_|su_dw_]atomic[BITS]_addr + * - ethr_native_[dw_|su_dw_]atomic[BITS]_cmpxchg[_<BARRIER>] + * (at least one cmpxchg of optional barrier) + * + * - Macros that needs to be defined: + * + * A macro informing about the presence of the native + * implementation: + * + * - ETHR_HAVE_NATIVE_[DW_|SU_DW_]ATOMIC[BITS] + * + * A macro naming (a string constant) the implementation: + * + * - ETHR_NATIVE_[DW_]ATOMIC[BITS]_IMPL + * + * Each implemented native atomic function has to + * be accompanied by a defined macro on the following + * form informing about its presence: + * + * - ETHR_HAVE_ETHR_NATIVE_[DW_|SU_DW_]ATOMIC[BITS]_<OP>[_<BARRIER>] + * + * A (sparc-v9 style) membar macro: + * + * - ETHR_MEMBAR(B) + * + * Which takes a combination of the following macros + * or:ed (using |) together: + * + * - ETHR_LoadLoad + * - ETHR_LoadStore + * - ETHR_StoreLoad + * - ETHR_StoreStore + * + */ + + #ifdef HAVE_CONFIG_H #include "config.h" #endif -#define ETHR_INLINE_FUNC_NAME_(X) X ## __ +#define ETHR_TRY_INLINE_FUNCS +#define ETHR_INLINE_DW_ATMC_FUNC_NAME_(X) X ## __ +#define ETHR_INLINE_ATMC_FUNC_NAME_(X) X ## __ +#define ETHR_INLINE_ATMC32_FUNC_NAME_(X) X ## __ #define ETHR_ATOMIC_IMPL__ #include "ethread.h" #include "ethr_internal.h" -#ifndef ETHR_HAVE_NATIVE_ATOMICS -ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATOMIC_ADDR_BITS]; +#if (!defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) \ + || !defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)) +/* + * Spinlock based fallback for atomics used in absence of a native + * implementation. + */ + +#define ETHR_ATMC_FLLBK_ADDR_BITS 10 +#define ETHR_ATMC_FLLBK_ADDR_SHIFT 6 + +typedef struct { + union { + ethr_spinlock_t lck; + char buf[ETHR_CACHE_LINE_ALIGN_SIZE(sizeof(ethr_spinlock_t))]; + } u; +} ethr_atomic_protection_t; + +extern ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATMC_FLLBK_ADDR_BITS]; + +#define ETHR_ATOMIC_PTR2LCK__(PTR) \ +(ðr_atomic_protection__[((((ethr_uint_t) (PTR)) >> ETHR_ATMC_FLLBK_ADDR_SHIFT) \ + & ((1 << ETHR_ATMC_FLLBK_ADDR_BITS) - 1))].u.lck) + + +#define ETHR_ATOMIC_OP_FALLBACK_IMPL__(AP, EXPS) \ +do { \ + ethr_spinlock_t *slp__ = ETHR_ATOMIC_PTR2LCK__((AP)); \ + ethr_spin_lock(slp__); \ + { EXPS; } \ + ethr_spin_unlock(slp__); \ +} while (0) + +ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATMC_FLLBK_ADDR_BITS]; + +#endif + + +#if defined(ETHR_AMC_FALLBACK__) + +/* + * Fallback for large sized (word and/or double word size) atomics using + * an "Atomic Modification Counter" based on smaller sized native atomics. + * + * We use a 63-bit modification counter and a one bit exclusive flag. + * If 32-bit native atomics are used, we need two 32-bit native atomics. + * The exclusive flag is the least significant bit, or if multiple atomics + * are used, the least significant bit of the least significant atomic. + * + * When using the AMC fallback the following is true: + * - Reads of the same atomic variable can be done in parallel. + * - Uncontended reads doesn't cause any cache line invalidations, + * since no modifications are done. + * - Assuming that the AMC atomic(s) and the integer(s) containing the + * value of the implemented atomic resides in the same cache line, + * modifications will only cause invalidations of one cache line. + * + * When using the spinlock based fallback none of the above is true, + * however, the spinlock based fallback consumes less memory. + */ + +# if ETHR_AMC_NO_ATMCS__ != 1 && ETHR_AMC_NO_ATMCS__ != 2 +# error "Not supported" +# endif +# define ETHR_AMC_MAX_TRY_READ__ 10 +# ifdef ETHR_DEBUG +# define ETHR_DBG_CHK_EXCL_STATE(ASP, S) \ +do { \ + ETHR_AMC_SINT_T__ act = ETHR_AMC_ATMC_FUNC__(read)(&(ASP)->atomic[0]); \ + ETHR_ASSERT(act == (S) + 1); \ + ETHR_ASSERT(act & 1); \ +} while (0) +# else +# define ETHR_DBG_CHK_EXCL_STATE(ASP, S) +# endif + +static ETHR_INLINE void +amc_init(ethr_amc_t *amc, int dw, ethr_sint_t *avar, ethr_sint_t *val) +{ + avar[0] = val[0]; + if (dw) + avar[1] = val[1]; +#if ETHR_AMC_NO_ATMCS__ == 2 + ETHR_AMC_ATMC_FUNC__(init)(&amc->atomic[1], 0); #endif + ETHR_AMC_ATMC_FUNC__(init_wb)(&amc->atomic[0], 0); +} + +static ETHR_INLINE ETHR_AMC_SINT_T__ +amc_set_excl(ethr_amc_t *amc, ETHR_AMC_SINT_T__ prev_state0) +{ + ETHR_AMC_SINT_T__ state0 = prev_state0; + /* Set exclusive flag. */ + while (1) { + ETHR_AMC_SINT_T__ act_state0, new_state0; + while (state0 & 1) { /* Wait until exclusive bit has been cleared */ + ETHR_SPIN_BODY; + state0 = ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[0]); + } + /* Try to set exclusive bit */ + new_state0 = state0 + 1; + act_state0 = ETHR_AMC_ATMC_FUNC__(cmpxchg_acqb)(&amc->atomic[0], + new_state0, + state0); + if (state0 == act_state0) + return state0; /* old state0 */ + state0 = act_state0; + } +} + +static ETHR_INLINE void +amc_inc_mc_unset_excl(ethr_amc_t *amc, ETHR_AMC_SINT_T__ old_state0) +{ + ETHR_AMC_SINT_T__ state0 = old_state0; + + /* Increment modification counter and reset exclusive flag. */ + + ETHR_DBG_CHK_EXCL_STATE(amc, state0); + + state0 += 2; + + ETHR_ASSERT((state0 & 1) == 0); + +#if ETHR_AMC_NO_ATMCS__ == 2 + if (state0 == 0) { + /* + * state0 wrapped, so we need to increment state1. There is no need + * for atomic inc op, since this is always done while having exclusive + * flag. + */ + ETHR_AMC_SINT_T__ state1 = ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[1]); + state1++; + ETHR_AMC_ATMC_FUNC__(set)(&amc->atomic[1], state1); + } +#endif + ETHR_AMC_ATMC_FUNC__(set_relb)(&amc->atomic[0], state0); +} + +static ETHR_INLINE void +amc_unset_excl(ethr_amc_t *amc, ETHR_AMC_SINT_T__ old_state0) +{ + ETHR_DBG_CHK_EXCL_STATE(amc, old_state0); + /* + * Reset exclusive flag, but leave modification counter unchanged, + * i.e., restore state to what it was before setting exclusive + * flag. + */ + ETHR_AMC_ATMC_FUNC__(set_relb)(&amc->atomic[0], old_state0); +} + +static ETHR_INLINE void +amc_set(ethr_amc_t *amc, int dw, ethr_sint_t *avar, ethr_sint_t *val) +{ + ETHR_AMC_SINT_T__ state0 = ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[0]); + + state0 = amc_set_excl(amc, state0); + + avar[0] = val[0]; + if (dw) + avar[1] = val[1]; + + amc_inc_mc_unset_excl(amc, state0); +} + +static ETHR_INLINE int +amc_try_read(ethr_amc_t *amc, int dw, ethr_sint_t *avar, + ethr_sint_t *val, ETHR_AMC_SINT_T__ *state0p) +{ + /* *state0p should contain last read value if aborting */ + ETHR_AMC_SINT_T__ old_state0; +#if ETHR_AMC_NO_ATMCS__ == 2 + ETHR_AMC_SINT_T__ state1; + int abrt; +#endif + + *state0p = ETHR_AMC_ATMC_FUNC__(read_rb)(&amc->atomic[0]); + if ((*state0p) & 1) + return 0; /* exclusive flag set; abort */ +#if ETHR_AMC_NO_ATMCS__ == 2 + state1 = ETHR_AMC_ATMC_FUNC__(read_rb)(&amc->atomic[1]); +#else + ETHR_COMPILER_BARRIER; +#endif + + val[0] = avar[0]; + if (dw) + val[1] = avar[1]; + + ETHR_READ_MEMORY_BARRIER; + + /* + * Abort if state has changed (i.e, either the exclusive + * flag is set, or modification counter changed). + */ + old_state0 = *state0p; +#if ETHR_AMC_NO_ATMCS__ == 2 + *state0p = ETHR_AMC_ATMC_FUNC__(read_rb)(&amc->atomic[0]); + abrt = (old_state0 != *state0p); + abrt |= (state1 != ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[1])); + return abrt == 0; +#else + *state0p = ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[0]); + return old_state0 == *state0p; +#endif +} + +static ETHR_INLINE void +amc_read(ethr_amc_t *amc, int dw, ethr_sint_t *avar, ethr_sint_t *val) +{ + ETHR_AMC_SINT_T__ state0; + int i; + +#if ETHR_AMC_MAX_TRY_READ__ == 0 + state0 = ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[0]); +#else + for (i = 0; i < ETHR_AMC_MAX_TRY_READ__; i++) { + if (amc_try_read(amc, dw, avar, val, &state0)) + return; /* read success */ + ETHR_SPIN_BODY; + } +#endif + + state0 = amc_set_excl(amc, state0); + + val[0] = avar[0]; + if (dw) + val[1] = avar[1]; + + amc_unset_excl(amc, state0); +} + +static ETHR_INLINE int +amc_cmpxchg(ethr_amc_t *amc, int dw, ethr_sint_t *avar, + ethr_sint_t *new, ethr_sint_t *xchg) +{ + ethr_sint_t val[2]; + ETHR_AMC_SINT_T__ state0; + + if (amc_try_read(amc, dw, avar, val, &state0)) { + if (val[0] != xchg[0] || (dw && val[1] != xchg[1])) { + xchg[0] = val[0]; + if (dw) + xchg[1] = val[1]; + return 0; /* failed */ + } + /* Operation will succeed if not interrupted */ + } + + state0 = amc_set_excl(amc, state0); + + if (xchg[0] != avar[0] || (dw && xchg[1] != avar[1])) { + xchg[0] = avar[0]; + if (dw) + xchg[1] = avar[1]; + + ETHR_DBG_CHK_EXCL_STATE(amc, state0); + + amc_unset_excl(amc, state0); + return 0; /* failed */ + } + + avar[0] = new[0]; + if (dw) + avar[1] = new[1]; + + amc_inc_mc_unset_excl(amc, state0); + return 1; +} + + +#define ETHR_AMC_MODIFICATION_OPS__(AMC, OPS) \ +do { \ + ETHR_AMC_SINT_T__ state0__; \ + state0__ = ETHR_AMC_ATMC_FUNC__(read)(&(AMC)->atomic[0]); \ + state0__ = amc_set_excl((AMC), state0__); \ + { OPS; } \ + amc_inc_mc_unset_excl((AMC), state0__); \ +} while (0) + +#endif /* amc fallback */ + int ethr_init_atomics(void) { -#ifndef ETHR_HAVE_NATIVE_ATOMICS - { - int i; - for (i = 0; i < (1 << ETHR_ATOMIC_ADDR_BITS); i++) { - int res = ethr_spinlock_init(ðr_atomic_protection__[i].u.lck); - if (res != 0) - return res; - } +#if (!defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) \ + || !defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)) + int i; + for (i = 0; i < (1 << ETHR_ATMC_FLLBK_ADDR_BITS); i++) { + int res = ethr_spinlock_init(ðr_atomic_protection__[i].u.lck); + if (res != 0) + return res; } #endif return 0; } + +/* ---------- Double word size atomic implementation ---------- */ + + + /* - * --- Pointer size atomics --------------------------------------------------- + * Double word atomics need runtime test. */ -ethr_sint_t * -ethr_atomic_addr(ethr_atomic_t *var) +int ethr_have_native_dw_atomic(void) +{ + return ethr_have_native_dw_atomic__(); +} + + +/* --- addr() --- */ + +ethr_sint_t *ETHR_DW_ATOMIC_FUNC__(addr)(ethr_dw_atomic_t *var) +{ + ethr_sint_t *res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + res = ethr_dw_atomic_addr__(var); +#elif defined(ETHR_AMC_FALLBACK__) + res = (ethr_sint_t *) ((&var->fallback))->sint; +#else + res = (ethr_sint_t *) (&var->fallback); +#endif + return res; +} + +#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ +ethr_sint_t *ethr_dw_atomic_addr(ethr_dw_atomic_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_dw_atomic_addr__(var); +} +#endif + + +/* -- cmpxchg() -- */ + + +int ETHR_DW_ATOMIC_FUNC__(cmpxchg)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val) +{ + int res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + res = ethr_dw_atomic_cmpxchg__(var, val, old_val); +#elif defined(ETHR_AMC_FALLBACK__) + res = amc_cmpxchg(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint, old_val->sint); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), + { + res = ((&var->fallback)->sint[0] == old_val->sint[0] && (&var->fallback)->sint[1] == old_val->sint[1]); + if (res) { + (&var->fallback)->sint[0] = val->sint[0]; + (&var->fallback)->sint[1] = val->sint[1]; + } + else { + old_val->sint[0] = (&var->fallback)->sint[0]; + old_val->sint[1] = (&var->fallback)->sint[1]; + } + }); +#endif + return res; +} + +#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ +int ethr_dw_atomic_cmpxchg(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_dw_atomic_cmpxchg__(var, val, old_val); +} +#endif + +int ETHR_DW_ATOMIC_FUNC__(cmpxchg_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val) +{ + int res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + res = ethr_dw_atomic_cmpxchg_rb__(var, val, old_val); +#elif defined(ETHR_AMC_FALLBACK__) + res = amc_cmpxchg(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint, old_val->sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), + { + res = ((&var->fallback)->sint[0] == old_val->sint[0] && (&var->fallback)->sint[1] == old_val->sint[1]); + if (res) { + (&var->fallback)->sint[0] = val->sint[0]; + (&var->fallback)->sint[1] = val->sint[1]; + } + else { + old_val->sint[0] = (&var->fallback)->sint[0]; + old_val->sint[1] = (&var->fallback)->sint[1]; + } + }); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + return res; +} + +#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ +int ethr_dw_atomic_cmpxchg_rb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_dw_atomic_cmpxchg_rb__(var, val, old_val); +} +#endif + +int ETHR_DW_ATOMIC_FUNC__(cmpxchg_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val) +{ + int res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + res = ethr_dw_atomic_cmpxchg_wb__(var, val, old_val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_StoreStore); + res = amc_cmpxchg(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint, old_val->sint); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), + { + res = ((&var->fallback)->sint[0] == old_val->sint[0] && (&var->fallback)->sint[1] == old_val->sint[1]); + if (res) { + (&var->fallback)->sint[0] = val->sint[0]; + (&var->fallback)->sint[1] = val->sint[1]; + } + else { + old_val->sint[0] = (&var->fallback)->sint[0]; + old_val->sint[1] = (&var->fallback)->sint[1]; + } + }); +#endif + return res; +} + +#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ +int ethr_dw_atomic_cmpxchg_wb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_dw_atomic_cmpxchg_wb__(var, val, old_val); +} +#endif + +int ETHR_DW_ATOMIC_FUNC__(cmpxchg_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val) +{ + int res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + res = ethr_dw_atomic_cmpxchg_acqb__(var, val, old_val); +#elif defined(ETHR_AMC_FALLBACK__) + res = amc_cmpxchg(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint, old_val->sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), + { + res = ((&var->fallback)->sint[0] == old_val->sint[0] && (&var->fallback)->sint[1] == old_val->sint[1]); + if (res) { + (&var->fallback)->sint[0] = val->sint[0]; + (&var->fallback)->sint[1] = val->sint[1]; + } + else { + old_val->sint[0] = (&var->fallback)->sint[0]; + old_val->sint[1] = (&var->fallback)->sint[1]; + } + }); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; +} + +#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ +int ethr_dw_atomic_cmpxchg_acqb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_dw_atomic_cmpxchg_acqb__(var, val, old_val); +} +#endif + +int ETHR_DW_ATOMIC_FUNC__(cmpxchg_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val) +{ + int res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + res = ethr_dw_atomic_cmpxchg_relb__(var, val, old_val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = amc_cmpxchg(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint, old_val->sint); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), + { + res = ((&var->fallback)->sint[0] == old_val->sint[0] && (&var->fallback)->sint[1] == old_val->sint[1]); + if (res) { + (&var->fallback)->sint[0] = val->sint[0]; + (&var->fallback)->sint[1] = val->sint[1]; + } + else { + old_val->sint[0] = (&var->fallback)->sint[0]; + old_val->sint[1] = (&var->fallback)->sint[1]; + } + }); +#endif + return res; +} + +#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ +int ethr_dw_atomic_cmpxchg_relb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_dw_atomic_cmpxchg_relb__(var, val, old_val); +} +#endif + +int ETHR_DW_ATOMIC_FUNC__(cmpxchg_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val) +{ + int res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + res = ethr_dw_atomic_cmpxchg_mb__(var, val, old_val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = amc_cmpxchg(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint, old_val->sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), + { + res = ((&var->fallback)->sint[0] == old_val->sint[0] && (&var->fallback)->sint[1] == old_val->sint[1]); + if (res) { + (&var->fallback)->sint[0] = val->sint[0]; + (&var->fallback)->sint[1] = val->sint[1]; + } + else { + old_val->sint[0] = (&var->fallback)->sint[0]; + old_val->sint[1] = (&var->fallback)->sint[1]; + } + }); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; +} + +#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ +int ethr_dw_atomic_cmpxchg_mb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_dw_atomic_cmpxchg_mb__(var, val, old_val); +} +#endif + + +/* -- set() -- */ + + +void ETHR_DW_ATOMIC_FUNC__(set)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + ethr_dw_atomic_set__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + amc_set(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]); +#endif + +} + +#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ +void ethr_dw_atomic_set(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_dw_atomic_set__(var, val); +} +#endif + +void ETHR_DW_ATOMIC_FUNC__(set_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + ethr_dw_atomic_set_rb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + amc_set(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + +} + +#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ +void ethr_dw_atomic_set_rb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_dw_atomic_set_rb__(var, val); +} +#endif + +void ETHR_DW_ATOMIC_FUNC__(set_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + ethr_dw_atomic_set_wb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_StoreStore); + amc_set(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]); +#endif + +} + +#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ +void ethr_dw_atomic_set_wb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_dw_atomic_set_wb__(var, val); +} +#endif + +void ETHR_DW_ATOMIC_FUNC__(set_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + ethr_dw_atomic_set_acqb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + amc_set(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#endif + +} + +#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ +void ethr_dw_atomic_set_acqb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_dw_atomic_set_acqb__(var, val); +} +#endif + +void ETHR_DW_ATOMIC_FUNC__(set_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + ethr_dw_atomic_set_relb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + amc_set(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint); +#else + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]); +#endif + +} + +#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ +void ethr_dw_atomic_set_relb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_dw_atomic_set_relb__(var, val); +} +#endif + +void ETHR_DW_ATOMIC_FUNC__(set_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + ethr_dw_atomic_set_mb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + amc_set(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#endif + +} + +#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ +void ethr_dw_atomic_set_mb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_dw_atomic_set_mb__(var, val); +} +#endif + + +/* -- read() -- */ + + +void ETHR_DW_ATOMIC_FUNC__(read)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + ethr_dw_atomic_read__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + amc_read(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), val->sint[0] = (&var->fallback)->sint[0]; val->sint[1] = (&var->fallback)->sint[1]); +#endif + +} + +#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ +void ethr_dw_atomic_read(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_dw_atomic_read__(var, val); +} +#endif + +void ETHR_DW_ATOMIC_FUNC__(read_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + ethr_dw_atomic_read_rb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + amc_read(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), val->sint[0] = (&var->fallback)->sint[0]; val->sint[1] = (&var->fallback)->sint[1]); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + +} + +#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ +void ethr_dw_atomic_read_rb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_dw_atomic_read_rb__(var, val); +} +#endif + +void ETHR_DW_ATOMIC_FUNC__(read_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + ethr_dw_atomic_read_wb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_StoreStore); + amc_read(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), val->sint[0] = (&var->fallback)->sint[0]; val->sint[1] = (&var->fallback)->sint[1]); +#endif + +} + +#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ +void ethr_dw_atomic_read_wb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_dw_atomic_read_wb__(var, val); +} +#endif + +void ETHR_DW_ATOMIC_FUNC__(read_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + ethr_dw_atomic_read_acqb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + amc_read(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint); + ETHR_MEMBAR(ETHR_LoadStore); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), val->sint[0] = (&var->fallback)->sint[0]; val->sint[1] = (&var->fallback)->sint[1]); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#endif + +} + +#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ +void ethr_dw_atomic_read_acqb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_dw_atomic_read_acqb__(var, val); +} +#endif + +void ETHR_DW_ATOMIC_FUNC__(read_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + ethr_dw_atomic_read_relb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + amc_read(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), val->sint[0] = (&var->fallback)->sint[0]; val->sint[1] = (&var->fallback)->sint[1]); +#endif + +} + +#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ +void ethr_dw_atomic_read_relb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_dw_atomic_read_relb__(var, val); +} +#endif + +void ETHR_DW_ATOMIC_FUNC__(read_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + ethr_dw_atomic_read_mb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + amc_read(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint); + ETHR_MEMBAR(ETHR_LoadStore); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), val->sint[0] = (&var->fallback)->sint[0]; val->sint[1] = (&var->fallback)->sint[1]); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#endif + +} + +#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ +void ethr_dw_atomic_read_mb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_dw_atomic_read_mb__(var, val); +} +#endif + + +/* -- init() -- */ + + +void ETHR_DW_ATOMIC_FUNC__(init)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + ethr_dw_atomic_init__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + amc_init(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]); +#endif + +} + +#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ +void ethr_dw_atomic_init(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(var); + ethr_dw_atomic_init__(var, val); +} +#endif + +void ETHR_DW_ATOMIC_FUNC__(init_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + ethr_dw_atomic_init_rb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + amc_init(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + +} + +#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ +void ethr_dw_atomic_init_rb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(var); + ethr_dw_atomic_init_rb__(var, val); +} +#endif + +void ETHR_DW_ATOMIC_FUNC__(init_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + ethr_dw_atomic_init_wb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_StoreStore); + amc_init(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]); +#endif + +} + +#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ +void ethr_dw_atomic_init_wb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(var); + ethr_dw_atomic_init_wb__(var, val); +} +#endif + +void ETHR_DW_ATOMIC_FUNC__(init_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + ethr_dw_atomic_init_acqb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + amc_init(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + +} + +#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ +void ethr_dw_atomic_init_acqb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(var); + ethr_dw_atomic_init_acqb__(var, val); +} +#endif + +void ETHR_DW_ATOMIC_FUNC__(init_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + ethr_dw_atomic_init_relb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + amc_init(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]); +#endif + +} + +#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ +void ethr_dw_atomic_init_relb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(var); + ethr_dw_atomic_init_relb__(var, val); +} +#endif + +void ETHR_DW_ATOMIC_FUNC__(init_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + ethr_dw_atomic_init_mb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + amc_init(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + +} + +#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ +void ethr_dw_atomic_init_mb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ + ETHR_ASSERT(var); + ethr_dw_atomic_init_mb__(var, val); +} +#endif + + +/* ---------- Word size atomic implementation ---------- */ + + + + +/* --- addr() --- */ + +ethr_sint_t *ethr_atomic_addr(ethr_atomic_t *var) +{ + ethr_sint_t *res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_addr__(var); +#elif defined(ETHR_AMC_FALLBACK__) + res = (ethr_sint_t *) (var)->sint; +#else + res = (ethr_sint_t *) var; +#endif + return res; +} + + +/* -- cmpxchg() -- */ + + +ethr_sint_t ethr_atomic_cmpxchg(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_cmpxchg__(var, val, old_val); +#elif defined(ETHR_AMC_FALLBACK__) + res = old_val; + (void) amc_cmpxchg(&var->amc, 0, &var->sint, &val, &res); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var)); +#endif + return res; +} + +ethr_sint_t ethr_atomic_cmpxchg_rb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_cmpxchg_rb__(var, val, old_val); +#elif defined(ETHR_AMC_FALLBACK__) + res = old_val; + (void) amc_cmpxchg(&var->amc, 0, &var->sint, &val, &res); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var)); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + return res; +} + +ethr_sint_t ethr_atomic_cmpxchg_wb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_cmpxchg_wb__(var, val, old_val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_StoreStore); + res = old_val; + (void) amc_cmpxchg(&var->amc, 0, &var->sint, &val, &res); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var)); +#endif + return res; +} + +ethr_sint_t ethr_atomic_cmpxchg_acqb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_cmpxchg_acqb__(var, val, old_val); +#elif defined(ETHR_AMC_FALLBACK__) + res = old_val; + (void) amc_cmpxchg(&var->amc, 0, &var->sint, &val, &res); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var)); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; +} + +ethr_sint_t ethr_atomic_cmpxchg_relb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_cmpxchg_relb__(var, val, old_val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = old_val; + (void) amc_cmpxchg(&var->amc, 0, &var->sint, &val, &res); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var)); +#endif + return res; +} + +ethr_sint_t ethr_atomic_cmpxchg_mb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_cmpxchg_mb__(var, val, old_val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + res = old_val; + (void) amc_cmpxchg(&var->amc, 0, &var->sint, &val, &res); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var)); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; +} + + +/* -- xchg() -- */ + + +ethr_sint_t ethr_atomic_xchg(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_xchg__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint = val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val); +#endif + return res; +} + +ethr_sint_t ethr_atomic_xchg_rb(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_xchg_rb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint = val); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + return res; +} + +ethr_sint_t ethr_atomic_xchg_wb(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_xchg_wb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint = val); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val); +#endif + return res; +} + +ethr_sint_t ethr_atomic_xchg_acqb(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_xchg_acqb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint = val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; +} + +ethr_sint_t ethr_atomic_xchg_relb(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_xchg_relb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint = val); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val); +#endif + return res; +} + +ethr_sint_t ethr_atomic_xchg_mb(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_xchg_mb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint = val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; +} + + +/* -- set() -- */ + + +void ethr_atomic_set(ethr_atomic_t *var, ethr_sint_t val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + ethr_atomic_set__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + amc_set(&var->amc, 0, &var->sint, &val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val); +#endif + +} + +void ethr_atomic_set_rb(ethr_atomic_t *var, ethr_sint_t val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + ethr_atomic_set_rb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + amc_set(&var->amc, 0, &var->sint, &val); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + +} + +void ethr_atomic_set_wb(ethr_atomic_t *var, ethr_sint_t val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + ethr_atomic_set_wb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_StoreStore); + amc_set(&var->amc, 0, &var->sint, &val); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val); +#endif + +} + +void ethr_atomic_set_acqb(ethr_atomic_t *var, ethr_sint_t val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + ethr_atomic_set_acqb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + amc_set(&var->amc, 0, &var->sint, &val); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#endif + +} + +void ethr_atomic_set_relb(ethr_atomic_t *var, ethr_sint_t val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + ethr_atomic_set_relb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + amc_set(&var->amc, 0, &var->sint, &val); +#else + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val); +#endif + +} + +void ethr_atomic_set_mb(ethr_atomic_t *var, ethr_sint_t val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + ethr_atomic_set_mb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + amc_set(&var->amc, 0, &var->sint, &val); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#endif + +} + + +/* -- init() -- */ + + +void ethr_atomic_init(ethr_atomic_t *var, ethr_sint_t val) +{ + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + ethr_atomic_init__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + amc_init(&var->amc, 0, &var->sint, &val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val); +#endif + +} + +void ethr_atomic_init_rb(ethr_atomic_t *var, ethr_sint_t val) +{ + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + ethr_atomic_init_rb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + amc_init(&var->amc, 0, &var->sint, &val); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + +} + +void ethr_atomic_init_wb(ethr_atomic_t *var, ethr_sint_t val) +{ + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + ethr_atomic_init_wb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_StoreStore); + amc_init(&var->amc, 0, &var->sint, &val); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val); +#endif + +} + +void ethr_atomic_init_acqb(ethr_atomic_t *var, ethr_sint_t val) +{ + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + ethr_atomic_init_acqb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + amc_init(&var->amc, 0, &var->sint, &val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + +} + +void ethr_atomic_init_relb(ethr_atomic_t *var, ethr_sint_t val) +{ + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + ethr_atomic_init_relb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + amc_init(&var->amc, 0, &var->sint, &val); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val); +#endif + +} + +void ethr_atomic_init_mb(ethr_atomic_t *var, ethr_sint_t val) +{ + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + ethr_atomic_init_mb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + amc_init(&var->amc, 0, &var->sint, &val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + +} + + +/* -- add_read() -- */ + + +ethr_sint_t ethr_atomic_add_read(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_add_read__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val; res = var->sint); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var); +#endif + return res; +} + +ethr_sint_t ethr_atomic_add_read_rb(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_add_read_rb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val; res = var->sint); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + return res; +} + +ethr_sint_t ethr_atomic_add_read_wb(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_add_read_wb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val; res = var->sint); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var); +#endif + return res; +} + +ethr_sint_t ethr_atomic_add_read_acqb(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_add_read_acqb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val; res = var->sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; +} + +ethr_sint_t ethr_atomic_add_read_relb(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_add_read_relb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val; res = var->sint); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var); +#endif + return res; +} + +ethr_sint_t ethr_atomic_add_read_mb(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_add_read_mb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val; res = var->sint); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; +} + + +/* -- read() -- */ + + +ethr_sint_t ethr_atomic_read(ethr_atomic_t *var) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_read__(var); +#elif defined(ETHR_AMC_FALLBACK__) + amc_read(&var->amc, 0, &var->sint, &res); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var); +#endif + return res; +} + +ethr_sint_t ethr_atomic_read_rb(ethr_atomic_t *var) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_read_rb__(var); +#elif defined(ETHR_AMC_FALLBACK__) + amc_read(&var->amc, 0, &var->sint, &res); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + return res; +} + +ethr_sint_t ethr_atomic_read_wb(ethr_atomic_t *var) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_read_wb__(var); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_StoreStore); + amc_read(&var->amc, 0, &var->sint, &res); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var); +#endif + return res; +} + +ethr_sint_t ethr_atomic_read_acqb(ethr_atomic_t *var) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_read_acqb__(var); +#elif defined(ETHR_AMC_FALLBACK__) + amc_read(&var->amc, 0, &var->sint, &res); + ETHR_MEMBAR(ETHR_LoadStore); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#endif + return res; +} + +ethr_sint_t ethr_atomic_read_relb(ethr_atomic_t *var) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_read_relb__(var); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + amc_read(&var->amc, 0, &var->sint, &res); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var); +#endif + return res; +} + +ethr_sint_t ethr_atomic_read_mb(ethr_atomic_t *var) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_read_mb__(var); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + amc_read(&var->amc, 0, &var->sint, &res); + ETHR_MEMBAR(ETHR_LoadStore); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#endif + return res; +} + + +/* -- inc_read() -- */ + + +ethr_sint_t ethr_atomic_inc_read(ethr_atomic_t *var) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_inc_read__(var); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = ++(var->sint)); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var)); +#endif + return res; +} + +ethr_sint_t ethr_atomic_inc_read_rb(ethr_atomic_t *var) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_inc_read_rb__(var); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = ++(var->sint)); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var)); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + return res; +} + +ethr_sint_t ethr_atomic_inc_read_wb(ethr_atomic_t *var) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_inc_read_wb__(var); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = ++(var->sint)); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var)); +#endif + return res; +} + +ethr_sint_t ethr_atomic_inc_read_acqb(ethr_atomic_t *var) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_inc_read_acqb__(var); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = ++(var->sint)); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var)); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; +} + +ethr_sint_t ethr_atomic_inc_read_relb(ethr_atomic_t *var) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_inc_read_relb__(var); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = ++(var->sint)); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var)); +#endif + return res; +} + +ethr_sint_t ethr_atomic_inc_read_mb(ethr_atomic_t *var) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_inc_read_mb__(var); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = ++(var->sint)); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var)); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; +} + + +/* -- dec_read() -- */ + + +ethr_sint_t ethr_atomic_dec_read(ethr_atomic_t *var) { + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - return ethr_atomic_addr__(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_dec_read__(var); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = --(var->sint)); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var)); +#endif + return res; } -void -ethr_atomic_init(ethr_atomic_t *var, ethr_sint_t i) +ethr_sint_t ethr_atomic_dec_read_rb(ethr_atomic_t *var) { + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - ethr_atomic_init__(var, i); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_dec_read_rb__(var); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = --(var->sint)); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var)); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + return res; } -void -ethr_atomic_set(ethr_atomic_t *var, ethr_sint_t i) +ethr_sint_t ethr_atomic_dec_read_wb(ethr_atomic_t *var) { + ethr_sint_t res; ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - ethr_atomic_set__(var, i); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_dec_read_wb__(var); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = --(var->sint)); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var)); +#endif + return res; } -ethr_sint_t -ethr_atomic_read(ethr_atomic_t *var) +ethr_sint_t ethr_atomic_dec_read_acqb(ethr_atomic_t *var) { + ethr_sint_t res; ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - return ethr_atomic_read__(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_dec_read_acqb__(var); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = --(var->sint)); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var)); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; } -ethr_sint_t -ethr_atomic_add_read(ethr_atomic_t *var, ethr_sint_t incr) +ethr_sint_t ethr_atomic_dec_read_relb(ethr_atomic_t *var) { + ethr_sint_t res; ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - return ethr_atomic_add_read__(var, incr); -} +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_dec_read_relb__(var); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = --(var->sint)); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var)); +#endif + return res; +} -ethr_sint_t -ethr_atomic_inc_read(ethr_atomic_t *var) +ethr_sint_t ethr_atomic_dec_read_mb(ethr_atomic_t *var) { + ethr_sint_t res; ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - return ethr_atomic_inc_read__(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_dec_read_mb__(var); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = --(var->sint)); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var)); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; } -ethr_sint_t -ethr_atomic_dec_read(ethr_atomic_t *var) + +/* -- add() -- */ + + +void ethr_atomic_add(ethr_atomic_t *var, ethr_sint_t val) { ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - return ethr_atomic_dec_read__(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + ethr_atomic_add__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val); +#endif + +} + +void ethr_atomic_add_rb(ethr_atomic_t *var, ethr_sint_t val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + ethr_atomic_add_rb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + } -void -ethr_atomic_add(ethr_atomic_t *var, ethr_sint_t incr) +void ethr_atomic_add_wb(ethr_atomic_t *var, ethr_sint_t val) { ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - ethr_atomic_add__(var, incr); -} - -void -ethr_atomic_inc(ethr_atomic_t *var) +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + ethr_atomic_add_wb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val); +#endif + +} + +void ethr_atomic_add_acqb(ethr_atomic_t *var, ethr_sint_t val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + ethr_atomic_add_acqb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + +} + +void ethr_atomic_add_relb(ethr_atomic_t *var, ethr_sint_t val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + ethr_atomic_add_relb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val); +#endif + +} + +void ethr_atomic_add_mb(ethr_atomic_t *var, ethr_sint_t val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + ethr_atomic_add_mb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + +} + + +/* -- inc() -- */ + + +void ethr_atomic_inc(ethr_atomic_t *var) { ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) ethr_atomic_inc__(var); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_AMC_MODIFICATION_OPS__(&var->amc, ++(var->sint)); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var)); +#endif + } -void -ethr_atomic_dec(ethr_atomic_t *var) +void ethr_atomic_inc_rb(ethr_atomic_t *var) { ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - ethr_atomic_dec__(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + ethr_atomic_inc_rb__(var); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_AMC_MODIFICATION_OPS__(&var->amc, ++(var->sint)); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var)); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + } -ethr_sint_t -ethr_atomic_read_band(ethr_atomic_t *var, ethr_sint_t mask) +void ethr_atomic_inc_wb(ethr_atomic_t *var) { ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - return ethr_atomic_read_band__(var, mask); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + ethr_atomic_inc_wb__(var); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_AMC_MODIFICATION_OPS__(&var->amc, ++(var->sint)); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var)); +#endif + } -ethr_sint_t -ethr_atomic_read_bor(ethr_atomic_t *var, ethr_sint_t mask) +void ethr_atomic_inc_acqb(ethr_atomic_t *var) { ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - return ethr_atomic_read_bor__(var, mask); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + ethr_atomic_inc_acqb__(var); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_AMC_MODIFICATION_OPS__(&var->amc, ++(var->sint)); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var)); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + } -ethr_sint_t -ethr_atomic_xchg(ethr_atomic_t *var, ethr_sint_t new) +void ethr_atomic_inc_relb(ethr_atomic_t *var) { ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - return ethr_atomic_xchg__(var, new); -} +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + ethr_atomic_inc_relb__(var); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_AMC_MODIFICATION_OPS__(&var->amc, ++(var->sint)); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var)); +#endif + +} -ethr_sint_t -ethr_atomic_cmpxchg(ethr_atomic_t *var, ethr_sint_t new, ethr_sint_t expected) +void ethr_atomic_inc_mb(ethr_atomic_t *var) { ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - return ethr_atomic_cmpxchg__(var, new, expected); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + ethr_atomic_inc_mb__(var); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_AMC_MODIFICATION_OPS__(&var->amc, ++(var->sint)); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var)); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + } -ethr_sint_t -ethr_atomic_read_acqb(ethr_atomic_t *var) + +/* -- dec() -- */ + + +void ethr_atomic_dec(ethr_atomic_t *var) { ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - return ethr_atomic_read_acqb__(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + ethr_atomic_dec__(var); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_AMC_MODIFICATION_OPS__(&var->amc, --(var->sint)); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var)); +#endif + } -ethr_sint_t -ethr_atomic_inc_read_acqb(ethr_atomic_t *var) +void ethr_atomic_dec_rb(ethr_atomic_t *var) { ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - return ethr_atomic_inc_read_acqb__(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + ethr_atomic_dec_rb__(var); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_AMC_MODIFICATION_OPS__(&var->amc, --(var->sint)); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var)); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + } -void -ethr_atomic_set_relb(ethr_atomic_t *var, ethr_sint_t i) +void ethr_atomic_dec_wb(ethr_atomic_t *var) { ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - ethr_atomic_set_relb__(var, i); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + ethr_atomic_dec_wb__(var); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_AMC_MODIFICATION_OPS__(&var->amc, --(var->sint)); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var)); +#endif + } -void -ethr_atomic_dec_relb(ethr_atomic_t *var) +void ethr_atomic_dec_acqb(ethr_atomic_t *var) { ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + ethr_atomic_dec_acqb__(var); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_AMC_MODIFICATION_OPS__(&var->amc, --(var->sint)); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var)); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + +} + +void ethr_atomic_dec_relb(ethr_atomic_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) ethr_atomic_dec_relb__(var); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_AMC_MODIFICATION_OPS__(&var->amc, --(var->sint)); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var)); +#endif + } -ethr_sint_t -ethr_atomic_dec_read_relb(ethr_atomic_t *var) +void ethr_atomic_dec_mb(ethr_atomic_t *var) { ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - return ethr_atomic_dec_read_relb__(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + ethr_atomic_dec_mb__(var); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_AMC_MODIFICATION_OPS__(&var->amc, --(var->sint)); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var)); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + } -ethr_sint_t -ethr_atomic_cmpxchg_acqb(ethr_atomic_t *var, ethr_sint_t new, ethr_sint_t exp) + +/* -- read_band() -- */ + + +ethr_sint_t ethr_atomic_read_band(ethr_atomic_t *var, ethr_sint_t val) { + ethr_sint_t res; ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - return ethr_atomic_cmpxchg_acqb__(var, new, exp); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_read_band__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint &= val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val); +#endif + return res; } -ethr_sint_t -ethr_atomic_cmpxchg_relb(ethr_atomic_t *var, ethr_sint_t new, ethr_sint_t exp) +ethr_sint_t ethr_atomic_read_band_rb(ethr_atomic_t *var, ethr_sint_t val) { + ethr_sint_t res; ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - return ethr_atomic_cmpxchg_relb__(var, new, exp); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_read_band_rb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint &= val); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + return res; } +ethr_sint_t ethr_atomic_read_band_wb(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_read_band_wb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint &= val); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val); +#endif + return res; +} + +ethr_sint_t ethr_atomic_read_band_acqb(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_read_band_acqb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint &= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; +} + +ethr_sint_t ethr_atomic_read_band_relb(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_read_band_relb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint &= val); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val); +#endif + return res; +} + +ethr_sint_t ethr_atomic_read_band_mb(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_read_band_mb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint &= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; +} + + +/* -- read_bor() -- */ + + +ethr_sint_t ethr_atomic_read_bor(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_read_bor__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint |= val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val); +#endif + return res; +} + +ethr_sint_t ethr_atomic_read_bor_rb(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_read_bor_rb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint |= val); + ETHR_MEMBAR(ETHR_LoadLoad); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + return res; +} + +ethr_sint_t ethr_atomic_read_bor_wb(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_read_bor_wb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint |= val); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val); +#endif + return res; +} + +ethr_sint_t ethr_atomic_read_bor_acqb(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_read_bor_acqb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint |= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; +} + +ethr_sint_t ethr_atomic_read_bor_relb(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_read_bor_relb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint |= val); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val); +#endif + return res; +} + +ethr_sint_t ethr_atomic_read_bor_mb(ethr_atomic_t *var, ethr_sint_t val) +{ + ethr_sint_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) + res = ethr_atomic_read_bor_mb__(var, val); +#elif defined(ETHR_AMC_FALLBACK__) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint |= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; +} + + +/* ---------- 32-bit atomic implementation ---------- */ + + + + +/* --- addr() --- */ + +ethr_sint32_t *ethr_atomic32_addr(ethr_atomic32_t *var) +{ + ethr_sint32_t *res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_addr__(var); + +#else + res = (ethr_sint32_t *) var; +#endif + return res; +} + + +/* -- cmpxchg() -- */ + + +ethr_sint32_t ethr_atomic32_cmpxchg(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_cmpxchg__(var, val, old_val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var)); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_cmpxchg_rb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_cmpxchg_rb__(var, val, old_val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var)); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_cmpxchg_wb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_cmpxchg_wb__(var, val, old_val); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var)); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_cmpxchg_acqb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_cmpxchg_acqb__(var, val, old_val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var)); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_cmpxchg_relb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_cmpxchg_relb__(var, val, old_val); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var)); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_cmpxchg_mb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_cmpxchg_mb__(var, val, old_val); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var)); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; +} + + +/* -- xchg() -- */ + + +ethr_sint32_t ethr_atomic32_xchg(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_xchg__(var, val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_xchg_rb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_xchg_rb__(var, val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_xchg_wb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_xchg_wb__(var, val); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_xchg_acqb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_xchg_acqb__(var, val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_xchg_relb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_xchg_relb__(var, val); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_xchg_mb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_xchg_mb__(var, val); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; +} + + +/* -- set() -- */ + + +void ethr_atomic32_set(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + ethr_atomic32_set__(var, val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val); +#endif + +} + +void ethr_atomic32_set_rb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + ethr_atomic32_set_rb__(var, val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + +} + +void ethr_atomic32_set_wb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + ethr_atomic32_set_wb__(var, val); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val); +#endif + +} + +void ethr_atomic32_set_acqb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + ethr_atomic32_set_acqb__(var, val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#endif + +} + +void ethr_atomic32_set_relb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + ethr_atomic32_set_relb__(var, val); +#else + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val); +#endif + +} + +void ethr_atomic32_set_mb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + ethr_atomic32_set_mb__(var, val); +#else + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val); + ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore); +#endif + +} + + +/* -- init() -- */ + + +void ethr_atomic32_init(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + ethr_atomic32_init__(var, val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val); +#endif + +} + +void ethr_atomic32_init_rb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + ethr_atomic32_init_rb__(var, val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + +} + +void ethr_atomic32_init_wb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + ethr_atomic32_init_wb__(var, val); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val); +#endif + +} + +void ethr_atomic32_init_acqb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + ethr_atomic32_init_acqb__(var, val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + +} + +void ethr_atomic32_init_relb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + ethr_atomic32_init_relb__(var, val); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val); +#endif + +} + +void ethr_atomic32_init_mb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + ethr_atomic32_init_mb__(var, val); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + +} + + +/* -- add_read() -- */ + + +ethr_sint32_t ethr_atomic32_add_read(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_add_read__(var, val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_add_read_rb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_add_read_rb__(var, val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_add_read_wb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_add_read_wb__(var, val); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_add_read_acqb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_add_read_acqb__(var, val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_add_read_relb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_add_read_relb__(var, val); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_add_read_mb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_add_read_mb__(var, val); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; +} + + +/* -- read() -- */ + + +ethr_sint32_t ethr_atomic32_read(ethr_atomic32_t *var) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_read__(var); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_read_rb(ethr_atomic32_t *var) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_read_rb__(var); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_read_wb(ethr_atomic32_t *var) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_read_wb__(var); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_read_acqb(ethr_atomic32_t *var) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_read_acqb__(var); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_read_relb(ethr_atomic32_t *var) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_read_relb__(var); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_read_mb(ethr_atomic32_t *var) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_read_mb__(var); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#endif + return res; +} + + +/* -- inc_read() -- */ + + +ethr_sint32_t ethr_atomic32_inc_read(ethr_atomic32_t *var) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_inc_read__(var); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var)); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_inc_read_rb(ethr_atomic32_t *var) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_inc_read_rb__(var); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var)); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_inc_read_wb(ethr_atomic32_t *var) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_inc_read_wb__(var); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var)); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_inc_read_acqb(ethr_atomic32_t *var) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_inc_read_acqb__(var); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var)); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_inc_read_relb(ethr_atomic32_t *var) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_inc_read_relb__(var); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var)); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_inc_read_mb(ethr_atomic32_t *var) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_inc_read_mb__(var); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var)); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; +} + + +/* -- dec_read() -- */ -/* - * --- 32-bit atomics --------------------------------------------------------- - */ -ethr_sint32_t * -ethr_atomic32_addr(ethr_atomic32_t *var) +ethr_sint32_t ethr_atomic32_dec_read(ethr_atomic32_t *var) { + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - return ethr_atomic32_addr__(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_dec_read__(var); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var)); +#endif + return res; } -void -ethr_atomic32_init(ethr_atomic32_t *var, ethr_sint32_t i) +ethr_sint32_t ethr_atomic32_dec_read_rb(ethr_atomic32_t *var) { + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - ethr_atomic32_init__(var, i); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_dec_read_rb__(var); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var)); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + return res; } -void -ethr_atomic32_set(ethr_atomic32_t *var, ethr_sint32_t i) +ethr_sint32_t ethr_atomic32_dec_read_wb(ethr_atomic32_t *var) { + ethr_sint32_t res; ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - ethr_atomic32_set__(var, i); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_dec_read_wb__(var); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var)); +#endif + return res; } -ethr_sint32_t -ethr_atomic32_read(ethr_atomic32_t *var) +ethr_sint32_t ethr_atomic32_dec_read_acqb(ethr_atomic32_t *var) { + ethr_sint32_t res; ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - return ethr_atomic32_read__(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_dec_read_acqb__(var); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var)); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; } +ethr_sint32_t ethr_atomic32_dec_read_relb(ethr_atomic32_t *var) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_dec_read_relb__(var); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var)); +#endif + return res; +} -ethr_sint32_t -ethr_atomic32_add_read(ethr_atomic32_t *var, ethr_sint32_t incr) +ethr_sint32_t ethr_atomic32_dec_read_mb(ethr_atomic32_t *var) { + ethr_sint32_t res; ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - return ethr_atomic32_add_read__(var, incr); -} +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_dec_read_mb__(var); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var)); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; +} + + +/* -- add() -- */ -ethr_sint32_t -ethr_atomic32_inc_read(ethr_atomic32_t *var) + +void ethr_atomic32_add(ethr_atomic32_t *var, ethr_sint32_t val) { ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - return ethr_atomic32_inc_read__(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + ethr_atomic32_add__(var, val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val); +#endif + } -ethr_sint32_t -ethr_atomic32_dec_read(ethr_atomic32_t *var) +void ethr_atomic32_add_rb(ethr_atomic32_t *var, ethr_sint32_t val) { ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - return ethr_atomic32_dec_read__(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + ethr_atomic32_add_rb__(var, val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + +} + +void ethr_atomic32_add_wb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + ethr_atomic32_add_wb__(var, val); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val); +#endif + +} + +void ethr_atomic32_add_acqb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + ethr_atomic32_add_acqb__(var, val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + +} + +void ethr_atomic32_add_relb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + ethr_atomic32_add_relb__(var, val); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val); +#endif + } -void -ethr_atomic32_add(ethr_atomic32_t *var, ethr_sint32_t incr) +void ethr_atomic32_add_mb(ethr_atomic32_t *var, ethr_sint32_t val) { ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - ethr_atomic32_add__(var, incr); -} - -void -ethr_atomic32_inc(ethr_atomic32_t *var) +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + ethr_atomic32_add_mb__(var, val); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + +} + + +/* -- inc() -- */ + + +void ethr_atomic32_inc(ethr_atomic32_t *var) { ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) ethr_atomic32_inc__(var); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var)); +#endif + } -void -ethr_atomic32_dec(ethr_atomic32_t *var) +void ethr_atomic32_inc_rb(ethr_atomic32_t *var) { ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - ethr_atomic32_dec__(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + ethr_atomic32_inc_rb__(var); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var)); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + } -ethr_sint32_t -ethr_atomic32_read_band(ethr_atomic32_t *var, ethr_sint32_t mask) +void ethr_atomic32_inc_wb(ethr_atomic32_t *var) { ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - return ethr_atomic32_read_band__(var, mask); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + ethr_atomic32_inc_wb__(var); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var)); +#endif + } -ethr_sint32_t -ethr_atomic32_read_bor(ethr_atomic32_t *var, ethr_sint32_t mask) +void ethr_atomic32_inc_acqb(ethr_atomic32_t *var) { ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - return ethr_atomic32_read_bor__(var, mask); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + ethr_atomic32_inc_acqb__(var); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var)); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + } -ethr_sint32_t -ethr_atomic32_xchg(ethr_atomic32_t *var, ethr_sint32_t new) +void ethr_atomic32_inc_relb(ethr_atomic32_t *var) { ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - return ethr_atomic32_xchg__(var, new); -} +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + ethr_atomic32_inc_relb__(var); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var)); +#endif -ethr_sint32_t -ethr_atomic32_cmpxchg(ethr_atomic32_t *var, - ethr_sint32_t new, - ethr_sint32_t expected) +} + +void ethr_atomic32_inc_mb(ethr_atomic32_t *var) { ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - return ethr_atomic32_cmpxchg__(var, new, expected); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + ethr_atomic32_inc_mb__(var); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var)); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + } -ethr_sint32_t -ethr_atomic32_read_acqb(ethr_atomic32_t *var) + +/* -- dec() -- */ + + +void ethr_atomic32_dec(ethr_atomic32_t *var) { ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - return ethr_atomic32_read_acqb__(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + ethr_atomic32_dec__(var); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var)); +#endif + +} + +void ethr_atomic32_dec_rb(ethr_atomic32_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + ethr_atomic32_dec_rb__(var); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var)); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + } -ethr_sint32_t -ethr_atomic32_inc_read_acqb(ethr_atomic32_t *var) +void ethr_atomic32_dec_wb(ethr_atomic32_t *var) { ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - return ethr_atomic32_inc_read_acqb__(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + ethr_atomic32_dec_wb__(var); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var)); +#endif + } -void -ethr_atomic32_set_relb(ethr_atomic32_t *var, ethr_sint32_t i) +void ethr_atomic32_dec_acqb(ethr_atomic32_t *var) { ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - ethr_atomic32_set_relb__(var, i); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + ethr_atomic32_dec_acqb__(var); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var)); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + } -void -ethr_atomic32_dec_relb(ethr_atomic32_t *var) +void ethr_atomic32_dec_relb(ethr_atomic32_t *var) { ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) ethr_atomic32_dec_relb__(var); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var)); +#endif + +} + +void ethr_atomic32_dec_mb(ethr_atomic32_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + ethr_atomic32_dec_mb__(var); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var)); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + +} + + +/* -- read_band() -- */ + + +ethr_sint32_t ethr_atomic32_read_band(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_read_band__(var, val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_read_band_rb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_read_band_rb__(var, val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_read_band_wb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_read_band_wb__(var, val); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_read_band_acqb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_read_band_acqb__(var, val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_read_band_relb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_read_band_relb__(var, val); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_read_band_mb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_read_band_mb__(var, val); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; +} + + +/* -- read_bor() -- */ + + +ethr_sint32_t ethr_atomic32_read_bor(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_read_bor__(var, val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_read_bor_rb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_read_bor_rb__(var, val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val); + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + return res; } -ethr_sint32_t -ethr_atomic32_dec_read_relb(ethr_atomic32_t *var) +ethr_sint32_t ethr_atomic32_read_bor_wb(ethr_atomic32_t *var, ethr_sint32_t val) { + ethr_sint32_t res; ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - return ethr_atomic32_dec_read_relb__(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_read_bor_wb__(var, val); +#else + ETHR_MEMBAR(ETHR_StoreStore); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val); +#endif + return res; } -ethr_sint32_t -ethr_atomic32_cmpxchg_acqb(ethr_atomic32_t *var, - ethr_sint32_t new, - ethr_sint32_t exp) +ethr_sint32_t ethr_atomic32_read_bor_acqb(ethr_atomic32_t *var, ethr_sint32_t val) { + ethr_sint32_t res; ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - return ethr_atomic32_cmpxchg_acqb__(var, new, exp); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_read_bor_acqb__(var, val); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; } -ethr_sint32_t -ethr_atomic32_cmpxchg_relb(ethr_atomic32_t *var, - ethr_sint32_t new, - ethr_sint32_t exp) +ethr_sint32_t ethr_atomic32_read_bor_relb(ethr_atomic32_t *var, ethr_sint32_t val) { + ethr_sint32_t res; ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(var); - return ethr_atomic32_cmpxchg_relb__(var, new, exp); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_read_bor_relb__(var, val); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val); +#endif + return res; +} + +ethr_sint32_t ethr_atomic32_read_bor_mb(ethr_atomic32_t *var, ethr_sint32_t val) +{ + ethr_sint32_t res; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); +#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) + res = ethr_atomic32_read_bor_mb__(var, val); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad); + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); +#endif + return res; +} + + + +/* --------- Info functions --------- */ + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) +char *zero_ops[] = {NULL}; +#endif + + +static char *native_su_dw_atomic_ops[] = { +#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG + "cmpxchg", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_RB + "cmpxchg_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_WB + "cmpxchg_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_ACQB + "cmpxchg_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_RELB + "cmpxchg_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_MB + "cmpxchg_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET + "set", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_RB + "set_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_WB + "set_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_ACQB + "set_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_RELB + "set_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_MB + "set_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ + "read", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_RB + "read_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_WB + "read_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_ACQB + "read_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_RELB + "read_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_MB + "read_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT + "init", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_RB + "init_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_WB + "init_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_ACQB + "init_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_RELB + "init_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_MB + "init_mb", +#endif + NULL +}; + +char ** +ethr_native_su_dw_atomic_ops(void) +{ + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + if (!ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + return &zero_ops[0]; +#endif + return &native_su_dw_atomic_ops[0]; +} + + +static char *native_dw_atomic_ops[] = { +#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG + "cmpxchg", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_RB + "cmpxchg_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_WB + "cmpxchg_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_ACQB + "cmpxchg_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_RELB + "cmpxchg_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_MB + "cmpxchg_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET + "set", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_RB + "set_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_WB + "set_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_ACQB + "set_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_RELB + "set_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_MB + "set_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ + "read", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_RB + "read_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_WB + "read_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_ACQB + "read_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_RELB + "read_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_MB + "read_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT + "init", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_RB + "init_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_WB + "init_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_ACQB + "init_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_RELB + "init_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_MB + "init_mb", +#endif + NULL +}; + +char ** +ethr_native_dw_atomic_ops(void) +{ + +#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + if (!ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) + return &zero_ops[0]; +#endif + return &native_dw_atomic_ops[0]; +} + + +static char *native_atomic64_ops[] = { +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG + "cmpxchg", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RB + "cmpxchg_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_WB + "cmpxchg_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_ACQB + "cmpxchg_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RELB + "cmpxchg_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_MB + "cmpxchg_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG + "xchg", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_RB + "xchg_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_WB + "xchg_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_ACQB + "xchg_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_RELB + "xchg_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_MB + "xchg_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET + "set", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RB + "set_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_WB + "set_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_ACQB + "set_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RELB + "set_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_MB + "set_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT + "init", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_RB + "init_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_WB + "init_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_ACQB + "init_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_RELB + "init_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_MB + "init_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN + "add_return", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_RB + "add_return_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_WB + "add_return_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_ACQB + "add_return_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_RELB + "add_return_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_MB + "add_return_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ + "read", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_RB + "read_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_WB + "read_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_ACQB + "read_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_RELB + "read_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_MB + "read_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN + "inc_return", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_RB + "inc_return_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_WB + "inc_return_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_ACQB + "inc_return_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_RELB + "inc_return_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_MB + "inc_return_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN + "dec_return", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_RB + "dec_return_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_WB + "dec_return_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_ACQB + "dec_return_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_RELB + "dec_return_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_MB + "dec_return_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD + "add", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RB + "add_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_WB + "add_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_ACQB + "add_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RELB + "add_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_MB + "add_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC + "inc", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RB + "inc_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_WB + "inc_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_ACQB + "inc_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RELB + "inc_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_MB + "inc_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC + "dec", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RB + "dec_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_WB + "dec_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_ACQB + "dec_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RELB + "dec_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_MB + "dec_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD + "and_retold", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_RB + "and_retold_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_WB + "and_retold_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_ACQB + "and_retold_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_RELB + "and_retold_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_MB + "and_retold_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD + "or_retold", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_RB + "or_retold_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_WB + "or_retold_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_ACQB + "or_retold_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_RELB + "or_retold_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_MB + "or_retold_mb", +#endif + NULL +}; + +char ** +ethr_native_atomic64_ops(void) +{ + + return &native_atomic64_ops[0]; } + +static char *native_atomic32_ops[] = { +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG + "cmpxchg", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_RB + "cmpxchg_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_WB + "cmpxchg_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_ACQB + "cmpxchg_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_RELB + "cmpxchg_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_MB + "cmpxchg_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG + "xchg", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_RB + "xchg_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_WB + "xchg_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_ACQB + "xchg_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_RELB + "xchg_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_MB + "xchg_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET + "set", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_RB + "set_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_WB + "set_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_ACQB + "set_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_RELB + "set_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_MB + "set_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT + "init", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_RB + "init_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_WB + "init_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_ACQB + "init_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_RELB + "init_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_MB + "init_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN + "add_return", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_RB + "add_return_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_WB + "add_return_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_ACQB + "add_return_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_RELB + "add_return_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_MB + "add_return_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ + "read", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_RB + "read_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_WB + "read_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_ACQB + "read_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_RELB + "read_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_MB + "read_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN + "inc_return", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_RB + "inc_return_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_WB + "inc_return_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_ACQB + "inc_return_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_RELB + "inc_return_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_MB + "inc_return_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN + "dec_return", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_RB + "dec_return_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_WB + "dec_return_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_ACQB + "dec_return_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_RELB + "dec_return_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_MB + "dec_return_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD + "add", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RB + "add_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_WB + "add_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_ACQB + "add_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RELB + "add_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_MB + "add_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC + "inc", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RB + "inc_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_WB + "inc_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_ACQB + "inc_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RELB + "inc_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_MB + "inc_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC + "dec", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RB + "dec_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_WB + "dec_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_ACQB + "dec_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RELB + "dec_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_MB + "dec_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD + "and_retold", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_RB + "and_retold_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_WB + "and_retold_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_ACQB + "and_retold_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_RELB + "and_retold_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_MB + "and_retold_mb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD + "or_retold", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_RB + "or_retold_rb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_WB + "or_retold_wb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_ACQB + "or_retold_acqb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_RELB + "or_retold_relb", +#endif +#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_MB + "or_retold_mb", +#endif + NULL +}; + +char ** +ethr_native_atomic32_ops(void) +{ + + return &native_atomic32_ops[0]; +} diff --git a/erts/lib_src/common/ethr_aux.c b/erts/lib_src/common/ethr_aux.c index 2c3e25a805..521640317e 100644 --- a/erts/lib_src/common/ethr_aux.c +++ b/erts/lib_src/common/ethr_aux.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2010. All Rights Reserved. + * Copyright Ericsson AB 2010-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -31,10 +31,6 @@ #define ETHR_INLINE_FUNC_NAME_(X) X ## __ #define ETHR_AUX_IMPL__ -#define ETHR_ATOMIC_IMPL__ /* Needed in order to pull in - native atomic implementations - for optimized fallbacks of - spinlocks and rwspinlocks */ #include "ethread.h" #include "ethr_internal.h" #include <string.h> @@ -75,10 +71,87 @@ static int main_threads; static int init_ts_event_alloc(void); +ethr_runtime_t ethr_runtime__ +#ifdef __GNUC__ +__attribute__ ((aligned (ETHR_CACHE_LINE_SIZE))) +#endif + ; + +#if defined(ETHR_X86_RUNTIME_CONF__) + +/* + * x86/x86_64 specifics shared between windows and + * pthread implementations. + */ + +#define ETHR_IS_X86_VENDOR(V, B, C, D) \ + (sizeof(V) == 13 && is_x86_vendor((V), (B), (C), (D))) + +static ETHR_INLINE int +is_x86_vendor(char *str, int ebx, int ecx, int edx) +{ + return (*((int *) &str[0]) == ebx + && *((int *) &str[sizeof(int)]) == edx + && *((int *) &str[sizeof(int)*2]) == ecx); +} + +static void +x86_init(void) +{ + int eax, ebx, ecx, edx; + + eax = ebx = ecx = edx = 0; + + ethr_x86_cpuid__(&eax, &ebx, &ecx, &edx); + + if (eax > 0 + && (ETHR_IS_X86_VENDOR("GenuineIntel", ebx, ecx, edx) + || ETHR_IS_X86_VENDOR("AuthenticAMD", ebx, ecx, edx))) { + eax = 1; + ethr_x86_cpuid__(&eax, &ebx, &ecx, &edx); + } + else { + /* + * The meaning of the feature flags for this + * vendor have not been verified. + */ + eax = ebx = ecx = edx = 0; + } + + /* + * The feature flags tested below have only been verified + * for vendors checked above. Also note that only these + * feature flags have been verified to have these specific + * meanings. If another feature flag test is introduced, + * it has to be verified to have the same meaning for all + * vendors above. + */ + +#if ETHR_SIZEOF_PTR == 8 + /* bit 13 of ecx is set if we have cmpxchg16b */ + ethr_runtime__.conf.have_dw_cmpxchg = (ecx & (1 << 13)); +#elif ETHR_SIZEOF_PTR == 4 + /* bit 8 of edx is set if we have cmpxchg8b */ + ethr_runtime__.conf.have_dw_cmpxchg = (edx & (1 << 8)); +#else +# error "Not supported" +#endif + /* bit 26 of edx is set if we have sse2 */ + ethr_runtime__.conf.have_sse2 = (edx & (1 << 26)); +} + +#endif /* ETHR_X86_RUNTIME_CONF__ */ + + int ethr_init_common__(ethr_init_data *id) { int res; + +#if defined(ETHR_X86_RUNTIME_CONF__) + x86_init(); +#endif + if (id) { ethr_thr_prepare_func__ = id->thread_create_prepare_func; ethr_thr_parent_func__ = id->thread_create_parent_func; diff --git a/erts/lib_src/common/ethr_mutex.c b/erts/lib_src/common/ethr_mutex.c index 2ddef32dfc..81fd6af80a 100644 --- a/erts/lib_src/common/ethr_mutex.c +++ b/erts/lib_src/common/ethr_mutex.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2010. All Rights Reserved. + * Copyright Ericsson AB 2010-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -26,8 +26,9 @@ #include "config.h" #endif -#define ETHR_INLINE_FUNC_NAME_(X) X ## __ +#define ETHR_INLINE_MTX_FUNC_NAME_(X) X ## __ #define ETHR_MUTEX_IMPL__ +#define ETHR_TRY_INLINE_FUNCS #include <limits.h> #include "ethread.h" diff --git a/erts/lib_src/pthread/ethr_x86_sse2_asm.c b/erts/lib_src/pthread/ethr_x86_sse2_asm.c new file mode 100644 index 0000000000..6cbe73cf16 --- /dev/null +++ b/erts/lib_src/pthread/ethr_x86_sse2_asm.c @@ -0,0 +1,31 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2011. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: sse2 asm:s + * Author: Rickard Green + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +/* ETHR_X86_SSE2_ASM_C__ will trigger asm:s to compile to be included */ +#define ETHR_X86_SSE2_ASM_C__ +#include "ethread.h" diff --git a/erts/lib_src/pthread/ethread.c b/erts/lib_src/pthread/ethread.c index f047104103..ad29249bac 100644 --- a/erts/lib_src/pthread/ethread.c +++ b/erts/lib_src/pthread/ethread.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2010. All Rights Reserved. + * Copyright Ericsson AB 2010-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -121,6 +121,98 @@ ethr_ts_event *ethr_get_tse__(void) return pthread_getspecific(ethr_ts_event_key__); } +#if defined(ETHR_PPC_RUNTIME_CONF__) + +static volatile int lwsync_caused_sigill; + +static void +handle_lwsync_sigill(int signum) +{ + lwsync_caused_sigill = 1; +} + +static int +ppc_init__(void) +{ + struct sigaction act, oact; + lwsync_caused_sigill = 0; + + sigemptyset(&act.sa_mask); + act.sa_flags = 0; + act.sa_handler = handle_lwsync_sigill; + if (sigaction(SIGILL, &act, &oact) != 0) + return errno; + + __asm__ __volatile__ ("lwsync\n\t" : : : "memory"); + + act.sa_flags = 0; + act.sa_handler = SIG_DFL; + if (sigaction(SIGILL, &act, &oact) != 0) + return errno; + + ethr_runtime__.conf.have_lwsync = (int) !lwsync_caused_sigill; + return 0; +} + +#endif + +#if defined(ETHR_X86_RUNTIME_CONF__) + +void +ethr_x86_cpuid__(int *eax, int *ebx, int *ecx, int *edx) +{ +#if ETHR_SIZEOF_PTR == 4 + int have_cpuid; + /* + * If it is possible to toggle eflags bit 21, + * we have the cpuid instruction. + */ + __asm__ ("pushf\n\t" + "popl %%eax\n\t" + "movl %%eax, %%ecx\n\t" + "xorl $0x200000, %%eax\n\t" + "pushl %%eax\n\t" + "popf\n\t" + "pushf\n\t" + "popl %%eax\n\t" + "movl $0x0, %0\n\t" + "xorl %%ecx, %%eax\n\t" + "jz no_cpuid\n\t" + "movl $0x1, %0\n\t" + "no_cpuid:\n\t" + : "=r"(have_cpuid) + : + : "%eax", "%ecx", "cc"); + if (!have_cpuid) { + *eax = *ebx = *ecx = *edx = 0; + return; + } +#endif +#if ETHR_SIZEOF_PTR == 4 && defined(__PIC__) && __PIC__ + /* + * When position independet code is used in 32-bit mode, the B register + * is used for storage of global offset table address, and we may not + * use it as input or output in an asm. We need to save and restore the + * B register explicitly (for some reason gcc doesn't provide this + * service to us). + */ + __asm__ ("pushl %%ebx\n\t" + "cpuid\n\t" + "movl %%ebx, %1\n\t" + "popl %%ebx\n\t" + : "=a"(*eax), "=r"(*ebx), "=c"(*ecx), "=d"(*edx) + : "0"(*eax) + : "cc"); +#else + __asm__ ("cpuid\n\t" + : "=a"(*eax), "=b"(*ebx), "=c"(*ecx), "=d"(*edx) + : "0"(*eax) + : "cc"); +#endif +} + +#endif /* ETHR_X86_RUNTIME_CONF__ */ + /* * -------------------------------------------------------------------------- * Exported functions @@ -137,6 +229,12 @@ ethr_init(ethr_init_data *id) ethr_not_inited__ = 0; +#if defined(ETHR_PPC_RUNTIME_CONF__) + res = ppc_init__(); + if (res != 0) + goto error; +#endif + res = ethr_init_common__(id); if (res != 0) goto error; @@ -146,6 +244,8 @@ ethr_init(ethr_init_data *id) child_wait_spin_count = 0; res = pthread_key_create(ðr_ts_event_key__, ethr_ts_event_destructor__); + if (res != 0) + goto error; return 0; error: diff --git a/erts/lib_src/utils/make_atomics_api b/erts/lib_src/utils/make_atomics_api new file mode 100755 index 0000000000..f4e71c7618 --- /dev/null +++ b/erts/lib_src/utils/make_atomics_api @@ -0,0 +1,2186 @@ +#!/usr/bin/env escript +%% -*- erlang -*- + +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2011. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +-mode(compile). + +%%%------------------------------------------------------------------- +%%% @author Rickard Green <[email protected]> +%%% @copyright (C) 2011, Rickard Green +%%% @doc +%%% Generation of the ethread atomic API +%%% @end +%%% Created : 17 Jan 2011 by Rickard Green <[email protected]> +%%%------------------------------------------------------------------- + +-define(H_FILE, "erts/include/internal/ethr_atomics.h"). +-define(C_FILE, "erts/lib_src/common/ethr_atomics.c"). + +%% These order constraints are important: +%% - 'cmpxchg' needs to appear before 'read' +%% - 'xchg' needs to apper before 'set' +%% - 'set' needs to apper before 'init' +%% - 'add_read' needs to apper before 'add', 'inc_read', and 'dec_read' +%% - 'inc_read' needs to apper before and 'inc' +%% - 'dec_read' needs to apper before and 'dec' +-define(ATOMIC_OPS, [cmpxchg, xchg, set, init, add_read, + read, inc_read, dec_read, add, inc, + dec, read_band, read_bor]). + +-define(DW_ATOMIC_OPS, [cmpxchg, set, read, init]). +-define(DW_FUNC_MACRO, "ETHR_DW_ATOMIC_FUNC__"). +-define(DW_RTCHK_MACRO, "ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__"). + +%% Barrier versions we implement +-define(BARRIERS, [none, rb, wb, acqb, relb, mb]). + +-define(ATOMIC_SIZES, ["dword", "word", "32"]). + +-define(HAVE_NATIVE_ATOMIC, "ETHR_HAVE_ETHR_NATIVE_ATOMIC"). + +-define(SU_DW_SINT_FIELD, "dw_sint"). +-define(DW_SINT_FIELD, "sint"). + +%% Fallback +-define(ETHR_ATMC_FLLBK_ADDR_BITS, "10"). +-define(ETHR_ATMC_FLLBK_ADDR_SHIFT, "6"). + +-record(atomic_context, {dw, + amc_fallback, + ret_type, + ret_var, + arg1, + arg2, + arg3, + have_native_atomic_ops, + atomic, + atomic_t, + addr_aint_t, + aint_t, + naint_t, + 'NATMC', + 'ATMC', + unusual_val}). + +atomic_context("dword") -> + #atomic_context{dw = true, + amc_fallback = true, + ret_type = "int", + ret_var = "res", + arg1 = "var", + arg2 = "val", + arg3 = "old_val", + have_native_atomic_ops = "ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS", + atomic = "ethr_dw_atomic", + atomic_t = "ethr_dw_atomic_t", + addr_aint_t = "ethr_sint_t", + aint_t = "ethr_dw_sint_t", + naint_t = "ETHR_SU_DW_NAINT_T__", + 'NATMC' = "DW_NATMC", + 'ATMC' = "DW_ATMC", + unusual_val = "ETHR_UNUSUAL_SINT_VAL__"}; +atomic_context(Size) -> + {SizeSuffix, HaveSize, AMC} = case Size of + "word" -> {"", "WORD_SZ", true}; + _ -> {Size, Size++"BIT", false} + end, + AintT = ["ethr_sint", SizeSuffix, "_t"], + #atomic_context{dw = false, + amc_fallback = AMC, + ret_type = AintT, + ret_var = "res", + arg1 = "var", + arg2 = "val", + arg3 = "old_val", + have_native_atomic_ops = ["ETHR_HAVE_", HaveSize, "_NATIVE_ATOMIC_OPS"], + atomic = ["ethr_atomic", SizeSuffix], + atomic_t = ["ethr_atomic", SizeSuffix, "_t"], + addr_aint_t = AintT, + aint_t = AintT, + naint_t = ["ETHR_NAINT", SizeSuffix, "_T__"], + 'NATMC' = ["NATMC", SizeSuffix], + 'ATMC' = ["ATMC", SizeSuffix], + unusual_val = ["ETHR_UNUSUAL_SINT", SizeSuffix, "_VAL__"]}. + +-record(op_context, {ret, var, val1, val2}). + +-define(POTENTIAL_NBITS, ["64", "32"]). + +is_return_op(#atomic_context{dw = false}, add) -> false; +is_return_op(#atomic_context{dw = false}, inc) -> false; +is_return_op(#atomic_context{dw = false}, dec) -> false; +is_return_op(#atomic_context{dw = true}, read) -> false; +is_return_op(_AC, init) -> false; +is_return_op(_AC, set) -> false; +is_return_op(_AC, _OP) -> true. + +native(add_read) -> add_return; +native(inc_read) -> inc_return; +native(dec_read) -> dec_return; +native(read_band) -> and_retold; +native(read_bor) -> or_retold; +native(Op) -> Op. + +op(Op, #op_context{var = Var, val1 = Val1}) when Op == init; Op == set -> + [Var, " = ", Val1]; +op(read, #op_context{ret = Ret, var = Var}) -> + [Ret, " = ", Var]; +op(add_read, OpC) -> + [op(add, OpC), "; ", op(read, OpC)]; +op(add, #op_context{var = Var, val1 = Val1}) -> + [Var, " += ", Val1]; +op(inc, #op_context{var = Var}) -> + ["++(", Var, ")"]; +op(dec, #op_context{var = Var}) -> + ["--(", Var, ")"]; +op(inc_read, #op_context{ret = Ret, var = Var}) -> + [Ret, " = ++(", Var, ")"]; +op(dec_read, #op_context{ret = Ret, var = Var}) -> + [Ret, " = --(", Var, ")"]; +op(read_band, #op_context{var = Var, val1 = Val1} = OpC) -> + [op(read, OpC), "; ", Var, " &= ", Val1]; +op(read_bor, #op_context{var = Var, val1 = Val1} = OpC) -> + [op(read, OpC), "; ", Var, " |= ", Val1]; +op(xchg, OpC) -> + [op(read, OpC), "; ", op(set, OpC)]; +op(cmpxchg, #op_context{ret = Ret, var = Var, val1 = Val1, val2 = Val2}) -> + [Ret, " = (", Var, " == ", Val2, " ? (", Var, " = ", Val1, ", ", Val2, ") : ", Var, ")"]. + +dw_op(Op, #op_context{var = Var, val1 = Val1}) when Op == init; Op == set -> + [Var, "[0] = ", Val1, "[0]; ", Var, "[1] = ", Val1, "[1]"]; +dw_op(read, #op_context{var = Var, val1 = Val1}) -> + [Val1, "[0] = ", Var, "[0]; ", Val1, "[1] = ", Var, "[1]"]; +dw_op(cmpxchg, #op_context{ret = Ret, var = Var, val1 = Val1, val2 = Val2}) -> + [" + { + ", Ret, " = (", Var, "[0] == ", Val2, "[0] && ", Var, "[1] == ", Val2, "[1]); + if (", Ret, ") { + ", Var, "[0] = ", Val1, "[0]; + ", Var, "[1] = ", Val1, "[1]; + } + else { + ", Val2, "[0] = ", Var, "[0]; + ", Val2, "[1] = ", Var, "[1]; + } + }"]. + +op_head_tail(init) -> {undef, undef}; +op_head_tail(set) -> {store, store}; +op_head_tail(read) -> {load, load}; +op_head_tail(_) -> {load, undef}. + +op_barrier_ext(none) -> ""; +op_barrier_ext(Barrier) -> [$_, a2l(Barrier)]. + +op_call(addr, _DW, Ret, Func, Arg1, _Arg2, _Arg3, _TypeCast) -> + [Ret, " ", Func, "(", Arg1, ");"]; +op_call(Op, false, Ret, Func, Arg1, _Arg2, _Arg3, _TypeCast) when Op == read; + Op == inc_read; + Op == inc_return; + Op == dec_read; + Op == dec_return -> + [Ret, " ", Func, "(", Arg1, ");"]; +op_call(Op, false, _Ret, Func, Arg1, _Arg2, _Arg3, _TypeCast) when Op == inc; + Op == dec -> + [Func, "(", Arg1, ");"]; +op_call(Op, false, Ret, Func, Arg1, Arg2, _Arg3, TypeCast) when Op == add_return; + Op == add_read; + Op == read_band; + Op == and_retold; + Op == read_bor; + Op == or_retold; + Op == xchg -> + [Ret, " ", Func, "(", Arg1, ",", TypeCast, " ", Arg2, ");"]; +op_call(cmpxchg, _DW, Ret, Func, Arg1, Arg2, Arg3, TypeCast) -> + [Ret, " ", Func, "(", Arg1, ",", TypeCast, " ", Arg2, ",", TypeCast, " ", Arg3, ");"]; +op_call(_Op, _DW, _Ret, Func, Arg1, Arg2, _Arg3, TypeCast) -> + [Func, "(", Arg1, ",", TypeCast, " ", Arg2, ");"]. % set, init, add (!= dw), read (== dw) + +native_op_call(#atomic_context{dw = DW, + ret_var = RetVar, + arg1 = Arg1, + arg2 = Arg2, + arg3 = Arg3, + aint_t = AintT, + 'NATMC' = NATMC, + naint_t = NAintT}, + Op, B, TypeCasts) -> + op_call(Op, + DW, + [RetVar, " =", + case TypeCasts of + true -> [" (", AintT, ")"]; + false -> "" + end], + ["ETHR_", NATMC, "_FUNC__(", opstr(native(Op)), op_barrier_ext(B), ")"], + Arg1, + Arg2, + Arg3, + case TypeCasts of + true -> [" (", NAintT, ")"]; + false -> "" + end). + +simple_fallback(#atomic_context{arg1 = Arg1, + arg2 = Arg2, + 'ATMC' = ATMC}, + init, B) -> %% Also double word + [" ETHR_", ATMC, "_FUNC__(set", op_barrier_ext(B),")(", Arg1, ", ", Arg2, ");\n"]; +simple_fallback(#atomic_context{dw = false, + arg1 = Arg1, + arg2 = Arg2, + 'ATMC' = ATMC}, + set, B) -> + [" (void) ETHR_", ATMC, "_FUNC__(xchg", op_barrier_ext(B),")(", Arg1, ", ", Arg2, ");\n"]; +simple_fallback(#atomic_context{dw = false, + arg1 = Arg1, + arg2 = Arg2, + 'ATMC' = ATMC}, + add, B) -> + [" (void) ETHR_", ATMC, "_FUNC__(add_read", op_barrier_ext(B), ")(", Arg1, ", ", Arg2, ");\n"]; +simple_fallback(#atomic_context{dw = false, + ret_var = RetVar, + arg1 = Arg1, + aint_t = AintT, + 'ATMC' = ATMC}, + inc_read, B) -> + [" ", RetVar, " = ETHR_", ATMC, "_FUNC__(add_read", op_barrier_ext(B), ")(", Arg1, ", (", AintT,") 1);\n"]; +simple_fallback(#atomic_context{dw = false, + ret_var = RetVar, + arg1 = Arg1, + aint_t = AintT, + 'ATMC' = ATMC}, + dec_read, B) -> + [" ", RetVar, " = ETHR_", ATMC, "_FUNC__(add_read", op_barrier_ext(B), ")(", Arg1, ", (", AintT,") -1);\n"]; +simple_fallback(#atomic_context{dw = false, + arg1 = Arg1, + 'ATMC' = ATMC}, + inc, B) -> + [" (void) ETHR_", ATMC, "_FUNC__(inc_read", op_barrier_ext(B), ")(", Arg1, ");\n"]; +simple_fallback(#atomic_context{dw = false, + arg1 = Arg1, + 'ATMC' = ATMC}, + dec, B) -> + [" (void) ETHR_", ATMC, "_FUNC__(dec_read", op_barrier_ext(B), ")(", Arg1, ");\n"]; +simple_fallback(#atomic_context{dw = false, + unusual_val = UnusualVal, + ret_var = RetVar, + arg1 = Arg1, + aint_t = AintT, + 'ATMC' = ATMC}, + read, B) -> + [" ", RetVar, " = ETHR_", ATMC, "_FUNC__(cmpxchg", op_barrier_ext(B), ")(", Arg1, ", (", AintT, ") ", UnusualVal, ", (", AintT,") ", UnusualVal, ");\n"]; +simple_fallback(#atomic_context{dw = true, + unusual_val = UnusualVal, + arg1 = Arg1, + arg2 = Arg2, + aint_t = AintT, + 'ATMC' = ATMC}, + read, B) -> + [" ", AintT, " tmp; + tmp.", ?DW_SINT_FIELD, "[0] = ", UnusualVal, "; + tmp.", ?DW_SINT_FIELD, "[1] = ", UnusualVal, "; + ", Arg2, "->", ?DW_SINT_FIELD, "[0] = ", UnusualVal, "; + ", Arg2, "->", ?DW_SINT_FIELD, "[1] = ", UnusualVal, "; + (void) ETHR_", ATMC, "_FUNC__(cmpxchg", op_barrier_ext(B), ")(", Arg1, ", &tmp, ", Arg2, "); +" + ]; +simple_fallback(_AC, _Op, _B) -> + []. + +func_header(AC, prototype, MacroName, Op, B) -> + [func_header(AC, implementation, MacroName, Op, B), ";"]; +func_header(#atomic_context{'ATMC' = ATMC} = AC, inline_implementation, _MacroName, Op, B) -> + do_func_header(AC, Op, "static ETHR_INLINE ", + ["ETHR_", ATMC, "_FUNC__(", opstr(Op), op_barrier_ext(B), ")"]); +func_header(#atomic_context{atomic = Atomic} = AC, implementation, false, Op, B) -> + do_func_header(AC, Op, "", [Atomic, "_", opstr(Op), op_barrier_ext(B)]); +func_header(AC, implementation, MacroName, Op, B) -> + do_func_header(AC, Op, "", [MacroName, "(", opstr(Op), op_barrier_ext(B), ")"]). + + +do_func_header(#atomic_context{atomic_t = AtomicT, + addr_aint_t = AddrAintT, + arg1 = Arg1}, + addr, Inline, Func) -> + [Inline, AddrAintT, " *", Func, "(", AtomicT, " *", Arg1, ")"]; +do_func_header(#atomic_context{dw = false, + atomic_t = AtomicT, + aint_t = AintT, + arg1 = Arg1, + arg2 = Arg2}, + Op, Inline, Func) when Op == init; + Op == set; + Op == add -> + [Inline, "void ", Func, "(", AtomicT, " *", Arg1, ", ", AintT, " ", Arg2, ")"]; +do_func_header(#atomic_context{dw = false, + atomic_t = AtomicT, + arg1 = Arg1}, + Op, Inline, Func) when Op == inc; + Op == dec -> + [Inline, "void ", Func, "(", AtomicT, " *", Arg1, ")"]; +do_func_header(#atomic_context{dw = false, + atomic_t = AtomicT, + aint_t = AintT, + arg1 = Arg1}, + Op, Inline, Func) when Op == read; + Op == inc_read; + Op == dec_read -> + [Inline, AintT, " ", Func, "(", AtomicT, " *", Arg1, ")"]; +do_func_header(#atomic_context{dw = false, + atomic_t = AtomicT, + aint_t = AintT, + arg1 = Arg1, + arg2 = Arg2}, + Op, Inline, Func) when Op == add_read; + Op == read_band; + Op == read_bor; + Op == xchg -> + [Inline, AintT, " ", Func, "(", AtomicT, " *", Arg1, ", ", AintT, " ", Arg2, ")"]; +do_func_header(#atomic_context{dw = false, + atomic_t = AtomicT, + aint_t = AintT, + arg1 = Arg1, + arg2 = Arg2, + arg3 = Arg3}, + cmpxchg, Inline, Func) -> + [Inline, AintT, " ", Func, "(", AtomicT, " *", Arg1, ", ", AintT, " ", Arg2, ", ", AintT, " ", Arg3, ")"]; +do_func_header(#atomic_context{dw = true, + atomic_t = AtomicT, + aint_t = AintT, + arg1 = Arg1, + arg2 = Arg2}, + Op, Inline, Func) when Op == init; + Op == set; + Op == read -> + [Inline, "void ", Func, "(", AtomicT, " *", Arg1, ", ", AintT, " *", Arg2, ")"]; +do_func_header(#atomic_context{dw = true, + atomic_t = AtomicT, + aint_t = AintT, + arg1 = Arg1, + arg2 = Arg2, + arg3 = Arg3}, + cmpxchg, Inline, Func) -> + [Inline, "int ", Func, "(", AtomicT, " *", Arg1, ", ", AintT, " *", Arg2, ", ", AintT, " *", Arg3, ")"]. + + +xbarriers(_Op, none, _NB) -> + {"", ""}; + +xbarriers(_Op, acqb, NB) when NB == acqb; NB == mb -> + {"", ""}; +xbarriers(Op, acqb, NB) -> + case {op_head_tail(Op), NB} of + {{_, load}, rb} -> {"", "ETHR_MEMBAR(ETHR_LoadStore);"}; + {{_, load}, _} -> {"", "ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);"}; + {{_, store}, _} -> {"", "ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);"}; + {_, rb} -> {"", "ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);"}; + _ -> {"", "ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);"} + end; + +xbarriers(_Op, relb, NB) when NB == relb; NB == mb -> + {"", ""}; +xbarriers(Op, relb, NB) -> + case {op_head_tail(Op), NB} of + {{store, _}, wb} -> {"ETHR_MEMBAR(ETHR_LoadStore);", ""}; + {{store, _}, _} -> {"ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);", ""}; + {{load, _}, _} -> {"ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);", ""}; + {_, wb} -> {"ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad);", ""}; + _ -> {"ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);", ""} + end; + +xbarriers(_Op, wb, NB) when NB == wb; NB == mb -> + {"", ""}; +xbarriers(_Op, wb, _NB) -> + {"ETHR_MEMBAR(ETHR_StoreStore);", ""}; + +xbarriers(_Op, rb, NB) when NB == rb; NB == mb -> + {"", ""}; +xbarriers(_Op, rb, _NB) -> + {"", "ETHR_MEMBAR(ETHR_LoadLoad);"}; + +xbarriers(_Op, mb, mb) -> + {"", ""}; +xbarriers(Op, mb, NB) -> + MB = "ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);", + {Head, Tail} = op_head_tail(Op), + PreOp = case {Head, NB} of + {_, relb} -> ""; + {store, wb} -> "ETHR_MEMBAR(ETHR_LoadStore);"; + {store, _} -> "ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);"; + {load, _} -> "ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);"; + {_, wb} -> "ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad);"; + _ -> MB + end, + PostOp = case {Tail, NB} of + {_, acqb} -> ""; + {load, rb} -> "ETHR_MEMBAR(ETHR_LoadStore);"; + {load, _} -> "ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);"; + {store, _} -> "ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);"; + {_, rb} -> "ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);"; + _ -> MB + end, + {PreOp, PostOp}. + +try_barrier_order_first(none) -> + [none, rb, wb, acqb, relb]; +try_barrier_order_first(acqb) -> + [acqb, rb, none, mb]; +try_barrier_order_first(relb) -> + [relb, wb, none, mb]; +try_barrier_order_first(rb) -> + [rb, none, mb]; +try_barrier_order_first(wb) -> + [wb, none, mb]; +try_barrier_order_first(mb) -> + [mb, relb, acqb, wb, rb, none]. + +try_barrier_order(B) -> + First = try_barrier_order_first(B), + First ++ (?BARRIERS -- First). + +native_barrier_op(#atomic_context{'NATMC' = NATMC} = AC, If, ExtraDecl, Op, B, NB, TypeCasts) -> + NOpStr = opstr(native(Op)), + CapNOpStr = to_upper(NOpStr), + NBExt = op_barrier_ext(NB), + CapNBExt = to_upper(NBExt), + {PreB, PostB} = xbarriers(Op, B, NB), + [If, " defined(ETHR_HAVE_", NATMC, "_", CapNOpStr, CapNBExt, ")\n", + ExtraDecl, + case PreB of + "" -> ""; + _ -> [" ", PreB, "\n"] + end, + " ", native_op_call(AC, Op, NB, TypeCasts), "\n", + case PostB of + "" -> ""; + _ -> [" ", PostB, "\n"] + end]. + +dw_native_barrier_op(#atomic_context{arg1 = Arg1, arg2 = Arg2, arg3 = Arg3} = AC, If, ExtraDecl, Op, B, NB) -> + native_barrier_op(AC#atomic_context{arg1 = ["&", Arg1, "->native"], + arg2 = [Arg2, "->", ?DW_SINT_FIELD], + arg3 = [Arg3, "->", ?DW_SINT_FIELD]}, + If, ExtraDecl, Op, B, NB, false). + +su_dw_native_barrier_op(#atomic_context{dw = true, + naint_t = NAintT, + ret_var = RetVar, + arg1 = Arg1, + arg2 = Arg2, + arg3 = Arg3, + 'NATMC' = NATMC} = AC, If, cmpxchg, B, NB) -> + SU = ["->", ?SU_DW_SINT_FIELD], + TmpVar = "act", + SUArg1 = ["&", Arg1, "->native"], + SUArg2 = [Arg2, SU], + SUArg3 = [Arg3, SU], + ExtraDecl = [" ", NAintT, " ", TmpVar, ";\n"], + [native_barrier_op(AC#atomic_context{dw = false, + ret_var = TmpVar, + arg1 = SUArg1, + arg2 = SUArg2, + arg3 = SUArg3, + 'NATMC' = ["SU_", NATMC]}, + If, ExtraDecl, cmpxchg, B, NB, false), + " ", RetVar, " = (", TmpVar, " == ", SUArg3, "); + ", SUArg3, " = ", TmpVar, "; +" + ]; +su_dw_native_barrier_op(#atomic_context{dw = true, + arg1 = Arg1, + arg2 = Arg2, + 'NATMC' = NATMC} = AC, If, Op, B, NB) -> + SUArg1 = ["&", Arg1, "->native"], + SUArg2 = [Arg2, "->", ?SU_DW_SINT_FIELD], + native_barrier_op(AC#atomic_context{dw = false, + ret_var = SUArg2, + arg1 = SUArg1, + arg2 = SUArg2, + arg3 = not_used, + 'NATMC' = ["SU_", NATMC]}, If, "", Op, B, NB, false). + +cmpxchg_fallback_define(#atomic_context{dw = false, aint_t = AintT} = AC) -> + do_cmpxchg_fallback_define(AC, true, AintT); +cmpxchg_fallback_define(#atomic_context{dw = true, + 'NATMC' = NATMC, + naint_t = NAintT} = AC) -> + ["\n\n#if defined(ETHR_HAVE_NATIVE_DW_ATOMIC)\n", + do_cmpxchg_fallback_define(AC, false, not_used), + "\n\n#elif defined(ETHR_HAVE_NATIVE_SU_DW_ATOMIC)\n", + do_cmpxchg_fallback_define(AC#atomic_context{'NATMC' = ["SU_", NATMC], + naint_t = NAintT}, + true, + NAintT), + " + +#else +# error \"?!?\" +#endif +"]. + +do_cmpxchg_fallback_define(#atomic_context{'NATMC' = NATMC, + aint_t = AintT, + naint_t = NAintT}, + SU, SUType) -> + + ReadFunc = fun (IF) -> + fun (B) -> + BExt = op_barrier_ext(B), + CapBExt = to_upper(BExt), + [IF, " defined(ETHR_HAVE_", NATMC, "_READ", CapBExt, ")", + case SU of + true -> [" +#define ETHR_", NATMC, "_CMPXCHG_FALLBACK_READ__(VAR) \\ + ETHR_", NATMC, "_FUNC__(read", BExt, ")(VAR) +" + ]; + false -> [" +#define ETHR_", NATMC, "_CMPXCHG_FALLBACK_READ__(VAR, VAL) \\ + ETHR_", NATMC, "_FUNC__(read", BExt, ")(VAR, VAL) +#elif defined(ETHR_HAVE_SU_", NATMC, "_READ", CapBExt, ") +#define ETHR_", NATMC, "_CMPXCHG_FALLBACK_READ__(VAR, VAL) \\ + VAL.", ?SU_DW_SINT_FIELD, " = ETHR_SU_", NATMC, "_FUNC__(read", BExt, ")(VAR) +" + ] + end] + end + end, + NotDefCMPXCHG = fun (B) -> + CapBExt = to_upper(op_barrier_ext(B)), + ["!defined(ETHR_HAVE_", NATMC, "_CMPXCHG", CapBExt, ")"] + end, + NoneTryBarrierOrder = try_barrier_order(none), + %% First a sanity check + [" +#if (", NotDefCMPXCHG(hd(?BARRIERS)) , + lists:map(fun (B) -> + [" \\ + && ", NotDefCMPXCHG(B)] + end, + tl(?BARRIERS)), ") +# error \"No native cmpxchg() op available\" +#endif + + +/* + * Read op used together with cmpxchg() fallback when no native op present. + */ +", + + %% Read op to use with cmpxchg fallback + (ReadFunc("#if"))(hd(NoneTryBarrierOrder)), + lists:map(ReadFunc("#elif"), tl(NoneTryBarrierOrder)), +"#else +/* + * We have no native read() op; guess zero and then use the + * the atomics actual value returned from cmpxchg(). + */", + case SU of + true -> [" +#define ETHR_", NATMC, "_CMPXCHG_FALLBACK_READ__(VAR) \\ + ((", NAintT, ") 0)"]; + false -> [" +#define ETHR_", NATMC, "_CMPXCHG_FALLBACK_READ__(VAR, VAL) \\ +do { \\ + VAL.", ?DW_SINT_FIELD, "[0] = (ethr_sint_t) 0; \\ + VAL.", ?DW_SINT_FIELD, "[1] = (ethr_sint_t) 0; \\ +} while (0)"] + end, " +#endif +", + + %% The fallback + " +/* + * Native cmpxchg() fallback used when no native op present. + */ +#define ETHR_", NATMC, "_CMPXCHG_FALLBACK__(CMPXCHG, VAR, AVAL, OPS) \\ +do { \\", + case SU of + true -> [" + ", SUType, " AVAL; \\ + ", NAintT, " new__, act__, exp__; \\ + act__ = ETHR_", NATMC, "_CMPXCHG_FALLBACK_READ__(VAR); \\ + do { \\ + exp__ = act__; \\ + AVAL = (", SUType, ") act__; \\ + { OPS; } \\ + new__ = (", NAintT, ") AVAL; \\ + act__ = CMPXCHG(VAR, new__, exp__); \\ + } while (__builtin_expect(act__ != exp__, 0)); \\"]; + false -> [" + int res__; \\ + ", AintT, " AVAL, exp_act__; \\ + ETHR_", NATMC, "_CMPXCHG_FALLBACK_READ__(VAR, exp_act__); \\ + do { \\ + AVAL.", ?DW_SINT_FIELD, "[0] = exp_act__.", ?DW_SINT_FIELD, "[0]; \\ + AVAL.", ?DW_SINT_FIELD, "[1] = exp_act__.", ?DW_SINT_FIELD, "[1]; \\ + { OPS; } \\ + res__ = CMPXCHG(VAR, AVAL.", ?DW_SINT_FIELD, ", exp_act__.", ?DW_SINT_FIELD, "); \\ + } while (__builtin_expect(res__ == 0, 0)); \\"] + end, " +} while (0) +" + ]. + +cmpxchg_fallbacks(#atomic_context{}, _SUDW, cmpxchg, _B) -> + ""; %% No need for a fallback +cmpxchg_fallbacks(#atomic_context{dw = DW, + ret_var = RetVar, + arg1 = Arg1, + arg2 = Arg2, + arg3 = Arg3, + 'NATMC' = NATMC}, + SUDW, Op, B) -> + Operation = case DW of + false -> + op(Op, #op_context{ret = RetVar, + var = "aval", + val1 = Arg2, + val2 = Arg3}); + true -> + case SUDW of + true -> + op(Op, #op_context{ret = [Arg2, "->", ?SU_DW_SINT_FIELD], + var = "aval", + val1 = [Arg2, "->", ?SU_DW_SINT_FIELD]}); + false -> + dw_op(Op, #op_context{ret = RetVar, + var = ["aval.", ?DW_SINT_FIELD], + val1 = [Arg2, "->", ?DW_SINT_FIELD]}) + end + end, + [lists:map(fun (NB) -> + NativeVar = case DW of + true -> ["&", Arg1, "->native"]; + false -> Arg1 + end, + NBExt = op_barrier_ext(NB), + CapNBExt = to_upper(NBExt), + {PreB, PostB} = xbarriers(cmpxchg, B, NB), + ["#elif defined(ETHR_HAVE_", NATMC, "_CMPXCHG", CapNBExt, ")\n", + case PreB of + "" -> ""; + _ -> [" ", PreB, "\n"] + end, + " ETHR_", NATMC, "_CMPXCHG_FALLBACK__(ETHR_", NATMC, "_FUNC__(cmpxchg", NBExt, "), ", NativeVar, ", aval, ", Operation, ");\n", + case PostB of + "" -> ""; + _ -> [" ", PostB, "\n"] + end] + end, + try_barrier_order(B))]. + +translate_have_defs(#atomic_context{dw = DW, 'NATMC' = NATMC}) -> + [" +#if !defined(ETHR_", NATMC, "_BITS__) +# error \"Missing native atomic implementation\"", + lists:map(fun (NBits) -> + {HaveInPrefix, + HaveOutPrefix, + HaveInPrefixExtra, + HaveOutPrefixExtra, + NativeTypeCheck} = case NBits of + "dw" -> + {"ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC", + ["ETHR_HAVE_", NATMC], + "ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC", + ["ETHR_HAVE_SU_", NATMC], + "\n#elif defined(ETHR_HAVE_NATIVE_DW_ATOMIC) || defined(ETHR_HAVE_NATIVE_SU_DW_ATOMIC)"}; + _ -> + {[?HAVE_NATIVE_ATOMIC, NBits], + case DW of + true -> ["ETHR_HAVE_SU_", NATMC]; + false -> ["ETHR_HAVE_", NATMC] + end, + false, + ["ETHR_HAVE_", NATMC], + ["\n#elif ETHR_", NATMC, "_BITS__ == ", NBits]} + end, + [NativeTypeCheck, + lists:map(fun (Op) -> + NOpStr = opstr(native(Op)), + CapNOpStr = to_upper(NOpStr), + lists:map(fun (B) -> + NBExt = op_barrier_ext(B), + CapNBExt = to_upper(NBExt), + HaveOutDef = [HaveOutPrefix, "_", CapNOpStr, CapNBExt], + HaveOutDefExtra = [HaveOutPrefixExtra, "_", CapNOpStr, CapNBExt], + [case DW of + true -> + ["\n# undef ", HaveOutDefExtra]; + false -> + "" + end, " +# undef ", HaveOutDef," +# ifdef ", HaveInPrefix, "_", CapNOpStr, CapNBExt, " +# define ", HaveOutDef, " 1 +# endif", + case HaveInPrefixExtra of + false -> ""; + _ -> [" +# ifdef ", HaveInPrefixExtra, "_", CapNOpStr, CapNBExt, " +# define ", HaveOutDefExtra, " 1 +# endif" + ] + end] + end, + ?BARRIERS) + end, + case DW of + true -> ?DW_ATOMIC_OPS; + false -> ?ATOMIC_OPS + end)] + end, + case DW of + true -> ["dw", "64"]; + false -> ?POTENTIAL_NBITS + end), + " +#else +# error \"Invalid native atomic size\" +#endif +"]. + + + +make_prototypes(#atomic_context{dw = DW, 'ATMC' = ATMC} = AC) -> + MkProt = fun (MacroName) -> + %% addr() is special + [func_header(AC, prototype, MacroName, addr, none), "\n", + lists:map(fun (Op) -> + lists:map(fun (B) -> + [func_header(AC, prototype, MacroName, Op, B), "\n"] + end, + ?BARRIERS) + end, + case DW of + true -> ?DW_ATOMIC_OPS; + false -> ?ATOMIC_OPS + end)] + end, + [" +#ifdef ETHR_NEED_", ATMC, "_PROTOTYPES__ +", + MkProt(false), + case DW of + true -> ["#if defined(", ?DW_RTCHK_MACRO, ")\n", + MkProt(?DW_FUNC_MACRO), + "#endif\n"]; + false -> "" + end, + "#endif /* ETHR_NEED_", ATMC, "_PROTOTYPES__ */\n"]. + +rtchk_fallback_call(Return, #atomic_context{dw = DW, + ret_var = RetVar, + arg1 = Arg1, + arg2 = Arg2, + arg3 = Arg3}, + Op, B) -> + op_call(Op, DW, case Return of + true -> "return"; + false -> [RetVar, " ="] + end, [?DW_FUNC_MACRO, "(", opstr(Op), op_barrier_ext(B), ")"], Arg1, Arg2, Arg3, ""). + +make_implementations(#atomic_context{dw = DW, + ret_type = RetType, + ret_var = RetVar, + arg1 = Arg1, + addr_aint_t = AddrAintT, + atomic = Atomic, + have_native_atomic_ops = HaveNativeAtomicOps, + 'ATMC' = ATMC, + 'NATMC' = NATMC} = AC) -> + NativeVar = case DW of + true -> ["(&", Arg1, "->native)"]; + false -> Arg1 + end, + RtchkBegin = [" +#if defined(", ?DW_RTCHK_MACRO, ") + if (", ?DW_RTCHK_MACRO, ") { +#endif +"], + RtchkEnd = fun (Return, Operation, Barrier) -> + [" +#if defined(", ?DW_RTCHK_MACRO, ") + } else { ", rtchk_fallback_call(Return, AC, Operation, Barrier), " } +#endif\n" + ] + end, + [" +#if (defined(", HaveNativeAtomicOps, ") \\ + && (defined(ETHR_", ATMC, "_INLINE__) || defined(ETHR_ATOMIC_IMPL__))) +", + translate_have_defs(AC), + cmpxchg_fallback_define(AC), + %% addr() is special + " + + +/* --- addr() --- */ + +", func_header(AC, inline_implementation, false, addr, none), " +{", case DW of + true -> RtchkBegin; + false -> "" + end, " + return (", AddrAintT, " *) ETHR_", NATMC, "_ADDR_FUNC__(", NativeVar, "); +",case DW of + true -> RtchkEnd(true, addr, none); + false -> "" + end, " +} +", + lists:map(fun (Op) -> + OpStr = opstr(Op), + [" + +/* --- ", OpStr, "() --- */ + +", + lists:map(fun (B) -> + TryBarriers = try_barrier_order(B), + [" +", func_header(AC, inline_implementation, false, Op, B), " +{ +", + case is_return_op(AC, Op) of + true -> + [" ", RetType, " ", RetVar, ";\n"]; + _ -> "" + end, + case DW of + true -> + [RtchkBegin, + "\n", + su_dw_native_barrier_op(AC, "#if", Op, B, hd(TryBarriers)), + lists:map(fun (NB) -> + su_dw_native_barrier_op(AC, "#elif", Op, B, NB) + end, + tl(TryBarriers)), + lists:map(fun (NB) -> + dw_native_barrier_op(AC, "#elif", "", Op, B, NB) + end, + TryBarriers), + case simple_fallback(AC, Op, B) of + "" -> + %% No simple fallback available; + %% use cmpxchg() fallbacks... + [cmpxchg_fallbacks(AC#atomic_context{'NATMC' = ["SU_", NATMC]}, true, Op, B), + cmpxchg_fallbacks(AC, false, Op, B), + "#else +#error \"Missing implementation of ", Atomic, "_", opstr(Op), op_barrier_ext(B), "()!\" +#endif +" + ]; + SimpleFallback -> + ["#else\n", SimpleFallback, "#endif\n"] + end, + RtchkEnd(false, Op, B), "\n"]; + false -> + [native_barrier_op(AC, "#if", "", Op, B, hd(TryBarriers), true), + lists:map(fun (NB) -> + native_barrier_op(AC, "#elif", "", Op, B, NB, true) + end, + tl(TryBarriers)), + case simple_fallback(AC, Op, B) of + "" -> + %% No simple fallback available; + %% use cmpxchg() fallbacks... + [cmpxchg_fallbacks(AC, false, Op, B), + "#else +#error \"Missing implementation of ", Atomic, "_", opstr(Op), op_barrier_ext(B), "()!\" +#endif +" + ]; + SimpleFallback -> + ["#else\n", SimpleFallback, "#endif\n"] + end] + end, + case is_return_op(AC, Op) of + true -> + [" return ", RetVar, ";\n"]; + false -> + "" + end, + "}\n"] + end, + ?BARRIERS)] + end, + case DW of + true -> ?DW_ATOMIC_OPS; + false -> ?ATOMIC_OPS + end), + " +#endif /* ETHR_", ATMC, "_INLINE__ */ +" + ]. + +atomic_implementation_comment(AtomicSize) -> + CSz = case AtomicSize of + "dword" -> "Double word size"; + "word" -> "Word size"; + _ -> AtomicSize ++ "-bit" + end, + [" + +/* ---------- ", CSz, " atomic implementation ---------- */ + +" + ]. + +write_h_file(FileName) -> + {ok, FD} = file:open(FileName, [write, latin1]), + ok = file:write(FD, comments()), + ok = file:write(FD, " +#ifndef ETHR_ATOMICS_H__ +#define ETHR_ATOMICS_H__ +" + ), + ok = file:write(FD, h_top()), + ok = lists:foreach(fun (AtomicSize) -> + AC = atomic_context(AtomicSize), + ok = file:write(FD, + [atomic_implementation_comment(AtomicSize), + make_prototypes(AC), + make_implementations(AC)]) + end, + ?ATOMIC_SIZES), + ok = file:write(FD, " +#endif /* ETHR_ATOMICS_H__ */ +" + ), + ok = file:close(FD). + + +make_native_impl_op(#atomic_context{dw = DW, + atomic = Atomic, + have_native_atomic_ops = HaveNativeAtomicOps, + ret_var = RetVar, + arg1 = Arg1, + arg2 = Arg2, + arg3 = Arg3}, Op, B) -> + ["#if defined(", HaveNativeAtomicOps, ")", + case DW of + true -> [" && !defined(", ?DW_RTCHK_MACRO, ")"]; + false -> "" + end, + "\n", + " ", op_call(Op, DW, [RetVar, " = "], [Atomic, "_", opstr(Op), op_barrier_ext(B), "__"], Arg1, Arg2, Arg3, ""), + "\n"]. + +amc_op_dw_arg(#atomic_context{dw = false}) -> + "0"; +amc_op_dw_arg(#atomic_context{dw = true}) -> + "1". + +amc_op_arg_prefix(#atomic_context{dw = false}) -> + "&"; +amc_op_arg_prefix(#atomic_context{dw = true}) -> + "". + +amc_sint_arg(#atomic_context{dw = DW, arg2 = Arg}, arg2) -> + amc_sint_arg(DW, Arg); +amc_sint_arg(#atomic_context{dw = DW, arg3 = Arg}, arg3) -> + amc_sint_arg(DW, Arg); +amc_sint_arg(#atomic_context{dw = DW, ret_var = Arg}, ret_var) -> + amc_sint_arg(DW, Arg); +amc_sint_arg(true, Arg) -> + [Arg, "->" ?DW_SINT_FIELD]; +amc_sint_arg(false, Arg) -> + ["&", Arg]. + +amc_op_call(#atomic_context{arg1 = Arg1} = AC, init) -> + [" amc_init(&", Arg1, "->amc, ", amc_op_dw_arg(AC), ", ", amc_op_arg_prefix(AC), Arg1, "->sint, ", amc_sint_arg(AC, arg2), ");\n"]; +amc_op_call(#atomic_context{arg1 = Arg1} = AC, set) -> + [" amc_set(&", Arg1, "->amc, ", amc_op_dw_arg(AC), ", ", amc_op_arg_prefix(AC), Arg1, "->sint, ", amc_sint_arg(AC, arg2), ");\n"]; +amc_op_call(#atomic_context{dw = false, arg1 = Arg1} = AC, read) -> + [" amc_read(&", Arg1, "->amc, ", amc_op_dw_arg(AC), ", ", amc_op_arg_prefix(AC), Arg1, "->sint, ", amc_sint_arg(AC, ret_var), ");\n"]; +amc_op_call(#atomic_context{dw = true, arg1 = Arg1} = AC, read) -> + [" amc_read(&", Arg1, "->amc, ", amc_op_dw_arg(AC), ", ", amc_op_arg_prefix(AC), Arg1, "->sint, ", amc_sint_arg(AC, arg2), ");\n"]; +amc_op_call(#atomic_context{dw = false, arg1 = Arg1, arg3 = Arg3, ret_var = RetVar} = AC, cmpxchg) -> + [" ", RetVar, " = ", Arg3, "; + (void) amc_cmpxchg(&", Arg1, "->amc, ", amc_op_dw_arg(AC), ", ", amc_op_arg_prefix(AC), Arg1, "->sint, ", amc_sint_arg(AC, arg2), ", ", amc_sint_arg(AC, ret_var), ");\n"]; +amc_op_call(#atomic_context{dw = true, arg1 = Arg1, ret_var = RetVar} = AC, cmpxchg) -> + [" ", RetVar, " = amc_cmpxchg(&", Arg1, "->amc, ", amc_op_dw_arg(AC), ", ", amc_op_arg_prefix(AC), Arg1, "->sint, ", amc_sint_arg(AC, arg2), ", ", amc_sint_arg(AC, arg3), ");\n"]; +amc_op_call(#atomic_context{dw = DW, arg1 = Arg1, arg2 = Arg2, arg3 = Arg3, ret_var = RetVar}, Op) -> + OpCtxt = #op_context{ret = RetVar, var = [Arg1,"->sint"], val1 = Arg2, val2 = Arg3}, + OpStr = case DW of + true -> dw_op(Op, OpCtxt); + false -> op(Op, OpCtxt) + end, + [" ETHR_AMC_MODIFICATION_OPS__(&", Arg1, "->amc, ", OpStr, ");\n"]. + +make_amc_fallback_op(#atomic_context{amc_fallback = false}, _Op, _B) -> + ""; +make_amc_fallback_op(#atomic_context{amc_fallback = true} = AC, Op, B) -> + NB = case Op of + read -> rb; + _ -> none + end, + {PreB, PostB} = xbarriers(Op, B, NB), + ["#elif defined(ETHR_AMC_FALLBACK__)\n", + case PreB of + "" -> ""; + _ -> [" ", PreB, "\n"] + end, + amc_op_call(AC, Op), + case PostB of + "" -> ""; + _ -> [" ", PostB, "\n"] + end]. + +make_locked_fallback_op(#atomic_context{dw = DW, + ret_var = RetVar, + arg1 = Arg1, + arg2 = Arg2, + arg3 = Arg3}, Op, B) -> + OpStr = case DW of + true -> + dw_op(Op, #op_context{ret = RetVar, + var = [Arg1, "->" ?DW_SINT_FIELD], + val1 = [Arg2, "->" ?DW_SINT_FIELD], + val2 = [Arg3, "->" ?DW_SINT_FIELD]}); + false -> + op(Op, #op_context{ret = RetVar, + var = ["*", Arg1], + val1 = Arg2, + val2 = Arg3}) + end, + {PreB, PostB} = xbarriers(Op, B, none), + ["#else\n", + case PreB of + "" -> ""; + _ -> [" ", PreB, "\n"] + end, + [" ETHR_ATOMIC_OP_FALLBACK_IMPL__(", Arg1, ", ", OpStr, ");\n"], + case PostB of + "" -> ""; + _ -> [" ", PostB, "\n"] + end, + "#endif\n"]. + +make_symbol_to_fallback_impl(#atomic_context{dw = true, + atomic = Atomic, + arg1 = Arg1, + arg2 = Arg2, + arg3 = Arg3} = AC, + Op, B) -> + [" +#ifdef ", ?DW_RTCHK_MACRO, " +", func_header(AC, implementation, false, Op, B), " +{", + case Op of + init -> ""; + _ -> ["\n ETHR_ASSERT(!ethr_not_inited__);"] + end, " + ETHR_ASSERT(", Arg1, "); + ", op_call(Op, true, "return", [Atomic, "_", opstr(Op), op_barrier_ext(B), "__"], Arg1, Arg2, Arg3, ""), " +} +#endif +" + ]; +make_symbol_to_fallback_impl(_, _, _) -> + "". + +make_symbol_implementations(#atomic_context{dw = DW, + amc_fallback = AMC, + ret_type = RetType, + addr_aint_t = AddrAintT, + ret_var = RetVar, + arg1 = Arg1} = AC) -> + FallbackVar = case DW of + true -> ["(&", Arg1, "->fallback)"]; + false -> Arg1 + end, + [" +", + case DW of + true -> [" +/* + * Double word atomics need runtime test. + */ + +int ethr_have_native_dw_atomic(void) +{ + return ethr_have_native_dw_atomic__(); +} + "]; + false -> "" + end, " + +/* --- addr() --- */ + +", func_header(AC, implementation, + case DW of + true -> ?DW_FUNC_MACRO; + false -> false + end, addr, none), " +{ + ", AddrAintT, " *", RetVar, "; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(", Arg1, "); +", make_native_impl_op(AC, addr, none), + case AMC of + true -> ["#elif defined(ETHR_AMC_FALLBACK__) + ", RetVar ," = (", AddrAintT, " *) (", FallbackVar, ")->sint;"]; + false -> "" + end, " +#else + ", RetVar, " = (", AddrAintT, " *) ", FallbackVar, "; +#endif + return ", RetVar, "; +} +", + make_symbol_to_fallback_impl(AC, addr, none), + lists:map(fun (Op) -> + [" + +/* -- ", opstr(Op), "() -- */ + +", + lists:map(fun (B) -> + ["\n", + func_header(AC, implementation, + case DW of + true -> ?DW_FUNC_MACRO; + false -> false + end, Op, B), + "\n{\n", + case is_return_op(AC, Op) of + true -> [" ", RetType, " ", RetVar, ";\n"]; + false -> "" + end, + case Op of + init -> ""; + _ -> [" ETHR_ASSERT(!ethr_not_inited__);\n"] + end, + [" ETHR_ASSERT(", Arg1, ");\n"], + make_native_impl_op(AC, Op, B), + make_amc_fallback_op(AC#atomic_context{arg1 = FallbackVar}, Op, B), + make_locked_fallback_op(AC#atomic_context{arg1 = FallbackVar}, Op, B), + case is_return_op(AC, Op) of + true -> [" return ", RetVar, ";" + ]; + false -> + "" + end, + "\n}\n", + make_symbol_to_fallback_impl(AC, Op, B)] + end, + ?BARRIERS)] + end, + case DW of + true -> ?DW_ATOMIC_OPS; + false -> ?ATOMIC_OPS + end)]. + +make_info_functions() -> + [" + + +/* --------- Info functions --------- */ + +#if defined(", ?DW_RTCHK_MACRO, ") +char *zero_ops[] = {NULL}; +#endif +", + [lists:map(fun (NBits) -> + {DW, Bits} = case NBits of + "su_dw" -> {"su_dw_", ""}; + "dw" -> {"dw_", ""}; + _ -> {"", NBits} + end, + [" + +static char *native_", DW, "atomic", Bits, "_ops[] = {", + lists:map(fun (Op) -> + NOpStr = opstr(native(Op)), + CapNOpStr = to_upper(NOpStr), + lists:map(fun (B) -> + HaveNative = case NBits of + "dw" -> + "ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC"; + "su_dw" -> + "ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC"; + _ -> + [?HAVE_NATIVE_ATOMIC, NBits] + end, + NBExt = op_barrier_ext(B), + CapNBExt = to_upper(NBExt), + [" +#ifdef ", HaveNative, "_", CapNOpStr, CapNBExt, " + \"", NOpStr, NBExt, "\", +#endif" + ] + end, + ?BARRIERS) + end, + case NBits of + "dw" -> ?DW_ATOMIC_OPS; + "su_dw" -> ?DW_ATOMIC_OPS; + _ -> ?ATOMIC_OPS + end), " + NULL +}; + +char ** +ethr_native_", DW, "atomic", Bits, "_ops(void) +{ +", + case DW of + "" -> ""; + _ -> [" +#if defined(", ?DW_RTCHK_MACRO, ") + if (!", ?DW_RTCHK_MACRO, ") + return &zero_ops[0]; +#endif" + ] + end, " + return &native_", DW, "atomic", Bits, "_ops[0]; +} +" + ] + end, ["su_dw", "dw" | ?POTENTIAL_NBITS])]]. + +write_c_file(FileName) -> + {ok, FD} = file:open(FileName, [write, latin1]), + ok = file:write(FD, comments()), + ok = file:write(FD, c_top()), + lists:foreach(fun (AtomicSize) -> + ok = file:write(FD, + [atomic_implementation_comment(AtomicSize), + make_symbol_implementations(atomic_context(AtomicSize))]) + end, + ?ATOMIC_SIZES), + ok = file:write(FD, make_info_functions()). + + +main([]) -> + case os:getenv("ERL_TOP") of + false -> + io:format("$ERL_TOP not set!~n", []), + halt(1); + ErlTop -> + HFile = filename:join(ErlTop, ?H_FILE), + WHFile = fun () -> + write_h_file(HFile) + end, + CFile = filename:join(ErlTop, ?C_FILE), + WCFile = fun () -> + write_c_file(CFile) + end, + case erlang:system_info(schedulers_online) of + 1 -> + WHFile(), + WCFile(); + _ -> + {HPid, HMon} = spawn_monitor(WHFile), + {CPid, CMon} = spawn_monitor(WCFile), + receive + {'DOWN', HMon, process, HPid, HReason} -> + normal = HReason + end, + receive + {'DOWN', CMon, process, CPid, CReason} -> + normal = CReason + end + end, + io:format("Wrote: ~s~n", [HFile]), + io:format("Wrote: ~s~n", [CFile]), + init:stop() + end. + +a2l(A) -> + atom_to_list(A). + +opstr(A) -> + a2l(A). + +to_upper([]) -> + []; +to_upper([C|Cs]) when is_list(C) -> + [to_upper(C)|to_upper(Cs)]; +to_upper([C|Cs]) when is_integer(C), 97 =< C, C =< 122 -> + [C-32|to_upper(Cs)]; +to_upper([C|Cs]) -> + [C|to_upper(Cs)]. + + +comments() -> + Years = case erlang:date() of + {2011, _, _} -> "2011"; + {Y, _, _} -> "2011-"++integer_to_list(Y) + end, + ["/* + * --------------- DO NOT EDIT THIS FILE! --------------- + * This file was automatically generated by the + * \$ERL_TOP/erts/lib_src/utils/make_atomics_api script. + * If you need to make changes, edit the script and + * regenerate this file. + * --------------- DO NOT EDIT THIS FILE! --------------- + */ + +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB ", Years, ". All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the \"License\"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an \"AS IS\" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: The ethread atomics API + * Author: Rickard Green + */ + +/* + * This file maps native atomic implementations to ethread + * API atomics. If no native atomic implementation + * is available, a less efficient fallback is used instead. + * The API consists of 32-bit size, word size (pointer size), + * and double word size atomics. + * + * The following atomic operations are implemented for + * 32-bit size, and word size atomics: +", + lists:map(fun (Op) -> + [" * - ", opstr(Op), "\n"] + end, + ?ATOMIC_OPS), + " * + * The following atomic operations are implemented for + * double word size atomics: +", + lists:map(fun (Op) -> + [" * - ", opstr(Op), "\n"] + end, + ?DW_ATOMIC_OPS), + " * + * Appart from a function implementing the atomic operation + * with unspecified memory barrier semantics, there are + * functions implementing each operation with the following + * memory barrier semantics: +", + lists:map(fun (none) -> + ""; + (rb) -> + [" * - rb (read barrier)\n"]; + (wb) -> + [" * - wb (write barrier)\n"]; + (acqb) -> + [" * - acqb (acquire barrier)\n"]; + (relb) -> + [" * - relb (release barrier)\n"]; + (mb) -> + [" * - mb (full memory barrier)\n"]; + (B) -> + [" * - ", a2l(B), "\n"] + end, + ?BARRIERS), + " * + * We implement all of these operation/barrier + * combinations, regardless of whether they are useful + * or not (some of them are useless). + * + * Double word size atomic functions are on the followning + * form: + * ethr_dw_atomic_<OP>[_<BARRIER>] + * + * Word size atomic functions are on the followning + * form: + * ethr_atomic_<OP>[_<BARRIER>] + * + * 32-bit size atomic functions are on the followning + * form: + * ethr_atomic32_<OP>[_<BARRIER>] + * + * Apart from the operation/barrier functions + * described above also 'addr' functions are implemented + * which return the actual memory address used of the + * atomic variable. The 'addr' functions have no barrier + * versions. + * + * The native atomic implementation does not need to + * implement all operation/barrier combinations. + * Functions that have no native implementation will be + * constructed from existing native functionality. These + * functions will perform the wanted operation and will + * produce sufficient memory barriers, but may + * in some cases be less efficient than pure native + * versions. + * + * When we create ethread API operation/barrier functions by + * adding barriers before and after native operations it is + * assumed that: + * - A native read operation begins, and ends with a load. + * - A native set operation begins, and ends with a store. + * - An init operation begins with either a load, or a store, + * and ends with either a load, or a store. + * - All other operations begins with a load, and ends with + * either a load, or a store. + * + * This is the minimum functionality that a native + * implementation needs to provide: + * + * - Functions that need to be implemented: + * + * - ethr_native_[dw_|su_dw_]atomic[BITS]_addr + * - ethr_native_[dw_|su_dw_]atomic[BITS]_cmpxchg[_<BARRIER>] + * (at least one cmpxchg of optional barrier) + * + * - Macros that needs to be defined: + * + * A macro informing about the presence of the native + * implementation: + * + * - ETHR_HAVE_NATIVE_[DW_|SU_DW_]ATOMIC[BITS] + * + * A macro naming (a string constant) the implementation: + * + * - ETHR_NATIVE_[DW_]ATOMIC[BITS]_IMPL + * + * Each implemented native atomic function has to + * be accompanied by a defined macro on the following + * form informing about its presence: + * + * - ETHR_HAVE_ETHR_NATIVE_[DW_|SU_DW_]ATOMIC[BITS]_<OP>[_<BARRIER>] + * + * A (sparc-v9 style) membar macro: + * + * - ETHR_MEMBAR(B) + * + * Which takes a combination of the following macros + * or:ed (using |) together: + * + * - ETHR_LoadLoad + * - ETHR_LoadStore + * - ETHR_StoreLoad + * - ETHR_StoreStore + * + */ +" + ]. + +h_top() -> + [" +#undef ETHR_AMC_FALLBACK__ +#undef ETHR_AMC_NO_ATMCS__ +#undef ETHR_AMC_ATMC_T__ +#undef ETHR_AMC_ATMC_FUNC__ + +/* -- 32-bit atomics -- */ + +#undef ETHR_NAINT32_T__ +#undef ETHR_NATMC32_FUNC__ +#undef ETHR_NATMC32_ADDR_FUNC__ +#undef ETHR_NATMC32_BITS__ +#if defined(ETHR_HAVE_NATIVE_ATOMIC32) +# define ETHR_NEED_NATMC32_ADDR +# define ETHR_NATMC32_ADDR_FUNC__ ethr_native_atomic32_addr +typedef ethr_native_atomic32_t ethr_atomic32_t; +# define ETHR_NAINT32_T__ ethr_sint32_t +# define ETHR_NATMC32_FUNC__(X) ethr_native_atomic32_ ## X +# define ETHR_NATMC32_BITS__ 32 +#elif defined(ETHR_HAVE_NATIVE_ATOMIC64) +# define ETHR_NEED_NATMC64_ADDR +#ifdef ETHR_BIGENDIAN +# define ETHR_NATMC32_ADDR_FUNC__(VAR) \\ + (((ethr_sint32_t *) ethr_native_atomic64_addr((VAR))) + 1) +#else +# define ETHR_NATMC32_ADDR_FUNC__(VAR) \\ + ((ethr_sint32_t *) ethr_native_atomic64_addr((VAR))) +#endif +typedef ethr_native_atomic64_t ethr_atomic32_t; +# define ETHR_NAINT32_T__ ethr_sint64_t +# define ETHR_NATMC32_FUNC__(X) ethr_native_atomic64_ ## X +# define ETHR_NATMC32_BITS__ 64 +#else +/* + * No native atomics usable for 32-bits atomics :( + * Use fallback... + */ +typedef ethr_sint32_t ethr_atomic32_t; +#endif + +#undef ETHR_ATMC32_INLINE__ +#ifdef ETHR_NATMC32_BITS__ +# ifdef ETHR_TRY_INLINE_FUNCS +# define ETHR_ATMC32_INLINE__ +# endif +# define ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS +#endif + +#if !defined(ETHR_ATMC32_INLINE__) || defined(ETHR_ATOMIC_IMPL__) +# define ETHR_NEED_ATMC32_PROTOTYPES__ +#endif + +#ifndef ETHR_INLINE_ATMC32_FUNC_NAME_ +# define ETHR_INLINE_ATMC32_FUNC_NAME_(X) X +#endif + +#undef ETHR_ATMC32_FUNC__ +#define ETHR_ATMC32_FUNC__(X) ETHR_INLINE_ATMC32_FUNC_NAME_(ethr_atomic32_ ## X) + + +/* -- Word size atomics -- */ + +#undef ETHR_NEED_NATMC32_ADDR +#undef ETHR_NEED_NATMC64_ADDR + +#undef ETHR_NAINT_T__ +#undef ETHR_NATMC_FUNC__ +#undef ETHR_NATMC_ADDR_FUNC__ +#undef ETHR_NATMC_BITS__ +#if ETHR_SIZEOF_PTR == 8 && defined(ETHR_HAVE_NATIVE_ATOMIC64) +# ifndef ETHR_NEED_NATMC64_ADDR +# define ETHR_NEED_NATMC64_ADDR +# endif +# define ETHR_NATMC_ADDR_FUNC__ ethr_native_atomic64_addr +typedef ethr_native_atomic64_t ethr_atomic_t; +# define ETHR_NAINT_T__ ethr_sint64_t +# define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X +# define ETHR_NATMC_BITS__ 64 +#elif ETHR_SIZEOF_PTR == 4 && defined(ETHR_HAVE_NATIVE_ATOMIC32) +# ifndef ETHR_NEED_NATMC64_ADDR +# define ETHR_NEED_NATMC32_ADDR +# endif +# define ETHR_NATMC_ADDR_FUNC__ ethr_native_atomic32_addr +typedef ethr_native_atomic32_t ethr_atomic_t; +# define ETHR_NAINT_T__ ethr_sint32_t +# define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X +# define ETHR_NATMC_BITS__ 32 +#elif ETHR_SIZEOF_PTR == 4 && defined(ETHR_HAVE_NATIVE_ATOMIC64) +# ifndef ETHR_NEED_NATMC64_ADDR +# define ETHR_NEED_NATMC64_ADDR +# endif +#ifdef ETHR_BIGENDIAN +# define ETHR_NATMC_ADDR_FUNC__(VAR) \\ + (((ethr_sint32_t *) ethr_native_atomic64_addr((VAR))) + 1) +#else +# define ETHR_NATMC_ADDR_FUNC__(VAR) \\ + ((ethr_sint32_t *) ethr_native_atomic64_addr((VAR))) +#endif +typedef ethr_native_atomic64_t ethr_atomic_t; +# define ETHR_NATMC_T__ ethr_native_atomic64_t +# define ETHR_NAINT_T__ ethr_sint64_t +# define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X +# define ETHR_NATMC_BITS__ 64 +#else +/* + * No native atomics usable for pointer size atomics :( + * Use fallback... + */ + +# if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) +# define ETHR_AMC_FALLBACK__ +# define ETHR_AMC_NO_ATMCS__ 2 +# define ETHR_AMC_SINT_T__ ethr_sint32_t +# define ETHR_AMC_ATMC_T__ ethr_atomic32_t +# define ETHR_AMC_ATMC_FUNC__(X) ETHR_INLINE_ATMC32_FUNC_NAME_(ethr_atomic32_ ## X) +typedef struct { + ETHR_AMC_ATMC_T__ atomic[ETHR_AMC_NO_ATMCS__]; +} ethr_amc_t; +typedef struct { + ethr_amc_t amc; + ethr_sint_t sint; +} ethr_atomic_t; +# else /* locked fallback */ +typedef ethr_sint_t ethr_atomic_t; +# endif +#endif + +#undef ETHR_ATMC_INLINE__ +#ifdef ETHR_NATMC_BITS__ +# ifdef ETHR_TRY_INLINE_FUNCS +# define ETHR_ATMC_INLINE__ +# endif +# define ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS +#endif + +#if !defined(ETHR_ATMC_INLINE__) || defined(ETHR_ATOMIC_IMPL__) +# define ETHR_NEED_ATMC_PROTOTYPES__ +#endif + +#ifndef ETHR_INLINE_ATMC_FUNC_NAME_ +# define ETHR_INLINE_ATMC_FUNC_NAME_(X) X +#endif + +#undef ETHR_ATMC_FUNC__ +#define ETHR_ATMC_FUNC__(X) ETHR_INLINE_ATMC_FUNC_NAME_(ethr_atomic_ ## X) + +/* -- Double word atomics -- */ + +#undef ETHR_SU_DW_NAINT_T__ +#undef ETHR_SU_DW_NATMC_FUNC__ +#undef ETHR_SU_DW_NATMC_ADDR_FUNC__ +#undef ETHR_DW_NATMC_FUNC__ +#undef ETHR_DW_NATMC_ADDR_FUNC__ +#undef ETHR_DW_NATMC_BITS__ +#if defined(ETHR_HAVE_NATIVE_DW_ATOMIC) || defined(ETHR_HAVE_NATIVE_SU_DW_ATOMIC) +# define ETHR_NEED_DW_NATMC_ADDR +# define ETHR_DW_NATMC_ADDR_FUNC__ ethr_native_dw_atomic_addr +# define ETHR_NATIVE_DW_ATOMIC_T__ ethr_native_dw_atomic_t +# define ETHR_DW_NATMC_FUNC__(X) ethr_native_dw_atomic_ ## X +# define ETHR_SU_DW_NATMC_FUNC__(X) ethr_native_su_dw_atomic_ ## X +# if ETHR_SIZEOF_PTR == 8 +# define ETHR_DW_NATMC_BITS__ 128 +# elif ETHR_SIZEOF_PTR == 4 +# define ETHR_DW_NATMC_BITS__ 64 +# else +# error \"Word size not supported\" +# endif +# ifdef ETHR_NATIVE_SU_DW_SINT_T +# define ETHR_SU_DW_NAINT_T__ ETHR_NATIVE_SU_DW_SINT_T +# endif +#elif ETHR_SIZEOF_PTR == 4 && defined(ETHR_HAVE_NATIVE_ATOMIC64) +# define ETHR_HAVE_NATIVE_SU_DW_ATOMIC +# ifndef ETHR_NEED_NATMC64_ADDR +# define ETHR_NEED_NATMC64_ADDR +# endif +# define ETHR_DW_NATMC_ADDR_FUNC__(VAR) \\ + ((ethr_dw_sint_t *) ethr_native_atomic64_addr((VAR))) +# define ETHR_NATIVE_DW_ATOMIC_T__ ethr_native_atomic64_t +# define ETHR_SU_DW_NAINT_T__ ethr_sint64_t +# define ETHR_SU_DW_NATMC_FUNC__(X) ethr_native_atomic64_ ## X +# define ETHR_DW_NATMC_BITS__ 64 +#endif + +#if defined(", ?DW_RTCHK_MACRO, ") +#define ", ?DW_FUNC_MACRO, "(X) ethr_dw_atomic_ ## X ## _fallback__ +#else +#define ", ?DW_FUNC_MACRO, "(X) ethr_dw_atomic_ ## X +#endif + +#if !defined(ETHR_DW_NATMC_BITS__) || defined(", ?DW_RTCHK_MACRO, ") +# define ETHR_NEED_DW_FALLBACK__ +#endif + +#if defined(ETHR_NEED_DW_FALLBACK__) +/* + * No native atomics usable for double word atomics :( + * Use fallback... + */ + +# ifndef ETHR_AMC_FALLBACK__ +# if ETHR_SIZEOF_PTR == 8 && defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) +# define ETHR_AMC_FALLBACK__ +# define ETHR_AMC_NO_ATMCS__ 1 +# define ETHR_AMC_SINT_T__ ethr_sint_t +# define ETHR_AMC_ATMC_T__ ethr_atomic_t +# define ETHR_AMC_ATMC_FUNC__(X) ETHR_INLINE_ATMC_FUNC_NAME_(ethr_atomic_ ## X) +# elif defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) +# define ETHR_AMC_FALLBACK__ +# define ETHR_AMC_NO_ATMCS__ 2 +# define ETHR_AMC_SINT_T__ ethr_sint32_t +# define ETHR_AMC_ATMC_T__ ethr_atomic32_t +# define ETHR_AMC_ATMC_FUNC__(X) ETHR_INLINE_ATMC32_FUNC_NAME_(ethr_atomic32_ ## X) +# endif +# ifdef ETHR_AMC_FALLBACK__ +typedef struct { + ETHR_AMC_ATMC_T__ atomic[ETHR_AMC_NO_ATMCS__]; +} ethr_amc_t; +# endif +# endif + +typedef struct { +#ifdef ETHR_AMC_FALLBACK__ + ethr_amc_t amc; +#endif + ethr_sint_t sint[2]; +} ethr_dw_atomic_fallback_t; + +#endif + +typedef union { +#ifdef ETHR_NATIVE_DW_ATOMIC_T__ + ETHR_NATIVE_DW_ATOMIC_T__ native; +#endif +#ifdef ETHR_NEED_DW_FALLBACK__ + ethr_dw_atomic_fallback_t fallback; +#endif + ethr_sint_t sint[2]; +} ethr_dw_atomic_t; + +typedef union { +#ifdef ETHR_SU_DW_NAINT_T__ + ETHR_SU_DW_NAINT_T__ ", ?SU_DW_SINT_FIELD, "; +#endif + ethr_sint_t ", ?DW_SINT_FIELD, "[2]; +} ethr_dw_sint_t; + +#ifdef ETHR_BIGENDIAN +# define ETHR_DW_SINT_LOW_WORD 1 +# define ETHR_DW_SINT_HIGH_WORD 0 +#else +# define ETHR_DW_SINT_LOW_WORD 0 +# define ETHR_DW_SINT_HIGH_WORD 1 +#endif + +#undef ETHR_DW_ATMC_INLINE__ +#ifdef ETHR_DW_NATMC_BITS__ +# ifdef ETHR_TRY_INLINE_FUNCS +# define ETHR_ATMC32_INLINE__ +# endif +# define ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS +#endif + +#if !defined(ETHR_DW_ATMC_INLINE__) || defined(ETHR_ATOMIC_IMPL__) +# define ETHR_NEED_DW_ATMC_PROTOTYPES__ +#endif + +#ifndef ETHR_INLINE_DW_ATMC_FUNC_NAME_ +# define ETHR_INLINE_DW_ATMC_FUNC_NAME_(X) X +#endif + +#undef ETHR_DW_ATMC_FUNC__ +#define ETHR_DW_ATMC_FUNC__(X) ETHR_INLINE_DW_ATMC_FUNC_NAME_(ethr_dw_atomic_ ## X) + +#if defined(ETHR_NEED_DW_ATMC_PROTOTYPES__) +int ethr_have_native_dw_atomic(void); +#endif +#if defined(ETHR_DW_ATMC_INLINE__) || defined(ETHR_ATOMIC_IMPL__) +static ETHR_INLINE int +ETHR_INLINE_DW_ATMC_FUNC_NAME_(ethr_have_native_dw_atomic)(void) +{ +#if defined(", ?DW_RTCHK_MACRO, ") + return ", ?DW_RTCHK_MACRO, "; +#elif defined(ETHR_DW_NATMC_BITS__) + return 1; +#else + return 0; +#endif +} +#endif + +/* -- Misc -- */ + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) +/* + * Unusual values are used by read() fallbacks implemented via cmpxchg(). + * We want to use an unusual value in hope that it is more efficient + * not to match the value in memory. + * + * - Negative integer values are probably more unusual. + * - Very large absolute integer values are probably more unusual. + * - Odd pointers are probably more unusual (only char pointers can be odd). + */ +# define ETHR_UNUSUAL_SINT32_VAL__ ((ethr_sint32_t) 0x81818181) +# if ETHR_SIZEOF_PTR == 4 +# define ETHR_UNUSUAL_SINT_VAL__ ((ethr_sint_t) ETHR_UNUSUAL_SINT32_VAL__) +# elif ETHR_SIZEOF_PTR == 8 +# define ETHR_UNUSUAL_SINT_VAL__ ((ethr_sint_t) 0x8181818181818181L) +# else +# error \"Word size not supported\" +# endif +# if defined(ETHR_NEED_DW_NATMC_ADDR) && !defined(ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_ADDR) +# error \"No ethr_native_dw_atomic_addr() available\" +# endif +# if defined(ETHR_NEED_NATMC32_ADDR) && !defined(ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADDR) +# error \"No ethr_native_atomic32_addr() available\" +# endif +# if defined(ETHR_NEED_NATMC64_ADDR) && !defined(ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADDR) +# error \"No ethr_native_atomic64_addr() available\" +# endif +#endif + +#if defined(__GNUC__) +# ifndef ETHR_COMPILER_BARRIER +# define ETHR_COMPILER_BARRIER __asm__ __volatile__(\"\" : : : \"memory\") +# endif +#elif defined(ETHR_WIN32_THREADS) +# ifndef ETHR_COMPILER_BARRIER +# include <intrin.h> +# pragma intrinsic(_ReadWriteBarrier) +# define ETHR_COMPILER_BARRIER _ReadWriteBarrier() +# endif +#endif + +void ethr_compiler_barrier_fallback(void); +#ifndef ETHR_COMPILER_BARRIER +# define ETHR_COMPILER_BARRIER ethr_compiler_barrier_fallback() +#endif + +int ethr_init_atomics(void); + +/* info */ +char **ethr_native_atomic32_ops(void); +char **ethr_native_atomic64_ops(void); +char **ethr_native_dw_atomic_ops(void); +char **ethr_native_su_dw_atomic_ops(void); + +#if !defined(ETHR_DW_NATMC_BITS__) && !defined(ETHR_NATMC_BITS__) && !defined(ETHR_NATMC32_BITS__) +/* + * ETHR_*MEMORY_BARRIER orders between locked and atomic accesses only, + * i.e. when no native atomic implementation exist and only our lock + * based atomic fallback is used, a noop is sufficient. + */ +# undef ETHR_MEMORY_BARRIER +# undef ETHR_WRITE_MEMORY_BARRIER +# undef ETHR_READ_MEMORY_BARRIER +# undef ETHR_READ_DEPEND_MEMORY_BARRIER +# undef ETHR_MEMBAR +# define ETHR_MEMBAR(B) do { } while (0) +#endif + +#ifndef ETHR_MEMBAR +# error \"No ETHR_MEMBAR defined\" +#endif + +#define ETHR_MEMORY_BARRIER ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore) +#define ETHR_WRITE_MEMORY_BARRIER ETHR_MEMBAR(ETHR_StoreStore) +#define ETHR_READ_MEMORY_BARRIER ETHR_MEMBAR(ETHR_LoadLoad) +#ifdef ETHR_READ_DEPEND_MEMORY_BARRIER +# undef ETHR_ORDERED_READ_DEPEND +#else +# define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_COMPILER_BARRIER +# define ETHR_ORDERED_READ_DEPEND +#endif +"]. + +c_top() -> + [" + +#ifdef HAVE_CONFIG_H +#include \"config.h\" +#endif + +#define ETHR_TRY_INLINE_FUNCS +#define ETHR_INLINE_DW_ATMC_FUNC_NAME_(X) X ## __ +#define ETHR_INLINE_ATMC_FUNC_NAME_(X) X ## __ +#define ETHR_INLINE_ATMC32_FUNC_NAME_(X) X ## __ +#define ETHR_ATOMIC_IMPL__ + +#include \"ethread.h\" +#include \"ethr_internal.h\" + +#if (!defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) \\ + || !defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)) +/* + * Spinlock based fallback for atomics used in absence of a native + * implementation. + */ + +#define ETHR_ATMC_FLLBK_ADDR_BITS ", ?ETHR_ATMC_FLLBK_ADDR_BITS, " +#define ETHR_ATMC_FLLBK_ADDR_SHIFT ", ?ETHR_ATMC_FLLBK_ADDR_SHIFT, " + +typedef struct { + union { + ethr_spinlock_t lck; + char buf[ETHR_CACHE_LINE_ALIGN_SIZE(sizeof(ethr_spinlock_t))]; + } u; +} ethr_atomic_protection_t; + +extern ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATMC_FLLBK_ADDR_BITS]; + +#define ETHR_ATOMIC_PTR2LCK__(PTR) \\ +(ðr_atomic_protection__[((((ethr_uint_t) (PTR)) >> ETHR_ATMC_FLLBK_ADDR_SHIFT) \\ + & ((1 << ETHR_ATMC_FLLBK_ADDR_BITS) - 1))].u.lck) + + +#define ETHR_ATOMIC_OP_FALLBACK_IMPL__(AP, EXPS) \\ +do { \\ + ethr_spinlock_t *slp__ = ETHR_ATOMIC_PTR2LCK__((AP)); \\ + ethr_spin_lock(slp__); \\ + { EXPS; } \\ + ethr_spin_unlock(slp__); \\ +} while (0) + +ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATMC_FLLBK_ADDR_BITS]; + +#endif + +", make_amc_fallback(), " + +int +ethr_init_atomics(void) +{ +#if (!defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) \\ + || !defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)) + int i; + for (i = 0; i < (1 << ETHR_ATMC_FLLBK_ADDR_BITS); i++) { + int res = ethr_spinlock_init(ðr_atomic_protection__[i].u.lck); + if (res != 0) + return res; + } +#endif + return 0; +} +"]. + +make_amc_fallback() -> + [" +#if defined(ETHR_AMC_FALLBACK__) + +/* + * Fallback for large sized (word and/or double word size) atomics using + * an \"Atomic Modification Counter\" based on smaller sized native atomics. + * + * We use a 63-bit modification counter and a one bit exclusive flag. + * If 32-bit native atomics are used, we need two 32-bit native atomics. + * The exclusive flag is the least significant bit, or if multiple atomics + * are used, the least significant bit of the least significant atomic. + * + * When using the AMC fallback the following is true: + * - Reads of the same atomic variable can be done in parallel. + * - Uncontended reads doesn't cause any cache line invalidations, + * since no modifications are done. + * - Assuming that the AMC atomic(s) and the integer(s) containing the + * value of the implemented atomic resides in the same cache line, + * modifications will only cause invalidations of one cache line. + * + * When using the spinlock based fallback none of the above is true, + * however, the spinlock based fallback consumes less memory. + */ + +# if ETHR_AMC_NO_ATMCS__ != 1 && ETHR_AMC_NO_ATMCS__ != 2 +# error \"Not supported\" +# endif +# define ETHR_AMC_MAX_TRY_READ__ 10 +# ifdef ETHR_DEBUG +# define ETHR_DBG_CHK_EXCL_STATE(ASP, S) \\ +do { \\ + ETHR_AMC_SINT_T__ act = ETHR_AMC_ATMC_FUNC__(read)(&(ASP)->atomic[0]); \\ + ETHR_ASSERT(act == (S) + 1); \\ + ETHR_ASSERT(act & 1); \\ +} while (0) +# else +# define ETHR_DBG_CHK_EXCL_STATE(ASP, S) +# endif + +static ETHR_INLINE void +amc_init(ethr_amc_t *amc, int dw, ethr_sint_t *avar, ethr_sint_t *val) +{ + avar[0] = val[0]; + if (dw) + avar[1] = val[1]; +#if ETHR_AMC_NO_ATMCS__ == 2 + ETHR_AMC_ATMC_FUNC__(init)(&amc->atomic[1], 0); +#endif + ETHR_AMC_ATMC_FUNC__(init_wb)(&amc->atomic[0], 0); +} + +static ETHR_INLINE ETHR_AMC_SINT_T__ +amc_set_excl(ethr_amc_t *amc, ETHR_AMC_SINT_T__ prev_state0) +{ + ETHR_AMC_SINT_T__ state0 = prev_state0; + /* Set exclusive flag. */ + while (1) { + ETHR_AMC_SINT_T__ act_state0, new_state0; + while (state0 & 1) { /* Wait until exclusive bit has been cleared */ + ETHR_SPIN_BODY; + state0 = ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[0]); + } + /* Try to set exclusive bit */ + new_state0 = state0 + 1; + act_state0 = ETHR_AMC_ATMC_FUNC__(cmpxchg_acqb)(&amc->atomic[0], + new_state0, + state0); + if (state0 == act_state0) + return state0; /* old state0 */ + state0 = act_state0; + } +} + +static ETHR_INLINE void +amc_inc_mc_unset_excl(ethr_amc_t *amc, ETHR_AMC_SINT_T__ old_state0) +{ + ETHR_AMC_SINT_T__ state0 = old_state0; + + /* Increment modification counter and reset exclusive flag. */ + + ETHR_DBG_CHK_EXCL_STATE(amc, state0); + + state0 += 2; + + ETHR_ASSERT((state0 & 1) == 0); + +#if ETHR_AMC_NO_ATMCS__ == 2 + if (state0 == 0) { + /* + * state0 wrapped, so we need to increment state1. There is no need + * for atomic inc op, since this is always done while having exclusive + * flag. + */ + ETHR_AMC_SINT_T__ state1 = ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[1]); + state1++; + ETHR_AMC_ATMC_FUNC__(set)(&amc->atomic[1], state1); + } +#endif + ETHR_AMC_ATMC_FUNC__(set_relb)(&amc->atomic[0], state0); +} + +static ETHR_INLINE void +amc_unset_excl(ethr_amc_t *amc, ETHR_AMC_SINT_T__ old_state0) +{ + ETHR_DBG_CHK_EXCL_STATE(amc, old_state0); + /* + * Reset exclusive flag, but leave modification counter unchanged, + * i.e., restore state to what it was before setting exclusive + * flag. + */ + ETHR_AMC_ATMC_FUNC__(set_relb)(&amc->atomic[0], old_state0); +} + +static ETHR_INLINE void +amc_set(ethr_amc_t *amc, int dw, ethr_sint_t *avar, ethr_sint_t *val) +{ + ETHR_AMC_SINT_T__ state0 = ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[0]); + + state0 = amc_set_excl(amc, state0); + + avar[0] = val[0]; + if (dw) + avar[1] = val[1]; + + amc_inc_mc_unset_excl(amc, state0); +} + +static ETHR_INLINE int +amc_try_read(ethr_amc_t *amc, int dw, ethr_sint_t *avar, + ethr_sint_t *val, ETHR_AMC_SINT_T__ *state0p) +{ + /* *state0p should contain last read value if aborting */ + ETHR_AMC_SINT_T__ old_state0; +#if ETHR_AMC_NO_ATMCS__ == 2 + ETHR_AMC_SINT_T__ state1; + int abrt; +#endif + + *state0p = ETHR_AMC_ATMC_FUNC__(read_rb)(&amc->atomic[0]); + if ((*state0p) & 1) + return 0; /* exclusive flag set; abort */ +#if ETHR_AMC_NO_ATMCS__ == 2 + state1 = ETHR_AMC_ATMC_FUNC__(read_rb)(&amc->atomic[1]); +#else + ETHR_COMPILER_BARRIER; +#endif + + val[0] = avar[0]; + if (dw) + val[1] = avar[1]; + + ETHR_READ_MEMORY_BARRIER; + + /* + * Abort if state has changed (i.e, either the exclusive + * flag is set, or modification counter changed). + */ + old_state0 = *state0p; +#if ETHR_AMC_NO_ATMCS__ == 2 + *state0p = ETHR_AMC_ATMC_FUNC__(read_rb)(&amc->atomic[0]); + abrt = (old_state0 != *state0p); + abrt |= (state1 != ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[1])); + return abrt == 0; +#else + *state0p = ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[0]); + return old_state0 == *state0p; +#endif +} + +static ETHR_INLINE void +amc_read(ethr_amc_t *amc, int dw, ethr_sint_t *avar, ethr_sint_t *val) +{ + ETHR_AMC_SINT_T__ state0; + int i; + +#if ETHR_AMC_MAX_TRY_READ__ == 0 + state0 = ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[0]); +#else + for (i = 0; i < ETHR_AMC_MAX_TRY_READ__; i++) { + if (amc_try_read(amc, dw, avar, val, &state0)) + return; /* read success */ + ETHR_SPIN_BODY; + } +#endif + + state0 = amc_set_excl(amc, state0); + + val[0] = avar[0]; + if (dw) + val[1] = avar[1]; + + amc_unset_excl(amc, state0); +} + +static ETHR_INLINE int +amc_cmpxchg(ethr_amc_t *amc, int dw, ethr_sint_t *avar, + ethr_sint_t *new, ethr_sint_t *xchg) +{ + ethr_sint_t val[2]; + ETHR_AMC_SINT_T__ state0; + + if (amc_try_read(amc, dw, avar, val, &state0)) { + if (val[0] != xchg[0] || (dw && val[1] != xchg[1])) { + xchg[0] = val[0]; + if (dw) + xchg[1] = val[1]; + return 0; /* failed */ + } + /* Operation will succeed if not interrupted */ + } + + state0 = amc_set_excl(amc, state0); + + if (xchg[0] != avar[0] || (dw && xchg[1] != avar[1])) { + xchg[0] = avar[0]; + if (dw) + xchg[1] = avar[1]; + + ETHR_DBG_CHK_EXCL_STATE(amc, state0); + + amc_unset_excl(amc, state0); + return 0; /* failed */ + } + + avar[0] = new[0]; + if (dw) + avar[1] = new[1]; + + amc_inc_mc_unset_excl(amc, state0); + return 1; +} + + +#define ETHR_AMC_MODIFICATION_OPS__(AMC, OPS) \\ +do { \\ + ETHR_AMC_SINT_T__ state0__; \\ + state0__ = ETHR_AMC_ATMC_FUNC__(read)(&(AMC)->atomic[0]); \\ + state0__ = amc_set_excl((AMC), state0__); \\ + { OPS; } \\ + amc_inc_mc_unset_excl((AMC), state0__); \\ +} while (0) + +#endif /* amc fallback */ +"]. + diff --git a/erts/lib_src/win/ethr_event.c b/erts/lib_src/win/ethr_event.c index 68f093f49c..bc2f635c26 100644 --- a/erts/lib_src/win/ethr_event.c +++ b/erts/lib_src/win/ethr_event.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2009-2010. All Rights Reserved. + * Copyright Ericsson AB 2009-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -28,13 +28,10 @@ /* --- Windows implementation of thread events ------------------------------ */ -#pragma intrinsic(_InterlockedExchangeAdd) -#pragma intrinsic(_InterlockedCompareExchange) - int ethr_event_init(ethr_event *e) { - e->state = ETHR_EVENT_OFF__; + ethr_atomic32_init(&e->state, ETHR_EVENT_OFF__); e->handle = CreateEvent(NULL, FALSE, FALSE, NULL); if (e->handle == INVALID_HANDLE_VALUE) return ethr_win_get_errno__(); @@ -63,7 +60,6 @@ ethr_event_reset(ethr_event *e) static ETHR_INLINE int wait(ethr_event *e, int spincount) { - LONG state; DWORD code; int sc, res, until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS; @@ -73,13 +69,9 @@ wait(ethr_event *e, int spincount) sc = spincount; while (1) { - long on; + ethr_sint32_t state; while (1) { -#if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ - state = e->state; -#else - state = _InterlockedExchangeAdd(&e->state, (LONG) 0); -#endif + state = ethr_atomic32_read(&e->state); if (state == ETHR_EVENT_ON__) return 0; if (sc == 0) @@ -95,9 +87,9 @@ wait(ethr_event *e, int spincount) } if (state != ETHR_EVENT_OFF_WAITER__) { - state = _InterlockedCompareExchange(&e->state, - ETHR_EVENT_OFF_WAITER__, - ETHR_EVENT_OFF__); + state = ethr_atomic32_cmpxchg(&e->state, + ETHR_EVENT_OFF_WAITER__, + ETHR_EVENT_OFF__); if (state == ETHR_EVENT_ON__) return 0; ETHR_ASSERT(state == ETHR_EVENT_OFF__); diff --git a/erts/lib_src/win/ethread.c b/erts/lib_src/win/ethread.c index 789a360b11..3abda6de4c 100644 --- a/erts/lib_src/win/ethread.c +++ b/erts/lib_src/win/ethread.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2010. All Rights Reserved. + * Copyright Ericsson AB 2010-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -35,6 +35,7 @@ #include <winerror.h> #include <stdio.h> #include <limits.h> +#include <intrin.h> #define ETHR_INLINE_FUNC_NAME_(X) X ## __ #define ETHREAD_IMPL__ @@ -158,6 +159,25 @@ ethr_abort__(void) #endif } +#if defined(ETHR_X86_RUNTIME_CONF__) + +#pragma intrinsic(__cpuid) + +void +ethr_x86_cpuid__(int *eax, int *ebx, int *ecx, int *edx) +{ + int CPUInfo[4]; + + __cpuid(CPUInfo, *eax); + + *eax = CPUInfo[0]; + *ebx = CPUInfo[1]; + *ecx = CPUInfo[2]; + *edx = CPUInfo[3]; +} + +#endif /* ETHR_X86_RUNTIME_CONF__ */ + /* * ---------------------------------------------------------------------------- * Exported functions diff --git a/erts/preloaded/ebin/erlang.beam b/erts/preloaded/ebin/erlang.beam Binary files differindex cd44e8232c..f704135ce8 100644 --- a/erts/preloaded/ebin/erlang.beam +++ b/erts/preloaded/ebin/erlang.beam diff --git a/erts/preloaded/src/erlang.erl b/erts/preloaded/src/erlang.erl index 5deb69edab..ff3633a3cc 100644 --- a/erts/preloaded/src/erlang.erl +++ b/erts/preloaded/src/erlang.erl @@ -35,14 +35,13 @@ -export([delay_trap/2]). -export([set_cookie/2, get_cookie/0]). -export([nodes/0]). --export([concat_binary/1]). + -export([list_to_integer/2,integer_to_list/2]). -export([flush_monitor_message/2]). -export([set_cpu_topology/1, format_cpu_topology/1]). -export([await_proc_exit/3]). -deprecated([hash/2]). --deprecated([concat_binary/1]). % Get rid of autoimports of spawn to avoid clashes with ourselves. -compile({no_auto_import,[spawn/1]}). @@ -535,11 +534,6 @@ set_cookie(Node, C) when Node =/= nonode@nohost, is_atom(Node) -> get_cookie() -> auth:get_cookie(). --spec concat_binary(ListOfBinaries) -> binary() when - ListOfBinaries :: iolist(). -concat_binary(List) -> - list_to_binary(List). - -spec integer_to_list(Integer, Base) -> string() when Integer :: integer(), Base :: 2..36. diff --git a/erts/test/ethread_SUITE.erl b/erts/test/ethread_SUITE.erl index 71d8c1c679..4206bebfe7 100644 --- a/erts/test/ethread_SUITE.erl +++ b/erts/test/ethread_SUITE.erl @@ -47,14 +47,29 @@ spinlock/1, rwspinlock/1, rwmutex/1, - atomic/1]). + atomic/1, + dw_atomic_massage/1]). -include_lib("test_server/include/test_server.hrl"). -tests() -> - [create_join_thread, equal_tids, mutex, try_lock_mutex, - cond_wait, broadcast, detached_thread, - max_threads, tsd, spinlock, rwspinlock, rwmutex, atomic]. +tests() -> + [create_join_thread, + equal_tids, + mutex, + try_lock_mutex, + cond_wait, + broadcast, + detached_thread, + max_threads, + tsd, + spinlock, + rwspinlock, + rwmutex, + atomic, + dw_atomic_massage]. + +all(doc) -> []; +all(suite) -> tests(). suite() -> [{ct_hooks,[ts_install_cth]}]. @@ -211,6 +226,13 @@ atomic(suite) -> atomic(Config) -> run_case(Config, "atomic", ""). +dw_atomic_massage(doc) -> + ["Massage double word atomics"]; +dw_atomic_massage(suite) -> + []; +dw_atomic_massage(Config) -> + run_case(Config, "dw_atomic_massage", ""). + %% %% %% Auxiliary functions diff --git a/erts/test/ethread_SUITE_data/ethread_tests.c b/erts/test/ethread_SUITE_data/ethread_tests.c index 0b59ff5aa6..7e7e133d6c 100644 --- a/erts/test/ethread_SUITE_data/ethread_tests.c +++ b/erts/test/ethread_SUITE_data/ethread_tests.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2004-2010. All Rights Reserved. + * Copyright Ericsson AB 2004-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -1310,6 +1310,9 @@ rwmutex_test(void) * Tests atomics. */ +#define AT_AINT32_MAX 0x7fffffff +#define AT_AINT32_MIN 0x80000000 + #define AT_THREADS 4 #define AT_ITER 10000 @@ -1320,12 +1323,428 @@ static ethr_atomic_t at_go; static ethr_atomic_t at_done; static ethr_atomic_t at_data; +#define AT_TEST_INIT(T, A, B) \ +do { \ + ethr_ ## A ## _init ## B(&A, 17); \ + ASSERT(ethr_ ## A ## _read ## B(&A) == 17); \ +} while (0) + +#define AT_TEST_SET(T, A, B) \ +do { \ + ethr_ ## A ## _set ## B(&A, 4711); \ + ASSERT(ethr_ ## A ## _read ## B(&A) == 4711); \ +} while (0) + +#define AT_TEST_XCHG(T, A, B) \ +do { \ + ethr_ ## A ## _set ## B(&A, 4711); \ + ASSERT(ethr_ ## A ## _xchg ## B(&A, 17) == 4711); \ + ASSERT(ethr_ ## A ## _read ## B(&A) == 17); \ +} while (0) + +#define AT_TEST_CMPXCHG(T, A, B) \ +do { \ + ethr_ ## A ## _set ## B(&A, 4711); \ + ASSERT(ethr_ ## A ## _cmpxchg ## B(&A, 17, 33) == 4711); \ + ASSERT(ethr_ ## A ## _read ## B(&A) == 4711); \ + ASSERT(ethr_ ## A ## _cmpxchg ## B(&A, 17, 4711) == 4711); \ + ASSERT(ethr_ ## A ## _read ## B(&A) == 17); \ +} while (0) + +#define AT_TEST_ADD_READ(T, A, B) \ +do { \ + T var_ = AT_AINT32_MAX; \ + var_ += 4711; \ + ethr_ ## A ## _set ## B(&A, AT_AINT32_MAX); \ + ASSERT(ethr_ ## A ## _add_read ## B(&A, 4711) == var_); \ + ASSERT(ethr_ ## A ## _read ## B(&A) == var_); \ + var_ = AT_AINT32_MIN; \ + var_ -= 4711; \ + ethr_ ## A ## _set ## B(&A, AT_AINT32_MIN); \ + ASSERT(ethr_ ## A ## _add_read ## B(&A, -4711) == var_); \ + ASSERT(ethr_ ## A ## _read ## B(&A) == var_); \ + ethr_ ## A ## _set ## B(&A, 4711); \ + ASSERT(ethr_ ## A ## _add_read ## B(&A, 10) == 4721); \ + ASSERT(ethr_ ## A ## _read ## B(&A) == 4721); \ +} while (0) + +#define AT_TEST_ADD(T, A, B) \ +do { \ + T var_ = AT_AINT32_MAX; \ + var_ += 4711; \ + ethr_ ## A ## _set ## B(&A, AT_AINT32_MAX); \ + ethr_ ## A ## _add ## B(&A, 4711); \ + ASSERT(ethr_ ## A ## _read ## B(&A) == var_); \ + var_ = AT_AINT32_MIN; \ + var_ -= 4711; \ + ethr_ ## A ## _set ## B(&A, AT_AINT32_MIN); \ + ethr_ ## A ## _add ## B(&A, -4711); \ + ASSERT(ethr_ ## A ## _read ## B(&A) == var_); \ + ethr_ ## A ## _set ## B(&A, 11); \ + ethr_ ## A ## _add ## B(&A, 4700); \ + ASSERT(ethr_ ## A ## _read ## B(&A) == 4711); \ +} while (0) + +#define AT_TEST_INC_READ(T, A, B) \ +do { \ + T var_ = AT_AINT32_MAX; \ + var_++; \ + ethr_ ## A ## _set ## B(&A, AT_AINT32_MAX); \ + ASSERT(ethr_ ## A ## _inc_read ## B(&A) == var_); \ + ASSERT(ethr_ ## A ## _read ## B(&A) == var_); \ + ethr_ ## A ## _set ## B(&A, 4710); \ + ASSERT(ethr_ ## A ## _inc_read ## B(&A) == 4711); \ + ASSERT(ethr_ ## A ## _read ## B(&A) == 4711); \ +} while (0) + +#define AT_TEST_DEC_READ(T, A, B) \ +do { \ + T var_ = AT_AINT32_MIN; \ + var_--; \ + ethr_ ## A ## _set ## B(&A, AT_AINT32_MIN); \ + ASSERT(ethr_ ## A ## _dec_read ## B(&A) == var_); \ + ASSERT(ethr_ ## A ## _read ## B(&A) == var_); \ + ethr_ ## A ## _set ## B(&A, 17); \ + ASSERT(ethr_ ## A ## _dec_read ## B(&A) == 16); \ + ASSERT(ethr_ ## A ## _read ## B(&A) == 16); \ +} while (0) + + +#define AT_TEST_INC(T, A, B) \ +do { \ + T var_ = AT_AINT32_MAX; \ + var_++; \ + ethr_ ## A ## _set ## B(&A, AT_AINT32_MAX); \ + ethr_ ## A ## _inc ## B(&A); \ + ASSERT(ethr_ ## A ## _read ## B(&A) == var_); \ + ethr_ ## A ## _set ## B(&A, 4710); \ + ethr_ ## A ## _inc ## B(&A); \ + ASSERT(ethr_ ## A ## _read ## B(&A) == 4711); \ +} while (0) + +#define AT_TEST_DEC(T, A, B) \ +do { \ + T var_ = AT_AINT32_MIN; \ + var_--; \ + ethr_ ## A ## _set ## B(&A, AT_AINT32_MIN); \ + ethr_ ## A ## _dec ## B(&A); \ + ASSERT(ethr_ ## A ## _read ## B(&A) == var_); \ + ethr_ ## A ## _set ## B(&A, 17); \ + ethr_ ## A ## _dec ## B(&A); \ + ASSERT(ethr_ ## A ## _read ## B(&A) == 16); \ +} while (0) + +#define AT_TEST_READ_BAND(T, A, B) \ +do { \ + ethr_ ## A ## _set ## B(&A, 0x13131313); \ + ASSERT(ethr_ ## A ## _read_band ## B(&A, 0x31313131) == 0x13131313); \ + ASSERT(ethr_ ## A ## _read ## B(&A) == 0x11111111); \ +} while (0) + +#define AT_TEST_READ_BOR(T, A, B) \ +do { \ + ethr_ ## A ## _set ## B(&A, 0x11111111); \ + ASSERT(ethr_ ## A ## _read_bor ## B(&A, 0x23232323) == 0x11111111); \ + ASSERT(ethr_ ## A ## _read ## B(&A) == 0x33333333); \ +} while (0) + + +static void +atomic_basic_test(void) +{ + /* + * Verify that each op does what it is expected + * to do for at least one input. + */ + ethr_atomic32_t atomic32; + ethr_atomic_t atomic; + + print_line("AT_AINT32_MAX=%d",AT_AINT32_MAX); + print_line("AT_AINT32_MIN=%d",AT_AINT32_MIN); + + AT_TEST_INIT(ethr_sint32_t, atomic32, ); + AT_TEST_SET(ethr_sint32_t, atomic32, ); + AT_TEST_XCHG(ethr_sint32_t, atomic32, ); + AT_TEST_CMPXCHG(ethr_sint32_t, atomic32, ); + AT_TEST_ADD_READ(ethr_sint32_t, atomic32, ); + AT_TEST_ADD(ethr_sint32_t, atomic32, ); + AT_TEST_INC_READ(ethr_sint32_t, atomic32, ); + AT_TEST_DEC_READ(ethr_sint32_t, atomic32, ); + AT_TEST_INC(ethr_sint32_t, atomic32, ); + AT_TEST_DEC(ethr_sint32_t, atomic32, ); + AT_TEST_READ_BAND(ethr_sint32_t, atomic32, ); + AT_TEST_READ_BOR(ethr_sint32_t, atomic32, ); + + AT_TEST_INIT(ethr_sint32_t, atomic32, _acqb); + AT_TEST_SET(ethr_sint32_t, atomic32, _acqb); + AT_TEST_XCHG(ethr_sint32_t, atomic32, _acqb); + AT_TEST_CMPXCHG(ethr_sint32_t, atomic32, _acqb); + AT_TEST_ADD_READ(ethr_sint32_t, atomic32, _acqb); + AT_TEST_ADD(ethr_sint32_t, atomic32, _acqb); + AT_TEST_INC_READ(ethr_sint32_t, atomic32, _acqb); + AT_TEST_DEC_READ(ethr_sint32_t, atomic32, _acqb); + AT_TEST_INC(ethr_sint32_t, atomic32, _acqb); + AT_TEST_DEC(ethr_sint32_t, atomic32, _acqb); + AT_TEST_READ_BAND(ethr_sint32_t, atomic32, _acqb); + AT_TEST_READ_BOR(ethr_sint32_t, atomic32, _acqb); + + AT_TEST_INIT(ethr_sint32_t, atomic32, _relb); + AT_TEST_SET(ethr_sint32_t, atomic32, _relb); + AT_TEST_XCHG(ethr_sint32_t, atomic32, _relb); + AT_TEST_CMPXCHG(ethr_sint32_t, atomic32, _relb); + AT_TEST_ADD_READ(ethr_sint32_t, atomic32, _relb); + AT_TEST_ADD(ethr_sint32_t, atomic32, _relb); + AT_TEST_INC_READ(ethr_sint32_t, atomic32, _relb); + AT_TEST_DEC_READ(ethr_sint32_t, atomic32, _relb); + AT_TEST_INC(ethr_sint32_t, atomic32, _relb); + AT_TEST_DEC(ethr_sint32_t, atomic32, _relb); + AT_TEST_READ_BAND(ethr_sint32_t, atomic32, _relb); + AT_TEST_READ_BOR(ethr_sint32_t, atomic32, _relb); + + AT_TEST_INIT(ethr_sint32_t, atomic32, _rb); + AT_TEST_SET(ethr_sint32_t, atomic32, _rb); + AT_TEST_XCHG(ethr_sint32_t, atomic32, _rb); + AT_TEST_CMPXCHG(ethr_sint32_t, atomic32, _rb); + AT_TEST_ADD_READ(ethr_sint32_t, atomic32, _rb); + AT_TEST_ADD(ethr_sint32_t, atomic32, _rb); + AT_TEST_INC_READ(ethr_sint32_t, atomic32, _rb); + AT_TEST_DEC_READ(ethr_sint32_t, atomic32, _rb); + AT_TEST_INC(ethr_sint32_t, atomic32, _rb); + AT_TEST_DEC(ethr_sint32_t, atomic32, _rb); + AT_TEST_READ_BAND(ethr_sint32_t, atomic32, _rb); + AT_TEST_READ_BOR(ethr_sint32_t, atomic32, _rb); + + AT_TEST_INIT(ethr_sint32_t, atomic32, _wb); + AT_TEST_SET(ethr_sint32_t, atomic32, _wb); + AT_TEST_XCHG(ethr_sint32_t, atomic32, _wb); + AT_TEST_CMPXCHG(ethr_sint32_t, atomic32, _wb); + AT_TEST_ADD_READ(ethr_sint32_t, atomic32, _wb); + AT_TEST_ADD(ethr_sint32_t, atomic32, _wb); + AT_TEST_INC_READ(ethr_sint32_t, atomic32, _wb); + AT_TEST_DEC_READ(ethr_sint32_t, atomic32, _wb); + AT_TEST_INC(ethr_sint32_t, atomic32, _wb); + AT_TEST_DEC(ethr_sint32_t, atomic32, _wb); + AT_TEST_READ_BAND(ethr_sint32_t, atomic32, _wb); + AT_TEST_READ_BOR(ethr_sint32_t, atomic32, _wb); + + AT_TEST_INIT(ethr_sint32_t, atomic32, _mb); + AT_TEST_SET(ethr_sint32_t, atomic32, _mb); + AT_TEST_XCHG(ethr_sint32_t, atomic32, _mb); + AT_TEST_CMPXCHG(ethr_sint32_t, atomic32, _mb); + AT_TEST_ADD_READ(ethr_sint32_t, atomic32, _mb); + AT_TEST_ADD(ethr_sint32_t, atomic32, _mb); + AT_TEST_INC_READ(ethr_sint32_t, atomic32, _mb); + AT_TEST_DEC_READ(ethr_sint32_t, atomic32, _mb); + AT_TEST_INC(ethr_sint32_t, atomic32, _mb); + AT_TEST_DEC(ethr_sint32_t, atomic32, _mb); + AT_TEST_READ_BAND(ethr_sint32_t, atomic32, _mb); + AT_TEST_READ_BOR(ethr_sint32_t, atomic32, _mb); + + AT_TEST_INIT(ethr_sint_t, atomic, ); + AT_TEST_SET(ethr_sint_t, atomic, ); + AT_TEST_XCHG(ethr_sint_t, atomic, ); + AT_TEST_CMPXCHG(ethr_sint_t, atomic, ); + AT_TEST_ADD_READ(ethr_sint_t, atomic, ); + AT_TEST_ADD(ethr_sint_t, atomic, ); + AT_TEST_INC_READ(ethr_sint_t, atomic, ); + AT_TEST_DEC_READ(ethr_sint_t, atomic, ); + AT_TEST_INC(ethr_sint_t, atomic, ); + AT_TEST_DEC(ethr_sint_t, atomic, ); + AT_TEST_READ_BAND(ethr_sint_t, atomic, ); + AT_TEST_READ_BOR(ethr_sint_t, atomic, ); + + AT_TEST_INIT(ethr_sint_t, atomic, _acqb); + AT_TEST_SET(ethr_sint_t, atomic, _acqb); + AT_TEST_XCHG(ethr_sint_t, atomic, _acqb); + AT_TEST_CMPXCHG(ethr_sint_t, atomic, _acqb); + AT_TEST_ADD_READ(ethr_sint_t, atomic, _acqb); + AT_TEST_ADD(ethr_sint_t, atomic, _acqb); + AT_TEST_INC_READ(ethr_sint_t, atomic, _acqb); + AT_TEST_DEC_READ(ethr_sint_t, atomic, _acqb); + AT_TEST_INC(ethr_sint_t, atomic, _acqb); + AT_TEST_DEC(ethr_sint_t, atomic, _acqb); + AT_TEST_READ_BAND(ethr_sint_t, atomic, _acqb); + AT_TEST_READ_BOR(ethr_sint_t, atomic, _acqb); + + AT_TEST_INIT(ethr_sint_t, atomic, _relb); + AT_TEST_SET(ethr_sint_t, atomic, _relb); + AT_TEST_XCHG(ethr_sint_t, atomic, _relb); + AT_TEST_CMPXCHG(ethr_sint_t, atomic, _relb); + AT_TEST_ADD_READ(ethr_sint_t, atomic, _relb); + AT_TEST_ADD(ethr_sint_t, atomic, _relb); + AT_TEST_INC_READ(ethr_sint_t, atomic, _relb); + AT_TEST_DEC_READ(ethr_sint_t, atomic, _relb); + AT_TEST_INC(ethr_sint_t, atomic, _relb); + AT_TEST_DEC(ethr_sint_t, atomic, _relb); + AT_TEST_READ_BAND(ethr_sint_t, atomic, _relb); + AT_TEST_READ_BOR(ethr_sint_t, atomic, _relb); + + AT_TEST_INIT(ethr_sint_t, atomic, _rb); + AT_TEST_SET(ethr_sint_t, atomic, _rb); + AT_TEST_XCHG(ethr_sint_t, atomic, _rb); + AT_TEST_CMPXCHG(ethr_sint_t, atomic, _rb); + AT_TEST_ADD_READ(ethr_sint_t, atomic, _rb); + AT_TEST_ADD(ethr_sint_t, atomic, _rb); + AT_TEST_INC_READ(ethr_sint_t, atomic, _rb); + AT_TEST_DEC_READ(ethr_sint_t, atomic, _rb); + AT_TEST_INC(ethr_sint_t, atomic, _rb); + AT_TEST_DEC(ethr_sint_t, atomic, _rb); + AT_TEST_READ_BAND(ethr_sint_t, atomic, _rb); + AT_TEST_READ_BOR(ethr_sint_t, atomic, _rb); + + AT_TEST_INIT(ethr_sint_t, atomic, _wb); + AT_TEST_SET(ethr_sint_t, atomic, _wb); + AT_TEST_XCHG(ethr_sint_t, atomic, _wb); + AT_TEST_CMPXCHG(ethr_sint_t, atomic, _wb); + AT_TEST_ADD_READ(ethr_sint_t, atomic, _wb); + AT_TEST_ADD(ethr_sint_t, atomic, _wb); + AT_TEST_INC_READ(ethr_sint_t, atomic, _wb); + AT_TEST_DEC_READ(ethr_sint_t, atomic, _wb); + AT_TEST_INC(ethr_sint_t, atomic, _wb); + AT_TEST_DEC(ethr_sint_t, atomic, _wb); + AT_TEST_READ_BAND(ethr_sint_t, atomic, _wb); + AT_TEST_READ_BOR(ethr_sint_t, atomic, _wb); + + AT_TEST_INIT(ethr_sint_t, atomic, _mb); + AT_TEST_SET(ethr_sint_t, atomic, _mb); + AT_TEST_XCHG(ethr_sint_t, atomic, _mb); + AT_TEST_CMPXCHG(ethr_sint_t, atomic, _mb); + AT_TEST_ADD_READ(ethr_sint_t, atomic, _mb); + AT_TEST_ADD(ethr_sint_t, atomic, _mb); + AT_TEST_INC_READ(ethr_sint_t, atomic, _mb); + AT_TEST_DEC_READ(ethr_sint_t, atomic, _mb); + AT_TEST_INC(ethr_sint_t, atomic, _mb); + AT_TEST_DEC(ethr_sint_t, atomic, _mb); + AT_TEST_READ_BAND(ethr_sint_t, atomic, _mb); + AT_TEST_READ_BOR(ethr_sint_t, atomic, _mb); + + /* Double word */ + { + ethr_dw_atomic_t dw_atomic; + ethr_dw_sint_t dw0, dw1; + dw0.sint[0] = 4711; + dw0.sint[1] = 4712; + + /* init */ + ethr_dw_atomic_init(&dw_atomic, &dw0); + ethr_dw_atomic_read(&dw_atomic, &dw1); + ETHR_ASSERT(dw1.sint[0] == 4711); + ETHR_ASSERT(dw1.sint[1] == 4712); + + /* set */ + dw0.sint[0] = 42; + dw0.sint[1] = ~((ethr_sint_t) 0); + ethr_dw_atomic_set(&dw_atomic, &dw0); + ethr_dw_atomic_read(&dw_atomic, &dw1); + ASSERT(dw1.sint[0] == 42); + ASSERT(dw1.sint[1] == ~((ethr_sint_t) 0)); + + /* cmpxchg */ + dw0.sint[0] = 17; + dw0.sint[1] = 18; + dw1.sint[0] = 19; + dw1.sint[1] = 20; + ASSERT(!ethr_dw_atomic_cmpxchg(&dw_atomic, &dw1, &dw0)); + ethr_dw_atomic_read(&dw_atomic, &dw0); + ASSERT(dw0.sint[0] == 42); + ASSERT(dw0.sint[1] == ~((ethr_sint_t) 0)); + + ASSERT(ethr_dw_atomic_cmpxchg(&dw_atomic, &dw1, &dw0)); + + ethr_dw_atomic_read(&dw_atomic, &dw0); + ASSERT(dw0.sint[0] == 19); + ASSERT(dw0.sint[1] == 20); + } +} + + +#define AT_DW_MIN 12 +#define AT_DW_MAX 42 +#define AT_DW_THREADS (AT_DW_MAX - AT_DW_MIN + 1) + +#define AT_DW_LOOPS 200000 +#define AT_DW_R_LOOPS 10 + +ethr_dw_atomic_t at_dw_atomic; + +void +at_dw_valid(ethr_dw_sint_t *dw) +{ + int i; + char c; + char *cp; + + ASSERT(dw->sint[0] == dw->sint[1]); + + cp = (char *) &dw->sint[0]; + c = cp[0]; + + ASSERT(AT_DW_MIN <= c && c <= AT_DW_MAX); + + for (i = 0; i < sizeof(ethr_sint_t); i++) + ASSERT(c == cp[i]); +} + +void * +at_dw_thr(void *vval) +{ + int l, r; + ethr_sint_t val = (ethr_sint_t) vval; + ethr_dw_sint_t dw; + ethr_dw_sint_t my_dw; + + my_dw.sint[0] = val; + my_dw.sint[1] = val; + + ethr_dw_atomic_set(&at_dw_atomic, &my_dw); + for (l = 0; l < AT_DW_LOOPS; l++) { + for (r = 0; r < AT_DW_R_LOOPS; r++) { + ethr_dw_atomic_read(&at_dw_atomic, &dw); + at_dw_valid(&dw); + } + ethr_dw_atomic_set(&at_dw_atomic, &my_dw); + for (r = 0; r < AT_DW_R_LOOPS; r++) { + ethr_dw_atomic_read(&at_dw_atomic, &dw); + at_dw_valid(&dw); + } + dw.sint[0] = 0; + dw.sint[1] = 0; + while (1) { + if (ethr_dw_atomic_cmpxchg(&at_dw_atomic, &my_dw, &dw)) + break; + } + } +} + +static void +dw_atomic_massage_test(void) +{ + int i, res; + ethr_tid tid[AT_DW_THREADS]; + ethr_thr_opts thr_opts = ETHR_THR_OPTS_DEFAULT_INITER; + ethr_dw_sint_t dw; + + dw.sint[0] = dw.sint[1] = 0; + + ethr_dw_atomic_init(&at_dw_atomic, &dw); + + for (i = AT_DW_MIN; i <= AT_DW_MAX; i++) { + ethr_sint_t val; + memset(&val, i, sizeof(ethr_sint_t)); + res = ethr_thr_create(&tid[i-AT_DW_MIN], at_dw_thr, (void *) val, &thr_opts); + ASSERT(res == 0); + } + for (i = AT_DW_MIN; i <= AT_DW_MAX; i++) { + res = ethr_thr_join(tid[i-AT_DW_MIN], NULL); + ASSERT(res == 0); + } +} + void * at_thread(void *unused) { int i; long val, go; - val = ethr_atomic_inc_read(&at_ready); ASSERT(val > 0); @@ -1373,7 +1792,6 @@ at_thread(void *unused) return NULL; } - static void atomic_test(void) { @@ -1382,6 +1800,8 @@ atomic_test(void) ethr_tid tid[AT_THREADS]; ethr_thr_opts thr_opts = ETHR_THR_OPTS_DEFAULT_INITER; + atomic_basic_test(); + #if ETHR_SIZEOF_PTR > 4 at_rm_val = ((long) 1) << 57; at_set_val = ((long) 1) << 60; @@ -1493,6 +1913,8 @@ main(int argc, char *argv[]) rwmutex_test(); else if (strcmp(testcase, "atomic") == 0) atomic_test(); + else if (strcmp(testcase, "dw_atomic_massage") == 0) + dw_atomic_massage_test(); else skip("Test case \"%s\" not implemented yet", testcase); diff --git a/lib/asn1/c_src/Makefile b/lib/asn1/c_src/Makefile index 9e9cb18524..60543df355 100644 --- a/lib/asn1/c_src/Makefile +++ b/lib/asn1/c_src/Makefile @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 2002-2010. All Rights Reserved. +# Copyright Ericsson AB 2002-2011. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in @@ -51,33 +51,26 @@ EI_LIBDIR = $(ERL_TOP)/lib/erl_interface/obj$(TYPEMARKER)/$(TARGET) # ---------------------------------------------------- # FLAGS # ---------------------------------------------------- -EI_INCLUDES = -I$(ERL_TOP)/lib/erl_interface/include CFLAGS = $(DED_INCLUDES) $(EI_INCLUDES) $(DED_CFLAGS) LDFLAGS += $(DED_LDFLAGS) -LD_INCL_EI = -L$(EI_LIBDIR) - # ---------------------------------------------------- # Target Specs # ---------------------------------------------------- -C_FILES = asn1_erl_driver.c +NIF_OBJ_FILES = $(OBJDIR)/asn1_erl_nif.o ifeq ($(TARGET),win32) -LD_EI = -lei_md -SHARED_OBJ_FILES = $(LIBDIR)/asn1_erl_drv.dll -OBJ_FILES = $(OBJDIR)/asn1_erl_drv.o +NIF_SHARED_OBJ_FILES = $(LIBDIR)/asn1_erl_nif.dll CLIB_FLAGS = LN=cp else -LD_EI = -lei -OBJ_FILES = $(OBJDIR)/asn1_erl_drv.o ifeq ($(findstring vxworks,$(TARGET)),vxworks) -SHARED_OBJ_FILES = $(LIBDIR)/asn1_erl_drv.eld +NIF_SHARED_OBJ_FILE = $(LIBDIR)/asn1_erl_nif.eld CLIB_FLAGS = else -SHARED_OBJ_FILES = $(LIBDIR)/asn1_erl_drv.so +NIF_SHARED_OBJ_FILE = $(LIBDIR)/asn1_erl_nif.so CLIB_FLAGS = -lc endif LN= ln -s @@ -87,7 +80,7 @@ endif # Targets # ---------------------------------------------------- -opt: $(OBJDIR) $(LIBDIR) $(SHARED_OBJ_FILES) +opt: $(OBJDIR) $(LIBDIR) $(NIF_SHARED_OBJ_FILE) debug: opt @@ -103,11 +96,11 @@ docs: # ---------------------------------------------------- -$(OBJ_FILES): $(C_FILES) - $(CC) -c $(CFLAGS) -o $(OBJ_FILES) $(C_FILES) +$(OBJDIR)/%.o: %.c + $(CC) -c $(CFLAGS) -O3 -o $@ $< -$(SHARED_OBJ_FILES): $(OBJ_FILES) - $(LD) $(LDFLAGS) $(LD_INCL_EI) -o $(SHARED_OBJ_FILES) $(OBJ_FILES) $(LD_EI) $(CLIB_FLAGS) $(LIBS) +$(NIF_SHARED_OBJ_FILE): $(NIF_OBJ_FILES) + $(LD) $(LDFLAGS) -o $(NIF_SHARED_OBJ_FILE) $(NIF_OBJ_FILES) $(CLIB_FLAGS) $(LIBS) $(LIBDIR): -mkdir -p $(LIBDIR) @@ -124,9 +117,9 @@ include $(ERL_TOP)/make/otp_release_targets.mk release_spec: opt $(INSTALL_DIR) $(RELSYSDIR)/priv/lib - $(INSTALL_PROGRAM) $(SHARED_OBJ_FILES) $(RELSYSDIR)/priv/lib + $(INSTALL_PROGRAM) $(NIF_SHARED_OBJ_FILE) $(RELSYSDIR)/priv/lib $(INSTALL_DIR) $(RELSYSDIR)/c_src - $(INSTALL_DATA) $(C_FILES) $(RELSYSDIR)/c_src + $(INSTALL_DATA) *.c $(RELSYSDIR)/c_src release_docs_spec: diff --git a/lib/asn1/c_src/asn1_erl_driver.c b/lib/asn1/c_src/asn1_erl_driver.c deleted file mode 100644 index 18d4157941..0000000000 --- a/lib/asn1/c_src/asn1_erl_driver.c +++ /dev/null @@ -1,1677 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 2002-2011. All Rights Reserved. - * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. - * - * %CopyrightEnd% - * - */ -#include <stdlib.h> -#include <stdio.h> -#include <string.h> -#include "erl_driver.h" -#include "ei.h" - - -/* #define ASN1_DEBUG 1 */ - -#define ASN1_OK 0 -#define ASN1_ERROR -1 -#define ASN1_COMPL_ERROR 1 -#define ASN1_MEMORY_ERROR 0 -#define ASN1_DECODE_ERROR 2 -#define ASN1_TAG_ERROR -3 -#define ASN1_LEN_ERROR -4 -#define ASN1_INDEF_LEN_ERROR -5 -#define ASN1_VALUE_ERROR -6 - - -#define ASN1_CLASS 0xc0 -#define ASN1_FORM 0x20 -#define ASN1_CLASSFORM (ASN1_CLASS | ASN1_FORM) -#define ASN1_TAG 0x1f -#define ASN1_LONG_TAG 0x7f - -#define ASN1_INDEFINITE_LENGTH 0x80 -#define ASN1_SHORT_DEFINITE_LENGTH 0 - -#define ASN1_PRIMITIVE 0 -#define ASN1_CONSTRUCTED 0x20 - -#define ASN1_COMPLETE 1 -#define ASN1_BER_TLV_DECODE 2 -#define ASN1_BER_TLV_PARTIAL_DECODE 3 - -#define ASN1_NOVALUE 0 - -#define ASN1_SKIPPED 0 -#define ASN1_OPTIONAL 1 -#define ASN1_CHOOSEN 2 - - -#define CEIL(X,Y) ((X-1) / Y + 1) - -#define INVMASK(X,M) (X & (M ^ 0xff)) -#define MASK(X,M) (X & M) - -typedef struct { - ErlDrvPort port; - int buffer_size; -} asn1_data; - -/* int min_alloc_bytes; */ - - -static ErlDrvData asn1_drv_start(ErlDrvPort, char *); - -static void asn1_drv_stop(ErlDrvData); - -int asn1_drv_control(ErlDrvData, unsigned int, char *, int, char **, int); - -int complete(ErlDrvBinary **,unsigned char *,unsigned char *, int); - -int insert_octets(int, unsigned char **, unsigned char **, int *); - -int insert_octets_except_unused(int, unsigned char **, unsigned char **, - int *, int); - -int insert_octets_as_bits_exact_len(int, int, unsigned char **, - unsigned char **, int *); - -int insert_octets_as_bits(int, unsigned char **, unsigned char **,int *); - -int pad_bits(int, unsigned char **, int *); - -int insert_least_sign_bits(int, unsigned char, unsigned char **, int *); - -int insert_most_sign_bits(int, unsigned char, unsigned char **, int *); - -int insert_bits_as_bits(int, int, unsigned char **, unsigned char **, int *); - -int insert_octets_unaligned(int, unsigned char **, unsigned char **, int); - -int realloc_decode_buf(ErlDrvBinary **,int); - -int realloc_memory(ErlDrvBinary **,int,unsigned char **,unsigned char **); - -int decode_begin(ErlDrvBinary **,unsigned char *, int, unsigned int *); - -int decode(ErlDrvBinary **,int *,unsigned char *,int *, int); - -int decode_tag(char *,int *,unsigned char *,int,int *); - -int decode_value(int *,unsigned char *,int *,ErlDrvBinary **,int ,int); - - -/* declaration of functions used for partial decode of a BER encoded - message */ - -int decode_partial(ErlDrvBinary **,unsigned char *, int); - -int skip_tag(unsigned char *,int *,int); - -int skip_length_and_value(unsigned char *,int *,int); - -int get_tag(unsigned char *,int *,int); - -int get_length(unsigned char *,int *,int *,int); - -int get_value(char *,unsigned char *,int *,int); - -static ErlDrvEntry asn1_drv_entry = { - NULL, /* init, always NULL for dynamic drivers */ - asn1_drv_start, /* start, called when port is opened */ - asn1_drv_stop, /* stop, called when port is closed */ - NULL, /* output, called when erlang has sent */ - NULL, /* ready_input, called when input descriptor ready */ - NULL, /* ready_output, called when output descriptor ready */ - "asn1_erl_drv", /* char *driver_name, the argument to open_port */ - NULL, /* finish, called when unloaded */ - NULL, /* void * that is not used (BC) */ - asn1_drv_control, /* control, port_control callback */ - NULL, /* timeout, called on timeouts */ - NULL, /* outputv, vector output interface */ - - NULL, /* ready_async */ - NULL, /* flush */ - NULL, /* call */ - NULL, /* event */ - ERL_DRV_EXTENDED_MARKER, - ERL_DRV_EXTENDED_MAJOR_VERSION, - ERL_DRV_EXTENDED_MINOR_VERSION, - ERL_DRV_FLAG_USE_PORT_LOCKING, - NULL, /* handle2 */ - NULL /* process_exit */ -}; - - - -DRIVER_INIT(asn1_erl_drv) /* must match name in driver_entry */ -{ - return &asn1_drv_entry; -} - -static ErlDrvData asn1_drv_start(ErlDrvPort port, char *buff) -{ - /* extern int min_alloc_bytes; */ - char *ptr; - asn1_data* d; - - d = (asn1_data*)driver_alloc(sizeof(asn1_data)); - set_port_control_flags(port, PORT_CONTROL_FLAG_BINARY); - d->port = port; - - if ((ptr = getenv("ASN1_MIN_BUF_SIZE")) == NULL) - d->buffer_size = 1024; - else - d->buffer_size = atoi(ptr); - return (ErlDrvData)d; -} - - -static void asn1_drv_stop(ErlDrvData handle) -{ - driver_free((char*)handle); -} - - - -int asn1_drv_control(ErlDrvData handle, - unsigned int command, - char *buf, - int buf_len, - char **res_buf, - int res_buf_len) -{ - unsigned char *complete_buf; - int complete_len, decode_len; - ErlDrvBinary *drv_binary; - ErlDrvBinary **drv_bin_ptr; - asn1_data* a_data; - int min_alloc_bytes; - unsigned int err_pos = 0; /* in case of error, return last correct position */ - int ret_err; /* return value in case of error in TLV decode, i.e. length of list in res_buf */ - - /* In case previous call to asn1_drv_control resulted in a change of - return value from binary to integer list */ - a_data = (asn1_data *)handle; - min_alloc_bytes = a_data->buffer_size; - set_port_control_flags(a_data->port, PORT_CONTROL_FLAG_BINARY); - - if (command == ASN1_COMPLETE) - { - if (buf_len==0) { - return 0; /* Avoid binary buffer overwrite (OTP-8451) */ - } - /* Do the PER complete encode step */ - if ((drv_binary = driver_alloc_binary(buf_len))==NULL) { - /* error handling */ - set_port_control_flags(a_data->port, 0); - return ASN1_MEMORY_ERROR; - } - complete_buf = (unsigned char*) drv_binary->orig_bytes; - if ((complete_len = complete(&drv_binary,complete_buf,(unsigned char*) buf,buf_len)) == ASN1_ERROR) - { - /* error handling due to failure in complete */ - /* printf("error when running complete\n\r"); */ - driver_free_binary(drv_binary); - set_port_control_flags(a_data->port, 0); - **res_buf = '1'; - return ASN1_COMPL_ERROR; - } - /* printf("complete_len=%dbuf_len=%d,orig_size=%d\n\r",complete_len,buf_len,drv_binary->orig_size); */ - /* now the message is complete packed, return to Erlang */ - /* if (complete_len < buf_len) {*/ - if (complete_len < drv_binary->orig_size) { - ErlDrvBinary *tmp; - if ((tmp=driver_realloc_binary(drv_binary,complete_len)) == NULL){ - /*error handling due to memory allocation failure */ - driver_free_binary(drv_binary); - set_port_control_flags(a_data->port, 0); - return ASN1_MEMORY_ERROR; - }else - drv_binary=tmp; - } - *res_buf = (char *)drv_binary; - return complete_len; - } else if (command == ASN1_BER_TLV_DECODE) { /* control == 2 */ - /* Do the tlv decode, - return the resulting term encoded on the Erlang - external format */ -/* printf("driver: buffer_len = %d, min_alloc_bytes = %d\r\n",buf_len,min_alloc_bytes); */ - if ((drv_binary = driver_alloc_binary((buf_len*5)+min_alloc_bytes))==NULL) { - /* error handling */ - set_port_control_flags(a_data->port, 0); - return ASN1_MEMORY_ERROR; - } - drv_bin_ptr = &drv_binary; - if ((decode_len = decode_begin(drv_bin_ptr,(unsigned char*)buf,buf_len,&err_pos)) <= ASN1_ERROR) - { - /* error handling due to failure in decode */ - char tmp_res_buf[5]; - driver_free_binary(*drv_bin_ptr); - set_port_control_flags(a_data->port, 0); - - if(decode_len==ASN1_ERROR) - tmp_res_buf[0]='1'; - else if(decode_len==ASN1_TAG_ERROR) - tmp_res_buf[0]='2'; - else if(decode_len==ASN1_LEN_ERROR) - tmp_res_buf[0]='3'; - else if(decode_len==ASN1_INDEF_LEN_ERROR) - tmp_res_buf[0]='4'; - else if(decode_len==ASN1_VALUE_ERROR) - tmp_res_buf[0]='5'; -/* printf("err_pos=%d\r\n",err_pos); */ -/* printf("decode_len:%d\r\n",decode_len); */ - ret_err = 1; - while(err_pos>0){ - tmp_res_buf[ret_err] =(char)err_pos;/* c;*/ - err_pos = err_pos >> 8; - ret_err++; - } - strncpy(*res_buf,tmp_res_buf,ret_err); - return ret_err; - } -/* printf("decode_len=%d\r\n",decode_len); */ - if (decode_len < ((buf_len * 5) + min_alloc_bytes)) { - /* not all memory was used => we have to reallocate */ - ErlDrvBinary *tmp; - if ((tmp=driver_realloc_binary(*drv_bin_ptr,decode_len)) == NULL){ - /*error handling due to memory allocation failure */ - driver_free_binary(*drv_bin_ptr); - set_port_control_flags(a_data->port, 0); - return ASN1_MEMORY_ERROR; - }else - *drv_bin_ptr=tmp; - } - *res_buf = (char *)(*drv_bin_ptr); - return decode_len; - } else { /*command == ASN1_BER_TLV_PARTIAL_DECODE */ - if ((drv_binary = driver_alloc_binary(buf_len))==NULL) { - /* error handling */ - set_port_control_flags(a_data->port, 0); - return ASN1_MEMORY_ERROR; - } - drv_bin_ptr = &drv_binary; - if ((decode_len = decode_partial(drv_bin_ptr,(unsigned char*)buf,buf_len)) - <= ASN1_ERROR) { - /* error handling due to failure in decode */ - driver_free_binary(*drv_bin_ptr); - set_port_control_flags(a_data->port, 0); - -/* printf("asn1_drv_control 1: decode_len=%d\r\n",decode_len); */ - - if(decode_len==ASN1_ERROR) - **res_buf = '1'; - return ASN1_DECODE_ERROR; - } - if (decode_len < buf_len) { - /* not all memory was used => we have to reallocate */ - ErlDrvBinary *tmp; -/* printf("asn1_drv_control 2: decode_len=%d\r\n",decode_len); */ - if ((tmp=driver_realloc_binary(*drv_bin_ptr,decode_len)) == NULL){ - /*error handling due to memory allocation failure */ - driver_free_binary(*drv_bin_ptr); - set_port_control_flags(a_data->port, 0); - return ASN1_MEMORY_ERROR; - }else - *drv_bin_ptr=tmp; - } - *res_buf = (char *)(*drv_bin_ptr); - return decode_len; - } -} - - - -/* - * - * This section defines functionality for the complete encode of a - * PER encoded message - * - */ - -int complete(ErlDrvBinary **drv_binary,unsigned char *complete_buf, - unsigned char *in_buf, int in_buf_len) -{ - int counter = in_buf_len; - /* counter keeps track of number of bytes left in the - input buffer */ - - int buf_space = in_buf_len; - /* This is the amount of allocated space left of the complete_buf. It - is possible when padding is applied that more space is needed than - was originally allocated. */ - - int buf_size = in_buf_len; - /* Size of the buffer. May become reallocated and thus other than - in_buf_len */ - - unsigned char *in_ptr, *ptr; - /* in_ptr points at the next byte in in_buf to be moved to - complete_buf. - ptr points into the new completed buffer, complete_buf, at the - position of the next byte that will be set */ - int unused = 8; - /* unused = [1,...,8] indicates how many of the rigthmost bits of - the byte that ptr points at that are unassigned */ - - int no_bits,no_bytes,in_unused,desired_len,ret, saved_mem, needed, pad_bits; - - unsigned char val; - - in_ptr = in_buf; - ptr = complete_buf; - *ptr = 0x00; - while(counter > 0) { - counter--; -/* printf("*in_ptr = %d\n\r",*in_ptr); */ - switch (*in_ptr) { - case 0: - /* just one zero-bit should be added to the buffer */ - if(unused == 1){ - unused = 8; - *++ptr = 0x00; - buf_space--; - } else - unused--; - break; - - case 1: - /* one one-bit should be added to the buffer */ - if(unused == 1){ - *ptr = *ptr | 1; - unused = 8; - *++ptr = 0x00; - buf_space--; - } else { - *ptr = *ptr | (1 << (unused - 1)); - unused--; - } - break; - - case 2: - /* align buffer to end of byte */ - if (unused != 8) { - *++ptr = 0x00; - buf_space--; - unused = 8; - } - break; - - case 10: - /* next byte in in_buf tells how many bits in the second next - byte that will be used */ - /* The leftmost unused bits in the value byte are supposed to be - zero bits */ - no_bits = (int)*(++in_ptr); - val = *(++in_ptr); - counter -= 2; - if ((ret=insert_least_sign_bits(no_bits,val,&ptr,&unused)) == ASN1_ERROR) - return ASN1_ERROR; - buf_space -= ret; - break; - - case 20: - /* in this case the next value in_ptr points at holds the number - of following bytes that holds the value that will be inserted - in the completed buffer */ - no_bytes = (int)*(++in_ptr); - counter -= (no_bytes + 1); - if ((counter<0) || - (ret=insert_octets(no_bytes,&in_ptr,&ptr,&unused)) == ASN1_ERROR) - return ASN1_ERROR; - buf_space -= ret; - break; - - case 21: - /* in this case the next two bytes in_ptr points at holds the number - of following bytes that holds the value that will be inserted - in the completed buffer */ - no_bytes = (int)*(++in_ptr); - no_bytes = no_bytes << 8; - no_bytes = no_bytes | (int)*(++in_ptr); - counter -= (2 + no_bytes); - if ((counter<0) || - (ret=insert_octets(no_bytes,&in_ptr,&ptr,&unused)) == ASN1_ERROR) - return ASN1_ERROR; - buf_space -= ret; - break; - - case 30: - /* If we call the following bytes, in the buffer in_ptr points at, - By1,By2,Rest then Rest is the value that will be transfered to - the completed buffer. By1 tells how many of the rightmost bits in - Rest that should not be used. By2 is the length of Rest in bytes.*/ - in_unused = (int)*(++in_ptr); - no_bytes = (int)*(++in_ptr); - counter -= (2 + no_bytes); -/* printf("%d: case 30: in_unused=%d, no_bytes=%d,counter=%d\n\r",__LINE__,in_unused,no_bytes,counter); */ - ret = -4711; - if ((counter<0) || - (ret=insert_octets_except_unused(no_bytes,&in_ptr,&ptr,&unused,in_unused)) == ASN1_ERROR) - return ASN1_ERROR; -/* printf("%d: ret=%d\n\r",__LINE__, ret); */ - buf_space -= ret; - break; - - case 31: - /* If we call the following bytes, in the buffer in_ptr points at, - By1,By2,By3,Rest then Rest is the value that will be transfered to - the completed buffer. By1 tells how many of the rightmost bits in - Rest that should not be used. By2 and By3 is the length of - Rest in bytes.*/ - in_unused = (int)*(++in_ptr); - no_bytes = (int)*(++in_ptr); - no_bytes = no_bytes << 8; - no_bytes = no_bytes | (int)*(++in_ptr); - counter -= (3 + no_bytes); - if ((counter<0) || - (ret=insert_octets_except_unused(no_bytes,&in_ptr,&ptr,&unused,in_unused)) == ASN1_ERROR) - return ASN1_ERROR; - buf_space -= ret; - break; - - case 40: - /* This case implies that next byte,By1,(..,By1,By2,Bin,...) - is the desired length of the completed value, maybe needs - padding zero bits or removal of trailing zero bits from Bin. - By2 is the length of Bin and Bin is the value that will be - put into the completed buffer. Each byte in Bin has the value - 1 or 0.*/ - desired_len = (int)*(++in_ptr); - no_bytes=(int)*(++in_ptr); - - /* This is the algorithm for need of memory reallocation: - Only when padding (cases 40 - 43,45 - 47) more memory may be - used than allocated. Therefore one has to keep track of how - much of the allocated memory that has been saved, i.e. the - difference between the number of parsed bytes of the input buffer - and the number of used bytes of the output buffer. - If saved memory is less than needed for the padding then we - need more memory. */ - saved_mem = buf_space - counter; - pad_bits = desired_len - no_bytes - unused; - needed = (pad_bits > 0) ? CEIL(pad_bits,8) : 0; - if (saved_mem < needed) { - /* Have to allocate more memory */ - buf_size += needed; - buf_space += needed; - if (realloc_memory(drv_binary,buf_size,&ptr, - &complete_buf) == ASN1_ERROR) - return ASN1_ERROR; - } - - counter -= (2 + no_bytes); - if ((counter<0) || - (ret=insert_octets_as_bits_exact_len(desired_len,no_bytes,&in_ptr, - &ptr,&unused)) == ASN1_ERROR) - return ASN1_ERROR; - buf_space -= ret; - break; - - case 41: - /* Same as case 40 apart from By2, the length of Bin, which is in - two bytes*/ - desired_len = (int)*(++in_ptr); - no_bytes=(int)*(++in_ptr); - no_bytes = no_bytes << 8; - no_bytes = no_bytes | (int)*(++in_ptr); - - saved_mem = buf_space - counter; - needed = CEIL((desired_len-unused),8) - no_bytes; - if (saved_mem < needed) { - /* Have to allocate more memory */ - buf_size += needed; - buf_space += needed; - if (realloc_memory(drv_binary,buf_size,&ptr, - &complete_buf) == ASN1_ERROR) - return ASN1_ERROR; - } - - counter -= (3 + no_bytes); - if ((counter<0) || - (ret=insert_octets_as_bits_exact_len(desired_len,no_bytes,&in_ptr, - &ptr,&unused)) == ASN1_ERROR) - return ASN1_ERROR; - buf_space -= ret; - break; - - case 42: - /* Same as case 40 apart from By1, the desired length, which is in - two bytes*/ - desired_len = (int)*(++in_ptr); - desired_len = desired_len << 8; - desired_len = desired_len | (int)*(++in_ptr); - no_bytes=(int)*(++in_ptr); - - saved_mem = buf_space - counter; - needed = CEIL((desired_len-unused),8) - no_bytes; - if (saved_mem < needed) { - /* Have to allocate more memory */ - buf_size += needed; - buf_space += needed; - if (realloc_memory(drv_binary,buf_size,&ptr, - &complete_buf) == ASN1_ERROR) - return ASN1_ERROR; - } - - counter -= (3 + no_bytes); - if ((counter<0) || - (ret=insert_octets_as_bits_exact_len(desired_len,no_bytes,&in_ptr, - &ptr,&unused)) == ASN1_ERROR) - return ASN1_ERROR; - buf_space -= ret; - break; - - case 43: - /* Same as case 40 apart from By1 and By2, the desired length and - the length of Bin, which are in two bytes each. */ - desired_len = (int)*(++in_ptr); - desired_len = desired_len << 8; - desired_len = desired_len | (int)*(++in_ptr); - no_bytes=(int)*(++in_ptr); - no_bytes = no_bytes << 8; - no_bytes = no_bytes | (int)*(++in_ptr); - - saved_mem = buf_space - counter; - needed = CEIL((desired_len-unused),8) - no_bytes; - if (saved_mem < needed) { - /* Have to allocate more memory */ - buf_size += needed; - buf_space += needed; - if (realloc_memory(drv_binary,buf_size,&ptr, - &complete_buf) == ASN1_ERROR) - return ASN1_ERROR; - } - - counter -= (4 + no_bytes); - if ((counter<0) || - (ret=insert_octets_as_bits_exact_len(desired_len,no_bytes,&in_ptr, - &ptr,&unused)) == ASN1_ERROR) - return ASN1_ERROR; - buf_space -= ret; - break; - - case 45: - /* This case assumes that the following bytes in the incoming buffer - (called By1,By2,Bin) is By1, which is the number of bits (n) that - will be inserted in the completed buffer. By2 is the number of - bytes in Bin. Each bit in the buffer Bin should be inserted from - the leftmost until the nth.*/ - desired_len = (int)*(++in_ptr); - no_bytes=(int)*(++in_ptr); - - saved_mem = buf_space - counter; - needed = CEIL((desired_len-unused),8) - no_bytes; -/* printf("buf_space=%d, counter=%d, needed=%d",buf_space,counter,needed); */ - if (saved_mem < needed) { - /* Have to allocate more memory */ - buf_size += needed; - buf_space += needed; - if (realloc_memory(drv_binary,buf_size,&ptr, - &complete_buf) == ASN1_ERROR) - return ASN1_ERROR; - } - - counter -= (2 + no_bytes); -/* printf("calling insert_bits_as_bits: desired_len=%d, no_bytes=%d\n\r",desired_len,no_bytes); */ -/* printf("1in_ptr=%d\n\r",in_ptr); */ - - if((counter<0) || - (ret=insert_bits_as_bits(desired_len,no_bytes,&in_ptr, - &ptr,&unused)) == ASN1_ERROR) - return ASN1_ERROR; -/* printf("2in_ptr=%d, ptr=%d, complete_buf=%d\n\r",in_ptr,ptr,complete_buf); */ -/* printf("buf_space=%d, ret=%d, counter=%d\n\r",buf_space,ret,counter); */ - buf_space -= ret; - break; - - case 46: - /* Same as case 45 apart from By1, the desired length, which is - in two bytes. */ - desired_len = (int)*(++in_ptr); - desired_len = desired_len << 8; - desired_len = desired_len | (int)*(++in_ptr); - no_bytes=(int)*(++in_ptr); - - saved_mem = buf_space - counter; - needed = CEIL((desired_len-unused),8) - no_bytes; - if (saved_mem < needed) { - /* Have to allocate more memory */ - buf_size += needed; - buf_space += needed; - if (realloc_memory(drv_binary,buf_size,&ptr, - &complete_buf) == ASN1_ERROR) - return ASN1_ERROR; - } - - counter -= (3 + no_bytes); - if((counter<0) || - (ret=insert_bits_as_bits(desired_len,no_bytes,&in_ptr, - &ptr,&unused)) == ASN1_ERROR) - return ASN1_ERROR; - buf_space -= ret; - break; - - case 47: - /* Same as case 45 apart from By1 and By2, the desired length - and the length of Bin, which are in two bytes each. */ - desired_len = (int)*(++in_ptr); - desired_len = desired_len << 8; - desired_len = desired_len | (int)*(++in_ptr); - no_bytes=(int)*(++in_ptr); - no_bytes = no_bytes << 8; - no_bytes = no_bytes | (int)*(++in_ptr); - - saved_mem = buf_space - counter; - needed = CEIL((desired_len-unused),8) - no_bytes; - if (saved_mem < needed) { - /* Have to allocate more memory */ - buf_size += needed; - buf_space += needed; - if (realloc_memory(drv_binary,buf_size,&ptr, - &complete_buf) == ASN1_ERROR) - return ASN1_ERROR; - } - - counter -= (4 + no_bytes); - if((counter<0) || - (ret=insert_bits_as_bits(desired_len,no_bytes,&in_ptr, - &ptr,&unused)) == ASN1_ERROR) - return ASN1_ERROR; - buf_space -= ret; - break; - - default: - return ASN1_ERROR; - } - in_ptr++; - } - /* The returned buffer must be at least one byte and - it must be octet aligned */ - if ((unused == 8) && (ptr != complete_buf)) - return (ptr - complete_buf); - else { - ptr++; /* octet align buffer */ - return (ptr - complete_buf); - } -} - - -int realloc_memory(ErlDrvBinary **drv_binary, - int amount, - unsigned char **ptr, - unsigned char **complete_buf) { - - ErlDrvBinary *tmp_bin; - int i; - -/* printf("realloc_momory: amount = %d\n",amount); */ - if ((tmp_bin=driver_realloc_binary(*drv_binary,amount)) == NULL) { - /*error handling due to memory allocation failure */ -/* printf("error when allocating memory\n"); */ - return ASN1_ERROR; - }else { - i = *ptr - *complete_buf; - *drv_binary=tmp_bin; - *complete_buf = (unsigned char*)(*drv_binary)->orig_bytes; - *ptr = *complete_buf + i; - } - return ASN1_OK; -} - - -int insert_most_sign_bits(int no_bits, - unsigned char val, - unsigned char **output_ptr, - int *unused) { - unsigned char *ptr = *output_ptr; - - if (no_bits < *unused){ - *ptr = *ptr | (val >> (8 - *unused)); - *unused -= no_bits; - } else if (no_bits == *unused) { - *ptr = *ptr | (val >> (8 - *unused)); - *unused = 8; - *++ptr = 0x00; - } else { - *ptr = *ptr | (val >> (8 - *unused)); - *++ptr = 0x00; - *ptr = *ptr | (val << *unused); - *unused = 8 - (no_bits - *unused); - } - *output_ptr = ptr; - return ASN1_OK; -} - - -int insert_least_sign_bits(int no_bits, - unsigned char val, - unsigned char **output_ptr, - int *unused) { - unsigned char *ptr = *output_ptr; - int ret = 0; - - if (no_bits < *unused){ - *ptr = *ptr | (val << (*unused - no_bits)); - *unused -= no_bits; - } else if (no_bits == *unused){ - *ptr = *ptr | val; - *unused = 8; - *++ptr = 0x00; - ret++; - } else { - /* first in the begun byte in the completed buffer insert - so many bits that fit, then insert the rest in next byte.*/ - *ptr = *ptr | (val >> (no_bits - *unused)); - *++ptr = 0x00; - ret++; - *ptr = *ptr | (val << (8 - (no_bits - *unused))); - *unused = 8 - (no_bits - *unused); - } - *output_ptr = ptr; - return ret; -} - -/* pad_bits adds no_bits bits in the buffer that output_ptr - points at. - */ -int pad_bits(int no_bits, unsigned char **output_ptr, int *unused) - { - unsigned char *ptr = *output_ptr; - int ret = 0; - - while (no_bits > 0) { - if(*unused == 1){ - *unused = 8; - *++ptr = 0x00; - ret++; - } else - (*unused)--; - no_bits--; - } - *output_ptr = ptr; - return ret; - } - - -/* insert_bits_as_bits removes no_bytes bytes from the buffer that in_ptr - points at and takes the desired_no leftmost bits from those removed - bytes and inserts them in the buffer(output buffer) that ptr points at. - The unused parameter tells how many bits that are not set in the - actual byte in the output buffer. If desired_no is more bits than the - input buffer has in no_bytes bytes, then zero bits is padded.*/ -int insert_bits_as_bits(int desired_no, - int no_bytes, - unsigned char **input_ptr, - unsigned char **output_ptr, - int *unused) -{ - unsigned char *in_ptr = *input_ptr; - unsigned char val; - int no_bits, ret, ret2; - - if (desired_no == (no_bytes * 8)) { - if(insert_octets_unaligned(no_bytes,&in_ptr,output_ptr,*unused) - == ASN1_ERROR) - return ASN1_ERROR; - ret = no_bytes; - } - else if (desired_no < (no_bytes * 8)) { -/* printf("insert_bits_as_bits 1\n\r"); */ - if(insert_octets_unaligned(desired_no/8,&in_ptr,output_ptr,*unused) - == ASN1_ERROR) - return ASN1_ERROR; -/* printf("insert_bits_as_bits 2\n\r"); */ - val = *++in_ptr; -/* printf("val = %d\n\r",(int)val); */ - no_bits = desired_no % 8; -/* printf("no_bits = %d\n\r",no_bits); */ - insert_most_sign_bits(no_bits,val,output_ptr,unused); - ret = CEIL(desired_no,8); - } - else { - if(insert_octets_unaligned(no_bytes,&in_ptr,output_ptr,*unused) - == ASN1_ERROR) - return ASN1_ERROR; - ret2 = pad_bits(desired_no - (no_bytes * 8),output_ptr,unused); -/* printf("ret2 = %d\n\r",ret2); */ - ret = CEIL(desired_no,8); -/* printf("ret = %d\n\r",ret); */ - } -/* printf("*unused = %d\n\r",*unused); */ - *input_ptr = in_ptr; - return ret; -} - - -/* insert_octets_as_bits_exact_len */ -int -insert_octets_as_bits_exact_len(int desired_len, - int in_buff_len, - unsigned char **in_ptr, - unsigned char **ptr, - int *unused) -{ - int ret = 0; - int ret2 = 0; - - if (desired_len == in_buff_len) { - if ((ret = insert_octets_as_bits(in_buff_len,in_ptr,ptr,unused)) == ASN1_ERROR) - return ASN1_ERROR; - } - else if(desired_len > in_buff_len) { - if((ret = insert_octets_as_bits(in_buff_len,in_ptr,ptr,unused)) == ASN1_ERROR) - return ASN1_ERROR; - /* now pad with zero bits */ -/* printf("~npad_bits: called with %d bits padding~n~n~r",desired_len - in_buff_len); */ - if ((ret2=pad_bits(desired_len - in_buff_len,ptr,unused)) == ASN1_ERROR) - return ASN1_ERROR; - } - else {/* desired_len < no_bits */ - if ((ret=insert_octets_as_bits(desired_len,in_ptr,ptr,unused)) == ASN1_ERROR) - return ASN1_ERROR; - /* now remove no_bits - desired_len bytes from in buffer */ - *in_ptr += (in_buff_len - desired_len); - } - return (ret+ret2); -} - - - -/* insert_octets_as_bits takes no_bytes bytes from the buffer that input_ptr - points at and inserts the least significant bit of it in the buffer that - output_ptr points at. Each byte in the input buffer must be 1 or 0 - otherwise the function returns ASN1_ERROR. The output buffer is concatenated - without alignment. - */ -int insert_octets_as_bits(int no_bytes, - unsigned char **input_ptr, - unsigned char **output_ptr, - int *unused) -{ - unsigned char *in_ptr = *input_ptr; - unsigned char *ptr = *output_ptr; - int used_bits = 8 - *unused; - - while (no_bytes > 0) { - switch (*++in_ptr) { - case 0: - if(*unused == 1){ - *unused = 8; - *++ptr = 0x00; - } else - (*unused)--; - break; - case 1: - if(*unused == 1){ - *ptr = *ptr | 1; - *unused = 8; - *++ptr = 0x00; - } else { - *ptr = *ptr | (1 << (*unused - 1)); - (*unused)--; - } - break; - default: - return ASN1_ERROR; - } - no_bytes--; - } - *input_ptr = in_ptr; - *output_ptr = ptr; - return ((used_bits+no_bytes) / 8); /*return number of new bytes - in completed buffer */ -} - -/* insert_octets inserts bytes from the input buffer, *input_ptr, - into the output buffer, *output_ptr. Before the first byte is - inserted the input buffer is aligned. - */ -int insert_octets(int no_bytes, - unsigned char **input_ptr, - unsigned char **output_ptr, - int *unused) -{ - unsigned char *in_ptr = *input_ptr; - unsigned char *ptr = *output_ptr; - int ret = 0; - - if (*unused != 8) {/* must align before octets are added */ - *++ptr = 0x00; - ret++; - *unused = 8; - } - while(no_bytes > 0) { - *ptr = *(++in_ptr); - *++ptr = 0x00; - /* *unused = *unused - 1; */ - no_bytes--; - } - *input_ptr = in_ptr; - *output_ptr = ptr; - return (ret + no_bytes); -} - -/* insert_octets_unaligned inserts bytes from the input buffer, *input_ptr, - into the output buffer, *output_ptr.No alignment is done. - */ -int insert_octets_unaligned(int no_bytes, - unsigned char **input_ptr, - unsigned char **output_ptr, - int unused) -{ - unsigned char *in_ptr = *input_ptr; - unsigned char *ptr = *output_ptr; - int n = no_bytes; - unsigned char val; - - while (n > 0) { - if (unused == 8) { - *ptr = *++in_ptr; - *++ptr = 0x00; - }else { - val = *++in_ptr; - *ptr = *ptr | val >> (8 - unused); - *++ptr = 0x00; - *ptr = val << unused; - } - n--; - } - *input_ptr = in_ptr; - *output_ptr = ptr; - return no_bytes; -} - - -int insert_octets_except_unused(int no_bytes, - unsigned char **input_ptr, - unsigned char **output_ptr, - int *unused, - int in_unused) -{ - unsigned char *in_ptr = *input_ptr; - unsigned char *ptr = *output_ptr; - int val, no_bits; - int ret = 0; - - if (in_unused == 0){ -/* printf("%d: insert_octets_except_unused: if\n\r",__LINE__); */ - if ((ret = insert_octets_unaligned(no_bytes,&in_ptr,&ptr, - *unused)) == ASN1_ERROR) - return ASN1_ERROR; - } - else { -/* printf("%d: insert_octets_except_unused: else\n\r",__LINE__); */ - if ((ret=insert_octets_unaligned(no_bytes - 1,&in_ptr,&ptr,*unused)) != ASN1_ERROR) { - val = (int) *(++in_ptr); - no_bits = 8 - in_unused; - /* no_bits is always less than *unused since the buffer is - octet aligned after insert:octets call, so the following - if clasuse is obsolete I think */ - if(no_bits < *unused){ - *ptr = *ptr | (val >> (8 - *unused)); - *unused = *unused - no_bits; - } else if (no_bits == *unused) { - *ptr = *ptr | (val >> (8 - *unused)); - *++ptr = 0x00; - ret++; - *unused = 8; - } else { - *ptr = *ptr | (val >> (8 - *unused)); - *++ptr = 0x00; - ret++; - *ptr = *ptr | (val << *unused); - *unused = 8 - (no_bits - *unused); - } - } else - return ASN1_ERROR; - } - *input_ptr = in_ptr; - *output_ptr = ptr; -/* printf("%d: insert_octets_except_unused: ret=%d\n\r",__LINE__,ret); */ - return ret; -} - - - -/* - * - * This section defines functionality for the partial decode of a - * BER encoded message - * - */ - -/* - * int decode(ErlDrvBinary **drv_binary,unsigned char *decode_buf, - * unsigned char *in_buf, int in_buf_len) - * drv_binary is a pointer to a pointer to an allocated driver binary. - * in_buf is a pointer into the buffer of incoming bytes. - * in_buf_len is the length of the incoming buffer. - * The function reads the bytes in the incoming buffer and structures - * it in a nested way as Erlang terms. The buffer contains data in the - * order tag - length - value. Tag, length and value has the following - * format: - * A tag is normally one byte but may be of any length, if the tag number - * is greater than 30. +----------+ - * |CL|C|NNNNN| - * +----------+ - * If NNNNN is 31 then will the 7 l.s.b of each of the following tag number - * bytes contain the tag number. Each tag number byte that is not the last one - * has the m.s.b. set to 1. - * The length can be short definite length (sdl), long definite length (ldl) - * or indefinite length (il). - * sdl: +---------+ the L bits is the length - * |0|LLLLLLL| - * +---------+ - * ldl: +---------+ +---------+ +---------+ +-----------+ - * |1|lllllll| |first len| | | |the Nth len| - * +---------+ +---------+ +---------+ ... +-----------+ - * The first byte tells how many len octets will follow, max 127 - * il: +---------+ +----------------------+ +--------+ +--------+ - * |1|0000000| |content octets (Value)| |00000000| |00000000| - * +---------+ +----------------------+ +--------+ +--------+ - * The value octets are preceded by one octet and followed by two - * exactly as above. The value must be some tag-length-value encoding. - * - * The function returns a value in Erlnag term format: - * {{TagNo,Value},Rest} - * TagNo is an integer ((CL bsl 16) + tag number) which limits the tag number - * to 65535. - * Value is a binary if the C bit in tag was unset, otherwise (if tag was - * constructed) Value is a list, List. - * List is like: [{TagNo,Value},{TagNo,Value},...] - * Rest is a binary, i.e. the undecoded part of the buffer. Most often Rest - * is the empty binary. - * If some error occured during the decoding of the in_buf an error is returned. - */ -int decode_begin(ErlDrvBinary **drv_binary,unsigned char *in_buf, int in_buf_len, unsigned int *err_pos) -{ - int maybe_ret; - char *decode_buf = (*drv_binary)->orig_bytes; - int ei_index = 0; - int ib_index = 0; - /* ei_index is the index used by the ei functions to encode an - Erlang term into the buffer decode_buf */ - /* ib_index is the index were to read the next byte from in_buf */ - - -#ifdef ASN1_DEBUG - printf("decode_begin1: ei_index=%d, ib_index=%d\n\r",ei_index,ib_index); -#endif - /* the first byte must be a "version magic" */ - if(ei_encode_version(decode_buf,&ei_index) == ASN1_ERROR) - return ASN1_ERROR; /* 1 byte */ -#ifdef ASN1_DEBUG - printf("decode_begin2: ei_index=%d, ib_index=%d\n\r",ei_index,ib_index); -#endif - if (ei_encode_tuple_header(decode_buf,&ei_index,2) == ASN1_ERROR) - return ASN1_ERROR; /* 2 bytes */ -#ifdef ASN1_DEBUG - printf("decode_begin3: ei_index=%d, ib_index=%d\n\r",ei_index,ib_index); -#endif - if((maybe_ret=decode(drv_binary,&ei_index,in_buf,&ib_index,in_buf_len)) <= ASN1_ERROR) - { - *err_pos = ib_index; -#ifdef ASN1_DEBUG - printf("err_pos=%d,ib_index=%d\r\n",*err_pos,ib_index); -#endif - return maybe_ret; - }; - - decode_buf = (*drv_binary)->orig_bytes; /* maybe a realloc during decode_value */ -#ifdef ASN1_DEBUG - printf("decode_begin4: in_buf_len=%d, ei_index=%d, ib_index=%d\n\r", - in_buf_len,ei_index,ib_index); -#endif - /* "{{TagNo,Value},Rest}" */ - if (ei_encode_binary(decode_buf,&ei_index,&(in_buf[ib_index]),in_buf_len-ib_index) - == ASN1_ERROR) /* at least 5 bytes */ - return ASN1_ERROR; -#ifdef ASN1_DEBUG - printf("decode_begin5: ei_index=%d, ib_index=%d\n\r",ei_index,ib_index); -#endif - return ei_index; -} - -int decode(ErlDrvBinary **drv_binary,int *ei_index,unsigned char *in_buf, - int *ib_index, int in_buf_len) -{ - int maybe_ret; - char *decode_buf = (*drv_binary)->orig_bytes; - int form; -#ifdef ASN1_DEBUG - printf("decode 1\n\r"); -#endif - if (((*drv_binary)->orig_size - *ei_index) < 19) {/* minimum amount of bytes */ - /* allocate more memory */ - if (realloc_decode_buf(drv_binary,(*drv_binary)->orig_size * 2) == - ASN1_ERROR) - return ASN1_ERROR; - decode_buf = (*drv_binary)->orig_bytes; - } -/* printf("decode 2\n\r"); */ - /* "{" */ - if (ei_encode_tuple_header(decode_buf,ei_index,2) == ASN1_ERROR) - return ASN1_ERROR; /* 2 bytes */ -#ifdef ASN1_DEBUG - printf("decode 3:orig_size=%ld, ei_index=%d, ib_index=%d\n\r",(*drv_binary)->orig_size,*ei_index,*ib_index); -#endif - - /*buffer must hold at least two bytes*/ - if ((*ib_index +2) > in_buf_len) - return ASN1_VALUE_ERROR; - /* "{{TagNo," */ - if ((form = decode_tag(decode_buf,ei_index,in_buf,in_buf_len,ib_index)) <= ASN1_ERROR) - return form; /* 5 bytes */ -#ifdef ASN1_DEBUG - printf("i_i=%d,in_buf_len=%d\r\n",*ei_index,in_buf_len); -#endif - if (*ib_index >= in_buf_len){ - return ASN1_TAG_ERROR; - } -#ifdef ASN1_DEBUG - printf("decode 5 ib_index=%d\n\r",*ib_index); -#endif - /* buffer must hold at least one byte (0 as length and nothing as - value) */ - /* "{{TagNo,Value}," */ - if ((maybe_ret=decode_value(ei_index,in_buf,ib_index,drv_binary,form, - in_buf_len)) <= ASN1_ERROR) - return maybe_ret; /* at least 5 bytes */ -#ifdef ASN1_DEBUG - printf("decode 7\n\r"); -#endif - return *ei_index; -} - -/* - * decode_tag decodes the BER encoded tag in in_buf and puts it in the - * decode_buf encoded by the Erlang extern format as an Erlang term. - */ -int decode_tag(char *decode_buf,int *db_index,unsigned char *in_buf, - int in_buf_len, int *ib_index) -{ - int tag_no, tmp_tag, form; - - - /* first get the class of tag and bit shift left 16*/ - tag_no = ((MASK(in_buf[*ib_index],ASN1_CLASS)) << 10); - - form = (MASK(in_buf[*ib_index],ASN1_FORM)); -#ifdef ASN1_DEBUG - printf("decode_tag0:ii=%d, tag_no=%d, form=%d.\r\n", - *ib_index,tag_no,form); -#endif - - /* then get the tag number */ - if((tmp_tag = (int) INVMASK(in_buf[*ib_index],ASN1_CLASSFORM)) < 31) { - ei_encode_ulong(decode_buf,db_index,tag_no+tmp_tag); /* usual case */ - (*ib_index)++; -#ifdef ASN1_DEBUG - printf("decode_tag1:ii=%d.\r\n",*ib_index); -#endif - } - else - { - int n = 0; /* n is used to check that the 64K limit is not - exceeded*/ -#ifdef ASN1_DEBUG - printf("decode_tag1:ii=%d, in_buf_len=%d.\r\n",*ib_index,in_buf_len); -#endif - - /* should check that at least three bytes are left in - in-buffer,at least two tag byte and at least one length byte */ - if ((*ib_index +3) > in_buf_len) - return ASN1_VALUE_ERROR; - (*ib_index)++; -#ifdef ASN1_DEBUG - printf("decode_tag2:ii=%d.\r\n",*ib_index); -#endif - /* The tag is in the following bytes in in_buf as - 1ttttttt 1ttttttt ... 0ttttttt, where the t-bits - is the tag number*/ - /* In practice is the tag size limited to 64K, i.e. 16 bits. If - the tag is greater then 64K return an error */ - while (((tmp_tag = (int)in_buf[*ib_index]) >= 128) && n < 2){ - /* m.s.b. = 1 */ - tag_no = tag_no + (MASK(tmp_tag,ASN1_LONG_TAG) << 7); - (*ib_index)++; -#ifdef ASN1_DEBUG - printf("decode_tag3:ii=%d.\r\n",*ib_index); -#endif - n++; - }; - if ((n==2) && in_buf[*ib_index] > 3) - return ASN1_TAG_ERROR; /* tag number > 64K */ - tag_no = tag_no + in_buf[*ib_index]; - (*ib_index)++; -#ifdef ASN1_DEBUG - printf("decode_tag4:ii=%d.\r\n",*ib_index); -#endif - ei_encode_ulong(decode_buf,db_index,tag_no); - } - return form; -} - - -/* - * decode_value decodes the BER encoded length and value fields in the - * in_buf and puts the value part in the decode_buf as an Erlang term - * encoded by the Erlang extern format - */ -int decode_value(int *ei_index,unsigned char *in_buf, - int *ib_index,ErlDrvBinary **drv_binary,int form, - int in_buf_len) -{ - int maybe_ret; - char *decode_buf = (*drv_binary)->orig_bytes; - unsigned int len = 0; - unsigned int lenoflen = 0; - int indef = 0; - -#ifdef ASN1_DEBUG - printf("decode_value1:ib_index=%d\n\r",*ib_index); -#endif - if (((in_buf[*ib_index]) & 0x80) == ASN1_SHORT_DEFINITE_LENGTH) { - len = in_buf[*ib_index]; - } - else if (in_buf[*ib_index] == ASN1_INDEFINITE_LENGTH) - indef = 1; - else /* long definite length */ { - lenoflen = (in_buf[*ib_index] & 0x7f); /*length of length */ -#ifdef ASN1_DEBUG - printf("decode_value,lenoflen:%d\r\n",lenoflen); -#endif - if (lenoflen > (in_buf_len - (*ib_index+1))) - return ASN1_LEN_ERROR; - len = 0; - while (lenoflen-- ) { - (*ib_index)++; -#ifdef ASN1_DEBUG - printf("decode_value1:*ib_index=%d, byte = %d.\r\n",*ib_index,in_buf[*ib_index]); -#endif - if (!(len < (1 << (sizeof(len)-1)*8))) - return ASN1_LEN_ERROR; /* length does not fit in 32 bits */ - len = (len << 8) + in_buf[*ib_index]; - } - } - if (len > (in_buf_len - (*ib_index + 1))) - return ASN1_VALUE_ERROR; - (*ib_index)++; -#ifdef ASN1_DEBUG - printf("decode_value2:ii=%d.\r\n",*ib_index); -#endif - if (indef == 1) - { /* in this case it is desireably to check that indefinite length - end bytes exist in inbuffer */ - while (!(in_buf[*ib_index]==0 && in_buf[*ib_index + 1]==0)) { -#ifdef ASN1_DEBUG - printf("decode_value while:ib_index=%d in_buf_len=%d\n\r", - *ib_index,in_buf_len); -#endif - if(*ib_index >= in_buf_len) - return ASN1_INDEF_LEN_ERROR; - ei_encode_list_header(decode_buf,ei_index,1); /* 5 bytes */ - if((maybe_ret=decode(drv_binary,ei_index,in_buf, - ib_index,in_buf_len)) <= ASN1_ERROR) - return maybe_ret; - decode_buf = (*drv_binary)->orig_bytes; - } - (*ib_index) += 2; /* skip the indefinite length end bytes */ -#ifdef ASN1_DEBUG - printf("decode_value3:ii=%d.\r\n",*ib_index); -#endif - ei_encode_empty_list(decode_buf,ei_index); /* 1 byte */ - } - else if (form == ASN1_CONSTRUCTED) - { - int end_index = *ib_index + len; - if(end_index > in_buf_len) - return ASN1_LEN_ERROR; - while (*ib_index < end_index) { - -#ifdef ASN1_DEBUG - printf("decode_value3:*ib_index=%d, end_index=%d\n\r",*ib_index,end_index); -#endif - ei_encode_list_header(decode_buf,ei_index,1); /* 5 bytes */ - if((maybe_ret=decode(drv_binary,ei_index,in_buf, - ib_index,in_buf_len))<=ASN1_ERROR) - return maybe_ret; - decode_buf = (*drv_binary)->orig_bytes; - } - ei_encode_empty_list(decode_buf,ei_index); /* 1 byte */ - } - else - { - if (((*drv_binary)->orig_size - *ei_index) < 10+len) { /* 5+len for the binary*/ - if (realloc_decode_buf(drv_binary,(*drv_binary)->orig_size * 2) == - ASN1_ERROR) - return ASN1_ERROR; - decode_buf = (*drv_binary)->orig_bytes; - } - if((*ib_index + len) > in_buf_len) - return ASN1_LEN_ERROR; - ei_encode_binary(decode_buf,ei_index,&in_buf[*ib_index],len); - *ib_index = *ib_index + len; -#ifdef ASN1_DEBUG - printf("decode_value4:ii=%d.\r\n",*ib_index); -#endif - } - return ASN1_OK; -} - -int realloc_decode_buf(ErlDrvBinary **drv_binary,int amount) { - ErlDrvBinary *tmp_bin; - - if ((tmp_bin=driver_realloc_binary(*drv_binary,amount)) == NULL) - return ASN1_ERROR; - *drv_binary = tmp_bin; - return ASN1_OK; -} - - - -/* - * int decode_partial(drv_binary,in_buf,in_buf_len) - */ -/* - * The in_buf contains two parts: first information about which value - * will be decoded, as a sequence of tags and tag codes, then the - * encoded BER value. First of all comes a length field that tells how - * many following bytes contains the sequence of tags. Then starts the - * BER encoded message. The tag sequence length field is a single - * byte. The sequence of tags/tag codes may be one of the codes - * ASN1_SKIPPED, ASN1_CHOOSEN and a tag or ASN1_OPTIONAL and a - * tag. ASN1_SKIPPED means that the following tag is mandatory and is - * skipped. ASN1_CHOOSEN means that the value of this tag shall, if - * this was the last tag in tag sequence, be returned or be searched - * in for the next tag. ASN1_OPTIONAL means that this tag shall be - * skipped but it may be missing. Each tag in the tag sequence - * correspond to a tag in the BER encoded message. If the decode - * arives to a position where there is no matching tag, an error is - * returned (if it wasn't the last tag and it was OPTIONAL). After the - * right value has been detected it is returned in the out_buf. - * - */ -int decode_partial(ErlDrvBinary **drv_binary,unsigned char *in_buf, int in_buf_len) -{ - char *out_buf = (*drv_binary)->orig_bytes; - int tag_index_val = 1; - int msg_index_val; - int *msg_index, *tag_index, tmp_index; - int tag_seq_length; - int wanted_tag, next_tag; - int buf_end_index = in_buf_len; - int ret = 0, length, old_index; - - tag_index = &tag_index_val; - tag_seq_length = in_buf[0]; - msg_index = &msg_index_val; - *msg_index = tag_seq_length + 1; - - -/* printf("decode_partial 1: in_buf_len=%d, tag_index=%d, msg_index=%d\r\n,tag_seq_length=%d\r\n",in_buf_len,*tag_index,*msg_index,tag_seq_length); */ - while(*tag_index < tag_seq_length) { - switch(in_buf[*tag_index]) { - case ASN1_SKIPPED: -/* printf("decode_partial ASN1_SKIPPED: in_buf[*msg_index]=%d\r\n",in_buf[*msg_index]); */ - (*tag_index)++; -/* printf("decode_partial ASN1_SKIPPED 2: *msg_index=%d\r\n",*msg_index); */ - skip_tag(in_buf,msg_index,buf_end_index); -/* printf("decode_partial ASN1_SKIPPED 3: *msg_index=%d\r\n",*msg_index); */ - skip_length_and_value(in_buf,msg_index,buf_end_index); -/* printf("decode_partial ASN1_SKIPPED 4: *msg_index=%d\r\n",*msg_index); */ - break; - case ASN1_OPTIONAL: - (*tag_index)++; -/* printf("decode_partial ASN1_OPTIONAL: in_buf[*tag_index]=%d\r\n",in_buf[*tag_index]); */ - wanted_tag = in_buf[*tag_index]; - (*tag_index)++; - tmp_index = *msg_index; - next_tag = get_tag(in_buf,msg_index,buf_end_index); - if (wanted_tag != next_tag) { - *msg_index = tmp_index; - } else - skip_length_and_value(in_buf,msg_index,buf_end_index); - break; - case ASN1_CHOOSEN: -/* printf("decode_partial ASN1_CHOOSEN: in_buf[*msg_index]=%d, *msg_index=%d\r\n",in_buf[*msg_index],*msg_index); */ - (*tag_index)++; - wanted_tag = in_buf[*tag_index]; - (*tag_index)++; - old_index = *msg_index; -/* printf("decode_partial ASN1_CHOOSEN 2: *msg_index=%d\r\n",*msg_index); */ - next_tag = get_tag(in_buf,msg_index,buf_end_index); -/* printf("decode_partial ASN1_CHOOSEN 3: *msg_index=%d\r\n,wanted_tag=%d, next_tag=%d\r\n",*msg_index,wanted_tag,next_tag); */ - if (wanted_tag != next_tag) - return ASN1_NOVALUE; /* an empty binary will be returned to Erlang */ - if (*tag_index == (tag_seq_length + 1)) { - /* get the value and return*/ - if((ret = get_value(out_buf,in_buf,msg_index,buf_end_index)) <= ASN1_ERROR) - return ASN1_ERROR; - return ret; - } - else { - /* calculate the length of the sub buffer and let *msg_index - be at the value part of this BER encoded type*/ - int indef; - indef = 0; - length = get_length(in_buf,msg_index,&indef,buf_end_index); -/* printf("decode_partial ASN1_CHOOSEN 4: length=%d, *msg_index=%d\r\n",length,*msg_index); */ - if ((length == 0) && (indef == 1)) { - /* indefinite length of value */ - old_index = *msg_index; - length = skip_length_and_value(in_buf,msg_index,buf_end_index); - *msg_index = old_index; - buf_end_index = *msg_index + length - 2; - /* remove two bytes due to indefinete length end zeros */ - } else - buf_end_index = (*msg_index + length); - } - break; - default: - return ASN1_ERROR; - } - } - return ASN1_ERROR; -} - - -/* - * int skip_tag(unsigned char *in_buf,int *index,int buf_len) - * steps past the BER encoded tag in in_buf and updates *index. - * Returns the number of skipped bytes. - */ -int skip_tag(unsigned char *in_buf,int *index,int buf_len) -{ - int start_index = *index; - if ((MASK(in_buf[*index],ASN1_TAG)) == 31){ - do { - (*index)++; - if (*index >= buf_len) - return ASN1_ERROR; - } - while(in_buf[*index] >=128); - } - (*index)++; - return (*index - start_index); -} - - -/* - * int skip_length_and_value(unsigned char *in_buf,int *index,int buf_len) - * steps past the BER encoded length and value in in_buf and updates *index. - * returns the length if the skipped "length value". - * Returns the number of skipped bytes. - */ -int skip_length_and_value(unsigned char *in_buf,int *index,int buf_len) -{ - long len; - int indef = 0, lenoflen; - int start_index = *index; - - if ((MASK(in_buf[*index],0x80)) == ASN1_SHORT_DEFINITE_LENGTH){ - len = in_buf[*index]; - if (len > (buf_len - (*index + 1))) - return ASN1_LEN_ERROR; - } else if (in_buf[*index] == ASN1_INDEFINITE_LENGTH) - indef = 1; - else /* long definite length */ { - lenoflen = (in_buf[*index] & 0x7f); /*length of length */ - len = 0; - while (lenoflen--) { - (*index)++; - len = (len << 8) + in_buf[*index]; - } - if (len > (buf_len - (*index + 1))) - return ASN1_LEN_ERROR; - } - (*index)++; - if (indef == 1) - { - while(!(in_buf[*index]==0 && in_buf[*index + 1]==0)) { - skip_tag(in_buf,index,buf_len); - skip_length_and_value(in_buf,index,buf_len); - } - (*index) += 2; - } - else - (*index) += len; - return (*index - start_index); -} - -/* int get_tag(unsigned char *in_buf,int *index) - * - * assumes next byte/bytes in in_buf is an encoded BER tag. A tag - * number has theoretically no upper limit in size. Here the tag - * number is assumed to be less than 64K. Returns an integer value - * on the format: - * xxxxxxxx xxxxxxcc tttttttt tttttttt - * the x-bits are 0 (insignificant) - * the c-bits are the class of the tag - * the t-bits are the tag number. This implies that the tag number - * is limited to 64K-1 - * - */ -int get_tag(unsigned char *in_buf,int *index,int buf_len) -{ - int tag_no = 0,tmp_tag = 0; - - tag_no = (MASK(in_buf[*index],ASN1_CLASSFORM)); - if ((MASK(in_buf[*index],ASN1_TAG)) == ASN1_TAG) { - /* long form of tag */ - do { - (*index)++; - if (*index >= buf_len) - return ASN1_TAG_ERROR; - tmp_tag = tmp_tag << 7; - tmp_tag += (MASK(in_buf[*index],ASN1_LONG_TAG)); - } while (in_buf[*index] >= 128); - (*index)++; - tag_no = tag_no + tmp_tag; - } else { - tag_no += (MASK(in_buf[*index],ASN1_TAG)); - (*index)++; - } - if (*index >= buf_len) - return ASN1_TAG_ERROR; - return tag_no; -} - - -/* - * int get_value(char *out_buf,unsigned char *in_buf, - * int *msg_index,int in_buf_len) - */ -/* assumes next byte/bytes in in_buf is an encoded BER value preceeded by a BER encoded length. Puts value in out_buf. - */ -int get_value(char *out_buf, - unsigned char *in_buf, - int *msg_index, - int in_buf_len) -{ - int len, lenoflen, indef=0, skip_len; - int ret=0; - int start_index; - -/* printf("get_value 1\n\r"); */ - if (in_buf[*msg_index] < 0x80){ /* short definite length */ - len = in_buf[*msg_index]; -/* printf("short definite length\r\n"); */ - } else if (in_buf[*msg_index] > 0x80) { /* long definite length */ - lenoflen = (in_buf[*msg_index] & 0x7f); /*length of length */ - len = 0; - while (lenoflen--) { - (*msg_index)++; - len = (len << 8) + in_buf[*msg_index]; - } - if (len > (in_buf_len - (*msg_index + 1))) - return ASN1_LEN_ERROR; - } else - indef = 1; - (*msg_index)++; -/* printf("get_value 2: len = %d, *msg_index = %d\r\n",len,*msg_index); */ - if (indef == 1) { - while(!(in_buf[*msg_index]==0 && in_buf[*msg_index + 1]==0)) { - start_index = *msg_index; - skip_len = skip_tag(in_buf,msg_index,in_buf_len); -/* printf("get_value 3: skip_len=%d,start_index=%d,*msg_index=%d\n\r", */ -/* skip_len,start_index,*msg_index); */ - memcpy(&out_buf[ret],&in_buf[start_index],skip_len); - ret += skip_len; - start_index = *msg_index; - skip_len = skip_length_and_value(in_buf,msg_index,in_buf_len); -/* printf("get_value 4: skip_len=%d,start_index=%d,*msg_index=%d\n\r", */ -/* skip_len,start_index,*msg_index); */ - memcpy(&out_buf[ret],&in_buf[start_index],skip_len); - ret += skip_len; - } - return ret; - } - else - memcpy(&out_buf[ret],&in_buf[*msg_index],len); - return len; -} - - -/* - * int get_length(unsigned char *in_buf,int *msg_index) - * assumes next byte/bytes contain a BER encoded length field, - * which is decoded. The value of the length is returned. If it - * is an indefinite length the *indef is set to one. - */ -int get_length(unsigned char *in_buf,int *msg_index, - int *indef,int in_buf_len) -{ - int len=0, lenoflen; - - if (in_buf[*msg_index] < 0x80) /* short definite length */ - len = in_buf[*msg_index]; - else if (in_buf[*msg_index] > 0x80) { /* long definite length */ - lenoflen = (in_buf[*msg_index] & 0x7f); /*length of length */ - len = 0; - while (lenoflen--) { - (*msg_index)++; - len = (len << 8) + in_buf[*msg_index]; - } - if (len > (in_buf_len - (*msg_index + 1))) - return ASN1_LEN_ERROR; - } else - *indef = 1; - (*msg_index)++; - return len; -} diff --git a/lib/asn1/c_src/asn1_erl_nif.c b/lib/asn1/c_src/asn1_erl_nif.c new file mode 100644 index 0000000000..58314e23c2 --- /dev/null +++ b/lib/asn1/c_src/asn1_erl_nif.c @@ -0,0 +1,1079 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2002-2011. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + * + */ +#include <stdlib.h> +#include <stdio.h> +#include <string.h> +#include "erl_nif.h" + +/* #define ASN1_DEBUG 1 */ + +#define ASN1_OK 0 +#define ASN1_ERROR -1 +#define ASN1_COMPL_ERROR 1 +#define ASN1_MEMORY_ERROR 0 +#define ASN1_DECODE_ERROR 2 +#define ASN1_TAG_ERROR -3 +#define ASN1_LEN_ERROR -4 +#define ASN1_INDEF_LEN_ERROR -5 +#define ASN1_VALUE_ERROR -6 + +#define ASN1_CLASS 0xc0 +#define ASN1_FORM 0x20 +#define ASN1_CLASSFORM (ASN1_CLASS | ASN1_FORM) +#define ASN1_TAG 0x1f +#define ASN1_LONG_TAG 0x7f + +#define ASN1_INDEFINITE_LENGTH 0x80 +#define ASN1_SHORT_DEFINITE_LENGTH 0 + +#define ASN1_PRIMITIVE 0 +#define ASN1_CONSTRUCTED 0x20 + +#define ASN1_COMPLETE 1 +#define ASN1_BER_TLV_DECODE 2 +#define ASN1_BER_TLV_PARTIAL_DECODE 3 + +#define ASN1_NOVALUE 0 + +#define ASN1_SKIPPED 0 +#define ASN1_OPTIONAL 1 +#define ASN1_CHOOSEN 2 + +#define CEIL(X,Y) ((X-1) / Y + 1) + +#define INVMASK(X,M) (X & (M ^ 0xff)) +#define MASK(X,M) (X & M) + +int complete(ErlNifBinary *, unsigned char *, int ); + +int insert_octets(int, unsigned char **, unsigned char **, int *); + +int insert_octets_except_unused(int, unsigned char **, unsigned char **, int *, + int); + +int insert_octets_as_bits_exact_len(int, int, unsigned char **, + unsigned char **, int *); + +int insert_octets_as_bits(int, unsigned char **, unsigned char **, int *); + +int pad_bits(int, unsigned char **, int *); + +int insert_least_sign_bits(int, unsigned char, unsigned char **, int *); + +int insert_most_sign_bits(int, unsigned char, unsigned char **, int *); + +int insert_bits_as_bits(int, int, unsigned char **, unsigned char **, int *); + +int insert_octets_unaligned(int, unsigned char **, unsigned char **, int); + +int realloc_memory(ErlNifBinary *, int, unsigned char **); + +int decode_begin(ErlNifEnv *, ERL_NIF_TERM *, unsigned char *, int, + unsigned int *); + +int decode(ErlNifEnv *, ERL_NIF_TERM *, unsigned char *, int *, int); + +int decode_tag(ErlNifEnv *, ERL_NIF_TERM *, unsigned char *, int, int *); + +int decode_value(ErlNifEnv*, ERL_NIF_TERM *, unsigned char *, int *, int, int); + +/* + * + * This section defines functionality for the complete encode of a + * PER encoded message + * + */ + +int complete(ErlNifBinary *out_binary, unsigned char *in_buf, int in_buf_len) { + int counter = in_buf_len; + /* counter keeps track of number of bytes left in the + input buffer */ + + int buf_space = in_buf_len; + /* This is the amount of allocated space left of the out_binary. It + is possible when padding is applied that more space is needed than + was originally allocated. */ + + int buf_size = in_buf_len; + /* Size of the buffer. May become reallocated and thus other than + in_buf_len */ + + unsigned char *in_ptr, *ptr; + /* in_ptr points at the next byte in in_buf to be moved to + complete_buf. + ptr points into the new completed buffer, complete_buf, at the + position of the next byte that will be set */ + int unused = 8; + /* unused = [1,...,8] indicates how many of the rigthmost bits of + the byte that ptr points at that are unassigned */ + + int no_bits, no_bytes, in_unused, desired_len, ret, saved_mem, needed, + pad_bits; + + unsigned char val; + + in_ptr = in_buf; + ptr = out_binary->data; + *ptr = 0x00; + while (counter > 0) { + counter--; + switch (*in_ptr) { + case 0: + /* just one zero-bit should be added to the buffer */ + if (unused == 1) { + unused = 8; + *++ptr = 0x00; + buf_space--; + } else + unused--; + break; + + case 1: + /* one one-bit should be added to the buffer */ + if (unused == 1) { + *ptr = *ptr | 1; + unused = 8; + *++ptr = 0x00; + buf_space--; + } else { + *ptr = *ptr | (1 << (unused - 1)); + unused--; + } + break; + + case 2: + /* align buffer to end of byte */ + if (unused != 8) { + *++ptr = 0x00; + buf_space--; + unused = 8; + } + break; + + case 10: + /* next byte in in_buf tells how many bits in the second next + byte that will be used */ + /* The leftmost unused bits in the value byte are supposed to be + zero bits */ + no_bits = (int) *(++in_ptr); + val = *(++in_ptr); + counter -= 2; + if ((ret = insert_least_sign_bits(no_bits, val, &ptr, &unused)) + == ASN1_ERROR + ) + return ASN1_ERROR; + buf_space -= ret; + break; + + case 20: + /* in this case the next value in_ptr points at holds the number + of following bytes that holds the value that will be inserted + in the completed buffer */ + no_bytes = (int) *(++in_ptr); + counter -= (no_bytes + 1); + if ((counter < 0) + || (ret = insert_octets(no_bytes, &in_ptr, &ptr, &unused)) + == ASN1_ERROR + ) + return ASN1_ERROR; + buf_space -= ret; + break; + + case 21: + /* in this case the next two bytes in_ptr points at holds the number + of following bytes that holds the value that will be inserted + in the completed buffer */ + no_bytes = (int) *(++in_ptr); + no_bytes = no_bytes << 8; + no_bytes = no_bytes | (int) *(++in_ptr); + counter -= (2 + no_bytes); + if ((counter < 0) + || (ret = insert_octets(no_bytes, &in_ptr, &ptr, &unused)) + == ASN1_ERROR + ) + return ASN1_ERROR; + buf_space -= ret; + break; + + case 30: + /* If we call the following bytes, in the buffer in_ptr points at, + By1,By2,Rest then Rest is the value that will be transfered to + the completed buffer. By1 tells how many of the rightmost bits in + Rest that should not be used. By2 is the length of Rest in bytes.*/ + in_unused = (int) *(++in_ptr); + no_bytes = (int) *(++in_ptr); + counter -= (2 + no_bytes); + ret = -4711; + if ((counter < 0) + || (ret = insert_octets_except_unused(no_bytes, &in_ptr, + &ptr, &unused, in_unused)) == ASN1_ERROR + ) + return ASN1_ERROR; + buf_space -= ret; + break; + + case 31: + /* If we call the following bytes, in the buffer in_ptr points at, + By1,By2,By3,Rest then Rest is the value that will be transfered to + the completed buffer. By1 tells how many of the rightmost bits in + Rest that should not be used. By2 and By3 is the length of + Rest in bytes.*/ + in_unused = (int) *(++in_ptr); + no_bytes = (int) *(++in_ptr); + no_bytes = no_bytes << 8; + no_bytes = no_bytes | (int) *(++in_ptr); + counter -= (3 + no_bytes); + if ((counter < 0) + || (ret = insert_octets_except_unused(no_bytes, &in_ptr, + &ptr, &unused, in_unused)) == ASN1_ERROR + ) + return ASN1_ERROR; + buf_space -= ret; + break; + + case 40: + /* This case implies that next byte,By1,(..,By1,By2,Bin,...) + is the desired length of the completed value, maybe needs + padding zero bits or removal of trailing zero bits from Bin. + By2 is the length of Bin and Bin is the value that will be + put into the completed buffer. Each byte in Bin has the value + 1 or 0.*/ + desired_len = (int) *(++in_ptr); + no_bytes = (int) *(++in_ptr); + + /* This is the algorithm for need of memory reallocation: + Only when padding (cases 40 - 43,45 - 47) more memory may be + used than allocated. Therefore one has to keep track of how + much of the allocated memory that has been saved, i.e. the + difference between the number of parsed bytes of the input buffer + and the number of used bytes of the output buffer. + If saved memory is less than needed for the padding then we + need more memory. */ + saved_mem = buf_space - counter; + pad_bits = desired_len - no_bytes - unused; + needed = (pad_bits > 0) ? CEIL(pad_bits,8) : 0; + if (saved_mem < needed) { + /* Have to allocate more memory */ + buf_size += needed; + buf_space += needed; + if (realloc_memory(out_binary, buf_size, &ptr) + == ASN1_ERROR + ) + return ASN1_ERROR; + } + + counter -= (2 + no_bytes); + if ((counter < 0) + || (ret = insert_octets_as_bits_exact_len(desired_len, + no_bytes, &in_ptr, &ptr, &unused)) == ASN1_ERROR + ) + return ASN1_ERROR; + buf_space -= ret; + break; + + case 41: + /* Same as case 40 apart from By2, the length of Bin, which is in + two bytes*/ + desired_len = (int) *(++in_ptr); + no_bytes = (int) *(++in_ptr); + no_bytes = no_bytes << 8; + no_bytes = no_bytes | (int) *(++in_ptr); + + saved_mem = buf_space - counter; + needed = CEIL((desired_len-unused),8) - no_bytes; + if (saved_mem < needed) { + /* Have to allocate more memory */ + buf_size += needed; + buf_space += needed; + if (realloc_memory(out_binary, buf_size, &ptr) + == ASN1_ERROR + ) + return ASN1_ERROR; + } + + counter -= (3 + no_bytes); + if ((counter < 0) + || (ret = insert_octets_as_bits_exact_len(desired_len, + no_bytes, &in_ptr, &ptr, &unused)) == ASN1_ERROR + ) + return ASN1_ERROR; + buf_space -= ret; + break; + + case 42: + /* Same as case 40 apart from By1, the desired length, which is in + two bytes*/ + desired_len = (int) *(++in_ptr); + desired_len = desired_len << 8; + desired_len = desired_len | (int) *(++in_ptr); + no_bytes = (int) *(++in_ptr); + + saved_mem = buf_space - counter; + needed = CEIL((desired_len-unused),8) - no_bytes; + if (saved_mem < needed) { + /* Have to allocate more memory */ + buf_size += needed; + buf_space += needed; + if (realloc_memory(out_binary, buf_size, &ptr) + == ASN1_ERROR + ) + return ASN1_ERROR; + } + + counter -= (3 + no_bytes); + if ((counter < 0) + || (ret = insert_octets_as_bits_exact_len(desired_len, + no_bytes, &in_ptr, &ptr, &unused)) == ASN1_ERROR + ) + return ASN1_ERROR; + buf_space -= ret; + break; + + case 43: + /* Same as case 40 apart from By1 and By2, the desired length and + the length of Bin, which are in two bytes each. */ + desired_len = (int) *(++in_ptr); + desired_len = desired_len << 8; + desired_len = desired_len | (int) *(++in_ptr); + no_bytes = (int) *(++in_ptr); + no_bytes = no_bytes << 8; + no_bytes = no_bytes | (int) *(++in_ptr); + + saved_mem = buf_space - counter; + needed = CEIL((desired_len-unused),8) - no_bytes; + if (saved_mem < needed) { + /* Have to allocate more memory */ + buf_size += needed; + buf_space += needed; + if (realloc_memory(out_binary, buf_size, &ptr) + == ASN1_ERROR + ) + return ASN1_ERROR; + } + + counter -= (4 + no_bytes); + if ((counter < 0) + || (ret = insert_octets_as_bits_exact_len(desired_len, + no_bytes, &in_ptr, &ptr, &unused)) == ASN1_ERROR + ) + return ASN1_ERROR; + buf_space -= ret; + break; + + case 45: + /* This case assumes that the following bytes in the incoming buffer + (called By1,By2,Bin) is By1, which is the number of bits (n) that + will be inserted in the completed buffer. By2 is the number of + bytes in Bin. Each bit in the buffer Bin should be inserted from + the leftmost until the nth.*/ + desired_len = (int) *(++in_ptr); + no_bytes = (int) *(++in_ptr); + + saved_mem = buf_space - counter; + needed = CEIL((desired_len-unused),8) - no_bytes; + if (saved_mem < needed) { + /* Have to allocate more memory */ + buf_size += needed; + buf_space += needed; + if (realloc_memory(out_binary, buf_size, &ptr) + == ASN1_ERROR + ) + return ASN1_ERROR; + } + + counter -= (2 + no_bytes); + + if ((counter < 0) + || (ret = insert_bits_as_bits(desired_len, no_bytes, + &in_ptr, &ptr, &unused)) == ASN1_ERROR + ) + return ASN1_ERROR; + buf_space -= ret; + break; + + case 46: + /* Same as case 45 apart from By1, the desired length, which is + in two bytes. */ + desired_len = (int) *(++in_ptr); + desired_len = desired_len << 8; + desired_len = desired_len | (int) *(++in_ptr); + no_bytes = (int) *(++in_ptr); + + saved_mem = buf_space - counter; + needed = CEIL((desired_len-unused),8) - no_bytes; + if (saved_mem < needed) { + /* Have to allocate more memory */ + buf_size += needed; + buf_space += needed; + if (realloc_memory(out_binary, buf_size, &ptr) + == ASN1_ERROR + ) + return ASN1_ERROR; + } + + counter -= (3 + no_bytes); + if ((counter < 0) + || (ret = insert_bits_as_bits(desired_len, no_bytes, + &in_ptr, &ptr, &unused)) == ASN1_ERROR + ) + return ASN1_ERROR; + buf_space -= ret; + break; + + case 47: + /* Same as case 45 apart from By1 and By2, the desired length + and the length of Bin, which are in two bytes each. */ + desired_len = (int) *(++in_ptr); + desired_len = desired_len << 8; + desired_len = desired_len | (int) *(++in_ptr); + no_bytes = (int) *(++in_ptr); + no_bytes = no_bytes << 8; + no_bytes = no_bytes | (int) *(++in_ptr); + + saved_mem = buf_space - counter; + needed = CEIL((desired_len-unused),8) - no_bytes; + if (saved_mem < needed) { + /* Have to allocate more memory */ + buf_size += needed; + buf_space += needed; + if (realloc_memory(out_binary, buf_size, &ptr) + == ASN1_ERROR + ) + return ASN1_ERROR; + } + + counter -= (4 + no_bytes); + if ((counter < 0) + || (ret = insert_bits_as_bits(desired_len, no_bytes, + &in_ptr, &ptr, &unused)) == ASN1_ERROR + ) + return ASN1_ERROR; + buf_space -= ret; + break; + + default: + return ASN1_ERROR; + } + in_ptr++; + } + /* The returned buffer must be at least one byte and + it must be octet aligned */ + if ((unused == 8) && (ptr != out_binary->data)) + return (ptr - out_binary->data); + else { + ptr++; /* octet align buffer */ + return (ptr - out_binary->data); + } +} + +int realloc_memory(ErlNifBinary *binary, int amount, unsigned char **ptr) { + + int i = *ptr - binary->data; + + if (!enif_realloc_binary(binary, amount)) { + /*error handling due to memory allocation failure */ + return ASN1_ERROR; + } else { + *ptr = binary->data + i; + } + return ASN1_OK; +} + +int insert_most_sign_bits(int no_bits, unsigned char val, + unsigned char **output_ptr, int *unused) { + unsigned char *ptr = *output_ptr; + + if (no_bits < *unused) { + *ptr = *ptr | (val >> (8 - *unused)); + *unused -= no_bits; + } else if (no_bits == *unused) { + *ptr = *ptr | (val >> (8 - *unused)); + *unused = 8; + *++ptr = 0x00; + } else { + *ptr = *ptr | (val >> (8 - *unused)); + *++ptr = 0x00; + *ptr = *ptr | (val << *unused); + *unused = 8 - (no_bits - *unused); + } + *output_ptr = ptr; + return ASN1_OK; +} + +int insert_least_sign_bits(int no_bits, unsigned char val, + unsigned char **output_ptr, int *unused) { + unsigned char *ptr = *output_ptr; + int ret = 0; + + if (no_bits < *unused) { + *ptr = *ptr | (val << (*unused - no_bits)); + *unused -= no_bits; + } else if (no_bits == *unused) { + *ptr = *ptr | val; + *unused = 8; + *++ptr = 0x00; + ret++; + } else { + /* first in the begun byte in the completed buffer insert + so many bits that fit, then insert the rest in next byte.*/ + *ptr = *ptr | (val >> (no_bits - *unused)); + *++ptr = 0x00; + ret++; + *ptr = *ptr | (val << (8 - (no_bits - *unused))); + *unused = 8 - (no_bits - *unused); + } + *output_ptr = ptr; + return ret; +} + +/* pad_bits adds no_bits bits in the buffer that output_ptr + points at. + */ +int pad_bits(int no_bits, unsigned char **output_ptr, int *unused) { + unsigned char *ptr = *output_ptr; + int ret = 0; + + while (no_bits > 0) { + if (*unused == 1) { + *unused = 8; + *++ptr = 0x00; + ret++; + } else + (*unused)--; + no_bits--; + } + *output_ptr = ptr; + return ret; +} + +/* insert_bits_as_bits removes no_bytes bytes from the buffer that in_ptr + points at and takes the desired_no leftmost bits from those removed + bytes and inserts them in the buffer(output buffer) that ptr points at. + The unused parameter tells how many bits that are not set in the + actual byte in the output buffer. If desired_no is more bits than the + input buffer has in no_bytes bytes, then zero bits is padded.*/ +int insert_bits_as_bits(int desired_no, int no_bytes, unsigned char **input_ptr, + unsigned char **output_ptr, int *unused) { + unsigned char *in_ptr = *input_ptr; + unsigned char val; + int no_bits, ret, ret2; + + if (desired_no == (no_bytes * 8)) { + if (insert_octets_unaligned(no_bytes, &in_ptr, output_ptr, *unused) + == ASN1_ERROR + ) + return ASN1_ERROR; + ret = no_bytes; + } else if (desired_no < (no_bytes * 8)) { + /* printf("insert_bits_as_bits 1\n\r"); */ + if (insert_octets_unaligned(desired_no / 8, &in_ptr, output_ptr, + *unused) == ASN1_ERROR + ) + return ASN1_ERROR; + /* printf("insert_bits_as_bits 2\n\r"); */ + val = *++in_ptr; + /* printf("val = %d\n\r",(int)val); */ + no_bits = desired_no % 8; + /* printf("no_bits = %d\n\r",no_bits); */ + insert_most_sign_bits(no_bits, val, output_ptr, unused); + ret = CEIL(desired_no,8); + } else { + if (insert_octets_unaligned(no_bytes, &in_ptr, output_ptr, *unused) + == ASN1_ERROR + ) + return ASN1_ERROR; + ret2 = pad_bits(desired_no - (no_bytes * 8), output_ptr, unused); + /* printf("ret2 = %d\n\r",ret2); */ + ret = CEIL(desired_no,8); + /* printf("ret = %d\n\r",ret); */ + } + /* printf("*unused = %d\n\r",*unused); */ + *input_ptr = in_ptr; + return ret; +} + +/* insert_octets_as_bits_exact_len */ +int insert_octets_as_bits_exact_len(int desired_len, int in_buff_len, + unsigned char **in_ptr, unsigned char **ptr, int *unused) { + int ret = 0; + int ret2 = 0; + + if (desired_len == in_buff_len) { + if ((ret = insert_octets_as_bits(in_buff_len, in_ptr, ptr, unused)) + == ASN1_ERROR + ) + return ASN1_ERROR; + } else if (desired_len > in_buff_len) { + if ((ret = insert_octets_as_bits(in_buff_len, in_ptr, ptr, unused)) + == ASN1_ERROR + ) + return ASN1_ERROR; + /* now pad with zero bits */ + /* printf("~npad_bits: called with %d bits padding~n~n~r",desired_len - in_buff_len); */ + if ((ret2 = pad_bits(desired_len - in_buff_len, ptr, unused)) + == ASN1_ERROR + ) + return ASN1_ERROR; + } else {/* desired_len < no_bits */ + if ((ret = insert_octets_as_bits(desired_len, in_ptr, ptr, unused)) + == ASN1_ERROR + ) + return ASN1_ERROR; + /* now remove no_bits - desired_len bytes from in buffer */ + *in_ptr += (in_buff_len - desired_len); + } + return (ret + ret2); +} + +/* insert_octets_as_bits takes no_bytes bytes from the buffer that input_ptr + points at and inserts the least significant bit of it in the buffer that + output_ptr points at. Each byte in the input buffer must be 1 or 0 + otherwise the function returns ASN1_ERROR. The output buffer is concatenated + without alignment. + */ +int insert_octets_as_bits(int no_bytes, unsigned char **input_ptr, + unsigned char **output_ptr, int *unused) { + unsigned char *in_ptr = *input_ptr; + unsigned char *ptr = *output_ptr; + int used_bits = 8 - *unused; + + while (no_bytes > 0) { + switch (*++in_ptr) { + case 0: + if (*unused == 1) { + *unused = 8; + *++ptr = 0x00; + } else + (*unused)--; + break; + case 1: + if (*unused == 1) { + *ptr = *ptr | 1; + *unused = 8; + *++ptr = 0x00; + } else { + *ptr = *ptr | (1 << (*unused - 1)); + (*unused)--; + } + break; + default: + return ASN1_ERROR; + } + no_bytes--; + } + *input_ptr = in_ptr; + *output_ptr = ptr; + return ((used_bits + no_bytes) / 8); /*return number of new bytes + in completed buffer */ +} + +/* insert_octets inserts bytes from the input buffer, *input_ptr, + into the output buffer, *output_ptr. Before the first byte is + inserted the input buffer is aligned. + */ +int insert_octets(int no_bytes, unsigned char **input_ptr, + unsigned char **output_ptr, int *unused) { + unsigned char *in_ptr = *input_ptr; + unsigned char *ptr = *output_ptr; + int ret = 0; + + if (*unused != 8) {/* must align before octets are added */ + *++ptr = 0x00; + ret++; + *unused = 8; + } + while (no_bytes > 0) { + *ptr = *(++in_ptr); + *++ptr = 0x00; + /* *unused = *unused - 1; */ + no_bytes--; + } + *input_ptr = in_ptr; + *output_ptr = ptr; + return (ret + no_bytes); +} + +/* insert_octets_unaligned inserts bytes from the input buffer, *input_ptr, + into the output buffer, *output_ptr.No alignment is done. + */ +int insert_octets_unaligned(int no_bytes, unsigned char **input_ptr, + unsigned char **output_ptr, int unused) { + unsigned char *in_ptr = *input_ptr; + unsigned char *ptr = *output_ptr; + int n = no_bytes; + unsigned char val; + + while (n > 0) { + if (unused == 8) { + *ptr = *++in_ptr; + *++ptr = 0x00; + } else { + val = *++in_ptr; + *ptr = *ptr | val >> (8 - unused); + *++ptr = 0x00; + *ptr = val << unused; + } + n--; + } + *input_ptr = in_ptr; + *output_ptr = ptr; + return no_bytes; +} + +int insert_octets_except_unused(int no_bytes, unsigned char **input_ptr, + unsigned char **output_ptr, int *unused, int in_unused) { + unsigned char *in_ptr = *input_ptr; + unsigned char *ptr = *output_ptr; + int val, no_bits; + int ret = 0; + + if (in_unused == 0) { + if ((ret = insert_octets_unaligned(no_bytes, &in_ptr, &ptr, *unused)) + == ASN1_ERROR + ) + return ASN1_ERROR; + } else { + if ((ret = insert_octets_unaligned(no_bytes - 1, &in_ptr, &ptr, *unused)) + != ASN1_ERROR) { + val = (int) *(++in_ptr); + no_bits = 8 - in_unused; + /* no_bits is always less than *unused since the buffer is + octet aligned after insert:octets call, so the following + if clasuse is obsolete I think */ + if (no_bits < *unused) { + *ptr = *ptr | (val >> (8 - *unused)); + *unused = *unused - no_bits; + } else if (no_bits == *unused) { + *ptr = *ptr | (val >> (8 - *unused)); + *++ptr = 0x00; + ret++; + *unused = 8; + } else { + *ptr = *ptr | (val >> (8 - *unused)); + *++ptr = 0x00; + ret++; + *ptr = *ptr | (val << *unused); + *unused = 8 - (no_bits - *unused); + } + } else + return ASN1_ERROR; + } + *input_ptr = in_ptr; + *output_ptr = ptr; + return ret; +} + +/* + * + * This section defines functionality for the partial decode of a + * BER encoded message + * + */ + +/* + * int decode(ErlNifEnv* env, ERL_NIF_TERM *term, unsigned char *in_buf, + int in_buf_len, unsigned int *err_pos) + * term is a pointer to the term which is to be returned to erlang + * in_buf is a pointer into the buffer of incoming bytes. + * in_buf_len is the length of the incoming buffer. + * The function reads the bytes in the incoming buffer and structures + * it in a nested way as Erlang terms. The buffer contains data in the + * order tag - length - value. Tag, length and value has the following + * format: + * A tag is normally one byte but may be of any length, if the tag number + * is greater than 30. +----------+ + * |CL|C|NNNNN| + * +----------+ + * If NNNNN is 31 then will the 7 l.s.b of each of the following tag number + * bytes contain the tag number. Each tag number byte that is not the last one + * has the m.s.b. set to 1. + * The length can be short definite length (sdl), long definite length (ldl) + * or indefinite length (il). + * sdl: +---------+ the L bits is the length + * |0|LLLLLLL| + * +---------+ + * ldl: +---------+ +---------+ +---------+ +-----------+ + * |1|lllllll| |first len| | | |the Nth len| + * +---------+ +---------+ +---------+ ... +-----------+ + * The first byte tells how many len octets will follow, max 127 + * il: +---------+ +----------------------+ +--------+ +--------+ + * |1|0000000| |content octets (Value)| |00000000| |00000000| + * +---------+ +----------------------+ +--------+ +--------+ + * The value octets are preceded by one octet and followed by two + * exactly as above. The value must be some tag-length-value encoding. + * + * The function returns a value in Erlang nif term format: + * {{TagNo,Value},Rest} + * TagNo is an integer ((CL bsl 16) + tag number) which limits the tag number + * to 65535. + * Value is a binary if the C bit in tag was unset, otherwise (if tag was + * constructed) Value is a list, List. + * List is like: [{TagNo,Value},{TagNo,Value},...] + * Rest is a binary, i.e. the undecoded part of the buffer. Most often Rest + * is the empty binary. + * If some error occured during the decoding of the in_buf an error is returned. + */ +int decode_begin(ErlNifEnv* env, ERL_NIF_TERM *term, unsigned char *in_buf, + int in_buf_len, unsigned int *err_pos) { + int maybe_ret; + int ib_index = 0; + unsigned char *rest_data; + ERL_NIF_TERM decoded_term, rest; + + if ((maybe_ret = decode(env, &decoded_term, in_buf, &ib_index, in_buf_len)) + <= ASN1_ERROR) + { + *err_pos = ib_index; + return maybe_ret; + }; + + // The remaining binary after one ASN1 segment has been decoded + if ((rest_data = enif_make_new_binary(env, in_buf_len - ib_index, &rest)) + == NULL) { + *term = enif_make_atom(env, "could_not_alloc_binary"); + return ASN1_ERROR; + } + + *term = enif_make_tuple2(env, decoded_term, rest); + return ASN1_OK; +} + +int decode(ErlNifEnv* env, ERL_NIF_TERM *term, unsigned char *in_buf, + int *ib_index, int in_buf_len) { + int maybe_ret; + int form; + ERL_NIF_TERM tag, value; + + /*buffer must hold at least two bytes*/ + if ((*ib_index + 2) > in_buf_len) + return ASN1_VALUE_ERROR; + /* "{{TagNo," */ + if ((form = decode_tag(env, &tag, in_buf, in_buf_len, ib_index)) + <= ASN1_ERROR + ) + return form; /* 5 bytes */ + if (*ib_index >= in_buf_len) { + return ASN1_TAG_ERROR; + } + /* buffer must hold at least one byte (0 as length and nothing as + value) */ + /* "{{TagNo,Value}," */ + if ((maybe_ret = decode_value(env, &value, in_buf, ib_index, form, + in_buf_len)) <= ASN1_ERROR + ) + return maybe_ret; /* at least 5 bytes */ + *term = enif_make_tuple2(env, tag, value); + return ASN1_OK; +} + +/* + * decode_tag decodes the BER encoded tag in in_buf and creates an + * nif term tag + */ +int decode_tag(ErlNifEnv* env, ERL_NIF_TERM *tag, unsigned char *in_buf, + int in_buf_len, int *ib_index) { + int tag_no, tmp_tag, form; + + /* first get the class of tag and bit shift left 16*/ + tag_no = ((MASK(in_buf[*ib_index],ASN1_CLASS)) << 10); + + form = (MASK(in_buf[*ib_index],ASN1_FORM)); + + /* then get the tag number */ + if ((tmp_tag = (int) INVMASK(in_buf[*ib_index],ASN1_CLASSFORM)) < 31) { + *tag = enif_make_uint(env, tag_no + tmp_tag); + (*ib_index)++; + } else { + int n = 0; /* n is used to check that the 64K limit is not + exceeded*/ + + /* should check that at least three bytes are left in + in-buffer,at least two tag byte and at least one length byte */ + if ((*ib_index + 3) > in_buf_len) + return ASN1_VALUE_ERROR; + (*ib_index)++; + /* The tag is in the following bytes in in_buf as + 1ttttttt 1ttttttt ... 0ttttttt, where the t-bits + is the tag number*/ + /* In practice is the tag size limited to 64K, i.e. 16 bits. If + the tag is greater then 64K return an error */ + while (((tmp_tag = (int) in_buf[*ib_index]) >= 128) && n < 2) { + /* m.s.b. = 1 */ + tag_no = tag_no + (MASK(tmp_tag,ASN1_LONG_TAG) << 7); + (*ib_index)++; + n++; + }; + if ((n == 2) && in_buf[*ib_index] > 3) + return ASN1_TAG_ERROR; /* tag number > 64K */ + tag_no = tag_no + in_buf[*ib_index]; + (*ib_index)++; + *tag = enif_make_uint(env, tag_no); + } + return form; +} + +/* + * decode_value decodes the BER encoded length and value fields in the + * in_buf and puts the value part in the decode_buf as an Erlang + * nif term into value + */ +int decode_value(ErlNifEnv* env, ERL_NIF_TERM *value, unsigned char *in_buf, + int *ib_index, int form, int in_buf_len) { + int maybe_ret; + unsigned int len = 0; + unsigned int lenoflen = 0; + int indef = 0; + unsigned char *tmp_out_buff; + ERL_NIF_TERM term = 0, curr_head = 0; + + if (((in_buf[*ib_index]) & 0x80) == ASN1_SHORT_DEFINITE_LENGTH) { + len = in_buf[*ib_index]; + } else if (in_buf[*ib_index] == ASN1_INDEFINITE_LENGTH + ) + indef = 1; + else /* long definite length */{ + lenoflen = (in_buf[*ib_index] & 0x7f); /*length of length */ + if (lenoflen > (in_buf_len - (*ib_index + 1))) + return ASN1_LEN_ERROR; + len = 0; + while (lenoflen--) { + (*ib_index)++; + if (!(len < (1 << (sizeof(len) - 1) * 8))) + return ASN1_LEN_ERROR; /* length does not fit in 32 bits */ + len = (len << 8) + in_buf[*ib_index]; + } + } + if (len > (in_buf_len - (*ib_index + 1))) + return ASN1_VALUE_ERROR; + (*ib_index)++; + if (indef == 1) { /* in this case it is desireably to check that indefinite length + end bytes exist in inbuffer */ + curr_head = enif_make_list(env, 0); + while (!(in_buf[*ib_index] == 0 && in_buf[*ib_index + 1] == 0)) { + if (*ib_index >= in_buf_len) + return ASN1_INDEF_LEN_ERROR; + + if ((maybe_ret = decode(env, &term, in_buf, ib_index, in_buf_len)) + <= ASN1_ERROR + ) + return maybe_ret; + curr_head = enif_make_list_cell(env, term, curr_head); + } + enif_make_reverse_list(env, curr_head, value); + (*ib_index) += 2; /* skip the indefinite length end bytes */ + } else if (form == ASN1_CONSTRUCTED) + { + int end_index = *ib_index + len; + if (end_index > in_buf_len) + return ASN1_LEN_ERROR; + curr_head = enif_make_list(env, 0); + while (*ib_index < end_index) { + + if ((maybe_ret = decode(env, &term, in_buf, ib_index, in_buf_len)) + <= ASN1_ERROR + ) + return maybe_ret; + curr_head = enif_make_list_cell(env, term, curr_head); + } + enif_make_reverse_list(env, curr_head, value); + } else { + if ((*ib_index + len) > in_buf_len) + return ASN1_LEN_ERROR; + tmp_out_buff = enif_make_new_binary(env, len, value); + memcpy(tmp_out_buff, in_buf + *ib_index, len); + *ib_index = *ib_index + len; + } + return ASN1_OK; +} + +static ERL_NIF_TERM encode_per_complete(ErlNifEnv* env, int argc, + const ERL_NIF_TERM argv[]) { + ERL_NIF_TERM err_code; + ErlNifBinary in_binary; + ErlNifBinary out_binary; + int complete_len; + if (!enif_inspect_iolist_as_binary(env, argv[0], &in_binary)) + return enif_make_atom(env, "badarg"); + + if (!enif_alloc_binary(in_binary.size, &out_binary)) + return enif_make_atom(env, "alloc_binary_failed"); + + if (in_binary.size == 0) + return enif_make_binary(env, &out_binary); + + if ((complete_len = complete(&out_binary, in_binary.data, in_binary.size)) + <= ASN1_ERROR) { + enif_release_binary(&out_binary); + if (complete_len == ASN1_ERROR + ) + err_code = enif_make_uint(env, '1'); + else + err_code = enif_make_uint(env, 0); + return enif_make_tuple2(env, enif_make_atom(env, "error"), err_code); + } + if (complete_len < out_binary.size) + enif_realloc_binary(&out_binary, complete_len); + + return enif_make_binary(env, &out_binary); +} + +static ERL_NIF_TERM decode_ber_tlv(ErlNifEnv* env, int argc, + const ERL_NIF_TERM argv[]) { + ErlNifBinary in_binary; + ERL_NIF_TERM return_term; + unsigned int err_pos = 0, return_code; + + if (!enif_inspect_iolist_as_binary(env, argv[0], &in_binary)) + return enif_make_badarg(env); + + if ((return_code = decode_begin(env, &return_term, in_binary.data, + in_binary.size, &err_pos)) != ASN1_OK + ) + return enif_make_tuple2(env, enif_make_atom(env,"error"), enif_make_tuple2(env, + enif_make_int(env, return_code),enif_make_int(env, err_pos))); + return return_term; +} + +static int is_ok_load_info(ErlNifEnv* env, ERL_NIF_TERM load_info) { + int i; + return enif_get_int(env, load_info, &i) && i == 1; +} + +static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) { + if (!is_ok_load_info(env, load_info)) + return -1; + return 0; +} + +static int upgrade(ErlNifEnv* env, void** priv_data, void** old_priv_data, + ERL_NIF_TERM load_info) { + if (!is_ok_load_info(env, load_info)) + return -1; + return 0; +} + +static void unload(ErlNifEnv* env, void* priv_data) { + +} + +static ErlNifFunc nif_funcs[] = { { "encode_per_complete", 1, + encode_per_complete }, { "decode_ber_tlv", 1, decode_ber_tlv } }; + +ERL_NIF_INIT(asn1rt_nif, nif_funcs, load, NULL, upgrade, unload) diff --git a/lib/asn1/doc/src/asn1_spec.xmlsrc b/lib/asn1/doc/src/asn1_spec.xmlsrc index 8d61834da8..07cba17816 100644 --- a/lib/asn1/doc/src/asn1_spec.xmlsrc +++ b/lib/asn1/doc/src/asn1_spec.xmlsrc @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2003</year><year>2009</year> + <year>2003</year><year>2011</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -43,7 +43,7 @@ <p>So far this functionality is only provided when using the optimized BER_BIN version, that is when compiling with the options <c>ber_bin</c> and <c>optimize</c>. It does also work - using the <c>driver</c> option. We have no intent to make this + using the <c>nif</c> option. We have no intent to make this available on the default BER version, but maybe in the PER_BIN version (<c>per_bin</c>). </p> @@ -661,7 +661,9 @@ ValAction = {'Action',17,{'Button',4711,false}}. <p>The ASN.1 specs in the test are compiled with the options <c>ber_bin, optimize, driver</c> and <c>asn1config</c>. If the <c>driver</c> option had been omitted there should have been - higher values for <c>decode</c> and <c>decode_part</c>. + higher values for <c>decode</c> and <c>decode_part</c>. These tests have + not been re-run using nifs, but are expected to perform about 5% better + than the linked-in driver. </p> <p>The test program runs 10000 decodes on the value, resulting in a printout with the elapsed time in microseconds for the diff --git a/lib/asn1/doc/src/asn1_ug.xml b/lib/asn1/doc/src/asn1_ug.xml index 12d986308f..61e4f6062f 100644 --- a/lib/asn1/doc/src/asn1_ug.xml +++ b/lib/asn1/doc/src/asn1_ug.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>1997</year><year>2010</year> + <year>1997</year><year>2011</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -347,7 +347,7 @@ erlc -o ../asnfiles -I ../asnfiles -I /usr/local/standards/asn1 Person.asn <p>This flag has effect only when used together with one of <c>per_bin</c> or <c>ber_bin</c> flags. It gives time optimized code in the generated modules and it uses another runtime module. - In the <c>per_bin</c> case a linked-in driver is used. The + In the <c>per_bin</c> case a nif is used. The result from an encode is a binary.</p> <p><em>When this flag is used you cannot use the old format</em><c>{TypeName,Value}</c> when you encode values. Since it is an unnecessary construct it has been removed in favor of @@ -362,9 +362,14 @@ erlc -o ../asnfiles -I ../asnfiles -I /usr/local/standards/asn1 Person.asn </item> <tag><c>+driver</c></tag> <item> - <p>Together with the flags <c>ber_bin</c> and <c>optimize</c> - you choose to use a linked in driver for considerable faster - decode.</p> + <p>As of R15B this means the same as the <c>nif</c> option. Kept for + backwards compatability reasons.</p> + </item> + <tag><c>+nif</c></tag> + <item> + <p>Together with the flags <c>ber_bin</c> + and <c>optimize</c> you choose to use a nif for considerable + faster decode. </p> </item> <tag><c>+asn1config</c></tag> <item> @@ -492,7 +497,7 @@ asn1ct:decode('H323-MESSAGES','SomeChoiceType',Bytes). </pre> </row> <row> <cell align="left" valign="middle">BER</cell> - <cell align="left" valign="middle"><em>[ber_bin, optimize, driver]</em></cell> + <cell align="left" valign="middle"><em>[ber_bin, optimize, nif]</em></cell> <cell align="left" valign="middle">EAVF</cell> <cell align="left" valign="middle">iolist</cell> <cell align="left" valign="middle">iolist / binary</cell> @@ -557,7 +562,7 @@ asn1ct:decode('H323-MESSAGES','SomeChoiceType',Bytes). </pre> </row> <row> <cell align="left" valign="middle">DER</cell> - <cell align="left" valign="middle"><em>[ber_bin, optimize, driver, der]</em></cell> + <cell align="left" valign="middle"><em>[ber_bin, optimize, nif, der]</em></cell> <cell align="left" valign="middle">EAVF</cell> <cell align="left" valign="middle">iolist</cell> <cell align="left" valign="middle">binary</cell> @@ -626,23 +631,24 @@ asn1ct:decode('H323-MESSAGES','SomeChoiceType',Bytes). </pre> </table> <p> - The sole compile options <c>ber</c>, <c>ber_bin</c> and <c>per</c> - are kept for backwards compatibility and should not be used in - new code. + The sole compile options <c>ber</c>, <c>ber_bin</c>, <c>per</c> and + <c>driver</c> are kept for backwards compatibility and should not be + used in new code. The nif implementation which replaces the linked-in + driver has been shown to be about 5-15% faster. </p> <p> You are strongly recommended to use the appropriate alternative of the bold typed options. The <c>optimize</c> and - <c>driver</c> options does not affect the encode or decode + <c>nif</c> options does not affect the encode or decode result, just the time spent in run-time. When <c>ber_bin</c> and - <c>driver</c> or <c>per_bin, optimize</c> and <c>driver</c> is - combined the C-code driver is used in chosen parts of encode / + <c>nif</c> or <c>per_bin</c> and <c>optimize</c> is + combined the C-code nif is used in chosen parts of encode / decode procedure. </p> <table> <row> <cell align="left" valign="middle"><em>Compile options, allowed combinations</em></cell> - <cell align="left" valign="middle"><em>use of linked-in driver</em></cell> + <cell align="left" valign="middle"><em>use of nif</em></cell> </row> <row> <cell align="left" valign="middle">[ber]</cell> @@ -657,7 +663,7 @@ asn1ct:decode('H323-MESSAGES','SomeChoiceType',Bytes). </pre> <cell align="left" valign="middle">no</cell> </row> <row> - <cell align="left" valign="middle"><em>[ber_bin, optimize, driver]</em></cell> + <cell align="left" valign="middle"><em>[ber_bin, optimize, nif]</em></cell> <cell align="left" valign="middle">yes</cell> </row> <row> @@ -690,12 +696,12 @@ asn1ct:decode('H323-MESSAGES','SomeChoiceType',Bytes). </pre> <cell align="left" valign="middle">no</cell> </row> <row> - <cell align="left" valign="middle"><em>[ber_bin, optimize, driver, der]</em></cell> + <cell align="left" valign="middle"><em>[ber_bin, optimize, nif, der]</em></cell> <cell align="left" valign="middle">yes</cell> </row> - <tcaption>When the ASN1 linked-in driver is used.</tcaption> + <tcaption>When the ASN1 nif is used.</tcaption> </table> </section> @@ -712,14 +718,14 @@ asn1rt:decode('H323-MESSAGES','SomeChoiceType',Bytes). </pre> <pre> 'H323-MESSAGES':encode('SomeChoiceType',{call,"octetstring"}). 'H323-MESSAGES':decode('SomeChoiceType',Bytes). </pre> - <p>The asn1 linked-in driver is enabled in two occasions: encoding of + <p>The asn1 nif is enabled in two occasions: encoding of asn1 values when the asn1 spec is compiled with <c>per_bin</c> and <c>optimize</c> or decode of encoded asn1 values when the asn1 spec is - compiled with <c>ber_bin</c>, <c>optimize</c> and <c>driver</c>. In - those cases the driver will be loaded automatically at the first call + compiled with <c>ber_bin</c>, <c>optimize</c> and <c>nif</c>. In + those cases the nif will be loaded automatically at the first call to <c>encode</c>/<c>decode</c>. If one doesn't want the performance - overhead of the driver being loaded at the first call it is possible - to load the driver separately by <c>asn1rt:load_driver()</c>. </p> + overhead of the nif being loaded at the first call it is possible + to load the nif separately by loading the <c>asn1rt_nif</c> module.</p> <p>By invoking the function <c>info/0</c> in a generated module, one gets information about which compiler options were used.</p> </section> diff --git a/lib/asn1/doc/src/asn1ct.xml b/lib/asn1/doc/src/asn1ct.xml index 265f8735c2..d7c2572dc8 100644 --- a/lib/asn1/doc/src/asn1ct.xml +++ b/lib/asn1/doc/src/asn1ct.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>1997</year><year>2010</year> + <year>1997</year><year>2011</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -52,7 +52,7 @@ <v>Options = [Option| OldOption]</v> <v>Option = ber_bin | per_bin | uper_bin | der | compact_bit_string | noobj | {n2n,EnumTypeName} |{outdir,Dir} | {i,IncludeDir} | optimize | - driver | asn1config | undec_rest | {inline,OutputName} | inline | + nif | asn1config | undec_rest | {inline,OutputName} | inline | {macro_name_prefix, Prefix} | {record_name_prefix, Prefix} | verbose</v> <v>OldOption = ber | per</v> <v>Reason = term()</v> @@ -212,16 +212,21 @@ Binary = binary() <c>per_bin</c> or <c>ber_bin</c> option. It gives time optimized code generated and it uses another runtime module and - in the <c>per_bin</c> case a linked-in driver. The result + in the <c>per_bin</c> case a nif. The result in the <c>per_bin</c> case from an encode when compiled with this option will be a binary.</p> </item> <tag><c>driver</c></tag> <item> + <p>As of R15B this means the same as the <c>nif</c> option. Kept for + backwards compatability reasons.</p> + </item> + <tag><c>nif</c></tag> + <item> <p>Option valid together with <c>ber_bin</c> and <c>optimize</c> - options. It enables the use of a linked-in driver that gives - considerable faster decode. In <c>ber_bin</c> the driver is - enabled only by explicit use of the option <c>driver</c>.</p> + options. It enables the use of several nifs that gives faster + decode. Nifs are only enabled by the explicit use of the option + <c>nif</c></p> </item> <tag><c>asn1config</c></tag> <item> @@ -343,18 +348,6 @@ Binary = binary() </desc> </func> <func> - <name>validate(Module,Type,Value) -> ok | {error,Reason}</name> - <fsummary>Validate an ASN.1 value.</fsummary> - <type> - <v>Module = Type = atom()</v> - <v>Value = term()</v> - </type> - <desc> - <p>Validates that <c>Value</c> conforms to <c>Type</c> - from <c>Module</c>. <em>Not implemented in this version of the ASN.1 application.</em></p> - </desc> - </func> - <func> <name>value(Module ,Type) -> {ok,Value} | {error,Reason}</name> <fsummary>Create an ASN.1 value for test purposes.</fsummary> <type> diff --git a/lib/asn1/doc/src/asn1rt.xml b/lib/asn1/doc/src/asn1rt.xml index 1217a07e9b..0c3c257189 100644 --- a/lib/asn1/doc/src/asn1rt.xml +++ b/lib/asn1/doc/src/asn1rt.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>1997</year><year>2009</year> + <year>1997</year><year>2011</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -42,36 +42,6 @@ <funcs> <func> - <name>start() -> ok |{error,Reason}</name> - <fsummary>Starts the asn1 server.</fsummary> - <type> - <v>Reason = term()</v> - </type> - <desc> - <p>Starts the asn1 server that loads the drivers.</p> - <p>The server schedules a driver that is not blocked by - another caller. The driver is used by the asn1 application if - specs are compiled with options <c>[per_bin, optimize]</c> or - <c>[ber_bin, optimize, driver]</c>. The server will be started - automatically at encode/decode if it isn't done explicitly. If - encode/decode with driver is used in test or industrial code - it is a performance gain to start it explicitly to avoid the - one time load in run-time.</p> - </desc> - </func> - - <func> - <name>stop() -> ok |{error,Reason}</name> - <fsummary>Stops the asn1 server.</fsummary> - <type> - <v>Reason = term()</v> - </type> - <desc> - <p>Stops the asn1 server and unloads the drivers.</p> - </desc> - </func> - - <func> <name>decode(Module,Type,Bytes) -> {ok,Value}|{error,Reason}</name> <fsummary>Decode from bytes into an ASN.1 value.</fsummary> <type> @@ -126,35 +96,23 @@ <func> <name>load_driver() -> ok | {error,Reason}</name> - <fsummary>Loads the linked-in driver.</fsummary> + <fsummary>Loads the linked-in driver. (deprecated)</fsummary> <type> <v>Reason = term()</v> </type> <desc> - <p>This function loads the linked-in driver before the first call - to encode. If this function is not called the driver will be loaded - automatically at the first call to encode. If one doesn't want the - performance cost of a driver load when the application is running, - this function makes it possible to load the driver in an - initialization.</p> - <p>The driver is only used when encoding/decoding ASN.1 files that - were compiled with the options <c>per_bin</c> and <c>optimize</c>.</p> + <p>This function is obsolete and will be removed in R16A</p> </desc> </func> <func> <name>unload_driver() -> ok | {error,Reason}</name> - <fsummary>Unloads the linked-in driver.</fsummary> + <fsummary>Unloads the linked-in driver. (deprecated)</fsummary> <type> <v>Reason = term()</v> </type> <desc> - <p>This function unloads the linked-in driver. - When the driver has been loaded it remains in the environment until - it is unloaded. Normally the driver should remain loaded, it is - crucial for the performance of ASN.1 encoding. </p> - <p>The driver is only used when ASN.1 modules have been compiled - with the flags <c>per_bin</c> and <c>optimize</c>.</p> + <p>This function is obsolete and will be removed in R16A</p> </desc> </func> @@ -188,19 +146,6 @@ value, to a UTF8 encoded binary.</p> </desc> </func> - - <func> - <name>validate(Module,Type,Value) -> ok | {error,Reason}</name> - <fsummary>Validate an ASN.1 value.</fsummary> - <type> - <v>Module = Type = atom()</v> - <v>Value = term()</v> - </type> - <desc> - <p>Validates that <c>Value</c> conforms to <c>Type</c> - from <c>Module</c>. <em>Not implemented in this version of the ASN.1 application.</em></p> - </desc> - </func> </funcs> diff --git a/lib/asn1/src/Makefile b/lib/asn1/src/Makefile index 2733cde3f8..3a59773d93 100644 --- a/lib/asn1/src/Makefile +++ b/lib/asn1/src/Makefile @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 1997-2009. All Rights Reserved. +# Copyright Ericsson AB 1997-2011. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in @@ -68,7 +68,7 @@ RT_MODULES= \ asn1rt_per_bin_rt2ct \ asn1rt_uper_bin \ asn1rt_check \ - asn1rt_driver_handler + asn1rt_nif # asn1_sup \ # asn1_app \ # asn1_server diff --git a/lib/asn1/src/asn1.app.src b/lib/asn1/src/asn1.app.src index abacb0a1e9..09144ba2f7 100644 --- a/lib/asn1/src/asn1.app.src +++ b/lib/asn1/src/asn1.app.src @@ -9,12 +9,11 @@ asn1rt_ber_bin, asn1rt_ber_bin_v2, asn1rt_check, - asn1rt_driver_handler + asn1rt_nif ]}, {registered, [ asn1_ns, - asn1db, - asn1_driver_owner + asn1db ]}, {env, []}, {applications, [kernel, stdlib]} diff --git a/lib/asn1/src/asn1_app.erl b/lib/asn1/src/asn1_app.erl index 2d3eed1743..9fff96e0bf 100644 --- a/lib/asn1/src/asn1_app.erl +++ b/lib/asn1/src/asn1_app.erl @@ -28,7 +28,7 @@ %% {error, Reason} %% start(_Type, _StartArgs) -> - asn1_sup:start_link(). + {ok, self()}. %% stop(State) %% diff --git a/lib/asn1/src/asn1_server.erl b/lib/asn1/src/asn1_server.erl deleted file mode 100644 index aeb59d8b0c..0000000000 --- a/lib/asn1/src/asn1_server.erl +++ /dev/null @@ -1,107 +0,0 @@ -%% ``The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved via the world wide web at http://www.erlang.org/. -%% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. -%% -%% The Initial Developer of the Original Code is Ericsson Utvecklings AB. -%% Portions created by Ericsson are Copyright 1999, Ericsson Utvecklings -%% AB. All Rights Reserved.'' -%% -%% $Id$ -%% - -%% Purpose: Provide complete encode/and pre-decode of asn1. --module(asn1_server). - - - --behaviour(gen_server). - --export([start_link/0,client_port/0]). - -%% Internal exports, call-back functions. --export([init/1,handle_call/3,handle_cast/2,handle_info/2,code_change/3, - terminate/2]). - - -%% Macros --define(port_names, - { asn1_drv01, asn1_drv02, asn1_drv03, asn1_drv04, - asn1_drv05, asn1_drv06, asn1_drv07, asn1_drv08, - asn1_drv09, asn1_drv10, asn1_drv11, asn1_drv12, - asn1_drv13, asn1_drv14, asn1_drv15, asn1_drv16 }). -%%% -------------------------------------------------------- -%%% Interface Functions. -%%% -------------------------------------------------------- - -start_link() -> - gen_server:start_link({local, asn1_server}, asn1_server, [], []). - -init([]) -> - process_flag(trap_exit, true), - erl_ddll:start(), - PrivDir = code:priv_dir(asn1), - LibDir1 = filename:join([PrivDir, "lib"]), - case erl_ddll:load_driver(LibDir1, asn1_erl_drv) of - ok -> ok; - {error,_} -> - LibDir2 = - filename:join(LibDir1, - erlang:system_info(system_architecture)), - erl_ddll:load_driver(LibDir2, asn1_erl_drv) - end, - open_ports("asn1_erl_drv",size(?port_names)). - -open_ports(_,0) -> - {ok, []}; -open_ports(Cmd,N) -> - Port = open_port({spawn, Cmd}, []), - %% check that driver is loaded, linked and working - case catch port_control(Port, 0, []) of - {'EXIT', _} -> - {stop, nodriver}; - _ -> - register(element(N,?port_names), Port), - open_ports(Cmd,N-1) - end. - -client_port() -> - element(erlang:system_info(scheduler_id) rem size(?port_names) + 1, - ?port_names). - - -%%% -------------------------------------------------------- -%%% The call-back functions. -%%% -------------------------------------------------------- - -handle_call(_, _, State) -> - {noreply, State}. - -handle_cast(_, State) -> - {noreply, State}. - -handle_info({'EXIT', Pid, _Reason}, State) when is_pid(Pid) -> - {noreply, State}; - -handle_info({'EXIT', Port, Reason}, State) when is_port(Port) -> - {stop, {port_died, Reason}, State}; -handle_info(_, State) -> - {noreply, State}. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -terminate(_Reason, _State) -> - close_ports(size(?port_names)). - -close_ports(0) -> - ok; -close_ports(N) -> - element(N,?port_names) ! {self(), close}, %% almost same as port_close(Name) - close_ports(N-1). diff --git a/lib/asn1/src/asn1ct.erl b/lib/asn1/src/asn1ct.erl index a167d27f82..a170dd8660 100644 --- a/lib/asn1/src/asn1ct.erl +++ b/lib/asn1/src/asn1ct.erl @@ -47,6 +47,10 @@ -import(asn1ct_gen_ber_bin_v2,[encode_tag_val/3,decode_class/1]). +-ifndef(vsn). +-define(vsn,"0.0.1"). +-endif. + -define(unique_names,0). -define(dupl_uniquedefs,1). -define(dupl_equaldefs,2). @@ -81,6 +85,12 @@ compile(File) -> compile(File,[]). compile(File,Options) when is_list(Options) -> + case lists:member(driver, Options) of %% remove me in R16A! + true -> + io:format("Warning: driver option is obsolete and will be removed in R16A, use nif instead!"); + false -> + ok + end, Options1 = optimize_ber_bin(Options), Options2 = includes(File,Options1), Includes=[I||{i,I}<-Options2], @@ -1082,7 +1092,7 @@ get_runtime_mod(Options) -> ber_bin_v2 -> ["asn1rt_ber_bin_v2.erl"]; uper_bin -> ["asn1rt_uper_bin.erl"] end, - RtMod1++["asn1rt_check.erl","asn1rt_driver_handler.erl","asn1rt.erl"]. + RtMod1++["asn1rt_check.erl","asn1rt.erl"]. erl_compile(OutFile,Options) -> diff --git a/lib/asn1/src/asn1ct_constructed_ber_bin_v2.erl b/lib/asn1/src/asn1ct_constructed_ber_bin_v2.erl index e3be914af4..e16873c717 100644 --- a/lib/asn1/src/asn1ct_constructed_ber_bin_v2.erl +++ b/lib/asn1/src/asn1ct_constructed_ber_bin_v2.erl @@ -1227,7 +1227,7 @@ gen_dec_call({typefield,_},_,_,_Cname,Type,BytesVar,Tag,_,_,false,_) -> emit([nl,indent(6),"begin",nl]), % emit([indent(9),{curr,opendec}," = ?RT_BER:decode_open_type(", emit([indent(9),{curr,tmptlv}," = ?RT_BER:decode_open_type(", - BytesVar,",",{asis,Tag},"),",nl]), + BytesVar,",",{asis,Tag},asn1ct_gen:nif_parameter(),"),",nl]), % emit([indent(9),"{",{curr,tmptlv},",_} = ?RT_BER:decode(", % {curr,opendec},"),",nl]), @@ -1242,7 +1242,8 @@ gen_dec_call({typefield,_},_,_,_Cname,Type,BytesVar,Tag,_,_,false,_) -> emit([indent(9),"end",nl,indent(6),"end",nl]), []; gen_dec_call({typefield,_},_,_,Cname,Type,BytesVar,Tag,_,_,_DecObjInf,OptOrMandComp) -> - emit(["?RT_BER:decode_open_type(",BytesVar,",",{asis,Tag},")"]), + emit(["?RT_BER:decode_open_type(",BytesVar,",",{asis,Tag}, + asn1ct_gen:nif_parameter(),")"]), RefedFieldName = % asn1ct_gen:get_constraint(Type#type.constraint, % tableconstraint_info), @@ -1250,7 +1251,8 @@ gen_dec_call({typefield,_},_,_,Cname,Type,BytesVar,Tag,_,_,_DecObjInf,OptOrMandC [{Cname,RefedFieldName,asn1ct_gen:mk_var(asn1ct_name:curr(term)), asn1ct_gen:mk_var(asn1ct_name:curr(tmpterm)),Tag,OptOrMandComp}]; gen_dec_call({objectfield,PrimFieldName,PFNList},_,_,Cname,_,BytesVar,Tag,_,_,_,OptOrMandComp) -> - emit(["?RT_BER:decode_open_type(",BytesVar,",",{asis,Tag},")"]), + emit(["?RT_BER:decode_open_type(",BytesVar,",",{asis,Tag}, + asn1ct_gen:nif_parameter(),")"]), [{Cname,{PrimFieldName,PFNList},asn1ct_gen:mk_var(asn1ct_name:curr(term)), asn1ct_gen:mk_var(asn1ct_name:curr(tmpterm)),Tag,OptOrMandComp}]; gen_dec_call(InnerType,Erules,TopType,Cname,Type,BytesVar,Tag,PrimOptOrMand, diff --git a/lib/asn1/src/asn1ct_constructed_per.erl b/lib/asn1/src/asn1ct_constructed_per.erl index c1b6aa5713..e07680f10b 100644 --- a/lib/asn1/src/asn1ct_constructed_per.erl +++ b/lib/asn1/src/asn1ct_constructed_per.erl @@ -73,16 +73,23 @@ gen_encode_constructed(Erule,Typename,D) when is_record(D,type) -> _ -> ok end, - case {Optionals = optionals(to_textual_order(CompList)),CompList} of - {[],EmptyCL} when EmptyCL == {[],[],[]};EmptyCL == {[],[]};EmptyCL == [] -> + case {Optionals = optionals(to_textual_order(CompList)),CompList, + is_optimized(Erule)} of + {[],EmptyCL,_} when EmptyCL == {[],[],[]};EmptyCL == {[],[]};EmptyCL == [] -> emit(["%%Variable setting just to eliminate ", "compiler warning for unused vars!",nl, "_Val = ",{curr,val},",",nl]); - {[],_} -> + {[],_,_} -> emit([{next,val}," = ?RT_PER:list_to_record("]), emit(["'",asn1ct_gen:list2rname(Typename),"'"]), emit([", ",{curr,val},"),",nl]); - _ -> + {_,_,true} -> + gen_fixoptionals(Optionals), + FixOpts = param_map(fun(Var) -> + {var,Var} + end,asn1ct_name:all(fixopt)), + emit({"{",{next,val},",Opt} = {",{curr,val},",[",FixOpts,"]},",nl}); + {_,_,false} -> Fixoptcall = ",Opt} = ?RT_PER:fixoptionals(", emit({"{",{next,val},Fixoptcall, {asis,Optionals},",",length(Optionals), @@ -439,9 +446,7 @@ gen_encode_sof(Erule,Typename,SeqOrSetOf,D) when is_record(D,type) -> _-> "" end, - emit({nl,indent(3),"?RT_PER:encode_length(", - {asis,SizeConstraint}, - ",length(Val)),",nl}), + gen_encode_length(SizeConstraint, is_optimized(Erule)), emit({indent(3),"'enc_",asn1ct_gen:list2name(Typename), "_components'(Val",ObjFun,", [])"}), emit({nl,"].",nl}), @@ -453,6 +458,42 @@ gen_encode_sof(Erule,Typename,SeqOrSetOf,D) when is_record(D,type) -> end, gen_encode_sof_components(Erule,Typename,SeqOrSetOf,NewComponentType). + +%% Logic copied from asn1_per_bin_rt2ct:encode_constrained_number +gen_encode_length({Lb,Ub},true) when Ub =< 65535, Lb >= 0 -> + Range = Ub - Lb + 1, + V2 = ["(length(Val) - ",Lb,")"], + Encode = if + Range == 1 -> + "[]"; + Range == 2 -> + {"[",V2,"]"}; + Range =< 4 -> + {"[10,2,",V2,"]"}; + Range =< 8 -> + {"[10,3,",V2,"]"}; + Range =< 16 -> + {"[10,4,",V2,"]"}; + Range =< 32 -> + {"[10,5,",V2,"]"}; + Range =< 64 -> + {"[10,6,",V2,"]"}; + Range =< 128 -> + {"[10,7,",V2,"]"}; + Range =< 255 -> + {"[10,8,",V2,"]"}; + Range =< 256 -> + {"[20,1,",V2,"]"}; + Range =< 65536 -> + {"[20,2,<<",V2,":16>>]"}; + true -> + {"?RT_PER:encode_length(",{asis,{Lb,Ub}},",length(Val))"} + end, + emit({nl,Encode,",",nl}); +gen_encode_length(SizeConstraint,_) -> + emit({nl,indent(3),"?RT_PER:encode_length(", + {asis,SizeConstraint},",length(Val)),",nl}). + gen_decode_sof(Erules,Typename,SeqOrSetOf,D) when is_record(D,type) -> asn1ct_name:start(), {_SeqOrSetOf,ComponentType} = D#type.def, @@ -469,7 +510,8 @@ gen_decode_sof(Erules,Typename,SeqOrSetOf,D) when is_record(D,type) -> _ -> "" end, - emit({nl,"{Num,Bytes1} = ?RT_PER:decode_length(Bytes,",{asis,SizeConstraint},"),",nl}), + gen_decode_length(SizeConstraint, + is_optimized(Erules)), emit({"'dec_",asn1ct_gen:list2name(Typename), "_components'(Num, Bytes1, telltype",ObjFun,", []).",nl}), NewComponentType = @@ -480,6 +522,41 @@ gen_decode_sof(Erules,Typename,SeqOrSetOf,D) when is_record(D,type) -> end, gen_decode_sof_components(Erules,Typename,SeqOrSetOf,NewComponentType). +%% Logic copied from asn1_per_bin_rt2ct:decode_constrained_number +gen_decode_length({Lb,Ub},true) when Ub =< 65535, Lb >= 0 -> + Range = Ub - Lb + 1, + Call = if + Range == 1 -> + "{0,Bytes}"; + Range == 2 -> + "?RT_PER:getbits(Bytes,1)"; + Range =< 4 -> + "?RT_PER:getbits(Bytes,2)"; + Range =< 8 -> + "?RT_PER:getbits(Bytes,3)"; + Range =< 16 -> + "?RT_PER:getbits(Bytes,4)"; + Range =< 32 -> + "?RT_PER:getbits(Bytes,5)"; + Range =< 64 -> + "?RT_PER:getbits(Bytes,6)"; + Range =< 128 -> + "?RT_PER:getbits(Bytes,7)"; + Range =< 255 -> + "?RT_PER:getbits(Bytes,8)"; + Range =< 256 -> + "?RT_PER:getoctets(Bytes,1)"; + Range =< 65536 -> + "?RT_PER:getoctets(Bytes,2)"; + true -> + ["exit({not_supported,{integer_range,",Range,"}}"] + end, + emit({nl,"{Val,Remain} = ",Call,",",nl}), + emit({nl,"{Num,Bytes1} = {Val+",Lb,",Remain},",nl}); +gen_decode_length(SizeConstraint,_) -> + emit({nl,"{Num,Bytes1} = ?RT_PER:decode_length(Bytes,", + {asis,SizeConstraint},"),",nl}). + gen_encode_sof_components(Erule,Typename,SeqOrSetOf,Cont) -> {ObjFun,ObjFun_Var} = case Cont#type.tablecinf of @@ -636,6 +713,27 @@ gen_dec_extension_value(_) -> emit({"{Ext,",{next,bytes},"} = ?RT_PER:getext(",{curr,bytes},")"}), asn1ct_name:new(bytes). +gen_fixoptionals([{Pos,Def}|R]) -> + asn1ct_name:new(fixopt), + emit({{curr,fixopt}," = case element(",{asis,Pos},",",{curr,val},") of",nl, + "asn1_DEFAULT -> 0;",nl, + {asis,Def}," -> 0;",nl, + "_ -> 1",nl, + "end,",nl}), + gen_fixoptionals(R); +gen_fixoptionals([Pos|R]) -> + gen_fixoptionals([{Pos,asn1_NOVALUE}|R]); +gen_fixoptionals([]) -> + ok. + + +param_map(Fun, [H]) -> + [Fun(H)]; +param_map(Fun, [H|T]) -> + [Fun(H),","|param_map(Fun,T)]. + + + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Produce a list with positions (in the Value record) where %% there are optional components, start with 2 because first element @@ -922,7 +1020,7 @@ gen_enc_line(Erule,TopType,Cname,Type,Element, _Pos,DynamicEnc,Ext) -> end, case Ext of {ext,_Ep2,_} -> - emit(["))"]); + emit("))"); _ -> true end. gen_dec_components_call(Erule,TopType,{Root1,ExtList,Root2},MaybeComma,DecInfObj,Ext,NumberOfOptionals) -> diff --git a/lib/asn1/src/asn1ct_gen.erl b/lib/asn1/src/asn1ct_gen.erl index e49829d82f..e18bc37058 100644 --- a/lib/asn1/src/asn1ct_gen.erl +++ b/lib/asn1/src/asn1ct_gen.erl @@ -47,6 +47,7 @@ un_hyphen_var/1]). -export([gen_encode_constructed/4, gen_decode_constructed/4]). +-export([nif_parameter/0]). %% pgen(Outfile, Erules, Module, TypeOrVal, Options) %% Generate Erlang module (.erl) and (.hrl) file corresponding to an ASN.1 module @@ -938,13 +939,13 @@ pgen_dispatcher(Erules,_Module,{Types,_Values,_,_,_Objects,_ObjectSets}) -> NoFinalPadding = lists:member(no_final_padding,get(encoding_options)), Call = case Erules of per -> "?RT_PER:complete(encode_disp(Type,Data))"; - per_bin -> "?RT_PER:complete(encode_disp(Type,Data))"; + per_bin -> ["?RT_PER:complete(encode_disp(Type,Data))"]; ber -> "encode_disp(Type,Data)"; ber_bin -> "encode_disp(Type,Data)"; ber_bin_v2 -> "encode_disp(Type,Data)"; uper_bin when NoFinalPadding == true -> "?RT_PER:complete_NFP(encode_disp(Type,Data))"; - uper_bin -> "?RT_PER:complete(encode_disp(Type,Data))" + uper_bin -> ["?RT_PER:complete(encode_disp(Type,Data))"] end, EncWrap = case Erules of ber -> "wrap_encode(Bytes)"; @@ -974,7 +975,7 @@ pgen_dispatcher(Erules,_Module,{Types,_Values,_,_,_Objects,_ObjectSets}) -> % case Erules of % ber_bin_v2 -> % emit(["decode(Type,Data0) ->",nl]), -% emit(["{Data,_RestBin} = ?RT_BER:decode(Data0",driver_parameter(),"),",nl]); +% emit(["{Data,_RestBin} = ?RT_BER:decode(Data0",nif_parameter(),"),",nl]); % _ -> % emit(["decode(Type,Data) ->",nl]) % end, @@ -991,10 +992,10 @@ pgen_dispatcher(Erules,_Module,{Types,_Values,_,_,_Objects,_ObjectSets}) -> {ber_bin_v2,false} -> io_lib:format("~s~s~s~n", ["element(1,?RT_BER:decode(Data", - driver_parameter(),"))"]); + nif_parameter(),"))"]); {ber_bin_v2,true} -> emit(["{Data,Rest} = ?RT_BER:decode(Data0", - driver_parameter(),"),",nl]), + nif_parameter(),"),",nl]), "Data"; _ -> "Data" @@ -1130,13 +1131,8 @@ gen_decode_partial_incomplete(Erule) when Erule == ber;Erule==ber_bin; "Data) of",nl]), EmitCaseClauses(), emit(["decode_part(Type,Data0) ->",nl]), - Driver = - case lists:member(driver,get(encoding_options)) of - true -> - ",driver"; - _ -> "" - end, - emit([" case catch decode_inc_disp(Type,element(1,?RT_BER:decode(Data0",Driver,"))) of",nl]), + emit([" case catch decode_inc_disp(Type,element(1," + "?RT_BER:decode(Data0",nif_parameter(),"))) of",nl]), % " {Data,_RestBin} = ?RT_BER:decode(Data0),",nl, % " case catch decode_inc_disp(Type,Data) of",nl]), EmitCaseClauses(); @@ -1179,12 +1175,12 @@ gen_partial_inc_dispatcher([],_) -> emit(["decode_partial_inc_disp(Type,_Data) ->",nl, " exit({error,{asn1,{undefined_type,Type}}}).",nl]). -driver_parameter() -> +nif_parameter() -> Options = get(encoding_options), - case lists:member(driver,Options) of - true -> - ",driver"; - _ -> "" + case {lists:member(driver,Options),lists:member(nif,Options)} of + {true,_} -> ",nif"; + {_,true} -> ",nif"; + _ -> "" end. gen_wrapper() -> diff --git a/lib/asn1/src/asn1ct_gen_ber_bin_v2.erl b/lib/asn1/src/asn1ct_gen_ber_bin_v2.erl index 9ec458e351..e8a4ad0cf1 100644 --- a/lib/asn1/src/asn1ct_gen_ber_bin_v2.erl +++ b/lib/asn1/src/asn1ct_gen_ber_bin_v2.erl @@ -416,7 +416,7 @@ gen_decode_selected(Erules,Type,FuncName) -> end, emit([" case ?RT_BER:decode_selective(",{asis,Pattern},",Bin) of",nl, " {ok,Bin2} when is_binary(Bin2) ->",nl, - " {Tlv,_} = ?RT_BER:decode(Bin2),",nl]), + " {Tlv,_} = ?RT_BER:decode(Bin2",asn1ct_gen:nif_parameter(),"),",nl]), emit("{ok,"), gen_decode_selected_type(Erules,Type), emit(["};",nl," Err -> exit({error,{selctive_decode,Err}})",nl, @@ -708,7 +708,7 @@ gen_dec_prim(Erules,Att,BytesVar,DoTag,TagIn,Form,OptOrMand) -> 'ASN1_OPEN_TYPE' -> emit(["?RT_BER:decode_open_type_as_binary(", BytesVar,","]), - add_func({decode_open_type_as_binary,2}); + add_func({decode_open_type_as_binary,3}); #'ObjectClassFieldType'{} -> case asn1ct_gen:get_inner(Att#type.def) of {fixedtypevaluefield,_,InnerType} -> @@ -716,7 +716,7 @@ gen_dec_prim(Erules,Att,BytesVar,DoTag,TagIn,Form,OptOrMand) -> 'ASN1_OPEN_TYPE' -> emit(["?RT_BER:decode_open_type_as_binary(", BytesVar,","]), - add_func({decode_open_type_as_binary,2}); + add_func({decode_open_type_as_binary,3}); Other -> exit({'can not decode' ,Other}) end; @@ -1064,7 +1064,7 @@ emit_tlv_format_function() -> end. emit_tlv_format_function1() -> emit(["tlv_format(Bytes) when is_binary(Bytes) ->",nl, - " {Tlv,_}=?RT_BER:decode(Bytes),",nl, + " {Tlv,_}=?RT_BER:decode(Bytes",asn1ct_gen:nif_parameter(),"),",nl, " Tlv;",nl, "tlv_format(Bytes) ->",nl, " Bytes.",nl]). diff --git a/lib/asn1/src/asn1ct_gen_per.erl b/lib/asn1/src/asn1ct_gen_per.erl index 8313cf1b60..b90a0adf81 100644 --- a/lib/asn1/src/asn1ct_gen_per.erl +++ b/lib/asn1/src/asn1ct_gen_per.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2010. All Rights Reserved. +%% Copyright Ericsson AB 1997-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -238,7 +238,8 @@ gen_encode_prim(Erules,D,DoTag,Value) when is_record(D,type) -> "?RT_PER:complete(enc_~s(~s))",[Tname,Value]); [#type{def=#'Externaltypereference'{type=Tname}}] -> io_lib:format( - "?RT_PER:complete(enc_~s(~s))",[Tname,Value]); + "?RT_PER:complete(enc_~s(~s))", + [Tname,Value]); _ -> Value end, emit(["?RT_PER:encode_open_type(", {asis,Constraint}, ",", diff --git a/lib/asn1/src/asn1ct_gen_per_rt2ct.erl b/lib/asn1/src/asn1ct_gen_per_rt2ct.erl index 4f4fcfafc3..1a0a0e211d 100644 --- a/lib/asn1/src/asn1ct_gen_per_rt2ct.erl +++ b/lib/asn1/src/asn1ct_gen_per_rt2ct.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2002-2010. All Rights Reserved. +%% Copyright Ericsson AB 2002-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -230,7 +230,8 @@ gen_encode_prim(Erules,D,DoTag,Value) when is_record(D,type) -> "?RT_PER:complete(enc_~s(~s))",[Tname,Value]); [#type{def=#'Externaltypereference'{type=Tname}}] -> io_lib:format( - "?RT_PER:complete(enc_~s(~s))",[Tname,Value]); + "?RT_PER:complete(enc_~s(~s))", + [Tname,Value]); _ -> Value end, emit(["?RT_PER:encode_open_type(", {asis,Constraint}, ",", diff --git a/lib/asn1/src/asn1rt.erl b/lib/asn1/src/asn1rt.erl index 9ef68efab5..e9d3ea9a72 100644 --- a/lib/asn1/src/asn1rt.erl +++ b/lib/asn1/src/asn1rt.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2009. All Rights Reserved. +%% Copyright Ericsson AB 1997-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -27,6 +27,8 @@ -export([utf8_binary_to_list/1,utf8_list_to_binary/1]). +-deprecated([load_driver/0,unload_driver/0]). + encode(Module,{Type,Term}) -> encode(Module,Type,Term). @@ -46,38 +48,12 @@ decode(Module,Type,Bytes) -> Result end. -%% asn1-1.6.8.1 -%% load_driver() -> -%% asn1rt_driver_handler:load_driver(), -%% receive -%% driver_ready -> -%% ok; -%% Err={error,_Reason} -> -%% Err; -%% Error -> -%% {error,Error} -%% end. - -%% asn1-1.6.9 - load_driver() -> - case catch asn1rt_driver_handler:load_driver() of - ok -> - ok; - {error,{already_started,asn1}} -> - ok; - Err -> - {error,Err} - end. - +%% Remove in R16A +load_driver() -> + ok. unload_driver() -> - case catch asn1rt_driver_handler:unload_driver() of - ok -> - ok; - Error -> - {error,Error} - end. - + ok. info(Module) -> case catch apply(Module,info,[]) of diff --git a/lib/asn1/src/asn1rt_ber_bin_v2.erl b/lib/asn1/src/asn1rt_ber_bin_v2.erl index a3bb570282..2df289e2b8 100644 --- a/lib/asn1/src/asn1rt_ber_bin_v2.erl +++ b/lib/asn1/src/asn1rt_ber_bin_v2.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2002-2009. All Rights Reserved. +%% Copyright Ericsson AB 2002-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -49,7 +49,8 @@ decode_tag_and_length/1]). -export([encode_open_type/1,encode_open_type/2, - decode_open_type/2,decode_open_type_as_binary/2]). + decode_open_type/2,decode_open_type/3, + decode_open_type_as_binary/2]). -export([decode_primitive_incomplete/2,decode_selective/2]). @@ -158,64 +159,47 @@ encode_tlv_list([],Acc) -> Bin=list_to_binary(lists:reverse(Acc)), {Bin,size(Bin)}. -%% asn1-1.6.8.1 -%% decode(B,driver) -> -%% case catch port_control(asn1_driver_port,2,B) of -%% Bin when is_binary(Bin) -> -%% binary_to_term(Bin); -%% List when is_list(List) -> handle_error(List,B); -%% {'EXIT',{badarg,Reason}} -> -%% asn1rt_driver_handler:load_driver(), -%% receive -%% driver_ready -> -%% case catch port_control(asn1_driver_port,2,B) of -%% Bin2 when is_binary(Bin2) -> binary_to_term(Bin2); -%% List when is_list(List) -> handle_error(List,B); -%% Error -> exit(Error) -%% end; -%% {error,Error} -> % error when loading driver -%% %% the driver could not be loaded -%% exit(Error); -%% Error={port_error,Reason} -> -%% exit(Error) -%% end; -%% {'EXIT',Reason} -> -%% exit(Reason) -%% end. - -%% asn1-1.6.9 -decode(B,driver) -> - case catch control(?TLV_DECODE,B) of - Bin when is_binary(Bin) -> - binary_to_term(Bin); - List when is_list(List) -> handle_error(List,B); - {'EXIT',{badarg,_Reason}} -> - case asn1rt:load_driver() of - ok -> - case control(?TLV_DECODE,B) of - Bin when is_binary(Bin) -> binary_to_term(Bin); - List when is_list(List) -> handle_error(List,B) - end; - Err -> - Err - end - end. +decode(B) -> + decode(B, erlang). +%% asn1-1.7 +decode(B, nif) -> + case application:get_env(asn1, nif_loadable) of + {ok, true} -> + case asn1rt_nif:decode_ber_tlv(B) of + {error, Reason} -> handle_error(Reason, B); + Else -> Else + end; + {ok, false} -> + decode(B); + undefined -> + case catch code:load_file(asn1rt_nif) of + {module, asn1rt_nif} -> + application:set_env(asn1, nif_loadable, true); + _Else -> + application:set_env(asn1, nif_loadable, false) + end, + decode(B, nif) + end; +decode(B,erlang) when is_binary(B) -> + decode_primitive(B); +decode(Tlv,erlang) -> + {Tlv,<<>>}. handle_error([],_)-> exit({error,{asn1,{"memory allocation problem"}}}); -handle_error([$1|_],L) -> % error in driver +handle_error({$1,_},L) -> % error in nif exit({error,{asn1,L}}); -handle_error([$2|T],L) -> % error in driver due to wrong tag +handle_error({$2,T},L) -> % error in nif due to wrong tag exit({error,{asn1,{"bad tag after byte:",error_pos(T),L}}}); -handle_error([$3|T],L) -> % error in driver due to length error +handle_error({$3,T},L) -> % error in driver due to length error exit({error,{asn1,{"bad length field after byte:", error_pos(T),L}}}); -handle_error([$4|T],L) -> % error in driver due to indefinite length error +handle_error({$4,T},L) -> % error in driver due to indefinite length error exit({error,{asn1, {"indefinite length without end bytes after byte:", error_pos(T),L}}}); -handle_error([$5|T],L) -> % error in driver due to indefinite length error +handle_error({$5,T},L) -> % error in driver due to indefinite length error exit({error,{asn1,{"bad encoded value after byte:", error_pos(T),L}}}); handle_error(ErrL,L) -> @@ -228,16 +212,6 @@ error_pos([B])-> error_pos([B|Bs]) -> BS = 8 * length(Bs), B bsl BS + error_pos(Bs). -%% asn1-1.6.9 -control(Cmd, Data) -> - Port = asn1rt_driver_handler:client_port(), - erlang:port_control(Port, Cmd, Data). - -decode(Bin) when is_binary(Bin) -> - decode_primitive(Bin); -decode(Tlv) -> % assume it is a tlv - {Tlv,<<>>}. - decode_primitive(Bin) -> {Form,TagNo,V,Rest} = decode_tag_and_length(Bin), @@ -796,9 +770,11 @@ encode_open_type(Val,Tag) -> %% Value = binary with decoded data (which must be decoded again as some type) %% decode_open_type(Tlv, TagIn) -> + decode_open_type(Tlv, TagIn, erlang). +decode_open_type(Tlv, TagIn, Method) -> case match_tags(Tlv,TagIn) of Bin when is_binary(Bin) -> - {InnerTlv,_} = decode(Bin), + {InnerTlv,_} = decode(Bin,Method), InnerTlv; TlvBytes -> TlvBytes end. @@ -1577,14 +1553,12 @@ e_object_identifier(V) when is_tuple(V) -> e_object_identifier([E1, E2 | Tail]) -> Head = 40*E1 + E2, % wow! {H,Lh} = mk_object_val(Head), - {R,Lr} = enc_obj_id_tail(Tail, [], 0), + {R,Lr} = lists:mapfoldl(fun enc_obj_id_tail/2,0,Tail), {[H|R], Lh+Lr}. -enc_obj_id_tail([], Ack, Len) -> - {lists:reverse(Ack), Len}; -enc_obj_id_tail([H|T], Ack, Len) -> +enc_obj_id_tail(H, Len) -> {B, L} = mk_object_val(H), - enc_obj_id_tail(T, [B|Ack], Len+L). + {B,Len+L}. %%%%%%%%%%% diff --git a/lib/asn1/src/asn1rt_driver_handler.erl b/lib/asn1/src/asn1rt_driver_handler.erl deleted file mode 100644 index 146d0043f9..0000000000 --- a/lib/asn1/src/asn1rt_driver_handler.erl +++ /dev/null @@ -1,144 +0,0 @@ -%% -%% %CopyrightBegin% -%% -%% Copyright Ericsson AB 2002-2011. All Rights Reserved. -%% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. -%% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. -%% -%% %CopyrightEnd% -%% -%% - --module(asn1rt_driver_handler). - --include("asn1_records.hrl"). - --export([load_driver/0,unload_driver/0,client_port/0]). - -%% Internal exports --export([init/2]). - -%% Macros --define(port_names, - { asn1_drv01, asn1_drv02, asn1_drv03, asn1_drv04, - asn1_drv05, asn1_drv06, asn1_drv07, asn1_drv08, - asn1_drv09, asn1_drv10, asn1_drv11, asn1_drv12, - asn1_drv13, asn1_drv14, asn1_drv15, asn1_drv16 }). - -%%% -------------------------------------------------------- -%%% Interface Functions. -%%% -------------------------------------------------------- -load_driver() -> - load_driver(noreason). - -load_driver(Reason) -> - Ref = make_ref(), - case whereis(asn1_driver_owner) of % to prevent unnecessary spawn - Pid when is_pid(Pid) -> - asn1_driver_owner ! {self(),Ref,are_you_ready}, - receive - {Ref,driver_ready} -> - ok - after 10000 -> - {error,{timeout,waiting_for_drivers}} - end; - _ -> - {_,Mref} = spawn_monitor(asn1rt_driver_handler, init, [self(),Ref]), - receive - {'DOWN', Mref, _, _, NewReason} -> - case NewReason of - Reason -> {error,Reason}; - _ -> load_driver(NewReason) - end; - {Ref,driver_ready} -> - erlang:demonitor(Mref), - ok; - {Ref,Error = {error,_Reason}} -> - erlang:demonitor(Mref), - Error - after 10000 -> %% 10 seconds - {error,{timeout,waiting_for_drivers}} - end - end. - -init(FromPid,FromRef) -> - case catch register(asn1_driver_owner,self()) of - true -> true; - _Other -> exit(normal) - end, - Dir = filename:join([code:priv_dir(asn1),"lib"]), - case catch erl_ddll:load_driver(Dir,asn1_erl_drv) of - ok -> - Result = open_named_ports(), - catch (FromPid ! {FromRef,Result}), - loop(Result); - {error,Err} -> % if erl_ddll:load_driver fails - ForErr = erl_ddll:format_error(Err), - OSDir = filename:join(Dir,erlang:system_info(system_architecture)), - case catch erl_ddll:load_driver(OSDir,asn1_erl_drv) of - ok -> - Result = open_named_ports(), - catch (FromPid ! {FromRef,Result}), - loop(Result); - {error,Err2} -> -% catch (FromPid ! {FromRef,Error}) - ForErr2 = erl_ddll:format_error(Err2), - catch (FromPid ! {FromRef,{error,{{Dir,ForErr},{OSDir,ForErr2}}}}) - end - end. - - -open_named_ports() -> - open_named_ports(size(?port_names)). - -open_named_ports(0) -> - driver_ready; -open_named_ports(N) -> - case catch open_port({spawn,"asn1_erl_drv"},[]) of - {'EXIT',Reason} -> - {error,{port_error,Reason}}; - Port -> - register(element(N,?port_names),Port), - open_named_ports(N-1) - end. - -loop(Result) -> - receive - {_FromPid,_FromRef,unload} -> - close_ports(size(?port_names)), - erl_ddll:unload_driver(asn1_erl_drv), - ok; - {FromPid,FromRef,are_you_ready} -> - catch (FromPid ! {FromRef,driver_ready}), - loop(Result); - _ -> - loop(Result) - end. - -unload_driver() -> - case whereis(asn1_driver_owner) of - Pid when is_pid(Pid) -> - Pid ! {self(),make_ref(),unload}, - ok; - _ -> - ok - end. - -close_ports(0) -> - ok; -close_ports(N) -> - element(N,?port_names) ! {self(), close}, %% almost same as port_close(Name) - close_ports(N-1). - -client_port() -> - element(erlang:system_info(scheduler_id) rem size(?port_names) + 1, - ?port_names). diff --git a/lib/asn1/src/asn1rt_nif.erl b/lib/asn1/src/asn1rt_nif.erl new file mode 100644 index 0000000000..8580c70e6b --- /dev/null +++ b/lib/asn1/src/asn1rt_nif.erl @@ -0,0 +1,83 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2002-2011. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +%% +-module(asn1rt_nif). + +%% Nif interface for asn1 + +-export([encode_per_complete/1, + decode_ber_tlv/1]). + +-on_load(load_nif/0). + +-define(ASN1_NIF_VSN,1). + +load_nif() -> + LibBaseName = "asn1_erl_nif", + PrivDir = code:priv_dir(asn1), + LibName = case erlang:system_info(build_type) of + opt -> + LibBaseName; + Type -> + LibTypeName = LibBaseName ++ "." ++ atom_to_list(Type), + case (filelib:wildcard( + filename:join( + [PrivDir, + "lib", + LibTypeName ++ "*"])) /= []) orelse + (filelib:wildcard( + filename:join( + [PrivDir, + "lib", + erlang:system_info(system_architecture), + LibTypeName ++ "*"])) /= []) of + true -> LibTypeName; + false -> LibBaseName + end + end, + Lib = filename:join([PrivDir, "lib", LibName]), + Status = case erlang:load_nif(Lib, ?ASN1_NIF_VSN) of + ok -> ok; + {error, {load_failed, _}}=Error1 -> + ArchLibDir = + filename:join([PrivDir, "lib", + erlang:system_info(system_architecture)]), + Candidate = + filelib:wildcard(filename:join([ArchLibDir,LibName ++ "*" ])), + case Candidate of + [] -> Error1; + _ -> + ArchLib = filename:join([ArchLibDir, LibName]), + erlang:load_nif(ArchLib, ?ASN1_NIF_VSN) + end; + Error1 -> Error1 + end, + case Status of + ok -> ok; + {error, {E, Str}} -> + error_logger:error_msg("Unable to load asn1 nif library. " + "Failed with error:~n\"~p, ~s\"~n",[E,Str]), + Status + end. + +encode_per_complete(_Binary) -> + erlang:nif_error({nif_not_loaded,module,?MODULE,line,?LINE}). + +decode_ber_tlv(_Binary) -> + erlang:nif_error({nif_not_loaded,module,?MODULE,line,?LINE}). diff --git a/lib/asn1/src/asn1rt_per_bin.erl b/lib/asn1/src/asn1rt_per_bin.erl index 6bbca26209..a124c7553d 100644 --- a/lib/asn1/src/asn1rt_per_bin.erl +++ b/lib/asn1/src/asn1rt_per_bin.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2001-2009. All Rights Reserved. +%% Copyright Ericsson AB 2001-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -57,7 +57,7 @@ encode_NumericString/2, decode_NumericString/2, encode_ObjectDescriptor/2, decode_ObjectDescriptor/1 ]). --export([complete_bytes/1]). +-export([complete_bytes/1, getbits/2, getoctets/2]). -define('16K',16384). -define('32K',32768). diff --git a/lib/asn1/src/asn1rt_per_bin_rt2ct.erl b/lib/asn1/src/asn1rt_per_bin_rt2ct.erl index f4aecf9322..c7ead680ce 100644 --- a/lib/asn1/src/asn1rt_per_bin_rt2ct.erl +++ b/lib/asn1/src/asn1rt_per_bin_rt2ct.erl @@ -1734,143 +1734,24 @@ get_constraint(C,Key) -> -ifdef(nodriver). complete(L) -> - case complete1(L) of - {[],[]} -> - <<0>>; - {Acc,[]} -> - Acc; - {Acc,Bacc} -> - [Acc|complete_bytes(Bacc)] - end. - - -% this function builds the ugly form of lists [E1|E2] to avoid having to reverse it at the end. -% this is done because it is efficient and that the result always will be sent on a port or -% converted by means of list_to_binary/1 - complete1(InList) when is_list(InList) -> - complete1(InList,[],[]); - complete1(InList) -> - complete1([InList],[],[]). - - complete1([],Acc,Bacc) -> - {Acc,Bacc}; - complete1([H|T],Acc,Bacc) when is_list(H) -> - {NewH,NewBacc} = complete1(H,Acc,Bacc), - complete1(T,NewH,NewBacc); - - complete1([{octets,Bin}|T],Acc,[]) -> - complete1(T,[Acc|Bin],[]); - - complete1([{octets,Bin}|T],Acc,Bacc) -> - complete1(T,[Acc|[complete_bytes(Bacc),Bin]],[]); - - complete1([{debug,_}|T], Acc,Bacc) -> - complete1(T,Acc,Bacc); - - complete1([{bits,N,Val}|T],Acc,Bacc) -> - complete1(T,Acc,complete_update_byte(Bacc,Val,N)); - - complete1([{bit,Val}|T],Acc,Bacc) -> - complete1(T,Acc,complete_update_byte(Bacc,Val,1)); - - complete1([align|T],Acc,[]) -> - complete1(T,Acc,[]); - complete1([align|T],Acc,Bacc) -> - complete1(T,[Acc|complete_bytes(Bacc)],[]); - complete1([{0,Bin}|T],Acc,[]) when is_binary(Bin) -> - complete1(T,[Acc|Bin],[]); - complete1([{Unused,Bin}|T],Acc,[]) when is_integer(Unused),is_binary(Bin) -> - Size = size(Bin)-1, - <<Bs:Size/binary,B>> = Bin, - NumBits = 8-Unused, - complete1(T,[Acc|Bs],[[B bsr Unused]|NumBits]); - complete1([{Unused,Bin}|T],Acc,Bacc) when is_integer(Unused),is_binary(Bin) -> - Size = size(Bin)-1, - <<Bs:Size/binary,B>> = Bin, - NumBits = 8 - Unused, - Bf = complete_bytes(Bacc), - complete1(T,[Acc|[Bf,Bs]],[[B bsr Unused]|NumBits]). - - - complete_update_byte([],Val,Len) -> - complete_update_byte([[0]|0],Val,Len); - complete_update_byte([[Byte|Bacc]|NumBits],Val,Len) when NumBits + Len == 8 -> - [[0,((Byte bsl Len) + Val) band 255|Bacc]|0]; - complete_update_byte([[Byte|Bacc]|NumBits],Val,Len) when NumBits + Len > 8 -> - Rem = 8 - NumBits, - Rest = Len - Rem, - complete_update_byte([[0,((Byte bsl Rem) + (Val bsr Rest)) band 255 |Bacc]|0],Val,Rest); - complete_update_byte([[Byte|Bacc]|NumBits],Val,Len) -> - [[((Byte bsl Len) + Val) band 255|Bacc]|NumBits+Len]. - - - complete_bytes([[Byte|Bacc]|0]) -> - lists:reverse(Bacc); - complete_bytes([[Byte|Bacc]|NumBytes]) -> - lists:reverse([(Byte bsl (8-NumBytes)) band 255|Bacc]); - complete_bytes([]) -> - []. + erlang_complete(L). -else. -%% asn1-1.6.8.1_dev -%% complete(L) -> -%% case catch port_control(asn1_driver_port,1,L) of -%% Bin when is_binary(Bin) -> -%% Bin; -%% List when is_list(List) -> handle_error(List,L); -%% {'EXIT',{badarg,Reason}} -> -%% asn1rt_driver_handler:load_driver(), -%% receive -%% driver_ready -> -%% case catch port_control(asn1_driver_port,1,L) of -%% Bin2 when is_binary(Bin2) -> Bin2; -%% List when is_list(List) -> handle_error(List,L); -%% {'EXIT',Reason2={badarg,_R}} -> -%% exit({"failed to call driver probably due to bad asn1 value",Reason2}); -%% Reason2 -> exit(Reason2) -%% end; -%% {error,Error} -> % error when loading driver -%% %% the driver could not be loaded -%% exit(Error); -%% Error={port_error,Reason} -> -%% exit(Error) -%% end; -%% {'EXIT',Reason} -> -%% exit(Reason) -%% end. - -%% asn1-1.6.9 +%% asn1-1.7 complete(L) -> - case catch control(?COMPLETE_ENCODE,L) of - Bin when is_binary(Bin) -> - Bin; - List when is_list(List) -> handle_error(List,L); - {'EXIT',{badarg,_Reason}} -> - case asn1rt:load_driver() of - ok -> - case control(?COMPLETE_ENCODE,L) of - Bin when is_binary(Bin) ->Bin; - List when is_list(List) -> handle_error(List,L) - end; - Err -> - Err - end + case asn1rt_nif:encode_per_complete(L) of + {error, Reason} -> handle_error(Reason, L); + Else when is_binary(Else) -> Else end. - handle_error([],_)-> exit({error,{asn1,{"memory allocation problem in driver"}}}); -handle_error("1",L) -> % error in complete in driver +handle_error($1,L) -> % error in complete in driver exit({error,{asn1,L}}); handle_error(ErrL,L) -> exit({error,{asn1,ErrL,L}}). -%% asn1-1.6.9 -control(Cmd, Data) -> - Port = asn1rt_driver_handler:client_port(), - erlang:port_control(Port, Cmd, Data). - -endif. diff --git a/lib/asn1/test/asn1.cover b/lib/asn1/test/asn1.cover index 589a8b7e3d..ad3a0f3db9 100644 --- a/lib/asn1/test/asn1.cover +++ b/lib/asn1/test/asn1.cover @@ -1,2 +1,3 @@ {incl_app,asn1,details}. +{excl_mods, asn1, [asn1rt_nif]}.
\ No newline at end of file diff --git a/lib/asn1/test/asn1_SUITE.erl.src b/lib/asn1/test/asn1_SUITE.erl.src index 582ccd877c..5339ad78c7 100644 --- a/lib/asn1/test/asn1_SUITE.erl.src +++ b/lib/asn1/test/asn1_SUITE.erl.src @@ -2036,11 +2036,7 @@ rtUI(Config) -> ?line {ok,_} = asn1rt:info('Prim'), ?line ok = asn1ct:compile(filename:join(DataDir,"Prim"),[?PER]), - ?line {ok,_} = asn1rt:info('Prim'), - - ?line ok = asn1rt:load_driver(), - ?line ok = asn1rt:load_driver(), - ?line ok = asn1rt:unload_driver(). + ?line {ok,_} = asn1rt:info('Prim'). testROSE(suite) -> []; testROSE(Config) -> diff --git a/lib/asn1/test/asn1_bin_v2_particular_SUITE.erl.src b/lib/asn1/test/asn1_bin_v2_particular_SUITE.erl.src index abd21b0d78..6e15aa9fdc 100644 --- a/lib/asn1/test/asn1_bin_v2_particular_SUITE.erl.src +++ b/lib/asn1/test/asn1_bin_v2_particular_SUITE.erl.src @@ -1,5 +1,5 @@ -particular() -> [smp, ticket7904]. +particular() -> [smp, per_performance, ber_performance, ticket7904]. smp(suite) -> []; @@ -11,37 +11,137 @@ smp(Config) -> ?line Msg = {initiatingMessage, testNBAPsystem:cell_setup_req_msg()}, ?line ok = testNBAPsystem:compile(Config,per_bin,[optimize]), - - Parent = self(), - ?line ok = asn1rt:load_driver(), - - smp2(Parent,NumOfProcs,Msg,2), + enc_dec(NumOfProcs,Msg,2), N = 10000, - ?line {Time1,ok} = timer:tc(?MODULE,smp2,[Parent,NumOfProcs,Msg, N]), - ?line {Time1S,ok} = timer:tc(?MODULE,sequential,[NumOfProcs * N,Msg]), + ?line {Time1,ok} = timer:tc(?MODULE,enc_dec,[NumOfProcs,Msg, N]), + ?line {Time1S,ok} = timer:tc(?MODULE,enc_dec,[1, Msg, NumOfProcs * N]), - ?line ok = testNBAPsystem:compile(Config,ber_bin,[optimize,driver]), - ?line {Time2,ok} = timer:tc(?MODULE,smp2,[Parent,NumOfProcs,Msg, N]), + ?line ok = testNBAPsystem:compile(Config,ber_bin,[optimize,nif]), + ?line {Time3,ok} = timer:tc(?MODULE,enc_dec,[NumOfProcs,Msg, N]), - ?line {Time2S,ok} = timer:tc(?MODULE,sequential,[NumOfProcs * N,Msg]), + ?line {Time3S,ok} = timer:tc(?MODULE,enc_dec,[1, Msg, NumOfProcs * N]), - {comment,lists:flatten(io_lib:format("Encode/decode time parallell with ~p cores: ~p [microsecs]~nEncode/decode time sequential: ~p [microsecs]",[NumOfProcs,Time1+Time2,Time1S+Time2S]))}; + {comment,lists:flatten( + io_lib:format( + "Encode/decode time parallell with ~p cores: ~p [microsecs]~n" + "Encode/decode time sequential: ~p [microsecs]", + [NumOfProcs,Time1+Time3,Time1S+Time3S]))}; false -> {skipped,"No smp support"} end. -smp2(Parent,NumOfProcs,Msg, N) -> - Pids = [spawn_link(fun() -> worker(Msg,Parent, N) end) - || _ <- lists:seq(1,NumOfProcs)], - ?line ok = wait_pids(Pids). +per_performance(Config) -> + + ?line Msg = {initiatingMessage, testNBAPsystem:cell_setup_req_msg()}, + ?line ok = testNBAPsystem:compile(Config,per_bin,[optimize]), + NumOfProcs = erlang:system_info(schedulers)*10, + N = 10000, + SmpN = lists:seq(1,round(N/NumOfProcs)), + + asn1_wrapper:encode('NBAP-PDU-Discriptions','NBAP-PDU', Msg), + + PerFun = fun() -> + [asn1_wrapper:encode('NBAP-PDU-Discriptions', + 'NBAP-PDU', + Msg) || _I <- lists:seq(1,N)], + ok + end, + + PerSMPFun = + fun() -> + pforeach(fun(_) -> + [asn1_wrapper:encode('NBAP-PDU-Discriptions', + 'NBAP-PDU', + Msg) || _I <- SmpN] + end,lists:seq(1,NumOfProcs)) + end, + + ?line {TimeN,ok} = timer:tc(PerFun), + ?line {TimeNS,ok} = timer:tc(PerSMPFun), + + ?line ok = testNBAPsystem:compile(Config,per_bin,[]), + + ?line {TimeE,ok} = timer:tc(PerFun), + ?line {TimeES,ok} = timer:tc(PerSMPFun), + + ct:log("Seq:<br/>" + "Nif : ~p (~.2f%)<br/>" + "Erlang: ~p (~.2f%)<br/>" + "Parallel:<br/>" + "Nif : ~p (~.2f%)<br/>" + "Erlang: ~p (~.2f%)<br/>", + [TimeN,TimeN/TimeN*100, + TimeE,TimeE/TimeN*100, + TimeNS,TimeNS/TimeNS*100, + TimeES,TimeES/TimeNS*100]), + + {comment, lists:flatten(io_lib:format("Nifs are ~.2f% faster than erlang!", + [faster(TimeN+TimeNS, + TimeE+TimeES)]))}. + +ber_performance(Config) -> + + ?line Msg = {initiatingMessage, testNBAPsystem:cell_setup_req_msg()}, + ?line ok = testNBAPsystem:compile(Config,ber_bin,[optimize,nif]), + NumOfProcs = erlang:system_info(schedulers)*10, + N = 10000, + SmpN = lists:seq(1,round(N/NumOfProcs)), + + {ok,B} = asn1_wrapper:encode('NBAP-PDU-Discriptions','NBAP-PDU', Msg), + + BerFun = fun() -> + [asn1_wrapper:decode( + 'NBAP-PDU-Discriptions', + 'NBAP-PDU', + B) || _I <- lists:seq(1,N)], + ok + end, + BerSMPFun = + fun() -> + pforeach(fun(_) -> + [asn1_wrapper:decode( + 'NBAP-PDU-Discriptions', + 'NBAP-PDU', + B) || _I <- SmpN] + end,lists:seq(1,NumOfProcs)) + end, + + ?line {TimeN,ok} = timer:tc(BerFun), + ?line {TimeNS,ok} = timer:tc(BerSMPFun), + + ?line ok = testNBAPsystem:compile(Config,ber_bin,[optimize]), + + ?line {TimeE,ok} = timer:tc(BerFun), + ?line {TimeES,ok} = timer:tc(BerSMPFun), + + ct:log("Seq:<br/>" + "Nif : ~p (~.2f%)<br/>" + "Erlang: ~p (~.2f%)<br/>" + "Parallel:<br/>" + "Nif : ~p (~.2f%)<br/>" + "Erlang: ~p (~.2f%)<br/>", + [TimeN,TimeN/TimeN*100, + TimeE,TimeE/TimeN*100, + TimeNS,TimeNS/TimeNS*100, + TimeES,TimeES/TimeNS*100]), + + {comment, lists:flatten(io_lib:format("Nifs are ~.2f% faster than erlang!", + [faster(TimeN+TimeNS, + TimeE+TimeES)]))}. -worker(Msg, Parent, N) -> - %% io:format("smp worker ~p with ~p worker loops.~n",[self(), N]), - worker_loop(N, Msg), - Parent ! self(). + +faster(A,B) -> + (B - A)/B * 100. + +enc_dec(1, Msg, N) -> + worker_loop(N, Msg); +enc_dec(NumOfProcs,Msg, N) -> + pforeach(fun(_) -> + worker_loop(N, Msg) + end, [I || I <- lists:seq(1,NumOfProcs)]). worker_loop(0, _Msg) -> ok; @@ -50,28 +150,24 @@ worker_loop(N, Msg) -> 'NBAP-PDU', Msg), ?line {ok,_Msg}=asn1_wrapper:decode('NBAP-PDU-Discriptions', - 'NBAP-PDU', - B), + 'NBAP-PDU', + B), worker_loop(N - 1, Msg). -wait_pids([]) -> - ok; -wait_pids(Pids) -> +pforeach(Fun, List) -> + pforeach(Fun, List, []). +pforeach(Fun, [], [{Pid,Ref}|Pids]) -> receive - Pid when is_pid(Pid) -> - ?line true = lists:member(Pid,Pids), - Others = lists:delete(Pid,Pids), - io:format("wait_pid got ~p, still waiting for ~p\n",[Pid,Others]), - wait_pids(Others); - Err -> - io:format("Err: ~p~n",[Err]), - ?line exit(Err) - end. - -sequential(N,Msg) -> - %%io:format("sequential encode/decode with N = ~p~n",[N]), - worker_loop(N,Msg). + {'DOWN', Ref, process, Pid, normal} -> + pforeach(Fun, [], Pids) + end; +pforeach(Fun, [H|T], Pids) -> + Pid = spawn(fun() -> Fun(H) end), + Ref = erlang:monitor(process, Pid), + pforeach(Fun, T, [{Pid, Ref}|Pids]); +pforeach(_Fun,[],[]) -> + ok. -record('InitiatingMessage',{procedureCode,criticality,value}). -record('Iu-ReleaseCommand',{first,second}). diff --git a/lib/asn1/test/ber_decode_error.erl b/lib/asn1/test/ber_decode_error.erl index 96d6545636..a566e0b07f 100644 --- a/lib/asn1/test/ber_decode_error.erl +++ b/lib/asn1/test/ber_decode_error.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1999-2010. All Rights Reserved. +%% Copyright Ericsson AB 1999-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -45,6 +45,10 @@ run([]) -> run([driver]) -> %% test of OTP-4797, bad indata to driver does not cause an EXIT ?line {error,_Reason} = asn1rt:decode('Constructed','S3',[3,5]), + ok; +run([nif]) -> + %% test of OTP-4797, bad indata to driver does not cause an EXIT + ?line {error,_Reason} = asn1rt:decode('Constructed','S3',[3,5]), ok. diff --git a/lib/asn1/test/testPrim.erl b/lib/asn1/test/testPrim.erl index 97f99e7b1c..39c1e4d1d8 100644 --- a/lib/asn1/test/testPrim.erl +++ b/lib/asn1/test/testPrim.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2010. All Rights Reserved. +%% Copyright Ericsson AB 1997-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -37,21 +37,10 @@ compile(Config,Rules,Opt) -> ?line DataDir = ?config(data_dir,Config), ?line OutDir = ?config(priv_dir,Config), ?line true = code:add_patha(?config(priv_dir,Config)), - case Opt of - [optimize] -> - ?line ok = asn1ct:compile(DataDir ++ "Prim", - [Rules,optimize,{outdir,OutDir}]), - ?line ok = asn1ct:compile(DataDir ++ "Real", - [Rules,optimize,{outdir,OutDir}]); - __ -> - ?line ok = asn1ct:compile(DataDir ++ "Prim", - [Rules,{outdir,OutDir}]), - ?line ok = asn1ct:compile(DataDir ++ "Real", - [Rules,{outdir,OutDir}]) - end. - - - + ?line ok = asn1ct:compile(DataDir ++ "Prim", + [Rules,{outdir,OutDir}] ++ Opt), + ?line ok = asn1ct:compile(DataDir ++ "Real", + [Rules,{outdir,OutDir}] ++ Opt). bool(Rules) -> diff --git a/lib/compiler/src/erl_bifs.erl b/lib/compiler/src/erl_bifs.erl index f8128702dd..2514c06360 100644 --- a/lib/compiler/src/erl_bifs.erl +++ b/lib/compiler/src/erl_bifs.erl @@ -72,7 +72,6 @@ is_pure(erlang, binary_to_list, 1) -> true; is_pure(erlang, binary_to_list, 3) -> true; is_pure(erlang, bit_size, 1) -> true; is_pure(erlang, byte_size, 1) -> true; -is_pure(erlang, concat_binary, 1) -> true; is_pure(erlang, element, 2) -> true; is_pure(erlang, float, 1) -> true; is_pure(erlang, float_to_list, 1) -> true; diff --git a/lib/debugger/src/dbg_iload.erl b/lib/debugger/src/dbg_iload.erl index 2ae0c333da..db5a17ad2e 100644 --- a/lib/debugger/src/dbg_iload.erl +++ b/lib/debugger/src/dbg_iload.erl @@ -635,7 +635,6 @@ bif_type(disconnect_node) -> safe; bif_type(binary_to_list) -> safe; bif_type(list_to_binary) -> safe; bif_type(split_binary) -> safe; -bif_type(concat_binary) -> safe; bif_type(term_to_atom) -> safe; bif_type(hash) -> safe; bif_type(pre_loaded) -> safe; diff --git a/lib/hipe/cerl/erl_bif_types.erl b/lib/hipe/cerl/erl_bif_types.erl index f1be658054..64a695129b 100644 --- a/lib/hipe/cerl/erl_bif_types.erl +++ b/lib/hipe/cerl/erl_bif_types.erl @@ -675,8 +675,6 @@ type(erlang, cancel_timer, 1, Xs) -> type(erlang, check_process_code, 2, Xs) -> strict(arg_types(erlang, check_process_code, 2), Xs, fun (_) -> t_boolean() end); -type(erlang, concat_binary, 1, Xs) -> - strict(arg_types(erlang, concat_binary, 1), Xs, fun (_) -> t_binary() end); type(erlang, crc32, 1, Xs) -> strict(arg_types(erlang, crc32, 1), Xs, fun (_) -> t_crc32() end); type(erlang, crc32, 2, Xs) -> @@ -3397,8 +3395,6 @@ arg_types(erlang, cancel_timer, 1) -> [t_reference()]; arg_types(erlang, check_process_code, 2) -> [t_pid(), t_atom()]; -arg_types(erlang, concat_binary, 1) -> - [t_list(t_binary())]; arg_types(erlang, crc32, 1) -> [t_iodata()]; arg_types(erlang, crc32, 2) -> diff --git a/lib/kernel/doc/src/gen_sctp.xml b/lib/kernel/doc/src/gen_sctp.xml index 5ceb82ae41..b761b6bd83 100644 --- a/lib/kernel/doc/src/gen_sctp.xml +++ b/lib/kernel/doc/src/gen_sctp.xml @@ -990,7 +990,7 @@ server(IP, Port) when is_tuple(IP) orelse IP == any orelse IP == loopback, is_integer(Port) -> - {ok,S} = gen_sctp:open([{ip,IP},{port,Port}],[{recbuf,65536}]), + {ok,S} = gen_sctp:open(Port, [{recbuf,65536}, {ip,IP}]), io:format("Listening on ~w:~w. ~w~n", [IP,Port,S]), ok = gen_sctp:listen(S, true), server_loop(S). diff --git a/lib/sasl/src/release_handler.erl b/lib/sasl/src/release_handler.erl index eb29787103..ab54b1d00b 100644 --- a/lib/sasl/src/release_handler.erl +++ b/lib/sasl/src/release_handler.erl @@ -291,7 +291,8 @@ check_script(Script, LibDirs) -> release_handler_1:check_script(Script, LibDirs). %%----------------------------------------------------------------- -%% eval_script(Script, Apps, LibDirs, Opts) -> {ok, UnPurged} | +%% eval_script(Script, Apps, LibDirs, NewLibs, Opts) -> +%% {ok, UnPurged} | %% restart_new_emulator | %% {error, Error} %% {'EXIT', Reason} @@ -299,9 +300,13 @@ check_script(Script, LibDirs) -> %% net_kernel:monitor_nodes(true) before calling this function. %% No! No other process than the release_handler can ever call this %% function, if sync_nodes is used. -%%----------------------------------------------------------------- -eval_script(Script, Apps, LibDirs, Opts) -> - catch release_handler_1:eval_script(Script, Apps, LibDirs, Opts). +%% +%% LibDirs is a list of all applications, while NewLibs is a list of +%% applications that have changed version between the current and the +%% new release. +%% ----------------------------------------------------------------- +eval_script(Script, Apps, LibDirs, NewLibs, Opts) -> + catch release_handler_1:eval_script(Script, Apps, LibDirs, NewLibs, Opts). %%----------------------------------------------------------------- %% Func: create_RELEASES(Root, RelFile, LibDirs) -> ok | {error, Reason} @@ -405,6 +410,7 @@ eval_appup_script(App, ToVsn, ToDir, Script) -> Res = release_handler_1:eval_script(Script, [], % [AppSpec] [{App, ToVsn, ToDir}], + [{App, ToVsn, ToDir}], []), % [Opt] case Res of {ok, _Unpurged} -> @@ -906,7 +912,9 @@ do_install_release(#state{start_prg = StartPrg, EnvBefore = application_controller:prep_config_change(), Apps = change_appl_data(RelDir, Release, Masters), LibDirs = Release#release.libs, - case eval_script(Script, Apps, LibDirs, Opts) of + NewLibs = get_new_libs(LatestRelease#release.libs, + Release#release.libs), + case eval_script(Script, Apps, LibDirs, NewLibs, Opts) of {ok, []} -> application_controller:config_change(EnvBefore), mon_nodes(false), @@ -1946,3 +1954,25 @@ safe_write_file_m(File, Data, Masters) -> filename:basename(File), filename:basename(Backup)}}) end. + +%%----------------------------------------------------------------- +%% Figure out which applications that have changed version between the +%% two releases. The paths for these applications must always be +%% updated, even if the relup script does not load any modules. See +%% OTP-9402. +%% +%% A different situation is when the same application version is used +%% in old and new release, but the path has changed. This is not +%% handled here - instead it must be explicitely indicated by the +%% 'update_paths' option to release_handler:install_release/2 if the +%% code path shall be updated then. +%% ----------------------------------------------------------------- +get_new_libs([{App,Vsn,LibDir}|CurrentLibs], NewLibs) -> + case lists:keyfind(App,1,NewLibs) of + {App,NewVsn,_} = LibInfo when NewVsn =/= Vsn -> + [LibInfo | get_new_libs(CurrentLibs,NewLibs)]; + _ -> + get_new_libs(CurrentLibs,NewLibs) + end; +get_new_libs([],_) -> + []. diff --git a/lib/sasl/src/release_handler_1.erl b/lib/sasl/src/release_handler_1.erl index 8d050fb7b0..ff62f847ac 100644 --- a/lib/sasl/src/release_handler_1.erl +++ b/lib/sasl/src/release_handler_1.erl @@ -19,7 +19,7 @@ -module(release_handler_1). %% External exports --export([eval_script/3, eval_script/4, check_script/2]). +-export([eval_script/1, eval_script/5, check_script/2]). -export([get_current_vsn/1]). %% exported because used in a test case -record(eval_state, {bins = [], stopped = [], suspended = [], apps = [], @@ -33,11 +33,11 @@ %% libdirs = [{Lib, LibVsn, LibDir}] - Maps Lib to Vsn and Directory %% unpurged = [{Mod, soft_purge | brutal_purge}] %% vsns = [{Mod, OldVsn, NewVsn}] - remember the old vsn of a mod -%% before it is removed/a new vsn is loaded; the new vsn +%% before a new vsn is loaded; the new vsn %% is kept in case of a downgrade, where the code_change %% function receives the vsn of the module to downgrade %% *to*. -%% newlibs = [{Lib, Dir}] - list of all new libs; used to change +%% newlibs = [{Lib, LibVsn, LibDir}] - list of all new libs; used to change %% the code path %% opts = [{Tag, Value}] - list of options %%----------------------------------------------------------------- @@ -63,10 +63,11 @@ check_script(Script, LibDirs) -> {error, {old_processes, Mod}} end. -eval_script(Script, Apps, LibDirs) -> - eval_script(Script, Apps, LibDirs, []). +%% eval_script/1 - For testing only - no apps added, just testing instructions +eval_script(Script) -> + eval_script(Script, [], [], [], []). -eval_script(Script, Apps, LibDirs, Opts) -> +eval_script(Script, Apps, LibDirs, NewLibs, Opts) -> case catch check_old_processes(Script) of ok -> {Before, After} = split_instructions(Script), @@ -75,6 +76,7 @@ eval_script(Script, Apps, LibDirs, Opts) -> end, #eval_state{apps = Apps, libdirs = LibDirs, + newlibs = NewLibs, opts = Opts}, Before) of EvalState2 when is_record(EvalState2, eval_state) -> @@ -214,16 +216,15 @@ check_old_code(Mod) -> %%----------------------------------------------------------------- eval({load_object_code, {Lib, LibVsn, Modules}}, EvalState) -> case lists:keysearch(Lib, 1, EvalState#eval_state.libdirs) of - {value, {Lib, LibVsn, LibDir}} -> - Ebin = filename:join(LibDir, "ebin"), + {value, {Lib, LibVsn, LibDir} = LibInfo} -> Ext = code:objfile_extension(), {NewBins, NewVsns} = lists:foldl(fun(Mod, {Bins, Vsns}) -> File = lists:concat([Mod, Ext]), - FName = filename:join(Ebin, File), + FName = filename:join([LibDir, "ebin", File]), case erl_prim_loader:get_file(FName) of {ok, Bin, FName2} -> - NVsns = add_new_vsn(Mod, Bin, Vsns), + NVsns = add_vsns(Mod, Bin, Vsns), {[{Mod, Bin, FName2} | Bins],NVsns}; error -> throw({error, {no_such_file,FName}}) @@ -232,7 +233,7 @@ eval({load_object_code, {Lib, LibVsn, Modules}}, EvalState) -> {EvalState#eval_state.bins, EvalState#eval_state.vsns}, Modules), - NewLibs = [{Lib, Ebin} | EvalState#eval_state.newlibs], + NewLibs = lists:keystore(Lib,1,EvalState#eval_state.newlibs,LibInfo), EvalState#eval_state{bins = NewBins, newlibs = NewLibs, vsns = NewVsns}; @@ -242,15 +243,14 @@ eval({load_object_code, {Lib, LibVsn, Modules}}, EvalState) -> eval(point_of_no_return, EvalState) -> Libs = case get_opt(update_paths, EvalState, false) of false -> - EvalState#eval_state.newlibs; % [{Lib, Path}] + EvalState#eval_state.newlibs; true -> - lists:map(fun({Lib, _LibVsn, LibDir}) -> - Ebin= filename:join(LibDir,"ebin"), - {Lib, Ebin} - end, - EvalState#eval_state.libdirs) + EvalState#eval_state.libdirs end, - lists:foreach(fun({Lib, Path}) -> code:replace_path(Lib, Path) end, + lists:foreach(fun({Lib, _LibVsn, LibDir}) -> + Ebin = filename:join(LibDir,"ebin"), + code:replace_path(Lib, Ebin) + end, Libs), EvalState; eval({load, {Mod, _PrePurgeMethod, PostPurgeMethod}}, EvalState) -> @@ -258,32 +258,21 @@ eval({load, {Mod, _PrePurgeMethod, PostPurgeMethod}}, EvalState) -> {value, {_Mod, Bin, File}} = lists:keysearch(Mod, 1, Bins), % load_binary kills all procs running old code % if soft_purge, we know that there are no such procs now - Vsns = EvalState#eval_state.vsns, - NewVsns = add_old_vsn(Mod, Vsns), code:load_binary(Mod, File, Bin), % Now, the prev current is old. There might be procs % running it. Find them. Unpurged = do_soft_purge(Mod,PostPurgeMethod,EvalState#eval_state.unpurged), EvalState#eval_state{bins = lists:keydelete(Mod, 1, Bins), - unpurged = Unpurged, - vsns = NewVsns}; + unpurged = Unpurged}; eval({remove, {Mod, _PrePurgeMethod, PostPurgeMethod}}, EvalState) -> - % purge kills all procs running old code - % if soft_purge, we know that there are no such procs now - Vsns = EvalState#eval_state.vsns, - NewVsns = add_old_vsn(Mod, Vsns), + %% purge kills all procs running old code + %% if soft_purge, we know that there are no such procs now code:purge(Mod), code:delete(Mod), - % Now, the prev current is old. There might be procs - % running it. Find them. - Unpurged = - case code:soft_purge(Mod) of - true -> EvalState#eval_state.unpurged; - false -> [{Mod, PostPurgeMethod} | EvalState#eval_state.unpurged] - end, -%% Bins = EvalState#eval_state.bins, -%% EvalState#eval_state{bins = lists:keydelete(Mod, 1, Bins), - EvalState#eval_state{unpurged = Unpurged, vsns = NewVsns}; + %% Now, the prev current is old. There might be procs + %% running it. Find them. + Unpurged = do_soft_purge(Mod,PostPurgeMethod,EvalState#eval_state.unpurged), + EvalState#eval_state{unpurged = Unpurged}; eval({purge, Modules}, EvalState) -> % Now, if there are any processes still executing old code, OR % if some new processes started after suspend but before load, @@ -606,26 +595,20 @@ sync_nodes(Id, Nodes) -> end, NNodes). -add_old_vsn(Mod, Vsns) -> +add_vsns(Mod, NewBin, Vsns) -> + OldVsn = get_current_vsn(Mod), + NewVsn = get_vsn(NewBin), case lists:keysearch(Mod, 1, Vsns) of - {value, {Mod, undefined, NewVsn}} -> - OldVsn = get_current_vsn(Mod), - lists:keyreplace(Mod, 1, Vsns, {Mod, OldVsn, NewVsn}); - {value, {Mod, _OldVsn, _NewVsn}} -> - Vsns; + {value, {Mod, OldVsn0, NewVsn0}} -> + lists:keyreplace(Mod, 1, Vsns, {Mod, + replace_undefined(OldVsn0,OldVsn), + replace_undefined(NewVsn0,NewVsn)}); false -> - OldVsn = get_current_vsn(Mod), - [{Mod, OldVsn, undefined} | Vsns] + [{Mod, OldVsn, NewVsn} | Vsns] end. -add_new_vsn(Mod, Bin, Vsns) -> - NewVsn = get_vsn(Bin), - case lists:keysearch(Mod, 1, Vsns) of - {value, {Mod, OldVsn, undefined}} -> - lists:keyreplace(Mod, 1, Vsns, {Mod, OldVsn, NewVsn}); - false -> - [{Mod, undefined, NewVsn} | Vsns] - end. +replace_undefined(undefined,Vsn) -> Vsn; +replace_undefined(Vsn,_) -> Vsn. %%----------------------------------------------------------------- %% Func: get_current_vsn/1 @@ -645,7 +628,9 @@ get_current_vsn(Mod) -> {ok, Bin, _File2} -> get_vsn(Bin); error -> - throw({error, {no_such_file, File}}) + %% This is the case when a new module is added, there will + %% be no current version of it at the time of this call. + undefined end. %%----------------------------------------------------------------- diff --git a/lib/sasl/test/release_handler_SUITE.erl b/lib/sasl/test/release_handler_SUITE.erl index 16267ba0d4..9c7733b7ec 100644 --- a/lib/sasl/test/release_handler_SUITE.erl +++ b/lib/sasl/test/release_handler_SUITE.erl @@ -55,7 +55,7 @@ win32_cases() -> %% Cases that can be run on all platforms cases() -> - [otp_2740, otp_2760, otp_5761, instructions, eval_appup]. + [otp_2740, otp_2760, otp_5761, otp_9402, otp_9417, instructions, eval_appup]. groups() -> [{release,[], @@ -393,7 +393,7 @@ instructions(Conf) when is_list(Conf) -> {stop, [aa]}, {apply, {?MODULE, no_cc, []}}, {start, [aa]}], - {ok, _} = release_handler_1:eval_script(S1, [], []), + {ok, _} = release_handler_1:eval_script(S1), case whereis(cc) of Pid2 when is_pid(Pid2) -> ok; @@ -403,17 +403,17 @@ instructions(Conf) when is_list(Conf) -> %% Make bb run old version of b. S2 = [point_of_no_return, {remove, {b, soft_purge, soft_purge}}], - {ok, [{b, soft_purge}]} = release_handler_1:eval_script(S2, [], []), + {ok, [{b, soft_purge}]} = release_handler_1:eval_script(S2), check_bstate("first", [FirstBB]), false = code:is_loaded(b), - {error,{old_processes,b}} = release_handler_1:eval_script(S2,[],[]), + {error,{old_processes,b}} = release_handler_1:eval_script(S2), check_bstate("first", [FirstBB]), %% Let supervisor restart bb with new code S3 = [point_of_no_return, {purge, [b]}], - {ok, []} = release_handler_1:eval_script(S3, [], []), + {ok, []} = release_handler_1:eval_script(S3), ok = wait_for(bb), check_bstate("second", []), SecondBB = whereis(bb), @@ -446,7 +446,7 @@ instructions(Conf) when is_list(Conf) -> %% Let supervisor restart bb yet another time S4 = [point_of_no_return, {remove, {b, brutal_purge, soft_purge}}], - {ok, HopefullyEmpty} = release_handler_1:eval_script(S4, [], []), + {ok, HopefullyEmpty} = release_handler_1:eval_script(S4), ok = wait_for(bb), FourthBB = whereis(bb), @@ -566,8 +566,7 @@ otp_2760(Conf) -> %% Execute the relup script and check that app1 is unloaded {ok, [{"after", [{_Rel1Vsn, _Descr, Script}], _}]} = file:consult(filename:join(Rel2Dir, "relup")), - {ok, []} = rpc:call(Node, release_handler_1, eval_script, - [Script, [], []]), + {ok, []} = rpc:call(Node, release_handler_1, eval_script, [Script]), false = rpc:call(Node, code, is_loaded, [app1]), true = stop_node(Node), @@ -658,6 +657,126 @@ otp_5761(Conf) when is_list(Conf) -> true = stop_node(Node), ok. + +%% When a new version of an application is added, but no module is +%% changed - the path was not updated - i.e. code:priv_dir would point +%% to the old location. +otp_9402(Conf) when is_list(Conf) -> + %% Set some paths + PrivDir = priv_dir(Conf), + Dir = filename:join(PrivDir,"otp_9402"), + LibDir = filename:join(?config(data_dir, Conf), "lib"), + + %% Create the releases + Rel1 = create_and_install_fake_first_release(Dir, + [{a,"1.1",LibDir}]), + Rel2 = create_fake_upgrade_release(Dir, + "2", + [{a,"1.2",LibDir}], + {Rel1,Rel1,[LibDir]}), + Rel1Dir = filename:dirname(Rel1), + Rel2Dir = filename:dirname(Rel2), + + %% Start a slave node + {ok, Node} = t_start_node(otp_9402, Rel1, filename:join(Rel1Dir,"sys.config")), + + %% Check path + Dir1 = filename:join([LibDir, "a-1.1"]), + Dir1 = rpc:call(Node, code, lib_dir, [a]), + ABeam = rpc:call(Node, code, which, [a]), + + %% Install second release, with no changed modules + {ok, RelVsn2} = + rpc:call(Node, release_handler, set_unpacked, + [Rel2++".rel", [{a,"1.2",LibDir}]]), + ok = rpc:call(Node, release_handler, install_file, + [RelVsn2, filename:join(Rel2Dir, "relup")]), + ok = rpc:call(Node, release_handler, install_file, + [RelVsn2, filename:join(Rel2Dir, "start.boot")]), + ok = rpc:call(Node, release_handler, install_file, + [RelVsn2, filename:join(Rel2Dir, "sys.config")]), + + {ok, RelVsn1, []} = + rpc:call(Node, release_handler, install_release, [RelVsn2]), + + %% Check path + Dir2 = filename:join([LibDir, "a-1.2"]), + Dir2 = rpc:call(Node, code, lib_dir, [a]), + APrivDir2 = rpc:call(Node, code, priv_dir, [a]), + true = filelib:is_regular(filename:join(APrivDir2,"file")), + + %% Just to make sure no modules have been re-loaded + ABeam = rpc:call(Node, code, which, [a]), + + %% Install RelVsn1 again + {ok, _OtherVsn, []} = + rpc:call(Node, release_handler, install_release, [RelVsn1]), + + %% Check path + Dir1 = rpc:call(Node, code, lib_dir, [a]), + APrivDir1 = rpc:call(Node, code, priv_dir, [a]), + false = filelib:is_regular(filename:join(APrivDir1,"file")), + + %% Just to make sure no modules have been re-loaded + ABeam = rpc:call(Node, code, which, [a]), + + ok. + + +%% When a module is deleted in an appup instruction, the upgrade +%% failed if the module was not loaded. +otp_9417(Conf) when is_list(Conf) -> + %% Set some paths + PrivDir = priv_dir(Conf), + Dir = filename:join(PrivDir,"otp_9417"), + LibDir = filename:join(?config(data_dir, Conf), "lib"), + + %% Create the releases + Rel1 = create_and_install_fake_first_release(Dir, + [{b,"1.0",LibDir}]), + Rel2 = create_fake_upgrade_release(Dir, + "2", + [{b,"2.0",LibDir}], + {Rel1,Rel1,[LibDir]}), + Rel1Dir = filename:dirname(Rel1), + Rel2Dir = filename:dirname(Rel2), + + %% Start a slave node + {ok, Node} = t_start_node(otp_9417, Rel1, filename:join(Rel1Dir,"sys.config")), + + %% Check paths + Dir1 = filename:join([LibDir, "b-1.0"]), + Dir1 = rpc:call(Node, code, lib_dir, [b]), + BLibBeam = filename:join([Dir1,"ebin","b_lib.beam"]), + BLibBeam = rpc:call(Node,code,which,[b_lib]), + false = rpc:call(Node,code,is_loaded,[b_lib]), + false = rpc:call(Node,code,is_loaded,[b_server]), + + %% Install second release, which removes b_lib module + {ok, RelVsn2} = + rpc:call(Node, release_handler, set_unpacked, + [Rel2++".rel", [{b,"2.0",LibDir}]]), + ok = rpc:call(Node, release_handler, install_file, + [RelVsn2, filename:join(Rel2Dir, "relup")]), + ok = rpc:call(Node, release_handler, install_file, + [RelVsn2, filename:join(Rel2Dir, "start.boot")]), + ok = rpc:call(Node, release_handler, install_file, + [RelVsn2, filename:join(Rel2Dir, "sys.config")]), + + {ok, _RelVsn1, []} = + rpc:call(Node, release_handler, install_release, [RelVsn2]), + + %% Check that the module does no longer exist + false = rpc:call(Node, code, is_loaded, [b_lib]), + non_existing = rpc:call(Node, code, which, [b_lib]), + + %% And check some paths + Dir2 = filename:join([LibDir, "b-2.0"]), + Dir2 = rpc:call(Node, code, lib_dir, [b]), + BServerBeam = filename:join([Dir2,"ebin","b_server.beam"]), + {file,BServerBeam} = rpc:call(Node,code,is_loaded,[b_server]), + ok. + %% Test upgrade and downgrade of applications eval_appup(Conf) when is_list(Conf) -> diff --git a/lib/sasl/test/release_handler_SUITE_data/Makefile.src b/lib/sasl/test/release_handler_SUITE_data/Makefile.src index 85e25fdc2f..a12e526d2e 100644 --- a/lib/sasl/test/release_handler_SUITE_data/Makefile.src +++ b/lib/sasl/test/release_handler_SUITE_data/Makefile.src @@ -1,14 +1,19 @@ EFLAGS=+debug_info P2B= \ - P2B/a-2.0/ebin/a.beam \ - P2B/a-2.0/ebin/a_sup.beam + P2B/a-2.0/ebin/a.@EMULATOR@ \ + P2B/a-2.0/ebin/a_sup.@EMULATOR@ LIB= \ - lib/a-1.1/ebin/a.beam \ - lib/a-1.1/ebin/a_sup.beam \ - lib/a-1.0/ebin/a.beam \ - lib/a-1.0/ebin/a_sup.beam \ + lib/a-1.2/ebin/a.@EMULATOR@ \ + lib/a-1.2/ebin/a_sup.@EMULATOR@ \ + lib/a-1.1/ebin/a.@EMULATOR@ \ + lib/a-1.1/ebin/a_sup.@EMULATOR@ \ + lib/a-1.0/ebin/a.@EMULATOR@ \ + lib/a-1.0/ebin/a_sup.@EMULATOR@ \ + lib/b-1.0/ebin/b_server.@EMULATOR@ \ + lib/b-1.0/ebin/b_lib.@EMULATOR@ \ + lib/b-2.0/ebin/b_server.@EMULATOR@ APP= \ app1_app2/lib1/app1-1.0/ebin/app1_sup.@EMULATOR@ \ @@ -57,6 +62,21 @@ lib/a-1.1/ebin/a_sup.@EMULATOR@: lib/a-1.1/src/a_sup.erl erlc $(EFLAGS) -olib/a-1.1/ebin lib/a-1.1/src/a_sup.erl +lib/a-1.2/ebin/a.@EMULATOR@: lib/a-1.2/src/a.erl + erlc $(EFLAGS) -olib/a-1.2/ebin lib/a-1.2/src/a.erl +lib/a-1.2/ebin/a_sup.@EMULATOR@: lib/a-1.2/src/a_sup.erl + erlc $(EFLAGS) -olib/a-1.2/ebin lib/a-1.2/src/a_sup.erl + +lib/b-1.0/ebin/b_server.@EMULATOR@: lib/b-1.0/src/b_server.erl + erlc $(EFLAGS) -olib/b-1.0/ebin lib/b-1.0/src/b_server.erl +lib/b-1.0/ebin/b_lib.@EMULATOR@: lib/b-1.0/src/b_lib.erl + erlc $(EFLAGS) -olib/b-1.0/ebin lib/b-1.0/src/b_lib.erl + +lib/b-2.0/ebin/b_server.@EMULATOR@: lib/b-2.0/src/b_server.erl + erlc $(EFLAGS) -olib/b-2.0/ebin lib/b-2.0/src/b_server.erl + + + app1_app2/lib1/app1-1.0/ebin/app1_sup.@EMULATOR@: app1_app2/lib1/app1-1.0/src/app1_sup.erl erlc $(EFLAGS) -oapp1_app2/lib1/app1-1.0/ebin app1_app2/lib1/app1-1.0/src/app1_sup.erl app1_app2/lib1/app1-1.0/ebin/app1_server.@EMULATOR@: app1_app2/lib1/app1-1.0/src/app1_server.erl diff --git a/lib/sasl/test/release_handler_SUITE_data/lib/README b/lib/sasl/test/release_handler_SUITE_data/lib/README new file mode 100644 index 0000000000..667d21d4cf --- /dev/null +++ b/lib/sasl/test/release_handler_SUITE_data/lib/README @@ -0,0 +1,16 @@ +a-1.0: +start version + +a-1.1: +can be upgraded to from a-1.0. Module a has changed + +a-1.2: +can be upgraded to from a-1.1. +No module have changed, but priv dir is added including one 'file' + +b-1.0: +start version, includes b_lib and b_server + +b-2.0: +can be upgraded to from b-1.0. +Removes b_lib and updates b_server diff --git a/lib/sasl/test/release_handler_SUITE_data/lib/a-1.0/src/a.app b/lib/sasl/test/release_handler_SUITE_data/lib/a-1.2/ebin/a.app index e938137f67..b38722f06d 100644 --- a/lib/sasl/test/release_handler_SUITE_data/lib/a-1.0/src/a.app +++ b/lib/sasl/test/release_handler_SUITE_data/lib/a-1.2/ebin/a.app @@ -1,7 +1,7 @@ {application, a, [{description, "A CXC 138 11"}, - {vsn, "1.0"}, - {modules, [{a, 1}, {a_sup,1}]}, + {vsn, "1.2"}, + {modules, [{a, 2}, {a_sup,1}]}, {registered, [a_sup]}, {applications, [kernel, stdlib]}, {env, [{key1, val1}]}, diff --git a/lib/sasl/test/release_handler_SUITE_data/lib/a-1.2/ebin/a.appup b/lib/sasl/test/release_handler_SUITE_data/lib/a-1.2/ebin/a.appup new file mode 100644 index 0000000000..3df0546316 --- /dev/null +++ b/lib/sasl/test/release_handler_SUITE_data/lib/a-1.2/ebin/a.appup @@ -0,0 +1,3 @@ +{"1.2", + [{"1.1",[]}], + [{"1.1",[]}]}. diff --git a/lib/sasl/test/release_handler_SUITE_data/lib/a-1.2/priv/file b/lib/sasl/test/release_handler_SUITE_data/lib/a-1.2/priv/file new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/lib/sasl/test/release_handler_SUITE_data/lib/a-1.2/priv/file diff --git a/lib/sasl/test/release_handler_SUITE_data/lib/a-1.2/src/a.erl b/lib/sasl/test/release_handler_SUITE_data/lib/a-1.2/src/a.erl new file mode 100644 index 0000000000..c082ad5339 --- /dev/null +++ b/lib/sasl/test/release_handler_SUITE_data/lib/a-1.2/src/a.erl @@ -0,0 +1,54 @@ +%% ``The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved via the world wide web at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% The Initial Developer of the Original Code is Ericsson Utvecklings AB. +%% Portions created by Ericsson are Copyright 1999, Ericsson Utvecklings +%% AB. All Rights Reserved.'' +%% +%% $Id$ +%% +-module(a). + + +-behaviour(gen_server). + +%% External exports +-export([start_link/0, a/0, b/0]). +%% Internal exports +-export([init/1, handle_call/3, handle_info/2, terminate/2, code_change/3]). + +start_link() -> gen_server:start_link({local, aa}, a, [], []). + +a() -> gen_server:call(aa, a). +b() -> gen_server:call(aa, b). + +%%----------------------------------------------------------------- +%% Callback functions from gen_server +%%----------------------------------------------------------------- +init([]) -> + process_flag(trap_exit, true), + {ok, {state, bval}}. + +handle_call(a, _From, State) -> + X = application:get_all_env(a), + {reply, X, State}; + +handle_call(b, _From, State) -> + {reply, {ok, element(2, State)}, State}. + +handle_info(_, State) -> + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +code_change(1, Extra, State) -> + {ok, {state, bval}}. diff --git a/lib/asn1/src/asn1_sup.erl b/lib/sasl/test/release_handler_SUITE_data/lib/a-1.2/src/a_sup.erl index a241dec6f4..a141c1767b 100644 --- a/lib/asn1/src/asn1_sup.erl +++ b/lib/sasl/test/release_handler_SUITE_data/lib/a-1.2/src/a_sup.erl @@ -3,35 +3,35 @@ %% compliance with the License. You should have received a copy of the %% Erlang Public License along with this software. If not, it can be %% retrieved via the world wide web at http://www.erlang.org/. -%% +%% %% Software distributed under the License is distributed on an "AS IS" %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See %% the License for the specific language governing rights and limitations %% under the License. -%% +%% %% The Initial Developer of the Original Code is Ericsson Utvecklings AB. %% Portions created by Ericsson are Copyright 1999, Ericsson Utvecklings %% AB. All Rights Reserved.'' -%% +%% %% $Id$ %% +-module(a_sup). -%% Purpose: Main supervisor in asn1 application. - --module(asn1_sup). -behaviour(supervisor). --export([start_link/0, init/1]). +%% External exports +-export([start/2]). -start_link() -> - supervisor:start_link({local, asn1_sup}, asn1_sup, []). +%% Internal exports +-export([init/1]). +start(_, _) -> + supervisor:start_link({local, a_sup}, a_sup, []). -%% init([]) -%% Returns: {ok, {SupFlags, [ChildSpec]}} -%% init([]) -> - Child = {asn1_server, {asn1_server, start_link, []}, - permanent, 2000, worker, [asn1_server]}, - {ok, {{one_for_all, 10, 3600}, [Child]}}. + SupFlags = {one_for_one, 4, 3600}, + Config = {a, + {a, start_link, []}, + permanent, 2000, worker, [a]}, + {ok, {SupFlags, [Config]}}. diff --git a/lib/sasl/test/release_handler_SUITE_data/lib/b-1.0/ebin/b.app b/lib/sasl/test/release_handler_SUITE_data/lib/b-1.0/ebin/b.app new file mode 100644 index 0000000000..00347b2754 --- /dev/null +++ b/lib/sasl/test/release_handler_SUITE_data/lib/b-1.0/ebin/b.app @@ -0,0 +1,7 @@ +%% -*- erlang -*- +{application, b, + [{description, "B CXC 138 12"}, + {vsn, "1.0"}, + {modules, [{b_server, 1},{b_lib, 1}]}, + {registered, [b_server]}, + {applications, [kernel, stdlib]}]}. diff --git a/lib/sasl/test/release_handler_SUITE_data/lib/b-1.0/src/b_lib.erl b/lib/sasl/test/release_handler_SUITE_data/lib/b-1.0/src/b_lib.erl new file mode 100644 index 0000000000..7e8a308a5e --- /dev/null +++ b/lib/sasl/test/release_handler_SUITE_data/lib/b-1.0/src/b_lib.erl @@ -0,0 +1,3 @@ +-module(b_lib). +-compile(export_all). +foo() -> ok. diff --git a/lib/sasl/test/release_handler_SUITE_data/lib/b-1.0/src/b_server.erl b/lib/sasl/test/release_handler_SUITE_data/lib/b-1.0/src/b_server.erl new file mode 100644 index 0000000000..e1a80a076f --- /dev/null +++ b/lib/sasl/test/release_handler_SUITE_data/lib/b-1.0/src/b_server.erl @@ -0,0 +1,37 @@ +-module(b_server). + +-behaviour(gen_server). + +%% API +-export([start_link/0]). + +%% gen_server callbacks +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, + terminate/2, code_change/3]). + +-define(SERVER, ?MODULE). + +-record(state, {}). + +start_link() -> + gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). + +init([]) -> + {ok, #state{}}. + +handle_call(_Request, _From, State) -> + Reply = ok, + {reply, Reply, State}. + +handle_cast(_Msg, State) -> + {noreply, State}. + +handle_info(_Info, State) -> + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +code_change(OldVsn, State, Extra) -> + file:write_file("/tmp/b_server",io_lib:format("~p~n",[{"1.0",OldVsn,Extra}])), + {ok, State}. diff --git a/lib/sasl/test/release_handler_SUITE_data/lib/b-2.0/ebin/b.app b/lib/sasl/test/release_handler_SUITE_data/lib/b-2.0/ebin/b.app new file mode 100644 index 0000000000..73c8e42b32 --- /dev/null +++ b/lib/sasl/test/release_handler_SUITE_data/lib/b-2.0/ebin/b.app @@ -0,0 +1,7 @@ +%% -*- erlang -*- +{application, b, + [{description, "B CXC 138 12"}, + {vsn, "2.0"}, + {modules, [{b_server, 1}]}, + {registered, [b_server]}, + {applications, [kernel, stdlib]}]}. diff --git a/lib/sasl/test/release_handler_SUITE_data/lib/b-2.0/ebin/b.appup b/lib/sasl/test/release_handler_SUITE_data/lib/b-2.0/ebin/b.appup new file mode 100644 index 0000000000..d261a37732 --- /dev/null +++ b/lib/sasl/test/release_handler_SUITE_data/lib/b-2.0/ebin/b.appup @@ -0,0 +1,3 @@ +{"2.0", + [{"1.0",[{delete_module,b_lib}, {update,b_server,{advanced,[]}}]}], + [{"1.0",[{add_module,b_lib},{update,b_server,{advanced,[]}}]}]}. diff --git a/lib/sasl/test/release_handler_SUITE_data/lib/b-2.0/src/b_lib.erl b/lib/sasl/test/release_handler_SUITE_data/lib/b-2.0/src/b_lib.erl new file mode 100644 index 0000000000..7e8a308a5e --- /dev/null +++ b/lib/sasl/test/release_handler_SUITE_data/lib/b-2.0/src/b_lib.erl @@ -0,0 +1,3 @@ +-module(b_lib). +-compile(export_all). +foo() -> ok. diff --git a/lib/sasl/test/release_handler_SUITE_data/lib/b-2.0/src/b_server.erl b/lib/sasl/test/release_handler_SUITE_data/lib/b-2.0/src/b_server.erl new file mode 100644 index 0000000000..f8bfbdaff7 --- /dev/null +++ b/lib/sasl/test/release_handler_SUITE_data/lib/b-2.0/src/b_server.erl @@ -0,0 +1,37 @@ +-module(b_server). + +-behaviour(gen_server). + +%% API +-export([start_link/0]). + +%% gen_server callbacks +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, + terminate/2, code_change/3]). + +-define(SERVER, ?MODULE). + +-record(state, {}). + +start_link() -> + gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). + +init([]) -> + {ok, #state{}}. + +handle_call(_Request, _From, State) -> + Reply = ok, + {reply, Reply, State}. + +handle_cast(_Msg, State) -> + {noreply, State}. + +handle_info(_Info, State) -> + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +code_change(OldVsn, State, Extra) -> + file:write_file("/tmp/b_server",io_lib:format("~p~n",[{"2.0",OldVsn,Extra}])), + {ok, State}. diff --git a/lib/snmp/Makefile b/lib/snmp/Makefile index 20e3d4692a..c55eff04c6 100644 --- a/lib/snmp/Makefile +++ b/lib/snmp/Makefile @@ -67,7 +67,7 @@ do_configure: configure configure: configure.in autoconf -.PHONY: info +.PHONY: info gclean info: @echo "OS: $(OS)" @@ -76,6 +76,9 @@ info: @echo "SNMP_VSN: $(SNMP_VSN)" @echo "APP_VSN: $(APP_VSN)" +gclean: + git clean -fXd + # ---------------------------------------------------- # Application (source) release targets diff --git a/lib/snmp/doc/src/Makefile b/lib/snmp/doc/src/Makefile index 35ed63e103..aa9431477c 100644 --- a/lib/snmp/doc/src/Makefile +++ b/lib/snmp/doc/src/Makefile @@ -152,6 +152,7 @@ $(TOP_PDF_FILE): $(XML_FILES) pdf: $(TOP_PDF_FILE) html: gifs $(HTML_REF_MAN_FILE) +html2: html $(INDEX_TARGET) clean clean_docs: clean_html clean_man clean_pdf rm -f errs core *~ @@ -228,6 +229,7 @@ clean_man: clean_html: @echo "cleaning html:" rm -rf $(HTMLDIR)/* + rm -f $(INDEX_TARGET) $(MAN7DIR)/%.7: $(MIBSDIR)/%.mib @echo "processing $*" @@ -286,7 +288,7 @@ release_docs_spec: docs $(INSTALL_DATA) $(INFO_FILE) $(RELSYSDIR) $(INSTALL_DIR) $(RELEASE_PATH)/man/man1 $(INSTALL_DATA) $(MAN1_FILES) $(RELEASE_PATH)/man/man1 - $(INSTALL_DIR) $(RELEASE_PATH)/man/man + $(INSTALL_DIR) $(RELEASE_PATH)/man/man3 $(INSTALL_DATA) $(MAN3_FILES) $(RELEASE_PATH)/man/man3 $(INSTALL_DIR) $(RELEASE_PATH)/man/man6 $(INSTALL_DATA) $(MAN6_FILES) $(RELEASE_PATH)/man/man6 diff --git a/lib/snmp/doc/src/notes.xml b/lib/snmp/doc/src/notes.xml index 6a20d8ee3a..4178192120 100644 --- a/lib/snmp/doc/src/notes.xml +++ b/lib/snmp/doc/src/notes.xml @@ -33,6 +33,136 @@ </header> <section> + <title>SNMP Development Toolkit 4.21</title> + <p>Version 4.21 supports code replacement in runtime from/to + version 4.20.1, 4.20 and 4.19. </p> + + <section> + <title>Improvements and new features</title> +<!-- + <p>-</p> +--> + <list type="bulleted"> + <item> + <p>[manager] There was no way to specify transport domain. + The transport domains was assumed to be IPv4 (transportDomainUdpIpv4). + This has now been changed so that it can also be IPv6 + (transportDomainUdpIpv6). + To facilitate this, the transport domain, <c>tdomain</c>, + is now a (new) valid option when + <seealso marker="snmpm#register_agent">registering</seealso> + a new agent (and + <seealso marker="snmpm#update_agent_info">updating</seealso> + agent info). </p> + <p>This also mean that the transport behaviour has changed. </p> + <p>Own Id: OTP-9305</p> + <p>Aux Id: Seq 11847</p> + </item> + + <item> + <p>[compiler] Added the option + <seealso marker="snmpc#compile">warnings_as_errors</seealso> + (for the SNMP MIB compiler (escript) frontend, the option + <seealso marker="snmpc(command)#option_wae">--wae</seealso> is used) + which specifies whether warnings should be treated as errors. </p> + <p>Tuncer Ayaz</p> + <p>Own Id: OTP-9437</p> + </item> + </list> + + </section> + + <section> + <title>Fixed Bugs and Malfunctions</title> +<!-- + <p>-</p> +--> + + <list type="bulleted"> + <item> + <p>The snmp config tool could not handle (manager) audit trail config + because the option seqno was not handled. </p> + <p>Own Id: OTP-9354</p> + </item> + + <item> + <p>[agent] The SNMP ACM cache was not properly updated when + changes where made to the VACM security-to-group, access and + view-tree-family tables. </p> + <p>Own Id: OTP-9367</p> + <p>Aux Id: Seq 11858</p> + </item> + + <item> + <p>Fixed install directory typo for man3. </p> + <p>Peter Lemenkov</p> + <p>Hans Ulrich Niedermann</p> + <p>Own Id: OTP-9442</p> + </item> + + </list> + </section> + + + <section> + <title>Incompatibilities</title> + <p>-</p> + </section> + + </section> <!-- 4.21 --> + + + <section> + <title>SNMP Development Toolkit 4.20.1</title> + <p>Version 4.20.1 supports code replacement in runtime from/to + version 4.20, 4.19 and 4.18.</p> + + <section> + <title>Improvements and new features</title> + <p>-</p> +<!-- + <list type="bulleted"> + <item> + <p>Added type specs for functions that do not return. </p> + <p>Kostis Sagonas</p> + <p>Own Id: OTP-9208</p> + </item> + </list> +--> + </section> + + <section> + <title>Fixed Bugs and Malfunctions</title> +<!-- + <p>-</p> +--> + <list type="bulleted"> + <item> + <p>[agent] Did not handle transport domains properly in some cases, + for instance trap sending. </p> + <p>Own Id: OTP-9400</p> + </item> + + <item> + <p>[agent] Wrong default transport domain, snmpUDPDomain, instead + of transportDomainUdpIpv4. </p> + <p>Own Id: OTP-9425</p> + <p>Aux Id: Seq 11874</p> + </item> + + </list> + </section> + + + <section> + <title>Incompatibilities</title> + <p>-</p> + </section> + + </section> <!-- 4.20.1 --> + + + <section> <title>SNMP Development Toolkit 4.20</title> <p>Version 4.20 supports code replacement in runtime from/to version 4.19 and 4.18.</p> diff --git a/lib/snmp/doc/src/snmpc.xml b/lib/snmp/doc/src/snmpc.xml index 771629492d..61d19251c5 100644 --- a/lib/snmp/doc/src/snmpc.xml +++ b/lib/snmp/doc/src/snmpc.xml @@ -48,7 +48,11 @@ <type> <v>File = string()</v> <v>Options = [opt()]</v> - <v>opt() = db() | relaxed_row_name_assign_check() | deprecated() | description() | reference() | group_check() | i() | il() | imports() | module() | module_identity() | module_compliance() | agent_capabilities() | outdir() | no_defs() | verbosity() | warnings()</v> + <v>opt() = db() | relaxed_row_name_assign_check() | deprecated() | + description() | reference() | group_check() | i() | il() | + imports() | module() | module_identity() | module_compliance() | + agent_capabilities() | outdir() | no_defs() | verbosity() | + warnings() | warnings_as_errors()</v> <v>db() = {db, volatile|persistent|mnesia}</v> <v>deprecated() = {deprecated, bool()}</v> <v>relaxed_row_name_assign_check() = relaxed_row_name_assign_check</v> @@ -66,6 +70,7 @@ <v>outdir() = {outdir, dir()}</v> <v>verbosity() = {verbosity, silence|warning|info|log|debug|trace}</v> <v>warnings() = {warnings, bool()}</v> + <v>warnings_as_errors() = warnings_as_errors</v> <v>dir() = string()</v> <v>BinFileName = string()</v> </type> @@ -200,11 +205,17 @@ <item> <p>The option <c>warnings</c> specifies whether warning - messages should be shown. </p> + messages should be shown. </p> <p>Default is <c>true</c>. </p> </item> + <item> + <p>The option <c>warnings_as_errors</c>, if present, specifies + whether warnings should be treated as errors.</p> + </item> + </list> + <p>The MIB compiler understands both SMIv1 and SMIv2 MIBs. It uses the <c>MODULE-IDENTITY</c> statement to determine if the MIB is version 1 or 2. diff --git a/lib/snmp/doc/src/snmpc_cmd.xml b/lib/snmp/doc/src/snmpc_cmd.xml index 9358382a10..72116f8981 100644 --- a/lib/snmp/doc/src/snmpc_cmd.xml +++ b/lib/snmp/doc/src/snmpc_cmd.xml @@ -50,6 +50,8 @@ with definitions of Erlang constants for the objects in the MIB, see <seealso marker="snmpc#mib_to_hrl">mib_to_hrl/1</seealso>. </p> + + <marker id="options"></marker> </desc> </func> </funcs> @@ -58,15 +60,18 @@ <title>Compiler options</title> <p>The following options are supported (note that most of these relate to the compilation of the MIB file):</p> + <marker id="option_help"></marker> <taglist> <tag>--help</tag> <item> <p>Prints help info.</p> + <marker id="option_version"></marker> </item> <tag>--version</tag> <item> <p>Prints application and mib format version.</p> + <marker id="option_verbosity"></marker> </item> <tag>--verbosity <em>verbosity</em></tag> @@ -74,11 +79,20 @@ <p>Print debug info. </p> <p><c>verbosity</c> = <c>trace</c> | <c>debug</c> | <c>log</c> | <c>info</c> | <c>silence</c></p> <p>Defaults to <c>silence</c>.</p> + <marker id="option_warnings"></marker> </item> <tag>--warnings</tag> <item> <p>Print warning messages. </p> + <marker id="option_wae"></marker> + </item> + + <tag>--wae</tag> + <item> + <p>Warnings as errors. + Indicates that warnings shall be treated as errors. </p> + <marker id="option_odir"></marker> </item> <tag>--o <em>directory</em></tag> @@ -86,6 +100,7 @@ <p>The directory where the compiler should place the output files. If not specified, output files will be placed in the current working directory.</p> + <marker id="option_idir"></marker> </item> <tag>--i <em>Directory</em></tag> @@ -94,6 +109,7 @@ By default, the current working directory is always included. </p> <p>This option can be present several times, each time specifying <em>one</em> path. </p> + <marker id="option_ildir"></marker> </item> <tag>--il <em>Directory</em></tag> @@ -106,6 +122,7 @@ the current version may be in the system). The current directory and the "snmp-home"/priv/mibs/ are always listed last in the include path. </p> + <marker id="option_sgc"></marker> </item> <tag>--sgc</tag> @@ -114,42 +131,50 @@ group check of the mib compiler. That is, should the OBJECT-GROUP and the NOTIFICATION-GROUP macro(s) be checked for correctness or not. </p> + <marker id="option_dep"></marker> </item> <tag>--dep</tag> <item> <p>Keep deprecated definition(s). If not specified the compiler will ignore deprecated definitions. </p> + <marker id="option_desc"></marker> </item> <tag>--desc</tag> <item> <p>The DESCRIPTION field will be included. </p> + <marker id="option_ref"></marker> </item> <tag>--ref</tag> <item> <p>The REFERENCE field will be included. </p> + <marker id="option_imp"></marker> </item> <tag>--imp</tag> <item> <p>The IMPORTS field will be included. </p> + <marker id="option_mi"></marker> </item> <tag>--mi</tag> <item> <p>The MODULE-IDENTITY field will be included. </p> + <marker id="option_mc"></marker> </item> <tag>--mc</tag> <item> <p>The MODULE-COMPLIANCE field will be included. </p> + <marker id="option_ac"></marker> </item> <tag>--ac</tag> <item> <p>The AGENT-CAPABILITIES field will be included. </p> + <marker id="option_mod"></marker> </item> <tag>--mod <em>module</em></tag> @@ -157,6 +182,7 @@ <p>The module which implements all the instrumentation functions. </p> <p>The name of all instrumentation functions must be the same as the corresponding managed object it implements. </p> + <marker id="option_nd"></marker> </item> <tag>--nd</tag> @@ -165,6 +191,7 @@ used if a managed object have no instrumentation function. Instead this will be reported as an error, and the compilation aborts. </p> + <marker id="option_rrnac"></marker> </item> <tag>--rrnac</tag> @@ -176,6 +203,7 @@ This means that the error will be converted to a warning. </p> <p>By default it is not included, but if this option is present it will be. </p> + <marker id="see_also"></marker> </item> </taglist> diff --git a/lib/snmp/doc/src/snmpm.xml b/lib/snmp/doc/src/snmpm.xml index b527d171b0..c36a1b2a24 100644 --- a/lib/snmp/doc/src/snmpm.xml +++ b/lib/snmp/doc/src/snmpm.xml @@ -283,27 +283,27 @@ sec_level = noAuthNoPriv | authNoPriv | authPriv <v>TargetName = target_name()</v> <v>Config = [agent_config()]</v> <v>agent_config() = {Item, Val}</v> - <v>Item = engine_id | address | port | community | timeout | max_message_size | version | sec_model | sec_name | sec_level</v> + <v>Item = engine_id | address | port | community | timeout | max_message_size | version | sec_model | sec_name | sec_level | tdomain</v> <v>Val = term()</v> <v>Reason = term()</v> </type> <desc> <p>Explicitly instruct the manager to handle this agent, with - <c>UserId</c> as the responsible user. </p> - <p>Called to instruct the manager that this agent - shall be handled. This function is used when - the user knows in advance which agents the - manager shall handle. - Note that there is an alternate way to do the same thing: - Add the agent to the manager config files (see - <seealso marker="snmp_manager_config_files#agents">agents.conf</seealso>).</p> + <c>UserId</c> as the responsible user. </p> + <p>Called to instruct the manager that this agent shall be handled. + This function is used when the user knows in advance which agents + the manager shall handle. + Note that there is an alternate way to do the same thing: + Add the agent to the manager config files (see + <seealso marker="snmp_manager_config_files#agents">agents.conf</seealso>).</p> <p><c>TargetName</c> is a non-empty string, - uniquely identifying the agent. </p> - <p>The type of <c>Val</c> depends on <c>Item</c>: </p> + uniquely identifying the agent. </p> + <p>The type of <c>Val</c> depends on <c>Item</c>: </p> <code type="none"><![CDATA[ [mandatory] engine_id = string() [mandatory] address = ip_address() [optional] port = integer() +[optional] tdomain = transportDomainUdpIpv4 | transportDomainUdpIpv6 [optional] community = string() [optional] timeout = integer() | snmp_timer() [optional] max_message_size = integer() @@ -312,7 +312,9 @@ sec_level = noAuthNoPriv | authNoPriv | authPriv [optional] sec_name = string() [optional] sec_level = noAuthNoPriv | authNoPriv | authPriv ]]></code> - <p>Note that if no <c>Port</c> is given, the default value is used.</p> + <p>Note that if no <c>tdomain</c> is given, the default value, + <c>transportDomainUdpIpv4</c>, is used.</p> + <p>Note that if no <c>port</c> is given, the default value is used.</p> <marker id="unregister_agent"></marker> </desc> @@ -348,17 +350,25 @@ sec_level = noAuthNoPriv | authNoPriv | authPriv </func> <func> + <name>update_agent_info(UserId, TargetName, Info) -> ok | {error, Reason}</name> <name>update_agent_info(UserId, TargetName, Item, Val) -> ok | {error, Reason}</name> <fsummary>Update agent config</fsummary> <type> <v>UserId = term()</v> <v>TargetName = target_name()</v> - <v>Item = atom()</v> - <v>Val = term()</v> + <v>Info = [{item(), item_value()}]</v> + <v>Item = item()</v> + <v>item() = atom()</v> + <v>Val = item_value()</v> + <v>item_value() = term()</v> <v>Reason = term()</v> </type> <desc> - <p>Update agent config.</p> + <p>Update agent config. The function <c>update_agent_info/3</c> + should be used when several values needs to be updated atomically. </p> + <p>See function + <seealso marker="#register_agent">register_agent</seealso>) + for more info about what kind of items are allowed. </p> <marker id="which_agents"></marker> </desc> diff --git a/lib/snmp/src/agent/snmp_view_based_acm_mib.erl b/lib/snmp/src/agent/snmp_view_based_acm_mib.erl index 28469a7b4e..37f6dd3f26 100644 --- a/lib/snmp/src/agent/snmp_view_based_acm_mib.erl +++ b/lib/snmp/src/agent/snmp_view_based_acm_mib.erl @@ -247,6 +247,7 @@ add_sec2group(SecModel, SecName, GroupName) -> Key = [Key1, length(Key2) | Key2], case table_cre_row(vacmSecurityToGroupTable, Key, Row) of true -> + snmpa_agent:invalidate_ca_cache(), {ok, Key}; false -> {error, create_failed} @@ -260,6 +261,7 @@ add_sec2group(SecModel, SecName, GroupName) -> delete_sec2group(Key) -> case table_del_row(vacmSecurityToGroupTable, Key) of true -> + snmpa_agent:invalidate_ca_cache(), ok; false -> {error, delete_failed} @@ -279,6 +281,7 @@ add_access(GroupName, Prefix, SecModel, SecLevel, Match, RV, WV, NV) -> Key3 = [SM, SL], Key = Key1 ++ Key2 ++ Key3, snmpa_vacm:insert([{Key, Row}], false), + snmpa_agent:invalidate_ca_cache(), {ok, Key}; {error, Reason} -> {error, Reason}; @@ -287,6 +290,7 @@ add_access(GroupName, Prefix, SecModel, SecLevel, Match, RV, WV, NV) -> end. delete_access(Key) -> + snmpa_agent:invalidate_ca_cache(), snmpa_vacm:delete(Key). @@ -299,6 +303,7 @@ add_view_tree_fam(ViewIndex, SubTree, Status, Mask) -> Key = [length(Key1) | Key1] ++ [length(Key2) | Key2], case table_cre_row(vacmViewTreeFamilyTable, Key, Row) of true -> + snmpa_agent:invalidate_ca_cache(), {ok, Key}; false -> {error, create_failed} @@ -312,6 +317,7 @@ add_view_tree_fam(ViewIndex, SubTree, Status, Mask) -> delete_view_tree_fam(Key) -> case table_del_row(vacmViewTreeFamilyTable, Key) of true -> + snmpa_agent:invalidate_ca_cache(), ok; false -> {error, delete_failed} diff --git a/lib/snmp/src/agent/snmpa_agent.erl b/lib/snmp/src/agent/snmpa_agent.erl index 82a7ec647b..6322f0f21d 100644 --- a/lib/snmp/src/agent/snmpa_agent.erl +++ b/lib/snmp/src/agent/snmpa_agent.erl @@ -1626,7 +1626,7 @@ invalidate_ca_cache() -> MasterAgent ! invalidate_ca_cache; false -> %% This is running on a sub-agent node, - %% so sent skip it + %% so skip it ok end; _ -> % Not on this node diff --git a/lib/snmp/src/agent/snmpa_conf.erl b/lib/snmp/src/agent/snmpa_conf.erl index 4b88eb69f7..c17a6abbd7 100644 --- a/lib/snmp/src/agent/snmpa_conf.erl +++ b/lib/snmp/src/agent/snmpa_conf.erl @@ -424,7 +424,8 @@ target_addr_entry(Name, EngineId, TMask) -> target_addr_entry(Name, Ip, 162, TagList, - ParamsName, EngineId, TMask, 2048). + ParamsName, EngineId, + TMask, 2048). target_addr_entry(Name, Ip, @@ -435,7 +436,8 @@ target_addr_entry(Name, TMask, MaxMessageSize) -> target_addr_entry(Name, Ip, Udp, 1500, 3, TagList, - ParamsName, EngineId, TMask, MaxMessageSize). + ParamsName, EngineId, + TMask, MaxMessageSize). target_addr_entry(Name, Ip, @@ -448,7 +450,8 @@ target_addr_entry(Name, TMask, MaxMessageSize) -> target_addr_entry(Name, snmp_target_mib:default_domain(), Ip, Udp, - Timeout, RetryCount, TagList, ParamsName, + Timeout, RetryCount, TagList, + ParamsName, EngineId, TMask, MaxMessageSize). target_addr_entry(Name, diff --git a/lib/snmp/src/agent/snmpa_mpd.erl b/lib/snmp/src/agent/snmpa_mpd.erl index 14f62b12f3..4f50b1a674 100644 --- a/lib/snmp/src/agent/snmpa_mpd.erl +++ b/lib/snmp/src/agent/snmpa_mpd.erl @@ -32,6 +32,7 @@ -include("SNMP-MPD-MIB.hrl"). -include("SNMPv2-TM.hrl"). -include("SNMP-FRAMEWORK-MIB.hrl"). +-include("TRANSPORT-ADDRESS-MIB.hrl"). -define(VMODULE,"MPD"). -include("snmp_verbosity.hrl"). @@ -981,12 +982,15 @@ generate_discovery_msg2(NoteStore, Pdu, discovery_note_timeout(Timeout) -> (Timeout div 100) + 1. -generate_discovery_msg(NoteStore, {?snmpUDPDomain, [A,B,C,D,U1,U2]}, +generate_discovery_msg(NoteStore, {TDomain, TAddress}, Pdu, ScopedPduBytes, ContextEngineID, ManagerEngineID, SecModel, SecName, SecLevelFlag, InitialUserName, ContextName, Timeout) -> + + {ok, {_Domain, Address}} = transform_taddr(TDomain, TAddress), + %% 7.1.7 ?vdebug("generate_discovery_msg -> 7.1.7 (~w)", [ManagerEngineID]), MsgID = generate_msg_id(), @@ -1027,7 +1031,7 @@ generate_discovery_msg(NoteStore, {?snmpUDPDomain, [A,B,C,D,U1,U2]}, %% Log(Packet), inc_snmp_out_vars(Pdu), ?vdebug("generate_discovery_msg -> done", []), - {Packet, {{A,B,C,D}, U1 bsl 8 + U2}}; + {Packet, Address}; Error -> throw(Error) @@ -1057,6 +1061,34 @@ generate_sec_discovery_msg(Message, SecModule, end. +transform_taddr(?snmpUDPDomain, TAddress) -> + transform_taddr(?transportDomainUdpIpv4, TAddress); +transform_taddr(?transportDomainUdpIpv4, [A, B, C, D, P1, P2]) -> + Domain = transportDomainUdpIpv4, + Addr = {A,B,C,D}, + Port = P1 bsl 8 + P2, + Address = {Addr, Port}, + {ok, {Domain, Address}}; +transform_taddr(?transportDomainUdpIpv4, BadAddr) -> + {error, {bad_transportDomainUdpIpv4_address, BadAddr}}; +transform_taddr(?transportDomainUdpIpv6, + [A1, A2, A3, A4, A5, A6, A7, A8, P1, P2]) -> + Domain = transportDomainUdpIpv6, + Addr = {A1, A2, A3, A4, A5, A6, A7, A8}, + Port = P1 bsl 8 + P2, + Address = {Addr, Port}, + {ok, {Domain, Address}}; +transform_taddr(?transportDomainUdpIpv6, BadAddr) -> + {error, {bad_transportDomainUdpIpv6_address, BadAddr}}; +transform_taddr(BadTDomain, TAddress) -> + case lists:member(BadTDomain, snmp_conf:all_tdomains()) of + true -> + {error, {unsupported_tdomain, BadTDomain, TAddress}}; + false -> + {error, {unknown_tdomain, BadTDomain, TAddress}} + end. + + process_taddrs(Dests) -> ?vtrace("process_taddrs -> entry with" "~n Dests: ~p", [Dests]), @@ -1066,46 +1098,44 @@ process_taddrs([], Acc) -> ?vtrace("process_taddrs -> entry when done with" "~n Acc: ~p", [Acc]), lists:reverse(Acc); - + %% v3 -process_taddrs([{{?snmpUDPDomain, [A,B,C,D,U1,U2]}, SecData} | T], Acc) -> +process_taddrs([{{TDomain, TAddress}, SecData} | T], Acc) -> ?vtrace("process_taddrs -> entry when v3 with" - "~n A: ~p" - "~n B: ~p" - "~n C: ~p" - "~n D: ~p" - "~n U1: ~p" - "~n U2: ~p" - "~n SecData: ~p", [A, B, C, D, U1, U2, SecData]), - Entry = {{snmpUDPDomain, {{A,B,C,D}, U1 bsl 8 + U2}}, SecData}, - process_taddrs(T, [Entry | Acc]); -%% Bad v3 -process_taddrs([{{TDomain, TAddr}, _SecData} | T], Acc) -> - ?vtrace("process_taddrs -> entry when bad v3 with" - "~n TDomain: ~p" - "~n TAddr: ~p", [TDomain, TAddr]), - user_err("Bad TDomain/TAddr: ~w/~w", [TDomain, TAddr]), - process_taddrs(T, Acc); + "~n TDomain: ~p" + "~n TAddress: ~p" + "~n SecData: ~p", [TDomain, TAddress, SecData]), + case transform_taddr(TDomain, TAddress) of + {ok, DestAddr} -> + ?vtrace("process_taddrs -> transformed: " + "~n DestAddr: ~p", [DestAddr]), + Entry = {DestAddr, SecData}, + process_taddrs(T, [Entry | Acc]); + {error, Reason} -> + ?vinfo("Failed transforming v3 domain and address" + "~n Reason: ~p", [Reason]), + user_err("Bad TDomain/TAddress: ~w/~w", [TDomain, TAddress]), + process_taddrs(T, Acc) + end; %% v1 & v2 -process_taddrs([{?snmpUDPDomain, [A,B,C,D,U1,U2]} | T], Acc) -> +process_taddrs([{TDomain, TAddress} | T], Acc) -> ?vtrace("process_taddrs -> entry when v1/v2 with" - "~n A: ~p" - "~n B: ~p" - "~n C: ~p" - "~n D: ~p" - "~n U1: ~p" - "~n U2: ~p", [A, B, C, D, U1, U2]), - Entry = {snmpUDPDomain, {{A,B,C,D}, U1 bsl 8 + U2}}, - process_taddrs(T, [Entry | Acc]); -%% Bad v1 or v2 -process_taddrs([{TDomain, TAddr} | T], Acc) -> - ?vtrace("process_taddrs -> entry when bad v1/v2 with" - "~n TDomain: ~p" - "~n TAddr: ~p", [TDomain, TAddr]), - user_err("Bad TDomain/TAddr: ~w/~w", [TDomain, TAddr]), - process_taddrs(T, Acc); + "~n TDomain: ~p" + "~n TAddress: ~p", [TDomain, TAddress]), + case transform_taddr(TDomain, TAddress) of + {ok, DestAddr} -> + ?vtrace("process_taddrs -> transformed: " + "~n DestAddr: ~p", [DestAddr]), + Entry = DestAddr, + process_taddrs(T, [Entry | Acc]); + {error, Reason} -> + ?vinfo("Failed transforming v1/v2 domain and address: " + "~n Reason: ~p", [Reason]), + user_err("Bad TDomain/TAddress: ~w/~w", [TDomain, TAddress]), + process_taddrs(T, Acc) + end; process_taddrs(Crap, Acc) -> - throw({error, {taddrs_crap, Crap, Acc}}). + throw({error, {bad_taddrs, Crap, Acc}}). mk_v1_v2_packet_list(To, Packet, Len, Pdu) -> diff --git a/lib/snmp/src/app/snmp.appup.src b/lib/snmp/src/app/snmp.appup.src index 5deb40be0f..8e1855b4df 100644 --- a/lib/snmp/src/app/snmp.appup.src +++ b/lib/snmp/src/app/snmp.appup.src @@ -22,82 +22,82 @@ %% ----- U p g r a d e ------------------------------------------------------- [ + {"4.20.1", + [ + {load_module, snmp_view_based_acm_mib, soft_purge, soft_purge, []}, + {load_module, snmpm, soft_purge, soft_purge, + [snmpm_server, snmpm_config, snmp_config]}, + {load_module, snmp_conf, soft_purge, soft_purge, []}, + {load_module, snmp_config, soft_purge, soft_purge, []}, + {load_module, snmpm_mpd, soft_purge, soft_purge, + [snmp_conf, snmp_config, snmpm_config]}, + {load_module, snmpa_mpd, soft_purge, soft_purge, + [snmp_conf, snmp_config]}, + {load_module, snmpa_conf, soft_purge, soft_purge, [snmp_config]}, + {update, snmpa_agent, soft, soft_purge, soft_purge, [snmpa_mpd]}, + {update, snmpm_config, soft, soft_purge, soft_purge, [snmp_conf]}, + {update, snmpm_server, soft, soft_purge, soft_purge, + [snmpm_net_if, snmpm_mpd, snmpm_config]}, + {update, snmpm_net_if, soft, soft_purge, soft_purge, + [snmp_conf, snmpm_mpd, snmpm_config]} + ] + }, + {"4.20", + [ + {load_module, snmp_view_based_acm_mib, soft_purge, soft_purge, []}, + {load_module, snmp_target_mib, soft_purge, soft_purge, [snmp_conf]}, + {load_module, snmpm, soft_purge, soft_purge, + [snmpm_server, snmpm_config, snmp_config]}, + {load_module, snmp_conf, soft_purge, soft_purge, []}, + {load_module, snmp_config, soft_purge, soft_purge, []}, + {load_module, snmpm_mpd, soft_purge, soft_purge, + [snmp_conf, snmp_config, snmpm_config]}, + {load_module, snmpa_mpd, soft_purge, soft_purge, + [snmp_conf, snmp_config]}, + {load_module, snmpa_conf, soft_purge, soft_purge, [snmp_config]}, + {update, snmpa_agent, soft, soft_purge, soft_purge, [snmpa_mpd]}, + {update, snmpm_config, soft, soft_purge, soft_purge, [snmp_conf]}, + {update, snmpm_server, soft, soft_purge, soft_purge, + [snmpm_net_if, snmpm_mpd, snmpm_config]}, + {update, snmpm_net_if, soft, soft_purge, soft_purge, + [snmp_conf, snmpm_mpd, snmpm_config]} + ] + }, {"4.19", [ {load_module, snmpa, soft_purge, soft_purge, []}, - {load_module, snmpm, soft_purge, soft_purge, [snmpm_server]}, + {load_module, snmpm, soft_purge, soft_purge, + [snmpm_server, snmpm_config, snmp_config]}, {load_module, snmpa_usm, soft_purge, soft_purge, []}, {load_module, snmpm_usm, soft_purge, soft_purge, []}, {load_module, snmp_log, soft_purge, soft_purge, []}, {load_module, snmp_pdus, soft_purge, soft_purge, []}, {load_module, snmp_conf, soft_purge, soft_purge, []}, - {load_module, snmpa_conf, soft_purge, soft_purge, [snmp_conf]}, + {load_module, snmpa_conf, soft_purge, soft_purge, + [snmp_conf, snmp_config]}, {load_module, snmp_misc, soft_purge, soft_purge, []}, {load_module, snmp_config, soft_purge, soft_purge, []}, - {load_module, snmpa_mpd, soft_purge, soft_purge, [snmp_conf]}, + {load_module, snmpa_mpd, soft_purge, soft_purge, + [snmp_conf, snmp_config]}, + {load_module, snmpm_mpd, soft_purge, soft_purge, + [snmp_conf, snmp_config, snmpm_config]}, {load_module, snmpa_trap, soft_purge, soft_purge, [snmpa_mpd, snmp_notification_mib, snmp_target_mib, snmpa_net_if]}, {load_module, snmpa_acm, soft_purge, soft_purge, [snmp_conf, snmpa_mpd, snmp_target_mib]}, {load_module, snmpa_conf, soft_purge, soft_purge, - [snmp_notification_mib]}, + [snmp_config, snmp_notification_mib]}, + {load_module, snmp_view_based_acm_mib, soft_purge, soft_purge, []}, {load_module, snmp_notification_mib, soft_purge, soft_purge, [snmp_conf, snmp_target_mib]}, {load_module, snmp_community_mib, soft_purge, soft_purge, []}, {load_module, snmp_target_mib, soft_purge, soft_purge, [snmp_conf]}, - {update, snmpm_net_if, soft, soft_purge, soft_purge, []}, - {update, snmpm_server, soft, soft_purge, soft_purge, [snmpm_net_if]}, - {update, snmpa_net_if, soft, soft_purge, soft_purge, - [snmp_conf, snmpa_mpd]}, - {update, snmpa_agent, soft, soft_purge, soft_purge, - [snmpa_acm, snmpa_mpd, snmpa_trap]} - ] - }, - {"4.18", - [ - {load_module, snmpa, soft_purge, soft_purge, []}, - {load_module, snmpm, soft_purge, soft_purge, [snmpm_server]}, - {load_module, snmpa_usm, soft_purge, soft_purge, []}, - {load_module, snmpm_usm, soft_purge, soft_purge, []}, - {load_module, snmp_misc, soft_purge, soft_purge, []}, - {load_module, snmp_log, soft_purge, soft_purge, []}, - {load_module, snmp_pdus, soft_purge, soft_purge, []}, - {load_module, snmp_conf, soft_purge, soft_purge, []}, - {load_module, snmp_config, soft_purge, soft_purge, [snmp_conf]}, - {load_module, snmpa_conf, soft_purge, soft_purge, []}, - {load_module, snmpa_mpd, soft_purge, soft_purge, [snmp_conf]}, - {load_module, snmpa_vacm, soft_purge, soft_purge, []}, - {load_module, snmpa_trap, soft_purge, soft_purge, - [snmpa_mpd, snmp_notification_mib, snmp_target_mib, snmpa_net_if]}, - {load_module, snmpa_acm, soft_purge, soft_purge, - [snmp_conf, snmpa_mpd, snmp_target_mib]}, - {load_module, snmpa, soft_purge, soft_purge, - [snmp_community_mib, - snmp_framework_mib, - snmp_standard_mib, - snmp_target_mib, - snmp_user_based_sm_mib, - snmp_view_based_acm_mib]}, - {load_module, snmp_notification_mib, soft_purge, soft_purge, - [snmp_conf, snmp_target_mib, snmpa_mib_lib]}, - {load_module, snmp_community_mib, soft_purge, soft_purge, - [snmpa_mib_lib]}, - {load_module, snmp_framework_mib, soft_purge, soft_purge, - [snmpa_mib_lib]}, - {load_module, snmp_standard_mib, soft_purge, soft_purge, - [snmpa_mib_lib]}, - {load_module, snmp_target_mib, soft_purge, soft_purge, - [snmpa_mib_lib]}, - {load_module, snmp_user_based_sm_mib, soft_purge, soft_purge, - [snmpa_mib_lib]}, - {load_module, snmp_view_based_acm_mib, soft_purge, soft_purge, - [snmpa_mib_lib, snmpa_vacm]}, - {load_module, snmpa_mib_lib, soft_purge, soft_purge, []}, - - {update, snmpm_net_if, soft, soft_purge, soft_purge, []}, - {update, snmpm_server, soft, soft_purge, soft_purge, [snmpm_net_if]}, - + {update, snmpm_net_if, soft, soft_purge, soft_purge, + [snmp_conf, snmpm_mpd, snmpm_config]}, + {update, snmpm_config, soft, soft_purge, soft_purge, [snmp_conf]}, + {update, snmpm_server, soft, soft_purge, soft_purge, + [snmpm_net_if, snmpm_mpd, snmpm_config]}, {update, snmpa_net_if, soft, soft_purge, soft_purge, [snmp_conf, snmpa_mpd]}, {update, snmpa_agent, soft, soft_purge, soft_purge, @@ -109,84 +109,82 @@ %% ------D o w n g r a d e --------------------------------------------------- [ + {"4.20.1", + [ + {load_module, snmp_view_based_acm_mib, soft_purge, soft_purge, []}, + {load_module, snmpm, soft_purge, soft_purge, + [snmpm_server, snmpm_config, snmp_config]}, + {load_module, snmp_conf, soft_purge, soft_purge, []}, + {load_module, snmp_config, soft_purge, soft_purge, []}, + {load_module, snmpm_mpd, soft_purge, soft_purge, + [snmp_conf, snmp_config, snmpm_config]}, + {load_module, snmpa_mpd, soft_purge, soft_purge, + [snmp_conf, snmp_config]}, + {load_module, snmpa_conf, soft_purge, soft_purge, [snmp_config]}, + {update, snmpa_agent, soft, soft_purge, soft_purge, [snmpa_mpd]}, + {update, snmpm_config, soft, soft_purge, soft_purge, [snmp_conf]}, + {update, snmpm_server, soft, soft_purge, soft_purge, + [snmpm_net_if, snmpm_mpd, snmpm_config]}, + {update, snmpm_net_if, soft, soft_purge, soft_purge, + [snmp_conf, snmpm_mpd, snmpm_config]} + ] + }, + {"4.20", + [ + {load_module, snmp_view_based_acm_mib, soft_purge, soft_purge, []}, + {load_module, snmp_target_mib, soft_purge, soft_purge, [snmp_conf]}, + {load_module, snmpm, soft_purge, soft_purge, + [snmpm_server, snmpm_config, snmp_config]}, + {load_module, snmp_conf, soft_purge, soft_purge, []}, + {load_module, snmp_config, soft_purge, soft_purge, []}, + {load_module, snmpm_mpd, soft_purge, soft_purge, + [snmp_conf, snmp_config, snmpm_config]}, + {load_module, snmpa_mpd, soft_purge, soft_purge, + [snmp_conf, snmp_config]}, + {load_module, snmpa_conf, soft_purge, soft_purge, [snmp_config]}, + {update, snmpa_agent, soft, soft_purge, soft_purge, [snmpa_mpd]}, + {update, snmpm_config, soft, soft_purge, soft_purge, [snmp_conf]}, + {update, snmpm_server, soft, soft_purge, soft_purge, + [snmpm_net_if, snmpm_mpd, snmpm_config]}, + {update, snmpm_net_if, soft, soft_purge, soft_purge, + [snmp_conf, snmpm_mpd, snmpm_config]} + ] + }, {"4.19", [ {load_module, snmpa, soft_purge, soft_purge, []}, - {load_module, snmpm, soft_purge, soft_purge, [snmpm_server]}, + {load_module, snmpm, soft_purge, soft_purge, + [snmpm_server, snmpm_config, snmp_config]}, {load_module, snmpa_usm, soft_purge, soft_purge, []}, {load_module, snmpm_usm, soft_purge, soft_purge, []}, {load_module, snmp_log, soft_purge, soft_purge, []}, {load_module, snmp_pdus, soft_purge, soft_purge, []}, {load_module, snmp_conf, soft_purge, soft_purge, []}, - {load_module, snmpa_conf, soft_purge, soft_purge, [snmp_conf]}, + {load_module, snmpa_conf, soft_purge, soft_purge, + [snmp_conf, snmp_config]}, {load_module, snmp_misc, soft_purge, soft_purge, []}, {load_module, snmp_config, soft_purge, soft_purge, []}, - {load_module, snmpa_mpd, soft_purge, soft_purge, [snmp_conf]}, + {load_module, snmpa_mpd, soft_purge, soft_purge, + [snmp_conf, snmp_config]}, + {load_module, snmpm_mpd, soft_purge, soft_purge, + [snmp_conf, snmp_config, snmpm_config]}, {load_module, snmpa_trap, soft_purge, soft_purge, [snmpa_mpd, snmp_notification_mib, snmp_target_mib, snmpa_net_if]}, {load_module, snmpa_acm, soft_purge, soft_purge, [snmp_conf, snmpa_mpd, snmp_target_mib]}, {load_module, snmpa_conf, soft_purge, soft_purge, - [snmp_notification_mib]}, + [snmp_config, snmp_notification_mib]}, + {load_module, snmp_view_based_acm_mib, soft_purge, soft_purge, []}, {load_module, snmp_notification_mib, soft_purge, soft_purge, [snmp_conf, snmp_target_mib]}, {load_module, snmp_community_mib, soft_purge, soft_purge, []}, {load_module, snmp_target_mib, soft_purge, soft_purge, [snmp_conf]}, - - {update, snmpm_net_if, soft, soft_purge, soft_purge, []}, - {update, snmpm_server, soft, soft_purge, soft_purge, [snmpm_net_if]}, - - {update, snmpa_net_if, soft, soft_purge, soft_purge, - [snmp_conf, snmpa_mpd]}, - {update, snmpa_agent, soft, soft_purge, soft_purge, - [snmpa_acm, snmpa_mpd, snmpa_trap]} - ] - }, - {"4.18", - [ - {load_module, snmpa, soft_purge, soft_purge, []}, - {load_module, snmpm, soft_purge, soft_purge, [snmpm_server]}, - {load_module, snmpa_usm, soft_purge, soft_purge, []}, - {load_module, snmpm_usm, soft_purge, soft_purge, []}, - {load_module, snmp_misc, soft_purge, soft_purge, []}, - {load_module, snmp_log, soft_purge, soft_purge, []}, - {load_module, snmp_pdus, soft_purge, soft_purge, []}, - {load_module, snmp_conf, soft_purge, soft_purge, []}, - {load_module, snmpa_conf, soft_purge, soft_purge, [snmp_conf]}, - {load_module, snmp_config, soft_purge, soft_purge, []}, - {load_module, snmpa_mpd, soft_purge, soft_purge, [snmp_conf]}, - {load_module, snmpa_vacm, soft_purge, soft_purge, []}, - {load_module, snmpa_trap, soft_purge, soft_purge, - [snmpa_mpd, snmp_notification_mib, snmp_target_mib, snmpa_net_if]}, - {load_module, snmpa_acm, soft_purge, soft_purge, - [snmp_conf, snmpa_mpd, snmp_target_mib]}, - {load_module, snmpa, soft_purge, soft_purge, - [snmp_community_mib, - snmp_framework_mib, - snmp_standard_mib, - snmp_target_mib, - snmp_user_based_sm_mib, - snmp_view_based_acm_mib]}, - {load_module, snmp_notification_mib, soft_purge, soft_purge, - [snmp_conf, snmp_target_mib, snmpa_mib_lib]}, - {load_module, snmp_community_mib, soft_purge, soft_purge, - [snmpa_mib_lib]}, - {load_module, snmp_framework_mib, soft_purge, soft_purge, - [snmpa_mib_lib]}, - {load_module, snmp_standard_mib, soft_purge, soft_purge, - [snmpa_mib_lib]}, - {load_module, snmp_target_mib, soft_purge, soft_purge, - [snmpa_mib_lib]}, - {load_module, snmp_user_based_sm_mib, soft_purge, soft_purge, - [snmpa_mib_lib]}, - {load_module, snmp_view_based_acm_mib, soft_purge, soft_purge, - [snmpa_mib_lib, snmpa_vacm]}, - {load_module, snmpa_mib_lib, soft_purge, soft_purge, []}, - - {update, snmpm_net_if, soft, soft_purge, soft_purge, []}, - {update, snmpm_server, soft, soft_purge, soft_purge, [snmpm_net_if]}, - + {update, snmpm_net_if, soft, soft_purge, soft_purge, + [snmp_conf, snmpm_mpd, snmpm_config]}, + {update, snmpm_config, soft, soft_purge, soft_purge, [snmp_conf]}, + {update, snmpm_server, soft, soft_purge, soft_purge, + [snmpm_net_if, snmpm_mpd, snmpm_config]}, {update, snmpa_net_if, soft, soft_purge, soft_purge, [snmp_conf, snmpa_mpd]}, {update, snmpa_agent, soft, soft_purge, soft_purge, diff --git a/lib/snmp/src/compile/Makefile b/lib/snmp/src/compile/Makefile index 0ceaf276a6..627af6f185 100644 --- a/lib/snmp/src/compile/Makefile +++ b/lib/snmp/src/compile/Makefile @@ -45,11 +45,10 @@ RELSYSDIR = $(RELEASE_PATH)/lib/snmp-$(VSN) include modules.mk -ESCRIPT_BIN = $(ESCRIPT_SRC:%.src=$(BIN)/%) - -ERL_FILES = $(MODULES:%=%.erl) - -TARGET_FILES = $(MODULES:%=$(EBIN)/%.$(EMULATOR)) $(ESCRIPT_BIN) +ESCRIPT_BIN = $(ESCRIPT_SRC:%.src=$(BIN)/%) +ERL_FILES = $(MODULES:%=%.erl) +EBIN_FILES = $(MODULES:%=$(EBIN)/%.$(EMULATOR)) +TARGET_FILES = $(EBIN_FILES) $(ESCRIPT_BIN) GENERATED_PARSER = $(PARSER_MODULE:%=%.erl) @@ -125,7 +124,7 @@ release_spec: opt $(INSTALL_DIR) $(RELSYSDIR)/src/compiler $(INSTALL_DATA) $(ESCRIPT_SRC) $(PARSER_SRC) $(ERL_FILES) $(INTERNAL_HRL_FILES) $(RELSYSDIR)/src/compiler $(INSTALL_DIR) $(RELSYSDIR)/ebin - $(INSTALL_DATA) $(TARGET_FILES) $(RELSYSDIR)/ebin + $(INSTALL_DATA) $(EBIN_FILES) $(RELSYSDIR)/ebin $(INSTALL_DIR) $(RELSYSDIR)/bin $(INSTALL_SCRIPT) $(ESCRIPT_BIN) $(RELSYSDIR)/bin diff --git a/lib/snmp/src/compile/snmpc.erl b/lib/snmp/src/compile/snmpc.erl index 195c238184..5e6b81f1ec 100644 --- a/lib/snmp/src/compile/snmpc.erl +++ b/lib/snmp/src/compile/snmpc.erl @@ -108,6 +108,7 @@ compile(FileName) -> %% {i, [import_dir_string()]} ["./"] %% {il, [import_lib_dir_string()]} [] %% {warnings, bool()} true +%% warnings_as_errors %% {outdir, string()} "./" %% description %% reference @@ -199,6 +200,8 @@ get_options([reference|Opts], Formats, Args) -> get_options(Opts, ["~n reference"|Formats], Args); get_options([{warnings, Val}|Opts], Formats, Args) -> get_options(Opts, ["~n warnings: ~w"|Formats], [Val|Args]); +get_options([warnings_as_errors|Opts], Formats, Args) -> + get_options(Opts, ["~n warnings_as_errors"|Formats], Args); get_options([{verbosity, Val}|Opts], Formats, Args) -> get_options(Opts, ["~n verbosity: ~w"|Formats], [Val|Args]); get_options([imports|Opts], Formats, Args) -> @@ -261,6 +264,8 @@ check_options([{group_check, Atom} | T]) when is_atom(Atom) -> check_options([{warnings, Bool} | T]) -> check_bool(warnings, Bool), check_options(T); +check_options([warnings_as_errors | T]) -> + check_options(T); check_options([{db, volatile} | T]) -> check_options(T); check_options([{db, persistent} | T]) -> @@ -331,6 +336,9 @@ get_agent_capabilities(Options) -> get_module_compliance(Options) -> get_bool_option(module_compliance, Options). +get_warnings_as_errors(Options) -> + lists:member(warnings_as_errors, Options). + get_relaxed_row_name_assign_check(Options) -> lists:member(relaxed_row_name_assign_check, Options). @@ -409,6 +417,7 @@ init(From, MibFileName, Options) -> put(reference, get_reference(Options)), put(agent_capabilities, get_agent_capabilities(Options)), put(module_compliance, get_module_compliance(Options)), + put(warnings_as_errors, get_warnings_as_errors(Options)), File = filename:rootname(MibFileName, ".mib"), put(filename, filename:basename(File ++ ".mib")), R = case catch c_impl(File) of diff --git a/lib/snmp/src/compile/snmpc.src b/lib/snmp/src/compile/snmpc.src index 5f9b154bfa..4e91ae9a03 100644 --- a/lib/snmp/src/compile/snmpc.src +++ b/lib/snmp/src/compile/snmpc.src @@ -50,7 +50,8 @@ %% The default verbosity (silence) will be filled in %% during argument processing. verbosity, - warnings = false + warnings = false, + warnings_as_errors = false }). @@ -74,6 +75,7 @@ %% --version %% --verbosity V %% --warnings +%% --wae main(Args) when is_list(Args) -> case (catch process_args(Args)) of ok -> @@ -156,7 +158,8 @@ mk_mib_options(#state{outdir = OutDir, %% The default verbosity (silence) will be filled in %% during argument processing. verbosity = V, - warnings = W}) -> + warnings = W, + warnings_as_errors = WAE}) -> [{outdir, OutDir}, {db, DB}, {i, IDs}, @@ -178,7 +181,8 @@ mk_mib_options(#state{outdir = OutDir, maybe_option(Imp, imports) ++ maybe_option(MI, module_identity) ++ maybe_option(MC, module_compliance) ++ - maybe_option(AC, agent_capabilities). + maybe_option(AC, agent_capabilities) ++ + maybe_option(WE, warnings_as_errors). maybe_option(true, Opt) -> [Opt]; maybe_option(_, _) -> []. @@ -292,6 +296,8 @@ process_args(["--nd"|Args], State) -> process_args(Args, State#state{no_defaults = true}); process_args(["--rrnac"|Args], State) -> process_args(Args, State#state{relaxed_row_name_assigne_check = true}); +process_args(["--wae"|Args], State) -> + process_args(Args, State#state{warnings_as_errors = true}); process_args([MIB], State) -> Ext = filename:extension(MIB), if @@ -371,6 +377,8 @@ usage() -> "~n a warning. " "~n By default it is not included, but if this option is " "~n present it will be. " + "~n --wae - Warnings as errors. " + "~n Indicates that warnings shall be treated as errors. " "~n " "~n", []), halt(1). diff --git a/lib/snmp/src/compile/snmpc_lib.hrl b/lib/snmp/src/compile/snmpc_lib.hrl index 000486e728..35ec9abd03 100644 --- a/lib/snmp/src/compile/snmpc_lib.hrl +++ b/lib/snmp/src/compile/snmpc_lib.hrl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2009. All Rights Reserved. +%% Copyright Ericsson AB 2009-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -20,8 +20,17 @@ -ifndef(snmpc_lib). -define(snmpc_lib, true). --define(vwarning(F, A), ?verbosity(warning, F, A, ignore)). --define(vwarning2(F, A, MibLine), ?verbosity(warning, F, A, MibLine)). +-define(vwarning(F, A), + case get(warnings_as_errors) of + true -> snmpc_lib:error(F, A); + _ -> ?verbosity(warning, F, A, ignore) + end). + +-define(vwarning2(F, A, MibLine), + case get(warnings_as_errors) of + true -> snmpc_lib:error(F, A, MibLine); + _ -> ?verbosity(warning, F, A, MibLine) + end). -define(vinfo(F, A), ?verbosity(info, F, A, ignore)). -define(vinfo2(F, A, MibLine), ?verbosity(info, F, A, MibLine)). -define(vlog(F, A), ?verbosity(log, F, A, ignore)). diff --git a/lib/snmp/src/manager/snmpm.erl b/lib/snmp/src/manager/snmpm.erl index 0d084332de..6d2ac8d747 100644 --- a/lib/snmp/src/manager/snmpm.erl +++ b/lib/snmp/src/manager/snmpm.erl @@ -50,7 +50,7 @@ register_agent/2, register_agent/3, register_agent/4, unregister_agent/2, unregister_agent/3, which_agents/0, which_agents/1, - agent_info/2, update_agent_info/4, + agent_info/2, update_agent_info/3, update_agent_info/4, register_usm_user/3, unregister_usm_user/2, which_usm_users/0, which_usm_users/1, @@ -167,6 +167,7 @@ -include_lib("snmp/include/snmp_types.hrl"). -include("snmpm_atl.hrl"). -include("snmpm_internal.hrl"). +-include("snmp_verbosity.hrl"). -define(DEFAULT_AGENT_PORT, 161). @@ -447,8 +448,11 @@ agent_info(Addr, Port, Item) -> Error end. +update_agent_info(UserId, TargetName, Info) when is_list(Info) -> + snmpm_config:update_agent_info(UserId, TargetName, Info). + update_agent_info(UserId, TargetName, Item, Val) -> - snmpm_config:update_agent_info(UserId, TargetName, Item, Val). + update_agent_info(UserId, TargetName, [{Item, Val}]). %% Backward compatibility functions update_agent_info(UserId, Addr, Port, Item, Val) -> diff --git a/lib/snmp/src/manager/snmpm_config.erl b/lib/snmp/src/manager/snmpm_config.erl index fd6da3e71a..c2e57abddb 100644 --- a/lib/snmp/src/manager/snmpm_config.erl +++ b/lib/snmp/src/manager/snmpm_config.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2004-2010. All Rights Reserved. +%% Copyright Ericsson AB 2004-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -36,7 +36,8 @@ user_info/0, user_info/1, user_info/2, register_agent/3, unregister_agent/2, - agent_info/0, agent_info/2, agent_info/3, update_agent_info/4, + agent_info/0, agent_info/2, agent_info/3, + update_agent_info/3, update_agent_info/4, which_agents/0, which_agents/1, is_known_engine_id/2, @@ -84,7 +85,9 @@ backup/1, - mk_target_name/3 + mk_target_name/3, + + default_transport_domain/0 ]). @@ -127,23 +130,24 @@ %% Macros and Constants: --define(SERVER, ?MODULE). --define(BACKUP_DB, snmpm_config_backup). --define(CONFIG_DB, snmpm_config_db). +-define(SERVER, ?MODULE). +-define(BACKUP_DB, snmpm_config_backup). +-define(CONFIG_DB, snmpm_config_db). -define(DEFAULT_USER, default_user). -define(DEFAULT_AGENT_PORT, 161). --define(IRB_DEFAULT, auto). -%% -define(IRB_DEFAULT, {user, timer:seconds(15)}). +-define(IRB_DEFAULT, auto). +%% -define(IRB_DEFAULT, {user, timer:seconds(15)}). --define(USER_MOD_DEFAULT, snmpm_user_default). --define(USER_DATA_DEFAULT, undefined). +-define(USER_MOD_DEFAULT, snmpm_user_default). +-define(USER_DATA_DEFAULT, undefined). %% -define(DEF_ADDR_TAG, default_addr_tag). -define(DEFAULT_TARGETNAME, default_agent). --define(DEF_PORT_TAG, default_port_tag). +-define(DEF_PORT_TAG, default_port_tag). +-define(SUPPORTED_DOMAINS, [transportDomainUdpIpv4, transportDomainUdpIpv6]). -ifdef(snmp_debug). -define(GS_START_LINK(Opts), @@ -159,6 +163,11 @@ %%%------------------------------------------------------------------- %%% API %%%------------------------------------------------------------------- + +default_transport_domain() -> + transportDomainUdpIpv4. + + start_link(Opts) -> ?d("start_link -> entry with" "~n Opts: ~p", [Opts]), @@ -269,9 +278,10 @@ do_user_info(_UserId, BadItem) -> error({not_found, BadItem}). -%% A target-name constructed in this way is a string with the following +%% A target-name constructed in this way is a string with the following: %% <IP-address>:<Port>-<Version> -%% +%% This is intended for backward compatibility and therefor has +%% only support for IPv4 addresses and *no* other transport domain. mk_target_name(Addr0, Port, Config) when is_list(Config) -> Version = case lists:keysearch(version, 1, Config) of @@ -280,7 +290,6 @@ mk_target_name(Addr0, Port, Config) when is_list(Config) -> false -> select_lowest_supported_version() end, -%% p("mk_target_name -> Version: ~p", [Version]), case normalize_address(Addr0) of {A, B, C, D} -> lists:flatten( @@ -308,57 +317,99 @@ select_lowest_supported_version([H|T], Versions) -> end. -register_agent(UserId, _TargetName, _Config) when (UserId =:= user_id) -> +register_agent(UserId, _TargetName, _Config0) when (UserId =:= user_id) -> {error, {bad_user_id, UserId}}; -register_agent(UserId, TargetName, Config) +register_agent(UserId, TargetName, Config0) when (is_list(TargetName) andalso (length(TargetName) > 0) andalso - is_list(Config)) -> + is_list(Config0)) -> -%% p("register_agent -> entry with" -%% "~n UserId: ~p" -%% "~n TargetName: ~p" -%% "~n Config: ~p", [UserId, TargetName, Config]), + ?vtrace("register_agent -> entry with" + "~n UserId: ~p" + "~n TargetName: ~p" + "~n Config0: ~p", [UserId, TargetName, Config0]), %% Check: %% 1) That the mandatory configs are present - %% 2) That the illegal config user_id (used internally) is - %% not present + %% 2) That no illegal config, e.g. user_id (used internally), + %% is not present %% 3) Check that there are no invalid or erroneous configs - %% 4) Chack that the manager is capable to use the selected version - case verify_agent_config(Config) of - ok -> + %% 4) Check that the manager is capable of using the selected version + case verify_agent_config(Config0) of + {ok, Config} -> call({register_agent, UserId, TargetName, Config}); Error -> Error end. -verify_agent_config(Conf) -> - ok = verify_mandatory(Conf, [engine_id, address, reg_type]), - case verify_invalid(Conf, [user_id]) of - ok -> - case verify_agent_config2(Conf) of - ok -> - {ok, Vsns} = system_info(versions), - Vsn = - case lists:keysearch(version, 1, Conf) of - {value, {version, V}} -> - V; - false -> - v1 - end, - case lists:member(Vsn, Vsns) of - true -> - ok; - false -> - {error, {version_not_supported_by_manager, Vsn, Vsns}} - end - end; - Error -> +verify_agent_config(Conf0) -> + try + begin + verify_mandatory(Conf0, [engine_id, address, reg_type]), + verify_invalid(Conf0, [user_id]), + Conf = verify_agent_config3(Conf0), + Vsns = versions(), + Vsn = which_version(Conf), + verify_version(Vsn, Vsns), + {ok, Conf} + end + catch + throw:Error -> Error end. +versions() -> + case system_info(versions) of + {ok, Vsns} -> + Vsns; + {error, _} = ERROR -> + throw(ERROR) + end. + +which_version(Conf) -> + case lists:keysearch(version, 1, Conf) of + {value, {version, V}} -> + V; + false -> + v1 + end. + +verify_version(Vsn, Vsns) -> + case lists:member(Vsn, Vsns) of + true -> + ok; + false -> + Reason = {version_not_supported_by_manager, Vsn, Vsns}, + throw({error, Reason}) + end. + +verify_agent_config3(Conf0) -> + %% Fix (transport) address and domain + {TDomain, Conf1} = + case lists:keysearch(tdomain, 1, Conf0) of + {value, {tdomain, Dom}} -> + {Dom, Conf0}; + false -> + Dom = default_transport_domain(), + {Dom, [{tdomain, Dom} | Conf0]} + end, + Conf2 = case lists:keysearch(address, 1, Conf1) of + {value, {address, Address}} -> + lists:keyreplace(address, 1, Conf1, + {address, {TDomain, Address}}); + false -> + %% This is a mandatory config option, + %% a later test will detect this + Conf1 + end, + case verify_agent2(Conf2) of + {ok, Conf} -> + Conf; + {error, _} = ERROR -> + throw(ERROR) + end. + verify_agent_config2(Conf) -> verify_agent2(Conf). @@ -366,6 +417,7 @@ verify_agent_config2(Conf) -> unregister_agent(UserId, TargetName) -> call({unregister_agent, UserId, TargetName}). +%% This is the old style agent unregistration (using Addr and Port). unregister_agent(UserId, Addr0, Port) -> Addr = normalize_address(Addr0), case do_agent_info(Addr, Port, target_name) of @@ -421,17 +473,51 @@ which_agents(UserId) -> Agents = ets:match(snmpm_agent_table, Pat), [TargetName || [TargetName] <- Agents]. - -update_agent_info(UserId, TargetName, Item, Val0) - when (Item =/= user_id) -> - case (catch verify_val(Item, Val0)) of - {ok, Val} -> - call({update_agent_info, UserId, TargetName, Item, Val}); - Error -> + +verify_agent_info(TargetName, Info0) -> + try + begin + verify_invalid(Info0, [user_id]), + %% Check if address is part of the list and + %% if so update it with the domain info. + Info = + case lists:keysearch(address, 1, Info0) of + {value, {address, Addr}} -> + %% If domain is part of the info, then use it. + %% If not, lookup what is already stored for + %% this agent and use that. + Domain = + case lists:keysearch(tdomain, 1, Info0) of + {value, {tdomain, Dom}} -> + Dom; + false -> + {ok, Dom} = + agent_info(TargetName, tdomain), + Dom + end, + Addr2 = {Domain, Addr}, + lists:keyreplace(address, 1, Info0, {address, Addr2}); + false -> + Info0 + end, + verify_agent2(Info) + end + catch + throw:Error -> Error end. -%% Backward compatibillity +update_agent_info(UserId, TargetName, Info) -> + call({update_agent_info, UserId, TargetName, Info}). + +%% <BACKWARD-COMPAT-2> +%% This is wrapped in the interface module, so this function is +%% only here to catch code-upgrade problems. +update_agent_info(UserId, TargetName, Item, Val) -> + update_agent_info(UserId, TargetName, [{Item, Val}]). +%% </BACKWARD-COMPAT-2> + +%% <BACKWARD-COMPAT-1> update_agent_info(UserId, Addr, Port, Item, Val) -> case agent_info(Addr, Port, target_name) of {ok, TargetName} -> @@ -439,6 +525,7 @@ update_agent_info(UserId, Addr, Port, Item, Val) -> Error -> Error end. +%% </BACKWARD-COMPAT-1> is_known_engine_id(EngineID, TargetName) -> case agent_info(TargetName, engine_id) of @@ -650,22 +737,14 @@ unregister_usm_user(EngineID, Name) call({unregister_usm_user, EngineID, Name}). verify_usm_user_config(EngineID, Name, Config) -> - %% case verify_mandatory(Config, []) of - %% ok -> - %% case verify_invalid(Config, [engine_id, name]) of - %% ok -> - %% verify_usm_user_config2(EngineID, Name, Config); - %% Error -> - %% Error - %% end; - %% Error -> - %% Error - %% end. - ok = verify_mandatory(Config, []), - case verify_invalid(Config, [engine_id, name]) of - ok -> - verify_usm_user_config2(EngineID, Name, Config); - Error -> + try + begin + verify_mandatory(Config, []), + verify_invalid(Config, [engine_id, name]), + verify_usm_user_config2(EngineID, Name, Config) + end + catch + throw:Error -> Error end. @@ -1590,6 +1669,7 @@ check_agent_config2(Agent) -> throw(Err) end. +%% For backward compatibility check_agent_config({UserId, TargetName, Community, @@ -1597,10 +1677,27 @@ check_agent_config({UserId, EngineId, Timeout, MaxMessageSize, Version, SecModel, SecName, SecLevel}) -> + TDomain = default_transport_domain(), + check_agent_config({UserId, + TargetName, + Community, + TDomain, Ip, Port, + EngineId, + Timeout, MaxMessageSize, + Version, SecModel, SecName, SecLevel}); + +check_agent_config({UserId, + TargetName, + Community, + TDomain, Ip, Port, + EngineId, + Timeout, MaxMessageSize, + Version, SecModel, SecName, SecLevel}) -> ?vtrace("check_agent_config -> entry with" "~n UserId: ~p" "~n TargetName: ~p" "~n Community: ~p" + "~n TDomain: ~p" "~n Ip: ~p" "~n Port: ~p" "~n EngineId: ~p" @@ -1610,15 +1707,16 @@ check_agent_config({UserId, "~n SecModel: ~p" "~n SecName: ~p" "~n SecLevel: ~p", - [UserId, TargetName, Community, Ip, Port, + [UserId, TargetName, Community, + TDomain, Ip, Port, EngineId, Timeout, MaxMessageSize, Version, SecModel, SecName, SecLevel]), - Addr = normalize_address(Ip), + Addr = normalize_address(TDomain, Ip), ?vtrace("check_agent_config -> Addr: ~p", [Addr]), Agent = {UserId, TargetName, Community, - Addr, Port, + TDomain, Addr, Port, EngineId, Timeout, MaxMessageSize, Version, SecModel, SecName, SecLevel}, @@ -1644,6 +1742,7 @@ init_agent_config({UserId, TargetName, Config}) -> end. +%% For backward compatibility verify_agent({UserId, TargetName, Comm, @@ -1651,48 +1750,68 @@ verify_agent({UserId, EngineId, Timeout, MMS, Version, SecModel, SecName, SecLevel}) -> - ?vtrace("verify_agent -> entry with" + TDomain = default_transport_domain(), + verify_agent({UserId, + TargetName, + Comm, + TDomain, Ip, Port, + EngineId, + Timeout, MMS, + Version, SecModel, SecName, SecLevel}); + +verify_agent({UserId, + TargetName, + Comm, + TDomain, Ip, Port, + EngineId, + Timeout, MMS, + Version, SecModel, SecName, SecLevel}) -> + ?vdebug("verify_agent -> entry with" "~n UserId: ~p" "~n TargetName: ~p", [UserId, TargetName]), snmp_conf:check_string(TargetName, {gt, 0}), - case verify_val(address, Ip) of - {ok, Addr} -> - snmp_conf:check_integer(Port, {gt, 0}), - Conf = - [{reg_type, target_name}, - {address, Addr}, - {port, Port}, - {community, Comm}, - {engine_id, EngineId}, - {timeout, Timeout}, - {max_message_size, MMS}, - {version, Version}, - {sec_model, SecModel}, - {sec_name, SecName}, - {sec_level, SecLevel} - ], - case verify_agent2(Conf) of - ok -> - {UserId, TargetName, Conf, Version}; - Err -> - throw(Err) - end; - - Error -> - ?vlog("verify_agent -> failed: ~n ~p", [Error]), - throw(Error) + snmp_conf:check_integer(Port, {gt, 0}), + %% Note that the order of Conf *is* important. + %% Some properties may depend on others, so that + %% in order to verify one property, another must + %% be already verified (and present). An example + %% of this is the property 'address', for which + %% the property tdomain is needed. + Conf0 = + [{reg_type, target_name}, + {tdomain, TDomain}, + %% This should be taddress, but what the*... + {address, {TDomain, Ip}}, + {port, Port}, + {community, Comm}, + {engine_id, EngineId}, + {timeout, Timeout}, + {max_message_size, MMS}, + {version, Version}, + {sec_model, SecModel}, + {sec_name, SecName}, + {sec_level, SecLevel} + ], + case verify_agent2(Conf0) of + {ok, Conf} -> + {UserId, TargetName, Conf, Version}; + Err -> + throw(Err) end. -verify_agent2([]) -> - ok; -verify_agent2([{Item, Val}|Items]) -> - case verify_val(Item, Val) of - {ok, _Val} -> - verify_agent2(Items); +verify_agent2(Conf) -> + verify_agent2(Conf, []). + +verify_agent2([], VerifiedConf) -> + {ok, VerifiedConf}; +verify_agent2([{Item, Val0}|Items], VerifiedConf) -> + case verify_val(Item, Val0) of + {ok, Val} -> + verify_agent2(Items, [{Item, Val} | VerifiedConf]); Err -> Err end; -verify_agent2([Bad|_]) -> +verify_agent2([Bad|_], _VerifiedConf) -> {error, {bad_agent_config, Bad}}. @@ -1708,14 +1827,28 @@ read_users_config_file(Dir) -> check_user_config({Id, Mod, Data}) -> + ?vtrace("check_user_config -> entry with" + "~n Id: ~p" + "~n Mod: ~p" + "~n Data: ~p", [Id, Mod, Data]), check_user_config({Id, Mod, Data, []}); -check_user_config({Id, Mod, _Data, DefaultAgentConfig} = User) +check_user_config({Id, Mod, Data, DefaultAgentConfig} = _User) when (Id =/= ?DEFAULT_USER) andalso is_list(DefaultAgentConfig) -> + ?vtrace("check_user_config -> entry with" + "~n Id: ~p" + "~n Mod: ~p" + "~n Data: ~p" + "~n DefaultAgentConfig: ~p", + [Id, Mod, Data, DefaultAgentConfig]), case (catch verify_user_behaviour(Mod)) of ok -> + ?vtrace("check_user_config -> user behaviour verified", []), case verify_user_agent_config(DefaultAgentConfig) of - ok -> - {ok, User}; + {ok, DefAgentConf} -> + ?vtrace("check_user_config -> " + "user agent (default) config verified", []), + User2 = {Id, Mod, Data, DefAgentConf}, + {ok, User2}; {error, Reason} -> error({bad_default_agent_config, Reason}) end; @@ -1756,16 +1889,16 @@ verify_user({Id, UserMod, UserData}) -> verify_user({Id, UserMod, UserData, DefaultAgentConfig}) when (Id =/= ?DEFAULT_USER) andalso is_list(DefaultAgentConfig) -> ?d("verify_user -> entry with" - "~n Id: ~p" - "~n UserMod: ~p" - "~n UserData: ~p" + "~n Id: ~p" + "~n UserMod: ~p" + "~n UserData: ~p" "~n DefaultAgentConfig: ~p", [Id, UserMod, UserData, DefaultAgentConfig]), case (catch verify_user_behaviour(UserMod)) of ok -> case verify_user_agent_config(DefaultAgentConfig) of - ok -> - Config = default_agent_config(DefaultAgentConfig), + {ok, DefAgentConf} -> + Config = default_agent_config(DefAgentConf), {ok, #user{id = Id, mod = UserMod, data = UserData, @@ -1783,10 +1916,15 @@ verify_user({Id, _, _, _}) -> {error, {bad_user_id, Id}}. verify_user_agent_config(Conf) -> - case verify_invalid(Conf, [user_id, engine_id, address]) of - ok -> - verify_agent_config2(Conf); - Error -> + try + begin + verify_invalid(Conf, [user_id, engine_id, address]), + verify_agent_config2(Conf) + end + catch + throw:Error -> + ?vdebug("verify_user_agent_config -> throw" + "~n Error: ~p", [Error]), Error end. @@ -2147,6 +2285,16 @@ handle_call({unregister_agent, UserId, TargetName}, _From, State) -> Reply = handle_unregister_agent(UserId, TargetName), {reply, Reply, State}; +handle_call({update_agent_info, UserId, TargetName, Info}, + _From, State) -> + ?vlog("received update_agent_info request: " + "~n UserId: ~p" + "~n TargetName: ~p" + "~n Info: ~p", [UserId, TargetName, Info]), + Reply = handle_update_agent_info(UserId, TargetName, Info), + {reply, Reply, State}; + +%% <BACKWARD-COMPAT> handle_call({update_agent_info, UserId, TargetName, Item, Val}, _From, State) -> ?vlog("received update_agent_info request: " @@ -2156,6 +2304,7 @@ handle_call({update_agent_info, UserId, TargetName, Item, Val}, "~n Val: ~p", [UserId, TargetName, Item, Val]), Reply = handle_update_agent_info(UserId, TargetName, Item, Val), {reply, Reply, State}; +%% </BACKWARD-COMPAT> handle_call({register_usm_user, User}, _From, State) -> ?vlog("received register_usm_user request: " @@ -2517,16 +2666,27 @@ handle_register_agent(UserId, TargetName, Config) -> "~n Config: ~p", [UserId, TargetName, Config]), case (catch agent_info(TargetName, user_id)) of {error, _} -> + ?vtrace("handle_register_agent -> user_id not found in config", []), case ets:lookup(snmpm_user_table, UserId) of [#user{default_agent_config = DefConfig}] -> + ?vtrace("handle_register_agent -> " + "~n DefConfig: ~p", [DefConfig]), + %% First, insert this users default config + ?vtrace("handle_register_agent -> store default config", []), do_handle_register_agent(TargetName, DefConfig), + %% Second, insert the config for this agent + ?vtrace("handle_register_agent -> store config", []), do_handle_register_agent(TargetName, [{user_id, UserId}|Config]), %% <DIRTY-BACKWARD-COMPATIBILLITY> %% And now for some (backward compatibillity) %% dirty crossref stuff + ?vtrace("handle_register_agent -> lookup address", []), {ok, Addr} = agent_info(TargetName, address), + ?vtrace("handle_register_agent -> Addr: ~p, lookup Port", + [Addr]), {ok, Port} = agent_info(TargetName, port), + ?vtrace("handle_register_agent -> register cross-ref fix", []), ets:insert(snmpm_agent_table, {{Addr, Port, target_name}, TargetName}), %% </DIRTY-BACKWARD-COMPATIBILLITY> @@ -2551,10 +2711,18 @@ handle_register_agent(UserId, TargetName, Config) -> do_handle_register_agent(_TargetName, []) -> ok; do_handle_register_agent(TargetName, [{Item, Val}|Rest]) -> + ?vtrace("handle_register_agent -> entry with" + "~n TargetName: ~p" + "~n Item: ~p" + "~n Val: ~p" + "~n Rest: ~p", [TargetName, Item, Val, Rest]), case (catch do_update_agent_info(TargetName, Item, Val)) of ok -> do_handle_register_agent(TargetName, Rest); {error, Reason} -> + ?vtrace("handle_register_agent -> failed updating ~p" + "~n Item: ~p" + "~n Reason: ~p", [Item, Reason]), ets:match_delete(snmpm_agent_table, {TargetName, '_'}), {error, Reason} end; @@ -2589,41 +2757,61 @@ handle_unregister_agent(UserId, TargetName) -> end. -handle_update_agent_info(UserId, TargetName, Item, Val) -> +handle_update_agent_info(UserId, TargetName, Info) -> ?vdebug("handle_update_agent_info -> entry with" "~n UserId: ~p" "~n TargetName: ~p" - "~n Item: ~p" - "~n Val: ~p", [UserId, TargetName, Item, Val]), + "~n Info: ~p", [UserId, TargetName, Info]), + %% Verify ownership case (catch agent_info(TargetName, user_id)) of - {ok, UserId} -> - do_update_agent_info(TargetName, Item, Val); + {ok, UserId} -> + handle_update_agent_info(TargetName, Info); {ok, OtherUserId} -> {error, {not_owner, OtherUserId}}; Error -> Error end. -do_update_agent_info(TargetName, Item, Val0) -> -%% p("do_update_agent_info -> entry with" -%% "~n TargetName: ~p" -%% "~n Item: ~p" -%% "~n Val0: ~p", [TargetName, Item, Val0]), - case verify_val(Item, Val0) of - {ok, Val} -> -%% p("do_update_agent_info -> verified value" -%% "~n Val: ~p", [Val]), - ets:insert(snmpm_agent_table, {{TargetName, Item}, Val}), - ok; +handle_update_agent_info(TargetName, Info0) -> + ?vtrace("handle_update_agent_info -> entry with" + "~n TargetName: ~p" + "~n Info0: ~p", [TargetName, Info0]), + %% Verify info + try verify_agent_info(TargetName, Info0) of + {ok, Info} -> + do_update_agent_info(TargetName, Info); Error -> - ?vlog("do_update_agent_info -> verify value failed: " - "~n TargetName: ~p" - "~n Item: ~p" - "~n Val0: ~p" - "~n Error: ~p", [TargetName, Item, Val0, Error]), - {error, {bad_agent_val, TargetName, Item, Val0}} + Error + catch + throw:Error -> + Error; + T:E -> + {error, {failed_info_verification, Info0, T, E}} end. +handle_update_agent_info(UserId, TargetName, Item, Val) -> + ?vdebug("handle_update_agent_info -> entry with" + "~n UserId: ~p" + "~n TargetName: ~p" + "~n Item: ~p" + "~n Val: ~p", [UserId, TargetName, Item, Val]), + handle_update_agent_info(TargetName, [{Item, Val}]). + +do_update_agent_info(TargetName, Info) -> + InsertItem = + fun({Item, Val}) -> + ets:insert(snmpm_agent_table, {{TargetName, Item}, Val}) + end, + lists:foreach(InsertItem, Info). + +do_update_agent_info(TargetName, Item, Val) -> + ?vtrace("do_update_agent_info -> entry with" + "~n TargetName: ~p" + "~n Item: ~p" + "~n Val: ~p", [TargetName, Item, Val]), + ets:insert(snmpm_agent_table, {{TargetName, Item}, Val}), + ok. + handle_register_usm_user(#usm_user{engine_id = EngineID, name = Name} = User) -> @@ -2791,7 +2979,7 @@ verify_mandatory(Conf, [Mand|Mands]) -> true -> verify_mandatory(Conf, Mands); false -> - {error, {missing_mandatory_config, Mand}} + throw({error, {missing_mandatory_config, Mand}}) end. verify_invalid(_, []) -> @@ -2801,7 +2989,7 @@ verify_invalid(Conf, [Inv|Invs]) -> false -> verify_invalid(Conf, Invs); true -> - {error, {illegal_config, Inv}} + throw({error, {illegal_config, Inv}}) end. @@ -2810,10 +2998,26 @@ verify_val(user_id, UserId) -> verify_val(reg_type, RegType) when (RegType =:= addr_port) orelse (RegType =:= target_name) -> {ok, RegType}; -verify_val(address, Addr0) -> - case normalize_address(Addr0) of +verify_val(tdomain = Item, snmpUDPDomain = _Domain) -> + verify_val(Item, transportDomainUdpIpv4); +verify_val(tdomain, Domain) -> + case lists:member(Domain, ?SUPPORTED_DOMAINS) of + true -> + {ok, Domain}; + false -> + case lists:member(Domain, snmp_conf:all_domains()) of + true -> + error({unsupported_domain, Domain}); + false -> + error({unknown_domain, Domain}) + end + end; +verify_val(address, {Domain, Addr0}) -> + case normalize_address(Domain, Addr0) of {_A1, _A2, _A3, _A4} = Addr -> {ok, Addr}; + {_A1, _A2, _A3, _A4, _A5, _A6, _A7, _A8} = Addr -> + {ok, Addr}; _ when is_list(Addr0) -> case (catch snmp_conf:check_ip(Addr0)) of ok -> @@ -2824,6 +3028,8 @@ verify_val(address, Addr0) -> _ -> error({bad_address, Addr0}) end; +verify_val(address, BadAddress) -> + error({bad_address, BadAddress}); verify_val(port, Port) -> case (catch snmp_conf:check_integer(Port, {gt, 0})) of ok -> @@ -2875,7 +3081,7 @@ verify_val(sec_name, BadName) -> verify_val(sec_level, Level) -> (catch snmp_conf:check_sec_level(Level)); verify_val(Item, _) -> - {error, {no_such_item, Item}}. + {error, {unknown_item, Item}}. %%%------------------------------------------------------------------- @@ -3034,11 +3240,17 @@ init_mini_mib_elems(MibName, [_|T], Res) -> %%---------------------------------------------------------------------- normalize_address(Addr) -> - case inet:getaddr(Addr, inet) of + normalize_address(snmpUDPDomain, Addr). + +normalize_address(snmpUDPDomain, Addr) -> + normalize_address(transportDomainUdpIpv4, Addr); + +normalize_address(Domain, Addr) -> + case inet:getaddr(Addr, td2fam(Domain)) of {ok, Addr2} -> Addr2; _ when is_list(Addr) -> - case (catch snmp_conf:check_ip(Addr)) of + case (catch snmp_conf:check_ip(Domain, Addr)) of ok -> list_to_tuple(Addr); _ -> @@ -3048,6 +3260,9 @@ normalize_address(Addr) -> Addr end. +td2fam(transportDomainUdpIpv4) -> inet; +td2fam(transportDomainUdpIpv6) -> inet6. + %%---------------------------------------------------------------------- diff --git a/lib/snmp/src/manager/snmpm_mpd.erl b/lib/snmp/src/manager/snmpm_mpd.erl index 7712370d28..627838e3d4 100644 --- a/lib/snmp/src/manager/snmpm_mpd.erl +++ b/lib/snmp/src/manager/snmpm_mpd.erl @@ -92,7 +92,7 @@ reset(#state{v3 = V3}) -> %% Purpose: This is the main Message Dispatching function. (see %% section 4.2.1 in rfc2272) %%----------------------------------------------------------------- -process_msg(Msg, TDomain, Addr, Port, State, NoteStore, Logger) -> +process_msg(Msg, Domain, Addr, Port, State, NoteStore, Logger) -> inc(snmpInPkts), @@ -102,18 +102,18 @@ process_msg(Msg, TDomain, Addr, Port, State, NoteStore, Logger) -> #message{version = 'version-1', vsn_hdr = Community, data = Data} when State#state.v1 =:= true -> HS = ?empty_msg_size + length(Community), - process_v1_v2c_msg('version-1', NoteStore, Msg, TDomain, - Addr, Port, + process_v1_v2c_msg('version-1', NoteStore, Msg, + Domain, Addr, Port, Community, Data, HS, Logger); %% Version 2 #message{version = 'version-2', vsn_hdr = Community, data = Data} when State#state.v2c =:= true -> HS = ?empty_msg_size + length(Community), - process_v1_v2c_msg('version-2', NoteStore, Msg, TDomain, - Addr, Port, - Community, Data, HS, Logger); - + (catch process_v1_v2c_msg('version-2', NoteStore, Msg, + Domain, Addr, Port, + Community, Data, HS, Logger)); + %% Version 3 #message{version = 'version-3', vsn_hdr = H, data = Data} when State#state.v3 =:= true -> @@ -148,17 +148,30 @@ process_msg(Msg, TDomain, Addr, Port, State, NoteStore, Logger) -> %%----------------------------------------------------------------- %% Handles a Community based message (v1 or v2c). %%----------------------------------------------------------------- -process_v1_v2c_msg(Vsn, _NoteStore, Msg, snmpUDPDomain, +process_v1_v2c_msg(Vsn, _NoteStore, Msg, Domain, Addr, Port, Community, Data, HS, Log) -> ?vdebug("process_v1_v2c_msg -> entry with" "~n Vsn: ~p" + "~n Domain: ~p" "~n Addr: ~p" "~n Port: ~p" "~n Community: ~p" - "~n HS: ~p", [Vsn, Addr, Port, Community, HS]), + "~n HS: ~p", [Vsn, Domain, Addr, Port, Community, HS]), + {TDomain, TAddress} = + try + begin + TD = snmp_conf:mk_tdomain(Domain), + TA = snmp_conf:mk_taddress(Domain, Addr, Port), + {TD, TA} + end + catch + throw:{error, TReason} -> + throw({discarded, {badarg, Domain, TReason}}) + end, + Max = get_max_message_size(), AgentMax = get_agent_max_message_size(Addr, Port), PduMS = pdu_ms(Max, AgentMax, HS), @@ -170,14 +183,14 @@ process_v1_v2c_msg(Vsn, _NoteStore, Msg, snmpUDPDomain, ?vtrace("process_v1_v2c_msg -> was a pdu", []), Log(Msg), inc_snmp_in(Pdu), - MsgData = {Community, sec_model(Vsn)}, + MsgData = {Community, sec_model(Vsn), TDomain, TAddress}, {ok, Vsn, Pdu, PduMS, MsgData}; Trap when is_record(Trap, trappdu) -> ?vtrace("process_v1_v2c_msg -> was a trap", []), Log(Msg), inc_snmp_in(Trap), - MsgData = {Community, sec_model(Vsn)}, + MsgData = {Community, sec_model(Vsn), TDomain, TAddress}, {ok, Vsn, Trap, PduMS, MsgData}; {'EXIT', Reason} -> @@ -185,11 +198,7 @@ process_v1_v2c_msg(Vsn, _NoteStore, Msg, snmpUDPDomain, "~n Reason: ~p", [Reason]), inc(snmpInASNParseErrs), {discarded, Reason} - end; -process_v1_v2c_msg(_Vsn, _NoteStore, _Msg, TDomain, - _Addr, _Port, - _Comm, _HS, _Data, _Log) -> - {discarded, {badarg, TDomain}}. + end. pdu_ms(MgrMMS, AgentMMS, HS) when AgentMMS < MgrMMS -> AgentMMS - HS; @@ -482,8 +491,8 @@ generate_msg('version-3', NoteStore, Pdu, generate_v3_msg(NoteStore, Pdu, SecModel, SecName, SecLevel, CtxEngineID, CtxName, TargetName, Log); -generate_msg(Vsn, _NoteStore, Pdu, {Community, _SecModel}, Log) -> - generate_v1_v2c_msg(Vsn, Pdu, Community, Log). +generate_msg(Vsn, _NoteStore, Pdu, {Comm, _SecModel}, Log) -> + generate_v1_v2c_msg(Vsn, Pdu, Comm, Log). generate_v3_msg(NoteStore, Pdu, @@ -627,6 +636,8 @@ generate_response_msg('version-3', Pdu, generate_v3_response_msg(Pdu, MsgID, SecModel, SecName, SecLevel, CtxEngineID, CtxName, SecData, Log); generate_response_msg(Vsn, Pdu, {Comm, _SecModel}, Log) -> + generate_v1_v2c_response_msg(Vsn, Pdu, Comm, Log); +generate_response_msg(Vsn, Pdu, {Comm, _SecModel, _TDomain, _TAddress}, Log) -> generate_v1_v2c_response_msg(Vsn, Pdu, Comm, Log). diff --git a/lib/snmp/src/manager/snmpm_net_if.erl b/lib/snmp/src/manager/snmpm_net_if.erl index a116c9f26b..4d6bd9aa33 100644 --- a/lib/snmp/src/manager/snmpm_net_if.erl +++ b/lib/snmp/src/manager/snmpm_net_if.erl @@ -28,7 +28,8 @@ start_link/2, stop/1, send_pdu/6, % Backward compatibillity - send_pdu/7, + send_pdu/7, % Backward compatibillity + send_pdu/8, inform_response/4, @@ -101,16 +102,21 @@ stop(Pid) -> send_pdu(Pid, Pdu, Vsn, MsgData, Addr, Port) -> send_pdu(Pid, Pdu, Vsn, MsgData, Addr, Port, ?DEFAULT_EXTRA_INFO). -send_pdu(Pid, Pdu, Vsn, MsgData, Addr, Port, ExtraInfo) +send_pdu(Pid, Pdu, Vsn, MsgData, Addr, Port, ExtraInfo) -> + Domain = snmpm_config:default_transport_domain(), + send_pdu(Pid, Pdu, Vsn, MsgData, Domain, Addr, Port, ExtraInfo). + +send_pdu(Pid, Pdu, Vsn, MsgData, Domain, Addr, Port, ExtraInfo) when is_record(Pdu, pdu) -> ?d("send_pdu -> entry with" "~n Pid: ~p" "~n Pdu: ~p" "~n Vsn: ~p" "~n MsgData: ~p" + "~n Domain: ~p" "~n Addr: ~p" - "~n Port: ~p", [Pid, Pdu, Vsn, MsgData, Addr, Port]), - cast(Pid, {send_pdu, Pdu, Vsn, MsgData, Addr, Port, ExtraInfo}). + "~n Port: ~p", [Pid, Pdu, Vsn, MsgData, Domain, Addr, Port]), + cast(Pid, {send_pdu, Pdu, Vsn, MsgData, Domain, Addr, Port, ExtraInfo}). note_store(Pid, NoteStore) -> call(Pid, {note_store, NoteStore}). @@ -380,15 +386,17 @@ handle_call(Req, From, State) -> %% {noreply, State, Timeout} | %% {stop, Reason, State} (terminate/2 is called) %%-------------------------------------------------------------------- -handle_cast({send_pdu, Pdu, Vsn, MsgData, Addr, Port, ExtraInfo}, State) -> +handle_cast({send_pdu, Pdu, Vsn, MsgData, Domain, Addr, Port, ExtraInfo}, + State) -> ?vlog("received send_pdu message with" "~n Pdu: ~p" "~n Vsn: ~p" "~n MsgData: ~p" + "~n Domain: ~p" "~n Addr: ~p" - "~n Port: ~p", [Pdu, Vsn, MsgData, Addr, Port]), + "~n Port: ~p", [Pdu, Vsn, MsgData, Domain, Addr, Port]), maybe_process_extra_info(ExtraInfo), - maybe_handle_send_pdu(Pdu, Vsn, MsgData, Addr, Port, State), + maybe_handle_send_pdu(Pdu, Vsn, MsgData, Domain, Addr, Port, State), {noreply, State}; handle_cast({inform_response, Ref, Addr, Port}, State) -> @@ -545,8 +553,9 @@ handle_recv_msg(Addr, Port, Bytes, mpd_state = MpdState, sock = Sock, log = Log} = State) -> + Domain = snmp_conf:which_domain(Addr), % What the ****... Logger = logger(Log, read, Addr, Port), - case (catch snmpm_mpd:process_msg(Bytes, snmpUDPDomain, Addr, Port, + case (catch snmpm_mpd:process_msg(Bytes, Domain, Addr, Port, MpdState, NoteStore, Logger)) of {ok, Vsn, Pdu, MS, ACM} -> @@ -734,17 +743,17 @@ irgc_stop(Ref) -> (catch erlang:cancel_timer(Ref)). -maybe_handle_send_pdu(Pdu, Vsn, MsgData, Addr, Port, +maybe_handle_send_pdu(Pdu, Vsn, MsgData, Domain, Addr, Port, #state{filter = FilterMod} = State) -> case (catch FilterMod:accept_send_pdu(Addr, Port, pdu_type_of(Pdu))) of false -> inc(netIfPduOutDrops), ok; _ -> - handle_send_pdu(Pdu, Vsn, MsgData, Addr, Port, State) + handle_send_pdu(Pdu, Vsn, MsgData, Domain, Addr, Port, State) end. -handle_send_pdu(Pdu, Vsn, MsgData, Addr, Port, +handle_send_pdu(Pdu, Vsn, MsgData, _Domain, Addr, Port, #state{server = Pid, note_store = NoteStore, sock = Sock, diff --git a/lib/snmp/src/manager/snmpm_server.erl b/lib/snmp/src/manager/snmpm_server.erl index 58a58507d6..484954addb 100644 --- a/lib/snmp/src/manager/snmpm_server.erl +++ b/lib/snmp/src/manager/snmpm_server.erl @@ -161,7 +161,8 @@ {id, user_id, reg_type, - target, + target, + domain, addr, port, type, @@ -1175,11 +1176,12 @@ handle_sync_get(Pid, UserId, TargetName, Oids, SendOpts, From, State) -> "~n From: ~p", [Pid, UserId, TargetName, Oids, SendOpts, From]), case agent_data(TargetName, SendOpts) of - {ok, RegType, Addr, Port, Vsn, MsgData} -> + {ok, RegType, Domain, Addr, Port, Vsn, MsgData} -> ?vtrace("handle_sync_get -> send a ~p message", [Vsn]), Extra = ?GET_EXTRA(SendOpts), ReqId = send_get_request(Oids, Vsn, MsgData, - Addr, Port, Extra, State), + Domain, Addr, Port, + Extra, State), ?vdebug("handle_sync_get -> ReqId: ~p", [ReqId]), Msg = {sync_timeout, ReqId, From}, Timeout = ?SYNC_GET_TIMEOUT(SendOpts), @@ -1190,6 +1192,7 @@ handle_sync_get(Pid, UserId, TargetName, Oids, SendOpts, From, State) -> user_id = UserId, reg_type = RegType, target = TargetName, + domain = Domain, addr = Addr, port = Port, type = get, @@ -1227,11 +1230,12 @@ handle_sync_get_next(Pid, UserId, TargetName, Oids, SendOpts, "~n From: ~p", [Pid, UserId, TargetName, Oids, SendOpts, From]), case agent_data(TargetName, SendOpts) of - {ok, RegType, Addr, Port, Vsn, MsgData} -> + {ok, RegType, Domain, Addr, Port, Vsn, MsgData} -> ?vtrace("handle_sync_get_next -> send a ~p message", [Vsn]), Extra = ?GET_EXTRA(SendOpts), ReqId = send_get_next_request(Oids, Vsn, MsgData, - Addr, Port, Extra, State), + Domain, Addr, Port, + Extra, State), ?vdebug("handle_sync_get_next -> ReqId: ~p", [ReqId]), Msg = {sync_timeout, ReqId, From}, Timeout = ?SYNC_GET_NEXT_TIMEOUT(SendOpts), @@ -1242,6 +1246,7 @@ handle_sync_get_next(Pid, UserId, TargetName, Oids, SendOpts, user_id = UserId, reg_type = RegType, target = TargetName, + domain = Domain, addr = Addr, port = Port, type = get_next, @@ -1285,10 +1290,11 @@ handle_sync_get_bulk(Pid, UserId, TargetName, NonRep, MaxRep, Oids, SendOpts, "~n From: ~p", [Pid, UserId, TargetName, NonRep, MaxRep, Oids, SendOpts, From]), case agent_data(TargetName, SendOpts) of - {ok, RegType, Addr, Port, Vsn, MsgData} -> + {ok, RegType, Domain, Addr, Port, Vsn, MsgData} -> ?vtrace("handle_sync_get_bulk -> send a ~p message", [Vsn]), Extra = ?GET_EXTRA(SendOpts), - ReqId = send_get_bulk_request(Oids, Vsn, MsgData, Addr, Port, + ReqId = send_get_bulk_request(Oids, Vsn, MsgData, + Domain, Addr, Port, NonRep, MaxRep, Extra, State), ?vdebug("handle_sync_get_bulk -> ReqId: ~p", [ReqId]), Msg = {sync_timeout, ReqId, From}, @@ -1300,6 +1306,7 @@ handle_sync_get_bulk(Pid, UserId, TargetName, NonRep, MaxRep, Oids, SendOpts, user_id = UserId, reg_type = RegType, target = TargetName, + domain = Domain, addr = Addr, port = Port, type = get_bulk, @@ -1339,11 +1346,12 @@ handle_sync_set(Pid, UserId, TargetName, VarsAndVals, SendOpts, From, State) -> "~n From: ~p", [Pid, UserId, TargetName, VarsAndVals, From]), case agent_data(TargetName, SendOpts) of - {ok, RegType, Addr, Port, Vsn, MsgData} -> + {ok, RegType, Domain, Addr, Port, Vsn, MsgData} -> ?vtrace("handle_sync_set -> send a ~p message", [Vsn]), Extra = ?GET_EXTRA(SendOpts), ReqId = send_set_request(VarsAndVals, Vsn, MsgData, - Addr, Port, Extra, State), + Domain, Addr, Port, + Extra, State), ?vdebug("handle_sync_set -> ReqId: ~p", [ReqId]), Msg = {sync_timeout, ReqId, From}, Timeout = ?SYNC_SET_TIMEOUT(SendOpts), @@ -1354,6 +1362,7 @@ handle_sync_set(Pid, UserId, TargetName, VarsAndVals, SendOpts, From, State) -> user_id = UserId, reg_type = RegType, target = TargetName, + domain = Domain, addr = Addr, port = Port, type = set, @@ -1391,10 +1400,11 @@ handle_async_get(Pid, UserId, TargetName, Oids, SendOpts, State) -> "~n SendOpts: ~p", [Pid, UserId, TargetName, Oids, SendOpts]), case agent_data(TargetName, SendOpts) of - {ok, RegType, Addr, Port, Vsn, MsgData} -> + {ok, RegType, Domain, Addr, Port, Vsn, MsgData} -> ?vtrace("handle_async_get -> send a ~p message", [Vsn]), Extra = ?GET_EXTRA(SendOpts), - ReqId = send_get_request(Oids, Vsn, MsgData, Addr, Port, + ReqId = send_get_request(Oids, Vsn, MsgData, + Domain, Addr, Port, Extra, State), ?vdebug("handle_async_get -> ReqId: ~p", [ReqId]), Expire = ?ASYNC_GET_TIMEOUT(SendOpts), @@ -1402,6 +1412,7 @@ handle_async_get(Pid, UserId, TargetName, Oids, SendOpts, State) -> user_id = UserId, reg_type = RegType, target = TargetName, + domain = Domain, addr = Addr, port = Port, type = get, @@ -1439,17 +1450,19 @@ handle_async_get_next(Pid, UserId, TargetName, Oids, SendOpts, State) -> "~n SendOpts: ~p", [Pid, UserId, TargetName, Oids, SendOpts]), case agent_data(TargetName, SendOpts) of - {ok, RegType, Addr, Port, Vsn, MsgData} -> + {ok, RegType, Domain, Addr, Port, Vsn, MsgData} -> ?vtrace("handle_async_get_next -> send a ~p message", [Vsn]), Extra = ?GET_EXTRA(SendOpts), ReqId = send_get_next_request(Oids, Vsn, MsgData, - Addr, Port, Extra, State), + Domain, Addr, Port, + Extra, State), ?vdebug("handle_async_get_next -> ReqId: ~p", [ReqId]), Expire = ?ASYNC_GET_NEXT_TIMEOUT(SendOpts), Req = #request{id = ReqId, user_id = UserId, reg_type = RegType, target = TargetName, + domain = Domain, addr = Addr, port = Port, type = get_next, @@ -1494,10 +1507,11 @@ handle_async_get_bulk(Pid, "~n SendOpts: ~p", [Pid, UserId, TargetName, NonRep, MaxRep, Oids, SendOpts]), case agent_data(TargetName, SendOpts) of - {ok, RegType, Addr, Port, Vsn, MsgData} -> + {ok, RegType, Domain, Addr, Port, Vsn, MsgData} -> ?vtrace("handle_async_get_bulk -> send a ~p message", [Vsn]), Extra = ?GET_EXTRA(SendOpts), - ReqId = send_get_bulk_request(Oids, Vsn, MsgData, Addr, Port, + ReqId = send_get_bulk_request(Oids, Vsn, MsgData, + Domain, Addr, Port, NonRep, MaxRep, Extra, State), ?vdebug("handle_async_get_bulk -> ReqId: ~p", [ReqId]), Expire = ?ASYNC_GET_BULK_TIMEOUT(SendOpts), @@ -1505,6 +1519,7 @@ handle_async_get_bulk(Pid, user_id = UserId, reg_type = RegType, target = TargetName, + domain = Domain, addr = Addr, port = Port, type = get_bulk, @@ -1541,17 +1556,19 @@ handle_async_set(Pid, UserId, TargetName, VarsAndVals, SendOpts, State) -> "~n SendOpts: ~p", [Pid, UserId, TargetName, VarsAndVals, SendOpts]), case agent_data(TargetName, SendOpts) of - {ok, RegType, Addr, Port, Vsn, MsgData} -> + {ok, RegType, Domain, Addr, Port, Vsn, MsgData} -> ?vtrace("handle_async_set -> send a ~p message", [Vsn]), Extra = ?GET_EXTRA(SendOpts), ReqId = send_set_request(VarsAndVals, Vsn, MsgData, - Addr, Port, Extra, State), + Domain, Addr, Port, + Extra, State), ?vdebug("handle_async_set -> ReqId: ~p", [ReqId]), Expire = ?ASYNC_SET_TIMEOUT(SendOpts), Req = #request{id = ReqId, user_id = UserId, reg_type = RegType, target = TargetName, + domain = Domain, addr = Addr, port = Port, type = set, @@ -2907,7 +2924,7 @@ do_gc(Key, Now) -> %% %%---------------------------------------------------------------------- -send_get_request(Oids, Vsn, MsgData, Addr, Port, ExtraInfo, +send_get_request(Oids, Vsn, MsgData, Domain, Addr, Port, ExtraInfo, #state{net_if = NetIf, net_if_mod = Mod, mini_mib = MiniMIB}) -> @@ -2918,34 +2935,39 @@ send_get_request(Oids, Vsn, MsgData, Addr, Port, ExtraInfo, "~n Pdu: ~p" "~n Vsn: ~p" "~n MsgData: ~p" + "~n Domain: ~p" "~n Addr: ~p" - "~n Port: ~p", [Mod, NetIf, Pdu, Vsn, MsgData, Addr, Port]), - (catch Mod:send_pdu(NetIf, Pdu, Vsn, MsgData, Addr, Port, ExtraInfo)), + "~n Port: ~p", + [Mod, NetIf, Pdu, Vsn, MsgData, Domain, Addr, Port]), + Res = (catch Mod:send_pdu(NetIf, Pdu, Vsn, MsgData, + Domain, Addr, Port, ExtraInfo)), + ?vtrace("send_get_request -> send result:" + "~n ~p", [Res]), Pdu#pdu.request_id. -send_get_next_request(Oids, Vsn, MsgData, Addr, Port, ExtraInfo, +send_get_next_request(Oids, Vsn, MsgData, Domain, Addr, Port, ExtraInfo, #state{mini_mib = MiniMIB, net_if = NetIf, net_if_mod = Mod}) -> Pdu = make_pdu(get_next, Oids, MiniMIB), - Mod:send_pdu(NetIf, Pdu, Vsn, MsgData, Addr, Port, ExtraInfo), + Mod:send_pdu(NetIf, Pdu, Vsn, MsgData, Domain, Addr, Port, ExtraInfo), Pdu#pdu.request_id. -send_get_bulk_request(Oids, Vsn, MsgData, Addr, Port, +send_get_bulk_request(Oids, Vsn, MsgData, Domain, Addr, Port, NonRep, MaxRep, ExtraInfo, #state{mini_mib = MiniMIB, net_if = NetIf, net_if_mod = Mod}) -> Pdu = make_pdu(bulk, {NonRep, MaxRep, Oids}, MiniMIB), - Mod:send_pdu(NetIf, Pdu, Vsn, MsgData, Addr, Port, ExtraInfo), + Mod:send_pdu(NetIf, Pdu, Vsn, MsgData, Domain, Addr, Port, ExtraInfo), Pdu#pdu.request_id. -send_set_request(VarsAndVals, Vsn, MsgData, Addr, Port, ExtraInfo, +send_set_request(VarsAndVals, Vsn, MsgData, Domain, Addr, Port, ExtraInfo, #state{mini_mib = MiniMIB, net_if = NetIf, net_if_mod = Mod}) -> Pdu = make_pdu(set, VarsAndVals, MiniMIB), - Mod:send_pdu(NetIf, Pdu, Vsn, MsgData, Addr, Port, ExtraInfo), + Mod:send_pdu(NetIf, Pdu, Vsn, MsgData, Domain, Addr, Port, ExtraInfo), Pdu#pdu.request_id. %% send_discovery(Vsn, MsgData, Addr, Port, ExtraInfo, @@ -3181,10 +3203,11 @@ agent_data(TargetName, SendOpts) -> {Comm, SecModel} end, + Domain = agent_data_item(tdomain, Info), Addr = agent_data_item(address, Info), Port = agent_data_item(port, Info), RegType = agent_data_item(reg_type, Info), - {ok, RegType, Addr, Port, version(Version), MsgData}; + {ok, RegType, Domain, Addr, Port, version(Version), MsgData}; Error -> Error end. diff --git a/lib/snmp/src/misc/snmp_conf.erl b/lib/snmp/src/misc/snmp_conf.erl index 20f4455d10..7249def24e 100644 --- a/lib/snmp/src/misc/snmp_conf.erl +++ b/lib/snmp/src/misc/snmp_conf.erl @@ -37,7 +37,9 @@ check_timer/1, + all_domains/0, check_domain/1, + all_tdomains/0, check_tdomain/1, mk_tdomain/1, which_domain/1, @@ -345,6 +347,25 @@ check_sec_level(BadSecLevel) -> %% --------- +all_tdomains() -> + [ + ?transportDomainUdpIpv4, + ?transportDomainUdpIpv6, + ?transportDomainUdpIpv4z, + ?transportDomainUdpIpv6z, + ?transportDomainTcpIpv4, + ?transportDomainTcpIpv6, + ?transportDomainTcpIpv4z, + ?transportDomainTcpIpv6z, + ?transportDomainSctpIpv4, + ?transportDomainSctpIpv6, + ?transportDomainSctpIpv4z, + ?transportDomainSctpIpv6z, + ?transportDomainLocal, + ?transportDomainUdpDns, + ?transportDomainTcpDns, + ?transportDomainSctpDns + ]. check_tdomain(TDomain) -> SupportedTDomains = @@ -353,25 +374,7 @@ check_tdomain(TDomain) -> ?transportDomainUdpIpv4, ?transportDomainUdpIpv6 ], - AllTDomains = - [ - ?transportDomainUdpIpv4, - ?transportDomainUdpIpv6, - ?transportDomainUdpIpv4z, - ?transportDomainUdpIpv6z, - ?transportDomainTcpIpv4, - ?transportDomainTcpIpv6, - ?transportDomainTcpIpv4z, - ?transportDomainTcpIpv6z, - ?transportDomainSctpIpv4, - ?transportDomainSctpIpv6, - ?transportDomainSctpIpv4z, - ?transportDomainSctpIpv6z, - ?transportDomainLocal, - ?transportDomainUdpDns, - ?transportDomainTcpDns, - ?transportDomainSctpDns - ], + AllTDomains = all_tdomains(), case lists:member(TDomain, SupportedTDomains) of true -> ok; @@ -388,7 +391,7 @@ check_tdomain(TDomain) -> %% --------- mk_tdomain(snmpUDPDomain) -> - ?snmpUDPDomain; + mk_tdomain(transportDomainUdpIpv4); mk_tdomain(transportDomainUdpIpv4) -> ?transportDomainUdpIpv4; mk_tdomain(transportDomainUdpIpv6) -> @@ -474,6 +477,26 @@ do_check_timer(WaitFor, Factor, Incr, Retry) -> %% --------- +all_domains() -> + [ + transportDomainUdpIpv4, + transportDomainUdpIpv6, + transportDomainUdpIpv4z, + transportDomainUdpIpv6z, + transportDomainTcpIpv4, + transportDomainTcpIpv6, + transportDomainTcpIpv4z, + transportDomainTcpIpv6z, + transportDomainSctpIpv4, + transportDomainSctpIpv6, + transportDomainSctpIpv4z, + transportDomainSctpIpv6z, + transportDomainLocal, + transportDomainUdpDns, + transportDomainTcpDns, + transportDomainSctpDns + ]. + check_domain(Domain) -> SupportedDomains = [ @@ -481,25 +504,7 @@ check_domain(Domain) -> transportDomainUdpIpv4, transportDomainUdpIpv6 ], - AllDomains = - [ - transportDomainUdpIpv4, - transportDomainUdpIpv6, - transportDomainUdpIpv4z, - transportDomainUdpIpv6z, - transportDomainTcpIpv4, - transportDomainTcpIpv6, - transportDomainTcpIpv4z, - transportDomainTcpIpv6z, - transportDomainSctpIpv4, - transportDomainSctpIpv6, - transportDomainSctpIpv4z, - transportDomainSctpIpv6z, - transportDomainLocal, - transportDomainUdpDns, - transportDomainTcpDns, - transportDomainSctpDns - ], + AllDomains = all_domains(), case lists:member(Domain, SupportedDomains) of true -> ok; diff --git a/lib/snmp/src/misc/snmp_config.erl b/lib/snmp/src/misc/snmp_config.erl index fcbc6a88c9..6ab20e3e48 100644 --- a/lib/snmp/src/misc/snmp_config.erl +++ b/lib/snmp/src/misc/snmp_config.erl @@ -337,7 +337,7 @@ config_agent_sys() -> {dir, ATLDir}, {size, ATLSize}, {repair, ATLRepair}, - {seqno, ATLSeqNo}]}]; + {seqno, ATLSeqNo}]}]; no -> [] end, @@ -568,7 +568,7 @@ config_agent_snmp(Dir, Vsns) -> false -> ok end, - i("The following agent files were written: agent.conf, " + i("The following agent files where written: agent.conf, " "community.conf,~n" "standard.conf, target_addr.conf, " "target_params.conf, ~n" @@ -776,7 +776,7 @@ config_manager_snmp(Dir, Vsns) -> Users, Agents, Usms)) of ok -> i("~n- - - - - - - - - - - - -"), - i("The following manager files were written: " + i("The following manager files where written: " "manager.conf, agents.conf " ++ case lists:member(v3, Vsns) of true -> @@ -2350,7 +2350,9 @@ write_sys_config_file_manager_atl_opt(Fid, {type, Type}) -> write_sys_config_file_manager_atl_opt(Fid, {size, Size}) -> ok = io:format(Fid, "{size, ~w}", [Size]); write_sys_config_file_manager_atl_opt(Fid, {repair, Rep}) -> - ok = io:format(Fid, "{repair, ~w}", [Rep]). + ok = io:format(Fid, "{repair, ~w}", [Rep]); +write_sys_config_file_manager_atl_opt(Fid, {seqno, SeqNo}) -> + ok = io:format(Fid, "{seqno, ~w}", [SeqNo]). header() -> diff --git a/lib/snmp/test/snmp_compiler_test.erl b/lib/snmp/test/snmp_compiler_test.erl index 2e6020ae7a..cee11ba97a 100644 --- a/lib/snmp/test/snmp_compiler_test.erl +++ b/lib/snmp/test/snmp_compiler_test.erl @@ -47,6 +47,7 @@ module_identity/1, agent_capabilities/1, module_compliance/1, + warnings_as_errors/1, otp_6150/1, otp_8574/1, @@ -97,9 +98,10 @@ all() -> description, oid_conflicts, imports, - module_identity, - agent_capabilities, - module_compliance, + module_identity, + agent_capabilities, + module_compliance, + warnings_as_errors, {group, tickets} ]. @@ -152,6 +154,8 @@ description(Config) when is_list(Config) -> ok. +%%====================================================================== + oid_conflicts(suite) -> []; oid_conflicts(Config) when is_list(Config) -> put(tname,oid_conflicts), @@ -165,18 +169,24 @@ oid_conflicts(Config) when is_list(Config) -> ok. +%%====================================================================== + imports(suite) -> []; imports(Config) when is_list(Config) -> ?SKIP(not_yet_implemented). +%%====================================================================== + module_identity(suite) -> []; module_identity(Config) when is_list(Config) -> ?SKIP(not_yet_implemented). +%%====================================================================== + agent_capabilities(suite) -> []; agent_capabilities(Config) when is_list(Config) -> @@ -218,6 +228,8 @@ agent_capabilities(Config) when is_list(Config) -> ok. +%%====================================================================== + module_compliance(suite) -> []; module_compliance(Config) when is_list(Config) -> @@ -259,6 +271,31 @@ module_compliance(Config) when is_list(Config) -> ok. +%%====================================================================== + +warnings_as_errors(suite) -> + ["OTP-9437"]; +warnings_as_errors(Config) when is_list(Config) -> + put(tname,warnings_as_errors), + p("starting with Config: ~p~n", [Config]), + Dir = ?config(comp_dir, Config), + MibDir = ?config(mib_dir, Config), + MibFile = join(MibDir, "OTP8574-MIB.mib"), + OutFile = join(Dir, "OTP8574-MIB.bin"), + Opts = [{group_check, false}, + {outdir, Dir}, + {verbosity, trace}, + relaxed_row_name_assign_check], + {error, compilation_failed} = + snmpc:compile(MibFile, [warnings_as_errors|Opts]), + false = filelib:is_regular(OutFile), + {ok, _} = snmpc:compile(MibFile, Opts), + true = filelib:is_regular(OutFile), + ok. + + +%%====================================================================== + otp_6150(suite) -> []; otp_6150(Config) when is_list(Config) -> @@ -273,6 +310,8 @@ otp_6150(Config) when is_list(Config) -> ok. +%%====================================================================== + otp_8574(suite) -> []; otp_8574(Config) when is_list(Config) -> @@ -304,6 +343,8 @@ otp_8574(Config) when is_list(Config) -> end. +%%====================================================================== + otp_8595(suite) -> []; otp_8595(Config) when is_list(Config) -> diff --git a/lib/snmp/test/snmp_manager_test.erl b/lib/snmp/test/snmp_manager_test.erl index 0b536748fb..d18f20d359 100644 --- a/lib/snmp/test/snmp_manager_test.erl +++ b/lib/snmp/test/snmp_manager_test.erl @@ -61,6 +61,7 @@ register_agent1/1, register_agent2/1, + register_agent3/1, info/1, @@ -383,12 +384,12 @@ end_per_testcase2(Case, Config) -> all() -> [ {group, start_and_stop_tests}, - {group, misc_tests}, + {group, misc_tests}, {group, user_tests}, - {group, agent_tests}, + {group, agent_tests}, {group, request_tests}, {group, event_tests}, - discovery, + discovery, {group, tickets} ]. @@ -417,7 +418,8 @@ groups() -> {agent_tests, [], [ register_agent1, - register_agent2 + register_agent2, + register_agent3 ] }, {request_tests, [], @@ -477,14 +479,14 @@ groups() -> }, {event_tests, [], [ - trap1, - trap2, - inform1, - inform2, - inform3, - inform4, - inform_swarm, - report + trap1%% , + %% trap2, + %% inform1, + %% inform2, + %% inform3, + %% inform4, + %% inform_swarm, + %% report ] }, {tickets, [], @@ -1134,6 +1136,7 @@ register_agent1(suite) -> register_agent1(Config) when is_list(Config) -> process_flag(trap_exit, true), put(tname,ra1), + p("starting with Config: ~p~n", [Config]), ManagerNode = start_manager_node(), @@ -1164,7 +1167,7 @@ register_agent1(Config) when is_list(Config) -> p("manager info: ~p~n", [mgr_info(ManagerNode)]), - p("register user(s) calvin & hobbe"), + p("register user(s) user_alfa & user_beta"), ?line ok = mgr_register_user(ManagerNode, user_alfa, snmpm_user_default, []), ?line ok = mgr_register_user(ManagerNode, user_beta, snmpm_user_default, []), p("manager info: ~p~n", [mgr_info(ManagerNode)]), @@ -1293,7 +1296,7 @@ register_agent2(Config) when is_list(Config) -> p("manager info: ~p~n", [mgr_info(ManagerNode)]), - p("register user(s) calvin & hobbe"), + p("register user(s) user_alfa & user_beta"), ?line ok = mgr_register_user(ManagerNode, user_alfa, snmpm_user_default, []), ?line ok = mgr_register_user(ManagerNode, user_beta, snmpm_user_default, []), p("manager info: ~p~n", [mgr_info(ManagerNode)]), @@ -1348,7 +1351,7 @@ register_agent2(Config) when is_list(Config) -> end, p("manager info: ~p~n", [mgr_info(ManagerNode)]), - + p("unregister user user_alfa"), ?line ok = mgr_unregister_user(ManagerNode, user_alfa), @@ -1377,7 +1380,157 @@ register_agent2(Config) when is_list(Config) -> p("manager info: ~p~n", [mgr_info(ManagerNode)]), - p("unregister user hobbe"), + p("unregister user user_beta"), + ?line ok = mgr_unregister_user(ManagerNode, user_beta), + + p("manager info: ~p~n", [mgr_info(ManagerNode)]), + + ?SLEEP(1000), + + p("stop snmp application (with only manager)"), + ?line ok = stop_snmp(ManagerNode), + + ?SLEEP(1000), + + stop_node(ManagerNode), + + ?SLEEP(1000), + + p("end"), + ok. + + +%%====================================================================== + +register_agent3(doc) -> + ["Test registration of agents with the NEW interface functions " + "and specifying transport domain"]; +register_agent3(suite) -> + []; +register_agent3(Config) when is_list(Config) -> + process_flag(trap_exit, true), + put(tname, ra3), + p("starting with Config: ~p~n", [Config]), + + ManagerNode = start_manager_node(), + + ConfDir = ?config(manager_conf_dir, Config), + DbDir = ?config(manager_db_dir, Config), + LocalHost = snmp_test_lib:localhost(), + + + write_manager_conf(ConfDir), + + Opts = [{server, [{verbosity, trace}]}, + {net_if, [{verbosity, trace}]}, + {note_store, [{verbosity, trace}]}, + {config, [{verbosity, trace}, {dir, ConfDir}, {db_dir, DbDir}]}], + + + p("load snmp application"), + ?line ok = load_snmp(ManagerNode), + + p("set manager env for the snmp application"), + ?line ok = set_mgr_env(ManagerNode, Opts), + + p("starting snmp application (with only manager)"), + ?line ok = start_snmp(ManagerNode), + + p("started"), + + ?SLEEP(1000), + + p("manager info: ~p~n", [mgr_info(ManagerNode)]), + + p("register user(s) user_alfa & user_beta"), + ?line ok = mgr_register_user(ManagerNode, user_alfa, snmpm_user_default, []), + ?line ok = mgr_register_user(ManagerNode, user_beta, snmpm_user_default, []), + p("manager info: ~p~n", [mgr_info(ManagerNode)]), + + p("register agent(s)"), + TargetName1 = "agent2", + ?line ok = mgr_register_agent(ManagerNode, user_alfa, TargetName1, + [{tdomain, transportDomainUdpIpv4}, + {address, LocalHost}, + {port, 5001}, + {engine_id, "agentEngineId-1"}]), + TargetName2 = "agent3", + ?line ok = mgr_register_agent(ManagerNode, user_alfa, TargetName2, + [{tdomain, transportDomainUdpIpv6}, + {address, LocalHost}, + {port, 5002}, + {engine_id, "agentEngineId-2"}]), + TargetName3 = "agent4", + ?line {error, {unsupported_domain, _} = Reason4} = + mgr_register_agent(ManagerNode, user_beta, TargetName3, + [{tdomain, transportDomainTcpIpv4}, + {address, LocalHost}, + {port, 5003}, + {engine_id, "agentEngineId-3"}]), + p("Expected registration failure: ~p", [Reason4]), + TargetName4 = "agent5", + ?line {error, {unknown_domain, _} = Reason5} = + mgr_register_agent(ManagerNode, user_beta, TargetName4, + [{tdomain, transportDomainUdpIpv4_bad}, + {address, LocalHost}, + {port, 5004}, + {engine_id, "agentEngineId-4"}]), + p("Expected registration failure: ~p", [Reason5]), + + p("verify all agent(s): expect 2"), + case mgr_which_agents(ManagerNode) of + Agents1 when length(Agents1) =:= 2 -> + p("all agents: ~p~n", [Agents1]), + ok; + Agents1 -> + ?FAIL({agent_registration_failure, Agents1}) + end, + + p("verify user_alfa agent(s)"), + case mgr_which_agents(ManagerNode, user_alfa) of + Agents2 when length(Agents2) =:= 2 -> + p("calvin agents: ~p~n", [Agents2]), + ok; + Agents2 -> + ?FAIL({agent_registration_failure, Agents2}) + end, + + p("verify user_beta agent(s)"), + case mgr_which_agents(ManagerNode, user_beta) of + Agents3 when length(Agents3) =:= 0 -> + p("hobbe agents: ~p~n", [Agents3]), + ok; + Agents3 -> + ?FAIL({agent_registration_failure, Agents3}) + end, + + p("manager info: ~p~n", [mgr_info(ManagerNode)]), + + p("unregister user user_alfa"), + ?line ok = mgr_unregister_user(ManagerNode, user_alfa), + + p("verify all agent(s): expect 0"), + case mgr_which_agents(ManagerNode) of + Agents4 when length(Agents4) =:= 0 -> + p("all agents: ~p~n", [Agents4]), + ok; + Agents4 -> + ?FAIL({agent_unregistration_failure, Agents4}) + end, + p("manager info: ~p~n", [mgr_info(ManagerNode)]), + + p("verify all agent(s): expect 0"), + case mgr_which_agents(ManagerNode) of + [] -> + ok; + Agents5 -> + p("all agents: ~p~n", [Agents5]), + ?FAIL({agent_unregistration_failure, Agents5}) + end, + + p("manager info: ~p~n", [mgr_info(ManagerNode)]), + + p("unregister user user_beta"), ?line ok = mgr_unregister_user(ManagerNode, user_beta), p("manager info: ~p~n", [mgr_info(ManagerNode)]), diff --git a/lib/snmp/vsn.mk b/lib/snmp/vsn.mk index 29228fc59b..08251ab9ea 100644 --- a/lib/snmp/vsn.mk +++ b/lib/snmp/vsn.mk @@ -17,6 +17,6 @@ # # %CopyrightEnd% -SNMP_VSN = 4.20 +SNMP_VSN = 4.21 PRE_VSN = APP_VSN = "snmp-$(SNMP_VSN)$(PRE_VSN)" diff --git a/lib/ssl/test/ssl_basic_SUITE.erl b/lib/ssl/test/ssl_basic_SUITE.erl index d9f4a76d80..37a021e7cf 100644 --- a/lib/ssl/test/ssl_basic_SUITE.erl +++ b/lib/ssl/test/ssl_basic_SUITE.erl @@ -257,7 +257,7 @@ all() -> %%different_ca_peer_sign, no_reuses_session_server_restart_new_cert, no_reuses_session_server_restart_new_cert_file, reuseaddr, - hibernate + hibernate, connect_twice ]. groups() -> @@ -3609,6 +3609,54 @@ hibernate(Config) -> ssl_test_lib:close(Client). %%-------------------------------------------------------------------- + +connect_twice(doc) -> + [""]; +connect_twice(suite) -> + []; +connect_twice(Config) when is_list(Config) -> + ClientOpts = ?config(client_opts, Config), + ServerOpts = ?config(server_opts, Config), + + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + + Server = + ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {?MODULE, send_recv_result, []}}, + {options, [{keepalive, true},{active, false} + | ServerOpts]}]), + Port = ssl_test_lib:inet_port(Server), + Client = + ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, + {host, Hostname}, + {from, self()}, + {mfa, {?MODULE, send_recv_result, []}}, + {options, [{keepalive, true},{active, false} + | ClientOpts]}]), + Server ! listen, + + {Client1, #sslsocket{}} = + ssl_test_lib:start_client([return_socket, + {node, ClientNode}, {port, Port}, + {host, Hostname}, + {from, self()}, + {mfa, {?MODULE, send_recv_result, []}}, + {options, [{keepalive, true},{active, false} + | ClientOpts]}]), + + test_server:format("Testcase ~p, Client ~p Server ~p ~n", + [self(), Client, Server]), + + ssl_test_lib:check_result(Server, ok, Client, ok), + ssl_test_lib:check_result(Server, ok, Client1, ok), + + ssl_test_lib:close(Server), + ssl_test_lib:close(Client), + ssl_test_lib:close(Client1). + + +%%-------------------------------------------------------------------- %%% Internal functions %%-------------------------------------------------------------------- send_recv_result(Socket) -> diff --git a/lib/ssl/test/ssl_session_cache_SUITE.erl b/lib/ssl/test/ssl_session_cache_SUITE.erl index f80ac3c1a9..5ea45018e6 100644 --- a/lib/ssl/test/ssl_session_cache_SUITE.erl +++ b/lib/ssl/test/ssl_session_cache_SUITE.erl @@ -223,15 +223,14 @@ session_cleanup(Config)when is_list(Config) -> %% Make sure session has expired and been cleaned up check_timer(SessionTimer), - test_server:sleep(?DELAY), %% Delay time + some extra time + test_server:sleep(?DELAY *2), %% Delay time + some extra time - {status, _, _, StatusInfo1} = sys:get_status(whereis(ssl_manager)), - [_, _,_, _, Prop1] = StatusInfo1, - State1 = state(Prop1), - DelayTimer = element(7, State1), + DelayTimer = get_delay_timer(), check_timer(DelayTimer), + test_server:sleep(?SLEEP), %% Make sure clean has had to run + undefined = ssl_session_cache:lookup(Cache, {{Hostname, Port}, Id}), undefined = ssl_session_cache:lookup(Cache, {Port, Id}), @@ -253,6 +252,18 @@ check_timer(Timer) -> test_server:sleep(Int), check_timer(Timer) end. + +get_delay_timer() -> + {status, _, _, StatusInfo} = sys:get_status(whereis(ssl_manager)), + [_, _,_, _, Prop] = StatusInfo, + State = state(Prop), + case element(7, State) of + undefined -> + test_server:sleep(?SLEEP), + get_delay_timer(); + DelayTimer -> + DelayTimer + end. %%-------------------------------------------------------------------- session_cache_process_list(doc) -> ["Test reuse of sessions (short handshake)"]; diff --git a/lib/stdlib/src/erl_internal.erl b/lib/stdlib/src/erl_internal.erl index 478f05e792..0b9b8b8e17 100644 --- a/lib/stdlib/src/erl_internal.erl +++ b/lib/stdlib/src/erl_internal.erl @@ -263,7 +263,6 @@ bif(bit_size, 1) -> true; bif(bitstring_to_list, 1) -> true; bif(byte_size, 1) -> true; bif(check_process_code, 2) -> true; -bif(concat_binary, 1) -> true; bif(date, 0) -> true; bif(delete_module, 1) -> true; bif(demonitor, 1) -> true; @@ -405,7 +404,6 @@ old_bif(bit_size, 1) -> true; old_bif(bitstring_to_list, 1) -> true; old_bif(byte_size, 1) -> true; old_bif(check_process_code, 2) -> true; -old_bif(concat_binary, 1) -> true; old_bif(date, 0) -> true; old_bif(delete_module, 1) -> true; old_bif(disconnect_node, 1) -> true; diff --git a/lib/stdlib/src/erl_scan.erl b/lib/stdlib/src/erl_scan.erl index 718ca2e91a..10b2ed2e49 100644 --- a/lib/stdlib/src/erl_scan.erl +++ b/lib/stdlib/src/erl_scan.erl @@ -408,7 +408,12 @@ set_attr(line, {Line,Column}, Fun) when ?ALINE(Line), ?COLUMN(Column) -> end; set_attr(line=Tag, Attrs, Fun) when is_list(Attrs) -> {line,Line} = lists:keyfind(Tag, 1, Attrs), - lists:keyreplace(Tag, 1, Attrs, {line,Fun(Line)}); + case lists:keyreplace(Tag, 1, Attrs, {line,Fun(Line)}) of + [{line,Ln}] when ?ALINE(Ln) -> + Ln; + As -> + As + end; set_attr(T1, T2, T3) -> erlang:error(badarg, [T1,T2,T3]). diff --git a/lib/stdlib/src/io_lib.erl b/lib/stdlib/src/io_lib.erl index 54c7283abf..165e03e506 100644 --- a/lib/stdlib/src/io_lib.erl +++ b/lib/stdlib/src/io_lib.erl @@ -100,7 +100,7 @@ fwrite(Format, Args) -> -spec fread(Format, String) -> Result when Format :: string(), String :: string(), - Result :: {'ok', InputList :: chars(), LeftOverChars :: string()} + Result :: {'ok', InputList :: [term()], LeftOverChars :: string()} | {'more', RestFormat :: string(), Nchars :: non_neg_integer(), InputStack :: chars()} @@ -115,7 +115,7 @@ fread(Chars, Format) -> Format :: string(), Return :: {'more', Continuation1 :: continuation()} | {'done', Result, LeftOverChars :: string()}, - Result :: {'ok', InputList :: chars()} + Result :: {'ok', InputList :: [term()]} | 'eof' | {'error', What :: term()}. diff --git a/lib/stdlib/src/io_lib_fread.erl b/lib/stdlib/src/io_lib_fread.erl index 52aa4d073c..ded1346097 100644 --- a/lib/stdlib/src/io_lib_fread.erl +++ b/lib/stdlib/src/io_lib_fread.erl @@ -24,6 +24,10 @@ -import(lists, [reverse/1,reverse/2]). +-define(is_whitespace(C), + ((C) =:= $\s orelse (C) =:= $\t + orelse (C) =:= $\r orelse (C) =:= $\n)). + %%----------------------------------------------------------------------- %% fread(Continuation, CharList, FormatString) @@ -106,31 +110,27 @@ fread_line(Format0, Line, N0, Results0, More, Newline) -> fread(Format, Line) -> fread(Format, Line, 0, []). -fread([$~|Format0], Line, N, Results) -> +fread([$~|Format0]=AllFormat, Line, N, Results) -> {Format,F,Sup,Unicode} = fread_field(Format0), - fread1(Format, F, Sup, Unicode, Line, N, Results, Format0); -fread([$\s|Format], Line, N, Results) -> - fread_skip_white(Format, Line, N, Results); -fread([$\t|Format], Line, N, Results) -> - fread_skip_white(Format, Line, N, Results); -fread([$\r|Format], Line, N, Results) -> - fread_skip_white(Format, Line, N, Results); -fread([$\n|Format], Line, N, Results) -> + fread1(Format, F, Sup, Unicode, Line, N, Results, AllFormat); +fread([C|Format], Line, N, Results) when ?is_whitespace(C) -> fread_skip_white(Format, Line, N, Results); fread([C|Format], [C|Line], N, Results) -> fread(Format, Line, N+1, Results); fread([_F|_Format], [_C|_Line], _N, _Results) -> fread_error(input); +fread([_|_]=Format, [], N, Results) -> + {more,Format,N,Results}; +fread([_|_], eof, 0, []) -> + %% This is at start of input so no error. + eof; +fread([_|_], eof, _N, _Results) -> + %% This is an error as there is no more input. + fread_error(input); fread([], Line, _N, Results) -> {ok,reverse(Results),Line}. -fread_skip_white(Format, [$\s|Line], N, Results) -> - fread_skip_white(Format, Line, N+1, Results); -fread_skip_white(Format, [$\t|Line], N, Results) -> - fread_skip_white(Format, Line, N+1, Results); -fread_skip_white(Format, [$\r|Line], N, Results) -> - fread_skip_white(Format, Line, N+1, Results); -fread_skip_white(Format, [$\n|Line], N, Results) -> +fread_skip_white(Format, [C|Line], N, Results) when ?is_whitespace(C) -> fread_skip_white(Format, Line, N+1, Results); fread_skip_white(Format, Line, N, Results) -> fread(Format, Line, N, Results). @@ -166,9 +166,9 @@ fread1([$l|Format], _F, Sup, _U, Line, N, Res, _AllFormat) -> fread(Format, Line, N, fread_result(Sup, N, Res)); fread1(_Format, _F, _Sup, _U, [], N, Res, AllFormat) -> %% Need more input here. - {more,[$~|AllFormat],N,Res}; -fread1(_Format, _F, _Sup, _U, eof, _N, [], _AllFormat) -> - %% This is at start of format string so no error. + {more,AllFormat,N,Res}; +fread1(_Format, _F, _Sup, _U, eof, 0, [], _AllFormat) -> + %% This is at start of input so no error. eof; fread1(_Format, _F, _Sup, _U, eof, _N, _Res, _AllFormat) -> %% This is an error as there is no more input. @@ -386,26 +386,16 @@ fread_string_cs(Line0, N0, true) -> %% fread_digits(Line, N, Base, Characters) %% Read segments of things, return "thing" characters in reverse order. -fread_skip_white([$\s|Line]) -> fread_skip_white(Line); -fread_skip_white([$\t|Line]) -> fread_skip_white(Line); -fread_skip_white([$\r|Line]) -> fread_skip_white(Line); -fread_skip_white([$\n|Line]) -> fread_skip_white(Line); +fread_skip_white([C|Line]) when ?is_whitespace(C) -> + fread_skip_white(Line); fread_skip_white(Line) -> Line. -fread_skip_white([$\s|Line], N) -> - fread_skip_white(Line, N+1); -fread_skip_white([$\t|Line], N) -> - fread_skip_white(Line, N+1); -fread_skip_white([$\r|Line], N) -> - fread_skip_white(Line, N+1); -fread_skip_white([$\n|Line], N) -> +fread_skip_white([C|Line], N) when ?is_whitespace(C) -> fread_skip_white(Line, N+1); fread_skip_white(Line, N) -> {Line,N}. -fread_skip_latin1_nonwhite([$\s|Line], N, Cs) -> {[$\s|Line],N,Cs}; -fread_skip_latin1_nonwhite([$\t|Line], N, Cs) -> {[$\t|Line],N,Cs}; -fread_skip_latin1_nonwhite([$\r|Line], N, Cs) -> {[$\r|Line],N,Cs}; -fread_skip_latin1_nonwhite([$\n|Line], N, Cs) -> {[$\n|Line],N,Cs}; +fread_skip_latin1_nonwhite([C|Line], N, Cs) when ?is_whitespace(C) -> + {[C|Line],N,Cs}; fread_skip_latin1_nonwhite([C|Line], N, []) when C > 255 -> {[C|Line],N,error}; fread_skip_latin1_nonwhite([C|Line], N, Cs) when C > 255 -> @@ -414,10 +404,8 @@ fread_skip_latin1_nonwhite([C|Line], N, Cs) -> fread_skip_latin1_nonwhite(Line, N+1, [C|Cs]); fread_skip_latin1_nonwhite([], N, Cs) -> {[],N,Cs}. -fread_skip_nonwhite([$\s|Line], N, Cs) -> {[$\s|Line],N,Cs}; -fread_skip_nonwhite([$\t|Line], N, Cs) -> {[$\t|Line],N,Cs}; -fread_skip_nonwhite([$\r|Line], N, Cs) -> {[$\r|Line],N,Cs}; -fread_skip_nonwhite([$\n|Line], N, Cs) -> {[$\n|Line],N,Cs}; +fread_skip_nonwhite([C|Line], N, Cs) when ?is_whitespace(C) -> + {[C|Line],N,Cs}; fread_skip_nonwhite([C|Line], N, Cs) -> fread_skip_nonwhite(Line, N+1, [C|Cs]); fread_skip_nonwhite([], N, Cs) -> {[],N,Cs}. diff --git a/lib/stdlib/src/otp_internal.erl b/lib/stdlib/src/otp_internal.erl index 39d017d430..db46670f61 100644 --- a/lib/stdlib/src/otp_internal.erl +++ b/lib/stdlib/src/otp_internal.erl @@ -449,7 +449,7 @@ obsolete_1(ssl_pkix, decode_cert, A) when A =:= 1; A =:= 2 -> %% Added in R13B04. obsolete_1(erlang, concat_binary, 1) -> - {deprecated,{erlang,list_to_binary,1},"R15B"}; + {removed,{erlang,list_to_binary,1},"R15B"}; %% Added in R14A. obsolete_1(ssl, peercert, 2) -> @@ -461,6 +461,10 @@ obsolete_1(public_key, pem_to_der, 1) -> obsolete_1(public_key, decode_private_key, A) when A =:= 1; A =:= 2 -> {deprecated,{public_key,pem_entry_decode,1},"R15A"}; +%% Added in R15B +obsolete_1(asn1rt, F, _) when F == load_driver; F == unload_driver -> + {deprecated,"deprecated (will be removed in R16A); has no effect as drivers are no longer used."}; + obsolete_1(_, _, _) -> no. diff --git a/lib/stdlib/src/queue.erl b/lib/stdlib/src/queue.erl index 4c6b4d710b..afe917b151 100644 --- a/lib/stdlib/src/queue.erl +++ b/lib/stdlib/src/queue.erl @@ -56,16 +56,14 @@ new() -> {[],[]}. %{RearList,FrontList} %% O(1) --spec is_queue(Term) -> boolean() when - Term :: term(). +-spec is_queue(Term :: term()) -> boolean(). is_queue({R,F}) when is_list(R), is_list(F) -> true; is_queue(_) -> false. %% O(1) --spec is_empty(Q) -> boolean() when - Q :: queue(). +-spec is_empty(Q :: queue()) -> boolean(). is_empty({[],[]}) -> true; is_empty({In,Out}) when is_list(In), is_list(Out) -> @@ -74,16 +72,14 @@ is_empty(Q) -> erlang:error(badarg, [Q]). %% O(len(Q)) --spec len(Q) -> non_neg_integer() when - Q :: queue(). +-spec len(Q :: queue()) -> non_neg_integer(). len({R,F}) when is_list(R), is_list(F) -> length(R)+length(F); len(Q) -> erlang:error(badarg, [Q]). %% O(len(Q)) --spec to_list(Q) -> list() when - Q :: queue(). +-spec to_list(Q :: queue()) -> list(). to_list({In,Out}) when is_list(In), is_list(Out) -> Out++lists:reverse(In, []); to_list(Q) -> @@ -92,8 +88,7 @@ to_list(Q) -> %% Create queue from list %% %% O(length(L)) --spec from_list(L) -> queue() when - L :: list(). +-spec from_list(L :: list()) -> queue(). from_list(L) when is_list(L) -> f2r(L); from_list(L) -> @@ -102,9 +97,7 @@ from_list(L) -> %% Return true or false depending on if element is in queue %% %% O(length(Q)) worst case --spec member(Item, Q) -> boolean() when - Item :: term(), - Q :: queue(). +-spec member(Item :: term(), Q :: queue()) -> boolean(). member(X, {R,F}) when is_list(R), is_list(F) -> lists:member(X, R) orelse lists:member(X, F); member(X, Q) -> @@ -117,10 +110,7 @@ member(X, Q) -> %% Put at least one element in each list, if it is cheap %% %% O(1) --spec in(Item, Q1) -> Q2 when - Item :: term(), - Q1 :: queue(), - Q2 :: queue(). +-spec in(Item :: term(), Q1 :: queue()) -> Q2 :: queue(). in(X, {[_]=In,[]}) -> {[X], In}; in(X, {In,Out}) when is_list(In), is_list(Out) -> @@ -132,10 +122,7 @@ in(X, Q) -> %% Put at least one element in each list, if it is cheap %% %% O(1) --spec in_r(Item, Q1) -> Q2 when - Item :: term(), - Q1 :: queue(), - Q2 :: queue(). +-spec in_r(Item :: term(), Q1 :: queue()) -> Q2 :: queue(). in_r(X, {[],[_]=F}) -> {F,[X]}; in_r(X, {R,F}) when is_list(R), is_list(F) -> @@ -146,10 +133,9 @@ in_r(X, Q) -> %% Take from head/front %% %% O(1) amortized, O(len(Q)) worst case --spec out(Q1) -> Result when - Q1 :: queue(), - Q2 :: queue(), - Result :: {{value, Item :: term()}, Q2} | {empty, Q1}. +-spec out(Q1 :: queue()) -> + {{value, Item :: term()}, Q2 :: queue()} | + {empty, Q1 :: queue()}. out({[],[]}=Q) -> {empty,Q}; out({[V],[]}) -> @@ -167,10 +153,9 @@ out(Q) -> %% Take from tail/rear %% %% O(1) amortized, O(len(Q)) worst case --spec out_r(Q1) -> Result when - Q1 :: queue(), - Q2 :: queue(), - Result :: {{value, Item :: term()}, Q2} | {empty, Q1}. +-spec out_r(Q1 :: queue()) -> + {{value, Item :: term()}, Q2 :: queue()} | + {empty, Q1 :: queue()}. out_r({[],[]}=Q) -> {empty,Q}; out_r({[],[V]}) -> @@ -191,9 +176,7 @@ out_r(Q) -> %% Return the first element in the queue %% %% O(1) since the queue is supposed to be well formed --spec get(Q) -> Item when - Q :: queue(), - Item :: term(). +-spec get(Q :: queue()) -> Item :: term(). get({[],[]}=Q) -> erlang:error(empty, [Q]); get({R,F}) when is_list(R), is_list(F) -> @@ -212,9 +195,7 @@ get([_|R], []) -> % malformed queue -> O(len(Q)) %% Return the last element in the queue %% %% O(1) since the queue is supposed to be well formed --spec get_r(Q) -> Item when - Q :: queue(), - Item :: term(). +-spec get_r(Q :: queue()) -> Item :: term(). get_r({[],[]}=Q) -> erlang:error(empty, [Q]); get_r({[H|_],F}) when is_list(F) -> @@ -229,9 +210,7 @@ get_r(Q) -> %% Return the first element in the queue %% %% O(1) since the queue is supposed to be well formed --spec peek(Q) -> 'empty' | {'value',Item} when - Q :: queue(), - Item :: term(). +-spec peek(Q :: queue()) -> empty | {value,Item :: term()}. peek({[],[]}) -> empty; peek({R,[H|_]}) when is_list(R) -> @@ -246,9 +225,7 @@ peek(Q) -> %% Return the last element in the queue %% %% O(1) since the queue is supposed to be well formed --spec peek_r(Q) -> 'empty' | {'value',Item} when - Q :: queue(), - Item :: term(). +-spec peek_r(Q :: queue()) -> empty | {value,Item :: term()}. peek_r({[],[]}) -> empty; peek_r({[H|_],F}) when is_list(F) -> @@ -263,9 +240,7 @@ peek_r(Q) -> %% Remove the first element and return resulting queue %% %% O(1) amortized --spec drop(Q1) -> Q2 when - Q1 :: queue(), - Q2 :: queue(). +-spec drop(Q1 :: queue()) -> Q2 :: queue(). drop({[],[]}=Q) -> erlang:error(empty, [Q]); drop({[_],[]}) -> @@ -283,9 +258,7 @@ drop(Q) -> %% Remove the last element and return resulting queue %% %% O(1) amortized --spec drop_r(Q1) -> Q2 when - Q1 :: queue(), - Q2 :: queue(). +-spec drop_r(Q1 :: queue()) -> Q2 :: queue(). drop_r({[],[]}=Q) -> erlang:error(empty, [Q]); drop_r({[],[_]}) -> @@ -306,9 +279,7 @@ drop_r(Q) -> %% Return reversed queue %% %% O(1) --spec reverse(Q1) -> Q2 when - Q1 :: queue(), - Q2 :: queue(). +-spec reverse(Q1 :: queue()) -> Q2 :: queue(). reverse({R,F}) when is_list(R), is_list(F) -> {F,R}; reverse(Q) -> @@ -318,10 +289,7 @@ reverse(Q) -> %% %% Q2 empty: O(1) %% else: O(len(Q1)) --spec join(Q1, Q2) -> Q3 when - Q1 :: queue(), - Q2 :: queue(), - Q3 :: queue(). +-spec join(Q1 :: queue(), Q2 :: queue()) -> Q3 :: queue(). join({R,F}=Q, {[],[]}) when is_list(R), is_list(F) -> Q; join({[],[]}, {R,F}=Q) when is_list(R), is_list(F) -> @@ -335,11 +303,8 @@ join(Q1, Q2) -> %% %% N = 0..len(Q) %% O(max(N, len(Q))) --spec split(N, Q1) -> {Q2,Q3} when - N :: non_neg_integer(), - Q1 :: queue(), - Q2 :: queue(), - Q3 :: queue(). +-spec split(N :: non_neg_integer(), Q1 :: queue()) -> + {Q2 :: queue(),Q3 :: queue()}. split(0, {R,F}=Q) when is_list(R), is_list(F) -> {{[],[]},Q}; split(N, {R,F}=Q) when is_integer(N), N >= 1, is_list(R), is_list(F) -> @@ -380,10 +345,8 @@ split_r1_to_f2(N, [X|R1], F1, R2, F2) -> %% %% Fun(_) -> List: O(length(List) * len(Q)) %% else: O(len(Q) --spec filter(Fun, Q1) -> Q2 when - Fun :: fun((Item :: term()) -> boolean() | list()), - Q1 :: queue(), - Q2 :: queue(). +-spec filter(Fun, Q1 :: queue()) -> Q2 :: queue() when + Fun :: fun((Item :: term()) -> boolean() | list()). filter(Fun, {R0,F0}) when is_function(Fun, 1), is_list(R0), is_list(F0) -> F = filter_f(Fun, F0), R = filter_r(Fun, R0), @@ -459,10 +422,7 @@ filter_r(Fun, [X|R0]) -> %% Cons to head %% --spec cons(Item, Q1) -> Q2 when - Item :: term(), - Q1 :: queue(), - Q2 :: queue(). +-spec cons(Item :: term(), Q1 :: queue()) -> Q2 :: queue(). cons(X, Q) -> in_r(X, Q). @@ -471,9 +431,7 @@ cons(X, Q) -> %% Return the first element in the queue %% %% O(1) since the queue is supposed to be well formed --spec head(Q) -> Item when - Q :: queue(), - Item :: term(). +-spec head(Q :: queue()) -> Item :: term(). head({[],[]}=Q) -> erlang:error(empty, [Q]); head({R,F}) when is_list(R), is_list(F) -> @@ -483,9 +441,7 @@ head(Q) -> %% Remove head element and return resulting queue %% --spec tail(Q1) -> Q2 when - Q1 :: queue(), - Q2 :: queue(). +-spec tail(Q1 :: queue()) -> Q2 :: queue(). tail(Q) -> drop(Q). @@ -493,35 +449,22 @@ tail(Q) -> %% Cons to tail %% --spec snoc(Q1, Item) -> Q2 when - Q1 :: queue(), - Q2 :: queue(), - Item :: term(). +-spec snoc(Q1 :: queue(), Item :: term()) -> Q2 :: queue(). snoc(Q, X) -> in(X, Q). %% Return last element --spec daeh(Q) -> Item when - Q :: queue(), - Item :: term(). +-spec daeh(Q :: queue()) -> Item :: term(). daeh(Q) -> get_r(Q). --spec last(Q) -> Item when - Q :: queue(), - Item :: term(). +-spec last(Q :: queue()) -> Item :: term(). last(Q) -> get_r(Q). %% Remove last element and return resulting queue --spec liat(Q1) -> Q2 when - Q1 :: queue(), - Q2 :: queue(). +-spec liat(Q1 :: queue()) -> Q2 :: queue(). liat(Q) -> drop_r(Q). --spec lait(Q1) -> Q2 when - Q1 :: queue(), - Q2 :: queue(). +-spec lait(Q1 :: queue()) -> Q2 :: queue(). lait(Q) -> drop_r(Q). %% Oops, mis-spelled 'tail' reversed. Forget this one. --spec init(Q1) -> Q2 when - Q1 :: queue(), - Q2 :: queue(). +-spec init(Q1 :: queue()) -> Q2 :: queue(). init(Q) -> drop_r(Q). %%-------------------------------------------------------------------------- diff --git a/lib/stdlib/test/erl_scan_SUITE.erl b/lib/stdlib/test/erl_scan_SUITE.erl index 31a4f94294..4298b2c701 100644 --- a/lib/stdlib/test/erl_scan_SUITE.erl +++ b/lib/stdlib/test/erl_scan_SUITE.erl @@ -737,6 +737,10 @@ set_attribute() -> (catch {foo, erl_scan:set_attribute(line, [], F2)}), % type error ?line {'EXIT',{badarg,_}} = (catch {foo, erl_scan:set_attribute(column, [], F2)}), % type error + + %% OTP-9412 + ?line 8 = erl_scan:set_attribute(line, [{line,{nos,'X',8}}], + fun({nos,_V,VL}) -> VL end), ok. column_errors() -> diff --git a/lib/stdlib/test/ets_SUITE.erl b/lib/stdlib/test/ets_SUITE.erl index 9d348b5f1a..9341300f90 100644 --- a/lib/stdlib/test/ets_SUITE.erl +++ b/lib/stdlib/test/ets_SUITE.erl @@ -72,6 +72,7 @@ exit_many_many_tables_owner/1]). -export([write_concurrency/1, heir/1, give_away/1, setopts/1]). -export([bad_table/1, types/1]). +-export([otp_9423/1]). -export([init_per_testcase/2, end_per_testcase/2]). %% Convenience for manual testing @@ -143,7 +144,8 @@ all() -> otp_8166, exit_large_table_owner, exit_many_large_table_owner, exit_many_tables_owner, exit_many_many_tables_owner, write_concurrency, heir, - give_away, setopts, bad_table, types]. + give_away, setopts, bad_table, types, + otp_9423]. groups() -> [{new, [], @@ -5420,7 +5422,39 @@ types_do(Opts) -> ?line verify_etsmem(EtsMem). - +otp_9423(doc) -> ["vm-deadlock caused by race between ets:delete and others on write_concurrency table"]; +otp_9423(Config) when is_list(Config) -> + InitF = fun(_) -> {0,0} end, + ExecF = fun({S,F}) -> + receive + stop -> + io:format("~p got stop\n", [self()]), + [end_of_work | {"Succeded=",S,"Failed=",F}] + after 0 -> + %%io:format("~p (~p) doing lookup\n", [self(), {S,F}]), + try ets:lookup(otp_9423, key) of + [] -> {S+1,F} + catch + error:badarg -> {S,F+1} + end + end + end, + FiniF = fun(R) -> R end, + case run_workers(InitF, ExecF, FiniF, infinite, 1) of + Pids when is_list(Pids) -> + %%[P ! start || P <- Pids], + repeat(fun() -> ets:new(otp_9423, [named_table, public, {write_concurrency,true}]), + ets:delete(otp_9423) + end, 10000), + [P ! stop || P <- Pids], + wait_pids(Pids), + ok; + + Skipped -> Skipped + end. + + + % % Utility functions: @@ -5434,21 +5468,30 @@ add_lists([E1|T1], [E2|T2], Acc) -> add_lists(T1, T2, [E1+E2 | Acc]). run_workers(InitF,ExecF,FiniF,Laps) -> + run_workers(InitF,ExecF,FiniF,Laps, 0). +run_workers(InitF,ExecF,FiniF,Laps, Exclude) -> case erlang:system_info(smp_support) of true -> - run_workers_do(InitF,ExecF,FiniF,Laps); + run_workers_do(InitF,ExecF,FiniF,Laps, Exclude); false -> {skipped,"No smp support"} end. - + run_workers_do(InitF,ExecF,FiniF,Laps) -> - NumOfProcs = erlang:system_info(schedulers), + run_workers_do(InitF,ExecF,FiniF,Laps, 0). +run_workers_do(InitF,ExecF,FiniF,Laps, Exclude) -> + ?line NumOfProcs = case erlang:system_info(schedulers) of + N when (N > Exclude) -> N - Exclude + end, io:format("smp starting ~p workers\n",[NumOfProcs]), Seeds = [{ProcN,random:uniform(9999)} || ProcN <- lists:seq(1,NumOfProcs)], Parent = self(), Pids = [spawn_link(fun()-> worker(Seed,InitF,ExecF,FiniF,Laps,Parent,NumOfProcs) end) || Seed <- Seeds], - wait_pids(Pids). + case Laps of + infinite -> Pids; + _ -> wait_pids(Pids) + end. worker({ProcN,Seed}, InitF, ExecF, FiniF, Laps, Parent, NumOfProcs) -> io:format("smp worker ~p, seed=~p~n",[self(),Seed]), @@ -5463,6 +5506,8 @@ worker_loop(0, _, State) -> State; worker_loop(_, _, [end_of_work|State]) -> State; +worker_loop(infinite, ExecF, State) -> + worker_loop(infinite,ExecF,ExecF(State)); worker_loop(N, ExecF, State) -> worker_loop(N-1,ExecF,ExecF(State)). @@ -5517,20 +5562,21 @@ etsmem() -> case erlang:system_info({allocator,ets_alloc}) of false -> undefined; MemInfo -> - MSBCS = lists:foldl( - fun ({instance, _, L}, Acc) -> - {value,{_,MBCS}} = lists:keysearch(mbcs, 1, L), - {value,{_,SBCS}} = lists:keysearch(sbcs, 1, L), - [MBCS,SBCS | Acc] - end, - [], - MemInfo), + CS = lists:foldl( + fun ({instance, _, L}, Acc) -> + {value,{_,SBMBCS}} = lists:keysearch(sbmbcs, 1, L), + {value,{_,MBCS}} = lists:keysearch(mbcs, 1, L), + {value,{_,SBCS}} = lists:keysearch(sbcs, 1, L), + [SBMBCS,MBCS,SBCS | Acc] + end, + [], + MemInfo), lists:foldl( fun(L, {Bl0,BlSz0}) -> {value,{_,Bl,_,_}} = lists:keysearch(blocks, 1, L), {value,{_,BlSz,_,_}} = lists:keysearch(blocks_size, 1, L), {Bl0+Bl,BlSz0+BlSz} - end, {0,0}, MSBCS) + end, {0,0}, CS) end}, {Mem,AllTabs}. diff --git a/lib/stdlib/test/io_SUITE.erl b/lib/stdlib/test/io_SUITE.erl index 54a98985cd..bb02a879c2 100644 --- a/lib/stdlib/test/io_SUITE.erl +++ b/lib/stdlib/test/io_SUITE.erl @@ -27,7 +27,7 @@ otp_6282/1, otp_6354/1, otp_6495/1, otp_6517/1, otp_6502/1, manpage/1, otp_6708/1, otp_7084/1, otp_7421/1, io_lib_collect_line_3_wb/1, cr_whitespace_in_string/1, - io_fread_newlines/1, otp_8989/1]). + io_fread_newlines/1, otp_8989/1, io_lib_fread_literal/1]). %-define(debug, true). @@ -62,7 +62,7 @@ all() -> otp_6282, otp_6354, otp_6495, otp_6517, otp_6502, manpage, otp_6708, otp_7084, otp_7421, io_lib_collect_line_3_wb, cr_whitespace_in_string, - io_fread_newlines, otp_8989]. + io_fread_newlines, otp_8989, io_lib_fread_literal]. groups() -> []. @@ -1995,3 +1995,29 @@ otp_8989(Suite) when is_list(Suite) -> ?line "Hel " = fmt("~-4.*s", [3,Hello]), ?line "Hel " = fmt("~*.*s", [-4,3,Hello]), ok. + +io_lib_fread_literal(doc) -> + "OTP-9439 io_lib:fread bug for literate at end"; +io_lib_fread_literal(Suite) when is_list(Suite) -> + ?line {more,"~d",0,""} = io_lib:fread("~d", ""), + ?line {error,{fread,integer}} = io_lib:fread("~d", " "), + ?line {more,"~d",1,""} = io_lib:fread(" ~d", " "), + ?line {ok,[17],"X"} = io_lib:fread(" ~d", " 17X"), + %% + ?line {more,"d",0,""} = io_lib:fread("d", ""), + ?line {error,{fread,input}} = io_lib:fread("d", " "), + ?line {more,"d",1,""} = io_lib:fread(" d", " "), + ?line {ok,[],"X"} = io_lib:fread(" d", " dX"), + %% + ?line {done,eof,_} = io_lib:fread([], eof, "~d"), + ?line {done,eof,_} = io_lib:fread([], eof, " ~d"), + ?line {more,C1} = io_lib:fread([], " \n", " ~d"), + ?line {done,{error,{fread,input}},_} = io_lib:fread(C1, eof, " ~d"), + ?line {done,{ok,[18]},""} = io_lib:fread(C1, "18\n", " ~d"), + %% + ?line {done,eof,_} = io_lib:fread([], eof, "d"), + ?line {done,eof,_} = io_lib:fread([], eof, " d"), + ?line {more,C2} = io_lib:fread([], " \n", " d"), + ?line {done,{error,{fread,input}},_} = io_lib:fread(C2, eof, " d"), + ?line {done,{ok,[]},[]} = io_lib:fread(C2, "d\n", " d"), + ok. diff --git a/lib/stdlib/test/supervisor_SUITE.erl b/lib/stdlib/test/supervisor_SUITE.erl index 969916abed..7a75114cb6 100644 --- a/lib/stdlib/test/supervisor_SUITE.erl +++ b/lib/stdlib/test/supervisor_SUITE.erl @@ -119,10 +119,9 @@ end_per_group(_GroupName, Config) -> Config. init_per_testcase(count_children_memory, Config) -> - MemoryState = erlang:system_info(allocator), - case count_children_allocator_test(MemoryState) of - true -> Config; - false -> + try erlang:memory() of + _ -> Config + catch error:notsup -> {skip, "+Meamin used during test; erlang:memory/1 not available"} end; init_per_testcase(_Case, Config) -> @@ -1118,17 +1117,6 @@ count_children_memory(Config) when is_list(Config) -> [terminate(SupPid, Pid, child, kill) || {undefined, Pid, worker, _Modules} <- Children3], [1,0,0,0] = get_child_counts(sup_test). -count_children_allocator_test(MemoryState) -> - Allocators = [temp_alloc, eheap_alloc, binary_alloc, ets_alloc, - driver_alloc, sl_alloc, ll_alloc, fix_alloc, std_alloc, - sys_alloc], - MemoryStateList = element(4, MemoryState), - AllocTypes = [lists:keyfind(Alloc, 1, MemoryStateList) - || Alloc <- Allocators], - AllocStates = [lists:keyfind(e, 1, AllocValue) - || {_Type, AllocValue} <- AllocTypes], - lists:all(fun(State) -> State == {e, true} end, AllocStates). - %%------------------------------------------------------------------------- do_not_save_start_parameters_for_temporary_children(doc) -> ["Temporary children shall not be restarted so they should not " diff --git a/lib/tools/doc/src/xref.xml b/lib/tools/doc/src/xref.xml index 75ffa25311..17de66bb22 100644 --- a/lib/tools/doc/src/xref.xml +++ b/lib/tools/doc/src/xref.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>2000</year><year>2010</year> + <year>2000</year><year>2011</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -1465,8 +1465,8 @@ Evaluates a predefined analysis. <name>start(NameOrOptions) -> Return</name> <fsummary>Create an Xref server.</fsummary> <type> - <v>Name = atom()()</v> - <v>XrefOrOptions = Xref | Options</v> + <v>NameOrOptions = Name | Options</v> + <v>Name = atom()</v> <v>Options = [Option] | Option</v> <v>Option = {xref_mode, mode()} | term()</v> <v>Return = {ok, pid()} | {error, {already_started, pid()}}</v> @@ -1483,7 +1483,7 @@ Evaluates a predefined analysis. <name>start(Name, Options) -> Return</name> <fsummary>Create an Xref server.</fsummary> <type> - <v>Name = atom()()</v> + <v>Name = atom()</v> <v>Options = [Option] | Option</v> <v>Option = {xref_mode, mode()} | term()</v> <v>Return = {ok, pid()} | {error, {already_started, pid()}}</v> |